summaryrefslogtreecommitdiff
path: root/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
blob: 80a37a60218136b5912ec7db9be5e8d21fb9ebd0 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
/*
 * Copyright 2013 Advanced Micro Devices, Inc.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * Authors: Christian König <christian.koenig@amd.com>
 */

#include <linux/firmware.h>
#include <drm/drmP.h>
#include "amdgpu.h"
#include "amdgpu_vce.h"
#include "cikd.h"

#include "vce/vce_2_0_d.h"
#include "vce/vce_2_0_sh_mask.h"

#include "oss/oss_2_0_d.h"
#include "oss/oss_2_0_sh_mask.h"

#define VCE_V2_0_FW_SIZE	(256 * 1024)
#define VCE_V2_0_STACK_SIZE	(64 * 1024)
#define VCE_V2_0_DATA_SIZE	(23552 * AMDGPU_MAX_VCE_HANDLES)

static void vce_v2_0_mc_resume(struct amdgpu_device *adev);
static void vce_v2_0_set_ring_funcs(struct amdgpu_device *adev);
static void vce_v2_0_set_irq_funcs(struct amdgpu_device *adev);
static int vce_v2_0_wait_for_idle(void *handle);
/**
 * vce_v2_0_ring_get_rptr - get read pointer
 *
 * @ring: amdgpu_ring pointer
 *
 * Returns the current hardware read pointer
 */
static uint32_t vce_v2_0_ring_get_rptr(struct amdgpu_ring *ring)
{
	struct amdgpu_device *adev = ring->adev;

	if (ring == &adev->vce.ring[0])
		return RREG32(mmVCE_RB_RPTR);
	else
		return RREG32(mmVCE_RB_RPTR2);
}

/**
 * vce_v2_0_ring_get_wptr - get write pointer
 *
 * @ring: amdgpu_ring pointer
 *
 * Returns the current hardware write pointer
 */
static uint32_t vce_v2_0_ring_get_wptr(struct amdgpu_ring *ring)
{
	struct amdgpu_device *adev = ring->adev;

	if (ring == &adev->vce.ring[0])
		return RREG32(mmVCE_RB_WPTR);
	else
		return RREG32(mmVCE_RB_WPTR2);
}

/**
 * vce_v2_0_ring_set_wptr - set write pointer
 *
 * @ring: amdgpu_ring pointer
 *
 * Commits the write pointer to the hardware
 */
static void vce_v2_0_ring_set_wptr(struct amdgpu_ring *ring)
{
	struct amdgpu_device *adev = ring->adev;

	if (ring == &adev->vce.ring[0])
		WREG32(mmVCE_RB_WPTR, ring->wptr);
	else
		WREG32(mmVCE_RB_WPTR2, ring->wptr);
}

/**
 * vce_v2_0_start - start VCE block
 *
 * @adev: amdgpu_device pointer
 *
 * Setup and start the VCE block
 */
static int vce_v2_0_start(struct amdgpu_device *adev)
{
	struct amdgpu_ring *ring;
	int i, j, r;

	vce_v2_0_mc_resume(adev);

	/* set BUSY flag */
	WREG32_P(mmVCE_STATUS, 1, ~1);

	ring = &adev->vce.ring[0];
	WREG32(mmVCE_RB_RPTR, ring->wptr);
	WREG32(mmVCE_RB_WPTR, ring->wptr);
	WREG32(mmVCE_RB_BASE_LO, ring->gpu_addr);
	WREG32(mmVCE_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
	WREG32(mmVCE_RB_SIZE, ring->ring_size / 4);

	ring = &adev->vce.ring[1];
	WREG32(mmVCE_RB_RPTR2, ring->wptr);
	WREG32(mmVCE_RB_WPTR2, ring->wptr);
	WREG32(mmVCE_RB_BASE_LO2, ring->gpu_addr);
	WREG32(mmVCE_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
	WREG32(mmVCE_RB_SIZE2, ring->ring_size / 4);

	WREG32_P(mmVCE_VCPU_CNTL, VCE_VCPU_CNTL__CLK_EN_MASK, ~VCE_VCPU_CNTL__CLK_EN_MASK);

	WREG32_P(mmVCE_SOFT_RESET,
		 VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,
		 ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);

	mdelay(100);

	WREG32_P(mmVCE_SOFT_RESET, 0, ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);

	for (i = 0; i < 10; ++i) {
		uint32_t status;
		for (j = 0; j < 100; ++j) {
			status = RREG32(mmVCE_STATUS);
			if (status & 2)
				break;
			mdelay(10);
		}
		r = 0;
		if (status & 2)
			break;

		DRM_ERROR("VCE not responding, trying to reset the ECPU!!!\n");
		WREG32_P(mmVCE_SOFT_RESET, VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,
				~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
		mdelay(10);
		WREG32_P(mmVCE_SOFT_RESET, 0, ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
		mdelay(10);
		r = -1;
	}

	/* clear BUSY flag */
	WREG32_P(mmVCE_STATUS, 0, ~1);

	if (r) {
		DRM_ERROR("VCE not responding, giving up!!!\n");
		return r;
	}

	return 0;
}

static int vce_v2_0_early_init(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	vce_v2_0_set_ring_funcs(adev);
	vce_v2_0_set_irq_funcs(adev);

	return 0;
}

static int vce_v2_0_sw_init(void *handle)
{
	struct amdgpu_ring *ring;
	int r;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	/* VCE */
	r = amdgpu_irq_add_id(adev, 167, &adev->vce.irq);
	if (r)
		return r;

	r = amdgpu_vce_sw_init(adev, VCE_V2_0_FW_SIZE +
		VCE_V2_0_STACK_SIZE + VCE_V2_0_DATA_SIZE);
	if (r)
		return r;

	r = amdgpu_vce_resume(adev);
	if (r)
		return r;

	ring = &adev->vce.ring[0];
	sprintf(ring->name, "vce0");
	r = amdgpu_ring_init(adev, ring, 512, VCE_CMD_NO_OP, 0xf,
			     &adev->vce.irq, 0, AMDGPU_RING_TYPE_VCE);
	if (r)
		return r;

	ring = &adev->vce.ring[1];
	sprintf(ring->name, "vce1");
	r = amdgpu_ring_init(adev, ring, 512, VCE_CMD_NO_OP, 0xf,
			     &adev->vce.irq, 0, AMDGPU_RING_TYPE_VCE);
	if (r)
		return r;

	return r;
}

static int vce_v2_0_sw_fini(void *handle)
{
	int r;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	r = amdgpu_vce_suspend(adev);
	if (r)
		return r;

	r = amdgpu_vce_sw_fini(adev);
	if (r)
		return r;

	return r;
}

static int vce_v2_0_hw_init(void *handle)
{
	struct amdgpu_ring *ring;
	int r;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	r = vce_v2_0_start(adev);
	if (r)
/* this error mean vcpu not in running state, so just skip ring test, not stop driver initialize */
		return 0;

	ring = &adev->vce.ring[0];
	ring->ready = true;
	r = amdgpu_ring_test_ring(ring);
	if (r) {
		ring->ready = false;
		return r;
	}

	ring = &adev->vce.ring[1];
	ring->ready = true;
	r = amdgpu_ring_test_ring(ring);
	if (r) {
		ring->ready = false;
		return r;
	}

	DRM_INFO("VCE initialized successfully.\n");

	return 0;
}

static int vce_v2_0_hw_fini(void *handle)
{
	return 0;
}

static int vce_v2_0_suspend(void *handle)
{
	int r;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	r = vce_v2_0_hw_fini(adev);
	if (r)
		return r;

	r = amdgpu_vce_suspend(adev);
	if (r)
		return r;

	return r;
}

static int vce_v2_0_resume(void *handle)
{
	int r;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	r = amdgpu_vce_resume(adev);
	if (r)
		return r;

	r = vce_v2_0_hw_init(adev);
	if (r)
		return r;

	return r;
}

static void vce_v2_0_set_sw_cg(struct amdgpu_device *adev, bool gated)
{
	u32 tmp;

	if (gated) {
		tmp = RREG32(mmVCE_CLOCK_GATING_B);
		tmp |= 0xe70000;
		WREG32(mmVCE_CLOCK_GATING_B, tmp);

		tmp = RREG32(mmVCE_UENC_CLOCK_GATING);
		tmp |= 0xff000000;
		WREG32(mmVCE_UENC_CLOCK_GATING, tmp);

		tmp = RREG32(mmVCE_UENC_REG_CLOCK_GATING);
		tmp &= ~0x3fc;
		WREG32(mmVCE_UENC_REG_CLOCK_GATING, tmp);

		WREG32(mmVCE_CGTT_CLK_OVERRIDE, 0);
	} else {
		tmp = RREG32(mmVCE_CLOCK_GATING_B);
		tmp |= 0xe7;
		tmp &= ~0xe70000;
		WREG32(mmVCE_CLOCK_GATING_B, tmp);

		tmp = RREG32(mmVCE_UENC_CLOCK_GATING);
		tmp |= 0x1fe000;
		tmp &= ~0xff000000;
		WREG32(mmVCE_UENC_CLOCK_GATING, tmp);

		tmp = RREG32(mmVCE_UENC_REG_CLOCK_GATING);
		tmp |= 0x3fc;
		WREG32(mmVCE_UENC_REG_CLOCK_GATING, tmp);
	}
}

static void vce_v2_0_set_dyn_cg(struct amdgpu_device *adev, bool gated)
{
	u32 orig, tmp;

	if (gated) {
		if (vce_v2_0_wait_for_idle(adev)) {
			DRM_INFO("VCE is busy, Can't set clock gateing");
			return;
		}
		WREG32_P(mmVCE_VCPU_CNTL, 0, ~VCE_VCPU_CNTL__CLK_EN_MASK);
		WREG32_P(mmVCE_SOFT_RESET, VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK, ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
		mdelay(100);
		WREG32(mmVCE_STATUS, 0);
	} else {
		WREG32_P(mmVCE_VCPU_CNTL, VCE_VCPU_CNTL__CLK_EN_MASK, ~VCE_VCPU_CNTL__CLK_EN_MASK);
		WREG32_P(mmVCE_SOFT_RESET, VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK, ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
		mdelay(100);
	}

	tmp = RREG32(mmVCE_CLOCK_GATING_B);
	tmp &= ~0x00060006;
	if (gated) {
		tmp |= 0xe10000;
	} else {
		tmp |= 0xe1;
		tmp &= ~0xe10000;
	}
	WREG32(mmVCE_CLOCK_GATING_B, tmp);

	orig = tmp = RREG32(mmVCE_UENC_CLOCK_GATING);
	tmp &= ~0x1fe000;
	tmp &= ~0xff000000;
	if (tmp != orig)
		WREG32(mmVCE_UENC_CLOCK_GATING, tmp);

	orig = tmp = RREG32(mmVCE_UENC_REG_CLOCK_GATING);
	tmp &= ~0x3fc;
	if (tmp != orig)
		WREG32(mmVCE_UENC_REG_CLOCK_GATING, tmp);

	if (gated)
		WREG32(mmVCE_CGTT_CLK_OVERRIDE, 0);
	WREG32_P(mmVCE_SOFT_RESET, 0, ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
}

static void vce_v2_0_disable_cg(struct amdgpu_device *adev)
{
	WREG32(mmVCE_CGTT_CLK_OVERRIDE, 7);
}

static void vce_v2_0_enable_mgcg(struct amdgpu_device *adev, bool enable)
{
	bool sw_cg = false;

	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG)) {
		if (sw_cg)
			vce_v2_0_set_sw_cg(adev, true);
		else
			vce_v2_0_set_dyn_cg(adev, true);
	} else {
		vce_v2_0_disable_cg(adev);

		if (sw_cg)
			vce_v2_0_set_sw_cg(adev, false);
		else
			vce_v2_0_set_dyn_cg(adev, false);
	}
}

static void vce_v2_0_init_cg(struct amdgpu_device *adev)
{
	u32 tmp;

	tmp = RREG32(mmVCE_CLOCK_GATING_A);
	tmp &= ~0xfff;
	tmp |= ((0 << 0) | (4 << 4));
	tmp |= 0x40000;
	WREG32(mmVCE_CLOCK_GATING_A, tmp);

	tmp = RREG32(mmVCE_UENC_CLOCK_GATING);
	tmp &= ~0xfff;
	tmp |= ((0 << 0) | (4 << 4));
	WREG32(mmVCE_UENC_CLOCK_GATING, tmp);

	tmp = RREG32(mmVCE_CLOCK_GATING_B);
	tmp |= 0x10;
	tmp &= ~0x100000;
	WREG32(mmVCE_CLOCK_GATING_B, tmp);
}

static void vce_v2_0_mc_resume(struct amdgpu_device *adev)
{
	uint64_t addr = adev->vce.gpu_addr;
	uint32_t size;

	WREG32_P(mmVCE_CLOCK_GATING_A, 0, ~(1 << 16));
	WREG32_P(mmVCE_UENC_CLOCK_GATING, 0x1FF000, ~0xFF9FF000);
	WREG32_P(mmVCE_UENC_REG_CLOCK_GATING, 0x3F, ~0x3F);
	WREG32(mmVCE_CLOCK_GATING_B, 0xf7);

	WREG32(mmVCE_LMI_CTRL, 0x00398000);
	WREG32_P(mmVCE_LMI_CACHE_CTRL, 0x0, ~0x1);
	WREG32(mmVCE_LMI_SWAP_CNTL, 0);
	WREG32(mmVCE_LMI_SWAP_CNTL1, 0);
	WREG32(mmVCE_LMI_VM_CTRL, 0);

	addr += AMDGPU_VCE_FIRMWARE_OFFSET;
	size = VCE_V2_0_FW_SIZE;
	WREG32(mmVCE_VCPU_CACHE_OFFSET0, addr & 0x7fffffff);
	WREG32(mmVCE_VCPU_CACHE_SIZE0, size);

	addr += size;
	size = VCE_V2_0_STACK_SIZE;
	WREG32(mmVCE_VCPU_CACHE_OFFSET1, addr & 0x7fffffff);
	WREG32(mmVCE_VCPU_CACHE_SIZE1, size);

	addr += size;
	size = VCE_V2_0_DATA_SIZE;
	WREG32(mmVCE_VCPU_CACHE_OFFSET2, addr & 0x7fffffff);
	WREG32(mmVCE_VCPU_CACHE_SIZE2, size);

	WREG32_P(mmVCE_LMI_CTRL2, 0x0, ~0x100);

	WREG32_P(mmVCE_SYS_INT_EN, VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK,
		 ~VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);

	vce_v2_0_init_cg(adev);
}

static bool vce_v2_0_is_idle(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	return !(RREG32(mmSRBM_STATUS2) & SRBM_STATUS2__VCE_BUSY_MASK);
}

static int vce_v2_0_wait_for_idle(void *handle)
{
	unsigned i;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	for (i = 0; i < adev->usec_timeout; i++) {
		if (!(RREG32(mmSRBM_STATUS2) & SRBM_STATUS2__VCE_BUSY_MASK))
			return 0;
	}
	return -ETIMEDOUT;
}

static int vce_v2_0_soft_reset(void *handle)
{
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	WREG32_P(mmSRBM_SOFT_RESET, SRBM_SOFT_RESET__SOFT_RESET_VCE_MASK,
			~SRBM_SOFT_RESET__SOFT_RESET_VCE_MASK);
	mdelay(5);

	return vce_v2_0_start(adev);
}

static int vce_v2_0_set_interrupt_state(struct amdgpu_device *adev,
					struct amdgpu_irq_src *source,
					unsigned type,
					enum amdgpu_interrupt_state state)
{
	uint32_t val = 0;

	if (state == AMDGPU_IRQ_STATE_ENABLE)
		val |= VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK;

	WREG32_P(mmVCE_SYS_INT_EN, val, ~VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);
	return 0;
}

static int vce_v2_0_process_interrupt(struct amdgpu_device *adev,
				      struct amdgpu_irq_src *source,
				      struct amdgpu_iv_entry *entry)
{
	DRM_DEBUG("IH: VCE\n");
	switch (entry->src_data) {
	case 0:
		amdgpu_fence_process(&adev->vce.ring[0]);
		break;
	case 1:
		amdgpu_fence_process(&adev->vce.ring[1]);
		break;
	default:
		DRM_ERROR("Unhandled interrupt: %d %d\n",
			  entry->src_id, entry->src_data);
		break;
	}

	return 0;
}

static int vce_v2_0_set_clockgating_state(void *handle,
					  enum amd_clockgating_state state)
{
	bool gate = false;
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	if (state == AMD_CG_STATE_GATE)
		gate = true;

	vce_v2_0_enable_mgcg(adev, gate);

	return 0;
}

static int vce_v2_0_set_powergating_state(void *handle,
					  enum amd_powergating_state state)
{
	/* This doesn't actually powergate the VCE block.
	 * That's done in the dpm code via the SMC.  This
	 * just re-inits the block as necessary.  The actual
	 * gating still happens in the dpm code.  We should
	 * revisit this when there is a cleaner line between
	 * the smc and the hw blocks
	 */
	struct amdgpu_device *adev = (struct amdgpu_device *)handle;

	if (!(adev->pg_flags & AMD_PG_SUPPORT_VCE))
		return 0;

	if (state == AMD_PG_STATE_GATE)
		/* XXX do we need a vce_v2_0_stop()? */
		return 0;
	else
		return vce_v2_0_start(adev);
}

const struct amd_ip_funcs vce_v2_0_ip_funcs = {
	.name = "vce_v2_0",
	.early_init = vce_v2_0_early_init,
	.late_init = NULL,
	.sw_init = vce_v2_0_sw_init,
	.sw_fini = vce_v2_0_sw_fini,
	.hw_init = vce_v2_0_hw_init,
	.hw_fini = vce_v2_0_hw_fini,
	.suspend = vce_v2_0_suspend,
	.resume = vce_v2_0_resume,
	.is_idle = vce_v2_0_is_idle,
	.wait_for_idle = vce_v2_0_wait_for_idle,
	.soft_reset = vce_v2_0_soft_reset,
	.set_clockgating_state = vce_v2_0_set_clockgating_state,
	.set_powergating_state = vce_v2_0_set_powergating_state,
};

static const struct amdgpu_ring_funcs vce_v2_0_ring_funcs = {
	.get_rptr = vce_v2_0_ring_get_rptr,
	.get_wptr = vce_v2_0_ring_get_wptr,
	.set_wptr = vce_v2_0_ring_set_wptr,
	.parse_cs = amdgpu_vce_ring_parse_cs,
	.emit_ib = amdgpu_vce_ring_emit_ib,
	.emit_fence = amdgpu_vce_ring_emit_fence,
	.test_ring = amdgpu_vce_ring_test_ring,
	.test_ib = amdgpu_vce_ring_test_ib,
	.insert_nop = amdgpu_ring_insert_nop,
	.pad_ib = amdgpu_ring_generic_pad_ib,
	.begin_use = amdgpu_vce_ring_begin_use,
	.end_use = amdgpu_vce_ring_end_use,
};

static void vce_v2_0_set_ring_funcs(struct amdgpu_device *adev)
{
	adev->vce.ring[0].funcs = &vce_v2_0_ring_funcs;
	adev->vce.ring[1].funcs = &vce_v2_0_ring_funcs;
}

static const struct amdgpu_irq_src_funcs vce_v2_0_irq_funcs = {
	.set = vce_v2_0_set_interrupt_state,
	.process = vce_v2_0_process_interrupt,
};

static void vce_v2_0_set_irq_funcs(struct amdgpu_device *adev)
{
	adev->vce.irq.num_types = 1;
	adev->vce.irq.funcs = &vce_v2_0_irq_funcs;
};
0%;'/> -rw-r--r--drivers/acpi/scan.c6
-rw-r--r--drivers/acpi/sleep.c187
-rw-r--r--drivers/acpi/tables.c21
-rw-r--r--drivers/acpi/thermal.c11
-rw-r--r--drivers/acpi/utils.c11
-rw-r--r--drivers/acpi/x86/utils.c4
-rw-r--r--drivers/amba/tegra-ahb.c11
-rw-r--r--drivers/android/binder.c262
-rw-r--r--drivers/android/binder_alloc.c44
-rw-r--r--drivers/android/binder_alloc.h22
-rw-r--r--drivers/android/binder_internal.h86
-rw-r--r--drivers/android/binderfs.c290
-rw-r--r--drivers/ata/acard-ahci.c39
-rw-r--r--drivers/ata/ahci.c149
-rw-r--r--drivers/ata/ahci.h2
-rw-r--r--drivers/ata/ahci_sunxi.c47
-rw-r--r--drivers/ata/libahci.c2
-rw-r--r--drivers/ata/libahci_platform.c4
-rw-r--r--drivers/ata/libata-core.c4
-rw-r--r--drivers/ata/libata-eh.c8
-rw-r--r--drivers/ata/libata-scsi.c21
-rw-r--r--drivers/ata/libata-sff.c14
-rw-r--r--drivers/ata/libata-zpodd.c2
-rw-r--r--drivers/ata/pata_atp867x.c7
-rw-r--r--drivers/ata/pata_buddha.c228
-rw-r--r--drivers/ata/pata_cs5520.c6
-rw-r--r--drivers/ata/pata_hpt3x3.c5
-rw-r--r--drivers/ata/pata_ninja32.c5
-rw-r--r--drivers/ata/pata_pdc2027x.c6
-rw-r--r--drivers/ata/pata_rb532_cf.c1
-rw-r--r--drivers/ata/pata_sil680.c5
-rw-r--r--drivers/ata/pdc_adma.c24
-rw-r--r--drivers/ata/sata_inic162x.c8
-rw-r--r--drivers/ata/sata_mv.c38
-rw-r--r--drivers/ata/sata_nv.c12
-rw-r--r--drivers/ata/sata_promise.c5
-rw-r--r--drivers/ata/sata_qstor.c35
-rw-r--r--drivers/ata/sata_sil.c5
-rw-r--r--drivers/ata/sata_sil24.c27
-rw-r--r--drivers/ata/sata_svw.c5
-rw-r--r--drivers/ata/sata_sx4.c5
-rw-r--r--drivers/ata/sata_via.c9
-rw-r--r--drivers/ata/sata_vsc.c5
-rw-r--r--drivers/atm/Kconfig2
-rw-r--r--drivers/atm/eni.c2
-rw-r--r--drivers/atm/he.c7
-rw-r--r--drivers/atm/idt77252.c1
-rw-r--r--drivers/atm/iphase.c8
-rw-r--r--drivers/atm/solos-pci.c9
-rw-r--r--drivers/auxdisplay/Kconfig9
-rw-r--r--drivers/auxdisplay/charlcd.c2
-rw-r--r--drivers/auxdisplay/charlcd.h44
-rw-r--r--drivers/auxdisplay/hd44780.c3
-rw-r--r--drivers/auxdisplay/ht16k33.c4
-rw-r--r--drivers/auxdisplay/panel.c4
-rw-r--r--drivers/base/Kconfig2
-rw-r--r--drivers/base/arch_topology.c311
-rw-r--r--drivers/base/base.h1
-rw-r--r--drivers/base/bus.c28
-rw-r--r--drivers/base/cacheinfo.c8
-rw-r--r--drivers/base/core.c348
-rw-r--r--drivers/base/dd.c69
-rw-r--r--drivers/base/devcon.c69
-rw-r--r--drivers/base/devcoredump.c13
-rw-r--r--drivers/base/devtmpfs.c3
-rw-r--r--drivers/base/driver.c4
-rw-r--r--drivers/base/firmware_loader/Kconfig18
-rw-r--r--drivers/base/firmware_loader/fallback.c65
-rw-r--r--drivers/base/firmware_loader/fallback_table.c13
-rw-r--r--drivers/base/firmware_loader/firmware.h16
-rw-r--r--drivers/base/firmware_loader/main.c224
-rw-r--r--drivers/base/memory.c219
-rw-r--r--drivers/base/node.c40
-rw-r--r--drivers/base/platform.c127
-rw-r--r--drivers/base/power/Makefile2
-rw-r--r--drivers/base/power/clock_ops.c6
-rw-r--r--drivers/base/power/domain.c33
-rw-r--r--drivers/base/power/domain_governor.c4
-rw-r--r--drivers/base/power/main.c71
-rw-r--r--drivers/base/power/power.h18
-rw-r--r--drivers/base/power/qos.c135
-rw-r--r--drivers/base/power/runtime.c21
-rw-r--r--drivers/base/power/sysfs.c6
-rw-r--r--drivers/base/power/wakeup.c78
-rw-r--r--drivers/base/power/wakeup_stats.c214
-rw-r--r--drivers/base/property.c24
-rw-r--r--drivers/base/regmap/Kconfig8
-rw-r--r--drivers/base/regmap/Makefile1
-rw-r--r--drivers/base/regmap/regcache-lzo.c8
-rw-r--r--drivers/base/regmap/regmap-debugfs.c14
-rw-r--r--drivers/base/regmap/regmap-i3c.c60
-rw-r--r--drivers/base/regmap/regmap-irq.c7
-rw-r--r--drivers/base/regmap/regmap.c2
-rw-r--r--drivers/base/soc.c9
-rw-r--r--drivers/base/swnode.c361
-rw-r--r--drivers/base/topology.c22
-rw-r--r--drivers/bcma/driver_mips.c16
-rw-r--r--drivers/bcma/driver_pci.c4
-rw-r--r--drivers/bcma/host_pci.c6
-rw-r--r--drivers/bcma/sprom.c10
-rw-r--r--drivers/block/Kconfig10
-rw-r--r--drivers/block/aoe/aoedev.c13
-rw-r--r--drivers/block/ataflop.c1
-rw-r--r--drivers/block/drbd/drbd_debugfs.c64
-rw-r--r--drivers/block/drbd/drbd_debugfs.h4
-rw-r--r--drivers/block/drbd/drbd_int.h2
-rw-r--r--drivers/block/drbd/drbd_main.c7
-rw-r--r--drivers/block/drbd/drbd_nl.c2
-rw-r--r--drivers/block/drbd/drbd_receiver.c14
-rw-r--r--drivers/block/floppy.c42
-rw-r--r--drivers/block/loop.c35
-rw-r--r--drivers/block/mtip32xx/mtip32xx.c5
-rw-r--r--drivers/block/nbd.c188
-rw-r--r--drivers/block/null_blk.h23
-rw-r--r--drivers/block/null_blk_main.c195
-rw-r--r--drivers/block/null_blk_zoned.c62
-rw-r--r--drivers/block/paride/pcd.c12
-rw-r--r--drivers/block/paride/pf.c2
-rw-r--r--drivers/block/rbd.c2199
-rw-r--r--drivers/block/rbd_types.h10
-rw-r--r--drivers/block/skd_main.c1
-rw-r--r--drivers/block/xen-blkback/xenbus.c6
-rw-r--r--drivers/block/zram/Kconfig6
-rw-r--r--drivers/bluetooth/Kconfig12
-rw-r--r--drivers/bluetooth/bpa10x.c5
-rw-r--r--drivers/bluetooth/btbcm.c4
-rw-r--r--drivers/bluetooth/btmtkuart.c51
-rw-r--r--drivers/bluetooth/btqca.c73
-rw-r--r--drivers/bluetooth/btqca.h17
-rw-r--r--drivers/bluetooth/btrtl.c153
-rw-r--r--drivers/bluetooth/btrtl.h6
-rw-r--r--drivers/bluetooth/btsdio.c1
-rw-r--r--drivers/bluetooth/btusb.c657
-rw-r--r--drivers/bluetooth/hci_ath.c3
-rw-r--r--drivers/bluetooth/hci_bcm.c36
-rw-r--r--drivers/bluetooth/hci_bcsp.c5
-rw-r--r--drivers/bluetooth/hci_intel.c3
-rw-r--r--drivers/bluetooth/hci_ldisc.c21
-rw-r--r--drivers/bluetooth/hci_ll.c109
-rw-r--r--drivers/bluetooth/hci_mrvl.c75
-rw-r--r--drivers/bluetooth/hci_qca.c121
-rw-r--r--drivers/bluetooth/hci_uart.h2
-rw-r--r--drivers/bus/Kconfig10
-rw-r--r--drivers/bus/Makefile1
-rw-r--r--drivers/bus/brcmstb_gisb.c4
-rw-r--r--drivers/bus/fsl-mc/dprc.c30
-rw-r--r--drivers/bus/fsl-mc/fsl-mc-allocator.c1
-rw-r--r--drivers/bus/fsl-mc/fsl-mc-bus.c15
-rw-r--r--drivers/bus/fsl-mc/fsl-mc-private.h17
-rw-r--r--drivers/bus/fsl-mc/mc-io.c1
-rw-r--r--drivers/bus/hisi_lpc.c47
-rw-r--r--drivers/bus/imx-weim.c34
-rw-r--r--drivers/bus/moxtet.c885
-rw-r--r--drivers/bus/sunxi-rsb.c4
-rw-r--r--drivers/bus/ti-sysc.c478
-rw-r--r--drivers/bus/uniphier-system-bus.c4
-rw-r--r--drivers/cdrom/cdrom.c2
-rw-r--r--drivers/char/Kconfig37
-rw-r--r--drivers/char/Makefile2
-rw-r--r--drivers/char/agp/Kconfig11
-rw-r--r--drivers/char/agp/Makefile1
-rw-r--r--drivers/char/agp/generic.c3
-rw-r--r--drivers/char/agp/sgi-agp.c338
-rw-r--r--drivers/char/bsr.c5
-rw-r--r--drivers/char/hpet.c3
-rw-r--r--drivers/char/hw_random/atmel-rng.c3
-rw-r--r--drivers/char/hw_random/cavium-rng-vf.c11
-rw-r--r--drivers/char/hw_random/core.c7
-rw-r--r--drivers/char/hw_random/exynos-trng.c3
-rw-r--r--drivers/char/hw_random/imx-rngc.c4
-rw-r--r--drivers/char/hw_random/iproc-rng200.c1
-rw-r--r--drivers/char/hw_random/meson-rng.c52
-rw-r--r--drivers/char/hw_random/mxc-rnga.c4
-rw-r--r--drivers/char/hw_random/n2-drv.c4
-rw-r--r--drivers/char/hw_random/nomadik-rng.c3
-rw-r--r--drivers/char/hw_random/omap-rng.c3
-rw-r--r--drivers/char/hw_random/powernv-rng.c10
-rw-r--r--drivers/char/hw_random/st-rng.c4
-rw-r--r--drivers/char/hw_random/timeriomem-rng.c4
-rw-r--r--drivers/char/hw_random/xgene-rng.c4
-rw-r--r--drivers/char/ipmi/Kconfig9
-rw-r--r--drivers/char/ipmi/Makefile1
-rw-r--r--drivers/char/ipmi/ipmb_dev_int.c364
-rw-r--r--drivers/char/ipmi/ipmi_msghandler.c8
-rw-r--r--drivers/char/ipmi/ipmi_si_intf.c4
-rw-r--r--drivers/char/ipmi/ipmi_si_platform.c9
-rw-r--r--drivers/char/ipmi/ipmi_ssif.c5
-rw-r--r--drivers/char/mbcs.c831
-rw-r--r--drivers/char/mbcs.h553
-rw-r--r--drivers/char/mem.c21
-rw-r--r--drivers/char/misc.c3
-rw-r--r--drivers/char/mspec.c155
-rw-r--r--drivers/char/ppdev.c2
-rw-r--r--drivers/char/random.c18
-rw-r--r--drivers/char/snsc.c469
-rw-r--r--drivers/char/snsc.h92
-rw-r--r--drivers/char/snsc_event.c303
-rw-r--r--drivers/char/toshiba.c8
-rw-r--r--drivers/char/tpm/Kconfig5
-rw-r--r--drivers/char/tpm/Makefile1
-rw-r--r--drivers/char/tpm/eventlog/efi.c59
-rw-r--r--drivers/char/tpm/eventlog/tpm2.c47
-rw-r--r--drivers/char/tpm/tpm-chip.c56
-rw-r--r--drivers/char/tpm/tpm-sysfs.c7
-rw-r--r--drivers/char/tpm/tpm.h2
-rw-r--r--drivers/char/tpm/tpm1-cmd.c43
-rw-r--r--drivers/char/tpm/tpm2-cmd.c13
-rw-r--r--drivers/char/tpm/tpm_ftpm_tee.c350
-rw-r--r--drivers/char/tpm/tpm_ftpm_tee.h40
-rw-r--r--drivers/char/tpm/tpm_tis_core.c3
-rw-r--r--drivers/clk/Kconfig28
-rw-r--r--drivers/clk/Makefile1
-rw-r--r--drivers/clk/at91/clk-generated.c2
-rw-r--r--drivers/clk/at91/sckc.c281
-rw-r--r--drivers/clk/bcm/Kconfig24
-rw-r--r--drivers/clk/bcm/Makefile6
-rw-r--r--drivers/clk/bcm/clk-bcm2835.c28
-rw-r--r--drivers/clk/bcm/clk-bcm63xx-gate.c238
-rw-r--r--drivers/clk/bcm/clk-raspberrypi.c315
-rw-r--r--drivers/clk/clk-bd718x7.c24
-rw-r--r--drivers/clk/clk-bulk.c23
-rw-r--r--drivers/clk/clk-cdce706.c2
-rw-r--r--drivers/clk/clk-devres.c22
-rw-r--r--drivers/clk/clk-lochnagar.c205
-rw-r--r--drivers/clk/clk-pwm.c14
-rw-r--r--drivers/clk/clk-qoriq.c12
-rw-r--r--drivers/clk/clk-rk808.c64
-rw-r--r--drivers/clk/clk-scmi.c2
-rw-r--r--drivers/clk/clk-si5341.c1346
-rw-r--r--drivers/clk/clk-si544.c102
-rw-r--r--drivers/clk/clk.c118
-rw-r--r--drivers/clk/clk.h4
-rw-r--r--drivers/clk/imx/Kconfig6
-rw-r--r--drivers/clk/imx/Makefile1
-rw-r--r--drivers/clk/imx/clk-busy.c30
-rw-r--r--drivers/clk/imx/clk-cpu.c14
-rw-r--r--drivers/clk/imx/clk-fixup-div.c15
-rw-r--r--drivers/clk/imx/clk-fixup-mux.c15
-rw-r--r--drivers/clk/imx/clk-gate-exclusive.c17
-rw-r--r--drivers/clk/imx/clk-gate2.c14
-rw-r--r--drivers/clk/imx/clk-imx6q.c782
-rw-r--r--drivers/clk/imx/clk-imx6sl.c409
-rw-r--r--drivers/clk/imx/clk-imx6sll.c434
-rw-r--r--drivers/clk/imx/clk-imx6sx.c662
-rw-r--r--drivers/clk/imx/clk-imx6ul.c580
-rw-r--r--drivers/clk/imx/clk-imx7d.c984
-rw-r--r--drivers/clk/imx/clk-imx7ulp.c2
-rw-r--r--drivers/clk/imx/clk-imx8mm.c35
-rw-r--r--drivers/clk/imx/clk-imx8mn.c636
-rw-r--r--drivers/clk/imx/clk-imx8mq.c27
-rw-r--r--drivers/clk/imx/clk-imx8qxp-lpcg.c5
-rw-r--r--drivers/clk/imx/clk-pfd.c14
-rw-r--r--drivers/clk/imx/clk-pllv3.c14
-rw-r--r--drivers/clk/imx/clk.c43
-rw-r--r--drivers/clk/imx/clk.h161
-rw-r--r--drivers/clk/ingenic/Makefile2
-rw-r--r--drivers/clk/ingenic/cgu.c41
-rw-r--r--drivers/clk/ingenic/cgu.h4
-rw-r--r--drivers/clk/ingenic/jz4725b-cgu.c41
-rw-r--r--drivers/clk/ingenic/jz4740-cgu.c105
-rw-r--r--drivers/clk/ingenic/jz4770-cgu.c67
-rw-r--r--drivers/clk/ingenic/jz4780-cgu.c3
-rw-r--r--drivers/clk/ingenic/pm.c45
-rw-r--r--drivers/clk/ingenic/pm.h12
-rw-r--r--drivers/clk/keystone/Kconfig11
-rw-r--r--drivers/clk/keystone/sci-clk.c239
-rw-r--r--drivers/clk/mediatek/Kconfig6
-rw-r--r--drivers/clk/mediatek/Makefile1
-rw-r--r--drivers/clk/mediatek/clk-mt8183.c65
-rw-r--r--drivers/clk/mediatek/clk-mt8516-aud.c65
-rw-r--r--drivers/clk/mediatek/clk-mt8516.c5
-rw-r--r--drivers/clk/meson/Kconfig11
-rw-r--r--drivers/clk/meson/Makefile2
-rw-r--r--drivers/clk/meson/axg-aoclk.c63
-rw-r--r--drivers/clk/meson/axg-audio.c261
-rw-r--r--drivers/clk/meson/axg.c217
-rw-r--r--drivers/clk/meson/clk-cpu-dyndiv.c73
-rw-r--r--drivers/clk/meson/clk-cpu-dyndiv.h20
-rw-r--r--drivers/clk/meson/clk-input.c49
-rw-r--r--drivers/clk/meson/clk-input.h19
-rw-r--r--drivers/clk/meson/clk-mpll.c36
-rw-r--r--drivers/clk/meson/clk-mpll.h3
-rw-r--r--drivers/clk/meson/clk-regmap.h12
-rw-r--r--drivers/clk/meson/g12a-aoclk.c81
-rw-r--r--drivers/clk/meson/g12a.c2267
-rw-r--r--drivers/clk/meson/g12a.h40
-rw-r--r--drivers/clk/meson/gxbb-aoclk.c55
-rw-r--r--drivers/clk/meson/gxbb.c662
-rw-r--r--drivers/clk/meson/meson-aoclk.c37
-rw-r--r--drivers/clk/meson/meson-aoclk.h8
-rw-r--r--drivers/clk/meson/meson-eeclk.c11
-rw-r--r--drivers/clk/meson/meson-eeclk.h4
-rw-r--r--drivers/clk/meson/meson8b.c836
-rw-r--r--drivers/clk/meson/meson8b.h8
-rw-r--r--drivers/clk/mmp/clk-frac.c3
-rw-r--r--drivers/clk/mvebu/kirkwood.c17
-rw-r--r--drivers/clk/qcom/clk-rpmh.c16
-rw-r--r--drivers/clk/qcom/gcc-msm8996.c36
-rw-r--r--drivers/clk/qcom/gcc-qcs404.c7
-rw-r--r--drivers/clk/qcom/gdsc.c4
-rw-r--r--drivers/clk/renesas/clk-div6.c19
-rw-r--r--drivers/clk/renesas/clk-mstp.c20
-rw-r--r--drivers/clk/renesas/r8a77470-cpg-mssr.c2
-rw-r--r--drivers/clk/renesas/r8a774a1-cpg-mssr.c5
-rw-r--r--drivers/clk/renesas/r8a7795-cpg-mssr.c5
-rw-r--r--drivers/clk/renesas/r8a7796-cpg-mssr.c4
-rw-r--r--drivers/clk/renesas/r8a77965-cpg-mssr.c4
-rw-r--r--drivers/clk/renesas/r8a77990-cpg-mssr.c2
-rw-r--r--drivers/clk/renesas/r8a77995-cpg-mssr.c2
-rw-r--r--drivers/clk/renesas/r9a06g032-clocks.c227
-rw-r--r--drivers/clk/renesas/renesas-cpg-mssr.c53
-rw-r--r--drivers/clk/rockchip/clk-mmc-phase.c14
-rw-r--r--drivers/clk/rockchip/clk-px30.c12
-rw-r--r--drivers/clk/rockchip/clk-rk3228.c3
-rw-r--r--drivers/clk/rockchip/clk-rk3288.c13
-rw-r--r--drivers/clk/rockchip/clk-rk3328.c3
-rw-r--r--drivers/clk/rockchip/clk-rk3368.c12
-rw-r--r--drivers/clk/rockchip/clk-rk3399.c12
-rw-r--r--drivers/clk/rockchip/clk.h4
-rw-r--r--drivers/clk/samsung/clk-exynos4.c1
-rw-r--r--drivers/clk/samsung/clk-exynos5-subcmu.c16
-rw-r--r--drivers/clk/samsung/clk-exynos5-subcmu.h2
-rw-r--r--drivers/clk/samsung/clk-exynos5250.c7
-rw-r--r--drivers/clk/samsung/clk-exynos5420.c240
-rw-r--r--drivers/clk/samsung/clk-exynos5433.c4
-rw-r--r--drivers/clk/socfpga/clk-periph-s10.c2
-rw-r--r--drivers/clk/socfpga/clk-s10.c6
-rw-r--r--drivers/clk/sprd/Kconfig1
-rw-r--r--drivers/clk/sprd/common.c9
-rw-r--r--drivers/clk/sprd/sc9860-clk.c5
-rw-r--r--drivers/clk/sunxi-ng/ccu-sun4i-a10.c39
-rw-r--r--drivers/clk/sunxi-ng/ccu-sun50i-a64.c41
-rw-r--r--drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c4
-rw-r--r--drivers/clk/sunxi-ng/ccu-sun50i-h6.c69
-rw-r--r--drivers/clk/sunxi-ng/ccu-sun5i.c34
-rw-r--r--drivers/clk/sunxi-ng/ccu-sun6i-a31.c39
-rw-r--r--drivers/clk/sunxi-ng/ccu-sun8i-a23.c34
-rw-r--r--drivers/clk/sunxi-ng/ccu-sun8i-a33.c34
-rw-r--r--drivers/clk/sunxi-ng/ccu-sun8i-h3.c29
-rw-r--r--drivers/clk/sunxi-ng/ccu-sun8i-r.c104
-rw-r--r--drivers/clk/sunxi-ng/ccu-sun8i-r40.c46
-rw-r--r--drivers/clk/sunxi-ng/ccu-sun8i-v3s.c29
-rw-r--r--drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c32
-rw-r--r--drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c29
-rw-r--r--drivers/clk/sunxi-ng/ccu_common.c2
-rw-r--r--drivers/clk/sunxi-ng/ccu_gate.h53
-rw-r--r--drivers/clk/sunxi/clk-sunxi.c2
-rw-r--r--drivers/clk/tegra/clk-tegra210.c20
-rw-r--r--drivers/clk/ti/divider.c85
-rw-r--r--drivers/clk/ti/gate.c30
-rw-r--r--drivers/clk/ti/mux.c31
-rw-r--r--drivers/clocksource/Kconfig16
-rw-r--r--drivers/clocksource/Makefile5
-rw-r--r--drivers/clocksource/arc_timer.c3
-rw-r--r--drivers/clocksource/arm_arch_timer.c15
-rw-r--r--drivers/clocksource/em_sti.c4
-rw-r--r--drivers/clocksource/exynos_mct.c4
-rw-r--r--drivers/clocksource/hyperv_timer.c326
-rw-r--r--drivers/clocksource/renesas-ostm.c2
-rw-r--r--drivers/clocksource/sh_cmt.c19
-rw-r--r--drivers/clocksource/sh_tmu.c5
-rw-r--r--drivers/clocksource/timer-atmel-tcb.c18
-rw-r--r--drivers/clocksource/timer-davinci.c369
-rw-r--r--drivers/clocksource/timer-imx-sysctr.c150
-rw-r--r--drivers/clocksource/timer-ixp4xx.c16
-rw-r--r--drivers/clocksource/timer-meson6.c5
-rw-r--r--drivers/clocksource/timer-npcm7xx.c9
-rw-r--r--drivers/clocksource/timer-of.c6
-rw-r--r--drivers/clocksource/timer-probe.c4
-rw-r--r--drivers/clocksource/timer-riscv.c23
-rw-r--r--drivers/clocksource/timer-sun4i.c4
-rw-r--r--drivers/clocksource/timer-tegra.c416
-rw-r--r--drivers/clocksource/timer-tegra20.c379
-rw-r--r--drivers/connector/connector.c6
-rw-r--r--drivers/counter/104-quad-8.c2
-rw-r--r--drivers/counter/ftm-quaddec.c34
-rw-r--r--drivers/cpufreq/Kconfig.arm33
-rw-r--r--drivers/cpufreq/Makefile5
-rw-r--r--drivers/cpufreq/armada-37xx-cpufreq.c4
-rw-r--r--drivers/cpufreq/armada-8k-cpufreq.c2
-rw-r--r--drivers/cpufreq/bmips-cpufreq.c17
-rw-r--r--drivers/cpufreq/brcmstb-avs-cpufreq.c12
-rw-r--r--drivers/cpufreq/cpufreq-dt-platdev.c10
-rw-r--r--drivers/cpufreq/cpufreq.c370
-rw-r--r--drivers/cpufreq/davinci-cpufreq.c3
-rw-r--r--drivers/cpufreq/imx-cpufreq-dt.c104
-rw-r--r--drivers/cpufreq/imx6q-cpufreq.c6
-rw-r--r--drivers/cpufreq/intel_pstate.c153
-rw-r--r--drivers/cpufreq/kirkwood-cpufreq.c3
-rw-r--r--drivers/cpufreq/loongson1-cpufreq.c8
-rw-r--r--drivers/cpufreq/loongson2_cpufreq.c3
-rw-r--r--drivers/cpufreq/maple-cpufreq.c3
-rw-r--r--drivers/cpufreq/mediatek-cpufreq.c4
-rw-r--r--drivers/cpufreq/omap-cpufreq.c15
-rw-r--r--drivers/cpufreq/pasemi-cpufreq.c26
-rw-r--r--drivers/cpufreq/pcc-cpufreq.c4
-rw-r--r--drivers/cpufreq/pmac32-cpufreq.c3
-rw-r--r--drivers/cpufreq/pmac64-cpufreq.c3
-rw-r--r--drivers/cpufreq/ppc_cbe_cpufreq.c19
-rw-r--r--drivers/cpufreq/ppc_cbe_cpufreq.h8
-rw-r--r--drivers/cpufreq/ppc_cbe_cpufreq_pmi.c96
-rw-r--r--drivers/cpufreq/qcom-cpufreq-hw.c23
-rw-r--r--drivers/cpufreq/qcom-cpufreq-kryo.c249
-rw-r--r--drivers/cpufreq/qcom-cpufreq-nvmem.c352
-rw-r--r--drivers/cpufreq/raspberrypi-cpufreq.c97
-rw-r--r--drivers/cpufreq/s3c2416-cpufreq.c9
-rw-r--r--drivers/cpufreq/s3c64xx-cpufreq.c15
-rw-r--r--drivers/cpufreq/s5pv210-cpufreq.c5
-rw-r--r--drivers/cpufreq/sa1100-cpufreq.c3
-rw-r--r--drivers/cpufreq/sa1110-cpufreq.c3
-rw-r--r--drivers/cpufreq/spear-cpufreq.c3
-rw-r--r--drivers/cpufreq/sun50i-cpufreq-nvmem.c226
-rw-r--r--drivers/cpufreq/tegra20-cpufreq.c8
-rw-r--r--drivers/cpufreq/ti-cpufreq.c1
-rw-r--r--drivers/cpuidle/Kconfig20
-rw-r--r--drivers/cpuidle/Kconfig.arm10
-rw-r--r--drivers/cpuidle/Makefile2
-rw-r--r--drivers/cpuidle/cpuidle-arm.c13
-rw-r--r--drivers/cpuidle/cpuidle-haltpoll.c134
-rw-r--r--drivers/cpuidle/cpuidle-psci.c236
-rw-r--r--drivers/cpuidle/cpuidle.c30
-rw-r--r--drivers/cpuidle/cpuidle.h2
-rw-r--r--drivers/cpuidle/driver.c25
-rw-r--r--drivers/cpuidle/governor.c9
-rw-r--r--drivers/cpuidle/governors/Makefile1
-rw-r--r--drivers/cpuidle/governors/haltpoll.c150
-rw-r--r--drivers/cpuidle/governors/ladder.c21
-rw-r--r--drivers/cpuidle/governors/menu.c21
-rw-r--r--drivers/cpuidle/governors/teo.c60
-rw-r--r--drivers/cpuidle/poll_state.c11
-rw-r--r--drivers/cpuidle/sysfs.c7
-rw-r--r--drivers/crypto/Kconfig96
-rw-r--r--drivers/crypto/Makefile2
-rw-r--r--drivers/crypto/amcc/crypto4xx_alg.c60
-rw-r--r--drivers/crypto/amcc/crypto4xx_core.c25
-rw-r--r--drivers/crypto/amcc/crypto4xx_core.h10
-rw-r--r--drivers/crypto/amcc/crypto4xx_trng.c1
-rw-r--r--drivers/crypto/atmel-aes.c1
-rw-r--r--drivers/crypto/atmel-ecc.c403
-rw-r--r--drivers/crypto/atmel-ecc.h116
-rw-r--r--drivers/crypto/atmel-i2c.c376
-rw-r--r--drivers/crypto/atmel-i2c.h185
-rw-r--r--drivers/crypto/atmel-sha.c1
-rw-r--r--drivers/crypto/atmel-sha204a.c170
-rw-r--r--drivers/crypto/atmel-tdes.c29
-rw-r--r--drivers/crypto/axis/artpec6_crypto.c4
-rw-r--r--drivers/crypto/bcm/cipher.c100
-rw-r--r--drivers/crypto/bcm/spu2.c10
-rw-r--r--drivers/crypto/caam/Kconfig50
-rw-r--r--drivers/crypto/caam/Makefile19
-rw-r--r--drivers/crypto/caam/caamalg.c600
-rw-r--r--drivers/crypto/caam/caamalg_desc.c203
-rw-r--r--drivers/crypto/caam/caamalg_desc.h8
-rw-r--r--drivers/crypto/caam/caamalg_qi.c518
-rw-r--r--drivers/crypto/caam/caamalg_qi2.c523
-rw-r--r--drivers/crypto/caam/caamalg_qi2.h31
-rw-r--r--drivers/crypto/caam/caamhash.c443
-rw-r--r--drivers/crypto/caam/caamhash_desc.c5
-rw-r--r--drivers/crypto/caam/caamhash_desc.h2
-rw-r--r--drivers/crypto/caam/caampkc.c274
-rw-r--r--drivers/crypto/caam/caampkc.h9
-rw-r--r--drivers/crypto/caam/caamrng.c95
-rw-r--r--drivers/crypto/caam/compat.h2
-rw-r--r--drivers/crypto/caam/ctrl.c303
-rw-r--r--drivers/crypto/caam/desc_constr.h92
-rw-r--r--drivers/crypto/caam/dpseci-debugfs.c79
-rw-r--r--drivers/crypto/caam/dpseci-debugfs.h18
-rw-r--r--drivers/crypto/caam/error.c72
-rw-r--r--drivers/crypto/caam/error.h4
-rw-r--r--drivers/crypto/caam/intern.h134
-rw-r--r--drivers/crypto/caam/jr.c167
-rw-r--r--drivers/crypto/caam/key_gen.c42
-rw-r--r--drivers/crypto/caam/pdb.h16
-rw-r--r--drivers/crypto/caam/pkc_desc.c8
-rw-r--r--drivers/crypto/caam/qi.c62
-rw-r--r--drivers/crypto/caam/qi.h26
-rw-r--r--drivers/crypto/caam/regs.h141
-rw-r--r--drivers/crypto/caam/sg_sw_qm.h18
-rw-r--r--drivers/crypto/caam/sg_sw_qm2.h18
-rw-r--r--drivers/crypto/caam/sg_sw_sec4.h26
-rw-r--r--drivers/crypto/cavium/cpt/cptvf_algs.c27
-rw-r--r--drivers/crypto/cavium/nitrox/Kconfig2
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_csr.h235
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_debugfs.c3
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_debugfs.h2
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_dev.h8
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_hal.c158
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_hal.h6
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_lib.c66
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_main.c148
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_mbx.h2
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_req.h30
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_skcipher.c4
-rw-r--r--drivers/crypto/cavium/nitrox/nitrox_sriov.c3
-rw-r--r--drivers/crypto/cavium/zip/zip_main.c3
-rw-r--r--drivers/crypto/ccp/Kconfig9
-rw-r--r--drivers/crypto/ccp/Makefile4
-rw-r--r--drivers/crypto/ccp/ccp-crypto-aes-cmac.c25
-rw-r--r--drivers/crypto/ccp/ccp-crypto-aes-galois.c14
-rw-r--r--drivers/crypto/ccp/ccp-crypto-aes-xts.c3
-rw-r--r--drivers/crypto/ccp/ccp-crypto-aes.c7
-rw-r--r--drivers/crypto/ccp/ccp-crypto-des3.c7
-rw-r--r--drivers/crypto/ccp/ccp-crypto-main.c4
-rw-r--r--drivers/crypto/ccp/ccp-crypto.h8
-rw-r--r--drivers/crypto/ccp/ccp-dev-v3.c3
-rw-r--r--drivers/crypto/ccp/ccp-dev-v5.c26
-rw-r--r--drivers/crypto/ccp/ccp-dev.c133
-rw-r--r--drivers/crypto/ccp/ccp-dev.h5
-rw-r--r--drivers/crypto/ccp/ccp-dmaengine.c13
-rw-r--r--drivers/crypto/ccp/ccp-ops.c122
-rw-r--r--drivers/crypto/ccp/psp-dev.c19
-rw-r--r--drivers/crypto/ccp/psp-dev.h1
-rw-r--r--drivers/crypto/ccp/sp-dev.h1
-rw-r--r--drivers/crypto/ccp/sp-platform.c4
-rw-r--r--drivers/crypto/ccree/Makefile2
-rw-r--r--drivers/crypto/ccree/cc_aead.c129
-rw-r--r--drivers/crypto/ccree/cc_aead.h3
-rw-r--r--drivers/crypto/ccree/cc_buffer_mgr.c21
-rw-r--r--drivers/crypto/ccree/cc_buffer_mgr.h2
-rw-r--r--drivers/crypto/ccree/cc_cipher.c31
-rw-r--r--drivers/crypto/ccree/cc_driver.c86
-rw-r--r--drivers/crypto/ccree/cc_driver.h16
-rw-r--r--drivers/crypto/ccree/cc_fips.c31
-rw-r--r--drivers/crypto/ccree/cc_hash.c153
-rw-r--r--drivers/crypto/ccree/cc_host_regs.h20
-rw-r--r--drivers/crypto/ccree/cc_ivgen.c276
-rw-r--r--drivers/crypto/ccree/cc_ivgen.h55
-rw-r--r--drivers/crypto/ccree/cc_pm.c13
-rw-r--r--drivers/crypto/ccree/cc_pm.h7
-rw-r--r--drivers/crypto/ccree/cc_request_mgr.c47
-rw-r--r--drivers/crypto/chelsio/Kconfig1
-rw-r--r--drivers/crypto/chelsio/chcr_algo.c46
-rw-r--r--drivers/crypto/chelsio/chcr_algo.h20
-rw-r--r--drivers/crypto/chelsio/chcr_crypto.h1
-rw-r--r--drivers/crypto/chelsio/chcr_ipsec.c19
-rw-r--r--drivers/crypto/chelsio/chtls/chtls_hw.c20
-rw-r--r--drivers/crypto/chelsio/chtls/chtls_io.c6
-rw-r--r--drivers/crypto/chelsio/chtls/chtls_main.c6
-rw-r--r--drivers/crypto/exynos-rng.c4
-rw-r--r--drivers/crypto/hifn_795x.c32
-rw-r--r--drivers/crypto/hisilicon/Kconfig25
-rw-r--r--drivers/crypto/hisilicon/Makefile3
-rw-r--r--drivers/crypto/hisilicon/qm.c1913
-rw-r--r--drivers/crypto/hisilicon/qm.h215
-rw-r--r--drivers/crypto/hisilicon/sec/sec_algs.c18
-rw-r--r--drivers/crypto/hisilicon/sec/sec_drv.h2
-rw-r--r--drivers/crypto/hisilicon/sgl.c214
-rw-r--r--drivers/crypto/hisilicon/sgl.h24
-rw-r--r--drivers/crypto/hisilicon/zip/Makefile2
-rw-r--r--drivers/crypto/hisilicon/zip/zip.h71
-rw-r--r--drivers/crypto/hisilicon/zip/zip_crypto.c653
-rw-r--r--drivers/crypto/hisilicon/zip/zip_main.c1013
-rw-r--r--drivers/crypto/img-hash.c5
-rw-r--r--drivers/crypto/inside-secure/safexcel.c1158
-rw-r--r--drivers/crypto/inside-secure/safexcel.h237
-rw-r--r--drivers/crypto/inside-secure/safexcel_cipher.c1538
-rw-r--r--drivers/crypto/inside-secure/safexcel_hash.c621
-rw-r--r--drivers/crypto/inside-secure/safexcel_ring.c12
-rw-r--r--drivers/crypto/ixp4xx_crypto.c42
-rw-r--r--drivers/crypto/marvell/cipher.c27
-rw-r--r--drivers/crypto/marvell/hash.c3
-rw-r--r--drivers/crypto/mediatek/mtk-aes.c143
-rw-r--r--drivers/crypto/mediatek/mtk-platform.c7
-rw-r--r--drivers/crypto/mediatek/mtk-sha.c4
-rw-r--r--drivers/crypto/mxs-dcp.c13
-rw-r--r--drivers/crypto/n2_core.c42
-rw-r--r--drivers/crypto/nx/nx-842-powernv.c9
-rw-r--r--drivers/crypto/nx/nx-842-pseries.c6
-rw-r--r--drivers/crypto/nx/nx.c4
-rw-r--r--drivers/crypto/nx/nx.h15
-rw-r--r--drivers/crypto/nx/nx_debugfs.c71
-rw-r--r--drivers/crypto/omap-aes.c1
-rw-r--r--drivers/crypto/omap-des.c28
-rw-r--r--drivers/crypto/omap-sham.c1
-rw-r--r--drivers/crypto/padlock-aes.c10
-rw-r--r--drivers/crypto/picoxcell_crypto.c29
-rw-r--r--drivers/crypto/qat/qat_common/adf_common_drv.h2
-rw-r--r--drivers/crypto/qat/qat_common/qat_algs.c294
-rw-r--r--drivers/crypto/qat/qat_common/qat_crypto.h2
-rw-r--r--drivers/crypto/qce/ablkcipher.c55
-rw-r--r--drivers/crypto/qce/core.c4
-rw-r--r--drivers/crypto/qcom-rng.c4
-rw-r--r--drivers/crypto/rockchip/rk3288_crypto.c4
-rw-r--r--drivers/crypto/rockchip/rk3288_crypto.h2
-rw-r--r--drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c21
-rw-r--r--drivers/crypto/s5p-sss.c7
-rw-r--r--drivers/crypto/sahara.c8
-rw-r--r--drivers/crypto/stm32/Kconfig2
-rw-r--r--drivers/crypto/stm32/Makefile2
-rw-r--r--drivers/crypto/stm32/stm32-crc32.c (renamed from drivers/crypto/stm32/stm32_crc32.c)4
-rw-r--r--drivers/crypto/stm32/stm32-cryp.c38
-rw-r--r--drivers/crypto/stm32/stm32-hash.c12
-rw-r--r--drivers/crypto/sunxi-ss/sun4i-ss-cipher.c75
-rw-r--r--drivers/crypto/sunxi-ss/sun4i-ss-core.c6
-rw-r--r--drivers/crypto/sunxi-ss/sun4i-ss-hash.c2
-rw-r--r--drivers/crypto/sunxi-ss/sun4i-ss.h4
-rw-r--r--drivers/crypto/talitos.c402
-rw-r--r--drivers/crypto/talitos.h73
-rw-r--r--drivers/crypto/ux500/Kconfig2
-rw-r--r--drivers/crypto/ux500/cryp/cryp.c6
-rw-r--r--drivers/crypto/ux500/cryp/cryp.h4
-rw-r--r--drivers/crypto/ux500/cryp/cryp_core.c39
-rw-r--r--drivers/crypto/ux500/hash/hash_alg.h4
-rw-r--r--drivers/crypto/ux500/hash/hash_core.c12
-rw-r--r--drivers/crypto/virtio/virtio_crypto_algs.c4
-rw-r--r--drivers/crypto/vmx/aes_cbc.c183
-rw-r--r--drivers/crypto/vmx/aes_ctr.c165
-rw-r--r--drivers/crypto/vmx/aes_xts.c181
-rw-r--r--drivers/crypto/vmx/aesp8-ppc.h2
-rw-r--r--drivers/crypto/vmx/aesp8-ppc.pl22
-rw-r--r--drivers/crypto/vmx/vmx.c72
-rw-r--r--drivers/dax/bus.c21
-rw-r--r--drivers/dax/dax-private.h6
-rw-r--r--drivers/dax/device.c41
-rw-r--r--drivers/dax/kmem.c46
-rw-r--r--drivers/dax/pmem/core.c2
-rw-r--r--drivers/dax/super.c42
-rw-r--r--drivers/devfreq/Kconfig19
-rw-r--r--drivers/devfreq/Makefile3
-rw-r--r--drivers/devfreq/devfreq.c12
-rw-r--r--drivers/devfreq/event/exynos-ppmu.c104
-rw-r--r--drivers/devfreq/exynos-bus.c153
-rw-r--r--drivers/devfreq/governor_passive.c7
-rw-r--r--drivers/devfreq/rk3399_dmc.c2
-rw-r--r--drivers/devfreq/tegra20-devfreq.c212
-rw-r--r--drivers/devfreq/tegra30-devfreq.c (renamed from drivers/devfreq/tegra-devfreq.c)315
-rw-r--r--drivers/dma-buf/Kconfig2
-rw-r--r--drivers/dma-buf/dma-buf.c181
-rw-r--r--drivers/dma-buf/dma-fence.c21
-rw-r--r--drivers/dma-buf/reservation.c4
-rw-r--r--drivers/dma-buf/sync_debug.c26
-rw-r--r--drivers/dma-buf/sync_debug.h1
-rw-r--r--drivers/dma/Kconfig9
-rw-r--r--drivers/dma/Makefile1
-rw-r--r--drivers/dma/acpi-dma.c12
-rw-r--r--drivers/dma/amba-pl08x.c5
-rw-r--r--drivers/dma/at_xdmac.c11
-rw-r--r--drivers/dma/bcm-sba-raid.c13
-rw-r--r--drivers/dma/bcm2835-dma.c42
-rw-r--r--drivers/dma/coh901318.c6
-rw-r--r--drivers/dma/dma-axi-dmac.c201
-rw-r--r--drivers/dma/dma-jz4780.c26
-rw-r--r--drivers/dma/dmaengine.c14
-rw-r--r--drivers/dma/dmatest.c41
-rw-r--r--drivers/dma/dw-edma/Kconfig19
-rw-r--r--drivers/dma/dw-edma/Makefile7
-rw-r--r--drivers/dma/dw-edma/dw-edma-core.c937
-rw-r--r--drivers/dma/dw-edma/dw-edma-core.h165
-rw-r--r--drivers/dma/dw-edma/dw-edma-pcie.c229
-rw-r--r--drivers/dma/dw-edma/dw-edma-v0-core.c350
-rw-r--r--drivers/dma/dw-edma/dw-edma-v0-core.h28
-rw-r--r--drivers/dma/dw-edma/dw-edma-v0-debugfs.c311
-rw-r--r--drivers/dma/dw-edma/dw-edma-v0-debugfs.h27
-rw-r--r--drivers/dma/dw-edma/dw-edma-v0-regs.h158
-rw-r--r--drivers/dma/dw/Makefile4
-rw-r--r--drivers/dma/dw/acpi.c53
-rw-r--r--drivers/dma/dw/internal.h51
-rw-r--r--drivers/dma/dw/of.c131
-rw-r--r--drivers/dma/dw/pci.c69
-rw-r--r--drivers/dma/dw/platform.c221
-rw-r--r--drivers/dma/fsl-edma-common.c85
-rw-r--r--drivers/dma/fsl-edma-common.h14
-rw-r--r--drivers/dma/fsl-edma.c124
-rw-r--r--drivers/dma/fsl-qdma.c27
-rw-r--r--drivers/dma/fsldma.c1
-rw-r--r--drivers/dma/hsu/hsu.c4
-rw-r--r--drivers/dma/imx-dma.c1
-rw-r--r--drivers/dma/imx-sdma.c13
-rw-r--r--drivers/dma/ioat/dca.c3
-rw-r--r--drivers/dma/iop-adma.c24
-rw-r--r--drivers/dma/iop-adma.h914
-rw-r--r--drivers/dma/mcf-edma.c11
-rw-r--r--drivers/dma/mediatek/Kconfig11
-rw-r--r--drivers/dma/mediatek/Makefile1
-rw-r--r--drivers/dma/mediatek/mtk-uart-apdma.c664
-rw-r--r--drivers/dma/mic_x100_dma.c6
-rw-r--r--drivers/dma/mmp_tdma.c10
-rw-r--r--drivers/dma/mv_xor_v2.c11
-rw-r--r--drivers/dma/mxs-dma.c33
-rw-r--r--drivers/dma/of-dma.c4
-rw-r--r--drivers/dma/pl330.c49
-rw-r--r--drivers/dma/pxa_dma.c56
-rw-r--r--drivers/dma/qcom/hidma.h5
-rw-r--r--drivers/dma/qcom/hidma_dbg.c37
-rw-r--r--drivers/dma/qcom/hidma_ll.c2
-rw-r--r--drivers/dma/qcom/hidma_mgmt.c9
-rw-r--r--drivers/dma/s3c24xx-dma.c5
-rw-r--r--drivers/dma/sh/Kconfig6
-rw-r--r--drivers/dma/sh/Makefile1
-rw-r--r--drivers/dma/sh/rcar-dmac.c40
-rw-r--r--drivers/dma/sh/sudmac.c414
-rw-r--r--drivers/dma/sh/usb-dmac.c12
-rw-r--r--drivers/dma/sprd-dma.c10
-rw-r--r--drivers/dma/st_fdma.c4
-rw-r--r--drivers/dma/ste_dma40.c4
-rw-r--r--drivers/dma/stm32-dma.c19
-rw-r--r--drivers/dma/stm32-dmamux.c9
-rw-r--r--drivers/dma/stm32-mdma.c9
-rw-r--r--drivers/dma/sun4i-dma.c4
-rw-r--r--drivers/dma/sun6i-dma.c151
-rw-r--r--drivers/dma/tegra20-apb-dma.c87
-rw-r--r--drivers/dma/tegra210-adma.c16
-rw-r--r--drivers/dma/ti/dma-crossbar.c4
-rw-r--r--drivers/dma/ti/edma.c228
-rw-r--r--drivers/dma/ti/omap-dma.c70
-rw-r--r--drivers/dma/uniphier-mdmac.c5
-rw-r--r--drivers/dma/virt-dma.c4
-rw-r--r--drivers/dma/virt-dma.h4
-rw-r--r--drivers/dma/xgene-dma.c8
-rw-r--r--drivers/dma/xilinx/xilinx_dma.c4
-rw-r--r--drivers/edac/Kconfig13
-rw-r--r--drivers/edac/Makefile2
-rw-r--r--drivers/edac/altera_edac.c101
-rw-r--r--drivers/edac/altera_edac.h25
-rw-r--r--drivers/edac/amd64_edac.c371
-rw-r--r--drivers/edac/amd64_edac.h15
-rw-r--r--drivers/edac/aspeed_edac.c4
-rw-r--r--drivers/edac/bluefield_edac.c356
-rw-r--r--drivers/edac/debugfs.c12
-rw-r--r--drivers/edac/edac_mc.c53
-rw-r--r--drivers/edac/edac_mc.h6
-rw-r--r--drivers/edac/edac_mc_sysfs.c116
-rw-r--r--drivers/edac/edac_module.h20
-rw-r--r--drivers/edac/ghes_edac.c2
-rw-r--r--drivers/edac/i10nm_base.c12
-rw-r--r--drivers/edac/i5100_edac.c16
-rw-r--r--drivers/edac/ie31200_edac.c78
-rw-r--r--drivers/edac/pnd2_edac.c9
-rw-r--r--drivers/edac/sb_edac.c3
-rw-r--r--drivers/edac/sifive_edac.c119
-rw-r--r--drivers/edac/skx_base.c2
-rw-r--r--drivers/edac/skx_common.c4
-rw-r--r--drivers/edac/skx_common.h2
-rw-r--r--drivers/extcon/Kconfig12
-rw-r--r--drivers/extcon/Makefile1
-rw-r--r--drivers/extcon/extcon-adc-jack.c4
-rw-r--r--drivers/extcon/extcon-arizona.c35
-rw-r--r--drivers/extcon/extcon-axp288.c16
-rw-r--r--drivers/extcon/extcon-fsa9480.c396
-rw-r--r--drivers/extcon/extcon-gpio.c29
-rw-r--r--drivers/extcon/extcon-max77843.c6
-rw-r--r--drivers/extcon/extcon-sm5502.c2
-rw-r--r--drivers/firewire/core-device.c2
-rw-r--r--drivers/firewire/core-iso.c2
-rw-r--r--drivers/firewire/core-topology.c1
-rw-r--r--drivers/firmware/Kconfig39
-rw-r--r--drivers/firmware/Makefile2
-rw-r--r--drivers/firmware/arm_scmi/Makefile2
-rw-r--r--drivers/firmware/arm_scmi/base.c2
-rw-r--r--drivers/firmware/arm_scmi/clock.c35
-rw-r--r--drivers/firmware/arm_scmi/common.h20
-rw-r--r--drivers/firmware/arm_scmi/driver.c366
-rw-r--r--drivers/firmware/arm_scmi/perf.c264
-rw-r--r--drivers/firmware/arm_scmi/power.c6
-rw-r--r--drivers/firmware/arm_scmi/reset.c231
-rw-r--r--drivers/firmware/arm_scmi/sensors.c67
-rw-r--r--drivers/firmware/arm_scpi.c5
-rw-r--r--drivers/firmware/efi/Kconfig13
-rw-r--r--drivers/firmware/efi/Makefile1
-rw-r--r--drivers/firmware/efi/cper.c15
-rw-r--r--drivers/firmware/efi/dev-path-parser.c4
-rw-r--r--drivers/firmware/efi/efi.c41
-rw-r--r--drivers/firmware/efi/libstub/efi-stub-helper.c31
-rw-r--r--drivers/firmware/efi/libstub/efistub.h2
-rw-r--r--drivers/firmware/efi/libstub/fdt.c27
-rw-r--r--drivers/firmware/efi/libstub/tpm.c80
-rw-r--r--drivers/firmware/efi/rci2-table.c147
-rw-r--r--drivers/firmware/efi/tpm.c63
-rw-r--r--drivers/firmware/google/coreboot_table.h11
-rw-r--r--drivers/firmware/google/framebuffer-coreboot.c14
-rw-r--r--drivers/firmware/google/memconsole-coreboot.c28
-rw-r--r--drivers/firmware/google/memconsole.c9
-rw-r--r--drivers/firmware/google/vpd.c18
-rw-r--r--drivers/firmware/google/vpd_decode.c57
-rw-r--r--drivers/firmware/google/vpd_decode.h6
-rw-r--r--drivers/firmware/imx/Kconfig11
-rw-r--r--drivers/firmware/imx/Makefile1
-rw-r--r--drivers/firmware/imx/imx-dsp.c155
-rw-r--r--drivers/firmware/imx/scu-pd.c4
-rw-r--r--drivers/firmware/iscsi_ibft.c4
-rw-r--r--drivers/firmware/psci/psci.c167
-rw-r--r--drivers/firmware/psci/psci_checker.c26
-rw-r--r--drivers/firmware/qcom_scm.c47
-rw-r--r--drivers/firmware/raspberrypi.c10
-rw-r--r--drivers/firmware/stratix10-rsu.c451
-rw-r--r--drivers/firmware/stratix10-svc.c76
-rw-r--r--drivers/firmware/tegra/bpmp.c4
-rw-r--r--drivers/firmware/ti_sci.c1028
-rw-r--r--drivers/firmware/ti_sci.h875
-rw-r--r--drivers/firmware/turris-mox-rwtm.c384
-rw-r--r--drivers/fmc/Kconfig52
-rw-r--r--drivers/fmc/Makefile15
-rw-r--r--drivers/fmc/fmc-chardev.c199
-rw-r--r--drivers/fmc/fmc-core.c388
-rw-r--r--drivers/fmc/fmc-debug.c172
-rw-r--r--drivers/fmc/fmc-dump.c58
-rw-r--r--drivers/fmc/fmc-fakedev.c355
-rw-r--r--drivers/fmc/fmc-match.c113
-rw-r--r--drivers/fmc/fmc-private.h8
-rw-r--r--drivers/fmc/fmc-sdb.c219
-rw-r--r--drivers/fmc/fmc-trivial.c103
-rw-r--r--drivers/fmc/fmc-write-eeprom.c175
-rw-r--r--drivers/fmc/fru-parse.c80
-rw-r--r--drivers/fpga/Kconfig13
-rw-r--r--drivers/fpga/Makefile3
-rw-r--r--drivers/fpga/altera-cvp.c342
-rw-r--r--drivers/fpga/altera-pr-ip-core-plat.c4
-rw-r--r--drivers/fpga/altera-pr-ip-core.c4
-rw-r--r--drivers/fpga/altera-ps-spi.c11
-rw-r--r--drivers/fpga/dfl-afu-dma-region.c53
-rw-r--r--drivers/fpga/dfl-afu-error.c230
-rw-r--r--drivers/fpga/dfl-afu-main.c381
-rw-r--r--drivers/fpga/dfl-afu.h9
-rw-r--r--drivers/fpga/dfl-fme-error.c359
-rw-r--r--drivers/fpga/dfl-fme-main.c128
-rw-r--r--drivers/fpga/dfl-fme-mgr.c4
-rw-r--r--drivers/fpga/dfl-fme-pr.c24
-rw-r--r--drivers/fpga/dfl-fme.h6
-rw-r--r--drivers/fpga/dfl-pci.c36
-rw-r--r--drivers/fpga/dfl.c226
-rw-r--r--drivers/fpga/dfl.h52
-rw-r--r--drivers/fpga/fpga-bridge.c8
-rw-r--r--drivers/fpga/fpga-mgr.c8
-rw-r--r--drivers/fpga/of-fpga-region.c7
-rw-r--r--drivers/fsi/cf-fsi-fw.h2
-rw-r--r--drivers/fsi/fsi-core.c32
-rw-r--r--drivers/fsi/fsi-occ.c15
-rw-r--r--drivers/fsi/fsi-sbefifo.c4
-rw-r--r--drivers/fsi/fsi-scom.c8
-rw-r--r--drivers/gpio/Kconfig55
-rw-r--r--drivers/gpio/Makefile297
-rw-r--r--drivers/gpio/TODO40
-rw-r--r--drivers/gpio/gpio-altera.c65
-rw-r--r--drivers/gpio/gpio-amd-fch.c4
-rw-r--r--drivers/gpio/gpio-amdpt.c10
-rw-r--r--drivers/gpio/gpio-arizona.c4
-rw-r--r--drivers/gpio/gpio-aspeed.c117
-rw-r--r--drivers/gpio/gpio-ath79.c66
-rw-r--r--drivers/gpio/gpio-bd70528.c232
-rw-r--r--drivers/gpio/gpio-brcmstb.c4
-rw-r--r--drivers/gpio/gpio-cadence.c36
-rw-r--r--drivers/gpio/gpio-creg-snps.c4
-rw-r--r--drivers/gpio/gpio-cs5535.c2
-rw-r--r--drivers/gpio/gpio-davinci.c12
-rw-r--r--drivers/gpio/gpio-dwapb.c1
-rw-r--r--drivers/gpio/gpio-eic-sprd.c13
-rw-r--r--drivers/gpio/gpio-em.c100
-rw-r--r--drivers/gpio/gpio-ep93xx.c147
-rw-r--r--drivers/gpio/gpio-ftgpio010.c37
-rw-r--r--drivers/gpio/gpio-grgpio.c6
-rw-r--r--drivers/gpio/gpio-hlwd.c58
-rw-r--r--drivers/gpio/gpio-htc-egpio.c14
-rw-r--r--drivers/gpio/gpio-intel-mid.c35
-rw-r--r--drivers/gpio/gpio-iop.c1
-rw-r--r--drivers/gpio/gpio-ixp4xx.c277
-rw-r--r--drivers/gpio/gpio-janz-ttl.c9
-rw-r--r--drivers/gpio/gpio-ks8695.c284
-rw-r--r--drivers/gpio/gpio-lpc32xx.c118
-rw-r--r--drivers/gpio/gpio-lynxpoint.c35
-rw-r--r--drivers/gpio/gpio-madera.c18
-rw-r--r--drivers/gpio/gpio-max732x.c45
-rw-r--r--drivers/gpio/gpio-max77620.c4
-rw-r--r--drivers/gpio/gpio-max77650.c1
-rw-r--r--drivers/gpio/gpio-mb86s7x.c52
-rw-r--r--drivers/gpio/gpio-merrifield.c28
-rw-r--r--drivers/gpio/gpio-mockup.c22
-rw-r--r--drivers/gpio/gpio-moxtet.c179
-rw-r--r--drivers/gpio/gpio-mpc8xxx.c32
-rw-r--r--drivers/gpio/gpio-mt7621.c33
-rw-r--r--drivers/gpio/gpio-mvebu.c11
-rw-r--r--drivers/gpio/gpio-mxc.c9
-rw-r--r--drivers/gpio/gpio-omap.c509
-rw-r--r--drivers/gpio/gpio-pca953x.c86
-rw-r--r--drivers/gpio/gpio-pch.c6
-rw-r--r--drivers/gpio/gpio-pl061.c30
-rw-r--r--drivers/gpio/gpio-pmic-eic-sprd.c4
-rw-r--r--drivers/gpio/gpio-rcar.c2
-rw-r--r--drivers/gpio/gpio-siox.c51
-rw-r--r--drivers/gpio/gpio-sprd.c4
-rw-r--r--drivers/gpio/gpio-stmpe.c36
-rw-r--r--drivers/gpio/gpio-stp-xway.c33
-rw-r--r--drivers/gpio/gpio-tb10x.c4
-rw-r--r--drivers/gpio/gpio-tegra.c8
-rw-r--r--drivers/gpio/gpio-thunderx.c163
-rw-r--r--drivers/gpio/gpio-tqmx86.c50
-rw-r--r--drivers/gpio/gpio-vf610.c40
-rw-r--r--drivers/gpio/gpio-viperboard.c20
-rw-r--r--drivers/gpio/gpio-vr41xx.c19
-rw-r--r--drivers/gpio/gpio-xgene-sb.c1
-rw-r--r--drivers/gpio/gpio-xilinx.c90
-rw-r--r--drivers/gpio/gpio-xlp.c29
-rw-r--r--drivers/gpio/gpio-zx.c35
-rw-r--r--drivers/gpio/gpio-zynq.c41
-rw-r--r--drivers/gpio/gpiolib-acpi.c75
-rw-r--r--drivers/gpio/gpiolib-acpi.h96
-rw-r--r--drivers/gpio/gpiolib-devres.c28
-rw-r--r--drivers/gpio/gpiolib-of.c236
-rw-r--r--drivers/gpio/gpiolib-of.h38
-rw-r--r--drivers/gpio/gpiolib.c833
-rw-r--r--drivers/gpio/gpiolib.h122
-rw-r--r--drivers/gpio/sgpio-aspeed.c533
-rw-r--r--drivers/gpu/drm/Kconfig15
-rw-r--r--drivers/gpu/drm/Makefile11
-rw-r--r--drivers/gpu/drm/amd/amdgpu/Kconfig6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/Makefile36
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu.h81
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c1
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_afmt.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c99
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h10
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c975
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c85
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c231
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c55
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c1
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c9
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c198
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c28
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c196
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h1
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_device.c507
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c415
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h34
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_display.c8
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c (renamed from drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c)133
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.h46
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h40
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c60
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h21
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c205
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_encoders.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c18
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c57
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h23
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c27
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h16
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c182
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h90
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c1
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c12
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c9
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ioc32.c3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_job.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_job.h3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c32
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h101
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c211
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h50
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_object.c9
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_object.h2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_pll.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c479
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_pm.h2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c280
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h37
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c230
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h61
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c319
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h15
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c15
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h17
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h98
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c29
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_socbb.h82
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c1
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_test.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c365
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h19
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c114
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h71
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c197
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h95
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c51
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h14
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c16
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h12
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c33
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c81
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/athub_v2_0.c101
-rw-r--r--drivers/gpu/drm/amd/amdgpu/athub_v2_0.h30
-rw-r--r--drivers/gpu/drm/amd/amdgpu/atom.h3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/atombios_crtc.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/atombios_dp.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/atombios_encoders.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/atombios_i2c.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/cik.c16
-rw-r--r--drivers/gpu/drm/amd/amdgpu/cik_ih.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/cik_sdma.c6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/clearstate_gfx10.h975
-rw-r--r--drivers/gpu/drm/amd/amdgpu/cz_ih.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v10_0.c5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v11_0.c5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v6_0.c7
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v8_0.c5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_virtual.c5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/df_v1_7.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/df_v3_6.c391
-rw-r--r--drivers/gpu/drm/amd/amdgpu/df_v3_6.h10
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c5216
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v10_0.h (renamed from drivers/gpu/drm/amd/amdgpu/vi_dpm.h)11
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c10
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c55
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c124
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c560
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c31
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c354
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.h35
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c918
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v10_0.h30
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c19
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c58
-rw-r--r--drivers/gpu/drm/amd/amdgpu/iceland_ih.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/kv_dpm.c1
-rw-r--r--drivers/gpu/drm/amd/amdgpu/kv_smc.c1
-rw-r--r--drivers/gpu/drm/amd/amdgpu/mes_v10_1.c366
-rw-r--r--drivers/gpu/drm/amd/amdgpu/mes_v10_1.h29
-rw-r--r--drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c28
-rw-r--r--drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c445
-rw-r--r--drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.h35
-rw-r--r--drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c18
-rw-r--r--drivers/gpu/drm/amd/amdgpu/navi10_ih.c486
-rw-r--r--drivers/gpu/drm/amd/amdgpu/navi10_ih.h29
-rw-r--r--drivers/gpu/drm/amd/amdgpu/navi10_reg_init.c68
-rw-r--r--drivers/gpu/drm/amd/amdgpu/navi10_sdma_pkt_open.h4806
-rw-r--r--drivers/gpu/drm/amd/amdgpu/nbio_v2_3.c334
-rw-r--r--drivers/gpu/drm/amd/amdgpu/nbio_v2_3.h (renamed from drivers/gpu/drm/i915/i915_gemfs.h)25
-rw-r--r--drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c15
-rw-r--r--drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c15
-rw-r--r--drivers/gpu/drm/amd/amdgpu/nv.c820
-rw-r--r--drivers/gpu/drm/amd/amdgpu/nv.h33
-rw-r--r--drivers/gpu/drm/amd/amdgpu/nvd.h418
-rw-r--r--drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h126
-rw-r--r--drivers/gpu/drm/amd/amdgpu/psp_v10_0.c3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/psp_v11_0.c173
-rw-r--r--drivers/gpu/drm/amd/amdgpu/psp_v3_1.c139
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c7
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c7
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c57
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c1687
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v5_0.h45
-rw-r--r--drivers/gpu/drm/amd/amdgpu/si.c20
-rw-r--r--drivers/gpu/drm/amd/amdgpu/si_dma.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/si_dpm.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/si_ih.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/si_smc.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/soc15.c129
-rw-r--r--drivers/gpu/drm/amd/amdgpu/soc15.h20
-rw-r--r--drivers/gpu/drm/amd/amdgpu/soc15_common.h68
-rw-r--r--drivers/gpu/drm/amd/amdgpu/ta_ras_if.h108
-rw-r--r--drivers/gpu/drm/amd/amdgpu/tonga_ih.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c9
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c8
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vce_v2_0.c3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vce_v3_0.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vce_v4_0.c3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c150
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c2288
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vcn_v2_0.h29
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vega10_ih.c95
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vi.c18
-rw-r--r--drivers/gpu/drm/amd/amdkfd/Makefile3
-rw-r--r--drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h782
-rw-r--r--drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm1124
-rw-r--r--drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm13
-rw-r--r--drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm63
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_chardev.c57
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_crat.c18
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_crat.h3
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c36
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device.c105
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c662
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h18
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c2
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v10.c88
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c3
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c2
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_events.c2
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c4
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_iommu.c10
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c25
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h1
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v10.c348
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c6
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c4
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_module.c6
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c90
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h24
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c134
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c497
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c155
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c143
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c13
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_ai.h16
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h7
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_priv.h73
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_process.c101
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c74
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_topology.c30
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_topology.h3
-rw-r--r--drivers/gpu/drm/amd/display/Kconfig22
-rw-r--r--drivers/gpu/drm/amd/display/Makefile1
-rw-r--r--drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c664
-rw-r--r--drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h48
-rw-r--r--drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c473
-rw-r--r--drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c1
-rw-r--r--drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c110
-rw-r--r--drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.h2
-rw-r--r--drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c11
-rw-r--r--drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c4
-rw-r--r--drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.c302
-rw-r--r--drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c1
-rw-r--r--drivers/gpu/drm/amd/display/dc/Makefile18
-rw-r--r--drivers/gpu/drm/amd/display/dc/basics/vector.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/bios/bios_parser.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c8
-rw-r--r--drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c5
-rw-r--r--drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_auto.h1
-rw-r--r--drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_math.c20
-rw-r--r--drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_math.h3
-rw-r--r--drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c75
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile87
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c143
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c471
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.h59
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dce110/dce110_clk_mgr.c276
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dce110/dce110_clk_mgr.h44
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dce112/dce112_clk_mgr.c239
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dce112/dce112_clk_mgr.h39
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dce120/dce120_clk_mgr.c153
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dce120/dce120_clk_mgr.h34
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c (renamed from drivers/gpu/drm/amd/display/dc/dcn10/dcn10_clk_mgr.c)198
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.h31
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_clk.c79
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_clk.h29
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_vbios_smu.c126
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_vbios_smu.h32
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv2_clk_mgr.c43
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv2_clk_mgr.h (renamed from drivers/gpu/drm/amd/display/dc/dcn10/dcn10_clk_mgr.h)13
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c394
-rw-r--r--drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.h48
-rw-r--r--drivers/gpu/drm/amd/display/dc/core/dc.c524
-rw-r--r--drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c31
-rw-r--r--drivers/gpu/drm/amd/display/dc/core/dc_link.c317
-rw-r--r--drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c16
-rw-r--r--drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c266
-rw-r--r--drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c144
-rw-r--r--drivers/gpu/drm/amd/display/dc/core/dc_resource.c204
-rw-r--r--drivers/gpu/drm/amd/display/dc/core/dc_sink.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/core/dc_stream.c261
-rw-r--r--drivers/gpu/drm/amd/display/dc/core/dc_surface.c75
-rw-r--r--drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c93
-rw-r--r--drivers/gpu/drm/amd/display/dc/dc.h144
-rw-r--r--drivers/gpu/drm/amd/display/dc/dc_dp_types.h127
-rw-r--r--drivers/gpu/drm/amd/display/dc/dc_dsc.h62
-rw-r--r--drivers/gpu/drm/amd/display/dc/dc_helper.c5
-rw-r--r--drivers/gpu/drm/amd/display/dc/dc_hw_types.h122
-rw-r--r--drivers/gpu/drm/amd/display/dc/dc_link.h11
-rw-r--r--drivers/gpu/drm/amd/display/dc/dc_stream.h76
-rw-r--r--drivers/gpu/drm/amd/display/dc/dc_types.h118
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/Makefile2
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_abm.c19
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_abm.h20
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_audio.c4
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_audio.h7
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_aux.c3
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_aux.h10
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c87
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.h42
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c97
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h10
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h127
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c109
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.h30
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c3
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c3
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_opp.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c16
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce100/dce100_hw_sequencer.c9
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c75
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.h5
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c3
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c216
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce110/dce110_opp_regamma_v.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c69
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.h5
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c5
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.h5
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator_v.c5
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce112/dce112_compressor.c3
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c33
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c39
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c96
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c52
-rw-r--r--drivers/gpu/drm/amd/display/dc/dce80/dce80_timing_generator.c7
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/Makefile2
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.h31
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c10
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h5
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c4
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c8
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dwb.c136
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dwb.h271
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c471
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h16
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c34
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h8
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c218
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h4
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_ipp.c26
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_ipp.h43
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c11
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h174
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c6
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c10
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c213
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h91
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c76
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.h5
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c129
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h79
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/Makefile23
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.c164
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h116
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp.c502
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp.h698
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp_cm.c990
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.c694
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.h575
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb.c332
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb.h458
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb_scl.c877
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.c592
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.h107
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c700
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.h277
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c2049
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.h103
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.c460
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.h173
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.c323
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.h544
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mpc.c526
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mpc.h285
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_opp.c355
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_opp.h158
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_optc.c542
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_optc.h116
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c3191
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.h133
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_stream_encoder.c610
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_stream_encoder.h107
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_vmid.c96
-rw-r--r--drivers/gpu/drm/amd/display/dc/dcn20/dcn20_vmid.h90
-rw-r--r--drivers/gpu/drm/amd/display/dc/dm_helpers.h7
-rw-r--r--drivers/gpu/drm/amd/display/dc/dm_pp_smu.h142
-rw-r--r--drivers/gpu/drm/amd/display/dc/dml/Makefile14
-rw-r--r--drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c5104
-rw-r--r--drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.h32
-rw-r--r--drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c1701
-rw-r--r--drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.h74
-rw-r--r--drivers/gpu/drm/amd/display/dc/dml/display_mode_enums.h12
-rw-r--r--drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c22
-rw-r--r--drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h36
-rw-r--r--drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h32
-rw-r--r--drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c839
-rw-r--r--drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h854
-rw-r--r--drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h8
-rw-r--r--drivers/gpu/drm/amd/display/dc/dsc/Makefile21
-rw-r--r--drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c858
-rw-r--r--drivers/gpu/drm/amd/display/dc/dsc/drm_dsc_dc.c388
-rw-r--r--drivers/gpu/drm/amd/display/dc/dsc/dscc_types.h54
-rw-r--r--drivers/gpu/drm/amd/display/dc/dsc/qp_tables.h706
-rw-r--r--drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c258
-rw-r--r--drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h85
-rw-r--r--drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c147
-rw-r--r--drivers/gpu/drm/amd/display/dc/gpio/Makefile11
-rw-r--r--drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_factory_dcn20.c212
-rw-r--r--drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_factory_dcn20.h33
-rw-r--r--drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_translate_dcn20.c382
-rw-r--r--drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_translate_dcn20.h35
-rw-r--r--drivers/gpu/drm/amd/display/dc/gpio/ddc_regs.h53
-rw-r--r--drivers/gpu/drm/amd/display/dc/gpio/gpio_base.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c18
-rw-r--r--drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c13
-rw-r--r--drivers/gpu/drm/amd/display/dc/gpio/hw_hpd.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c11
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/core_status.h5
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/core_types.h95
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h7
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h2
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw/abm.h2
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw/audio.h1
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h31
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h (renamed from drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.h)220
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h58
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h70
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw/dsc.h101
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw/dwb.h180
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h30
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h51
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h28
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw/mcif_wb.h105
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h52
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw/opp.h29
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h66
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h60
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw/vmid.h1
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h58
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/resource.h8
-rw-r--r--drivers/gpu/drm/amd/display/dc/inc/vm_helper.h16
-rw-r--r--drivers/gpu/drm/amd/display/dc/irq/Makefile10
-rw-r--r--drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/irq/dce120/irq_service_dce120.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/irq/dce80/irq_service_dce80.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c4
-rw-r--r--drivers/gpu/drm/amd/display/dc/irq/dcn20/irq_service_dcn20.c375
-rw-r--r--drivers/gpu/drm/amd/display/dc/irq/dcn20/irq_service_dcn20.h34
-rw-r--r--drivers/gpu/drm/amd/display/dc/irq/irq_service.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/os_types.h8
-rw-r--r--drivers/gpu/drm/amd/display/dc/virtual/virtual_link_encoder.c2
-rw-r--r--drivers/gpu/drm/amd/display/dc/virtual/virtual_stream_encoder.c17
-rw-r--r--drivers/gpu/drm/amd/display/include/bios_parser_types.h3
-rw-r--r--drivers/gpu/drm/amd/display/include/dal_asic_id.h20
-rw-r--r--drivers/gpu/drm/amd/display/include/dal_types.h5
-rw-r--r--drivers/gpu/drm/amd/display/include/dpcd_defs.h2
-rw-r--r--drivers/gpu/drm/amd/display/include/logger_types.h10
-rw-r--r--drivers/gpu/drm/amd/display/include/set_mode_types.h5
-rw-r--r--drivers/gpu/drm/amd/display/modules/color/color_gamma.c62
-rw-r--r--drivers/gpu/drm/amd/display/modules/color/color_gamma.h1
-rw-r--r--drivers/gpu/drm/amd/display/modules/freesync/freesync.c2
-rw-r--r--drivers/gpu/drm/amd/display/modules/inc/mod_shared.h60
-rw-r--r--drivers/gpu/drm/amd/display/modules/inc/mod_vmid.h46
-rw-r--r--drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c4
-rw-r--r--drivers/gpu/drm/amd/display/modules/power/Makefile2
-rw-r--r--drivers/gpu/drm/amd/display/modules/vmid/vmid.c167
-rw-r--r--drivers/gpu/drm/amd/include/amd_shared.h13
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/athub/athub_2_0_0_default.h272
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/athub/athub_2_0_0_offset.h514
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/athub/athub_2_0_0_sh_mask.h2264
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/clk/clk_11_0_0_offset.h33
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/clk/clk_11_0_0_sh_mask.h38
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_2_0_0_offset.h17535
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_2_0_0_sh_mask.h68024
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_offset.h18
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_1_0_default.h6028
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_1_0_offset.h11339
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_1_0_sh_mask.h43963
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_offset.h31
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/hdp/hdp_5_0_0_offset.h217
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/hdp/hdp_5_0_0_sh_mask.h659
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_2_0_0_default.h927
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_2_0_0_offset.h1799
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_2_0_0_sh_mask.h7567
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/mp/mp_11_0_sh_mask.h429
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_2_3_default.h18521
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_2_3_offset.h14663
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_2_3_sh_mask.h120339
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_6_1_smn.h3
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_0_smn.h3
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_4_0_smn.h3
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/oss/osssys_5_0_0_offset.h353
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/oss/osssys_5_0_0_sh_mask.h1305
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_11_0_0_offset.h323
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_11_0_0_sh_mask.h689
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/vcn/vcn_2_0_0_offset.h1008
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/vcn/vcn_2_0_0_sh_mask.h3815
-rw-r--r--drivers/gpu/drm/amd/include/atomfirmware.h188
-rw-r--r--drivers/gpu/drm/amd/include/cik_structs.h3
-rw-r--r--drivers/gpu/drm/amd/include/discovery.h165
-rw-r--r--drivers/gpu/drm/amd/include/ivsrcid/dcn/irqsrcs_dcn_1_0.h (renamed from drivers/gpu/drm/amd/include/ivsrcid/irqsrcs_dcn_1_0.h)0
-rw-r--r--drivers/gpu/drm/amd/include/ivsrcid/gfx/irqsrcs_gfx_10_1.h53
-rw-r--r--drivers/gpu/drm/amd/include/ivsrcid/sdma0/irqsrcs_sdma0_5_0.h43
-rw-r--r--drivers/gpu/drm/amd/include/ivsrcid/sdma1/irqsrcs_sdma1_5_0.h44
-rw-r--r--drivers/gpu/drm/amd/include/ivsrcid/vcn/irqsrcs_vcn_2_0.h32
-rw-r--r--drivers/gpu/drm/amd/include/kgd_kfd_interface.h1
-rw-r--r--drivers/gpu/drm/amd/include/kgd_pp_interface.h12
-rw-r--r--drivers/gpu/drm/amd/include/navi10_enum.h22764
-rw-r--r--drivers/gpu/drm/amd/include/navi10_ip_offset.h855
-rw-r--r--drivers/gpu/drm/amd/include/soc15_hw_ip.h4
-rw-r--r--drivers/gpu/drm/amd/include/v10_structs.h1258
-rw-r--r--drivers/gpu/drm/amd/include/v9_structs.h3
-rw-r--r--drivers/gpu/drm/amd/include/vi_structs.h3
-rw-r--r--drivers/gpu/drm/amd/powerplay/Makefile2
-rw-r--r--drivers/gpu/drm/amd/powerplay/amdgpu_smu.c557
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c18
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c1
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c2
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c9
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c8
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/smu_helper.c3
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c157
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/vega10_processpptables.c25
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/vega10_processpptables.h1
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c123
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.h3
-rw-r--r--drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.c150
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/amdgpu_smu.h370
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/hwmgr.h1
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/power_state.h7
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/pp_thermal.h12
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/smu11_driver_if.h6
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/smu11_driver_if_navi10.h1069
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/smu_v11_0.h29
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_ppsmc.h39
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_pptable.h2
-rw-r--r--drivers/gpu/drm/amd/powerplay/inc/smumgr.h1
-rw-r--r--drivers/gpu/drm/amd/powerplay/navi10_ppt.c1706
-rw-r--r--drivers/gpu/drm/amd/powerplay/navi10_ppt.h32
-rw-r--r--drivers/gpu/drm/amd/powerplay/smu_v11_0.c1270
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c6
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c3
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c6
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c5
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/smu10_smumgr.c3
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/smu8_smumgr.c1
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c6
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c3
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/vega12_smumgr.c22
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c1
-rw-r--r--drivers/gpu/drm/amd/powerplay/smumgr/vegam_smumgr.c3
-rw-r--r--drivers/gpu/drm/amd/powerplay/vega20_ppt.c1319
-rw-r--r--drivers/gpu/drm/amd/powerplay/vega20_ppt.h50
-rw-r--r--drivers/gpu/drm/arm/display/include/malidp_io.h7
-rw-r--r--drivers/gpu/drm/arm/display/include/malidp_utils.h5
-rw-r--r--drivers/gpu/drm/arm/display/komeda/Makefile2
-rw-r--r--drivers/gpu/drm/arm/display/komeda/d71/d71_component.c582
-rw-r--r--drivers/gpu/drm/arm/display/komeda/d71/d71_dev.c142
-rw-r--r--drivers/gpu/drm/arm/display/komeda/d71/d71_dev.h2
-rw-r--r--drivers/gpu/drm/arm/display/komeda/komeda_color_mgmt.c67
-rw-r--r--drivers/gpu/drm/arm/display/komeda/komeda_color_mgmt.h17
-rw-r--r--drivers/gpu/drm/arm/display/komeda/komeda_crtc.c91
-rw-r--r--drivers/gpu/drm/arm/display/komeda/komeda_dev.c70
-rw-r--r--drivers/gpu/drm/arm/display/komeda/komeda_dev.h13
-rw-r--r--drivers/gpu/drm/arm/display/komeda/komeda_format_caps.c77
-rw-r--r--drivers/gpu/drm/arm/display/komeda/komeda_format_caps.h27
-rw-r--r--drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.c176
-rw-r--r--drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.h13
-rw-r--r--drivers/gpu/drm/arm/display/komeda/komeda_kms.c150
-rw-r--r--drivers/gpu/drm/arm/display/komeda/komeda_kms.h57
-rw-r--r--drivers/gpu/drm/arm/display/komeda/komeda_pipeline.c66
-rw-r--r--drivers/gpu/drm/arm/display/komeda/komeda_pipeline.h113
-rw-r--r--drivers/gpu/drm/arm/display/komeda/komeda_pipeline_state.c692
-rw-r--r--drivers/gpu/drm/arm/display/komeda/komeda_plane.c109
-rw-r--r--drivers/gpu/drm/arm/display/komeda/komeda_private_obj.c154
-rw-r--r--drivers/gpu/drm/arm/display/komeda/komeda_wb_connector.c191
-rw-r--r--drivers/gpu/drm/arm/malidp_crtc.c28
-rw-r--r--drivers/gpu/drm/arm/malidp_drv.c11
-rw-r--r--drivers/gpu/drm/arm/malidp_hw.c3
-rw-r--r--drivers/gpu/drm/arm/malidp_mw.c2
-rw-r--r--drivers/gpu/drm/arm/malidp_planes.c8
-rw-r--r--drivers/gpu/drm/armada/armada_510.c130
-rw-r--r--drivers/gpu/drm/armada/armada_crtc.c214
-rw-r--r--drivers/gpu/drm/armada/armada_crtc.h21
-rw-r--r--drivers/gpu/drm/armada/armada_debugfs.c98
-rw-r--r--drivers/gpu/drm/armada/armada_drm.h1
-rw-r--r--drivers/gpu/drm/armada/armada_drv.c38
-rw-r--r--drivers/gpu/drm/armada/armada_fb.c3
-rw-r--r--drivers/gpu/drm/armada/armada_hw.h29
-rw-r--r--drivers/gpu/drm/armada/armada_overlay.c56
-rw-r--r--drivers/gpu/drm/armada/armada_plane.c124
-rw-r--r--drivers/gpu/drm/armada/armada_plane.h23
-rw-r--r--drivers/gpu/drm/ast/Kconfig3
-rw-r--r--drivers/gpu/drm/ast/ast_drv.c13
-rw-r--r--drivers/gpu/drm/ast/ast_drv.h78
-rw-r--r--drivers/gpu/drm/ast/ast_fb.c61
-rw-r--r--drivers/gpu/drm/ast/ast_main.c82
-rw-r--r--drivers/gpu/drm/ast/ast_mode.c159
-rw-r--r--drivers/gpu/drm/ast/ast_post.c2
-rw-r--r--drivers/gpu/drm/ast/ast_ttm.c302
-rw-r--r--drivers/gpu/drm/ati_pcigart.c5
-rw-r--r--drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c18
-rw-r--r--drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c120
-rw-r--r--drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h2
-rw-r--r--drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c11
-rw-r--r--drivers/gpu/drm/bochs/Kconfig2
-rw-r--r--drivers/gpu/drm/bochs/bochs.h56
-rw-r--r--drivers/gpu/drm/bochs/bochs_drv.c24
-rw-r--r--drivers/gpu/drm/bochs/bochs_hw.c14
-rw-r--r--drivers/gpu/drm/bochs/bochs_kms.c20
-rw-r--r--drivers/gpu/drm/bochs/bochs_mm.c427
-rw-r--r--drivers/gpu/drm/bridge/Kconfig4
-rw-r--r--drivers/gpu/drm/bridge/adv7511/adv7511_drv.c8
-rw-r--r--drivers/gpu/drm/bridge/analogix-anx78xx.c9
-rw-r--r--drivers/gpu/drm/bridge/analogix/analogix_dp_core.c58
-rw-r--r--drivers/gpu/drm/bridge/analogix/analogix_dp_core.h6
-rw-r--r--drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c24
-rw-r--r--drivers/gpu/drm/bridge/dumb-vga-dac.c2
-rw-r--r--drivers/gpu/drm/bridge/lvds-encoder.c10
-rw-r--r--drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c3
-rw-r--r--drivers/gpu/drm/bridge/nxp-ptn3460.c3
-rw-r--r--drivers/gpu/drm/bridge/panel.c5
-rw-r--r--drivers/gpu/drm/bridge/parade-ps8622.c3
-rw-r--r--drivers/gpu/drm/bridge/sii902x.c491
-rw-r--r--drivers/gpu/drm/bridge/sii9234.c4
-rw-r--r--drivers/gpu/drm/bridge/synopsys/dw-hdmi.c193
-rw-r--r--drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c17
-rw-r--r--drivers/gpu/drm/bridge/tc358764.c14
-rw-r--r--drivers/gpu/drm/bridge/tc358767.c593
-rw-r--r--drivers/gpu/drm/bridge/thc63lvd1024.c64
-rw-r--r--drivers/gpu/drm/bridge/ti-sn65dsi86.c18
-rw-r--r--drivers/gpu/drm/bridge/ti-tfp410.c14
-rw-r--r--drivers/gpu/drm/cirrus/cirrus_ttm.c337
-rw-r--r--drivers/gpu/drm/drm_agpsupport.c11
-rw-r--r--drivers/gpu/drm/drm_atomic.c248
-rw-r--r--drivers/gpu/drm/drm_atomic_helper.c232
-rw-r--r--drivers/gpu/drm/drm_atomic_state_helper.c70
-rw-r--r--drivers/gpu/drm/drm_atomic_uapi.c23
-rw-r--r--drivers/gpu/drm/drm_auth.c30
-rw-r--r--drivers/gpu/drm/drm_blend.c9
-rw-r--r--drivers/gpu/drm/drm_bridge.c110
-rw-r--r--drivers/gpu/drm/drm_bufs.c21
-rw-r--r--drivers/gpu/drm/drm_client.c75
-rw-r--r--drivers/gpu/drm/drm_client_modeset.c1126
-rw-r--r--drivers/gpu/drm/drm_color_mgmt.c8
-rw-r--r--drivers/gpu/drm/drm_connector.c99
-rw-r--r--drivers/gpu/drm/drm_context.c8
-rw-r--r--drivers/gpu/drm/drm_crtc.c4
-rw-r--r--drivers/gpu/drm/drm_crtc_helper.c14
-rw-r--r--drivers/gpu/drm/drm_crtc_internal.h31
-rw-r--r--drivers/gpu/drm/drm_damage_helper.c2
-rw-r--r--drivers/gpu/drm/drm_debugfs.c92
-rw-r--r--drivers/gpu/drm/drm_debugfs_crc.c46
-rw-r--r--drivers/gpu/drm/drm_dma.c6
-rw-r--r--drivers/gpu/drm/drm_dp_aux_dev.c8
-rw-r--r--drivers/gpu/drm/drm_dp_dual_mode_helper.c4
-rw-r--r--drivers/gpu/drm/drm_dp_helper.c16
-rw-r--r--drivers/gpu/drm/drm_dp_mst_topology.c19
-rw-r--r--drivers/gpu/drm/drm_drv.c34
-rw-r--r--drivers/gpu/drm/drm_dumb_buffers.c4
-rw-r--r--drivers/gpu/drm/drm_edid.c232
-rw-r--r--drivers/gpu/drm/drm_edid_load.c9
-rw-r--r--drivers/gpu/drm/drm_encoder.c4
-rw-r--r--drivers/gpu/drm/drm_fb_helper.c1459
-rw-r--r--drivers/gpu/drm/drm_file.c133
-rw-r--r--drivers/gpu/drm/drm_flip_work.c6
-rw-r--r--drivers/gpu/drm/drm_format_helper.c4
-rw-r--r--drivers/gpu/drm/drm_fourcc.c120
-rw-r--r--drivers/gpu/drm/drm_framebuffer.c15
-rw-r--r--drivers/gpu/drm/drm_gem.c40
-rw-r--r--drivers/gpu/drm/drm_gem_cma_helper.c11
-rw-r--r--drivers/gpu/drm/drm_gem_framebuffer_helper.c7
-rw-r--r--drivers/gpu/drm/drm_gem_vram_helper.c641
-rw-r--r--drivers/gpu/drm/drm_hashtab.c10
-rw-r--r--drivers/gpu/drm/drm_hdcp.c382
-rw-r--r--drivers/gpu/drm/drm_internal.h42
-rw-r--r--drivers/gpu/drm/drm_ioc32.c9
-rw-r--r--drivers/gpu/drm/drm_ioctl.c24
-rw-r--r--drivers/gpu/drm/drm_irq.c13
-rw-r--r--drivers/gpu/drm/drm_kms_helper_common.c3
-rw-r--r--drivers/gpu/drm/drm_lease.c15
-rw-r--r--drivers/gpu/drm/drm_legacy.h6
-rw-r--r--drivers/gpu/drm/drm_legacy_misc.c27
-rw-r--r--drivers/gpu/drm/drm_lock.c8
-rw-r--r--drivers/gpu/drm/drm_memory.c9
-rw-r--r--drivers/gpu/drm/drm_mipi_dsi.c7
-rw-r--r--drivers/gpu/drm/drm_mm.c9
-rw-r--r--drivers/gpu/drm/drm_mode_config.c6
-rw-r--r--drivers/gpu/drm/drm_mode_object.c9
-rw-r--r--drivers/gpu/drm/drm_modes.c532
-rw-r--r--drivers/gpu/drm/drm_modeset_lock.c4
-rw-r--r--drivers/gpu/drm/drm_of.c5
-rw-r--r--drivers/gpu/drm/drm_panel_orientation_quirks.c12
-rw-r--r--drivers/gpu/drm/drm_pci.c11
-rw-r--r--drivers/gpu/drm/drm_plane_helper.c9
-rw-r--r--drivers/gpu/drm/drm_prime.c84
-rw-r--r--drivers/gpu/drm/drm_print.c7
-rw-r--r--drivers/gpu/drm/drm_probe_helper.c16
-rw-r--r--drivers/gpu/drm/drm_property.c7
-rw-r--r--drivers/gpu/drm/drm_rect.c4
-rw-r--r--drivers/gpu/drm/drm_scatter.c9
-rw-r--r--drivers/gpu/drm/drm_scdc_helper.c2
-rw-r--r--drivers/gpu/drm/drm_self_refresh_helper.c218
-rw-r--r--drivers/gpu/drm/drm_simple_kms_helper.c5
-rw-r--r--drivers/gpu/drm/drm_syncobj.c13
-rw-r--r--drivers/gpu/drm/drm_sysfs.c17
-rw-r--r--drivers/gpu/drm/drm_trace.h2
-rw-r--r--drivers/gpu/drm/drm_trace_points.c3
-rw-r--r--drivers/gpu/drm/drm_vblank.c22
-rw-r--r--drivers/gpu/drm/drm_vm.c19
-rw-r--r--drivers/gpu/drm/drm_vma_manager.c6
-rw-r--r--drivers/gpu/drm/drm_vram_helper_common.c96
-rw-r--r--drivers/gpu/drm/drm_vram_mm_helper.c297
-rw-r--r--drivers/gpu/drm/drm_writeback.c6
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_dump.c5
-rw-r--r--drivers/gpu/drm/etnaviv/etnaviv_sched.c2
-rw-r--r--drivers/gpu/drm/exynos/Kconfig7
-rw-r--r--drivers/gpu/drm/exynos/exynos5433_drm_decon.c7
-rw-r--r--drivers/gpu/drm/exynos/exynos7_drm_decon.c8
-rw-r--r--drivers/gpu/drm/exynos/exynos_dp.c13
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_crtc.c2
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_dma.c6
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_dpi.c8
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_drv.c21
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_drv.h8
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_dsi.c21
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_fb.c6
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_fbdev.c8
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_fimc.c17
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_fimd.c14
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_g2d.c13
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_gem.c7
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_gsc.c15
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_ipp.c3
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_mic.c22
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_plane.c4
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_rotator.c10
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_scaler.c16
-rw-r--r--drivers/gpu/drm/exynos/exynos_drm_vidi.c9
-rw-r--r--drivers/gpu/drm/exynos/exynos_hdmi.c41
-rw-r--r--drivers/gpu/drm/exynos/exynos_mixer.c31
-rw-r--r--drivers/gpu/drm/gma500/accel_2d.c18
-rw-r--r--drivers/gpu/drm/gma500/blitter.h2
-rw-r--r--drivers/gpu/drm/gma500/cdv_device.c13
-rw-r--r--drivers/gpu/drm/gma500/cdv_device.h4
-rw-r--r--drivers/gpu/drm/gma500/cdv_intel_crt.c8
-rw-r--r--drivers/gpu/drm/gma500/cdv_intel_display.c10
-rw-r--r--drivers/gpu/drm/gma500/cdv_intel_dp.c9
-rw-r--r--drivers/gpu/drm/gma500/cdv_intel_hdmi.c9
-rw-r--r--drivers/gpu/drm/gma500/cdv_intel_lvds.c9
-rw-r--r--drivers/gpu/drm/gma500/framebuffer.c26
-rw-r--r--drivers/gpu/drm/gma500/framebuffer.h1
-rw-r--r--drivers/gpu/drm/gma500/gem.c5
-rw-r--r--drivers/gpu/drm/gma500/gma_device.c1
-rw-r--r--drivers/gpu/drm/gma500/gma_device.h1
-rw-r--r--drivers/gpu/drm/gma500/gma_display.c12
-rw-r--r--drivers/gpu/drm/gma500/gma_display.h3
-rw-r--r--drivers/gpu/drm/gma500/gtt.c5
-rw-r--r--drivers/gpu/drm/gma500/gtt.h1
-rw-r--r--drivers/gpu/drm/gma500/intel_bios.c6
-rw-r--r--drivers/gpu/drm/gma500/intel_bios.h3
-rw-r--r--drivers/gpu/drm/gma500/intel_gmbus.c11
-rw-r--r--drivers/gpu/drm/gma500/intel_i2c.c4
-rw-r--r--drivers/gpu/drm/gma500/mdfld_device.c16
-rw-r--r--drivers/gpu/drm/gma500/mdfld_dsi_dpi.c4
-rw-r--r--drivers/gpu/drm/gma500/mdfld_dsi_output.c12
-rw-r--r--drivers/gpu/drm/gma500/mdfld_dsi_output.h8
-rw-r--r--drivers/gpu/drm/gma500/mdfld_dsi_pkg_sender.c4
-rw-r--r--drivers/gpu/drm/gma500/mdfld_intel_display.c11
-rw-r--r--drivers/gpu/drm/gma500/mdfld_tmd_vid.c2
-rw-r--r--drivers/gpu/drm/gma500/mid_bios.c5
-rw-r--r--drivers/gpu/drm/gma500/mid_bios.h1
-rw-r--r--drivers/gpu/drm/gma500/mmu.c6
-rw-r--r--drivers/gpu/drm/gma500/oaktrail.h2
-rw-r--r--drivers/gpu/drm/gma500/oaktrail_crtc.c8
-rw-r--r--drivers/gpu/drm/gma500/oaktrail_device.c20
-rw-r--r--drivers/gpu/drm/gma500/oaktrail_hdmi.c8
-rw-r--r--drivers/gpu/drm/gma500/oaktrail_lvds.c6
-rw-r--r--drivers/gpu/drm/gma500/oaktrail_lvds_i2c.c11
-rw-r--r--drivers/gpu/drm/gma500/power.h4
-rw-r--r--drivers/gpu/drm/gma500/psb_device.c12
-rw-r--r--drivers/gpu/drm/gma500/psb_drv.c33
-rw-r--r--drivers/gpu/drm/gma500/psb_drv.h16
-rw-r--r--drivers/gpu/drm/gma500/psb_intel_display.c7
-rw-r--r--drivers/gpu/drm/gma500/psb_intel_lvds.c5
-rw-r--r--drivers/gpu/drm/gma500/psb_intel_modes.c2
-rw-r--r--drivers/gpu/drm/gma500/psb_intel_sdvo.c15
-rw-r--r--drivers/gpu/drm/gma500/psb_irq.c9
-rw-r--r--drivers/gpu/drm/gma500/psb_irq.h2
-rw-r--r--drivers/gpu/drm/gma500/psb_lid.c6
-rw-r--r--drivers/gpu/drm/gma500/tc35876x-dsi-lvds.c13
-rw-r--r--drivers/gpu/drm/hisilicon/hibmc/Kconfig2
-rw-r--r--drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c19
-rw-r--r--drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c14
-rw-r--r--drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h33
-rw-r--r--drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_fbdev.c37
-rw-r--r--drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c341
-rw-r--r--drivers/gpu/drm/i2c/tda998x_drv.c450
-rw-r--r--drivers/gpu/drm/i915/.gitignore1
-rw-r--r--drivers/gpu/drm/i915/Kconfig35
-rw-r--r--drivers/gpu/drm/i915/Kconfig.debug15
-rw-r--r--drivers/gpu/drm/i915/Kconfig.profile27
-rw-r--r--drivers/gpu/drm/i915/Makefile202
-rw-r--r--drivers/gpu/drm/i915/Makefile.header-test49
-rw-r--r--drivers/gpu/drm/i915/display/Makefile2
-rw-r--r--drivers/gpu/drm/i915/display/Makefile.header-test16
-rw-r--r--drivers/gpu/drm/i915/display/dvo_ch7017.c (renamed from drivers/gpu/drm/i915/dvo_ch7017.c)3
-rw-r--r--drivers/gpu/drm/i915/display/dvo_ch7xxx.c (renamed from drivers/gpu/drm/i915/dvo_ch7xxx.c)3
-rw-r--r--drivers/gpu/drm/i915/display/dvo_ivch.c (renamed from drivers/gpu/drm/i915/dvo_ivch.c)3
-rw-r--r--drivers/gpu/drm/i915/display/dvo_ns2501.c (renamed from drivers/gpu/drm/i915/dvo_ns2501.c)5
-rw-r--r--drivers/gpu/drm/i915/display/dvo_sil164.c (renamed from drivers/gpu/drm/i915/dvo_sil164.c)3
-rw-r--r--drivers/gpu/drm/i915/display/dvo_tfp410.c (renamed from drivers/gpu/drm/i915/dvo_tfp410.c)3
-rw-r--r--drivers/gpu/drm/i915/display/icl_dsi.c (renamed from drivers/gpu/drm/i915/icl_dsi.c)171
-rw-r--r--drivers/gpu/drm/i915/display/intel_acpi.c (renamed from drivers/gpu/drm/i915/intel_acpi.c)3
-rw-r--r--drivers/gpu/drm/i915/display/intel_acpi.h17
-rw-r--r--drivers/gpu/drm/i915/display/intel_atomic.c (renamed from drivers/gpu/drm/i915/intel_atomic.c)35
-rw-r--r--drivers/gpu/drm/i915/display/intel_atomic.h49
-rw-r--r--drivers/gpu/drm/i915/display/intel_atomic_plane.c (renamed from drivers/gpu/drm/i915/intel_atomic_plane.c)72
-rw-r--r--drivers/gpu/drm/i915/display/intel_atomic_plane.h (renamed from drivers/gpu/drm/i915/intel_atomic_plane.h)10
-rw-r--r--drivers/gpu/drm/i915/display/intel_audio.c (renamed from drivers/gpu/drm/i915/intel_audio.c)61
-rw-r--r--drivers/gpu/drm/i915/display/intel_audio.h (renamed from drivers/gpu/drm/i915/intel_audio.h)0
-rw-r--r--drivers/gpu/drm/i915/display/intel_bios.c (renamed from drivers/gpu/drm/i915/intel_bios.c)214
-rw-r--r--drivers/gpu/drm/i915/display/intel_bios.h (renamed from drivers/gpu/drm/i915/intel_bios.h)21
-rw-r--r--drivers/gpu/drm/i915/display/intel_bw.c426
-rw-r--r--drivers/gpu/drm/i915/display/intel_bw.h47
-rw-r--r--drivers/gpu/drm/i915/display/intel_cdclk.c (renamed from drivers/gpu/drm/i915/intel_cdclk.c)307
-rw-r--r--drivers/gpu/drm/i915/display/intel_cdclk.h (renamed from drivers/gpu/drm/i915/intel_cdclk.h)0
-rw-r--r--drivers/gpu/drm/i915/display/intel_color.c (renamed from drivers/gpu/drm/i915/intel_color.c)248
-rw-r--r--drivers/gpu/drm/i915/display/intel_color.h (renamed from drivers/gpu/drm/i915/intel_color.h)1
-rw-r--r--drivers/gpu/drm/i915/display/intel_combo_phy.c (renamed from drivers/gpu/drm/i915/intel_combo_phy.c)87
-rw-r--r--drivers/gpu/drm/i915/display/intel_combo_phy.h20
-rw-r--r--drivers/gpu/drm/i915/display/intel_connector.c (renamed from drivers/gpu/drm/i915/intel_connector.c)3
-rw-r--r--drivers/gpu/drm/i915/display/intel_connector.h (renamed from drivers/gpu/drm/i915/intel_connector.h)0
-rw-r--r--drivers/gpu/drm/i915/display/intel_crt.c (renamed from drivers/gpu/drm/i915/intel_crt.c)44
-rw-r--r--drivers/gpu/drm/i915/display/intel_crt.h (renamed from drivers/gpu/drm/i915/intel_crt.h)0
-rw-r--r--drivers/gpu/drm/i915/display/intel_ddi.c (renamed from drivers/gpu/drm/i915/intel_ddi.c)97
-rw-r--r--drivers/gpu/drm/i915/display/intel_ddi.h (renamed from drivers/gpu/drm/i915/intel_ddi.h)1
-rw-r--r--drivers/gpu/drm/i915/display/intel_display.c (renamed from drivers/gpu/drm/i915/intel_display.c)1780
-rw-r--r--drivers/gpu/drm/i915/display/intel_display.h (renamed from drivers/gpu/drm/i915/intel_display.h)90
-rw-r--r--drivers/gpu/drm/i915/display/intel_display_power.c4625
-rw-r--r--drivers/gpu/drm/i915/display/intel_display_power.h288
-rw-r--r--drivers/gpu/drm/i915/display/intel_dp.c (renamed from drivers/gpu/drm/i915/intel_dp.c)311
-rw-r--r--drivers/gpu/drm/i915/display/intel_dp.h (renamed from drivers/gpu/drm/i915/intel_dp.h)1
-rw-r--r--drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c (renamed from drivers/gpu/drm/i915/intel_dp_aux_backlight.c)1
-rw-r--r--drivers/gpu/drm/i915/display/intel_dp_aux_backlight.h13
-rw-r--r--drivers/gpu/drm/i915/display/intel_dp_link_training.c (renamed from drivers/gpu/drm/i915/intel_dp_link_training.c)1
-rw-r--r--drivers/gpu/drm/i915/display/intel_dp_link_training.h14
-rw-r--r--drivers/gpu/drm/i915/display/intel_dp_mst.c (renamed from drivers/gpu/drm/i915/intel_dp_mst.c)30
-rw-r--r--drivers/gpu/drm/i915/display/intel_dp_mst.h14
-rw-r--r--drivers/gpu/drm/i915/display/intel_dpio_phy.c (renamed from drivers/gpu/drm/i915/intel_dpio_phy.c)42
-rw-r--r--drivers/gpu/drm/i915/display/intel_dpio_phy.h58
-rw-r--r--drivers/gpu/drm/i915/display/intel_dpll_mgr.c (renamed from drivers/gpu/drm/i915/intel_dpll_mgr.c)87
-rw-r--r--drivers/gpu/drm/i915/display/intel_dpll_mgr.h (renamed from drivers/gpu/drm/i915/intel_dpll_mgr.h)12
-rw-r--r--drivers/gpu/drm/i915/display/intel_dsi.c (renamed from drivers/gpu/drm/i915/intel_dsi.c)0
-rw-r--r--drivers/gpu/drm/i915/display/intel_dsi.h (renamed from drivers/gpu/drm/i915/intel_dsi.h)8
-rw-r--r--drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.c (renamed from drivers/gpu/drm/i915/intel_dsi_dcs_backlight.c)8
-rw-r--r--drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.h13
-rw-r--r--drivers/gpu/drm/i915/display/intel_dsi_vbt.c (renamed from drivers/gpu/drm/i915/intel_dsi_vbt.c)364
-rw-r--r--drivers/gpu/drm/i915/display/intel_dvo.c (renamed from drivers/gpu/drm/i915/intel_dvo.c)8
-rw-r--r--drivers/gpu/drm/i915/display/intel_dvo.h (renamed from drivers/gpu/drm/i915/intel_dvo.h)0
-rw-r--r--drivers/gpu/drm/i915/display/intel_dvo_dev.h (renamed from drivers/gpu/drm/i915/dvo.h)10
-rw-r--r--drivers/gpu/drm/i915/display/intel_fbc.c (renamed from drivers/gpu/drm/i915/intel_fbc.c)4
-rw-r--r--drivers/gpu/drm/i915/display/intel_fbc.h (renamed from drivers/gpu/drm/i915/intel_fbc.h)0
-rw-r--r--drivers/gpu/drm/i915/display/intel_fbdev.c (renamed from drivers/gpu/drm/i915/intel_fbdev.c)8
-rw-r--r--drivers/gpu/drm/i915/display/intel_fbdev.h (renamed from drivers/gpu/drm/i915/intel_fbdev.h)0
-rw-r--r--drivers/gpu/drm/i915/display/intel_fifo_underrun.c (renamed from drivers/gpu/drm/i915/intel_fifo_underrun.c)1
-rw-r--r--drivers/gpu/drm/i915/display/intel_fifo_underrun.h27
-rw-r--r--drivers/gpu/drm/i915/display/intel_frontbuffer.c (renamed from drivers/gpu/drm/i915/intel_frontbuffer.c)7
-rw-r--r--drivers/gpu/drm/i915/display/intel_frontbuffer.h (renamed from drivers/gpu/drm/i915/intel_frontbuffer.h)2
-rw-r--r--drivers/gpu/drm/i915/display/intel_gmbus.c (renamed from drivers/gpu/drm/i915/intel_i2c.c)100
-rw-r--r--drivers/gpu/drm/i915/display/intel_gmbus.h27
-rw-r--r--drivers/gpu/drm/i915/display/intel_hdcp.c (renamed from drivers/gpu/drm/i915/intel_hdcp.c)56
-rw-r--r--drivers/gpu/drm/i915/display/intel_hdcp.h (renamed from drivers/gpu/drm/i915/intel_hdcp.h)1
-rw-r--r--drivers/gpu/drm/i915/display/intel_hdmi.c (renamed from drivers/gpu/drm/i915/intel_hdmi.c)175
-rw-r--r--drivers/gpu/drm/i915/display/intel_hdmi.h (renamed from drivers/gpu/drm/i915/intel_hdmi.h)0
-rw-r--r--drivers/gpu/drm/i915/display/intel_hotplug.c (renamed from drivers/gpu/drm/i915/intel_hotplug.c)5
-rw-r--r--drivers/gpu/drm/i915/display/intel_hotplug.h30
-rw-r--r--drivers/gpu/drm/i915/display/intel_lpe_audio.c (renamed from drivers/gpu/drm/i915/intel_lpe_audio.c)8
-rw-r--r--drivers/gpu/drm/i915/display/intel_lpe_audio.h22
-rw-r--r--drivers/gpu/drm/i915/display/intel_lspcon.c (renamed from drivers/gpu/drm/i915/intel_lspcon.c)0
-rw-r--r--drivers/gpu/drm/i915/display/intel_lspcon.h (renamed from drivers/gpu/drm/i915/intel_lspcon.h)0
-rw-r--r--drivers/gpu/drm/i915/display/intel_lvds.c (renamed from drivers/gpu/drm/i915/intel_lvds.c)2
-rw-r--r--drivers/gpu/drm/i915/display/intel_lvds.h (renamed from drivers/gpu/drm/i915/intel_lvds.h)0
-rw-r--r--drivers/gpu/drm/i915/display/intel_opregion.c (renamed from drivers/gpu/drm/i915/intel_opregion.c)3
-rw-r--r--drivers/gpu/drm/i915/display/intel_opregion.h (renamed from drivers/gpu/drm/i915/intel_opregion.h)0
-rw-r--r--drivers/gpu/drm/i915/display/intel_overlay.c (renamed from drivers/gpu/drm/i915/intel_overlay.c)40
-rw-r--r--drivers/gpu/drm/i915/display/intel_overlay.h29
-rw-r--r--drivers/gpu/drm/i915/display/intel_panel.c (renamed from drivers/gpu/drm/i915/intel_panel.c)4
-rw-r--r--drivers/gpu/drm/i915/display/intel_panel.h (renamed from drivers/gpu/drm/i915/intel_panel.h)0
-rw-r--r--drivers/gpu/drm/i915/display/intel_pipe_crc.c (renamed from drivers/gpu/drm/i915/intel_pipe_crc.c)14
-rw-r--r--drivers/gpu/drm/i915/display/intel_pipe_crc.h (renamed from drivers/gpu/drm/i915/intel_pipe_crc.h)3
-rw-r--r--drivers/gpu/drm/i915/display/intel_psr.c (renamed from drivers/gpu/drm/i915/intel_psr.c)51
-rw-r--r--drivers/gpu/drm/i915/display/intel_psr.h (renamed from drivers/gpu/drm/i915/intel_psr.h)0
-rw-r--r--drivers/gpu/drm/i915/display/intel_quirks.c (renamed from drivers/gpu/drm/i915/intel_quirks.c)1
-rw-r--r--drivers/gpu/drm/i915/display/intel_quirks.h13
-rw-r--r--drivers/gpu/drm/i915/display/intel_sdvo.c (renamed from drivers/gpu/drm/i915/intel_sdvo.c)34
-rw-r--r--drivers/gpu/drm/i915/display/intel_sdvo.h (renamed from drivers/gpu/drm/i915/intel_sdvo.h)0
-rw-r--r--drivers/gpu/drm/i915/display/intel_sdvo_regs.h (renamed from drivers/gpu/drm/i915/intel_sdvo_regs.h)8
-rw-r--r--drivers/gpu/drm/i915/display/intel_sprite.c (renamed from drivers/gpu/drm/i915/intel_sprite.c)45
-rw-r--r--drivers/gpu/drm/i915/display/intel_sprite.h (renamed from drivers/gpu/drm/i915/intel_sprite.h)12
-rw-r--r--drivers/gpu/drm/i915/display/intel_tv.c (renamed from drivers/gpu/drm/i915/intel_tv.c)9
-rw-r--r--drivers/gpu/drm/i915/display/intel_tv.h (renamed from drivers/gpu/drm/i915/intel_tv.h)0
-rw-r--r--drivers/gpu/drm/i915/display/intel_vbt_defs.h (renamed from drivers/gpu/drm/i915/intel_vbt_defs.h)633
-rw-r--r--drivers/gpu/drm/i915/display/intel_vdsc.c (renamed from drivers/gpu/drm/i915/intel_vdsc.c)4
-rw-r--r--drivers/gpu/drm/i915/display/intel_vdsc.h21
-rw-r--r--drivers/gpu/drm/i915/display/vlv_dsi.c (renamed from drivers/gpu/drm/i915/vlv_dsi.c)230
-rw-r--r--drivers/gpu/drm/i915/display/vlv_dsi_pll.c (renamed from drivers/gpu/drm/i915/vlv_dsi_pll.c)22
-rw-r--r--drivers/gpu/drm/i915/gem/Makefile1
-rw-r--r--drivers/gpu/drm/i915/gem/Makefile.header-test16
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_busy.c139
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_clflush.c (renamed from drivers/gpu/drm/i915/i915_gem_clflush.c)34
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_clflush.h20
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_client_blt.c304
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_client_blt.h21
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_context.c (renamed from drivers/gpu/drm/i915/i915_gem_context.c)1200
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_context.h (renamed from drivers/gpu/drm/i915/i915_gem_context.h)106
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_context_types.h (renamed from drivers/gpu/drm/i915/i915_gem_context_types.h)59
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c (renamed from drivers/gpu/drm/i915/i915_gem_dmabuf.c)42
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_domain.c796
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c (renamed from drivers/gpu/drm/i915/i915_gem_execbuffer.c)379
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_fence.c96
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_internal.c (renamed from drivers/gpu/drm/i915/i915_gem_internal.c)34
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_ioctls.h52
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_mman.c508
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_object.c398
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_object.h430
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_object_blt.c107
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_object_blt.h24
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_object_types.h262
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_pages.c544
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_phys.c212
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_pm.c295
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_pm.h25
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_shmem.c571
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_shrinker.c (renamed from drivers/gpu/drm/i915/i915_gem_shrinker.c)181
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_stolen.c (renamed from drivers/gpu/drm/i915/i915_gem_stolen.c)41
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_throttle.c73
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_tiling.c (renamed from drivers/gpu/drm/i915/i915_gem_tiling.c)31
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_userptr.c (renamed from drivers/gpu/drm/i915/i915_gem_userptr.c)40
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gem_wait.c278
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gemfs.c (renamed from drivers/gpu/drm/i915/i915_gemfs.c)22
-rw-r--r--drivers/gpu/drm/i915/gem/i915_gemfs.h16
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c (renamed from drivers/gpu/drm/i915/selftests/huge_gem_object.c)24
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/huge_gem_object.h27
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/huge_pages.c (renamed from drivers/gpu/drm/i915/selftests/huge_pages.c)105
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c127
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c (renamed from drivers/gpu/drm/i915/selftests/i915_gem_coherency.c)56
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c (renamed from drivers/gpu/drm/i915/selftests/i915_gem_context.c)379
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c (renamed from drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c)35
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c (renamed from drivers/gpu/drm/i915/selftests/i915_gem_object.c)237
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c99
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/i915_gem_object_blt.c110
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/i915_gem_phys.c80
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c34
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.h17
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/mock_context.c (renamed from drivers/gpu/drm/i915/selftests/mock_context.c)45
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/mock_context.h24
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c (renamed from drivers/gpu/drm/i915/selftests/mock_dmabuf.c)22
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.h22
-rw-r--r--drivers/gpu/drm/i915/gem/selftests/mock_gem_object.h (renamed from drivers/gpu/drm/i915/selftests/mock_gem_object.h)7
-rw-r--r--drivers/gpu/drm/i915/gt/Makefile2
-rw-r--r--drivers/gpu/drm/i915/gt/Makefile.header-test16
-rw-r--r--drivers/gpu/drm/i915/gt/intel_breadcrumbs.c (renamed from drivers/gpu/drm/i915/intel_breadcrumbs.c)19
-rw-r--r--drivers/gpu/drm/i915/gt/intel_context.c250
-rw-r--r--drivers/gpu/drm/i915/gt/intel_context.h134
-rw-r--r--drivers/gpu/drm/i915/gt/intel_context_types.h (renamed from drivers/gpu/drm/i915/intel_context_types.h)29
-rw-r--r--drivers/gpu/drm/i915/gt/intel_engine.h (renamed from drivers/gpu/drm/i915/intel_ringbuffer.h)81
-rw-r--r--drivers/gpu/drm/i915/gt/intel_engine_cs.c (renamed from drivers/gpu/drm/i915/intel_engine_cs.c)537
-rw-r--r--drivers/gpu/drm/i915/gt/intel_engine_pm.c144
-rw-r--r--drivers/gpu/drm/i915/gt/intel_engine_pm.h28
-rw-r--r--drivers/gpu/drm/i915/gt/intel_engine_types.h (renamed from drivers/gpu/drm/i915/intel_engine_types.h)72
-rw-r--r--drivers/gpu/drm/i915/gt/intel_gpu_commands.h (renamed from drivers/gpu/drm/i915/intel_gpu_commands.h)1
-rw-r--r--drivers/gpu/drm/i915/gt/intel_gt_pm.c162
-rw-r--r--drivers/gpu/drm/i915/gt/intel_gt_pm.h27
-rw-r--r--drivers/gpu/drm/i915/gt/intel_hangcheck.c (renamed from drivers/gpu/drm/i915/intel_hangcheck.c)35
-rw-r--r--drivers/gpu/drm/i915/gt/intel_lrc.c (renamed from drivers/gpu/drm/i915/intel_lrc.c)1384
-rw-r--r--drivers/gpu/drm/i915/gt/intel_lrc.h (renamed from drivers/gpu/drm/i915/intel_lrc.h)34
-rw-r--r--drivers/gpu/drm/i915/gt/intel_lrc_reg.h (renamed from drivers/gpu/drm/i915/intel_lrc_reg.h)2
-rw-r--r--drivers/gpu/drm/i915/gt/intel_mocs.c (renamed from drivers/gpu/drm/i915/intel_mocs.c)12
-rw-r--r--drivers/gpu/drm/i915/gt/intel_mocs.h (renamed from drivers/gpu/drm/i915/intel_mocs.h)4
-rw-r--r--drivers/gpu/drm/i915/gt/intel_reset.c (renamed from drivers/gpu/drm/i915/i915_reset.c)272
-rw-r--r--drivers/gpu/drm/i915/gt/intel_reset.h (renamed from drivers/gpu/drm/i915/i915_reset.h)5
-rw-r--r--drivers/gpu/drm/i915/gt/intel_ringbuffer.c (renamed from drivers/gpu/drm/i915/intel_ringbuffer.c)497
-rw-r--r--drivers/gpu/drm/i915/gt/intel_sseu.c159
-rw-r--r--drivers/gpu/drm/i915/gt/intel_sseu.h75
-rw-r--r--drivers/gpu/drm/i915/gt/intel_workarounds.c (renamed from drivers/gpu/drm/i915/intel_workarounds.c)444
-rw-r--r--drivers/gpu/drm/i915/gt/intel_workarounds.h (renamed from drivers/gpu/drm/i915/intel_workarounds.h)10
-rw-r--r--drivers/gpu/drm/i915/gt/intel_workarounds_types.h (renamed from drivers/gpu/drm/i915/intel_workarounds_types.h)7
-rw-r--r--drivers/gpu/drm/i915/gt/mock_engine.c (renamed from drivers/gpu/drm/i915/selftests/mock_engine.c)68
-rw-r--r--drivers/gpu/drm/i915/gt/mock_engine.h (renamed from drivers/gpu/drm/i915/selftests/mock_engine.h)4
-rw-r--r--drivers/gpu/drm/i915/gt/selftest_engine_cs.c (renamed from drivers/gpu/drm/i915/selftests/intel_engine_cs.c)0
-rw-r--r--drivers/gpu/drm/i915/gt/selftest_hangcheck.c (renamed from drivers/gpu/drm/i915/selftests/intel_hangcheck.c)270
-rw-r--r--drivers/gpu/drm/i915/gt/selftest_lrc.c (renamed from drivers/gpu/drm/i915/selftests/intel_lrc.c)583
-rw-r--r--drivers/gpu/drm/i915/gt/selftest_reset.c121
-rw-r--r--drivers/gpu/drm/i915/gt/selftest_workarounds.c (renamed from drivers/gpu/drm/i915/selftests/intel_workarounds.c)502
-rw-r--r--drivers/gpu/drm/i915/gvt/aperture_gm.c24
-rw-r--r--drivers/gpu/drm/i915/gvt/cmd_parser.c37
-rw-r--r--drivers/gpu/drm/i915/gvt/debugfs.c4
-rw-r--r--drivers/gpu/drm/i915/gvt/fb_decoder.c6
-rw-r--r--drivers/gpu/drm/i915/gvt/firmware.c5
-rw-r--r--drivers/gpu/drm/i915/gvt/gtt.c9
-rw-r--r--drivers/gpu/drm/i915/gvt/gvt.h10
-rw-r--r--drivers/gpu/drm/i915/gvt/kvmgt.c14
-rw-r--r--drivers/gpu/drm/i915/gvt/mmio_context.c4
-rw-r--r--drivers/gpu/drm/i915/gvt/opregion.c2
-rw-r--r--drivers/gpu/drm/i915/gvt/sched_policy.c4
-rw-r--r--drivers/gpu/drm/i915/gvt/scheduler.c243
-rw-r--r--drivers/gpu/drm/i915/gvt/trace_points.c2
-rw-r--r--drivers/gpu/drm/i915/i915_active.c96
-rw-r--r--drivers/gpu/drm/i915/i915_active.h7
-rw-r--r--drivers/gpu/drm/i915/i915_active_types.h3
-rw-r--r--drivers/gpu/drm/i915/i915_cmd_parser.c26
-rw-r--r--drivers/gpu/drm/i915/i915_debugfs.c555
-rw-r--r--drivers/gpu/drm/i915/i915_debugfs.h20
-rw-r--r--drivers/gpu/drm/i915/i915_drv.c167
-rw-r--r--drivers/gpu/drm/i915/i915_drv.h1034
-rw-r--r--drivers/gpu/drm/i915/i915_fixed.h6
-rw-r--r--drivers/gpu/drm/i915/i915_gem.c4095
-rw-r--r--drivers/gpu/drm/i915/i915_gem.h8
-rw-r--r--drivers/gpu/drm/i915/i915_gem_batch_pool.c6
-rw-r--r--drivers/gpu/drm/i915/i915_gem_batch_pool.h3
-rw-r--r--drivers/gpu/drm/i915/i915_gem_clflush.h36
-rw-r--r--drivers/gpu/drm/i915/i915_gem_evict.c49
-rw-r--r--drivers/gpu/drm/i915/i915_gem_fence_reg.c207
-rw-r--r--drivers/gpu/drm/i915/i915_gem_fence_reg.h19
-rw-r--r--drivers/gpu/drm/i915/i915_gem_gtt.c1020
-rw-r--r--drivers/gpu/drm/i915/i915_gem_gtt.h167
-rw-r--r--drivers/gpu/drm/i915/i915_gem_object.c90
-rw-r--r--drivers/gpu/drm/i915/i915_gem_object.h509
-rw-r--r--drivers/gpu/drm/i915/i915_gem_render_state.c8
-rw-r--r--drivers/gpu/drm/i915/i915_globals.c4
-rw-r--r--drivers/gpu/drm/i915/i915_gpu_error.c149
-rw-r--r--drivers/gpu/drm/i915/i915_gpu_error.h7
-rw-r--r--drivers/gpu/drm/i915/i915_irq.c170
-rw-r--r--drivers/gpu/drm/i915/i915_irq.h117
-rw-r--r--drivers/gpu/drm/i915/i915_mm.c3
-rw-r--r--drivers/gpu/drm/i915/i915_params.c7
-rw-r--r--drivers/gpu/drm/i915/i915_params.h3
-rw-r--r--drivers/gpu/drm/i915/i915_pci.c63
-rw-r--r--drivers/gpu/drm/i915/i915_perf.c175
-rw-r--r--drivers/gpu/drm/i915/i915_pmu.c28
-rw-r--r--drivers/gpu/drm/i915/i915_query.c66
-rw-r--r--drivers/gpu/drm/i915/i915_reg.h107
-rw-r--r--drivers/gpu/drm/i915/i915_request.c548
-rw-r--r--drivers/gpu/drm/i915/i915_request.h19
-rw-r--r--drivers/gpu/drm/i915/i915_scatterlist.c39
-rw-r--r--drivers/gpu/drm/i915/i915_scatterlist.h127
-rw-r--r--drivers/gpu/drm/i915/i915_scheduler.c91
-rw-r--r--drivers/gpu/drm/i915/i915_scheduler.h18
-rw-r--r--drivers/gpu/drm/i915/i915_scheduler_types.h2
-rw-r--r--drivers/gpu/drm/i915/i915_suspend.c6
-rw-r--r--drivers/gpu/drm/i915/i915_sysfs.c65
-rw-r--r--drivers/gpu/drm/i915/i915_timeline.c14
-rw-r--r--drivers/gpu/drm/i915/i915_timeline.h19
-rw-r--r--drivers/gpu/drm/i915/i915_timeline_types.h3
-rw-r--r--drivers/gpu/drm/i915/i915_trace.h85
-rw-r--r--drivers/gpu/drm/i915/i915_utils.h187
-rw-r--r--drivers/gpu/drm/i915/i915_vgpu.c3
-rw-r--r--drivers/gpu/drm/i915/i915_vma.c134
-rw-r--r--drivers/gpu/drm/i915/i915_vma.h38
-rw-r--r--drivers/gpu/drm/i915/intel_context.c270
-rw-r--r--drivers/gpu/drm/i915/intel_context.h87
-rw-r--r--drivers/gpu/drm/i915/intel_csr.c411
-rw-r--r--drivers/gpu/drm/i915/intel_csr.h4
-rw-r--r--drivers/gpu/drm/i915/intel_device_info.c78
-rw-r--r--drivers/gpu/drm/i915/intel_device_info.h90
-rw-r--r--drivers/gpu/drm/i915/intel_drv.h449
-rw-r--r--drivers/gpu/drm/i915/intel_guc.c196
-rw-r--r--drivers/gpu/drm/i915/intel_guc.h20
-rw-r--r--drivers/gpu/drm/i915/intel_guc_ads.c167
-rw-r--r--drivers/gpu/drm/i915/intel_guc_ads.h1
-rw-r--r--drivers/gpu/drm/i915/intel_guc_ct.c16
-rw-r--r--drivers/gpu/drm/i915/intel_guc_ct.h5
-rw-r--r--drivers/gpu/drm/i915/intel_guc_fw.c117
-rw-r--r--drivers/gpu/drm/i915/intel_guc_fwif.h201
-rw-r--r--drivers/gpu/drm/i915/intel_guc_log.c23
-rw-r--r--drivers/gpu/drm/i915/intel_guc_reg.h25
-rw-r--r--drivers/gpu/drm/i915/intel_guc_submission.c62
-rw-r--r--drivers/gpu/drm/i915/intel_guc_submission.h3
-rw-r--r--drivers/gpu/drm/i915/intel_huc.c102
-rw-r--r--drivers/gpu/drm/i915/intel_huc.h13
-rw-r--r--drivers/gpu/drm/i915/intel_huc_fw.c73
-rw-r--r--drivers/gpu/drm/i915/intel_pm.c567
-rw-r--r--drivers/gpu/drm/i915/intel_pm.h19
-rw-r--r--drivers/gpu/drm/i915/intel_runtime_pm.c4252
-rw-r--r--drivers/gpu/drm/i915/intel_runtime_pm.h213
-rw-r--r--drivers/gpu/drm/i915/intel_sideband.c483
-rw-r--r--drivers/gpu/drm/i915/intel_sideband.h141
-rw-r--r--drivers/gpu/drm/i915/intel_uc.c148
-rw-r--r--drivers/gpu/drm/i915/intel_uc.h3
-rw-r--r--drivers/gpu/drm/i915/intel_uc_fw.c126
-rw-r--r--drivers/gpu/drm/i915/intel_uc_fw.h10
-rw-r--r--drivers/gpu/drm/i915/intel_uncore.c55
-rw-r--r--drivers/gpu/drm/i915/intel_uncore.h4
-rw-r--r--drivers/gpu/drm/i915/intel_wakeref.c138
-rw-r--r--drivers/gpu/drm/i915/intel_wakeref.h179
-rw-r--r--drivers/gpu/drm/i915/intel_wopcm.c27
-rw-r--r--drivers/gpu/drm/i915/intel_wopcm.h15
-rw-r--r--drivers/gpu/drm/i915/selftests/huge_gem_object.h45
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_active.c14
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_gem.c35
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_gem_evict.c34
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_gem_gtt.c31
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_live_selftests.h5
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_mock_selftests.h1
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_request.c86
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_timeline.c30
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_vma.c258
-rw-r--r--drivers/gpu/drm/i915/selftests/igt_atomic.h56
-rw-r--r--drivers/gpu/drm/i915/selftests/igt_flush_test.c38
-rw-r--r--drivers/gpu/drm/i915/selftests/igt_reset.c11
-rw-r--r--drivers/gpu/drm/i915/selftests/igt_reset.h1
-rw-r--r--drivers/gpu/drm/i915/selftests/igt_spinner.c20
-rw-r--r--drivers/gpu/drm/i915/selftests/igt_spinner.h10
-rw-r--r--drivers/gpu/drm/i915/selftests/intel_guc.c11
-rw-r--r--drivers/gpu/drm/i915/selftests/intel_uncore.c4
-rw-r--r--drivers/gpu/drm/i915/selftests/lib_sw_fence.c3
-rw-r--r--drivers/gpu/drm/i915/selftests/mock_context.h42
-rw-r--r--drivers/gpu/drm/i915/selftests/mock_dmabuf.h41
-rw-r--r--drivers/gpu/drm/i915/selftests/mock_gem_device.c48
-rw-r--r--drivers/gpu/drm/i915/selftests/mock_gtt.c7
-rw-r--r--drivers/gpu/drm/i915/selftests/mock_gtt.h4
-rw-r--r--drivers/gpu/drm/i915/selftests/mock_request.c6
-rw-r--r--drivers/gpu/drm/i915/selftests/mock_timeline.c1
-rw-r--r--drivers/gpu/drm/i915/selftests/scatterlist.c3
-rw-r--r--drivers/gpu/drm/imx/ipuv3-plane.c16
-rw-r--r--drivers/gpu/drm/ingenic/Kconfig16
-rw-r--r--drivers/gpu/drm/ingenic/Makefile1
-rw-r--r--drivers/gpu/drm/ingenic/ingenic-drm.c817
-rw-r--r--drivers/gpu/drm/lima/lima_drv.c2
-rw-r--r--drivers/gpu/drm/lima/lima_gem.c2
-rw-r--r--drivers/gpu/drm/lima/lima_pp.c8
-rw-r--r--drivers/gpu/drm/lima/lima_sched.c13
-rw-r--r--drivers/gpu/drm/mcde/Kconfig18
-rw-r--r--drivers/gpu/drm/mcde/Makefile3
-rw-r--r--drivers/gpu/drm/mcde/mcde_display.c1142
-rw-r--r--drivers/gpu/drm/mcde/mcde_display_regs.h518
-rw-r--r--drivers/gpu/drm/mcde/mcde_drm.h44
-rw-r--r--drivers/gpu/drm/mcde/mcde_drv.c571
-rw-r--r--drivers/gpu/drm/mcde/mcde_dsi.c1044
-rw-r--r--drivers/gpu/drm/mcde/mcde_dsi_regs.h385
-rw-r--r--drivers/gpu/drm/mediatek/mtk_drm_drv.c55
-rw-r--r--drivers/gpu/drm/mediatek/mtk_drm_drv.h2
-rw-r--r--drivers/gpu/drm/mediatek/mtk_drm_fb.c8
-rw-r--r--drivers/gpu/drm/mediatek/mtk_hdmi.c3
-rw-r--r--drivers/gpu/drm/meson/Kconfig1
-rw-r--r--drivers/gpu/drm/meson/meson_overlay.c17
-rw-r--r--drivers/gpu/drm/meson/meson_plane.c19
-rw-r--r--drivers/gpu/drm/mgag200/Kconfig2
-rw-r--r--drivers/gpu/drm/mgag200/mgag200_cursor.c183
-rw-r--r--drivers/gpu/drm/mgag200/mgag200_drv.c13
-rw-r--r--drivers/gpu/drm/mgag200/mgag200_drv.h75
-rw-r--r--drivers/gpu/drm/mgag200/mgag200_fb.c59
-rw-r--r--drivers/gpu/drm/mgag200/mgag200_main.c91
-rw-r--r--drivers/gpu/drm/mgag200/mgag200_mode.c59
-rw-r--r--drivers/gpu/drm/mgag200/mgag200_ttm.c301
-rw-r--r--drivers/gpu/drm/msm/adreno/a3xx_gpu.c24
-rw-r--r--drivers/gpu/drm/msm/adreno/a5xx.xml.h28
-rw-r--r--drivers/gpu/drm/msm/adreno/a5xx_debugfs.c8
-rw-r--r--drivers/gpu/drm/msm/adreno/a5xx_gpu.c42
-rw-r--r--drivers/gpu/drm/msm/adreno/a5xx_power.c76
-rw-r--r--drivers/gpu/drm/msm/adreno/a6xx_gmu.c70
-rw-r--r--drivers/gpu/drm/msm/adreno/a6xx_gmu.h1
-rw-r--r--drivers/gpu/drm/msm/adreno/a6xx_gpu.c17
-rw-r--r--drivers/gpu/drm/msm/adreno/a6xx_gpu.h2
-rw-r--r--drivers/gpu/drm/msm/adreno/adreno_device.c20
-rw-r--r--drivers/gpu/drm/msm/adreno/adreno_gpu.c9
-rw-r--r--drivers/gpu/drm/msm/adreno/adreno_gpu.h6
-rw-r--r--drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c176
-rw-r--r--drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.h4
-rw-r--r--drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c20
-rw-r--r--drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c8
-rw-r--r--drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c119
-rw-r--r--drivers/gpu/drm/msm/disp/dpu1/dpu_io_util.c6
-rw-r--r--drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c46
-rw-r--r--drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h6
-rw-r--r--drivers/gpu/drm/msm/disp/dpu1/dpu_mdss.c57
-rw-r--r--drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c15
-rw-r--r--drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h22
-rw-r--r--drivers/gpu/drm/msm/disp/dpu1/dpu_vbif.c4
-rw-r--r--drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c3
-rw-r--r--drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.c2
-rw-r--r--drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c47
-rw-r--r--drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c40
-rw-r--r--drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c27
-rw-r--r--drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c7
-rw-r--r--drivers/gpu/drm/msm/dsi/dsi.c2
-rw-r--r--drivers/gpu/drm/msm/dsi/dsi.h7
-rw-r--r--drivers/gpu/drm/msm/dsi/dsi_cfg.c21
-rw-r--r--drivers/gpu/drm/msm/dsi/dsi_cfg.h1
-rw-r--r--drivers/gpu/drm/msm/dsi/dsi_host.c19
-rw-r--r--drivers/gpu/drm/msm/dsi/dsi_manager.c149
-rw-r--r--drivers/gpu/drm/msm/dsi/phy/dsi_phy.c6
-rw-r--r--drivers/gpu/drm/msm/dsi/phy/dsi_phy.h5
-rw-r--r--drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c30
-rw-r--r--drivers/gpu/drm/msm/dsi/pll/dsi_pll_10nm.c106
-rw-r--r--drivers/gpu/drm/msm/msm_drv.c37
-rw-r--r--drivers/gpu/drm/msm/msm_drv.h1
-rw-r--r--drivers/gpu/drm/msm/msm_fb.c20
-rw-r--r--drivers/gpu/drm/msm/msm_gem.c53
-rw-r--r--drivers/gpu/drm/msm/msm_gem.h1
-rw-r--r--drivers/gpu/drm/msm/msm_gem_submit.c13
-rw-r--r--drivers/gpu/drm/msm/msm_gpu.c5
-rw-r--r--drivers/gpu/drm/msm/msm_iommu.c2
-rw-r--r--drivers/gpu/drm/msm/msm_perf.c15
-rw-r--r--drivers/gpu/drm/msm/msm_rd.c16
-rw-r--r--drivers/gpu/drm/nouveau/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/Kconfig6
-rw-r--r--drivers/gpu/drm/nouveau/dispnv04/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/dispnv04/cursor.c2
-rw-r--r--drivers/gpu/drm/nouveau/dispnv04/disp.h2
-rw-r--r--drivers/gpu/drm/nouveau/dispnv50/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/dispnv50/disp.c36
-rw-r--r--drivers/gpu/drm/nouveau/dispnv50/head.c41
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl0002.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl0046.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl006b.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl0080.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl506e.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl506f.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl5070.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl507a.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl507b.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl507c.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl507d.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl507e.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl826e.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl826f.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl906f.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cl9097.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/cla06f.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/class.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/clc36f.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/clc37b.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/clc37e.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/client.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/device.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/driver.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/event.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/if0000.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/if0001.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/if0002.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/if0003.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/if0004.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/if0005.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/ioctl.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/notify.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/object.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/os.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvif/unpack.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/client.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/debug.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/device.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/engine.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/enum.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/event.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/firmware.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/gpuobj.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/ioctl.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/memory.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/mm.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/notify.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/object.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/oproxy.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/option.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/os.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/pci.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/ramht.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/core/tegra.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/bsp.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/cipher.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/dma.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/mpeg.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/msenc.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/mspdec.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/msppp.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/msvld.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/nvdec.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/nvenc.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/pm.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/sec.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/sec2.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/sw.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/vic.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/vp.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/engine/xtensa.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bar.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/M0203.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/M0205.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/M0209.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/P0260.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/bit.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/bmp.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/boost.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/conn.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/cstep.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/dcb.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/disp.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/dp.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/extdev.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/fan.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/gpio.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/i2c.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/iccsense.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/image.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/init.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/mxm.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/npde.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/pcir.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/perf.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/pll.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/pmu.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/power_budget.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/ramcfg.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/rammap.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/therm.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/timing.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/vmap.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/volt.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/vpstate.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/xpio.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/bus.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/clk.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/devinit.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/fuse.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/gpio.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/i2c.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/ibus.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/iccsense.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/instmem.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/mc.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/mxm.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/pmu.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/timer.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/top.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/vga.h2
-rw-r--r--drivers/gpu/drm/nouveau/include/nvkm/subdev/volt.h2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_abi16.h2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_acpi.c2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_acpi.h2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_bo.h2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_chan.h2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_connector.c9
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_debugfs.h2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_display.h2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_dmem.c106
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_drv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_fence.h2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_gem.h2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_hwmon.c10
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_ioctl.h2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_reg.h2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_sgdma.c2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_svm.c47
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_ttm.h2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_usif.h2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_vga.c2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_vga.h2
-rw-r--r--drivers/gpu/drm/nouveau/nv10_fence.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvif/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/core/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/bsp/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gf100.fuc3.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gt215.fuc3.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/ce/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/cipher/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/device/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/device/acpi.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/device/base.c38
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/conn.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmi.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmi.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/disp/sortu102.c1
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/dma/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/dma/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/dma/user.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/regsnv04.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxnv40.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf100.fuc3.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf117.fuc3.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk104.fuc3.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk110.fuc3.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk208.fuc5.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgm107.fuc5.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf100.fuc3.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf117.fuc3.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk104.fuc3.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk110.fuc3.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk208.fuc5.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgm107.fuc5.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/os.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv10.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv25.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv2a.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv30.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv34.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv35.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/nv50.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/gr/regs.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/mpeg/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/mpeg/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/msenc/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/mspdec/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/mspdec/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/msppp/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/msppp/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/msvld/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/msvld/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/nvdec/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/nvdec/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/nvenc/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/pm/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/pm/gf100.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/pm/nv40.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/pm/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/sec/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/sec/fuc/g98.fuc0s.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/sec2/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/sw/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/sw/nv50.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/sw/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/vic/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/vp/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/falcon/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/falcon/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/falcon/v1.c36
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bar/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bios/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bios/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bus/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bus/hwsq.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bus/nv04.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/bus/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/clk/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/clk/nv50.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/clk/pll.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/clk/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/clk/seq.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/devinit/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv04.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/devinit/nv50.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/devinit/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fault/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramfuc.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv40.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramseq.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fb/regsnv04.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fuse/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/fuse/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/gpio/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/gpio/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/gsp/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/i2c/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.c24
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c20
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/i2c/bus.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/i2c/pad.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/i2c/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/ibus/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/ibus/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/iccsense/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/iccsense/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/instmem/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mmu/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mxm/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mxm/mxms.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/mxm/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/pci/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/pci/agp.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/pci/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/pmu/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf119.fuc4.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/os.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/pmu/memx.c2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/secboot/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp102.c12
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/secboot/ls_ucode_msgqueue.c29
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/timer/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/timer/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/timer/regsnv04.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/top/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/top/priv.h2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/volt/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/subdev/volt/priv.h2
-rw-r--r--drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c18
-rw-r--r--drivers/gpu/drm/omapdrm/dss/omapdss-boot-init.c1
-rw-r--r--drivers/gpu/drm/omapdrm/dss/output.c4
-rw-r--r--drivers/gpu/drm/omapdrm/omap_crtc.c180
-rw-r--r--drivers/gpu/drm/omapdrm/omap_crtc.h2
-rw-r--r--drivers/gpu/drm/omapdrm/omap_drv.c18
-rw-r--r--drivers/gpu/drm/omapdrm/omap_drv.h4
-rw-r--r--drivers/gpu/drm/omapdrm/omap_fb.c25
-rw-r--r--drivers/gpu/drm/omapdrm/omap_irq.c25
-rw-r--r--drivers/gpu/drm/omapdrm/omap_irq.h1
-rw-r--r--drivers/gpu/drm/panel/Kconfig18
-rw-r--r--drivers/gpu/drm/panel/Makefile2
-rw-r--r--drivers/gpu/drm/panel/panel-arm-versatile.c6
-rw-r--r--drivers/gpu/drm/panel/panel-ilitek-ili9322.c9
-rw-r--r--drivers/gpu/drm/panel/panel-innolux-p079zca.c10
-rw-r--r--drivers/gpu/drm/panel/panel-jdi-lt070me05000.c8
-rw-r--r--drivers/gpu/drm/panel/panel-kingdisplay-kd097d04.c9
-rw-r--r--drivers/gpu/drm/panel/panel-lg-lg4573.c9
-rw-r--r--drivers/gpu/drm/panel/panel-lvds.c7
-rw-r--r--drivers/gpu/drm/panel/panel-olimex-lcd-olinuxino.c8
-rw-r--r--drivers/gpu/drm/panel/panel-orisetech-otm8009a.c11
-rw-r--r--drivers/gpu/drm/panel/panel-osd-osd101t2587-53ts.c254
-rw-r--r--drivers/gpu/drm/panel/panel-panasonic-vvx10f034n00.c7
-rw-r--r--drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c3
-rw-r--r--drivers/gpu/drm/panel/panel-raydium-rm68200.c5
-rw-r--r--drivers/gpu/drm/panel/panel-rocktech-jh057n00900.c14
-rw-r--r--drivers/gpu/drm/panel/panel-samsung-ld9040.c10
-rw-r--r--drivers/gpu/drm/panel/panel-samsung-s6e3ha2.c10
-rw-r--r--drivers/gpu/drm/panel/panel-samsung-s6e63j0x03.c11
-rw-r--r--drivers/gpu/drm/panel/panel-samsung-s6e63m0.c514
-rw-r--r--drivers/gpu/drm/panel/panel-samsung-s6e8aa0.c12
-rw-r--r--drivers/gpu/drm/panel/panel-seiko-43wvf1g.c10
-rw-r--r--drivers/gpu/drm/panel/panel-sharp-lq101r1sx01.c7
-rw-r--r--drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c7
-rw-r--r--drivers/gpu/drm/panel/panel-simple.c276
-rw-r--r--drivers/gpu/drm/panel/panel-sitronix-st7701.c6
-rw-r--r--drivers/gpu/drm/panel/panel-sitronix-st7789v.c10
-rw-r--r--drivers/gpu/drm/panel/panel-truly-nt35597.c13
-rw-r--r--drivers/gpu/drm/panfrost/Makefile3
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_device.c30
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_device.h11
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_drv.c15
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_gem.c8
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_gpu.c10
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_job.c2
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_mmu.c24
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_perfcnt.c329
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_perfcnt.h18
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_regs.h22
-rw-r--r--drivers/gpu/drm/qxl/qxl_drv.c20
-rw-r--r--drivers/gpu/drm/qxl/qxl_release.c2
-rw-r--r--drivers/gpu/drm/r128/r128_cce.c28
-rw-r--r--drivers/gpu/drm/r128/r128_drv.c9
-rw-r--r--drivers/gpu/drm/r128/r128_drv.h16
-rw-r--r--drivers/gpu/drm/r128/r128_state.c25
-rw-r--r--drivers/gpu/drm/radeon/atom.c2
-rw-r--r--drivers/gpu/drm/radeon/atom.h1
-rw-r--r--drivers/gpu/drm/radeon/atombios_crtc.c7
-rw-r--r--drivers/gpu/drm/radeon/atombios_dp.c2
-rw-r--r--drivers/gpu/drm/radeon/atombios_encoders.c14
-rw-r--r--drivers/gpu/drm/radeon/atombios_i2c.c2
-rw-r--r--drivers/gpu/drm/radeon/btc_dpm.c16
-rw-r--r--drivers/gpu/drm/radeon/btc_dpm.h3
-rw-r--r--drivers/gpu/drm/radeon/ci_dpm.c14
-rw-r--r--drivers/gpu/drm/radeon/ci_dpm.h1
-rw-r--r--drivers/gpu/drm/radeon/ci_smc.c2
-rw-r--r--drivers/gpu/drm/radeon/cik.c18
-rw-r--r--drivers/gpu/drm/radeon/cik_sdma.c6
-rw-r--r--drivers/gpu/drm/radeon/clearstate_cayman.h2
-rw-r--r--drivers/gpu/drm/radeon/clearstate_ci.h2
-rw-r--r--drivers/gpu/drm/radeon/clearstate_si.h2
-rw-r--r--drivers/gpu/drm/radeon/cypress_dpm.c11
-rw-r--r--drivers/gpu/drm/radeon/dce3_1_afmt.c2
-rw-r--r--drivers/gpu/drm/radeon/dce6_afmt.c2
-rw-r--r--drivers/gpu/drm/radeon/evergreen.c16
-rw-r--r--drivers/gpu/drm/radeon/evergreen_cs.c2
-rw-r--r--drivers/gpu/drm/radeon/evergreen_dma.c2
-rw-r--r--drivers/gpu/drm/radeon/evergreen_hdmi.c2
-rw-r--r--drivers/gpu/drm/radeon/kv_dpm.c10
-rw-r--r--drivers/gpu/drm/radeon/kv_smc.c1
-rw-r--r--drivers/gpu/drm/radeon/ni.c17
-rw-r--r--drivers/gpu/drm/radeon/ni_dma.c2
-rw-r--r--drivers/gpu/drm/radeon/ni_dpm.c16
-rw-r--r--drivers/gpu/drm/radeon/r100.c36
-rw-r--r--drivers/gpu/drm/radeon/r100_track.h2
-rw-r--r--drivers/gpu/drm/radeon/r200.c2
-rw-r--r--drivers/gpu/drm/radeon/r300.c18
-rw-r--r--drivers/gpu/drm/radeon/r420.c16
-rw-r--r--drivers/gpu/drm/radeon/r520.c4
-rw-r--r--drivers/gpu/drm/radeon/r600.c18
-rw-r--r--drivers/gpu/drm/radeon/r600_cs.c2
-rw-r--r--drivers/gpu/drm/radeon/r600_dma.c6
-rw-r--r--drivers/gpu/drm/radeon/r600_dpm.c1
-rw-r--r--drivers/gpu/drm/radeon/r600_dpm.h2
-rw-r--r--drivers/gpu/drm/radeon/r600_hdmi.c2
-rw-r--r--drivers/gpu/drm/radeon/radeon_acpi.c13
-rw-r--r--drivers/gpu/drm/radeon/radeon_agp.c8
-rw-r--r--drivers/gpu/drm/radeon/radeon_asic.c10
-rw-r--r--drivers/gpu/drm/radeon/radeon_atombios.c5
-rw-r--r--drivers/gpu/drm/radeon/radeon_audio.c2
-rw-r--r--drivers/gpu/drm/radeon/radeon_benchmark.c2
-rw-r--r--drivers/gpu/drm/radeon/radeon_bios.c12
-rw-r--r--drivers/gpu/drm/radeon/radeon_clocks.c9
-rw-r--r--drivers/gpu/drm/radeon/radeon_combios.c5
-rw-r--r--drivers/gpu/drm/radeon/radeon_connectors.c2
-rw-r--r--drivers/gpu/drm/radeon/radeon_cs.c10
-rw-r--r--drivers/gpu/drm/radeon/radeon_cursor.c4
-rw-r--r--drivers/gpu/drm/radeon/radeon_device.c18
-rw-r--r--drivers/gpu/drm/radeon/radeon_display.c21
-rw-r--r--drivers/gpu/drm/radeon/radeon_dp_auxch.c2
-rw-r--r--drivers/gpu/drm/radeon/radeon_dp_mst.c5
-rw-r--r--drivers/gpu/drm/radeon/radeon_drv.c19
-rw-r--r--drivers/gpu/drm/radeon/radeon_encoders.c5
-rw-r--r--drivers/gpu/drm/radeon/radeon_fb.c17
-rw-r--r--drivers/gpu/drm/radeon/radeon_fence.c16
-rw-r--r--drivers/gpu/drm/radeon/radeon_gart.c5
-rw-r--r--drivers/gpu/drm/radeon/radeon_gem.c9
-rw-r--r--drivers/gpu/drm/radeon/radeon_i2c.c5
-rw-r--r--drivers/gpu/drm/radeon/radeon_ib.c5
-rw-r--r--drivers/gpu/drm/radeon/radeon_irq_kms.c14
-rw-r--r--drivers/gpu/drm/radeon/radeon_kms.c17
-rw-r--r--drivers/gpu/drm/radeon/radeon_legacy_crtc.c9
-rw-r--r--drivers/gpu/drm/radeon/radeon_legacy_encoders.c11
-rw-r--r--drivers/gpu/drm/radeon/radeon_legacy_tv.c4
-rw-r--r--drivers/gpu/drm/radeon/radeon_mn.c2
-rw-r--r--drivers/gpu/drm/radeon/radeon_object.c11
-rw-r--r--drivers/gpu/drm/radeon/radeon_pm.c17
-rw-r--r--drivers/gpu/drm/radeon/radeon_prime.c8
-rw-r--r--drivers/gpu/drm/radeon/radeon_ring.c6
-rw-r--r--drivers/gpu/drm/radeon/radeon_sa.c2
-rw-r--r--drivers/gpu/drm/radeon/radeon_semaphore.c2
-rw-r--r--drivers/gpu/drm/radeon/radeon_sync.c1
-rw-r--r--drivers/gpu/drm/radeon/radeon_test.c2
-rw-r--r--drivers/gpu/drm/radeon/radeon_trace.h4
-rw-r--r--drivers/gpu/drm/radeon/radeon_trace_points.c2
-rw-r--r--drivers/gpu/drm/radeon/radeon_ttm.c47
-rw-r--r--drivers/gpu/drm/radeon/radeon_ucode.c2
-rw-r--r--drivers/gpu/drm/radeon/radeon_uvd.c2
-rw-r--r--drivers/gpu/drm/radeon/radeon_vce.c4
-rw-r--r--drivers/gpu/drm/radeon/radeon_vm.c2
-rw-r--r--drivers/gpu/drm/radeon/rs400.c11
-rw-r--r--drivers/gpu/drm/radeon/rs600.c13
-rw-r--r--drivers/gpu/drm/radeon/rs690.c6
-rw-r--r--drivers/gpu/drm/radeon/rs780_dpm.c12
-rw-r--r--drivers/gpu/drm/radeon/rv515.c13
-rw-r--r--drivers/gpu/drm/radeon/rv6xx_dpm.c1
-rw-r--r--drivers/gpu/drm/radeon/rv730_dpm.c1
-rw-r--r--drivers/gpu/drm/radeon/rv740_dpm.c1
-rw-r--r--drivers/gpu/drm/radeon/rv770.c12
-rw-r--r--drivers/gpu/drm/radeon/rv770_dma.c2
-rw-r--r--drivers/gpu/drm/radeon/rv770_dpm.c1
-rw-r--r--drivers/gpu/drm/radeon/rv770_dpm.h1
-rw-r--r--drivers/gpu/drm/radeon/rv770_smc.c2
-rw-r--r--drivers/gpu/drm/radeon/si.c16
-rw-r--r--drivers/gpu/drm/radeon/si_dma.c2
-rw-r--r--drivers/gpu/drm/radeon/si_dpm.c14
-rw-r--r--drivers/gpu/drm/radeon/si_smc.c2
-rw-r--r--drivers/gpu/drm/radeon/sumo_dpm.c1
-rw-r--r--drivers/gpu/drm/radeon/sumo_dpm.h1
-rw-r--r--drivers/gpu/drm/radeon/sumo_smc.c1
-rw-r--r--drivers/gpu/drm/radeon/trinity_dpm.c10
-rw-r--r--drivers/gpu/drm/radeon/trinity_smc.c1
-rw-r--r--drivers/gpu/drm/radeon/uvd_v1_0.c4
-rw-r--r--drivers/gpu/drm/radeon/uvd_v2_2.c2
-rw-r--r--drivers/gpu/drm/radeon/uvd_v3_1.c1
-rw-r--r--drivers/gpu/drm/radeon/uvd_v4_2.c2
-rw-r--r--drivers/gpu/drm/radeon/vce_v1_0.c2
-rw-r--r--drivers/gpu/drm/radeon/vce_v2_0.c2
-rw-r--r--drivers/gpu/drm/rcar-du/rcar_du_drv.c30
-rw-r--r--drivers/gpu/drm/rcar-du/rcar_du_encoder.c12
-rw-r--r--drivers/gpu/drm/rcar-du/rcar_du_kms.c82
-rw-r--r--drivers/gpu/drm/rcar-du/rcar_du_writeback.c1
-rw-r--r--drivers/gpu/drm/rcar-du/rcar_lvds.c133
-rw-r--r--drivers/gpu/drm/rcar-du/rcar_lvds.h5
-rw-r--r--drivers/gpu/drm/rockchip/analogix_dp-rockchip.c2
-rw-r--r--drivers/gpu/drm/rockchip/cdn-dp-reg.c4
-rw-r--r--drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c67
-rw-r--r--drivers/gpu/drm/rockchip/rockchip_drm_drv.c3
-rw-r--r--drivers/gpu/drm/rockchip/rockchip_drm_fb.c30
-rw-r--r--drivers/gpu/drm/rockchip/rockchip_drm_vop.c42
-rw-r--r--drivers/gpu/drm/savage/savage_bci.c25
-rw-r--r--drivers/gpu/drm/savage/savage_drv.c9
-rw-r--r--drivers/gpu/drm/savage/savage_drv.h10
-rw-r--r--drivers/gpu/drm/savage/savage_state.c9
-rw-r--r--drivers/gpu/drm/scheduler/sched_entity.c4
-rw-r--r--drivers/gpu/drm/scheduler/sched_main.c179
-rw-r--r--drivers/gpu/drm/selftests/Makefile2
-rw-r--r--drivers/gpu/drm/selftests/drm_cmdline_selftests.h62
-rw-r--r--drivers/gpu/drm/selftests/test-drm_cmdline_parser.c1008
-rw-r--r--drivers/gpu/drm/sis/sis_drv.c8
-rw-r--r--drivers/gpu/drm/sis/sis_drv.h10
-rw-r--r--drivers/gpu/drm/sis/sis_mm.c7
-rw-r--r--drivers/gpu/drm/sti/sti_awg_utils.c2
-rw-r--r--drivers/gpu/drm/sti/sti_awg_utils.h2
-rw-r--r--drivers/gpu/drm/sti/sti_compositor.c5
-rw-r--r--drivers/gpu/drm/sti/sti_crtc.c4
-rw-r--r--drivers/gpu/drm/sti/sti_crtc.h6
-rw-r--r--drivers/gpu/drm/sti/sti_cursor.c2
-rw-r--r--drivers/gpu/drm/sti/sti_cursor.h3
-rw-r--r--drivers/gpu/drm/sti/sti_drv.c21
-rw-r--r--drivers/gpu/drm/sti/sti_drv.h5
-rw-r--r--drivers/gpu/drm/sti/sti_dvo.c3
-rw-r--r--drivers/gpu/drm/sti/sti_gdp.c4
-rw-r--r--drivers/gpu/drm/sti/sti_gdp.h5
-rw-r--r--drivers/gpu/drm/sti/sti_hda.c6
-rw-r--r--drivers/gpu/drm/sti/sti_hdmi.c5
-rw-r--r--drivers/gpu/drm/sti/sti_hdmi.h4
-rw-r--r--drivers/gpu/drm/sti/sti_hdmi_tx3g4c28phy.c2
-rw-r--r--drivers/gpu/drm/sti/sti_hqvdp.c8
-rw-r--r--drivers/gpu/drm/sti/sti_mixer.c4
-rw-r--r--drivers/gpu/drm/sti/sti_mixer.h7
-rw-r--r--drivers/gpu/drm/sti/sti_plane.c4
-rw-r--r--drivers/gpu/drm/sti/sti_plane.h1
-rw-r--r--drivers/gpu/drm/sti/sti_tvout.c6
-rw-r--r--drivers/gpu/drm/sti/sti_vid.c4
-rw-r--r--drivers/gpu/drm/sti/sti_vtg.c4
-rw-r--r--drivers/gpu/drm/sti/sti_vtg.h1
-rw-r--r--drivers/gpu/drm/stm/drv.c44
-rw-r--r--drivers/gpu/drm/stm/dw_mipi_dsi-stm.c105
-rw-r--r--drivers/gpu/drm/stm/ltdc.c142
-rw-r--r--drivers/gpu/drm/sun4i/sun4i_drv.c16
-rw-r--r--drivers/gpu/drm/sun4i/sun4i_tcon.c1
-rw-r--r--drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c2
-rw-r--r--drivers/gpu/drm/tegra/dc.c21
-rw-r--r--drivers/gpu/drm/tegra/dpaux.c13
-rw-r--r--drivers/gpu/drm/tegra/drm.h3
-rw-r--r--drivers/gpu/drm/tegra/fb.c14
-rw-r--r--drivers/gpu/drm/tegra/output.c54
-rw-r--r--drivers/gpu/drm/ttm/ttm_bo.c270
-rw-r--r--drivers/gpu/drm/ttm/ttm_bo_util.c4
-rw-r--r--drivers/gpu/drm/ttm/ttm_execbuf_util.c20
-rw-r--r--drivers/gpu/drm/ttm/ttm_page_alloc_dma.c6
-rw-r--r--drivers/gpu/drm/v3d/v3d_debugfs.c35
-rw-r--r--drivers/gpu/drm/v3d/v3d_drv.c17
-rw-r--r--drivers/gpu/drm/v3d/v3d_drv.h106
-rw-r--r--drivers/gpu/drm/v3d/v3d_fence.c2
-rw-r--r--drivers/gpu/drm/v3d/v3d_gem.c552
-rw-r--r--drivers/gpu/drm/v3d/v3d_irq.c55
-rw-r--r--drivers/gpu/drm/v3d/v3d_mmu.c7
-rw-r--r--drivers/gpu/drm/v3d/v3d_regs.h122
-rw-r--r--drivers/gpu/drm/v3d/v3d_sched.c382
-rw-r--r--drivers/gpu/drm/v3d/v3d_trace.h94
-rw-r--r--drivers/gpu/drm/vboxvideo/Kconfig2
-rw-r--r--drivers/gpu/drm/vboxvideo/vbox_drv.c12
-rw-r--r--drivers/gpu/drm/vboxvideo/vbox_drv.h75
-rw-r--r--drivers/gpu/drm/vboxvideo/vbox_fb.c22
-rw-r--r--drivers/gpu/drm/vboxvideo/vbox_main.c75
-rw-r--r--drivers/gpu/drm/vboxvideo/vbox_mode.c36
-rw-r--r--drivers/gpu/drm/vboxvideo/vbox_ttm.c355
-rw-r--r--drivers/gpu/drm/vc4/vc4_bo.c31
-rw-r--r--drivers/gpu/drm/vc4/vc4_debugfs.c8
-rw-r--r--drivers/gpu/drm/vc4/vc4_drv.c9
-rw-r--r--drivers/gpu/drm/vc4/vc4_drv.h14
-rw-r--r--drivers/gpu/drm/vc4/vc4_gem.c11
-rw-r--r--drivers/gpu/drm/vc4/vc4_hdmi.c27
-rw-r--r--drivers/gpu/drm/vc4/vc4_irq.c20
-rw-r--r--drivers/gpu/drm/vc4/vc4_plane.c15
-rw-r--r--drivers/gpu/drm/vc4/vc4_txp.c7
-rw-r--r--drivers/gpu/drm/vc4/vc4_v3d.c72
-rw-r--r--drivers/gpu/drm/virtio/Makefile4
-rw-r--r--drivers/gpu/drm/virtio/virtgpu_display.c20
-rw-r--r--drivers/gpu/drm/virtio/virtgpu_drv.h10
-rw-r--r--drivers/gpu/drm/virtio/virtgpu_fb.c150
-rw-r--r--drivers/gpu/drm/virtio/virtgpu_fence.c25
-rw-r--r--drivers/gpu/drm/virtio/virtgpu_ioctl.c38
-rw-r--r--drivers/gpu/drm/virtio/virtgpu_object.c10
-rw-r--r--drivers/gpu/drm/virtio/virtgpu_trace.h52
-rw-r--r--drivers/gpu/drm/virtio/virtgpu_trace_points.c5
-rw-r--r--drivers/gpu/drm/virtio/virtgpu_vq.c36
-rw-r--r--drivers/gpu/drm/vkms/vkms_crc.c9
-rw-r--r--drivers/gpu/drm/vkms/vkms_crtc.c56
-rw-r--r--drivers/gpu/drm/vkms/vkms_drv.h10
-rw-r--r--drivers/gpu/drm/vkms/vkms_output.c10
-rw-r--r--drivers/gpu/drm/vkms/vkms_plane.c8
-rw-r--r--drivers/gpu/drm/vmwgfx/vmwgfx_msg.c27
-rw-r--r--drivers/gpu/drm/vmwgfx/vmwgfx_msg.h35
-rw-r--r--drivers/gpu/drm/vmwgfx/vmwgfx_resource.c3
-rw-r--r--drivers/gpu/drm/vmwgfx/vmwgfx_validation.h2
-rw-r--r--drivers/gpu/drm/zte/zx_plane.c6
-rw-r--r--drivers/gpu/host1x/bus.c35
-rw-r--r--drivers/gpu/host1x/debug.c3
-rw-r--r--drivers/gpu/host1x/dev.c5
-rw-r--r--drivers/gpu/ipu-v3/Makefile4
-rw-r--r--drivers/gpu/ipu-v3/ipu-ic-csc.c409
-rw-r--r--drivers/gpu/ipu-v3/ipu-ic.c138
-rw-r--r--drivers/gpu/ipu-v3/ipu-image-convert.c37
-rw-r--r--drivers/gpu/vga/Kconfig1
-rw-r--r--drivers/gpu/vga/vga_switcheroo.c45
-rw-r--r--drivers/greybus/Kconfig32
-rw-r--r--drivers/greybus/Makefile26
-rw-r--r--drivers/greybus/arpc.h63
-rw-r--r--drivers/greybus/bundle.c (renamed from drivers/staging/greybus/bundle.c)2
-rw-r--r--drivers/greybus/connection.c (renamed from drivers/staging/greybus/connection.c)2
-rw-r--r--drivers/greybus/control.c (renamed from drivers/staging/greybus/control.c)2
-rw-r--r--drivers/greybus/core.c (renamed from drivers/staging/greybus/core.c)2
-rw-r--r--drivers/greybus/debugfs.c (renamed from drivers/staging/greybus/debugfs.c)3
-rw-r--r--drivers/greybus/es2.c (renamed from drivers/staging/greybus/es2.c)3
-rw-r--r--drivers/greybus/greybus_trace.h (renamed from drivers/staging/greybus/greybus_trace.h)2
-rw-r--r--drivers/greybus/hd.c (renamed from drivers/staging/greybus/hd.c)12
-rw-r--r--drivers/greybus/interface.c (renamed from drivers/staging/greybus/interface.c)2
-rw-r--r--drivers/greybus/manifest.c (renamed from drivers/staging/greybus/manifest.c)41
-rw-r--r--drivers/greybus/module.c (renamed from drivers/staging/greybus/module.c)2
-rw-r--r--drivers/greybus/operation.c (renamed from drivers/staging/greybus/operation.c)2
-rw-r--r--drivers/greybus/svc.c (renamed from drivers/staging/greybus/svc.c)3
-rw-r--r--drivers/greybus/svc_watchdog.c (renamed from drivers/staging/greybus/svc_watchdog.c)2
-rw-r--r--drivers/hid/hid-a4tech.c30
-rw-r--r--drivers/hid/hid-asus.c2
-rw-r--r--drivers/hid/hid-cp2112.c15
-rw-r--r--drivers/hid/hid-holtek-kbd.c9
-rw-r--r--drivers/hid/hid-ids.h7
-rw-r--r--drivers/hid/hid-lg.c2
-rw-r--r--drivers/hid/hid-logitech-dj.c27
-rw-r--r--drivers/hid/hid-logitech-hidpp.c16
-rw-r--r--drivers/hid/hid-microsoft.c2
-rw-r--r--drivers/hid/hid-picolcd_fb.c4
-rw-r--r--drivers/hid/hid-quirks.c25
-rw-r--r--drivers/hid/hid-sensor-custom.c2
-rw-r--r--drivers/hid/hid-sony.c15
-rw-r--r--drivers/hid/hid-tmff.c12
-rw-r--r--drivers/hid/hid-uclogic-core.c2
-rw-r--r--drivers/hid/hid-uclogic-params.c2
-rw-r--r--drivers/hid/intel-ish-hid/ipc/hw-ish.h1
-rw-r--r--drivers/hid/intel-ish-hid/ipc/pci-ish.c2
-rw-r--r--drivers/hid/usbhid/hiddev.c12
-rw-r--r--drivers/hid/wacom_sys.c13
-rw-r--r--drivers/hid/wacom_wac.c171
-rw-r--r--drivers/hid/wacom_wac.h3
-rw-r--r--drivers/hsi/clients/ssi_protocol.c3
-rw-r--r--drivers/hv/Kconfig5
-rw-r--r--drivers/hv/channel.c2
-rw-r--r--drivers/hv/hv.c156
-rw-r--r--drivers/hv/hv_trace.h2
-rw-r--r--drivers/hv/hv_util.c1
-rw-r--r--drivers/hv/hyperv_vmbus.h13
-rw-r--r--drivers/hv/vmbus_drv.c49
-rw-r--r--drivers/hwmon/Kconfig31
-rw-r--r--drivers/hwmon/Makefile2
-rw-r--r--drivers/hwmon/acpi_power_meter.c4
-rw-r--r--drivers/hwmon/adm1029.c10
-rw-r--r--drivers/hwmon/ads1015.c324
-rw-r--r--drivers/hwmon/adt7475.c146
-rw-r--r--drivers/hwmon/as370-hwmon.c145
-rw-r--r--drivers/hwmon/asb100.c12
-rw-r--r--drivers/hwmon/asus_atk0110.c23
-rw-r--r--drivers/hwmon/atxp1.c2
-rw-r--r--drivers/hwmon/coretemp.c39
-rw-r--r--drivers/hwmon/gpio-fan.c22
-rw-r--r--drivers/hwmon/hwmon.c6
-rw-r--r--drivers/hwmon/iio_hwmon.c18
-rw-r--r--drivers/hwmon/ina3221.c4
-rw-r--r--drivers/hwmon/k10temp.c1
-rw-r--r--drivers/hwmon/k8temp.c233
-rw-r--r--drivers/hwmon/lm75.c462
-rw-r--r--drivers/hwmon/lm90.c106
-rw-r--r--drivers/hwmon/ltc2990.c10
-rw-r--r--drivers/hwmon/max6650.c710
-rw-r--r--drivers/hwmon/nct6775.c183
-rw-r--r--drivers/hwmon/nct7802.c6
-rw-r--r--drivers/hwmon/nct7904.c555
-rw-r--r--drivers/hwmon/npcm750-pwm-fan.c4
-rw-r--r--drivers/hwmon/occ/common.c16
-rw-r--r--drivers/hwmon/occ/common.h1
-rw-r--r--drivers/hwmon/pmbus/Kconfig27
-rw-r--r--drivers/hwmon/pmbus/Makefile3
-rw-r--r--drivers/hwmon/pmbus/adm1275.c105
-rw-r--r--drivers/hwmon/pmbus/ibm-cffps.c110
-rw-r--r--drivers/hwmon/pmbus/inspur-ipsps.c228
-rw-r--r--drivers/hwmon/pmbus/irps5401.c67
-rw-r--r--drivers/hwmon/pmbus/max31785.c2
-rw-r--r--drivers/hwmon/pmbus/pxe1610.c139
-rw-r--r--drivers/hwmon/pmbus/ucd9000.c1
-rw-r--r--drivers/hwmon/pwm-fan.c12
-rw-r--r--drivers/hwmon/raspberrypi-hwmon.c2
-rw-r--r--drivers/hwmon/scmi-hwmon.c50
-rw-r--r--drivers/hwmon/scpi-hwmon.c10
-rw-r--r--drivers/hwmon/shtc1.c57
-rw-r--r--drivers/hwmon/smm665.c8
-rw-r--r--drivers/hwmon/smsc47m1.c2
-rw-r--r--drivers/hwmon/w83781d.c6
-rw-r--r--drivers/hwmon/w83791d.c32
-rw-r--r--drivers/hwmon/w83792d.c32
-rw-r--r--drivers/hwmon/w83793.c30
-rw-r--r--drivers/hwspinlock/Kconfig2
-rw-r--r--drivers/hwspinlock/hwspinlock_core.c48
-rw-r--r--drivers/hwspinlock/omap_hwspinlock.c4
-rw-r--r--drivers/hwspinlock/stm32_hwspinlock.c7
-rw-r--r--drivers/hwtracing/coresight/Kconfig3
-rw-r--r--drivers/hwtracing/coresight/Makefile3
-rw-r--r--drivers/hwtracing/coresight/coresight-catu.c40
-rw-r--r--drivers/hwtracing/coresight/coresight-catu.h1
-rw-r--r--drivers/hwtracing/coresight/coresight-cpu-debug.c54
-rw-r--r--drivers/hwtracing/coresight/coresight-etb10.c78
-rw-r--r--drivers/hwtracing/coresight/coresight-etm-perf.c9
-rw-r--r--drivers/hwtracing/coresight/coresight-etm.h6
-rw-r--r--drivers/hwtracing/coresight/coresight-etm3x-sysfs.c12
-rw-r--r--drivers/hwtracing/coresight/coresight-etm3x.c49
-rw-r--r--drivers/hwtracing/coresight/coresight-etm4x-sysfs.c13
-rw-r--r--drivers/hwtracing/coresight/coresight-etm4x.c78
-rw-r--r--drivers/hwtracing/coresight/coresight-etm4x.h2
-rw-r--r--drivers/hwtracing/coresight/coresight-funnel.c45
-rw-r--r--drivers/hwtracing/coresight/coresight-platform.c808
-rw-r--r--drivers/hwtracing/coresight/coresight-priv.h12
-rw-r--r--drivers/hwtracing/coresight/coresight-replicator.c44
-rw-r--r--drivers/hwtracing/coresight/coresight-stm.c118
-rw-r--r--drivers/hwtracing/coresight/coresight-tmc-etf.c66
-rw-r--r--drivers/hwtracing/coresight/coresight-tmc-etr.c171
-rw-r--r--drivers/hwtracing/coresight/coresight-tmc.c136
-rw-r--r--drivers/hwtracing/coresight/coresight-tmc.h13
-rw-r--r--drivers/hwtracing/coresight/coresight-tpiu.c24
-rw-r--r--drivers/hwtracing/coresight/coresight.c168
-rw-r--r--drivers/hwtracing/coresight/of_coresight.c297
-rw-r--r--drivers/hwtracing/intel_th/Makefile3
-rw-r--r--drivers/hwtracing/intel_th/core.c11
-rw-r--r--drivers/hwtracing/intel_th/msu-sink.c116
-rw-r--r--drivers/hwtracing/intel_th/msu.c647
-rw-r--r--drivers/hwtracing/intel_th/msu.h22
-rw-r--r--drivers/hwtracing/intel_th/pci.c15
-rw-r--r--drivers/hwtracing/intel_th/pti.h2
-rw-r--r--drivers/hwtracing/stm/core.c10
-rw-r--r--drivers/i2c/Kconfig4
-rw-r--r--drivers/i2c/busses/Kconfig17
-rw-r--r--drivers/i2c/busses/Makefile1
-rw-r--r--drivers/i2c/busses/i2c-amd-mp2-pci.c8
-rw-r--r--drivers/i2c/busses/i2c-at91-core.c2
-rw-r--r--drivers/i2c/busses/i2c-at91-master.c9
-rw-r--r--drivers/i2c/busses/i2c-bcm-iproc.c132
-rw-r--r--drivers/i2c/busses/i2c-bcm2835.c135
-rw-r--r--drivers/i2c/busses/i2c-cpm.c4
-rw-r--r--drivers/i2c/busses/i2c-designware-slave.c1
-rw-r--r--drivers/i2c/busses/i2c-emev2.c16
-rw-r--r--drivers/i2c/busses/i2c-fsi.c32
-rw-r--r--drivers/i2c/busses/i2c-i801.c217
-rw-r--r--drivers/i2c/busses/i2c-imx.c29
-rw-r--r--drivers/i2c/busses/i2c-iop3xx.c32
-rw-r--r--drivers/i2c/busses/i2c-iop3xx.h2
-rw-r--r--drivers/i2c/busses/i2c-mt65xx.c11
-rw-r--r--drivers/i2c/busses/i2c-mt7621.c356
-rw-r--r--drivers/i2c/busses/i2c-nvidia-gpu.c64
-rw-r--r--drivers/i2c/busses/i2c-ocores.c33
-rw-r--r--drivers/i2c/busses/i2c-piix4.c12
-rw-r--r--drivers/i2c/busses/i2c-qcom-geni.c17
-rw-r--r--drivers/i2c/busses/i2c-rcar.c11
-rw-r--r--drivers/i2c/busses/i2c-s3c2410.c48
-rw-r--r--drivers/i2c/busses/i2c-stm32.h2
-rw-r--r--drivers/i2c/busses/i2c-stm32f7.c32
-rw-r--r--drivers/i2c/busses/i2c-taos-evm.c2
-rw-r--r--drivers/i2c/busses/i2c-tegra.c136
-rw-r--r--drivers/i2c/i2c-core-acpi.c78
-rw-r--r--drivers/i2c/i2c-core-base.c26
-rw-r--r--drivers/i2c/i2c-core-of.c10
-rw-r--r--drivers/i2c/i2c-core.h9
-rw-r--r--drivers/i2c/i2c-mux.c4
-rw-r--r--drivers/i2c/muxes/i2c-arb-gpio-challenge.c79
-rw-r--r--drivers/i2c/muxes/i2c-mux-gpio.c116
-rw-r--r--drivers/i2c/muxes/i2c-mux-pinctrl.c5
-rw-r--r--drivers/i3c/device.c53
-rw-r--r--drivers/i3c/master.c149
-rw-r--r--drivers/i3c/master/dw-i3c-master.c11
-rw-r--r--drivers/i3c/master/i3c-master-cdns.c38
-rw-r--r--drivers/ide/Kconfig29
-rw-r--r--drivers/ide/Makefile1
-rw-r--r--drivers/ide/ide-cd.c2
-rw-r--r--drivers/ide/sgiioc4.c630
-rw-r--r--drivers/idle/intel_idle.c28
-rw-r--r--drivers/iio/Kconfig2
-rw-r--r--drivers/iio/accel/Kconfig4
-rw-r--r--drivers/iio/accel/adis16201.c4
-rw-r--r--drivers/iio/accel/adis16209.c4
-rw-r--r--drivers/iio/accel/adxl372.c27
-rw-r--r--drivers/iio/accel/adxl372_spi.c9
-rw-r--r--drivers/iio/accel/cros_ec_accel_legacy.c353
-rw-r--r--drivers/iio/accel/kxcjk-1013.c3
-rw-r--r--drivers/iio/accel/kxsd9-spi.c9
-rw-r--r--drivers/iio/accel/mxc4005.c40
-rw-r--r--drivers/iio/accel/sca3000.c9
-rw-r--r--drivers/iio/accel/st_accel.h1
-rw-r--r--drivers/iio/accel/st_accel_buffer.c48
-rw-r--r--drivers/iio/accel/st_accel_core.c32
-rw-r--r--drivers/iio/accel/st_accel_i2c.c23
-rw-r--r--drivers/iio/accel/st_accel_spi.c20
-rw-r--r--drivers/iio/adc/Kconfig3
-rw-r--r--drivers/iio/adc/ad7124.c33
-rw-r--r--drivers/iio/adc/ad7606.c157
-rw-r--r--drivers/iio/adc/ad7606.h60
-rw-r--r--drivers/iio/adc/ad7606_par.c4
-rw-r--r--drivers/iio/adc/ad7606_spi.c282
-rw-r--r--drivers/iio/adc/ad_sigma_delta.c3
-rw-r--r--drivers/iio/adc/at91-sama5d2_adc.c12
-rw-r--r--drivers/iio/adc/at91_adc.c8
-rw-r--r--drivers/iio/adc/axp288_adc.c4
-rw-r--r--drivers/iio/adc/bcm_iproc_adc.c7
-rw-r--r--drivers/iio/adc/da9150-gpadc.c4
-rw-r--r--drivers/iio/adc/envelope-detector.c5
-rw-r--r--drivers/iio/adc/exynos_adc.c4
-rw-r--r--drivers/iio/adc/fsl-imx25-gcq.c1
-rw-r--r--drivers/iio/adc/hi8435.c34
-rw-r--r--drivers/iio/adc/imx7d_adc.c28
-rw-r--r--drivers/iio/adc/ingenic-adc.c54
-rw-r--r--drivers/iio/adc/lpc32xx_adc.c4
-rw-r--r--drivers/iio/adc/max1027.c38
-rw-r--r--drivers/iio/adc/max9611.c4
-rw-r--r--drivers/iio/adc/meson_saradc.c2
-rw-r--r--drivers/iio/adc/mt6577_auxadc.c54
-rw-r--r--drivers/iio/adc/npcm_adc.c1
-rw-r--r--drivers/iio/adc/rcar-gyroadc.c8
-rw-r--r--drivers/iio/adc/rockchip_saradc.c4
-rw-r--r--drivers/iio/adc/sc27xx_adc.c122
-rw-r--r--drivers/iio/adc/spear_adc.c1
-rw-r--r--drivers/iio/adc/stm32-adc-core.c213
-rw-r--r--drivers/iio/adc/stm32-adc.c4
-rw-r--r--drivers/iio/adc/stm32-dfsdm-adc.c236
-rw-r--r--drivers/iio/adc/stm32-dfsdm-core.c8
-rw-r--r--drivers/iio/adc/stm32-dfsdm.h24
-rw-r--r--drivers/iio/adc/stmpe-adc.c40
-rw-r--r--drivers/iio/adc/sun4i-gpadc-iio.c6
-rw-r--r--drivers/iio/adc/twl6030-gpadc.c4
-rw-r--r--drivers/iio/adc/vf610_adc.c4
-rw-r--r--drivers/iio/amplifiers/Kconfig13
-rw-r--r--drivers/iio/amplifiers/ad8366.c146
-rw-r--r--drivers/iio/common/cros_ec_sensors/Kconfig9
-rw-r--r--drivers/iio/common/cros_ec_sensors/Makefile1
-rw-r--r--drivers/iio/common/cros_ec_sensors/cros_ec_lid_angle.c139
-rw-r--r--drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c54
-rw-r--r--drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c167
-rw-r--r--drivers/iio/common/hid-sensors/hid-sensor-attributes.c69
-rw-r--r--drivers/iio/common/st_sensors/Kconfig2
-rw-r--r--drivers/iio/common/st_sensors/st_sensors_buffer.c10
-rw-r--r--drivers/iio/common/st_sensors/st_sensors_core.c118
-rw-r--r--drivers/iio/common/st_sensors/st_sensors_i2c.c82
-rw-r--r--drivers/iio/common/st_sensors/st_sensors_spi.c148
-rw-r--r--drivers/iio/common/st_sensors/st_sensors_trigger.c31
-rw-r--r--drivers/iio/dac/ad5380.c2
-rw-r--r--drivers/iio/dac/ad5758.c18
-rw-r--r--drivers/iio/dac/ds4424.c6
-rw-r--r--drivers/iio/dummy/iio_simple_dummy.c4
-rw-r--r--drivers/iio/frequency/Kconfig10
-rw-r--r--drivers/iio/frequency/Makefile1
-rw-r--r--drivers/iio/frequency/ad9523.c8
-rw-r--r--drivers/iio/frequency/adf4371.c632
-rw-r--r--drivers/iio/gyro/st_gyro.h1
-rw-r--r--drivers/iio/gyro/st_gyro_buffer.c48
-rw-r--r--drivers/iio/gyro/st_gyro_core.c32
-rw-r--r--drivers/iio/gyro/st_gyro_i2c.c22
-rw-r--r--drivers/iio/gyro/st_gyro_spi.c20
-rw-r--r--drivers/iio/humidity/am2315.c24
-rw-r--r--drivers/iio/humidity/dht11.c36
-rw-r--r--drivers/iio/humidity/hdc100x.c19
-rw-r--r--drivers/iio/imu/Kconfig12
-rw-r--r--drivers/iio/imu/Makefile1
-rw-r--r--drivers/iio/imu/adis.c12
-rw-r--r--drivers/iio/imu/adis16460.c489
-rw-r--r--drivers/iio/imu/inv_mpu6050/Kconfig10
-rw-r--r--drivers/iio/imu/inv_mpu6050/inv_mpu_core.c46
-rw-r--r--drivers/iio/imu/st_lsm6dsx/Kconfig11
-rw-r--r--drivers/iio/imu/st_lsm6dsx/Makefile1
-rw-r--r--drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h38
-rw-r--r--drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c36
-rw-r--r--drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c675
-rw-r--r--drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c18
-rw-r--r--drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i3c.c57
-rw-r--r--drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c18
-rw-r--r--drivers/iio/industrialio-core.c41
-rw-r--r--drivers/iio/inkern.c2
-rw-r--r--drivers/iio/light/Kconfig11
-rw-r--r--drivers/iio/light/Makefile1
-rw-r--r--drivers/iio/light/apds9960.c2
-rw-r--r--drivers/iio/light/bh1780.c2
-rw-r--r--drivers/iio/light/cm3323.c33
-rw-r--r--drivers/iio/light/cm36651.c12
-rw-r--r--drivers/iio/light/cros_ec_light_prox.c51
-rw-r--r--drivers/iio/light/noa1305.c313
-rw-r--r--drivers/iio/light/si1145.c42
-rw-r--r--drivers/iio/light/stk3310.c15
-rw-r--r--drivers/iio/light/tsl2772.c124
-rw-r--r--drivers/iio/light/veml6070.c6
-rw-r--r--drivers/iio/magnetometer/mmc35240.c4
-rw-r--r--drivers/iio/magnetometer/st_magn.h1
-rw-r--r--drivers/iio/magnetometer/st_magn_buffer.c31
-rw-r--r--drivers/iio/magnetometer/st_magn_core.c32
-rw-r--r--drivers/iio/magnetometer/st_magn_i2c.c22
-rw-r--r--drivers/iio/magnetometer/st_magn_spi.c20
-rw-r--r--drivers/iio/potentiometer/Kconfig11
-rw-r--r--drivers/iio/potentiometer/Makefile1
-rw-r--r--drivers/iio/potentiometer/max5432.c135
-rw-r--r--drivers/iio/pressure/Kconfig11
-rw-r--r--drivers/iio/pressure/Makefile1
-rw-r--r--drivers/iio/pressure/cros_ec_baro.c19
-rw-r--r--drivers/iio/pressure/dps310.c827
-rw-r--r--drivers/iio/pressure/hp03.c6
-rw-r--r--drivers/iio/pressure/st_pressure.h1
-rw-r--r--drivers/iio/pressure/st_pressure_buffer.c39
-rw-r--r--drivers/iio/pressure/st_pressure_core.c32
-rw-r--r--drivers/iio/pressure/st_pressure_i2c.c31
-rw-r--r--drivers/iio/pressure/st_pressure_spi.c22
-rw-r--r--drivers/iio/proximity/Kconfig2
-rw-r--r--drivers/iio/temperature/maxim_thermocouple.c35
-rw-r--r--drivers/iio/trigger/stm32-timer-trigger.c84
-rw-r--r--drivers/infiniband/Kconfig14
-rw-r--r--drivers/infiniband/core/Makefile5
-rw-r--r--drivers/infiniband/core/addr.c2
-rw-r--r--drivers/infiniband/core/cma.c6
-rw-r--r--drivers/infiniband/core/core_priv.h15
-rw-r--r--drivers/infiniband/core/counters.c641
-rw-r--r--drivers/infiniband/core/cq.c95
-rw-r--r--drivers/infiniband/core/device.c260
-rw-r--r--drivers/infiniband/core/mad.c20
-rw-r--r--drivers/infiniband/core/mr_pool.c8
-rw-r--r--drivers/infiniband/core/nldev.c807
-rw-r--r--drivers/infiniband/core/restrack.c62
-rw-r--r--drivers/infiniband/core/restrack.h3
-rw-r--r--drivers/infiniband/core/roce_gid_mgmt.c5
-rw-r--r--drivers/infiniband/core/rw.c201
-rw-r--r--drivers/infiniband/core/sysfs.c16
-rw-r--r--drivers/infiniband/core/ucm.c1350
-rw-r--r--drivers/infiniband/core/ucma.c114
-rw-r--r--drivers/infiniband/core/umem.c20
-rw-r--r--drivers/infiniband/core/umem_odp.c110
-rw-r--r--drivers/infiniband/core/user_mad.c59
-rw-r--r--drivers/infiniband/core/uverbs_cmd.c28
-rw-r--r--drivers/infiniband/core/uverbs_ioctl.c2
-rw-r--r--drivers/infiniband/core/uverbs_main.c40
-rw-r--r--drivers/infiniband/core/uverbs_std_types_cq.c19
-rw-r--r--drivers/infiniband/core/uverbs_std_types_mr.c1
-rw-r--r--drivers/infiniband/core/uverbs_uapi.c4
-rw-r--r--drivers/infiniband/core/verbs.c165
-rw-r--r--drivers/infiniband/hw/Makefile1
-rw-r--r--drivers/infiniband/hw/bnxt_re/ib_verbs.c73
-rw-r--r--drivers/infiniband/hw/bnxt_re/ib_verbs.h9
-rw-r--r--drivers/infiniband/hw/bnxt_re/main.c8
-rw-r--r--drivers/infiniband/hw/bnxt_re/qplib_rcfw.c8
-rw-r--r--drivers/infiniband/hw/bnxt_re/qplib_rcfw.h11
-rw-r--r--drivers/infiniband/hw/bnxt_re/qplib_res.c13
-rw-r--r--drivers/infiniband/hw/bnxt_re/qplib_res.h2
-rw-r--r--drivers/infiniband/hw/bnxt_re/qplib_sp.c14
-rw-r--r--drivers/infiniband/hw/bnxt_re/qplib_sp.h7
-rw-r--r--drivers/infiniband/hw/cxgb3/cxio_hal.c33
-rw-r--r--drivers/infiniband/hw/cxgb3/cxio_hal.h3
-rw-r--r--drivers/infiniband/hw/cxgb3/iwch_cm.c2
-rw-r--r--drivers/infiniband/hw/cxgb3/iwch_provider.c160
-rw-r--r--drivers/infiniband/hw/cxgb4/cm.c30
-rw-r--r--drivers/infiniband/hw/cxgb4/cq.c55
-rw-r--r--drivers/infiniband/hw/cxgb4/device.c9
-rw-r--r--drivers/infiniband/hw/cxgb4/iw_cxgb4.h11
-rw-r--r--drivers/infiniband/hw/cxgb4/mem.c8
-rw-r--r--drivers/infiniband/hw/cxgb4/provider.c9
-rw-r--r--drivers/infiniband/hw/cxgb4/qp.c95
-rw-r--r--drivers/infiniband/hw/cxgb4/resource.c16
-rw-r--r--drivers/infiniband/hw/efa/efa.h9
-rw-r--r--drivers/infiniband/hw/efa/efa_com.c118
-rw-r--r--drivers/infiniband/hw/efa/efa_com.h1
-rw-r--r--drivers/infiniband/hw/efa/efa_com_cmd.c8
-rw-r--r--drivers/infiniband/hw/efa/efa_main.c10
-rw-r--r--drivers/infiniband/hw/efa/efa_verbs.c248
-rw-r--r--drivers/infiniband/hw/hfi1/Makefile1
-rw-r--r--drivers/infiniband/hw/hfi1/affinity.c6
-rw-r--r--drivers/infiniband/hw/hfi1/aspm.c270
-rw-r--r--drivers/infiniband/hw/hfi1/aspm.h262
-rw-r--r--drivers/infiniband/hw/hfi1/chip.c11
-rw-r--r--drivers/infiniband/hw/hfi1/debugfs.c5
-rw-r--r--drivers/infiniband/hw/hfi1/fault.c12
-rw-r--r--drivers/infiniband/hw/hfi1/mad.c9
-rw-r--r--drivers/infiniband/hw/hfi1/pcie.c6
-rw-r--r--drivers/infiniband/hw/hfi1/pio.c3
-rw-r--r--drivers/infiniband/hw/hfi1/qp.c8
-rw-r--r--drivers/infiniband/hw/hfi1/rc.c31
-rw-r--r--drivers/infiniband/hw/hfi1/sdma.c3
-rw-r--r--drivers/infiniband/hw/hfi1/tid_rdma.c124
-rw-r--r--drivers/infiniband/hw/hfi1/trace_ibhdrs.h2
-rw-r--r--drivers/infiniband/hw/hfi1/uc.c3
-rw-r--r--drivers/infiniband/hw/hfi1/ud.c36
-rw-r--r--drivers/infiniband/hw/hfi1/user_pages.c11
-rw-r--r--drivers/infiniband/hw/hfi1/verbs.c8
-rw-r--r--drivers/infiniband/hw/hfi1/vnic_sdma.c4
-rw-r--r--drivers/infiniband/hw/hns/Kconfig13
-rw-r--r--drivers/infiniband/hw/hns/Makefile7
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_alloc.c101
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_cmd.c6
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_cq.c81
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_db.c27
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_device.h108
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_hem.c504
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_hem.h16
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_hw_v1.c91
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_hw_v2.c280
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_hw_v2.h23
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_main.c31
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_mr.c166
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_pd.c4
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_qp.c220
-rw-r--r--drivers/infiniband/hw/hns/hns_roce_srq.c40
-rw-r--r--drivers/infiniband/hw/i40iw/i40iw_cm.c11
-rw-r--r--drivers/infiniband/hw/i40iw/i40iw_main.c6
-rw-r--r--drivers/infiniband/hw/i40iw/i40iw_utils.c12
-rw-r--r--drivers/infiniband/hw/i40iw/i40iw_verbs.c56
-rw-r--r--drivers/infiniband/hw/mlx4/alias_GUID.c6
-rw-r--r--drivers/infiniband/hw/mlx4/cq.c43
-rw-r--r--drivers/infiniband/hw/mlx4/mad.c4
-rw-r--r--drivers/infiniband/hw/mlx4/main.c21
-rw-r--r--drivers/infiniband/hw/mlx4/mlx4_ib.h9
-rw-r--r--drivers/infiniband/hw/mlx4/mr.c16
-rw-r--r--drivers/infiniband/hw/mlx4/qp.c11
-rw-r--r--drivers/infiniband/hw/mlx4/srq.c9
-rw-r--r--drivers/infiniband/hw/mlx5/cmd.c130
-rw-r--r--drivers/infiniband/hw/mlx5/cmd.h4
-rw-r--r--drivers/infiniband/hw/mlx5/cq.c69
-rw-r--r--drivers/infiniband/hw/mlx5/devx.c1069
-rw-r--r--drivers/infiniband/hw/mlx5/flow.c34
-rw-r--r--drivers/infiniband/hw/mlx5/ib_rep.c39
-rw-r--r--drivers/infiniband/hw/mlx5/ib_rep.h4
-rw-r--r--drivers/infiniband/hw/mlx5/mad.c60
-rw-r--r--drivers/infiniband/hw/mlx5/main.c358
-rw-r--r--drivers/infiniband/hw/mlx5/mem.c23
-rw-r--r--drivers/infiniband/hw/mlx5/mlx5_ib.h72
-rw-r--r--drivers/infiniband/hw/mlx5/mr.c615
-rw-r--r--drivers/infiniband/hw/mlx5/odp.c104
-rw-r--r--drivers/infiniband/hw/mlx5/qp.c347
-rw-r--r--drivers/infiniband/hw/mlx5/srq_cmd.c6
-rw-r--r--drivers/infiniband/hw/mthca/mthca_allocator.c2
-rw-r--r--drivers/infiniband/hw/mthca/mthca_memfree.c6
-rw-r--r--drivers/infiniband/hw/mthca/mthca_provider.c52
-rw-r--r--drivers/infiniband/hw/nes/Kconfig16
-rw-r--r--drivers/infiniband/hw/nes/Makefile4
-rw-r--r--drivers/infiniband/hw/nes/nes.c1205
-rw-r--r--drivers/infiniband/hw/nes/nes.h574
-rw-r--r--drivers/infiniband/hw/nes/nes_cm.c3992
-rw-r--r--drivers/infiniband/hw/nes/nes_cm.h470
-rw-r--r--drivers/infiniband/hw/nes/nes_context.h193
-rw-r--r--drivers/infiniband/hw/nes/nes_hw.c3887
-rw-r--r--drivers/infiniband/hw/nes/nes_hw.h1380
-rw-r--r--drivers/infiniband/hw/nes/nes_mgt.c1155
-rw-r--r--drivers/infiniband/hw/nes/nes_mgt.h97
-rw-r--r--drivers/infiniband/hw/nes/nes_nic.c1870
-rw-r--r--drivers/infiniband/hw/nes/nes_utils.c916
-rw-r--r--drivers/infiniband/hw/nes/nes_verbs.c3759
-rw-r--r--drivers/infiniband/hw/nes/nes_verbs.h198
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_hw.c11
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_hw.h2
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_main.c8
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_verbs.c38
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_verbs.h7
-rw-r--r--drivers/infiniband/hw/qedr/main.c43
-rw-r--r--drivers/infiniband/hw/qedr/qedr.h2
-rw-r--r--drivers/infiniband/hw/qedr/verbs.c82
-rw-r--r--drivers/infiniband/hw/qedr/verbs.h7
-rw-r--r--drivers/infiniband/hw/qib/qib_file_ops.c7
-rw-r--r--drivers/infiniband/hw/qib/qib_fs.c28
-rw-r--r--drivers/infiniband/hw/qib/qib_qp.c4
-rw-r--r--drivers/infiniband/hw/qib/qib_rc.c29
-rw-r--r--drivers/infiniband/hw/qib/qib_uc.c3
-rw-r--r--drivers/infiniband/hw/qib/qib_ud.c28
-rw-r--r--drivers/infiniband/hw/qib/qib_user_pages.c11
-rw-r--r--drivers/infiniband/hw/qib/qib_user_sdma.c11
-rw-r--r--drivers/infiniband/hw/qib/qib_verbs.c6
-rw-r--r--drivers/infiniband/hw/usnic/usnic_ib.h4
-rw-r--r--drivers/infiniband/hw/usnic/usnic_ib_main.c23
-rw-r--r--drivers/infiniband/hw/usnic/usnic_ib_verbs.c22
-rw-r--r--drivers/infiniband/hw/usnic/usnic_ib_verbs.h7
-rw-r--r--drivers/infiniband/hw/usnic/usnic_uiom.c7
-rw-r--r--drivers/infiniband/hw/vmw_pvrdma/pvrdma.h2
-rw-r--r--drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c46
-rw-r--r--drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c8
-rw-r--r--drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c3
-rw-r--r--drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c16
-rw-r--r--drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h7
-rw-r--r--drivers/infiniband/sw/Makefile1
-rw-r--r--drivers/infiniband/sw/rdmavt/ah.c6
-rw-r--r--drivers/infiniband/sw/rdmavt/cq.c250
-rw-r--r--drivers/infiniband/sw/rdmavt/cq.h7
-rw-r--r--drivers/infiniband/sw/rdmavt/mr.c6
-rw-r--r--drivers/infiniband/sw/rdmavt/qp.c402
-rw-r--r--drivers/infiniband/sw/rdmavt/qp.h2
-rw-r--r--drivers/infiniband/sw/rdmavt/rc.c41
-rw-r--r--drivers/infiniband/sw/rdmavt/srq.c69
-rw-r--r--drivers/infiniband/sw/rdmavt/trace_mr.h56
-rw-r--r--drivers/infiniband/sw/rdmavt/vt.c7
-rw-r--r--drivers/infiniband/sw/rdmavt/vt.h9
-rw-r--r--drivers/infiniband/sw/rxe/rxe_comp.c2
-rw-r--r--drivers/infiniband/sw/rxe/rxe_mr.c3
-rw-r--r--drivers/infiniband/sw/rxe/rxe_pool.c1
-rw-r--r--drivers/infiniband/sw/rxe/rxe_resp.c5
-rw-r--r--drivers/infiniband/sw/rxe/rxe_verbs.c40
-rw-r--r--drivers/infiniband/sw/rxe/rxe_verbs.h3
-rw-r--r--drivers/infiniband/sw/siw/Kconfig18
-rw-r--r--drivers/infiniband/sw/siw/Makefile11
-rw-r--r--drivers/infiniband/sw/siw/iwarp.h380
-rw-r--r--drivers/infiniband/sw/siw/siw.h745
-rw-r--r--drivers/infiniband/sw/siw/siw_cm.c2074
-rw-r--r--drivers/infiniband/sw/siw/siw_cm.h133
-rw-r--r--drivers/infiniband/sw/siw/siw_cq.c102
-rw-r--r--drivers/infiniband/sw/siw/siw_main.c684
-rw-r--r--drivers/infiniband/sw/siw/siw_mem.c460
-rw-r--r--drivers/infiniband/sw/siw/siw_mem.h74
-rw-r--r--drivers/infiniband/sw/siw/siw_qp.c1330
-rw-r--r--drivers/infiniband/sw/siw/siw_qp_rx.c1460
-rw-r--r--drivers/infiniband/sw/siw/siw_qp_tx.c1263
-rw-r--r--drivers/infiniband/sw/siw/siw_verbs.c1766
-rw-r--r--drivers/infiniband/sw/siw/siw_verbs.h91
-rw-r--r--drivers/infiniband/ulp/ipoib/Kconfig2
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib_cm.c1
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib_ethtool.c3
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib_ib.c3
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib_main.c35
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib_verbs.c7
-rw-r--r--drivers/infiniband/ulp/iser/iscsi_iser.c47
-rw-r--r--drivers/infiniband/ulp/iser/iscsi_iser.h64
-rw-r--r--drivers/infiniband/ulp/iser/iser_initiator.c12
-rw-r--r--drivers/infiniband/ulp/iser/iser_memory.c121
-rw-r--r--drivers/infiniband/ulp/iser/iser_verbs.c156
-rw-r--r--drivers/infiniband/ulp/isert/ib_isert.c19
-rw-r--r--drivers/infiniband/ulp/srp/ib_srp.c60
-rw-r--r--drivers/input/joydev.c24
-rw-r--r--drivers/input/joystick/iforce/Kconfig8
-rw-r--r--drivers/input/joystick/iforce/Makefile7
-rw-r--r--drivers/input/joystick/iforce/iforce-ff.c21
-rw-r--r--drivers/input/joystick/iforce/iforce-main.c181
-rw-r--r--drivers/input/joystick/iforce/iforce-packets.c218
-rw-r--r--drivers/input/joystick/iforce/iforce-serio.c164
-rw-r--r--drivers/input/joystick/iforce/iforce-usb.c200
-rw-r--r--drivers/input/joystick/iforce/iforce.h58
-rw-r--r--drivers/input/keyboard/Kconfig18
-rw-r--r--drivers/input/keyboard/Makefile1
-rw-r--r--drivers/input/keyboard/adp5589-keys.c1
-rw-r--r--drivers/input/keyboard/applespi.c1980
-rw-r--r--drivers/input/keyboard/applespi.h29
-rw-r--r--drivers/input/keyboard/applespi_trace.h93
-rw-r--r--drivers/input/keyboard/cros_ec_keyb.c2
-rw-r--r--drivers/input/keyboard/gpio_keys.c6
-rw-r--r--drivers/input/keyboard/gpio_keys_polled.c10
-rw-r--r--drivers/input/keyboard/imx_keypad.c4
-rw-r--r--drivers/input/keyboard/mtk-pmic-keys.c9
-rw-r--r--drivers/input/keyboard/sun4i-lradc-keys.c3
-rw-r--r--drivers/input/keyboard/tca8418_keypad.c3
-rw-r--r--drivers/input/misc/da9063_onkey.c11
-rw-r--r--drivers/input/misc/max77650-onkey.c1
-rw-r--r--drivers/input/mouse/alps.c32
-rw-r--r--drivers/input/mouse/elan_i2c_core.c122
-rw-r--r--drivers/input/mouse/elantech.c377
-rw-r--r--drivers/input/mouse/elantech.h8
-rw-r--r--drivers/input/mouse/synaptics.c5
-rw-r--r--drivers/input/mouse/trackpoint.h3
-rw-r--r--drivers/input/mouse/vmmouse.c6
-rw-r--r--drivers/input/mousedev.c2
-rw-r--r--drivers/input/rmi4/rmi_f12.c6
-rw-r--r--drivers/input/serio/hyperv-keyboard.c39
-rw-r--r--drivers/input/serio/i8042.c2
-rw-r--r--drivers/input/tablet/gtco.c20
-rw-r--r--drivers/input/tablet/kbtab.c6
-rw-r--r--drivers/input/touchscreen/atmel_mxt_ts.c23
-rw-r--r--drivers/input/touchscreen/auo-pixcir-ts.c3
-rw-r--r--drivers/input/touchscreen/edt-ft5x06.c18
-rw-r--r--drivers/input/touchscreen/eeti_ts.c71
-rw-r--r--drivers/input/touchscreen/imx6ul_tsc.c8
-rw-r--r--drivers/input/touchscreen/iqs5xx.c2
-rw-r--r--drivers/input/touchscreen/sun4i-ts.c2
-rw-r--r--drivers/input/touchscreen/sur40.c10
-rw-r--r--drivers/input/touchscreen/usbtouchscreen.c2
-rw-r--r--drivers/interconnect/core.c27
-rw-r--r--drivers/interconnect/qcom/Kconfig12
-rw-r--r--drivers/interconnect/qcom/Makefile4
-rw-r--r--drivers/interconnect/qcom/qcs404.c539
-rw-r--r--drivers/interconnect/qcom/sdm845.c160
-rw-r--r--drivers/interconnect/qcom/smd-rpm.c77
-rw-r--r--drivers/interconnect/qcom/smd-rpm.h15
-rw-r--r--drivers/iommu/Kconfig14
-rw-r--r--drivers/iommu/Makefile6
-rw-r--r--drivers/iommu/amd_iommu.c174
-rw-r--r--drivers/iommu/amd_iommu.h14
-rw-r--r--drivers/iommu/amd_iommu_init.c140
-rw-r--r--drivers/iommu/amd_iommu_quirks.c92
-rw-r--r--drivers/iommu/amd_iommu_types.h18
-rw-r--r--drivers/iommu/arm-smmu-impl.c174
-rw-r--r--drivers/iommu/arm-smmu-regs.h210
-rw-r--r--drivers/iommu/arm-smmu-v3.c1054
-rw-r--r--drivers/iommu/arm-smmu.c675
-rw-r--r--drivers/iommu/arm-smmu.h402
-rw-r--r--drivers/iommu/dma-iommu.c475
-rw-r--r--drivers/iommu/dmar.c77
-rw-r--r--drivers/iommu/exynos-iommu.c9
-rw-r--r--drivers/iommu/intel-iommu-debugfs.c137
-rw-r--r--drivers/iommu/intel-iommu.c1198
-rw-r--r--drivers/iommu/intel-pasid.c17
-rw-r--r--drivers/iommu/intel-pasid.h26
-rw-r--r--drivers/iommu/intel-svm.c51
-rw-r--r--drivers/iommu/intel-trace.c14
-rw-r--r--drivers/iommu/intel_irq_remapping.c10
-rw-r--r--drivers/iommu/io-pgtable-arm-v7s.c162
-rw-r--r--drivers/iommu/io-pgtable-arm.c88
-rw-r--r--drivers/iommu/iommu.c505
-rw-r--r--drivers/iommu/iova.c27
-rw-r--r--drivers/iommu/ipmmu-vmsa.c288
-rw-r--r--drivers/iommu/msm_iommu.c43
-rw-r--r--drivers/iommu/mtk_iommu.c213
-rw-r--r--drivers/iommu/mtk_iommu.h21
-rw-r--r--drivers/iommu/mtk_iommu_v1.c9
-rw-r--r--drivers/iommu/omap-iommu-debug.c35
-rw-r--r--drivers/iommu/omap-iommu.c327
-rw-r--r--drivers/iommu/omap-iommu.h9
-rw-r--r--drivers/iommu/qcom_iommu.c72
-rw-r--r--drivers/iommu/rockchip-iommu.c2
-rw-r--r--drivers/iommu/s390-iommu.c3
-rw-r--r--drivers/iommu/tegra-gart.c12
-rw-r--r--drivers/iommu/tegra-smmu.c2
-rw-r--r--drivers/iommu/virtio-iommu.c1177
-rw-r--r--drivers/ipack/devices/ipoctal.h1
-rw-r--r--drivers/irqchip/Kconfig32
-rw-r--r--drivers/irqchip/Makefile2
-rw-r--r--drivers/irqchip/irq-al-fic.c278
-rw-r--r--drivers/irqchip/irq-csky-mpintc.c86
-rw-r--r--drivers/irqchip/irq-gic-common.c35
-rw-r--r--drivers/irqchip/irq-gic-common.h2
-rw-r--r--drivers/irqchip/irq-gic-v2m.c87
-rw-r--r--drivers/irqchip/irq-gic-v3-its.c17
-rw-r--r--drivers/irqchip/irq-gic-v3.c398
-rw-r--r--drivers/irqchip/irq-gic.c14
-rw-r--r--drivers/irqchip/irq-hip04.c9
-rw-r--r--drivers/irqchip/irq-imgpdc.c8
-rw-r--r--drivers/irqchip/irq-imx-gpcv2.c1
-rw-r--r--drivers/irqchip/irq-ixp4xx.c2
-rw-r--r--drivers/irqchip/irq-keystone.c4
-rw-r--r--drivers/irqchip/irq-mbigen.c12
-rw-r--r--drivers/irqchip/irq-meson-gpio.c53
-rw-r--r--drivers/irqchip/irq-mmp.c86
-rw-r--r--drivers/irqchip/irq-renesas-intc-irqpin.c3
-rw-r--r--drivers/irqchip/irq-renesas-irqc.c91
-rw-r--r--drivers/irqchip/irq-renesas-rza1.c284
-rw-r--r--drivers/irqchip/irq-sifive-plic.c12
-rw-r--r--drivers/irqchip/irq-sni-exiu.c142
-rw-r--r--drivers/irqchip/irq-uniphier-aidet.c4
-rw-r--r--drivers/irqchip/qcom-irq-combiner.c9
-rw-r--r--drivers/isdn/Kconfig51
-rw-r--r--drivers/isdn/Makefile6
-rw-r--r--drivers/isdn/capi/Kconfig29
-rw-r--r--drivers/isdn/capi/Makefile2
-rw-r--r--drivers/isdn/capi/capi.c10
-rw-r--r--drivers/isdn/capi/capidrv.c2525
-rw-r--r--drivers/isdn/capi/capidrv.h140
-rw-r--r--drivers/isdn/divert/Makefile10
-rw-r--r--drivers/isdn/divert/divert_init.c82
-rw-r--r--drivers/isdn/divert/divert_procfs.c336
-rw-r--r--drivers/isdn/divert/isdn_divert.c846
-rw-r--r--drivers/isdn/divert/isdn_divert.h132
-rw-r--r--drivers/isdn/gigaset/i4l.c692
-rw-r--r--drivers/isdn/hardware/Kconfig8
-rw-r--r--drivers/isdn/hardware/Makefile1
-rw-r--r--drivers/isdn/hardware/mISDN/Kconfig7
-rw-r--r--drivers/isdn/hardware/mISDN/Makefile2
-rw-r--r--drivers/isdn/hardware/mISDN/avmfritz.c3
-rw-r--r--drivers/isdn/hardware/mISDN/hfcpci.c6
-rw-r--r--drivers/isdn/hardware/mISDN/hfcsusb.c16
-rw-r--r--drivers/isdn/hardware/mISDN/isdnhdlc.c (renamed from drivers/isdn/i4l/isdnhdlc.c)2
-rw-r--r--drivers/isdn/hardware/mISDN/isdnhdlc.h69
-rw-r--r--drivers/isdn/hardware/mISDN/mISDNipac.c12
-rw-r--r--drivers/isdn/hardware/mISDN/mISDNisar.c3
-rw-r--r--drivers/isdn/hardware/mISDN/netjet.c5
-rw-r--r--drivers/isdn/hardware/mISDN/w6692.c9
-rw-r--r--drivers/isdn/hisax/Kconfig423
-rw-r--r--drivers/isdn/hisax/Makefile60
-rw-r--r--drivers/isdn/hisax/amd7930_fn.c794
-rw-r--r--drivers/isdn/hisax/amd7930_fn.h37
-rw-r--r--drivers/isdn/hisax/arcofi.c131
-rw-r--r--drivers/isdn/hisax/arcofi.h27
-rw-r--r--drivers/isdn/hisax/asuscom.c423
-rw-r--r--drivers/isdn/hisax/avm_a1.c307
-rw-r--r--drivers/isdn/hisax/avm_a1p.c267
-rw-r--r--drivers/isdn/hisax/avm_pci.c904
-rw-r--r--drivers/isdn/hisax/avma1_cs.c162
-rw-r--r--drivers/isdn/hisax/bkm_a4t.c358
-rw-r--r--drivers/isdn/hisax/bkm_a8.c433
-rw-r--r--drivers/isdn/hisax/bkm_ax.h119
-rw-r--r--drivers/isdn/hisax/callc.c1792
-rw-r--r--drivers/isdn/hisax/config.c1993
-rw-r--r--drivers/isdn/hisax/diva.c1282
-rw-r--r--drivers/isdn/hisax/elsa.c1245
-rw-r--r--drivers/isdn/hisax/elsa_cs.c218
-rw-r--r--drivers/isdn/hisax/elsa_ser.c659
-rw-r--r--drivers/isdn/hisax/enternow_pci.c420
-rw-r--r--drivers/isdn/hisax/fsm.c161
-rw-r--r--drivers/isdn/hisax/fsm.h61
-rw-r--r--drivers/isdn/hisax/gazel.c691
-rw-r--r--drivers/isdn/hisax/hfc4s8s_l1.c1584
-rw-r--r--drivers/isdn/hisax/hfc4s8s_l1.h89
-rw-r--r--drivers/isdn/hisax/hfc_2bds0.c1078
-rw-r--r--drivers/isdn/hisax/hfc_2bds0.h128
-rw-r--r--drivers/isdn/hisax/hfc_2bs0.c591
-rw-r--r--drivers/isdn/hisax/hfc_2bs0.h60
-rw-r--r--drivers/isdn/hisax/hfc_pci.c1755
-rw-r--r--drivers/isdn/hisax/hfc_pci.h235
-rw-r--r--drivers/isdn/hisax/hfc_sx.c1517
-rw-r--r--drivers/isdn/hisax/hfc_sx.h196
-rw-r--r--drivers/isdn/hisax/hfc_usb.c1594
-rw-r--r--drivers/isdn/hisax/hfc_usb.h208
-rw-r--r--drivers/isdn/hisax/hfcscard.c261
-rw-r--r--drivers/isdn/hisax/hisax.h1352
-rw-r--r--drivers/isdn/hisax/hisax_cfg.h66
-rw-r--r--drivers/isdn/hisax/hisax_debug.h80
-rw-r--r--drivers/isdn/hisax/hisax_fcpcipnp.c1024
-rw-r--r--drivers/isdn/hisax/hisax_fcpcipnp.h58
-rw-r--r--drivers/isdn/hisax/hisax_if.h66
-rw-r--r--drivers/isdn/hisax/hisax_isac.c895
-rw-r--r--drivers/isdn/hisax/hisax_isac.h46
-rw-r--r--drivers/isdn/hisax/hscx.c277
-rw-r--r--drivers/isdn/hisax/hscx.h41
-rw-r--r--drivers/isdn/hisax/hscx_irq.c294
-rw-r--r--drivers/isdn/hisax/icc.c680
-rw-r--r--drivers/isdn/hisax/icc.h72
-rw-r--r--drivers/isdn/hisax/ipac.h29
-rw-r--r--drivers/isdn/hisax/ipacx.c913
-rw-r--r--drivers/isdn/hisax/ipacx.h162
-rw-r--r--drivers/isdn/hisax/isac.c681
-rw-r--r--drivers/isdn/hisax/isac.h70
-rw-r--r--drivers/isdn/hisax/isar.c1910
-rw-r--r--drivers/isdn/hisax/isar.h222
-rw-r--r--drivers/isdn/hisax/isdnl1.c930
-rw-r--r--drivers/isdn/hisax/isdnl1.h32
-rw-r--r--drivers/isdn/hisax/isdnl2.c1839
-rw-r--r--drivers/isdn/hisax/isdnl2.h25
-rw-r--r--drivers/isdn/hisax/isdnl3.c594
-rw-r--r--drivers/isdn/hisax/isdnl3.h42
-rw-r--r--drivers/isdn/hisax/isurf.c305
-rw-r--r--drivers/isdn/hisax/ix1_micro.c316
-rw-r--r--drivers/isdn/hisax/jade.c305
-rw-r--r--drivers/isdn/hisax/jade.h134
-rw-r--r--drivers/isdn/hisax/jade_irq.c238
-rw-r--r--drivers/isdn/hisax/l3_1tr6.c932
-rw-r--r--drivers/isdn/hisax/l3_1tr6.h164
-rw-r--r--drivers/isdn/hisax/l3dss1.c3227
-rw-r--r--drivers/isdn/hisax/l3dss1.h124
-rw-r--r--drivers/isdn/hisax/l3ni1.c3182
-rw-r--r--drivers/isdn/hisax/l3ni1.h136
-rw-r--r--drivers/isdn/hisax/lmgr.c50
-rw-r--r--drivers/isdn/hisax/mic.c235
-rw-r--r--drivers/isdn/hisax/netjet.c985
-rw-r--r--drivers/isdn/hisax/netjet.h69
-rw-r--r--drivers/isdn/hisax/niccy.c380
-rw-r--r--drivers/isdn/hisax/nj_s.c294
-rw-r--r--drivers/isdn/hisax/nj_u.c258
-rw-r--r--drivers/isdn/hisax/q931.c1513
-rw-r--r--drivers/isdn/hisax/s0box.c260
-rw-r--r--drivers/isdn/hisax/saphir.c296
-rw-r--r--drivers/isdn/hisax/sedlbauer.c873
-rw-r--r--drivers/isdn/hisax/sedlbauer_cs.c209
-rw-r--r--drivers/isdn/hisax/sportster.c267
-rw-r--r--drivers/isdn/hisax/st5481.h529
-rw-r--r--drivers/isdn/hisax/st5481_b.c380
-rw-r--r--drivers/isdn/hisax/st5481_d.c780
-rw-r--r--drivers/isdn/hisax/st5481_init.c221
-rw-r--r--drivers/isdn/hisax/st5481_usb.c659
-rw-r--r--drivers/isdn/hisax/tei.c465
-rw-r--r--drivers/isdn/hisax/teleint.c334
-rw-r--r--drivers/isdn/hisax/teles0.c364
-rw-r--r--drivers/isdn/hisax/teles3.c498
-rw-r--r--drivers/isdn/hisax/teles_cs.c201
-rw-r--r--drivers/isdn/hisax/telespci.c349
-rw-r--r--drivers/isdn/hisax/w6692.c1085
-rw-r--r--drivers/isdn/hisax/w6692.h184
-rw-r--r--drivers/isdn/i4l/Kconfig129
-rw-r--r--drivers/isdn/i4l/Makefile20
-rw-r--r--drivers/isdn/i4l/isdn_audio.c711
-rw-r--r--drivers/isdn/i4l/isdn_audio.h44
-rw-r--r--drivers/isdn/i4l/isdn_bsdcomp.c930
-rw-r--r--drivers/isdn/i4l/isdn_common.c2368
-rw-r--r--drivers/isdn/i4l/isdn_common.h47
-rw-r--r--drivers/isdn/i4l/isdn_concap.c99
-rw-r--r--drivers/isdn/i4l/isdn_concap.h11
-rw-r--r--drivers/isdn/i4l/isdn_net.c3198
-rw-r--r--drivers/isdn/i4l/isdn_net.h151
-rw-r--r--drivers/isdn/i4l/isdn_ppp.c3046
-rw-r--r--drivers/isdn/i4l/isdn_ppp.h41
-rw-r--r--drivers/isdn/i4l/isdn_tty.c3756
-rw-r--r--drivers/isdn/i4l/isdn_tty.h120
-rw-r--r--drivers/isdn/i4l/isdn_ttyfax.c1123
-rw-r--r--drivers/isdn/i4l/isdn_ttyfax.h17
-rw-r--r--drivers/isdn/i4l/isdn_v110.c625
-rw-r--r--drivers/isdn/i4l/isdn_v110.h29
-rw-r--r--drivers/isdn/i4l/isdn_x25iface.c332
-rw-r--r--drivers/isdn/i4l/isdn_x25iface.h30
-rw-r--r--drivers/isdn/isdnloop/Makefile6
-rw-r--r--drivers/isdn/isdnloop/isdnloop.c1528
-rw-r--r--drivers/isdn/isdnloop/isdnloop.h112
-rw-r--r--drivers/isdn/mISDN/dsp_core.c2
-rw-r--r--drivers/isdn/mISDN/l1oip_core.c3
-rw-r--r--drivers/isdn/mISDN/layer2.c9
-rw-r--r--drivers/isdn/mISDN/stack.c6
-rw-r--r--drivers/isdn/mISDN/tei.c6
-rw-r--r--drivers/leds/Kconfig41
-rw-r--r--drivers/leds/Makefile4
-rw-r--r--drivers/leds/led-class-flash.c9
-rw-r--r--drivers/leds/led-class.c62
-rw-r--r--drivers/leds/led-core.c136
-rw-r--r--drivers/leds/led-triggers.c5
-rw-r--r--drivers/leds/leds-aat1290.c16
-rw-r--r--drivers/leds/leds-an30259a.c29
-rw-r--r--drivers/leds/leds-apu.c157
-rw-r--r--drivers/leds/leds-as3645a.c74
-rw-r--r--drivers/leds/leds-cr0014114.c33
-rw-r--r--drivers/leds/leds-gpio.c26
-rw-r--r--drivers/leds/leds-is31fl319x.c3
-rw-r--r--drivers/leds/leds-is31fl32xx.c8
-rw-r--r--drivers/leds/leds-ktd2692.c4
-rw-r--r--drivers/leds/leds-lm3532.c138
-rw-r--r--drivers/leds/leds-lm3601x.c38
-rw-r--r--drivers/leds/leds-lm36274.c170
-rw-r--r--drivers/leds/leds-lm3692x.c22
-rw-r--r--drivers/leds/leds-lm3697.c392
-rw-r--r--drivers/leds/leds-lp5562.c6
-rw-r--r--drivers/leds/leds-lp8860.c35
-rw-r--r--drivers/leds/leds-lt3593.c20
-rw-r--r--drivers/leds/leds-max77650.c46
-rw-r--r--drivers/leds/leds-netxbig.c76
-rw-r--r--drivers/leds/leds-ns2.c21
-rw-r--r--drivers/leds/leds-pca9532.c2
-rw-r--r--drivers/leds/leds-pca955x.c2
-rw-r--r--drivers/leds/leds-pwm.c52
-rw-r--r--drivers/leds/leds-sc27xx-bltc.c22
-rw-r--r--drivers/leds/leds-spi-byte.c161
-rw-r--r--drivers/leds/leds-syscon.c2
-rw-r--r--drivers/leds/leds-tca6507.c2
-rw-r--r--drivers/leds/leds-ti-lmu-common.c153
-rw-r--r--drivers/leds/leds.h1
-rw-r--r--drivers/leds/trigger/Kconfig2
-rw-r--r--drivers/leds/trigger/ledtrig-activity.c2
-rw-r--r--drivers/leds/trigger/ledtrig-gpio.c12
-rw-r--r--drivers/leds/trigger/ledtrig-transient.c2
-rw-r--r--drivers/lightnvm/core.c99
-rw-r--r--drivers/lightnvm/pblk-core.c132
-rw-r--r--drivers/lightnvm/pblk-gc.c19
-rw-r--r--drivers/lightnvm/pblk-init.c38
-rw-r--r--drivers/lightnvm/pblk-read.c26
-rw-r--r--drivers/lightnvm/pblk-recovery.c42
-rw-r--r--drivers/lightnvm/pblk-write.c20
-rw-r--r--drivers/lightnvm/pblk.h31
-rw-r--r--drivers/macintosh/smu.c5
-rw-r--r--drivers/macintosh/windfarm_cpufreq_clamp.c77
-rw-r--r--drivers/mailbox/Kconfig2
-rw-r--r--drivers/mailbox/arm_mhu.c11
-rw-r--r--drivers/mailbox/bcm-flexrm-mailbox.c34
-rw-r--r--drivers/mailbox/bcm-pdc-mailbox.c8
-rw-r--r--drivers/mailbox/imx-mailbox.c4
-rw-r--r--drivers/mailbox/mailbox.c6
-rw-r--r--drivers/mailbox/omap-mailbox.c43
-rw-r--r--drivers/mailbox/stm32-ipcc.c37
-rw-r--r--drivers/mailbox/tegra-hsp.c20
-rw-r--r--drivers/md/Kconfig2
-rw-r--r--drivers/md/bcache/alloc.c9
-rw-r--r--drivers/md/bcache/bcache.h6
-rw-r--r--drivers/md/bcache/bset.c61
-rw-r--r--drivers/md/bcache/btree.c53
-rw-r--r--drivers/md/bcache/btree.h2
-rw-r--r--drivers/md/bcache/closure.c10
-rw-r--r--drivers/md/bcache/debug.c5
-rw-r--r--drivers/md/bcache/io.c12
-rw-r--r--drivers/md/bcache/journal.c141
-rw-r--r--drivers/md/bcache/journal.h4
-rw-r--r--drivers/md/bcache/super.c230
-rw-r--r--drivers/md/bcache/sysfs.c48
-rw-r--r--drivers/md/bcache/util.h2
-rw-r--r--drivers/md/bcache/writeback.c8
-rw-r--r--drivers/md/dm-crypt.c101
-rw-r--r--drivers/md/dm-dust.c11
-rw-r--r--drivers/md/dm-flakey.c5
-rw-r--r--drivers/md/dm-init.c2
-rw-r--r--drivers/md/dm-integrity.c22
-rw-r--r--drivers/md/dm-kcopyd.c39
-rw-r--r--drivers/md/dm-linear.c5
-rw-r--r--drivers/md/dm-log-writes.c4
-rw-r--r--drivers/md/dm-raid.c4
-rw-r--r--drivers/md/dm-rq.c5
-rw-r--r--drivers/md/dm-snap.c196
-rw-r--r--drivers/md/dm-table.c33
-rw-r--r--drivers/md/dm-thin-metadata.c7
-rw-r--r--drivers/md/dm-zoned-metadata.c108
-rw-r--r--drivers/md/dm-zoned-reclaim.c47
-rw-r--r--drivers/md/dm-zoned-target.c68
-rw-r--r--drivers/md/dm-zoned.h39
-rw-r--r--drivers/md/dm.c11
-rw-r--r--drivers/md/dm.h5
-rw-r--r--drivers/md/md-bitmap.c20
-rw-r--r--drivers/md/md-linear.c5
-rw-r--r--drivers/md/md.c228
-rw-r--r--drivers/md/md.h43
-rw-r--r--drivers/md/persistent-data/dm-btree.c31
-rw-r--r--drivers/md/persistent-data/dm-space-map-metadata.c2
-rw-r--r--drivers/md/raid0.c41
-rw-r--r--drivers/md/raid0.h14
-rw-r--r--drivers/md/raid1-10.c30
-rw-r--r--drivers/md/raid1.c208
-rw-r--r--drivers/md/raid10.c118
-rw-r--r--drivers/md/raid5.c39
-rw-r--r--drivers/md/raid5.h5
-rw-r--r--drivers/media/Kconfig44
-rw-r--r--drivers/media/Makefile13
-rw-r--r--drivers/media/cec/cec-adap.c144
-rw-r--r--drivers/media/cec/cec-api.c8
-rw-r--r--drivers/media/cec/cec-core.c8
-rw-r--r--drivers/media/cec/cec-notifier.c114
-rw-r--r--drivers/media/cec/cec-priv.h5
-rw-r--r--drivers/media/common/saa7146/saa7146_fops.c9
-rw-r--r--drivers/media/common/saa7146/saa7146_video.c34
-rw-r--r--drivers/media/common/v4l2-tpg/v4l2-tpg-core.c107
-rw-r--r--drivers/media/common/videobuf2/videobuf2-core.c5
-rw-r--r--drivers/media/common/videobuf2/videobuf2-dma-contig.c26
-rw-r--r--drivers/media/common/videobuf2/videobuf2-dma-sg.c5
-rw-r--r--drivers/media/common/videobuf2/videobuf2-memops.c9
-rw-r--r--drivers/media/common/videobuf2/videobuf2-v4l2.c18
-rw-r--r--drivers/media/common/videobuf2/videobuf2-vmalloc.c3
-rw-r--r--drivers/media/dvb-core/Kconfig3
-rw-r--r--drivers/media/dvb-core/dvb_frontend.c144
-rw-r--r--drivers/media/dvb-core/dvbdev.c7
-rw-r--r--drivers/media/dvb-frontends/Kconfig6
-rw-r--r--drivers/media/dvb-frontends/cx24117.c4
-rw-r--r--drivers/media/dvb-frontends/cx24123.c2
-rw-r--r--drivers/media/dvb-frontends/cxd2099.c2
-rw-r--r--drivers/media/dvb-frontends/cxd2820r_core.c9
-rw-r--r--drivers/media/dvb-frontends/dib7000p.c7
-rw-r--r--drivers/media/dvb-frontends/drx39xyj/drxj.c5
-rw-r--r--drivers/media/dvb-frontends/dvb-pll.c40
-rw-r--r--drivers/media/dvb-frontends/mn88443x.c6
-rw-r--r--drivers/media/dvb-frontends/mn88472.c18
-rw-r--r--drivers/media/dvb-frontends/mn88473.c18
-rw-r--r--drivers/media/dvb-frontends/rtl2832_sdr.c8
-rw-r--r--drivers/media/dvb-frontends/si2168.c171
-rw-r--r--drivers/media/dvb-frontends/si2168.h1
-rw-r--r--drivers/media/dvb-frontends/stv0297.c2
-rw-r--r--drivers/media/dvb-frontends/stv0900_core.c2
-rw-r--r--drivers/media/dvb-frontends/stv090x.c197
-rw-r--r--drivers/media/dvb-frontends/stv090x.h3
-rw-r--r--drivers/media/dvb-frontends/stv090x_priv.h2
-rw-r--r--drivers/media/dvb-frontends/stv6110x.c135
-rw-r--r--drivers/media/dvb-frontends/stv6110x.h3
-rw-r--r--drivers/media/dvb-frontends/stv6110x_priv.h3
-rw-r--r--drivers/media/dvb-frontends/tua6100.c22
-rw-r--r--drivers/media/dvb-frontends/zd1301_demod.c3
-rw-r--r--drivers/media/firewire/firedtv-ci.c2
-rw-r--r--drivers/media/i2c/Kconfig21
-rw-r--r--drivers/media/i2c/Makefile3
-rw-r--r--drivers/media/i2c/ad9389b.c6
-rw-r--r--drivers/media/i2c/adv7180.c12
-rw-r--r--drivers/media/i2c/adv7343.c5
-rw-r--r--drivers/media/i2c/adv748x/adv748x-core.c13
-rw-r--r--drivers/media/i2c/adv7511-v4l2.c (renamed from drivers/media/i2c/adv7511.c)33
-rw-r--r--drivers/media/i2c/adv7842.c42
-rw-r--r--drivers/media/i2c/ak881x.c2
-rw-r--r--drivers/media/i2c/cx25840/cx25840-core.c1409
-rw-r--r--drivers/media/i2c/cx25840/cx25840-core.h30
-rw-r--r--drivers/media/i2c/cx25840/cx25840-vbi.c4
-rw-r--r--drivers/media/i2c/et8ek8/et8ek8_driver.c5
-rw-r--r--drivers/media/i2c/imx214.c2
-rw-r--r--drivers/media/i2c/imx274.c5
-rw-r--r--drivers/media/i2c/ir-kbd-i2c.c14
-rw-r--r--drivers/media/i2c/max2175.c5
-rw-r--r--drivers/media/i2c/mt9m001.c7
-rw-r--r--drivers/media/i2c/mt9m111.c47
-rw-r--r--drivers/media/i2c/mt9p031.c2
-rw-r--r--drivers/media/i2c/ov13858.c4
-rw-r--r--drivers/media/i2c/ov2640.c9
-rw-r--r--drivers/media/i2c/ov2659.c9
-rw-r--r--drivers/media/i2c/ov2680.c9
-rw-r--r--drivers/media/i2c/ov2685.c2
-rw-r--r--drivers/media/i2c/ov5640.c14
-rw-r--r--drivers/media/i2c/ov5645.c151
-rw-r--r--drivers/media/i2c/ov5647.c5
-rw-r--r--drivers/media/i2c/ov5675.c1183
-rw-r--r--drivers/media/i2c/ov5695.c7
-rw-r--r--drivers/media/i2c/ov6650.c1
-rw-r--r--drivers/media/i2c/ov7670.c6
-rw-r--r--drivers/media/i2c/ov772x.c5
-rw-r--r--drivers/media/i2c/ov7740.c37
-rw-r--r--drivers/media/i2c/ov8856.c17
-rw-r--r--drivers/media/i2c/ov9640.c4
-rw-r--r--drivers/media/i2c/ov9650.c10
-rw-r--r--drivers/media/i2c/s5c73m3/s5c73m3-core.c5
-rw-r--r--drivers/media/i2c/s5k5baf.c5
-rw-r--r--drivers/media/i2c/s5k6a3.c5
-rw-r--r--drivers/media/i2c/smiapp/smiapp-core.c5
-rw-r--r--drivers/media/i2c/smiapp/smiapp-quirk.c2
-rw-r--r--drivers/media/i2c/st-mipid02.c60
-rw-r--r--drivers/media/i2c/tc358743.c5
-rw-r--r--drivers/media/i2c/tda1997x.c9
-rw-r--r--drivers/media/i2c/tda7432.c3
-rw-r--r--drivers/media/i2c/ths8200.c5
-rw-r--r--drivers/media/i2c/tvp5150.c9
-rw-r--r--drivers/media/i2c/tvp7002.c4
-rw-r--r--drivers/media/i2c/tw9910.c3
-rw-r--r--drivers/media/i2c/video-i2c.c8
-rw-r--r--drivers/media/mc/Kconfig33
-rw-r--r--drivers/media/mc/Makefile10
-rw-r--r--drivers/media/mc/mc-dev-allocator.c (renamed from drivers/media/media-dev-allocator.c)0
-rw-r--r--drivers/media/mc/mc-device.c (renamed from drivers/media/media-device.c)10
-rw-r--r--drivers/media/mc/mc-devnode.c (renamed from drivers/media/media-devnode.c)0
-rw-r--r--drivers/media/mc/mc-entity.c (renamed from drivers/media/media-entity.c)0
-rw-r--r--drivers/media/mc/mc-request.c (renamed from drivers/media/media-request.c)0
-rw-r--r--drivers/media/pci/bt8xx/bttv-audio-hook.c2
-rw-r--r--drivers/media/pci/bt8xx/bttv-audio-hook.h2
-rw-r--r--drivers/media/pci/bt8xx/bttv-driver.c69
-rw-r--r--drivers/media/pci/bt8xx/bttv-input.c4
-rw-r--r--drivers/media/pci/bt8xx/bttv-risc.c8
-rw-r--r--drivers/media/pci/bt8xx/bttvp.h1
-rw-r--r--drivers/media/pci/bt8xx/dvb-bt8xx.c2
-rw-r--r--drivers/media/pci/cobalt/Kconfig2
-rw-r--r--drivers/media/pci/cobalt/cobalt-driver.c40
-rw-r--r--drivers/media/pci/cobalt/cobalt-driver.h63
-rw-r--r--drivers/media/pci/cobalt/cobalt-flash.c2
-rw-r--r--drivers/media/pci/cobalt/cobalt-v4l2.c20
-rw-r--r--drivers/media/pci/cx18/cx18-ioctl.c7
-rw-r--r--drivers/media/pci/cx18/cx18-mailbox.c2
-rw-r--r--drivers/media/pci/cx18/cx18-streams.c1
-rw-r--r--drivers/media/pci/cx23885/cx23885-417.c14
-rw-r--r--drivers/media/pci/cx23885/cx23885-dvb.c2
-rw-r--r--drivers/media/pci/cx23885/cx23885-video.c29
-rw-r--r--drivers/media/pci/cx23885/cx23885.h1
-rw-r--r--drivers/media/pci/cx25821/cx25821-video.c21
-rw-r--r--drivers/media/pci/cx25821/cx25821.h1
-rw-r--r--drivers/media/pci/cx88/cx88-alsa.c2
-rw-r--r--drivers/media/pci/cx88/cx88-blackbird.c8
-rw-r--r--drivers/media/pci/cx88/cx88-core.c2
-rw-r--r--drivers/media/pci/cx88/cx88-i2c.c1
-rw-r--r--drivers/media/pci/cx88/cx88-input.c4
-rw-r--r--drivers/media/pci/cx88/cx88-video.c51
-rw-r--r--drivers/media/pci/cx88/cx88.h1
-rw-r--r--drivers/media/pci/ddbridge/Kconfig1
-rw-r--r--drivers/media/pci/dt3155/Kconfig1
-rw-r--r--drivers/media/pci/dt3155/dt3155.c6
-rw-r--r--drivers/media/pci/intel/ipu3/ipu3-cio2.c102
-rw-r--r--drivers/media/pci/ivtv/Kconfig2
-rw-r--r--drivers/media/pci/ivtv/ivtv-cards.h3
-rw-r--r--drivers/media/pci/ivtv/ivtv-driver.c2
-rw-r--r--drivers/media/pci/ivtv/ivtv-ioctl.c11
-rw-r--r--drivers/media/pci/ivtv/ivtv-irq.h28
-rw-r--r--drivers/media/pci/ivtv/ivtv-mailbox.c2
-rw-r--r--drivers/media/pci/ivtv/ivtv-streams.c14
-rw-r--r--drivers/media/pci/ivtv/ivtvfb.c16
-rw-r--r--drivers/media/pci/mantis/mantis_reg.h152
-rw-r--r--drivers/media/pci/meye/Kconfig3
-rw-r--r--drivers/media/pci/meye/meye.c9
-rw-r--r--drivers/media/pci/ngene/ngene-core.c4
-rw-r--r--drivers/media/pci/pt1/pt1.c6
-rw-r--r--drivers/media/pci/pt3/pt3.c6
-rw-r--r--drivers/media/pci/saa7134/saa7134-core.c15
-rw-r--r--drivers/media/pci/saa7134/saa7134-dvb.c14
-rw-r--r--drivers/media/pci/saa7134/saa7134-empress.c6
-rw-r--r--drivers/media/pci/saa7134/saa7134-i2c.c12
-rw-r--r--drivers/media/pci/saa7134/saa7134-tvaudio.c1
-rw-r--r--drivers/media/pci/saa7134/saa7134-video.c74
-rw-r--r--drivers/media/pci/saa7134/saa7134.h1
-rw-r--r--drivers/media/pci/saa7146/hexium_gemini.c3
-rw-r--r--drivers/media/pci/saa7164/saa7164-core.c33
-rw-r--r--drivers/media/pci/saa7164/saa7164-encoder.c16
-rw-r--r--drivers/media/pci/saa7164/saa7164-vbi.c15
-rw-r--r--drivers/media/pci/solo6x10/solo6x10-gpio.c6
-rw-r--r--drivers/media/pci/solo6x10/solo6x10-regs.h286
-rw-r--r--drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c14
-rw-r--r--drivers/media/pci/solo6x10/solo6x10-v4l2.c9
-rw-r--r--drivers/media/pci/sta2x11/sta2x11_vip.c8
-rw-r--r--drivers/media/pci/ttpci/Kconfig3
-rw-r--r--drivers/media/pci/ttpci/av7110.c14
-rw-r--r--drivers/media/pci/ttpci/av7110.h21
-rw-r--r--drivers/media/pci/ttpci/av7110_hw.c1
-rw-r--r--drivers/media/pci/ttpci/av7110_ir.c421
-rw-r--r--drivers/media/pci/tw68/tw68-video.c22
-rw-r--r--drivers/media/pci/tw68/tw68.h1
-rw-r--r--drivers/media/pci/tw686x/tw686x-video.c5
-rw-r--r--drivers/media/platform/Kconfig18
-rw-r--r--drivers/media/platform/Makefile2
-rw-r--r--drivers/media/platform/am437x/am437x-vpfe.c30
-rw-r--r--drivers/media/platform/am437x/am437x-vpfe.h6
-rw-r--r--drivers/media/platform/am437x/am437x-vpfe_regs.h26
-rw-r--r--drivers/media/platform/aspeed-video.c162
-rw-r--r--drivers/media/platform/atmel/Makefile4
-rw-r--r--drivers/media/platform/atmel/atmel-isc-base.c (renamed from drivers/media/platform/atmel/atmel-isc.c)783
-rw-r--r--drivers/media/platform/atmel/atmel-isc-regs.h6
-rw-r--r--drivers/media/platform/atmel/atmel-isc.h245
-rw-r--r--drivers/media/platform/atmel/atmel-isi.c2
-rw-r--r--drivers/media/platform/atmel/atmel-sama5d2-isc.c345
-rw-r--r--drivers/media/platform/cadence/cdns-csi2tx.c155
-rw-r--r--drivers/media/platform/cec-gpio/cec-gpio.c28
-rw-r--r--drivers/media/platform/coda/Makefile5
-rw-r--r--drivers/media/platform/coda/coda-bit.c452
-rw-r--r--drivers/media/platform/coda/coda-common.c396
-rw-r--r--drivers/media/platform/coda/coda-h264.c3
-rw-r--r--drivers/media/platform/coda/coda-mpeg2.c87
-rw-r--r--drivers/media/platform/coda/coda-mpeg4.c87
-rw-r--r--drivers/media/platform/coda/coda.h47
-rw-r--r--drivers/media/platform/coda/coda_regs.h20
-rw-r--r--drivers/media/platform/coda/trace.h2
-rw-r--r--drivers/media/platform/cros-ec-cec/cros-ec-cec.c69
-rw-r--r--drivers/media/platform/davinci/dm644x_ccdc_regs.h20
-rw-r--r--drivers/media/platform/davinci/vpbe_display.c21
-rw-r--r--drivers/media/platform/davinci/vpbe_osd.c5
-rw-r--r--drivers/media/platform/davinci/vpbe_venc.c5
-rw-r--r--drivers/media/platform/davinci/vpfe_capture.c54
-rw-r--r--drivers/media/platform/davinci/vpif_capture.c49
-rw-r--r--drivers/media/platform/davinci/vpif_display.c7
-rw-r--r--drivers/media/platform/davinci/vpss.c7
-rw-r--r--drivers/media/platform/exynos-gsc/gsc-core.c24
-rw-r--r--drivers/media/platform/exynos-gsc/gsc-core.h4
-rw-r--r--drivers/media/platform/exynos-gsc/gsc-m2m.c14
-rw-r--r--drivers/media/platform/exynos4-is/common.c5
-rw-r--r--drivers/media/platform/exynos4-is/common.h3
-rw-r--r--drivers/media/platform/exynos4-is/fimc-capture.c13
-rw-r--r--drivers/media/platform/exynos4-is/fimc-core.c20
-rw-r--r--drivers/media/platform/exynos4-is/fimc-is.c2
-rw-r--r--drivers/media/platform/exynos4-is/fimc-isp-video.c10
-rw-r--r--drivers/media/platform/exynos4-is/fimc-isp.c3
-rw-r--r--drivers/media/platform/exynos4-is/fimc-lite-reg.h80
-rw-r--r--drivers/media/platform/exynos4-is/fimc-lite.c18
-rw-r--r--drivers/media/platform/exynos4-is/fimc-m2m.c13
-rw-r--r--drivers/media/platform/exynos4-is/fimc-reg.h138
-rw-r--r--drivers/media/platform/exynos4-is/media-dev.c8
-rw-r--r--drivers/media/platform/exynos4-is/mipi-csis.c10
-rw-r--r--drivers/media/platform/fsl-viu.c11
-rw-r--r--drivers/media/platform/imx-pxp.c4
-rw-r--r--drivers/media/platform/m2m-deinterlace.c140
-rw-r--r--drivers/media/platform/marvell-ccic/Kconfig2
-rw-r--r--drivers/media/platform/marvell-ccic/cafe-driver.c58
-rw-r--r--drivers/media/platform/marvell-ccic/mcam-core.c363
-rw-r--r--drivers/media/platform/marvell-ccic/mcam-core.h12
-rw-r--r--drivers/media/platform/marvell-ccic/mmp-driver.c239
-rw-r--r--drivers/media/platform/meson/ao-cec-g12a.c105
-rw-r--r--drivers/media/platform/meson/ao-cec.c44
-rw-r--r--drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c6
-rw-r--r--drivers/media/platform/mtk-mdp/mtk_mdp_core.c9
-rw-r--r--drivers/media/platform/mtk-mdp/mtk_mdp_m2m.c18
-rw-r--r--drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.c48
-rw-r--r--drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.h2
-rw-r--r--drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_drv.c2
-rw-r--r--drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_pm.c4
-rw-r--r--drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_pm.h2
-rw-r--r--drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h7
-rw-r--r--drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c47
-rw-r--r--drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.h2
-rw-r--r--drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c2
-rw-r--r--drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_pm.c2
-rw-r--r--drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_pm.h2
-rw-r--r--drivers/media/platform/mtk-vcodec/mtk_vcodec_intr.c2
-rw-r--r--drivers/media/platform/mtk-vcodec/mtk_vcodec_intr.h2
-rw-r--r--drivers/media/platform/mtk-vcodec/mtk_vcodec_util.c2
-rw-r--r--drivers/media/platform/mtk-vcodec/mtk_vcodec_util.h2
-rw-r--r--drivers/media/platform/mtk-vcodec/vdec/vdec_h264_if.c39
-rw-r--r--drivers/media/platform/mtk-vcodec/vdec/vdec_vp8_if.c23
-rw-r--r--drivers/media/platform/mtk-vcodec/vdec/vdec_vp9_if.c25
-rw-r--r--drivers/media/platform/mtk-vcodec/vdec_drv_base.h10
-rw-r--r--drivers/media/platform/mtk-vcodec/vdec_drv_if.c22
-rw-r--r--drivers/media/platform/mtk-vcodec/vdec_drv_if.h6
-rw-r--r--drivers/media/platform/mtk-vcodec/vdec_ipi_msg.h2
-rw-r--r--drivers/media/platform/mtk-vcodec/vdec_vpu_if.c2
-rw-r--r--drivers/media/platform/mtk-vcodec/vdec_vpu_if.h2
-rw-r--r--drivers/media/platform/mtk-vcodec/venc/venc_h264_if.c21
-rw-r--r--drivers/media/platform/mtk-vcodec/venc/venc_vp8_if.c21
-rw-r--r--drivers/media/platform/mtk-vcodec/venc_drv_base.h10
-rw-r--r--drivers/media/platform/mtk-vcodec/venc_drv_if.c15
-rw-r--r--drivers/media/platform/mtk-vcodec/venc_drv_if.h5
-rw-r--r--drivers/media/platform/mtk-vcodec/venc_ipi_msg.h2
-rw-r--r--drivers/media/platform/mtk-vcodec/venc_vpu_if.c2
-rw-r--r--drivers/media/platform/mtk-vcodec/venc_vpu_if.h2
-rw-r--r--drivers/media/platform/mtk-vpu/mtk_vpu.c2
-rw-r--r--drivers/media/platform/mx2_emmaprp.c141
-rw-r--r--drivers/media/platform/omap/Kconfig4
-rw-r--r--drivers/media/platform/omap/omap_vout.c1045
-rw-r--r--drivers/media/platform/omap/omap_vout_vrfb.c29
-rw-r--r--drivers/media/platform/omap/omap_vout_vrfb.h4
-rw-r--r--drivers/media/platform/omap/omap_voutdef.h45
-rw-r--r--drivers/media/platform/omap/omap_voutlib.c6
-rw-r--r--drivers/media/platform/omap3isp/isp.c358
-rw-r--r--drivers/media/platform/omap3isp/ispccdc.c1
-rw-r--r--drivers/media/platform/omap3isp/ispccp2.c1
-rw-r--r--drivers/media/platform/omap3isp/ispcsi2.c1
-rw-r--r--drivers/media/platform/omap3isp/isph3a_aewb.c24
-rw-r--r--drivers/media/platform/omap3isp/isph3a_af.c24
-rw-r--r--drivers/media/platform/omap3isp/isphist.c11
-rw-r--r--drivers/media/platform/omap3isp/isppreview.c1
-rw-r--r--drivers/media/platform/omap3isp/ispreg.h584
-rw-r--r--drivers/media/platform/omap3isp/ispresizer.c1
-rw-r--r--drivers/media/platform/omap3isp/ispstat.c6
-rw-r--r--drivers/media/platform/omap3isp/ispvideo.c18
-rw-r--r--drivers/media/platform/pxa_camera.c9
-rw-r--r--drivers/media/platform/qcom/camss/camss-video.c2
-rw-r--r--drivers/media/platform/qcom/camss/camss.c2
-rw-r--r--drivers/media/platform/qcom/venus/core.c6
-rw-r--r--drivers/media/platform/qcom/venus/core.h33
-rw-r--r--drivers/media/platform/qcom/venus/firmware.c6
-rw-r--r--drivers/media/platform/qcom/venus/helpers.c206
-rw-r--r--drivers/media/platform/qcom/venus/helpers.h11
-rw-r--r--drivers/media/platform/qcom/venus/hfi.c11
-rw-r--r--drivers/media/platform/qcom/venus/hfi.h2
-rw-r--r--drivers/media/platform/qcom/venus/hfi_cmds.c2
-rw-r--r--drivers/media/platform/qcom/venus/vdec.c552
-rw-r--r--drivers/media/platform/qcom/venus/vdec_ctrls.c9
-rw-r--r--drivers/media/platform/qcom/venus/venc.c17
-rw-r--r--drivers/media/platform/qcom/venus/venc_ctrls.c23
-rw-r--r--drivers/media/platform/rcar-vin/rcar-core.c55
-rw-r--r--drivers/media/platform/rcar-vin/rcar-csi2.c4
-rw-r--r--drivers/media/platform/rcar-vin/rcar-dma.c56
-rw-r--r--drivers/media/platform/rcar-vin/rcar-v4l2.c226
-rw-r--r--drivers/media/platform/rcar-vin/rcar-vin.h9
-rw-r--r--drivers/media/platform/rcar_fdp1.c15
-rw-r--r--drivers/media/platform/rcar_jpu.c10
-rw-r--r--drivers/media/platform/renesas-ceu.c6
-rw-r--r--drivers/media/platform/rockchip/rga/rga.c1
-rw-r--r--drivers/media/platform/s3c-camif/camif-capture.c16
-rw-r--r--drivers/media/platform/s3c-camif/camif-core.c10
-rw-r--r--drivers/media/platform/s3c-camif/camif-core.h1
-rw-r--r--drivers/media/platform/s3c-camif/camif-regs.h118
-rw-r--r--drivers/media/platform/s5p-cec/s5p_cec.c23
-rw-r--r--drivers/media/platform/s5p-g2d/g2d.c13
-rw-r--r--drivers/media/platform/s5p-g2d/g2d.h1
-rw-r--r--drivers/media/platform/s5p-jpeg/jpeg-core.c31
-rw-r--r--drivers/media/platform/s5p-jpeg/jpeg-core.h2
-rw-r--r--drivers/media/platform/s5p-jpeg/jpeg-regs.h10
-rw-r--r--drivers/media/platform/s5p-mfc/s5p_mfc.c5
-rw-r--r--drivers/media/platform/s5p-mfc/s5p_mfc_common.h1
-rw-r--r--drivers/media/platform/s5p-mfc/s5p_mfc_dec.c38
-rw-r--r--drivers/media/platform/s5p-mfc/s5p_mfc_enc.c35
-rw-r--r--drivers/media/platform/s5p-mfc/s5p_mfc_opr_v5.c8
-rw-r--r--drivers/media/platform/s5p-mfc/s5p_mfc_opr_v6.c10
-rw-r--r--drivers/media/platform/s5p-mfc/s5p_mfc_pm.c5
-rw-r--r--drivers/media/platform/seco-cec/seco-cec.c57
-rw-r--r--drivers/media/platform/sh_veu.c28
-rw-r--r--drivers/media/platform/sh_vou.c17
-rw-r--r--drivers/media/platform/sti/bdisp/bdisp-v4l2.c6
-rw-r--r--drivers/media/platform/sti/c8sectpfe/c8sectpfe-core.c8
-rw-r--r--drivers/media/platform/sti/c8sectpfe/c8sectpfe-dvb.c4
-rw-r--r--drivers/media/platform/sti/cec/stih-cec.c32
-rw-r--r--drivers/media/platform/sti/hva/hva-hw.c8
-rw-r--r--drivers/media/platform/sti/hva/hva-v4l2.c4
-rw-r--r--drivers/media/platform/stm32/stm32-dcmi.c325
-rw-r--r--drivers/media/platform/sunxi/Kconfig2
-rw-r--r--drivers/media/platform/sunxi/Makefile2
-rw-r--r--drivers/media/platform/sunxi/sun4i-csi/Kconfig11
-rw-r--r--drivers/media/platform/sunxi/sun4i-csi/Makefile5
-rw-r--r--drivers/media/platform/sunxi/sun4i-csi/sun4i_csi.c314
-rw-r--r--drivers/media/platform/sunxi/sun4i-csi/sun4i_csi.h160
-rw-r--r--drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c454
-rw-r--r--drivers/media/platform/sunxi/sun4i-csi/sun4i_v4l2.c385
-rw-r--r--drivers/media/platform/sunxi/sun6i-csi/sun6i_csi.c8
-rw-r--r--drivers/media/platform/tegra-cec/tegra_cec.c35
-rw-r--r--drivers/media/platform/tegra-cec/tegra_cec.h82
-rw-r--r--drivers/media/platform/ti-vpe/cal.c6
-rw-r--r--drivers/media/platform/ti-vpe/vpdma.c10
-rw-r--r--drivers/media/platform/ti-vpe/vpe.c19
-rw-r--r--drivers/media/platform/ti-vpe/vpe_regs.h94
-rw-r--r--drivers/media/platform/via-camera.c579
-rw-r--r--drivers/media/platform/vicodec/Kconfig1
-rw-r--r--drivers/media/platform/vicodec/codec-v4l2-fwht.c30
-rw-r--r--drivers/media/platform/vicodec/vicodec-core.c327
-rw-r--r--drivers/media/platform/vim2m.c6
-rw-r--r--drivers/media/platform/vimc/Kconfig1
-rw-r--r--drivers/media/platform/vimc/Makefile12
-rw-r--r--drivers/media/platform/vimc/vimc-capture.c81
-rw-r--r--drivers/media/platform/vimc/vimc-common.c313
-rw-r--r--drivers/media/platform/vimc/vimc-common.h58
-rw-r--r--drivers/media/platform/vimc/vimc-core.c7
-rw-r--r--drivers/media/platform/vimc/vimc-debayer.c83
-rw-r--r--drivers/media/platform/vimc/vimc-scaler.c63
-rw-r--r--drivers/media/platform/vimc/vimc-sensor.c52
-rw-r--r--drivers/media/platform/vimc/vimc-streamer.c46
-rw-r--r--drivers/media/platform/vimc/vimc-streamer.h14
-rw-r--r--drivers/media/platform/vivid/Kconfig1
-rw-r--r--drivers/media/platform/vivid/vivid-core.c138
-rw-r--r--drivers/media/platform/vivid/vivid-core.h44
-rw-r--r--drivers/media/platform/vivid/vivid-ctrls.c110
-rw-r--r--drivers/media/platform/vivid/vivid-kthread-cap.c17
-rw-r--r--drivers/media/platform/vivid/vivid-osd.c2
-rw-r--r--drivers/media/platform/vivid/vivid-vbi-cap.c16
-rw-r--r--drivers/media/platform/vivid/vivid-vid-cap.c142
-rw-r--r--drivers/media/platform/vivid/vivid-vid-common.c160
-rw-r--r--drivers/media/platform/vivid/vivid-vid-common.h2
-rw-r--r--drivers/media/platform/vivid/vivid-vid-out.c6
-rw-r--r--drivers/media/platform/vsp1/vsp1_dl.c4
-rw-r--r--drivers/media/platform/vsp1/vsp1_histo.c3
-rw-r--r--drivers/media/platform/vsp1/vsp1_regs.h224
-rw-r--r--drivers/media/platform/vsp1/vsp1_video.c10
-rw-r--r--drivers/media/platform/xilinx/xilinx-dma.c18
-rw-r--r--drivers/media/platform/xilinx/xilinx-vip.c16
-rw-r--r--drivers/media/platform/xilinx/xilinx-vip.h31
-rw-r--r--drivers/media/platform/xilinx/xilinx-vipp.c2
-rw-r--r--drivers/media/radio/Kconfig1
-rw-r--r--drivers/media/radio/dsbr100.c3
-rw-r--r--drivers/media/radio/radio-cadet.c5
-rw-r--r--drivers/media/radio/radio-gemtek.c2
-rw-r--r--drivers/media/radio/radio-isa.c4
-rw-r--r--drivers/media/radio/radio-keene.c3
-rw-r--r--drivers/media/radio/radio-ma901.c3
-rw-r--r--drivers/media/radio/radio-miropcm20.c4
-rw-r--r--drivers/media/radio/radio-mr800.c5
-rw-r--r--drivers/media/radio/radio-raremono.c33
-rw-r--r--drivers/media/radio/radio-sf16fmi.c3
-rw-r--r--drivers/media/radio/radio-si476x.c21
-rw-r--r--drivers/media/radio/radio-tea5764.c3
-rw-r--r--drivers/media/radio/radio-tea5777.c5
-rw-r--r--drivers/media/radio/radio-timb.c3
-rw-r--r--drivers/media/radio/radio-trust.c1
-rw-r--r--drivers/media/radio/radio-wl1273.c12
-rw-r--r--drivers/media/radio/si470x/radio-si470x-i2c.c12
-rw-r--r--drivers/media/radio/si470x/radio-si470x-usb.c11
-rw-r--r--drivers/media/radio/si4713/radio-platform-si4713.c4
-rw-r--r--drivers/media/radio/si4713/radio-usb-si4713.c4
-rw-r--r--drivers/media/radio/si4713/si4713.c5
-rw-r--r--drivers/media/radio/tea575x.c7
-rw-r--r--drivers/media/radio/wl128x/fmdrv_common.h88
-rw-r--r--drivers/media/radio/wl128x/fmdrv_v4l2.c13
-rw-r--r--drivers/media/rc/bpf-lirc.c30
-rw-r--r--drivers/media/rc/iguanair.c15
-rw-r--r--drivers/media/rc/img-ir/img-ir-core.c4
-rw-r--r--drivers/media/rc/imon.c7
-rw-r--r--drivers/media/rc/imon_raw.c43
-rw-r--r--drivers/media/rc/ir-hix5hd2.c4
-rw-r--r--drivers/media/rc/ir-spi.c1
-rw-r--r--drivers/media/rc/keymaps/Makefile7
-rw-r--r--drivers/media/rc/keymaps/rc-adstech-dvb-t-pci.c20
-rw-r--r--drivers/media/rc/keymaps/rc-alink-dtu-m.c20
-rw-r--r--drivers/media/rc/keymaps/rc-anysee.c20
-rw-r--r--drivers/media/rc/keymaps/rc-apac-viewcomp.c20
-rw-r--r--drivers/media/rc/keymaps/rc-astrometa-t2hybrid.c20
-rw-r--r--drivers/media/rc/keymaps/rc-asus-pc39.c20
-rw-r--r--drivers/media/rc/keymaps/rc-asus-ps3-100.c20
-rw-r--r--drivers/media/rc/keymaps/rc-ati-x10.c20
-rw-r--r--drivers/media/rc/keymaps/rc-avermedia-a16d.c20
-rw-r--r--drivers/media/rc/keymaps/rc-avermedia-cardbus.c20
-rw-r--r--drivers/media/rc/keymaps/rc-avermedia-dvbt.c20
-rw-r--r--drivers/media/rc/keymaps/rc-avermedia-m135a.c40
-rw-r--r--drivers/media/rc/keymaps/rc-avermedia-m733a-rm-k6.c20
-rw-r--r--drivers/media/rc/keymaps/rc-avermedia-rm-ks.c20
-rw-r--r--drivers/media/rc/keymaps/rc-avermedia.c20
-rw-r--r--drivers/media/rc/keymaps/rc-avertv-303.c20
-rw-r--r--drivers/media/rc/keymaps/rc-azurewave-ad-tu700.c20
-rw-r--r--drivers/media/rc/keymaps/rc-behold-columbus.c20
-rw-r--r--drivers/media/rc/keymaps/rc-behold.c20
-rw-r--r--drivers/media/rc/keymaps/rc-budget-ci-old.c20
-rw-r--r--drivers/media/rc/keymaps/rc-cinergy-1400.c20
-rw-r--r--drivers/media/rc/keymaps/rc-cinergy.c20
-rw-r--r--drivers/media/rc/keymaps/rc-d680-dmb.c20
-rw-r--r--drivers/media/rc/keymaps/rc-delock-61959.c20
-rw-r--r--drivers/media/rc/keymaps/rc-dib0700-nec.c40
-rw-r--r--drivers/media/rc/keymaps/rc-dib0700-rc5.c100
-rw-r--r--drivers/media/rc/keymaps/rc-digitalnow-tinytwin.c20
-rw-r--r--drivers/media/rc/keymaps/rc-digittrade.c20
-rw-r--r--drivers/media/rc/keymaps/rc-dm1105-nec.c20
-rw-r--r--drivers/media/rc/keymaps/rc-dntv-live-dvb-t.c20
-rw-r--r--drivers/media/rc/keymaps/rc-dntv-live-dvbt-pro.c20
-rw-r--r--drivers/media/rc/keymaps/rc-dtt200u.c20
-rw-r--r--drivers/media/rc/keymaps/rc-dvbsky.c20
-rw-r--r--drivers/media/rc/keymaps/rc-dvico-mce.c20
-rw-r--r--drivers/media/rc/keymaps/rc-dvico-portable.c20
-rw-r--r--drivers/media/rc/keymaps/rc-em-terratec.c20
-rw-r--r--drivers/media/rc/keymaps/rc-encore-enltv-fm53.c20
-rw-r--r--drivers/media/rc/keymaps/rc-encore-enltv.c20
-rw-r--r--drivers/media/rc/keymaps/rc-encore-enltv2.c20
-rw-r--r--drivers/media/rc/keymaps/rc-eztv.c20
-rw-r--r--drivers/media/rc/keymaps/rc-flydvb.c20
-rw-r--r--drivers/media/rc/keymaps/rc-flyvideo.c20
-rw-r--r--drivers/media/rc/keymaps/rc-fusionhdtv-mce.c20
-rw-r--r--drivers/media/rc/keymaps/rc-gadmei-rm008z.c20
-rw-r--r--drivers/media/rc/keymaps/rc-genius-tvgo-a11mce.c20
-rw-r--r--drivers/media/rc/keymaps/rc-gotview7135.c20
-rw-r--r--drivers/media/rc/keymaps/rc-hauppauge.c101
-rw-r--r--drivers/media/rc/keymaps/rc-hisi-poplar.c20
-rw-r--r--drivers/media/rc/keymaps/rc-hisi-tv-demo.c20
-rw-r--r--drivers/media/rc/keymaps/rc-imon-rsc.c7
-rw-r--r--drivers/media/rc/keymaps/rc-iodata-bctv7e.c20
-rw-r--r--drivers/media/rc/keymaps/rc-it913x-v1.c40
-rw-r--r--drivers/media/rc/keymaps/rc-it913x-v2.c40
-rw-r--r--drivers/media/rc/keymaps/rc-kaiomy.c20
-rw-r--r--drivers/media/rc/keymaps/rc-khadas.c54
-rw-r--r--drivers/media/rc/keymaps/rc-kworld-315u.c20
-rw-r--r--drivers/media/rc/keymaps/rc-kworld-pc150u.c20
-rw-r--r--drivers/media/rc/keymaps/rc-kworld-plus-tv-analog.c24
-rw-r--r--drivers/media/rc/keymaps/rc-leadtek-y04g0051.c20
-rw-r--r--drivers/media/rc/keymaps/rc-lme2510.c60
-rw-r--r--drivers/media/rc/keymaps/rc-manli.c20
-rw-r--r--drivers/media/rc/keymaps/rc-medion-x10-digitainer.c20
-rw-r--r--drivers/media/rc/keymaps/rc-medion-x10-or2x.c20
-rw-r--r--drivers/media/rc/keymaps/rc-medion-x10.c20
-rw-r--r--drivers/media/rc/keymaps/rc-msi-digivox-ii.c20
-rw-r--r--drivers/media/rc/keymaps/rc-msi-digivox-iii.c20
-rw-r--r--drivers/media/rc/keymaps/rc-msi-tvanywhere-plus.c20
-rw-r--r--drivers/media/rc/keymaps/rc-msi-tvanywhere.c20
-rw-r--r--drivers/media/rc/keymaps/rc-nebula.c20
-rw-r--r--drivers/media/rc/keymaps/rc-nec-terratec-cinergy-xs.c40
-rw-r--r--drivers/media/rc/keymaps/rc-norwood.c20
-rw-r--r--drivers/media/rc/keymaps/rc-npgtech.c20
-rw-r--r--drivers/media/rc/keymaps/rc-odroid.c54
-rw-r--r--drivers/media/rc/keymaps/rc-pctv-sedna.c20
-rw-r--r--drivers/media/rc/keymaps/rc-pinnacle-color.c20
-rw-r--r--drivers/media/rc/keymaps/rc-pinnacle-grey.c20
-rw-r--r--drivers/media/rc/keymaps/rc-pinnacle-pctv-hd.c20
-rw-r--r--drivers/media/rc/keymaps/rc-pixelview-002t.c20
-rw-r--r--drivers/media/rc/keymaps/rc-pixelview-mk12.c20
-rw-r--r--drivers/media/rc/keymaps/rc-pixelview-new.c20
-rw-r--r--drivers/media/rc/keymaps/rc-pixelview.c20
-rw-r--r--drivers/media/rc/keymaps/rc-powercolor-real-angel.c20
-rw-r--r--drivers/media/rc/keymaps/rc-proteus-2309.c20
-rw-r--r--drivers/media/rc/keymaps/rc-purpletv.c20
-rw-r--r--drivers/media/rc/keymaps/rc-pv951.c20
-rw-r--r--drivers/media/rc/keymaps/rc-real-audio-220-32-keys.c20
-rw-r--r--drivers/media/rc/keymaps/rc-reddo.c20
-rw-r--r--drivers/media/rc/keymaps/rc-snapstream-firefly.c20
-rw-r--r--drivers/media/rc/keymaps/rc-su3000.c20
-rw-r--r--drivers/media/rc/keymaps/rc-tango.c20
-rw-r--r--drivers/media/rc/keymaps/rc-tanix-tx3mini.c77
-rw-r--r--drivers/media/rc/keymaps/rc-tanix-tx5max.c68
-rw-r--r--drivers/media/rc/keymaps/rc-tbs-nec.c20
-rw-r--r--drivers/media/rc/keymaps/rc-technisat-ts35.c20
-rw-r--r--drivers/media/rc/keymaps/rc-technisat-usb2.c20
-rw-r--r--drivers/media/rc/keymaps/rc-terratec-cinergy-c-pci.c20
-rw-r--r--drivers/media/rc/keymaps/rc-terratec-cinergy-s2-hd.c20
-rw-r--r--drivers/media/rc/keymaps/rc-terratec-cinergy-xs.c20
-rw-r--r--drivers/media/rc/keymaps/rc-terratec-slim-2.c20
-rw-r--r--drivers/media/rc/keymaps/rc-terratec-slim.c20
-rw-r--r--drivers/media/rc/keymaps/rc-tevii-nec.c20
-rw-r--r--drivers/media/rc/keymaps/rc-total-media-in-hand-02.c20
-rw-r--r--drivers/media/rc/keymaps/rc-total-media-in-hand.c20
-rw-r--r--drivers/media/rc/keymaps/rc-trekstor.c20
-rw-r--r--drivers/media/rc/keymaps/rc-tt-1500.c20
-rw-r--r--drivers/media/rc/keymaps/rc-twinhan-dtv-cab-ci.c20
-rw-r--r--drivers/media/rc/keymaps/rc-twinhan1027.c20
-rw-r--r--drivers/media/rc/keymaps/rc-videomate-m1f.c20
-rw-r--r--drivers/media/rc/keymaps/rc-videomate-s350.c20
-rw-r--r--drivers/media/rc/keymaps/rc-videomate-tv-pvr.c20
-rw-r--r--drivers/media/rc/keymaps/rc-wetek-hub.c53
-rw-r--r--drivers/media/rc/keymaps/rc-wetek-play2.c93
-rw-r--r--drivers/media/rc/keymaps/rc-winfast-usbii-deluxe.c20
-rw-r--r--drivers/media/rc/keymaps/rc-winfast.c20
-rw-r--r--drivers/media/rc/keymaps/rc-x96max.c83
-rw-r--r--drivers/media/rc/keymaps/rc-xbox-dvd.c20
-rw-r--r--drivers/media/rc/keymaps/rc-zx-irdec.c20
-rw-r--r--drivers/media/rc/lirc_dev.c2
-rw-r--r--drivers/media/rc/mceusb.c405
-rw-r--r--drivers/media/rc/meson-ir.c10
-rw-r--r--drivers/media/rc/mtk-cir.c18
-rw-r--r--drivers/media/rc/rc-main.c6
-rw-r--r--drivers/media/rc/sunxi-cir.c90
-rw-r--r--drivers/media/spi/Kconfig5
-rw-r--r--drivers/media/tuners/Kconfig6
-rw-r--r--drivers/media/tuners/si2157.c6
-rw-r--r--drivers/media/tuners/si2157_priv.h3
-rw-r--r--drivers/media/tuners/tuner-xc2028.c3
-rw-r--r--drivers/media/tuners/xc4000.c3
-rw-r--r--drivers/media/usb/airspy/airspy.c9
-rw-r--r--drivers/media/usb/au0828/au0828-core.c12
-rw-r--r--drivers/media/usb/au0828/au0828-video.c27
-rw-r--r--drivers/media/usb/cpia2/cpia2_usb.c7
-rw-r--r--drivers/media/usb/cpia2/cpia2_v4l.c34
-rw-r--r--drivers/media/usb/cx231xx/cx231xx-417.c2
-rw-r--r--drivers/media/usb/cx231xx/cx231xx-cards.c6
-rw-r--r--drivers/media/usb/cx231xx/cx231xx-core.c2
-rw-r--r--drivers/media/usb/cx231xx/cx231xx-dvb.c1
-rw-r--r--drivers/media/usb/cx231xx/cx231xx-video.c32
-rw-r--r--drivers/media/usb/cx231xx/cx231xx.h1
-rw-r--r--drivers/media/usb/dvb-usb-v2/af9035.c2
-rw-r--r--drivers/media/usb/dvb-usb-v2/anysee.c2
-rw-r--r--drivers/media/usb/dvb-usb-v2/dvb_usb_urb.c15
-rw-r--r--drivers/media/usb/dvb-usb-v2/dvbsky.c37
-rw-r--r--drivers/media/usb/dvb-usb-v2/ec168.c2
-rw-r--r--drivers/media/usb/dvb-usb-v2/gl861.c2
-rw-r--r--drivers/media/usb/dvb-usb/Kconfig16
-rw-r--r--drivers/media/usb/dvb-usb/Makefile3
-rw-r--r--drivers/media/usb/dvb-usb/cxusb-analog.c1845
-rw-r--r--drivers/media/usb/dvb-usb/cxusb.c933
-rw-r--r--drivers/media/usb/dvb-usb/cxusb.h158
-rw-r--r--drivers/media/usb/dvb-usb/dib0700_devices.c8
-rw-r--r--drivers/media/usb/dvb-usb/dvb-usb-dvb.c5
-rw-r--r--drivers/media/usb/dvb-usb/dvb-usb-init.c20
-rw-r--r--drivers/media/usb/dvb-usb/dvb-usb.h10
-rw-r--r--drivers/media/usb/dvb-usb/pctv452e.c8
-rw-r--r--drivers/media/usb/dvb-usb/technisat-usb2.c22
-rw-r--r--drivers/media/usb/em28xx/em28xx-cards.c4
-rw-r--r--drivers/media/usb/em28xx/em28xx-core.c2
-rw-r--r--drivers/media/usb/em28xx/em28xx-input.c35
-rw-r--r--drivers/media/usb/em28xx/em28xx-video.c43
-rw-r--r--drivers/media/usb/em28xx/em28xx.h4
-rw-r--r--drivers/media/usb/go7007/go7007-v4l2.c26
-rw-r--r--drivers/media/usb/go7007/s2250-board.c6
-rw-r--r--drivers/media/usb/go7007/snd-go7007.c2
-rw-r--r--drivers/media/usb/gspca/gspca.c43
-rw-r--r--drivers/media/usb/gspca/konica.c5
-rw-r--r--drivers/media/usb/gspca/nw80x.c5
-rw-r--r--drivers/media/usb/gspca/ov519.c10
-rw-r--r--drivers/media/usb/gspca/ov534.c5
-rw-r--r--drivers/media/usb/gspca/ov534_9.c1
-rw-r--r--drivers/media/usb/gspca/se401.c5
-rw-r--r--drivers/media/usb/gspca/sn9c20x.c12
-rw-r--r--drivers/media/usb/gspca/sonixb.c5
-rw-r--r--drivers/media/usb/gspca/sonixj.c5
-rw-r--r--drivers/media/usb/gspca/spca1528.c5
-rw-r--r--drivers/media/usb/gspca/sq930x.c5
-rw-r--r--drivers/media/usb/gspca/sunplus.c5
-rw-r--r--drivers/media/usb/gspca/vc032x.c5
-rw-r--r--drivers/media/usb/gspca/w996Xcf.c5
-rw-r--r--drivers/media/usb/hackrf/hackrf.c14
-rw-r--r--drivers/media/usb/hdpvr/hdpvr-core.c13
-rw-r--r--drivers/media/usb/hdpvr/hdpvr-i2c.c6
-rw-r--r--drivers/media/usb/hdpvr/hdpvr-video.c25
-rw-r--r--drivers/media/usb/msi2500/msi2500.c13
-rw-r--r--drivers/media/usb/pvrusb2/Kconfig2
-rw-r--r--drivers/media/usb/pvrusb2/pvrusb2-cx2584x-v4l.c25
-rw-r--r--drivers/media/usb/pvrusb2/pvrusb2-devattr.c212
-rw-r--r--drivers/media/usb/pvrusb2/pvrusb2-devattr.h1
-rw-r--r--drivers/media/usb/pvrusb2/pvrusb2-dvb.c88
-rw-r--r--drivers/media/usb/pvrusb2/pvrusb2-dvb.h5
-rw-r--r--drivers/media/usb/pvrusb2/pvrusb2-eeprom.c3
-rw-r--r--drivers/media/usb/pvrusb2/pvrusb2-fx2-cmd.h4
-rw-r--r--drivers/media/usb/pvrusb2/pvrusb2-hdw.c59
-rw-r--r--drivers/media/usb/pvrusb2/pvrusb2-i2c-core.c6
-rw-r--r--drivers/media/usb/pvrusb2/pvrusb2-std.c2
-rw-r--r--drivers/media/usb/pvrusb2/pvrusb2-sysfs.c3
-rw-r--r--drivers/media/usb/pvrusb2/pvrusb2-v4l2.c21
-rw-r--r--drivers/media/usb/pwc/pwc-if.c2
-rw-r--r--drivers/media/usb/pwc/pwc-v4l.c8
-rw-r--r--drivers/media/usb/pwc/pwc.h18
-rw-r--r--drivers/media/usb/s2255/Kconfig1
-rw-r--r--drivers/media/usb/s2255/s2255drv.c15
-rw-r--r--drivers/media/usb/stk1160/stk1160-v4l.c9
-rw-r--r--drivers/media/usb/stk1160/stk1160.h1
-rw-r--r--drivers/media/usb/stkwebcam/stk-webcam.c11
-rw-r--r--drivers/media/usb/tm6000/tm6000-cards.c2
-rw-r--r--drivers/media/usb/tm6000/tm6000-dvb.c3
-rw-r--r--drivers/media/usb/tm6000/tm6000-video.c24
-rw-r--r--drivers/media/usb/tm6000/tm6000.h1
-rw-r--r--drivers/media/usb/ttusb-dec/ttusb_dec.c2
-rw-r--r--drivers/media/usb/usbtv/usbtv-video.c7
-rw-r--r--drivers/media/usb/usbvision/usbvision-video.c48
-rw-r--r--drivers/media/usb/usbvision/usbvision.h1
-rw-r--r--drivers/media/usb/uvc/uvc_ctrl.c4
-rw-r--r--drivers/media/usb/uvc/uvc_debugfs.c5
-rw-r--r--drivers/media/usb/uvc/uvc_v4l2.c2
-rw-r--r--drivers/media/usb/zr364xx/zr364xx.c21
-rw-r--r--drivers/media/v4l2-core/Kconfig7
-rw-r--r--drivers/media/v4l2-core/Makefile15
-rw-r--r--drivers/media/v4l2-core/v4l2-async.c28
-rw-r--r--drivers/media/v4l2-core/v4l2-common.c250
-rw-r--r--drivers/media/v4l2-core/v4l2-ctrls.c417
-rw-r--r--drivers/media/v4l2-core/v4l2-dev.c9
-rw-r--r--drivers/media/v4l2-core/v4l2-device.c39
-rw-r--r--drivers/media/v4l2-core/v4l2-fwnode.c41
-rw-r--r--drivers/media/v4l2-core/v4l2-i2c.c184
-rw-r--r--drivers/media/v4l2-core/v4l2-ioctl.c99
-rw-r--r--drivers/media/v4l2-core/v4l2-mem2mem.c76
-rw-r--r--drivers/media/v4l2-core/v4l2-spi.c78
-rw-r--r--drivers/media/v4l2-core/v4l2-subdev.c274
-rw-r--r--drivers/media/v4l2-core/videobuf-core.c5
-rw-r--r--drivers/media/v4l2-core/videobuf-dma-contig.c4
-rw-r--r--drivers/media/v4l2-core/videobuf-vmalloc.c2
-rw-r--r--drivers/memory/.gitignore1
-rw-r--r--drivers/memory/Kconfig10
-rw-r--r--drivers/memory/Makefile6
-rw-r--r--drivers/memory/brcmstb_dpfe.c317
-rw-r--r--drivers/memory/emif.c3
-rw-r--r--drivers/memory/jedec_ddr.h172
-rw-r--r--drivers/memory/jedec_ddr_data.c133
-rw-r--r--drivers/memory/jz4780-nemc.c28
-rw-r--r--drivers/memory/mtk-smi.c268
-rw-r--r--drivers/memory/of_memory.c3
-rw-r--r--drivers/memory/omap-gpmc.c4
-rw-r--r--drivers/memory/tegra/tegra124.c44
-rw-r--r--drivers/memory/ti-emif-sram-pm.S2
-rw-r--r--drivers/memstick/core/memstick.c13
-rw-r--r--drivers/memstick/core/ms_block.c2
-rw-r--r--drivers/memstick/host/r592.c6
-rw-r--r--drivers/message/fusion/mptbase.c3
-rw-r--r--drivers/mfd/Kconfig44
-rw-r--r--drivers/mfd/Makefile8
-rw-r--r--drivers/mfd/aat2870-core.c13
-rw-r--r--drivers/mfd/ab3100-core.c45
-rw-r--r--drivers/mfd/ab3100-otp.c21
-rw-r--r--drivers/mfd/ab8500-debugfs.c324
-rw-r--r--drivers/mfd/altera-sysmgr.c14
-rw-r--r--drivers/mfd/arizona-core.c2
-rw-r--r--drivers/mfd/cros_ec.c6
-rw-r--r--drivers/mfd/cros_ec_dev.c92
-rw-r--r--drivers/mfd/cs47l15-tables.c1299
-rw-r--r--drivers/mfd/cs47l35-tables.c60
-rw-r--r--drivers/mfd/cs47l85-tables.c128
-rw-r--r--drivers/mfd/cs47l90-tables.c82
-rw-r--r--drivers/mfd/cs47l92-tables.c1947
-rw-r--r--drivers/mfd/cs5535-mfd.c24
-rw-r--r--drivers/mfd/db8500-prcmu.c2
-rw-r--r--drivers/mfd/hi655x-pmic.c2
-rw-r--r--drivers/mfd/intel-lpss-pci.c21
-rw-r--r--drivers/mfd/intel-lpss.c1
-rw-r--r--drivers/mfd/lp87565.c4
-rw-r--r--drivers/mfd/madera-core.c129
-rw-r--r--drivers/mfd/madera-i2c.c24
-rw-r--r--drivers/mfd/madera-spi.c24
-rw-r--r--drivers/mfd/madera.h13
-rw-r--r--drivers/mfd/menelaus.c2
-rw-r--r--drivers/mfd/mfd-core.c1
-rw-r--r--drivers/mfd/omap-usb-host.c4
-rw-r--r--drivers/mfd/rk808.c257
-rw-r--r--drivers/mfd/rohm-bd70528.c316
-rw-r--r--drivers/mfd/rohm-bd718x7.c80
-rw-r--r--drivers/mfd/syscon.c21
-rw-r--r--drivers/mfd/ti-lmu.c23
-rw-r--r--drivers/misc/Kconfig59
-rw-r--r--drivers/misc/Makefile4
-rw-r--r--drivers/misc/altera-stapl/Kconfig1
-rw-r--r--drivers/misc/c2port/Kconfig2
-rw-r--r--drivers/misc/cardreader/alcor_pci.c6
-rw-r--r--drivers/misc/cb710/Kconfig1
-rw-r--r--drivers/misc/cxl/Kconfig3
-rw-r--r--drivers/misc/cxl/api.c13
-rw-r--r--drivers/misc/cxl/cxl.h15
-rw-r--r--drivers/misc/cxl/debugfs.c36
-rw-r--r--drivers/misc/echo/Kconfig1
-rw-r--r--drivers/misc/eeprom/Kconfig8
-rw-r--r--drivers/misc/eeprom/at24.c88
-rw-r--r--drivers/misc/eeprom/ee1004.c49
-rw-r--r--drivers/misc/eeprom/idt_89hpesx.c6
-rw-r--r--drivers/misc/eeprom/max6875.c6
-rw-r--r--drivers/misc/fastrpc.c79
-rw-r--r--drivers/misc/fsa9480.c547
-rw-r--r--drivers/misc/genwqe/Kconfig1
-rw-r--r--drivers/misc/genwqe/card_base.c5
-rw-r--r--drivers/misc/genwqe/card_base.h2
-rw-r--r--drivers/misc/genwqe/card_debugfs.c165
-rw-r--r--drivers/misc/genwqe/card_dev.c6
-rw-r--r--drivers/misc/habanalabs/asid.c2
-rw-r--r--drivers/misc/habanalabs/command_buffer.c3
-rw-r--r--drivers/misc/habanalabs/command_submission.c37
-rw-r--r--drivers/misc/habanalabs/context.c49
-rw-r--r--drivers/misc/habanalabs/debugfs.c68
-rw-r--r--drivers/misc/habanalabs/device.c660
-rw-r--r--drivers/misc/habanalabs/firmware_if.c71
-rw-r--r--drivers/misc/habanalabs/goya/goya.c803
-rw-r--r--drivers/misc/habanalabs/goya/goyaP.h37
-rw-r--r--drivers/misc/habanalabs/goya/goya_coresight.c89
-rw-r--r--drivers/misc/habanalabs/goya/goya_hwmgr.c109
-rw-r--r--drivers/misc/habanalabs/goya/goya_security.c16
-rw-r--r--drivers/misc/habanalabs/habanalabs.h227
-rw-r--r--drivers/misc/habanalabs/habanalabs_drv.c229
-rw-r--r--drivers/misc/habanalabs/habanalabs_ioctl.c191
-rw-r--r--drivers/misc/habanalabs/hw_queue.c34
-rw-r--r--drivers/misc/habanalabs/hwmon.c24
-rw-r--r--drivers/misc/habanalabs/include/armcp_if.h85
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_0_masks.h418
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/goya_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/goya.h2
-rw-r--r--drivers/misc/habanalabs/include/goya/goya_packets.h13
-rw-r--r--drivers/misc/habanalabs/include/goya/goya_reg_map.h34
-rw-r--r--drivers/misc/habanalabs/irq.c31
-rw-r--r--drivers/misc/habanalabs/memory.c15
-rw-r--r--drivers/misc/habanalabs/mmu.c20
-rw-r--r--drivers/misc/habanalabs/pci.c10
-rw-r--r--drivers/misc/habanalabs/sysfs.c130
-rw-r--r--drivers/misc/ibmasm/ibmasmfs.c21
-rw-r--r--drivers/misc/ioc4.c498
-rw-r--r--drivers/misc/isl29003.c4
-rw-r--r--drivers/misc/lis3lv02d/Kconfig2
-rw-r--r--drivers/misc/lkdtm/Makefile4
-rw-r--r--drivers/misc/lkdtm/bugs.c79
-rw-r--r--drivers/misc/lkdtm/cfi.c42
-rw-r--r--drivers/misc/lkdtm/core.c25
-rw-r--r--drivers/misc/lkdtm/heap.c72
-rw-r--r--drivers/misc/lkdtm/lkdtm.h10
-rw-r--r--drivers/misc/mei/debugfs.c223
-rw-r--r--drivers/misc/mei/hdcp/mei_hdcp.c13
-rw-r--r--drivers/misc/mei/hw-me-regs.h5
-rw-r--r--drivers/misc/mei/main.c17
-rw-r--r--drivers/misc/mei/mei_dev.h7
-rw-r--r--drivers/misc/mei/pci-me.c24
-rw-r--r--drivers/misc/mei/pci-txe.c19
-rw-r--r--drivers/misc/mic/card/mic_debugfs.c18
-rw-r--r--drivers/misc/mic/card/mic_x100.c28
-rw-r--r--drivers/misc/mic/cosm/cosm_debugfs.c4
-rw-r--r--drivers/misc/mic/host/mic_debugfs.c4
-rw-r--r--drivers/misc/mic/scif/scif_debugfs.c5
-rw-r--r--drivers/misc/mic/scif/scif_epd.h5
-rw-r--r--drivers/misc/mic/scif/scif_main.c1
-rw-r--r--drivers/misc/mic/vop/vop_debugfs.c4
-rw-r--r--drivers/misc/ocxl/Kconfig1
-rw-r--r--drivers/misc/ocxl/config.c181
-rw-r--r--drivers/misc/ocxl/context.c9
-rw-r--r--drivers/misc/ocxl/link.c28
-rw-r--r--drivers/misc/ocxl/pci.c2
-rw-r--r--drivers/misc/pci_endpoint_test.c2
-rw-r--r--drivers/misc/sgi-xp/Makefile13
-rw-r--r--drivers/misc/sgi-xp/xp.h19
-rw-r--r--drivers/misc/sgi-xp/xp_main.c8
-rw-r--r--drivers/misc/sgi-xp/xp_nofault.S35
-rw-r--r--drivers/misc/sgi-xp/xp_sn2.c190
-rw-r--r--drivers/misc/sgi-xp/xp_uv.c9
-rw-r--r--drivers/misc/sgi-xp/xpc.h273
-rw-r--r--drivers/misc/sgi-xp/xpc_main.c31
-rw-r--r--drivers/misc/sgi-xp/xpc_partition.c7
-rw-r--r--drivers/misc/sgi-xp/xpc_sn2.c2459
-rw-r--r--drivers/misc/sgi-xp/xpc_uv.c20
-rw-r--r--drivers/misc/sgi-xp/xpnet.c2
-rw-r--r--drivers/misc/spear13xx_pcie_gadget.c797
-rw-r--r--drivers/misc/ti-st/st_kim.c4
-rw-r--r--drivers/misc/tsl2550.c2
-rw-r--r--drivers/misc/vmw_balloon.c507
-rw-r--r--drivers/misc/vmw_vmci/vmci_context.c80
-rw-r--r--drivers/misc/vmw_vmci/vmci_doorbell.c6
-rw-r--r--drivers/misc/vmw_vmci/vmci_handle_array.c38
-rw-r--r--drivers/misc/vmw_vmci/vmci_handle_array.h29
-rw-r--r--drivers/misc/xilinx_sdfec.c1509
-rw-r--r--drivers/mmc/core/debugfs.c56
-rw-r--r--drivers/mmc/core/mmc_ops.c2
-rw-r--r--drivers/mmc/core/mmc_test.c10
-rw-r--r--drivers/mmc/core/queue.c12
-rw-r--r--drivers/mmc/core/sd.c6
-rw-r--r--drivers/mmc/core/sdio.c96
-rw-r--r--drivers/mmc/core/sdio_irq.c60
-rw-r--r--drivers/mmc/host/Kconfig14
-rw-r--r--drivers/mmc/host/Makefile1
-rw-r--r--drivers/mmc/host/alcor.c2
-rw-r--r--drivers/mmc/host/android-goldfish.c31
-rw-r--r--drivers/mmc/host/atmel-mci.c42
-rw-r--r--drivers/mmc/host/bcm2835.c5
-rw-r--r--drivers/mmc/host/cavium.c4
-rw-r--r--drivers/mmc/host/dw_mmc-hi3798cv200.c2
-rw-r--r--drivers/mmc/host/dw_mmc.c43
-rw-r--r--drivers/mmc/host/jz4740_mmc.c19
-rw-r--r--drivers/mmc/host/meson-gx-mmc.c71
-rw-r--r--drivers/mmc/host/meson-mx-sdio.c2
-rw-r--r--drivers/mmc/host/mmc_spi.c2
-rw-r--r--drivers/mmc/host/mmci.c69
-rw-r--r--drivers/mmc/host/mtk-sd.c29
-rw-r--r--drivers/mmc/host/mxcmmc.c4
-rw-r--r--drivers/mmc/host/mxs-mmc.c4
-rw-r--r--drivers/mmc/host/renesas_sdhi_core.c26
-rw-r--r--drivers/mmc/host/renesas_sdhi_sys_dmac.c36
-rw-r--r--drivers/mmc/host/s3cmci.c28
-rw-r--r--drivers/mmc/host/s3cmci.h2
-rw-r--r--drivers/mmc/host/sdhci-acpi.c2
-rw-r--r--drivers/mmc/host/sdhci-cadence.c8
-rw-r--r--drivers/mmc/host/sdhci-esdhc-imx.c36
-rw-r--r--drivers/mmc/host/sdhci-iproc.c9
-rw-r--r--drivers/mmc/host/sdhci-msm.c11
-rw-r--r--drivers/mmc/host/sdhci-of-arasan.c15
-rw-r--r--drivers/mmc/host/sdhci-of-aspeed.c342
-rw-r--r--drivers/mmc/host/sdhci-of-at91.c5
-rw-r--r--drivers/mmc/host/sdhci-of-esdhc.c18
-rw-r--r--drivers/mmc/host/sdhci-pci-core.c21
-rw-r--r--drivers/mmc/host/sdhci-pci-o2micro.c325
-rw-r--r--drivers/mmc/host/sdhci-pci.h3
-rw-r--r--drivers/mmc/host/sdhci-pltfm.c5
-rw-r--r--drivers/mmc/host/sdhci-pxav3.c2
-rw-r--r--drivers/mmc/host/sdhci-s3c.c7
-rw-r--r--drivers/mmc/host/sdhci-sprd.c282
-rw-r--r--drivers/mmc/host/sdhci-tegra.c19
-rw-r--r--drivers/mmc/host/sdhci-xenon.c2
-rw-r--r--drivers/mmc/host/sdhci.c80
-rw-r--r--drivers/mmc/host/sdhci.h16
-rw-r--r--drivers/mmc/host/sdhci_am654.c293
-rw-r--r--drivers/mmc/host/sdhci_f_sdh30.c4
-rw-r--r--drivers/mmc/host/tmio_mmc.h1
-rw-r--r--drivers/mmc/host/tmio_mmc_core.c34
-rw-r--r--drivers/mmc/host/uniphier-sd.c4
-rw-r--r--drivers/mtd/Kconfig2
-rw-r--r--drivers/mtd/Makefile1
-rw-r--r--drivers/mtd/chips/cfi_cmdset_0002.c143
-rw-r--r--drivers/mtd/devices/Kconfig2
-rw-r--r--drivers/mtd/hyperbus/Kconfig25
-rw-r--r--drivers/mtd/hyperbus/Makefile4
-rw-r--r--drivers/mtd/hyperbus/hbmc-am654.c147
-rw-r--r--drivers/mtd/hyperbus/hyperbus-core.c153
-rw-r--r--drivers/mtd/maps/sa1100-flash.c1
-rw-r--r--drivers/mtd/mtdconcat.c37
-rw-r--r--drivers/mtd/mtdcore.c3
-rw-r--r--drivers/mtd/mtdcore.h1
-rw-r--r--drivers/mtd/mtdsuper.c179
-rw-r--r--drivers/mtd/nand/onenand/onenand_base.c6
-rw-r--r--drivers/mtd/nand/raw/brcmnand/brcmnand.c263
-rw-r--r--drivers/mtd/nand/raw/fsmc_nand.c19
-rw-r--r--drivers/mtd/nand/raw/gpmi-nand/Makefile1
-rw-r--r--drivers/mtd/nand/raw/gpmi-nand/gpmi-lib.c934
-rw-r--r--drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c1709
-rw-r--r--drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.h64
-rw-r--r--drivers/mtd/nand/raw/mtk_ecc.c4
-rw-r--r--drivers/mtd/nand/raw/mtk_ecc.h2
-rw-r--r--drivers/mtd/nand/raw/mtk_nand.c88
-rw-r--r--drivers/mtd/nand/raw/nand_base.c80
-rw-r--r--drivers/mtd/nand/raw/nand_bch.c3
-rw-r--r--drivers/mtd/nand/raw/nand_ecc.c2
-rw-r--r--drivers/mtd/nand/raw/nand_macronix.c45
-rw-r--r--drivers/mtd/nand/raw/nand_micron.c14
-rw-r--r--drivers/mtd/nand/raw/stm32_fmc2_nand.c21
-rw-r--r--drivers/mtd/nand/spi/Makefile2
-rw-r--r--drivers/mtd/nand/spi/core.c5
-rw-r--r--drivers/mtd/nand/spi/gigadevice.c79
-rw-r--r--drivers/mtd/nand/spi/paragon.c147
-rw-r--r--drivers/mtd/parsers/afs.c3
-rw-r--r--drivers/mtd/spi-nor/Kconfig7
-rw-r--r--drivers/mtd/spi-nor/Makefile1
-rw-r--r--drivers/mtd/spi-nor/cadence-quadspi.c21
-rw-r--r--drivers/mtd/spi-nor/intel-spi-pci.c1
-rw-r--r--drivers/mtd/spi-nor/spi-nor.c42
-rw-r--r--drivers/mtd/spi-nor/stm32-quadspi.c707
-rw-r--r--drivers/mux/Kconfig12
-rw-r--r--drivers/mux/core.c7
-rw-r--r--drivers/mux/mmio.c6
-rw-r--r--drivers/net/arcnet/arc-rimi.c3
-rw-r--r--drivers/net/arcnet/capmode.c2
-rw-r--r--drivers/net/arcnet/com20020-isa.c6
-rw-r--r--drivers/net/arcnet/com90io.c2
-rw-r--r--drivers/net/arcnet/com90xx.c3
-rw-r--r--drivers/net/bonding/bond_3ad.c222
-rw-r--r--drivers/net/bonding/bond_alb.c30
-rw-r--r--drivers/net/bonding/bond_debugfs.c5
-rw-r--r--drivers/net/bonding/bond_main.c399
-rw-r--r--drivers/net/bonding/bond_netlink.c14
-rw-r--r--drivers/net/bonding/bond_options.c101
-rw-r--r--drivers/net/bonding/bond_procfs.c2
-rw-r--r--drivers/net/bonding/bond_sysfs.c13
-rw-r--r--drivers/net/caif/caif_hsi.c2
-rw-r--r--drivers/net/caif/caif_serial.c26
-rw-r--r--drivers/net/caif/caif_virtio.c6
-rw-r--r--drivers/net/can/Kconfig13
-rw-r--r--drivers/net/can/Makefile1
-rw-r--r--drivers/net/can/at91_can.c6
-rw-r--r--drivers/net/can/dev.c157
-rw-r--r--drivers/net/can/flexcan.c175
-rw-r--r--drivers/net/can/janz-ican3.c1
-rw-r--r--drivers/net/can/kvaser_pciefd.c1911
-rw-r--r--drivers/net/can/m_can/Kconfig22
-rw-r--r--drivers/net/can/m_can/Makefile2
-rw-r--r--drivers/net/can/m_can/m_can.c1079
-rw-r--r--drivers/net/can/m_can/m_can.h110
-rw-r--r--drivers/net/can/m_can/m_can_platform.c201
-rw-r--r--drivers/net/can/m_can/tcan4x5x.c505
-rw-r--r--drivers/net/can/peak_canfd/peak_pciefd_main.c2
-rw-r--r--drivers/net/can/rcar/rcar_can.c23
-rw-r--r--drivers/net/can/rcar/rcar_canfd.c11
-rw-r--r--drivers/net/can/sja1000/Kconfig79
-rw-r--r--drivers/net/can/sja1000/Makefile11
-rw-r--r--drivers/net/can/sja1000/f81601.c211
-rw-r--r--drivers/net/can/sja1000/peak_pci.c2
-rw-r--r--drivers/net/can/sja1000/peak_pcmcia.c2
-rw-r--r--drivers/net/can/slcan.c6
-rw-r--r--drivers/net/can/softing/softing_main.c4
-rw-r--r--drivers/net/can/spi/hi311x.c62
-rw-r--r--drivers/net/can/spi/mcp251x.c239
-rw-r--r--drivers/net/can/sun4i_can.c1
-rw-r--r--drivers/net/can/ti_hecc.c268
-rw-r--r--drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c3
-rw-r--r--drivers/net/can/usb/peak_usb/pcan_usb.c2
-rw-r--r--drivers/net/can/usb/peak_usb/pcan_usb_core.c10
-rw-r--r--drivers/net/can/usb/peak_usb/pcan_usb_fd.c2
-rw-r--r--drivers/net/can/usb/peak_usb/pcan_usb_pro.c2
-rw-r--r--drivers/net/can/vcan.c26
-rw-r--r--drivers/net/can/vxcan.c4
-rw-r--r--drivers/net/can/xilinx_can.c292
-rw-r--r--drivers/net/dsa/Kconfig24
-rw-r--r--drivers/net/dsa/Makefile4
-rw-r--r--drivers/net/dsa/b53/b53_common.c44
-rw-r--r--drivers/net/dsa/b53/b53_priv.h2
-rw-r--r--drivers/net/dsa/b53/b53_srab.c8
-rw-r--r--drivers/net/dsa/bcm_sf2.c17
-rw-r--r--drivers/net/dsa/lan9303-core.c6
-rw-r--r--drivers/net/dsa/lantiq_gswip.c16
-rw-r--r--drivers/net/dsa/microchip/Kconfig26
-rw-r--r--drivers/net/dsa/microchip/Makefile3
-rw-r--r--drivers/net/dsa/microchip/ksz8795.c1310
-rw-r--r--drivers/net/dsa/microchip/ksz8795_reg.h1004
-rw-r--r--drivers/net/dsa/microchip/ksz8795_spi.c103
-rw-r--r--drivers/net/dsa/microchip/ksz9477.c239
-rw-r--r--drivers/net/dsa/microchip/ksz9477_i2c.c101
-rw-r--r--drivers/net/dsa/microchip/ksz9477_spi.c117
-rw-r--r--drivers/net/dsa/microchip/ksz_common.c29
-rw-r--r--drivers/net/dsa/microchip/ksz_common.h323
-rw-r--r--drivers/net/dsa/microchip/ksz_priv.h172
-rw-r--r--drivers/net/dsa/microchip/ksz_spi.h69
-rw-r--r--drivers/net/dsa/mt7530.c413
-rw-r--r--drivers/net/dsa/mt7530.h65
-rw-r--r--drivers/net/dsa/mv88e6xxx/Makefile1
-rw-r--r--drivers/net/dsa/mv88e6xxx/chip.c1107
-rw-r--r--drivers/net/dsa/mv88e6xxx/chip.h92
-rw-r--r--drivers/net/dsa/mv88e6xxx/global1.c130
-rw-r--r--drivers/net/dsa/mv88e6xxx/global1.h64
-rw-r--r--drivers/net/dsa/mv88e6xxx/global1_atu.c24
-rw-r--r--drivers/net/dsa/mv88e6xxx/global1_vtu.c70
-rw-r--r--drivers/net/dsa/mv88e6xxx/global2.c118
-rw-r--r--drivers/net/dsa/mv88e6xxx/global2.h26
-rw-r--r--drivers/net/dsa/mv88e6xxx/global2_avb.c29
-rw-r--r--drivers/net/dsa/mv88e6xxx/global2_scratch.c3
-rw-r--r--drivers/net/dsa/mv88e6xxx/hwtstamp.c28
-rw-r--r--drivers/net/dsa/mv88e6xxx/phy.c4
-rw-r--r--drivers/net/dsa/mv88e6xxx/port.c286
-rw-r--r--drivers/net/dsa/mv88e6xxx/port.h63
-rw-r--r--drivers/net/dsa/mv88e6xxx/port_hidden.c70
-rw-r--r--drivers/net/dsa/mv88e6xxx/ptp.c127
-rw-r--r--drivers/net/dsa/mv88e6xxx/ptp.h6
-rw-r--r--drivers/net/dsa/mv88e6xxx/serdes.c473
-rw-r--r--drivers/net/dsa/mv88e6xxx/serdes.h100
-rw-r--r--drivers/net/dsa/mv88e6xxx/smi.c29
-rw-r--r--drivers/net/dsa/qca8k.c25
-rw-r--r--drivers/net/dsa/qca8k.h2
-rw-r--r--drivers/net/dsa/sja1105/Kconfig17
-rw-r--r--drivers/net/dsa/sja1105/Makefile8
-rw-r--r--drivers/net/dsa/sja1105/sja1105.h60
-rw-r--r--drivers/net/dsa/sja1105/sja1105_clocking.c100
-rw-r--r--drivers/net/dsa/sja1105/sja1105_dynamic_config.c318
-rw-r--r--drivers/net/dsa/sja1105/sja1105_dynamic_config.h11
-rw-r--r--drivers/net/dsa/sja1105/sja1105_main.c887
-rw-r--r--drivers/net/dsa/sja1105/sja1105_ptp.c392
-rw-r--r--drivers/net/dsa/sja1105/sja1105_ptp.h64
-rw-r--r--drivers/net/dsa/sja1105/sja1105_spi.c70
-rw-r--r--drivers/net/dsa/sja1105/sja1105_static_config.c255
-rw-r--r--drivers/net/dsa/sja1105/sja1105_static_config.h85
-rw-r--r--drivers/net/dsa/sja1105/sja1105_tas.c423
-rw-r--r--drivers/net/dsa/sja1105/sja1105_tas.h41
-rw-r--r--drivers/net/dsa/vitesse-vsc73xx-core.c (renamed from drivers/net/dsa/vitesse-vsc73xx.c)206
-rw-r--r--drivers/net/dsa/vitesse-vsc73xx-platform.c164
-rw-r--r--drivers/net/dsa/vitesse-vsc73xx-spi.c203
-rw-r--r--drivers/net/dsa/vitesse-vsc73xx.h29
-rw-r--r--drivers/net/ethernet/3com/3c59x.c8
-rw-r--r--drivers/net/ethernet/8390/Kconfig4
-rw-r--r--drivers/net/ethernet/Kconfig3
-rw-r--r--drivers/net/ethernet/Makefile3
-rw-r--r--drivers/net/ethernet/aeroflex/greth.c2
-rw-r--r--drivers/net/ethernet/agere/et131x.c8
-rw-r--r--drivers/net/ethernet/allwinner/sun4i-emac.c9
-rw-r--r--drivers/net/ethernet/amazon/ena/ena_admin_defs.h69
-rw-r--r--drivers/net/ethernet/amazon/ena/ena_com.c320
-rw-r--r--drivers/net/ethernet/amazon/ena/ena_com.h168
-rw-r--r--drivers/net/ethernet/amazon/ena/ena_eth_com.c54
-rw-r--r--drivers/net/ethernet/amazon/ena/ena_eth_com.h73
-rw-r--r--drivers/net/ethernet/amazon/ena/ena_ethtool.c124
-rw-r--r--drivers/net/ethernet/amazon/ena/ena_netdev.c475
-rw-r--r--drivers/net/ethernet/amazon/ena/ena_netdev.h48
-rw-r--r--drivers/net/ethernet/amd/Kconfig2
-rw-r--r--drivers/net/ethernet/amd/au1000_eth.c1
-rw-r--r--drivers/net/ethernet/amd/ni65.c6
-rw-r--r--drivers/net/ethernet/amd/xgbe/xgbe-debugfs.c107
-rw-r--r--drivers/net/ethernet/amd/xgbe/xgbe-desc.c2
-rw-r--r--drivers/net/ethernet/amd/xgbe/xgbe-drv.c2
-rw-r--r--drivers/net/ethernet/amd/xgbe/xgbe-main.c10
-rw-r--r--drivers/net/ethernet/amd/xgbe/xgbe-platform.c33
-rw-r--r--drivers/net/ethernet/apm/xgene-v2/main.c4
-rw-r--r--drivers/net/ethernet/apm/xgene/xgene_enet_hw.c10
-rw-r--r--drivers/net/ethernet/apm/xgene/xgene_enet_main.c8
-rw-r--r--drivers/net/ethernet/apm/xgene/xgene_enet_sgmac.c10
-rw-r--r--drivers/net/ethernet/apm/xgene/xgene_enet_xgmac.c10
-rw-r--r--drivers/net/ethernet/apple/Kconfig4
-rw-r--r--drivers/net/ethernet/apple/bmac.c4
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/aq_cfg.h7
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/aq_drvinfo.c2
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/aq_drvinfo.h2
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/aq_filters.c7
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/aq_filters.h2
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/aq_main.c38
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/aq_nic.c30
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/aq_nic.h2
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/aq_ring.c4
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/aq_ring.h9
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/aq_vec.c3
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.c2
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c62
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0_internal.h7
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.c16
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.h5
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh_internal.h18
-rw-r--r--drivers/net/ethernet/aquantia/atlantic/ver.h5
-rw-r--r--drivers/net/ethernet/atheros/Kconfig10
-rw-r--r--drivers/net/ethernet/atheros/Makefile1
-rw-r--r--drivers/net/ethernet/atheros/ag71xx.c1901
-rw-r--r--drivers/net/ethernet/atheros/alx/main.c10
-rw-r--r--drivers/net/ethernet/atheros/atl1c/atl1c_main.c14
-rw-r--r--drivers/net/ethernet/atheros/atl1e/atl1e_main.c3
-rw-r--r--drivers/net/ethernet/atheros/atlx/atl1.c13
-rw-r--r--drivers/net/ethernet/atheros/atlx/atl2.c1
-rw-r--r--drivers/net/ethernet/aurora/nb8800.c4
-rw-r--r--drivers/net/ethernet/broadcom/Kconfig8
-rw-r--r--drivers/net/ethernet/broadcom/bcm63xx_enet.c10
-rw-r--r--drivers/net/ethernet/broadcom/bcmsysport.c27
-rw-r--r--drivers/net/ethernet/broadcom/bcmsysport.h4
-rw-r--r--drivers/net/ethernet/broadcom/bgmac-platform.c4
-rw-r--r--drivers/net/ethernet/broadcom/bgmac.c2
-rw-r--r--drivers/net/ethernet/broadcom/bnx2.c6
-rw-r--r--drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c20
-rw-r--r--drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.h2
-rw-r--r--drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c4
-rw-r--r--drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c50
-rw-r--r--drivers/net/ethernet/broadcom/bnx2x/bnx2x_stats.h3
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt.c1671
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt.h212
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c8
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.c43
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c206
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.h1
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c9
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c217
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h247
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c181
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.h1
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c43
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt_tc.h10
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c7
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c29
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c144
-rw-r--r--drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h7
-rw-r--r--drivers/net/ethernet/broadcom/cnic.c5
-rw-r--r--drivers/net/ethernet/broadcom/genet/bcmgenet.c91
-rw-r--r--drivers/net/ethernet/broadcom/genet/bcmgenet.h4
-rw-r--r--drivers/net/ethernet/broadcom/tg3.c8
-rw-r--r--drivers/net/ethernet/brocade/bna/bnad.c2
-rw-r--r--drivers/net/ethernet/cadence/Kconfig10
-rw-r--r--drivers/net/ethernet/cadence/macb.h12
-rw-r--r--drivers/net/ethernet/cadence/macb_main.c143
-rw-r--r--drivers/net/ethernet/cadence/macb_ptp.c7
-rw-r--r--drivers/net/ethernet/calxeda/xgmac.c6
-rw-r--r--drivers/net/ethernet/cavium/common/cavium_ptp.c2
-rw-r--r--drivers/net/ethernet/cavium/liquidio/lio_main.c23
-rw-r--r--drivers/net/ethernet/cavium/liquidio/lio_vf_main.c23
-rw-r--r--drivers/net/ethernet/cavium/liquidio/octeon_mailbox.c4
-rw-r--r--drivers/net/ethernet/cavium/liquidio/request_manager.c10
-rw-r--r--drivers/net/ethernet/cavium/thunder/nicvf_queues.c6
-rw-r--r--drivers/net/ethernet/cavium/thunder/thunder_bgx.c18
-rw-r--r--drivers/net/ethernet/chelsio/cxgb/my3126.c4
-rw-r--r--drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c5
-rw-r--r--drivers/net/ethernet/chelsio/cxgb3/sge.c4
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/Makefile2
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c19
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/cxgb4.h62
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c9
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c49
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.h2
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c249
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/cxgb4_mps.c241
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c25
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.h6
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c21
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h2
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/sched.c1
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/smt.c18
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/smt.h2
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/t4_hw.c118
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/t4_regs.h4
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h28
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c21
-rw-r--r--drivers/net/ethernet/chelsio/libcxgb/libcxgb_ppm.c47
-rw-r--r--drivers/net/ethernet/chelsio/libcxgb/libcxgb_ppm.h7
-rw-r--r--drivers/net/ethernet/cirrus/cs89x0.c7
-rw-r--r--drivers/net/ethernet/cortina/gemini.c9
-rw-r--r--drivers/net/ethernet/davicom/dm9000.c2
-rw-r--r--drivers/net/ethernet/emulex/benet/be.h2
-rw-r--r--drivers/net/ethernet/emulex/benet/be_cmds.c10
-rw-r--r--drivers/net/ethernet/emulex/benet/be_ethtool.c7
-rw-r--r--drivers/net/ethernet/emulex/benet/be_main.c34
-rw-r--r--drivers/net/ethernet/ezchip/nps_enet.c4
-rw-r--r--drivers/net/ethernet/ezchip/nps_enet.h4
-rw-r--r--drivers/net/ethernet/faraday/Kconfig1
-rw-r--r--drivers/net/ethernet/faraday/ftgmac100.c41
-rw-r--r--drivers/net/ethernet/freescale/dpaa/dpaa_eth.c8
-rw-r--r--drivers/net/ethernet/freescale/dpaa/dpaa_eth.h3
-rw-r--r--drivers/net/ethernet/freescale/dpaa2/Kconfig3
-rw-r--r--drivers/net/ethernet/freescale/dpaa2/dpaa2-eth-debugfs.c54
-rw-r--r--drivers/net/ethernet/freescale/dpaa2/dpaa2-eth-debugfs.h3
-rw-r--r--drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c264
-rw-r--r--drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.h16
-rw-r--r--drivers/net/ethernet/freescale/dpaa2/dpaa2-ethtool.c134
-rw-r--r--drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c242
-rw-r--r--drivers/net/ethernet/freescale/dpaa2/dpni-cmd.h3
-rw-r--r--drivers/net/ethernet/freescale/dpaa2/dpni.c42
-rw-r--r--drivers/net/ethernet/freescale/dpaa2/dpni.h45
-rw-r--r--drivers/net/ethernet/freescale/dpaa2/dprtc-cmd.h48
-rw-r--r--drivers/net/ethernet/freescale/dpaa2/dprtc.c191
-rw-r--r--drivers/net/ethernet/freescale/dpaa2/dprtc.h62
-rw-r--r--drivers/net/ethernet/freescale/enetc/Kconfig21
-rw-r--r--drivers/net/ethernet/freescale/enetc/Makefile19
-rw-r--r--drivers/net/ethernet/freescale/enetc/enetc.c218
-rw-r--r--drivers/net/ethernet/freescale/enetc/enetc.h18
-rw-r--r--drivers/net/ethernet/freescale/enetc/enetc_ethtool.c31
-rw-r--r--drivers/net/ethernet/freescale/enetc/enetc_hw.h25
-rw-r--r--drivers/net/ethernet/freescale/enetc/enetc_mdio.c97
-rw-r--r--drivers/net/ethernet/freescale/enetc/enetc_mdio.h12
-rw-r--r--drivers/net/ethernet/freescale/enetc/enetc_pci_mdio.c101
-rw-r--r--drivers/net/ethernet/freescale/enetc/enetc_pf.c7
-rw-r--r--drivers/net/ethernet/freescale/enetc/enetc_ptp.c10
-rw-r--r--drivers/net/ethernet/freescale/enetc/enetc_vf.c2
-rw-r--r--drivers/net/ethernet/freescale/fec_main.c96
-rw-r--r--drivers/net/ethernet/freescale/fec_ptp.c2
-rw-r--r--drivers/net/ethernet/freescale/fman/fman.c3
-rw-r--r--drivers/net/ethernet/freescale/fman/fman_keygen.c3
-rw-r--r--drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c2
-rw-r--r--drivers/net/ethernet/freescale/gianfar.c2762
-rw-r--r--drivers/net/ethernet/freescale/gianfar.h45
-rw-r--r--drivers/net/ethernet/freescale/gianfar_ethtool.c13
-rw-r--r--drivers/net/ethernet/google/Kconfig27
-rw-r--r--drivers/net/ethernet/google/Makefile5
-rw-r--r--drivers/net/ethernet/google/gve/Makefile4
-rw-r--r--drivers/net/ethernet/google/gve/gve.h457
-rw-r--r--drivers/net/ethernet/google/gve/gve_adminq.c387
-rw-r--r--drivers/net/ethernet/google/gve/gve_adminq.h217
-rw-r--r--drivers/net/ethernet/google/gve/gve_desc.h113
-rw-r--r--drivers/net/ethernet/google/gve/gve_ethtool.c245
-rw-r--r--drivers/net/ethernet/google/gve/gve_main.c1231
-rw-r--r--drivers/net/ethernet/google/gve/gve_register.h27
-rw-r--r--drivers/net/ethernet/google/gve/gve_rx.c442
-rw-r--r--drivers/net/ethernet/google/gve/gve_tx.c584
-rw-r--r--drivers/net/ethernet/hisilicon/Kconfig10
-rw-r--r--drivers/net/ethernet/hisilicon/hip04_eth.c173
-rw-r--r--drivers/net/ethernet/hisilicon/hisi_femac.c8
-rw-r--r--drivers/net/ethernet/hisilicon/hix5hd2_gmac.c9
-rw-r--r--drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c2
-rw-r--r--drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c8
-rw-r--r--drivers/net/ethernet/hisilicon/hns/hns_enet.c7
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h6
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hnae3.c51
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hnae3.h72
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3_dcbnl.c12
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c57
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3_enet.c864
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3_enet.h63
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c226
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.c111
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h103
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_dcb.c65
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c523
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.h19
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.c890
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.h22
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c2418
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h135
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c81
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c17
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c195
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h3
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3vf/Makefile2
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.c77
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.h21
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c457
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h16
-rw-r--r--drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_mbx.c17
-rw-r--r--drivers/net/ethernet/hisilicon/hns_mdio.c4
-rw-r--r--drivers/net/ethernet/huawei/hinic/Makefile2
-rw-r--r--drivers/net/ethernet/huawei/hinic/hinic_dev.h28
-rw-r--r--drivers/net/ethernet/huawei/hinic/hinic_ethtool.c762
-rw-r--r--drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c12
-rw-r--r--drivers/net/ethernet/huawei/hinic/hinic_hw_dev.h56
-rw-r--r--drivers/net/ethernet/huawei/hinic/hinic_hw_io.c60
-rw-r--r--drivers/net/ethernet/huawei/hinic/hinic_hw_qp_ctxt.h5
-rw-r--r--drivers/net/ethernet/huawei/hinic/hinic_hw_wqe.h53
-rw-r--r--drivers/net/ethernet/huawei/hinic/hinic_main.c339
-rw-r--r--drivers/net/ethernet/huawei/hinic/hinic_port.c638
-rw-r--r--drivers/net/ethernet/huawei/hinic/hinic_port.h371
-rw-r--r--drivers/net/ethernet/huawei/hinic/hinic_rx.c82
-rw-r--r--drivers/net/ethernet/huawei/hinic/hinic_rx.h7
-rw-r--r--drivers/net/ethernet/huawei/hinic/hinic_tx.c27
-rw-r--r--drivers/net/ethernet/huawei/hinic/hinic_tx.h1
-rw-r--r--drivers/net/ethernet/ibm/ehea/ehea_main.c14
-rw-r--r--drivers/net/ethernet/ibm/emac/core.c2
-rw-r--r--drivers/net/ethernet/ibm/ibmveth.c9
-rw-r--r--drivers/net/ethernet/ibm/ibmvnic.c20
-rw-r--r--drivers/net/ethernet/intel/e1000/e1000_ethtool.c3
-rw-r--r--drivers/net/ethernet/intel/e1000/e1000_main.c12
-rw-r--r--drivers/net/ethernet/intel/e1000e/80003es2lan.c2
-rw-r--r--drivers/net/ethernet/intel/e1000e/82571.c2
-rw-r--r--drivers/net/ethernet/intel/e1000e/defines.h3
-rw-r--r--drivers/net/ethernet/intel/e1000e/e1000.h5
-rw-r--r--drivers/net/ethernet/intel/e1000e/ethtool.c20
-rw-r--r--drivers/net/ethernet/intel/e1000e/ich8lan.c30
-rw-r--r--drivers/net/ethernet/intel/e1000e/ich8lan.h2
-rw-r--r--drivers/net/ethernet/intel/e1000e/mac.c2
-rw-r--r--drivers/net/ethernet/intel/e1000e/netdev.c121
-rw-r--r--drivers/net/ethernet/intel/e1000e/nvm.c2
-rw-r--r--drivers/net/ethernet/intel/fm10k/fm10k.h10
-rw-r--r--drivers/net/ethernet/intel/fm10k/fm10k_dcbnl.c6
-rw-r--r--drivers/net/ethernet/intel/fm10k/fm10k_debugfs.c2
-rw-r--r--drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c15
-rw-r--r--drivers/net/ethernet/intel/fm10k/fm10k_iov.c5
-rw-r--r--drivers/net/ethernet/intel/fm10k/fm10k_main.c24
-rw-r--r--drivers/net/ethernet/intel/fm10k/fm10k_mbx.c11
-rw-r--r--drivers/net/ethernet/intel/fm10k/fm10k_netdev.c23
-rw-r--r--drivers/net/ethernet/intel/fm10k/fm10k_pci.c24
-rw-r--r--drivers/net/ethernet/intel/fm10k/fm10k_pf.c15
-rw-r--r--drivers/net/ethernet/intel/fm10k/fm10k_tlv.c9
-rw-r--r--drivers/net/ethernet/intel/fm10k/fm10k_type.h2
-rw-r--r--drivers/net/ethernet/intel/fm10k/fm10k_vf.c25
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e.h44
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_adminq.c12
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h38
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_common.c154
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_dcb.c18
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_dcb.h2
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_debugfs.c35
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_ethtool.c166
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_hmc.c1
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_lan_hmc.c21
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_main.c771
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_nvm.c101
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_osdep.h5
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_prototype.h12
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_ptp.c5
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_register.h30
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_txrx.c8
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_txrx.h2
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_type.h3
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c182
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h1
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_xsk.c70
-rw-r--r--drivers/net/ethernet/intel/i40e/i40e_xsk.h2
-rw-r--r--drivers/net/ethernet/intel/iavf/Makefile2
-rw-r--r--drivers/net/ethernet/intel/iavf/i40e_adminq_cmd.h530
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf.h14
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_adminq.c (renamed from drivers/net/ethernet/intel/iavf/i40e_adminq.c)267
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_adminq.h (renamed from drivers/net/ethernet/intel/iavf/i40e_adminq.h)80
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_adminq_cmd.h528
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_alloc.h17
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_client.c127
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_client.h104
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_common.c499
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_ethtool.c16
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_main.c893
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_osdep.h11
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_prototype.h58
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_status.h136
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_trace.h4
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_txrx.c47
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_txrx.h2
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_type.h4
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_virtchnl.c84
-rw-r--r--drivers/net/ethernet/intel/ice/Makefile1
-rw-r--r--drivers/net/ethernet/intel/ice/ice.h127
-rw-r--r--drivers/net/ethernet/intel/ice/ice_adminq_cmd.h126
-rw-r--r--drivers/net/ethernet/intel/ice/ice_common.c571
-rw-r--r--drivers/net/ethernet/intel/ice/ice_common.h29
-rw-r--r--drivers/net/ethernet/intel/ice/ice_controlq.c114
-rw-r--r--drivers/net/ethernet/intel/ice/ice_controlq.h2
-rw-r--r--drivers/net/ethernet/intel/ice/ice_dcb.c85
-rw-r--r--drivers/net/ethernet/intel/ice/ice_dcb.h23
-rw-r--r--drivers/net/ethernet/intel/ice/ice_dcb_lib.c377
-rw-r--r--drivers/net/ethernet/intel/ice/ice_dcb_lib.h7
-rw-r--r--drivers/net/ethernet/intel/ice/ice_ethtool.c1285
-rw-r--r--drivers/net/ethernet/intel/ice/ice_flex_pipe.c1549
-rw-r--r--drivers/net/ethernet/intel/ice/ice_flex_pipe.h29
-rw-r--r--drivers/net/ethernet/intel/ice/ice_flex_type.h374
-rw-r--r--drivers/net/ethernet/intel/ice/ice_hw_autogen.h40
-rw-r--r--drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h2
-rw-r--r--drivers/net/ethernet/intel/ice/ice_lib.c1126
-rw-r--r--drivers/net/ethernet/intel/ice/ice_lib.h49
-rw-r--r--drivers/net/ethernet/intel/ice/ice_main.c1678
-rw-r--r--drivers/net/ethernet/intel/ice/ice_nvm.c35
-rw-r--r--drivers/net/ethernet/intel/ice/ice_sched.c63
-rw-r--r--drivers/net/ethernet/intel/ice/ice_status.h1
-rw-r--r--drivers/net/ethernet/intel/ice/ice_switch.c70
-rw-r--r--drivers/net/ethernet/intel/ice/ice_switch.h7
-rw-r--r--drivers/net/ethernet/intel/ice/ice_txrx.c220
-rw-r--r--drivers/net/ethernet/intel/ice/ice_txrx.h48
-rw-r--r--drivers/net/ethernet/intel/ice/ice_type.h64
-rw-r--r--drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c848
-rw-r--r--drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h72
-rw-r--r--drivers/net/ethernet/intel/igb/e1000_82575.c2
-rw-r--r--drivers/net/ethernet/intel/igb/e1000_regs.h2
-rw-r--r--drivers/net/ethernet/intel/igb/igb_ethtool.c75
-rw-r--r--drivers/net/ethernet/intel/igb/igb_main.c58
-rw-r--r--drivers/net/ethernet/intel/igbvf/netdev.c2
-rw-r--r--drivers/net/ethernet/intel/igc/igc.h4
-rw-r--r--drivers/net/ethernet/intel/igc/igc_base.c54
-rw-r--r--drivers/net/ethernet/intel/igc/igc_base.h8
-rw-r--r--drivers/net/ethernet/intel/igc/igc_defines.h29
-rw-r--r--drivers/net/ethernet/intel/igc/igc_hw.h17
-rw-r--r--drivers/net/ethernet/intel/igc/igc_mac.c96
-rw-r--r--drivers/net/ethernet/intel/igc/igc_main.c151
-rw-r--r--drivers/net/ethernet/intel/igc/igc_phy.c192
-rw-r--r--drivers/net/ethernet/intel/ixgb/ixgb_main.c4
-rw-r--r--drivers/net/ethernet/intel/ixgbe/ixgbe.h14
-rw-r--r--drivers/net/ethernet/intel/ixgbe/ixgbe_debugfs.c22
-rw-r--r--drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c3
-rw-r--r--drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c12
-rw-r--r--drivers/net/ethernet/intel/ixgbe/ixgbe_main.c88
-rw-r--r--drivers/net/ethernet/intel/ixgbe/ixgbe_phy.h1
-rw-r--r--drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c181
-rw-r--r--drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c2
-rw-r--r--drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h2
-rw-r--r--drivers/net/ethernet/intel/ixgbe/ixgbe_type.h14
-rw-r--r--drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c174
-rw-r--r--drivers/net/ethernet/intel/ixgbevf/ethtool.c10
-rw-r--r--drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c30
-rw-r--r--drivers/net/ethernet/intel/ixgbevf/vf.c5
-rw-r--r--drivers/net/ethernet/jme.c20
-rw-r--r--drivers/net/ethernet/lantiq_xrx200.c10
-rw-r--r--drivers/net/ethernet/marvell/mv643xx_eth.c2
-rw-r--r--drivers/net/ethernet/marvell/mvmdio.c34
-rw-r--r--drivers/net/ethernet/marvell/mvneta.c46
-rw-r--r--drivers/net/ethernet/marvell/mvneta_bm.c8
-rw-r--r--drivers/net/ethernet/marvell/mvpp2/mvpp2.h46
-rw-r--r--drivers/net/ethernet/marvell/mvpp2/mvpp2_cls.c400
-rw-r--r--drivers/net/ethernet/marvell/mvpp2/mvpp2_cls.h43
-rw-r--r--drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c19
-rw-r--r--drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c670
-rw-r--r--drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.c3
-rw-r--r--drivers/net/ethernet/marvell/pxa168_eth.c3
-rw-r--r--drivers/net/ethernet/marvell/skge.c47
-rw-r--r--drivers/net/ethernet/marvell/sky2.c24
-rw-r--r--drivers/net/ethernet/mediatek/Kconfig5
-rw-r--r--drivers/net/ethernet/mediatek/Makefile3
-rw-r--r--drivers/net/ethernet/mediatek/mtk_eth_path.c289
-rw-r--r--drivers/net/ethernet/mediatek/mtk_eth_soc.c1091
-rw-r--r--drivers/net/ethernet/mediatek/mtk_eth_soc.h310
-rw-r--r--drivers/net/ethernet/mediatek/mtk_sgmii.c126
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/catas.c2
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/crdump.c7
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/en_ethtool.c2
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/en_netdev.c43
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/en_rx.c3
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/en_tx.c4
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/eq.c2
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/main.c112
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/mlx4.h3
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/Kconfig62
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/Makefile36
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec.c9
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec.h7
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/accel/tls.c45
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/accel/tls.h51
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/cmd.c59
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/cq.c21
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/debugfs.c102
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/dev.c11
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/devlink.c228
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/devlink.h14
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/diag/crdump.c115
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/diag/en_rep_tracepoint.h54
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/diag/en_tc_tracepoint.c58
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/diag/en_tc_tracepoint.h114
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.h4
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c146
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.h20
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/ecpf.c27
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/ecpf.h4
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en.h366
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/fs.h25
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/health.c205
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/health.h53
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/hv_vhca_stats.c161
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/hv_vhca_stats.h25
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/params.c123
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/params.h121
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/port.c27
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/port.h6
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/reporter.h15
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c404
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c273
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c351
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.h43
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_geneve.c335
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_gre.c95
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c151
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h226
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c269
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h103
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/xsk/Makefile1
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c193
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h41
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c242
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h25
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c118
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h27
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c267
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.h31
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h1
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h1
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c93
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h99
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c442
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c17
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h11
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c7
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h1
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c97
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_common.c12
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c4
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_dim.c14
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c213
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_fs.c50
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c32
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_main.c1052
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_rep.c632
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_rep.h28
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_rx.c226
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c2
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_stats.c347
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_stats.h51
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_tc.c1281
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_tc.h47
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_tx.c110
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c77
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/eq.c534
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/eswitch.c445
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/eswitch.h156
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c892
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c277
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/events.c4
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c8
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c8
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.h75
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c204
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h41
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/fs_core.c356
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/fs_core.h48
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c461
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/fw.c237
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/health.c577
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c49
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h2
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c10
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lag.c4
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lag_mp.c36
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c74
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c223
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h16
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lib/geneve.c157
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lib/geneve.h33
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lib/hv.c64
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lib/hv.h22
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lib/hv_vhca.c371
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lib/hv_vhca.h104
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h8
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.c33
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lib/pci_vsc.c316
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lib/pci_vsc.h32
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lib/port_tun.c23
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c9
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/main.c144
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h31
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/mr.c27
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c334
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/qp.c8
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/rdma.c16
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/sriov.c52
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/steering/Makefile2
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c1588
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c480
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/steering/dr_crc32.c98
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/steering/dr_domain.c393
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/steering/dr_fw.c93
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/steering/dr_icm_pool.c570
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c770
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c1243
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c975
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c2308
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/steering/dr_table.c294
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h1060
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c600
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.h60
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5_ifc_dr.h604
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h212
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/vport.c50
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/wq.c5
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/wq.h6
-rw-r--r--drivers/net/ethernet/mellanox/mlxfw/mlxfw.h11
-rw-r--r--drivers/net/ethernet/mellanox/mlxfw/mlxfw_fsm.c57
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/Kconfig8
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/Makefile3
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/cmd.h12
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/core.c214
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/core.h47
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c18
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h22
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/core_env.c27
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/core_hwmon.c143
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/core_thermal.c248
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/i2c.c76
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/minimal.c18
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/pci.c50
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/pci.h1
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/pci_hw.h3
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/reg.h535
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/spectrum.c875
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/spectrum.h77
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c28
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.c10
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c55
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/spectrum_dcb.c16
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c10
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c102
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/spectrum_nve.c1
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/spectrum_nve.h1
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/spectrum_nve_vxlan.c76
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c1167
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.h218
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c277
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c6
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/spectrum_trap.c267
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/switchx2.c8
-rw-r--r--drivers/net/ethernet/mellanox/mlxsw/trap.h13
-rw-r--r--drivers/net/ethernet/micrel/Kconfig11
-rw-r--r--drivers/net/ethernet/micrel/Makefile1
-rw-r--r--drivers/net/ethernet/micrel/ks8695net.c1632
-rw-r--r--drivers/net/ethernet/micrel/ks8695net.h108
-rw-r--r--drivers/net/ethernet/micrel/ks8842.c4
-rw-r--r--drivers/net/ethernet/micrel/ks8851_mll.c7
-rw-r--r--drivers/net/ethernet/micrel/ksz884x.c2
-rw-r--r--drivers/net/ethernet/microchip/lan743x_main.c8
-rw-r--r--drivers/net/ethernet/microchip/lan743x_ptp.c3
-rw-r--r--drivers/net/ethernet/mscc/Makefile2
-rw-r--r--drivers/net/ethernet/mscc/ocelot.c428
-rw-r--r--drivers/net/ethernet/mscc/ocelot.h60
-rw-r--r--drivers/net/ethernet/mscc/ocelot_ace.c782
-rw-r--r--drivers/net/ethernet/mscc/ocelot_ace.h232
-rw-r--r--drivers/net/ethernet/mscc/ocelot_board.c151
-rw-r--r--drivers/net/ethernet/mscc/ocelot_flower.c356
-rw-r--r--drivers/net/ethernet/mscc/ocelot_police.c227
-rw-r--r--drivers/net/ethernet/mscc/ocelot_police.h22
-rw-r--r--drivers/net/ethernet/mscc/ocelot_ptp.h41
-rw-r--r--drivers/net/ethernet/mscc/ocelot_regs.c22
-rw-r--r--drivers/net/ethernet/mscc/ocelot_s2.h64
-rw-r--r--drivers/net/ethernet/mscc/ocelot_tc.c197
-rw-r--r--drivers/net/ethernet/mscc/ocelot_tc.h22
-rw-r--r--drivers/net/ethernet/mscc/ocelot_vcap.h403
-rw-r--r--drivers/net/ethernet/myricom/myri10ge/myri10ge.c21
-rw-r--r--drivers/net/ethernet/natsemi/sonic.c6
-rw-r--r--drivers/net/ethernet/neterion/s2io.c1
-rw-r--r--drivers/net/ethernet/netronome/Kconfig1
-rw-r--r--drivers/net/ethernet/netronome/nfp/Makefile7
-rw-r--r--drivers/net/ethernet/netronome/nfp/abm/cls.c22
-rw-r--r--drivers/net/ethernet/netronome/nfp/abm/main.h2
-rw-r--r--drivers/net/ethernet/netronome/nfp/bpf/cmsg.c187
-rw-r--r--drivers/net/ethernet/netronome/nfp/bpf/fw.h1
-rw-r--r--drivers/net/ethernet/netronome/nfp/bpf/jit.c132
-rw-r--r--drivers/net/ethernet/netronome/nfp/bpf/main.c63
-rw-r--r--drivers/net/ethernet/netronome/nfp/bpf/main.h26
-rw-r--r--drivers/net/ethernet/netronome/nfp/bpf/offload.c3
-rw-r--r--drivers/net/ethernet/netronome/nfp/bpf/verifier.c12
-rw-r--r--drivers/net/ethernet/netronome/nfp/ccm.c3
-rw-r--r--drivers/net/ethernet/netronome/nfp/ccm.h60
-rw-r--r--drivers/net/ethernet/netronome/nfp/ccm_mbox.c743
-rw-r--r--drivers/net/ethernet/netronome/nfp/crypto/crypto.h27
-rw-r--r--drivers/net/ethernet/netronome/nfp/crypto/fw.h84
-rw-r--r--drivers/net/ethernet/netronome/nfp/crypto/tls.c522
-rw-r--r--drivers/net/ethernet/netronome/nfp/devlink_param.c255
-rw-r--r--drivers/net/ethernet/netronome/nfp/flower/action.c406
-rw-r--r--drivers/net/ethernet/netronome/nfp/flower/cmsg.c10
-rw-r--r--drivers/net/ethernet/netronome/nfp/flower/cmsg.h82
-rw-r--r--drivers/net/ethernet/netronome/nfp/flower/lag_conf.c4
-rw-r--r--drivers/net/ethernet/netronome/nfp/flower/main.c1
-rw-r--r--drivers/net/ethernet/netronome/nfp/flower/main.h37
-rw-r--r--drivers/net/ethernet/netronome/nfp/flower/match.c149
-rw-r--r--drivers/net/ethernet/netronome/nfp/flower/metadata.c30
-rw-r--r--drivers/net/ethernet/netronome/nfp/flower/offload.c557
-rw-r--r--drivers/net/ethernet/netronome/nfp/flower/qos_conf.c2
-rw-r--r--drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c211
-rw-r--r--drivers/net/ethernet/netronome/nfp/nfp_main.c148
-rw-r--r--drivers/net/ethernet/netronome/nfp/nfp_main.h5
-rw-r--r--drivers/net/ethernet/netronome/nfp/nfp_net.h75
-rw-r--r--drivers/net/ethernet/netronome/nfp/nfp_net_common.c227
-rw-r--r--drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.c15
-rw-r--r--drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h21
-rw-r--r--drivers/net/ethernet/netronome/nfp/nfp_net_debugfs.c17
-rw-r--r--drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c26
-rw-r--r--drivers/net/ethernet/netronome/nfp/nfp_net_main.c11
-rw-r--r--drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cpplib.c16
-rw-r--r--drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.c84
-rw-r--r--drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.h29
-rw-r--r--drivers/net/ethernet/ni/Kconfig2
-rw-r--r--drivers/net/ethernet/ni/nixge.c2
-rw-r--r--drivers/net/ethernet/nuvoton/Kconfig29
-rw-r--r--drivers/net/ethernet/nuvoton/Makefile6
-rw-r--r--drivers/net/ethernet/nuvoton/w90p910_ether.c1082
-rw-r--r--drivers/net/ethernet/nvidia/forcedeth.c146
-rw-r--r--drivers/net/ethernet/nxp/Kconfig2
-rw-r--r--drivers/net/ethernet/nxp/lpc_eth.c45
-rw-r--r--drivers/net/ethernet/packetengines/Kconfig6
-rw-r--r--drivers/net/ethernet/packetengines/Makefile2
-rw-r--r--drivers/net/ethernet/packetengines/yellowfin.c3
-rw-r--r--drivers/net/ethernet/pasemi/pasemi_mac.c2
-rw-r--r--drivers/net/ethernet/pensando/Kconfig32
-rw-r--r--drivers/net/ethernet/pensando/Makefile6
-rw-r--r--drivers/net/ethernet/pensando/ionic/Makefile8
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic.h73
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_bus.h16
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c292
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_debugfs.c248
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_debugfs.h34
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_dev.c500
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_dev.h299
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_devlink.c99
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_devlink.h14
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_ethtool.c779
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_ethtool.h9
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_if.h2482
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_lif.c2274
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_lif.h277
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_main.c548
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_regs.h136
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c150
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_rx_filter.h35
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_stats.c310
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_stats.h53
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_txrx.c925
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_txrx.h15
-rw-r--r--drivers/net/ethernet/qlogic/Kconfig10
-rw-r--r--drivers/net/ethernet/qlogic/Makefile1
-rw-r--r--drivers/net/ethernet/qlogic/netxen/netxen_nic_ctx.c3
-rw-r--r--drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c12
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed.h24
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_cxt.c5
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_debug.c84
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_dev.c1276
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_dev_api.h113
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_fcoe.c26
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_hsi.h48
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_hw.c44
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_init_ops.c9
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_int.c10
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_iscsi.c35
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_iwarp.c69
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_iwarp.h4
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_l2.c38
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_ll2.c406
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_main.c310
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_mcp.c126
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_mcp.h51
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_ptp.c11
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_rdma.c82
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_reg_addr.h6
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_sp_commands.c2
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_sriov.c3
-rw-r--r--drivers/net/ethernet/qlogic/qed/qed_vf.c2
-rw-r--r--drivers/net/ethernet/qlogic/qede/qede.h19
-rw-r--r--drivers/net/ethernet/qlogic/qede/qede_ethtool.c120
-rw-r--r--drivers/net/ethernet/qlogic/qede/qede_filter.c2
-rw-r--r--drivers/net/ethernet/qlogic/qede/qede_main.c42
-rw-r--r--drivers/net/ethernet/qlogic/qede/qede_ptp.c37
-rw-r--r--drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c2
-rw-r--r--drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c5
-rw-r--r--drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_pf.c2
-rw-r--r--drivers/net/ethernet/qualcomm/emac/emac-mac.c12
-rw-r--r--drivers/net/ethernet/qualcomm/emac/emac.c12
-rw-r--r--drivers/net/ethernet/qualcomm/qca_debug.c13
-rw-r--r--drivers/net/ethernet/qualcomm/qca_spi.c3
-rw-r--r--drivers/net/ethernet/qualcomm/qca_uart.c3
-rw-r--r--drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h25
-rw-r--r--drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c13
-rw-r--r--drivers/net/ethernet/realtek/Kconfig9
-rw-r--r--drivers/net/ethernet/realtek/Makefile1
-rw-r--r--drivers/net/ethernet/realtek/r8169_firmware.c231
-rw-r--r--drivers/net/ethernet/realtek/r8169_firmware.h39
-rw-r--r--drivers/net/ethernet/realtek/r8169_main.c (renamed from drivers/net/ethernet/realtek/r8169.c)2373
-rw-r--r--drivers/net/ethernet/renesas/ravb.h9
-rw-r--r--drivers/net/ethernet/renesas/ravb_main.c29
-rw-r--r--drivers/net/ethernet/rocker/rocker_main.c9
-rw-r--r--drivers/net/ethernet/rocker/rocker_ofdpa.c25
-rw-r--r--drivers/net/ethernet/samsung/Kconfig2
-rw-r--r--drivers/net/ethernet/samsung/sxgbe/sxgbe_platform.c4
-rw-r--r--drivers/net/ethernet/seeq/sgiseeq.c7
-rw-r--r--drivers/net/ethernet/sfc/ef10.c4
-rw-r--r--drivers/net/ethernet/sfc/efx.c16
-rw-r--r--drivers/net/ethernet/sfc/falcon/efx.c6
-rw-r--r--drivers/net/ethernet/sfc/falcon/falcon_boards.c18
-rw-r--r--drivers/net/ethernet/sfc/falcon/rx.c5
-rw-r--r--drivers/net/ethernet/sfc/rx.c5
-rw-r--r--drivers/net/ethernet/sfc/tx.c2
-rw-r--r--drivers/net/ethernet/sgi/ioc3-eth.c1038
-rw-r--r--drivers/net/ethernet/sgi/meth.c3
-rw-r--r--drivers/net/ethernet/sis/sis900.c98
-rw-r--r--drivers/net/ethernet/smsc/Kconfig6
-rw-r--r--drivers/net/ethernet/smsc/smc911x.c1
-rw-r--r--drivers/net/ethernet/smsc/smc91x.c3
-rw-r--r--drivers/net/ethernet/socionext/Kconfig1
-rw-r--r--drivers/net/ethernet/socionext/netsec.c577
-rw-r--r--drivers/net/ethernet/socionext/sni_ave.c8
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/Kconfig17
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/Makefile2
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/common.h38
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/descs.h2
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac-anarion.c4
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c14
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c8
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac-meson.c4
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c4
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c6
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c127
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c53
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac1000.h1
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c22
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c8
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c13
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c8
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac4.h33
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c175
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c56
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.h9
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c14
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c4
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h192
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c1104
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c118
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c90
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/hwif.c11
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/hwif.h91
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/mmc.h13
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/mmc_core.c205
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/stmmac.h76
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c220
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/stmmac_main.c1274
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c108
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c245
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c118
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c3
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c1871
-rw-r--r--drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c251
-rw-r--r--drivers/net/ethernet/sun/cassini.c8
-rw-r--r--drivers/net/ethernet/sun/niu.c4
-rw-r--r--drivers/net/ethernet/sun/sunvnet_common.c7
-rw-r--r--drivers/net/ethernet/synopsys/dwc-xlgmac-desc.c2
-rw-r--r--drivers/net/ethernet/synopsys/dwc-xlgmac-net.c2
-rw-r--r--drivers/net/ethernet/tehuti/tehuti.c2
-rw-r--r--drivers/net/ethernet/ti/Kconfig2
-rw-r--r--drivers/net/ethernet/ti/cpsw-phy-sel.c4
-rw-r--r--drivers/net/ethernet/ti/cpsw.c592
-rw-r--r--drivers/net/ethernet/ti/cpsw_ethtool.c97
-rw-r--r--drivers/net/ethernet/ti/cpsw_priv.h8
-rw-r--r--drivers/net/ethernet/ti/cpts.c88
-rw-r--r--drivers/net/ethernet/ti/cpts.h2
-rw-r--r--drivers/net/ethernet/ti/davinci_cpdma.c187
-rw-r--r--drivers/net/ethernet/ti/davinci_cpdma.h9
-rw-r--r--drivers/net/ethernet/ti/davinci_emac.c6
-rw-r--r--drivers/net/ethernet/ti/netcp_core.c2
-rw-r--r--drivers/net/ethernet/ti/netcp_ethss.c9
-rw-r--r--drivers/net/ethernet/ti/tlan.c1
-rw-r--r--drivers/net/ethernet/toshiba/ps3_gelic_net.h2
-rw-r--r--drivers/net/ethernet/toshiba/spider_net.c7
-rw-r--r--drivers/net/ethernet/toshiba/tc35815.c6
-rw-r--r--drivers/net/ethernet/tundra/tsi108_eth.c5
-rw-r--r--drivers/net/ethernet/via/via-rhine.c4
-rw-r--r--drivers/net/ethernet/via/via-velocity.h2
-rw-r--r--drivers/net/ethernet/wiznet/w5100-spi.c24
-rw-r--r--drivers/net/ethernet/xilinx/Kconfig6
-rw-r--r--drivers/net/ethernet/xilinx/ll_temac.h5
-rw-r--r--drivers/net/ethernet/xilinx/ll_temac_main.c258
-rw-r--r--drivers/net/ethernet/xilinx/ll_temac_mdio.c20
-rw-r--r--drivers/net/ethernet/xilinx/xilinx_axienet.h35
-rw-r--r--drivers/net/ethernet/xilinx/xilinx_axienet_main.c678
-rw-r--r--drivers/net/ethernet/xilinx/xilinx_axienet_mdio.c111
-rw-r--r--drivers/net/ethernet/xscale/Kconfig2
-rw-r--r--drivers/net/fddi/defza.c1
-rw-r--r--drivers/net/fddi/skfp/drvfbi.c3
-rw-r--r--drivers/net/fddi/skfp/h/skfbi.h231
-rw-r--r--drivers/net/fjes/fjes_debugfs.c15
-rw-r--r--drivers/net/geneve.c2
-rw-r--r--drivers/net/gtp.c37
-rw-r--r--drivers/net/hamradio/6pack.c4
-rw-r--r--drivers/net/hamradio/baycom_epp.c6
-rw-r--r--drivers/net/hamradio/hdlcdrv.c3
-rw-r--r--drivers/net/hamradio/mkiss.c11
-rw-r--r--drivers/net/hippi/rrunner.c2
-rw-r--r--drivers/net/hyperv/hyperv_net.h2
-rw-r--r--drivers/net/hyperv/netvsc_drv.c40
-rw-r--r--drivers/net/hyperv/rndis_filter.c1
-rw-r--r--drivers/net/ieee802154/adf7242.c13
-rw-r--r--drivers/net/ieee802154/at86rf230.c20
-rw-r--r--drivers/net/ieee802154/ca8210.c9
-rw-r--r--drivers/net/ieee802154/mac802154_hwsim.c8
-rw-r--r--drivers/net/ipvlan/ipvlan_main.c1
-rw-r--r--drivers/net/loopback.c78
-rw-r--r--drivers/net/macsec.c6
-rw-r--r--drivers/net/macvlan.c2
-rw-r--r--drivers/net/netdevsim/dev.c535
-rw-r--r--drivers/net/netdevsim/fib.c102
-rw-r--r--drivers/net/netdevsim/netdev.c38
-rw-r--r--drivers/net/netdevsim/netdevsim.h15
-rw-r--r--drivers/net/phy/Kconfig32
-rw-r--r--drivers/net/phy/Makefile3
-rw-r--r--drivers/net/phy/adin.c781
-rw-r--r--drivers/net/phy/aquantia_main.c8
-rw-r--r--drivers/net/phy/at803x.c36
-rw-r--r--drivers/net/phy/bcm87xx.c20
-rw-r--r--drivers/net/phy/broadcom.c2
-rw-r--r--drivers/net/phy/dp83822.c5
-rw-r--r--drivers/net/phy/dp83848.c11
-rw-r--r--drivers/net/phy/dp83867.c212
-rw-r--r--drivers/net/phy/dp83tc811.c4
-rw-r--r--drivers/net/phy/fixed_phy.c6
-rw-r--r--drivers/net/phy/lxt.c6
-rw-r--r--drivers/net/phy/mdio-aspeed.c157
-rw-r--r--drivers/net/phy/mdio-bcm-iproc.c4
-rw-r--r--drivers/net/phy/mdio-cavium.h2
-rw-r--r--drivers/net/phy/mdio-hisi-femac.c4
-rw-r--r--drivers/net/phy/mdio-moxart.c4
-rw-r--r--drivers/net/phy/mdio-mux-meson-g12a.c4
-rw-r--r--drivers/net/phy/mdio-sun4i.c4
-rw-r--r--drivers/net/phy/mdio-xgene.c4
-rw-r--r--drivers/net/phy/mdio_bus.c29
-rw-r--r--drivers/net/phy/meson-gxl.c2
-rw-r--r--drivers/net/phy/microchip.c1
-rw-r--r--drivers/net/phy/microchip_t1.c1
-rw-r--r--drivers/net/phy/mscc.c20
-rw-r--r--drivers/net/phy/nxp-tja11xx.c403
-rw-r--r--drivers/net/phy/phy-c45.c40
-rw-r--r--drivers/net/phy/phy-core.c70
-rw-r--r--drivers/net/phy/phy.c198
-rw-r--r--drivers/net/phy/phy_device.c238
-rw-r--r--drivers/net/phy/phy_led_triggers.c3
-rw-r--r--drivers/net/phy/phylink.c304
-rw-r--r--drivers/net/phy/realtek.c188
-rw-r--r--drivers/net/phy/sfp-bus.c14
-rw-r--r--drivers/net/phy/sfp.c147
-rw-r--r--drivers/net/phy/swphy.c8
-rw-r--r--drivers/net/phy/vitesse.c6
-rw-r--r--drivers/net/phy/xilinx_gmii2rgmii.c4
-rw-r--r--drivers/net/plip/plip.c4
-rw-r--r--drivers/net/ppp/Kconfig3
-rw-r--r--drivers/net/ppp/ppp_mppe.c98
-rw-r--r--drivers/net/ppp/pppoe.c3
-rw-r--r--drivers/net/ppp/pppox.c13
-rw-r--r--drivers/net/ppp/pptp.c3
-rw-r--r--drivers/net/slip/slhc.c30
-rw-r--r--drivers/net/tap.c5
-rw-r--r--drivers/net/team/team.c27
-rw-r--r--drivers/net/thunderbolt.c2
-rw-r--r--drivers/net/tun.c33
-rw-r--r--drivers/net/usb/asix_common.c9
-rw-r--r--drivers/net/usb/asix_devices.c6
-rw-r--r--drivers/net/usb/ax88179_178a.c14
-rw-r--r--drivers/net/usb/cdc_ether.c10
-rw-r--r--drivers/net/usb/cx82310_eth.c3
-rw-r--r--drivers/net/usb/kalmia.c6
-rw-r--r--drivers/net/usb/lan78xx.c31
-rw-r--r--drivers/net/usb/lg-vl600.c4
-rw-r--r--drivers/net/usb/pegasus.c2
-rw-r--r--drivers/net/usb/qmi_wwan.c2
-rw-r--r--drivers/net/usb/r8152.c1068
-rw-r--r--drivers/net/usb/rtl8150.c6
-rw-r--r--drivers/net/usb/smsc75xx.c20
-rw-r--r--drivers/net/usb/sr9800.c9
-rw-r--r--drivers/net/usb/usbnet.c6
-rw-r--r--drivers/net/veth.c61
-rw-r--r--drivers/net/virtio_net.c4
-rw-r--r--drivers/net/vmxnet3/vmxnet3_drv.c30
-rw-r--r--drivers/net/vmxnet3/vmxnet3_ethtool.c10
-rw-r--r--drivers/net/vmxnet3/vmxnet3_int.h7
-rw-r--r--drivers/net/vrf.c63
-rw-r--r--drivers/net/vxlan.c131
-rw-r--r--drivers/net/wan/Kconfig14
-rw-r--r--drivers/net/wan/Makefile1
-rw-r--r--drivers/net/wan/dscc4.c2057
-rw-r--r--drivers/net/wan/hdlc_cisco.c11
-rw-r--r--drivers/net/wan/lmc/lmc_main.c2
-rw-r--r--drivers/net/wan/sdla.c1
-rw-r--r--drivers/net/wan/x25_asy.c4
-rw-r--r--drivers/net/wimax/i2400m/debugfs.c150
-rw-r--r--drivers/net/wimax/i2400m/driver.c7
-rw-r--r--drivers/net/wimax/i2400m/fw.c13
-rw-r--r--drivers/net/wimax/i2400m/i2400m.h7
-rw-r--r--drivers/net/wimax/i2400m/op-rfkill.c1
-rw-r--r--drivers/net/wimax/i2400m/rx.c1
-rw-r--r--drivers/net/wimax/i2400m/usb.c64
-rw-r--r--drivers/net/wireless/ath/Kconfig2
-rw-r--r--drivers/net/wireless/ath/Makefile2
-rw-r--r--drivers/net/wireless/ath/ar5523/Kconfig2
-rw-r--r--drivers/net/wireless/ath/ar5523/Makefile2
-rw-r--r--drivers/net/wireless/ath/ath10k/Kconfig2
-rw-r--r--drivers/net/wireless/ath/ath10k/ahb.c2
-rw-r--r--drivers/net/wireless/ath/ath10k/core.c80
-rw-r--r--drivers/net/wireless/ath/ath10k/core.h27
-rw-r--r--drivers/net/wireless/ath/ath10k/coredump.c4
-rw-r--r--drivers/net/wireless/ath/ath10k/debug.c58
-rw-r--r--drivers/net/wireless/ath/ath10k/debug.h25
-rw-r--r--drivers/net/wireless/ath/ath10k/debugfs_sta.c7
-rw-r--r--drivers/net/wireless/ath/ath10k/hif.h15
-rw-r--r--drivers/net/wireless/ath/ath10k/htc.c1
-rw-r--r--drivers/net/wireless/ath/ath10k/htt.c2
-rw-r--r--drivers/net/wireless/ath/ath10k/htt.h76
-rw-r--r--drivers/net/wireless/ath/ath10k/htt_rx.c488
-rw-r--r--drivers/net/wireless/ath/ath10k/htt_tx.c46
-rw-r--r--drivers/net/wireless/ath/ath10k/hw.c6
-rw-r--r--drivers/net/wireless/ath/ath10k/hw.h13
-rw-r--r--drivers/net/wireless/ath/ath10k/mac.c237
-rw-r--r--drivers/net/wireless/ath/ath10k/pci.c27
-rw-r--r--drivers/net/wireless/ath/ath10k/qmi.c61
-rw-r--r--drivers/net/wireless/ath/ath10k/qmi.h1
-rw-r--r--drivers/net/wireless/ath/ath10k/sdio.c64
-rw-r--r--drivers/net/wireless/ath/ath10k/snoc.c19
-rw-r--r--drivers/net/wireless/ath/ath10k/swap.c4
-rw-r--r--drivers/net/wireless/ath/ath10k/testmode.c17
-rw-r--r--drivers/net/wireless/ath/ath10k/trace.c1
-rw-r--r--drivers/net/wireless/ath/ath10k/trace.h6
-rw-r--r--drivers/net/wireless/ath/ath10k/txrx.c3
-rw-r--r--drivers/net/wireless/ath/ath10k/usb.c4
-rw-r--r--drivers/net/wireless/ath/ath10k/wmi-tlv.c63
-rw-r--r--drivers/net/wireless/ath/ath10k/wmi-tlv.h36
-rw-r--r--drivers/net/wireless/ath/ath10k/wmi.c37
-rw-r--r--drivers/net/wireless/ath/ath10k/wmi.h31
-rw-r--r--drivers/net/wireless/ath/ath5k/Kconfig2
-rw-r--r--drivers/net/wireless/ath/ath5k/Makefile2
-rw-r--r--drivers/net/wireless/ath/ath6kl/Kconfig2
-rw-r--r--drivers/net/wireless/ath/ath6kl/cfg80211.c4
-rw-r--r--drivers/net/wireless/ath/ath6kl/debug.c3
-rw-r--r--drivers/net/wireless/ath/ath6kl/htc_mbox.c4
-rw-r--r--drivers/net/wireless/ath/ath6kl/htc_pipe.c3
-rw-r--r--drivers/net/wireless/ath/ath6kl/trace.h2
-rw-r--r--drivers/net/wireless/ath/ath6kl/usb.c8
-rw-r--r--drivers/net/wireless/ath/ath6kl/wmi.c13
-rw-r--r--drivers/net/wireless/ath/ath9k/Kconfig18
-rw-r--r--drivers/net/wireless/ath/ath9k/Makefile4
-rw-r--r--drivers/net/wireless/ath/ath9k/ar9003_phy.c24
-rw-r--r--drivers/net/wireless/ath/ath9k/ath9k_pci_owl_loader.c215
-rw-r--r--drivers/net/wireless/ath/ath9k/dynack.c101
-rw-r--r--drivers/net/wireless/ath/ath9k/eeprom.c2
-rw-r--r--drivers/net/wireless/ath/ath9k/eeprom_4k.c1
-rw-r--r--drivers/net/wireless/ath/ath9k/htc_drv_init.c4
-rw-r--r--drivers/net/wireless/ath/ath9k/htc_hst.c3
-rw-r--r--drivers/net/wireless/ath/ath9k/hw.c40
-rw-r--r--drivers/net/wireless/ath/ath9k/hw.h1
-rw-r--r--drivers/net/wireless/ath/ath9k/init.c2
-rw-r--r--drivers/net/wireless/ath/ath9k/main.c3
-rw-r--r--drivers/net/wireless/ath/ath9k/recv.c6
-rw-r--r--drivers/net/wireless/ath/ath9k/wmi.c1
-rw-r--r--drivers/net/wireless/ath/ath9k/xmit.c18
-rw-r--r--drivers/net/wireless/ath/carl9170/mac.c2
-rw-r--r--drivers/net/wireless/ath/carl9170/main.c9
-rw-r--r--drivers/net/wireless/ath/carl9170/rx.c2
-rw-r--r--drivers/net/wireless/ath/carl9170/usb.c41
-rw-r--r--drivers/net/wireless/ath/dfs_pattern_detector.c2
-rw-r--r--drivers/net/wireless/ath/regd.h1
-rw-r--r--drivers/net/wireless/ath/wcn36xx/Kconfig2
-rw-r--r--drivers/net/wireless/ath/wcn36xx/Makefile2
-rw-r--r--drivers/net/wireless/ath/wcn36xx/smd.c186
-rw-r--r--drivers/net/wireless/ath/wil6210/Kconfig2
-rw-r--r--drivers/net/wireless/ath/wil6210/Makefile2
-rw-r--r--drivers/net/wireless/ath/wil6210/cfg80211.c251
-rw-r--r--drivers/net/wireless/ath/wil6210/debugfs.c257
-rw-r--r--drivers/net/wireless/ath/wil6210/fw.h11
-rw-r--r--drivers/net/wireless/ath/wil6210/fw_inc.c148
-rw-r--r--drivers/net/wireless/ath/wil6210/interrupt.c67
-rw-r--r--drivers/net/wireless/ath/wil6210/main.c41
-rw-r--r--drivers/net/wireless/ath/wil6210/netdev.c4
-rw-r--r--drivers/net/wireless/ath/wil6210/pcie_bus.c7
-rw-r--r--drivers/net/wireless/ath/wil6210/pmc.c26
-rw-r--r--drivers/net/wireless/ath/wil6210/pmc.h1
-rw-r--r--drivers/net/wireless/ath/wil6210/rx_reorder.c34
-rw-r--r--drivers/net/wireless/ath/wil6210/txrx.c288
-rw-r--r--drivers/net/wireless/ath/wil6210/txrx.h42
-rw-r--r--drivers/net/wireless/ath/wil6210/txrx_edma.c68
-rw-r--r--drivers/net/wireless/ath/wil6210/txrx_edma.h14
-rw-r--r--drivers/net/wireless/ath/wil6210/wil6210.h62
-rw-r--r--drivers/net/wireless/ath/wil6210/wmi.c184
-rw-r--r--drivers/net/wireless/ath/wil6210/wmi.h76
-rw-r--r--drivers/net/wireless/broadcom/b43/dma.c69
-rw-r--r--drivers/net/wireless/broadcom/b43/main.c7
-rw-r--r--drivers/net/wireless/broadcom/b43legacy/dma.c57
-rw-r--r--drivers/net/wireless/broadcom/b43legacy/phy.c21
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/Kconfig52
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/Makefile14
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/Kconfig50
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile14
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c24
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.h19
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h17
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c97
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h14
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c19
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h14
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c14
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h16
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/commonring.c16
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/commonring.h16
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c106
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h14
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h17
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/dmi.c13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c14
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h17
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c16
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.h16
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c22
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.c15
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c29
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h17
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c24
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h16
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c38
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.h16
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.c23
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.h16
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c28
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/tracepoint.c13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/tracepoint.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c47
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c14
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.c13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_cmn.c13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_hal.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_int.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_n.c29
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_qmath.c13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_qmath.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_radio.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phyreg_n.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phytbl_lcn.c13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phytbl_lcn.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phytbl_n.c13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phytbl_n.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmutil/Makefile13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/brcmutil/utils.c13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/include/brcmu_d11.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/include/brcmu_utils.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/include/defs.h13
-rw-r--r--drivers/net/wireless/broadcom/brcm80211/include/soc.h13
-rw-r--r--drivers/net/wireless/cisco/Kconfig2
-rw-r--r--drivers/net/wireless/cisco/airo.c68
-rw-r--r--drivers/net/wireless/intel/ipw2x00/ipw2100.c2
-rw-r--r--drivers/net/wireless/intel/ipw2x00/ipw2200.c2
-rw-r--r--drivers/net/wireless/intel/iwlegacy/3945-rs.c17
-rw-r--r--drivers/net/wireless/intel/iwlegacy/3945.h3
-rw-r--r--drivers/net/wireless/intel/iwlegacy/4965-rs.c35
-rw-r--r--drivers/net/wireless/intel/iwlegacy/Kconfig4
-rw-r--r--drivers/net/wireless/intel/iwlegacy/common.c3
-rw-r--r--drivers/net/wireless/intel/iwlegacy/common.h4
-rw-r--r--drivers/net/wireless/intel/iwlwifi/Kconfig16
-rw-r--r--drivers/net/wireless/intel/iwlwifi/cfg/1000.c14
-rw-r--r--drivers/net/wireless/intel/iwlwifi/cfg/2000.c26
-rw-r--r--drivers/net/wireless/intel/iwlwifi/cfg/22000.c267
-rw-r--r--drivers/net/wireless/intel/iwlwifi/cfg/5000.c18
-rw-r--r--drivers/net/wireless/intel/iwlwifi/cfg/6000.c44
-rw-r--r--drivers/net/wireless/intel/iwlwifi/cfg/7000.c10
-rw-r--r--drivers/net/wireless/intel/iwlwifi/cfg/8000.c10
-rw-r--r--drivers/net/wireless/intel/iwlwifi/cfg/9000.c10
-rw-r--r--drivers/net/wireless/intel/iwlwifi/dvm/devices.c3
-rw-r--r--drivers/net/wireless/intel/iwlwifi/dvm/led.c5
-rw-r--r--drivers/net/wireless/intel/iwlwifi/dvm/lib.c3
-rw-r--r--drivers/net/wireless/intel/iwlwifi/dvm/mac80211.c4
-rw-r--r--drivers/net/wireless/intel/iwlwifi/dvm/main.c12
-rw-r--r--drivers/net/wireless/intel/iwlwifi/dvm/power.c3
-rw-r--r--drivers/net/wireless/intel/iwlwifi/dvm/rs.c33
-rw-r--r--drivers/net/wireless/intel/iwlwifi/dvm/rs.h4
-rw-r--r--drivers/net/wireless/intel/iwlwifi/dvm/tx.c5
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/acpi.c28
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/acpi.h17
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/api/commands.h7
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/api/d3.h6
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/api/dbg-tlv.h124
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/api/debug.h83
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/api/location.h15
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/api/mac.h4
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/api/phy.h7
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/api/power.h24
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/api/rs.h18
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/api/rx.h35
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/api/scan.h70
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/dbg.c1258
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/dbg.h160
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/error-dump.h141
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/file.h44
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/img.h9
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/init.c9
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/paging.c6
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/runtime.h49
-rw-r--r--drivers/net/wireless/intel/iwlwifi/fw/smem.c14
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-config.h74
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-csr.h3
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c235
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.h36
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-devtrace.h1
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-drv.c65
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.c21
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-eeprom-parse.h4
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-eeprom-read.c14
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-fh.h6
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-io.c21
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-io.h18
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-modparams.h9
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c60
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-op-mode.h27
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-prph.h5
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-trans.c16
-rw-r--r--drivers/net/wireless/intel/iwlwifi/iwl-trans.h209
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/constants.h2
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/d3.c234
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c220
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c2
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/fw.c282
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/led.c6
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c58
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c543
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/mvm.h170
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/nvm.c13
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/ops.c494
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c11
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/power.c82
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c44
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/rs.c546
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/rs.h3
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/rx.c20
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c280
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/scan.c110
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/sta.c35
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/sta.h16
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/tdls.c9
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/time-event.c8
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/tt.c12
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/tx.c28
-rw-r--r--drivers/net/wireless/intel/iwlwifi/mvm/utils.c45
-rw-r--r--drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info-gen3.c12
-rw-r--r--drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info.c2
-rw-r--r--drivers/net/wireless/intel/iwlwifi/pcie/drv.c582
-rw-r--r--drivers/net/wireless/intel/iwlwifi/pcie/internal.h65
-rw-r--r--drivers/net/wireless/intel/iwlwifi/pcie/rx.c143
-rw-r--r--drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c28
-rw-r--r--drivers/net/wireless/intel/iwlwifi/pcie/trans.c562
-rw-r--r--drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c70
-rw-r--r--drivers/net/wireless/intel/iwlwifi/pcie/tx.c178
-rw-r--r--drivers/net/wireless/intersil/hostap/hostap_download.c6
-rw-r--r--drivers/net/wireless/intersil/hostap/hostap_plx.c3
-rw-r--r--drivers/net/wireless/intersil/hostap/hostap_proc.c3
-rw-r--r--drivers/net/wireless/intersil/p54/main.c9
-rw-r--r--drivers/net/wireless/intersil/p54/p54usb.c43
-rw-r--r--drivers/net/wireless/intersil/p54/txrx.c11
-rw-r--r--drivers/net/wireless/mac80211_hwsim.c315
-rw-r--r--drivers/net/wireless/marvell/libertas/dev.h2
-rw-r--r--drivers/net/wireless/marvell/libertas/if_spi.c14
-rw-r--r--drivers/net/wireless/marvell/libertas/if_usb.c5
-rw-r--r--drivers/net/wireless/marvell/libertas/main.c2
-rw-r--r--drivers/net/wireless/marvell/libertas/mesh.c31
-rw-r--r--drivers/net/wireless/marvell/libertas/mesh.h3
-rw-r--r--drivers/net/wireless/marvell/libertas_tf/cmd.c2
-rw-r--r--drivers/net/wireless/marvell/libertas_tf/if_usb.c2
-rw-r--r--drivers/net/wireless/marvell/mwifiex/11n.c53
-rw-r--r--drivers/net/wireless/marvell/mwifiex/11n.h5
-rw-r--r--drivers/net/wireless/marvell/mwifiex/11n_aggr.c26
-rw-r--r--drivers/net/wireless/marvell/mwifiex/11n_aggr.h2
-rw-r--r--drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c125
-rw-r--r--drivers/net/wireless/marvell/mwifiex/cfg80211.c37
-rw-r--r--drivers/net/wireless/marvell/mwifiex/cmdevt.c103
-rw-r--r--drivers/net/wireless/marvell/mwifiex/fw.h12
-rw-r--r--drivers/net/wireless/marvell/mwifiex/ie.c3
-rw-r--r--drivers/net/wireless/marvell/mwifiex/init.c34
-rw-r--r--drivers/net/wireless/marvell/mwifiex/main.c35
-rw-r--r--drivers/net/wireless/marvell/mwifiex/main.h3
-rw-r--r--drivers/net/wireless/marvell/mwifiex/pcie.c13
-rw-r--r--drivers/net/wireless/marvell/mwifiex/scan.c81
-rw-r--r--drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c5
-rw-r--r--drivers/net/wireless/marvell/mwifiex/sta_event.c10
-rw-r--r--drivers/net/wireless/marvell/mwifiex/sta_ioctl.c4
-rw-r--r--drivers/net/wireless/marvell/mwifiex/tdls.c71
-rw-r--r--drivers/net/wireless/marvell/mwifiex/txrx.c5
-rw-r--r--drivers/net/wireless/marvell/mwifiex/uap_cmd.c9
-rw-r--r--drivers/net/wireless/marvell/mwifiex/uap_txrx.c10
-rw-r--r--drivers/net/wireless/marvell/mwifiex/usb.c10
-rw-r--r--drivers/net/wireless/marvell/mwifiex/util.c15
-rw-r--r--drivers/net/wireless/marvell/mwifiex/wmm.c111
-rw-r--r--drivers/net/wireless/mediatek/mt76/agg-rx.c36
-rw-r--r--drivers/net/wireless/mediatek/mt76/debugfs.c13
-rw-r--r--drivers/net/wireless/mediatek/mt76/dma.c22
-rw-r--r--drivers/net/wireless/mediatek/mt76/dma.h13
-rw-r--r--drivers/net/wireless/mediatek/mt76/eeprom.c13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mac80211.c140
-rw-r--r--drivers/net/wireless/mediatek/mt76/mcu.c13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mmio.c28
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76.h67
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7603/Kconfig6
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7603/beacon.c2
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7603/core.c4
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7603/debugfs.c32
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7603/dma.c36
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7603/eeprom.c2
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7603/eeprom.h2
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7603/init.c32
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7603/mac.c201
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7603/main.c49
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7603/mcu.c4
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7603/mt7603.h16
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7603/pci.c2
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7603/regs.h6
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7603/soc.c9
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7615/Kconfig7
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7615/Makefile3
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7615/debugfs.c91
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7615/dma.c25
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7615/eeprom.c135
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7615/eeprom.h61
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7615/init.c147
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7615/mac.c818
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7615/mac.h33
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7615/main.c159
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7615/mcu.c1661
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7615/mcu.h110
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h115
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7615/pci.c8
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt7615/regs.h75
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x0/Kconfig12
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x0/eeprom.c5
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x0/init.c5
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x0/main.c19
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x0/mt76x0.h2
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x0/pci.c30
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x0/pci_mcu.c13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x0/phy.c62
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x0/phy.h10
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x0/usb.c38
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x0/usb_mcu.c13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02.h46
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_beacon.c87
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_debugfs.c23
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_dfs.c31
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_dfs.h15
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_dma.h13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_eeprom.c13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_eeprom.h14
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_mac.c131
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_mac.h17
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_mcu.c26
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_mcu.h13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_mmio.c42
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_phy.c16
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_phy.h13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_regs.h44
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_trace.c13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_trace.h16
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_txrx.c22
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_usb.h13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_usb_core.c26
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_usb_mcu.c13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x02_util.c75
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/Kconfig14
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/eeprom.c23
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/eeprom.h13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/init.c22
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/mac.c13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/mac.h13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/mcu.c13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/mcu.h16
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/mt76x2.h13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/mt76x2u.h13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/pci.c13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/pci_init.c14
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/pci_main.c31
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/pci_mcu.c17
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/pci_phy.c21
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/phy.c18
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/usb.c14
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/usb_init.c22
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/usb_mac.c13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/usb_main.c43
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/usb_mcu.c13
-rw-r--r--drivers/net/wireless/mediatek/mt76/mt76x2/usb_phy.c20
-rw-r--r--drivers/net/wireless/mediatek/mt76/trace.c13
-rw-r--r--drivers/net/wireless/mediatek/mt76/trace.h22
-rw-r--r--drivers/net/wireless/mediatek/mt76/tx.c44
-rw-r--r--drivers/net/wireless/mediatek/mt76/usb.c124
-rw-r--r--drivers/net/wireless/mediatek/mt76/usb_trace.c13
-rw-r--r--drivers/net/wireless/mediatek/mt76/usb_trace.h24
-rw-r--r--drivers/net/wireless/mediatek/mt76/util.c13
-rw-r--r--drivers/net/wireless/mediatek/mt76/util.h4
-rw-r--r--drivers/net/wireless/mediatek/mt7601u/dma.c54
-rw-r--r--drivers/net/wireless/mediatek/mt7601u/init.c3
-rw-r--r--drivers/net/wireless/mediatek/mt7601u/main.c4
-rw-r--r--drivers/net/wireless/mediatek/mt7601u/tx.c4
-rw-r--r--drivers/net/wireless/quantenna/qtnfmac/commands.c5
-rw-r--r--drivers/net/wireless/quantenna/qtnfmac/pcie/pcie.c4
-rw-r--r--drivers/net/wireless/ralink/rt2x00/rt2800lib.c138
-rw-r--r--drivers/net/wireless/ralink/rt2x00/rt2800lib.h11
-rw-r--r--drivers/net/wireless/ralink/rt2x00/rt2800mmio.c31
-rw-r--r--drivers/net/wireless/ralink/rt2x00/rt2800mmio.h2
-rw-r--r--drivers/net/wireless/ralink/rt2x00/rt2800pci.c3
-rw-r--r--drivers/net/wireless/ralink/rt2x00/rt2800soc.c3
-rw-r--r--drivers/net/wireless/ralink/rt2x00/rt2800usb.c12
-rw-r--r--drivers/net/wireless/ralink/rt2x00/rt2x00.h11
-rw-r--r--drivers/net/wireless/ralink/rt2x00/rt2x00debug.c159
-rw-r--r--drivers/net/wireless/ralink/rt2x00/rt2x00dev.c26
-rw-r--r--drivers/net/wireless/ralink/rt2x00/rt2x00link.c15
-rw-r--r--drivers/net/wireless/ralink/rt2x00/rt2x00queue.h6
-rw-r--r--drivers/net/wireless/ralink/rt2x00/rt2x00usb.c21
-rw-r--r--drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723b.c11
-rw-r--r--drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c3
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/base.h27
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.c43
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.h1
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/btcoexist/rtl_btc.c3
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/debug.c2
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/efuse.c22
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/pci.c8
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rc.c3
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/regd.c18
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8188ee/def.h29
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8188ee/dm.c7
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8188ee/hw.c2
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c4
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8188ee/trx.c257
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8188ee/trx.h1046
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8192ce/def.h33
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c2
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8192ce/trx.c238
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8192ce/trx.h803
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8192cu/mac.c26
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c2
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.c272
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.h529
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8192de/def.h31
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8192de/dm.c695
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8192de/trx.c18
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.c314
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.h861
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c2
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8723ae/def.h31
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c4
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8723ae/trx.c212
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8723ae/trx.h794
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c4
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8723be/trx.c236
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8723be/trx.h718
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8821ae/def.h31
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8821ae/dm.c8
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8821ae/phy.c4
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c4
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8821ae/trx.c253
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/rtl8821ae/trx.h708
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/usb.c21
-rw-r--r--drivers/net/wireless/realtek/rtlwifi/wifi.h3
-rw-r--r--drivers/net/wireless/realtek/rtw88/Makefile1
-rw-r--r--drivers/net/wireless/realtek/rtw88/coex.c2502
-rw-r--r--drivers/net/wireless/realtek/rtw88/coex.h370
-rw-r--r--drivers/net/wireless/realtek/rtw88/debug.c116
-rw-r--r--drivers/net/wireless/realtek/rtw88/fw.c135
-rw-r--r--drivers/net/wireless/realtek/rtw88/fw.h73
-rw-r--r--drivers/net/wireless/realtek/rtw88/hci.h2
-rw-r--r--drivers/net/wireless/realtek/rtw88/mac.c8
-rw-r--r--drivers/net/wireless/realtek/rtw88/mac80211.c53
-rw-r--r--drivers/net/wireless/realtek/rtw88/main.c82
-rw-r--r--drivers/net/wireless/realtek/rtw88/main.h327
-rw-r--r--drivers/net/wireless/realtek/rtw88/pci.c154
-rw-r--r--drivers/net/wireless/realtek/rtw88/phy.c1425
-rw-r--r--drivers/net/wireless/realtek/rtw88/phy.h29
-rw-r--r--drivers/net/wireless/realtek/rtw88/ps.c9
-rw-r--r--drivers/net/wireless/realtek/rtw88/reg.h79
-rw-r--r--drivers/net/wireless/realtek/rtw88/regd.c69
-rw-r--r--drivers/net/wireless/realtek/rtw88/regd.h4
-rw-r--r--drivers/net/wireless/realtek/rtw88/rtw8822b.c470
-rw-r--r--drivers/net/wireless/realtek/rtw88/rtw8822b_table.c1564
-rw-r--r--drivers/net/wireless/realtek/rtw88/rtw8822c.c1979
-rw-r--r--drivers/net/wireless/realtek/rtw88/rtw8822c.h109
-rw-r--r--drivers/net/wireless/realtek/rtw88/rtw8822c_table.c8714
-rw-r--r--drivers/net/wireless/realtek/rtw88/rtw8822c_table.h3
-rw-r--r--drivers/net/wireless/realtek/rtw88/rx.c5
-rw-r--r--drivers/net/wireless/realtek/rtw88/tx.c2
-rw-r--r--drivers/net/wireless/rsi/rsi_91x_mac80211.c3
-rw-r--r--drivers/net/wireless/rsi/rsi_91x_sdio.c31
-rw-r--r--drivers/net/wireless/rsi/rsi_91x_usb.c1
-rw-r--r--drivers/net/wireless/ti/wl18xx/main.c38
-rw-r--r--drivers/net/wireless/ti/wlcore/main.c5
-rw-r--r--drivers/net/wireless/ti/wlcore/rx.c2
-rw-r--r--drivers/net/wireless/ti/wlcore/tx.c2
-rw-r--r--drivers/net/wireless/ti/wlcore/vendor_cmd.c3
-rw-r--r--drivers/net/wireless/virt_wifi.c2
-rw-r--r--drivers/net/wireless/wl3501_cs.c4
-rw-r--r--drivers/net/wireless/zydas/zd1211rw/zd_chip.c3
-rw-r--r--drivers/net/wireless/zydas/zd1211rw/zd_mac.c1
-rw-r--r--drivers/net/wireless/zydas/zd1211rw/zd_usb.c11
-rw-r--r--drivers/net/xen-netback/interface.c2
-rw-r--r--drivers/net/xen-netback/netback.c15
-rw-r--r--drivers/net/xen-netback/xenbus.c46
-rw-r--r--drivers/net/xen-netfront.c10
-rw-r--r--drivers/nfc/nfcmrvl/main.c4
-rw-r--r--drivers/nfc/nfcmrvl/uart.c4
-rw-r--r--drivers/nfc/nfcmrvl/usb.c1
-rw-r--r--drivers/nfc/nxp-nci/Kconfig7
-rw-r--r--drivers/nfc/nxp-nci/core.c2
-rw-r--r--drivers/nfc/nxp-nci/i2c.c134
-rw-r--r--drivers/nfc/nxp-nci/nxp-nci.h1
-rw-r--r--drivers/nfc/st-nci/i2c.c2
-rw-r--r--drivers/nfc/st-nci/se.c2
-rw-r--r--drivers/nfc/st21nfca/se.c2
-rw-r--r--drivers/nfc/st95hf/core.c2
-rw-r--r--drivers/ntb/Kconfig11
-rw-r--r--drivers/ntb/Makefile3
-rw-r--r--drivers/ntb/core.c (renamed from drivers/ntb/ntb.c)0
-rw-r--r--drivers/ntb/hw/amd/ntb_hw_amd.c10
-rw-r--r--drivers/ntb/hw/intel/ntb_hw_gen3.c6
-rw-r--r--drivers/ntb/hw/mscc/ntb_hw_switchtec.c82
-rw-r--r--drivers/ntb/msi.c410
-rw-r--r--drivers/ntb/ntb_transport.c170
-rw-r--r--drivers/ntb/test/Kconfig9
-rw-r--r--drivers/ntb/test/Makefile1
-rw-r--r--drivers/ntb/test/ntb_msi_test.c433
-rw-r--r--drivers/ntb/test/ntb_perf.c14
-rw-r--r--drivers/nvdimm/Kconfig2
-rw-r--r--drivers/nvdimm/Makefile1
-rw-r--r--drivers/nvdimm/btt_devs.c16
-rw-r--r--drivers/nvdimm/bus.c210
-rw-r--r--drivers/nvdimm/claim.c6
-rw-r--r--drivers/nvdimm/core.c10
-rw-r--r--drivers/nvdimm/dax_devs.c2
-rw-r--r--drivers/nvdimm/dimm_devs.c4
-rw-r--r--drivers/nvdimm/namespace_devs.c44
-rw-r--r--drivers/nvdimm/nd-core.h71
-rw-r--r--drivers/nvdimm/nd.h1
-rw-r--r--drivers/nvdimm/nd_virtio.c125
-rw-r--r--drivers/nvdimm/pfn.h15
-rw-r--r--drivers/nvdimm/pfn_devs.c123
-rw-r--r--drivers/nvdimm/pmem.c73
-rw-r--r--drivers/nvdimm/region.c24
-rw-r--r--drivers/nvdimm/region_devs.c45
-rw-r--r--drivers/nvdimm/virtio_pmem.c122
-rw-r--r--drivers/nvdimm/virtio_pmem.h55
-rw-r--r--drivers/nvme/host/Kconfig1
-rw-r--r--drivers/nvme/host/core.c330
-rw-r--r--drivers/nvme/host/fabrics.c40
-rw-r--r--drivers/nvme/host/fabrics.h3
-rw-r--r--drivers/nvme/host/fault_inject.c41
-rw-r--r--drivers/nvme/host/fc.c137
-rw-r--r--drivers/nvme/host/lightnvm.c47
-rw-r--r--drivers/nvme/host/multipath.c105
-rw-r--r--drivers/nvme/host/nvme.h111
-rw-r--r--drivers/nvme/host/pci.c276
-rw-r--r--drivers/nvme/host/rdma.c86
-rw-r--r--drivers/nvme/host/tcp.c153
-rw-r--r--drivers/nvme/host/trace.c108
-rw-r--r--drivers/nvme/host/trace.h66
-rw-r--r--drivers/nvme/target/Makefile3
-rw-r--r--drivers/nvme/target/admin-cmd.c25
-rw-r--r--drivers/nvme/target/configfs.c5
-rw-r--r--drivers/nvme/target/core.c27
-rw-r--r--drivers/nvme/target/discovery.c8
-rw-r--r--drivers/nvme/target/fabrics-cmd.c2
-rw-r--r--drivers/nvme/target/fc.c13
-rw-r--r--drivers/nvme/target/fcloop.c81
-rw-r--r--drivers/nvme/target/io-cmd-bdev.c39
-rw-r--r--drivers/nvme/target/loop.c42
-rw-r--r--drivers/nvme/target/nvmet.h13
-rw-r--r--drivers/nvme/target/tcp.c24
-rw-r--r--drivers/nvme/target/trace.c219
-rw-r--r--drivers/nvme/target/trace.h141
-rw-r--r--drivers/nvmem/Kconfig9
-rw-r--r--drivers/nvmem/Makefile2
-rw-r--r--drivers/nvmem/core.c7
-rw-r--r--drivers/nvmem/imx-ocotp-scu.c168
-rw-r--r--drivers/nvmem/imx-ocotp.c59
-rw-r--r--drivers/nvmem/meson-mx-efuse.c3
-rw-r--r--drivers/nvmem/mxs-ocotp.c2
-rw-r--r--drivers/nvmem/nvmem-sysfs.c15
-rw-r--r--drivers/nvmem/sunxi_sid.c1
-rw-r--r--drivers/of/base.c10
-rw-r--r--drivers/of/fdt.c155
-rw-r--r--drivers/of/irq.c2
-rw-r--r--drivers/of/of_mdio.c8
-rw-r--r--drivers/of/of_reserved_mem.c3
-rw-r--r--drivers/of/platform.c10
-rw-r--r--drivers/of/resolver.c12
-rw-r--r--drivers/of/unittest.c2
-rw-r--r--drivers/opp/Kconfig2
-rw-r--r--drivers/opp/core.c241
-rw-r--r--drivers/opp/of.c60
-rw-r--r--drivers/oprofile/oprofilefs.c20
-rw-r--r--drivers/parisc/dino.c30
-rw-r--r--drivers/parisc/eisa_enumerator.c10
-rw-r--r--drivers/parisc/hppb.c11
-rw-r--r--drivers/parport/Kconfig2
-rw-r--r--drivers/parport/Makefile2
-rw-r--r--drivers/parport/parport_serial.c50
-rw-r--r--drivers/pci/Kconfig1
-rw-r--r--drivers/pci/ats.c2
-rw-r--r--drivers/pci/controller/Kconfig11
-rw-r--r--drivers/pci/controller/Makefile1
-rw-r--r--drivers/pci/controller/dwc/Kconfig2
-rw-r--r--drivers/pci/controller/dwc/pci-dra7xx.c1
-rw-r--r--drivers/pci/controller/dwc/pcie-armada8k.c84
-rw-r--r--drivers/pci/controller/dwc/pcie-designware-host.c12
-rw-r--r--drivers/pci/controller/dwc/pcie-designware.c61
-rw-r--r--drivers/pci/controller/dwc/pcie-designware.h39
-rw-r--r--drivers/pci/controller/dwc/pcie-kirin.c2
-rw-r--r--drivers/pci/controller/dwc/pcie-qcom.c115
-rw-r--r--drivers/pci/controller/pci-aardvark.c2
-rw-r--r--drivers/pci/controller/pci-hyperv-intf.c67
-rw-r--r--drivers/pci/controller/pci-hyperv.c333
-rw-r--r--drivers/pci/controller/pci-tegra.c589
-rw-r--r--drivers/pci/controller/pcie-altera-msi.c10
-rw-r--r--drivers/pci/controller/pcie-altera.c69
-rw-r--r--drivers/pci/controller/pcie-iproc-platform.c2
-rw-r--r--drivers/pci/controller/pcie-iproc.c2
-rw-r--r--drivers/pci/controller/pcie-mobiveil.c525
-rw-r--r--drivers/pci/controller/pcie-xilinx-nwl.c11
-rw-r--r--drivers/pci/controller/vmd.c2
-rw-r--r--drivers/pci/endpoint/functions/pci-epf-test.c35
-rw-r--r--drivers/pci/endpoint/pci-epc-core.c3
-rw-r--r--drivers/pci/hotplug/Kconfig9
-rw-r--r--drivers/pci/hotplug/Makefile1
-rw-r--r--drivers/pci/hotplug/sgi_hotplug.c700
-rw-r--r--drivers/pci/iov.c2
-rw-r--r--drivers/pci/mmap.c2
-rw-r--r--drivers/pci/msi.c97
-rw-r--r--drivers/pci/of.c8
-rw-r--r--drivers/pci/p2pdma.c68
-rw-r--r--drivers/pci/pci-acpi.c14
-rw-r--r--drivers/pci/pci-bridge-emul.c2
-rw-r--r--drivers/pci/pci-driver.c90
-rw-r--r--drivers/pci/pci-pf-stub.c2
-rw-r--r--drivers/pci/pci-sysfs.c5
-rw-r--r--drivers/pci/pci.c93
-rw-r--r--drivers/pci/pci.h8
-rw-r--r--drivers/pci/pcie/aer_inject.c2
-rw-r--r--drivers/pci/pcie/aspm.c40
-rw-r--r--drivers/pci/probe.c35
-rw-r--r--drivers/pci/proc.c2
-rw-r--r--drivers/pci/quirks.c112
-rw-r--r--drivers/pci/search.c4
-rw-r--r--drivers/pci/setup-bus.c60
-rw-r--r--drivers/pci/slot.c1
-rw-r--r--drivers/pci/switch/Kconfig2
-rw-r--r--drivers/pci/switch/switchtec.c12
-rw-r--r--drivers/pcmcia/db1xxx_ss.c4
-rw-r--r--drivers/pcmcia/ds.c2
-rw-r--r--drivers/pcmcia/i82092.c6
-rw-r--r--drivers/perf/Kconfig8
-rw-r--r--drivers/perf/Makefile1
-rw-r--r--drivers/perf/arm_pmu.c2
-rw-r--r--drivers/perf/arm_pmu_acpi.c72
-rw-r--r--drivers/perf/arm_smmuv3_pmu.c65
-rw-r--r--drivers/perf/arm_spe_pmu.c12
-rw-r--r--drivers/perf/fsl_imx8_ddr_perf.c625
-rw-r--r--drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c4
-rw-r--r--drivers/perf/hisilicon/hisi_uncore_hha_pmu.c4
-rw-r--r--drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c4
-rw-r--r--drivers/perf/qcom_l2_pmu.c6
-rw-r--r--drivers/perf/qcom_l3_pmu.c2
-rw-r--r--drivers/perf/xgene_pmu.c4
-rw-r--r--drivers/phy/Makefile2
-rw-r--r--drivers/phy/amlogic/phy-meson-g12a-usb3-pcie.c2
-rw-r--r--drivers/phy/broadcom/phy-brcm-usb.c9
-rw-r--r--drivers/phy/freescale/Kconfig10
-rw-r--r--drivers/phy/freescale/Makefile1
-rw-r--r--drivers/phy/freescale/phy-fsl-imx8-mipi-dphy.c497
-rw-r--r--drivers/phy/lantiq/Kconfig11
-rw-r--r--drivers/phy/lantiq/Makefile1
-rw-r--r--drivers/phy/lantiq/phy-lantiq-vrx200-pcie.c494
-rw-r--r--drivers/phy/marvell/Kconfig1
-rw-r--r--drivers/phy/marvell/phy-armada38x-comphy.c4
-rw-r--r--drivers/phy/marvell/phy-mvebu-a3700-comphy.c17
-rw-r--r--drivers/phy/marvell/phy-mvebu-cp110-comphy.c525
-rw-r--r--drivers/phy/phy-core.c10
-rw-r--r--drivers/phy/qualcomm/Kconfig8
-rw-r--r--drivers/phy/qualcomm/Makefile1
-rw-r--r--drivers/phy/qualcomm/phy-qcom-pcie2.c331
-rw-r--r--drivers/phy/qualcomm/phy-qcom-qmp.c43
-rw-r--r--drivers/phy/qualcomm/phy-qcom-qusb2.c2
-rw-r--r--drivers/phy/renesas/phy-rcar-gen2.c2
-rw-r--r--drivers/phy/renesas/phy-rcar-gen3-usb2.c21
-rw-r--r--drivers/phy/rockchip/phy-rockchip-inno-hdmi.c2
-rw-r--r--drivers/phy/samsung/phy-exynos-dp-video.c1
-rw-r--r--drivers/phy/samsung/phy-exynos-mipi-video.c1
-rw-r--r--drivers/phy/samsung/phy-exynos-pcie.c1
-rw-r--r--drivers/phy/samsung/phy-exynos5-usbdrd.c1
-rw-r--r--drivers/phy/samsung/phy-exynos5250-sata.c1
-rw-r--r--drivers/phy/samsung/phy-samsung-usb2.c6
-rw-r--r--drivers/phy/tegra/xusb-tegra124.c9
-rw-r--r--drivers/phy/tegra/xusb-tegra210.c9
-rw-r--r--drivers/phy/ti/phy-am654-serdes.c37
-rw-r--r--drivers/pinctrl/aspeed/Makefile2
-rw-r--r--drivers/pinctrl/aspeed/pinctrl-aspeed-g4.c94
-rw-r--r--drivers/pinctrl/aspeed/pinctrl-aspeed-g5.c237
-rw-r--r--drivers/pinctrl/aspeed/pinctrl-aspeed.c252
-rw-r--r--drivers/pinctrl/aspeed/pinctrl-aspeed.h542
-rw-r--r--drivers/pinctrl/aspeed/pinmux-aspeed.c99
-rw-r--r--drivers/pinctrl/aspeed/pinmux-aspeed.h737
-rw-r--r--drivers/pinctrl/bcm/Kconfig6
-rw-r--r--drivers/pinctrl/bcm/pinctrl-ns2-mux.c2
-rw-r--r--drivers/pinctrl/cirrus/pinctrl-cs47l35.c6
-rw-r--r--drivers/pinctrl/cirrus/pinctrl-cs47l85.c6
-rw-r--r--drivers/pinctrl/cirrus/pinctrl-cs47l90.c6
-rw-r--r--drivers/pinctrl/cirrus/pinctrl-madera-core.c6
-rw-r--r--drivers/pinctrl/cirrus/pinctrl-madera.h6
-rw-r--r--drivers/pinctrl/core.c44
-rw-r--r--drivers/pinctrl/devicetree.c7
-rw-r--r--drivers/pinctrl/freescale/Kconfig7
-rw-r--r--drivers/pinctrl/freescale/Makefile1
-rw-r--r--drivers/pinctrl/freescale/pinctrl-imx8mn.c348
-rw-r--r--drivers/pinctrl/intel/pinctrl-baytrail.c338
-rw-r--r--drivers/pinctrl/intel/pinctrl-cherryview.c42
-rw-r--r--drivers/pinctrl/mediatek/pinctrl-mt8183.c1
-rw-r--r--drivers/pinctrl/mediatek/pinctrl-paris.c19
-rw-r--r--drivers/pinctrl/mediatek/pinctrl-paris.h2
-rw-r--r--drivers/pinctrl/meson/pinctrl-meson-g12a.c36
-rw-r--r--drivers/pinctrl/meson/pinctrl-meson.c350
-rw-r--r--drivers/pinctrl/meson/pinctrl-meson.h18
-rw-r--r--drivers/pinctrl/mvebu/pinctrl-kirkwood.c576
-rw-r--r--drivers/pinctrl/pinconf-generic.c2
-rw-r--r--drivers/pinctrl/pinctrl-bm1880.c733
-rw-r--r--drivers/pinctrl/pinctrl-rockchip.c1
-rw-r--r--drivers/pinctrl/pinctrl-stmfx.c19
-rw-r--r--drivers/pinctrl/pinctrl-tb10x.c12
-rw-r--r--drivers/pinctrl/qcom/Kconfig12
-rw-r--r--drivers/pinctrl/qcom/Makefile1
-rw-r--r--drivers/pinctrl/qcom/pinctrl-msm.c60
-rw-r--r--drivers/pinctrl/qcom/pinctrl-msm.h1
-rw-r--r--drivers/pinctrl/qcom/pinctrl-sdm845.c46
-rw-r--r--drivers/pinctrl/qcom/pinctrl-sm8150.c1548
-rw-r--r--drivers/pinctrl/qcom/pinctrl-spmi-gpio.c92
-rw-r--r--drivers/pinctrl/sh-pfc/core.c60
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-emev2.c70
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-r8a73a4.c2
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-r8a7740.c2
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-r8a77470.c2
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-r8a7778.c125
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-r8a7779.c2
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-r8a7790.c36
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-r8a7791.c2
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-r8a7792.c2
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-r8a7794.c2
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-r8a7795-es1.c434
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-r8a7795.c414
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-r8a7796.c414
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-r8a77965.c410
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-r8a77970.c26
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-r8a77980.c2
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-r8a77990.c181
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-r8a77995.c2
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-sh73a0.c21
-rw-r--r--drivers/pinctrl/sh-pfc/pfc-sh7734.c2
-rw-r--r--drivers/pinctrl/sh-pfc/pinctrl.c3
-rw-r--r--drivers/pinctrl/sh-pfc/sh_pfc.h90
-rw-r--r--drivers/pinctrl/stm32/pinctrl-stm32.c184
-rw-r--r--drivers/pinctrl/stm32/pinctrl-stm32.h2
-rw-r--r--drivers/pinctrl/stm32/pinctrl-stm32mp157.c5
-rw-r--r--drivers/pinctrl/tegra/Kconfig4
-rw-r--r--drivers/pinctrl/tegra/Makefile1
-rw-r--r--drivers/pinctrl/tegra/pinctrl-tegra.c26
-rw-r--r--drivers/pinctrl/tegra/pinctrl-tegra.h12
-rw-r--r--drivers/pinctrl/tegra/pinctrl-tegra114.c6
-rw-r--r--drivers/pinctrl/tegra/pinctrl-tegra124.c6
-rw-r--r--drivers/pinctrl/tegra/pinctrl-tegra194.c170
-rw-r--r--drivers/pinctrl/tegra/pinctrl-tegra20.c6
-rw-r--r--drivers/pinctrl/tegra/pinctrl-tegra210.c62
-rw-r--r--drivers/pinctrl/tegra/pinctrl-tegra30.c4
-rw-r--r--drivers/platform/Kconfig2
-rw-r--r--drivers/platform/Makefile2
-rw-r--r--drivers/platform/chrome/Kconfig42
-rw-r--r--drivers/platform/chrome/Makefile4
-rw-r--r--drivers/platform/chrome/cros_ec_debugfs.c48
-rw-r--r--drivers/platform/chrome/cros_ec_ishtp.c763
-rw-r--r--drivers/platform/chrome/cros_ec_lightbar.c6
-rw-r--r--drivers/platform/chrome/cros_ec_lpc.c165
-rw-r--r--drivers/platform/chrome/cros_ec_lpc_mec.c14
-rw-r--r--drivers/platform/chrome/cros_ec_lpc_reg.c101
-rw-r--r--drivers/platform/chrome/cros_ec_lpc_reg.h45
-rw-r--r--drivers/platform/chrome/cros_ec_spi.c68
-rw-r--r--drivers/platform/chrome/cros_ec_sysfs.c2
-rw-r--r--drivers/platform/chrome/cros_ec_vbc.c2
-rw-r--r--drivers/platform/chrome/wilco_ec/Kconfig18
-rw-r--r--drivers/platform/chrome/wilco_ec/Makefile6
-rw-r--r--drivers/platform/chrome/wilco_ec/core.c26
-rw-r--r--drivers/platform/chrome/wilco_ec/debugfs.c12
-rw-r--r--drivers/platform/chrome/wilco_ec/event.c581
-rw-r--r--drivers/platform/chrome/wilco_ec/mailbox.c21
-rw-r--r--drivers/platform/chrome/wilco_ec/properties.c132
-rw-r--r--drivers/platform/chrome/wilco_ec/sysfs.c156
-rw-r--r--drivers/platform/chrome/wilco_ec/telemetry.c450
-rw-r--r--drivers/platform/mellanox/mlxreg-hotplug.c5
-rw-r--r--drivers/platform/olpc/Kconfig29
-rw-r--r--drivers/platform/olpc/Makefile3
-rw-r--r--drivers/platform/olpc/olpc-ec.c174
-rw-r--r--drivers/platform/olpc/olpc-xo175-ec.c759
-rw-r--r--drivers/platform/x86/Kconfig29
-rw-r--r--drivers/platform/x86/Makefile4
-rw-r--r--drivers/platform/x86/acer-wmi.c82
-rw-r--r--drivers/platform/x86/asus-nb-wmi.c12
-rw-r--r--drivers/platform/x86/asus-wmi.c981
-rw-r--r--drivers/platform/x86/compal-laptop.c2
-rw-r--r--drivers/platform/x86/dcdbas.c2
-rw-r--r--drivers/platform/x86/dell-laptop.c5
-rw-r--r--drivers/platform/x86/dell-smbios-wmi.c2
-rw-r--r--drivers/platform/x86/dell-smo8800.c1
-rw-r--r--drivers/platform/x86/dell-wmi-descriptor.c3
-rw-r--r--drivers/platform/x86/dell-wmi.c14
-rw-r--r--drivers/platform/x86/dell_rbu.c2
-rw-r--r--drivers/platform/x86/hp-wmi.c47
-rw-r--r--drivers/platform/x86/hp_accel.c2
-rw-r--r--drivers/platform/x86/huawei-wmi.c2
-rw-r--r--drivers/platform/x86/i2c-multi-instantiate.c4
-rw-r--r--drivers/platform/x86/ideapad-laptop.c36
-rw-r--r--drivers/platform/x86/intel-hid.c36
-rw-r--r--drivers/platform/x86/intel-vbtn.c20
-rw-r--r--drivers/platform/x86/intel-wmi-thunderbolt.c3
-rw-r--r--drivers/platform/x86/intel_bxtwc_tmu.c5
-rw-r--r--drivers/platform/x86/intel_cht_int33fe.c254
-rw-r--r--drivers/platform/x86/intel_int0002_vgpio.c48
-rw-r--r--drivers/platform/x86/intel_menlow.c8
-rw-r--r--drivers/platform/x86/intel_pmc_core.c84
-rw-r--r--drivers/platform/x86/intel_pmc_core_pltdrv.c70
-rw-r--r--drivers/platform/x86/intel_pmc_ipc.c4
-rw-r--r--drivers/platform/x86/intel_speed_select_if/Kconfig17
-rw-r--r--drivers/platform/x86/intel_speed_select_if/Makefile10
-rw-r--r--drivers/platform/x86/intel_speed_select_if/isst_if_common.c674
-rw-r--r--drivers/platform/x86/intel_speed_select_if/isst_if_common.h69
-rw-r--r--drivers/platform/x86/intel_speed_select_if/isst_if_mbox_msr.c216
-rw-r--r--drivers/platform/x86/intel_speed_select_if/isst_if_mbox_pci.c214
-rw-r--r--drivers/platform/x86/intel_speed_select_if/isst_if_mmio.c176
-rw-r--r--drivers/platform/x86/intel_telemetry_debugfs.c78
-rw-r--r--drivers/platform/x86/mlx-platform.c216
-rw-r--r--drivers/platform/x86/pcengines-apuv2.c23
-rw-r--r--drivers/platform/x86/pmc_atom.c58
-rw-r--r--drivers/platform/x86/samsung-laptop.c89
-rw-r--r--drivers/platform/x86/thinkpad_acpi.c122
-rw-r--r--drivers/platform/x86/touchscreen_dmi.c86
-rw-r--r--drivers/platform/x86/wmi-bmof.c2
-rw-r--r--drivers/platform/x86/wmi.c48
-rw-r--r--drivers/platform/x86/xiaomi-wmi.c92
-rw-r--r--drivers/pnp/isapnp/Kconfig2
-rw-r--r--drivers/power/avs/smartreflex.c41
-rw-r--r--drivers/power/reset/Kconfig10
-rw-r--r--drivers/power/reset/Makefile1
-rw-r--r--drivers/power/reset/nvmem-reboot-mode.c76
-rw-r--r--drivers/power/reset/qcom-pon.c12
-rw-r--r--drivers/power/supply/Kconfig34
-rw-r--r--drivers/power/supply/Makefile3
-rw-r--r--drivers/power/supply/ab8500_charger.c1
-rw-r--r--drivers/power/supply/bd70528-charger.c743
-rw-r--r--drivers/power/supply/bq24190_charger.c2
-rw-r--r--drivers/power/supply/bq24257_charger.c2
-rw-r--r--drivers/power/supply/bq25890_charger.c2
-rw-r--r--drivers/power/supply/cros_usbpd-charger.c116
-rw-r--r--drivers/power/supply/max14656_charger_detector.c2
-rw-r--r--drivers/power/supply/max17040_battery.c2
-rw-r--r--drivers/power/supply/max17042_battery.c2
-rw-r--r--drivers/power/supply/olpc_battery.c1
-rw-r--r--drivers/power/supply/power_supply_core.c9
-rw-r--r--drivers/power/supply/power_supply_hwmon.c355
-rw-r--r--drivers/power/supply/power_supply_sysfs.c2
-rw-r--r--drivers/power/supply/rt5033_battery.c2
-rw-r--r--drivers/power/supply/rt9455_charger.c2
-rw-r--r--drivers/power/supply/sbs-manager.c2
-rw-r--r--drivers/power/supply/ucs1002_power.c2
-rw-r--r--drivers/power/supply/wilco-charger.c187
-rw-r--r--drivers/powercap/Kconfig11
-rw-r--r--drivers/powercap/Makefile3
-rw-r--r--drivers/powercap/idle_inject.c53
-rw-r--r--drivers/powercap/intel_rapl_common.c (renamed from drivers/powercap/intel_rapl.c)838
-rw-r--r--drivers/powercap/intel_rapl_msr.c183
-rw-r--r--drivers/powercap/powercap_sys.c2
-rw-r--r--drivers/pps/pps.c8
-rw-r--r--drivers/ptp/Kconfig2
-rw-r--r--drivers/ptp/ptp_chardev.c63
-rw-r--r--drivers/ptp/ptp_clock.c3
-rw-r--r--drivers/ptp/ptp_dte.c5
-rw-r--r--drivers/pwm/Kconfig11
-rw-r--r--drivers/pwm/Makefile1
-rw-r--r--drivers/pwm/core.c175
-rw-r--r--drivers/pwm/pwm-atmel-hlcdc.c1
-rw-r--r--drivers/pwm/pwm-bcm2835.c8
-rw-r--r--drivers/pwm/pwm-fsl-ftm.c383
-rw-r--r--drivers/pwm/pwm-jz4740.c49
-rw-r--r--drivers/pwm/pwm-meson.c386
-rw-r--r--drivers/pwm/pwm-rcar.c39
-rw-r--r--drivers/pwm/pwm-sifive.c339
-rw-r--r--drivers/pwm/pwm-stm32-lp.c25
-rw-r--r--drivers/pwm/pwm-stm32.c2
-rw-r--r--drivers/pwm/sysfs.c102
-rw-r--r--drivers/rapidio/Kconfig2
-rw-r--r--drivers/rapidio/devices/rio_mport_cdev.c2
-rw-r--r--drivers/ras/Makefile3
-rw-r--r--drivers/ras/cec.c133
-rw-r--r--drivers/ras/debugfs.c2
-rw-r--r--drivers/regulator/88pm800-regulator.c (renamed from drivers/regulator/88pm800.c)0
-rw-r--r--drivers/regulator/Kconfig60
-rw-r--r--drivers/regulator/Makefile6
-rw-r--r--drivers/regulator/act8865-regulator.c316
-rw-r--r--drivers/regulator/act8945a-regulator.c8
-rw-r--r--drivers/regulator/arizona-ldo1.c83
-rw-r--r--drivers/regulator/arizona-micsupp.c72
-rw-r--r--drivers/regulator/axp20x-regulator.c10
-rw-r--r--drivers/regulator/bd70528-regulator.c1
-rw-r--r--drivers/regulator/bd718x7-regulator.c26
-rw-r--r--drivers/regulator/core.c336
-rw-r--r--drivers/regulator/cpcap-regulator.c2
-rw-r--r--drivers/regulator/da9062-regulator.c44
-rw-r--r--drivers/regulator/da9063-regulator.c65
-rw-r--r--drivers/regulator/da9211-regulator.c4
-rw-r--r--drivers/regulator/fixed.c83
-rw-r--r--drivers/regulator/helpers.c32
-rw-r--r--drivers/regulator/lm363x-regulator.c80
-rw-r--r--drivers/regulator/lp87565-regulator.c25
-rw-r--r--drivers/regulator/lp8788-ldo.c2
-rw-r--r--drivers/regulator/max77620-regulator.c28
-rw-r--r--drivers/regulator/max77650-regulator.c170
-rw-r--r--drivers/regulator/max77686-regulator.c2
-rw-r--r--drivers/regulator/max77802-regulator.c2
-rw-r--r--drivers/regulator/max8660.c1
-rw-r--r--drivers/regulator/max8952.c64
-rw-r--r--drivers/regulator/mt6358-regulator.c549
-rw-r--r--drivers/regulator/of_regulator.c74
-rw-r--r--drivers/regulator/qcom-rpmh-regulator.c193
-rw-r--r--drivers/regulator/qcom_spmi-regulator.c252
-rw-r--r--drivers/regulator/rk808-regulator.c646
-rw-r--r--drivers/regulator/s2mps11.c257
-rw-r--r--drivers/regulator/s5m8767.c4
-rw-r--r--drivers/regulator/slg51000-regulator.c524
-rw-r--r--drivers/regulator/slg51000-regulator.h505
-rw-r--r--drivers/regulator/stm32-booster.c128
-rw-r--r--drivers/regulator/sy8824x.c232
-rw-r--r--drivers/regulator/tps65090-regulator.c7
-rw-r--r--drivers/regulator/tps65132-regulator.c4
-rw-r--r--drivers/regulator/twl-regulator.c23
-rw-r--r--drivers/regulator/twl6030-regulator.c21
-rw-r--r--drivers/regulator/uniphier-regulator.c4
-rw-r--r--drivers/regulator/wm831x-dcdc.c29
-rw-r--r--drivers/remoteproc/Kconfig18
-rw-r--r--drivers/remoteproc/Makefile1
-rw-r--r--drivers/remoteproc/imx_rproc.c8
-rw-r--r--drivers/remoteproc/qcom_q6v5_adsp.c73
-rw-r--r--drivers/remoteproc/qcom_q6v5_mss.c33
-rw-r--r--drivers/remoteproc/remoteproc_core.c15
-rw-r--r--drivers/remoteproc/remoteproc_elf_loader.c3
-rw-r--r--drivers/remoteproc/remoteproc_internal.h11
-rw-r--r--drivers/remoteproc/stm32_rproc.c628
-rw-r--r--drivers/reset/Kconfig14
-rw-r--r--drivers/reset/Makefile1
-rw-r--r--drivers/reset/core.c3
-rw-r--r--drivers/reset/reset-imx7.c12
-rw-r--r--drivers/reset/reset-meson.c51
-rw-r--r--drivers/reset/reset-scmi.c124
-rw-r--r--drivers/reset/reset-simple.c5
-rw-r--r--drivers/rpmsg/rpmsg_core.c3
-rw-r--r--drivers/rtc/Kconfig15
-rw-r--r--drivers/rtc/Makefile1
-rw-r--r--drivers/rtc/interface.c13
-rw-r--r--drivers/rtc/rtc-bd70528.c500
-rw-r--r--drivers/rtc/rtc-ds1307.c129
-rw-r--r--drivers/rtc/rtc-ds1374.c2
-rw-r--r--drivers/rtc/rtc-ds2404.c5
-rw-r--r--drivers/rtc/rtc-fm3130.c8
-rw-r--r--drivers/rtc/rtc-imx-sc.c87
-rw-r--r--drivers/rtc/rtc-m41t80.c2
-rw-r--r--drivers/rtc/rtc-pcf2123.c354
-rw-r--r--drivers/rtc/rtc-pcf8563.c13
-rw-r--r--drivers/rtc/rtc-rk808.c68
-rw-r--r--drivers/rtc/rtc-rv8803.c2
-rw-r--r--drivers/rtc/rtc-rx8010.c2
-rw-r--r--drivers/rtc/rtc-rx8025.c2
-rw-r--r--drivers/rtc/rtc-s35390a.c55
-rw-r--r--drivers/rtc/rtc-st-lpc.c4
-rw-r--r--drivers/rtc/rtc-stm32.c6
-rw-r--r--drivers/rtc/rtc-sun6i.c1
-rw-r--r--drivers/rtc/rtc-tegra.c253
-rw-r--r--drivers/rtc/rtc-test.c1
-rw-r--r--drivers/rtc/rtc-tps65910.c2
-rw-r--r--drivers/rtc/rtc-wm831x.c3
-rw-r--r--drivers/s390/Makefile3
-rw-r--r--drivers/s390/block/Kconfig2
-rw-r--r--drivers/s390/block/dasd.c233
-rw-r--r--drivers/s390/block/dasd_alias.c22
-rw-r--r--drivers/s390/block/dasd_devmap.c72
-rw-r--r--drivers/s390/block/dasd_diag.c22
-rw-r--r--drivers/s390/block/dasd_eckd.c966
-rw-r--r--drivers/s390/block/dasd_eckd.h175
-rw-r--r--drivers/s390/block/dasd_eer.c1
-rw-r--r--drivers/s390/block/dasd_fba.c45
-rw-r--r--drivers/s390/block/dasd_fba.h5
-rw-r--r--drivers/s390/block/dasd_int.h33
-rw-r--r--drivers/s390/block/dasd_ioctl.c56
-rw-r--r--drivers/s390/block/dcssblk.c2
-rw-r--r--drivers/s390/char/Kconfig22
-rw-r--r--drivers/s390/char/Makefile4
-rw-r--r--drivers/s390/char/con3215.c1
-rw-r--r--drivers/s390/char/sclp_async.c189
-rw-r--r--drivers/s390/char/sclp_early.c3
-rw-r--r--drivers/s390/char/tape_core.c3
-rw-r--r--drivers/s390/char/vmcp.c2
-rw-r--r--drivers/s390/char/zcore.c2
-rw-r--r--drivers/s390/cio/airq.c37
-rw-r--r--drivers/s390/cio/ccwgroup.c18
-rw-r--r--drivers/s390/cio/ccwreq.c9
-rw-r--r--drivers/s390/cio/chsc.c30
-rw-r--r--drivers/s390/cio/chsc_sch.c2
-rw-r--r--drivers/s390/cio/cio.h3
-rw-r--r--drivers/s390/cio/css.c191
-rw-r--r--drivers/s390/cio/device.c89
-rw-r--r--drivers/s390/cio/device_fsm.c49
-rw-r--r--drivers/s390/cio/device_id.c20
-rw-r--r--drivers/s390/cio/device_ops.c21
-rw-r--r--drivers/s390/cio/device_pgid.c22
-rw-r--r--drivers/s390/cio/device_status.c24
-rw-r--r--drivers/s390/cio/io_sch.h20
-rw-r--r--drivers/s390/cio/qdio.h3
-rw-r--r--drivers/s390/cio/qdio_main.c100
-rw-r--r--drivers/s390/cio/qdio_setup.c4
-rw-r--r--drivers/s390/cio/qdio_thinint.c6
-rw-r--r--drivers/s390/cio/scm.c4
-rw-r--r--drivers/s390/cio/vfio_ccw_async.c2
-rw-r--r--drivers/s390/cio/vfio_ccw_cp.c542
-rw-r--r--drivers/s390/cio/vfio_ccw_cp.h7
-rw-r--r--drivers/s390/cio/vfio_ccw_drv.c73
-rw-r--r--drivers/s390/cio/vfio_ccw_fsm.c51
-rw-r--r--drivers/s390/cio/vfio_ccw_ops.c10
-rw-r--r--drivers/s390/cio/vfio_ccw_private.h17
-rw-r--r--drivers/s390/crypto/Makefile2
-rw-r--r--drivers/s390/crypto/ap_bus.c9
-rw-r--r--drivers/s390/crypto/ap_queue.c1
-rw-r--r--drivers/s390/crypto/pkey_api.c1646
-rw-r--r--drivers/s390/crypto/vfio_ap_drv.c34
-rw-r--r--drivers/s390/crypto/vfio_ap_ops.c379
-rw-r--r--drivers/s390/crypto/vfio_ap_private.h15
-rw-r--r--drivers/s390/crypto/zcrypt_api.c52
-rw-r--r--drivers/s390/crypto/zcrypt_api.h7
-rw-r--r--drivers/s390/crypto/zcrypt_ccamisc.c1765
-rw-r--r--drivers/s390/crypto/zcrypt_ccamisc.h217
-rw-r--r--drivers/s390/crypto/zcrypt_cex4.c106
-rw-r--r--drivers/s390/crypto/zcrypt_msgtype6.c21
-rw-r--r--drivers/s390/net/Kconfig8
-rw-r--r--drivers/s390/net/ctcm_fsms.c43
-rw-r--r--drivers/s390/net/ctcm_main.c12
-rw-r--r--drivers/s390/net/ctcm_mpc.c9
-rw-r--r--drivers/s390/net/lcs.c6
-rw-r--r--drivers/s390/net/qeth_core.h192
-rw-r--r--drivers/s390/net/qeth_core_main.c1646
-rw-r--r--drivers/s390/net/qeth_core_mpc.h52
-rw-r--r--drivers/s390/net/qeth_ethtool.c2
-rw-r--r--drivers/s390/net/qeth_l2_main.c338
-rw-r--r--drivers/s390/net/qeth_l3_main.c282
-rw-r--r--drivers/s390/scsi/zfcp_erp.c7
-rw-r--r--drivers/s390/scsi/zfcp_fc.c4
-rw-r--r--drivers/s390/scsi/zfcp_fsf.c55
-rw-r--r--drivers/s390/virtio/virtio_ccw.c250
-rw-r--r--drivers/scsi/Kconfig65
-rw-r--r--drivers/scsi/Makefile6
-rw-r--r--drivers/scsi/NCR5380.c59
-rw-r--r--drivers/scsi/NCR5380.h2
-rw-r--r--drivers/scsi/advansys.c2
-rw-r--r--drivers/scsi/aha152x.c46
-rw-r--r--drivers/scsi/aic7xxx/aic7xxx.reg2
-rw-r--r--drivers/scsi/aic94xx/aic94xx_dev.c4
-rw-r--r--drivers/scsi/arm/fas216.c8
-rw-r--r--drivers/scsi/bnx2fc/bnx2fc.h14
-rw-r--r--drivers/scsi/bnx2fc/bnx2fc_els.c60
-rw-r--r--drivers/scsi/bnx2fc/bnx2fc_fcoe.c5
-rw-r--r--drivers/scsi/bnx2fc/bnx2fc_io.c116
-rw-r--r--drivers/scsi/bnx2fc/bnx2fc_tgt.c10
-rw-r--r--drivers/scsi/cxgbi/cxgb3i/cxgb3i.c10
-rw-r--r--drivers/scsi/cxgbi/cxgb4i/cxgb4i.c26
-rw-r--r--drivers/scsi/cxgbi/libcxgbi.c15
-rw-r--r--drivers/scsi/cxgbi/libcxgbi.h9
-rw-r--r--drivers/scsi/cxlflash/ocxl_hw.c23
-rw-r--r--drivers/scsi/device_handler/scsi_dh_alua.c7
-rw-r--r--drivers/scsi/esp_scsi.c20
-rw-r--r--drivers/scsi/esp_scsi.h2
-rw-r--r--drivers/scsi/fcoe/fcoe.c3
-rw-r--r--drivers/scsi/fcoe/fcoe_ctlr.c140
-rw-r--r--drivers/scsi/fcoe/fcoe_transport.c4
-rw-r--r--drivers/scsi/fdomain.c597
-rw-r--r--drivers/scsi/fdomain.h114
-rw-r--r--drivers/scsi/fdomain_isa.c222
-rw-r--r--drivers/scsi/fdomain_pci.c68
-rw-r--r--drivers/scsi/hisi_sas/hisi_sas.h8
-rw-r--r--drivers/scsi/hisi_sas/hisi_sas_main.c16
-rw-r--r--drivers/scsi/hisi_sas/hisi_sas_v2_hw.c50
-rw-r--r--drivers/scsi/hisi_sas/hisi_sas_v3_hw.c50
-rw-r--r--drivers/scsi/hosts.c3
-rw-r--r--drivers/scsi/hpsa.c294
-rw-r--r--drivers/scsi/hpsa.h6
-rw-r--r--drivers/scsi/hpsa_cmd.h2
-rw-r--r--drivers/scsi/ibmvscsi/ibmvfc.c2
-rw-r--r--drivers/scsi/ibmvscsi/ibmvscsi.c77
-rw-r--r--drivers/scsi/ibmvscsi/ibmvscsi.h10
-rw-r--r--drivers/scsi/imm.c2
-rw-r--r--drivers/scsi/ipr.c29
-rw-r--r--drivers/scsi/isci/remote_device.c4
-rw-r--r--drivers/scsi/isci/remote_device.h5
-rw-r--r--drivers/scsi/isci/request.c8
-rw-r--r--drivers/scsi/isci/task.c2
-rw-r--r--drivers/scsi/libfc/fc_exch.c2
-rw-r--r--drivers/scsi/libfc/fc_rport.c5
-rw-r--r--drivers/scsi/libiscsi_tcp.c2
-rw-r--r--drivers/scsi/libsas/sas_discover.c24
-rw-r--r--drivers/scsi/libsas/sas_event.c18
-rw-r--r--drivers/scsi/libsas/sas_expander.c71
-rw-r--r--drivers/scsi/libsas/sas_init.c2
-rw-r--r--drivers/scsi/libsas/sas_internal.h2
-rw-r--r--drivers/scsi/libsas/sas_phy.c18
-rw-r--r--drivers/scsi/libsas/sas_port.c24
-rw-r--r--drivers/scsi/libsas/sas_scsi_host.c3
-rw-r--r--drivers/scsi/lpfc/lpfc.h3
-rw-r--r--drivers/scsi/lpfc/lpfc_attr.c109
-rw-r--r--drivers/scsi/lpfc/lpfc_bsg.c2
-rw-r--r--drivers/scsi/lpfc/lpfc_crtn.h7
-rw-r--r--drivers/scsi/lpfc/lpfc_ct.c14
-rw-r--r--drivers/scsi/lpfc/lpfc_debugfs.h2
-rw-r--r--drivers/scsi/lpfc/lpfc_els.c128
-rw-r--r--drivers/scsi/lpfc/lpfc_hbadisc.c35
-rw-r--r--drivers/scsi/lpfc/lpfc_hw.h2
-rw-r--r--drivers/scsi/lpfc/lpfc_init.c547
-rw-r--r--drivers/scsi/lpfc/lpfc_nvme.c60
-rw-r--r--drivers/scsi/lpfc/lpfc_nvmet.c352
-rw-r--r--drivers/scsi/lpfc/lpfc_nvmet.h1
-rw-r--r--drivers/scsi/lpfc/lpfc_scsi.c16
-rw-r--r--drivers/scsi/lpfc/lpfc_sli.c77
-rw-r--r--drivers/scsi/lpfc/lpfc_sli4.h16
-rw-r--r--drivers/scsi/lpfc/lpfc_version.h2
-rw-r--r--drivers/scsi/mac_scsi.c421
-rw-r--r--drivers/scsi/megaraid/Kconfig.megaraid1
-rw-r--r--drivers/scsi/megaraid/Makefile2
-rw-r--r--drivers/scsi/megaraid/megaraid_sas.h101
-rw-r--r--drivers/scsi/megaraid/megaraid_sas_base.c746
-rw-r--r--drivers/scsi/megaraid/megaraid_sas_debugfs.c179
-rw-r--r--drivers/scsi/megaraid/megaraid_sas_fp.c82
-rw-r--r--drivers/scsi/megaraid/megaraid_sas_fusion.c578
-rw-r--r--drivers/scsi/megaraid/megaraid_sas_fusion.h33
-rw-r--r--drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h2
-rw-r--r--drivers/scsi/mpt3sas/mpt3sas_base.c509
-rw-r--r--drivers/scsi/mpt3sas/mpt3sas_base.h35
-rw-r--r--drivers/scsi/mpt3sas/mpt3sas_config.c73
-rw-r--r--drivers/scsi/mpt3sas/mpt3sas_ctl.c234
-rw-r--r--drivers/scsi/mpt3sas/mpt3sas_scsih.c53
-rw-r--r--drivers/scsi/mpt3sas/mpt3sas_transport.c8
-rw-r--r--drivers/scsi/mvsas/mv_sas.c2
-rw-r--r--drivers/scsi/mvsas/mv_sas.h3
-rw-r--r--drivers/scsi/mvumi.c11
-rw-r--r--drivers/scsi/osst.c6108
-rw-r--r--drivers/scsi/osst.h651
-rw-r--r--drivers/scsi/osst_detect.h7
-rw-r--r--drivers/scsi/osst_options.h107
-rw-r--r--drivers/scsi/pcmcia/Kconfig10
-rw-r--r--drivers/scsi/pcmcia/Makefile1
-rw-r--r--drivers/scsi/pcmcia/fdomain_cs.c95
-rw-r--r--drivers/scsi/pcmcia/nsp_cs.c4
-rw-r--r--drivers/scsi/pm8001/pm8001_ctl.c52
-rw-r--r--drivers/scsi/pm8001/pm8001_hwi.c4
-rw-r--r--drivers/scsi/pm8001/pm8001_sas.c10
-rw-r--r--drivers/scsi/pm8001/pm8001_sas.h1
-rw-r--r--drivers/scsi/pm8001/pm80xx_hwi.c6
-rw-r--r--drivers/scsi/pm8001/pm80xx_hwi.h2
-rw-r--r--drivers/scsi/pmcraid.c14
-rw-r--r--drivers/scsi/ppa.c2
-rw-r--r--drivers/scsi/qedf/qedf_main.c41
-rw-r--r--drivers/scsi/qedi/qedi_main.c34
-rw-r--r--drivers/scsi/qla1280.c37
-rw-r--r--drivers/scsi/qla1280.h3
-rw-r--r--drivers/scsi/qla2xxx/qla_attr.c2
-rw-r--r--drivers/scsi/qla2xxx/qla_def.h5
-rw-r--r--drivers/scsi/qla2xxx/qla_gbl.h2
-rw-r--r--drivers/scsi/qla2xxx/qla_init.c17
-rw-r--r--drivers/scsi/qla2xxx/qla_nvme.c236
-rw-r--r--drivers/scsi/qla2xxx/qla_nvme.h2
-rw-r--r--drivers/scsi/qla2xxx/qla_os.c12
-rw-r--r--drivers/scsi/qla2xxx/qla_target.c16
-rw-r--r--drivers/scsi/scsi.c12
-rw-r--r--drivers/scsi/scsi_debugfs.h1
-rw-r--r--drivers/scsi/scsi_devinfo.c2
-rw-r--r--drivers/scsi/scsi_error.c26
-rw-r--r--drivers/scsi/scsi_lib.c67
-rw-r--r--drivers/scsi/scsi_pm.c9
-rw-r--r--drivers/scsi/scsi_priv.h1
-rw-r--r--drivers/scsi/scsi_proc.c9
-rw-r--r--drivers/scsi/scsi_sysfs.c7
-rw-r--r--drivers/scsi/scsi_transport_fc.c3
-rw-r--r--drivers/scsi/sd.c116
-rw-r--r--drivers/scsi/sd.h8
-rw-r--r--drivers/scsi/sd_zbc.c122
-rw-r--r--drivers/scsi/ses.c7
-rw-r--r--drivers/scsi/st.c6
-rw-r--r--drivers/scsi/storvsc_drv.c16
-rw-r--r--drivers/scsi/ufs/ufs-qcom.c23
-rw-r--r--drivers/scsi/ufs/ufs-sysfs.c6
-rw-r--r--drivers/scsi/ufs/ufs_bsg.c6
-rw-r--r--drivers/scsi/ufs/ufshcd-pci.c2
-rw-r--r--drivers/scsi/ufs/ufshcd.c41
-rw-r--r--drivers/scsi/ufs/ufshcd.h5
-rw-r--r--drivers/scsi/ufs/ufshci.h6
-rw-r--r--drivers/scsi/virtio_scsi.c5
-rw-r--r--drivers/scsi/vmw_pvscsi.c2
-rw-r--r--drivers/scsi/wd33c93.c2
-rw-r--r--drivers/scsi/wd719x.c42
-rw-r--r--drivers/slimbus/core.c5
-rw-r--r--drivers/slimbus/qcom-ctrl.c4
-rw-r--r--drivers/slimbus/qcom-ngd-ctrl.c5
-rw-r--r--drivers/slimbus/slimbus.h2
-rw-r--r--drivers/slimbus/stream.c12
-rw-r--r--drivers/sn/Kconfig22
-rw-r--r--drivers/sn/Makefile7
-rw-r--r--drivers/sn/ioc3.c844
-rw-r--r--drivers/soc/amlogic/Kconfig11
-rw-r--r--drivers/soc/amlogic/Makefile1
-rw-r--r--drivers/soc/amlogic/meson-canvas.c14
-rw-r--r--drivers/soc/amlogic/meson-clk-measure.c148
-rw-r--r--drivers/soc/amlogic/meson-ee-pwrc.c492
-rw-r--r--drivers/soc/amlogic/meson-gx-socinfo.c7
-rw-r--r--drivers/soc/aspeed/aspeed-lpc-ctrl.c61
-rw-r--r--drivers/soc/fsl/Kconfig10
-rw-r--r--drivers/soc/fsl/Makefile1
-rw-r--r--drivers/soc/fsl/dpaa2-console.c329
-rw-r--r--drivers/soc/fsl/dpio/dpio-driver.c23
-rw-r--r--drivers/soc/fsl/dpio/dpio-service.c2
-rw-r--r--drivers/soc/fsl/dpio/qbman-portal.c148
-rw-r--r--drivers/soc/fsl/dpio/qbman-portal.h9
-rw-r--r--drivers/soc/fsl/guts.c12
-rw-r--r--drivers/soc/fsl/qbman/bman.c17
-rw-r--r--drivers/soc/fsl/qbman/bman_ccsr.c36
-rw-r--r--drivers/soc/fsl/qbman/bman_portal.c42
-rw-r--r--drivers/soc/fsl/qbman/bman_priv.h5
-rw-r--r--drivers/soc/fsl/qbman/dpaa_sys.c63
-rw-r--r--drivers/soc/fsl/qbman/qman.c83
-rw-r--r--drivers/soc/fsl/qbman/qman_ccsr.c70
-rw-r--r--drivers/soc/fsl/qbman/qman_portal.c43
-rw-r--r--drivers/soc/fsl/qbman/qman_priv.h17
-rw-r--r--drivers/soc/fsl/qe/qe.c165
-rw-r--r--drivers/soc/imx/Kconfig9
-rw-r--r--drivers/soc/imx/Makefile1
-rw-r--r--drivers/soc/imx/gpcv2.c2
-rw-r--r--drivers/soc/imx/soc-imx-scu.c183
-rw-r--r--drivers/soc/imx/soc-imx8.c111
-rw-r--r--drivers/soc/ixp4xx/Kconfig4
-rw-r--r--drivers/soc/mediatek/mtk-cmdq-helper.c10
-rw-r--r--drivers/soc/qcom/Kconfig20
-rw-r--r--drivers/soc/qcom/Makefile2
-rw-r--r--drivers/soc/qcom/apr.c76
-rw-r--r--drivers/soc/qcom/mdt_loader.c88
-rw-r--r--drivers/soc/qcom/qcom-geni-se.c27
-rw-r--r--drivers/soc/qcom/qcom_aoss.c613
-rw-r--r--drivers/soc/qcom/rpmpd.c134
-rw-r--r--drivers/soc/qcom/smem.c11
-rw-r--r--drivers/soc/qcom/socinfo.c476
-rw-r--r--drivers/soc/renesas/Kconfig15
-rw-r--r--drivers/soc/renesas/rcar-sysc.c7
-rw-r--r--drivers/soc/renesas/rmobile-sysc.c31
-rw-r--r--drivers/soc/rockchip/pm_domains.c230
-rw-r--r--drivers/soc/samsung/Kconfig6
-rw-r--r--drivers/soc/samsung/Makefile2
-rw-r--r--drivers/soc/samsung/exynos-chipid.c105
-rw-r--r--drivers/soc/tegra/Kconfig1
-rw-r--r--drivers/soc/tegra/fuse/fuse-tegra.c6
-rw-r--r--drivers/soc/tegra/fuse/fuse-tegra20.c2
-rw-r--r--drivers/soc/tegra/pmc.c19
-rw-r--r--drivers/soc/ti/Kconfig5
-rw-r--r--drivers/soc/ti/pm33xx.c20
-rw-r--r--drivers/soc/ti/ti_sci_pm_domains.c23
-rw-r--r--drivers/soc/ux500/ux500-soc-id.c5
-rw-r--r--drivers/soundwire/Kconfig7
-rw-r--r--drivers/soundwire/Makefile2
-rw-r--r--drivers/soundwire/bus.c6
-rw-r--r--drivers/soundwire/cadence_master.c38
-rw-r--r--drivers/soundwire/intel.c17
-rw-r--r--drivers/soundwire/intel.h2
-rw-r--r--drivers/soundwire/intel_init.c25
-rw-r--r--drivers/soundwire/mipi_disco.c35
-rw-r--r--drivers/soundwire/stream.c8
-rw-r--r--drivers/spi/Kconfig33
-rw-r--r--drivers/spi/Makefile3
-rw-r--r--drivers/spi/atmel-quadspi.c22
-rw-r--r--drivers/spi/spi-altera.c4
-rw-r--r--drivers/spi/spi-armada-3700.c5
-rw-r--r--drivers/spi/spi-at91-usart.c221
-rw-r--r--drivers/spi/spi-ath79.c4
-rw-r--r--drivers/spi/spi-atmel.c5
-rw-r--r--drivers/spi/spi-axi-spi-engine.c4
-rw-r--r--drivers/spi/spi-bcm-qspi.c5
-rw-r--r--drivers/spi/spi-bcm2835.c763
-rw-r--r--drivers/spi/spi-bcm2835aux.c9
-rw-r--r--drivers/spi/spi-bcm63xx-hsspi.c8
-rw-r--r--drivers/spi/spi-bcm63xx.c4
-rw-r--r--drivers/spi/spi-butterfly.c2
-rw-r--r--drivers/spi/spi-cadence.c5
-rw-r--r--drivers/spi/spi-cavium-octeon.c4
-rw-r--r--drivers/spi/spi-clps711x.c4
-rw-r--r--drivers/spi/spi-coldfire-qspi.c4
-rw-r--r--drivers/spi/spi-dw-mmio.c18
-rw-r--r--drivers/spi/spi-dw-pci.c20
-rw-r--r--drivers/spi/spi-efm32.c4
-rw-r--r--drivers/spi/spi-ep93xx.c4
-rw-r--r--drivers/spi/spi-fsl-cpm.c6
-rw-r--r--drivers/spi/spi-fsl-dspi.c718
-rw-r--r--drivers/spi/spi-fsl-lib.h3
-rw-r--r--drivers/spi/spi-fsl-qspi.c6
-rw-r--r--drivers/spi/spi-fsl-spi.c193
-rw-r--r--drivers/spi/spi-geni-qcom.c8
-rw-r--r--drivers/spi/spi-gpio.c11
-rw-r--r--drivers/spi/spi-lantiq-ssc.c12
-rw-r--r--drivers/spi/spi-lm70llp.c2
-rw-r--r--drivers/spi/spi-lp8841-rtc.c4
-rw-r--r--drivers/spi/spi-meson-spicc.c4
-rw-r--r--drivers/spi/spi-meson-spifc.c16
-rw-r--r--drivers/spi/spi-mt65xx.c69
-rw-r--r--drivers/spi/spi-mt7621.c4
-rw-r--r--drivers/spi/spi-mxs.c4
-rw-r--r--drivers/spi/spi-npcm-fiu.c769
-rw-r--r--drivers/spi/spi-npcm-pspi.c5
-rw-r--r--drivers/spi/spi-nuc900.c429
-rw-r--r--drivers/spi/spi-nxp-fspi.c4
-rw-r--r--drivers/spi/spi-oc-tiny.c4
-rw-r--r--drivers/spi/spi-pic32-sqi.c5
-rw-r--r--drivers/spi/spi-pic32.c12
-rw-r--r--drivers/spi/spi-pxa2xx.c28
-rw-r--r--drivers/spi/spi-qcom-qspi.c8
-rw-r--r--drivers/spi/spi-qup.c55
-rw-r--r--drivers/spi/spi-rb4xx.c4
-rw-r--r--drivers/spi/spi-rockchip.c4
-rw-r--r--drivers/spi/spi-s3c24xx.c5
-rw-r--r--drivers/spi/spi-sh-msiof.c7
-rw-r--r--drivers/spi/spi-sh.c4
-rw-r--r--drivers/spi/spi-sifive.c5
-rw-r--r--drivers/spi/spi-sirf.c4
-rw-r--r--drivers/spi/spi-slave-mt27xx.c1
-rw-r--r--drivers/spi/spi-sprd-adi.c92
-rw-r--r--drivers/spi/spi-sprd.c4
-rw-r--r--drivers/spi/spi-st-ssc4.c4
-rw-r--r--drivers/spi/spi-stm32-qspi.c19
-rw-r--r--drivers/spi/spi-sun4i.c5
-rw-r--r--drivers/spi/spi-sun6i.c5
-rw-r--r--drivers/spi/spi-synquacer.c826
-rw-r--r--drivers/spi/spi-tegra114.c170
-rw-r--r--drivers/spi/spi-tegra20-sflash.c4
-rw-r--r--drivers/spi/spi-ti-qspi.c1
-rw-r--r--drivers/spi/spi-uniphier.c99
-rw-r--r--drivers/spi/spi-xlp.c8
-rw-r--r--drivers/spi/spi-zynq-qspi.c7
-rw-r--r--drivers/spi/spi-zynqmp-gqspi.c5
-rw-r--r--drivers/spi/spi.c311
-rw-r--r--drivers/spi/spidev.c2
-rw-r--r--drivers/ssb/driver_gpio.c6
-rw-r--r--drivers/ssb/pci.c2
-rw-r--r--drivers/staging/Kconfig11
-rw-r--r--drivers/staging/Makefile6
-rw-r--r--drivers/staging/android/TODO2
-rw-r--r--drivers/staging/android/ion/Kconfig18
-rw-r--r--drivers/staging/android/ion/Makefile2
-rw-r--r--drivers/staging/android/ion/ion.c36
-rw-r--r--drivers/staging/android/ion/ion.h10
-rw-r--r--drivers/staging/android/ion/ion_carveout_heap.c133
-rw-r--r--drivers/staging/android/ion/ion_chunk_heap.c146
-rw-r--r--drivers/staging/android/ion/ion_page_pool.c3
-rw-r--r--drivers/staging/android/vsoc.c8
-rw-r--r--drivers/staging/comedi/comedi_buf.c150
-rw-r--r--drivers/staging/comedi/comedi_fops.c39
-rw-r--r--drivers/staging/comedi/drivers/amplc_dio200_common.c16
-rw-r--r--drivers/staging/comedi/drivers/amplc_pci230.c3
-rw-r--r--drivers/staging/comedi/drivers/daqboard2000.c7
-rw-r--r--drivers/staging/comedi/drivers/dt282x.c3
-rw-r--r--drivers/staging/comedi/drivers/dt3000.c8
-rw-r--r--drivers/staging/comedi/drivers/mite.c27
-rw-r--r--drivers/staging/comedi/drivers/ni_mio_common.c2
-rw-r--r--drivers/staging/comedi/drivers/usbdux.c2
-rw-r--r--drivers/staging/comedi/drivers/usbduxsigma.c2
-rw-r--r--drivers/staging/emxx_udc/emxx_udc.c80
-rw-r--r--drivers/staging/emxx_udc/emxx_udc.h5
-rw-r--r--drivers/staging/erofs/Documentation/filesystems/erofs.txt209
-rw-r--r--drivers/staging/erofs/Kconfig151
-rw-r--r--drivers/staging/erofs/Makefile13
-rw-r--r--drivers/staging/erofs/TODO46
-rw-r--r--drivers/staging/erofs/data.c400
-rw-r--r--drivers/staging/erofs/dir.c152
-rw-r--r--drivers/staging/erofs/erofs_fs.h282
-rw-r--r--drivers/staging/erofs/include/linux/tagptr.h110
-rw-r--r--drivers/staging/erofs/include/trace/events/erofs.h256
-rw-r--r--drivers/staging/erofs/inode.c310
-rw-r--r--drivers/staging/erofs/internal.h610
-rw-r--r--drivers/staging/erofs/namei.c255
-rw-r--r--drivers/staging/erofs/super.c701
-rw-r--r--drivers/staging/erofs/unzip_pagevec.h172
-rw-r--r--drivers/staging/erofs/unzip_vle.c1888
-rw-r--r--drivers/staging/erofs/unzip_vle.h228
-rw-r--r--drivers/staging/erofs/unzip_vle_lz4.c229
-rw-r--r--drivers/staging/erofs/utils.c341
-rw-r--r--drivers/staging/erofs/xattr.c704
-rw-r--r--drivers/staging/erofs/xattr.h97
-rw-r--r--drivers/staging/exfat/Kconfig49
-rw-r--r--drivers/staging/exfat/Makefile10
-rw-r--r--drivers/staging/exfat/TODO12
-rw-r--r--drivers/staging/exfat/exfat.h971
-rw-r--r--drivers/staging/exfat/exfat_blkdev.c136
-rw-r--r--drivers/staging/exfat/exfat_cache.c724
-rw-r--r--drivers/staging/exfat/exfat_core.c3701
-rw-r--r--drivers/staging/exfat/exfat_nls.c404
-rw-r--r--drivers/staging/exfat/exfat_super.c4049
-rw-r--r--drivers/staging/exfat/exfat_upcase.c740
-rw-r--r--drivers/staging/fbtft/fb_bd663474.c2
-rw-r--r--drivers/staging/fbtft/fb_hx8340bn.c2
-rw-r--r--drivers/staging/fbtft/fb_hx8347d.c2
-rw-r--r--drivers/staging/fbtft/fb_ili9163.c4
-rw-r--r--drivers/staging/fbtft/fb_ili9320.c2
-rw-r--r--drivers/staging/fbtft/fb_ili9325.c4
-rw-r--r--drivers/staging/fbtft/fb_pcd8544.c4
-rw-r--r--drivers/staging/fbtft/fb_s6d1121.c4
-rw-r--r--drivers/staging/fbtft/fb_ssd1289.c4
-rw-r--r--drivers/staging/fbtft/fb_ssd1331.c4
-rw-r--r--drivers/staging/fbtft/fb_upd161704.c2
-rw-r--r--drivers/staging/fbtft/fbtft-bus.c2
-rw-r--r--drivers/staging/fbtft/fbtft-core.c51
-rw-r--r--drivers/staging/fieldbus/Documentation/devicetree/bindings/fieldbus/arcx,anybus-controller.txt71
-rw-r--r--drivers/staging/fieldbus/Documentation/fieldbus_dev.txt4
-rw-r--r--drivers/staging/fieldbus/anybuss/Kconfig1
-rw-r--r--drivers/staging/fieldbus/anybuss/arcx-anybus.c44
-rw-r--r--drivers/staging/fieldbus/dev_core.c6
-rw-r--r--drivers/staging/fsl-dpaa2/Kconfig8
-rw-r--r--drivers/staging/fsl-dpaa2/ethsw/TODO1
-rw-r--r--drivers/staging/fsl-dpaa2/ethsw/dpsw-cmd.h15
-rw-r--r--drivers/staging/fsl-dpaa2/ethsw/dpsw.c51
-rw-r--r--drivers/staging/fsl-dpaa2/ethsw/dpsw.h56
-rw-r--r--drivers/staging/fsl-dpaa2/ethsw/ethsw-ethtool.c44
-rw-r--r--drivers/staging/fsl-dpaa2/ethsw/ethsw.c310
-rw-r--r--drivers/staging/fsl-dpaa2/ethsw/ethsw.h4
-rw-r--r--drivers/staging/gasket/apex_driver.c22
-rw-r--r--drivers/staging/gasket/gasket_core.c6
-rw-r--r--drivers/staging/gasket/gasket_ioctl.c15
-rw-r--r--drivers/staging/gasket/gasket_page_table.c14
-rw-r--r--drivers/staging/goldfish/goldfish_audio.c4
-rw-r--r--drivers/staging/greybus/Documentation/firmware/authenticate.c46
-rw-r--r--drivers/staging/greybus/Documentation/firmware/firmware.c46
-rw-r--r--drivers/staging/greybus/Kconfig27
-rw-r--r--drivers/staging/greybus/Makefile22
-rw-r--r--drivers/staging/greybus/arche-platform.c2
-rw-r--r--drivers/staging/greybus/arpc.h109
-rw-r--r--drivers/staging/greybus/audio_apbridgea.c3
-rw-r--r--drivers/staging/greybus/audio_apbridgea.h26
-rw-r--r--drivers/staging/greybus/audio_codec.h4
-rw-r--r--drivers/staging/greybus/audio_gb.c4
-rw-r--r--drivers/staging/greybus/audio_manager.c2
-rw-r--r--drivers/staging/greybus/authentication.c3
-rw-r--r--drivers/staging/greybus/bootrom.c2
-rw-r--r--drivers/staging/greybus/bundle.h89
-rw-r--r--drivers/staging/greybus/camera.c2
-rw-r--r--drivers/staging/greybus/connection.h128
-rw-r--r--drivers/staging/greybus/control.h57
-rw-r--r--drivers/staging/greybus/firmware.h4
-rw-r--r--drivers/staging/greybus/fw-core.c2
-rw-r--r--drivers/staging/greybus/fw-download.c2
-rw-r--r--drivers/staging/greybus/fw-management.c2
-rw-r--r--drivers/staging/greybus/gb-camera.h2
-rw-r--r--drivers/staging/greybus/gbphy.c2
-rw-r--r--drivers/staging/greybus/gbphy.h2
-rw-r--r--drivers/staging/greybus/gpio.c2
-rw-r--r--drivers/staging/greybus/greybus.h152
-rw-r--r--drivers/staging/greybus/greybus_authentication.h48
-rw-r--r--drivers/staging/greybus/greybus_firmware.h48
-rw-r--r--drivers/staging/greybus/greybus_id.h27
-rw-r--r--drivers/staging/greybus/greybus_manifest.h178
-rw-r--r--drivers/staging/greybus/greybus_protocols.h2222
-rw-r--r--drivers/staging/greybus/hd.h82
-rw-r--r--drivers/staging/greybus/hid.c3
-rw-r--r--drivers/staging/greybus/i2c.c24
-rw-r--r--drivers/staging/greybus/interface.h82
-rw-r--r--drivers/staging/greybus/light.c16
-rw-r--r--drivers/staging/greybus/log.c9
-rw-r--r--drivers/staging/greybus/loopback.c9
-rw-r--r--drivers/staging/greybus/manifest.h15
-rw-r--r--drivers/staging/greybus/module.h33
-rw-r--r--drivers/staging/greybus/operation.h224
-rw-r--r--drivers/staging/greybus/power_supply.c3
-rw-r--r--drivers/staging/greybus/pwm.c2
-rw-r--r--drivers/staging/greybus/raw.c3
-rw-r--r--drivers/staging/greybus/sdio.c2
-rw-r--r--drivers/staging/greybus/spi.c2
-rw-r--r--drivers/staging/greybus/spilib.c2
-rw-r--r--drivers/staging/greybus/spilib.h2
-rw-r--r--drivers/staging/greybus/svc.h101
-rw-r--r--drivers/staging/greybus/tools/loopback_test.c8
-rw-r--r--drivers/staging/greybus/uart.c2
-rw-r--r--drivers/staging/greybus/usb.c2
-rw-r--r--drivers/staging/greybus/vibrator.c3
-rw-r--r--drivers/staging/iio/accel/adis16203.c12
-rw-r--r--drivers/staging/iio/accel/adis16240.c14
-rw-r--r--drivers/staging/iio/adc/Kconfig3
-rw-r--r--drivers/staging/iio/adc/ad7192.c175
-rw-r--r--drivers/staging/iio/adc/ad7192.h37
-rw-r--r--drivers/staging/iio/addac/adt7316-spi.c13
-rw-r--r--drivers/staging/iio/addac/adt7316.c2
-rw-r--r--drivers/staging/iio/cdc/ad7150.c58
-rw-r--r--drivers/staging/iio/cdc/ad7746.c10
-rw-r--r--drivers/staging/iio/frequency/ad9834.c11
-rw-r--r--drivers/staging/iio/resolver/ad2s1210.c15
-rw-r--r--drivers/staging/isdn/Kconfig12
-rw-r--r--drivers/staging/isdn/Makefile8
-rw-r--r--drivers/staging/isdn/TODO22
-rw-r--r--drivers/staging/isdn/avm/Kconfig (renamed from drivers/isdn/hardware/avm/Kconfig)0
-rw-r--r--drivers/staging/isdn/avm/Makefile (renamed from drivers/isdn/hardware/avm/Makefile)0
-rw-r--r--drivers/staging/isdn/avm/avm_cs.c (renamed from drivers/isdn/hardware/avm/avm_cs.c)0
-rw-r--r--drivers/staging/isdn/avm/avmcard.h (renamed from drivers/isdn/hardware/avm/avmcard.h)0
-rw-r--r--drivers/staging/isdn/avm/b1.c (renamed from drivers/isdn/hardware/avm/b1.c)0
-rw-r--r--drivers/staging/isdn/avm/b1dma.c (renamed from drivers/isdn/hardware/avm/b1dma.c)0
-rw-r--r--drivers/staging/isdn/avm/b1isa.c (renamed from drivers/isdn/hardware/avm/b1isa.c)0
-rw-r--r--drivers/staging/isdn/avm/b1pci.c (renamed from drivers/isdn/hardware/avm/b1pci.c)0
-rw-r--r--drivers/staging/isdn/avm/b1pcmcia.c (renamed from drivers/isdn/hardware/avm/b1pcmcia.c)0
-rw-r--r--drivers/staging/isdn/avm/c4.c (renamed from drivers/isdn/hardware/avm/c4.c)0
-rw-r--r--drivers/staging/isdn/avm/t1isa.c (renamed from drivers/isdn/hardware/avm/t1isa.c)0
-rw-r--r--drivers/staging/isdn/avm/t1pci.c (renamed from drivers/isdn/hardware/avm/t1pci.c)0
-rw-r--r--drivers/staging/isdn/gigaset/Kconfig (renamed from drivers/isdn/gigaset/Kconfig)9
-rw-r--r--drivers/staging/isdn/gigaset/Makefile (renamed from drivers/isdn/gigaset/Makefile)10
-rw-r--r--drivers/staging/isdn/gigaset/asyncdata.c (renamed from drivers/isdn/gigaset/asyncdata.c)0
-rw-r--r--drivers/staging/isdn/gigaset/bas-gigaset.c (renamed from drivers/isdn/gigaset/bas-gigaset.c)0
-rw-r--r--drivers/staging/isdn/gigaset/capi.c (renamed from drivers/isdn/gigaset/capi.c)0
-rw-r--r--drivers/staging/isdn/gigaset/common.c (renamed from drivers/isdn/gigaset/common.c)0
-rw-r--r--drivers/staging/isdn/gigaset/dummyll.c (renamed from drivers/isdn/gigaset/dummyll.c)0
-rw-r--r--drivers/staging/isdn/gigaset/ev-layer.c (renamed from drivers/isdn/gigaset/ev-layer.c)0
-rw-r--r--drivers/staging/isdn/gigaset/gigaset.h (renamed from drivers/isdn/gigaset/gigaset.h)0
-rw-r--r--drivers/staging/isdn/gigaset/interface.c (renamed from drivers/isdn/gigaset/interface.c)0
-rw-r--r--drivers/staging/isdn/gigaset/isocdata.c (renamed from drivers/isdn/gigaset/isocdata.c)0
-rw-r--r--drivers/staging/isdn/gigaset/proc.c (renamed from drivers/isdn/gigaset/proc.c)0
-rw-r--r--drivers/staging/isdn/gigaset/ser-gigaset.c (renamed from drivers/isdn/gigaset/ser-gigaset.c)0
-rw-r--r--drivers/staging/isdn/gigaset/usb-gigaset.c (renamed from drivers/isdn/gigaset/usb-gigaset.c)0
-rw-r--r--drivers/staging/isdn/hysdn/Kconfig (renamed from drivers/isdn/hysdn/Kconfig)2
-rw-r--r--drivers/staging/isdn/hysdn/Makefile (renamed from drivers/isdn/hysdn/Makefile)0
-rw-r--r--drivers/staging/isdn/hysdn/boardergo.c (renamed from drivers/isdn/hysdn/boardergo.c)0
-rw-r--r--drivers/staging/isdn/hysdn/boardergo.h (renamed from drivers/isdn/hysdn/boardergo.h)0
-rw-r--r--drivers/staging/isdn/hysdn/hycapi.c (renamed from drivers/isdn/hysdn/hycapi.c)0
-rw-r--r--drivers/staging/isdn/hysdn/hysdn_boot.c (renamed from drivers/isdn/hysdn/hysdn_boot.c)0
-rw-r--r--drivers/staging/isdn/hysdn/hysdn_defs.h (renamed from drivers/isdn/hysdn/hysdn_defs.h)0
-rw-r--r--drivers/staging/isdn/hysdn/hysdn_init.c (renamed from drivers/isdn/hysdn/hysdn_init.c)0
-rw-r--r--drivers/staging/isdn/hysdn/hysdn_net.c (renamed from drivers/isdn/hysdn/hysdn_net.c)8
-rw-r--r--drivers/staging/isdn/hysdn/hysdn_pof.h (renamed from drivers/isdn/hysdn/hysdn_pof.h)0
-rw-r--r--drivers/staging/isdn/hysdn/hysdn_procconf.c (renamed from drivers/isdn/hysdn/hysdn_procconf.c)2
-rw-r--r--drivers/staging/isdn/hysdn/hysdn_proclog.c (renamed from drivers/isdn/hysdn/hysdn_proclog.c)0
-rw-r--r--drivers/staging/isdn/hysdn/hysdn_sched.c (renamed from drivers/isdn/hysdn/hysdn_sched.c)0
-rw-r--r--drivers/staging/isdn/hysdn/ince1pc.h (renamed from drivers/isdn/hysdn/ince1pc.h)0
-rw-r--r--drivers/staging/kpc2000/Kconfig4
-rw-r--r--drivers/staging/kpc2000/Makefile4
-rw-r--r--drivers/staging/kpc2000/TODO6
-rw-r--r--drivers/staging/kpc2000/kpc2000/Makefile2
-rw-r--r--drivers/staging/kpc2000/kpc2000/cell_probe.c750
-rw-r--r--drivers/staging/kpc2000/kpc2000/core.c892
-rw-r--r--drivers/staging/kpc2000/kpc2000/dma_common_defs.h19
-rw-r--r--drivers/staging/kpc2000/kpc2000/fileops.c131
-rw-r--r--drivers/staging/kpc2000/kpc2000/kp2000_module.c54
-rw-r--r--drivers/staging/kpc2000/kpc2000/pcie.h119
-rw-r--r--drivers/staging/kpc2000/kpc2000/uapi.h22
-rw-r--r--drivers/staging/kpc2000/kpc2000_i2c.c651
-rw-r--r--drivers/staging/kpc2000/kpc2000_spi.c519
-rw-r--r--drivers/staging/kpc2000/kpc_dma/dma.c142
-rw-r--r--drivers/staging/kpc2000/kpc_dma/fileops.c277
-rw-r--r--drivers/staging/kpc2000/kpc_dma/kpc_dma_driver.c122
-rw-r--r--drivers/staging/kpc2000/kpc_dma/kpc_dma_driver.h34
-rw-r--r--drivers/staging/kpc2000/kpc_i2c/Makefile4
-rw-r--r--drivers/staging/kpc2000/kpc_i2c/fileops.c181
-rw-r--r--drivers/staging/kpc2000/kpc_i2c/i2c_driver.c699
-rw-r--r--drivers/staging/kpc2000/kpc_spi/Makefile4
-rw-r--r--drivers/staging/kpc2000/kpc_spi/spi_driver.c507
-rw-r--r--drivers/staging/kpc2000/kpc_spi/spi_parts.h48
-rw-r--r--drivers/staging/ks7010/ks7010_sdio.c2
-rw-r--r--drivers/staging/ks7010/ks_hostif.c7
-rw-r--r--drivers/staging/media/Kconfig8
-rw-r--r--drivers/staging/media/Makefile6
-rw-r--r--drivers/staging/media/allegro-dvt/Kconfig16
-rw-r--r--drivers/staging/media/allegro-dvt/Makefile5
-rw-r--r--drivers/staging/media/allegro-dvt/TODO4
-rw-r--r--drivers/staging/media/allegro-dvt/allegro-core.c3012
-rw-r--r--drivers/staging/media/allegro-dvt/nal-h264.c1001
-rw-r--r--drivers/staging/media/allegro-dvt/nal-h264.h208
-rw-r--r--drivers/staging/media/bcm2048/Kconfig14
-rw-r--r--drivers/staging/media/bcm2048/Makefile2
-rw-r--r--drivers/staging/media/bcm2048/TODO24
-rw-r--r--drivers/staging/media/bcm2048/radio-bcm2048.c2692
-rw-r--r--drivers/staging/media/bcm2048/radio-bcm2048.h26
-rw-r--r--drivers/staging/media/davinci_vpfe/Kconfig13
-rw-r--r--drivers/staging/media/davinci_vpfe/Makefile11
-rw-r--r--drivers/staging/media/davinci_vpfe/TODO38
-rw-r--r--drivers/staging/media/davinci_vpfe/davinci-vpfe-mc.txt154
-rw-r--r--drivers/staging/media/davinci_vpfe/davinci_vpfe_user.h1287
-rw-r--r--drivers/staging/media/davinci_vpfe/dm365_ipipe.c1847
-rw-r--r--drivers/staging/media/davinci_vpfe/dm365_ipipe.h174
-rw-r--r--drivers/staging/media/davinci_vpfe/dm365_ipipe_hw.c1038
-rw-r--r--drivers/staging/media/davinci_vpfe/dm365_ipipe_hw.h556
-rw-r--r--drivers/staging/media/davinci_vpfe/dm365_ipipeif.c1070
-rw-r--r--drivers/staging/media/davinci_vpfe/dm365_ipipeif.h228
-rw-r--r--drivers/staging/media/davinci_vpfe/dm365_ipipeif_user.h90
-rw-r--r--drivers/staging/media/davinci_vpfe/dm365_isif.c2097
-rw-r--r--drivers/staging/media/davinci_vpfe/dm365_isif.h200
-rw-r--r--drivers/staging/media/davinci_vpfe/dm365_isif_regs.h291
-rw-r--r--drivers/staging/media/davinci_vpfe/dm365_resizer.c1995
-rw-r--r--drivers/staging/media/davinci_vpfe/dm365_resizer.h241
-rw-r--r--drivers/staging/media/davinci_vpfe/vpfe.h83
-rw-r--r--drivers/staging/media/davinci_vpfe/vpfe_mc_capture.c716
-rw-r--r--drivers/staging/media/davinci_vpfe/vpfe_mc_capture.h90
-rw-r--r--drivers/staging/media/davinci_vpfe/vpfe_video.c1642
-rw-r--r--drivers/staging/media/davinci_vpfe/vpfe_video.h150
-rw-r--r--drivers/staging/media/hantro/Kconfig23
-rw-r--r--drivers/staging/media/hantro/Makefile20
-rw-r--r--drivers/staging/media/hantro/TODO (renamed from drivers/staging/media/rockchip/vpu/TODO)0
-rw-r--r--drivers/staging/media/hantro/hantro.h396
-rw-r--r--drivers/staging/media/hantro/hantro_drv.c938
-rw-r--r--drivers/staging/media/hantro/hantro_g1_h264_dec.c292
-rw-r--r--drivers/staging/media/hantro/hantro_g1_mpeg2_dec.c254
-rw-r--r--drivers/staging/media/hantro/hantro_g1_regs.h301
-rw-r--r--drivers/staging/media/hantro/hantro_g1_vp8_dec.c503
-rw-r--r--drivers/staging/media/hantro/hantro_h1_jpeg_enc.c128
-rw-r--r--drivers/staging/media/hantro/hantro_h1_regs.h154
-rw-r--r--drivers/staging/media/hantro/hantro_h264.c646
-rw-r--r--drivers/staging/media/hantro/hantro_hw.h182
-rw-r--r--drivers/staging/media/hantro/hantro_jpeg.c (renamed from drivers/staging/media/rockchip/vpu/rockchip_vpu_jpeg.c)41
-rw-r--r--drivers/staging/media/hantro/hantro_jpeg.h13
-rw-r--r--drivers/staging/media/hantro/hantro_mpeg2.c61
-rw-r--r--drivers/staging/media/hantro/hantro_v4l2.c697
-rw-r--r--drivers/staging/media/hantro/hantro_v4l2.h26
-rw-r--r--drivers/staging/media/hantro/hantro_vp8.c201
-rw-r--r--drivers/staging/media/hantro/rk3288_vpu_hw.c226
-rw-r--r--drivers/staging/media/hantro/rk3399_vpu_hw.c223
-rw-r--r--drivers/staging/media/hantro/rk3399_vpu_hw_jpeg_enc.c (renamed from drivers/staging/media/rockchip/vpu/rk3399_vpu_hw_jpeg_enc.c)44
-rw-r--r--drivers/staging/media/hantro/rk3399_vpu_hw_mpeg2_dec.c260
-rw-r--r--drivers/staging/media/hantro/rk3399_vpu_hw_vp8_dec.c595
-rw-r--r--drivers/staging/media/hantro/rk3399_vpu_regs.h (renamed from drivers/staging/media/rockchip/vpu/rk3399_vpu_regs.h)2
-rw-r--r--drivers/staging/media/imx/Kconfig5
-rw-r--r--drivers/staging/media/imx/Makefile19
-rw-r--r--drivers/staging/media/imx/imx-ic-common.c68
-rw-r--r--drivers/staging/media/imx/imx-ic-prp.c42
-rw-r--r--drivers/staging/media/imx/imx-ic-prpencvf.c132
-rw-r--r--drivers/staging/media/imx/imx-ic.h6
-rw-r--r--drivers/staging/media/imx/imx-media-capture.c97
-rw-r--r--drivers/staging/media/imx/imx-media-csc-scaler.c925
-rw-r--r--drivers/staging/media/imx/imx-media-csi.c70
-rw-r--r--drivers/staging/media/imx/imx-media-dev-common.c346
-rw-r--r--drivers/staging/media/imx/imx-media-dev.c447
-rw-r--r--drivers/staging/media/imx/imx-media-fim.c9
-rw-r--r--drivers/staging/media/imx/imx-media-internal-sd.c361
-rw-r--r--drivers/staging/media/imx/imx-media-of.c41
-rw-r--r--drivers/staging/media/imx/imx-media-utils.c243
-rw-r--r--drivers/staging/media/imx/imx-media-vdic.c89
-rw-r--r--drivers/staging/media/imx/imx-media.h127
-rw-r--r--drivers/staging/media/imx/imx6-mipi-csi2.c12
-rw-r--r--drivers/staging/media/imx/imx7-media-csi.c209
-rw-r--r--drivers/staging/media/imx/imx7-mipi-csis.c45
-rw-r--r--drivers/staging/media/ipu3/include/intel-ipu3.h2
-rw-r--r--drivers/staging/media/ipu3/ipu3-css-fw.c6
-rw-r--r--drivers/staging/media/ipu3/ipu3-css.c14
-rw-r--r--drivers/staging/media/ipu3/ipu3-dmamap.c15
-rw-r--r--drivers/staging/media/ipu3/ipu3-mmu.c125
-rw-r--r--drivers/staging/media/ipu3/ipu3-mmu.h5
-rw-r--r--drivers/staging/media/ipu3/ipu3-tables.h4
-rw-r--r--drivers/staging/media/ipu3/ipu3-v4l2.c4
-rw-r--r--drivers/staging/media/ipu3/ipu3.c3
-rw-r--r--drivers/staging/media/meson/vdec/Kconfig11
-rw-r--r--drivers/staging/media/meson/vdec/Makefile8
-rw-r--r--drivers/staging/media/meson/vdec/TODO8
-rw-r--r--drivers/staging/media/meson/vdec/codec_mpeg12.c210
-rw-r--r--drivers/staging/media/meson/vdec/codec_mpeg12.h14
-rw-r--r--drivers/staging/media/meson/vdec/dos_regs.h98
-rw-r--r--drivers/staging/media/meson/vdec/esparser.c322
-rw-r--r--drivers/staging/media/meson/vdec/esparser.h32
-rw-r--r--drivers/staging/media/meson/vdec/vdec.c1099
-rw-r--r--drivers/staging/media/meson/vdec/vdec.h267
-rw-r--r--drivers/staging/media/meson/vdec/vdec_1.c230
-rw-r--r--drivers/staging/media/meson/vdec/vdec_1.h14
-rw-r--r--drivers/staging/media/meson/vdec/vdec_helpers.c449
-rw-r--r--drivers/staging/media/meson/vdec/vdec_helpers.h83
-rw-r--r--drivers/staging/media/meson/vdec/vdec_platform.c101
-rw-r--r--drivers/staging/media/meson/vdec/vdec_platform.h30
-rw-r--r--drivers/staging/media/omap4iss/iss.c1
-rw-r--r--drivers/staging/media/omap4iss/iss_video.c51
-rw-r--r--drivers/staging/media/omap4iss/iss_video.h2
-rw-r--r--drivers/staging/media/rockchip/vpu/Kconfig13
-rw-r--r--drivers/staging/media/rockchip/vpu/Makefile11
-rw-r--r--drivers/staging/media/rockchip/vpu/rk3288_vpu_hw.c118
-rw-r--r--drivers/staging/media/rockchip/vpu/rk3288_vpu_hw_jpeg_enc.c125
-rw-r--r--drivers/staging/media/rockchip/vpu/rk3288_vpu_regs.h442
-rw-r--r--drivers/staging/media/rockchip/vpu/rk3399_vpu_hw.c118
-rw-r--r--drivers/staging/media/rockchip/vpu/rockchip_vpu.h232
-rw-r--r--drivers/staging/media/rockchip/vpu/rockchip_vpu_common.h29
-rw-r--r--drivers/staging/media/rockchip/vpu/rockchip_vpu_drv.c542
-rw-r--r--drivers/staging/media/rockchip/vpu/rockchip_vpu_enc.c671
-rw-r--r--drivers/staging/media/rockchip/vpu/rockchip_vpu_hw.h58
-rw-r--r--drivers/staging/media/rockchip/vpu/rockchip_vpu_jpeg.h14
-rw-r--r--drivers/staging/media/soc_camera/imx074.c2
-rw-r--r--drivers/staging/media/soc_camera/mt9t031.c2
-rw-r--r--drivers/staging/media/soc_camera/soc_camera.c2
-rw-r--r--drivers/staging/media/soc_camera/soc_mt9v022.c2
-rw-r--r--drivers/staging/media/soc_camera/soc_ov5642.c6
-rw-r--r--drivers/staging/media/sunxi/cedrus/Makefile3
-rw-r--r--drivers/staging/media/sunxi/cedrus/cedrus.c85
-rw-r--r--drivers/staging/media/sunxi/cedrus/cedrus.h44
-rw-r--r--drivers/staging/media/sunxi/cedrus/cedrus_dec.c13
-rw-r--r--drivers/staging/media/sunxi/cedrus/cedrus_h264.c576
-rw-r--r--drivers/staging/media/sunxi/cedrus/cedrus_hw.c14
-rw-r--r--drivers/staging/media/sunxi/cedrus/cedrus_hw.h2
-rw-r--r--drivers/staging/media/sunxi/cedrus/cedrus_regs.h93
-rw-r--r--drivers/staging/media/sunxi/cedrus/cedrus_video.c31
-rw-r--r--drivers/staging/media/tegra-vde/Kconfig1
-rw-r--r--drivers/staging/media/tegra-vde/Makefile1
-rw-r--r--drivers/staging/media/tegra-vde/dmabuf-cache.c226
-rw-r--r--drivers/staging/media/tegra-vde/iommu.c157
-rw-r--r--drivers/staging/media/tegra-vde/trace.h2
-rw-r--r--drivers/staging/media/tegra-vde/uapi.h48
-rw-r--r--drivers/staging/media/tegra-vde/vde.c (renamed from drivers/staging/media/tegra-vde/tegra-vde.c)212
-rw-r--r--drivers/staging/media/tegra-vde/vde.h107
-rw-r--r--drivers/staging/most/Documentation/ABI/configfs-most.txt16
-rw-r--r--drivers/staging/most/Documentation/driver_usage.txt8
-rw-r--r--drivers/staging/most/Kconfig2
-rw-r--r--drivers/staging/most/cdev/cdev.c4
-rw-r--r--drivers/staging/most/configfs.c14
-rw-r--r--drivers/staging/most/core.c13
-rw-r--r--drivers/staging/most/dim2/dim2.c21
-rw-r--r--drivers/staging/most/dim2/hal.c99
-rw-r--r--drivers/staging/most/dim2/hal.h4
-rw-r--r--drivers/staging/most/net/net.c16
-rw-r--r--drivers/staging/most/sound/sound.c5
-rw-r--r--drivers/staging/most/video/video.c22
-rw-r--r--drivers/staging/mt7621-dma/mtk-hsdma.c9
-rw-r--r--drivers/staging/mt7621-dts/Kconfig7
-rw-r--r--drivers/staging/mt7621-dts/Makefile1
-rw-r--r--drivers/staging/mt7621-dts/TODO2
-rw-r--r--drivers/staging/mt7621-dts/gbpc1.dts2
-rw-r--r--drivers/staging/mt7621-dts/gbpc2.dts21
-rw-r--r--drivers/staging/mt7621-dts/mt7621.dtsi55
-rw-r--r--drivers/staging/mt7621-pci-phy/pci-mt7621-phy.c8
-rw-r--r--drivers/staging/mt7621-pci/mediatek,mt7621-pci.txt2
-rw-r--r--drivers/staging/mt7621-pci/pci-mt7621.c132
-rw-r--r--drivers/staging/mt7621-pinctrl/pinctrl-rt2880.c5
-rw-r--r--drivers/staging/netlogic/xlr_net.c2
-rw-r--r--drivers/staging/nvec/nvec.c8
-rw-r--r--drivers/staging/octeon-usb/octeon-hcd.c5
-rw-r--r--drivers/staging/octeon/Kconfig3
-rw-r--r--drivers/staging/octeon/ethernet-defines.h2
-rw-r--r--drivers/staging/octeon/ethernet-mdio.c6
-rw-r--r--drivers/staging/octeon/ethernet-mem.c5
-rw-r--r--drivers/staging/octeon/ethernet-rgmii.c10
-rw-r--r--drivers/staging/octeon/ethernet-rx.c13
-rw-r--r--drivers/staging/octeon/ethernet-rx.h2
-rw-r--r--drivers/staging/octeon/ethernet-sgmii.c8
-rw-r--r--drivers/staging/octeon/ethernet-spi.c10
-rw-r--r--drivers/staging/octeon/ethernet-tx.c19
-rw-r--r--drivers/staging/octeon/ethernet-util.h4
-rw-r--r--drivers/staging/octeon/ethernet.c28
-rw-r--r--drivers/staging/octeon/octeon-ethernet.h29
-rw-r--r--drivers/staging/octeon/octeon-stubs.h1429
-rw-r--r--drivers/staging/olpc_dcon/TODO11
-rw-r--r--drivers/staging/olpc_dcon/olpc_dcon.c6
-rw-r--r--drivers/staging/pi433/Documentation/pi433.txt2
-rw-r--r--drivers/staging/pi433/pi433_if.c5
-rw-r--r--drivers/staging/pi433/rf69.c4
-rw-r--r--drivers/staging/pi433/rf69_registers.h2
-rw-r--r--drivers/staging/qlge/Kconfig10
-rw-r--r--drivers/staging/qlge/Makefile (renamed from drivers/net/ethernet/qlogic/qlge/Makefile)0
-rw-r--r--drivers/staging/qlge/TODO46
-rw-r--r--drivers/staging/qlge/qlge.h (renamed from drivers/net/ethernet/qlogic/qlge/qlge.h)0
-rw-r--r--drivers/staging/qlge/qlge_dbg.c (renamed from drivers/net/ethernet/qlogic/qlge/qlge_dbg.c)0
-rw-r--r--drivers/staging/qlge/qlge_ethtool.c (renamed from drivers/net/ethernet/qlogic/qlge/qlge_ethtool.c)0
-rw-r--r--drivers/staging/qlge/qlge_main.c (renamed from drivers/net/ethernet/qlogic/qlge/qlge_main.c)0
-rw-r--r--drivers/staging/qlge/qlge_mpi.c (renamed from drivers/net/ethernet/qlogic/qlge/qlge_mpi.c)0
-rw-r--r--drivers/staging/ralink-gdma/ralink-gdma.c7
-rw-r--r--drivers/staging/rtl8188eu/Kconfig4
-rw-r--r--drivers/staging/rtl8188eu/core/rtw_cmd.c2
-rw-r--r--drivers/staging/rtl8188eu/core/rtw_efuse.c141
-rw-r--r--drivers/staging/rtl8188eu/core/rtw_ieee80211.c8
-rw-r--r--drivers/staging/rtl8188eu/core/rtw_mlme.c46
-rw-r--r--drivers/staging/rtl8188eu/core/rtw_mlme_ext.c4
-rw-r--r--drivers/staging/rtl8188eu/core/rtw_recv.c28
-rw-r--r--drivers/staging/rtl8188eu/core/rtw_security.c41
-rw-r--r--drivers/staging/rtl8188eu/core/rtw_wlan_util.c65
-rw-r--r--drivers/staging/rtl8188eu/hal/bb_cfg.c2
-rw-r--r--drivers/staging/rtl8188eu/hal/hal_com.c6
-rw-r--r--drivers/staging/rtl8188eu/hal/odm.c10
-rw-r--r--drivers/staging/rtl8188eu/hal/odm_hwconfig.c15
-rw-r--r--drivers/staging/rtl8188eu/hal/rf_cfg.c14
-rw-r--r--drivers/staging/rtl8188eu/hal/usb_halinit.c167
-rw-r--r--drivers/staging/rtl8188eu/include/hal8188e_phy_reg.h881
-rw-r--r--drivers/staging/rtl8188eu/include/hal_com.h1
-rw-r--r--drivers/staging/rtl8188eu/include/hal_intf.h2
-rw-r--r--drivers/staging/rtl8188eu/include/ieee80211.h10
-rw-r--r--drivers/staging/rtl8188eu/include/mlme_osdep.h1
-rw-r--r--drivers/staging/rtl8188eu/include/odm_precomp.h2
-rw-r--r--drivers/staging/rtl8188eu/include/osdep_service.h2
-rw-r--r--drivers/staging/rtl8188eu/include/rtw_eeprom.h6
-rw-r--r--drivers/staging/rtl8188eu/include/rtw_mlme.h5
-rw-r--r--drivers/staging/rtl8188eu/include/rtw_mlme_ext.h3
-rw-r--r--drivers/staging/rtl8188eu/os_dep/ioctl_linux.c22
-rw-r--r--drivers/staging/rtl8188eu/os_dep/mlme_linux.c13
-rw-r--r--drivers/staging/rtl8188eu/os_dep/os_intfs.c1
-rw-r--r--drivers/staging/rtl8188eu/os_dep/osdep_service.c5
-rw-r--r--drivers/staging/rtl8188eu/os_dep/rtw_android.c2
-rw-r--r--drivers/staging/rtl8188eu/os_dep/usb_ops_linux.c3
-rw-r--r--drivers/staging/rtl8192e/Kconfig1
-rw-r--r--drivers/staging/rtl8192e/rtl8192e/r8192E_phy.c2
-rw-r--r--drivers/staging/rtl8192e/rtl8192e/rtl_dm.c16
-rw-r--r--drivers/staging/rtl8192e/rtllib.h5
-rw-r--r--drivers/staging/rtl8192e/rtllib_crypt_ccmp.c206
-rw-r--r--drivers/staging/rtl8192e/rtllib_module.c1
-rw-r--r--drivers/staging/rtl8192e/rtllib_rx.c2
-rw-r--r--drivers/staging/rtl8192e/rtllib_softmac.c24
-rw-r--r--drivers/staging/rtl8192u/Kconfig2
-rw-r--r--drivers/staging/rtl8192u/ieee80211/dot11d.c10
-rw-r--r--drivers/staging/rtl8192u/ieee80211/ieee80211.h42
-rw-r--r--drivers/staging/rtl8192u/ieee80211/ieee80211_crypt.c2
-rw-r--r--drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_ccmp.c204
-rw-r--r--drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c22
-rw-r--r--drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_wep.c4
-rw-r--r--drivers/staging/rtl8192u/ieee80211/ieee80211_module.c3
-rw-r--r--drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c662
-rw-r--r--drivers/staging/rtl8192u/ieee80211/ieee80211_softmac.c14
-rw-r--r--drivers/staging/rtl8192u/ieee80211/ieee80211_softmac_wx.c18
-rw-r--r--drivers/staging/rtl8192u/ieee80211/ieee80211_tx.c194
-rw-r--r--drivers/staging/rtl8192u/ieee80211/ieee80211_wx.c73
-rw-r--r--drivers/staging/rtl8192u/ieee80211/rtl819x_BAProc.c12
-rw-r--r--drivers/staging/rtl8192u/ieee80211/rtl819x_HT.h17
-rw-r--r--drivers/staging/rtl8192u/ieee80211/rtl819x_HTProc.c4
-rw-r--r--drivers/staging/rtl8192u/ieee80211/rtl819x_TSProc.c28
-rw-r--r--drivers/staging/rtl8192u/r8180_93cx6.c11
-rw-r--r--drivers/staging/rtl8192u/r8190_rtl8256.c42
-rw-r--r--drivers/staging/rtl8192u/r8192U_core.c122
-rw-r--r--drivers/staging/rtl8192u/r8192U_dm.c111
-rw-r--r--drivers/staging/rtl8192u/r8192U_dm.h1
-rw-r--r--drivers/staging/rtl8192u/r819xU_firmware.c2
-rw-r--r--drivers/staging/rtl8192u/r819xU_phy.c59
-rw-r--r--drivers/staging/rtl8712/drv_types.h13
-rw-r--r--drivers/staging/rtl8712/hal_init.c174
-rw-r--r--drivers/staging/rtl8712/ieee80211.c74
-rw-r--r--drivers/staging/rtl8712/mlme_linux.c36
-rw-r--r--drivers/staging/rtl8712/os_intfs.c40
-rw-r--r--drivers/staging/rtl8712/recv_linux.c68
-rw-r--r--drivers/staging/rtl8712/recv_osdep.h16
-rw-r--r--drivers/staging/rtl8712/rtl8712_cmd.c14
-rw-r--r--drivers/staging/rtl8712/rtl8712_efuse.c152
-rw-r--r--drivers/staging/rtl8712/rtl8712_recv.c54
-rw-r--r--drivers/staging/rtl8712/rtl8712_recv.h2
-rw-r--r--drivers/staging/rtl8712/rtl8712_xmit.c48
-rw-r--r--drivers/staging/rtl8712/rtl8712_xmit.h8
-rw-r--r--drivers/staging/rtl8712/rtl871x_cmd.c306
-rw-r--r--drivers/staging/rtl8712/rtl871x_cmd.h43
-rw-r--r--drivers/staging/rtl8712/rtl871x_eeprom.c6
-rw-r--r--drivers/staging/rtl8712/rtl871x_io.c4
-rw-r--r--drivers/staging/rtl8712/rtl871x_io.h7
-rw-r--r--drivers/staging/rtl8712/rtl871x_ioctl_linux.c176
-rw-r--r--drivers/staging/rtl8712/rtl871x_ioctl_rtl.c4
-rw-r--r--drivers/staging/rtl8712/rtl871x_ioctl_set.c20
-rw-r--r--drivers/staging/rtl8712/rtl871x_ioctl_set.h4
-rw-r--r--drivers/staging/rtl8712/rtl871x_mlme.c79
-rw-r--r--drivers/staging/rtl8712/rtl871x_mlme.h11
-rw-r--r--drivers/staging/rtl8712/rtl871x_mp.c23
-rw-r--r--drivers/staging/rtl8712/rtl871x_mp_ioctl.c27
-rw-r--r--drivers/staging/rtl8712/rtl871x_mp_ioctl.h2
-rw-r--r--drivers/staging/rtl8712/rtl871x_pwrctrl.c10
-rw-r--r--drivers/staging/rtl8712/rtl871x_pwrctrl.h2
-rw-r--r--drivers/staging/rtl8712/rtl871x_recv.c36
-rw-r--r--drivers/staging/rtl8712/rtl871x_recv.h2
-rw-r--r--drivers/staging/rtl8712/rtl871x_rf.h3
-rw-r--r--drivers/staging/rtl8712/rtl871x_security.c33
-rw-r--r--drivers/staging/rtl8712/rtl871x_security.h4
-rw-r--r--drivers/staging/rtl8712/rtl871x_sta_mgt.c9
-rw-r--r--drivers/staging/rtl8712/rtl871x_xmit.c80
-rw-r--r--drivers/staging/rtl8712/rtl871x_xmit.h24
-rw-r--r--drivers/staging/rtl8712/sta_info.h4
-rw-r--r--drivers/staging/rtl8712/usb_halinit.c288
-rw-r--r--drivers/staging/rtl8712/usb_intf.c6
-rw-r--r--drivers/staging/rtl8712/usb_ops.c84
-rw-r--r--drivers/staging/rtl8712/usb_osintf.h4
-rw-r--r--drivers/staging/rtl8712/wifi.h19
-rw-r--r--drivers/staging/rtl8712/xmit_linux.c56
-rw-r--r--drivers/staging/rtl8723bs/Kconfig2
-rw-r--r--drivers/staging/rtl8723bs/Makefile1
-rw-r--r--drivers/staging/rtl8723bs/TODO3
-rw-r--r--drivers/staging/rtl8723bs/core/rtw_ap.c138
-rw-r--r--drivers/staging/rtl8723bs/core/rtw_btcoex.c147
-rw-r--r--drivers/staging/rtl8723bs/core/rtw_cmd.c48
-rw-r--r--drivers/staging/rtl8723bs/core/rtw_debug.c1308
-rw-r--r--drivers/staging/rtl8723bs/core/rtw_eeprom.c139
-rw-r--r--drivers/staging/rtl8723bs/core/rtw_ieee80211.c4
-rw-r--r--drivers/staging/rtl8723bs/core/rtw_io.c2
-rw-r--r--drivers/staging/rtl8723bs/core/rtw_ioctl_set.c9
-rw-r--r--drivers/staging/rtl8723bs/core/rtw_mlme.c24
-rw-r--r--drivers/staging/rtl8723bs/core/rtw_mlme_ext.c98
-rw-r--r--drivers/staging/rtl8723bs/core/rtw_pwrctrl.c139
-rw-r--r--drivers/staging/rtl8723bs/core/rtw_recv.c56
-rw-r--r--drivers/staging/rtl8723bs/core/rtw_security.c3
-rw-r--r--drivers/staging/rtl8723bs/core/rtw_wlan_util.c26
-rw-r--r--drivers/staging/rtl8723bs/core/rtw_xmit.c70
-rw-r--r--drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.c14
-rw-r--r--drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.c742
-rw-r--r--drivers/staging/rtl8723bs/hal/HalBtcOutSrc.h1
-rw-r--r--drivers/staging/rtl8723bs/hal/HalHWImg8723B_BB.c4
-rw-r--r--drivers/staging/rtl8723bs/hal/HalHWImg8723B_MAC.c2
-rw-r--r--drivers/staging/rtl8723bs/hal/HalHWImg8723B_RF.c2
-rw-r--r--drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c6
-rw-r--r--drivers/staging/rtl8723bs/hal/hal_btcoex.c110
-rw-r--r--drivers/staging/rtl8723bs/hal/hal_com.c13
-rw-r--r--drivers/staging/rtl8723bs/hal/hal_com_phycfg.c18
-rw-r--r--drivers/staging/rtl8723bs/hal/hal_intf.c7
-rw-r--r--drivers/staging/rtl8723bs/hal/hal_phy.c216
-rw-r--r--drivers/staging/rtl8723bs/hal/odm.c19
-rw-r--r--drivers/staging/rtl8723bs/hal/odm.h4
-rw-r--r--drivers/staging/rtl8723bs/hal/odm_AntDiv.c62
-rw-r--r--drivers/staging/rtl8723bs/hal/odm_AntDiv.h30
-rw-r--r--drivers/staging/rtl8723bs/hal/odm_CfoTracking.c5
-rw-r--r--drivers/staging/rtl8723bs/hal/odm_DIG.c11
-rw-r--r--drivers/staging/rtl8723bs/hal/odm_HWConfig.c43
-rw-r--r--drivers/staging/rtl8723bs/hal/odm_HWConfig.h2
-rw-r--r--drivers/staging/rtl8723bs/hal/odm_RegConfig8723B.h61
-rw-r--r--drivers/staging/rtl8723bs/hal/odm_precomp.h1
-rw-r--r--drivers/staging/rtl8723bs/hal/rtl8723b_cmd.c58
-rw-r--r--drivers/staging/rtl8723bs/hal/rtl8723b_dm.c4
-rw-r--r--drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c76
-rw-r--r--drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c12
-rw-r--r--drivers/staging/rtl8723bs/hal/rtl8723b_rf6052.c10
-rw-r--r--drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c17
-rw-r--r--drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c39
-rw-r--r--drivers/staging/rtl8723bs/hal/sdio_halinit.c145
-rw-r--r--drivers/staging/rtl8723bs/hal/sdio_ops.c68
-rw-r--r--drivers/staging/rtl8723bs/include/autoconf.h4
-rw-r--r--drivers/staging/rtl8723bs/include/drv_types.h10
-rw-r--r--drivers/staging/rtl8723bs/include/hal_btcoex.h12
-rw-r--r--drivers/staging/rtl8723bs/include/hal_com.h1
-rw-r--r--drivers/staging/rtl8723bs/include/hal_com_phycfg.h17
-rw-r--r--drivers/staging/rtl8723bs/include/hal_intf.h3
-rw-r--r--drivers/staging/rtl8723bs/include/hal_phy_cfg.h4
-rw-r--r--drivers/staging/rtl8723bs/include/ieee80211.h2
-rw-r--r--drivers/staging/rtl8723bs/include/osdep_intf.h4
-rw-r--r--drivers/staging/rtl8723bs/include/osdep_service.h12
-rw-r--r--drivers/staging/rtl8723bs/include/osdep_service_linux.h14
-rw-r--r--drivers/staging/rtl8723bs/include/recv_osdep.h4
-rw-r--r--drivers/staging/rtl8723bs/include/rtw_ap.h4
-rw-r--r--drivers/staging/rtl8723bs/include/rtw_btcoex.h28
-rw-r--r--drivers/staging/rtl8723bs/include/rtw_debug.h77
-rw-r--r--drivers/staging/rtl8723bs/include/rtw_mlme.h23
-rw-r--r--drivers/staging/rtl8723bs/include/rtw_mlme_ext.h5
-rw-r--r--drivers/staging/rtl8723bs/include/rtw_recv.h16
-rw-r--r--drivers/staging/rtl8723bs/include/sdio_ops.h2
-rw-r--r--drivers/staging/rtl8723bs/include/sta_info.h2
-rw-r--r--drivers/staging/rtl8723bs/include/wifi.h14
-rw-r--r--drivers/staging/rtl8723bs/include/wlan_bssdef.h2
-rw-r--r--drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c44
-rw-r--r--drivers/staging/rtl8723bs/os_dep/ioctl_linux.c527
-rw-r--r--drivers/staging/rtl8723bs/os_dep/mlme_linux.c20
-rw-r--r--drivers/staging/rtl8723bs/os_dep/os_intfs.c51
-rw-r--r--drivers/staging/rtl8723bs/os_dep/osdep_service.c45
-rw-r--r--drivers/staging/rtl8723bs/os_dep/recv_linux.c101
-rw-r--r--drivers/staging/rtl8723bs/os_dep/rtw_proc.c781
-rw-r--r--drivers/staging/rtl8723bs/os_dep/rtw_proc.h37
-rw-r--r--drivers/staging/rtl8723bs/os_dep/sdio_intf.c78
-rw-r--r--drivers/staging/rtl8723bs/os_dep/sdio_ops_linux.c24
-rw-r--r--drivers/staging/rtl8723bs/os_dep/wifi_regd.c5
-rw-r--r--drivers/staging/rtl8723bs/os_dep/xmit_linux.c10
-rw-r--r--drivers/staging/rts5208/TODO2
-rw-r--r--drivers/staging/rts5208/ms.c2
-rw-r--r--drivers/staging/rts5208/rtsx_chip.c20
-rw-r--r--drivers/staging/rts5208/rtsx_transport.c12
-rw-r--r--drivers/staging/rts5208/sd.c58
-rw-r--r--drivers/staging/rts5208/sd.h1
-rw-r--r--drivers/staging/rts5208/xd.c8
-rw-r--r--drivers/staging/sm750fb/Kconfig2
-rw-r--r--drivers/staging/sm750fb/ddk750.h3
-rw-r--r--drivers/staging/sm750fb/ddk750_swi2c.c3
-rw-r--r--drivers/staging/sm750fb/ddk750_swi2c.h3
-rw-r--r--drivers/staging/speakup/serialio.h3
-rw-r--r--drivers/staging/unisys/Documentation/overview.txt4
-rw-r--r--drivers/staging/unisys/Kconfig4
-rw-r--r--drivers/staging/unisys/visorhba/visorhba_main.c9
-rw-r--r--drivers/staging/unisys/visornic/visornic_main.c13
-rw-r--r--drivers/staging/uwb/Kconfig (renamed from drivers/uwb/Kconfig)0
-rw-r--r--drivers/staging/uwb/Makefile (renamed from drivers/uwb/Makefile)0
-rw-r--r--drivers/staging/uwb/TODO8
-rw-r--r--drivers/staging/uwb/address.c (renamed from drivers/uwb/address.c)0
-rw-r--r--drivers/staging/uwb/allocator.c (renamed from drivers/uwb/allocator.c)2
-rw-r--r--drivers/staging/uwb/beacon.c (renamed from drivers/uwb/beacon.c)0
-rw-r--r--drivers/staging/uwb/driver.c (renamed from drivers/uwb/driver.c)0
-rw-r--r--drivers/staging/uwb/drp-avail.c (renamed from drivers/uwb/drp-avail.c)0
-rw-r--r--drivers/staging/uwb/drp-ie.c (renamed from drivers/uwb/drp-ie.c)2
-rw-r--r--drivers/staging/uwb/drp.c (renamed from drivers/uwb/drp.c)0
-rw-r--r--drivers/staging/uwb/est.c (renamed from drivers/uwb/est.c)0
-rw-r--r--drivers/staging/uwb/hwa-rc.c (renamed from drivers/uwb/hwa-rc.c)6
-rw-r--r--drivers/staging/uwb/i1480/Makefile (renamed from drivers/uwb/i1480/Makefile)0
-rw-r--r--drivers/staging/uwb/i1480/dfu/Makefile (renamed from drivers/uwb/i1480/dfu/Makefile)0
-rw-r--r--drivers/staging/uwb/i1480/dfu/dfu.c (renamed from drivers/uwb/i1480/dfu/dfu.c)2
-rw-r--r--drivers/staging/uwb/i1480/dfu/i1480-dfu.h (renamed from drivers/uwb/i1480/dfu/i1480-dfu.h)2
-rw-r--r--drivers/staging/uwb/i1480/dfu/mac.c (renamed from drivers/uwb/i1480/dfu/mac.c)2
-rw-r--r--drivers/staging/uwb/i1480/dfu/phy.c (renamed from drivers/uwb/i1480/dfu/phy.c)2
-rw-r--r--drivers/staging/uwb/i1480/dfu/usb.c (renamed from drivers/uwb/i1480/dfu/usb.c)6
-rw-r--r--drivers/staging/uwb/i1480/i1480-est.c (renamed from drivers/uwb/i1480/i1480-est.c)2
-rw-r--r--drivers/staging/uwb/ie-rcv.c (renamed from drivers/uwb/ie-rcv.c)0
-rw-r--r--drivers/staging/uwb/ie.c (renamed from drivers/uwb/ie.c)0
-rw-r--r--drivers/staging/uwb/include/debug-cmd.h57
-rw-r--r--drivers/staging/uwb/include/spec.h767
-rw-r--r--drivers/staging/uwb/include/umc.h192
-rw-r--r--drivers/staging/uwb/include/whci.h102
-rw-r--r--drivers/staging/uwb/lc-dev.c (renamed from drivers/uwb/lc-dev.c)0
-rw-r--r--drivers/staging/uwb/lc-rc.c (renamed from drivers/uwb/lc-rc.c)0
-rw-r--r--drivers/staging/uwb/neh.c (renamed from drivers/uwb/neh.c)0
-rw-r--r--drivers/staging/uwb/pal.c (renamed from drivers/uwb/pal.c)2
-rw-r--r--drivers/staging/uwb/radio.c (renamed from drivers/uwb/radio.c)2
-rw-r--r--drivers/staging/uwb/reset.c (renamed from drivers/uwb/reset.c)0
-rw-r--r--drivers/staging/uwb/rsv.c (renamed from drivers/uwb/rsv.c)2
-rw-r--r--drivers/staging/uwb/scan.c (renamed from drivers/uwb/scan.c)0
-rw-r--r--drivers/staging/uwb/umc-bus.c (renamed from drivers/uwb/umc-bus.c)2
-rw-r--r--drivers/staging/uwb/umc-dev.c (renamed from drivers/uwb/umc-dev.c)2
-rw-r--r--drivers/staging/uwb/umc-drv.c (renamed from drivers/uwb/umc-drv.c)2
-rw-r--r--drivers/staging/uwb/uwb-debug.c (renamed from drivers/uwb/uwb-debug.c)3
-rw-r--r--drivers/staging/uwb/uwb-internal.h (renamed from drivers/uwb/uwb-internal.h)2
-rw-r--r--drivers/staging/uwb/uwb.h817
-rw-r--r--drivers/staging/uwb/uwbd.c (renamed from drivers/uwb/uwbd.c)0
-rw-r--r--drivers/staging/uwb/whc-rc.c (renamed from drivers/uwb/whc-rc.c)6
-rw-r--r--drivers/staging/uwb/whci.c (renamed from drivers/uwb/whci.c)4
-rw-r--r--drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.c410
-rw-r--r--drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.h32
-rw-r--r--drivers/staging/vc04_services/bcm2835-camera/controls.c208
-rw-r--r--drivers/staging/vc04_services/bcm2835-camera/mmal-common.h13
-rw-r--r--drivers/staging/vc04_services/bcm2835-camera/mmal-encodings.h9
-rw-r--r--drivers/staging/vc04_services/bcm2835-camera/mmal-msg-common.h9
-rw-r--r--drivers/staging/vc04_services/bcm2835-camera/mmal-msg-format.h104
-rw-r--r--drivers/staging/vc04_services/bcm2835-camera/mmal-msg-port.h133
-rw-r--r--drivers/staging/vc04_services/bcm2835-camera/mmal-msg.h154
-rw-r--r--drivers/staging/vc04_services/bcm2835-camera/mmal-parameters.h286
-rw-r--r--drivers/staging/vc04_services/bcm2835-camera/mmal-vchiq.c159
-rw-r--r--drivers/staging/vc04_services/bcm2835-camera/mmal-vchiq.h24
-rw-r--r--drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c6
-rw-r--r--drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c361
-rw-r--r--drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.h6
-rw-r--r--drivers/staging/vc04_services/interface/vchiq_arm/vchiq_connected.c4
-rw-r--r--drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c144
-rw-r--r--drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.h9
-rw-r--r--drivers/staging/vc04_services/interface/vchiq_arm/vchiq_debugfs.c4
-rw-r--r--drivers/staging/vc04_services/interface/vchiq_arm/vchiq_if.h2
-rw-r--r--drivers/staging/vc04_services/interface/vchiq_arm/vchiq_shim.c4
-rw-r--r--drivers/staging/vc04_services/interface/vchiq_arm/vchiq_util.c11
-rw-r--r--drivers/staging/vc04_services/interface/vchiq_arm/vchiq_util.h1
-rw-r--r--drivers/staging/vt6655/Kconfig5
-rw-r--r--drivers/staging/vt6655/card.c8
-rw-r--r--drivers/staging/vt6655/card.h2
-rw-r--r--drivers/staging/vt6655/test2
-rw-r--r--drivers/staging/vt6656/Kconfig5
-rw-r--r--drivers/staging/vt6656/baseband.c130
-rw-r--r--drivers/staging/vt6656/baseband.h8
-rw-r--r--drivers/staging/vt6656/card.c22
-rw-r--r--drivers/staging/vt6656/firmware.c91
-rw-r--r--drivers/staging/vt6656/int.c8
-rw-r--r--drivers/staging/vt6656/int.h2
-rw-r--r--drivers/staging/vt6656/mac.c19
-rw-r--r--drivers/staging/vt6656/mac.h6
-rw-r--r--drivers/staging/vt6656/main_usb.c230
-rw-r--r--drivers/staging/vt6656/rf.c38
-rw-r--r--drivers/staging/vt6656/rf.h2
-rw-r--r--drivers/staging/vt6656/rxtx.c10
-rw-r--r--drivers/staging/vt6656/usbpipe.c117
-rw-r--r--drivers/staging/vt6656/usbpipe.h4
-rw-r--r--drivers/staging/wilc1000/Makefile2
-rw-r--r--drivers/staging/wilc1000/microchip,wilc1000,sdio.txt8
-rw-r--r--drivers/staging/wilc1000/microchip,wilc1000,spi.txt8
-rw-r--r--drivers/staging/wilc1000/wilc_hif.c (renamed from drivers/staging/wilc1000/host_interface.c)256
-rw-r--r--drivers/staging/wilc1000/wilc_hif.h (renamed from drivers/staging/wilc1000/host_interface.h)8
-rw-r--r--drivers/staging/wilc1000/wilc_mon.c12
-rw-r--r--drivers/staging/wilc1000/wilc_netdev.c346
-rw-r--r--drivers/staging/wilc1000/wilc_sdio.c25
-rw-r--r--drivers/staging/wilc1000/wilc_spi.c3
-rw-r--r--drivers/staging/wilc1000/wilc_wfi_cfgoperations.c600
-rw-r--r--drivers/staging/wilc1000/wilc_wfi_cfgoperations.h13
-rw-r--r--drivers/staging/wilc1000/wilc_wfi_netdevice.h27
-rw-r--r--drivers/staging/wilc1000/wilc_wlan.c161
-rw-r--r--drivers/staging/wilc1000/wilc_wlan.h27
-rw-r--r--drivers/staging/wilc1000/wilc_wlan_cfg.c78
-rw-r--r--drivers/staging/wilc1000/wilc_wlan_cfg.h4
-rw-r--r--drivers/staging/wilc1000/wilc_wlan_if.h5
-rw-r--r--drivers/staging/wlan-ng/cfg80211.c14
-rw-r--r--drivers/staging/wlan-ng/hfa384x_usb.c210
-rw-r--r--drivers/staging/wlan-ng/prism2mib.c44
-rw-r--r--drivers/staging/wlan-ng/prism2sta.c2
-rw-r--r--drivers/staging/wusbcore/Documentation/wusb-cbaf130
-rw-r--r--drivers/staging/wusbcore/Documentation/wusb-design-overview.rst457
-rw-r--r--drivers/staging/wusbcore/Kconfig (renamed from drivers/usb/wusbcore/Kconfig)11
-rw-r--r--drivers/staging/wusbcore/Makefile (renamed from drivers/usb/wusbcore/Makefile)2
-rw-r--r--drivers/staging/wusbcore/TODO8
-rw-r--r--drivers/staging/wusbcore/cbaf.c (renamed from drivers/usb/wusbcore/cbaf.c)6
-rw-r--r--drivers/staging/wusbcore/crypto.c (renamed from drivers/usb/wusbcore/crypto.c)173
-rw-r--r--drivers/staging/wusbcore/dev-sysfs.c (renamed from drivers/usb/wusbcore/dev-sysfs.c)0
-rw-r--r--drivers/staging/wusbcore/devconnect.c (renamed from drivers/usb/wusbcore/devconnect.c)2
-rw-r--r--drivers/staging/wusbcore/host/Kconfig28
-rw-r--r--drivers/staging/wusbcore/host/Makefile3
-rw-r--r--drivers/staging/wusbcore/host/hwa-hc.c (renamed from drivers/usb/host/hwa-hc.c)6
-rw-r--r--drivers/staging/wusbcore/host/whci/Makefile (renamed from drivers/usb/host/whci/Makefile)0
-rw-r--r--drivers/staging/wusbcore/host/whci/asl.c (renamed from drivers/usb/host/whci/asl.c)4
-rw-r--r--drivers/staging/wusbcore/host/whci/debug.c (renamed from drivers/usb/host/whci/debug.c)2
-rw-r--r--drivers/staging/wusbcore/host/whci/hcd.c (renamed from drivers/usb/host/whci/hcd.c)4
-rw-r--r--drivers/staging/wusbcore/host/whci/hw.c (renamed from drivers/usb/host/whci/hw.c)4
-rw-r--r--drivers/staging/wusbcore/host/whci/init.c (renamed from drivers/usb/host/whci/init.c)4
-rw-r--r--drivers/staging/wusbcore/host/whci/int.c (renamed from drivers/usb/host/whci/int.c)4
-rw-r--r--drivers/staging/wusbcore/host/whci/pzl.c (renamed from drivers/usb/host/whci/pzl.c)4
-rw-r--r--drivers/staging/wusbcore/host/whci/qset.c (renamed from drivers/usb/host/whci/qset.c)4
-rw-r--r--drivers/staging/wusbcore/host/whci/whcd.h (renamed from drivers/usb/host/whci/whcd.h)4
-rw-r--r--drivers/staging/wusbcore/host/whci/whci-hc.h (renamed from drivers/usb/host/whci/whci-hc.h)0
-rw-r--r--drivers/staging/wusbcore/host/whci/wusb.c (renamed from drivers/usb/host/whci/wusb.c)4
-rw-r--r--drivers/staging/wusbcore/include/association.h151
-rw-r--r--drivers/staging/wusbcore/include/wusb-wa.h304
-rw-r--r--drivers/staging/wusbcore/include/wusb.h362
-rw-r--r--drivers/staging/wusbcore/mmc.c (renamed from drivers/usb/wusbcore/mmc.c)2
-rw-r--r--drivers/staging/wusbcore/pal.c (renamed from drivers/usb/wusbcore/pal.c)0
-rw-r--r--drivers/staging/wusbcore/reservation.c (renamed from drivers/usb/wusbcore/reservation.c)2
-rw-r--r--drivers/staging/wusbcore/rh.c (renamed from drivers/usb/wusbcore/rh.c)0
-rw-r--r--drivers/staging/wusbcore/security.c (renamed from drivers/usb/wusbcore/security.c)0
-rw-r--r--drivers/staging/wusbcore/wa-hc.c (renamed from drivers/usb/wusbcore/wa-hc.c)0
-rw-r--r--drivers/staging/wusbcore/wa-hc.h (renamed from drivers/usb/wusbcore/wa-hc.h)6
-rw-r--r--drivers/staging/wusbcore/wa-nep.c (renamed from drivers/usb/wusbcore/wa-nep.c)0
-rw-r--r--drivers/staging/wusbcore/wa-rpipe.c (renamed from drivers/usb/wusbcore/wa-rpipe.c)0
-rw-r--r--drivers/staging/wusbcore/wa-xfer.c (renamed from drivers/usb/wusbcore/wa-xfer.c)0
-rw-r--r--drivers/staging/wusbcore/wusbhc.c (renamed from drivers/usb/wusbcore/wusbhc.c)0
-rw-r--r--drivers/staging/wusbcore/wusbhc.h (renamed from drivers/usb/wusbcore/wusbhc.h)4
-rw-r--r--drivers/target/iscsi/cxgbit/cxgbit_cm.c8
-rw-r--r--drivers/target/iscsi/cxgbit/cxgbit_ddp.c6
-rw-r--r--drivers/target/iscsi/cxgbit/cxgbit_main.c3
-rw-r--r--drivers/target/iscsi/cxgbit/cxgbit_target.c13
-rw-r--r--drivers/target/iscsi/iscsi_target_nego.c15
-rw-r--r--drivers/target/target_core_user.c25
-rw-r--r--drivers/tee/optee/call.c1
-rw-r--r--drivers/thermal/broadcom/bcm2835_thermal.c2
-rw-r--r--drivers/thermal/cpu_cooling.c110
-rw-r--r--drivers/thermal/fair_share.c12
-rw-r--r--drivers/thermal/gov_bang_bang.c11
-rw-r--r--drivers/thermal/intel/int340x_thermal/Kconfig6
-rw-r--r--drivers/thermal/intel/int340x_thermal/processor_thermal_device.c195
-rw-r--r--drivers/thermal/intel/intel_powerclamp.c14
-rw-r--r--drivers/thermal/intel/x86_pkg_temp_thermal.c167
-rw-r--r--drivers/thermal/power_allocator.c11
-rw-r--r--drivers/thermal/step_wise.c11
-rw-r--r--drivers/thermal/tegra/soctherm.c14
-rw-r--r--drivers/thermal/thermal_core.c52
-rw-r--r--drivers/thermal/thermal_core.h55
-rw-r--r--drivers/thermal/user_space.c12
-rw-r--r--drivers/thunderbolt/Makefile2
-rw-r--r--drivers/thunderbolt/ctl.c23
-rw-r--r--drivers/thunderbolt/eeprom.c6
-rw-r--r--drivers/thunderbolt/icm.c194
-rw-r--r--drivers/thunderbolt/nhi.c134
-rw-r--r--drivers/thunderbolt/nhi.h22
-rw-r--r--drivers/thunderbolt/nhi_ops.c179
-rw-r--r--drivers/thunderbolt/nhi_regs.h37
-rw-r--r--drivers/thunderbolt/switch.c56
-rw-r--r--drivers/thunderbolt/tb_msgs.h16
-rw-r--r--drivers/thunderbolt/tunnel.c4
-rw-r--r--drivers/thunderbolt/xdomain.c2
-rw-r--r--drivers/tty/Kconfig8
-rw-r--r--drivers/tty/hvc/hvc_vio.c16
-rw-r--r--drivers/tty/hvc/hvcs.c6
-rw-r--r--drivers/tty/isicom.c1
-rw-r--r--drivers/tty/n_gsm.c25
-rw-r--r--drivers/tty/nozomi.c4
-rw-r--r--drivers/tty/serial/8250/8250.h90
-rw-r--r--drivers/tty/serial/8250/8250_bcm2835aux.c4
-rw-r--r--drivers/tty/serial/8250/8250_core.c18
-rw-r--r--drivers/tty/serial/8250/8250_dma.c11
-rw-r--r--drivers/tty/serial/8250/8250_dw.c173
-rw-r--r--drivers/tty/serial/8250/8250_dwlib.c126
-rw-r--r--drivers/tty/serial/8250/8250_dwlib.h19
-rw-r--r--drivers/tty/serial/8250/8250_exar.c114
-rw-r--r--drivers/tty/serial/8250/8250_lpc18xx.c4
-rw-r--r--drivers/tty/serial/8250/8250_lpss.c76
-rw-r--r--drivers/tty/serial/8250/8250_moxa.c155
-rw-r--r--drivers/tty/serial/8250/8250_mtk.c73
-rw-r--r--drivers/tty/serial/8250/8250_of.c14
-rw-r--r--drivers/tty/serial/8250/8250_omap.c54
-rw-r--r--drivers/tty/serial/8250/8250_pci.c748
-rw-r--r--drivers/tty/serial/8250/8250_pnp.c24
-rw-r--r--drivers/tty/serial/8250/8250_port.c143
-rw-r--r--drivers/tty/serial/8250/8250_uniphier.c4
-rw-r--r--drivers/tty/serial/8250/Kconfig16
-rw-r--r--drivers/tty/serial/8250/Makefile2
-rw-r--r--drivers/tty/serial/Kconfig99
-rw-r--r--drivers/tty/serial/Makefile7
-rw-r--r--drivers/tty/serial/amba-pl011.c7
-rw-r--r--drivers/tty/serial/atmel_serial.c51
-rw-r--r--drivers/tty/serial/cpm_uart/cpm_uart_core.c19
-rw-r--r--drivers/tty/serial/digicolor-usart.c6
-rw-r--r--drivers/tty/serial/fsl_linflexuart.c937
-rw-r--r--drivers/tty/serial/fsl_lpuart.c632
-rw-r--r--drivers/tty/serial/icom.c2
-rw-r--r--drivers/tty/serial/imx.c162
-rw-r--r--drivers/tty/serial/ioc3_serial.c2195
-rw-r--r--drivers/tty/serial/ioc4_serial.c2955
-rw-r--r--drivers/tty/serial/kgdboc.c4
-rw-r--r--drivers/tty/serial/lantiq.c261
-rw-r--r--drivers/tty/serial/lpc32xx_hs.c42
-rw-r--r--drivers/tty/serial/max310x.c187
-rw-r--r--drivers/tty/serial/mpsc.c2138
-rw-r--r--drivers/tty/serial/msm_serial.c4
-rw-r--r--drivers/tty/serial/mvebu-uart.c12
-rw-r--r--drivers/tty/serial/mxs-auart.c6
-rw-r--r--drivers/tty/serial/netx-serial.c733
-rw-r--r--drivers/tty/serial/owl-uart.c4
-rw-r--r--drivers/tty/serial/qcom_geni_serial.c23
-rw-r--r--drivers/tty/serial/rda-uart.c4
-rw-r--r--drivers/tty/serial/sa1100.c46
-rw-r--r--drivers/tty/serial/sccnxp.c1
-rw-r--r--drivers/tty/serial/serial-tegra.c407
-rw-r--r--drivers/tty/serial/serial_core.c7
-rw-r--r--drivers/tty/serial/serial_ks8695.c698
-rw-r--r--drivers/tty/serial/serial_mctrl_gpio.c48
-rw-r--r--drivers/tty/serial/serial_mctrl_gpio.h6
-rw-r--r--drivers/tty/serial/sh-sci.c83
-rw-r--r--drivers/tty/serial/sifive.c4
-rw-r--r--drivers/tty/serial/sn_console.c1036
-rw-r--r--drivers/tty/serial/sprd_serial.c58
-rw-r--r--drivers/tty/serial/stm32-usart.c394
-rw-r--r--drivers/tty/serial/stm32-usart.h33
-rw-r--r--drivers/tty/serial/sunhv.c2
-rw-r--r--drivers/tty/serial/ucc_uart.c2
-rw-r--r--drivers/tty/serial/xilinx_uartps.c39
-rw-r--r--drivers/tty/tty_io.c12
-rw-r--r--drivers/tty/tty_ldisc.c14
-rw-r--r--drivers/tty/tty_ldsem.c5
-rw-r--r--drivers/tty/vt/vt.c24
-rw-r--r--drivers/uio/uio_dmem_genirq.c4
-rw-r--r--drivers/uio/uio_fsl_elbc_gpcm.c23
-rw-r--r--drivers/uio/uio_pdrv_genirq.c14
-rw-r--r--drivers/usb/Kconfig42
-rw-r--r--drivers/usb/Makefile7
-rw-r--r--drivers/usb/atm/Kconfig1
-rw-r--r--drivers/usb/atm/cxacru.c58
-rw-r--r--drivers/usb/atm/ueagle-atm.c64
-rw-r--r--drivers/usb/cdns3/Kconfig46
-rw-r--r--drivers/usb/cdns3/Makefile16
-rw-r--r--drivers/usb/cdns3/cdns3-pci-wrap.c203
-rw-r--r--drivers/usb/cdns3/core.c651
-rw-r--r--drivers/usb/cdns3/core.h98
-rw-r--r--drivers/usb/cdns3/debug.h161
-rw-r--r--drivers/usb/cdns3/drd.c381
-rw-r--r--drivers/usb/cdns3/drd.h167
-rw-r--r--drivers/usb/cdns3/ep0.c886
-rw-r--r--drivers/usb/cdns3/gadget-export.h28
-rw-r--r--drivers/usb/cdns3/gadget.c2744
-rw-r--r--drivers/usb/cdns3/gadget.h1338
-rw-r--r--drivers/usb/cdns3/host-export.h28
-rw-r--r--drivers/usb/cdns3/host.c74
-rw-r--r--drivers/usb/cdns3/trace.c11
-rw-r--r--drivers/usb/cdns3/trace.h493
-rw-r--r--drivers/usb/chipidea/Kconfig1
-rw-r--r--drivers/usb/chipidea/ci.h12
-rw-r--r--drivers/usb/chipidea/ci_hdrc_imx.c48
-rw-r--r--drivers/usb/chipidea/ci_hdrc_msm.c13
-rw-r--r--drivers/usb/chipidea/core.c102
-rw-r--r--drivers/usb/chipidea/otg.c8
-rw-r--r--drivers/usb/chipidea/udc.c35
-rw-r--r--drivers/usb/chipidea/usbmisc_imx.c8
-rw-r--r--drivers/usb/class/Kconfig2
-rw-r--r--drivers/usb/class/cdc-acm.c12
-rw-r--r--drivers/usb/class/cdc-wdm.c18
-rw-r--r--drivers/usb/class/usblp.c13
-rw-r--r--drivers/usb/class/usbtmc.c16
-rw-r--r--drivers/usb/common/Kconfig51
-rw-r--r--drivers/usb/common/Makefile2
-rw-r--r--drivers/usb/common/common.c21
-rw-r--r--drivers/usb/common/common.h14
-rw-r--r--drivers/usb/common/debug.c268
-rw-r--r--drivers/usb/common/led.c9
-rw-r--r--drivers/usb/common/usb-conn-gpio.c284
-rw-r--r--drivers/usb/core/Kconfig1
-rw-r--r--drivers/usb/core/buffer.c21
-rw-r--r--drivers/usb/core/config.c12
-rw-r--r--drivers/usb/core/devio.c233
-rw-r--r--drivers/usb/core/driver.c2
-rw-r--r--drivers/usb/core/file.c10
-rw-r--r--drivers/usb/core/generic.c5
-rw-r--r--drivers/usb/core/hcd-pci.c33
-rw-r--r--drivers/usb/core/hcd.c189
-rw-r--r--drivers/usb/core/hub.c45
-rw-r--r--drivers/usb/core/message.c4
-rw-r--r--drivers/usb/core/notify.c3
-rw-r--r--drivers/usb/core/phy.c21
-rw-r--r--drivers/usb/core/phy.h1
-rw-r--r--drivers/usb/core/port.c9
-rw-r--r--drivers/usb/core/sysfs.c121
-rw-r--r--drivers/usb/core/usb.c236
-rw-r--r--drivers/usb/core/usb.h9
-rw-r--r--drivers/usb/dwc2/Kconfig1
-rw-r--r--drivers/usb/dwc2/core.c2
-rw-r--r--drivers/usb/dwc2/core.h8
-rw-r--r--drivers/usb/dwc2/gadget.c9
-rw-r--r--drivers/usb/dwc2/hcd.c28
-rw-r--r--drivers/usb/dwc2/hcd.h1
-rw-r--r--drivers/usb/dwc2/params.c6
-rw-r--r--drivers/usb/dwc2/platform.c27
-rw-r--r--drivers/usb/dwc3/Kconfig2
-rw-r--r--drivers/usb/dwc3/core.c75
-rw-r--r--drivers/usb/dwc3/core.h8
-rw-r--r--drivers/usb/dwc3/debug.h252
-rw-r--r--drivers/usb/dwc3/dwc3-keystone.c5
-rw-r--r--drivers/usb/dwc3/dwc3-meson-g12a.c54
-rw-r--r--drivers/usb/dwc3/dwc3-omap.c15
-rw-r--r--drivers/usb/dwc3/dwc3-pci.c8
-rw-r--r--drivers/usb/dwc3/dwc3-qcom.c224
-rw-r--r--drivers/usb/dwc3/dwc3-st.c10
-rw-r--r--drivers/usb/dwc3/ep0.c9
-rw-r--r--drivers/usb/dwc3/gadget.c46
-rw-r--r--drivers/usb/dwc3/gadget.h6
-rw-r--r--drivers/usb/dwc3/host.c22
-rw-r--r--drivers/usb/dwc3/trace.h2
-rw-r--r--drivers/usb/gadget/Kconfig6
-rw-r--r--drivers/usb/gadget/composite.c39
-rw-r--r--drivers/usb/gadget/function/f_eem.c3
-rw-r--r--drivers/usb/gadget/function/f_fs.c9
-rw-r--r--drivers/usb/gadget/function/f_mass_storage.c30
-rw-r--r--drivers/usb/gadget/function/f_uvc.c1
-rw-r--r--drivers/usb/gadget/function/u_audio.c4
-rw-r--r--drivers/usb/gadget/function/u_ether.c10
-rw-r--r--drivers/usb/gadget/function/uvc_v4l2.c4
-rw-r--r--drivers/usb/gadget/legacy/Kconfig8
-rw-r--r--drivers/usb/gadget/legacy/inode.c21
-rw-r--r--drivers/usb/gadget/udc/Kconfig3
-rw-r--r--drivers/usb/gadget/udc/aspeed-vhub/core.c7
-rw-r--r--drivers/usb/gadget/udc/aspeed-vhub/dev.c80
-rw-r--r--drivers/usb/gadget/udc/aspeed-vhub/ep0.c59
-rw-r--r--drivers/usb/gadget/udc/aspeed-vhub/epn.c2
-rw-r--r--drivers/usb/gadget/udc/aspeed-vhub/hub.c15
-rw-r--r--drivers/usb/gadget/udc/aspeed-vhub/vhub.h3
-rw-r--r--drivers/usb/gadget/udc/at91_udc.c3
-rw-r--r--drivers/usb/gadget/udc/atmel_usba_udc.c1
-rw-r--r--drivers/usb/gadget/udc/bcm63xx_udc.c8
-rw-r--r--drivers/usb/gadget/udc/bdc/bdc_core.c4
-rw-r--r--drivers/usb/gadget/udc/core.c2
-rw-r--r--drivers/usb/gadget/udc/fotg210-udc.c3
-rw-r--r--drivers/usb/gadget/udc/gr_udc.c8
-rw-r--r--drivers/usb/gadget/udc/lpc32xx_udc.c19
-rw-r--r--drivers/usb/gadget/udc/net2272.c5
-rw-r--r--drivers/usb/gadget/udc/net2280.c38
-rw-r--r--drivers/usb/gadget/udc/net2280.h3
-rw-r--r--drivers/usb/gadget/udc/omap_udc.c3
-rw-r--r--drivers/usb/gadget/udc/pch_udc.c3
-rw-r--r--drivers/usb/gadget/udc/renesas_usb3.c102
-rw-r--r--drivers/usb/gadget/udc/s3c-hsudc.c4
-rw-r--r--drivers/usb/gadget/udc/s3c2410_udc.c2
-rw-r--r--drivers/usb/gadget/udc/udc-xilinx.c4
-rw-r--r--drivers/usb/host/Kconfig42
-rw-r--r--drivers/usb/host/Makefile4
-rw-r--r--drivers/usb/host/ehci-atmel.c3
-rw-r--r--drivers/usb/host/ehci-exynos.c28
-rw-r--r--drivers/usb/host/ehci-fsl.c52
-rw-r--r--drivers/usb/host/ehci-fsl.h3
-rw-r--r--drivers/usb/host/ehci-grlib.c2
-rw-r--r--drivers/usb/host/ehci-hcd.c4
-rw-r--r--drivers/usb/host/ehci-omap.c4
-rw-r--r--drivers/usb/host/ehci-orion.c3
-rw-r--r--drivers/usb/host/ehci-pci.c4
-rw-r--r--drivers/usb/host/ehci-platform.c4
-rw-r--r--drivers/usb/host/ehci-pmcmsp.c2
-rw-r--r--drivers/usb/host/ehci-ppc-of.c2
-rw-r--r--drivers/usb/host/ehci-ps3.c2
-rw-r--r--drivers/usb/host/ehci-sh.c5
-rw-r--r--drivers/usb/host/ehci-st.c6
-rw-r--r--drivers/usb/host/ehci-w90x900.c130
-rw-r--r--drivers/usb/host/ehci-xilinx-of.c2
-rw-r--r--drivers/usb/host/fhci-hcd.c2
-rw-r--r--drivers/usb/host/fotg210-hcd.c18
-rw-r--r--drivers/usb/host/fsl-mph-dr-of.c10
-rw-r--r--drivers/usb/host/imx21-hcd.c6
-rw-r--r--drivers/usb/host/isp116x-hcd.c6
-rw-r--r--drivers/usb/host/isp1362-hcd.c5
-rw-r--r--drivers/usb/host/isp1362.h2
-rw-r--r--drivers/usb/host/max3421-hcd.c17
-rw-r--r--drivers/usb/host/ohci-exynos.c28
-rw-r--r--drivers/usb/host/ohci-hcd.c42
-rw-r--r--drivers/usb/host/ohci-mem.c37
-rw-r--r--drivers/usb/host/ohci-nxp.c25
-rw-r--r--drivers/usb/host/ohci-pci.c4
-rw-r--r--drivers/usb/host/ohci-platform.c4
-rw-r--r--drivers/usb/host/ohci-ppc-of.c2
-rw-r--r--drivers/usb/host/ohci-ps3.c2
-rw-r--r--drivers/usb/host/ohci-pxa27x.c2
-rw-r--r--drivers/usb/host/ohci-s3c2410.c2
-rw-r--r--drivers/usb/host/ohci-sa1111.c2
-rw-r--r--drivers/usb/host/ohci-sm501.c50
-rw-r--r--drivers/usb/host/ohci-spear.c3
-rw-r--r--drivers/usb/host/ohci-st.c6
-rw-r--r--drivers/usb/host/ohci-tmio.c18
-rw-r--r--drivers/usb/host/ohci.h2
-rw-r--r--drivers/usb/host/oxu210hp-hcd.c446
-rw-r--r--drivers/usb/host/oxu210hp.h448
-rw-r--r--drivers/usb/host/pci-quirks.c45
-rw-r--r--drivers/usb/host/pci-quirks.h2
-rw-r--r--drivers/usb/host/r8a66597-hcd.c6
-rw-r--r--drivers/usb/host/sl811-hcd.c6
-rw-r--r--drivers/usb/host/u132-hcd.c5
-rw-r--r--drivers/usb/host/uhci-grlib.c2
-rw-r--r--drivers/usb/host/uhci-hcd.c2
-rw-r--r--drivers/usb/host/uhci-pci.c2
-rw-r--r--drivers/usb/host/uhci-platform.c2
-rw-r--r--drivers/usb/host/xhci-dbgcap.c1
-rw-r--r--drivers/usb/host/xhci-dbgtty.c4
-rw-r--r--drivers/usb/host/xhci-ext-caps.c18
-rw-r--r--drivers/usb/host/xhci-hub.c2
-rw-r--r--drivers/usb/host/xhci-mem.c1
-rw-r--r--drivers/usb/host/xhci-mtk.c13
-rw-r--r--drivers/usb/host/xhci-mtk.h1
-rw-r--r--drivers/usb/host/xhci-pci.c2
-rw-r--r--drivers/usb/host/xhci-plat.c15
-rw-r--r--drivers/usb/host/xhci-plat.h2
-rw-r--r--drivers/usb/host/xhci-rcar.c25
-rw-r--r--drivers/usb/host/xhci-rcar.h21
-rw-r--r--drivers/usb/host/xhci-ring.c27
-rw-r--r--drivers/usb/host/xhci-tegra.c33
-rw-r--r--drivers/usb/host/xhci.c38
-rw-r--r--drivers/usb/host/xhci.h11
-rw-r--r--drivers/usb/image/microtek.c20
-rw-r--r--drivers/usb/image/microtek.h2
-rw-r--r--drivers/usb/isp1760/isp1760-core.c3
-rw-r--r--drivers/usb/isp1760/isp1760-if.c1
-rw-r--r--drivers/usb/misc/Kconfig6
-rw-r--r--drivers/usb/misc/adutux.c16
-rw-r--r--drivers/usb/misc/cypress_cy7c63.c29
-rw-r--r--drivers/usb/misc/cytherm.c64
-rw-r--r--drivers/usb/misc/ftdi-elan.c7
-rw-r--r--drivers/usb/misc/iowarrior.c7
-rw-r--r--drivers/usb/misc/lvstest.c19
-rw-r--r--drivers/usb/misc/rio500.c66
-rw-r--r--drivers/usb/misc/sisusbvga/sisusb.c10
-rw-r--r--drivers/usb/misc/trancevibrator.c15
-rw-r--r--drivers/usb/misc/usb251xb.c15
-rw-r--r--drivers/usb/misc/usbsevseg.c17
-rw-r--r--drivers/usb/misc/yurex.c2
-rw-r--r--drivers/usb/mon/Kconfig2
-rw-r--r--drivers/usb/mtu3/Kconfig1
-rw-r--r--drivers/usb/mtu3/mtu3.h5
-rw-r--r--drivers/usb/mtu3/mtu3_core.c4
-rw-r--r--drivers/usb/mtu3/mtu3_debugfs.c7
-rw-r--r--drivers/usb/mtu3/mtu3_dr.c48
-rw-r--r--drivers/usb/mtu3/mtu3_dr.h6
-rw-r--r--drivers/usb/mtu3/mtu3_plat.c3
-rw-r--r--drivers/usb/musb/musb_core.c24
-rw-r--r--drivers/usb/musb/musb_host.c2
-rw-r--r--drivers/usb/phy/phy-ab8500-usb.c18
-rw-r--r--drivers/usb/phy/phy-am335x-control.c4
-rw-r--r--drivers/usb/phy/phy-fsl-usb.c14
-rw-r--r--drivers/usb/phy/phy-isp1301.c4
-rw-r--r--drivers/usb/phy/phy-mv-usb.c19
-rw-r--r--drivers/usb/phy/phy-mxs-usb.c67
-rw-r--r--drivers/usb/phy/phy-tahvo.c18
-rw-r--r--drivers/usb/phy/phy-twl6030-usb.c12
-rw-r--r--drivers/usb/renesas_usbhs/Kconfig1
-rw-r--r--drivers/usb/renesas_usbhs/Makefile2
-rw-r--r--drivers/usb/renesas_usbhs/common.c214
-rw-r--r--drivers/usb/renesas_usbhs/common.h9
-rw-r--r--drivers/usb/renesas_usbhs/fifo.c52
-rw-r--r--drivers/usb/renesas_usbhs/mod.c23
-rw-r--r--drivers/usb/renesas_usbhs/mod.h26
-rw-r--r--drivers/usb/renesas_usbhs/mod_gadget.c7
-rw-r--r--drivers/usb/renesas_usbhs/mod_host.c2
-rw-r--r--drivers/usb/renesas_usbhs/rcar2.c22
-rw-r--r--drivers/usb/renesas_usbhs/rcar2.h3
-rw-r--r--drivers/usb/renesas_usbhs/rcar3.c33
-rw-r--r--drivers/usb/renesas_usbhs/rcar3.h5
-rw-r--r--drivers/usb/renesas_usbhs/rza.c18
-rw-r--r--drivers/usb/renesas_usbhs/rza.h3
-rw-r--r--drivers/usb/renesas_usbhs/rza2.c74
-rw-r--r--drivers/usb/roles/class.c59
-rw-r--r--drivers/usb/roles/intel-xhci-usb-role-switch.c54
-rw-r--r--drivers/usb/serial/Kconfig10
-rw-r--r--drivers/usb/serial/belkin_sa.c2
-rw-r--r--drivers/usb/serial/belkin_sa.h2
-rw-r--r--drivers/usb/serial/cypress_m8.c2
-rw-r--r--drivers/usb/serial/empeg.c2
-rw-r--r--drivers/usb/serial/ftdi_sio.c46
-rw-r--r--drivers/usb/serial/ftdi_sio_ids.h6
-rw-r--r--drivers/usb/serial/ir-usb.c2
-rw-r--r--drivers/usb/serial/keyspan_pda.c2
-rw-r--r--drivers/usb/serial/omninet.c2
-rw-r--r--drivers/usb/serial/option.c11
-rw-r--r--drivers/usb/serial/oti6858.c2
-rw-r--r--drivers/usb/serial/pl2303.c2
-rw-r--r--drivers/usb/serial/usb-serial.c2
-rw-r--r--drivers/usb/serial/visor.c2
-rw-r--r--drivers/usb/serial/visor.h2
-rw-r--r--drivers/usb/serial/whiteheat.c2
-rw-r--r--drivers/usb/serial/whiteheat.h2
-rw-r--r--drivers/usb/storage/debug.h2
-rw-r--r--drivers/usb/storage/isd200.c2
-rw-r--r--drivers/usb/storage/realtek_cr.c15
-rw-r--r--drivers/usb/storage/scsiglue.c26
-rw-r--r--drivers/usb/storage/unusual_devs.h2
-rw-r--r--drivers/usb/typec/Kconfig1
-rw-r--r--drivers/usb/typec/bus.h15
-rw-r--r--drivers/usb/typec/class.c33
-rw-r--r--drivers/usb/typec/mux.c240
-rw-r--r--drivers/usb/typec/mux/pi3usb30532.c46
-rw-r--r--drivers/usb/typec/tcpm/fusb302.c104
-rw-r--r--drivers/usb/typec/tcpm/tcpm.c75
-rw-r--r--drivers/usb/typec/tcpm/wcove.c4
-rw-r--r--drivers/usb/typec/tps6598x.c6
-rw-r--r--drivers/usb/typec/ucsi/ucsi.c10
-rw-r--r--drivers/usb/typec/ucsi/ucsi.h1
-rw-r--r--drivers/usb/typec/ucsi/ucsi_ccg.c131
-rw-r--r--drivers/usb/usb-skeleton.c8
-rw-r--r--drivers/usb/usbip/stub.h7
-rw-r--r--drivers/usb/usbip/stub_dev.c50
-rw-r--r--drivers/usb/usbip/stub_main.c65
-rw-r--r--drivers/usb/usbip/stub_rx.c204
-rw-r--r--drivers/usb/usbip/stub_tx.c99
-rw-r--r--drivers/usb/usbip/usbip_common.c59
-rw-r--r--drivers/usb/usbip/vhci_hcd.c12
-rw-r--r--drivers/usb/usbip/vhci_rx.c3
-rw-r--r--drivers/usb/usbip/vhci_tx.c78
-rw-r--r--drivers/usb/usbip/vudc.h2
-rw-r--r--drivers/usb/usbip/vudc_dev.c9
-rw-r--r--drivers/usb/usbip/vudc_main.c1
-rw-r--r--drivers/usb/usbip/vudc_sysfs.c7
-rw-r--r--drivers/vfio/Kconfig2
-rw-r--r--drivers/vfio/mdev/Kconfig2
-rw-r--r--drivers/vfio/mdev/mdev_core.c9
-rw-r--r--drivers/vfio/pci/vfio_pci_nvlink2.c3
-rw-r--r--drivers/vfio/vfio_iommu_spapr_tce.c54
-rw-r--r--drivers/vfio/vfio_iommu_type1.c44
-rw-r--r--drivers/vhost/net.c6
-rw-r--r--drivers/vhost/test.c13
-rw-r--r--drivers/vhost/vhost.c346
-rw-r--r--drivers/vhost/vhost.h2
-rw-r--r--drivers/vhost/vsock.c68
-rw-r--r--drivers/video/backlight/backlight.c2
-rw-r--r--drivers/video/backlight/gpio_backlight.c23
-rw-r--r--drivers/video/backlight/lcd.c12
-rw-r--r--drivers/video/backlight/pwm_bl.c30
-rw-r--r--drivers/video/console/dummycon.c6
-rw-r--r--drivers/video/fbdev/Kconfig86
-rw-r--r--drivers/video/fbdev/Makefile2
-rw-r--r--drivers/video/fbdev/acornfb.c1
-rw-r--r--drivers/video/fbdev/amifb.c4
-rw-r--r--drivers/video/fbdev/arkfb.c4
-rw-r--r--drivers/video/fbdev/atafb.c63
-rw-r--r--drivers/video/fbdev/atmel_lcdfb.c10
-rw-r--r--drivers/video/fbdev/aty/aty128fb.c69
-rw-r--r--drivers/video/fbdev/aty/atyfb_base.c13
-rw-r--r--drivers/video/fbdev/aty/radeon_base.c2
-rw-r--r--drivers/video/fbdev/au1100fb.c24
-rw-r--r--drivers/video/fbdev/au1100fb.h1
-rw-r--r--drivers/video/fbdev/au1200fb.c19
-rw-r--r--drivers/video/fbdev/chipsfb.c1
-rw-r--r--drivers/video/fbdev/cirrusfb.c5
-rw-r--r--drivers/video/fbdev/controlfb.c8
-rw-r--r--drivers/video/fbdev/core/fbcmap.c6
-rw-r--r--drivers/video/fbdev/core/fbcon.c314
-rw-r--r--drivers/video/fbdev/core/fbcon.h6
-rw-r--r--drivers/video/fbdev/core/fbmem.c399
-rw-r--r--drivers/video/fbdev/core/fbsysfs.c20
-rw-r--r--drivers/video/fbdev/cyber2000fb.c6
-rw-r--r--drivers/video/fbdev/da8xx-fb.c119
-rw-r--r--drivers/video/fbdev/efifb.c6
-rw-r--r--drivers/video/fbdev/gbefb.c19
-rw-r--r--drivers/video/fbdev/grvga.c4
-rw-r--r--drivers/video/fbdev/gxt4500.c5
-rw-r--r--drivers/video/fbdev/hyperv_fb.c4
-rw-r--r--drivers/video/fbdev/i740fb.c4
-rw-r--r--drivers/video/fbdev/imsttfb.c5
-rw-r--r--drivers/video/fbdev/imxfb.c11
-rw-r--r--drivers/video/fbdev/intelfb/intelfbdrv.c7
-rw-r--r--drivers/video/fbdev/jz4740_fb.c11
-rw-r--r--drivers/video/fbdev/matrox/matroxfb_base.c2
-rw-r--r--drivers/video/fbdev/mb862xx/mb862xxfbdrv.c5
-rw-r--r--drivers/video/fbdev/mbx/mbxfb.c4
-rw-r--r--drivers/video/fbdev/mmp/hw/mmp_ctrl.c8
-rw-r--r--drivers/video/fbdev/mxsfb.c1028
-rw-r--r--drivers/video/fbdev/neofb.c9
-rw-r--r--drivers/video/fbdev/nuc900fb.c760
-rw-r--r--drivers/video/fbdev/nuc900fb.h51
-rw-r--r--drivers/video/fbdev/omap/omapfb_main.c10
-rw-r--r--drivers/video/fbdev/omap2/omapfb/dss/Kconfig12
-rw-r--r--drivers/video/fbdev/omap2/omapfb/dss/Makefile1
-rw-r--r--drivers/video/fbdev/omap2/omapfb/dss/core.c6
-rw-r--r--drivers/video/fbdev/omap2/omapfb/dss/dss.h4
-rw-r--r--drivers/video/fbdev/omap2/omapfb/dss/rfbi.c1067
-rw-r--r--drivers/video/fbdev/omap2/omapfb/omapfb-main.c6
-rw-r--r--drivers/video/fbdev/omap2/omapfb/omapfb-sysfs.c21
-rw-r--r--drivers/video/fbdev/platinumfb.c5
-rw-r--r--drivers/video/fbdev/pmag-aa-fb.c4
-rw-r--r--drivers/video/fbdev/pmag-ba-fb.c4
-rw-r--r--drivers/video/fbdev/pmagb-b-fb.c4
-rw-r--r--drivers/video/fbdev/pvr2fb.c188
-rw-r--r--drivers/video/fbdev/pxafb.c23
-rw-r--r--drivers/video/fbdev/pxafb.h1
-rw-r--r--drivers/video/fbdev/riva/fbdev.c1
-rw-r--r--drivers/video/fbdev/s3c-fb.c24
-rw-r--r--drivers/video/fbdev/s3fb.c4
-rw-r--r--drivers/video/fbdev/sa1100fb.c52
-rw-r--r--drivers/video/fbdev/sa1100fb.h1
-rw-r--r--drivers/video/fbdev/savage/savagefb_driver.c9
-rw-r--r--drivers/video/fbdev/sh7760fb.c2
-rw-r--r--drivers/video/fbdev/sh_mobile_lcdcfb.c140
-rw-r--r--drivers/video/fbdev/sh_mobile_lcdcfb.h5
-rw-r--r--drivers/video/fbdev/sm501fb.c41
-rw-r--r--drivers/video/fbdev/sm712fb.c1
-rw-r--r--drivers/video/fbdev/smscufx.c4
-rw-r--r--drivers/video/fbdev/ssd1307fb.c4
-rw-r--r--drivers/video/fbdev/sunxvr1000.c1
-rw-r--r--drivers/video/fbdev/sunxvr2500.c1
-rw-r--r--drivers/video/fbdev/sunxvr500.c1
-rw-r--r--drivers/video/fbdev/tgafb.c4
-rw-r--r--drivers/video/fbdev/udlfb.c4
-rw-r--r--drivers/video/fbdev/via/viafbdev.c6
-rw-r--r--drivers/video/fbdev/vt8623fb.c4
-rw-r--r--drivers/video/fbdev/w100fb.c23
-rw-r--r--drivers/video/fbdev/wm8505fb.c13
-rw-r--r--drivers/video/hdmi.c275
-rw-r--r--drivers/virtio/Kconfig11
-rw-r--r--drivers/virtio/virtio_balloon.c13
-rw-r--r--drivers/virtio/virtio_mmio.c7
-rw-r--r--drivers/virtio/virtio_ring.c8
-rw-r--r--drivers/visorbus/visorbus_main.c4
-rw-r--r--drivers/w1/Kconfig2
-rw-r--r--drivers/w1/masters/Kconfig9
-rw-r--r--drivers/w1/masters/Makefile1
-rw-r--r--drivers/w1/masters/mxc_w1.c4
-rw-r--r--drivers/w1/masters/omap_hdq.c4
-rw-r--r--drivers/w1/masters/sgi_w1.c130
-rw-r--r--drivers/w1/slaves/Kconfig6
-rw-r--r--drivers/w1/slaves/Makefile1
-rw-r--r--drivers/w1/slaves/w1_ds2413.c65
-rw-r--r--drivers/w1/slaves/w1_ds250x.c290
-rw-r--r--drivers/w1/slaves/w1_ds2805.c6
-rw-r--r--drivers/watchdog/Kconfig18
-rw-r--r--drivers/watchdog/acquirewdt.c6
-rw-r--r--drivers/watchdog/advantechwdt.c6
-rw-r--r--drivers/watchdog/ar7_wdt.c1
-rw-r--r--drivers/watchdog/aspeed_wdt.c8
-rw-r--r--drivers/watchdog/bcm2835_wdt.c5
-rw-r--r--drivers/watchdog/bcm7038_wdt.c4
-rw-r--r--drivers/watchdog/bcm_kona_wdt.c18
-rw-r--r--drivers/watchdog/cadence_wdt.c4
-rw-r--r--drivers/watchdog/da9052_wdt.c9
-rw-r--r--drivers/watchdog/da9062_wdt.c5
-rw-r--r--drivers/watchdog/davinci_wdt.c14
-rw-r--r--drivers/watchdog/digicolor_wdt.c9
-rw-r--r--drivers/watchdog/ebc-c384_wdt.c9
-rw-r--r--drivers/watchdog/eurotechwdt.c6
-rw-r--r--drivers/watchdog/ftwdt010_wdt.c4
-rw-r--r--drivers/watchdog/gpio_wdt.c7
-rw-r--r--drivers/watchdog/hpwdt.c59
-rw-r--r--drivers/watchdog/i6300esb.c5
-rw-r--r--drivers/watchdog/iTCO_vendor_support.c7
-rw-r--r--drivers/watchdog/iTCO_wdt.c6
-rw-r--r--drivers/watchdog/ib700wdt.c6
-rw-r--r--drivers/watchdog/ie6xx_wdt.c8
-rw-r--r--drivers/watchdog/imx2_wdt.c4
-rw-r--r--drivers/watchdog/imx_sc_wdt.c123
-rw-r--r--drivers/watchdog/intel-mid_wdt.c4
-rw-r--r--drivers/watchdog/jz4740_wdt.c57
-rw-r--r--drivers/watchdog/loongson1_wdt.c4
-rw-r--r--drivers/watchdog/max77620_wdt.c8
-rw-r--r--drivers/watchdog/mei_wdt.c34
-rw-r--r--drivers/watchdog/mena21_wdt.c4
-rw-r--r--drivers/watchdog/menf21bmc_wdt.c4
-rw-r--r--drivers/watchdog/mpc8xxx_wdt.c5
-rw-r--r--drivers/watchdog/mv64x60_wdt.c6
-rw-r--r--drivers/watchdog/ni903x_wdt.c4
-rw-r--r--drivers/watchdog/nic7018_wdt.c1
-rw-r--r--drivers/watchdog/npcm_wdt.c4
-rw-r--r--drivers/watchdog/nv_tco.h6
-rw-r--r--drivers/watchdog/octeon-wdt-main.c11
-rw-r--r--drivers/watchdog/of_xilinx_wdt.c4
-rw-r--r--drivers/watchdog/omap_wdt.c6
-rw-r--r--drivers/watchdog/omap_wdt.h21
-rw-r--r--drivers/watchdog/pc87413_wdt.c6
-rw-r--r--drivers/watchdog/pcwd.c2
-rw-r--r--drivers/watchdog/pcwd_pci.c6
-rw-r--r--drivers/watchdog/pcwd_usb.c6
-rw-r--r--drivers/watchdog/pic32-dmt.c4
-rw-r--r--drivers/watchdog/pic32-wdt.c4
-rw-r--r--drivers/watchdog/pnx4008_wdt.c10
-rw-r--r--drivers/watchdog/qcom-wdt.c4
-rw-r--r--drivers/watchdog/rave-sp-wdt.c1
-rw-r--r--drivers/watchdog/renesas_wdt.c35
-rw-r--r--drivers/watchdog/retu_wdt.c10
-rw-r--r--drivers/watchdog/riowd.c2
-rw-r--r--drivers/watchdog/s3c2410_wdt.c4
-rw-r--r--drivers/watchdog/sa1100_wdt.c6
-rw-r--r--drivers/watchdog/sama5d4_wdt.c29
-rw-r--r--drivers/watchdog/sb_wdog.c1
-rw-r--r--drivers/watchdog/sbc7240_wdt.c11
-rw-r--r--drivers/watchdog/sbc8360.c6
-rw-r--r--drivers/watchdog/sch311x_wdt.c6
-rw-r--r--drivers/watchdog/scx200_wdt.c1
-rw-r--r--drivers/watchdog/smsc37b787_wdt.c2
-rw-r--r--drivers/watchdog/softdog.c6
-rw-r--r--drivers/watchdog/sp5100_tco.c4
-rw-r--r--drivers/watchdog/sp805_wdt.c5
-rw-r--r--drivers/watchdog/sprd_wdt.c1
-rw-r--r--drivers/watchdog/st_lpc_wdt.c4
-rw-r--r--drivers/watchdog/stm32_iwdg.c4
-rw-r--r--drivers/watchdog/stmp3xxx_rtc_wdt.c4
-rw-r--r--drivers/watchdog/tegra_wdt.c4
-rw-r--r--drivers/watchdog/ts4800_wdt.c4
-rw-r--r--drivers/watchdog/w83627hf_wdt.c6
-rw-r--r--drivers/watchdog/wafer5823wdt.c6
-rw-r--r--drivers/watchdog/watchdog_core.c22
-rw-r--r--drivers/watchdog/watchdog_core.h6
-rw-r--r--drivers/watchdog/watchdog_dev.c54
-rw-r--r--drivers/watchdog/wd501p.h6
-rw-r--r--drivers/watchdog/wdt.c8
-rw-r--r--drivers/watchdog/wdt285.c2
-rw-r--r--drivers/watchdog/wdt977.c2
-rw-r--r--drivers/watchdog/wdt_pci.c6
-rw-r--r--drivers/watchdog/wm831x_wdt.c9
-rw-r--r--drivers/watchdog/xen_wdt.c4
-rw-r--r--drivers/xen/Kconfig23
-rw-r--r--drivers/xen/Makefile2
-rw-r--r--drivers/xen/balloon.c23
-rw-r--r--drivers/xen/events/events_base.c12
-rw-r--r--drivers/xen/evtchn.c2
-rw-r--r--drivers/xen/gntdev.c8
-rw-r--r--drivers/xen/privcmd.c39
-rw-r--r--drivers/xen/swiotlb-xen.c44
-rw-r--r--drivers/xen/tmem.c419
-rw-r--r--drivers/xen/xen-balloon.c2
-rw-r--r--drivers/xen/xen-pciback/conf_space_capability.c3
-rw-r--r--drivers/xen/xen-selfballoon.c579
-rw-r--r--drivers/xen/xenfs/super.c21
-rw-r--r--drivers/xen/xlate_mmu.c35
8419 files changed, 857620 insertions, 335669 deletions
diff --git a/drivers/Kconfig b/drivers/Kconfig
index e8231663f201..8befa53f43be 100644
--- a/drivers/Kconfig
+++ b/drivers/Kconfig
@@ -30,8 +30,6 @@ source "drivers/block/Kconfig"
source "drivers/nvme/Kconfig"
-# misc before ide - BLK_DEV_SGIIOC4 depends on SGI_IOC4
-
source "drivers/misc/Kconfig"
source "drivers/ide/Kconfig"
@@ -108,8 +106,6 @@ source "drivers/hid/Kconfig"
source "drivers/usb/Kconfig"
-source "drivers/uwb/Kconfig"
-
source "drivers/mmc/Kconfig"
source "drivers/memstick/Kconfig"
@@ -146,6 +142,8 @@ source "drivers/hv/Kconfig"
source "drivers/xen/Kconfig"
+source "drivers/greybus/Kconfig"
+
source "drivers/staging/Kconfig"
source "drivers/platform/Kconfig"
@@ -188,8 +186,6 @@ source "drivers/ipack/Kconfig"
source "drivers/reset/Kconfig"
-source "drivers/fmc/Kconfig"
-
source "drivers/phy/Kconfig"
source "drivers/powercap/Kconfig"
diff --git a/drivers/Makefile b/drivers/Makefile
index 28b030d7988d..aaef17cc6512 100644
--- a/drivers/Makefile
+++ b/drivers/Makefile
@@ -100,7 +100,6 @@ obj-$(CONFIG_ZORRO) += zorro/
obj-$(CONFIG_ATA_OVER_ETH) += block/aoe/
obj-$(CONFIG_PARIDE) += block/paride/
obj-$(CONFIG_TC) += tc/
-obj-$(CONFIG_UWB) += uwb/
obj-$(CONFIG_USB_PHY) += usb/
obj-$(CONFIG_USB) += usb/
obj-$(CONFIG_USB_SUPPORT) += usb/
@@ -132,7 +131,6 @@ obj-y += mmc/
obj-$(CONFIG_MEMSTICK) += memstick/
obj-$(CONFIG_NEW_LEDS) += leds/
obj-$(CONFIG_INFINIBAND) += infiniband/
-obj-$(CONFIG_SGI_SN) += sn/
obj-y += firmware/
obj-$(CONFIG_CRYPTO) += crypto/
obj-$(CONFIG_SUPERH) += sh/
@@ -148,6 +146,7 @@ obj-$(CONFIG_BCMA) += bcma/
obj-$(CONFIG_VHOST_RING) += vhost/
obj-$(CONFIG_VHOST) += vhost/
obj-$(CONFIG_VLYNQ) += vlynq/
+obj-$(CONFIG_GREYBUS) += greybus/
obj-$(CONFIG_STAGING) += staging/
obj-y += platform/
@@ -168,7 +167,6 @@ obj-$(CONFIG_IIO) += iio/
obj-$(CONFIG_VME_BUS) += vme/
obj-$(CONFIG_IPACK_BUS) += ipack/
obj-$(CONFIG_NTB) += ntb/
-obj-$(CONFIG_FMC) += fmc/
obj-$(CONFIG_POWERCAP) += powercap/
obj-$(CONFIG_MCB) += mcb/
obj-$(CONFIG_PERF_EVENTS) += perf/
diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig
index 283ee94224c6..ebe1e9e5fd81 100644
--- a/drivers/acpi/Kconfig
+++ b/drivers/acpi/Kconfig
@@ -155,7 +155,6 @@ config ACPI_EC_DEBUGFS
config ACPI_AC
tristate "AC Adapter"
- depends on X86
select POWER_SUPPLY
default y
help
@@ -168,7 +167,6 @@ config ACPI_AC
config ACPI_BATTERY
tristate "Battery"
- depends on X86
select POWER_SUPPLY
default y
help
@@ -325,7 +323,7 @@ config ACPI_NUMA
bool "NUMA support"
depends on NUMA
depends on (X86 || IA64 || ARM64)
- default y if IA64_GENERIC || IA64_SGI_SN2 || ARM64
+ default y if IA64 || ARM64
config ACPI_CUSTOM_DSDT_FILE
string "Custom DSDT Table file to include"
@@ -333,7 +331,7 @@ config ACPI_CUSTOM_DSDT_FILE
depends on !STANDALONE
help
This option supports a custom DSDT by linking it into the kernel.
- See Documentation/acpi/dsdt-override.txt
+ See Documentation/admin-guide/acpi/dsdt-override.rst
Enter the full path name to the file which includes the AmlCode
or dsdt_aml_code declaration.
@@ -355,7 +353,7 @@ config ACPI_TABLE_UPGRADE
This option provides functionality to upgrade arbitrary ACPI tables
via initrd. No functional change if no ACPI tables are passed via
initrd, therefore it's safe to say Y.
- See Documentation/acpi/initrd_table_override.txt for details
+ See Documentation/admin-guide/acpi/initrd_table_override.rst for details
config ACPI_TABLE_OVERRIDE_VIA_BUILTIN_INITRD
bool "Override ACPI tables from built-in initrd"
@@ -365,7 +363,7 @@ config ACPI_TABLE_OVERRIDE_VIA_BUILTIN_INITRD
This option provides functionality to override arbitrary ACPI tables
from built-in uncompressed initrd.
- See Documentation/acpi/initrd_table_override.txt for details
+ See Documentation/admin-guide/acpi/initrd_table_override.rst for details
config ACPI_DEBUG
bool "Debug Statements"
@@ -374,7 +372,7 @@ config ACPI_DEBUG
output and increases the kernel size by around 50K.
Use the acpi.debug_layer and acpi.debug_level kernel command-line
- parameters documented in Documentation/acpi/debug.txt and
+ parameters documented in Documentation/firmware-guide/acpi/debug.rst and
Documentation/admin-guide/kernel-parameters.rst to control the type and
amount of debug output.
@@ -445,7 +443,7 @@ config ACPI_CUSTOM_METHOD
help
This debug facility allows ACPI AML methods to be inserted and/or
replaced without rebooting the system. For details refer to:
- Documentation/acpi/method-customizing.txt.
+ Documentation/firmware-guide/acpi/method-customizing.rst.
NOTE: This option is security sensitive, because it allows arbitrary
kernel memory to be written to by root (uid=0) users, allowing them
diff --git a/drivers/acpi/acpi_amba.c b/drivers/acpi/acpi_amba.c
index 8159f0a669b8..49b781a9cd97 100644
--- a/drivers/acpi/acpi_amba.c
+++ b/drivers/acpi/acpi_amba.c
@@ -21,6 +21,15 @@
static const struct acpi_device_id amba_id_list[] = {
{"ARMH0061", 0}, /* PL061 GPIO Device */
+ {"ARMHC500", 0}, /* ARM CoreSight ETM4x */
+ {"ARMHC501", 0}, /* ARM CoreSight ETR */
+ {"ARMHC502", 0}, /* ARM CoreSight STM */
+ {"ARMHC503", 0}, /* ARM CoreSight Debug */
+ {"ARMHC979", 0}, /* ARM CoreSight TPIU */
+ {"ARMHC97C", 0}, /* ARM CoreSight SoC-400 TMC, SoC-600 ETF/ETB */
+ {"ARMHC98D", 0}, /* ARM CoreSight Dynamic Replicator */
+ {"ARMHC9CA", 0}, /* ARM CoreSight CATU */
+ {"ARMHC9FF", 0}, /* ARM CoreSight Dynamic Funnel */
{"", 0},
};
diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c
index ff47317d8ef1..7cd0c9ac71ea 100644
--- a/drivers/acpi/acpi_apd.c
+++ b/drivers/acpi/acpi_apd.c
@@ -57,7 +57,7 @@ struct apd_private_data {
static int acpi_apd_setup(struct apd_private_data *pdata)
{
const struct apd_device_desc *dev_desc = pdata->dev_desc;
- struct clk *clk = ERR_PTR(-ENODEV);
+ struct clk *clk;
if (dev_desc->fixed_clk_rate) {
clk = clk_register_fixed_rate(&pdata->adev->dev,
diff --git a/drivers/acpi/acpi_configfs.c b/drivers/acpi/acpi_configfs.c
index 9c6ff0f5a25e..57d9d574d4dd 100644
--- a/drivers/acpi/acpi_configfs.c
+++ b/drivers/acpi/acpi_configfs.c
@@ -53,11 +53,7 @@ static ssize_t acpi_table_aml_write(struct config_item *cfg,
if (!table->header)
return -ENOMEM;
- ACPI_INFO(("Host-directed Dynamic ACPI Table Load:"));
- ret = acpi_tb_install_and_load_table(
- ACPI_PTR_TO_PHYSADDR(table->header),
- ACPI_TABLE_ORIGIN_EXTERNAL_VIRTUAL, FALSE,
- &table->index);
+ ret = acpi_load_table(table->header);
if (ret) {
kfree(table->header);
table->header = NULL;
diff --git a/drivers/acpi/acpi_lpit.c b/drivers/acpi/acpi_lpit.c
index 6116b0fb86d4..433376e819bb 100644
--- a/drivers/acpi/acpi_lpit.c
+++ b/drivers/acpi/acpi_lpit.c
@@ -129,7 +129,7 @@ static void lpit_update_residency(struct lpit_residency_info *info,
static void lpit_process(u64 begin, u64 end)
{
- while (begin + sizeof(struct acpi_lpit_native) < end) {
+ while (begin + sizeof(struct acpi_lpit_native) <= end) {
struct acpi_lpit_native *lpit_native = (struct acpi_lpit_native *)begin;
if (!lpit_native->header.type && !lpit_native->header.flags) {
@@ -148,7 +148,6 @@ static void lpit_process(u64 begin, u64 end)
void acpi_init_lpit(void)
{
acpi_status status;
- u64 lpit_begin;
struct acpi_table_lpit *lpit;
status = acpi_get_table(ACPI_SIG_LPIT, 0, (struct acpi_table_header **)&lpit);
@@ -156,6 +155,6 @@ void acpi_init_lpit(void)
if (ACPI_FAILURE(status))
return;
- lpit_begin = (u64)lpit + sizeof(*lpit);
- lpit_process(lpit_begin, lpit_begin + lpit->header.length);
+ lpit_process((u64)lpit + sizeof(*lpit),
+ (u64)lpit + lpit->header.length);
}
diff --git a/drivers/acpi/acpi_lpss.c b/drivers/acpi/acpi_lpss.c
index 23484aa877b6..60bbc5090abe 100644
--- a/drivers/acpi/acpi_lpss.c
+++ b/drivers/acpi/acpi_lpss.c
@@ -219,12 +219,13 @@ static void bsw_pwm_setup(struct lpss_private_data *pdata)
}
static const struct lpss_device_desc lpt_dev_desc = {
- .flags = LPSS_CLK | LPSS_CLK_GATE | LPSS_CLK_DIVIDER | LPSS_LTR,
+ .flags = LPSS_CLK | LPSS_CLK_GATE | LPSS_CLK_DIVIDER | LPSS_LTR
+ | LPSS_SAVE_CTX,
.prv_offset = 0x800,
};
static const struct lpss_device_desc lpt_i2c_dev_desc = {
- .flags = LPSS_CLK | LPSS_CLK_GATE | LPSS_LTR,
+ .flags = LPSS_CLK | LPSS_CLK_GATE | LPSS_LTR | LPSS_SAVE_CTX,
.prv_offset = 0x800,
};
@@ -236,7 +237,8 @@ static struct property_entry uart_properties[] = {
};
static const struct lpss_device_desc lpt_uart_dev_desc = {
- .flags = LPSS_CLK | LPSS_CLK_GATE | LPSS_CLK_DIVIDER | LPSS_LTR,
+ .flags = LPSS_CLK | LPSS_CLK_GATE | LPSS_CLK_DIVIDER | LPSS_LTR
+ | LPSS_SAVE_CTX,
.clk_con_id = "baudclk",
.prv_offset = 0x800,
.setup = lpss_uart_setup,
@@ -508,10 +510,10 @@ struct hid_uid {
const char *uid;
};
-static int match_hid_uid(struct device *dev, void *data)
+static int match_hid_uid(struct device *dev, const void *data)
{
struct acpi_device *adev = ACPI_COMPANION(dev);
- struct hid_uid *id = data;
+ const struct hid_uid *id = data;
if (!adev)
return 0;
@@ -1061,6 +1063,13 @@ static int acpi_lpss_suspend_noirq(struct device *dev)
int ret;
if (pdata->dev_desc->resume_from_noirq) {
+ /*
+ * The driver's ->suspend_late callback will be invoked by
+ * acpi_lpss_do_suspend_late(), with the assumption that the
+ * driver really wanted to run that code in ->suspend_noirq, but
+ * it could not run after acpi_dev_suspend() and the driver
+ * expected the latter to be called in the "late" phase.
+ */
ret = acpi_lpss_do_suspend_late(dev);
if (ret)
return ret;
@@ -1091,16 +1100,99 @@ static int acpi_lpss_resume_noirq(struct device *dev)
struct lpss_private_data *pdata = acpi_driver_data(ACPI_COMPANION(dev));
int ret;
- ret = acpi_subsys_resume_noirq(dev);
+ /* Follow acpi_subsys_resume_noirq(). */
+ if (dev_pm_may_skip_resume(dev))
+ return 0;
+
+ if (dev_pm_smart_suspend_and_suspended(dev))
+ pm_runtime_set_active(dev);
+
+ ret = pm_generic_resume_noirq(dev);
if (ret)
return ret;
- if (!dev_pm_may_skip_resume(dev) && pdata->dev_desc->resume_from_noirq)
- ret = acpi_lpss_do_resume_early(dev);
+ if (!pdata->dev_desc->resume_from_noirq)
+ return 0;
- return ret;
+ /*
+ * The driver's ->resume_early callback will be invoked by
+ * acpi_lpss_do_resume_early(), with the assumption that the driver
+ * really wanted to run that code in ->resume_noirq, but it could not
+ * run before acpi_dev_resume() and the driver expected the latter to be
+ * called in the "early" phase.
+ */
+ return acpi_lpss_do_resume_early(dev);
+}
+
+static int acpi_lpss_do_restore_early(struct device *dev)
+{
+ int ret = acpi_lpss_resume(dev);
+
+ return ret ? ret : pm_generic_restore_early(dev);
}
+static int acpi_lpss_restore_early(struct device *dev)
+{
+ struct lpss_private_data *pdata = acpi_driver_data(ACPI_COMPANION(dev));
+
+ if (pdata->dev_desc->resume_from_noirq)
+ return 0;
+
+ return acpi_lpss_do_restore_early(dev);
+}
+
+static int acpi_lpss_restore_noirq(struct device *dev)
+{
+ struct lpss_private_data *pdata = acpi_driver_data(ACPI_COMPANION(dev));
+ int ret;
+
+ ret = pm_generic_restore_noirq(dev);
+ if (ret)
+ return ret;
+
+ if (!pdata->dev_desc->resume_from_noirq)
+ return 0;
+
+ /* This is analogous to what happens in acpi_lpss_resume_noirq(). */
+ return acpi_lpss_do_restore_early(dev);
+}
+
+static int acpi_lpss_do_poweroff_late(struct device *dev)
+{
+ int ret = pm_generic_poweroff_late(dev);
+
+ return ret ? ret : acpi_lpss_suspend(dev, device_may_wakeup(dev));
+}
+
+static int acpi_lpss_poweroff_late(struct device *dev)
+{
+ struct lpss_private_data *pdata = acpi_driver_data(ACPI_COMPANION(dev));
+
+ if (dev_pm_smart_suspend_and_suspended(dev))
+ return 0;
+
+ if (pdata->dev_desc->resume_from_noirq)
+ return 0;
+
+ return acpi_lpss_do_poweroff_late(dev);
+}
+
+static int acpi_lpss_poweroff_noirq(struct device *dev)
+{
+ struct lpss_private_data *pdata = acpi_driver_data(ACPI_COMPANION(dev));
+
+ if (dev_pm_smart_suspend_and_suspended(dev))
+ return 0;
+
+ if (pdata->dev_desc->resume_from_noirq) {
+ /* This is analogous to the acpi_lpss_suspend_noirq() case. */
+ int ret = acpi_lpss_do_poweroff_late(dev);
+ if (ret)
+ return ret;
+ }
+
+ return pm_generic_poweroff_noirq(dev);
+}
#endif /* CONFIG_PM_SLEEP */
static int acpi_lpss_runtime_suspend(struct device *dev)
@@ -1134,14 +1226,11 @@ static struct dev_pm_domain acpi_lpss_pm_domain = {
.resume_noirq = acpi_lpss_resume_noirq,
.resume_early = acpi_lpss_resume_early,
.freeze = acpi_subsys_freeze,
- .freeze_late = acpi_subsys_freeze_late,
- .freeze_noirq = acpi_subsys_freeze_noirq,
- .thaw_noirq = acpi_subsys_thaw_noirq,
- .poweroff = acpi_subsys_suspend,
- .poweroff_late = acpi_lpss_suspend_late,
- .poweroff_noirq = acpi_lpss_suspend_noirq,
- .restore_noirq = acpi_lpss_resume_noirq,
- .restore_early = acpi_lpss_resume_early,
+ .poweroff = acpi_subsys_poweroff,
+ .poweroff_late = acpi_lpss_poweroff_late,
+ .poweroff_noirq = acpi_lpss_poweroff_noirq,
+ .restore_noirq = acpi_lpss_restore_noirq,
+ .restore_early = acpi_lpss_restore_early,
#endif
.runtime_suspend = acpi_lpss_runtime_suspend,
.runtime_resume = acpi_lpss_runtime_resume,
diff --git a/drivers/acpi/acpi_memhotplug.c b/drivers/acpi/acpi_memhotplug.c
index db013dc21c02..e294f44a7850 100644
--- a/drivers/acpi/acpi_memhotplug.c
+++ b/drivers/acpi/acpi_memhotplug.c
@@ -155,16 +155,6 @@ static int acpi_memory_check_device(struct acpi_memory_device *mem_device)
return 0;
}
-static unsigned long acpi_meminfo_start_pfn(struct acpi_memory_info *info)
-{
- return PFN_DOWN(info->start_addr);
-}
-
-static unsigned long acpi_meminfo_end_pfn(struct acpi_memory_info *info)
-{
- return PFN_UP(info->start_addr + info->length-1);
-}
-
static int acpi_bind_memblk(struct memory_block *mem, void *arg)
{
return acpi_bind_one(&mem->dev, arg);
@@ -173,9 +163,8 @@ static int acpi_bind_memblk(struct memory_block *mem, void *arg)
static int acpi_bind_memory_blocks(struct acpi_memory_info *info,
struct acpi_device *adev)
{
- return walk_memory_range(acpi_meminfo_start_pfn(info),
- acpi_meminfo_end_pfn(info), adev,
- acpi_bind_memblk);
+ return walk_memory_blocks(info->start_addr, info->length, adev,
+ acpi_bind_memblk);
}
static int acpi_unbind_memblk(struct memory_block *mem, void *arg)
@@ -186,8 +175,8 @@ static int acpi_unbind_memblk(struct memory_block *mem, void *arg)
static void acpi_unbind_memory_blocks(struct acpi_memory_info *info)
{
- walk_memory_range(acpi_meminfo_start_pfn(info),
- acpi_meminfo_end_pfn(info), NULL, acpi_unbind_memblk);
+ walk_memory_blocks(info->start_addr, info->length, NULL,
+ acpi_unbind_memblk);
}
static int acpi_memory_enable_device(struct acpi_memory_device *mem_device)
diff --git a/drivers/acpi/acpi_pad.c b/drivers/acpi/acpi_pad.c
index 6b3f1217a237..e7dc0133f817 100644
--- a/drivers/acpi/acpi_pad.c
+++ b/drivers/acpi/acpi_pad.c
@@ -64,6 +64,7 @@ static void power_saving_mwait_init(void)
case X86_VENDOR_HYGON:
case X86_VENDOR_AMD:
case X86_VENDOR_INTEL:
+ case X86_VENDOR_ZHAOXIN:
/*
* AMD Fam10h TSC will tick in all
* C/P/S0/S1 states when this bit is set.
diff --git a/drivers/acpi/acpi_processor.c b/drivers/acpi/acpi_processor.c
index 24f065114d42..2c4dda0787e8 100644
--- a/drivers/acpi/acpi_processor.c
+++ b/drivers/acpi/acpi_processor.c
@@ -279,9 +279,13 @@ static int acpi_processor_get_info(struct acpi_device *device)
}
if (acpi_duplicate_processor_id(pr->acpi_id)) {
- dev_err(&device->dev,
- "Failed to get unique processor _UID (0x%x)\n",
- pr->acpi_id);
+ if (pr->acpi_id == 0xff)
+ dev_info_once(&device->dev,
+ "Entry not well-defined, consider updating BIOS\n");
+ else
+ dev_err(&device->dev,
+ "Failed to get unique processor _UID (0x%x)\n",
+ pr->acpi_id);
return -ENODEV;
}
diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c
index 9489ffc06411..4f325e47519f 100644
--- a/drivers/acpi/acpi_video.c
+++ b/drivers/acpi/acpi_video.c
@@ -60,6 +60,12 @@ module_param(report_key_events, int, 0644);
MODULE_PARM_DESC(report_key_events,
"0: none, 1: output changes, 2: brightness changes, 3: all");
+static int hw_changes_brightness = -1;
+module_param(hw_changes_brightness, int, 0644);
+MODULE_PARM_DESC(hw_changes_brightness,
+ "Set this to 1 on buggy hw which changes the brightness itself when "
+ "a hotkey is pressed: -1: auto, 0: normal 1: hw-changes-brightness");
+
/*
* Whether the struct acpi_video_device_attrib::device_id_scheme bit should be
* assumed even if not actually set.
@@ -405,6 +411,14 @@ static int video_set_report_key_events(const struct dmi_system_id *id)
return 0;
}
+static int video_hw_changes_brightness(
+ const struct dmi_system_id *d)
+{
+ if (hw_changes_brightness == -1)
+ hw_changes_brightness = 1;
+ return 0;
+}
+
static const struct dmi_system_id video_dmi_table[] = {
/*
* Broken _BQC workaround http://bugzilla.kernel.org/show_bug.cgi?id=13121
@@ -529,6 +543,21 @@ static const struct dmi_system_id video_dmi_table[] = {
DMI_MATCH(DMI_PRODUCT_NAME, "Vostro V131"),
},
},
+ /*
+ * Some machines change the brightness themselves when a brightness
+ * hotkey gets pressed, despite us telling them not to. In this case
+ * acpi_video_device_notify() should only call backlight_force_update(
+ * BACKLIGHT_UPDATE_HOTKEY) and not do anything else.
+ */
+ {
+ /* https://bugzilla.kernel.org/show_bug.cgi?id=204077 */
+ .callback = video_hw_changes_brightness,
+ .ident = "Packard Bell EasyNote MZ35",
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "Packard Bell"),
+ DMI_MATCH(DMI_PRODUCT_NAME, "EasyNote MZ35"),
+ },
+ },
{}
};
@@ -1612,6 +1641,14 @@ static void acpi_video_device_notify(acpi_handle handle, u32 event, void *data)
bus = video_device->video;
input = bus->input;
+ if (hw_changes_brightness > 0) {
+ if (video_device->backlight)
+ backlight_force_update(video_device->backlight,
+ BACKLIGHT_UPDATE_HOTKEY);
+ acpi_notifier_call_chain(device, event, 0);
+ return;
+ }
+
switch (event) {
case ACPI_VIDEO_NOTIFY_CYCLE_BRIGHTNESS: /* Cycle brightness */
brightness_switch_event(video_device, event);
diff --git a/drivers/acpi/acpica/acevents.h b/drivers/acpi/acpica/acevents.h
index 831660179662..c8652f91054e 100644
--- a/drivers/acpi/acpica/acevents.h
+++ b/drivers/acpi/acpica/acevents.h
@@ -69,7 +69,8 @@ acpi_status
acpi_ev_mask_gpe(struct acpi_gpe_event_info *gpe_event_info, u8 is_masked);
acpi_status
-acpi_ev_add_gpe_reference(struct acpi_gpe_event_info *gpe_event_info);
+acpi_ev_add_gpe_reference(struct acpi_gpe_event_info *gpe_event_info,
+ u8 clear_on_enable);
acpi_status
acpi_ev_remove_gpe_reference(struct acpi_gpe_event_info *gpe_event_info);
diff --git a/drivers/acpi/acpica/acglobal.h b/drivers/acpi/acpica/acglobal.h
index d056a1845613..fd3beea93421 100644
--- a/drivers/acpi/acpica/acglobal.h
+++ b/drivers/acpi/acpica/acglobal.h
@@ -178,7 +178,6 @@ ACPI_GLOBAL(u8, acpi_gbl_verbose_leak_dump);
ACPI_GLOBAL(struct acpi_namespace_node, acpi_gbl_root_node_struct);
ACPI_GLOBAL(struct acpi_namespace_node *, acpi_gbl_root_node);
ACPI_GLOBAL(struct acpi_namespace_node *, acpi_gbl_fadt_gpe_device);
-ACPI_GLOBAL(union acpi_operand_object *, acpi_gbl_module_code_list);
extern const u8 acpi_gbl_ns_properties[ACPI_NUM_NS_TYPES];
extern const struct acpi_predefined_names
diff --git a/drivers/acpi/acpica/aclocal.h b/drivers/acpi/acpica/aclocal.h
index 13d513b81589..1ea52576f0a2 100644
--- a/drivers/acpi/acpica/aclocal.h
+++ b/drivers/acpi/acpica/aclocal.h
@@ -134,12 +134,12 @@ struct acpi_namespace_node {
union acpi_operand_object *object; /* Interpreter object */
u8 descriptor_type; /* Differentiate object descriptor types */
u8 type; /* ACPI Type associated with this name */
- u8 flags; /* Miscellaneous flags */
- acpi_owner_id owner_id; /* Node creator */
+ u16 flags; /* Miscellaneous flags */
union acpi_name_union name; /* ACPI Name, always 4 chars per ACPI spec */
struct acpi_namespace_node *parent; /* Parent node */
struct acpi_namespace_node *child; /* First child */
struct acpi_namespace_node *peer; /* First peer */
+ acpi_owner_id owner_id; /* Node creator */
/*
* The following fields are used by the ASL compiler and disassembler only
diff --git a/drivers/acpi/acpica/acnamesp.h b/drivers/acpi/acpica/acnamesp.h
index 39812fc4386a..7da1864798a0 100644
--- a/drivers/acpi/acpica/acnamesp.h
+++ b/drivers/acpi/acpica/acnamesp.h
@@ -207,8 +207,6 @@ acpi_ns_dump_object_paths(acpi_object_type type,
*/
acpi_status acpi_ns_evaluate(struct acpi_evaluate_info *info);
-void acpi_ns_exec_module_code_list(void);
-
/*
* nsarguments - Argument count/type checking for predefined/reserved names
*/
diff --git a/drivers/acpi/acpica/acobject.h b/drivers/acpi/acpica/acobject.h
index b2ef703d7df8..8def0e3d690f 100644
--- a/drivers/acpi/acpica/acobject.h
+++ b/drivers/acpi/acpica/acobject.h
@@ -153,8 +153,8 @@ struct acpi_object_method {
} dispatch;
u32 aml_length;
- u8 thread_count;
acpi_owner_id owner_id;
+ u8 thread_count;
};
/* Flags for info_flags field above */
diff --git a/drivers/acpi/acpica/acstruct.h b/drivers/acpi/acpica/acstruct.h
index 8a4e6b4aaf2c..218ff4c8b817 100644
--- a/drivers/acpi/acpica/acstruct.h
+++ b/drivers/acpi/acpica/acstruct.h
@@ -167,9 +167,9 @@ struct acpi_evaluate_info {
u32 return_flags; /* Used for return value analysis */
u32 return_btype; /* Bitmapped type of the returned object */
u16 param_count; /* Count of the input argument list */
+ u16 node_flags; /* Same as Node->Flags */
u8 pass_number; /* Parser pass number */
u8 return_object_type; /* Object type of the returned object */
- u8 node_flags; /* Same as Node->Flags */
u8 flags; /* General flags */
};
diff --git a/drivers/acpi/acpica/acutils.h b/drivers/acpi/acpica/acutils.h
index 9022537567e9..601808be86d1 100644
--- a/drivers/acpi/acpica/acutils.h
+++ b/drivers/acpi/acpica/acutils.h
@@ -686,22 +686,26 @@ void acpi_ut_delete_address_lists(void);
/*
* utxferror - various error/warning output functions
*/
+ACPI_PRINTF_LIKE(5)
void ACPI_INTERNAL_VAR_XFACE
acpi_ut_predefined_warning(const char *module_name,
u32 line_number,
char *pathname,
- u8 node_flags, const char *format, ...);
+ u16 node_flags, const char *format, ...);
+ACPI_PRINTF_LIKE(5)
void ACPI_INTERNAL_VAR_XFACE
acpi_ut_predefined_info(const char *module_name,
u32 line_number,
- char *pathname, u8 node_flags, const char *format, ...);
+ char *pathname,
+ u16 node_flags, const char *format, ...);
+ACPI_PRINTF_LIKE(5)
void ACPI_INTERNAL_VAR_XFACE
acpi_ut_predefined_bios_error(const char *module_name,
u32 line_number,
char *pathname,
- u8 node_flags, const char *format, ...);
+ u16 node_flags, const char *format, ...);
void
acpi_ut_prefixed_namespace_error(const char *module_name,
diff --git a/drivers/acpi/acpica/dbhistry.c b/drivers/acpi/acpica/dbhistry.c
index 7809bd94a18d..47d2e5059849 100644
--- a/drivers/acpi/acpica/dbhistry.c
+++ b/drivers/acpi/acpica/dbhistry.c
@@ -121,7 +121,7 @@ void acpi_db_display_history(void)
for (i = 0; i < acpi_gbl_num_history; i++) {
if (acpi_gbl_history_buffer[history_index].command) {
- acpi_os_printf("%3ld %s\n",
+ acpi_os_printf("%3u %s\n",
acpi_gbl_history_buffer[history_index].
cmd_num,
acpi_gbl_history_buffer[history_index].
diff --git a/drivers/acpi/acpica/dbinput.c b/drivers/acpi/acpica/dbinput.c
index 3e5f95390f0d..55a7e10998d8 100644
--- a/drivers/acpi/acpica/dbinput.c
+++ b/drivers/acpi/acpica/dbinput.c
@@ -593,7 +593,7 @@ static u32 acpi_db_get_line(char *input_buffer)
input_buffer)) {
acpi_os_printf
("Buffer overflow while parsing input line (max %u characters)\n",
- sizeof(acpi_gbl_db_parsed_buf));
+ (u32)sizeof(acpi_gbl_db_parsed_buf));
return (0);
}
@@ -853,24 +853,24 @@ acpi_db_command_dispatch(char *input_buffer,
if (param_count == 0) {
acpi_os_printf
- ("Current debug level for file output is: %8.8lX\n",
+ ("Current debug level for file output is: %8.8X\n",
acpi_gbl_db_debug_level);
acpi_os_printf
- ("Current debug level for console output is: %8.8lX\n",
+ ("Current debug level for console output is: %8.8X\n",
acpi_gbl_db_console_debug_level);
} else if (param_count == 2) {
temp = acpi_gbl_db_console_debug_level;
acpi_gbl_db_console_debug_level =
strtoul(acpi_gbl_db_args[1], NULL, 16);
acpi_os_printf
- ("Debug Level for console output was %8.8lX, now %8.8lX\n",
+ ("Debug Level for console output was %8.8X, now %8.8X\n",
temp, acpi_gbl_db_console_debug_level);
} else {
temp = acpi_gbl_db_debug_level;
acpi_gbl_db_debug_level =
strtoul(acpi_gbl_db_args[1], NULL, 16);
acpi_os_printf
- ("Debug Level for file output was %8.8lX, now %8.8lX\n",
+ ("Debug Level for file output was %8.8X, now %8.8X\n",
temp, acpi_gbl_db_debug_level);
}
break;
diff --git a/drivers/acpi/acpica/dbmethod.c b/drivers/acpi/acpica/dbmethod.c
index d8b7a0fe92ec..76a15b6ffc5d 100644
--- a/drivers/acpi/acpica/dbmethod.c
+++ b/drivers/acpi/acpica/dbmethod.c
@@ -302,6 +302,10 @@ acpi_status acpi_db_disassemble_method(char *name)
}
status = acpi_ut_allocate_owner_id(&obj_desc->method.owner_id);
+ if (ACPI_FAILURE(status)) {
+ return (status);
+ }
+
walk_state->owner_id = obj_desc->method.owner_id;
/* Push start scope on scope stack and make it current */
diff --git a/drivers/acpi/acpica/dbobject.c b/drivers/acpi/acpica/dbobject.c
index d220168dca01..f9fc84bc3e84 100644
--- a/drivers/acpi/acpica/dbobject.c
+++ b/drivers/acpi/acpica/dbobject.c
@@ -394,7 +394,6 @@ void acpi_db_decode_locals(struct acpi_walk_state *walk_state)
u8 display_locals = FALSE;
node = walk_state->method_node;
- obj_desc = walk_state->method_desc;
/* There are no locals for the module-level code case */
diff --git a/drivers/acpi/acpica/dbstats.c b/drivers/acpi/acpica/dbstats.c
index bf620937c79b..3af88e70238c 100644
--- a/drivers/acpi/acpica/dbstats.c
+++ b/drivers/acpi/acpica/dbstats.c
@@ -341,17 +341,17 @@ acpi_status acpi_db_display_statistics(char *type_arg)
"ACPI_TYPE", "NODES", "OBJECTS");
for (i = 0; i < ACPI_TYPE_NS_NODE_MAX; i++) {
- acpi_os_printf("%16.16s % 10ld% 10ld\n",
+ acpi_os_printf("%16.16s %10u %10u\n",
acpi_ut_get_type_name(i),
acpi_gbl_node_type_count[i],
acpi_gbl_obj_type_count[i]);
}
- acpi_os_printf("%16.16s % 10ld% 10ld\n", "Misc/Unknown",
+ acpi_os_printf("%16.16s %10u %10u\n", "Misc/Unknown",
acpi_gbl_node_type_count_misc,
acpi_gbl_obj_type_count_misc);
- acpi_os_printf("%16.16s % 10ld% 10ld\n", "TOTALS:",
+ acpi_os_printf("%16.16s %10u %10u\n", "TOTALS:",
acpi_gbl_num_nodes, acpi_gbl_num_objects);
break;
@@ -379,16 +379,14 @@ acpi_status acpi_db_display_statistics(char *type_arg)
case CMD_STAT_MISC:
acpi_os_printf("\nMiscellaneous Statistics:\n\n");
- acpi_os_printf("Calls to AcpiPsFind:.. ........% 7ld\n",
+ acpi_os_printf("%-28s: %7u\n", "Calls to AcpiPsFind",
acpi_gbl_ps_find_count);
- acpi_os_printf("Calls to AcpiNsLookup:..........% 7ld\n",
+ acpi_os_printf("%-28s: %7u\n", "Calls to AcpiNsLookup",
acpi_gbl_ns_lookup_count);
- acpi_os_printf("\n");
-
- acpi_os_printf("Mutex usage:\n\n");
+ acpi_os_printf("\nMutex usage:\n\n");
for (i = 0; i < ACPI_NUM_MUTEX; i++) {
- acpi_os_printf("%-28s: % 7ld\n",
+ acpi_os_printf("%-28s: %7u\n",
acpi_ut_get_mutex_name(i),
acpi_gbl_mutex_info[i].use_count);
}
@@ -399,87 +397,87 @@ acpi_status acpi_db_display_statistics(char *type_arg)
acpi_os_printf("\nInternal object sizes:\n\n");
acpi_os_printf("Common %3d\n",
- sizeof(struct acpi_object_common));
+ (u32)sizeof(struct acpi_object_common));
acpi_os_printf("Number %3d\n",
- sizeof(struct acpi_object_integer));
+ (u32)sizeof(struct acpi_object_integer));
acpi_os_printf("String %3d\n",
- sizeof(struct acpi_object_string));
+ (u32)sizeof(struct acpi_object_string));
acpi_os_printf("Buffer %3d\n",
- sizeof(struct acpi_object_buffer));
+ (u32)sizeof(struct acpi_object_buffer));
acpi_os_printf("Package %3d\n",
- sizeof(struct acpi_object_package));
+ (u32)sizeof(struct acpi_object_package));
acpi_os_printf("BufferField %3d\n",
- sizeof(struct acpi_object_buffer_field));
+ (u32)sizeof(struct acpi_object_buffer_field));
acpi_os_printf("Device %3d\n",
- sizeof(struct acpi_object_device));
+ (u32)sizeof(struct acpi_object_device));
acpi_os_printf("Event %3d\n",
- sizeof(struct acpi_object_event));
+ (u32)sizeof(struct acpi_object_event));
acpi_os_printf("Method %3d\n",
- sizeof(struct acpi_object_method));
+ (u32)sizeof(struct acpi_object_method));
acpi_os_printf("Mutex %3d\n",
- sizeof(struct acpi_object_mutex));
+ (u32)sizeof(struct acpi_object_mutex));
acpi_os_printf("Region %3d\n",
- sizeof(struct acpi_object_region));
+ (u32)sizeof(struct acpi_object_region));
acpi_os_printf("PowerResource %3d\n",
- sizeof(struct acpi_object_power_resource));
+ (u32)sizeof(struct acpi_object_power_resource));
acpi_os_printf("Processor %3d\n",
- sizeof(struct acpi_object_processor));
+ (u32)sizeof(struct acpi_object_processor));
acpi_os_printf("ThermalZone %3d\n",
- sizeof(struct acpi_object_thermal_zone));
+ (u32)sizeof(struct acpi_object_thermal_zone));
acpi_os_printf("RegionField %3d\n",
- sizeof(struct acpi_object_region_field));
+ (u32)sizeof(struct acpi_object_region_field));
acpi_os_printf("BankField %3d\n",
- sizeof(struct acpi_object_bank_field));
+ (u32)sizeof(struct acpi_object_bank_field));
acpi_os_printf("IndexField %3d\n",
- sizeof(struct acpi_object_index_field));
+ (u32)sizeof(struct acpi_object_index_field));
acpi_os_printf("Reference %3d\n",
- sizeof(struct acpi_object_reference));
+ (u32)sizeof(struct acpi_object_reference));
acpi_os_printf("Notify %3d\n",
- sizeof(struct acpi_object_notify_handler));
+ (u32)sizeof(struct acpi_object_notify_handler));
acpi_os_printf("AddressSpace %3d\n",
- sizeof(struct acpi_object_addr_handler));
+ (u32)sizeof(struct acpi_object_addr_handler));
acpi_os_printf("Extra %3d\n",
- sizeof(struct acpi_object_extra));
+ (u32)sizeof(struct acpi_object_extra));
acpi_os_printf("Data %3d\n",
- sizeof(struct acpi_object_data));
+ (u32)sizeof(struct acpi_object_data));
acpi_os_printf("\n");
acpi_os_printf("ParseObject %3d\n",
- sizeof(struct acpi_parse_obj_common));
+ (u32)sizeof(struct acpi_parse_obj_common));
acpi_os_printf("ParseObjectNamed %3d\n",
- sizeof(struct acpi_parse_obj_named));
+ (u32)sizeof(struct acpi_parse_obj_named));
acpi_os_printf("ParseObjectAsl %3d\n",
- sizeof(struct acpi_parse_obj_asl));
+ (u32)sizeof(struct acpi_parse_obj_asl));
acpi_os_printf("OperandObject %3d\n",
- sizeof(union acpi_operand_object));
+ (u32)sizeof(union acpi_operand_object));
acpi_os_printf("NamespaceNode %3d\n",
- sizeof(struct acpi_namespace_node));
+ (u32)sizeof(struct acpi_namespace_node));
acpi_os_printf("AcpiObject %3d\n",
- sizeof(union acpi_object));
+ (u32)sizeof(union acpi_object));
acpi_os_printf("\n");
acpi_os_printf("Generic State %3d\n",
- sizeof(union acpi_generic_state));
+ (u32)sizeof(union acpi_generic_state));
acpi_os_printf("Common State %3d\n",
- sizeof(struct acpi_common_state));
+ (u32)sizeof(struct acpi_common_state));
acpi_os_printf("Control State %3d\n",
- sizeof(struct acpi_control_state));
+ (u32)sizeof(struct acpi_control_state));
acpi_os_printf("Update State %3d\n",
- sizeof(struct acpi_update_state));
+ (u32)sizeof(struct acpi_update_state));
acpi_os_printf("Scope State %3d\n",
- sizeof(struct acpi_scope_state));
+ (u32)sizeof(struct acpi_scope_state));
acpi_os_printf("Parse Scope %3d\n",
- sizeof(struct acpi_pscope_state));
+ (u32)sizeof(struct acpi_pscope_state));
acpi_os_printf("Package State %3d\n",
- sizeof(struct acpi_pkg_state));
+ (u32)sizeof(struct acpi_pkg_state));
acpi_os_printf("Thread State %3d\n",
- sizeof(struct acpi_thread_state));
+ (u32)sizeof(struct acpi_thread_state));
acpi_os_printf("Result Values %3d\n",
- sizeof(struct acpi_result_values));
+ (u32)sizeof(struct acpi_result_values));
acpi_os_printf("Notify Info %3d\n",
- sizeof(struct acpi_notify_info));
+ (u32)sizeof(struct acpi_notify_info));
break;
case CMD_STAT_STACK:
diff --git a/drivers/acpi/acpica/dsinit.c b/drivers/acpi/acpica/dsinit.c
index 4ebd23700bbc..a1ffed29903b 100644
--- a/drivers/acpi/acpica/dsinit.c
+++ b/drivers/acpi/acpica/dsinit.c
@@ -202,7 +202,7 @@ acpi_ds_initialize_objects(u32 table_index,
if (ACPI_COMPARE_NAMESEG(table->signature, ACPI_SIG_DSDT)) {
ACPI_DEBUG_PRINT_RAW((ACPI_DB_INIT,
- "\nInitializing Namespace objects:\n"));
+ "\nACPI table initialization:\n"));
}
/* Summary of objects initialized */
diff --git a/drivers/acpi/acpica/evgpe.c b/drivers/acpi/acpica/evgpe.c
index 62d3aa74277b..344feba29063 100644
--- a/drivers/acpi/acpica/evgpe.c
+++ b/drivers/acpi/acpica/evgpe.c
@@ -146,6 +146,7 @@ acpi_ev_mask_gpe(struct acpi_gpe_event_info *gpe_event_info, u8 is_masked)
* FUNCTION: acpi_ev_add_gpe_reference
*
* PARAMETERS: gpe_event_info - Add a reference to this GPE
+ * clear_on_enable - Clear GPE status before enabling it
*
* RETURN: Status
*
@@ -155,7 +156,8 @@ acpi_ev_mask_gpe(struct acpi_gpe_event_info *gpe_event_info, u8 is_masked)
******************************************************************************/
acpi_status
-acpi_ev_add_gpe_reference(struct acpi_gpe_event_info *gpe_event_info)
+acpi_ev_add_gpe_reference(struct acpi_gpe_event_info *gpe_event_info,
+ u8 clear_on_enable)
{
acpi_status status = AE_OK;
@@ -170,6 +172,10 @@ acpi_ev_add_gpe_reference(struct acpi_gpe_event_info *gpe_event_info)
/* Enable on first reference */
+ if (clear_on_enable) {
+ (void)acpi_hw_clear_gpe(gpe_event_info);
+ }
+
status = acpi_ev_update_gpe_enable_mask(gpe_event_info);
if (ACPI_SUCCESS(status)) {
status = acpi_ev_enable_gpe(gpe_event_info);
diff --git a/drivers/acpi/acpica/evgpeblk.c b/drivers/acpi/acpica/evgpeblk.c
index 328d1d6123ad..fb15e9e2373b 100644
--- a/drivers/acpi/acpica/evgpeblk.c
+++ b/drivers/acpi/acpica/evgpeblk.c
@@ -453,7 +453,7 @@ acpi_ev_initialize_gpe_block(struct acpi_gpe_xrupt_info *gpe_xrupt_info,
continue;
}
- status = acpi_ev_add_gpe_reference(gpe_event_info);
+ status = acpi_ev_add_gpe_reference(gpe_event_info, FALSE);
if (ACPI_FAILURE(status)) {
ACPI_EXCEPTION((AE_INFO, status,
"Could not enable GPE 0x%02X",
diff --git a/drivers/acpi/acpica/evxface.c b/drivers/acpi/acpica/evxface.c
index 3df00eb6621b..279ef0557aa3 100644
--- a/drivers/acpi/acpica/evxface.c
+++ b/drivers/acpi/acpica/evxface.c
@@ -971,7 +971,7 @@ acpi_remove_gpe_handler(acpi_handle gpe_device,
ACPI_GPE_DISPATCH_METHOD) ||
(ACPI_GPE_DISPATCH_TYPE(handler->original_flags) ==
ACPI_GPE_DISPATCH_NOTIFY)) && handler->originally_enabled) {
- (void)acpi_ev_add_gpe_reference(gpe_event_info);
+ (void)acpi_ev_add_gpe_reference(gpe_event_info, FALSE);
if (ACPI_GPE_IS_POLLING_NEEDED(gpe_event_info)) {
/* Poll edge triggered GPEs to handle existing events */
diff --git a/drivers/acpi/acpica/evxfgpe.c b/drivers/acpi/acpica/evxfgpe.c
index 30a083902f52..04a40d563dd6 100644
--- a/drivers/acpi/acpica/evxfgpe.c
+++ b/drivers/acpi/acpica/evxfgpe.c
@@ -108,7 +108,7 @@ acpi_status acpi_enable_gpe(acpi_handle gpe_device, u32 gpe_number)
if (gpe_event_info) {
if (ACPI_GPE_DISPATCH_TYPE(gpe_event_info->flags) !=
ACPI_GPE_DISPATCH_NONE) {
- status = acpi_ev_add_gpe_reference(gpe_event_info);
+ status = acpi_ev_add_gpe_reference(gpe_event_info, TRUE);
if (ACPI_SUCCESS(status) &&
ACPI_GPE_IS_POLLING_NEEDED(gpe_event_info)) {
@@ -644,17 +644,17 @@ ACPI_EXPORT_SYMBOL(acpi_get_gpe_status)
* PARAMETERS: gpe_device - Parent GPE Device. NULL for GPE0/GPE1
* gpe_number - GPE level within the GPE block
*
- * RETURN: None
+ * RETURN: INTERRUPT_HANDLED or INTERRUPT_NOT_HANDLED
*
* DESCRIPTION: Detect and dispatch a General Purpose Event to either a function
* (e.g. EC) or method (e.g. _Lxx/_Exx) handler.
*
******************************************************************************/
-void acpi_dispatch_gpe(acpi_handle gpe_device, u32 gpe_number)
+u32 acpi_dispatch_gpe(acpi_handle gpe_device, u32 gpe_number)
{
ACPI_FUNCTION_TRACE(acpi_dispatch_gpe);
- acpi_ev_detect_gpe(gpe_device, NULL, gpe_number);
+ return acpi_ev_detect_gpe(gpe_device, NULL, gpe_number);
}
ACPI_EXPORT_SYMBOL(acpi_dispatch_gpe)
diff --git a/drivers/acpi/acpica/exconfig.c b/drivers/acpi/acpica/exconfig.c
index 587aeeeb5070..46a8baf28bd0 100644
--- a/drivers/acpi/acpica/exconfig.c
+++ b/drivers/acpi/acpica/exconfig.c
@@ -174,12 +174,11 @@ acpi_ex_load_table_op(struct acpi_walk_state *walk_state,
return_ACPI_STATUS(status);
}
- /* Complete the initialization/resolution of package objects */
+ /* Complete the initialization/resolution of new objects */
- status = acpi_ns_walk_namespace(ACPI_TYPE_PACKAGE, ACPI_ROOT_OBJECT,
- ACPI_UINT32_MAX, 0,
- acpi_ns_init_one_package, NULL, NULL,
- NULL);
+ acpi_ex_exit_interpreter();
+ acpi_ns_initialize_objects();
+ acpi_ex_enter_interpreter();
/* Parameter Data (optional) */
@@ -437,12 +436,11 @@ acpi_ex_load_op(union acpi_operand_object *obj_desc,
return_ACPI_STATUS(status);
}
- /* Complete the initialization/resolution of package objects */
+ /* Complete the initialization/resolution of new objects */
- status = acpi_ns_walk_namespace(ACPI_TYPE_PACKAGE, ACPI_ROOT_OBJECT,
- ACPI_UINT32_MAX, 0,
- acpi_ns_init_one_package, NULL, NULL,
- NULL);
+ acpi_ex_exit_interpreter();
+ acpi_ns_initialize_objects();
+ acpi_ex_enter_interpreter();
/* Store the ddb_handle into the Target operand */
diff --git a/drivers/acpi/acpica/exdump.c b/drivers/acpi/acpica/exdump.c
index 6526b2deeaad..a9bc938a3b55 100644
--- a/drivers/acpi/acpica/exdump.c
+++ b/drivers/acpi/acpica/exdump.c
@@ -94,7 +94,7 @@ static struct acpi_exdump_info acpi_ex_dump_method[9] = {
"Parameter Count"},
{ACPI_EXD_UINT8, ACPI_EXD_OFFSET(method.sync_level), "Sync Level"},
{ACPI_EXD_POINTER, ACPI_EXD_OFFSET(method.mutex), "Mutex"},
- {ACPI_EXD_UINT8, ACPI_EXD_OFFSET(method.owner_id), "Owner Id"},
+ {ACPI_EXD_UINT16, ACPI_EXD_OFFSET(method.owner_id), "Owner Id"},
{ACPI_EXD_UINT8, ACPI_EXD_OFFSET(method.thread_count), "Thread Count"},
{ACPI_EXD_UINT32, ACPI_EXD_OFFSET(method.aml_length), "Aml Length"},
{ACPI_EXD_POINTER, ACPI_EXD_OFFSET(method.aml_start), "Aml Start"}
@@ -269,8 +269,8 @@ static struct acpi_exdump_info acpi_ex_dump_field_common[7] = {
static struct acpi_exdump_info acpi_ex_dump_node[7] = {
{ACPI_EXD_INIT, ACPI_EXD_TABLE_SIZE(acpi_ex_dump_node), NULL},
- {ACPI_EXD_UINT8, ACPI_EXD_NSOFFSET(flags), "Flags"},
- {ACPI_EXD_UINT8, ACPI_EXD_NSOFFSET(owner_id), "Owner Id"},
+ {ACPI_EXD_UINT16, ACPI_EXD_NSOFFSET(flags), "Flags"},
+ {ACPI_EXD_UINT16, ACPI_EXD_NSOFFSET(owner_id), "Owner Id"},
{ACPI_EXD_LIST, ACPI_EXD_NSOFFSET(object), "Object List"},
{ACPI_EXD_NODE, ACPI_EXD_NSOFFSET(parent), "Parent"},
{ACPI_EXD_NODE, ACPI_EXD_NSOFFSET(child), "Child"},
diff --git a/drivers/acpi/acpica/nsaccess.c b/drivers/acpi/acpica/nsaccess.c
index 7b855603f81a..3f045b5953b2 100644
--- a/drivers/acpi/acpica/nsaccess.c
+++ b/drivers/acpi/acpica/nsaccess.c
@@ -36,6 +36,7 @@ acpi_status acpi_ns_root_initialize(void)
acpi_status status;
const struct acpi_predefined_names *init_val = NULL;
struct acpi_namespace_node *new_node;
+ struct acpi_namespace_node *prev_node = NULL;
union acpi_operand_object *obj_desc;
acpi_string val = NULL;
@@ -61,12 +62,28 @@ acpi_status acpi_ns_root_initialize(void)
*/
acpi_gbl_root_node = &acpi_gbl_root_node_struct;
- /* Enter the pre-defined names in the name table */
+ /* Enter the predefined names in the name table */
ACPI_DEBUG_PRINT((ACPI_DB_INFO,
"Entering predefined entries into namespace\n"));
+ /*
+ * Create the initial (default) namespace.
+ * This namespace looks like something similar to this:
+ *
+ * ACPI Namespace (from Namespace Root):
+ * 0 _GPE Scope 00203160 00
+ * 0 _PR_ Scope 002031D0 00
+ * 0 _SB_ Device 00203240 00 Notify Object: 0020ADD8
+ * 0 _SI_ Scope 002032B0 00
+ * 0 _TZ_ Device 00203320 00
+ * 0 _REV Integer 00203390 00 = 0000000000000002
+ * 0 _OS_ String 00203488 00 Len 14 "Microsoft Windows NT"
+ * 0 _GL_ Mutex 00203580 00 Object 002035F0
+ * 0 _OSI Method 00203678 00 Args 1 Len 0000 Aml 00000000
+ */
for (init_val = acpi_gbl_pre_defined_names; init_val->name; init_val++) {
+ status = AE_OK;
/* _OSI is optional for now, will be permanent later */
@@ -75,17 +92,32 @@ acpi_status acpi_ns_root_initialize(void)
continue;
}
- status =
- acpi_ns_lookup(NULL, ACPI_CAST_PTR(char, init_val->name),
- init_val->type, ACPI_IMODE_LOAD_PASS2,
- ACPI_NS_NO_UPSEARCH, NULL, &new_node);
- if (ACPI_FAILURE(status)) {
- ACPI_EXCEPTION((AE_INFO, status,
- "Could not create predefined name %s",
- init_val->name));
- continue;
+ /*
+ * Create, init, and link the new predefined name
+ * Note: No need to use acpi_ns_lookup here because all the
+ * predefined names are at the root level. It is much easier to
+ * just create and link the new node(s) here.
+ */
+ new_node =
+ ACPI_ALLOCATE_ZEROED(sizeof(struct acpi_namespace_node));
+ if (!new_node) {
+ status = AE_NO_MEMORY;
+ goto unlock_and_exit;
}
+ ACPI_COPY_NAMESEG(new_node->name.ascii, init_val->name);
+ new_node->descriptor_type = ACPI_DESC_TYPE_NAMED;
+ new_node->type = init_val->type;
+
+ if (!prev_node) {
+ acpi_gbl_root_node_struct.child = new_node;
+ } else {
+ prev_node->peer = new_node;
+ }
+
+ new_node->parent = &acpi_gbl_root_node_struct;
+ prev_node = new_node;
+
/*
* Name entered successfully. If entry in pre_defined_names[] specifies
* an initial value, create the initial value.
@@ -131,7 +163,7 @@ acpi_status acpi_ns_root_initialize(void)
new_node->value = obj_desc->method.param_count;
#else
- /* Mark this as a very SPECIAL method */
+ /* Mark this as a very SPECIAL method (_OSI) */
obj_desc->method.info_flags =
ACPI_METHOD_INTERNAL_ONLY;
@@ -566,7 +598,7 @@ acpi_ns_lookup(union acpi_generic_state *scope_info,
if (flags & ACPI_NS_PREFIX_MUST_EXIST) {
acpi_os_printf(ACPI_MSG_BIOS_ERROR
"Object does not exist: %4.4s\n",
- &simple_name);
+ (char *)&simple_name);
}
#endif
/* Name not found in ACPI namespace */
diff --git a/drivers/acpi/acpica/nsalloc.c b/drivers/acpi/acpica/nsalloc.c
index 6eb63db72249..fe9b3639a87d 100644
--- a/drivers/acpi/acpica/nsalloc.c
+++ b/drivers/acpi/acpica/nsalloc.c
@@ -241,7 +241,7 @@ void acpi_ns_install_node(struct acpi_walk_state *walk_state, struct acpi_namesp
node->type = (u8) type;
ACPI_DEBUG_PRINT((ACPI_DB_NAMES,
- "%4.4s (%s) [Node %p Owner %X] added to %4.4s (%s) [Node %p]\n",
+ "%4.4s (%s) [Node %p Owner %3.3X] added to %4.4s (%s) [Node %p]\n",
acpi_ut_get_node_name(node),
acpi_ut_get_type_name(node->type), node, owner_id,
acpi_ut_get_node_name(parent_node),
diff --git a/drivers/acpi/acpica/nsdump.c b/drivers/acpi/acpica/nsdump.c
index 1b12c172e115..9731d7cf1b83 100644
--- a/drivers/acpi/acpica/nsdump.c
+++ b/drivers/acpi/acpica/nsdump.c
@@ -197,7 +197,7 @@ acpi_ns_dump_one_object(acpi_handle obj_handle,
/* Now we can print out the pertinent information */
- acpi_os_printf(" %-12s %p %2.2X ",
+ acpi_os_printf(" %-12s %p %3.3X ",
acpi_ut_get_type_name(type), this_node,
this_node->owner_id);
diff --git a/drivers/acpi/acpica/nseval.c b/drivers/acpi/acpica/nseval.c
index 6390b7951ebf..63748ac699f7 100644
--- a/drivers/acpi/acpica/nseval.c
+++ b/drivers/acpi/acpica/nseval.c
@@ -14,11 +14,6 @@
#define _COMPONENT ACPI_NAMESPACE
ACPI_MODULE_NAME("nseval")
-/* Local prototypes */
-static void
-acpi_ns_exec_module_code(union acpi_operand_object *method_obj,
- struct acpi_evaluate_info *info);
-
/*******************************************************************************
*
* FUNCTION: acpi_ns_evaluate
@@ -44,7 +39,6 @@ acpi_ns_exec_module_code(union acpi_operand_object *method_obj,
* MUTEX: Locks interpreter
*
******************************************************************************/
-
acpi_status acpi_ns_evaluate(struct acpi_evaluate_info *info)
{
acpi_status status;
@@ -310,187 +304,3 @@ cleanup:
info->full_pathname = NULL;
return_ACPI_STATUS(status);
}
-
-/*******************************************************************************
- *
- * FUNCTION: acpi_ns_exec_module_code_list
- *
- * PARAMETERS: None
- *
- * RETURN: None. Exceptions during method execution are ignored, since
- * we cannot abort a table load.
- *
- * DESCRIPTION: Execute all elements of the global module-level code list.
- * Each element is executed as a single control method.
- *
- * NOTE: With this option enabled, each block of detected executable AML
- * code that is outside of any control method is wrapped with a temporary
- * control method object and placed on a global list. The methods on this
- * list are executed below.
- *
- * This function executes the module-level code for all tables only after
- * all of the tables have been loaded. It is a legacy option and is
- * not compatible with other ACPI implementations. See acpi_ns_load_table.
- *
- * This function will be removed when the legacy option is removed.
- *
- ******************************************************************************/
-
-void acpi_ns_exec_module_code_list(void)
-{
- union acpi_operand_object *prev;
- union acpi_operand_object *next;
- struct acpi_evaluate_info *info;
- u32 method_count = 0;
-
- ACPI_FUNCTION_TRACE(ns_exec_module_code_list);
-
- /* Exit now if the list is empty */
-
- next = acpi_gbl_module_code_list;
- if (!next) {
- ACPI_DEBUG_PRINT((ACPI_DB_INIT_NAMES,
- "Legacy MLC block list is empty\n"));
-
- return_VOID;
- }
-
- /* Allocate the evaluation information block */
-
- info = ACPI_ALLOCATE(sizeof(struct acpi_evaluate_info));
- if (!info) {
- return_VOID;
- }
-
- /* Walk the list, executing each "method" */
-
- while (next) {
- prev = next;
- next = next->method.mutex;
-
- /* Clear the link field and execute the method */
-
- prev->method.mutex = NULL;
- acpi_ns_exec_module_code(prev, info);
- method_count++;
-
- /* Delete the (temporary) method object */
-
- acpi_ut_remove_reference(prev);
- }
-
- ACPI_INFO(("Executed %u blocks of module-level executable AML code",
- method_count));
-
- ACPI_FREE(info);
- acpi_gbl_module_code_list = NULL;
- return_VOID;
-}
-
-/*******************************************************************************
- *
- * FUNCTION: acpi_ns_exec_module_code
- *
- * PARAMETERS: method_obj - Object container for the module-level code
- * info - Info block for method evaluation
- *
- * RETURN: None. Exceptions during method execution are ignored, since
- * we cannot abort a table load.
- *
- * DESCRIPTION: Execute a control method containing a block of module-level
- * executable AML code. The control method is temporarily
- * installed to the root node, then evaluated.
- *
- ******************************************************************************/
-
-static void
-acpi_ns_exec_module_code(union acpi_operand_object *method_obj,
- struct acpi_evaluate_info *info)
-{
- union acpi_operand_object *parent_obj;
- struct acpi_namespace_node *parent_node;
- acpi_object_type type;
- acpi_status status;
-
- ACPI_FUNCTION_TRACE(ns_exec_module_code);
-
- /*
- * Get the parent node. We cheat by using the next_object field
- * of the method object descriptor.
- */
- parent_node =
- ACPI_CAST_PTR(struct acpi_namespace_node,
- method_obj->method.next_object);
- type = acpi_ns_get_type(parent_node);
-
- /*
- * Get the region handler and save it in the method object. We may need
- * this if an operation region declaration causes a _REG method to be run.
- *
- * We can't do this in acpi_ps_link_module_code because
- * acpi_gbl_root_node->Object is NULL at PASS1.
- */
- if ((type == ACPI_TYPE_DEVICE) && parent_node->object) {
- method_obj->method.dispatch.handler =
- parent_node->object->device.handler;
- }
-
- /* Must clear next_object (acpi_ns_attach_object needs the field) */
-
- method_obj->method.next_object = NULL;
-
- /* Initialize the evaluation information block */
-
- memset(info, 0, sizeof(struct acpi_evaluate_info));
- info->prefix_node = parent_node;
-
- /*
- * Get the currently attached parent object. Add a reference,
- * because the ref count will be decreased when the method object
- * is installed to the parent node.
- */
- parent_obj = acpi_ns_get_attached_object(parent_node);
- if (parent_obj) {
- acpi_ut_add_reference(parent_obj);
- }
-
- /* Install the method (module-level code) in the parent node */
-
- status =
- acpi_ns_attach_object(parent_node, method_obj, ACPI_TYPE_METHOD);
- if (ACPI_FAILURE(status)) {
- goto exit;
- }
-
- /* Execute the parent node as a control method */
-
- status = acpi_ns_evaluate(info);
-
- ACPI_DEBUG_PRINT((ACPI_DB_INIT_NAMES,
- "Executed module-level code at %p\n",
- method_obj->method.aml_start));
-
- /* Delete a possible implicit return value (in slack mode) */
-
- if (info->return_object) {
- acpi_ut_remove_reference(info->return_object);
- }
-
- /* Detach the temporary method object */
-
- acpi_ns_detach_object(parent_node);
-
- /* Restore the original parent object */
-
- if (parent_obj) {
- status = acpi_ns_attach_object(parent_node, parent_obj, type);
- } else {
- parent_node->type = (u8)type;
- }
-
-exit:
- if (parent_obj) {
- acpi_ut_remove_reference(parent_obj);
- }
- return_VOID;
-}
diff --git a/drivers/acpi/acpica/nsinit.c b/drivers/acpi/acpica/nsinit.c
index 53e5d00d3a5e..61e9dfc9fe8c 100644
--- a/drivers/acpi/acpica/nsinit.c
+++ b/drivers/acpi/acpica/nsinit.c
@@ -55,14 +55,19 @@ acpi_status acpi_ns_initialize_objects(void)
ACPI_DEBUG_PRINT((ACPI_DB_DISPATCH,
"**** Starting initialization of namespace objects ****\n"));
ACPI_DEBUG_PRINT_RAW((ACPI_DB_INIT,
- "Completing Region/Field/Buffer/Package initialization:\n"));
+ "Final data object initialization: "));
- /* Set all init info to zero */
+ /* Clear the info block */
memset(&info, 0, sizeof(struct acpi_init_walk_info));
/* Walk entire namespace from the supplied root */
+ /*
+ * TBD: will become ACPI_TYPE_PACKAGE as this type object
+ * is now the only one that supports deferred initialization
+ * (forward references).
+ */
status = acpi_walk_namespace(ACPI_TYPE_ANY, ACPI_ROOT_OBJECT,
ACPI_UINT32_MAX, acpi_ns_init_one_object,
NULL, &info, NULL);
@@ -71,13 +76,8 @@ acpi_status acpi_ns_initialize_objects(void)
}
ACPI_DEBUG_PRINT_RAW((ACPI_DB_INIT,
- " Initialized %u/%u Regions %u/%u Fields %u/%u "
- "Buffers %u/%u Packages (%u nodes)\n",
- info.op_region_init, info.op_region_count,
- info.field_init, info.field_count,
- info.buffer_init, info.buffer_count,
- info.package_init, info.package_count,
- info.object_count));
+ "Namespace contains %u (0x%X) objects\n",
+ info.object_count, info.object_count));
ACPI_DEBUG_PRINT((ACPI_DB_DISPATCH,
"%u Control Methods found\n%u Op Regions found\n",
@@ -382,34 +382,18 @@ acpi_ns_init_one_object(acpi_handle obj_handle,
acpi_ex_enter_interpreter();
/*
- * Each of these types can contain executable AML code within the
- * declaration.
+ * Only initialization of Package objects can be deferred, in order
+ * to support forward references.
*/
switch (type) {
- case ACPI_TYPE_REGION:
-
- info->op_region_init++;
- status = acpi_ds_get_region_arguments(obj_desc);
- break;
-
- case ACPI_TYPE_BUFFER_FIELD:
-
- info->field_init++;
- status = acpi_ds_get_buffer_field_arguments(obj_desc);
- break;
-
case ACPI_TYPE_LOCAL_BANK_FIELD:
+ /* TBD: bank_fields do not require deferred init, remove this code */
+
info->field_init++;
status = acpi_ds_get_bank_field_arguments(obj_desc);
break;
- case ACPI_TYPE_BUFFER:
-
- info->buffer_init++;
- status = acpi_ds_get_buffer_arguments(obj_desc);
- break;
-
case ACPI_TYPE_PACKAGE:
/* Complete the initialization/resolution of the package object */
@@ -421,8 +405,13 @@ acpi_ns_init_one_object(acpi_handle obj_handle,
default:
- /* No other types can get here */
+ /* No other types should get here */
+ status = AE_TYPE;
+ ACPI_EXCEPTION((AE_INFO, status,
+ "Opcode is not deferred [%4.4s] (%s)",
+ acpi_ut_get_node_name(node),
+ acpi_ut_get_type_name(type)));
break;
}
diff --git a/drivers/acpi/acpica/nsload.c b/drivers/acpi/acpica/nsload.c
index 35fff5c75da1..d7c4d6e8e21e 100644
--- a/drivers/acpi/acpica/nsload.c
+++ b/drivers/acpi/acpica/nsload.c
@@ -109,18 +109,6 @@ unlock:
ACPI_DEBUG_PRINT((ACPI_DB_INFO,
"**** Completed Table Object Initialization\n"));
- /*
- * This case handles the legacy option that groups all module-level
- * code blocks together and defers execution until all of the tables
- * are loaded. Execute all of these blocks at this time.
- * Execute any module-level code that was detected during the table
- * load phase.
- *
- * Note: this option is deprecated and will be eliminated in the
- * future. Use of this option can cause problems with AML code that
- * depends upon in-order immediate execution of module-level code.
- */
- acpi_ns_exec_module_code_list();
return_ACPI_STATUS(status);
}
diff --git a/drivers/acpi/acpica/nsrepair2.c b/drivers/acpi/acpica/nsrepair2.c
index 8d776256b213..663d85e0adba 100644
--- a/drivers/acpi/acpica/nsrepair2.c
+++ b/drivers/acpi/acpica/nsrepair2.c
@@ -126,7 +126,7 @@ static const struct acpi_repair_info acpi_ns_repairable_names[] = {
#define ACPI_FDE_FIELD_COUNT 5
#define ACPI_FDE_BYTE_BUFFER_SIZE 5
-#define ACPI_FDE_DWORD_BUFFER_SIZE (ACPI_FDE_FIELD_COUNT * sizeof (u32))
+#define ACPI_FDE_DWORD_BUFFER_SIZE (ACPI_FDE_FIELD_COUNT * (u32) sizeof (u32))
/******************************************************************************
*
diff --git a/drivers/acpi/acpica/nsutils.c b/drivers/acpi/acpica/nsutils.c
index 6bc90d46db5c..b8d007c84d32 100644
--- a/drivers/acpi/acpica/nsutils.c
+++ b/drivers/acpi/acpica/nsutils.c
@@ -560,21 +560,9 @@ struct acpi_namespace_node *acpi_ns_validate_handle(acpi_handle handle)
void acpi_ns_terminate(void)
{
acpi_status status;
- union acpi_operand_object *prev;
- union acpi_operand_object *next;
ACPI_FUNCTION_TRACE(ns_terminate);
- /* Delete any module-level code blocks */
-
- next = acpi_gbl_module_code_list;
- while (next) {
- prev = next;
- next = next->method.mutex;
- prev->method.mutex = NULL; /* Clear the Mutex (cheated) field */
- acpi_ut_remove_reference(prev);
- }
-
/*
* Free the entire namespace -- all nodes and all objects
* attached to the nodes
diff --git a/drivers/acpi/acpica/tbdata.c b/drivers/acpi/acpica/tbdata.c
index 933f81316ad2..309440010ab2 100644
--- a/drivers/acpi/acpica/tbdata.c
+++ b/drivers/acpi/acpica/tbdata.c
@@ -750,6 +750,7 @@ acpi_status acpi_tb_delete_namespace_by_owner(u32 table_index)
if (ACPI_FAILURE(status)) {
return_ACPI_STATUS(status);
}
+
acpi_ns_delete_namespace_by_owner(owner_id);
acpi_ut_release_write_lock(&acpi_gbl_namespace_rw_lock);
return_ACPI_STATUS(status);
@@ -934,19 +935,6 @@ acpi_tb_load_table(u32 table_index, struct acpi_namespace_node *parent_node)
status = acpi_ns_load_table(table_index, parent_node);
/*
- * This case handles the legacy option that groups all module-level
- * code blocks together and defers execution until all of the tables
- * are loaded. Execute all of these blocks at this time.
- * Execute any module-level code that was detected during the table
- * load phase.
- *
- * Note: this option is deprecated and will be eliminated in the
- * future. Use of this option can cause problems with AML code that
- * depends upon in-order immediate execution of module-level code.
- */
- acpi_ns_exec_module_code_list();
-
- /*
* Update GPEs for any new _Lxx/_Exx methods. Ignore errors. The host is
* responsible for discovering any new wake GPEs by running _PRW methods
* that may have been loaded by this table.
diff --git a/drivers/acpi/acpica/tbxfload.c b/drivers/acpi/acpica/tbxfload.c
index 4f30f06a6f78..86f1693f6d29 100644
--- a/drivers/acpi/acpica/tbxfload.c
+++ b/drivers/acpi/acpica/tbxfload.c
@@ -297,6 +297,13 @@ acpi_status acpi_load_table(struct acpi_table_header *table)
status = acpi_tb_install_and_load_table(ACPI_PTR_TO_PHYSADDR(table),
ACPI_TABLE_ORIGIN_EXTERNAL_VIRTUAL,
FALSE, &table_index);
+ if (ACPI_SUCCESS(status)) {
+
+ /* Complete the initialization/resolution of new objects */
+
+ acpi_ns_initialize_objects();
+ }
+
return_ACPI_STATUS(status);
}
diff --git a/drivers/acpi/acpica/utdebug.c b/drivers/acpi/acpica/utdebug.c
index 01b1b36c8a8e..5b169b5f0f1a 100644
--- a/drivers/acpi/acpica/utdebug.c
+++ b/drivers/acpi/acpica/utdebug.c
@@ -158,7 +158,7 @@ acpi_debug_print(u32 requested_debug_level,
* Display the module name, current line number, thread ID (if requested),
* current procedure nesting level, and the current procedure name
*/
- acpi_os_printf("%9s-%04ld ", module_name, line_number);
+ acpi_os_printf("%9s-%04d ", module_name, line_number);
#ifdef ACPI_APPLICATION
/*
@@ -177,7 +177,7 @@ acpi_debug_print(u32 requested_debug_level,
fill_count = 0;
}
- acpi_os_printf("[%02ld] %*s",
+ acpi_os_printf("[%02d] %*s",
acpi_gbl_nesting_level, acpi_gbl_nesting_level + 1, " ");
acpi_os_printf("%s%*s: ",
acpi_ut_trim_function_name(function_name), fill_count,
diff --git a/drivers/acpi/acpica/uterror.c b/drivers/acpi/acpica/uterror.c
index 075457341bad..918aca7c4db4 100644
--- a/drivers/acpi/acpica/uterror.c
+++ b/drivers/acpi/acpica/uterror.c
@@ -39,7 +39,7 @@ void ACPI_INTERNAL_VAR_XFACE
acpi_ut_predefined_warning(const char *module_name,
u32 line_number,
char *pathname,
- u8 node_flags, const char *format, ...)
+ u16 node_flags, const char *format, ...)
{
va_list arg_list;
@@ -81,7 +81,7 @@ acpi_ut_predefined_warning(const char *module_name,
void ACPI_INTERNAL_VAR_XFACE
acpi_ut_predefined_info(const char *module_name,
u32 line_number,
- char *pathname, u8 node_flags, const char *format, ...)
+ char *pathname, u16 node_flags, const char *format, ...)
{
va_list arg_list;
@@ -124,7 +124,7 @@ void ACPI_INTERNAL_VAR_XFACE
acpi_ut_predefined_bios_error(const char *module_name,
u32 line_number,
char *pathname,
- u8 node_flags, const char *format, ...)
+ u16 node_flags, const char *format, ...)
{
va_list arg_list;
diff --git a/drivers/acpi/acpica/utinit.c b/drivers/acpi/acpica/utinit.c
index bc124591320e..6f33e7c72327 100644
--- a/drivers/acpi/acpica/utinit.c
+++ b/drivers/acpi/acpica/utinit.c
@@ -180,7 +180,6 @@ acpi_status acpi_ut_init_globals(void)
/* Namespace */
- acpi_gbl_module_code_list = NULL;
acpi_gbl_root_node = NULL;
acpi_gbl_root_node_struct.name.integer = ACPI_ROOT_NAME;
acpi_gbl_root_node_struct.descriptor_type = ACPI_DESC_TYPE_NAMED;
diff --git a/drivers/acpi/acpica/utosi.c b/drivers/acpi/acpica/utosi.c
index 688c61a90725..ad2b218039d0 100644
--- a/drivers/acpi/acpica/utosi.c
+++ b/drivers/acpi/acpica/utosi.c
@@ -65,13 +65,14 @@ static struct acpi_interface_info acpi_default_supported_interfaces[] = {
{"Windows 2006 SP2", NULL, 0, ACPI_OSI_WIN_VISTA_SP2}, /* Windows Vista SP2 - Added 09/2010 */
{"Windows 2009", NULL, 0, ACPI_OSI_WIN_7}, /* Windows 7 and Server 2008 R2 - Added 09/2009 */
{"Windows 2012", NULL, 0, ACPI_OSI_WIN_8}, /* Windows 8 and Server 2012 - Added 08/2012 */
- {"Windows 2013", NULL, 0, ACPI_OSI_WIN_8}, /* Windows 8.1 and Server 2012 R2 - Added 01/2014 */
+ {"Windows 2013", NULL, 0, ACPI_OSI_WIN_8_1}, /* Windows 8.1 and Server 2012 R2 - Added 01/2014 */
{"Windows 2015", NULL, 0, ACPI_OSI_WIN_10}, /* Windows 10 - Added 03/2015 */
{"Windows 2016", NULL, 0, ACPI_OSI_WIN_10_RS1}, /* Windows 10 version 1607 - Added 12/2017 */
{"Windows 2017", NULL, 0, ACPI_OSI_WIN_10_RS2}, /* Windows 10 version 1703 - Added 12/2017 */
{"Windows 2017.2", NULL, 0, ACPI_OSI_WIN_10_RS3}, /* Windows 10 version 1709 - Added 02/2018 */
{"Windows 2018", NULL, 0, ACPI_OSI_WIN_10_RS4}, /* Windows 10 version 1803 - Added 11/2018 */
{"Windows 2018.2", NULL, 0, ACPI_OSI_WIN_10_RS5}, /* Windows 10 version 1809 - Added 11/2018 */
+ {"Windows 2019", NULL, 0, ACPI_OSI_WIN_10_19H1}, /* Windows 10 version 1903 - Added 08/2019 */
/* Feature Group Strings */
diff --git a/drivers/acpi/acpica/utownerid.c b/drivers/acpi/acpica/utownerid.c
index 5eb8b76ce9d8..d3525ef8ed28 100644
--- a/drivers/acpi/acpica/utownerid.c
+++ b/drivers/acpi/acpica/utownerid.c
@@ -38,7 +38,7 @@ acpi_status acpi_ut_allocate_owner_id(acpi_owner_id *owner_id)
if (*owner_id) {
ACPI_ERROR((AE_INFO,
- "Owner ID [0x%2.2X] already exists", *owner_id));
+ "Owner ID [0x%3.3X] already exists", *owner_id));
return_ACPI_STATUS(AE_ALREADY_EXISTS);
}
@@ -88,14 +88,14 @@ acpi_status acpi_ut_allocate_owner_id(acpi_owner_id *owner_id)
/*
* Construct encoded ID from the index and bit position
*
- * Note: Last [j].k (bit 255) is never used and is marked
+ * Note: Last [j].k (bit 4095) is never used and is marked
* permanently allocated (prevents +1 overflow)
*/
*owner_id =
(acpi_owner_id)((k + 1) + ACPI_MUL_32(j));
ACPI_DEBUG_PRINT((ACPI_DB_VALUES,
- "Allocated OwnerId: %2.2X\n",
+ "Allocated OwnerId: 0x%3.3X\n",
(unsigned int)*owner_id));
goto exit;
}
@@ -116,7 +116,7 @@ acpi_status acpi_ut_allocate_owner_id(acpi_owner_id *owner_id)
*/
status = AE_OWNER_ID_LIMIT;
ACPI_ERROR((AE_INFO,
- "Could not allocate new OwnerId (255 max), AE_OWNER_ID_LIMIT"));
+ "Could not allocate new OwnerId (4095 max), AE_OWNER_ID_LIMIT"));
exit:
(void)acpi_ut_release_mutex(ACPI_MTX_CACHES);
@@ -153,7 +153,7 @@ void acpi_ut_release_owner_id(acpi_owner_id *owner_id_ptr)
/* Zero is not a valid owner_ID */
if (owner_id == 0) {
- ACPI_ERROR((AE_INFO, "Invalid OwnerId: 0x%2.2X", owner_id));
+ ACPI_ERROR((AE_INFO, "Invalid OwnerId: 0x%3.3X", owner_id));
return_VOID;
}
@@ -179,7 +179,7 @@ void acpi_ut_release_owner_id(acpi_owner_id *owner_id_ptr)
acpi_gbl_owner_id_mask[index] ^= bit;
} else {
ACPI_ERROR((AE_INFO,
- "Release of non-allocated OwnerId: 0x%2.2X",
+ "Attempted release of non-allocated OwnerId: 0x%3.3X",
owner_id + 1));
}
diff --git a/drivers/acpi/acpica/utxfinit.c b/drivers/acpi/acpica/utxfinit.c
index 9f3b1e3a09de..cf769e94fe0f 100644
--- a/drivers/acpi/acpica/utxfinit.c
+++ b/drivers/acpi/acpica/utxfinit.c
@@ -211,24 +211,17 @@ acpi_status ACPI_INIT_FUNCTION acpi_initialize_objects(u32 flags)
ACPI_FUNCTION_TRACE(acpi_initialize_objects);
+#ifdef ACPI_OBSOLETE_BEHAVIOR
/*
- * This case handles the legacy option that groups all module-level
- * code blocks together and defers execution until all of the tables
- * are loaded. Execute all of these blocks at this time.
- * Execute any module-level code that was detected during the table
- * load phase.
- *
- * Note: this option is deprecated and will be eliminated in the
- * future. Use of this option can cause problems with AML code that
- * depends upon in-order immediate execution of module-level code.
+ * 05/2019: Removed, initialization now happens at both object
+ * creation and table load time
*/
- acpi_ns_exec_module_code_list();
/*
* Initialize the objects that remain uninitialized. This
* runs the executable AML that may be part of the
- * declaration of these objects:
- * operation_regions, buffer_fields, Buffers, and Packages.
+ * declaration of these objects: operation_regions, buffer_fields,
+ * bank_fields, Buffers, and Packages.
*/
if (!(flags & ACPI_NO_OBJECT_INIT)) {
status = acpi_ns_initialize_objects();
@@ -236,6 +229,7 @@ acpi_status ACPI_INIT_FUNCTION acpi_initialize_objects(u32 flags)
return_ACPI_STATUS(status);
}
}
+#endif
/*
* Initialize all device/region objects in the namespace. This runs
diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c
index 993940d582f5..777f6f7122b4 100644
--- a/drivers/acpi/apei/ghes.c
+++ b/drivers/acpi/apei/ghes.c
@@ -153,6 +153,7 @@ static void ghes_unmap(void __iomem *vaddr, enum fixed_addresses fixmap_idx)
int ghes_estatus_pool_init(int num_ghes)
{
unsigned long addr, len;
+ int rc;
ghes_estatus_pool = gen_pool_create(GHES_ESTATUS_POOL_MIN_ALLOC_ORDER, -1);
if (!ghes_estatus_pool)
@@ -164,7 +165,7 @@ int ghes_estatus_pool_init(int num_ghes)
ghes_estatus_pool_size_request = PAGE_ALIGN(len);
addr = (unsigned long)vmalloc(PAGE_ALIGN(len));
if (!addr)
- return -ENOMEM;
+ goto err_pool_alloc;
/*
* New allocation must be visible in all pgd before it can be found by
@@ -172,7 +173,19 @@ int ghes_estatus_pool_init(int num_ghes)
*/
vmalloc_sync_all();
- return gen_pool_add(ghes_estatus_pool, addr, PAGE_ALIGN(len), -1);
+ rc = gen_pool_add(ghes_estatus_pool, addr, PAGE_ALIGN(len), -1);
+ if (rc)
+ goto err_pool_add;
+
+ return 0;
+
+err_pool_add:
+ vfree((void *)addr);
+
+err_pool_alloc:
+ gen_pool_destroy(ghes_estatus_pool);
+
+ return -ENOMEM;
}
static int map_gen_v2(struct ghes *ghes)
@@ -345,7 +358,7 @@ static int __ghes_peek_estatus(struct ghes *ghes,
return -ENOENT;
}
- return __ghes_check_estatus(ghes, estatus);
+ return 0;
}
static int __ghes_read_estatus(struct acpi_hest_generic_status *estatus,
@@ -483,7 +496,7 @@ static void ghes_do_proc(struct ghes *ghes,
int sev, sec_sev;
struct acpi_hest_generic_data *gdata;
guid_t *sec_type;
- guid_t *fru_id = &NULL_UUID_LE;
+ const guid_t *fru_id = &guid_null;
char *fru_text = "";
sev = ghes_severity(estatus->error_severity);
diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c
index d4551e33fa71..5a7551d060f2 100644
--- a/drivers/acpi/arm64/iort.c
+++ b/drivers/acpi/arm64/iort.c
@@ -611,8 +611,8 @@ static int iort_dev_find_its_id(struct device *dev, u32 req_id,
/* Move to ITS specific data */
its = (struct acpi_iort_its_group *)node->node_data;
- if (idx > its->its_count) {
- dev_err(dev, "requested ITS ID index [%d] is greater than available [%d]\n",
+ if (idx >= its->its_count) {
+ dev_err(dev, "requested ITS ID index [%d] overruns ITS entries [%d]\n",
idx, its->its_count);
return -ENXIO;
}
@@ -1256,12 +1256,12 @@ static int __init arm_smmu_v3_set_proximity(struct device *dev,
smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
if (smmu->flags & ACPI_IORT_SMMU_V3_PXM_VALID) {
- int node = acpi_map_pxm_to_node(smmu->pxm);
+ int dev_node = acpi_map_pxm_to_node(smmu->pxm);
- if (node != NUMA_NO_NODE && !node_online(node))
+ if (dev_node != NUMA_NO_NODE && !node_online(dev_node))
return -EINVAL;
- set_dev_node(dev, node);
+ set_dev_node(dev, dev_node);
pr_info("SMMU-v3[%llx] Mapped to Proximity domain %d\n",
smmu->base_address,
smmu->pxm);
diff --git a/drivers/acpi/blacklist.c b/drivers/acpi/blacklist.c
index ad2c565f5cbe..a86a770c9b79 100644
--- a/drivers/acpi/blacklist.c
+++ b/drivers/acpi/blacklist.c
@@ -17,7 +17,9 @@
#include "internal.h"
+#ifdef CONFIG_DMI
static const struct dmi_system_id acpi_rev_dmi_table[] __initconst;
+#endif
/*
* POLICY: If *anything* doesn't work, put it on the blacklist.
@@ -61,7 +63,9 @@ int __init acpi_blacklisted(void)
}
(void)early_acpi_osi_init();
+#ifdef CONFIG_DMI
dmi_check_system(acpi_rev_dmi_table);
+#endif
return blacklisted;
}
diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c
index 15f103d7532b..3b2525908dd8 100644
--- a/drivers/acpi/cppc_acpi.c
+++ b/drivers/acpi/cppc_acpi.c
@@ -365,8 +365,10 @@ static int acpi_get_psd(struct cpc_desc *cpc_ptr, acpi_handle handle)
union acpi_object *psd = NULL;
struct acpi_psd_package *pdomain;
- status = acpi_evaluate_object_typed(handle, "_PSD", NULL, &buffer,
- ACPI_TYPE_PACKAGE);
+ status = acpi_evaluate_object_typed(handle, "_PSD", NULL,
+ &buffer, ACPI_TYPE_PACKAGE);
+ if (status == AE_NOT_FOUND) /* _PSD is optional */
+ return 0;
if (ACPI_FAILURE(status))
return -ENODEV;
diff --git a/drivers/acpi/custom_method.c b/drivers/acpi/custom_method.c
index b2ef4c2ec955..fd66a736621c 100644
--- a/drivers/acpi/custom_method.c
+++ b/drivers/acpi/custom_method.c
@@ -49,8 +49,10 @@ static ssize_t cm_write(struct file *file, const char __user * user_buf,
if ((*ppos > max_size) ||
(*ppos + count > max_size) ||
(*ppos + count < count) ||
- (count > uncopied_bytes))
+ (count > uncopied_bytes)) {
+ kfree(buf);
return -EINVAL;
+ }
if (copy_from_user(buf + (*ppos), user_buf, count)) {
kfree(buf);
@@ -70,6 +72,7 @@ static ssize_t cm_write(struct file *file, const char __user * user_buf,
add_taint(TAINT_OVERRIDDEN_ACPI_TABLE, LOCKDEP_NOW_UNRELIABLE);
}
+ kfree(buf);
return count;
}
diff --git a/drivers/acpi/device_pm.c b/drivers/acpi/device_pm.c
index e54956ae93d3..08bb9f2f2d23 100644
--- a/drivers/acpi/device_pm.c
+++ b/drivers/acpi/device_pm.c
@@ -45,6 +45,19 @@ const char *acpi_power_state_string(int state)
}
}
+static int acpi_dev_pm_explicit_get(struct acpi_device *device, int *state)
+{
+ unsigned long long psc;
+ acpi_status status;
+
+ status = acpi_evaluate_integer(device->handle, "_PSC", NULL, &psc);
+ if (ACPI_FAILURE(status))
+ return -ENODEV;
+
+ *state = psc;
+ return 0;
+}
+
/**
* acpi_device_get_power - Get power state of an ACPI device.
* @device: Device to get the power state of.
@@ -53,10 +66,16 @@ const char *acpi_power_state_string(int state)
* This function does not update the device's power.state field, but it may
* update its parent's power.state field (when the parent's power state is
* unknown and the device's power state turns out to be D0).
+ *
+ * Also, it does not update power resource reference counters to ensure that
+ * the power state returned by it will be persistent and it may return a power
+ * state shallower than previously set by acpi_device_set_power() for @device
+ * (if that power state depends on any power resources).
*/
int acpi_device_get_power(struct acpi_device *device, int *state)
{
int result = ACPI_STATE_UNKNOWN;
+ int error;
if (!device || !state)
return -EINVAL;
@@ -73,18 +92,16 @@ int acpi_device_get_power(struct acpi_device *device, int *state)
* if available.
*/
if (device->power.flags.power_resources) {
- int error = acpi_power_get_inferred_state(device, &result);
+ error = acpi_power_get_inferred_state(device, &result);
if (error)
return error;
}
if (device->power.flags.explicit_get) {
- acpi_handle handle = device->handle;
- unsigned long long psc;
- acpi_status status;
+ int psc;
- status = acpi_evaluate_integer(handle, "_PSC", NULL, &psc);
- if (ACPI_FAILURE(status))
- return -ENODEV;
+ error = acpi_dev_pm_explicit_get(device, &psc);
+ if (error)
+ return error;
/*
* The power resources settings may indicate a power state
@@ -118,7 +135,6 @@ int acpi_device_get_power(struct acpi_device *device, int *state)
return 0;
}
-EXPORT_SYMBOL(acpi_device_get_power);
static int acpi_dev_pm_explicit_set(struct acpi_device *adev, int state)
{
@@ -150,9 +166,14 @@ int acpi_device_set_power(struct acpi_device *device, int state)
|| (state < ACPI_STATE_D0) || (state > ACPI_STATE_D3_COLD))
return -EINVAL;
+ acpi_handle_debug(device->handle, "Power state change: %s -> %s\n",
+ acpi_power_state_string(device->power.state),
+ acpi_power_state_string(state));
+
/* Make sure this is a valid target state */
- if (state == device->power.state) {
+ /* There is a special case for D0 addressed below. */
+ if (state > ACPI_STATE_D0 && state == device->power.state) {
ACPI_DEBUG_PRINT((ACPI_DB_INFO, "Device [%s] already in %s\n",
device->pnp.bus_id,
acpi_power_state_string(state)));
@@ -202,18 +223,50 @@ int acpi_device_set_power(struct acpi_device *device, int state)
return -ENODEV;
}
- result = acpi_dev_pm_explicit_set(device, state);
- if (result)
- goto end;
+ /*
+ * If the device goes from D3hot to D3cold, _PS3 has been
+ * evaluated for it already, so skip it in that case.
+ */
+ if (device->power.state < ACPI_STATE_D3_HOT) {
+ result = acpi_dev_pm_explicit_set(device, state);
+ if (result)
+ goto end;
+ }
if (device->power.flags.power_resources)
result = acpi_power_transition(device, target_state);
} else {
+ int cur_state = device->power.state;
+
if (device->power.flags.power_resources) {
result = acpi_power_transition(device, ACPI_STATE_D0);
if (result)
goto end;
}
+
+ if (cur_state == ACPI_STATE_D0) {
+ int psc;
+
+ /* Nothing to do here if _PSC is not present. */
+ if (!device->power.flags.explicit_get)
+ return 0;
+
+ /*
+ * The power state of the device was set to D0 last
+ * time, but that might have happened before a
+ * system-wide transition involving the platform
+ * firmware, so it may be necessary to evaluate _PS0
+ * for the device here. However, use extra care here
+ * and evaluate _PSC to check the device's current power
+ * state, and only invoke _PS0 if the evaluation of _PSC
+ * is successful and it returns a power state different
+ * from D0.
+ */
+ result = acpi_dev_pm_explicit_get(device, &psc);
+ if (result || psc == ACPI_STATE_D0)
+ return 0;
+ }
+
result = acpi_dev_pm_explicit_set(device, ACPI_STATE_D0);
}
@@ -448,7 +501,8 @@ acpi_status acpi_add_pm_notifier(struct acpi_device *adev, struct device *dev,
goto out;
mutex_lock(&acpi_pm_notifier_lock);
- adev->wakeup.ws = wakeup_source_register(dev_name(&adev->dev));
+ adev->wakeup.ws = wakeup_source_register(&adev->dev,
+ dev_name(&adev->dev));
adev->wakeup.context.dev = dev;
adev->wakeup.context.func = func;
adev->wakeup.flags.notifier_present = true;
@@ -1073,7 +1127,7 @@ EXPORT_SYMBOL_GPL(acpi_subsys_suspend_noirq);
* acpi_subsys_resume_noirq - Run the device driver's "noirq" resume callback.
* @dev: Device to handle.
*/
-int acpi_subsys_resume_noirq(struct device *dev)
+static int acpi_subsys_resume_noirq(struct device *dev)
{
if (dev_pm_may_skip_resume(dev))
return 0;
@@ -1088,7 +1142,6 @@ int acpi_subsys_resume_noirq(struct device *dev)
return pm_generic_resume_noirq(dev);
}
-EXPORT_SYMBOL_GPL(acpi_subsys_resume_noirq);
/**
* acpi_subsys_resume_early - Resume device using ACPI.
@@ -1098,12 +1151,11 @@ EXPORT_SYMBOL_GPL(acpi_subsys_resume_noirq);
* generic early resume procedure for it during system transition into the
* working state.
*/
-int acpi_subsys_resume_early(struct device *dev)
+static int acpi_subsys_resume_early(struct device *dev)
{
int ret = acpi_dev_resume(dev);
return ret ? ret : pm_generic_resume_early(dev);
}
-EXPORT_SYMBOL_GPL(acpi_subsys_resume_early);
/**
* acpi_subsys_freeze - Run the device driver's freeze callback.
@@ -1112,65 +1164,81 @@ EXPORT_SYMBOL_GPL(acpi_subsys_resume_early);
int acpi_subsys_freeze(struct device *dev)
{
/*
- * This used to be done in acpi_subsys_prepare() for all devices and
- * some drivers may depend on it, so do it here. Ideally, however,
- * runtime-suspended devices should not be touched during freeze/thaw
- * transitions.
+ * Resume all runtime-suspended devices before creating a snapshot
+ * image of system memory, because the restore kernel generally cannot
+ * be expected to always handle them consistently and they need to be
+ * put into the runtime-active metastate during system resume anyway,
+ * so it is better to ensure that the state saved in the image will be
+ * always consistent with that.
*/
- if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND))
- pm_runtime_resume(dev);
+ pm_runtime_resume(dev);
return pm_generic_freeze(dev);
}
EXPORT_SYMBOL_GPL(acpi_subsys_freeze);
/**
- * acpi_subsys_freeze_late - Run the device driver's "late" freeze callback.
- * @dev: Device to handle.
+ * acpi_subsys_restore_early - Restore device using ACPI.
+ * @dev: Device to restore.
*/
-int acpi_subsys_freeze_late(struct device *dev)
+int acpi_subsys_restore_early(struct device *dev)
{
+ int ret = acpi_dev_resume(dev);
+ return ret ? ret : pm_generic_restore_early(dev);
+}
+EXPORT_SYMBOL_GPL(acpi_subsys_restore_early);
- if (dev_pm_smart_suspend_and_suspended(dev))
- return 0;
+/**
+ * acpi_subsys_poweroff - Run the device driver's poweroff callback.
+ * @dev: Device to handle.
+ *
+ * Follow PCI and resume devices from runtime suspend before running their
+ * system poweroff callbacks, unless the driver can cope with runtime-suspended
+ * devices during system suspend and there are no ACPI-specific reasons for
+ * resuming them.
+ */
+int acpi_subsys_poweroff(struct device *dev)
+{
+ if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) ||
+ acpi_dev_needs_resume(dev, ACPI_COMPANION(dev)))
+ pm_runtime_resume(dev);
- return pm_generic_freeze_late(dev);
+ return pm_generic_poweroff(dev);
}
-EXPORT_SYMBOL_GPL(acpi_subsys_freeze_late);
+EXPORT_SYMBOL_GPL(acpi_subsys_poweroff);
/**
- * acpi_subsys_freeze_noirq - Run the device driver's "noirq" freeze callback.
+ * acpi_subsys_poweroff_late - Run the device driver's poweroff callback.
* @dev: Device to handle.
+ *
+ * Carry out the generic late poweroff procedure for @dev and use ACPI to put
+ * it into a low-power state during system transition into a sleep state.
*/
-int acpi_subsys_freeze_noirq(struct device *dev)
+static int acpi_subsys_poweroff_late(struct device *dev)
{
+ int ret;
if (dev_pm_smart_suspend_and_suspended(dev))
return 0;
- return pm_generic_freeze_noirq(dev);
+ ret = pm_generic_poweroff_late(dev);
+ if (ret)
+ return ret;
+
+ return acpi_dev_suspend(dev, device_may_wakeup(dev));
}
-EXPORT_SYMBOL_GPL(acpi_subsys_freeze_noirq);
/**
- * acpi_subsys_thaw_noirq - Run the device driver's "noirq" thaw callback.
- * @dev: Device to handle.
+ * acpi_subsys_poweroff_noirq - Run the driver's "noirq" poweroff callback.
+ * @dev: Device to suspend.
*/
-int acpi_subsys_thaw_noirq(struct device *dev)
+static int acpi_subsys_poweroff_noirq(struct device *dev)
{
- /*
- * If the device is in runtime suspend, the "thaw" code may not work
- * correctly with it, so skip the driver callback and make the PM core
- * skip all of the subsequent "thaw" callbacks for the device.
- */
- if (dev_pm_smart_suspend_and_suspended(dev)) {
- dev_pm_skip_next_resume_phases(dev);
+ if (dev_pm_smart_suspend_and_suspended(dev))
return 0;
- }
- return pm_generic_thaw_noirq(dev);
+ return pm_generic_poweroff_noirq(dev);
}
-EXPORT_SYMBOL_GPL(acpi_subsys_thaw_noirq);
#endif /* CONFIG_PM_SLEEP */
static struct dev_pm_domain acpi_general_pm_domain = {
@@ -1186,14 +1254,10 @@ static struct dev_pm_domain acpi_general_pm_domain = {
.resume_noirq = acpi_subsys_resume_noirq,
.resume_early = acpi_subsys_resume_early,
.freeze = acpi_subsys_freeze,
- .freeze_late = acpi_subsys_freeze_late,
- .freeze_noirq = acpi_subsys_freeze_noirq,
- .thaw_noirq = acpi_subsys_thaw_noirq,
- .poweroff = acpi_subsys_suspend,
- .poweroff_late = acpi_subsys_suspend_late,
- .poweroff_noirq = acpi_subsys_suspend_noirq,
- .restore_noirq = acpi_subsys_resume_noirq,
- .restore_early = acpi_subsys_resume_early,
+ .poweroff = acpi_subsys_poweroff,
+ .poweroff_late = acpi_subsys_poweroff_late,
+ .poweroff_noirq = acpi_subsys_poweroff_noirq,
+ .restore_early = acpi_subsys_restore_early,
#endif
},
};
diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c
index c33756ed3304..da1e5c5ce150 100644
--- a/drivers/acpi/ec.c
+++ b/drivers/acpi/ec.c
@@ -25,6 +25,7 @@
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
+#include <linux/suspend.h>
#include <linux/acpi.h>
#include <linux/dmi.h>
#include <asm/io.h>
@@ -1048,24 +1049,6 @@ void acpi_ec_unblock_transactions(void)
acpi_ec_start(first_ec, true);
}
-void acpi_ec_mark_gpe_for_wake(void)
-{
- if (first_ec && !ec_no_wakeup)
- acpi_mark_gpe_for_wake(NULL, first_ec->gpe);
-}
-
-void acpi_ec_set_gpe_wake_mask(u8 action)
-{
- if (first_ec && !ec_no_wakeup)
- acpi_set_gpe_wake_mask(NULL, first_ec->gpe, action);
-}
-
-void acpi_ec_dispatch_gpe(void)
-{
- if (first_ec)
- acpi_dispatch_gpe(NULL, first_ec->gpe);
-}
-
/* --------------------------------------------------------------------------
Event Management
-------------------------------------------------------------------------- */
@@ -1931,7 +1914,7 @@ static int acpi_ec_suspend(struct device *dev)
struct acpi_ec *ec =
acpi_driver_data(to_acpi_device(dev));
- if (acpi_sleep_no_ec_events() && ec_freeze_events)
+ if (!pm_suspend_no_platform() && ec_freeze_events)
acpi_ec_disable_event(ec);
return 0;
}
@@ -1948,8 +1931,7 @@ static int acpi_ec_suspend_noirq(struct device *dev)
ec->reference_count >= 1)
acpi_set_gpe(NULL, ec->gpe, ACPI_GPE_DISABLE);
- if (acpi_sleep_no_ec_events())
- acpi_ec_enter_noirq(ec);
+ acpi_ec_enter_noirq(ec);
return 0;
}
@@ -1958,8 +1940,7 @@ static int acpi_ec_resume_noirq(struct device *dev)
{
struct acpi_ec *ec = acpi_driver_data(to_acpi_device(dev));
- if (acpi_sleep_no_ec_events())
- acpi_ec_leave_noirq(ec);
+ acpi_ec_leave_noirq(ec);
if (ec_no_wakeup && test_bit(EC_FLAGS_STARTED, &ec->flags) &&
ec->reference_count >= 1)
@@ -1976,7 +1957,35 @@ static int acpi_ec_resume(struct device *dev)
acpi_ec_enable_event(ec);
return 0;
}
-#endif
+
+void acpi_ec_mark_gpe_for_wake(void)
+{
+ if (first_ec && !ec_no_wakeup)
+ acpi_mark_gpe_for_wake(NULL, first_ec->gpe);
+}
+EXPORT_SYMBOL_GPL(acpi_ec_mark_gpe_for_wake);
+
+void acpi_ec_set_gpe_wake_mask(u8 action)
+{
+ if (pm_suspend_no_platform() && first_ec && !ec_no_wakeup)
+ acpi_set_gpe_wake_mask(NULL, first_ec->gpe, action);
+}
+
+bool acpi_ec_dispatch_gpe(void)
+{
+ u32 ret;
+
+ if (!first_ec)
+ return false;
+
+ ret = acpi_dispatch_gpe(NULL, first_ec->gpe);
+ if (ret == ACPI_INTERRUPT_HANDLED) {
+ pm_pr_dbg("EC GPE dispatched\n");
+ return true;
+ }
+ return false;
+}
+#endif /* CONFIG_PM_SLEEP */
static const struct dev_pm_ops acpi_ec_pm = {
SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(acpi_ec_suspend_noirq, acpi_ec_resume_noirq)
diff --git a/drivers/acpi/hmat/hmat.c b/drivers/acpi/hmat/hmat.c
index 96b7d39a97c6..8f9a28a870b0 100644
--- a/drivers/acpi/hmat/hmat.c
+++ b/drivers/acpi/hmat/hmat.c
@@ -14,14 +14,18 @@
#include <linux/init.h>
#include <linux/list.h>
#include <linux/list_sort.h>
+#include <linux/memory.h>
+#include <linux/mutex.h>
#include <linux/node.h>
#include <linux/sysfs.h>
-static __initdata u8 hmat_revision;
+static u8 hmat_revision;
-static __initdata LIST_HEAD(targets);
-static __initdata LIST_HEAD(initiators);
-static __initdata LIST_HEAD(localities);
+static LIST_HEAD(targets);
+static LIST_HEAD(initiators);
+static LIST_HEAD(localities);
+
+static DEFINE_MUTEX(target_lock);
/*
* The defined enum order is used to prioritize attributes to break ties when
@@ -36,11 +40,19 @@ enum locality_types {
static struct memory_locality *localities_types[4];
+struct target_cache {
+ struct list_head node;
+ struct node_cache_attrs cache_attrs;
+};
+
struct memory_target {
struct list_head node;
unsigned int memory_pxm;
unsigned int processor_pxm;
struct node_hmem_attrs hmem_attrs;
+ struct list_head caches;
+ struct node_cache_attrs cache_attrs;
+ bool registered;
};
struct memory_initiator {
@@ -53,7 +65,7 @@ struct memory_locality {
struct acpi_hmat_locality *hmat_loc;
};
-static __init struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm)
+static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm)
{
struct memory_initiator *initiator;
@@ -63,7 +75,7 @@ static __init struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm)
return NULL;
}
-static __init struct memory_target *find_mem_target(unsigned int mem_pxm)
+static struct memory_target *find_mem_target(unsigned int mem_pxm)
{
struct memory_target *target;
@@ -96,9 +108,6 @@ static __init void alloc_memory_target(unsigned int mem_pxm)
{
struct memory_target *target;
- if (pxm_to_node(mem_pxm) == NUMA_NO_NODE)
- return;
-
target = find_mem_target(mem_pxm);
if (target)
return;
@@ -110,6 +119,7 @@ static __init void alloc_memory_target(unsigned int mem_pxm)
target->memory_pxm = mem_pxm;
target->processor_pxm = PXM_INVAL;
list_add_tail(&target->node, &targets);
+ INIT_LIST_HEAD(&target->caches);
}
static __init const char *hmat_data_type(u8 type)
@@ -148,7 +158,7 @@ static __init const char *hmat_data_type_suffix(u8 type)
}
}
-static __init u32 hmat_normalize(u16 entry, u64 base, u8 type)
+static u32 hmat_normalize(u16 entry, u64 base, u8 type)
{
u32 value;
@@ -183,7 +193,7 @@ static __init u32 hmat_normalize(u16 entry, u64 base, u8 type)
return value;
}
-static __init void hmat_update_target_access(struct memory_target *target,
+static void hmat_update_target_access(struct memory_target *target,
u8 type, u32 value)
{
switch (type) {
@@ -314,7 +324,8 @@ static __init int hmat_parse_cache(union acpi_subtable_headers *header,
const unsigned long end)
{
struct acpi_hmat_cache *cache = (void *)header;
- struct node_cache_attrs cache_attrs;
+ struct memory_target *target;
+ struct target_cache *tcache;
u32 attrs;
if (cache->header.length < sizeof(*cache)) {
@@ -328,37 +339,47 @@ static __init int hmat_parse_cache(union acpi_subtable_headers *header,
cache->memory_PD, cache->cache_size, attrs,
cache->number_of_SMBIOShandles);
- cache_attrs.size = cache->cache_size;
- cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4;
- cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16;
+ target = find_mem_target(cache->memory_PD);
+ if (!target)
+ return 0;
+
+ tcache = kzalloc(sizeof(*tcache), GFP_KERNEL);
+ if (!tcache) {
+ pr_notice_once("Failed to allocate HMAT cache info\n");
+ return 0;
+ }
+
+ tcache->cache_attrs.size = cache->cache_size;
+ tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4;
+ tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16;
switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) {
case ACPI_HMAT_CA_DIRECT_MAPPED:
- cache_attrs.indexing = NODE_CACHE_DIRECT_MAP;
+ tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP;
break;
case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING:
- cache_attrs.indexing = NODE_CACHE_INDEXED;
+ tcache->cache_attrs.indexing = NODE_CACHE_INDEXED;
break;
case ACPI_HMAT_CA_NONE:
default:
- cache_attrs.indexing = NODE_CACHE_OTHER;
+ tcache->cache_attrs.indexing = NODE_CACHE_OTHER;
break;
}
switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) {
case ACPI_HMAT_CP_WB:
- cache_attrs.write_policy = NODE_CACHE_WRITE_BACK;
+ tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK;
break;
case ACPI_HMAT_CP_WT:
- cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH;
+ tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH;
break;
case ACPI_HMAT_CP_NONE:
default:
- cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER;
+ tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER;
break;
}
+ list_add_tail(&tcache->node, &target->caches);
- node_add_cache(pxm_to_node(cache->memory_PD), &cache_attrs);
return 0;
}
@@ -435,7 +456,7 @@ static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header,
return 0;
}
-static __init u32 hmat_initiator_perf(struct memory_target *target,
+static u32 hmat_initiator_perf(struct memory_target *target,
struct memory_initiator *initiator,
struct acpi_hmat_locality *hmat_loc)
{
@@ -473,7 +494,7 @@ static __init u32 hmat_initiator_perf(struct memory_target *target,
hmat_loc->data_type);
}
-static __init bool hmat_update_best(u8 type, u32 value, u32 *best)
+static bool hmat_update_best(u8 type, u32 value, u32 *best)
{
bool updated = false;
@@ -517,7 +538,7 @@ static int initiator_cmp(void *priv, struct list_head *a, struct list_head *b)
return ia->processor_pxm - ib->processor_pxm;
}
-static __init void hmat_register_target_initiators(struct memory_target *target)
+static void hmat_register_target_initiators(struct memory_target *target)
{
static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
struct memory_initiator *initiator;
@@ -577,29 +598,89 @@ static __init void hmat_register_target_initiators(struct memory_target *target)
}
}
-static __init void hmat_register_target_perf(struct memory_target *target)
+static void hmat_register_target_cache(struct memory_target *target)
+{
+ unsigned mem_nid = pxm_to_node(target->memory_pxm);
+ struct target_cache *tcache;
+
+ list_for_each_entry(tcache, &target->caches, node)
+ node_add_cache(mem_nid, &tcache->cache_attrs);
+}
+
+static void hmat_register_target_perf(struct memory_target *target)
{
unsigned mem_nid = pxm_to_node(target->memory_pxm);
node_set_perf_attrs(mem_nid, &target->hmem_attrs, 0);
}
-static __init void hmat_register_targets(void)
+static void hmat_register_target(struct memory_target *target)
{
- struct memory_target *target;
+ int nid = pxm_to_node(target->memory_pxm);
- list_for_each_entry(target, &targets, node) {
+ /*
+ * Skip offline nodes. This can happen when memory
+ * marked EFI_MEMORY_SP, "specific purpose", is applied
+ * to all the memory in a promixity domain leading to
+ * the node being marked offline / unplugged, or if
+ * memory-only "hotplug" node is offline.
+ */
+ if (nid == NUMA_NO_NODE || !node_online(nid))
+ return;
+
+ mutex_lock(&target_lock);
+ if (!target->registered) {
hmat_register_target_initiators(target);
+ hmat_register_target_cache(target);
hmat_register_target_perf(target);
+ target->registered = true;
}
+ mutex_unlock(&target_lock);
+}
+
+static void hmat_register_targets(void)
+{
+ struct memory_target *target;
+
+ list_for_each_entry(target, &targets, node)
+ hmat_register_target(target);
+}
+
+static int hmat_callback(struct notifier_block *self,
+ unsigned long action, void *arg)
+{
+ struct memory_target *target;
+ struct memory_notify *mnb = arg;
+ int pxm, nid = mnb->status_change_nid;
+
+ if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+ return NOTIFY_OK;
+
+ pxm = node_to_pxm(nid);
+ target = find_mem_target(pxm);
+ if (!target)
+ return NOTIFY_OK;
+
+ hmat_register_target(target);
+ return NOTIFY_OK;
}
+static struct notifier_block hmat_callback_nb = {
+ .notifier_call = hmat_callback,
+ .priority = 2,
+};
+
static __init void hmat_free_structures(void)
{
struct memory_target *target, *tnext;
struct memory_locality *loc, *lnext;
struct memory_initiator *initiator, *inext;
+ struct target_cache *tcache, *cnext;
list_for_each_entry_safe(target, tnext, &targets, node) {
+ list_for_each_entry_safe(tcache, cnext, &target->caches, node) {
+ list_del(&tcache->node);
+ kfree(tcache);
+ }
list_del(&target->node);
kfree(target);
}
@@ -658,6 +739,10 @@ static __init int hmat_init(void)
}
}
hmat_register_targets();
+
+ /* Keep the table and structures if the notifier may use them */
+ if (!register_hotmemory_notifier(&hmat_callback_nb))
+ return 0;
out_put:
hmat_free_structures();
acpi_put_table(tbl);
diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h
index f6157d4d637a..afe6636f9ad3 100644
--- a/drivers/acpi/internal.h
+++ b/drivers/acpi/internal.h
@@ -139,8 +139,15 @@ int acpi_power_get_inferred_state(struct acpi_device *device, int *state);
int acpi_power_on_resources(struct acpi_device *device, int state);
int acpi_power_transition(struct acpi_device *device, int state);
+/* --------------------------------------------------------------------------
+ Device Power Management
+ -------------------------------------------------------------------------- */
+int acpi_device_get_power(struct acpi_device *device, int *state);
int acpi_wakeup_device_init(void);
+/* --------------------------------------------------------------------------
+ Processor
+ -------------------------------------------------------------------------- */
#ifdef CONFIG_ARCH_MIGHT_HAVE_ACPI_PDC
void acpi_early_processor_set_pdc(void);
#else
@@ -187,9 +194,6 @@ void acpi_ec_ecdt_probe(void);
void acpi_ec_dsdt_probe(void);
void acpi_ec_block_transactions(void);
void acpi_ec_unblock_transactions(void);
-void acpi_ec_mark_gpe_for_wake(void);
-void acpi_ec_set_gpe_wake_mask(u8 action);
-void acpi_ec_dispatch_gpe(void);
int acpi_ec_add_query_handler(struct acpi_ec *ec, u8 query_bit,
acpi_handle handle, acpi_ec_query_func func,
void *data);
@@ -197,6 +201,7 @@ void acpi_ec_remove_query_handler(struct acpi_ec *ec, u8 query_bit);
#ifdef CONFIG_PM_SLEEP
void acpi_ec_flush_work(void);
+bool acpi_ec_dispatch_gpe(void);
#endif
@@ -205,11 +210,9 @@ void acpi_ec_flush_work(void);
-------------------------------------------------------------------------- */
#ifdef CONFIG_ACPI_SYSTEM_POWER_STATES_SUPPORT
extern bool acpi_s2idle_wakeup(void);
-extern bool acpi_sleep_no_ec_events(void);
extern int acpi_sleep_init(void);
#else
static inline bool acpi_s2idle_wakeup(void) { return false; }
-static inline bool acpi_sleep_no_ec_events(void) { return true; }
static inline int acpi_sleep_init(void) { return -ENXIO; }
#endif
diff --git a/drivers/acpi/irq.c b/drivers/acpi/irq.c
index 89690a471360..e209081d644b 100644
--- a/drivers/acpi/irq.c
+++ b/drivers/acpi/irq.c
@@ -292,3 +292,29 @@ void __init acpi_set_irq_model(enum acpi_irq_model_id model,
acpi_irq_model = model;
acpi_gsi_domain_id = fwnode;
}
+
+/**
+ * acpi_irq_create_hierarchy - Create a hierarchical IRQ domain with the default
+ * GSI domain as its parent.
+ * @flags: Irq domain flags associated with the domain
+ * @size: Size of the domain.
+ * @fwnode: Optional fwnode of the interrupt controller
+ * @ops: Pointer to the interrupt domain callbacks
+ * @host_data: Controller private data pointer
+ */
+struct irq_domain *acpi_irq_create_hierarchy(unsigned int flags,
+ unsigned int size,
+ struct fwnode_handle *fwnode,
+ const struct irq_domain_ops *ops,
+ void *host_data)
+{
+ struct irq_domain *d = irq_find_matching_fwnode(acpi_gsi_domain_id,
+ DOMAIN_BUS_ANY);
+
+ if (!d)
+ return NULL;
+
+ return irq_domain_create_hierarchy(d, flags, size, fwnode, ops,
+ host_data);
+}
+EXPORT_SYMBOL_GPL(acpi_irq_create_hierarchy);
diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c
index 23022cf20d26..1413324982f0 100644
--- a/drivers/acpi/nfit/core.c
+++ b/drivers/acpi/nfit/core.c
@@ -1282,7 +1282,7 @@ static ssize_t hw_error_scrub_store(struct device *dev,
if (rc)
return rc;
- device_lock(dev);
+ nfit_device_lock(dev);
nd_desc = dev_get_drvdata(dev);
if (nd_desc) {
struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
@@ -1299,7 +1299,7 @@ static ssize_t hw_error_scrub_store(struct device *dev,
break;
}
}
- device_unlock(dev);
+ nfit_device_unlock(dev);
if (rc)
return rc;
return size;
@@ -1319,7 +1319,7 @@ static ssize_t scrub_show(struct device *dev,
ssize_t rc = -ENXIO;
bool busy;
- device_lock(dev);
+ nfit_device_lock(dev);
nd_desc = dev_get_drvdata(dev);
if (!nd_desc) {
device_unlock(dev);
@@ -1339,7 +1339,7 @@ static ssize_t scrub_show(struct device *dev,
}
mutex_unlock(&acpi_desc->init_mutex);
- device_unlock(dev);
+ nfit_device_unlock(dev);
return rc;
}
@@ -1356,14 +1356,14 @@ static ssize_t scrub_store(struct device *dev,
if (val != 1)
return -EINVAL;
- device_lock(dev);
+ nfit_device_lock(dev);
nd_desc = dev_get_drvdata(dev);
if (nd_desc) {
struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
rc = acpi_nfit_ars_rescan(acpi_desc, ARS_REQ_LONG);
}
- device_unlock(dev);
+ nfit_device_unlock(dev);
if (rc)
return rc;
return size;
@@ -1749,9 +1749,9 @@ static void acpi_nvdimm_notify(acpi_handle handle, u32 event, void *data)
struct acpi_device *adev = data;
struct device *dev = &adev->dev;
- device_lock(dev->parent);
+ nfit_device_lock(dev->parent);
__acpi_nvdimm_notify(dev, event);
- device_unlock(dev->parent);
+ nfit_device_unlock(dev->parent);
}
static bool acpi_nvdimm_has_method(struct acpi_device *adev, char *method)
@@ -2426,7 +2426,7 @@ static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw,
offset = to_interleave_offset(offset, mmio);
writeq(cmd, mmio->addr.base + offset);
- nvdimm_flush(nfit_blk->nd_region);
+ nvdimm_flush(nfit_blk->nd_region, NULL);
if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH)
readq(mmio->addr.base + offset);
@@ -2475,7 +2475,7 @@ static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk,
}
if (rw)
- nvdimm_flush(nfit_blk->nd_region);
+ nvdimm_flush(nfit_blk->nd_region, NULL);
rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0;
return rc;
@@ -3457,8 +3457,8 @@ static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc)
struct device *dev = acpi_desc->dev;
/* Bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */
- device_lock(dev);
- device_unlock(dev);
+ nfit_device_lock(dev);
+ nfit_device_unlock(dev);
/* Bounce the init_mutex to complete initial registration */
mutex_lock(&acpi_desc->init_mutex);
@@ -3602,8 +3602,8 @@ void acpi_nfit_shutdown(void *data)
* acpi_nfit_ars_rescan() submissions have had a chance to
* either submit or see ->cancel set.
*/
- device_lock(bus_dev);
- device_unlock(bus_dev);
+ nfit_device_lock(bus_dev);
+ nfit_device_unlock(bus_dev);
flush_workqueue(nfit_wq);
}
@@ -3746,9 +3746,9 @@ EXPORT_SYMBOL_GPL(__acpi_nfit_notify);
static void acpi_nfit_notify(struct acpi_device *adev, u32 event)
{
- device_lock(&adev->dev);
+ nfit_device_lock(&adev->dev);
__acpi_nfit_notify(&adev->dev, adev->handle, event);
- device_unlock(&adev->dev);
+ nfit_device_unlock(&adev->dev);
}
static const struct acpi_device_id acpi_nfit_ids[] = {
diff --git a/drivers/acpi/nfit/nfit.h b/drivers/acpi/nfit/nfit.h
index 6ee2b02af73e..24241941181c 100644
--- a/drivers/acpi/nfit/nfit.h
+++ b/drivers/acpi/nfit/nfit.h
@@ -312,6 +312,30 @@ static inline struct acpi_nfit_desc *to_acpi_desc(
return container_of(nd_desc, struct acpi_nfit_desc, nd_desc);
}
+#ifdef CONFIG_PROVE_LOCKING
+static inline void nfit_device_lock(struct device *dev)
+{
+ device_lock(dev);
+ mutex_lock(&dev->lockdep_mutex);
+}
+
+static inline void nfit_device_unlock(struct device *dev)
+{
+ mutex_unlock(&dev->lockdep_mutex);
+ device_unlock(dev);
+}
+#else
+static inline void nfit_device_lock(struct device *dev)
+{
+ device_lock(dev);
+}
+
+static inline void nfit_device_unlock(struct device *dev)
+{
+ device_unlock(dev);
+}
+#endif
+
const guid_t *to_nfit_uuid(enum nfit_uuids id);
int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *nfit, acpi_size sz);
void acpi_nfit_shutdown(void *data);
diff --git a/drivers/acpi/osl.c b/drivers/acpi/osl.c
index cc7507091dec..2f9d0d20b836 100644
--- a/drivers/acpi/osl.c
+++ b/drivers/acpi/osl.c
@@ -14,6 +14,7 @@
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/highmem.h>
+#include <linux/lockdep.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/kmod.h>
@@ -80,6 +81,7 @@ struct acpi_ioremap {
static LIST_HEAD(acpi_ioremaps);
static DEFINE_MUTEX(acpi_ioremap_lock);
+#define acpi_ioremap_lock_held() lock_is_held(&acpi_ioremap_lock.dep_map)
static void __init acpi_request_region (struct acpi_generic_address *gas,
unsigned int length, char *desc)
@@ -206,7 +208,7 @@ acpi_map_lookup(acpi_physical_address phys, acpi_size size)
{
struct acpi_ioremap *map;
- list_for_each_entry_rcu(map, &acpi_ioremaps, list)
+ list_for_each_entry_rcu(map, &acpi_ioremaps, list, acpi_ioremap_lock_held())
if (map->phys <= phys &&
phys + size <= map->phys + map->size)
return map;
@@ -249,7 +251,7 @@ acpi_map_lookup_virt(void __iomem *virt, acpi_size size)
{
struct acpi_ioremap *map;
- list_for_each_entry_rcu(map, &acpi_ioremaps, list)
+ list_for_each_entry_rcu(map, &acpi_ioremaps, list, acpi_ioremap_lock_held())
if (map->virt <= virt &&
virt + size <= map->virt + map->size)
return map;
@@ -301,8 +303,8 @@ static void acpi_unmap(acpi_physical_address pg_off, void __iomem *vaddr)
* During early init (when acpi_permanent_mmap has not been set yet) this
* routine simply calls __acpi_map_table() to get the job done.
*/
-void __iomem *__ref
-acpi_os_map_iomem(acpi_physical_address phys, acpi_size size)
+void __iomem __ref
+*acpi_os_map_iomem(acpi_physical_address phys, acpi_size size)
{
struct acpi_ioremap *map;
void __iomem *virt;
diff --git a/drivers/acpi/pci_irq.c b/drivers/acpi/pci_irq.c
index d2549ae65e1b..dea8a60e18a4 100644
--- a/drivers/acpi/pci_irq.c
+++ b/drivers/acpi/pci_irq.c
@@ -449,8 +449,10 @@ int acpi_pci_irq_enable(struct pci_dev *dev)
* No IRQ known to the ACPI subsystem - maybe the BIOS /
* driver reported one, then use it. Exit in any case.
*/
- if (!acpi_pci_irq_valid(dev, pin))
+ if (!acpi_pci_irq_valid(dev, pin)) {
+ kfree(entry);
return 0;
+ }
if (acpi_isa_register_gsi(dev))
dev_warn(&dev->dev, "PCI INT %c: no GSI\n",
diff --git a/drivers/acpi/pci_link.c b/drivers/acpi/pci_link.c
index db11f7771ef1..00a6da2121be 100644
--- a/drivers/acpi/pci_link.c
+++ b/drivers/acpi/pci_link.c
@@ -661,7 +661,7 @@ int acpi_pci_link_allocate_irq(acpi_handle handle, int index, int *triggering,
ACPI_DEBUG_PRINT((ACPI_DB_INFO,
"Link %s is referenced\n",
acpi_device_bid(link->device)));
- return (link->irq.active);
+ return link->irq.active;
}
/*
@@ -712,7 +712,7 @@ int acpi_pci_link_free_irq(acpi_handle handle)
acpi_evaluate_object(link->device->handle, "_DIS", NULL, NULL);
mutex_unlock(&acpi_link_lock);
- return (link->irq.active);
+ return link->irq.active;
}
/* --------------------------------------------------------------------------
diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c
index 39f5d172e84f..314a187ed572 100644
--- a/drivers/acpi/pci_root.c
+++ b/drivers/acpi/pci_root.c
@@ -881,6 +881,7 @@ struct pci_bus *acpi_pci_root_create(struct acpi_pci_root *root,
int node = acpi_get_node(device->handle);
struct pci_bus *bus;
struct pci_host_bridge *host_bridge;
+ union acpi_object *obj;
info->root = root;
info->bridge = device;
@@ -917,6 +918,17 @@ struct pci_bus *acpi_pci_root_create(struct acpi_pci_root *root,
if (!(root->osc_control_set & OSC_PCI_EXPRESS_LTR_CONTROL))
host_bridge->native_ltr = 0;
+ /*
+ * Evaluate the "PCI Boot Configuration" _DSM Function. If it
+ * exists and returns 0, we must preserve any PCI resource
+ * assignments made by firmware for this host bridge.
+ */
+ obj = acpi_evaluate_dsm(ACPI_HANDLE(bus->bridge), &pci_acpi_dsm_guid, 1,
+ IGNORE_PCI_BOOT_CONFIG_DSM, NULL);
+ if (obj && obj->type == ACPI_TYPE_INTEGER && obj->integer.value == 0)
+ host_bridge->preserve_config = 1;
+ ACPI_FREE(obj);
+
pci_scan_child_bus(bus);
pci_set_host_bridge_release(host_bridge, acpi_pci_root_release_info,
info);
diff --git a/drivers/acpi/pmic/intel_pmic.c b/drivers/acpi/pmic/intel_pmic.c
index 1b722fd57d5e..452041398b34 100644
--- a/drivers/acpi/pmic/intel_pmic.c
+++ b/drivers/acpi/pmic/intel_pmic.c
@@ -284,8 +284,6 @@ int intel_pmic_install_opregion_handler(struct device *dev, acpi_handle handle,
intel_pmic_thermal_handler,
NULL, opregion);
if (ACPI_FAILURE(status)) {
- acpi_remove_address_space_handler(handle, PMIC_POWER_OPREGION_ID,
- intel_pmic_power_handler);
ret = -ENODEV;
goto out_remove_power_handler;
}
diff --git a/drivers/acpi/power.c b/drivers/acpi/power.c
index a916417b9e70..fe1e7bc91a5e 100644
--- a/drivers/acpi/power.c
+++ b/drivers/acpi/power.c
@@ -42,6 +42,11 @@ ACPI_MODULE_NAME("power");
#define ACPI_POWER_RESOURCE_STATE_ON 0x01
#define ACPI_POWER_RESOURCE_STATE_UNKNOWN 0xFF
+struct acpi_power_dependent_device {
+ struct device *dev;
+ struct list_head node;
+};
+
struct acpi_power_resource {
struct acpi_device device;
struct list_head list_node;
@@ -51,6 +56,7 @@ struct acpi_power_resource {
unsigned int ref_count;
bool wakeup_enabled;
struct mutex resource_lock;
+ struct list_head dependents;
};
struct acpi_power_resource_entry {
@@ -232,8 +238,121 @@ static int acpi_power_get_list_state(struct list_head *list, int *state)
return 0;
}
+static int
+acpi_power_resource_add_dependent(struct acpi_power_resource *resource,
+ struct device *dev)
+{
+ struct acpi_power_dependent_device *dep;
+ int ret = 0;
+
+ mutex_lock(&resource->resource_lock);
+ list_for_each_entry(dep, &resource->dependents, node) {
+ /* Only add it once */
+ if (dep->dev == dev)
+ goto unlock;
+ }
+
+ dep = kzalloc(sizeof(*dep), GFP_KERNEL);
+ if (!dep) {
+ ret = -ENOMEM;
+ goto unlock;
+ }
+
+ dep->dev = dev;
+ list_add_tail(&dep->node, &resource->dependents);
+ dev_dbg(dev, "added power dependency to [%s]\n", resource->name);
+
+unlock:
+ mutex_unlock(&resource->resource_lock);
+ return ret;
+}
+
+static void
+acpi_power_resource_remove_dependent(struct acpi_power_resource *resource,
+ struct device *dev)
+{
+ struct acpi_power_dependent_device *dep;
+
+ mutex_lock(&resource->resource_lock);
+ list_for_each_entry(dep, &resource->dependents, node) {
+ if (dep->dev == dev) {
+ list_del(&dep->node);
+ kfree(dep);
+ dev_dbg(dev, "removed power dependency to [%s]\n",
+ resource->name);
+ break;
+ }
+ }
+ mutex_unlock(&resource->resource_lock);
+}
+
+/**
+ * acpi_device_power_add_dependent - Add dependent device of this ACPI device
+ * @adev: ACPI device pointer
+ * @dev: Dependent device
+ *
+ * If @adev has non-empty _PR0 the @dev is added as dependent device to all
+ * power resources returned by it. This means that whenever these power
+ * resources are turned _ON the dependent devices get runtime resumed. This
+ * is needed for devices such as PCI to allow its driver to re-initialize
+ * it after it went to D0uninitialized.
+ *
+ * If @adev does not have _PR0 this does nothing.
+ *
+ * Returns %0 in case of success and negative errno otherwise.
+ */
+int acpi_device_power_add_dependent(struct acpi_device *adev,
+ struct device *dev)
+{
+ struct acpi_power_resource_entry *entry;
+ struct list_head *resources;
+ int ret;
+
+ if (!adev->flags.power_manageable)
+ return 0;
+
+ resources = &adev->power.states[ACPI_STATE_D0].resources;
+ list_for_each_entry(entry, resources, node) {
+ ret = acpi_power_resource_add_dependent(entry->resource, dev);
+ if (ret)
+ goto err;
+ }
+
+ return 0;
+
+err:
+ list_for_each_entry(entry, resources, node)
+ acpi_power_resource_remove_dependent(entry->resource, dev);
+
+ return ret;
+}
+
+/**
+ * acpi_device_power_remove_dependent - Remove dependent device
+ * @adev: ACPI device pointer
+ * @dev: Dependent device
+ *
+ * Does the opposite of acpi_device_power_add_dependent() and removes the
+ * dependent device if it is found. Can be called to @adev that does not
+ * have _PR0 as well.
+ */
+void acpi_device_power_remove_dependent(struct acpi_device *adev,
+ struct device *dev)
+{
+ struct acpi_power_resource_entry *entry;
+ struct list_head *resources;
+
+ if (!adev->flags.power_manageable)
+ return;
+
+ resources = &adev->power.states[ACPI_STATE_D0].resources;
+ list_for_each_entry_reverse(entry, resources, node)
+ acpi_power_resource_remove_dependent(entry->resource, dev);
+}
+
static int __acpi_power_on(struct acpi_power_resource *resource)
{
+ struct acpi_power_dependent_device *dep;
acpi_status status = AE_OK;
status = acpi_evaluate_object(resource->device.handle, "_ON", NULL, NULL);
@@ -243,6 +362,21 @@ static int __acpi_power_on(struct acpi_power_resource *resource)
ACPI_DEBUG_PRINT((ACPI_DB_INFO, "Power resource [%s] turned on\n",
resource->name));
+ /*
+ * If there are other dependents on this power resource we need to
+ * resume them now so that their drivers can re-initialize the
+ * hardware properly after it went back to D0.
+ */
+ if (list_empty(&resource->dependents) ||
+ list_is_singular(&resource->dependents))
+ return 0;
+
+ list_for_each_entry(dep, &resource->dependents, node) {
+ dev_dbg(dep->dev, "runtime resuming because [%s] turned on\n",
+ resource->name);
+ pm_request_resume(dep->dev);
+ }
+
return 0;
}
@@ -810,6 +944,7 @@ int acpi_add_power_resource(acpi_handle handle)
ACPI_STA_DEFAULT);
mutex_init(&resource->resource_lock);
INIT_LIST_HEAD(&resource->list_node);
+ INIT_LIST_HEAD(&resource->dependents);
resource->name = device->pnp.bus_id;
strcpy(acpi_device_name(device), ACPI_POWER_DEVICE_NAME);
strcpy(acpi_device_class(device), ACPI_POWER_CLASS);
diff --git a/drivers/acpi/pptt.c b/drivers/acpi/pptt.c
index b72e6afaa8fb..f31544d3656e 100644
--- a/drivers/acpi/pptt.c
+++ b/drivers/acpi/pptt.c
@@ -432,17 +432,40 @@ static void cache_setup_acpi_cpu(struct acpi_table_header *table,
}
}
+static bool flag_identical(struct acpi_table_header *table_hdr,
+ struct acpi_pptt_processor *cpu)
+{
+ struct acpi_pptt_processor *next;
+
+ /* heterogeneous machines must use PPTT revision > 1 */
+ if (table_hdr->revision < 2)
+ return false;
+
+ /* Locate the last node in the tree with IDENTICAL set */
+ if (cpu->flags & ACPI_PPTT_ACPI_IDENTICAL) {
+ next = fetch_pptt_node(table_hdr, cpu->parent);
+ if (!(next && next->flags & ACPI_PPTT_ACPI_IDENTICAL))
+ return true;
+ }
+
+ return false;
+}
+
/* Passing level values greater than this will result in search termination */
#define PPTT_ABORT_PACKAGE 0xFF
-static struct acpi_pptt_processor *acpi_find_processor_package_id(struct acpi_table_header *table_hdr,
- struct acpi_pptt_processor *cpu,
- int level, int flag)
+static struct acpi_pptt_processor *acpi_find_processor_tag(struct acpi_table_header *table_hdr,
+ struct acpi_pptt_processor *cpu,
+ int level, int flag)
{
struct acpi_pptt_processor *prev_node;
while (cpu && level) {
- if (cpu->flags & flag)
+ /* special case the identical flag to find last identical */
+ if (flag == ACPI_PPTT_ACPI_IDENTICAL) {
+ if (flag_identical(table_hdr, cpu))
+ break;
+ } else if (cpu->flags & flag)
break;
pr_debug("level %d\n", level);
prev_node = fetch_pptt_node(table_hdr, cpu->parent);
@@ -480,8 +503,8 @@ static int topology_get_acpi_cpu_tag(struct acpi_table_header *table,
cpu_node = acpi_find_processor_node(table, acpi_cpu_id);
if (cpu_node) {
- cpu_node = acpi_find_processor_package_id(table, cpu_node,
- level, flag);
+ cpu_node = acpi_find_processor_tag(table, cpu_node,
+ level, flag);
/*
* As per specification if the processor structure represents
* an actual processor, then ACPI processor ID must be valid.
@@ -518,6 +541,44 @@ static int find_acpi_cpu_topology_tag(unsigned int cpu, int level, int flag)
}
/**
+ * check_acpi_cpu_flag() - Determine if CPU node has a flag set
+ * @cpu: Kernel logical CPU number
+ * @rev: The minimum PPTT revision defining the flag
+ * @flag: The flag itself
+ *
+ * Check the node representing a CPU for a given flag.
+ *
+ * Return: -ENOENT if the PPTT doesn't exist, the CPU cannot be found or
+ * the table revision isn't new enough.
+ * 1, any passed flag set
+ * 0, flag unset
+ */
+static int check_acpi_cpu_flag(unsigned int cpu, int rev, u32 flag)
+{
+ struct acpi_table_header *table;
+ acpi_status status;
+ u32 acpi_cpu_id = get_acpi_id_for_cpu(cpu);
+ struct acpi_pptt_processor *cpu_node = NULL;
+ int ret = -ENOENT;
+
+ status = acpi_get_table(ACPI_SIG_PPTT, 0, &table);
+ if (ACPI_FAILURE(status)) {
+ acpi_pptt_warn_missing();
+ return ret;
+ }
+
+ if (table->revision >= rev)
+ cpu_node = acpi_find_processor_node(table, acpi_cpu_id);
+
+ if (cpu_node)
+ ret = (cpu_node->flags & flag) != 0;
+
+ acpi_put_table(table);
+
+ return ret;
+}
+
+/**
* acpi_find_last_cache_level() - Determines the number of cache levels for a PE
* @cpu: Kernel logical CPU number
*
@@ -582,6 +643,20 @@ int cache_setup_acpi(unsigned int cpu)
}
/**
+ * acpi_pptt_cpu_is_thread() - Determine if CPU is a thread
+ * @cpu: Kernel logical CPU number
+ *
+ * Return: 1, a thread
+ * 0, not a thread
+ * -ENOENT ,if the PPTT doesn't exist, the CPU cannot be found or
+ * the table revision isn't new enough.
+ */
+int acpi_pptt_cpu_is_thread(unsigned int cpu)
+{
+ return check_acpi_cpu_flag(cpu, 2, ACPI_PPTT_ACPI_PROCESSOR_IS_THREAD);
+}
+
+/**
* find_acpi_cpu_topology() - Determine a unique topology value for a given CPU
* @cpu: Kernel logical CPU number
* @level: The topological level for which we would like a unique ID
@@ -641,7 +716,6 @@ int find_acpi_cpu_cache_topology(unsigned int cpu, int level)
return ret;
}
-
/**
* find_acpi_cpu_topology_package() - Determine a unique CPU package value
* @cpu: Kernel logical CPU number
@@ -660,3 +734,29 @@ int find_acpi_cpu_topology_package(unsigned int cpu)
return find_acpi_cpu_topology_tag(cpu, PPTT_ABORT_PACKAGE,
ACPI_PPTT_PHYSICAL_PACKAGE);
}
+
+/**
+ * find_acpi_cpu_topology_hetero_id() - Get a core architecture tag
+ * @cpu: Kernel logical CPU number
+ *
+ * Determine a unique heterogeneous tag for the given CPU. CPUs with the same
+ * implementation should have matching tags.
+ *
+ * The returned tag can be used to group peers with identical implementation.
+ *
+ * The search terminates when a level is found with the identical implementation
+ * flag set or we reach a root node.
+ *
+ * Due to limitations in the PPTT data structure, there may be rare situations
+ * where two cores in a heterogeneous machine may be identical, but won't have
+ * the same tag.
+ *
+ * Return: -ENOENT if the PPTT doesn't exist, or the CPU cannot be found.
+ * Otherwise returns a value which represents a group of identical cores
+ * similar to this CPU.
+ */
+int find_acpi_cpu_topology_hetero_id(unsigned int cpu)
+{
+ return find_acpi_cpu_topology_tag(cpu, PPTT_ABORT_PACKAGE,
+ ACPI_PPTT_ACPI_IDENTICAL);
+}
diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c
index aea8d674a33d..08da9c29f1e9 100644
--- a/drivers/acpi/processor_driver.c
+++ b/drivers/acpi/processor_driver.c
@@ -284,6 +284,29 @@ static int acpi_processor_stop(struct device *dev)
return 0;
}
+bool acpi_processor_cpufreq_init;
+
+static int acpi_processor_notifier(struct notifier_block *nb,
+ unsigned long event, void *data)
+{
+ struct cpufreq_policy *policy = data;
+ int cpu = policy->cpu;
+
+ if (event == CPUFREQ_CREATE_POLICY) {
+ acpi_thermal_cpufreq_init(cpu);
+ acpi_processor_ppc_init(cpu);
+ } else if (event == CPUFREQ_REMOVE_POLICY) {
+ acpi_processor_ppc_exit(cpu);
+ acpi_thermal_cpufreq_exit(cpu);
+ }
+
+ return 0;
+}
+
+static struct notifier_block acpi_processor_notifier_block = {
+ .notifier_call = acpi_processor_notifier,
+};
+
/*
* We keep the driver loaded even when ACPI is not running.
* This is needed for the powernow-k8 driver, that works even without
@@ -310,8 +333,12 @@ static int __init acpi_processor_driver_init(void)
cpuhp_setup_state_nocalls(CPUHP_ACPI_CPUDRV_DEAD, "acpi/cpu-drv:dead",
NULL, acpi_soft_cpu_dead);
- acpi_thermal_cpufreq_init();
- acpi_processor_ppc_init();
+ if (!cpufreq_register_notifier(&acpi_processor_notifier_block,
+ CPUFREQ_POLICY_NOTIFIER)) {
+ acpi_processor_cpufreq_init = true;
+ acpi_processor_ignore_ppc_init();
+ }
+
acpi_processor_throttling_init();
return 0;
err:
@@ -324,8 +351,12 @@ static void __exit acpi_processor_driver_exit(void)
if (acpi_disabled)
return;
- acpi_processor_ppc_exit();
- acpi_thermal_cpufreq_exit();
+ if (acpi_processor_cpufreq_init) {
+ cpufreq_unregister_notifier(&acpi_processor_notifier_block,
+ CPUFREQ_POLICY_NOTIFIER);
+ acpi_processor_cpufreq_init = false;
+ }
+
cpuhp_remove_state_nocalls(hp_online);
cpuhp_remove_state_nocalls(CPUHP_ACPI_CPUDRV_DEAD);
driver_unregister(&acpi_processor_driver);
diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c
index e387a258d649..ed56c6d20b08 100644
--- a/drivers/acpi/processor_idle.c
+++ b/drivers/acpi/processor_idle.c
@@ -196,6 +196,7 @@ static void tsc_check_state(int state)
case X86_VENDOR_AMD:
case X86_VENDOR_INTEL:
case X86_VENDOR_CENTAUR:
+ case X86_VENDOR_ZHAOXIN:
/*
* AMD Fam10h TSC will tick in all
* C/P/S0/S1 states when this bit is set.
diff --git a/drivers/acpi/processor_perflib.c b/drivers/acpi/processor_perflib.c
index ee87cb6f6e59..2261713d1aec 100644
--- a/drivers/acpi/processor_perflib.c
+++ b/drivers/acpi/processor_perflib.c
@@ -50,57 +50,13 @@ module_param(ignore_ppc, int, 0644);
MODULE_PARM_DESC(ignore_ppc, "If the frequency of your machine gets wrongly" \
"limited by BIOS, this should help");
-#define PPC_REGISTERED 1
-#define PPC_IN_USE 2
-
-static int acpi_processor_ppc_status;
-
-static int acpi_processor_ppc_notifier(struct notifier_block *nb,
- unsigned long event, void *data)
-{
- struct cpufreq_policy *policy = data;
- struct acpi_processor *pr;
- unsigned int ppc = 0;
-
- if (ignore_ppc < 0)
- ignore_ppc = 0;
-
- if (ignore_ppc)
- return 0;
-
- if (event != CPUFREQ_ADJUST)
- return 0;
-
- mutex_lock(&performance_mutex);
-
- pr = per_cpu(processors, policy->cpu);
- if (!pr || !pr->performance)
- goto out;
-
- ppc = (unsigned int)pr->performance_platform_limit;
-
- if (ppc >= pr->performance->state_count)
- goto out;
-
- cpufreq_verify_within_limits(policy, 0,
- pr->performance->states[ppc].
- core_frequency * 1000);
-
- out:
- mutex_unlock(&performance_mutex);
-
- return 0;
-}
-
-static struct notifier_block acpi_ppc_notifier_block = {
- .notifier_call = acpi_processor_ppc_notifier,
-};
+static bool acpi_processor_ppc_in_use;
static int acpi_processor_get_platform_limit(struct acpi_processor *pr)
{
acpi_status status = 0;
unsigned long long ppc = 0;
-
+ int ret;
if (!pr)
return -EINVAL;
@@ -112,7 +68,7 @@ static int acpi_processor_get_platform_limit(struct acpi_processor *pr)
status = acpi_evaluate_integer(pr->handle, "_PPC", NULL, &ppc);
if (status != AE_NOT_FOUND)
- acpi_processor_ppc_status |= PPC_IN_USE;
+ acpi_processor_ppc_in_use = true;
if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) {
ACPI_EXCEPTION((AE_INFO, status, "Evaluating _PPC"));
@@ -124,6 +80,17 @@ static int acpi_processor_get_platform_limit(struct acpi_processor *pr)
pr->performance_platform_limit = (int)ppc;
+ if (ppc >= pr->performance->state_count ||
+ unlikely(!dev_pm_qos_request_active(&pr->perflib_req)))
+ return 0;
+
+ ret = dev_pm_qos_update_request(&pr->perflib_req,
+ pr->performance->states[ppc].core_frequency * 1000);
+ if (ret < 0) {
+ pr_warn("Failed to update perflib freq constraint: CPU%d (%d)\n",
+ pr->id, ret);
+ }
+
return 0;
}
@@ -184,23 +151,32 @@ int acpi_processor_get_bios_limit(int cpu, unsigned int *limit)
}
EXPORT_SYMBOL(acpi_processor_get_bios_limit);
-void acpi_processor_ppc_init(void)
+void acpi_processor_ignore_ppc_init(void)
{
- if (!cpufreq_register_notifier
- (&acpi_ppc_notifier_block, CPUFREQ_POLICY_NOTIFIER))
- acpi_processor_ppc_status |= PPC_REGISTERED;
- else
- printk(KERN_DEBUG
- "Warning: Processor Platform Limit not supported.\n");
+ if (ignore_ppc < 0)
+ ignore_ppc = 0;
+}
+
+void acpi_processor_ppc_init(int cpu)
+{
+ struct acpi_processor *pr = per_cpu(processors, cpu);
+ int ret;
+
+ ret = dev_pm_qos_add_request(get_cpu_device(cpu),
+ &pr->perflib_req, DEV_PM_QOS_MAX_FREQUENCY,
+ INT_MAX);
+ if (ret < 0) {
+ pr_err("Failed to add freq constraint for CPU%d (%d)\n", cpu,
+ ret);
+ return;
+ }
}
-void acpi_processor_ppc_exit(void)
+void acpi_processor_ppc_exit(int cpu)
{
- if (acpi_processor_ppc_status & PPC_REGISTERED)
- cpufreq_unregister_notifier(&acpi_ppc_notifier_block,
- CPUFREQ_POLICY_NOTIFIER);
+ struct acpi_processor *pr = per_cpu(processors, cpu);
- acpi_processor_ppc_status &= ~PPC_REGISTERED;
+ dev_pm_qos_remove_request(&pr->perflib_req);
}
static int acpi_processor_get_performance_control(struct acpi_processor *pr)
@@ -477,7 +453,7 @@ int acpi_processor_notify_smm(struct module *calling_module)
static int is_done = 0;
int result;
- if (!(acpi_processor_ppc_status & PPC_REGISTERED))
+ if (!acpi_processor_cpufreq_init)
return -EBUSY;
if (!try_module_get(calling_module))
@@ -513,7 +489,7 @@ int acpi_processor_notify_smm(struct module *calling_module)
* we can allow the cpufreq driver to be rmmod'ed. */
is_done = 1;
- if (!(acpi_processor_ppc_status & PPC_IN_USE))
+ if (!acpi_processor_ppc_in_use)
module_put(calling_module);
return 0;
@@ -742,7 +718,7 @@ acpi_processor_register_performance(struct acpi_processor_performance
{
struct acpi_processor *pr;
- if (!(acpi_processor_ppc_status & PPC_REGISTERED))
+ if (!acpi_processor_cpufreq_init)
return -EINVAL;
mutex_lock(&performance_mutex);
diff --git a/drivers/acpi/processor_thermal.c b/drivers/acpi/processor_thermal.c
index 50fb0107375e..ec2638f1df4f 100644
--- a/drivers/acpi/processor_thermal.c
+++ b/drivers/acpi/processor_thermal.c
@@ -35,7 +35,6 @@ ACPI_MODULE_NAME("processor_thermal");
#define CPUFREQ_THERMAL_MAX_STEP 3
static DEFINE_PER_CPU(unsigned int, cpufreq_thermal_reduction_pctg);
-static unsigned int acpi_thermal_cpufreq_is_init = 0;
#define reduction_pctg(cpu) \
per_cpu(cpufreq_thermal_reduction_pctg, phys_package_first_cpu(cpu))
@@ -61,35 +60,11 @@ static int phys_package_first_cpu(int cpu)
static int cpu_has_cpufreq(unsigned int cpu)
{
struct cpufreq_policy policy;
- if (!acpi_thermal_cpufreq_is_init || cpufreq_get_policy(&policy, cpu))
+ if (!acpi_processor_cpufreq_init || cpufreq_get_policy(&policy, cpu))
return 0;
return 1;
}
-static int acpi_thermal_cpufreq_notifier(struct notifier_block *nb,
- unsigned long event, void *data)
-{
- struct cpufreq_policy *policy = data;
- unsigned long max_freq = 0;
-
- if (event != CPUFREQ_ADJUST)
- goto out;
-
- max_freq = (
- policy->cpuinfo.max_freq *
- (100 - reduction_pctg(policy->cpu) * 20)
- ) / 100;
-
- cpufreq_verify_within_limits(policy, 0, max_freq);
-
- out:
- return 0;
-}
-
-static struct notifier_block acpi_thermal_cpufreq_notifier_block = {
- .notifier_call = acpi_thermal_cpufreq_notifier,
-};
-
static int cpufreq_get_max_state(unsigned int cpu)
{
if (!cpu_has_cpufreq(cpu))
@@ -108,7 +83,10 @@ static int cpufreq_get_cur_state(unsigned int cpu)
static int cpufreq_set_cur_state(unsigned int cpu, int state)
{
- int i;
+ struct cpufreq_policy *policy;
+ struct acpi_processor *pr;
+ unsigned long max_freq;
+ int i, ret;
if (!cpu_has_cpufreq(cpu))
return 0;
@@ -121,33 +99,53 @@ static int cpufreq_set_cur_state(unsigned int cpu, int state)
* frequency.
*/
for_each_online_cpu(i) {
- if (topology_physical_package_id(i) ==
+ if (topology_physical_package_id(i) !=
topology_physical_package_id(cpu))
- cpufreq_update_policy(i);
+ continue;
+
+ pr = per_cpu(processors, i);
+
+ if (unlikely(!dev_pm_qos_request_active(&pr->thermal_req)))
+ continue;
+
+ policy = cpufreq_cpu_get(i);
+ if (!policy)
+ return -EINVAL;
+
+ max_freq = (policy->cpuinfo.max_freq * (100 - reduction_pctg(i) * 20)) / 100;
+
+ cpufreq_cpu_put(policy);
+
+ ret = dev_pm_qos_update_request(&pr->thermal_req, max_freq);
+ if (ret < 0) {
+ pr_warn("Failed to update thermal freq constraint: CPU%d (%d)\n",
+ pr->id, ret);
+ }
}
return 0;
}
-void acpi_thermal_cpufreq_init(void)
+void acpi_thermal_cpufreq_init(int cpu)
{
- int i;
-
- i = cpufreq_register_notifier(&acpi_thermal_cpufreq_notifier_block,
- CPUFREQ_POLICY_NOTIFIER);
- if (!i)
- acpi_thermal_cpufreq_is_init = 1;
+ struct acpi_processor *pr = per_cpu(processors, cpu);
+ int ret;
+
+ ret = dev_pm_qos_add_request(get_cpu_device(cpu),
+ &pr->thermal_req, DEV_PM_QOS_MAX_FREQUENCY,
+ INT_MAX);
+ if (ret < 0) {
+ pr_err("Failed to add freq constraint for CPU%d (%d)\n", cpu,
+ ret);
+ return;
+ }
}
-void acpi_thermal_cpufreq_exit(void)
+void acpi_thermal_cpufreq_exit(int cpu)
{
- if (acpi_thermal_cpufreq_is_init)
- cpufreq_unregister_notifier
- (&acpi_thermal_cpufreq_notifier_block,
- CPUFREQ_POLICY_NOTIFIER);
+ struct acpi_processor *pr = per_cpu(processors, cpu);
- acpi_thermal_cpufreq_is_init = 0;
+ dev_pm_qos_remove_request(&pr->thermal_req);
}
-
#else /* ! CONFIG_CPU_FREQ */
static int cpufreq_get_max_state(unsigned int cpu)
{
diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c
index da3ced297f19..3eacf474e1e3 100644
--- a/drivers/acpi/property.c
+++ b/drivers/acpi/property.c
@@ -39,6 +39,12 @@ static const guid_t prp_guids[] = {
/* External facing port GUID: efcc06cc-73ac-4bc3-bff0-76143807c389 */
GUID_INIT(0xefcc06cc, 0x73ac, 0x4bc3,
0xbf, 0xf0, 0x76, 0x14, 0x38, 0x07, 0xc3, 0x89),
+ /* Thunderbolt GUID for IMR_VALID: c44d002f-69f9-4e7d-a904-a7baabdf43f7 */
+ GUID_INIT(0xc44d002f, 0x69f9, 0x4e7d,
+ 0xa9, 0x04, 0xa7, 0xba, 0xab, 0xdf, 0x43, 0xf7),
+ /* Thunderbolt GUID for WAKE_SUPPORTED: 6c501103-c189-4296-ba72-9bf5a26ebe5d */
+ GUID_INIT(0x6c501103, 0xc189, 0x4296,
+ 0xba, 0x72, 0x9b, 0xf5, 0xa2, 0x6e, 0xbe, 0x5d),
};
/* ACPI _DSD data subnodes GUID: dbb8e3e6-5886-4ba6-8795-1319f52a966b */
@@ -600,15 +606,29 @@ static struct fwnode_handle *
acpi_fwnode_get_named_child_node(const struct fwnode_handle *fwnode,
const char *childname)
{
+ char name[ACPI_PATH_SEGMENT_LENGTH];
struct fwnode_handle *child;
+ struct acpi_buffer path;
+ acpi_status status;
- /*
- * Find first matching named child node of this fwnode.
- * For ACPI this will be a data only sub-node.
- */
- fwnode_for_each_child_node(fwnode, child)
- if (acpi_data_node_match(child, childname))
+ path.length = sizeof(name);
+ path.pointer = name;
+
+ fwnode_for_each_child_node(fwnode, child) {
+ if (is_acpi_data_node(child)) {
+ if (acpi_data_node_match(child, childname))
+ return child;
+ continue;
+ }
+
+ status = acpi_get_name(ACPI_HANDLE_FWNODE(child),
+ ACPI_SINGLE_NAME, &path);
+ if (ACPI_FAILURE(status))
+ break;
+
+ if (!strncmp(name, childname, ACPI_NAMESEG_SIZE))
return child;
+ }
return NULL;
}
@@ -1196,7 +1216,7 @@ static struct fwnode_handle *acpi_graph_get_child_prop_value(
/**
- * acpi_graph_get_remote_enpoint - Parses and returns remote end of an endpoint
+ * acpi_graph_get_remote_endpoint - Parses and returns remote end of an endpoint
* @fwnode: Endpoint firmware node pointing to a remote device
* @endpoint: Firmware node of remote endpoint is filled here if not %NULL
*
diff --git a/drivers/acpi/sbshc.h b/drivers/acpi/sbshc.h
index 06372a37df10..c3522bb82792 100644
--- a/drivers/acpi/sbshc.h
+++ b/drivers/acpi/sbshc.h
@@ -15,8 +15,6 @@ enum acpi_smb_protocol {
SMBUS_BLOCK_PROCESS_CALL = 0xd,
};
-static const u8 SMBUS_PEC = 0x80;
-
enum acpi_sbs_device_addr {
ACPI_SBS_CHARGER = 0x9,
ACPI_SBS_MANAGER = 0xa,
diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c
index 0e28270b0fd8..aad6be5c0af0 100644
--- a/drivers/acpi/scan.c
+++ b/drivers/acpi/scan.c
@@ -2204,6 +2204,12 @@ int __init acpi_scan_init(void)
acpi_gpe_apply_masked_gpes();
acpi_update_all_gpes();
+ /*
+ * Although we call __add_memory() that is documented to require the
+ * device_hotplug_lock, it is not necessary here because this is an
+ * early code when userspace or any other code path cannot trigger
+ * hotplug/hotunplug operations.
+ */
mutex_lock(&acpi_scan_lock);
/*
* Enumerate devices in the ACPI namespace.
diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c
index 8ff08e531443..9fa77d72ef27 100644
--- a/drivers/acpi/sleep.c
+++ b/drivers/acpi/sleep.c
@@ -77,7 +77,7 @@ static int acpi_sleep_prepare(u32 acpi_state)
return 0;
}
-static bool acpi_sleep_state_supported(u8 sleep_state)
+bool acpi_sleep_state_supported(u8 sleep_state)
{
acpi_status status;
u8 type_a, type_b;
@@ -89,6 +89,10 @@ static bool acpi_sleep_state_supported(u8 sleep_state)
}
#ifdef CONFIG_ACPI_SLEEP
+static bool sleep_no_lps0 __read_mostly;
+module_param(sleep_no_lps0, bool, 0644);
+MODULE_PARM_DESC(sleep_no_lps0, "Do not use the special LPS0 device interface");
+
static u32 acpi_target_sleep_state = ACPI_STATE_S0;
u32 acpi_target_system_state(void)
@@ -158,11 +162,11 @@ static int __init init_nvs_nosave(const struct dmi_system_id *d)
return 0;
}
-static bool acpi_sleep_no_lps0;
+static bool acpi_sleep_default_s3;
-static int __init init_no_lps0(const struct dmi_system_id *d)
+static int __init init_default_s3(const struct dmi_system_id *d)
{
- acpi_sleep_no_lps0 = true;
+ acpi_sleep_default_s3 = true;
return 0;
}
@@ -363,7 +367,7 @@ static const struct dmi_system_id acpisleep_dmi_table[] __initconst = {
* S0 Idle firmware interface.
*/
{
- .callback = init_no_lps0,
+ .callback = init_default_s3,
.ident = "Dell XPS13 9360",
.matches = {
DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
@@ -376,7 +380,7 @@ static const struct dmi_system_id acpisleep_dmi_table[] __initconst = {
* https://bugzilla.kernel.org/show_bug.cgi?id=199057).
*/
{
- .callback = init_no_lps0,
+ .callback = init_default_s3,
.ident = "ThinkPad X1 Tablet(2016)",
.matches = {
DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
@@ -452,14 +456,6 @@ static int acpi_pm_prepare(void)
return error;
}
-static int find_powerf_dev(struct device *dev, void *data)
-{
- struct acpi_device *device = to_acpi_device(dev);
- const char *hid = acpi_device_hid(device);
-
- return !strcmp(hid, ACPI_BUTTON_HID_POWERF);
-}
-
/**
* acpi_pm_finish - Instruct the platform to leave a sleep state.
*
@@ -468,7 +464,7 @@ static int find_powerf_dev(struct device *dev, void *data)
*/
static void acpi_pm_finish(void)
{
- struct device *pwr_btn_dev;
+ struct acpi_device *pwr_btn_adev;
u32 acpi_state = acpi_target_sleep_state;
acpi_ec_unblock_transactions();
@@ -499,11 +495,11 @@ static void acpi_pm_finish(void)
return;
pwr_btn_event_pending = false;
- pwr_btn_dev = bus_find_device(&acpi_bus_type, NULL, NULL,
- find_powerf_dev);
- if (pwr_btn_dev) {
- pm_wakeup_event(pwr_btn_dev, 0);
- put_device(pwr_btn_dev);
+ pwr_btn_adev = acpi_dev_get_first_match_dev(ACPI_BUTTON_HID_POWERF,
+ NULL, -1);
+ if (pwr_btn_adev) {
+ pm_wakeup_event(&pwr_btn_adev->dev, 0);
+ acpi_dev_put(pwr_btn_adev);
}
}
@@ -532,8 +528,9 @@ static void acpi_pm_end(void)
acpi_sleep_tts_switch(acpi_target_sleep_state);
}
#else /* !CONFIG_ACPI_SLEEP */
+#define sleep_no_lps0 (1)
#define acpi_target_sleep_state ACPI_STATE_S0
-#define acpi_sleep_no_lps0 (false)
+#define acpi_sleep_default_s3 (1)
static inline void acpi_sleep_dmi_check(void) {}
#endif /* CONFIG_ACPI_SLEEP */
@@ -699,7 +696,6 @@ static const struct platform_suspend_ops acpi_suspend_ops_old = {
.recover = acpi_pm_finish,
};
-static bool s2idle_in_progress;
static bool s2idle_wakeup;
/*
@@ -912,42 +908,43 @@ static int lps0_device_attach(struct acpi_device *adev,
if (lps0_device_handle)
return 0;
- if (acpi_sleep_no_lps0) {
- acpi_handle_info(adev->handle,
- "Low Power S0 Idle interface disabled\n");
- return 0;
- }
-
if (!(acpi_gbl_FADT.flags & ACPI_FADT_LOW_POWER_S0))
return 0;
guid_parse(ACPI_LPS0_DSM_UUID, &lps0_dsm_guid);
/* Check if the _DSM is present and as expected. */
out_obj = acpi_evaluate_dsm(adev->handle, &lps0_dsm_guid, 1, 0, NULL);
- if (out_obj && out_obj->type == ACPI_TYPE_BUFFER) {
- char bitmask = *(char *)out_obj->buffer.pointer;
-
- lps0_dsm_func_mask = bitmask;
- lps0_device_handle = adev->handle;
- /*
- * Use suspend-to-idle by default if the default
- * suspend mode was not set from the command line.
- */
- if (mem_sleep_default > PM_SUSPEND_MEM)
- mem_sleep_current = PM_SUSPEND_TO_IDLE;
-
- acpi_handle_debug(adev->handle, "_DSM function mask: 0x%x\n",
- bitmask);
-
- acpi_ec_mark_gpe_for_wake();
- } else {
+ if (!out_obj || out_obj->type != ACPI_TYPE_BUFFER) {
acpi_handle_debug(adev->handle,
"_DSM function 0 evaluation failed\n");
+ return 0;
}
+
+ lps0_dsm_func_mask = *(char *)out_obj->buffer.pointer;
+
ACPI_FREE(out_obj);
+ acpi_handle_debug(adev->handle, "_DSM function mask: 0x%x\n",
+ lps0_dsm_func_mask);
+
+ lps0_device_handle = adev->handle;
+
lpi_device_get_constraints();
+ /*
+ * Use suspend-to-idle by default if the default suspend mode was not
+ * set from the command line.
+ */
+ if (mem_sleep_default > PM_SUSPEND_MEM && !acpi_sleep_default_s3)
+ mem_sleep_current = PM_SUSPEND_TO_IDLE;
+
+ /*
+ * Some LPS0 systems, like ASUS Zenbook UX430UNR/i7-8550U, require the
+ * EC GPE to be enabled while suspended for certain wakeup devices to
+ * work, so mark it as wakeup-capable.
+ */
+ acpi_ec_mark_gpe_for_wake();
+
return 0;
}
@@ -959,98 +956,110 @@ static struct acpi_scan_handler lps0_handler = {
static int acpi_s2idle_begin(void)
{
acpi_scan_lock_acquire();
- s2idle_in_progress = true;
return 0;
}
static int acpi_s2idle_prepare(void)
{
- if (lps0_device_handle) {
- acpi_sleep_run_lps0_dsm(ACPI_LPS0_SCREEN_OFF);
- acpi_sleep_run_lps0_dsm(ACPI_LPS0_ENTRY);
-
+ if (acpi_sci_irq_valid()) {
+ enable_irq_wake(acpi_sci_irq);
acpi_ec_set_gpe_wake_mask(ACPI_GPE_ENABLE);
}
- if (acpi_sci_irq_valid())
- enable_irq_wake(acpi_sci_irq);
-
acpi_enable_wakeup_devices(ACPI_STATE_S0);
/* Change the configuration of GPEs to avoid spurious wakeup. */
acpi_enable_all_wakeup_gpes();
acpi_os_wait_events_complete();
+
+ s2idle_wakeup = true;
return 0;
}
-static void acpi_s2idle_wake(void)
+static int acpi_s2idle_prepare_late(void)
{
- if (!lps0_device_handle)
- return;
+ if (!lps0_device_handle || sleep_no_lps0)
+ return 0;
if (pm_debug_messages_on)
lpi_check_constraints();
+ acpi_sleep_run_lps0_dsm(ACPI_LPS0_SCREEN_OFF);
+ acpi_sleep_run_lps0_dsm(ACPI_LPS0_ENTRY);
+
+ return 0;
+}
+
+static void acpi_s2idle_wake(void)
+{
/*
- * If IRQD_WAKEUP_ARMED is not set for the SCI at this point, it means
- * that the SCI has triggered while suspended, so cancel the wakeup in
- * case it has not been a wakeup event (the GPEs will be checked later).
+ * If IRQD_WAKEUP_ARMED is set for the SCI at this point, the SCI has
+ * not triggered while suspended, so bail out.
*/
- if (acpi_sci_irq_valid() &&
- !irqd_is_wakeup_armed(irq_get_irq_data(acpi_sci_irq))) {
+ if (!acpi_sci_irq_valid() ||
+ irqd_is_wakeup_armed(irq_get_irq_data(acpi_sci_irq)))
+ return;
+
+ /*
+ * If there are EC events to process, the wakeup may be a spurious one
+ * coming from the EC.
+ */
+ if (acpi_ec_dispatch_gpe()) {
+ /*
+ * Cancel the wakeup and process all pending events in case
+ * there are any wakeup ones in there.
+ *
+ * Note that if any non-EC GPEs are active at this point, the
+ * SCI will retrigger after the rearming below, so no events
+ * should be missed by canceling the wakeup here.
+ */
pm_system_cancel_wakeup();
- s2idle_wakeup = true;
/*
- * On some platforms with the LPS0 _DSM device noirq resume
- * takes too much time for EC wakeup events to survive, so look
- * for them now.
+ * The EC driver uses the system workqueue and an additional
+ * special one, so those need to be flushed too.
*/
- acpi_ec_dispatch_gpe();
+ acpi_os_wait_events_complete(); /* synchronize EC GPE processing */
+ acpi_ec_flush_work();
+ acpi_os_wait_events_complete(); /* synchronize Notify handling */
+
+ rearm_wake_irq(acpi_sci_irq);
}
}
-static void acpi_s2idle_sync(void)
+static void acpi_s2idle_restore_early(void)
{
- /*
- * Process all pending events in case there are any wakeup ones.
- *
- * The EC driver uses the system workqueue and an additional special
- * one, so those need to be flushed too.
- */
- acpi_os_wait_events_complete(); /* synchronize SCI IRQ handling */
- acpi_ec_flush_work();
- acpi_os_wait_events_complete(); /* synchronize Notify handling */
- s2idle_wakeup = false;
+ if (!lps0_device_handle || sleep_no_lps0)
+ return;
+
+ acpi_sleep_run_lps0_dsm(ACPI_LPS0_EXIT);
+ acpi_sleep_run_lps0_dsm(ACPI_LPS0_SCREEN_ON);
}
static void acpi_s2idle_restore(void)
{
+ s2idle_wakeup = false;
+
acpi_enable_all_runtime_gpes();
acpi_disable_wakeup_devices(ACPI_STATE_S0);
- if (acpi_sci_irq_valid())
- disable_irq_wake(acpi_sci_irq);
-
- if (lps0_device_handle) {
+ if (acpi_sci_irq_valid()) {
acpi_ec_set_gpe_wake_mask(ACPI_GPE_DISABLE);
-
- acpi_sleep_run_lps0_dsm(ACPI_LPS0_EXIT);
- acpi_sleep_run_lps0_dsm(ACPI_LPS0_SCREEN_ON);
+ disable_irq_wake(acpi_sci_irq);
}
}
static void acpi_s2idle_end(void)
{
- s2idle_in_progress = false;
acpi_scan_lock_release();
}
static const struct platform_s2idle_ops acpi_s2idle_ops = {
.begin = acpi_s2idle_begin,
.prepare = acpi_s2idle_prepare,
+ .prepare_late = acpi_s2idle_prepare_late,
.wake = acpi_s2idle_wake,
- .sync = acpi_s2idle_sync,
+ .restore_early = acpi_s2idle_restore_early,
.restore = acpi_s2idle_restore,
.end = acpi_s2idle_end,
};
@@ -1071,7 +1080,6 @@ static void acpi_sleep_suspend_setup(void)
}
#else /* !CONFIG_SUSPEND */
-#define s2idle_in_progress (false)
#define s2idle_wakeup (false)
#define lps0_device_handle (NULL)
static inline void acpi_sleep_suspend_setup(void) {}
@@ -1082,11 +1090,6 @@ bool acpi_s2idle_wakeup(void)
return s2idle_wakeup;
}
-bool acpi_sleep_no_ec_events(void)
-{
- return !s2idle_in_progress || !lps0_device_handle;
-}
-
#ifdef CONFIG_PM_SLEEP
static u32 saved_bm_rld;
diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c
index de974322a197..b32327759380 100644
--- a/drivers/acpi/tables.c
+++ b/drivers/acpi/tables.c
@@ -490,16 +490,17 @@ static u8 __init acpi_table_checksum(u8 *buffer, u32 length)
/* All but ACPI_SIG_RSDP and ACPI_SIG_FACS: */
static const char * const table_sigs[] = {
- ACPI_SIG_BERT, ACPI_SIG_CPEP, ACPI_SIG_ECDT, ACPI_SIG_EINJ,
- ACPI_SIG_ERST, ACPI_SIG_HEST, ACPI_SIG_MADT, ACPI_SIG_MSCT,
- ACPI_SIG_SBST, ACPI_SIG_SLIT, ACPI_SIG_SRAT, ACPI_SIG_ASF,
- ACPI_SIG_BOOT, ACPI_SIG_DBGP, ACPI_SIG_DMAR, ACPI_SIG_HPET,
- ACPI_SIG_IBFT, ACPI_SIG_IVRS, ACPI_SIG_MCFG, ACPI_SIG_MCHI,
- ACPI_SIG_SLIC, ACPI_SIG_SPCR, ACPI_SIG_SPMI, ACPI_SIG_TCPA,
- ACPI_SIG_UEFI, ACPI_SIG_WAET, ACPI_SIG_WDAT, ACPI_SIG_WDDT,
- ACPI_SIG_WDRT, ACPI_SIG_DSDT, ACPI_SIG_FADT, ACPI_SIG_PSDT,
- ACPI_SIG_RSDT, ACPI_SIG_XSDT, ACPI_SIG_SSDT, ACPI_SIG_IORT,
- ACPI_SIG_NFIT, ACPI_SIG_HMAT, ACPI_SIG_PPTT, NULL };
+ ACPI_SIG_BERT, ACPI_SIG_BGRT, ACPI_SIG_CPEP, ACPI_SIG_ECDT,
+ ACPI_SIG_EINJ, ACPI_SIG_ERST, ACPI_SIG_HEST, ACPI_SIG_MADT,
+ ACPI_SIG_MSCT, ACPI_SIG_SBST, ACPI_SIG_SLIT, ACPI_SIG_SRAT,
+ ACPI_SIG_ASF, ACPI_SIG_BOOT, ACPI_SIG_DBGP, ACPI_SIG_DMAR,
+ ACPI_SIG_HPET, ACPI_SIG_IBFT, ACPI_SIG_IVRS, ACPI_SIG_MCFG,
+ ACPI_SIG_MCHI, ACPI_SIG_SLIC, ACPI_SIG_SPCR, ACPI_SIG_SPMI,
+ ACPI_SIG_TCPA, ACPI_SIG_UEFI, ACPI_SIG_WAET, ACPI_SIG_WDAT,
+ ACPI_SIG_WDDT, ACPI_SIG_WDRT, ACPI_SIG_DSDT, ACPI_SIG_FADT,
+ ACPI_SIG_PSDT, ACPI_SIG_RSDT, ACPI_SIG_XSDT, ACPI_SIG_SSDT,
+ ACPI_SIG_IORT, ACPI_SIG_NFIT, ACPI_SIG_HMAT, ACPI_SIG_PPTT,
+ NULL };
#define ACPI_HEADER_SIZE sizeof(struct acpi_table_header)
diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c
index 00f12a86ecbd..d831a61e0010 100644
--- a/drivers/acpi/thermal.c
+++ b/drivers/acpi/thermal.c
@@ -225,13 +225,9 @@ static int acpi_thermal_set_cooling_mode(struct acpi_thermal *tz, int mode)
if (!tz)
return -EINVAL;
- if (!acpi_has_method(tz->device->handle, "_SCP")) {
- ACPI_DEBUG_PRINT((ACPI_DB_INFO, "_SCP not present\n"));
+ if (ACPI_FAILURE(acpi_execute_simple_method(tz->device->handle,
+ "_SCP", mode)))
return -ENODEV;
- } else if (ACPI_FAILURE(acpi_execute_simple_method(tz->device->handle,
- "_SCP", mode))) {
- return -ENODEV;
- }
return 0;
}
@@ -463,8 +459,7 @@ static int acpi_thermal_trips_update(struct acpi_thermal *tz, int flag)
break;
}
- if ((flag & ACPI_TRIPS_DEVICES)
- && acpi_has_method(tz->device->handle, "_TZD")) {
+ if (flag & ACPI_TRIPS_DEVICES) {
memset(&devices, 0, sizeof(devices));
status = acpi_evaluate_reference(tz->device->handle, "_TZD",
NULL, &devices);
diff --git a/drivers/acpi/utils.c b/drivers/acpi/utils.c
index 7def63ab00c0..e3974a8f8fd4 100644
--- a/drivers/acpi/utils.c
+++ b/drivers/acpi/utils.c
@@ -725,17 +725,15 @@ bool acpi_dev_found(const char *hid)
EXPORT_SYMBOL(acpi_dev_found);
struct acpi_dev_match_info {
- const char *dev_name;
- struct acpi_device *adev;
struct acpi_device_id hid[2];
const char *uid;
s64 hrv;
};
-static int acpi_dev_match_cb(struct device *dev, void *data)
+static int acpi_dev_match_cb(struct device *dev, const void *data)
{
struct acpi_device *adev = to_acpi_device(dev);
- struct acpi_dev_match_info *match = data;
+ const struct acpi_dev_match_info *match = data;
unsigned long long hrv;
acpi_status status;
@@ -746,9 +744,6 @@ static int acpi_dev_match_cb(struct device *dev, void *data)
strcmp(adev->pnp.unique_id, match->uid)))
return 0;
- match->dev_name = acpi_dev_name(adev);
- match->adev = adev;
-
if (match->hrv == -1)
return 1;
@@ -818,7 +813,7 @@ acpi_dev_get_first_match_dev(const char *hid, const char *uid, s64 hrv)
match.hrv = hrv;
dev = bus_find_device(&acpi_bus_type, NULL, &match, acpi_dev_match_cb);
- return dev ? match.adev : NULL;
+ return dev ? to_acpi_device(dev) : NULL;
}
EXPORT_SYMBOL(acpi_dev_get_first_match_dev);
diff --git a/drivers/acpi/x86/utils.c b/drivers/acpi/x86/utils.c
index ba277cd5c7fa..697a6b12d6b9 100644
--- a/drivers/acpi/x86/utils.c
+++ b/drivers/acpi/x86/utils.c
@@ -69,11 +69,11 @@ static const struct always_present_id always_present_ids[] = {
* after _SB.PCI0.GFX0.LCD.LCD1._ON gets called has passed
* *and* _STA has been called at least 3 times since.
*/
- ENTRY("SYNA7500", "1", ICPU(INTEL_FAM6_HASWELL_ULT), {
+ ENTRY("SYNA7500", "1", ICPU(INTEL_FAM6_HASWELL_L), {
DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
DMI_MATCH(DMI_PRODUCT_NAME, "Venue 11 Pro 7130"),
}),
- ENTRY("SYNA7500", "1", ICPU(INTEL_FAM6_HASWELL_ULT), {
+ ENTRY("SYNA7500", "1", ICPU(INTEL_FAM6_HASWELL_L), {
DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
DMI_MATCH(DMI_PRODUCT_NAME, "Venue 11 Pro 7139"),
}),
diff --git a/drivers/amba/tegra-ahb.c b/drivers/amba/tegra-ahb.c
index 3eaa459ae057..57d3b2e2d007 100644
--- a/drivers/amba/tegra-ahb.c
+++ b/drivers/amba/tegra-ahb.c
@@ -134,22 +134,13 @@ static inline void gizmo_writel(struct tegra_ahb *ahb, u32 value, u32 offset)
}
#ifdef CONFIG_TEGRA_IOMMU_SMMU
-static int tegra_ahb_match_by_smmu(struct device *dev, void *data)
-{
- struct tegra_ahb *ahb = dev_get_drvdata(dev);
- struct device_node *dn = data;
-
- return (ahb->dev->of_node == dn) ? 1 : 0;
-}
-
int tegra_ahb_enable_smmu(struct device_node *dn)
{
struct device *dev;
u32 val;
struct tegra_ahb *ahb;
- dev = driver_find_device(&tegra_ahb_driver.driver, NULL, dn,
- tegra_ahb_match_by_smmu);
+ dev = driver_find_device_by_of_node(&tegra_ahb_driver.driver, dn);
if (!dev)
return -EPROBE_DEFER;
ahb = dev_get_drvdata(dev);
diff --git a/drivers/android/binder.c b/drivers/android/binder.c
index bc26b5511f0a..c0a491277aca 100644
--- a/drivers/android/binder.c
+++ b/drivers/android/binder.c
@@ -122,7 +122,7 @@ static uint32_t binder_debug_mask = BINDER_DEBUG_USER_ERROR |
BINDER_DEBUG_FAILED_TRANSACTION | BINDER_DEBUG_DEAD_TRANSACTION;
module_param_named(debug_mask, binder_debug_mask, uint, 0644);
-static char *binder_devices_param = CONFIG_ANDROID_BINDER_DEVICES;
+char *binder_devices_param = CONFIG_ANDROID_BINDER_DEVICES;
module_param_named(devices, binder_devices_param, charp, 0444);
static DECLARE_WAIT_QUEUE_HEAD(binder_user_error_wait);
@@ -196,30 +196,8 @@ static inline void binder_stats_created(enum binder_stat_types type)
atomic_inc(&binder_stats.obj_created[type]);
}
-struct binder_transaction_log_entry {
- int debug_id;
- int debug_id_done;
- int call_type;
- int from_proc;
- int from_thread;
- int target_handle;
- int to_proc;
- int to_thread;
- int to_node;
- int data_size;
- int offsets_size;
- int return_error_line;
- uint32_t return_error;
- uint32_t return_error_param;
- const char *context_name;
-};
-struct binder_transaction_log {
- atomic_t cur;
- bool full;
- struct binder_transaction_log_entry entry[32];
-};
-static struct binder_transaction_log binder_transaction_log;
-static struct binder_transaction_log binder_transaction_log_failed;
+struct binder_transaction_log binder_transaction_log;
+struct binder_transaction_log binder_transaction_log_failed;
static struct binder_transaction_log_entry *binder_transaction_log_add(
struct binder_transaction_log *log)
@@ -480,6 +458,7 @@ enum binder_deferred_state {
* @inner_lock: can nest under outer_lock and/or node lock
* @outer_lock: no nesting under innor or node lock
* Lock order: 1) outer, 2) node, 3) inner
+ * @binderfs_entry: process-specific binderfs log file
*
* Bookkeeping structure for binder processes
*/
@@ -509,6 +488,7 @@ struct binder_proc {
struct binder_context *context;
spinlock_t inner_lock;
spinlock_t outer_lock;
+ struct dentry *binderfs_entry;
};
enum {
@@ -2059,10 +2039,9 @@ static size_t binder_get_object(struct binder_proc *proc,
read_size = min_t(size_t, sizeof(*object), buffer->data_size - offset);
if (offset > buffer->data_size || read_size < sizeof(*hdr) ||
- !IS_ALIGNED(offset, sizeof(u32)))
+ binder_alloc_copy_from_buffer(&proc->alloc, object, buffer,
+ offset, read_size))
return 0;
- binder_alloc_copy_from_buffer(&proc->alloc, object, buffer,
- offset, read_size);
/* Ok, now see if we read a complete object. */
hdr = &object->hdr;
@@ -2131,8 +2110,10 @@ static struct binder_buffer_object *binder_validate_ptr(
return NULL;
buffer_offset = start_offset + sizeof(binder_size_t) * index;
- binder_alloc_copy_from_buffer(&proc->alloc, &object_offset,
- b, buffer_offset, sizeof(object_offset));
+ if (binder_alloc_copy_from_buffer(&proc->alloc, &object_offset,
+ b, buffer_offset,
+ sizeof(object_offset)))
+ return NULL;
object_size = binder_get_object(proc, b, object_offset, object);
if (!object_size || object->hdr.type != BINDER_TYPE_PTR)
return NULL;
@@ -2212,10 +2193,12 @@ static bool binder_validate_fixup(struct binder_proc *proc,
return false;
last_min_offset = last_bbo->parent_offset + sizeof(uintptr_t);
buffer_offset = objects_start_offset +
- sizeof(binder_size_t) * last_bbo->parent,
- binder_alloc_copy_from_buffer(&proc->alloc, &last_obj_offset,
- b, buffer_offset,
- sizeof(last_obj_offset));
+ sizeof(binder_size_t) * last_bbo->parent;
+ if (binder_alloc_copy_from_buffer(&proc->alloc,
+ &last_obj_offset,
+ b, buffer_offset,
+ sizeof(last_obj_offset)))
+ return false;
}
return (fixup_offset >= last_min_offset);
}
@@ -2301,15 +2284,15 @@ static void binder_transaction_buffer_release(struct binder_proc *proc,
for (buffer_offset = off_start_offset; buffer_offset < off_end_offset;
buffer_offset += sizeof(binder_size_t)) {
struct binder_object_header *hdr;
- size_t object_size;
+ size_t object_size = 0;
struct binder_object object;
binder_size_t object_offset;
- binder_alloc_copy_from_buffer(&proc->alloc, &object_offset,
- buffer, buffer_offset,
- sizeof(object_offset));
- object_size = binder_get_object(proc, buffer,
- object_offset, &object);
+ if (!binder_alloc_copy_from_buffer(&proc->alloc, &object_offset,
+ buffer, buffer_offset,
+ sizeof(object_offset)))
+ object_size = binder_get_object(proc, buffer,
+ object_offset, &object);
if (object_size == 0) {
pr_err("transaction release %d bad object at offset %lld, size %zd\n",
debug_id, (u64)object_offset, buffer->data_size);
@@ -2432,15 +2415,16 @@ static void binder_transaction_buffer_release(struct binder_proc *proc,
for (fd_index = 0; fd_index < fda->num_fds;
fd_index++) {
u32 fd;
+ int err;
binder_size_t offset = fda_offset +
fd_index * sizeof(fd);
- binder_alloc_copy_from_buffer(&proc->alloc,
- &fd,
- buffer,
- offset,
- sizeof(fd));
- binder_deferred_fd_close(fd);
+ err = binder_alloc_copy_from_buffer(
+ &proc->alloc, &fd, buffer,
+ offset, sizeof(fd));
+ WARN_ON(err);
+ if (!err)
+ binder_deferred_fd_close(fd);
}
} break;
default:
@@ -2683,11 +2667,12 @@ static int binder_translate_fd_array(struct binder_fd_array_object *fda,
int ret;
binder_size_t offset = fda_offset + fdi * sizeof(fd);
- binder_alloc_copy_from_buffer(&target_proc->alloc,
- &fd, t->buffer,
- offset, sizeof(fd));
- ret = binder_translate_fd(fd, offset, t, thread,
- in_reply_to);
+ ret = binder_alloc_copy_from_buffer(&target_proc->alloc,
+ &fd, t->buffer,
+ offset, sizeof(fd));
+ if (!ret)
+ ret = binder_translate_fd(fd, offset, t, thread,
+ in_reply_to);
if (ret < 0)
return ret;
}
@@ -2740,8 +2725,12 @@ static int binder_fixup_parent(struct binder_transaction *t,
}
buffer_offset = bp->parent_offset +
(uintptr_t)parent->buffer - (uintptr_t)b->user_data;
- binder_alloc_copy_to_buffer(&target_proc->alloc, b, buffer_offset,
- &bp->buffer, sizeof(bp->buffer));
+ if (binder_alloc_copy_to_buffer(&target_proc->alloc, b, buffer_offset,
+ &bp->buffer, sizeof(bp->buffer))) {
+ binder_user_error("%d:%d got transaction with invalid parent offset\n",
+ proc->pid, thread->pid);
+ return -EINVAL;
+ }
return 0;
}
@@ -2979,7 +2968,7 @@ static void binder_transaction(struct binder_proc *proc,
else
return_error = BR_DEAD_REPLY;
mutex_unlock(&context->context_mgr_node_lock);
- if (target_node && target_proc == proc) {
+ if (target_node && target_proc->pid == proc->pid) {
binder_user_error("%d:%d got transaction to context manager from process owning it\n",
proc->pid, thread->pid);
return_error = BR_FAILED_REPLY;
@@ -3160,15 +3149,20 @@ static void binder_transaction(struct binder_proc *proc,
goto err_binder_alloc_buf_failed;
}
if (secctx) {
+ int err;
size_t buf_offset = ALIGN(tr->data_size, sizeof(void *)) +
ALIGN(tr->offsets_size, sizeof(void *)) +
ALIGN(extra_buffers_size, sizeof(void *)) -
ALIGN(secctx_sz, sizeof(u64));
t->security_ctx = (uintptr_t)t->buffer->user_data + buf_offset;
- binder_alloc_copy_to_buffer(&target_proc->alloc,
- t->buffer, buf_offset,
- secctx, secctx_sz);
+ err = binder_alloc_copy_to_buffer(&target_proc->alloc,
+ t->buffer, buf_offset,
+ secctx, secctx_sz);
+ if (err) {
+ t->security_ctx = 0;
+ WARN_ON(1);
+ }
security_release_secctx(secctx, secctx_sz);
secctx = NULL;
}
@@ -3225,7 +3219,8 @@ static void binder_transaction(struct binder_proc *proc,
buffer_offset = off_start_offset;
off_end_offset = off_start_offset + tr->offsets_size;
sg_buf_offset = ALIGN(off_end_offset, sizeof(void *));
- sg_buf_end_offset = sg_buf_offset + extra_buffers_size;
+ sg_buf_end_offset = sg_buf_offset + extra_buffers_size -
+ ALIGN(secctx_sz, sizeof(u64));
off_min = 0;
for (buffer_offset = off_start_offset; buffer_offset < off_end_offset;
buffer_offset += sizeof(binder_size_t)) {
@@ -3234,11 +3229,16 @@ static void binder_transaction(struct binder_proc *proc,
struct binder_object object;
binder_size_t object_offset;
- binder_alloc_copy_from_buffer(&target_proc->alloc,
- &object_offset,
- t->buffer,
- buffer_offset,
- sizeof(object_offset));
+ if (binder_alloc_copy_from_buffer(&target_proc->alloc,
+ &object_offset,
+ t->buffer,
+ buffer_offset,
+ sizeof(object_offset))) {
+ return_error = BR_FAILED_REPLY;
+ return_error_param = -EINVAL;
+ return_error_line = __LINE__;
+ goto err_bad_offset;
+ }
object_size = binder_get_object(target_proc, t->buffer,
object_offset, &object);
if (object_size == 0 || object_offset < off_min) {
@@ -3262,15 +3262,17 @@ static void binder_transaction(struct binder_proc *proc,
fp = to_flat_binder_object(hdr);
ret = binder_translate_binder(fp, t, thread);
- if (ret < 0) {
+
+ if (ret < 0 ||
+ binder_alloc_copy_to_buffer(&target_proc->alloc,
+ t->buffer,
+ object_offset,
+ fp, sizeof(*fp))) {
return_error = BR_FAILED_REPLY;
return_error_param = ret;
return_error_line = __LINE__;
goto err_translate_failed;
}
- binder_alloc_copy_to_buffer(&target_proc->alloc,
- t->buffer, object_offset,
- fp, sizeof(*fp));
} break;
case BINDER_TYPE_HANDLE:
case BINDER_TYPE_WEAK_HANDLE: {
@@ -3278,15 +3280,16 @@ static void binder_transaction(struct binder_proc *proc,
fp = to_flat_binder_object(hdr);
ret = binder_translate_handle(fp, t, thread);
- if (ret < 0) {
+ if (ret < 0 ||
+ binder_alloc_copy_to_buffer(&target_proc->alloc,
+ t->buffer,
+ object_offset,
+ fp, sizeof(*fp))) {
return_error = BR_FAILED_REPLY;
return_error_param = ret;
return_error_line = __LINE__;
goto err_translate_failed;
}
- binder_alloc_copy_to_buffer(&target_proc->alloc,
- t->buffer, object_offset,
- fp, sizeof(*fp));
} break;
case BINDER_TYPE_FD: {
@@ -3296,16 +3299,17 @@ static void binder_transaction(struct binder_proc *proc,
int ret = binder_translate_fd(fp->fd, fd_offset, t,
thread, in_reply_to);
- if (ret < 0) {
+ fp->pad_binder = 0;
+ if (ret < 0 ||
+ binder_alloc_copy_to_buffer(&target_proc->alloc,
+ t->buffer,
+ object_offset,
+ fp, sizeof(*fp))) {
return_error = BR_FAILED_REPLY;
return_error_param = ret;
return_error_line = __LINE__;
goto err_translate_failed;
}
- fp->pad_binder = 0;
- binder_alloc_copy_to_buffer(&target_proc->alloc,
- t->buffer, object_offset,
- fp, sizeof(*fp));
} break;
case BINDER_TYPE_FDA: {
struct binder_object ptr_object;
@@ -3393,15 +3397,16 @@ static void binder_transaction(struct binder_proc *proc,
num_valid,
last_fixup_obj_off,
last_fixup_min_off);
- if (ret < 0) {
+ if (ret < 0 ||
+ binder_alloc_copy_to_buffer(&target_proc->alloc,
+ t->buffer,
+ object_offset,
+ bp, sizeof(*bp))) {
return_error = BR_FAILED_REPLY;
return_error_param = ret;
return_error_line = __LINE__;
goto err_translate_failed;
}
- binder_alloc_copy_to_buffer(&target_proc->alloc,
- t->buffer, object_offset,
- bp, sizeof(*bp));
last_fixup_obj_off = object_offset;
last_fixup_min_off = 0;
} break;
@@ -4140,20 +4145,27 @@ static int binder_apply_fd_fixups(struct binder_proc *proc,
trace_binder_transaction_fd_recv(t, fd, fixup->offset);
fd_install(fd, fixup->file);
fixup->file = NULL;
- binder_alloc_copy_to_buffer(&proc->alloc, t->buffer,
- fixup->offset, &fd,
- sizeof(u32));
+ if (binder_alloc_copy_to_buffer(&proc->alloc, t->buffer,
+ fixup->offset, &fd,
+ sizeof(u32))) {
+ ret = -EINVAL;
+ break;
+ }
}
list_for_each_entry_safe(fixup, tmp, &t->fd_fixups, fixup_entry) {
if (fixup->file) {
fput(fixup->file);
} else if (ret) {
u32 fd;
-
- binder_alloc_copy_from_buffer(&proc->alloc, &fd,
- t->buffer, fixup->offset,
- sizeof(fd));
- binder_deferred_fd_close(fd);
+ int err;
+
+ err = binder_alloc_copy_from_buffer(&proc->alloc, &fd,
+ t->buffer,
+ fixup->offset,
+ sizeof(fd));
+ WARN_ON(err);
+ if (!err)
+ binder_deferred_fd_close(fd);
}
list_del(&fixup->fixup_entry);
kfree(fixup);
@@ -4268,6 +4280,8 @@ retry:
case BINDER_WORK_TRANSACTION_COMPLETE: {
binder_inner_proc_unlock(proc);
cmd = BR_TRANSACTION_COMPLETE;
+ kfree(w);
+ binder_stats_deleted(BINDER_STAT_TRANSACTION_COMPLETE);
if (put_user(cmd, (uint32_t __user *)ptr))
return -EFAULT;
ptr += sizeof(uint32_t);
@@ -4276,8 +4290,6 @@ retry:
binder_debug(BINDER_DEBUG_TRANSACTION_COMPLETE,
"%d:%d BR_TRANSACTION_COMPLETE\n",
proc->pid, thread->pid);
- kfree(w);
- binder_stats_deleted(BINDER_STAT_TRANSACTION_COMPLETE);
} break;
case BINDER_WORK_NODE: {
struct binder_node *node = container_of(w, struct binder_node, work);
@@ -5198,6 +5210,8 @@ static int binder_open(struct inode *nodp, struct file *filp)
{
struct binder_proc *proc;
struct binder_device *binder_dev;
+ struct binderfs_info *info;
+ struct dentry *binder_binderfs_dir_entry_proc = NULL;
binder_debug(BINDER_DEBUG_OPEN_CLOSE, "%s: %d:%d\n", __func__,
current->group_leader->pid, current->pid);
@@ -5212,11 +5226,14 @@ static int binder_open(struct inode *nodp, struct file *filp)
INIT_LIST_HEAD(&proc->todo);
proc->default_priority = task_nice(current);
/* binderfs stashes devices in i_private */
- if (is_binderfs_device(nodp))
+ if (is_binderfs_device(nodp)) {
binder_dev = nodp->i_private;
- else
+ info = nodp->i_sb->s_fs_info;
+ binder_binderfs_dir_entry_proc = info->proc_log_dir;
+ } else {
binder_dev = container_of(filp->private_data,
struct binder_device, miscdev);
+ }
proc->context = &binder_dev->context;
binder_alloc_init(&proc->alloc);
@@ -5247,6 +5264,35 @@ static int binder_open(struct inode *nodp, struct file *filp)
&proc_fops);
}
+ if (binder_binderfs_dir_entry_proc) {
+ char strbuf[11];
+ struct dentry *binderfs_entry;
+
+ snprintf(strbuf, sizeof(strbuf), "%u", proc->pid);
+ /*
+ * Similar to debugfs, the process specific log file is shared
+ * between contexts. If the file has already been created for a
+ * process, the following binderfs_create_file() call will
+ * fail with error code EEXIST if another context of the same
+ * process invoked binder_open(). This is ok since same as
+ * debugfs, the log file will contain information on all
+ * contexts of a given PID.
+ */
+ binderfs_entry = binderfs_create_file(binder_binderfs_dir_entry_proc,
+ strbuf, &proc_fops, (void *)(unsigned long)proc->pid);
+ if (!IS_ERR(binderfs_entry)) {
+ proc->binderfs_entry = binderfs_entry;
+ } else {
+ int error;
+
+ error = PTR_ERR(binderfs_entry);
+ if (error != -EEXIST) {
+ pr_warn("Unable to create file %s in binderfs (error %d)\n",
+ strbuf, error);
+ }
+ }
+ }
+
return 0;
}
@@ -5286,6 +5332,12 @@ static int binder_release(struct inode *nodp, struct file *filp)
struct binder_proc *proc = filp->private_data;
debugfs_remove(proc->debugfs_entry);
+
+ if (proc->binderfs_entry) {
+ binderfs_remove_file(proc->binderfs_entry);
+ proc->binderfs_entry = NULL;
+ }
+
binder_defer_work(proc, BINDER_DEFERRED_RELEASE);
return 0;
@@ -5875,7 +5927,7 @@ static void print_binder_proc_stats(struct seq_file *m,
}
-static int state_show(struct seq_file *m, void *unused)
+int binder_state_show(struct seq_file *m, void *unused)
{
struct binder_proc *proc;
struct binder_node *node;
@@ -5914,7 +5966,7 @@ static int state_show(struct seq_file *m, void *unused)
return 0;
}
-static int stats_show(struct seq_file *m, void *unused)
+int binder_stats_show(struct seq_file *m, void *unused)
{
struct binder_proc *proc;
@@ -5930,7 +5982,7 @@ static int stats_show(struct seq_file *m, void *unused)
return 0;
}
-static int transactions_show(struct seq_file *m, void *unused)
+int binder_transactions_show(struct seq_file *m, void *unused)
{
struct binder_proc *proc;
@@ -5986,7 +6038,7 @@ static void print_binder_transaction_log_entry(struct seq_file *m,
"\n" : " (incomplete)\n");
}
-static int transaction_log_show(struct seq_file *m, void *unused)
+int binder_transaction_log_show(struct seq_file *m, void *unused)
{
struct binder_transaction_log *log = m->private;
unsigned int log_cur = atomic_read(&log->cur);
@@ -6018,11 +6070,6 @@ const struct file_operations binder_fops = {
.release = binder_release,
};
-DEFINE_SHOW_ATTRIBUTE(state);
-DEFINE_SHOW_ATTRIBUTE(stats);
-DEFINE_SHOW_ATTRIBUTE(transactions);
-DEFINE_SHOW_ATTRIBUTE(transaction_log);
-
static int __init init_binder_device(const char *name)
{
int ret;
@@ -6076,30 +6123,31 @@ static int __init binder_init(void)
0444,
binder_debugfs_dir_entry_root,
NULL,
- &state_fops);
+ &binder_state_fops);
debugfs_create_file("stats",
0444,
binder_debugfs_dir_entry_root,
NULL,
- &stats_fops);
+ &binder_stats_fops);
debugfs_create_file("transactions",
0444,
binder_debugfs_dir_entry_root,
NULL,
- &transactions_fops);
+ &binder_transactions_fops);
debugfs_create_file("transaction_log",
0444,
binder_debugfs_dir_entry_root,
&binder_transaction_log,
- &transaction_log_fops);
+ &binder_transaction_log_fops);
debugfs_create_file("failed_transaction_log",
0444,
binder_debugfs_dir_entry_root,
&binder_transaction_log_failed,
- &transaction_log_fops);
+ &binder_transaction_log_fops);
}
- if (strcmp(binder_devices_param, "") != 0) {
+ if (!IS_ENABLED(CONFIG_ANDROID_BINDERFS) &&
+ strcmp(binder_devices_param, "") != 0) {
/*
* Copy the module_parameter string, because we don't want to
* tokenize it in-place.
diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c
index ce5603c2291c..6d79a1b0d446 100644
--- a/drivers/android/binder_alloc.c
+++ b/drivers/android/binder_alloc.c
@@ -1119,15 +1119,16 @@ binder_alloc_copy_user_to_buffer(struct binder_alloc *alloc,
return 0;
}
-static void binder_alloc_do_buffer_copy(struct binder_alloc *alloc,
- bool to_buffer,
- struct binder_buffer *buffer,
- binder_size_t buffer_offset,
- void *ptr,
- size_t bytes)
+static int binder_alloc_do_buffer_copy(struct binder_alloc *alloc,
+ bool to_buffer,
+ struct binder_buffer *buffer,
+ binder_size_t buffer_offset,
+ void *ptr,
+ size_t bytes)
{
/* All copies must be 32-bit aligned and 32-bit size */
- BUG_ON(!check_buffer(alloc, buffer, buffer_offset, bytes));
+ if (!check_buffer(alloc, buffer, buffer_offset, bytes))
+ return -EINVAL;
while (bytes) {
unsigned long size;
@@ -1155,25 +1156,26 @@ static void binder_alloc_do_buffer_copy(struct binder_alloc *alloc,
ptr = ptr + size;
buffer_offset += size;
}
+ return 0;
}
-void binder_alloc_copy_to_buffer(struct binder_alloc *alloc,
- struct binder_buffer *buffer,
- binder_size_t buffer_offset,
- void *src,
- size_t bytes)
+int binder_alloc_copy_to_buffer(struct binder_alloc *alloc,
+ struct binder_buffer *buffer,
+ binder_size_t buffer_offset,
+ void *src,
+ size_t bytes)
{
- binder_alloc_do_buffer_copy(alloc, true, buffer, buffer_offset,
- src, bytes);
+ return binder_alloc_do_buffer_copy(alloc, true, buffer, buffer_offset,
+ src, bytes);
}
-void binder_alloc_copy_from_buffer(struct binder_alloc *alloc,
- void *dest,
- struct binder_buffer *buffer,
- binder_size_t buffer_offset,
- size_t bytes)
+int binder_alloc_copy_from_buffer(struct binder_alloc *alloc,
+ void *dest,
+ struct binder_buffer *buffer,
+ binder_size_t buffer_offset,
+ size_t bytes)
{
- binder_alloc_do_buffer_copy(alloc, false, buffer, buffer_offset,
- dest, bytes);
+ return binder_alloc_do_buffer_copy(alloc, false, buffer, buffer_offset,
+ dest, bytes);
}
diff --git a/drivers/android/binder_alloc.h b/drivers/android/binder_alloc.h
index 71bfa95f8e09..db9c1b984695 100644
--- a/drivers/android/binder_alloc.h
+++ b/drivers/android/binder_alloc.h
@@ -159,17 +159,17 @@ binder_alloc_copy_user_to_buffer(struct binder_alloc *alloc,
const void __user *from,
size_t bytes);
-void binder_alloc_copy_to_buffer(struct binder_alloc *alloc,
- struct binder_buffer *buffer,
- binder_size_t buffer_offset,
- void *src,
- size_t bytes);
-
-void binder_alloc_copy_from_buffer(struct binder_alloc *alloc,
- void *dest,
- struct binder_buffer *buffer,
- binder_size_t buffer_offset,
- size_t bytes);
+int binder_alloc_copy_to_buffer(struct binder_alloc *alloc,
+ struct binder_buffer *buffer,
+ binder_size_t buffer_offset,
+ void *src,
+ size_t bytes);
+
+int binder_alloc_copy_from_buffer(struct binder_alloc *alloc,
+ void *dest,
+ struct binder_buffer *buffer,
+ binder_size_t buffer_offset,
+ size_t bytes);
#endif /* _LINUX_BINDER_ALLOC_H */
diff --git a/drivers/android/binder_internal.h b/drivers/android/binder_internal.h
index 045b3e42d98b..bd47f7f72075 100644
--- a/drivers/android/binder_internal.h
+++ b/drivers/android/binder_internal.h
@@ -35,15 +35,63 @@ struct binder_device {
struct inode *binderfs_inode;
};
+/**
+ * binderfs_mount_opts - mount options for binderfs
+ * @max: maximum number of allocatable binderfs binder devices
+ * @stats_mode: enable binder stats in binderfs.
+ */
+struct binderfs_mount_opts {
+ int max;
+ int stats_mode;
+};
+
+/**
+ * binderfs_info - information about a binderfs mount
+ * @ipc_ns: The ipc namespace the binderfs mount belongs to.
+ * @control_dentry: This records the dentry of this binderfs mount
+ * binder-control device.
+ * @root_uid: uid that needs to be used when a new binder device is
+ * created.
+ * @root_gid: gid that needs to be used when a new binder device is
+ * created.
+ * @mount_opts: The mount options in use.
+ * @device_count: The current number of allocated binder devices.
+ * @proc_log_dir: Pointer to the directory dentry containing process-specific
+ * logs.
+ */
+struct binderfs_info {
+ struct ipc_namespace *ipc_ns;
+ struct dentry *control_dentry;
+ kuid_t root_uid;
+ kgid_t root_gid;
+ struct binderfs_mount_opts mount_opts;
+ int device_count;
+ struct dentry *proc_log_dir;
+};
+
extern const struct file_operations binder_fops;
+extern char *binder_devices_param;
+
#ifdef CONFIG_ANDROID_BINDERFS
extern bool is_binderfs_device(const struct inode *inode);
+extern struct dentry *binderfs_create_file(struct dentry *dir, const char *name,
+ const struct file_operations *fops,
+ void *data);
+extern void binderfs_remove_file(struct dentry *dentry);
#else
static inline bool is_binderfs_device(const struct inode *inode)
{
return false;
}
+static inline struct dentry *binderfs_create_file(struct dentry *dir,
+ const char *name,
+ const struct file_operations *fops,
+ void *data)
+{
+ return NULL;
+}
+static inline void binderfs_remove_file(struct dentry *dentry) {}
#endif
#ifdef CONFIG_ANDROID_BINDERFS
@@ -55,4 +103,42 @@ static inline int __init init_binderfs(void)
}
#endif
+int binder_stats_show(struct seq_file *m, void *unused);
+DEFINE_SHOW_ATTRIBUTE(binder_stats);
+
+int binder_state_show(struct seq_file *m, void *unused);
+DEFINE_SHOW_ATTRIBUTE(binder_state);
+
+int binder_transactions_show(struct seq_file *m, void *unused);
+DEFINE_SHOW_ATTRIBUTE(binder_transactions);
+
+int binder_transaction_log_show(struct seq_file *m, void *unused);
+DEFINE_SHOW_ATTRIBUTE(binder_transaction_log);
+
+struct binder_transaction_log_entry {
+ int debug_id;
+ int debug_id_done;
+ int call_type;
+ int from_proc;
+ int from_thread;
+ int target_handle;
+ int to_proc;
+ int to_thread;
+ int to_node;
+ int data_size;
+ int offsets_size;
+ int return_error_line;
+ uint32_t return_error;
+ uint32_t return_error_param;
+ const char *context_name;
+};
+
+struct binder_transaction_log {
+ atomic_t cur;
+ bool full;
+ struct binder_transaction_log_entry entry[32];
+};
+
+extern struct binder_transaction_log binder_transaction_log;
+extern struct binder_transaction_log binder_transaction_log_failed;
#endif /* _LINUX_BINDER_INTERNAL_H */
diff --git a/drivers/android/binderfs.c b/drivers/android/binderfs.c
index e773f45d19d9..e2580e5316a2 100644
--- a/drivers/android/binderfs.c
+++ b/drivers/android/binderfs.c
@@ -48,45 +48,23 @@ static dev_t binderfs_dev;
static DEFINE_MUTEX(binderfs_minors_mutex);
static DEFINE_IDA(binderfs_minors);
-/**
- * binderfs_mount_opts - mount options for binderfs
- * @max: maximum number of allocatable binderfs binder devices
- */
-struct binderfs_mount_opts {
- int max;
-};
-
enum {
Opt_max,
+ Opt_stats_mode,
Opt_err
};
+enum binderfs_stats_mode {
+ STATS_NONE,
+ STATS_GLOBAL,
+};
+
static const match_table_t tokens = {
{ Opt_max, "max=%d" },
+ { Opt_stats_mode, "stats=%s" },
{ Opt_err, NULL }
};
-/**
- * binderfs_info - information about a binderfs mount
- * @ipc_ns: The ipc namespace the binderfs mount belongs to.
- * @control_dentry: This records the dentry of this binderfs mount
- * binder-control device.
- * @root_uid: uid that needs to be used when a new binder device is
- * created.
- * @root_gid: gid that needs to be used when a new binder device is
- * created.
- * @mount_opts: The mount options in use.
- * @device_count: The current number of allocated binder devices.
- */
-struct binderfs_info {
- struct ipc_namespace *ipc_ns;
- struct dentry *control_dentry;
- kuid_t root_uid;
- kgid_t root_gid;
- struct binderfs_mount_opts mount_opts;
- int device_count;
-};
-
static inline struct binderfs_info *BINDERFS_I(const struct inode *inode)
{
return inode->i_sb->s_fs_info;
@@ -186,8 +164,7 @@ static int binderfs_binder_device_create(struct inode *ref_inode,
req->major = MAJOR(binderfs_dev);
req->minor = minor;
- ret = copy_to_user(userp, req, sizeof(*req));
- if (ret) {
+ if (userp && copy_to_user(userp, req, sizeof(*req))) {
ret = -EFAULT;
goto err;
}
@@ -272,7 +249,7 @@ static void binderfs_evict_inode(struct inode *inode)
clear_inode(inode);
- if (!device)
+ if (!S_ISCHR(inode->i_mode) || !device)
return;
mutex_lock(&binderfs_minors_mutex);
@@ -291,8 +268,9 @@ static void binderfs_evict_inode(struct inode *inode)
static int binderfs_parse_mount_opts(char *data,
struct binderfs_mount_opts *opts)
{
- char *p;
+ char *p, *stats;
opts->max = BINDERFS_MAX_MINOR;
+ opts->stats_mode = STATS_NONE;
while ((p = strsep(&data, ",")) != NULL) {
substring_t args[MAX_OPT_ARGS];
@@ -312,6 +290,22 @@ static int binderfs_parse_mount_opts(char *data,
opts->max = max_devices;
break;
+ case Opt_stats_mode:
+ if (!capable(CAP_SYS_ADMIN))
+ return -EINVAL;
+
+ stats = match_strdup(&args[0]);
+ if (!stats)
+ return -ENOMEM;
+
+ if (strcmp(stats, "global") != 0) {
+ kfree(stats);
+ return -EINVAL;
+ }
+
+ opts->stats_mode = STATS_GLOBAL;
+ kfree(stats);
+ break;
default:
pr_err("Invalid mount options\n");
return -EINVAL;
@@ -323,8 +317,21 @@ static int binderfs_parse_mount_opts(char *data,
static int binderfs_remount(struct super_block *sb, int *flags, char *data)
{
+ int prev_stats_mode, ret;
struct binderfs_info *info = sb->s_fs_info;
- return binderfs_parse_mount_opts(data, &info->mount_opts);
+
+ prev_stats_mode = info->mount_opts.stats_mode;
+ ret = binderfs_parse_mount_opts(data, &info->mount_opts);
+ if (ret)
+ return ret;
+
+ if (prev_stats_mode != info->mount_opts.stats_mode) {
+ pr_err("Binderfs stats mode cannot be changed during a remount\n");
+ info->mount_opts.stats_mode = prev_stats_mode;
+ return -EINVAL;
+ }
+
+ return 0;
}
static int binderfs_show_mount_opts(struct seq_file *seq, struct dentry *root)
@@ -334,6 +341,8 @@ static int binderfs_show_mount_opts(struct seq_file *seq, struct dentry *root)
info = root->d_sb->s_fs_info;
if (info->mount_opts.max <= BINDERFS_MAX_MINOR)
seq_printf(seq, ",max=%d", info->mount_opts.max);
+ if (info->mount_opts.stats_mode == STATS_GLOBAL)
+ seq_printf(seq, ",stats=global");
return 0;
}
@@ -462,11 +471,192 @@ static const struct inode_operations binderfs_dir_inode_operations = {
.unlink = binderfs_unlink,
};
+static struct inode *binderfs_make_inode(struct super_block *sb, int mode)
+{
+ struct inode *ret;
+
+ ret = new_inode(sb);
+ if (ret) {
+ ret->i_ino = iunique(sb, BINDERFS_MAX_MINOR + INODE_OFFSET);
+ ret->i_mode = mode;
+ ret->i_atime = ret->i_mtime = ret->i_ctime = current_time(ret);
+ }
+ return ret;
+}
+
+static struct dentry *binderfs_create_dentry(struct dentry *parent,
+ const char *name)
+{
+ struct dentry *dentry;
+
+ dentry = lookup_one_len(name, parent, strlen(name));
+ if (IS_ERR(dentry))
+ return dentry;
+
+ /* Return error if the file/dir already exists. */
+ if (d_really_is_positive(dentry)) {
+ dput(dentry);
+ return ERR_PTR(-EEXIST);
+ }
+
+ return dentry;
+}
+
+void binderfs_remove_file(struct dentry *dentry)
+{
+ struct inode *parent_inode;
+
+ parent_inode = d_inode(dentry->d_parent);
+ inode_lock(parent_inode);
+ if (simple_positive(dentry)) {
+ dget(dentry);
+ simple_unlink(parent_inode, dentry);
+ d_delete(dentry);
+ dput(dentry);
+ }
+ inode_unlock(parent_inode);
+}
+
+struct dentry *binderfs_create_file(struct dentry *parent, const char *name,
+ const struct file_operations *fops,
+ void *data)
+{
+ struct dentry *dentry;
+ struct inode *new_inode, *parent_inode;
+ struct super_block *sb;
+
+ parent_inode = d_inode(parent);
+ inode_lock(parent_inode);
+
+ dentry = binderfs_create_dentry(parent, name);
+ if (IS_ERR(dentry))
+ goto out;
+
+ sb = parent_inode->i_sb;
+ new_inode = binderfs_make_inode(sb, S_IFREG | 0444);
+ if (!new_inode) {
+ dput(dentry);
+ dentry = ERR_PTR(-ENOMEM);
+ goto out;
+ }
+
+ new_inode->i_fop = fops;
+ new_inode->i_private = data;
+ d_instantiate(dentry, new_inode);
+ fsnotify_create(parent_inode, dentry);
+
+out:
+ inode_unlock(parent_inode);
+ return dentry;
+}
+
+static struct dentry *binderfs_create_dir(struct dentry *parent,
+ const char *name)
+{
+ struct dentry *dentry;
+ struct inode *new_inode, *parent_inode;
+ struct super_block *sb;
+
+ parent_inode = d_inode(parent);
+ inode_lock(parent_inode);
+
+ dentry = binderfs_create_dentry(parent, name);
+ if (IS_ERR(dentry))
+ goto out;
+
+ sb = parent_inode->i_sb;
+ new_inode = binderfs_make_inode(sb, S_IFDIR | 0755);
+ if (!new_inode) {
+ dput(dentry);
+ dentry = ERR_PTR(-ENOMEM);
+ goto out;
+ }
+
+ new_inode->i_fop = &simple_dir_operations;
+ new_inode->i_op = &simple_dir_inode_operations;
+
+ set_nlink(new_inode, 2);
+ d_instantiate(dentry, new_inode);
+ inc_nlink(parent_inode);
+ fsnotify_mkdir(parent_inode, dentry);
+
+out:
+ inode_unlock(parent_inode);
+ return dentry;
+}
+
+static int init_binder_logs(struct super_block *sb)
+{
+ struct dentry *binder_logs_root_dir, *dentry, *proc_log_dir;
+ struct binderfs_info *info;
+ int ret = 0;
+
+ binder_logs_root_dir = binderfs_create_dir(sb->s_root,
+ "binder_logs");
+ if (IS_ERR(binder_logs_root_dir)) {
+ ret = PTR_ERR(binder_logs_root_dir);
+ goto out;
+ }
+
+ dentry = binderfs_create_file(binder_logs_root_dir, "stats",
+ &binder_stats_fops, NULL);
+ if (IS_ERR(dentry)) {
+ ret = PTR_ERR(dentry);
+ goto out;
+ }
+
+ dentry = binderfs_create_file(binder_logs_root_dir, "state",
+ &binder_state_fops, NULL);
+ if (IS_ERR(dentry)) {
+ ret = PTR_ERR(dentry);
+ goto out;
+ }
+
+ dentry = binderfs_create_file(binder_logs_root_dir, "transactions",
+ &binder_transactions_fops, NULL);
+ if (IS_ERR(dentry)) {
+ ret = PTR_ERR(dentry);
+ goto out;
+ }
+
+ dentry = binderfs_create_file(binder_logs_root_dir,
+ "transaction_log",
+ &binder_transaction_log_fops,
+ &binder_transaction_log);
+ if (IS_ERR(dentry)) {
+ ret = PTR_ERR(dentry);
+ goto out;
+ }
+
+ dentry = binderfs_create_file(binder_logs_root_dir,
+ "failed_transaction_log",
+ &binder_transaction_log_fops,
+ &binder_transaction_log_failed);
+ if (IS_ERR(dentry)) {
+ ret = PTR_ERR(dentry);
+ goto out;
+ }
+
+ proc_log_dir = binderfs_create_dir(binder_logs_root_dir, "proc");
+ if (IS_ERR(proc_log_dir)) {
+ ret = PTR_ERR(proc_log_dir);
+ goto out;
+ }
+ info = sb->s_fs_info;
+ info->proc_log_dir = proc_log_dir;
+
+out:
+ return ret;
+}
+
static int binderfs_fill_super(struct super_block *sb, void *data, int silent)
{
int ret;
struct binderfs_info *info;
struct inode *inode = NULL;
+ struct binderfs_device device_info = { 0 };
+ const char *name;
+ size_t len;
sb->s_blocksize = PAGE_SIZE;
sb->s_blocksize_bits = PAGE_SHIFT;
@@ -521,7 +711,25 @@ static int binderfs_fill_super(struct super_block *sb, void *data, int silent)
if (!sb->s_root)
return -ENOMEM;
- return binderfs_binder_ctl_create(sb);
+ ret = binderfs_binder_ctl_create(sb);
+ if (ret)
+ return ret;
+
+ name = binder_devices_param;
+ for (len = strcspn(name, ","); len > 0; len = strcspn(name, ",")) {
+ strscpy(device_info.name, name, len + 1);
+ ret = binderfs_binder_device_create(inode, NULL, &device_info);
+ if (ret)
+ return ret;
+ name += len;
+ if (*name == ',')
+ name++;
+ }
+
+ if (info->mount_opts.stats_mode == STATS_GLOBAL)
+ return init_binder_logs(sb);
+
+ return 0;
}
static struct dentry *binderfs_mount(struct file_system_type *fs_type,
@@ -553,6 +761,18 @@ static struct file_system_type binder_fs_type = {
int __init init_binderfs(void)
{
int ret;
+ const char *name;
+ size_t len;
+
+ /* Verify that the default binderfs device names are valid. */
+ name = binder_devices_param;
+ for (len = strcspn(name, ","); len > 0; len = strcspn(name, ",")) {
+ if (len > BINDERFS_MAX_NAME)
+ return -E2BIG;
+ name += len;
+ if (*name == ',')
+ name++;
+ }
/* Allocate new major number for binderfs. */
ret = alloc_chrdev_region(&binderfs_dev, 0, BINDERFS_MAX_MINOR,
diff --git a/drivers/ata/acard-ahci.c b/drivers/ata/acard-ahci.c
index b1b49dbd0b14..753985c01517 100644
--- a/drivers/ata/acard-ahci.c
+++ b/drivers/ata/acard-ahci.c
@@ -160,37 +160,6 @@ static int acard_ahci_pci_device_resume(struct pci_dev *pdev)
}
#endif
-static int acard_ahci_configure_dma_masks(struct pci_dev *pdev, int using_dac)
-{
- int rc;
-
- if (using_dac &&
- !dma_set_mask(&pdev->dev, DMA_BIT_MASK(64))) {
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
- if (rc) {
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev,
- "64-bit DMA enable failed\n");
- return rc;
- }
- }
- } else {
- rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev, "32-bit DMA enable failed\n");
- return rc;
- }
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev,
- "32-bit consistent DMA enable failed\n");
- return rc;
- }
- }
- return 0;
-}
-
static void acard_ahci_pci_print_info(struct ata_host *host)
{
struct pci_dev *pdev = to_pci_dev(host->dev);
@@ -344,7 +313,6 @@ static int acard_ahci_port_start(struct ata_port *ap)
mem = dmam_alloc_coherent(dev, dma_sz, &mem_dma, GFP_KERNEL);
if (!mem)
return -ENOMEM;
- memset(mem, 0, dma_sz);
/*
* First item in chunk of DMA memory: 32-slot command table,
@@ -472,9 +440,12 @@ static int acard_ahci_init_one(struct pci_dev *pdev, const struct pci_device_id
}
/* initialize adapter */
- rc = acard_ahci_configure_dma_masks(pdev, hpriv->cap & HOST_CAP_64);
- if (rc)
+ rc = dma_set_mask_and_coherent(&pdev->dev,
+ DMA_BIT_MASK((hpriv->cap & HOST_CAP_64) ? 64 : 32));
+ if (rc) {
+ dev_err(&pdev->dev, "DMA enable failed\n");
return rc;
+ }
rc = ahci_reset_controller(host);
if (rc)
diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c
index f7652baa6337..dd92faf197d5 100644
--- a/drivers/ata/ahci.c
+++ b/drivers/ata/ahci.c
@@ -65,6 +65,12 @@ enum board_ids {
board_ahci_sb700, /* for SB700 and SB800 */
board_ahci_vt8251,
+ /*
+ * board IDs for Intel chipsets that support more than 6 ports
+ * *and* end up needing the PCS quirk.
+ */
+ board_ahci_pcs7,
+
/* aliases */
board_ahci_mcp_linux = board_ahci_mcp65,
board_ahci_mcp67 = board_ahci_mcp65,
@@ -220,6 +226,12 @@ static const struct ata_port_info ahci_port_info[] = {
.udma_mask = ATA_UDMA6,
.port_ops = &ahci_vt8251_ops,
},
+ [board_ahci_pcs7] = {
+ .flags = AHCI_FLAG_COMMON,
+ .pio_mask = ATA_PIO4,
+ .udma_mask = ATA_UDMA6,
+ .port_ops = &ahci_ops,
+ },
};
static const struct pci_device_id ahci_pci_tbl[] = {
@@ -264,26 +276,26 @@ static const struct pci_device_id ahci_pci_tbl[] = {
{ PCI_VDEVICE(INTEL, 0x3b2b), board_ahci }, /* PCH RAID */
{ PCI_VDEVICE(INTEL, 0x3b2c), board_ahci_mobile }, /* PCH M RAID */
{ PCI_VDEVICE(INTEL, 0x3b2f), board_ahci }, /* PCH AHCI */
- { PCI_VDEVICE(INTEL, 0x19b0), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19b1), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19b2), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19b3), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19b4), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19b5), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19b6), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19b7), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19bE), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19bF), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19c0), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19c1), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19c2), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19c3), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19c4), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19c5), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19c6), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19c7), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19cE), board_ahci }, /* DNV AHCI */
- { PCI_VDEVICE(INTEL, 0x19cF), board_ahci }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19b0), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19b1), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19b2), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19b3), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19b4), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19b5), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19b6), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19b7), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19bE), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19bF), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19c0), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19c1), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19c2), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19c3), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19c4), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19c5), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19c6), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19c7), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19cE), board_ahci_pcs7 }, /* DNV AHCI */
+ { PCI_VDEVICE(INTEL, 0x19cF), board_ahci_pcs7 }, /* DNV AHCI */
{ PCI_VDEVICE(INTEL, 0x1c02), board_ahci }, /* CPT AHCI */
{ PCI_VDEVICE(INTEL, 0x1c03), board_ahci_mobile }, /* CPT M AHCI */
{ PCI_VDEVICE(INTEL, 0x1c04), board_ahci }, /* CPT RAID */
@@ -623,30 +635,6 @@ static void ahci_pci_save_initial_config(struct pci_dev *pdev,
ahci_save_initial_config(&pdev->dev, hpriv);
}
-static int ahci_pci_reset_controller(struct ata_host *host)
-{
- struct pci_dev *pdev = to_pci_dev(host->dev);
- int rc;
-
- rc = ahci_reset_controller(host);
- if (rc)
- return rc;
-
- if (pdev->vendor == PCI_VENDOR_ID_INTEL) {
- struct ahci_host_priv *hpriv = host->private_data;
- u16 tmp16;
-
- /* configure PCS */
- pci_read_config_word(pdev, 0x92, &tmp16);
- if ((tmp16 & hpriv->port_map) != hpriv->port_map) {
- tmp16 |= hpriv->port_map;
- pci_write_config_word(pdev, 0x92, tmp16);
- }
- }
-
- return 0;
-}
-
static void ahci_pci_init_controller(struct ata_host *host)
{
struct ahci_host_priv *hpriv = host->private_data;
@@ -849,7 +837,7 @@ static int ahci_pci_device_runtime_resume(struct device *dev)
struct ata_host *host = pci_get_drvdata(pdev);
int rc;
- rc = ahci_pci_reset_controller(host);
+ rc = ahci_reset_controller(host);
if (rc)
return rc;
ahci_pci_init_controller(host);
@@ -884,7 +872,7 @@ static int ahci_pci_device_resume(struct device *dev)
ahci_mcp89_apple_enable(pdev);
if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
- rc = ahci_pci_reset_controller(host);
+ rc = ahci_reset_controller(host);
if (rc)
return rc;
@@ -901,40 +889,23 @@ static int ahci_pci_device_resume(struct device *dev)
static int ahci_configure_dma_masks(struct pci_dev *pdev, int using_dac)
{
+ const int dma_bits = using_dac ? 64 : 32;
int rc;
/*
* If the device fixup already set the dma_mask to some non-standard
* value, don't extend it here. This happens on STA2X11, for example.
+ *
+ * XXX: manipulating the DMA mask from platform code is completely
+ * bogus, platform code should use dev->bus_dma_mask instead..
*/
if (pdev->dma_mask && pdev->dma_mask < DMA_BIT_MASK(32))
return 0;
- if (using_dac &&
- !dma_set_mask(&pdev->dev, DMA_BIT_MASK(64))) {
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
- if (rc) {
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev,
- "64-bit DMA enable failed\n");
- return rc;
- }
- }
- } else {
- rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev, "32-bit DMA enable failed\n");
- return rc;
- }
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev,
- "32-bit consistent DMA enable failed\n");
- return rc;
- }
- }
- return 0;
+ rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(dma_bits));
+ if (rc)
+ dev_err(&pdev->dev, "DMA enable failed\n");
+ return rc;
}
static void ahci_pci_print_info(struct ata_host *host)
@@ -1619,6 +1590,34 @@ update_policy:
ap->target_lpm_policy = policy;
}
+static void ahci_intel_pcs_quirk(struct pci_dev *pdev, struct ahci_host_priv *hpriv)
+{
+ const struct pci_device_id *id = pci_match_id(ahci_pci_tbl, pdev);
+ u16 tmp16;
+
+ /*
+ * Only apply the 6-port PCS quirk for known legacy platforms.
+ */
+ if (!id || id->vendor != PCI_VENDOR_ID_INTEL)
+ return;
+ if (((enum board_ids) id->driver_data) < board_ahci_pcs7)
+ return;
+
+ /*
+ * port_map is determined from PORTS_IMPL PCI register which is
+ * implemented as write or write-once register. If the register
+ * isn't programmed, ahci automatically generates it from number
+ * of ports, which is good enough for PCS programming. It is
+ * otherwise expected that platform firmware enables the ports
+ * before the OS boots.
+ */
+ pci_read_config_word(pdev, PCS_6, &tmp16);
+ if ((tmp16 & hpriv->port_map) != hpriv->port_map) {
+ tmp16 |= hpriv->port_map;
+ pci_write_config_word(pdev, PCS_6, tmp16);
+ }
+}
+
static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
{
unsigned int board_id = ent->driver_data;
@@ -1731,6 +1730,12 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
/* save initial config */
ahci_pci_save_initial_config(pdev, hpriv);
+ /*
+ * If platform firmware failed to enable ports, try to enable
+ * them here.
+ */
+ ahci_intel_pcs_quirk(pdev, hpriv);
+
/* prepare host */
if (hpriv->cap & HOST_CAP_NCQ) {
pi.flags |= ATA_FLAG_NCQ;
@@ -1840,7 +1845,7 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
if (rc)
return rc;
- rc = ahci_pci_reset_controller(host);
+ rc = ahci_reset_controller(host);
if (rc)
return rc;
diff --git a/drivers/ata/ahci.h b/drivers/ata/ahci.h
index 0570629d719d..3dbf398c92ea 100644
--- a/drivers/ata/ahci.h
+++ b/drivers/ata/ahci.h
@@ -247,6 +247,8 @@ enum {
ATA_FLAG_ACPI_SATA | ATA_FLAG_AN,
ICH_MAP = 0x90, /* ICH MAP register */
+ PCS_6 = 0x92, /* 6 port PCS */
+ PCS_7 = 0x94, /* 7+ port PCS (Denverton) */
/* em constants */
EM_MAX_SLOTS = 8,
diff --git a/drivers/ata/ahci_sunxi.c b/drivers/ata/ahci_sunxi.c
index 4100e904376b..cb69b737cb49 100644
--- a/drivers/ata/ahci_sunxi.c
+++ b/drivers/ata/ahci_sunxi.c
@@ -149,8 +149,51 @@ static void ahci_sunxi_start_engine(struct ata_port *ap)
void __iomem *port_mmio = ahci_port_base(ap);
struct ahci_host_priv *hpriv = ap->host->private_data;
- /* Setup DMA before DMA start */
- sunxi_clrsetbits(hpriv->mmio + AHCI_P0DMACR, 0x0000ff00, 0x00004400);
+ /* Setup DMA before DMA start
+ *
+ * NOTE: A similar SoC with SATA/AHCI by Texas Instruments documents
+ * this Vendor Specific Port (P0DMACR, aka PxDMACR) in its
+ * User's Guide document (TMS320C674x/OMAP-L1x Processor
+ * Serial ATA (SATA) Controller, Literature Number: SPRUGJ8C,
+ * March 2011, Chapter 4.33 Port DMA Control Register (P0DMACR),
+ * p.68, https://www.ti.com/lit/ug/sprugj8c/sprugj8c.pdf)
+ * as equivalent to the following struct:
+ *
+ * struct AHCI_P0DMACR_t
+ * {
+ * unsigned TXTS : 4;
+ * unsigned RXTS : 4;
+ * unsigned TXABL : 4;
+ * unsigned RXABL : 4;
+ * unsigned Reserved : 16;
+ * };
+ *
+ * TXTS: Transmit Transaction Size (TX_TRANSACTION_SIZE).
+ * This field defines the DMA transaction size in DWORDs for
+ * transmit (system bus read, device write) operation. [...]
+ *
+ * RXTS: Receive Transaction Size (RX_TRANSACTION_SIZE).
+ * This field defines the Port DMA transaction size in DWORDs
+ * for receive (system bus write, device read) operation. [...]
+ *
+ * TXABL: Transmit Burst Limit.
+ * This field allows software to limit the VBUSP master read
+ * burst size. [...]
+ *
+ * RXABL: Receive Burst Limit.
+ * Allows software to limit the VBUSP master write burst
+ * size. [...]
+ *
+ * Reserved: Reserved.
+ *
+ *
+ * NOTE: According to the above document, the following alternative
+ * to the code below could perhaps be a better option
+ * (or preparation) for possible further improvements later:
+ * sunxi_clrsetbits(hpriv->mmio + AHCI_P0DMACR, 0x0000ffff,
+ * 0x00000033);
+ */
+ sunxi_clrsetbits(hpriv->mmio + AHCI_P0DMACR, 0x0000ffff, 0x00004433);
/* Start DMA */
sunxi_setbits(port_mmio + PORT_CMD, PORT_CMD_START);
diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c
index 0984c4b76d7e..bff369d9a1a7 100644
--- a/drivers/ata/libahci.c
+++ b/drivers/ata/libahci.c
@@ -175,7 +175,6 @@ struct ata_port_operations ahci_pmp_retry_srst_ops = {
EXPORT_SYMBOL_GPL(ahci_pmp_retry_srst_ops);
static bool ahci_em_messages __read_mostly = true;
-EXPORT_SYMBOL_GPL(ahci_em_messages);
module_param(ahci_em_messages, bool, 0444);
/* add other LED protocol types when they become supported */
MODULE_PARM_DESC(ahci_em_messages,
@@ -2365,7 +2364,6 @@ static int ahci_port_start(struct ata_port *ap)
mem = dmam_alloc_coherent(dev, dma_sz, &mem_dma, GFP_KERNEL);
if (!mem)
return -ENOMEM;
- memset(mem, 0, dma_sz);
/*
* First item in chunk of DMA memory: 32-slot command table,
diff --git a/drivers/ata/libahci_platform.c b/drivers/ata/libahci_platform.c
index 72312ad2e142..9e9583a6bba9 100644
--- a/drivers/ata/libahci_platform.c
+++ b/drivers/ata/libahci_platform.c
@@ -338,6 +338,9 @@ static int ahci_platform_get_phy(struct ahci_host_priv *hpriv, u32 port,
hpriv->phys[port] = NULL;
rc = 0;
break;
+ case -EPROBE_DEFER:
+ /* Do not complain yet */
+ break;
default:
dev_err(dev,
@@ -408,7 +411,6 @@ struct ahci_host_priv *ahci_platform_get_resources(struct platform_device *pdev,
hpriv->mmio = devm_ioremap_resource(dev,
platform_get_resource(pdev, IORESOURCE_MEM, 0));
if (IS_ERR(hpriv->mmio)) {
- dev_err(dev, "no mmio space\n");
rc = PTR_ERR(hpriv->mmio);
goto err_out;
}
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
index 4a2dff303865..28c492be0a57 100644
--- a/drivers/ata/libata-core.c
+++ b/drivers/ata/libata-core.c
@@ -4462,9 +4462,7 @@ static const struct ata_blacklist_entry ata_device_blacklist [] = {
/* drives which fail FPDMA_AA activation (some may freeze afterwards)
the ST disks also have LPM issues */
- { "ST1000LM024 HN-M101MBB", "2AR10001", ATA_HORKAGE_BROKEN_FPDMA_AA |
- ATA_HORKAGE_NOLPM, },
- { "ST1000LM024 HN-M101MBB", "2BA30001", ATA_HORKAGE_BROKEN_FPDMA_AA |
+ { "ST1000LM024 HN-M101MBB", NULL, ATA_HORKAGE_BROKEN_FPDMA_AA |
ATA_HORKAGE_NOLPM, },
{ "VB0250EAVER", "HPG7", ATA_HORKAGE_BROKEN_FPDMA_AA },
diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c
index 9d687e1d4325..3bfd9da58473 100644
--- a/drivers/ata/libata-eh.c
+++ b/drivers/ata/libata-eh.c
@@ -1469,7 +1469,7 @@ static int ata_eh_read_log_10h(struct ata_device *dev,
tf->hob_lbah = buf[10];
tf->nsect = buf[12];
tf->hob_nsect = buf[13];
- if (ata_id_has_ncq_autosense(dev->id))
+ if (dev->class == ATA_DEV_ZAC && ata_id_has_ncq_autosense(dev->id))
tf->auxiliary = buf[14] << 16 | buf[15] << 8 | buf[16];
return 0;
@@ -1716,7 +1716,8 @@ void ata_eh_analyze_ncq_error(struct ata_link *link)
memcpy(&qc->result_tf, &tf, sizeof(tf));
qc->result_tf.flags = ATA_TFLAG_ISADDR | ATA_TFLAG_LBA | ATA_TFLAG_LBA48;
qc->err_mask |= AC_ERR_DEV | AC_ERR_NCQ;
- if ((qc->result_tf.command & ATA_SENSE) || qc->result_tf.auxiliary) {
+ if (dev->class == ATA_DEV_ZAC &&
+ ((qc->result_tf.command & ATA_SENSE) || qc->result_tf.auxiliary)) {
char sense_key, asc, ascq;
sense_key = (qc->result_tf.auxiliary >> 16) & 0xff;
@@ -1770,10 +1771,11 @@ static unsigned int ata_eh_analyze_tf(struct ata_queued_cmd *qc,
}
switch (qc->dev->class) {
- case ATA_DEV_ATA:
case ATA_DEV_ZAC:
if (stat & ATA_SENSE)
ata_eh_request_sense(qc, qc->scsicmd);
+ /* fall through */
+ case ATA_DEV_ATA:
if (err & ATA_ICRC)
qc->err_mask |= AC_ERR_ATA_BUS;
if (err & (ATA_UNC | ATA_AMNF))
diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c
index 391ac0503dc0..76d0f9de767b 100644
--- a/drivers/ata/libata-scsi.c
+++ b/drivers/ata/libata-scsi.c
@@ -1786,6 +1786,21 @@ nothing_to_do:
return 1;
}
+static bool ata_check_nblocks(struct scsi_cmnd *scmd, u32 n_blocks)
+{
+ struct request *rq = scmd->request;
+ u32 req_blocks;
+
+ if (!blk_rq_is_passthrough(rq))
+ return true;
+
+ req_blocks = blk_rq_bytes(rq) / scmd->device->sector_size;
+ if (n_blocks > req_blocks)
+ return false;
+
+ return true;
+}
+
/**
* ata_scsi_rw_xlat - Translate SCSI r/w command into an ATA one
* @qc: Storage for translated ATA taskfile
@@ -1830,6 +1845,8 @@ static unsigned int ata_scsi_rw_xlat(struct ata_queued_cmd *qc)
scsi_10_lba_len(cdb, &block, &n_block);
if (cdb[1] & (1 << 3))
tf_flags |= ATA_TFLAG_FUA;
+ if (!ata_check_nblocks(scmd, n_block))
+ goto invalid_fld;
break;
case READ_6:
case WRITE_6:
@@ -1844,6 +1861,8 @@ static unsigned int ata_scsi_rw_xlat(struct ata_queued_cmd *qc)
*/
if (!n_block)
n_block = 256;
+ if (!ata_check_nblocks(scmd, n_block))
+ goto invalid_fld;
break;
case READ_16:
case WRITE_16:
@@ -1854,6 +1873,8 @@ static unsigned int ata_scsi_rw_xlat(struct ata_queued_cmd *qc)
scsi_16_lba_len(cdb, &block, &n_block);
if (cdb[1] & (1 << 3))
tf_flags |= ATA_TFLAG_FUA;
+ if (!ata_check_nblocks(scmd, n_block))
+ goto invalid_fld;
break;
default:
DPRINTK("no-byte command\n");
diff --git a/drivers/ata/libata-sff.c b/drivers/ata/libata-sff.c
index 10aa27882142..4ed682da52ae 100644
--- a/drivers/ata/libata-sff.c
+++ b/drivers/ata/libata-sff.c
@@ -658,6 +658,10 @@ static void ata_pio_sector(struct ata_queued_cmd *qc)
unsigned int offset;
unsigned char *buf;
+ if (!qc->cursg) {
+ qc->curbytes = qc->nbytes;
+ return;
+ }
if (qc->curbytes == qc->nbytes - qc->sect_size)
ap->hsm_task_state = HSM_ST_LAST;
@@ -683,6 +687,8 @@ static void ata_pio_sector(struct ata_queued_cmd *qc)
if (qc->cursg_ofs == qc->cursg->length) {
qc->cursg = sg_next(qc->cursg);
+ if (!qc->cursg)
+ ap->hsm_task_state = HSM_ST_LAST;
qc->cursg_ofs = 0;
}
}
@@ -3147,15 +3153,9 @@ void ata_pci_bmdma_init(struct ata_host *host)
* ->sff_irq_clear method. Try to initialize bmdma_addr
* regardless of dma masks.
*/
- rc = dma_set_mask(&pdev->dev, ATA_DMA_MASK);
+ rc = dma_set_mask_and_coherent(&pdev->dev, ATA_DMA_MASK);
if (rc)
ata_bmdma_nodma(host, "failed to set dma mask");
- if (!rc) {
- rc = dma_set_coherent_mask(&pdev->dev, ATA_DMA_MASK);
- if (rc)
- ata_bmdma_nodma(host,
- "failed to set consistent dma mask");
- }
/* request and iomap DMA region */
rc = pcim_iomap_regions(pdev, 1 << 4, dev_driver_string(gdev));
diff --git a/drivers/ata/libata-zpodd.c b/drivers/ata/libata-zpodd.c
index 173e6f2dd9af..eefda51f97d3 100644
--- a/drivers/ata/libata-zpodd.c
+++ b/drivers/ata/libata-zpodd.c
@@ -56,7 +56,7 @@ static enum odd_mech_type zpodd_get_mech_type(struct ata_device *dev)
unsigned int ret;
struct rm_feature_desc *desc;
struct ata_taskfile tf;
- static const char cdb[] = { GPCMD_GET_CONFIGURATION,
+ static const char cdb[ATAPI_CDB_LEN] = { GPCMD_GET_CONFIGURATION,
2, /* only 1 feature descriptor requested */
0, 3, /* 3, removable medium feature */
0, 0, 0,/* reserved */
diff --git a/drivers/ata/pata_atp867x.c b/drivers/ata/pata_atp867x.c
index 2b9ed4ddef8d..cfd0cf2cbca6 100644
--- a/drivers/ata/pata_atp867x.c
+++ b/drivers/ata/pata_atp867x.c
@@ -463,12 +463,7 @@ static int atp867x_ata_pci_sff_init_host(struct ata_host *host)
atp867x_fixup(host);
- rc = dma_set_mask(&pdev->dev, ATA_DMA_MASK);
- if (rc)
- return rc;
-
- rc = dma_set_coherent_mask(&pdev->dev, ATA_DMA_MASK);
- return rc;
+ return dma_set_mask_and_coherent(&pdev->dev, ATA_DMA_MASK);
}
static int atp867x_init_one(struct pci_dev *pdev,
diff --git a/drivers/ata/pata_buddha.c b/drivers/ata/pata_buddha.c
index 11a8044ff633..27d4c417fc60 100644
--- a/drivers/ata/pata_buddha.c
+++ b/drivers/ata/pata_buddha.c
@@ -18,7 +18,9 @@
#include <linux/kernel.h>
#include <linux/libata.h>
#include <linux/mm.h>
+#include <linux/mod_devicetable.h>
#include <linux/module.h>
+#include <linux/types.h>
#include <linux/zorro.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_host.h>
@@ -29,7 +31,7 @@
#include <asm/setup.h>
#define DRV_NAME "pata_buddha"
-#define DRV_VERSION "0.1.0"
+#define DRV_VERSION "0.1.1"
#define BUDDHA_BASE1 0x800
#define BUDDHA_BASE2 0xa00
@@ -47,11 +49,11 @@ enum {
BOARD_XSURF
};
-static unsigned int buddha_bases[3] __initdata = {
+static unsigned int buddha_bases[3] = {
BUDDHA_BASE1, BUDDHA_BASE2, BUDDHA_BASE3
};
-static unsigned int xsurf_bases[2] __initdata = {
+static unsigned int xsurf_bases[2] = {
XSURF_BASE1, XSURF_BASE2
};
@@ -145,111 +147,151 @@ static struct ata_port_operations pata_xsurf_ops = {
.set_mode = pata_buddha_set_mode,
};
-static int __init pata_buddha_init_one(void)
+static int pata_buddha_probe(struct zorro_dev *z,
+ const struct zorro_device_id *ent)
{
- struct zorro_dev *z = NULL;
+ static const char * const board_name[] = {
+ "Buddha", "Catweasel", "X-Surf"
+ };
+ struct ata_host *host;
+ void __iomem *buddha_board;
+ unsigned long board;
+ unsigned int type = ent->driver_data;
+ unsigned int nr_ports = (type == BOARD_CATWEASEL) ? 3 : 2;
+ void *old_drvdata;
+ int i;
+
+ dev_info(&z->dev, "%s IDE controller\n", board_name[type]);
+
+ board = z->resource.start;
+
+ if (type != BOARD_XSURF) {
+ if (!devm_request_mem_region(&z->dev,
+ board + BUDDHA_BASE1,
+ 0x800, DRV_NAME))
+ return -ENXIO;
+ } else {
+ if (!devm_request_mem_region(&z->dev,
+ board + XSURF_BASE1,
+ 0x1000, DRV_NAME))
+ return -ENXIO;
+ if (!devm_request_mem_region(&z->dev,
+ board + XSURF_BASE2,
+ 0x1000, DRV_NAME)) {
+ }
+ }
+
+ /* Workaround for X-Surf: Save drvdata in case zorro8390 has set it */
+ if (type == BOARD_XSURF)
+ old_drvdata = dev_get_drvdata(&z->dev);
+
+ /* allocate host */
+ host = ata_host_alloc(&z->dev, nr_ports);
+ if (type == BOARD_XSURF)
+ dev_set_drvdata(&z->dev, old_drvdata);
+ if (!host)
+ return -ENXIO;
+
+ buddha_board = ZTWO_VADDR(board);
+
+ /* enable the board IRQ on Buddha/Catweasel */
+ if (type != BOARD_XSURF)
+ z_writeb(0, buddha_board + BUDDHA_IRQ_MR);
- while ((z = zorro_find_device(ZORRO_WILDCARD, z))) {
- static const char *board_name[]
- = { "Buddha", "Catweasel", "X-Surf" };
- struct ata_host *host;
- void __iomem *buddha_board;
- unsigned long board;
- unsigned int type, nr_ports = 2;
- int i;
-
- if (z->id == ZORRO_PROD_INDIVIDUAL_COMPUTERS_BUDDHA) {
- type = BOARD_BUDDHA;
- } else if (z->id == ZORRO_PROD_INDIVIDUAL_COMPUTERS_CATWEASEL) {
- type = BOARD_CATWEASEL;
- nr_ports++;
- } else if (z->id == ZORRO_PROD_INDIVIDUAL_COMPUTERS_X_SURF) {
- type = BOARD_XSURF;
- } else
- continue;
-
- dev_info(&z->dev, "%s IDE controller\n", board_name[type]);
-
- board = z->resource.start;
+ for (i = 0; i < nr_ports; i++) {
+ struct ata_port *ap = host->ports[i];
+ void __iomem *base, *irqport;
+ unsigned long ctl = 0;
if (type != BOARD_XSURF) {
- if (!devm_request_mem_region(&z->dev,
- board + BUDDHA_BASE1,
- 0x800, DRV_NAME))
- continue;
+ ap->ops = &pata_buddha_ops;
+ base = buddha_board + buddha_bases[i];
+ ctl = BUDDHA_CONTROL;
+ irqport = buddha_board + BUDDHA_IRQ + i * 0x40;
} else {
- if (!devm_request_mem_region(&z->dev,
- board + XSURF_BASE1,
- 0x1000, DRV_NAME))
- continue;
- if (!devm_request_mem_region(&z->dev,
- board + XSURF_BASE2,
- 0x1000, DRV_NAME))
- continue;
+ ap->ops = &pata_xsurf_ops;
+ base = buddha_board + xsurf_bases[i];
+ /* X-Surf has no CS1* (Control/AltStat) */
+ irqport = buddha_board + XSURF_IRQ;
}
- /* allocate host */
- host = ata_host_alloc(&z->dev, nr_ports);
- if (!host)
- continue;
-
- buddha_board = ZTWO_VADDR(board);
-
- /* enable the board IRQ on Buddha/Catweasel */
- if (type != BOARD_XSURF)
- z_writeb(0, buddha_board + BUDDHA_IRQ_MR);
-
- for (i = 0; i < nr_ports; i++) {
- struct ata_port *ap = host->ports[i];
- void __iomem *base, *irqport;
- unsigned long ctl = 0;
-
- if (type != BOARD_XSURF) {
- ap->ops = &pata_buddha_ops;
- base = buddha_board + buddha_bases[i];
- ctl = BUDDHA_CONTROL;
- irqport = buddha_board + BUDDHA_IRQ + i * 0x40;
- } else {
- ap->ops = &pata_xsurf_ops;
- base = buddha_board + xsurf_bases[i];
- /* X-Surf has no CS1* (Control/AltStat) */
- irqport = buddha_board + XSURF_IRQ;
- }
-
- ap->pio_mask = ATA_PIO4;
- ap->flags |= ATA_FLAG_SLAVE_POSS | ATA_FLAG_NO_IORDY;
-
- ap->ioaddr.data_addr = base;
- ap->ioaddr.error_addr = base + 2 + 1 * 4;
- ap->ioaddr.feature_addr = base + 2 + 1 * 4;
- ap->ioaddr.nsect_addr = base + 2 + 2 * 4;
- ap->ioaddr.lbal_addr = base + 2 + 3 * 4;
- ap->ioaddr.lbam_addr = base + 2 + 4 * 4;
- ap->ioaddr.lbah_addr = base + 2 + 5 * 4;
- ap->ioaddr.device_addr = base + 2 + 6 * 4;
- ap->ioaddr.status_addr = base + 2 + 7 * 4;
- ap->ioaddr.command_addr = base + 2 + 7 * 4;
-
- if (ctl) {
- ap->ioaddr.altstatus_addr = base + ctl;
- ap->ioaddr.ctl_addr = base + ctl;
- }
-
- ap->private_data = (void *)irqport;
-
- ata_port_desc(ap, "cmd 0x%lx ctl 0x%lx", board,
- ctl ? board + buddha_bases[i] + ctl : 0);
+ ap->pio_mask = ATA_PIO4;
+ ap->flags |= ATA_FLAG_SLAVE_POSS | ATA_FLAG_NO_IORDY;
+
+ ap->ioaddr.data_addr = base;
+ ap->ioaddr.error_addr = base + 2 + 1 * 4;
+ ap->ioaddr.feature_addr = base + 2 + 1 * 4;
+ ap->ioaddr.nsect_addr = base + 2 + 2 * 4;
+ ap->ioaddr.lbal_addr = base + 2 + 3 * 4;
+ ap->ioaddr.lbam_addr = base + 2 + 4 * 4;
+ ap->ioaddr.lbah_addr = base + 2 + 5 * 4;
+ ap->ioaddr.device_addr = base + 2 + 6 * 4;
+ ap->ioaddr.status_addr = base + 2 + 7 * 4;
+ ap->ioaddr.command_addr = base + 2 + 7 * 4;
+
+ if (ctl) {
+ ap->ioaddr.altstatus_addr = base + ctl;
+ ap->ioaddr.ctl_addr = base + ctl;
}
- ata_host_activate(host, IRQ_AMIGA_PORTS, ata_sff_interrupt,
- IRQF_SHARED, &pata_buddha_sht);
+ ap->private_data = (void *)irqport;
+ ata_port_desc(ap, "cmd 0x%lx ctl 0x%lx", board,
+ ctl ? board + buddha_bases[i] + ctl : 0);
}
+ ata_host_activate(host, IRQ_AMIGA_PORTS, ata_sff_interrupt,
+ IRQF_SHARED, &pata_buddha_sht);
+
return 0;
}
-module_init(pata_buddha_init_one);
+static void pata_buddha_remove(struct zorro_dev *z)
+{
+ struct ata_host *host = dev_get_drvdata(&z->dev);
+
+ ata_host_detach(host);
+}
+
+static const struct zorro_device_id pata_buddha_zorro_tbl[] = {
+ { ZORRO_PROD_INDIVIDUAL_COMPUTERS_BUDDHA, BOARD_BUDDHA},
+ { ZORRO_PROD_INDIVIDUAL_COMPUTERS_CATWEASEL, BOARD_CATWEASEL},
+ { 0 }
+};
+MODULE_DEVICE_TABLE(zorro, pata_buddha_zorro_tbl);
+
+static struct zorro_driver pata_buddha_driver = {
+ .name = "pata_buddha",
+ .id_table = pata_buddha_zorro_tbl,
+ .probe = pata_buddha_probe,
+ .remove = pata_buddha_remove,
+};
+
+/*
+ * We cannot have a modalias for X-Surf boards, as it competes with the
+ * zorro8390 network driver. As a stopgap measure until we have proper
+ * MFD support for this board, we manually attach to it late after Zorro
+ * has enumerated its boards.
+ */
+static int __init pata_buddha_late_init(void)
+{
+ struct zorro_dev *z = NULL;
+
+ /* Auto-bind to regular boards */
+ zorro_register_driver(&pata_buddha_driver);
+
+ /* Manually bind to all X-Surf boards */
+ while ((z = zorro_find_device(ZORRO_PROD_INDIVIDUAL_COMPUTERS_X_SURF, z))) {
+ static struct zorro_device_id xsurf_ent = {
+ ZORRO_PROD_INDIVIDUAL_COMPUTERS_X_SURF, BOARD_XSURF
+ };
+
+ pata_buddha_probe(z, &xsurf_ent);
+ }
+
+ return 0;
+}
+late_initcall(pata_buddha_late_init);
MODULE_AUTHOR("Bartlomiej Zolnierkiewicz");
MODULE_DESCRIPTION("low-level driver for Buddha/Catweasel/X-Surf PATA");
diff --git a/drivers/ata/pata_cs5520.c b/drivers/ata/pata_cs5520.c
index 099a5c68a4c9..9052148b306d 100644
--- a/drivers/ata/pata_cs5520.c
+++ b/drivers/ata/pata_cs5520.c
@@ -155,14 +155,10 @@ static int cs5520_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
return -ENODEV;
}
- if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(32))) {
+ if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) {
printk(KERN_ERR DRV_NAME ": unable to configure DMA mask.\n");
return -ENODEV;
}
- if (dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32))) {
- printk(KERN_ERR DRV_NAME ": unable to configure consistent DMA mask.\n");
- return -ENODEV;
- }
/* Map IO ports and initialize host accordingly */
iomap[0] = devm_ioport_map(&pdev->dev, cmd_port[0], 8);
diff --git a/drivers/ata/pata_hpt3x3.c b/drivers/ata/pata_hpt3x3.c
index b2fc023783b1..83974d5eb387 100644
--- a/drivers/ata/pata_hpt3x3.c
+++ b/drivers/ata/pata_hpt3x3.c
@@ -221,10 +221,7 @@ static int hpt3x3_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
if (rc)
return rc;
host->iomap = pcim_iomap_table(pdev);
- rc = dma_set_mask(&pdev->dev, ATA_DMA_MASK);
- if (rc)
- return rc;
- rc = dma_set_coherent_mask(&pdev->dev, ATA_DMA_MASK);
+ rc = dma_set_mask_and_coherent(&pdev->dev, ATA_DMA_MASK);
if (rc)
return rc;
diff --git a/drivers/ata/pata_ninja32.c b/drivers/ata/pata_ninja32.c
index 607db1f05f9a..f9255d6fd194 100644
--- a/drivers/ata/pata_ninja32.c
+++ b/drivers/ata/pata_ninja32.c
@@ -123,10 +123,7 @@ static int ninja32_init_one(struct pci_dev *dev, const struct pci_device_id *id)
return rc;
host->iomap = pcim_iomap_table(dev);
- rc = dma_set_mask(&dev->dev, ATA_DMA_MASK);
- if (rc)
- return rc;
- rc = dma_set_coherent_mask(&dev->dev, ATA_DMA_MASK);
+ rc = dma_set_mask_and_coherent(&dev->dev, ATA_DMA_MASK);
if (rc)
return rc;
pci_set_master(dev);
diff --git a/drivers/ata/pata_pdc2027x.c b/drivers/ata/pata_pdc2027x.c
index b656e1536855..de834fbb6dfe 100644
--- a/drivers/ata/pata_pdc2027x.c
+++ b/drivers/ata/pata_pdc2027x.c
@@ -722,11 +722,7 @@ static int pdc2027x_init_one(struct pci_dev *pdev,
return rc;
host->iomap = pcim_iomap_table(pdev);
- rc = dma_set_mask(&pdev->dev, ATA_DMA_MASK);
- if (rc)
- return rc;
-
- rc = dma_set_coherent_mask(&pdev->dev, ATA_DMA_MASK);
+ rc = dma_set_mask_and_coherent(&pdev->dev, ATA_DMA_MASK);
if (rc)
return rc;
diff --git a/drivers/ata/pata_rb532_cf.c b/drivers/ata/pata_rb532_cf.c
index 7c37f2ff09e4..deae466395de 100644
--- a/drivers/ata/pata_rb532_cf.c
+++ b/drivers/ata/pata_rb532_cf.c
@@ -158,7 +158,6 @@ static int rb532_pata_driver_probe(struct platform_device *pdev)
static int rb532_pata_driver_remove(struct platform_device *pdev)
{
struct ata_host *ah = platform_get_drvdata(pdev);
- struct rb532_cf_info *info = ah->private_data;
ata_host_detach(ah);
diff --git a/drivers/ata/pata_sil680.c b/drivers/ata/pata_sil680.c
index c14071be4f55..7ab9aea3b630 100644
--- a/drivers/ata/pata_sil680.c
+++ b/drivers/ata/pata_sil680.c
@@ -374,10 +374,7 @@ static int sil680_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
host->iomap = pcim_iomap_table(pdev);
/* Setup DMA masks */
- rc = dma_set_mask(&pdev->dev, ATA_DMA_MASK);
- if (rc)
- return rc;
- rc = dma_set_coherent_mask(&pdev->dev, ATA_DMA_MASK);
+ rc = dma_set_mask_and_coherent(&pdev->dev, ATA_DMA_MASK);
if (rc)
return rc;
pci_set_master(pdev);
diff --git a/drivers/ata/pdc_adma.c b/drivers/ata/pdc_adma.c
index 52fa8606a25f..cb490531b62e 100644
--- a/drivers/ata/pdc_adma.c
+++ b/drivers/ata/pdc_adma.c
@@ -550,7 +550,6 @@ static int adma_port_start(struct ata_port *ap)
(u32)pp->pkt_dma);
return -ENOMEM;
}
- memset(pp->pkt, 0, ADMA_PKT_BYTES);
ap->private_data = pp;
adma_reinit_engine(ap);
return 0;
@@ -573,23 +572,6 @@ static void adma_host_init(struct ata_host *host, unsigned int chip_id)
adma_reset_engine(host->ports[port_no]);
}
-static int adma_set_dma_masks(struct pci_dev *pdev, void __iomem *mmio_base)
-{
- int rc;
-
- rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev, "32-bit DMA enable failed\n");
- return rc;
- }
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev, "32-bit consistent DMA enable failed\n");
- return rc;
- }
- return 0;
-}
-
static int adma_ata_init_one(struct pci_dev *pdev,
const struct pci_device_id *ent)
{
@@ -620,9 +602,11 @@ static int adma_ata_init_one(struct pci_dev *pdev,
host->iomap = pcim_iomap_table(pdev);
mmio_base = host->iomap[ADMA_MMIO_BAR];
- rc = adma_set_dma_masks(pdev, mmio_base);
- if (rc)
+ rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+ if (rc) {
+ dev_err(&pdev->dev, "32-bit DMA enable failed\n");
return rc;
+ }
for (port_no = 0; port_no < ADMA_PORTS; ++port_no) {
struct ata_port *ap = host->ports[port_no];
diff --git a/drivers/ata/sata_inic162x.c b/drivers/ata/sata_inic162x.c
index 790968497dfe..7f99e23bff88 100644
--- a/drivers/ata/sata_inic162x.c
+++ b/drivers/ata/sata_inic162x.c
@@ -862,18 +862,12 @@ static int inic_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
}
/* Set dma_mask. This devices doesn't support 64bit addressing. */
- rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
+ rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
if (rc) {
dev_err(&pdev->dev, "32-bit DMA enable failed\n");
return rc;
}
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev, "32-bit consistent DMA enable failed\n");
- return rc;
- }
-
rc = init_controller(hpriv->mmio_base, hpriv->cached_hctl);
if (rc) {
dev_err(&pdev->dev, "failed to initialize controller\n");
diff --git a/drivers/ata/sata_mv.c b/drivers/ata/sata_mv.c
index da585d2bded6..ad385a113391 100644
--- a/drivers/ata/sata_mv.c
+++ b/drivers/ata/sata_mv.c
@@ -4314,38 +4314,6 @@ static struct pci_driver mv_pci_driver = {
};
-/* move to PCI layer or libata core? */
-static int pci_go_64(struct pci_dev *pdev)
-{
- int rc;
-
- if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64))) {
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
- if (rc) {
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev,
- "64-bit DMA enable failed\n");
- return rc;
- }
- }
- } else {
- rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev, "32-bit DMA enable failed\n");
- return rc;
- }
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev,
- "32-bit consistent DMA enable failed\n");
- return rc;
- }
- }
-
- return rc;
-}
-
/**
* mv_print_info - Dump key info to kernel log for perusal.
* @host: ATA host to print info about
@@ -4430,9 +4398,11 @@ static int mv_pci_init_one(struct pci_dev *pdev,
host->iomap = pcim_iomap_table(pdev);
hpriv->base = host->iomap[MV_PRIMARY_BAR];
- rc = pci_go_64(pdev);
- if (rc)
+ rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+ if (rc) {
+ dev_err(&pdev->dev, "DMA enable failed\n");
return rc;
+ }
rc = mv_create_dma_pools(hpriv, &pdev->dev);
if (rc)
diff --git a/drivers/ata/sata_nv.c b/drivers/ata/sata_nv.c
index 54bfab15c74a..56946012d113 100644
--- a/drivers/ata/sata_nv.c
+++ b/drivers/ata/sata_nv.c
@@ -1122,21 +1122,16 @@ static int nv_adma_port_start(struct ata_port *ap)
/*
* Now that the legacy PRD and padding buffer are allocated we can
- * try to raise the DMA mask to allocate the CPB/APRD table.
+ * raise the DMA mask to allocate the CPB/APRD table.
*/
- rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
- if (rc) {
- rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
- if (rc)
- return rc;
- }
+ dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+
pp->adma_dma_mask = *dev->dma_mask;
mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
&mem_dma, GFP_KERNEL);
if (!mem)
return -ENOMEM;
- memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
/*
* First item in chunk of DMA memory:
@@ -1946,7 +1941,6 @@ static int nv_swncq_port_start(struct ata_port *ap)
&pp->prd_dma, GFP_KERNEL);
if (!pp->prd)
return -ENOMEM;
- memset(pp->prd, 0, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE);
ap->private_data = pp;
pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
diff --git a/drivers/ata/sata_promise.c b/drivers/ata/sata_promise.c
index f4dfec3b6e42..5fd464765ddc 100644
--- a/drivers/ata/sata_promise.c
+++ b/drivers/ata/sata_promise.c
@@ -1230,10 +1230,7 @@ static int pdc_ata_init_one(struct pci_dev *pdev,
/* initialize adapter */
pdc_host_init(host);
- rc = dma_set_mask(&pdev->dev, ATA_DMA_MASK);
- if (rc)
- return rc;
- rc = dma_set_coherent_mask(&pdev->dev, ATA_DMA_MASK);
+ rc = dma_set_mask_and_coherent(&pdev->dev, ATA_DMA_MASK);
if (rc)
return rc;
diff --git a/drivers/ata/sata_qstor.c b/drivers/ata/sata_qstor.c
index 7ec0c216a6a6..c53c5a47204d 100644
--- a/drivers/ata/sata_qstor.c
+++ b/drivers/ata/sata_qstor.c
@@ -477,7 +477,6 @@ static int qs_port_start(struct ata_port *ap)
GFP_KERNEL);
if (!pp->pkt)
return -ENOMEM;
- memset(pp->pkt, 0, QS_PKT_BYTES);
ap->private_data = pp;
qs_enter_reg_mode(ap);
@@ -538,33 +537,13 @@ static void qs_host_init(struct ata_host *host, unsigned int chip_id)
static int qs_set_dma_masks(struct pci_dev *pdev, void __iomem *mmio_base)
{
u32 bus_info = readl(mmio_base + QS_HID_HPHY);
- int rc, have_64bit_bus = (bus_info & QS_HPHY_64BIT);
-
- if (have_64bit_bus &&
- !dma_set_mask(&pdev->dev, DMA_BIT_MASK(64))) {
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
- if (rc) {
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev,
- "64-bit DMA enable failed\n");
- return rc;
- }
- }
- } else {
- rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev, "32-bit DMA enable failed\n");
- return rc;
- }
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev,
- "32-bit consistent DMA enable failed\n");
- return rc;
- }
- }
- return 0;
+ int dma_bits = (bus_info & QS_HPHY_64BIT) ? 64 : 32;
+ int rc;
+
+ rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(dma_bits));
+ if (rc)
+ dev_err(&pdev->dev, "%d-bit DMA enable failed\n", dma_bits);
+ return rc;
}
static int qs_ata_init_one(struct pci_dev *pdev,
diff --git a/drivers/ata/sata_sil.c b/drivers/ata/sata_sil.c
index 25b6a52be5ab..e6fbae2f645a 100644
--- a/drivers/ata/sata_sil.c
+++ b/drivers/ata/sata_sil.c
@@ -757,10 +757,7 @@ static int sil_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
return rc;
host->iomap = pcim_iomap_table(pdev);
- rc = dma_set_mask(&pdev->dev, ATA_DMA_MASK);
- if (rc)
- return rc;
- rc = dma_set_coherent_mask(&pdev->dev, ATA_DMA_MASK);
+ rc = dma_set_mask_and_coherent(&pdev->dev, ATA_DMA_MASK);
if (rc)
return rc;
diff --git a/drivers/ata/sata_sil24.c b/drivers/ata/sata_sil24.c
index bfdf41912588..7bef82de53ca 100644
--- a/drivers/ata/sata_sil24.c
+++ b/drivers/ata/sata_sil24.c
@@ -1202,7 +1202,6 @@ static int sil24_port_start(struct ata_port *ap)
cb = dmam_alloc_coherent(dev, cb_size, &cb_dma, GFP_KERNEL);
if (!cb)
return -ENOMEM;
- memset(cb, 0, cb_size);
pp->cmd_block = cb;
pp->cmd_block_dma = cb_dma;
@@ -1302,28 +1301,10 @@ static int sil24_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
host->iomap = iomap;
/* configure and activate the device */
- if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64))) {
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
- if (rc) {
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev,
- "64-bit DMA enable failed\n");
- return rc;
- }
- }
- } else {
- rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev, "32-bit DMA enable failed\n");
- return rc;
- }
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(&pdev->dev,
- "32-bit consistent DMA enable failed\n");
- return rc;
- }
+ rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+ if (rc) {
+ dev_err(&pdev->dev, "DMA enable failed\n");
+ return rc;
}
/* Set max read request size to 4096. This slightly increases
diff --git a/drivers/ata/sata_svw.c b/drivers/ata/sata_svw.c
index b903d55c6c20..f8552559db7f 100644
--- a/drivers/ata/sata_svw.c
+++ b/drivers/ata/sata_svw.c
@@ -471,10 +471,7 @@ static int k2_sata_init_one(struct pci_dev *pdev, const struct pci_device_id *en
ata_port_pbar_desc(ap, 5, offset, "port");
}
- rc = dma_set_mask(&pdev->dev, ATA_DMA_MASK);
- if (rc)
- return rc;
- rc = dma_set_coherent_mask(&pdev->dev, ATA_DMA_MASK);
+ rc = dma_set_mask_and_coherent(&pdev->dev, ATA_DMA_MASK);
if (rc)
return rc;
diff --git a/drivers/ata/sata_sx4.c b/drivers/ata/sata_sx4.c
index ae8e374d0a77..2277ba0c9c7f 100644
--- a/drivers/ata/sata_sx4.c
+++ b/drivers/ata/sata_sx4.c
@@ -1470,10 +1470,7 @@ static int pdc_sata_init_one(struct pci_dev *pdev,
}
/* configure and activate */
- rc = dma_set_mask(&pdev->dev, ATA_DMA_MASK);
- if (rc)
- return rc;
- rc = dma_set_coherent_mask(&pdev->dev, ATA_DMA_MASK);
+ rc = dma_set_mask_and_coherent(&pdev->dev, ATA_DMA_MASK);
if (rc)
return rc;
diff --git a/drivers/ata/sata_via.c b/drivers/ata/sata_via.c
index fcb9245b184f..c7891cc84ea0 100644
--- a/drivers/ata/sata_via.c
+++ b/drivers/ata/sata_via.c
@@ -505,14 +505,7 @@ static int vt6421_prepare_host(struct pci_dev *pdev, struct ata_host **r_host)
for (i = 0; i < host->n_ports; i++)
vt6421_init_addrs(host->ports[i]);
- rc = dma_set_mask(&pdev->dev, ATA_DMA_MASK);
- if (rc)
- return rc;
- rc = dma_set_coherent_mask(&pdev->dev, ATA_DMA_MASK);
- if (rc)
- return rc;
-
- return 0;
+ return dma_set_mask_and_coherent(&pdev->dev, ATA_DMA_MASK);
}
static int vt8251_prepare_host(struct pci_dev *pdev, struct ata_host **r_host)
diff --git a/drivers/ata/sata_vsc.c b/drivers/ata/sata_vsc.c
index fd401e9164ef..8fa952cb9f7f 100644
--- a/drivers/ata/sata_vsc.c
+++ b/drivers/ata/sata_vsc.c
@@ -371,10 +371,7 @@ static int vsc_sata_init_one(struct pci_dev *pdev,
/*
* Use 32 bit DMA mask, because 64 bit address support is poor.
*/
- rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
- if (rc)
- return rc;
- rc = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
+ rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
if (rc)
return rc;
diff --git a/drivers/atm/Kconfig b/drivers/atm/Kconfig
index 2e2efa577437..8c37294f1d1e 100644
--- a/drivers/atm/Kconfig
+++ b/drivers/atm/Kconfig
@@ -200,7 +200,7 @@ config ATM_NICSTAR_USE_SUNI
make the card work).
config ATM_NICSTAR_USE_IDT77105
- bool "Use IDT77015 PHY driver (25Mbps)"
+ bool "Use IDT77105 PHY driver (25Mbps)"
depends on ATM_NICSTAR
help
Support for the PHYsical layer chip in ForeRunner LE25 cards. In
diff --git a/drivers/atm/eni.c b/drivers/atm/eni.c
index 79b718430cd1..b23d1e4bad33 100644
--- a/drivers/atm/eni.c
+++ b/drivers/atm/eni.c
@@ -1136,7 +1136,7 @@ DPRINTK("doing direct send\n"); /* @@@ well, this doesn't work anyway */
else
put_dma(tx->index,eni_dev->dma,&j,(unsigned long)
skb_frag_page(&skb_shinfo(skb)->frags[i]) +
- skb_shinfo(skb)->frags[i].page_offset,
+ skb_frag_off(&skb_shinfo(skb)->frags[i]),
skb_frag_size(&skb_shinfo(skb)->frags[i]));
}
if (skb->len & 3) {
diff --git a/drivers/atm/he.c b/drivers/atm/he.c
index 211607986134..70b00ae4ec38 100644
--- a/drivers/atm/he.c
+++ b/drivers/atm/he.c
@@ -2580,10 +2580,9 @@ he_send(struct atm_vcc *vcc, struct sk_buff *skb)
slot = 0;
}
- tpd->iovec[slot].addr = dma_map_single(&he_dev->pci_dev->dev,
- (void *) page_address(frag->page) + frag->page_offset,
- frag->size, DMA_TO_DEVICE);
- tpd->iovec[slot].len = frag->size;
+ tpd->iovec[slot].addr = skb_frag_dma_map(&he_dev->pci_dev->dev,
+ frag, 0, skb_frag_size(frag), DMA_TO_DEVICE);
+ tpd->iovec[slot].len = skb_frag_size(frag);
++slot;
}
diff --git a/drivers/atm/idt77252.c b/drivers/atm/idt77252.c
index 43a14579e80e..df51680e8931 100644
--- a/drivers/atm/idt77252.c
+++ b/drivers/atm/idt77252.c
@@ -1379,7 +1379,6 @@ init_tsq(struct idt77252_dev *card)
printk("%s: can't allocate TSQ.\n", card->name);
return -1;
}
- memset(card->tsq.base, 0, TSQSIZE);
card->tsq.last = card->tsq.base + TSQ_NUM_ENTRIES - 1;
card->tsq.next = card->tsq.last;
diff --git a/drivers/atm/iphase.c b/drivers/atm/iphase.c
index 302cf0ba1600..8c7a996d1f16 100644
--- a/drivers/atm/iphase.c
+++ b/drivers/atm/iphase.c
@@ -63,6 +63,7 @@
#include <asm/byteorder.h>
#include <linux/vmalloc.h>
#include <linux/jiffies.h>
+#include <linux/nospec.h>
#include "iphase.h"
#include "suni.h"
#define swap_byte_order(x) (((x & 0xff) << 8) | ((x & 0xff00) >> 8))
@@ -2760,8 +2761,11 @@ static int ia_ioctl(struct atm_dev *dev, unsigned int cmd, void __user *arg)
}
if (copy_from_user(&ia_cmds, arg, sizeof ia_cmds)) return -EFAULT;
board = ia_cmds.status;
- if ((board < 0) || (board > iadev_count))
- board = 0;
+
+ if ((board < 0) || (board > iadev_count))
+ board = 0;
+ board = array_index_nospec(board, iadev_count + 1);
+
iadev = ia_dev[board];
switch (ia_cmds.cmd) {
case MEMDUMP:
diff --git a/drivers/atm/solos-pci.c b/drivers/atm/solos-pci.c
index 5c4c6eeb505c..c32f7dd9879a 100644
--- a/drivers/atm/solos-pci.c
+++ b/drivers/atm/solos-pci.c
@@ -516,9 +516,8 @@ struct geos_gpio_attr {
static ssize_t geos_gpio_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count)
{
- struct pci_dev *pdev = to_pci_dev(dev);
struct geos_gpio_attr *gattr = container_of(attr, struct geos_gpio_attr, attr);
- struct solos_card *card = pci_get_drvdata(pdev);
+ struct solos_card *card = dev_get_drvdata(dev);
uint32_t data32;
if (count != 1 && (count != 2 || buf[1] != '\n'))
@@ -542,9 +541,8 @@ static ssize_t geos_gpio_store(struct device *dev, struct device_attribute *attr
static ssize_t geos_gpio_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
- struct pci_dev *pdev = to_pci_dev(dev);
struct geos_gpio_attr *gattr = container_of(attr, struct geos_gpio_attr, attr);
- struct solos_card *card = pci_get_drvdata(pdev);
+ struct solos_card *card = dev_get_drvdata(dev);
uint32_t data32;
data32 = ioread32(card->config_regs + GPIO_STATUS);
@@ -556,9 +554,8 @@ static ssize_t geos_gpio_show(struct device *dev, struct device_attribute *attr,
static ssize_t hardware_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
- struct pci_dev *pdev = to_pci_dev(dev);
struct geos_gpio_attr *gattr = container_of(attr, struct geos_gpio_attr, attr);
- struct solos_card *card = pci_get_drvdata(pdev);
+ struct solos_card *card = dev_get_drvdata(dev);
uint32_t data32;
data32 = ioread32(card->config_regs + GPIO_STATUS);
diff --git a/drivers/auxdisplay/Kconfig b/drivers/auxdisplay/Kconfig
index c52c738e554a..b8313a04422d 100644
--- a/drivers/auxdisplay/Kconfig
+++ b/drivers/auxdisplay/Kconfig
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: GPL-2.0
#
# For a description of the syntax of this configuration file,
-# see Documentation/kbuild/kconfig-language.txt.
+# see Documentation/kbuild/kconfig-language.rst.
#
# Auxiliary display drivers configuration.
#
@@ -97,7 +97,7 @@ config CFAG12864B
say Y. You also need the ks0108 LCD Controller driver.
For help about how to wire your LCD to the parallel port,
- check Documentation/auxdisplay/cfag12864b
+ check Documentation/admin-guide/auxdisplay/cfag12864b.rst
Depends on the x86 arch and the framebuffer support.
@@ -448,6 +448,11 @@ config PANEL_BOOT_MESSAGE
choice
prompt "Backlight initial state"
default CHARLCD_BL_FLASH
+ ---help---
+ Select the initial backlight state on boot or module load.
+
+ Previously, there was no option for this: the backlight flashed
+ briefly on init. Now you can also turn it off/on.
config CHARLCD_BL_OFF
bool "Off"
diff --git a/drivers/auxdisplay/charlcd.c b/drivers/auxdisplay/charlcd.c
index 92745efefb54..bef6b85778b6 100644
--- a/drivers/auxdisplay/charlcd.c
+++ b/drivers/auxdisplay/charlcd.c
@@ -20,7 +20,7 @@
#include <generated/utsrelease.h>
-#include <misc/charlcd.h>
+#include "charlcd.h"
#define LCD_MINOR 156
diff --git a/drivers/auxdisplay/charlcd.h b/drivers/auxdisplay/charlcd.h
new file mode 100644
index 000000000000..00911ad0f3de
--- /dev/null
+++ b/drivers/auxdisplay/charlcd.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Character LCD driver for Linux
+ *
+ * Copyright (C) 2000-2008, Willy Tarreau <w@1wt.eu>
+ * Copyright (C) 2016-2017 Glider bvba
+ */
+
+#ifndef _CHARLCD_H
+#define _CHARLCD_H
+
+struct charlcd {
+ const struct charlcd_ops *ops;
+ const unsigned char *char_conv; /* Optional */
+
+ int ifwidth; /* 4-bit or 8-bit (default) */
+ int height;
+ int width;
+ int bwidth; /* Default set by charlcd_alloc() */
+ int hwidth; /* Default set by charlcd_alloc() */
+
+ void *drvdata; /* Set by charlcd_alloc() */
+};
+
+struct charlcd_ops {
+ /* Required */
+ void (*write_cmd)(struct charlcd *lcd, int cmd);
+ void (*write_data)(struct charlcd *lcd, int data);
+
+ /* Optional */
+ void (*write_cmd_raw4)(struct charlcd *lcd, int cmd); /* 4-bit only */
+ void (*clear_fast)(struct charlcd *lcd);
+ void (*backlight)(struct charlcd *lcd, int on);
+};
+
+struct charlcd *charlcd_alloc(unsigned int drvdata_size);
+void charlcd_free(struct charlcd *lcd);
+
+int charlcd_register(struct charlcd *lcd);
+int charlcd_unregister(struct charlcd *lcd);
+
+void charlcd_poke(struct charlcd *lcd);
+
+#endif /* CHARLCD_H */
diff --git a/drivers/auxdisplay/hd44780.c b/drivers/auxdisplay/hd44780.c
index ab15b64707ad..bcbe13092327 100644
--- a/drivers/auxdisplay/hd44780.c
+++ b/drivers/auxdisplay/hd44780.c
@@ -14,8 +14,7 @@
#include <linux/property.h>
#include <linux/slab.h>
-#include <misc/charlcd.h>
-
+#include "charlcd.h"
enum hd44780_pin {
/* Order does matter due to writing to GPIO array subsets! */
diff --git a/drivers/auxdisplay/ht16k33.c b/drivers/auxdisplay/ht16k33.c
index 9c0bb771751d..a2fcde582e2a 100644
--- a/drivers/auxdisplay/ht16k33.c
+++ b/drivers/auxdisplay/ht16k33.c
@@ -74,7 +74,7 @@ struct ht16k33_priv {
struct ht16k33_fbdev fbdev;
};
-static struct fb_fix_screeninfo ht16k33_fb_fix = {
+static const struct fb_fix_screeninfo ht16k33_fb_fix = {
.id = DRIVER_NAME,
.type = FB_TYPE_PACKED_PIXELS,
.visual = FB_VISUAL_MONO10,
@@ -85,7 +85,7 @@ static struct fb_fix_screeninfo ht16k33_fb_fix = {
.accel = FB_ACCEL_NONE,
};
-static struct fb_var_screeninfo ht16k33_fb_var = {
+static const struct fb_var_screeninfo ht16k33_fb_var = {
.xres = HT16K33_MATRIX_LED_MAX_ROWS,
.yres = HT16K33_MATRIX_LED_MAX_COLS,
.xres_virtual = HT16K33_MATRIX_LED_MAX_ROWS,
diff --git a/drivers/auxdisplay/panel.c b/drivers/auxdisplay/panel.c
index e06de63497cf..85965953683e 100644
--- a/drivers/auxdisplay/panel.c
+++ b/drivers/auxdisplay/panel.c
@@ -55,7 +55,7 @@
#include <linux/io.h>
#include <linux/uaccess.h>
-#include <misc/charlcd.h>
+#include "charlcd.h"
#define KEYPAD_MINOR 185
@@ -1617,6 +1617,8 @@ static void panel_attach(struct parport *port)
return;
err_lcd_unreg:
+ if (scan_timer.function)
+ del_timer_sync(&scan_timer);
if (lcd.enabled)
charlcd_unregister(lcd.charlcd);
err_unreg_device:
diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
index dc404492381d..28b92e3cc570 100644
--- a/drivers/base/Kconfig
+++ b/drivers/base/Kconfig
@@ -202,7 +202,7 @@ config GENERIC_ARCH_TOPOLOGY
help
Enable support for architectures common topology code: e.g., parsing
CPU capacity information from DT, usage of such information for
- appropriate scaling, sysfs interface for changing capacity values at
+ appropriate scaling, sysfs interface for reading capacity values at
runtime.
endmenu
diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c
index 1739d7e1952a..1eb81f113786 100644
--- a/drivers/base/arch_topology.c
+++ b/drivers/base/arch_topology.c
@@ -15,6 +15,11 @@
#include <linux/string.h>
#include <linux/sched/topology.h>
#include <linux/cpuset.h>
+#include <linux/cpumask.h>
+#include <linux/init.h>
+#include <linux/percpu.h>
+#include <linux/sched.h>
+#include <linux/smp.h>
DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE;
@@ -43,7 +48,7 @@ static ssize_t cpu_capacity_show(struct device *dev,
{
struct cpu *cpu = container_of(dev, struct cpu, dev);
- return sprintf(buf, "%lu\n", topology_get_cpu_scale(NULL, cpu->dev.id));
+ return sprintf(buf, "%lu\n", topology_get_cpu_scale(cpu->dev.id));
}
static void update_topology_flags_workfn(struct work_struct *work);
@@ -116,7 +121,7 @@ void topology_normalize_cpu_scale(void)
/ capacity_scale;
topology_set_cpu_scale(cpu, capacity);
pr_debug("cpu_capacity: CPU%d cpu_capacity=%lu\n",
- cpu, topology_get_cpu_scale(NULL, cpu));
+ cpu, topology_get_cpu_scale(cpu));
}
}
@@ -137,7 +142,6 @@ bool __init topology_parse_cpu_capacity(struct device_node *cpu_node, int cpu)
sizeof(*raw_capacity),
GFP_KERNEL);
if (!raw_capacity) {
- pr_err("cpu_capacity: failed to allocate memory for raw capacities\n");
cap_parsing_failed = true;
return false;
}
@@ -175,7 +179,7 @@ init_cpu_capacity_callback(struct notifier_block *nb,
if (!raw_capacity)
return 0;
- if (val != CPUFREQ_NOTIFY)
+ if (val != CPUFREQ_CREATE_POLICY)
return 0;
pr_debug("cpu_capacity: init cpu capacity for CPUs [%*pbl] (to_visit=%*pbl)\n",
@@ -185,7 +189,7 @@ init_cpu_capacity_callback(struct notifier_block *nb,
cpumask_andnot(cpus_to_visit, cpus_to_visit, policy->related_cpus);
for_each_cpu(cpu, policy->related_cpus) {
- raw_capacity[cpu] = topology_get_cpu_scale(NULL, cpu) *
+ raw_capacity[cpu] = topology_get_cpu_scale(cpu) *
policy->cpuinfo.max_freq / 1000UL;
capacity_scale = max(raw_capacity[cpu], capacity_scale);
}
@@ -217,10 +221,8 @@ static int __init register_cpufreq_notifier(void)
if (!acpi_disabled || !raw_capacity)
return -EINVAL;
- if (!alloc_cpumask_var(&cpus_to_visit, GFP_KERNEL)) {
- pr_err("cpu_capacity: failed to allocate memory for cpus_to_visit\n");
+ if (!alloc_cpumask_var(&cpus_to_visit, GFP_KERNEL))
return -ENOMEM;
- }
cpumask_copy(cpus_to_visit, cpu_possible_mask);
@@ -244,3 +246,296 @@ static void parsing_done_workfn(struct work_struct *work)
#else
core_initcall(free_raw_capacity);
#endif
+
+#if defined(CONFIG_ARM64) || defined(CONFIG_RISCV)
+static int __init get_cpu_for_node(struct device_node *node)
+{
+ struct device_node *cpu_node;
+ int cpu;
+
+ cpu_node = of_parse_phandle(node, "cpu", 0);
+ if (!cpu_node)
+ return -1;
+
+ cpu = of_cpu_node_to_id(cpu_node);
+ if (cpu >= 0)
+ topology_parse_cpu_capacity(cpu_node, cpu);
+ else
+ pr_crit("Unable to find CPU node for %pOF\n", cpu_node);
+
+ of_node_put(cpu_node);
+ return cpu;
+}
+
+static int __init parse_core(struct device_node *core, int package_id,
+ int core_id)
+{
+ char name[10];
+ bool leaf = true;
+ int i = 0;
+ int cpu;
+ struct device_node *t;
+
+ do {
+ snprintf(name, sizeof(name), "thread%d", i);
+ t = of_get_child_by_name(core, name);
+ if (t) {
+ leaf = false;
+ cpu = get_cpu_for_node(t);
+ if (cpu >= 0) {
+ cpu_topology[cpu].package_id = package_id;
+ cpu_topology[cpu].core_id = core_id;
+ cpu_topology[cpu].thread_id = i;
+ } else {
+ pr_err("%pOF: Can't get CPU for thread\n",
+ t);
+ of_node_put(t);
+ return -EINVAL;
+ }
+ of_node_put(t);
+ }
+ i++;
+ } while (t);
+
+ cpu = get_cpu_for_node(core);
+ if (cpu >= 0) {
+ if (!leaf) {
+ pr_err("%pOF: Core has both threads and CPU\n",
+ core);
+ return -EINVAL;
+ }
+
+ cpu_topology[cpu].package_id = package_id;
+ cpu_topology[cpu].core_id = core_id;
+ } else if (leaf) {
+ pr_err("%pOF: Can't get CPU for leaf core\n", core);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int __init parse_cluster(struct device_node *cluster, int depth)
+{
+ char name[10];
+ bool leaf = true;
+ bool has_cores = false;
+ struct device_node *c;
+ static int package_id __initdata;
+ int core_id = 0;
+ int i, ret;
+
+ /*
+ * First check for child clusters; we currently ignore any
+ * information about the nesting of clusters and present the
+ * scheduler with a flat list of them.
+ */
+ i = 0;
+ do {
+ snprintf(name, sizeof(name), "cluster%d", i);
+ c = of_get_child_by_name(cluster, name);
+ if (c) {
+ leaf = false;
+ ret = parse_cluster(c, depth + 1);
+ of_node_put(c);
+ if (ret != 0)
+ return ret;
+ }
+ i++;
+ } while (c);
+
+ /* Now check for cores */
+ i = 0;
+ do {
+ snprintf(name, sizeof(name), "core%d", i);
+ c = of_get_child_by_name(cluster, name);
+ if (c) {
+ has_cores = true;
+
+ if (depth == 0) {
+ pr_err("%pOF: cpu-map children should be clusters\n",
+ c);
+ of_node_put(c);
+ return -EINVAL;
+ }
+
+ if (leaf) {
+ ret = parse_core(c, package_id, core_id++);
+ } else {
+ pr_err("%pOF: Non-leaf cluster with core %s\n",
+ cluster, name);
+ ret = -EINVAL;
+ }
+
+ of_node_put(c);
+ if (ret != 0)
+ return ret;
+ }
+ i++;
+ } while (c);
+
+ if (leaf && !has_cores)
+ pr_warn("%pOF: empty cluster\n", cluster);
+
+ if (leaf)
+ package_id++;
+
+ return 0;
+}
+
+static int __init parse_dt_topology(void)
+{
+ struct device_node *cn, *map;
+ int ret = 0;
+ int cpu;
+
+ cn = of_find_node_by_path("/cpus");
+ if (!cn) {
+ pr_err("No CPU information found in DT\n");
+ return 0;
+ }
+
+ /*
+ * When topology is provided cpu-map is essentially a root
+ * cluster with restricted subnodes.
+ */
+ map = of_get_child_by_name(cn, "cpu-map");
+ if (!map)
+ goto out;
+
+ ret = parse_cluster(map, 0);
+ if (ret != 0)
+ goto out_map;
+
+ topology_normalize_cpu_scale();
+
+ /*
+ * Check that all cores are in the topology; the SMP code will
+ * only mark cores described in the DT as possible.
+ */
+ for_each_possible_cpu(cpu)
+ if (cpu_topology[cpu].package_id == -1)
+ ret = -EINVAL;
+
+out_map:
+ of_node_put(map);
+out:
+ of_node_put(cn);
+ return ret;
+}
+#endif
+
+/*
+ * cpu topology table
+ */
+struct cpu_topology cpu_topology[NR_CPUS];
+EXPORT_SYMBOL_GPL(cpu_topology);
+
+const struct cpumask *cpu_coregroup_mask(int cpu)
+{
+ const cpumask_t *core_mask = cpumask_of_node(cpu_to_node(cpu));
+
+ /* Find the smaller of NUMA, core or LLC siblings */
+ if (cpumask_subset(&cpu_topology[cpu].core_sibling, core_mask)) {
+ /* not numa in package, lets use the package siblings */
+ core_mask = &cpu_topology[cpu].core_sibling;
+ }
+ if (cpu_topology[cpu].llc_id != -1) {
+ if (cpumask_subset(&cpu_topology[cpu].llc_sibling, core_mask))
+ core_mask = &cpu_topology[cpu].llc_sibling;
+ }
+
+ return core_mask;
+}
+
+void update_siblings_masks(unsigned int cpuid)
+{
+ struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid];
+ int cpu;
+
+ /* update core and thread sibling masks */
+ for_each_online_cpu(cpu) {
+ cpu_topo = &cpu_topology[cpu];
+
+ if (cpuid_topo->llc_id == cpu_topo->llc_id) {
+ cpumask_set_cpu(cpu, &cpuid_topo->llc_sibling);
+ cpumask_set_cpu(cpuid, &cpu_topo->llc_sibling);
+ }
+
+ if (cpuid_topo->package_id != cpu_topo->package_id)
+ continue;
+
+ cpumask_set_cpu(cpuid, &cpu_topo->core_sibling);
+ cpumask_set_cpu(cpu, &cpuid_topo->core_sibling);
+
+ if (cpuid_topo->core_id != cpu_topo->core_id)
+ continue;
+
+ cpumask_set_cpu(cpuid, &cpu_topo->thread_sibling);
+ cpumask_set_cpu(cpu, &cpuid_topo->thread_sibling);
+ }
+}
+
+static void clear_cpu_topology(int cpu)
+{
+ struct cpu_topology *cpu_topo = &cpu_topology[cpu];
+
+ cpumask_clear(&cpu_topo->llc_sibling);
+ cpumask_set_cpu(cpu, &cpu_topo->llc_sibling);
+
+ cpumask_clear(&cpu_topo->core_sibling);
+ cpumask_set_cpu(cpu, &cpu_topo->core_sibling);
+ cpumask_clear(&cpu_topo->thread_sibling);
+ cpumask_set_cpu(cpu, &cpu_topo->thread_sibling);
+}
+
+void __init reset_cpu_topology(void)
+{
+ unsigned int cpu;
+
+ for_each_possible_cpu(cpu) {
+ struct cpu_topology *cpu_topo = &cpu_topology[cpu];
+
+ cpu_topo->thread_id = -1;
+ cpu_topo->core_id = -1;
+ cpu_topo->package_id = -1;
+ cpu_topo->llc_id = -1;
+
+ clear_cpu_topology(cpu);
+ }
+}
+
+void remove_cpu_topology(unsigned int cpu)
+{
+ int sibling;
+
+ for_each_cpu(sibling, topology_core_cpumask(cpu))
+ cpumask_clear_cpu(cpu, topology_core_cpumask(sibling));
+ for_each_cpu(sibling, topology_sibling_cpumask(cpu))
+ cpumask_clear_cpu(cpu, topology_sibling_cpumask(sibling));
+ for_each_cpu(sibling, topology_llc_cpumask(cpu))
+ cpumask_clear_cpu(cpu, topology_llc_cpumask(sibling));
+
+ clear_cpu_topology(cpu);
+}
+
+__weak int __init parse_acpi_topology(void)
+{
+ return 0;
+}
+
+#if defined(CONFIG_ARM64) || defined(CONFIG_RISCV)
+void __init init_cpu_topology(void)
+{
+ reset_cpu_topology();
+
+ /*
+ * Discard anything that was parsed if we hit an error so we
+ * don't use partial information.
+ */
+ if (parse_acpi_topology())
+ reset_cpu_topology();
+ else if (of_have_populated_dt() && parse_dt_topology())
+ reset_cpu_topology();
+}
+#endif
diff --git a/drivers/base/base.h b/drivers/base/base.h
index b405436ee28e..0d32544b6f91 100644
--- a/drivers/base/base.h
+++ b/drivers/base/base.h
@@ -165,6 +165,7 @@ static inline int devtmpfs_init(void) { return 0; }
/* Device links support */
extern int device_links_read_lock(void);
extern void device_links_read_unlock(int idx);
+extern int device_links_read_lock_held(void);
extern int device_links_check_suppliers(struct device *dev);
extern void device_links_driver_bound(struct device *dev);
extern void device_links_driver_cleanup(struct device *dev);
diff --git a/drivers/base/bus.c b/drivers/base/bus.c
index 0a58e969f8b7..a1d1e8256324 100644
--- a/drivers/base/bus.c
+++ b/drivers/base/bus.c
@@ -323,8 +323,8 @@ EXPORT_SYMBOL_GPL(bus_for_each_dev);
* return to the caller and not iterate over any more devices.
*/
struct device *bus_find_device(struct bus_type *bus,
- struct device *start, void *data,
- int (*match)(struct device *dev, void *data))
+ struct device *start, const void *data,
+ int (*match)(struct device *dev, const void *data))
{
struct klist_iter i;
struct device *dev;
@@ -342,30 +342,6 @@ struct device *bus_find_device(struct bus_type *bus,
}
EXPORT_SYMBOL_GPL(bus_find_device);
-static int match_name(struct device *dev, void *data)
-{
- const char *name = data;
-
- return sysfs_streq(name, dev_name(dev));
-}
-
-/**
- * bus_find_device_by_name - device iterator for locating a particular device of a specific name
- * @bus: bus type
- * @start: Device to begin with
- * @name: name of the device to match
- *
- * This is similar to the bus_find_device() function above, but it handles
- * searching by a name automatically, no need to write another strcmp matching
- * function.
- */
-struct device *bus_find_device_by_name(struct bus_type *bus,
- struct device *start, const char *name)
-{
- return bus_find_device(bus, start, (void *)name, match_name);
-}
-EXPORT_SYMBOL_GPL(bus_find_device_by_name);
-
/**
* subsys_find_device_by_id - find a device with a specific enumeration number
* @subsys: subsystem
diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c
index a7359535caf5..8d553c92cd32 100644
--- a/drivers/base/cacheinfo.c
+++ b/drivers/base/cacheinfo.c
@@ -213,6 +213,8 @@ int __weak cache_setup_acpi(unsigned int cpu)
return -ENOTSUPP;
}
+unsigned int coherency_max_size;
+
static int cache_shared_cpu_map_setup(unsigned int cpu)
{
struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
@@ -251,6 +253,9 @@ static int cache_shared_cpu_map_setup(unsigned int cpu)
cpumask_set_cpu(i, &this_leaf->shared_cpu_map);
}
}
+ /* record the maximum cache line size */
+ if (this_leaf->coherency_line_size > coherency_max_size)
+ coherency_max_size = this_leaf->coherency_line_size;
}
return 0;
@@ -655,7 +660,8 @@ static int cacheinfo_cpu_pre_down(unsigned int cpu)
static int __init cacheinfo_sysfs_init(void)
{
- return cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "base/cacheinfo:online",
+ return cpuhp_setup_state(CPUHP_AP_BASE_CACHEINFO_ONLINE,
+ "base/cacheinfo:online",
cacheinfo_cpu_online, cacheinfo_cpu_pre_down);
}
device_initcall(cacheinfo_sysfs_init);
diff --git a/drivers/base/core.c b/drivers/base/core.c
index fd7511e04e62..2db62d98e395 100644
--- a/drivers/base/core.c
+++ b/drivers/base/core.c
@@ -68,6 +68,11 @@ void device_links_read_unlock(int idx)
{
srcu_read_unlock(&device_links_srcu, idx);
}
+
+int device_links_read_lock_held(void)
+{
+ return srcu_read_lock_held(&device_links_srcu);
+}
#else /* !CONFIG_SRCU */
static DECLARE_RWSEM(device_links_lock);
@@ -91,6 +96,13 @@ void device_links_read_unlock(int not_used)
{
up_read(&device_links_lock);
}
+
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+int device_links_read_lock_held(void)
+{
+ return lockdep_is_held(&device_links_lock);
+}
+#endif
#endif /* !CONFIG_SRCU */
/**
@@ -124,6 +136,50 @@ static int device_is_dependent(struct device *dev, void *target)
return ret;
}
+static void device_link_init_status(struct device_link *link,
+ struct device *consumer,
+ struct device *supplier)
+{
+ switch (supplier->links.status) {
+ case DL_DEV_PROBING:
+ switch (consumer->links.status) {
+ case DL_DEV_PROBING:
+ /*
+ * A consumer driver can create a link to a supplier
+ * that has not completed its probing yet as long as it
+ * knows that the supplier is already functional (for
+ * example, it has just acquired some resources from the
+ * supplier).
+ */
+ link->status = DL_STATE_CONSUMER_PROBE;
+ break;
+ default:
+ link->status = DL_STATE_DORMANT;
+ break;
+ }
+ break;
+ case DL_DEV_DRIVER_BOUND:
+ switch (consumer->links.status) {
+ case DL_DEV_PROBING:
+ link->status = DL_STATE_CONSUMER_PROBE;
+ break;
+ case DL_DEV_DRIVER_BOUND:
+ link->status = DL_STATE_ACTIVE;
+ break;
+ default:
+ link->status = DL_STATE_AVAILABLE;
+ break;
+ }
+ break;
+ case DL_DEV_UNBINDING:
+ link->status = DL_STATE_SUPPLIER_UNBIND;
+ break;
+ default:
+ link->status = DL_STATE_DORMANT;
+ break;
+ }
+}
+
static int device_reorder_to_tail(struct device *dev, void *not_used)
{
struct device_link *link;
@@ -165,6 +221,13 @@ void device_pm_move_to_tail(struct device *dev)
device_links_read_unlock(idx);
}
+#define DL_MANAGED_LINK_FLAGS (DL_FLAG_AUTOREMOVE_CONSUMER | \
+ DL_FLAG_AUTOREMOVE_SUPPLIER | \
+ DL_FLAG_AUTOPROBE_CONSUMER)
+
+#define DL_ADD_VALID_FLAGS (DL_MANAGED_LINK_FLAGS | DL_FLAG_STATELESS | \
+ DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE)
+
/**
* device_link_add - Create a link between two devices.
* @consumer: Consumer end of the link.
@@ -179,9 +242,9 @@ void device_pm_move_to_tail(struct device *dev)
* of the link. If DL_FLAG_PM_RUNTIME is not set, DL_FLAG_RPM_ACTIVE will be
* ignored.
*
- * If DL_FLAG_STATELESS is set in @flags, the link is not going to be managed by
- * the driver core and, in particular, the caller of this function is expected
- * to drop the reference to the link acquired by it directly.
+ * If DL_FLAG_STATELESS is set in @flags, the caller of this function is
+ * expected to release the link returned by it directly with the help of either
+ * device_link_del() or device_link_remove().
*
* If that flag is not set, however, the caller of this function is handing the
* management of the link over to the driver core entirely and its return value
@@ -201,9 +264,16 @@ void device_pm_move_to_tail(struct device *dev)
* be used to request the driver core to automaticall probe for a consmer
* driver after successfully binding a driver to the supplier device.
*
- * The combination of DL_FLAG_STATELESS and either DL_FLAG_AUTOREMOVE_CONSUMER
- * or DL_FLAG_AUTOREMOVE_SUPPLIER set in @flags at the same time is invalid and
- * will cause NULL to be returned upfront.
+ * The combination of DL_FLAG_STATELESS and one of DL_FLAG_AUTOREMOVE_CONSUMER,
+ * DL_FLAG_AUTOREMOVE_SUPPLIER, or DL_FLAG_AUTOPROBE_CONSUMER set in @flags at
+ * the same time is invalid and will cause NULL to be returned upfront.
+ * However, if a device link between the given @consumer and @supplier pair
+ * exists already when this function is called for them, the existing link will
+ * be returned regardless of its current type and status (the link's flags may
+ * be modified then). The caller of this function is then expected to treat
+ * the link as though it has just been created, so (in particular) if
+ * DL_FLAG_STATELESS was passed in @flags, the link needs to be released
+ * explicitly when not needed any more (as stated above).
*
* A side effect of the link creation is re-ordering of dpm_list and the
* devices_kset list by moving the consumer device and all devices depending
@@ -219,11 +289,8 @@ struct device_link *device_link_add(struct device *consumer,
{
struct device_link *link;
- if (!consumer || !supplier ||
- (flags & DL_FLAG_STATELESS &&
- flags & (DL_FLAG_AUTOREMOVE_CONSUMER |
- DL_FLAG_AUTOREMOVE_SUPPLIER |
- DL_FLAG_AUTOPROBE_CONSUMER)) ||
+ if (!consumer || !supplier || flags & ~DL_ADD_VALID_FLAGS ||
+ (flags & DL_FLAG_STATELESS && flags & DL_MANAGED_LINK_FLAGS) ||
(flags & DL_FLAG_AUTOPROBE_CONSUMER &&
flags & (DL_FLAG_AUTOREMOVE_CONSUMER |
DL_FLAG_AUTOREMOVE_SUPPLIER)))
@@ -236,6 +303,9 @@ struct device_link *device_link_add(struct device *consumer,
}
}
+ if (!(flags & DL_FLAG_STATELESS))
+ flags |= DL_FLAG_MANAGED;
+
device_links_write_lock();
device_pm_lock();
@@ -262,15 +332,6 @@ struct device_link *device_link_add(struct device *consumer,
if (link->consumer != consumer)
continue;
- /*
- * Don't return a stateless link if the caller wants a stateful
- * one and vice versa.
- */
- if (WARN_ON((flags & DL_FLAG_STATELESS) != (link->flags & DL_FLAG_STATELESS))) {
- link = NULL;
- goto out;
- }
-
if (flags & DL_FLAG_PM_RUNTIME) {
if (!(link->flags & DL_FLAG_PM_RUNTIME)) {
pm_runtime_new_link(consumer);
@@ -281,6 +342,7 @@ struct device_link *device_link_add(struct device *consumer,
}
if (flags & DL_FLAG_STATELESS) {
+ link->flags |= DL_FLAG_STATELESS;
kref_get(&link->kref);
goto out;
}
@@ -299,6 +361,11 @@ struct device_link *device_link_add(struct device *consumer,
link->flags &= ~(DL_FLAG_AUTOREMOVE_CONSUMER |
DL_FLAG_AUTOREMOVE_SUPPLIER);
}
+ if (!(link->flags & DL_FLAG_MANAGED)) {
+ kref_get(&link->kref);
+ link->flags |= DL_FLAG_MANAGED;
+ device_link_init_status(link, consumer, supplier);
+ }
goto out;
}
@@ -325,48 +392,10 @@ struct device_link *device_link_add(struct device *consumer,
kref_init(&link->kref);
/* Determine the initial link state. */
- if (flags & DL_FLAG_STATELESS) {
+ if (flags & DL_FLAG_STATELESS)
link->status = DL_STATE_NONE;
- } else {
- switch (supplier->links.status) {
- case DL_DEV_PROBING:
- switch (consumer->links.status) {
- case DL_DEV_PROBING:
- /*
- * A consumer driver can create a link to a
- * supplier that has not completed its probing
- * yet as long as it knows that the supplier is
- * already functional (for example, it has just
- * acquired some resources from the supplier).
- */
- link->status = DL_STATE_CONSUMER_PROBE;
- break;
- default:
- link->status = DL_STATE_DORMANT;
- break;
- }
- break;
- case DL_DEV_DRIVER_BOUND:
- switch (consumer->links.status) {
- case DL_DEV_PROBING:
- link->status = DL_STATE_CONSUMER_PROBE;
- break;
- case DL_DEV_DRIVER_BOUND:
- link->status = DL_STATE_ACTIVE;
- break;
- default:
- link->status = DL_STATE_AVAILABLE;
- break;
- }
- break;
- case DL_DEV_UNBINDING:
- link->status = DL_STATE_SUPPLIER_UNBIND;
- break;
- default:
- link->status = DL_STATE_DORMANT;
- break;
- }
- }
+ else
+ device_link_init_status(link, consumer, supplier);
/*
* Some callers expect the link creation during consumer driver probe to
@@ -528,7 +557,7 @@ static void device_links_missing_supplier(struct device *dev)
* mark the link as "consumer probe in progress" to make the supplier removal
* wait for us to complete (or bad things may happen).
*
- * Links with the DL_FLAG_STATELESS flag set are ignored.
+ * Links without the DL_FLAG_MANAGED flag set are ignored.
*/
int device_links_check_suppliers(struct device *dev)
{
@@ -538,7 +567,7 @@ int device_links_check_suppliers(struct device *dev)
device_links_write_lock();
list_for_each_entry(link, &dev->links.suppliers, c_node) {
- if (link->flags & DL_FLAG_STATELESS)
+ if (!(link->flags & DL_FLAG_MANAGED))
continue;
if (link->status != DL_STATE_AVAILABLE) {
@@ -563,7 +592,7 @@ int device_links_check_suppliers(struct device *dev)
*
* Also change the status of @dev's links to suppliers to "active".
*
- * Links with the DL_FLAG_STATELESS flag set are ignored.
+ * Links without the DL_FLAG_MANAGED flag set are ignored.
*/
void device_links_driver_bound(struct device *dev)
{
@@ -572,7 +601,7 @@ void device_links_driver_bound(struct device *dev)
device_links_write_lock();
list_for_each_entry(link, &dev->links.consumers, s_node) {
- if (link->flags & DL_FLAG_STATELESS)
+ if (!(link->flags & DL_FLAG_MANAGED))
continue;
/*
@@ -593,7 +622,7 @@ void device_links_driver_bound(struct device *dev)
}
list_for_each_entry(link, &dev->links.suppliers, c_node) {
- if (link->flags & DL_FLAG_STATELESS)
+ if (!(link->flags & DL_FLAG_MANAGED))
continue;
WARN_ON(link->status != DL_STATE_CONSUMER_PROBE);
@@ -605,6 +634,13 @@ void device_links_driver_bound(struct device *dev)
device_links_write_unlock();
}
+static void device_link_drop_managed(struct device_link *link)
+{
+ link->flags &= ~DL_FLAG_MANAGED;
+ WRITE_ONCE(link->status, DL_STATE_NONE);
+ kref_put(&link->kref, __device_link_del);
+}
+
/**
* __device_links_no_driver - Update links of a device without a driver.
* @dev: Device without a drvier.
@@ -615,18 +651,18 @@ void device_links_driver_bound(struct device *dev)
* unless they already are in the "supplier unbind in progress" state in which
* case they need not be updated.
*
- * Links with the DL_FLAG_STATELESS flag set are ignored.
+ * Links without the DL_FLAG_MANAGED flag set are ignored.
*/
static void __device_links_no_driver(struct device *dev)
{
struct device_link *link, *ln;
list_for_each_entry_safe_reverse(link, ln, &dev->links.suppliers, c_node) {
- if (link->flags & DL_FLAG_STATELESS)
+ if (!(link->flags & DL_FLAG_MANAGED))
continue;
if (link->flags & DL_FLAG_AUTOREMOVE_CONSUMER)
- __device_link_del(&link->kref);
+ device_link_drop_managed(link);
else if (link->status == DL_STATE_CONSUMER_PROBE ||
link->status == DL_STATE_ACTIVE)
WRITE_ONCE(link->status, DL_STATE_AVAILABLE);
@@ -643,7 +679,7 @@ static void __device_links_no_driver(struct device *dev)
* %__device_links_no_driver() to update links to suppliers for it as
* appropriate.
*
- * Links with the DL_FLAG_STATELESS flag set are ignored.
+ * Links without the DL_FLAG_MANAGED flag set are ignored.
*/
void device_links_no_driver(struct device *dev)
{
@@ -652,7 +688,7 @@ void device_links_no_driver(struct device *dev)
device_links_write_lock();
list_for_each_entry(link, &dev->links.consumers, s_node) {
- if (link->flags & DL_FLAG_STATELESS)
+ if (!(link->flags & DL_FLAG_MANAGED))
continue;
/*
@@ -680,7 +716,7 @@ void device_links_no_driver(struct device *dev)
* invoke %__device_links_no_driver() to update links to suppliers for it as
* appropriate.
*
- * Links with the DL_FLAG_STATELESS flag set are ignored.
+ * Links without the DL_FLAG_MANAGED flag set are ignored.
*/
void device_links_driver_cleanup(struct device *dev)
{
@@ -689,7 +725,7 @@ void device_links_driver_cleanup(struct device *dev)
device_links_write_lock();
list_for_each_entry_safe(link, ln, &dev->links.consumers, s_node) {
- if (link->flags & DL_FLAG_STATELESS)
+ if (!(link->flags & DL_FLAG_MANAGED))
continue;
WARN_ON(link->flags & DL_FLAG_AUTOREMOVE_CONSUMER);
@@ -702,7 +738,7 @@ void device_links_driver_cleanup(struct device *dev)
*/
if (link->status == DL_STATE_SUPPLIER_UNBIND &&
link->flags & DL_FLAG_AUTOREMOVE_SUPPLIER)
- __device_link_del(&link->kref);
+ device_link_drop_managed(link);
WRITE_ONCE(link->status, DL_STATE_DORMANT);
}
@@ -724,7 +760,7 @@ void device_links_driver_cleanup(struct device *dev)
*
* Return 'false' if there are no probing or active consumers.
*
- * Links with the DL_FLAG_STATELESS flag set are ignored.
+ * Links without the DL_FLAG_MANAGED flag set are ignored.
*/
bool device_links_busy(struct device *dev)
{
@@ -734,7 +770,7 @@ bool device_links_busy(struct device *dev)
device_links_write_lock();
list_for_each_entry(link, &dev->links.consumers, s_node) {
- if (link->flags & DL_FLAG_STATELESS)
+ if (!(link->flags & DL_FLAG_MANAGED))
continue;
if (link->status == DL_STATE_CONSUMER_PROBE
@@ -764,7 +800,7 @@ bool device_links_busy(struct device *dev)
* driver to unbind and start over (the consumer will not re-probe as we have
* changed the state of the link already).
*
- * Links with the DL_FLAG_STATELESS flag set are ignored.
+ * Links without the DL_FLAG_MANAGED flag set are ignored.
*/
void device_links_unbind_consumers(struct device *dev)
{
@@ -776,7 +812,7 @@ void device_links_unbind_consumers(struct device *dev)
list_for_each_entry(link, &dev->links.consumers, s_node) {
enum device_link_state status;
- if (link->flags & DL_FLAG_STATELESS)
+ if (!(link->flags & DL_FLAG_MANAGED))
continue;
status = link->status;
@@ -1663,6 +1699,9 @@ void device_initialize(struct device *dev)
kobject_init(&dev->kobj, &device_ktype);
INIT_LIST_HEAD(&dev->dma_pools);
mutex_init(&dev->mutex);
+#ifdef CONFIG_PROVE_LOCKING
+ mutex_init(&dev->lockdep_mutex);
+#endif
lockdep_set_novalidate_class(&dev->mutex);
spin_lock_init(&dev->devres_lock);
INIT_LIST_HEAD(&dev->devres_head);
@@ -1820,12 +1859,63 @@ static inline struct kobject *get_glue_dir(struct device *dev)
*/
static void cleanup_glue_dir(struct device *dev, struct kobject *glue_dir)
{
+ unsigned int ref;
+
/* see if we live in a "glue" directory */
if (!live_in_glue_dir(glue_dir, dev))
return;
mutex_lock(&gdp_mutex);
- if (!kobject_has_children(glue_dir))
+ /**
+ * There is a race condition between removing glue directory
+ * and adding a new device under the glue directory.
+ *
+ * CPU1: CPU2:
+ *
+ * device_add()
+ * get_device_parent()
+ * class_dir_create_and_add()
+ * kobject_add_internal()
+ * create_dir() // create glue_dir
+ *
+ * device_add()
+ * get_device_parent()
+ * kobject_get() // get glue_dir
+ *
+ * device_del()
+ * cleanup_glue_dir()
+ * kobject_del(glue_dir)
+ *
+ * kobject_add()
+ * kobject_add_internal()
+ * create_dir() // in glue_dir
+ * sysfs_create_dir_ns()
+ * kernfs_create_dir_ns(sd)
+ *
+ * sysfs_remove_dir() // glue_dir->sd=NULL
+ * sysfs_put() // free glue_dir->sd
+ *
+ * // sd is freed
+ * kernfs_new_node(sd)
+ * kernfs_get(glue_dir)
+ * kernfs_add_one()
+ * kernfs_put()
+ *
+ * Before CPU1 remove last child device under glue dir, if CPU2 add
+ * a new device under glue dir, the glue_dir kobject reference count
+ * will be increase to 2 in kobject_get(k). And CPU2 has been called
+ * kernfs_create_dir_ns(). Meanwhile, CPU1 call sysfs_remove_dir()
+ * and sysfs_put(). This result in glue_dir->sd is freed.
+ *
+ * Then the CPU2 will see a stale "empty" but still potentially used
+ * glue dir around in kernfs_new_node().
+ *
+ * In order to avoid this happening, we also should make sure that
+ * kernfs_node for glue_dir is released in CPU1 only when refcount
+ * for glue_dir kobj is 1.
+ */
+ ref = kref_read(&glue_dir->kref);
+ if (!kobject_has_children(glue_dir) && !--ref)
kobject_del(glue_dir);
kobject_put(glue_dir);
mutex_unlock(&gdp_mutex);
@@ -2211,6 +2301,24 @@ void put_device(struct device *dev)
}
EXPORT_SYMBOL_GPL(put_device);
+bool kill_device(struct device *dev)
+{
+ /*
+ * Require the device lock and set the "dead" flag to guarantee that
+ * the update behavior is consistent with the other bitfields near
+ * it and that we cannot have an asynchronous probe routine trying
+ * to run while we are tearing out the bus/class/sysfs from
+ * underneath the device.
+ */
+ lockdep_assert_held(&dev->mutex);
+
+ if (dev->p->dead)
+ return false;
+ dev->p->dead = true;
+ return true;
+}
+EXPORT_SYMBOL_GPL(kill_device);
+
/**
* device_del - delete device from system.
* @dev: device.
@@ -2230,15 +2338,8 @@ void device_del(struct device *dev)
struct kobject *glue_dir = NULL;
struct class_interface *class_intf;
- /*
- * Hold the device lock and set the "dead" flag to guarantee that
- * the update behavior is consistent with the other bitfields near
- * it and that we cannot have an asynchronous probe routine trying
- * to run while we are tearing out the bus/class/sysfs from
- * underneath the device.
- */
device_lock(dev);
- dev->p->dead = true;
+ kill_device(dev);
device_unlock(dev);
/* Notify clients of device removal. This call must come
@@ -2474,6 +2575,34 @@ struct device *device_find_child(struct device *parent, void *data,
}
EXPORT_SYMBOL_GPL(device_find_child);
+/**
+ * device_find_child_by_name - device iterator for locating a child device.
+ * @parent: parent struct device
+ * @name: name of the child device
+ *
+ * This is similar to the device_find_child() function above, but it
+ * returns a reference to a device that has the name @name.
+ *
+ * NOTE: you will need to drop the reference with put_device() after use.
+ */
+struct device *device_find_child_by_name(struct device *parent,
+ const char *name)
+{
+ struct klist_iter i;
+ struct device *child;
+
+ if (!parent)
+ return NULL;
+
+ klist_iter_init(&parent->p->klist_children, &i);
+ while ((child = next_device(&i)))
+ if (!strcmp(dev_name(child), name) && get_device(child))
+ break;
+ klist_iter_exit(&i);
+ return child;
+}
+EXPORT_SYMBOL_GPL(device_find_child_by_name);
+
int __init devices_init(void)
{
devices_kset = kset_create_and_add("devices", &device_uevent_ops, NULL);
@@ -2839,13 +2968,6 @@ struct device *device_create_with_groups(struct class *class,
}
EXPORT_SYMBOL_GPL(device_create_with_groups);
-static int __match_devt(struct device *dev, const void *data)
-{
- const dev_t *devt = data;
-
- return dev->devt == *devt;
-}
-
/**
* device_destroy - removes a device that was created with device_create()
* @class: pointer to the struct class that this device was registered with
@@ -2858,7 +2980,7 @@ void device_destroy(struct class *class, dev_t devt)
{
struct device *dev;
- dev = class_find_device(class, NULL, &devt, __match_devt);
+ dev = class_find_device_by_devt(class, devt);
if (dev) {
put_device(dev);
device_unregister(dev);
@@ -3328,3 +3450,39 @@ void device_set_of_node_from_dev(struct device *dev, const struct device *dev2)
dev->of_node_reused = true;
}
EXPORT_SYMBOL_GPL(device_set_of_node_from_dev);
+
+int device_match_name(struct device *dev, const void *name)
+{
+ return sysfs_streq(dev_name(dev), name);
+}
+EXPORT_SYMBOL_GPL(device_match_name);
+
+int device_match_of_node(struct device *dev, const void *np)
+{
+ return dev->of_node == np;
+}
+EXPORT_SYMBOL_GPL(device_match_of_node);
+
+int device_match_fwnode(struct device *dev, const void *fwnode)
+{
+ return dev_fwnode(dev) == fwnode;
+}
+EXPORT_SYMBOL_GPL(device_match_fwnode);
+
+int device_match_devt(struct device *dev, const void *pdevt)
+{
+ return dev->devt == *(dev_t *)pdevt;
+}
+EXPORT_SYMBOL_GPL(device_match_devt);
+
+int device_match_acpi_dev(struct device *dev, const void *adev)
+{
+ return ACPI_COMPANION(dev) == adev;
+}
+EXPORT_SYMBOL(device_match_acpi_dev);
+
+int device_match_any(struct device *dev, const void *unused)
+{
+ return 1;
+}
+EXPORT_SYMBOL_GPL(device_match_any);
diff --git a/drivers/base/dd.c b/drivers/base/dd.c
index 0df9b4461766..d811e60610d3 100644
--- a/drivers/base/dd.c
+++ b/drivers/base/dd.c
@@ -235,6 +235,19 @@ static int __init deferred_probe_timeout_setup(char *str)
}
__setup("deferred_probe_timeout=", deferred_probe_timeout_setup);
+static int __driver_deferred_probe_check_state(struct device *dev)
+{
+ if (!initcalls_done)
+ return -EPROBE_DEFER;
+
+ if (!deferred_probe_timeout) {
+ dev_WARN(dev, "deferred probe timeout, ignoring dependency");
+ return -ETIMEDOUT;
+ }
+
+ return 0;
+}
+
/**
* driver_deferred_probe_check_state() - Check deferred probe state
* @dev: device to check
@@ -248,14 +261,40 @@ __setup("deferred_probe_timeout=", deferred_probe_timeout_setup);
*/
int driver_deferred_probe_check_state(struct device *dev)
{
- if (initcalls_done) {
- if (!deferred_probe_timeout) {
- dev_WARN(dev, "deferred probe timeout, ignoring dependency");
- return -ETIMEDOUT;
- }
- dev_warn(dev, "ignoring dependency for device, assuming no driver");
- return -ENODEV;
- }
+ int ret;
+
+ ret = __driver_deferred_probe_check_state(dev);
+ if (ret < 0)
+ return ret;
+
+ dev_warn(dev, "ignoring dependency for device, assuming no driver");
+
+ return -ENODEV;
+}
+
+/**
+ * driver_deferred_probe_check_state_continue() - check deferred probe state
+ * @dev: device to check
+ *
+ * Returns -ETIMEDOUT if deferred probe debug timeout has expired, or
+ * -EPROBE_DEFER otherwise.
+ *
+ * Drivers or subsystems can opt-in to calling this function instead of
+ * directly returning -EPROBE_DEFER.
+ *
+ * This is similar to driver_deferred_probe_check_state(), but it allows the
+ * subsystem to keep deferring probe after built-in drivers have had a chance
+ * to probe. One scenario where that is useful is if built-in drivers rely on
+ * resources that are provided by modular drivers.
+ */
+int driver_deferred_probe_check_state_continue(struct device *dev)
+{
+ int ret;
+
+ ret = __driver_deferred_probe_check_state(dev);
+ if (ret < 0)
+ return ret;
+
return -EPROBE_DEFER;
}
@@ -515,9 +554,16 @@ re_probe:
goto probe_failed;
}
+ if (device_add_groups(dev, drv->dev_groups)) {
+ dev_err(dev, "device_add_groups() failed\n");
+ goto dev_groups_failed;
+ }
+
if (test_remove) {
test_remove = false;
+ device_remove_groups(dev, drv->dev_groups);
+
if (dev->bus->remove)
dev->bus->remove(dev);
else if (drv->remove)
@@ -545,6 +591,11 @@ re_probe:
drv->bus->name, __func__, dev_name(dev), drv->name);
goto done;
+dev_groups_failed:
+ if (dev->bus->remove)
+ dev->bus->remove(dev);
+ else if (drv->remove)
+ drv->remove(dev);
probe_failed:
if (dev->bus)
blocking_notifier_call_chain(&dev->bus->p->bus_notifier,
@@ -1075,6 +1126,8 @@ static void __device_release_driver(struct device *dev, struct device *parent)
pm_runtime_put_sync(dev);
+ device_remove_groups(dev, drv->dev_groups);
+
if (dev->bus && dev->bus->remove)
dev->bus->remove(dev);
else if (drv->remove)
diff --git a/drivers/base/devcon.c b/drivers/base/devcon.c
index 04db9ae235e4..14e2178e09f8 100644
--- a/drivers/base/devcon.c
+++ b/drivers/base/devcon.c
@@ -12,9 +12,6 @@
static DEFINE_MUTEX(devcon_lock);
static LIST_HEAD(devcon_list);
-typedef void *(*devcon_match_fn_t)(struct device_connection *con, int ep,
- void *data);
-
static void *
fwnode_graph_devcon_match(struct fwnode_handle *fwnode, const char *con_id,
void *data, devcon_match_fn_t match)
@@ -38,6 +35,56 @@ fwnode_graph_devcon_match(struct fwnode_handle *fwnode, const char *con_id,
return NULL;
}
+static void *
+fwnode_devcon_match(struct fwnode_handle *fwnode, const char *con_id,
+ void *data, devcon_match_fn_t match)
+{
+ struct device_connection con = { };
+ void *ret;
+ int i;
+
+ for (i = 0; ; i++) {
+ con.fwnode = fwnode_find_reference(fwnode, con_id, i);
+ if (IS_ERR(con.fwnode))
+ break;
+
+ ret = match(&con, -1, data);
+ fwnode_handle_put(con.fwnode);
+ if (ret)
+ return ret;
+ }
+
+ return NULL;
+}
+
+/**
+ * fwnode_connection_find_match - Find connection from a device node
+ * @fwnode: Device node with the connection
+ * @con_id: Identifier for the connection
+ * @data: Data for the match function
+ * @match: Function to check and convert the connection description
+ *
+ * Find a connection with unique identifier @con_id between @fwnode and another
+ * device node. @match will be used to convert the connection description to
+ * data the caller is expecting to be returned.
+ */
+void *fwnode_connection_find_match(struct fwnode_handle *fwnode,
+ const char *con_id, void *data,
+ devcon_match_fn_t match)
+{
+ void *ret;
+
+ if (!fwnode || !match)
+ return NULL;
+
+ ret = fwnode_graph_devcon_match(fwnode, con_id, data, match);
+ if (ret)
+ return ret;
+
+ return fwnode_devcon_match(fwnode, con_id, data, match);
+}
+EXPORT_SYMBOL_GPL(fwnode_connection_find_match);
+
/**
* device_connection_find_match - Find physical connection to a device
* @dev: Device with the connection
@@ -61,11 +108,9 @@ void *device_connection_find_match(struct device *dev, const char *con_id,
if (!match)
return NULL;
- if (fwnode) {
- ret = fwnode_graph_devcon_match(fwnode, con_id, data, match);
- if (ret)
- return ret;
- }
+ ret = fwnode_connection_find_match(fwnode, con_id, data, match);
+ if (ret)
+ return ret;
mutex_lock(&devcon_lock);
@@ -107,19 +152,13 @@ static struct bus_type *generic_match_buses[] = {
NULL,
};
-static int device_fwnode_match(struct device *dev, void *fwnode)
-{
- return dev_fwnode(dev) == fwnode;
-}
-
static void *device_connection_fwnode_match(struct device_connection *con)
{
struct bus_type *bus;
struct device *dev;
for (bus = generic_match_buses[0]; bus; bus++) {
- dev = bus_find_device(bus, NULL, (void *)con->fwnode,
- device_fwnode_match);
+ dev = bus_find_device_by_fwnode(bus, con->fwnode);
if (dev && !strncmp(dev_name(dev), con->id, strlen(con->id)))
return dev;
diff --git a/drivers/base/devcoredump.c b/drivers/base/devcoredump.c
index f1a3353f3494..e42d0b514384 100644
--- a/drivers/base/devcoredump.c
+++ b/drivers/base/devcoredump.c
@@ -164,16 +164,7 @@ static struct class devcd_class = {
static ssize_t devcd_readv(char *buffer, loff_t offset, size_t count,
void *data, size_t datalen)
{
- if (offset > datalen)
- return -EINVAL;
-
- if (offset + count > datalen)
- count = datalen - offset;
-
- if (count)
- memcpy(buffer, ((u8 *)data) + offset, count);
-
- return count;
+ return memory_read_from_buffer(buffer, count, &offset, data, datalen);
}
static void devcd_freev(void *data)
@@ -323,7 +314,7 @@ void dev_coredumpm(struct device *dev, struct module *owner,
EXPORT_SYMBOL_GPL(dev_coredumpm);
/**
- * dev_coredumpmsg - create device coredump that uses scatterlist as data
+ * dev_coredumpsg - create device coredump that uses scatterlist as data
* parameter
* @dev: the struct device for the crashed device
* @table: the dump data
diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c
index 0dbc43068eeb..ba5c80903efe 100644
--- a/drivers/base/devtmpfs.c
+++ b/drivers/base/devtmpfs.c
@@ -357,8 +357,7 @@ int devtmpfs_mount(const char *mntdir)
if (!thread)
return 0;
- err = ksys_mount("devtmpfs", (char *)mntdir, "devtmpfs", MS_SILENT,
- NULL);
+ err = ksys_mount("devtmpfs", mntdir, "devtmpfs", MS_SILENT, NULL);
if (err)
printk(KERN_INFO "devtmpfs: error mounting %i\n", err);
else
diff --git a/drivers/base/driver.c b/drivers/base/driver.c
index 857c8f1b876e..4e5ca632f35e 100644
--- a/drivers/base/driver.c
+++ b/drivers/base/driver.c
@@ -73,8 +73,8 @@ EXPORT_SYMBOL_GPL(driver_for_each_device);
* return to the caller and not iterate over any more devices.
*/
struct device *driver_find_device(struct device_driver *drv,
- struct device *start, void *data,
- int (*match)(struct device *dev, void *data))
+ struct device *start, const void *data,
+ int (*match)(struct device *dev, const void *data))
{
struct klist_iter i;
struct device *dev;
diff --git a/drivers/base/firmware_loader/Kconfig b/drivers/base/firmware_loader/Kconfig
index 38f2da6f5c2b..3f9e274e2ed3 100644
--- a/drivers/base/firmware_loader/Kconfig
+++ b/drivers/base/firmware_loader/Kconfig
@@ -26,6 +26,9 @@ config FW_LOADER
if FW_LOADER
+config FW_LOADER_PAGED_BUF
+ bool
+
config EXTRA_FIRMWARE
string "Build named firmware blobs into the kernel binary"
help
@@ -67,6 +70,7 @@ config EXTRA_FIRMWARE_DIR
config FW_LOADER_USER_HELPER
bool "Enable the firmware sysfs fallback mechanism"
+ select FW_LOADER_PAGED_BUF
help
This option enables a sysfs loading facility to enable firmware
loading to the kernel through userspace as a fallback mechanism
@@ -151,5 +155,19 @@ config FW_LOADER_USER_HELPER_FALLBACK
If you are unsure about this, say N here.
+config FW_LOADER_COMPRESS
+ bool "Enable compressed firmware support"
+ select FW_LOADER_PAGED_BUF
+ select XZ_DEC
+ help
+ This option enables the support for loading compressed firmware
+ files. The caller of firmware API receives the decompressed file
+ content. The compressed file is loaded as a fallback, only after
+ loading the raw file failed at first.
+
+ Currently only XZ-compressed files are supported, and they have to
+ be compressed with either none or crc32 integrity check type (pass
+ "-C crc32" option to xz command).
+
endif # FW_LOADER
endmenu
diff --git a/drivers/base/firmware_loader/fallback.c b/drivers/base/firmware_loader/fallback.c
index f962488546b6..62ee90b4db56 100644
--- a/drivers/base/firmware_loader/fallback.c
+++ b/drivers/base/firmware_loader/fallback.c
@@ -219,20 +219,6 @@ static ssize_t firmware_loading_show(struct device *dev,
return sprintf(buf, "%d\n", loading);
}
-/* one pages buffer should be mapped/unmapped only once */
-static int map_fw_priv_pages(struct fw_priv *fw_priv)
-{
- if (!fw_priv->is_paged_buf)
- return 0;
-
- vunmap(fw_priv->data);
- fw_priv->data = vmap(fw_priv->pages, fw_priv->nr_pages, 0,
- PAGE_KERNEL_RO);
- if (!fw_priv->data)
- return -ENOMEM;
- return 0;
-}
-
/**
* firmware_loading_store() - set value in the 'loading' control file
* @dev: device pointer
@@ -254,7 +240,6 @@ static ssize_t firmware_loading_store(struct device *dev,
struct fw_priv *fw_priv;
ssize_t written = count;
int loading = simple_strtol(buf, NULL, 10);
- int i;
mutex_lock(&fw_lock);
fw_priv = fw_sysfs->fw_priv;
@@ -265,12 +250,7 @@ static ssize_t firmware_loading_store(struct device *dev,
case 1:
/* discarding any previous partial load */
if (!fw_sysfs_done(fw_priv)) {
- for (i = 0; i < fw_priv->nr_pages; i++)
- __free_page(fw_priv->pages[i]);
- vfree(fw_priv->pages);
- fw_priv->pages = NULL;
- fw_priv->page_array_size = 0;
- fw_priv->nr_pages = 0;
+ fw_free_paged_buf(fw_priv);
fw_state_start(fw_priv);
}
break;
@@ -284,7 +264,7 @@ static ssize_t firmware_loading_store(struct device *dev,
* see the mapped 'buf->data' once the loading
* is completed.
* */
- rc = map_fw_priv_pages(fw_priv);
+ rc = fw_map_paged_buf(fw_priv);
if (rc)
dev_err(dev, "%s: map pages failed\n",
__func__);
@@ -389,40 +369,13 @@ out:
static int fw_realloc_pages(struct fw_sysfs *fw_sysfs, int min_size)
{
- struct fw_priv *fw_priv= fw_sysfs->fw_priv;
- int pages_needed = PAGE_ALIGN(min_size) >> PAGE_SHIFT;
-
- /* If the array of pages is too small, grow it... */
- if (fw_priv->page_array_size < pages_needed) {
- int new_array_size = max(pages_needed,
- fw_priv->page_array_size * 2);
- struct page **new_pages;
+ int err;
- new_pages = vmalloc(array_size(new_array_size, sizeof(void *)));
- if (!new_pages) {
- fw_load_abort(fw_sysfs);
- return -ENOMEM;
- }
- memcpy(new_pages, fw_priv->pages,
- fw_priv->page_array_size * sizeof(void *));
- memset(&new_pages[fw_priv->page_array_size], 0, sizeof(void *) *
- (new_array_size - fw_priv->page_array_size));
- vfree(fw_priv->pages);
- fw_priv->pages = new_pages;
- fw_priv->page_array_size = new_array_size;
- }
-
- while (fw_priv->nr_pages < pages_needed) {
- fw_priv->pages[fw_priv->nr_pages] =
- alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
-
- if (!fw_priv->pages[fw_priv->nr_pages]) {
- fw_load_abort(fw_sysfs);
- return -ENOMEM;
- }
- fw_priv->nr_pages++;
- }
- return 0;
+ err = fw_grow_paged_buf(fw_sysfs->fw_priv,
+ PAGE_ALIGN(min_size) >> PAGE_SHIFT);
+ if (err)
+ fw_load_abort(fw_sysfs);
+ return err;
}
/**
@@ -659,7 +612,7 @@ static bool fw_run_sysfs_fallback(enum fw_opt opt_flags)
/* Also permit LSMs and IMA to fail firmware sysfs fallback */
ret = security_kernel_load_data(LOADING_FIRMWARE);
if (ret < 0)
- return ret;
+ return false;
return fw_force_sysfs_fallback(opt_flags);
}
diff --git a/drivers/base/firmware_loader/fallback_table.c b/drivers/base/firmware_loader/fallback_table.c
index 776dd69cf5be..ba9d30b28edc 100644
--- a/drivers/base/firmware_loader/fallback_table.c
+++ b/drivers/base/firmware_loader/fallback_table.c
@@ -16,9 +16,6 @@
* firmware fallback configuration table
*/
-static unsigned int zero;
-static unsigned int one = 1;
-
struct firmware_fallback_config fw_fallback_config = {
.force_sysfs_fallback = IS_ENABLED(CONFIG_FW_LOADER_USER_HELPER_FALLBACK),
.loading_timeout = 60,
@@ -26,6 +23,7 @@ struct firmware_fallback_config fw_fallback_config = {
};
EXPORT_SYMBOL_GPL(fw_fallback_config);
+#ifdef CONFIG_SYSCTL
struct ctl_table firmware_config_table[] = {
{
.procname = "force_sysfs_fallback",
@@ -33,8 +31,8 @@ struct ctl_table firmware_config_table[] = {
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_douintvec_minmax,
- .extra1 = &zero,
- .extra2 = &one,
+ .extra1 = SYSCTL_ZERO,
+ .extra2 = SYSCTL_ONE,
},
{
.procname = "ignore_sysfs_fallback",
@@ -42,9 +40,10 @@ struct ctl_table firmware_config_table[] = {
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_douintvec_minmax,
- .extra1 = &zero,
- .extra2 = &one,
+ .extra1 = SYSCTL_ZERO,
+ .extra2 = SYSCTL_ONE,
},
{ }
};
EXPORT_SYMBOL_GPL(firmware_config_table);
+#endif
diff --git a/drivers/base/firmware_loader/firmware.h b/drivers/base/firmware_loader/firmware.h
index 4c1395f8e7ed..7ecd590e67fe 100644
--- a/drivers/base/firmware_loader/firmware.h
+++ b/drivers/base/firmware_loader/firmware.h
@@ -64,12 +64,14 @@ struct fw_priv {
void *data;
size_t size;
size_t allocated_size;
-#ifdef CONFIG_FW_LOADER_USER_HELPER
+#ifdef CONFIG_FW_LOADER_PAGED_BUF
bool is_paged_buf;
- bool need_uevent;
struct page **pages;
int nr_pages;
int page_array_size;
+#endif
+#ifdef CONFIG_FW_LOADER_USER_HELPER
+ bool need_uevent;
struct list_head pending_list;
#endif
const char *fw_name;
@@ -133,4 +135,14 @@ static inline void fw_state_done(struct fw_priv *fw_priv)
int assign_fw(struct firmware *fw, struct device *device,
enum fw_opt opt_flags);
+#ifdef CONFIG_FW_LOADER_PAGED_BUF
+void fw_free_paged_buf(struct fw_priv *fw_priv);
+int fw_grow_paged_buf(struct fw_priv *fw_priv, int pages_needed);
+int fw_map_paged_buf(struct fw_priv *fw_priv);
+#else
+static inline void fw_free_paged_buf(struct fw_priv *fw_priv) {}
+static inline int fw_grow_paged_buf(struct fw_priv *fw_priv, int pages_needed) { return -ENXIO; }
+static inline int fw_map_paged_buf(struct fw_priv *fw_priv) { return -ENXIO; }
+#endif
+
#endif /* __FIRMWARE_LOADER_H */
diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c
index 7eaaf5ee5ba6..bf44c79beae9 100644
--- a/drivers/base/firmware_loader/main.c
+++ b/drivers/base/firmware_loader/main.c
@@ -33,6 +33,7 @@
#include <linux/syscore_ops.h>
#include <linux/reboot.h>
#include <linux/security.h>
+#include <linux/xz.h>
#include <generated/utsrelease.h>
@@ -251,15 +252,7 @@ static void __free_fw_priv(struct kref *ref)
list_del(&fw_priv->list);
spin_unlock(&fwc->lock);
-#ifdef CONFIG_FW_LOADER_USER_HELPER
- if (fw_priv->is_paged_buf) {
- int i;
- vunmap(fw_priv->data);
- for (i = 0; i < fw_priv->nr_pages; i++)
- __free_page(fw_priv->pages[i]);
- vfree(fw_priv->pages);
- } else
-#endif
+ fw_free_paged_buf(fw_priv); /* free leftover pages */
if (!fw_priv->allocated_size)
vfree(fw_priv->data);
kfree_const(fw_priv->fw_name);
@@ -274,6 +267,174 @@ static void free_fw_priv(struct fw_priv *fw_priv)
spin_unlock(&fwc->lock);
}
+#ifdef CONFIG_FW_LOADER_PAGED_BUF
+void fw_free_paged_buf(struct fw_priv *fw_priv)
+{
+ int i;
+
+ if (!fw_priv->pages)
+ return;
+
+ for (i = 0; i < fw_priv->nr_pages; i++)
+ __free_page(fw_priv->pages[i]);
+ kvfree(fw_priv->pages);
+ fw_priv->pages = NULL;
+ fw_priv->page_array_size = 0;
+ fw_priv->nr_pages = 0;
+}
+
+int fw_grow_paged_buf(struct fw_priv *fw_priv, int pages_needed)
+{
+ /* If the array of pages is too small, grow it */
+ if (fw_priv->page_array_size < pages_needed) {
+ int new_array_size = max(pages_needed,
+ fw_priv->page_array_size * 2);
+ struct page **new_pages;
+
+ new_pages = kvmalloc_array(new_array_size, sizeof(void *),
+ GFP_KERNEL);
+ if (!new_pages)
+ return -ENOMEM;
+ memcpy(new_pages, fw_priv->pages,
+ fw_priv->page_array_size * sizeof(void *));
+ memset(&new_pages[fw_priv->page_array_size], 0, sizeof(void *) *
+ (new_array_size - fw_priv->page_array_size));
+ kvfree(fw_priv->pages);
+ fw_priv->pages = new_pages;
+ fw_priv->page_array_size = new_array_size;
+ }
+
+ while (fw_priv->nr_pages < pages_needed) {
+ fw_priv->pages[fw_priv->nr_pages] =
+ alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
+
+ if (!fw_priv->pages[fw_priv->nr_pages])
+ return -ENOMEM;
+ fw_priv->nr_pages++;
+ }
+
+ return 0;
+}
+
+int fw_map_paged_buf(struct fw_priv *fw_priv)
+{
+ /* one pages buffer should be mapped/unmapped only once */
+ if (!fw_priv->pages)
+ return 0;
+
+ vunmap(fw_priv->data);
+ fw_priv->data = vmap(fw_priv->pages, fw_priv->nr_pages, 0,
+ PAGE_KERNEL_RO);
+ if (!fw_priv->data)
+ return -ENOMEM;
+
+ /* page table is no longer needed after mapping, let's free */
+ kvfree(fw_priv->pages);
+ fw_priv->pages = NULL;
+
+ return 0;
+}
+#endif
+
+/*
+ * XZ-compressed firmware support
+ */
+#ifdef CONFIG_FW_LOADER_COMPRESS
+/* show an error and return the standard error code */
+static int fw_decompress_xz_error(struct device *dev, enum xz_ret xz_ret)
+{
+ if (xz_ret != XZ_STREAM_END) {
+ dev_warn(dev, "xz decompression failed (xz_ret=%d)\n", xz_ret);
+ return xz_ret == XZ_MEM_ERROR ? -ENOMEM : -EINVAL;
+ }
+ return 0;
+}
+
+/* single-shot decompression onto the pre-allocated buffer */
+static int fw_decompress_xz_single(struct device *dev, struct fw_priv *fw_priv,
+ size_t in_size, const void *in_buffer)
+{
+ struct xz_dec *xz_dec;
+ struct xz_buf xz_buf;
+ enum xz_ret xz_ret;
+
+ xz_dec = xz_dec_init(XZ_SINGLE, (u32)-1);
+ if (!xz_dec)
+ return -ENOMEM;
+
+ xz_buf.in_size = in_size;
+ xz_buf.in = in_buffer;
+ xz_buf.in_pos = 0;
+ xz_buf.out_size = fw_priv->allocated_size;
+ xz_buf.out = fw_priv->data;
+ xz_buf.out_pos = 0;
+
+ xz_ret = xz_dec_run(xz_dec, &xz_buf);
+ xz_dec_end(xz_dec);
+
+ fw_priv->size = xz_buf.out_pos;
+ return fw_decompress_xz_error(dev, xz_ret);
+}
+
+/* decompression on paged buffer and map it */
+static int fw_decompress_xz_pages(struct device *dev, struct fw_priv *fw_priv,
+ size_t in_size, const void *in_buffer)
+{
+ struct xz_dec *xz_dec;
+ struct xz_buf xz_buf;
+ enum xz_ret xz_ret;
+ struct page *page;
+ int err = 0;
+
+ xz_dec = xz_dec_init(XZ_DYNALLOC, (u32)-1);
+ if (!xz_dec)
+ return -ENOMEM;
+
+ xz_buf.in_size = in_size;
+ xz_buf.in = in_buffer;
+ xz_buf.in_pos = 0;
+
+ fw_priv->is_paged_buf = true;
+ fw_priv->size = 0;
+ do {
+ if (fw_grow_paged_buf(fw_priv, fw_priv->nr_pages + 1)) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ /* decompress onto the new allocated page */
+ page = fw_priv->pages[fw_priv->nr_pages - 1];
+ xz_buf.out = kmap(page);
+ xz_buf.out_pos = 0;
+ xz_buf.out_size = PAGE_SIZE;
+ xz_ret = xz_dec_run(xz_dec, &xz_buf);
+ kunmap(page);
+ fw_priv->size += xz_buf.out_pos;
+ /* partial decompression means either end or error */
+ if (xz_buf.out_pos != PAGE_SIZE)
+ break;
+ } while (xz_ret == XZ_OK);
+
+ err = fw_decompress_xz_error(dev, xz_ret);
+ if (!err)
+ err = fw_map_paged_buf(fw_priv);
+
+ out:
+ xz_dec_end(xz_dec);
+ return err;
+}
+
+static int fw_decompress_xz(struct device *dev, struct fw_priv *fw_priv,
+ size_t in_size, const void *in_buffer)
+{
+ /* if the buffer is pre-allocated, we can perform in single-shot mode */
+ if (fw_priv->data)
+ return fw_decompress_xz_single(dev, fw_priv, in_size, in_buffer);
+ else
+ return fw_decompress_xz_pages(dev, fw_priv, in_size, in_buffer);
+}
+#endif /* CONFIG_FW_LOADER_COMPRESS */
+
/* direct firmware loading support */
static char fw_path_para[256];
static const char * const fw_path[] = {
@@ -293,7 +454,12 @@ module_param_string(path, fw_path_para, sizeof(fw_path_para), 0644);
MODULE_PARM_DESC(path, "customized firmware image search path with a higher priority than default path");
static int
-fw_get_filesystem_firmware(struct device *device, struct fw_priv *fw_priv)
+fw_get_filesystem_firmware(struct device *device, struct fw_priv *fw_priv,
+ const char *suffix,
+ int (*decompress)(struct device *dev,
+ struct fw_priv *fw_priv,
+ size_t in_size,
+ const void *in_buffer))
{
loff_t size;
int i, len;
@@ -301,9 +467,11 @@ fw_get_filesystem_firmware(struct device *device, struct fw_priv *fw_priv)
char *path;
enum kernel_read_file_id id = READING_FIRMWARE;
size_t msize = INT_MAX;
+ void *buffer = NULL;
/* Already populated data member means we're loading into a buffer */
- if (fw_priv->data) {
+ if (!decompress && fw_priv->data) {
+ buffer = fw_priv->data;
id = READING_FIRMWARE_PREALLOC_BUFFER;
msize = fw_priv->allocated_size;
}
@@ -317,15 +485,15 @@ fw_get_filesystem_firmware(struct device *device, struct fw_priv *fw_priv)
if (!fw_path[i][0])
continue;
- len = snprintf(path, PATH_MAX, "%s/%s",
- fw_path[i], fw_priv->fw_name);
+ len = snprintf(path, PATH_MAX, "%s/%s%s",
+ fw_path[i], fw_priv->fw_name, suffix);
if (len >= PATH_MAX) {
rc = -ENAMETOOLONG;
break;
}
fw_priv->size = 0;
- rc = kernel_read_file_from_path(path, &fw_priv->data, &size,
+ rc = kernel_read_file_from_path(path, &buffer, &size,
msize, id);
if (rc) {
if (rc != -ENOENT)
@@ -336,8 +504,24 @@ fw_get_filesystem_firmware(struct device *device, struct fw_priv *fw_priv)
path);
continue;
}
- dev_dbg(device, "direct-loading %s\n", fw_priv->fw_name);
- fw_priv->size = size;
+ if (decompress) {
+ dev_dbg(device, "f/w decompressing %s\n",
+ fw_priv->fw_name);
+ rc = decompress(device, fw_priv, size, buffer);
+ /* discard the superfluous original content */
+ vfree(buffer);
+ buffer = NULL;
+ if (rc) {
+ fw_free_paged_buf(fw_priv);
+ continue;
+ }
+ } else {
+ dev_dbg(device, "direct-loading %s\n",
+ fw_priv->fw_name);
+ if (!fw_priv->data)
+ fw_priv->data = buffer;
+ fw_priv->size = size;
+ }
fw_state_done(fw_priv);
break;
}
@@ -584,7 +768,13 @@ _request_firmware(const struct firmware **firmware_p, const char *name,
if (ret <= 0) /* error or already assigned */
goto out;
- ret = fw_get_filesystem_firmware(device, fw->priv);
+ ret = fw_get_filesystem_firmware(device, fw->priv, "", NULL);
+#ifdef CONFIG_FW_LOADER_COMPRESS
+ if (ret == -ENOENT)
+ ret = fw_get_filesystem_firmware(device, fw->priv, ".xz",
+ fw_decompress_xz);
+#endif
+
if (ret) {
if (!(opt_flags & FW_OPT_NO_WARN))
dev_warn(device,
diff --git a/drivers/base/memory.c b/drivers/base/memory.c
index f180427e48f4..20c39d1bcef8 100644
--- a/drivers/base/memory.c
+++ b/drivers/base/memory.c
@@ -34,11 +34,21 @@ static DEFINE_MUTEX(mem_sysfs_mutex);
static int sections_per_block;
-static inline int base_memory_block_id(int section_nr)
+static inline unsigned long base_memory_block_id(unsigned long section_nr)
{
return section_nr / sections_per_block;
}
+static inline unsigned long pfn_to_block_id(unsigned long pfn)
+{
+ return base_memory_block_id(pfn_to_section_nr(pfn));
+}
+
+static inline unsigned long phys_to_block_id(unsigned long phys)
+{
+ return pfn_to_block_id(PFN_DOWN(phys));
+}
+
static int memory_subsys_online(struct device *dev);
static int memory_subsys_offline(struct device *dev);
@@ -126,9 +136,9 @@ static ssize_t phys_index_show(struct device *dev,
static ssize_t removable_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
- unsigned long i, pfn;
- int ret = 1;
struct memory_block *mem = to_memory_block(dev);
+ unsigned long pfn;
+ int ret = 1, i;
if (mem->state != MEM_ONLINE)
goto out;
@@ -578,23 +588,13 @@ int __weak arch_get_memory_phys_device(unsigned long start_pfn)
return 0;
}
-/*
- * A reference for the returned object is held and the reference for the
- * hinted object is released.
- */
-struct memory_block *find_memory_block_hinted(struct mem_section *section,
- struct memory_block *hint)
+/* A reference for the returned memory block device is acquired. */
+static struct memory_block *find_memory_block_by_id(unsigned long block_id)
{
- int block_id = base_memory_block_id(__section_nr(section));
- struct device *hintdev = hint ? &hint->dev : NULL;
struct device *dev;
- dev = subsys_find_device_by_id(&memory_subsys, block_id, hintdev);
- if (hint)
- put_device(&hint->dev);
- if (!dev)
- return NULL;
- return to_memory_block(dev);
+ dev = subsys_find_device_by_id(&memory_subsys, block_id, NULL);
+ return dev ? to_memory_block(dev) : NULL;
}
/*
@@ -607,7 +607,9 @@ struct memory_block *find_memory_block_hinted(struct mem_section *section,
*/
struct memory_block *find_memory_block(struct mem_section *section)
{
- return find_memory_block_hinted(section, NULL);
+ unsigned long block_id = base_memory_block_id(__section_nr(section));
+
+ return find_memory_block_by_id(block_id);
}
static struct attribute *memory_memblk_attrs[] = {
@@ -652,20 +654,22 @@ int register_memory(struct memory_block *memory)
}
static int init_memory_block(struct memory_block **memory,
- struct mem_section *section, unsigned long state)
+ unsigned long block_id, unsigned long state)
{
struct memory_block *mem;
unsigned long start_pfn;
- int scn_nr;
int ret = 0;
+ mem = find_memory_block_by_id(block_id);
+ if (mem) {
+ put_device(&mem->dev);
+ return -EEXIST;
+ }
mem = kzalloc(sizeof(*mem), GFP_KERNEL);
if (!mem)
return -ENOMEM;
- scn_nr = __section_nr(section);
- mem->start_section_nr =
- base_memory_block_id(scn_nr) * sections_per_block;
+ mem->start_section_nr = block_id * sections_per_block;
mem->end_section_nr = mem->start_section_nr + sections_per_block - 1;
mem->state = state;
start_pfn = section_nr_to_pfn(mem->start_section_nr);
@@ -677,97 +681,101 @@ static int init_memory_block(struct memory_block **memory,
return ret;
}
-static int add_memory_block(int base_section_nr)
+static int add_memory_block(unsigned long base_section_nr)
{
+ int ret, section_count = 0;
struct memory_block *mem;
- int i, ret, section_count = 0, section_nr;
+ unsigned long nr;
- for (i = base_section_nr;
- i < base_section_nr + sections_per_block;
- i++) {
- if (!present_section_nr(i))
- continue;
- if (section_count == 0)
- section_nr = i;
- section_count++;
- }
+ for (nr = base_section_nr; nr < base_section_nr + sections_per_block;
+ nr++)
+ if (present_section_nr(nr))
+ section_count++;
if (section_count == 0)
return 0;
- ret = init_memory_block(&mem, __nr_to_section(section_nr), MEM_ONLINE);
+ ret = init_memory_block(&mem, base_memory_block_id(base_section_nr),
+ MEM_ONLINE);
if (ret)
return ret;
mem->section_count = section_count;
return 0;
}
+static void unregister_memory(struct memory_block *memory)
+{
+ if (WARN_ON_ONCE(memory->dev.bus != &memory_subsys))
+ return;
+
+ /* drop the ref. we got via find_memory_block() */
+ put_device(&memory->dev);
+ device_unregister(&memory->dev);
+}
+
/*
- * need an interface for the VM to add new memory regions,
- * but without onlining it.
+ * Create memory block devices for the given memory area. Start and size
+ * have to be aligned to memory block granularity. Memory block devices
+ * will be initialized as offline.
*/
-int hotplug_memory_register(int nid, struct mem_section *section)
+int create_memory_block_devices(unsigned long start, unsigned long size)
{
- int ret = 0;
+ const unsigned long start_block_id = pfn_to_block_id(PFN_DOWN(start));
+ unsigned long end_block_id = pfn_to_block_id(PFN_DOWN(start + size));
struct memory_block *mem;
+ unsigned long block_id;
+ int ret = 0;
- mutex_lock(&mem_sysfs_mutex);
+ if (WARN_ON_ONCE(!IS_ALIGNED(start, memory_block_size_bytes()) ||
+ !IS_ALIGNED(size, memory_block_size_bytes())))
+ return -EINVAL;
- mem = find_memory_block(section);
- if (mem) {
- mem->section_count++;
- put_device(&mem->dev);
- } else {
- ret = init_memory_block(&mem, section, MEM_OFFLINE);
+ mutex_lock(&mem_sysfs_mutex);
+ for (block_id = start_block_id; block_id != end_block_id; block_id++) {
+ ret = init_memory_block(&mem, block_id, MEM_OFFLINE);
if (ret)
- goto out;
- mem->section_count++;
+ break;
+ mem->section_count = sections_per_block;
+ }
+ if (ret) {
+ end_block_id = block_id;
+ for (block_id = start_block_id; block_id != end_block_id;
+ block_id++) {
+ mem = find_memory_block_by_id(block_id);
+ mem->section_count = 0;
+ unregister_memory(mem);
+ }
}
-
-out:
mutex_unlock(&mem_sysfs_mutex);
return ret;
}
-#ifdef CONFIG_MEMORY_HOTREMOVE
-static void
-unregister_memory(struct memory_block *memory)
-{
- BUG_ON(memory->dev.bus != &memory_subsys);
-
- /* drop the ref. we got via find_memory_block() */
- put_device(&memory->dev);
- device_unregister(&memory->dev);
-}
-
-void unregister_memory_section(struct mem_section *section)
+/*
+ * Remove memory block devices for the given memory area. Start and size
+ * have to be aligned to memory block granularity. Memory block devices
+ * have to be offline.
+ */
+void remove_memory_block_devices(unsigned long start, unsigned long size)
{
+ const unsigned long start_block_id = pfn_to_block_id(PFN_DOWN(start));
+ const unsigned long end_block_id = pfn_to_block_id(PFN_DOWN(start + size));
struct memory_block *mem;
+ unsigned long block_id;
- if (WARN_ON_ONCE(!present_section(section)))
+ if (WARN_ON_ONCE(!IS_ALIGNED(start, memory_block_size_bytes()) ||
+ !IS_ALIGNED(size, memory_block_size_bytes())))
return;
mutex_lock(&mem_sysfs_mutex);
-
- /*
- * Some users of the memory hotplug do not want/need memblock to
- * track all sections. Skip over those.
- */
- mem = find_memory_block(section);
- if (!mem)
- goto out_unlock;
-
- unregister_mem_sect_under_nodes(mem, __section_nr(section));
-
- mem->section_count--;
- if (mem->section_count == 0)
+ for (block_id = start_block_id; block_id != end_block_id; block_id++) {
+ mem = find_memory_block_by_id(block_id);
+ if (WARN_ON_ONCE(!mem))
+ continue;
+ mem->section_count = 0;
+ unregister_memory_block_under_nodes(mem);
unregister_memory(mem);
- else
- put_device(&mem->dev);
-
-out_unlock:
+ }
mutex_unlock(&mem_sysfs_mutex);
}
-#endif /* CONFIG_MEMORY_HOTREMOVE */
/* return true if the memory block is offlined, otherwise, return false */
bool is_memblock_offlined(struct memory_block *mem)
@@ -804,10 +812,9 @@ static const struct attribute_group *memory_root_attr_groups[] = {
*/
int __init memory_dev_init(void)
{
- unsigned int i;
int ret;
int err;
- unsigned long block_sz;
+ unsigned long block_sz, nr;
ret = subsys_system_register(&memory_subsys, memory_root_attr_groups);
if (ret)
@@ -821,9 +828,9 @@ int __init memory_dev_init(void)
* during boot and have been initialized
*/
mutex_lock(&mem_sysfs_mutex);
- for (i = 0; i <= __highest_present_section_nr;
- i += sections_per_block) {
- err = add_memory_block(i);
+ for (nr = 0; nr <= __highest_present_section_nr;
+ nr += sections_per_block) {
+ err = add_memory_block(nr);
if (!ret)
ret = err;
}
@@ -834,3 +841,43 @@ out:
printk(KERN_ERR "%s() failed: %d\n", __func__, ret);
return ret;
}
+
+/**
+ * walk_memory_blocks - walk through all present memory blocks overlapped
+ * by the range [start, start + size)
+ *
+ * @start: start address of the memory range
+ * @size: size of the memory range
+ * @arg: argument passed to func
+ * @func: callback for each memory section walked
+ *
+ * This function walks through all present memory blocks overlapped by the
+ * range [start, start + size), calling func on each memory block.
+ *
+ * In case func() returns an error, walking is aborted and the error is
+ * returned.
+ */
+int walk_memory_blocks(unsigned long start, unsigned long size,
+ void *arg, walk_memory_blocks_func_t func)
+{
+ const unsigned long start_block_id = phys_to_block_id(start);
+ const unsigned long end_block_id = phys_to_block_id(start + size - 1);
+ struct memory_block *mem;
+ unsigned long block_id;
+ int ret = 0;
+
+ if (!size)
+ return 0;
+
+ for (block_id = start_block_id; block_id <= end_block_id; block_id++) {
+ mem = find_memory_block_by_id(block_id);
+ if (!mem)
+ continue;
+
+ ret = func(mem, arg);
+ put_device(&mem->dev);
+ if (ret)
+ break;
+ }
+ return ret;
+}
diff --git a/drivers/base/node.c b/drivers/base/node.c
index 8598fcbd2a17..75b7e6f6535b 100644
--- a/drivers/base/node.c
+++ b/drivers/base/node.c
@@ -66,6 +66,7 @@ static DEVICE_ATTR(cpulist, S_IRUGO, node_read_cpulist, NULL);
* @dev: Device for this memory access class
* @list_node: List element in the node's access list
* @access: The access class rank
+ * @hmem_attrs: Heterogeneous memory performance attributes
*/
struct node_access_nodes {
struct device dev;
@@ -673,8 +674,8 @@ int register_cpu_under_node(unsigned int cpu, unsigned int nid)
/**
* register_memory_node_under_compute_node - link memory node to its compute
* node for a given access class.
- * @mem_node: Memory node number
- * @cpu_node: Cpu node number
+ * @mem_nid: Memory node number
+ * @cpu_nid: Cpu node number
* @access: Access class to register
*
* Description:
@@ -752,7 +753,8 @@ static int __ref get_nid_for_pfn(unsigned long pfn)
}
/* register memory section under specified node if it spans that node */
-int register_mem_sect_under_node(struct memory_block *mem_blk, void *arg)
+static int register_mem_sect_under_node(struct memory_block *mem_blk,
+ void *arg)
{
int ret, nid = *(int *)arg;
unsigned long pfn, sect_start_pfn, sect_end_pfn;
@@ -801,23 +803,18 @@ int register_mem_sect_under_node(struct memory_block *mem_blk, void *arg)
return 0;
}
-/* unregister memory section under all nodes that it spans */
-int unregister_mem_sect_under_nodes(struct memory_block *mem_blk,
- unsigned long phys_index)
+/*
+ * Unregister memory block device under all nodes that it spans.
+ * Has to be called with mem_sysfs_mutex held (due to unlinked_nodes).
+ */
+void unregister_memory_block_under_nodes(struct memory_block *mem_blk)
{
- NODEMASK_ALLOC(nodemask_t, unlinked_nodes, GFP_KERNEL);
unsigned long pfn, sect_start_pfn, sect_end_pfn;
+ static nodemask_t unlinked_nodes;
- if (!mem_blk) {
- NODEMASK_FREE(unlinked_nodes);
- return -EFAULT;
- }
- if (!unlinked_nodes)
- return -ENOMEM;
- nodes_clear(*unlinked_nodes);
-
- sect_start_pfn = section_nr_to_pfn(phys_index);
- sect_end_pfn = sect_start_pfn + PAGES_PER_SECTION - 1;
+ nodes_clear(unlinked_nodes);
+ sect_start_pfn = section_nr_to_pfn(mem_blk->start_section_nr);
+ sect_end_pfn = section_nr_to_pfn(mem_blk->end_section_nr);
for (pfn = sect_start_pfn; pfn <= sect_end_pfn; pfn++) {
int nid;
@@ -826,21 +823,20 @@ int unregister_mem_sect_under_nodes(struct memory_block *mem_blk,
continue;
if (!node_online(nid))
continue;
- if (node_test_and_set(nid, *unlinked_nodes))
+ if (node_test_and_set(nid, unlinked_nodes))
continue;
sysfs_remove_link(&node_devices[nid]->dev.kobj,
kobject_name(&mem_blk->dev.kobj));
sysfs_remove_link(&mem_blk->dev.kobj,
kobject_name(&node_devices[nid]->dev.kobj));
}
- NODEMASK_FREE(unlinked_nodes);
- return 0;
}
int link_mem_sections(int nid, unsigned long start_pfn, unsigned long end_pfn)
{
- return walk_memory_range(start_pfn, end_pfn, (void *)&nid,
- register_mem_sect_under_node);
+ return walk_memory_blocks(PFN_PHYS(start_pfn),
+ PFN_PHYS(end_pfn - start_pfn), (void *)&nid,
+ register_mem_sect_under_node);
}
#ifdef CONFIG_HUGETLBFS
diff --git a/drivers/base/platform.c b/drivers/base/platform.c
index 4d1729853d1a..b6c6c7d97d5b 100644
--- a/drivers/base/platform.c
+++ b/drivers/base/platform.c
@@ -5,7 +5,7 @@
* Copyright (c) 2002-3 Patrick Mochel
* Copyright (c) 2002-3 Open Source Development Labs
*
- * Please see Documentation/driver-model/platform.txt for more
+ * Please see Documentation/driver-api/driver-model/platform.rst for more
* information.
*/
@@ -40,25 +40,6 @@ struct device platform_bus = {
EXPORT_SYMBOL_GPL(platform_bus);
/**
- * arch_setup_pdev_archdata - Allow manipulation of archdata before its used
- * @pdev: platform device
- *
- * This is called before platform_device_add() such that any pdev_archdata may
- * be setup before the platform_notifier is called. So if a user needs to
- * manipulate any relevant information in the pdev_archdata they can do:
- *
- * platform_device_alloc()
- * ... manipulate ...
- * platform_device_add()
- *
- * And if they don't care they can just call platform_device_register() and
- * everything will just work out.
- */
-void __weak arch_setup_pdev_archdata(struct platform_device *pdev)
-{
-}
-
-/**
* platform_get_resource - get a resource for a device
* @dev: platform device
* @type: resource type
@@ -99,12 +80,7 @@ void __iomem *devm_platform_ioremap_resource(struct platform_device *pdev,
EXPORT_SYMBOL_GPL(devm_platform_ioremap_resource);
#endif /* CONFIG_HAS_IOMEM */
-/**
- * platform_get_irq - get an IRQ for a device
- * @dev: platform device
- * @num: IRQ number index
- */
-int platform_get_irq(struct platform_device *dev, unsigned int num)
+static int __platform_get_irq(struct platform_device *dev, unsigned int num)
{
#ifdef CONFIG_SPARC
/* sparc does not have irqs represented as IORESOURCE_IRQ resources */
@@ -157,15 +133,70 @@ int platform_get_irq(struct platform_device *dev, unsigned int num)
* the device will only expose one IRQ, and this fallback
* allows a common code path across either kind of resource.
*/
- if (num == 0 && has_acpi_companion(&dev->dev))
- return acpi_dev_gpio_irq_get(ACPI_COMPANION(&dev->dev), num);
+ if (num == 0 && has_acpi_companion(&dev->dev)) {
+ int ret = acpi_dev_gpio_irq_get(ACPI_COMPANION(&dev->dev), num);
+
+ /* Our callers expect -ENXIO for missing IRQs. */
+ if (ret >= 0 || ret == -EPROBE_DEFER)
+ return ret;
+ }
return -ENXIO;
#endif
}
+
+/**
+ * platform_get_irq - get an IRQ for a device
+ * @dev: platform device
+ * @num: IRQ number index
+ *
+ * Gets an IRQ for a platform device and prints an error message if finding the
+ * IRQ fails. Device drivers should check the return value for errors so as to
+ * not pass a negative integer value to the request_irq() APIs.
+ *
+ * Example:
+ * int irq = platform_get_irq(pdev, 0);
+ * if (irq < 0)
+ * return irq;
+ *
+ * Return: IRQ number on success, negative error number on failure.
+ */
+int platform_get_irq(struct platform_device *dev, unsigned int num)
+{
+ int ret;
+
+ ret = __platform_get_irq(dev, num);
+ if (ret < 0 && ret != -EPROBE_DEFER)
+ dev_err(&dev->dev, "IRQ index %u not found\n", num);
+
+ return ret;
+}
EXPORT_SYMBOL_GPL(platform_get_irq);
/**
+ * platform_get_irq_optional - get an optional IRQ for a device
+ * @dev: platform device
+ * @num: IRQ number index
+ *
+ * Gets an IRQ for a platform device. Device drivers should check the return
+ * value for errors so as to not pass a negative integer value to the
+ * request_irq() APIs. This is the same as platform_get_irq(), except that it
+ * does not print an error message if an IRQ can not be obtained.
+ *
+ * Example:
+ * int irq = platform_get_irq_optional(pdev, 0);
+ * if (irq < 0)
+ * return irq;
+ *
+ * Return: IRQ number on success, negative error number on failure.
+ */
+int platform_get_irq_optional(struct platform_device *dev, unsigned int num)
+{
+ return __platform_get_irq(dev, num);
+}
+EXPORT_SYMBOL_GPL(platform_get_irq_optional);
+
+/**
* platform_irq_count - Count the number of IRQs a platform device uses
* @dev: platform device
*
@@ -175,7 +206,7 @@ int platform_irq_count(struct platform_device *dev)
{
int ret, nr = 0;
- while ((ret = platform_get_irq(dev, nr)) >= 0)
+ while ((ret = __platform_get_irq(dev, nr)) >= 0)
nr++;
if (ret == -EPROBE_DEFER)
@@ -228,7 +259,11 @@ int platform_get_irq_byname(struct platform_device *dev, const char *name)
}
r = platform_get_resource_byname(dev, IORESOURCE_IRQ, name);
- return r ? r->start : -ENXIO;
+ if (r)
+ return r->start;
+
+ dev_err(&dev->dev, "IRQ %s not found\n", name);
+ return -ENXIO;
}
EXPORT_SYMBOL_GPL(platform_get_irq_byname);
@@ -259,6 +294,20 @@ struct platform_object {
char name[];
};
+/*
+ * Set up default DMA mask for platform devices if the they weren't
+ * previously set by the architecture / DT.
+ */
+static void setup_pdev_dma_masks(struct platform_device *pdev)
+{
+ if (!pdev->dev.coherent_dma_mask)
+ pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
+ if (!pdev->dma_mask)
+ pdev->dma_mask = DMA_BIT_MASK(32);
+ if (!pdev->dev.dma_mask)
+ pdev->dev.dma_mask = &pdev->dma_mask;
+};
+
/**
* platform_device_put - destroy a platform device
* @pdev: platform device to free
@@ -305,7 +354,7 @@ struct platform_device *platform_device_alloc(const char *name, int id)
pa->pdev.id = id;
device_initialize(&pa->pdev.dev);
pa->pdev.dev.release = platform_device_release;
- arch_setup_pdev_archdata(&pa->pdev);
+ setup_pdev_dma_masks(&pa->pdev);
}
return pa ? &pa->pdev : NULL;
@@ -507,7 +556,7 @@ EXPORT_SYMBOL_GPL(platform_device_del);
int platform_device_register(struct platform_device *pdev)
{
device_initialize(&pdev->dev);
- arch_setup_pdev_archdata(pdev);
+ setup_pdev_dma_masks(pdev);
return platform_device_add(pdev);
}
EXPORT_SYMBOL_GPL(platform_device_register);
@@ -1197,6 +1246,20 @@ struct bus_type platform_bus_type = {
};
EXPORT_SYMBOL_GPL(platform_bus_type);
+/**
+ * platform_find_device_by_driver - Find a platform device with a given
+ * driver.
+ * @start: The device to start the search from.
+ * @drv: The device driver to look for.
+ */
+struct device *platform_find_device_by_driver(struct device *start,
+ const struct device_driver *drv)
+{
+ return bus_find_device(&platform_bus_type, start, drv,
+ (void *)platform_match);
+}
+EXPORT_SYMBOL_GPL(platform_find_device_by_driver);
+
int __init platform_bus_init(void)
{
int error;
diff --git a/drivers/base/power/Makefile b/drivers/base/power/Makefile
index e1bb691cf8f1..ec5bb190b9d0 100644
--- a/drivers/base/power/Makefile
+++ b/drivers/base/power/Makefile
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_PM) += sysfs.o generic_ops.o common.o qos.o runtime.o wakeirq.o
-obj-$(CONFIG_PM_SLEEP) += main.o wakeup.o
+obj-$(CONFIG_PM_SLEEP) += main.o wakeup.o wakeup_stats.o
obj-$(CONFIG_PM_TRACE_RTC) += trace.o
obj-$(CONFIG_PM_GENERIC_DOMAINS) += domain.o domain_governor.o
obj-$(CONFIG_HAVE_CLK) += clock_ops.o
diff --git a/drivers/base/power/clock_ops.c b/drivers/base/power/clock_ops.c
index 59d19dd64928..ced6863a16a5 100644
--- a/drivers/base/power/clock_ops.c
+++ b/drivers/base/power/clock_ops.c
@@ -12,6 +12,7 @@
#include <linux/pm_clock.h>
#include <linux/clk.h>
#include <linux/clkdev.h>
+#include <linux/of_clk.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/pm_domain.h>
@@ -92,8 +93,6 @@ static int __pm_clk_add(struct device *dev, const char *con_id,
if (con_id) {
ce->con_id = kstrdup(con_id, GFP_KERNEL);
if (!ce->con_id) {
- dev_err(dev,
- "Not enough memory for clock connection ID.\n");
kfree(ce);
return -ENOMEM;
}
@@ -195,8 +194,7 @@ int of_pm_clk_add_clks(struct device *dev)
if (!dev || !dev->of_node)
return -EINVAL;
- count = of_count_phandle_with_args(dev->of_node, "clocks",
- "#clock-cells");
+ count = of_clk_get_parent_count(dev->of_node);
if (count <= 0)
return -ENODEV;
diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c
index 33c30c1e6a30..cc85e87eaf05 100644
--- a/drivers/base/power/domain.c
+++ b/drivers/base/power/domain.c
@@ -149,29 +149,24 @@ static inline bool irq_safe_dev_in_no_sleep_domain(struct device *dev,
return ret;
}
+static int genpd_runtime_suspend(struct device *dev);
+
/*
* Get the generic PM domain for a particular struct device.
* This validates the struct device pointer, the PM domain pointer,
* and checks that the PM domain pointer is a real generic PM domain.
* Any failure results in NULL being returned.
*/
-static struct generic_pm_domain *genpd_lookup_dev(struct device *dev)
+static struct generic_pm_domain *dev_to_genpd_safe(struct device *dev)
{
- struct generic_pm_domain *genpd = NULL, *gpd;
-
if (IS_ERR_OR_NULL(dev) || IS_ERR_OR_NULL(dev->pm_domain))
return NULL;
- mutex_lock(&gpd_list_lock);
- list_for_each_entry(gpd, &gpd_list, gpd_list_node) {
- if (&gpd->domain == dev->pm_domain) {
- genpd = gpd;
- break;
- }
- }
- mutex_unlock(&gpd_list_lock);
+ /* A genpd's always have its ->runtime_suspend() callback assigned. */
+ if (dev->pm_domain->ops.runtime_suspend == genpd_runtime_suspend)
+ return pd_to_genpd(dev->pm_domain);
- return genpd;
+ return NULL;
}
/*
@@ -385,8 +380,8 @@ int dev_pm_genpd_set_performance_state(struct device *dev, unsigned int state)
unsigned int prev;
int ret;
- genpd = dev_to_genpd(dev);
- if (IS_ERR(genpd))
+ genpd = dev_to_genpd_safe(dev);
+ if (!genpd)
return -ENODEV;
if (unlikely(!genpd->set_performance_state))
@@ -1536,7 +1531,8 @@ static int genpd_add_device(struct generic_pm_domain *genpd, struct device *dev,
if (ret)
genpd_free_dev_data(dev, gpd_data);
else
- dev_pm_qos_add_notifier(dev, &gpd_data->nb);
+ dev_pm_qos_add_notifier(dev, &gpd_data->nb,
+ DEV_PM_QOS_RESUME_LATENCY);
return ret;
}
@@ -1569,7 +1565,8 @@ static int genpd_remove_device(struct generic_pm_domain *genpd,
pdd = dev->power.subsys_data->domain_data;
gpd_data = to_gpd_data(pdd);
- dev_pm_qos_remove_notifier(dev, &gpd_data->nb);
+ dev_pm_qos_remove_notifier(dev, &gpd_data->nb,
+ DEV_PM_QOS_RESUME_LATENCY);
genpd_lock(genpd);
@@ -1597,7 +1594,7 @@ static int genpd_remove_device(struct generic_pm_domain *genpd,
out:
genpd_unlock(genpd);
- dev_pm_qos_add_notifier(dev, &gpd_data->nb);
+ dev_pm_qos_add_notifier(dev, &gpd_data->nb, DEV_PM_QOS_RESUME_LATENCY);
return ret;
}
@@ -1608,7 +1605,7 @@ static int genpd_remove_device(struct generic_pm_domain *genpd,
*/
int pm_genpd_remove_device(struct device *dev)
{
- struct generic_pm_domain *genpd = genpd_lookup_dev(dev);
+ struct generic_pm_domain *genpd = dev_to_genpd_safe(dev);
if (!genpd)
return -EINVAL;
diff --git a/drivers/base/power/domain_governor.c b/drivers/base/power/domain_governor.c
index 3838045c9277..daa8c7689f7e 100644
--- a/drivers/base/power/domain_governor.c
+++ b/drivers/base/power/domain_governor.c
@@ -33,7 +33,7 @@ static int dev_update_qos_constraint(struct device *dev, void *data)
* take its current PM QoS constraint (that's the only thing
* known at this point anyway).
*/
- constraint_ns = dev_pm_qos_read_value(dev);
+ constraint_ns = dev_pm_qos_read_value(dev, DEV_PM_QOS_RESUME_LATENCY);
constraint_ns *= NSEC_PER_USEC;
}
@@ -66,7 +66,7 @@ static bool default_suspend_ok(struct device *dev)
td->constraint_changed = false;
td->cached_suspend_ok = false;
td->effective_constraint_ns = 0;
- constraint_ns = __dev_pm_qos_read_value(dev);
+ constraint_ns = __dev_pm_qos_resume_latency(dev);
spin_unlock_irqrestore(&dev->power.lock, flags);
diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
index dcfc0a36c8f7..134a8af51511 100644
--- a/drivers/base/power/main.c
+++ b/drivers/base/power/main.c
@@ -530,21 +530,6 @@ static void dpm_watchdog_clear(struct dpm_watchdog *wd)
/*------------------------- Resume routines -------------------------*/
/**
- * dev_pm_skip_next_resume_phases - Skip next system resume phases for device.
- * @dev: Target device.
- *
- * Make the core skip the "early resume" and "resume" phases for @dev.
- *
- * This function can be called by middle-layer code during the "noirq" phase of
- * system resume if necessary, but not by device drivers.
- */
-void dev_pm_skip_next_resume_phases(struct device *dev)
-{
- dev->power.is_late_suspended = false;
- dev->power.is_suspended = false;
-}
-
-/**
* suspend_event - Return a "suspend" message for given "resume" one.
* @resume_msg: PM message representing a system-wide resume transition.
*/
@@ -681,6 +666,9 @@ Skip:
dev->power.is_noirq_suspended = false;
if (skip_resume) {
+ /* Make the next phases of resume skip the device. */
+ dev->power.is_late_suspended = false;
+ dev->power.is_suspended = false;
/*
* The device is going to be left in suspend, but it might not
* have been in runtime suspend before the system suspended, so
@@ -689,7 +677,6 @@ Skip:
* device again.
*/
pm_runtime_set_suspended(dev);
- dev_pm_skip_next_resume_phases(dev);
}
Out:
@@ -729,7 +716,7 @@ static void async_resume_noirq(void *data, async_cookie_t cookie)
put_device(dev);
}
-void dpm_noirq_resume_devices(pm_message_t state)
+static void dpm_noirq_resume_devices(pm_message_t state)
{
struct device *dev;
ktime_t starttime = ktime_get();
@@ -773,13 +760,6 @@ void dpm_noirq_resume_devices(pm_message_t state)
trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, false);
}
-void dpm_noirq_end(void)
-{
- resume_device_irqs();
- device_wakeup_disarm_wake_irqs();
- cpuidle_resume();
-}
-
/**
* dpm_resume_noirq - Execute "noirq resume" callbacks for all devices.
* @state: PM transition of the system being carried out.
@@ -790,7 +770,11 @@ void dpm_noirq_end(void)
void dpm_resume_noirq(pm_message_t state)
{
dpm_noirq_resume_devices(state);
- dpm_noirq_end();
+
+ resume_device_irqs();
+ device_wakeup_disarm_wake_irqs();
+
+ cpuidle_resume();
}
static pm_callback_t dpm_subsys_resume_early_cb(struct device *dev,
@@ -1304,11 +1288,6 @@ static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool a
if (async_error)
goto Complete;
- if (pm_wakeup_pending()) {
- async_error = -EBUSY;
- goto Complete;
- }
-
if (dev->power.syscore || dev->power.direct_complete)
goto Complete;
@@ -1375,14 +1354,7 @@ static int device_suspend_noirq(struct device *dev)
return __device_suspend_noirq(dev, pm_transition, false);
}
-void dpm_noirq_begin(void)
-{
- cpuidle_pause();
- device_wakeup_arm_wake_irqs();
- suspend_device_irqs();
-}
-
-int dpm_noirq_suspend_devices(pm_message_t state)
+static int dpm_noirq_suspend_devices(pm_message_t state)
{
ktime_t starttime = ktime_get();
int error = 0;
@@ -1439,7 +1411,11 @@ int dpm_suspend_noirq(pm_message_t state)
{
int ret;
- dpm_noirq_begin();
+ cpuidle_pause();
+
+ device_wakeup_arm_wake_irqs();
+ suspend_device_irqs();
+
ret = dpm_noirq_suspend_devices(state);
if (ret)
dpm_resume_noirq(resume_event(state));
@@ -1631,17 +1607,20 @@ int dpm_suspend_late(pm_message_t state)
*/
int dpm_suspend_end(pm_message_t state)
{
- int error = dpm_suspend_late(state);
+ ktime_t starttime = ktime_get();
+ int error;
+
+ error = dpm_suspend_late(state);
if (error)
- return error;
+ goto out;
error = dpm_suspend_noirq(state);
- if (error) {
+ if (error)
dpm_resume_early(resume_event(state));
- return error;
- }
- return 0;
+out:
+ dpm_show_time(starttime, state, error, "end");
+ return error;
}
EXPORT_SYMBOL_GPL(dpm_suspend_end);
@@ -2034,6 +2013,7 @@ int dpm_prepare(pm_message_t state)
*/
int dpm_suspend_start(pm_message_t state)
{
+ ktime_t starttime = ktime_get();
int error;
error = dpm_prepare(state);
@@ -2042,6 +2022,7 @@ int dpm_suspend_start(pm_message_t state)
dpm_save_failed_step(SUSPEND_PREPARE);
} else
error = dpm_suspend(state);
+ dpm_show_time(starttime, state, error, "start");
return error;
}
EXPORT_SYMBOL_GPL(dpm_suspend_start);
diff --git a/drivers/base/power/power.h b/drivers/base/power/power.h
index ec33fbdb919b..39a06a0cfdaa 100644
--- a/drivers/base/power/power.h
+++ b/drivers/base/power/power.h
@@ -149,3 +149,21 @@ static inline void device_pm_init(struct device *dev)
device_pm_sleep_init(dev);
pm_runtime_init(dev);
}
+
+#ifdef CONFIG_PM_SLEEP
+
+/* drivers/base/power/wakeup_stats.c */
+extern int wakeup_source_sysfs_add(struct device *parent,
+ struct wakeup_source *ws);
+extern void wakeup_source_sysfs_remove(struct wakeup_source *ws);
+
+extern int pm_wakeup_source_sysfs_add(struct device *parent);
+
+#else /* !CONFIG_PM_SLEEP */
+
+static inline int pm_wakeup_source_sysfs_add(struct device *parent)
+{
+ return 0;
+}
+
+#endif /* CONFIG_PM_SLEEP */
diff --git a/drivers/base/power/qos.c b/drivers/base/power/qos.c
index 6c91f8df1d59..6c90fd7e2ff8 100644
--- a/drivers/base/power/qos.c
+++ b/drivers/base/power/qos.c
@@ -90,29 +90,49 @@ enum pm_qos_flags_status dev_pm_qos_flags(struct device *dev, s32 mask)
EXPORT_SYMBOL_GPL(dev_pm_qos_flags);
/**
- * __dev_pm_qos_read_value - Get PM QoS constraint for a given device.
+ * __dev_pm_qos_resume_latency - Get resume latency constraint for a given device.
* @dev: Device to get the PM QoS constraint value for.
*
* This routine must be called with dev->power.lock held.
*/
-s32 __dev_pm_qos_read_value(struct device *dev)
+s32 __dev_pm_qos_resume_latency(struct device *dev)
{
lockdep_assert_held(&dev->power.lock);
- return dev_pm_qos_raw_read_value(dev);
+ return dev_pm_qos_raw_resume_latency(dev);
}
/**
* dev_pm_qos_read_value - Get PM QoS constraint for a given device (locked).
* @dev: Device to get the PM QoS constraint value for.
+ * @type: QoS request type.
*/
-s32 dev_pm_qos_read_value(struct device *dev)
+s32 dev_pm_qos_read_value(struct device *dev, enum dev_pm_qos_req_type type)
{
+ struct dev_pm_qos *qos = dev->power.qos;
unsigned long flags;
s32 ret;
spin_lock_irqsave(&dev->power.lock, flags);
- ret = __dev_pm_qos_read_value(dev);
+
+ switch (type) {
+ case DEV_PM_QOS_RESUME_LATENCY:
+ ret = IS_ERR_OR_NULL(qos) ? PM_QOS_RESUME_LATENCY_NO_CONSTRAINT
+ : pm_qos_read_value(&qos->resume_latency);
+ break;
+ case DEV_PM_QOS_MIN_FREQUENCY:
+ ret = IS_ERR_OR_NULL(qos) ? PM_QOS_MIN_FREQUENCY_DEFAULT_VALUE
+ : pm_qos_read_value(&qos->min_frequency);
+ break;
+ case DEV_PM_QOS_MAX_FREQUENCY:
+ ret = IS_ERR_OR_NULL(qos) ? PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE
+ : pm_qos_read_value(&qos->max_frequency);
+ break;
+ default:
+ WARN_ON(1);
+ ret = 0;
+ }
+
spin_unlock_irqrestore(&dev->power.lock, flags);
return ret;
@@ -149,6 +169,14 @@ static int apply_constraint(struct dev_pm_qos_request *req,
req->dev->power.set_latency_tolerance(req->dev, value);
}
break;
+ case DEV_PM_QOS_MIN_FREQUENCY:
+ ret = pm_qos_update_target(&qos->min_frequency,
+ &req->data.pnode, action, value);
+ break;
+ case DEV_PM_QOS_MAX_FREQUENCY:
+ ret = pm_qos_update_target(&qos->max_frequency,
+ &req->data.pnode, action, value);
+ break;
case DEV_PM_QOS_FLAGS:
ret = pm_qos_update_flags(&qos->flags, &req->data.flr,
action, value);
@@ -177,12 +205,11 @@ static int dev_pm_qos_constraints_allocate(struct device *dev)
if (!qos)
return -ENOMEM;
- n = kzalloc(sizeof(*n), GFP_KERNEL);
+ n = kzalloc(3 * sizeof(*n), GFP_KERNEL);
if (!n) {
kfree(qos);
return -ENOMEM;
}
- BLOCKING_INIT_NOTIFIER_HEAD(n);
c = &qos->resume_latency;
plist_head_init(&c->list);
@@ -191,6 +218,7 @@ static int dev_pm_qos_constraints_allocate(struct device *dev)
c->no_constraint_value = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT;
c->type = PM_QOS_MIN;
c->notifiers = n;
+ BLOCKING_INIT_NOTIFIER_HEAD(n);
c = &qos->latency_tolerance;
plist_head_init(&c->list);
@@ -199,6 +227,24 @@ static int dev_pm_qos_constraints_allocate(struct device *dev)
c->no_constraint_value = PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT;
c->type = PM_QOS_MIN;
+ c = &qos->min_frequency;
+ plist_head_init(&c->list);
+ c->target_value = PM_QOS_MIN_FREQUENCY_DEFAULT_VALUE;
+ c->default_value = PM_QOS_MIN_FREQUENCY_DEFAULT_VALUE;
+ c->no_constraint_value = PM_QOS_MIN_FREQUENCY_DEFAULT_VALUE;
+ c->type = PM_QOS_MAX;
+ c->notifiers = ++n;
+ BLOCKING_INIT_NOTIFIER_HEAD(n);
+
+ c = &qos->max_frequency;
+ plist_head_init(&c->list);
+ c->target_value = PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE;
+ c->default_value = PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE;
+ c->no_constraint_value = PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE;
+ c->type = PM_QOS_MIN;
+ c->notifiers = ++n;
+ BLOCKING_INIT_NOTIFIER_HEAD(n);
+
INIT_LIST_HEAD(&qos->flags.list);
spin_lock_irq(&dev->power.lock);
@@ -252,11 +298,25 @@ void dev_pm_qos_constraints_destroy(struct device *dev)
apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE);
memset(req, 0, sizeof(*req));
}
+
c = &qos->latency_tolerance;
plist_for_each_entry_safe(req, tmp, &c->list, data.pnode) {
apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE);
memset(req, 0, sizeof(*req));
}
+
+ c = &qos->min_frequency;
+ plist_for_each_entry_safe(req, tmp, &c->list, data.pnode) {
+ apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_MIN_FREQUENCY_DEFAULT_VALUE);
+ memset(req, 0, sizeof(*req));
+ }
+
+ c = &qos->max_frequency;
+ plist_for_each_entry_safe(req, tmp, &c->list, data.pnode) {
+ apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE);
+ memset(req, 0, sizeof(*req));
+ }
+
f = &qos->flags;
list_for_each_entry_safe(req, tmp, &f->list, data.flr.node) {
apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE);
@@ -368,6 +428,8 @@ static int __dev_pm_qos_update_request(struct dev_pm_qos_request *req,
switch(req->type) {
case DEV_PM_QOS_RESUME_LATENCY:
case DEV_PM_QOS_LATENCY_TOLERANCE:
+ case DEV_PM_QOS_MIN_FREQUENCY:
+ case DEV_PM_QOS_MAX_FREQUENCY:
curr_value = req->data.pnode.prio;
break;
case DEV_PM_QOS_FLAGS:
@@ -467,6 +529,7 @@ EXPORT_SYMBOL_GPL(dev_pm_qos_remove_request);
*
* @dev: target device for the constraint
* @notifier: notifier block managed by caller.
+ * @type: request type.
*
* Will register the notifier into a notification chain that gets called
* upon changes to the target value for the device.
@@ -474,7 +537,8 @@ EXPORT_SYMBOL_GPL(dev_pm_qos_remove_request);
* If the device's constraints object doesn't exist when this routine is called,
* it will be created (or error code will be returned if that fails).
*/
-int dev_pm_qos_add_notifier(struct device *dev, struct notifier_block *notifier)
+int dev_pm_qos_add_notifier(struct device *dev, struct notifier_block *notifier,
+ enum dev_pm_qos_req_type type)
{
int ret = 0;
@@ -485,10 +549,28 @@ int dev_pm_qos_add_notifier(struct device *dev, struct notifier_block *notifier)
else if (!dev->power.qos)
ret = dev_pm_qos_constraints_allocate(dev);
- if (!ret)
+ if (ret)
+ goto unlock;
+
+ switch (type) {
+ case DEV_PM_QOS_RESUME_LATENCY:
ret = blocking_notifier_chain_register(dev->power.qos->resume_latency.notifiers,
notifier);
+ break;
+ case DEV_PM_QOS_MIN_FREQUENCY:
+ ret = blocking_notifier_chain_register(dev->power.qos->min_frequency.notifiers,
+ notifier);
+ break;
+ case DEV_PM_QOS_MAX_FREQUENCY:
+ ret = blocking_notifier_chain_register(dev->power.qos->max_frequency.notifiers,
+ notifier);
+ break;
+ default:
+ WARN_ON(1);
+ ret = -EINVAL;
+ }
+unlock:
mutex_unlock(&dev_pm_qos_mtx);
return ret;
}
@@ -500,24 +582,44 @@ EXPORT_SYMBOL_GPL(dev_pm_qos_add_notifier);
*
* @dev: target device for the constraint
* @notifier: notifier block to be removed.
+ * @type: request type.
*
* Will remove the notifier from the notification chain that gets called
* upon changes to the target value.
*/
int dev_pm_qos_remove_notifier(struct device *dev,
- struct notifier_block *notifier)
+ struct notifier_block *notifier,
+ enum dev_pm_qos_req_type type)
{
- int retval = 0;
+ int ret = 0;
mutex_lock(&dev_pm_qos_mtx);
/* Silently return if the constraints object is not present. */
- if (!IS_ERR_OR_NULL(dev->power.qos))
- retval = blocking_notifier_chain_unregister(dev->power.qos->resume_latency.notifiers,
- notifier);
+ if (IS_ERR_OR_NULL(dev->power.qos))
+ goto unlock;
+
+ switch (type) {
+ case DEV_PM_QOS_RESUME_LATENCY:
+ ret = blocking_notifier_chain_unregister(dev->power.qos->resume_latency.notifiers,
+ notifier);
+ break;
+ case DEV_PM_QOS_MIN_FREQUENCY:
+ ret = blocking_notifier_chain_unregister(dev->power.qos->min_frequency.notifiers,
+ notifier);
+ break;
+ case DEV_PM_QOS_MAX_FREQUENCY:
+ ret = blocking_notifier_chain_unregister(dev->power.qos->max_frequency.notifiers,
+ notifier);
+ break;
+ default:
+ WARN_ON(1);
+ ret = -EINVAL;
+ }
+unlock:
mutex_unlock(&dev_pm_qos_mtx);
- return retval;
+ return ret;
}
EXPORT_SYMBOL_GPL(dev_pm_qos_remove_notifier);
@@ -577,6 +679,9 @@ static void __dev_pm_qos_drop_user_request(struct device *dev,
req = dev->power.qos->flags_req;
dev->power.qos->flags_req = NULL;
break;
+ default:
+ WARN_ON(1);
+ return;
}
__dev_pm_qos_remove_request(req);
kfree(req);
diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c
index 952a1e7057c7..48616f358854 100644
--- a/drivers/base/power/runtime.c
+++ b/drivers/base/power/runtime.c
@@ -275,7 +275,7 @@ static int rpm_check_suspend_allowed(struct device *dev)
|| (dev->power.request_pending
&& dev->power.request == RPM_REQ_RESUME))
retval = -EAGAIN;
- else if (__dev_pm_qos_read_value(dev) == 0)
+ else if (__dev_pm_qos_resume_latency(dev) == 0)
retval = -EPERM;
else if (dev->power.runtime_status == RPM_SUSPENDED)
retval = 1;
@@ -287,7 +287,8 @@ static int rpm_get_suppliers(struct device *dev)
{
struct device_link *link;
- list_for_each_entry_rcu(link, &dev->links.suppliers, c_node) {
+ list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
+ device_links_read_lock_held()) {
int retval;
if (!(link->flags & DL_FLAG_PM_RUNTIME) ||
@@ -309,7 +310,8 @@ static void rpm_put_suppliers(struct device *dev)
{
struct device_link *link;
- list_for_each_entry_rcu(link, &dev->links.suppliers, c_node) {
+ list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
+ device_links_read_lock_held()) {
if (READ_ONCE(link->status) == DL_STATE_SUPPLIER_UNBIND)
continue;
@@ -1624,7 +1626,7 @@ void pm_runtime_remove(struct device *dev)
* runtime PM references to the device, drop the usage counter of the device
* (as many times as needed).
*
- * Links with the DL_FLAG_STATELESS flag set are ignored.
+ * Links with the DL_FLAG_MANAGED flag unset are ignored.
*
* Since the device is guaranteed to be runtime-active at the point this is
* called, nothing else needs to be done here.
@@ -1640,8 +1642,9 @@ void pm_runtime_clean_up_links(struct device *dev)
idx = device_links_read_lock();
- list_for_each_entry_rcu(link, &dev->links.consumers, s_node) {
- if (link->flags & DL_FLAG_STATELESS)
+ list_for_each_entry_rcu(link, &dev->links.consumers, s_node,
+ device_links_read_lock_held()) {
+ if (!(link->flags & DL_FLAG_MANAGED))
continue;
while (refcount_dec_not_one(&link->rpm_active))
@@ -1662,7 +1665,8 @@ void pm_runtime_get_suppliers(struct device *dev)
idx = device_links_read_lock();
- list_for_each_entry_rcu(link, &dev->links.suppliers, c_node)
+ list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
+ device_links_read_lock_held())
if (link->flags & DL_FLAG_PM_RUNTIME) {
link->supplier_preactivated = true;
refcount_inc(&link->rpm_active);
@@ -1683,7 +1687,8 @@ void pm_runtime_put_suppliers(struct device *dev)
idx = device_links_read_lock();
- list_for_each_entry_rcu(link, &dev->links.suppliers, c_node)
+ list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
+ device_links_read_lock_held())
if (link->supplier_preactivated) {
link->supplier_preactivated = false;
if (refcount_dec_not_one(&link->rpm_active))
diff --git a/drivers/base/power/sysfs.c b/drivers/base/power/sysfs.c
index 1b9c281cbe41..d7d82db2e4bc 100644
--- a/drivers/base/power/sysfs.c
+++ b/drivers/base/power/sysfs.c
@@ -5,6 +5,7 @@
#include <linux/export.h>
#include <linux/pm_qos.h>
#include <linux/pm_runtime.h>
+#include <linux/pm_wakeup.h>
#include <linux/atomic.h>
#include <linux/jiffies.h>
#include "power.h"
@@ -667,8 +668,13 @@ int dpm_sysfs_add(struct device *dev)
if (rc)
goto err_wakeup;
}
+ rc = pm_wakeup_source_sysfs_add(dev);
+ if (rc)
+ goto err_latency;
return 0;
+ err_latency:
+ sysfs_unmerge_group(&dev->kobj, &pm_qos_latency_tolerance_attr_group);
err_wakeup:
sysfs_unmerge_group(&dev->kobj, &pm_wakeup_attr_group);
err_runtime:
diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c
index 5b2b6a05a4f3..5817b51d2b15 100644
--- a/drivers/base/power/wakeup.c
+++ b/drivers/base/power/wakeup.c
@@ -72,22 +72,7 @@ static struct wakeup_source deleted_ws = {
.lock = __SPIN_LOCK_UNLOCKED(deleted_ws.lock),
};
-/**
- * wakeup_source_prepare - Prepare a new wakeup source for initialization.
- * @ws: Wakeup source to prepare.
- * @name: Pointer to the name of the new wakeup source.
- *
- * Callers must ensure that the @name string won't be freed when @ws is still in
- * use.
- */
-void wakeup_source_prepare(struct wakeup_source *ws, const char *name)
-{
- if (ws) {
- memset(ws, 0, sizeof(*ws));
- ws->name = name;
- }
-}
-EXPORT_SYMBOL_GPL(wakeup_source_prepare);
+static DEFINE_IDA(wakeup_ida);
/**
* wakeup_source_create - Create a struct wakeup_source object.
@@ -96,13 +81,31 @@ EXPORT_SYMBOL_GPL(wakeup_source_prepare);
struct wakeup_source *wakeup_source_create(const char *name)
{
struct wakeup_source *ws;
+ const char *ws_name;
+ int id;
- ws = kmalloc(sizeof(*ws), GFP_KERNEL);
+ ws = kzalloc(sizeof(*ws), GFP_KERNEL);
if (!ws)
- return NULL;
+ goto err_ws;
+
+ ws_name = kstrdup_const(name, GFP_KERNEL);
+ if (!ws_name)
+ goto err_name;
+ ws->name = ws_name;
+
+ id = ida_alloc(&wakeup_ida, GFP_KERNEL);
+ if (id < 0)
+ goto err_id;
+ ws->id = id;
- wakeup_source_prepare(ws, name ? kstrdup_const(name, GFP_KERNEL) : NULL);
return ws;
+
+err_id:
+ kfree_const(ws->name);
+err_name:
+ kfree(ws);
+err_ws:
+ return NULL;
}
EXPORT_SYMBOL_GPL(wakeup_source_create);
@@ -134,6 +137,13 @@ static void wakeup_source_record(struct wakeup_source *ws)
spin_unlock_irqrestore(&deleted_ws.lock, flags);
}
+static void wakeup_source_free(struct wakeup_source *ws)
+{
+ ida_free(&wakeup_ida, ws->id);
+ kfree_const(ws->name);
+ kfree(ws);
+}
+
/**
* wakeup_source_destroy - Destroy a struct wakeup_source object.
* @ws: Wakeup source to destroy.
@@ -147,8 +157,7 @@ void wakeup_source_destroy(struct wakeup_source *ws)
__pm_relax(ws);
wakeup_source_record(ws);
- kfree_const(ws->name);
- kfree(ws);
+ wakeup_source_free(ws);
}
EXPORT_SYMBOL_GPL(wakeup_source_destroy);
@@ -200,16 +209,26 @@ EXPORT_SYMBOL_GPL(wakeup_source_remove);
/**
* wakeup_source_register - Create wakeup source and add it to the list.
+ * @dev: Device this wakeup source is associated with (or NULL if virtual).
* @name: Name of the wakeup source to register.
*/
-struct wakeup_source *wakeup_source_register(const char *name)
+struct wakeup_source *wakeup_source_register(struct device *dev,
+ const char *name)
{
struct wakeup_source *ws;
+ int ret;
ws = wakeup_source_create(name);
- if (ws)
+ if (ws) {
+ if (!dev || device_is_registered(dev)) {
+ ret = wakeup_source_sysfs_add(dev, ws);
+ if (ret) {
+ wakeup_source_free(ws);
+ return NULL;
+ }
+ }
wakeup_source_add(ws);
-
+ }
return ws;
}
EXPORT_SYMBOL_GPL(wakeup_source_register);
@@ -222,6 +241,7 @@ void wakeup_source_unregister(struct wakeup_source *ws)
{
if (ws) {
wakeup_source_remove(ws);
+ wakeup_source_sysfs_remove(ws);
wakeup_source_destroy(ws);
}
}
@@ -265,7 +285,7 @@ int device_wakeup_enable(struct device *dev)
if (pm_suspend_target_state != PM_SUSPEND_ON)
dev_dbg(dev, "Suspicious %s() during system transition!\n", __func__);
- ws = wakeup_source_register(dev_name(dev));
+ ws = wakeup_source_register(dev, dev_name(dev));
if (!ws)
return -ENOMEM;
@@ -859,7 +879,7 @@ EXPORT_SYMBOL_GPL(pm_system_wakeup);
void pm_system_cancel_wakeup(void)
{
- atomic_dec(&pm_abort_suspend);
+ atomic_dec_if_positive(&pm_abort_suspend);
}
void pm_wakeup_clear(bool reset)
@@ -968,8 +988,6 @@ void pm_wakep_autosleep_enabled(bool set)
}
#endif /* CONFIG_PM_AUTOSLEEP */
-static struct dentry *wakeup_sources_stats_dentry;
-
/**
* print_wakeup_source_stats - Print wakeup source statistics information.
* @m: seq_file to print the statistics into.
@@ -1099,8 +1117,8 @@ static const struct file_operations wakeup_sources_stats_fops = {
static int __init wakeup_sources_debugfs_init(void)
{
- wakeup_sources_stats_dentry = debugfs_create_file("wakeup_sources",
- S_IRUGO, NULL, NULL, &wakeup_sources_stats_fops);
+ debugfs_create_file("wakeup_sources", S_IRUGO, NULL, NULL,
+ &wakeup_sources_stats_fops);
return 0;
}
diff --git a/drivers/base/power/wakeup_stats.c b/drivers/base/power/wakeup_stats.c
new file mode 100644
index 000000000000..c7734914d914
--- /dev/null
+++ b/drivers/base/power/wakeup_stats.c
@@ -0,0 +1,214 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Wakeup statistics in sysfs
+ *
+ * Copyright (c) 2019 Linux Foundation
+ * Copyright (c) 2019 Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+ * Copyright (c) 2019 Google Inc.
+ */
+
+#include <linux/device.h>
+#include <linux/idr.h>
+#include <linux/init.h>
+#include <linux/kdev_t.h>
+#include <linux/kernel.h>
+#include <linux/kobject.h>
+#include <linux/slab.h>
+#include <linux/timekeeping.h>
+
+#include "power.h"
+
+static struct class *wakeup_class;
+
+#define wakeup_attr(_name) \
+static ssize_t _name##_show(struct device *dev, \
+ struct device_attribute *attr, char *buf) \
+{ \
+ struct wakeup_source *ws = dev_get_drvdata(dev); \
+ \
+ return sprintf(buf, "%lu\n", ws->_name); \
+} \
+static DEVICE_ATTR_RO(_name)
+
+wakeup_attr(active_count);
+wakeup_attr(event_count);
+wakeup_attr(wakeup_count);
+wakeup_attr(expire_count);
+
+static ssize_t active_time_ms_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct wakeup_source *ws = dev_get_drvdata(dev);
+ ktime_t active_time =
+ ws->active ? ktime_sub(ktime_get(), ws->last_time) : 0;
+
+ return sprintf(buf, "%lld\n", ktime_to_ms(active_time));
+}
+static DEVICE_ATTR_RO(active_time_ms);
+
+static ssize_t total_time_ms_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct wakeup_source *ws = dev_get_drvdata(dev);
+ ktime_t active_time;
+ ktime_t total_time = ws->total_time;
+
+ if (ws->active) {
+ active_time = ktime_sub(ktime_get(), ws->last_time);
+ total_time = ktime_add(total_time, active_time);
+ }
+ return sprintf(buf, "%lld\n", ktime_to_ms(total_time));
+}
+static DEVICE_ATTR_RO(total_time_ms);
+
+static ssize_t max_time_ms_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct wakeup_source *ws = dev_get_drvdata(dev);
+ ktime_t active_time;
+ ktime_t max_time = ws->max_time;
+
+ if (ws->active) {
+ active_time = ktime_sub(ktime_get(), ws->last_time);
+ if (active_time > max_time)
+ max_time = active_time;
+ }
+ return sprintf(buf, "%lld\n", ktime_to_ms(max_time));
+}
+static DEVICE_ATTR_RO(max_time_ms);
+
+static ssize_t last_change_ms_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct wakeup_source *ws = dev_get_drvdata(dev);
+
+ return sprintf(buf, "%lld\n", ktime_to_ms(ws->last_time));
+}
+static DEVICE_ATTR_RO(last_change_ms);
+
+static ssize_t name_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct wakeup_source *ws = dev_get_drvdata(dev);
+
+ return sprintf(buf, "%s\n", ws->name);
+}
+static DEVICE_ATTR_RO(name);
+
+static ssize_t prevent_suspend_time_ms_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct wakeup_source *ws = dev_get_drvdata(dev);
+ ktime_t prevent_sleep_time = ws->prevent_sleep_time;
+
+ if (ws->active && ws->autosleep_enabled) {
+ prevent_sleep_time = ktime_add(prevent_sleep_time,
+ ktime_sub(ktime_get(), ws->start_prevent_time));
+ }
+ return sprintf(buf, "%lld\n", ktime_to_ms(prevent_sleep_time));
+}
+static DEVICE_ATTR_RO(prevent_suspend_time_ms);
+
+static struct attribute *wakeup_source_attrs[] = {
+ &dev_attr_name.attr,
+ &dev_attr_active_count.attr,
+ &dev_attr_event_count.attr,
+ &dev_attr_wakeup_count.attr,
+ &dev_attr_expire_count.attr,
+ &dev_attr_active_time_ms.attr,
+ &dev_attr_total_time_ms.attr,
+ &dev_attr_max_time_ms.attr,
+ &dev_attr_last_change_ms.attr,
+ &dev_attr_prevent_suspend_time_ms.attr,
+ NULL,
+};
+ATTRIBUTE_GROUPS(wakeup_source);
+
+static void device_create_release(struct device *dev)
+{
+ kfree(dev);
+}
+
+static struct device *wakeup_source_device_create(struct device *parent,
+ struct wakeup_source *ws)
+{
+ struct device *dev = NULL;
+ int retval = -ENODEV;
+
+ dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+ if (!dev) {
+ retval = -ENOMEM;
+ goto error;
+ }
+
+ device_initialize(dev);
+ dev->devt = MKDEV(0, 0);
+ dev->class = wakeup_class;
+ dev->parent = parent;
+ dev->groups = wakeup_source_groups;
+ dev->release = device_create_release;
+ dev_set_drvdata(dev, ws);
+ device_set_pm_not_required(dev);
+
+ retval = kobject_set_name(&dev->kobj, "wakeup%d", ws->id);
+ if (retval)
+ goto error;
+
+ retval = device_add(dev);
+ if (retval)
+ goto error;
+
+ return dev;
+
+error:
+ put_device(dev);
+ return ERR_PTR(retval);
+}
+
+/**
+ * wakeup_source_sysfs_add - Add wakeup_source attributes to sysfs.
+ * @parent: Device given wakeup source is associated with (or NULL if virtual).
+ * @ws: Wakeup source to be added in sysfs.
+ */
+int wakeup_source_sysfs_add(struct device *parent, struct wakeup_source *ws)
+{
+ struct device *dev;
+
+ dev = wakeup_source_device_create(parent, ws);
+ if (IS_ERR(dev))
+ return PTR_ERR(dev);
+ ws->dev = dev;
+
+ return 0;
+}
+
+/**
+ * pm_wakeup_source_sysfs_add - Add wakeup_source attributes to sysfs
+ * for a device if they're missing.
+ * @parent: Device given wakeup source is associated with
+ */
+int pm_wakeup_source_sysfs_add(struct device *parent)
+{
+ if (!parent->power.wakeup || parent->power.wakeup->dev)
+ return 0;
+
+ return wakeup_source_sysfs_add(parent, parent->power.wakeup);
+}
+
+/**
+ * wakeup_source_sysfs_remove - Remove wakeup_source attributes from sysfs.
+ * @ws: Wakeup source to be removed from sysfs.
+ */
+void wakeup_source_sysfs_remove(struct wakeup_source *ws)
+{
+ device_unregister(ws->dev);
+}
+
+static int __init wakeup_sources_sysfs_init(void)
+{
+ wakeup_class = class_create(THIS_MODULE, "wakeup");
+
+ return PTR_ERR_OR_ZERO(wakeup_class);
+}
+postcore_initcall(wakeup_sources_sysfs_init);
diff --git a/drivers/base/property.c b/drivers/base/property.c
index 348b37e64944..81bd01ed4042 100644
--- a/drivers/base/property.c
+++ b/drivers/base/property.c
@@ -485,6 +485,30 @@ int fwnode_property_get_reference_args(const struct fwnode_handle *fwnode,
EXPORT_SYMBOL_GPL(fwnode_property_get_reference_args);
/**
+ * fwnode_find_reference - Find named reference to a fwnode_handle
+ * @fwnode: Firmware node where to look for the reference
+ * @name: The name of the reference
+ * @index: Index of the reference
+ *
+ * @index can be used when the named reference holds a table of references.
+ *
+ * Returns pointer to the reference fwnode, or ERR_PTR. Caller is responsible to
+ * call fwnode_handle_put() on the returned fwnode pointer.
+ */
+struct fwnode_handle *fwnode_find_reference(const struct fwnode_handle *fwnode,
+ const char *name,
+ unsigned int index)
+{
+ struct fwnode_reference_args args;
+ int ret;
+
+ ret = fwnode_property_get_reference_args(fwnode, name, NULL, 0, index,
+ &args);
+ return ret ? ERR_PTR(ret) : args.fwnode;
+}
+EXPORT_SYMBOL_GPL(fwnode_find_reference);
+
+/**
* device_remove_properties - Remove properties from a device object.
* @dev: Device whose properties to remove.
*
diff --git a/drivers/base/regmap/Kconfig b/drivers/base/regmap/Kconfig
index 6ad5ef48b61e..0fd6f97ee523 100644
--- a/drivers/base/regmap/Kconfig
+++ b/drivers/base/regmap/Kconfig
@@ -4,7 +4,7 @@
# subsystems should select the appropriate symbols.
config REGMAP
- default y if (REGMAP_I2C || REGMAP_SPI || REGMAP_SPMI || REGMAP_W1 || REGMAP_AC97 || REGMAP_MMIO || REGMAP_IRQ)
+ default y if (REGMAP_I2C || REGMAP_SPI || REGMAP_SPMI || REGMAP_W1 || REGMAP_AC97 || REGMAP_MMIO || REGMAP_IRQ || REGMAP_SCCB || REGMAP_I3C)
select IRQ_DOMAIN if REGMAP_IRQ
bool
@@ -44,8 +44,12 @@ config REGMAP_IRQ
config REGMAP_SOUNDWIRE
tristate
- depends on SOUNDWIRE_BUS
+ depends on SOUNDWIRE
config REGMAP_SCCB
tristate
depends on I2C
+
+config REGMAP_I3C
+ tristate
+ depends on I3C
diff --git a/drivers/base/regmap/Makefile b/drivers/base/regmap/Makefile
index f5b4e8851d00..ff6c7d8ec1cd 100644
--- a/drivers/base/regmap/Makefile
+++ b/drivers/base/regmap/Makefile
@@ -16,3 +16,4 @@ obj-$(CONFIG_REGMAP_IRQ) += regmap-irq.o
obj-$(CONFIG_REGMAP_W1) += regmap-w1.o
obj-$(CONFIG_REGMAP_SOUNDWIRE) += regmap-sdw.o
obj-$(CONFIG_REGMAP_SCCB) += regmap-sccb.o
+obj-$(CONFIG_REGMAP_I3C) += regmap-i3c.o
diff --git a/drivers/base/regmap/regcache-lzo.c b/drivers/base/regmap/regcache-lzo.c
index fc14e8b9344f..7886303eb026 100644
--- a/drivers/base/regmap/regcache-lzo.c
+++ b/drivers/base/regmap/regcache-lzo.c
@@ -148,20 +148,18 @@ static int regcache_lzo_init(struct regmap *map)
* that register.
*/
bmp_size = map->num_reg_defaults_raw;
- sync_bmp = kmalloc_array(BITS_TO_LONGS(bmp_size), sizeof(long),
- GFP_KERNEL);
+ sync_bmp = bitmap_zalloc(bmp_size, GFP_KERNEL);
if (!sync_bmp) {
ret = -ENOMEM;
goto err;
}
- bitmap_zero(sync_bmp, bmp_size);
/* allocate the lzo blocks and initialize them */
for (i = 0; i < blkcount; i++) {
lzo_blocks[i] = kzalloc(sizeof **lzo_blocks,
GFP_KERNEL);
if (!lzo_blocks[i]) {
- kfree(sync_bmp);
+ bitmap_free(sync_bmp);
ret = -ENOMEM;
goto err;
}
@@ -213,7 +211,7 @@ static int regcache_lzo_exit(struct regmap *map)
* only once.
*/
if (lzo_blocks[0])
- kfree(lzo_blocks[0]->sync_bmp);
+ bitmap_free(lzo_blocks[0]->sync_bmp);
for (i = 0; i < blkcount; i++) {
if (lzo_blocks[i]) {
kfree(lzo_blocks[i]->wmem);
diff --git a/drivers/base/regmap/regmap-debugfs.c b/drivers/base/regmap/regmap-debugfs.c
index 263f82516ff4..e72843fe41df 100644
--- a/drivers/base/regmap/regmap-debugfs.c
+++ b/drivers/base/regmap/regmap-debugfs.c
@@ -579,6 +579,8 @@ void regmap_debugfs_init(struct regmap *map, const char *name)
}
if (!strcmp(name, "dummy")) {
+ kfree(map->debugfs_name);
+
map->debugfs_name = kasprintf(GFP_KERNEL, "dummy%d",
dummy_index);
name = map->debugfs_name;
@@ -586,14 +588,6 @@ void regmap_debugfs_init(struct regmap *map, const char *name)
}
map->debugfs = debugfs_create_dir(name, regmap_debugfs_root);
- if (!map->debugfs) {
- dev_warn(map->dev,
- "Failed to create %s debugfs directory\n", name);
-
- kfree(map->debugfs_name);
- map->debugfs_name = NULL;
- return;
- }
debugfs_create_file("name", 0400, map->debugfs,
map, &regmap_name_fops);
@@ -670,10 +664,6 @@ void regmap_debugfs_initcall(void)
struct regmap_debugfs_node *node, *tmp;
regmap_debugfs_root = debugfs_create_dir("regmap", NULL);
- if (!regmap_debugfs_root) {
- pr_warn("regmap: Failed to create debugfs root\n");
- return;
- }
mutex_lock(&regmap_debugfs_early_lock);
list_for_each_entry_safe(node, tmp, &regmap_debugfs_early_list, link) {
diff --git a/drivers/base/regmap/regmap-i3c.c b/drivers/base/regmap/regmap-i3c.c
new file mode 100644
index 000000000000..1578fb506683
--- /dev/null
+++ b/drivers/base/regmap/regmap-i3c.c
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2018 Synopsys, Inc. and/or its affiliates.
+
+#include <linux/regmap.h>
+#include <linux/i3c/device.h>
+#include <linux/i3c/master.h>
+#include <linux/module.h>
+
+static int regmap_i3c_write(void *context, const void *data, size_t count)
+{
+ struct device *dev = context;
+ struct i3c_device *i3c = dev_to_i3cdev(dev);
+ struct i3c_priv_xfer xfers[] = {
+ {
+ .rnw = false,
+ .len = count,
+ .data.out = data,
+ },
+ };
+
+ return i3c_device_do_priv_xfers(i3c, xfers, 1);
+}
+
+static int regmap_i3c_read(void *context,
+ const void *reg, size_t reg_size,
+ void *val, size_t val_size)
+{
+ struct device *dev = context;
+ struct i3c_device *i3c = dev_to_i3cdev(dev);
+ struct i3c_priv_xfer xfers[2];
+
+ xfers[0].rnw = false;
+ xfers[0].len = reg_size;
+ xfers[0].data.out = reg;
+
+ xfers[1].rnw = true;
+ xfers[1].len = val_size;
+ xfers[1].data.in = val;
+
+ return i3c_device_do_priv_xfers(i3c, xfers, 2);
+}
+
+static struct regmap_bus regmap_i3c = {
+ .write = regmap_i3c_write,
+ .read = regmap_i3c_read,
+};
+
+struct regmap *__devm_regmap_init_i3c(struct i3c_device *i3c,
+ const struct regmap_config *config,
+ struct lock_class_key *lock_key,
+ const char *lock_name)
+{
+ return __devm_regmap_init(&i3c->dev, &regmap_i3c, &i3c->dev, config,
+ lock_key, lock_name);
+}
+EXPORT_SYMBOL_GPL(__devm_regmap_init_i3c);
+
+MODULE_AUTHOR("Vitor Soares <vitor.soares@synopsys.com>");
+MODULE_DESCRIPTION("Regmap I3C Module");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c
index c9dc70ceca5f..3d64c9331a82 100644
--- a/drivers/base/regmap/regmap-irq.c
+++ b/drivers/base/regmap/regmap-irq.c
@@ -370,7 +370,6 @@ static irqreturn_t regmap_irq_thread(int irq, void *d)
if (ret < 0) {
dev_err(map->dev, "IRQ thread failed to resume: %d\n",
ret);
- pm_runtime_put(map->dev);
goto exit;
}
}
@@ -425,8 +424,6 @@ static irqreturn_t regmap_irq_thread(int irq, void *d)
dev_err(map->dev,
"Failed to read IRQ status %d\n",
ret);
- if (chip->runtime_pm)
- pm_runtime_put(map->dev);
goto exit;
}
}
@@ -478,8 +475,6 @@ static irqreturn_t regmap_irq_thread(int irq, void *d)
dev_err(map->dev,
"Failed to read IRQ status: %d\n",
ret);
- if (chip->runtime_pm)
- pm_runtime_put(map->dev);
goto exit;
}
}
@@ -513,10 +508,10 @@ static irqreturn_t regmap_irq_thread(int irq, void *d)
}
}
+exit:
if (chip->runtime_pm)
pm_runtime_put(map->dev);
-exit:
if (chip->handle_post_irq)
chip->handle_post_irq(chip->irq_drv_data);
diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c
index f1025452bb39..19f57ccfbe1d 100644
--- a/drivers/base/regmap/regmap.c
+++ b/drivers/base/regmap/regmap.c
@@ -1637,6 +1637,8 @@ static int _regmap_raw_write_impl(struct regmap *map, unsigned int reg,
map->format.reg_bytes +
map->format.pad_bytes,
val, val_len);
+ else
+ ret = -ENOTSUPP;
/* If that didn't work fall back on linearising by hand. */
if (ret == -ENOTSUPP) {
diff --git a/drivers/base/soc.c b/drivers/base/soc.c
index 10b280f30217..7c0c5ca5953d 100644
--- a/drivers/base/soc.c
+++ b/drivers/base/soc.c
@@ -33,6 +33,7 @@ static struct bus_type soc_bus_type = {
static DEVICE_ATTR(machine, S_IRUGO, soc_info_get, NULL);
static DEVICE_ATTR(family, S_IRUGO, soc_info_get, NULL);
+static DEVICE_ATTR(serial_number, S_IRUGO, soc_info_get, NULL);
static DEVICE_ATTR(soc_id, S_IRUGO, soc_info_get, NULL);
static DEVICE_ATTR(revision, S_IRUGO, soc_info_get, NULL);
@@ -57,6 +58,9 @@ static umode_t soc_attribute_mode(struct kobject *kobj,
if ((attr == &dev_attr_revision.attr)
&& (soc_dev->attr->revision != NULL))
return attr->mode;
+ if ((attr == &dev_attr_serial_number.attr)
+ && (soc_dev->attr->serial_number != NULL))
+ return attr->mode;
if ((attr == &dev_attr_soc_id.attr)
&& (soc_dev->attr->soc_id != NULL))
return attr->mode;
@@ -77,6 +81,8 @@ static ssize_t soc_info_get(struct device *dev,
return sprintf(buf, "%s\n", soc_dev->attr->family);
if (attr == &dev_attr_revision)
return sprintf(buf, "%s\n", soc_dev->attr->revision);
+ if (attr == &dev_attr_serial_number)
+ return sprintf(buf, "%s\n", soc_dev->attr->serial_number);
if (attr == &dev_attr_soc_id)
return sprintf(buf, "%s\n", soc_dev->attr->soc_id);
@@ -87,6 +93,7 @@ static ssize_t soc_info_get(struct device *dev,
static struct attribute *soc_attr[] = {
&dev_attr_machine.attr,
&dev_attr_family.attr,
+ &dev_attr_serial_number.attr,
&dev_attr_soc_id.attr,
&dev_attr_revision.attr,
NULL,
@@ -157,6 +164,7 @@ out2:
out1:
return ERR_PTR(ret);
}
+EXPORT_SYMBOL_GPL(soc_device_register);
/* Ensure soc_dev->attr is freed prior to calling soc_device_unregister. */
void soc_device_unregister(struct soc_device *soc_dev)
@@ -166,6 +174,7 @@ void soc_device_unregister(struct soc_device *soc_dev)
device_unregister(&soc_dev->dev);
early_soc_dev_attr = NULL;
}
+EXPORT_SYMBOL_GPL(soc_device_unregister);
static int __init soc_bus_register(void)
{
diff --git a/drivers/base/swnode.c b/drivers/base/swnode.c
index 7fc5a18e02ad..a1f3f0994f9f 100644
--- a/drivers/base/swnode.c
+++ b/drivers/base/swnode.c
@@ -11,25 +11,25 @@
#include <linux/property.h>
#include <linux/slab.h>
-struct software_node {
+struct swnode {
int id;
struct kobject kobj;
struct fwnode_handle fwnode;
+ const struct software_node *node;
/* hierarchy */
struct ida child_ids;
struct list_head entry;
struct list_head children;
- struct software_node *parent;
+ struct swnode *parent;
- /* properties */
- const struct property_entry *properties;
+ unsigned int allocated:1;
};
static DEFINE_IDA(swnode_root_ids);
static struct kset *swnode_kset;
-#define kobj_to_swnode(_kobj_) container_of(_kobj_, struct software_node, kobj)
+#define kobj_to_swnode(_kobj_) container_of(_kobj_, struct swnode, kobj)
static const struct fwnode_operations software_node_ops;
@@ -37,17 +37,56 @@ bool is_software_node(const struct fwnode_handle *fwnode)
{
return !IS_ERR_OR_NULL(fwnode) && fwnode->ops == &software_node_ops;
}
+EXPORT_SYMBOL_GPL(is_software_node);
-#define to_software_node(__fwnode) \
+#define to_swnode(__fwnode) \
({ \
- typeof(__fwnode) __to_software_node_fwnode = __fwnode; \
+ typeof(__fwnode) __to_swnode_fwnode = __fwnode; \
\
- is_software_node(__to_software_node_fwnode) ? \
- container_of(__to_software_node_fwnode, \
- struct software_node, fwnode) : \
- NULL; \
+ is_software_node(__to_swnode_fwnode) ? \
+ container_of(__to_swnode_fwnode, \
+ struct swnode, fwnode) : NULL; \
})
+static struct swnode *
+software_node_to_swnode(const struct software_node *node)
+{
+ struct swnode *swnode = NULL;
+ struct kobject *k;
+
+ if (!node)
+ return NULL;
+
+ spin_lock(&swnode_kset->list_lock);
+
+ list_for_each_entry(k, &swnode_kset->list, entry) {
+ swnode = kobj_to_swnode(k);
+ if (swnode->node == node)
+ break;
+ swnode = NULL;
+ }
+
+ spin_unlock(&swnode_kset->list_lock);
+
+ return swnode;
+}
+
+const struct software_node *to_software_node(struct fwnode_handle *fwnode)
+{
+ struct swnode *swnode = to_swnode(fwnode);
+
+ return swnode ? swnode->node : NULL;
+}
+EXPORT_SYMBOL_GPL(to_software_node);
+
+struct fwnode_handle *software_node_fwnode(const struct software_node *node)
+{
+ struct swnode *swnode = software_node_to_swnode(node);
+
+ return swnode ? &swnode->fwnode : NULL;
+}
+EXPORT_SYMBOL_GPL(software_node_fwnode);
+
/* -------------------------------------------------------------------------- */
/* property_entry processing */
@@ -383,6 +422,9 @@ property_entries_dup(const struct property_entry *properties)
int i, n = 0;
int ret;
+ if (!properties)
+ return NULL;
+
while (properties[n].name)
n++;
@@ -430,7 +472,7 @@ EXPORT_SYMBOL_GPL(property_entries_free);
static struct fwnode_handle *software_node_get(struct fwnode_handle *fwnode)
{
- struct software_node *swnode = to_software_node(fwnode);
+ struct swnode *swnode = to_swnode(fwnode);
kobject_get(&swnode->kobj);
@@ -439,7 +481,7 @@ static struct fwnode_handle *software_node_get(struct fwnode_handle *fwnode)
static void software_node_put(struct fwnode_handle *fwnode)
{
- struct software_node *swnode = to_software_node(fwnode);
+ struct swnode *swnode = to_swnode(fwnode);
kobject_put(&swnode->kobj);
}
@@ -447,8 +489,9 @@ static void software_node_put(struct fwnode_handle *fwnode)
static bool software_node_property_present(const struct fwnode_handle *fwnode,
const char *propname)
{
- return !!property_entry_get(to_software_node(fwnode)->properties,
- propname);
+ struct swnode *swnode = to_swnode(fwnode);
+
+ return !!property_entry_get(swnode->node->properties, propname);
}
static int software_node_read_int_array(const struct fwnode_handle *fwnode,
@@ -456,9 +499,9 @@ static int software_node_read_int_array(const struct fwnode_handle *fwnode,
unsigned int elem_size, void *val,
size_t nval)
{
- struct software_node *swnode = to_software_node(fwnode);
+ struct swnode *swnode = to_swnode(fwnode);
- return property_entry_read_int_array(swnode->properties, propname,
+ return property_entry_read_int_array(swnode->node->properties, propname,
elem_size, val, nval);
}
@@ -466,27 +509,26 @@ static int software_node_read_string_array(const struct fwnode_handle *fwnode,
const char *propname,
const char **val, size_t nval)
{
- struct software_node *swnode = to_software_node(fwnode);
+ struct swnode *swnode = to_swnode(fwnode);
- return property_entry_read_string_array(swnode->properties, propname,
- val, nval);
+ return property_entry_read_string_array(swnode->node->properties,
+ propname, val, nval);
}
static struct fwnode_handle *
software_node_get_parent(const struct fwnode_handle *fwnode)
{
- struct software_node *swnode = to_software_node(fwnode);
+ struct swnode *swnode = to_swnode(fwnode);
- return swnode ? (swnode->parent ? &swnode->parent->fwnode : NULL) :
- NULL;
+ return swnode ? (swnode->parent ? &swnode->parent->fwnode : NULL) : NULL;
}
static struct fwnode_handle *
software_node_get_next_child(const struct fwnode_handle *fwnode,
struct fwnode_handle *child)
{
- struct software_node *p = to_software_node(fwnode);
- struct software_node *c = to_software_node(child);
+ struct swnode *p = to_swnode(fwnode);
+ struct swnode *c = to_swnode(child);
if (!p || list_empty(&p->children) ||
(c && list_is_last(&c->entry, &p->children)))
@@ -495,7 +537,7 @@ software_node_get_next_child(const struct fwnode_handle *fwnode,
if (c)
c = list_next_entry(c, entry);
else
- c = list_first_entry(&p->children, struct software_node, entry);
+ c = list_first_entry(&p->children, struct swnode, entry);
return &c->fwnode;
}
@@ -503,18 +545,14 @@ static struct fwnode_handle *
software_node_get_named_child_node(const struct fwnode_handle *fwnode,
const char *childname)
{
- struct software_node *swnode = to_software_node(fwnode);
- const struct property_entry *prop;
- struct software_node *child;
+ struct swnode *swnode = to_swnode(fwnode);
+ struct swnode *child;
if (!swnode || list_empty(&swnode->children))
return NULL;
list_for_each_entry(child, &swnode->children, entry) {
- prop = property_entry_get(child->properties, "name");
- if (!prop)
- continue;
- if (!strcmp(childname, prop->value.str)) {
+ if (!strcmp(childname, kobject_name(&child->kobj))) {
kobject_get(&child->kobj);
return &child->fwnode;
}
@@ -522,6 +560,52 @@ software_node_get_named_child_node(const struct fwnode_handle *fwnode,
return NULL;
}
+static int
+software_node_get_reference_args(const struct fwnode_handle *fwnode,
+ const char *propname, const char *nargs_prop,
+ unsigned int nargs, unsigned int index,
+ struct fwnode_reference_args *args)
+{
+ struct swnode *swnode = to_swnode(fwnode);
+ const struct software_node_reference *ref;
+ const struct property_entry *prop;
+ struct fwnode_handle *refnode;
+ int i;
+
+ if (!swnode || !swnode->node->references)
+ return -ENOENT;
+
+ for (ref = swnode->node->references; ref->name; ref++)
+ if (!strcmp(ref->name, propname))
+ break;
+
+ if (!ref->name || index > (ref->nrefs - 1))
+ return -ENOENT;
+
+ refnode = software_node_fwnode(ref->refs[index].node);
+ if (!refnode)
+ return -ENOENT;
+
+ if (nargs_prop) {
+ prop = property_entry_get(swnode->node->properties, nargs_prop);
+ if (!prop)
+ return -EINVAL;
+
+ nargs = prop->value.u32_data;
+ }
+
+ if (nargs > NR_FWNODE_REFERENCE_ARGS)
+ return -EINVAL;
+
+ args->fwnode = software_node_get(refnode);
+ args->nargs = nargs;
+
+ for (i = 0; i < nargs; i++)
+ args->args[i] = ref->refs[index].args[i];
+
+ return 0;
+}
+
static const struct fwnode_operations software_node_ops = {
.get = software_node_get,
.put = software_node_put,
@@ -531,12 +615,50 @@ static const struct fwnode_operations software_node_ops = {
.get_parent = software_node_get_parent,
.get_next_child_node = software_node_get_next_child,
.get_named_child_node = software_node_get_named_child_node,
+ .get_reference_args = software_node_get_reference_args
};
/* -------------------------------------------------------------------------- */
+/**
+ * software_node_find_by_name - Find software node by name
+ * @parent: Parent of the software node
+ * @name: Name of the software node
+ *
+ * The function will find a node that is child of @parent and that is named
+ * @name. If no node is found, the function returns NULL.
+ *
+ * NOTE: you will need to drop the reference with fwnode_handle_put() after use.
+ */
+const struct software_node *
+software_node_find_by_name(const struct software_node *parent, const char *name)
+{
+ struct swnode *swnode = NULL;
+ struct kobject *k;
+
+ if (!name)
+ return NULL;
+
+ spin_lock(&swnode_kset->list_lock);
+
+ list_for_each_entry(k, &swnode_kset->list, entry) {
+ swnode = kobj_to_swnode(k);
+ if (parent == swnode->node->parent && swnode->node->name &&
+ !strcmp(name, swnode->node->name)) {
+ kobject_get(&swnode->kobj);
+ break;
+ }
+ swnode = NULL;
+ }
+
+ spin_unlock(&swnode_kset->list_lock);
+
+ return swnode ? swnode->node : NULL;
+}
+EXPORT_SYMBOL_GPL(software_node_find_by_name);
+
static int
-software_node_register_properties(struct software_node *swnode,
+software_node_register_properties(struct software_node *node,
const struct property_entry *properties)
{
struct property_entry *props;
@@ -545,24 +667,20 @@ software_node_register_properties(struct software_node *swnode,
if (IS_ERR(props))
return PTR_ERR(props);
- swnode->properties = props;
+ node->properties = props;
return 0;
}
static void software_node_release(struct kobject *kobj)
{
- struct software_node *swnode = kobj_to_swnode(kobj);
+ struct swnode *swnode = kobj_to_swnode(kobj);
- if (swnode->parent) {
- ida_simple_remove(&swnode->parent->child_ids, swnode->id);
- list_del(&swnode->entry);
- } else {
- ida_simple_remove(&swnode_root_ids, swnode->id);
+ if (swnode->allocated) {
+ property_entries_free(swnode->node->properties);
+ kfree(swnode->node);
}
-
ida_destroy(&swnode->child_ids);
- property_entries_free(swnode->properties);
kfree(swnode);
}
@@ -571,70 +689,165 @@ static struct kobj_type software_node_type = {
.sysfs_ops = &kobj_sysfs_ops,
};
-struct fwnode_handle *
-fwnode_create_software_node(const struct property_entry *properties,
- const struct fwnode_handle *parent)
+static struct fwnode_handle *
+swnode_register(const struct software_node *node, struct swnode *parent,
+ unsigned int allocated)
{
- struct software_node *p = NULL;
- struct software_node *swnode;
+ struct swnode *swnode;
int ret;
- if (parent) {
- if (IS_ERR(parent))
- return ERR_CAST(parent);
- if (!is_software_node(parent))
- return ERR_PTR(-EINVAL);
- p = to_software_node(parent);
- }
-
swnode = kzalloc(sizeof(*swnode), GFP_KERNEL);
- if (!swnode)
- return ERR_PTR(-ENOMEM);
+ if (!swnode) {
+ ret = -ENOMEM;
+ goto out_err;
+ }
- ret = ida_simple_get(p ? &p->child_ids : &swnode_root_ids, 0, 0,
- GFP_KERNEL);
+ ret = ida_simple_get(parent ? &parent->child_ids : &swnode_root_ids,
+ 0, 0, GFP_KERNEL);
if (ret < 0) {
kfree(swnode);
- return ERR_PTR(ret);
+ goto out_err;
}
swnode->id = ret;
+ swnode->node = node;
+ swnode->parent = parent;
+ swnode->allocated = allocated;
swnode->kobj.kset = swnode_kset;
swnode->fwnode.ops = &software_node_ops;
ida_init(&swnode->child_ids);
INIT_LIST_HEAD(&swnode->entry);
INIT_LIST_HEAD(&swnode->children);
- swnode->parent = p;
-
- if (p)
- list_add_tail(&swnode->entry, &p->children);
- ret = kobject_init_and_add(&swnode->kobj, &software_node_type,
- p ? &p->kobj : NULL, "node%d", swnode->id);
+ if (node->name)
+ ret = kobject_init_and_add(&swnode->kobj, &software_node_type,
+ parent ? &parent->kobj : NULL,
+ "%s", node->name);
+ else
+ ret = kobject_init_and_add(&swnode->kobj, &software_node_type,
+ parent ? &parent->kobj : NULL,
+ "node%d", swnode->id);
if (ret) {
kobject_put(&swnode->kobj);
return ERR_PTR(ret);
}
- ret = software_node_register_properties(swnode, properties);
+ if (parent)
+ list_add_tail(&swnode->entry, &parent->children);
+
+ kobject_uevent(&swnode->kobj, KOBJ_ADD);
+ return &swnode->fwnode;
+
+out_err:
+ if (allocated)
+ property_entries_free(node->properties);
+ return ERR_PTR(ret);
+}
+
+/**
+ * software_node_register_nodes - Register an array of software nodes
+ * @nodes: Zero terminated array of software nodes to be registered
+ *
+ * Register multiple software nodes at once.
+ */
+int software_node_register_nodes(const struct software_node *nodes)
+{
+ int ret;
+ int i;
+
+ for (i = 0; nodes[i].name; i++) {
+ ret = software_node_register(&nodes[i]);
+ if (ret) {
+ software_node_unregister_nodes(nodes);
+ return ret;
+ }
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(software_node_register_nodes);
+
+/**
+ * software_node_unregister_nodes - Unregister an array of software nodes
+ * @nodes: Zero terminated array of software nodes to be unregistered
+ *
+ * Unregister multiple software nodes at once.
+ */
+void software_node_unregister_nodes(const struct software_node *nodes)
+{
+ struct swnode *swnode;
+ int i;
+
+ for (i = 0; nodes[i].name; i++) {
+ swnode = software_node_to_swnode(&nodes[i]);
+ if (swnode)
+ fwnode_remove_software_node(&swnode->fwnode);
+ }
+}
+EXPORT_SYMBOL_GPL(software_node_unregister_nodes);
+
+/**
+ * software_node_register - Register static software node
+ * @node: The software node to be registered
+ */
+int software_node_register(const struct software_node *node)
+{
+ struct swnode *parent = software_node_to_swnode(node->parent);
+
+ if (software_node_to_swnode(node))
+ return -EEXIST;
+
+ return PTR_ERR_OR_ZERO(swnode_register(node, parent, 0));
+}
+EXPORT_SYMBOL_GPL(software_node_register);
+
+struct fwnode_handle *
+fwnode_create_software_node(const struct property_entry *properties,
+ const struct fwnode_handle *parent)
+{
+ struct software_node *node;
+ struct swnode *p = NULL;
+ int ret;
+
+ if (parent) {
+ if (IS_ERR(parent))
+ return ERR_CAST(parent);
+ if (!is_software_node(parent))
+ return ERR_PTR(-EINVAL);
+ p = to_swnode(parent);
+ }
+
+ node = kzalloc(sizeof(*node), GFP_KERNEL);
+ if (!node)
+ return ERR_PTR(-ENOMEM);
+
+ ret = software_node_register_properties(node, properties);
if (ret) {
- kobject_put(&swnode->kobj);
+ kfree(node);
return ERR_PTR(ret);
}
- kobject_uevent(&swnode->kobj, KOBJ_ADD);
- return &swnode->fwnode;
+ node->parent = p ? p->node : NULL;
+
+ return swnode_register(node, p, 1);
}
EXPORT_SYMBOL_GPL(fwnode_create_software_node);
void fwnode_remove_software_node(struct fwnode_handle *fwnode)
{
- struct software_node *swnode = to_software_node(fwnode);
+ struct swnode *swnode = to_swnode(fwnode);
if (!swnode)
return;
+ if (swnode->parent) {
+ ida_simple_remove(&swnode->parent->child_ids, swnode->id);
+ list_del(&swnode->entry);
+ } else {
+ ida_simple_remove(&swnode_root_ids, swnode->id);
+ }
+
kobject_put(&swnode->kobj);
}
EXPORT_SYMBOL_GPL(fwnode_remove_software_node);
@@ -642,7 +855,7 @@ EXPORT_SYMBOL_GPL(fwnode_remove_software_node);
int software_node_notify(struct device *dev, unsigned long action)
{
struct fwnode_handle *fwnode = dev_fwnode(dev);
- struct software_node *swnode;
+ struct swnode *swnode;
int ret;
if (!fwnode)
@@ -653,7 +866,7 @@ int software_node_notify(struct device *dev, unsigned long action)
if (!is_software_node(fwnode))
return 0;
- swnode = to_software_node(fwnode);
+ swnode = to_swnode(fwnode);
switch (action) {
case KOBJ_ADD:
diff --git a/drivers/base/topology.c b/drivers/base/topology.c
index 5fd9f167ecc1..4e033d4cc0dc 100644
--- a/drivers/base/topology.c
+++ b/drivers/base/topology.c
@@ -43,6 +43,9 @@ static ssize_t name##_list_show(struct device *dev, \
define_id_show_func(physical_package_id);
static DEVICE_ATTR_RO(physical_package_id);
+define_id_show_func(die_id);
+static DEVICE_ATTR_RO(die_id);
+
define_id_show_func(core_id);
static DEVICE_ATTR_RO(core_id);
@@ -50,10 +53,22 @@ define_siblings_show_func(thread_siblings, sibling_cpumask);
static DEVICE_ATTR_RO(thread_siblings);
static DEVICE_ATTR_RO(thread_siblings_list);
+define_siblings_show_func(core_cpus, sibling_cpumask);
+static DEVICE_ATTR_RO(core_cpus);
+static DEVICE_ATTR_RO(core_cpus_list);
+
define_siblings_show_func(core_siblings, core_cpumask);
static DEVICE_ATTR_RO(core_siblings);
static DEVICE_ATTR_RO(core_siblings_list);
+define_siblings_show_func(die_cpus, die_cpumask);
+static DEVICE_ATTR_RO(die_cpus);
+static DEVICE_ATTR_RO(die_cpus_list);
+
+define_siblings_show_func(package_cpus, core_cpumask);
+static DEVICE_ATTR_RO(package_cpus);
+static DEVICE_ATTR_RO(package_cpus_list);
+
#ifdef CONFIG_SCHED_BOOK
define_id_show_func(book_id);
static DEVICE_ATTR_RO(book_id);
@@ -72,11 +87,18 @@ static DEVICE_ATTR_RO(drawer_siblings_list);
static struct attribute *default_attrs[] = {
&dev_attr_physical_package_id.attr,
+ &dev_attr_die_id.attr,
&dev_attr_core_id.attr,
&dev_attr_thread_siblings.attr,
&dev_attr_thread_siblings_list.attr,
+ &dev_attr_core_cpus.attr,
+ &dev_attr_core_cpus_list.attr,
&dev_attr_core_siblings.attr,
&dev_attr_core_siblings_list.attr,
+ &dev_attr_die_cpus.attr,
+ &dev_attr_die_cpus_list.attr,
+ &dev_attr_package_cpus.attr,
+ &dev_attr_package_cpus_list.attr,
#ifdef CONFIG_SCHED_BOOK
&dev_attr_book_id.attr,
&dev_attr_book_siblings.attr,
diff --git a/drivers/bcma/driver_mips.c b/drivers/bcma/driver_mips.c
index 27e9686b6d3a..87760aa60446 100644
--- a/drivers/bcma/driver_mips.c
+++ b/drivers/bcma/driver_mips.c
@@ -59,22 +59,6 @@ static inline void mips_write32(struct bcma_drv_mips *mcore,
bcma_write32(mcore->core, offset, value);
}
-static const u32 ipsflag_irq_mask[] = {
- 0,
- BCMA_MIPS_IPSFLAG_IRQ1,
- BCMA_MIPS_IPSFLAG_IRQ2,
- BCMA_MIPS_IPSFLAG_IRQ3,
- BCMA_MIPS_IPSFLAG_IRQ4,
-};
-
-static const u32 ipsflag_irq_shift[] = {
- 0,
- BCMA_MIPS_IPSFLAG_IRQ1_SHIFT,
- BCMA_MIPS_IPSFLAG_IRQ2_SHIFT,
- BCMA_MIPS_IPSFLAG_IRQ3_SHIFT,
- BCMA_MIPS_IPSFLAG_IRQ4_SHIFT,
-};
-
static u32 bcma_core_mips_irqflag(struct bcma_device *dev)
{
u32 flag;
diff --git a/drivers/bcma/driver_pci.c b/drivers/bcma/driver_pci.c
index f499a469e66d..12b2cc9a3fbe 100644
--- a/drivers/bcma/driver_pci.c
+++ b/drivers/bcma/driver_pci.c
@@ -78,7 +78,7 @@ static u16 bcma_pcie_mdio_read(struct bcma_drv_pci *pc, u16 device, u8 address)
v |= (address << BCMA_CORE_PCI_MDIODATA_REGADDR_SHF_OLD);
}
- v = BCMA_CORE_PCI_MDIODATA_START;
+ v |= BCMA_CORE_PCI_MDIODATA_START;
v |= BCMA_CORE_PCI_MDIODATA_READ;
v |= BCMA_CORE_PCI_MDIODATA_TA;
@@ -121,7 +121,7 @@ static void bcma_pcie_mdio_write(struct bcma_drv_pci *pc, u16 device,
v |= (address << BCMA_CORE_PCI_MDIODATA_REGADDR_SHF_OLD);
}
- v = BCMA_CORE_PCI_MDIODATA_START;
+ v |= BCMA_CORE_PCI_MDIODATA_START;
v |= BCMA_CORE_PCI_MDIODATA_WRITE;
v |= BCMA_CORE_PCI_MDIODATA_TA;
v |= data;
diff --git a/drivers/bcma/host_pci.c b/drivers/bcma/host_pci.c
index f52239feb4cb..69c10a7b7c61 100644
--- a/drivers/bcma/host_pci.c
+++ b/drivers/bcma/host_pci.c
@@ -260,8 +260,7 @@ static void bcma_host_pci_remove(struct pci_dev *dev)
#ifdef CONFIG_PM_SLEEP
static int bcma_host_pci_suspend(struct device *dev)
{
- struct pci_dev *pdev = to_pci_dev(dev);
- struct bcma_bus *bus = pci_get_drvdata(pdev);
+ struct bcma_bus *bus = dev_get_drvdata(dev);
bus->mapped_core = NULL;
@@ -270,8 +269,7 @@ static int bcma_host_pci_suspend(struct device *dev)
static int bcma_host_pci_resume(struct device *dev)
{
- struct pci_dev *pdev = to_pci_dev(dev);
- struct bcma_bus *bus = pci_get_drvdata(pdev);
+ struct bcma_bus *bus = dev_get_drvdata(dev);
return bcma_bus_resume(bus);
}
diff --git a/drivers/bcma/sprom.c b/drivers/bcma/sprom.c
index 206edd3ba668..bd2c923a6586 100644
--- a/drivers/bcma/sprom.c
+++ b/drivers/bcma/sprom.c
@@ -222,7 +222,7 @@ static void bcma_sprom_extract_r8(struct bcma_bus *bus, const u16 *sprom)
{
u16 v, o;
int i;
- u16 pwr_info_offset[] = {
+ static const u16 pwr_info_offset[] = {
SSB_SROM8_PWR_INFO_CORE0, SSB_SROM8_PWR_INFO_CORE1,
SSB_SROM8_PWR_INFO_CORE2, SSB_SROM8_PWR_INFO_CORE3
};
@@ -578,9 +578,11 @@ int bcma_sprom_get(struct bcma_bus *bus)
{
u16 offset = BCMA_CC_SPROM;
u16 *sprom;
- size_t sprom_sizes[] = { SSB_SPROMSIZE_WORDS_R4,
- SSB_SPROMSIZE_WORDS_R10,
- SSB_SPROMSIZE_WORDS_R11, };
+ static const size_t sprom_sizes[] = {
+ SSB_SPROMSIZE_WORDS_R4,
+ SSB_SPROMSIZE_WORDS_R10,
+ SSB_SPROMSIZE_WORDS_R11,
+ };
int i, err = 0;
if (!bus->drv_cc.core)
diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig
index 20bb4bfa4be6..1bb8ec575352 100644
--- a/drivers/block/Kconfig
+++ b/drivers/block/Kconfig
@@ -31,7 +31,7 @@ config BLK_DEV_FD
If you want to use the floppy disk drive(s) of your PC under Linux,
say Y. Information about this driver, especially important for IBM
Thinkpad users, is contained in
- <file:Documentation/blockdev/floppy.txt>.
+ <file:Documentation/admin-guide/blockdev/floppy.rst>.
That file also contains the location of the Floppy driver FAQ as
well as location of the fdutils package used to configure additional
parameters of the driver at run time.
@@ -96,7 +96,7 @@ config PARIDE
your computer's parallel port. Most of them are actually IDE devices
using a parallel port IDE adapter. This option enables the PARIDE
subsystem which contains drivers for many of these external drives.
- Read <file:Documentation/blockdev/paride.txt> for more information.
+ Read <file:Documentation/admin-guide/blockdev/paride.rst> for more information.
If you have said Y to the "Parallel-port support" configuration
option, you may share a single port between your printer and other
@@ -261,7 +261,7 @@ config BLK_DEV_NBD
userland (making server and client physically the same computer,
communicating using the loopback network device).
- Read <file:Documentation/blockdev/nbd.txt> for more information,
+ Read <file:Documentation/admin-guide/blockdev/nbd.rst> for more information,
especially about where to find the server code, which runs in user
space and does not need special kernel support.
@@ -303,7 +303,7 @@ config BLK_DEV_RAM
during the initial install of Linux.
Note that the kernel command line option "ramdisk=XX" is now obsolete.
- For details, read <file:Documentation/blockdev/ramdisk.txt>.
+ For details, read <file:Documentation/admin-guide/blockdev/ramdisk.rst>.
To compile this driver as a module, choose M here: the
module will be called brd. An alias "rd" has been defined
@@ -347,7 +347,7 @@ config CDROM_PKTCDVD
is possible.
DVD-RW disks must be in restricted overwrite mode.
- See the file <file:Documentation/cdrom/packet-writing.txt>
+ See the file <file:Documentation/cdrom/packet-writing.rst>
for further information on the use of this driver.
To compile this driver as a module, choose M here: the
diff --git a/drivers/block/aoe/aoedev.c b/drivers/block/aoe/aoedev.c
index 5b49f1b33ebe..e2ea2356da06 100644
--- a/drivers/block/aoe/aoedev.c
+++ b/drivers/block/aoe/aoedev.c
@@ -323,10 +323,14 @@ flush(const char __user *str, size_t cnt, int exiting)
}
flush_scheduled_work();
- /* pass one: without sleeping, do aoedev_downdev */
+ /* pass one: do aoedev_downdev, which might sleep */
+restart1:
spin_lock_irqsave(&devlist_lock, flags);
for (d = devlist; d; d = d->next) {
spin_lock(&d->lock);
+ if (d->flags & DEVFL_TKILL)
+ goto cont;
+
if (exiting) {
/* unconditionally take each device down */
} else if (specified) {
@@ -338,8 +342,11 @@ flush(const char __user *str, size_t cnt, int exiting)
|| d->ref)
goto cont;
+ spin_unlock(&d->lock);
+ spin_unlock_irqrestore(&devlist_lock, flags);
aoedev_downdev(d);
d->flags |= DEVFL_TKILL;
+ goto restart1;
cont:
spin_unlock(&d->lock);
}
@@ -348,7 +355,7 @@ cont:
/* pass two: call freedev, which might sleep,
* for aoedevs marked with DEVFL_TKILL
*/
-restart:
+restart2:
spin_lock_irqsave(&devlist_lock, flags);
for (d = devlist; d; d = d->next) {
spin_lock(&d->lock);
@@ -357,7 +364,7 @@ restart:
spin_unlock(&d->lock);
spin_unlock_irqrestore(&devlist_lock, flags);
freedev(d);
- goto restart;
+ goto restart2;
}
spin_unlock(&d->lock);
}
diff --git a/drivers/block/ataflop.c b/drivers/block/ataflop.c
index 85f20e371f2f..bd7d3bb8b890 100644
--- a/drivers/block/ataflop.c
+++ b/drivers/block/ataflop.c
@@ -1726,6 +1726,7 @@ static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode,
/* MSch: invalidate default_params */
default_params[drive].blocks = 0;
set_capacity(floppy->disk, MAX_DISK_SIZE * 2);
+ /* Fall through */
case FDFMTEND:
case FDFLUSH:
/* invalidate the buffer track to force a reread */
diff --git a/drivers/block/drbd/drbd_debugfs.c b/drivers/block/drbd/drbd_debugfs.c
index f13b48ff5f43..b3b9cd5628fd 100644
--- a/drivers/block/drbd/drbd_debugfs.c
+++ b/drivers/block/drbd/drbd_debugfs.c
@@ -465,35 +465,20 @@ static const struct file_operations in_flight_summary_fops = {
void drbd_debugfs_resource_add(struct drbd_resource *resource)
{
struct dentry *dentry;
- if (!drbd_debugfs_resources)
- return;
dentry = debugfs_create_dir(resource->name, drbd_debugfs_resources);
- if (IS_ERR_OR_NULL(dentry))
- goto fail;
resource->debugfs_res = dentry;
dentry = debugfs_create_dir("volumes", resource->debugfs_res);
- if (IS_ERR_OR_NULL(dentry))
- goto fail;
resource->debugfs_res_volumes = dentry;
dentry = debugfs_create_dir("connections", resource->debugfs_res);
- if (IS_ERR_OR_NULL(dentry))
- goto fail;
resource->debugfs_res_connections = dentry;
dentry = debugfs_create_file("in_flight_summary", 0440,
resource->debugfs_res, resource,
&in_flight_summary_fops);
- if (IS_ERR_OR_NULL(dentry))
- goto fail;
resource->debugfs_res_in_flight_summary = dentry;
- return;
-
-fail:
- drbd_debugfs_resource_cleanup(resource);
- drbd_err(resource, "failed to create debugfs dentry\n");
}
static void drbd_debugfs_remove(struct dentry **dp)
@@ -636,35 +621,22 @@ void drbd_debugfs_connection_add(struct drbd_connection *connection)
{
struct dentry *conns_dir = connection->resource->debugfs_res_connections;
struct dentry *dentry;
- if (!conns_dir)
- return;
/* Once we enable mutliple peers,
* these connections will have descriptive names.
* For now, it is just the one connection to the (only) "peer". */
dentry = debugfs_create_dir("peer", conns_dir);
- if (IS_ERR_OR_NULL(dentry))
- goto fail;
connection->debugfs_conn = dentry;
dentry = debugfs_create_file("callback_history", 0440,
connection->debugfs_conn, connection,
&connection_callback_history_fops);
- if (IS_ERR_OR_NULL(dentry))
- goto fail;
connection->debugfs_conn_callback_history = dentry;
dentry = debugfs_create_file("oldest_requests", 0440,
connection->debugfs_conn, connection,
&connection_oldest_requests_fops);
- if (IS_ERR_OR_NULL(dentry))
- goto fail;
connection->debugfs_conn_oldest_requests = dentry;
- return;
-
-fail:
- drbd_debugfs_connection_cleanup(connection);
- drbd_err(connection, "failed to create debugfs dentry\n");
}
void drbd_debugfs_connection_cleanup(struct drbd_connection *connection)
@@ -809,8 +781,6 @@ void drbd_debugfs_device_add(struct drbd_device *device)
snprintf(vnr_buf, sizeof(vnr_buf), "%u", device->vnr);
dentry = debugfs_create_dir(vnr_buf, vols_dir);
- if (IS_ERR_OR_NULL(dentry))
- goto fail;
device->debugfs_vol = dentry;
snprintf(minor_buf, sizeof(minor_buf), "%u", device->minor);
@@ -819,18 +789,14 @@ void drbd_debugfs_device_add(struct drbd_device *device)
if (!slink_name)
goto fail;
dentry = debugfs_create_symlink(minor_buf, drbd_debugfs_minors, slink_name);
+ device->debugfs_minor = dentry;
kfree(slink_name);
slink_name = NULL;
- if (IS_ERR_OR_NULL(dentry))
- goto fail;
- device->debugfs_minor = dentry;
#define DCF(name) do { \
dentry = debugfs_create_file(#name, 0440, \
device->debugfs_vol, device, \
&device_ ## name ## _fops); \
- if (IS_ERR_OR_NULL(dentry)) \
- goto fail; \
device->debugfs_vol_ ## name = dentry; \
} while (0)
@@ -864,19 +830,9 @@ void drbd_debugfs_peer_device_add(struct drbd_peer_device *peer_device)
struct dentry *dentry;
char vnr_buf[8];
- if (!conn_dir)
- return;
-
snprintf(vnr_buf, sizeof(vnr_buf), "%u", peer_device->device->vnr);
dentry = debugfs_create_dir(vnr_buf, conn_dir);
- if (IS_ERR_OR_NULL(dentry))
- goto fail;
peer_device->debugfs_peer_dev = dentry;
- return;
-
-fail:
- drbd_debugfs_peer_device_cleanup(peer_device);
- drbd_err(peer_device, "failed to create debugfs entries\n");
}
void drbd_debugfs_peer_device_cleanup(struct drbd_peer_device *peer_device)
@@ -917,35 +873,19 @@ void drbd_debugfs_cleanup(void)
drbd_debugfs_remove(&drbd_debugfs_root);
}
-int __init drbd_debugfs_init(void)
+void __init drbd_debugfs_init(void)
{
struct dentry *dentry;
dentry = debugfs_create_dir("drbd", NULL);
- if (IS_ERR_OR_NULL(dentry))
- goto fail;
drbd_debugfs_root = dentry;
dentry = debugfs_create_file("version", 0444, drbd_debugfs_root, NULL, &drbd_version_fops);
- if (IS_ERR_OR_NULL(dentry))
- goto fail;
drbd_debugfs_version = dentry;
dentry = debugfs_create_dir("resources", drbd_debugfs_root);
- if (IS_ERR_OR_NULL(dentry))
- goto fail;
drbd_debugfs_resources = dentry;
dentry = debugfs_create_dir("minors", drbd_debugfs_root);
- if (IS_ERR_OR_NULL(dentry))
- goto fail;
drbd_debugfs_minors = dentry;
- return 0;
-
-fail:
- drbd_debugfs_cleanup();
- if (dentry)
- return PTR_ERR(dentry);
- else
- return -EINVAL;
}
diff --git a/drivers/block/drbd/drbd_debugfs.h b/drivers/block/drbd/drbd_debugfs.h
index 4ecfbb3358d7..58e31cef0844 100644
--- a/drivers/block/drbd/drbd_debugfs.h
+++ b/drivers/block/drbd/drbd_debugfs.h
@@ -6,7 +6,7 @@
#include "drbd_int.h"
#ifdef CONFIG_DEBUG_FS
-int __init drbd_debugfs_init(void);
+void __init drbd_debugfs_init(void);
void drbd_debugfs_cleanup(void);
void drbd_debugfs_resource_add(struct drbd_resource *resource);
@@ -22,7 +22,7 @@ void drbd_debugfs_peer_device_add(struct drbd_peer_device *peer_device);
void drbd_debugfs_peer_device_cleanup(struct drbd_peer_device *peer_device);
#else
-static inline int __init drbd_debugfs_init(void) { return -ENODEV; }
+static inline void __init drbd_debugfs_init(void) { }
static inline void drbd_debugfs_cleanup(void) { }
static inline void drbd_debugfs_resource_add(struct drbd_resource *resource) { }
diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index 31237f45247a..ddbf56014c51 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -1960,7 +1960,7 @@ static inline void wake_ack_receiver(struct drbd_connection *connection)
{
struct task_struct *task = connection->ack_receiver.task;
if (task && get_t_state(&connection->ack_receiver) == RUNNING)
- force_sig(SIGXCPU, task);
+ send_sig(SIGXCPU, task, 1);
}
static inline void request_ping(struct drbd_connection *connection)
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index 541b31fa42b3..5b248763a672 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -322,6 +322,8 @@ static int drbd_thread_setup(void *arg)
thi->name[0],
resource->name);
+ allow_kernel_signal(DRBD_SIGKILL);
+ allow_kernel_signal(SIGXCPU);
restart:
retval = thi->function(thi);
@@ -465,7 +467,7 @@ void _drbd_thread_stop(struct drbd_thread *thi, int restart, int wait)
smp_mb();
init_completion(&thi->stop);
if (thi->task != current)
- force_sig(DRBD_SIGKILL, thi->task);
+ send_sig(DRBD_SIGKILL, thi->task, 1);
}
spin_unlock_irqrestore(&thi->t_lock, flags);
@@ -3009,8 +3011,7 @@ static int __init drbd_init(void)
spin_lock_init(&retry.lock);
INIT_LIST_HEAD(&retry.writes);
- if (drbd_debugfs_init())
- pr_notice("failed to initialize debugfs -- will not be available\n");
+ drbd_debugfs_init();
pr_info("initialized. "
"Version: " REL_VERSION " (api:%d/proto:%d-%d)\n",
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index cdd748b8116d..5d52a2d32155 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -599,7 +599,7 @@ void conn_try_outdate_peer_async(struct drbd_connection *connection)
struct task_struct *opa;
kref_get(&connection->kref);
- /* We may just have force_sig()'ed this thread
+ /* We may have just sent a signal to this thread
* to get it out of some blocking network function.
* Clear signals; otherwise kthread_run(), which internally uses
* wait_on_completion_killable(), will mistake our pending signal
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index 90ebfcae0ce6..2b3103c30857 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -5417,7 +5417,7 @@ static int drbd_do_auth(struct drbd_connection *connection)
unsigned int key_len;
char secret[SHARED_SECRET_MAX]; /* 64 byte */
unsigned int resp_size;
- SHASH_DESC_ON_STACK(desc, connection->cram_hmac_tfm);
+ struct shash_desc *desc;
struct packet_info pi;
struct net_conf *nc;
int err, rv;
@@ -5430,6 +5430,13 @@ static int drbd_do_auth(struct drbd_connection *connection)
memcpy(secret, nc->shared_secret, key_len);
rcu_read_unlock();
+ desc = kmalloc(sizeof(struct shash_desc) +
+ crypto_shash_descsize(connection->cram_hmac_tfm),
+ GFP_KERNEL);
+ if (!desc) {
+ rv = -1;
+ goto fail;
+ }
desc->tfm = connection->cram_hmac_tfm;
rv = crypto_shash_setkey(connection->cram_hmac_tfm, (u8 *)secret, key_len);
@@ -5571,7 +5578,10 @@ static int drbd_do_auth(struct drbd_connection *connection)
kfree(peers_ch);
kfree(response);
kfree(right_response);
- shash_desc_zero(desc);
+ if (desc) {
+ shash_desc_zero(desc);
+ kfree(desc);
+ }
return rv;
}
diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c
index 9fb9b312ab6b..485865fd0412 100644
--- a/drivers/block/floppy.c
+++ b/drivers/block/floppy.c
@@ -2120,6 +2120,9 @@ static void setup_format_params(int track)
raw_cmd->kernel_data = floppy_track_buffer;
raw_cmd->length = 4 * F_SECT_PER_TRACK;
+ if (!F_SECT_PER_TRACK)
+ return;
+
/* allow for about 30ms for data transport per track */
head_shift = (F_SECT_PER_TRACK + 5) / 6;
@@ -3230,8 +3233,12 @@ static int set_geometry(unsigned int cmd, struct floppy_struct *g,
int cnt;
/* sanity checking for parameters. */
- if (g->sect <= 0 ||
- g->head <= 0 ||
+ if ((int)g->sect <= 0 ||
+ (int)g->head <= 0 ||
+ /* check for overflow in max_sector */
+ (int)(g->sect * g->head) <= 0 ||
+ /* check for zero in F_SECT_PER_TRACK */
+ (unsigned char)((g->sect << 2) >> FD_SIZECODE(g)) == 0 ||
g->track <= 0 || g->track > UDP->tracks >> STRETCH(g) ||
/* check if reserved bits are set */
(g->stretch & ~(FD_STRETCH | FD_SWAPSIDES | FD_SECTBASEMASK)) != 0)
@@ -3375,6 +3382,24 @@ static int fd_getgeo(struct block_device *bdev, struct hd_geometry *geo)
return 0;
}
+static bool valid_floppy_drive_params(const short autodetect[8],
+ int native_format)
+{
+ size_t floppy_type_size = ARRAY_SIZE(floppy_type);
+ size_t i = 0;
+
+ for (i = 0; i < 8; ++i) {
+ if (autodetect[i] < 0 ||
+ autodetect[i] >= floppy_type_size)
+ return false;
+ }
+
+ if (native_format < 0 || native_format >= floppy_type_size)
+ return false;
+
+ return true;
+}
+
static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
unsigned long param)
{
@@ -3501,6 +3526,9 @@ static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode, unsigned int
SUPBOUND(size, strlen((const char *)outparam) + 1);
break;
case FDSETDRVPRM:
+ if (!valid_floppy_drive_params(inparam.dp.autodetect,
+ inparam.dp.native_format))
+ return -EINVAL;
*UDP = inparam.dp;
break;
case FDGETDRVPRM:
@@ -3698,6 +3726,8 @@ static int compat_setdrvprm(int drive,
return -EPERM;
if (copy_from_user(&v, arg, sizeof(struct compat_floppy_drive_params)))
return -EFAULT;
+ if (!valid_floppy_drive_params(v.autodetect, v.native_format))
+ return -EINVAL;
mutex_lock(&floppy_mutex);
UDP->cmos = v.cmos;
UDP->max_dtr = v.max_dtr;
@@ -3750,7 +3780,7 @@ static int compat_getdrvprm(int drive,
v.native_format = UDP->native_format;
mutex_unlock(&floppy_mutex);
- if (copy_from_user(arg, &v, sizeof(struct compat_floppy_drive_params)))
+ if (copy_to_user(arg, &v, sizeof(struct compat_floppy_drive_params)))
return -EFAULT;
return 0;
}
@@ -3786,7 +3816,7 @@ static int compat_getdrvstat(int drive, bool poll,
v.bufblocks = UDRS->bufblocks;
mutex_unlock(&floppy_mutex);
- if (copy_from_user(arg, &v, sizeof(struct compat_floppy_drive_struct)))
+ if (copy_to_user(arg, &v, sizeof(struct compat_floppy_drive_struct)))
return -EFAULT;
return 0;
Eintr:
@@ -3900,7 +3930,7 @@ static void __init config_types(void)
if (!UDP->cmos)
UDP->cmos = FLOPPY0_TYPE;
drive = 1;
- if (!UDP->cmos && FLOPPY1_TYPE)
+ if (!UDP->cmos)
UDP->cmos = FLOPPY1_TYPE;
/* FIXME: additional physical CMOS drive detection should go here */
@@ -4424,7 +4454,7 @@ static int __init floppy_setup(char *str)
pr_cont("\n");
} else
DPRINT("botched floppy option\n");
- DPRINT("Read Documentation/blockdev/floppy.txt\n");
+ DPRINT("Read Documentation/admin-guide/blockdev/floppy.rst\n");
return 0;
}
diff --git a/drivers/block/loop.c b/drivers/block/loop.c
index f11b7dc16e9d..1410fa893653 100644
--- a/drivers/block/loop.c
+++ b/drivers/block/loop.c
@@ -264,20 +264,12 @@ lo_do_transfer(struct loop_device *lo, int cmd,
return ret;
}
-static inline void loop_iov_iter_bvec(struct iov_iter *i,
- unsigned int direction, const struct bio_vec *bvec,
- unsigned long nr_segs, size_t count)
-{
- iov_iter_bvec(i, direction, bvec, nr_segs, count);
- i->type |= ITER_BVEC_FLAG_NO_REF;
-}
-
static int lo_write_bvec(struct file *file, struct bio_vec *bvec, loff_t *ppos)
{
struct iov_iter i;
ssize_t bw;
- loop_iov_iter_bvec(&i, WRITE, bvec, 1, bvec->bv_len);
+ iov_iter_bvec(&i, WRITE, bvec, 1, bvec->bv_len);
file_start_write(file);
bw = vfs_iter_write(file, &i, ppos, 0);
@@ -355,7 +347,7 @@ static int lo_read_simple(struct loop_device *lo, struct request *rq,
ssize_t len;
rq_for_each_segment(bvec, rq, iter) {
- loop_iov_iter_bvec(&i, READ, &bvec, 1, bvec.bv_len);
+ iov_iter_bvec(&i, READ, &bvec, 1, bvec.bv_len);
len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0);
if (len < 0)
return len;
@@ -396,7 +388,7 @@ static int lo_read_transfer(struct loop_device *lo, struct request *rq,
b.bv_offset = 0;
b.bv_len = bvec.bv_len;
- loop_iov_iter_bvec(&i, READ, &b, 1, b.bv_len);
+ iov_iter_bvec(&i, READ, &b, 1, b.bv_len);
len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0);
if (len < 0) {
ret = len;
@@ -563,7 +555,7 @@ static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd,
}
atomic_set(&cmd->ref, 2);
- loop_iov_iter_bvec(&iter, rw, bvec, nr_bvec, blk_rq_bytes(rq));
+ iov_iter_bvec(&iter, rw, bvec, nr_bvec, blk_rq_bytes(rq));
iter.iov_offset = offset;
cmd->iocb.ki_pos = pos;
@@ -893,7 +885,7 @@ static void loop_unprepare_queue(struct loop_device *lo)
static int loop_kthread_worker_fn(void *worker_ptr)
{
- current->flags |= PF_LESS_THROTTLE;
+ current->flags |= PF_LESS_THROTTLE | PF_MEMALLOC_NOIO;
return kthread_worker_fn(worker_ptr);
}
@@ -932,6 +924,7 @@ static int loop_set_fd(struct loop_device *lo, fmode_t mode,
struct file *file;
struct inode *inode;
struct address_space *mapping;
+ struct block_device *claimed_bdev = NULL;
int lo_flags = 0;
int error;
loff_t size;
@@ -950,10 +943,11 @@ static int loop_set_fd(struct loop_device *lo, fmode_t mode,
* here to avoid changing device under exclusive owner.
*/
if (!(mode & FMODE_EXCL)) {
- bdgrab(bdev);
- error = blkdev_get(bdev, mode | FMODE_EXCL, loop_set_fd);
- if (error)
+ claimed_bdev = bd_start_claiming(bdev, loop_set_fd);
+ if (IS_ERR(claimed_bdev)) {
+ error = PTR_ERR(claimed_bdev);
goto out_putf;
+ }
}
error = mutex_lock_killable(&loop_ctl_mutex);
@@ -1023,15 +1017,15 @@ static int loop_set_fd(struct loop_device *lo, fmode_t mode,
mutex_unlock(&loop_ctl_mutex);
if (partscan)
loop_reread_partitions(lo, bdev);
- if (!(mode & FMODE_EXCL))
- blkdev_put(bdev, mode | FMODE_EXCL);
+ if (claimed_bdev)
+ bd_abort_claiming(bdev, claimed_bdev, loop_set_fd);
return 0;
out_unlock:
mutex_unlock(&loop_ctl_mutex);
out_bdev:
- if (!(mode & FMODE_EXCL))
- blkdev_put(bdev, mode | FMODE_EXCL);
+ if (claimed_bdev)
+ bd_abort_claiming(bdev, claimed_bdev, loop_set_fd);
out_putf:
fput(file);
out:
@@ -1761,6 +1755,7 @@ static int lo_compat_ioctl(struct block_device *bdev, fmode_t mode,
case LOOP_SET_FD:
case LOOP_CHANGE_FD:
case LOOP_SET_BLOCK_SIZE:
+ case LOOP_SET_DIRECT_IO:
err = lo_ioctl(bdev, mode, cmd, arg);
break;
default:
diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c
index a14b09ab3a41..964f78cfffa0 100644
--- a/drivers/block/mtip32xx/mtip32xx.c
+++ b/drivers/block/mtip32xx/mtip32xx.c
@@ -1577,7 +1577,6 @@ static int exec_drive_command(struct mtip_port *port, u8 *command,
ATA_SECT_SIZE * xfer_sz);
return -ENOMEM;
}
- memset(buf, 0, ATA_SECT_SIZE * xfer_sz);
}
/* Build the FIS. */
@@ -2776,7 +2775,6 @@ static int mtip_dma_alloc(struct driver_data *dd)
&port->block1_dma, GFP_KERNEL);
if (!port->block1)
return -ENOMEM;
- memset(port->block1, 0, BLOCK_DMA_ALLOC_SZ);
/* Allocate dma memory for command list */
port->command_list =
@@ -2789,7 +2787,6 @@ static int mtip_dma_alloc(struct driver_data *dd)
port->block1_dma = 0;
return -ENOMEM;
}
- memset(port->command_list, 0, AHCI_CMD_TBL_SZ);
/* Setup all pointers into first DMA region */
port->rxfis = port->block1 + AHCI_RX_FIS_OFFSET;
@@ -3529,8 +3526,6 @@ static int mtip_init_cmd(struct blk_mq_tag_set *set, struct request *rq,
if (!cmd->command)
return -ENOMEM;
- memset(cmd->command, 0, CMD_DMA_ALLOC_SZ);
-
sg_init_table(cmd->sg, MTIP_MAX_SG);
return 0;
}
diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index 3a9bca3aa093..a8e3815295fe 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -108,6 +108,7 @@ struct nbd_device {
struct nbd_config *config;
struct mutex config_lock;
struct gendisk *disk;
+ struct workqueue_struct *recv_workq;
struct list_head list;
struct task_struct *task_recv;
@@ -121,6 +122,7 @@ struct nbd_cmd {
struct mutex lock;
int index;
int cookie;
+ int retries;
blk_status_t status;
unsigned long flags;
u32 cmd_cookie;
@@ -134,9 +136,10 @@ static struct dentry *nbd_dbg_dir;
#define NBD_MAGIC 0x68797548
+#define NBD_DEF_BLKSIZE 1024
+
static unsigned int nbds_max = 16;
static int max_part = 16;
-static struct workqueue_struct *recv_workqueue;
static int part_shift;
static int nbd_dev_dbg_init(struct nbd_device *nbd);
@@ -342,6 +345,22 @@ static void sock_shutdown(struct nbd_device *nbd)
dev_warn(disk_to_dev(nbd->disk), "shutting down sockets\n");
}
+static u32 req_to_nbd_cmd_type(struct request *req)
+{
+ switch (req_op(req)) {
+ case REQ_OP_DISCARD:
+ return NBD_CMD_TRIM;
+ case REQ_OP_FLUSH:
+ return NBD_CMD_FLUSH;
+ case REQ_OP_WRITE:
+ return NBD_CMD_WRITE;
+ case REQ_OP_READ:
+ return NBD_CMD_READ;
+ default:
+ return U32_MAX;
+ }
+}
+
static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req,
bool reserved)
{
@@ -355,8 +374,10 @@ static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req,
}
config = nbd->config;
- if (!mutex_trylock(&cmd->lock))
+ if (!mutex_trylock(&cmd->lock)) {
+ nbd_config_put(nbd);
return BLK_EH_RESET_TIMER;
+ }
if (config->num_connections > 1) {
dev_err_ratelimited(nbd_to_dev(nbd),
@@ -387,10 +408,25 @@ static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req,
nbd_config_put(nbd);
return BLK_EH_DONE;
}
- } else {
- dev_err_ratelimited(nbd_to_dev(nbd),
- "Connection timed out\n");
}
+
+ if (!nbd->tag_set.timeout) {
+ /*
+ * Userspace sets timeout=0 to disable socket disconnection,
+ * so just warn and reset the timer.
+ */
+ cmd->retries++;
+ dev_info(nbd_to_dev(nbd), "Possible stuck request %p: control (%s@%llu,%uB). Runtime %u seconds\n",
+ req, nbdcmd_to_ascii(req_to_nbd_cmd_type(req)),
+ (unsigned long long)blk_rq_pos(req) << 9,
+ blk_rq_bytes(req), (req->timeout / HZ) * cmd->retries);
+
+ mutex_unlock(&cmd->lock);
+ nbd_config_put(nbd);
+ return BLK_EH_RESET_TIMER;
+ }
+
+ dev_err_ratelimited(nbd_to_dev(nbd), "Connection timed out\n");
set_bit(NBD_TIMEDOUT, &config->runtime_flags);
cmd->status = BLK_STS_IOERR;
mutex_unlock(&cmd->lock);
@@ -478,22 +514,9 @@ static int nbd_send_cmd(struct nbd_device *nbd, struct nbd_cmd *cmd, int index)
iov_iter_kvec(&from, WRITE, &iov, 1, sizeof(request));
- switch (req_op(req)) {
- case REQ_OP_DISCARD:
- type = NBD_CMD_TRIM;
- break;
- case REQ_OP_FLUSH:
- type = NBD_CMD_FLUSH;
- break;
- case REQ_OP_WRITE:
- type = NBD_CMD_WRITE;
- break;
- case REQ_OP_READ:
- type = NBD_CMD_READ;
- break;
- default:
+ type = req_to_nbd_cmd_type(req);
+ if (type == U32_MAX)
return -EIO;
- }
if (rq_data_dir(req) == WRITE &&
(config->flags & NBD_FLAG_READ_ONLY)) {
@@ -524,6 +547,7 @@ static int nbd_send_cmd(struct nbd_device *nbd, struct nbd_cmd *cmd, int index)
}
cmd->index = index;
cmd->cookie = nsock->cookie;
+ cmd->retries = 0;
request.type = htonl(type | nbd_cmd_flags);
if (type != NBD_CMD_FLUSH) {
request.from = cpu_to_be64((u64)blk_rq_pos(req) << 9);
@@ -1034,7 +1058,7 @@ static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg)
/* We take the tx_mutex in an error path in the recv_work, so we
* need to queue_work outside of the tx_mutex.
*/
- queue_work(recv_workqueue, &args->work);
+ queue_work(nbd->recv_workq, &args->work);
atomic_inc(&config->live_connections);
wake_up(&config->conn_wait);
@@ -1135,6 +1159,10 @@ static void nbd_config_put(struct nbd_device *nbd)
kfree(nbd->config);
nbd->config = NULL;
+ if (nbd->recv_workq)
+ destroy_workqueue(nbd->recv_workq);
+ nbd->recv_workq = NULL;
+
nbd->tag_set.timeout = 0;
nbd->disk->queue->limits.discard_granularity = 0;
nbd->disk->queue->limits.discard_alignment = 0;
@@ -1163,6 +1191,14 @@ static int nbd_start_device(struct nbd_device *nbd)
return -EINVAL;
}
+ nbd->recv_workq = alloc_workqueue("knbd%d-recv",
+ WQ_MEM_RECLAIM | WQ_HIGHPRI |
+ WQ_UNBOUND, 0, nbd->index);
+ if (!nbd->recv_workq) {
+ dev_err(disk_to_dev(nbd->disk), "Could not allocate knbd recv work queue.\n");
+ return -ENOMEM;
+ }
+
blk_mq_update_nr_hw_queues(&nbd->tag_set, config->num_connections);
nbd->task_recv = current;
@@ -1193,7 +1229,7 @@ static int nbd_start_device(struct nbd_device *nbd)
INIT_WORK(&args->work, recv_work);
args->nbd = nbd;
args->index = i;
- queue_work(recv_workqueue, &args->work);
+ queue_work(nbd->recv_workq, &args->work);
}
nbd_size_update(nbd);
return error;
@@ -1213,8 +1249,10 @@ static int nbd_start_device_ioctl(struct nbd_device *nbd, struct block_device *b
mutex_unlock(&nbd->config_lock);
ret = wait_event_interruptible(config->recv_wq,
atomic_read(&config->recv_threads) == 0);
- if (ret)
+ if (ret) {
sock_shutdown(nbd);
+ flush_workqueue(nbd->recv_workq);
+ }
mutex_lock(&nbd->config_lock);
nbd_bdev_reset(bdev);
/* user requested, ignore socket errors */
@@ -1229,13 +1267,28 @@ static void nbd_clear_sock_ioctl(struct nbd_device *nbd,
struct block_device *bdev)
{
sock_shutdown(nbd);
- kill_bdev(bdev);
+ __invalidate_device(bdev, true);
nbd_bdev_reset(bdev);
if (test_and_clear_bit(NBD_HAS_CONFIG_REF,
&nbd->config->runtime_flags))
nbd_config_put(nbd);
}
+static bool nbd_is_valid_blksize(unsigned long blksize)
+{
+ if (!blksize || !is_power_of_2(blksize) || blksize < 512 ||
+ blksize > PAGE_SIZE)
+ return false;
+ return true;
+}
+
+static void nbd_set_cmd_timeout(struct nbd_device *nbd, u64 timeout)
+{
+ nbd->tag_set.timeout = timeout * HZ;
+ if (timeout)
+ blk_queue_rq_timeout(nbd->disk->queue, timeout * HZ);
+}
+
/* Must be called with config_lock held */
static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
unsigned int cmd, unsigned long arg)
@@ -1251,8 +1304,9 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
case NBD_SET_SOCK:
return nbd_add_socket(nbd, arg, false);
case NBD_SET_BLKSIZE:
- if (!arg || !is_power_of_2(arg) || arg < 512 ||
- arg > PAGE_SIZE)
+ if (!arg)
+ arg = NBD_DEF_BLKSIZE;
+ if (!nbd_is_valid_blksize(arg))
return -EINVAL;
nbd_size_set(nbd, arg,
div_s64(config->bytesize, arg));
@@ -1265,10 +1319,7 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
nbd_size_set(nbd, config->blksize, arg);
return 0;
case NBD_SET_TIMEOUT:
- if (arg) {
- nbd->tag_set.timeout = arg * HZ;
- blk_queue_rq_timeout(nbd->disk->queue, arg * HZ);
- }
+ nbd_set_cmd_timeout(nbd, arg);
return 0;
case NBD_SET_FLAGS:
@@ -1332,7 +1383,7 @@ static struct nbd_config *nbd_alloc_config(void)
atomic_set(&config->recv_threads, 0);
init_waitqueue_head(&config->recv_wq);
init_waitqueue_head(&config->conn_wait);
- config->blksize = 1024;
+ config->blksize = NBD_DEF_BLKSIZE;
atomic_set(&config->live_connections, 0);
try_module_get(THIS_MODULE);
return config;
@@ -1673,6 +1724,30 @@ nbd_device_policy[NBD_DEVICE_ATTR_MAX + 1] = {
[NBD_DEVICE_CONNECTED] = { .type = NLA_U8 },
};
+static int nbd_genl_size_set(struct genl_info *info, struct nbd_device *nbd)
+{
+ struct nbd_config *config = nbd->config;
+ u64 bsize = config->blksize;
+ u64 bytes = config->bytesize;
+
+ if (info->attrs[NBD_ATTR_SIZE_BYTES])
+ bytes = nla_get_u64(info->attrs[NBD_ATTR_SIZE_BYTES]);
+
+ if (info->attrs[NBD_ATTR_BLOCK_SIZE_BYTES]) {
+ bsize = nla_get_u64(info->attrs[NBD_ATTR_BLOCK_SIZE_BYTES]);
+ if (!bsize)
+ bsize = NBD_DEF_BLKSIZE;
+ if (!nbd_is_valid_blksize(bsize)) {
+ printk(KERN_ERR "Invalid block size %llu\n", bsize);
+ return -EINVAL;
+ }
+ }
+
+ if (bytes != config->bytesize || bsize != config->blksize)
+ nbd_size_set(nbd, bsize, div64_u64(bytes, bsize));
+ return 0;
+}
+
static int nbd_genl_connect(struct sk_buff *skb, struct genl_info *info)
{
struct nbd_device *nbd = NULL;
@@ -1760,21 +1835,13 @@ again:
refcount_set(&nbd->config_refs, 1);
set_bit(NBD_BOUND, &config->runtime_flags);
- if (info->attrs[NBD_ATTR_SIZE_BYTES]) {
- u64 bytes = nla_get_u64(info->attrs[NBD_ATTR_SIZE_BYTES]);
- nbd_size_set(nbd, config->blksize,
- div64_u64(bytes, config->blksize));
- }
- if (info->attrs[NBD_ATTR_BLOCK_SIZE_BYTES]) {
- u64 bsize =
- nla_get_u64(info->attrs[NBD_ATTR_BLOCK_SIZE_BYTES]);
- nbd_size_set(nbd, bsize, div64_u64(config->bytesize, bsize));
- }
- if (info->attrs[NBD_ATTR_TIMEOUT]) {
- u64 timeout = nla_get_u64(info->attrs[NBD_ATTR_TIMEOUT]);
- nbd->tag_set.timeout = timeout * HZ;
- blk_queue_rq_timeout(nbd->disk->queue, timeout * HZ);
- }
+ ret = nbd_genl_size_set(info, nbd);
+ if (ret)
+ goto out;
+
+ if (info->attrs[NBD_ATTR_TIMEOUT])
+ nbd_set_cmd_timeout(nbd,
+ nla_get_u64(info->attrs[NBD_ATTR_TIMEOUT]));
if (info->attrs[NBD_ATTR_DEAD_CONN_TIMEOUT]) {
config->dead_conn_timeout =
nla_get_u64(info->attrs[NBD_ATTR_DEAD_CONN_TIMEOUT]);
@@ -1846,6 +1913,12 @@ static void nbd_disconnect_and_put(struct nbd_device *nbd)
nbd_disconnect(nbd);
nbd_clear_sock(nbd);
mutex_unlock(&nbd->config_lock);
+ /*
+ * Make sure recv thread has finished, so it does not drop the last
+ * config ref and try to destroy the workqueue from inside the work
+ * queue.
+ */
+ flush_workqueue(nbd->recv_workq);
if (test_and_clear_bit(NBD_HAS_CONFIG_REF,
&nbd->config->runtime_flags))
nbd_config_put(nbd);
@@ -1938,11 +2011,13 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info)
goto out;
}
- if (info->attrs[NBD_ATTR_TIMEOUT]) {
- u64 timeout = nla_get_u64(info->attrs[NBD_ATTR_TIMEOUT]);
- nbd->tag_set.timeout = timeout * HZ;
- blk_queue_rq_timeout(nbd->disk->queue, timeout * HZ);
- }
+ ret = nbd_genl_size_set(info, nbd);
+ if (ret)
+ goto out;
+
+ if (info->attrs[NBD_ATTR_TIMEOUT])
+ nbd_set_cmd_timeout(nbd,
+ nla_get_u64(info->attrs[NBD_ATTR_TIMEOUT]));
if (info->attrs[NBD_ATTR_DEAD_CONN_TIMEOUT]) {
config->dead_conn_timeout =
nla_get_u64(info->attrs[NBD_ATTR_DEAD_CONN_TIMEOUT]);
@@ -2228,20 +2303,12 @@ static int __init nbd_init(void)
if (nbds_max > 1UL << (MINORBITS - part_shift))
return -EINVAL;
- recv_workqueue = alloc_workqueue("knbd-recv",
- WQ_MEM_RECLAIM | WQ_HIGHPRI |
- WQ_UNBOUND, 0);
- if (!recv_workqueue)
- return -ENOMEM;
- if (register_blkdev(NBD_MAJOR, "nbd")) {
- destroy_workqueue(recv_workqueue);
+ if (register_blkdev(NBD_MAJOR, "nbd"))
return -EIO;
- }
if (genl_register_family(&nbd_genl_family)) {
unregister_blkdev(NBD_MAJOR, "nbd");
- destroy_workqueue(recv_workqueue);
return -EINVAL;
}
nbd_dbg_init();
@@ -2283,7 +2350,6 @@ static void __exit nbd_cleanup(void)
idr_destroy(&nbd_index_idr);
genl_unregister_family(&nbd_genl_family);
- destroy_workqueue(recv_workqueue);
unregister_blkdev(NBD_MAJOR, "nbd");
}
diff --git a/drivers/block/null_blk.h b/drivers/block/null_blk.h
index 34b22d6523ba..a235c45e22a7 100644
--- a/drivers/block/null_blk.h
+++ b/drivers/block/null_blk.h
@@ -2,6 +2,9 @@
#ifndef __BLK_NULL_BLK_H
#define __BLK_NULL_BLK_H
+#undef pr_fmt
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
#include <linux/blkdev.h>
#include <linux/slab.h>
#include <linux/blk-mq.h>
@@ -89,28 +92,28 @@ struct nullb {
int null_zone_init(struct nullb_device *dev);
void null_zone_exit(struct nullb_device *dev);
int null_zone_report(struct gendisk *disk, sector_t sector,
- struct blk_zone *zones, unsigned int *nr_zones,
- gfp_t gfp_mask);
-void null_zone_write(struct nullb_cmd *cmd, sector_t sector,
- unsigned int nr_sectors);
-void null_zone_reset(struct nullb_cmd *cmd, sector_t sector);
+ struct blk_zone *zones, unsigned int *nr_zones);
+blk_status_t null_handle_zoned(struct nullb_cmd *cmd,
+ enum req_opf op, sector_t sector,
+ sector_t nr_sectors);
#else
static inline int null_zone_init(struct nullb_device *dev)
{
- pr_err("null_blk: CONFIG_BLK_DEV_ZONED not enabled\n");
+ pr_err("CONFIG_BLK_DEV_ZONED not enabled\n");
return -EINVAL;
}
static inline void null_zone_exit(struct nullb_device *dev) {}
static inline int null_zone_report(struct gendisk *disk, sector_t sector,
struct blk_zone *zones,
- unsigned int *nr_zones, gfp_t gfp_mask)
+ unsigned int *nr_zones)
{
return -EOPNOTSUPP;
}
-static inline void null_zone_write(struct nullb_cmd *cmd, sector_t sector,
- unsigned int nr_sectors)
+static inline blk_status_t null_handle_zoned(struct nullb_cmd *cmd,
+ enum req_opf op, sector_t sector,
+ sector_t nr_sectors)
{
+ return BLK_STS_NOTSUPP;
}
-static inline void null_zone_reset(struct nullb_cmd *cmd, sector_t sector) {}
#endif /* CONFIG_BLK_DEV_ZONED */
#endif /* __NULL_BLK_H */
diff --git a/drivers/block/null_blk_main.c b/drivers/block/null_blk_main.c
index 447d635c79a2..0e7da5015ccd 100644
--- a/drivers/block/null_blk_main.c
+++ b/drivers/block/null_blk_main.c
@@ -141,8 +141,8 @@ static int g_bs = 512;
module_param_named(bs, g_bs, int, 0444);
MODULE_PARM_DESC(bs, "Block size (in bytes)");
-static int nr_devices = 1;
-module_param(nr_devices, int, 0444);
+static unsigned int nr_devices = 1;
+module_param(nr_devices, uint, 0444);
MODULE_PARM_DESC(nr_devices, "Number of devices to register");
static bool g_blocking;
@@ -327,11 +327,12 @@ static ssize_t nullb_device_power_store(struct config_item *item,
set_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
dev->power = newp;
} else if (dev->power && !newp) {
- mutex_lock(&lock);
- dev->power = newp;
- null_del_dev(dev->nullb);
- mutex_unlock(&lock);
- clear_bit(NULLB_DEV_FL_UP, &dev->flags);
+ if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
+ mutex_lock(&lock);
+ dev->power = newp;
+ null_del_dev(dev->nullb);
+ mutex_unlock(&lock);
+ }
clear_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
}
@@ -1132,93 +1133,61 @@ static void null_restart_queue_async(struct nullb *nullb)
blk_mq_start_stopped_hw_queues(q, true);
}
-static blk_status_t null_handle_cmd(struct nullb_cmd *cmd)
+static inline blk_status_t null_handle_throttled(struct nullb_cmd *cmd)
{
struct nullb_device *dev = cmd->nq->dev;
struct nullb *nullb = dev->nullb;
- int err = 0;
+ blk_status_t sts = BLK_STS_OK;
+ struct request *rq = cmd->rq;
- if (test_bit(NULLB_DEV_FL_THROTTLED, &dev->flags)) {
- struct request *rq = cmd->rq;
-
- if (!hrtimer_active(&nullb->bw_timer))
- hrtimer_restart(&nullb->bw_timer);
-
- if (atomic_long_sub_return(blk_rq_bytes(rq),
- &nullb->cur_bytes) < 0) {
- null_stop_queue(nullb);
- /* race with timer */
- if (atomic_long_read(&nullb->cur_bytes) > 0)
- null_restart_queue_async(nullb);
- /* requeue request */
- return BLK_STS_DEV_RESOURCE;
- }
- }
+ if (!hrtimer_active(&nullb->bw_timer))
+ hrtimer_restart(&nullb->bw_timer);
- if (nullb->dev->badblocks.shift != -1) {
- int bad_sectors;
- sector_t sector, size, first_bad;
- bool is_flush = true;
-
- if (dev->queue_mode == NULL_Q_BIO &&
- bio_op(cmd->bio) != REQ_OP_FLUSH) {
- is_flush = false;
- sector = cmd->bio->bi_iter.bi_sector;
- size = bio_sectors(cmd->bio);
- }
- if (dev->queue_mode != NULL_Q_BIO &&
- req_op(cmd->rq) != REQ_OP_FLUSH) {
- is_flush = false;
- sector = blk_rq_pos(cmd->rq);
- size = blk_rq_sectors(cmd->rq);
- }
- if (!is_flush && badblocks_check(&nullb->dev->badblocks, sector,
- size, &first_bad, &bad_sectors)) {
- cmd->error = BLK_STS_IOERR;
- goto out;
- }
+ if (atomic_long_sub_return(blk_rq_bytes(rq), &nullb->cur_bytes) < 0) {
+ null_stop_queue(nullb);
+ /* race with timer */
+ if (atomic_long_read(&nullb->cur_bytes) > 0)
+ null_restart_queue_async(nullb);
+ /* requeue request */
+ sts = BLK_STS_DEV_RESOURCE;
}
+ return sts;
+}
- if (dev->memory_backed) {
- if (dev->queue_mode == NULL_Q_BIO) {
- if (bio_op(cmd->bio) == REQ_OP_FLUSH)
- err = null_handle_flush(nullb);
- else
- err = null_handle_bio(cmd);
- } else {
- if (req_op(cmd->rq) == REQ_OP_FLUSH)
- err = null_handle_flush(nullb);
- else
- err = null_handle_rq(cmd);
- }
- }
- cmd->error = errno_to_blk_status(err);
+static inline blk_status_t null_handle_badblocks(struct nullb_cmd *cmd,
+ sector_t sector,
+ sector_t nr_sectors)
+{
+ struct badblocks *bb = &cmd->nq->dev->badblocks;
+ sector_t first_bad;
+ int bad_sectors;
- if (!cmd->error && dev->zoned) {
- sector_t sector;
- unsigned int nr_sectors;
- int op;
+ if (badblocks_check(bb, sector, nr_sectors, &first_bad, &bad_sectors))
+ return BLK_STS_IOERR;
- if (dev->queue_mode == NULL_Q_BIO) {
- op = bio_op(cmd->bio);
- sector = cmd->bio->bi_iter.bi_sector;
- nr_sectors = cmd->bio->bi_iter.bi_size >> 9;
- } else {
- op = req_op(cmd->rq);
- sector = blk_rq_pos(cmd->rq);
- nr_sectors = blk_rq_sectors(cmd->rq);
- }
+ return BLK_STS_OK;
+}
- if (op == REQ_OP_WRITE)
- null_zone_write(cmd, sector, nr_sectors);
- else if (op == REQ_OP_ZONE_RESET)
- null_zone_reset(cmd, sector);
- }
-out:
+static inline blk_status_t null_handle_memory_backed(struct nullb_cmd *cmd,
+ enum req_opf op)
+{
+ struct nullb_device *dev = cmd->nq->dev;
+ int err;
+
+ if (dev->queue_mode == NULL_Q_BIO)
+ err = null_handle_bio(cmd);
+ else
+ err = null_handle_rq(cmd);
+
+ return errno_to_blk_status(err);
+}
+
+static inline void nullb_complete_cmd(struct nullb_cmd *cmd)
+{
/* Complete IO by inline, softirq or timer */
- switch (dev->irqmode) {
+ switch (cmd->nq->dev->irqmode) {
case NULL_IRQ_SOFTIRQ:
- switch (dev->queue_mode) {
+ switch (cmd->nq->dev->queue_mode) {
case NULL_Q_MQ:
blk_mq_complete_request(cmd->rq);
break;
@@ -1237,6 +1206,40 @@ out:
null_cmd_end_timer(cmd);
break;
}
+}
+
+static blk_status_t null_handle_cmd(struct nullb_cmd *cmd, sector_t sector,
+ sector_t nr_sectors, enum req_opf op)
+{
+ struct nullb_device *dev = cmd->nq->dev;
+ struct nullb *nullb = dev->nullb;
+ blk_status_t sts;
+
+ if (test_bit(NULLB_DEV_FL_THROTTLED, &dev->flags)) {
+ sts = null_handle_throttled(cmd);
+ if (sts != BLK_STS_OK)
+ return sts;
+ }
+
+ if (op == REQ_OP_FLUSH) {
+ cmd->error = errno_to_blk_status(null_handle_flush(nullb));
+ goto out;
+ }
+
+ if (nullb->dev->badblocks.shift != -1) {
+ cmd->error = null_handle_badblocks(cmd, sector, nr_sectors);
+ if (cmd->error != BLK_STS_OK)
+ goto out;
+ }
+
+ if (dev->memory_backed)
+ cmd->error = null_handle_memory_backed(cmd, op);
+
+ if (!cmd->error && dev->zoned)
+ cmd->error = null_handle_zoned(cmd, op, sector, nr_sectors);
+
+out:
+ nullb_complete_cmd(cmd);
return BLK_STS_OK;
}
@@ -1279,6 +1282,8 @@ static struct nullb_queue *nullb_to_queue(struct nullb *nullb)
static blk_qc_t null_queue_bio(struct request_queue *q, struct bio *bio)
{
+ sector_t sector = bio->bi_iter.bi_sector;
+ sector_t nr_sectors = bio_sectors(bio);
struct nullb *nullb = q->queuedata;
struct nullb_queue *nq = nullb_to_queue(nullb);
struct nullb_cmd *cmd;
@@ -1286,7 +1291,7 @@ static blk_qc_t null_queue_bio(struct request_queue *q, struct bio *bio)
cmd = alloc_cmd(nq, 1);
cmd->bio = bio;
- null_handle_cmd(cmd);
+ null_handle_cmd(cmd, sector, nr_sectors, bio_op(bio));
return BLK_QC_T_NONE;
}
@@ -1310,7 +1315,7 @@ static bool should_requeue_request(struct request *rq)
static enum blk_eh_timer_return null_timeout_rq(struct request *rq, bool res)
{
- pr_info("null: rq %p timed out\n", rq);
+ pr_info("rq %p timed out\n", rq);
blk_mq_complete_request(rq);
return BLK_EH_DONE;
}
@@ -1320,6 +1325,8 @@ static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
{
struct nullb_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
struct nullb_queue *nq = hctx->driver_data;
+ sector_t nr_sectors = blk_rq_sectors(bd->rq);
+ sector_t sector = blk_rq_pos(bd->rq);
might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
@@ -1348,7 +1355,7 @@ static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
if (should_timeout_request(bd->rq))
return BLK_STS_OK;
- return null_handle_cmd(cmd);
+ return null_handle_cmd(cmd, sector, nr_sectors, req_op(bd->rq));
}
static const struct blk_mq_ops null_mq_ops = {
@@ -1488,7 +1495,6 @@ static int setup_queues(struct nullb *nullb)
if (!nullb->queues)
return -ENOMEM;
- nullb->nr_queues = 0;
nullb->queue_depth = nullb->dev->hw_queue_depth;
return 0;
@@ -1688,6 +1694,9 @@ static int null_add_dev(struct nullb_device *dev)
blk_queue_chunk_sectors(nullb->q, dev->zone_size_sects);
nullb->q->limits.zoned = BLK_ZONED_HM;
+ blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, nullb->q);
+ blk_queue_required_elevator_features(nullb->q,
+ ELEVATOR_F_ZBD_SEQ_WRITE);
}
nullb->q->queuedata = nullb;
@@ -1739,28 +1748,28 @@ static int __init null_init(void)
struct nullb_device *dev;
if (g_bs > PAGE_SIZE) {
- pr_warn("null_blk: invalid block size\n");
- pr_warn("null_blk: defaults block size to %lu\n", PAGE_SIZE);
+ pr_warn("invalid block size\n");
+ pr_warn("defaults block size to %lu\n", PAGE_SIZE);
g_bs = PAGE_SIZE;
}
if (!is_power_of_2(g_zone_size)) {
- pr_err("null_blk: zone_size must be power-of-two\n");
+ pr_err("zone_size must be power-of-two\n");
return -EINVAL;
}
if (g_home_node != NUMA_NO_NODE && g_home_node >= nr_online_nodes) {
- pr_err("null_blk: invalid home_node value\n");
+ pr_err("invalid home_node value\n");
g_home_node = NUMA_NO_NODE;
}
if (g_queue_mode == NULL_Q_RQ) {
- pr_err("null_blk: legacy IO path no longer available\n");
+ pr_err("legacy IO path no longer available\n");
return -EINVAL;
}
if (g_queue_mode == NULL_Q_MQ && g_use_per_node_hctx) {
if (g_submit_queues != nr_online_nodes) {
- pr_warn("null_blk: submit_queues param is set to %u.\n",
+ pr_warn("submit_queues param is set to %u.\n",
nr_online_nodes);
g_submit_queues = nr_online_nodes;
}
@@ -1803,7 +1812,7 @@ static int __init null_init(void)
}
}
- pr_info("null: module loaded\n");
+ pr_info("module loaded\n");
return 0;
err_dev:
diff --git a/drivers/block/null_blk_zoned.c b/drivers/block/null_blk_zoned.c
index fca0c97ff1aa..eabc116832a7 100644
--- a/drivers/block/null_blk_zoned.c
+++ b/drivers/block/null_blk_zoned.c
@@ -17,7 +17,7 @@ int null_zone_init(struct nullb_device *dev)
unsigned int i;
if (!is_power_of_2(dev->zone_size)) {
- pr_err("null_blk: zone_size must be power-of-two\n");
+ pr_err("zone_size must be power-of-two\n");
return -EINVAL;
}
@@ -31,7 +31,7 @@ int null_zone_init(struct nullb_device *dev)
if (dev->zone_nr_conv >= dev->nr_zones) {
dev->zone_nr_conv = dev->nr_zones - 1;
- pr_info("null_blk: changed the number of conventional zones to %u",
+ pr_info("changed the number of conventional zones to %u",
dev->zone_nr_conv);
}
@@ -67,8 +67,7 @@ void null_zone_exit(struct nullb_device *dev)
}
int null_zone_report(struct gendisk *disk, sector_t sector,
- struct blk_zone *zones, unsigned int *nr_zones,
- gfp_t gfp_mask)
+ struct blk_zone *zones, unsigned int *nr_zones)
{
struct nullb *nullb = disk->private_data;
struct nullb_device *dev = nullb->dev;
@@ -85,7 +84,7 @@ int null_zone_report(struct gendisk *disk, sector_t sector,
return 0;
}
-void null_zone_write(struct nullb_cmd *cmd, sector_t sector,
+static blk_status_t null_zone_write(struct nullb_cmd *cmd, sector_t sector,
unsigned int nr_sectors)
{
struct nullb_device *dev = cmd->nq->dev;
@@ -96,14 +95,12 @@ void null_zone_write(struct nullb_cmd *cmd, sector_t sector,
case BLK_ZONE_COND_FULL:
/* Cannot write to a full zone */
cmd->error = BLK_STS_IOERR;
- break;
+ return BLK_STS_IOERR;
case BLK_ZONE_COND_EMPTY:
case BLK_ZONE_COND_IMP_OPEN:
/* Writes must be at the write pointer position */
- if (sector != zone->wp) {
- cmd->error = BLK_STS_IOERR;
- break;
- }
+ if (sector != zone->wp)
+ return BLK_STS_IOERR;
if (zone->cond == BLK_ZONE_COND_EMPTY)
zone->cond = BLK_ZONE_COND_IMP_OPEN;
@@ -116,22 +113,51 @@ void null_zone_write(struct nullb_cmd *cmd, sector_t sector,
break;
default:
/* Invalid zone condition */
- cmd->error = BLK_STS_IOERR;
- break;
+ return BLK_STS_IOERR;
}
+ return BLK_STS_OK;
}
-void null_zone_reset(struct nullb_cmd *cmd, sector_t sector)
+static blk_status_t null_zone_reset(struct nullb_cmd *cmd, sector_t sector)
{
struct nullb_device *dev = cmd->nq->dev;
unsigned int zno = null_zone_no(dev, sector);
struct blk_zone *zone = &dev->zones[zno];
+ size_t i;
+
+ switch (req_op(cmd->rq)) {
+ case REQ_OP_ZONE_RESET_ALL:
+ for (i = 0; i < dev->nr_zones; i++) {
+ if (zone[i].type == BLK_ZONE_TYPE_CONVENTIONAL)
+ continue;
+ zone[i].cond = BLK_ZONE_COND_EMPTY;
+ zone[i].wp = zone[i].start;
+ }
+ break;
+ case REQ_OP_ZONE_RESET:
+ if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL)
+ return BLK_STS_IOERR;
- if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL) {
- cmd->error = BLK_STS_IOERR;
- return;
+ zone->cond = BLK_ZONE_COND_EMPTY;
+ zone->wp = zone->start;
+ break;
+ default:
+ cmd->error = BLK_STS_NOTSUPP;
+ break;
}
+ return BLK_STS_OK;
+}
- zone->cond = BLK_ZONE_COND_EMPTY;
- zone->wp = zone->start;
+blk_status_t null_handle_zoned(struct nullb_cmd *cmd, enum req_opf op,
+ sector_t sector, sector_t nr_sectors)
+{
+ switch (op) {
+ case REQ_OP_WRITE:
+ return null_zone_write(cmd, sector, nr_sectors);
+ case REQ_OP_ZONE_RESET:
+ case REQ_OP_ZONE_RESET_ALL:
+ return null_zone_reset(cmd, sector);
+ default:
+ return BLK_STS_OK;
+ }
}
diff --git a/drivers/block/paride/pcd.c b/drivers/block/paride/pcd.c
index 001dbdcbf355..636bfea2de6f 100644
--- a/drivers/block/paride/pcd.c
+++ b/drivers/block/paride/pcd.c
@@ -314,8 +314,8 @@ static void pcd_init_units(void)
disk->queue = blk_mq_init_sq_queue(&cd->tag_set, &pcd_mq_ops,
1, BLK_MQ_F_SHOULD_MERGE);
if (IS_ERR(disk->queue)) {
- put_disk(disk);
disk->queue = NULL;
+ put_disk(disk);
continue;
}
@@ -723,9 +723,9 @@ static int pcd_detect(void)
k = 0;
if (pcd_drive_count == 0) { /* nothing spec'd - so autoprobe for 1 */
cd = pcd;
- if (pi_init(cd->pi, 1, -1, -1, -1, -1, -1, pcd_buffer,
- PI_PCD, verbose, cd->name)) {
- if (!pcd_probe(cd, -1, id) && cd->disk) {
+ if (cd->disk && pi_init(cd->pi, 1, -1, -1, -1, -1, -1,
+ pcd_buffer, PI_PCD, verbose, cd->name)) {
+ if (!pcd_probe(cd, -1, id)) {
cd->present = 1;
k++;
} else
@@ -736,11 +736,13 @@ static int pcd_detect(void)
int *conf = *drives[unit];
if (!conf[D_PRT])
continue;
+ if (!cd->disk)
+ continue;
if (!pi_init(cd->pi, 0, conf[D_PRT], conf[D_MOD],
conf[D_UNI], conf[D_PRO], conf[D_DLY],
pcd_buffer, PI_PCD, verbose, cd->name))
continue;
- if (!pcd_probe(cd, conf[D_SLV], id) && cd->disk) {
+ if (!pcd_probe(cd, conf[D_SLV], id)) {
cd->present = 1;
k++;
} else
diff --git a/drivers/block/paride/pf.c b/drivers/block/paride/pf.c
index 1e9c50a7256c..6b7d4cab3687 100644
--- a/drivers/block/paride/pf.c
+++ b/drivers/block/paride/pf.c
@@ -300,8 +300,8 @@ static void __init pf_init_units(void)
disk->queue = blk_mq_init_sq_queue(&pf->tag_set, &pf_mq_ops,
1, BLK_MQ_F_SHOULD_MERGE);
if (IS_ERR(disk->queue)) {
- put_disk(disk);
disk->queue = NULL;
+ put_disk(disk);
continue;
}
diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
index e5009a34f9c2..c8fb886aebd4 100644
--- a/drivers/block/rbd.c
+++ b/drivers/block/rbd.c
@@ -115,6 +115,8 @@ static int atomic_dec_return_safe(atomic_t *v)
#define RBD_FEATURE_LAYERING (1ULL<<0)
#define RBD_FEATURE_STRIPINGV2 (1ULL<<1)
#define RBD_FEATURE_EXCLUSIVE_LOCK (1ULL<<2)
+#define RBD_FEATURE_OBJECT_MAP (1ULL<<3)
+#define RBD_FEATURE_FAST_DIFF (1ULL<<4)
#define RBD_FEATURE_DEEP_FLATTEN (1ULL<<5)
#define RBD_FEATURE_DATA_POOL (1ULL<<7)
#define RBD_FEATURE_OPERATIONS (1ULL<<8)
@@ -122,6 +124,8 @@ static int atomic_dec_return_safe(atomic_t *v)
#define RBD_FEATURES_ALL (RBD_FEATURE_LAYERING | \
RBD_FEATURE_STRIPINGV2 | \
RBD_FEATURE_EXCLUSIVE_LOCK | \
+ RBD_FEATURE_OBJECT_MAP | \
+ RBD_FEATURE_FAST_DIFF | \
RBD_FEATURE_DEEP_FLATTEN | \
RBD_FEATURE_DATA_POOL | \
RBD_FEATURE_OPERATIONS)
@@ -203,6 +207,11 @@ struct rbd_client {
struct list_head node;
};
+struct pending_result {
+ int result; /* first nonzero result */
+ int num_pending;
+};
+
struct rbd_img_request;
enum obj_request_type {
@@ -219,6 +228,18 @@ enum obj_operation_type {
OBJ_OP_ZEROOUT,
};
+#define RBD_OBJ_FLAG_DELETION (1U << 0)
+#define RBD_OBJ_FLAG_COPYUP_ENABLED (1U << 1)
+#define RBD_OBJ_FLAG_COPYUP_ZEROS (1U << 2)
+#define RBD_OBJ_FLAG_MAY_EXIST (1U << 3)
+#define RBD_OBJ_FLAG_NOOP_FOR_NONEXISTENT (1U << 4)
+
+enum rbd_obj_read_state {
+ RBD_OBJ_READ_START = 1,
+ RBD_OBJ_READ_OBJECT,
+ RBD_OBJ_READ_PARENT,
+};
+
/*
* Writes go through the following state machine to deal with
* layering:
@@ -245,17 +266,28 @@ enum obj_operation_type {
* even if there is a parent).
*/
enum rbd_obj_write_state {
- RBD_OBJ_WRITE_FLAT = 1,
- RBD_OBJ_WRITE_GUARD,
- RBD_OBJ_WRITE_READ_FROM_PARENT,
- RBD_OBJ_WRITE_COPYUP_EMPTY_SNAPC,
- RBD_OBJ_WRITE_COPYUP_OPS,
+ RBD_OBJ_WRITE_START = 1,
+ RBD_OBJ_WRITE_PRE_OBJECT_MAP,
+ RBD_OBJ_WRITE_OBJECT,
+ __RBD_OBJ_WRITE_COPYUP,
+ RBD_OBJ_WRITE_COPYUP,
+ RBD_OBJ_WRITE_POST_OBJECT_MAP,
+};
+
+enum rbd_obj_copyup_state {
+ RBD_OBJ_COPYUP_START = 1,
+ RBD_OBJ_COPYUP_READ_PARENT,
+ __RBD_OBJ_COPYUP_OBJECT_MAPS,
+ RBD_OBJ_COPYUP_OBJECT_MAPS,
+ __RBD_OBJ_COPYUP_WRITE_OBJECT,
+ RBD_OBJ_COPYUP_WRITE_OBJECT,
};
struct rbd_obj_request {
struct ceph_object_extent ex;
+ unsigned int flags; /* RBD_OBJ_FLAG_* */
union {
- bool tried_parent; /* for reads */
+ enum rbd_obj_read_state read_state; /* for reads */
enum rbd_obj_write_state write_state; /* for writes */
};
@@ -271,14 +303,15 @@ struct rbd_obj_request {
u32 bvec_idx;
};
};
+
+ enum rbd_obj_copyup_state copyup_state;
struct bio_vec *copyup_bvecs;
u32 copyup_bvec_count;
- struct ceph_osd_request *osd_req;
-
- u64 xferred; /* bytes transferred */
- int result;
+ struct list_head osd_reqs; /* w/ r_private_item */
+ struct mutex state_mutex;
+ struct pending_result pending;
struct kref kref;
};
@@ -287,11 +320,19 @@ enum img_req_flags {
IMG_REQ_LAYERED, /* ENOENT handling: normal = 0, layered = 1 */
};
+enum rbd_img_state {
+ RBD_IMG_START = 1,
+ RBD_IMG_EXCLUSIVE_LOCK,
+ __RBD_IMG_OBJECT_REQUESTS,
+ RBD_IMG_OBJECT_REQUESTS,
+};
+
struct rbd_img_request {
struct rbd_device *rbd_dev;
enum obj_operation_type op_type;
enum obj_request_type data_type;
unsigned long flags;
+ enum rbd_img_state state;
union {
u64 snap_id; /* for reads */
struct ceph_snap_context *snapc; /* for writes */
@@ -300,13 +341,14 @@ struct rbd_img_request {
struct request *rq; /* block request */
struct rbd_obj_request *obj_request; /* obj req initiator */
};
- spinlock_t completion_lock;
- u64 xferred;/* aggregate bytes transferred */
- int result; /* first nonzero obj_request result */
+ struct list_head lock_item;
struct list_head object_extents; /* obj_req.ex structs */
- u32 pending_count;
+ struct mutex state_mutex;
+ struct pending_result pending;
+ struct work_struct work;
+ int work_result;
struct kref kref;
};
@@ -380,7 +422,17 @@ struct rbd_device {
struct work_struct released_lock_work;
struct delayed_work lock_dwork;
struct work_struct unlock_work;
- wait_queue_head_t lock_waitq;
+ spinlock_t lock_lists_lock;
+ struct list_head acquiring_list;
+ struct list_head running_list;
+ struct completion acquire_wait;
+ int acquire_err;
+ struct completion releasing_wait;
+
+ spinlock_t object_map_lock;
+ u8 *object_map;
+ u64 object_map_size; /* in objects */
+ u64 object_map_flags;
struct workqueue_struct *task_wq;
@@ -408,12 +460,10 @@ struct rbd_device {
* Flag bits for rbd_dev->flags:
* - REMOVING (which is coupled with rbd_dev->open_count) is protected
* by rbd_dev->lock
- * - BLACKLISTED is protected by rbd_dev->lock_rwsem
*/
enum rbd_dev_flags {
RBD_DEV_FLAG_EXISTS, /* mapped snapshot has not been deleted */
RBD_DEV_FLAG_REMOVING, /* this mapping is being removed */
- RBD_DEV_FLAG_BLACKLISTED, /* our ceph_client is blacklisted */
};
static DEFINE_MUTEX(client_mutex); /* Serialize client creation */
@@ -466,6 +516,8 @@ static int minor_to_rbd_dev_id(int minor)
static bool __rbd_is_lock_owner(struct rbd_device *rbd_dev)
{
+ lockdep_assert_held(&rbd_dev->lock_rwsem);
+
return rbd_dev->lock_state == RBD_LOCK_STATE_LOCKED ||
rbd_dev->lock_state == RBD_LOCK_STATE_RELEASING;
}
@@ -583,6 +635,26 @@ static int _rbd_dev_v2_snap_size(struct rbd_device *rbd_dev, u64 snap_id,
u8 *order, u64 *snap_size);
static int _rbd_dev_v2_snap_features(struct rbd_device *rbd_dev, u64 snap_id,
u64 *snap_features);
+static int rbd_dev_v2_get_flags(struct rbd_device *rbd_dev);
+
+static void rbd_obj_handle_request(struct rbd_obj_request *obj_req, int result);
+static void rbd_img_handle_request(struct rbd_img_request *img_req, int result);
+
+/*
+ * Return true if nothing else is pending.
+ */
+static bool pending_result_dec(struct pending_result *pending, int *result)
+{
+ rbd_assert(pending->num_pending > 0);
+
+ if (*result && !pending->result)
+ pending->result = *result;
+ if (--pending->num_pending)
+ return false;
+
+ *result = pending->result;
+ return true;
+}
static int rbd_open(struct block_device *bdev, fmode_t mode)
{
@@ -1317,6 +1389,8 @@ static void zero_bvecs(struct ceph_bvec_iter *bvec_pos, u32 off, u32 bytes)
static void rbd_obj_zero_range(struct rbd_obj_request *obj_req, u32 off,
u32 bytes)
{
+ dout("%s %p data buf %u~%u\n", __func__, obj_req, off, bytes);
+
switch (obj_req->img_request->data_type) {
case OBJ_REQUEST_BIO:
zero_bios(&obj_req->bio_pos, off, bytes);
@@ -1339,13 +1413,6 @@ static void rbd_obj_request_put(struct rbd_obj_request *obj_request)
kref_put(&obj_request->kref, rbd_obj_request_destroy);
}
-static void rbd_img_request_get(struct rbd_img_request *img_request)
-{
- dout("%s: img %p (was %d)\n", __func__, img_request,
- kref_read(&img_request->kref));
- kref_get(&img_request->kref);
-}
-
static void rbd_img_request_destroy(struct kref *kref);
static void rbd_img_request_put(struct rbd_img_request *img_request)
{
@@ -1362,7 +1429,6 @@ static inline void rbd_img_obj_request_add(struct rbd_img_request *img_request,
/* Image request now owns object's original reference */
obj_request->img_request = img_request;
- img_request->pending_count++;
dout("%s: img %p obj %p\n", __func__, img_request, obj_request);
}
@@ -1375,13 +1441,13 @@ static inline void rbd_img_obj_request_del(struct rbd_img_request *img_request,
rbd_obj_request_put(obj_request);
}
-static void rbd_obj_request_submit(struct rbd_obj_request *obj_request)
+static void rbd_osd_submit(struct ceph_osd_request *osd_req)
{
- struct ceph_osd_request *osd_req = obj_request->osd_req;
+ struct rbd_obj_request *obj_req = osd_req->r_priv;
- dout("%s %p object_no %016llx %llu~%llu osd_req %p\n", __func__,
- obj_request, obj_request->ex.oe_objno, obj_request->ex.oe_off,
- obj_request->ex.oe_len, osd_req);
+ dout("%s osd_req %p for obj_req %p objno %llu %llu~%llu\n",
+ __func__, osd_req, obj_req, obj_req->ex.oe_objno,
+ obj_req->ex.oe_off, obj_req->ex.oe_len);
ceph_osdc_start_request(osd_req->r_osdc, osd_req, false);
}
@@ -1457,41 +1523,38 @@ static bool rbd_img_is_write(struct rbd_img_request *img_req)
}
}
-static void rbd_obj_handle_request(struct rbd_obj_request *obj_req);
-
static void rbd_osd_req_callback(struct ceph_osd_request *osd_req)
{
struct rbd_obj_request *obj_req = osd_req->r_priv;
+ int result;
dout("%s osd_req %p result %d for obj_req %p\n", __func__, osd_req,
osd_req->r_result, obj_req);
- rbd_assert(osd_req == obj_req->osd_req);
- obj_req->result = osd_req->r_result < 0 ? osd_req->r_result : 0;
- if (!obj_req->result && !rbd_img_is_write(obj_req->img_request))
- obj_req->xferred = osd_req->r_result;
+ /*
+ * Writes aren't allowed to return a data payload. In some
+ * guarded write cases (e.g. stat + zero on an empty object)
+ * a stat response makes it through, but we don't care.
+ */
+ if (osd_req->r_result > 0 && rbd_img_is_write(obj_req->img_request))
+ result = 0;
else
- /*
- * Writes aren't allowed to return a data payload. In some
- * guarded write cases (e.g. stat + zero on an empty object)
- * a stat response makes it through, but we don't care.
- */
- obj_req->xferred = 0;
+ result = osd_req->r_result;
- rbd_obj_handle_request(obj_req);
+ rbd_obj_handle_request(obj_req, result);
}
-static void rbd_osd_req_format_read(struct rbd_obj_request *obj_request)
+static void rbd_osd_format_read(struct ceph_osd_request *osd_req)
{
- struct ceph_osd_request *osd_req = obj_request->osd_req;
+ struct rbd_obj_request *obj_request = osd_req->r_priv;
osd_req->r_flags = CEPH_OSD_FLAG_READ;
osd_req->r_snapid = obj_request->img_request->snap_id;
}
-static void rbd_osd_req_format_write(struct rbd_obj_request *obj_request)
+static void rbd_osd_format_write(struct ceph_osd_request *osd_req)
{
- struct ceph_osd_request *osd_req = obj_request->osd_req;
+ struct rbd_obj_request *obj_request = osd_req->r_priv;
osd_req->r_flags = CEPH_OSD_FLAG_WRITE;
ktime_get_real_ts64(&osd_req->r_mtime);
@@ -1499,19 +1562,21 @@ static void rbd_osd_req_format_write(struct rbd_obj_request *obj_request)
}
static struct ceph_osd_request *
-__rbd_osd_req_create(struct rbd_obj_request *obj_req,
- struct ceph_snap_context *snapc, unsigned int num_ops)
+__rbd_obj_add_osd_request(struct rbd_obj_request *obj_req,
+ struct ceph_snap_context *snapc, int num_ops)
{
struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
struct ceph_osd_request *req;
const char *name_format = rbd_dev->image_format == 1 ?
RBD_V1_DATA_FORMAT : RBD_V2_DATA_FORMAT;
+ int ret;
req = ceph_osdc_alloc_request(osdc, snapc, num_ops, false, GFP_NOIO);
if (!req)
- return NULL;
+ return ERR_PTR(-ENOMEM);
+ list_add_tail(&req->r_private_item, &obj_req->osd_reqs);
req->r_callback = rbd_osd_req_callback;
req->r_priv = obj_req;
@@ -1522,27 +1587,20 @@ __rbd_osd_req_create(struct rbd_obj_request *obj_req,
ceph_oloc_copy(&req->r_base_oloc, &rbd_dev->header_oloc);
req->r_base_oloc.pool = rbd_dev->layout.pool_id;
- if (ceph_oid_aprintf(&req->r_base_oid, GFP_NOIO, name_format,
- rbd_dev->header.object_prefix, obj_req->ex.oe_objno))
- goto err_req;
+ ret = ceph_oid_aprintf(&req->r_base_oid, GFP_NOIO, name_format,
+ rbd_dev->header.object_prefix,
+ obj_req->ex.oe_objno);
+ if (ret)
+ return ERR_PTR(ret);
return req;
-
-err_req:
- ceph_osdc_put_request(req);
- return NULL;
}
static struct ceph_osd_request *
-rbd_osd_req_create(struct rbd_obj_request *obj_req, unsigned int num_ops)
-{
- return __rbd_osd_req_create(obj_req, obj_req->img_request->snapc,
- num_ops);
-}
-
-static void rbd_osd_req_destroy(struct ceph_osd_request *osd_req)
+rbd_obj_add_osd_request(struct rbd_obj_request *obj_req, int num_ops)
{
- ceph_osdc_put_request(osd_req);
+ return __rbd_obj_add_osd_request(obj_req, obj_req->img_request->snapc,
+ num_ops);
}
static struct rbd_obj_request *rbd_obj_request_create(void)
@@ -1554,6 +1612,8 @@ static struct rbd_obj_request *rbd_obj_request_create(void)
return NULL;
ceph_object_extent_init(&obj_request->ex);
+ INIT_LIST_HEAD(&obj_request->osd_reqs);
+ mutex_init(&obj_request->state_mutex);
kref_init(&obj_request->kref);
dout("%s %p\n", __func__, obj_request);
@@ -1563,14 +1623,19 @@ static struct rbd_obj_request *rbd_obj_request_create(void)
static void rbd_obj_request_destroy(struct kref *kref)
{
struct rbd_obj_request *obj_request;
+ struct ceph_osd_request *osd_req;
u32 i;
obj_request = container_of(kref, struct rbd_obj_request, kref);
dout("%s: obj %p\n", __func__, obj_request);
- if (obj_request->osd_req)
- rbd_osd_req_destroy(obj_request->osd_req);
+ while (!list_empty(&obj_request->osd_reqs)) {
+ osd_req = list_first_entry(&obj_request->osd_reqs,
+ struct ceph_osd_request, r_private_item);
+ list_del_init(&osd_req->r_private_item);
+ ceph_osdc_put_request(osd_req);
+ }
switch (obj_request->img_request->data_type) {
case OBJ_REQUEST_NODATA:
@@ -1684,8 +1749,9 @@ static struct rbd_img_request *rbd_img_request_create(
if (rbd_dev_parent_get(rbd_dev))
img_request_layered_set(img_request);
- spin_lock_init(&img_request->completion_lock);
+ INIT_LIST_HEAD(&img_request->lock_item);
INIT_LIST_HEAD(&img_request->object_extents);
+ mutex_init(&img_request->state_mutex);
kref_init(&img_request->kref);
dout("%s: rbd_dev %p %s -> img %p\n", __func__, rbd_dev,
@@ -1703,6 +1769,7 @@ static void rbd_img_request_destroy(struct kref *kref)
dout("%s: img %p\n", __func__, img_request);
+ WARN_ON(!list_empty(&img_request->lock_item));
for_each_obj_request_safe(img_request, obj_request, next_obj_request)
rbd_img_obj_request_del(img_request, obj_request);
@@ -1717,6 +1784,466 @@ static void rbd_img_request_destroy(struct kref *kref)
kmem_cache_free(rbd_img_request_cache, img_request);
}
+#define BITS_PER_OBJ 2
+#define OBJS_PER_BYTE (BITS_PER_BYTE / BITS_PER_OBJ)
+#define OBJ_MASK ((1 << BITS_PER_OBJ) - 1)
+
+static void __rbd_object_map_index(struct rbd_device *rbd_dev, u64 objno,
+ u64 *index, u8 *shift)
+{
+ u32 off;
+
+ rbd_assert(objno < rbd_dev->object_map_size);
+ *index = div_u64_rem(objno, OBJS_PER_BYTE, &off);
+ *shift = (OBJS_PER_BYTE - off - 1) * BITS_PER_OBJ;
+}
+
+static u8 __rbd_object_map_get(struct rbd_device *rbd_dev, u64 objno)
+{
+ u64 index;
+ u8 shift;
+
+ lockdep_assert_held(&rbd_dev->object_map_lock);
+ __rbd_object_map_index(rbd_dev, objno, &index, &shift);
+ return (rbd_dev->object_map[index] >> shift) & OBJ_MASK;
+}
+
+static void __rbd_object_map_set(struct rbd_device *rbd_dev, u64 objno, u8 val)
+{
+ u64 index;
+ u8 shift;
+ u8 *p;
+
+ lockdep_assert_held(&rbd_dev->object_map_lock);
+ rbd_assert(!(val & ~OBJ_MASK));
+
+ __rbd_object_map_index(rbd_dev, objno, &index, &shift);
+ p = &rbd_dev->object_map[index];
+ *p = (*p & ~(OBJ_MASK << shift)) | (val << shift);
+}
+
+static u8 rbd_object_map_get(struct rbd_device *rbd_dev, u64 objno)
+{
+ u8 state;
+
+ spin_lock(&rbd_dev->object_map_lock);
+ state = __rbd_object_map_get(rbd_dev, objno);
+ spin_unlock(&rbd_dev->object_map_lock);
+ return state;
+}
+
+static bool use_object_map(struct rbd_device *rbd_dev)
+{
+ return ((rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP) &&
+ !(rbd_dev->object_map_flags & RBD_FLAG_OBJECT_MAP_INVALID));
+}
+
+static bool rbd_object_map_may_exist(struct rbd_device *rbd_dev, u64 objno)
+{
+ u8 state;
+
+ /* fall back to default logic if object map is disabled or invalid */
+ if (!use_object_map(rbd_dev))
+ return true;
+
+ state = rbd_object_map_get(rbd_dev, objno);
+ return state != OBJECT_NONEXISTENT;
+}
+
+static void rbd_object_map_name(struct rbd_device *rbd_dev, u64 snap_id,
+ struct ceph_object_id *oid)
+{
+ if (snap_id == CEPH_NOSNAP)
+ ceph_oid_printf(oid, "%s%s", RBD_OBJECT_MAP_PREFIX,
+ rbd_dev->spec->image_id);
+ else
+ ceph_oid_printf(oid, "%s%s.%016llx", RBD_OBJECT_MAP_PREFIX,
+ rbd_dev->spec->image_id, snap_id);
+}
+
+static int rbd_object_map_lock(struct rbd_device *rbd_dev)
+{
+ struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
+ CEPH_DEFINE_OID_ONSTACK(oid);
+ u8 lock_type;
+ char *lock_tag;
+ struct ceph_locker *lockers;
+ u32 num_lockers;
+ bool broke_lock = false;
+ int ret;
+
+ rbd_object_map_name(rbd_dev, CEPH_NOSNAP, &oid);
+
+again:
+ ret = ceph_cls_lock(osdc, &oid, &rbd_dev->header_oloc, RBD_LOCK_NAME,
+ CEPH_CLS_LOCK_EXCLUSIVE, "", "", "", 0);
+ if (ret != -EBUSY || broke_lock) {
+ if (ret == -EEXIST)
+ ret = 0; /* already locked by myself */
+ if (ret)
+ rbd_warn(rbd_dev, "failed to lock object map: %d", ret);
+ return ret;
+ }
+
+ ret = ceph_cls_lock_info(osdc, &oid, &rbd_dev->header_oloc,
+ RBD_LOCK_NAME, &lock_type, &lock_tag,
+ &lockers, &num_lockers);
+ if (ret) {
+ if (ret == -ENOENT)
+ goto again;
+
+ rbd_warn(rbd_dev, "failed to get object map lockers: %d", ret);
+ return ret;
+ }
+
+ kfree(lock_tag);
+ if (num_lockers == 0)
+ goto again;
+
+ rbd_warn(rbd_dev, "breaking object map lock owned by %s%llu",
+ ENTITY_NAME(lockers[0].id.name));
+
+ ret = ceph_cls_break_lock(osdc, &oid, &rbd_dev->header_oloc,
+ RBD_LOCK_NAME, lockers[0].id.cookie,
+ &lockers[0].id.name);
+ ceph_free_lockers(lockers, num_lockers);
+ if (ret) {
+ if (ret == -ENOENT)
+ goto again;
+
+ rbd_warn(rbd_dev, "failed to break object map lock: %d", ret);
+ return ret;
+ }
+
+ broke_lock = true;
+ goto again;
+}
+
+static void rbd_object_map_unlock(struct rbd_device *rbd_dev)
+{
+ struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
+ CEPH_DEFINE_OID_ONSTACK(oid);
+ int ret;
+
+ rbd_object_map_name(rbd_dev, CEPH_NOSNAP, &oid);
+
+ ret = ceph_cls_unlock(osdc, &oid, &rbd_dev->header_oloc, RBD_LOCK_NAME,
+ "");
+ if (ret && ret != -ENOENT)
+ rbd_warn(rbd_dev, "failed to unlock object map: %d", ret);
+}
+
+static int decode_object_map_header(void **p, void *end, u64 *object_map_size)
+{
+ u8 struct_v;
+ u32 struct_len;
+ u32 header_len;
+ void *header_end;
+ int ret;
+
+ ceph_decode_32_safe(p, end, header_len, e_inval);
+ header_end = *p + header_len;
+
+ ret = ceph_start_decoding(p, end, 1, "BitVector header", &struct_v,
+ &struct_len);
+ if (ret)
+ return ret;
+
+ ceph_decode_64_safe(p, end, *object_map_size, e_inval);
+
+ *p = header_end;
+ return 0;
+
+e_inval:
+ return -EINVAL;
+}
+
+static int __rbd_object_map_load(struct rbd_device *rbd_dev)
+{
+ struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
+ CEPH_DEFINE_OID_ONSTACK(oid);
+ struct page **pages;
+ void *p, *end;
+ size_t reply_len;
+ u64 num_objects;
+ u64 object_map_bytes;
+ u64 object_map_size;
+ int num_pages;
+ int ret;
+
+ rbd_assert(!rbd_dev->object_map && !rbd_dev->object_map_size);
+
+ num_objects = ceph_get_num_objects(&rbd_dev->layout,
+ rbd_dev->mapping.size);
+ object_map_bytes = DIV_ROUND_UP_ULL(num_objects * BITS_PER_OBJ,
+ BITS_PER_BYTE);
+ num_pages = calc_pages_for(0, object_map_bytes) + 1;
+ pages = ceph_alloc_page_vector(num_pages, GFP_KERNEL);
+ if (IS_ERR(pages))
+ return PTR_ERR(pages);
+
+ reply_len = num_pages * PAGE_SIZE;
+ rbd_object_map_name(rbd_dev, rbd_dev->spec->snap_id, &oid);
+ ret = ceph_osdc_call(osdc, &oid, &rbd_dev->header_oloc,
+ "rbd", "object_map_load", CEPH_OSD_FLAG_READ,
+ NULL, 0, pages, &reply_len);
+ if (ret)
+ goto out;
+
+ p = page_address(pages[0]);
+ end = p + min(reply_len, (size_t)PAGE_SIZE);
+ ret = decode_object_map_header(&p, end, &object_map_size);
+ if (ret)
+ goto out;
+
+ if (object_map_size != num_objects) {
+ rbd_warn(rbd_dev, "object map size mismatch: %llu vs %llu",
+ object_map_size, num_objects);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ if (offset_in_page(p) + object_map_bytes > reply_len) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ rbd_dev->object_map = kvmalloc(object_map_bytes, GFP_KERNEL);
+ if (!rbd_dev->object_map) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ rbd_dev->object_map_size = object_map_size;
+ ceph_copy_from_page_vector(pages, rbd_dev->object_map,
+ offset_in_page(p), object_map_bytes);
+
+out:
+ ceph_release_page_vector(pages, num_pages);
+ return ret;
+}
+
+static void rbd_object_map_free(struct rbd_device *rbd_dev)
+{
+ kvfree(rbd_dev->object_map);
+ rbd_dev->object_map = NULL;
+ rbd_dev->object_map_size = 0;
+}
+
+static int rbd_object_map_load(struct rbd_device *rbd_dev)
+{
+ int ret;
+
+ ret = __rbd_object_map_load(rbd_dev);
+ if (ret)
+ return ret;
+
+ ret = rbd_dev_v2_get_flags(rbd_dev);
+ if (ret) {
+ rbd_object_map_free(rbd_dev);
+ return ret;
+ }
+
+ if (rbd_dev->object_map_flags & RBD_FLAG_OBJECT_MAP_INVALID)
+ rbd_warn(rbd_dev, "object map is invalid");
+
+ return 0;
+}
+
+static int rbd_object_map_open(struct rbd_device *rbd_dev)
+{
+ int ret;
+
+ ret = rbd_object_map_lock(rbd_dev);
+ if (ret)
+ return ret;
+
+ ret = rbd_object_map_load(rbd_dev);
+ if (ret) {
+ rbd_object_map_unlock(rbd_dev);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void rbd_object_map_close(struct rbd_device *rbd_dev)
+{
+ rbd_object_map_free(rbd_dev);
+ rbd_object_map_unlock(rbd_dev);
+}
+
+/*
+ * This function needs snap_id (or more precisely just something to
+ * distinguish between HEAD and snapshot object maps), new_state and
+ * current_state that were passed to rbd_object_map_update().
+ *
+ * To avoid allocating and stashing a context we piggyback on the OSD
+ * request. A HEAD update has two ops (assert_locked). For new_state
+ * and current_state we decode our own object_map_update op, encoded in
+ * rbd_cls_object_map_update().
+ */
+static int rbd_object_map_update_finish(struct rbd_obj_request *obj_req,
+ struct ceph_osd_request *osd_req)
+{
+ struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
+ struct ceph_osd_data *osd_data;
+ u64 objno;
+ u8 state, new_state, current_state;
+ bool has_current_state;
+ void *p;
+
+ if (osd_req->r_result)
+ return osd_req->r_result;
+
+ /*
+ * Nothing to do for a snapshot object map.
+ */
+ if (osd_req->r_num_ops == 1)
+ return 0;
+
+ /*
+ * Update in-memory HEAD object map.
+ */
+ rbd_assert(osd_req->r_num_ops == 2);
+ osd_data = osd_req_op_data(osd_req, 1, cls, request_data);
+ rbd_assert(osd_data->type == CEPH_OSD_DATA_TYPE_PAGES);
+
+ p = page_address(osd_data->pages[0]);
+ objno = ceph_decode_64(&p);
+ rbd_assert(objno == obj_req->ex.oe_objno);
+ rbd_assert(ceph_decode_64(&p) == objno + 1);
+ new_state = ceph_decode_8(&p);
+ has_current_state = ceph_decode_8(&p);
+ if (has_current_state)
+ current_state = ceph_decode_8(&p);
+
+ spin_lock(&rbd_dev->object_map_lock);
+ state = __rbd_object_map_get(rbd_dev, objno);
+ if (!has_current_state || current_state == state ||
+ (current_state == OBJECT_EXISTS && state == OBJECT_EXISTS_CLEAN))
+ __rbd_object_map_set(rbd_dev, objno, new_state);
+ spin_unlock(&rbd_dev->object_map_lock);
+
+ return 0;
+}
+
+static void rbd_object_map_callback(struct ceph_osd_request *osd_req)
+{
+ struct rbd_obj_request *obj_req = osd_req->r_priv;
+ int result;
+
+ dout("%s osd_req %p result %d for obj_req %p\n", __func__, osd_req,
+ osd_req->r_result, obj_req);
+
+ result = rbd_object_map_update_finish(obj_req, osd_req);
+ rbd_obj_handle_request(obj_req, result);
+}
+
+static bool update_needed(struct rbd_device *rbd_dev, u64 objno, u8 new_state)
+{
+ u8 state = rbd_object_map_get(rbd_dev, objno);
+
+ if (state == new_state ||
+ (new_state == OBJECT_PENDING && state == OBJECT_NONEXISTENT) ||
+ (new_state == OBJECT_NONEXISTENT && state != OBJECT_PENDING))
+ return false;
+
+ return true;
+}
+
+static int rbd_cls_object_map_update(struct ceph_osd_request *req,
+ int which, u64 objno, u8 new_state,
+ const u8 *current_state)
+{
+ struct page **pages;
+ void *p, *start;
+ int ret;
+
+ ret = osd_req_op_cls_init(req, which, "rbd", "object_map_update");
+ if (ret)
+ return ret;
+
+ pages = ceph_alloc_page_vector(1, GFP_NOIO);
+ if (IS_ERR(pages))
+ return PTR_ERR(pages);
+
+ p = start = page_address(pages[0]);
+ ceph_encode_64(&p, objno);
+ ceph_encode_64(&p, objno + 1);
+ ceph_encode_8(&p, new_state);
+ if (current_state) {
+ ceph_encode_8(&p, 1);
+ ceph_encode_8(&p, *current_state);
+ } else {
+ ceph_encode_8(&p, 0);
+ }
+
+ osd_req_op_cls_request_data_pages(req, which, pages, p - start, 0,
+ false, true);
+ return 0;
+}
+
+/*
+ * Return:
+ * 0 - object map update sent
+ * 1 - object map update isn't needed
+ * <0 - error
+ */
+static int rbd_object_map_update(struct rbd_obj_request *obj_req, u64 snap_id,
+ u8 new_state, const u8 *current_state)
+{
+ struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
+ struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
+ struct ceph_osd_request *req;
+ int num_ops = 1;
+ int which = 0;
+ int ret;
+
+ if (snap_id == CEPH_NOSNAP) {
+ if (!update_needed(rbd_dev, obj_req->ex.oe_objno, new_state))
+ return 1;
+
+ num_ops++; /* assert_locked */
+ }
+
+ req = ceph_osdc_alloc_request(osdc, NULL, num_ops, false, GFP_NOIO);
+ if (!req)
+ return -ENOMEM;
+
+ list_add_tail(&req->r_private_item, &obj_req->osd_reqs);
+ req->r_callback = rbd_object_map_callback;
+ req->r_priv = obj_req;
+
+ rbd_object_map_name(rbd_dev, snap_id, &req->r_base_oid);
+ ceph_oloc_copy(&req->r_base_oloc, &rbd_dev->header_oloc);
+ req->r_flags = CEPH_OSD_FLAG_WRITE;
+ ktime_get_real_ts64(&req->r_mtime);
+
+ if (snap_id == CEPH_NOSNAP) {
+ /*
+ * Protect against possible race conditions during lock
+ * ownership transitions.
+ */
+ ret = ceph_cls_assert_locked(req, which++, RBD_LOCK_NAME,
+ CEPH_CLS_LOCK_EXCLUSIVE, "", "");
+ if (ret)
+ return ret;
+ }
+
+ ret = rbd_cls_object_map_update(req, which, obj_req->ex.oe_objno,
+ new_state, current_state);
+ if (ret)
+ return ret;
+
+ ret = ceph_osdc_alloc_messages(req, GFP_NOIO);
+ if (ret)
+ return ret;
+
+ ceph_osdc_start_request(osdc, req, false);
+ return 0;
+}
+
static void prune_extents(struct ceph_file_extent *img_extents,
u32 *num_img_extents, u64 overlap)
{
@@ -1764,11 +2291,13 @@ static int rbd_obj_calc_img_extents(struct rbd_obj_request *obj_req,
return 0;
}
-static void rbd_osd_req_setup_data(struct rbd_obj_request *obj_req, u32 which)
+static void rbd_osd_setup_data(struct ceph_osd_request *osd_req, int which)
{
+ struct rbd_obj_request *obj_req = osd_req->r_priv;
+
switch (obj_req->img_request->data_type) {
case OBJ_REQUEST_BIO:
- osd_req_op_extent_osd_data_bio(obj_req->osd_req, which,
+ osd_req_op_extent_osd_data_bio(osd_req, which,
&obj_req->bio_pos,
obj_req->ex.oe_len);
break;
@@ -1777,7 +2306,7 @@ static void rbd_osd_req_setup_data(struct rbd_obj_request *obj_req, u32 which)
rbd_assert(obj_req->bvec_pos.iter.bi_size ==
obj_req->ex.oe_len);
rbd_assert(obj_req->bvec_idx == obj_req->bvec_count);
- osd_req_op_extent_osd_data_bvec_pos(obj_req->osd_req, which,
+ osd_req_op_extent_osd_data_bvec_pos(osd_req, which,
&obj_req->bvec_pos);
break;
default:
@@ -1785,22 +2314,7 @@ static void rbd_osd_req_setup_data(struct rbd_obj_request *obj_req, u32 which)
}
}
-static int rbd_obj_setup_read(struct rbd_obj_request *obj_req)
-{
- obj_req->osd_req = __rbd_osd_req_create(obj_req, NULL, 1);
- if (!obj_req->osd_req)
- return -ENOMEM;
-
- osd_req_op_extent_init(obj_req->osd_req, 0, CEPH_OSD_OP_READ,
- obj_req->ex.oe_off, obj_req->ex.oe_len, 0, 0);
- rbd_osd_req_setup_data(obj_req, 0);
-
- rbd_osd_req_format_read(obj_req);
- return 0;
-}
-
-static int __rbd_obj_setup_stat(struct rbd_obj_request *obj_req,
- unsigned int which)
+static int rbd_osd_setup_stat(struct ceph_osd_request *osd_req, int which)
{
struct page **pages;
@@ -1816,45 +2330,60 @@ static int __rbd_obj_setup_stat(struct rbd_obj_request *obj_req,
if (IS_ERR(pages))
return PTR_ERR(pages);
- osd_req_op_init(obj_req->osd_req, which, CEPH_OSD_OP_STAT, 0);
- osd_req_op_raw_data_in_pages(obj_req->osd_req, which, pages,
+ osd_req_op_init(osd_req, which, CEPH_OSD_OP_STAT, 0);
+ osd_req_op_raw_data_in_pages(osd_req, which, pages,
8 + sizeof(struct ceph_timespec),
0, false, true);
return 0;
}
-static int count_write_ops(struct rbd_obj_request *obj_req)
+static int rbd_osd_setup_copyup(struct ceph_osd_request *osd_req, int which,
+ u32 bytes)
{
- return 2; /* setallochint + write/writefull */
+ struct rbd_obj_request *obj_req = osd_req->r_priv;
+ int ret;
+
+ ret = osd_req_op_cls_init(osd_req, which, "rbd", "copyup");
+ if (ret)
+ return ret;
+
+ osd_req_op_cls_request_data_bvecs(osd_req, which, obj_req->copyup_bvecs,
+ obj_req->copyup_bvec_count, bytes);
+ return 0;
}
-static void __rbd_obj_setup_write(struct rbd_obj_request *obj_req,
- unsigned int which)
+static int rbd_obj_init_read(struct rbd_obj_request *obj_req)
{
+ obj_req->read_state = RBD_OBJ_READ_START;
+ return 0;
+}
+
+static void __rbd_osd_setup_write_ops(struct ceph_osd_request *osd_req,
+ int which)
+{
+ struct rbd_obj_request *obj_req = osd_req->r_priv;
struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
u16 opcode;
- osd_req_op_alloc_hint_init(obj_req->osd_req, which++,
- rbd_dev->layout.object_size,
- rbd_dev->layout.object_size);
+ if (!use_object_map(rbd_dev) ||
+ !(obj_req->flags & RBD_OBJ_FLAG_MAY_EXIST)) {
+ osd_req_op_alloc_hint_init(osd_req, which++,
+ rbd_dev->layout.object_size,
+ rbd_dev->layout.object_size);
+ }
if (rbd_obj_is_entire(obj_req))
opcode = CEPH_OSD_OP_WRITEFULL;
else
opcode = CEPH_OSD_OP_WRITE;
- osd_req_op_extent_init(obj_req->osd_req, which, opcode,
+ osd_req_op_extent_init(osd_req, which, opcode,
obj_req->ex.oe_off, obj_req->ex.oe_len, 0, 0);
- rbd_osd_req_setup_data(obj_req, which++);
-
- rbd_assert(which == obj_req->osd_req->r_num_ops);
- rbd_osd_req_format_write(obj_req);
+ rbd_osd_setup_data(osd_req, which);
}
-static int rbd_obj_setup_write(struct rbd_obj_request *obj_req)
+static int rbd_obj_init_write(struct rbd_obj_request *obj_req)
{
- unsigned int num_osd_ops, which = 0;
- bool need_guard;
int ret;
/* reverse map the entire object onto the parent */
@@ -1862,24 +2391,10 @@ static int rbd_obj_setup_write(struct rbd_obj_request *obj_req)
if (ret)
return ret;
- need_guard = rbd_obj_copyup_enabled(obj_req);
- num_osd_ops = need_guard + count_write_ops(obj_req);
+ if (rbd_obj_copyup_enabled(obj_req))
+ obj_req->flags |= RBD_OBJ_FLAG_COPYUP_ENABLED;
- obj_req->osd_req = rbd_osd_req_create(obj_req, num_osd_ops);
- if (!obj_req->osd_req)
- return -ENOMEM;
-
- if (need_guard) {
- ret = __rbd_obj_setup_stat(obj_req, which++);
- if (ret)
- return ret;
-
- obj_req->write_state = RBD_OBJ_WRITE_GUARD;
- } else {
- obj_req->write_state = RBD_OBJ_WRITE_FLAT;
- }
-
- __rbd_obj_setup_write(obj_req, which);
+ obj_req->write_state = RBD_OBJ_WRITE_START;
return 0;
}
@@ -1889,11 +2404,26 @@ static u16 truncate_or_zero_opcode(struct rbd_obj_request *obj_req)
CEPH_OSD_OP_ZERO;
}
-static int rbd_obj_setup_discard(struct rbd_obj_request *obj_req)
+static void __rbd_osd_setup_discard_ops(struct ceph_osd_request *osd_req,
+ int which)
+{
+ struct rbd_obj_request *obj_req = osd_req->r_priv;
+
+ if (rbd_obj_is_entire(obj_req) && !obj_req->num_img_extents) {
+ rbd_assert(obj_req->flags & RBD_OBJ_FLAG_DELETION);
+ osd_req_op_init(osd_req, which, CEPH_OSD_OP_DELETE, 0);
+ } else {
+ osd_req_op_extent_init(osd_req, which,
+ truncate_or_zero_opcode(obj_req),
+ obj_req->ex.oe_off, obj_req->ex.oe_len,
+ 0, 0);
+ }
+}
+
+static int rbd_obj_init_discard(struct rbd_obj_request *obj_req)
{
struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
- u64 off = obj_req->ex.oe_off;
- u64 next_off = obj_req->ex.oe_off + obj_req->ex.oe_len;
+ u64 off, next_off;
int ret;
/*
@@ -1906,10 +2436,17 @@ static int rbd_obj_setup_discard(struct rbd_obj_request *obj_req)
*/
if (rbd_dev->opts->alloc_size != rbd_dev->layout.object_size ||
!rbd_obj_is_tail(obj_req)) {
- off = round_up(off, rbd_dev->opts->alloc_size);
- next_off = round_down(next_off, rbd_dev->opts->alloc_size);
+ off = round_up(obj_req->ex.oe_off, rbd_dev->opts->alloc_size);
+ next_off = round_down(obj_req->ex.oe_off + obj_req->ex.oe_len,
+ rbd_dev->opts->alloc_size);
if (off >= next_off)
return 1;
+
+ dout("%s %p %llu~%llu -> %llu~%llu\n", __func__,
+ obj_req, obj_req->ex.oe_off, obj_req->ex.oe_len,
+ off, next_off - off);
+ obj_req->ex.oe_off = off;
+ obj_req->ex.oe_len = next_off - off;
}
/* reverse map the entire object onto the parent */
@@ -1917,52 +2454,29 @@ static int rbd_obj_setup_discard(struct rbd_obj_request *obj_req)
if (ret)
return ret;
- obj_req->osd_req = rbd_osd_req_create(obj_req, 1);
- if (!obj_req->osd_req)
- return -ENOMEM;
+ obj_req->flags |= RBD_OBJ_FLAG_NOOP_FOR_NONEXISTENT;
+ if (rbd_obj_is_entire(obj_req) && !obj_req->num_img_extents)
+ obj_req->flags |= RBD_OBJ_FLAG_DELETION;
- if (rbd_obj_is_entire(obj_req) && !obj_req->num_img_extents) {
- osd_req_op_init(obj_req->osd_req, 0, CEPH_OSD_OP_DELETE, 0);
- } else {
- dout("%s %p %llu~%llu -> %llu~%llu\n", __func__,
- obj_req, obj_req->ex.oe_off, obj_req->ex.oe_len,
- off, next_off - off);
- osd_req_op_extent_init(obj_req->osd_req, 0,
- truncate_or_zero_opcode(obj_req),
- off, next_off - off, 0, 0);
- }
-
- obj_req->write_state = RBD_OBJ_WRITE_FLAT;
- rbd_osd_req_format_write(obj_req);
+ obj_req->write_state = RBD_OBJ_WRITE_START;
return 0;
}
-static int count_zeroout_ops(struct rbd_obj_request *obj_req)
-{
- int num_osd_ops;
-
- if (rbd_obj_is_entire(obj_req) && obj_req->num_img_extents &&
- !rbd_obj_copyup_enabled(obj_req))
- num_osd_ops = 2; /* create + truncate */
- else
- num_osd_ops = 1; /* delete/truncate/zero */
-
- return num_osd_ops;
-}
-
-static void __rbd_obj_setup_zeroout(struct rbd_obj_request *obj_req,
- unsigned int which)
+static void __rbd_osd_setup_zeroout_ops(struct ceph_osd_request *osd_req,
+ int which)
{
+ struct rbd_obj_request *obj_req = osd_req->r_priv;
u16 opcode;
if (rbd_obj_is_entire(obj_req)) {
if (obj_req->num_img_extents) {
- if (!rbd_obj_copyup_enabled(obj_req))
- osd_req_op_init(obj_req->osd_req, which++,
+ if (!(obj_req->flags & RBD_OBJ_FLAG_COPYUP_ENABLED))
+ osd_req_op_init(osd_req, which++,
CEPH_OSD_OP_CREATE, 0);
opcode = CEPH_OSD_OP_TRUNCATE;
} else {
- osd_req_op_init(obj_req->osd_req, which++,
+ rbd_assert(obj_req->flags & RBD_OBJ_FLAG_DELETION);
+ osd_req_op_init(osd_req, which++,
CEPH_OSD_OP_DELETE, 0);
opcode = 0;
}
@@ -1971,18 +2485,13 @@ static void __rbd_obj_setup_zeroout(struct rbd_obj_request *obj_req,
}
if (opcode)
- osd_req_op_extent_init(obj_req->osd_req, which++, opcode,
+ osd_req_op_extent_init(osd_req, which, opcode,
obj_req->ex.oe_off, obj_req->ex.oe_len,
0, 0);
-
- rbd_assert(which == obj_req->osd_req->r_num_ops);
- rbd_osd_req_format_write(obj_req);
}
-static int rbd_obj_setup_zeroout(struct rbd_obj_request *obj_req)
+static int rbd_obj_init_zeroout(struct rbd_obj_request *obj_req)
{
- unsigned int num_osd_ops, which = 0;
- bool need_guard;
int ret;
/* reverse map the entire object onto the parent */
@@ -1990,31 +2499,66 @@ static int rbd_obj_setup_zeroout(struct rbd_obj_request *obj_req)
if (ret)
return ret;
- need_guard = rbd_obj_copyup_enabled(obj_req);
- num_osd_ops = need_guard + count_zeroout_ops(obj_req);
+ if (rbd_obj_copyup_enabled(obj_req))
+ obj_req->flags |= RBD_OBJ_FLAG_COPYUP_ENABLED;
+ if (!obj_req->num_img_extents) {
+ obj_req->flags |= RBD_OBJ_FLAG_NOOP_FOR_NONEXISTENT;
+ if (rbd_obj_is_entire(obj_req))
+ obj_req->flags |= RBD_OBJ_FLAG_DELETION;
+ }
- obj_req->osd_req = rbd_osd_req_create(obj_req, num_osd_ops);
- if (!obj_req->osd_req)
- return -ENOMEM;
+ obj_req->write_state = RBD_OBJ_WRITE_START;
+ return 0;
+}
- if (need_guard) {
- ret = __rbd_obj_setup_stat(obj_req, which++);
- if (ret)
- return ret;
+static int count_write_ops(struct rbd_obj_request *obj_req)
+{
+ struct rbd_img_request *img_req = obj_req->img_request;
- obj_req->write_state = RBD_OBJ_WRITE_GUARD;
- } else {
- obj_req->write_state = RBD_OBJ_WRITE_FLAT;
+ switch (img_req->op_type) {
+ case OBJ_OP_WRITE:
+ if (!use_object_map(img_req->rbd_dev) ||
+ !(obj_req->flags & RBD_OBJ_FLAG_MAY_EXIST))
+ return 2; /* setallochint + write/writefull */
+
+ return 1; /* write/writefull */
+ case OBJ_OP_DISCARD:
+ return 1; /* delete/truncate/zero */
+ case OBJ_OP_ZEROOUT:
+ if (rbd_obj_is_entire(obj_req) && obj_req->num_img_extents &&
+ !(obj_req->flags & RBD_OBJ_FLAG_COPYUP_ENABLED))
+ return 2; /* create + truncate */
+
+ return 1; /* delete/truncate/zero */
+ default:
+ BUG();
}
+}
- __rbd_obj_setup_zeroout(obj_req, which);
- return 0;
+static void rbd_osd_setup_write_ops(struct ceph_osd_request *osd_req,
+ int which)
+{
+ struct rbd_obj_request *obj_req = osd_req->r_priv;
+
+ switch (obj_req->img_request->op_type) {
+ case OBJ_OP_WRITE:
+ __rbd_osd_setup_write_ops(osd_req, which);
+ break;
+ case OBJ_OP_DISCARD:
+ __rbd_osd_setup_discard_ops(osd_req, which);
+ break;
+ case OBJ_OP_ZEROOUT:
+ __rbd_osd_setup_zeroout_ops(osd_req, which);
+ break;
+ default:
+ BUG();
+ }
}
/*
- * For each object request in @img_req, allocate an OSD request, add
- * individual OSD ops and prepare them for submission. The number of
- * OSD ops depends on op_type and the overlap point (if any).
+ * Prune the list of object requests (adjust offset and/or length, drop
+ * redundant requests). Prepare object request state machines and image
+ * request state machine for execution.
*/
static int __rbd_img_fill_request(struct rbd_img_request *img_req)
{
@@ -2024,16 +2568,16 @@ static int __rbd_img_fill_request(struct rbd_img_request *img_req)
for_each_obj_request_safe(img_req, obj_req, next_obj_req) {
switch (img_req->op_type) {
case OBJ_OP_READ:
- ret = rbd_obj_setup_read(obj_req);
+ ret = rbd_obj_init_read(obj_req);
break;
case OBJ_OP_WRITE:
- ret = rbd_obj_setup_write(obj_req);
+ ret = rbd_obj_init_write(obj_req);
break;
case OBJ_OP_DISCARD:
- ret = rbd_obj_setup_discard(obj_req);
+ ret = rbd_obj_init_discard(obj_req);
break;
case OBJ_OP_ZEROOUT:
- ret = rbd_obj_setup_zeroout(obj_req);
+ ret = rbd_obj_init_zeroout(obj_req);
break;
default:
BUG();
@@ -2041,17 +2585,12 @@ static int __rbd_img_fill_request(struct rbd_img_request *img_req)
if (ret < 0)
return ret;
if (ret > 0) {
- img_req->xferred += obj_req->ex.oe_len;
- img_req->pending_count--;
rbd_img_obj_request_del(img_req, obj_req);
continue;
}
-
- ret = ceph_osdc_alloc_messages(obj_req->osd_req, GFP_NOIO);
- if (ret)
- return ret;
}
+ img_req->state = RBD_IMG_START;
return 0;
}
@@ -2340,17 +2879,55 @@ static int rbd_img_fill_from_bvecs(struct rbd_img_request *img_req,
&it);
}
-static void rbd_img_request_submit(struct rbd_img_request *img_request)
+static void rbd_img_handle_request_work(struct work_struct *work)
{
- struct rbd_obj_request *obj_request;
+ struct rbd_img_request *img_req =
+ container_of(work, struct rbd_img_request, work);
- dout("%s: img %p\n", __func__, img_request);
+ rbd_img_handle_request(img_req, img_req->work_result);
+}
- rbd_img_request_get(img_request);
- for_each_obj_request(img_request, obj_request)
- rbd_obj_request_submit(obj_request);
+static void rbd_img_schedule(struct rbd_img_request *img_req, int result)
+{
+ INIT_WORK(&img_req->work, rbd_img_handle_request_work);
+ img_req->work_result = result;
+ queue_work(rbd_wq, &img_req->work);
+}
- rbd_img_request_put(img_request);
+static bool rbd_obj_may_exist(struct rbd_obj_request *obj_req)
+{
+ struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
+
+ if (rbd_object_map_may_exist(rbd_dev, obj_req->ex.oe_objno)) {
+ obj_req->flags |= RBD_OBJ_FLAG_MAY_EXIST;
+ return true;
+ }
+
+ dout("%s %p objno %llu assuming dne\n", __func__, obj_req,
+ obj_req->ex.oe_objno);
+ return false;
+}
+
+static int rbd_obj_read_object(struct rbd_obj_request *obj_req)
+{
+ struct ceph_osd_request *osd_req;
+ int ret;
+
+ osd_req = __rbd_obj_add_osd_request(obj_req, NULL, 1);
+ if (IS_ERR(osd_req))
+ return PTR_ERR(osd_req);
+
+ osd_req_op_extent_init(osd_req, 0, CEPH_OSD_OP_READ,
+ obj_req->ex.oe_off, obj_req->ex.oe_len, 0, 0);
+ rbd_osd_setup_data(osd_req, 0);
+ rbd_osd_format_read(osd_req);
+
+ ret = ceph_osdc_alloc_messages(osd_req, GFP_NOIO);
+ if (ret)
+ return ret;
+
+ rbd_osd_submit(osd_req);
+ return 0;
}
static int rbd_obj_read_from_parent(struct rbd_obj_request *obj_req)
@@ -2396,51 +2973,155 @@ static int rbd_obj_read_from_parent(struct rbd_obj_request *obj_req)
return ret;
}
- rbd_img_request_submit(child_img_req);
+ /* avoid parent chain recursion */
+ rbd_img_schedule(child_img_req, 0);
return 0;
}
-static bool rbd_obj_handle_read(struct rbd_obj_request *obj_req)
+static bool rbd_obj_advance_read(struct rbd_obj_request *obj_req, int *result)
{
struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
int ret;
- if (obj_req->result == -ENOENT &&
- rbd_dev->parent_overlap && !obj_req->tried_parent) {
- /* reverse map this object extent onto the parent */
- ret = rbd_obj_calc_img_extents(obj_req, false);
+again:
+ switch (obj_req->read_state) {
+ case RBD_OBJ_READ_START:
+ rbd_assert(!*result);
+
+ if (!rbd_obj_may_exist(obj_req)) {
+ *result = -ENOENT;
+ obj_req->read_state = RBD_OBJ_READ_OBJECT;
+ goto again;
+ }
+
+ ret = rbd_obj_read_object(obj_req);
if (ret) {
- obj_req->result = ret;
+ *result = ret;
return true;
}
-
- if (obj_req->num_img_extents) {
- obj_req->tried_parent = true;
- ret = rbd_obj_read_from_parent(obj_req);
+ obj_req->read_state = RBD_OBJ_READ_OBJECT;
+ return false;
+ case RBD_OBJ_READ_OBJECT:
+ if (*result == -ENOENT && rbd_dev->parent_overlap) {
+ /* reverse map this object extent onto the parent */
+ ret = rbd_obj_calc_img_extents(obj_req, false);
if (ret) {
- obj_req->result = ret;
+ *result = ret;
return true;
}
- return false;
+ if (obj_req->num_img_extents) {
+ ret = rbd_obj_read_from_parent(obj_req);
+ if (ret) {
+ *result = ret;
+ return true;
+ }
+ obj_req->read_state = RBD_OBJ_READ_PARENT;
+ return false;
+ }
+ }
+
+ /*
+ * -ENOENT means a hole in the image -- zero-fill the entire
+ * length of the request. A short read also implies zero-fill
+ * to the end of the request.
+ */
+ if (*result == -ENOENT) {
+ rbd_obj_zero_range(obj_req, 0, obj_req->ex.oe_len);
+ *result = 0;
+ } else if (*result >= 0) {
+ if (*result < obj_req->ex.oe_len)
+ rbd_obj_zero_range(obj_req, *result,
+ obj_req->ex.oe_len - *result);
+ else
+ rbd_assert(*result == obj_req->ex.oe_len);
+ *result = 0;
}
+ return true;
+ case RBD_OBJ_READ_PARENT:
+ /*
+ * The parent image is read only up to the overlap -- zero-fill
+ * from the overlap to the end of the request.
+ */
+ if (!*result) {
+ u32 obj_overlap = rbd_obj_img_extents_bytes(obj_req);
+
+ if (obj_overlap < obj_req->ex.oe_len)
+ rbd_obj_zero_range(obj_req, obj_overlap,
+ obj_req->ex.oe_len - obj_overlap);
+ }
+ return true;
+ default:
+ BUG();
}
+}
- /*
- * -ENOENT means a hole in the image -- zero-fill the entire
- * length of the request. A short read also implies zero-fill
- * to the end of the request. In both cases we update xferred
- * count to indicate the whole request was satisfied.
- */
- if (obj_req->result == -ENOENT ||
- (!obj_req->result && obj_req->xferred < obj_req->ex.oe_len)) {
- rbd_assert(!obj_req->xferred || !obj_req->result);
- rbd_obj_zero_range(obj_req, obj_req->xferred,
- obj_req->ex.oe_len - obj_req->xferred);
- obj_req->result = 0;
- obj_req->xferred = obj_req->ex.oe_len;
+static bool rbd_obj_write_is_noop(struct rbd_obj_request *obj_req)
+{
+ struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
+
+ if (rbd_object_map_may_exist(rbd_dev, obj_req->ex.oe_objno))
+ obj_req->flags |= RBD_OBJ_FLAG_MAY_EXIST;
+
+ if (!(obj_req->flags & RBD_OBJ_FLAG_MAY_EXIST) &&
+ (obj_req->flags & RBD_OBJ_FLAG_NOOP_FOR_NONEXISTENT)) {
+ dout("%s %p noop for nonexistent\n", __func__, obj_req);
+ return true;
}
- return true;
+ return false;
+}
+
+/*
+ * Return:
+ * 0 - object map update sent
+ * 1 - object map update isn't needed
+ * <0 - error
+ */
+static int rbd_obj_write_pre_object_map(struct rbd_obj_request *obj_req)
+{
+ struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
+ u8 new_state;
+
+ if (!(rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP))
+ return 1;
+
+ if (obj_req->flags & RBD_OBJ_FLAG_DELETION)
+ new_state = OBJECT_PENDING;
+ else
+ new_state = OBJECT_EXISTS;
+
+ return rbd_object_map_update(obj_req, CEPH_NOSNAP, new_state, NULL);
+}
+
+static int rbd_obj_write_object(struct rbd_obj_request *obj_req)
+{
+ struct ceph_osd_request *osd_req;
+ int num_ops = count_write_ops(obj_req);
+ int which = 0;
+ int ret;
+
+ if (obj_req->flags & RBD_OBJ_FLAG_COPYUP_ENABLED)
+ num_ops++; /* stat */
+
+ osd_req = rbd_obj_add_osd_request(obj_req, num_ops);
+ if (IS_ERR(osd_req))
+ return PTR_ERR(osd_req);
+
+ if (obj_req->flags & RBD_OBJ_FLAG_COPYUP_ENABLED) {
+ ret = rbd_osd_setup_stat(osd_req, which++);
+ if (ret)
+ return ret;
+ }
+
+ rbd_osd_setup_write_ops(osd_req, which);
+ rbd_osd_format_write(osd_req);
+
+ ret = ceph_osdc_alloc_messages(osd_req, GFP_NOIO);
+ if (ret)
+ return ret;
+
+ rbd_osd_submit(osd_req);
+ return 0;
}
/*
@@ -2463,123 +3144,67 @@ static bool is_zero_bvecs(struct bio_vec *bvecs, u32 bytes)
#define MODS_ONLY U32_MAX
-static int rbd_obj_issue_copyup_empty_snapc(struct rbd_obj_request *obj_req,
- u32 bytes)
+static int rbd_obj_copyup_empty_snapc(struct rbd_obj_request *obj_req,
+ u32 bytes)
{
+ struct ceph_osd_request *osd_req;
int ret;
dout("%s obj_req %p bytes %u\n", __func__, obj_req, bytes);
- rbd_assert(obj_req->osd_req->r_ops[0].op == CEPH_OSD_OP_STAT);
rbd_assert(bytes > 0 && bytes != MODS_ONLY);
- rbd_osd_req_destroy(obj_req->osd_req);
- obj_req->osd_req = __rbd_osd_req_create(obj_req, &rbd_empty_snapc, 1);
- if (!obj_req->osd_req)
- return -ENOMEM;
+ osd_req = __rbd_obj_add_osd_request(obj_req, &rbd_empty_snapc, 1);
+ if (IS_ERR(osd_req))
+ return PTR_ERR(osd_req);
- ret = osd_req_op_cls_init(obj_req->osd_req, 0, "rbd", "copyup");
+ ret = rbd_osd_setup_copyup(osd_req, 0, bytes);
if (ret)
return ret;
- osd_req_op_cls_request_data_bvecs(obj_req->osd_req, 0,
- obj_req->copyup_bvecs,
- obj_req->copyup_bvec_count,
- bytes);
- rbd_osd_req_format_write(obj_req);
+ rbd_osd_format_write(osd_req);
- ret = ceph_osdc_alloc_messages(obj_req->osd_req, GFP_NOIO);
+ ret = ceph_osdc_alloc_messages(osd_req, GFP_NOIO);
if (ret)
return ret;
- rbd_obj_request_submit(obj_req);
+ rbd_osd_submit(osd_req);
return 0;
}
-static int rbd_obj_issue_copyup_ops(struct rbd_obj_request *obj_req, u32 bytes)
+static int rbd_obj_copyup_current_snapc(struct rbd_obj_request *obj_req,
+ u32 bytes)
{
- struct rbd_img_request *img_req = obj_req->img_request;
- unsigned int num_osd_ops = (bytes != MODS_ONLY);
- unsigned int which = 0;
+ struct ceph_osd_request *osd_req;
+ int num_ops = count_write_ops(obj_req);
+ int which = 0;
int ret;
dout("%s obj_req %p bytes %u\n", __func__, obj_req, bytes);
- rbd_assert(obj_req->osd_req->r_ops[0].op == CEPH_OSD_OP_STAT ||
- obj_req->osd_req->r_ops[0].op == CEPH_OSD_OP_CALL);
- rbd_osd_req_destroy(obj_req->osd_req);
- switch (img_req->op_type) {
- case OBJ_OP_WRITE:
- num_osd_ops += count_write_ops(obj_req);
- break;
- case OBJ_OP_ZEROOUT:
- num_osd_ops += count_zeroout_ops(obj_req);
- break;
- default:
- BUG();
- }
+ if (bytes != MODS_ONLY)
+ num_ops++; /* copyup */
- obj_req->osd_req = rbd_osd_req_create(obj_req, num_osd_ops);
- if (!obj_req->osd_req)
- return -ENOMEM;
+ osd_req = rbd_obj_add_osd_request(obj_req, num_ops);
+ if (IS_ERR(osd_req))
+ return PTR_ERR(osd_req);
if (bytes != MODS_ONLY) {
- ret = osd_req_op_cls_init(obj_req->osd_req, which, "rbd",
- "copyup");
+ ret = rbd_osd_setup_copyup(osd_req, which++, bytes);
if (ret)
return ret;
-
- osd_req_op_cls_request_data_bvecs(obj_req->osd_req, which++,
- obj_req->copyup_bvecs,
- obj_req->copyup_bvec_count,
- bytes);
}
- switch (img_req->op_type) {
- case OBJ_OP_WRITE:
- __rbd_obj_setup_write(obj_req, which);
- break;
- case OBJ_OP_ZEROOUT:
- __rbd_obj_setup_zeroout(obj_req, which);
- break;
- default:
- BUG();
- }
+ rbd_osd_setup_write_ops(osd_req, which);
+ rbd_osd_format_write(osd_req);
- ret = ceph_osdc_alloc_messages(obj_req->osd_req, GFP_NOIO);
+ ret = ceph_osdc_alloc_messages(osd_req, GFP_NOIO);
if (ret)
return ret;
- rbd_obj_request_submit(obj_req);
+ rbd_osd_submit(osd_req);
return 0;
}
-static int rbd_obj_issue_copyup(struct rbd_obj_request *obj_req, u32 bytes)
-{
- /*
- * Only send non-zero copyup data to save some I/O and network
- * bandwidth -- zero copyup data is equivalent to the object not
- * existing.
- */
- if (is_zero_bvecs(obj_req->copyup_bvecs, bytes)) {
- dout("%s obj_req %p detected zeroes\n", __func__, obj_req);
- bytes = 0;
- }
-
- if (obj_req->img_request->snapc->num_snaps && bytes > 0) {
- /*
- * Send a copyup request with an empty snapshot context to
- * deep-copyup the object through all existing snapshots.
- * A second request with the current snapshot context will be
- * sent for the actual modification.
- */
- obj_req->write_state = RBD_OBJ_WRITE_COPYUP_EMPTY_SNAPC;
- return rbd_obj_issue_copyup_empty_snapc(obj_req, bytes);
- }
-
- obj_req->write_state = RBD_OBJ_WRITE_COPYUP_OPS;
- return rbd_obj_issue_copyup_ops(obj_req, bytes);
-}
-
static int setup_copyup_bvecs(struct rbd_obj_request *obj_req, u64 obj_overlap)
{
u32 i;
@@ -2608,7 +3233,12 @@ static int setup_copyup_bvecs(struct rbd_obj_request *obj_req, u64 obj_overlap)
return 0;
}
-static int rbd_obj_handle_write_guard(struct rbd_obj_request *obj_req)
+/*
+ * The target object doesn't exist. Read the data for the entire
+ * target object up to the overlap point (if any) from the parent,
+ * so we can use it for a copyup.
+ */
+static int rbd_obj_copyup_read_parent(struct rbd_obj_request *obj_req)
{
struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
int ret;
@@ -2623,178 +3253,492 @@ static int rbd_obj_handle_write_guard(struct rbd_obj_request *obj_req)
* request -- pass MODS_ONLY since the copyup isn't needed
* anymore.
*/
- obj_req->write_state = RBD_OBJ_WRITE_COPYUP_OPS;
- return rbd_obj_issue_copyup_ops(obj_req, MODS_ONLY);
+ return rbd_obj_copyup_current_snapc(obj_req, MODS_ONLY);
}
ret = setup_copyup_bvecs(obj_req, rbd_obj_img_extents_bytes(obj_req));
if (ret)
return ret;
- obj_req->write_state = RBD_OBJ_WRITE_READ_FROM_PARENT;
return rbd_obj_read_from_parent(obj_req);
}
-static bool rbd_obj_handle_write(struct rbd_obj_request *obj_req)
+static void rbd_obj_copyup_object_maps(struct rbd_obj_request *obj_req)
{
+ struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
+ struct ceph_snap_context *snapc = obj_req->img_request->snapc;
+ u8 new_state;
+ u32 i;
int ret;
- switch (obj_req->write_state) {
- case RBD_OBJ_WRITE_GUARD:
- rbd_assert(!obj_req->xferred);
- if (obj_req->result == -ENOENT) {
- /*
- * The target object doesn't exist. Read the data for
- * the entire target object up to the overlap point (if
- * any) from the parent, so we can use it for a copyup.
- */
- ret = rbd_obj_handle_write_guard(obj_req);
- if (ret) {
- obj_req->result = ret;
- return true;
- }
- return false;
+ rbd_assert(!obj_req->pending.result && !obj_req->pending.num_pending);
+
+ if (!(rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP))
+ return;
+
+ if (obj_req->flags & RBD_OBJ_FLAG_COPYUP_ZEROS)
+ return;
+
+ for (i = 0; i < snapc->num_snaps; i++) {
+ if ((rbd_dev->header.features & RBD_FEATURE_FAST_DIFF) &&
+ i + 1 < snapc->num_snaps)
+ new_state = OBJECT_EXISTS_CLEAN;
+ else
+ new_state = OBJECT_EXISTS;
+
+ ret = rbd_object_map_update(obj_req, snapc->snaps[i],
+ new_state, NULL);
+ if (ret < 0) {
+ obj_req->pending.result = ret;
+ return;
}
- /* fall through */
- case RBD_OBJ_WRITE_FLAT:
- case RBD_OBJ_WRITE_COPYUP_OPS:
- if (!obj_req->result)
- /*
- * There is no such thing as a successful short
- * write -- indicate the whole request was satisfied.
- */
- obj_req->xferred = obj_req->ex.oe_len;
- return true;
- case RBD_OBJ_WRITE_READ_FROM_PARENT:
- if (obj_req->result)
- return true;
- rbd_assert(obj_req->xferred);
- ret = rbd_obj_issue_copyup(obj_req, obj_req->xferred);
+ rbd_assert(!ret);
+ obj_req->pending.num_pending++;
+ }
+}
+
+static void rbd_obj_copyup_write_object(struct rbd_obj_request *obj_req)
+{
+ u32 bytes = rbd_obj_img_extents_bytes(obj_req);
+ int ret;
+
+ rbd_assert(!obj_req->pending.result && !obj_req->pending.num_pending);
+
+ /*
+ * Only send non-zero copyup data to save some I/O and network
+ * bandwidth -- zero copyup data is equivalent to the object not
+ * existing.
+ */
+ if (obj_req->flags & RBD_OBJ_FLAG_COPYUP_ZEROS)
+ bytes = 0;
+
+ if (obj_req->img_request->snapc->num_snaps && bytes > 0) {
+ /*
+ * Send a copyup request with an empty snapshot context to
+ * deep-copyup the object through all existing snapshots.
+ * A second request with the current snapshot context will be
+ * sent for the actual modification.
+ */
+ ret = rbd_obj_copyup_empty_snapc(obj_req, bytes);
if (ret) {
- obj_req->result = ret;
- obj_req->xferred = 0;
+ obj_req->pending.result = ret;
+ return;
+ }
+
+ obj_req->pending.num_pending++;
+ bytes = MODS_ONLY;
+ }
+
+ ret = rbd_obj_copyup_current_snapc(obj_req, bytes);
+ if (ret) {
+ obj_req->pending.result = ret;
+ return;
+ }
+
+ obj_req->pending.num_pending++;
+}
+
+static bool rbd_obj_advance_copyup(struct rbd_obj_request *obj_req, int *result)
+{
+ struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
+ int ret;
+
+again:
+ switch (obj_req->copyup_state) {
+ case RBD_OBJ_COPYUP_START:
+ rbd_assert(!*result);
+
+ ret = rbd_obj_copyup_read_parent(obj_req);
+ if (ret) {
+ *result = ret;
return true;
}
+ if (obj_req->num_img_extents)
+ obj_req->copyup_state = RBD_OBJ_COPYUP_READ_PARENT;
+ else
+ obj_req->copyup_state = RBD_OBJ_COPYUP_WRITE_OBJECT;
return false;
- case RBD_OBJ_WRITE_COPYUP_EMPTY_SNAPC:
- if (obj_req->result)
+ case RBD_OBJ_COPYUP_READ_PARENT:
+ if (*result)
return true;
- obj_req->write_state = RBD_OBJ_WRITE_COPYUP_OPS;
- ret = rbd_obj_issue_copyup_ops(obj_req, MODS_ONLY);
- if (ret) {
- obj_req->result = ret;
+ if (is_zero_bvecs(obj_req->copyup_bvecs,
+ rbd_obj_img_extents_bytes(obj_req))) {
+ dout("%s %p detected zeros\n", __func__, obj_req);
+ obj_req->flags |= RBD_OBJ_FLAG_COPYUP_ZEROS;
+ }
+
+ rbd_obj_copyup_object_maps(obj_req);
+ if (!obj_req->pending.num_pending) {
+ *result = obj_req->pending.result;
+ obj_req->copyup_state = RBD_OBJ_COPYUP_OBJECT_MAPS;
+ goto again;
+ }
+ obj_req->copyup_state = __RBD_OBJ_COPYUP_OBJECT_MAPS;
+ return false;
+ case __RBD_OBJ_COPYUP_OBJECT_MAPS:
+ if (!pending_result_dec(&obj_req->pending, result))
+ return false;
+ /* fall through */
+ case RBD_OBJ_COPYUP_OBJECT_MAPS:
+ if (*result) {
+ rbd_warn(rbd_dev, "snap object map update failed: %d",
+ *result);
return true;
}
+
+ rbd_obj_copyup_write_object(obj_req);
+ if (!obj_req->pending.num_pending) {
+ *result = obj_req->pending.result;
+ obj_req->copyup_state = RBD_OBJ_COPYUP_WRITE_OBJECT;
+ goto again;
+ }
+ obj_req->copyup_state = __RBD_OBJ_COPYUP_WRITE_OBJECT;
return false;
+ case __RBD_OBJ_COPYUP_WRITE_OBJECT:
+ if (!pending_result_dec(&obj_req->pending, result))
+ return false;
+ /* fall through */
+ case RBD_OBJ_COPYUP_WRITE_OBJECT:
+ return true;
default:
BUG();
}
}
/*
- * Returns true if @obj_req is completed, or false otherwise.
+ * Return:
+ * 0 - object map update sent
+ * 1 - object map update isn't needed
+ * <0 - error
*/
-static bool __rbd_obj_handle_request(struct rbd_obj_request *obj_req)
+static int rbd_obj_write_post_object_map(struct rbd_obj_request *obj_req)
{
- switch (obj_req->img_request->op_type) {
- case OBJ_OP_READ:
- return rbd_obj_handle_read(obj_req);
- case OBJ_OP_WRITE:
- return rbd_obj_handle_write(obj_req);
- case OBJ_OP_DISCARD:
- case OBJ_OP_ZEROOUT:
- if (rbd_obj_handle_write(obj_req)) {
+ struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
+ u8 current_state = OBJECT_PENDING;
+
+ if (!(rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP))
+ return 1;
+
+ if (!(obj_req->flags & RBD_OBJ_FLAG_DELETION))
+ return 1;
+
+ return rbd_object_map_update(obj_req, CEPH_NOSNAP, OBJECT_NONEXISTENT,
+ &current_state);
+}
+
+static bool rbd_obj_advance_write(struct rbd_obj_request *obj_req, int *result)
+{
+ struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;
+ int ret;
+
+again:
+ switch (obj_req->write_state) {
+ case RBD_OBJ_WRITE_START:
+ rbd_assert(!*result);
+
+ if (rbd_obj_write_is_noop(obj_req))
+ return true;
+
+ ret = rbd_obj_write_pre_object_map(obj_req);
+ if (ret < 0) {
+ *result = ret;
+ return true;
+ }
+ obj_req->write_state = RBD_OBJ_WRITE_PRE_OBJECT_MAP;
+ if (ret > 0)
+ goto again;
+ return false;
+ case RBD_OBJ_WRITE_PRE_OBJECT_MAP:
+ if (*result) {
+ rbd_warn(rbd_dev, "pre object map update failed: %d",
+ *result);
+ return true;
+ }
+ ret = rbd_obj_write_object(obj_req);
+ if (ret) {
+ *result = ret;
+ return true;
+ }
+ obj_req->write_state = RBD_OBJ_WRITE_OBJECT;
+ return false;
+ case RBD_OBJ_WRITE_OBJECT:
+ if (*result == -ENOENT) {
+ if (obj_req->flags & RBD_OBJ_FLAG_COPYUP_ENABLED) {
+ *result = 0;
+ obj_req->copyup_state = RBD_OBJ_COPYUP_START;
+ obj_req->write_state = __RBD_OBJ_WRITE_COPYUP;
+ goto again;
+ }
/*
- * Hide -ENOENT from delete/truncate/zero -- discarding
- * a non-existent object is not a problem.
+ * On a non-existent object:
+ * delete - -ENOENT, truncate/zero - 0
*/
- if (obj_req->result == -ENOENT) {
- obj_req->result = 0;
- obj_req->xferred = obj_req->ex.oe_len;
- }
+ if (obj_req->flags & RBD_OBJ_FLAG_DELETION)
+ *result = 0;
+ }
+ if (*result)
+ return true;
+
+ obj_req->write_state = RBD_OBJ_WRITE_COPYUP;
+ goto again;
+ case __RBD_OBJ_WRITE_COPYUP:
+ if (!rbd_obj_advance_copyup(obj_req, result))
+ return false;
+ /* fall through */
+ case RBD_OBJ_WRITE_COPYUP:
+ if (*result) {
+ rbd_warn(rbd_dev, "copyup failed: %d", *result);
return true;
}
+ ret = rbd_obj_write_post_object_map(obj_req);
+ if (ret < 0) {
+ *result = ret;
+ return true;
+ }
+ obj_req->write_state = RBD_OBJ_WRITE_POST_OBJECT_MAP;
+ if (ret > 0)
+ goto again;
return false;
+ case RBD_OBJ_WRITE_POST_OBJECT_MAP:
+ if (*result)
+ rbd_warn(rbd_dev, "post object map update failed: %d",
+ *result);
+ return true;
default:
BUG();
}
}
-static void rbd_obj_end_request(struct rbd_obj_request *obj_req)
+/*
+ * Return true if @obj_req is completed.
+ */
+static bool __rbd_obj_handle_request(struct rbd_obj_request *obj_req,
+ int *result)
{
struct rbd_img_request *img_req = obj_req->img_request;
+ struct rbd_device *rbd_dev = img_req->rbd_dev;
+ bool done;
- rbd_assert((!obj_req->result &&
- obj_req->xferred == obj_req->ex.oe_len) ||
- (obj_req->result < 0 && !obj_req->xferred));
- if (!obj_req->result) {
- img_req->xferred += obj_req->xferred;
- return;
- }
+ mutex_lock(&obj_req->state_mutex);
+ if (!rbd_img_is_write(img_req))
+ done = rbd_obj_advance_read(obj_req, result);
+ else
+ done = rbd_obj_advance_write(obj_req, result);
+ mutex_unlock(&obj_req->state_mutex);
- rbd_warn(img_req->rbd_dev,
- "%s at objno %llu %llu~%llu result %d xferred %llu",
- obj_op_name(img_req->op_type), obj_req->ex.oe_objno,
- obj_req->ex.oe_off, obj_req->ex.oe_len, obj_req->result,
- obj_req->xferred);
- if (!img_req->result) {
- img_req->result = obj_req->result;
- img_req->xferred = 0;
+ if (done && *result) {
+ rbd_assert(*result < 0);
+ rbd_warn(rbd_dev, "%s at objno %llu %llu~%llu result %d",
+ obj_op_name(img_req->op_type), obj_req->ex.oe_objno,
+ obj_req->ex.oe_off, obj_req->ex.oe_len, *result);
}
+ return done;
}
-static void rbd_img_end_child_request(struct rbd_img_request *img_req)
+/*
+ * This is open-coded in rbd_img_handle_request() to avoid parent chain
+ * recursion.
+ */
+static void rbd_obj_handle_request(struct rbd_obj_request *obj_req, int result)
{
- struct rbd_obj_request *obj_req = img_req->obj_request;
+ if (__rbd_obj_handle_request(obj_req, &result))
+ rbd_img_handle_request(obj_req->img_request, result);
+}
- rbd_assert(test_bit(IMG_REQ_CHILD, &img_req->flags));
- rbd_assert((!img_req->result &&
- img_req->xferred == rbd_obj_img_extents_bytes(obj_req)) ||
- (img_req->result < 0 && !img_req->xferred));
+static bool need_exclusive_lock(struct rbd_img_request *img_req)
+{
+ struct rbd_device *rbd_dev = img_req->rbd_dev;
+
+ if (!(rbd_dev->header.features & RBD_FEATURE_EXCLUSIVE_LOCK))
+ return false;
+
+ if (rbd_dev->spec->snap_id != CEPH_NOSNAP)
+ return false;
+
+ rbd_assert(!test_bit(IMG_REQ_CHILD, &img_req->flags));
+ if (rbd_dev->opts->lock_on_read ||
+ (rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP))
+ return true;
- obj_req->result = img_req->result;
- obj_req->xferred = img_req->xferred;
- rbd_img_request_put(img_req);
+ return rbd_img_is_write(img_req);
}
-static void rbd_img_end_request(struct rbd_img_request *img_req)
+static bool rbd_lock_add_request(struct rbd_img_request *img_req)
{
- rbd_assert(!test_bit(IMG_REQ_CHILD, &img_req->flags));
- rbd_assert((!img_req->result &&
- img_req->xferred == blk_rq_bytes(img_req->rq)) ||
- (img_req->result < 0 && !img_req->xferred));
+ struct rbd_device *rbd_dev = img_req->rbd_dev;
+ bool locked;
+
+ lockdep_assert_held(&rbd_dev->lock_rwsem);
+ locked = rbd_dev->lock_state == RBD_LOCK_STATE_LOCKED;
+ spin_lock(&rbd_dev->lock_lists_lock);
+ rbd_assert(list_empty(&img_req->lock_item));
+ if (!locked)
+ list_add_tail(&img_req->lock_item, &rbd_dev->acquiring_list);
+ else
+ list_add_tail(&img_req->lock_item, &rbd_dev->running_list);
+ spin_unlock(&rbd_dev->lock_lists_lock);
+ return locked;
+}
+
+static void rbd_lock_del_request(struct rbd_img_request *img_req)
+{
+ struct rbd_device *rbd_dev = img_req->rbd_dev;
+ bool need_wakeup;
- blk_mq_end_request(img_req->rq,
- errno_to_blk_status(img_req->result));
- rbd_img_request_put(img_req);
+ lockdep_assert_held(&rbd_dev->lock_rwsem);
+ spin_lock(&rbd_dev->lock_lists_lock);
+ rbd_assert(!list_empty(&img_req->lock_item));
+ list_del_init(&img_req->lock_item);
+ need_wakeup = (rbd_dev->lock_state == RBD_LOCK_STATE_RELEASING &&
+ list_empty(&rbd_dev->running_list));
+ spin_unlock(&rbd_dev->lock_lists_lock);
+ if (need_wakeup)
+ complete(&rbd_dev->releasing_wait);
}
-static void rbd_obj_handle_request(struct rbd_obj_request *obj_req)
+static int rbd_img_exclusive_lock(struct rbd_img_request *img_req)
{
- struct rbd_img_request *img_req;
+ struct rbd_device *rbd_dev = img_req->rbd_dev;
+
+ if (!need_exclusive_lock(img_req))
+ return 1;
+
+ if (rbd_lock_add_request(img_req))
+ return 1;
+
+ if (rbd_dev->opts->exclusive) {
+ WARN_ON(1); /* lock got released? */
+ return -EROFS;
+ }
+
+ /*
+ * Note the use of mod_delayed_work() in rbd_acquire_lock()
+ * and cancel_delayed_work() in wake_lock_waiters().
+ */
+ dout("%s rbd_dev %p queueing lock_dwork\n", __func__, rbd_dev);
+ queue_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork, 0);
+ return 0;
+}
+
+static void rbd_img_object_requests(struct rbd_img_request *img_req)
+{
+ struct rbd_obj_request *obj_req;
+
+ rbd_assert(!img_req->pending.result && !img_req->pending.num_pending);
+
+ for_each_obj_request(img_req, obj_req) {
+ int result = 0;
+
+ if (__rbd_obj_handle_request(obj_req, &result)) {
+ if (result) {
+ img_req->pending.result = result;
+ return;
+ }
+ } else {
+ img_req->pending.num_pending++;
+ }
+ }
+}
+
+static bool rbd_img_advance(struct rbd_img_request *img_req, int *result)
+{
+ struct rbd_device *rbd_dev = img_req->rbd_dev;
+ int ret;
again:
- if (!__rbd_obj_handle_request(obj_req))
- return;
+ switch (img_req->state) {
+ case RBD_IMG_START:
+ rbd_assert(!*result);
- img_req = obj_req->img_request;
- spin_lock(&img_req->completion_lock);
- rbd_obj_end_request(obj_req);
- rbd_assert(img_req->pending_count);
- if (--img_req->pending_count) {
- spin_unlock(&img_req->completion_lock);
- return;
+ ret = rbd_img_exclusive_lock(img_req);
+ if (ret < 0) {
+ *result = ret;
+ return true;
+ }
+ img_req->state = RBD_IMG_EXCLUSIVE_LOCK;
+ if (ret > 0)
+ goto again;
+ return false;
+ case RBD_IMG_EXCLUSIVE_LOCK:
+ if (*result)
+ return true;
+
+ rbd_assert(!need_exclusive_lock(img_req) ||
+ __rbd_is_lock_owner(rbd_dev));
+
+ rbd_img_object_requests(img_req);
+ if (!img_req->pending.num_pending) {
+ *result = img_req->pending.result;
+ img_req->state = RBD_IMG_OBJECT_REQUESTS;
+ goto again;
+ }
+ img_req->state = __RBD_IMG_OBJECT_REQUESTS;
+ return false;
+ case __RBD_IMG_OBJECT_REQUESTS:
+ if (!pending_result_dec(&img_req->pending, result))
+ return false;
+ /* fall through */
+ case RBD_IMG_OBJECT_REQUESTS:
+ return true;
+ default:
+ BUG();
+ }
+}
+
+/*
+ * Return true if @img_req is completed.
+ */
+static bool __rbd_img_handle_request(struct rbd_img_request *img_req,
+ int *result)
+{
+ struct rbd_device *rbd_dev = img_req->rbd_dev;
+ bool done;
+
+ if (need_exclusive_lock(img_req)) {
+ down_read(&rbd_dev->lock_rwsem);
+ mutex_lock(&img_req->state_mutex);
+ done = rbd_img_advance(img_req, result);
+ if (done)
+ rbd_lock_del_request(img_req);
+ mutex_unlock(&img_req->state_mutex);
+ up_read(&rbd_dev->lock_rwsem);
+ } else {
+ mutex_lock(&img_req->state_mutex);
+ done = rbd_img_advance(img_req, result);
+ mutex_unlock(&img_req->state_mutex);
}
- spin_unlock(&img_req->completion_lock);
+ if (done && *result) {
+ rbd_assert(*result < 0);
+ rbd_warn(rbd_dev, "%s%s result %d",
+ test_bit(IMG_REQ_CHILD, &img_req->flags) ? "child " : "",
+ obj_op_name(img_req->op_type), *result);
+ }
+ return done;
+}
+
+static void rbd_img_handle_request(struct rbd_img_request *img_req, int result)
+{
+again:
+ if (!__rbd_img_handle_request(img_req, &result))
+ return;
+
if (test_bit(IMG_REQ_CHILD, &img_req->flags)) {
- obj_req = img_req->obj_request;
- rbd_img_end_child_request(img_req);
- goto again;
+ struct rbd_obj_request *obj_req = img_req->obj_request;
+
+ rbd_img_request_put(img_req);
+ if (__rbd_obj_handle_request(obj_req, &result)) {
+ img_req = obj_req->img_request;
+ goto again;
+ }
+ } else {
+ struct request *rq = img_req->rq;
+
+ rbd_img_request_put(img_req);
+ blk_mq_end_request(rq, errno_to_blk_status(result));
}
- rbd_img_end_request(img_req);
}
static const struct rbd_client_id rbd_empty_cid;
@@ -2839,6 +3783,7 @@ static void __rbd_lock(struct rbd_device *rbd_dev, const char *cookie)
{
struct rbd_client_id cid = rbd_get_cid(rbd_dev);
+ rbd_dev->lock_state = RBD_LOCK_STATE_LOCKED;
strcpy(rbd_dev->lock_cookie, cookie);
rbd_set_owner_cid(rbd_dev, &cid);
queue_work(rbd_dev->task_wq, &rbd_dev->acquired_lock_work);
@@ -2863,7 +3808,6 @@ static int rbd_lock(struct rbd_device *rbd_dev)
if (ret)
return ret;
- rbd_dev->lock_state = RBD_LOCK_STATE_LOCKED;
__rbd_lock(rbd_dev, cookie);
return 0;
}
@@ -2882,7 +3826,7 @@ static void rbd_unlock(struct rbd_device *rbd_dev)
ret = ceph_cls_unlock(osdc, &rbd_dev->header_oid, &rbd_dev->header_oloc,
RBD_LOCK_NAME, rbd_dev->lock_cookie);
if (ret && ret != -ENOENT)
- rbd_warn(rbd_dev, "failed to unlock: %d", ret);
+ rbd_warn(rbd_dev, "failed to unlock header: %d", ret);
/* treat errors as the image is unlocked */
rbd_dev->lock_state = RBD_LOCK_STATE_UNLOCKED;
@@ -3009,15 +3953,34 @@ e_inval:
goto out;
}
-static void wake_requests(struct rbd_device *rbd_dev, bool wake_all)
+/*
+ * Either image request state machine(s) or rbd_add_acquire_lock()
+ * (i.e. "rbd map").
+ */
+static void wake_lock_waiters(struct rbd_device *rbd_dev, int result)
{
- dout("%s rbd_dev %p wake_all %d\n", __func__, rbd_dev, wake_all);
+ struct rbd_img_request *img_req;
+
+ dout("%s rbd_dev %p result %d\n", __func__, rbd_dev, result);
+ lockdep_assert_held_write(&rbd_dev->lock_rwsem);
cancel_delayed_work(&rbd_dev->lock_dwork);
- if (wake_all)
- wake_up_all(&rbd_dev->lock_waitq);
- else
- wake_up(&rbd_dev->lock_waitq);
+ if (!completion_done(&rbd_dev->acquire_wait)) {
+ rbd_assert(list_empty(&rbd_dev->acquiring_list) &&
+ list_empty(&rbd_dev->running_list));
+ rbd_dev->acquire_err = result;
+ complete_all(&rbd_dev->acquire_wait);
+ return;
+ }
+
+ list_for_each_entry(img_req, &rbd_dev->acquiring_list, lock_item) {
+ mutex_lock(&img_req->state_mutex);
+ rbd_assert(img_req->state == RBD_IMG_EXCLUSIVE_LOCK);
+ rbd_img_schedule(img_req, result);
+ mutex_unlock(&img_req->state_mutex);
+ }
+
+ list_splice_tail_init(&rbd_dev->acquiring_list, &rbd_dev->running_list);
}
static int get_lock_owner_info(struct rbd_device *rbd_dev,
@@ -3132,13 +4095,10 @@ static int rbd_try_lock(struct rbd_device *rbd_dev)
goto again;
ret = find_watcher(rbd_dev, lockers);
- if (ret) {
- if (ret > 0)
- ret = 0; /* have to request lock */
- goto out;
- }
+ if (ret)
+ goto out; /* request lock or error */
- rbd_warn(rbd_dev, "%s%llu seems dead, breaking lock",
+ rbd_warn(rbd_dev, "breaking header lock owned by %s%llu",
ENTITY_NAME(lockers[0].id.name));
ret = ceph_monc_blacklist_add(&client->monc,
@@ -3165,53 +4125,90 @@ out:
return ret;
}
+static int rbd_post_acquire_action(struct rbd_device *rbd_dev)
+{
+ int ret;
+
+ if (rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP) {
+ ret = rbd_object_map_open(rbd_dev);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
/*
- * ret is set only if lock_state is RBD_LOCK_STATE_UNLOCKED
+ * Return:
+ * 0 - lock acquired
+ * 1 - caller should call rbd_request_lock()
+ * <0 - error
*/
-static enum rbd_lock_state rbd_try_acquire_lock(struct rbd_device *rbd_dev,
- int *pret)
+static int rbd_try_acquire_lock(struct rbd_device *rbd_dev)
{
- enum rbd_lock_state lock_state;
+ int ret;
down_read(&rbd_dev->lock_rwsem);
dout("%s rbd_dev %p read lock_state %d\n", __func__, rbd_dev,
rbd_dev->lock_state);
if (__rbd_is_lock_owner(rbd_dev)) {
- lock_state = rbd_dev->lock_state;
up_read(&rbd_dev->lock_rwsem);
- return lock_state;
+ return 0;
}
up_read(&rbd_dev->lock_rwsem);
down_write(&rbd_dev->lock_rwsem);
dout("%s rbd_dev %p write lock_state %d\n", __func__, rbd_dev,
rbd_dev->lock_state);
- if (!__rbd_is_lock_owner(rbd_dev)) {
- *pret = rbd_try_lock(rbd_dev);
- if (*pret)
- rbd_warn(rbd_dev, "failed to acquire lock: %d", *pret);
+ if (__rbd_is_lock_owner(rbd_dev)) {
+ up_write(&rbd_dev->lock_rwsem);
+ return 0;
+ }
+
+ ret = rbd_try_lock(rbd_dev);
+ if (ret < 0) {
+ rbd_warn(rbd_dev, "failed to lock header: %d", ret);
+ if (ret == -EBLACKLISTED)
+ goto out;
+
+ ret = 1; /* request lock anyway */
+ }
+ if (ret > 0) {
+ up_write(&rbd_dev->lock_rwsem);
+ return ret;
+ }
+
+ rbd_assert(rbd_dev->lock_state == RBD_LOCK_STATE_LOCKED);
+ rbd_assert(list_empty(&rbd_dev->running_list));
+
+ ret = rbd_post_acquire_action(rbd_dev);
+ if (ret) {
+ rbd_warn(rbd_dev, "post-acquire action failed: %d", ret);
+ /*
+ * Can't stay in RBD_LOCK_STATE_LOCKED because
+ * rbd_lock_add_request() would let the request through,
+ * assuming that e.g. object map is locked and loaded.
+ */
+ rbd_unlock(rbd_dev);
}
- lock_state = rbd_dev->lock_state;
+out:
+ wake_lock_waiters(rbd_dev, ret);
up_write(&rbd_dev->lock_rwsem);
- return lock_state;
+ return ret;
}
static void rbd_acquire_lock(struct work_struct *work)
{
struct rbd_device *rbd_dev = container_of(to_delayed_work(work),
struct rbd_device, lock_dwork);
- enum rbd_lock_state lock_state;
- int ret = 0;
+ int ret;
dout("%s rbd_dev %p\n", __func__, rbd_dev);
again:
- lock_state = rbd_try_acquire_lock(rbd_dev, &ret);
- if (lock_state != RBD_LOCK_STATE_UNLOCKED || ret == -EBLACKLISTED) {
- if (lock_state == RBD_LOCK_STATE_LOCKED)
- wake_requests(rbd_dev, true);
- dout("%s rbd_dev %p lock_state %d ret %d - done\n", __func__,
- rbd_dev, lock_state, ret);
+ ret = rbd_try_acquire_lock(rbd_dev);
+ if (ret <= 0) {
+ dout("%s rbd_dev %p ret %d - done\n", __func__, rbd_dev, ret);
return;
}
@@ -3220,16 +4217,9 @@ again:
goto again; /* treat this as a dead client */
} else if (ret == -EROFS) {
rbd_warn(rbd_dev, "peer will not release lock");
- /*
- * If this is rbd_add_acquire_lock(), we want to fail
- * immediately -- reuse BLACKLISTED flag. Otherwise we
- * want to block.
- */
- if (!(rbd_dev->disk->flags & GENHD_FL_UP)) {
- set_bit(RBD_DEV_FLAG_BLACKLISTED, &rbd_dev->flags);
- /* wake "rbd map --exclusive" process */
- wake_requests(rbd_dev, false);
- }
+ down_write(&rbd_dev->lock_rwsem);
+ wake_lock_waiters(rbd_dev, ret);
+ up_write(&rbd_dev->lock_rwsem);
} else if (ret < 0) {
rbd_warn(rbd_dev, "error requesting lock: %d", ret);
mod_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork,
@@ -3246,43 +4236,67 @@ again:
}
}
-/*
- * lock_rwsem must be held for write
- */
-static bool rbd_release_lock(struct rbd_device *rbd_dev)
+static bool rbd_quiesce_lock(struct rbd_device *rbd_dev)
{
- dout("%s rbd_dev %p read lock_state %d\n", __func__, rbd_dev,
- rbd_dev->lock_state);
+ bool need_wait;
+
+ dout("%s rbd_dev %p\n", __func__, rbd_dev);
+ lockdep_assert_held_write(&rbd_dev->lock_rwsem);
+
if (rbd_dev->lock_state != RBD_LOCK_STATE_LOCKED)
return false;
- rbd_dev->lock_state = RBD_LOCK_STATE_RELEASING;
- downgrade_write(&rbd_dev->lock_rwsem);
/*
* Ensure that all in-flight IO is flushed.
- *
- * FIXME: ceph_osdc_sync() flushes the entire OSD client, which
- * may be shared with other devices.
*/
- ceph_osdc_sync(&rbd_dev->rbd_client->client->osdc);
+ rbd_dev->lock_state = RBD_LOCK_STATE_RELEASING;
+ rbd_assert(!completion_done(&rbd_dev->releasing_wait));
+ need_wait = !list_empty(&rbd_dev->running_list);
+ downgrade_write(&rbd_dev->lock_rwsem);
+ if (need_wait)
+ wait_for_completion(&rbd_dev->releasing_wait);
up_read(&rbd_dev->lock_rwsem);
down_write(&rbd_dev->lock_rwsem);
- dout("%s rbd_dev %p write lock_state %d\n", __func__, rbd_dev,
- rbd_dev->lock_state);
if (rbd_dev->lock_state != RBD_LOCK_STATE_RELEASING)
return false;
+ rbd_assert(list_empty(&rbd_dev->running_list));
+ return true;
+}
+
+static void rbd_pre_release_action(struct rbd_device *rbd_dev)
+{
+ if (rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP)
+ rbd_object_map_close(rbd_dev);
+}
+
+static void __rbd_release_lock(struct rbd_device *rbd_dev)
+{
+ rbd_assert(list_empty(&rbd_dev->running_list));
+
+ rbd_pre_release_action(rbd_dev);
rbd_unlock(rbd_dev);
+}
+
+/*
+ * lock_rwsem must be held for write
+ */
+static void rbd_release_lock(struct rbd_device *rbd_dev)
+{
+ if (!rbd_quiesce_lock(rbd_dev))
+ return;
+
+ __rbd_release_lock(rbd_dev);
+
/*
* Give others a chance to grab the lock - we would re-acquire
- * almost immediately if we got new IO during ceph_osdc_sync()
- * otherwise. We need to ack our own notifications, so this
- * lock_dwork will be requeued from rbd_wait_state_locked()
- * after wake_requests() in rbd_handle_released_lock().
+ * almost immediately if we got new IO while draining the running
+ * list otherwise. We need to ack our own notifications, so this
+ * lock_dwork will be requeued from rbd_handle_released_lock() by
+ * way of maybe_kick_acquire().
*/
cancel_delayed_work(&rbd_dev->lock_dwork);
- return true;
}
static void rbd_release_lock_work(struct work_struct *work)
@@ -3295,6 +4309,23 @@ static void rbd_release_lock_work(struct work_struct *work)
up_write(&rbd_dev->lock_rwsem);
}
+static void maybe_kick_acquire(struct rbd_device *rbd_dev)
+{
+ bool have_requests;
+
+ dout("%s rbd_dev %p\n", __func__, rbd_dev);
+ if (__rbd_is_lock_owner(rbd_dev))
+ return;
+
+ spin_lock(&rbd_dev->lock_lists_lock);
+ have_requests = !list_empty(&rbd_dev->acquiring_list);
+ spin_unlock(&rbd_dev->lock_lists_lock);
+ if (have_requests || delayed_work_pending(&rbd_dev->lock_dwork)) {
+ dout("%s rbd_dev %p kicking lock_dwork\n", __func__, rbd_dev);
+ mod_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork, 0);
+ }
+}
+
static void rbd_handle_acquired_lock(struct rbd_device *rbd_dev, u8 struct_v,
void **p)
{
@@ -3324,8 +4355,7 @@ static void rbd_handle_acquired_lock(struct rbd_device *rbd_dev, u8 struct_v,
down_read(&rbd_dev->lock_rwsem);
}
- if (!__rbd_is_lock_owner(rbd_dev))
- wake_requests(rbd_dev, false);
+ maybe_kick_acquire(rbd_dev);
up_read(&rbd_dev->lock_rwsem);
}
@@ -3357,8 +4387,7 @@ static void rbd_handle_released_lock(struct rbd_device *rbd_dev, u8 struct_v,
down_read(&rbd_dev->lock_rwsem);
}
- if (!__rbd_is_lock_owner(rbd_dev))
- wake_requests(rbd_dev, false);
+ maybe_kick_acquire(rbd_dev);
up_read(&rbd_dev->lock_rwsem);
}
@@ -3608,7 +4637,6 @@ static void cancel_tasks_sync(struct rbd_device *rbd_dev)
static void rbd_unregister_watch(struct rbd_device *rbd_dev)
{
- WARN_ON(waitqueue_active(&rbd_dev->lock_waitq));
cancel_tasks_sync(rbd_dev);
mutex_lock(&rbd_dev->watch_mutex);
@@ -3630,7 +4658,8 @@ static void rbd_reacquire_lock(struct rbd_device *rbd_dev)
char cookie[32];
int ret;
- WARN_ON(rbd_dev->lock_state != RBD_LOCK_STATE_LOCKED);
+ if (!rbd_quiesce_lock(rbd_dev))
+ return;
format_lock_cookie(rbd_dev, cookie);
ret = ceph_cls_set_cookie(osdc, &rbd_dev->header_oid,
@@ -3646,11 +4675,11 @@ static void rbd_reacquire_lock(struct rbd_device *rbd_dev)
* Lock cookie cannot be updated on older OSDs, so do
* a manual release and queue an acquire.
*/
- if (rbd_release_lock(rbd_dev))
- queue_delayed_work(rbd_dev->task_wq,
- &rbd_dev->lock_dwork, 0);
+ __rbd_release_lock(rbd_dev);
+ queue_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork, 0);
} else {
__rbd_lock(rbd_dev, cookie);
+ wake_lock_waiters(rbd_dev, 0);
}
}
@@ -3671,15 +4700,18 @@ static void rbd_reregister_watch(struct work_struct *work)
ret = __rbd_register_watch(rbd_dev);
if (ret) {
rbd_warn(rbd_dev, "failed to reregister watch: %d", ret);
- if (ret == -EBLACKLISTED || ret == -ENOENT) {
- set_bit(RBD_DEV_FLAG_BLACKLISTED, &rbd_dev->flags);
- wake_requests(rbd_dev, true);
- } else {
+ if (ret != -EBLACKLISTED && ret != -ENOENT) {
queue_delayed_work(rbd_dev->task_wq,
&rbd_dev->watch_dwork,
RBD_RETRY_DELAY);
+ mutex_unlock(&rbd_dev->watch_mutex);
+ return;
}
+
mutex_unlock(&rbd_dev->watch_mutex);
+ down_write(&rbd_dev->lock_rwsem);
+ wake_lock_waiters(rbd_dev, ret);
+ up_write(&rbd_dev->lock_rwsem);
return;
}
@@ -3742,7 +4774,7 @@ static int rbd_obj_method_sync(struct rbd_device *rbd_dev,
ret = ceph_osdc_call(osdc, oid, oloc, RBD_DRV_NAME, method_name,
CEPH_OSD_FLAG_READ, req_page, outbound_size,
- reply_page, &inbound_size);
+ &reply_page, &inbound_size);
if (!ret) {
memcpy(inbound, page_address(reply_page), inbound_size);
ret = inbound_size;
@@ -3754,54 +4786,6 @@ static int rbd_obj_method_sync(struct rbd_device *rbd_dev,
return ret;
}
-/*
- * lock_rwsem must be held for read
- */
-static int rbd_wait_state_locked(struct rbd_device *rbd_dev, bool may_acquire)
-{
- DEFINE_WAIT(wait);
- unsigned long timeout;
- int ret = 0;
-
- if (test_bit(RBD_DEV_FLAG_BLACKLISTED, &rbd_dev->flags))
- return -EBLACKLISTED;
-
- if (rbd_dev->lock_state == RBD_LOCK_STATE_LOCKED)
- return 0;
-
- if (!may_acquire) {
- rbd_warn(rbd_dev, "exclusive lock required");
- return -EROFS;
- }
-
- do {
- /*
- * Note the use of mod_delayed_work() in rbd_acquire_lock()
- * and cancel_delayed_work() in wake_requests().
- */
- dout("%s rbd_dev %p queueing lock_dwork\n", __func__, rbd_dev);
- queue_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork, 0);
- prepare_to_wait_exclusive(&rbd_dev->lock_waitq, &wait,
- TASK_UNINTERRUPTIBLE);
- up_read(&rbd_dev->lock_rwsem);
- timeout = schedule_timeout(ceph_timeout_jiffies(
- rbd_dev->opts->lock_timeout));
- down_read(&rbd_dev->lock_rwsem);
- if (test_bit(RBD_DEV_FLAG_BLACKLISTED, &rbd_dev->flags)) {
- ret = -EBLACKLISTED;
- break;
- }
- if (!timeout) {
- rbd_warn(rbd_dev, "timed out waiting for lock");
- ret = -ETIMEDOUT;
- break;
- }
- } while (rbd_dev->lock_state != RBD_LOCK_STATE_LOCKED);
-
- finish_wait(&rbd_dev->lock_waitq, &wait);
- return ret;
-}
-
static void rbd_queue_workfn(struct work_struct *work)
{
struct request *rq = blk_mq_rq_from_pdu(work);
@@ -3812,7 +4796,6 @@ static void rbd_queue_workfn(struct work_struct *work)
u64 length = blk_rq_bytes(rq);
enum obj_operation_type op_type;
u64 mapping_size;
- bool must_be_locked;
int result;
switch (req_op(rq)) {
@@ -3886,21 +4869,10 @@ static void rbd_queue_workfn(struct work_struct *work)
goto err_rq;
}
- must_be_locked =
- (rbd_dev->header.features & RBD_FEATURE_EXCLUSIVE_LOCK) &&
- (op_type != OBJ_OP_READ || rbd_dev->opts->lock_on_read);
- if (must_be_locked) {
- down_read(&rbd_dev->lock_rwsem);
- result = rbd_wait_state_locked(rbd_dev,
- !rbd_dev->opts->exclusive);
- if (result)
- goto err_unlock;
- }
-
img_request = rbd_img_request_create(rbd_dev, op_type, snapc);
if (!img_request) {
result = -ENOMEM;
- goto err_unlock;
+ goto err_rq;
}
img_request->rq = rq;
snapc = NULL; /* img_request consumes a ref */
@@ -3910,19 +4882,14 @@ static void rbd_queue_workfn(struct work_struct *work)
else
result = rbd_img_fill_from_bio(img_request, offset, length,
rq->bio);
- if (result || !img_request->pending_count)
+ if (result)
goto err_img_request;
- rbd_img_request_submit(img_request);
- if (must_be_locked)
- up_read(&rbd_dev->lock_rwsem);
+ rbd_img_handle_request(img_request, 0);
return;
err_img_request:
rbd_img_request_put(img_request);
-err_unlock:
- if (must_be_locked)
- up_read(&rbd_dev->lock_rwsem);
err_rq:
if (result)
rbd_warn(rbd_dev, "%s %llx at %llx result %d",
@@ -4589,7 +5556,13 @@ static struct rbd_device *__rbd_dev_create(struct rbd_client *rbdc,
INIT_WORK(&rbd_dev->released_lock_work, rbd_notify_released_lock);
INIT_DELAYED_WORK(&rbd_dev->lock_dwork, rbd_acquire_lock);
INIT_WORK(&rbd_dev->unlock_work, rbd_release_lock_work);
- init_waitqueue_head(&rbd_dev->lock_waitq);
+ spin_lock_init(&rbd_dev->lock_lists_lock);
+ INIT_LIST_HEAD(&rbd_dev->acquiring_list);
+ INIT_LIST_HEAD(&rbd_dev->running_list);
+ init_completion(&rbd_dev->acquire_wait);
+ init_completion(&rbd_dev->releasing_wait);
+
+ spin_lock_init(&rbd_dev->object_map_lock);
rbd_dev->dev.bus = &rbd_bus_type;
rbd_dev->dev.type = &rbd_device_type;
@@ -4772,6 +5745,32 @@ static int rbd_dev_v2_features(struct rbd_device *rbd_dev)
&rbd_dev->header.features);
}
+/*
+ * These are generic image flags, but since they are used only for
+ * object map, store them in rbd_dev->object_map_flags.
+ *
+ * For the same reason, this function is called only on object map
+ * (re)load and not on header refresh.
+ */
+static int rbd_dev_v2_get_flags(struct rbd_device *rbd_dev)
+{
+ __le64 snapid = cpu_to_le64(rbd_dev->spec->snap_id);
+ __le64 flags;
+ int ret;
+
+ ret = rbd_obj_method_sync(rbd_dev, &rbd_dev->header_oid,
+ &rbd_dev->header_oloc, "get_flags",
+ &snapid, sizeof(snapid),
+ &flags, sizeof(flags));
+ if (ret < 0)
+ return ret;
+ if (ret < sizeof(flags))
+ return -EBADMSG;
+
+ rbd_dev->object_map_flags = le64_to_cpu(flags);
+ return 0;
+}
+
struct parent_image_info {
u64 pool_id;
const char *pool_ns;
@@ -4829,7 +5828,7 @@ static int __get_parent_info(struct rbd_device *rbd_dev,
ret = ceph_osdc_call(osdc, &rbd_dev->header_oid, &rbd_dev->header_oloc,
"rbd", "parent_get", CEPH_OSD_FLAG_READ,
- req_page, sizeof(u64), reply_page, &reply_len);
+ req_page, sizeof(u64), &reply_page, &reply_len);
if (ret)
return ret == -EOPNOTSUPP ? 1 : ret;
@@ -4841,7 +5840,7 @@ static int __get_parent_info(struct rbd_device *rbd_dev,
ret = ceph_osdc_call(osdc, &rbd_dev->header_oid, &rbd_dev->header_oloc,
"rbd", "parent_overlap_get", CEPH_OSD_FLAG_READ,
- req_page, sizeof(u64), reply_page, &reply_len);
+ req_page, sizeof(u64), &reply_page, &reply_len);
if (ret)
return ret;
@@ -4872,7 +5871,7 @@ static int __get_parent_info_legacy(struct rbd_device *rbd_dev,
ret = ceph_osdc_call(osdc, &rbd_dev->header_oid, &rbd_dev->header_oloc,
"rbd", "get_parent", CEPH_OSD_FLAG_READ,
- req_page, sizeof(u64), reply_page, &reply_len);
+ req_page, sizeof(u64), &reply_page, &reply_len);
if (ret)
return ret;
@@ -5605,28 +6604,49 @@ static void rbd_dev_image_unlock(struct rbd_device *rbd_dev)
{
down_write(&rbd_dev->lock_rwsem);
if (__rbd_is_lock_owner(rbd_dev))
- rbd_unlock(rbd_dev);
+ __rbd_release_lock(rbd_dev);
up_write(&rbd_dev->lock_rwsem);
}
+/*
+ * If the wait is interrupted, an error is returned even if the lock
+ * was successfully acquired. rbd_dev_image_unlock() will release it
+ * if needed.
+ */
static int rbd_add_acquire_lock(struct rbd_device *rbd_dev)
{
- int ret;
+ long ret;
if (!(rbd_dev->header.features & RBD_FEATURE_EXCLUSIVE_LOCK)) {
+ if (!rbd_dev->opts->exclusive && !rbd_dev->opts->lock_on_read)
+ return 0;
+
rbd_warn(rbd_dev, "exclusive-lock feature is not enabled");
return -EINVAL;
}
- /* FIXME: "rbd map --exclusive" should be in interruptible */
- down_read(&rbd_dev->lock_rwsem);
- ret = rbd_wait_state_locked(rbd_dev, true);
- up_read(&rbd_dev->lock_rwsem);
+ if (rbd_dev->spec->snap_id != CEPH_NOSNAP)
+ return 0;
+
+ rbd_assert(!rbd_is_lock_owner(rbd_dev));
+ queue_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork, 0);
+ ret = wait_for_completion_killable_timeout(&rbd_dev->acquire_wait,
+ ceph_timeout_jiffies(rbd_dev->opts->lock_timeout));
+ if (ret > 0)
+ ret = rbd_dev->acquire_err;
+ else if (!ret)
+ ret = -ETIMEDOUT;
+
if (ret) {
- rbd_warn(rbd_dev, "failed to acquire exclusive lock");
- return -EROFS;
+ rbd_warn(rbd_dev, "failed to acquire exclusive lock: %ld", ret);
+ return ret;
}
+ /*
+ * The lock may have been released by now, unless automatic lock
+ * transitions are disabled.
+ */
+ rbd_assert(!rbd_dev->opts->exclusive || rbd_is_lock_owner(rbd_dev));
return 0;
}
@@ -5724,6 +6744,8 @@ static void rbd_dev_unprobe(struct rbd_device *rbd_dev)
struct rbd_image_header *header;
rbd_dev_parent_put(rbd_dev);
+ rbd_object_map_free(rbd_dev);
+ rbd_dev_mapping_clear(rbd_dev);
/* Free dynamic fields from the header, then zero it out */
@@ -5824,7 +6846,6 @@ out_err:
static void rbd_dev_device_release(struct rbd_device *rbd_dev)
{
clear_bit(RBD_DEV_FLAG_EXISTS, &rbd_dev->flags);
- rbd_dev_mapping_clear(rbd_dev);
rbd_free_disk(rbd_dev);
if (!single_major)
unregister_blkdev(rbd_dev->major, rbd_dev->name);
@@ -5858,23 +6879,17 @@ static int rbd_dev_device_setup(struct rbd_device *rbd_dev)
if (ret)
goto err_out_blkdev;
- ret = rbd_dev_mapping_set(rbd_dev);
- if (ret)
- goto err_out_disk;
-
set_capacity(rbd_dev->disk, rbd_dev->mapping.size / SECTOR_SIZE);
set_disk_ro(rbd_dev->disk, rbd_dev->opts->read_only);
ret = dev_set_name(&rbd_dev->dev, "%d", rbd_dev->dev_id);
if (ret)
- goto err_out_mapping;
+ goto err_out_disk;
set_bit(RBD_DEV_FLAG_EXISTS, &rbd_dev->flags);
up_write(&rbd_dev->header_rwsem);
return 0;
-err_out_mapping:
- rbd_dev_mapping_clear(rbd_dev);
err_out_disk:
rbd_free_disk(rbd_dev);
err_out_blkdev:
@@ -5975,6 +6990,17 @@ static int rbd_dev_image_probe(struct rbd_device *rbd_dev, int depth)
goto err_out_probe;
}
+ ret = rbd_dev_mapping_set(rbd_dev);
+ if (ret)
+ goto err_out_probe;
+
+ if (rbd_dev->spec->snap_id != CEPH_NOSNAP &&
+ (rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP)) {
+ ret = rbd_object_map_load(rbd_dev);
+ if (ret)
+ goto err_out_probe;
+ }
+
if (rbd_dev->header.features & RBD_FEATURE_LAYERING) {
ret = rbd_dev_v2_parent_info(rbd_dev);
if (ret)
@@ -6071,11 +7097,9 @@ static ssize_t do_rbd_add(struct bus_type *bus,
if (rc)
goto err_out_image_probe;
- if (rbd_dev->opts->exclusive) {
- rc = rbd_add_acquire_lock(rbd_dev);
- if (rc)
- goto err_out_device_setup;
- }
+ rc = rbd_add_acquire_lock(rbd_dev);
+ if (rc)
+ goto err_out_image_lock;
/* Everything's ready. Announce the disk to the world. */
@@ -6101,7 +7125,6 @@ out:
err_out_image_lock:
rbd_dev_image_unlock(rbd_dev);
-err_out_device_setup:
rbd_dev_device_release(rbd_dev);
err_out_image_probe:
rbd_dev_image_release(rbd_dev);
diff --git a/drivers/block/rbd_types.h b/drivers/block/rbd_types.h
index 62ff50d3e7a6..ac98ab6ccd3b 100644
--- a/drivers/block/rbd_types.h
+++ b/drivers/block/rbd_types.h
@@ -18,6 +18,7 @@
/* For format version 2, rbd image 'foo' consists of objects
* rbd_id.foo - id of image
* rbd_header.<id> - image metadata
+ * rbd_object_map.<id> - optional image object map
* rbd_data.<id>.0000000000000000
* rbd_data.<id>.0000000000000001
* ... - data
@@ -25,6 +26,7 @@
*/
#define RBD_HEADER_PREFIX "rbd_header."
+#define RBD_OBJECT_MAP_PREFIX "rbd_object_map."
#define RBD_ID_PREFIX "rbd_id."
#define RBD_V2_DATA_FORMAT "%s.%016llx"
@@ -39,6 +41,14 @@ enum rbd_notify_op {
RBD_NOTIFY_OP_HEADER_UPDATE = 3,
};
+#define OBJECT_NONEXISTENT 0
+#define OBJECT_EXISTS 1
+#define OBJECT_PENDING 2
+#define OBJECT_EXISTS_CLEAN 3
+
+#define RBD_FLAG_OBJECT_MAP_INVALID (1ULL << 0)
+#define RBD_FLAG_FAST_DIFF_INVALID (1ULL << 1)
+
/*
* For format version 1, rbd image 'foo' consists of objects
* foo.rbd - image metadata
diff --git a/drivers/block/skd_main.c b/drivers/block/skd_main.c
index c479235862e5..51569c199a6c 100644
--- a/drivers/block/skd_main.c
+++ b/drivers/block/skd_main.c
@@ -2694,7 +2694,6 @@ static int skd_cons_skmsg(struct skd_device *skdev)
(FIT_QCMD_ALIGN - 1),
"not aligned: msg_buf %p mb_dma_address %pad\n",
skmsg->msg_buf, &skmsg->mb_dma_address);
- memset(skmsg->msg_buf, 0, SKD_N_FITMSG_BYTES);
}
err_out:
diff --git a/drivers/block/xen-blkback/xenbus.c b/drivers/block/xen-blkback/xenbus.c
index 3ac6a5d18071..b90dbcd99c03 100644
--- a/drivers/block/xen-blkback/xenbus.c
+++ b/drivers/block/xen-blkback/xenbus.c
@@ -965,6 +965,7 @@ static int read_per_ring_refs(struct xen_blkif_ring *ring, const char *dir)
}
}
+ err = -ENOMEM;
for (i = 0; i < nr_grefs * XEN_BLKIF_REQS_PER_PAGE; i++) {
req = kzalloc(sizeof(*req), GFP_KERNEL);
if (!req)
@@ -987,7 +988,7 @@ static int read_per_ring_refs(struct xen_blkif_ring *ring, const char *dir)
err = xen_blkif_map(ring, ring_ref, nr_grefs, evtchn);
if (err) {
xenbus_dev_fatal(dev, err, "mapping ring-ref port %u", evtchn);
- return err;
+ goto fail;
}
return 0;
@@ -1007,8 +1008,7 @@ fail:
}
kfree(req);
}
- return -ENOMEM;
-
+ return err;
}
static int connect_ring(struct backend_info *be)
diff --git a/drivers/block/zram/Kconfig b/drivers/block/zram/Kconfig
index 1ffc64770643..fe7a4b7d30cf 100644
--- a/drivers/block/zram/Kconfig
+++ b/drivers/block/zram/Kconfig
@@ -12,7 +12,7 @@ config ZRAM
It has several use cases, for example: /tmp storage, use as swap
disks and maybe many more.
- See Documentation/blockdev/zram.txt for more information.
+ See Documentation/admin-guide/blockdev/zram.rst for more information.
config ZRAM_WRITEBACK
bool "Write back incompressible or idle page to backing device"
@@ -26,7 +26,7 @@ config ZRAM_WRITEBACK
With /sys/block/zramX/{idle,writeback}, application could ask
idle page's writeback to the backing device to save in memory.
- See Documentation/blockdev/zram.txt for more information.
+ See Documentation/admin-guide/blockdev/zram.rst for more information.
config ZRAM_MEMORY_TRACKING
bool "Track zRam block status"
@@ -36,4 +36,4 @@ config ZRAM_MEMORY_TRACKING
of zRAM. Admin could see the information via
/sys/kernel/debug/zram/zramX/block_state.
- See Documentation/blockdev/zram.txt for more information.
+ See Documentation/admin-guide/blockdev/zram.rst for more information.
diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig
index b9c34ff9a0d3..aae665a3a254 100644
--- a/drivers/bluetooth/Kconfig
+++ b/drivers/bluetooth/Kconfig
@@ -52,6 +52,17 @@ config BT_HCIBTUSB_BCM
Say Y here to compile support for Broadcom protocol.
+config BT_HCIBTUSB_MTK
+ bool "MediaTek protocol support"
+ depends on BT_HCIBTUSB
+ default n
+ help
+ The MediaTek protocol support enables firmware download
+ support and chip initialization for MediaTek Bluetooth
+ USB controllers.
+
+ Say Y here to compile support for MediaTek protocol.
+
config BT_HCIBTUSB_RTL
bool "Realtek protocol support"
depends on BT_HCIBTUSB
@@ -237,6 +248,7 @@ config BT_HCIUART_AG6XX
config BT_HCIUART_MRVL
bool "Marvell protocol support"
depends on BT_HCIUART
+ depends on BT_HCIUART_SERDEV
select BT_HCIUART_H4
help
Marvell is serial protocol for communication between Bluetooth
diff --git a/drivers/bluetooth/bpa10x.c b/drivers/bluetooth/bpa10x.c
index a346ccb5450d..1fa58c059cbf 100644
--- a/drivers/bluetooth/bpa10x.c
+++ b/drivers/bluetooth/bpa10x.c
@@ -337,7 +337,7 @@ static int bpa10x_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
usb_free_urb(urb);
- return 0;
+ return err;
}
static int bpa10x_set_diag(struct hci_dev *hdev, bool enable)
@@ -359,7 +359,8 @@ static int bpa10x_set_diag(struct hci_dev *hdev, bool enable)
return 0;
}
-static int bpa10x_probe(struct usb_interface *intf, const struct usb_device_id *id)
+static int bpa10x_probe(struct usb_interface *intf,
+ const struct usb_device_id *id)
{
struct bpa10x_data *data;
struct hci_dev *hdev;
diff --git a/drivers/bluetooth/btbcm.c b/drivers/bluetooth/btbcm.c
index 3fe941539a1f..2d2e6d862068 100644
--- a/drivers/bluetooth/btbcm.c
+++ b/drivers/bluetooth/btbcm.c
@@ -23,6 +23,7 @@
#define BDADDR_BCM43430A0 (&(bdaddr_t) {{0xac, 0x1f, 0x12, 0xa0, 0x43, 0x43}})
#define BDADDR_BCM4324B3 (&(bdaddr_t) {{0x00, 0x00, 0x00, 0xb3, 0x24, 0x43}})
#define BDADDR_BCM4330B1 (&(bdaddr_t) {{0x00, 0x00, 0x00, 0xb1, 0x30, 0x43}})
+#define BDADDR_BCM4345C5 (&(bdaddr_t) {{0xac, 0x1f, 0x00, 0xc5, 0x45, 0x43}})
#define BDADDR_BCM43341B (&(bdaddr_t) {{0xac, 0x1f, 0x00, 0x1b, 0x34, 0x43}})
int btbcm_check_bdaddr(struct hci_dev *hdev)
@@ -73,6 +74,7 @@ int btbcm_check_bdaddr(struct hci_dev *hdev)
!bacmp(&bda->bdaddr, BDADDR_BCM2076B1) ||
!bacmp(&bda->bdaddr, BDADDR_BCM4324B3) ||
!bacmp(&bda->bdaddr, BDADDR_BCM4330B1) ||
+ !bacmp(&bda->bdaddr, BDADDR_BCM4345C5) ||
!bacmp(&bda->bdaddr, BDADDR_BCM43430A0) ||
!bacmp(&bda->bdaddr, BDADDR_BCM43341B)) {
bt_dev_info(hdev, "BCM: Using default device address (%pMR)",
@@ -332,9 +334,11 @@ static const struct bcm_subver_table bcm_uart_subver_table[] = {
{ 0x2122, "BCM4343A0" }, /* 001.001.034 */
{ 0x2209, "BCM43430A1" }, /* 001.002.009 */
{ 0x6119, "BCM4345C0" }, /* 003.001.025 */
+ { 0x6606, "BCM4345C5" }, /* 003.006.006 */
{ 0x230f, "BCM4356A2" }, /* 001.003.015 */
{ 0x220e, "BCM20702A1" }, /* 001.002.014 */
{ 0x4217, "BCM4329B1" }, /* 002.002.023 */
+ { 0x6106, "BCM4359C0" }, /* 003.001.006 */
{ }
};
diff --git a/drivers/bluetooth/btmtkuart.c b/drivers/bluetooth/btmtkuart.c
index f5dbeec8e274..e11169ad8247 100644
--- a/drivers/bluetooth/btmtkuart.c
+++ b/drivers/bluetooth/btmtkuart.c
@@ -115,10 +115,12 @@ struct btmtk_hci_wmt_params {
struct btmtkuart_dev {
struct hci_dev *hdev;
struct serdev_device *serdev;
- struct clk *clk;
+ struct clk *clk;
+ struct clk *osc;
struct regulator *vcc;
struct gpio_desc *reset;
+ struct gpio_desc *boot;
struct pinctrl *pinctrl;
struct pinctrl_state *pins_runtime;
struct pinctrl_state *pins_boot;
@@ -911,6 +913,19 @@ static int btmtkuart_parse_dt(struct serdev_device *serdev)
return err;
}
+ bdev->osc = devm_clk_get_optional(&serdev->dev, "osc");
+ if (IS_ERR(bdev->osc)) {
+ err = PTR_ERR(bdev->osc);
+ return err;
+ }
+
+ bdev->boot = devm_gpiod_get_optional(&serdev->dev, "boot",
+ GPIOD_OUT_LOW);
+ if (IS_ERR(bdev->boot)) {
+ err = PTR_ERR(bdev->boot);
+ return err;
+ }
+
bdev->pinctrl = devm_pinctrl_get(&serdev->dev);
if (IS_ERR(bdev->pinctrl)) {
err = PTR_ERR(bdev->pinctrl);
@@ -919,8 +934,10 @@ static int btmtkuart_parse_dt(struct serdev_device *serdev)
bdev->pins_boot = pinctrl_lookup_state(bdev->pinctrl,
"default");
- if (IS_ERR(bdev->pins_boot)) {
+ if (IS_ERR(bdev->pins_boot) && !bdev->boot) {
err = PTR_ERR(bdev->pins_boot);
+ dev_err(&serdev->dev,
+ "Should assign RXD to LOW at boot stage\n");
return err;
}
@@ -996,13 +1013,25 @@ static int btmtkuart_probe(struct serdev_device *serdev)
set_bit(HCI_QUIRK_NON_PERSISTENT_SETUP, &hdev->quirks);
if (btmtkuart_is_standalone(bdev)) {
- /* Switch to the specific pin state for the booting requires */
- pinctrl_select_state(bdev->pinctrl, bdev->pins_boot);
+ err = clk_prepare_enable(bdev->osc);
+ if (err < 0)
+ return err;
+
+ if (bdev->boot) {
+ gpiod_set_value_cansleep(bdev->boot, 1);
+ } else {
+ /* Switch to the specific pin state for the booting
+ * requires.
+ */
+ pinctrl_select_state(bdev->pinctrl, bdev->pins_boot);
+ }
/* Power on */
err = regulator_enable(bdev->vcc);
- if (err < 0)
+ if (err < 0) {
+ clk_disable_unprepare(bdev->osc);
return err;
+ }
/* Reset if the reset-gpios is available otherwise the board
* -level design should be guaranteed.
@@ -1017,6 +1046,10 @@ static int btmtkuart_probe(struct serdev_device *serdev)
* mode the device requires for UART transfers.
*/
msleep(50);
+
+ if (bdev->boot)
+ devm_gpiod_put(&serdev->dev, bdev->boot);
+
pinctrl_select_state(bdev->pinctrl, bdev->pins_runtime);
/* A standalone device doesn't depends on power domain on SoC,
@@ -1037,10 +1070,8 @@ static int btmtkuart_probe(struct serdev_device *serdev)
return 0;
err_regulator_disable:
- if (btmtkuart_is_standalone(bdev)) {
- pinctrl_select_state(bdev->pinctrl, bdev->pins_boot);
+ if (btmtkuart_is_standalone(bdev))
regulator_disable(bdev->vcc);
- }
return err;
}
@@ -1050,9 +1081,9 @@ static void btmtkuart_remove(struct serdev_device *serdev)
struct btmtkuart_dev *bdev = serdev_device_get_drvdata(serdev);
struct hci_dev *hdev = bdev->hdev;
- if (btmtkuart_is_standalone(bdev)) {
- pinctrl_select_state(bdev->pinctrl, bdev->pins_boot);
+ if (btmtkuart_is_standalone(bdev)) {
regulator_disable(bdev->vcc);
+ clk_disable_unprepare(bdev->osc);
}
hci_unregister_dev(hdev);
diff --git a/drivers/bluetooth/btqca.c b/drivers/bluetooth/btqca.c
index aff1d22223bd..8cc21ad7cf29 100644
--- a/drivers/bluetooth/btqca.c
+++ b/drivers/bluetooth/btqca.c
@@ -99,6 +99,28 @@ static int qca_send_reset(struct hci_dev *hdev)
return 0;
}
+int qca_send_pre_shutdown_cmd(struct hci_dev *hdev)
+{
+ struct sk_buff *skb;
+ int err;
+
+ bt_dev_dbg(hdev, "QCA pre shutdown cmd");
+
+ skb = __hci_cmd_sync_ev(hdev, QCA_PRE_SHUTDOWN_CMD, 0,
+ NULL, HCI_EV_CMD_COMPLETE, HCI_INIT_TIMEOUT);
+
+ if (IS_ERR(skb)) {
+ err = PTR_ERR(skb);
+ bt_dev_err(hdev, "QCA preshutdown_cmd failed (%d)", err);
+ return err;
+ }
+
+ kfree_skb(skb);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(qca_send_pre_shutdown_cmd);
+
static void qca_tlv_check_data(struct rome_config *config,
const struct firmware *fw)
{
@@ -119,6 +141,7 @@ static void qca_tlv_check_data(struct rome_config *config,
BT_DBG("Length\t\t : %d bytes", length);
config->dnld_mode = ROME_SKIP_EVT_NONE;
+ config->dnld_type = ROME_SKIP_EVT_NONE;
switch (config->type) {
case TLV_TYPE_PATCH:
@@ -131,6 +154,7 @@ static void qca_tlv_check_data(struct rome_config *config,
* In case VSE is skipped, only the last segment is acked.
*/
config->dnld_mode = tlv_patch->download_mode;
+ config->dnld_type = config->dnld_mode;
BT_DBG("Total Length : %d bytes",
le32_to_cpu(tlv_patch->total_size));
@@ -251,6 +275,31 @@ out:
return err;
}
+static int qca_inject_cmd_complete_event(struct hci_dev *hdev)
+{
+ struct hci_event_hdr *hdr;
+ struct hci_ev_cmd_complete *evt;
+ struct sk_buff *skb;
+
+ skb = bt_skb_alloc(sizeof(*hdr) + sizeof(*evt) + 1, GFP_KERNEL);
+ if (!skb)
+ return -ENOMEM;
+
+ hdr = skb_put(skb, sizeof(*hdr));
+ hdr->evt = HCI_EV_CMD_COMPLETE;
+ hdr->plen = sizeof(*evt) + 1;
+
+ evt = skb_put(skb, sizeof(*evt));
+ evt->ncmd = 1;
+ evt->opcode = cpu_to_le16(QCA_HCI_CC_OPCODE);
+
+ skb_put_u8(skb, QCA_HCI_CC_SUCCESS);
+
+ hci_skb_pkt_type(skb) = HCI_EVENT_PKT;
+
+ return hci_recv_frame(hdev, skb);
+}
+
static int qca_download_firmware(struct hci_dev *hdev,
struct rome_config *config)
{
@@ -284,11 +333,22 @@ static int qca_download_firmware(struct hci_dev *hdev,
ret = qca_tlv_send_segment(hdev, segsize, segment,
config->dnld_mode);
if (ret)
- break;
+ goto out;
segment += segsize;
}
+ /* Latest qualcomm chipsets are not sending a command complete event
+ * for every fw packet sent. They only respond with a vendor specific
+ * event for the last packet. This optimization in the chip will
+ * decrease the BT in initialization time. Here we will inject a command
+ * complete event to avoid a command timeout error message.
+ */
+ if (config->dnld_type == ROME_SKIP_EVT_VSE_CC ||
+ config->dnld_type == ROME_SKIP_EVT_VSE)
+ ret = qca_inject_cmd_complete_event(hdev);
+
+out:
release_firmware(fw);
return ret;
@@ -319,7 +379,8 @@ int qca_set_bdaddr_rome(struct hci_dev *hdev, const bdaddr_t *bdaddr)
EXPORT_SYMBOL_GPL(qca_set_bdaddr_rome);
int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate,
- enum qca_btsoc_type soc_type, u32 soc_ver)
+ enum qca_btsoc_type soc_type, u32 soc_ver,
+ const char *firmware_name)
{
struct rome_config config;
int err;
@@ -350,9 +411,15 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate,
return err;
}
+ /* Give the controller some time to get ready to receive the NVM */
+ msleep(10);
+
/* Download NVM configuration */
config.type = TLV_TYPE_NVM;
- if (qca_is_wcn399x(soc_type))
+ if (firmware_name)
+ snprintf(config.fwname, sizeof(config.fwname),
+ "qca/%s", firmware_name);
+ else if (qca_is_wcn399x(soc_type))
snprintf(config.fwname, sizeof(config.fwname),
"qca/crnv%02x.bin", rom_ver);
else
diff --git a/drivers/bluetooth/btqca.h b/drivers/bluetooth/btqca.h
index e9c999959603..69c5315a65fd 100644
--- a/drivers/bluetooth/btqca.h
+++ b/drivers/bluetooth/btqca.h
@@ -13,6 +13,7 @@
#define EDL_PATCH_TLV_REQ_CMD (0x1E)
#define EDL_NVM_ACCESS_SET_REQ_CMD (0x01)
#define MAX_SIZE_PER_TLV_SEGMENT (243)
+#define QCA_PRE_SHUTDOWN_CMD (0xFC08)
#define EDL_CMD_REQ_RES_EVT (0x00)
#define EDL_PATCH_VER_RES_EVT (0x19)
@@ -28,6 +29,9 @@
#define QCA_WCN3990_POWERON_PULSE 0xFC
#define QCA_WCN3990_POWEROFF_PULSE 0xC0
+#define QCA_HCI_CC_OPCODE 0xFC00
+#define QCA_HCI_CC_SUCCESS 0x00
+
enum qca_baudrate {
QCA_BAUDRATE_115200 = 0,
QCA_BAUDRATE_57600,
@@ -69,6 +73,7 @@ struct rome_config {
char fwname[64];
uint8_t user_baud_rate;
enum rome_tlv_dnld_mode dnld_mode;
+ enum rome_tlv_dnld_mode dnld_type;
};
struct edl_event_hdr {
@@ -127,9 +132,11 @@ enum qca_btsoc_type {
int qca_set_bdaddr_rome(struct hci_dev *hdev, const bdaddr_t *bdaddr);
int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate,
- enum qca_btsoc_type soc_type, u32 soc_ver);
+ enum qca_btsoc_type soc_type, u32 soc_ver,
+ const char *firmware_name);
int qca_read_soc_version(struct hci_dev *hdev, u32 *soc_version);
int qca_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr);
+int qca_send_pre_shutdown_cmd(struct hci_dev *hdev);
static inline bool qca_is_wcn399x(enum qca_btsoc_type soc_type)
{
return soc_type == QCA_WCN3990 || soc_type == QCA_WCN3998;
@@ -142,7 +149,8 @@ static inline int qca_set_bdaddr_rome(struct hci_dev *hdev, const bdaddr_t *bdad
}
static inline int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate,
- enum qca_btsoc_type soc_type, u32 soc_ver)
+ enum qca_btsoc_type soc_type, u32 soc_ver,
+ const char *firmware_name)
{
return -EOPNOTSUPP;
}
@@ -161,4 +169,9 @@ static inline bool qca_is_wcn399x(enum qca_btsoc_type soc_type)
{
return false;
}
+
+static inline int qca_send_pre_shutdown_cmd(struct hci_dev *hdev)
+{
+ return -EOPNOTSUPP;
+}
#endif
diff --git a/drivers/bluetooth/btrtl.c b/drivers/bluetooth/btrtl.c
index 208feef63de4..bf3c02be6930 100644
--- a/drivers/bluetooth/btrtl.c
+++ b/drivers/bluetooth/btrtl.c
@@ -21,6 +21,7 @@
#define RTL_ROM_LMP_3499 0x3499
#define RTL_ROM_LMP_8723A 0x1200
#define RTL_ROM_LMP_8723B 0x8723
+#define RTL_ROM_LMP_8723D 0x8873
#define RTL_ROM_LMP_8821A 0x8821
#define RTL_ROM_LMP_8761A 0x8761
#define RTL_ROM_LMP_8822B 0x8822
@@ -107,6 +108,13 @@ static const struct id_table ic_id_table[] = {
.fw_name = "rtl_bt/rtl8723ds_fw.bin",
.cfg_name = "rtl_bt/rtl8723ds_config" },
+ /* 8723DU */
+ { IC_INFO(RTL_ROM_LMP_8723D, 0x826C),
+ .config_needed = true,
+ .has_rom_version = true,
+ .fw_name = "rtl_bt/rtl8723d_fw.bin",
+ .cfg_name = "rtl_bt/rtl8723d_config" },
+
/* 8821A */
{ IC_INFO(RTL_ROM_LMP_8821A, 0xa),
.config_needed = false,
@@ -170,6 +178,27 @@ static const struct id_table *btrtl_match_ic(u16 lmp_subver, u16 hci_rev,
return &ic_id_table[i];
}
+static struct sk_buff *btrtl_read_local_version(struct hci_dev *hdev)
+{
+ struct sk_buff *skb;
+
+ skb = __hci_cmd_sync(hdev, HCI_OP_READ_LOCAL_VERSION, 0, NULL,
+ HCI_INIT_TIMEOUT);
+ if (IS_ERR(skb)) {
+ rtl_dev_err(hdev, "HCI_OP_READ_LOCAL_VERSION failed (%ld)",
+ PTR_ERR(skb));
+ return skb;
+ }
+
+ if (skb->len != sizeof(struct hci_rp_read_local_version)) {
+ rtl_dev_err(hdev, "HCI_OP_READ_LOCAL_VERSION event length mismatch");
+ kfree_skb(skb);
+ return ERR_PTR(-EIO);
+ }
+
+ return skb;
+}
+
static int rtl_read_rom_version(struct hci_dev *hdev, u8 *version)
{
struct rtl_rom_version_evt *rom_version;
@@ -178,19 +207,19 @@ static int rtl_read_rom_version(struct hci_dev *hdev, u8 *version)
/* Read RTL ROM version command */
skb = __hci_cmd_sync(hdev, 0xfc6d, 0, NULL, HCI_INIT_TIMEOUT);
if (IS_ERR(skb)) {
- rtl_dev_err(hdev, "Read ROM version failed (%ld)\n",
+ rtl_dev_err(hdev, "Read ROM version failed (%ld)",
PTR_ERR(skb));
return PTR_ERR(skb);
}
if (skb->len != sizeof(*rom_version)) {
- rtl_dev_err(hdev, "RTL version event length mismatch\n");
+ rtl_dev_err(hdev, "version event length mismatch");
kfree_skb(skb);
return -EIO;
}
rom_version = (struct rtl_rom_version_evt *)skb->data;
- rtl_dev_info(hdev, "rom_version status=%x version=%x\n",
+ rtl_dev_info(hdev, "rom_version status=%x version=%x",
rom_version->status, rom_version->version);
*version = rom_version->version;
@@ -234,7 +263,7 @@ static int rtlbt_parse_firmware(struct hci_dev *hdev,
fwptr = btrtl_dev->fw_data + btrtl_dev->fw_len - sizeof(extension_sig);
if (memcmp(fwptr, extension_sig, sizeof(extension_sig)) != 0) {
- rtl_dev_err(hdev, "extension section signature mismatch\n");
+ rtl_dev_err(hdev, "extension section signature mismatch");
return -EINVAL;
}
@@ -255,7 +284,7 @@ static int rtlbt_parse_firmware(struct hci_dev *hdev,
break;
if (length == 0) {
- rtl_dev_err(hdev, "found instruction with length 0\n");
+ rtl_dev_err(hdev, "found instruction with length 0");
return -EINVAL;
}
@@ -268,7 +297,7 @@ static int rtlbt_parse_firmware(struct hci_dev *hdev,
}
if (project_id < 0) {
- rtl_dev_err(hdev, "failed to find version instruction\n");
+ rtl_dev_err(hdev, "failed to find version instruction");
return -EINVAL;
}
@@ -279,13 +308,13 @@ static int rtlbt_parse_firmware(struct hci_dev *hdev,
}
if (i >= ARRAY_SIZE(project_id_to_lmp_subver)) {
- rtl_dev_err(hdev, "unknown project id %d\n", project_id);
+ rtl_dev_err(hdev, "unknown project id %d", project_id);
return -EINVAL;
}
if (btrtl_dev->ic_info->lmp_subver !=
project_id_to_lmp_subver[i].lmp_subver) {
- rtl_dev_err(hdev, "firmware is for %x but this is a %x\n",
+ rtl_dev_err(hdev, "firmware is for %x but this is a %x",
project_id_to_lmp_subver[i].lmp_subver,
btrtl_dev->ic_info->lmp_subver);
return -EINVAL;
@@ -293,7 +322,7 @@ static int rtlbt_parse_firmware(struct hci_dev *hdev,
epatch_info = (struct rtl_epatch_header *)btrtl_dev->fw_data;
if (memcmp(epatch_info->signature, RTL_EPATCH_SIGNATURE, 8) != 0) {
- rtl_dev_err(hdev, "bad EPATCH signature\n");
+ rtl_dev_err(hdev, "bad EPATCH signature");
return -EINVAL;
}
@@ -360,6 +389,8 @@ static int rtl_download_firmware(struct hci_dev *hdev,
int frag_len = RTL_FRAG_LEN;
int ret = 0;
int i;
+ struct sk_buff *skb;
+ struct hci_rp_read_local_version *rp;
dl_cmd = kmalloc(sizeof(struct rtl_download_cmd), GFP_KERNEL);
if (!dl_cmd)
@@ -370,7 +401,11 @@ static int rtl_download_firmware(struct hci_dev *hdev,
BT_DBG("download fw (%d/%d)", i, frag_num);
- dl_cmd->index = i;
+ if (i > 0x7f)
+ dl_cmd->index = (i & 0x7f) + 1;
+ else
+ dl_cmd->index = i;
+
if (i == (frag_num - 1)) {
dl_cmd->index |= 0x80; /* data end */
frag_len = fw_len % RTL_FRAG_LEN;
@@ -381,14 +416,14 @@ static int rtl_download_firmware(struct hci_dev *hdev,
skb = __hci_cmd_sync(hdev, 0xfc20, frag_len + 1, dl_cmd,
HCI_INIT_TIMEOUT);
if (IS_ERR(skb)) {
- rtl_dev_err(hdev, "download fw command failed (%ld)\n",
+ rtl_dev_err(hdev, "download fw command failed (%ld)",
PTR_ERR(skb));
ret = -PTR_ERR(skb);
goto out;
}
if (skb->len != sizeof(struct rtl_download_response)) {
- rtl_dev_err(hdev, "download fw event length mismatch\n");
+ rtl_dev_err(hdev, "download fw event length mismatch");
kfree_skb(skb);
ret = -EIO;
goto out;
@@ -398,6 +433,18 @@ static int rtl_download_firmware(struct hci_dev *hdev,
data += RTL_FRAG_LEN;
}
+ skb = btrtl_read_local_version(hdev);
+ if (IS_ERR(skb)) {
+ ret = PTR_ERR(skb);
+ rtl_dev_err(hdev, "read local version failed");
+ goto out;
+ }
+
+ rp = (struct hci_rp_read_local_version *)skb->data;
+ rtl_dev_info(hdev, "fw version 0x%04x%04x",
+ __le16_to_cpu(rp->hci_rev), __le16_to_cpu(rp->lmp_subver));
+ kfree_skb(skb);
+
out:
kfree(dl_cmd);
return ret;
@@ -408,7 +455,7 @@ static int rtl_load_file(struct hci_dev *hdev, const char *name, u8 **buff)
const struct firmware *fw;
int ret;
- rtl_dev_info(hdev, "rtl: loading %s\n", name);
+ rtl_dev_info(hdev, "loading %s", name);
ret = request_firmware(&fw, name, &hdev->dev);
if (ret < 0)
return ret;
@@ -432,7 +479,7 @@ static int btrtl_setup_rtl8723a(struct hci_dev *hdev,
* (which is only for RTL8723B and newer).
*/
if (!memcmp(btrtl_dev->fw_data, RTL_EPATCH_SIGNATURE, 8)) {
- rtl_dev_err(hdev, "unexpected EPATCH signature!\n");
+ rtl_dev_err(hdev, "unexpected EPATCH signature!");
return -EINVAL;
}
@@ -467,7 +514,7 @@ static int btrtl_setup_rtl8723b(struct hci_dev *hdev,
fw_data = tbuff;
}
- rtl_dev_info(hdev, "cfg_sz %d, total sz %d\n", btrtl_dev->cfg_len, ret);
+ rtl_dev_info(hdev, "cfg_sz %d, total sz %d", btrtl_dev->cfg_len, ret);
ret = rtl_download_firmware(hdev, fw_data, ret);
@@ -476,27 +523,6 @@ out:
return ret;
}
-static struct sk_buff *btrtl_read_local_version(struct hci_dev *hdev)
-{
- struct sk_buff *skb;
-
- skb = __hci_cmd_sync(hdev, HCI_OP_READ_LOCAL_VERSION, 0, NULL,
- HCI_INIT_TIMEOUT);
- if (IS_ERR(skb)) {
- rtl_dev_err(hdev, "HCI_OP_READ_LOCAL_VERSION failed (%ld)\n",
- PTR_ERR(skb));
- return skb;
- }
-
- if (skb->len != sizeof(struct hci_rp_read_local_version)) {
- rtl_dev_err(hdev, "HCI_OP_READ_LOCAL_VERSION event length mismatch\n");
- kfree_skb(skb);
- return ERR_PTR(-EIO);
- }
-
- return skb;
-}
-
void btrtl_free(struct btrtl_device_info *btrtl_dev)
{
kfree(btrtl_dev->fw_data);
@@ -529,7 +555,7 @@ struct btrtl_device_info *btrtl_initialize(struct hci_dev *hdev,
}
resp = (struct hci_rp_read_local_version *)skb->data;
- rtl_dev_info(hdev, "rtl: examining hci_ver=%02x hci_rev=%04x lmp_ver=%02x lmp_subver=%04x\n",
+ rtl_dev_info(hdev, "examining hci_ver=%02x hci_rev=%04x lmp_ver=%02x lmp_subver=%04x",
resp->hci_ver, resp->hci_rev,
resp->lmp_ver, resp->lmp_subver);
@@ -542,7 +568,7 @@ struct btrtl_device_info *btrtl_initialize(struct hci_dev *hdev,
hdev->bus);
if (!btrtl_dev->ic_info) {
- rtl_dev_info(hdev, "rtl: unknown IC info, lmp subver %04x, hci rev %04x, hci ver %04x",
+ rtl_dev_info(hdev, "unknown IC info, lmp subver %04x, hci rev %04x, hci ver %04x",
lmp_subver, hci_rev, hci_ver);
return btrtl_dev;
}
@@ -556,7 +582,7 @@ struct btrtl_device_info *btrtl_initialize(struct hci_dev *hdev,
btrtl_dev->fw_len = rtl_load_file(hdev, btrtl_dev->ic_info->fw_name,
&btrtl_dev->fw_data);
if (btrtl_dev->fw_len < 0) {
- rtl_dev_err(hdev, "firmware file %s not found\n",
+ rtl_dev_err(hdev, "firmware file %s not found",
btrtl_dev->ic_info->fw_name);
ret = btrtl_dev->fw_len;
goto err_free;
@@ -574,7 +600,7 @@ struct btrtl_device_info *btrtl_initialize(struct hci_dev *hdev,
&btrtl_dev->cfg_data);
if (btrtl_dev->ic_info->config_needed &&
btrtl_dev->cfg_len <= 0) {
- rtl_dev_err(hdev, "mandatory config file %s not found\n",
+ rtl_dev_err(hdev, "mandatory config file %s not found",
btrtl_dev->ic_info->cfg_name);
ret = btrtl_dev->cfg_len;
goto err_free;
@@ -600,7 +626,7 @@ int btrtl_download_firmware(struct hci_dev *hdev,
* to a different value.
*/
if (!btrtl_dev->ic_info) {
- rtl_dev_info(hdev, "rtl: assuming no firmware upload needed\n");
+ rtl_dev_info(hdev, "assuming no firmware upload needed");
return 0;
}
@@ -614,7 +640,7 @@ int btrtl_download_firmware(struct hci_dev *hdev,
case RTL_ROM_LMP_8822B:
return btrtl_setup_rtl8723b(hdev, btrtl_dev);
default:
- rtl_dev_info(hdev, "rtl: assuming no firmware upload needed\n");
+ rtl_dev_info(hdev, "assuming no firmware upload needed");
return 0;
}
}
@@ -633,10 +659,35 @@ int btrtl_setup_realtek(struct hci_dev *hdev)
btrtl_free(btrtl_dev);
+ /* Enable controller to do both LE scan and BR/EDR inquiry
+ * simultaneously.
+ */
+ set_bit(HCI_QUIRK_SIMULTANEOUS_DISCOVERY, &hdev->quirks);
+
return ret;
}
EXPORT_SYMBOL_GPL(btrtl_setup_realtek);
+int btrtl_shutdown_realtek(struct hci_dev *hdev)
+{
+ struct sk_buff *skb;
+ int ret;
+
+ /* According to the vendor driver, BT must be reset on close to avoid
+ * firmware crash.
+ */
+ skb = __hci_cmd_sync(hdev, HCI_OP_RESET, 0, NULL, HCI_INIT_TIMEOUT);
+ if (IS_ERR(skb)) {
+ ret = PTR_ERR(skb);
+ bt_dev_err(hdev, "HCI reset during shutdown failed");
+ return ret;
+ }
+ kfree_skb(skb);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(btrtl_shutdown_realtek);
+
static unsigned int btrtl_convert_baudrate(u32 device_baudrate)
{
switch (device_baudrate) {
@@ -686,18 +737,18 @@ int btrtl_get_uart_settings(struct hci_dev *hdev,
total_data_len = btrtl_dev->cfg_len - sizeof(*config);
if (total_data_len <= 0) {
- rtl_dev_warn(hdev, "no config loaded\n");
+ rtl_dev_warn(hdev, "no config loaded");
return -EINVAL;
}
config = (struct rtl_vendor_config *)btrtl_dev->cfg_data;
if (le32_to_cpu(config->signature) != RTL_CONFIG_MAGIC) {
- rtl_dev_err(hdev, "invalid config magic\n");
+ rtl_dev_err(hdev, "invalid config magic");
return -EINVAL;
}
if (total_data_len < le16_to_cpu(config->total_len)) {
- rtl_dev_err(hdev, "config is too short\n");
+ rtl_dev_err(hdev, "config is too short");
return -EINVAL;
}
@@ -707,7 +758,7 @@ int btrtl_get_uart_settings(struct hci_dev *hdev,
switch (le16_to_cpu(entry->offset)) {
case 0xc:
if (entry->len < sizeof(*device_baudrate)) {
- rtl_dev_err(hdev, "invalid UART config entry\n");
+ rtl_dev_err(hdev, "invalid UART config entry");
return -EINVAL;
}
@@ -724,7 +775,7 @@ int btrtl_get_uart_settings(struct hci_dev *hdev,
break;
default:
- rtl_dev_dbg(hdev, "skipping config entry 0x%x (len %u)\n",
+ rtl_dev_dbg(hdev, "skipping config entry 0x%x (len %u)",
le16_to_cpu(entry->offset), entry->len);
break;
};
@@ -733,13 +784,13 @@ int btrtl_get_uart_settings(struct hci_dev *hdev,
}
if (!found) {
- rtl_dev_err(hdev, "no UART config entry found\n");
+ rtl_dev_err(hdev, "no UART config entry found");
return -ENOENT;
}
- rtl_dev_dbg(hdev, "device baudrate = 0x%08x\n", *device_baudrate);
- rtl_dev_dbg(hdev, "controller baudrate = %u\n", *controller_baudrate);
- rtl_dev_dbg(hdev, "flow control %d\n", *flow_control);
+ rtl_dev_dbg(hdev, "device baudrate = 0x%08x", *device_baudrate);
+ rtl_dev_dbg(hdev, "controller baudrate = %u", *controller_baudrate);
+ rtl_dev_dbg(hdev, "flow control %d", *flow_control);
return 0;
}
diff --git a/drivers/bluetooth/btrtl.h b/drivers/bluetooth/btrtl.h
index f1676144fce8..10ad40c3e42c 100644
--- a/drivers/bluetooth/btrtl.h
+++ b/drivers/bluetooth/btrtl.h
@@ -55,6 +55,7 @@ void btrtl_free(struct btrtl_device_info *btrtl_dev);
int btrtl_download_firmware(struct hci_dev *hdev,
struct btrtl_device_info *btrtl_dev);
int btrtl_setup_realtek(struct hci_dev *hdev);
+int btrtl_shutdown_realtek(struct hci_dev *hdev);
int btrtl_get_uart_settings(struct hci_dev *hdev,
struct btrtl_device_info *btrtl_dev,
unsigned int *controller_baudrate,
@@ -83,6 +84,11 @@ static inline int btrtl_setup_realtek(struct hci_dev *hdev)
return -EOPNOTSUPP;
}
+static inline int btrtl_shutdown_realtek(struct hci_dev *hdev)
+{
+ return -EOPNOTSUPP;
+}
+
static inline int btrtl_get_uart_settings(struct hci_dev *hdev,
struct btrtl_device_info *btrtl_dev,
unsigned int *controller_baudrate,
diff --git a/drivers/bluetooth/btsdio.c b/drivers/bluetooth/btsdio.c
index 83748b7b2033..fd9571d5fdac 100644
--- a/drivers/bluetooth/btsdio.c
+++ b/drivers/bluetooth/btsdio.c
@@ -286,6 +286,7 @@ static int btsdio_probe(struct sdio_func *func,
switch (func->device) {
case SDIO_DEVICE_ID_BROADCOM_43341:
case SDIO_DEVICE_ID_BROADCOM_43430:
+ case SDIO_DEVICE_ID_BROADCOM_4356:
return -ENODEV;
}
}
diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c
index 50aed5259c2b..a9c35ebb30f8 100644
--- a/drivers/bluetooth/btusb.c
+++ b/drivers/bluetooth/btusb.c
@@ -11,6 +11,7 @@
#include <linux/usb.h>
#include <linux/usb/quirks.h>
#include <linux/firmware.h>
+#include <linux/iopoll.h>
#include <linux/of_device.h>
#include <linux/of_irq.h>
#include <linux/suspend.h>
@@ -55,6 +56,7 @@ static struct usb_driver btusb_driver;
#define BTUSB_BCM2045 0x40000
#define BTUSB_IFNUM_2 0x80000
#define BTUSB_CW6622 0x100000
+#define BTUSB_MEDIATEK 0x200000
static const struct usb_device_id btusb_table[] = {
/* Generic Bluetooth USB device */
@@ -264,7 +266,9 @@ static const struct usb_device_id blacklist_table[] = {
{ USB_DEVICE(0x04ca, 0x3015), .driver_info = BTUSB_QCA_ROME },
{ USB_DEVICE(0x04ca, 0x3016), .driver_info = BTUSB_QCA_ROME },
{ USB_DEVICE(0x04ca, 0x301a), .driver_info = BTUSB_QCA_ROME },
+ { USB_DEVICE(0x13d3, 0x3491), .driver_info = BTUSB_QCA_ROME },
{ USB_DEVICE(0x13d3, 0x3496), .driver_info = BTUSB_QCA_ROME },
+ { USB_DEVICE(0x13d3, 0x3501), .driver_info = BTUSB_QCA_ROME },
/* Broadcom BCM2035 */
{ USB_DEVICE(0x0a5c, 0x2009), .driver_info = BTUSB_BCM92035 },
@@ -346,6 +350,10 @@ static const struct usb_device_id blacklist_table[] = {
{ USB_VENDOR_AND_INTERFACE_INFO(0x0bda, 0xe0, 0x01, 0x01),
.driver_info = BTUSB_REALTEK },
+ /* MediaTek Bluetooth devices */
+ { USB_VENDOR_AND_INTERFACE_INFO(0x0e8d, 0xe0, 0x01, 0x01),
+ .driver_info = BTUSB_MEDIATEK },
+
/* Additional Realtek 8723AE Bluetooth devices */
{ USB_DEVICE(0x0930, 0x021d), .driver_info = BTUSB_REALTEK },
{ USB_DEVICE(0x13d3, 0x3394), .driver_info = BTUSB_REALTEK },
@@ -376,6 +384,9 @@ static const struct usb_device_id blacklist_table[] = {
{ USB_DEVICE(0x13d3, 0x3526), .driver_info = BTUSB_REALTEK },
{ USB_DEVICE(0x0b05, 0x185c), .driver_info = BTUSB_REALTEK },
+ /* Additional Realtek 8822CE Bluetooth devices */
+ { USB_DEVICE(0x04ca, 0x4005), .driver_info = BTUSB_REALTEK },
+
/* Silicon Wave based devices */
{ USB_DEVICE(0x0c10, 0x0000), .driver_info = BTUSB_SWAVE },
@@ -426,6 +437,8 @@ static const struct dmi_system_id btusb_needs_reset_resume_table[] = {
#define BTUSB_DIAG_RUNNING 10
#define BTUSB_OOB_WAKE_ENABLED 11
#define BTUSB_HW_RESET_ACTIVE 12
+#define BTUSB_TX_WAIT_VND_EVT 13
+#define BTUSB_WAKEUP_DISABLE 14
struct btusb_data {
struct hci_dev *hdev;
@@ -449,6 +462,7 @@ struct btusb_data {
struct usb_anchor bulk_anchor;
struct usb_anchor isoc_anchor;
struct usb_anchor diag_anchor;
+ struct usb_anchor ctrl_anchor;
spinlock_t rxlock;
struct sk_buff *evt_skb;
@@ -513,6 +527,36 @@ static void btusb_intel_cmd_timeout(struct hci_dev *hdev)
gpiod_set_value_cansleep(reset_gpio, 0);
}
+static void btusb_rtl_cmd_timeout(struct hci_dev *hdev)
+{
+ struct btusb_data *data = hci_get_drvdata(hdev);
+ struct gpio_desc *reset_gpio = data->reset_gpio;
+
+ if (++data->cmd_timeout_cnt < 5)
+ return;
+
+ if (!reset_gpio) {
+ bt_dev_err(hdev, "No gpio to reset Realtek device, ignoring");
+ return;
+ }
+
+ /* Toggle the hard reset line. The Realtek device is going to
+ * yank itself off the USB and then replug. The cleanup is handled
+ * correctly on the way out (standard USB disconnect), and the new
+ * device is detected cleanly and bound to the driver again like
+ * it should be.
+ */
+ if (test_and_set_bit(BTUSB_HW_RESET_ACTIVE, &data->flags)) {
+ bt_dev_err(hdev, "last reset failed? Not resetting again");
+ return;
+ }
+
+ bt_dev_err(hdev, "Reset Realtek device via gpio");
+ gpiod_set_value_cansleep(reset_gpio, 0);
+ msleep(200);
+ gpiod_set_value_cansleep(reset_gpio, 1);
+}
+
static inline void btusb_free_frags(struct btusb_data *data)
{
unsigned long flags;
@@ -1160,10 +1204,13 @@ static int btusb_open(struct hci_dev *hdev)
}
data->intf->needs_remote_wakeup = 1;
- /* device specific wakeup source enabled and required for USB
- * remote wakeup while host is suspended
+
+ /* Disable device remote wakeup when host is suspended
+ * For Realtek chips, global suspend without
+ * SET_FEATURE (DEVICE_REMOTE_WAKEUP) can save more power in device.
*/
- device_wakeup_enable(&data->udev->dev);
+ if (test_bit(BTUSB_WAKEUP_DISABLE, &data->flags))
+ device_wakeup_disable(&data->udev->dev);
if (test_and_set_bit(BTUSB_INTR_RUNNING, &data->flags))
goto done;
@@ -1202,6 +1249,7 @@ static void btusb_stop_traffic(struct btusb_data *data)
usb_kill_anchored_urbs(&data->bulk_anchor);
usb_kill_anchored_urbs(&data->isoc_anchor);
usb_kill_anchored_urbs(&data->diag_anchor);
+ usb_kill_anchored_urbs(&data->ctrl_anchor);
}
static int btusb_close(struct hci_dev *hdev)
@@ -1227,7 +1275,11 @@ static int btusb_close(struct hci_dev *hdev)
goto failed;
data->intf->needs_remote_wakeup = 0;
- device_wakeup_disable(&data->udev->dev);
+
+ /* Enable remote wake up for auto-suspend */
+ if (test_bit(BTUSB_WAKEUP_DISABLE, &data->flags))
+ data->intf->needs_remote_wakeup = 1;
+
usb_autopm_put_interface(data->intf);
failed:
@@ -2437,6 +2489,570 @@ static int btusb_shutdown_intel_new(struct hci_dev *hdev)
return 0;
}
+#ifdef CONFIG_BT_HCIBTUSB_MTK
+
+#define FIRMWARE_MT7663 "mediatek/mt7663pr2h.bin"
+#define FIRMWARE_MT7668 "mediatek/mt7668pr2h.bin"
+
+#define HCI_WMT_MAX_EVENT_SIZE 64
+
+enum {
+ BTMTK_WMT_PATCH_DWNLD = 0x1,
+ BTMTK_WMT_FUNC_CTRL = 0x6,
+ BTMTK_WMT_RST = 0x7,
+ BTMTK_WMT_SEMAPHORE = 0x17,
+};
+
+enum {
+ BTMTK_WMT_INVALID,
+ BTMTK_WMT_PATCH_UNDONE,
+ BTMTK_WMT_PATCH_DONE,
+ BTMTK_WMT_ON_UNDONE,
+ BTMTK_WMT_ON_DONE,
+ BTMTK_WMT_ON_PROGRESS,
+};
+
+struct btmtk_wmt_hdr {
+ u8 dir;
+ u8 op;
+ __le16 dlen;
+ u8 flag;
+} __packed;
+
+struct btmtk_hci_wmt_cmd {
+ struct btmtk_wmt_hdr hdr;
+ u8 data[256];
+} __packed;
+
+struct btmtk_hci_wmt_evt {
+ struct hci_event_hdr hhdr;
+ struct btmtk_wmt_hdr whdr;
+} __packed;
+
+struct btmtk_hci_wmt_evt_funcc {
+ struct btmtk_hci_wmt_evt hwhdr;
+ __be16 status;
+} __packed;
+
+struct btmtk_tci_sleep {
+ u8 mode;
+ __le16 duration;
+ __le16 host_duration;
+ u8 host_wakeup_pin;
+ u8 time_compensation;
+} __packed;
+
+struct btmtk_hci_wmt_params {
+ u8 op;
+ u8 flag;
+ u16 dlen;
+ const void *data;
+ u32 *status;
+};
+
+static void btusb_mtk_wmt_recv(struct urb *urb)
+{
+ struct hci_dev *hdev = urb->context;
+ struct btusb_data *data = hci_get_drvdata(hdev);
+ struct hci_event_hdr *hdr;
+ struct sk_buff *skb;
+ int err;
+
+ if (urb->status == 0 && urb->actual_length > 0) {
+ hdev->stat.byte_rx += urb->actual_length;
+
+ /* WMT event shouldn't be fragmented and the size should be
+ * less than HCI_WMT_MAX_EVENT_SIZE.
+ */
+ skb = bt_skb_alloc(HCI_WMT_MAX_EVENT_SIZE, GFP_ATOMIC);
+ if (!skb) {
+ hdev->stat.err_rx++;
+ goto err_out;
+ }
+
+ hci_skb_pkt_type(skb) = HCI_EVENT_PKT;
+ skb_put_data(skb, urb->transfer_buffer, urb->actual_length);
+
+ hdr = (void *)skb->data;
+ /* Fix up the vendor event id with 0xff for vendor specific
+ * instead of 0xe4 so that event send via monitoring socket can
+ * be parsed properly.
+ */
+ hdr->evt = 0xff;
+
+ /* When someone waits for the WMT event, the skb is being cloned
+ * and being processed the events from there then.
+ */
+ if (test_bit(BTUSB_TX_WAIT_VND_EVT, &data->flags)) {
+ data->evt_skb = skb_clone(skb, GFP_KERNEL);
+ if (!data->evt_skb)
+ goto err_out;
+ }
+
+ err = hci_recv_frame(hdev, skb);
+ if (err < 0)
+ goto err_free_skb;
+
+ if (test_and_clear_bit(BTUSB_TX_WAIT_VND_EVT,
+ &data->flags)) {
+ /* Barrier to sync with other CPUs */
+ smp_mb__after_atomic();
+ wake_up_bit(&data->flags,
+ BTUSB_TX_WAIT_VND_EVT);
+ }
+err_out:
+ return;
+err_free_skb:
+ kfree_skb(data->evt_skb);
+ data->evt_skb = NULL;
+ return;
+ } else if (urb->status == -ENOENT) {
+ /* Avoid suspend failed when usb_kill_urb */
+ return;
+ }
+
+ usb_mark_last_busy(data->udev);
+
+ /* The URB complete handler is still called with urb->actual_length = 0
+ * when the event is not available, so we should keep re-submitting
+ * URB until WMT event returns, Also, It's necessary to wait some time
+ * between the two consecutive control URBs to relax the target device
+ * to generate the event. Otherwise, the WMT event cannot return from
+ * the device successfully.
+ */
+ udelay(100);
+
+ usb_anchor_urb(urb, &data->ctrl_anchor);
+ err = usb_submit_urb(urb, GFP_ATOMIC);
+ if (err < 0) {
+ /* -EPERM: urb is being killed;
+ * -ENODEV: device got disconnected
+ */
+ if (err != -EPERM && err != -ENODEV)
+ bt_dev_err(hdev, "urb %p failed to resubmit (%d)",
+ urb, -err);
+ usb_unanchor_urb(urb);
+ }
+}
+
+static int btusb_mtk_submit_wmt_recv_urb(struct hci_dev *hdev)
+{
+ struct btusb_data *data = hci_get_drvdata(hdev);
+ struct usb_ctrlrequest *dr;
+ unsigned char *buf;
+ int err, size = 64;
+ unsigned int pipe;
+ struct urb *urb;
+
+ urb = usb_alloc_urb(0, GFP_KERNEL);
+ if (!urb)
+ return -ENOMEM;
+
+ dr = kmalloc(sizeof(*dr), GFP_KERNEL);
+ if (!dr) {
+ usb_free_urb(urb);
+ return -ENOMEM;
+ }
+
+ dr->bRequestType = USB_TYPE_VENDOR | USB_DIR_IN;
+ dr->bRequest = 1;
+ dr->wIndex = cpu_to_le16(0);
+ dr->wValue = cpu_to_le16(48);
+ dr->wLength = cpu_to_le16(size);
+
+ buf = kmalloc(size, GFP_KERNEL);
+ if (!buf) {
+ kfree(dr);
+ return -ENOMEM;
+ }
+
+ pipe = usb_rcvctrlpipe(data->udev, 0);
+
+ usb_fill_control_urb(urb, data->udev, pipe, (void *)dr,
+ buf, size, btusb_mtk_wmt_recv, hdev);
+
+ urb->transfer_flags |= URB_FREE_BUFFER;
+
+ usb_anchor_urb(urb, &data->ctrl_anchor);
+ err = usb_submit_urb(urb, GFP_KERNEL);
+ if (err < 0) {
+ if (err != -EPERM && err != -ENODEV)
+ bt_dev_err(hdev, "urb %p submission failed (%d)",
+ urb, -err);
+ usb_unanchor_urb(urb);
+ }
+
+ usb_free_urb(urb);
+
+ return err;
+}
+
+static int btusb_mtk_hci_wmt_sync(struct hci_dev *hdev,
+ struct btmtk_hci_wmt_params *wmt_params)
+{
+ struct btusb_data *data = hci_get_drvdata(hdev);
+ struct btmtk_hci_wmt_evt_funcc *wmt_evt_funcc;
+ u32 hlen, status = BTMTK_WMT_INVALID;
+ struct btmtk_hci_wmt_evt *wmt_evt;
+ struct btmtk_hci_wmt_cmd wc;
+ struct btmtk_wmt_hdr *hdr;
+ int err;
+
+ /* Submit control IN URB on demand to process the WMT event */
+ err = btusb_mtk_submit_wmt_recv_urb(hdev);
+ if (err < 0)
+ return err;
+
+ /* Send the WMT command and wait until the WMT event returns */
+ hlen = sizeof(*hdr) + wmt_params->dlen;
+ if (hlen > 255)
+ return -EINVAL;
+
+ hdr = (struct btmtk_wmt_hdr *)&wc;
+ hdr->dir = 1;
+ hdr->op = wmt_params->op;
+ hdr->dlen = cpu_to_le16(wmt_params->dlen + 1);
+ hdr->flag = wmt_params->flag;
+ memcpy(wc.data, wmt_params->data, wmt_params->dlen);
+
+ set_bit(BTUSB_TX_WAIT_VND_EVT, &data->flags);
+
+ err = __hci_cmd_send(hdev, 0xfc6f, hlen, &wc);
+
+ if (err < 0) {
+ clear_bit(BTUSB_TX_WAIT_VND_EVT, &data->flags);
+ return err;
+ }
+
+ /* The vendor specific WMT commands are all answered by a vendor
+ * specific event and will have the Command Status or Command
+ * Complete as with usual HCI command flow control.
+ *
+ * After sending the command, wait for BTUSB_TX_WAIT_VND_EVT
+ * state to be cleared. The driver specific event receive routine
+ * will clear that state and with that indicate completion of the
+ * WMT command.
+ */
+ err = wait_on_bit_timeout(&data->flags, BTUSB_TX_WAIT_VND_EVT,
+ TASK_INTERRUPTIBLE, HCI_INIT_TIMEOUT);
+ if (err == -EINTR) {
+ bt_dev_err(hdev, "Execution of wmt command interrupted");
+ clear_bit(BTUSB_TX_WAIT_VND_EVT, &data->flags);
+ return err;
+ }
+
+ if (err) {
+ bt_dev_err(hdev, "Execution of wmt command timed out");
+ clear_bit(BTUSB_TX_WAIT_VND_EVT, &data->flags);
+ return -ETIMEDOUT;
+ }
+
+ /* Parse and handle the return WMT event */
+ wmt_evt = (struct btmtk_hci_wmt_evt *)data->evt_skb->data;
+ if (wmt_evt->whdr.op != hdr->op) {
+ bt_dev_err(hdev, "Wrong op received %d expected %d",
+ wmt_evt->whdr.op, hdr->op);
+ err = -EIO;
+ goto err_free_skb;
+ }
+
+ switch (wmt_evt->whdr.op) {
+ case BTMTK_WMT_SEMAPHORE:
+ if (wmt_evt->whdr.flag == 2)
+ status = BTMTK_WMT_PATCH_UNDONE;
+ else
+ status = BTMTK_WMT_PATCH_DONE;
+ break;
+ case BTMTK_WMT_FUNC_CTRL:
+ wmt_evt_funcc = (struct btmtk_hci_wmt_evt_funcc *)wmt_evt;
+ if (be16_to_cpu(wmt_evt_funcc->status) == 0x404)
+ status = BTMTK_WMT_ON_DONE;
+ else if (be16_to_cpu(wmt_evt_funcc->status) == 0x420)
+ status = BTMTK_WMT_ON_PROGRESS;
+ else
+ status = BTMTK_WMT_ON_UNDONE;
+ break;
+ }
+
+ if (wmt_params->status)
+ *wmt_params->status = status;
+
+err_free_skb:
+ kfree_skb(data->evt_skb);
+ data->evt_skb = NULL;
+
+ return err;
+}
+
+static int btusb_mtk_setup_firmware(struct hci_dev *hdev, const char *fwname)
+{
+ struct btmtk_hci_wmt_params wmt_params;
+ const struct firmware *fw;
+ const u8 *fw_ptr;
+ size_t fw_size;
+ int err, dlen;
+ u8 flag;
+
+ err = request_firmware(&fw, fwname, &hdev->dev);
+ if (err < 0) {
+ bt_dev_err(hdev, "Failed to load firmware file (%d)", err);
+ return err;
+ }
+
+ fw_ptr = fw->data;
+ fw_size = fw->size;
+
+ /* The size of patch header is 30 bytes, should be skip */
+ if (fw_size < 30) {
+ err = -EINVAL;
+ goto err_release_fw;
+ }
+
+ fw_size -= 30;
+ fw_ptr += 30;
+ flag = 1;
+
+ wmt_params.op = BTMTK_WMT_PATCH_DWNLD;
+ wmt_params.status = NULL;
+
+ while (fw_size > 0) {
+ dlen = min_t(int, 250, fw_size);
+
+ /* Tell deivice the position in sequence */
+ if (fw_size - dlen <= 0)
+ flag = 3;
+ else if (fw_size < fw->size - 30)
+ flag = 2;
+
+ wmt_params.flag = flag;
+ wmt_params.dlen = dlen;
+ wmt_params.data = fw_ptr;
+
+ err = btusb_mtk_hci_wmt_sync(hdev, &wmt_params);
+ if (err < 0) {
+ bt_dev_err(hdev, "Failed to send wmt patch dwnld (%d)",
+ err);
+ goto err_release_fw;
+ }
+
+ fw_size -= dlen;
+ fw_ptr += dlen;
+ }
+
+ wmt_params.op = BTMTK_WMT_RST;
+ wmt_params.flag = 4;
+ wmt_params.dlen = 0;
+ wmt_params.data = NULL;
+ wmt_params.status = NULL;
+
+ /* Activate funciton the firmware providing to */
+ err = btusb_mtk_hci_wmt_sync(hdev, &wmt_params);
+ if (err < 0) {
+ bt_dev_err(hdev, "Failed to send wmt rst (%d)", err);
+ return err;
+ }
+
+ /* Wait a few moments for firmware activation done */
+ usleep_range(10000, 12000);
+
+err_release_fw:
+ release_firmware(fw);
+
+ return err;
+}
+
+static int btusb_mtk_func_query(struct hci_dev *hdev)
+{
+ struct btmtk_hci_wmt_params wmt_params;
+ int status, err;
+ u8 param = 0;
+
+ /* Query whether the function is enabled */
+ wmt_params.op = BTMTK_WMT_FUNC_CTRL;
+ wmt_params.flag = 4;
+ wmt_params.dlen = sizeof(param);
+ wmt_params.data = &param;
+ wmt_params.status = &status;
+
+ err = btusb_mtk_hci_wmt_sync(hdev, &wmt_params);
+ if (err < 0) {
+ bt_dev_err(hdev, "Failed to query function status (%d)", err);
+ return err;
+ }
+
+ return status;
+}
+
+static int btusb_mtk_reg_read(struct btusb_data *data, u32 reg, u32 *val)
+{
+ int pipe, err, size = sizeof(u32);
+ void *buf;
+
+ buf = kzalloc(size, GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
+
+ pipe = usb_rcvctrlpipe(data->udev, 0);
+ err = usb_control_msg(data->udev, pipe, 0x63,
+ USB_TYPE_VENDOR | USB_DIR_IN,
+ reg >> 16, reg & 0xffff,
+ buf, size, USB_CTRL_SET_TIMEOUT);
+ if (err < 0)
+ goto err_free_buf;
+
+ *val = get_unaligned_le32(buf);
+
+err_free_buf:
+ kfree(buf);
+
+ return err;
+}
+
+static int btusb_mtk_id_get(struct btusb_data *data, u32 *id)
+{
+ return btusb_mtk_reg_read(data, 0x80000008, id);
+}
+
+static int btusb_mtk_setup(struct hci_dev *hdev)
+{
+ struct btusb_data *data = hci_get_drvdata(hdev);
+ struct btmtk_hci_wmt_params wmt_params;
+ ktime_t calltime, delta, rettime;
+ struct btmtk_tci_sleep tci_sleep;
+ unsigned long long duration;
+ struct sk_buff *skb;
+ const char *fwname;
+ int err, status;
+ u32 dev_id;
+ u8 param;
+
+ calltime = ktime_get();
+
+ err = btusb_mtk_id_get(data, &dev_id);
+ if (err < 0) {
+ bt_dev_err(hdev, "Failed to get device id (%d)", err);
+ return err;
+ }
+
+ switch (dev_id) {
+ case 0x7663:
+ fwname = FIRMWARE_MT7663;
+ break;
+ case 0x7668:
+ fwname = FIRMWARE_MT7668;
+ break;
+ default:
+ bt_dev_err(hdev, "Unsupported support hardware variant (%08x)",
+ dev_id);
+ return -ENODEV;
+ }
+
+ /* Query whether the firmware is already download */
+ wmt_params.op = BTMTK_WMT_SEMAPHORE;
+ wmt_params.flag = 1;
+ wmt_params.dlen = 0;
+ wmt_params.data = NULL;
+ wmt_params.status = &status;
+
+ err = btusb_mtk_hci_wmt_sync(hdev, &wmt_params);
+ if (err < 0) {
+ bt_dev_err(hdev, "Failed to query firmware status (%d)", err);
+ return err;
+ }
+
+ if (status == BTMTK_WMT_PATCH_DONE) {
+ bt_dev_info(hdev, "firmware already downloaded");
+ goto ignore_setup_fw;
+ }
+
+ /* Setup a firmware which the device definitely requires */
+ err = btusb_mtk_setup_firmware(hdev, fwname);
+ if (err < 0)
+ return err;
+
+ignore_setup_fw:
+ err = readx_poll_timeout(btusb_mtk_func_query, hdev, status,
+ status < 0 || status != BTMTK_WMT_ON_PROGRESS,
+ 2000, 5000000);
+ /* -ETIMEDOUT happens */
+ if (err < 0)
+ return err;
+
+ /* The other errors happen in btusb_mtk_func_query */
+ if (status < 0)
+ return status;
+
+ if (status == BTMTK_WMT_ON_DONE) {
+ bt_dev_info(hdev, "function already on");
+ goto ignore_func_on;
+ }
+
+ /* Enable Bluetooth protocol */
+ param = 1;
+ wmt_params.op = BTMTK_WMT_FUNC_CTRL;
+ wmt_params.flag = 0;
+ wmt_params.dlen = sizeof(param);
+ wmt_params.data = &param;
+ wmt_params.status = NULL;
+
+ err = btusb_mtk_hci_wmt_sync(hdev, &wmt_params);
+ if (err < 0) {
+ bt_dev_err(hdev, "Failed to send wmt func ctrl (%d)", err);
+ return err;
+ }
+
+ignore_func_on:
+ /* Apply the low power environment setup */
+ tci_sleep.mode = 0x5;
+ tci_sleep.duration = cpu_to_le16(0x640);
+ tci_sleep.host_duration = cpu_to_le16(0x640);
+ tci_sleep.host_wakeup_pin = 0;
+ tci_sleep.time_compensation = 0;
+
+ skb = __hci_cmd_sync(hdev, 0xfc7a, sizeof(tci_sleep), &tci_sleep,
+ HCI_INIT_TIMEOUT);
+ if (IS_ERR(skb)) {
+ err = PTR_ERR(skb);
+ bt_dev_err(hdev, "Failed to apply low power setting (%d)", err);
+ return err;
+ }
+ kfree_skb(skb);
+
+ rettime = ktime_get();
+ delta = ktime_sub(rettime, calltime);
+ duration = (unsigned long long)ktime_to_ns(delta) >> 10;
+
+ bt_dev_info(hdev, "Device setup in %llu usecs", duration);
+
+ return 0;
+}
+
+static int btusb_mtk_shutdown(struct hci_dev *hdev)
+{
+ struct btmtk_hci_wmt_params wmt_params;
+ u8 param = 0;
+ int err;
+
+ /* Disable the device */
+ wmt_params.op = BTMTK_WMT_FUNC_CTRL;
+ wmt_params.flag = 0;
+ wmt_params.dlen = sizeof(param);
+ wmt_params.data = &param;
+ wmt_params.status = NULL;
+
+ err = btusb_mtk_hci_wmt_sync(hdev, &wmt_params);
+ if (err < 0) {
+ bt_dev_err(hdev, "Failed to send wmt func ctrl (%d)", err);
+ return err;
+ }
+
+ return 0;
+}
+
+MODULE_FIRMWARE(FIRMWARE_MT7663);
+MODULE_FIRMWARE(FIRMWARE_MT7668);
+#endif
+
#ifdef CONFIG_PM
/* Configure an out-of-band gpio as wake-up pin, if specified in device tree */
static int marvell_config_oob_wake(struct hci_dev *hdev)
@@ -3044,6 +3660,7 @@ static int btusb_probe(struct usb_interface *intf,
init_usb_anchor(&data->bulk_anchor);
init_usb_anchor(&data->isoc_anchor);
init_usb_anchor(&data->diag_anchor);
+ init_usb_anchor(&data->ctrl_anchor);
spin_lock_init(&data->rxlock);
if (id->driver_info & BTUSB_INTEL_NEW) {
@@ -3157,6 +3774,15 @@ static int btusb_probe(struct usb_interface *intf,
if (id->driver_info & BTUSB_MARVELL)
hdev->set_bdaddr = btusb_set_bdaddr_marvell;
+#ifdef CONFIG_BT_HCIBTUSB_MTK
+ if (id->driver_info & BTUSB_MEDIATEK) {
+ hdev->setup = btusb_mtk_setup;
+ hdev->shutdown = btusb_mtk_shutdown;
+ hdev->manufacturer = 70;
+ set_bit(HCI_QUIRK_NON_PERSISTENT_SETUP, &hdev->quirks);
+ }
+#endif
+
if (id->driver_info & BTUSB_SWAVE) {
set_bit(HCI_QUIRK_FIXUP_INQUIRY_MODE, &hdev->quirks);
set_bit(HCI_QUIRK_BROKEN_LOCAL_COMMANDS, &hdev->quirks);
@@ -3184,12 +3810,14 @@ static int btusb_probe(struct usb_interface *intf,
#ifdef CONFIG_BT_HCIBTUSB_RTL
if (id->driver_info & BTUSB_REALTEK) {
hdev->setup = btrtl_setup_realtek;
+ hdev->shutdown = btrtl_shutdown_realtek;
+ hdev->cmd_timeout = btusb_rtl_cmd_timeout;
- /* Realtek devices lose their updated firmware over suspend,
- * but the USB hub doesn't notice any status change.
- * Explicitly request a device reset on resume.
+ /* Realtek devices lose their updated firmware over global
+ * suspend that means host doesn't send SET_FEATURE
+ * (DEVICE_REMOTE_WAKEUP)
*/
- interface_to_usbdev(intf)->quirks |= USB_QUIRK_RESET_RESUME;
+ set_bit(BTUSB_WAKEUP_DISABLE, &data->flags);
}
#endif
@@ -3363,6 +3991,19 @@ static int btusb_suspend(struct usb_interface *intf, pm_message_t message)
enable_irq(data->oob_wake_irq);
}
+ /* For global suspend, Realtek devices lose the loaded fw
+ * in them. But for autosuspend, firmware should remain.
+ * Actually, it depends on whether the usb host sends
+ * set feature (enable wakeup) or not.
+ */
+ if (test_bit(BTUSB_WAKEUP_DISABLE, &data->flags)) {
+ if (PMSG_IS_AUTO(message) &&
+ device_can_wakeup(&data->udev->dev))
+ data->udev->do_remote_wakeup = 1;
+ else if (!PMSG_IS_AUTO(message))
+ data->udev->reset_resume = 1;
+ }
+
return 0;
}
diff --git a/drivers/bluetooth/hci_ath.c b/drivers/bluetooth/hci_ath.c
index a55be205b91a..dbfe34664633 100644
--- a/drivers/bluetooth/hci_ath.c
+++ b/drivers/bluetooth/hci_ath.c
@@ -98,6 +98,9 @@ static int ath_open(struct hci_uart *hu)
BT_DBG("hu %p", hu);
+ if (!hci_uart_has_flow_control(hu))
+ return -EOPNOTSUPP;
+
ath = kzalloc(sizeof(*ath), GFP_KERNEL);
if (!ath)
return -ENOMEM;
diff --git a/drivers/bluetooth/hci_bcm.c b/drivers/bluetooth/hci_bcm.c
index 8905ad2edde7..7646636f2d18 100644
--- a/drivers/bluetooth/hci_bcm.c
+++ b/drivers/bluetooth/hci_bcm.c
@@ -260,7 +260,7 @@ static int bcm_gpio_set_power(struct bcm_device *dev, bool powered)
}
/* wait for device to power on and come out of reset */
- usleep_range(10000, 20000);
+ usleep_range(100000, 120000);
dev->res_enabled = powered;
@@ -406,6 +406,9 @@ static int bcm_open(struct hci_uart *hu)
bt_dev_dbg(hu->hdev, "hu %p", hu);
+ if (!hci_uart_has_flow_control(hu))
+ return -EOPNOTSUPP;
+
bcm = kzalloc(sizeof(*bcm), GFP_KERNEL);
if (!bcm)
return -ENOMEM;
@@ -821,6 +824,21 @@ unlock:
}
#endif
+/* Some firmware reports an IRQ which does not work (wrong pin in fw table?) */
+static const struct dmi_system_id bcm_broken_irq_dmi_table[] = {
+ {
+ .ident = "Meegopad T08",
+ .matches = {
+ DMI_EXACT_MATCH(DMI_BOARD_VENDOR,
+ "To be filled by OEM."),
+ DMI_EXACT_MATCH(DMI_BOARD_NAME, "T3 MRD"),
+ DMI_EXACT_MATCH(DMI_BOARD_VERSION, "V1.1"),
+ },
+ },
+ { }
+};
+
+#ifdef CONFIG_ACPI
static const struct acpi_gpio_params first_gpio = { 0, 0, false };
static const struct acpi_gpio_params second_gpio = { 1, 0, false };
static const struct acpi_gpio_params third_gpio = { 2, 0, false };
@@ -839,21 +857,6 @@ static const struct acpi_gpio_mapping acpi_bcm_int_first_gpios[] = {
{ },
};
-/* Some firmware reports an IRQ which does not work (wrong pin in fw table?) */
-static const struct dmi_system_id bcm_broken_irq_dmi_table[] = {
- {
- .ident = "Meegopad T08",
- .matches = {
- DMI_EXACT_MATCH(DMI_BOARD_VENDOR,
- "To be filled by OEM."),
- DMI_EXACT_MATCH(DMI_BOARD_NAME, "T3 MRD"),
- DMI_EXACT_MATCH(DMI_BOARD_VERSION, "V1.1"),
- },
- },
- { }
-};
-
-#ifdef CONFIG_ACPI
static int bcm_resource(struct acpi_resource *ares, void *data)
{
struct bcm_device *dev = data;
@@ -1416,6 +1419,7 @@ static void bcm_serdev_remove(struct serdev_device *serdev)
#ifdef CONFIG_OF
static const struct of_device_id bcm_bluetooth_of_match[] = {
{ .compatible = "brcm,bcm20702a1" },
+ { .compatible = "brcm,bcm4345c5" },
{ .compatible = "brcm,bcm4330-bt" },
{ .compatible = "brcm,bcm43438-bt" },
{ },
diff --git a/drivers/bluetooth/hci_bcsp.c b/drivers/bluetooth/hci_bcsp.c
index 82b13faa9422..fe2e307009f4 100644
--- a/drivers/bluetooth/hci_bcsp.c
+++ b/drivers/bluetooth/hci_bcsp.c
@@ -744,6 +744,11 @@ static int bcsp_close(struct hci_uart *hu)
skb_queue_purge(&bcsp->rel);
skb_queue_purge(&bcsp->unrel);
+ if (bcsp->rx_skb) {
+ kfree_skb(bcsp->rx_skb);
+ bcsp->rx_skb = NULL;
+ }
+
kfree(bcsp);
return 0;
}
diff --git a/drivers/bluetooth/hci_intel.c b/drivers/bluetooth/hci_intel.c
index 207bae5e0d46..31f25153087d 100644
--- a/drivers/bluetooth/hci_intel.c
+++ b/drivers/bluetooth/hci_intel.c
@@ -391,6 +391,9 @@ static int intel_open(struct hci_uart *hu)
BT_DBG("hu %p", hu);
+ if (!hci_uart_has_flow_control(hu))
+ return -EOPNOTSUPP;
+
intel = kzalloc(sizeof(*intel), GFP_KERNEL);
if (!intel)
return -ENOMEM;
diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c
index c84f985f348d..85a30fb9177b 100644
--- a/drivers/bluetooth/hci_ldisc.c
+++ b/drivers/bluetooth/hci_ldisc.c
@@ -178,6 +178,7 @@ restart:
goto restart;
clear_bit(HCI_UART_SENDING, &hu->tx_state);
+ wake_up_bit(&hu->tx_state, HCI_UART_SENDING);
}
void hci_uart_init_work(struct work_struct *work)
@@ -213,6 +214,13 @@ int hci_uart_init_ready(struct hci_uart *hu)
return 0;
}
+int hci_uart_wait_until_sent(struct hci_uart *hu)
+{
+ return wait_on_bit_timeout(&hu->tx_state, HCI_UART_SENDING,
+ TASK_INTERRUPTIBLE,
+ msecs_to_jiffies(2000));
+}
+
/* ------- Interface to HCI layer ------ */
/* Reset device */
static int hci_uart_flush(struct hci_dev *hdev)
@@ -284,6 +292,19 @@ static int hci_uart_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
return 0;
}
+/* Check the underlying device or tty has flow control support */
+bool hci_uart_has_flow_control(struct hci_uart *hu)
+{
+ /* serdev nodes check if the needed operations are present */
+ if (hu->serdev)
+ return true;
+
+ if (hu->tty->driver->ops->tiocmget && hu->tty->driver->ops->tiocmset)
+ return true;
+
+ return false;
+}
+
/* Flow control or un-flow control the device */
void hci_uart_set_flow_control(struct hci_uart *hu, bool enable)
{
diff --git a/drivers/bluetooth/hci_ll.c b/drivers/bluetooth/hci_ll.c
index c04f5f9e1ed0..285706618f8a 100644
--- a/drivers/bluetooth/hci_ll.c
+++ b/drivers/bluetooth/hci_ll.c
@@ -128,6 +128,7 @@ static int ll_open(struct hci_uart *hu)
if (hu->serdev) {
struct ll_device *lldev = serdev_device_get_drvdata(hu->serdev);
+
if (!IS_ERR(lldev->ext_clk))
clk_prepare_enable(lldev->ext_clk);
}
@@ -162,6 +163,7 @@ static int ll_close(struct hci_uart *hu)
if (hu->serdev) {
struct ll_device *lldev = serdev_device_get_drvdata(hu->serdev);
+
gpiod_set_value_cansleep(lldev->enable_gpio, 0);
clk_disable_unprepare(lldev->ext_clk);
@@ -227,7 +229,8 @@ static void ll_device_want_to_wakeup(struct hci_uart *hu)
break;
default:
/* any other state is illegal */
- BT_ERR("received HCILL_WAKE_UP_IND in state %ld", ll->hcill_state);
+ BT_ERR("received HCILL_WAKE_UP_IND in state %ld",
+ ll->hcill_state);
break;
}
@@ -256,7 +259,8 @@ static void ll_device_want_to_sleep(struct hci_uart *hu)
/* sanity check */
if (ll->hcill_state != HCILL_AWAKE)
- BT_ERR("ERR: HCILL_GO_TO_SLEEP_IND in state %ld", ll->hcill_state);
+ BT_ERR("ERR: HCILL_GO_TO_SLEEP_IND in state %ld",
+ ll->hcill_state);
/* acknowledge device sleep */
if (send_hcill_cmd(HCILL_GO_TO_SLEEP_ACK, hu) < 0) {
@@ -289,7 +293,8 @@ static void ll_device_woke_up(struct hci_uart *hu)
/* sanity check */
if (ll->hcill_state != HCILL_ASLEEP_TO_AWAKE)
- BT_ERR("received HCILL_WAKE_UP_ACK in state %ld", ll->hcill_state);
+ BT_ERR("received HCILL_WAKE_UP_ACK in state %ld",
+ ll->hcill_state);
/* send pending packets and change state to HCILL_AWAKE */
__ll_do_awake(ll);
@@ -338,7 +343,8 @@ static int ll_enqueue(struct hci_uart *hu, struct sk_buff *skb)
skb_queue_tail(&ll->tx_wait_q, skb);
break;
default:
- BT_ERR("illegal hcill state: %ld (losing packet)", ll->hcill_state);
+ BT_ERR("illegal hcill state: %ld (losing packet)",
+ ll->hcill_state);
kfree_skb(skb);
break;
}
@@ -438,6 +444,7 @@ static int ll_recv(struct hci_uart *hu, const void *data, int count)
static struct sk_buff *ll_dequeue(struct hci_uart *hu)
{
struct ll_struct *ll = hu->priv;
+
return skb_dequeue(&ll->txq);
}
@@ -449,7 +456,8 @@ static int read_local_version(struct hci_dev *hdev)
struct sk_buff *skb;
struct hci_rp_read_local_version *ver;
- skb = __hci_cmd_sync(hdev, HCI_OP_READ_LOCAL_VERSION, 0, NULL, HCI_INIT_TIMEOUT);
+ skb = __hci_cmd_sync(hdev, HCI_OP_READ_LOCAL_VERSION, 0, NULL,
+ HCI_INIT_TIMEOUT);
if (IS_ERR(skb)) {
bt_dev_err(hdev, "Reading TI version information failed (%ld)",
PTR_ERR(skb));
@@ -469,11 +477,38 @@ static int read_local_version(struct hci_dev *hdev)
version = le16_to_cpu(ver->lmp_subver);
out:
- if (err) bt_dev_err(hdev, "Failed to read TI version info: %d", err);
+ if (err)
+ bt_dev_err(hdev, "Failed to read TI version info: %d", err);
kfree_skb(skb);
return err ? err : version;
}
+static int send_command_from_firmware(struct ll_device *lldev,
+ struct hci_command *cmd)
+{
+ struct sk_buff *skb;
+
+ if (cmd->opcode == HCI_VS_UPDATE_UART_HCI_BAUDRATE) {
+ /* ignore remote change
+ * baud rate HCI VS command
+ */
+ bt_dev_warn(lldev->hu.hdev,
+ "change remote baud rate command in firmware");
+ return 0;
+ }
+ if (cmd->prefix != 1)
+ bt_dev_dbg(lldev->hu.hdev, "command type %d", cmd->prefix);
+
+ skb = __hci_cmd_sync(lldev->hu.hdev, cmd->opcode, cmd->plen,
+ &cmd->speed, HCI_INIT_TIMEOUT);
+ if (IS_ERR(skb)) {
+ bt_dev_err(lldev->hu.hdev, "send command failed");
+ return PTR_ERR(skb);
+ }
+ kfree_skb(skb);
+ return 0;
+}
+
/**
* download_firmware -
* internal function which parses through the .bts firmware
@@ -486,7 +521,6 @@ static int download_firmware(struct ll_device *lldev)
unsigned char *ptr, *action_ptr;
unsigned char bts_scr_name[40]; /* 40 char long bts scr name? */
const struct firmware *fw;
- struct sk_buff *skb;
struct hci_command *cmd;
version = read_local_version(lldev->hu.hdev);
@@ -528,23 +562,9 @@ static int download_firmware(struct ll_device *lldev)
case ACTION_SEND_COMMAND: /* action send */
bt_dev_dbg(lldev->hu.hdev, "S");
cmd = (struct hci_command *)action_ptr;
- if (cmd->opcode == HCI_VS_UPDATE_UART_HCI_BAUDRATE) {
- /* ignore remote change
- * baud rate HCI VS command
- */
- bt_dev_warn(lldev->hu.hdev, "change remote baud rate command in firmware");
- break;
- }
- if (cmd->prefix != 1)
- bt_dev_dbg(lldev->hu.hdev, "command type %d", cmd->prefix);
-
- skb = __hci_cmd_sync(lldev->hu.hdev, cmd->opcode, cmd->plen, &cmd->speed, HCI_INIT_TIMEOUT);
- if (IS_ERR(skb)) {
- bt_dev_err(lldev->hu.hdev, "send command failed");
- err = PTR_ERR(skb);
+ err = send_command_from_firmware(lldev, cmd);
+ if (err)
goto out_rel_fw;
- }
- kfree_skb(skb);
break;
case ACTION_WAIT_EVENT: /* wait */
/* no need to wait as command was synchronous */
@@ -601,6 +621,13 @@ static int ll_setup(struct hci_uart *hu)
serdev_device_set_flow_control(serdev, true);
+ if (hu->oper_speed)
+ speed = hu->oper_speed;
+ else if (hu->proto->oper_speed)
+ speed = hu->proto->oper_speed;
+ else
+ speed = 0;
+
do {
/* Reset the Bluetooth device */
gpiod_set_value_cansleep(lldev->enable_gpio, 0);
@@ -612,6 +639,20 @@ static int ll_setup(struct hci_uart *hu)
return err;
}
+ if (speed) {
+ __le32 speed_le = cpu_to_le32(speed);
+ struct sk_buff *skb;
+
+ skb = __hci_cmd_sync(hu->hdev,
+ HCI_VS_UPDATE_UART_HCI_BAUDRATE,
+ sizeof(speed_le), &speed_le,
+ HCI_INIT_TIMEOUT);
+ if (!IS_ERR(skb)) {
+ kfree_skb(skb);
+ serdev_device_set_baudrate(serdev, speed);
+ }
+ }
+
err = download_firmware(lldev);
if (!err)
break;
@@ -636,25 +677,7 @@ static int ll_setup(struct hci_uart *hu)
}
/* Operational speed if any */
- if (hu->oper_speed)
- speed = hu->oper_speed;
- else if (hu->proto->oper_speed)
- speed = hu->proto->oper_speed;
- else
- speed = 0;
-
- if (speed) {
- __le32 speed_le = cpu_to_le32(speed);
- struct sk_buff *skb;
- skb = __hci_cmd_sync(hu->hdev, HCI_VS_UPDATE_UART_HCI_BAUDRATE,
- sizeof(speed_le), &speed_le,
- HCI_INIT_TIMEOUT);
- if (!IS_ERR(skb)) {
- kfree_skb(skb);
- serdev_device_set_baudrate(serdev, speed);
- }
- }
return 0;
}
@@ -676,7 +699,9 @@ static int hci_ti_probe(struct serdev_device *serdev)
serdev_device_set_drvdata(serdev, lldev);
lldev->serdev = hu->serdev = serdev;
- lldev->enable_gpio = devm_gpiod_get_optional(&serdev->dev, "enable", GPIOD_OUT_LOW);
+ lldev->enable_gpio = devm_gpiod_get_optional(&serdev->dev,
+ "enable",
+ GPIOD_OUT_LOW);
if (IS_ERR(lldev->enable_gpio))
return PTR_ERR(lldev->enable_gpio);
diff --git a/drivers/bluetooth/hci_mrvl.c b/drivers/bluetooth/hci_mrvl.c
index 50212ac629e3..fbc3f7c3a5c7 100644
--- a/drivers/bluetooth/hci_mrvl.c
+++ b/drivers/bluetooth/hci_mrvl.c
@@ -13,6 +13,8 @@
#include <linux/firmware.h>
#include <linux/module.h>
#include <linux/tty.h>
+#include <linux/of.h>
+#include <linux/serdev.h>
#include <net/bluetooth/bluetooth.h>
#include <net/bluetooth/hci_core.h>
@@ -40,6 +42,10 @@ struct mrvl_data {
u8 id, rev;
};
+struct mrvl_serdev {
+ struct hci_uart hu;
+};
+
struct hci_mrvl_pkt {
__le16 lhs;
__le16 rhs;
@@ -49,9 +55,13 @@ struct hci_mrvl_pkt {
static int mrvl_open(struct hci_uart *hu)
{
struct mrvl_data *mrvl;
+ int ret;
BT_DBG("hu %p", hu);
+ if (!hci_uart_has_flow_control(hu))
+ return -EOPNOTSUPP;
+
mrvl = kzalloc(sizeof(*mrvl), GFP_KERNEL);
if (!mrvl)
return -ENOMEM;
@@ -62,7 +72,18 @@ static int mrvl_open(struct hci_uart *hu)
set_bit(STATE_CHIP_VER_PENDING, &mrvl->flags);
hu->priv = mrvl;
+
+ if (hu->serdev) {
+ ret = serdev_device_open(hu->serdev);
+ if (ret)
+ goto err;
+ }
+
return 0;
+err:
+ kfree(mrvl);
+
+ return ret;
}
static int mrvl_close(struct hci_uart *hu)
@@ -71,6 +92,9 @@ static int mrvl_close(struct hci_uart *hu)
BT_DBG("hu %p", hu);
+ if (hu->serdev)
+ serdev_device_close(hu->serdev);
+
skb_queue_purge(&mrvl->txq);
skb_queue_purge(&mrvl->rawq);
kfree_skb(mrvl->rx_skb);
@@ -339,7 +363,14 @@ static int mrvl_setup(struct hci_uart *hu)
return -EINVAL;
}
- hci_uart_set_baudrate(hu, 3000000);
+ /* Let the final ack go out before switching the baudrate */
+ hci_uart_wait_until_sent(hu);
+
+ if (hu->serdev)
+ serdev_device_set_baudrate(hu->serdev, 3000000);
+ else
+ hci_uart_set_baudrate(hu, 3000000);
+
hci_uart_set_flow_control(hu, false);
err = mrvl_load_firmware(hu->hdev, "mrvl/uart8897_bt.bin");
@@ -362,12 +393,54 @@ static const struct hci_uart_proto mrvl_proto = {
.dequeue = mrvl_dequeue,
};
+static int mrvl_serdev_probe(struct serdev_device *serdev)
+{
+ struct mrvl_serdev *mrvldev;
+
+ mrvldev = devm_kzalloc(&serdev->dev, sizeof(*mrvldev), GFP_KERNEL);
+ if (!mrvldev)
+ return -ENOMEM;
+
+ mrvldev->hu.serdev = serdev;
+ serdev_device_set_drvdata(serdev, mrvldev);
+
+ return hci_uart_register_device(&mrvldev->hu, &mrvl_proto);
+}
+
+static void mrvl_serdev_remove(struct serdev_device *serdev)
+{
+ struct mrvl_serdev *mrvldev = serdev_device_get_drvdata(serdev);
+
+ hci_uart_unregister_device(&mrvldev->hu);
+}
+
+#ifdef CONFIG_OF
+static const struct of_device_id mrvl_bluetooth_of_match[] = {
+ { .compatible = "mrvl,88w8897" },
+ { },
+};
+MODULE_DEVICE_TABLE(of, mrvl_bluetooth_of_match);
+#endif
+
+static struct serdev_device_driver mrvl_serdev_driver = {
+ .probe = mrvl_serdev_probe,
+ .remove = mrvl_serdev_remove,
+ .driver = {
+ .name = "hci_uart_mrvl",
+ .of_match_table = of_match_ptr(mrvl_bluetooth_of_match),
+ },
+};
+
int __init mrvl_init(void)
{
+ serdev_device_driver_register(&mrvl_serdev_driver);
+
return hci_uart_register_proto(&mrvl_proto);
}
int __exit mrvl_deinit(void)
{
+ serdev_device_driver_unregister(&mrvl_serdev_driver);
+
return hci_uart_unregister_proto(&mrvl_proto);
}
diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c
index 9d273cdde563..e3164c200eac 100644
--- a/drivers/bluetooth/hci_qca.c
+++ b/drivers/bluetooth/hci_qca.c
@@ -17,6 +17,7 @@
#include <linux/kernel.h>
#include <linux/clk.h>
+#include <linux/completion.h>
#include <linux/debugfs.h>
#include <linux/delay.h>
#include <linux/device.h>
@@ -53,6 +54,7 @@
enum qca_flags {
QCA_IBS_ENABLED,
+ QCA_DROP_VENDOR_EVENT,
};
/* HCI_IBS transmit side sleep protocol states */
@@ -97,6 +99,7 @@ struct qca_data {
struct work_struct ws_rx_vote_off;
struct work_struct ws_tx_vote_off;
unsigned long flags;
+ struct completion drop_ev_comp;
/* For debugging purpose */
u64 ibs_sent_wacks;
@@ -156,6 +159,7 @@ struct qca_serdev {
struct qca_power *bt_power;
u32 init_speed;
u32 oper_speed;
+ const char *firmware_name;
};
static int qca_power_setup(struct hci_uart *hu, bool on);
@@ -177,6 +181,17 @@ static enum qca_btsoc_type qca_soc_type(struct hci_uart *hu)
return soc_type;
}
+static const char *qca_get_firmware_name(struct hci_uart *hu)
+{
+ if (hu->serdev) {
+ struct qca_serdev *qsd = serdev_device_get_drvdata(hu->serdev);
+
+ return qsd->firmware_name;
+ } else {
+ return NULL;
+ }
+}
+
static void __serial_clock_on(struct tty_struct *tty)
{
/* TODO: Some chipset requires to enable UART clock on client
@@ -294,13 +309,14 @@ static void qca_wq_awake_device(struct work_struct *work)
ws_awake_device);
struct hci_uart *hu = qca->hu;
unsigned long retrans_delay;
+ unsigned long flags;
BT_DBG("hu %p wq awake device", hu);
/* Vote for serial clock */
serial_clock_vote(HCI_IBS_TX_VOTE_CLOCK_ON, hu);
- spin_lock(&qca->hci_ibs_lock);
+ spin_lock_irqsave(&qca->hci_ibs_lock, flags);
/* Send wake indication to device */
if (send_hci_ibs_cmd(HCI_IBS_WAKE_IND, hu) < 0)
@@ -312,7 +328,7 @@ static void qca_wq_awake_device(struct work_struct *work)
retrans_delay = msecs_to_jiffies(qca->wake_retrans);
mod_timer(&qca->wake_retrans_timer, jiffies + retrans_delay);
- spin_unlock(&qca->hci_ibs_lock);
+ spin_unlock_irqrestore(&qca->hci_ibs_lock, flags);
/* Actually send the packets */
hci_uart_tx_wakeup(hu);
@@ -323,12 +339,13 @@ static void qca_wq_awake_rx(struct work_struct *work)
struct qca_data *qca = container_of(work, struct qca_data,
ws_awake_rx);
struct hci_uart *hu = qca->hu;
+ unsigned long flags;
BT_DBG("hu %p wq awake rx", hu);
serial_clock_vote(HCI_IBS_RX_VOTE_CLOCK_ON, hu);
- spin_lock(&qca->hci_ibs_lock);
+ spin_lock_irqsave(&qca->hci_ibs_lock, flags);
qca->rx_ibs_state = HCI_IBS_RX_AWAKE;
/* Always acknowledge device wake up,
@@ -339,7 +356,7 @@ static void qca_wq_awake_rx(struct work_struct *work)
qca->ibs_sent_wacks++;
- spin_unlock(&qca->hci_ibs_lock);
+ spin_unlock_irqrestore(&qca->hci_ibs_lock, flags);
/* Actually send the packets */
hci_uart_tx_wakeup(hu);
@@ -458,6 +475,9 @@ static int qca_open(struct hci_uart *hu)
BT_DBG("hu %p qca_open", hu);
+ if (!hci_uart_has_flow_control(hu))
+ return -EOPNOTSUPP;
+
qca = kzalloc(sizeof(struct qca_data), GFP_KERNEL);
if (!qca)
return -ENOMEM;
@@ -478,31 +498,13 @@ static int qca_open(struct hci_uart *hu)
INIT_WORK(&qca->ws_tx_vote_off, qca_wq_serial_tx_clock_vote_off);
qca->hu = hu;
+ init_completion(&qca->drop_ev_comp);
/* Assume we start with both sides asleep -- extra wakes OK */
qca->tx_ibs_state = HCI_IBS_TX_ASLEEP;
qca->rx_ibs_state = HCI_IBS_RX_ASLEEP;
- /* clocks actually on, but we start votes off */
- qca->tx_vote = false;
- qca->rx_vote = false;
- qca->flags = 0;
-
- qca->ibs_sent_wacks = 0;
- qca->ibs_sent_slps = 0;
- qca->ibs_sent_wakes = 0;
- qca->ibs_recv_wacks = 0;
- qca->ibs_recv_slps = 0;
- qca->ibs_recv_wakes = 0;
qca->vote_last_jif = jiffies;
- qca->vote_on_ms = 0;
- qca->vote_off_ms = 0;
- qca->votes_on = 0;
- qca->votes_off = 0;
- qca->tx_votes_on = 0;
- qca->tx_votes_off = 0;
- qca->rx_votes_on = 0;
- qca->rx_votes_off = 0;
hu->priv = qca;
@@ -686,7 +688,7 @@ static void device_want_to_sleep(struct hci_uart *hu)
unsigned long flags;
struct qca_data *qca = hu->priv;
- BT_DBG("hu %p want to sleep", hu);
+ BT_DBG("hu %p want to sleep in %d state", hu, qca->rx_ibs_state);
spin_lock_irqsave(&qca->hci_ibs_lock, flags);
@@ -701,7 +703,7 @@ static void device_want_to_sleep(struct hci_uart *hu)
break;
case HCI_IBS_RX_ASLEEP:
- /* Fall through */
+ break;
default:
/* Any other state is illegal */
@@ -872,6 +874,35 @@ static int qca_recv_acl_data(struct hci_dev *hdev, struct sk_buff *skb)
return hci_recv_frame(hdev, skb);
}
+static int qca_recv_event(struct hci_dev *hdev, struct sk_buff *skb)
+{
+ struct hci_uart *hu = hci_get_drvdata(hdev);
+ struct qca_data *qca = hu->priv;
+
+ if (test_bit(QCA_DROP_VENDOR_EVENT, &qca->flags)) {
+ struct hci_event_hdr *hdr = (void *)skb->data;
+
+ /* For the WCN3990 the vendor command for a baudrate change
+ * isn't sent as synchronous HCI command, because the
+ * controller sends the corresponding vendor event with the
+ * new baudrate. The event is received and properly decoded
+ * after changing the baudrate of the host port. It needs to
+ * be dropped, otherwise it can be misinterpreted as
+ * response to a later firmware download command (also a
+ * vendor command).
+ */
+
+ if (hdr->evt == HCI_EV_VENDOR)
+ complete(&qca->drop_ev_comp);
+
+ kfree_skb(skb);
+
+ return 0;
+ }
+
+ return hci_recv_frame(hdev, skb);
+}
+
#define QCA_IBS_SLEEP_IND_EVENT \
.type = HCI_IBS_SLEEP_IND, \
.hlen = 0, \
@@ -896,7 +927,7 @@ static int qca_recv_acl_data(struct hci_dev *hdev, struct sk_buff *skb)
static const struct h4_recv_pkt qca_recv_pkts[] = {
{ H4_RECV_ACL, .recv = qca_recv_acl_data },
{ H4_RECV_SCO, .recv = hci_recv_frame },
- { H4_RECV_EVENT, .recv = hci_recv_frame },
+ { H4_RECV_EVENT, .recv = qca_recv_event },
{ QCA_IBS_WAKE_IND_EVENT, .recv = qca_ibs_wake_ind },
{ QCA_IBS_WAKE_ACK_EVENT, .recv = qca_ibs_wake_ack },
{ QCA_IBS_SLEEP_IND_EVENT, .recv = qca_ibs_sleep_ind },
@@ -1091,6 +1122,7 @@ static int qca_check_speeds(struct hci_uart *hu)
static int qca_set_speed(struct hci_uart *hu, enum qca_speed_type speed_type)
{
unsigned int speed, qca_baudrate;
+ struct qca_data *qca = hu->priv;
int ret = 0;
if (speed_type == QCA_INIT_SPEED) {
@@ -1110,6 +1142,11 @@ static int qca_set_speed(struct hci_uart *hu, enum qca_speed_type speed_type)
if (qca_is_wcn399x(soc_type))
hci_uart_set_flow_control(hu, true);
+ if (soc_type == QCA_WCN3990) {
+ reinit_completion(&qca->drop_ev_comp);
+ set_bit(QCA_DROP_VENDOR_EVENT, &qca->flags);
+ }
+
qca_baudrate = qca_get_baudrate_value(speed);
bt_dev_dbg(hu->hdev, "Set UART speed to %d", speed);
ret = qca_set_baudrate(hu->hdev, qca_baudrate);
@@ -1121,6 +1158,20 @@ static int qca_set_speed(struct hci_uart *hu, enum qca_speed_type speed_type)
error:
if (qca_is_wcn399x(soc_type))
hci_uart_set_flow_control(hu, false);
+
+ if (soc_type == QCA_WCN3990) {
+ /* Wait for the controller to send the vendor event
+ * for the baudrate change command.
+ */
+ if (!wait_for_completion_timeout(&qca->drop_ev_comp,
+ msecs_to_jiffies(100))) {
+ bt_dev_err(hu->hdev,
+ "Failed to change controller baudrate\n");
+ ret = -ETIMEDOUT;
+ }
+
+ clear_bit(QCA_DROP_VENDOR_EVENT, &qca->flags);
+ }
}
return ret;
@@ -1182,6 +1233,7 @@ static int qca_setup(struct hci_uart *hu)
struct qca_data *qca = hu->priv;
unsigned int speed, qca_baudrate = QCA_BAUDRATE_115200;
enum qca_btsoc_type soc_type = qca_soc_type(hu);
+ const char *firmware_name = qca_get_firmware_name(hu);
int ret;
int soc_ver = 0;
@@ -1192,6 +1244,11 @@ static int qca_setup(struct hci_uart *hu)
/* Patch downloading has to be done without IBS mode */
clear_bit(QCA_IBS_ENABLED, &qca->flags);
+ /* Enable controller to do both LE scan and BR/EDR inquiry
+ * simultaneously.
+ */
+ set_bit(HCI_QUIRK_SIMULTANEOUS_DISCOVERY, &hdev->quirks);
+
if (qca_is_wcn399x(soc_type)) {
bt_dev_info(hdev, "setting up wcn3990");
@@ -1232,7 +1289,8 @@ static int qca_setup(struct hci_uart *hu)
bt_dev_info(hdev, "QCA controller version 0x%08x", soc_ver);
/* Setup patch / NVM configurations */
- ret = qca_uart_setup(hdev, qca_baudrate, soc_type, soc_ver);
+ ret = qca_uart_setup(hdev, qca_baudrate, soc_type, soc_ver,
+ firmware_name);
if (!ret) {
set_bit(QCA_IBS_ENABLED, &qca->flags);
qca_debugfs_init(hdev);
@@ -1256,7 +1314,7 @@ static int qca_setup(struct hci_uart *hu)
return ret;
}
-static struct hci_uart_proto qca_proto = {
+static const struct hci_uart_proto qca_proto = {
.id = HCI_UART_QCA,
.name = "QCA",
.manufacturer = 29,
@@ -1316,6 +1374,11 @@ static int qca_power_off(struct hci_dev *hdev)
{
struct hci_uart *hu = hci_get_drvdata(hdev);
+ /* Perform pre shutdown command */
+ qca_send_pre_shutdown_cmd(hdev);
+
+ usleep_range(8000, 10000);
+
qca_power_shutdown(hu);
return 0;
}
@@ -1426,6 +1489,8 @@ static int qca_serdev_probe(struct serdev_device *serdev)
qcadev->serdev_hu.serdev = serdev;
data = of_device_get_match_data(&serdev->dev);
serdev_device_set_drvdata(serdev, qcadev);
+ device_property_read_string(&serdev->dev, "firmware-name",
+ &qcadev->firmware_name);
if (data && qca_is_wcn399x(data->soc_type)) {
qcadev->btsoc_type = data->soc_type;
qcadev->bt_power = devm_kzalloc(&serdev->dev,
diff --git a/drivers/bluetooth/hci_uart.h b/drivers/bluetooth/hci_uart.h
index d8cf005e3c5d..6ab631101019 100644
--- a/drivers/bluetooth/hci_uart.h
+++ b/drivers/bluetooth/hci_uart.h
@@ -100,9 +100,11 @@ int hci_uart_register_device(struct hci_uart *hu, const struct hci_uart_proto *p
void hci_uart_unregister_device(struct hci_uart *hu);
int hci_uart_tx_wakeup(struct hci_uart *hu);
+int hci_uart_wait_until_sent(struct hci_uart *hu);
int hci_uart_init_ready(struct hci_uart *hu);
void hci_uart_init_work(struct work_struct *work);
void hci_uart_set_baudrate(struct hci_uart *hu, unsigned int speed);
+bool hci_uart_has_flow_control(struct hci_uart *hu);
void hci_uart_set_flow_control(struct hci_uart *hu, bool enable);
void hci_uart_set_speeds(struct hci_uart *hu, unsigned int init_speed,
unsigned int oper_speed);
diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig
index 1851112ccc29..6b331061d34b 100644
--- a/drivers/bus/Kconfig
+++ b/drivers/bus/Kconfig
@@ -29,6 +29,16 @@ config BRCMSTB_GISB_ARB
arbiter. This driver provides timeout and target abort error handling
and internal bus master decoding.
+config MOXTET
+ tristate "CZ.NIC Turris Mox module configuration bus"
+ depends on SPI_MASTER && OF
+ help
+ Say yes here to add support for the module configuration bus found
+ on CZ.NIC's Turris Mox. This is needed for the ability to discover
+ the order in which the modules are connected and to get/set some of
+ their settings. For example the GPIOs on Mox SFP module are
+ configured through this bus.
+
config HISILICON_LPC
bool "Support for ISA I/O space on HiSilicon Hip06/7"
depends on ARM64 && (ARCH_HISI || COMPILE_TEST)
diff --git a/drivers/bus/Makefile b/drivers/bus/Makefile
index ca300b1914ce..16b43d3468c6 100644
--- a/drivers/bus/Makefile
+++ b/drivers/bus/Makefile
@@ -8,6 +8,7 @@ obj-$(CONFIG_ARM_CCI) += arm-cci.o
obj-$(CONFIG_HISILICON_LPC) += hisi_lpc.o
obj-$(CONFIG_BRCMSTB_GISB_ARB) += brcmstb_gisb.o
+obj-$(CONFIG_MOXTET) += moxtet.o
# DPAA2 fsl-mc bus
obj-$(CONFIG_FSL_MC_BUS) += fsl-mc/
diff --git a/drivers/bus/brcmstb_gisb.c b/drivers/bus/brcmstb_gisb.c
index 972854ca1d9a..ec1004c858b8 100644
--- a/drivers/bus/brcmstb_gisb.c
+++ b/drivers/bus/brcmstb_gisb.c
@@ -399,8 +399,8 @@ static int __init brcmstb_gisb_arb_probe(struct platform_device *pdev)
&gisb_panic_notifier);
}
- dev_info(&pdev->dev, "registered mem: %p, irqs: %d, %d\n",
- gdev->base, timeout_irq, tea_irq);
+ dev_info(&pdev->dev, "registered irqs: %d, %d\n",
+ timeout_irq, tea_irq);
return 0;
}
diff --git a/drivers/bus/fsl-mc/dprc.c b/drivers/bus/fsl-mc/dprc.c
index 1c3f62182266..0fe3f52ae0de 100644
--- a/drivers/bus/fsl-mc/dprc.c
+++ b/drivers/bus/fsl-mc/dprc.c
@@ -443,11 +443,31 @@ int dprc_get_obj_region(struct fsl_mc_io *mc_io,
struct fsl_mc_command cmd = { 0 };
struct dprc_cmd_get_obj_region *cmd_params;
struct dprc_rsp_get_obj_region *rsp_params;
+ u16 major_ver, minor_ver;
int err;
/* prepare command */
- cmd.header = mc_encode_cmd_header(DPRC_CMDID_GET_OBJ_REG,
- cmd_flags, token);
+ err = dprc_get_api_version(mc_io, 0,
+ &major_ver,
+ &minor_ver);
+ if (err)
+ return err;
+
+ /**
+ * MC API version 6.3 introduced a new field to the region
+ * descriptor: base_address. If the older API is in use then the base
+ * address is set to zero to indicate it needs to be obtained elsewhere
+ * (typically the device tree).
+ */
+ if (major_ver > 6 || (major_ver == 6 && minor_ver >= 3))
+ cmd.header =
+ mc_encode_cmd_header(DPRC_CMDID_GET_OBJ_REG_V2,
+ cmd_flags, token);
+ else
+ cmd.header =
+ mc_encode_cmd_header(DPRC_CMDID_GET_OBJ_REG,
+ cmd_flags, token);
+
cmd_params = (struct dprc_cmd_get_obj_region *)cmd.params;
cmd_params->obj_id = cpu_to_le32(obj_id);
cmd_params->region_index = region_index;
@@ -461,8 +481,12 @@ int dprc_get_obj_region(struct fsl_mc_io *mc_io,
/* retrieve response parameters */
rsp_params = (struct dprc_rsp_get_obj_region *)cmd.params;
- region_desc->base_offset = le64_to_cpu(rsp_params->base_addr);
+ region_desc->base_offset = le64_to_cpu(rsp_params->base_offset);
region_desc->size = le32_to_cpu(rsp_params->size);
+ if (major_ver > 6 || (major_ver == 6 && minor_ver >= 3))
+ region_desc->base_address = le64_to_cpu(rsp_params->base_addr);
+ else
+ region_desc->base_address = 0;
return 0;
}
diff --git a/drivers/bus/fsl-mc/fsl-mc-allocator.c b/drivers/bus/fsl-mc/fsl-mc-allocator.c
index 8ad77246f322..cc7bb900f524 100644
--- a/drivers/bus/fsl-mc/fsl-mc-allocator.c
+++ b/drivers/bus/fsl-mc/fsl-mc-allocator.c
@@ -330,7 +330,6 @@ void fsl_mc_object_free(struct fsl_mc_device *mc_adev)
fsl_mc_resource_free(resource);
- device_link_del(mc_adev->consumer_link);
mc_adev->consumer_link = NULL;
}
EXPORT_SYMBOL_GPL(fsl_mc_object_free);
diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c
index f0404c6d1ff4..5c9bf2e06552 100644
--- a/drivers/bus/fsl-mc/fsl-mc-bus.c
+++ b/drivers/bus/fsl-mc/fsl-mc-bus.c
@@ -487,10 +487,19 @@ static int fsl_mc_device_get_mmio_regions(struct fsl_mc_device *mc_dev,
"dprc_get_obj_region() failed: %d\n", error);
goto error_cleanup_regions;
}
-
- error = translate_mc_addr(mc_dev, mc_region_type,
+ /*
+ * Older MC only returned region offset and no base address
+ * If base address is in the region_desc use it otherwise
+ * revert to old mechanism
+ */
+ if (region_desc.base_address)
+ regions[i].start = region_desc.base_address +
+ region_desc.base_offset;
+ else
+ error = translate_mc_addr(mc_dev, mc_region_type,
region_desc.base_offset,
&regions[i].start);
+
if (error < 0) {
dev_err(parent_dev,
"Invalid MC offset: %#x (for %s.%d\'s region %d)\n",
@@ -504,6 +513,8 @@ static int fsl_mc_device_get_mmio_regions(struct fsl_mc_device *mc_dev,
regions[i].flags = IORESOURCE_IO;
if (region_desc.flags & DPRC_REGION_CACHEABLE)
regions[i].flags |= IORESOURCE_CACHEABLE;
+ if (region_desc.flags & DPRC_REGION_SHAREABLE)
+ regions[i].flags |= IORESOURCE_MEM;
}
mc_dev->regions = regions;
diff --git a/drivers/bus/fsl-mc/fsl-mc-private.h b/drivers/bus/fsl-mc/fsl-mc-private.h
index ea11b4fe59f7..020fcc04ec8b 100644
--- a/drivers/bus/fsl-mc/fsl-mc-private.h
+++ b/drivers/bus/fsl-mc/fsl-mc-private.h
@@ -79,9 +79,11 @@ int dpmcp_reset(struct fsl_mc_io *mc_io,
/* DPRC command versioning */
#define DPRC_CMD_BASE_VERSION 1
+#define DPRC_CMD_2ND_VERSION 2
#define DPRC_CMD_ID_OFFSET 4
#define DPRC_CMD(id) (((id) << DPRC_CMD_ID_OFFSET) | DPRC_CMD_BASE_VERSION)
+#define DPRC_CMD_V2(id) (((id) << DPRC_CMD_ID_OFFSET) | DPRC_CMD_2ND_VERSION)
/* DPRC command IDs */
#define DPRC_CMDID_CLOSE DPRC_CMD(0x800)
@@ -100,6 +102,7 @@ int dpmcp_reset(struct fsl_mc_io *mc_io,
#define DPRC_CMDID_GET_OBJ_COUNT DPRC_CMD(0x159)
#define DPRC_CMDID_GET_OBJ DPRC_CMD(0x15A)
#define DPRC_CMDID_GET_OBJ_REG DPRC_CMD(0x15E)
+#define DPRC_CMDID_GET_OBJ_REG_V2 DPRC_CMD_V2(0x15E)
#define DPRC_CMDID_SET_OBJ_IRQ DPRC_CMD(0x15F)
struct dprc_cmd_open {
@@ -199,9 +202,16 @@ struct dprc_rsp_get_obj_region {
/* response word 0 */
__le64 pad;
/* response word 1 */
- __le64 base_addr;
+ __le64 base_offset;
/* response word 2 */
__le32 size;
+ __le32 pad2;
+ /* response word 3 */
+ __le32 flags;
+ __le32 pad3;
+ /* response word 4 */
+ /* base_addr may be zero if older MC firmware is used */
+ __le64 base_addr;
};
struct dprc_cmd_set_obj_irq {
@@ -334,6 +344,7 @@ int dprc_set_obj_irq(struct fsl_mc_io *mc_io,
/* Region flags */
/* Cacheable - Indicates that region should be mapped as cacheable */
#define DPRC_REGION_CACHEABLE 0x00000001
+#define DPRC_REGION_SHAREABLE 0x00000002
/**
* enum dprc_region_type - Region type
@@ -342,7 +353,8 @@ int dprc_set_obj_irq(struct fsl_mc_io *mc_io,
*/
enum dprc_region_type {
DPRC_REGION_TYPE_MC_PORTAL,
- DPRC_REGION_TYPE_QBMAN_PORTAL
+ DPRC_REGION_TYPE_QBMAN_PORTAL,
+ DPRC_REGION_TYPE_QBMAN_MEM_BACKED_PORTAL
};
/**
@@ -360,6 +372,7 @@ struct dprc_region_desc {
u32 size;
u32 flags;
enum dprc_region_type type;
+ u64 base_address;
};
int dprc_get_obj_region(struct fsl_mc_io *mc_io,
diff --git a/drivers/bus/fsl-mc/mc-io.c b/drivers/bus/fsl-mc/mc-io.c
index 3ae574a58cce..d9629fc13a15 100644
--- a/drivers/bus/fsl-mc/mc-io.c
+++ b/drivers/bus/fsl-mc/mc-io.c
@@ -255,7 +255,6 @@ void fsl_mc_portal_free(struct fsl_mc_io *mc_io)
fsl_destroy_mc_io(mc_io);
fsl_mc_resource_free(resource);
- device_link_del(dpmcp_dev->consumer_link);
dpmcp_dev->consumer_link = NULL;
}
EXPORT_SYMBOL_GPL(fsl_mc_portal_free);
diff --git a/drivers/bus/hisi_lpc.c b/drivers/bus/hisi_lpc.c
index 19d7b6ff2f17..20c957185af2 100644
--- a/drivers/bus/hisi_lpc.c
+++ b/drivers/bus/hisi_lpc.c
@@ -456,6 +456,17 @@ struct hisi_lpc_acpi_cell {
size_t pdata_size;
};
+static void hisi_lpc_acpi_remove(struct device *hostdev)
+{
+ struct acpi_device *adev = ACPI_COMPANION(hostdev);
+ struct acpi_device *child;
+
+ device_for_each_child(hostdev, NULL, hisi_lpc_acpi_remove_subdev);
+
+ list_for_each_entry(child, &adev->children, node)
+ acpi_device_clear_enumerated(child);
+}
+
/*
* hisi_lpc_acpi_probe - probe children for ACPI FW
* @hostdev: LPC host device pointer
@@ -555,8 +566,7 @@ static int hisi_lpc_acpi_probe(struct device *hostdev)
return 0;
fail:
- device_for_each_child(hostdev, NULL,
- hisi_lpc_acpi_remove_subdev);
+ hisi_lpc_acpi_remove(hostdev);
return ret;
}
@@ -569,6 +579,10 @@ static int hisi_lpc_acpi_probe(struct device *dev)
{
return -ENODEV;
}
+
+static void hisi_lpc_acpi_remove(struct device *hostdev)
+{
+}
#endif // CONFIG_ACPI
/*
@@ -606,24 +620,27 @@ static int hisi_lpc_probe(struct platform_device *pdev)
range->fwnode = dev->fwnode;
range->flags = LOGIC_PIO_INDIRECT;
range->size = PIO_INDIRECT_SIZE;
+ range->hostdata = lpcdev;
+ range->ops = &hisi_lpc_ops;
+ lpcdev->io_host = range;
ret = logic_pio_register_range(range);
if (ret) {
dev_err(dev, "register IO range failed (%d)!\n", ret);
return ret;
}
- lpcdev->io_host = range;
/* register the LPC host PIO resources */
if (acpi_device)
ret = hisi_lpc_acpi_probe(dev);
else
ret = of_platform_populate(dev->of_node, NULL, NULL, dev);
- if (ret)
+ if (ret) {
+ logic_pio_unregister_range(range);
return ret;
+ }
- lpcdev->io_host->hostdata = lpcdev;
- lpcdev->io_host->ops = &hisi_lpc_ops;
+ dev_set_drvdata(dev, lpcdev);
io_end = lpcdev->io_host->io_start + lpcdev->io_host->size;
dev_info(dev, "registered range [%pa - %pa]\n",
@@ -632,6 +649,23 @@ static int hisi_lpc_probe(struct platform_device *pdev)
return ret;
}
+static int hisi_lpc_remove(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct acpi_device *acpi_device = ACPI_COMPANION(dev);
+ struct hisi_lpc_dev *lpcdev = dev_get_drvdata(dev);
+ struct logic_pio_hwaddr *range = lpcdev->io_host;
+
+ if (acpi_device)
+ hisi_lpc_acpi_remove(dev);
+ else
+ of_platform_depopulate(dev);
+
+ logic_pio_unregister_range(range);
+
+ return 0;
+}
+
static const struct of_device_id hisi_lpc_of_match[] = {
{ .compatible = "hisilicon,hip06-lpc", },
{ .compatible = "hisilicon,hip07-lpc", },
@@ -645,5 +679,6 @@ static struct platform_driver hisi_lpc_driver = {
.acpi_match_table = ACPI_PTR(hisi_lpc_acpi_match),
},
.probe = hisi_lpc_probe,
+ .remove = hisi_lpc_remove,
};
builtin_platform_driver(hisi_lpc_driver);
diff --git a/drivers/bus/imx-weim.c b/drivers/bus/imx-weim.c
index db74334ca5ef..28bb65a5613f 100644
--- a/drivers/bus/imx-weim.c
+++ b/drivers/bus/imx-weim.c
@@ -19,6 +19,8 @@ struct imx_weim_devtype {
unsigned int cs_count;
unsigned int cs_regs_count;
unsigned int cs_stride;
+ unsigned int wcr_offset;
+ unsigned int wcr_bcm;
};
static const struct imx_weim_devtype imx1_weim_devtype = {
@@ -37,6 +39,8 @@ static const struct imx_weim_devtype imx50_weim_devtype = {
.cs_count = 4,
.cs_regs_count = 6,
.cs_stride = 0x18,
+ .wcr_offset = 0x90,
+ .wcr_bcm = BIT(0),
};
static const struct imx_weim_devtype imx51_weim_devtype = {
@@ -72,7 +76,7 @@ static const struct of_device_id weim_id_table[] = {
};
MODULE_DEVICE_TABLE(of, weim_id_table);
-static int __init imx_weim_gpr_setup(struct platform_device *pdev)
+static int imx_weim_gpr_setup(struct platform_device *pdev)
{
struct device_node *np = pdev->dev.of_node;
struct property *prop;
@@ -122,10 +126,10 @@ err:
}
/* Parse and set the timing for this device. */
-static int __init weim_timing_setup(struct device *dev,
- struct device_node *np, void __iomem *base,
- const struct imx_weim_devtype *devtype,
- struct cs_timing_state *ts)
+static int weim_timing_setup(struct device *dev,
+ struct device_node *np, void __iomem *base,
+ const struct imx_weim_devtype *devtype,
+ struct cs_timing_state *ts)
{
u32 cs_idx, value[MAX_CS_REGS_COUNT];
int i, ret;
@@ -183,8 +187,7 @@ static int __init weim_timing_setup(struct device *dev,
return 0;
}
-static int __init weim_parse_dt(struct platform_device *pdev,
- void __iomem *base)
+static int weim_parse_dt(struct platform_device *pdev, void __iomem *base)
{
const struct of_device_id *of_id = of_match_device(weim_id_table,
&pdev->dev);
@@ -192,6 +195,7 @@ static int __init weim_parse_dt(struct platform_device *pdev,
struct device_node *child;
int ret, have_child = 0;
struct cs_timing_state ts = {};
+ u32 reg;
if (devtype == &imx50_weim_devtype) {
ret = imx_weim_gpr_setup(pdev);
@@ -199,6 +203,17 @@ static int __init weim_parse_dt(struct platform_device *pdev,
return ret;
}
+ if (of_property_read_bool(pdev->dev.of_node, "fsl,burst-clk-enable")) {
+ if (devtype->wcr_bcm) {
+ reg = readl(base + devtype->wcr_offset);
+ writel(reg | devtype->wcr_bcm,
+ base + devtype->wcr_offset);
+ } else {
+ dev_err(&pdev->dev, "burst clk mode not supported.\n");
+ return -EINVAL;
+ }
+ }
+
for_each_available_child_of_node(pdev->dev.of_node, child) {
ret = weim_timing_setup(&pdev->dev, child, base, devtype, &ts);
if (ret)
@@ -217,7 +232,7 @@ static int __init weim_parse_dt(struct platform_device *pdev,
return ret;
}
-static int __init weim_probe(struct platform_device *pdev)
+static int weim_probe(struct platform_device *pdev)
{
struct resource *res;
struct clk *clk;
@@ -254,8 +269,9 @@ static struct platform_driver weim_driver = {
.name = "imx-weim",
.of_match_table = weim_id_table,
},
+ .probe = weim_probe,
};
-module_platform_driver_probe(weim_driver, weim_probe);
+module_platform_driver(weim_driver);
MODULE_AUTHOR("Freescale Semiconductor Inc.");
MODULE_DESCRIPTION("i.MX EIM Controller Driver");
diff --git a/drivers/bus/moxtet.c b/drivers/bus/moxtet.c
new file mode 100644
index 000000000000..36cf13eee6b8
--- /dev/null
+++ b/drivers/bus/moxtet.c
@@ -0,0 +1,885 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Turris Mox module configuration bus driver
+ *
+ * Copyright (C) 2019 Marek Behun <marek.behun@nic.cz>
+ */
+
+#include <dt-bindings/bus/moxtet.h>
+#include <linux/bitops.h>
+#include <linux/debugfs.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/moxtet.h>
+#include <linux/mutex.h>
+#include <linux/of_device.h>
+#include <linux/of_irq.h>
+#include <linux/spi/spi.h>
+
+/*
+ * @name: module name for sysfs
+ * @hwirq_base: base index for IRQ for this module (-1 if no IRQs)
+ * @nirqs: how many interrupts does the shift register provide
+ * @desc: module description for kernel log
+ */
+static const struct {
+ const char *name;
+ int hwirq_base;
+ int nirqs;
+ const char *desc;
+} mox_module_table[] = {
+ /* do not change order of this array! */
+ { NULL, 0, 0, NULL },
+ { "sfp", -1, 0, "MOX D (SFP cage)" },
+ { "pci", MOXTET_IRQ_PCI, 1, "MOX B (Mini-PCIe)" },
+ { "topaz", MOXTET_IRQ_TOPAZ, 1, "MOX C (4 port switch)" },
+ { "peridot", MOXTET_IRQ_PERIDOT(0), 1, "MOX E (8 port switch)" },
+ { "usb3", MOXTET_IRQ_USB3, 2, "MOX F (USB 3.0)" },
+ { "pci-bridge", -1, 0, "MOX G (Mini-PCIe bridge)" },
+};
+
+static inline bool mox_module_known(unsigned int id)
+{
+ return id >= TURRIS_MOX_MODULE_FIRST && id <= TURRIS_MOX_MODULE_LAST;
+}
+
+static inline const char *mox_module_name(unsigned int id)
+{
+ if (mox_module_known(id))
+ return mox_module_table[id].name;
+ else
+ return "unknown";
+}
+
+#define DEF_MODULE_ATTR(name, fmt, ...) \
+static ssize_t \
+module_##name##_show(struct device *dev, struct device_attribute *a, \
+ char *buf) \
+{ \
+ struct moxtet_device *mdev = to_moxtet_device(dev); \
+ return sprintf(buf, (fmt), __VA_ARGS__); \
+} \
+static DEVICE_ATTR_RO(module_##name)
+
+DEF_MODULE_ATTR(id, "0x%x\n", mdev->id);
+DEF_MODULE_ATTR(name, "%s\n", mox_module_name(mdev->id));
+DEF_MODULE_ATTR(description, "%s\n",
+ mox_module_known(mdev->id) ? mox_module_table[mdev->id].desc
+ : "");
+
+static struct attribute *moxtet_dev_attrs[] = {
+ &dev_attr_module_id.attr,
+ &dev_attr_module_name.attr,
+ &dev_attr_module_description.attr,
+ NULL,
+};
+
+static const struct attribute_group moxtet_dev_group = {
+ .attrs = moxtet_dev_attrs,
+};
+
+static const struct attribute_group *moxtet_dev_groups[] = {
+ &moxtet_dev_group,
+ NULL,
+};
+
+static int moxtet_match(struct device *dev, struct device_driver *drv)
+{
+ struct moxtet_device *mdev = to_moxtet_device(dev);
+ struct moxtet_driver *tdrv = to_moxtet_driver(drv);
+ const enum turris_mox_module_id *t;
+
+ if (of_driver_match_device(dev, drv))
+ return 1;
+
+ if (!tdrv->id_table)
+ return 0;
+
+ for (t = tdrv->id_table; *t; ++t)
+ if (*t == mdev->id)
+ return 1;
+
+ return 0;
+}
+
+struct bus_type moxtet_bus_type = {
+ .name = "moxtet",
+ .dev_groups = moxtet_dev_groups,
+ .match = moxtet_match,
+};
+EXPORT_SYMBOL_GPL(moxtet_bus_type);
+
+int __moxtet_register_driver(struct module *owner,
+ struct moxtet_driver *mdrv)
+{
+ mdrv->driver.owner = owner;
+ mdrv->driver.bus = &moxtet_bus_type;
+ return driver_register(&mdrv->driver);
+}
+EXPORT_SYMBOL_GPL(__moxtet_register_driver);
+
+static int moxtet_dev_check(struct device *dev, void *data)
+{
+ struct moxtet_device *mdev = to_moxtet_device(dev);
+ struct moxtet_device *new_dev = data;
+
+ if (mdev->moxtet == new_dev->moxtet && mdev->id == new_dev->id &&
+ mdev->idx == new_dev->idx)
+ return -EBUSY;
+ return 0;
+}
+
+static void moxtet_dev_release(struct device *dev)
+{
+ struct moxtet_device *mdev = to_moxtet_device(dev);
+
+ put_device(mdev->moxtet->dev);
+ kfree(mdev);
+}
+
+static struct moxtet_device *
+moxtet_alloc_device(struct moxtet *moxtet)
+{
+ struct moxtet_device *dev;
+
+ if (!get_device(moxtet->dev))
+ return NULL;
+
+ dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+ if (!dev) {
+ put_device(moxtet->dev);
+ return NULL;
+ }
+
+ dev->moxtet = moxtet;
+ dev->dev.parent = moxtet->dev;
+ dev->dev.bus = &moxtet_bus_type;
+ dev->dev.release = moxtet_dev_release;
+
+ device_initialize(&dev->dev);
+
+ return dev;
+}
+
+static int moxtet_add_device(struct moxtet_device *dev)
+{
+ static DEFINE_MUTEX(add_mutex);
+ int ret;
+
+ if (dev->idx >= TURRIS_MOX_MAX_MODULES || dev->id > 0xf)
+ return -EINVAL;
+
+ dev_set_name(&dev->dev, "moxtet-%s.%u", mox_module_name(dev->id),
+ dev->idx);
+
+ mutex_lock(&add_mutex);
+
+ ret = bus_for_each_dev(&moxtet_bus_type, NULL, dev,
+ moxtet_dev_check);
+ if (ret)
+ goto done;
+
+ ret = device_add(&dev->dev);
+ if (ret < 0)
+ dev_err(dev->moxtet->dev, "can't add %s, status %d\n",
+ dev_name(dev->moxtet->dev), ret);
+
+done:
+ mutex_unlock(&add_mutex);
+ return ret;
+}
+
+static int __unregister(struct device *dev, void *null)
+{
+ if (dev->of_node) {
+ of_node_clear_flag(dev->of_node, OF_POPULATED);
+ of_node_put(dev->of_node);
+ }
+
+ device_unregister(dev);
+
+ return 0;
+}
+
+static struct moxtet_device *
+of_register_moxtet_device(struct moxtet *moxtet, struct device_node *nc)
+{
+ struct moxtet_device *dev;
+ u32 val;
+ int ret;
+
+ dev = moxtet_alloc_device(moxtet);
+ if (!dev) {
+ dev_err(moxtet->dev,
+ "Moxtet device alloc error for %pOF\n", nc);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ ret = of_property_read_u32(nc, "reg", &val);
+ if (ret) {
+ dev_err(moxtet->dev, "%pOF has no valid 'reg' property (%d)\n",
+ nc, ret);
+ goto err_put;
+ }
+
+ dev->idx = val;
+
+ if (dev->idx >= TURRIS_MOX_MAX_MODULES) {
+ dev_err(moxtet->dev, "%pOF Moxtet address 0x%x out of range\n",
+ nc, dev->idx);
+ ret = -EINVAL;
+ goto err_put;
+ }
+
+ dev->id = moxtet->modules[dev->idx];
+
+ if (!dev->id) {
+ dev_err(moxtet->dev, "%pOF Moxtet address 0x%x is empty\n", nc,
+ dev->idx);
+ ret = -ENODEV;
+ goto err_put;
+ }
+
+ of_node_get(nc);
+ dev->dev.of_node = nc;
+
+ ret = moxtet_add_device(dev);
+ if (ret) {
+ dev_err(moxtet->dev,
+ "Moxtet device register error for %pOF\n", nc);
+ of_node_put(nc);
+ goto err_put;
+ }
+
+ return dev;
+
+err_put:
+ put_device(&dev->dev);
+ return ERR_PTR(ret);
+}
+
+static void of_register_moxtet_devices(struct moxtet *moxtet)
+{
+ struct moxtet_device *dev;
+ struct device_node *nc;
+
+ if (!moxtet->dev->of_node)
+ return;
+
+ for_each_available_child_of_node(moxtet->dev->of_node, nc) {
+ if (of_node_test_and_set_flag(nc, OF_POPULATED))
+ continue;
+ dev = of_register_moxtet_device(moxtet, nc);
+ if (IS_ERR(dev)) {
+ dev_warn(moxtet->dev,
+ "Failed to create Moxtet device for %pOF\n",
+ nc);
+ of_node_clear_flag(nc, OF_POPULATED);
+ }
+ }
+}
+
+static void
+moxtet_register_devices_from_topology(struct moxtet *moxtet)
+{
+ struct moxtet_device *dev;
+ int i, ret;
+
+ for (i = 0; i < moxtet->count; ++i) {
+ dev = moxtet_alloc_device(moxtet);
+ if (!dev) {
+ dev_err(moxtet->dev, "Moxtet device %u alloc error\n",
+ i);
+ continue;
+ }
+
+ dev->idx = i;
+ dev->id = moxtet->modules[i];
+
+ ret = moxtet_add_device(dev);
+ if (ret && ret != -EBUSY) {
+ put_device(&dev->dev);
+ dev_err(moxtet->dev,
+ "Moxtet device %u register error: %i\n", i,
+ ret);
+ }
+ }
+}
+
+/*
+ * @nsame: how many modules with same id are already in moxtet->modules
+ */
+static int moxtet_set_irq(struct moxtet *moxtet, int idx, int id, int nsame)
+{
+ int i, first;
+ struct moxtet_irqpos *pos;
+
+ first = mox_module_table[id].hwirq_base +
+ nsame * mox_module_table[id].nirqs;
+
+ if (first + mox_module_table[id].nirqs > MOXTET_NIRQS)
+ return -EINVAL;
+
+ for (i = 0; i < mox_module_table[id].nirqs; ++i) {
+ pos = &moxtet->irq.position[first + i];
+ pos->idx = idx;
+ pos->bit = i;
+ moxtet->irq.exists |= BIT(first + i);
+ }
+
+ return 0;
+}
+
+static int moxtet_find_topology(struct moxtet *moxtet)
+{
+ u8 buf[TURRIS_MOX_MAX_MODULES];
+ int cnts[TURRIS_MOX_MODULE_LAST];
+ int i, ret;
+
+ memset(cnts, 0, sizeof(cnts));
+
+ ret = spi_read(to_spi_device(moxtet->dev), buf, TURRIS_MOX_MAX_MODULES);
+ if (ret < 0)
+ return ret;
+
+ if (buf[0] == TURRIS_MOX_CPU_ID_EMMC) {
+ dev_info(moxtet->dev, "Found MOX A (eMMC CPU) module\n");
+ } else if (buf[0] == TURRIS_MOX_CPU_ID_SD) {
+ dev_info(moxtet->dev, "Found MOX A (CPU) module\n");
+ } else {
+ dev_err(moxtet->dev, "Invalid Turris MOX A CPU module 0x%02x\n",
+ buf[0]);
+ return -ENODEV;
+ }
+
+ moxtet->count = 0;
+
+ for (i = 1; i < TURRIS_MOX_MAX_MODULES; ++i) {
+ int id;
+
+ if (buf[i] == 0xff)
+ break;
+
+ id = buf[i] & 0xf;
+
+ moxtet->modules[i-1] = id;
+ ++moxtet->count;
+
+ if (mox_module_known(id)) {
+ dev_info(moxtet->dev, "Found %s module\n",
+ mox_module_table[id].desc);
+
+ if (moxtet_set_irq(moxtet, i-1, id, cnts[id]++) < 0)
+ dev_err(moxtet->dev,
+ " Cannot set IRQ for module %s\n",
+ mox_module_table[id].desc);
+ } else {
+ dev_warn(moxtet->dev,
+ "Unknown Moxtet module found (ID 0x%02x)\n",
+ id);
+ }
+ }
+
+ return 0;
+}
+
+static int moxtet_spi_read(struct moxtet *moxtet, u8 *buf)
+{
+ struct spi_transfer xfer = {
+ .rx_buf = buf,
+ .tx_buf = moxtet->tx,
+ .len = moxtet->count + 1
+ };
+ int ret;
+
+ mutex_lock(&moxtet->lock);
+
+ ret = spi_sync_transfer(to_spi_device(moxtet->dev), &xfer, 1);
+
+ mutex_unlock(&moxtet->lock);
+
+ return ret;
+}
+
+int moxtet_device_read(struct device *dev)
+{
+ struct moxtet_device *mdev = to_moxtet_device(dev);
+ struct moxtet *moxtet = mdev->moxtet;
+ u8 buf[TURRIS_MOX_MAX_MODULES];
+ int ret;
+
+ if (mdev->idx >= moxtet->count)
+ return -EINVAL;
+
+ ret = moxtet_spi_read(moxtet, buf);
+ if (ret < 0)
+ return ret;
+
+ return buf[mdev->idx + 1] >> 4;
+}
+EXPORT_SYMBOL_GPL(moxtet_device_read);
+
+int moxtet_device_write(struct device *dev, u8 val)
+{
+ struct moxtet_device *mdev = to_moxtet_device(dev);
+ struct moxtet *moxtet = mdev->moxtet;
+ int ret;
+
+ if (mdev->idx >= moxtet->count)
+ return -EINVAL;
+
+ mutex_lock(&moxtet->lock);
+
+ moxtet->tx[moxtet->count - mdev->idx] = val;
+
+ ret = spi_write(to_spi_device(moxtet->dev), moxtet->tx,
+ moxtet->count + 1);
+
+ mutex_unlock(&moxtet->lock);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(moxtet_device_write);
+
+int moxtet_device_written(struct device *dev)
+{
+ struct moxtet_device *mdev = to_moxtet_device(dev);
+ struct moxtet *moxtet = mdev->moxtet;
+
+ if (mdev->idx >= moxtet->count)
+ return -EINVAL;
+
+ return moxtet->tx[moxtet->count - mdev->idx];
+}
+EXPORT_SYMBOL_GPL(moxtet_device_written);
+
+#ifdef CONFIG_DEBUG_FS
+static int moxtet_debug_open(struct inode *inode, struct file *file)
+{
+ file->private_data = inode->i_private;
+
+ return nonseekable_open(inode, file);
+}
+
+static ssize_t input_read(struct file *file, char __user *buf, size_t len,
+ loff_t *ppos)
+{
+ struct moxtet *moxtet = file->private_data;
+ u8 bin[TURRIS_MOX_MAX_MODULES];
+ u8 hex[sizeof(buf) * 2 + 1];
+ int ret, n;
+
+ ret = moxtet_spi_read(moxtet, bin);
+ if (ret < 0)
+ return ret;
+
+ n = moxtet->count + 1;
+ bin2hex(hex, bin, n);
+
+ hex[2*n] = '\n';
+
+ return simple_read_from_buffer(buf, len, ppos, hex, 2*n + 1);
+}
+
+static const struct file_operations input_fops = {
+ .owner = THIS_MODULE,
+ .open = moxtet_debug_open,
+ .read = input_read,
+ .llseek = no_llseek,
+};
+
+static ssize_t output_read(struct file *file, char __user *buf, size_t len,
+ loff_t *ppos)
+{
+ struct moxtet *moxtet = file->private_data;
+ u8 hex[TURRIS_MOX_MAX_MODULES * 2 + 1];
+ u8 *p = hex;
+ int i;
+
+ mutex_lock(&moxtet->lock);
+
+ for (i = 0; i < moxtet->count; ++i)
+ p = hex_byte_pack(p, moxtet->tx[moxtet->count - i]);
+
+ mutex_unlock(&moxtet->lock);
+
+ *p++ = '\n';
+
+ return simple_read_from_buffer(buf, len, ppos, hex, p - hex);
+}
+
+static ssize_t output_write(struct file *file, const char __user *buf,
+ size_t len, loff_t *ppos)
+{
+ struct moxtet *moxtet = file->private_data;
+ u8 bin[TURRIS_MOX_MAX_MODULES];
+ u8 hex[sizeof(bin) * 2 + 1];
+ ssize_t res;
+ loff_t dummy = 0;
+ int err, i;
+
+ if (len > 2 * moxtet->count + 1 || len < 2 * moxtet->count)
+ return -EINVAL;
+
+ res = simple_write_to_buffer(hex, sizeof(hex), &dummy, buf, len);
+ if (res < 0)
+ return res;
+
+ if (len % 2 == 1 && hex[len - 1] != '\n')
+ return -EINVAL;
+
+ err = hex2bin(bin, hex, moxtet->count);
+ if (err < 0)
+ return -EINVAL;
+
+ mutex_lock(&moxtet->lock);
+
+ for (i = 0; i < moxtet->count; ++i)
+ moxtet->tx[moxtet->count - i] = bin[i];
+
+ err = spi_write(to_spi_device(moxtet->dev), moxtet->tx,
+ moxtet->count + 1);
+
+ mutex_unlock(&moxtet->lock);
+
+ return err < 0 ? err : len;
+}
+
+static const struct file_operations output_fops = {
+ .owner = THIS_MODULE,
+ .open = moxtet_debug_open,
+ .read = output_read,
+ .write = output_write,
+ .llseek = no_llseek,
+};
+
+static int moxtet_register_debugfs(struct moxtet *moxtet)
+{
+ struct dentry *root, *entry;
+
+ root = debugfs_create_dir("moxtet", NULL);
+
+ if (IS_ERR(root))
+ return PTR_ERR(root);
+
+ entry = debugfs_create_file_unsafe("input", 0444, root, moxtet,
+ &input_fops);
+ if (IS_ERR(entry))
+ goto err_remove;
+
+ entry = debugfs_create_file_unsafe("output", 0644, root, moxtet,
+ &output_fops);
+ if (IS_ERR(entry))
+ goto err_remove;
+
+ moxtet->debugfs_root = root;
+
+ return 0;
+err_remove:
+ debugfs_remove_recursive(root);
+ return PTR_ERR(entry);
+}
+
+static void moxtet_unregister_debugfs(struct moxtet *moxtet)
+{
+ debugfs_remove_recursive(moxtet->debugfs_root);
+}
+#else
+static inline int moxtet_register_debugfs(struct moxtet *moxtet)
+{
+ return 0;
+}
+
+static inline void moxtet_unregister_debugfs(struct moxtet *moxtet)
+{
+}
+#endif
+
+static int moxtet_irq_domain_map(struct irq_domain *d, unsigned int irq,
+ irq_hw_number_t hw)
+{
+ struct moxtet *moxtet = d->host_data;
+
+ if (hw >= MOXTET_NIRQS || !(moxtet->irq.exists & BIT(hw))) {
+ dev_err(moxtet->dev, "Invalid hw irq number\n");
+ return -EINVAL;
+ }
+
+ irq_set_chip_data(irq, d->host_data);
+ irq_set_chip_and_handler(irq, &moxtet->irq.chip, handle_level_irq);
+
+ return 0;
+}
+
+static int moxtet_irq_domain_xlate(struct irq_domain *d,
+ struct device_node *ctrlr,
+ const u32 *intspec, unsigned int intsize,
+ unsigned long *out_hwirq,
+ unsigned int *out_type)
+{
+ struct moxtet *moxtet = d->host_data;
+ int irq;
+
+ if (WARN_ON(intsize < 1))
+ return -EINVAL;
+
+ irq = intspec[0];
+
+ if (irq >= MOXTET_NIRQS || !(moxtet->irq.exists & BIT(irq)))
+ return -EINVAL;
+
+ *out_hwirq = irq;
+ *out_type = IRQ_TYPE_NONE;
+ return 0;
+}
+
+static const struct irq_domain_ops moxtet_irq_domain = {
+ .map = moxtet_irq_domain_map,
+ .xlate = moxtet_irq_domain_xlate,
+};
+
+static void moxtet_irq_mask(struct irq_data *d)
+{
+ struct moxtet *moxtet = irq_data_get_irq_chip_data(d);
+
+ moxtet->irq.masked |= BIT(d->hwirq);
+}
+
+static void moxtet_irq_unmask(struct irq_data *d)
+{
+ struct moxtet *moxtet = irq_data_get_irq_chip_data(d);
+
+ moxtet->irq.masked &= ~BIT(d->hwirq);
+}
+
+static void moxtet_irq_print_chip(struct irq_data *d, struct seq_file *p)
+{
+ struct moxtet *moxtet = irq_data_get_irq_chip_data(d);
+ struct moxtet_irqpos *pos = &moxtet->irq.position[d->hwirq];
+ int id;
+
+ id = moxtet->modules[pos->idx];
+
+ seq_printf(p, " moxtet-%s.%i#%i", mox_module_name(id), pos->idx,
+ pos->bit);
+}
+
+static const struct irq_chip moxtet_irq_chip = {
+ .name = "moxtet",
+ .irq_mask = moxtet_irq_mask,
+ .irq_unmask = moxtet_irq_unmask,
+ .irq_print_chip = moxtet_irq_print_chip,
+};
+
+static int moxtet_irq_read(struct moxtet *moxtet, unsigned long *map)
+{
+ struct moxtet_irqpos *pos = moxtet->irq.position;
+ u8 buf[TURRIS_MOX_MAX_MODULES];
+ int i, ret;
+
+ ret = moxtet_spi_read(moxtet, buf);
+ if (ret < 0)
+ return ret;
+
+ *map = 0;
+
+ for_each_set_bit(i, &moxtet->irq.exists, MOXTET_NIRQS) {
+ if (!(buf[pos[i].idx + 1] & BIT(4 + pos[i].bit)))
+ set_bit(i, map);
+ }
+
+ return 0;
+}
+
+static irqreturn_t moxtet_irq_thread_fn(int irq, void *data)
+{
+ struct moxtet *moxtet = data;
+ unsigned long set;
+ int nhandled = 0, i, sub_irq, ret;
+
+ ret = moxtet_irq_read(moxtet, &set);
+ if (ret < 0)
+ goto out;
+
+ set &= ~moxtet->irq.masked;
+
+ do {
+ for_each_set_bit(i, &set, MOXTET_NIRQS) {
+ sub_irq = irq_find_mapping(moxtet->irq.domain, i);
+ handle_nested_irq(sub_irq);
+ dev_dbg(moxtet->dev, "%i irq\n", i);
+ ++nhandled;
+ }
+
+ ret = moxtet_irq_read(moxtet, &set);
+ if (ret < 0)
+ goto out;
+
+ set &= ~moxtet->irq.masked;
+ } while (set);
+
+out:
+ return (nhandled > 0 ? IRQ_HANDLED : IRQ_NONE);
+}
+
+static void moxtet_irq_free(struct moxtet *moxtet)
+{
+ int i, irq;
+
+ for (i = 0; i < MOXTET_NIRQS; ++i) {
+ if (moxtet->irq.exists & BIT(i)) {
+ irq = irq_find_mapping(moxtet->irq.domain, i);
+ irq_dispose_mapping(irq);
+ }
+ }
+
+ irq_domain_remove(moxtet->irq.domain);
+}
+
+static int moxtet_irq_setup(struct moxtet *moxtet)
+{
+ int i, ret;
+
+ moxtet->irq.domain = irq_domain_add_simple(moxtet->dev->of_node,
+ MOXTET_NIRQS, 0,
+ &moxtet_irq_domain, moxtet);
+ if (moxtet->irq.domain == NULL) {
+ dev_err(moxtet->dev, "Could not add IRQ domain\n");
+ return -ENOMEM;
+ }
+
+ for (i = 0; i < MOXTET_NIRQS; ++i)
+ if (moxtet->irq.exists & BIT(i))
+ irq_create_mapping(moxtet->irq.domain, i);
+
+ moxtet->irq.chip = moxtet_irq_chip;
+ moxtet->irq.masked = ~0;
+
+ ret = request_threaded_irq(moxtet->dev_irq, NULL, moxtet_irq_thread_fn,
+ IRQF_ONESHOT, "moxtet", moxtet);
+ if (ret < 0)
+ goto err_free;
+
+ return 0;
+
+err_free:
+ moxtet_irq_free(moxtet);
+ return ret;
+}
+
+static int moxtet_probe(struct spi_device *spi)
+{
+ struct moxtet *moxtet;
+ int ret;
+
+ ret = spi_setup(spi);
+ if (ret < 0)
+ return ret;
+
+ moxtet = devm_kzalloc(&spi->dev, sizeof(struct moxtet),
+ GFP_KERNEL);
+ if (!moxtet)
+ return -ENOMEM;
+
+ moxtet->dev = &spi->dev;
+ spi_set_drvdata(spi, moxtet);
+
+ mutex_init(&moxtet->lock);
+
+ moxtet->dev_irq = of_irq_get(moxtet->dev->of_node, 0);
+ if (moxtet->dev_irq == -EPROBE_DEFER)
+ return -EPROBE_DEFER;
+
+ if (moxtet->dev_irq <= 0) {
+ dev_err(moxtet->dev, "No IRQ resource found\n");
+ return -ENXIO;
+ }
+
+ ret = moxtet_find_topology(moxtet);
+ if (ret < 0)
+ return ret;
+
+ if (moxtet->irq.exists) {
+ ret = moxtet_irq_setup(moxtet);
+ if (ret < 0)
+ return ret;
+ }
+
+ of_register_moxtet_devices(moxtet);
+ moxtet_register_devices_from_topology(moxtet);
+
+ ret = moxtet_register_debugfs(moxtet);
+ if (ret < 0)
+ dev_warn(moxtet->dev, "Failed creating debugfs entries: %i\n",
+ ret);
+
+ return 0;
+}
+
+static int moxtet_remove(struct spi_device *spi)
+{
+ struct moxtet *moxtet = spi_get_drvdata(spi);
+
+ free_irq(moxtet->dev_irq, moxtet);
+
+ moxtet_irq_free(moxtet);
+
+ moxtet_unregister_debugfs(moxtet);
+
+ device_for_each_child(moxtet->dev, NULL, __unregister);
+
+ mutex_destroy(&moxtet->lock);
+
+ return 0;
+}
+
+static const struct of_device_id moxtet_dt_ids[] = {
+ { .compatible = "cznic,moxtet" },
+ {},
+};
+MODULE_DEVICE_TABLE(of, moxtet_dt_ids);
+
+static struct spi_driver moxtet_spi_driver = {
+ .driver = {
+ .name = "moxtet",
+ .of_match_table = moxtet_dt_ids,
+ },
+ .probe = moxtet_probe,
+ .remove = moxtet_remove,
+};
+
+static int __init moxtet_init(void)
+{
+ int ret;
+
+ ret = bus_register(&moxtet_bus_type);
+ if (ret < 0) {
+ pr_err("moxtet bus registration failed: %d\n", ret);
+ goto error;
+ }
+
+ ret = spi_register_driver(&moxtet_spi_driver);
+ if (ret < 0) {
+ pr_err("moxtet spi driver registration failed: %d\n", ret);
+ goto error_bus;
+ }
+
+ return 0;
+
+error_bus:
+ bus_unregister(&moxtet_bus_type);
+error:
+ return ret;
+}
+postcore_initcall_sync(moxtet_init);
+
+static void __exit moxtet_exit(void)
+{
+ spi_unregister_driver(&moxtet_spi_driver);
+ bus_unregister(&moxtet_bus_type);
+}
+module_exit(moxtet_exit);
+
+MODULE_AUTHOR("Marek Behun <marek.behun@nic.cz>");
+MODULE_DESCRIPTION("CZ.NIC's Turris Mox module configuration bus");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/bus/sunxi-rsb.c b/drivers/bus/sunxi-rsb.c
index 1b76d9585902..be79d6c6a4e4 100644
--- a/drivers/bus/sunxi-rsb.c
+++ b/drivers/bus/sunxi-rsb.c
@@ -651,10 +651,8 @@ static int sunxi_rsb_probe(struct platform_device *pdev)
return PTR_ERR(rsb->regs);
irq = platform_get_irq(pdev, 0);
- if (irq < 0) {
- dev_err(dev, "failed to retrieve irq: %d\n", irq);
+ if (irq < 0)
return irq;
- }
rsb->clk = devm_clk_get(dev, NULL);
if (IS_ERR(rsb->clk)) {
diff --git a/drivers/bus/ti-sysc.c b/drivers/bus/ti-sysc.c
index b72741668c92..2db474ab4c6b 100644
--- a/drivers/bus/ti-sysc.c
+++ b/drivers/bus/ti-sysc.c
@@ -71,6 +71,9 @@ static const char * const clock_names[SYSC_MAX_CLOCKS] = {
* @name: name if available
* @revision: interconnect target module revision
* @needs_resume: runtime resume needed on resume from suspend
+ * @clk_enable_quirk: module specific clock enable quirk
+ * @clk_disable_quirk: module specific clock disable quirk
+ * @reset_done_quirk: module specific reset done quirk
*/
struct sysc {
struct device *dev;
@@ -89,10 +92,14 @@ struct sysc {
struct ti_sysc_cookie cookie;
const char *name;
u32 revision;
- bool enabled;
- bool needs_resume;
- bool child_needs_resume;
+ unsigned int enabled:1;
+ unsigned int needs_resume:1;
+ unsigned int child_needs_resume:1;
+ unsigned int disable_on_idle:1;
struct delayed_work idle_work;
+ void (*clk_enable_quirk)(struct sysc *sysc);
+ void (*clk_disable_quirk)(struct sysc *sysc);
+ void (*reset_done_quirk)(struct sysc *sysc);
};
static void sysc_parse_dts_quirks(struct sysc *ddata, struct device_node *np,
@@ -100,6 +107,20 @@ static void sysc_parse_dts_quirks(struct sysc *ddata, struct device_node *np,
static void sysc_write(struct sysc *ddata, int offset, u32 value)
{
+ if (ddata->cfg.quirks & SYSC_QUIRK_16BIT) {
+ writew_relaxed(value & 0xffff, ddata->module_va + offset);
+
+ /* Only i2c revision has LO and HI register with stride of 4 */
+ if (ddata->offsets[SYSC_REVISION] >= 0 &&
+ offset == ddata->offsets[SYSC_REVISION]) {
+ u16 hi = value >> 16;
+
+ writew_relaxed(hi, ddata->module_va + offset + 4);
+ }
+
+ return;
+ }
+
writel_relaxed(value, ddata->module_va + offset);
}
@@ -109,7 +130,14 @@ static u32 sysc_read(struct sysc *ddata, int offset)
u32 val;
val = readw_relaxed(ddata->module_va + offset);
- val |= (readw_relaxed(ddata->module_va + offset + 4) << 16);
+
+ /* Only i2c revision has LO and HI register with stride of 4 */
+ if (ddata->offsets[SYSC_REVISION] >= 0 &&
+ offset == ddata->offsets[SYSC_REVISION]) {
+ u16 tmp = readw_relaxed(ddata->module_va + offset + 4);
+
+ val |= tmp << 16;
+ }
return val;
}
@@ -132,6 +160,26 @@ static u32 sysc_read_revision(struct sysc *ddata)
return sysc_read(ddata, offset);
}
+static u32 sysc_read_sysconfig(struct sysc *ddata)
+{
+ int offset = ddata->offsets[SYSC_SYSCONFIG];
+
+ if (offset < 0)
+ return 0;
+
+ return sysc_read(ddata, offset);
+}
+
+static u32 sysc_read_sysstatus(struct sysc *ddata)
+{
+ int offset = ddata->offsets[SYSC_SYSSTATUS];
+
+ if (offset < 0)
+ return 0;
+
+ return sysc_read(ddata, offset);
+}
+
static int sysc_add_named_clock_from_child(struct sysc *ddata,
const char *name,
const char *optfck_name)
@@ -422,6 +470,30 @@ static void sysc_disable_opt_clocks(struct sysc *ddata)
}
}
+static void sysc_clkdm_deny_idle(struct sysc *ddata)
+{
+ struct ti_sysc_platform_data *pdata;
+
+ if (ddata->legacy_mode)
+ return;
+
+ pdata = dev_get_platdata(ddata->dev);
+ if (pdata && pdata->clkdm_deny_idle)
+ pdata->clkdm_deny_idle(ddata->dev, &ddata->cookie);
+}
+
+static void sysc_clkdm_allow_idle(struct sysc *ddata)
+{
+ struct ti_sysc_platform_data *pdata;
+
+ if (ddata->legacy_mode)
+ return;
+
+ pdata = dev_get_platdata(ddata->dev);
+ if (pdata && pdata->clkdm_allow_idle)
+ pdata->clkdm_allow_idle(ddata->dev, &ddata->cookie);
+}
+
/**
* sysc_init_resets - init rstctrl reset line if configured
* @ddata: device driver data
@@ -431,7 +503,7 @@ static void sysc_disable_opt_clocks(struct sysc *ddata)
static int sysc_init_resets(struct sysc *ddata)
{
ddata->rsts =
- devm_reset_control_array_get_optional_exclusive(ddata->dev);
+ devm_reset_control_get_optional(ddata->dev, "rstctrl");
if (IS_ERR(ddata->rsts))
return PTR_ERR(ddata->rsts);
@@ -694,8 +766,11 @@ static int sysc_ioremap(struct sysc *ddata)
ddata->offsets[SYSC_SYSCONFIG],
ddata->offsets[SYSC_SYSSTATUS]);
+ if (size < SZ_1K)
+ size = SZ_1K;
+
if ((size + sizeof(u32)) > ddata->module_size)
- return -EINVAL;
+ size = ddata->module_size;
}
ddata->module_va = devm_ioremap(ddata->dev,
@@ -794,7 +869,9 @@ static void sysc_show_registers(struct sysc *ddata)
}
#define SYSC_IDLE_MASK (SYSC_NR_IDLEMODES - 1)
+#define SYSC_CLOCACT_ICK 2
+/* Caller needs to manage sysc_clkdm_deny_idle() and sysc_clkdm_allow_idle() */
static int sysc_enable_module(struct device *dev)
{
struct sysc *ddata;
@@ -805,23 +882,34 @@ static int sysc_enable_module(struct device *dev)
if (ddata->offsets[SYSC_SYSCONFIG] == -ENODEV)
return 0;
- /*
- * TODO: Need to prevent clockdomain autoidle?
- * See clkdm_deny_idle() in arch/mach-omap2/omap_hwmod.c
- */
-
regbits = ddata->cap->regbits;
reg = sysc_read(ddata, ddata->offsets[SYSC_SYSCONFIG]);
+ /* Set CLOCKACTIVITY, we only use it for ick */
+ if (regbits->clkact_shift >= 0 &&
+ (ddata->cfg.quirks & SYSC_QUIRK_USE_CLOCKACT ||
+ ddata->cfg.sysc_val & BIT(regbits->clkact_shift)))
+ reg |= SYSC_CLOCACT_ICK << regbits->clkact_shift;
+
/* Set SIDLE mode */
idlemodes = ddata->cfg.sidlemodes;
if (!idlemodes || regbits->sidle_shift < 0)
goto set_midle;
- best_mode = fls(ddata->cfg.sidlemodes) - 1;
- if (best_mode > SYSC_IDLE_MASK) {
- dev_err(dev, "%s: invalid sidlemode\n", __func__);
- return -EINVAL;
+ if (ddata->cfg.quirks & (SYSC_QUIRK_SWSUP_SIDLE |
+ SYSC_QUIRK_SWSUP_SIDLE_ACT)) {
+ best_mode = SYSC_IDLE_NO;
+ } else {
+ best_mode = fls(ddata->cfg.sidlemodes) - 1;
+ if (best_mode > SYSC_IDLE_MASK) {
+ dev_err(dev, "%s: invalid sidlemode\n", __func__);
+ return -EINVAL;
+ }
+
+ /* Set WAKEUP */
+ if (regbits->enwkup_shift >= 0 &&
+ ddata->cfg.sysc_val & BIT(regbits->enwkup_shift))
+ reg |= BIT(regbits->enwkup_shift);
}
reg &= ~(SYSC_IDLE_MASK << regbits->sidle_shift);
@@ -832,7 +920,7 @@ set_midle:
/* Set MIDLE mode */
idlemodes = ddata->cfg.midlemodes;
if (!idlemodes || regbits->midle_shift < 0)
- return 0;
+ goto set_autoidle;
best_mode = fls(ddata->cfg.midlemodes) - 1;
if (best_mode > SYSC_IDLE_MASK) {
@@ -844,6 +932,14 @@ set_midle:
reg |= best_mode << regbits->midle_shift;
sysc_write(ddata, ddata->offsets[SYSC_SYSCONFIG], reg);
+set_autoidle:
+ /* Autoidle bit must enabled separately if available */
+ if (regbits->autoidle_shift >= 0 &&
+ ddata->cfg.sysc_val & BIT(regbits->autoidle_shift)) {
+ reg |= 1 << regbits->autoidle_shift;
+ sysc_write(ddata, ddata->offsets[SYSC_SYSCONFIG], reg);
+ }
+
return 0;
}
@@ -853,7 +949,7 @@ static int sysc_best_idle_mode(u32 idlemodes, u32 *best_mode)
*best_mode = SYSC_IDLE_SMART_WKUP;
else if (idlemodes & BIT(SYSC_IDLE_SMART))
*best_mode = SYSC_IDLE_SMART;
- else if (idlemodes & SYSC_IDLE_FORCE)
+ else if (idlemodes & BIT(SYSC_IDLE_FORCE))
*best_mode = SYSC_IDLE_FORCE;
else
return -EINVAL;
@@ -861,6 +957,7 @@ static int sysc_best_idle_mode(u32 idlemodes, u32 *best_mode)
return 0;
}
+/* Caller needs to manage sysc_clkdm_deny_idle() and sysc_clkdm_allow_idle() */
static int sysc_disable_module(struct device *dev)
{
struct sysc *ddata;
@@ -872,11 +969,6 @@ static int sysc_disable_module(struct device *dev)
if (ddata->offsets[SYSC_SYSCONFIG] == -ENODEV)
return 0;
- /*
- * TODO: Need to prevent clockdomain autoidle?
- * See clkdm_deny_idle() in arch/mach-omap2/omap_hwmod.c
- */
-
regbits = ddata->cap->regbits;
reg = sysc_read(ddata, ddata->offsets[SYSC_SYSCONFIG]);
@@ -901,14 +993,21 @@ set_sidle:
if (!idlemodes || regbits->sidle_shift < 0)
return 0;
- ret = sysc_best_idle_mode(idlemodes, &best_mode);
- if (ret) {
- dev_err(dev, "%s: invalid sidlemode\n", __func__);
- return ret;
+ if (ddata->cfg.quirks & SYSC_QUIRK_SWSUP_SIDLE) {
+ best_mode = SYSC_IDLE_FORCE;
+ } else {
+ ret = sysc_best_idle_mode(idlemodes, &best_mode);
+ if (ret) {
+ dev_err(dev, "%s: invalid sidlemode\n", __func__);
+ return ret;
+ }
}
reg &= ~(SYSC_IDLE_MASK << regbits->sidle_shift);
reg |= best_mode << regbits->sidle_shift;
+ if (regbits->autoidle_shift >= 0 &&
+ ddata->cfg.sysc_val & BIT(regbits->autoidle_shift))
+ reg |= 1 << regbits->autoidle_shift;
sysc_write(ddata, ddata->offsets[SYSC_SYSCONFIG], reg);
return 0;
@@ -932,6 +1031,9 @@ static int __maybe_unused sysc_runtime_suspend_legacy(struct device *dev,
dev_err(dev, "%s: could not idle: %i\n",
__func__, error);
+ if (ddata->disable_on_idle)
+ reset_control_assert(ddata->rsts);
+
return 0;
}
@@ -941,6 +1043,9 @@ static int __maybe_unused sysc_runtime_resume_legacy(struct device *dev,
struct ti_sysc_platform_data *pdata;
int error;
+ if (ddata->disable_on_idle)
+ reset_control_deassert(ddata->rsts);
+
pdata = dev_get_platdata(ddata->dev);
if (!pdata)
return 0;
@@ -966,14 +1071,16 @@ static int __maybe_unused sysc_runtime_suspend(struct device *dev)
if (!ddata->enabled)
return 0;
+ sysc_clkdm_deny_idle(ddata);
+
if (ddata->legacy_mode) {
error = sysc_runtime_suspend_legacy(dev, ddata);
if (error)
- return error;
+ goto err_allow_idle;
} else {
error = sysc_disable_module(dev);
if (error)
- return error;
+ goto err_allow_idle;
}
sysc_disable_main_clocks(ddata);
@@ -983,6 +1090,12 @@ static int __maybe_unused sysc_runtime_suspend(struct device *dev)
ddata->enabled = false;
+err_allow_idle:
+ sysc_clkdm_allow_idle(ddata);
+
+ if (ddata->disable_on_idle)
+ reset_control_assert(ddata->rsts);
+
return error;
}
@@ -996,10 +1109,15 @@ static int __maybe_unused sysc_runtime_resume(struct device *dev)
if (ddata->enabled)
return 0;
+ if (ddata->disable_on_idle)
+ reset_control_deassert(ddata->rsts);
+
+ sysc_clkdm_deny_idle(ddata);
+
if (sysc_opt_clks_needed(ddata)) {
error = sysc_enable_opt_clocks(ddata);
if (error)
- return error;
+ goto err_allow_idle;
}
error = sysc_enable_main_clocks(ddata);
@@ -1018,6 +1136,8 @@ static int __maybe_unused sysc_runtime_resume(struct device *dev)
ddata->enabled = true;
+ sysc_clkdm_allow_idle(ddata);
+
return 0;
err_main_clocks:
@@ -1025,6 +1145,8 @@ err_main_clocks:
err_opt_clocks:
if (sysc_opt_clks_needed(ddata))
sysc_disable_opt_clocks(ddata);
+err_allow_idle:
+ sysc_clkdm_allow_idle(ddata);
return error;
}
@@ -1106,8 +1228,10 @@ static const struct sysc_revision_quirk sysc_revision_quirks[] = {
0),
SYSC_QUIRK("timer", 0, 0, 0x10, -1, 0x4fff1301, 0xffff00ff,
0),
+ SYSC_QUIRK("uart", 0, 0x50, 0x54, 0x58, 0x00000046, 0xffffffff,
+ SYSC_QUIRK_SWSUP_SIDLE | SYSC_QUIRK_LEGACY_IDLE),
SYSC_QUIRK("uart", 0, 0x50, 0x54, 0x58, 0x00000052, 0xffffffff,
- SYSC_QUIRK_SWSUP_SIDLE_ACT | SYSC_QUIRK_LEGACY_IDLE),
+ SYSC_QUIRK_SWSUP_SIDLE | SYSC_QUIRK_LEGACY_IDLE),
/* Uarts on omap4 and later */
SYSC_QUIRK("uart", 0, 0x50, 0x54, 0x58, 0x50411e03, 0xffff00ff,
SYSC_QUIRK_SWSUP_SIDLE_ACT | SYSC_QUIRK_LEGACY_IDLE),
@@ -1119,6 +1243,22 @@ static const struct sysc_revision_quirk sysc_revision_quirks[] = {
SYSC_QUIRK_EXT_OPT_CLOCK | SYSC_QUIRK_NO_RESET_ON_INIT |
SYSC_QUIRK_SWSUP_SIDLE),
+ /* Quirks that need to be set based on detected module */
+ SYSC_QUIRK("hdq1w", 0, 0, 0x14, 0x18, 0x00000006, 0xffffffff,
+ SYSC_MODULE_QUIRK_HDQ1W),
+ SYSC_QUIRK("hdq1w", 0, 0, 0x14, 0x18, 0x0000000a, 0xffffffff,
+ SYSC_MODULE_QUIRK_HDQ1W),
+ SYSC_QUIRK("i2c", 0, 0, 0x20, 0x10, 0x00000036, 0x000000ff,
+ SYSC_MODULE_QUIRK_I2C),
+ SYSC_QUIRK("i2c", 0, 0, 0x20, 0x10, 0x0000003c, 0x000000ff,
+ SYSC_MODULE_QUIRK_I2C),
+ SYSC_QUIRK("i2c", 0, 0, 0x20, 0x10, 0x00000040, 0x000000ff,
+ SYSC_MODULE_QUIRK_I2C),
+ SYSC_QUIRK("i2c", 0, 0, 0x10, 0x90, 0x5040000a, 0xfffff0f0,
+ SYSC_MODULE_QUIRK_I2C),
+ SYSC_QUIRK("wdt", 0, 0, 0x10, 0x14, 0x502a0500, 0xfffff0f0,
+ SYSC_MODULE_QUIRK_WDT),
+
#ifdef DEBUG
SYSC_QUIRK("adc", 0, 0, 0x10, -1, 0x47300001, 0xffffffff, 0),
SYSC_QUIRK("atl", 0, 0, -1, -1, 0x0a070100, 0xffffffff, 0),
@@ -1127,16 +1267,14 @@ static const struct sysc_revision_quirk sysc_revision_quirks[] = {
SYSC_QUIRK("control", 0, 0, 0x10, -1, 0x40000900, 0xffffffff, 0),
SYSC_QUIRK("cpgmac", 0, 0x1200, 0x1208, 0x1204, 0x4edb1902,
0xffff00f0, 0),
- SYSC_QUIRK("dcan", 0, 0, -1, -1, 0xffffffff, 0xffffffff, 0),
+ SYSC_QUIRK("dcan", 0, 0x20, -1, -1, 0xa3170504, 0xffffffff, 0),
+ SYSC_QUIRK("dcan", 0, 0x20, -1, -1, 0x4edb1902, 0xffffffff, 0),
SYSC_QUIRK("dmic", 0, 0, 0x10, -1, 0x50010000, 0xffffffff, 0),
SYSC_QUIRK("dwc3", 0, 0, 0x10, -1, 0x500a0200, 0xffffffff, 0),
SYSC_QUIRK("epwmss", 0, 0, 0x4, -1, 0x47400001, 0xffffffff, 0),
SYSC_QUIRK("gpu", 0, 0x1fc00, 0x1fc10, -1, 0, 0, 0),
- SYSC_QUIRK("hdq1w", 0, 0, 0x14, 0x18, 0x00000006, 0xffffffff, 0),
- SYSC_QUIRK("hdq1w", 0, 0, 0x14, 0x18, 0x0000000a, 0xffffffff, 0),
SYSC_QUIRK("hsi", 0, 0, 0x10, 0x14, 0x50043101, 0xffffffff, 0),
SYSC_QUIRK("iss", 0, 0, 0x10, -1, 0x40000101, 0xffffffff, 0),
- SYSC_QUIRK("i2c", 0, 0, 0x10, 0x90, 0x5040000a, 0xfffff0f0, 0),
SYSC_QUIRK("lcdc", 0, 0, 0x54, -1, 0x4f201000, 0xffffffff, 0),
SYSC_QUIRK("mcasp", 0, 0, 0x4, -1, 0x44306302, 0xffffffff, 0),
SYSC_QUIRK("mcasp", 0, 0, 0x4, -1, 0x44307b02, 0xffffffff, 0),
@@ -1172,7 +1310,6 @@ static const struct sysc_revision_quirk sysc_revision_quirks[] = {
SYSC_QUIRK("usb_host_hs", 0, 0, 0x10, -1, 0x50700101, 0xffffffff, 0),
SYSC_QUIRK("usb_otg_hs", 0, 0x400, 0x404, 0x408, 0x00000050,
0xffffffff, 0),
- SYSC_QUIRK("wdt", 0, 0, 0x10, 0x14, 0x502a0500, 0xfffff0f0, 0),
SYSC_QUIRK("vfpe", 0, 0, 0x104, -1, 0x4d001200, 0xffffffff, 0),
#endif
};
@@ -1245,6 +1382,121 @@ static void sysc_init_revision_quirks(struct sysc *ddata)
}
}
+/* 1-wire needs module's internal clocks enabled for reset */
+static void sysc_clk_enable_quirk_hdq1w(struct sysc *ddata)
+{
+ int offset = 0x0c; /* HDQ_CTRL_STATUS */
+ u16 val;
+
+ val = sysc_read(ddata, offset);
+ val |= BIT(5);
+ sysc_write(ddata, offset, val);
+}
+
+/* I2C needs extra enable bit toggling for reset */
+static void sysc_clk_quirk_i2c(struct sysc *ddata, bool enable)
+{
+ int offset;
+ u16 val;
+
+ /* I2C_CON, omap2/3 is different from omap4 and later */
+ if ((ddata->revision & 0xffffff00) == 0x001f0000)
+ offset = 0x24;
+ else
+ offset = 0xa4;
+
+ /* I2C_EN */
+ val = sysc_read(ddata, offset);
+ if (enable)
+ val |= BIT(15);
+ else
+ val &= ~BIT(15);
+ sysc_write(ddata, offset, val);
+}
+
+static void sysc_clk_enable_quirk_i2c(struct sysc *ddata)
+{
+ sysc_clk_quirk_i2c(ddata, true);
+}
+
+static void sysc_clk_disable_quirk_i2c(struct sysc *ddata)
+{
+ sysc_clk_quirk_i2c(ddata, false);
+}
+
+/* Watchdog timer needs a disable sequence after reset */
+static void sysc_reset_done_quirk_wdt(struct sysc *ddata)
+{
+ int wps, spr, error;
+ u32 val;
+
+ wps = 0x34;
+ spr = 0x48;
+
+ sysc_write(ddata, spr, 0xaaaa);
+ error = readl_poll_timeout(ddata->module_va + wps, val,
+ !(val & 0x10), 100,
+ MAX_MODULE_SOFTRESET_WAIT);
+ if (error)
+ dev_warn(ddata->dev, "wdt disable spr failed\n");
+
+ sysc_write(ddata, wps, 0x5555);
+ error = readl_poll_timeout(ddata->module_va + wps, val,
+ !(val & 0x10), 100,
+ MAX_MODULE_SOFTRESET_WAIT);
+ if (error)
+ dev_warn(ddata->dev, "wdt disable wps failed\n");
+}
+
+static void sysc_init_module_quirks(struct sysc *ddata)
+{
+ if (ddata->legacy_mode || !ddata->name)
+ return;
+
+ if (ddata->cfg.quirks & SYSC_MODULE_QUIRK_HDQ1W) {
+ ddata->clk_enable_quirk = sysc_clk_enable_quirk_hdq1w;
+
+ return;
+ }
+
+ if (ddata->cfg.quirks & SYSC_MODULE_QUIRK_I2C) {
+ ddata->clk_enable_quirk = sysc_clk_enable_quirk_i2c;
+ ddata->clk_disable_quirk = sysc_clk_disable_quirk_i2c;
+
+ return;
+ }
+
+ if (ddata->cfg.quirks & SYSC_MODULE_QUIRK_WDT)
+ ddata->reset_done_quirk = sysc_reset_done_quirk_wdt;
+}
+
+static int sysc_clockdomain_init(struct sysc *ddata)
+{
+ struct ti_sysc_platform_data *pdata = dev_get_platdata(ddata->dev);
+ struct clk *fck = NULL, *ick = NULL;
+ int error;
+
+ if (!pdata || !pdata->init_clockdomain)
+ return 0;
+
+ switch (ddata->nr_clocks) {
+ case 2:
+ ick = ddata->clocks[SYSC_ICK];
+ /* fallthrough */
+ case 1:
+ fck = ddata->clocks[SYSC_FCK];
+ break;
+ case 0:
+ return 0;
+ }
+
+ error = pdata->init_clockdomain(ddata->dev, fck, ick, &ddata->cookie);
+ if (!error || error == -ENODEV)
+ return 0;
+
+ return error;
+}
+
/*
* Note that pdata->init_module() typically does a reset first. After
* pdata->init_module() is done, PM runtime can be used for the interconnect
@@ -1255,7 +1507,7 @@ static int sysc_legacy_init(struct sysc *ddata)
struct ti_sysc_platform_data *pdata = dev_get_platdata(ddata->dev);
int error;
- if (!ddata->legacy_mode || !pdata || !pdata->init_module)
+ if (!pdata || !pdata->init_module)
return 0;
error = pdata->init_module(ddata->dev, ddata->mdata, &ddata->cookie);
@@ -1280,7 +1532,7 @@ static int sysc_legacy_init(struct sysc *ddata)
*/
static int sysc_rstctrl_reset_deassert(struct sysc *ddata, bool reset)
{
- int error;
+ int error, val;
if (!ddata->rsts)
return 0;
@@ -1291,37 +1543,68 @@ static int sysc_rstctrl_reset_deassert(struct sysc *ddata, bool reset)
return error;
}
- return reset_control_deassert(ddata->rsts);
+ error = reset_control_deassert(ddata->rsts);
+ if (error == -EEXIST)
+ return 0;
+
+ error = readx_poll_timeout(reset_control_status, ddata->rsts, val,
+ val == 0, 100, MAX_MODULE_SOFTRESET_WAIT);
+
+ return error;
}
+/*
+ * Note that the caller must ensure the interconnect target module is enabled
+ * before calling reset. Otherwise reset will not complete.
+ */
static int sysc_reset(struct sysc *ddata)
{
- int offset = ddata->offsets[SYSC_SYSCONFIG];
- int val;
+ int sysc_offset, syss_offset, sysc_val, rstval, quirks, error = 0;
+ u32 sysc_mask, syss_done;
- if (ddata->legacy_mode || offset < 0 ||
+ sysc_offset = ddata->offsets[SYSC_SYSCONFIG];
+ syss_offset = ddata->offsets[SYSC_SYSSTATUS];
+ quirks = ddata->cfg.quirks;
+
+ if (ddata->legacy_mode || sysc_offset < 0 ||
+ ddata->cap->regbits->srst_shift < 0 ||
ddata->cfg.quirks & SYSC_QUIRK_NO_RESET_ON_INIT)
return 0;
- /*
- * Currently only support reset status in sysstatus.
- * Warn and return error in all other cases
- */
- if (!ddata->cfg.syss_mask) {
- dev_err(ddata->dev, "No ti,syss-mask. Reset failed\n");
- return -EINVAL;
- }
+ sysc_mask = BIT(ddata->cap->regbits->srst_shift);
- val = sysc_read(ddata, offset);
- val |= (0x1 << ddata->cap->regbits->srst_shift);
- sysc_write(ddata, offset, val);
+ if (ddata->cfg.quirks & SYSS_QUIRK_RESETDONE_INVERTED)
+ syss_done = 0;
+ else
+ syss_done = ddata->cfg.syss_mask;
+
+ if (ddata->clk_disable_quirk)
+ ddata->clk_disable_quirk(ddata);
+
+ sysc_val = sysc_read_sysconfig(ddata);
+ sysc_val |= sysc_mask;
+ sysc_write(ddata, sysc_offset, sysc_val);
+
+ if (ddata->clk_enable_quirk)
+ ddata->clk_enable_quirk(ddata);
/* Poll on reset status */
- offset = ddata->offsets[SYSC_SYSSTATUS];
+ if (syss_offset >= 0) {
+ error = readx_poll_timeout(sysc_read_sysstatus, ddata, rstval,
+ (rstval & ddata->cfg.syss_mask) ==
+ syss_done,
+ 100, MAX_MODULE_SOFTRESET_WAIT);
+
+ } else if (ddata->cfg.quirks & SYSC_QUIRK_RESET_STATUS) {
+ error = readx_poll_timeout(sysc_read_sysconfig, ddata, rstval,
+ !(rstval & sysc_mask),
+ 100, MAX_MODULE_SOFTRESET_WAIT);
+ }
+
+ if (ddata->reset_done_quirk)
+ ddata->reset_done_quirk(ddata);
- return readl_poll_timeout(ddata->module_va + offset, val,
- (val & ddata->cfg.syss_mask) == 0x0,
- 100, MAX_MODULE_SOFTRESET_WAIT);
+ return error;
}
/*
@@ -1334,12 +1617,8 @@ static int sysc_init_module(struct sysc *ddata)
{
int error = 0;
bool manage_clocks = true;
- bool reset = true;
-
- if (ddata->cfg.quirks & SYSC_QUIRK_NO_RESET_ON_INIT)
- reset = false;
- error = sysc_rstctrl_reset_deassert(ddata, reset);
+ error = sysc_rstctrl_reset_deassert(ddata, false);
if (error)
return error;
@@ -1347,7 +1626,13 @@ static int sysc_init_module(struct sysc *ddata)
(SYSC_QUIRK_NO_IDLE | SYSC_QUIRK_NO_IDLE_ON_INIT))
manage_clocks = false;
+ error = sysc_clockdomain_init(ddata);
+ if (error)
+ return error;
+
if (manage_clocks) {
+ sysc_clkdm_deny_idle(ddata);
+
error = sysc_enable_opt_clocks(ddata);
if (error)
return error;
@@ -1357,23 +1642,43 @@ static int sysc_init_module(struct sysc *ddata)
goto err_opt_clocks;
}
+ if (!(ddata->cfg.quirks & SYSC_QUIRK_NO_RESET_ON_INIT)) {
+ error = sysc_rstctrl_reset_deassert(ddata, true);
+ if (error)
+ goto err_main_clocks;
+ }
+
ddata->revision = sysc_read_revision(ddata);
sysc_init_revision_quirks(ddata);
+ sysc_init_module_quirks(ddata);
- error = sysc_legacy_init(ddata);
- if (error)
- goto err_main_clocks;
+ if (ddata->legacy_mode) {
+ error = sysc_legacy_init(ddata);
+ if (error)
+ goto err_main_clocks;
+ }
+
+ if (!ddata->legacy_mode && manage_clocks) {
+ error = sysc_enable_module(ddata->dev);
+ if (error)
+ goto err_main_clocks;
+ }
error = sysc_reset(ddata);
if (error)
dev_err(ddata->dev, "Reset failed with %d\n", error);
+ if (!ddata->legacy_mode && manage_clocks)
+ sysc_disable_module(ddata->dev);
+
err_main_clocks:
if (manage_clocks)
sysc_disable_main_clocks(ddata);
err_opt_clocks:
- if (manage_clocks)
+ if (manage_clocks) {
sysc_disable_opt_clocks(ddata);
+ sysc_clkdm_allow_idle(ddata);
+ }
return error;
}
@@ -1388,10 +1693,7 @@ static int sysc_init_sysc_mask(struct sysc *ddata)
if (error)
return 0;
- if (val)
- ddata->cfg.sysc_val = val & ddata->cap->sysc_mask;
- else
- ddata->cfg.sysc_val = ddata->cap->sysc_mask;
+ ddata->cfg.sysc_val = val & ddata->cap->sysc_mask;
return 0;
}
@@ -1663,9 +1965,6 @@ static struct dev_pm_domain sysc_child_pm_domain = {
*/
static void sysc_legacy_idle_quirk(struct sysc *ddata, struct device *child)
{
- if (!ddata->legacy_mode)
- return;
-
if (ddata->cfg.quirks & SYSC_QUIRK_LEGACY_IDLE)
dev_pm_domain_set(child, &sysc_child_pm_domain);
}
@@ -2005,6 +2304,7 @@ static const struct sysc_capabilities sysc_dra7_mcan = {
.type = TI_SYSC_DRA7_MCAN,
.sysc_mask = SYSC_DRA7_MCAN_ENAWAKEUP | SYSC_OMAP4_SOFTRESET,
.regbits = &sysc_regbits_dra7_mcan,
+ .mod_quirks = SYSS_QUIRK_RESETDONE_INVERTED,
};
static int sysc_init_pdata(struct sysc *ddata)
@@ -2012,20 +2312,22 @@ static int sysc_init_pdata(struct sysc *ddata)
struct ti_sysc_platform_data *pdata = dev_get_platdata(ddata->dev);
struct ti_sysc_module_data *mdata;
- if (!pdata || !ddata->legacy_mode)
+ if (!pdata)
return 0;
mdata = devm_kzalloc(ddata->dev, sizeof(*mdata), GFP_KERNEL);
if (!mdata)
return -ENOMEM;
- mdata->name = ddata->legacy_mode;
- mdata->module_pa = ddata->module_pa;
- mdata->module_size = ddata->module_size;
- mdata->offsets = ddata->offsets;
- mdata->nr_offsets = SYSC_MAX_REGS;
- mdata->cap = ddata->cap;
- mdata->cfg = &ddata->cfg;
+ if (ddata->legacy_mode) {
+ mdata->name = ddata->legacy_mode;
+ mdata->module_pa = ddata->module_pa;
+ mdata->module_size = ddata->module_size;
+ mdata->offsets = ddata->offsets;
+ mdata->nr_offsets = SYSC_MAX_REGS;
+ mdata->cap = ddata->cap;
+ mdata->cfg = &ddata->cfg;
+ }
ddata->mdata = mdata;
@@ -2081,27 +2383,27 @@ static int sysc_probe(struct platform_device *pdev)
error = sysc_init_dts_quirks(ddata);
if (error)
- goto unprepare;
+ return error;
error = sysc_map_and_check_registers(ddata);
if (error)
- goto unprepare;
+ return error;
error = sysc_init_sysc_mask(ddata);
if (error)
- goto unprepare;
+ return error;
error = sysc_init_idlemodes(ddata);
if (error)
- goto unprepare;
+ return error;
error = sysc_init_syss_mask(ddata);
if (error)
- goto unprepare;
+ return error;
error = sysc_init_pdata(ddata);
if (error)
- goto unprepare;
+ return error;
sysc_init_early_quirks(ddata);
@@ -2111,7 +2413,7 @@ static int sysc_probe(struct platform_device *pdev)
error = sysc_init_resets(ddata);
if (error)
- return error;
+ goto unprepare;
error = sysc_init_module(ddata);
if (error)
@@ -2145,7 +2447,7 @@ static int sysc_probe(struct platform_device *pdev)
}
if (!of_get_available_child_count(ddata->dev->of_node))
- reset_control_assert(ddata->rsts);
+ ddata->disable_on_idle = true;
return 0;
diff --git a/drivers/bus/uniphier-system-bus.c b/drivers/bus/uniphier-system-bus.c
index e845c1a93f21..f70dedace20b 100644
--- a/drivers/bus/uniphier-system-bus.c
+++ b/drivers/bus/uniphier-system-bus.c
@@ -176,7 +176,6 @@ static int uniphier_system_bus_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct uniphier_system_bus_priv *priv;
- struct resource *regs;
const __be32 *ranges;
u32 cells, addr, size;
u64 paddr;
@@ -186,8 +185,7 @@ static int uniphier_system_bus_probe(struct platform_device *pdev)
if (!priv)
return -ENOMEM;
- regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- priv->membase = devm_ioremap_resource(dev, regs);
+ priv->membase = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(priv->membase))
return PTR_ERR(priv->membase);
diff --git a/drivers/cdrom/cdrom.c b/drivers/cdrom/cdrom.c
index 933268b8d6a5..ac42ae4651ce 100644
--- a/drivers/cdrom/cdrom.c
+++ b/drivers/cdrom/cdrom.c
@@ -7,7 +7,7 @@
License. See linux/COPYING for more information.
Uniform CD-ROM driver for Linux.
- See Documentation/cdrom/cdrom-standard.tex for usage information.
+ See Documentation/cdrom/cdrom-standard.rst for usage information.
The routines in the file provide a uniform interface between the
software that uses CD-ROMs and the various low-level drivers that
diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig
index 466ebd84ad17..df0fc997dc3e 100644
--- a/drivers/char/Kconfig
+++ b/drivers/char/Kconfig
@@ -26,28 +26,6 @@ config DEVKMEM
kind of kernel debugging operations.
When in doubt, say "N".
-config SGI_SNSC
- bool "SGI Altix system controller communication support"
- depends on (IA64_SGI_SN2 || IA64_GENERIC)
- help
- If you have an SGI Altix and you want to enable system
- controller communication from user space (you want this!),
- say Y. Otherwise, say N.
-
-config SGI_TIOCX
- bool "SGI TIO CX driver support"
- depends on (IA64_SGI_SN2 || IA64_GENERIC)
- help
- If you have an SGI Altix and you have fpga devices attached
- to your TIO, say Y here, otherwise say N.
-
-config SGI_MBCS
- tristate "SGI FPGA Core Services driver support"
- depends on SGI_TIOCX
- help
- If you have an SGI Altix with an attached SABrick
- say Y or M here, otherwise say N.
-
source "drivers/tty/serial/Kconfig"
source "drivers/tty/serdev/Kconfig"
@@ -291,7 +269,7 @@ config RTC
and set the RTC in an SMP compatible fashion.
If you think you have a use for such a device (such as periodic data
- sampling), then say Y here, and read <file:Documentation/rtc.txt>
+ sampling), then say Y here, and read <file:Documentation/admin-guide/rtc.rst>
for details.
To compile this driver as a module, choose M here: the
@@ -313,7 +291,7 @@ config JS_RTC
/dev/rtc.
If you think you have a use for such a device (such as periodic data
- sampling), then say Y here, and read <file:Documentation/rtc.txt>
+ sampling), then say Y here, and read <file:Documentation/admin-guide/rtc.rst>
for details.
To compile this driver as a module, choose M here: the
@@ -382,7 +360,7 @@ config SONYPI
Device which can be found in many (all ?) Sony Vaio laptops.
If you have one of those laptops, read
- <file:Documentation/laptops/sonypi.txt>, and say Y or M here.
+ <file:Documentation/admin-guide/laptops/sonypi.rst>, and say Y or M here.
To compile this driver as a module, choose M here: the
module will be called sonypi.
@@ -573,3 +551,12 @@ config RANDOM_TRUST_CPU
has not installed a hidden back door to compromise the CPU's
random number generation facilities. This can also be configured
at boot with "random.trust_cpu=on/off".
+
+config RANDOM_TRUST_BOOTLOADER
+ bool "Trust the bootloader to initialize Linux's CRNG"
+ help
+ Some bootloaders can provide entropy to increase the kernel's initial
+ device randomness. Say Y here to assume the entropy provided by the
+ booloader is trustworthy so it will be added to the kernel's entropy
+ pool. Otherwise, say N here so it will be regarded as device input that
+ only mixes the entropy pool. \ No newline at end of file
diff --git a/drivers/char/Makefile b/drivers/char/Makefile
index fbea7dd12932..7c5ea6f9df14 100644
--- a/drivers/char/Makefile
+++ b/drivers/char/Makefile
@@ -9,11 +9,9 @@ obj-y += misc.o
obj-$(CONFIG_ATARI_DSP56K) += dsp56k.o
obj-$(CONFIG_VIRTIO_CONSOLE) += virtio_console.o
obj-$(CONFIG_RAW_DRIVER) += raw.o
-obj-$(CONFIG_SGI_SNSC) += snsc.o snsc_event.o
obj-$(CONFIG_MSPEC) += mspec.o
obj-$(CONFIG_UV_MMTIMER) += uv_mmtimer.o
obj-$(CONFIG_IBM_BSR) += bsr.o
-obj-$(CONFIG_SGI_MBCS) += mbcs.o
obj-$(CONFIG_PRINTER) += lp.o
diff --git a/drivers/char/agp/Kconfig b/drivers/char/agp/Kconfig
index 6231714ef3c8..812d6aa6e013 100644
--- a/drivers/char/agp/Kconfig
+++ b/drivers/char/agp/Kconfig
@@ -111,14 +111,14 @@ config AGP_VIA
config AGP_I460
tristate "Intel 460GX chipset support"
- depends on AGP && (IA64_DIG || IA64_GENERIC)
+ depends on AGP && IA64
help
This option gives you AGP GART support for the Intel 460GX chipset
for IA64 processors.
config AGP_HP_ZX1
tristate "HP ZX1 chipset AGP support"
- depends on AGP && (IA64_HP_ZX1 || IA64_HP_ZX1_SWIOTLB || IA64_GENERIC)
+ depends on AGP && IA64
help
This option gives you AGP GART support for the HP ZX1 chipset
for IA64 processors.
@@ -150,13 +150,6 @@ config AGP_EFFICEON
This option gives you AGP support for the Transmeta Efficeon
series processors with integrated northbridges.
-config AGP_SGI_TIOCA
- tristate "SGI TIO chipset AGP support"
- depends on AGP && (IA64_SGI_SN2 || IA64_GENERIC)
- help
- This option gives you AGP GART support for the SGI TIO chipset
- for IA64 processors.
-
config INTEL_GTT
tristate
depends on X86 && PCI
diff --git a/drivers/char/agp/Makefile b/drivers/char/agp/Makefile
index 4a786ffd9dee..cb2497d157f6 100644
--- a/drivers/char/agp/Makefile
+++ b/drivers/char/agp/Makefile
@@ -16,7 +16,6 @@ obj-$(CONFIG_AGP_I460) += i460-agp.o
obj-$(CONFIG_AGP_INTEL) += intel-agp.o
obj-$(CONFIG_INTEL_GTT) += intel-gtt.o
obj-$(CONFIG_AGP_NVIDIA) += nvidia-agp.o
-obj-$(CONFIG_AGP_SGI_TIOCA) += sgi-agp.o
obj-$(CONFIG_AGP_SIS) += sis-agp.o
obj-$(CONFIG_AGP_SWORKS) += sworks-agp.o
obj-$(CONFIG_AGP_UNINORTH) += uninorth-agp.o
diff --git a/drivers/char/agp/generic.c b/drivers/char/agp/generic.c
index 658664a5a5aa..df1edb5ec0ad 100644
--- a/drivers/char/agp/generic.c
+++ b/drivers/char/agp/generic.c
@@ -1311,8 +1311,7 @@ static void ipi_handler(void *null)
void global_cache_flush(void)
{
- if (on_each_cpu(ipi_handler, NULL, 1) != 0)
- panic(PFX "timed out waiting for the other CPUs!\n");
+ on_each_cpu(ipi_handler, NULL, 1);
}
EXPORT_SYMBOL(global_cache_flush);
diff --git a/drivers/char/agp/sgi-agp.c b/drivers/char/agp/sgi-agp.c
deleted file mode 100644
index e7d5bdc02d93..000000000000
--- a/drivers/char/agp/sgi-agp.c
+++ /dev/null
@@ -1,338 +0,0 @@
-/*
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file "COPYING" in the main directory of this archive
- * for more details.
- *
- * Copyright (C) 2003-2005 Silicon Graphics, Inc. All Rights Reserved.
- */
-
-/*
- * SGI TIOCA AGPGART routines.
- *
- */
-
-#include <linux/acpi.h>
-#include <linux/module.h>
-#include <linux/pci.h>
-#include <linux/slab.h>
-#include <linux/agp_backend.h>
-#include <asm/sn/addrs.h>
-#include <asm/sn/io.h>
-#include <asm/sn/pcidev.h>
-#include <asm/sn/pcibus_provider_defs.h>
-#include <asm/sn/tioca_provider.h>
-#include "agp.h"
-
-extern int agp_memory_reserved;
-extern uint32_t tioca_gart_found;
-extern struct list_head tioca_list;
-static struct agp_bridge_data **sgi_tioca_agp_bridges;
-
-/*
- * The aperature size and related information is set up at TIOCA init time.
- * Values for this table will be extracted and filled in at
- * sgi_tioca_fetch_size() time.
- */
-
-static struct aper_size_info_fixed sgi_tioca_sizes[] = {
- {0, 0, 0},
-};
-
-static struct page *sgi_tioca_alloc_page(struct agp_bridge_data *bridge)
-{
- struct page *page;
- int nid;
- struct tioca_kernel *info =
- (struct tioca_kernel *)bridge->dev_private_data;
-
- nid = info->ca_closest_node;
- page = alloc_pages_node(nid, GFP_KERNEL, 0);
- if (!page)
- return NULL;
-
- get_page(page);
- atomic_inc(&agp_bridge->current_memory_agp);
- return page;
-}
-
-/*
- * Flush GART tlb's. Cannot selectively flush based on memory so the mem
- * arg is ignored.
- */
-
-static void sgi_tioca_tlbflush(struct agp_memory *mem)
-{
- tioca_tlbflush(mem->bridge->dev_private_data);
-}
-
-/*
- * Given an address of a host physical page, turn it into a valid gart
- * entry.
- */
-static unsigned long
-sgi_tioca_mask_memory(struct agp_bridge_data *bridge, dma_addr_t addr,
- int type)
-{
- return tioca_physpage_to_gart(addr);
-}
-
-static void sgi_tioca_agp_enable(struct agp_bridge_data *bridge, u32 mode)
-{
- tioca_fastwrite_enable(bridge->dev_private_data);
-}
-
-/*
- * sgi_tioca_configure() doesn't have anything to do since the base CA driver
- * has alreay set up the GART.
- */
-
-static int sgi_tioca_configure(void)
-{
- return 0;
-}
-
-/*
- * Determine gfx aperature size. This has already been determined by the
- * CA driver init, so just need to set agp_bridge values accordingly.
- */
-
-static int sgi_tioca_fetch_size(void)
-{
- struct tioca_kernel *info =
- (struct tioca_kernel *)agp_bridge->dev_private_data;
-
- sgi_tioca_sizes[0].size = info->ca_gfxap_size / MB(1);
- sgi_tioca_sizes[0].num_entries = info->ca_gfxgart_entries;
-
- return sgi_tioca_sizes[0].size;
-}
-
-static int sgi_tioca_create_gatt_table(struct agp_bridge_data *bridge)
-{
- struct tioca_kernel *info =
- (struct tioca_kernel *)bridge->dev_private_data;
-
- bridge->gatt_table_real = (u32 *) info->ca_gfxgart;
- bridge->gatt_table = bridge->gatt_table_real;
- bridge->gatt_bus_addr = info->ca_gfxgart_base;
-
- return 0;
-}
-
-static int sgi_tioca_free_gatt_table(struct agp_bridge_data *bridge)
-{
- return 0;
-}
-
-static int sgi_tioca_insert_memory(struct agp_memory *mem, off_t pg_start,
- int type)
-{
- int num_entries;
- size_t i;
- off_t j;
- void *temp;
- struct agp_bridge_data *bridge;
- u64 *table;
-
- bridge = mem->bridge;
- if (!bridge)
- return -EINVAL;
-
- table = (u64 *)bridge->gatt_table;
-
- temp = bridge->current_size;
-
- switch (bridge->driver->size_type) {
- case U8_APER_SIZE:
- num_entries = A_SIZE_8(temp)->num_entries;
- break;
- case U16_APER_SIZE:
- num_entries = A_SIZE_16(temp)->num_entries;
- break;
- case U32_APER_SIZE:
- num_entries = A_SIZE_32(temp)->num_entries;
- break;
- case FIXED_APER_SIZE:
- num_entries = A_SIZE_FIX(temp)->num_entries;
- break;
- case LVL2_APER_SIZE:
- return -EINVAL;
- default:
- num_entries = 0;
- break;
- }
-
- num_entries -= agp_memory_reserved / PAGE_SIZE;
- if (num_entries < 0)
- num_entries = 0;
-
- if (type != 0 || mem->type != 0) {
- return -EINVAL;
- }
-
- if ((pg_start + mem->page_count) > num_entries)
- return -EINVAL;
-
- j = pg_start;
-
- while (j < (pg_start + mem->page_count)) {
- if (table[j])
- return -EBUSY;
- j++;
- }
-
- if (!mem->is_flushed) {
- bridge->driver->cache_flush();
- mem->is_flushed = true;
- }
-
- for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
- table[j] =
- bridge->driver->mask_memory(bridge,
- page_to_phys(mem->pages[i]),
- mem->type);
- }
-
- bridge->driver->tlb_flush(mem);
- return 0;
-}
-
-static int sgi_tioca_remove_memory(struct agp_memory *mem, off_t pg_start,
- int type)
-{
- size_t i;
- struct agp_bridge_data *bridge;
- u64 *table;
-
- bridge = mem->bridge;
- if (!bridge)
- return -EINVAL;
-
- if (type != 0 || mem->type != 0) {
- return -EINVAL;
- }
-
- table = (u64 *)bridge->gatt_table;
-
- for (i = pg_start; i < (mem->page_count + pg_start); i++) {
- table[i] = 0;
- }
-
- bridge->driver->tlb_flush(mem);
- return 0;
-}
-
-static void sgi_tioca_cache_flush(void)
-{
-}
-
-/*
- * Cleanup. Nothing to do as the CA driver owns the GART.
- */
-
-static void sgi_tioca_cleanup(void)
-{
-}
-
-static struct agp_bridge_data *sgi_tioca_find_bridge(struct pci_dev *pdev)
-{
- struct agp_bridge_data *bridge;
-
- list_for_each_entry(bridge, &agp_bridges, list) {
- if (bridge->dev->bus == pdev->bus)
- break;
- }
- return bridge;
-}
-
-const struct agp_bridge_driver sgi_tioca_driver = {
- .owner = THIS_MODULE,
- .size_type = U16_APER_SIZE,
- .configure = sgi_tioca_configure,
- .fetch_size = sgi_tioca_fetch_size,
- .cleanup = sgi_tioca_cleanup,
- .tlb_flush = sgi_tioca_tlbflush,
- .mask_memory = sgi_tioca_mask_memory,
- .agp_enable = sgi_tioca_agp_enable,
- .cache_flush = sgi_tioca_cache_flush,
- .create_gatt_table = sgi_tioca_create_gatt_table,
- .free_gatt_table = sgi_tioca_free_gatt_table,
- .insert_memory = sgi_tioca_insert_memory,
- .remove_memory = sgi_tioca_remove_memory,
- .alloc_by_type = agp_generic_alloc_by_type,
- .free_by_type = agp_generic_free_by_type,
- .agp_alloc_page = sgi_tioca_alloc_page,
- .agp_destroy_page = agp_generic_destroy_page,
- .agp_type_to_mask_type = agp_generic_type_to_mask_type,
- .cant_use_aperture = true,
- .needs_scratch_page = false,
- .num_aperture_sizes = 1,
-};
-
-static int agp_sgi_init(void)
-{
- unsigned int j;
- struct tioca_kernel *info;
- struct pci_dev *pdev = NULL;
-
- if (tioca_gart_found)
- printk(KERN_INFO PFX "SGI TIO CA GART driver initialized.\n");
- else
- return 0;
-
- sgi_tioca_agp_bridges = kmalloc_array(tioca_gart_found,
- sizeof(struct agp_bridge_data *),
- GFP_KERNEL);
- if (!sgi_tioca_agp_bridges)
- return -ENOMEM;
-
- j = 0;
- list_for_each_entry(info, &tioca_list, ca_list) {
- if (list_empty(info->ca_devices))
- continue;
- list_for_each_entry(pdev, info->ca_devices, bus_list) {
- u8 cap_ptr;
-
- if (pdev->class != (PCI_CLASS_DISPLAY_VGA << 8))
- continue;
- cap_ptr = pci_find_capability(pdev, PCI_CAP_ID_AGP);
- if (!cap_ptr)
- continue;
- }
- sgi_tioca_agp_bridges[j] = agp_alloc_bridge();
- printk(KERN_INFO PFX "bridge %d = 0x%p\n", j,
- sgi_tioca_agp_bridges[j]);
- if (sgi_tioca_agp_bridges[j]) {
- sgi_tioca_agp_bridges[j]->dev = pdev;
- sgi_tioca_agp_bridges[j]->dev_private_data = info;
- sgi_tioca_agp_bridges[j]->driver = &sgi_tioca_driver;
- sgi_tioca_agp_bridges[j]->gart_bus_addr =
- info->ca_gfxap_base;
- sgi_tioca_agp_bridges[j]->mode = (0x7D << 24) | /* 126 requests */
- (0x1 << 9) | /* SBA supported */
- (0x1 << 5) | /* 64-bit addresses supported */
- (0x1 << 4) | /* FW supported */
- (0x1 << 3) | /* AGP 3.0 mode */
- 0x2; /* 8x transfer only */
- sgi_tioca_agp_bridges[j]->current_size =
- sgi_tioca_agp_bridges[j]->previous_size =
- (void *)&sgi_tioca_sizes[0];
- agp_add_bridge(sgi_tioca_agp_bridges[j]);
- }
- j++;
- }
-
- agp_find_bridge = &sgi_tioca_find_bridge;
- return 0;
-}
-
-static void agp_sgi_cleanup(void)
-{
- kfree(sgi_tioca_agp_bridges);
- sgi_tioca_agp_bridges = NULL;
-}
-
-module_init(agp_sgi_init);
-module_exit(agp_sgi_cleanup);
-
-MODULE_LICENSE("GPL and additional rights");
diff --git a/drivers/char/bsr.c b/drivers/char/bsr.c
index 167e7370d43a..e5e5333f302d 100644
--- a/drivers/char/bsr.c
+++ b/drivers/char/bsr.c
@@ -134,7 +134,7 @@ static int bsr_mmap(struct file *filp, struct vm_area_struct *vma)
return 0;
}
-static int bsr_open(struct inode * inode, struct file * filp)
+static int bsr_open(struct inode *inode, struct file *filp)
{
struct cdev *cdev = inode->i_cdev;
struct bsr_dev *dev = container_of(cdev, struct bsr_dev, bsr_cdev);
@@ -309,7 +309,8 @@ static int __init bsr_init(void)
goto out_err_2;
}
- if ((ret = bsr_create_devs(np)) < 0) {
+ ret = bsr_create_devs(np);
+ if (ret < 0) {
np = NULL;
goto out_err_3;
}
diff --git a/drivers/char/hpet.c b/drivers/char/hpet.c
index 5c39f20378b8..9ac6671bb514 100644
--- a/drivers/char/hpet.c
+++ b/drivers/char/hpet.c
@@ -567,8 +567,7 @@ static inline unsigned long hpet_time_div(struct hpets *hpets,
unsigned long long m;
m = hpets->hp_tick_freq + (dis >> 1);
- do_div(m, dis);
- return (unsigned long)m;
+ return div64_ul(m, dis);
}
static int
diff --git a/drivers/char/hw_random/atmel-rng.c b/drivers/char/hw_random/atmel-rng.c
index 433426242b87..e55705745d5e 100644
--- a/drivers/char/hw_random/atmel-rng.c
+++ b/drivers/char/hw_random/atmel-rng.c
@@ -86,7 +86,7 @@ static int atmel_trng_probe(struct platform_device *pdev)
trng->rng.name = pdev->name;
trng->rng.read = atmel_trng_read;
- ret = hwrng_register(&trng->rng);
+ ret = devm_hwrng_register(&pdev->dev, &trng->rng);
if (ret)
goto err_register;
@@ -103,7 +103,6 @@ static int atmel_trng_remove(struct platform_device *pdev)
{
struct atmel_trng *trng = platform_get_drvdata(pdev);
- hwrng_unregister(&trng->rng);
atmel_trng_disable(trng);
clk_disable_unprepare(trng->clk);
diff --git a/drivers/char/hw_random/cavium-rng-vf.c b/drivers/char/hw_random/cavium-rng-vf.c
index 2d1352b67168..3de4a6a443ef 100644
--- a/drivers/char/hw_random/cavium-rng-vf.c
+++ b/drivers/char/hw_random/cavium-rng-vf.c
@@ -67,7 +67,7 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev,
pci_set_drvdata(pdev, rng);
- ret = hwrng_register(&rng->ops);
+ ret = devm_hwrng_register(&pdev->dev, &rng->ops);
if (ret) {
dev_err(&pdev->dev, "Error registering device as HWRNG.\n");
return ret;
@@ -76,14 +76,6 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev,
return 0;
}
-/* Remove the VF */
-static void cavium_rng_remove_vf(struct pci_dev *pdev)
-{
- struct cavium_rng *rng;
-
- rng = pci_get_drvdata(pdev);
- hwrng_unregister(&rng->ops);
-}
static const struct pci_device_id cavium_rng_vf_id_table[] = {
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xa033), 0, 0, 0},
@@ -95,7 +87,6 @@ static struct pci_driver cavium_rng_vf_driver = {
.name = "cavium_rng_vf",
.id_table = cavium_rng_vf_id_table,
.probe = cavium_rng_probe_vf,
- .remove = cavium_rng_remove_vf,
};
module_pci_driver(cavium_rng_vf_driver);
diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c
index 95be7228f327..bdab5d9af8d2 100644
--- a/drivers/char/hw_random/core.c
+++ b/drivers/char/hw_random/core.c
@@ -4,7 +4,7 @@
* Copyright 2006 Michael Buesch <m@bues.ch>
* Copyright 2005 (c) MontaVista Software, Inc.
*
- * Please read Documentation/hw_random.txt for details on use.
+ * Please read Documentation/admin-guide/hw_random.rst for details on use.
*
* This software may be used and distributed according to the terms
* of the GNU General Public License, incorporated herein by reference.
@@ -13,6 +13,7 @@
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/err.h>
+#include <linux/freezer.h>
#include <linux/fs.h>
#include <linux/hw_random.h>
#include <linux/kernel.h>
@@ -421,7 +422,9 @@ static int hwrng_fillfn(void *unused)
{
long rc;
- while (!kthread_should_stop()) {
+ set_freezable();
+
+ while (!kthread_freezable_should_stop(NULL)) {
struct hwrng *rng;
rng = get_current_rng();
diff --git a/drivers/char/hw_random/exynos-trng.c b/drivers/char/hw_random/exynos-trng.c
index 94235761955c..b4b52ab23b6b 100644
--- a/drivers/char/hw_random/exynos-trng.c
+++ b/drivers/char/hw_random/exynos-trng.c
@@ -153,7 +153,7 @@ static int exynos_trng_probe(struct platform_device *pdev)
goto err_clock;
}
- ret = hwrng_register(&trng->rng);
+ ret = devm_hwrng_register(&pdev->dev, &trng->rng);
if (ret) {
dev_err(&pdev->dev, "Could not register hwrng device.\n");
goto err_register;
@@ -179,7 +179,6 @@ static int exynos_trng_remove(struct platform_device *pdev)
{
struct exynos_trng_dev *trng = platform_get_drvdata(pdev);
- hwrng_unregister(&trng->rng);
clk_disable_unprepare(trng->clk);
pm_runtime_put_sync(&pdev->dev);
diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c
index 69f537980004..30cf00f8e9a0 100644
--- a/drivers/char/hw_random/imx-rngc.c
+++ b/drivers/char/hw_random/imx-rngc.c
@@ -196,7 +196,6 @@ static int imx_rngc_init(struct hwrng *rng)
static int imx_rngc_probe(struct platform_device *pdev)
{
struct imx_rngc *rngc;
- struct resource *res;
int ret;
int irq;
@@ -204,8 +203,7 @@ static int imx_rngc_probe(struct platform_device *pdev)
if (!rngc)
return -ENOMEM;
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- rngc->base = devm_ioremap_resource(&pdev->dev, res);
+ rngc->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(rngc->base))
return PTR_ERR(rngc->base);
diff --git a/drivers/char/hw_random/iproc-rng200.c b/drivers/char/hw_random/iproc-rng200.c
index 8b5a20b35293..92be1c0ab99f 100644
--- a/drivers/char/hw_random/iproc-rng200.c
+++ b/drivers/char/hw_random/iproc-rng200.c
@@ -220,6 +220,7 @@ static int iproc_rng200_probe(struct platform_device *pdev)
}
static const struct of_device_id iproc_rng200_of_match[] = {
+ { .compatible = "brcm,bcm7211-rng200", },
{ .compatible = "brcm,bcm7278-rng200", },
{ .compatible = "brcm,iproc-rng200", },
{},
diff --git a/drivers/char/hw_random/meson-rng.c b/drivers/char/hw_random/meson-rng.c
index 2e23be802a62..76e693da5dde 100644
--- a/drivers/char/hw_random/meson-rng.c
+++ b/drivers/char/hw_random/meson-rng.c
@@ -1,58 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
/*
- * This file is provided under a dual BSD/GPLv2 license. When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
* Copyright (c) 2016 BayLibre, SAS.
* Author: Neil Armstrong <narmstrong@baylibre.com>
* Copyright (C) 2014 Amlogic, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, see <http://www.gnu.org/licenses/>.
- * The full GNU General Public License is included in this distribution
- * in the file called COPYING.
- *
- * BSD LICENSE
- *
- * Copyright (c) 2016 BayLibre, SAS.
- * Author: Neil Armstrong <narmstrong@baylibre.com>
- * Copyright (C) 2014 Amlogic, Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <linux/err.h>
#include <linux/module.h>
diff --git a/drivers/char/hw_random/mxc-rnga.c b/drivers/char/hw_random/mxc-rnga.c
index ea2bf18b1fbb..025083c838f5 100644
--- a/drivers/char/hw_random/mxc-rnga.c
+++ b/drivers/char/hw_random/mxc-rnga.c
@@ -134,7 +134,6 @@ static void mxc_rnga_cleanup(struct hwrng *rng)
static int __init mxc_rnga_probe(struct platform_device *pdev)
{
int err;
- struct resource *res;
struct mxc_rng *mxc_rng;
mxc_rng = devm_kzalloc(&pdev->dev, sizeof(*mxc_rng), GFP_KERNEL);
@@ -158,8 +157,7 @@ static int __init mxc_rnga_probe(struct platform_device *pdev)
if (err)
return err;
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- mxc_rng->mem = devm_ioremap_resource(&pdev->dev, res);
+ mxc_rng->mem = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(mxc_rng->mem)) {
err = PTR_ERR(mxc_rng->mem);
goto err_ioremap;
diff --git a/drivers/char/hw_random/n2-drv.c b/drivers/char/hw_random/n2-drv.c
index d4cab105796f..73e408146420 100644
--- a/drivers/char/hw_random/n2-drv.c
+++ b/drivers/char/hw_random/n2-drv.c
@@ -768,7 +768,7 @@ static int n2rng_probe(struct platform_device *op)
np->hwrng.data_read = n2rng_data_read;
np->hwrng.priv = (unsigned long) np;
- err = hwrng_register(&np->hwrng);
+ err = devm_hwrng_register(&op->dev, &np->hwrng);
if (err)
goto out_hvapi_unregister;
@@ -793,8 +793,6 @@ static int n2rng_remove(struct platform_device *op)
cancel_delayed_work_sync(&np->work);
- hwrng_unregister(&np->hwrng);
-
sun4v_hvapi_unregister(HV_GRP_RNG);
return 0;
diff --git a/drivers/char/hw_random/nomadik-rng.c b/drivers/char/hw_random/nomadik-rng.c
index fc0f6b0cb80d..74ed29f42e4f 100644
--- a/drivers/char/hw_random/nomadik-rng.c
+++ b/drivers/char/hw_random/nomadik-rng.c
@@ -57,7 +57,7 @@ static int nmk_rng_probe(struct amba_device *dev, const struct amba_id *id)
if (!base)
goto out_release;
nmk_rng.priv = (unsigned long)base;
- ret = hwrng_register(&nmk_rng);
+ ret = devm_hwrng_register(&dev->dev, &nmk_rng);
if (ret)
goto out_release;
return 0;
@@ -71,7 +71,6 @@ out_clk:
static int nmk_rng_remove(struct amba_device *dev)
{
- hwrng_unregister(&nmk_rng);
amba_release_regions(dev);
clk_disable(rng_clk);
return 0;
diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c
index e9b6ac61fb7f..b27f39688b5e 100644
--- a/drivers/char/hw_random/omap-rng.c
+++ b/drivers/char/hw_random/omap-rng.c
@@ -500,7 +500,7 @@ static int omap_rng_probe(struct platform_device *pdev)
if (ret)
goto err_register;
- ret = hwrng_register(&priv->rng);
+ ret = devm_hwrng_register(&pdev->dev, &priv->rng);
if (ret)
goto err_register;
@@ -525,7 +525,6 @@ static int omap_rng_remove(struct platform_device *pdev)
{
struct omap_rng_dev *priv = platform_get_drvdata(pdev);
- hwrng_unregister(&priv->rng);
priv->pdata->cleanup(priv);
diff --git a/drivers/char/hw_random/powernv-rng.c b/drivers/char/hw_random/powernv-rng.c
index f2e8272e276a..8da1d7917bdc 100644
--- a/drivers/char/hw_random/powernv-rng.c
+++ b/drivers/char/hw_random/powernv-rng.c
@@ -33,18 +33,11 @@ static struct hwrng powernv_hwrng = {
.read = powernv_rng_read,
};
-static int powernv_rng_remove(struct platform_device *pdev)
-{
- hwrng_unregister(&powernv_hwrng);
-
- return 0;
-}
-
static int powernv_rng_probe(struct platform_device *pdev)
{
int rc;
- rc = hwrng_register(&powernv_hwrng);
+ rc = devm_hwrng_register(&pdev->dev, &powernv_hwrng);
if (rc) {
/* We only register one device, ignore any others */
if (rc == -EEXIST)
@@ -70,7 +63,6 @@ static struct platform_driver powernv_rng_driver = {
.of_match_table = powernv_rng_match,
},
.probe = powernv_rng_probe,
- .remove = powernv_rng_remove,
};
module_platform_driver(powernv_rng_driver);
diff --git a/drivers/char/hw_random/st-rng.c b/drivers/char/hw_random/st-rng.c
index bd6a98b3479b..863448360a7d 100644
--- a/drivers/char/hw_random/st-rng.c
+++ b/drivers/char/hw_random/st-rng.c
@@ -102,7 +102,7 @@ static int st_rng_probe(struct platform_device *pdev)
dev_set_drvdata(&pdev->dev, ddata);
- ret = hwrng_register(&ddata->ops);
+ ret = devm_hwrng_register(&pdev->dev, &ddata->ops);
if (ret) {
dev_err(&pdev->dev, "Failed to register HW RNG\n");
clk_disable_unprepare(clk);
@@ -118,8 +118,6 @@ static int st_rng_remove(struct platform_device *pdev)
{
struct st_rng_data *ddata = dev_get_drvdata(&pdev->dev);
- hwrng_unregister(&ddata->ops);
-
clk_disable_unprepare(ddata->clk);
return 0;
diff --git a/drivers/char/hw_random/timeriomem-rng.c b/drivers/char/hw_random/timeriomem-rng.c
index ccd1f6e0696b..e262445fed5f 100644
--- a/drivers/char/hw_random/timeriomem-rng.c
+++ b/drivers/char/hw_random/timeriomem-rng.c
@@ -117,9 +117,9 @@ static int timeriomem_rng_probe(struct platform_device *pdev)
if (!res)
return -ENXIO;
- if (res->start % 4 != 0 || resource_size(res) != 4) {
+ if (res->start % 4 != 0 || resource_size(res) < 4) {
dev_err(&pdev->dev,
- "address must be four bytes wide and aligned\n");
+ "address must be at least four bytes wide and 32-bit aligned\n");
return -EINVAL;
}
diff --git a/drivers/char/hw_random/xgene-rng.c b/drivers/char/hw_random/xgene-rng.c
index 8c6f9f63da5e..7e568db87ae2 100644
--- a/drivers/char/hw_random/xgene-rng.c
+++ b/drivers/char/hw_random/xgene-rng.c
@@ -361,7 +361,7 @@ static int xgene_rng_probe(struct platform_device *pdev)
xgene_rng_func.priv = (unsigned long) ctx;
- rc = hwrng_register(&xgene_rng_func);
+ rc = devm_hwrng_register(&pdev->dev, &xgene_rng_func);
if (rc) {
dev_err(&pdev->dev, "RNG registering failed error %d\n", rc);
if (!IS_ERR(ctx->clk))
@@ -375,7 +375,6 @@ static int xgene_rng_probe(struct platform_device *pdev)
rc);
if (!IS_ERR(ctx->clk))
clk_disable_unprepare(ctx->clk);
- hwrng_unregister(&xgene_rng_func);
return rc;
}
@@ -392,7 +391,6 @@ static int xgene_rng_remove(struct platform_device *pdev)
dev_err(&pdev->dev, "RNG init wakeup failed error %d\n", rc);
if (!IS_ERR(ctx->clk))
clk_disable_unprepare(ctx->clk);
- hwrng_unregister(&xgene_rng_func);
return rc;
}
diff --git a/drivers/char/ipmi/Kconfig b/drivers/char/ipmi/Kconfig
index caac5d24baa4..4bad0614109b 100644
--- a/drivers/char/ipmi/Kconfig
+++ b/drivers/char/ipmi/Kconfig
@@ -132,3 +132,12 @@ config ASPEED_BT_IPMI_BMC
Provides a driver for the BT (Block Transfer) IPMI interface
found on Aspeed SOCs (AST2400 and AST2500). The driver
implements the BMC side of the BT interface.
+
+config IPMB_DEVICE_INTERFACE
+ tristate 'IPMB Interface handler'
+ depends on I2C
+ depends on I2C_SLAVE
+ help
+ Provides a driver for a device (Satellite MC) to
+ receive requests and send responses back to the BMC via
+ the IPMB interface. This module requires I2C support.
diff --git a/drivers/char/ipmi/Makefile b/drivers/char/ipmi/Makefile
index 3f06b2062475..0822adc2ec41 100644
--- a/drivers/char/ipmi/Makefile
+++ b/drivers/char/ipmi/Makefile
@@ -26,3 +26,4 @@ obj-$(CONFIG_IPMI_KCS_BMC) += kcs_bmc.o
obj-$(CONFIG_ASPEED_BT_IPMI_BMC) += bt-bmc.o
obj-$(CONFIG_ASPEED_KCS_IPMI_BMC) += kcs_bmc_aspeed.o
obj-$(CONFIG_NPCM7XX_KCS_IPMI_BMC) += kcs_bmc_npcm7xx.o
+obj-$(CONFIG_IPMB_DEVICE_INTERFACE) += ipmb_dev_int.o
diff --git a/drivers/char/ipmi/ipmb_dev_int.c b/drivers/char/ipmi/ipmb_dev_int.c
new file mode 100644
index 000000000000..285e0b8f9a97
--- /dev/null
+++ b/drivers/char/ipmi/ipmb_dev_int.c
@@ -0,0 +1,364 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * IPMB driver to receive a request and send a response
+ *
+ * Copyright (C) 2019 Mellanox Techologies, Ltd.
+ *
+ * This was inspired by Brendan Higgins' ipmi-bmc-bt-i2c driver.
+ */
+
+#include <linux/acpi.h>
+#include <linux/errno.h>
+#include <linux/i2c.h>
+#include <linux/miscdevice.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/poll.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/wait.h>
+
+#define MAX_MSG_LEN 128
+#define IPMB_REQUEST_LEN_MIN 7
+#define NETFN_RSP_BIT_MASK 0x4
+#define REQUEST_QUEUE_MAX_LEN 256
+
+#define IPMB_MSG_LEN_IDX 0
+#define RQ_SA_8BIT_IDX 1
+#define NETFN_LUN_IDX 2
+
+#define GET_7BIT_ADDR(addr_8bit) (addr_8bit >> 1)
+#define GET_8BIT_ADDR(addr_7bit) ((addr_7bit << 1) & 0xff)
+
+#define IPMB_MSG_PAYLOAD_LEN_MAX (MAX_MSG_LEN - IPMB_REQUEST_LEN_MIN - 1)
+
+#define SMBUS_MSG_HEADER_LENGTH 2
+#define SMBUS_MSG_IDX_OFFSET (SMBUS_MSG_HEADER_LENGTH + 1)
+
+struct ipmb_msg {
+ u8 len;
+ u8 rs_sa;
+ u8 netfn_rs_lun;
+ u8 checksum1;
+ u8 rq_sa;
+ u8 rq_seq_rq_lun;
+ u8 cmd;
+ u8 payload[IPMB_MSG_PAYLOAD_LEN_MAX];
+ /* checksum2 is included in payload */
+} __packed;
+
+struct ipmb_request_elem {
+ struct list_head list;
+ struct ipmb_msg request;
+};
+
+struct ipmb_dev {
+ struct i2c_client *client;
+ struct miscdevice miscdev;
+ struct ipmb_msg request;
+ struct list_head request_queue;
+ atomic_t request_queue_len;
+ size_t msg_idx;
+ spinlock_t lock;
+ wait_queue_head_t wait_queue;
+ struct mutex file_mutex;
+};
+
+static inline struct ipmb_dev *to_ipmb_dev(struct file *file)
+{
+ return container_of(file->private_data, struct ipmb_dev, miscdev);
+}
+
+static ssize_t ipmb_read(struct file *file, char __user *buf, size_t count,
+ loff_t *ppos)
+{
+ struct ipmb_dev *ipmb_dev = to_ipmb_dev(file);
+ struct ipmb_request_elem *queue_elem;
+ struct ipmb_msg msg;
+ ssize_t ret = 0;
+
+ memset(&msg, 0, sizeof(msg));
+
+ spin_lock_irq(&ipmb_dev->lock);
+
+ while (list_empty(&ipmb_dev->request_queue)) {
+ spin_unlock_irq(&ipmb_dev->lock);
+
+ if (file->f_flags & O_NONBLOCK)
+ return -EAGAIN;
+
+ ret = wait_event_interruptible(ipmb_dev->wait_queue,
+ !list_empty(&ipmb_dev->request_queue));
+ if (ret)
+ return ret;
+
+ spin_lock_irq(&ipmb_dev->lock);
+ }
+
+ queue_elem = list_first_entry(&ipmb_dev->request_queue,
+ struct ipmb_request_elem, list);
+ memcpy(&msg, &queue_elem->request, sizeof(msg));
+ list_del(&queue_elem->list);
+ kfree(queue_elem);
+ atomic_dec(&ipmb_dev->request_queue_len);
+
+ spin_unlock_irq(&ipmb_dev->lock);
+
+ count = min_t(size_t, count, msg.len + 1);
+ if (copy_to_user(buf, &msg, count))
+ ret = -EFAULT;
+
+ return ret < 0 ? ret : count;
+}
+
+static ssize_t ipmb_write(struct file *file, const char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct ipmb_dev *ipmb_dev = to_ipmb_dev(file);
+ u8 rq_sa, netf_rq_lun, msg_len;
+ union i2c_smbus_data data;
+ u8 msg[MAX_MSG_LEN];
+ ssize_t ret;
+
+ if (count > sizeof(msg))
+ return -EINVAL;
+
+ if (copy_from_user(&msg, buf, count))
+ return -EFAULT;
+
+ if (count < msg[0])
+ return -EINVAL;
+
+ rq_sa = GET_7BIT_ADDR(msg[RQ_SA_8BIT_IDX]);
+ netf_rq_lun = msg[NETFN_LUN_IDX];
+
+ if (!(netf_rq_lun & NETFN_RSP_BIT_MASK))
+ return -EINVAL;
+
+ /*
+ * subtract rq_sa and netf_rq_lun from the length of the msg passed to
+ * i2c_smbus_xfer
+ */
+ msg_len = msg[IPMB_MSG_LEN_IDX] - SMBUS_MSG_HEADER_LENGTH;
+ if (msg_len > I2C_SMBUS_BLOCK_MAX)
+ msg_len = I2C_SMBUS_BLOCK_MAX;
+
+ data.block[0] = msg_len;
+ memcpy(&data.block[1], msg + SMBUS_MSG_IDX_OFFSET, msg_len);
+ ret = i2c_smbus_xfer(ipmb_dev->client->adapter, rq_sa,
+ ipmb_dev->client->flags,
+ I2C_SMBUS_WRITE, netf_rq_lun,
+ I2C_SMBUS_BLOCK_DATA, &data);
+
+ return ret ? : count;
+}
+
+static unsigned int ipmb_poll(struct file *file, poll_table *wait)
+{
+ struct ipmb_dev *ipmb_dev = to_ipmb_dev(file);
+ unsigned int mask = POLLOUT;
+
+ mutex_lock(&ipmb_dev->file_mutex);
+ poll_wait(file, &ipmb_dev->wait_queue, wait);
+
+ if (atomic_read(&ipmb_dev->request_queue_len))
+ mask |= POLLIN;
+ mutex_unlock(&ipmb_dev->file_mutex);
+
+ return mask;
+}
+
+static const struct file_operations ipmb_fops = {
+ .owner = THIS_MODULE,
+ .read = ipmb_read,
+ .write = ipmb_write,
+ .poll = ipmb_poll,
+};
+
+/* Called with ipmb_dev->lock held. */
+static void ipmb_handle_request(struct ipmb_dev *ipmb_dev)
+{
+ struct ipmb_request_elem *queue_elem;
+
+ if (atomic_read(&ipmb_dev->request_queue_len) >=
+ REQUEST_QUEUE_MAX_LEN)
+ return;
+
+ queue_elem = kmalloc(sizeof(*queue_elem), GFP_ATOMIC);
+ if (!queue_elem)
+ return;
+
+ memcpy(&queue_elem->request, &ipmb_dev->request,
+ sizeof(struct ipmb_msg));
+ list_add(&queue_elem->list, &ipmb_dev->request_queue);
+ atomic_inc(&ipmb_dev->request_queue_len);
+ wake_up_all(&ipmb_dev->wait_queue);
+}
+
+static u8 ipmb_verify_checksum1(struct ipmb_dev *ipmb_dev, u8 rs_sa)
+{
+ /* The 8 lsb of the sum is 0 when the checksum is valid */
+ return (rs_sa + ipmb_dev->request.netfn_rs_lun +
+ ipmb_dev->request.checksum1);
+}
+
+static bool is_ipmb_request(struct ipmb_dev *ipmb_dev, u8 rs_sa)
+{
+ if (ipmb_dev->msg_idx >= IPMB_REQUEST_LEN_MIN) {
+ if (ipmb_verify_checksum1(ipmb_dev, rs_sa))
+ return false;
+
+ /*
+ * Check whether this is an IPMB request or
+ * response.
+ * The 6 MSB of netfn_rs_lun are dedicated to the netfn
+ * while the remaining bits are dedicated to the lun.
+ * If the LSB of the netfn is cleared, it is associated
+ * with an IPMB request.
+ * If the LSB of the netfn is set, it is associated with
+ * an IPMB response.
+ */
+ if (!(ipmb_dev->request.netfn_rs_lun & NETFN_RSP_BIT_MASK))
+ return true;
+ }
+ return false;
+}
+
+/*
+ * The IPMB protocol only supports I2C Writes so there is no need
+ * to support I2C_SLAVE_READ* events.
+ * This i2c callback function only monitors IPMB request messages
+ * and adds them in a queue, so that they can be handled by
+ * receive_ipmb_request.
+ */
+static int ipmb_slave_cb(struct i2c_client *client,
+ enum i2c_slave_event event, u8 *val)
+{
+ struct ipmb_dev *ipmb_dev = i2c_get_clientdata(client);
+ u8 *buf = (u8 *)&ipmb_dev->request;
+ unsigned long flags;
+
+ spin_lock_irqsave(&ipmb_dev->lock, flags);
+ switch (event) {
+ case I2C_SLAVE_WRITE_REQUESTED:
+ memset(&ipmb_dev->request, 0, sizeof(ipmb_dev->request));
+ ipmb_dev->msg_idx = 0;
+
+ /*
+ * At index 0, ipmb_msg stores the length of msg,
+ * skip it for now.
+ * The len will be populated once the whole
+ * buf is populated.
+ *
+ * The I2C bus driver's responsibility is to pass the
+ * data bytes to the backend driver; it does not
+ * forward the i2c slave address.
+ * Since the first byte in the IPMB message is the
+ * address of the responder, it is the responsibility
+ * of the IPMB driver to format the message properly.
+ * So this driver prepends the address of the responder
+ * to the received i2c data before the request message
+ * is handled in userland.
+ */
+ buf[++ipmb_dev->msg_idx] = GET_8BIT_ADDR(client->addr);
+ break;
+
+ case I2C_SLAVE_WRITE_RECEIVED:
+ if (ipmb_dev->msg_idx >= sizeof(struct ipmb_msg))
+ break;
+
+ buf[++ipmb_dev->msg_idx] = *val;
+ break;
+
+ case I2C_SLAVE_STOP:
+ ipmb_dev->request.len = ipmb_dev->msg_idx;
+
+ if (is_ipmb_request(ipmb_dev, GET_8BIT_ADDR(client->addr)))
+ ipmb_handle_request(ipmb_dev);
+ break;
+
+ default:
+ break;
+ }
+ spin_unlock_irqrestore(&ipmb_dev->lock, flags);
+
+ return 0;
+}
+
+static int ipmb_probe(struct i2c_client *client,
+ const struct i2c_device_id *id)
+{
+ struct ipmb_dev *ipmb_dev;
+ int ret;
+
+ ipmb_dev = devm_kzalloc(&client->dev, sizeof(*ipmb_dev),
+ GFP_KERNEL);
+ if (!ipmb_dev)
+ return -ENOMEM;
+
+ spin_lock_init(&ipmb_dev->lock);
+ init_waitqueue_head(&ipmb_dev->wait_queue);
+ atomic_set(&ipmb_dev->request_queue_len, 0);
+ INIT_LIST_HEAD(&ipmb_dev->request_queue);
+
+ mutex_init(&ipmb_dev->file_mutex);
+
+ ipmb_dev->miscdev.minor = MISC_DYNAMIC_MINOR;
+
+ ipmb_dev->miscdev.name = devm_kasprintf(&client->dev, GFP_KERNEL,
+ "%s%d", "ipmb-",
+ client->adapter->nr);
+ ipmb_dev->miscdev.fops = &ipmb_fops;
+ ipmb_dev->miscdev.parent = &client->dev;
+ ret = misc_register(&ipmb_dev->miscdev);
+ if (ret)
+ return ret;
+
+ ipmb_dev->client = client;
+ i2c_set_clientdata(client, ipmb_dev);
+ ret = i2c_slave_register(client, ipmb_slave_cb);
+ if (ret) {
+ misc_deregister(&ipmb_dev->miscdev);
+ return ret;
+ }
+
+ return 0;
+}
+
+static int ipmb_remove(struct i2c_client *client)
+{
+ struct ipmb_dev *ipmb_dev = i2c_get_clientdata(client);
+
+ i2c_slave_unregister(client);
+ misc_deregister(&ipmb_dev->miscdev);
+
+ return 0;
+}
+
+static const struct i2c_device_id ipmb_id[] = {
+ { "ipmb-dev", 0 },
+ {},
+};
+MODULE_DEVICE_TABLE(i2c, ipmb_id);
+
+static const struct acpi_device_id acpi_ipmb_id[] = {
+ { "IPMB0001", 0 },
+ {},
+};
+MODULE_DEVICE_TABLE(acpi, acpi_ipmb_id);
+
+static struct i2c_driver ipmb_driver = {
+ .driver = {
+ .name = "ipmb-dev",
+ .acpi_match_table = ACPI_PTR(acpi_ipmb_id),
+ },
+ .probe = ipmb_probe,
+ .remove = ipmb_remove,
+ .id_table = ipmb_id,
+};
+module_i2c_driver(ipmb_driver);
+
+MODULE_AUTHOR("Mellanox Technologies");
+MODULE_DESCRIPTION("IPMB driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c
index 1dc10740fc0f..6707659cffd6 100644
--- a/drivers/char/ipmi/ipmi_msghandler.c
+++ b/drivers/char/ipmi/ipmi_msghandler.c
@@ -2819,9 +2819,9 @@ static const struct device_type bmc_device_type = {
.groups = bmc_dev_attr_groups,
};
-static int __find_bmc_guid(struct device *dev, void *data)
+static int __find_bmc_guid(struct device *dev, const void *data)
{
- guid_t *guid = data;
+ const guid_t *guid = data;
struct bmc_device *bmc;
int rv;
@@ -2857,9 +2857,9 @@ struct prod_dev_id {
unsigned char device_id;
};
-static int __find_bmc_prod_dev_id(struct device *dev, void *data)
+static int __find_bmc_prod_dev_id(struct device *dev, const void *data)
{
- struct prod_dev_id *cid = data;
+ const struct prod_dev_id *cid = data;
struct bmc_device *bmc;
int rv;
diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c
index f124a2d2bb9f..da5b6723329a 100644
--- a/drivers/char/ipmi/ipmi_si_intf.c
+++ b/drivers/char/ipmi/ipmi_si_intf.c
@@ -71,7 +71,7 @@ enum si_intf_state {
static const char * const si_to_str[] = { "invalid", "kcs", "smic", "bt" };
-static int initialized;
+static bool initialized;
/*
* Indexes into stats[] in smi_info below.
@@ -2124,7 +2124,7 @@ static int __init init_ipmi_si(void)
}
skip_fallback_noirq:
- initialized = 1;
+ initialized = true;
mutex_unlock(&smi_infos_lock);
if (type)
diff --git a/drivers/char/ipmi/ipmi_si_platform.c b/drivers/char/ipmi/ipmi_si_platform.c
index f2a91c4d8cab..22f6c9b20e9a 100644
--- a/drivers/char/ipmi/ipmi_si_platform.c
+++ b/drivers/char/ipmi/ipmi_si_platform.c
@@ -19,6 +19,7 @@
#include "ipmi_si.h"
#include "ipmi_dmi.h"
+static bool platform_registered;
static bool si_tryplatform = true;
#ifdef CONFIG_ACPI
static bool si_tryacpi = true;
@@ -426,7 +427,7 @@ static int ipmi_remove(struct platform_device *pdev)
return ipmi_si_remove_by_dev(&pdev->dev);
}
-static int pdev_match_name(struct device *dev, void *data)
+static int pdev_match_name(struct device *dev, const void *data)
{
struct platform_device *pdev = to_platform_device(dev);
const char *name = data;
@@ -443,6 +444,7 @@ void ipmi_remove_platform_device_by_name(char *name)
struct platform_device *pdev = to_platform_device(dev);
platform_device_unregister(pdev);
+ put_device(dev);
}
}
@@ -469,9 +471,12 @@ void ipmi_si_platform_init(void)
int rv = platform_driver_register(&ipmi_platform_driver);
if (rv)
pr_err("Unable to register driver: %d\n", rv);
+ else
+ platform_registered = true;
}
void ipmi_si_platform_shutdown(void)
{
- platform_driver_unregister(&ipmi_platform_driver);
+ if (platform_registered)
+ platform_driver_unregister(&ipmi_platform_driver);
}
diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c
index cf8156d6bc07..305fa5054274 100644
--- a/drivers/char/ipmi/ipmi_ssif.c
+++ b/drivers/char/ipmi/ipmi_ssif.c
@@ -303,6 +303,7 @@ struct ssif_info {
((unsigned int) atomic_read(&(ssif)->stats[SSIF_STAT_ ## stat]))
static bool initialized;
+static bool platform_registered;
static void return_hosed_msg(struct ssif_info *ssif_info,
struct ipmi_smi_msg *msg);
@@ -2088,6 +2089,8 @@ static int init_ipmi_ssif(void)
rv = platform_driver_register(&ipmi_driver);
if (rv)
pr_err("Unable to register driver: %d\n", rv);
+ else
+ platform_registered = true;
}
ssif_i2c_driver.address_list = ssif_address_list();
@@ -2111,7 +2114,7 @@ static void cleanup_ipmi_ssif(void)
kfree(ssif_i2c_driver.address_list);
- if (ssif_trydmi)
+ if (ssif_trydmi && platform_registered)
platform_driver_unregister(&ipmi_driver);
free_ssif_clients();
diff --git a/drivers/char/mbcs.c b/drivers/char/mbcs.c
deleted file mode 100644
index 0a31b60bee7b..000000000000
--- a/drivers/char/mbcs.c
+++ /dev/null
@@ -1,831 +0,0 @@
-/*
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file "COPYING" in the main directory of this archive
- * for more details.
- *
- * Copyright (c) 2005 Silicon Graphics, Inc. All rights reserved.
- */
-
-/*
- * MOATB Core Services driver.
- */
-
-#include <linux/interrupt.h>
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/types.h>
-#include <linux/ioport.h>
-#include <linux/kernel.h>
-#include <linux/notifier.h>
-#include <linux/reboot.h>
-#include <linux/init.h>
-#include <linux/fs.h>
-#include <linux/delay.h>
-#include <linux/device.h>
-#include <linux/mm.h>
-#include <linux/uio.h>
-#include <linux/mutex.h>
-#include <linux/slab.h>
-#include <linux/pagemap.h>
-#include <asm/io.h>
-#include <linux/uaccess.h>
-#include <asm/pgtable.h>
-#include <asm/sn/addrs.h>
-#include <asm/sn/intr.h>
-#include <asm/sn/tiocx.h>
-#include "mbcs.h"
-
-#define MBCS_DEBUG 0
-#if MBCS_DEBUG
-#define DBG(fmt...) printk(KERN_ALERT fmt)
-#else
-#define DBG(fmt...)
-#endif
-static DEFINE_MUTEX(mbcs_mutex);
-static int mbcs_major;
-
-static LIST_HEAD(soft_list);
-
-/*
- * file operations
- */
-static const struct file_operations mbcs_ops = {
- .owner = THIS_MODULE,
- .open = mbcs_open,
- .llseek = mbcs_sram_llseek,
- .read = mbcs_sram_read,
- .write = mbcs_sram_write,
- .mmap = mbcs_gscr_mmap,
-};
-
-struct mbcs_callback_arg {
- int minor;
- struct cx_dev *cx_dev;
-};
-
-static inline void mbcs_getdma_init(struct getdma *gdma)
-{
- memset(gdma, 0, sizeof(struct getdma));
- gdma->DoneIntEnable = 1;
-}
-
-static inline void mbcs_putdma_init(struct putdma *pdma)
-{
- memset(pdma, 0, sizeof(struct putdma));
- pdma->DoneIntEnable = 1;
-}
-
-static inline void mbcs_algo_init(struct algoblock *algo_soft)
-{
- memset(algo_soft, 0, sizeof(struct algoblock));
-}
-
-static inline void mbcs_getdma_set(void *mmr,
- uint64_t hostAddr,
- uint64_t localAddr,
- uint64_t localRamSel,
- uint64_t numPkts,
- uint64_t amoEnable,
- uint64_t intrEnable,
- uint64_t peerIO,
- uint64_t amoHostDest,
- uint64_t amoModType, uint64_t intrHostDest,
- uint64_t intrVector)
-{
- union dma_control rdma_control;
- union dma_amo_dest amo_dest;
- union intr_dest intr_dest;
- union dma_localaddr local_addr;
- union dma_hostaddr host_addr;
-
- rdma_control.dma_control_reg = 0;
- amo_dest.dma_amo_dest_reg = 0;
- intr_dest.intr_dest_reg = 0;
- local_addr.dma_localaddr_reg = 0;
- host_addr.dma_hostaddr_reg = 0;
-
- host_addr.dma_sys_addr = hostAddr;
- MBCS_MMR_SET(mmr, MBCS_RD_DMA_SYS_ADDR, host_addr.dma_hostaddr_reg);
-
- local_addr.dma_ram_addr = localAddr;
- local_addr.dma_ram_sel = localRamSel;
- MBCS_MMR_SET(mmr, MBCS_RD_DMA_LOC_ADDR, local_addr.dma_localaddr_reg);
-
- rdma_control.dma_op_length = numPkts;
- rdma_control.done_amo_en = amoEnable;
- rdma_control.done_int_en = intrEnable;
- rdma_control.pio_mem_n = peerIO;
- MBCS_MMR_SET(mmr, MBCS_RD_DMA_CTRL, rdma_control.dma_control_reg);
-
- amo_dest.dma_amo_sys_addr = amoHostDest;
- amo_dest.dma_amo_mod_type = amoModType;
- MBCS_MMR_SET(mmr, MBCS_RD_DMA_AMO_DEST, amo_dest.dma_amo_dest_reg);
-
- intr_dest.address = intrHostDest;
- intr_dest.int_vector = intrVector;
- MBCS_MMR_SET(mmr, MBCS_RD_DMA_INT_DEST, intr_dest.intr_dest_reg);
-
-}
-
-static inline void mbcs_putdma_set(void *mmr,
- uint64_t hostAddr,
- uint64_t localAddr,
- uint64_t localRamSel,
- uint64_t numPkts,
- uint64_t amoEnable,
- uint64_t intrEnable,
- uint64_t peerIO,
- uint64_t amoHostDest,
- uint64_t amoModType,
- uint64_t intrHostDest, uint64_t intrVector)
-{
- union dma_control wdma_control;
- union dma_amo_dest amo_dest;
- union intr_dest intr_dest;
- union dma_localaddr local_addr;
- union dma_hostaddr host_addr;
-
- wdma_control.dma_control_reg = 0;
- amo_dest.dma_amo_dest_reg = 0;
- intr_dest.intr_dest_reg = 0;
- local_addr.dma_localaddr_reg = 0;
- host_addr.dma_hostaddr_reg = 0;
-
- host_addr.dma_sys_addr = hostAddr;
- MBCS_MMR_SET(mmr, MBCS_WR_DMA_SYS_ADDR, host_addr.dma_hostaddr_reg);
-
- local_addr.dma_ram_addr = localAddr;
- local_addr.dma_ram_sel = localRamSel;
- MBCS_MMR_SET(mmr, MBCS_WR_DMA_LOC_ADDR, local_addr.dma_localaddr_reg);
-
- wdma_control.dma_op_length = numPkts;
- wdma_control.done_amo_en = amoEnable;
- wdma_control.done_int_en = intrEnable;
- wdma_control.pio_mem_n = peerIO;
- MBCS_MMR_SET(mmr, MBCS_WR_DMA_CTRL, wdma_control.dma_control_reg);
-
- amo_dest.dma_amo_sys_addr = amoHostDest;
- amo_dest.dma_amo_mod_type = amoModType;
- MBCS_MMR_SET(mmr, MBCS_WR_DMA_AMO_DEST, amo_dest.dma_amo_dest_reg);
-
- intr_dest.address = intrHostDest;
- intr_dest.int_vector = intrVector;
- MBCS_MMR_SET(mmr, MBCS_WR_DMA_INT_DEST, intr_dest.intr_dest_reg);
-
-}
-
-static inline void mbcs_algo_set(void *mmr,
- uint64_t amoHostDest,
- uint64_t amoModType,
- uint64_t intrHostDest,
- uint64_t intrVector, uint64_t algoStepCount)
-{
- union dma_amo_dest amo_dest;
- union intr_dest intr_dest;
- union algo_step step;
-
- step.algo_step_reg = 0;
- intr_dest.intr_dest_reg = 0;
- amo_dest.dma_amo_dest_reg = 0;
-
- amo_dest.dma_amo_sys_addr = amoHostDest;
- amo_dest.dma_amo_mod_type = amoModType;
- MBCS_MMR_SET(mmr, MBCS_ALG_AMO_DEST, amo_dest.dma_amo_dest_reg);
-
- intr_dest.address = intrHostDest;
- intr_dest.int_vector = intrVector;
- MBCS_MMR_SET(mmr, MBCS_ALG_INT_DEST, intr_dest.intr_dest_reg);
-
- step.alg_step_cnt = algoStepCount;
- MBCS_MMR_SET(mmr, MBCS_ALG_STEP, step.algo_step_reg);
-}
-
-static inline int mbcs_getdma_start(struct mbcs_soft *soft)
-{
- void *mmr_base;
- struct getdma *gdma;
- uint64_t numPkts;
- union cm_control cm_control;
-
- mmr_base = soft->mmr_base;
- gdma = &soft->getdma;
-
- /* check that host address got setup */
- if (!gdma->hostAddr)
- return -1;
-
- numPkts =
- (gdma->bytes + (MBCS_CACHELINE_SIZE - 1)) / MBCS_CACHELINE_SIZE;
-
- /* program engine */
- mbcs_getdma_set(mmr_base, tiocx_dma_addr(gdma->hostAddr),
- gdma->localAddr,
- (gdma->localAddr < MB2) ? 0 :
- (gdma->localAddr < MB4) ? 1 :
- (gdma->localAddr < MB6) ? 2 : 3,
- numPkts,
- gdma->DoneAmoEnable,
- gdma->DoneIntEnable,
- gdma->peerIO,
- gdma->amoHostDest,
- gdma->amoModType,
- gdma->intrHostDest, gdma->intrVector);
-
- /* start engine */
- cm_control.cm_control_reg = MBCS_MMR_GET(mmr_base, MBCS_CM_CONTROL);
- cm_control.rd_dma_go = 1;
- MBCS_MMR_SET(mmr_base, MBCS_CM_CONTROL, cm_control.cm_control_reg);
-
- return 0;
-
-}
-
-static inline int mbcs_putdma_start(struct mbcs_soft *soft)
-{
- void *mmr_base;
- struct putdma *pdma;
- uint64_t numPkts;
- union cm_control cm_control;
-
- mmr_base = soft->mmr_base;
- pdma = &soft->putdma;
-
- /* check that host address got setup */
- if (!pdma->hostAddr)
- return -1;
-
- numPkts =
- (pdma->bytes + (MBCS_CACHELINE_SIZE - 1)) / MBCS_CACHELINE_SIZE;
-
- /* program engine */
- mbcs_putdma_set(mmr_base, tiocx_dma_addr(pdma->hostAddr),
- pdma->localAddr,
- (pdma->localAddr < MB2) ? 0 :
- (pdma->localAddr < MB4) ? 1 :
- (pdma->localAddr < MB6) ? 2 : 3,
- numPkts,
- pdma->DoneAmoEnable,
- pdma->DoneIntEnable,
- pdma->peerIO,
- pdma->amoHostDest,
- pdma->amoModType,
- pdma->intrHostDest, pdma->intrVector);
-
- /* start engine */
- cm_control.cm_control_reg = MBCS_MMR_GET(mmr_base, MBCS_CM_CONTROL);
- cm_control.wr_dma_go = 1;
- MBCS_MMR_SET(mmr_base, MBCS_CM_CONTROL, cm_control.cm_control_reg);
-
- return 0;
-
-}
-
-static inline int mbcs_algo_start(struct mbcs_soft *soft)
-{
- struct algoblock *algo_soft = &soft->algo;
- void *mmr_base = soft->mmr_base;
- union cm_control cm_control;
-
- if (mutex_lock_interruptible(&soft->algolock))
- return -ERESTARTSYS;
-
- atomic_set(&soft->algo_done, 0);
-
- mbcs_algo_set(mmr_base,
- algo_soft->amoHostDest,
- algo_soft->amoModType,
- algo_soft->intrHostDest,
- algo_soft->intrVector, algo_soft->algoStepCount);
-
- /* start algorithm */
- cm_control.cm_control_reg = MBCS_MMR_GET(mmr_base, MBCS_CM_CONTROL);
- cm_control.alg_done_int_en = 1;
- cm_control.alg_go = 1;
- MBCS_MMR_SET(mmr_base, MBCS_CM_CONTROL, cm_control.cm_control_reg);
-
- mutex_unlock(&soft->algolock);
-
- return 0;
-}
-
-static inline ssize_t
-do_mbcs_sram_dmawrite(struct mbcs_soft *soft, uint64_t hostAddr,
- size_t len, loff_t * off)
-{
- int rv = 0;
-
- if (mutex_lock_interruptible(&soft->dmawritelock))
- return -ERESTARTSYS;
-
- atomic_set(&soft->dmawrite_done, 0);
-
- soft->putdma.hostAddr = hostAddr;
- soft->putdma.localAddr = *off;
- soft->putdma.bytes = len;
-
- if (mbcs_putdma_start(soft) < 0) {
- DBG(KERN_ALERT "do_mbcs_sram_dmawrite: "
- "mbcs_putdma_start failed\n");
- rv = -EAGAIN;
- goto dmawrite_exit;
- }
-
- if (wait_event_interruptible(soft->dmawrite_queue,
- atomic_read(&soft->dmawrite_done))) {
- rv = -ERESTARTSYS;
- goto dmawrite_exit;
- }
-
- rv = len;
- *off += len;
-
-dmawrite_exit:
- mutex_unlock(&soft->dmawritelock);
-
- return rv;
-}
-
-static inline ssize_t
-do_mbcs_sram_dmaread(struct mbcs_soft *soft, uint64_t hostAddr,
- size_t len, loff_t * off)
-{
- int rv = 0;
-
- if (mutex_lock_interruptible(&soft->dmareadlock))
- return -ERESTARTSYS;
-
- atomic_set(&soft->dmawrite_done, 0);
-
- soft->getdma.hostAddr = hostAddr;
- soft->getdma.localAddr = *off;
- soft->getdma.bytes = len;
-
- if (mbcs_getdma_start(soft) < 0) {
- DBG(KERN_ALERT "mbcs_strategy: mbcs_getdma_start failed\n");
- rv = -EAGAIN;
- goto dmaread_exit;
- }
-
- if (wait_event_interruptible(soft->dmaread_queue,
- atomic_read(&soft->dmaread_done))) {
- rv = -ERESTARTSYS;
- goto dmaread_exit;
- }
-
- rv = len;
- *off += len;
-
-dmaread_exit:
- mutex_unlock(&soft->dmareadlock);
-
- return rv;
-}
-
-static int mbcs_open(struct inode *ip, struct file *fp)
-{
- struct mbcs_soft *soft;
- int minor;
-
- mutex_lock(&mbcs_mutex);
- minor = iminor(ip);
-
- /* Nothing protects access to this list... */
- list_for_each_entry(soft, &soft_list, list) {
- if (soft->nasid == minor) {
- fp->private_data = soft->cxdev;
- mutex_unlock(&mbcs_mutex);
- return 0;
- }
- }
-
- mutex_unlock(&mbcs_mutex);
- return -ENODEV;
-}
-
-static ssize_t mbcs_sram_read(struct file * fp, char __user *buf, size_t len, loff_t * off)
-{
- struct cx_dev *cx_dev = fp->private_data;
- struct mbcs_soft *soft = cx_dev->soft;
- uint64_t hostAddr;
- int rv = 0;
-
- hostAddr = __get_dma_pages(GFP_KERNEL, get_order(len));
- if (hostAddr == 0)
- return -ENOMEM;
-
- rv = do_mbcs_sram_dmawrite(soft, hostAddr, len, off);
- if (rv < 0)
- goto exit;
-
- if (copy_to_user(buf, (void *)hostAddr, len))
- rv = -EFAULT;
-
- exit:
- free_pages(hostAddr, get_order(len));
-
- return rv;
-}
-
-static ssize_t
-mbcs_sram_write(struct file * fp, const char __user *buf, size_t len, loff_t * off)
-{
- struct cx_dev *cx_dev = fp->private_data;
- struct mbcs_soft *soft = cx_dev->soft;
- uint64_t hostAddr;
- int rv = 0;
-
- hostAddr = __get_dma_pages(GFP_KERNEL, get_order(len));
- if (hostAddr == 0)
- return -ENOMEM;
-
- if (copy_from_user((void *)hostAddr, buf, len)) {
- rv = -EFAULT;
- goto exit;
- }
-
- rv = do_mbcs_sram_dmaread(soft, hostAddr, len, off);
-
- exit:
- free_pages(hostAddr, get_order(len));
-
- return rv;
-}
-
-static loff_t mbcs_sram_llseek(struct file * filp, loff_t off, int whence)
-{
- return generic_file_llseek_size(filp, off, whence, MAX_LFS_FILESIZE,
- MBCS_SRAM_SIZE);
-}
-
-static uint64_t mbcs_pioaddr(struct mbcs_soft *soft, uint64_t offset)
-{
- uint64_t mmr_base;
-
- mmr_base = (uint64_t) (soft->mmr_base + offset);
-
- return mmr_base;
-}
-
-static void mbcs_debug_pioaddr_set(struct mbcs_soft *soft)
-{
- soft->debug_addr = mbcs_pioaddr(soft, MBCS_DEBUG_START);
-}
-
-static void mbcs_gscr_pioaddr_set(struct mbcs_soft *soft)
-{
- soft->gscr_addr = mbcs_pioaddr(soft, MBCS_GSCR_START);
-}
-
-static int mbcs_gscr_mmap(struct file *fp, struct vm_area_struct *vma)
-{
- struct cx_dev *cx_dev = fp->private_data;
- struct mbcs_soft *soft = cx_dev->soft;
-
- if (vma->vm_pgoff != 0)
- return -EINVAL;
-
- vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
-
- /* Remap-pfn-range will mark the range VM_IO */
- if (remap_pfn_range(vma,
- vma->vm_start,
- __pa(soft->gscr_addr) >> PAGE_SHIFT,
- PAGE_SIZE,
- vma->vm_page_prot))
- return -EAGAIN;
-
- return 0;
-}
-
-/**
- * mbcs_completion_intr_handler - Primary completion handler.
- * @irq: irq
- * @arg: soft struct for device
- *
- */
-static irqreturn_t
-mbcs_completion_intr_handler(int irq, void *arg)
-{
- struct mbcs_soft *soft = (struct mbcs_soft *)arg;
- void *mmr_base;
- union cm_status cm_status;
- union cm_control cm_control;
-
- mmr_base = soft->mmr_base;
- cm_status.cm_status_reg = MBCS_MMR_GET(mmr_base, MBCS_CM_STATUS);
-
- if (cm_status.rd_dma_done) {
- /* stop dma-read engine, clear status */
- cm_control.cm_control_reg =
- MBCS_MMR_GET(mmr_base, MBCS_CM_CONTROL);
- cm_control.rd_dma_clr = 1;
- MBCS_MMR_SET(mmr_base, MBCS_CM_CONTROL,
- cm_control.cm_control_reg);
- atomic_set(&soft->dmaread_done, 1);
- wake_up(&soft->dmaread_queue);
- }
- if (cm_status.wr_dma_done) {
- /* stop dma-write engine, clear status */
- cm_control.cm_control_reg =
- MBCS_MMR_GET(mmr_base, MBCS_CM_CONTROL);
- cm_control.wr_dma_clr = 1;
- MBCS_MMR_SET(mmr_base, MBCS_CM_CONTROL,
- cm_control.cm_control_reg);
- atomic_set(&soft->dmawrite_done, 1);
- wake_up(&soft->dmawrite_queue);
- }
- if (cm_status.alg_done) {
- /* clear status */
- cm_control.cm_control_reg =
- MBCS_MMR_GET(mmr_base, MBCS_CM_CONTROL);
- cm_control.alg_done_clr = 1;
- MBCS_MMR_SET(mmr_base, MBCS_CM_CONTROL,
- cm_control.cm_control_reg);
- atomic_set(&soft->algo_done, 1);
- wake_up(&soft->algo_queue);
- }
-
- return IRQ_HANDLED;
-}
-
-/**
- * mbcs_intr_alloc - Allocate interrupts.
- * @dev: device pointer
- *
- */
-static int mbcs_intr_alloc(struct cx_dev *dev)
-{
- struct sn_irq_info *sn_irq;
- struct mbcs_soft *soft;
- struct getdma *getdma;
- struct putdma *putdma;
- struct algoblock *algo;
-
- soft = dev->soft;
- getdma = &soft->getdma;
- putdma = &soft->putdma;
- algo = &soft->algo;
-
- soft->get_sn_irq = NULL;
- soft->put_sn_irq = NULL;
- soft->algo_sn_irq = NULL;
-
- sn_irq = tiocx_irq_alloc(dev->cx_id.nasid, TIOCX_CORELET, -1, -1, -1);
- if (sn_irq == NULL)
- return -EAGAIN;
- soft->get_sn_irq = sn_irq;
- getdma->intrHostDest = sn_irq->irq_xtalkaddr;
- getdma->intrVector = sn_irq->irq_irq;
- if (request_irq(sn_irq->irq_irq,
- (void *)mbcs_completion_intr_handler, IRQF_SHARED,
- "MBCS get intr", (void *)soft)) {
- tiocx_irq_free(soft->get_sn_irq);
- return -EAGAIN;
- }
-
- sn_irq = tiocx_irq_alloc(dev->cx_id.nasid, TIOCX_CORELET, -1, -1, -1);
- if (sn_irq == NULL) {
- free_irq(soft->get_sn_irq->irq_irq, soft);
- tiocx_irq_free(soft->get_sn_irq);
- return -EAGAIN;
- }
- soft->put_sn_irq = sn_irq;
- putdma->intrHostDest = sn_irq->irq_xtalkaddr;
- putdma->intrVector = sn_irq->irq_irq;
- if (request_irq(sn_irq->irq_irq,
- (void *)mbcs_completion_intr_handler, IRQF_SHARED,
- "MBCS put intr", (void *)soft)) {
- tiocx_irq_free(soft->put_sn_irq);
- free_irq(soft->get_sn_irq->irq_irq, soft);
- tiocx_irq_free(soft->get_sn_irq);
- return -EAGAIN;
- }
-
- sn_irq = tiocx_irq_alloc(dev->cx_id.nasid, TIOCX_CORELET, -1, -1, -1);
- if (sn_irq == NULL) {
- free_irq(soft->put_sn_irq->irq_irq, soft);
- tiocx_irq_free(soft->put_sn_irq);
- free_irq(soft->get_sn_irq->irq_irq, soft);
- tiocx_irq_free(soft->get_sn_irq);
- return -EAGAIN;
- }
- soft->algo_sn_irq = sn_irq;
- algo->intrHostDest = sn_irq->irq_xtalkaddr;
- algo->intrVector = sn_irq->irq_irq;
- if (request_irq(sn_irq->irq_irq,
- (void *)mbcs_completion_intr_handler, IRQF_SHARED,
- "MBCS algo intr", (void *)soft)) {
- tiocx_irq_free(soft->algo_sn_irq);
- free_irq(soft->put_sn_irq->irq_irq, soft);
- tiocx_irq_free(soft->put_sn_irq);
- free_irq(soft->get_sn_irq->irq_irq, soft);
- tiocx_irq_free(soft->get_sn_irq);
- return -EAGAIN;
- }
-
- return 0;
-}
-
-/**
- * mbcs_intr_dealloc - Remove interrupts.
- * @dev: device pointer
- *
- */
-static void mbcs_intr_dealloc(struct cx_dev *dev)
-{
- struct mbcs_soft *soft;
-
- soft = dev->soft;
-
- free_irq(soft->get_sn_irq->irq_irq, soft);
- tiocx_irq_free(soft->get_sn_irq);
- free_irq(soft->put_sn_irq->irq_irq, soft);
- tiocx_irq_free(soft->put_sn_irq);
- free_irq(soft->algo_sn_irq->irq_irq, soft);
- tiocx_irq_free(soft->algo_sn_irq);
-}
-
-static inline int mbcs_hw_init(struct mbcs_soft *soft)
-{
- void *mmr_base = soft->mmr_base;
- union cm_control cm_control;
- union cm_req_timeout cm_req_timeout;
- uint64_t err_stat;
-
- cm_req_timeout.cm_req_timeout_reg =
- MBCS_MMR_GET(mmr_base, MBCS_CM_REQ_TOUT);
-
- cm_req_timeout.time_out = MBCS_CM_CONTROL_REQ_TOUT_MASK;
- MBCS_MMR_SET(mmr_base, MBCS_CM_REQ_TOUT,
- cm_req_timeout.cm_req_timeout_reg);
-
- mbcs_gscr_pioaddr_set(soft);
- mbcs_debug_pioaddr_set(soft);
-
- /* clear errors */
- err_stat = MBCS_MMR_GET(mmr_base, MBCS_CM_ERR_STAT);
- MBCS_MMR_SET(mmr_base, MBCS_CM_CLR_ERR_STAT, err_stat);
- MBCS_MMR_ZERO(mmr_base, MBCS_CM_ERROR_DETAIL1);
-
- /* enable interrupts */
- /* turn off 2^23 (INT_EN_PIO_REQ_ADDR_INV) */
- MBCS_MMR_SET(mmr_base, MBCS_CM_ERR_INT_EN, 0x3ffffff7e00ffUL);
-
- /* arm status regs and clear engines */
- cm_control.cm_control_reg = MBCS_MMR_GET(mmr_base, MBCS_CM_CONTROL);
- cm_control.rearm_stat_regs = 1;
- cm_control.alg_clr = 1;
- cm_control.wr_dma_clr = 1;
- cm_control.rd_dma_clr = 1;
-
- MBCS_MMR_SET(mmr_base, MBCS_CM_CONTROL, cm_control.cm_control_reg);
-
- return 0;
-}
-
-static ssize_t show_algo(struct device *dev, struct device_attribute *attr, char *buf)
-{
- struct cx_dev *cx_dev = to_cx_dev(dev);
- struct mbcs_soft *soft = cx_dev->soft;
- uint64_t debug0;
-
- /*
- * By convention, the first debug register contains the
- * algorithm number and revision.
- */
- debug0 = *(uint64_t *) soft->debug_addr;
-
- return sprintf(buf, "0x%x 0x%x\n",
- upper_32_bits(debug0), lower_32_bits(debug0));
-}
-
-static ssize_t store_algo(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
-{
- int n;
- struct cx_dev *cx_dev = to_cx_dev(dev);
- struct mbcs_soft *soft = cx_dev->soft;
-
- if (count <= 0)
- return 0;
-
- n = simple_strtoul(buf, NULL, 0);
-
- if (n == 1) {
- mbcs_algo_start(soft);
- if (wait_event_interruptible(soft->algo_queue,
- atomic_read(&soft->algo_done)))
- return -ERESTARTSYS;
- }
-
- return count;
-}
-
-DEVICE_ATTR(algo, 0644, show_algo, store_algo);
-
-/**
- * mbcs_probe - Initialize for device
- * @dev: device pointer
- * @device_id: id table pointer
- *
- */
-static int mbcs_probe(struct cx_dev *dev, const struct cx_device_id *id)
-{
- struct mbcs_soft *soft;
-
- dev->soft = NULL;
-
- soft = kzalloc(sizeof(struct mbcs_soft), GFP_KERNEL);
- if (soft == NULL)
- return -ENOMEM;
-
- soft->nasid = dev->cx_id.nasid;
- list_add(&soft->list, &soft_list);
- soft->mmr_base = (void *)tiocx_swin_base(dev->cx_id.nasid);
- dev->soft = soft;
- soft->cxdev = dev;
-
- init_waitqueue_head(&soft->dmawrite_queue);
- init_waitqueue_head(&soft->dmaread_queue);
- init_waitqueue_head(&soft->algo_queue);
-
- mutex_init(&soft->dmawritelock);
- mutex_init(&soft->dmareadlock);
- mutex_init(&soft->algolock);
-
- mbcs_getdma_init(&soft->getdma);
- mbcs_putdma_init(&soft->putdma);
- mbcs_algo_init(&soft->algo);
-
- mbcs_hw_init(soft);
-
- /* Allocate interrupts */
- mbcs_intr_alloc(dev);
-
- device_create_file(&dev->dev, &dev_attr_algo);
-
- return 0;
-}
-
-static int mbcs_remove(struct cx_dev *dev)
-{
- if (dev->soft) {
- mbcs_intr_dealloc(dev);
- kfree(dev->soft);
- }
-
- device_remove_file(&dev->dev, &dev_attr_algo);
-
- return 0;
-}
-
-static const struct cx_device_id mbcs_id_table[] = {
- {
- .part_num = MBCS_PART_NUM,
- .mfg_num = MBCS_MFG_NUM,
- },
- {
- .part_num = MBCS_PART_NUM_ALG0,
- .mfg_num = MBCS_MFG_NUM,
- },
- {0, 0}
-};
-
-MODULE_DEVICE_TABLE(cx, mbcs_id_table);
-
-static struct cx_drv mbcs_driver = {
- .name = DEVICE_NAME,
- .id_table = mbcs_id_table,
- .probe = mbcs_probe,
- .remove = mbcs_remove,
-};
-
-static void __exit mbcs_exit(void)
-{
- unregister_chrdev(mbcs_major, DEVICE_NAME);
- cx_driver_unregister(&mbcs_driver);
-}
-
-static int __init mbcs_init(void)
-{
- int rv;
-
- if (!ia64_platform_is("sn2"))
- return -ENODEV;
-
- // Put driver into chrdevs[]. Get major number.
- rv = register_chrdev(mbcs_major, DEVICE_NAME, &mbcs_ops);
- if (rv < 0) {
- DBG(KERN_ALERT "mbcs_init: can't get major number. %d\n", rv);
- return rv;
- }
- mbcs_major = rv;
-
- return cx_driver_register(&mbcs_driver);
-}
-
-module_init(mbcs_init);
-module_exit(mbcs_exit);
-
-MODULE_AUTHOR("Bruce Losure <blosure@sgi.com>");
-MODULE_DESCRIPTION("Driver for MOATB Core Services");
-MODULE_LICENSE("GPL");
diff --git a/drivers/char/mbcs.h b/drivers/char/mbcs.h
deleted file mode 100644
index 1a36884c48b5..000000000000
--- a/drivers/char/mbcs.h
+++ /dev/null
@@ -1,553 +0,0 @@
-/*
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file "COPYING" in the main directory of this archive
- * for more details.
- *
- * Copyright (c) 2005 Silicon Graphics, Inc. All rights reserved.
- */
-
-#ifndef __MBCS_H__
-#define __MBCS_H__
-
-/*
- * General macros
- */
-#define MB (1024*1024)
-#define MB2 (2*MB)
-#define MB4 (4*MB)
-#define MB6 (6*MB)
-
-/*
- * Offsets and masks
- */
-#define MBCS_CM_ID 0x0000 /* Identification */
-#define MBCS_CM_STATUS 0x0008 /* Status */
-#define MBCS_CM_ERROR_DETAIL1 0x0010 /* Error Detail1 */
-#define MBCS_CM_ERROR_DETAIL2 0x0018 /* Error Detail2 */
-#define MBCS_CM_CONTROL 0x0020 /* Control */
-#define MBCS_CM_REQ_TOUT 0x0028 /* Request Time-out */
-#define MBCS_CM_ERR_INT_DEST 0x0038 /* Error Interrupt Destination */
-#define MBCS_CM_TARG_FL 0x0050 /* Target Flush */
-#define MBCS_CM_ERR_STAT 0x0060 /* Error Status */
-#define MBCS_CM_CLR_ERR_STAT 0x0068 /* Clear Error Status */
-#define MBCS_CM_ERR_INT_EN 0x0070 /* Error Interrupt Enable */
-#define MBCS_RD_DMA_SYS_ADDR 0x0100 /* Read DMA System Address */
-#define MBCS_RD_DMA_LOC_ADDR 0x0108 /* Read DMA Local Address */
-#define MBCS_RD_DMA_CTRL 0x0110 /* Read DMA Control */
-#define MBCS_RD_DMA_AMO_DEST 0x0118 /* Read DMA AMO Destination */
-#define MBCS_RD_DMA_INT_DEST 0x0120 /* Read DMA Interrupt Destination */
-#define MBCS_RD_DMA_AUX_STAT 0x0130 /* Read DMA Auxiliary Status */
-#define MBCS_WR_DMA_SYS_ADDR 0x0200 /* Write DMA System Address */
-#define MBCS_WR_DMA_LOC_ADDR 0x0208 /* Write DMA Local Address */
-#define MBCS_WR_DMA_CTRL 0x0210 /* Write DMA Control */
-#define MBCS_WR_DMA_AMO_DEST 0x0218 /* Write DMA AMO Destination */
-#define MBCS_WR_DMA_INT_DEST 0x0220 /* Write DMA Interrupt Destination */
-#define MBCS_WR_DMA_AUX_STAT 0x0230 /* Write DMA Auxiliary Status */
-#define MBCS_ALG_AMO_DEST 0x0300 /* Algorithm AMO Destination */
-#define MBCS_ALG_INT_DEST 0x0308 /* Algorithm Interrupt Destination */
-#define MBCS_ALG_OFFSETS 0x0310
-#define MBCS_ALG_STEP 0x0318 /* Algorithm Step */
-
-#define MBCS_GSCR_START 0x0000000
-#define MBCS_DEBUG_START 0x0100000
-#define MBCS_RAM0_START 0x0200000
-#define MBCS_RAM1_START 0x0400000
-#define MBCS_RAM2_START 0x0600000
-
-#define MBCS_CM_CONTROL_REQ_TOUT_MASK 0x0000000000ffffffUL
-//#define PIO_BASE_ADDR_BASE_OFFSET_MASK 0x00fffffffff00000UL
-
-#define MBCS_SRAM_SIZE (1024*1024)
-#define MBCS_CACHELINE_SIZE 128
-
-/*
- * MMR get's and put's
- */
-#define MBCS_MMR_ADDR(mmr_base, offset)((uint64_t *)(mmr_base + offset))
-#define MBCS_MMR_SET(mmr_base, offset, value) { \
- uint64_t *mbcs_mmr_set_u64p, readback; \
- mbcs_mmr_set_u64p = (uint64_t *)(mmr_base + offset); \
- *mbcs_mmr_set_u64p = value; \
- readback = *mbcs_mmr_set_u64p; \
-}
-#define MBCS_MMR_GET(mmr_base, offset) *(uint64_t *)(mmr_base + offset)
-#define MBCS_MMR_ZERO(mmr_base, offset) MBCS_MMR_SET(mmr_base, offset, 0)
-
-/*
- * MBCS mmr structures
- */
-union cm_id {
- uint64_t cm_id_reg;
- struct {
- uint64_t always_one:1, // 0
- mfg_id:11, // 11:1
- part_num:16, // 27:12
- bitstream_rev:8, // 35:28
- :28; // 63:36
- };
-};
-
-union cm_status {
- uint64_t cm_status_reg;
- struct {
- uint64_t pending_reads:8, // 7:0
- pending_writes:8, // 15:8
- ice_rsp_credits:8, // 23:16
- ice_req_credits:8, // 31:24
- cm_req_credits:8, // 39:32
- :1, // 40
- rd_dma_in_progress:1, // 41
- rd_dma_done:1, // 42
- :1, // 43
- wr_dma_in_progress:1, // 44
- wr_dma_done:1, // 45
- alg_waiting:1, // 46
- alg_pipe_running:1, // 47
- alg_done:1, // 48
- :3, // 51:49
- pending_int_reqs:8, // 59:52
- :3, // 62:60
- alg_half_speed_sel:1; // 63
- };
-};
-
-union cm_error_detail1 {
- uint64_t cm_error_detail1_reg;
- struct {
- uint64_t packet_type:4, // 3:0
- source_id:2, // 5:4
- data_size:2, // 7:6
- tnum:8, // 15:8
- byte_enable:8, // 23:16
- gfx_cred:8, // 31:24
- read_type:2, // 33:32
- pio_or_memory:1, // 34
- head_cw_error:1, // 35
- :12, // 47:36
- head_error_bit:1, // 48
- data_error_bit:1, // 49
- :13, // 62:50
- valid:1; // 63
- };
-};
-
-union cm_error_detail2 {
- uint64_t cm_error_detail2_reg;
- struct {
- uint64_t address:56, // 55:0
- :8; // 63:56
- };
-};
-
-union cm_control {
- uint64_t cm_control_reg;
- struct {
- uint64_t cm_id:2, // 1:0
- :2, // 3:2
- max_trans:5, // 8:4
- :3, // 11:9
- address_mode:1, // 12
- :7, // 19:13
- credit_limit:8, // 27:20
- :5, // 32:28
- rearm_stat_regs:1, // 33
- prescalar_byp:1, // 34
- force_gap_war:1, // 35
- rd_dma_go:1, // 36
- wr_dma_go:1, // 37
- alg_go:1, // 38
- rd_dma_clr:1, // 39
- wr_dma_clr:1, // 40
- alg_clr:1, // 41
- :2, // 43:42
- alg_wait_step:1, // 44
- alg_done_amo_en:1, // 45
- alg_done_int_en:1, // 46
- :1, // 47
- alg_sram0_locked:1, // 48
- alg_sram1_locked:1, // 49
- alg_sram2_locked:1, // 50
- alg_done_clr:1, // 51
- :12; // 63:52
- };
-};
-
-union cm_req_timeout {
- uint64_t cm_req_timeout_reg;
- struct {
- uint64_t time_out:24, // 23:0
- :40; // 63:24
- };
-};
-
-union intr_dest {
- uint64_t intr_dest_reg;
- struct {
- uint64_t address:56, // 55:0
- int_vector:8; // 63:56
- };
-};
-
-union cm_error_status {
- uint64_t cm_error_status_reg;
- struct {
- uint64_t ecc_sbe:1, // 0
- ecc_mbe:1, // 1
- unsupported_req:1, // 2
- unexpected_rsp:1, // 3
- bad_length:1, // 4
- bad_datavalid:1, // 5
- buffer_overflow:1, // 6
- request_timeout:1, // 7
- :8, // 15:8
- head_inv_data_size:1, // 16
- rsp_pactype_inv:1, // 17
- head_sb_err:1, // 18
- missing_head:1, // 19
- head_inv_rd_type:1, // 20
- head_cmd_err_bit:1, // 21
- req_addr_align_inv:1, // 22
- pio_req_addr_inv:1, // 23
- req_range_dsize_inv:1, // 24
- early_term:1, // 25
- early_tail:1, // 26
- missing_tail:1, // 27
- data_flit_sb_err:1, // 28
- cm2hcm_req_cred_of:1, // 29
- cm2hcm_rsp_cred_of:1, // 30
- rx_bad_didn:1, // 31
- rd_dma_err_rsp:1, // 32
- rd_dma_tnum_tout:1, // 33
- rd_dma_multi_tnum_tou:1, // 34
- wr_dma_err_rsp:1, // 35
- wr_dma_tnum_tout:1, // 36
- wr_dma_multi_tnum_tou:1, // 37
- alg_data_overflow:1, // 38
- alg_data_underflow:1, // 39
- ram0_access_conflict:1, // 40
- ram1_access_conflict:1, // 41
- ram2_access_conflict:1, // 42
- ram0_perr:1, // 43
- ram1_perr:1, // 44
- ram2_perr:1, // 45
- int_gen_rsp_err:1, // 46
- int_gen_tnum_tout:1, // 47
- rd_dma_prog_err:1, // 48
- wr_dma_prog_err:1, // 49
- :14; // 63:50
- };
-};
-
-union cm_clr_error_status {
- uint64_t cm_clr_error_status_reg;
- struct {
- uint64_t clr_ecc_sbe:1, // 0
- clr_ecc_mbe:1, // 1
- clr_unsupported_req:1, // 2
- clr_unexpected_rsp:1, // 3
- clr_bad_length:1, // 4
- clr_bad_datavalid:1, // 5
- clr_buffer_overflow:1, // 6
- clr_request_timeout:1, // 7
- :8, // 15:8
- clr_head_inv_data_siz:1, // 16
- clr_rsp_pactype_inv:1, // 17
- clr_head_sb_err:1, // 18
- clr_missing_head:1, // 19
- clr_head_inv_rd_type:1, // 20
- clr_head_cmd_err_bit:1, // 21
- clr_req_addr_align_in:1, // 22
- clr_pio_req_addr_inv:1, // 23
- clr_req_range_dsize_i:1, // 24
- clr_early_term:1, // 25
- clr_early_tail:1, // 26
- clr_missing_tail:1, // 27
- clr_data_flit_sb_err:1, // 28
- clr_cm2hcm_req_cred_o:1, // 29
- clr_cm2hcm_rsp_cred_o:1, // 30
- clr_rx_bad_didn:1, // 31
- clr_rd_dma_err_rsp:1, // 32
- clr_rd_dma_tnum_tout:1, // 33
- clr_rd_dma_multi_tnum:1, // 34
- clr_wr_dma_err_rsp:1, // 35
- clr_wr_dma_tnum_tout:1, // 36
- clr_wr_dma_multi_tnum:1, // 37
- clr_alg_data_overflow:1, // 38
- clr_alg_data_underflo:1, // 39
- clr_ram0_access_confl:1, // 40
- clr_ram1_access_confl:1, // 41
- clr_ram2_access_confl:1, // 42
- clr_ram0_perr:1, // 43
- clr_ram1_perr:1, // 44
- clr_ram2_perr:1, // 45
- clr_int_gen_rsp_err:1, // 46
- clr_int_gen_tnum_tout:1, // 47
- clr_rd_dma_prog_err:1, // 48
- clr_wr_dma_prog_err:1, // 49
- :14; // 63:50
- };
-};
-
-union cm_error_intr_enable {
- uint64_t cm_error_intr_enable_reg;
- struct {
- uint64_t int_en_ecc_sbe:1, // 0
- int_en_ecc_mbe:1, // 1
- int_en_unsupported_re:1, // 2
- int_en_unexpected_rsp:1, // 3
- int_en_bad_length:1, // 4
- int_en_bad_datavalid:1, // 5
- int_en_buffer_overflo:1, // 6
- int_en_request_timeou:1, // 7
- :8, // 15:8
- int_en_head_inv_data_:1, // 16
- int_en_rsp_pactype_in:1, // 17
- int_en_head_sb_err:1, // 18
- int_en_missing_head:1, // 19
- int_en_head_inv_rd_ty:1, // 20
- int_en_head_cmd_err_b:1, // 21
- int_en_req_addr_align:1, // 22
- int_en_pio_req_addr_i:1, // 23
- int_en_req_range_dsiz:1, // 24
- int_en_early_term:1, // 25
- int_en_early_tail:1, // 26
- int_en_missing_tail:1, // 27
- int_en_data_flit_sb_e:1, // 28
- int_en_cm2hcm_req_cre:1, // 29
- int_en_cm2hcm_rsp_cre:1, // 30
- int_en_rx_bad_didn:1, // 31
- int_en_rd_dma_err_rsp:1, // 32
- int_en_rd_dma_tnum_to:1, // 33
- int_en_rd_dma_multi_t:1, // 34
- int_en_wr_dma_err_rsp:1, // 35
- int_en_wr_dma_tnum_to:1, // 36
- int_en_wr_dma_multi_t:1, // 37
- int_en_alg_data_overf:1, // 38
- int_en_alg_data_under:1, // 39
- int_en_ram0_access_co:1, // 40
- int_en_ram1_access_co:1, // 41
- int_en_ram2_access_co:1, // 42
- int_en_ram0_perr:1, // 43
- int_en_ram1_perr:1, // 44
- int_en_ram2_perr:1, // 45
- int_en_int_gen_rsp_er:1, // 46
- int_en_int_gen_tnum_t:1, // 47
- int_en_rd_dma_prog_er:1, // 48
- int_en_wr_dma_prog_er:1, // 49
- :14; // 63:50
- };
-};
-
-struct cm_mmr {
- union cm_id id;
- union cm_status status;
- union cm_error_detail1 err_detail1;
- union cm_error_detail2 err_detail2;
- union cm_control control;
- union cm_req_timeout req_timeout;
- uint64_t reserved1[1];
- union intr_dest int_dest;
- uint64_t reserved2[2];
- uint64_t targ_flush;
- uint64_t reserved3[1];
- union cm_error_status err_status;
- union cm_clr_error_status clr_err_status;
- union cm_error_intr_enable int_enable;
-};
-
-union dma_hostaddr {
- uint64_t dma_hostaddr_reg;
- struct {
- uint64_t dma_sys_addr:56, // 55:0
- :8; // 63:56
- };
-};
-
-union dma_localaddr {
- uint64_t dma_localaddr_reg;
- struct {
- uint64_t dma_ram_addr:21, // 20:0
- dma_ram_sel:2, // 22:21
- :41; // 63:23
- };
-};
-
-union dma_control {
- uint64_t dma_control_reg;
- struct {
- uint64_t dma_op_length:16, // 15:0
- :18, // 33:16
- done_amo_en:1, // 34
- done_int_en:1, // 35
- :1, // 36
- pio_mem_n:1, // 37
- :26; // 63:38
- };
-};
-
-union dma_amo_dest {
- uint64_t dma_amo_dest_reg;
- struct {
- uint64_t dma_amo_sys_addr:56, // 55:0
- dma_amo_mod_type:3, // 58:56
- :5; // 63:59
- };
-};
-
-union rdma_aux_status {
- uint64_t rdma_aux_status_reg;
- struct {
- uint64_t op_num_pacs_left:17, // 16:0
- :5, // 21:17
- lrsp_buff_empty:1, // 22
- :17, // 39:23
- pending_reqs_left:6, // 45:40
- :18; // 63:46
- };
-};
-
-struct rdma_mmr {
- union dma_hostaddr host_addr;
- union dma_localaddr local_addr;
- union dma_control control;
- union dma_amo_dest amo_dest;
- union intr_dest intr_dest;
- union rdma_aux_status aux_status;
-};
-
-union wdma_aux_status {
- uint64_t wdma_aux_status_reg;
- struct {
- uint64_t op_num_pacs_left:17, // 16:0
- :4, // 20:17
- lreq_buff_empty:1, // 21
- :18, // 39:22
- pending_reqs_left:6, // 45:40
- :18; // 63:46
- };
-};
-
-struct wdma_mmr {
- union dma_hostaddr host_addr;
- union dma_localaddr local_addr;
- union dma_control control;
- union dma_amo_dest amo_dest;
- union intr_dest intr_dest;
- union wdma_aux_status aux_status;
-};
-
-union algo_step {
- uint64_t algo_step_reg;
- struct {
- uint64_t alg_step_cnt:16, // 15:0
- :48; // 63:16
- };
-};
-
-struct algo_mmr {
- union dma_amo_dest amo_dest;
- union intr_dest intr_dest;
- union {
- uint64_t algo_offset_reg;
- struct {
- uint64_t sram0_offset:7, // 6:0
- reserved0:1, // 7
- sram1_offset:7, // 14:8
- reserved1:1, // 15
- sram2_offset:7, // 22:16
- reserved2:14; // 63:23
- };
- } sram_offset;
- union algo_step step;
-};
-
-struct mbcs_mmr {
- struct cm_mmr cm;
- uint64_t reserved1[17];
- struct rdma_mmr rdDma;
- uint64_t reserved2[25];
- struct wdma_mmr wrDma;
- uint64_t reserved3[25];
- struct algo_mmr algo;
- uint64_t reserved4[156];
-};
-
-/*
- * defines
- */
-#define DEVICE_NAME "mbcs"
-#define MBCS_PART_NUM 0xfff0
-#define MBCS_PART_NUM_ALG0 0xf001
-#define MBCS_MFG_NUM 0x1
-
-struct algoblock {
- uint64_t amoHostDest;
- uint64_t amoModType;
- uint64_t intrHostDest;
- uint64_t intrVector;
- uint64_t algoStepCount;
-};
-
-struct getdma {
- uint64_t hostAddr;
- uint64_t localAddr;
- uint64_t bytes;
- uint64_t DoneAmoEnable;
- uint64_t DoneIntEnable;
- uint64_t peerIO;
- uint64_t amoHostDest;
- uint64_t amoModType;
- uint64_t intrHostDest;
- uint64_t intrVector;
-};
-
-struct putdma {
- uint64_t hostAddr;
- uint64_t localAddr;
- uint64_t bytes;
- uint64_t DoneAmoEnable;
- uint64_t DoneIntEnable;
- uint64_t peerIO;
- uint64_t amoHostDest;
- uint64_t amoModType;
- uint64_t intrHostDest;
- uint64_t intrVector;
-};
-
-struct mbcs_soft {
- struct list_head list;
- struct cx_dev *cxdev;
- int major;
- int nasid;
- void *mmr_base;
- wait_queue_head_t dmawrite_queue;
- wait_queue_head_t dmaread_queue;
- wait_queue_head_t algo_queue;
- struct sn_irq_info *get_sn_irq;
- struct sn_irq_info *put_sn_irq;
- struct sn_irq_info *algo_sn_irq;
- struct getdma getdma;
- struct putdma putdma;
- struct algoblock algo;
- uint64_t gscr_addr; // pio addr
- uint64_t ram0_addr; // pio addr
- uint64_t ram1_addr; // pio addr
- uint64_t ram2_addr; // pio addr
- uint64_t debug_addr; // pio addr
- atomic_t dmawrite_done;
- atomic_t dmaread_done;
- atomic_t algo_done;
- struct mutex dmawritelock;
- struct mutex dmareadlock;
- struct mutex algolock;
-};
-
-static int mbcs_open(struct inode *ip, struct file *fp);
-static ssize_t mbcs_sram_read(struct file *fp, char __user *buf, size_t len,
- loff_t * off);
-static ssize_t mbcs_sram_write(struct file *fp, const char __user *buf, size_t len,
- loff_t * off);
-static loff_t mbcs_sram_llseek(struct file *filp, loff_t off, int whence);
-static int mbcs_gscr_mmap(struct file *fp, struct vm_area_struct *vma);
-
-#endif // __MBCS_H__
diff --git a/drivers/char/mem.c b/drivers/char/mem.c
index b08dc50f9f26..9eb564c002f6 100644
--- a/drivers/char/mem.c
+++ b/drivers/char/mem.c
@@ -97,6 +97,13 @@ void __weak unxlate_dev_mem_ptr(phys_addr_t phys, void *addr)
}
#endif
+static inline bool should_stop_iteration(void)
+{
+ if (need_resched())
+ cond_resched();
+ return fatal_signal_pending(current);
+}
+
/*
* This funcion reads the *physical* memory. The f_pos points directly to the
* memory location.
@@ -175,6 +182,8 @@ static ssize_t read_mem(struct file *file, char __user *buf,
p += sz;
count -= sz;
read += sz;
+ if (should_stop_iteration())
+ break;
}
kfree(bounce);
@@ -251,6 +260,8 @@ static ssize_t write_mem(struct file *file, const char __user *buf,
p += sz;
count -= sz;
written += sz;
+ if (should_stop_iteration())
+ break;
}
*ppos += written;
@@ -468,6 +479,10 @@ static ssize_t read_kmem(struct file *file, char __user *buf,
read += sz;
low_count -= sz;
count -= sz;
+ if (should_stop_iteration()) {
+ count = 0;
+ break;
+ }
}
}
@@ -492,6 +507,8 @@ static ssize_t read_kmem(struct file *file, char __user *buf,
buf += sz;
read += sz;
p += sz;
+ if (should_stop_iteration())
+ break;
}
free_page((unsigned long)kbuf);
}
@@ -544,6 +561,8 @@ static ssize_t do_write_kmem(unsigned long p, const char __user *buf,
p += sz;
count -= sz;
written += sz;
+ if (should_stop_iteration())
+ break;
}
*ppos += written;
@@ -595,6 +614,8 @@ static ssize_t write_kmem(struct file *file, const char __user *buf,
buf += sz;
virtr += sz;
p += sz;
+ if (should_stop_iteration())
+ break;
}
free_page((unsigned long)kbuf);
}
diff --git a/drivers/char/misc.c b/drivers/char/misc.c
index 53cfe574d8d4..f6a147427029 100644
--- a/drivers/char/misc.c
+++ b/drivers/char/misc.c
@@ -226,6 +226,7 @@ int misc_register(struct miscdevice *misc)
mutex_unlock(&misc_mtx);
return err;
}
+EXPORT_SYMBOL(misc_register);
/**
* misc_deregister - unregister a miscellaneous device
@@ -249,8 +250,6 @@ void misc_deregister(struct miscdevice *misc)
clear_bit(i, misc_minors);
mutex_unlock(&misc_mtx);
}
-
-EXPORT_SYMBOL(misc_register);
EXPORT_SYMBOL(misc_deregister);
static char *misc_devnode(struct device *dev, umode_t *mode)
diff --git a/drivers/char/mspec.c b/drivers/char/mspec.c
index e75c9df7c2d8..a9d9f074fbd6 100644
--- a/drivers/char/mspec.c
+++ b/drivers/char/mspec.c
@@ -9,11 +9,8 @@
*
* This driver exports the SN special memory (mspec) facility to user
* processes.
- * There are three types of memory made available thru this driver:
- * fetchops, uncached and cached.
- *
- * Fetchops are atomic memory operations that are implemented in the
- * memory controller on SGI SN hardware.
+ * There are two types of memory made available thru this driver:
+ * uncached and cached.
*
* Uncached are used for memory write combining feature of the ia64
* cpu.
@@ -46,16 +43,8 @@
#include <linux/atomic.h>
#include <asm/tlbflush.h>
#include <asm/uncached.h>
-#include <asm/sn/addrs.h>
-#include <asm/sn/arch.h>
-#include <asm/sn/mspec.h>
-#include <asm/sn/sn_cpuid.h>
-#include <asm/sn/io.h>
-#include <asm/sn/bte.h>
-#include <asm/sn/shubio.h>
-#define FETCHOP_ID "SGI Fetchop,"
#define CACHED_ID "Cached,"
#define UNCACHED_ID "Uncached"
#define REVISION "4.0"
@@ -65,17 +54,10 @@
* Page types allocated by the device.
*/
enum mspec_page_type {
- MSPEC_FETCHOP = 1,
- MSPEC_CACHED,
+ MSPEC_CACHED = 2,
MSPEC_UNCACHED
};
-#ifdef CONFIG_SGI_SN
-static int is_sn2;
-#else
-#define is_sn2 0
-#endif
-
/*
* One of these structures is allocated when an mspec region is mmaped. The
* structure is pointed to by the vma->vm_private_data field in the vma struct.
@@ -96,39 +78,6 @@ struct vma_data {
unsigned long maddr[0]; /* Array of MSPEC addresses. */
};
-/* used on shub2 to clear FOP cache in the HUB */
-static unsigned long scratch_page[MAX_NUMNODES];
-#define SH2_AMO_CACHE_ENTRIES 4
-
-static inline int
-mspec_zero_block(unsigned long addr, int len)
-{
- int status;
-
- if (is_sn2) {
- if (is_shub2()) {
- int nid;
- void *p;
- int i;
-
- nid = nasid_to_cnodeid(get_node_number(__pa(addr)));
- p = (void *)TO_AMO(scratch_page[nid]);
-
- for (i=0; i < SH2_AMO_CACHE_ENTRIES; i++) {
- FETCHOP_LOAD_OP(p, FETCHOP_LOAD);
- p += FETCHOP_VAR_SIZE;
- }
- }
-
- status = bte_copy(0, addr & ~__IA64_UNCACHED_OFFSET, len,
- BTE_WACQUIRE | BTE_ZERO_FILL, NULL);
- } else {
- memset((char *) addr, 0, len);
- status = 0;
- }
- return status;
-}
-
/*
* mspec_open
*
@@ -173,11 +122,8 @@ mspec_close(struct vm_area_struct *vma)
*/
my_page = vdata->maddr[index];
vdata->maddr[index] = 0;
- if (!mspec_zero_block(my_page, PAGE_SIZE))
- uncached_free_page(my_page, 1);
- else
- printk(KERN_WARNING "mspec_close(): "
- "failed to zero page %ld\n", my_page);
+ memset((char *)my_page, 0, PAGE_SIZE);
+ uncached_free_page(my_page, 1);
}
kvfree(vdata);
@@ -213,11 +159,7 @@ mspec_fault(struct vm_fault *vmf)
spin_unlock(&vdata->lock);
}
- if (vdata->type == MSPEC_FETCHOP)
- paddr = TO_AMO(maddr);
- else
- paddr = maddr & ~__IA64_UNCACHED_OFFSET;
-
+ paddr = maddr & ~__IA64_UNCACHED_OFFSET;
pfn = paddr >> PAGE_SHIFT;
return vmf_insert_pfn(vmf->vma, vmf->address, pfn);
@@ -269,7 +211,7 @@ mspec_mmap(struct file *file, struct vm_area_struct *vma,
vma->vm_private_data = vdata;
vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
- if (vdata->type == MSPEC_FETCHOP || vdata->type == MSPEC_UNCACHED)
+ if (vdata->type == MSPEC_UNCACHED)
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
vma->vm_ops = &mspec_vm_ops;
@@ -277,12 +219,6 @@ mspec_mmap(struct file *file, struct vm_area_struct *vma,
}
static int
-fetchop_mmap(struct file *file, struct vm_area_struct *vma)
-{
- return mspec_mmap(file, vma, MSPEC_FETCHOP);
-}
-
-static int
cached_mmap(struct file *file, struct vm_area_struct *vma)
{
return mspec_mmap(file, vma, MSPEC_CACHED);
@@ -294,18 +230,6 @@ uncached_mmap(struct file *file, struct vm_area_struct *vma)
return mspec_mmap(file, vma, MSPEC_UNCACHED);
}
-static const struct file_operations fetchop_fops = {
- .owner = THIS_MODULE,
- .mmap = fetchop_mmap,
- .llseek = noop_llseek,
-};
-
-static struct miscdevice fetchop_miscdev = {
- .minor = MISC_DYNAMIC_MINOR,
- .name = "sgi_fetchop",
- .fops = &fetchop_fops
-};
-
static const struct file_operations cached_fops = {
.owner = THIS_MODULE,
.mmap = cached_mmap,
@@ -339,89 +263,32 @@ static int __init
mspec_init(void)
{
int ret;
- int nid;
-
- /*
- * The fetchop device only works on SN2 hardware, uncached and cached
- * memory drivers should both be valid on all ia64 hardware
- */
-#ifdef CONFIG_SGI_SN
- if (ia64_platform_is("sn2")) {
- is_sn2 = 1;
- if (is_shub2()) {
- ret = -ENOMEM;
- for_each_node_state(nid, N_ONLINE) {
- int actual_nid;
- int nasid;
- unsigned long phys;
-
- scratch_page[nid] = uncached_alloc_page(nid, 1);
- if (scratch_page[nid] == 0)
- goto free_scratch_pages;
- phys = __pa(scratch_page[nid]);
- nasid = get_node_number(phys);
- actual_nid = nasid_to_cnodeid(nasid);
- if (actual_nid != nid)
- goto free_scratch_pages;
- }
- }
- ret = misc_register(&fetchop_miscdev);
- if (ret) {
- printk(KERN_ERR
- "%s: failed to register device %i\n",
- FETCHOP_ID, ret);
- goto free_scratch_pages;
- }
- }
-#endif
ret = misc_register(&cached_miscdev);
if (ret) {
printk(KERN_ERR "%s: failed to register device %i\n",
CACHED_ID, ret);
- if (is_sn2)
- misc_deregister(&fetchop_miscdev);
- goto free_scratch_pages;
+ return ret;
}
ret = misc_register(&uncached_miscdev);
if (ret) {
printk(KERN_ERR "%s: failed to register device %i\n",
UNCACHED_ID, ret);
misc_deregister(&cached_miscdev);
- if (is_sn2)
- misc_deregister(&fetchop_miscdev);
- goto free_scratch_pages;
+ return ret;
}
- printk(KERN_INFO "%s %s initialized devices: %s %s %s\n",
- MSPEC_BASENAME, REVISION, is_sn2 ? FETCHOP_ID : "",
- CACHED_ID, UNCACHED_ID);
+ printk(KERN_INFO "%s %s initialized devices: %s %s\n",
+ MSPEC_BASENAME, REVISION, CACHED_ID, UNCACHED_ID);
return 0;
-
- free_scratch_pages:
- for_each_node(nid) {
- if (scratch_page[nid] != 0)
- uncached_free_page(scratch_page[nid], 1);
- }
- return ret;
}
static void __exit
mspec_exit(void)
{
- int nid;
-
misc_deregister(&uncached_miscdev);
misc_deregister(&cached_miscdev);
- if (is_sn2) {
- misc_deregister(&fetchop_miscdev);
-
- for_each_node(nid) {
- if (scratch_page[nid] != 0)
- uncached_free_page(scratch_page[nid], 1);
- }
- }
}
module_init(mspec_init);
diff --git a/drivers/char/ppdev.c b/drivers/char/ppdev.c
index f0a8adca1eee..c86f18aa8985 100644
--- a/drivers/char/ppdev.c
+++ b/drivers/char/ppdev.c
@@ -737,7 +737,7 @@ static int pp_release(struct inode *inode, struct file *file)
"negotiated back to compatibility mode because user-space forgot\n");
}
- if (pp->flags & PP_CLAIMED) {
+ if ((pp->flags & PP_CLAIMED) && pp->pdev) {
struct ieee1284_info *info;
info = &pp->pdev->port->ieee1284;
diff --git a/drivers/char/random.c b/drivers/char/random.c
index 5d5ea4ce1442..d3beed084c0a 100644
--- a/drivers/char/random.c
+++ b/drivers/char/random.c
@@ -327,6 +327,7 @@
#include <linux/percpu.h>
#include <linux/cryptohash.h>
#include <linux/fips.h>
+#include <linux/freezer.h>
#include <linux/ptrace.h>
#include <linux/workqueue.h>
#include <linux/irq.h>
@@ -2439,9 +2440,24 @@ void add_hwgenerator_randomness(const char *buffer, size_t count,
* We'll be woken up again once below random_write_wakeup_thresh,
* or when the calling thread is about to terminate.
*/
- wait_event_interruptible(random_write_wait, kthread_should_stop() ||
+ wait_event_freezable(random_write_wait,
+ kthread_should_stop() ||
ENTROPY_BITS(&input_pool) <= random_write_wakeup_bits);
mix_pool_bytes(poolp, buffer, count);
credit_entropy_bits(poolp, entropy);
}
EXPORT_SYMBOL_GPL(add_hwgenerator_randomness);
+
+/* Handle random seed passed by bootloader.
+ * If the seed is trustworthy, it would be regarded as hardware RNGs. Otherwise
+ * it would be regarded as device data.
+ * The decision is controlled by CONFIG_RANDOM_TRUST_BOOTLOADER.
+ */
+void add_bootloader_randomness(const void *buf, unsigned int size)
+{
+ if (IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER))
+ add_hwgenerator_randomness(buf, size, size * 8);
+ else
+ add_device_randomness(buf, size);
+}
+EXPORT_SYMBOL_GPL(add_bootloader_randomness); \ No newline at end of file
diff --git a/drivers/char/snsc.c b/drivers/char/snsc.c
deleted file mode 100644
index 5918ea7499bb..000000000000
--- a/drivers/char/snsc.c
+++ /dev/null
@@ -1,469 +0,0 @@
-/*
- * SN Platform system controller communication support
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file "COPYING" in the main directory of this archive
- * for more details.
- *
- * Copyright (C) 2004, 2006 Silicon Graphics, Inc. All rights reserved.
- */
-
-/*
- * System controller communication driver
- *
- * This driver allows a user process to communicate with the system
- * controller (a.k.a. "IRouter") network in an SGI SN system.
- */
-
-#include <linux/interrupt.h>
-#include <linux/sched/signal.h>
-#include <linux/device.h>
-#include <linux/poll.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/mutex.h>
-#include <asm/sn/io.h>
-#include <asm/sn/sn_sal.h>
-#include <asm/sn/module.h>
-#include <asm/sn/geo.h>
-#include <asm/sn/nodepda.h>
-#include "snsc.h"
-
-#define SYSCTL_BASENAME "snsc"
-
-#define SCDRV_BUFSZ 2048
-#define SCDRV_TIMEOUT 1000
-
-static DEFINE_MUTEX(scdrv_mutex);
-static irqreturn_t
-scdrv_interrupt(int irq, void *subch_data)
-{
- struct subch_data_s *sd = subch_data;
- unsigned long flags;
- int status;
-
- spin_lock_irqsave(&sd->sd_rlock, flags);
- spin_lock(&sd->sd_wlock);
- status = ia64_sn_irtr_intr(sd->sd_nasid, sd->sd_subch);
-
- if (status > 0) {
- if (status & SAL_IROUTER_INTR_RECV) {
- wake_up(&sd->sd_rq);
- }
- if (status & SAL_IROUTER_INTR_XMIT) {
- ia64_sn_irtr_intr_disable
- (sd->sd_nasid, sd->sd_subch,
- SAL_IROUTER_INTR_XMIT);
- wake_up(&sd->sd_wq);
- }
- }
- spin_unlock(&sd->sd_wlock);
- spin_unlock_irqrestore(&sd->sd_rlock, flags);
- return IRQ_HANDLED;
-}
-
-/*
- * scdrv_open
- *
- * Reserve a subchannel for system controller communication.
- */
-
-static int
-scdrv_open(struct inode *inode, struct file *file)
-{
- struct sysctl_data_s *scd;
- struct subch_data_s *sd;
- int rv;
-
- /* look up device info for this device file */
- scd = container_of(inode->i_cdev, struct sysctl_data_s, scd_cdev);
-
- /* allocate memory for subchannel data */
- sd = kzalloc(sizeof (struct subch_data_s), GFP_KERNEL);
- if (sd == NULL) {
- printk("%s: couldn't allocate subchannel data\n",
- __func__);
- return -ENOMEM;
- }
-
- /* initialize subch_data_s fields */
- sd->sd_nasid = scd->scd_nasid;
- sd->sd_subch = ia64_sn_irtr_open(scd->scd_nasid);
-
- if (sd->sd_subch < 0) {
- kfree(sd);
- printk("%s: couldn't allocate subchannel\n", __func__);
- return -EBUSY;
- }
-
- spin_lock_init(&sd->sd_rlock);
- spin_lock_init(&sd->sd_wlock);
- init_waitqueue_head(&sd->sd_rq);
- init_waitqueue_head(&sd->sd_wq);
- sema_init(&sd->sd_rbs, 1);
- sema_init(&sd->sd_wbs, 1);
-
- file->private_data = sd;
-
- /* hook this subchannel up to the system controller interrupt */
- mutex_lock(&scdrv_mutex);
- rv = request_irq(SGI_UART_VECTOR, scdrv_interrupt,
- IRQF_SHARED, SYSCTL_BASENAME, sd);
- if (rv) {
- ia64_sn_irtr_close(sd->sd_nasid, sd->sd_subch);
- kfree(sd);
- printk("%s: irq request failed (%d)\n", __func__, rv);
- mutex_unlock(&scdrv_mutex);
- return -EBUSY;
- }
- mutex_unlock(&scdrv_mutex);
- return 0;
-}
-
-/*
- * scdrv_release
- *
- * Release a previously-reserved subchannel.
- */
-
-static int
-scdrv_release(struct inode *inode, struct file *file)
-{
- struct subch_data_s *sd = (struct subch_data_s *) file->private_data;
- int rv;
-
- /* free the interrupt */
- free_irq(SGI_UART_VECTOR, sd);
-
- /* ask SAL to close the subchannel */
- rv = ia64_sn_irtr_close(sd->sd_nasid, sd->sd_subch);
-
- kfree(sd);
- return rv;
-}
-
-/*
- * scdrv_read
- *
- * Called to read bytes from the open IRouter pipe.
- *
- */
-
-static inline int
-read_status_check(struct subch_data_s *sd, int *len)
-{
- return ia64_sn_irtr_recv(sd->sd_nasid, sd->sd_subch, sd->sd_rb, len);
-}
-
-static ssize_t
-scdrv_read(struct file *file, char __user *buf, size_t count, loff_t *f_pos)
-{
- int status;
- int len;
- unsigned long flags;
- struct subch_data_s *sd = (struct subch_data_s *) file->private_data;
-
- /* try to get control of the read buffer */
- if (down_trylock(&sd->sd_rbs)) {
- /* somebody else has it now;
- * if we're non-blocking, then exit...
- */
- if (file->f_flags & O_NONBLOCK) {
- return -EAGAIN;
- }
- /* ...or if we want to block, then do so here */
- if (down_interruptible(&sd->sd_rbs)) {
- /* something went wrong with wait */
- return -ERESTARTSYS;
- }
- }
-
- /* anything to read? */
- len = CHUNKSIZE;
- spin_lock_irqsave(&sd->sd_rlock, flags);
- status = read_status_check(sd, &len);
-
- /* if not, and we're blocking I/O, loop */
- while (status < 0) {
- DECLARE_WAITQUEUE(wait, current);
-
- if (file->f_flags & O_NONBLOCK) {
- spin_unlock_irqrestore(&sd->sd_rlock, flags);
- up(&sd->sd_rbs);
- return -EAGAIN;
- }
-
- len = CHUNKSIZE;
- set_current_state(TASK_INTERRUPTIBLE);
- add_wait_queue(&sd->sd_rq, &wait);
- spin_unlock_irqrestore(&sd->sd_rlock, flags);
-
- schedule_timeout(msecs_to_jiffies(SCDRV_TIMEOUT));
-
- remove_wait_queue(&sd->sd_rq, &wait);
- if (signal_pending(current)) {
- /* wait was interrupted */
- up(&sd->sd_rbs);
- return -ERESTARTSYS;
- }
-
- spin_lock_irqsave(&sd->sd_rlock, flags);
- status = read_status_check(sd, &len);
- }
- spin_unlock_irqrestore(&sd->sd_rlock, flags);
-
- if (len > 0) {
- /* we read something in the last read_status_check(); copy
- * it out to user space
- */
- if (count < len) {
- pr_debug("%s: only accepting %d of %d bytes\n",
- __func__, (int) count, len);
- }
- len = min((int) count, len);
- if (copy_to_user(buf, sd->sd_rb, len))
- len = -EFAULT;
- }
-
- /* release the read buffer and wake anyone who might be
- * waiting for it
- */
- up(&sd->sd_rbs);
-
- /* return the number of characters read in */
- return len;
-}
-
-/*
- * scdrv_write
- *
- * Writes a chunk of an IRouter packet (or other system controller data)
- * to the system controller.
- *
- */
-static inline int
-write_status_check(struct subch_data_s *sd, int count)
-{
- return ia64_sn_irtr_send(sd->sd_nasid, sd->sd_subch, sd->sd_wb, count);
-}
-
-static ssize_t
-scdrv_write(struct file *file, const char __user *buf,
- size_t count, loff_t *f_pos)
-{
- unsigned long flags;
- int status;
- struct subch_data_s *sd = (struct subch_data_s *) file->private_data;
-
- /* try to get control of the write buffer */
- if (down_trylock(&sd->sd_wbs)) {
- /* somebody else has it now;
- * if we're non-blocking, then exit...
- */
- if (file->f_flags & O_NONBLOCK) {
- return -EAGAIN;
- }
- /* ...or if we want to block, then do so here */
- if (down_interruptible(&sd->sd_wbs)) {
- /* something went wrong with wait */
- return -ERESTARTSYS;
- }
- }
-
- count = min((int) count, CHUNKSIZE);
- if (copy_from_user(sd->sd_wb, buf, count)) {
- up(&sd->sd_wbs);
- return -EFAULT;
- }
-
- /* try to send the buffer */
- spin_lock_irqsave(&sd->sd_wlock, flags);
- status = write_status_check(sd, count);
-
- /* if we failed, and we want to block, then loop */
- while (status <= 0) {
- DECLARE_WAITQUEUE(wait, current);
-
- if (file->f_flags & O_NONBLOCK) {
- spin_unlock_irqrestore(&sd->sd_wlock, flags);
- up(&sd->sd_wbs);
- return -EAGAIN;
- }
-
- set_current_state(TASK_INTERRUPTIBLE);
- add_wait_queue(&sd->sd_wq, &wait);
- spin_unlock_irqrestore(&sd->sd_wlock, flags);
-
- schedule_timeout(msecs_to_jiffies(SCDRV_TIMEOUT));
-
- remove_wait_queue(&sd->sd_wq, &wait);
- if (signal_pending(current)) {
- /* wait was interrupted */
- up(&sd->sd_wbs);
- return -ERESTARTSYS;
- }
-
- spin_lock_irqsave(&sd->sd_wlock, flags);
- status = write_status_check(sd, count);
- }
- spin_unlock_irqrestore(&sd->sd_wlock, flags);
-
- /* release the write buffer and wake anyone who's waiting for it */
- up(&sd->sd_wbs);
-
- /* return the number of characters accepted (should be the complete
- * "chunk" as requested)
- */
- if ((status >= 0) && (status < count)) {
- pr_debug("Didn't accept the full chunk; %d of %d\n",
- status, (int) count);
- }
- return status;
-}
-
-static __poll_t
-scdrv_poll(struct file *file, struct poll_table_struct *wait)
-{
- __poll_t mask = 0;
- int status = 0;
- struct subch_data_s *sd = (struct subch_data_s *) file->private_data;
- unsigned long flags;
-
- poll_wait(file, &sd->sd_rq, wait);
- poll_wait(file, &sd->sd_wq, wait);
-
- spin_lock_irqsave(&sd->sd_rlock, flags);
- spin_lock(&sd->sd_wlock);
- status = ia64_sn_irtr_intr(sd->sd_nasid, sd->sd_subch);
- spin_unlock(&sd->sd_wlock);
- spin_unlock_irqrestore(&sd->sd_rlock, flags);
-
- if (status > 0) {
- if (status & SAL_IROUTER_INTR_RECV) {
- mask |= EPOLLIN | EPOLLRDNORM;
- }
- if (status & SAL_IROUTER_INTR_XMIT) {
- mask |= EPOLLOUT | EPOLLWRNORM;
- }
- }
-
- return mask;
-}
-
-static const struct file_operations scdrv_fops = {
- .owner = THIS_MODULE,
- .read = scdrv_read,
- .write = scdrv_write,
- .poll = scdrv_poll,
- .open = scdrv_open,
- .release = scdrv_release,
- .llseek = noop_llseek,
-};
-
-static struct class *snsc_class;
-
-/*
- * scdrv_init
- *
- * Called at boot time to initialize the system controller communication
- * facility.
- */
-int __init
-scdrv_init(void)
-{
- geoid_t geoid;
- cnodeid_t cnode;
- char devname[32];
- char *devnamep;
- struct sysctl_data_s *scd;
- void *salbuf;
- dev_t first_dev, dev;
- nasid_t event_nasid;
-
- if (!ia64_platform_is("sn2"))
- return -ENODEV;
-
- event_nasid = ia64_sn_get_console_nasid();
-
- snsc_class = class_create(THIS_MODULE, SYSCTL_BASENAME);
- if (IS_ERR(snsc_class)) {
- printk("%s: failed to allocate class\n", __func__);
- return PTR_ERR(snsc_class);
- }
-
- if (alloc_chrdev_region(&first_dev, 0, num_cnodes,
- SYSCTL_BASENAME) < 0) {
- printk("%s: failed to register SN system controller device\n",
- __func__);
- return -ENODEV;
- }
-
- for (cnode = 0; cnode < num_cnodes; cnode++) {
- geoid = cnodeid_get_geoid(cnode);
- devnamep = devname;
- format_module_id(devnamep, geo_module(geoid),
- MODULE_FORMAT_BRIEF);
- devnamep = devname + strlen(devname);
- sprintf(devnamep, "^%d#%d", geo_slot(geoid),
- geo_slab(geoid));
-
- /* allocate sysctl device data */
- scd = kzalloc(sizeof (struct sysctl_data_s),
- GFP_KERNEL);
- if (!scd) {
- printk("%s: failed to allocate device info"
- "for %s/%s\n", __func__,
- SYSCTL_BASENAME, devname);
- continue;
- }
-
- /* initialize sysctl device data fields */
- scd->scd_nasid = cnodeid_to_nasid(cnode);
- if (!(salbuf = kmalloc(SCDRV_BUFSZ, GFP_KERNEL))) {
- printk("%s: failed to allocate driver buffer"
- "(%s%s)\n", __func__,
- SYSCTL_BASENAME, devname);
- kfree(scd);
- continue;
- }
-
- if (ia64_sn_irtr_init(scd->scd_nasid, salbuf,
- SCDRV_BUFSZ) < 0) {
- printk
- ("%s: failed to initialize SAL for"
- " system controller communication"
- " (%s/%s): outdated PROM?\n",
- __func__, SYSCTL_BASENAME, devname);
- kfree(scd);
- kfree(salbuf);
- continue;
- }
-
- dev = first_dev + cnode;
- cdev_init(&scd->scd_cdev, &scdrv_fops);
- if (cdev_add(&scd->scd_cdev, dev, 1)) {
- printk("%s: failed to register system"
- " controller device (%s%s)\n",
- __func__, SYSCTL_BASENAME, devname);
- kfree(scd);
- kfree(salbuf);
- continue;
- }
-
- device_create(snsc_class, NULL, dev, NULL,
- "%s", devname);
-
- ia64_sn_irtr_intr_enable(scd->scd_nasid,
- 0 /*ignored */ ,
- SAL_IROUTER_INTR_RECV);
-
- /* on the console nasid, prepare to receive
- * system controller environmental events
- */
- if(scd->scd_nasid == event_nasid) {
- scdrv_event_init(scd);
- }
- }
- return 0;
-}
-device_initcall(scdrv_init);
diff --git a/drivers/char/snsc.h b/drivers/char/snsc.h
deleted file mode 100644
index e8c52c882b21..000000000000
--- a/drivers/char/snsc.h
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * SN Platform system controller communication support
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file "COPYING" in the main directory of this archive
- * for more details.
- *
- * Copyright (C) 2004-2006 Silicon Graphics, Inc. All rights reserved.
- */
-
-/*
- * This file contains macros and data types for communication with the
- * system controllers in SGI SN systems.
- */
-
-#ifndef _SN_SYSCTL_H_
-#define _SN_SYSCTL_H_
-
-#include <linux/types.h>
-#include <linux/spinlock.h>
-#include <linux/wait.h>
-#include <linux/fs.h>
-#include <linux/cdev.h>
-#include <linux/semaphore.h>
-#include <asm/sn/types.h>
-
-#define CHUNKSIZE 127
-
-/* This structure is used to track an open subchannel. */
-struct subch_data_s {
- nasid_t sd_nasid; /* node on which the subchannel was opened */
- int sd_subch; /* subchannel number */
- spinlock_t sd_rlock; /* monitor lock for rsv */
- spinlock_t sd_wlock; /* monitor lock for wsv */
- wait_queue_head_t sd_rq; /* wait queue for readers */
- wait_queue_head_t sd_wq; /* wait queue for writers */
- struct semaphore sd_rbs; /* semaphore for read buffer */
- struct semaphore sd_wbs; /* semaphore for write buffer */
-
- char sd_rb[CHUNKSIZE]; /* read buffer */
- char sd_wb[CHUNKSIZE]; /* write buffer */
-};
-
-struct sysctl_data_s {
- struct cdev scd_cdev; /* Character device info */
- nasid_t scd_nasid; /* Node on which subchannels are opened. */
-};
-
-
-/* argument types */
-#define IR_ARG_INT 0x00 /* 4-byte integer (big-endian) */
-#define IR_ARG_ASCII 0x01 /* null-terminated ASCII string */
-#define IR_ARG_UNKNOWN 0x80 /* unknown data type. The low
- * 7 bits will contain the data
- * length. */
-#define IR_ARG_UNKNOWN_LENGTH_MASK 0x7f
-
-
-/* system controller event codes */
-#define EV_CLASS_MASK 0xf000ul
-#define EV_SEVERITY_MASK 0x0f00ul
-#define EV_COMPONENT_MASK 0x00fful
-
-#define EV_CLASS_POWER 0x1000ul
-#define EV_CLASS_FAN 0x2000ul
-#define EV_CLASS_TEMP 0x3000ul
-#define EV_CLASS_ENV 0x4000ul
-#define EV_CLASS_TEST_FAULT 0x5000ul
-#define EV_CLASS_TEST_WARNING 0x6000ul
-#define EV_CLASS_PWRD_NOTIFY 0x8000ul
-
-/* ENV class codes */
-#define ENV_PWRDN_PEND 0x4101ul
-
-#define EV_SEVERITY_POWER_STABLE 0x0000ul
-#define EV_SEVERITY_POWER_LOW_WARNING 0x0100ul
-#define EV_SEVERITY_POWER_HIGH_WARNING 0x0200ul
-#define EV_SEVERITY_POWER_HIGH_FAULT 0x0300ul
-#define EV_SEVERITY_POWER_LOW_FAULT 0x0400ul
-
-#define EV_SEVERITY_FAN_STABLE 0x0000ul
-#define EV_SEVERITY_FAN_WARNING 0x0100ul
-#define EV_SEVERITY_FAN_FAULT 0x0200ul
-
-#define EV_SEVERITY_TEMP_STABLE 0x0000ul
-#define EV_SEVERITY_TEMP_ADVISORY 0x0100ul
-#define EV_SEVERITY_TEMP_CRITICAL 0x0200ul
-#define EV_SEVERITY_TEMP_FAULT 0x0300ul
-
-void scdrv_event_init(struct sysctl_data_s *);
-
-#endif /* _SN_SYSCTL_H_ */
diff --git a/drivers/char/snsc_event.c b/drivers/char/snsc_event.c
deleted file mode 100644
index e452673dff66..000000000000
--- a/drivers/char/snsc_event.c
+++ /dev/null
@@ -1,303 +0,0 @@
-/*
- * SN Platform system controller communication support
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file "COPYING" in the main directory of this archive
- * for more details.
- *
- * Copyright (C) 2004-2006 Silicon Graphics, Inc. All rights reserved.
- */
-
-/*
- * System controller event handler
- *
- * These routines deal with environmental events arriving from the
- * system controllers.
- */
-
-#include <linux/interrupt.h>
-#include <linux/sched/signal.h>
-#include <linux/slab.h>
-#include <asm/byteorder.h>
-#include <asm/sn/sn_sal.h>
-#include <asm/unaligned.h>
-#include "snsc.h"
-
-static struct subch_data_s *event_sd;
-
-void scdrv_event(unsigned long);
-DECLARE_TASKLET(sn_sysctl_event, scdrv_event, 0);
-
-/*
- * scdrv_event_interrupt
- *
- * Pull incoming environmental events off the physical link to the
- * system controller and put them in a temporary holding area in SAL.
- * Schedule scdrv_event() to move them along to their ultimate
- * destination.
- */
-static irqreturn_t
-scdrv_event_interrupt(int irq, void *subch_data)
-{
- struct subch_data_s *sd = subch_data;
- unsigned long flags;
- int status;
-
- spin_lock_irqsave(&sd->sd_rlock, flags);
- status = ia64_sn_irtr_intr(sd->sd_nasid, sd->sd_subch);
-
- if ((status > 0) && (status & SAL_IROUTER_INTR_RECV)) {
- tasklet_schedule(&sn_sysctl_event);
- }
- spin_unlock_irqrestore(&sd->sd_rlock, flags);
- return IRQ_HANDLED;
-}
-
-
-/*
- * scdrv_parse_event
- *
- * Break an event (as read from SAL) into useful pieces so we can decide
- * what to do with it.
- */
-static int
-scdrv_parse_event(char *event, int *src, int *code, int *esp_code, char *desc)
-{
- char *desc_end;
-
- /* record event source address */
- *src = get_unaligned_be32(event);
- event += 4; /* move on to event code */
-
- /* record the system controller's event code */
- *code = get_unaligned_be32(event);
- event += 4; /* move on to event arguments */
-
- /* how many arguments are in the packet? */
- if (*event++ != 2) {
- /* if not 2, give up */
- return -1;
- }
-
- /* parse out the ESP code */
- if (*event++ != IR_ARG_INT) {
- /* not an integer argument, so give up */
- return -1;
- }
- *esp_code = get_unaligned_be32(event);
- event += 4;
-
- /* parse out the event description */
- if (*event++ != IR_ARG_ASCII) {
- /* not an ASCII string, so give up */
- return -1;
- }
- event[CHUNKSIZE-1] = '\0'; /* ensure this string ends! */
- event += 2; /* skip leading CR/LF */
- desc_end = desc + sprintf(desc, "%s", event);
-
- /* strip trailing CR/LF (if any) */
- for (desc_end--;
- (desc_end != desc) && ((*desc_end == 0xd) || (*desc_end == 0xa));
- desc_end--) {
- *desc_end = '\0';
- }
-
- return 0;
-}
-
-
-/*
- * scdrv_event_severity
- *
- * Figure out how urgent a message we should write to the console/syslog
- * via printk.
- */
-static char *
-scdrv_event_severity(int code)
-{
- int ev_class = (code & EV_CLASS_MASK);
- int ev_severity = (code & EV_SEVERITY_MASK);
- char *pk_severity = KERN_NOTICE;
-
- switch (ev_class) {
- case EV_CLASS_POWER:
- switch (ev_severity) {
- case EV_SEVERITY_POWER_LOW_WARNING:
- case EV_SEVERITY_POWER_HIGH_WARNING:
- pk_severity = KERN_WARNING;
- break;
- case EV_SEVERITY_POWER_HIGH_FAULT:
- case EV_SEVERITY_POWER_LOW_FAULT:
- pk_severity = KERN_ALERT;
- break;
- }
- break;
- case EV_CLASS_FAN:
- switch (ev_severity) {
- case EV_SEVERITY_FAN_WARNING:
- pk_severity = KERN_WARNING;
- break;
- case EV_SEVERITY_FAN_FAULT:
- pk_severity = KERN_CRIT;
- break;
- }
- break;
- case EV_CLASS_TEMP:
- switch (ev_severity) {
- case EV_SEVERITY_TEMP_ADVISORY:
- pk_severity = KERN_WARNING;
- break;
- case EV_SEVERITY_TEMP_CRITICAL:
- pk_severity = KERN_CRIT;
- break;
- case EV_SEVERITY_TEMP_FAULT:
- pk_severity = KERN_ALERT;
- break;
- }
- break;
- case EV_CLASS_ENV:
- pk_severity = KERN_ALERT;
- break;
- case EV_CLASS_TEST_FAULT:
- pk_severity = KERN_ALERT;
- break;
- case EV_CLASS_TEST_WARNING:
- pk_severity = KERN_WARNING;
- break;
- case EV_CLASS_PWRD_NOTIFY:
- pk_severity = KERN_ALERT;
- break;
- }
-
- return pk_severity;
-}
-
-
-/*
- * scdrv_dispatch_event
- *
- * Do the right thing with an incoming event. That's often nothing
- * more than printing it to the system log. For power-down notifications
- * we start a graceful shutdown.
- */
-static void
-scdrv_dispatch_event(char *event, int len)
-{
- static int snsc_shutting_down = 0;
- int code, esp_code, src, class;
- char desc[CHUNKSIZE];
- char *severity;
-
- if (scdrv_parse_event(event, &src, &code, &esp_code, desc) < 0) {
- /* ignore uninterpretible event */
- return;
- }
-
- /* how urgent is the message? */
- severity = scdrv_event_severity(code);
-
- class = (code & EV_CLASS_MASK);
-
- if (class == EV_CLASS_PWRD_NOTIFY || code == ENV_PWRDN_PEND) {
- if (snsc_shutting_down)
- return;
-
- snsc_shutting_down = 1;
-
- /* give a message for each type of event */
- if (class == EV_CLASS_PWRD_NOTIFY)
- printk(KERN_NOTICE "Power off indication received."
- " Sending SIGPWR to init...\n");
- else if (code == ENV_PWRDN_PEND)
- printk(KERN_CRIT "WARNING: Shutting down the system"
- " due to a critical environmental condition."
- " Sending SIGPWR to init...\n");
-
- /* give a SIGPWR signal to init proc */
- kill_cad_pid(SIGPWR, 0);
- } else {
- /* print to system log */
- printk("%s|$(0x%x)%s\n", severity, esp_code, desc);
- }
-}
-
-
-/*
- * scdrv_event
- *
- * Called as a tasklet when an event arrives from the L1. Read the event
- * from where it's temporarily stored in SAL and call scdrv_dispatch_event()
- * to send it on its way. Keep trying to read events until SAL indicates
- * that there are no more immediately available.
- */
-void
-scdrv_event(unsigned long dummy)
-{
- int status;
- int len;
- unsigned long flags;
- struct subch_data_s *sd = event_sd;
-
- /* anything to read? */
- len = CHUNKSIZE;
- spin_lock_irqsave(&sd->sd_rlock, flags);
- status = ia64_sn_irtr_recv(sd->sd_nasid, sd->sd_subch,
- sd->sd_rb, &len);
-
- while (!(status < 0)) {
- spin_unlock_irqrestore(&sd->sd_rlock, flags);
- scdrv_dispatch_event(sd->sd_rb, len);
- len = CHUNKSIZE;
- spin_lock_irqsave(&sd->sd_rlock, flags);
- status = ia64_sn_irtr_recv(sd->sd_nasid, sd->sd_subch,
- sd->sd_rb, &len);
- }
- spin_unlock_irqrestore(&sd->sd_rlock, flags);
-}
-
-
-/*
- * scdrv_event_init
- *
- * Sets up a system controller subchannel to begin receiving event
- * messages. This is sort of a specialized version of scdrv_open()
- * in drivers/char/sn_sysctl.c.
- */
-void
-scdrv_event_init(struct sysctl_data_s *scd)
-{
- int rv;
-
- event_sd = kzalloc(sizeof (struct subch_data_s), GFP_KERNEL);
- if (event_sd == NULL) {
- printk(KERN_WARNING "%s: couldn't allocate subchannel info"
- " for event monitoring\n", __func__);
- return;
- }
-
- /* initialize subch_data_s fields */
- event_sd->sd_nasid = scd->scd_nasid;
- spin_lock_init(&event_sd->sd_rlock);
-
- /* ask the system controllers to send events to this node */
- event_sd->sd_subch = ia64_sn_sysctl_event_init(scd->scd_nasid);
-
- if (event_sd->sd_subch < 0) {
- kfree(event_sd);
- printk(KERN_WARNING "%s: couldn't open event subchannel\n",
- __func__);
- return;
- }
-
- /* hook event subchannel up to the system controller interrupt */
- rv = request_irq(SGI_UART_VECTOR, scdrv_event_interrupt,
- IRQF_SHARED, "system controller events", event_sd);
- if (rv) {
- printk(KERN_WARNING "%s: irq request failed (%d)\n",
- __func__, rv);
- ia64_sn_irtr_close(event_sd->sd_nasid, event_sd->sd_subch);
- kfree(event_sd);
- return;
- }
-}
diff --git a/drivers/char/toshiba.c b/drivers/char/toshiba.c
index 0bdc602f0d48..98f3150e0048 100644
--- a/drivers/char/toshiba.c
+++ b/drivers/char/toshiba.c
@@ -373,7 +373,7 @@ static int tosh_get_machine_id(void __iomem *bios)
value. This has been verified on a Satellite Pro 430CDT,
Tecra 750CDT, Tecra 780DVD and Satellite 310CDT. */
#if TOSH_DEBUG
- printk("toshiba: debugging ID ebx=0x%04x\n", regs.ebx);
+ pr_debug("toshiba: debugging ID ebx=0x%04x\n", regs.ebx);
#endif
bx = 0xe6f5;
@@ -417,7 +417,7 @@ static int tosh_probe(void)
for (i=0;i<7;i++) {
if (readb(bios+0xe010+i)!=signature[i]) {
- printk("toshiba: not a supported Toshiba laptop\n");
+ pr_err("toshiba: not a supported Toshiba laptop\n");
iounmap(bios);
return -ENODEV;
}
@@ -433,7 +433,7 @@ static int tosh_probe(void)
/* if this is not a Toshiba laptop carry flag is set and ah=0x86 */
if ((flag==1) || ((regs.eax & 0xff00)==0x8600)) {
- printk("toshiba: not a supported Toshiba laptop\n");
+ pr_err("toshiba: not a supported Toshiba laptop\n");
iounmap(bios);
return -ENODEV;
}
@@ -486,7 +486,7 @@ static int __init toshiba_init(void)
if (tosh_probe())
return -ENODEV;
- printk(KERN_INFO "Toshiba System Management Mode driver v" TOSH_VERSION "\n");
+ pr_info("Toshiba System Management Mode driver v" TOSH_VERSION "\n");
/* set the port to use for Fn status if not specified as a parameter */
if (tosh_fn==0x00)
diff --git a/drivers/char/tpm/Kconfig b/drivers/char/tpm/Kconfig
index 88a3c06fc153..9c37047f4b56 100644
--- a/drivers/char/tpm/Kconfig
+++ b/drivers/char/tpm/Kconfig
@@ -164,6 +164,11 @@ config TCG_VTPM_PROXY
/dev/vtpmX and a server-side file descriptor on which the vTPM
can receive commands.
+config TCG_FTPM_TEE
+ tristate "TEE based fTPM Interface"
+ depends on TEE && OPTEE
+ help
+ This driver proxies for firmware TPM running in TEE.
source "drivers/char/tpm/st33zp24/Kconfig"
endif # TCG_TPM
diff --git a/drivers/char/tpm/Makefile b/drivers/char/tpm/Makefile
index a01c4cab902a..c354cdff9c62 100644
--- a/drivers/char/tpm/Makefile
+++ b/drivers/char/tpm/Makefile
@@ -33,3 +33,4 @@ obj-$(CONFIG_TCG_TIS_ST33ZP24) += st33zp24/
obj-$(CONFIG_TCG_XEN) += xen-tpmfront.o
obj-$(CONFIG_TCG_CRB) += tpm_crb.o
obj-$(CONFIG_TCG_VTPM_PROXY) += tpm_vtpm_proxy.o
+obj-$(CONFIG_TCG_FTPM_TEE) += tpm_ftpm_tee.o
diff --git a/drivers/char/tpm/eventlog/efi.c b/drivers/char/tpm/eventlog/efi.c
index 3e44362e469c..6bb023de17f1 100644
--- a/drivers/char/tpm/eventlog/efi.c
+++ b/drivers/char/tpm/eventlog/efi.c
@@ -16,10 +16,13 @@
int tpm_read_log_efi(struct tpm_chip *chip)
{
+ struct efi_tcg2_final_events_table *final_tbl = NULL;
struct linux_efi_tpm_eventlog *log_tbl;
struct tpm_bios_log *log;
u32 log_size;
u8 tpm_log_version;
+ void *tmp;
+ int ret;
if (!(chip->flags & TPM_CHIP_FLAG_TPM2))
return -ENODEV;
@@ -47,15 +50,57 @@ int tpm_read_log_efi(struct tpm_chip *chip)
/* malloc EventLog space */
log->bios_event_log = kmemdup(log_tbl->log, log_size, GFP_KERNEL);
- if (!log->bios_event_log)
- goto err_memunmap;
- log->bios_event_log_end = log->bios_event_log + log_size;
+ if (!log->bios_event_log) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ log->bios_event_log_end = log->bios_event_log + log_size;
tpm_log_version = log_tbl->version;
- memunmap(log_tbl);
- return tpm_log_version;
-err_memunmap:
+ ret = tpm_log_version;
+
+ if (efi.tpm_final_log == EFI_INVALID_TABLE_ADDR ||
+ efi_tpm_final_log_size == 0 ||
+ tpm_log_version != EFI_TCG2_EVENT_LOG_FORMAT_TCG_2)
+ goto out;
+
+ final_tbl = memremap(efi.tpm_final_log,
+ sizeof(*final_tbl) + efi_tpm_final_log_size,
+ MEMREMAP_WB);
+ if (!final_tbl) {
+ pr_err("Could not map UEFI TPM final log\n");
+ kfree(log->bios_event_log);
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ efi_tpm_final_log_size -= log_tbl->final_events_preboot_size;
+
+ tmp = krealloc(log->bios_event_log,
+ log_size + efi_tpm_final_log_size,
+ GFP_KERNEL);
+ if (!tmp) {
+ kfree(log->bios_event_log);
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ log->bios_event_log = tmp;
+
+ /*
+ * Copy any of the final events log that didn't also end up in the
+ * main log. Events can be logged in both if events are generated
+ * between GetEventLog() and ExitBootServices().
+ */
+ memcpy((void *)log->bios_event_log + log_size,
+ final_tbl->events + log_tbl->final_events_preboot_size,
+ efi_tpm_final_log_size);
+ log->bios_event_log_end = log->bios_event_log +
+ log_size + efi_tpm_final_log_size;
+
+out:
+ memunmap(final_tbl);
memunmap(log_tbl);
- return -ENOMEM;
+ return ret;
}
diff --git a/drivers/char/tpm/eventlog/tpm2.c b/drivers/char/tpm/eventlog/tpm2.c
index d506362e046f..b9aeda1cbcd7 100644
--- a/drivers/char/tpm/eventlog/tpm2.c
+++ b/drivers/char/tpm/eventlog/tpm2.c
@@ -36,52 +36,7 @@
static size_t calc_tpm2_event_size(struct tcg_pcr_event2_head *event,
struct tcg_pcr_event *event_header)
{
- struct tcg_efi_specid_event_head *efispecid;
- struct tcg_event_field *event_field;
- void *marker;
- void *marker_start;
- u32 halg_size;
- size_t size;
- u16 halg;
- int i;
- int j;
-
- marker = event;
- marker_start = marker;
- marker = marker + sizeof(event->pcr_idx) + sizeof(event->event_type)
- + sizeof(event->count);
-
- efispecid = (struct tcg_efi_specid_event_head *)event_header->event;
-
- /* Check if event is malformed. */
- if (event->count > efispecid->num_algs)
- return 0;
-
- for (i = 0; i < event->count; i++) {
- halg_size = sizeof(event->digests[i].alg_id);
- memcpy(&halg, marker, halg_size);
- marker = marker + halg_size;
- for (j = 0; j < efispecid->num_algs; j++) {
- if (halg == efispecid->digest_sizes[j].alg_id) {
- marker +=
- efispecid->digest_sizes[j].digest_size;
- break;
- }
- }
- /* Algorithm without known length. Such event is unparseable. */
- if (j == efispecid->num_algs)
- return 0;
- }
-
- event_field = (struct tcg_event_field *)marker;
- marker = marker + sizeof(event_field->event_size)
- + event_field->event_size;
- size = marker - marker_start;
-
- if ((event->event_type == 0) && (event_field->event_size == 0))
- return 0;
-
- return size;
+ return __calc_tpm2_event_size(event, event_header, false);
}
static void *tpm2_bios_measurements_start(struct seq_file *m, loff_t *pos)
diff --git a/drivers/char/tpm/tpm-chip.c b/drivers/char/tpm/tpm-chip.c
index 90325e1749fb..3d6d394a8661 100644
--- a/drivers/char/tpm/tpm-chip.c
+++ b/drivers/char/tpm/tpm-chip.c
@@ -77,6 +77,18 @@ static int tpm_go_idle(struct tpm_chip *chip)
return chip->ops->go_idle(chip);
}
+static void tpm_clk_enable(struct tpm_chip *chip)
+{
+ if (chip->ops->clk_enable)
+ chip->ops->clk_enable(chip, true);
+}
+
+static void tpm_clk_disable(struct tpm_chip *chip)
+{
+ if (chip->ops->clk_enable)
+ chip->ops->clk_enable(chip, false);
+}
+
/**
* tpm_chip_start() - power on the TPM
* @chip: a TPM chip to use
@@ -89,13 +101,12 @@ int tpm_chip_start(struct tpm_chip *chip)
{
int ret;
- if (chip->ops->clk_enable)
- chip->ops->clk_enable(chip, true);
+ tpm_clk_enable(chip);
if (chip->locality == -1) {
ret = tpm_request_locality(chip);
if (ret) {
- chip->ops->clk_enable(chip, false);
+ tpm_clk_disable(chip);
return ret;
}
}
@@ -103,8 +114,7 @@ int tpm_chip_start(struct tpm_chip *chip)
ret = tpm_cmd_ready(chip);
if (ret) {
tpm_relinquish_locality(chip);
- if (chip->ops->clk_enable)
- chip->ops->clk_enable(chip, false);
+ tpm_clk_disable(chip);
return ret;
}
@@ -124,8 +134,7 @@ void tpm_chip_stop(struct tpm_chip *chip)
{
tpm_go_idle(chip);
tpm_relinquish_locality(chip);
- if (chip->ops->clk_enable)
- chip->ops->clk_enable(chip, false);
+ tpm_clk_disable(chip);
}
EXPORT_SYMBOL_GPL(tpm_chip_stop);
@@ -278,26 +287,23 @@ static void tpm_devs_release(struct device *dev)
* @dev: device to which the chip is associated.
*
* Issues a TPM2_Shutdown command prior to loss of power, as required by the
- * TPM 2.0 spec.
- * Then, calls bus- and device- specific shutdown code.
+ * TPM 2.0 spec. Then, calls bus- and device- specific shutdown code.
*
- * XXX: This codepath relies on the fact that sysfs is not enabled for
- * TPM2: sysfs uses an implicit lock on chip->ops, so this could race if TPM2
- * has sysfs support enabled before TPM sysfs's implicit locking is fixed.
+ * Return: always 0 (i.e. success)
*/
static int tpm_class_shutdown(struct device *dev)
{
struct tpm_chip *chip = container_of(dev, struct tpm_chip, dev);
+ down_write(&chip->ops_sem);
if (chip->flags & TPM_CHIP_FLAG_TPM2) {
- down_write(&chip->ops_sem);
if (!tpm_chip_start(chip)) {
tpm2_shutdown(chip, TPM2_SU_CLEAR);
tpm_chip_stop(chip);
}
- chip->ops = NULL;
- up_write(&chip->ops_sem);
}
+ chip->ops = NULL;
+ up_write(&chip->ops_sem);
return 0;
}
@@ -545,6 +551,20 @@ static int tpm_add_hwrng(struct tpm_chip *chip)
return hwrng_register(&chip->hwrng);
}
+static int tpm_get_pcr_allocation(struct tpm_chip *chip)
+{
+ int rc;
+
+ rc = (chip->flags & TPM_CHIP_FLAG_TPM2) ?
+ tpm2_get_pcr_allocation(chip) :
+ tpm1_get_pcr_allocation(chip);
+
+ if (rc > 0)
+ return -ENODEV;
+
+ return rc;
+}
+
/*
* tpm_chip_register() - create a character device for the TPM chip
* @chip: TPM chip to use.
@@ -564,6 +584,12 @@ int tpm_chip_register(struct tpm_chip *chip)
if (rc)
return rc;
rc = tpm_auto_startup(chip);
+ if (rc) {
+ tpm_chip_stop(chip);
+ return rc;
+ }
+
+ rc = tpm_get_pcr_allocation(chip);
tpm_chip_stop(chip);
if (rc)
return rc;
diff --git a/drivers/char/tpm/tpm-sysfs.c b/drivers/char/tpm/tpm-sysfs.c
index d9caedda075b..edfa89160010 100644
--- a/drivers/char/tpm/tpm-sysfs.c
+++ b/drivers/char/tpm/tpm-sysfs.c
@@ -329,16 +329,9 @@ static const struct attribute_group tpm_dev_group = {
void tpm_sysfs_add_device(struct tpm_chip *chip)
{
- /* XXX: If you wish to remove this restriction, you must first update
- * tpm_sysfs to explicitly lock chip->ops.
- */
if (chip->flags & TPM_CHIP_FLAG_TPM2)
return;
- /* The sysfs routines rely on an implicit tpm_try_get_ops, device_del
- * is called before ops is null'd and the sysfs core synchronizes this
- * removal so that no callbacks are running or can run again
- */
WARN_ON(chip->groups_cnt != 0);
chip->groups[chip->groups_cnt++] = &tpm_dev_group;
}
diff --git a/drivers/char/tpm/tpm.h b/drivers/char/tpm/tpm.h
index e503ffc3aa39..a7fea3e0ca86 100644
--- a/drivers/char/tpm/tpm.h
+++ b/drivers/char/tpm/tpm.h
@@ -394,6 +394,7 @@ int tpm1_pcr_read(struct tpm_chip *chip, u32 pcr_idx, u8 *res_buf);
ssize_t tpm1_getcap(struct tpm_chip *chip, u32 subcap_id, cap_t *cap,
const char *desc, size_t min_cap_length);
int tpm1_get_random(struct tpm_chip *chip, u8 *out, size_t max);
+int tpm1_get_pcr_allocation(struct tpm_chip *chip);
unsigned long tpm_calc_ordinal_duration(struct tpm_chip *chip, u32 ordinal);
int tpm_pm_suspend(struct device *dev);
int tpm_pm_resume(struct device *dev);
@@ -449,6 +450,7 @@ int tpm2_unseal_trusted(struct tpm_chip *chip,
ssize_t tpm2_get_tpm_pt(struct tpm_chip *chip, u32 property_id,
u32 *value, const char *desc);
+ssize_t tpm2_get_pcr_allocation(struct tpm_chip *chip);
int tpm2_auto_startup(struct tpm_chip *chip);
void tpm2_shutdown(struct tpm_chip *chip, u16 shutdown_type);
unsigned long tpm2_calc_ordinal_duration(struct tpm_chip *chip, u32 ordinal);
diff --git a/drivers/char/tpm/tpm1-cmd.c b/drivers/char/tpm/tpm1-cmd.c
index 85dcf2654d11..149e953ca369 100644
--- a/drivers/char/tpm/tpm1-cmd.c
+++ b/drivers/char/tpm/tpm1-cmd.c
@@ -510,7 +510,7 @@ struct tpm1_get_random_out {
*
* Return:
* * number of bytes read
- * * -errno or a TPM return code otherwise
+ * * -errno (positive TPM return codes are masked to -EIO)
*/
int tpm1_get_random(struct tpm_chip *chip, u8 *dest, size_t max)
{
@@ -531,8 +531,11 @@ int tpm1_get_random(struct tpm_chip *chip, u8 *dest, size_t max)
rc = tpm_transmit_cmd(chip, &buf, sizeof(out->rng_data_len),
"attempting get random");
- if (rc)
+ if (rc) {
+ if (rc > 0)
+ rc = -EIO;
goto out;
+ }
out = (struct tpm1_get_random_out *)&buf.data[TPM_HEADER_SIZE];
@@ -696,18 +699,6 @@ int tpm1_auto_startup(struct tpm_chip *chip)
goto out;
}
- chip->allocated_banks = kcalloc(1, sizeof(*chip->allocated_banks),
- GFP_KERNEL);
- if (!chip->allocated_banks) {
- rc = -ENOMEM;
- goto out;
- }
-
- chip->allocated_banks[0].alg_id = TPM_ALG_SHA1;
- chip->allocated_banks[0].digest_size = hash_digest_size[HASH_ALGO_SHA1];
- chip->allocated_banks[0].crypto_id = HASH_ALGO_SHA1;
- chip->nr_allocated_banks = 1;
-
return rc;
out:
if (rc > 0)
@@ -776,3 +767,27 @@ int tpm1_pm_suspend(struct tpm_chip *chip, u32 tpm_suspend_pcr)
return rc;
}
+/**
+ * tpm1_get_pcr_allocation() - initialize the allocated bank
+ * @chip: TPM chip to use.
+ *
+ * The function initializes the SHA1 allocated bank to extend PCR
+ *
+ * Return:
+ * * 0 on success,
+ * * < 0 on error.
+ */
+int tpm1_get_pcr_allocation(struct tpm_chip *chip)
+{
+ chip->allocated_banks = kcalloc(1, sizeof(*chip->allocated_banks),
+ GFP_KERNEL);
+ if (!chip->allocated_banks)
+ return -ENOMEM;
+
+ chip->allocated_banks[0].alg_id = TPM_ALG_SHA1;
+ chip->allocated_banks[0].digest_size = hash_digest_size[HASH_ALGO_SHA1];
+ chip->allocated_banks[0].crypto_id = HASH_ALGO_SHA1;
+ chip->nr_allocated_banks = 1;
+
+ return 0;
+}
diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c
index 4de49924cfc4..ba9acae83bff 100644
--- a/drivers/char/tpm/tpm2-cmd.c
+++ b/drivers/char/tpm/tpm2-cmd.c
@@ -297,7 +297,7 @@ struct tpm2_get_random_out {
*
* Return:
* size of the buffer on success,
- * -errno otherwise
+ * -errno otherwise (positive TPM return codes are masked to -EIO)
*/
int tpm2_get_random(struct tpm_chip *chip, u8 *dest, size_t max)
{
@@ -324,8 +324,11 @@ int tpm2_get_random(struct tpm_chip *chip, u8 *dest, size_t max)
offsetof(struct tpm2_get_random_out,
buffer),
"attempting get random");
- if (err)
+ if (err) {
+ if (err > 0)
+ err = -EIO;
goto out;
+ }
out = (struct tpm2_get_random_out *)
&buf.data[TPM_HEADER_SIZE];
@@ -837,7 +840,7 @@ struct tpm2_pcr_selection {
u8 pcr_select[3];
} __packed;
-static ssize_t tpm2_get_pcr_allocation(struct tpm_chip *chip)
+ssize_t tpm2_get_pcr_allocation(struct tpm_chip *chip)
{
struct tpm2_pcr_selection pcr_selection;
struct tpm_buf buf;
@@ -1037,10 +1040,6 @@ int tpm2_auto_startup(struct tpm_chip *chip)
goto out;
}
- rc = tpm2_get_pcr_allocation(chip);
- if (rc)
- goto out;
-
rc = tpm2_get_cc_attrs_tbl(chip);
out:
diff --git a/drivers/char/tpm/tpm_ftpm_tee.c b/drivers/char/tpm/tpm_ftpm_tee.c
new file mode 100644
index 000000000000..6640a14dbe48
--- /dev/null
+++ b/drivers/char/tpm/tpm_ftpm_tee.c
@@ -0,0 +1,350 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) Microsoft Corporation
+ *
+ * Implements a firmware TPM as described here:
+ * https://www.microsoft.com/en-us/research/publication/ftpm-software-implementation-tpm-chip/
+ *
+ * A reference implementation is available here:
+ * https://github.com/microsoft/ms-tpm-20-ref/tree/master/Samples/ARM32-FirmwareTPM/optee_ta/fTPM
+ */
+
+#include <linux/acpi.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/tee_drv.h>
+#include <linux/tpm.h>
+#include <linux/uuid.h>
+
+#include "tpm.h"
+#include "tpm_ftpm_tee.h"
+
+/*
+ * TA_FTPM_UUID: BC50D971-D4C9-42C4-82CB-343FB7F37896
+ *
+ * Randomly generated, and must correspond to the GUID on the TA side.
+ * Defined here in the reference implementation:
+ * https://github.com/microsoft/ms-tpm-20-ref/blob/master/Samples/ARM32-FirmwareTPM/optee_ta/fTPM/include/fTPM.h#L42
+ */
+static const uuid_t ftpm_ta_uuid =
+ UUID_INIT(0xBC50D971, 0xD4C9, 0x42C4,
+ 0x82, 0xCB, 0x34, 0x3F, 0xB7, 0xF3, 0x78, 0x96);
+
+/**
+ * ftpm_tee_tpm_op_recv - retrieve fTPM response.
+ * @chip: the tpm_chip description as specified in driver/char/tpm/tpm.h.
+ * @buf: the buffer to store data.
+ * @count: the number of bytes to read.
+ *
+ * Return:
+ * In case of success the number of bytes received.
+ * On failure, -errno.
+ */
+static int ftpm_tee_tpm_op_recv(struct tpm_chip *chip, u8 *buf, size_t count)
+{
+ struct ftpm_tee_private *pvt_data = dev_get_drvdata(chip->dev.parent);
+ size_t len;
+
+ len = pvt_data->resp_len;
+ if (count < len) {
+ dev_err(&chip->dev,
+ "%s: Invalid size in recv: count=%zd, resp_len=%zd\n",
+ __func__, count, len);
+ return -EIO;
+ }
+
+ memcpy(buf, pvt_data->resp_buf, len);
+ pvt_data->resp_len = 0;
+
+ return len;
+}
+
+/**
+ * ftpm_tee_tpm_op_send - send TPM commands through the TEE shared memory.
+ * @chip: the tpm_chip description as specified in driver/char/tpm/tpm.h
+ * @buf: the buffer to send.
+ * @len: the number of bytes to send.
+ *
+ * Return:
+ * In case of success, returns 0.
+ * On failure, -errno
+ */
+static int ftpm_tee_tpm_op_send(struct tpm_chip *chip, u8 *buf, size_t len)
+{
+ struct ftpm_tee_private *pvt_data = dev_get_drvdata(chip->dev.parent);
+ size_t resp_len;
+ int rc;
+ u8 *temp_buf;
+ struct tpm_header *resp_header;
+ struct tee_ioctl_invoke_arg transceive_args;
+ struct tee_param command_params[4];
+ struct tee_shm *shm = pvt_data->shm;
+
+ if (len > MAX_COMMAND_SIZE) {
+ dev_err(&chip->dev,
+ "%s: len=%zd exceeds MAX_COMMAND_SIZE supported by fTPM TA\n",
+ __func__, len);
+ return -EIO;
+ }
+
+ memset(&transceive_args, 0, sizeof(transceive_args));
+ memset(command_params, 0, sizeof(command_params));
+ pvt_data->resp_len = 0;
+
+ /* Invoke FTPM_OPTEE_TA_SUBMIT_COMMAND function of fTPM TA */
+ transceive_args = (struct tee_ioctl_invoke_arg) {
+ .func = FTPM_OPTEE_TA_SUBMIT_COMMAND,
+ .session = pvt_data->session,
+ .num_params = 4,
+ };
+
+ /* Fill FTPM_OPTEE_TA_SUBMIT_COMMAND parameters */
+ command_params[0] = (struct tee_param) {
+ .attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT,
+ .u.memref = {
+ .shm = shm,
+ .size = len,
+ .shm_offs = 0,
+ },
+ };
+
+ temp_buf = tee_shm_get_va(shm, 0);
+ if (IS_ERR(temp_buf)) {
+ dev_err(&chip->dev, "%s: tee_shm_get_va failed for transmit\n",
+ __func__);
+ return PTR_ERR(temp_buf);
+ }
+ memset(temp_buf, 0, (MAX_COMMAND_SIZE + MAX_RESPONSE_SIZE));
+ memcpy(temp_buf, buf, len);
+
+ command_params[1] = (struct tee_param) {
+ .attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT,
+ .u.memref = {
+ .shm = shm,
+ .size = MAX_RESPONSE_SIZE,
+ .shm_offs = MAX_COMMAND_SIZE,
+ },
+ };
+
+ rc = tee_client_invoke_func(pvt_data->ctx, &transceive_args,
+ command_params);
+ if ((rc < 0) || (transceive_args.ret != 0)) {
+ dev_err(&chip->dev, "%s: SUBMIT_COMMAND invoke error: 0x%x\n",
+ __func__, transceive_args.ret);
+ return (rc < 0) ? rc : transceive_args.ret;
+ }
+
+ temp_buf = tee_shm_get_va(shm, command_params[1].u.memref.shm_offs);
+ if (IS_ERR(temp_buf)) {
+ dev_err(&chip->dev, "%s: tee_shm_get_va failed for receive\n",
+ __func__);
+ return PTR_ERR(temp_buf);
+ }
+
+ resp_header = (struct tpm_header *)temp_buf;
+ resp_len = be32_to_cpu(resp_header->length);
+
+ /* sanity check resp_len */
+ if (resp_len < TPM_HEADER_SIZE) {
+ dev_err(&chip->dev, "%s: tpm response header too small\n",
+ __func__);
+ return -EIO;
+ }
+ if (resp_len > MAX_RESPONSE_SIZE) {
+ dev_err(&chip->dev,
+ "%s: resp_len=%zd exceeds MAX_RESPONSE_SIZE\n",
+ __func__, resp_len);
+ return -EIO;
+ }
+
+ /* sanity checks look good, cache the response */
+ memcpy(pvt_data->resp_buf, temp_buf, resp_len);
+ pvt_data->resp_len = resp_len;
+
+ return 0;
+}
+
+static void ftpm_tee_tpm_op_cancel(struct tpm_chip *chip)
+{
+ /* not supported */
+}
+
+static u8 ftpm_tee_tpm_op_status(struct tpm_chip *chip)
+{
+ return 0;
+}
+
+static bool ftpm_tee_tpm_req_canceled(struct tpm_chip *chip, u8 status)
+{
+ return 0;
+}
+
+static const struct tpm_class_ops ftpm_tee_tpm_ops = {
+ .flags = TPM_OPS_AUTO_STARTUP,
+ .recv = ftpm_tee_tpm_op_recv,
+ .send = ftpm_tee_tpm_op_send,
+ .cancel = ftpm_tee_tpm_op_cancel,
+ .status = ftpm_tee_tpm_op_status,
+ .req_complete_mask = 0,
+ .req_complete_val = 0,
+ .req_canceled = ftpm_tee_tpm_req_canceled,
+};
+
+/*
+ * Check whether this driver supports the fTPM TA in the TEE instance
+ * represented by the params (ver/data) to this function.
+ */
+static int ftpm_tee_match(struct tee_ioctl_version_data *ver, const void *data)
+{
+ /*
+ * Currently this driver only support GP Complaint OPTEE based fTPM TA
+ */
+ if ((ver->impl_id == TEE_IMPL_ID_OPTEE) &&
+ (ver->gen_caps & TEE_GEN_CAP_GP))
+ return 1;
+ else
+ return 0;
+}
+
+/**
+ * ftpm_tee_probe - initialize the fTPM
+ * @pdev: the platform_device description.
+ *
+ * Return:
+ * On success, 0. On failure, -errno.
+ */
+static int ftpm_tee_probe(struct platform_device *pdev)
+{
+ int rc;
+ struct tpm_chip *chip;
+ struct device *dev = &pdev->dev;
+ struct ftpm_tee_private *pvt_data = NULL;
+ struct tee_ioctl_open_session_arg sess_arg;
+
+ pvt_data = devm_kzalloc(dev, sizeof(struct ftpm_tee_private),
+ GFP_KERNEL);
+ if (!pvt_data)
+ return -ENOMEM;
+
+ dev_set_drvdata(dev, pvt_data);
+
+ /* Open context with TEE driver */
+ pvt_data->ctx = tee_client_open_context(NULL, ftpm_tee_match, NULL,
+ NULL);
+ if (IS_ERR(pvt_data->ctx)) {
+ if (PTR_ERR(pvt_data->ctx) == -ENOENT)
+ return -EPROBE_DEFER;
+ dev_err(dev, "%s: tee_client_open_context failed\n", __func__);
+ return PTR_ERR(pvt_data->ctx);
+ }
+
+ /* Open a session with fTPM TA */
+ memset(&sess_arg, 0, sizeof(sess_arg));
+ memcpy(sess_arg.uuid, ftpm_ta_uuid.b, TEE_IOCTL_UUID_LEN);
+ sess_arg.clnt_login = TEE_IOCTL_LOGIN_PUBLIC;
+ sess_arg.num_params = 0;
+
+ rc = tee_client_open_session(pvt_data->ctx, &sess_arg, NULL);
+ if ((rc < 0) || (sess_arg.ret != 0)) {
+ dev_err(dev, "%s: tee_client_open_session failed, err=%x\n",
+ __func__, sess_arg.ret);
+ rc = -EINVAL;
+ goto out_tee_session;
+ }
+ pvt_data->session = sess_arg.session;
+
+ /* Allocate dynamic shared memory with fTPM TA */
+ pvt_data->shm = tee_shm_alloc(pvt_data->ctx,
+ MAX_COMMAND_SIZE + MAX_RESPONSE_SIZE,
+ TEE_SHM_MAPPED | TEE_SHM_DMA_BUF);
+ if (IS_ERR(pvt_data->shm)) {
+ dev_err(dev, "%s: tee_shm_alloc failed\n", __func__);
+ rc = -ENOMEM;
+ goto out_shm_alloc;
+ }
+
+ /* Allocate new struct tpm_chip instance */
+ chip = tpm_chip_alloc(dev, &ftpm_tee_tpm_ops);
+ if (IS_ERR(chip)) {
+ dev_err(dev, "%s: tpm_chip_alloc failed\n", __func__);
+ rc = PTR_ERR(chip);
+ goto out_chip_alloc;
+ }
+
+ pvt_data->chip = chip;
+ pvt_data->chip->flags |= TPM_CHIP_FLAG_TPM2;
+
+ /* Create a character device for the fTPM */
+ rc = tpm_chip_register(pvt_data->chip);
+ if (rc) {
+ dev_err(dev, "%s: tpm_chip_register failed with rc=%d\n",
+ __func__, rc);
+ goto out_chip;
+ }
+
+ return 0;
+
+out_chip:
+ put_device(&pvt_data->chip->dev);
+out_chip_alloc:
+ tee_shm_free(pvt_data->shm);
+out_shm_alloc:
+ tee_client_close_session(pvt_data->ctx, pvt_data->session);
+out_tee_session:
+ tee_client_close_context(pvt_data->ctx);
+
+ return rc;
+}
+
+/**
+ * ftpm_tee_remove - remove the TPM device
+ * @pdev: the platform_device description.
+ *
+ * Return:
+ * 0 always.
+ */
+static int ftpm_tee_remove(struct platform_device *pdev)
+{
+ struct ftpm_tee_private *pvt_data = dev_get_drvdata(&pdev->dev);
+
+ /* Release the chip */
+ tpm_chip_unregister(pvt_data->chip);
+
+ /* frees chip */
+ put_device(&pvt_data->chip->dev);
+
+ /* Free the shared memory pool */
+ tee_shm_free(pvt_data->shm);
+
+ /* close the existing session with fTPM TA*/
+ tee_client_close_session(pvt_data->ctx, pvt_data->session);
+
+ /* close the context with TEE driver */
+ tee_client_close_context(pvt_data->ctx);
+
+ /* memory allocated with devm_kzalloc() is freed automatically */
+
+ return 0;
+}
+
+static const struct of_device_id of_ftpm_tee_ids[] = {
+ { .compatible = "microsoft,ftpm" },
+ { }
+};
+MODULE_DEVICE_TABLE(of, of_ftpm_tee_ids);
+
+static struct platform_driver ftpm_tee_driver = {
+ .driver = {
+ .name = "ftpm-tee",
+ .of_match_table = of_match_ptr(of_ftpm_tee_ids),
+ },
+ .probe = ftpm_tee_probe,
+ .remove = ftpm_tee_remove,
+};
+
+module_platform_driver(ftpm_tee_driver);
+
+MODULE_AUTHOR("Thirupathaiah Annapureddy <thiruan@microsoft.com>");
+MODULE_DESCRIPTION("TPM Driver for fTPM TA in TEE");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/char/tpm/tpm_ftpm_tee.h b/drivers/char/tpm/tpm_ftpm_tee.h
new file mode 100644
index 000000000000..f98daa7bf68c
--- /dev/null
+++ b/drivers/char/tpm/tpm_ftpm_tee.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) Microsoft Corporation
+ */
+
+#ifndef __TPM_FTPM_TEE_H__
+#define __TPM_FTPM_TEE_H__
+
+#include <linux/tee_drv.h>
+#include <linux/tpm.h>
+#include <linux/uuid.h>
+
+/* The TAFs ID implemented in this TA */
+#define FTPM_OPTEE_TA_SUBMIT_COMMAND (0)
+#define FTPM_OPTEE_TA_EMULATE_PPI (1)
+
+/* max. buffer size supported by fTPM */
+#define MAX_COMMAND_SIZE 4096
+#define MAX_RESPONSE_SIZE 4096
+
+/**
+ * struct ftpm_tee_private - fTPM's private data
+ * @chip: struct tpm_chip instance registered with tpm framework.
+ * @state: internal state
+ * @session: fTPM TA session identifier.
+ * @resp_len: cached response buffer length.
+ * @resp_buf: cached response buffer.
+ * @ctx: TEE context handler.
+ * @shm: Memory pool shared with fTPM TA in TEE.
+ */
+struct ftpm_tee_private {
+ struct tpm_chip *chip;
+ u32 session;
+ size_t resp_len;
+ u8 resp_buf[MAX_RESPONSE_SIZE];
+ struct tee_context *ctx;
+ struct tee_shm *shm;
+};
+
+#endif /* __TPM_FTPM_TEE_H__ */
diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c
index c3181ea9f271..270f43acbb77 100644
--- a/drivers/char/tpm/tpm_tis_core.c
+++ b/drivers/char/tpm/tpm_tis_core.c
@@ -980,6 +980,8 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq,
goto out_err;
}
+ tpm_chip_start(chip);
+ chip->flags |= TPM_CHIP_FLAG_IRQ;
if (irq) {
tpm_tis_probe_irq_single(chip, intmask, IRQF_SHARED,
irq);
@@ -989,6 +991,7 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq,
} else {
tpm_tis_probe_irq(chip, intmask);
}
+ tpm_chip_stop(chip);
}
rc = tpm_chip_register(chip);
diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig
index fc1e0cf44995..801fa1cd0321 100644
--- a/drivers/clk/Kconfig
+++ b/drivers/clk/Kconfig
@@ -53,13 +53,12 @@ config COMMON_CLK_MAX9485
This driver supports Maxim 9485 Programmable Audio Clock Generator
config COMMON_CLK_RK808
- tristate "Clock driver for RK805/RK808/RK818"
+ tristate "Clock driver for RK805/RK808/RK809/RK817/RK818"
depends on MFD_RK808
---help---
- This driver supports RK805, RK808 and RK818 crystal oscillator clock. These
- multi-function devices have two fixed-rate oscillators,
- clocked at 32KHz each. Clkout1 is always on, Clkout2 can off
- by control register.
+ This driver supports RK805, RK809 and RK817, RK808 and RK818 crystal oscillator clock.
+ These multi-function devices have two fixed-rate oscillators, clocked at 32KHz each.
+ Clkout1 is always on, Clkout2 can off by control register.
config COMMON_CLK_HI655X
tristate "Clock driver for Hi655x" if EXPERT
@@ -91,6 +90,17 @@ config COMMON_CLK_SCPI
This driver uses SCPI Message Protocol to interact with the
firmware providing all the clock controls.
+config COMMON_CLK_SI5341
+ tristate "Clock driver for SiLabs 5341 and 5340 A/B/C/D devices"
+ depends on I2C
+ select REGMAP_I2C
+ help
+ This driver supports Silicon Labs Si5341 and Si5340 programmable clock
+ generators. Not all features of these chips are currently supported
+ by the driver, in particular it only supports XTAL input. The chip can
+ be pre-programmed to support other configurations and features not yet
+ implemented in the driver.
+
config COMMON_CLK_SI5351
tristate "Clock driver for SiLabs 5351A/B/C"
depends on I2C
@@ -215,7 +225,7 @@ config CLK_QORIQ
config COMMON_CLK_XGENE
bool "Clock driver for APM XGene SoC"
- default y
+ default ARCH_XGENE
depends on ARM64 || COMPILE_TEST
---help---
Sypport for the APM X-Gene SoC reference, PLL, and device clocks.
@@ -293,10 +303,10 @@ config COMMON_CLK_STM32H7
config COMMON_CLK_BD718XX
tristate "Clock driver for ROHM BD718x7 PMIC"
- depends on MFD_ROHM_BD718XX
+ depends on MFD_ROHM_BD718XX || MFD_ROHM_BD70528
help
- This driver supports ROHM BD71837 and ROHM BD71847
- PMICs clock gates.
+ This driver supports ROHM BD71837, ROHM BD71847 and
+ ROHM BD70528 PMICs clock gates.
config COMMON_CLK_FIXED_MMIO
bool "Clock driver for Memory Mapped Fixed values"
diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile
index 9ef4305d55e0..0cad76021297 100644
--- a/drivers/clk/Makefile
+++ b/drivers/clk/Makefile
@@ -49,6 +49,7 @@ obj-$(CONFIG_COMMON_CLK_HI655X) += clk-hi655x.o
obj-$(CONFIG_COMMON_CLK_S2MPS11) += clk-s2mps11.o
obj-$(CONFIG_COMMON_CLK_SCMI) += clk-scmi.o
obj-$(CONFIG_COMMON_CLK_SCPI) += clk-scpi.o
+obj-$(CONFIG_COMMON_CLK_SI5341) += clk-si5341.o
obj-$(CONFIG_COMMON_CLK_SI5351) += clk-si5351.o
obj-$(CONFIG_COMMON_CLK_SI514) += clk-si514.o
obj-$(CONFIG_COMMON_CLK_SI544) += clk-si544.o
diff --git a/drivers/clk/at91/clk-generated.c b/drivers/clk/at91/clk-generated.c
index 44db83a6d01c..44a46dcc0518 100644
--- a/drivers/clk/at91/clk-generated.c
+++ b/drivers/clk/at91/clk-generated.c
@@ -141,6 +141,8 @@ static int clk_generated_determine_rate(struct clk_hw *hw,
continue;
div = DIV_ROUND_CLOSEST(parent_rate, req->rate);
+ if (div > GENERATED_MAX_DIV + 1)
+ div = GENERATED_MAX_DIV + 1;
clk_generated_best_diff(req, parent, parent_rate, div,
&best_diff, &best_rate);
diff --git a/drivers/clk/at91/sckc.c b/drivers/clk/at91/sckc.c
index 45526f56f1ba..9bfe9a28294a 100644
--- a/drivers/clk/at91/sckc.c
+++ b/drivers/clk/at91/sckc.c
@@ -18,14 +18,18 @@
SLOW_CLOCK_FREQ)
#define AT91_SCKC_CR 0x00
-#define AT91_SCKC_RCEN (1 << 0)
-#define AT91_SCKC_OSC32EN (1 << 1)
-#define AT91_SCKC_OSC32BYP (1 << 2)
-#define AT91_SCKC_OSCSEL (1 << 3)
+
+struct clk_slow_bits {
+ u32 cr_rcen;
+ u32 cr_osc32en;
+ u32 cr_osc32byp;
+ u32 cr_oscsel;
+};
struct clk_slow_osc {
struct clk_hw hw;
void __iomem *sckcr;
+ const struct clk_slow_bits *bits;
unsigned long startup_usec;
};
@@ -34,6 +38,7 @@ struct clk_slow_osc {
struct clk_sama5d4_slow_osc {
struct clk_hw hw;
void __iomem *sckcr;
+ const struct clk_slow_bits *bits;
unsigned long startup_usec;
bool prepared;
};
@@ -43,6 +48,7 @@ struct clk_sama5d4_slow_osc {
struct clk_slow_rc_osc {
struct clk_hw hw;
void __iomem *sckcr;
+ const struct clk_slow_bits *bits;
unsigned long frequency;
unsigned long accuracy;
unsigned long startup_usec;
@@ -53,6 +59,7 @@ struct clk_slow_rc_osc {
struct clk_sam9x5_slow {
struct clk_hw hw;
void __iomem *sckcr;
+ const struct clk_slow_bits *bits;
u8 parent;
};
@@ -64,10 +71,10 @@ static int clk_slow_osc_prepare(struct clk_hw *hw)
void __iomem *sckcr = osc->sckcr;
u32 tmp = readl(sckcr);
- if (tmp & (AT91_SCKC_OSC32BYP | AT91_SCKC_OSC32EN))
+ if (tmp & (osc->bits->cr_osc32byp | osc->bits->cr_osc32en))
return 0;
- writel(tmp | AT91_SCKC_OSC32EN, sckcr);
+ writel(tmp | osc->bits->cr_osc32en, sckcr);
usleep_range(osc->startup_usec, osc->startup_usec + 1);
@@ -80,10 +87,10 @@ static void clk_slow_osc_unprepare(struct clk_hw *hw)
void __iomem *sckcr = osc->sckcr;
u32 tmp = readl(sckcr);
- if (tmp & AT91_SCKC_OSC32BYP)
+ if (tmp & osc->bits->cr_osc32byp)
return;
- writel(tmp & ~AT91_SCKC_OSC32EN, sckcr);
+ writel(tmp & ~osc->bits->cr_osc32en, sckcr);
}
static int clk_slow_osc_is_prepared(struct clk_hw *hw)
@@ -92,10 +99,10 @@ static int clk_slow_osc_is_prepared(struct clk_hw *hw)
void __iomem *sckcr = osc->sckcr;
u32 tmp = readl(sckcr);
- if (tmp & AT91_SCKC_OSC32BYP)
+ if (tmp & osc->bits->cr_osc32byp)
return 1;
- return !!(tmp & AT91_SCKC_OSC32EN);
+ return !!(tmp & osc->bits->cr_osc32en);
}
static const struct clk_ops slow_osc_ops = {
@@ -109,7 +116,8 @@ at91_clk_register_slow_osc(void __iomem *sckcr,
const char *name,
const char *parent_name,
unsigned long startup,
- bool bypass)
+ bool bypass,
+ const struct clk_slow_bits *bits)
{
struct clk_slow_osc *osc;
struct clk_hw *hw;
@@ -132,10 +140,11 @@ at91_clk_register_slow_osc(void __iomem *sckcr,
osc->hw.init = &init;
osc->sckcr = sckcr;
osc->startup_usec = startup;
+ osc->bits = bits;
if (bypass)
- writel((readl(sckcr) & ~AT91_SCKC_OSC32EN) | AT91_SCKC_OSC32BYP,
- sckcr);
+ writel((readl(sckcr) & ~osc->bits->cr_osc32en) |
+ osc->bits->cr_osc32byp, sckcr);
hw = &osc->hw;
ret = clk_hw_register(NULL, &osc->hw);
@@ -147,6 +156,14 @@ at91_clk_register_slow_osc(void __iomem *sckcr,
return hw;
}
+static void at91_clk_unregister_slow_osc(struct clk_hw *hw)
+{
+ struct clk_slow_osc *osc = to_clk_slow_osc(hw);
+
+ clk_hw_unregister(hw);
+ kfree(osc);
+}
+
static unsigned long clk_slow_rc_osc_recalc_rate(struct clk_hw *hw,
unsigned long parent_rate)
{
@@ -168,7 +185,7 @@ static int clk_slow_rc_osc_prepare(struct clk_hw *hw)
struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw);
void __iomem *sckcr = osc->sckcr;
- writel(readl(sckcr) | AT91_SCKC_RCEN, sckcr);
+ writel(readl(sckcr) | osc->bits->cr_rcen, sckcr);
usleep_range(osc->startup_usec, osc->startup_usec + 1);
@@ -180,14 +197,14 @@ static void clk_slow_rc_osc_unprepare(struct clk_hw *hw)
struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw);
void __iomem *sckcr = osc->sckcr;
- writel(readl(sckcr) & ~AT91_SCKC_RCEN, sckcr);
+ writel(readl(sckcr) & ~osc->bits->cr_rcen, sckcr);
}
static int clk_slow_rc_osc_is_prepared(struct clk_hw *hw)
{
struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw);
- return !!(readl(osc->sckcr) & AT91_SCKC_RCEN);
+ return !!(readl(osc->sckcr) & osc->bits->cr_rcen);
}
static const struct clk_ops slow_rc_osc_ops = {
@@ -203,7 +220,8 @@ at91_clk_register_slow_rc_osc(void __iomem *sckcr,
const char *name,
unsigned long frequency,
unsigned long accuracy,
- unsigned long startup)
+ unsigned long startup,
+ const struct clk_slow_bits *bits)
{
struct clk_slow_rc_osc *osc;
struct clk_hw *hw;
@@ -225,6 +243,7 @@ at91_clk_register_slow_rc_osc(void __iomem *sckcr,
osc->hw.init = &init;
osc->sckcr = sckcr;
+ osc->bits = bits;
osc->frequency = frequency;
osc->accuracy = accuracy;
osc->startup_usec = startup;
@@ -239,6 +258,14 @@ at91_clk_register_slow_rc_osc(void __iomem *sckcr,
return hw;
}
+static void at91_clk_unregister_slow_rc_osc(struct clk_hw *hw)
+{
+ struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw);
+
+ clk_hw_unregister(hw);
+ kfree(osc);
+}
+
static int clk_sam9x5_slow_set_parent(struct clk_hw *hw, u8 index)
{
struct clk_sam9x5_slow *slowck = to_clk_sam9x5_slow(hw);
@@ -250,14 +277,14 @@ static int clk_sam9x5_slow_set_parent(struct clk_hw *hw, u8 index)
tmp = readl(sckcr);
- if ((!index && !(tmp & AT91_SCKC_OSCSEL)) ||
- (index && (tmp & AT91_SCKC_OSCSEL)))
+ if ((!index && !(tmp & slowck->bits->cr_oscsel)) ||
+ (index && (tmp & slowck->bits->cr_oscsel)))
return 0;
if (index)
- tmp |= AT91_SCKC_OSCSEL;
+ tmp |= slowck->bits->cr_oscsel;
else
- tmp &= ~AT91_SCKC_OSCSEL;
+ tmp &= ~slowck->bits->cr_oscsel;
writel(tmp, sckcr);
@@ -270,7 +297,7 @@ static u8 clk_sam9x5_slow_get_parent(struct clk_hw *hw)
{
struct clk_sam9x5_slow *slowck = to_clk_sam9x5_slow(hw);
- return !!(readl(slowck->sckcr) & AT91_SCKC_OSCSEL);
+ return !!(readl(slowck->sckcr) & slowck->bits->cr_oscsel);
}
static const struct clk_ops sam9x5_slow_ops = {
@@ -282,7 +309,8 @@ static struct clk_hw * __init
at91_clk_register_sam9x5_slow(void __iomem *sckcr,
const char *name,
const char **parent_names,
- int num_parents)
+ int num_parents,
+ const struct clk_slow_bits *bits)
{
struct clk_sam9x5_slow *slowck;
struct clk_hw *hw;
@@ -304,7 +332,8 @@ at91_clk_register_sam9x5_slow(void __iomem *sckcr,
slowck->hw.init = &init;
slowck->sckcr = sckcr;
- slowck->parent = !!(readl(sckcr) & AT91_SCKC_OSCSEL);
+ slowck->bits = bits;
+ slowck->parent = !!(readl(sckcr) & slowck->bits->cr_oscsel);
hw = &slowck->hw;
ret = clk_hw_register(NULL, &slowck->hw);
@@ -316,22 +345,33 @@ at91_clk_register_sam9x5_slow(void __iomem *sckcr,
return hw;
}
+static void at91_clk_unregister_sam9x5_slow(struct clk_hw *hw)
+{
+ struct clk_sam9x5_slow *slowck = to_clk_sam9x5_slow(hw);
+
+ clk_hw_unregister(hw);
+ kfree(slowck);
+}
+
static void __init at91sam9x5_sckc_register(struct device_node *np,
- unsigned int rc_osc_startup_us)
+ unsigned int rc_osc_startup_us,
+ const struct clk_slow_bits *bits)
{
const char *parent_names[2] = { "slow_rc_osc", "slow_osc" };
void __iomem *regbase = of_iomap(np, 0);
struct device_node *child = NULL;
const char *xtal_name;
- struct clk_hw *hw;
+ struct clk_hw *slow_rc, *slow_osc, *slowck;
bool bypass;
+ int ret;
if (!regbase)
return;
- hw = at91_clk_register_slow_rc_osc(regbase, parent_names[0], 32768,
- 50000000, rc_osc_startup_us);
- if (IS_ERR(hw))
+ slow_rc = at91_clk_register_slow_rc_osc(regbase, parent_names[0],
+ 32768, 50000000,
+ rc_osc_startup_us, bits);
+ if (IS_ERR(slow_rc))
return;
xtal_name = of_clk_get_parent_name(np, 0);
@@ -339,7 +379,7 @@ static void __init at91sam9x5_sckc_register(struct device_node *np,
/* DT backward compatibility */
child = of_get_compatible_child(np, "atmel,at91sam9x5-clk-slow-osc");
if (!child)
- return;
+ goto unregister_slow_rc;
xtal_name = of_clk_get_parent_name(child, 0);
bypass = of_property_read_bool(child, "atmel,osc-bypass");
@@ -350,38 +390,133 @@ static void __init at91sam9x5_sckc_register(struct device_node *np,
}
if (!xtal_name)
- return;
-
- hw = at91_clk_register_slow_osc(regbase, parent_names[1], xtal_name,
- 1200000, bypass);
- if (IS_ERR(hw))
- return;
+ goto unregister_slow_rc;
- hw = at91_clk_register_sam9x5_slow(regbase, "slowck", parent_names, 2);
- if (IS_ERR(hw))
- return;
+ slow_osc = at91_clk_register_slow_osc(regbase, parent_names[1],
+ xtal_name, 1200000, bypass, bits);
+ if (IS_ERR(slow_osc))
+ goto unregister_slow_rc;
- of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
+ slowck = at91_clk_register_sam9x5_slow(regbase, "slowck", parent_names,
+ 2, bits);
+ if (IS_ERR(slowck))
+ goto unregister_slow_osc;
/* DT backward compatibility */
if (child)
- of_clk_add_hw_provider(child, of_clk_hw_simple_get, hw);
+ ret = of_clk_add_hw_provider(child, of_clk_hw_simple_get,
+ slowck);
+ else
+ ret = of_clk_add_hw_provider(np, of_clk_hw_simple_get, slowck);
+
+ if (WARN_ON(ret))
+ goto unregister_slowck;
+
+ return;
+
+unregister_slowck:
+ at91_clk_unregister_sam9x5_slow(slowck);
+unregister_slow_osc:
+ at91_clk_unregister_slow_osc(slow_osc);
+unregister_slow_rc:
+ at91_clk_unregister_slow_rc_osc(slow_rc);
}
+static const struct clk_slow_bits at91sam9x5_bits = {
+ .cr_rcen = BIT(0),
+ .cr_osc32en = BIT(1),
+ .cr_osc32byp = BIT(2),
+ .cr_oscsel = BIT(3),
+};
+
static void __init of_at91sam9x5_sckc_setup(struct device_node *np)
{
- at91sam9x5_sckc_register(np, 75);
+ at91sam9x5_sckc_register(np, 75, &at91sam9x5_bits);
}
CLK_OF_DECLARE(at91sam9x5_clk_sckc, "atmel,at91sam9x5-sckc",
of_at91sam9x5_sckc_setup);
static void __init of_sama5d3_sckc_setup(struct device_node *np)
{
- at91sam9x5_sckc_register(np, 500);
+ at91sam9x5_sckc_register(np, 500, &at91sam9x5_bits);
}
CLK_OF_DECLARE(sama5d3_clk_sckc, "atmel,sama5d3-sckc",
of_sama5d3_sckc_setup);
+static const struct clk_slow_bits at91sam9x60_bits = {
+ .cr_osc32en = BIT(1),
+ .cr_osc32byp = BIT(2),
+ .cr_oscsel = BIT(24),
+};
+
+static void __init of_sam9x60_sckc_setup(struct device_node *np)
+{
+ void __iomem *regbase = of_iomap(np, 0);
+ struct clk_hw_onecell_data *clk_data;
+ struct clk_hw *slow_rc, *slow_osc;
+ const char *xtal_name;
+ const char *parent_names[2] = { "slow_rc_osc", "slow_osc" };
+ bool bypass;
+ int ret;
+
+ if (!regbase)
+ return;
+
+ slow_rc = clk_hw_register_fixed_rate(NULL, parent_names[0], NULL, 0,
+ 32768);
+ if (IS_ERR(slow_rc))
+ return;
+
+ xtal_name = of_clk_get_parent_name(np, 0);
+ if (!xtal_name)
+ goto unregister_slow_rc;
+
+ bypass = of_property_read_bool(np, "atmel,osc-bypass");
+ slow_osc = at91_clk_register_slow_osc(regbase, parent_names[1],
+ xtal_name, 5000000, bypass,
+ &at91sam9x60_bits);
+ if (IS_ERR(slow_osc))
+ goto unregister_slow_rc;
+
+ clk_data = kzalloc(sizeof(*clk_data) + (2 * sizeof(struct clk_hw *)),
+ GFP_KERNEL);
+ if (!clk_data)
+ goto unregister_slow_osc;
+
+ /* MD_SLCK and TD_SLCK. */
+ clk_data->num = 2;
+ clk_data->hws[0] = clk_hw_register_fixed_rate(NULL, "md_slck",
+ parent_names[0],
+ 0, 32768);
+ if (IS_ERR(clk_data->hws[0]))
+ goto clk_data_free;
+
+ clk_data->hws[1] = at91_clk_register_sam9x5_slow(regbase, "td_slck",
+ parent_names, 2,
+ &at91sam9x60_bits);
+ if (IS_ERR(clk_data->hws[1]))
+ goto unregister_md_slck;
+
+ ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_data);
+ if (WARN_ON(ret))
+ goto unregister_td_slck;
+
+ return;
+
+unregister_td_slck:
+ at91_clk_unregister_sam9x5_slow(clk_data->hws[1]);
+unregister_md_slck:
+ clk_hw_unregister(clk_data->hws[0]);
+clk_data_free:
+ kfree(clk_data);
+unregister_slow_osc:
+ at91_clk_unregister_slow_osc(slow_osc);
+unregister_slow_rc:
+ clk_hw_unregister(slow_rc);
+}
+CLK_OF_DECLARE(sam9x60_clk_sckc, "microchip,sam9x60-sckc",
+ of_sam9x60_sckc_setup);
+
static int clk_sama5d4_slow_osc_prepare(struct clk_hw *hw)
{
struct clk_sama5d4_slow_osc *osc = to_clk_sama5d4_slow_osc(hw);
@@ -393,7 +528,7 @@ static int clk_sama5d4_slow_osc_prepare(struct clk_hw *hw)
* Assume that if it has already been selected (for example by the
* bootloader), enough time has aready passed.
*/
- if ((readl(osc->sckcr) & AT91_SCKC_OSCSEL)) {
+ if ((readl(osc->sckcr) & osc->bits->cr_oscsel)) {
osc->prepared = true;
return 0;
}
@@ -416,33 +551,35 @@ static const struct clk_ops sama5d4_slow_osc_ops = {
.is_prepared = clk_sama5d4_slow_osc_is_prepared,
};
+static const struct clk_slow_bits at91sama5d4_bits = {
+ .cr_oscsel = BIT(3),
+};
+
static void __init of_sama5d4_sckc_setup(struct device_node *np)
{
void __iomem *regbase = of_iomap(np, 0);
- struct clk_hw *hw;
+ struct clk_hw *slow_rc, *slowck;
struct clk_sama5d4_slow_osc *osc;
struct clk_init_data init;
const char *xtal_name;
const char *parent_names[2] = { "slow_rc_osc", "slow_osc" };
- bool bypass;
int ret;
if (!regbase)
return;
- hw = clk_hw_register_fixed_rate_with_accuracy(NULL, parent_names[0],
- NULL, 0, 32768,
- 250000000);
- if (IS_ERR(hw))
+ slow_rc = clk_hw_register_fixed_rate_with_accuracy(NULL,
+ parent_names[0],
+ NULL, 0, 32768,
+ 250000000);
+ if (IS_ERR(slow_rc))
return;
xtal_name = of_clk_get_parent_name(np, 0);
- bypass = of_property_read_bool(np, "atmel,osc-bypass");
-
osc = kzalloc(sizeof(*osc), GFP_KERNEL);
if (!osc)
- return;
+ goto unregister_slow_rc;
init.name = parent_names[1];
init.ops = &sama5d4_slow_osc_ops;
@@ -453,22 +590,32 @@ static void __init of_sama5d4_sckc_setup(struct device_node *np)
osc->hw.init = &init;
osc->sckcr = regbase;
osc->startup_usec = 1200000;
+ osc->bits = &at91sama5d4_bits;
- if (bypass)
- writel((readl(regbase) | AT91_SCKC_OSC32BYP), regbase);
-
- hw = &osc->hw;
ret = clk_hw_register(NULL, &osc->hw);
- if (ret) {
- kfree(osc);
- return;
- }
-
- hw = at91_clk_register_sam9x5_slow(regbase, "slowck", parent_names, 2);
- if (IS_ERR(hw))
- return;
-
- of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
+ if (ret)
+ goto free_slow_osc_data;
+
+ slowck = at91_clk_register_sam9x5_slow(regbase, "slowck",
+ parent_names, 2,
+ &at91sama5d4_bits);
+ if (IS_ERR(slowck))
+ goto unregister_slow_osc;
+
+ ret = of_clk_add_hw_provider(np, of_clk_hw_simple_get, slowck);
+ if (WARN_ON(ret))
+ goto unregister_slowck;
+
+ return;
+
+unregister_slowck:
+ at91_clk_unregister_sam9x5_slow(slowck);
+unregister_slow_osc:
+ clk_hw_unregister(&osc->hw);
+free_slow_osc_data:
+ kfree(osc);
+unregister_slow_rc:
+ clk_hw_unregister(slow_rc);
}
CLK_OF_DECLARE(sama5d4_clk_sckc, "atmel,sama5d4-sckc",
of_sama5d4_sckc_setup);
diff --git a/drivers/clk/bcm/Kconfig b/drivers/clk/bcm/Kconfig
index 29ee7b776cd4..8c83977a7dc4 100644
--- a/drivers/clk/bcm/Kconfig
+++ b/drivers/clk/bcm/Kconfig
@@ -1,4 +1,13 @@
# SPDX-License-Identifier: GPL-2.0-only
+config CLK_BCM2835
+ bool "Broadcom BCM2835 clock support"
+ depends on ARCH_BCM2835 || ARCH_BRCMSTB || COMPILE_TEST
+ depends on COMMON_CLK
+ default ARCH_BCM2835 || ARCH_BRCMSTB
+ help
+ Enable common clock framework support for Broadcom BCM2835
+ SoCs.
+
config CLK_BCM_63XX
bool "Broadcom BCM63xx clock support"
depends on ARCH_BCM_63XX || COMPILE_TEST
@@ -8,6 +17,14 @@ config CLK_BCM_63XX
Enable common clock framework support for Broadcom BCM63xx DSL SoCs
based on the ARM architecture
+config CLK_BCM_63XX_GATE
+ bool "Broadcom BCM63xx gated clock support"
+ depends on BMIPS_GENERIC || COMPILE_TEST
+ default BMIPS_GENERIC
+ help
+ Enable common clock framework support for Broadcom BCM63xx DSL SoCs
+ based on the MIPS architecture
+
config CLK_BCM_KONA
bool "Broadcom Kona CCU clock support"
depends on ARCH_BCM_MOBILE || COMPILE_TEST
@@ -64,3 +81,10 @@ config CLK_BCM_SR
default ARCH_BCM_IPROC
help
Enable common clock framework support for the Broadcom Stingray SoC
+
+config CLK_RASPBERRYPI
+ tristate "Raspberry Pi firmware based clock support"
+ depends on RASPBERRYPI_FIRMWARE || (COMPILE_TEST && !RASPBERRYPI_FIRMWARE)
+ help
+ Enable common clock framework support for Raspberry Pi's firmware
+ dependent clocks
diff --git a/drivers/clk/bcm/Makefile b/drivers/clk/bcm/Makefile
index 002661d39128..0070ddf6cdd2 100644
--- a/drivers/clk/bcm/Makefile
+++ b/drivers/clk/bcm/Makefile
@@ -1,12 +1,14 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_CLK_BCM_63XX) += clk-bcm63xx.o
+obj-$(CONFIG_CLK_BCM_63XX_GATE) += clk-bcm63xx-gate.o
obj-$(CONFIG_CLK_BCM_KONA) += clk-kona.o
obj-$(CONFIG_CLK_BCM_KONA) += clk-kona-setup.o
obj-$(CONFIG_CLK_BCM_KONA) += clk-bcm281xx.o
obj-$(CONFIG_CLK_BCM_KONA) += clk-bcm21664.o
obj-$(CONFIG_COMMON_CLK_IPROC) += clk-iproc-armpll.o clk-iproc-pll.o clk-iproc-asiu.o
-obj-$(CONFIG_ARCH_BCM2835) += clk-bcm2835.o
-obj-$(CONFIG_ARCH_BCM2835) += clk-bcm2835-aux.o
+obj-$(CONFIG_CLK_BCM2835) += clk-bcm2835.o
+obj-$(CONFIG_CLK_BCM2835) += clk-bcm2835-aux.o
+obj-$(CONFIG_CLK_RASPBERRYPI) += clk-raspberrypi.o
obj-$(CONFIG_ARCH_BCM_53573) += clk-bcm53573-ilp.o
obj-$(CONFIG_CLK_BCM_CYGNUS) += clk-cygnus.o
obj-$(CONFIG_CLK_BCM_HR2) += clk-hr2.o
diff --git a/drivers/clk/bcm/clk-bcm2835.c b/drivers/clk/bcm/clk-bcm2835.c
index 770bb01f523e..867ae3c20041 100644
--- a/drivers/clk/bcm/clk-bcm2835.c
+++ b/drivers/clk/bcm/clk-bcm2835.c
@@ -1651,30 +1651,10 @@ static const struct bcm2835_clk_desc clk_desc_array[] = {
.fixed_divider = 1,
.flags = CLK_SET_RATE_PARENT),
- /* PLLB is used for the ARM's clock. */
- [BCM2835_PLLB] = REGISTER_PLL(
- .name = "pllb",
- .cm_ctrl_reg = CM_PLLB,
- .a2w_ctrl_reg = A2W_PLLB_CTRL,
- .frac_reg = A2W_PLLB_FRAC,
- .ana_reg_base = A2W_PLLB_ANA0,
- .reference_enable_mask = A2W_XOSC_CTRL_PLLB_ENABLE,
- .lock_mask = CM_LOCK_FLOCKB,
-
- .ana = &bcm2835_ana_default,
-
- .min_rate = 600000000u,
- .max_rate = 3000000000u,
- .max_fb_rate = BCM2835_MAX_FB_RATE),
- [BCM2835_PLLB_ARM] = REGISTER_PLL_DIV(
- .name = "pllb_arm",
- .source_pll = "pllb",
- .cm_reg = CM_PLLB,
- .a2w_reg = A2W_PLLB_ARM,
- .load_mask = CM_PLLB_LOADARM,
- .hold_mask = CM_PLLB_HOLDARM,
- .fixed_divider = 1,
- .flags = CLK_SET_RATE_PARENT),
+ /*
+ * PLLB is used for the ARM's clock. Controlled by firmware, see
+ * clk-raspberrypi.c.
+ */
/*
* PLLC is the core PLL, used to drive the core VPU clock.
diff --git a/drivers/clk/bcm/clk-bcm63xx-gate.c b/drivers/clk/bcm/clk-bcm63xx-gate.c
new file mode 100644
index 000000000000..9e1dcd43258c
--- /dev/null
+++ b/drivers/clk/bcm/clk-bcm63xx-gate.c
@@ -0,0 +1,238 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/clk-provider.h>
+#include <linux/init.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+
+struct clk_bcm63xx_table_entry {
+ const char * const name;
+ u8 bit;
+ unsigned long flags;
+};
+
+struct clk_bcm63xx_hw {
+ void __iomem *regs;
+ spinlock_t lock;
+
+ struct clk_hw_onecell_data data;
+};
+
+static const struct clk_bcm63xx_table_entry bcm3368_clocks[] = {
+ { .name = "mac", .bit = 3, },
+ { .name = "tc", .bit = 5, },
+ { .name = "us_top", .bit = 6, },
+ { .name = "ds_top", .bit = 7, },
+ { .name = "acm", .bit = 8, },
+ { .name = "spi", .bit = 9, },
+ { .name = "usbs", .bit = 10, },
+ { .name = "bmu", .bit = 11, },
+ { .name = "pcm", .bit = 12, },
+ { .name = "ntp", .bit = 13, },
+ { .name = "acp_b", .bit = 14, },
+ { .name = "acp_a", .bit = 15, },
+ { .name = "emusb", .bit = 17, },
+ { .name = "enet0", .bit = 18, },
+ { .name = "enet1", .bit = 19, },
+ { .name = "usbsu", .bit = 20, },
+ { .name = "ephy", .bit = 21, },
+ { },
+};
+
+static const struct clk_bcm63xx_table_entry bcm6328_clocks[] = {
+ { .name = "phy_mips", .bit = 0, },
+ { .name = "adsl_qproc", .bit = 1, },
+ { .name = "adsl_afe", .bit = 2, },
+ { .name = "adsl", .bit = 3, },
+ { .name = "mips", .bit = 4, .flags = CLK_IS_CRITICAL, },
+ { .name = "sar", .bit = 5, },
+ { .name = "pcm", .bit = 6, },
+ { .name = "usbd", .bit = 7, },
+ { .name = "usbh", .bit = 8, },
+ { .name = "hsspi", .bit = 9, },
+ { .name = "pcie", .bit = 10, },
+ { .name = "robosw", .bit = 11, },
+ { },
+};
+
+static const struct clk_bcm63xx_table_entry bcm6358_clocks[] = {
+ { .name = "enet", .bit = 4, },
+ { .name = "adslphy", .bit = 5, },
+ { .name = "pcm", .bit = 8, },
+ { .name = "spi", .bit = 9, },
+ { .name = "usbs", .bit = 10, },
+ { .name = "sar", .bit = 11, },
+ { .name = "emusb", .bit = 17, },
+ { .name = "enet0", .bit = 18, },
+ { .name = "enet1", .bit = 19, },
+ { .name = "usbsu", .bit = 20, },
+ { .name = "ephy", .bit = 21, },
+ { },
+};
+
+static const struct clk_bcm63xx_table_entry bcm6362_clocks[] = {
+ { .name = "adsl_qproc", .bit = 1, },
+ { .name = "adsl_afe", .bit = 2, },
+ { .name = "adsl", .bit = 3, },
+ { .name = "mips", .bit = 4, .flags = CLK_IS_CRITICAL, },
+ { .name = "wlan_ocp", .bit = 5, },
+ { .name = "swpkt_usb", .bit = 7, },
+ { .name = "swpkt_sar", .bit = 8, },
+ { .name = "sar", .bit = 9, },
+ { .name = "robosw", .bit = 10, },
+ { .name = "pcm", .bit = 11, },
+ { .name = "usbd", .bit = 12, },
+ { .name = "usbh", .bit = 13, },
+ { .name = "ipsec", .bit = 14, },
+ { .name = "spi", .bit = 15, },
+ { .name = "hsspi", .bit = 16, },
+ { .name = "pcie", .bit = 17, },
+ { .name = "fap", .bit = 18, },
+ { .name = "phymips", .bit = 19, },
+ { .name = "nand", .bit = 20, },
+ { },
+};
+
+static const struct clk_bcm63xx_table_entry bcm6368_clocks[] = {
+ { .name = "vdsl_qproc", .bit = 2, },
+ { .name = "vdsl_afe", .bit = 3, },
+ { .name = "vdsl_bonding", .bit = 4, },
+ { .name = "vdsl", .bit = 5, },
+ { .name = "phymips", .bit = 6, },
+ { .name = "swpkt_usb", .bit = 7, },
+ { .name = "swpkt_sar", .bit = 8, },
+ { .name = "spi", .bit = 9, },
+ { .name = "usbd", .bit = 10, },
+ { .name = "sar", .bit = 11, },
+ { .name = "robosw", .bit = 12, },
+ { .name = "utopia", .bit = 13, },
+ { .name = "pcm", .bit = 14, },
+ { .name = "usbh", .bit = 15, },
+ { .name = "disable_gless", .bit = 16, },
+ { .name = "nand", .bit = 17, },
+ { .name = "ipsec", .bit = 18, },
+ { },
+};
+
+static const struct clk_bcm63xx_table_entry bcm63268_clocks[] = {
+ { .name = "disable_gless", .bit = 0, },
+ { .name = "vdsl_qproc", .bit = 1, },
+ { .name = "vdsl_afe", .bit = 2, },
+ { .name = "vdsl", .bit = 3, },
+ { .name = "mips", .bit = 4, .flags = CLK_IS_CRITICAL, },
+ { .name = "wlan_ocp", .bit = 5, },
+ { .name = "dect", .bit = 6, },
+ { .name = "fap0", .bit = 7, },
+ { .name = "fap1", .bit = 8, },
+ { .name = "sar", .bit = 9, },
+ { .name = "robosw", .bit = 10, },
+ { .name = "pcm", .bit = 11, },
+ { .name = "usbd", .bit = 12, },
+ { .name = "usbh", .bit = 13, },
+ { .name = "ipsec", .bit = 14, },
+ { .name = "spi", .bit = 15, },
+ { .name = "hsspi", .bit = 16, },
+ { .name = "pcie", .bit = 17, },
+ { .name = "phymips", .bit = 18, },
+ { .name = "gmac", .bit = 19, },
+ { .name = "nand", .bit = 20, },
+ { .name = "tbus", .bit = 27, },
+ { .name = "robosw250", .bit = 31, },
+ { },
+};
+
+static int clk_bcm63xx_probe(struct platform_device *pdev)
+{
+ const struct clk_bcm63xx_table_entry *entry, *table;
+ struct clk_bcm63xx_hw *hw;
+ struct resource *r;
+ u8 maxbit = 0;
+ int i, ret;
+
+ table = of_device_get_match_data(&pdev->dev);
+ if (!table)
+ return -EINVAL;
+
+ for (entry = table; entry->name; entry++)
+ maxbit = max_t(u8, maxbit, entry->bit);
+
+ hw = devm_kzalloc(&pdev->dev, struct_size(hw, data.hws, maxbit),
+ GFP_KERNEL);
+ if (!hw)
+ return -ENOMEM;
+
+ platform_set_drvdata(pdev, hw);
+
+ spin_lock_init(&hw->lock);
+
+ hw->data.num = maxbit;
+ for (i = 0; i < maxbit; i++)
+ hw->data.hws[i] = ERR_PTR(-ENODEV);
+
+ r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ hw->regs = devm_ioremap_resource(&pdev->dev, r);
+ if (IS_ERR(hw->regs))
+ return PTR_ERR(hw->regs);
+
+ for (entry = table; entry->name; entry++) {
+ struct clk_hw *clk;
+
+ clk = clk_hw_register_gate(&pdev->dev, entry->name, NULL,
+ entry->flags, hw->regs, entry->bit,
+ CLK_GATE_BIG_ENDIAN, &hw->lock);
+ if (IS_ERR(clk)) {
+ ret = PTR_ERR(clk);
+ goto out_err;
+ }
+
+ hw->data.hws[entry->bit] = clk;
+ }
+
+ ret = of_clk_add_hw_provider(pdev->dev.of_node, of_clk_hw_onecell_get,
+ &hw->data);
+ if (!ret)
+ return 0;
+out_err:
+ for (i = 0; i < hw->data.num; i++) {
+ if (!IS_ERR(hw->data.hws[i]))
+ clk_hw_unregister_gate(hw->data.hws[i]);
+ }
+
+ return ret;
+}
+
+static int clk_bcm63xx_remove(struct platform_device *pdev)
+{
+ struct clk_bcm63xx_hw *hw = platform_get_drvdata(pdev);
+ int i;
+
+ of_clk_del_provider(pdev->dev.of_node);
+
+ for (i = 0; i < hw->data.num; i++) {
+ if (!IS_ERR(hw->data.hws[i]))
+ clk_hw_unregister_gate(hw->data.hws[i]);
+ }
+
+ return 0;
+}
+
+static const struct of_device_id clk_bcm63xx_dt_ids[] = {
+ { .compatible = "brcm,bcm3368-clocks", .data = &bcm3368_clocks, },
+ { .compatible = "brcm,bcm6328-clocks", .data = &bcm6328_clocks, },
+ { .compatible = "brcm,bcm6358-clocks", .data = &bcm6358_clocks, },
+ { .compatible = "brcm,bcm6362-clocks", .data = &bcm6362_clocks, },
+ { .compatible = "brcm,bcm6368-clocks", .data = &bcm6368_clocks, },
+ { .compatible = "brcm,bcm63268-clocks", .data = &bcm63268_clocks, },
+ { }
+};
+
+static struct platform_driver clk_bcm63xx = {
+ .probe = clk_bcm63xx_probe,
+ .remove = clk_bcm63xx_remove,
+ .driver = {
+ .name = "bcm63xx-clock",
+ .of_match_table = clk_bcm63xx_dt_ids,
+ },
+};
+builtin_platform_driver(clk_bcm63xx);
diff --git a/drivers/clk/bcm/clk-raspberrypi.c b/drivers/clk/bcm/clk-raspberrypi.c
new file mode 100644
index 000000000000..1654fd0eedc9
--- /dev/null
+++ b/drivers/clk/bcm/clk-raspberrypi.c
@@ -0,0 +1,315 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Raspberry Pi driver for firmware controlled clocks
+ *
+ * Even though clk-bcm2835 provides an interface to the hardware registers for
+ * the system clocks we've had to factor out 'pllb' as the firmware 'owns' it.
+ * We're not allowed to change it directly as we might race with the
+ * over-temperature and under-voltage protections provided by the firmware.
+ *
+ * Copyright (C) 2019 Nicolas Saenz Julienne <nsaenzjulienne@suse.de>
+ */
+
+#include <linux/clkdev.h>
+#include <linux/clk-provider.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include <soc/bcm2835/raspberrypi-firmware.h>
+
+#define RPI_FIRMWARE_ARM_CLK_ID 0x00000003
+
+#define RPI_FIRMWARE_STATE_ENABLE_BIT BIT(0)
+#define RPI_FIRMWARE_STATE_WAIT_BIT BIT(1)
+
+/*
+ * Even though the firmware interface alters 'pllb' the frequencies are
+ * provided as per 'pllb_arm'. We need to scale before passing them trough.
+ */
+#define RPI_FIRMWARE_PLLB_ARM_DIV_RATE 2
+
+#define A2W_PLL_FRAC_BITS 20
+
+struct raspberrypi_clk {
+ struct device *dev;
+ struct rpi_firmware *firmware;
+ struct platform_device *cpufreq;
+
+ unsigned long min_rate;
+ unsigned long max_rate;
+
+ struct clk_hw pllb;
+ struct clk_hw *pllb_arm;
+ struct clk_lookup *pllb_arm_lookup;
+};
+
+/*
+ * Structure of the message passed to Raspberry Pi's firmware in order to
+ * change clock rates. The 'disable_turbo' option is only available to the ARM
+ * clock (pllb) which we enable by default as turbo mode will alter multiple
+ * clocks at once.
+ *
+ * Even though we're able to access the clock registers directly we're bound to
+ * use the firmware interface as the firmware ultimately takes care of
+ * mitigating overheating/undervoltage situations and we would be changing
+ * frequencies behind his back.
+ *
+ * For more information on the firmware interface check:
+ * https://github.com/raspberrypi/firmware/wiki/Mailbox-property-interface
+ */
+struct raspberrypi_firmware_prop {
+ __le32 id;
+ __le32 val;
+ __le32 disable_turbo;
+} __packed;
+
+static int raspberrypi_clock_property(struct rpi_firmware *firmware, u32 tag,
+ u32 clk, u32 *val)
+{
+ struct raspberrypi_firmware_prop msg = {
+ .id = cpu_to_le32(clk),
+ .val = cpu_to_le32(*val),
+ .disable_turbo = cpu_to_le32(1),
+ };
+ int ret;
+
+ ret = rpi_firmware_property(firmware, tag, &msg, sizeof(msg));
+ if (ret)
+ return ret;
+
+ *val = le32_to_cpu(msg.val);
+
+ return 0;
+}
+
+static int raspberrypi_fw_pll_is_on(struct clk_hw *hw)
+{
+ struct raspberrypi_clk *rpi = container_of(hw, struct raspberrypi_clk,
+ pllb);
+ u32 val = 0;
+ int ret;
+
+ ret = raspberrypi_clock_property(rpi->firmware,
+ RPI_FIRMWARE_GET_CLOCK_STATE,
+ RPI_FIRMWARE_ARM_CLK_ID, &val);
+ if (ret)
+ return 0;
+
+ return !!(val & RPI_FIRMWARE_STATE_ENABLE_BIT);
+}
+
+
+static unsigned long raspberrypi_fw_pll_get_rate(struct clk_hw *hw,
+ unsigned long parent_rate)
+{
+ struct raspberrypi_clk *rpi = container_of(hw, struct raspberrypi_clk,
+ pllb);
+ u32 val = 0;
+ int ret;
+
+ ret = raspberrypi_clock_property(rpi->firmware,
+ RPI_FIRMWARE_GET_CLOCK_RATE,
+ RPI_FIRMWARE_ARM_CLK_ID,
+ &val);
+ if (ret)
+ return ret;
+
+ return val * RPI_FIRMWARE_PLLB_ARM_DIV_RATE;
+}
+
+static int raspberrypi_fw_pll_set_rate(struct clk_hw *hw, unsigned long rate,
+ unsigned long parent_rate)
+{
+ struct raspberrypi_clk *rpi = container_of(hw, struct raspberrypi_clk,
+ pllb);
+ u32 new_rate = rate / RPI_FIRMWARE_PLLB_ARM_DIV_RATE;
+ int ret;
+
+ ret = raspberrypi_clock_property(rpi->firmware,
+ RPI_FIRMWARE_SET_CLOCK_RATE,
+ RPI_FIRMWARE_ARM_CLK_ID,
+ &new_rate);
+ if (ret)
+ dev_err_ratelimited(rpi->dev, "Failed to change %s frequency: %d",
+ clk_hw_get_name(hw), ret);
+
+ return ret;
+}
+
+/*
+ * Sadly there is no firmware rate rounding interface. We borrowed it from
+ * clk-bcm2835.
+ */
+static int raspberrypi_pll_determine_rate(struct clk_hw *hw,
+ struct clk_rate_request *req)
+{
+ struct raspberrypi_clk *rpi = container_of(hw, struct raspberrypi_clk,
+ pllb);
+ u64 div, final_rate;
+ u32 ndiv, fdiv;
+
+ /* We can't use req->rate directly as it would overflow */
+ final_rate = clamp(req->rate, rpi->min_rate, rpi->max_rate);
+
+ div = (u64)final_rate << A2W_PLL_FRAC_BITS;
+ do_div(div, req->best_parent_rate);
+
+ ndiv = div >> A2W_PLL_FRAC_BITS;
+ fdiv = div & ((1 << A2W_PLL_FRAC_BITS) - 1);
+
+ final_rate = ((u64)req->best_parent_rate *
+ ((ndiv << A2W_PLL_FRAC_BITS) + fdiv));
+
+ req->rate = final_rate >> A2W_PLL_FRAC_BITS;
+
+ return 0;
+}
+
+static const struct clk_ops raspberrypi_firmware_pll_clk_ops = {
+ .is_prepared = raspberrypi_fw_pll_is_on,
+ .recalc_rate = raspberrypi_fw_pll_get_rate,
+ .set_rate = raspberrypi_fw_pll_set_rate,
+ .determine_rate = raspberrypi_pll_determine_rate,
+};
+
+static int raspberrypi_register_pllb(struct raspberrypi_clk *rpi)
+{
+ u32 min_rate = 0, max_rate = 0;
+ struct clk_init_data init;
+ int ret;
+
+ memset(&init, 0, sizeof(init));
+
+ /* All of the PLLs derive from the external oscillator. */
+ init.parent_names = (const char *[]){ "osc" };
+ init.num_parents = 1;
+ init.name = "pllb";
+ init.ops = &raspberrypi_firmware_pll_clk_ops;
+ init.flags = CLK_GET_RATE_NOCACHE | CLK_IGNORE_UNUSED;
+
+ /* Get min & max rates set by the firmware */
+ ret = raspberrypi_clock_property(rpi->firmware,
+ RPI_FIRMWARE_GET_MIN_CLOCK_RATE,
+ RPI_FIRMWARE_ARM_CLK_ID,
+ &min_rate);
+ if (ret) {
+ dev_err(rpi->dev, "Failed to get %s min freq: %d\n",
+ init.name, ret);
+ return ret;
+ }
+
+ ret = raspberrypi_clock_property(rpi->firmware,
+ RPI_FIRMWARE_GET_MAX_CLOCK_RATE,
+ RPI_FIRMWARE_ARM_CLK_ID,
+ &max_rate);
+ if (ret) {
+ dev_err(rpi->dev, "Failed to get %s max freq: %d\n",
+ init.name, ret);
+ return ret;
+ }
+
+ if (!min_rate || !max_rate) {
+ dev_err(rpi->dev, "Unexpected frequency range: min %u, max %u\n",
+ min_rate, max_rate);
+ return -EINVAL;
+ }
+
+ dev_info(rpi->dev, "CPU frequency range: min %u, max %u\n",
+ min_rate, max_rate);
+
+ rpi->min_rate = min_rate * RPI_FIRMWARE_PLLB_ARM_DIV_RATE;
+ rpi->max_rate = max_rate * RPI_FIRMWARE_PLLB_ARM_DIV_RATE;
+
+ rpi->pllb.init = &init;
+
+ return devm_clk_hw_register(rpi->dev, &rpi->pllb);
+}
+
+static int raspberrypi_register_pllb_arm(struct raspberrypi_clk *rpi)
+{
+ rpi->pllb_arm = clk_hw_register_fixed_factor(rpi->dev,
+ "pllb_arm", "pllb",
+ CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE,
+ 1, 2);
+ if (IS_ERR(rpi->pllb_arm)) {
+ dev_err(rpi->dev, "Failed to initialize pllb_arm\n");
+ return PTR_ERR(rpi->pllb_arm);
+ }
+
+ rpi->pllb_arm_lookup = clkdev_hw_create(rpi->pllb_arm, NULL, "cpu0");
+ if (!rpi->pllb_arm_lookup) {
+ dev_err(rpi->dev, "Failed to initialize pllb_arm_lookup\n");
+ clk_hw_unregister_fixed_factor(rpi->pllb_arm);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static int raspberrypi_clk_probe(struct platform_device *pdev)
+{
+ struct device_node *firmware_node;
+ struct device *dev = &pdev->dev;
+ struct rpi_firmware *firmware;
+ struct raspberrypi_clk *rpi;
+ int ret;
+
+ firmware_node = of_find_compatible_node(NULL, NULL,
+ "raspberrypi,bcm2835-firmware");
+ if (!firmware_node) {
+ dev_err(dev, "Missing firmware node\n");
+ return -ENOENT;
+ }
+
+ firmware = rpi_firmware_get(firmware_node);
+ of_node_put(firmware_node);
+ if (!firmware)
+ return -EPROBE_DEFER;
+
+ rpi = devm_kzalloc(dev, sizeof(*rpi), GFP_KERNEL);
+ if (!rpi)
+ return -ENOMEM;
+
+ rpi->dev = dev;
+ rpi->firmware = firmware;
+ platform_set_drvdata(pdev, rpi);
+
+ ret = raspberrypi_register_pllb(rpi);
+ if (ret) {
+ dev_err(dev, "Failed to initialize pllb, %d\n", ret);
+ return ret;
+ }
+
+ ret = raspberrypi_register_pllb_arm(rpi);
+ if (ret)
+ return ret;
+
+ rpi->cpufreq = platform_device_register_data(dev, "raspberrypi-cpufreq",
+ -1, NULL, 0);
+
+ return 0;
+}
+
+static int raspberrypi_clk_remove(struct platform_device *pdev)
+{
+ struct raspberrypi_clk *rpi = platform_get_drvdata(pdev);
+
+ platform_device_unregister(rpi->cpufreq);
+
+ return 0;
+}
+
+static struct platform_driver raspberrypi_clk_driver = {
+ .driver = {
+ .name = "raspberrypi-clk",
+ },
+ .probe = raspberrypi_clk_probe,
+ .remove = raspberrypi_clk_remove,
+};
+module_platform_driver(raspberrypi_clk_driver);
+
+MODULE_AUTHOR("Nicolas Saenz Julienne <nsaenzjulienne@suse.de>");
+MODULE_DESCRIPTION("Raspberry Pi firmware clock driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:raspberrypi-clk");
diff --git a/drivers/clk/clk-bd718x7.c b/drivers/clk/clk-bd718x7.c
index 60422c72d142..ae6e5baee330 100644
--- a/drivers/clk/clk-bd718x7.c
+++ b/drivers/clk/clk-bd718x7.c
@@ -8,6 +8,7 @@
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/mfd/rohm-bd718x7.h>
+#include <linux/mfd/rohm-bd70528.h>
#include <linux/clk-provider.h>
#include <linux/clkdev.h>
#include <linux/regmap.h>
@@ -17,7 +18,7 @@ struct bd718xx_clk {
u8 reg;
u8 mask;
struct platform_device *pdev;
- struct bd718xx *mfd;
+ struct rohm_regmap_dev *mfd;
};
static int bd71837_clk_set(struct clk_hw *hw, int status)
@@ -68,7 +69,7 @@ static int bd71837_clk_probe(struct platform_device *pdev)
int rval = -ENOMEM;
const char *parent_clk;
struct device *parent = pdev->dev.parent;
- struct bd718xx *mfd = dev_get_drvdata(parent);
+ struct rohm_regmap_dev *mfd = dev_get_drvdata(parent);
struct clk_init_data init = {
.name = "bd718xx-32k-out",
.ops = &bd71837_clk_ops,
@@ -86,9 +87,20 @@ static int bd71837_clk_probe(struct platform_device *pdev)
dev_err(&pdev->dev, "No parent clk found\n");
return -EINVAL;
}
-
- c->reg = BD718XX_REG_OUT32K;
- c->mask = BD718XX_OUT32K_EN;
+ switch (mfd->chip_type) {
+ case ROHM_CHIP_TYPE_BD71837:
+ case ROHM_CHIP_TYPE_BD71847:
+ c->reg = BD718XX_REG_OUT32K;
+ c->mask = BD718XX_OUT32K_EN;
+ break;
+ case ROHM_CHIP_TYPE_BD70528:
+ c->reg = BD70528_REG_CLK_OUT;
+ c->mask = BD70528_CLK_OUT_EN_MASK;
+ break;
+ default:
+ dev_err(&pdev->dev, "Unknown clk chip\n");
+ return -EINVAL;
+ }
c->mfd = mfd;
c->pdev = pdev;
c->hw.init = &init;
@@ -119,5 +131,5 @@ static struct platform_driver bd71837_clk = {
module_platform_driver(bd71837_clk);
MODULE_AUTHOR("Matti Vaittinen <matti.vaittinen@fi.rohmeurope.com>");
-MODULE_DESCRIPTION("BD71837 chip clk driver");
+MODULE_DESCRIPTION("BD71837/BD71847/BD70528 chip clk driver");
MODULE_LICENSE("GPL");
diff --git a/drivers/clk/clk-bulk.c b/drivers/clk/clk-bulk.c
index 06499568cf07..524bf9a53098 100644
--- a/drivers/clk/clk-bulk.c
+++ b/drivers/clk/clk-bulk.c
@@ -75,8 +75,8 @@ void clk_bulk_put(int num_clks, struct clk_bulk_data *clks)
}
EXPORT_SYMBOL_GPL(clk_bulk_put);
-int __must_check clk_bulk_get(struct device *dev, int num_clks,
- struct clk_bulk_data *clks)
+static int __clk_bulk_get(struct device *dev, int num_clks,
+ struct clk_bulk_data *clks, bool optional)
{
int ret;
int i;
@@ -88,10 +88,14 @@ int __must_check clk_bulk_get(struct device *dev, int num_clks,
clks[i].clk = clk_get(dev, clks[i].id);
if (IS_ERR(clks[i].clk)) {
ret = PTR_ERR(clks[i].clk);
+ clks[i].clk = NULL;
+
+ if (ret == -ENOENT && optional)
+ continue;
+
if (ret != -EPROBE_DEFER)
dev_err(dev, "Failed to get clk '%s': %d\n",
clks[i].id, ret);
- clks[i].clk = NULL;
goto err;
}
}
@@ -103,8 +107,21 @@ err:
return ret;
}
+
+int __must_check clk_bulk_get(struct device *dev, int num_clks,
+ struct clk_bulk_data *clks)
+{
+ return __clk_bulk_get(dev, num_clks, clks, false);
+}
EXPORT_SYMBOL(clk_bulk_get);
+int __must_check clk_bulk_get_optional(struct device *dev, int num_clks,
+ struct clk_bulk_data *clks)
+{
+ return __clk_bulk_get(dev, num_clks, clks, true);
+}
+EXPORT_SYMBOL_GPL(clk_bulk_get_optional);
+
void clk_bulk_put_all(int num_clks, struct clk_bulk_data *clks)
{
if (IS_ERR_OR_NULL(clks))
diff --git a/drivers/clk/clk-cdce706.c b/drivers/clk/clk-cdce706.c
index 0443dfc82794..239102e37e2f 100644
--- a/drivers/clk/clk-cdce706.c
+++ b/drivers/clk/clk-cdce706.c
@@ -630,7 +630,7 @@ of_clk_cdce_get(struct of_phandle_args *clkspec, void *data)
static int cdce706_probe(struct i2c_client *client,
const struct i2c_device_id *id)
{
- struct i2c_adapter *adapter = to_i2c_adapter(client->dev.parent);
+ struct i2c_adapter *adapter = client->adapter;
struct cdce706_dev_data *cdce;
int ret;
diff --git a/drivers/clk/clk-devres.c b/drivers/clk/clk-devres.c
index daa1fc8fba53..be160764911b 100644
--- a/drivers/clk/clk-devres.c
+++ b/drivers/clk/clk-devres.c
@@ -52,8 +52,8 @@ static void devm_clk_bulk_release(struct device *dev, void *res)
clk_bulk_put(devres->num_clks, devres->clks);
}
-int __must_check devm_clk_bulk_get(struct device *dev, int num_clks,
- struct clk_bulk_data *clks)
+static int __devm_clk_bulk_get(struct device *dev, int num_clks,
+ struct clk_bulk_data *clks, bool optional)
{
struct clk_bulk_devres *devres;
int ret;
@@ -63,7 +63,10 @@ int __must_check devm_clk_bulk_get(struct device *dev, int num_clks,
if (!devres)
return -ENOMEM;
- ret = clk_bulk_get(dev, num_clks, clks);
+ if (optional)
+ ret = clk_bulk_get_optional(dev, num_clks, clks);
+ else
+ ret = clk_bulk_get(dev, num_clks, clks);
if (!ret) {
devres->clks = clks;
devres->num_clks = num_clks;
@@ -74,8 +77,21 @@ int __must_check devm_clk_bulk_get(struct device *dev, int num_clks,
return ret;
}
+
+int __must_check devm_clk_bulk_get(struct device *dev, int num_clks,
+ struct clk_bulk_data *clks)
+{
+ return __devm_clk_bulk_get(dev, num_clks, clks, false);
+}
EXPORT_SYMBOL_GPL(devm_clk_bulk_get);
+int __must_check devm_clk_bulk_get_optional(struct device *dev, int num_clks,
+ struct clk_bulk_data *clks)
+{
+ return __devm_clk_bulk_get(dev, num_clks, clks, true);
+}
+EXPORT_SYMBOL_GPL(devm_clk_bulk_get_optional);
+
int __must_check devm_clk_bulk_get_all(struct device *dev,
struct clk_bulk_data **clks)
{
diff --git a/drivers/clk/clk-lochnagar.c b/drivers/clk/clk-lochnagar.c
index a2f31e58ee48..fa8c91758b1d 100644
--- a/drivers/clk/clk-lochnagar.c
+++ b/drivers/clk/clk-lochnagar.c
@@ -16,7 +16,6 @@
#include <linux/platform_device.h>
#include <linux/regmap.h>
-#include <linux/mfd/lochnagar.h>
#include <linux/mfd/lochnagar1_regs.h>
#include <linux/mfd/lochnagar2_regs.h>
@@ -40,48 +39,46 @@ struct lochnagar_clk {
struct lochnagar_clk_priv {
struct device *dev;
struct regmap *regmap;
- enum lochnagar_type type;
-
- const char **parents;
- unsigned int nparents;
struct lochnagar_clk lclks[LOCHNAGAR_NUM_CLOCKS];
};
-static const char * const lochnagar1_clk_parents[] = {
- "ln-none",
- "ln-spdif-mclk",
- "ln-psia1-mclk",
- "ln-psia2-mclk",
- "ln-cdc-clkout",
- "ln-dsp-clkout",
- "ln-pmic-32k",
- "ln-gf-mclk1",
- "ln-gf-mclk3",
- "ln-gf-mclk2",
- "ln-gf-mclk4",
+#define LN_PARENT(NAME) { .name = NAME, .fw_name = NAME }
+
+static const struct clk_parent_data lochnagar1_clk_parents[] = {
+ LN_PARENT("ln-none"),
+ LN_PARENT("ln-spdif-mclk"),
+ LN_PARENT("ln-psia1-mclk"),
+ LN_PARENT("ln-psia2-mclk"),
+ LN_PARENT("ln-cdc-clkout"),
+ LN_PARENT("ln-dsp-clkout"),
+ LN_PARENT("ln-pmic-32k"),
+ LN_PARENT("ln-gf-mclk1"),
+ LN_PARENT("ln-gf-mclk3"),
+ LN_PARENT("ln-gf-mclk2"),
+ LN_PARENT("ln-gf-mclk4"),
};
-static const char * const lochnagar2_clk_parents[] = {
- "ln-none",
- "ln-cdc-clkout",
- "ln-dsp-clkout",
- "ln-pmic-32k",
- "ln-spdif-mclk",
- "ln-clk-12m",
- "ln-clk-11m",
- "ln-clk-24m",
- "ln-clk-22m",
- "ln-clk-8m",
- "ln-usb-clk-24m",
- "ln-gf-mclk1",
- "ln-gf-mclk3",
- "ln-gf-mclk2",
- "ln-psia1-mclk",
- "ln-psia2-mclk",
- "ln-spdif-clkout",
- "ln-adat-mclk",
- "ln-usb-clk-12m",
+static const struct clk_parent_data lochnagar2_clk_parents[] = {
+ LN_PARENT("ln-none"),
+ LN_PARENT("ln-cdc-clkout"),
+ LN_PARENT("ln-dsp-clkout"),
+ LN_PARENT("ln-pmic-32k"),
+ LN_PARENT("ln-spdif-mclk"),
+ LN_PARENT("ln-clk-12m"),
+ LN_PARENT("ln-clk-11m"),
+ LN_PARENT("ln-clk-24m"),
+ LN_PARENT("ln-clk-22m"),
+ LN_PARENT("ln-clk-8m"),
+ LN_PARENT("ln-usb-clk-24m"),
+ LN_PARENT("ln-gf-mclk1"),
+ LN_PARENT("ln-gf-mclk3"),
+ LN_PARENT("ln-gf-mclk2"),
+ LN_PARENT("ln-psia1-mclk"),
+ LN_PARENT("ln-psia2-mclk"),
+ LN_PARENT("ln-spdif-clkout"),
+ LN_PARENT("ln-adat-mclk"),
+ LN_PARENT("ln-usb-clk-12m"),
};
#define LN1_CLK(ID, NAME, REG) \
@@ -122,6 +119,24 @@ static const struct lochnagar_clk lochnagar2_clks[LOCHNAGAR_NUM_CLOCKS] = {
LN2_CLK(SOUNDCARD_MCLK, "ln-soundcard-mclk"),
};
+struct lochnagar_config {
+ const struct clk_parent_data *parents;
+ int nparents;
+ const struct lochnagar_clk *clks;
+};
+
+static const struct lochnagar_config lochnagar1_conf = {
+ .parents = lochnagar1_clk_parents,
+ .nparents = ARRAY_SIZE(lochnagar1_clk_parents),
+ .clks = lochnagar1_clks,
+};
+
+static const struct lochnagar_config lochnagar2_conf = {
+ .parents = lochnagar2_clk_parents,
+ .nparents = ARRAY_SIZE(lochnagar2_clk_parents),
+ .clks = lochnagar2_clks,
+};
+
static inline struct lochnagar_clk *lochnagar_hw_to_lclk(struct clk_hw *hw)
{
return container_of(hw, struct lochnagar_clk, hw);
@@ -183,7 +198,7 @@ static u8 lochnagar_clk_get_parent(struct clk_hw *hw)
if (ret < 0) {
dev_dbg(priv->dev, "Failed to read parent of %s: %d\n",
lclk->name, ret);
- return priv->nparents;
+ return hw->init->num_parents;
}
val &= lclk->src_mask;
@@ -198,46 +213,6 @@ static const struct clk_ops lochnagar_clk_ops = {
.get_parent = lochnagar_clk_get_parent,
};
-static int lochnagar_init_parents(struct lochnagar_clk_priv *priv)
-{
- struct device_node *np = priv->dev->of_node;
- int i, j;
-
- switch (priv->type) {
- case LOCHNAGAR1:
- memcpy(priv->lclks, lochnagar1_clks, sizeof(lochnagar1_clks));
-
- priv->nparents = ARRAY_SIZE(lochnagar1_clk_parents);
- priv->parents = devm_kmemdup(priv->dev, lochnagar1_clk_parents,
- sizeof(lochnagar1_clk_parents),
- GFP_KERNEL);
- break;
- case LOCHNAGAR2:
- memcpy(priv->lclks, lochnagar2_clks, sizeof(lochnagar2_clks));
-
- priv->nparents = ARRAY_SIZE(lochnagar2_clk_parents);
- priv->parents = devm_kmemdup(priv->dev, lochnagar2_clk_parents,
- sizeof(lochnagar2_clk_parents),
- GFP_KERNEL);
- break;
- default:
- dev_err(priv->dev, "Unknown Lochnagar type: %d\n", priv->type);
- return -EINVAL;
- }
-
- if (!priv->parents)
- return -ENOMEM;
-
- for (i = 0; i < priv->nparents; i++) {
- j = of_property_match_string(np, "clock-names",
- priv->parents[i]);
- if (j >= 0)
- priv->parents[i] = of_clk_get_parent_name(np, j);
- }
-
- return 0;
-}
-
static struct clk_hw *
lochnagar_of_clk_hw_get(struct of_phandle_args *clkspec, void *data)
{
@@ -252,16 +227,42 @@ lochnagar_of_clk_hw_get(struct of_phandle_args *clkspec, void *data)
return &priv->lclks[idx].hw;
}
-static int lochnagar_init_clks(struct lochnagar_clk_priv *priv)
+static const struct of_device_id lochnagar_of_match[] = {
+ { .compatible = "cirrus,lochnagar1-clk", .data = &lochnagar1_conf },
+ { .compatible = "cirrus,lochnagar2-clk", .data = &lochnagar2_conf },
+ {}
+};
+MODULE_DEVICE_TABLE(of, lochnagar_of_match);
+
+static int lochnagar_clk_probe(struct platform_device *pdev)
{
struct clk_init_data clk_init = {
.ops = &lochnagar_clk_ops,
- .parent_names = priv->parents,
- .num_parents = priv->nparents,
};
+ struct device *dev = &pdev->dev;
+ struct lochnagar_clk_priv *priv;
+ const struct of_device_id *of_id;
struct lochnagar_clk *lclk;
+ struct lochnagar_config *conf;
int ret, i;
+ of_id = of_match_device(lochnagar_of_match, dev);
+ if (!of_id)
+ return -EINVAL;
+
+ priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ priv->dev = dev;
+ priv->regmap = dev_get_regmap(dev->parent, NULL);
+ conf = (struct lochnagar_config *)of_id->data;
+
+ memcpy(priv->lclks, conf->clks, sizeof(priv->lclks));
+
+ clk_init.parent_data = conf->parents;
+ clk_init.num_parents = conf->nparents;
+
for (i = 0; i < ARRAY_SIZE(priv->lclks); i++) {
lclk = &priv->lclks[i];
@@ -273,55 +274,21 @@ static int lochnagar_init_clks(struct lochnagar_clk_priv *priv)
lclk->priv = priv;
lclk->hw.init = &clk_init;
- ret = devm_clk_hw_register(priv->dev, &lclk->hw);
+ ret = devm_clk_hw_register(dev, &lclk->hw);
if (ret) {
- dev_err(priv->dev, "Failed to register %s: %d\n",
+ dev_err(dev, "Failed to register %s: %d\n",
lclk->name, ret);
return ret;
}
}
- ret = devm_of_clk_add_hw_provider(priv->dev, lochnagar_of_clk_hw_get,
- priv);
+ ret = devm_of_clk_add_hw_provider(dev, lochnagar_of_clk_hw_get, priv);
if (ret < 0)
- dev_err(priv->dev, "Failed to register provider: %d\n", ret);
+ dev_err(dev, "Failed to register provider: %d\n", ret);
return ret;
}
-static const struct of_device_id lochnagar_of_match[] = {
- { .compatible = "cirrus,lochnagar1-clk", .data = (void *)LOCHNAGAR1 },
- { .compatible = "cirrus,lochnagar2-clk", .data = (void *)LOCHNAGAR2 },
- {}
-};
-MODULE_DEVICE_TABLE(of, lochnagar_of_match);
-
-static int lochnagar_clk_probe(struct platform_device *pdev)
-{
- struct device *dev = &pdev->dev;
- struct lochnagar_clk_priv *priv;
- const struct of_device_id *of_id;
- int ret;
-
- of_id = of_match_device(lochnagar_of_match, dev);
- if (!of_id)
- return -EINVAL;
-
- priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
- if (!priv)
- return -ENOMEM;
-
- priv->dev = dev;
- priv->regmap = dev_get_regmap(dev->parent, NULL);
- priv->type = (enum lochnagar_type)of_id->data;
-
- ret = lochnagar_init_parents(priv);
- if (ret)
- return ret;
-
- return lochnagar_init_clks(priv);
-}
-
static struct platform_driver lochnagar_clk_driver = {
.driver = {
.name = "lochnagar-clk",
diff --git a/drivers/clk/clk-pwm.c b/drivers/clk/clk-pwm.c
index 5f0490b8f6cb..87fe0b0e01a3 100644
--- a/drivers/clk/clk-pwm.c
+++ b/drivers/clk/clk-pwm.c
@@ -44,10 +44,24 @@ static unsigned long clk_pwm_recalc_rate(struct clk_hw *hw,
return clk_pwm->fixed_rate;
}
+static int clk_pwm_get_duty_cycle(struct clk_hw *hw, struct clk_duty *duty)
+{
+ struct clk_pwm *clk_pwm = to_clk_pwm(hw);
+ struct pwm_state state;
+
+ pwm_get_state(clk_pwm->pwm, &state);
+
+ duty->num = state.duty_cycle;
+ duty->den = state.period;
+
+ return 0;
+}
+
static const struct clk_ops clk_pwm_ops = {
.prepare = clk_pwm_prepare,
.unprepare = clk_pwm_unprepare,
.recalc_rate = clk_pwm_recalc_rate,
+ .get_duty_cycle = clk_pwm_get_duty_cycle,
};
static int clk_pwm_probe(struct platform_device *pdev)
diff --git a/drivers/clk/clk-qoriq.c b/drivers/clk/clk-qoriq.c
index dd93d3acc67d..07f3b252f3e0 100644
--- a/drivers/clk/clk-qoriq.c
+++ b/drivers/clk/clk-qoriq.c
@@ -635,6 +635,17 @@ static const struct clockgen_chipinfo chipinfo[] = {
.flags = CG_VER3 | CG_LITTLE_ENDIAN,
},
{
+ .compat = "fsl,lx2160a-clockgen",
+ .cmux_groups = {
+ &clockgen2_cmux_cga12, &clockgen2_cmux_cgb
+ },
+ .cmux_to_group = {
+ 0, 0, 0, 0, 1, 1, 1, 1, -1
+ },
+ .pll_mask = 0x37,
+ .flags = CG_VER3 | CG_LITTLE_ENDIAN,
+ },
+ {
.compat = "fsl,p2041-clockgen",
.guts_compat = "fsl,qoriq-device-config-1.0",
.init_periph = p2041_init_periph,
@@ -1493,6 +1504,7 @@ CLK_OF_DECLARE(qoriq_clockgen_ls1043a, "fsl,ls1043a-clockgen", clockgen_init);
CLK_OF_DECLARE(qoriq_clockgen_ls1046a, "fsl,ls1046a-clockgen", clockgen_init);
CLK_OF_DECLARE(qoriq_clockgen_ls1088a, "fsl,ls1088a-clockgen", clockgen_init);
CLK_OF_DECLARE(qoriq_clockgen_ls2080a, "fsl,ls2080a-clockgen", clockgen_init);
+CLK_OF_DECLARE(qoriq_clockgen_lx2160a, "fsl,lx2160a-clockgen", clockgen_init);
CLK_OF_DECLARE(qoriq_clockgen_p2041, "fsl,p2041-clockgen", clockgen_init);
CLK_OF_DECLARE(qoriq_clockgen_p3041, "fsl,p3041-clockgen", clockgen_init);
CLK_OF_DECLARE(qoriq_clockgen_p4080, "fsl,p4080-clockgen", clockgen_init);
diff --git a/drivers/clk/clk-rk808.c b/drivers/clk/clk-rk808.c
index b1d7e4a296ef..32f833d732ed 100644
--- a/drivers/clk/clk-rk808.c
+++ b/drivers/clk/clk-rk808.c
@@ -88,6 +88,68 @@ of_clk_rk808_get(struct of_phandle_args *clkspec, void *data)
return idx ? &rk808_clkout->clkout2_hw : &rk808_clkout->clkout1_hw;
}
+static int rk817_clkout2_enable(struct clk_hw *hw, bool enable)
+{
+ struct rk808_clkout *rk808_clkout = container_of(hw,
+ struct rk808_clkout,
+ clkout2_hw);
+ struct rk808 *rk808 = rk808_clkout->rk808;
+
+ return regmap_update_bits(rk808->regmap, RK817_SYS_CFG(1),
+ RK817_CLK32KOUT2_EN,
+ enable ? RK817_CLK32KOUT2_EN : 0);
+}
+
+static int rk817_clkout2_prepare(struct clk_hw *hw)
+{
+ return rk817_clkout2_enable(hw, true);
+}
+
+static void rk817_clkout2_unprepare(struct clk_hw *hw)
+{
+ rk817_clkout2_enable(hw, false);
+}
+
+static int rk817_clkout2_is_prepared(struct clk_hw *hw)
+{
+ struct rk808_clkout *rk808_clkout = container_of(hw,
+ struct rk808_clkout,
+ clkout2_hw);
+ struct rk808 *rk808 = rk808_clkout->rk808;
+ unsigned int val;
+
+ int ret = regmap_read(rk808->regmap, RK817_SYS_CFG(1), &val);
+
+ if (ret < 0)
+ return 0;
+
+ return (val & RK817_CLK32KOUT2_EN) ? 1 : 0;
+}
+
+static const struct clk_ops rk817_clkout2_ops = {
+ .prepare = rk817_clkout2_prepare,
+ .unprepare = rk817_clkout2_unprepare,
+ .is_prepared = rk817_clkout2_is_prepared,
+ .recalc_rate = rk808_clkout_recalc_rate,
+};
+
+static const struct clk_ops *rkpmic_get_ops(long variant)
+{
+ switch (variant) {
+ case RK809_ID:
+ case RK817_ID:
+ return &rk817_clkout2_ops;
+ /*
+ * For the default case, it match the following PMIC type.
+ * RK805_ID
+ * RK808_ID
+ * RK818_ID
+ */
+ default:
+ return &rk808_clkout2_ops;
+ }
+}
+
static int rk808_clkout_probe(struct platform_device *pdev)
{
struct rk808 *rk808 = dev_get_drvdata(pdev->dev.parent);
@@ -119,7 +181,7 @@ static int rk808_clkout_probe(struct platform_device *pdev)
return ret;
init.name = "rk808-clkout2";
- init.ops = &rk808_clkout2_ops;
+ init.ops = rkpmic_get_ops(rk808->variant);
rk808_clkout->clkout2_hw.init = &init;
/* optional override of the clockname */
diff --git a/drivers/clk/clk-scmi.c b/drivers/clk/clk-scmi.c
index a2287c770d5c..886f7c5df51a 100644
--- a/drivers/clk/clk-scmi.c
+++ b/drivers/clk/clk-scmi.c
@@ -69,7 +69,7 @@ static int scmi_clk_set_rate(struct clk_hw *hw, unsigned long rate,
{
struct scmi_clk *clk = to_scmi_clk(hw);
- return clk->handle->clk_ops->rate_set(clk->handle, clk->id, 0, rate);
+ return clk->handle->clk_ops->rate_set(clk->handle, clk->id, rate);
}
static int scmi_clk_enable(struct clk_hw *hw)
diff --git a/drivers/clk/clk-si5341.c b/drivers/clk/clk-si5341.c
new file mode 100644
index 000000000000..72424eb7e5f8
--- /dev/null
+++ b/drivers/clk/clk-si5341.c
@@ -0,0 +1,1346 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for Silicon Labs Si5341/Si5340 Clock generator
+ * Copyright (C) 2019 Topic Embedded Products
+ * Author: Mike Looijmans <mike.looijmans@topic.nl>
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/gcd.h>
+#include <linux/math64.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <asm/unaligned.h>
+
+#define SI5341_MAX_NUM_OUTPUTS 10
+#define SI5340_MAX_NUM_OUTPUTS 4
+
+#define SI5341_NUM_SYNTH 5
+#define SI5340_NUM_SYNTH 4
+
+/* Range of the synthesizer fractional divider */
+#define SI5341_SYNTH_N_MIN 10
+#define SI5341_SYNTH_N_MAX 4095
+
+/* The chip can get its input clock from 3 input pins or an XTAL */
+
+/* There is one PLL running at 13500–14256 MHz */
+#define SI5341_PLL_VCO_MIN 13500000000ull
+#define SI5341_PLL_VCO_MAX 14256000000ull
+
+/* The 5 frequency synthesizers obtain their input from the PLL */
+struct clk_si5341_synth {
+ struct clk_hw hw;
+ struct clk_si5341 *data;
+ u8 index;
+};
+#define to_clk_si5341_synth(_hw) \
+ container_of(_hw, struct clk_si5341_synth, hw)
+
+/* The output stages can be connected to any synth (full mux) */
+struct clk_si5341_output {
+ struct clk_hw hw;
+ struct clk_si5341 *data;
+ u8 index;
+};
+#define to_clk_si5341_output(_hw) \
+ container_of(_hw, struct clk_si5341_output, hw)
+
+struct clk_si5341 {
+ struct clk_hw hw;
+ struct regmap *regmap;
+ struct i2c_client *i2c_client;
+ struct clk_si5341_synth synth[SI5341_NUM_SYNTH];
+ struct clk_si5341_output clk[SI5341_MAX_NUM_OUTPUTS];
+ struct clk *pxtal;
+ const char *pxtal_name;
+ const u16 *reg_output_offset;
+ const u16 *reg_rdiv_offset;
+ u64 freq_vco; /* 13500–14256 MHz */
+ u8 num_outputs;
+ u8 num_synth;
+};
+#define to_clk_si5341(_hw) container_of(_hw, struct clk_si5341, hw)
+
+struct clk_si5341_output_config {
+ u8 out_format_drv_bits;
+ u8 out_cm_ampl_bits;
+ bool synth_master;
+ bool always_on;
+};
+
+#define SI5341_PAGE 0x0001
+#define SI5341_PN_BASE 0x0002
+#define SI5341_DEVICE_REV 0x0005
+#define SI5341_STATUS 0x000C
+#define SI5341_SOFT_RST 0x001C
+
+/* Input dividers (48-bit) */
+#define SI5341_IN_PDIV(x) (0x0208 + ((x) * 10))
+#define SI5341_IN_PSET(x) (0x020E + ((x) * 10))
+
+/* PLL configuration */
+#define SI5341_PLL_M_NUM 0x0235
+#define SI5341_PLL_M_DEN 0x023B
+
+/* Output configuration */
+#define SI5341_OUT_CONFIG(output) \
+ ((output)->data->reg_output_offset[(output)->index])
+#define SI5341_OUT_FORMAT(output) (SI5341_OUT_CONFIG(output) + 1)
+#define SI5341_OUT_CM(output) (SI5341_OUT_CONFIG(output) + 2)
+#define SI5341_OUT_MUX_SEL(output) (SI5341_OUT_CONFIG(output) + 3)
+#define SI5341_OUT_R_REG(output) \
+ ((output)->data->reg_rdiv_offset[(output)->index])
+
+/* Synthesize N divider */
+#define SI5341_SYNTH_N_NUM(x) (0x0302 + ((x) * 11))
+#define SI5341_SYNTH_N_DEN(x) (0x0308 + ((x) * 11))
+#define SI5341_SYNTH_N_UPD(x) (0x030C + ((x) * 11))
+
+/* Synthesizer output enable, phase bypass, power mode */
+#define SI5341_SYNTH_N_CLK_TO_OUTX_EN 0x0A03
+#define SI5341_SYNTH_N_PIBYP 0x0A04
+#define SI5341_SYNTH_N_PDNB 0x0A05
+#define SI5341_SYNTH_N_CLK_DIS 0x0B4A
+
+#define SI5341_REGISTER_MAX 0xBFF
+
+/* SI5341_OUT_CONFIG bits */
+#define SI5341_OUT_CFG_PDN BIT(0)
+#define SI5341_OUT_CFG_OE BIT(1)
+#define SI5341_OUT_CFG_RDIV_FORCE2 BIT(2)
+
+/* Static configuration (to be moved to firmware) */
+struct si5341_reg_default {
+ u16 address;
+ u8 value;
+};
+
+/* Output configuration registers 0..9 are not quite logically organized */
+static const u16 si5341_reg_output_offset[] = {
+ 0x0108,
+ 0x010D,
+ 0x0112,
+ 0x0117,
+ 0x011C,
+ 0x0121,
+ 0x0126,
+ 0x012B,
+ 0x0130,
+ 0x013A,
+};
+
+static const u16 si5340_reg_output_offset[] = {
+ 0x0112,
+ 0x0117,
+ 0x0126,
+ 0x012B,
+};
+
+/* The location of the R divider registers */
+static const u16 si5341_reg_rdiv_offset[] = {
+ 0x024A,
+ 0x024D,
+ 0x0250,
+ 0x0253,
+ 0x0256,
+ 0x0259,
+ 0x025C,
+ 0x025F,
+ 0x0262,
+ 0x0268,
+};
+static const u16 si5340_reg_rdiv_offset[] = {
+ 0x0250,
+ 0x0253,
+ 0x025C,
+ 0x025F,
+};
+
+/*
+ * Programming sequence from ClockBuilder, settings to initialize the system
+ * using only the XTAL input, without pre-divider.
+ * This also contains settings that aren't mentioned anywhere in the datasheet.
+ * The "known" settings like synth and output configuration are done later.
+ */
+static const struct si5341_reg_default si5341_reg_defaults[] = {
+ { 0x0017, 0x3A }, /* INT mask (disable interrupts) */
+ { 0x0018, 0xFF }, /* INT mask */
+ { 0x0021, 0x0F }, /* Select XTAL as input */
+ { 0x0022, 0x00 }, /* Not in datasheet */
+ { 0x002B, 0x02 }, /* SPI config */
+ { 0x002C, 0x20 }, /* LOS enable for XTAL */
+ { 0x002D, 0x00 }, /* LOS timing */
+ { 0x002E, 0x00 },
+ { 0x002F, 0x00 },
+ { 0x0030, 0x00 },
+ { 0x0031, 0x00 },
+ { 0x0032, 0x00 },
+ { 0x0033, 0x00 },
+ { 0x0034, 0x00 },
+ { 0x0035, 0x00 },
+ { 0x0036, 0x00 },
+ { 0x0037, 0x00 },
+ { 0x0038, 0x00 }, /* LOS setting (thresholds) */
+ { 0x0039, 0x00 },
+ { 0x003A, 0x00 },
+ { 0x003B, 0x00 },
+ { 0x003C, 0x00 },
+ { 0x003D, 0x00 }, /* LOS setting (thresholds) end */
+ { 0x0041, 0x00 }, /* LOS0_DIV_SEL */
+ { 0x0042, 0x00 }, /* LOS1_DIV_SEL */
+ { 0x0043, 0x00 }, /* LOS2_DIV_SEL */
+ { 0x0044, 0x00 }, /* LOS3_DIV_SEL */
+ { 0x009E, 0x00 }, /* Not in datasheet */
+ { 0x0102, 0x01 }, /* Enable outputs */
+ { 0x013F, 0x00 }, /* Not in datasheet */
+ { 0x0140, 0x00 }, /* Not in datasheet */
+ { 0x0141, 0x40 }, /* OUT LOS */
+ { 0x0202, 0x00 }, /* XAXB_FREQ_OFFSET (=0)*/
+ { 0x0203, 0x00 },
+ { 0x0204, 0x00 },
+ { 0x0205, 0x00 },
+ { 0x0206, 0x00 }, /* PXAXB (2^x) */
+ { 0x0208, 0x00 }, /* Px divider setting (usually 0) */
+ { 0x0209, 0x00 },
+ { 0x020A, 0x00 },
+ { 0x020B, 0x00 },
+ { 0x020C, 0x00 },
+ { 0x020D, 0x00 },
+ { 0x020E, 0x00 },
+ { 0x020F, 0x00 },
+ { 0x0210, 0x00 },
+ { 0x0211, 0x00 },
+ { 0x0212, 0x00 },
+ { 0x0213, 0x00 },
+ { 0x0214, 0x00 },
+ { 0x0215, 0x00 },
+ { 0x0216, 0x00 },
+ { 0x0217, 0x00 },
+ { 0x0218, 0x00 },
+ { 0x0219, 0x00 },
+ { 0x021A, 0x00 },
+ { 0x021B, 0x00 },
+ { 0x021C, 0x00 },
+ { 0x021D, 0x00 },
+ { 0x021E, 0x00 },
+ { 0x021F, 0x00 },
+ { 0x0220, 0x00 },
+ { 0x0221, 0x00 },
+ { 0x0222, 0x00 },
+ { 0x0223, 0x00 },
+ { 0x0224, 0x00 },
+ { 0x0225, 0x00 },
+ { 0x0226, 0x00 },
+ { 0x0227, 0x00 },
+ { 0x0228, 0x00 },
+ { 0x0229, 0x00 },
+ { 0x022A, 0x00 },
+ { 0x022B, 0x00 },
+ { 0x022C, 0x00 },
+ { 0x022D, 0x00 },
+ { 0x022E, 0x00 },
+ { 0x022F, 0x00 }, /* Px divider setting (usually 0) end */
+ { 0x026B, 0x00 }, /* DESIGN_ID (ASCII string) */
+ { 0x026C, 0x00 },
+ { 0x026D, 0x00 },
+ { 0x026E, 0x00 },
+ { 0x026F, 0x00 },
+ { 0x0270, 0x00 },
+ { 0x0271, 0x00 },
+ { 0x0272, 0x00 }, /* DESIGN_ID (ASCII string) end */
+ { 0x0339, 0x1F }, /* N_FSTEP_MSK */
+ { 0x033B, 0x00 }, /* Nx_FSTEPW (Frequency step) */
+ { 0x033C, 0x00 },
+ { 0x033D, 0x00 },
+ { 0x033E, 0x00 },
+ { 0x033F, 0x00 },
+ { 0x0340, 0x00 },
+ { 0x0341, 0x00 },
+ { 0x0342, 0x00 },
+ { 0x0343, 0x00 },
+ { 0x0344, 0x00 },
+ { 0x0345, 0x00 },
+ { 0x0346, 0x00 },
+ { 0x0347, 0x00 },
+ { 0x0348, 0x00 },
+ { 0x0349, 0x00 },
+ { 0x034A, 0x00 },
+ { 0x034B, 0x00 },
+ { 0x034C, 0x00 },
+ { 0x034D, 0x00 },
+ { 0x034E, 0x00 },
+ { 0x034F, 0x00 },
+ { 0x0350, 0x00 },
+ { 0x0351, 0x00 },
+ { 0x0352, 0x00 },
+ { 0x0353, 0x00 },
+ { 0x0354, 0x00 },
+ { 0x0355, 0x00 },
+ { 0x0356, 0x00 },
+ { 0x0357, 0x00 },
+ { 0x0358, 0x00 }, /* Nx_FSTEPW (Frequency step) end */
+ { 0x0359, 0x00 }, /* Nx_DELAY */
+ { 0x035A, 0x00 },
+ { 0x035B, 0x00 },
+ { 0x035C, 0x00 },
+ { 0x035D, 0x00 },
+ { 0x035E, 0x00 },
+ { 0x035F, 0x00 },
+ { 0x0360, 0x00 },
+ { 0x0361, 0x00 },
+ { 0x0362, 0x00 }, /* Nx_DELAY end */
+ { 0x0802, 0x00 }, /* Not in datasheet */
+ { 0x0803, 0x00 }, /* Not in datasheet */
+ { 0x0804, 0x00 }, /* Not in datasheet */
+ { 0x090E, 0x02 }, /* XAXB_EXTCLK_EN=0 XAXB_PDNB=1 (use XTAL) */
+ { 0x091C, 0x04 }, /* ZDM_EN=4 (Normal mode) */
+ { 0x0943, 0x00 }, /* IO_VDD_SEL=0 (0=1v8, use 1=3v3) */
+ { 0x0949, 0x00 }, /* IN_EN (disable input clocks) */
+ { 0x094A, 0x00 }, /* INx_TO_PFD_EN (disabled) */
+ { 0x0A02, 0x00 }, /* Not in datasheet */
+ { 0x0B44, 0x0F }, /* PDIV_ENB (datasheet does not mention what it is) */
+};
+
+/* Read and interpret a 44-bit followed by a 32-bit value in the regmap */
+static int si5341_decode_44_32(struct regmap *regmap, unsigned int reg,
+ u64 *val1, u32 *val2)
+{
+ int err;
+ u8 r[10];
+
+ err = regmap_bulk_read(regmap, reg, r, 10);
+ if (err < 0)
+ return err;
+
+ *val1 = ((u64)((r[5] & 0x0f) << 8 | r[4]) << 32) |
+ (get_unaligned_le32(r));
+ *val2 = get_unaligned_le32(&r[6]);
+
+ return 0;
+}
+
+static int si5341_encode_44_32(struct regmap *regmap, unsigned int reg,
+ u64 n_num, u32 n_den)
+{
+ u8 r[10];
+
+ /* Shift left as far as possible without overflowing */
+ while (!(n_num & BIT_ULL(43)) && !(n_den & BIT(31))) {
+ n_num <<= 1;
+ n_den <<= 1;
+ }
+
+ /* 44 bits (6 bytes) numerator */
+ put_unaligned_le32(n_num, r);
+ r[4] = (n_num >> 32) & 0xff;
+ r[5] = (n_num >> 40) & 0x0f;
+ /* 32 bits denominator */
+ put_unaligned_le32(n_den, &r[6]);
+
+ /* Program the fraction */
+ return regmap_bulk_write(regmap, reg, r, sizeof(r));
+}
+
+/* VCO, we assume it runs at a constant frequency */
+static unsigned long si5341_clk_recalc_rate(struct clk_hw *hw,
+ unsigned long parent_rate)
+{
+ struct clk_si5341 *data = to_clk_si5341(hw);
+ int err;
+ u64 res;
+ u64 m_num;
+ u32 m_den;
+ unsigned int shift;
+
+ /* Assume that PDIV is not being used, just read the PLL setting */
+ err = si5341_decode_44_32(data->regmap, SI5341_PLL_M_NUM,
+ &m_num, &m_den);
+ if (err < 0)
+ return 0;
+
+ if (!m_num || !m_den)
+ return 0;
+
+ /*
+ * Though m_num is 64-bit, only the upper bits are actually used. While
+ * calculating m_num and m_den, they are shifted as far as possible to
+ * the left. To avoid 96-bit division here, we just shift them back so
+ * we can do with just 64 bits.
+ */
+ shift = 0;
+ res = m_num;
+ while (res & 0xffff00000000ULL) {
+ ++shift;
+ res >>= 1;
+ }
+ res *= parent_rate;
+ do_div(res, (m_den >> shift));
+
+ /* We cannot return the actual frequency in 32 bit, store it locally */
+ data->freq_vco = res;
+
+ /* Report kHz since the value is out of range */
+ do_div(res, 1000);
+
+ return (unsigned long)res;
+}
+
+static const struct clk_ops si5341_clk_ops = {
+ .recalc_rate = si5341_clk_recalc_rate,
+};
+
+/* Synthesizers, there are 5 synthesizers that connect to any of the outputs */
+
+/* The synthesizer is on if all power and enable bits are set */
+static int si5341_synth_clk_is_on(struct clk_hw *hw)
+{
+ struct clk_si5341_synth *synth = to_clk_si5341_synth(hw);
+ int err;
+ u32 val;
+ u8 index = synth->index;
+
+ err = regmap_read(synth->data->regmap,
+ SI5341_SYNTH_N_CLK_TO_OUTX_EN, &val);
+ if (err < 0)
+ return 0;
+
+ if (!(val & BIT(index)))
+ return 0;
+
+ err = regmap_read(synth->data->regmap, SI5341_SYNTH_N_PDNB, &val);
+ if (err < 0)
+ return 0;
+
+ if (!(val & BIT(index)))
+ return 0;
+
+ /* This bit must be 0 for the synthesizer to receive clock input */
+ err = regmap_read(synth->data->regmap, SI5341_SYNTH_N_CLK_DIS, &val);
+ if (err < 0)
+ return 0;
+
+ return !(val & BIT(index));
+}
+
+static void si5341_synth_clk_unprepare(struct clk_hw *hw)
+{
+ struct clk_si5341_synth *synth = to_clk_si5341_synth(hw);
+ u8 index = synth->index; /* In range 0..5 */
+ u8 mask = BIT(index);
+
+ /* Disable output */
+ regmap_update_bits(synth->data->regmap,
+ SI5341_SYNTH_N_CLK_TO_OUTX_EN, mask, 0);
+ /* Power down */
+ regmap_update_bits(synth->data->regmap,
+ SI5341_SYNTH_N_PDNB, mask, 0);
+ /* Disable clock input to synth (set to 1 to disable) */
+ regmap_update_bits(synth->data->regmap,
+ SI5341_SYNTH_N_CLK_DIS, mask, mask);
+}
+
+static int si5341_synth_clk_prepare(struct clk_hw *hw)
+{
+ struct clk_si5341_synth *synth = to_clk_si5341_synth(hw);
+ int err;
+ u8 index = synth->index;
+ u8 mask = BIT(index);
+
+ /* Power up */
+ err = regmap_update_bits(synth->data->regmap,
+ SI5341_SYNTH_N_PDNB, mask, mask);
+ if (err < 0)
+ return err;
+
+ /* Enable clock input to synth (set bit to 0 to enable) */
+ err = regmap_update_bits(synth->data->regmap,
+ SI5341_SYNTH_N_CLK_DIS, mask, 0);
+ if (err < 0)
+ return err;
+
+ /* Enable output */
+ return regmap_update_bits(synth->data->regmap,
+ SI5341_SYNTH_N_CLK_TO_OUTX_EN, mask, mask);
+}
+
+/* Synth clock frequency: Fvco * n_den / n_den, with Fvco in 13500-14256 MHz */
+static unsigned long si5341_synth_clk_recalc_rate(struct clk_hw *hw,
+ unsigned long parent_rate)
+{
+ struct clk_si5341_synth *synth = to_clk_si5341_synth(hw);
+ u64 f;
+ u64 n_num;
+ u32 n_den;
+ int err;
+
+ err = si5341_decode_44_32(synth->data->regmap,
+ SI5341_SYNTH_N_NUM(synth->index), &n_num, &n_den);
+ if (err < 0)
+ return err;
+
+ /*
+ * n_num and n_den are shifted left as much as possible, so to prevent
+ * overflow in 64-bit math, we shift n_den 4 bits to the right
+ */
+ f = synth->data->freq_vco;
+ f *= n_den >> 4;
+
+ /* Now we need to to 64-bit division: f/n_num */
+ /* And compensate for the 4 bits we dropped */
+ f = div64_u64(f, (n_num >> 4));
+
+ return f;
+}
+
+static long si5341_synth_clk_round_rate(struct clk_hw *hw, unsigned long rate,
+ unsigned long *parent_rate)
+{
+ struct clk_si5341_synth *synth = to_clk_si5341_synth(hw);
+ u64 f;
+
+ /* The synthesizer accuracy is such that anything in range will work */
+ f = synth->data->freq_vco;
+ do_div(f, SI5341_SYNTH_N_MAX);
+ if (rate < f)
+ return f;
+
+ f = synth->data->freq_vco;
+ do_div(f, SI5341_SYNTH_N_MIN);
+ if (rate > f)
+ return f;
+
+ return rate;
+}
+
+static int si5341_synth_program(struct clk_si5341_synth *synth,
+ u64 n_num, u32 n_den, bool is_integer)
+{
+ int err;
+ u8 index = synth->index;
+
+ err = si5341_encode_44_32(synth->data->regmap,
+ SI5341_SYNTH_N_NUM(index), n_num, n_den);
+
+ err = regmap_update_bits(synth->data->regmap,
+ SI5341_SYNTH_N_PIBYP, BIT(index), is_integer ? BIT(index) : 0);
+ if (err < 0)
+ return err;
+
+ return regmap_write(synth->data->regmap,
+ SI5341_SYNTH_N_UPD(index), 0x01);
+}
+
+
+static int si5341_synth_clk_set_rate(struct clk_hw *hw, unsigned long rate,
+ unsigned long parent_rate)
+{
+ struct clk_si5341_synth *synth = to_clk_si5341_synth(hw);
+ u64 n_num;
+ u32 n_den;
+ u32 r;
+ u32 g;
+ bool is_integer;
+
+ n_num = synth->data->freq_vco;
+ n_den = rate;
+
+ /* see if there's an integer solution */
+ r = do_div(n_num, rate);
+ is_integer = (r == 0);
+ if (is_integer) {
+ /* Integer divider equal to n_num */
+ n_den = 1;
+ } else {
+ /* Calculate a fractional solution */
+ g = gcd(r, rate);
+ n_den = rate / g;
+ n_num *= n_den;
+ n_num += r / g;
+ }
+
+ dev_dbg(&synth->data->i2c_client->dev,
+ "%s(%u): n=0x%llx d=0x%x %s\n", __func__,
+ synth->index, n_num, n_den,
+ is_integer ? "int" : "frac");
+
+ return si5341_synth_program(synth, n_num, n_den, is_integer);
+}
+
+static const struct clk_ops si5341_synth_clk_ops = {
+ .is_prepared = si5341_synth_clk_is_on,
+ .prepare = si5341_synth_clk_prepare,
+ .unprepare = si5341_synth_clk_unprepare,
+ .recalc_rate = si5341_synth_clk_recalc_rate,
+ .round_rate = si5341_synth_clk_round_rate,
+ .set_rate = si5341_synth_clk_set_rate,
+};
+
+static int si5341_output_clk_is_on(struct clk_hw *hw)
+{
+ struct clk_si5341_output *output = to_clk_si5341_output(hw);
+ int err;
+ u32 val;
+
+ err = regmap_read(output->data->regmap,
+ SI5341_OUT_CONFIG(output), &val);
+ if (err < 0)
+ return err;
+
+ /* Bit 0=PDN, 1=OE so only a value of 0x2 enables the output */
+ return (val & 0x03) == SI5341_OUT_CFG_OE;
+}
+
+/* Disables and then powers down the output */
+static void si5341_output_clk_unprepare(struct clk_hw *hw)
+{
+ struct clk_si5341_output *output = to_clk_si5341_output(hw);
+
+ regmap_update_bits(output->data->regmap,
+ SI5341_OUT_CONFIG(output),
+ SI5341_OUT_CFG_OE, 0);
+ regmap_update_bits(output->data->regmap,
+ SI5341_OUT_CONFIG(output),
+ SI5341_OUT_CFG_PDN, SI5341_OUT_CFG_PDN);
+}
+
+/* Powers up and then enables the output */
+static int si5341_output_clk_prepare(struct clk_hw *hw)
+{
+ struct clk_si5341_output *output = to_clk_si5341_output(hw);
+ int err;
+
+ err = regmap_update_bits(output->data->regmap,
+ SI5341_OUT_CONFIG(output),
+ SI5341_OUT_CFG_PDN, 0);
+ if (err < 0)
+ return err;
+
+ return regmap_update_bits(output->data->regmap,
+ SI5341_OUT_CONFIG(output),
+ SI5341_OUT_CFG_OE, SI5341_OUT_CFG_OE);
+}
+
+static unsigned long si5341_output_clk_recalc_rate(struct clk_hw *hw,
+ unsigned long parent_rate)
+{
+ struct clk_si5341_output *output = to_clk_si5341_output(hw);
+ int err;
+ u32 val;
+ u32 r_divider;
+ u8 r[3];
+
+ err = regmap_bulk_read(output->data->regmap,
+ SI5341_OUT_R_REG(output), r, 3);
+ if (err < 0)
+ return err;
+
+ /* Calculate value as 24-bit integer*/
+ r_divider = r[2] << 16 | r[1] << 8 | r[0];
+
+ /* If Rx_REG is zero, the divider is disabled, so return a "0" rate */
+ if (!r_divider)
+ return 0;
+
+ /* Divider is 2*(Rx_REG+1) */
+ r_divider += 1;
+ r_divider <<= 1;
+
+ err = regmap_read(output->data->regmap,
+ SI5341_OUT_CONFIG(output), &val);
+ if (err < 0)
+ return err;
+
+ if (val & SI5341_OUT_CFG_RDIV_FORCE2)
+ r_divider = 2;
+
+ return parent_rate / r_divider;
+}
+
+static long si5341_output_clk_round_rate(struct clk_hw *hw, unsigned long rate,
+ unsigned long *parent_rate)
+{
+ unsigned long r;
+
+ r = *parent_rate >> 1;
+
+ /* If rate is an even divisor, no changes to parent required */
+ if (r && !(r % rate))
+ return (long)rate;
+
+ if (clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT) {
+ if (rate > 200000000) {
+ /* minimum r-divider is 2 */
+ r = 2;
+ } else {
+ /* Take a parent frequency near 400 MHz */
+ r = (400000000u / rate) & ~1;
+ }
+ *parent_rate = r * rate;
+ } else {
+ /* We cannot change our parent's rate, report what we can do */
+ r /= rate;
+ rate = *parent_rate / (r << 1);
+ }
+
+ return rate;
+}
+
+static int si5341_output_clk_set_rate(struct clk_hw *hw, unsigned long rate,
+ unsigned long parent_rate)
+{
+ struct clk_si5341_output *output = to_clk_si5341_output(hw);
+ /* Frequency divider is (r_div + 1) * 2 */
+ u32 r_div = (parent_rate / rate) >> 1;
+ int err;
+ u8 r[3];
+
+ if (r_div <= 1)
+ r_div = 0;
+ else if (r_div >= BIT(24))
+ r_div = BIT(24) - 1;
+ else
+ --r_div;
+
+ /* For a value of "2", we set the "OUT0_RDIV_FORCE2" bit */
+ err = regmap_update_bits(output->data->regmap,
+ SI5341_OUT_CONFIG(output),
+ SI5341_OUT_CFG_RDIV_FORCE2,
+ (r_div == 0) ? SI5341_OUT_CFG_RDIV_FORCE2 : 0);
+ if (err < 0)
+ return err;
+
+ /* Always write Rx_REG, because a zero value disables the divider */
+ r[0] = r_div ? (r_div & 0xff) : 1;
+ r[1] = (r_div >> 8) & 0xff;
+ r[2] = (r_div >> 16) & 0xff;
+ err = regmap_bulk_write(output->data->regmap,
+ SI5341_OUT_R_REG(output), r, 3);
+
+ return 0;
+}
+
+static int si5341_output_reparent(struct clk_si5341_output *output, u8 index)
+{
+ return regmap_update_bits(output->data->regmap,
+ SI5341_OUT_MUX_SEL(output), 0x07, index);
+}
+
+static int si5341_output_set_parent(struct clk_hw *hw, u8 index)
+{
+ struct clk_si5341_output *output = to_clk_si5341_output(hw);
+
+ if (index >= output->data->num_synth)
+ return -EINVAL;
+
+ return si5341_output_reparent(output, index);
+}
+
+static u8 si5341_output_get_parent(struct clk_hw *hw)
+{
+ struct clk_si5341_output *output = to_clk_si5341_output(hw);
+ int err;
+ u32 val;
+
+ err = regmap_read(output->data->regmap,
+ SI5341_OUT_MUX_SEL(output), &val);
+
+ return val & 0x7;
+}
+
+static const struct clk_ops si5341_output_clk_ops = {
+ .is_prepared = si5341_output_clk_is_on,
+ .prepare = si5341_output_clk_prepare,
+ .unprepare = si5341_output_clk_unprepare,
+ .recalc_rate = si5341_output_clk_recalc_rate,
+ .round_rate = si5341_output_clk_round_rate,
+ .set_rate = si5341_output_clk_set_rate,
+ .set_parent = si5341_output_set_parent,
+ .get_parent = si5341_output_get_parent,
+};
+
+/*
+ * The chip can be bought in a pre-programmed version, or one can program the
+ * NVM in the chip to boot up in a preset mode. This routine tries to determine
+ * if that's the case, or if we need to reset and program everything from
+ * scratch. Returns negative error, or true/false.
+ */
+static int si5341_is_programmed_already(struct clk_si5341 *data)
+{
+ int err;
+ u8 r[4];
+
+ /* Read the PLL divider value, it must have a non-zero value */
+ err = regmap_bulk_read(data->regmap, SI5341_PLL_M_DEN,
+ r, ARRAY_SIZE(r));
+ if (err < 0)
+ return err;
+
+ return !!get_unaligned_le32(r);
+}
+
+static struct clk_hw *
+of_clk_si5341_get(struct of_phandle_args *clkspec, void *_data)
+{
+ struct clk_si5341 *data = _data;
+ unsigned int idx = clkspec->args[1];
+ unsigned int group = clkspec->args[0];
+
+ switch (group) {
+ case 0:
+ if (idx >= data->num_outputs) {
+ dev_err(&data->i2c_client->dev,
+ "invalid output index %u\n", idx);
+ return ERR_PTR(-EINVAL);
+ }
+ return &data->clk[idx].hw;
+ case 1:
+ if (idx >= data->num_synth) {
+ dev_err(&data->i2c_client->dev,
+ "invalid synthesizer index %u\n", idx);
+ return ERR_PTR(-EINVAL);
+ }
+ return &data->synth[idx].hw;
+ case 2:
+ if (idx > 0) {
+ dev_err(&data->i2c_client->dev,
+ "invalid PLL index %u\n", idx);
+ return ERR_PTR(-EINVAL);
+ }
+ return &data->hw;
+ default:
+ dev_err(&data->i2c_client->dev, "invalid group %u\n", group);
+ return ERR_PTR(-EINVAL);
+ }
+}
+
+static int si5341_probe_chip_id(struct clk_si5341 *data)
+{
+ int err;
+ u8 reg[4];
+ u16 model;
+
+ err = regmap_bulk_read(data->regmap, SI5341_PN_BASE, reg,
+ ARRAY_SIZE(reg));
+ if (err < 0) {
+ dev_err(&data->i2c_client->dev, "Failed to read chip ID\n");
+ return err;
+ }
+
+ model = get_unaligned_le16(reg);
+
+ dev_info(&data->i2c_client->dev, "Chip: %x Grade: %u Rev: %u\n",
+ model, reg[2], reg[3]);
+
+ switch (model) {
+ case 0x5340:
+ data->num_outputs = SI5340_MAX_NUM_OUTPUTS;
+ data->num_synth = SI5340_NUM_SYNTH;
+ data->reg_output_offset = si5340_reg_output_offset;
+ data->reg_rdiv_offset = si5340_reg_rdiv_offset;
+ break;
+ case 0x5341:
+ data->num_outputs = SI5341_MAX_NUM_OUTPUTS;
+ data->num_synth = SI5341_NUM_SYNTH;
+ data->reg_output_offset = si5341_reg_output_offset;
+ data->reg_rdiv_offset = si5341_reg_rdiv_offset;
+ break;
+ default:
+ dev_err(&data->i2c_client->dev, "Model '%x' not supported\n",
+ model);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/* Read active settings into the regmap cache for later reference */
+static int si5341_read_settings(struct clk_si5341 *data)
+{
+ int err;
+ u8 i;
+ u8 r[10];
+
+ err = regmap_bulk_read(data->regmap, SI5341_PLL_M_NUM, r, 10);
+ if (err < 0)
+ return err;
+
+ err = regmap_bulk_read(data->regmap,
+ SI5341_SYNTH_N_CLK_TO_OUTX_EN, r, 3);
+ if (err < 0)
+ return err;
+
+ err = regmap_bulk_read(data->regmap,
+ SI5341_SYNTH_N_CLK_DIS, r, 1);
+ if (err < 0)
+ return err;
+
+ for (i = 0; i < data->num_synth; ++i) {
+ err = regmap_bulk_read(data->regmap,
+ SI5341_SYNTH_N_NUM(i), r, 10);
+ if (err < 0)
+ return err;
+ }
+
+ for (i = 0; i < data->num_outputs; ++i) {
+ err = regmap_bulk_read(data->regmap,
+ data->reg_output_offset[i], r, 4);
+ if (err < 0)
+ return err;
+
+ err = regmap_bulk_read(data->regmap,
+ data->reg_rdiv_offset[i], r, 3);
+ if (err < 0)
+ return err;
+ }
+
+ return 0;
+}
+
+static int si5341_write_multiple(struct clk_si5341 *data,
+ const struct si5341_reg_default *values, unsigned int num_values)
+{
+ unsigned int i;
+ int res;
+
+ for (i = 0; i < num_values; ++i) {
+ res = regmap_write(data->regmap,
+ values[i].address, values[i].value);
+ if (res < 0) {
+ dev_err(&data->i2c_client->dev,
+ "Failed to write %#x:%#x\n",
+ values[i].address, values[i].value);
+ return res;
+ }
+ }
+
+ return 0;
+}
+
+static const struct si5341_reg_default si5341_preamble[] = {
+ { 0x0B25, 0x00 },
+ { 0x0502, 0x01 },
+ { 0x0505, 0x03 },
+ { 0x0957, 0x1F },
+ { 0x0B4E, 0x1A },
+};
+
+static int si5341_send_preamble(struct clk_si5341 *data)
+{
+ int res;
+ u32 revision;
+
+ /* For revision 2 and up, the values are slightly different */
+ res = regmap_read(data->regmap, SI5341_DEVICE_REV, &revision);
+ if (res < 0)
+ return res;
+
+ /* Write "preamble" as specified by datasheet */
+ res = regmap_write(data->regmap, 0xB24, revision < 2 ? 0xD8 : 0xC0);
+ if (res < 0)
+ return res;
+ res = si5341_write_multiple(data,
+ si5341_preamble, ARRAY_SIZE(si5341_preamble));
+ if (res < 0)
+ return res;
+
+ /* Datasheet specifies a 300ms wait after sending the preamble */
+ msleep(300);
+
+ return 0;
+}
+
+/* Perform a soft reset and write post-amble */
+static int si5341_finalize_defaults(struct clk_si5341 *data)
+{
+ int res;
+ u32 revision;
+
+ res = regmap_read(data->regmap, SI5341_DEVICE_REV, &revision);
+ if (res < 0)
+ return res;
+
+ dev_dbg(&data->i2c_client->dev, "%s rev=%u\n", __func__, revision);
+
+ res = regmap_write(data->regmap, SI5341_SOFT_RST, 0x01);
+ if (res < 0)
+ return res;
+
+ /* Datasheet does not explain these nameless registers */
+ res = regmap_write(data->regmap, 0xB24, revision < 2 ? 0xDB : 0xC3);
+ if (res < 0)
+ return res;
+ res = regmap_write(data->regmap, 0x0B25, 0x02);
+ if (res < 0)
+ return res;
+
+ return 0;
+}
+
+
+static const struct regmap_range si5341_regmap_volatile_range[] = {
+ regmap_reg_range(0x000C, 0x0012), /* Status */
+ regmap_reg_range(0x001C, 0x001E), /* reset, finc/fdec */
+ regmap_reg_range(0x00E2, 0x00FE), /* NVM, interrupts, device ready */
+ /* Update bits for synth config */
+ regmap_reg_range(SI5341_SYNTH_N_UPD(0), SI5341_SYNTH_N_UPD(0)),
+ regmap_reg_range(SI5341_SYNTH_N_UPD(1), SI5341_SYNTH_N_UPD(1)),
+ regmap_reg_range(SI5341_SYNTH_N_UPD(2), SI5341_SYNTH_N_UPD(2)),
+ regmap_reg_range(SI5341_SYNTH_N_UPD(3), SI5341_SYNTH_N_UPD(3)),
+ regmap_reg_range(SI5341_SYNTH_N_UPD(4), SI5341_SYNTH_N_UPD(4)),
+};
+
+static const struct regmap_access_table si5341_regmap_volatile = {
+ .yes_ranges = si5341_regmap_volatile_range,
+ .n_yes_ranges = ARRAY_SIZE(si5341_regmap_volatile_range),
+};
+
+/* Pages 0, 1, 2, 3, 9, A, B are valid, so there are 12 pages */
+static const struct regmap_range_cfg si5341_regmap_ranges[] = {
+ {
+ .range_min = 0,
+ .range_max = SI5341_REGISTER_MAX,
+ .selector_reg = SI5341_PAGE,
+ .selector_mask = 0xff,
+ .selector_shift = 0,
+ .window_start = 0,
+ .window_len = 256,
+ },
+};
+
+static const struct regmap_config si5341_regmap_config = {
+ .reg_bits = 8,
+ .val_bits = 8,
+ .cache_type = REGCACHE_RBTREE,
+ .ranges = si5341_regmap_ranges,
+ .num_ranges = ARRAY_SIZE(si5341_regmap_ranges),
+ .max_register = SI5341_REGISTER_MAX,
+ .volatile_table = &si5341_regmap_volatile,
+};
+
+static int si5341_dt_parse_dt(struct i2c_client *client,
+ struct clk_si5341_output_config *config)
+{
+ struct device_node *child;
+ struct device_node *np = client->dev.of_node;
+ u32 num;
+ u32 val;
+
+ memset(config, 0, sizeof(struct clk_si5341_output_config) *
+ SI5341_MAX_NUM_OUTPUTS);
+
+ for_each_child_of_node(np, child) {
+ if (of_property_read_u32(child, "reg", &num)) {
+ dev_err(&client->dev, "missing reg property of %s\n",
+ child->name);
+ goto put_child;
+ }
+
+ if (num >= SI5341_MAX_NUM_OUTPUTS) {
+ dev_err(&client->dev, "invalid clkout %d\n", num);
+ goto put_child;
+ }
+
+ if (!of_property_read_u32(child, "silabs,format", &val)) {
+ /* Set cm and ampl conservatively to 3v3 settings */
+ switch (val) {
+ case 1: /* normal differential */
+ config[num].out_cm_ampl_bits = 0x33;
+ break;
+ case 2: /* low-power differential */
+ config[num].out_cm_ampl_bits = 0x13;
+ break;
+ case 4: /* LVCMOS */
+ config[num].out_cm_ampl_bits = 0x33;
+ /* Set SI recommended impedance for LVCMOS */
+ config[num].out_format_drv_bits |= 0xc0;
+ break;
+ default:
+ dev_err(&client->dev,
+ "invalid silabs,format %u for %u\n",
+ val, num);
+ goto put_child;
+ }
+ config[num].out_format_drv_bits &= ~0x07;
+ config[num].out_format_drv_bits |= val & 0x07;
+ /* Always enable the SYNC feature */
+ config[num].out_format_drv_bits |= 0x08;
+ }
+
+ if (!of_property_read_u32(child, "silabs,common-mode", &val)) {
+ if (val > 0xf) {
+ dev_err(&client->dev,
+ "invalid silabs,common-mode %u\n",
+ val);
+ goto put_child;
+ }
+ config[num].out_cm_ampl_bits &= 0xf0;
+ config[num].out_cm_ampl_bits |= val & 0x0f;
+ }
+
+ if (!of_property_read_u32(child, "silabs,amplitude", &val)) {
+ if (val > 0xf) {
+ dev_err(&client->dev,
+ "invalid silabs,amplitude %u\n",
+ val);
+ goto put_child;
+ }
+ config[num].out_cm_ampl_bits &= 0x0f;
+ config[num].out_cm_ampl_bits |= (val << 4) & 0xf0;
+ }
+
+ if (of_property_read_bool(child, "silabs,disable-high"))
+ config[num].out_format_drv_bits |= 0x10;
+
+ config[num].synth_master =
+ of_property_read_bool(child, "silabs,synth-master");
+
+ config[num].always_on =
+ of_property_read_bool(child, "always-on");
+ }
+
+ return 0;
+
+put_child:
+ of_node_put(child);
+ return -EINVAL;
+}
+
+/*
+ * If not pre-configured, calculate and set the PLL configuration manually.
+ * For low-jitter performance, the PLL should be set such that the synthesizers
+ * only need integer division.
+ * Without any user guidance, we'll set the PLL to 14GHz, which still allows
+ * the chip to generate any frequency on its outputs, but jitter performance
+ * may be sub-optimal.
+ */
+static int si5341_initialize_pll(struct clk_si5341 *data)
+{
+ struct device_node *np = data->i2c_client->dev.of_node;
+ u32 m_num = 0;
+ u32 m_den = 0;
+
+ if (of_property_read_u32(np, "silabs,pll-m-num", &m_num)) {
+ dev_err(&data->i2c_client->dev,
+ "PLL configuration requires silabs,pll-m-num\n");
+ }
+ if (of_property_read_u32(np, "silabs,pll-m-den", &m_den)) {
+ dev_err(&data->i2c_client->dev,
+ "PLL configuration requires silabs,pll-m-den\n");
+ }
+
+ if (!m_num || !m_den) {
+ dev_err(&data->i2c_client->dev,
+ "PLL configuration invalid, assume 14GHz\n");
+ m_den = clk_get_rate(data->pxtal) / 10;
+ m_num = 1400000000;
+ }
+
+ return si5341_encode_44_32(data->regmap,
+ SI5341_PLL_M_NUM, m_num, m_den);
+}
+
+static int si5341_probe(struct i2c_client *client,
+ const struct i2c_device_id *id)
+{
+ struct clk_si5341 *data;
+ struct clk_init_data init;
+ const char *root_clock_name;
+ const char *synth_clock_names[SI5341_NUM_SYNTH];
+ int err;
+ unsigned int i;
+ struct clk_si5341_output_config config[SI5341_MAX_NUM_OUTPUTS];
+ bool initialization_required;
+
+ data = devm_kzalloc(&client->dev, sizeof(*data), GFP_KERNEL);
+ if (!data)
+ return -ENOMEM;
+
+ data->i2c_client = client;
+
+ data->pxtal = devm_clk_get(&client->dev, "xtal");
+ if (IS_ERR(data->pxtal)) {
+ if (PTR_ERR(data->pxtal) == -EPROBE_DEFER)
+ return -EPROBE_DEFER;
+
+ dev_err(&client->dev, "Missing xtal clock input\n");
+ }
+
+ err = si5341_dt_parse_dt(client, config);
+ if (err)
+ return err;
+
+ if (of_property_read_string(client->dev.of_node, "clock-output-names",
+ &init.name))
+ init.name = client->dev.of_node->name;
+ root_clock_name = init.name;
+
+ data->regmap = devm_regmap_init_i2c(client, &si5341_regmap_config);
+ if (IS_ERR(data->regmap))
+ return PTR_ERR(data->regmap);
+
+ i2c_set_clientdata(client, data);
+
+ err = si5341_probe_chip_id(data);
+ if (err < 0)
+ return err;
+
+ /* "Activate" the xtal (usually a fixed clock) */
+ clk_prepare_enable(data->pxtal);
+
+ if (of_property_read_bool(client->dev.of_node, "silabs,reprogram")) {
+ initialization_required = true;
+ } else {
+ err = si5341_is_programmed_already(data);
+ if (err < 0)
+ return err;
+
+ initialization_required = !err;
+ }
+
+ if (initialization_required) {
+ /* Populate the regmap cache in preparation for "cache only" */
+ err = si5341_read_settings(data);
+ if (err < 0)
+ return err;
+
+ err = si5341_send_preamble(data);
+ if (err < 0)
+ return err;
+
+ /*
+ * We intend to send all 'final' register values in a single
+ * transaction. So cache all register writes until we're done
+ * configuring.
+ */
+ regcache_cache_only(data->regmap, true);
+
+ /* Write the configuration pairs from the firmware blob */
+ err = si5341_write_multiple(data, si5341_reg_defaults,
+ ARRAY_SIZE(si5341_reg_defaults));
+ if (err < 0)
+ return err;
+
+ /* PLL configuration is required */
+ err = si5341_initialize_pll(data);
+ if (err < 0)
+ return err;
+ }
+
+ /* Register the PLL */
+ data->pxtal_name = __clk_get_name(data->pxtal);
+ init.parent_names = &data->pxtal_name;
+ init.num_parents = 1; /* For now, only XTAL input supported */
+ init.ops = &si5341_clk_ops;
+ init.flags = 0;
+ data->hw.init = &init;
+
+ err = devm_clk_hw_register(&client->dev, &data->hw);
+ if (err) {
+ dev_err(&client->dev, "clock registration failed\n");
+ return err;
+ }
+
+ init.num_parents = 1;
+ init.parent_names = &root_clock_name;
+ init.ops = &si5341_synth_clk_ops;
+ for (i = 0; i < data->num_synth; ++i) {
+ synth_clock_names[i] = devm_kasprintf(&client->dev, GFP_KERNEL,
+ "%s.N%u", client->dev.of_node->name, i);
+ init.name = synth_clock_names[i];
+ data->synth[i].index = i;
+ data->synth[i].data = data;
+ data->synth[i].hw.init = &init;
+ err = devm_clk_hw_register(&client->dev, &data->synth[i].hw);
+ if (err) {
+ dev_err(&client->dev,
+ "synth N%u registration failed\n", i);
+ }
+ }
+
+ init.num_parents = data->num_synth;
+ init.parent_names = synth_clock_names;
+ init.ops = &si5341_output_clk_ops;
+ for (i = 0; i < data->num_outputs; ++i) {
+ init.name = kasprintf(GFP_KERNEL, "%s.%d",
+ client->dev.of_node->name, i);
+ init.flags = config[i].synth_master ? CLK_SET_RATE_PARENT : 0;
+ data->clk[i].index = i;
+ data->clk[i].data = data;
+ data->clk[i].hw.init = &init;
+ if (config[i].out_format_drv_bits & 0x07) {
+ regmap_write(data->regmap,
+ SI5341_OUT_FORMAT(&data->clk[i]),
+ config[i].out_format_drv_bits);
+ regmap_write(data->regmap,
+ SI5341_OUT_CM(&data->clk[i]),
+ config[i].out_cm_ampl_bits);
+ }
+ err = devm_clk_hw_register(&client->dev, &data->clk[i].hw);
+ kfree(init.name); /* clock framework made a copy of the name */
+ if (err) {
+ dev_err(&client->dev,
+ "output %u registration failed\n", i);
+ return err;
+ }
+ if (config[i].always_on)
+ clk_prepare(data->clk[i].hw.clk);
+ }
+
+ err = of_clk_add_hw_provider(client->dev.of_node, of_clk_si5341_get,
+ data);
+ if (err) {
+ dev_err(&client->dev, "unable to add clk provider\n");
+ return err;
+ }
+
+ if (initialization_required) {
+ /* Synchronize */
+ regcache_cache_only(data->regmap, false);
+ err = regcache_sync(data->regmap);
+ if (err < 0)
+ return err;
+
+ err = si5341_finalize_defaults(data);
+ if (err < 0)
+ return err;
+ }
+
+ /* Free the names, clk framework makes copies */
+ for (i = 0; i < data->num_synth; ++i)
+ devm_kfree(&client->dev, (void *)synth_clock_names[i]);
+
+ return 0;
+}
+
+static const struct i2c_device_id si5341_id[] = {
+ { "si5340", 0 },
+ { "si5341", 1 },
+ { }
+};
+MODULE_DEVICE_TABLE(i2c, si5341_id);
+
+static const struct of_device_id clk_si5341_of_match[] = {
+ { .compatible = "silabs,si5340" },
+ { .compatible = "silabs,si5341" },
+ { }
+};
+MODULE_DEVICE_TABLE(of, clk_si5341_of_match);
+
+static struct i2c_driver si5341_driver = {
+ .driver = {
+ .name = "si5341",
+ .of_match_table = clk_si5341_of_match,
+ },
+ .probe = si5341_probe,
+ .id_table = si5341_id,
+};
+module_i2c_driver(si5341_driver);
+
+MODULE_AUTHOR("Mike Looijmans <mike.looijmans@topic.nl>");
+MODULE_DESCRIPTION("Si5341 driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/clk-si544.c b/drivers/clk/clk-si544.c
index 64e607f3232a..d9ec9086184d 100644
--- a/drivers/clk/clk-si544.c
+++ b/drivers/clk/clk-si544.c
@@ -7,6 +7,7 @@
#include <linux/clk-provider.h>
#include <linux/delay.h>
+#include <linux/math64.h>
#include <linux/module.h>
#include <linux/i2c.h>
#include <linux/regmap.h>
@@ -50,6 +51,11 @@
/* Lowest frequency synthesizeable using only the HS divider */
#define MIN_HSDIV_FREQ (FVCO_MIN / HS_DIV_MAX)
+/* Range and interpretation of the adjustment value */
+#define DELTA_M_MAX 8161512
+#define DELTA_M_FRAC_NUM 19
+#define DELTA_M_FRAC_DEN 20000
+
enum si544_speed_grade {
si544a,
si544b,
@@ -71,12 +77,14 @@ struct clk_si544 {
* @hs_div: 1st divider, 5..2046, must be even when >33
* @ls_div_bits: 2nd divider, as 2^x, range 0..5
* If ls_div_bits is non-zero, hs_div must be even
+ * @delta_m: Frequency shift for small -950..+950 ppm changes, 24 bit
*/
struct clk_si544_muldiv {
u32 fb_div_frac;
u16 fb_div_int;
u16 hs_div;
u8 ls_div_bits;
+ s32 delta_m;
};
/* Enables or disables the output driver */
@@ -134,9 +142,30 @@ static int si544_get_muldiv(struct clk_si544 *data,
settings->fb_div_int = reg[4] | (reg[5] & 0x07) << 8;
settings->fb_div_frac = reg[0] | reg[1] << 8 | reg[2] << 16 |
reg[3] << 24;
+
+ err = regmap_bulk_read(data->regmap, SI544_REG_ADPLL_DELTA_M0, reg, 3);
+ if (err)
+ return err;
+
+ /* Interpret as 24-bit signed number */
+ settings->delta_m = reg[0] << 8 | reg[1] << 16 | reg[2] << 24;
+ settings->delta_m >>= 8;
+
return 0;
}
+static int si544_set_delta_m(struct clk_si544 *data, s32 delta_m)
+{
+ u8 reg[3];
+
+ reg[0] = delta_m;
+ reg[1] = delta_m >> 8;
+ reg[2] = delta_m >> 16;
+
+ return regmap_bulk_write(data->regmap, SI544_REG_ADPLL_DELTA_M0,
+ reg, 3);
+}
+
static int si544_set_muldiv(struct clk_si544 *data,
struct clk_si544_muldiv *settings)
{
@@ -238,11 +267,15 @@ static int si544_calc_muldiv(struct clk_si544_muldiv *settings,
do_div(vco, FXO);
settings->fb_div_frac = vco;
+ /* Reset the frequency adjustment */
+ settings->delta_m = 0;
+
return 0;
}
/* Calculate resulting frequency given the register settings */
-static unsigned long si544_calc_rate(struct clk_si544_muldiv *settings)
+static unsigned long si544_calc_center_rate(
+ const struct clk_si544_muldiv *settings)
{
u32 d = settings->hs_div * BIT(settings->ls_div_bits);
u64 vco;
@@ -261,6 +294,25 @@ static unsigned long si544_calc_rate(struct clk_si544_muldiv *settings)
return vco;
}
+static unsigned long si544_calc_rate(const struct clk_si544_muldiv *settings)
+{
+ unsigned long rate = si544_calc_center_rate(settings);
+ s64 delta = (s64)rate * (DELTA_M_FRAC_NUM * settings->delta_m);
+
+ /*
+ * The clock adjustment is much smaller than 1 Hz, round to the
+ * nearest multiple. Apparently div64_s64 rounds towards zero, hence
+ * check the sign and adjust into the proper direction.
+ */
+ if (settings->delta_m < 0)
+ delta -= ((s64)DELTA_M_MAX * DELTA_M_FRAC_DEN) / 2;
+ else
+ delta += ((s64)DELTA_M_MAX * DELTA_M_FRAC_DEN) / 2;
+ delta = div64_s64(delta, ((s64)DELTA_M_MAX * DELTA_M_FRAC_DEN));
+
+ return rate + delta;
+}
+
static unsigned long si544_recalc_rate(struct clk_hw *hw,
unsigned long parent_rate)
{
@@ -279,33 +331,60 @@ static long si544_round_rate(struct clk_hw *hw, unsigned long rate,
unsigned long *parent_rate)
{
struct clk_si544 *data = to_clk_si544(hw);
- struct clk_si544_muldiv settings;
- int err;
if (!is_valid_frequency(data, rate))
return -EINVAL;
- err = si544_calc_muldiv(&settings, rate);
- if (err)
- return err;
+ /* The accuracy is less than 1 Hz, so any rate is possible */
+ return rate;
+}
- return si544_calc_rate(&settings);
+/* Calculates the maximum "small" change, 950 * rate / 1000000 */
+static unsigned long si544_max_delta(unsigned long rate)
+{
+ u64 num = rate;
+
+ num *= DELTA_M_FRAC_NUM;
+ do_div(num, DELTA_M_FRAC_DEN);
+
+ return num;
+}
+
+static s32 si544_calc_delta(s32 delta, s32 max_delta)
+{
+ s64 n = (s64)delta * DELTA_M_MAX;
+
+ return div_s64(n, max_delta);
}
-/*
- * Update output frequency for "big" frequency changes
- */
static int si544_set_rate(struct clk_hw *hw, unsigned long rate,
unsigned long parent_rate)
{
struct clk_si544 *data = to_clk_si544(hw);
struct clk_si544_muldiv settings;
+ unsigned long center;
+ long max_delta;
+ long delta;
unsigned int old_oe_state;
int err;
if (!is_valid_frequency(data, rate))
return -EINVAL;
+ /* Try using the frequency adjustment feature for a <= 950ppm change */
+ err = si544_get_muldiv(data, &settings);
+ if (err)
+ return err;
+
+ center = si544_calc_center_rate(&settings);
+ max_delta = si544_max_delta(center);
+ delta = rate - center;
+
+ if (abs(delta) <= max_delta)
+ return si544_set_delta_m(data,
+ si544_calc_delta(delta, max_delta));
+
+ /* Too big for the delta adjustment, need to reprogram */
err = si544_calc_muldiv(&settings, rate);
if (err)
return err;
@@ -321,6 +400,9 @@ static int si544_set_rate(struct clk_hw *hw, unsigned long rate,
if (err < 0)
return err;
+ err = si544_set_delta_m(data, settings.delta_m);
+ if (err < 0)
+ return err;
err = si544_set_muldiv(data, &settings);
if (err < 0)
diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c
index 87b410d6e51d..ca99e9db6575 100644
--- a/drivers/clk/clk.c
+++ b/drivers/clk/clk.c
@@ -324,6 +324,25 @@ static struct clk_core *clk_core_lookup(const char *name)
return NULL;
}
+#ifdef CONFIG_OF
+static int of_parse_clkspec(const struct device_node *np, int index,
+ const char *name, struct of_phandle_args *out_args);
+static struct clk_hw *
+of_clk_get_hw_from_clkspec(struct of_phandle_args *clkspec);
+#else
+static inline int of_parse_clkspec(const struct device_node *np, int index,
+ const char *name,
+ struct of_phandle_args *out_args)
+{
+ return -ENOENT;
+}
+static inline struct clk_hw *
+of_clk_get_hw_from_clkspec(struct of_phandle_args *clkspec)
+{
+ return ERR_PTR(-ENOENT);
+}
+#endif
+
/**
* clk_core_get - Find the clk_core parent of a clk
* @core: clk to find parent of
@@ -355,8 +374,9 @@ static struct clk_core *clk_core_lookup(const char *name)
* };
*
* Returns: -ENOENT when the provider can't be found or the clk doesn't
- * exist in the provider. -EINVAL when the name can't be found. NULL when the
- * provider knows about the clk but it isn't provided on this system.
+ * exist in the provider or the name can't be found in the DT node or
+ * in a clkdev lookup. NULL when the provider knows about the clk but it
+ * isn't provided on this system.
* A valid clk_core pointer when the clk can be found in the provider.
*/
static struct clk_core *clk_core_get(struct clk_core *core, u8 p_index)
@@ -367,17 +387,19 @@ static struct clk_core *clk_core_get(struct clk_core *core, u8 p_index)
struct device *dev = core->dev;
const char *dev_id = dev ? dev_name(dev) : NULL;
struct device_node *np = core->of_node;
+ struct of_phandle_args clkspec;
- if (np && (name || index >= 0))
- hw = of_clk_get_hw(np, index, name);
-
- /*
- * If the DT search above couldn't find the provider or the provider
- * didn't know about this clk, fallback to looking up via clkdev based
- * clk_lookups
- */
- if (PTR_ERR(hw) == -ENOENT && name)
+ if (np && (name || index >= 0) &&
+ !of_parse_clkspec(np, index, name, &clkspec)) {
+ hw = of_clk_get_hw_from_clkspec(&clkspec);
+ of_node_put(clkspec.np);
+ } else if (name) {
+ /*
+ * If the DT search above couldn't find the provider fallback to
+ * looking up via clkdev based clk_lookups.
+ */
hw = clk_find_hw(dev_id, name);
+ }
if (IS_ERR(hw))
return ERR_CAST(hw);
@@ -401,7 +423,7 @@ static void clk_core_fill_parent_index(struct clk_core *core, u8 index)
parent = ERR_PTR(-EPROBE_DEFER);
} else {
parent = clk_core_get(core, index);
- if (IS_ERR(parent) && PTR_ERR(parent) == -ENOENT)
+ if (IS_ERR(parent) && PTR_ERR(parent) == -ENOENT && entry->name)
parent = clk_core_lookup(entry->name);
}
@@ -1324,10 +1346,7 @@ static void clk_core_init_rate_req(struct clk_core * const core,
static bool clk_core_can_round(struct clk_core * const core)
{
- if (core->ops->determine_rate || core->ops->round_rate)
- return true;
-
- return false;
+ return core->ops->determine_rate || core->ops->round_rate;
}
static int clk_core_round_rate_nolock(struct clk_core *core,
@@ -1635,7 +1654,8 @@ static int clk_fetch_parent_index(struct clk_core *core,
break;
/* Fallback to comparing globally unique names */
- if (!strcmp(parent->name, core->parents[i].name))
+ if (core->parents[i].name &&
+ !strcmp(parent->name, core->parents[i].name))
break;
}
@@ -2194,7 +2214,7 @@ int clk_set_rate(struct clk *clk, unsigned long rate)
EXPORT_SYMBOL_GPL(clk_set_rate);
/**
- * clk_set_rate_exclusive - specify a new rate get exclusive control
+ * clk_set_rate_exclusive - specify a new rate and get exclusive control
* @clk: the clk whose rate is being changed
* @rate: the new rate for clk
*
@@ -2202,7 +2222,7 @@ EXPORT_SYMBOL_GPL(clk_set_rate);
* within a critical section
*
* This can be used initially to ensure that at least 1 consumer is
- * statisfied when several consumers are competing for exclusivity over the
+ * satisfied when several consumers are competing for exclusivity over the
* same clock provider.
*
* The exclusivity is not applied if setting the rate failed.
@@ -2490,6 +2510,12 @@ runtime_put:
return ret;
}
+int clk_hw_set_parent(struct clk_hw *hw, struct clk_hw *parent)
+{
+ return clk_core_set_parent_nolock(hw->core, parent->core);
+}
+EXPORT_SYMBOL_GPL(clk_hw_set_parent);
+
/**
* clk_set_parent - switch the parent of a mux clk
* @clk: the mux clk whose input we are switching
@@ -2997,20 +3023,65 @@ static int clk_flags_show(struct seq_file *s, void *data)
}
DEFINE_SHOW_ATTRIBUTE(clk_flags);
+static void possible_parent_show(struct seq_file *s, struct clk_core *core,
+ unsigned int i, char terminator)
+{
+ struct clk_core *parent;
+
+ /*
+ * Go through the following options to fetch a parent's name.
+ *
+ * 1. Fetch the registered parent clock and use its name
+ * 2. Use the global (fallback) name if specified
+ * 3. Use the local fw_name if provided
+ * 4. Fetch parent clock's clock-output-name if DT index was set
+ *
+ * This may still fail in some cases, such as when the parent is
+ * specified directly via a struct clk_hw pointer, but it isn't
+ * registered (yet).
+ */
+ parent = clk_core_get_parent_by_index(core, i);
+ if (parent)
+ seq_printf(s, "%s", parent->name);
+ else if (core->parents[i].name)
+ seq_printf(s, "%s", core->parents[i].name);
+ else if (core->parents[i].fw_name)
+ seq_printf(s, "<%s>(fw)", core->parents[i].fw_name);
+ else if (core->parents[i].index >= 0)
+ seq_printf(s, "%s",
+ of_clk_get_parent_name(core->of_node,
+ core->parents[i].index));
+ else
+ seq_puts(s, "(missing)");
+
+ seq_putc(s, terminator);
+}
+
static int possible_parents_show(struct seq_file *s, void *data)
{
struct clk_core *core = s->private;
int i;
for (i = 0; i < core->num_parents - 1; i++)
- seq_printf(s, "%s ", core->parents[i].name);
+ possible_parent_show(s, core, i, ' ');
- seq_printf(s, "%s\n", core->parents[i].name);
+ possible_parent_show(s, core, i, '\n');
return 0;
}
DEFINE_SHOW_ATTRIBUTE(possible_parents);
+static int current_parent_show(struct seq_file *s, void *data)
+{
+ struct clk_core *core = s->private;
+
+ if (core->parent)
+ seq_printf(s, "%s\n", core->parent->name);
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(current_parent);
+
static int clk_duty_cycle_show(struct seq_file *s, void *data)
{
struct clk_core *core = s->private;
@@ -3043,6 +3114,10 @@ static void clk_debug_create_one(struct clk_core *core, struct dentry *pdentry)
debugfs_create_file("clk_duty_cycle", 0444, root, core,
&clk_duty_cycle_fops);
+ if (core->num_parents > 0)
+ debugfs_create_file("clk_parent", 0444, root, core,
+ &current_parent_fops);
+
if (core->num_parents > 1)
debugfs_create_file("clk_possible_parents", 0444, root, core,
&possible_parents_fops);
@@ -4038,6 +4113,7 @@ struct of_clk_provider {
void *data;
};
+extern struct of_device_id __clk_of_table;
static const struct of_device_id __clk_of_table_sentinel
__used __section(__clk_of_table_end);
diff --git a/drivers/clk/clk.h b/drivers/clk/clk.h
index d8400d623b34..2d801900cad5 100644
--- a/drivers/clk/clk.h
+++ b/drivers/clk/clk.h
@@ -33,10 +33,6 @@ clk_hw_create_clk(struct device *dev, struct clk_hw *hw, const char *dev_id,
{
return (struct clk *)hw;
}
-static struct clk_hw *__clk_get_hw(struct clk *clk)
-{
- return (struct clk_hw *)clk;
-}
static inline void __clk_put(struct clk *clk) { }
#endif
diff --git a/drivers/clk/imx/Kconfig b/drivers/clk/imx/Kconfig
index 0eaf41848280..1ac0c7990392 100644
--- a/drivers/clk/imx/Kconfig
+++ b/drivers/clk/imx/Kconfig
@@ -14,6 +14,12 @@ config CLK_IMX8MM
help
Build the driver for i.MX8MM CCM Clock Driver
+config CLK_IMX8MN
+ bool "IMX8MN CCM Clock Driver"
+ depends on ARCH_MXC && ARM64
+ help
+ Build the driver for i.MX8MN CCM Clock Driver
+
config CLK_IMX8MQ
bool "IMX8MQ CCM Clock Driver"
depends on ARCH_MXC && ARM64
diff --git a/drivers/clk/imx/Makefile b/drivers/clk/imx/Makefile
index 05641c64b317..77a3d714f1d5 100644
--- a/drivers/clk/imx/Makefile
+++ b/drivers/clk/imx/Makefile
@@ -26,6 +26,7 @@ obj-$(CONFIG_MXC_CLK_SCU) += \
clk-lpcg-scu.o
obj-$(CONFIG_CLK_IMX8MM) += clk-imx8mm.o
+obj-$(CONFIG_CLK_IMX8MN) += clk-imx8mn.o
obj-$(CONFIG_CLK_IMX8MQ) += clk-imx8mq.o
obj-$(CONFIG_CLK_IMX8QXP) += clk-imx8qxp.o clk-imx8qxp-lpcg.o
diff --git a/drivers/clk/imx/clk-busy.c b/drivers/clk/imx/clk-busy.c
index 7b35a11238ca..25c863da32c7 100644
--- a/drivers/clk/imx/clk-busy.c
+++ b/drivers/clk/imx/clk-busy.c
@@ -72,13 +72,14 @@ static const struct clk_ops clk_busy_divider_ops = {
.set_rate = clk_busy_divider_set_rate,
};
-struct clk *imx_clk_busy_divider(const char *name, const char *parent_name,
+struct clk_hw *imx_clk_hw_busy_divider(const char *name, const char *parent_name,
void __iomem *reg, u8 shift, u8 width,
void __iomem *busy_reg, u8 busy_shift)
{
struct clk_busy_divider *busy;
- struct clk *clk;
+ struct clk_hw *hw;
struct clk_init_data init;
+ int ret;
busy = kzalloc(sizeof(*busy), GFP_KERNEL);
if (!busy)
@@ -101,11 +102,15 @@ struct clk *imx_clk_busy_divider(const char *name, const char *parent_name,
busy->div.hw.init = &init;
- clk = clk_register(NULL, &busy->div.hw);
- if (IS_ERR(clk))
+ hw = &busy->div.hw;
+
+ ret = clk_hw_register(NULL, hw);
+ if (ret) {
kfree(busy);
+ return ERR_PTR(ret);
+ }
- return clk;
+ return hw;
}
struct clk_busy_mux {
@@ -146,13 +151,14 @@ static const struct clk_ops clk_busy_mux_ops = {
.set_parent = clk_busy_mux_set_parent,
};
-struct clk *imx_clk_busy_mux(const char *name, void __iomem *reg, u8 shift,
+struct clk_hw *imx_clk_hw_busy_mux(const char *name, void __iomem *reg, u8 shift,
u8 width, void __iomem *busy_reg, u8 busy_shift,
const char * const *parent_names, int num_parents)
{
struct clk_busy_mux *busy;
- struct clk *clk;
+ struct clk_hw *hw;
struct clk_init_data init;
+ int ret;
busy = kzalloc(sizeof(*busy), GFP_KERNEL);
if (!busy)
@@ -175,9 +181,13 @@ struct clk *imx_clk_busy_mux(const char *name, void __iomem *reg, u8 shift,
busy->mux.hw.init = &init;
- clk = clk_register(NULL, &busy->mux.hw);
- if (IS_ERR(clk))
+ hw = &busy->mux.hw;
+
+ ret = clk_hw_register(NULL, hw);
+ if (ret) {
kfree(busy);
+ return ERR_PTR(ret);
+ }
- return clk;
+ return hw;
}
diff --git a/drivers/clk/imx/clk-cpu.c b/drivers/clk/imx/clk-cpu.c
index 00d026eb7891..cb182bec79ba 100644
--- a/drivers/clk/imx/clk-cpu.c
+++ b/drivers/clk/imx/clk-cpu.c
@@ -69,13 +69,14 @@ static const struct clk_ops clk_cpu_ops = {
.set_rate = clk_cpu_set_rate,
};
-struct clk *imx_clk_cpu(const char *name, const char *parent_name,
+struct clk_hw *imx_clk_hw_cpu(const char *name, const char *parent_name,
struct clk *div, struct clk *mux, struct clk *pll,
struct clk *step)
{
struct clk_cpu *cpu;
- struct clk *clk;
+ struct clk_hw *hw;
struct clk_init_data init;
+ int ret;
cpu = kzalloc(sizeof(*cpu), GFP_KERNEL);
if (!cpu)
@@ -93,10 +94,13 @@ struct clk *imx_clk_cpu(const char *name, const char *parent_name,
init.num_parents = 1;
cpu->hw.init = &init;
+ hw = &cpu->hw;
- clk = clk_register(NULL, &cpu->hw);
- if (IS_ERR(clk))
+ ret = clk_hw_register(NULL, hw);
+ if (ret) {
kfree(cpu);
+ return ERR_PTR(ret);
+ }
- return clk;
+ return hw;
}
diff --git a/drivers/clk/imx/clk-fixup-div.c b/drivers/clk/imx/clk-fixup-div.c
index bab46c6fd3db..4b17b91504ed 100644
--- a/drivers/clk/imx/clk-fixup-div.c
+++ b/drivers/clk/imx/clk-fixup-div.c
@@ -85,13 +85,14 @@ static const struct clk_ops clk_fixup_div_ops = {
.set_rate = clk_fixup_div_set_rate,
};
-struct clk *imx_clk_fixup_divider(const char *name, const char *parent,
+struct clk_hw *imx_clk_hw_fixup_divider(const char *name, const char *parent,
void __iomem *reg, u8 shift, u8 width,
void (*fixup)(u32 *val))
{
struct clk_fixup_div *fixup_div;
- struct clk *clk;
+ struct clk_hw *hw;
struct clk_init_data init;
+ int ret;
if (!fixup)
return ERR_PTR(-EINVAL);
@@ -114,9 +115,13 @@ struct clk *imx_clk_fixup_divider(const char *name, const char *parent,
fixup_div->ops = &clk_divider_ops;
fixup_div->fixup = fixup;
- clk = clk_register(NULL, &fixup_div->divider.hw);
- if (IS_ERR(clk))
+ hw = &fixup_div->divider.hw;
+
+ ret = clk_hw_register(NULL, hw);
+ if (ret) {
kfree(fixup_div);
+ return ERR_PTR(ret);
+ }
- return clk;
+ return hw;
}
diff --git a/drivers/clk/imx/clk-fixup-mux.c b/drivers/clk/imx/clk-fixup-mux.c
index 1aa3e8d9abf3..b569d919c645 100644
--- a/drivers/clk/imx/clk-fixup-mux.c
+++ b/drivers/clk/imx/clk-fixup-mux.c
@@ -63,13 +63,14 @@ static const struct clk_ops clk_fixup_mux_ops = {
.set_parent = clk_fixup_mux_set_parent,
};
-struct clk *imx_clk_fixup_mux(const char *name, void __iomem *reg,
+struct clk_hw *imx_clk_hw_fixup_mux(const char *name, void __iomem *reg,
u8 shift, u8 width, const char * const *parents,
int num_parents, void (*fixup)(u32 *val))
{
struct clk_fixup_mux *fixup_mux;
- struct clk *clk;
+ struct clk_hw *hw;
struct clk_init_data init;
+ int ret;
if (!fixup)
return ERR_PTR(-EINVAL);
@@ -92,9 +93,13 @@ struct clk *imx_clk_fixup_mux(const char *name, void __iomem *reg,
fixup_mux->ops = &clk_mux_ops;
fixup_mux->fixup = fixup;
- clk = clk_register(NULL, &fixup_mux->mux.hw);
- if (IS_ERR(clk))
+ hw = &fixup_mux->mux.hw;
+
+ ret = clk_hw_register(NULL, hw);
+ if (ret) {
kfree(fixup_mux);
+ return ERR_PTR(ret);
+ }
- return clk;
+ return hw;
}
diff --git a/drivers/clk/imx/clk-gate-exclusive.c b/drivers/clk/imx/clk-gate-exclusive.c
index cffa4966568d..77342893bb71 100644
--- a/drivers/clk/imx/clk-gate-exclusive.c
+++ b/drivers/clk/imx/clk-gate-exclusive.c
@@ -55,13 +55,14 @@ static const struct clk_ops clk_gate_exclusive_ops = {
.is_enabled = clk_gate_exclusive_is_enabled,
};
-struct clk *imx_clk_gate_exclusive(const char *name, const char *parent,
+struct clk_hw *imx_clk_hw_gate_exclusive(const char *name, const char *parent,
void __iomem *reg, u8 shift, u32 exclusive_mask)
{
struct clk_gate_exclusive *exgate;
struct clk_gate *gate;
- struct clk *clk;
+ struct clk_hw *hw;
struct clk_init_data init;
+ int ret;
if (exclusive_mask == 0)
return ERR_PTR(-EINVAL);
@@ -83,9 +84,13 @@ struct clk *imx_clk_gate_exclusive(const char *name, const char *parent,
gate->hw.init = &init;
exgate->exclusive_mask = exclusive_mask;
- clk = clk_register(NULL, &gate->hw);
- if (IS_ERR(clk))
- kfree(exgate);
+ hw = &gate->hw;
- return clk;
+ ret = clk_hw_register(NULL, hw);
+ if (ret) {
+ kfree(gate);
+ return ERR_PTR(ret);
+ }
+
+ return hw;
}
diff --git a/drivers/clk/imx/clk-gate2.c b/drivers/clk/imx/clk-gate2.c
index ec08fda547a3..7d44ce814806 100644
--- a/drivers/clk/imx/clk-gate2.c
+++ b/drivers/clk/imx/clk-gate2.c
@@ -122,15 +122,16 @@ static const struct clk_ops clk_gate2_ops = {
.is_enabled = clk_gate2_is_enabled,
};
-struct clk *clk_register_gate2(struct device *dev, const char *name,
+struct clk_hw *clk_hw_register_gate2(struct device *dev, const char *name,
const char *parent_name, unsigned long flags,
void __iomem *reg, u8 bit_idx, u8 cgr_val,
u8 clk_gate2_flags, spinlock_t *lock,
unsigned int *share_count)
{
struct clk_gate2 *gate;
- struct clk *clk;
+ struct clk_hw *hw;
struct clk_init_data init;
+ int ret;
gate = kzalloc(sizeof(struct clk_gate2), GFP_KERNEL);
if (!gate)
@@ -151,10 +152,13 @@ struct clk *clk_register_gate2(struct device *dev, const char *name,
init.num_parents = parent_name ? 1 : 0;
gate->hw.init = &init;
+ hw = &gate->hw;
- clk = clk_register(dev, &gate->hw);
- if (IS_ERR(clk))
+ ret = clk_hw_register(NULL, hw);
+ if (ret) {
kfree(gate);
+ return ERR_PTR(ret);
+ }
- return clk;
+ return hw;
}
diff --git a/drivers/clk/imx/clk-imx6q.c b/drivers/clk/imx/clk-imx6q.c
index d5de733f336e..60f2de851f39 100644
--- a/drivers/clk/imx/clk-imx6q.c
+++ b/drivers/clk/imx/clk-imx6q.c
@@ -8,6 +8,7 @@
#include <linux/types.h>
#include <linux/clk.h>
#include <linux/clkdev.h>
+#include <linux/clk-provider.h>
#include <linux/err.h>
#include <linux/io.h>
#include <linux/of.h>
@@ -87,8 +88,8 @@ static const char *pll5_bypass_sels[] = { "pll5", "pll5_bypass_src", };
static const char *pll6_bypass_sels[] = { "pll6", "pll6_bypass_src", };
static const char *pll7_bypass_sels[] = { "pll7", "pll7_bypass_src", };
-static struct clk *clk[IMX6QDL_CLK_END];
-static struct clk_onecell_data clk_data;
+static struct clk_hw **hws;
+static struct clk_hw_onecell_data *clk_hw_data;
static struct clk_div_table clk_enet_ref_table[] = {
{ .val = 0, .div = 20, },
@@ -138,12 +139,13 @@ static inline int clk_on_imx6dl(void)
return of_machine_is_compatible("fsl,imx6dl");
}
-static struct clk ** const uart_clks[] __initconst = {
- &clk[IMX6QDL_CLK_UART_IPG],
- &clk[IMX6QDL_CLK_UART_SERIAL],
- NULL
+static const int uart_clk_ids[] __initconst = {
+ IMX6QDL_CLK_UART_IPG,
+ IMX6QDL_CLK_UART_SERIAL,
};
+static struct clk **uart_clks[ARRAY_SIZE(uart_clk_ids) + 1] __initdata;
+
static int ldb_di_sel_by_clock_id(int clock_id)
{
switch (clock_id) {
@@ -254,25 +256,14 @@ static bool pll6_bypassed(struct device_node *node)
return false;
}
-#define CCM_CCDR 0x04
#define CCM_CCSR 0x0c
#define CCM_CS2CDR 0x2c
-#define CCDR_MMDC_CH1_MASK BIT(16)
#define CCSR_PLL3_SW_CLK_SEL BIT(0)
#define CS2CDR_LDB_DI0_CLK_SEL_SHIFT 9
#define CS2CDR_LDB_DI1_CLK_SEL_SHIFT 12
-static void __init imx6q_mmdc_ch1_mask_handshake(void __iomem *ccm_base)
-{
- unsigned int reg;
-
- reg = readl_relaxed(ccm_base + CCM_CCDR);
- reg |= CCDR_MMDC_CH1_MASK;
- writel_relaxed(reg, ccm_base + CCM_CCDR);
-}
-
/*
* The only way to disable the MMDC_CH1 clock is to move it to pll3_sw_clk
* via periph2_clk2_sel and then to disable pll3_sw_clk by selecting the
@@ -282,14 +273,8 @@ static void mmdc_ch1_disable(void __iomem *ccm_base)
{
unsigned int reg;
- clk_set_parent(clk[IMX6QDL_CLK_PERIPH2_CLK2_SEL],
- clk[IMX6QDL_CLK_PLL3_USB_OTG]);
-
- /*
- * Handshake with mmdc_ch1 module must be masked when changing
- * periph2_clk_sel.
- */
- clk_set_parent(clk[IMX6QDL_CLK_PERIPH2], clk[IMX6QDL_CLK_PERIPH2_CLK2]);
+ clk_set_parent(hws[IMX6QDL_CLK_PERIPH2_CLK2_SEL]->clk,
+ hws[IMX6QDL_CLK_PLL3_USB_OTG]->clk);
/* Disable pll3_sw_clk by selecting the bypass clock source */
reg = readl_relaxed(ccm_base + CCM_CCSR);
@@ -305,8 +290,6 @@ static void mmdc_ch1_reenable(void __iomem *ccm_base)
reg = readl_relaxed(ccm_base + CCM_CCSR);
reg &= ~CCSR_PLL3_SW_CLK_SEL;
writel_relaxed(reg, ccm_base + CCM_CCSR);
-
- clk_set_parent(clk[IMX6QDL_CLK_PERIPH2], clk[IMX6QDL_CLK_PERIPH2_PRE]);
}
/*
@@ -365,8 +348,8 @@ static void init_ldb_clks(struct device_node *np, void __iomem *ccm_base)
/* Only switch to or from pll2_pfd2_396m if it is disabled */
if ((sel[i][0] == 2 || sel[i][3] == 2) &&
- (clk_get_parent(clk[IMX6QDL_CLK_PERIPH_PRE]) ==
- clk[IMX6QDL_CLK_PLL2_PFD2_396M])) {
+ (clk_get_parent(hws[IMX6QDL_CLK_PERIPH_PRE]->clk) ==
+ hws[IMX6QDL_CLK_PLL2_PFD2_396M]->clk)) {
pr_err("ccm: ldb_di%d_sel: couldn't disable pll2_pfd2_396m\n",
i);
sel[i][3] = sel[i][2] = sel[i][1] = sel[i][0];
@@ -418,8 +401,8 @@ static void disable_anatop_clocks(void __iomem *anatop_base)
/* Make sure PLL2 PFDs 0-2 are gated */
reg = readl_relaxed(anatop_base + CCM_ANALOG_PFD_528);
/* Cannot gate PFD2 if pll2_pfd2_396m is the parent of MMDC clock */
- if (clk_get_parent(clk[IMX6QDL_CLK_PERIPH_PRE]) ==
- clk[IMX6QDL_CLK_PLL2_PFD2_396M])
+ if (clk_get_parent(hws[IMX6QDL_CLK_PERIPH_PRE]->clk) ==
+ hws[IMX6QDL_CLK_PLL2_PFD2_396M]->clk)
reg |= PFD0_CLKGATE | PFD1_CLKGATE;
else
reg |= PFD0_CLKGATE | PFD1_CLKGATE | PFD2_CLKGATE;
@@ -436,31 +419,45 @@ static void disable_anatop_clocks(void __iomem *anatop_base)
writel_relaxed(reg, anatop_base + CCM_ANALOG_PLL_VIDEO);
}
+static struct clk_hw * __init imx6q_obtain_fixed_clk_hw(struct device_node *np,
+ const char *name,
+ unsigned long rate)
+{
+ struct clk *clk = of_clk_get_by_name(np, name);
+ struct clk_hw *hw;
+
+ if (IS_ERR(clk))
+ hw = imx_obtain_fixed_clock_hw(name, rate);
+ else
+ hw = __clk_get_hw(clk);
+
+ return hw;
+}
+
static void __init imx6q_clocks_init(struct device_node *ccm_node)
{
struct device_node *np;
void __iomem *anatop_base, *base;
int ret;
+ int i;
+
+ clk_hw_data = kzalloc(struct_size(clk_hw_data, hws,
+ IMX6QDL_CLK_END), GFP_KERNEL);
+ if (WARN_ON(!clk_hw_data))
+ return;
- clk[IMX6QDL_CLK_DUMMY] = imx_clk_fixed("dummy", 0);
- clk[IMX6QDL_CLK_CKIL] = of_clk_get_by_name(ccm_node, "ckil");
- if (IS_ERR(clk[IMX6QDL_CLK_CKIL]))
- clk[IMX6QDL_CLK_CKIL] = imx_obtain_fixed_clock("ckil", 0);
- clk[IMX6QDL_CLK_CKIH] = of_clk_get_by_name(ccm_node, "ckih1");
- if (IS_ERR(clk[IMX6QDL_CLK_CKIH]))
- clk[IMX6QDL_CLK_CKIH] = imx_obtain_fixed_clock("ckih1", 0);
- clk[IMX6QDL_CLK_OSC] = of_clk_get_by_name(ccm_node, "osc");
- if (IS_ERR(clk[IMX6QDL_CLK_OSC]))
- clk[IMX6QDL_CLK_OSC] = imx_obtain_fixed_clock("osc", 0);
+ clk_hw_data->num = IMX6QDL_CLK_END;
+ hws = clk_hw_data->hws;
- /* Clock source from external clock via CLK1/2 PADs */
- clk[IMX6QDL_CLK_ANACLK1] = of_clk_get_by_name(ccm_node, "anaclk1");
- if (IS_ERR(clk[IMX6QDL_CLK_ANACLK1]))
- clk[IMX6QDL_CLK_ANACLK1] = imx_obtain_fixed_clock("anaclk1", 0);
+ hws[IMX6QDL_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0);
- clk[IMX6QDL_CLK_ANACLK2] = of_clk_get_by_name(ccm_node, "anaclk2");
- if (IS_ERR(clk[IMX6QDL_CLK_ANACLK2]))
- clk[IMX6QDL_CLK_ANACLK2] = imx_obtain_fixed_clock("anaclk2", 0);
+ hws[IMX6QDL_CLK_CKIL] = imx6q_obtain_fixed_clk_hw(ccm_node, "ckil", 0);
+ hws[IMX6QDL_CLK_CKIH] = imx6q_obtain_fixed_clk_hw(ccm_node, "ckih1", 0);
+ hws[IMX6QDL_CLK_OSC] = imx6q_obtain_fixed_clk_hw(ccm_node, "osc", 0);
+
+ /* Clock source from external clock via CLK1/2 PADs */
+ hws[IMX6QDL_CLK_ANACLK1] = imx6q_obtain_fixed_clk_hw(ccm_node, "anaclk1", 0);
+ hws[IMX6QDL_CLK_ANACLK2] = imx6q_obtain_fixed_clk_hw(ccm_node, "anaclk2", 0);
np = of_find_compatible_node(NULL, NULL, "fsl,imx6q-anatop");
anatop_base = base = of_iomap(np, 0);
@@ -475,47 +472,47 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node)
video_div_table[3].div = 1;
}
- clk[IMX6QDL_PLL1_BYPASS_SRC] = imx_clk_mux("pll1_bypass_src", base + 0x00, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clk[IMX6QDL_PLL2_BYPASS_SRC] = imx_clk_mux("pll2_bypass_src", base + 0x30, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clk[IMX6QDL_PLL3_BYPASS_SRC] = imx_clk_mux("pll3_bypass_src", base + 0x10, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clk[IMX6QDL_PLL4_BYPASS_SRC] = imx_clk_mux("pll4_bypass_src", base + 0x70, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clk[IMX6QDL_PLL5_BYPASS_SRC] = imx_clk_mux("pll5_bypass_src", base + 0xa0, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clk[IMX6QDL_PLL6_BYPASS_SRC] = imx_clk_mux("pll6_bypass_src", base + 0xe0, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clk[IMX6QDL_PLL7_BYPASS_SRC] = imx_clk_mux("pll7_bypass_src", base + 0x20, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6QDL_PLL1_BYPASS_SRC] = imx_clk_hw_mux("pll1_bypass_src", base + 0x00, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6QDL_PLL2_BYPASS_SRC] = imx_clk_hw_mux("pll2_bypass_src", base + 0x30, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6QDL_PLL3_BYPASS_SRC] = imx_clk_hw_mux("pll3_bypass_src", base + 0x10, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6QDL_PLL4_BYPASS_SRC] = imx_clk_hw_mux("pll4_bypass_src", base + 0x70, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6QDL_PLL5_BYPASS_SRC] = imx_clk_hw_mux("pll5_bypass_src", base + 0xa0, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6QDL_PLL6_BYPASS_SRC] = imx_clk_hw_mux("pll6_bypass_src", base + 0xe0, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6QDL_PLL7_BYPASS_SRC] = imx_clk_hw_mux("pll7_bypass_src", base + 0x20, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
/* type name parent_name base div_mask */
- clk[IMX6QDL_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f);
- clk[IMX6QDL_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1);
- clk[IMX6QDL_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3);
- clk[IMX6QDL_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f);
- clk[IMX6QDL_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f);
- clk[IMX6QDL_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3);
- clk[IMX6QDL_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3);
-
- clk[IMX6QDL_PLL1_BYPASS] = imx_clk_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_PLL2_BYPASS] = imx_clk_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_PLL3_BYPASS] = imx_clk_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_PLL4_BYPASS] = imx_clk_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_PLL5_BYPASS] = imx_clk_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_PLL6_BYPASS] = imx_clk_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_PLL7_BYPASS] = imx_clk_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_CLK_PLL1] = imx_clk_hw_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f);
+ hws[IMX6QDL_CLK_PLL2] = imx_clk_hw_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1);
+ hws[IMX6QDL_CLK_PLL3] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3);
+ hws[IMX6QDL_CLK_PLL4] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f);
+ hws[IMX6QDL_CLK_PLL5] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f);
+ hws[IMX6QDL_CLK_PLL6] = imx_clk_hw_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3);
+ hws[IMX6QDL_CLK_PLL7] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3);
+
+ hws[IMX6QDL_PLL1_BYPASS] = imx_clk_hw_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_PLL2_BYPASS] = imx_clk_hw_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_PLL3_BYPASS] = imx_clk_hw_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_PLL4_BYPASS] = imx_clk_hw_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_PLL5_BYPASS] = imx_clk_hw_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_PLL6_BYPASS] = imx_clk_hw_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_PLL7_BYPASS] = imx_clk_hw_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT);
/* Do not bypass PLLs initially */
- clk_set_parent(clk[IMX6QDL_PLL1_BYPASS], clk[IMX6QDL_CLK_PLL1]);
- clk_set_parent(clk[IMX6QDL_PLL2_BYPASS], clk[IMX6QDL_CLK_PLL2]);
- clk_set_parent(clk[IMX6QDL_PLL3_BYPASS], clk[IMX6QDL_CLK_PLL3]);
- clk_set_parent(clk[IMX6QDL_PLL4_BYPASS], clk[IMX6QDL_CLK_PLL4]);
- clk_set_parent(clk[IMX6QDL_PLL5_BYPASS], clk[IMX6QDL_CLK_PLL5]);
- clk_set_parent(clk[IMX6QDL_PLL6_BYPASS], clk[IMX6QDL_CLK_PLL6]);
- clk_set_parent(clk[IMX6QDL_PLL7_BYPASS], clk[IMX6QDL_CLK_PLL7]);
-
- clk[IMX6QDL_CLK_PLL1_SYS] = imx_clk_gate("pll1_sys", "pll1_bypass", base + 0x00, 13);
- clk[IMX6QDL_CLK_PLL2_BUS] = imx_clk_gate("pll2_bus", "pll2_bypass", base + 0x30, 13);
- clk[IMX6QDL_CLK_PLL3_USB_OTG] = imx_clk_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13);
- clk[IMX6QDL_CLK_PLL4_AUDIO] = imx_clk_gate("pll4_audio", "pll4_bypass", base + 0x70, 13);
- clk[IMX6QDL_CLK_PLL5_VIDEO] = imx_clk_gate("pll5_video", "pll5_bypass", base + 0xa0, 13);
- clk[IMX6QDL_CLK_PLL6_ENET] = imx_clk_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13);
- clk[IMX6QDL_CLK_PLL7_USB_HOST] = imx_clk_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13);
+ clk_set_parent(hws[IMX6QDL_PLL1_BYPASS]->clk, hws[IMX6QDL_CLK_PLL1]->clk);
+ clk_set_parent(hws[IMX6QDL_PLL2_BYPASS]->clk, hws[IMX6QDL_CLK_PLL2]->clk);
+ clk_set_parent(hws[IMX6QDL_PLL3_BYPASS]->clk, hws[IMX6QDL_CLK_PLL3]->clk);
+ clk_set_parent(hws[IMX6QDL_PLL4_BYPASS]->clk, hws[IMX6QDL_CLK_PLL4]->clk);
+ clk_set_parent(hws[IMX6QDL_PLL5_BYPASS]->clk, hws[IMX6QDL_CLK_PLL5]->clk);
+ clk_set_parent(hws[IMX6QDL_PLL6_BYPASS]->clk, hws[IMX6QDL_CLK_PLL6]->clk);
+ clk_set_parent(hws[IMX6QDL_PLL7_BYPASS]->clk, hws[IMX6QDL_CLK_PLL7]->clk);
+
+ hws[IMX6QDL_CLK_PLL1_SYS] = imx_clk_hw_gate("pll1_sys", "pll1_bypass", base + 0x00, 13);
+ hws[IMX6QDL_CLK_PLL2_BUS] = imx_clk_hw_gate("pll2_bus", "pll2_bypass", base + 0x30, 13);
+ hws[IMX6QDL_CLK_PLL3_USB_OTG] = imx_clk_hw_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13);
+ hws[IMX6QDL_CLK_PLL4_AUDIO] = imx_clk_hw_gate("pll4_audio", "pll4_bypass", base + 0x70, 13);
+ hws[IMX6QDL_CLK_PLL5_VIDEO] = imx_clk_hw_gate("pll5_video", "pll5_bypass", base + 0xa0, 13);
+ hws[IMX6QDL_CLK_PLL6_ENET] = imx_clk_hw_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13);
+ hws[IMX6QDL_CLK_PLL7_USB_HOST] = imx_clk_hw_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13);
/*
* Bit 20 is the reserved and read-only bit, we do this only for:
@@ -523,15 +520,15 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node)
* - Keep refcount when do usbphy clk_enable/disable, in that case,
* the clk framework may need to enable/disable usbphy's parent
*/
- clk[IMX6QDL_CLK_USBPHY1] = imx_clk_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20);
- clk[IMX6QDL_CLK_USBPHY2] = imx_clk_gate("usbphy2", "pll7_usb_host", base + 0x20, 20);
+ hws[IMX6QDL_CLK_USBPHY1] = imx_clk_hw_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20);
+ hws[IMX6QDL_CLK_USBPHY2] = imx_clk_hw_gate("usbphy2", "pll7_usb_host", base + 0x20, 20);
/*
* usbphy*_gate needs to be on after system boots up, and software
* never needs to control it anymore.
*/
- clk[IMX6QDL_CLK_USBPHY1_GATE] = imx_clk_gate("usbphy1_gate", "dummy", base + 0x10, 6);
- clk[IMX6QDL_CLK_USBPHY2_GATE] = imx_clk_gate("usbphy2_gate", "dummy", base + 0x20, 6);
+ hws[IMX6QDL_CLK_USBPHY1_GATE] = imx_clk_hw_gate("usbphy1_gate", "dummy", base + 0x10, 6);
+ hws[IMX6QDL_CLK_USBPHY2_GATE] = imx_clk_hw_gate("usbphy2_gate", "dummy", base + 0x20, 6);
/*
* The ENET PLL is special in that is has multiple outputs with
@@ -545,22 +542,22 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node)
*
*/
if (!pll6_bypassed(ccm_node)) {
- clk[IMX6QDL_CLK_SATA_REF] = imx_clk_fixed_factor("sata_ref", "pll6_enet", 1, 5);
- clk[IMX6QDL_CLK_PCIE_REF] = imx_clk_fixed_factor("pcie_ref", "pll6_enet", 1, 4);
- clk[IMX6QDL_CLK_ENET_REF] = clk_register_divider_table(NULL, "enet_ref", "pll6_enet", 0,
+ hws[IMX6QDL_CLK_SATA_REF] = imx_clk_hw_fixed_factor("sata_ref", "pll6_enet", 1, 5);
+ hws[IMX6QDL_CLK_PCIE_REF] = imx_clk_hw_fixed_factor("pcie_ref", "pll6_enet", 1, 4);
+ hws[IMX6QDL_CLK_ENET_REF] = clk_hw_register_divider_table(NULL, "enet_ref", "pll6_enet", 0,
base + 0xe0, 0, 2, 0, clk_enet_ref_table,
&imx_ccm_lock);
} else {
- clk[IMX6QDL_CLK_SATA_REF] = imx_clk_fixed_factor("sata_ref", "pll6_enet", 1, 1);
- clk[IMX6QDL_CLK_PCIE_REF] = imx_clk_fixed_factor("pcie_ref", "pll6_enet", 1, 1);
- clk[IMX6QDL_CLK_ENET_REF] = imx_clk_fixed_factor("enet_ref", "pll6_enet", 1, 1);
+ hws[IMX6QDL_CLK_SATA_REF] = imx_clk_hw_fixed_factor("sata_ref", "pll6_enet", 1, 1);
+ hws[IMX6QDL_CLK_PCIE_REF] = imx_clk_hw_fixed_factor("pcie_ref", "pll6_enet", 1, 1);
+ hws[IMX6QDL_CLK_ENET_REF] = imx_clk_hw_fixed_factor("enet_ref", "pll6_enet", 1, 1);
}
- clk[IMX6QDL_CLK_SATA_REF_100M] = imx_clk_gate("sata_ref_100m", "sata_ref", base + 0xe0, 20);
- clk[IMX6QDL_CLK_PCIE_REF_125M] = imx_clk_gate("pcie_ref_125m", "pcie_ref", base + 0xe0, 19);
+ hws[IMX6QDL_CLK_SATA_REF_100M] = imx_clk_hw_gate("sata_ref_100m", "sata_ref", base + 0xe0, 20);
+ hws[IMX6QDL_CLK_PCIE_REF_125M] = imx_clk_hw_gate("pcie_ref_125m", "pcie_ref", base + 0xe0, 19);
- clk[IMX6QDL_CLK_LVDS1_SEL] = imx_clk_mux("lvds1_sel", base + 0x160, 0, 5, lvds_sels, ARRAY_SIZE(lvds_sels));
- clk[IMX6QDL_CLK_LVDS2_SEL] = imx_clk_mux("lvds2_sel", base + 0x160, 5, 5, lvds_sels, ARRAY_SIZE(lvds_sels));
+ hws[IMX6QDL_CLK_LVDS1_SEL] = imx_clk_hw_mux("lvds1_sel", base + 0x160, 0, 5, lvds_sels, ARRAY_SIZE(lvds_sels));
+ hws[IMX6QDL_CLK_LVDS2_SEL] = imx_clk_hw_mux("lvds2_sel", base + 0x160, 5, 5, lvds_sels, ARRAY_SIZE(lvds_sels));
/*
* lvds1_gate and lvds2_gate are pseudo-gates. Both can be
@@ -572,84 +569,84 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node)
* it.
*/
writel(readl(base + 0x160) & ~0x3c00, base + 0x160);
- clk[IMX6QDL_CLK_LVDS1_GATE] = imx_clk_gate_exclusive("lvds1_gate", "lvds1_sel", base + 0x160, 10, BIT(12));
- clk[IMX6QDL_CLK_LVDS2_GATE] = imx_clk_gate_exclusive("lvds2_gate", "lvds2_sel", base + 0x160, 11, BIT(13));
+ hws[IMX6QDL_CLK_LVDS1_GATE] = imx_clk_hw_gate_exclusive("lvds1_gate", "lvds1_sel", base + 0x160, 10, BIT(12));
+ hws[IMX6QDL_CLK_LVDS2_GATE] = imx_clk_hw_gate_exclusive("lvds2_gate", "lvds2_sel", base + 0x160, 11, BIT(13));
- clk[IMX6QDL_CLK_LVDS1_IN] = imx_clk_gate_exclusive("lvds1_in", "anaclk1", base + 0x160, 12, BIT(10));
- clk[IMX6QDL_CLK_LVDS2_IN] = imx_clk_gate_exclusive("lvds2_in", "anaclk2", base + 0x160, 13, BIT(11));
+ hws[IMX6QDL_CLK_LVDS1_IN] = imx_clk_hw_gate_exclusive("lvds1_in", "anaclk1", base + 0x160, 12, BIT(10));
+ hws[IMX6QDL_CLK_LVDS2_IN] = imx_clk_hw_gate_exclusive("lvds2_in", "anaclk2", base + 0x160, 13, BIT(11));
/* name parent_name reg idx */
- clk[IMX6QDL_CLK_PLL2_PFD0_352M] = imx_clk_pfd("pll2_pfd0_352m", "pll2_bus", base + 0x100, 0);
- clk[IMX6QDL_CLK_PLL2_PFD1_594M] = imx_clk_pfd("pll2_pfd1_594m", "pll2_bus", base + 0x100, 1);
- clk[IMX6QDL_CLK_PLL2_PFD2_396M] = imx_clk_pfd("pll2_pfd2_396m", "pll2_bus", base + 0x100, 2);
- clk[IMX6QDL_CLK_PLL3_PFD0_720M] = imx_clk_pfd("pll3_pfd0_720m", "pll3_usb_otg", base + 0xf0, 0);
- clk[IMX6QDL_CLK_PLL3_PFD1_540M] = imx_clk_pfd("pll3_pfd1_540m", "pll3_usb_otg", base + 0xf0, 1);
- clk[IMX6QDL_CLK_PLL3_PFD2_508M] = imx_clk_pfd("pll3_pfd2_508m", "pll3_usb_otg", base + 0xf0, 2);
- clk[IMX6QDL_CLK_PLL3_PFD3_454M] = imx_clk_pfd("pll3_pfd3_454m", "pll3_usb_otg", base + 0xf0, 3);
+ hws[IMX6QDL_CLK_PLL2_PFD0_352M] = imx_clk_hw_pfd("pll2_pfd0_352m", "pll2_bus", base + 0x100, 0);
+ hws[IMX6QDL_CLK_PLL2_PFD1_594M] = imx_clk_hw_pfd("pll2_pfd1_594m", "pll2_bus", base + 0x100, 1);
+ hws[IMX6QDL_CLK_PLL2_PFD2_396M] = imx_clk_hw_pfd("pll2_pfd2_396m", "pll2_bus", base + 0x100, 2);
+ hws[IMX6QDL_CLK_PLL3_PFD0_720M] = imx_clk_hw_pfd("pll3_pfd0_720m", "pll3_usb_otg", base + 0xf0, 0);
+ hws[IMX6QDL_CLK_PLL3_PFD1_540M] = imx_clk_hw_pfd("pll3_pfd1_540m", "pll3_usb_otg", base + 0xf0, 1);
+ hws[IMX6QDL_CLK_PLL3_PFD2_508M] = imx_clk_hw_pfd("pll3_pfd2_508m", "pll3_usb_otg", base + 0xf0, 2);
+ hws[IMX6QDL_CLK_PLL3_PFD3_454M] = imx_clk_hw_pfd("pll3_pfd3_454m", "pll3_usb_otg", base + 0xf0, 3);
/* name parent_name mult div */
- clk[IMX6QDL_CLK_PLL2_198M] = imx_clk_fixed_factor("pll2_198m", "pll2_pfd2_396m", 1, 2);
- clk[IMX6QDL_CLK_PLL3_120M] = imx_clk_fixed_factor("pll3_120m", "pll3_usb_otg", 1, 4);
- clk[IMX6QDL_CLK_PLL3_80M] = imx_clk_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6);
- clk[IMX6QDL_CLK_PLL3_60M] = imx_clk_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8);
- clk[IMX6QDL_CLK_TWD] = imx_clk_fixed_factor("twd", "arm", 1, 2);
- clk[IMX6QDL_CLK_GPT_3M] = imx_clk_fixed_factor("gpt_3m", "osc", 1, 8);
- clk[IMX6QDL_CLK_VIDEO_27M] = imx_clk_fixed_factor("video_27m", "pll3_pfd1_540m", 1, 20);
+ hws[IMX6QDL_CLK_PLL2_198M] = imx_clk_hw_fixed_factor("pll2_198m", "pll2_pfd2_396m", 1, 2);
+ hws[IMX6QDL_CLK_PLL3_120M] = imx_clk_hw_fixed_factor("pll3_120m", "pll3_usb_otg", 1, 4);
+ hws[IMX6QDL_CLK_PLL3_80M] = imx_clk_hw_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6);
+ hws[IMX6QDL_CLK_PLL3_60M] = imx_clk_hw_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8);
+ hws[IMX6QDL_CLK_TWD] = imx_clk_hw_fixed_factor("twd", "arm", 1, 2);
+ hws[IMX6QDL_CLK_GPT_3M] = imx_clk_hw_fixed_factor("gpt_3m", "osc", 1, 8);
+ hws[IMX6QDL_CLK_VIDEO_27M] = imx_clk_hw_fixed_factor("video_27m", "pll3_pfd1_540m", 1, 20);
if (clk_on_imx6dl() || clk_on_imx6qp()) {
- clk[IMX6QDL_CLK_GPU2D_AXI] = imx_clk_fixed_factor("gpu2d_axi", "mmdc_ch0_axi_podf", 1, 1);
- clk[IMX6QDL_CLK_GPU3D_AXI] = imx_clk_fixed_factor("gpu3d_axi", "mmdc_ch0_axi_podf", 1, 1);
+ hws[IMX6QDL_CLK_GPU2D_AXI] = imx_clk_hw_fixed_factor("gpu2d_axi", "mmdc_ch0_axi_podf", 1, 1);
+ hws[IMX6QDL_CLK_GPU3D_AXI] = imx_clk_hw_fixed_factor("gpu3d_axi", "mmdc_ch0_axi_podf", 1, 1);
}
- clk[IMX6QDL_CLK_PLL4_POST_DIV] = clk_register_divider_table(NULL, "pll4_post_div", "pll4_audio", CLK_SET_RATE_PARENT, base + 0x70, 19, 2, 0, post_div_table, &imx_ccm_lock);
- clk[IMX6QDL_CLK_PLL4_AUDIO_DIV] = clk_register_divider(NULL, "pll4_audio_div", "pll4_post_div", CLK_SET_RATE_PARENT, base + 0x170, 15, 1, 0, &imx_ccm_lock);
- clk[IMX6QDL_CLK_PLL5_POST_DIV] = clk_register_divider_table(NULL, "pll5_post_div", "pll5_video", CLK_SET_RATE_PARENT, base + 0xa0, 19, 2, 0, post_div_table, &imx_ccm_lock);
- clk[IMX6QDL_CLK_PLL5_VIDEO_DIV] = clk_register_divider_table(NULL, "pll5_video_div", "pll5_post_div", CLK_SET_RATE_PARENT, base + 0x170, 30, 2, 0, video_div_table, &imx_ccm_lock);
+ hws[IMX6QDL_CLK_PLL4_POST_DIV] = clk_hw_register_divider_table(NULL, "pll4_post_div", "pll4_audio", CLK_SET_RATE_PARENT, base + 0x70, 19, 2, 0, post_div_table, &imx_ccm_lock);
+ hws[IMX6QDL_CLK_PLL4_AUDIO_DIV] = clk_hw_register_divider(NULL, "pll4_audio_div", "pll4_post_div", CLK_SET_RATE_PARENT, base + 0x170, 15, 1, 0, &imx_ccm_lock);
+ hws[IMX6QDL_CLK_PLL5_POST_DIV] = clk_hw_register_divider_table(NULL, "pll5_post_div", "pll5_video", CLK_SET_RATE_PARENT, base + 0xa0, 19, 2, 0, post_div_table, &imx_ccm_lock);
+ hws[IMX6QDL_CLK_PLL5_VIDEO_DIV] = clk_hw_register_divider_table(NULL, "pll5_video_div", "pll5_post_div", CLK_SET_RATE_PARENT, base + 0x170, 30, 2, 0, video_div_table, &imx_ccm_lock);
np = ccm_node;
base = of_iomap(np, 0);
WARN_ON(!base);
/* name reg shift width parent_names num_parents */
- clk[IMX6QDL_CLK_STEP] = imx_clk_mux("step", base + 0xc, 8, 1, step_sels, ARRAY_SIZE(step_sels));
- clk[IMX6QDL_CLK_PLL1_SW] = imx_clk_mux("pll1_sw", base + 0xc, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels));
- clk[IMX6QDL_CLK_PERIPH_PRE] = imx_clk_mux("periph_pre", base + 0x18, 18, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels));
- clk[IMX6QDL_CLK_PERIPH2_PRE] = imx_clk_mux("periph2_pre", base + 0x18, 21, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels));
- clk[IMX6QDL_CLK_PERIPH_CLK2_SEL] = imx_clk_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels));
- clk[IMX6QDL_CLK_PERIPH2_CLK2_SEL] = imx_clk_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels));
- clk[IMX6QDL_CLK_AXI_SEL] = imx_clk_mux("axi_sel", base + 0x14, 6, 2, axi_sels, ARRAY_SIZE(axi_sels));
- clk[IMX6QDL_CLK_ESAI_SEL] = imx_clk_mux("esai_sel", base + 0x20, 19, 2, audio_sels, ARRAY_SIZE(audio_sels));
- clk[IMX6QDL_CLK_ASRC_SEL] = imx_clk_mux("asrc_sel", base + 0x30, 7, 2, audio_sels, ARRAY_SIZE(audio_sels));
- clk[IMX6QDL_CLK_SPDIF_SEL] = imx_clk_mux("spdif_sel", base + 0x30, 20, 2, audio_sels, ARRAY_SIZE(audio_sels));
+ hws[IMX6QDL_CLK_STEP] = imx_clk_hw_mux("step", base + 0xc, 8, 1, step_sels, ARRAY_SIZE(step_sels));
+ hws[IMX6QDL_CLK_PLL1_SW] = imx_clk_hw_mux("pll1_sw", base + 0xc, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels));
+ hws[IMX6QDL_CLK_PERIPH_PRE] = imx_clk_hw_mux("periph_pre", base + 0x18, 18, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels));
+ hws[IMX6QDL_CLK_PERIPH2_PRE] = imx_clk_hw_mux("periph2_pre", base + 0x18, 21, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels));
+ hws[IMX6QDL_CLK_PERIPH_CLK2_SEL] = imx_clk_hw_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels));
+ hws[IMX6QDL_CLK_PERIPH2_CLK2_SEL] = imx_clk_hw_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels));
+ hws[IMX6QDL_CLK_AXI_SEL] = imx_clk_hw_mux("axi_sel", base + 0x14, 6, 2, axi_sels, ARRAY_SIZE(axi_sels));
+ hws[IMX6QDL_CLK_ESAI_SEL] = imx_clk_hw_mux("esai_sel", base + 0x20, 19, 2, audio_sels, ARRAY_SIZE(audio_sels));
+ hws[IMX6QDL_CLK_ASRC_SEL] = imx_clk_hw_mux("asrc_sel", base + 0x30, 7, 2, audio_sels, ARRAY_SIZE(audio_sels));
+ hws[IMX6QDL_CLK_SPDIF_SEL] = imx_clk_hw_mux("spdif_sel", base + 0x30, 20, 2, audio_sels, ARRAY_SIZE(audio_sels));
if (clk_on_imx6q()) {
- clk[IMX6QDL_CLK_GPU2D_AXI] = imx_clk_mux("gpu2d_axi", base + 0x18, 0, 1, gpu_axi_sels, ARRAY_SIZE(gpu_axi_sels));
- clk[IMX6QDL_CLK_GPU3D_AXI] = imx_clk_mux("gpu3d_axi", base + 0x18, 1, 1, gpu_axi_sels, ARRAY_SIZE(gpu_axi_sels));
+ hws[IMX6QDL_CLK_GPU2D_AXI] = imx_clk_hw_mux("gpu2d_axi", base + 0x18, 0, 1, gpu_axi_sels, ARRAY_SIZE(gpu_axi_sels));
+ hws[IMX6QDL_CLK_GPU3D_AXI] = imx_clk_hw_mux("gpu3d_axi", base + 0x18, 1, 1, gpu_axi_sels, ARRAY_SIZE(gpu_axi_sels));
}
if (clk_on_imx6qp()) {
- clk[IMX6QDL_CLK_CAN_SEL] = imx_clk_mux("can_sel", base + 0x20, 8, 2, can_sels, ARRAY_SIZE(can_sels));
- clk[IMX6QDL_CLK_ECSPI_SEL] = imx_clk_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels));
- clk[IMX6QDL_CLK_IPG_PER_SEL] = imx_clk_mux("ipg_per_sel", base + 0x1c, 6, 1, ipg_per_sels, ARRAY_SIZE(ipg_per_sels));
- clk[IMX6QDL_CLK_UART_SEL] = imx_clk_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels));
- clk[IMX6QDL_CLK_GPU2D_CORE_SEL] = imx_clk_mux("gpu2d_core_sel", base + 0x18, 16, 2, gpu2d_core_sels_2, ARRAY_SIZE(gpu2d_core_sels_2));
+ hws[IMX6QDL_CLK_CAN_SEL] = imx_clk_hw_mux("can_sel", base + 0x20, 8, 2, can_sels, ARRAY_SIZE(can_sels));
+ hws[IMX6QDL_CLK_ECSPI_SEL] = imx_clk_hw_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels));
+ hws[IMX6QDL_CLK_IPG_PER_SEL] = imx_clk_hw_mux("ipg_per_sel", base + 0x1c, 6, 1, ipg_per_sels, ARRAY_SIZE(ipg_per_sels));
+ hws[IMX6QDL_CLK_UART_SEL] = imx_clk_hw_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels));
+ hws[IMX6QDL_CLK_GPU2D_CORE_SEL] = imx_clk_hw_mux("gpu2d_core_sel", base + 0x18, 16, 2, gpu2d_core_sels_2, ARRAY_SIZE(gpu2d_core_sels_2));
} else if (clk_on_imx6dl()) {
- clk[IMX6QDL_CLK_MLB_SEL] = imx_clk_mux("mlb_sel", base + 0x18, 16, 2, gpu2d_core_sels, ARRAY_SIZE(gpu2d_core_sels));
+ hws[IMX6QDL_CLK_MLB_SEL] = imx_clk_hw_mux("mlb_sel", base + 0x18, 16, 2, gpu2d_core_sels, ARRAY_SIZE(gpu2d_core_sels));
} else {
- clk[IMX6QDL_CLK_GPU2D_CORE_SEL] = imx_clk_mux("gpu2d_core_sel", base + 0x18, 16, 2, gpu2d_core_sels, ARRAY_SIZE(gpu2d_core_sels));
+ hws[IMX6QDL_CLK_GPU2D_CORE_SEL] = imx_clk_hw_mux("gpu2d_core_sel", base + 0x18, 16, 2, gpu2d_core_sels, ARRAY_SIZE(gpu2d_core_sels));
}
- clk[IMX6QDL_CLK_GPU3D_CORE_SEL] = imx_clk_mux("gpu3d_core_sel", base + 0x18, 4, 2, gpu3d_core_sels, ARRAY_SIZE(gpu3d_core_sels));
+ hws[IMX6QDL_CLK_GPU3D_CORE_SEL] = imx_clk_hw_mux("gpu3d_core_sel", base + 0x18, 4, 2, gpu3d_core_sels, ARRAY_SIZE(gpu3d_core_sels));
if (clk_on_imx6dl())
- clk[IMX6QDL_CLK_GPU2D_CORE_SEL] = imx_clk_mux("gpu2d_core_sel", base + 0x18, 8, 2, gpu3d_shader_sels, ARRAY_SIZE(gpu3d_shader_sels));
+ hws[IMX6QDL_CLK_GPU2D_CORE_SEL] = imx_clk_hw_mux("gpu2d_core_sel", base + 0x18, 8, 2, gpu3d_shader_sels, ARRAY_SIZE(gpu3d_shader_sels));
else
- clk[IMX6QDL_CLK_GPU3D_SHADER_SEL] = imx_clk_mux("gpu3d_shader_sel", base + 0x18, 8, 2, gpu3d_shader_sels, ARRAY_SIZE(gpu3d_shader_sels));
- clk[IMX6QDL_CLK_IPU1_SEL] = imx_clk_mux("ipu1_sel", base + 0x3c, 9, 2, ipu_sels, ARRAY_SIZE(ipu_sels));
- clk[IMX6QDL_CLK_IPU2_SEL] = imx_clk_mux("ipu2_sel", base + 0x3c, 14, 2, ipu_sels, ARRAY_SIZE(ipu_sels));
+ hws[IMX6QDL_CLK_GPU3D_SHADER_SEL] = imx_clk_hw_mux("gpu3d_shader_sel", base + 0x18, 8, 2, gpu3d_shader_sels, ARRAY_SIZE(gpu3d_shader_sels));
+ hws[IMX6QDL_CLK_IPU1_SEL] = imx_clk_hw_mux("ipu1_sel", base + 0x3c, 9, 2, ipu_sels, ARRAY_SIZE(ipu_sels));
+ hws[IMX6QDL_CLK_IPU2_SEL] = imx_clk_hw_mux("ipu2_sel", base + 0x3c, 14, 2, ipu_sels, ARRAY_SIZE(ipu_sels));
disable_anatop_clocks(anatop_base);
- imx6q_mmdc_ch1_mask_handshake(base);
+ imx_mmdc_mask_handshake(base, 1);
if (clk_on_imx6qp()) {
- clk[IMX6QDL_CLK_LDB_DI0_SEL] = imx_clk_mux_flags("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_CLK_LDB_DI1_SEL] = imx_clk_mux_flags("ldb_di1_sel", base + 0x2c, 12, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_CLK_LDB_DI0_SEL] = imx_clk_hw_mux_flags("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_CLK_LDB_DI1_SEL] = imx_clk_hw_mux_flags("ldb_di1_sel", base + 0x2c, 12, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels), CLK_SET_RATE_PARENT);
} else {
/*
* The LDB_DI0/1_SEL muxes are registered read-only due to a hardware
@@ -658,322 +655,333 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node)
*/
init_ldb_clks(np, base);
- clk[IMX6QDL_CLK_LDB_DI0_SEL] = imx_clk_mux_ldb("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels));
- clk[IMX6QDL_CLK_LDB_DI1_SEL] = imx_clk_mux_ldb("ldb_di1_sel", base + 0x2c, 12, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels));
+ hws[IMX6QDL_CLK_LDB_DI0_SEL] = imx_clk_hw_mux_ldb("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels));
+ hws[IMX6QDL_CLK_LDB_DI1_SEL] = imx_clk_hw_mux_ldb("ldb_di1_sel", base + 0x2c, 12, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels));
}
- clk[IMX6QDL_CLK_IPU1_DI0_PRE_SEL] = imx_clk_mux_flags("ipu1_di0_pre_sel", base + 0x34, 6, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_CLK_IPU1_DI1_PRE_SEL] = imx_clk_mux_flags("ipu1_di1_pre_sel", base + 0x34, 15, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_CLK_IPU2_DI0_PRE_SEL] = imx_clk_mux_flags("ipu2_di0_pre_sel", base + 0x38, 6, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_CLK_IPU2_DI1_PRE_SEL] = imx_clk_mux_flags("ipu2_di1_pre_sel", base + 0x38, 15, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_CLK_HSI_TX_SEL] = imx_clk_mux("hsi_tx_sel", base + 0x30, 28, 1, hsi_tx_sels, ARRAY_SIZE(hsi_tx_sels));
- clk[IMX6QDL_CLK_PCIE_AXI_SEL] = imx_clk_mux("pcie_axi_sel", base + 0x18, 10, 1, pcie_axi_sels, ARRAY_SIZE(pcie_axi_sels));
+
+ hws[IMX6QDL_CLK_IPU1_DI0_PRE_SEL] = imx_clk_hw_mux_flags("ipu1_di0_pre_sel", base + 0x34, 6, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_CLK_IPU1_DI1_PRE_SEL] = imx_clk_hw_mux_flags("ipu1_di1_pre_sel", base + 0x34, 15, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_CLK_IPU2_DI0_PRE_SEL] = imx_clk_hw_mux_flags("ipu2_di0_pre_sel", base + 0x38, 6, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_CLK_IPU2_DI1_PRE_SEL] = imx_clk_hw_mux_flags("ipu2_di1_pre_sel", base + 0x38, 15, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_CLK_HSI_TX_SEL] = imx_clk_hw_mux("hsi_tx_sel", base + 0x30, 28, 1, hsi_tx_sels, ARRAY_SIZE(hsi_tx_sels));
+ hws[IMX6QDL_CLK_PCIE_AXI_SEL] = imx_clk_hw_mux("pcie_axi_sel", base + 0x18, 10, 1, pcie_axi_sels, ARRAY_SIZE(pcie_axi_sels));
+
if (clk_on_imx6qp()) {
- clk[IMX6QDL_CLK_IPU1_DI0_SEL] = imx_clk_mux_flags("ipu1_di0_sel", base + 0x34, 0, 3, ipu1_di0_sels_2, ARRAY_SIZE(ipu1_di0_sels_2), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_CLK_IPU1_DI1_SEL] = imx_clk_mux_flags("ipu1_di1_sel", base + 0x34, 9, 3, ipu1_di1_sels_2, ARRAY_SIZE(ipu1_di1_sels_2), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_CLK_IPU2_DI0_SEL] = imx_clk_mux_flags("ipu2_di0_sel", base + 0x38, 0, 3, ipu2_di0_sels_2, ARRAY_SIZE(ipu2_di0_sels_2), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_CLK_IPU2_DI1_SEL] = imx_clk_mux_flags("ipu2_di1_sel", base + 0x38, 9, 3, ipu2_di1_sels_2, ARRAY_SIZE(ipu2_di1_sels_2), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_CLK_SSI1_SEL] = imx_clk_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
- clk[IMX6QDL_CLK_SSI2_SEL] = imx_clk_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
- clk[IMX6QDL_CLK_SSI3_SEL] = imx_clk_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
- clk[IMX6QDL_CLK_USDHC1_SEL] = imx_clk_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
- clk[IMX6QDL_CLK_USDHC2_SEL] = imx_clk_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
- clk[IMX6QDL_CLK_USDHC3_SEL] = imx_clk_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
- clk[IMX6QDL_CLK_USDHC4_SEL] = imx_clk_mux("usdhc4_sel", base + 0x1c, 19, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
- clk[IMX6QDL_CLK_ENFC_SEL] = imx_clk_mux("enfc_sel", base + 0x2c, 15, 3, enfc_sels_2, ARRAY_SIZE(enfc_sels_2));
- clk[IMX6QDL_CLK_EIM_SEL] = imx_clk_mux("eim_sel", base + 0x1c, 27, 2, eim_sels, ARRAY_SIZE(eim_sels));
- clk[IMX6QDL_CLK_EIM_SLOW_SEL] = imx_clk_mux("eim_slow_sel", base + 0x1c, 29, 2, eim_slow_sels, ARRAY_SIZE(eim_slow_sels));
- clk[IMX6QDL_CLK_PRE_AXI] = imx_clk_mux("pre_axi", base + 0x18, 1, 1, pre_axi_sels, ARRAY_SIZE(pre_axi_sels));
+ hws[IMX6QDL_CLK_IPU1_DI0_SEL] = imx_clk_hw_mux_flags("ipu1_di0_sel", base + 0x34, 0, 3, ipu1_di0_sels_2, ARRAY_SIZE(ipu1_di0_sels_2), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_CLK_IPU1_DI1_SEL] = imx_clk_hw_mux_flags("ipu1_di1_sel", base + 0x34, 9, 3, ipu1_di1_sels_2, ARRAY_SIZE(ipu1_di1_sels_2), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_CLK_IPU2_DI0_SEL] = imx_clk_hw_mux_flags("ipu2_di0_sel", base + 0x38, 0, 3, ipu2_di0_sels_2, ARRAY_SIZE(ipu2_di0_sels_2), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_CLK_IPU2_DI1_SEL] = imx_clk_hw_mux_flags("ipu2_di1_sel", base + 0x38, 9, 3, ipu2_di1_sels_2, ARRAY_SIZE(ipu2_di1_sels_2), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_CLK_SSI1_SEL] = imx_clk_hw_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
+ hws[IMX6QDL_CLK_SSI2_SEL] = imx_clk_hw_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
+ hws[IMX6QDL_CLK_SSI3_SEL] = imx_clk_hw_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
+ hws[IMX6QDL_CLK_USDHC1_SEL] = imx_clk_hw_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
+ hws[IMX6QDL_CLK_USDHC2_SEL] = imx_clk_hw_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
+ hws[IMX6QDL_CLK_USDHC3_SEL] = imx_clk_hw_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
+ hws[IMX6QDL_CLK_USDHC4_SEL] = imx_clk_hw_mux("usdhc4_sel", base + 0x1c, 19, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
+ hws[IMX6QDL_CLK_ENFC_SEL] = imx_clk_hw_mux("enfc_sel", base + 0x2c, 15, 3, enfc_sels_2, ARRAY_SIZE(enfc_sels_2));
+ hws[IMX6QDL_CLK_EIM_SEL] = imx_clk_hw_mux("eim_sel", base + 0x1c, 27, 2, eim_sels, ARRAY_SIZE(eim_sels));
+ hws[IMX6QDL_CLK_EIM_SLOW_SEL] = imx_clk_hw_mux("eim_slow_sel", base + 0x1c, 29, 2, eim_slow_sels, ARRAY_SIZE(eim_slow_sels));
+ hws[IMX6QDL_CLK_PRE_AXI] = imx_clk_hw_mux("pre_axi", base + 0x18, 1, 1, pre_axi_sels, ARRAY_SIZE(pre_axi_sels));
} else {
- clk[IMX6QDL_CLK_IPU1_DI0_SEL] = imx_clk_mux_flags("ipu1_di0_sel", base + 0x34, 0, 3, ipu1_di0_sels, ARRAY_SIZE(ipu1_di0_sels), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_CLK_IPU1_DI1_SEL] = imx_clk_mux_flags("ipu1_di1_sel", base + 0x34, 9, 3, ipu1_di1_sels, ARRAY_SIZE(ipu1_di1_sels), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_CLK_IPU2_DI0_SEL] = imx_clk_mux_flags("ipu2_di0_sel", base + 0x38, 0, 3, ipu2_di0_sels, ARRAY_SIZE(ipu2_di0_sels), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_CLK_IPU2_DI1_SEL] = imx_clk_mux_flags("ipu2_di1_sel", base + 0x38, 9, 3, ipu2_di1_sels, ARRAY_SIZE(ipu2_di1_sels), CLK_SET_RATE_PARENT);
- clk[IMX6QDL_CLK_SSI1_SEL] = imx_clk_fixup_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup);
- clk[IMX6QDL_CLK_SSI2_SEL] = imx_clk_fixup_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup);
- clk[IMX6QDL_CLK_SSI3_SEL] = imx_clk_fixup_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup);
- clk[IMX6QDL_CLK_USDHC1_SEL] = imx_clk_fixup_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup);
- clk[IMX6QDL_CLK_USDHC2_SEL] = imx_clk_fixup_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup);
- clk[IMX6QDL_CLK_USDHC3_SEL] = imx_clk_fixup_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup);
- clk[IMX6QDL_CLK_USDHC4_SEL] = imx_clk_fixup_mux("usdhc4_sel", base + 0x1c, 19, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup);
- clk[IMX6QDL_CLK_ENFC_SEL] = imx_clk_mux("enfc_sel", base + 0x2c, 16, 2, enfc_sels, ARRAY_SIZE(enfc_sels));
- clk[IMX6QDL_CLK_EIM_SEL] = imx_clk_fixup_mux("eim_sel", base + 0x1c, 27, 2, eim_sels, ARRAY_SIZE(eim_sels), imx_cscmr1_fixup);
- clk[IMX6QDL_CLK_EIM_SLOW_SEL] = imx_clk_fixup_mux("eim_slow_sel", base + 0x1c, 29, 2, eim_slow_sels, ARRAY_SIZE(eim_slow_sels), imx_cscmr1_fixup);
+ hws[IMX6QDL_CLK_IPU1_DI0_SEL] = imx_clk_hw_mux_flags("ipu1_di0_sel", base + 0x34, 0, 3, ipu1_di0_sels, ARRAY_SIZE(ipu1_di0_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_CLK_IPU1_DI1_SEL] = imx_clk_hw_mux_flags("ipu1_di1_sel", base + 0x34, 9, 3, ipu1_di1_sels, ARRAY_SIZE(ipu1_di1_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_CLK_IPU2_DI0_SEL] = imx_clk_hw_mux_flags("ipu2_di0_sel", base + 0x38, 0, 3, ipu2_di0_sels, ARRAY_SIZE(ipu2_di0_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_CLK_IPU2_DI1_SEL] = imx_clk_hw_mux_flags("ipu2_di1_sel", base + 0x38, 9, 3, ipu2_di1_sels, ARRAY_SIZE(ipu2_di1_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6QDL_CLK_SSI1_SEL] = imx_clk_hw_fixup_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup);
+ hws[IMX6QDL_CLK_SSI2_SEL] = imx_clk_hw_fixup_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup);
+ hws[IMX6QDL_CLK_SSI3_SEL] = imx_clk_hw_fixup_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup);
+ hws[IMX6QDL_CLK_USDHC1_SEL] = imx_clk_hw_fixup_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup);
+ hws[IMX6QDL_CLK_USDHC2_SEL] = imx_clk_hw_fixup_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup);
+ hws[IMX6QDL_CLK_USDHC3_SEL] = imx_clk_hw_fixup_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup);
+ hws[IMX6QDL_CLK_USDHC4_SEL] = imx_clk_hw_fixup_mux("usdhc4_sel", base + 0x1c, 19, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup);
+ hws[IMX6QDL_CLK_ENFC_SEL] = imx_clk_hw_mux("enfc_sel", base + 0x2c, 16, 2, enfc_sels, ARRAY_SIZE(enfc_sels));
+ hws[IMX6QDL_CLK_EIM_SEL] = imx_clk_hw_fixup_mux("eim_sel", base + 0x1c, 27, 2, eim_sels, ARRAY_SIZE(eim_sels), imx_cscmr1_fixup);
+ hws[IMX6QDL_CLK_EIM_SLOW_SEL] = imx_clk_hw_fixup_mux("eim_slow_sel", base + 0x1c, 29, 2, eim_slow_sels, ARRAY_SIZE(eim_slow_sels), imx_cscmr1_fixup);
}
- clk[IMX6QDL_CLK_VDO_AXI_SEL] = imx_clk_mux("vdo_axi_sel", base + 0x18, 11, 1, vdo_axi_sels, ARRAY_SIZE(vdo_axi_sels));
- clk[IMX6QDL_CLK_VPU_AXI_SEL] = imx_clk_mux("vpu_axi_sel", base + 0x18, 14, 2, vpu_axi_sels, ARRAY_SIZE(vpu_axi_sels));
- clk[IMX6QDL_CLK_CKO1_SEL] = imx_clk_mux("cko1_sel", base + 0x60, 0, 4, cko1_sels, ARRAY_SIZE(cko1_sels));
- clk[IMX6QDL_CLK_CKO2_SEL] = imx_clk_mux("cko2_sel", base + 0x60, 16, 5, cko2_sels, ARRAY_SIZE(cko2_sels));
- clk[IMX6QDL_CLK_CKO] = imx_clk_mux("cko", base + 0x60, 8, 1, cko_sels, ARRAY_SIZE(cko_sels));
+
+ hws[IMX6QDL_CLK_VDO_AXI_SEL] = imx_clk_hw_mux("vdo_axi_sel", base + 0x18, 11, 1, vdo_axi_sels, ARRAY_SIZE(vdo_axi_sels));
+ hws[IMX6QDL_CLK_VPU_AXI_SEL] = imx_clk_hw_mux("vpu_axi_sel", base + 0x18, 14, 2, vpu_axi_sels, ARRAY_SIZE(vpu_axi_sels));
+ hws[IMX6QDL_CLK_CKO1_SEL] = imx_clk_hw_mux("cko1_sel", base + 0x60, 0, 4, cko1_sels, ARRAY_SIZE(cko1_sels));
+ hws[IMX6QDL_CLK_CKO2_SEL] = imx_clk_hw_mux("cko2_sel", base + 0x60, 16, 5, cko2_sels, ARRAY_SIZE(cko2_sels));
+ hws[IMX6QDL_CLK_CKO] = imx_clk_hw_mux("cko", base + 0x60, 8, 1, cko_sels, ARRAY_SIZE(cko_sels));
/* name reg shift width busy: reg, shift parent_names num_parents */
- clk[IMX6QDL_CLK_PERIPH] = imx_clk_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels));
- clk[IMX6QDL_CLK_PERIPH2] = imx_clk_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels));
+ hws[IMX6QDL_CLK_PERIPH] = imx_clk_hw_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels));
+ hws[IMX6QDL_CLK_PERIPH2] = imx_clk_hw_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels));
/* name parent_name reg shift width */
- clk[IMX6QDL_CLK_PERIPH_CLK2] = imx_clk_divider("periph_clk2", "periph_clk2_sel", base + 0x14, 27, 3);
- clk[IMX6QDL_CLK_PERIPH2_CLK2] = imx_clk_divider("periph2_clk2", "periph2_clk2_sel", base + 0x14, 0, 3);
- clk[IMX6QDL_CLK_IPG] = imx_clk_divider("ipg", "ahb", base + 0x14, 8, 2);
- clk[IMX6QDL_CLK_ESAI_PRED] = imx_clk_divider("esai_pred", "esai_sel", base + 0x28, 9, 3);
- clk[IMX6QDL_CLK_ESAI_PODF] = imx_clk_divider("esai_podf", "esai_pred", base + 0x28, 25, 3);
- clk[IMX6QDL_CLK_ASRC_PRED] = imx_clk_divider("asrc_pred", "asrc_sel", base + 0x30, 12, 3);
- clk[IMX6QDL_CLK_ASRC_PODF] = imx_clk_divider("asrc_podf", "asrc_pred", base + 0x30, 9, 3);
- clk[IMX6QDL_CLK_SPDIF_PRED] = imx_clk_divider("spdif_pred", "spdif_sel", base + 0x30, 25, 3);
- clk[IMX6QDL_CLK_SPDIF_PODF] = imx_clk_divider("spdif_podf", "spdif_pred", base + 0x30, 22, 3);
+ hws[IMX6QDL_CLK_PERIPH_CLK2] = imx_clk_hw_divider("periph_clk2", "periph_clk2_sel", base + 0x14, 27, 3);
+ hws[IMX6QDL_CLK_PERIPH2_CLK2] = imx_clk_hw_divider("periph2_clk2", "periph2_clk2_sel", base + 0x14, 0, 3);
+ hws[IMX6QDL_CLK_IPG] = imx_clk_hw_divider("ipg", "ahb", base + 0x14, 8, 2);
+ hws[IMX6QDL_CLK_ESAI_PRED] = imx_clk_hw_divider("esai_pred", "esai_sel", base + 0x28, 9, 3);
+ hws[IMX6QDL_CLK_ESAI_PODF] = imx_clk_hw_divider("esai_podf", "esai_pred", base + 0x28, 25, 3);
+ hws[IMX6QDL_CLK_ASRC_PRED] = imx_clk_hw_divider("asrc_pred", "asrc_sel", base + 0x30, 12, 3);
+ hws[IMX6QDL_CLK_ASRC_PODF] = imx_clk_hw_divider("asrc_podf", "asrc_pred", base + 0x30, 9, 3);
+ hws[IMX6QDL_CLK_SPDIF_PRED] = imx_clk_hw_divider("spdif_pred", "spdif_sel", base + 0x30, 25, 3);
+ hws[IMX6QDL_CLK_SPDIF_PODF] = imx_clk_hw_divider("spdif_podf", "spdif_pred", base + 0x30, 22, 3);
+
if (clk_on_imx6qp()) {
- clk[IMX6QDL_CLK_IPG_PER] = imx_clk_divider("ipg_per", "ipg_per_sel", base + 0x1c, 0, 6);
- clk[IMX6QDL_CLK_ECSPI_ROOT] = imx_clk_divider("ecspi_root", "ecspi_sel", base + 0x38, 19, 6);
- clk[IMX6QDL_CLK_CAN_ROOT] = imx_clk_divider("can_root", "can_sel", base + 0x20, 2, 6);
- clk[IMX6QDL_CLK_UART_SERIAL_PODF] = imx_clk_divider("uart_serial_podf", "uart_sel", base + 0x24, 0, 6);
- clk[IMX6QDL_CLK_LDB_DI0_DIV_3_5] = imx_clk_fixed_factor("ldb_di0_div_3_5", "ldb_di0", 2, 7);
- clk[IMX6QDL_CLK_LDB_DI1_DIV_3_5] = imx_clk_fixed_factor("ldb_di1_div_3_5", "ldb_di1", 2, 7);
+ hws[IMX6QDL_CLK_IPG_PER] = imx_clk_hw_divider("ipg_per", "ipg_per_sel", base + 0x1c, 0, 6);
+ hws[IMX6QDL_CLK_ECSPI_ROOT] = imx_clk_hw_divider("ecspi_root", "ecspi_sel", base + 0x38, 19, 6);
+ hws[IMX6QDL_CLK_CAN_ROOT] = imx_clk_hw_divider("can_root", "can_sel", base + 0x20, 2, 6);
+ hws[IMX6QDL_CLK_UART_SERIAL_PODF] = imx_clk_hw_divider("uart_serial_podf", "uart_sel", base + 0x24, 0, 6);
+ hws[IMX6QDL_CLK_LDB_DI0_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di0_div_3_5", "ldb_di0", 2, 7);
+ hws[IMX6QDL_CLK_LDB_DI1_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di1_div_3_5", "ldb_di1", 2, 7);
} else {
- clk[IMX6QDL_CLK_ECSPI_ROOT] = imx_clk_divider("ecspi_root", "pll3_60m", base + 0x38, 19, 6);
- clk[IMX6QDL_CLK_CAN_ROOT] = imx_clk_divider("can_root", "pll3_60m", base + 0x20, 2, 6);
- clk[IMX6QDL_CLK_IPG_PER] = imx_clk_fixup_divider("ipg_per", "ipg", base + 0x1c, 0, 6, imx_cscmr1_fixup);
- clk[IMX6QDL_CLK_UART_SERIAL_PODF] = imx_clk_divider("uart_serial_podf", "pll3_80m", base + 0x24, 0, 6);
- clk[IMX6QDL_CLK_LDB_DI0_DIV_3_5] = imx_clk_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7);
- clk[IMX6QDL_CLK_LDB_DI1_DIV_3_5] = imx_clk_fixed_factor("ldb_di1_div_3_5", "ldb_di1_sel", 2, 7);
+ hws[IMX6QDL_CLK_ECSPI_ROOT] = imx_clk_hw_divider("ecspi_root", "pll3_60m", base + 0x38, 19, 6);
+ hws[IMX6QDL_CLK_CAN_ROOT] = imx_clk_hw_divider("can_root", "pll3_60m", base + 0x20, 2, 6);
+ hws[IMX6QDL_CLK_IPG_PER] = imx_clk_hw_fixup_divider("ipg_per", "ipg", base + 0x1c, 0, 6, imx_cscmr1_fixup);
+ hws[IMX6QDL_CLK_UART_SERIAL_PODF] = imx_clk_hw_divider("uart_serial_podf", "pll3_80m", base + 0x24, 0, 6);
+ hws[IMX6QDL_CLK_LDB_DI0_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7);
+ hws[IMX6QDL_CLK_LDB_DI1_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di1_div_3_5", "ldb_di1_sel", 2, 7);
}
+
if (clk_on_imx6dl())
- clk[IMX6QDL_CLK_MLB_PODF] = imx_clk_divider("mlb_podf", "mlb_sel", base + 0x18, 23, 3);
+ hws[IMX6QDL_CLK_MLB_PODF] = imx_clk_hw_divider("mlb_podf", "mlb_sel", base + 0x18, 23, 3);
else
- clk[IMX6QDL_CLK_GPU2D_CORE_PODF] = imx_clk_divider("gpu2d_core_podf", "gpu2d_core_sel", base + 0x18, 23, 3);
- clk[IMX6QDL_CLK_GPU3D_CORE_PODF] = imx_clk_divider("gpu3d_core_podf", "gpu3d_core_sel", base + 0x18, 26, 3);
+ hws[IMX6QDL_CLK_GPU2D_CORE_PODF] = imx_clk_hw_divider("gpu2d_core_podf", "gpu2d_core_sel", base + 0x18, 23, 3);
+ hws[IMX6QDL_CLK_GPU3D_CORE_PODF] = imx_clk_hw_divider("gpu3d_core_podf", "gpu3d_core_sel", base + 0x18, 26, 3);
if (clk_on_imx6dl())
- clk[IMX6QDL_CLK_GPU2D_CORE_PODF] = imx_clk_divider("gpu2d_core_podf", "gpu2d_core_sel", base + 0x18, 29, 3);
+ hws[IMX6QDL_CLK_GPU2D_CORE_PODF] = imx_clk_hw_divider("gpu2d_core_podf", "gpu2d_core_sel", base + 0x18, 29, 3);
else
- clk[IMX6QDL_CLK_GPU3D_SHADER] = imx_clk_divider("gpu3d_shader", "gpu3d_shader_sel", base + 0x18, 29, 3);
- clk[IMX6QDL_CLK_IPU1_PODF] = imx_clk_divider("ipu1_podf", "ipu1_sel", base + 0x3c, 11, 3);
- clk[IMX6QDL_CLK_IPU2_PODF] = imx_clk_divider("ipu2_podf", "ipu2_sel", base + 0x3c, 16, 3);
- clk[IMX6QDL_CLK_LDB_DI0_PODF] = imx_clk_divider_flags("ldb_di0_podf", "ldb_di0_div_3_5", base + 0x20, 10, 1, 0);
- clk[IMX6QDL_CLK_LDB_DI1_PODF] = imx_clk_divider_flags("ldb_di1_podf", "ldb_di1_div_3_5", base + 0x20, 11, 1, 0);
- clk[IMX6QDL_CLK_IPU1_DI0_PRE] = imx_clk_divider("ipu1_di0_pre", "ipu1_di0_pre_sel", base + 0x34, 3, 3);
- clk[IMX6QDL_CLK_IPU1_DI1_PRE] = imx_clk_divider("ipu1_di1_pre", "ipu1_di1_pre_sel", base + 0x34, 12, 3);
- clk[IMX6QDL_CLK_IPU2_DI0_PRE] = imx_clk_divider("ipu2_di0_pre", "ipu2_di0_pre_sel", base + 0x38, 3, 3);
- clk[IMX6QDL_CLK_IPU2_DI1_PRE] = imx_clk_divider("ipu2_di1_pre", "ipu2_di1_pre_sel", base + 0x38, 12, 3);
- clk[IMX6QDL_CLK_HSI_TX_PODF] = imx_clk_divider("hsi_tx_podf", "hsi_tx_sel", base + 0x30, 29, 3);
- clk[IMX6QDL_CLK_SSI1_PRED] = imx_clk_divider("ssi1_pred", "ssi1_sel", base + 0x28, 6, 3);
- clk[IMX6QDL_CLK_SSI1_PODF] = imx_clk_divider("ssi1_podf", "ssi1_pred", base + 0x28, 0, 6);
- clk[IMX6QDL_CLK_SSI2_PRED] = imx_clk_divider("ssi2_pred", "ssi2_sel", base + 0x2c, 6, 3);
- clk[IMX6QDL_CLK_SSI2_PODF] = imx_clk_divider("ssi2_podf", "ssi2_pred", base + 0x2c, 0, 6);
- clk[IMX6QDL_CLK_SSI3_PRED] = imx_clk_divider("ssi3_pred", "ssi3_sel", base + 0x28, 22, 3);
- clk[IMX6QDL_CLK_SSI3_PODF] = imx_clk_divider("ssi3_podf", "ssi3_pred", base + 0x28, 16, 6);
- clk[IMX6QDL_CLK_USDHC1_PODF] = imx_clk_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3);
- clk[IMX6QDL_CLK_USDHC2_PODF] = imx_clk_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3);
- clk[IMX6QDL_CLK_USDHC3_PODF] = imx_clk_divider("usdhc3_podf", "usdhc3_sel", base + 0x24, 19, 3);
- clk[IMX6QDL_CLK_USDHC4_PODF] = imx_clk_divider("usdhc4_podf", "usdhc4_sel", base + 0x24, 22, 3);
- clk[IMX6QDL_CLK_ENFC_PRED] = imx_clk_divider("enfc_pred", "enfc_sel", base + 0x2c, 18, 3);
- clk[IMX6QDL_CLK_ENFC_PODF] = imx_clk_divider("enfc_podf", "enfc_pred", base + 0x2c, 21, 6);
+ hws[IMX6QDL_CLK_GPU3D_SHADER] = imx_clk_hw_divider("gpu3d_shader", "gpu3d_shader_sel", base + 0x18, 29, 3);
+ hws[IMX6QDL_CLK_IPU1_PODF] = imx_clk_hw_divider("ipu1_podf", "ipu1_sel", base + 0x3c, 11, 3);
+ hws[IMX6QDL_CLK_IPU2_PODF] = imx_clk_hw_divider("ipu2_podf", "ipu2_sel", base + 0x3c, 16, 3);
+ hws[IMX6QDL_CLK_LDB_DI0_PODF] = imx_clk_hw_divider_flags("ldb_di0_podf", "ldb_di0_div_3_5", base + 0x20, 10, 1, 0);
+ hws[IMX6QDL_CLK_LDB_DI1_PODF] = imx_clk_hw_divider_flags("ldb_di1_podf", "ldb_di1_div_3_5", base + 0x20, 11, 1, 0);
+ hws[IMX6QDL_CLK_IPU1_DI0_PRE] = imx_clk_hw_divider("ipu1_di0_pre", "ipu1_di0_pre_sel", base + 0x34, 3, 3);
+ hws[IMX6QDL_CLK_IPU1_DI1_PRE] = imx_clk_hw_divider("ipu1_di1_pre", "ipu1_di1_pre_sel", base + 0x34, 12, 3);
+ hws[IMX6QDL_CLK_IPU2_DI0_PRE] = imx_clk_hw_divider("ipu2_di0_pre", "ipu2_di0_pre_sel", base + 0x38, 3, 3);
+ hws[IMX6QDL_CLK_IPU2_DI1_PRE] = imx_clk_hw_divider("ipu2_di1_pre", "ipu2_di1_pre_sel", base + 0x38, 12, 3);
+ hws[IMX6QDL_CLK_HSI_TX_PODF] = imx_clk_hw_divider("hsi_tx_podf", "hsi_tx_sel", base + 0x30, 29, 3);
+ hws[IMX6QDL_CLK_SSI1_PRED] = imx_clk_hw_divider("ssi1_pred", "ssi1_sel", base + 0x28, 6, 3);
+ hws[IMX6QDL_CLK_SSI1_PODF] = imx_clk_hw_divider("ssi1_podf", "ssi1_pred", base + 0x28, 0, 6);
+ hws[IMX6QDL_CLK_SSI2_PRED] = imx_clk_hw_divider("ssi2_pred", "ssi2_sel", base + 0x2c, 6, 3);
+ hws[IMX6QDL_CLK_SSI2_PODF] = imx_clk_hw_divider("ssi2_podf", "ssi2_pred", base + 0x2c, 0, 6);
+ hws[IMX6QDL_CLK_SSI3_PRED] = imx_clk_hw_divider("ssi3_pred", "ssi3_sel", base + 0x28, 22, 3);
+ hws[IMX6QDL_CLK_SSI3_PODF] = imx_clk_hw_divider("ssi3_podf", "ssi3_pred", base + 0x28, 16, 6);
+ hws[IMX6QDL_CLK_USDHC1_PODF] = imx_clk_hw_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3);
+ hws[IMX6QDL_CLK_USDHC2_PODF] = imx_clk_hw_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3);
+ hws[IMX6QDL_CLK_USDHC3_PODF] = imx_clk_hw_divider("usdhc3_podf", "usdhc3_sel", base + 0x24, 19, 3);
+ hws[IMX6QDL_CLK_USDHC4_PODF] = imx_clk_hw_divider("usdhc4_podf", "usdhc4_sel", base + 0x24, 22, 3);
+ hws[IMX6QDL_CLK_ENFC_PRED] = imx_clk_hw_divider("enfc_pred", "enfc_sel", base + 0x2c, 18, 3);
+ hws[IMX6QDL_CLK_ENFC_PODF] = imx_clk_hw_divider("enfc_podf", "enfc_pred", base + 0x2c, 21, 6);
if (clk_on_imx6qp()) {
- clk[IMX6QDL_CLK_EIM_PODF] = imx_clk_divider("eim_podf", "eim_sel", base + 0x1c, 20, 3);
- clk[IMX6QDL_CLK_EIM_SLOW_PODF] = imx_clk_divider("eim_slow_podf", "eim_slow_sel", base + 0x1c, 23, 3);
+ hws[IMX6QDL_CLK_EIM_PODF] = imx_clk_hw_divider("eim_podf", "eim_sel", base + 0x1c, 20, 3);
+ hws[IMX6QDL_CLK_EIM_SLOW_PODF] = imx_clk_hw_divider("eim_slow_podf", "eim_slow_sel", base + 0x1c, 23, 3);
} else {
- clk[IMX6QDL_CLK_EIM_PODF] = imx_clk_fixup_divider("eim_podf", "eim_sel", base + 0x1c, 20, 3, imx_cscmr1_fixup);
- clk[IMX6QDL_CLK_EIM_SLOW_PODF] = imx_clk_fixup_divider("eim_slow_podf", "eim_slow_sel", base + 0x1c, 23, 3, imx_cscmr1_fixup);
+ hws[IMX6QDL_CLK_EIM_PODF] = imx_clk_hw_fixup_divider("eim_podf", "eim_sel", base + 0x1c, 20, 3, imx_cscmr1_fixup);
+ hws[IMX6QDL_CLK_EIM_SLOW_PODF] = imx_clk_hw_fixup_divider("eim_slow_podf", "eim_slow_sel", base + 0x1c, 23, 3, imx_cscmr1_fixup);
}
- clk[IMX6QDL_CLK_VPU_AXI_PODF] = imx_clk_divider("vpu_axi_podf", "vpu_axi_sel", base + 0x24, 25, 3);
- clk[IMX6QDL_CLK_CKO1_PODF] = imx_clk_divider("cko1_podf", "cko1_sel", base + 0x60, 4, 3);
- clk[IMX6QDL_CLK_CKO2_PODF] = imx_clk_divider("cko2_podf", "cko2_sel", base + 0x60, 21, 3);
+
+ hws[IMX6QDL_CLK_VPU_AXI_PODF] = imx_clk_hw_divider("vpu_axi_podf", "vpu_axi_sel", base + 0x24, 25, 3);
+ hws[IMX6QDL_CLK_CKO1_PODF] = imx_clk_hw_divider("cko1_podf", "cko1_sel", base + 0x60, 4, 3);
+ hws[IMX6QDL_CLK_CKO2_PODF] = imx_clk_hw_divider("cko2_podf", "cko2_sel", base + 0x60, 21, 3);
/* name parent_name reg shift width busy: reg, shift */
- clk[IMX6QDL_CLK_AXI] = imx_clk_busy_divider("axi", "axi_sel", base + 0x14, 16, 3, base + 0x48, 0);
- clk[IMX6QDL_CLK_MMDC_CH0_AXI_PODF] = imx_clk_busy_divider("mmdc_ch0_axi_podf", "periph", base + 0x14, 19, 3, base + 0x48, 4);
+ hws[IMX6QDL_CLK_AXI] = imx_clk_hw_busy_divider("axi", "axi_sel", base + 0x14, 16, 3, base + 0x48, 0);
+ hws[IMX6QDL_CLK_MMDC_CH0_AXI_PODF] = imx_clk_hw_busy_divider("mmdc_ch0_axi_podf", "periph", base + 0x14, 19, 3, base + 0x48, 4);
if (clk_on_imx6qp()) {
- clk[IMX6QDL_CLK_MMDC_CH1_AXI_CG] = imx_clk_gate("mmdc_ch1_axi_cg", "periph2", base + 0x4, 18);
- clk[IMX6QDL_CLK_MMDC_CH1_AXI_PODF] = imx_clk_busy_divider("mmdc_ch1_axi_podf", "mmdc_ch1_axi_cg", base + 0x14, 3, 3, base + 0x48, 2);
+ hws[IMX6QDL_CLK_MMDC_CH1_AXI_CG] = imx_clk_hw_gate("mmdc_ch1_axi_cg", "periph2", base + 0x4, 18);
+ hws[IMX6QDL_CLK_MMDC_CH1_AXI_PODF] = imx_clk_hw_busy_divider("mmdc_ch1_axi_podf", "mmdc_ch1_axi_cg", base + 0x14, 3, 3, base + 0x48, 2);
} else {
- clk[IMX6QDL_CLK_MMDC_CH1_AXI_PODF] = imx_clk_busy_divider("mmdc_ch1_axi_podf", "periph2", base + 0x14, 3, 3, base + 0x48, 2);
+ hws[IMX6QDL_CLK_MMDC_CH1_AXI_PODF] = imx_clk_hw_busy_divider("mmdc_ch1_axi_podf", "periph2", base + 0x14, 3, 3, base + 0x48, 2);
}
- clk[IMX6QDL_CLK_ARM] = imx_clk_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16);
- clk[IMX6QDL_CLK_AHB] = imx_clk_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1);
+ hws[IMX6QDL_CLK_ARM] = imx_clk_hw_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16);
+ hws[IMX6QDL_CLK_AHB] = imx_clk_hw_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1);
/* name parent_name reg shift */
- clk[IMX6QDL_CLK_APBH_DMA] = imx_clk_gate2("apbh_dma", "usdhc3", base + 0x68, 4);
- clk[IMX6QDL_CLK_ASRC] = imx_clk_gate2_shared("asrc", "asrc_podf", base + 0x68, 6, &share_count_asrc);
- clk[IMX6QDL_CLK_ASRC_IPG] = imx_clk_gate2_shared("asrc_ipg", "ahb", base + 0x68, 6, &share_count_asrc);
- clk[IMX6QDL_CLK_ASRC_MEM] = imx_clk_gate2_shared("asrc_mem", "ahb", base + 0x68, 6, &share_count_asrc);
- clk[IMX6QDL_CLK_CAAM_MEM] = imx_clk_gate2("caam_mem", "ahb", base + 0x68, 8);
- clk[IMX6QDL_CLK_CAAM_ACLK] = imx_clk_gate2("caam_aclk", "ahb", base + 0x68, 10);
- clk[IMX6QDL_CLK_CAAM_IPG] = imx_clk_gate2("caam_ipg", "ipg", base + 0x68, 12);
- clk[IMX6QDL_CLK_CAN1_IPG] = imx_clk_gate2("can1_ipg", "ipg", base + 0x68, 14);
- clk[IMX6QDL_CLK_CAN1_SERIAL] = imx_clk_gate2("can1_serial", "can_root", base + 0x68, 16);
- clk[IMX6QDL_CLK_CAN2_IPG] = imx_clk_gate2("can2_ipg", "ipg", base + 0x68, 18);
- clk[IMX6QDL_CLK_CAN2_SERIAL] = imx_clk_gate2("can2_serial", "can_root", base + 0x68, 20);
- clk[IMX6QDL_CLK_DCIC1] = imx_clk_gate2("dcic1", "ipu1_podf", base + 0x68, 24);
- clk[IMX6QDL_CLK_DCIC2] = imx_clk_gate2("dcic2", "ipu2_podf", base + 0x68, 26);
- clk[IMX6QDL_CLK_ECSPI1] = imx_clk_gate2("ecspi1", "ecspi_root", base + 0x6c, 0);
- clk[IMX6QDL_CLK_ECSPI2] = imx_clk_gate2("ecspi2", "ecspi_root", base + 0x6c, 2);
- clk[IMX6QDL_CLK_ECSPI3] = imx_clk_gate2("ecspi3", "ecspi_root", base + 0x6c, 4);
- clk[IMX6QDL_CLK_ECSPI4] = imx_clk_gate2("ecspi4", "ecspi_root", base + 0x6c, 6);
+ hws[IMX6QDL_CLK_APBH_DMA] = imx_clk_hw_gate2("apbh_dma", "usdhc3", base + 0x68, 4);
+ hws[IMX6QDL_CLK_ASRC] = imx_clk_hw_gate2_shared("asrc", "asrc_podf", base + 0x68, 6, &share_count_asrc);
+ hws[IMX6QDL_CLK_ASRC_IPG] = imx_clk_hw_gate2_shared("asrc_ipg", "ahb", base + 0x68, 6, &share_count_asrc);
+ hws[IMX6QDL_CLK_ASRC_MEM] = imx_clk_hw_gate2_shared("asrc_mem", "ahb", base + 0x68, 6, &share_count_asrc);
+ hws[IMX6QDL_CLK_CAAM_MEM] = imx_clk_hw_gate2("caam_mem", "ahb", base + 0x68, 8);
+ hws[IMX6QDL_CLK_CAAM_ACLK] = imx_clk_hw_gate2("caam_aclk", "ahb", base + 0x68, 10);
+ hws[IMX6QDL_CLK_CAAM_IPG] = imx_clk_hw_gate2("caam_ipg", "ipg", base + 0x68, 12);
+ hws[IMX6QDL_CLK_CAN1_IPG] = imx_clk_hw_gate2("can1_ipg", "ipg", base + 0x68, 14);
+ hws[IMX6QDL_CLK_CAN1_SERIAL] = imx_clk_hw_gate2("can1_serial", "can_root", base + 0x68, 16);
+ hws[IMX6QDL_CLK_CAN2_IPG] = imx_clk_hw_gate2("can2_ipg", "ipg", base + 0x68, 18);
+ hws[IMX6QDL_CLK_CAN2_SERIAL] = imx_clk_hw_gate2("can2_serial", "can_root", base + 0x68, 20);
+ hws[IMX6QDL_CLK_DCIC1] = imx_clk_hw_gate2("dcic1", "ipu1_podf", base + 0x68, 24);
+ hws[IMX6QDL_CLK_DCIC2] = imx_clk_hw_gate2("dcic2", "ipu2_podf", base + 0x68, 26);
+ hws[IMX6QDL_CLK_ECSPI1] = imx_clk_hw_gate2("ecspi1", "ecspi_root", base + 0x6c, 0);
+ hws[IMX6QDL_CLK_ECSPI2] = imx_clk_hw_gate2("ecspi2", "ecspi_root", base + 0x6c, 2);
+ hws[IMX6QDL_CLK_ECSPI3] = imx_clk_hw_gate2("ecspi3", "ecspi_root", base + 0x6c, 4);
+ hws[IMX6QDL_CLK_ECSPI4] = imx_clk_hw_gate2("ecspi4", "ecspi_root", base + 0x6c, 6);
if (clk_on_imx6dl())
- clk[IMX6DL_CLK_I2C4] = imx_clk_gate2("i2c4", "ipg_per", base + 0x6c, 8);
+ hws[IMX6DL_CLK_I2C4] = imx_clk_hw_gate2("i2c4", "ipg_per", base + 0x6c, 8);
else
- clk[IMX6Q_CLK_ECSPI5] = imx_clk_gate2("ecspi5", "ecspi_root", base + 0x6c, 8);
- clk[IMX6QDL_CLK_ENET] = imx_clk_gate2("enet", "ipg", base + 0x6c, 10);
- clk[IMX6QDL_CLK_EPIT1] = imx_clk_gate2("epit1", "ipg", base + 0x6c, 12);
- clk[IMX6QDL_CLK_EPIT2] = imx_clk_gate2("epit2", "ipg", base + 0x6c, 14);
- clk[IMX6QDL_CLK_ESAI_EXTAL] = imx_clk_gate2_shared("esai_extal", "esai_podf", base + 0x6c, 16, &share_count_esai);
- clk[IMX6QDL_CLK_ESAI_IPG] = imx_clk_gate2_shared("esai_ipg", "ahb", base + 0x6c, 16, &share_count_esai);
- clk[IMX6QDL_CLK_ESAI_MEM] = imx_clk_gate2_shared("esai_mem", "ahb", base + 0x6c, 16, &share_count_esai);
- clk[IMX6QDL_CLK_GPT_IPG] = imx_clk_gate2("gpt_ipg", "ipg", base + 0x6c, 20);
- clk[IMX6QDL_CLK_GPT_IPG_PER] = imx_clk_gate2("gpt_ipg_per", "ipg_per", base + 0x6c, 22);
- clk[IMX6QDL_CLK_GPU2D_CORE] = imx_clk_gate2("gpu2d_core", "gpu2d_core_podf", base + 0x6c, 24);
- clk[IMX6QDL_CLK_GPU3D_CORE] = imx_clk_gate2("gpu3d_core", "gpu3d_core_podf", base + 0x6c, 26);
- clk[IMX6QDL_CLK_HDMI_IAHB] = imx_clk_gate2("hdmi_iahb", "ahb", base + 0x70, 0);
- clk[IMX6QDL_CLK_HDMI_ISFR] = imx_clk_gate2("hdmi_isfr", "mipi_core_cfg", base + 0x70, 4);
- clk[IMX6QDL_CLK_I2C1] = imx_clk_gate2("i2c1", "ipg_per", base + 0x70, 6);
- clk[IMX6QDL_CLK_I2C2] = imx_clk_gate2("i2c2", "ipg_per", base + 0x70, 8);
- clk[IMX6QDL_CLK_I2C3] = imx_clk_gate2("i2c3", "ipg_per", base + 0x70, 10);
- clk[IMX6QDL_CLK_IIM] = imx_clk_gate2("iim", "ipg", base + 0x70, 12);
- clk[IMX6QDL_CLK_ENFC] = imx_clk_gate2("enfc", "enfc_podf", base + 0x70, 14);
- clk[IMX6QDL_CLK_VDOA] = imx_clk_gate2("vdoa", "vdo_axi", base + 0x70, 26);
- clk[IMX6QDL_CLK_IPU1] = imx_clk_gate2("ipu1", "ipu1_podf", base + 0x74, 0);
- clk[IMX6QDL_CLK_IPU1_DI0] = imx_clk_gate2("ipu1_di0", "ipu1_di0_sel", base + 0x74, 2);
- clk[IMX6QDL_CLK_IPU1_DI1] = imx_clk_gate2("ipu1_di1", "ipu1_di1_sel", base + 0x74, 4);
- clk[IMX6QDL_CLK_IPU2] = imx_clk_gate2("ipu2", "ipu2_podf", base + 0x74, 6);
- clk[IMX6QDL_CLK_IPU2_DI0] = imx_clk_gate2("ipu2_di0", "ipu2_di0_sel", base + 0x74, 8);
+ hws[IMX6Q_CLK_ECSPI5] = imx_clk_hw_gate2("ecspi5", "ecspi_root", base + 0x6c, 8);
+ hws[IMX6QDL_CLK_ENET] = imx_clk_hw_gate2("enet", "ipg", base + 0x6c, 10);
+ hws[IMX6QDL_CLK_EPIT1] = imx_clk_hw_gate2("epit1", "ipg", base + 0x6c, 12);
+ hws[IMX6QDL_CLK_EPIT2] = imx_clk_hw_gate2("epit2", "ipg", base + 0x6c, 14);
+ hws[IMX6QDL_CLK_ESAI_EXTAL] = imx_clk_hw_gate2_shared("esai_extal", "esai_podf", base + 0x6c, 16, &share_count_esai);
+ hws[IMX6QDL_CLK_ESAI_IPG] = imx_clk_hw_gate2_shared("esai_ipg", "ahb", base + 0x6c, 16, &share_count_esai);
+ hws[IMX6QDL_CLK_ESAI_MEM] = imx_clk_hw_gate2_shared("esai_mem", "ahb", base + 0x6c, 16, &share_count_esai);
+ hws[IMX6QDL_CLK_GPT_IPG] = imx_clk_hw_gate2("gpt_ipg", "ipg", base + 0x6c, 20);
+ hws[IMX6QDL_CLK_GPT_IPG_PER] = imx_clk_hw_gate2("gpt_ipg_per", "ipg_per", base + 0x6c, 22);
+ hws[IMX6QDL_CLK_GPU2D_CORE] = imx_clk_hw_gate2("gpu2d_core", "gpu2d_core_podf", base + 0x6c, 24);
+ hws[IMX6QDL_CLK_GPU3D_CORE] = imx_clk_hw_gate2("gpu3d_core", "gpu3d_core_podf", base + 0x6c, 26);
+ hws[IMX6QDL_CLK_HDMI_IAHB] = imx_clk_hw_gate2("hdmi_iahb", "ahb", base + 0x70, 0);
+ hws[IMX6QDL_CLK_HDMI_ISFR] = imx_clk_hw_gate2("hdmi_isfr", "mipi_core_cfg", base + 0x70, 4);
+ hws[IMX6QDL_CLK_I2C1] = imx_clk_hw_gate2("i2c1", "ipg_per", base + 0x70, 6);
+ hws[IMX6QDL_CLK_I2C2] = imx_clk_hw_gate2("i2c2", "ipg_per", base + 0x70, 8);
+ hws[IMX6QDL_CLK_I2C3] = imx_clk_hw_gate2("i2c3", "ipg_per", base + 0x70, 10);
+ hws[IMX6QDL_CLK_IIM] = imx_clk_hw_gate2("iim", "ipg", base + 0x70, 12);
+ hws[IMX6QDL_CLK_ENFC] = imx_clk_hw_gate2("enfc", "enfc_podf", base + 0x70, 14);
+ hws[IMX6QDL_CLK_VDOA] = imx_clk_hw_gate2("vdoa", "vdo_axi", base + 0x70, 26);
+ hws[IMX6QDL_CLK_IPU1] = imx_clk_hw_gate2("ipu1", "ipu1_podf", base + 0x74, 0);
+ hws[IMX6QDL_CLK_IPU1_DI0] = imx_clk_hw_gate2("ipu1_di0", "ipu1_di0_sel", base + 0x74, 2);
+ hws[IMX6QDL_CLK_IPU1_DI1] = imx_clk_hw_gate2("ipu1_di1", "ipu1_di1_sel", base + 0x74, 4);
+ hws[IMX6QDL_CLK_IPU2] = imx_clk_hw_gate2("ipu2", "ipu2_podf", base + 0x74, 6);
+ hws[IMX6QDL_CLK_IPU2_DI0] = imx_clk_hw_gate2("ipu2_di0", "ipu2_di0_sel", base + 0x74, 8);
if (clk_on_imx6qp()) {
- clk[IMX6QDL_CLK_LDB_DI0] = imx_clk_gate2("ldb_di0", "ldb_di0_sel", base + 0x74, 12);
- clk[IMX6QDL_CLK_LDB_DI1] = imx_clk_gate2("ldb_di1", "ldb_di1_sel", base + 0x74, 14);
+ hws[IMX6QDL_CLK_LDB_DI0] = imx_clk_hw_gate2("ldb_di0", "ldb_di0_sel", base + 0x74, 12);
+ hws[IMX6QDL_CLK_LDB_DI1] = imx_clk_hw_gate2("ldb_di1", "ldb_di1_sel", base + 0x74, 14);
} else {
- clk[IMX6QDL_CLK_LDB_DI0] = imx_clk_gate2("ldb_di0", "ldb_di0_podf", base + 0x74, 12);
- clk[IMX6QDL_CLK_LDB_DI1] = imx_clk_gate2("ldb_di1", "ldb_di1_podf", base + 0x74, 14);
+ hws[IMX6QDL_CLK_LDB_DI0] = imx_clk_hw_gate2("ldb_di0", "ldb_di0_podf", base + 0x74, 12);
+ hws[IMX6QDL_CLK_LDB_DI1] = imx_clk_hw_gate2("ldb_di1", "ldb_di1_podf", base + 0x74, 14);
}
- clk[IMX6QDL_CLK_IPU2_DI1] = imx_clk_gate2("ipu2_di1", "ipu2_di1_sel", base + 0x74, 10);
- clk[IMX6QDL_CLK_HSI_TX] = imx_clk_gate2_shared("hsi_tx", "hsi_tx_podf", base + 0x74, 16, &share_count_mipi_core_cfg);
- clk[IMX6QDL_CLK_MIPI_CORE_CFG] = imx_clk_gate2_shared("mipi_core_cfg", "video_27m", base + 0x74, 16, &share_count_mipi_core_cfg);
- clk[IMX6QDL_CLK_MIPI_IPG] = imx_clk_gate2_shared("mipi_ipg", "ipg", base + 0x74, 16, &share_count_mipi_core_cfg);
+ hws[IMX6QDL_CLK_IPU2_DI1] = imx_clk_hw_gate2("ipu2_di1", "ipu2_di1_sel", base + 0x74, 10);
+ hws[IMX6QDL_CLK_HSI_TX] = imx_clk_hw_gate2_shared("hsi_tx", "hsi_tx_podf", base + 0x74, 16, &share_count_mipi_core_cfg);
+ hws[IMX6QDL_CLK_MIPI_CORE_CFG] = imx_clk_hw_gate2_shared("mipi_core_cfg", "video_27m", base + 0x74, 16, &share_count_mipi_core_cfg);
+ hws[IMX6QDL_CLK_MIPI_IPG] = imx_clk_hw_gate2_shared("mipi_ipg", "ipg", base + 0x74, 16, &share_count_mipi_core_cfg);
+
if (clk_on_imx6dl())
/*
* The multiplexer and divider of the imx6q clock gpu2d get
* redefined/reused as mlb_sys_sel and mlb_sys_clk_podf on imx6dl.
*/
- clk[IMX6QDL_CLK_MLB] = imx_clk_gate2("mlb", "mlb_podf", base + 0x74, 18);
+ hws[IMX6QDL_CLK_MLB] = imx_clk_hw_gate2("mlb", "mlb_podf", base + 0x74, 18);
else
- clk[IMX6QDL_CLK_MLB] = imx_clk_gate2("mlb", "axi", base + 0x74, 18);
- clk[IMX6QDL_CLK_MMDC_CH0_AXI] = imx_clk_gate2_flags("mmdc_ch0_axi", "mmdc_ch0_axi_podf", base + 0x74, 20, CLK_IS_CRITICAL);
- clk[IMX6QDL_CLK_MMDC_CH1_AXI] = imx_clk_gate2("mmdc_ch1_axi", "mmdc_ch1_axi_podf", base + 0x74, 22);
- clk[IMX6QDL_CLK_MMDC_P0_IPG] = imx_clk_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL);
- clk[IMX6QDL_CLK_OCRAM] = imx_clk_gate2("ocram", "ahb", base + 0x74, 28);
- clk[IMX6QDL_CLK_OPENVG_AXI] = imx_clk_gate2("openvg_axi", "axi", base + 0x74, 30);
- clk[IMX6QDL_CLK_PCIE_AXI] = imx_clk_gate2("pcie_axi", "pcie_axi_sel", base + 0x78, 0);
- clk[IMX6QDL_CLK_PER1_BCH] = imx_clk_gate2("per1_bch", "usdhc3", base + 0x78, 12);
- clk[IMX6QDL_CLK_PWM1] = imx_clk_gate2("pwm1", "ipg_per", base + 0x78, 16);
- clk[IMX6QDL_CLK_PWM2] = imx_clk_gate2("pwm2", "ipg_per", base + 0x78, 18);
- clk[IMX6QDL_CLK_PWM3] = imx_clk_gate2("pwm3", "ipg_per", base + 0x78, 20);
- clk[IMX6QDL_CLK_PWM4] = imx_clk_gate2("pwm4", "ipg_per", base + 0x78, 22);
- clk[IMX6QDL_CLK_GPMI_BCH_APB] = imx_clk_gate2("gpmi_bch_apb", "usdhc3", base + 0x78, 24);
- clk[IMX6QDL_CLK_GPMI_BCH] = imx_clk_gate2("gpmi_bch", "usdhc4", base + 0x78, 26);
- clk[IMX6QDL_CLK_GPMI_IO] = imx_clk_gate2("gpmi_io", "enfc", base + 0x78, 28);
- clk[IMX6QDL_CLK_GPMI_APB] = imx_clk_gate2("gpmi_apb", "usdhc3", base + 0x78, 30);
- clk[IMX6QDL_CLK_ROM] = imx_clk_gate2_flags("rom", "ahb", base + 0x7c, 0, CLK_IS_CRITICAL);
- clk[IMX6QDL_CLK_SATA] = imx_clk_gate2("sata", "ahb", base + 0x7c, 4);
- clk[IMX6QDL_CLK_SDMA] = imx_clk_gate2("sdma", "ahb", base + 0x7c, 6);
- clk[IMX6QDL_CLK_SPBA] = imx_clk_gate2("spba", "ipg", base + 0x7c, 12);
- clk[IMX6QDL_CLK_SPDIF] = imx_clk_gate2_shared("spdif", "spdif_podf", base + 0x7c, 14, &share_count_spdif);
- clk[IMX6QDL_CLK_SPDIF_GCLK] = imx_clk_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_spdif);
- clk[IMX6QDL_CLK_SSI1_IPG] = imx_clk_gate2_shared("ssi1_ipg", "ipg", base + 0x7c, 18, &share_count_ssi1);
- clk[IMX6QDL_CLK_SSI2_IPG] = imx_clk_gate2_shared("ssi2_ipg", "ipg", base + 0x7c, 20, &share_count_ssi2);
- clk[IMX6QDL_CLK_SSI3_IPG] = imx_clk_gate2_shared("ssi3_ipg", "ipg", base + 0x7c, 22, &share_count_ssi3);
- clk[IMX6QDL_CLK_SSI1] = imx_clk_gate2_shared("ssi1", "ssi1_podf", base + 0x7c, 18, &share_count_ssi1);
- clk[IMX6QDL_CLK_SSI2] = imx_clk_gate2_shared("ssi2", "ssi2_podf", base + 0x7c, 20, &share_count_ssi2);
- clk[IMX6QDL_CLK_SSI3] = imx_clk_gate2_shared("ssi3", "ssi3_podf", base + 0x7c, 22, &share_count_ssi3);
- clk[IMX6QDL_CLK_UART_IPG] = imx_clk_gate2("uart_ipg", "ipg", base + 0x7c, 24);
- clk[IMX6QDL_CLK_UART_SERIAL] = imx_clk_gate2("uart_serial", "uart_serial_podf", base + 0x7c, 26);
- clk[IMX6QDL_CLK_USBOH3] = imx_clk_gate2("usboh3", "ipg", base + 0x80, 0);
- clk[IMX6QDL_CLK_USDHC1] = imx_clk_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2);
- clk[IMX6QDL_CLK_USDHC2] = imx_clk_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4);
- clk[IMX6QDL_CLK_USDHC3] = imx_clk_gate2("usdhc3", "usdhc3_podf", base + 0x80, 6);
- clk[IMX6QDL_CLK_USDHC4] = imx_clk_gate2("usdhc4", "usdhc4_podf", base + 0x80, 8);
- clk[IMX6QDL_CLK_EIM_SLOW] = imx_clk_gate2("eim_slow", "eim_slow_podf", base + 0x80, 10);
- clk[IMX6QDL_CLK_VDO_AXI] = imx_clk_gate2("vdo_axi", "vdo_axi_sel", base + 0x80, 12);
- clk[IMX6QDL_CLK_VPU_AXI] = imx_clk_gate2("vpu_axi", "vpu_axi_podf", base + 0x80, 14);
+ hws[IMX6QDL_CLK_MLB] = imx_clk_hw_gate2("mlb", "axi", base + 0x74, 18);
+ hws[IMX6QDL_CLK_MMDC_CH0_AXI] = imx_clk_hw_gate2_flags("mmdc_ch0_axi", "mmdc_ch0_axi_podf", base + 0x74, 20, CLK_IS_CRITICAL);
+ hws[IMX6QDL_CLK_MMDC_CH1_AXI] = imx_clk_hw_gate2("mmdc_ch1_axi", "mmdc_ch1_axi_podf", base + 0x74, 22);
+ hws[IMX6QDL_CLK_MMDC_P0_IPG] = imx_clk_hw_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL);
+ hws[IMX6QDL_CLK_OCRAM] = imx_clk_hw_gate2("ocram", "ahb", base + 0x74, 28);
+ hws[IMX6QDL_CLK_OPENVG_AXI] = imx_clk_hw_gate2("openvg_axi", "axi", base + 0x74, 30);
+ hws[IMX6QDL_CLK_PCIE_AXI] = imx_clk_hw_gate2("pcie_axi", "pcie_axi_sel", base + 0x78, 0);
+ hws[IMX6QDL_CLK_PER1_BCH] = imx_clk_hw_gate2("per1_bch", "usdhc3", base + 0x78, 12);
+ hws[IMX6QDL_CLK_PWM1] = imx_clk_hw_gate2("pwm1", "ipg_per", base + 0x78, 16);
+ hws[IMX6QDL_CLK_PWM2] = imx_clk_hw_gate2("pwm2", "ipg_per", base + 0x78, 18);
+ hws[IMX6QDL_CLK_PWM3] = imx_clk_hw_gate2("pwm3", "ipg_per", base + 0x78, 20);
+ hws[IMX6QDL_CLK_PWM4] = imx_clk_hw_gate2("pwm4", "ipg_per", base + 0x78, 22);
+ hws[IMX6QDL_CLK_GPMI_BCH_APB] = imx_clk_hw_gate2("gpmi_bch_apb", "usdhc3", base + 0x78, 24);
+ hws[IMX6QDL_CLK_GPMI_BCH] = imx_clk_hw_gate2("gpmi_bch", "usdhc4", base + 0x78, 26);
+ hws[IMX6QDL_CLK_GPMI_IO] = imx_clk_hw_gate2("gpmi_io", "enfc", base + 0x78, 28);
+ hws[IMX6QDL_CLK_GPMI_APB] = imx_clk_hw_gate2("gpmi_apb", "usdhc3", base + 0x78, 30);
+ hws[IMX6QDL_CLK_ROM] = imx_clk_hw_gate2_flags("rom", "ahb", base + 0x7c, 0, CLK_IS_CRITICAL);
+ hws[IMX6QDL_CLK_SATA] = imx_clk_hw_gate2("sata", "ahb", base + 0x7c, 4);
+ hws[IMX6QDL_CLK_SDMA] = imx_clk_hw_gate2("sdma", "ahb", base + 0x7c, 6);
+ hws[IMX6QDL_CLK_SPBA] = imx_clk_hw_gate2("spba", "ipg", base + 0x7c, 12);
+ hws[IMX6QDL_CLK_SPDIF] = imx_clk_hw_gate2_shared("spdif", "spdif_podf", base + 0x7c, 14, &share_count_spdif);
+ hws[IMX6QDL_CLK_SPDIF_GCLK] = imx_clk_hw_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_spdif);
+ hws[IMX6QDL_CLK_SSI1_IPG] = imx_clk_hw_gate2_shared("ssi1_ipg", "ipg", base + 0x7c, 18, &share_count_ssi1);
+ hws[IMX6QDL_CLK_SSI2_IPG] = imx_clk_hw_gate2_shared("ssi2_ipg", "ipg", base + 0x7c, 20, &share_count_ssi2);
+ hws[IMX6QDL_CLK_SSI3_IPG] = imx_clk_hw_gate2_shared("ssi3_ipg", "ipg", base + 0x7c, 22, &share_count_ssi3);
+ hws[IMX6QDL_CLK_SSI1] = imx_clk_hw_gate2_shared("ssi1", "ssi1_podf", base + 0x7c, 18, &share_count_ssi1);
+ hws[IMX6QDL_CLK_SSI2] = imx_clk_hw_gate2_shared("ssi2", "ssi2_podf", base + 0x7c, 20, &share_count_ssi2);
+ hws[IMX6QDL_CLK_SSI3] = imx_clk_hw_gate2_shared("ssi3", "ssi3_podf", base + 0x7c, 22, &share_count_ssi3);
+ hws[IMX6QDL_CLK_UART_IPG] = imx_clk_hw_gate2("uart_ipg", "ipg", base + 0x7c, 24);
+ hws[IMX6QDL_CLK_UART_SERIAL] = imx_clk_hw_gate2("uart_serial", "uart_serial_podf", base + 0x7c, 26);
+ hws[IMX6QDL_CLK_USBOH3] = imx_clk_hw_gate2("usboh3", "ipg", base + 0x80, 0);
+ hws[IMX6QDL_CLK_USDHC1] = imx_clk_hw_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2);
+ hws[IMX6QDL_CLK_USDHC2] = imx_clk_hw_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4);
+ hws[IMX6QDL_CLK_USDHC3] = imx_clk_hw_gate2("usdhc3", "usdhc3_podf", base + 0x80, 6);
+ hws[IMX6QDL_CLK_USDHC4] = imx_clk_hw_gate2("usdhc4", "usdhc4_podf", base + 0x80, 8);
+ hws[IMX6QDL_CLK_EIM_SLOW] = imx_clk_hw_gate2("eim_slow", "eim_slow_podf", base + 0x80, 10);
+ hws[IMX6QDL_CLK_VDO_AXI] = imx_clk_hw_gate2("vdo_axi", "vdo_axi_sel", base + 0x80, 12);
+ hws[IMX6QDL_CLK_VPU_AXI] = imx_clk_hw_gate2("vpu_axi", "vpu_axi_podf", base + 0x80, 14);
if (clk_on_imx6qp()) {
- clk[IMX6QDL_CLK_PRE0] = imx_clk_gate2("pre0", "pre_axi", base + 0x80, 16);
- clk[IMX6QDL_CLK_PRE1] = imx_clk_gate2("pre1", "pre_axi", base + 0x80, 18);
- clk[IMX6QDL_CLK_PRE2] = imx_clk_gate2("pre2", "pre_axi", base + 0x80, 20);
- clk[IMX6QDL_CLK_PRE3] = imx_clk_gate2("pre3", "pre_axi", base + 0x80, 22);
- clk[IMX6QDL_CLK_PRG0_AXI] = imx_clk_gate2_shared("prg0_axi", "ipu1_podf", base + 0x80, 24, &share_count_prg0);
- clk[IMX6QDL_CLK_PRG1_AXI] = imx_clk_gate2_shared("prg1_axi", "ipu2_podf", base + 0x80, 26, &share_count_prg1);
- clk[IMX6QDL_CLK_PRG0_APB] = imx_clk_gate2_shared("prg0_apb", "ipg", base + 0x80, 24, &share_count_prg0);
- clk[IMX6QDL_CLK_PRG1_APB] = imx_clk_gate2_shared("prg1_apb", "ipg", base + 0x80, 26, &share_count_prg1);
+ hws[IMX6QDL_CLK_PRE0] = imx_clk_hw_gate2("pre0", "pre_axi", base + 0x80, 16);
+ hws[IMX6QDL_CLK_PRE1] = imx_clk_hw_gate2("pre1", "pre_axi", base + 0x80, 18);
+ hws[IMX6QDL_CLK_PRE2] = imx_clk_hw_gate2("pre2", "pre_axi", base + 0x80, 20);
+ hws[IMX6QDL_CLK_PRE3] = imx_clk_hw_gate2("pre3", "pre_axi", base + 0x80, 22);
+ hws[IMX6QDL_CLK_PRG0_AXI] = imx_clk_hw_gate2_shared("prg0_axi", "ipu1_podf", base + 0x80, 24, &share_count_prg0);
+ hws[IMX6QDL_CLK_PRG1_AXI] = imx_clk_hw_gate2_shared("prg1_axi", "ipu2_podf", base + 0x80, 26, &share_count_prg1);
+ hws[IMX6QDL_CLK_PRG0_APB] = imx_clk_hw_gate2_shared("prg0_apb", "ipg", base + 0x80, 24, &share_count_prg0);
+ hws[IMX6QDL_CLK_PRG1_APB] = imx_clk_hw_gate2_shared("prg1_apb", "ipg", base + 0x80, 26, &share_count_prg1);
}
- clk[IMX6QDL_CLK_CKO1] = imx_clk_gate("cko1", "cko1_podf", base + 0x60, 7);
- clk[IMX6QDL_CLK_CKO2] = imx_clk_gate("cko2", "cko2_podf", base + 0x60, 24);
+ hws[IMX6QDL_CLK_CKO1] = imx_clk_hw_gate("cko1", "cko1_podf", base + 0x60, 7);
+ hws[IMX6QDL_CLK_CKO2] = imx_clk_hw_gate("cko2", "cko2_podf", base + 0x60, 24);
/*
* The gpt_3m clock is not available on i.MX6Q TO1.0. Let's point it
* to clock gpt_ipg_per to ease the gpt driver code.
*/
if (clk_on_imx6q() && imx_get_soc_revision() == IMX_CHIP_REVISION_1_0)
- clk[IMX6QDL_CLK_GPT_3M] = clk[IMX6QDL_CLK_GPT_IPG_PER];
+ hws[IMX6QDL_CLK_GPT_3M] = hws[IMX6QDL_CLK_GPT_IPG_PER];
- imx_check_clocks(clk, ARRAY_SIZE(clk));
+ imx_check_clk_hws(hws, IMX6QDL_CLK_END);
- clk_data.clks = clk;
- clk_data.clk_num = ARRAY_SIZE(clk);
- of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
+ of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data);
- clk_register_clkdev(clk[IMX6QDL_CLK_ENET_REF], "enet_ref", NULL);
+ clk_hw_register_clkdev(hws[IMX6QDL_CLK_ENET_REF], "enet_ref", NULL);
- clk_set_rate(clk[IMX6QDL_CLK_PLL3_PFD1_540M], 540000000);
+ clk_set_rate(hws[IMX6QDL_CLK_PLL3_PFD1_540M]->clk, 540000000);
if (clk_on_imx6dl())
- clk_set_parent(clk[IMX6QDL_CLK_IPU1_SEL], clk[IMX6QDL_CLK_PLL3_PFD1_540M]);
+ clk_set_parent(hws[IMX6QDL_CLK_IPU1_SEL]->clk, hws[IMX6QDL_CLK_PLL3_PFD1_540M]->clk);
- clk_set_parent(clk[IMX6QDL_CLK_IPU1_DI0_PRE_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]);
- clk_set_parent(clk[IMX6QDL_CLK_IPU1_DI1_PRE_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]);
- clk_set_parent(clk[IMX6QDL_CLK_IPU2_DI0_PRE_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]);
- clk_set_parent(clk[IMX6QDL_CLK_IPU2_DI1_PRE_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]);
- clk_set_parent(clk[IMX6QDL_CLK_IPU1_DI0_SEL], clk[IMX6QDL_CLK_IPU1_DI0_PRE]);
- clk_set_parent(clk[IMX6QDL_CLK_IPU1_DI1_SEL], clk[IMX6QDL_CLK_IPU1_DI1_PRE]);
- clk_set_parent(clk[IMX6QDL_CLK_IPU2_DI0_SEL], clk[IMX6QDL_CLK_IPU2_DI0_PRE]);
- clk_set_parent(clk[IMX6QDL_CLK_IPU2_DI1_SEL], clk[IMX6QDL_CLK_IPU2_DI1_PRE]);
+ clk_set_parent(hws[IMX6QDL_CLK_IPU1_DI0_PRE_SEL]->clk, hws[IMX6QDL_CLK_PLL5_VIDEO_DIV]->clk);
+ clk_set_parent(hws[IMX6QDL_CLK_IPU1_DI1_PRE_SEL]->clk, hws[IMX6QDL_CLK_PLL5_VIDEO_DIV]->clk);
+ clk_set_parent(hws[IMX6QDL_CLK_IPU2_DI0_PRE_SEL]->clk, hws[IMX6QDL_CLK_PLL5_VIDEO_DIV]->clk);
+ clk_set_parent(hws[IMX6QDL_CLK_IPU2_DI1_PRE_SEL]->clk, hws[IMX6QDL_CLK_PLL5_VIDEO_DIV]->clk);
+ clk_set_parent(hws[IMX6QDL_CLK_IPU1_DI0_SEL]->clk, hws[IMX6QDL_CLK_IPU1_DI0_PRE]->clk);
+ clk_set_parent(hws[IMX6QDL_CLK_IPU1_DI1_SEL]->clk, hws[IMX6QDL_CLK_IPU1_DI1_PRE]->clk);
+ clk_set_parent(hws[IMX6QDL_CLK_IPU2_DI0_SEL]->clk, hws[IMX6QDL_CLK_IPU2_DI0_PRE]->clk);
+ clk_set_parent(hws[IMX6QDL_CLK_IPU2_DI1_SEL]->clk, hws[IMX6QDL_CLK_IPU2_DI1_PRE]->clk);
/*
* The gpmi needs 100MHz frequency in the EDO/Sync mode,
* We can not get the 100MHz from the pll2_pfd0_352m.
* So choose pll2_pfd2_396m as enfc_sel's parent.
*/
- clk_set_parent(clk[IMX6QDL_CLK_ENFC_SEL], clk[IMX6QDL_CLK_PLL2_PFD2_396M]);
+ clk_set_parent(hws[IMX6QDL_CLK_ENFC_SEL]->clk, hws[IMX6QDL_CLK_PLL2_PFD2_396M]->clk);
if (IS_ENABLED(CONFIG_USB_MXS_PHY)) {
- clk_prepare_enable(clk[IMX6QDL_CLK_USBPHY1_GATE]);
- clk_prepare_enable(clk[IMX6QDL_CLK_USBPHY2_GATE]);
+ clk_prepare_enable(hws[IMX6QDL_CLK_USBPHY1_GATE]->clk);
+ clk_prepare_enable(hws[IMX6QDL_CLK_USBPHY2_GATE]->clk);
}
/*
* Let's initially set up CLKO with OSC24M, since this configuration
* is widely used by imx6q board designs to clock audio codec.
*/
- ret = clk_set_parent(clk[IMX6QDL_CLK_CKO2_SEL], clk[IMX6QDL_CLK_OSC]);
+ ret = clk_set_parent(hws[IMX6QDL_CLK_CKO2_SEL]->clk, hws[IMX6QDL_CLK_OSC]->clk);
if (!ret)
- ret = clk_set_parent(clk[IMX6QDL_CLK_CKO], clk[IMX6QDL_CLK_CKO2]);
+ ret = clk_set_parent(hws[IMX6QDL_CLK_CKO]->clk, hws[IMX6QDL_CLK_CKO2]->clk);
if (ret)
pr_warn("failed to set up CLKO: %d\n", ret);
/* Audio-related clocks configuration */
- clk_set_parent(clk[IMX6QDL_CLK_SPDIF_SEL], clk[IMX6QDL_CLK_PLL3_PFD3_454M]);
+ clk_set_parent(hws[IMX6QDL_CLK_SPDIF_SEL]->clk, hws[IMX6QDL_CLK_PLL3_PFD3_454M]->clk);
/* All existing boards with PCIe use LVDS1 */
if (IS_ENABLED(CONFIG_PCI_IMX6))
- clk_set_parent(clk[IMX6QDL_CLK_LVDS1_SEL], clk[IMX6QDL_CLK_SATA_REF_100M]);
+ clk_set_parent(hws[IMX6QDL_CLK_LVDS1_SEL]->clk, hws[IMX6QDL_CLK_SATA_REF_100M]->clk);
/*
* Initialize the GPU clock muxes, so that the maximum specified clock
* rates for the respective SoC are not exceeded.
*/
if (clk_on_imx6dl()) {
- clk_set_parent(clk[IMX6QDL_CLK_GPU3D_CORE_SEL],
- clk[IMX6QDL_CLK_PLL2_PFD1_594M]);
- clk_set_parent(clk[IMX6QDL_CLK_GPU2D_CORE_SEL],
- clk[IMX6QDL_CLK_PLL2_PFD1_594M]);
+ clk_set_parent(hws[IMX6QDL_CLK_GPU3D_CORE_SEL]->clk,
+ hws[IMX6QDL_CLK_PLL2_PFD1_594M]->clk);
+ clk_set_parent(hws[IMX6QDL_CLK_GPU2D_CORE_SEL]->clk,
+ hws[IMX6QDL_CLK_PLL2_PFD1_594M]->clk);
} else if (clk_on_imx6q()) {
- clk_set_parent(clk[IMX6QDL_CLK_GPU3D_CORE_SEL],
- clk[IMX6QDL_CLK_MMDC_CH0_AXI]);
- clk_set_parent(clk[IMX6QDL_CLK_GPU3D_SHADER_SEL],
- clk[IMX6QDL_CLK_PLL2_PFD1_594M]);
- clk_set_parent(clk[IMX6QDL_CLK_GPU2D_CORE_SEL],
- clk[IMX6QDL_CLK_PLL3_USB_OTG]);
+ clk_set_parent(hws[IMX6QDL_CLK_GPU3D_CORE_SEL]->clk,
+ hws[IMX6QDL_CLK_MMDC_CH0_AXI]->clk);
+ clk_set_parent(hws[IMX6QDL_CLK_GPU3D_SHADER_SEL]->clk,
+ hws[IMX6QDL_CLK_PLL2_PFD1_594M]->clk);
+ clk_set_parent(hws[IMX6QDL_CLK_GPU2D_CORE_SEL]->clk,
+ hws[IMX6QDL_CLK_PLL3_USB_OTG]->clk);
+ }
+
+ for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) {
+ int index = uart_clk_ids[i];
+
+ uart_clks[i] = &hws[index]->clk;
}
imx_register_uart_clocks(uart_clks);
diff --git a/drivers/clk/imx/clk-imx6sl.c b/drivers/clk/imx/clk-imx6sl.c
index f9f1f8a95d92..4bd44d89eaaa 100644
--- a/drivers/clk/imx/clk-imx6sl.c
+++ b/drivers/clk/imx/clk-imx6sl.c
@@ -13,8 +13,6 @@
#include "clk.h"
-#define CCDR 0x4
-#define BM_CCM_CCDR_MMDC_CH0_MASK (1 << 17)
#define CCSR 0xc
#define BM_CCSR_PLL1_SW_CLK_SEL (1 << 2)
#define CACRR 0x10
@@ -97,8 +95,8 @@ static unsigned int share_count_ssi2;
static unsigned int share_count_ssi3;
static unsigned int share_count_spdif;
-static struct clk *clks[IMX6SL_CLK_END];
-static struct clk_onecell_data clk_data;
+static struct clk_hw **hws;
+static struct clk_hw_onecell_data *clk_hw_data;
static void __iomem *ccm_base;
static void __iomem *anatop_base;
@@ -179,74 +177,84 @@ void imx6sl_set_wait_clk(bool enter)
imx6sl_enable_pll_arm(false);
}
-static struct clk ** const uart_clks[] __initconst = {
- &clks[IMX6SL_CLK_UART],
- &clks[IMX6SL_CLK_UART_SERIAL],
- NULL
+static const int uart_clk_ids[] __initconst = {
+ IMX6SL_CLK_UART,
+ IMX6SL_CLK_UART_SERIAL,
};
+static struct clk **uart_clks[ARRAY_SIZE(uart_clk_ids) + 1] __initdata;
+
static void __init imx6sl_clocks_init(struct device_node *ccm_node)
{
struct device_node *np;
void __iomem *base;
int ret;
+ int i;
+
+ clk_hw_data = kzalloc(struct_size(clk_hw_data, hws,
+ IMX6SL_CLK_END), GFP_KERNEL);
+ if (WARN_ON(!clk_hw_data))
+ return;
+
+ clk_hw_data->num = IMX6SL_CLK_END;
+ hws = clk_hw_data->hws;
- clks[IMX6SL_CLK_DUMMY] = imx_clk_fixed("dummy", 0);
- clks[IMX6SL_CLK_CKIL] = imx_obtain_fixed_clock("ckil", 0);
- clks[IMX6SL_CLK_OSC] = imx_obtain_fixed_clock("osc", 0);
+ hws[IMX6SL_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0);
+ hws[IMX6SL_CLK_CKIL] = imx_obtain_fixed_clock_hw("ckil", 0);
+ hws[IMX6SL_CLK_OSC] = imx_obtain_fixed_clock_hw("osc", 0);
/* Clock source from external clock via CLK1 PAD */
- clks[IMX6SL_CLK_ANACLK1] = imx_obtain_fixed_clock("anaclk1", 0);
+ hws[IMX6SL_CLK_ANACLK1] = imx_obtain_fixed_clock_hw("anaclk1", 0);
np = of_find_compatible_node(NULL, NULL, "fsl,imx6sl-anatop");
base = of_iomap(np, 0);
WARN_ON(!base);
anatop_base = base;
- clks[IMX6SL_PLL1_BYPASS_SRC] = imx_clk_mux("pll1_bypass_src", base + 0x00, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SL_PLL2_BYPASS_SRC] = imx_clk_mux("pll2_bypass_src", base + 0x30, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SL_PLL3_BYPASS_SRC] = imx_clk_mux("pll3_bypass_src", base + 0x10, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SL_PLL4_BYPASS_SRC] = imx_clk_mux("pll4_bypass_src", base + 0x70, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SL_PLL5_BYPASS_SRC] = imx_clk_mux("pll5_bypass_src", base + 0xa0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SL_PLL6_BYPASS_SRC] = imx_clk_mux("pll6_bypass_src", base + 0xe0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SL_PLL7_BYPASS_SRC] = imx_clk_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SL_PLL1_BYPASS_SRC] = imx_clk_hw_mux("pll1_bypass_src", base + 0x00, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SL_PLL2_BYPASS_SRC] = imx_clk_hw_mux("pll2_bypass_src", base + 0x30, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SL_PLL3_BYPASS_SRC] = imx_clk_hw_mux("pll3_bypass_src", base + 0x10, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SL_PLL4_BYPASS_SRC] = imx_clk_hw_mux("pll4_bypass_src", base + 0x70, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SL_PLL5_BYPASS_SRC] = imx_clk_hw_mux("pll5_bypass_src", base + 0xa0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SL_PLL6_BYPASS_SRC] = imx_clk_hw_mux("pll6_bypass_src", base + 0xe0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SL_PLL7_BYPASS_SRC] = imx_clk_hw_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
/* type name parent_name base div_mask */
- clks[IMX6SL_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f);
- clks[IMX6SL_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1);
- clks[IMX6SL_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3);
- clks[IMX6SL_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f);
- clks[IMX6SL_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f);
- clks[IMX6SL_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3);
- clks[IMX6SL_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3);
-
- clks[IMX6SL_PLL1_BYPASS] = imx_clk_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SL_PLL2_BYPASS] = imx_clk_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SL_PLL3_BYPASS] = imx_clk_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SL_PLL4_BYPASS] = imx_clk_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SL_PLL5_BYPASS] = imx_clk_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SL_PLL6_BYPASS] = imx_clk_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SL_PLL7_BYPASS] = imx_clk_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SL_CLK_PLL1] = imx_clk_hw_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f);
+ hws[IMX6SL_CLK_PLL2] = imx_clk_hw_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1);
+ hws[IMX6SL_CLK_PLL3] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3);
+ hws[IMX6SL_CLK_PLL4] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f);
+ hws[IMX6SL_CLK_PLL5] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f);
+ hws[IMX6SL_CLK_PLL6] = imx_clk_hw_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3);
+ hws[IMX6SL_CLK_PLL7] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3);
+
+ hws[IMX6SL_PLL1_BYPASS] = imx_clk_hw_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SL_PLL2_BYPASS] = imx_clk_hw_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SL_PLL3_BYPASS] = imx_clk_hw_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SL_PLL4_BYPASS] = imx_clk_hw_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SL_PLL5_BYPASS] = imx_clk_hw_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SL_PLL6_BYPASS] = imx_clk_hw_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SL_PLL7_BYPASS] = imx_clk_hw_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT);
/* Do not bypass PLLs initially */
- clk_set_parent(clks[IMX6SL_PLL1_BYPASS], clks[IMX6SL_CLK_PLL1]);
- clk_set_parent(clks[IMX6SL_PLL2_BYPASS], clks[IMX6SL_CLK_PLL2]);
- clk_set_parent(clks[IMX6SL_PLL3_BYPASS], clks[IMX6SL_CLK_PLL3]);
- clk_set_parent(clks[IMX6SL_PLL4_BYPASS], clks[IMX6SL_CLK_PLL4]);
- clk_set_parent(clks[IMX6SL_PLL5_BYPASS], clks[IMX6SL_CLK_PLL5]);
- clk_set_parent(clks[IMX6SL_PLL6_BYPASS], clks[IMX6SL_CLK_PLL6]);
- clk_set_parent(clks[IMX6SL_PLL7_BYPASS], clks[IMX6SL_CLK_PLL7]);
-
- clks[IMX6SL_CLK_PLL1_SYS] = imx_clk_gate("pll1_sys", "pll1_bypass", base + 0x00, 13);
- clks[IMX6SL_CLK_PLL2_BUS] = imx_clk_gate("pll2_bus", "pll2_bypass", base + 0x30, 13);
- clks[IMX6SL_CLK_PLL3_USB_OTG] = imx_clk_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13);
- clks[IMX6SL_CLK_PLL4_AUDIO] = imx_clk_gate("pll4_audio", "pll4_bypass", base + 0x70, 13);
- clks[IMX6SL_CLK_PLL5_VIDEO] = imx_clk_gate("pll5_video", "pll5_bypass", base + 0xa0, 13);
- clks[IMX6SL_CLK_PLL6_ENET] = imx_clk_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13);
- clks[IMX6SL_CLK_PLL7_USB_HOST] = imx_clk_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13);
-
- clks[IMX6SL_CLK_LVDS1_SEL] = imx_clk_mux("lvds1_sel", base + 0x160, 0, 5, lvds_sels, ARRAY_SIZE(lvds_sels));
- clks[IMX6SL_CLK_LVDS1_OUT] = imx_clk_gate_exclusive("lvds1_out", "lvds1_sel", base + 0x160, 10, BIT(12));
- clks[IMX6SL_CLK_LVDS1_IN] = imx_clk_gate_exclusive("lvds1_in", "anaclk1", base + 0x160, 12, BIT(10));
+ clk_set_parent(hws[IMX6SL_PLL1_BYPASS]->clk, hws[IMX6SL_CLK_PLL1]->clk);
+ clk_set_parent(hws[IMX6SL_PLL2_BYPASS]->clk, hws[IMX6SL_CLK_PLL2]->clk);
+ clk_set_parent(hws[IMX6SL_PLL3_BYPASS]->clk, hws[IMX6SL_CLK_PLL3]->clk);
+ clk_set_parent(hws[IMX6SL_PLL4_BYPASS]->clk, hws[IMX6SL_CLK_PLL4]->clk);
+ clk_set_parent(hws[IMX6SL_PLL5_BYPASS]->clk, hws[IMX6SL_CLK_PLL5]->clk);
+ clk_set_parent(hws[IMX6SL_PLL6_BYPASS]->clk, hws[IMX6SL_CLK_PLL6]->clk);
+ clk_set_parent(hws[IMX6SL_PLL7_BYPASS]->clk, hws[IMX6SL_CLK_PLL7]->clk);
+
+ hws[IMX6SL_CLK_PLL1_SYS] = imx_clk_hw_gate("pll1_sys", "pll1_bypass", base + 0x00, 13);
+ hws[IMX6SL_CLK_PLL2_BUS] = imx_clk_hw_gate("pll2_bus", "pll2_bypass", base + 0x30, 13);
+ hws[IMX6SL_CLK_PLL3_USB_OTG] = imx_clk_hw_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13);
+ hws[IMX6SL_CLK_PLL4_AUDIO] = imx_clk_hw_gate("pll4_audio", "pll4_bypass", base + 0x70, 13);
+ hws[IMX6SL_CLK_PLL5_VIDEO] = imx_clk_hw_gate("pll5_video", "pll5_bypass", base + 0xa0, 13);
+ hws[IMX6SL_CLK_PLL6_ENET] = imx_clk_hw_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13);
+ hws[IMX6SL_CLK_PLL7_USB_HOST] = imx_clk_hw_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13);
+
+ hws[IMX6SL_CLK_LVDS1_SEL] = imx_clk_hw_mux("lvds1_sel", base + 0x160, 0, 5, lvds_sels, ARRAY_SIZE(lvds_sels));
+ hws[IMX6SL_CLK_LVDS1_OUT] = imx_clk_hw_gate_exclusive("lvds1_out", "lvds1_sel", base + 0x160, 10, BIT(12));
+ hws[IMX6SL_CLK_LVDS1_IN] = imx_clk_hw_gate_exclusive("lvds1_in", "anaclk1", base + 0x160, 12, BIT(10));
/*
* usbphy1 and usbphy2 are implemented as dummy gates using reserve
@@ -255,32 +263,32 @@ static void __init imx6sl_clocks_init(struct device_node *ccm_node)
* turned on during boot, and software will not need to control it
* anymore after that.
*/
- clks[IMX6SL_CLK_USBPHY1] = imx_clk_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20);
- clks[IMX6SL_CLK_USBPHY2] = imx_clk_gate("usbphy2", "pll7_usb_host", base + 0x20, 20);
- clks[IMX6SL_CLK_USBPHY1_GATE] = imx_clk_gate("usbphy1_gate", "dummy", base + 0x10, 6);
- clks[IMX6SL_CLK_USBPHY2_GATE] = imx_clk_gate("usbphy2_gate", "dummy", base + 0x20, 6);
+ hws[IMX6SL_CLK_USBPHY1] = imx_clk_hw_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20);
+ hws[IMX6SL_CLK_USBPHY2] = imx_clk_hw_gate("usbphy2", "pll7_usb_host", base + 0x20, 20);
+ hws[IMX6SL_CLK_USBPHY1_GATE] = imx_clk_hw_gate("usbphy1_gate", "dummy", base + 0x10, 6);
+ hws[IMX6SL_CLK_USBPHY2_GATE] = imx_clk_hw_gate("usbphy2_gate", "dummy", base + 0x20, 6);
/* dev name parent_name flags reg shift width div: flags, div_table lock */
- clks[IMX6SL_CLK_PLL4_POST_DIV] = clk_register_divider_table(NULL, "pll4_post_div", "pll4_audio", CLK_SET_RATE_PARENT, base + 0x70, 19, 2, 0, post_div_table, &imx_ccm_lock);
- clks[IMX6SL_CLK_PLL4_AUDIO_DIV] = clk_register_divider(NULL, "pll4_audio_div", "pll4_post_div", CLK_SET_RATE_PARENT, base + 0x170, 15, 1, 0, &imx_ccm_lock);
- clks[IMX6SL_CLK_PLL5_POST_DIV] = clk_register_divider_table(NULL, "pll5_post_div", "pll5_video", CLK_SET_RATE_PARENT, base + 0xa0, 19, 2, 0, post_div_table, &imx_ccm_lock);
- clks[IMX6SL_CLK_PLL5_VIDEO_DIV] = clk_register_divider_table(NULL, "pll5_video_div", "pll5_post_div", CLK_SET_RATE_PARENT, base + 0x170, 30, 2, 0, video_div_table, &imx_ccm_lock);
- clks[IMX6SL_CLK_ENET_REF] = clk_register_divider_table(NULL, "enet_ref", "pll6_enet", 0, base + 0xe0, 0, 2, 0, clk_enet_ref_table, &imx_ccm_lock);
+ hws[IMX6SL_CLK_PLL4_POST_DIV] = clk_hw_register_divider_table(NULL, "pll4_post_div", "pll4_audio", CLK_SET_RATE_PARENT, base + 0x70, 19, 2, 0, post_div_table, &imx_ccm_lock);
+ hws[IMX6SL_CLK_PLL4_AUDIO_DIV] = clk_hw_register_divider(NULL, "pll4_audio_div", "pll4_post_div", CLK_SET_RATE_PARENT, base + 0x170, 15, 1, 0, &imx_ccm_lock);
+ hws[IMX6SL_CLK_PLL5_POST_DIV] = clk_hw_register_divider_table(NULL, "pll5_post_div", "pll5_video", CLK_SET_RATE_PARENT, base + 0xa0, 19, 2, 0, post_div_table, &imx_ccm_lock);
+ hws[IMX6SL_CLK_PLL5_VIDEO_DIV] = clk_hw_register_divider_table(NULL, "pll5_video_div", "pll5_post_div", CLK_SET_RATE_PARENT, base + 0x170, 30, 2, 0, video_div_table, &imx_ccm_lock);
+ hws[IMX6SL_CLK_ENET_REF] = clk_hw_register_divider_table(NULL, "enet_ref", "pll6_enet", 0, base + 0xe0, 0, 2, 0, clk_enet_ref_table, &imx_ccm_lock);
/* name parent_name reg idx */
- clks[IMX6SL_CLK_PLL2_PFD0] = imx_clk_pfd("pll2_pfd0", "pll2_bus", base + 0x100, 0);
- clks[IMX6SL_CLK_PLL2_PFD1] = imx_clk_pfd("pll2_pfd1", "pll2_bus", base + 0x100, 1);
- clks[IMX6SL_CLK_PLL2_PFD2] = imx_clk_pfd("pll2_pfd2", "pll2_bus", base + 0x100, 2);
- clks[IMX6SL_CLK_PLL3_PFD0] = imx_clk_pfd("pll3_pfd0", "pll3_usb_otg", base + 0xf0, 0);
- clks[IMX6SL_CLK_PLL3_PFD1] = imx_clk_pfd("pll3_pfd1", "pll3_usb_otg", base + 0xf0, 1);
- clks[IMX6SL_CLK_PLL3_PFD2] = imx_clk_pfd("pll3_pfd2", "pll3_usb_otg", base + 0xf0, 2);
- clks[IMX6SL_CLK_PLL3_PFD3] = imx_clk_pfd("pll3_pfd3", "pll3_usb_otg", base + 0xf0, 3);
+ hws[IMX6SL_CLK_PLL2_PFD0] = imx_clk_hw_pfd("pll2_pfd0", "pll2_bus", base + 0x100, 0);
+ hws[IMX6SL_CLK_PLL2_PFD1] = imx_clk_hw_pfd("pll2_pfd1", "pll2_bus", base + 0x100, 1);
+ hws[IMX6SL_CLK_PLL2_PFD2] = imx_clk_hw_pfd("pll2_pfd2", "pll2_bus", base + 0x100, 2);
+ hws[IMX6SL_CLK_PLL3_PFD0] = imx_clk_hw_pfd("pll3_pfd0", "pll3_usb_otg", base + 0xf0, 0);
+ hws[IMX6SL_CLK_PLL3_PFD1] = imx_clk_hw_pfd("pll3_pfd1", "pll3_usb_otg", base + 0xf0, 1);
+ hws[IMX6SL_CLK_PLL3_PFD2] = imx_clk_hw_pfd("pll3_pfd2", "pll3_usb_otg", base + 0xf0, 2);
+ hws[IMX6SL_CLK_PLL3_PFD3] = imx_clk_hw_pfd("pll3_pfd3", "pll3_usb_otg", base + 0xf0, 3);
/* name parent_name mult div */
- clks[IMX6SL_CLK_PLL2_198M] = imx_clk_fixed_factor("pll2_198m", "pll2_pfd2", 1, 2);
- clks[IMX6SL_CLK_PLL3_120M] = imx_clk_fixed_factor("pll3_120m", "pll3_usb_otg", 1, 4);
- clks[IMX6SL_CLK_PLL3_80M] = imx_clk_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6);
- clks[IMX6SL_CLK_PLL3_60M] = imx_clk_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8);
+ hws[IMX6SL_CLK_PLL2_198M] = imx_clk_hw_fixed_factor("pll2_198m", "pll2_pfd2", 1, 2);
+ hws[IMX6SL_CLK_PLL3_120M] = imx_clk_hw_fixed_factor("pll3_120m", "pll3_usb_otg", 1, 4);
+ hws[IMX6SL_CLK_PLL3_80M] = imx_clk_hw_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6);
+ hws[IMX6SL_CLK_PLL3_60M] = imx_clk_hw_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8);
np = ccm_node;
base = of_iomap(np, 0);
@@ -288,157 +296,160 @@ static void __init imx6sl_clocks_init(struct device_node *ccm_node)
ccm_base = base;
/* name reg shift width parent_names num_parents */
- clks[IMX6SL_CLK_STEP] = imx_clk_mux("step", base + 0xc, 8, 1, step_sels, ARRAY_SIZE(step_sels));
- clks[IMX6SL_CLK_PLL1_SW] = imx_clk_mux("pll1_sw", base + 0xc, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels));
- clks[IMX6SL_CLK_OCRAM_ALT_SEL] = imx_clk_mux("ocram_alt_sel", base + 0x14, 7, 1, ocram_alt_sels, ARRAY_SIZE(ocram_alt_sels));
- clks[IMX6SL_CLK_OCRAM_SEL] = imx_clk_mux("ocram_sel", base + 0x14, 6, 1, ocram_sels, ARRAY_SIZE(ocram_sels));
- clks[IMX6SL_CLK_PRE_PERIPH2_SEL] = imx_clk_mux("pre_periph2_sel", base + 0x18, 21, 2, pre_periph_sels, ARRAY_SIZE(pre_periph_sels));
- clks[IMX6SL_CLK_PRE_PERIPH_SEL] = imx_clk_mux("pre_periph_sel", base + 0x18, 18, 2, pre_periph_sels, ARRAY_SIZE(pre_periph_sels));
- clks[IMX6SL_CLK_PERIPH2_CLK2_SEL] = imx_clk_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels));
- clks[IMX6SL_CLK_PERIPH_CLK2_SEL] = imx_clk_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels));
- clks[IMX6SL_CLK_CSI_SEL] = imx_clk_mux("csi_sel", base + 0x3c, 9, 2, csi_sels, ARRAY_SIZE(csi_sels));
- clks[IMX6SL_CLK_LCDIF_AXI_SEL] = imx_clk_mux("lcdif_axi_sel", base + 0x3c, 14, 2, lcdif_axi_sels, ARRAY_SIZE(lcdif_axi_sels));
- clks[IMX6SL_CLK_USDHC1_SEL] = imx_clk_fixup_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup);
- clks[IMX6SL_CLK_USDHC2_SEL] = imx_clk_fixup_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup);
- clks[IMX6SL_CLK_USDHC3_SEL] = imx_clk_fixup_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup);
- clks[IMX6SL_CLK_USDHC4_SEL] = imx_clk_fixup_mux("usdhc4_sel", base + 0x1c, 19, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup);
- clks[IMX6SL_CLK_SSI1_SEL] = imx_clk_fixup_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup);
- clks[IMX6SL_CLK_SSI2_SEL] = imx_clk_fixup_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup);
- clks[IMX6SL_CLK_SSI3_SEL] = imx_clk_fixup_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup);
- clks[IMX6SL_CLK_PERCLK_SEL] = imx_clk_fixup_mux("perclk_sel", base + 0x1c, 6, 1, perclk_sels, ARRAY_SIZE(perclk_sels), imx_cscmr1_fixup);
- clks[IMX6SL_CLK_PXP_AXI_SEL] = imx_clk_mux("pxp_axi_sel", base + 0x34, 6, 3, pxp_axi_sels, ARRAY_SIZE(pxp_axi_sels));
- clks[IMX6SL_CLK_EPDC_AXI_SEL] = imx_clk_mux("epdc_axi_sel", base + 0x34, 15, 3, epdc_axi_sels, ARRAY_SIZE(epdc_axi_sels));
- clks[IMX6SL_CLK_GPU2D_OVG_SEL] = imx_clk_mux("gpu2d_ovg_sel", base + 0x18, 4, 2, gpu2d_ovg_sels, ARRAY_SIZE(gpu2d_ovg_sels));
- clks[IMX6SL_CLK_GPU2D_SEL] = imx_clk_mux("gpu2d_sel", base + 0x18, 8, 2, gpu2d_sels, ARRAY_SIZE(gpu2d_sels));
- clks[IMX6SL_CLK_LCDIF_PIX_SEL] = imx_clk_mux("lcdif_pix_sel", base + 0x38, 6, 3, lcdif_pix_sels, ARRAY_SIZE(lcdif_pix_sels));
- clks[IMX6SL_CLK_EPDC_PIX_SEL] = imx_clk_mux("epdc_pix_sel", base + 0x38, 15, 3, epdc_pix_sels, ARRAY_SIZE(epdc_pix_sels));
- clks[IMX6SL_CLK_SPDIF0_SEL] = imx_clk_mux("spdif0_sel", base + 0x30, 20, 2, audio_sels, ARRAY_SIZE(audio_sels));
- clks[IMX6SL_CLK_SPDIF1_SEL] = imx_clk_mux("spdif1_sel", base + 0x30, 7, 2, audio_sels, ARRAY_SIZE(audio_sels));
- clks[IMX6SL_CLK_EXTERN_AUDIO_SEL] = imx_clk_mux("extern_audio_sel", base + 0x20, 19, 2, audio_sels, ARRAY_SIZE(audio_sels));
- clks[IMX6SL_CLK_ECSPI_SEL] = imx_clk_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels));
- clks[IMX6SL_CLK_UART_SEL] = imx_clk_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels));
+ hws[IMX6SL_CLK_STEP] = imx_clk_hw_mux("step", base + 0xc, 8, 1, step_sels, ARRAY_SIZE(step_sels));
+ hws[IMX6SL_CLK_PLL1_SW] = imx_clk_hw_mux("pll1_sw", base + 0xc, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels));
+ hws[IMX6SL_CLK_OCRAM_ALT_SEL] = imx_clk_hw_mux("ocram_alt_sel", base + 0x14, 7, 1, ocram_alt_sels, ARRAY_SIZE(ocram_alt_sels));
+ hws[IMX6SL_CLK_OCRAM_SEL] = imx_clk_hw_mux("ocram_sel", base + 0x14, 6, 1, ocram_sels, ARRAY_SIZE(ocram_sels));
+ hws[IMX6SL_CLK_PRE_PERIPH2_SEL] = imx_clk_hw_mux("pre_periph2_sel", base + 0x18, 21, 2, pre_periph_sels, ARRAY_SIZE(pre_periph_sels));
+ hws[IMX6SL_CLK_PRE_PERIPH_SEL] = imx_clk_hw_mux("pre_periph_sel", base + 0x18, 18, 2, pre_periph_sels, ARRAY_SIZE(pre_periph_sels));
+ hws[IMX6SL_CLK_PERIPH2_CLK2_SEL] = imx_clk_hw_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels));
+ hws[IMX6SL_CLK_PERIPH_CLK2_SEL] = imx_clk_hw_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels));
+ hws[IMX6SL_CLK_CSI_SEL] = imx_clk_hw_mux("csi_sel", base + 0x3c, 9, 2, csi_sels, ARRAY_SIZE(csi_sels));
+ hws[IMX6SL_CLK_LCDIF_AXI_SEL] = imx_clk_hw_mux("lcdif_axi_sel", base + 0x3c, 14, 2, lcdif_axi_sels, ARRAY_SIZE(lcdif_axi_sels));
+ hws[IMX6SL_CLK_USDHC1_SEL] = imx_clk_hw_fixup_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup);
+ hws[IMX6SL_CLK_USDHC2_SEL] = imx_clk_hw_fixup_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup);
+ hws[IMX6SL_CLK_USDHC3_SEL] = imx_clk_hw_fixup_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup);
+ hws[IMX6SL_CLK_USDHC4_SEL] = imx_clk_hw_fixup_mux("usdhc4_sel", base + 0x1c, 19, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup);
+ hws[IMX6SL_CLK_SSI1_SEL] = imx_clk_hw_fixup_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup);
+ hws[IMX6SL_CLK_SSI2_SEL] = imx_clk_hw_fixup_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup);
+ hws[IMX6SL_CLK_SSI3_SEL] = imx_clk_hw_fixup_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup);
+ hws[IMX6SL_CLK_PERCLK_SEL] = imx_clk_hw_fixup_mux("perclk_sel", base + 0x1c, 6, 1, perclk_sels, ARRAY_SIZE(perclk_sels), imx_cscmr1_fixup);
+ hws[IMX6SL_CLK_PXP_AXI_SEL] = imx_clk_hw_mux("pxp_axi_sel", base + 0x34, 6, 3, pxp_axi_sels, ARRAY_SIZE(pxp_axi_sels));
+ hws[IMX6SL_CLK_EPDC_AXI_SEL] = imx_clk_hw_mux("epdc_axi_sel", base + 0x34, 15, 3, epdc_axi_sels, ARRAY_SIZE(epdc_axi_sels));
+ hws[IMX6SL_CLK_GPU2D_OVG_SEL] = imx_clk_hw_mux("gpu2d_ovg_sel", base + 0x18, 4, 2, gpu2d_ovg_sels, ARRAY_SIZE(gpu2d_ovg_sels));
+ hws[IMX6SL_CLK_GPU2D_SEL] = imx_clk_hw_mux("gpu2d_sel", base + 0x18, 8, 2, gpu2d_sels, ARRAY_SIZE(gpu2d_sels));
+ hws[IMX6SL_CLK_LCDIF_PIX_SEL] = imx_clk_hw_mux("lcdif_pix_sel", base + 0x38, 6, 3, lcdif_pix_sels, ARRAY_SIZE(lcdif_pix_sels));
+ hws[IMX6SL_CLK_EPDC_PIX_SEL] = imx_clk_hw_mux("epdc_pix_sel", base + 0x38, 15, 3, epdc_pix_sels, ARRAY_SIZE(epdc_pix_sels));
+ hws[IMX6SL_CLK_SPDIF0_SEL] = imx_clk_hw_mux("spdif0_sel", base + 0x30, 20, 2, audio_sels, ARRAY_SIZE(audio_sels));
+ hws[IMX6SL_CLK_SPDIF1_SEL] = imx_clk_hw_mux("spdif1_sel", base + 0x30, 7, 2, audio_sels, ARRAY_SIZE(audio_sels));
+ hws[IMX6SL_CLK_EXTERN_AUDIO_SEL] = imx_clk_hw_mux("extern_audio_sel", base + 0x20, 19, 2, audio_sels, ARRAY_SIZE(audio_sels));
+ hws[IMX6SL_CLK_ECSPI_SEL] = imx_clk_hw_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels));
+ hws[IMX6SL_CLK_UART_SEL] = imx_clk_hw_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels));
/* name reg shift width busy: reg, shift parent_names num_parents */
- clks[IMX6SL_CLK_PERIPH] = imx_clk_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels));
- clks[IMX6SL_CLK_PERIPH2] = imx_clk_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels));
+ hws[IMX6SL_CLK_PERIPH] = imx_clk_hw_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels));
+ hws[IMX6SL_CLK_PERIPH2] = imx_clk_hw_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels));
/* name parent_name reg shift width */
- clks[IMX6SL_CLK_OCRAM_PODF] = imx_clk_busy_divider("ocram_podf", "ocram_sel", base + 0x14, 16, 3, base + 0x48, 0);
- clks[IMX6SL_CLK_PERIPH_CLK2_PODF] = imx_clk_divider("periph_clk2_podf", "periph_clk2_sel", base + 0x14, 27, 3);
- clks[IMX6SL_CLK_PERIPH2_CLK2_PODF] = imx_clk_divider("periph2_clk2_podf", "periph2_clk2_sel", base + 0x14, 0, 3);
- clks[IMX6SL_CLK_IPG] = imx_clk_divider("ipg", "ahb", base + 0x14, 8, 2);
- clks[IMX6SL_CLK_CSI_PODF] = imx_clk_divider("csi_podf", "csi_sel", base + 0x3c, 11, 3);
- clks[IMX6SL_CLK_LCDIF_AXI_PODF] = imx_clk_divider("lcdif_axi_podf", "lcdif_axi_sel", base + 0x3c, 16, 3);
- clks[IMX6SL_CLK_USDHC1_PODF] = imx_clk_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3);
- clks[IMX6SL_CLK_USDHC2_PODF] = imx_clk_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3);
- clks[IMX6SL_CLK_USDHC3_PODF] = imx_clk_divider("usdhc3_podf", "usdhc3_sel", base + 0x24, 19, 3);
- clks[IMX6SL_CLK_USDHC4_PODF] = imx_clk_divider("usdhc4_podf", "usdhc4_sel", base + 0x24, 22, 3);
- clks[IMX6SL_CLK_SSI1_PRED] = imx_clk_divider("ssi1_pred", "ssi1_sel", base + 0x28, 6, 3);
- clks[IMX6SL_CLK_SSI1_PODF] = imx_clk_divider("ssi1_podf", "ssi1_pred", base + 0x28, 0, 6);
- clks[IMX6SL_CLK_SSI2_PRED] = imx_clk_divider("ssi2_pred", "ssi2_sel", base + 0x2c, 6, 3);
- clks[IMX6SL_CLK_SSI2_PODF] = imx_clk_divider("ssi2_podf", "ssi2_pred", base + 0x2c, 0, 6);
- clks[IMX6SL_CLK_SSI3_PRED] = imx_clk_divider("ssi3_pred", "ssi3_sel", base + 0x28, 22, 3);
- clks[IMX6SL_CLK_SSI3_PODF] = imx_clk_divider("ssi3_podf", "ssi3_pred", base + 0x28, 16, 6);
- clks[IMX6SL_CLK_PERCLK] = imx_clk_fixup_divider("perclk", "perclk_sel", base + 0x1c, 0, 6, imx_cscmr1_fixup);
- clks[IMX6SL_CLK_PXP_AXI_PODF] = imx_clk_divider("pxp_axi_podf", "pxp_axi_sel", base + 0x34, 3, 3);
- clks[IMX6SL_CLK_EPDC_AXI_PODF] = imx_clk_divider("epdc_axi_podf", "epdc_axi_sel", base + 0x34, 12, 3);
- clks[IMX6SL_CLK_GPU2D_OVG_PODF] = imx_clk_divider("gpu2d_ovg_podf", "gpu2d_ovg_sel", base + 0x18, 26, 3);
- clks[IMX6SL_CLK_GPU2D_PODF] = imx_clk_divider("gpu2d_podf", "gpu2d_sel", base + 0x18, 29, 3);
- clks[IMX6SL_CLK_LCDIF_PIX_PRED] = imx_clk_divider("lcdif_pix_pred", "lcdif_pix_sel", base + 0x38, 3, 3);
- clks[IMX6SL_CLK_EPDC_PIX_PRED] = imx_clk_divider("epdc_pix_pred", "epdc_pix_sel", base + 0x38, 12, 3);
- clks[IMX6SL_CLK_LCDIF_PIX_PODF] = imx_clk_fixup_divider("lcdif_pix_podf", "lcdif_pix_pred", base + 0x1c, 20, 3, imx_cscmr1_fixup);
- clks[IMX6SL_CLK_EPDC_PIX_PODF] = imx_clk_divider("epdc_pix_podf", "epdc_pix_pred", base + 0x18, 23, 3);
- clks[IMX6SL_CLK_SPDIF0_PRED] = imx_clk_divider("spdif0_pred", "spdif0_sel", base + 0x30, 25, 3);
- clks[IMX6SL_CLK_SPDIF0_PODF] = imx_clk_divider("spdif0_podf", "spdif0_pred", base + 0x30, 22, 3);
- clks[IMX6SL_CLK_SPDIF1_PRED] = imx_clk_divider("spdif1_pred", "spdif1_sel", base + 0x30, 12, 3);
- clks[IMX6SL_CLK_SPDIF1_PODF] = imx_clk_divider("spdif1_podf", "spdif1_pred", base + 0x30, 9, 3);
- clks[IMX6SL_CLK_EXTERN_AUDIO_PRED] = imx_clk_divider("extern_audio_pred", "extern_audio_sel", base + 0x28, 9, 3);
- clks[IMX6SL_CLK_EXTERN_AUDIO_PODF] = imx_clk_divider("extern_audio_podf", "extern_audio_pred", base + 0x28, 25, 3);
- clks[IMX6SL_CLK_ECSPI_ROOT] = imx_clk_divider("ecspi_root", "ecspi_sel", base + 0x38, 19, 6);
- clks[IMX6SL_CLK_UART_ROOT] = imx_clk_divider("uart_root", "uart_sel", base + 0x24, 0, 6);
+ hws[IMX6SL_CLK_OCRAM_PODF] = imx_clk_hw_busy_divider("ocram_podf", "ocram_sel", base + 0x14, 16, 3, base + 0x48, 0);
+ hws[IMX6SL_CLK_PERIPH_CLK2_PODF] = imx_clk_hw_divider("periph_clk2_podf", "periph_clk2_sel", base + 0x14, 27, 3);
+ hws[IMX6SL_CLK_PERIPH2_CLK2_PODF] = imx_clk_hw_divider("periph2_clk2_podf", "periph2_clk2_sel", base + 0x14, 0, 3);
+ hws[IMX6SL_CLK_IPG] = imx_clk_hw_divider("ipg", "ahb", base + 0x14, 8, 2);
+ hws[IMX6SL_CLK_CSI_PODF] = imx_clk_hw_divider("csi_podf", "csi_sel", base + 0x3c, 11, 3);
+ hws[IMX6SL_CLK_LCDIF_AXI_PODF] = imx_clk_hw_divider("lcdif_axi_podf", "lcdif_axi_sel", base + 0x3c, 16, 3);
+ hws[IMX6SL_CLK_USDHC1_PODF] = imx_clk_hw_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3);
+ hws[IMX6SL_CLK_USDHC2_PODF] = imx_clk_hw_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3);
+ hws[IMX6SL_CLK_USDHC3_PODF] = imx_clk_hw_divider("usdhc3_podf", "usdhc3_sel", base + 0x24, 19, 3);
+ hws[IMX6SL_CLK_USDHC4_PODF] = imx_clk_hw_divider("usdhc4_podf", "usdhc4_sel", base + 0x24, 22, 3);
+ hws[IMX6SL_CLK_SSI1_PRED] = imx_clk_hw_divider("ssi1_pred", "ssi1_sel", base + 0x28, 6, 3);
+ hws[IMX6SL_CLK_SSI1_PODF] = imx_clk_hw_divider("ssi1_podf", "ssi1_pred", base + 0x28, 0, 6);
+ hws[IMX6SL_CLK_SSI2_PRED] = imx_clk_hw_divider("ssi2_pred", "ssi2_sel", base + 0x2c, 6, 3);
+ hws[IMX6SL_CLK_SSI2_PODF] = imx_clk_hw_divider("ssi2_podf", "ssi2_pred", base + 0x2c, 0, 6);
+ hws[IMX6SL_CLK_SSI3_PRED] = imx_clk_hw_divider("ssi3_pred", "ssi3_sel", base + 0x28, 22, 3);
+ hws[IMX6SL_CLK_SSI3_PODF] = imx_clk_hw_divider("ssi3_podf", "ssi3_pred", base + 0x28, 16, 6);
+ hws[IMX6SL_CLK_PERCLK] = imx_clk_hw_fixup_divider("perclk", "perclk_sel", base + 0x1c, 0, 6, imx_cscmr1_fixup);
+ hws[IMX6SL_CLK_PXP_AXI_PODF] = imx_clk_hw_divider("pxp_axi_podf", "pxp_axi_sel", base + 0x34, 3, 3);
+ hws[IMX6SL_CLK_EPDC_AXI_PODF] = imx_clk_hw_divider("epdc_axi_podf", "epdc_axi_sel", base + 0x34, 12, 3);
+ hws[IMX6SL_CLK_GPU2D_OVG_PODF] = imx_clk_hw_divider("gpu2d_ovg_podf", "gpu2d_ovg_sel", base + 0x18, 26, 3);
+ hws[IMX6SL_CLK_GPU2D_PODF] = imx_clk_hw_divider("gpu2d_podf", "gpu2d_sel", base + 0x18, 29, 3);
+ hws[IMX6SL_CLK_LCDIF_PIX_PRED] = imx_clk_hw_divider("lcdif_pix_pred", "lcdif_pix_sel", base + 0x38, 3, 3);
+ hws[IMX6SL_CLK_EPDC_PIX_PRED] = imx_clk_hw_divider("epdc_pix_pred", "epdc_pix_sel", base + 0x38, 12, 3);
+ hws[IMX6SL_CLK_LCDIF_PIX_PODF] = imx_clk_hw_fixup_divider("lcdif_pix_podf", "lcdif_pix_pred", base + 0x1c, 20, 3, imx_cscmr1_fixup);
+ hws[IMX6SL_CLK_EPDC_PIX_PODF] = imx_clk_hw_divider("epdc_pix_podf", "epdc_pix_pred", base + 0x18, 23, 3);
+ hws[IMX6SL_CLK_SPDIF0_PRED] = imx_clk_hw_divider("spdif0_pred", "spdif0_sel", base + 0x30, 25, 3);
+ hws[IMX6SL_CLK_SPDIF0_PODF] = imx_clk_hw_divider("spdif0_podf", "spdif0_pred", base + 0x30, 22, 3);
+ hws[IMX6SL_CLK_SPDIF1_PRED] = imx_clk_hw_divider("spdif1_pred", "spdif1_sel", base + 0x30, 12, 3);
+ hws[IMX6SL_CLK_SPDIF1_PODF] = imx_clk_hw_divider("spdif1_podf", "spdif1_pred", base + 0x30, 9, 3);
+ hws[IMX6SL_CLK_EXTERN_AUDIO_PRED] = imx_clk_hw_divider("extern_audio_pred", "extern_audio_sel", base + 0x28, 9, 3);
+ hws[IMX6SL_CLK_EXTERN_AUDIO_PODF] = imx_clk_hw_divider("extern_audio_podf", "extern_audio_pred", base + 0x28, 25, 3);
+ hws[IMX6SL_CLK_ECSPI_ROOT] = imx_clk_hw_divider("ecspi_root", "ecspi_sel", base + 0x38, 19, 6);
+ hws[IMX6SL_CLK_UART_ROOT] = imx_clk_hw_divider("uart_root", "uart_sel", base + 0x24, 0, 6);
/* name parent_name reg shift width busy: reg, shift */
- clks[IMX6SL_CLK_AHB] = imx_clk_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1);
- clks[IMX6SL_CLK_MMDC_ROOT] = imx_clk_busy_divider("mmdc", "periph2", base + 0x14, 3, 3, base + 0x48, 2);
- clks[IMX6SL_CLK_ARM] = imx_clk_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16);
+ hws[IMX6SL_CLK_AHB] = imx_clk_hw_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1);
+ hws[IMX6SL_CLK_MMDC_ROOT] = imx_clk_hw_busy_divider("mmdc", "periph2", base + 0x14, 3, 3, base + 0x48, 2);
+ hws[IMX6SL_CLK_ARM] = imx_clk_hw_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16);
/* name parent_name reg shift */
- clks[IMX6SL_CLK_ECSPI1] = imx_clk_gate2("ecspi1", "ecspi_root", base + 0x6c, 0);
- clks[IMX6SL_CLK_ECSPI2] = imx_clk_gate2("ecspi2", "ecspi_root", base + 0x6c, 2);
- clks[IMX6SL_CLK_ECSPI3] = imx_clk_gate2("ecspi3", "ecspi_root", base + 0x6c, 4);
- clks[IMX6SL_CLK_ECSPI4] = imx_clk_gate2("ecspi4", "ecspi_root", base + 0x6c, 6);
- clks[IMX6SL_CLK_ENET] = imx_clk_gate2("enet", "ipg", base + 0x6c, 10);
- clks[IMX6SL_CLK_EPIT1] = imx_clk_gate2("epit1", "perclk", base + 0x6c, 12);
- clks[IMX6SL_CLK_EPIT2] = imx_clk_gate2("epit2", "perclk", base + 0x6c, 14);
- clks[IMX6SL_CLK_EXTERN_AUDIO] = imx_clk_gate2("extern_audio", "extern_audio_podf", base + 0x6c, 16);
- clks[IMX6SL_CLK_GPT] = imx_clk_gate2("gpt", "perclk", base + 0x6c, 20);
- clks[IMX6SL_CLK_GPT_SERIAL] = imx_clk_gate2("gpt_serial", "perclk", base + 0x6c, 22);
- clks[IMX6SL_CLK_GPU2D_OVG] = imx_clk_gate2("gpu2d_ovg", "gpu2d_ovg_podf", base + 0x6c, 26);
- clks[IMX6SL_CLK_I2C1] = imx_clk_gate2("i2c1", "perclk", base + 0x70, 6);
- clks[IMX6SL_CLK_I2C2] = imx_clk_gate2("i2c2", "perclk", base + 0x70, 8);
- clks[IMX6SL_CLK_I2C3] = imx_clk_gate2("i2c3", "perclk", base + 0x70, 10);
- clks[IMX6SL_CLK_OCOTP] = imx_clk_gate2("ocotp", "ipg", base + 0x70, 12);
- clks[IMX6SL_CLK_CSI] = imx_clk_gate2("csi", "csi_podf", base + 0x74, 0);
- clks[IMX6SL_CLK_PXP_AXI] = imx_clk_gate2("pxp_axi", "pxp_axi_podf", base + 0x74, 2);
- clks[IMX6SL_CLK_EPDC_AXI] = imx_clk_gate2("epdc_axi", "epdc_axi_podf", base + 0x74, 4);
- clks[IMX6SL_CLK_LCDIF_AXI] = imx_clk_gate2("lcdif_axi", "lcdif_axi_podf", base + 0x74, 6);
- clks[IMX6SL_CLK_LCDIF_PIX] = imx_clk_gate2("lcdif_pix", "lcdif_pix_podf", base + 0x74, 8);
- clks[IMX6SL_CLK_EPDC_PIX] = imx_clk_gate2("epdc_pix", "epdc_pix_podf", base + 0x74, 10);
- clks[IMX6SL_CLK_MMDC_P0_IPG] = imx_clk_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL);
- clks[IMX6SL_CLK_MMDC_P1_IPG] = imx_clk_gate2("mmdc_p1_ipg", "ipg", base + 0x74, 26);
- clks[IMX6SL_CLK_OCRAM] = imx_clk_gate2("ocram", "ocram_podf", base + 0x74, 28);
- clks[IMX6SL_CLK_PWM1] = imx_clk_gate2("pwm1", "perclk", base + 0x78, 16);
- clks[IMX6SL_CLK_PWM2] = imx_clk_gate2("pwm2", "perclk", base + 0x78, 18);
- clks[IMX6SL_CLK_PWM3] = imx_clk_gate2("pwm3", "perclk", base + 0x78, 20);
- clks[IMX6SL_CLK_PWM4] = imx_clk_gate2("pwm4", "perclk", base + 0x78, 22);
- clks[IMX6SL_CLK_SDMA] = imx_clk_gate2("sdma", "ipg", base + 0x7c, 6);
- clks[IMX6SL_CLK_SPBA] = imx_clk_gate2("spba", "ipg", base + 0x7c, 12);
- clks[IMX6SL_CLK_SPDIF] = imx_clk_gate2_shared("spdif", "spdif0_podf", base + 0x7c, 14, &share_count_spdif);
- clks[IMX6SL_CLK_SPDIF_GCLK] = imx_clk_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_spdif);
- clks[IMX6SL_CLK_SSI1_IPG] = imx_clk_gate2_shared("ssi1_ipg", "ipg", base + 0x7c, 18, &share_count_ssi1);
- clks[IMX6SL_CLK_SSI2_IPG] = imx_clk_gate2_shared("ssi2_ipg", "ipg", base + 0x7c, 20, &share_count_ssi2);
- clks[IMX6SL_CLK_SSI3_IPG] = imx_clk_gate2_shared("ssi3_ipg", "ipg", base + 0x7c, 22, &share_count_ssi3);
- clks[IMX6SL_CLK_SSI1] = imx_clk_gate2_shared("ssi1", "ssi1_podf", base + 0x7c, 18, &share_count_ssi1);
- clks[IMX6SL_CLK_SSI2] = imx_clk_gate2_shared("ssi2", "ssi2_podf", base + 0x7c, 20, &share_count_ssi2);
- clks[IMX6SL_CLK_SSI3] = imx_clk_gate2_shared("ssi3", "ssi3_podf", base + 0x7c, 22, &share_count_ssi3);
- clks[IMX6SL_CLK_UART] = imx_clk_gate2("uart", "ipg", base + 0x7c, 24);
- clks[IMX6SL_CLK_UART_SERIAL] = imx_clk_gate2("uart_serial", "uart_root", base + 0x7c, 26);
- clks[IMX6SL_CLK_USBOH3] = imx_clk_gate2("usboh3", "ipg", base + 0x80, 0);
- clks[IMX6SL_CLK_USDHC1] = imx_clk_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2);
- clks[IMX6SL_CLK_USDHC2] = imx_clk_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4);
- clks[IMX6SL_CLK_USDHC3] = imx_clk_gate2("usdhc3", "usdhc3_podf", base + 0x80, 6);
- clks[IMX6SL_CLK_USDHC4] = imx_clk_gate2("usdhc4", "usdhc4_podf", base + 0x80, 8);
+ hws[IMX6SL_CLK_ECSPI1] = imx_clk_hw_gate2("ecspi1", "ecspi_root", base + 0x6c, 0);
+ hws[IMX6SL_CLK_ECSPI2] = imx_clk_hw_gate2("ecspi2", "ecspi_root", base + 0x6c, 2);
+ hws[IMX6SL_CLK_ECSPI3] = imx_clk_hw_gate2("ecspi3", "ecspi_root", base + 0x6c, 4);
+ hws[IMX6SL_CLK_ECSPI4] = imx_clk_hw_gate2("ecspi4", "ecspi_root", base + 0x6c, 6);
+ hws[IMX6SL_CLK_ENET] = imx_clk_hw_gate2("enet", "ipg", base + 0x6c, 10);
+ hws[IMX6SL_CLK_EPIT1] = imx_clk_hw_gate2("epit1", "perclk", base + 0x6c, 12);
+ hws[IMX6SL_CLK_EPIT2] = imx_clk_hw_gate2("epit2", "perclk", base + 0x6c, 14);
+ hws[IMX6SL_CLK_EXTERN_AUDIO] = imx_clk_hw_gate2("extern_audio", "extern_audio_podf", base + 0x6c, 16);
+ hws[IMX6SL_CLK_GPT] = imx_clk_hw_gate2("gpt", "perclk", base + 0x6c, 20);
+ hws[IMX6SL_CLK_GPT_SERIAL] = imx_clk_hw_gate2("gpt_serial", "perclk", base + 0x6c, 22);
+ hws[IMX6SL_CLK_GPU2D_OVG] = imx_clk_hw_gate2("gpu2d_ovg", "gpu2d_ovg_podf", base + 0x6c, 26);
+ hws[IMX6SL_CLK_I2C1] = imx_clk_hw_gate2("i2c1", "perclk", base + 0x70, 6);
+ hws[IMX6SL_CLK_I2C2] = imx_clk_hw_gate2("i2c2", "perclk", base + 0x70, 8);
+ hws[IMX6SL_CLK_I2C3] = imx_clk_hw_gate2("i2c3", "perclk", base + 0x70, 10);
+ hws[IMX6SL_CLK_OCOTP] = imx_clk_hw_gate2("ocotp", "ipg", base + 0x70, 12);
+ hws[IMX6SL_CLK_CSI] = imx_clk_hw_gate2("csi", "csi_podf", base + 0x74, 0);
+ hws[IMX6SL_CLK_PXP_AXI] = imx_clk_hw_gate2("pxp_axi", "pxp_axi_podf", base + 0x74, 2);
+ hws[IMX6SL_CLK_EPDC_AXI] = imx_clk_hw_gate2("epdc_axi", "epdc_axi_podf", base + 0x74, 4);
+ hws[IMX6SL_CLK_LCDIF_AXI] = imx_clk_hw_gate2("lcdif_axi", "lcdif_axi_podf", base + 0x74, 6);
+ hws[IMX6SL_CLK_LCDIF_PIX] = imx_clk_hw_gate2("lcdif_pix", "lcdif_pix_podf", base + 0x74, 8);
+ hws[IMX6SL_CLK_EPDC_PIX] = imx_clk_hw_gate2("epdc_pix", "epdc_pix_podf", base + 0x74, 10);
+ hws[IMX6SL_CLK_MMDC_P0_IPG] = imx_clk_hw_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL);
+ hws[IMX6SL_CLK_MMDC_P1_IPG] = imx_clk_hw_gate2("mmdc_p1_ipg", "ipg", base + 0x74, 26);
+ hws[IMX6SL_CLK_OCRAM] = imx_clk_hw_gate2("ocram", "ocram_podf", base + 0x74, 28);
+ hws[IMX6SL_CLK_PWM1] = imx_clk_hw_gate2("pwm1", "perclk", base + 0x78, 16);
+ hws[IMX6SL_CLK_PWM2] = imx_clk_hw_gate2("pwm2", "perclk", base + 0x78, 18);
+ hws[IMX6SL_CLK_PWM3] = imx_clk_hw_gate2("pwm3", "perclk", base + 0x78, 20);
+ hws[IMX6SL_CLK_PWM4] = imx_clk_hw_gate2("pwm4", "perclk", base + 0x78, 22);
+ hws[IMX6SL_CLK_SDMA] = imx_clk_hw_gate2("sdma", "ipg", base + 0x7c, 6);
+ hws[IMX6SL_CLK_SPBA] = imx_clk_hw_gate2("spba", "ipg", base + 0x7c, 12);
+ hws[IMX6SL_CLK_SPDIF] = imx_clk_hw_gate2_shared("spdif", "spdif0_podf", base + 0x7c, 14, &share_count_spdif);
+ hws[IMX6SL_CLK_SPDIF_GCLK] = imx_clk_hw_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_spdif);
+ hws[IMX6SL_CLK_SSI1_IPG] = imx_clk_hw_gate2_shared("ssi1_ipg", "ipg", base + 0x7c, 18, &share_count_ssi1);
+ hws[IMX6SL_CLK_SSI2_IPG] = imx_clk_hw_gate2_shared("ssi2_ipg", "ipg", base + 0x7c, 20, &share_count_ssi2);
+ hws[IMX6SL_CLK_SSI3_IPG] = imx_clk_hw_gate2_shared("ssi3_ipg", "ipg", base + 0x7c, 22, &share_count_ssi3);
+ hws[IMX6SL_CLK_SSI1] = imx_clk_hw_gate2_shared("ssi1", "ssi1_podf", base + 0x7c, 18, &share_count_ssi1);
+ hws[IMX6SL_CLK_SSI2] = imx_clk_hw_gate2_shared("ssi2", "ssi2_podf", base + 0x7c, 20, &share_count_ssi2);
+ hws[IMX6SL_CLK_SSI3] = imx_clk_hw_gate2_shared("ssi3", "ssi3_podf", base + 0x7c, 22, &share_count_ssi3);
+ hws[IMX6SL_CLK_UART] = imx_clk_hw_gate2("uart", "ipg", base + 0x7c, 24);
+ hws[IMX6SL_CLK_UART_SERIAL] = imx_clk_hw_gate2("uart_serial", "uart_root", base + 0x7c, 26);
+ hws[IMX6SL_CLK_USBOH3] = imx_clk_hw_gate2("usboh3", "ipg", base + 0x80, 0);
+ hws[IMX6SL_CLK_USDHC1] = imx_clk_hw_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2);
+ hws[IMX6SL_CLK_USDHC2] = imx_clk_hw_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4);
+ hws[IMX6SL_CLK_USDHC3] = imx_clk_hw_gate2("usdhc3", "usdhc3_podf", base + 0x80, 6);
+ hws[IMX6SL_CLK_USDHC4] = imx_clk_hw_gate2("usdhc4", "usdhc4_podf", base + 0x80, 8);
/* Ensure the MMDC CH0 handshake is bypassed */
- writel_relaxed(readl_relaxed(base + CCDR) |
- BM_CCM_CCDR_MMDC_CH0_MASK, base + CCDR);
+ imx_mmdc_mask_handshake(base, 0);
- imx_check_clocks(clks, ARRAY_SIZE(clks));
+ imx_check_clk_hws(hws, IMX6SL_CLK_END);
- clk_data.clks = clks;
- clk_data.clk_num = ARRAY_SIZE(clks);
- of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
+ of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data);
/* Ensure the AHB clk is at 132MHz. */
- ret = clk_set_rate(clks[IMX6SL_CLK_AHB], 132000000);
+ ret = clk_set_rate(hws[IMX6SL_CLK_AHB]->clk, 132000000);
if (ret)
pr_warn("%s: failed to set AHB clock rate %d!\n",
__func__, ret);
if (IS_ENABLED(CONFIG_USB_MXS_PHY)) {
- clk_prepare_enable(clks[IMX6SL_CLK_USBPHY1_GATE]);
- clk_prepare_enable(clks[IMX6SL_CLK_USBPHY2_GATE]);
+ clk_prepare_enable(hws[IMX6SL_CLK_USBPHY1_GATE]->clk);
+ clk_prepare_enable(hws[IMX6SL_CLK_USBPHY2_GATE]->clk);
}
/* Audio-related clocks configuration */
- clk_set_parent(clks[IMX6SL_CLK_SPDIF0_SEL], clks[IMX6SL_CLK_PLL3_PFD3]);
+ clk_set_parent(hws[IMX6SL_CLK_SPDIF0_SEL]->clk, hws[IMX6SL_CLK_PLL3_PFD3]->clk);
/* set PLL5 video as lcdif pix parent clock */
- clk_set_parent(clks[IMX6SL_CLK_LCDIF_PIX_SEL],
- clks[IMX6SL_CLK_PLL5_VIDEO_DIV]);
+ clk_set_parent(hws[IMX6SL_CLK_LCDIF_PIX_SEL]->clk,
+ hws[IMX6SL_CLK_PLL5_VIDEO_DIV]->clk);
- clk_set_parent(clks[IMX6SL_CLK_LCDIF_AXI_SEL],
- clks[IMX6SL_CLK_PLL2_PFD2]);
+ clk_set_parent(hws[IMX6SL_CLK_LCDIF_AXI_SEL]->clk,
+ hws[IMX6SL_CLK_PLL2_PFD2]->clk);
+
+ for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) {
+ int index = uart_clk_ids[i];
+
+ uart_clks[i] = &hws[index]->clk;
+ }
imx_register_uart_clocks(uart_clks);
}
diff --git a/drivers/clk/imx/clk-imx6sll.c b/drivers/clk/imx/clk-imx6sll.c
index 7eea448cb9a9..5f3e92c09a5e 100644
--- a/drivers/clk/imx/clk-imx6sll.c
+++ b/drivers/clk/imx/clk-imx6sll.c
@@ -7,6 +7,7 @@
#include <dt-bindings/clock/imx6sll-clock.h>
#include <linux/clk.h>
#include <linux/clkdev.h>
+#include <linux/clk-provider.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/io.h>
@@ -16,7 +17,6 @@
#include "clk.h"
#define CCM_ANALOG_PLL_BYPASS (0x1 << 16)
-#define BM_CCM_CCDR_MMDC_CH0_MASK (0x2 << 16)
#define xPLL_CLR(offset) (offset + 0x8)
static const char *pll_bypass_src_sels[] = { "osc", "dummy", };
@@ -53,8 +53,8 @@ static const char *lcdif_sels[] = { "lcdif_podf", "ipp_di0", "ipp_di1", "ldb_di0
static const char *epdc_pre_sels[] = { "pll2_bus", "pll3_usb_otg", "pll5_video_div", "pll2_pfd0_352m", "pll2_pfd2_396m", "pll3_pfd2_508m", };
static const char *epdc_sels[] = { "epdc_podf", "ipp_di0", "ipp_di1", "ldb_di0", "ldb_di1", };
-static struct clk *clks[IMX6SLL_CLK_END];
-static struct clk_onecell_data clk_data;
+static struct clk_hw **hws;
+static struct clk_hw_onecell_data *clk_hw_data;
static const struct clk_div_table post_div_table[] = {
{ .val = 2, .div = 1, },
@@ -76,33 +76,43 @@ static u32 share_count_ssi1;
static u32 share_count_ssi2;
static u32 share_count_ssi3;
-static struct clk ** const uart_clks[] __initconst = {
- &clks[IMX6SLL_CLK_UART1_IPG],
- &clks[IMX6SLL_CLK_UART1_SERIAL],
- &clks[IMX6SLL_CLK_UART2_IPG],
- &clks[IMX6SLL_CLK_UART2_SERIAL],
- &clks[IMX6SLL_CLK_UART3_IPG],
- &clks[IMX6SLL_CLK_UART3_SERIAL],
- &clks[IMX6SLL_CLK_UART4_IPG],
- &clks[IMX6SLL_CLK_UART4_SERIAL],
- &clks[IMX6SLL_CLK_UART5_IPG],
- &clks[IMX6SLL_CLK_UART5_SERIAL],
- NULL
+static const int uart_clk_ids[] __initconst = {
+ IMX6SLL_CLK_UART1_IPG,
+ IMX6SLL_CLK_UART1_SERIAL,
+ IMX6SLL_CLK_UART2_IPG,
+ IMX6SLL_CLK_UART2_SERIAL,
+ IMX6SLL_CLK_UART3_IPG,
+ IMX6SLL_CLK_UART3_SERIAL,
+ IMX6SLL_CLK_UART4_IPG,
+ IMX6SLL_CLK_UART4_SERIAL,
+ IMX6SLL_CLK_UART5_IPG,
+ IMX6SLL_CLK_UART5_SERIAL,
};
+static struct clk **uart_clks[ARRAY_SIZE(uart_clk_ids) + 1] __initdata;
+
static void __init imx6sll_clocks_init(struct device_node *ccm_node)
{
struct device_node *np;
void __iomem *base;
+ int i;
+
+ clk_hw_data = kzalloc(struct_size(clk_hw_data, hws,
+ IMX6SLL_CLK_END), GFP_KERNEL);
+ if (WARN_ON(!clk_hw_data))
+ return;
+
+ clk_hw_data->num = IMX6SLL_CLK_END;
+ hws = clk_hw_data->hws;
- clks[IMX6SLL_CLK_DUMMY] = imx_clk_fixed("dummy", 0);
+ hws[IMX6SLL_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0);
- clks[IMX6SLL_CLK_CKIL] = of_clk_get_by_name(ccm_node, "ckil");
- clks[IMX6SLL_CLK_OSC] = of_clk_get_by_name(ccm_node, "osc");
+ hws[IMX6SLL_CLK_CKIL] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ckil"));
+ hws[IMX6SLL_CLK_OSC] = __clk_get_hw(of_clk_get_by_name(ccm_node, "osc"));
/* ipp_di clock is external input */
- clks[IMX6SLL_CLK_IPP_DI0] = of_clk_get_by_name(ccm_node, "ipp_di0");
- clks[IMX6SLL_CLK_IPP_DI1] = of_clk_get_by_name(ccm_node, "ipp_di1");
+ hws[IMX6SLL_CLK_IPP_DI0] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ipp_di0"));
+ hws[IMX6SLL_CLK_IPP_DI1] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ipp_di1"));
np = of_find_compatible_node(NULL, NULL, "fsl,imx6sll-anatop");
base = of_iomap(np, 0);
@@ -118,37 +128,37 @@ static void __init imx6sll_clocks_init(struct device_node *ccm_node)
writel_relaxed(CCM_ANALOG_PLL_BYPASS, base + xPLL_CLR(0xa0));
writel_relaxed(CCM_ANALOG_PLL_BYPASS, base + xPLL_CLR(0xe0));
- clks[IMX6SLL_PLL1_BYPASS_SRC] = imx_clk_mux("pll1_bypass_src", base + 0x00, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SLL_PLL2_BYPASS_SRC] = imx_clk_mux("pll2_bypass_src", base + 0x30, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SLL_PLL3_BYPASS_SRC] = imx_clk_mux("pll3_bypass_src", base + 0x10, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SLL_PLL4_BYPASS_SRC] = imx_clk_mux("pll4_bypass_src", base + 0x70, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SLL_PLL5_BYPASS_SRC] = imx_clk_mux("pll5_bypass_src", base + 0xa0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SLL_PLL6_BYPASS_SRC] = imx_clk_mux("pll6_bypass_src", base + 0xe0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SLL_PLL7_BYPASS_SRC] = imx_clk_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
-
- clks[IMX6SLL_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "pll1_bypass_src", base + 0x00, 0x7f);
- clks[IMX6SLL_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "pll2_bypass_src", base + 0x30, 0x1);
- clks[IMX6SLL_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "pll3_bypass_src", base + 0x10, 0x3);
- clks[IMX6SLL_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "pll4_bypass_src", base + 0x70, 0x7f);
- clks[IMX6SLL_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "pll5_bypass_src", base + 0xa0, 0x7f);
- clks[IMX6SLL_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "pll6_bypass_src", base + 0xe0, 0x3);
- clks[IMX6SLL_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "pll7_bypass_src", base + 0x20, 0x3);
-
- clks[IMX6SLL_PLL1_BYPASS] = imx_clk_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SLL_PLL2_BYPASS] = imx_clk_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SLL_PLL3_BYPASS] = imx_clk_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SLL_PLL4_BYPASS] = imx_clk_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SLL_PLL5_BYPASS] = imx_clk_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SLL_PLL6_BYPASS] = imx_clk_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SLL_PLL7_BYPASS] = imx_clk_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT);
-
- clks[IMX6SLL_CLK_PLL1_SYS] = imx_clk_fixed_factor("pll1_sys", "pll1_bypass", 1, 1);
- clks[IMX6SLL_CLK_PLL2_BUS] = imx_clk_gate("pll2_bus", "pll2_bypass", base + 0x30, 13);
- clks[IMX6SLL_CLK_PLL3_USB_OTG] = imx_clk_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13);
- clks[IMX6SLL_CLK_PLL4_AUDIO] = imx_clk_gate("pll4_audio", "pll4_bypass", base + 0x70, 13);
- clks[IMX6SLL_CLK_PLL5_VIDEO] = imx_clk_gate("pll5_video", "pll5_bypass", base + 0xa0, 13);
- clks[IMX6SLL_CLK_PLL6_ENET] = imx_clk_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13);
- clks[IMX6SLL_CLK_PLL7_USB_HOST] = imx_clk_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13);
+ hws[IMX6SLL_PLL1_BYPASS_SRC] = imx_clk_hw_mux("pll1_bypass_src", base + 0x00, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SLL_PLL2_BYPASS_SRC] = imx_clk_hw_mux("pll2_bypass_src", base + 0x30, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SLL_PLL3_BYPASS_SRC] = imx_clk_hw_mux("pll3_bypass_src", base + 0x10, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SLL_PLL4_BYPASS_SRC] = imx_clk_hw_mux("pll4_bypass_src", base + 0x70, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SLL_PLL5_BYPASS_SRC] = imx_clk_hw_mux("pll5_bypass_src", base + 0xa0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SLL_PLL6_BYPASS_SRC] = imx_clk_hw_mux("pll6_bypass_src", base + 0xe0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SLL_PLL7_BYPASS_SRC] = imx_clk_hw_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+
+ hws[IMX6SLL_CLK_PLL1] = imx_clk_hw_pllv3(IMX_PLLV3_SYS, "pll1", "pll1_bypass_src", base + 0x00, 0x7f);
+ hws[IMX6SLL_CLK_PLL2] = imx_clk_hw_pllv3(IMX_PLLV3_GENERIC, "pll2", "pll2_bypass_src", base + 0x30, 0x1);
+ hws[IMX6SLL_CLK_PLL3] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll3", "pll3_bypass_src", base + 0x10, 0x3);
+ hws[IMX6SLL_CLK_PLL4] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll4", "pll4_bypass_src", base + 0x70, 0x7f);
+ hws[IMX6SLL_CLK_PLL5] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll5", "pll5_bypass_src", base + 0xa0, 0x7f);
+ hws[IMX6SLL_CLK_PLL6] = imx_clk_hw_pllv3(IMX_PLLV3_ENET, "pll6", "pll6_bypass_src", base + 0xe0, 0x3);
+ hws[IMX6SLL_CLK_PLL7] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll7", "pll7_bypass_src", base + 0x20, 0x3);
+
+ hws[IMX6SLL_PLL1_BYPASS] = imx_clk_hw_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SLL_PLL2_BYPASS] = imx_clk_hw_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SLL_PLL3_BYPASS] = imx_clk_hw_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SLL_PLL4_BYPASS] = imx_clk_hw_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SLL_PLL5_BYPASS] = imx_clk_hw_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SLL_PLL6_BYPASS] = imx_clk_hw_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SLL_PLL7_BYPASS] = imx_clk_hw_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT);
+
+ hws[IMX6SLL_CLK_PLL1_SYS] = imx_clk_hw_fixed_factor("pll1_sys", "pll1_bypass", 1, 1);
+ hws[IMX6SLL_CLK_PLL2_BUS] = imx_clk_hw_gate("pll2_bus", "pll2_bypass", base + 0x30, 13);
+ hws[IMX6SLL_CLK_PLL3_USB_OTG] = imx_clk_hw_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13);
+ hws[IMX6SLL_CLK_PLL4_AUDIO] = imx_clk_hw_gate("pll4_audio", "pll4_bypass", base + 0x70, 13);
+ hws[IMX6SLL_CLK_PLL5_VIDEO] = imx_clk_hw_gate("pll5_video", "pll5_bypass", base + 0xa0, 13);
+ hws[IMX6SLL_CLK_PLL6_ENET] = imx_clk_hw_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13);
+ hws[IMX6SLL_CLK_PLL7_USB_HOST] = imx_clk_hw_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13);
/*
* Bit 20 is the reserved and read-only bit, we do this only for:
@@ -156,209 +166,213 @@ static void __init imx6sll_clocks_init(struct device_node *ccm_node)
* - Keep refcount when do usbphy clk_enable/disable, in that case,
* the clk framework many need to enable/disable usbphy's parent
*/
- clks[IMX6SLL_CLK_USBPHY1] = imx_clk_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20);
- clks[IMX6SLL_CLK_USBPHY2] = imx_clk_gate("usbphy2", "pll7_usb_host", base + 0x20, 20);
+ hws[IMX6SLL_CLK_USBPHY1] = imx_clk_hw_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20);
+ hws[IMX6SLL_CLK_USBPHY2] = imx_clk_hw_gate("usbphy2", "pll7_usb_host", base + 0x20, 20);
/*
* usbphy*_gate needs to be on after system boots up, and software
* never needs to control it anymore.
*/
if (IS_ENABLED(CONFIG_USB_MXS_PHY)) {
- clks[IMX6SLL_CLK_USBPHY1_GATE] = imx_clk_gate_flags("usbphy1_gate", "dummy", base + 0x10, 6, CLK_IS_CRITICAL);
- clks[IMX6SLL_CLK_USBPHY2_GATE] = imx_clk_gate_flags("usbphy2_gate", "dummy", base + 0x20, 6, CLK_IS_CRITICAL);
+ hws[IMX6SLL_CLK_USBPHY1_GATE] = imx_clk_hw_gate_flags("usbphy1_gate", "dummy", base + 0x10, 6, CLK_IS_CRITICAL);
+ hws[IMX6SLL_CLK_USBPHY2_GATE] = imx_clk_hw_gate_flags("usbphy2_gate", "dummy", base + 0x20, 6, CLK_IS_CRITICAL);
}
/* name parent_name reg idx */
- clks[IMX6SLL_CLK_PLL2_PFD0] = imx_clk_pfd("pll2_pfd0_352m", "pll2_bus", base + 0x100, 0);
- clks[IMX6SLL_CLK_PLL2_PFD1] = imx_clk_pfd("pll2_pfd1_594m", "pll2_bus", base + 0x100, 1);
- clks[IMX6SLL_CLK_PLL2_PFD2] = imx_clk_pfd("pll2_pfd2_396m", "pll2_bus", base + 0x100, 2);
- clks[IMX6SLL_CLK_PLL2_PFD3] = imx_clk_pfd("pll2_pfd3_594m", "pll2_bus", base + 0x100, 3);
- clks[IMX6SLL_CLK_PLL3_PFD0] = imx_clk_pfd("pll3_pfd0_720m", "pll3_usb_otg", base + 0xf0, 0);
- clks[IMX6SLL_CLK_PLL3_PFD1] = imx_clk_pfd("pll3_pfd1_540m", "pll3_usb_otg", base + 0xf0, 1);
- clks[IMX6SLL_CLK_PLL3_PFD2] = imx_clk_pfd("pll3_pfd2_508m", "pll3_usb_otg", base + 0xf0, 2);
- clks[IMX6SLL_CLK_PLL3_PFD3] = imx_clk_pfd("pll3_pfd3_454m", "pll3_usb_otg", base + 0xf0, 3);
-
- clks[IMX6SLL_CLK_PLL4_POST_DIV] = clk_register_divider_table(NULL, "pll4_post_div", "pll4_audio",
+ hws[IMX6SLL_CLK_PLL2_PFD0] = imx_clk_hw_pfd("pll2_pfd0_352m", "pll2_bus", base + 0x100, 0);
+ hws[IMX6SLL_CLK_PLL2_PFD1] = imx_clk_hw_pfd("pll2_pfd1_594m", "pll2_bus", base + 0x100, 1);
+ hws[IMX6SLL_CLK_PLL2_PFD2] = imx_clk_hw_pfd("pll2_pfd2_396m", "pll2_bus", base + 0x100, 2);
+ hws[IMX6SLL_CLK_PLL2_PFD3] = imx_clk_hw_pfd("pll2_pfd3_594m", "pll2_bus", base + 0x100, 3);
+ hws[IMX6SLL_CLK_PLL3_PFD0] = imx_clk_hw_pfd("pll3_pfd0_720m", "pll3_usb_otg", base + 0xf0, 0);
+ hws[IMX6SLL_CLK_PLL3_PFD1] = imx_clk_hw_pfd("pll3_pfd1_540m", "pll3_usb_otg", base + 0xf0, 1);
+ hws[IMX6SLL_CLK_PLL3_PFD2] = imx_clk_hw_pfd("pll3_pfd2_508m", "pll3_usb_otg", base + 0xf0, 2);
+ hws[IMX6SLL_CLK_PLL3_PFD3] = imx_clk_hw_pfd("pll3_pfd3_454m", "pll3_usb_otg", base + 0xf0, 3);
+
+ hws[IMX6SLL_CLK_PLL4_POST_DIV] = clk_hw_register_divider_table(NULL, "pll4_post_div", "pll4_audio",
CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x70, 19, 2, 0, post_div_table, &imx_ccm_lock);
- clks[IMX6SLL_CLK_PLL4_AUDIO_DIV] = clk_register_divider(NULL, "pll4_audio_div", "pll4_post_div",
+ hws[IMX6SLL_CLK_PLL4_AUDIO_DIV] = clk_hw_register_divider(NULL, "pll4_audio_div", "pll4_post_div",
CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x170, 15, 1, 0, &imx_ccm_lock);
- clks[IMX6SLL_CLK_PLL5_POST_DIV] = clk_register_divider_table(NULL, "pll5_post_div", "pll5_video",
+ hws[IMX6SLL_CLK_PLL5_POST_DIV] = clk_hw_register_divider_table(NULL, "pll5_post_div", "pll5_video",
CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0xa0, 19, 2, 0, post_div_table, &imx_ccm_lock);
- clks[IMX6SLL_CLK_PLL5_VIDEO_DIV] = clk_register_divider_table(NULL, "pll5_video_div", "pll5_post_div",
+ hws[IMX6SLL_CLK_PLL5_VIDEO_DIV] = clk_hw_register_divider_table(NULL, "pll5_video_div", "pll5_post_div",
CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x170, 30, 2, 0, video_div_table, &imx_ccm_lock);
/* name parent_name mult div */
- clks[IMX6SLL_CLK_PLL2_198M] = imx_clk_fixed_factor("pll2_198m", "pll2_pfd2_396m", 1, 2);
- clks[IMX6SLL_CLK_PLL3_120M] = imx_clk_fixed_factor("pll3_120m", "pll3_usb_otg", 1, 4);
- clks[IMX6SLL_CLK_PLL3_80M] = imx_clk_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6);
- clks[IMX6SLL_CLK_PLL3_60M] = imx_clk_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8);
+ hws[IMX6SLL_CLK_PLL2_198M] = imx_clk_hw_fixed_factor("pll2_198m", "pll2_pfd2_396m", 1, 2);
+ hws[IMX6SLL_CLK_PLL3_120M] = imx_clk_hw_fixed_factor("pll3_120m", "pll3_usb_otg", 1, 4);
+ hws[IMX6SLL_CLK_PLL3_80M] = imx_clk_hw_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6);
+ hws[IMX6SLL_CLK_PLL3_60M] = imx_clk_hw_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8);
np = ccm_node;
base = of_iomap(np, 0);
WARN_ON(!base);
- clks[IMX6SLL_CLK_STEP] = imx_clk_mux("step", base + 0x0c, 8, 1, step_sels, ARRAY_SIZE(step_sels));
- clks[IMX6SLL_CLK_PLL1_SW] = imx_clk_mux_flags("pll1_sw", base + 0x0c, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels), 0);
- clks[IMX6SLL_CLK_AXI_ALT_SEL] = imx_clk_mux("axi_alt_sel", base + 0x14, 7, 1, axi_alt_sels, ARRAY_SIZE(axi_alt_sels));
- clks[IMX6SLL_CLK_AXI_SEL] = imx_clk_mux_flags("axi_sel", base + 0x14, 6, 1, axi_sels, ARRAY_SIZE(axi_sels), 0);
- clks[IMX6SLL_CLK_PERIPH_PRE] = imx_clk_mux("periph_pre", base + 0x18, 18, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels));
- clks[IMX6SLL_CLK_PERIPH2_PRE] = imx_clk_mux("periph2_pre", base + 0x18, 21, 2, periph2_pre_sels, ARRAY_SIZE(periph2_pre_sels));
- clks[IMX6SLL_CLK_PERIPH_CLK2_SEL] = imx_clk_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels));
- clks[IMX6SLL_CLK_PERIPH2_CLK2_SEL] = imx_clk_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels));
- clks[IMX6SLL_CLK_USDHC1_SEL] = imx_clk_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
- clks[IMX6SLL_CLK_USDHC2_SEL] = imx_clk_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
- clks[IMX6SLL_CLK_USDHC3_SEL] = imx_clk_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
- clks[IMX6SLL_CLK_SSI1_SEL] = imx_clk_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
- clks[IMX6SLL_CLK_SSI2_SEL] = imx_clk_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
- clks[IMX6SLL_CLK_SSI3_SEL] = imx_clk_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
- clks[IMX6SLL_CLK_PERCLK_SEL] = imx_clk_mux("perclk_sel", base + 0x1c, 6, 1, perclk_sels, ARRAY_SIZE(perclk_sels));
- clks[IMX6SLL_CLK_UART_SEL] = imx_clk_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels));
- clks[IMX6SLL_CLK_SPDIF_SEL] = imx_clk_mux("spdif_sel", base + 0x30, 20, 2, spdif_sels, ARRAY_SIZE(spdif_sels));
- clks[IMX6SLL_CLK_EXTERN_AUDIO_SEL] = imx_clk_mux("extern_audio_sel", base + 0x30, 7, 2, spdif_sels, ARRAY_SIZE(spdif_sels));
- clks[IMX6SLL_CLK_EPDC_PRE_SEL] = imx_clk_mux("epdc_pre_sel", base + 0x34, 15, 3, epdc_pre_sels, ARRAY_SIZE(epdc_pre_sels));
- clks[IMX6SLL_CLK_EPDC_SEL] = imx_clk_mux("epdc_sel", base + 0x34, 9, 3, epdc_sels, ARRAY_SIZE(epdc_sels));
- clks[IMX6SLL_CLK_ECSPI_SEL] = imx_clk_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels));
- clks[IMX6SLL_CLK_LCDIF_PRE_SEL] = imx_clk_mux("lcdif_pre_sel", base + 0x38, 15, 3, lcdif_pre_sels, ARRAY_SIZE(lcdif_pre_sels));
- clks[IMX6SLL_CLK_LCDIF_SEL] = imx_clk_mux("lcdif_sel", base + 0x38, 9, 3, lcdif_sels, ARRAY_SIZE(lcdif_sels));
-
- clks[IMX6SLL_CLK_PERIPH] = imx_clk_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels));
- clks[IMX6SLL_CLK_PERIPH2] = imx_clk_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels));
-
- clks[IMX6SLL_CLK_PERIPH_CLK2] = imx_clk_divider("periph_clk2", "periph_clk2_sel", base + 0x14, 27, 3);
- clks[IMX6SLL_CLK_PERIPH2_CLK2] = imx_clk_divider("periph2_clk2", "periph2_clk2_sel", base + 0x14, 0, 3);
- clks[IMX6SLL_CLK_IPG] = imx_clk_divider("ipg", "ahb", base + 0x14, 8, 2);
- clks[IMX6SLL_CLK_LCDIF_PODF] = imx_clk_divider("lcdif_podf", "lcdif_pred", base + 0x18, 23, 3);
- clks[IMX6SLL_CLK_PERCLK] = imx_clk_divider("perclk", "perclk_sel", base + 0x1c, 0, 6);
- clks[IMX6SLL_CLK_USDHC3_PODF] = imx_clk_divider("usdhc3_podf", "usdhc3_sel", base + 0x24, 19, 3);
- clks[IMX6SLL_CLK_USDHC2_PODF] = imx_clk_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3);
- clks[IMX6SLL_CLK_USDHC1_PODF] = imx_clk_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3);
- clks[IMX6SLL_CLK_UART_PODF] = imx_clk_divider("uart_podf", "uart_sel", base + 0x24, 0, 6);
- clks[IMX6SLL_CLK_SSI3_PRED] = imx_clk_divider("ssi3_pred", "ssi3_sel", base + 0x28, 22, 3);
- clks[IMX6SLL_CLK_SSI3_PODF] = imx_clk_divider("ssi3_podf", "ssi3_pred", base + 0x28, 16, 6);
- clks[IMX6SLL_CLK_SSI1_PRED] = imx_clk_divider("ssi1_pred", "ssi1_sel", base + 0x28, 6, 3);
- clks[IMX6SLL_CLK_SSI1_PODF] = imx_clk_divider("ssi1_podf", "ssi1_pred", base + 0x28, 0, 6);
- clks[IMX6SLL_CLK_SSI2_PRED] = imx_clk_divider("ssi2_pred", "ssi2_sel", base + 0x2c, 6, 3);
- clks[IMX6SLL_CLK_SSI2_PODF] = imx_clk_divider("ssi2_podf", "ssi2_pred", base + 0x2c, 0, 6);
- clks[IMX6SLL_CLK_SPDIF_PRED] = imx_clk_divider("spdif_pred", "spdif_sel", base + 0x30, 25, 3);
- clks[IMX6SLL_CLK_SPDIF_PODF] = imx_clk_divider("spdif_podf", "spdif_pred", base + 0x30, 22, 3);
- clks[IMX6SLL_CLK_EXTERN_AUDIO_PRED] = imx_clk_divider("extern_audio_pred", "extern_audio_sel", base + 0x30, 12, 3);
- clks[IMX6SLL_CLK_EXTERN_AUDIO_PODF] = imx_clk_divider("extern_audio_podf", "extern_audio_pred", base + 0x30, 9, 3);
- clks[IMX6SLL_CLK_EPDC_PODF] = imx_clk_divider("epdc_podf", "epdc_pre_sel", base + 0x34, 12, 3);
- clks[IMX6SLL_CLK_ECSPI_PODF] = imx_clk_divider("ecspi_podf", "ecspi_sel", base + 0x38, 19, 6);
- clks[IMX6SLL_CLK_LCDIF_PRED] = imx_clk_divider("lcdif_pred", "lcdif_pre_sel", base + 0x38, 12, 3);
-
- clks[IMX6SLL_CLK_ARM] = imx_clk_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16);
- clks[IMX6SLL_CLK_MMDC_PODF] = imx_clk_busy_divider("mmdc_podf", "periph2", base + 0x14, 3, 3, base + 0x48, 2);
- clks[IMX6SLL_CLK_AXI_PODF] = imx_clk_busy_divider("axi", "axi_sel", base + 0x14, 16, 3, base + 0x48, 0);
- clks[IMX6SLL_CLK_AHB] = imx_clk_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1);
-
- clks[IMX6SLL_CLK_LDB_DI0_DIV_3_5] = imx_clk_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7);
- clks[IMX6SLL_CLK_LDB_DI0_DIV_7] = imx_clk_fixed_factor("ldb_di0_div_7", "ldb_di0_sel", 1, 7);
- clks[IMX6SLL_CLK_LDB_DI1_DIV_3_5] = imx_clk_fixed_factor("ldb_di1_div_3_5", "ldb_di1_sel", 2, 7);
- clks[IMX6SLL_CLK_LDB_DI1_DIV_7] = imx_clk_fixed_factor("ldb_di1_div_7", "ldb_di1_sel", 1, 7);
-
- clks[IMX6SLL_CLK_LDB_DI0_SEL] = imx_clk_mux("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di0_sels, ARRAY_SIZE(ldb_di0_sels));
- clks[IMX6SLL_CLK_LDB_DI1_SEL] = imx_clk_mux("ldb_di1_sel", base + 0x1c, 7, 3, ldb_di1_sels, ARRAY_SIZE(ldb_di1_sels));
- clks[IMX6SLL_CLK_LDB_DI0_DIV_SEL] = imx_clk_mux("ldb_di0_div_sel", base + 0x20, 10, 1, ldb_di0_div_sels, ARRAY_SIZE(ldb_di0_div_sels));
- clks[IMX6SLL_CLK_LDB_DI1_DIV_SEL] = imx_clk_mux("ldb_di1_div_sel", base + 0x20, 10, 1, ldb_di1_div_sels, ARRAY_SIZE(ldb_di1_div_sels));
+ hws[IMX6SLL_CLK_STEP] = imx_clk_hw_mux("step", base + 0x0c, 8, 1, step_sels, ARRAY_SIZE(step_sels));
+ hws[IMX6SLL_CLK_PLL1_SW] = imx_clk_hw_mux_flags("pll1_sw", base + 0x0c, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels), 0);
+ hws[IMX6SLL_CLK_AXI_ALT_SEL] = imx_clk_hw_mux("axi_alt_sel", base + 0x14, 7, 1, axi_alt_sels, ARRAY_SIZE(axi_alt_sels));
+ hws[IMX6SLL_CLK_AXI_SEL] = imx_clk_hw_mux_flags("axi_sel", base + 0x14, 6, 1, axi_sels, ARRAY_SIZE(axi_sels), 0);
+ hws[IMX6SLL_CLK_PERIPH_PRE] = imx_clk_hw_mux("periph_pre", base + 0x18, 18, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels));
+ hws[IMX6SLL_CLK_PERIPH2_PRE] = imx_clk_hw_mux("periph2_pre", base + 0x18, 21, 2, periph2_pre_sels, ARRAY_SIZE(periph2_pre_sels));
+ hws[IMX6SLL_CLK_PERIPH_CLK2_SEL] = imx_clk_hw_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels));
+ hws[IMX6SLL_CLK_PERIPH2_CLK2_SEL] = imx_clk_hw_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels));
+ hws[IMX6SLL_CLK_USDHC1_SEL] = imx_clk_hw_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
+ hws[IMX6SLL_CLK_USDHC2_SEL] = imx_clk_hw_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
+ hws[IMX6SLL_CLK_USDHC3_SEL] = imx_clk_hw_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
+ hws[IMX6SLL_CLK_SSI1_SEL] = imx_clk_hw_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
+ hws[IMX6SLL_CLK_SSI2_SEL] = imx_clk_hw_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
+ hws[IMX6SLL_CLK_SSI3_SEL] = imx_clk_hw_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
+ hws[IMX6SLL_CLK_PERCLK_SEL] = imx_clk_hw_mux("perclk_sel", base + 0x1c, 6, 1, perclk_sels, ARRAY_SIZE(perclk_sels));
+ hws[IMX6SLL_CLK_UART_SEL] = imx_clk_hw_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels));
+ hws[IMX6SLL_CLK_SPDIF_SEL] = imx_clk_hw_mux("spdif_sel", base + 0x30, 20, 2, spdif_sels, ARRAY_SIZE(spdif_sels));
+ hws[IMX6SLL_CLK_EXTERN_AUDIO_SEL] = imx_clk_hw_mux("extern_audio_sel", base + 0x30, 7, 2, spdif_sels, ARRAY_SIZE(spdif_sels));
+ hws[IMX6SLL_CLK_EPDC_PRE_SEL] = imx_clk_hw_mux("epdc_pre_sel", base + 0x34, 15, 3, epdc_pre_sels, ARRAY_SIZE(epdc_pre_sels));
+ hws[IMX6SLL_CLK_EPDC_SEL] = imx_clk_hw_mux("epdc_sel", base + 0x34, 9, 3, epdc_sels, ARRAY_SIZE(epdc_sels));
+ hws[IMX6SLL_CLK_ECSPI_SEL] = imx_clk_hw_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels));
+ hws[IMX6SLL_CLK_LCDIF_PRE_SEL] = imx_clk_hw_mux("lcdif_pre_sel", base + 0x38, 15, 3, lcdif_pre_sels, ARRAY_SIZE(lcdif_pre_sels));
+ hws[IMX6SLL_CLK_LCDIF_SEL] = imx_clk_hw_mux("lcdif_sel", base + 0x38, 9, 3, lcdif_sels, ARRAY_SIZE(lcdif_sels));
+
+ hws[IMX6SLL_CLK_PERIPH] = imx_clk_hw_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels));
+ hws[IMX6SLL_CLK_PERIPH2] = imx_clk_hw_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels));
+
+ hws[IMX6SLL_CLK_PERIPH_CLK2] = imx_clk_hw_divider("periph_clk2", "periph_clk2_sel", base + 0x14, 27, 3);
+ hws[IMX6SLL_CLK_PERIPH2_CLK2] = imx_clk_hw_divider("periph2_clk2", "periph2_clk2_sel", base + 0x14, 0, 3);
+ hws[IMX6SLL_CLK_IPG] = imx_clk_hw_divider("ipg", "ahb", base + 0x14, 8, 2);
+ hws[IMX6SLL_CLK_LCDIF_PODF] = imx_clk_hw_divider("lcdif_podf", "lcdif_pred", base + 0x18, 23, 3);
+ hws[IMX6SLL_CLK_PERCLK] = imx_clk_hw_divider("perclk", "perclk_sel", base + 0x1c, 0, 6);
+ hws[IMX6SLL_CLK_USDHC3_PODF] = imx_clk_hw_divider("usdhc3_podf", "usdhc3_sel", base + 0x24, 19, 3);
+ hws[IMX6SLL_CLK_USDHC2_PODF] = imx_clk_hw_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3);
+ hws[IMX6SLL_CLK_USDHC1_PODF] = imx_clk_hw_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3);
+ hws[IMX6SLL_CLK_UART_PODF] = imx_clk_hw_divider("uart_podf", "uart_sel", base + 0x24, 0, 6);
+ hws[IMX6SLL_CLK_SSI3_PRED] = imx_clk_hw_divider("ssi3_pred", "ssi3_sel", base + 0x28, 22, 3);
+ hws[IMX6SLL_CLK_SSI3_PODF] = imx_clk_hw_divider("ssi3_podf", "ssi3_pred", base + 0x28, 16, 6);
+ hws[IMX6SLL_CLK_SSI1_PRED] = imx_clk_hw_divider("ssi1_pred", "ssi1_sel", base + 0x28, 6, 3);
+ hws[IMX6SLL_CLK_SSI1_PODF] = imx_clk_hw_divider("ssi1_podf", "ssi1_pred", base + 0x28, 0, 6);
+ hws[IMX6SLL_CLK_SSI2_PRED] = imx_clk_hw_divider("ssi2_pred", "ssi2_sel", base + 0x2c, 6, 3);
+ hws[IMX6SLL_CLK_SSI2_PODF] = imx_clk_hw_divider("ssi2_podf", "ssi2_pred", base + 0x2c, 0, 6);
+ hws[IMX6SLL_CLK_SPDIF_PRED] = imx_clk_hw_divider("spdif_pred", "spdif_sel", base + 0x30, 25, 3);
+ hws[IMX6SLL_CLK_SPDIF_PODF] = imx_clk_hw_divider("spdif_podf", "spdif_pred", base + 0x30, 22, 3);
+ hws[IMX6SLL_CLK_EXTERN_AUDIO_PRED] = imx_clk_hw_divider("extern_audio_pred", "extern_audio_sel", base + 0x30, 12, 3);
+ hws[IMX6SLL_CLK_EXTERN_AUDIO_PODF] = imx_clk_hw_divider("extern_audio_podf", "extern_audio_pred", base + 0x30, 9, 3);
+ hws[IMX6SLL_CLK_EPDC_PODF] = imx_clk_hw_divider("epdc_podf", "epdc_pre_sel", base + 0x34, 12, 3);
+ hws[IMX6SLL_CLK_ECSPI_PODF] = imx_clk_hw_divider("ecspi_podf", "ecspi_sel", base + 0x38, 19, 6);
+ hws[IMX6SLL_CLK_LCDIF_PRED] = imx_clk_hw_divider("lcdif_pred", "lcdif_pre_sel", base + 0x38, 12, 3);
+
+ hws[IMX6SLL_CLK_ARM] = imx_clk_hw_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16);
+ hws[IMX6SLL_CLK_MMDC_PODF] = imx_clk_hw_busy_divider("mmdc_podf", "periph2", base + 0x14, 3, 3, base + 0x48, 2);
+ hws[IMX6SLL_CLK_AXI_PODF] = imx_clk_hw_busy_divider("axi", "axi_sel", base + 0x14, 16, 3, base + 0x48, 0);
+ hws[IMX6SLL_CLK_AHB] = imx_clk_hw_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1);
+
+ hws[IMX6SLL_CLK_LDB_DI0_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7);
+ hws[IMX6SLL_CLK_LDB_DI0_DIV_7] = imx_clk_hw_fixed_factor("ldb_di0_div_7", "ldb_di0_sel", 1, 7);
+ hws[IMX6SLL_CLK_LDB_DI1_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di1_div_3_5", "ldb_di1_sel", 2, 7);
+ hws[IMX6SLL_CLK_LDB_DI1_DIV_7] = imx_clk_hw_fixed_factor("ldb_di1_div_7", "ldb_di1_sel", 1, 7);
+
+ hws[IMX6SLL_CLK_LDB_DI0_SEL] = imx_clk_hw_mux("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di0_sels, ARRAY_SIZE(ldb_di0_sels));
+ hws[IMX6SLL_CLK_LDB_DI1_SEL] = imx_clk_hw_mux("ldb_di1_sel", base + 0x1c, 7, 3, ldb_di1_sels, ARRAY_SIZE(ldb_di1_sels));
+ hws[IMX6SLL_CLK_LDB_DI0_DIV_SEL] = imx_clk_hw_mux("ldb_di0_div_sel", base + 0x20, 10, 1, ldb_di0_div_sels, ARRAY_SIZE(ldb_di0_div_sels));
+ hws[IMX6SLL_CLK_LDB_DI1_DIV_SEL] = imx_clk_hw_mux("ldb_di1_div_sel", base + 0x20, 10, 1, ldb_di1_div_sels, ARRAY_SIZE(ldb_di1_div_sels));
/* CCGR0 */
- clks[IMX6SLL_CLK_AIPSTZ1] = imx_clk_gate2_flags("aips_tz1", "ahb", base + 0x68, 0, CLK_IS_CRITICAL);
- clks[IMX6SLL_CLK_AIPSTZ2] = imx_clk_gate2_flags("aips_tz2", "ahb", base + 0x68, 2, CLK_IS_CRITICAL);
- clks[IMX6SLL_CLK_DCP] = imx_clk_gate2("dcp", "ahb", base + 0x68, 10);
- clks[IMX6SLL_CLK_UART2_IPG] = imx_clk_gate2("uart2_ipg", "ipg", base + 0x68, 28);
- clks[IMX6SLL_CLK_UART2_SERIAL] = imx_clk_gate2("uart2_serial", "uart_podf", base + 0x68, 28);
- clks[IMX6SLL_CLK_GPIO2] = imx_clk_gate2("gpio2", "ipg", base + 0x68, 30);
+ hws[IMX6SLL_CLK_AIPSTZ1] = imx_clk_hw_gate2_flags("aips_tz1", "ahb", base + 0x68, 0, CLK_IS_CRITICAL);
+ hws[IMX6SLL_CLK_AIPSTZ2] = imx_clk_hw_gate2_flags("aips_tz2", "ahb", base + 0x68, 2, CLK_IS_CRITICAL);
+ hws[IMX6SLL_CLK_DCP] = imx_clk_hw_gate2("dcp", "ahb", base + 0x68, 10);
+ hws[IMX6SLL_CLK_UART2_IPG] = imx_clk_hw_gate2("uart2_ipg", "ipg", base + 0x68, 28);
+ hws[IMX6SLL_CLK_UART2_SERIAL] = imx_clk_hw_gate2("uart2_serial", "uart_podf", base + 0x68, 28);
+ hws[IMX6SLL_CLK_GPIO2] = imx_clk_hw_gate2("gpio2", "ipg", base + 0x68, 30);
/* CCGR1 */
- clks[IMX6SLL_CLK_ECSPI1] = imx_clk_gate2("ecspi1", "ecspi_podf", base + 0x6c, 0);
- clks[IMX6SLL_CLK_ECSPI2] = imx_clk_gate2("ecspi2", "ecspi_podf", base + 0x6c, 2);
- clks[IMX6SLL_CLK_ECSPI3] = imx_clk_gate2("ecspi3", "ecspi_podf", base + 0x6c, 4);
- clks[IMX6SLL_CLK_ECSPI4] = imx_clk_gate2("ecspi4", "ecspi_podf", base + 0x6c, 6);
- clks[IMX6SLL_CLK_UART3_IPG] = imx_clk_gate2("uart3_ipg", "ipg", base + 0x6c, 10);
- clks[IMX6SLL_CLK_UART3_SERIAL] = imx_clk_gate2("uart3_serial", "uart_podf", base + 0x6c, 10);
- clks[IMX6SLL_CLK_EPIT1] = imx_clk_gate2("epit1", "perclk", base + 0x6c, 12);
- clks[IMX6SLL_CLK_EPIT2] = imx_clk_gate2("epit2", "perclk", base + 0x6c, 14);
- clks[IMX6SLL_CLK_GPT_BUS] = imx_clk_gate2("gpt1_bus", "perclk", base + 0x6c, 20);
- clks[IMX6SLL_CLK_GPT_SERIAL] = imx_clk_gate2("gpt1_serial", "perclk", base + 0x6c, 22);
- clks[IMX6SLL_CLK_UART4_IPG] = imx_clk_gate2("uart4_ipg", "ipg", base + 0x6c, 24);
- clks[IMX6SLL_CLK_UART4_SERIAL] = imx_clk_gate2("uart4_serial", "uart_podf", base + 0x6c, 24);
- clks[IMX6SLL_CLK_GPIO1] = imx_clk_gate2("gpio1", "ipg", base + 0x6c, 26);
- clks[IMX6SLL_CLK_GPIO5] = imx_clk_gate2("gpio5", "ipg", base + 0x6c, 30);
+ hws[IMX6SLL_CLK_ECSPI1] = imx_clk_hw_gate2("ecspi1", "ecspi_podf", base + 0x6c, 0);
+ hws[IMX6SLL_CLK_ECSPI2] = imx_clk_hw_gate2("ecspi2", "ecspi_podf", base + 0x6c, 2);
+ hws[IMX6SLL_CLK_ECSPI3] = imx_clk_hw_gate2("ecspi3", "ecspi_podf", base + 0x6c, 4);
+ hws[IMX6SLL_CLK_ECSPI4] = imx_clk_hw_gate2("ecspi4", "ecspi_podf", base + 0x6c, 6);
+ hws[IMX6SLL_CLK_UART3_IPG] = imx_clk_hw_gate2("uart3_ipg", "ipg", base + 0x6c, 10);
+ hws[IMX6SLL_CLK_UART3_SERIAL] = imx_clk_hw_gate2("uart3_serial", "uart_podf", base + 0x6c, 10);
+ hws[IMX6SLL_CLK_EPIT1] = imx_clk_hw_gate2("epit1", "perclk", base + 0x6c, 12);
+ hws[IMX6SLL_CLK_EPIT2] = imx_clk_hw_gate2("epit2", "perclk", base + 0x6c, 14);
+ hws[IMX6SLL_CLK_GPT_BUS] = imx_clk_hw_gate2("gpt1_bus", "perclk", base + 0x6c, 20);
+ hws[IMX6SLL_CLK_GPT_SERIAL] = imx_clk_hw_gate2("gpt1_serial", "perclk", base + 0x6c, 22);
+ hws[IMX6SLL_CLK_UART4_IPG] = imx_clk_hw_gate2("uart4_ipg", "ipg", base + 0x6c, 24);
+ hws[IMX6SLL_CLK_UART4_SERIAL] = imx_clk_hw_gate2("uart4_serial", "uart_podf", base + 0x6c, 24);
+ hws[IMX6SLL_CLK_GPIO1] = imx_clk_hw_gate2("gpio1", "ipg", base + 0x6c, 26);
+ hws[IMX6SLL_CLK_GPIO5] = imx_clk_hw_gate2("gpio5", "ipg", base + 0x6c, 30);
/* CCGR2 */
- clks[IMX6SLL_CLK_GPIO6] = imx_clk_gate2("gpio6", "ipg", base + 0x70, 0);
- clks[IMX6SLL_CLK_CSI] = imx_clk_gate2("csi", "axi", base + 0x70, 2);
- clks[IMX6SLL_CLK_I2C1] = imx_clk_gate2("i2c1", "perclk", base + 0x70, 6);
- clks[IMX6SLL_CLK_I2C2] = imx_clk_gate2("i2c2", "perclk", base + 0x70, 8);
- clks[IMX6SLL_CLK_I2C3] = imx_clk_gate2("i2c3", "perclk", base + 0x70, 10);
- clks[IMX6SLL_CLK_OCOTP] = imx_clk_gate2("ocotp", "ipg", base + 0x70, 12);
- clks[IMX6SLL_CLK_GPIO3] = imx_clk_gate2("gpio3", "ipg", base + 0x70, 26);
- clks[IMX6SLL_CLK_LCDIF_APB] = imx_clk_gate2("lcdif_apb", "axi", base + 0x70, 28);
- clks[IMX6SLL_CLK_PXP] = imx_clk_gate2("pxp", "axi", base + 0x70, 30);
+ hws[IMX6SLL_CLK_GPIO6] = imx_clk_hw_gate2("gpio6", "ipg", base + 0x70, 0);
+ hws[IMX6SLL_CLK_CSI] = imx_clk_hw_gate2("csi", "axi", base + 0x70, 2);
+ hws[IMX6SLL_CLK_I2C1] = imx_clk_hw_gate2("i2c1", "perclk", base + 0x70, 6);
+ hws[IMX6SLL_CLK_I2C2] = imx_clk_hw_gate2("i2c2", "perclk", base + 0x70, 8);
+ hws[IMX6SLL_CLK_I2C3] = imx_clk_hw_gate2("i2c3", "perclk", base + 0x70, 10);
+ hws[IMX6SLL_CLK_OCOTP] = imx_clk_hw_gate2("ocotp", "ipg", base + 0x70, 12);
+ hws[IMX6SLL_CLK_GPIO3] = imx_clk_hw_gate2("gpio3", "ipg", base + 0x70, 26);
+ hws[IMX6SLL_CLK_LCDIF_APB] = imx_clk_hw_gate2("lcdif_apb", "axi", base + 0x70, 28);
+ hws[IMX6SLL_CLK_PXP] = imx_clk_hw_gate2("pxp", "axi", base + 0x70, 30);
/* CCGR3 */
- clks[IMX6SLL_CLK_UART5_IPG] = imx_clk_gate2("uart5_ipg", "ipg", base + 0x74, 2);
- clks[IMX6SLL_CLK_UART5_SERIAL] = imx_clk_gate2("uart5_serial", "uart_podf", base + 0x74, 2);
- clks[IMX6SLL_CLK_EPDC_AXI] = imx_clk_gate2("epdc_aclk", "axi", base + 0x74, 4);
- clks[IMX6SLL_CLK_EPDC_PIX] = imx_clk_gate2("epdc_pix", "epdc_podf", base + 0x74, 4);
- clks[IMX6SLL_CLK_LCDIF_PIX] = imx_clk_gate2("lcdif_pix", "lcdif_podf", base + 0x74, 10);
- clks[IMX6SLL_CLK_GPIO4] = imx_clk_gate2("gpio4", "ipg", base + 0x74, 12);
- clks[IMX6SLL_CLK_WDOG1] = imx_clk_gate2("wdog1", "ipg", base + 0x74, 16);
- clks[IMX6SLL_CLK_MMDC_P0_FAST] = imx_clk_gate_flags("mmdc_p0_fast", "mmdc_podf", base + 0x74, 20, CLK_IS_CRITICAL);
- clks[IMX6SLL_CLK_MMDC_P0_IPG] = imx_clk_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL);
- clks[IMX6SLL_CLK_MMDC_P1_IPG] = imx_clk_gate2("mmdc_p1_ipg", "ipg", base + 0x74, 26);
- clks[IMX6SLL_CLK_OCRAM] = imx_clk_gate_flags("ocram","ahb", base + 0x74, 28, CLK_IS_CRITICAL);
+ hws[IMX6SLL_CLK_UART5_IPG] = imx_clk_hw_gate2("uart5_ipg", "ipg", base + 0x74, 2);
+ hws[IMX6SLL_CLK_UART5_SERIAL] = imx_clk_hw_gate2("uart5_serial", "uart_podf", base + 0x74, 2);
+ hws[IMX6SLL_CLK_EPDC_AXI] = imx_clk_hw_gate2("epdc_aclk", "axi", base + 0x74, 4);
+ hws[IMX6SLL_CLK_EPDC_PIX] = imx_clk_hw_gate2("epdc_pix", "epdc_podf", base + 0x74, 4);
+ hws[IMX6SLL_CLK_LCDIF_PIX] = imx_clk_hw_gate2("lcdif_pix", "lcdif_podf", base + 0x74, 10);
+ hws[IMX6SLL_CLK_GPIO4] = imx_clk_hw_gate2("gpio4", "ipg", base + 0x74, 12);
+ hws[IMX6SLL_CLK_WDOG1] = imx_clk_hw_gate2("wdog1", "ipg", base + 0x74, 16);
+ hws[IMX6SLL_CLK_MMDC_P0_FAST] = imx_clk_hw_gate_flags("mmdc_p0_fast", "mmdc_podf", base + 0x74, 20, CLK_IS_CRITICAL);
+ hws[IMX6SLL_CLK_MMDC_P0_IPG] = imx_clk_hw_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL);
+ hws[IMX6SLL_CLK_MMDC_P1_IPG] = imx_clk_hw_gate2_flags("mmdc_p1_ipg", "ipg", base + 0x74, 26, CLK_IS_CRITICAL);
+ hws[IMX6SLL_CLK_OCRAM] = imx_clk_hw_gate_flags("ocram", "ahb", base + 0x74, 28, CLK_IS_CRITICAL);
/* CCGR4 */
- clks[IMX6SLL_CLK_PWM1] = imx_clk_gate2("pwm1", "perclk", base + 0x78, 16);
- clks[IMX6SLL_CLK_PWM2] = imx_clk_gate2("pwm2", "perclk", base + 0x78, 18);
- clks[IMX6SLL_CLK_PWM3] = imx_clk_gate2("pwm3", "perclk", base + 0x78, 20);
- clks[IMX6SLL_CLK_PWM4] = imx_clk_gate2("pwm4", "perclk", base + 0x78, 22);
+ hws[IMX6SLL_CLK_PWM1] = imx_clk_hw_gate2("pwm1", "perclk", base + 0x78, 16);
+ hws[IMX6SLL_CLK_PWM2] = imx_clk_hw_gate2("pwm2", "perclk", base + 0x78, 18);
+ hws[IMX6SLL_CLK_PWM3] = imx_clk_hw_gate2("pwm3", "perclk", base + 0x78, 20);
+ hws[IMX6SLL_CLK_PWM4] = imx_clk_hw_gate2("pwm4", "perclk", base + 0x78, 22);
/* CCGR5 */
- clks[IMX6SLL_CLK_ROM] = imx_clk_gate2_flags("rom", "ahb", base + 0x7c, 0, CLK_IS_CRITICAL);
- clks[IMX6SLL_CLK_SDMA] = imx_clk_gate2("sdma", "ahb", base + 0x7c, 6);
- clks[IMX6SLL_CLK_WDOG2] = imx_clk_gate2("wdog2", "ipg", base + 0x7c, 10);
- clks[IMX6SLL_CLK_SPBA] = imx_clk_gate2("spba", "ipg", base + 0x7c, 12);
- clks[IMX6SLL_CLK_EXTERN_AUDIO] = imx_clk_gate2_shared("extern_audio", "extern_audio_podf", base + 0x7c, 14, &share_count_audio);
- clks[IMX6SLL_CLK_SPDIF] = imx_clk_gate2_shared("spdif", "spdif_podf", base + 0x7c, 14, &share_count_audio);
- clks[IMX6SLL_CLK_SPDIF_GCLK] = imx_clk_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_audio);
- clks[IMX6SLL_CLK_SSI1] = imx_clk_gate2_shared("ssi1", "ssi1_podf", base + 0x7c, 18, &share_count_ssi1);
- clks[IMX6SLL_CLK_SSI1_IPG] = imx_clk_gate2_shared("ssi1_ipg", "ipg", base + 0x7c, 18, &share_count_ssi1);
- clks[IMX6SLL_CLK_SSI2] = imx_clk_gate2_shared("ssi2", "ssi2_podf", base + 0x7c, 20, &share_count_ssi2);
- clks[IMX6SLL_CLK_SSI2_IPG] = imx_clk_gate2_shared("ssi2_ipg", "ipg", base + 0x7c, 20, &share_count_ssi2);
- clks[IMX6SLL_CLK_SSI3] = imx_clk_gate2_shared("ssi3", "ssi3_podf", base + 0x7c, 22, &share_count_ssi3);
- clks[IMX6SLL_CLK_SSI3_IPG] = imx_clk_gate2_shared("ssi3_ipg", "ipg", base + 0x7c, 22, &share_count_ssi3);
- clks[IMX6SLL_CLK_UART1_IPG] = imx_clk_gate2("uart1_ipg", "ipg", base + 0x7c, 24);
- clks[IMX6SLL_CLK_UART1_SERIAL] = imx_clk_gate2("uart1_serial", "uart_podf", base + 0x7c, 24);
+ hws[IMX6SLL_CLK_ROM] = imx_clk_hw_gate2_flags("rom", "ahb", base + 0x7c, 0, CLK_IS_CRITICAL);
+ hws[IMX6SLL_CLK_SDMA] = imx_clk_hw_gate2("sdma", "ahb", base + 0x7c, 6);
+ hws[IMX6SLL_CLK_WDOG2] = imx_clk_hw_gate2("wdog2", "ipg", base + 0x7c, 10);
+ hws[IMX6SLL_CLK_SPBA] = imx_clk_hw_gate2("spba", "ipg", base + 0x7c, 12);
+ hws[IMX6SLL_CLK_EXTERN_AUDIO] = imx_clk_hw_gate2_shared("extern_audio", "extern_audio_podf", base + 0x7c, 14, &share_count_audio);
+ hws[IMX6SLL_CLK_SPDIF] = imx_clk_hw_gate2_shared("spdif", "spdif_podf", base + 0x7c, 14, &share_count_audio);
+ hws[IMX6SLL_CLK_SPDIF_GCLK] = imx_clk_hw_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_audio);
+ hws[IMX6SLL_CLK_SSI1] = imx_clk_hw_gate2_shared("ssi1", "ssi1_podf", base + 0x7c, 18, &share_count_ssi1);
+ hws[IMX6SLL_CLK_SSI1_IPG] = imx_clk_hw_gate2_shared("ssi1_ipg", "ipg", base + 0x7c, 18, &share_count_ssi1);
+ hws[IMX6SLL_CLK_SSI2] = imx_clk_hw_gate2_shared("ssi2", "ssi2_podf", base + 0x7c, 20, &share_count_ssi2);
+ hws[IMX6SLL_CLK_SSI2_IPG] = imx_clk_hw_gate2_shared("ssi2_ipg", "ipg", base + 0x7c, 20, &share_count_ssi2);
+ hws[IMX6SLL_CLK_SSI3] = imx_clk_hw_gate2_shared("ssi3", "ssi3_podf", base + 0x7c, 22, &share_count_ssi3);
+ hws[IMX6SLL_CLK_SSI3_IPG] = imx_clk_hw_gate2_shared("ssi3_ipg", "ipg", base + 0x7c, 22, &share_count_ssi3);
+ hws[IMX6SLL_CLK_UART1_IPG] = imx_clk_hw_gate2("uart1_ipg", "ipg", base + 0x7c, 24);
+ hws[IMX6SLL_CLK_UART1_SERIAL] = imx_clk_hw_gate2("uart1_serial", "uart_podf", base + 0x7c, 24);
/* CCGR6 */
- clks[IMX6SLL_CLK_USBOH3] = imx_clk_gate2("usboh3", "ipg", base + 0x80, 0);
- clks[IMX6SLL_CLK_USDHC1] = imx_clk_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2);
- clks[IMX6SLL_CLK_USDHC2] = imx_clk_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4);
- clks[IMX6SLL_CLK_USDHC3] = imx_clk_gate2("usdhc3", "usdhc3_podf", base + 0x80, 6);
+ hws[IMX6SLL_CLK_USBOH3] = imx_clk_hw_gate2("usboh3", "ipg", base + 0x80, 0);
+ hws[IMX6SLL_CLK_USDHC1] = imx_clk_hw_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2);
+ hws[IMX6SLL_CLK_USDHC2] = imx_clk_hw_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4);
+ hws[IMX6SLL_CLK_USDHC3] = imx_clk_hw_gate2("usdhc3", "usdhc3_podf", base + 0x80, 6);
/* mask handshake of mmdc */
- writel_relaxed(BM_CCM_CCDR_MMDC_CH0_MASK, base + 0x4);
+ imx_mmdc_mask_handshake(base, 0);
- imx_check_clocks(clks, ARRAY_SIZE(clks));
+ imx_check_clk_hws(hws, IMX6SLL_CLK_END);
- clk_data.clks = clks;
- clk_data.clk_num = ARRAY_SIZE(clks);
- of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
+ of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data);
+
+ for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) {
+ int index = uart_clk_ids[i];
+
+ uart_clks[i] = &hws[index]->clk;
+ }
imx_register_uart_clocks(uart_clks);
/* Lower the AHB clock rate before changing the clock source. */
- clk_set_rate(clks[IMX6SLL_CLK_AHB], 99000000);
+ clk_set_rate(hws[IMX6SLL_CLK_AHB]->clk, 99000000);
/* Change periph_pre clock to pll2_bus to adjust AXI rate to 264MHz */
- clk_set_parent(clks[IMX6SLL_CLK_PERIPH_CLK2_SEL], clks[IMX6SLL_CLK_PLL3_USB_OTG]);
- clk_set_parent(clks[IMX6SLL_CLK_PERIPH], clks[IMX6SLL_CLK_PERIPH_CLK2]);
- clk_set_parent(clks[IMX6SLL_CLK_PERIPH_PRE], clks[IMX6SLL_CLK_PLL2_BUS]);
- clk_set_parent(clks[IMX6SLL_CLK_PERIPH], clks[IMX6SLL_CLK_PERIPH_PRE]);
+ clk_set_parent(hws[IMX6SLL_CLK_PERIPH_CLK2_SEL]->clk, hws[IMX6SLL_CLK_PLL3_USB_OTG]->clk);
+ clk_set_parent(hws[IMX6SLL_CLK_PERIPH]->clk, hws[IMX6SLL_CLK_PERIPH_CLK2]->clk);
+ clk_set_parent(hws[IMX6SLL_CLK_PERIPH_PRE]->clk, hws[IMX6SLL_CLK_PLL2_BUS]->clk);
+ clk_set_parent(hws[IMX6SLL_CLK_PERIPH]->clk, hws[IMX6SLL_CLK_PERIPH_PRE]->clk);
- clk_set_rate(clks[IMX6SLL_CLK_AHB], 132000000);
+ clk_set_rate(hws[IMX6SLL_CLK_AHB]->clk, 132000000);
}
CLK_OF_DECLARE_DRIVER(imx6sll, "fsl,imx6sll-ccm", imx6sll_clocks_init);
diff --git a/drivers/clk/imx/clk-imx6sx.c b/drivers/clk/imx/clk-imx6sx.c
index d243e3483e24..c4685c01929a 100644
--- a/drivers/clk/imx/clk-imx6sx.c
+++ b/drivers/clk/imx/clk-imx6sx.c
@@ -6,6 +6,7 @@
#include <dt-bindings/clock/imx6sx-clock.h>
#include <linux/clk.h>
#include <linux/clkdev.h>
+#include <linux/clk-provider.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/io.h>
@@ -16,9 +17,6 @@
#include "clk.h"
-#define CCDR 0x4
-#define BM_CCM_CCDR_MMDC_CH0_MASK (0x2 << 16)
-
static const char *step_sels[] = { "osc", "pll2_pfd2_396m", };
static const char *pll1_sw_sels[] = { "pll1_sys", "step", };
static const char *periph_pre_sels[] = { "pll2_bus", "pll2_pfd2_396m", "pll2_pfd0_352m", "pll2_198m", };
@@ -83,8 +81,8 @@ static const char *pll5_bypass_sels[] = { "pll5", "pll5_bypass_src", };
static const char *pll6_bypass_sels[] = { "pll6", "pll6_bypass_src", };
static const char *pll7_bypass_sels[] = { "pll7", "pll7_bypass_src", };
-static struct clk *clks[IMX6SX_CLK_CLK_END];
-static struct clk_onecell_data clk_data;
+static struct clk_hw **hws;
+static struct clk_hw_onecell_data *clk_hw_data;
static const struct clk_div_table clk_enet_ref_table[] = {
{ .val = 0, .div = 20, },
@@ -118,76 +116,86 @@ static u32 share_count_ssi3;
static u32 share_count_sai1;
static u32 share_count_sai2;
-static struct clk ** const uart_clks[] __initconst = {
- &clks[IMX6SX_CLK_UART_IPG],
- &clks[IMX6SX_CLK_UART_SERIAL],
- NULL
+static const int uart_clk_ids[] __initconst = {
+ IMX6SX_CLK_UART_IPG,
+ IMX6SX_CLK_UART_SERIAL,
};
+static struct clk **uart_clks[ARRAY_SIZE(uart_clk_ids) + 1] __initdata;
+
static void __init imx6sx_clocks_init(struct device_node *ccm_node)
{
struct device_node *np;
void __iomem *base;
+ int i;
+
+ clk_hw_data = kzalloc(struct_size(clk_hw_data, hws,
+ IMX6SX_CLK_CLK_END), GFP_KERNEL);
+ if (WARN_ON(!clk_hw_data))
+ return;
+
+ clk_hw_data->num = IMX6SX_CLK_CLK_END;
+ hws = clk_hw_data->hws;
- clks[IMX6SX_CLK_DUMMY] = imx_clk_fixed("dummy", 0);
+ hws[IMX6SX_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0);
- clks[IMX6SX_CLK_CKIL] = of_clk_get_by_name(ccm_node, "ckil");
- clks[IMX6SX_CLK_OSC] = of_clk_get_by_name(ccm_node, "osc");
+ hws[IMX6SX_CLK_CKIL] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ckil"));
+ hws[IMX6SX_CLK_OSC] = __clk_get_hw(of_clk_get_by_name(ccm_node, "osc"));
/* ipp_di clock is external input */
- clks[IMX6SX_CLK_IPP_DI0] = of_clk_get_by_name(ccm_node, "ipp_di0");
- clks[IMX6SX_CLK_IPP_DI1] = of_clk_get_by_name(ccm_node, "ipp_di1");
+ hws[IMX6SX_CLK_IPP_DI0] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ipp_di0"));
+ hws[IMX6SX_CLK_IPP_DI1] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ipp_di1"));
/* Clock source from external clock via CLK1/2 PAD */
- clks[IMX6SX_CLK_ANACLK1] = of_clk_get_by_name(ccm_node, "anaclk1");
- clks[IMX6SX_CLK_ANACLK2] = of_clk_get_by_name(ccm_node, "anaclk2");
+ hws[IMX6SX_CLK_ANACLK1] = __clk_get_hw(of_clk_get_by_name(ccm_node, "anaclk1"));
+ hws[IMX6SX_CLK_ANACLK2] = __clk_get_hw(of_clk_get_by_name(ccm_node, "anaclk2"));
np = of_find_compatible_node(NULL, NULL, "fsl,imx6sx-anatop");
base = of_iomap(np, 0);
WARN_ON(!base);
of_node_put(np);
- clks[IMX6SX_PLL1_BYPASS_SRC] = imx_clk_mux("pll1_bypass_src", base + 0x00, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SX_PLL2_BYPASS_SRC] = imx_clk_mux("pll2_bypass_src", base + 0x30, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SX_PLL3_BYPASS_SRC] = imx_clk_mux("pll3_bypass_src", base + 0x10, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SX_PLL4_BYPASS_SRC] = imx_clk_mux("pll4_bypass_src", base + 0x70, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SX_PLL5_BYPASS_SRC] = imx_clk_mux("pll5_bypass_src", base + 0xa0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SX_PLL6_BYPASS_SRC] = imx_clk_mux("pll6_bypass_src", base + 0xe0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6SX_PLL7_BYPASS_SRC] = imx_clk_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SX_PLL1_BYPASS_SRC] = imx_clk_hw_mux("pll1_bypass_src", base + 0x00, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SX_PLL2_BYPASS_SRC] = imx_clk_hw_mux("pll2_bypass_src", base + 0x30, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SX_PLL3_BYPASS_SRC] = imx_clk_hw_mux("pll3_bypass_src", base + 0x10, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SX_PLL4_BYPASS_SRC] = imx_clk_hw_mux("pll4_bypass_src", base + 0x70, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SX_PLL5_BYPASS_SRC] = imx_clk_hw_mux("pll5_bypass_src", base + 0xa0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SX_PLL6_BYPASS_SRC] = imx_clk_hw_mux("pll6_bypass_src", base + 0xe0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6SX_PLL7_BYPASS_SRC] = imx_clk_hw_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
/* type name parent_name base div_mask */
- clks[IMX6SX_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f);
- clks[IMX6SX_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1);
- clks[IMX6SX_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3);
- clks[IMX6SX_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f);
- clks[IMX6SX_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f);
- clks[IMX6SX_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3);
- clks[IMX6SX_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3);
-
- clks[IMX6SX_PLL1_BYPASS] = imx_clk_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SX_PLL2_BYPASS] = imx_clk_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SX_PLL3_BYPASS] = imx_clk_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SX_PLL4_BYPASS] = imx_clk_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SX_PLL5_BYPASS] = imx_clk_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SX_PLL6_BYPASS] = imx_clk_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SX_PLL7_BYPASS] = imx_clk_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SX_CLK_PLL1] = imx_clk_hw_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f);
+ hws[IMX6SX_CLK_PLL2] = imx_clk_hw_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1);
+ hws[IMX6SX_CLK_PLL3] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3);
+ hws[IMX6SX_CLK_PLL4] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f);
+ hws[IMX6SX_CLK_PLL5] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f);
+ hws[IMX6SX_CLK_PLL6] = imx_clk_hw_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3);
+ hws[IMX6SX_CLK_PLL7] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3);
+
+ hws[IMX6SX_PLL1_BYPASS] = imx_clk_hw_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SX_PLL2_BYPASS] = imx_clk_hw_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SX_PLL3_BYPASS] = imx_clk_hw_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SX_PLL4_BYPASS] = imx_clk_hw_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SX_PLL5_BYPASS] = imx_clk_hw_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SX_PLL6_BYPASS] = imx_clk_hw_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SX_PLL7_BYPASS] = imx_clk_hw_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT);
/* Do not bypass PLLs initially */
- clk_set_parent(clks[IMX6SX_PLL1_BYPASS], clks[IMX6SX_CLK_PLL1]);
- clk_set_parent(clks[IMX6SX_PLL2_BYPASS], clks[IMX6SX_CLK_PLL2]);
- clk_set_parent(clks[IMX6SX_PLL3_BYPASS], clks[IMX6SX_CLK_PLL3]);
- clk_set_parent(clks[IMX6SX_PLL4_BYPASS], clks[IMX6SX_CLK_PLL4]);
- clk_set_parent(clks[IMX6SX_PLL5_BYPASS], clks[IMX6SX_CLK_PLL5]);
- clk_set_parent(clks[IMX6SX_PLL6_BYPASS], clks[IMX6SX_CLK_PLL6]);
- clk_set_parent(clks[IMX6SX_PLL7_BYPASS], clks[IMX6SX_CLK_PLL7]);
-
- clks[IMX6SX_CLK_PLL1_SYS] = imx_clk_gate("pll1_sys", "pll1_bypass", base + 0x00, 13);
- clks[IMX6SX_CLK_PLL2_BUS] = imx_clk_gate("pll2_bus", "pll2_bypass", base + 0x30, 13);
- clks[IMX6SX_CLK_PLL3_USB_OTG] = imx_clk_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13);
- clks[IMX6SX_CLK_PLL4_AUDIO] = imx_clk_gate("pll4_audio", "pll4_bypass", base + 0x70, 13);
- clks[IMX6SX_CLK_PLL5_VIDEO] = imx_clk_gate("pll5_video", "pll5_bypass", base + 0xa0, 13);
- clks[IMX6SX_CLK_PLL6_ENET] = imx_clk_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13);
- clks[IMX6SX_CLK_PLL7_USB_HOST] = imx_clk_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13);
+ clk_set_parent(hws[IMX6SX_PLL1_BYPASS]->clk, hws[IMX6SX_CLK_PLL1]->clk);
+ clk_set_parent(hws[IMX6SX_PLL2_BYPASS]->clk, hws[IMX6SX_CLK_PLL2]->clk);
+ clk_set_parent(hws[IMX6SX_PLL3_BYPASS]->clk, hws[IMX6SX_CLK_PLL3]->clk);
+ clk_set_parent(hws[IMX6SX_PLL4_BYPASS]->clk, hws[IMX6SX_CLK_PLL4]->clk);
+ clk_set_parent(hws[IMX6SX_PLL5_BYPASS]->clk, hws[IMX6SX_CLK_PLL5]->clk);
+ clk_set_parent(hws[IMX6SX_PLL6_BYPASS]->clk, hws[IMX6SX_CLK_PLL6]->clk);
+ clk_set_parent(hws[IMX6SX_PLL7_BYPASS]->clk, hws[IMX6SX_CLK_PLL7]->clk);
+
+ hws[IMX6SX_CLK_PLL1_SYS] = imx_clk_hw_gate("pll1_sys", "pll1_bypass", base + 0x00, 13);
+ hws[IMX6SX_CLK_PLL2_BUS] = imx_clk_hw_gate("pll2_bus", "pll2_bypass", base + 0x30, 13);
+ hws[IMX6SX_CLK_PLL3_USB_OTG] = imx_clk_hw_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13);
+ hws[IMX6SX_CLK_PLL4_AUDIO] = imx_clk_hw_gate("pll4_audio", "pll4_bypass", base + 0x70, 13);
+ hws[IMX6SX_CLK_PLL5_VIDEO] = imx_clk_hw_gate("pll5_video", "pll5_bypass", base + 0xa0, 13);
+ hws[IMX6SX_CLK_PLL6_ENET] = imx_clk_hw_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13);
+ hws[IMX6SX_CLK_PLL7_USB_HOST] = imx_clk_hw_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13);
/*
* Bit 20 is the reserved and read-only bit, we do this only for:
@@ -195,361 +203,363 @@ static void __init imx6sx_clocks_init(struct device_node *ccm_node)
* - Keep refcount when do usbphy clk_enable/disable, in that case,
* the clk framework may need to enable/disable usbphy's parent
*/
- clks[IMX6SX_CLK_USBPHY1] = imx_clk_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20);
- clks[IMX6SX_CLK_USBPHY2] = imx_clk_gate("usbphy2", "pll7_usb_host", base + 0x20, 20);
+ hws[IMX6SX_CLK_USBPHY1] = imx_clk_hw_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20);
+ hws[IMX6SX_CLK_USBPHY2] = imx_clk_hw_gate("usbphy2", "pll7_usb_host", base + 0x20, 20);
/*
* usbphy*_gate needs to be on after system boots up, and software
* never needs to control it anymore.
*/
- clks[IMX6SX_CLK_USBPHY1_GATE] = imx_clk_gate("usbphy1_gate", "dummy", base + 0x10, 6);
- clks[IMX6SX_CLK_USBPHY2_GATE] = imx_clk_gate("usbphy2_gate", "dummy", base + 0x20, 6);
+ hws[IMX6SX_CLK_USBPHY1_GATE] = imx_clk_hw_gate("usbphy1_gate", "dummy", base + 0x10, 6);
+ hws[IMX6SX_CLK_USBPHY2_GATE] = imx_clk_hw_gate("usbphy2_gate", "dummy", base + 0x20, 6);
/* FIXME 100MHz is used for pcie ref for all imx6 pcie, excepted imx6q */
- clks[IMX6SX_CLK_PCIE_REF] = imx_clk_fixed_factor("pcie_ref", "pll6_enet", 1, 5);
- clks[IMX6SX_CLK_PCIE_REF_125M] = imx_clk_gate("pcie_ref_125m", "pcie_ref", base + 0xe0, 19);
+ hws[IMX6SX_CLK_PCIE_REF] = imx_clk_hw_fixed_factor("pcie_ref", "pll6_enet", 1, 5);
+ hws[IMX6SX_CLK_PCIE_REF_125M] = imx_clk_hw_gate("pcie_ref_125m", "pcie_ref", base + 0xe0, 19);
- clks[IMX6SX_CLK_LVDS1_OUT] = imx_clk_gate_exclusive("lvds1_out", "lvds1_sel", base + 0x160, 10, BIT(12));
- clks[IMX6SX_CLK_LVDS2_OUT] = imx_clk_gate_exclusive("lvds2_out", "lvds2_sel", base + 0x160, 11, BIT(13));
- clks[IMX6SX_CLK_LVDS1_IN] = imx_clk_gate_exclusive("lvds1_in", "anaclk1", base + 0x160, 12, BIT(10));
- clks[IMX6SX_CLK_LVDS2_IN] = imx_clk_gate_exclusive("lvds2_in", "anaclk2", base + 0x160, 13, BIT(11));
+ hws[IMX6SX_CLK_LVDS1_OUT] = imx_clk_hw_gate_exclusive("lvds1_out", "lvds1_sel", base + 0x160, 10, BIT(12));
+ hws[IMX6SX_CLK_LVDS2_OUT] = imx_clk_hw_gate_exclusive("lvds2_out", "lvds2_sel", base + 0x160, 11, BIT(13));
+ hws[IMX6SX_CLK_LVDS1_IN] = imx_clk_hw_gate_exclusive("lvds1_in", "anaclk1", base + 0x160, 12, BIT(10));
+ hws[IMX6SX_CLK_LVDS2_IN] = imx_clk_hw_gate_exclusive("lvds2_in", "anaclk2", base + 0x160, 13, BIT(11));
- clks[IMX6SX_CLK_ENET_REF] = clk_register_divider_table(NULL, "enet_ref", "pll6_enet", 0,
+ hws[IMX6SX_CLK_ENET_REF] = clk_hw_register_divider_table(NULL, "enet_ref", "pll6_enet", 0,
base + 0xe0, 0, 2, 0, clk_enet_ref_table,
&imx_ccm_lock);
- clks[IMX6SX_CLK_ENET2_REF] = clk_register_divider_table(NULL, "enet2_ref", "pll6_enet", 0,
+ hws[IMX6SX_CLK_ENET2_REF] = clk_hw_register_divider_table(NULL, "enet2_ref", "pll6_enet", 0,
base + 0xe0, 2, 2, 0, clk_enet_ref_table,
&imx_ccm_lock);
- clks[IMX6SX_CLK_ENET2_REF_125M] = imx_clk_gate("enet2_ref_125m", "enet2_ref", base + 0xe0, 20);
+ hws[IMX6SX_CLK_ENET2_REF_125M] = imx_clk_hw_gate("enet2_ref_125m", "enet2_ref", base + 0xe0, 20);
- clks[IMX6SX_CLK_ENET_PTP_REF] = imx_clk_fixed_factor("enet_ptp_ref", "pll6_enet", 1, 20);
- clks[IMX6SX_CLK_ENET_PTP] = imx_clk_gate("enet_ptp_25m", "enet_ptp_ref", base + 0xe0, 21);
+ hws[IMX6SX_CLK_ENET_PTP_REF] = imx_clk_hw_fixed_factor("enet_ptp_ref", "pll6_enet", 1, 20);
+ hws[IMX6SX_CLK_ENET_PTP] = imx_clk_hw_gate("enet_ptp_25m", "enet_ptp_ref", base + 0xe0, 21);
/* name parent_name reg idx */
- clks[IMX6SX_CLK_PLL2_PFD0] = imx_clk_pfd("pll2_pfd0_352m", "pll2_bus", base + 0x100, 0);
- clks[IMX6SX_CLK_PLL2_PFD1] = imx_clk_pfd("pll2_pfd1_594m", "pll2_bus", base + 0x100, 1);
- clks[IMX6SX_CLK_PLL2_PFD2] = imx_clk_pfd("pll2_pfd2_396m", "pll2_bus", base + 0x100, 2);
- clks[IMX6SX_CLK_PLL2_PFD3] = imx_clk_pfd("pll2_pfd3_594m", "pll2_bus", base + 0x100, 3);
- clks[IMX6SX_CLK_PLL3_PFD0] = imx_clk_pfd("pll3_pfd0_720m", "pll3_usb_otg", base + 0xf0, 0);
- clks[IMX6SX_CLK_PLL3_PFD1] = imx_clk_pfd("pll3_pfd1_540m", "pll3_usb_otg", base + 0xf0, 1);
- clks[IMX6SX_CLK_PLL3_PFD2] = imx_clk_pfd("pll3_pfd2_508m", "pll3_usb_otg", base + 0xf0, 2);
- clks[IMX6SX_CLK_PLL3_PFD3] = imx_clk_pfd("pll3_pfd3_454m", "pll3_usb_otg", base + 0xf0, 3);
+ hws[IMX6SX_CLK_PLL2_PFD0] = imx_clk_hw_pfd("pll2_pfd0_352m", "pll2_bus", base + 0x100, 0);
+ hws[IMX6SX_CLK_PLL2_PFD1] = imx_clk_hw_pfd("pll2_pfd1_594m", "pll2_bus", base + 0x100, 1);
+ hws[IMX6SX_CLK_PLL2_PFD2] = imx_clk_hw_pfd("pll2_pfd2_396m", "pll2_bus", base + 0x100, 2);
+ hws[IMX6SX_CLK_PLL2_PFD3] = imx_clk_hw_pfd("pll2_pfd3_594m", "pll2_bus", base + 0x100, 3);
+ hws[IMX6SX_CLK_PLL3_PFD0] = imx_clk_hw_pfd("pll3_pfd0_720m", "pll3_usb_otg", base + 0xf0, 0);
+ hws[IMX6SX_CLK_PLL3_PFD1] = imx_clk_hw_pfd("pll3_pfd1_540m", "pll3_usb_otg", base + 0xf0, 1);
+ hws[IMX6SX_CLK_PLL3_PFD2] = imx_clk_hw_pfd("pll3_pfd2_508m", "pll3_usb_otg", base + 0xf0, 2);
+ hws[IMX6SX_CLK_PLL3_PFD3] = imx_clk_hw_pfd("pll3_pfd3_454m", "pll3_usb_otg", base + 0xf0, 3);
/* name parent_name mult div */
- clks[IMX6SX_CLK_PLL2_198M] = imx_clk_fixed_factor("pll2_198m", "pll2_pfd2_396m", 1, 2);
- clks[IMX6SX_CLK_PLL3_120M] = imx_clk_fixed_factor("pll3_120m", "pll3_usb_otg", 1, 4);
- clks[IMX6SX_CLK_PLL3_80M] = imx_clk_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6);
- clks[IMX6SX_CLK_PLL3_60M] = imx_clk_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8);
- clks[IMX6SX_CLK_TWD] = imx_clk_fixed_factor("twd", "arm", 1, 2);
- clks[IMX6SX_CLK_GPT_3M] = imx_clk_fixed_factor("gpt_3m", "osc", 1, 8);
-
- clks[IMX6SX_CLK_PLL4_POST_DIV] = clk_register_divider_table(NULL, "pll4_post_div", "pll4_audio",
+ hws[IMX6SX_CLK_PLL2_198M] = imx_clk_hw_fixed_factor("pll2_198m", "pll2_pfd2_396m", 1, 2);
+ hws[IMX6SX_CLK_PLL3_120M] = imx_clk_hw_fixed_factor("pll3_120m", "pll3_usb_otg", 1, 4);
+ hws[IMX6SX_CLK_PLL3_80M] = imx_clk_hw_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6);
+ hws[IMX6SX_CLK_PLL3_60M] = imx_clk_hw_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8);
+ hws[IMX6SX_CLK_TWD] = imx_clk_hw_fixed_factor("twd", "arm", 1, 2);
+ hws[IMX6SX_CLK_GPT_3M] = imx_clk_hw_fixed_factor("gpt_3m", "osc", 1, 8);
+
+ hws[IMX6SX_CLK_PLL4_POST_DIV] = clk_hw_register_divider_table(NULL, "pll4_post_div", "pll4_audio",
CLK_SET_RATE_PARENT, base + 0x70, 19, 2, 0, post_div_table, &imx_ccm_lock);
- clks[IMX6SX_CLK_PLL4_AUDIO_DIV] = clk_register_divider(NULL, "pll4_audio_div", "pll4_post_div",
+ hws[IMX6SX_CLK_PLL4_AUDIO_DIV] = clk_hw_register_divider(NULL, "pll4_audio_div", "pll4_post_div",
CLK_SET_RATE_PARENT, base + 0x170, 15, 1, 0, &imx_ccm_lock);
- clks[IMX6SX_CLK_PLL5_POST_DIV] = clk_register_divider_table(NULL, "pll5_post_div", "pll5_video",
+ hws[IMX6SX_CLK_PLL5_POST_DIV] = clk_hw_register_divider_table(NULL, "pll5_post_div", "pll5_video",
CLK_SET_RATE_PARENT, base + 0xa0, 19, 2, 0, post_div_table, &imx_ccm_lock);
- clks[IMX6SX_CLK_PLL5_VIDEO_DIV] = clk_register_divider_table(NULL, "pll5_video_div", "pll5_post_div",
+ hws[IMX6SX_CLK_PLL5_VIDEO_DIV] = clk_hw_register_divider_table(NULL, "pll5_video_div", "pll5_post_div",
CLK_SET_RATE_PARENT, base + 0x170, 30, 2, 0, video_div_table, &imx_ccm_lock);
/* name reg shift width parent_names num_parents */
- clks[IMX6SX_CLK_LVDS1_SEL] = imx_clk_mux("lvds1_sel", base + 0x160, 0, 5, lvds_sels, ARRAY_SIZE(lvds_sels));
- clks[IMX6SX_CLK_LVDS2_SEL] = imx_clk_mux("lvds2_sel", base + 0x160, 5, 5, lvds_sels, ARRAY_SIZE(lvds_sels));
+ hws[IMX6SX_CLK_LVDS1_SEL] = imx_clk_hw_mux("lvds1_sel", base + 0x160, 0, 5, lvds_sels, ARRAY_SIZE(lvds_sels));
+ hws[IMX6SX_CLK_LVDS2_SEL] = imx_clk_hw_mux("lvds2_sel", base + 0x160, 5, 5, lvds_sels, ARRAY_SIZE(lvds_sels));
np = ccm_node;
base = of_iomap(np, 0);
WARN_ON(!base);
/* name reg shift width parent_names num_parents */
- clks[IMX6SX_CLK_STEP] = imx_clk_mux("step", base + 0xc, 8, 1, step_sels, ARRAY_SIZE(step_sels));
- clks[IMX6SX_CLK_PLL1_SW] = imx_clk_mux("pll1_sw", base + 0xc, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels));
- clks[IMX6SX_CLK_OCRAM_SEL] = imx_clk_mux("ocram_sel", base + 0x14, 6, 2, ocram_sels, ARRAY_SIZE(ocram_sels));
- clks[IMX6SX_CLK_PERIPH_PRE] = imx_clk_mux("periph_pre", base + 0x18, 18, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels));
- clks[IMX6SX_CLK_PERIPH2_PRE] = imx_clk_mux("periph2_pre", base + 0x18, 21, 2, periph2_pre_sels, ARRAY_SIZE(periph2_pre_sels));
- clks[IMX6SX_CLK_PERIPH_CLK2_SEL] = imx_clk_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels));
- clks[IMX6SX_CLK_PERIPH2_CLK2_SEL] = imx_clk_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels));
- clks[IMX6SX_CLK_PCIE_AXI_SEL] = imx_clk_mux("pcie_axi_sel", base + 0x18, 10, 1, pcie_axi_sels, ARRAY_SIZE(pcie_axi_sels));
- clks[IMX6SX_CLK_GPU_AXI_SEL] = imx_clk_mux("gpu_axi_sel", base + 0x18, 8, 2, gpu_axi_sels, ARRAY_SIZE(gpu_axi_sels));
- clks[IMX6SX_CLK_GPU_CORE_SEL] = imx_clk_mux("gpu_core_sel", base + 0x18, 4, 2, gpu_core_sels, ARRAY_SIZE(gpu_core_sels));
- clks[IMX6SX_CLK_EIM_SLOW_SEL] = imx_clk_mux("eim_slow_sel", base + 0x1c, 29, 2, eim_slow_sels, ARRAY_SIZE(eim_slow_sels));
- clks[IMX6SX_CLK_USDHC1_SEL] = imx_clk_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
- clks[IMX6SX_CLK_USDHC2_SEL] = imx_clk_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
- clks[IMX6SX_CLK_USDHC3_SEL] = imx_clk_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
- clks[IMX6SX_CLK_USDHC4_SEL] = imx_clk_mux("usdhc4_sel", base + 0x1c, 19, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
- clks[IMX6SX_CLK_SSI3_SEL] = imx_clk_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
- clks[IMX6SX_CLK_SSI2_SEL] = imx_clk_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
- clks[IMX6SX_CLK_SSI1_SEL] = imx_clk_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
- clks[IMX6SX_CLK_QSPI1_SEL] = imx_clk_mux_flags("qspi1_sel", base + 0x1c, 7, 3, qspi1_sels, ARRAY_SIZE(qspi1_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SX_CLK_PERCLK_SEL] = imx_clk_mux("perclk_sel", base + 0x1c, 6, 1, perclk_sels, ARRAY_SIZE(perclk_sels));
- clks[IMX6SX_CLK_VID_SEL] = imx_clk_mux("vid_sel", base + 0x20, 21, 3, vid_sels, ARRAY_SIZE(vid_sels));
- clks[IMX6SX_CLK_ESAI_SEL] = imx_clk_mux("esai_sel", base + 0x20, 19, 2, audio_sels, ARRAY_SIZE(audio_sels));
- clks[IMX6SX_CLK_CAN_SEL] = imx_clk_mux("can_sel", base + 0x20, 8, 2, can_sels, ARRAY_SIZE(can_sels));
- clks[IMX6SX_CLK_UART_SEL] = imx_clk_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels));
- clks[IMX6SX_CLK_QSPI2_SEL] = imx_clk_mux_flags("qspi2_sel", base + 0x2c, 15, 3, qspi2_sels, ARRAY_SIZE(qspi2_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SX_CLK_SPDIF_SEL] = imx_clk_mux("spdif_sel", base + 0x30, 20, 2, audio_sels, ARRAY_SIZE(audio_sels));
- clks[IMX6SX_CLK_AUDIO_SEL] = imx_clk_mux("audio_sel", base + 0x30, 7, 2, audio_sels, ARRAY_SIZE(audio_sels));
- clks[IMX6SX_CLK_ENET_PRE_SEL] = imx_clk_mux("enet_pre_sel", base + 0x34, 15, 3, enet_pre_sels, ARRAY_SIZE(enet_pre_sels));
- clks[IMX6SX_CLK_ENET_SEL] = imx_clk_mux("enet_sel", base + 0x34, 9, 3, enet_sels, ARRAY_SIZE(enet_sels));
- clks[IMX6SX_CLK_M4_PRE_SEL] = imx_clk_mux("m4_pre_sel", base + 0x34, 6, 3, m4_pre_sels, ARRAY_SIZE(m4_pre_sels));
- clks[IMX6SX_CLK_M4_SEL] = imx_clk_mux("m4_sel", base + 0x34, 0, 3, m4_sels, ARRAY_SIZE(m4_sels));
- clks[IMX6SX_CLK_ECSPI_SEL] = imx_clk_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels));
- clks[IMX6SX_CLK_LCDIF2_PRE_SEL] = imx_clk_mux("lcdif2_pre_sel", base + 0x38, 6, 3, lcdif2_pre_sels, ARRAY_SIZE(lcdif2_pre_sels));
- clks[IMX6SX_CLK_LCDIF2_SEL] = imx_clk_mux("lcdif2_sel", base + 0x38, 0, 3, lcdif2_sels, ARRAY_SIZE(lcdif2_sels));
- clks[IMX6SX_CLK_DISPLAY_SEL] = imx_clk_mux("display_sel", base + 0x3c, 14, 2, display_sels, ARRAY_SIZE(display_sels));
- clks[IMX6SX_CLK_CSI_SEL] = imx_clk_mux("csi_sel", base + 0x3c, 9, 2, csi_sels, ARRAY_SIZE(csi_sels));
- clks[IMX6SX_CLK_CKO1_SEL] = imx_clk_mux("cko1_sel", base + 0x60, 0, 4, cko1_sels, ARRAY_SIZE(cko1_sels));
- clks[IMX6SX_CLK_CKO2_SEL] = imx_clk_mux("cko2_sel", base + 0x60, 16, 5, cko2_sels, ARRAY_SIZE(cko2_sels));
- clks[IMX6SX_CLK_CKO] = imx_clk_mux("cko", base + 0x60, 8, 1, cko_sels, ARRAY_SIZE(cko_sels));
-
- clks[IMX6SX_CLK_LDB_DI1_DIV_SEL] = imx_clk_mux_flags("ldb_di1_div_sel", base + 0x20, 11, 1, ldb_di1_div_sels, ARRAY_SIZE(ldb_di1_div_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SX_CLK_LDB_DI0_DIV_SEL] = imx_clk_mux_flags("ldb_di0_div_sel", base + 0x20, 10, 1, ldb_di0_div_sels, ARRAY_SIZE(ldb_di0_div_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SX_CLK_LDB_DI1_SEL] = imx_clk_mux_flags("ldb_di1_sel", base + 0x2c, 12, 3, ldb_di1_sels, ARRAY_SIZE(ldb_di1_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SX_CLK_LDB_DI0_SEL] = imx_clk_mux_flags("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di0_sels, ARRAY_SIZE(ldb_di0_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SX_CLK_LCDIF1_PRE_SEL] = imx_clk_mux_flags("lcdif1_pre_sel", base + 0x38, 15, 3, lcdif1_pre_sels, ARRAY_SIZE(lcdif1_pre_sels), CLK_SET_RATE_PARENT);
- clks[IMX6SX_CLK_LCDIF1_SEL] = imx_clk_mux_flags("lcdif1_sel", base + 0x38, 9, 3, lcdif1_sels, ARRAY_SIZE(lcdif1_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SX_CLK_STEP] = imx_clk_hw_mux("step", base + 0xc, 8, 1, step_sels, ARRAY_SIZE(step_sels));
+ hws[IMX6SX_CLK_PLL1_SW] = imx_clk_hw_mux("pll1_sw", base + 0xc, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels));
+ hws[IMX6SX_CLK_OCRAM_SEL] = imx_clk_hw_mux("ocram_sel", base + 0x14, 6, 2, ocram_sels, ARRAY_SIZE(ocram_sels));
+ hws[IMX6SX_CLK_PERIPH_PRE] = imx_clk_hw_mux("periph_pre", base + 0x18, 18, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels));
+ hws[IMX6SX_CLK_PERIPH2_PRE] = imx_clk_hw_mux("periph2_pre", base + 0x18, 21, 2, periph2_pre_sels, ARRAY_SIZE(periph2_pre_sels));
+ hws[IMX6SX_CLK_PERIPH_CLK2_SEL] = imx_clk_hw_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels));
+ hws[IMX6SX_CLK_PERIPH2_CLK2_SEL] = imx_clk_hw_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels));
+ hws[IMX6SX_CLK_PCIE_AXI_SEL] = imx_clk_hw_mux("pcie_axi_sel", base + 0x18, 10, 1, pcie_axi_sels, ARRAY_SIZE(pcie_axi_sels));
+ hws[IMX6SX_CLK_GPU_AXI_SEL] = imx_clk_hw_mux("gpu_axi_sel", base + 0x18, 8, 2, gpu_axi_sels, ARRAY_SIZE(gpu_axi_sels));
+ hws[IMX6SX_CLK_GPU_CORE_SEL] = imx_clk_hw_mux("gpu_core_sel", base + 0x18, 4, 2, gpu_core_sels, ARRAY_SIZE(gpu_core_sels));
+ hws[IMX6SX_CLK_EIM_SLOW_SEL] = imx_clk_hw_mux("eim_slow_sel", base + 0x1c, 29, 2, eim_slow_sels, ARRAY_SIZE(eim_slow_sels));
+ hws[IMX6SX_CLK_USDHC1_SEL] = imx_clk_hw_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
+ hws[IMX6SX_CLK_USDHC2_SEL] = imx_clk_hw_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
+ hws[IMX6SX_CLK_USDHC3_SEL] = imx_clk_hw_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
+ hws[IMX6SX_CLK_USDHC4_SEL] = imx_clk_hw_mux("usdhc4_sel", base + 0x1c, 19, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
+ hws[IMX6SX_CLK_SSI3_SEL] = imx_clk_hw_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
+ hws[IMX6SX_CLK_SSI2_SEL] = imx_clk_hw_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
+ hws[IMX6SX_CLK_SSI1_SEL] = imx_clk_hw_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels));
+ hws[IMX6SX_CLK_QSPI1_SEL] = imx_clk_hw_mux_flags("qspi1_sel", base + 0x1c, 7, 3, qspi1_sels, ARRAY_SIZE(qspi1_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SX_CLK_PERCLK_SEL] = imx_clk_hw_mux("perclk_sel", base + 0x1c, 6, 1, perclk_sels, ARRAY_SIZE(perclk_sels));
+ hws[IMX6SX_CLK_VID_SEL] = imx_clk_hw_mux("vid_sel", base + 0x20, 21, 3, vid_sels, ARRAY_SIZE(vid_sels));
+ hws[IMX6SX_CLK_ESAI_SEL] = imx_clk_hw_mux("esai_sel", base + 0x20, 19, 2, audio_sels, ARRAY_SIZE(audio_sels));
+ hws[IMX6SX_CLK_CAN_SEL] = imx_clk_hw_mux("can_sel", base + 0x20, 8, 2, can_sels, ARRAY_SIZE(can_sels));
+ hws[IMX6SX_CLK_UART_SEL] = imx_clk_hw_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels));
+ hws[IMX6SX_CLK_QSPI2_SEL] = imx_clk_hw_mux_flags("qspi2_sel", base + 0x2c, 15, 3, qspi2_sels, ARRAY_SIZE(qspi2_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SX_CLK_SPDIF_SEL] = imx_clk_hw_mux("spdif_sel", base + 0x30, 20, 2, audio_sels, ARRAY_SIZE(audio_sels));
+ hws[IMX6SX_CLK_AUDIO_SEL] = imx_clk_hw_mux("audio_sel", base + 0x30, 7, 2, audio_sels, ARRAY_SIZE(audio_sels));
+ hws[IMX6SX_CLK_ENET_PRE_SEL] = imx_clk_hw_mux("enet_pre_sel", base + 0x34, 15, 3, enet_pre_sels, ARRAY_SIZE(enet_pre_sels));
+ hws[IMX6SX_CLK_ENET_SEL] = imx_clk_hw_mux("enet_sel", base + 0x34, 9, 3, enet_sels, ARRAY_SIZE(enet_sels));
+ hws[IMX6SX_CLK_M4_PRE_SEL] = imx_clk_hw_mux("m4_pre_sel", base + 0x34, 6, 3, m4_pre_sels, ARRAY_SIZE(m4_pre_sels));
+ hws[IMX6SX_CLK_M4_SEL] = imx_clk_hw_mux("m4_sel", base + 0x34, 0, 3, m4_sels, ARRAY_SIZE(m4_sels));
+ hws[IMX6SX_CLK_ECSPI_SEL] = imx_clk_hw_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels));
+ hws[IMX6SX_CLK_LCDIF2_PRE_SEL] = imx_clk_hw_mux("lcdif2_pre_sel", base + 0x38, 6, 3, lcdif2_pre_sels, ARRAY_SIZE(lcdif2_pre_sels));
+ hws[IMX6SX_CLK_LCDIF2_SEL] = imx_clk_hw_mux("lcdif2_sel", base + 0x38, 0, 3, lcdif2_sels, ARRAY_SIZE(lcdif2_sels));
+ hws[IMX6SX_CLK_DISPLAY_SEL] = imx_clk_hw_mux("display_sel", base + 0x3c, 14, 2, display_sels, ARRAY_SIZE(display_sels));
+ hws[IMX6SX_CLK_CSI_SEL] = imx_clk_hw_mux("csi_sel", base + 0x3c, 9, 2, csi_sels, ARRAY_SIZE(csi_sels));
+ hws[IMX6SX_CLK_CKO1_SEL] = imx_clk_hw_mux("cko1_sel", base + 0x60, 0, 4, cko1_sels, ARRAY_SIZE(cko1_sels));
+ hws[IMX6SX_CLK_CKO2_SEL] = imx_clk_hw_mux("cko2_sel", base + 0x60, 16, 5, cko2_sels, ARRAY_SIZE(cko2_sels));
+ hws[IMX6SX_CLK_CKO] = imx_clk_hw_mux("cko", base + 0x60, 8, 1, cko_sels, ARRAY_SIZE(cko_sels));
+
+ hws[IMX6SX_CLK_LDB_DI1_DIV_SEL] = imx_clk_hw_mux_flags("ldb_di1_div_sel", base + 0x20, 11, 1, ldb_di1_div_sels, ARRAY_SIZE(ldb_di1_div_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SX_CLK_LDB_DI0_DIV_SEL] = imx_clk_hw_mux_flags("ldb_di0_div_sel", base + 0x20, 10, 1, ldb_di0_div_sels, ARRAY_SIZE(ldb_di0_div_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SX_CLK_LDB_DI1_SEL] = imx_clk_hw_mux_flags("ldb_di1_sel", base + 0x2c, 12, 3, ldb_di1_sels, ARRAY_SIZE(ldb_di1_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SX_CLK_LDB_DI0_SEL] = imx_clk_hw_mux_flags("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di0_sels, ARRAY_SIZE(ldb_di0_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SX_CLK_LCDIF1_PRE_SEL] = imx_clk_hw_mux_flags("lcdif1_pre_sel", base + 0x38, 15, 3, lcdif1_pre_sels, ARRAY_SIZE(lcdif1_pre_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6SX_CLK_LCDIF1_SEL] = imx_clk_hw_mux_flags("lcdif1_sel", base + 0x38, 9, 3, lcdif1_sels, ARRAY_SIZE(lcdif1_sels), CLK_SET_RATE_PARENT);
/* name parent_name reg shift width */
- clks[IMX6SX_CLK_PERIPH_CLK2] = imx_clk_divider("periph_clk2", "periph_clk2_sel", base + 0x14, 27, 3);
- clks[IMX6SX_CLK_PERIPH2_CLK2] = imx_clk_divider("periph2_clk2", "periph2_clk2_sel", base + 0x14, 0, 3);
- clks[IMX6SX_CLK_IPG] = imx_clk_divider("ipg", "ahb", base + 0x14, 8, 2);
- clks[IMX6SX_CLK_GPU_CORE_PODF] = imx_clk_divider("gpu_core_podf", "gpu_core_sel", base + 0x18, 29, 3);
- clks[IMX6SX_CLK_GPU_AXI_PODF] = imx_clk_divider("gpu_axi_podf", "gpu_axi_sel", base + 0x18, 26, 3);
- clks[IMX6SX_CLK_LCDIF1_PODF] = imx_clk_divider("lcdif1_podf", "lcdif1_pred", base + 0x18, 23, 3);
- clks[IMX6SX_CLK_QSPI1_PODF] = imx_clk_divider("qspi1_podf", "qspi1_sel", base + 0x1c, 26, 3);
- clks[IMX6SX_CLK_EIM_SLOW_PODF] = imx_clk_divider("eim_slow_podf", "eim_slow_sel", base + 0x1c, 23, 3);
- clks[IMX6SX_CLK_LCDIF2_PODF] = imx_clk_divider("lcdif2_podf", "lcdif2_pred", base + 0x1c, 20, 3);
- clks[IMX6SX_CLK_PERCLK] = imx_clk_divider_flags("perclk", "perclk_sel", base + 0x1c, 0, 6, CLK_IS_CRITICAL);
- clks[IMX6SX_CLK_VID_PODF] = imx_clk_divider("vid_podf", "vid_sel", base + 0x20, 24, 2);
- clks[IMX6SX_CLK_CAN_PODF] = imx_clk_divider("can_podf", "can_sel", base + 0x20, 2, 6);
- clks[IMX6SX_CLK_USDHC4_PODF] = imx_clk_divider("usdhc4_podf", "usdhc4_sel", base + 0x24, 22, 3);
- clks[IMX6SX_CLK_USDHC3_PODF] = imx_clk_divider("usdhc3_podf", "usdhc3_sel", base + 0x24, 19, 3);
- clks[IMX6SX_CLK_USDHC2_PODF] = imx_clk_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3);
- clks[IMX6SX_CLK_USDHC1_PODF] = imx_clk_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3);
- clks[IMX6SX_CLK_UART_PODF] = imx_clk_divider("uart_podf", "uart_sel", base + 0x24, 0, 6);
- clks[IMX6SX_CLK_ESAI_PRED] = imx_clk_divider("esai_pred", "esai_sel", base + 0x28, 9, 3);
- clks[IMX6SX_CLK_ESAI_PODF] = imx_clk_divider("esai_podf", "esai_pred", base + 0x28, 25, 3);
- clks[IMX6SX_CLK_SSI3_PRED] = imx_clk_divider("ssi3_pred", "ssi3_sel", base + 0x28, 22, 3);
- clks[IMX6SX_CLK_SSI3_PODF] = imx_clk_divider("ssi3_podf", "ssi3_pred", base + 0x28, 16, 6);
- clks[IMX6SX_CLK_SSI1_PRED] = imx_clk_divider("ssi1_pred", "ssi1_sel", base + 0x28, 6, 3);
- clks[IMX6SX_CLK_SSI1_PODF] = imx_clk_divider("ssi1_podf", "ssi1_pred", base + 0x28, 0, 6);
- clks[IMX6SX_CLK_QSPI2_PRED] = imx_clk_divider("qspi2_pred", "qspi2_sel", base + 0x2c, 18, 3);
- clks[IMX6SX_CLK_QSPI2_PODF] = imx_clk_divider("qspi2_podf", "qspi2_pred", base + 0x2c, 21, 6);
- clks[IMX6SX_CLK_SSI2_PRED] = imx_clk_divider("ssi2_pred", "ssi2_sel", base + 0x2c, 6, 3);
- clks[IMX6SX_CLK_SSI2_PODF] = imx_clk_divider("ssi2_podf", "ssi2_pred", base + 0x2c, 0, 6);
- clks[IMX6SX_CLK_SPDIF_PRED] = imx_clk_divider("spdif_pred", "spdif_sel", base + 0x30, 25, 3);
- clks[IMX6SX_CLK_SPDIF_PODF] = imx_clk_divider("spdif_podf", "spdif_pred", base + 0x30, 22, 3);
- clks[IMX6SX_CLK_AUDIO_PRED] = imx_clk_divider("audio_pred", "audio_sel", base + 0x30, 12, 3);
- clks[IMX6SX_CLK_AUDIO_PODF] = imx_clk_divider("audio_podf", "audio_pred", base + 0x30, 9, 3);
- clks[IMX6SX_CLK_ENET_PODF] = imx_clk_divider("enet_podf", "enet_pre_sel", base + 0x34, 12, 3);
- clks[IMX6SX_CLK_M4_PODF] = imx_clk_divider("m4_podf", "m4_sel", base + 0x34, 3, 3);
- clks[IMX6SX_CLK_ECSPI_PODF] = imx_clk_divider("ecspi_podf", "ecspi_sel", base + 0x38, 19, 6);
- clks[IMX6SX_CLK_LCDIF1_PRED] = imx_clk_divider("lcdif1_pred", "lcdif1_pre_sel", base + 0x38, 12, 3);
- clks[IMX6SX_CLK_LCDIF2_PRED] = imx_clk_divider("lcdif2_pred", "lcdif2_pre_sel", base + 0x38, 3, 3);
- clks[IMX6SX_CLK_DISPLAY_PODF] = imx_clk_divider("display_podf", "display_sel", base + 0x3c, 16, 3);
- clks[IMX6SX_CLK_CSI_PODF] = imx_clk_divider("csi_podf", "csi_sel", base + 0x3c, 11, 3);
- clks[IMX6SX_CLK_CKO1_PODF] = imx_clk_divider("cko1_podf", "cko1_sel", base + 0x60, 4, 3);
- clks[IMX6SX_CLK_CKO2_PODF] = imx_clk_divider("cko2_podf", "cko2_sel", base + 0x60, 21, 3);
-
- clks[IMX6SX_CLK_LDB_DI0_DIV_3_5] = imx_clk_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7);
- clks[IMX6SX_CLK_LDB_DI0_DIV_7] = imx_clk_fixed_factor("ldb_di0_div_7", "ldb_di0_sel", 1, 7);
- clks[IMX6SX_CLK_LDB_DI1_DIV_3_5] = imx_clk_fixed_factor("ldb_di1_div_3_5", "ldb_di1_sel", 2, 7);
- clks[IMX6SX_CLK_LDB_DI1_DIV_7] = imx_clk_fixed_factor("ldb_di1_div_7", "ldb_di1_sel", 1, 7);
+ hws[IMX6SX_CLK_PERIPH_CLK2] = imx_clk_hw_divider("periph_clk2", "periph_clk2_sel", base + 0x14, 27, 3);
+ hws[IMX6SX_CLK_PERIPH2_CLK2] = imx_clk_hw_divider("periph2_clk2", "periph2_clk2_sel", base + 0x14, 0, 3);
+ hws[IMX6SX_CLK_IPG] = imx_clk_hw_divider("ipg", "ahb", base + 0x14, 8, 2);
+ hws[IMX6SX_CLK_GPU_CORE_PODF] = imx_clk_hw_divider("gpu_core_podf", "gpu_core_sel", base + 0x18, 29, 3);
+ hws[IMX6SX_CLK_GPU_AXI_PODF] = imx_clk_hw_divider("gpu_axi_podf", "gpu_axi_sel", base + 0x18, 26, 3);
+ hws[IMX6SX_CLK_LCDIF1_PODF] = imx_clk_hw_divider("lcdif1_podf", "lcdif1_pred", base + 0x18, 23, 3);
+ hws[IMX6SX_CLK_QSPI1_PODF] = imx_clk_hw_divider("qspi1_podf", "qspi1_sel", base + 0x1c, 26, 3);
+ hws[IMX6SX_CLK_EIM_SLOW_PODF] = imx_clk_hw_divider("eim_slow_podf", "eim_slow_sel", base + 0x1c, 23, 3);
+ hws[IMX6SX_CLK_LCDIF2_PODF] = imx_clk_hw_divider("lcdif2_podf", "lcdif2_pred", base + 0x1c, 20, 3);
+ hws[IMX6SX_CLK_PERCLK] = imx_clk_hw_divider_flags("perclk", "perclk_sel", base + 0x1c, 0, 6, CLK_IS_CRITICAL);
+ hws[IMX6SX_CLK_VID_PODF] = imx_clk_hw_divider("vid_podf", "vid_sel", base + 0x20, 24, 2);
+ hws[IMX6SX_CLK_CAN_PODF] = imx_clk_hw_divider("can_podf", "can_sel", base + 0x20, 2, 6);
+ hws[IMX6SX_CLK_USDHC4_PODF] = imx_clk_hw_divider("usdhc4_podf", "usdhc4_sel", base + 0x24, 22, 3);
+ hws[IMX6SX_CLK_USDHC3_PODF] = imx_clk_hw_divider("usdhc3_podf", "usdhc3_sel", base + 0x24, 19, 3);
+ hws[IMX6SX_CLK_USDHC2_PODF] = imx_clk_hw_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3);
+ hws[IMX6SX_CLK_USDHC1_PODF] = imx_clk_hw_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3);
+ hws[IMX6SX_CLK_UART_PODF] = imx_clk_hw_divider("uart_podf", "uart_sel", base + 0x24, 0, 6);
+ hws[IMX6SX_CLK_ESAI_PRED] = imx_clk_hw_divider("esai_pred", "esai_sel", base + 0x28, 9, 3);
+ hws[IMX6SX_CLK_ESAI_PODF] = imx_clk_hw_divider("esai_podf", "esai_pred", base + 0x28, 25, 3);
+ hws[IMX6SX_CLK_SSI3_PRED] = imx_clk_hw_divider("ssi3_pred", "ssi3_sel", base + 0x28, 22, 3);
+ hws[IMX6SX_CLK_SSI3_PODF] = imx_clk_hw_divider("ssi3_podf", "ssi3_pred", base + 0x28, 16, 6);
+ hws[IMX6SX_CLK_SSI1_PRED] = imx_clk_hw_divider("ssi1_pred", "ssi1_sel", base + 0x28, 6, 3);
+ hws[IMX6SX_CLK_SSI1_PODF] = imx_clk_hw_divider("ssi1_podf", "ssi1_pred", base + 0x28, 0, 6);
+ hws[IMX6SX_CLK_QSPI2_PRED] = imx_clk_hw_divider("qspi2_pred", "qspi2_sel", base + 0x2c, 18, 3);
+ hws[IMX6SX_CLK_QSPI2_PODF] = imx_clk_hw_divider("qspi2_podf", "qspi2_pred", base + 0x2c, 21, 6);
+ hws[IMX6SX_CLK_SSI2_PRED] = imx_clk_hw_divider("ssi2_pred", "ssi2_sel", base + 0x2c, 6, 3);
+ hws[IMX6SX_CLK_SSI2_PODF] = imx_clk_hw_divider("ssi2_podf", "ssi2_pred", base + 0x2c, 0, 6);
+ hws[IMX6SX_CLK_SPDIF_PRED] = imx_clk_hw_divider("spdif_pred", "spdif_sel", base + 0x30, 25, 3);
+ hws[IMX6SX_CLK_SPDIF_PODF] = imx_clk_hw_divider("spdif_podf", "spdif_pred", base + 0x30, 22, 3);
+ hws[IMX6SX_CLK_AUDIO_PRED] = imx_clk_hw_divider("audio_pred", "audio_sel", base + 0x30, 12, 3);
+ hws[IMX6SX_CLK_AUDIO_PODF] = imx_clk_hw_divider("audio_podf", "audio_pred", base + 0x30, 9, 3);
+ hws[IMX6SX_CLK_ENET_PODF] = imx_clk_hw_divider("enet_podf", "enet_pre_sel", base + 0x34, 12, 3);
+ hws[IMX6SX_CLK_M4_PODF] = imx_clk_hw_divider("m4_podf", "m4_sel", base + 0x34, 3, 3);
+ hws[IMX6SX_CLK_ECSPI_PODF] = imx_clk_hw_divider("ecspi_podf", "ecspi_sel", base + 0x38, 19, 6);
+ hws[IMX6SX_CLK_LCDIF1_PRED] = imx_clk_hw_divider("lcdif1_pred", "lcdif1_pre_sel", base + 0x38, 12, 3);
+ hws[IMX6SX_CLK_LCDIF2_PRED] = imx_clk_hw_divider("lcdif2_pred", "lcdif2_pre_sel", base + 0x38, 3, 3);
+ hws[IMX6SX_CLK_DISPLAY_PODF] = imx_clk_hw_divider("display_podf", "display_sel", base + 0x3c, 16, 3);
+ hws[IMX6SX_CLK_CSI_PODF] = imx_clk_hw_divider("csi_podf", "csi_sel", base + 0x3c, 11, 3);
+ hws[IMX6SX_CLK_CKO1_PODF] = imx_clk_hw_divider("cko1_podf", "cko1_sel", base + 0x60, 4, 3);
+ hws[IMX6SX_CLK_CKO2_PODF] = imx_clk_hw_divider("cko2_podf", "cko2_sel", base + 0x60, 21, 3);
+
+ hws[IMX6SX_CLK_LDB_DI0_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7);
+ hws[IMX6SX_CLK_LDB_DI0_DIV_7] = imx_clk_hw_fixed_factor("ldb_di0_div_7", "ldb_di0_sel", 1, 7);
+ hws[IMX6SX_CLK_LDB_DI1_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di1_div_3_5", "ldb_di1_sel", 2, 7);
+ hws[IMX6SX_CLK_LDB_DI1_DIV_7] = imx_clk_hw_fixed_factor("ldb_di1_div_7", "ldb_di1_sel", 1, 7);
/* name reg shift width busy: reg, shift parent_names num_parents */
- clks[IMX6SX_CLK_PERIPH] = imx_clk_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels));
- clks[IMX6SX_CLK_PERIPH2] = imx_clk_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels));
+ hws[IMX6SX_CLK_PERIPH] = imx_clk_hw_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels));
+ hws[IMX6SX_CLK_PERIPH2] = imx_clk_hw_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels));
/* name parent_name reg shift width busy: reg, shift */
- clks[IMX6SX_CLK_OCRAM_PODF] = imx_clk_busy_divider("ocram_podf", "ocram_sel", base + 0x14, 16, 3, base + 0x48, 0);
- clks[IMX6SX_CLK_AHB] = imx_clk_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1);
- clks[IMX6SX_CLK_MMDC_PODF] = imx_clk_busy_divider("mmdc_podf", "periph2", base + 0x14, 3, 3, base + 0x48, 2);
- clks[IMX6SX_CLK_ARM] = imx_clk_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16);
+ hws[IMX6SX_CLK_OCRAM_PODF] = imx_clk_hw_busy_divider("ocram_podf", "ocram_sel", base + 0x14, 16, 3, base + 0x48, 0);
+ hws[IMX6SX_CLK_AHB] = imx_clk_hw_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1);
+ hws[IMX6SX_CLK_MMDC_PODF] = imx_clk_hw_busy_divider("mmdc_podf", "periph2", base + 0x14, 3, 3, base + 0x48, 2);
+ hws[IMX6SX_CLK_ARM] = imx_clk_hw_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16);
/* name parent_name reg shift */
/* CCGR0 */
- clks[IMX6SX_CLK_AIPS_TZ1] = imx_clk_gate2_flags("aips_tz1", "ahb", base + 0x68, 0, CLK_IS_CRITICAL);
- clks[IMX6SX_CLK_AIPS_TZ2] = imx_clk_gate2_flags("aips_tz2", "ahb", base + 0x68, 2, CLK_IS_CRITICAL);
- clks[IMX6SX_CLK_APBH_DMA] = imx_clk_gate2("apbh_dma", "usdhc3", base + 0x68, 4);
- clks[IMX6SX_CLK_ASRC_MEM] = imx_clk_gate2_shared("asrc_mem", "ahb", base + 0x68, 6, &share_count_asrc);
- clks[IMX6SX_CLK_ASRC_IPG] = imx_clk_gate2_shared("asrc_ipg", "ahb", base + 0x68, 6, &share_count_asrc);
- clks[IMX6SX_CLK_CAAM_MEM] = imx_clk_gate2("caam_mem", "ahb", base + 0x68, 8);
- clks[IMX6SX_CLK_CAAM_ACLK] = imx_clk_gate2("caam_aclk", "ahb", base + 0x68, 10);
- clks[IMX6SX_CLK_CAAM_IPG] = imx_clk_gate2("caam_ipg", "ipg", base + 0x68, 12);
- clks[IMX6SX_CLK_CAN1_IPG] = imx_clk_gate2("can1_ipg", "ipg", base + 0x68, 14);
- clks[IMX6SX_CLK_CAN1_SERIAL] = imx_clk_gate2("can1_serial", "can_podf", base + 0x68, 16);
- clks[IMX6SX_CLK_CAN2_IPG] = imx_clk_gate2("can2_ipg", "ipg", base + 0x68, 18);
- clks[IMX6SX_CLK_CAN2_SERIAL] = imx_clk_gate2("can2_serial", "can_podf", base + 0x68, 20);
- clks[IMX6SX_CLK_DCIC1] = imx_clk_gate2("dcic1", "display_podf", base + 0x68, 24);
- clks[IMX6SX_CLK_DCIC2] = imx_clk_gate2("dcic2", "display_podf", base + 0x68, 26);
- clks[IMX6SX_CLK_AIPS_TZ3] = imx_clk_gate2_flags("aips_tz3", "ahb", base + 0x68, 30, CLK_IS_CRITICAL);
+ hws[IMX6SX_CLK_AIPS_TZ1] = imx_clk_hw_gate2_flags("aips_tz1", "ahb", base + 0x68, 0, CLK_IS_CRITICAL);
+ hws[IMX6SX_CLK_AIPS_TZ2] = imx_clk_hw_gate2_flags("aips_tz2", "ahb", base + 0x68, 2, CLK_IS_CRITICAL);
+ hws[IMX6SX_CLK_APBH_DMA] = imx_clk_hw_gate2("apbh_dma", "usdhc3", base + 0x68, 4);
+ hws[IMX6SX_CLK_ASRC_MEM] = imx_clk_hw_gate2_shared("asrc_mem", "ahb", base + 0x68, 6, &share_count_asrc);
+ hws[IMX6SX_CLK_ASRC_IPG] = imx_clk_hw_gate2_shared("asrc_ipg", "ahb", base + 0x68, 6, &share_count_asrc);
+ hws[IMX6SX_CLK_CAAM_MEM] = imx_clk_hw_gate2("caam_mem", "ahb", base + 0x68, 8);
+ hws[IMX6SX_CLK_CAAM_ACLK] = imx_clk_hw_gate2("caam_aclk", "ahb", base + 0x68, 10);
+ hws[IMX6SX_CLK_CAAM_IPG] = imx_clk_hw_gate2("caam_ipg", "ipg", base + 0x68, 12);
+ hws[IMX6SX_CLK_CAN1_IPG] = imx_clk_hw_gate2("can1_ipg", "ipg", base + 0x68, 14);
+ hws[IMX6SX_CLK_CAN1_SERIAL] = imx_clk_hw_gate2("can1_serial", "can_podf", base + 0x68, 16);
+ hws[IMX6SX_CLK_CAN2_IPG] = imx_clk_hw_gate2("can2_ipg", "ipg", base + 0x68, 18);
+ hws[IMX6SX_CLK_CAN2_SERIAL] = imx_clk_hw_gate2("can2_serial", "can_podf", base + 0x68, 20);
+ hws[IMX6SX_CLK_DCIC1] = imx_clk_hw_gate2("dcic1", "display_podf", base + 0x68, 24);
+ hws[IMX6SX_CLK_DCIC2] = imx_clk_hw_gate2("dcic2", "display_podf", base + 0x68, 26);
+ hws[IMX6SX_CLK_AIPS_TZ3] = imx_clk_hw_gate2_flags("aips_tz3", "ahb", base + 0x68, 30, CLK_IS_CRITICAL);
/* CCGR1 */
- clks[IMX6SX_CLK_ECSPI1] = imx_clk_gate2("ecspi1", "ecspi_podf", base + 0x6c, 0);
- clks[IMX6SX_CLK_ECSPI2] = imx_clk_gate2("ecspi2", "ecspi_podf", base + 0x6c, 2);
- clks[IMX6SX_CLK_ECSPI3] = imx_clk_gate2("ecspi3", "ecspi_podf", base + 0x6c, 4);
- clks[IMX6SX_CLK_ECSPI4] = imx_clk_gate2("ecspi4", "ecspi_podf", base + 0x6c, 6);
- clks[IMX6SX_CLK_ECSPI5] = imx_clk_gate2("ecspi5", "ecspi_podf", base + 0x6c, 8);
- clks[IMX6SX_CLK_EPIT1] = imx_clk_gate2("epit1", "perclk", base + 0x6c, 12);
- clks[IMX6SX_CLK_EPIT2] = imx_clk_gate2("epit2", "perclk", base + 0x6c, 14);
- clks[IMX6SX_CLK_ESAI_EXTAL] = imx_clk_gate2_shared("esai_extal", "esai_podf", base + 0x6c, 16, &share_count_esai);
- clks[IMX6SX_CLK_ESAI_IPG] = imx_clk_gate2_shared("esai_ipg", "ahb", base + 0x6c, 16, &share_count_esai);
- clks[IMX6SX_CLK_ESAI_MEM] = imx_clk_gate2_shared("esai_mem", "ahb", base + 0x6c, 16, &share_count_esai);
- clks[IMX6SX_CLK_WAKEUP] = imx_clk_gate2_flags("wakeup", "ipg", base + 0x6c, 18, CLK_IS_CRITICAL);
- clks[IMX6SX_CLK_GPT_BUS] = imx_clk_gate2("gpt_bus", "perclk", base + 0x6c, 20);
- clks[IMX6SX_CLK_GPT_SERIAL] = imx_clk_gate2("gpt_serial", "perclk", base + 0x6c, 22);
- clks[IMX6SX_CLK_GPU] = imx_clk_gate2("gpu", "gpu_core_podf", base + 0x6c, 26);
- clks[IMX6SX_CLK_OCRAM_S] = imx_clk_gate2("ocram_s", "ahb", base + 0x6c, 28);
- clks[IMX6SX_CLK_CANFD] = imx_clk_gate2("canfd", "can_podf", base + 0x6c, 30);
+ hws[IMX6SX_CLK_ECSPI1] = imx_clk_hw_gate2("ecspi1", "ecspi_podf", base + 0x6c, 0);
+ hws[IMX6SX_CLK_ECSPI2] = imx_clk_hw_gate2("ecspi2", "ecspi_podf", base + 0x6c, 2);
+ hws[IMX6SX_CLK_ECSPI3] = imx_clk_hw_gate2("ecspi3", "ecspi_podf", base + 0x6c, 4);
+ hws[IMX6SX_CLK_ECSPI4] = imx_clk_hw_gate2("ecspi4", "ecspi_podf", base + 0x6c, 6);
+ hws[IMX6SX_CLK_ECSPI5] = imx_clk_hw_gate2("ecspi5", "ecspi_podf", base + 0x6c, 8);
+ hws[IMX6SX_CLK_EPIT1] = imx_clk_hw_gate2("epit1", "perclk", base + 0x6c, 12);
+ hws[IMX6SX_CLK_EPIT2] = imx_clk_hw_gate2("epit2", "perclk", base + 0x6c, 14);
+ hws[IMX6SX_CLK_ESAI_EXTAL] = imx_clk_hw_gate2_shared("esai_extal", "esai_podf", base + 0x6c, 16, &share_count_esai);
+ hws[IMX6SX_CLK_ESAI_IPG] = imx_clk_hw_gate2_shared("esai_ipg", "ahb", base + 0x6c, 16, &share_count_esai);
+ hws[IMX6SX_CLK_ESAI_MEM] = imx_clk_hw_gate2_shared("esai_mem", "ahb", base + 0x6c, 16, &share_count_esai);
+ hws[IMX6SX_CLK_WAKEUP] = imx_clk_hw_gate2_flags("wakeup", "ipg", base + 0x6c, 18, CLK_IS_CRITICAL);
+ hws[IMX6SX_CLK_GPT_BUS] = imx_clk_hw_gate2("gpt_bus", "perclk", base + 0x6c, 20);
+ hws[IMX6SX_CLK_GPT_SERIAL] = imx_clk_hw_gate2("gpt_serial", "perclk", base + 0x6c, 22);
+ hws[IMX6SX_CLK_GPU] = imx_clk_hw_gate2("gpu", "gpu_core_podf", base + 0x6c, 26);
+ hws[IMX6SX_CLK_OCRAM_S] = imx_clk_hw_gate2("ocram_s", "ahb", base + 0x6c, 28);
+ hws[IMX6SX_CLK_CANFD] = imx_clk_hw_gate2("canfd", "can_podf", base + 0x6c, 30);
/* CCGR2 */
- clks[IMX6SX_CLK_CSI] = imx_clk_gate2("csi", "csi_podf", base + 0x70, 2);
- clks[IMX6SX_CLK_I2C1] = imx_clk_gate2("i2c1", "perclk", base + 0x70, 6);
- clks[IMX6SX_CLK_I2C2] = imx_clk_gate2("i2c2", "perclk", base + 0x70, 8);
- clks[IMX6SX_CLK_I2C3] = imx_clk_gate2("i2c3", "perclk", base + 0x70, 10);
- clks[IMX6SX_CLK_OCOTP] = imx_clk_gate2("ocotp", "ipg", base + 0x70, 12);
- clks[IMX6SX_CLK_IOMUXC] = imx_clk_gate2("iomuxc", "lcdif1_podf", base + 0x70, 14);
- clks[IMX6SX_CLK_IPMUX1] = imx_clk_gate2_flags("ipmux1", "ahb", base + 0x70, 16, CLK_IS_CRITICAL);
- clks[IMX6SX_CLK_IPMUX2] = imx_clk_gate2_flags("ipmux2", "ahb", base + 0x70, 18, CLK_IS_CRITICAL);
- clks[IMX6SX_CLK_IPMUX3] = imx_clk_gate2_flags("ipmux3", "ahb", base + 0x70, 20, CLK_IS_CRITICAL);
- clks[IMX6SX_CLK_TZASC1] = imx_clk_gate2_flags("tzasc1", "mmdc_podf", base + 0x70, 22, CLK_IS_CRITICAL);
- clks[IMX6SX_CLK_LCDIF_APB] = imx_clk_gate2("lcdif_apb", "display_podf", base + 0x70, 28);
- clks[IMX6SX_CLK_PXP_AXI] = imx_clk_gate2("pxp_axi", "display_podf", base + 0x70, 30);
+ hws[IMX6SX_CLK_CSI] = imx_clk_hw_gate2("csi", "csi_podf", base + 0x70, 2);
+ hws[IMX6SX_CLK_I2C1] = imx_clk_hw_gate2("i2c1", "perclk", base + 0x70, 6);
+ hws[IMX6SX_CLK_I2C2] = imx_clk_hw_gate2("i2c2", "perclk", base + 0x70, 8);
+ hws[IMX6SX_CLK_I2C3] = imx_clk_hw_gate2("i2c3", "perclk", base + 0x70, 10);
+ hws[IMX6SX_CLK_OCOTP] = imx_clk_hw_gate2("ocotp", "ipg", base + 0x70, 12);
+ hws[IMX6SX_CLK_IOMUXC] = imx_clk_hw_gate2("iomuxc", "lcdif1_podf", base + 0x70, 14);
+ hws[IMX6SX_CLK_IPMUX1] = imx_clk_hw_gate2_flags("ipmux1", "ahb", base + 0x70, 16, CLK_IS_CRITICAL);
+ hws[IMX6SX_CLK_IPMUX2] = imx_clk_hw_gate2_flags("ipmux2", "ahb", base + 0x70, 18, CLK_IS_CRITICAL);
+ hws[IMX6SX_CLK_IPMUX3] = imx_clk_hw_gate2_flags("ipmux3", "ahb", base + 0x70, 20, CLK_IS_CRITICAL);
+ hws[IMX6SX_CLK_TZASC1] = imx_clk_hw_gate2_flags("tzasc1", "mmdc_podf", base + 0x70, 22, CLK_IS_CRITICAL);
+ hws[IMX6SX_CLK_LCDIF_APB] = imx_clk_hw_gate2("lcdif_apb", "display_podf", base + 0x70, 28);
+ hws[IMX6SX_CLK_PXP_AXI] = imx_clk_hw_gate2("pxp_axi", "display_podf", base + 0x70, 30);
/* CCGR3 */
- clks[IMX6SX_CLK_M4] = imx_clk_gate2("m4", "m4_podf", base + 0x74, 2);
- clks[IMX6SX_CLK_ENET] = imx_clk_gate2("enet", "ipg", base + 0x74, 4);
- clks[IMX6SX_CLK_ENET_AHB] = imx_clk_gate2("enet_ahb", "enet_sel", base + 0x74, 4);
- clks[IMX6SX_CLK_DISPLAY_AXI] = imx_clk_gate2("display_axi", "display_podf", base + 0x74, 6);
- clks[IMX6SX_CLK_LCDIF2_PIX] = imx_clk_gate2("lcdif2_pix", "lcdif2_sel", base + 0x74, 8);
- clks[IMX6SX_CLK_LCDIF1_PIX] = imx_clk_gate2("lcdif1_pix", "lcdif1_sel", base + 0x74, 10);
- clks[IMX6SX_CLK_LDB_DI0] = imx_clk_gate2("ldb_di0", "ldb_di0_div_sel", base + 0x74, 12);
- clks[IMX6SX_CLK_QSPI1] = imx_clk_gate2("qspi1", "qspi1_podf", base + 0x74, 14);
- clks[IMX6SX_CLK_MLB] = imx_clk_gate2("mlb", "ahb", base + 0x74, 18);
- clks[IMX6SX_CLK_MMDC_P0_FAST] = imx_clk_gate2_flags("mmdc_p0_fast", "mmdc_podf", base + 0x74, 20, CLK_IS_CRITICAL);
- clks[IMX6SX_CLK_MMDC_P0_IPG] = imx_clk_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL);
- clks[IMX6SX_CLK_MMDC_P1_IPG] = imx_clk_gate2("mmdc_p1_ipg", "ipg", base + 0x74, 26);
- clks[IMX6SX_CLK_OCRAM] = imx_clk_gate2_flags("ocram", "ocram_podf", base + 0x74, 28, CLK_IS_CRITICAL);
+ hws[IMX6SX_CLK_M4] = imx_clk_hw_gate2("m4", "m4_podf", base + 0x74, 2);
+ hws[IMX6SX_CLK_ENET] = imx_clk_hw_gate2("enet", "ipg", base + 0x74, 4);
+ hws[IMX6SX_CLK_ENET_AHB] = imx_clk_hw_gate2("enet_ahb", "enet_sel", base + 0x74, 4);
+ hws[IMX6SX_CLK_DISPLAY_AXI] = imx_clk_hw_gate2("display_axi", "display_podf", base + 0x74, 6);
+ hws[IMX6SX_CLK_LCDIF2_PIX] = imx_clk_hw_gate2("lcdif2_pix", "lcdif2_sel", base + 0x74, 8);
+ hws[IMX6SX_CLK_LCDIF1_PIX] = imx_clk_hw_gate2("lcdif1_pix", "lcdif1_sel", base + 0x74, 10);
+ hws[IMX6SX_CLK_LDB_DI0] = imx_clk_hw_gate2("ldb_di0", "ldb_di0_div_sel", base + 0x74, 12);
+ hws[IMX6SX_CLK_QSPI1] = imx_clk_hw_gate2("qspi1", "qspi1_podf", base + 0x74, 14);
+ hws[IMX6SX_CLK_MLB] = imx_clk_hw_gate2("mlb", "ahb", base + 0x74, 18);
+ hws[IMX6SX_CLK_MMDC_P0_FAST] = imx_clk_hw_gate2_flags("mmdc_p0_fast", "mmdc_podf", base + 0x74, 20, CLK_IS_CRITICAL);
+ hws[IMX6SX_CLK_MMDC_P0_IPG] = imx_clk_hw_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL);
+ hws[IMX6SX_CLK_MMDC_P1_IPG] = imx_clk_hw_gate2_flags("mmdc_p1_ipg", "ipg", base + 0x74, 26, CLK_IS_CRITICAL);
+ hws[IMX6SX_CLK_OCRAM] = imx_clk_hw_gate2_flags("ocram", "ocram_podf", base + 0x74, 28, CLK_IS_CRITICAL);
/* CCGR4 */
- clks[IMX6SX_CLK_PCIE_AXI] = imx_clk_gate2("pcie_axi", "display_podf", base + 0x78, 0);
- clks[IMX6SX_CLK_QSPI2] = imx_clk_gate2("qspi2", "qspi2_podf", base + 0x78, 10);
- clks[IMX6SX_CLK_PER1_BCH] = imx_clk_gate2("per1_bch", "usdhc3", base + 0x78, 12);
- clks[IMX6SX_CLK_PER2_MAIN] = imx_clk_gate2_flags("per2_main", "ahb", base + 0x78, 14, CLK_IS_CRITICAL);
- clks[IMX6SX_CLK_PWM1] = imx_clk_gate2("pwm1", "perclk", base + 0x78, 16);
- clks[IMX6SX_CLK_PWM2] = imx_clk_gate2("pwm2", "perclk", base + 0x78, 18);
- clks[IMX6SX_CLK_PWM3] = imx_clk_gate2("pwm3", "perclk", base + 0x78, 20);
- clks[IMX6SX_CLK_PWM4] = imx_clk_gate2("pwm4", "perclk", base + 0x78, 22);
- clks[IMX6SX_CLK_GPMI_BCH_APB] = imx_clk_gate2("gpmi_bch_apb", "usdhc3", base + 0x78, 24);
- clks[IMX6SX_CLK_GPMI_BCH] = imx_clk_gate2("gpmi_bch", "usdhc4", base + 0x78, 26);
- clks[IMX6SX_CLK_GPMI_IO] = imx_clk_gate2("gpmi_io", "qspi2_podf", base + 0x78, 28);
- clks[IMX6SX_CLK_GPMI_APB] = imx_clk_gate2("gpmi_apb", "usdhc3", base + 0x78, 30);
+ hws[IMX6SX_CLK_PCIE_AXI] = imx_clk_hw_gate2("pcie_axi", "display_podf", base + 0x78, 0);
+ hws[IMX6SX_CLK_QSPI2] = imx_clk_hw_gate2("qspi2", "qspi2_podf", base + 0x78, 10);
+ hws[IMX6SX_CLK_PER1_BCH] = imx_clk_hw_gate2("per1_bch", "usdhc3", base + 0x78, 12);
+ hws[IMX6SX_CLK_PER2_MAIN] = imx_clk_hw_gate2_flags("per2_main", "ahb", base + 0x78, 14, CLK_IS_CRITICAL);
+ hws[IMX6SX_CLK_PWM1] = imx_clk_hw_gate2("pwm1", "perclk", base + 0x78, 16);
+ hws[IMX6SX_CLK_PWM2] = imx_clk_hw_gate2("pwm2", "perclk", base + 0x78, 18);
+ hws[IMX6SX_CLK_PWM3] = imx_clk_hw_gate2("pwm3", "perclk", base + 0x78, 20);
+ hws[IMX6SX_CLK_PWM4] = imx_clk_hw_gate2("pwm4", "perclk", base + 0x78, 22);
+ hws[IMX6SX_CLK_GPMI_BCH_APB] = imx_clk_hw_gate2("gpmi_bch_apb", "usdhc3", base + 0x78, 24);
+ hws[IMX6SX_CLK_GPMI_BCH] = imx_clk_hw_gate2("gpmi_bch", "usdhc4", base + 0x78, 26);
+ hws[IMX6SX_CLK_GPMI_IO] = imx_clk_hw_gate2("gpmi_io", "qspi2_podf", base + 0x78, 28);
+ hws[IMX6SX_CLK_GPMI_APB] = imx_clk_hw_gate2("gpmi_apb", "usdhc3", base + 0x78, 30);
/* CCGR5 */
- clks[IMX6SX_CLK_ROM] = imx_clk_gate2_flags("rom", "ahb", base + 0x7c, 0, CLK_IS_CRITICAL);
- clks[IMX6SX_CLK_SDMA] = imx_clk_gate2("sdma", "ahb", base + 0x7c, 6);
- clks[IMX6SX_CLK_SPBA] = imx_clk_gate2("spba", "ipg", base + 0x7c, 12);
- clks[IMX6SX_CLK_AUDIO] = imx_clk_gate2_shared("audio", "audio_podf", base + 0x7c, 14, &share_count_audio);
- clks[IMX6SX_CLK_SPDIF] = imx_clk_gate2_shared("spdif", "spdif_podf", base + 0x7c, 14, &share_count_audio);
- clks[IMX6SX_CLK_SPDIF_GCLK] = imx_clk_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_audio);
- clks[IMX6SX_CLK_SSI1_IPG] = imx_clk_gate2_shared("ssi1_ipg", "ipg", base + 0x7c, 18, &share_count_ssi1);
- clks[IMX6SX_CLK_SSI2_IPG] = imx_clk_gate2_shared("ssi2_ipg", "ipg", base + 0x7c, 20, &share_count_ssi2);
- clks[IMX6SX_CLK_SSI3_IPG] = imx_clk_gate2_shared("ssi3_ipg", "ipg", base + 0x7c, 22, &share_count_ssi3);
- clks[IMX6SX_CLK_SSI1] = imx_clk_gate2_shared("ssi1", "ssi1_podf", base + 0x7c, 18, &share_count_ssi1);
- clks[IMX6SX_CLK_SSI2] = imx_clk_gate2_shared("ssi2", "ssi2_podf", base + 0x7c, 20, &share_count_ssi2);
- clks[IMX6SX_CLK_SSI3] = imx_clk_gate2_shared("ssi3", "ssi3_podf", base + 0x7c, 22, &share_count_ssi3);
- clks[IMX6SX_CLK_UART_IPG] = imx_clk_gate2("uart_ipg", "ipg", base + 0x7c, 24);
- clks[IMX6SX_CLK_UART_SERIAL] = imx_clk_gate2("uart_serial", "uart_podf", base + 0x7c, 26);
- clks[IMX6SX_CLK_SAI1_IPG] = imx_clk_gate2_shared("sai1_ipg", "ipg", base + 0x7c, 28, &share_count_sai1);
- clks[IMX6SX_CLK_SAI2_IPG] = imx_clk_gate2_shared("sai2_ipg", "ipg", base + 0x7c, 30, &share_count_sai2);
- clks[IMX6SX_CLK_SAI1] = imx_clk_gate2_shared("sai1", "ssi1_podf", base + 0x7c, 28, &share_count_sai1);
- clks[IMX6SX_CLK_SAI2] = imx_clk_gate2_shared("sai2", "ssi2_podf", base + 0x7c, 30, &share_count_sai2);
+ hws[IMX6SX_CLK_ROM] = imx_clk_hw_gate2_flags("rom", "ahb", base + 0x7c, 0, CLK_IS_CRITICAL);
+ hws[IMX6SX_CLK_SDMA] = imx_clk_hw_gate2("sdma", "ahb", base + 0x7c, 6);
+ hws[IMX6SX_CLK_SPBA] = imx_clk_hw_gate2("spba", "ipg", base + 0x7c, 12);
+ hws[IMX6SX_CLK_AUDIO] = imx_clk_hw_gate2_shared("audio", "audio_podf", base + 0x7c, 14, &share_count_audio);
+ hws[IMX6SX_CLK_SPDIF] = imx_clk_hw_gate2_shared("spdif", "spdif_podf", base + 0x7c, 14, &share_count_audio);
+ hws[IMX6SX_CLK_SPDIF_GCLK] = imx_clk_hw_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_audio);
+ hws[IMX6SX_CLK_SSI1_IPG] = imx_clk_hw_gate2_shared("ssi1_ipg", "ipg", base + 0x7c, 18, &share_count_ssi1);
+ hws[IMX6SX_CLK_SSI2_IPG] = imx_clk_hw_gate2_shared("ssi2_ipg", "ipg", base + 0x7c, 20, &share_count_ssi2);
+ hws[IMX6SX_CLK_SSI3_IPG] = imx_clk_hw_gate2_shared("ssi3_ipg", "ipg", base + 0x7c, 22, &share_count_ssi3);
+ hws[IMX6SX_CLK_SSI1] = imx_clk_hw_gate2_shared("ssi1", "ssi1_podf", base + 0x7c, 18, &share_count_ssi1);
+ hws[IMX6SX_CLK_SSI2] = imx_clk_hw_gate2_shared("ssi2", "ssi2_podf", base + 0x7c, 20, &share_count_ssi2);
+ hws[IMX6SX_CLK_SSI3] = imx_clk_hw_gate2_shared("ssi3", "ssi3_podf", base + 0x7c, 22, &share_count_ssi3);
+ hws[IMX6SX_CLK_UART_IPG] = imx_clk_hw_gate2("uart_ipg", "ipg", base + 0x7c, 24);
+ hws[IMX6SX_CLK_UART_SERIAL] = imx_clk_hw_gate2("uart_serial", "uart_podf", base + 0x7c, 26);
+ hws[IMX6SX_CLK_SAI1_IPG] = imx_clk_hw_gate2_shared("sai1_ipg", "ipg", base + 0x7c, 28, &share_count_sai1);
+ hws[IMX6SX_CLK_SAI2_IPG] = imx_clk_hw_gate2_shared("sai2_ipg", "ipg", base + 0x7c, 30, &share_count_sai2);
+ hws[IMX6SX_CLK_SAI1] = imx_clk_hw_gate2_shared("sai1", "ssi1_podf", base + 0x7c, 28, &share_count_sai1);
+ hws[IMX6SX_CLK_SAI2] = imx_clk_hw_gate2_shared("sai2", "ssi2_podf", base + 0x7c, 30, &share_count_sai2);
/* CCGR6 */
- clks[IMX6SX_CLK_USBOH3] = imx_clk_gate2("usboh3", "ipg", base + 0x80, 0);
- clks[IMX6SX_CLK_USDHC1] = imx_clk_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2);
- clks[IMX6SX_CLK_USDHC2] = imx_clk_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4);
- clks[IMX6SX_CLK_USDHC3] = imx_clk_gate2("usdhc3", "usdhc3_podf", base + 0x80, 6);
- clks[IMX6SX_CLK_USDHC4] = imx_clk_gate2("usdhc4", "usdhc4_podf", base + 0x80, 8);
- clks[IMX6SX_CLK_EIM_SLOW] = imx_clk_gate2("eim_slow", "eim_slow_podf", base + 0x80, 10);
- clks[IMX6SX_CLK_PWM8] = imx_clk_gate2("pwm8", "perclk", base + 0x80, 16);
- clks[IMX6SX_CLK_VADC] = imx_clk_gate2("vadc", "vid_podf", base + 0x80, 20);
- clks[IMX6SX_CLK_GIS] = imx_clk_gate2("gis", "display_podf", base + 0x80, 22);
- clks[IMX6SX_CLK_I2C4] = imx_clk_gate2("i2c4", "perclk", base + 0x80, 24);
- clks[IMX6SX_CLK_PWM5] = imx_clk_gate2("pwm5", "perclk", base + 0x80, 26);
- clks[IMX6SX_CLK_PWM6] = imx_clk_gate2("pwm6", "perclk", base + 0x80, 28);
- clks[IMX6SX_CLK_PWM7] = imx_clk_gate2("pwm7", "perclk", base + 0x80, 30);
-
- clks[IMX6SX_CLK_CKO1] = imx_clk_gate("cko1", "cko1_podf", base + 0x60, 7);
- clks[IMX6SX_CLK_CKO2] = imx_clk_gate("cko2", "cko2_podf", base + 0x60, 24);
+ hws[IMX6SX_CLK_USBOH3] = imx_clk_hw_gate2("usboh3", "ipg", base + 0x80, 0);
+ hws[IMX6SX_CLK_USDHC1] = imx_clk_hw_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2);
+ hws[IMX6SX_CLK_USDHC2] = imx_clk_hw_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4);
+ hws[IMX6SX_CLK_USDHC3] = imx_clk_hw_gate2("usdhc3", "usdhc3_podf", base + 0x80, 6);
+ hws[IMX6SX_CLK_USDHC4] = imx_clk_hw_gate2("usdhc4", "usdhc4_podf", base + 0x80, 8);
+ hws[IMX6SX_CLK_EIM_SLOW] = imx_clk_hw_gate2("eim_slow", "eim_slow_podf", base + 0x80, 10);
+ hws[IMX6SX_CLK_PWM8] = imx_clk_hw_gate2("pwm8", "perclk", base + 0x80, 16);
+ hws[IMX6SX_CLK_VADC] = imx_clk_hw_gate2("vadc", "vid_podf", base + 0x80, 20);
+ hws[IMX6SX_CLK_GIS] = imx_clk_hw_gate2("gis", "display_podf", base + 0x80, 22);
+ hws[IMX6SX_CLK_I2C4] = imx_clk_hw_gate2("i2c4", "perclk", base + 0x80, 24);
+ hws[IMX6SX_CLK_PWM5] = imx_clk_hw_gate2("pwm5", "perclk", base + 0x80, 26);
+ hws[IMX6SX_CLK_PWM6] = imx_clk_hw_gate2("pwm6", "perclk", base + 0x80, 28);
+ hws[IMX6SX_CLK_PWM7] = imx_clk_hw_gate2("pwm7", "perclk", base + 0x80, 30);
+
+ hws[IMX6SX_CLK_CKO1] = imx_clk_hw_gate("cko1", "cko1_podf", base + 0x60, 7);
+ hws[IMX6SX_CLK_CKO2] = imx_clk_hw_gate("cko2", "cko2_podf", base + 0x60, 24);
/* mask handshake of mmdc */
- writel_relaxed(BM_CCM_CCDR_MMDC_CH0_MASK, base + CCDR);
+ imx_mmdc_mask_handshake(base, 0);
- imx_check_clocks(clks, ARRAY_SIZE(clks));
+ imx_check_clk_hws(hws, IMX6SX_CLK_CLK_END);
- clk_data.clks = clks;
- clk_data.clk_num = ARRAY_SIZE(clks);
- of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
+ of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data);
if (IS_ENABLED(CONFIG_USB_MXS_PHY)) {
- clk_prepare_enable(clks[IMX6SX_CLK_USBPHY1_GATE]);
- clk_prepare_enable(clks[IMX6SX_CLK_USBPHY2_GATE]);
+ clk_prepare_enable(hws[IMX6SX_CLK_USBPHY1_GATE]->clk);
+ clk_prepare_enable(hws[IMX6SX_CLK_USBPHY2_GATE]->clk);
}
/* Set the default 132MHz for EIM module */
- clk_set_parent(clks[IMX6SX_CLK_EIM_SLOW_SEL], clks[IMX6SX_CLK_PLL2_PFD2]);
- clk_set_rate(clks[IMX6SX_CLK_EIM_SLOW], 132000000);
+ clk_set_parent(hws[IMX6SX_CLK_EIM_SLOW_SEL]->clk, hws[IMX6SX_CLK_PLL2_PFD2]->clk);
+ clk_set_rate(hws[IMX6SX_CLK_EIM_SLOW]->clk, 132000000);
/* set parent clock for LCDIF1 pixel clock */
- clk_set_parent(clks[IMX6SX_CLK_LCDIF1_PRE_SEL], clks[IMX6SX_CLK_PLL5_VIDEO_DIV]);
- clk_set_parent(clks[IMX6SX_CLK_LCDIF1_SEL], clks[IMX6SX_CLK_LCDIF1_PODF]);
+ clk_set_parent(hws[IMX6SX_CLK_LCDIF1_PRE_SEL]->clk, hws[IMX6SX_CLK_PLL5_VIDEO_DIV]->clk);
+ clk_set_parent(hws[IMX6SX_CLK_LCDIF1_SEL]->clk, hws[IMX6SX_CLK_LCDIF1_PODF]->clk);
/* Set the parent clks of PCIe lvds1 and pcie_axi to be pcie ref, axi */
- if (clk_set_parent(clks[IMX6SX_CLK_LVDS1_SEL], clks[IMX6SX_CLK_PCIE_REF_125M]))
+ if (clk_set_parent(hws[IMX6SX_CLK_LVDS1_SEL]->clk, hws[IMX6SX_CLK_PCIE_REF_125M]->clk))
pr_err("Failed to set pcie bus parent clk.\n");
- if (clk_set_parent(clks[IMX6SX_CLK_PCIE_AXI_SEL], clks[IMX6SX_CLK_AXI]))
- pr_err("Failed to set pcie parent clk.\n");
/*
* Init enet system AHB clock, set to 200MHz
* pll2_pfd2_396m-> ENET_PODF-> ENET_AHB
*/
- clk_set_parent(clks[IMX6SX_CLK_ENET_PRE_SEL], clks[IMX6SX_CLK_PLL2_PFD2]);
- clk_set_parent(clks[IMX6SX_CLK_ENET_SEL], clks[IMX6SX_CLK_ENET_PODF]);
- clk_set_rate(clks[IMX6SX_CLK_ENET_PODF], 200000000);
- clk_set_rate(clks[IMX6SX_CLK_ENET_REF], 125000000);
- clk_set_rate(clks[IMX6SX_CLK_ENET2_REF], 125000000);
+ clk_set_parent(hws[IMX6SX_CLK_ENET_PRE_SEL]->clk, hws[IMX6SX_CLK_PLL2_PFD2]->clk);
+ clk_set_parent(hws[IMX6SX_CLK_ENET_SEL]->clk, hws[IMX6SX_CLK_ENET_PODF]->clk);
+ clk_set_rate(hws[IMX6SX_CLK_ENET_PODF]->clk, 200000000);
+ clk_set_rate(hws[IMX6SX_CLK_ENET_REF]->clk, 125000000);
+ clk_set_rate(hws[IMX6SX_CLK_ENET2_REF]->clk, 125000000);
/* Audio clocks */
- clk_set_rate(clks[IMX6SX_CLK_PLL4_AUDIO_DIV], 393216000);
+ clk_set_rate(hws[IMX6SX_CLK_PLL4_AUDIO_DIV]->clk, 393216000);
- clk_set_parent(clks[IMX6SX_CLK_SPDIF_SEL], clks[IMX6SX_CLK_PLL4_AUDIO_DIV]);
- clk_set_rate(clks[IMX6SX_CLK_SPDIF_PODF], 98304000);
+ clk_set_parent(hws[IMX6SX_CLK_SPDIF_SEL]->clk, hws[IMX6SX_CLK_PLL4_AUDIO_DIV]->clk);
+ clk_set_rate(hws[IMX6SX_CLK_SPDIF_PODF]->clk, 98304000);
- clk_set_parent(clks[IMX6SX_CLK_AUDIO_SEL], clks[IMX6SX_CLK_PLL3_USB_OTG]);
- clk_set_rate(clks[IMX6SX_CLK_AUDIO_PODF], 24000000);
+ clk_set_parent(hws[IMX6SX_CLK_AUDIO_SEL]->clk, hws[IMX6SX_CLK_PLL3_USB_OTG]->clk);
+ clk_set_rate(hws[IMX6SX_CLK_AUDIO_PODF]->clk, 24000000);
- clk_set_parent(clks[IMX6SX_CLK_SSI1_SEL], clks[IMX6SX_CLK_PLL4_AUDIO_DIV]);
- clk_set_parent(clks[IMX6SX_CLK_SSI2_SEL], clks[IMX6SX_CLK_PLL4_AUDIO_DIV]);
- clk_set_parent(clks[IMX6SX_CLK_SSI3_SEL], clks[IMX6SX_CLK_PLL4_AUDIO_DIV]);
- clk_set_rate(clks[IMX6SX_CLK_SSI1_PODF], 24576000);
- clk_set_rate(clks[IMX6SX_CLK_SSI2_PODF], 24576000);
- clk_set_rate(clks[IMX6SX_CLK_SSI3_PODF], 24576000);
+ clk_set_parent(hws[IMX6SX_CLK_SSI1_SEL]->clk, hws[IMX6SX_CLK_PLL4_AUDIO_DIV]->clk);
+ clk_set_parent(hws[IMX6SX_CLK_SSI2_SEL]->clk, hws[IMX6SX_CLK_PLL4_AUDIO_DIV]->clk);
+ clk_set_parent(hws[IMX6SX_CLK_SSI3_SEL]->clk, hws[IMX6SX_CLK_PLL4_AUDIO_DIV]->clk);
+ clk_set_rate(hws[IMX6SX_CLK_SSI1_PODF]->clk, 24576000);
+ clk_set_rate(hws[IMX6SX_CLK_SSI2_PODF]->clk, 24576000);
+ clk_set_rate(hws[IMX6SX_CLK_SSI3_PODF]->clk, 24576000);
- clk_set_parent(clks[IMX6SX_CLK_ESAI_SEL], clks[IMX6SX_CLK_PLL4_AUDIO_DIV]);
- clk_set_rate(clks[IMX6SX_CLK_ESAI_PODF], 24576000);
+ clk_set_parent(hws[IMX6SX_CLK_ESAI_SEL]->clk, hws[IMX6SX_CLK_PLL4_AUDIO_DIV]->clk);
+ clk_set_rate(hws[IMX6SX_CLK_ESAI_PODF]->clk, 24576000);
/* Set parent clock for vadc */
- clk_set_parent(clks[IMX6SX_CLK_VID_SEL], clks[IMX6SX_CLK_PLL3_USB_OTG]);
+ clk_set_parent(hws[IMX6SX_CLK_VID_SEL]->clk, hws[IMX6SX_CLK_PLL3_USB_OTG]->clk);
/* default parent of can_sel clock is invalid, manually set it here */
- clk_set_parent(clks[IMX6SX_CLK_CAN_SEL], clks[IMX6SX_CLK_PLL3_60M]);
+ clk_set_parent(hws[IMX6SX_CLK_CAN_SEL]->clk, hws[IMX6SX_CLK_PLL3_60M]->clk);
/* Update gpu clock from default 528M to 720M */
- clk_set_parent(clks[IMX6SX_CLK_GPU_CORE_SEL], clks[IMX6SX_CLK_PLL3_PFD0]);
- clk_set_parent(clks[IMX6SX_CLK_GPU_AXI_SEL], clks[IMX6SX_CLK_PLL3_PFD0]);
+ clk_set_parent(hws[IMX6SX_CLK_GPU_CORE_SEL]->clk, hws[IMX6SX_CLK_PLL3_PFD0]->clk);
+ clk_set_parent(hws[IMX6SX_CLK_GPU_AXI_SEL]->clk, hws[IMX6SX_CLK_PLL3_PFD0]->clk);
- clk_set_parent(clks[IMX6SX_CLK_QSPI1_SEL], clks[IMX6SX_CLK_PLL2_BUS]);
- clk_set_parent(clks[IMX6SX_CLK_QSPI2_SEL], clks[IMX6SX_CLK_PLL2_BUS]);
+ clk_set_parent(hws[IMX6SX_CLK_QSPI1_SEL]->clk, hws[IMX6SX_CLK_PLL2_BUS]->clk);
+ clk_set_parent(hws[IMX6SX_CLK_QSPI2_SEL]->clk, hws[IMX6SX_CLK_PLL2_BUS]->clk);
+
+ for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) {
+ int index = uart_clk_ids[i];
+
+ uart_clks[i] = &hws[index]->clk;
+ }
imx_register_uart_clocks(uart_clks);
}
diff --git a/drivers/clk/imx/clk-imx6ul.c b/drivers/clk/imx/clk-imx6ul.c
index 8fd52e103cc2..bc931988fe7b 100644
--- a/drivers/clk/imx/clk-imx6ul.c
+++ b/drivers/clk/imx/clk-imx6ul.c
@@ -6,6 +6,7 @@
#include <dt-bindings/clock/imx6ul-clock.h>
#include <linux/clk.h>
#include <linux/clkdev.h>
+#include <linux/clk-provider.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/io.h>
@@ -16,9 +17,6 @@
#include "clk.h"
-#define BM_CCM_CCDR_MMDC_CH0_MASK (0x2 << 16)
-#define CCDR 0x4
-
static const char *pll_bypass_src_sels[] = { "osc", "dummy", };
static const char *pll1_bypass_sels[] = { "pll1", "pll1_bypass_src", };
static const char *pll2_bypass_sels[] = { "pll2", "pll2_bypass_src", };
@@ -70,8 +68,8 @@ static const char *cko2_sels[] = { "dummy", "dummy", "dummy", "usdhc1", "dummy",
"dummy", "dummy", "dummy", "dummy", "uart_serial", "spdif", "dummy", "dummy", };
static const char *cko_sels[] = { "cko1", "cko2", };
-static struct clk *clks[IMX6UL_CLK_END];
-static struct clk_onecell_data clk_data;
+static struct clk_hw **hws;
+static struct clk_hw_onecell_data *clk_hw_data;
static const struct clk_div_table clk_enet_ref_table[] = {
{ .val = 0, .div = 20, },
@@ -118,61 +116,69 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
struct device_node *np;
void __iomem *base;
- clks[IMX6UL_CLK_DUMMY] = imx_clk_fixed("dummy", 0);
+ clk_hw_data = kzalloc(struct_size(clk_hw_data, hws,
+ IMX6UL_CLK_END), GFP_KERNEL);
+ if (WARN_ON(!clk_hw_data))
+ return;
+
+ clk_hw_data->num = IMX6UL_CLK_END;
+ hws = clk_hw_data->hws;
+
+ hws[IMX6UL_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0);
- clks[IMX6UL_CLK_CKIL] = of_clk_get_by_name(ccm_node, "ckil");
- clks[IMX6UL_CLK_OSC] = of_clk_get_by_name(ccm_node, "osc");
+ hws[IMX6UL_CLK_CKIL] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ckil"));
+ hws[IMX6UL_CLK_OSC] = __clk_get_hw(of_clk_get_by_name(ccm_node, "osc"));
/* ipp_di clock is external input */
- clks[IMX6UL_CLK_IPP_DI0] = of_clk_get_by_name(ccm_node, "ipp_di0");
- clks[IMX6UL_CLK_IPP_DI1] = of_clk_get_by_name(ccm_node, "ipp_di1");
+ hws[IMX6UL_CLK_IPP_DI0] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ipp_di0"));
+ hws[IMX6UL_CLK_IPP_DI1] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ipp_di1"));
np = of_find_compatible_node(NULL, NULL, "fsl,imx6ul-anatop");
base = of_iomap(np, 0);
of_node_put(np);
WARN_ON(!base);
- clks[IMX6UL_PLL1_BYPASS_SRC] = imx_clk_mux("pll1_bypass_src", base + 0x00, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6UL_PLL2_BYPASS_SRC] = imx_clk_mux("pll2_bypass_src", base + 0x30, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6UL_PLL3_BYPASS_SRC] = imx_clk_mux("pll3_bypass_src", base + 0x10, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6UL_PLL4_BYPASS_SRC] = imx_clk_mux("pll4_bypass_src", base + 0x70, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6UL_PLL5_BYPASS_SRC] = imx_clk_mux("pll5_bypass_src", base + 0xa0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6UL_PLL6_BYPASS_SRC] = imx_clk_mux("pll6_bypass_src", base + 0xe0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
- clks[IMX6UL_PLL7_BYPASS_SRC] = imx_clk_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
-
- clks[IMX6UL_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f);
- clks[IMX6UL_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1);
- clks[IMX6UL_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3);
- clks[IMX6UL_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f);
- clks[IMX6UL_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f);
- clks[IMX6UL_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3);
- clks[IMX6UL_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3);
-
- clks[IMX6UL_PLL1_BYPASS] = imx_clk_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6UL_PLL2_BYPASS] = imx_clk_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6UL_PLL3_BYPASS] = imx_clk_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6UL_PLL4_BYPASS] = imx_clk_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6UL_PLL5_BYPASS] = imx_clk_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6UL_PLL6_BYPASS] = imx_clk_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6UL_PLL7_BYPASS] = imx_clk_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT);
- clks[IMX6UL_CLK_CSI_SEL] = imx_clk_mux_flags("csi_sel", base + 0x3c, 9, 2, csi_sels, ARRAY_SIZE(csi_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6UL_PLL1_BYPASS_SRC] = imx_clk_hw_mux("pll1_bypass_src", base + 0x00, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6UL_PLL2_BYPASS_SRC] = imx_clk_hw_mux("pll2_bypass_src", base + 0x30, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6UL_PLL3_BYPASS_SRC] = imx_clk_hw_mux("pll3_bypass_src", base + 0x10, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6UL_PLL4_BYPASS_SRC] = imx_clk_hw_mux("pll4_bypass_src", base + 0x70, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6UL_PLL5_BYPASS_SRC] = imx_clk_hw_mux("pll5_bypass_src", base + 0xa0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6UL_PLL6_BYPASS_SRC] = imx_clk_hw_mux("pll6_bypass_src", base + 0xe0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+ hws[IMX6UL_PLL7_BYPASS_SRC] = imx_clk_hw_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels));
+
+ hws[IMX6UL_CLK_PLL1] = imx_clk_hw_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f);
+ hws[IMX6UL_CLK_PLL2] = imx_clk_hw_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1);
+ hws[IMX6UL_CLK_PLL3] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3);
+ hws[IMX6UL_CLK_PLL4] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f);
+ hws[IMX6UL_CLK_PLL5] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f);
+ hws[IMX6UL_CLK_PLL6] = imx_clk_hw_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3);
+ hws[IMX6UL_CLK_PLL7] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3);
+
+ hws[IMX6UL_PLL1_BYPASS] = imx_clk_hw_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6UL_PLL2_BYPASS] = imx_clk_hw_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6UL_PLL3_BYPASS] = imx_clk_hw_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6UL_PLL4_BYPASS] = imx_clk_hw_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6UL_PLL5_BYPASS] = imx_clk_hw_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6UL_PLL6_BYPASS] = imx_clk_hw_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6UL_PLL7_BYPASS] = imx_clk_hw_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6UL_CLK_CSI_SEL] = imx_clk_hw_mux_flags("csi_sel", base + 0x3c, 9, 2, csi_sels, ARRAY_SIZE(csi_sels), CLK_SET_RATE_PARENT);
/* Do not bypass PLLs initially */
- clk_set_parent(clks[IMX6UL_PLL1_BYPASS], clks[IMX6UL_CLK_PLL1]);
- clk_set_parent(clks[IMX6UL_PLL2_BYPASS], clks[IMX6UL_CLK_PLL2]);
- clk_set_parent(clks[IMX6UL_PLL3_BYPASS], clks[IMX6UL_CLK_PLL3]);
- clk_set_parent(clks[IMX6UL_PLL4_BYPASS], clks[IMX6UL_CLK_PLL4]);
- clk_set_parent(clks[IMX6UL_PLL5_BYPASS], clks[IMX6UL_CLK_PLL5]);
- clk_set_parent(clks[IMX6UL_PLL6_BYPASS], clks[IMX6UL_CLK_PLL6]);
- clk_set_parent(clks[IMX6UL_PLL7_BYPASS], clks[IMX6UL_CLK_PLL7]);
-
- clks[IMX6UL_CLK_PLL1_SYS] = imx_clk_fixed_factor("pll1_sys", "pll1_bypass", 1, 1);
- clks[IMX6UL_CLK_PLL2_BUS] = imx_clk_gate("pll2_bus", "pll2_bypass", base + 0x30, 13);
- clks[IMX6UL_CLK_PLL3_USB_OTG] = imx_clk_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13);
- clks[IMX6UL_CLK_PLL4_AUDIO] = imx_clk_gate("pll4_audio", "pll4_bypass", base + 0x70, 13);
- clks[IMX6UL_CLK_PLL5_VIDEO] = imx_clk_gate("pll5_video", "pll5_bypass", base + 0xa0, 13);
- clks[IMX6UL_CLK_PLL6_ENET] = imx_clk_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13);
- clks[IMX6UL_CLK_PLL7_USB_HOST] = imx_clk_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13);
+ clk_set_parent(hws[IMX6UL_PLL1_BYPASS]->clk, hws[IMX6UL_CLK_PLL1]->clk);
+ clk_set_parent(hws[IMX6UL_PLL2_BYPASS]->clk, hws[IMX6UL_CLK_PLL2]->clk);
+ clk_set_parent(hws[IMX6UL_PLL3_BYPASS]->clk, hws[IMX6UL_CLK_PLL3]->clk);
+ clk_set_parent(hws[IMX6UL_PLL4_BYPASS]->clk, hws[IMX6UL_CLK_PLL4]->clk);
+ clk_set_parent(hws[IMX6UL_PLL5_BYPASS]->clk, hws[IMX6UL_CLK_PLL5]->clk);
+ clk_set_parent(hws[IMX6UL_PLL6_BYPASS]->clk, hws[IMX6UL_CLK_PLL6]->clk);
+ clk_set_parent(hws[IMX6UL_PLL7_BYPASS]->clk, hws[IMX6UL_CLK_PLL7]->clk);
+
+ hws[IMX6UL_CLK_PLL1_SYS] = imx_clk_hw_fixed_factor("pll1_sys", "pll1_bypass", 1, 1);
+ hws[IMX6UL_CLK_PLL2_BUS] = imx_clk_hw_gate("pll2_bus", "pll2_bypass", base + 0x30, 13);
+ hws[IMX6UL_CLK_PLL3_USB_OTG] = imx_clk_hw_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13);
+ hws[IMX6UL_CLK_PLL4_AUDIO] = imx_clk_hw_gate("pll4_audio", "pll4_bypass", base + 0x70, 13);
+ hws[IMX6UL_CLK_PLL5_VIDEO] = imx_clk_hw_gate("pll5_video", "pll5_bypass", base + 0xa0, 13);
+ hws[IMX6UL_CLK_PLL6_ENET] = imx_clk_hw_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13);
+ hws[IMX6UL_CLK_PLL7_USB_HOST] = imx_clk_hw_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13);
/*
* Bit 20 is the reserved and read-only bit, we do this only for:
@@ -180,291 +186,289 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
* - Keep refcount when do usbphy clk_enable/disable, in that case,
* the clk framework many need to enable/disable usbphy's parent
*/
- clks[IMX6UL_CLK_USBPHY1] = imx_clk_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20);
- clks[IMX6UL_CLK_USBPHY2] = imx_clk_gate("usbphy2", "pll7_usb_host", base + 0x20, 20);
+ hws[IMX6UL_CLK_USBPHY1] = imx_clk_hw_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20);
+ hws[IMX6UL_CLK_USBPHY2] = imx_clk_hw_gate("usbphy2", "pll7_usb_host", base + 0x20, 20);
/*
* usbphy*_gate needs to be on after system boots up, and software
* never needs to control it anymore.
*/
- clks[IMX6UL_CLK_USBPHY1_GATE] = imx_clk_gate("usbphy1_gate", "dummy", base + 0x10, 6);
- clks[IMX6UL_CLK_USBPHY2_GATE] = imx_clk_gate("usbphy2_gate", "dummy", base + 0x20, 6);
+ hws[IMX6UL_CLK_USBPHY1_GATE] = imx_clk_hw_gate("usbphy1_gate", "dummy", base + 0x10, 6);
+ hws[IMX6UL_CLK_USBPHY2_GATE] = imx_clk_hw_gate("usbphy2_gate", "dummy", base + 0x20, 6);
/* name parent_name reg idx */
- clks[IMX6UL_CLK_PLL2_PFD0] = imx_clk_pfd("pll2_pfd0_352m", "pll2_bus", base + 0x100, 0);
- clks[IMX6UL_CLK_PLL2_PFD1] = imx_clk_pfd("pll2_pfd1_594m", "pll2_bus", base + 0x100, 1);
- clks[IMX6UL_CLK_PLL2_PFD2] = imx_clk_pfd("pll2_pfd2_396m", "pll2_bus", base + 0x100, 2);
- clks[IMX6UL_CLK_PLL2_PFD3] = imx_clk_pfd("pll2_pfd3_594m", "pll2_bus", base + 0x100, 3);
- clks[IMX6UL_CLK_PLL3_PFD0] = imx_clk_pfd("pll3_pfd0_720m", "pll3_usb_otg", base + 0xf0, 0);
- clks[IMX6UL_CLK_PLL3_PFD1] = imx_clk_pfd("pll3_pfd1_540m", "pll3_usb_otg", base + 0xf0, 1);
- clks[IMX6UL_CLK_PLL3_PFD2] = imx_clk_pfd("pll3_pfd2_508m", "pll3_usb_otg", base + 0xf0, 2);
- clks[IMX6UL_CLK_PLL3_PFD3] = imx_clk_pfd("pll3_pfd3_454m", "pll3_usb_otg", base + 0xf0, 3);
-
- clks[IMX6UL_CLK_ENET_REF] = clk_register_divider_table(NULL, "enet_ref", "pll6_enet", 0,
+ hws[IMX6UL_CLK_PLL2_PFD0] = imx_clk_hw_pfd("pll2_pfd0_352m", "pll2_bus", base + 0x100, 0);
+ hws[IMX6UL_CLK_PLL2_PFD1] = imx_clk_hw_pfd("pll2_pfd1_594m", "pll2_bus", base + 0x100, 1);
+ hws[IMX6UL_CLK_PLL2_PFD2] = imx_clk_hw_pfd("pll2_pfd2_396m", "pll2_bus", base + 0x100, 2);
+ hws[IMX6UL_CLK_PLL2_PFD3] = imx_clk_hw_pfd("pll2_pfd3_594m", "pll2_bus", base + 0x100, 3);
+ hws[IMX6UL_CLK_PLL3_PFD0] = imx_clk_hw_pfd("pll3_pfd0_720m", "pll3_usb_otg", base + 0xf0, 0);
+ hws[IMX6UL_CLK_PLL3_PFD1] = imx_clk_hw_pfd("pll3_pfd1_540m", "pll3_usb_otg", base + 0xf0, 1);
+ hws[IMX6UL_CLK_PLL3_PFD2] = imx_clk_hw_pfd("pll3_pfd2_508m", "pll3_usb_otg", base + 0xf0, 2);
+ hws[IMX6UL_CLK_PLL3_PFD3] = imx_clk_hw_pfd("pll3_pfd3_454m", "pll3_usb_otg", base + 0xf0, 3);
+
+ hws[IMX6UL_CLK_ENET_REF] = clk_hw_register_divider_table(NULL, "enet_ref", "pll6_enet", 0,
base + 0xe0, 0, 2, 0, clk_enet_ref_table, &imx_ccm_lock);
- clks[IMX6UL_CLK_ENET2_REF] = clk_register_divider_table(NULL, "enet2_ref", "pll6_enet", 0,
+ hws[IMX6UL_CLK_ENET2_REF] = clk_hw_register_divider_table(NULL, "enet2_ref", "pll6_enet", 0,
base + 0xe0, 2, 2, 0, clk_enet_ref_table, &imx_ccm_lock);
- clks[IMX6UL_CLK_ENET2_REF_125M] = imx_clk_gate("enet_ref_125m", "enet2_ref", base + 0xe0, 20);
- clks[IMX6UL_CLK_ENET_PTP_REF] = imx_clk_fixed_factor("enet_ptp_ref", "pll6_enet", 1, 20);
- clks[IMX6UL_CLK_ENET_PTP] = imx_clk_gate("enet_ptp", "enet_ptp_ref", base + 0xe0, 21);
+ hws[IMX6UL_CLK_ENET2_REF_125M] = imx_clk_hw_gate("enet_ref_125m", "enet2_ref", base + 0xe0, 20);
+ hws[IMX6UL_CLK_ENET_PTP_REF] = imx_clk_hw_fixed_factor("enet_ptp_ref", "pll6_enet", 1, 20);
+ hws[IMX6UL_CLK_ENET_PTP] = imx_clk_hw_gate("enet_ptp", "enet_ptp_ref", base + 0xe0, 21);
- clks[IMX6UL_CLK_PLL4_POST_DIV] = clk_register_divider_table(NULL, "pll4_post_div", "pll4_audio",
+ hws[IMX6UL_CLK_PLL4_POST_DIV] = clk_hw_register_divider_table(NULL, "pll4_post_div", "pll4_audio",
CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x70, 19, 2, 0, post_div_table, &imx_ccm_lock);
- clks[IMX6UL_CLK_PLL4_AUDIO_DIV] = clk_register_divider(NULL, "pll4_audio_div", "pll4_post_div",
+ hws[IMX6UL_CLK_PLL4_AUDIO_DIV] = clk_hw_register_divider(NULL, "pll4_audio_div", "pll4_post_div",
CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x170, 15, 1, 0, &imx_ccm_lock);
- clks[IMX6UL_CLK_PLL5_POST_DIV] = clk_register_divider_table(NULL, "pll5_post_div", "pll5_video",
+ hws[IMX6UL_CLK_PLL5_POST_DIV] = clk_hw_register_divider_table(NULL, "pll5_post_div", "pll5_video",
CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0xa0, 19, 2, 0, post_div_table, &imx_ccm_lock);
- clks[IMX6UL_CLK_PLL5_VIDEO_DIV] = clk_register_divider_table(NULL, "pll5_video_div", "pll5_post_div",
+ hws[IMX6UL_CLK_PLL5_VIDEO_DIV] = clk_hw_register_divider_table(NULL, "pll5_video_div", "pll5_post_div",
CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x170, 30, 2, 0, video_div_table, &imx_ccm_lock);
/* name parent_name mult div */
- clks[IMX6UL_CLK_PLL2_198M] = imx_clk_fixed_factor("pll2_198m", "pll2_pfd2_396m", 1, 2);
- clks[IMX6UL_CLK_PLL3_80M] = imx_clk_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6);
- clks[IMX6UL_CLK_PLL3_60M] = imx_clk_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8);
- clks[IMX6UL_CLK_GPT_3M] = imx_clk_fixed_factor("gpt_3m", "osc", 1, 8);
+ hws[IMX6UL_CLK_PLL2_198M] = imx_clk_hw_fixed_factor("pll2_198m", "pll2_pfd2_396m", 1, 2);
+ hws[IMX6UL_CLK_PLL3_80M] = imx_clk_hw_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6);
+ hws[IMX6UL_CLK_PLL3_60M] = imx_clk_hw_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8);
+ hws[IMX6UL_CLK_GPT_3M] = imx_clk_hw_fixed_factor("gpt_3m", "osc", 1, 8);
np = ccm_node;
base = of_iomap(np, 0);
WARN_ON(!base);
- clks[IMX6UL_CA7_SECONDARY_SEL] = imx_clk_mux("ca7_secondary_sel", base + 0xc, 3, 1, ca7_secondary_sels, ARRAY_SIZE(ca7_secondary_sels));
- clks[IMX6UL_CLK_STEP] = imx_clk_mux("step", base + 0x0c, 8, 1, step_sels, ARRAY_SIZE(step_sels));
- clks[IMX6UL_CLK_PLL1_SW] = imx_clk_mux_flags("pll1_sw", base + 0x0c, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels), 0);
- clks[IMX6UL_CLK_AXI_ALT_SEL] = imx_clk_mux("axi_alt_sel", base + 0x14, 7, 1, axi_alt_sels, ARRAY_SIZE(axi_alt_sels));
- clks[IMX6UL_CLK_AXI_SEL] = imx_clk_mux_flags("axi_sel", base + 0x14, 6, 1, axi_sels, ARRAY_SIZE(axi_sels), 0);
- clks[IMX6UL_CLK_PERIPH_PRE] = imx_clk_mux("periph_pre", base + 0x18, 18, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels));
- clks[IMX6UL_CLK_PERIPH2_PRE] = imx_clk_mux("periph2_pre", base + 0x18, 21, 2, periph2_pre_sels, ARRAY_SIZE(periph2_pre_sels));
- clks[IMX6UL_CLK_PERIPH_CLK2_SEL] = imx_clk_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels));
- clks[IMX6UL_CLK_PERIPH2_CLK2_SEL] = imx_clk_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels));
- clks[IMX6UL_CLK_EIM_SLOW_SEL] = imx_clk_mux("eim_slow_sel", base + 0x1c, 29, 2, eim_slow_sels, ARRAY_SIZE(eim_slow_sels));
- clks[IMX6UL_CLK_GPMI_SEL] = imx_clk_mux("gpmi_sel", base + 0x1c, 19, 1, gpmi_sels, ARRAY_SIZE(gpmi_sels));
- clks[IMX6UL_CLK_BCH_SEL] = imx_clk_mux("bch_sel", base + 0x1c, 18, 1, bch_sels, ARRAY_SIZE(bch_sels));
- clks[IMX6UL_CLK_USDHC2_SEL] = imx_clk_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
- clks[IMX6UL_CLK_USDHC1_SEL] = imx_clk_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
- clks[IMX6UL_CLK_SAI3_SEL] = imx_clk_mux("sai3_sel", base + 0x1c, 14, 2, sai_sels, ARRAY_SIZE(sai_sels));
- clks[IMX6UL_CLK_SAI2_SEL] = imx_clk_mux("sai2_sel", base + 0x1c, 12, 2, sai_sels, ARRAY_SIZE(sai_sels));
- clks[IMX6UL_CLK_SAI1_SEL] = imx_clk_mux("sai1_sel", base + 0x1c, 10, 2, sai_sels, ARRAY_SIZE(sai_sels));
- clks[IMX6UL_CLK_QSPI1_SEL] = imx_clk_mux("qspi1_sel", base + 0x1c, 7, 3, qspi1_sels, ARRAY_SIZE(qspi1_sels));
- clks[IMX6UL_CLK_PERCLK_SEL] = imx_clk_mux("perclk_sel", base + 0x1c, 6, 1, perclk_sels, ARRAY_SIZE(perclk_sels));
- clks[IMX6UL_CLK_CAN_SEL] = imx_clk_mux("can_sel", base + 0x20, 8, 2, can_sels, ARRAY_SIZE(can_sels));
+ hws[IMX6UL_CA7_SECONDARY_SEL] = imx_clk_hw_mux("ca7_secondary_sel", base + 0xc, 3, 1, ca7_secondary_sels, ARRAY_SIZE(ca7_secondary_sels));
+ hws[IMX6UL_CLK_STEP] = imx_clk_hw_mux("step", base + 0x0c, 8, 1, step_sels, ARRAY_SIZE(step_sels));
+ hws[IMX6UL_CLK_PLL1_SW] = imx_clk_hw_mux_flags("pll1_sw", base + 0x0c, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels), 0);
+ hws[IMX6UL_CLK_AXI_ALT_SEL] = imx_clk_hw_mux("axi_alt_sel", base + 0x14, 7, 1, axi_alt_sels, ARRAY_SIZE(axi_alt_sels));
+ hws[IMX6UL_CLK_AXI_SEL] = imx_clk_hw_mux_flags("axi_sel", base + 0x14, 6, 1, axi_sels, ARRAY_SIZE(axi_sels), 0);
+ hws[IMX6UL_CLK_PERIPH_PRE] = imx_clk_hw_mux("periph_pre", base + 0x18, 18, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels));
+ hws[IMX6UL_CLK_PERIPH2_PRE] = imx_clk_hw_mux("periph2_pre", base + 0x18, 21, 2, periph2_pre_sels, ARRAY_SIZE(periph2_pre_sels));
+ hws[IMX6UL_CLK_PERIPH_CLK2_SEL] = imx_clk_hw_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels));
+ hws[IMX6UL_CLK_PERIPH2_CLK2_SEL] = imx_clk_hw_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels));
+ hws[IMX6UL_CLK_EIM_SLOW_SEL] = imx_clk_hw_mux("eim_slow_sel", base + 0x1c, 29, 2, eim_slow_sels, ARRAY_SIZE(eim_slow_sels));
+ hws[IMX6UL_CLK_GPMI_SEL] = imx_clk_hw_mux("gpmi_sel", base + 0x1c, 19, 1, gpmi_sels, ARRAY_SIZE(gpmi_sels));
+ hws[IMX6UL_CLK_BCH_SEL] = imx_clk_hw_mux("bch_sel", base + 0x1c, 18, 1, bch_sels, ARRAY_SIZE(bch_sels));
+ hws[IMX6UL_CLK_USDHC2_SEL] = imx_clk_hw_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
+ hws[IMX6UL_CLK_USDHC1_SEL] = imx_clk_hw_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels));
+ hws[IMX6UL_CLK_SAI3_SEL] = imx_clk_hw_mux("sai3_sel", base + 0x1c, 14, 2, sai_sels, ARRAY_SIZE(sai_sels));
+ hws[IMX6UL_CLK_SAI2_SEL] = imx_clk_hw_mux("sai2_sel", base + 0x1c, 12, 2, sai_sels, ARRAY_SIZE(sai_sels));
+ hws[IMX6UL_CLK_SAI1_SEL] = imx_clk_hw_mux("sai1_sel", base + 0x1c, 10, 2, sai_sels, ARRAY_SIZE(sai_sels));
+ hws[IMX6UL_CLK_QSPI1_SEL] = imx_clk_hw_mux("qspi1_sel", base + 0x1c, 7, 3, qspi1_sels, ARRAY_SIZE(qspi1_sels));
+ hws[IMX6UL_CLK_PERCLK_SEL] = imx_clk_hw_mux("perclk_sel", base + 0x1c, 6, 1, perclk_sels, ARRAY_SIZE(perclk_sels));
+ hws[IMX6UL_CLK_CAN_SEL] = imx_clk_hw_mux("can_sel", base + 0x20, 8, 2, can_sels, ARRAY_SIZE(can_sels));
if (clk_on_imx6ull())
- clks[IMX6ULL_CLK_ESAI_SEL] = imx_clk_mux("esai_sel", base + 0x20, 19, 2, esai_sels, ARRAY_SIZE(esai_sels));
- clks[IMX6UL_CLK_UART_SEL] = imx_clk_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels));
- clks[IMX6UL_CLK_ENFC_SEL] = imx_clk_mux("enfc_sel", base + 0x2c, 15, 3, enfc_sels, ARRAY_SIZE(enfc_sels));
- clks[IMX6UL_CLK_LDB_DI0_SEL] = imx_clk_mux("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di0_sels, ARRAY_SIZE(ldb_di0_sels));
- clks[IMX6UL_CLK_SPDIF_SEL] = imx_clk_mux("spdif_sel", base + 0x30, 20, 2, spdif_sels, ARRAY_SIZE(spdif_sels));
+ hws[IMX6ULL_CLK_ESAI_SEL] = imx_clk_hw_mux("esai_sel", base + 0x20, 19, 2, esai_sels, ARRAY_SIZE(esai_sels));
+ hws[IMX6UL_CLK_UART_SEL] = imx_clk_hw_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels));
+ hws[IMX6UL_CLK_ENFC_SEL] = imx_clk_hw_mux("enfc_sel", base + 0x2c, 15, 3, enfc_sels, ARRAY_SIZE(enfc_sels));
+ hws[IMX6UL_CLK_LDB_DI0_SEL] = imx_clk_hw_mux("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di0_sels, ARRAY_SIZE(ldb_di0_sels));
+ hws[IMX6UL_CLK_SPDIF_SEL] = imx_clk_hw_mux("spdif_sel", base + 0x30, 20, 2, spdif_sels, ARRAY_SIZE(spdif_sels));
if (clk_on_imx6ul()) {
- clks[IMX6UL_CLK_SIM_PRE_SEL] = imx_clk_mux("sim_pre_sel", base + 0x34, 15, 3, sim_pre_sels, ARRAY_SIZE(sim_pre_sels));
- clks[IMX6UL_CLK_SIM_SEL] = imx_clk_mux("sim_sel", base + 0x34, 9, 3, sim_sels, ARRAY_SIZE(sim_sels));
+ hws[IMX6UL_CLK_SIM_PRE_SEL] = imx_clk_hw_mux("sim_pre_sel", base + 0x34, 15, 3, sim_pre_sels, ARRAY_SIZE(sim_pre_sels));
+ hws[IMX6UL_CLK_SIM_SEL] = imx_clk_hw_mux("sim_sel", base + 0x34, 9, 3, sim_sels, ARRAY_SIZE(sim_sels));
} else if (clk_on_imx6ull()) {
- clks[IMX6ULL_CLK_EPDC_PRE_SEL] = imx_clk_mux("epdc_pre_sel", base + 0x34, 15, 3, epdc_pre_sels, ARRAY_SIZE(epdc_pre_sels));
- clks[IMX6ULL_CLK_EPDC_SEL] = imx_clk_mux("epdc_sel", base + 0x34, 9, 3, epdc_sels, ARRAY_SIZE(epdc_sels));
+ hws[IMX6ULL_CLK_EPDC_PRE_SEL] = imx_clk_hw_mux("epdc_pre_sel", base + 0x34, 15, 3, epdc_pre_sels, ARRAY_SIZE(epdc_pre_sels));
+ hws[IMX6ULL_CLK_EPDC_SEL] = imx_clk_hw_mux("epdc_sel", base + 0x34, 9, 3, epdc_sels, ARRAY_SIZE(epdc_sels));
}
- clks[IMX6UL_CLK_ECSPI_SEL] = imx_clk_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels));
- clks[IMX6UL_CLK_LCDIF_PRE_SEL] = imx_clk_mux_flags("lcdif_pre_sel", base + 0x38, 15, 3, lcdif_pre_sels, ARRAY_SIZE(lcdif_pre_sels), CLK_SET_RATE_PARENT);
- clks[IMX6UL_CLK_LCDIF_SEL] = imx_clk_mux("lcdif_sel", base + 0x38, 9, 3, lcdif_sels, ARRAY_SIZE(lcdif_sels));
-
- clks[IMX6UL_CLK_LDB_DI0_DIV_SEL] = imx_clk_mux("ldb_di0", base + 0x20, 10, 1, ldb_di0_div_sels, ARRAY_SIZE(ldb_di0_div_sels));
- clks[IMX6UL_CLK_LDB_DI1_DIV_SEL] = imx_clk_mux("ldb_di1", base + 0x20, 11, 1, ldb_di1_div_sels, ARRAY_SIZE(ldb_di1_div_sels));
-
- clks[IMX6UL_CLK_CKO1_SEL] = imx_clk_mux("cko1_sel", base + 0x60, 0, 4, cko1_sels, ARRAY_SIZE(cko1_sels));
- clks[IMX6UL_CLK_CKO2_SEL] = imx_clk_mux("cko2_sel", base + 0x60, 16, 5, cko2_sels, ARRAY_SIZE(cko2_sels));
- clks[IMX6UL_CLK_CKO] = imx_clk_mux("cko", base + 0x60, 8, 1, cko_sels, ARRAY_SIZE(cko_sels));
-
- clks[IMX6UL_CLK_LDB_DI0_DIV_3_5] = imx_clk_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7);
- clks[IMX6UL_CLK_LDB_DI0_DIV_7] = imx_clk_fixed_factor("ldb_di0_div_7", "ldb_di0_sel", 1, 7);
- clks[IMX6UL_CLK_LDB_DI1_DIV_3_5] = imx_clk_fixed_factor("ldb_di1_div_3_5", "qspi1_sel", 2, 7);
- clks[IMX6UL_CLK_LDB_DI1_DIV_7] = imx_clk_fixed_factor("ldb_di1_div_7", "qspi1_sel", 1, 7);
-
- clks[IMX6UL_CLK_PERIPH] = imx_clk_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels));
- clks[IMX6UL_CLK_PERIPH2] = imx_clk_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels));
-
- clks[IMX6UL_CLK_PERIPH_CLK2] = imx_clk_divider("periph_clk2", "periph_clk2_sel", base + 0x14, 27, 3);
- clks[IMX6UL_CLK_PERIPH2_CLK2] = imx_clk_divider("periph2_clk2", "periph2_clk2_sel", base + 0x14, 0, 3);
- clks[IMX6UL_CLK_IPG] = imx_clk_divider("ipg", "ahb", base + 0x14, 8, 2);
- clks[IMX6UL_CLK_LCDIF_PODF] = imx_clk_divider("lcdif_podf", "lcdif_pred", base + 0x18, 23, 3);
- clks[IMX6UL_CLK_QSPI1_PDOF] = imx_clk_divider("qspi1_podf", "qspi1_sel", base + 0x1c, 26, 3);
- clks[IMX6UL_CLK_EIM_SLOW_PODF] = imx_clk_divider("eim_slow_podf", "eim_slow_sel", base + 0x1c, 23, 3);
- clks[IMX6UL_CLK_PERCLK] = imx_clk_divider("perclk", "perclk_sel", base + 0x1c, 0, 6);
- clks[IMX6UL_CLK_CAN_PODF] = imx_clk_divider("can_podf", "can_sel", base + 0x20, 2, 6);
- clks[IMX6UL_CLK_GPMI_PODF] = imx_clk_divider("gpmi_podf", "gpmi_sel", base + 0x24, 22, 3);
- clks[IMX6UL_CLK_BCH_PODF] = imx_clk_divider("bch_podf", "bch_sel", base + 0x24, 19, 3);
- clks[IMX6UL_CLK_USDHC2_PODF] = imx_clk_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3);
- clks[IMX6UL_CLK_USDHC1_PODF] = imx_clk_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3);
- clks[IMX6UL_CLK_UART_PODF] = imx_clk_divider("uart_podf", "uart_sel", base + 0x24, 0, 6);
- clks[IMX6UL_CLK_SAI3_PRED] = imx_clk_divider("sai3_pred", "sai3_sel", base + 0x28, 22, 3);
- clks[IMX6UL_CLK_SAI3_PODF] = imx_clk_divider("sai3_podf", "sai3_pred", base + 0x28, 16, 6);
- clks[IMX6UL_CLK_SAI1_PRED] = imx_clk_divider("sai1_pred", "sai1_sel", base + 0x28, 6, 3);
- clks[IMX6UL_CLK_SAI1_PODF] = imx_clk_divider("sai1_podf", "sai1_pred", base + 0x28, 0, 6);
+ hws[IMX6UL_CLK_ECSPI_SEL] = imx_clk_hw_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels));
+ hws[IMX6UL_CLK_LCDIF_PRE_SEL] = imx_clk_hw_mux_flags("lcdif_pre_sel", base + 0x38, 15, 3, lcdif_pre_sels, ARRAY_SIZE(lcdif_pre_sels), CLK_SET_RATE_PARENT);
+ hws[IMX6UL_CLK_LCDIF_SEL] = imx_clk_hw_mux("lcdif_sel", base + 0x38, 9, 3, lcdif_sels, ARRAY_SIZE(lcdif_sels));
+
+ hws[IMX6UL_CLK_LDB_DI0_DIV_SEL] = imx_clk_hw_mux("ldb_di0", base + 0x20, 10, 1, ldb_di0_div_sels, ARRAY_SIZE(ldb_di0_div_sels));
+ hws[IMX6UL_CLK_LDB_DI1_DIV_SEL] = imx_clk_hw_mux("ldb_di1", base + 0x20, 11, 1, ldb_di1_div_sels, ARRAY_SIZE(ldb_di1_div_sels));
+
+ hws[IMX6UL_CLK_CKO1_SEL] = imx_clk_hw_mux("cko1_sel", base + 0x60, 0, 4, cko1_sels, ARRAY_SIZE(cko1_sels));
+ hws[IMX6UL_CLK_CKO2_SEL] = imx_clk_hw_mux("cko2_sel", base + 0x60, 16, 5, cko2_sels, ARRAY_SIZE(cko2_sels));
+ hws[IMX6UL_CLK_CKO] = imx_clk_hw_mux("cko", base + 0x60, 8, 1, cko_sels, ARRAY_SIZE(cko_sels));
+
+ hws[IMX6UL_CLK_LDB_DI0_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7);
+ hws[IMX6UL_CLK_LDB_DI0_DIV_7] = imx_clk_hw_fixed_factor("ldb_di0_div_7", "ldb_di0_sel", 1, 7);
+ hws[IMX6UL_CLK_LDB_DI1_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di1_div_3_5", "qspi1_sel", 2, 7);
+ hws[IMX6UL_CLK_LDB_DI1_DIV_7] = imx_clk_hw_fixed_factor("ldb_di1_div_7", "qspi1_sel", 1, 7);
+
+ hws[IMX6UL_CLK_PERIPH] = imx_clk_hw_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels));
+ hws[IMX6UL_CLK_PERIPH2] = imx_clk_hw_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels));
+
+ hws[IMX6UL_CLK_PERIPH_CLK2] = imx_clk_hw_divider("periph_clk2", "periph_clk2_sel", base + 0x14, 27, 3);
+ hws[IMX6UL_CLK_PERIPH2_CLK2] = imx_clk_hw_divider("periph2_clk2", "periph2_clk2_sel", base + 0x14, 0, 3);
+ hws[IMX6UL_CLK_IPG] = imx_clk_hw_divider("ipg", "ahb", base + 0x14, 8, 2);
+ hws[IMX6UL_CLK_LCDIF_PODF] = imx_clk_hw_divider("lcdif_podf", "lcdif_pred", base + 0x18, 23, 3);
+ hws[IMX6UL_CLK_QSPI1_PDOF] = imx_clk_hw_divider("qspi1_podf", "qspi1_sel", base + 0x1c, 26, 3);
+ hws[IMX6UL_CLK_EIM_SLOW_PODF] = imx_clk_hw_divider("eim_slow_podf", "eim_slow_sel", base + 0x1c, 23, 3);
+ hws[IMX6UL_CLK_PERCLK] = imx_clk_hw_divider("perclk", "perclk_sel", base + 0x1c, 0, 6);
+ hws[IMX6UL_CLK_CAN_PODF] = imx_clk_hw_divider("can_podf", "can_sel", base + 0x20, 2, 6);
+ hws[IMX6UL_CLK_GPMI_PODF] = imx_clk_hw_divider("gpmi_podf", "gpmi_sel", base + 0x24, 22, 3);
+ hws[IMX6UL_CLK_BCH_PODF] = imx_clk_hw_divider("bch_podf", "bch_sel", base + 0x24, 19, 3);
+ hws[IMX6UL_CLK_USDHC2_PODF] = imx_clk_hw_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3);
+ hws[IMX6UL_CLK_USDHC1_PODF] = imx_clk_hw_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3);
+ hws[IMX6UL_CLK_UART_PODF] = imx_clk_hw_divider("uart_podf", "uart_sel", base + 0x24, 0, 6);
+ hws[IMX6UL_CLK_SAI3_PRED] = imx_clk_hw_divider("sai3_pred", "sai3_sel", base + 0x28, 22, 3);
+ hws[IMX6UL_CLK_SAI3_PODF] = imx_clk_hw_divider("sai3_podf", "sai3_pred", base + 0x28, 16, 6);
+ hws[IMX6UL_CLK_SAI1_PRED] = imx_clk_hw_divider("sai1_pred", "sai1_sel", base + 0x28, 6, 3);
+ hws[IMX6UL_CLK_SAI1_PODF] = imx_clk_hw_divider("sai1_podf", "sai1_pred", base + 0x28, 0, 6);
if (clk_on_imx6ull()) {
- clks[IMX6ULL_CLK_ESAI_PRED] = imx_clk_divider("esai_pred", "esai_sel", base + 0x28, 9, 3);
- clks[IMX6ULL_CLK_ESAI_PODF] = imx_clk_divider("esai_podf", "esai_pred", base + 0x28, 25, 3);
+ hws[IMX6ULL_CLK_ESAI_PRED] = imx_clk_hw_divider("esai_pred", "esai_sel", base + 0x28, 9, 3);
+ hws[IMX6ULL_CLK_ESAI_PODF] = imx_clk_hw_divider("esai_podf", "esai_pred", base + 0x28, 25, 3);
}
- clks[IMX6UL_CLK_ENFC_PRED] = imx_clk_divider("enfc_pred", "enfc_sel", base + 0x2c, 18, 3);
- clks[IMX6UL_CLK_ENFC_PODF] = imx_clk_divider("enfc_podf", "enfc_pred", base + 0x2c, 21, 6);
- clks[IMX6UL_CLK_SAI2_PRED] = imx_clk_divider("sai2_pred", "sai2_sel", base + 0x2c, 6, 3);
- clks[IMX6UL_CLK_SAI2_PODF] = imx_clk_divider("sai2_podf", "sai2_pred", base + 0x2c, 0, 6);
- clks[IMX6UL_CLK_SPDIF_PRED] = imx_clk_divider("spdif_pred", "spdif_sel", base + 0x30, 25, 3);
- clks[IMX6UL_CLK_SPDIF_PODF] = imx_clk_divider("spdif_podf", "spdif_pred", base + 0x30, 22, 3);
+ hws[IMX6UL_CLK_ENFC_PRED] = imx_clk_hw_divider("enfc_pred", "enfc_sel", base + 0x2c, 18, 3);
+ hws[IMX6UL_CLK_ENFC_PODF] = imx_clk_hw_divider("enfc_podf", "enfc_pred", base + 0x2c, 21, 6);
+ hws[IMX6UL_CLK_SAI2_PRED] = imx_clk_hw_divider("sai2_pred", "sai2_sel", base + 0x2c, 6, 3);
+ hws[IMX6UL_CLK_SAI2_PODF] = imx_clk_hw_divider("sai2_podf", "sai2_pred", base + 0x2c, 0, 6);
+ hws[IMX6UL_CLK_SPDIF_PRED] = imx_clk_hw_divider("spdif_pred", "spdif_sel", base + 0x30, 25, 3);
+ hws[IMX6UL_CLK_SPDIF_PODF] = imx_clk_hw_divider("spdif_podf", "spdif_pred", base + 0x30, 22, 3);
if (clk_on_imx6ul())
- clks[IMX6UL_CLK_SIM_PODF] = imx_clk_divider("sim_podf", "sim_pre_sel", base + 0x34, 12, 3);
+ hws[IMX6UL_CLK_SIM_PODF] = imx_clk_hw_divider("sim_podf", "sim_pre_sel", base + 0x34, 12, 3);
else if (clk_on_imx6ull())
- clks[IMX6ULL_CLK_EPDC_PODF] = imx_clk_divider("epdc_podf", "epdc_pre_sel", base + 0x34, 12, 3);
- clks[IMX6UL_CLK_ECSPI_PODF] = imx_clk_divider("ecspi_podf", "ecspi_sel", base + 0x38, 19, 6);
- clks[IMX6UL_CLK_LCDIF_PRED] = imx_clk_divider("lcdif_pred", "lcdif_pre_sel", base + 0x38, 12, 3);
- clks[IMX6UL_CLK_CSI_PODF] = imx_clk_divider("csi_podf", "csi_sel", base + 0x3c, 11, 3);
+ hws[IMX6ULL_CLK_EPDC_PODF] = imx_clk_hw_divider("epdc_podf", "epdc_pre_sel", base + 0x34, 12, 3);
+ hws[IMX6UL_CLK_ECSPI_PODF] = imx_clk_hw_divider("ecspi_podf", "ecspi_sel", base + 0x38, 19, 6);
+ hws[IMX6UL_CLK_LCDIF_PRED] = imx_clk_hw_divider("lcdif_pred", "lcdif_pre_sel", base + 0x38, 12, 3);
+ hws[IMX6UL_CLK_CSI_PODF] = imx_clk_hw_divider("csi_podf", "csi_sel", base + 0x3c, 11, 3);
- clks[IMX6UL_CLK_CKO1_PODF] = imx_clk_divider("cko1_podf", "cko1_sel", base + 0x60, 4, 3);
- clks[IMX6UL_CLK_CKO2_PODF] = imx_clk_divider("cko2_podf", "cko2_sel", base + 0x60, 21, 3);
+ hws[IMX6UL_CLK_CKO1_PODF] = imx_clk_hw_divider("cko1_podf", "cko1_sel", base + 0x60, 4, 3);
+ hws[IMX6UL_CLK_CKO2_PODF] = imx_clk_hw_divider("cko2_podf", "cko2_sel", base + 0x60, 21, 3);
- clks[IMX6UL_CLK_ARM] = imx_clk_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16);
- clks[IMX6UL_CLK_MMDC_PODF] = imx_clk_busy_divider("mmdc_podf", "periph2", base + 0x14, 3, 3, base + 0x48, 2);
- clks[IMX6UL_CLK_AXI_PODF] = imx_clk_busy_divider("axi_podf", "axi_sel", base + 0x14, 16, 3, base + 0x48, 0);
- clks[IMX6UL_CLK_AHB] = imx_clk_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1);
+ hws[IMX6UL_CLK_ARM] = imx_clk_hw_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16);
+ hws[IMX6UL_CLK_MMDC_PODF] = imx_clk_hw_busy_divider("mmdc_podf", "periph2", base + 0x14, 3, 3, base + 0x48, 2);
+ hws[IMX6UL_CLK_AXI_PODF] = imx_clk_hw_busy_divider("axi_podf", "axi_sel", base + 0x14, 16, 3, base + 0x48, 0);
+ hws[IMX6UL_CLK_AHB] = imx_clk_hw_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1);
/* CCGR0 */
- clks[IMX6UL_CLK_AIPSTZ1] = imx_clk_gate2_flags("aips_tz1", "ahb", base + 0x68, 0, CLK_IS_CRITICAL);
- clks[IMX6UL_CLK_AIPSTZ2] = imx_clk_gate2_flags("aips_tz2", "ahb", base + 0x68, 2, CLK_IS_CRITICAL);
- clks[IMX6UL_CLK_APBHDMA] = imx_clk_gate2("apbh_dma", "bch_podf", base + 0x68, 4);
- clks[IMX6UL_CLK_ASRC_IPG] = imx_clk_gate2_shared("asrc_ipg", "ahb", base + 0x68, 6, &share_count_asrc);
- clks[IMX6UL_CLK_ASRC_MEM] = imx_clk_gate2_shared("asrc_mem", "ahb", base + 0x68, 6, &share_count_asrc);
+ hws[IMX6UL_CLK_AIPSTZ1] = imx_clk_hw_gate2_flags("aips_tz1", "ahb", base + 0x68, 0, CLK_IS_CRITICAL);
+ hws[IMX6UL_CLK_AIPSTZ2] = imx_clk_hw_gate2_flags("aips_tz2", "ahb", base + 0x68, 2, CLK_IS_CRITICAL);
+ hws[IMX6UL_CLK_APBHDMA] = imx_clk_hw_gate2("apbh_dma", "bch_podf", base + 0x68, 4);
+ hws[IMX6UL_CLK_ASRC_IPG] = imx_clk_hw_gate2_shared("asrc_ipg", "ahb", base + 0x68, 6, &share_count_asrc);
+ hws[IMX6UL_CLK_ASRC_MEM] = imx_clk_hw_gate2_shared("asrc_mem", "ahb", base + 0x68, 6, &share_count_asrc);
if (clk_on_imx6ul()) {
- clks[IMX6UL_CLK_CAAM_MEM] = imx_clk_gate2("caam_mem", "ahb", base + 0x68, 8);
- clks[IMX6UL_CLK_CAAM_ACLK] = imx_clk_gate2("caam_aclk", "ahb", base + 0x68, 10);
- clks[IMX6UL_CLK_CAAM_IPG] = imx_clk_gate2("caam_ipg", "ipg", base + 0x68, 12);
+ hws[IMX6UL_CLK_CAAM_MEM] = imx_clk_hw_gate2("caam_mem", "ahb", base + 0x68, 8);
+ hws[IMX6UL_CLK_CAAM_ACLK] = imx_clk_hw_gate2("caam_aclk", "ahb", base + 0x68, 10);
+ hws[IMX6UL_CLK_CAAM_IPG] = imx_clk_hw_gate2("caam_ipg", "ipg", base + 0x68, 12);
} else if (clk_on_imx6ull()) {
- clks[IMX6ULL_CLK_DCP_CLK] = imx_clk_gate2("dcp", "ahb", base + 0x68, 10);
- clks[IMX6UL_CLK_ENET] = imx_clk_gate2("enet", "ipg", base + 0x68, 12);
- clks[IMX6UL_CLK_ENET_AHB] = imx_clk_gate2("enet_ahb", "ahb", base + 0x68, 12);
+ hws[IMX6ULL_CLK_DCP_CLK] = imx_clk_hw_gate2("dcp", "ahb", base + 0x68, 10);
+ hws[IMX6UL_CLK_ENET] = imx_clk_hw_gate2("enet", "ipg", base + 0x68, 12);
+ hws[IMX6UL_CLK_ENET_AHB] = imx_clk_hw_gate2("enet_ahb", "ahb", base + 0x68, 12);
}
- clks[IMX6UL_CLK_CAN1_IPG] = imx_clk_gate2("can1_ipg", "ipg", base + 0x68, 14);
- clks[IMX6UL_CLK_CAN1_SERIAL] = imx_clk_gate2("can1_serial", "can_podf", base + 0x68, 16);
- clks[IMX6UL_CLK_CAN2_IPG] = imx_clk_gate2("can2_ipg", "ipg", base + 0x68, 18);
- clks[IMX6UL_CLK_CAN2_SERIAL] = imx_clk_gate2("can2_serial", "can_podf", base + 0x68, 20);
- clks[IMX6UL_CLK_GPT2_BUS] = imx_clk_gate2("gpt2_bus", "perclk", base + 0x68, 24);
- clks[IMX6UL_CLK_GPT2_SERIAL] = imx_clk_gate2("gpt2_serial", "perclk", base + 0x68, 26);
- clks[IMX6UL_CLK_UART2_IPG] = imx_clk_gate2("uart2_ipg", "ipg", base + 0x68, 28);
- clks[IMX6UL_CLK_UART2_SERIAL] = imx_clk_gate2("uart2_serial", "uart_podf", base + 0x68, 28);
+ hws[IMX6UL_CLK_CAN1_IPG] = imx_clk_hw_gate2("can1_ipg", "ipg", base + 0x68, 14);
+ hws[IMX6UL_CLK_CAN1_SERIAL] = imx_clk_hw_gate2("can1_serial", "can_podf", base + 0x68, 16);
+ hws[IMX6UL_CLK_CAN2_IPG] = imx_clk_hw_gate2("can2_ipg", "ipg", base + 0x68, 18);
+ hws[IMX6UL_CLK_CAN2_SERIAL] = imx_clk_hw_gate2("can2_serial", "can_podf", base + 0x68, 20);
+ hws[IMX6UL_CLK_GPT2_BUS] = imx_clk_hw_gate2("gpt2_bus", "perclk", base + 0x68, 24);
+ hws[IMX6UL_CLK_GPT2_SERIAL] = imx_clk_hw_gate2("gpt2_serial", "perclk", base + 0x68, 26);
+ hws[IMX6UL_CLK_UART2_IPG] = imx_clk_hw_gate2("uart2_ipg", "ipg", base + 0x68, 28);
+ hws[IMX6UL_CLK_UART2_SERIAL] = imx_clk_hw_gate2("uart2_serial", "uart_podf", base + 0x68, 28);
if (clk_on_imx6ull())
- clks[IMX6UL_CLK_AIPSTZ3] = imx_clk_gate2("aips_tz3", "ahb", base + 0x80, 18);
- clks[IMX6UL_CLK_GPIO2] = imx_clk_gate2("gpio2", "ipg", base + 0x68, 30);
+ hws[IMX6UL_CLK_AIPSTZ3] = imx_clk_hw_gate2("aips_tz3", "ahb", base + 0x80, 18);
+ hws[IMX6UL_CLK_GPIO2] = imx_clk_hw_gate2("gpio2", "ipg", base + 0x68, 30);
/* CCGR1 */
- clks[IMX6UL_CLK_ECSPI1] = imx_clk_gate2("ecspi1", "ecspi_podf", base + 0x6c, 0);
- clks[IMX6UL_CLK_ECSPI2] = imx_clk_gate2("ecspi2", "ecspi_podf", base + 0x6c, 2);
- clks[IMX6UL_CLK_ECSPI3] = imx_clk_gate2("ecspi3", "ecspi_podf", base + 0x6c, 4);
- clks[IMX6UL_CLK_ECSPI4] = imx_clk_gate2("ecspi4", "ecspi_podf", base + 0x6c, 6);
- clks[IMX6UL_CLK_ADC2] = imx_clk_gate2("adc2", "ipg", base + 0x6c, 8);
- clks[IMX6UL_CLK_UART3_IPG] = imx_clk_gate2("uart3_ipg", "ipg", base + 0x6c, 10);
- clks[IMX6UL_CLK_UART3_SERIAL] = imx_clk_gate2("uart3_serial", "uart_podf", base + 0x6c, 10);
- clks[IMX6UL_CLK_EPIT1] = imx_clk_gate2("epit1", "perclk", base + 0x6c, 12);
- clks[IMX6UL_CLK_EPIT2] = imx_clk_gate2("epit2", "perclk", base + 0x6c, 14);
- clks[IMX6UL_CLK_ADC1] = imx_clk_gate2("adc1", "ipg", base + 0x6c, 16);
- clks[IMX6UL_CLK_GPT1_BUS] = imx_clk_gate2("gpt1_bus", "perclk", base + 0x6c, 20);
- clks[IMX6UL_CLK_GPT1_SERIAL] = imx_clk_gate2("gpt1_serial", "perclk", base + 0x6c, 22);
- clks[IMX6UL_CLK_UART4_IPG] = imx_clk_gate2("uart4_ipg", "ipg", base + 0x6c, 24);
- clks[IMX6UL_CLK_UART4_SERIAL] = imx_clk_gate2("uart4_serial", "uart_podf", base + 0x6c, 24);
- clks[IMX6UL_CLK_GPIO1] = imx_clk_gate2("gpio1", "ipg", base + 0x6c, 26);
- clks[IMX6UL_CLK_GPIO5] = imx_clk_gate2("gpio5", "ipg", base + 0x6c, 30);
+ hws[IMX6UL_CLK_ECSPI1] = imx_clk_hw_gate2("ecspi1", "ecspi_podf", base + 0x6c, 0);
+ hws[IMX6UL_CLK_ECSPI2] = imx_clk_hw_gate2("ecspi2", "ecspi_podf", base + 0x6c, 2);
+ hws[IMX6UL_CLK_ECSPI3] = imx_clk_hw_gate2("ecspi3", "ecspi_podf", base + 0x6c, 4);
+ hws[IMX6UL_CLK_ECSPI4] = imx_clk_hw_gate2("ecspi4", "ecspi_podf", base + 0x6c, 6);
+ hws[IMX6UL_CLK_ADC2] = imx_clk_hw_gate2("adc2", "ipg", base + 0x6c, 8);
+ hws[IMX6UL_CLK_UART3_IPG] = imx_clk_hw_gate2("uart3_ipg", "ipg", base + 0x6c, 10);
+ hws[IMX6UL_CLK_UART3_SERIAL] = imx_clk_hw_gate2("uart3_serial", "uart_podf", base + 0x6c, 10);
+ hws[IMX6UL_CLK_EPIT1] = imx_clk_hw_gate2("epit1", "perclk", base + 0x6c, 12);
+ hws[IMX6UL_CLK_EPIT2] = imx_clk_hw_gate2("epit2", "perclk", base + 0x6c, 14);
+ hws[IMX6UL_CLK_ADC1] = imx_clk_hw_gate2("adc1", "ipg", base + 0x6c, 16);
+ hws[IMX6UL_CLK_GPT1_BUS] = imx_clk_hw_gate2("gpt1_bus", "perclk", base + 0x6c, 20);
+ hws[IMX6UL_CLK_GPT1_SERIAL] = imx_clk_hw_gate2("gpt1_serial", "perclk", base + 0x6c, 22);
+ hws[IMX6UL_CLK_UART4_IPG] = imx_clk_hw_gate2("uart4_ipg", "ipg", base + 0x6c, 24);
+ hws[IMX6UL_CLK_UART4_SERIAL] = imx_clk_hw_gate2("uart4_serial", "uart_podf", base + 0x6c, 24);
+ hws[IMX6UL_CLK_GPIO1] = imx_clk_hw_gate2("gpio1", "ipg", base + 0x6c, 26);
+ hws[IMX6UL_CLK_GPIO5] = imx_clk_hw_gate2("gpio5", "ipg", base + 0x6c, 30);
/* CCGR2 */
if (clk_on_imx6ull()) {
- clks[IMX6ULL_CLK_ESAI_EXTAL] = imx_clk_gate2_shared("esai_extal", "esai_podf", base + 0x70, 0, &share_count_esai);
- clks[IMX6ULL_CLK_ESAI_IPG] = imx_clk_gate2_shared("esai_ipg", "ahb", base + 0x70, 0, &share_count_esai);
- clks[IMX6ULL_CLK_ESAI_MEM] = imx_clk_gate2_shared("esai_mem", "ahb", base + 0x70, 0, &share_count_esai);
+ hws[IMX6ULL_CLK_ESAI_EXTAL] = imx_clk_hw_gate2_shared("esai_extal", "esai_podf", base + 0x70, 0, &share_count_esai);
+ hws[IMX6ULL_CLK_ESAI_IPG] = imx_clk_hw_gate2_shared("esai_ipg", "ahb", base + 0x70, 0, &share_count_esai);
+ hws[IMX6ULL_CLK_ESAI_MEM] = imx_clk_hw_gate2_shared("esai_mem", "ahb", base + 0x70, 0, &share_count_esai);
}
- clks[IMX6UL_CLK_CSI] = imx_clk_gate2("csi", "csi_podf", base + 0x70, 2);
- clks[IMX6UL_CLK_I2C1] = imx_clk_gate2("i2c1", "perclk", base + 0x70, 6);
- clks[IMX6UL_CLK_I2C2] = imx_clk_gate2("i2c2", "perclk", base + 0x70, 8);
- clks[IMX6UL_CLK_I2C3] = imx_clk_gate2("i2c3", "perclk", base + 0x70, 10);
- clks[IMX6UL_CLK_OCOTP] = imx_clk_gate2("ocotp", "ipg", base + 0x70, 12);
- clks[IMX6UL_CLK_IOMUXC] = imx_clk_gate2("iomuxc", "lcdif_podf", base + 0x70, 14);
- clks[IMX6UL_CLK_GPIO3] = imx_clk_gate2("gpio3", "ipg", base + 0x70, 26);
- clks[IMX6UL_CLK_LCDIF_APB] = imx_clk_gate2("lcdif_apb", "axi", base + 0x70, 28);
- clks[IMX6UL_CLK_PXP] = imx_clk_gate2("pxp", "axi", base + 0x70, 30);
+ hws[IMX6UL_CLK_CSI] = imx_clk_hw_gate2("csi", "csi_podf", base + 0x70, 2);
+ hws[IMX6UL_CLK_I2C1] = imx_clk_hw_gate2("i2c1", "perclk", base + 0x70, 6);
+ hws[IMX6UL_CLK_I2C2] = imx_clk_hw_gate2("i2c2", "perclk", base + 0x70, 8);
+ hws[IMX6UL_CLK_I2C3] = imx_clk_hw_gate2("i2c3", "perclk", base + 0x70, 10);
+ hws[IMX6UL_CLK_OCOTP] = imx_clk_hw_gate2("ocotp", "ipg", base + 0x70, 12);
+ hws[IMX6UL_CLK_IOMUXC] = imx_clk_hw_gate2("iomuxc", "lcdif_podf", base + 0x70, 14);
+ hws[IMX6UL_CLK_GPIO3] = imx_clk_hw_gate2("gpio3", "ipg", base + 0x70, 26);
+ hws[IMX6UL_CLK_LCDIF_APB] = imx_clk_hw_gate2("lcdif_apb", "axi", base + 0x70, 28);
+ hws[IMX6UL_CLK_PXP] = imx_clk_hw_gate2("pxp", "axi", base + 0x70, 30);
/* CCGR3 */
- clks[IMX6UL_CLK_UART5_IPG] = imx_clk_gate2("uart5_ipg", "ipg", base + 0x74, 2);
- clks[IMX6UL_CLK_UART5_SERIAL] = imx_clk_gate2("uart5_serial", "uart_podf", base + 0x74, 2);
+ hws[IMX6UL_CLK_UART5_IPG] = imx_clk_hw_gate2("uart5_ipg", "ipg", base + 0x74, 2);
+ hws[IMX6UL_CLK_UART5_SERIAL] = imx_clk_hw_gate2("uart5_serial", "uart_podf", base + 0x74, 2);
if (clk_on_imx6ul()) {
- clks[IMX6UL_CLK_ENET] = imx_clk_gate2("enet", "ipg", base + 0x74, 4);
- clks[IMX6UL_CLK_ENET_AHB] = imx_clk_gate2("enet_ahb", "ahb", base + 0x74, 4);
+ hws[IMX6UL_CLK_ENET] = imx_clk_hw_gate2("enet", "ipg", base + 0x74, 4);
+ hws[IMX6UL_CLK_ENET_AHB] = imx_clk_hw_gate2("enet_ahb", "ahb", base + 0x74, 4);
} else if (clk_on_imx6ull()) {
- clks[IMX6ULL_CLK_EPDC_ACLK] = imx_clk_gate2("epdc_aclk", "axi", base + 0x74, 4);
- clks[IMX6ULL_CLK_EPDC_PIX] = imx_clk_gate2("epdc_pix", "epdc_podf", base + 0x74, 4);
+ hws[IMX6ULL_CLK_EPDC_ACLK] = imx_clk_hw_gate2("epdc_aclk", "axi", base + 0x74, 4);
+ hws[IMX6ULL_CLK_EPDC_PIX] = imx_clk_hw_gate2("epdc_pix", "epdc_podf", base + 0x74, 4);
}
- clks[IMX6UL_CLK_UART6_IPG] = imx_clk_gate2("uart6_ipg", "ipg", base + 0x74, 6);
- clks[IMX6UL_CLK_UART6_SERIAL] = imx_clk_gate2("uart6_serial", "uart_podf", base + 0x74, 6);
- clks[IMX6UL_CLK_LCDIF_PIX] = imx_clk_gate2("lcdif_pix", "lcdif_podf", base + 0x74, 10);
- clks[IMX6UL_CLK_GPIO4] = imx_clk_gate2("gpio4", "ipg", base + 0x74, 12);
- clks[IMX6UL_CLK_QSPI] = imx_clk_gate2("qspi1", "qspi1_podf", base + 0x74, 14);
- clks[IMX6UL_CLK_WDOG1] = imx_clk_gate2("wdog1", "ipg", base + 0x74, 16);
- clks[IMX6UL_CLK_MMDC_P0_FAST] = imx_clk_gate_flags("mmdc_p0_fast", "mmdc_podf", base + 0x74, 20, CLK_IS_CRITICAL);
- clks[IMX6UL_CLK_MMDC_P0_IPG] = imx_clk_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL);
- clks[IMX6UL_CLK_MMDC_P1_IPG] = imx_clk_gate2("mmdc_p1_ipg", "ipg", base + 0x74, 26);
- clks[IMX6UL_CLK_AXI] = imx_clk_gate_flags("axi", "axi_podf", base + 0x74, 28, CLK_IS_CRITICAL);
+ hws[IMX6UL_CLK_UART6_IPG] = imx_clk_hw_gate2("uart6_ipg", "ipg", base + 0x74, 6);
+ hws[IMX6UL_CLK_UART6_SERIAL] = imx_clk_hw_gate2("uart6_serial", "uart_podf", base + 0x74, 6);
+ hws[IMX6UL_CLK_LCDIF_PIX] = imx_clk_hw_gate2("lcdif_pix", "lcdif_podf", base + 0x74, 10);
+ hws[IMX6UL_CLK_GPIO4] = imx_clk_hw_gate2("gpio4", "ipg", base + 0x74, 12);
+ hws[IMX6UL_CLK_QSPI] = imx_clk_hw_gate2("qspi1", "qspi1_podf", base + 0x74, 14);
+ hws[IMX6UL_CLK_WDOG1] = imx_clk_hw_gate2("wdog1", "ipg", base + 0x74, 16);
+ hws[IMX6UL_CLK_MMDC_P0_FAST] = imx_clk_hw_gate_flags("mmdc_p0_fast", "mmdc_podf", base + 0x74, 20, CLK_IS_CRITICAL);
+ hws[IMX6UL_CLK_MMDC_P0_IPG] = imx_clk_hw_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL);
+ hws[IMX6UL_CLK_MMDC_P1_IPG] = imx_clk_hw_gate2_flags("mmdc_p1_ipg", "ipg", base + 0x74, 26, CLK_IS_CRITICAL);
+ hws[IMX6UL_CLK_AXI] = imx_clk_hw_gate_flags("axi", "axi_podf", base + 0x74, 28, CLK_IS_CRITICAL);
/* CCGR4 */
- clks[IMX6UL_CLK_PER_BCH] = imx_clk_gate2("per_bch", "bch_podf", base + 0x78, 12);
- clks[IMX6UL_CLK_PWM1] = imx_clk_gate2("pwm1", "perclk", base + 0x78, 16);
- clks[IMX6UL_CLK_PWM2] = imx_clk_gate2("pwm2", "perclk", base + 0x78, 18);
- clks[IMX6UL_CLK_PWM3] = imx_clk_gate2("pwm3", "perclk", base + 0x78, 20);
- clks[IMX6UL_CLK_PWM4] = imx_clk_gate2("pwm4", "perclk", base + 0x78, 22);
- clks[IMX6UL_CLK_GPMI_BCH_APB] = imx_clk_gate2("gpmi_bch_apb", "bch_podf", base + 0x78, 24);
- clks[IMX6UL_CLK_GPMI_BCH] = imx_clk_gate2("gpmi_bch", "gpmi_podf", base + 0x78, 26);
- clks[IMX6UL_CLK_GPMI_IO] = imx_clk_gate2("gpmi_io", "enfc_podf", base + 0x78, 28);
- clks[IMX6UL_CLK_GPMI_APB] = imx_clk_gate2("gpmi_apb", "bch_podf", base + 0x78, 30);
+ hws[IMX6UL_CLK_PER_BCH] = imx_clk_hw_gate2("per_bch", "bch_podf", base + 0x78, 12);
+ hws[IMX6UL_CLK_PWM1] = imx_clk_hw_gate2("pwm1", "perclk", base + 0x78, 16);
+ hws[IMX6UL_CLK_PWM2] = imx_clk_hw_gate2("pwm2", "perclk", base + 0x78, 18);
+ hws[IMX6UL_CLK_PWM3] = imx_clk_hw_gate2("pwm3", "perclk", base + 0x78, 20);
+ hws[IMX6UL_CLK_PWM4] = imx_clk_hw_gate2("pwm4", "perclk", base + 0x78, 22);
+ hws[IMX6UL_CLK_GPMI_BCH_APB] = imx_clk_hw_gate2("gpmi_bch_apb", "bch_podf", base + 0x78, 24);
+ hws[IMX6UL_CLK_GPMI_BCH] = imx_clk_hw_gate2("gpmi_bch", "gpmi_podf", base + 0x78, 26);
+ hws[IMX6UL_CLK_GPMI_IO] = imx_clk_hw_gate2("gpmi_io", "enfc_podf", base + 0x78, 28);
+ hws[IMX6UL_CLK_GPMI_APB] = imx_clk_hw_gate2("gpmi_apb", "bch_podf", base + 0x78, 30);
/* CCGR5 */
- clks[IMX6UL_CLK_ROM] = imx_clk_gate2_flags("rom", "ahb", base + 0x7c, 0, CLK_IS_CRITICAL);
- clks[IMX6UL_CLK_SDMA] = imx_clk_gate2("sdma", "ahb", base + 0x7c, 6);
- clks[IMX6UL_CLK_KPP] = imx_clk_gate2("kpp", "ipg", base + 0x7c, 8);
- clks[IMX6UL_CLK_WDOG2] = imx_clk_gate2("wdog2", "ipg", base + 0x7c, 10);
- clks[IMX6UL_CLK_SPBA] = imx_clk_gate2("spba", "ipg", base + 0x7c, 12);
- clks[IMX6UL_CLK_SPDIF] = imx_clk_gate2_shared("spdif", "spdif_podf", base + 0x7c, 14, &share_count_audio);
- clks[IMX6UL_CLK_SPDIF_GCLK] = imx_clk_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_audio);
- clks[IMX6UL_CLK_SAI3] = imx_clk_gate2_shared("sai3", "sai3_podf", base + 0x7c, 22, &share_count_sai3);
- clks[IMX6UL_CLK_SAI3_IPG] = imx_clk_gate2_shared("sai3_ipg", "ipg", base + 0x7c, 22, &share_count_sai3);
- clks[IMX6UL_CLK_UART1_IPG] = imx_clk_gate2("uart1_ipg", "ipg", base + 0x7c, 24);
- clks[IMX6UL_CLK_UART1_SERIAL] = imx_clk_gate2("uart1_serial", "uart_podf", base + 0x7c, 24);
- clks[IMX6UL_CLK_UART7_IPG] = imx_clk_gate2("uart7_ipg", "ipg", base + 0x7c, 26);
- clks[IMX6UL_CLK_UART7_SERIAL] = imx_clk_gate2("uart7_serial", "uart_podf", base + 0x7c, 26);
- clks[IMX6UL_CLK_SAI1] = imx_clk_gate2_shared("sai1", "sai1_podf", base + 0x7c, 28, &share_count_sai1);
- clks[IMX6UL_CLK_SAI1_IPG] = imx_clk_gate2_shared("sai1_ipg", "ipg", base + 0x7c, 28, &share_count_sai1);
- clks[IMX6UL_CLK_SAI2] = imx_clk_gate2_shared("sai2", "sai2_podf", base + 0x7c, 30, &share_count_sai2);
- clks[IMX6UL_CLK_SAI2_IPG] = imx_clk_gate2_shared("sai2_ipg", "ipg", base + 0x7c, 30, &share_count_sai2);
+ hws[IMX6UL_CLK_ROM] = imx_clk_hw_gate2_flags("rom", "ahb", base + 0x7c, 0, CLK_IS_CRITICAL);
+ hws[IMX6UL_CLK_SDMA] = imx_clk_hw_gate2("sdma", "ahb", base + 0x7c, 6);
+ hws[IMX6UL_CLK_KPP] = imx_clk_hw_gate2("kpp", "ipg", base + 0x7c, 8);
+ hws[IMX6UL_CLK_WDOG2] = imx_clk_hw_gate2("wdog2", "ipg", base + 0x7c, 10);
+ hws[IMX6UL_CLK_SPBA] = imx_clk_hw_gate2("spba", "ipg", base + 0x7c, 12);
+ hws[IMX6UL_CLK_SPDIF] = imx_clk_hw_gate2_shared("spdif", "spdif_podf", base + 0x7c, 14, &share_count_audio);
+ hws[IMX6UL_CLK_SPDIF_GCLK] = imx_clk_hw_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_audio);
+ hws[IMX6UL_CLK_SAI3] = imx_clk_hw_gate2_shared("sai3", "sai3_podf", base + 0x7c, 22, &share_count_sai3);
+ hws[IMX6UL_CLK_SAI3_IPG] = imx_clk_hw_gate2_shared("sai3_ipg", "ipg", base + 0x7c, 22, &share_count_sai3);
+ hws[IMX6UL_CLK_UART1_IPG] = imx_clk_hw_gate2("uart1_ipg", "ipg", base + 0x7c, 24);
+ hws[IMX6UL_CLK_UART1_SERIAL] = imx_clk_hw_gate2("uart1_serial", "uart_podf", base + 0x7c, 24);
+ hws[IMX6UL_CLK_UART7_IPG] = imx_clk_hw_gate2("uart7_ipg", "ipg", base + 0x7c, 26);
+ hws[IMX6UL_CLK_UART7_SERIAL] = imx_clk_hw_gate2("uart7_serial", "uart_podf", base + 0x7c, 26);
+ hws[IMX6UL_CLK_SAI1] = imx_clk_hw_gate2_shared("sai1", "sai1_podf", base + 0x7c, 28, &share_count_sai1);
+ hws[IMX6UL_CLK_SAI1_IPG] = imx_clk_hw_gate2_shared("sai1_ipg", "ipg", base + 0x7c, 28, &share_count_sai1);
+ hws[IMX6UL_CLK_SAI2] = imx_clk_hw_gate2_shared("sai2", "sai2_podf", base + 0x7c, 30, &share_count_sai2);
+ hws[IMX6UL_CLK_SAI2_IPG] = imx_clk_hw_gate2_shared("sai2_ipg", "ipg", base + 0x7c, 30, &share_count_sai2);
/* CCGR6 */
- clks[IMX6UL_CLK_USBOH3] = imx_clk_gate2("usboh3", "ipg", base + 0x80, 0);
- clks[IMX6UL_CLK_USDHC1] = imx_clk_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2);
- clks[IMX6UL_CLK_USDHC2] = imx_clk_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4);
+ hws[IMX6UL_CLK_USBOH3] = imx_clk_hw_gate2("usboh3", "ipg", base + 0x80, 0);
+ hws[IMX6UL_CLK_USDHC1] = imx_clk_hw_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2);
+ hws[IMX6UL_CLK_USDHC2] = imx_clk_hw_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4);
if (clk_on_imx6ul()) {
- clks[IMX6UL_CLK_SIM1] = imx_clk_gate2("sim1", "sim_sel", base + 0x80, 6);
- clks[IMX6UL_CLK_SIM2] = imx_clk_gate2("sim2", "sim_sel", base + 0x80, 8);
+ hws[IMX6UL_CLK_SIM1] = imx_clk_hw_gate2("sim1", "sim_sel", base + 0x80, 6);
+ hws[IMX6UL_CLK_SIM2] = imx_clk_hw_gate2("sim2", "sim_sel", base + 0x80, 8);
}
- clks[IMX6UL_CLK_EIM] = imx_clk_gate2("eim", "eim_slow_podf", base + 0x80, 10);
- clks[IMX6UL_CLK_PWM8] = imx_clk_gate2("pwm8", "perclk", base + 0x80, 16);
- clks[IMX6UL_CLK_UART8_IPG] = imx_clk_gate2("uart8_ipg", "ipg", base + 0x80, 14);
- clks[IMX6UL_CLK_UART8_SERIAL] = imx_clk_gate2("uart8_serial", "uart_podf", base + 0x80, 14);
- clks[IMX6UL_CLK_WDOG3] = imx_clk_gate2("wdog3", "ipg", base + 0x80, 20);
- clks[IMX6UL_CLK_I2C4] = imx_clk_gate2("i2c4", "perclk", base + 0x80, 24);
- clks[IMX6UL_CLK_PWM5] = imx_clk_gate2("pwm5", "perclk", base + 0x80, 26);
- clks[IMX6UL_CLK_PWM6] = imx_clk_gate2("pwm6", "perclk", base + 0x80, 28);
- clks[IMX6UL_CLK_PWM7] = imx_clk_gate2("pwm7", "perclk", base + 0x80, 30);
+ hws[IMX6UL_CLK_EIM] = imx_clk_hw_gate2("eim", "eim_slow_podf", base + 0x80, 10);
+ hws[IMX6UL_CLK_PWM8] = imx_clk_hw_gate2("pwm8", "perclk", base + 0x80, 16);
+ hws[IMX6UL_CLK_UART8_IPG] = imx_clk_hw_gate2("uart8_ipg", "ipg", base + 0x80, 14);
+ hws[IMX6UL_CLK_UART8_SERIAL] = imx_clk_hw_gate2("uart8_serial", "uart_podf", base + 0x80, 14);
+ hws[IMX6UL_CLK_WDOG3] = imx_clk_hw_gate2("wdog3", "ipg", base + 0x80, 20);
+ hws[IMX6UL_CLK_I2C4] = imx_clk_hw_gate2("i2c4", "perclk", base + 0x80, 24);
+ hws[IMX6UL_CLK_PWM5] = imx_clk_hw_gate2("pwm5", "perclk", base + 0x80, 26);
+ hws[IMX6UL_CLK_PWM6] = imx_clk_hw_gate2("pwm6", "perclk", base + 0x80, 28);
+ hws[IMX6UL_CLK_PWM7] = imx_clk_hw_gate2("pwm7", "perclk", base + 0x80, 30);
/* CCOSR */
- clks[IMX6UL_CLK_CKO1] = imx_clk_gate("cko1", "cko1_podf", base + 0x60, 7);
- clks[IMX6UL_CLK_CKO2] = imx_clk_gate("cko2", "cko2_podf", base + 0x60, 24);
+ hws[IMX6UL_CLK_CKO1] = imx_clk_hw_gate("cko1", "cko1_podf", base + 0x60, 7);
+ hws[IMX6UL_CLK_CKO2] = imx_clk_hw_gate("cko2", "cko2_podf", base + 0x60, 24);
/* mask handshake of mmdc */
- writel_relaxed(BM_CCM_CCDR_MMDC_CH0_MASK, base + CCDR);
+ imx_mmdc_mask_handshake(base, 0);
- imx_check_clocks(clks, ARRAY_SIZE(clks));
+ imx_check_clk_hws(hws, IMX6UL_CLK_END);
- clk_data.clks = clks;
- clk_data.clk_num = ARRAY_SIZE(clks);
- of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
+ of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data);
/*
* Lower the AHB clock rate before changing the parent clock source,
@@ -473,39 +477,39 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
* AXI clock rate, so we need to lower AHB rate first to make sure at
* any time, AHB rate is <= 133MHz.
*/
- clk_set_rate(clks[IMX6UL_CLK_AHB], 99000000);
+ clk_set_rate(hws[IMX6UL_CLK_AHB]->clk, 99000000);
/* Change periph_pre clock to pll2_bus to adjust AXI rate to 264MHz */
- clk_set_parent(clks[IMX6UL_CLK_PERIPH_CLK2_SEL], clks[IMX6UL_CLK_OSC]);
- clk_set_parent(clks[IMX6UL_CLK_PERIPH], clks[IMX6UL_CLK_PERIPH_CLK2]);
- clk_set_parent(clks[IMX6UL_CLK_PERIPH_PRE], clks[IMX6UL_CLK_PLL2_BUS]);
- clk_set_parent(clks[IMX6UL_CLK_PERIPH], clks[IMX6UL_CLK_PERIPH_PRE]);
+ clk_set_parent(hws[IMX6UL_CLK_PERIPH_CLK2_SEL]->clk, hws[IMX6UL_CLK_OSC]->clk);
+ clk_set_parent(hws[IMX6UL_CLK_PERIPH]->clk, hws[IMX6UL_CLK_PERIPH_CLK2]->clk);
+ clk_set_parent(hws[IMX6UL_CLK_PERIPH_PRE]->clk, hws[IMX6UL_CLK_PLL2_BUS]->clk);
+ clk_set_parent(hws[IMX6UL_CLK_PERIPH]->clk, hws[IMX6UL_CLK_PERIPH_PRE]->clk);
/* Make sure AHB rate is 132MHz */
- clk_set_rate(clks[IMX6UL_CLK_AHB], 132000000);
+ clk_set_rate(hws[IMX6UL_CLK_AHB]->clk, 132000000);
/* set perclk to from OSC */
- clk_set_parent(clks[IMX6UL_CLK_PERCLK_SEL], clks[IMX6UL_CLK_OSC]);
+ clk_set_parent(hws[IMX6UL_CLK_PERCLK_SEL]->clk, hws[IMX6UL_CLK_OSC]->clk);
- clk_set_rate(clks[IMX6UL_CLK_ENET_REF], 50000000);
- clk_set_rate(clks[IMX6UL_CLK_ENET2_REF], 50000000);
- clk_set_rate(clks[IMX6UL_CLK_CSI], 24000000);
+ clk_set_rate(hws[IMX6UL_CLK_ENET_REF]->clk, 50000000);
+ clk_set_rate(hws[IMX6UL_CLK_ENET2_REF]->clk, 50000000);
+ clk_set_rate(hws[IMX6UL_CLK_CSI]->clk, 24000000);
if (clk_on_imx6ull())
- clk_prepare_enable(clks[IMX6UL_CLK_AIPSTZ3]);
+ clk_prepare_enable(hws[IMX6UL_CLK_AIPSTZ3]->clk);
if (IS_ENABLED(CONFIG_USB_MXS_PHY)) {
- clk_prepare_enable(clks[IMX6UL_CLK_USBPHY1_GATE]);
- clk_prepare_enable(clks[IMX6UL_CLK_USBPHY2_GATE]);
+ clk_prepare_enable(hws[IMX6UL_CLK_USBPHY1_GATE]->clk);
+ clk_prepare_enable(hws[IMX6UL_CLK_USBPHY2_GATE]->clk);
}
- clk_set_parent(clks[IMX6UL_CLK_CAN_SEL], clks[IMX6UL_CLK_PLL3_60M]);
+ clk_set_parent(hws[IMX6UL_CLK_CAN_SEL]->clk, hws[IMX6UL_CLK_PLL3_60M]->clk);
if (clk_on_imx6ul())
- clk_set_parent(clks[IMX6UL_CLK_SIM_PRE_SEL], clks[IMX6UL_CLK_PLL3_USB_OTG]);
+ clk_set_parent(hws[IMX6UL_CLK_SIM_PRE_SEL]->clk, hws[IMX6UL_CLK_PLL3_USB_OTG]->clk);
else if (clk_on_imx6ull())
- clk_set_parent(clks[IMX6ULL_CLK_EPDC_PRE_SEL], clks[IMX6UL_CLK_PLL3_PFD2]);
+ clk_set_parent(hws[IMX6ULL_CLK_EPDC_PRE_SEL]->clk, hws[IMX6UL_CLK_PLL3_PFD2]->clk);
- clk_set_parent(clks[IMX6UL_CLK_ENFC_SEL], clks[IMX6UL_CLK_PLL2_PFD2]);
+ clk_set_parent(hws[IMX6UL_CLK_ENFC_SEL]->clk, hws[IMX6UL_CLK_PLL2_PFD2]->clk);
}
CLK_OF_DECLARE(imx6ul, "fsl,imx6ul-ccm", imx6ul_clocks_init);
diff --git a/drivers/clk/imx/clk-imx7d.c b/drivers/clk/imx/clk-imx7d.c
index 8f3aa994c8f7..fbea774ef687 100644
--- a/drivers/clk/imx/clk-imx7d.c
+++ b/drivers/clk/imx/clk-imx7d.c
@@ -6,6 +6,7 @@
#include <dt-bindings/clock/imx7d-clock.h>
#include <linux/clk.h>
#include <linux/clkdev.h>
+#include <linux/clk-provider.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/io.h>
@@ -39,7 +40,6 @@ static const struct clk_div_table post_div_table[] = {
{ }
};
-static struct clk *clks[IMX7D_CLK_END];
static const char *arm_a7_sel[] = { "osc", "pll_arm_main_clk",
"pll_enet_500m_clk", "pll_dram_main_clk",
"pll_sys_main_clk", "pll_sys_pfd0_392m_clk", "pll_audio_post_div",
@@ -373,517 +373,533 @@ static const char *pll_enet_bypass_sel[] = { "pll_enet_main", "pll_enet_main_src
static const char *pll_audio_bypass_sel[] = { "pll_audio_main", "pll_audio_main_src", };
static const char *pll_video_bypass_sel[] = { "pll_video_main", "pll_video_main_src", };
-static struct clk_onecell_data clk_data;
-
-static struct clk ** const uart_clks[] __initconst = {
- &clks[IMX7D_UART1_ROOT_CLK],
- &clks[IMX7D_UART2_ROOT_CLK],
- &clks[IMX7D_UART3_ROOT_CLK],
- &clks[IMX7D_UART4_ROOT_CLK],
- &clks[IMX7D_UART5_ROOT_CLK],
- &clks[IMX7D_UART6_ROOT_CLK],
- &clks[IMX7D_UART7_ROOT_CLK],
- NULL
+static struct clk_hw **hws;
+static struct clk_hw_onecell_data *clk_hw_data;
+
+static const int uart_clk_ids[] __initconst = {
+ IMX7D_UART1_ROOT_CLK,
+ IMX7D_UART2_ROOT_CLK,
+ IMX7D_UART3_ROOT_CLK,
+ IMX7D_UART4_ROOT_CLK,
+ IMX7D_UART5_ROOT_CLK,
+ IMX7D_UART6_ROOT_CLK,
+ IMX7D_UART7_ROOT_CLK,
};
+static struct clk **uart_clks[ARRAY_SIZE(uart_clk_ids) + 1] __initdata;
+
static void __init imx7d_clocks_init(struct device_node *ccm_node)
{
struct device_node *np;
void __iomem *base;
+ int i;
+
+ clk_hw_data = kzalloc(struct_size(clk_hw_data, hws,
+ IMX7D_CLK_END), GFP_KERNEL);
+ if (WARN_ON(!clk_hw_data))
+ return;
+
+ clk_hw_data->num = IMX7D_CLK_END;
+ hws = clk_hw_data->hws;
- clks[IMX7D_CLK_DUMMY] = imx_clk_fixed("dummy", 0);
- clks[IMX7D_OSC_24M_CLK] = of_clk_get_by_name(ccm_node, "osc");
- clks[IMX7D_CKIL] = of_clk_get_by_name(ccm_node, "ckil");
+ hws[IMX7D_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0);
+ hws[IMX7D_OSC_24M_CLK] = __clk_get_hw(of_clk_get_by_name(ccm_node, "osc"));
+ hws[IMX7D_CKIL] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ckil"));
np = of_find_compatible_node(NULL, NULL, "fsl,imx7d-anatop");
base = of_iomap(np, 0);
WARN_ON(!base);
of_node_put(np);
- clks[IMX7D_PLL_ARM_MAIN_SRC] = imx_clk_mux("pll_arm_main_src", base + 0x60, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel));
- clks[IMX7D_PLL_DRAM_MAIN_SRC] = imx_clk_mux("pll_dram_main_src", base + 0x70, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel));
- clks[IMX7D_PLL_SYS_MAIN_SRC] = imx_clk_mux("pll_sys_main_src", base + 0xb0, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel));
- clks[IMX7D_PLL_ENET_MAIN_SRC] = imx_clk_mux("pll_enet_main_src", base + 0xe0, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel));
- clks[IMX7D_PLL_AUDIO_MAIN_SRC] = imx_clk_mux("pll_audio_main_src", base + 0xf0, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel));
- clks[IMX7D_PLL_VIDEO_MAIN_SRC] = imx_clk_mux("pll_video_main_src", base + 0x130, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel));
-
- clks[IMX7D_PLL_ARM_MAIN] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll_arm_main", "osc", base + 0x60, 0x7f);
- clks[IMX7D_PLL_DRAM_MAIN] = imx_clk_pllv3(IMX_PLLV3_DDR_IMX7, "pll_dram_main", "osc", base + 0x70, 0x7f);
- clks[IMX7D_PLL_SYS_MAIN] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll_sys_main", "osc", base + 0xb0, 0x1);
- clks[IMX7D_PLL_ENET_MAIN] = imx_clk_pllv3(IMX_PLLV3_ENET_IMX7, "pll_enet_main", "osc", base + 0xe0, 0x0);
- clks[IMX7D_PLL_AUDIO_MAIN] = imx_clk_pllv3(IMX_PLLV3_AV_IMX7, "pll_audio_main", "osc", base + 0xf0, 0x7f);
- clks[IMX7D_PLL_VIDEO_MAIN] = imx_clk_pllv3(IMX_PLLV3_AV_IMX7, "pll_video_main", "osc", base + 0x130, 0x7f);
-
- clks[IMX7D_PLL_ARM_MAIN_BYPASS] = imx_clk_mux_flags("pll_arm_main_bypass", base + 0x60, 16, 1, pll_arm_bypass_sel, ARRAY_SIZE(pll_arm_bypass_sel), CLK_SET_RATE_PARENT);
- clks[IMX7D_PLL_DRAM_MAIN_BYPASS] = imx_clk_mux_flags("pll_dram_main_bypass", base + 0x70, 16, 1, pll_dram_bypass_sel, ARRAY_SIZE(pll_dram_bypass_sel), CLK_SET_RATE_PARENT);
- clks[IMX7D_PLL_SYS_MAIN_BYPASS] = imx_clk_mux_flags("pll_sys_main_bypass", base + 0xb0, 16, 1, pll_sys_bypass_sel, ARRAY_SIZE(pll_sys_bypass_sel), CLK_SET_RATE_PARENT);
- clks[IMX7D_PLL_ENET_MAIN_BYPASS] = imx_clk_mux_flags("pll_enet_main_bypass", base + 0xe0, 16, 1, pll_enet_bypass_sel, ARRAY_SIZE(pll_enet_bypass_sel), CLK_SET_RATE_PARENT);
- clks[IMX7D_PLL_AUDIO_MAIN_BYPASS] = imx_clk_mux_flags("pll_audio_main_bypass", base + 0xf0, 16, 1, pll_audio_bypass_sel, ARRAY_SIZE(pll_audio_bypass_sel), CLK_SET_RATE_PARENT);
- clks[IMX7D_PLL_VIDEO_MAIN_BYPASS] = imx_clk_mux_flags("pll_video_main_bypass", base + 0x130, 16, 1, pll_video_bypass_sel, ARRAY_SIZE(pll_video_bypass_sel), CLK_SET_RATE_PARENT);
-
- clks[IMX7D_PLL_ARM_MAIN_CLK] = imx_clk_gate("pll_arm_main_clk", "pll_arm_main_bypass", base + 0x60, 13);
- clks[IMX7D_PLL_DRAM_MAIN_CLK] = imx_clk_gate("pll_dram_main_clk", "pll_dram_test_div", base + 0x70, 13);
- clks[IMX7D_PLL_SYS_MAIN_CLK] = imx_clk_gate("pll_sys_main_clk", "pll_sys_main_bypass", base + 0xb0, 13);
- clks[IMX7D_PLL_AUDIO_MAIN_CLK] = imx_clk_gate("pll_audio_main_clk", "pll_audio_main_bypass", base + 0xf0, 13);
- clks[IMX7D_PLL_VIDEO_MAIN_CLK] = imx_clk_gate("pll_video_main_clk", "pll_video_main_bypass", base + 0x130, 13);
-
- clks[IMX7D_PLL_DRAM_TEST_DIV] = clk_register_divider_table(NULL, "pll_dram_test_div", "pll_dram_main_bypass",
+ hws[IMX7D_PLL_ARM_MAIN_SRC] = imx_clk_hw_mux("pll_arm_main_src", base + 0x60, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel));
+ hws[IMX7D_PLL_DRAM_MAIN_SRC] = imx_clk_hw_mux("pll_dram_main_src", base + 0x70, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel));
+ hws[IMX7D_PLL_SYS_MAIN_SRC] = imx_clk_hw_mux("pll_sys_main_src", base + 0xb0, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel));
+ hws[IMX7D_PLL_ENET_MAIN_SRC] = imx_clk_hw_mux("pll_enet_main_src", base + 0xe0, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel));
+ hws[IMX7D_PLL_AUDIO_MAIN_SRC] = imx_clk_hw_mux("pll_audio_main_src", base + 0xf0, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel));
+ hws[IMX7D_PLL_VIDEO_MAIN_SRC] = imx_clk_hw_mux("pll_video_main_src", base + 0x130, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel));
+
+ hws[IMX7D_PLL_ARM_MAIN] = imx_clk_hw_pllv3(IMX_PLLV3_SYS, "pll_arm_main", "osc", base + 0x60, 0x7f);
+ hws[IMX7D_PLL_DRAM_MAIN] = imx_clk_hw_pllv3(IMX_PLLV3_DDR_IMX7, "pll_dram_main", "osc", base + 0x70, 0x7f);
+ hws[IMX7D_PLL_SYS_MAIN] = imx_clk_hw_pllv3(IMX_PLLV3_GENERIC, "pll_sys_main", "osc", base + 0xb0, 0x1);
+ hws[IMX7D_PLL_ENET_MAIN] = imx_clk_hw_pllv3(IMX_PLLV3_ENET_IMX7, "pll_enet_main", "osc", base + 0xe0, 0x0);
+ hws[IMX7D_PLL_AUDIO_MAIN] = imx_clk_hw_pllv3(IMX_PLLV3_AV_IMX7, "pll_audio_main", "osc", base + 0xf0, 0x7f);
+ hws[IMX7D_PLL_VIDEO_MAIN] = imx_clk_hw_pllv3(IMX_PLLV3_AV_IMX7, "pll_video_main", "osc", base + 0x130, 0x7f);
+
+ hws[IMX7D_PLL_ARM_MAIN_BYPASS] = imx_clk_hw_mux_flags("pll_arm_main_bypass", base + 0x60, 16, 1, pll_arm_bypass_sel, ARRAY_SIZE(pll_arm_bypass_sel), CLK_SET_RATE_PARENT);
+ hws[IMX7D_PLL_DRAM_MAIN_BYPASS] = imx_clk_hw_mux_flags("pll_dram_main_bypass", base + 0x70, 16, 1, pll_dram_bypass_sel, ARRAY_SIZE(pll_dram_bypass_sel), CLK_SET_RATE_PARENT);
+ hws[IMX7D_PLL_SYS_MAIN_BYPASS] = imx_clk_hw_mux_flags("pll_sys_main_bypass", base + 0xb0, 16, 1, pll_sys_bypass_sel, ARRAY_SIZE(pll_sys_bypass_sel), CLK_SET_RATE_PARENT);
+ hws[IMX7D_PLL_ENET_MAIN_BYPASS] = imx_clk_hw_mux_flags("pll_enet_main_bypass", base + 0xe0, 16, 1, pll_enet_bypass_sel, ARRAY_SIZE(pll_enet_bypass_sel), CLK_SET_RATE_PARENT);
+ hws[IMX7D_PLL_AUDIO_MAIN_BYPASS] = imx_clk_hw_mux_flags("pll_audio_main_bypass", base + 0xf0, 16, 1, pll_audio_bypass_sel, ARRAY_SIZE(pll_audio_bypass_sel), CLK_SET_RATE_PARENT);
+ hws[IMX7D_PLL_VIDEO_MAIN_BYPASS] = imx_clk_hw_mux_flags("pll_video_main_bypass", base + 0x130, 16, 1, pll_video_bypass_sel, ARRAY_SIZE(pll_video_bypass_sel), CLK_SET_RATE_PARENT);
+
+ hws[IMX7D_PLL_ARM_MAIN_CLK] = imx_clk_hw_gate("pll_arm_main_clk", "pll_arm_main_bypass", base + 0x60, 13);
+ hws[IMX7D_PLL_DRAM_MAIN_CLK] = imx_clk_hw_gate("pll_dram_main_clk", "pll_dram_test_div", base + 0x70, 13);
+ hws[IMX7D_PLL_SYS_MAIN_CLK] = imx_clk_hw_gate("pll_sys_main_clk", "pll_sys_main_bypass", base + 0xb0, 13);
+ hws[IMX7D_PLL_AUDIO_MAIN_CLK] = imx_clk_hw_gate("pll_audio_main_clk", "pll_audio_main_bypass", base + 0xf0, 13);
+ hws[IMX7D_PLL_VIDEO_MAIN_CLK] = imx_clk_hw_gate("pll_video_main_clk", "pll_video_main_bypass", base + 0x130, 13);
+
+ hws[IMX7D_PLL_DRAM_TEST_DIV] = clk_hw_register_divider_table(NULL, "pll_dram_test_div", "pll_dram_main_bypass",
CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x70, 21, 2, 0, test_div_table, &imx_ccm_lock);
- clks[IMX7D_PLL_AUDIO_TEST_DIV] = clk_register_divider_table(NULL, "pll_audio_test_div", "pll_audio_main_clk",
+ hws[IMX7D_PLL_AUDIO_TEST_DIV] = clk_hw_register_divider_table(NULL, "pll_audio_test_div", "pll_audio_main_clk",
CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0xf0, 19, 2, 0, test_div_table, &imx_ccm_lock);
- clks[IMX7D_PLL_AUDIO_POST_DIV] = clk_register_divider_table(NULL, "pll_audio_post_div", "pll_audio_test_div",
+ hws[IMX7D_PLL_AUDIO_POST_DIV] = clk_hw_register_divider_table(NULL, "pll_audio_post_div", "pll_audio_test_div",
CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0xf0, 22, 2, 0, post_div_table, &imx_ccm_lock);
- clks[IMX7D_PLL_VIDEO_TEST_DIV] = clk_register_divider_table(NULL, "pll_video_test_div", "pll_video_main_clk",
+ hws[IMX7D_PLL_VIDEO_TEST_DIV] = clk_hw_register_divider_table(NULL, "pll_video_test_div", "pll_video_main_clk",
CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x130, 19, 2, 0, test_div_table, &imx_ccm_lock);
- clks[IMX7D_PLL_VIDEO_POST_DIV] = clk_register_divider_table(NULL, "pll_video_post_div", "pll_video_test_div",
+ hws[IMX7D_PLL_VIDEO_POST_DIV] = clk_hw_register_divider_table(NULL, "pll_video_post_div", "pll_video_test_div",
CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x130, 22, 2, 0, post_div_table, &imx_ccm_lock);
- clks[IMX7D_PLL_SYS_PFD0_392M_CLK] = imx_clk_pfd("pll_sys_pfd0_392m_clk", "pll_sys_main_clk", base + 0xc0, 0);
- clks[IMX7D_PLL_SYS_PFD1_332M_CLK] = imx_clk_pfd("pll_sys_pfd1_332m_clk", "pll_sys_main_clk", base + 0xc0, 1);
- clks[IMX7D_PLL_SYS_PFD2_270M_CLK] = imx_clk_pfd("pll_sys_pfd2_270m_clk", "pll_sys_main_clk", base + 0xc0, 2);
-
- clks[IMX7D_PLL_SYS_PFD3_CLK] = imx_clk_pfd("pll_sys_pfd3_clk", "pll_sys_main_clk", base + 0xc0, 3);
- clks[IMX7D_PLL_SYS_PFD4_CLK] = imx_clk_pfd("pll_sys_pfd4_clk", "pll_sys_main_clk", base + 0xd0, 0);
- clks[IMX7D_PLL_SYS_PFD5_CLK] = imx_clk_pfd("pll_sys_pfd5_clk", "pll_sys_main_clk", base + 0xd0, 1);
- clks[IMX7D_PLL_SYS_PFD6_CLK] = imx_clk_pfd("pll_sys_pfd6_clk", "pll_sys_main_clk", base + 0xd0, 2);
- clks[IMX7D_PLL_SYS_PFD7_CLK] = imx_clk_pfd("pll_sys_pfd7_clk", "pll_sys_main_clk", base + 0xd0, 3);
-
- clks[IMX7D_PLL_SYS_MAIN_480M] = imx_clk_fixed_factor("pll_sys_main_480m", "pll_sys_main_clk", 1, 1);
- clks[IMX7D_PLL_SYS_MAIN_240M] = imx_clk_fixed_factor("pll_sys_main_240m", "pll_sys_main_clk", 1, 2);
- clks[IMX7D_PLL_SYS_MAIN_120M] = imx_clk_fixed_factor("pll_sys_main_120m", "pll_sys_main_clk", 1, 4);
- clks[IMX7D_PLL_DRAM_MAIN_533M] = imx_clk_fixed_factor("pll_dram_533m", "pll_dram_main_clk", 1, 2);
-
- clks[IMX7D_PLL_SYS_MAIN_480M_CLK] = imx_clk_gate_dis_flags("pll_sys_main_480m_clk", "pll_sys_main_480m", base + 0xb0, 4, CLK_IS_CRITICAL);
- clks[IMX7D_PLL_SYS_MAIN_240M_CLK] = imx_clk_gate_dis("pll_sys_main_240m_clk", "pll_sys_main_240m", base + 0xb0, 5);
- clks[IMX7D_PLL_SYS_MAIN_120M_CLK] = imx_clk_gate_dis("pll_sys_main_120m_clk", "pll_sys_main_120m", base + 0xb0, 6);
- clks[IMX7D_PLL_DRAM_MAIN_533M_CLK] = imx_clk_gate("pll_dram_533m_clk", "pll_dram_533m", base + 0x70, 12);
-
- clks[IMX7D_PLL_SYS_PFD0_196M] = imx_clk_fixed_factor("pll_sys_pfd0_196m", "pll_sys_pfd0_392m_clk", 1, 2);
- clks[IMX7D_PLL_SYS_PFD1_166M] = imx_clk_fixed_factor("pll_sys_pfd1_166m", "pll_sys_pfd1_332m_clk", 1, 2);
- clks[IMX7D_PLL_SYS_PFD2_135M] = imx_clk_fixed_factor("pll_sys_pfd2_135m", "pll_sys_pfd2_270m_clk", 1, 2);
-
- clks[IMX7D_PLL_SYS_PFD0_196M_CLK] = imx_clk_gate_dis("pll_sys_pfd0_196m_clk", "pll_sys_pfd0_196m", base + 0xb0, 26);
- clks[IMX7D_PLL_SYS_PFD1_166M_CLK] = imx_clk_gate_dis("pll_sys_pfd1_166m_clk", "pll_sys_pfd1_166m", base + 0xb0, 27);
- clks[IMX7D_PLL_SYS_PFD2_135M_CLK] = imx_clk_gate_dis("pll_sys_pfd2_135m_clk", "pll_sys_pfd2_135m", base + 0xb0, 28);
-
- clks[IMX7D_PLL_ENET_MAIN_CLK] = imx_clk_fixed_factor("pll_enet_main_clk", "pll_enet_main_bypass", 1, 1);
- clks[IMX7D_PLL_ENET_MAIN_500M] = imx_clk_fixed_factor("pll_enet_500m", "pll_enet_main_clk", 1, 2);
- clks[IMX7D_PLL_ENET_MAIN_250M] = imx_clk_fixed_factor("pll_enet_250m", "pll_enet_main_clk", 1, 4);
- clks[IMX7D_PLL_ENET_MAIN_125M] = imx_clk_fixed_factor("pll_enet_125m", "pll_enet_main_clk", 1, 8);
- clks[IMX7D_PLL_ENET_MAIN_100M] = imx_clk_fixed_factor("pll_enet_100m", "pll_enet_main_clk", 1, 10);
- clks[IMX7D_PLL_ENET_MAIN_50M] = imx_clk_fixed_factor("pll_enet_50m", "pll_enet_main_clk", 1, 20);
- clks[IMX7D_PLL_ENET_MAIN_40M] = imx_clk_fixed_factor("pll_enet_40m", "pll_enet_main_clk", 1, 25);
- clks[IMX7D_PLL_ENET_MAIN_25M] = imx_clk_fixed_factor("pll_enet_25m", "pll_enet_main_clk", 1, 40);
-
- clks[IMX7D_PLL_ENET_MAIN_500M_CLK] = imx_clk_gate("pll_enet_500m_clk", "pll_enet_500m", base + 0xe0, 12);
- clks[IMX7D_PLL_ENET_MAIN_250M_CLK] = imx_clk_gate("pll_enet_250m_clk", "pll_enet_250m", base + 0xe0, 11);
- clks[IMX7D_PLL_ENET_MAIN_125M_CLK] = imx_clk_gate("pll_enet_125m_clk", "pll_enet_125m", base + 0xe0, 10);
- clks[IMX7D_PLL_ENET_MAIN_100M_CLK] = imx_clk_gate("pll_enet_100m_clk", "pll_enet_100m", base + 0xe0, 9);
- clks[IMX7D_PLL_ENET_MAIN_50M_CLK] = imx_clk_gate("pll_enet_50m_clk", "pll_enet_50m", base + 0xe0, 8);
- clks[IMX7D_PLL_ENET_MAIN_40M_CLK] = imx_clk_gate("pll_enet_40m_clk", "pll_enet_40m", base + 0xe0, 7);
- clks[IMX7D_PLL_ENET_MAIN_25M_CLK] = imx_clk_gate("pll_enet_25m_clk", "pll_enet_25m", base + 0xe0, 6);
-
- clks[IMX7D_LVDS1_OUT_SEL] = imx_clk_mux("lvds1_sel", base + 0x170, 0, 5, lvds1_sel, ARRAY_SIZE(lvds1_sel));
- clks[IMX7D_LVDS1_OUT_CLK] = imx_clk_gate_exclusive("lvds1_out", "lvds1_sel", base + 0x170, 5, BIT(6));
+ hws[IMX7D_PLL_SYS_PFD0_392M_CLK] = imx_clk_hw_pfd("pll_sys_pfd0_392m_clk", "pll_sys_main_clk", base + 0xc0, 0);
+ hws[IMX7D_PLL_SYS_PFD1_332M_CLK] = imx_clk_hw_pfd("pll_sys_pfd1_332m_clk", "pll_sys_main_clk", base + 0xc0, 1);
+ hws[IMX7D_PLL_SYS_PFD2_270M_CLK] = imx_clk_hw_pfd("pll_sys_pfd2_270m_clk", "pll_sys_main_clk", base + 0xc0, 2);
+
+ hws[IMX7D_PLL_SYS_PFD3_CLK] = imx_clk_hw_pfd("pll_sys_pfd3_clk", "pll_sys_main_clk", base + 0xc0, 3);
+ hws[IMX7D_PLL_SYS_PFD4_CLK] = imx_clk_hw_pfd("pll_sys_pfd4_clk", "pll_sys_main_clk", base + 0xd0, 0);
+ hws[IMX7D_PLL_SYS_PFD5_CLK] = imx_clk_hw_pfd("pll_sys_pfd5_clk", "pll_sys_main_clk", base + 0xd0, 1);
+ hws[IMX7D_PLL_SYS_PFD6_CLK] = imx_clk_hw_pfd("pll_sys_pfd6_clk", "pll_sys_main_clk", base + 0xd0, 2);
+ hws[IMX7D_PLL_SYS_PFD7_CLK] = imx_clk_hw_pfd("pll_sys_pfd7_clk", "pll_sys_main_clk", base + 0xd0, 3);
+
+ hws[IMX7D_PLL_SYS_MAIN_480M] = imx_clk_hw_fixed_factor("pll_sys_main_480m", "pll_sys_main_clk", 1, 1);
+ hws[IMX7D_PLL_SYS_MAIN_240M] = imx_clk_hw_fixed_factor("pll_sys_main_240m", "pll_sys_main_clk", 1, 2);
+ hws[IMX7D_PLL_SYS_MAIN_120M] = imx_clk_hw_fixed_factor("pll_sys_main_120m", "pll_sys_main_clk", 1, 4);
+ hws[IMX7D_PLL_DRAM_MAIN_533M] = imx_clk_hw_fixed_factor("pll_dram_533m", "pll_dram_main_clk", 1, 2);
+
+ hws[IMX7D_PLL_SYS_MAIN_480M_CLK] = imx_clk_hw_gate_dis_flags("pll_sys_main_480m_clk", "pll_sys_main_480m", base + 0xb0, 4, CLK_IS_CRITICAL);
+ hws[IMX7D_PLL_SYS_MAIN_240M_CLK] = imx_clk_hw_gate_dis("pll_sys_main_240m_clk", "pll_sys_main_240m", base + 0xb0, 5);
+ hws[IMX7D_PLL_SYS_MAIN_120M_CLK] = imx_clk_hw_gate_dis("pll_sys_main_120m_clk", "pll_sys_main_120m", base + 0xb0, 6);
+ hws[IMX7D_PLL_DRAM_MAIN_533M_CLK] = imx_clk_hw_gate("pll_dram_533m_clk", "pll_dram_533m", base + 0x70, 12);
+
+ hws[IMX7D_PLL_SYS_PFD0_196M] = imx_clk_hw_fixed_factor("pll_sys_pfd0_196m", "pll_sys_pfd0_392m_clk", 1, 2);
+ hws[IMX7D_PLL_SYS_PFD1_166M] = imx_clk_hw_fixed_factor("pll_sys_pfd1_166m", "pll_sys_pfd1_332m_clk", 1, 2);
+ hws[IMX7D_PLL_SYS_PFD2_135M] = imx_clk_hw_fixed_factor("pll_sys_pfd2_135m", "pll_sys_pfd2_270m_clk", 1, 2);
+
+ hws[IMX7D_PLL_SYS_PFD0_196M_CLK] = imx_clk_hw_gate_dis("pll_sys_pfd0_196m_clk", "pll_sys_pfd0_196m", base + 0xb0, 26);
+ hws[IMX7D_PLL_SYS_PFD1_166M_CLK] = imx_clk_hw_gate_dis("pll_sys_pfd1_166m_clk", "pll_sys_pfd1_166m", base + 0xb0, 27);
+ hws[IMX7D_PLL_SYS_PFD2_135M_CLK] = imx_clk_hw_gate_dis("pll_sys_pfd2_135m_clk", "pll_sys_pfd2_135m", base + 0xb0, 28);
+
+ hws[IMX7D_PLL_ENET_MAIN_CLK] = imx_clk_hw_fixed_factor("pll_enet_main_clk", "pll_enet_main_bypass", 1, 1);
+ hws[IMX7D_PLL_ENET_MAIN_500M] = imx_clk_hw_fixed_factor("pll_enet_500m", "pll_enet_main_clk", 1, 2);
+ hws[IMX7D_PLL_ENET_MAIN_250M] = imx_clk_hw_fixed_factor("pll_enet_250m", "pll_enet_main_clk", 1, 4);
+ hws[IMX7D_PLL_ENET_MAIN_125M] = imx_clk_hw_fixed_factor("pll_enet_125m", "pll_enet_main_clk", 1, 8);
+ hws[IMX7D_PLL_ENET_MAIN_100M] = imx_clk_hw_fixed_factor("pll_enet_100m", "pll_enet_main_clk", 1, 10);
+ hws[IMX7D_PLL_ENET_MAIN_50M] = imx_clk_hw_fixed_factor("pll_enet_50m", "pll_enet_main_clk", 1, 20);
+ hws[IMX7D_PLL_ENET_MAIN_40M] = imx_clk_hw_fixed_factor("pll_enet_40m", "pll_enet_main_clk", 1, 25);
+ hws[IMX7D_PLL_ENET_MAIN_25M] = imx_clk_hw_fixed_factor("pll_enet_25m", "pll_enet_main_clk", 1, 40);
+
+ hws[IMX7D_PLL_ENET_MAIN_500M_CLK] = imx_clk_hw_gate("pll_enet_500m_clk", "pll_enet_500m", base + 0xe0, 12);
+ hws[IMX7D_PLL_ENET_MAIN_250M_CLK] = imx_clk_hw_gate("pll_enet_250m_clk", "pll_enet_250m", base + 0xe0, 11);
+ hws[IMX7D_PLL_ENET_MAIN_125M_CLK] = imx_clk_hw_gate("pll_enet_125m_clk", "pll_enet_125m", base + 0xe0, 10);
+ hws[IMX7D_PLL_ENET_MAIN_100M_CLK] = imx_clk_hw_gate("pll_enet_100m_clk", "pll_enet_100m", base + 0xe0, 9);
+ hws[IMX7D_PLL_ENET_MAIN_50M_CLK] = imx_clk_hw_gate("pll_enet_50m_clk", "pll_enet_50m", base + 0xe0, 8);
+ hws[IMX7D_PLL_ENET_MAIN_40M_CLK] = imx_clk_hw_gate("pll_enet_40m_clk", "pll_enet_40m", base + 0xe0, 7);
+ hws[IMX7D_PLL_ENET_MAIN_25M_CLK] = imx_clk_hw_gate("pll_enet_25m_clk", "pll_enet_25m", base + 0xe0, 6);
+
+ hws[IMX7D_LVDS1_OUT_SEL] = imx_clk_hw_mux("lvds1_sel", base + 0x170, 0, 5, lvds1_sel, ARRAY_SIZE(lvds1_sel));
+ hws[IMX7D_LVDS1_OUT_CLK] = imx_clk_hw_gate_exclusive("lvds1_out", "lvds1_sel", base + 0x170, 5, BIT(6));
np = ccm_node;
base = of_iomap(np, 0);
WARN_ON(!base);
- clks[IMX7D_ARM_A7_ROOT_SRC] = imx_clk_mux2("arm_a7_src", base + 0x8000, 24, 3, arm_a7_sel, ARRAY_SIZE(arm_a7_sel));
- clks[IMX7D_ARM_M4_ROOT_SRC] = imx_clk_mux2("arm_m4_src", base + 0x8080, 24, 3, arm_m4_sel, ARRAY_SIZE(arm_m4_sel));
- clks[IMX7D_MAIN_AXI_ROOT_SRC] = imx_clk_mux2("axi_src", base + 0x8800, 24, 3, axi_sel, ARRAY_SIZE(axi_sel));
- clks[IMX7D_DISP_AXI_ROOT_SRC] = imx_clk_mux2("disp_axi_src", base + 0x8880, 24, 3, disp_axi_sel, ARRAY_SIZE(disp_axi_sel));
- clks[IMX7D_ENET_AXI_ROOT_SRC] = imx_clk_mux2("enet_axi_src", base + 0x8900, 24, 3, enet_axi_sel, ARRAY_SIZE(enet_axi_sel));
- clks[IMX7D_NAND_USDHC_BUS_ROOT_SRC] = imx_clk_mux2("nand_usdhc_src", base + 0x8980, 24, 3, nand_usdhc_bus_sel, ARRAY_SIZE(nand_usdhc_bus_sel));
- clks[IMX7D_AHB_CHANNEL_ROOT_SRC] = imx_clk_mux2("ahb_src", base + 0x9000, 24, 3, ahb_channel_sel, ARRAY_SIZE(ahb_channel_sel));
- clks[IMX7D_DRAM_PHYM_ROOT_SRC] = imx_clk_mux2("dram_phym_src", base + 0x9800, 24, 1, dram_phym_sel, ARRAY_SIZE(dram_phym_sel));
- clks[IMX7D_DRAM_ROOT_SRC] = imx_clk_mux2("dram_src", base + 0x9880, 24, 1, dram_sel, ARRAY_SIZE(dram_sel));
- clks[IMX7D_DRAM_PHYM_ALT_ROOT_SRC] = imx_clk_mux2("dram_phym_alt_src", base + 0xa000, 24, 3, dram_phym_alt_sel, ARRAY_SIZE(dram_phym_alt_sel));
- clks[IMX7D_DRAM_ALT_ROOT_SRC] = imx_clk_mux2("dram_alt_src", base + 0xa080, 24, 3, dram_alt_sel, ARRAY_SIZE(dram_alt_sel));
- clks[IMX7D_USB_HSIC_ROOT_SRC] = imx_clk_mux2("usb_hsic_src", base + 0xa100, 24, 3, usb_hsic_sel, ARRAY_SIZE(usb_hsic_sel));
- clks[IMX7D_PCIE_CTRL_ROOT_SRC] = imx_clk_mux2("pcie_ctrl_src", base + 0xa180, 24, 3, pcie_ctrl_sel, ARRAY_SIZE(pcie_ctrl_sel));
- clks[IMX7D_PCIE_PHY_ROOT_SRC] = imx_clk_mux2("pcie_phy_src", base + 0xa200, 24, 3, pcie_phy_sel, ARRAY_SIZE(pcie_phy_sel));
- clks[IMX7D_EPDC_PIXEL_ROOT_SRC] = imx_clk_mux2("epdc_pixel_src", base + 0xa280, 24, 3, epdc_pixel_sel, ARRAY_SIZE(epdc_pixel_sel));
- clks[IMX7D_LCDIF_PIXEL_ROOT_SRC] = imx_clk_mux2("lcdif_pixel_src", base + 0xa300, 24, 3, lcdif_pixel_sel, ARRAY_SIZE(lcdif_pixel_sel));
- clks[IMX7D_MIPI_DSI_ROOT_SRC] = imx_clk_mux2("mipi_dsi_src", base + 0xa380, 24, 3, mipi_dsi_sel, ARRAY_SIZE(mipi_dsi_sel));
- clks[IMX7D_MIPI_CSI_ROOT_SRC] = imx_clk_mux2("mipi_csi_src", base + 0xa400, 24, 3, mipi_csi_sel, ARRAY_SIZE(mipi_csi_sel));
- clks[IMX7D_MIPI_DPHY_ROOT_SRC] = imx_clk_mux2("mipi_dphy_src", base + 0xa480, 24, 3, mipi_dphy_sel, ARRAY_SIZE(mipi_dphy_sel));
- clks[IMX7D_SAI1_ROOT_SRC] = imx_clk_mux2("sai1_src", base + 0xa500, 24, 3, sai1_sel, ARRAY_SIZE(sai1_sel));
- clks[IMX7D_SAI2_ROOT_SRC] = imx_clk_mux2("sai2_src", base + 0xa580, 24, 3, sai2_sel, ARRAY_SIZE(sai2_sel));
- clks[IMX7D_SAI3_ROOT_SRC] = imx_clk_mux2("sai3_src", base + 0xa600, 24, 3, sai3_sel, ARRAY_SIZE(sai3_sel));
- clks[IMX7D_SPDIF_ROOT_SRC] = imx_clk_mux2("spdif_src", base + 0xa680, 24, 3, spdif_sel, ARRAY_SIZE(spdif_sel));
- clks[IMX7D_ENET1_REF_ROOT_SRC] = imx_clk_mux2("enet1_ref_src", base + 0xa700, 24, 3, enet1_ref_sel, ARRAY_SIZE(enet1_ref_sel));
- clks[IMX7D_ENET1_TIME_ROOT_SRC] = imx_clk_mux2("enet1_time_src", base + 0xa780, 24, 3, enet1_time_sel, ARRAY_SIZE(enet1_time_sel));
- clks[IMX7D_ENET2_REF_ROOT_SRC] = imx_clk_mux2("enet2_ref_src", base + 0xa800, 24, 3, enet2_ref_sel, ARRAY_SIZE(enet2_ref_sel));
- clks[IMX7D_ENET2_TIME_ROOT_SRC] = imx_clk_mux2("enet2_time_src", base + 0xa880, 24, 3, enet2_time_sel, ARRAY_SIZE(enet2_time_sel));
- clks[IMX7D_ENET_PHY_REF_ROOT_SRC] = imx_clk_mux2("enet_phy_ref_src", base + 0xa900, 24, 3, enet_phy_ref_sel, ARRAY_SIZE(enet_phy_ref_sel));
- clks[IMX7D_EIM_ROOT_SRC] = imx_clk_mux2("eim_src", base + 0xa980, 24, 3, eim_sel, ARRAY_SIZE(eim_sel));
- clks[IMX7D_NAND_ROOT_SRC] = imx_clk_mux2("nand_src", base + 0xaa00, 24, 3, nand_sel, ARRAY_SIZE(nand_sel));
- clks[IMX7D_QSPI_ROOT_SRC] = imx_clk_mux2("qspi_src", base + 0xaa80, 24, 3, qspi_sel, ARRAY_SIZE(qspi_sel));
- clks[IMX7D_USDHC1_ROOT_SRC] = imx_clk_mux2("usdhc1_src", base + 0xab00, 24, 3, usdhc1_sel, ARRAY_SIZE(usdhc1_sel));
- clks[IMX7D_USDHC2_ROOT_SRC] = imx_clk_mux2("usdhc2_src", base + 0xab80, 24, 3, usdhc2_sel, ARRAY_SIZE(usdhc2_sel));
- clks[IMX7D_USDHC3_ROOT_SRC] = imx_clk_mux2("usdhc3_src", base + 0xac00, 24, 3, usdhc3_sel, ARRAY_SIZE(usdhc3_sel));
- clks[IMX7D_CAN1_ROOT_SRC] = imx_clk_mux2("can1_src", base + 0xac80, 24, 3, can1_sel, ARRAY_SIZE(can1_sel));
- clks[IMX7D_CAN2_ROOT_SRC] = imx_clk_mux2("can2_src", base + 0xad00, 24, 3, can2_sel, ARRAY_SIZE(can2_sel));
- clks[IMX7D_I2C1_ROOT_SRC] = imx_clk_mux2("i2c1_src", base + 0xad80, 24, 3, i2c1_sel, ARRAY_SIZE(i2c1_sel));
- clks[IMX7D_I2C2_ROOT_SRC] = imx_clk_mux2("i2c2_src", base + 0xae00, 24, 3, i2c2_sel, ARRAY_SIZE(i2c2_sel));
- clks[IMX7D_I2C3_ROOT_SRC] = imx_clk_mux2("i2c3_src", base + 0xae80, 24, 3, i2c3_sel, ARRAY_SIZE(i2c3_sel));
- clks[IMX7D_I2C4_ROOT_SRC] = imx_clk_mux2("i2c4_src", base + 0xaf00, 24, 3, i2c4_sel, ARRAY_SIZE(i2c4_sel));
- clks[IMX7D_UART1_ROOT_SRC] = imx_clk_mux2("uart1_src", base + 0xaf80, 24, 3, uart1_sel, ARRAY_SIZE(uart1_sel));
- clks[IMX7D_UART2_ROOT_SRC] = imx_clk_mux2("uart2_src", base + 0xb000, 24, 3, uart2_sel, ARRAY_SIZE(uart2_sel));
- clks[IMX7D_UART3_ROOT_SRC] = imx_clk_mux2("uart3_src", base + 0xb080, 24, 3, uart3_sel, ARRAY_SIZE(uart3_sel));
- clks[IMX7D_UART4_ROOT_SRC] = imx_clk_mux2("uart4_src", base + 0xb100, 24, 3, uart4_sel, ARRAY_SIZE(uart4_sel));
- clks[IMX7D_UART5_ROOT_SRC] = imx_clk_mux2("uart5_src", base + 0xb180, 24, 3, uart5_sel, ARRAY_SIZE(uart5_sel));
- clks[IMX7D_UART6_ROOT_SRC] = imx_clk_mux2("uart6_src", base + 0xb200, 24, 3, uart6_sel, ARRAY_SIZE(uart6_sel));
- clks[IMX7D_UART7_ROOT_SRC] = imx_clk_mux2("uart7_src", base + 0xb280, 24, 3, uart7_sel, ARRAY_SIZE(uart7_sel));
- clks[IMX7D_ECSPI1_ROOT_SRC] = imx_clk_mux2("ecspi1_src", base + 0xb300, 24, 3, ecspi1_sel, ARRAY_SIZE(ecspi1_sel));
- clks[IMX7D_ECSPI2_ROOT_SRC] = imx_clk_mux2("ecspi2_src", base + 0xb380, 24, 3, ecspi2_sel, ARRAY_SIZE(ecspi2_sel));
- clks[IMX7D_ECSPI3_ROOT_SRC] = imx_clk_mux2("ecspi3_src", base + 0xb400, 24, 3, ecspi3_sel, ARRAY_SIZE(ecspi3_sel));
- clks[IMX7D_ECSPI4_ROOT_SRC] = imx_clk_mux2("ecspi4_src", base + 0xb480, 24, 3, ecspi4_sel, ARRAY_SIZE(ecspi4_sel));
- clks[IMX7D_PWM1_ROOT_SRC] = imx_clk_mux2("pwm1_src", base + 0xb500, 24, 3, pwm1_sel, ARRAY_SIZE(pwm1_sel));
- clks[IMX7D_PWM2_ROOT_SRC] = imx_clk_mux2("pwm2_src", base + 0xb580, 24, 3, pwm2_sel, ARRAY_SIZE(pwm2_sel));
- clks[IMX7D_PWM3_ROOT_SRC] = imx_clk_mux2("pwm3_src", base + 0xb600, 24, 3, pwm3_sel, ARRAY_SIZE(pwm3_sel));
- clks[IMX7D_PWM4_ROOT_SRC] = imx_clk_mux2("pwm4_src", base + 0xb680, 24, 3, pwm4_sel, ARRAY_SIZE(pwm4_sel));
- clks[IMX7D_FLEXTIMER1_ROOT_SRC] = imx_clk_mux2("flextimer1_src", base + 0xb700, 24, 3, flextimer1_sel, ARRAY_SIZE(flextimer1_sel));
- clks[IMX7D_FLEXTIMER2_ROOT_SRC] = imx_clk_mux2("flextimer2_src", base + 0xb780, 24, 3, flextimer2_sel, ARRAY_SIZE(flextimer2_sel));
- clks[IMX7D_SIM1_ROOT_SRC] = imx_clk_mux2("sim1_src", base + 0xb800, 24, 3, sim1_sel, ARRAY_SIZE(sim1_sel));
- clks[IMX7D_SIM2_ROOT_SRC] = imx_clk_mux2("sim2_src", base + 0xb880, 24, 3, sim2_sel, ARRAY_SIZE(sim2_sel));
- clks[IMX7D_GPT1_ROOT_SRC] = imx_clk_mux2("gpt1_src", base + 0xb900, 24, 3, gpt1_sel, ARRAY_SIZE(gpt1_sel));
- clks[IMX7D_GPT2_ROOT_SRC] = imx_clk_mux2("gpt2_src", base + 0xb980, 24, 3, gpt2_sel, ARRAY_SIZE(gpt2_sel));
- clks[IMX7D_GPT3_ROOT_SRC] = imx_clk_mux2("gpt3_src", base + 0xba00, 24, 3, gpt3_sel, ARRAY_SIZE(gpt3_sel));
- clks[IMX7D_GPT4_ROOT_SRC] = imx_clk_mux2("gpt4_src", base + 0xba80, 24, 3, gpt4_sel, ARRAY_SIZE(gpt4_sel));
- clks[IMX7D_TRACE_ROOT_SRC] = imx_clk_mux2("trace_src", base + 0xbb00, 24, 3, trace_sel, ARRAY_SIZE(trace_sel));
- clks[IMX7D_WDOG_ROOT_SRC] = imx_clk_mux2("wdog_src", base + 0xbb80, 24, 3, wdog_sel, ARRAY_SIZE(wdog_sel));
- clks[IMX7D_CSI_MCLK_ROOT_SRC] = imx_clk_mux2("csi_mclk_src", base + 0xbc00, 24, 3, csi_mclk_sel, ARRAY_SIZE(csi_mclk_sel));
- clks[IMX7D_AUDIO_MCLK_ROOT_SRC] = imx_clk_mux2("audio_mclk_src", base + 0xbc80, 24, 3, audio_mclk_sel, ARRAY_SIZE(audio_mclk_sel));
- clks[IMX7D_WRCLK_ROOT_SRC] = imx_clk_mux2("wrclk_src", base + 0xbd00, 24, 3, wrclk_sel, ARRAY_SIZE(wrclk_sel));
- clks[IMX7D_CLKO1_ROOT_SRC] = imx_clk_mux2("clko1_src", base + 0xbd80, 24, 3, clko1_sel, ARRAY_SIZE(clko1_sel));
- clks[IMX7D_CLKO2_ROOT_SRC] = imx_clk_mux2("clko2_src", base + 0xbe00, 24, 3, clko2_sel, ARRAY_SIZE(clko2_sel));
-
- clks[IMX7D_ARM_A7_ROOT_CG] = imx_clk_gate3("arm_a7_cg", "arm_a7_src", base + 0x8000, 28);
- clks[IMX7D_ARM_M4_ROOT_CG] = imx_clk_gate3("arm_m4_cg", "arm_m4_src", base + 0x8080, 28);
- clks[IMX7D_MAIN_AXI_ROOT_CG] = imx_clk_gate3("axi_cg", "axi_src", base + 0x8800, 28);
- clks[IMX7D_DISP_AXI_ROOT_CG] = imx_clk_gate3("disp_axi_cg", "disp_axi_src", base + 0x8880, 28);
- clks[IMX7D_ENET_AXI_ROOT_CG] = imx_clk_gate3("enet_axi_cg", "enet_axi_src", base + 0x8900, 28);
- clks[IMX7D_NAND_USDHC_BUS_ROOT_CG] = imx_clk_gate3("nand_usdhc_cg", "nand_usdhc_src", base + 0x8980, 28);
- clks[IMX7D_AHB_CHANNEL_ROOT_CG] = imx_clk_gate3("ahb_cg", "ahb_src", base + 0x9000, 28);
- clks[IMX7D_DRAM_PHYM_ROOT_CG] = imx_clk_gate3("dram_phym_cg", "dram_phym_src", base + 0x9800, 28);
- clks[IMX7D_DRAM_ROOT_CG] = imx_clk_gate3("dram_cg", "dram_src", base + 0x9880, 28);
- clks[IMX7D_DRAM_PHYM_ALT_ROOT_CG] = imx_clk_gate3("dram_phym_alt_cg", "dram_phym_alt_src", base + 0xa000, 28);
- clks[IMX7D_DRAM_ALT_ROOT_CG] = imx_clk_gate3("dram_alt_cg", "dram_alt_src", base + 0xa080, 28);
- clks[IMX7D_USB_HSIC_ROOT_CG] = imx_clk_gate3("usb_hsic_cg", "usb_hsic_src", base + 0xa100, 28);
- clks[IMX7D_PCIE_CTRL_ROOT_CG] = imx_clk_gate3("pcie_ctrl_cg", "pcie_ctrl_src", base + 0xa180, 28);
- clks[IMX7D_PCIE_PHY_ROOT_CG] = imx_clk_gate3("pcie_phy_cg", "pcie_phy_src", base + 0xa200, 28);
- clks[IMX7D_EPDC_PIXEL_ROOT_CG] = imx_clk_gate3("epdc_pixel_cg", "epdc_pixel_src", base + 0xa280, 28);
- clks[IMX7D_LCDIF_PIXEL_ROOT_CG] = imx_clk_gate3("lcdif_pixel_cg", "lcdif_pixel_src", base + 0xa300, 28);
- clks[IMX7D_MIPI_DSI_ROOT_CG] = imx_clk_gate3("mipi_dsi_cg", "mipi_dsi_src", base + 0xa380, 28);
- clks[IMX7D_MIPI_CSI_ROOT_CG] = imx_clk_gate3("mipi_csi_cg", "mipi_csi_src", base + 0xa400, 28);
- clks[IMX7D_MIPI_DPHY_ROOT_CG] = imx_clk_gate3("mipi_dphy_cg", "mipi_dphy_src", base + 0xa480, 28);
- clks[IMX7D_SAI1_ROOT_CG] = imx_clk_gate3("sai1_cg", "sai1_src", base + 0xa500, 28);
- clks[IMX7D_SAI2_ROOT_CG] = imx_clk_gate3("sai2_cg", "sai2_src", base + 0xa580, 28);
- clks[IMX7D_SAI3_ROOT_CG] = imx_clk_gate3("sai3_cg", "sai3_src", base + 0xa600, 28);
- clks[IMX7D_SPDIF_ROOT_CG] = imx_clk_gate3("spdif_cg", "spdif_src", base + 0xa680, 28);
- clks[IMX7D_ENET1_REF_ROOT_CG] = imx_clk_gate3("enet1_ref_cg", "enet1_ref_src", base + 0xa700, 28);
- clks[IMX7D_ENET1_TIME_ROOT_CG] = imx_clk_gate3("enet1_time_cg", "enet1_time_src", base + 0xa780, 28);
- clks[IMX7D_ENET2_REF_ROOT_CG] = imx_clk_gate3("enet2_ref_cg", "enet2_ref_src", base + 0xa800, 28);
- clks[IMX7D_ENET2_TIME_ROOT_CG] = imx_clk_gate3("enet2_time_cg", "enet2_time_src", base + 0xa880, 28);
- clks[IMX7D_ENET_PHY_REF_ROOT_CG] = imx_clk_gate3("enet_phy_ref_cg", "enet_phy_ref_src", base + 0xa900, 28);
- clks[IMX7D_EIM_ROOT_CG] = imx_clk_gate3("eim_cg", "eim_src", base + 0xa980, 28);
- clks[IMX7D_NAND_ROOT_CG] = imx_clk_gate3("nand_cg", "nand_src", base + 0xaa00, 28);
- clks[IMX7D_QSPI_ROOT_CG] = imx_clk_gate3("qspi_cg", "qspi_src", base + 0xaa80, 28);
- clks[IMX7D_USDHC1_ROOT_CG] = imx_clk_gate3("usdhc1_cg", "usdhc1_src", base + 0xab00, 28);
- clks[IMX7D_USDHC2_ROOT_CG] = imx_clk_gate3("usdhc2_cg", "usdhc2_src", base + 0xab80, 28);
- clks[IMX7D_USDHC3_ROOT_CG] = imx_clk_gate3("usdhc3_cg", "usdhc3_src", base + 0xac00, 28);
- clks[IMX7D_CAN1_ROOT_CG] = imx_clk_gate3("can1_cg", "can1_src", base + 0xac80, 28);
- clks[IMX7D_CAN2_ROOT_CG] = imx_clk_gate3("can2_cg", "can2_src", base + 0xad00, 28);
- clks[IMX7D_I2C1_ROOT_CG] = imx_clk_gate3("i2c1_cg", "i2c1_src", base + 0xad80, 28);
- clks[IMX7D_I2C2_ROOT_CG] = imx_clk_gate3("i2c2_cg", "i2c2_src", base + 0xae00, 28);
- clks[IMX7D_I2C3_ROOT_CG] = imx_clk_gate3("i2c3_cg", "i2c3_src", base + 0xae80, 28);
- clks[IMX7D_I2C4_ROOT_CG] = imx_clk_gate3("i2c4_cg", "i2c4_src", base + 0xaf00, 28);
- clks[IMX7D_UART1_ROOT_CG] = imx_clk_gate3("uart1_cg", "uart1_src", base + 0xaf80, 28);
- clks[IMX7D_UART2_ROOT_CG] = imx_clk_gate3("uart2_cg", "uart2_src", base + 0xb000, 28);
- clks[IMX7D_UART3_ROOT_CG] = imx_clk_gate3("uart3_cg", "uart3_src", base + 0xb080, 28);
- clks[IMX7D_UART4_ROOT_CG] = imx_clk_gate3("uart4_cg", "uart4_src", base + 0xb100, 28);
- clks[IMX7D_UART5_ROOT_CG] = imx_clk_gate3("uart5_cg", "uart5_src", base + 0xb180, 28);
- clks[IMX7D_UART6_ROOT_CG] = imx_clk_gate3("uart6_cg", "uart6_src", base + 0xb200, 28);
- clks[IMX7D_UART7_ROOT_CG] = imx_clk_gate3("uart7_cg", "uart7_src", base + 0xb280, 28);
- clks[IMX7D_ECSPI1_ROOT_CG] = imx_clk_gate3("ecspi1_cg", "ecspi1_src", base + 0xb300, 28);
- clks[IMX7D_ECSPI2_ROOT_CG] = imx_clk_gate3("ecspi2_cg", "ecspi2_src", base + 0xb380, 28);
- clks[IMX7D_ECSPI3_ROOT_CG] = imx_clk_gate3("ecspi3_cg", "ecspi3_src", base + 0xb400, 28);
- clks[IMX7D_ECSPI4_ROOT_CG] = imx_clk_gate3("ecspi4_cg", "ecspi4_src", base + 0xb480, 28);
- clks[IMX7D_PWM1_ROOT_CG] = imx_clk_gate3("pwm1_cg", "pwm1_src", base + 0xb500, 28);
- clks[IMX7D_PWM2_ROOT_CG] = imx_clk_gate3("pwm2_cg", "pwm2_src", base + 0xb580, 28);
- clks[IMX7D_PWM3_ROOT_CG] = imx_clk_gate3("pwm3_cg", "pwm3_src", base + 0xb600, 28);
- clks[IMX7D_PWM4_ROOT_CG] = imx_clk_gate3("pwm4_cg", "pwm4_src", base + 0xb680, 28);
- clks[IMX7D_FLEXTIMER1_ROOT_CG] = imx_clk_gate3("flextimer1_cg", "flextimer1_src", base + 0xb700, 28);
- clks[IMX7D_FLEXTIMER2_ROOT_CG] = imx_clk_gate3("flextimer2_cg", "flextimer2_src", base + 0xb780, 28);
- clks[IMX7D_SIM1_ROOT_CG] = imx_clk_gate3("sim1_cg", "sim1_src", base + 0xb800, 28);
- clks[IMX7D_SIM2_ROOT_CG] = imx_clk_gate3("sim2_cg", "sim2_src", base + 0xb880, 28);
- clks[IMX7D_GPT1_ROOT_CG] = imx_clk_gate3("gpt1_cg", "gpt1_src", base + 0xb900, 28);
- clks[IMX7D_GPT2_ROOT_CG] = imx_clk_gate3("gpt2_cg", "gpt2_src", base + 0xb980, 28);
- clks[IMX7D_GPT3_ROOT_CG] = imx_clk_gate3("gpt3_cg", "gpt3_src", base + 0xbA00, 28);
- clks[IMX7D_GPT4_ROOT_CG] = imx_clk_gate3("gpt4_cg", "gpt4_src", base + 0xbA80, 28);
- clks[IMX7D_TRACE_ROOT_CG] = imx_clk_gate3("trace_cg", "trace_src", base + 0xbb00, 28);
- clks[IMX7D_WDOG_ROOT_CG] = imx_clk_gate3("wdog_cg", "wdog_src", base + 0xbb80, 28);
- clks[IMX7D_CSI_MCLK_ROOT_CG] = imx_clk_gate3("csi_mclk_cg", "csi_mclk_src", base + 0xbc00, 28);
- clks[IMX7D_AUDIO_MCLK_ROOT_CG] = imx_clk_gate3("audio_mclk_cg", "audio_mclk_src", base + 0xbc80, 28);
- clks[IMX7D_WRCLK_ROOT_CG] = imx_clk_gate3("wrclk_cg", "wrclk_src", base + 0xbd00, 28);
- clks[IMX7D_CLKO1_ROOT_CG] = imx_clk_gate3("clko1_cg", "clko1_src", base + 0xbd80, 28);
- clks[IMX7D_CLKO2_ROOT_CG] = imx_clk_gate3("clko2_cg", "clko2_src", base + 0xbe00, 28);
-
- clks[IMX7D_MAIN_AXI_ROOT_PRE_DIV] = imx_clk_divider2("axi_pre_div", "axi_cg", base + 0x8800, 16, 3);
- clks[IMX7D_DISP_AXI_ROOT_PRE_DIV] = imx_clk_divider2("disp_axi_pre_div", "disp_axi_cg", base + 0x8880, 16, 3);
- clks[IMX7D_ENET_AXI_ROOT_PRE_DIV] = imx_clk_divider2("enet_axi_pre_div", "enet_axi_cg", base + 0x8900, 16, 3);
- clks[IMX7D_NAND_USDHC_BUS_ROOT_PRE_DIV] = imx_clk_divider2("nand_usdhc_pre_div", "nand_usdhc_cg", base + 0x8980, 16, 3);
- clks[IMX7D_AHB_CHANNEL_ROOT_PRE_DIV] = imx_clk_divider2("ahb_pre_div", "ahb_cg", base + 0x9000, 16, 3);
- clks[IMX7D_DRAM_PHYM_ALT_ROOT_PRE_DIV] = imx_clk_divider2("dram_phym_alt_pre_div", "dram_phym_alt_cg", base + 0xa000, 16, 3);
- clks[IMX7D_DRAM_ALT_ROOT_PRE_DIV] = imx_clk_divider2("dram_alt_pre_div", "dram_alt_cg", base + 0xa080, 16, 3);
- clks[IMX7D_USB_HSIC_ROOT_PRE_DIV] = imx_clk_divider2("usb_hsic_pre_div", "usb_hsic_cg", base + 0xa100, 16, 3);
- clks[IMX7D_PCIE_CTRL_ROOT_PRE_DIV] = imx_clk_divider2("pcie_ctrl_pre_div", "pcie_ctrl_cg", base + 0xa180, 16, 3);
- clks[IMX7D_PCIE_PHY_ROOT_PRE_DIV] = imx_clk_divider2("pcie_phy_pre_div", "pcie_phy_cg", base + 0xa200, 16, 3);
- clks[IMX7D_EPDC_PIXEL_ROOT_PRE_DIV] = imx_clk_divider2("epdc_pixel_pre_div", "epdc_pixel_cg", base + 0xa280, 16, 3);
- clks[IMX7D_LCDIF_PIXEL_ROOT_PRE_DIV] = imx_clk_divider2("lcdif_pixel_pre_div", "lcdif_pixel_cg", base + 0xa300, 16, 3);
- clks[IMX7D_MIPI_DSI_ROOT_PRE_DIV] = imx_clk_divider2("mipi_dsi_pre_div", "mipi_dsi_cg", base + 0xa380, 16, 3);
- clks[IMX7D_MIPI_CSI_ROOT_PRE_DIV] = imx_clk_divider2("mipi_csi_pre_div", "mipi_csi_cg", base + 0xa400, 16, 3);
- clks[IMX7D_MIPI_DPHY_ROOT_PRE_DIV] = imx_clk_divider2("mipi_dphy_pre_div", "mipi_dphy_cg", base + 0xa480, 16, 3);
- clks[IMX7D_SAI1_ROOT_PRE_DIV] = imx_clk_divider2("sai1_pre_div", "sai1_cg", base + 0xa500, 16, 3);
- clks[IMX7D_SAI2_ROOT_PRE_DIV] = imx_clk_divider2("sai2_pre_div", "sai2_cg", base + 0xa580, 16, 3);
- clks[IMX7D_SAI3_ROOT_PRE_DIV] = imx_clk_divider2("sai3_pre_div", "sai3_cg", base + 0xa600, 16, 3);
- clks[IMX7D_SPDIF_ROOT_PRE_DIV] = imx_clk_divider2("spdif_pre_div", "spdif_cg", base + 0xa680, 16, 3);
- clks[IMX7D_ENET1_REF_ROOT_PRE_DIV] = imx_clk_divider2("enet1_ref_pre_div", "enet1_ref_cg", base + 0xa700, 16, 3);
- clks[IMX7D_ENET1_TIME_ROOT_PRE_DIV] = imx_clk_divider2("enet1_time_pre_div", "enet1_time_cg", base + 0xa780, 16, 3);
- clks[IMX7D_ENET2_REF_ROOT_PRE_DIV] = imx_clk_divider2("enet2_ref_pre_div", "enet2_ref_cg", base + 0xa800, 16, 3);
- clks[IMX7D_ENET2_TIME_ROOT_PRE_DIV] = imx_clk_divider2("enet2_time_pre_div", "enet2_time_cg", base + 0xa880, 16, 3);
- clks[IMX7D_ENET_PHY_REF_ROOT_PRE_DIV] = imx_clk_divider2("enet_phy_ref_pre_div", "enet_phy_ref_cg", base + 0xa900, 16, 3);
- clks[IMX7D_EIM_ROOT_PRE_DIV] = imx_clk_divider2("eim_pre_div", "eim_cg", base + 0xa980, 16, 3);
- clks[IMX7D_NAND_ROOT_PRE_DIV] = imx_clk_divider2("nand_pre_div", "nand_cg", base + 0xaa00, 16, 3);
- clks[IMX7D_QSPI_ROOT_PRE_DIV] = imx_clk_divider2("qspi_pre_div", "qspi_cg", base + 0xaa80, 16, 3);
- clks[IMX7D_USDHC1_ROOT_PRE_DIV] = imx_clk_divider2("usdhc1_pre_div", "usdhc1_cg", base + 0xab00, 16, 3);
- clks[IMX7D_USDHC2_ROOT_PRE_DIV] = imx_clk_divider2("usdhc2_pre_div", "usdhc2_cg", base + 0xab80, 16, 3);
- clks[IMX7D_USDHC3_ROOT_PRE_DIV] = imx_clk_divider2("usdhc3_pre_div", "usdhc3_cg", base + 0xac00, 16, 3);
- clks[IMX7D_CAN1_ROOT_PRE_DIV] = imx_clk_divider2("can1_pre_div", "can1_cg", base + 0xac80, 16, 3);
- clks[IMX7D_CAN2_ROOT_PRE_DIV] = imx_clk_divider2("can2_pre_div", "can2_cg", base + 0xad00, 16, 3);
- clks[IMX7D_I2C1_ROOT_PRE_DIV] = imx_clk_divider2("i2c1_pre_div", "i2c1_cg", base + 0xad80, 16, 3);
- clks[IMX7D_I2C2_ROOT_PRE_DIV] = imx_clk_divider2("i2c2_pre_div", "i2c2_cg", base + 0xae00, 16, 3);
- clks[IMX7D_I2C3_ROOT_PRE_DIV] = imx_clk_divider2("i2c3_pre_div", "i2c3_cg", base + 0xae80, 16, 3);
- clks[IMX7D_I2C4_ROOT_PRE_DIV] = imx_clk_divider2("i2c4_pre_div", "i2c4_cg", base + 0xaf00, 16, 3);
- clks[IMX7D_UART1_ROOT_PRE_DIV] = imx_clk_divider2("uart1_pre_div", "uart1_cg", base + 0xaf80, 16, 3);
- clks[IMX7D_UART2_ROOT_PRE_DIV] = imx_clk_divider2("uart2_pre_div", "uart2_cg", base + 0xb000, 16, 3);
- clks[IMX7D_UART3_ROOT_PRE_DIV] = imx_clk_divider2("uart3_pre_div", "uart3_cg", base + 0xb080, 16, 3);
- clks[IMX7D_UART4_ROOT_PRE_DIV] = imx_clk_divider2("uart4_pre_div", "uart4_cg", base + 0xb100, 16, 3);
- clks[IMX7D_UART5_ROOT_PRE_DIV] = imx_clk_divider2("uart5_pre_div", "uart5_cg", base + 0xb180, 16, 3);
- clks[IMX7D_UART6_ROOT_PRE_DIV] = imx_clk_divider2("uart6_pre_div", "uart6_cg", base + 0xb200, 16, 3);
- clks[IMX7D_UART7_ROOT_PRE_DIV] = imx_clk_divider2("uart7_pre_div", "uart7_cg", base + 0xb280, 16, 3);
- clks[IMX7D_ECSPI1_ROOT_PRE_DIV] = imx_clk_divider2("ecspi1_pre_div", "ecspi1_cg", base + 0xb300, 16, 3);
- clks[IMX7D_ECSPI2_ROOT_PRE_DIV] = imx_clk_divider2("ecspi2_pre_div", "ecspi2_cg", base + 0xb380, 16, 3);
- clks[IMX7D_ECSPI3_ROOT_PRE_DIV] = imx_clk_divider2("ecspi3_pre_div", "ecspi3_cg", base + 0xb400, 16, 3);
- clks[IMX7D_ECSPI4_ROOT_PRE_DIV] = imx_clk_divider2("ecspi4_pre_div", "ecspi4_cg", base + 0xb480, 16, 3);
- clks[IMX7D_PWM1_ROOT_PRE_DIV] = imx_clk_divider2("pwm1_pre_div", "pwm1_cg", base + 0xb500, 16, 3);
- clks[IMX7D_PWM2_ROOT_PRE_DIV] = imx_clk_divider2("pwm2_pre_div", "pwm2_cg", base + 0xb580, 16, 3);
- clks[IMX7D_PWM3_ROOT_PRE_DIV] = imx_clk_divider2("pwm3_pre_div", "pwm3_cg", base + 0xb600, 16, 3);
- clks[IMX7D_PWM4_ROOT_PRE_DIV] = imx_clk_divider2("pwm4_pre_div", "pwm4_cg", base + 0xb680, 16, 3);
- clks[IMX7D_FLEXTIMER1_ROOT_PRE_DIV] = imx_clk_divider2("flextimer1_pre_div", "flextimer1_cg", base + 0xb700, 16, 3);
- clks[IMX7D_FLEXTIMER2_ROOT_PRE_DIV] = imx_clk_divider2("flextimer2_pre_div", "flextimer2_cg", base + 0xb780, 16, 3);
- clks[IMX7D_SIM1_ROOT_PRE_DIV] = imx_clk_divider2("sim1_pre_div", "sim1_cg", base + 0xb800, 16, 3);
- clks[IMX7D_SIM2_ROOT_PRE_DIV] = imx_clk_divider2("sim2_pre_div", "sim2_cg", base + 0xb880, 16, 3);
- clks[IMX7D_GPT1_ROOT_PRE_DIV] = imx_clk_divider2("gpt1_pre_div", "gpt1_cg", base + 0xb900, 16, 3);
- clks[IMX7D_GPT2_ROOT_PRE_DIV] = imx_clk_divider2("gpt2_pre_div", "gpt2_cg", base + 0xb980, 16, 3);
- clks[IMX7D_GPT3_ROOT_PRE_DIV] = imx_clk_divider2("gpt3_pre_div", "gpt3_cg", base + 0xba00, 16, 3);
- clks[IMX7D_GPT4_ROOT_PRE_DIV] = imx_clk_divider2("gpt4_pre_div", "gpt4_cg", base + 0xba80, 16, 3);
- clks[IMX7D_TRACE_ROOT_PRE_DIV] = imx_clk_divider2("trace_pre_div", "trace_cg", base + 0xbb00, 16, 3);
- clks[IMX7D_WDOG_ROOT_PRE_DIV] = imx_clk_divider2("wdog_pre_div", "wdog_cg", base + 0xbb80, 16, 3);
- clks[IMX7D_CSI_MCLK_ROOT_PRE_DIV] = imx_clk_divider2("csi_mclk_pre_div", "csi_mclk_cg", base + 0xbc00, 16, 3);
- clks[IMX7D_AUDIO_MCLK_ROOT_PRE_DIV] = imx_clk_divider2("audio_mclk_pre_div", "audio_mclk_cg", base + 0xbc80, 16, 3);
- clks[IMX7D_WRCLK_ROOT_PRE_DIV] = imx_clk_divider2("wrclk_pre_div", "wrclk_cg", base + 0xbd00, 16, 3);
- clks[IMX7D_CLKO1_ROOT_PRE_DIV] = imx_clk_divider2("clko1_pre_div", "clko1_cg", base + 0xbd80, 16, 3);
- clks[IMX7D_CLKO2_ROOT_PRE_DIV] = imx_clk_divider2("clko2_pre_div", "clko2_cg", base + 0xbe00, 16, 3);
-
- clks[IMX7D_ARM_A7_ROOT_DIV] = imx_clk_divider2("arm_a7_div", "arm_a7_cg", base + 0x8000, 0, 3);
- clks[IMX7D_ARM_M4_ROOT_DIV] = imx_clk_divider2("arm_m4_div", "arm_m4_cg", base + 0x8080, 0, 3);
- clks[IMX7D_MAIN_AXI_ROOT_DIV] = imx_clk_divider2("axi_post_div", "axi_pre_div", base + 0x8800, 0, 6);
- clks[IMX7D_DISP_AXI_ROOT_DIV] = imx_clk_divider2("disp_axi_post_div", "disp_axi_pre_div", base + 0x8880, 0, 6);
- clks[IMX7D_ENET_AXI_ROOT_DIV] = imx_clk_divider2("enet_axi_post_div", "enet_axi_pre_div", base + 0x8900, 0, 6);
- clks[IMX7D_NAND_USDHC_BUS_ROOT_CLK] = imx_clk_divider2("nand_usdhc_root_clk", "nand_usdhc_pre_div", base + 0x8980, 0, 6);
- clks[IMX7D_AHB_CHANNEL_ROOT_DIV] = imx_clk_divider2("ahb_root_clk", "ahb_pre_div", base + 0x9000, 0, 6);
- clks[IMX7D_IPG_ROOT_CLK] = imx_clk_divider_flags("ipg_root_clk", "ahb_root_clk", base + 0x9080, 0, 2, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE | CLK_SET_RATE_PARENT);
- clks[IMX7D_DRAM_ROOT_DIV] = imx_clk_divider2("dram_post_div", "dram_cg", base + 0x9880, 0, 3);
- clks[IMX7D_DRAM_PHYM_ALT_ROOT_DIV] = imx_clk_divider2("dram_phym_alt_post_div", "dram_phym_alt_pre_div", base + 0xa000, 0, 3);
- clks[IMX7D_DRAM_ALT_ROOT_DIV] = imx_clk_divider2("dram_alt_post_div", "dram_alt_pre_div", base + 0xa080, 0, 3);
- clks[IMX7D_USB_HSIC_ROOT_DIV] = imx_clk_divider2("usb_hsic_post_div", "usb_hsic_pre_div", base + 0xa100, 0, 6);
- clks[IMX7D_PCIE_CTRL_ROOT_DIV] = imx_clk_divider2("pcie_ctrl_post_div", "pcie_ctrl_pre_div", base + 0xa180, 0, 6);
- clks[IMX7D_PCIE_PHY_ROOT_DIV] = imx_clk_divider2("pcie_phy_post_div", "pcie_phy_pre_div", base + 0xa200, 0, 6);
- clks[IMX7D_EPDC_PIXEL_ROOT_DIV] = imx_clk_divider2("epdc_pixel_post_div", "epdc_pixel_pre_div", base + 0xa280, 0, 6);
- clks[IMX7D_LCDIF_PIXEL_ROOT_DIV] = imx_clk_divider2("lcdif_pixel_post_div", "lcdif_pixel_pre_div", base + 0xa300, 0, 6);
- clks[IMX7D_MIPI_DSI_ROOT_DIV] = imx_clk_divider2("mipi_dsi_post_div", "mipi_dsi_pre_div", base + 0xa380, 0, 6);
- clks[IMX7D_MIPI_CSI_ROOT_DIV] = imx_clk_divider2("mipi_csi_post_div", "mipi_csi_pre_div", base + 0xa400, 0, 6);
- clks[IMX7D_MIPI_DPHY_ROOT_DIV] = imx_clk_divider2("mipi_dphy_post_div", "mipi_dphy_pre_div", base + 0xa480, 0, 6);
- clks[IMX7D_SAI1_ROOT_DIV] = imx_clk_divider2("sai1_post_div", "sai1_pre_div", base + 0xa500, 0, 6);
- clks[IMX7D_SAI2_ROOT_DIV] = imx_clk_divider2("sai2_post_div", "sai2_pre_div", base + 0xa580, 0, 6);
- clks[IMX7D_SAI3_ROOT_DIV] = imx_clk_divider2("sai3_post_div", "sai3_pre_div", base + 0xa600, 0, 6);
- clks[IMX7D_SPDIF_ROOT_DIV] = imx_clk_divider2("spdif_post_div", "spdif_pre_div", base + 0xa680, 0, 6);
- clks[IMX7D_ENET1_REF_ROOT_DIV] = imx_clk_divider2("enet1_ref_post_div", "enet1_ref_pre_div", base + 0xa700, 0, 6);
- clks[IMX7D_ENET1_TIME_ROOT_DIV] = imx_clk_divider2("enet1_time_post_div", "enet1_time_pre_div", base + 0xa780, 0, 6);
- clks[IMX7D_ENET2_REF_ROOT_DIV] = imx_clk_divider2("enet2_ref_post_div", "enet2_ref_pre_div", base + 0xa800, 0, 6);
- clks[IMX7D_ENET2_TIME_ROOT_DIV] = imx_clk_divider2("enet2_time_post_div", "enet2_time_pre_div", base + 0xa880, 0, 6);
- clks[IMX7D_ENET_PHY_REF_ROOT_CLK] = imx_clk_divider2("enet_phy_ref_root_clk", "enet_phy_ref_pre_div", base + 0xa900, 0, 6);
- clks[IMX7D_EIM_ROOT_DIV] = imx_clk_divider2("eim_post_div", "eim_pre_div", base + 0xa980, 0, 6);
- clks[IMX7D_NAND_ROOT_CLK] = imx_clk_divider2("nand_root_clk", "nand_pre_div", base + 0xaa00, 0, 6);
- clks[IMX7D_QSPI_ROOT_DIV] = imx_clk_divider2("qspi_post_div", "qspi_pre_div", base + 0xaa80, 0, 6);
- clks[IMX7D_USDHC1_ROOT_DIV] = imx_clk_divider2("usdhc1_post_div", "usdhc1_pre_div", base + 0xab00, 0, 6);
- clks[IMX7D_USDHC2_ROOT_DIV] = imx_clk_divider2("usdhc2_post_div", "usdhc2_pre_div", base + 0xab80, 0, 6);
- clks[IMX7D_USDHC3_ROOT_DIV] = imx_clk_divider2("usdhc3_post_div", "usdhc3_pre_div", base + 0xac00, 0, 6);
- clks[IMX7D_CAN1_ROOT_DIV] = imx_clk_divider2("can1_post_div", "can1_pre_div", base + 0xac80, 0, 6);
- clks[IMX7D_CAN2_ROOT_DIV] = imx_clk_divider2("can2_post_div", "can2_pre_div", base + 0xad00, 0, 6);
- clks[IMX7D_I2C1_ROOT_DIV] = imx_clk_divider2("i2c1_post_div", "i2c1_pre_div", base + 0xad80, 0, 6);
- clks[IMX7D_I2C2_ROOT_DIV] = imx_clk_divider2("i2c2_post_div", "i2c2_pre_div", base + 0xae00, 0, 6);
- clks[IMX7D_I2C3_ROOT_DIV] = imx_clk_divider2("i2c3_post_div", "i2c3_pre_div", base + 0xae80, 0, 6);
- clks[IMX7D_I2C4_ROOT_DIV] = imx_clk_divider2("i2c4_post_div", "i2c4_pre_div", base + 0xaf00, 0, 6);
- clks[IMX7D_UART1_ROOT_DIV] = imx_clk_divider2("uart1_post_div", "uart1_pre_div", base + 0xaf80, 0, 6);
- clks[IMX7D_UART2_ROOT_DIV] = imx_clk_divider2("uart2_post_div", "uart2_pre_div", base + 0xb000, 0, 6);
- clks[IMX7D_UART3_ROOT_DIV] = imx_clk_divider2("uart3_post_div", "uart3_pre_div", base + 0xb080, 0, 6);
- clks[IMX7D_UART4_ROOT_DIV] = imx_clk_divider2("uart4_post_div", "uart4_pre_div", base + 0xb100, 0, 6);
- clks[IMX7D_UART5_ROOT_DIV] = imx_clk_divider2("uart5_post_div", "uart5_pre_div", base + 0xb180, 0, 6);
- clks[IMX7D_UART6_ROOT_DIV] = imx_clk_divider2("uart6_post_div", "uart6_pre_div", base + 0xb200, 0, 6);
- clks[IMX7D_UART7_ROOT_DIV] = imx_clk_divider2("uart7_post_div", "uart7_pre_div", base + 0xb280, 0, 6);
- clks[IMX7D_ECSPI1_ROOT_DIV] = imx_clk_divider2("ecspi1_post_div", "ecspi1_pre_div", base + 0xb300, 0, 6);
- clks[IMX7D_ECSPI2_ROOT_DIV] = imx_clk_divider2("ecspi2_post_div", "ecspi2_pre_div", base + 0xb380, 0, 6);
- clks[IMX7D_ECSPI3_ROOT_DIV] = imx_clk_divider2("ecspi3_post_div", "ecspi3_pre_div", base + 0xb400, 0, 6);
- clks[IMX7D_ECSPI4_ROOT_DIV] = imx_clk_divider2("ecspi4_post_div", "ecspi4_pre_div", base + 0xb480, 0, 6);
- clks[IMX7D_PWM1_ROOT_DIV] = imx_clk_divider2("pwm1_post_div", "pwm1_pre_div", base + 0xb500, 0, 6);
- clks[IMX7D_PWM2_ROOT_DIV] = imx_clk_divider2("pwm2_post_div", "pwm2_pre_div", base + 0xb580, 0, 6);
- clks[IMX7D_PWM3_ROOT_DIV] = imx_clk_divider2("pwm3_post_div", "pwm3_pre_div", base + 0xb600, 0, 6);
- clks[IMX7D_PWM4_ROOT_DIV] = imx_clk_divider2("pwm4_post_div", "pwm4_pre_div", base + 0xb680, 0, 6);
- clks[IMX7D_FLEXTIMER1_ROOT_DIV] = imx_clk_divider2("flextimer1_post_div", "flextimer1_pre_div", base + 0xb700, 0, 6);
- clks[IMX7D_FLEXTIMER2_ROOT_DIV] = imx_clk_divider2("flextimer2_post_div", "flextimer2_pre_div", base + 0xb780, 0, 6);
- clks[IMX7D_SIM1_ROOT_DIV] = imx_clk_divider2("sim1_post_div", "sim1_pre_div", base + 0xb800, 0, 6);
- clks[IMX7D_SIM2_ROOT_DIV] = imx_clk_divider2("sim2_post_div", "sim2_pre_div", base + 0xb880, 0, 6);
- clks[IMX7D_GPT1_ROOT_DIV] = imx_clk_divider2("gpt1_post_div", "gpt1_pre_div", base + 0xb900, 0, 6);
- clks[IMX7D_GPT2_ROOT_DIV] = imx_clk_divider2("gpt2_post_div", "gpt2_pre_div", base + 0xb980, 0, 6);
- clks[IMX7D_GPT3_ROOT_DIV] = imx_clk_divider2("gpt3_post_div", "gpt3_pre_div", base + 0xba00, 0, 6);
- clks[IMX7D_GPT4_ROOT_DIV] = imx_clk_divider2("gpt4_post_div", "gpt4_pre_div", base + 0xba80, 0, 6);
- clks[IMX7D_TRACE_ROOT_DIV] = imx_clk_divider2("trace_post_div", "trace_pre_div", base + 0xbb00, 0, 6);
- clks[IMX7D_WDOG_ROOT_DIV] = imx_clk_divider2("wdog_post_div", "wdog_pre_div", base + 0xbb80, 0, 6);
- clks[IMX7D_CSI_MCLK_ROOT_DIV] = imx_clk_divider2("csi_mclk_post_div", "csi_mclk_pre_div", base + 0xbc00, 0, 6);
- clks[IMX7D_AUDIO_MCLK_ROOT_DIV] = imx_clk_divider2("audio_mclk_post_div", "audio_mclk_pre_div", base + 0xbc80, 0, 6);
- clks[IMX7D_WRCLK_ROOT_DIV] = imx_clk_divider2("wrclk_post_div", "wrclk_pre_div", base + 0xbd00, 0, 6);
- clks[IMX7D_CLKO1_ROOT_DIV] = imx_clk_divider2("clko1_post_div", "clko1_pre_div", base + 0xbd80, 0, 6);
- clks[IMX7D_CLKO2_ROOT_DIV] = imx_clk_divider2("clko2_post_div", "clko2_pre_div", base + 0xbe00, 0, 6);
-
- clks[IMX7D_ARM_A7_ROOT_CLK] = imx_clk_gate2_flags("arm_a7_root_clk", "arm_a7_div", base + 0x4000, 0, CLK_OPS_PARENT_ENABLE);
- clks[IMX7D_ARM_M4_ROOT_CLK] = imx_clk_gate4("arm_m4_root_clk", "arm_m4_div", base + 0x4010, 0);
- clks[IMX7D_MAIN_AXI_ROOT_CLK] = imx_clk_gate2_flags("main_axi_root_clk", "axi_post_div", base + 0x4040, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE);
- clks[IMX7D_DISP_AXI_ROOT_CLK] = imx_clk_gate4("disp_axi_root_clk", "disp_axi_post_div", base + 0x4050, 0);
- clks[IMX7D_ENET_AXI_ROOT_CLK] = imx_clk_gate4("enet_axi_root_clk", "enet_axi_post_div", base + 0x4060, 0);
- clks[IMX7D_OCRAM_CLK] = imx_clk_gate4("ocram_clk", "main_axi_root_clk", base + 0x4110, 0);
- clks[IMX7D_OCRAM_S_CLK] = imx_clk_gate4("ocram_s_clk", "ahb_root_clk", base + 0x4120, 0);
- clks[IMX7D_DRAM_ROOT_CLK] = imx_clk_gate2_flags("dram_root_clk", "dram_post_div", base + 0x4130, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE);
- clks[IMX7D_DRAM_PHYM_ROOT_CLK] = imx_clk_gate2_flags("dram_phym_root_clk", "dram_phym_cg", base + 0x4130, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE);
- clks[IMX7D_DRAM_PHYM_ALT_ROOT_CLK] = imx_clk_gate2_flags("dram_phym_alt_root_clk", "dram_phym_alt_post_div", base + 0x4130, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE);
- clks[IMX7D_DRAM_ALT_ROOT_CLK] = imx_clk_gate2_flags("dram_alt_root_clk", "dram_alt_post_div", base + 0x4130, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE);
- clks[IMX7D_OCOTP_CLK] = imx_clk_gate4("ocotp_clk", "ipg_root_clk", base + 0x4230, 0);
- clks[IMX7D_SNVS_CLK] = imx_clk_gate4("snvs_clk", "ipg_root_clk", base + 0x4250, 0);
- clks[IMX7D_MU_ROOT_CLK] = imx_clk_gate4("mu_root_clk", "ipg_root_clk", base + 0x4270, 0);
- clks[IMX7D_CAAM_CLK] = imx_clk_gate4("caam_clk", "ipg_root_clk", base + 0x4240, 0);
- clks[IMX7D_USB_HSIC_ROOT_CLK] = imx_clk_gate4("usb_hsic_root_clk", "usb_hsic_post_div", base + 0x4690, 0);
- clks[IMX7D_SDMA_CORE_CLK] = imx_clk_gate4("sdma_root_clk", "ahb_root_clk", base + 0x4480, 0);
- clks[IMX7D_PCIE_CTRL_ROOT_CLK] = imx_clk_gate4("pcie_ctrl_root_clk", "pcie_ctrl_post_div", base + 0x4600, 0);
- clks[IMX7D_PCIE_PHY_ROOT_CLK] = imx_clk_gate4("pcie_phy_root_clk", "pcie_phy_post_div", base + 0x4600, 0);
- clks[IMX7D_EPDC_PIXEL_ROOT_CLK] = imx_clk_gate4("epdc_pixel_root_clk", "epdc_pixel_post_div", base + 0x44a0, 0);
- clks[IMX7D_LCDIF_PIXEL_ROOT_CLK] = imx_clk_gate4("lcdif_pixel_root_clk", "lcdif_pixel_post_div", base + 0x44b0, 0);
- clks[IMX7D_MIPI_DSI_ROOT_CLK] = imx_clk_gate4("mipi_dsi_root_clk", "mipi_dsi_post_div", base + 0x4650, 0);
- clks[IMX7D_MIPI_CSI_ROOT_CLK] = imx_clk_gate4("mipi_csi_root_clk", "mipi_csi_post_div", base + 0x4640, 0);
- clks[IMX7D_MIPI_DPHY_ROOT_CLK] = imx_clk_gate4("mipi_dphy_root_clk", "mipi_dphy_post_div", base + 0x4660, 0);
- clks[IMX7D_ENET1_IPG_ROOT_CLK] = imx_clk_gate2_shared2("enet1_ipg_root_clk", "enet_axi_post_div", base + 0x4700, 0, &share_count_enet1);
- clks[IMX7D_ENET1_TIME_ROOT_CLK] = imx_clk_gate2_shared2("enet1_time_root_clk", "enet1_time_post_div", base + 0x4700, 0, &share_count_enet1);
- clks[IMX7D_ENET2_IPG_ROOT_CLK] = imx_clk_gate2_shared2("enet2_ipg_root_clk", "enet_axi_post_div", base + 0x4710, 0, &share_count_enet2);
- clks[IMX7D_ENET2_TIME_ROOT_CLK] = imx_clk_gate2_shared2("enet2_time_root_clk", "enet2_time_post_div", base + 0x4710, 0, &share_count_enet2);
- clks[IMX7D_SAI1_ROOT_CLK] = imx_clk_gate2_shared2("sai1_root_clk", "sai1_post_div", base + 0x48c0, 0, &share_count_sai1);
- clks[IMX7D_SAI1_IPG_CLK] = imx_clk_gate2_shared2("sai1_ipg_clk", "ipg_root_clk", base + 0x48c0, 0, &share_count_sai1);
- clks[IMX7D_SAI2_ROOT_CLK] = imx_clk_gate2_shared2("sai2_root_clk", "sai2_post_div", base + 0x48d0, 0, &share_count_sai2);
- clks[IMX7D_SAI2_IPG_CLK] = imx_clk_gate2_shared2("sai2_ipg_clk", "ipg_root_clk", base + 0x48d0, 0, &share_count_sai2);
- clks[IMX7D_SAI3_ROOT_CLK] = imx_clk_gate2_shared2("sai3_root_clk", "sai3_post_div", base + 0x48e0, 0, &share_count_sai3);
- clks[IMX7D_SAI3_IPG_CLK] = imx_clk_gate2_shared2("sai3_ipg_clk", "ipg_root_clk", base + 0x48e0, 0, &share_count_sai3);
- clks[IMX7D_SPDIF_ROOT_CLK] = imx_clk_gate4("spdif_root_clk", "spdif_post_div", base + 0x44d0, 0);
- clks[IMX7D_EIM_ROOT_CLK] = imx_clk_gate4("eim_root_clk", "eim_post_div", base + 0x4160, 0);
- clks[IMX7D_NAND_RAWNAND_CLK] = imx_clk_gate2_shared2("nand_rawnand_clk", "nand_root_clk", base + 0x4140, 0, &share_count_nand);
- clks[IMX7D_NAND_USDHC_BUS_RAWNAND_CLK] = imx_clk_gate2_shared2("nand_usdhc_rawnand_clk", "nand_usdhc_root_clk", base + 0x4140, 0, &share_count_nand);
- clks[IMX7D_QSPI_ROOT_CLK] = imx_clk_gate4("qspi_root_clk", "qspi_post_div", base + 0x4150, 0);
- clks[IMX7D_USDHC1_ROOT_CLK] = imx_clk_gate4("usdhc1_root_clk", "usdhc1_post_div", base + 0x46c0, 0);
- clks[IMX7D_USDHC2_ROOT_CLK] = imx_clk_gate4("usdhc2_root_clk", "usdhc2_post_div", base + 0x46d0, 0);
- clks[IMX7D_USDHC3_ROOT_CLK] = imx_clk_gate4("usdhc3_root_clk", "usdhc3_post_div", base + 0x46e0, 0);
- clks[IMX7D_CAN1_ROOT_CLK] = imx_clk_gate4("can1_root_clk", "can1_post_div", base + 0x4740, 0);
- clks[IMX7D_CAN2_ROOT_CLK] = imx_clk_gate4("can2_root_clk", "can2_post_div", base + 0x4750, 0);
- clks[IMX7D_I2C1_ROOT_CLK] = imx_clk_gate4("i2c1_root_clk", "i2c1_post_div", base + 0x4880, 0);
- clks[IMX7D_I2C2_ROOT_CLK] = imx_clk_gate4("i2c2_root_clk", "i2c2_post_div", base + 0x4890, 0);
- clks[IMX7D_I2C3_ROOT_CLK] = imx_clk_gate4("i2c3_root_clk", "i2c3_post_div", base + 0x48a0, 0);
- clks[IMX7D_I2C4_ROOT_CLK] = imx_clk_gate4("i2c4_root_clk", "i2c4_post_div", base + 0x48b0, 0);
- clks[IMX7D_UART1_ROOT_CLK] = imx_clk_gate4("uart1_root_clk", "uart1_post_div", base + 0x4940, 0);
- clks[IMX7D_UART2_ROOT_CLK] = imx_clk_gate4("uart2_root_clk", "uart2_post_div", base + 0x4950, 0);
- clks[IMX7D_UART3_ROOT_CLK] = imx_clk_gate4("uart3_root_clk", "uart3_post_div", base + 0x4960, 0);
- clks[IMX7D_UART4_ROOT_CLK] = imx_clk_gate4("uart4_root_clk", "uart4_post_div", base + 0x4970, 0);
- clks[IMX7D_UART5_ROOT_CLK] = imx_clk_gate4("uart5_root_clk", "uart5_post_div", base + 0x4980, 0);
- clks[IMX7D_UART6_ROOT_CLK] = imx_clk_gate4("uart6_root_clk", "uart6_post_div", base + 0x4990, 0);
- clks[IMX7D_UART7_ROOT_CLK] = imx_clk_gate4("uart7_root_clk", "uart7_post_div", base + 0x49a0, 0);
- clks[IMX7D_ECSPI1_ROOT_CLK] = imx_clk_gate4("ecspi1_root_clk", "ecspi1_post_div", base + 0x4780, 0);
- clks[IMX7D_ECSPI2_ROOT_CLK] = imx_clk_gate4("ecspi2_root_clk", "ecspi2_post_div", base + 0x4790, 0);
- clks[IMX7D_ECSPI3_ROOT_CLK] = imx_clk_gate4("ecspi3_root_clk", "ecspi3_post_div", base + 0x47a0, 0);
- clks[IMX7D_ECSPI4_ROOT_CLK] = imx_clk_gate4("ecspi4_root_clk", "ecspi4_post_div", base + 0x47b0, 0);
- clks[IMX7D_PWM1_ROOT_CLK] = imx_clk_gate4("pwm1_root_clk", "pwm1_post_div", base + 0x4840, 0);
- clks[IMX7D_PWM2_ROOT_CLK] = imx_clk_gate4("pwm2_root_clk", "pwm2_post_div", base + 0x4850, 0);
- clks[IMX7D_PWM3_ROOT_CLK] = imx_clk_gate4("pwm3_root_clk", "pwm3_post_div", base + 0x4860, 0);
- clks[IMX7D_PWM4_ROOT_CLK] = imx_clk_gate4("pwm4_root_clk", "pwm4_post_div", base + 0x4870, 0);
- clks[IMX7D_FLEXTIMER1_ROOT_CLK] = imx_clk_gate4("flextimer1_root_clk", "flextimer1_post_div", base + 0x4800, 0);
- clks[IMX7D_FLEXTIMER2_ROOT_CLK] = imx_clk_gate4("flextimer2_root_clk", "flextimer2_post_div", base + 0x4810, 0);
- clks[IMX7D_SIM1_ROOT_CLK] = imx_clk_gate4("sim1_root_clk", "sim1_post_div", base + 0x4900, 0);
- clks[IMX7D_SIM2_ROOT_CLK] = imx_clk_gate4("sim2_root_clk", "sim2_post_div", base + 0x4910, 0);
- clks[IMX7D_GPT1_ROOT_CLK] = imx_clk_gate4("gpt1_root_clk", "gpt1_post_div", base + 0x47c0, 0);
- clks[IMX7D_GPT2_ROOT_CLK] = imx_clk_gate4("gpt2_root_clk", "gpt2_post_div", base + 0x47d0, 0);
- clks[IMX7D_GPT3_ROOT_CLK] = imx_clk_gate4("gpt3_root_clk", "gpt3_post_div", base + 0x47e0, 0);
- clks[IMX7D_GPT4_ROOT_CLK] = imx_clk_gate4("gpt4_root_clk", "gpt4_post_div", base + 0x47f0, 0);
- clks[IMX7D_TRACE_ROOT_CLK] = imx_clk_gate4("trace_root_clk", "trace_post_div", base + 0x4300, 0);
- clks[IMX7D_WDOG1_ROOT_CLK] = imx_clk_gate4("wdog1_root_clk", "wdog_post_div", base + 0x49c0, 0);
- clks[IMX7D_WDOG2_ROOT_CLK] = imx_clk_gate4("wdog2_root_clk", "wdog_post_div", base + 0x49d0, 0);
- clks[IMX7D_WDOG3_ROOT_CLK] = imx_clk_gate4("wdog3_root_clk", "wdog_post_div", base + 0x49e0, 0);
- clks[IMX7D_WDOG4_ROOT_CLK] = imx_clk_gate4("wdog4_root_clk", "wdog_post_div", base + 0x49f0, 0);
- clks[IMX7D_KPP_ROOT_CLK] = imx_clk_gate4("kpp_root_clk", "ipg_root_clk", base + 0x4aa0, 0);
- clks[IMX7D_CSI_MCLK_ROOT_CLK] = imx_clk_gate4("csi_mclk_root_clk", "csi_mclk_post_div", base + 0x4490, 0);
- clks[IMX7D_AUDIO_MCLK_ROOT_CLK] = imx_clk_gate4("audio_mclk_root_clk", "audio_mclk_post_div", base + 0x4790, 0);
- clks[IMX7D_WRCLK_ROOT_CLK] = imx_clk_gate4("wrclk_root_clk", "wrclk_post_div", base + 0x47a0, 0);
- clks[IMX7D_USB_CTRL_CLK] = imx_clk_gate4("usb_ctrl_clk", "ahb_root_clk", base + 0x4680, 0);
- clks[IMX7D_USB_PHY1_CLK] = imx_clk_gate4("usb_phy1_clk", "pll_usb1_main_clk", base + 0x46a0, 0);
- clks[IMX7D_USB_PHY2_CLK] = imx_clk_gate4("usb_phy2_clk", "pll_usb_main_clk", base + 0x46b0, 0);
- clks[IMX7D_ADC_ROOT_CLK] = imx_clk_gate4("adc_root_clk", "ipg_root_clk", base + 0x4200, 0);
-
- clks[IMX7D_GPT_3M_CLK] = imx_clk_fixed_factor("gpt_3m", "osc", 1, 8);
-
- clks[IMX7D_CLK_ARM] = imx_clk_cpu("arm", "arm_a7_root_clk",
- clks[IMX7D_ARM_A7_ROOT_CLK],
- clks[IMX7D_ARM_A7_ROOT_SRC],
- clks[IMX7D_PLL_ARM_MAIN_CLK],
- clks[IMX7D_PLL_SYS_MAIN_CLK]);
-
- imx_check_clocks(clks, ARRAY_SIZE(clks));
-
- clk_data.clks = clks;
- clk_data.clk_num = ARRAY_SIZE(clks);
- of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
-
- clk_set_parent(clks[IMX7D_PLL_ARM_MAIN_BYPASS], clks[IMX7D_PLL_ARM_MAIN]);
- clk_set_parent(clks[IMX7D_PLL_DRAM_MAIN_BYPASS], clks[IMX7D_PLL_DRAM_MAIN]);
- clk_set_parent(clks[IMX7D_PLL_SYS_MAIN_BYPASS], clks[IMX7D_PLL_SYS_MAIN]);
- clk_set_parent(clks[IMX7D_PLL_ENET_MAIN_BYPASS], clks[IMX7D_PLL_ENET_MAIN]);
- clk_set_parent(clks[IMX7D_PLL_AUDIO_MAIN_BYPASS], clks[IMX7D_PLL_AUDIO_MAIN]);
- clk_set_parent(clks[IMX7D_PLL_VIDEO_MAIN_BYPASS], clks[IMX7D_PLL_VIDEO_MAIN]);
-
- clk_set_parent(clks[IMX7D_MIPI_CSI_ROOT_SRC], clks[IMX7D_PLL_SYS_PFD3_CLK]);
+ hws[IMX7D_ARM_A7_ROOT_SRC] = imx_clk_hw_mux2("arm_a7_src", base + 0x8000, 24, 3, arm_a7_sel, ARRAY_SIZE(arm_a7_sel));
+ hws[IMX7D_ARM_M4_ROOT_SRC] = imx_clk_hw_mux2("arm_m4_src", base + 0x8080, 24, 3, arm_m4_sel, ARRAY_SIZE(arm_m4_sel));
+ hws[IMX7D_MAIN_AXI_ROOT_SRC] = imx_clk_hw_mux2("axi_src", base + 0x8800, 24, 3, axi_sel, ARRAY_SIZE(axi_sel));
+ hws[IMX7D_DISP_AXI_ROOT_SRC] = imx_clk_hw_mux2("disp_axi_src", base + 0x8880, 24, 3, disp_axi_sel, ARRAY_SIZE(disp_axi_sel));
+ hws[IMX7D_ENET_AXI_ROOT_SRC] = imx_clk_hw_mux2("enet_axi_src", base + 0x8900, 24, 3, enet_axi_sel, ARRAY_SIZE(enet_axi_sel));
+ hws[IMX7D_NAND_USDHC_BUS_ROOT_SRC] = imx_clk_hw_mux2("nand_usdhc_src", base + 0x8980, 24, 3, nand_usdhc_bus_sel, ARRAY_SIZE(nand_usdhc_bus_sel));
+ hws[IMX7D_AHB_CHANNEL_ROOT_SRC] = imx_clk_hw_mux2("ahb_src", base + 0x9000, 24, 3, ahb_channel_sel, ARRAY_SIZE(ahb_channel_sel));
+ hws[IMX7D_DRAM_PHYM_ROOT_SRC] = imx_clk_hw_mux2("dram_phym_src", base + 0x9800, 24, 1, dram_phym_sel, ARRAY_SIZE(dram_phym_sel));
+ hws[IMX7D_DRAM_ROOT_SRC] = imx_clk_hw_mux2("dram_src", base + 0x9880, 24, 1, dram_sel, ARRAY_SIZE(dram_sel));
+ hws[IMX7D_DRAM_PHYM_ALT_ROOT_SRC] = imx_clk_hw_mux2("dram_phym_alt_src", base + 0xa000, 24, 3, dram_phym_alt_sel, ARRAY_SIZE(dram_phym_alt_sel));
+ hws[IMX7D_DRAM_ALT_ROOT_SRC] = imx_clk_hw_mux2("dram_alt_src", base + 0xa080, 24, 3, dram_alt_sel, ARRAY_SIZE(dram_alt_sel));
+ hws[IMX7D_USB_HSIC_ROOT_SRC] = imx_clk_hw_mux2("usb_hsic_src", base + 0xa100, 24, 3, usb_hsic_sel, ARRAY_SIZE(usb_hsic_sel));
+ hws[IMX7D_PCIE_CTRL_ROOT_SRC] = imx_clk_hw_mux2("pcie_ctrl_src", base + 0xa180, 24, 3, pcie_ctrl_sel, ARRAY_SIZE(pcie_ctrl_sel));
+ hws[IMX7D_PCIE_PHY_ROOT_SRC] = imx_clk_hw_mux2("pcie_phy_src", base + 0xa200, 24, 3, pcie_phy_sel, ARRAY_SIZE(pcie_phy_sel));
+ hws[IMX7D_EPDC_PIXEL_ROOT_SRC] = imx_clk_hw_mux2("epdc_pixel_src", base + 0xa280, 24, 3, epdc_pixel_sel, ARRAY_SIZE(epdc_pixel_sel));
+ hws[IMX7D_LCDIF_PIXEL_ROOT_SRC] = imx_clk_hw_mux2("lcdif_pixel_src", base + 0xa300, 24, 3, lcdif_pixel_sel, ARRAY_SIZE(lcdif_pixel_sel));
+ hws[IMX7D_MIPI_DSI_ROOT_SRC] = imx_clk_hw_mux2("mipi_dsi_src", base + 0xa380, 24, 3, mipi_dsi_sel, ARRAY_SIZE(mipi_dsi_sel));
+ hws[IMX7D_MIPI_CSI_ROOT_SRC] = imx_clk_hw_mux2("mipi_csi_src", base + 0xa400, 24, 3, mipi_csi_sel, ARRAY_SIZE(mipi_csi_sel));
+ hws[IMX7D_MIPI_DPHY_ROOT_SRC] = imx_clk_hw_mux2("mipi_dphy_src", base + 0xa480, 24, 3, mipi_dphy_sel, ARRAY_SIZE(mipi_dphy_sel));
+ hws[IMX7D_SAI1_ROOT_SRC] = imx_clk_hw_mux2("sai1_src", base + 0xa500, 24, 3, sai1_sel, ARRAY_SIZE(sai1_sel));
+ hws[IMX7D_SAI2_ROOT_SRC] = imx_clk_hw_mux2("sai2_src", base + 0xa580, 24, 3, sai2_sel, ARRAY_SIZE(sai2_sel));
+ hws[IMX7D_SAI3_ROOT_SRC] = imx_clk_hw_mux2("sai3_src", base + 0xa600, 24, 3, sai3_sel, ARRAY_SIZE(sai3_sel));
+ hws[IMX7D_SPDIF_ROOT_SRC] = imx_clk_hw_mux2("spdif_src", base + 0xa680, 24, 3, spdif_sel, ARRAY_SIZE(spdif_sel));
+ hws[IMX7D_ENET1_REF_ROOT_SRC] = imx_clk_hw_mux2("enet1_ref_src", base + 0xa700, 24, 3, enet1_ref_sel, ARRAY_SIZE(enet1_ref_sel));
+ hws[IMX7D_ENET1_TIME_ROOT_SRC] = imx_clk_hw_mux2("enet1_time_src", base + 0xa780, 24, 3, enet1_time_sel, ARRAY_SIZE(enet1_time_sel));
+ hws[IMX7D_ENET2_REF_ROOT_SRC] = imx_clk_hw_mux2("enet2_ref_src", base + 0xa800, 24, 3, enet2_ref_sel, ARRAY_SIZE(enet2_ref_sel));
+ hws[IMX7D_ENET2_TIME_ROOT_SRC] = imx_clk_hw_mux2("enet2_time_src", base + 0xa880, 24, 3, enet2_time_sel, ARRAY_SIZE(enet2_time_sel));
+ hws[IMX7D_ENET_PHY_REF_ROOT_SRC] = imx_clk_hw_mux2("enet_phy_ref_src", base + 0xa900, 24, 3, enet_phy_ref_sel, ARRAY_SIZE(enet_phy_ref_sel));
+ hws[IMX7D_EIM_ROOT_SRC] = imx_clk_hw_mux2("eim_src", base + 0xa980, 24, 3, eim_sel, ARRAY_SIZE(eim_sel));
+ hws[IMX7D_NAND_ROOT_SRC] = imx_clk_hw_mux2("nand_src", base + 0xaa00, 24, 3, nand_sel, ARRAY_SIZE(nand_sel));
+ hws[IMX7D_QSPI_ROOT_SRC] = imx_clk_hw_mux2("qspi_src", base + 0xaa80, 24, 3, qspi_sel, ARRAY_SIZE(qspi_sel));
+ hws[IMX7D_USDHC1_ROOT_SRC] = imx_clk_hw_mux2("usdhc1_src", base + 0xab00, 24, 3, usdhc1_sel, ARRAY_SIZE(usdhc1_sel));
+ hws[IMX7D_USDHC2_ROOT_SRC] = imx_clk_hw_mux2("usdhc2_src", base + 0xab80, 24, 3, usdhc2_sel, ARRAY_SIZE(usdhc2_sel));
+ hws[IMX7D_USDHC3_ROOT_SRC] = imx_clk_hw_mux2("usdhc3_src", base + 0xac00, 24, 3, usdhc3_sel, ARRAY_SIZE(usdhc3_sel));
+ hws[IMX7D_CAN1_ROOT_SRC] = imx_clk_hw_mux2("can1_src", base + 0xac80, 24, 3, can1_sel, ARRAY_SIZE(can1_sel));
+ hws[IMX7D_CAN2_ROOT_SRC] = imx_clk_hw_mux2("can2_src", base + 0xad00, 24, 3, can2_sel, ARRAY_SIZE(can2_sel));
+ hws[IMX7D_I2C1_ROOT_SRC] = imx_clk_hw_mux2("i2c1_src", base + 0xad80, 24, 3, i2c1_sel, ARRAY_SIZE(i2c1_sel));
+ hws[IMX7D_I2C2_ROOT_SRC] = imx_clk_hw_mux2("i2c2_src", base + 0xae00, 24, 3, i2c2_sel, ARRAY_SIZE(i2c2_sel));
+ hws[IMX7D_I2C3_ROOT_SRC] = imx_clk_hw_mux2("i2c3_src", base + 0xae80, 24, 3, i2c3_sel, ARRAY_SIZE(i2c3_sel));
+ hws[IMX7D_I2C4_ROOT_SRC] = imx_clk_hw_mux2("i2c4_src", base + 0xaf00, 24, 3, i2c4_sel, ARRAY_SIZE(i2c4_sel));
+ hws[IMX7D_UART1_ROOT_SRC] = imx_clk_hw_mux2("uart1_src", base + 0xaf80, 24, 3, uart1_sel, ARRAY_SIZE(uart1_sel));
+ hws[IMX7D_UART2_ROOT_SRC] = imx_clk_hw_mux2("uart2_src", base + 0xb000, 24, 3, uart2_sel, ARRAY_SIZE(uart2_sel));
+ hws[IMX7D_UART3_ROOT_SRC] = imx_clk_hw_mux2("uart3_src", base + 0xb080, 24, 3, uart3_sel, ARRAY_SIZE(uart3_sel));
+ hws[IMX7D_UART4_ROOT_SRC] = imx_clk_hw_mux2("uart4_src", base + 0xb100, 24, 3, uart4_sel, ARRAY_SIZE(uart4_sel));
+ hws[IMX7D_UART5_ROOT_SRC] = imx_clk_hw_mux2("uart5_src", base + 0xb180, 24, 3, uart5_sel, ARRAY_SIZE(uart5_sel));
+ hws[IMX7D_UART6_ROOT_SRC] = imx_clk_hw_mux2("uart6_src", base + 0xb200, 24, 3, uart6_sel, ARRAY_SIZE(uart6_sel));
+ hws[IMX7D_UART7_ROOT_SRC] = imx_clk_hw_mux2("uart7_src", base + 0xb280, 24, 3, uart7_sel, ARRAY_SIZE(uart7_sel));
+ hws[IMX7D_ECSPI1_ROOT_SRC] = imx_clk_hw_mux2("ecspi1_src", base + 0xb300, 24, 3, ecspi1_sel, ARRAY_SIZE(ecspi1_sel));
+ hws[IMX7D_ECSPI2_ROOT_SRC] = imx_clk_hw_mux2("ecspi2_src", base + 0xb380, 24, 3, ecspi2_sel, ARRAY_SIZE(ecspi2_sel));
+ hws[IMX7D_ECSPI3_ROOT_SRC] = imx_clk_hw_mux2("ecspi3_src", base + 0xb400, 24, 3, ecspi3_sel, ARRAY_SIZE(ecspi3_sel));
+ hws[IMX7D_ECSPI4_ROOT_SRC] = imx_clk_hw_mux2("ecspi4_src", base + 0xb480, 24, 3, ecspi4_sel, ARRAY_SIZE(ecspi4_sel));
+ hws[IMX7D_PWM1_ROOT_SRC] = imx_clk_hw_mux2("pwm1_src", base + 0xb500, 24, 3, pwm1_sel, ARRAY_SIZE(pwm1_sel));
+ hws[IMX7D_PWM2_ROOT_SRC] = imx_clk_hw_mux2("pwm2_src", base + 0xb580, 24, 3, pwm2_sel, ARRAY_SIZE(pwm2_sel));
+ hws[IMX7D_PWM3_ROOT_SRC] = imx_clk_hw_mux2("pwm3_src", base + 0xb600, 24, 3, pwm3_sel, ARRAY_SIZE(pwm3_sel));
+ hws[IMX7D_PWM4_ROOT_SRC] = imx_clk_hw_mux2("pwm4_src", base + 0xb680, 24, 3, pwm4_sel, ARRAY_SIZE(pwm4_sel));
+ hws[IMX7D_FLEXTIMER1_ROOT_SRC] = imx_clk_hw_mux2("flextimer1_src", base + 0xb700, 24, 3, flextimer1_sel, ARRAY_SIZE(flextimer1_sel));
+ hws[IMX7D_FLEXTIMER2_ROOT_SRC] = imx_clk_hw_mux2("flextimer2_src", base + 0xb780, 24, 3, flextimer2_sel, ARRAY_SIZE(flextimer2_sel));
+ hws[IMX7D_SIM1_ROOT_SRC] = imx_clk_hw_mux2("sim1_src", base + 0xb800, 24, 3, sim1_sel, ARRAY_SIZE(sim1_sel));
+ hws[IMX7D_SIM2_ROOT_SRC] = imx_clk_hw_mux2("sim2_src", base + 0xb880, 24, 3, sim2_sel, ARRAY_SIZE(sim2_sel));
+ hws[IMX7D_GPT1_ROOT_SRC] = imx_clk_hw_mux2("gpt1_src", base + 0xb900, 24, 3, gpt1_sel, ARRAY_SIZE(gpt1_sel));
+ hws[IMX7D_GPT2_ROOT_SRC] = imx_clk_hw_mux2("gpt2_src", base + 0xb980, 24, 3, gpt2_sel, ARRAY_SIZE(gpt2_sel));
+ hws[IMX7D_GPT3_ROOT_SRC] = imx_clk_hw_mux2("gpt3_src", base + 0xba00, 24, 3, gpt3_sel, ARRAY_SIZE(gpt3_sel));
+ hws[IMX7D_GPT4_ROOT_SRC] = imx_clk_hw_mux2("gpt4_src", base + 0xba80, 24, 3, gpt4_sel, ARRAY_SIZE(gpt4_sel));
+ hws[IMX7D_TRACE_ROOT_SRC] = imx_clk_hw_mux2("trace_src", base + 0xbb00, 24, 3, trace_sel, ARRAY_SIZE(trace_sel));
+ hws[IMX7D_WDOG_ROOT_SRC] = imx_clk_hw_mux2("wdog_src", base + 0xbb80, 24, 3, wdog_sel, ARRAY_SIZE(wdog_sel));
+ hws[IMX7D_CSI_MCLK_ROOT_SRC] = imx_clk_hw_mux2("csi_mclk_src", base + 0xbc00, 24, 3, csi_mclk_sel, ARRAY_SIZE(csi_mclk_sel));
+ hws[IMX7D_AUDIO_MCLK_ROOT_SRC] = imx_clk_hw_mux2("audio_mclk_src", base + 0xbc80, 24, 3, audio_mclk_sel, ARRAY_SIZE(audio_mclk_sel));
+ hws[IMX7D_WRCLK_ROOT_SRC] = imx_clk_hw_mux2("wrclk_src", base + 0xbd00, 24, 3, wrclk_sel, ARRAY_SIZE(wrclk_sel));
+ hws[IMX7D_CLKO1_ROOT_SRC] = imx_clk_hw_mux2("clko1_src", base + 0xbd80, 24, 3, clko1_sel, ARRAY_SIZE(clko1_sel));
+ hws[IMX7D_CLKO2_ROOT_SRC] = imx_clk_hw_mux2("clko2_src", base + 0xbe00, 24, 3, clko2_sel, ARRAY_SIZE(clko2_sel));
+
+ hws[IMX7D_ARM_A7_ROOT_CG] = imx_clk_hw_gate3("arm_a7_cg", "arm_a7_src", base + 0x8000, 28);
+ hws[IMX7D_ARM_M4_ROOT_CG] = imx_clk_hw_gate3("arm_m4_cg", "arm_m4_src", base + 0x8080, 28);
+ hws[IMX7D_MAIN_AXI_ROOT_CG] = imx_clk_hw_gate3("axi_cg", "axi_src", base + 0x8800, 28);
+ hws[IMX7D_DISP_AXI_ROOT_CG] = imx_clk_hw_gate3("disp_axi_cg", "disp_axi_src", base + 0x8880, 28);
+ hws[IMX7D_ENET_AXI_ROOT_CG] = imx_clk_hw_gate3("enet_axi_cg", "enet_axi_src", base + 0x8900, 28);
+ hws[IMX7D_NAND_USDHC_BUS_ROOT_CG] = imx_clk_hw_gate3("nand_usdhc_cg", "nand_usdhc_src", base + 0x8980, 28);
+ hws[IMX7D_AHB_CHANNEL_ROOT_CG] = imx_clk_hw_gate3("ahb_cg", "ahb_src", base + 0x9000, 28);
+ hws[IMX7D_DRAM_PHYM_ROOT_CG] = imx_clk_hw_gate3("dram_phym_cg", "dram_phym_src", base + 0x9800, 28);
+ hws[IMX7D_DRAM_ROOT_CG] = imx_clk_hw_gate3("dram_cg", "dram_src", base + 0x9880, 28);
+ hws[IMX7D_DRAM_PHYM_ALT_ROOT_CG] = imx_clk_hw_gate3("dram_phym_alt_cg", "dram_phym_alt_src", base + 0xa000, 28);
+ hws[IMX7D_DRAM_ALT_ROOT_CG] = imx_clk_hw_gate3("dram_alt_cg", "dram_alt_src", base + 0xa080, 28);
+ hws[IMX7D_USB_HSIC_ROOT_CG] = imx_clk_hw_gate3("usb_hsic_cg", "usb_hsic_src", base + 0xa100, 28);
+ hws[IMX7D_PCIE_CTRL_ROOT_CG] = imx_clk_hw_gate3("pcie_ctrl_cg", "pcie_ctrl_src", base + 0xa180, 28);
+ hws[IMX7D_PCIE_PHY_ROOT_CG] = imx_clk_hw_gate3("pcie_phy_cg", "pcie_phy_src", base + 0xa200, 28);
+ hws[IMX7D_EPDC_PIXEL_ROOT_CG] = imx_clk_hw_gate3("epdc_pixel_cg", "epdc_pixel_src", base + 0xa280, 28);
+ hws[IMX7D_LCDIF_PIXEL_ROOT_CG] = imx_clk_hw_gate3("lcdif_pixel_cg", "lcdif_pixel_src", base + 0xa300, 28);
+ hws[IMX7D_MIPI_DSI_ROOT_CG] = imx_clk_hw_gate3("mipi_dsi_cg", "mipi_dsi_src", base + 0xa380, 28);
+ hws[IMX7D_MIPI_CSI_ROOT_CG] = imx_clk_hw_gate3("mipi_csi_cg", "mipi_csi_src", base + 0xa400, 28);
+ hws[IMX7D_MIPI_DPHY_ROOT_CG] = imx_clk_hw_gate3("mipi_dphy_cg", "mipi_dphy_src", base + 0xa480, 28);
+ hws[IMX7D_SAI1_ROOT_CG] = imx_clk_hw_gate3("sai1_cg", "sai1_src", base + 0xa500, 28);
+ hws[IMX7D_SAI2_ROOT_CG] = imx_clk_hw_gate3("sai2_cg", "sai2_src", base + 0xa580, 28);
+ hws[IMX7D_SAI3_ROOT_CG] = imx_clk_hw_gate3("sai3_cg", "sai3_src", base + 0xa600, 28);
+ hws[IMX7D_SPDIF_ROOT_CG] = imx_clk_hw_gate3("spdif_cg", "spdif_src", base + 0xa680, 28);
+ hws[IMX7D_ENET1_REF_ROOT_CG] = imx_clk_hw_gate3("enet1_ref_cg", "enet1_ref_src", base + 0xa700, 28);
+ hws[IMX7D_ENET1_TIME_ROOT_CG] = imx_clk_hw_gate3("enet1_time_cg", "enet1_time_src", base + 0xa780, 28);
+ hws[IMX7D_ENET2_REF_ROOT_CG] = imx_clk_hw_gate3("enet2_ref_cg", "enet2_ref_src", base + 0xa800, 28);
+ hws[IMX7D_ENET2_TIME_ROOT_CG] = imx_clk_hw_gate3("enet2_time_cg", "enet2_time_src", base + 0xa880, 28);
+ hws[IMX7D_ENET_PHY_REF_ROOT_CG] = imx_clk_hw_gate3("enet_phy_ref_cg", "enet_phy_ref_src", base + 0xa900, 28);
+ hws[IMX7D_EIM_ROOT_CG] = imx_clk_hw_gate3("eim_cg", "eim_src", base + 0xa980, 28);
+ hws[IMX7D_NAND_ROOT_CG] = imx_clk_hw_gate3("nand_cg", "nand_src", base + 0xaa00, 28);
+ hws[IMX7D_QSPI_ROOT_CG] = imx_clk_hw_gate3("qspi_cg", "qspi_src", base + 0xaa80, 28);
+ hws[IMX7D_USDHC1_ROOT_CG] = imx_clk_hw_gate3("usdhc1_cg", "usdhc1_src", base + 0xab00, 28);
+ hws[IMX7D_USDHC2_ROOT_CG] = imx_clk_hw_gate3("usdhc2_cg", "usdhc2_src", base + 0xab80, 28);
+ hws[IMX7D_USDHC3_ROOT_CG] = imx_clk_hw_gate3("usdhc3_cg", "usdhc3_src", base + 0xac00, 28);
+ hws[IMX7D_CAN1_ROOT_CG] = imx_clk_hw_gate3("can1_cg", "can1_src", base + 0xac80, 28);
+ hws[IMX7D_CAN2_ROOT_CG] = imx_clk_hw_gate3("can2_cg", "can2_src", base + 0xad00, 28);
+ hws[IMX7D_I2C1_ROOT_CG] = imx_clk_hw_gate3("i2c1_cg", "i2c1_src", base + 0xad80, 28);
+ hws[IMX7D_I2C2_ROOT_CG] = imx_clk_hw_gate3("i2c2_cg", "i2c2_src", base + 0xae00, 28);
+ hws[IMX7D_I2C3_ROOT_CG] = imx_clk_hw_gate3("i2c3_cg", "i2c3_src", base + 0xae80, 28);
+ hws[IMX7D_I2C4_ROOT_CG] = imx_clk_hw_gate3("i2c4_cg", "i2c4_src", base + 0xaf00, 28);
+ hws[IMX7D_UART1_ROOT_CG] = imx_clk_hw_gate3("uart1_cg", "uart1_src", base + 0xaf80, 28);
+ hws[IMX7D_UART2_ROOT_CG] = imx_clk_hw_gate3("uart2_cg", "uart2_src", base + 0xb000, 28);
+ hws[IMX7D_UART3_ROOT_CG] = imx_clk_hw_gate3("uart3_cg", "uart3_src", base + 0xb080, 28);
+ hws[IMX7D_UART4_ROOT_CG] = imx_clk_hw_gate3("uart4_cg", "uart4_src", base + 0xb100, 28);
+ hws[IMX7D_UART5_ROOT_CG] = imx_clk_hw_gate3("uart5_cg", "uart5_src", base + 0xb180, 28);
+ hws[IMX7D_UART6_ROOT_CG] = imx_clk_hw_gate3("uart6_cg", "uart6_src", base + 0xb200, 28);
+ hws[IMX7D_UART7_ROOT_CG] = imx_clk_hw_gate3("uart7_cg", "uart7_src", base + 0xb280, 28);
+ hws[IMX7D_ECSPI1_ROOT_CG] = imx_clk_hw_gate3("ecspi1_cg", "ecspi1_src", base + 0xb300, 28);
+ hws[IMX7D_ECSPI2_ROOT_CG] = imx_clk_hw_gate3("ecspi2_cg", "ecspi2_src", base + 0xb380, 28);
+ hws[IMX7D_ECSPI3_ROOT_CG] = imx_clk_hw_gate3("ecspi3_cg", "ecspi3_src", base + 0xb400, 28);
+ hws[IMX7D_ECSPI4_ROOT_CG] = imx_clk_hw_gate3("ecspi4_cg", "ecspi4_src", base + 0xb480, 28);
+ hws[IMX7D_PWM1_ROOT_CG] = imx_clk_hw_gate3("pwm1_cg", "pwm1_src", base + 0xb500, 28);
+ hws[IMX7D_PWM2_ROOT_CG] = imx_clk_hw_gate3("pwm2_cg", "pwm2_src", base + 0xb580, 28);
+ hws[IMX7D_PWM3_ROOT_CG] = imx_clk_hw_gate3("pwm3_cg", "pwm3_src", base + 0xb600, 28);
+ hws[IMX7D_PWM4_ROOT_CG] = imx_clk_hw_gate3("pwm4_cg", "pwm4_src", base + 0xb680, 28);
+ hws[IMX7D_FLEXTIMER1_ROOT_CG] = imx_clk_hw_gate3("flextimer1_cg", "flextimer1_src", base + 0xb700, 28);
+ hws[IMX7D_FLEXTIMER2_ROOT_CG] = imx_clk_hw_gate3("flextimer2_cg", "flextimer2_src", base + 0xb780, 28);
+ hws[IMX7D_SIM1_ROOT_CG] = imx_clk_hw_gate3("sim1_cg", "sim1_src", base + 0xb800, 28);
+ hws[IMX7D_SIM2_ROOT_CG] = imx_clk_hw_gate3("sim2_cg", "sim2_src", base + 0xb880, 28);
+ hws[IMX7D_GPT1_ROOT_CG] = imx_clk_hw_gate3("gpt1_cg", "gpt1_src", base + 0xb900, 28);
+ hws[IMX7D_GPT2_ROOT_CG] = imx_clk_hw_gate3("gpt2_cg", "gpt2_src", base + 0xb980, 28);
+ hws[IMX7D_GPT3_ROOT_CG] = imx_clk_hw_gate3("gpt3_cg", "gpt3_src", base + 0xbA00, 28);
+ hws[IMX7D_GPT4_ROOT_CG] = imx_clk_hw_gate3("gpt4_cg", "gpt4_src", base + 0xbA80, 28);
+ hws[IMX7D_TRACE_ROOT_CG] = imx_clk_hw_gate3("trace_cg", "trace_src", base + 0xbb00, 28);
+ hws[IMX7D_WDOG_ROOT_CG] = imx_clk_hw_gate3("wdog_cg", "wdog_src", base + 0xbb80, 28);
+ hws[IMX7D_CSI_MCLK_ROOT_CG] = imx_clk_hw_gate3("csi_mclk_cg", "csi_mclk_src", base + 0xbc00, 28);
+ hws[IMX7D_AUDIO_MCLK_ROOT_CG] = imx_clk_hw_gate3("audio_mclk_cg", "audio_mclk_src", base + 0xbc80, 28);
+ hws[IMX7D_WRCLK_ROOT_CG] = imx_clk_hw_gate3("wrclk_cg", "wrclk_src", base + 0xbd00, 28);
+ hws[IMX7D_CLKO1_ROOT_CG] = imx_clk_hw_gate3("clko1_cg", "clko1_src", base + 0xbd80, 28);
+ hws[IMX7D_CLKO2_ROOT_CG] = imx_clk_hw_gate3("clko2_cg", "clko2_src", base + 0xbe00, 28);
+
+ hws[IMX7D_MAIN_AXI_ROOT_PRE_DIV] = imx_clk_hw_divider2("axi_pre_div", "axi_cg", base + 0x8800, 16, 3);
+ hws[IMX7D_DISP_AXI_ROOT_PRE_DIV] = imx_clk_hw_divider2("disp_axi_pre_div", "disp_axi_cg", base + 0x8880, 16, 3);
+ hws[IMX7D_ENET_AXI_ROOT_PRE_DIV] = imx_clk_hw_divider2("enet_axi_pre_div", "enet_axi_cg", base + 0x8900, 16, 3);
+ hws[IMX7D_NAND_USDHC_BUS_ROOT_PRE_DIV] = imx_clk_hw_divider2("nand_usdhc_pre_div", "nand_usdhc_cg", base + 0x8980, 16, 3);
+ hws[IMX7D_AHB_CHANNEL_ROOT_PRE_DIV] = imx_clk_hw_divider2("ahb_pre_div", "ahb_cg", base + 0x9000, 16, 3);
+ hws[IMX7D_DRAM_PHYM_ALT_ROOT_PRE_DIV] = imx_clk_hw_divider2("dram_phym_alt_pre_div", "dram_phym_alt_cg", base + 0xa000, 16, 3);
+ hws[IMX7D_DRAM_ALT_ROOT_PRE_DIV] = imx_clk_hw_divider2("dram_alt_pre_div", "dram_alt_cg", base + 0xa080, 16, 3);
+ hws[IMX7D_USB_HSIC_ROOT_PRE_DIV] = imx_clk_hw_divider2("usb_hsic_pre_div", "usb_hsic_cg", base + 0xa100, 16, 3);
+ hws[IMX7D_PCIE_CTRL_ROOT_PRE_DIV] = imx_clk_hw_divider2("pcie_ctrl_pre_div", "pcie_ctrl_cg", base + 0xa180, 16, 3);
+ hws[IMX7D_PCIE_PHY_ROOT_PRE_DIV] = imx_clk_hw_divider2("pcie_phy_pre_div", "pcie_phy_cg", base + 0xa200, 16, 3);
+ hws[IMX7D_EPDC_PIXEL_ROOT_PRE_DIV] = imx_clk_hw_divider2("epdc_pixel_pre_div", "epdc_pixel_cg", base + 0xa280, 16, 3);
+ hws[IMX7D_LCDIF_PIXEL_ROOT_PRE_DIV] = imx_clk_hw_divider2("lcdif_pixel_pre_div", "lcdif_pixel_cg", base + 0xa300, 16, 3);
+ hws[IMX7D_MIPI_DSI_ROOT_PRE_DIV] = imx_clk_hw_divider2("mipi_dsi_pre_div", "mipi_dsi_cg", base + 0xa380, 16, 3);
+ hws[IMX7D_MIPI_CSI_ROOT_PRE_DIV] = imx_clk_hw_divider2("mipi_csi_pre_div", "mipi_csi_cg", base + 0xa400, 16, 3);
+ hws[IMX7D_MIPI_DPHY_ROOT_PRE_DIV] = imx_clk_hw_divider2("mipi_dphy_pre_div", "mipi_dphy_cg", base + 0xa480, 16, 3);
+ hws[IMX7D_SAI1_ROOT_PRE_DIV] = imx_clk_hw_divider2("sai1_pre_div", "sai1_cg", base + 0xa500, 16, 3);
+ hws[IMX7D_SAI2_ROOT_PRE_DIV] = imx_clk_hw_divider2("sai2_pre_div", "sai2_cg", base + 0xa580, 16, 3);
+ hws[IMX7D_SAI3_ROOT_PRE_DIV] = imx_clk_hw_divider2("sai3_pre_div", "sai3_cg", base + 0xa600, 16, 3);
+ hws[IMX7D_SPDIF_ROOT_PRE_DIV] = imx_clk_hw_divider2("spdif_pre_div", "spdif_cg", base + 0xa680, 16, 3);
+ hws[IMX7D_ENET1_REF_ROOT_PRE_DIV] = imx_clk_hw_divider2("enet1_ref_pre_div", "enet1_ref_cg", base + 0xa700, 16, 3);
+ hws[IMX7D_ENET1_TIME_ROOT_PRE_DIV] = imx_clk_hw_divider2("enet1_time_pre_div", "enet1_time_cg", base + 0xa780, 16, 3);
+ hws[IMX7D_ENET2_REF_ROOT_PRE_DIV] = imx_clk_hw_divider2("enet2_ref_pre_div", "enet2_ref_cg", base + 0xa800, 16, 3);
+ hws[IMX7D_ENET2_TIME_ROOT_PRE_DIV] = imx_clk_hw_divider2("enet2_time_pre_div", "enet2_time_cg", base + 0xa880, 16, 3);
+ hws[IMX7D_ENET_PHY_REF_ROOT_PRE_DIV] = imx_clk_hw_divider2("enet_phy_ref_pre_div", "enet_phy_ref_cg", base + 0xa900, 16, 3);
+ hws[IMX7D_EIM_ROOT_PRE_DIV] = imx_clk_hw_divider2("eim_pre_div", "eim_cg", base + 0xa980, 16, 3);
+ hws[IMX7D_NAND_ROOT_PRE_DIV] = imx_clk_hw_divider2("nand_pre_div", "nand_cg", base + 0xaa00, 16, 3);
+ hws[IMX7D_QSPI_ROOT_PRE_DIV] = imx_clk_hw_divider2("qspi_pre_div", "qspi_cg", base + 0xaa80, 16, 3);
+ hws[IMX7D_USDHC1_ROOT_PRE_DIV] = imx_clk_hw_divider2("usdhc1_pre_div", "usdhc1_cg", base + 0xab00, 16, 3);
+ hws[IMX7D_USDHC2_ROOT_PRE_DIV] = imx_clk_hw_divider2("usdhc2_pre_div", "usdhc2_cg", base + 0xab80, 16, 3);
+ hws[IMX7D_USDHC3_ROOT_PRE_DIV] = imx_clk_hw_divider2("usdhc3_pre_div", "usdhc3_cg", base + 0xac00, 16, 3);
+ hws[IMX7D_CAN1_ROOT_PRE_DIV] = imx_clk_hw_divider2("can1_pre_div", "can1_cg", base + 0xac80, 16, 3);
+ hws[IMX7D_CAN2_ROOT_PRE_DIV] = imx_clk_hw_divider2("can2_pre_div", "can2_cg", base + 0xad00, 16, 3);
+ hws[IMX7D_I2C1_ROOT_PRE_DIV] = imx_clk_hw_divider2("i2c1_pre_div", "i2c1_cg", base + 0xad80, 16, 3);
+ hws[IMX7D_I2C2_ROOT_PRE_DIV] = imx_clk_hw_divider2("i2c2_pre_div", "i2c2_cg", base + 0xae00, 16, 3);
+ hws[IMX7D_I2C3_ROOT_PRE_DIV] = imx_clk_hw_divider2("i2c3_pre_div", "i2c3_cg", base + 0xae80, 16, 3);
+ hws[IMX7D_I2C4_ROOT_PRE_DIV] = imx_clk_hw_divider2("i2c4_pre_div", "i2c4_cg", base + 0xaf00, 16, 3);
+ hws[IMX7D_UART1_ROOT_PRE_DIV] = imx_clk_hw_divider2("uart1_pre_div", "uart1_cg", base + 0xaf80, 16, 3);
+ hws[IMX7D_UART2_ROOT_PRE_DIV] = imx_clk_hw_divider2("uart2_pre_div", "uart2_cg", base + 0xb000, 16, 3);
+ hws[IMX7D_UART3_ROOT_PRE_DIV] = imx_clk_hw_divider2("uart3_pre_div", "uart3_cg", base + 0xb080, 16, 3);
+ hws[IMX7D_UART4_ROOT_PRE_DIV] = imx_clk_hw_divider2("uart4_pre_div", "uart4_cg", base + 0xb100, 16, 3);
+ hws[IMX7D_UART5_ROOT_PRE_DIV] = imx_clk_hw_divider2("uart5_pre_div", "uart5_cg", base + 0xb180, 16, 3);
+ hws[IMX7D_UART6_ROOT_PRE_DIV] = imx_clk_hw_divider2("uart6_pre_div", "uart6_cg", base + 0xb200, 16, 3);
+ hws[IMX7D_UART7_ROOT_PRE_DIV] = imx_clk_hw_divider2("uart7_pre_div", "uart7_cg", base + 0xb280, 16, 3);
+ hws[IMX7D_ECSPI1_ROOT_PRE_DIV] = imx_clk_hw_divider2("ecspi1_pre_div", "ecspi1_cg", base + 0xb300, 16, 3);
+ hws[IMX7D_ECSPI2_ROOT_PRE_DIV] = imx_clk_hw_divider2("ecspi2_pre_div", "ecspi2_cg", base + 0xb380, 16, 3);
+ hws[IMX7D_ECSPI3_ROOT_PRE_DIV] = imx_clk_hw_divider2("ecspi3_pre_div", "ecspi3_cg", base + 0xb400, 16, 3);
+ hws[IMX7D_ECSPI4_ROOT_PRE_DIV] = imx_clk_hw_divider2("ecspi4_pre_div", "ecspi4_cg", base + 0xb480, 16, 3);
+ hws[IMX7D_PWM1_ROOT_PRE_DIV] = imx_clk_hw_divider2("pwm1_pre_div", "pwm1_cg", base + 0xb500, 16, 3);
+ hws[IMX7D_PWM2_ROOT_PRE_DIV] = imx_clk_hw_divider2("pwm2_pre_div", "pwm2_cg", base + 0xb580, 16, 3);
+ hws[IMX7D_PWM3_ROOT_PRE_DIV] = imx_clk_hw_divider2("pwm3_pre_div", "pwm3_cg", base + 0xb600, 16, 3);
+ hws[IMX7D_PWM4_ROOT_PRE_DIV] = imx_clk_hw_divider2("pwm4_pre_div", "pwm4_cg", base + 0xb680, 16, 3);
+ hws[IMX7D_FLEXTIMER1_ROOT_PRE_DIV] = imx_clk_hw_divider2("flextimer1_pre_div", "flextimer1_cg", base + 0xb700, 16, 3);
+ hws[IMX7D_FLEXTIMER2_ROOT_PRE_DIV] = imx_clk_hw_divider2("flextimer2_pre_div", "flextimer2_cg", base + 0xb780, 16, 3);
+ hws[IMX7D_SIM1_ROOT_PRE_DIV] = imx_clk_hw_divider2("sim1_pre_div", "sim1_cg", base + 0xb800, 16, 3);
+ hws[IMX7D_SIM2_ROOT_PRE_DIV] = imx_clk_hw_divider2("sim2_pre_div", "sim2_cg", base + 0xb880, 16, 3);
+ hws[IMX7D_GPT1_ROOT_PRE_DIV] = imx_clk_hw_divider2("gpt1_pre_div", "gpt1_cg", base + 0xb900, 16, 3);
+ hws[IMX7D_GPT2_ROOT_PRE_DIV] = imx_clk_hw_divider2("gpt2_pre_div", "gpt2_cg", base + 0xb980, 16, 3);
+ hws[IMX7D_GPT3_ROOT_PRE_DIV] = imx_clk_hw_divider2("gpt3_pre_div", "gpt3_cg", base + 0xba00, 16, 3);
+ hws[IMX7D_GPT4_ROOT_PRE_DIV] = imx_clk_hw_divider2("gpt4_pre_div", "gpt4_cg", base + 0xba80, 16, 3);
+ hws[IMX7D_TRACE_ROOT_PRE_DIV] = imx_clk_hw_divider2("trace_pre_div", "trace_cg", base + 0xbb00, 16, 3);
+ hws[IMX7D_WDOG_ROOT_PRE_DIV] = imx_clk_hw_divider2("wdog_pre_div", "wdog_cg", base + 0xbb80, 16, 3);
+ hws[IMX7D_CSI_MCLK_ROOT_PRE_DIV] = imx_clk_hw_divider2("csi_mclk_pre_div", "csi_mclk_cg", base + 0xbc00, 16, 3);
+ hws[IMX7D_AUDIO_MCLK_ROOT_PRE_DIV] = imx_clk_hw_divider2("audio_mclk_pre_div", "audio_mclk_cg", base + 0xbc80, 16, 3);
+ hws[IMX7D_WRCLK_ROOT_PRE_DIV] = imx_clk_hw_divider2("wrclk_pre_div", "wrclk_cg", base + 0xbd00, 16, 3);
+ hws[IMX7D_CLKO1_ROOT_PRE_DIV] = imx_clk_hw_divider2("clko1_pre_div", "clko1_cg", base + 0xbd80, 16, 3);
+ hws[IMX7D_CLKO2_ROOT_PRE_DIV] = imx_clk_hw_divider2("clko2_pre_div", "clko2_cg", base + 0xbe00, 16, 3);
+
+ hws[IMX7D_ARM_A7_ROOT_DIV] = imx_clk_hw_divider2("arm_a7_div", "arm_a7_cg", base + 0x8000, 0, 3);
+ hws[IMX7D_ARM_M4_ROOT_DIV] = imx_clk_hw_divider2("arm_m4_div", "arm_m4_cg", base + 0x8080, 0, 3);
+ hws[IMX7D_MAIN_AXI_ROOT_DIV] = imx_clk_hw_divider2("axi_post_div", "axi_pre_div", base + 0x8800, 0, 6);
+ hws[IMX7D_DISP_AXI_ROOT_DIV] = imx_clk_hw_divider2("disp_axi_post_div", "disp_axi_pre_div", base + 0x8880, 0, 6);
+ hws[IMX7D_ENET_AXI_ROOT_DIV] = imx_clk_hw_divider2("enet_axi_post_div", "enet_axi_pre_div", base + 0x8900, 0, 6);
+ hws[IMX7D_NAND_USDHC_BUS_ROOT_CLK] = imx_clk_hw_divider2("nand_usdhc_root_clk", "nand_usdhc_pre_div", base + 0x8980, 0, 6);
+ hws[IMX7D_AHB_CHANNEL_ROOT_DIV] = imx_clk_hw_divider2("ahb_root_clk", "ahb_pre_div", base + 0x9000, 0, 6);
+ hws[IMX7D_IPG_ROOT_CLK] = imx_clk_hw_divider_flags("ipg_root_clk", "ahb_root_clk", base + 0x9080, 0, 2, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE | CLK_SET_RATE_PARENT);
+ hws[IMX7D_DRAM_ROOT_DIV] = imx_clk_hw_divider2("dram_post_div", "dram_cg", base + 0x9880, 0, 3);
+ hws[IMX7D_DRAM_PHYM_ALT_ROOT_DIV] = imx_clk_hw_divider2("dram_phym_alt_post_div", "dram_phym_alt_pre_div", base + 0xa000, 0, 3);
+ hws[IMX7D_DRAM_ALT_ROOT_DIV] = imx_clk_hw_divider2("dram_alt_post_div", "dram_alt_pre_div", base + 0xa080, 0, 3);
+ hws[IMX7D_USB_HSIC_ROOT_DIV] = imx_clk_hw_divider2("usb_hsic_post_div", "usb_hsic_pre_div", base + 0xa100, 0, 6);
+ hws[IMX7D_PCIE_CTRL_ROOT_DIV] = imx_clk_hw_divider2("pcie_ctrl_post_div", "pcie_ctrl_pre_div", base + 0xa180, 0, 6);
+ hws[IMX7D_PCIE_PHY_ROOT_DIV] = imx_clk_hw_divider2("pcie_phy_post_div", "pcie_phy_pre_div", base + 0xa200, 0, 6);
+ hws[IMX7D_EPDC_PIXEL_ROOT_DIV] = imx_clk_hw_divider2("epdc_pixel_post_div", "epdc_pixel_pre_div", base + 0xa280, 0, 6);
+ hws[IMX7D_LCDIF_PIXEL_ROOT_DIV] = imx_clk_hw_divider2("lcdif_pixel_post_div", "lcdif_pixel_pre_div", base + 0xa300, 0, 6);
+ hws[IMX7D_MIPI_DSI_ROOT_DIV] = imx_clk_hw_divider2("mipi_dsi_post_div", "mipi_dsi_pre_div", base + 0xa380, 0, 6);
+ hws[IMX7D_MIPI_CSI_ROOT_DIV] = imx_clk_hw_divider2("mipi_csi_post_div", "mipi_csi_pre_div", base + 0xa400, 0, 6);
+ hws[IMX7D_MIPI_DPHY_ROOT_DIV] = imx_clk_hw_divider2("mipi_dphy_post_div", "mipi_dphy_pre_div", base + 0xa480, 0, 6);
+ hws[IMX7D_SAI1_ROOT_DIV] = imx_clk_hw_divider2("sai1_post_div", "sai1_pre_div", base + 0xa500, 0, 6);
+ hws[IMX7D_SAI2_ROOT_DIV] = imx_clk_hw_divider2("sai2_post_div", "sai2_pre_div", base + 0xa580, 0, 6);
+ hws[IMX7D_SAI3_ROOT_DIV] = imx_clk_hw_divider2("sai3_post_div", "sai3_pre_div", base + 0xa600, 0, 6);
+ hws[IMX7D_SPDIF_ROOT_DIV] = imx_clk_hw_divider2("spdif_post_div", "spdif_pre_div", base + 0xa680, 0, 6);
+ hws[IMX7D_ENET1_REF_ROOT_DIV] = imx_clk_hw_divider2("enet1_ref_post_div", "enet1_ref_pre_div", base + 0xa700, 0, 6);
+ hws[IMX7D_ENET1_TIME_ROOT_DIV] = imx_clk_hw_divider2("enet1_time_post_div", "enet1_time_pre_div", base + 0xa780, 0, 6);
+ hws[IMX7D_ENET2_REF_ROOT_DIV] = imx_clk_hw_divider2("enet2_ref_post_div", "enet2_ref_pre_div", base + 0xa800, 0, 6);
+ hws[IMX7D_ENET2_TIME_ROOT_DIV] = imx_clk_hw_divider2("enet2_time_post_div", "enet2_time_pre_div", base + 0xa880, 0, 6);
+ hws[IMX7D_ENET_PHY_REF_ROOT_CLK] = imx_clk_hw_divider2("enet_phy_ref_root_clk", "enet_phy_ref_pre_div", base + 0xa900, 0, 6);
+ hws[IMX7D_EIM_ROOT_DIV] = imx_clk_hw_divider2("eim_post_div", "eim_pre_div", base + 0xa980, 0, 6);
+ hws[IMX7D_NAND_ROOT_CLK] = imx_clk_hw_divider2("nand_root_clk", "nand_pre_div", base + 0xaa00, 0, 6);
+ hws[IMX7D_QSPI_ROOT_DIV] = imx_clk_hw_divider2("qspi_post_div", "qspi_pre_div", base + 0xaa80, 0, 6);
+ hws[IMX7D_USDHC1_ROOT_DIV] = imx_clk_hw_divider2("usdhc1_post_div", "usdhc1_pre_div", base + 0xab00, 0, 6);
+ hws[IMX7D_USDHC2_ROOT_DIV] = imx_clk_hw_divider2("usdhc2_post_div", "usdhc2_pre_div", base + 0xab80, 0, 6);
+ hws[IMX7D_USDHC3_ROOT_DIV] = imx_clk_hw_divider2("usdhc3_post_div", "usdhc3_pre_div", base + 0xac00, 0, 6);
+ hws[IMX7D_CAN1_ROOT_DIV] = imx_clk_hw_divider2("can1_post_div", "can1_pre_div", base + 0xac80, 0, 6);
+ hws[IMX7D_CAN2_ROOT_DIV] = imx_clk_hw_divider2("can2_post_div", "can2_pre_div", base + 0xad00, 0, 6);
+ hws[IMX7D_I2C1_ROOT_DIV] = imx_clk_hw_divider2("i2c1_post_div", "i2c1_pre_div", base + 0xad80, 0, 6);
+ hws[IMX7D_I2C2_ROOT_DIV] = imx_clk_hw_divider2("i2c2_post_div", "i2c2_pre_div", base + 0xae00, 0, 6);
+ hws[IMX7D_I2C3_ROOT_DIV] = imx_clk_hw_divider2("i2c3_post_div", "i2c3_pre_div", base + 0xae80, 0, 6);
+ hws[IMX7D_I2C4_ROOT_DIV] = imx_clk_hw_divider2("i2c4_post_div", "i2c4_pre_div", base + 0xaf00, 0, 6);
+ hws[IMX7D_UART1_ROOT_DIV] = imx_clk_hw_divider2("uart1_post_div", "uart1_pre_div", base + 0xaf80, 0, 6);
+ hws[IMX7D_UART2_ROOT_DIV] = imx_clk_hw_divider2("uart2_post_div", "uart2_pre_div", base + 0xb000, 0, 6);
+ hws[IMX7D_UART3_ROOT_DIV] = imx_clk_hw_divider2("uart3_post_div", "uart3_pre_div", base + 0xb080, 0, 6);
+ hws[IMX7D_UART4_ROOT_DIV] = imx_clk_hw_divider2("uart4_post_div", "uart4_pre_div", base + 0xb100, 0, 6);
+ hws[IMX7D_UART5_ROOT_DIV] = imx_clk_hw_divider2("uart5_post_div", "uart5_pre_div", base + 0xb180, 0, 6);
+ hws[IMX7D_UART6_ROOT_DIV] = imx_clk_hw_divider2("uart6_post_div", "uart6_pre_div", base + 0xb200, 0, 6);
+ hws[IMX7D_UART7_ROOT_DIV] = imx_clk_hw_divider2("uart7_post_div", "uart7_pre_div", base + 0xb280, 0, 6);
+ hws[IMX7D_ECSPI1_ROOT_DIV] = imx_clk_hw_divider2("ecspi1_post_div", "ecspi1_pre_div", base + 0xb300, 0, 6);
+ hws[IMX7D_ECSPI2_ROOT_DIV] = imx_clk_hw_divider2("ecspi2_post_div", "ecspi2_pre_div", base + 0xb380, 0, 6);
+ hws[IMX7D_ECSPI3_ROOT_DIV] = imx_clk_hw_divider2("ecspi3_post_div", "ecspi3_pre_div", base + 0xb400, 0, 6);
+ hws[IMX7D_ECSPI4_ROOT_DIV] = imx_clk_hw_divider2("ecspi4_post_div", "ecspi4_pre_div", base + 0xb480, 0, 6);
+ hws[IMX7D_PWM1_ROOT_DIV] = imx_clk_hw_divider2("pwm1_post_div", "pwm1_pre_div", base + 0xb500, 0, 6);
+ hws[IMX7D_PWM2_ROOT_DIV] = imx_clk_hw_divider2("pwm2_post_div", "pwm2_pre_div", base + 0xb580, 0, 6);
+ hws[IMX7D_PWM3_ROOT_DIV] = imx_clk_hw_divider2("pwm3_post_div", "pwm3_pre_div", base + 0xb600, 0, 6);
+ hws[IMX7D_PWM4_ROOT_DIV] = imx_clk_hw_divider2("pwm4_post_div", "pwm4_pre_div", base + 0xb680, 0, 6);
+ hws[IMX7D_FLEXTIMER1_ROOT_DIV] = imx_clk_hw_divider2("flextimer1_post_div", "flextimer1_pre_div", base + 0xb700, 0, 6);
+ hws[IMX7D_FLEXTIMER2_ROOT_DIV] = imx_clk_hw_divider2("flextimer2_post_div", "flextimer2_pre_div", base + 0xb780, 0, 6);
+ hws[IMX7D_SIM1_ROOT_DIV] = imx_clk_hw_divider2("sim1_post_div", "sim1_pre_div", base + 0xb800, 0, 6);
+ hws[IMX7D_SIM2_ROOT_DIV] = imx_clk_hw_divider2("sim2_post_div", "sim2_pre_div", base + 0xb880, 0, 6);
+ hws[IMX7D_GPT1_ROOT_DIV] = imx_clk_hw_divider2("gpt1_post_div", "gpt1_pre_div", base + 0xb900, 0, 6);
+ hws[IMX7D_GPT2_ROOT_DIV] = imx_clk_hw_divider2("gpt2_post_div", "gpt2_pre_div", base + 0xb980, 0, 6);
+ hws[IMX7D_GPT3_ROOT_DIV] = imx_clk_hw_divider2("gpt3_post_div", "gpt3_pre_div", base + 0xba00, 0, 6);
+ hws[IMX7D_GPT4_ROOT_DIV] = imx_clk_hw_divider2("gpt4_post_div", "gpt4_pre_div", base + 0xba80, 0, 6);
+ hws[IMX7D_TRACE_ROOT_DIV] = imx_clk_hw_divider2("trace_post_div", "trace_pre_div", base + 0xbb00, 0, 6);
+ hws[IMX7D_WDOG_ROOT_DIV] = imx_clk_hw_divider2("wdog_post_div", "wdog_pre_div", base + 0xbb80, 0, 6);
+ hws[IMX7D_CSI_MCLK_ROOT_DIV] = imx_clk_hw_divider2("csi_mclk_post_div", "csi_mclk_pre_div", base + 0xbc00, 0, 6);
+ hws[IMX7D_AUDIO_MCLK_ROOT_DIV] = imx_clk_hw_divider2("audio_mclk_post_div", "audio_mclk_pre_div", base + 0xbc80, 0, 6);
+ hws[IMX7D_WRCLK_ROOT_DIV] = imx_clk_hw_divider2("wrclk_post_div", "wrclk_pre_div", base + 0xbd00, 0, 6);
+ hws[IMX7D_CLKO1_ROOT_DIV] = imx_clk_hw_divider2("clko1_post_div", "clko1_pre_div", base + 0xbd80, 0, 6);
+ hws[IMX7D_CLKO2_ROOT_DIV] = imx_clk_hw_divider2("clko2_post_div", "clko2_pre_div", base + 0xbe00, 0, 6);
+
+ hws[IMX7D_ARM_A7_ROOT_CLK] = imx_clk_hw_gate2_flags("arm_a7_root_clk", "arm_a7_div", base + 0x4000, 0, CLK_OPS_PARENT_ENABLE);
+ hws[IMX7D_ARM_M4_ROOT_CLK] = imx_clk_hw_gate4("arm_m4_root_clk", "arm_m4_div", base + 0x4010, 0);
+ hws[IMX7D_MAIN_AXI_ROOT_CLK] = imx_clk_hw_gate2_flags("main_axi_root_clk", "axi_post_div", base + 0x4040, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE);
+ hws[IMX7D_DISP_AXI_ROOT_CLK] = imx_clk_hw_gate4("disp_axi_root_clk", "disp_axi_post_div", base + 0x4050, 0);
+ hws[IMX7D_ENET_AXI_ROOT_CLK] = imx_clk_hw_gate4("enet_axi_root_clk", "enet_axi_post_div", base + 0x4060, 0);
+ hws[IMX7D_OCRAM_CLK] = imx_clk_hw_gate4("ocram_clk", "main_axi_root_clk", base + 0x4110, 0);
+ hws[IMX7D_OCRAM_S_CLK] = imx_clk_hw_gate4("ocram_s_clk", "ahb_root_clk", base + 0x4120, 0);
+ hws[IMX7D_DRAM_ROOT_CLK] = imx_clk_hw_gate2_flags("dram_root_clk", "dram_post_div", base + 0x4130, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE);
+ hws[IMX7D_DRAM_PHYM_ROOT_CLK] = imx_clk_hw_gate2_flags("dram_phym_root_clk", "dram_phym_cg", base + 0x4130, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE);
+ hws[IMX7D_DRAM_PHYM_ALT_ROOT_CLK] = imx_clk_hw_gate2_flags("dram_phym_alt_root_clk", "dram_phym_alt_post_div", base + 0x4130, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE);
+ hws[IMX7D_DRAM_ALT_ROOT_CLK] = imx_clk_hw_gate2_flags("dram_alt_root_clk", "dram_alt_post_div", base + 0x4130, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE);
+ hws[IMX7D_OCOTP_CLK] = imx_clk_hw_gate4("ocotp_clk", "ipg_root_clk", base + 0x4230, 0);
+ hws[IMX7D_SNVS_CLK] = imx_clk_hw_gate4("snvs_clk", "ipg_root_clk", base + 0x4250, 0);
+ hws[IMX7D_MU_ROOT_CLK] = imx_clk_hw_gate4("mu_root_clk", "ipg_root_clk", base + 0x4270, 0);
+ hws[IMX7D_CAAM_CLK] = imx_clk_hw_gate4("caam_clk", "ipg_root_clk", base + 0x4240, 0);
+ hws[IMX7D_USB_HSIC_ROOT_CLK] = imx_clk_hw_gate4("usb_hsic_root_clk", "usb_hsic_post_div", base + 0x4690, 0);
+ hws[IMX7D_SDMA_CORE_CLK] = imx_clk_hw_gate4("sdma_root_clk", "ahb_root_clk", base + 0x4480, 0);
+ hws[IMX7D_PCIE_CTRL_ROOT_CLK] = imx_clk_hw_gate4("pcie_ctrl_root_clk", "pcie_ctrl_post_div", base + 0x4600, 0);
+ hws[IMX7D_PCIE_PHY_ROOT_CLK] = imx_clk_hw_gate4("pcie_phy_root_clk", "pcie_phy_post_div", base + 0x4600, 0);
+ hws[IMX7D_EPDC_PIXEL_ROOT_CLK] = imx_clk_hw_gate4("epdc_pixel_root_clk", "epdc_pixel_post_div", base + 0x44a0, 0);
+ hws[IMX7D_LCDIF_PIXEL_ROOT_CLK] = imx_clk_hw_gate4("lcdif_pixel_root_clk", "lcdif_pixel_post_div", base + 0x44b0, 0);
+ hws[IMX7D_MIPI_DSI_ROOT_CLK] = imx_clk_hw_gate4("mipi_dsi_root_clk", "mipi_dsi_post_div", base + 0x4650, 0);
+ hws[IMX7D_MIPI_CSI_ROOT_CLK] = imx_clk_hw_gate4("mipi_csi_root_clk", "mipi_csi_post_div", base + 0x4640, 0);
+ hws[IMX7D_MIPI_DPHY_ROOT_CLK] = imx_clk_hw_gate4("mipi_dphy_root_clk", "mipi_dphy_post_div", base + 0x4660, 0);
+ hws[IMX7D_ENET1_IPG_ROOT_CLK] = imx_clk_hw_gate2_shared2("enet1_ipg_root_clk", "enet_axi_post_div", base + 0x4700, 0, &share_count_enet1);
+ hws[IMX7D_ENET1_TIME_ROOT_CLK] = imx_clk_hw_gate2_shared2("enet1_time_root_clk", "enet1_time_post_div", base + 0x4700, 0, &share_count_enet1);
+ hws[IMX7D_ENET2_IPG_ROOT_CLK] = imx_clk_hw_gate2_shared2("enet2_ipg_root_clk", "enet_axi_post_div", base + 0x4710, 0, &share_count_enet2);
+ hws[IMX7D_ENET2_TIME_ROOT_CLK] = imx_clk_hw_gate2_shared2("enet2_time_root_clk", "enet2_time_post_div", base + 0x4710, 0, &share_count_enet2);
+ hws[IMX7D_SAI1_ROOT_CLK] = imx_clk_hw_gate2_shared2("sai1_root_clk", "sai1_post_div", base + 0x48c0, 0, &share_count_sai1);
+ hws[IMX7D_SAI1_IPG_CLK] = imx_clk_hw_gate2_shared2("sai1_ipg_clk", "ipg_root_clk", base + 0x48c0, 0, &share_count_sai1);
+ hws[IMX7D_SAI2_ROOT_CLK] = imx_clk_hw_gate2_shared2("sai2_root_clk", "sai2_post_div", base + 0x48d0, 0, &share_count_sai2);
+ hws[IMX7D_SAI2_IPG_CLK] = imx_clk_hw_gate2_shared2("sai2_ipg_clk", "ipg_root_clk", base + 0x48d0, 0, &share_count_sai2);
+ hws[IMX7D_SAI3_ROOT_CLK] = imx_clk_hw_gate2_shared2("sai3_root_clk", "sai3_post_div", base + 0x48e0, 0, &share_count_sai3);
+ hws[IMX7D_SAI3_IPG_CLK] = imx_clk_hw_gate2_shared2("sai3_ipg_clk", "ipg_root_clk", base + 0x48e0, 0, &share_count_sai3);
+ hws[IMX7D_SPDIF_ROOT_CLK] = imx_clk_hw_gate4("spdif_root_clk", "spdif_post_div", base + 0x44d0, 0);
+ hws[IMX7D_EIM_ROOT_CLK] = imx_clk_hw_gate4("eim_root_clk", "eim_post_div", base + 0x4160, 0);
+ hws[IMX7D_NAND_RAWNAND_CLK] = imx_clk_hw_gate2_shared2("nand_rawnand_clk", "nand_root_clk", base + 0x4140, 0, &share_count_nand);
+ hws[IMX7D_NAND_USDHC_BUS_RAWNAND_CLK] = imx_clk_hw_gate2_shared2("nand_usdhc_rawnand_clk", "nand_usdhc_root_clk", base + 0x4140, 0, &share_count_nand);
+ hws[IMX7D_QSPI_ROOT_CLK] = imx_clk_hw_gate4("qspi_root_clk", "qspi_post_div", base + 0x4150, 0);
+ hws[IMX7D_USDHC1_ROOT_CLK] = imx_clk_hw_gate4("usdhc1_root_clk", "usdhc1_post_div", base + 0x46c0, 0);
+ hws[IMX7D_USDHC2_ROOT_CLK] = imx_clk_hw_gate4("usdhc2_root_clk", "usdhc2_post_div", base + 0x46d0, 0);
+ hws[IMX7D_USDHC3_ROOT_CLK] = imx_clk_hw_gate4("usdhc3_root_clk", "usdhc3_post_div", base + 0x46e0, 0);
+ hws[IMX7D_CAN1_ROOT_CLK] = imx_clk_hw_gate4("can1_root_clk", "can1_post_div", base + 0x4740, 0);
+ hws[IMX7D_CAN2_ROOT_CLK] = imx_clk_hw_gate4("can2_root_clk", "can2_post_div", base + 0x4750, 0);
+ hws[IMX7D_I2C1_ROOT_CLK] = imx_clk_hw_gate4("i2c1_root_clk", "i2c1_post_div", base + 0x4880, 0);
+ hws[IMX7D_I2C2_ROOT_CLK] = imx_clk_hw_gate4("i2c2_root_clk", "i2c2_post_div", base + 0x4890, 0);
+ hws[IMX7D_I2C3_ROOT_CLK] = imx_clk_hw_gate4("i2c3_root_clk", "i2c3_post_div", base + 0x48a0, 0);
+ hws[IMX7D_I2C4_ROOT_CLK] = imx_clk_hw_gate4("i2c4_root_clk", "i2c4_post_div", base + 0x48b0, 0);
+ hws[IMX7D_UART1_ROOT_CLK] = imx_clk_hw_gate4("uart1_root_clk", "uart1_post_div", base + 0x4940, 0);
+ hws[IMX7D_UART2_ROOT_CLK] = imx_clk_hw_gate4("uart2_root_clk", "uart2_post_div", base + 0x4950, 0);
+ hws[IMX7D_UART3_ROOT_CLK] = imx_clk_hw_gate4("uart3_root_clk", "uart3_post_div", base + 0x4960, 0);
+ hws[IMX7D_UART4_ROOT_CLK] = imx_clk_hw_gate4("uart4_root_clk", "uart4_post_div", base + 0x4970, 0);
+ hws[IMX7D_UART5_ROOT_CLK] = imx_clk_hw_gate4("uart5_root_clk", "uart5_post_div", base + 0x4980, 0);
+ hws[IMX7D_UART6_ROOT_CLK] = imx_clk_hw_gate4("uart6_root_clk", "uart6_post_div", base + 0x4990, 0);
+ hws[IMX7D_UART7_ROOT_CLK] = imx_clk_hw_gate4("uart7_root_clk", "uart7_post_div", base + 0x49a0, 0);
+ hws[IMX7D_ECSPI1_ROOT_CLK] = imx_clk_hw_gate4("ecspi1_root_clk", "ecspi1_post_div", base + 0x4780, 0);
+ hws[IMX7D_ECSPI2_ROOT_CLK] = imx_clk_hw_gate4("ecspi2_root_clk", "ecspi2_post_div", base + 0x4790, 0);
+ hws[IMX7D_ECSPI3_ROOT_CLK] = imx_clk_hw_gate4("ecspi3_root_clk", "ecspi3_post_div", base + 0x47a0, 0);
+ hws[IMX7D_ECSPI4_ROOT_CLK] = imx_clk_hw_gate4("ecspi4_root_clk", "ecspi4_post_div", base + 0x47b0, 0);
+ hws[IMX7D_PWM1_ROOT_CLK] = imx_clk_hw_gate4("pwm1_root_clk", "pwm1_post_div", base + 0x4840, 0);
+ hws[IMX7D_PWM2_ROOT_CLK] = imx_clk_hw_gate4("pwm2_root_clk", "pwm2_post_div", base + 0x4850, 0);
+ hws[IMX7D_PWM3_ROOT_CLK] = imx_clk_hw_gate4("pwm3_root_clk", "pwm3_post_div", base + 0x4860, 0);
+ hws[IMX7D_PWM4_ROOT_CLK] = imx_clk_hw_gate4("pwm4_root_clk", "pwm4_post_div", base + 0x4870, 0);
+ hws[IMX7D_FLEXTIMER1_ROOT_CLK] = imx_clk_hw_gate4("flextimer1_root_clk", "flextimer1_post_div", base + 0x4800, 0);
+ hws[IMX7D_FLEXTIMER2_ROOT_CLK] = imx_clk_hw_gate4("flextimer2_root_clk", "flextimer2_post_div", base + 0x4810, 0);
+ hws[IMX7D_SIM1_ROOT_CLK] = imx_clk_hw_gate4("sim1_root_clk", "sim1_post_div", base + 0x4900, 0);
+ hws[IMX7D_SIM2_ROOT_CLK] = imx_clk_hw_gate4("sim2_root_clk", "sim2_post_div", base + 0x4910, 0);
+ hws[IMX7D_GPT1_ROOT_CLK] = imx_clk_hw_gate4("gpt1_root_clk", "gpt1_post_div", base + 0x47c0, 0);
+ hws[IMX7D_GPT2_ROOT_CLK] = imx_clk_hw_gate4("gpt2_root_clk", "gpt2_post_div", base + 0x47d0, 0);
+ hws[IMX7D_GPT3_ROOT_CLK] = imx_clk_hw_gate4("gpt3_root_clk", "gpt3_post_div", base + 0x47e0, 0);
+ hws[IMX7D_GPT4_ROOT_CLK] = imx_clk_hw_gate4("gpt4_root_clk", "gpt4_post_div", base + 0x47f0, 0);
+ hws[IMX7D_TRACE_ROOT_CLK] = imx_clk_hw_gate4("trace_root_clk", "trace_post_div", base + 0x4300, 0);
+ hws[IMX7D_WDOG1_ROOT_CLK] = imx_clk_hw_gate4("wdog1_root_clk", "wdog_post_div", base + 0x49c0, 0);
+ hws[IMX7D_WDOG2_ROOT_CLK] = imx_clk_hw_gate4("wdog2_root_clk", "wdog_post_div", base + 0x49d0, 0);
+ hws[IMX7D_WDOG3_ROOT_CLK] = imx_clk_hw_gate4("wdog3_root_clk", "wdog_post_div", base + 0x49e0, 0);
+ hws[IMX7D_WDOG4_ROOT_CLK] = imx_clk_hw_gate4("wdog4_root_clk", "wdog_post_div", base + 0x49f0, 0);
+ hws[IMX7D_KPP_ROOT_CLK] = imx_clk_hw_gate4("kpp_root_clk", "ipg_root_clk", base + 0x4aa0, 0);
+ hws[IMX7D_CSI_MCLK_ROOT_CLK] = imx_clk_hw_gate4("csi_mclk_root_clk", "csi_mclk_post_div", base + 0x4490, 0);
+ hws[IMX7D_AUDIO_MCLK_ROOT_CLK] = imx_clk_hw_gate4("audio_mclk_root_clk", "audio_mclk_post_div", base + 0x4790, 0);
+ hws[IMX7D_WRCLK_ROOT_CLK] = imx_clk_hw_gate4("wrclk_root_clk", "wrclk_post_div", base + 0x47a0, 0);
+ hws[IMX7D_USB_CTRL_CLK] = imx_clk_hw_gate4("usb_ctrl_clk", "ahb_root_clk", base + 0x4680, 0);
+ hws[IMX7D_USB_PHY1_CLK] = imx_clk_hw_gate4("usb_phy1_clk", "pll_usb1_main_clk", base + 0x46a0, 0);
+ hws[IMX7D_USB_PHY2_CLK] = imx_clk_hw_gate4("usb_phy2_clk", "pll_usb_main_clk", base + 0x46b0, 0);
+ hws[IMX7D_ADC_ROOT_CLK] = imx_clk_hw_gate4("adc_root_clk", "ipg_root_clk", base + 0x4200, 0);
+
+ hws[IMX7D_GPT_3M_CLK] = imx_clk_hw_fixed_factor("gpt_3m", "osc", 1, 8);
+
+ hws[IMX7D_CLK_ARM] = imx_clk_hw_cpu("arm", "arm_a7_root_clk",
+ hws[IMX7D_ARM_A7_ROOT_CLK]->clk,
+ hws[IMX7D_ARM_A7_ROOT_SRC]->clk,
+ hws[IMX7D_PLL_ARM_MAIN_CLK]->clk,
+ hws[IMX7D_PLL_SYS_MAIN_CLK]->clk);
+
+ imx_check_clk_hws(hws, IMX7D_CLK_END);
+
+ of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data);
+
+ clk_set_parent(hws[IMX7D_PLL_ARM_MAIN_BYPASS]->clk, hws[IMX7D_PLL_ARM_MAIN]->clk);
+ clk_set_parent(hws[IMX7D_PLL_DRAM_MAIN_BYPASS]->clk, hws[IMX7D_PLL_DRAM_MAIN]->clk);
+ clk_set_parent(hws[IMX7D_PLL_SYS_MAIN_BYPASS]->clk, hws[IMX7D_PLL_SYS_MAIN]->clk);
+ clk_set_parent(hws[IMX7D_PLL_ENET_MAIN_BYPASS]->clk, hws[IMX7D_PLL_ENET_MAIN]->clk);
+ clk_set_parent(hws[IMX7D_PLL_AUDIO_MAIN_BYPASS]->clk, hws[IMX7D_PLL_AUDIO_MAIN]->clk);
+ clk_set_parent(hws[IMX7D_PLL_VIDEO_MAIN_BYPASS]->clk, hws[IMX7D_PLL_VIDEO_MAIN]->clk);
+
+ clk_set_parent(hws[IMX7D_MIPI_CSI_ROOT_SRC]->clk, hws[IMX7D_PLL_SYS_PFD3_CLK]->clk);
/* use old gpt clk setting, gpt1 root clk must be twice as gpt counter freq */
- clk_set_parent(clks[IMX7D_GPT1_ROOT_SRC], clks[IMX7D_OSC_24M_CLK]);
+ clk_set_parent(hws[IMX7D_GPT1_ROOT_SRC]->clk, hws[IMX7D_OSC_24M_CLK]->clk);
/* Set clock rate for USBPHY, the USB_PLL at CCM is from USBOTG2 */
- clks[IMX7D_USB1_MAIN_480M_CLK] = imx_clk_fixed_factor("pll_usb1_main_clk", "osc", 20, 1);
- clks[IMX7D_USB_MAIN_480M_CLK] = imx_clk_fixed_factor("pll_usb_main_clk", "osc", 20, 1);
+ hws[IMX7D_USB1_MAIN_480M_CLK] = imx_clk_hw_fixed_factor("pll_usb1_main_clk", "osc", 20, 1);
+ hws[IMX7D_USB_MAIN_480M_CLK] = imx_clk_hw_fixed_factor("pll_usb_main_clk", "osc", 20, 1);
+
+ for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) {
+ int index = uart_clk_ids[i];
+
+ uart_clks[i] = &hws[index]->clk;
+ }
+
imx_register_uart_clocks(uart_clks);
diff --git a/drivers/clk/imx/clk-imx7ulp.c b/drivers/clk/imx/clk-imx7ulp.c
index 66682100f14c..42e4667f22fd 100644
--- a/drivers/clk/imx/clk-imx7ulp.c
+++ b/drivers/clk/imx/clk-imx7ulp.c
@@ -115,7 +115,7 @@ static void __init imx7ulp_clk_scg1_init(struct device_node *np)
clks[IMX7ULP_CLK_NIC0_DIV] = imx_clk_hw_divider_flags("nic0_clk", "nic_sel", base + 0x40, 24, 4, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL);
clks[IMX7ULP_CLK_NIC1_DIV] = imx_clk_hw_divider_flags("nic1_clk", "nic0_clk", base + 0x40, 16, 4, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL);
- clks[IMX7ULP_CLK_NIC1_BUS_DIV] = imx_clk_hw_divider_flags("nic1_bus_clk", "nic1_clk", base + 0x40, 4, 4, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL);
+ clks[IMX7ULP_CLK_NIC1_BUS_DIV] = imx_clk_hw_divider_flags("nic1_bus_clk", "nic0_clk", base + 0x40, 4, 4, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL);
clks[IMX7ULP_CLK_GPU_DIV] = imx_clk_hw_divider("gpu_clk", "nic0_clk", base + 0x40, 20, 4);
diff --git a/drivers/clk/imx/clk-imx8mm.c b/drivers/clk/imx/clk-imx8mm.c
index 122a81ab8e48..43fa9c361fcb 100644
--- a/drivers/clk/imx/clk-imx8mm.c
+++ b/drivers/clk/imx/clk-imx8mm.c
@@ -26,23 +26,6 @@ static u32 share_count_dcss;
static u32 share_count_pdm;
static u32 share_count_nand;
-#define PLL_1416X_RATE(_rate, _m, _p, _s) \
- { \
- .rate = (_rate), \
- .mdiv = (_m), \
- .pdiv = (_p), \
- .sdiv = (_s), \
- }
-
-#define PLL_1443X_RATE(_rate, _m, _p, _s, _k) \
- { \
- .rate = (_rate), \
- .mdiv = (_m), \
- .pdiv = (_p), \
- .sdiv = (_s), \
- .kdiv = (_k), \
- }
-
static const struct imx_pll14xx_rate_table imx8mm_pll1416x_tbl[] = {
PLL_1416X_RATE(1800000000U, 225, 3, 0),
PLL_1416X_RATE(1600000000U, 200, 3, 0),
@@ -288,6 +271,9 @@ static const char *imx8mm_usb_core_sels[] = {"osc_24m", "sys_pll1_100m", "sys_pl
static const char *imx8mm_usb_phy_sels[] = {"osc_24m", "sys_pll1_100m", "sys_pll1_40m", "sys_pll2_100m",
"sys_pll2_200m", "clk_ext2", "clk_ext3", "audio_pll2_out", };
+static const char *imx8mm_gic_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m", "sys_pll2_100m",
+ "sys_pll1_800m", "clk_ext2", "clk_ext4", "audio_pll2_out" };
+
static const char *imx8mm_ecspi1_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m", "sys_pll1_160m",
"sys_pll1_800m", "sys_pll3_out", "sys_pll2_250m", "audio_pll2_out", };
@@ -325,7 +311,7 @@ static const char *imx8mm_dsi_dbi_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll
"sys_pll2_1000m", "sys_pll3_out", "audio_pll2_out", "video_pll1_out", };
static const char *imx8mm_usdhc3_sels[] = {"osc_24m", "sys_pll1_400m", "sys_pll1_800m", "sys_pll2_500m",
- "sys_pll3_out", "sys_pll1_266m", "audio_pll2_clk", "sys_pll1_100m", };
+ "sys_pll3_out", "sys_pll1_266m", "audio_pll2_out", "sys_pll1_100m", };
static const char *imx8mm_csi1_core_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll2_250m", "sys_pll1_800m",
"sys_pll2_1000m", "sys_pll3_out", "audio_pll2_out", "video_pll1_out", };
@@ -361,11 +347,11 @@ static const char *imx8mm_pdm_sels[] = {"osc_24m", "sys_pll2_100m", "audio_pll1_
"sys_pll2_1000m", "sys_pll3_out", "clk_ext3", "audio_pll2_out", };
static const char *imx8mm_vpu_h1_sels[] = {"osc_24m", "vpu_pll_out", "sys_pll1_800m", "sys_pll2_1000m",
- "audio_pll2_clk", "sys_pll2_125m", "sys_pll3_clk", "audio_pll1_out", };
+ "audio_pll2_out", "sys_pll2_125m", "sys_pll3_clk", "audio_pll1_out", };
static const char *imx8mm_dram_core_sels[] = {"dram_pll_out", "dram_alt_root", };
-static const char *imx8mm_clko1_sels[] = {"osc_24m", "sys_pll1_800m", "osc_27m", "sys_pll1_200m", "audio_pll2_clk",
+static const char *imx8mm_clko1_sels[] = {"osc_24m", "sys_pll1_800m", "osc_27m", "sys_pll1_200m", "audio_pll2_out",
"vpu_pll", "sys_pll1_80m", };
static struct clk *clks[IMX8MM_CLK_END];
@@ -523,7 +509,7 @@ static int __init imx8mm_clocks_init(struct device_node *ccm_node)
/* IP */
clks[IMX8MM_CLK_DRAM_ALT] = imx8m_clk_composite("dram_alt", imx8mm_dram_alt_sels, base + 0xa000);
- clks[IMX8MM_CLK_DRAM_APB] = imx8m_clk_composite("dram_apb", imx8mm_dram_apb_sels, base + 0xa080);
+ clks[IMX8MM_CLK_DRAM_APB] = imx8m_clk_composite_critical("dram_apb", imx8mm_dram_apb_sels, base + 0xa080);
clks[IMX8MM_CLK_VPU_G1] = imx8m_clk_composite("vpu_g1", imx8mm_vpu_g1_sels, base + 0xa100);
clks[IMX8MM_CLK_VPU_G2] = imx8m_clk_composite("vpu_g2", imx8mm_vpu_g2_sels, base + 0xa180);
clks[IMX8MM_CLK_DISP_DTRC] = imx8m_clk_composite("disp_dtrc", imx8mm_disp_dtrc_sels, base + 0xa200);
@@ -558,6 +544,7 @@ static int __init imx8mm_clocks_init(struct device_node *ccm_node)
clks[IMX8MM_CLK_UART4] = imx8m_clk_composite("uart4", imx8mm_uart4_sels, base + 0xb080);
clks[IMX8MM_CLK_USB_CORE_REF] = imx8m_clk_composite("usb_core_ref", imx8mm_usb_core_sels, base + 0xb100);
clks[IMX8MM_CLK_USB_PHY_REF] = imx8m_clk_composite("usb_phy_ref", imx8mm_usb_phy_sels, base + 0xb180);
+ clks[IMX8MM_CLK_GIC] = imx8m_clk_composite_critical("gic", imx8mm_gic_sels, base + 0xb200);
clks[IMX8MM_CLK_ECSPI1] = imx8m_clk_composite("ecspi1", imx8mm_ecspi1_sels, base + 0xb280);
clks[IMX8MM_CLK_ECSPI2] = imx8m_clk_composite("ecspi2", imx8mm_ecspi2_sels, base + 0xb300);
clks[IMX8MM_CLK_PWM1] = imx8m_clk_composite("pwm1", imx8mm_pwm1_sels, base + 0xb380);
@@ -590,6 +577,11 @@ static int __init imx8mm_clocks_init(struct device_node *ccm_node)
clks[IMX8MM_CLK_ECSPI2_ROOT] = imx_clk_gate4("ecspi2_root_clk", "ecspi2", base + 0x4080, 0);
clks[IMX8MM_CLK_ECSPI3_ROOT] = imx_clk_gate4("ecspi3_root_clk", "ecspi3", base + 0x4090, 0);
clks[IMX8MM_CLK_ENET1_ROOT] = imx_clk_gate4("enet1_root_clk", "enet_axi", base + 0x40a0, 0);
+ clks[IMX8MM_CLK_GPIO1_ROOT] = imx_clk_gate4("gpio1_root_clk", "ipg_root", base + 0x40b0, 0);
+ clks[IMX8MM_CLK_GPIO2_ROOT] = imx_clk_gate4("gpio2_root_clk", "ipg_root", base + 0x40c0, 0);
+ clks[IMX8MM_CLK_GPIO3_ROOT] = imx_clk_gate4("gpio3_root_clk", "ipg_root", base + 0x40d0, 0);
+ clks[IMX8MM_CLK_GPIO4_ROOT] = imx_clk_gate4("gpio4_root_clk", "ipg_root", base + 0x40e0, 0);
+ clks[IMX8MM_CLK_GPIO5_ROOT] = imx_clk_gate4("gpio5_root_clk", "ipg_root", base + 0x40f0, 0);
clks[IMX8MM_CLK_GPT1_ROOT] = imx_clk_gate4("gpt1_root_clk", "gpt1", base + 0x4100, 0);
clks[IMX8MM_CLK_I2C1_ROOT] = imx_clk_gate4("i2c1_root_clk", "i2c1", base + 0x4170, 0);
clks[IMX8MM_CLK_I2C2_ROOT] = imx_clk_gate4("i2c2_root_clk", "i2c2", base + 0x4180, 0);
@@ -617,6 +609,7 @@ static int __init imx8mm_clocks_init(struct device_node *ccm_node)
clks[IMX8MM_CLK_SAI5_IPG] = imx_clk_gate2_shared2("sai5_ipg_clk", "ipg_audio_root", base + 0x4370, 0, &share_count_sai5);
clks[IMX8MM_CLK_SAI6_ROOT] = imx_clk_gate2_shared2("sai6_root_clk", "sai6", base + 0x4380, 0, &share_count_sai6);
clks[IMX8MM_CLK_SAI6_IPG] = imx_clk_gate2_shared2("sai6_ipg_clk", "ipg_audio_root", base + 0x4380, 0, &share_count_sai6);
+ clks[IMX8MM_CLK_SNVS_ROOT] = imx_clk_gate4("snvs_root_clk", "ipg_root", base + 0x4470, 0);
clks[IMX8MM_CLK_UART1_ROOT] = imx_clk_gate4("uart1_root_clk", "uart1", base + 0x4490, 0);
clks[IMX8MM_CLK_UART2_ROOT] = imx_clk_gate4("uart2_root_clk", "uart2", base + 0x44a0, 0);
clks[IMX8MM_CLK_UART3_ROOT] = imx_clk_gate4("uart3_root_clk", "uart3", base + 0x44b0, 0);
diff --git a/drivers/clk/imx/clk-imx8mn.c b/drivers/clk/imx/clk-imx8mn.c
new file mode 100644
index 000000000000..07481a53e336
--- /dev/null
+++ b/drivers/clk/imx/clk-imx8mn.c
@@ -0,0 +1,636 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018-2019 NXP.
+ */
+
+#include <dt-bindings/clock/imx8mn-clock.h>
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/types.h>
+
+#include "clk.h"
+
+static u32 share_count_sai2;
+static u32 share_count_sai3;
+static u32 share_count_sai5;
+static u32 share_count_sai6;
+static u32 share_count_sai7;
+static u32 share_count_disp;
+static u32 share_count_pdm;
+static u32 share_count_nand;
+
+enum {
+ ARM_PLL,
+ GPU_PLL,
+ VPU_PLL,
+ SYS_PLL1,
+ SYS_PLL2,
+ SYS_PLL3,
+ DRAM_PLL,
+ AUDIO_PLL1,
+ AUDIO_PLL2,
+ VIDEO_PLL2,
+ NR_PLLS,
+};
+
+static const struct imx_pll14xx_rate_table imx8mn_pll1416x_tbl[] = {
+ PLL_1416X_RATE(1800000000U, 225, 3, 0),
+ PLL_1416X_RATE(1600000000U, 200, 3, 0),
+ PLL_1416X_RATE(1200000000U, 300, 3, 1),
+ PLL_1416X_RATE(1000000000U, 250, 3, 1),
+ PLL_1416X_RATE(800000000U, 200, 3, 1),
+ PLL_1416X_RATE(750000000U, 250, 2, 2),
+ PLL_1416X_RATE(700000000U, 350, 3, 2),
+ PLL_1416X_RATE(600000000U, 300, 3, 2),
+};
+
+static const struct imx_pll14xx_rate_table imx8mn_audiopll_tbl[] = {
+ PLL_1443X_RATE(786432000U, 655, 5, 2, 23593),
+ PLL_1443X_RATE(722534400U, 301, 5, 1, 3670),
+};
+
+static const struct imx_pll14xx_rate_table imx8mn_videopll_tbl[] = {
+ PLL_1443X_RATE(650000000U, 325, 3, 2, 0),
+ PLL_1443X_RATE(594000000U, 198, 2, 2, 0),
+};
+
+static const struct imx_pll14xx_rate_table imx8mn_drampll_tbl[] = {
+ PLL_1443X_RATE(650000000U, 325, 3, 2, 0),
+};
+
+static struct imx_pll14xx_clk imx8mn_audio_pll = {
+ .type = PLL_1443X,
+ .rate_table = imx8mn_audiopll_tbl,
+};
+
+static struct imx_pll14xx_clk imx8mn_video_pll = {
+ .type = PLL_1443X,
+ .rate_table = imx8mn_videopll_tbl,
+};
+
+static struct imx_pll14xx_clk imx8mn_dram_pll = {
+ .type = PLL_1443X,
+ .rate_table = imx8mn_drampll_tbl,
+};
+
+static struct imx_pll14xx_clk imx8mn_arm_pll = {
+ .type = PLL_1416X,
+ .rate_table = imx8mn_pll1416x_tbl,
+};
+
+static struct imx_pll14xx_clk imx8mn_gpu_pll = {
+ .type = PLL_1416X,
+ .rate_table = imx8mn_pll1416x_tbl,
+};
+
+static struct imx_pll14xx_clk imx8mn_vpu_pll = {
+ .type = PLL_1416X,
+ .rate_table = imx8mn_pll1416x_tbl,
+};
+
+static struct imx_pll14xx_clk imx8mn_sys_pll = {
+ .type = PLL_1416X,
+ .rate_table = imx8mn_pll1416x_tbl,
+};
+
+static const char * const pll_ref_sels[] = { "osc_24m", "dummy", "dummy", "dummy", };
+static const char * const audio_pll1_bypass_sels[] = {"audio_pll1", "audio_pll1_ref_sel", };
+static const char * const audio_pll2_bypass_sels[] = {"audio_pll2", "audio_pll2_ref_sel", };
+static const char * const video_pll1_bypass_sels[] = {"video_pll1", "video_pll1_ref_sel", };
+static const char * const dram_pll_bypass_sels[] = {"dram_pll", "dram_pll_ref_sel", };
+static const char * const gpu_pll_bypass_sels[] = {"gpu_pll", "gpu_pll_ref_sel", };
+static const char * const vpu_pll_bypass_sels[] = {"vpu_pll", "vpu_pll_ref_sel", };
+static const char * const arm_pll_bypass_sels[] = {"arm_pll", "arm_pll_ref_sel", };
+static const char * const sys_pll1_bypass_sels[] = {"sys_pll1", "sys_pll1_ref_sel", };
+static const char * const sys_pll2_bypass_sels[] = {"sys_pll2", "sys_pll2_ref_sel", };
+static const char * const sys_pll3_bypass_sels[] = {"sys_pll3", "sys_pll3_ref_sel", };
+
+static const char * const imx8mn_a53_sels[] = {"osc_24m", "arm_pll_out", "sys_pll2_500m",
+ "sys_pll2_1000m", "sys_pll1_800m", "sys_pll1_400m",
+ "audio_pll1_out", "sys_pll3_out", };
+
+static const char * const imx8mn_gpu_core_sels[] = {"osc_24m", "gpu_pll_out", "sys_pll1_800m",
+ "sys_pll3_out", "sys_pll2_1000m", "audio_pll1_out",
+ "video_pll1_out", "audio_pll2_out", };
+
+static const char * const imx8mn_gpu_shader_sels[] = {"osc_24m", "gpu_pll_out", "sys_pll1_800m",
+ "sys_pll3_out", "sys_pll2_1000m", "audio_pll1_out",
+ "video_pll1_out", "audio_pll2_out", };
+
+static const char * const imx8mn_main_axi_sels[] = {"osc_24m", "sys_pll2_333m", "sys_pll1_800m",
+ "sys_pll2_250m", "sys_pll2_1000m", "audio_pll1_out",
+ "video_pll1_out", "sys_pll1_100m",};
+
+static const char * const imx8mn_enet_axi_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll1_800m",
+ "sys_pll2_250m", "sys_pll2_200m", "audio_pll1_out",
+ "video_pll1_out", "sys_pll3_out", };
+
+static const char * const imx8mn_nand_usdhc_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll1_800m",
+ "sys_pll2_200m", "sys_pll1_133m", "sys_pll3_out",
+ "sys_pll2_250m", "audio_pll1_out", };
+
+static const char * const imx8mn_disp_axi_sels[] = {"osc_24m", "sys_pll2_1000m", "sys_pll1_800m",
+ "sys_pll3_out", "sys_pll1_40m", "audio_pll2_out",
+ "clk_ext1", "clk_ext4", };
+
+static const char * const imx8mn_disp_apb_sels[] = {"osc_24m", "sys_pll2_125m", "sys_pll1_800m",
+ "sys_pll3_out", "sys1_pll_40m", "audio_pll2_out",
+ "clk_ext1", "clk_ext3", };
+
+static const char * const imx8mn_usb_bus_sels[] = {"osc_24m", "sys_pll2_500m", "sys_pll1_800m",
+ "sys_pll2_100m", "sys_pll2_200m", "clk_ext2",
+ "clk_ext4", "audio_pll2_out", };
+
+static const char * const imx8mn_gpu_axi_sels[] = {"osc_24m", "sys_pll1_800m", "gpu_pll_out",
+ "sys_pll3_out", "sys_pll2_1000m", "audio_pll1_out",
+ "video_pll1_out", "audio_pll2_out", };
+
+static const char * const imx8mn_gpu_ahb_sels[] = {"osc_24m", "sys_pll1_800m", "gpu_pll_out",
+ "sys_pll3_out", "sys_pll2_1000m", "audio_pll1_out",
+ "video_pll1_out", "audio_pll2_out", };
+
+static const char * const imx8mn_noc_sels[] = {"osc_24m", "sys_pll1_800m", "sys_pll3_out",
+ "sys_pll2_1000m", "sys_pll2_500m", "audio_pll1_out",
+ "video_pll1_out", "audio_pll2_out", };
+
+static const char * const imx8mn_ahb_sels[] = {"osc_24m", "sys_pll1_133m", "sys_pll1_800m",
+ "sys_pll1_400m", "sys_pll2_125m", "sys_pll3_out",
+ "audio_pll1_out", "video_pll1_out", };
+
+static const char * const imx8mn_audio_ahb_sels[] = {"osc_24m", "sys_pll2_500m", "sys_pll1_800m",
+ "sys_pll2_1000m", "sys_pll2_166m", "sys_pll3_out",
+ "audio_pll1_out", "video_pll1_out", };
+
+static const char * const imx8mn_dram_alt_sels[] = {"osc_24m", "sys_pll1_800m", "sys_pll1_100m",
+ "sys_pll2_500m", "sys_pll2_1000m", "sys_pll3_out",
+ "audio_pll1_out", "sys_pll1_266m", };
+
+static const char * const imx8mn_dram_apb_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m",
+ "sys_pll1_160m", "sys_pll1_800m", "sys_pll3_out",
+ "sys_pll2_250m", "audio_pll2_out", };
+
+static const char * const imx8mn_disp_pixel_sels[] = {"osc_24m", "video_pll1_out", "audio_pll2_out",
+ "audio_pll1_out", "sys_pll1_800m", "sys_pll2_1000m",
+ "sys_pll3_out", "clk_ext4", };
+
+static const char * const imx8mn_sai2_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out",
+ "video_pll1_out", "sys_pll1_133m", "osc_hdmi",
+ "clk_ext3", "clk_ext4", };
+
+static const char * const imx8mn_sai3_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out",
+ "video_pll1_out", "sys_pll1_133m", "osc_hdmi",
+ "clk_ext3", "clk_ext4", };
+
+static const char * const imx8mn_sai5_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out",
+ "video_pll1_out", "sys_pll1_133m", "osc_hdmi",
+ "clk_ext2", "clk_ext3", };
+
+static const char * const imx8mn_sai6_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out",
+ "video_pll1_out", "sys_pll1_133m", "osc_hdmi",
+ "clk_ext3", "clk_ext4", };
+
+static const char * const imx8mn_sai7_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out",
+ "video_pll1_out", "sys_pll1_133m", "osc_hdmi",
+ "clk_ext3", "clk_ext4", };
+
+static const char * const imx8mn_spdif1_sels[] = {"osc_24m", "audio_pll1_out", "audio_pll2_out",
+ "video_pll1_out", "sys_pll1_133m", "osc_hdmi",
+ "clk_ext2", "clk_ext3", };
+
+static const char * const imx8mn_enet_ref_sels[] = {"osc_24m", "sys_pll2_125m", "sys_pll2_50m",
+ "sys_pll2_100m", "sys_pll1_160m", "audio_pll1_out",
+ "video_pll1_out", "clk_ext4", };
+
+static const char * const imx8mn_enet_timer_sels[] = {"osc_24m", "sys_pll2_100m", "audio_pll1_out",
+ "clk_ext1", "clk_ext2", "clk_ext3",
+ "clk_ext4", "video_pll1_out", };
+
+static const char * const imx8mn_enet_phy_sels[] = {"osc_24m", "sys_pll2_50m", "sys_pll2_125m",
+ "sys_pll2_200m", "sys_pll2_500m", "video_pll1_out",
+ "audio_pll2_out", };
+
+static const char * const imx8mn_nand_sels[] = {"osc_24m", "sys_pll2_500m", "audio_pll1_out",
+ "sys_pll1_400m", "audio_pll2_out", "sys_pll3_out",
+ "sys_pll2_250m", "video_pll1_out", };
+
+static const char * const imx8mn_qspi_sels[] = {"osc_24m", "sys1_pll_400m", "sys_pll1_800m",
+ "sys2_pll_500m", "audio_pll2_out", "sys1_pll_266m",
+ "sys3_pll2_out", "sys1_pll_100m", };
+
+static const char * const imx8mn_usdhc1_sels[] = {"osc_24m", "sys_pll1_400m", "sys_pll1_800m",
+ "sys_pll2_500m", "sys_pll3_out", "sys_pll1_266m",
+ "audio_pll2_out", "sys_pll1_100m", };
+
+static const char * const imx8mn_usdhc2_sels[] = {"osc_24m", "sys_pll1_400m", "sys_pll1_800m",
+ "sys_pll2_500m", "sys_pll3_out", "sys_pll1_266m",
+ "audio_pll2_out", "sys_pll1_100m", };
+
+static const char * const imx8mn_i2c1_sels[] = {"osc_24m", "sys_pll1_160m", "sys_pll2_50m",
+ "sys_pll3_out", "audio_pll1_out", "video_pll1_out",
+ "audio_pll2_out", "sys_pll1_133m", };
+
+static const char * const imx8mn_i2c2_sels[] = {"osc_24m", "sys_pll1_160m", "sys_pll2_50m",
+ "sys_pll3_out", "audio_pll1_out", "video_pll1_out",
+ "audio_pll2_out", "sys_pll1_133m", };
+
+static const char * const imx8mn_i2c3_sels[] = {"osc_24m", "sys_pll1_160m", "sys_pll2_50m",
+ "sys_pll3_out", "audio_pll1_out", "video_pll1_out",
+ "audio_pll2_out", "sys_pll1_133m", };
+
+static const char * const imx8mn_i2c4_sels[] = {"osc_24m", "sys_pll1_160m", "sys_pll2_50m",
+ "sys_pll3_out", "audio_pll1_out", "video_pll1_out",
+ "audio_pll2_out", "sys_pll1_133m", };
+
+static const char * const imx8mn_uart1_sels[] = {"osc_24m", "sys_pll1_80m", "sys_pll2_200m",
+ "sys_pll2_100m", "sys_pll3_out", "clk_ext2",
+ "clk_ext4", "audio_pll2_out", };
+
+static const char * const imx8mn_uart2_sels[] = {"osc_24m", "sys_pll1_80m", "sys_pll2_200m",
+ "sys_pll2_100m", "sys_pll3_out", "clk_ext2",
+ "clk_ext3", "audio_pll2_out", };
+
+static const char * const imx8mn_uart3_sels[] = {"osc_24m", "sys_pll1_80m", "sys_pll2_200m",
+ "sys_pll2_100m", "sys_pll3_out", "clk_ext2",
+ "clk_ext4", "audio_pll2_out", };
+
+static const char * const imx8mn_uart4_sels[] = {"osc_24m", "sys_pll1_80m", "sys_pll2_200m",
+ "sys_pll2_100m", "sys_pll3_out", "clk_ext2",
+ "clk_ext3", "audio_pll2_out", };
+
+static const char * const imx8mn_usb_core_sels[] = {"osc_24m", "sys_pll1_100m", "sys_pll1_40m",
+ "sys_pll2_100m", "sys_pll2_200m", "clk_ext2",
+ "clk_ext3", "audio_pll2_out", };
+
+static const char * const imx8mn_usb_phy_sels[] = {"osc_24m", "sys_pll1_100m", "sys_pll1_40m",
+ "sys_pll2_100m", "sys_pll2_200m", "clk_ext2",
+ "clk_ext3", "audio_pll2_out", };
+
+static const char * const imx8mn_ecspi1_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m",
+ "sys_pll1_160m", "sys_pll1_800m", "sys_pll3_out",
+ "sys_pll2_250m", "audio_pll2_out", };
+
+static const char * const imx8mn_ecspi2_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m",
+ "sys_pll1_160m", "sys_pll1_800m", "sys_pll3_out",
+ "sys_pll2_250m", "audio_pll2_out", };
+
+static const char * const imx8mn_pwm1_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_160m",
+ "sys_pll1_40m", "sys_pll3_out", "clk_ext1",
+ "sys_pll1_80m", "video_pll1_out", };
+
+static const char * const imx8mn_pwm2_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_160m",
+ "sys_pll1_40m", "sys_pll3_out", "clk_ext1",
+ "sys_pll1_80m", "video_pll1_out", };
+
+static const char * const imx8mn_pwm3_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_160m",
+ "sys_pll1_40m", "sys3_pll2_out", "clk_ext2",
+ "sys_pll1_80m", "video_pll1_out", };
+
+static const char * const imx8mn_pwm4_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_160m",
+ "sys_pll1_40m", "sys_pll3_out", "clk_ext2",
+ "sys_pll1_80m", "video_pll1_out", };
+
+static const char * const imx8mn_wdog_sels[] = {"osc_24m", "sys_pll1_133m", "sys_pll1_160m",
+ "vpu_pll_out", "sys_pll2_125m", "sys_pll3_out",
+ "sys_pll1_80m", "sys_pll2_166m", };
+
+static const char * const imx8mn_wrclk_sels[] = {"osc_24m", "sys_pll1_40m", "vpu_pll_out",
+ "sys_pll3_out", "sys_pll2_200m", "sys_pll1_266m",
+ "sys_pll2_500m", "sys_pll1_100m", };
+
+static const char * const imx8mn_dsi_core_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll2_250m",
+ "sys_pll1_800m", "sys_pll2_1000m", "sys_pll3_out",
+ "audio_pll2_out", "video_pll1_out", };
+
+static const char * const imx8mn_dsi_phy_sels[] = {"osc_24m", "sys_pll2_125m", "sys_pll2_100m",
+ "sys_pll1_800m", "sys_pll2_1000m", "clk_ext2",
+ "audio_pll2_out", "video_pll1_out", };
+
+static const char * const imx8mn_dsi_dbi_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll2_100m",
+ "sys_pll1_800m", "sys_pll2_1000m", "sys_pll3_out",
+ "audio_pll2_out", "video_pll1_out", };
+
+static const char * const imx8mn_usdhc3_sels[] = {"osc_24m", "sys_pll1_400m", "sys_pll1_800m",
+ "sys_pll2_500m", "sys_pll3_out", "sys_pll1_266m",
+ "audio_pll2_clk", "sys_pll1_100m", };
+
+static const char * const imx8mn_camera_pixel_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll2_250m",
+ "sys_pll1_800m", "sys_pll2_1000m", "sys_pll3_out",
+ "audio_pll2_out", "video_pll1_out", };
+
+static const char * const imx8mn_csi1_phy_sels[] = {"osc_24m", "sys_pll2_333m", "sys_pll2_100m",
+ "sys_pll1_800m", "sys_pll2_1000m", "clk_ext2",
+ "audio_pll2_out", "video_pll1_out", };
+
+static const char * const imx8mn_csi2_phy_sels[] = {"osc_24m", "sys_pll2_333m", "sys_pll2_100m",
+ "sys_pll1_800m", "sys_pll2_1000m", "clk_ext2",
+ "audio_pll2_out", "video_pll1_out", };
+
+static const char * const imx8mn_csi2_esc_sels[] = {"osc_24m", "sys_pll2_100m", "sys_pll1_80m",
+ "sys_pll1_800m", "sys_pll2_1000m", "sys_pll3_out",
+ "clk_ext3", "audio_pll2_out", };
+
+static const char * const imx8mn_ecspi3_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m",
+ "sys_pll1_160m", "sys_pll1_800m", "sys_pll3_out",
+ "sys_pll2_250m", "audio_pll2_out", };
+
+static const char * const imx8mn_pdm_sels[] = {"osc_24m", "sys_pll2_100m", "audio_pll1_out",
+ "sys_pll1_800m", "sys_pll2_1000m", "sys_pll3_out",
+ "clk_ext3", "audio_pll2_out", };
+
+static const char * const imx8mn_dram_core_sels[] = {"dram_pll_out", "dram_alt_root", };
+
+static const char * const imx8mn_clko1_sels[] = {"osc_24m", "sys_pll1_800m", "osc_27m",
+ "sys_pll1_200m", "audio_pll2_clk", "vpu_pll",
+ "sys_pll1_80m", };
+static const char * const imx8mn_clko2_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_400m",
+ "sys_pll2_166m", "sys_pll3_out", "audio_pll1_out",
+ "video_pll1_out", "osc_32k", };
+
+static struct clk *clks[IMX8MN_CLK_END];
+static struct clk_onecell_data clk_data;
+
+static int imx8mn_clocks_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct device_node *np = dev->of_node;
+ void __iomem *base;
+ int ret;
+
+ clks[IMX8MN_CLK_DUMMY] = imx_clk_fixed("dummy", 0);
+ clks[IMX8MN_CLK_24M] = of_clk_get_by_name(np, "osc_24m");
+ clks[IMX8MN_CLK_32K] = of_clk_get_by_name(np, "osc_32k");
+ clks[IMX8MN_CLK_EXT1] = of_clk_get_by_name(np, "clk_ext1");
+ clks[IMX8MN_CLK_EXT2] = of_clk_get_by_name(np, "clk_ext2");
+ clks[IMX8MN_CLK_EXT3] = of_clk_get_by_name(np, "clk_ext3");
+ clks[IMX8MN_CLK_EXT4] = of_clk_get_by_name(np, "clk_ext4");
+
+ np = of_find_compatible_node(NULL, NULL, "fsl,imx8mn-anatop");
+ base = of_iomap(np, 0);
+ if (WARN_ON(!base)) {
+ ret = -ENOMEM;
+ goto unregister_clks;
+ }
+
+ clks[IMX8MN_AUDIO_PLL1_REF_SEL] = imx_clk_mux("audio_pll1_ref_sel", base + 0x0, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels));
+ clks[IMX8MN_AUDIO_PLL2_REF_SEL] = imx_clk_mux("audio_pll2_ref_sel", base + 0x14, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels));
+ clks[IMX8MN_VIDEO_PLL1_REF_SEL] = imx_clk_mux("video_pll1_ref_sel", base + 0x28, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels));
+ clks[IMX8MN_DRAM_PLL_REF_SEL] = imx_clk_mux("dram_pll_ref_sel", base + 0x50, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels));
+ clks[IMX8MN_GPU_PLL_REF_SEL] = imx_clk_mux("gpu_pll_ref_sel", base + 0x64, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels));
+ clks[IMX8MN_VPU_PLL_REF_SEL] = imx_clk_mux("vpu_pll_ref_sel", base + 0x74, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels));
+ clks[IMX8MN_ARM_PLL_REF_SEL] = imx_clk_mux("arm_pll_ref_sel", base + 0x84, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels));
+ clks[IMX8MN_SYS_PLL1_REF_SEL] = imx_clk_mux("sys_pll1_ref_sel", base + 0x94, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels));
+ clks[IMX8MN_SYS_PLL2_REF_SEL] = imx_clk_mux("sys_pll2_ref_sel", base + 0x104, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels));
+ clks[IMX8MN_SYS_PLL3_REF_SEL] = imx_clk_mux("sys_pll3_ref_sel", base + 0x114, 0, 2, pll_ref_sels, ARRAY_SIZE(pll_ref_sels));
+
+ clks[IMX8MN_AUDIO_PLL1] = imx_clk_pll14xx("audio_pll1", "audio_pll1_ref_sel", base, &imx8mn_audio_pll);
+ clks[IMX8MN_AUDIO_PLL2] = imx_clk_pll14xx("audio_pll2", "audio_pll2_ref_sel", base + 0x14, &imx8mn_audio_pll);
+ clks[IMX8MN_VIDEO_PLL1] = imx_clk_pll14xx("video_pll1", "video_pll1_ref_sel", base + 0x28, &imx8mn_video_pll);
+ clks[IMX8MN_DRAM_PLL] = imx_clk_pll14xx("dram_pll", "dram_pll_ref_sel", base + 0x50, &imx8mn_dram_pll);
+ clks[IMX8MN_GPU_PLL] = imx_clk_pll14xx("gpu_pll", "gpu_pll_ref_sel", base + 0x64, &imx8mn_gpu_pll);
+ clks[IMX8MN_VPU_PLL] = imx_clk_pll14xx("vpu_pll", "vpu_pll_ref_sel", base + 0x74, &imx8mn_vpu_pll);
+ clks[IMX8MN_ARM_PLL] = imx_clk_pll14xx("arm_pll", "arm_pll_ref_sel", base + 0x84, &imx8mn_arm_pll);
+ clks[IMX8MN_SYS_PLL1] = imx_clk_pll14xx("sys_pll1", "sys_pll1_ref_sel", base + 0x94, &imx8mn_sys_pll);
+ clks[IMX8MN_SYS_PLL2] = imx_clk_pll14xx("sys_pll2", "sys_pll2_ref_sel", base + 0x104, &imx8mn_sys_pll);
+ clks[IMX8MN_SYS_PLL3] = imx_clk_pll14xx("sys_pll3", "sys_pll3_ref_sel", base + 0x114, &imx8mn_sys_pll);
+
+ /* PLL bypass out */
+ clks[IMX8MN_AUDIO_PLL1_BYPASS] = imx_clk_mux_flags("audio_pll1_bypass", base, 4, 1, audio_pll1_bypass_sels, ARRAY_SIZE(audio_pll1_bypass_sels), CLK_SET_RATE_PARENT);
+ clks[IMX8MN_AUDIO_PLL2_BYPASS] = imx_clk_mux_flags("audio_pll2_bypass", base + 0x14, 4, 1, audio_pll2_bypass_sels, ARRAY_SIZE(audio_pll2_bypass_sels), CLK_SET_RATE_PARENT);
+ clks[IMX8MN_VIDEO_PLL1_BYPASS] = imx_clk_mux_flags("video_pll1_bypass", base + 0x28, 4, 1, video_pll1_bypass_sels, ARRAY_SIZE(video_pll1_bypass_sels), CLK_SET_RATE_PARENT);
+ clks[IMX8MN_DRAM_PLL_BYPASS] = imx_clk_mux_flags("dram_pll_bypass", base + 0x50, 4, 1, dram_pll_bypass_sels, ARRAY_SIZE(dram_pll_bypass_sels), CLK_SET_RATE_PARENT);
+ clks[IMX8MN_GPU_PLL_BYPASS] = imx_clk_mux_flags("gpu_pll_bypass", base + 0x64, 4, 1, gpu_pll_bypass_sels, ARRAY_SIZE(gpu_pll_bypass_sels), CLK_SET_RATE_PARENT);
+ clks[IMX8MN_VPU_PLL_BYPASS] = imx_clk_mux_flags("vpu_pll_bypass", base + 0x74, 4, 1, vpu_pll_bypass_sels, ARRAY_SIZE(vpu_pll_bypass_sels), CLK_SET_RATE_PARENT);
+ clks[IMX8MN_ARM_PLL_BYPASS] = imx_clk_mux_flags("arm_pll_bypass", base + 0x84, 4, 1, arm_pll_bypass_sels, ARRAY_SIZE(arm_pll_bypass_sels), CLK_SET_RATE_PARENT);
+ clks[IMX8MN_SYS_PLL1_BYPASS] = imx_clk_mux_flags("sys_pll1_bypass", base + 0x94, 4, 1, sys_pll1_bypass_sels, ARRAY_SIZE(sys_pll1_bypass_sels), CLK_SET_RATE_PARENT);
+ clks[IMX8MN_SYS_PLL2_BYPASS] = imx_clk_mux_flags("sys_pll2_bypass", base + 0x104, 4, 1, sys_pll2_bypass_sels, ARRAY_SIZE(sys_pll2_bypass_sels), CLK_SET_RATE_PARENT);
+ clks[IMX8MN_SYS_PLL3_BYPASS] = imx_clk_mux_flags("sys_pll3_bypass", base + 0x114, 4, 1, sys_pll3_bypass_sels, ARRAY_SIZE(sys_pll3_bypass_sels), CLK_SET_RATE_PARENT);
+
+ /* unbypass all the plls */
+ clk_set_parent(clks[IMX8MN_AUDIO_PLL1_BYPASS], clks[IMX8MN_AUDIO_PLL1]);
+ clk_set_parent(clks[IMX8MN_AUDIO_PLL2_BYPASS], clks[IMX8MN_AUDIO_PLL2]);
+ clk_set_parent(clks[IMX8MN_VIDEO_PLL1_BYPASS], clks[IMX8MN_VIDEO_PLL1]);
+ clk_set_parent(clks[IMX8MN_DRAM_PLL_BYPASS], clks[IMX8MN_DRAM_PLL]);
+ clk_set_parent(clks[IMX8MN_GPU_PLL_BYPASS], clks[IMX8MN_GPU_PLL]);
+ clk_set_parent(clks[IMX8MN_VPU_PLL_BYPASS], clks[IMX8MN_VPU_PLL]);
+ clk_set_parent(clks[IMX8MN_ARM_PLL_BYPASS], clks[IMX8MN_ARM_PLL]);
+ clk_set_parent(clks[IMX8MN_SYS_PLL1_BYPASS], clks[IMX8MN_SYS_PLL1]);
+ clk_set_parent(clks[IMX8MN_SYS_PLL2_BYPASS], clks[IMX8MN_SYS_PLL2]);
+ clk_set_parent(clks[IMX8MN_SYS_PLL3_BYPASS], clks[IMX8MN_SYS_PLL3]);
+
+ /* PLL out gate */
+ clks[IMX8MN_AUDIO_PLL1_OUT] = imx_clk_gate("audio_pll1_out", "audio_pll1_bypass", base, 13);
+ clks[IMX8MN_AUDIO_PLL2_OUT] = imx_clk_gate("audio_pll2_out", "audio_pll2_bypass", base + 0x14, 13);
+ clks[IMX8MN_VIDEO_PLL1_OUT] = imx_clk_gate("video_pll1_out", "video_pll1_bypass", base + 0x28, 13);
+ clks[IMX8MN_DRAM_PLL_OUT] = imx_clk_gate("dram_pll_out", "dram_pll_bypass", base + 0x50, 13);
+ clks[IMX8MN_GPU_PLL_OUT] = imx_clk_gate("gpu_pll_out", "gpu_pll_bypass", base + 0x64, 13);
+ clks[IMX8MN_VPU_PLL_OUT] = imx_clk_gate("vpu_pll_out", "vpu_pll_bypass", base + 0x74, 13);
+ clks[IMX8MN_ARM_PLL_OUT] = imx_clk_gate("arm_pll_out", "arm_pll_bypass", base + 0x84, 13);
+ clks[IMX8MN_SYS_PLL1_OUT] = imx_clk_gate("sys_pll1_out", "sys_pll1_bypass", base + 0x94, 13);
+ clks[IMX8MN_SYS_PLL2_OUT] = imx_clk_gate("sys_pll2_out", "sys_pll2_bypass", base + 0x104, 13);
+ clks[IMX8MN_SYS_PLL3_OUT] = imx_clk_gate("sys_pll3_out", "sys_pll3_bypass", base + 0x114, 13);
+
+ /* SYS PLL fixed output */
+ clks[IMX8MN_SYS_PLL1_40M] = imx_clk_fixed_factor("sys_pll1_40m", "sys_pll1_out", 1, 20);
+ clks[IMX8MN_SYS_PLL1_80M] = imx_clk_fixed_factor("sys_pll1_80m", "sys_pll1_out", 1, 10);
+ clks[IMX8MN_SYS_PLL1_100M] = imx_clk_fixed_factor("sys_pll1_100m", "sys_pll1_out", 1, 8);
+ clks[IMX8MN_SYS_PLL1_133M] = imx_clk_fixed_factor("sys_pll1_133m", "sys_pll1_out", 1, 6);
+ clks[IMX8MN_SYS_PLL1_160M] = imx_clk_fixed_factor("sys_pll1_160m", "sys_pll1_out", 1, 5);
+ clks[IMX8MN_SYS_PLL1_200M] = imx_clk_fixed_factor("sys_pll1_200m", "sys_pll1_out", 1, 4);
+ clks[IMX8MN_SYS_PLL1_266M] = imx_clk_fixed_factor("sys_pll1_266m", "sys_pll1_out", 1, 3);
+ clks[IMX8MN_SYS_PLL1_400M] = imx_clk_fixed_factor("sys_pll1_400m", "sys_pll1_out", 1, 2);
+ clks[IMX8MN_SYS_PLL1_800M] = imx_clk_fixed_factor("sys_pll1_800m", "sys_pll1_out", 1, 1);
+
+ clks[IMX8MN_SYS_PLL2_50M] = imx_clk_fixed_factor("sys_pll2_50m", "sys_pll2_out", 1, 20);
+ clks[IMX8MN_SYS_PLL2_100M] = imx_clk_fixed_factor("sys_pll2_100m", "sys_pll2_out", 1, 10);
+ clks[IMX8MN_SYS_PLL2_125M] = imx_clk_fixed_factor("sys_pll2_125m", "sys_pll2_out", 1, 8);
+ clks[IMX8MN_SYS_PLL2_166M] = imx_clk_fixed_factor("sys_pll2_166m", "sys_pll2_out", 1, 6);
+ clks[IMX8MN_SYS_PLL2_200M] = imx_clk_fixed_factor("sys_pll2_200m", "sys_pll2_out", 1, 5);
+ clks[IMX8MN_SYS_PLL2_250M] = imx_clk_fixed_factor("sys_pll2_250m", "sys_pll2_out", 1, 4);
+ clks[IMX8MN_SYS_PLL2_333M] = imx_clk_fixed_factor("sys_pll2_333m", "sys_pll2_out", 1, 3);
+ clks[IMX8MN_SYS_PLL2_500M] = imx_clk_fixed_factor("sys_pll2_500m", "sys_pll2_out", 1, 2);
+ clks[IMX8MN_SYS_PLL2_1000M] = imx_clk_fixed_factor("sys_pll2_1000m", "sys_pll2_out", 1, 1);
+
+ np = dev->of_node;
+ base = devm_platform_ioremap_resource(pdev, 0);
+ if (WARN_ON(IS_ERR(base))) {
+ ret = PTR_ERR(base);
+ goto unregister_clks;
+ }
+
+ /* CORE */
+ clks[IMX8MN_CLK_A53_SRC] = imx_clk_mux2("arm_a53_src", base + 0x8000, 24, 3, imx8mn_a53_sels, ARRAY_SIZE(imx8mn_a53_sels));
+ clks[IMX8MN_CLK_GPU_CORE_SRC] = imx_clk_mux2("gpu_core_src", base + 0x8180, 24, 3, imx8mn_gpu_core_sels, ARRAY_SIZE(imx8mn_gpu_core_sels));
+ clks[IMX8MN_CLK_GPU_SHADER_SRC] = imx_clk_mux2("gpu_shader_src", base + 0x8200, 24, 3, imx8mn_gpu_shader_sels, ARRAY_SIZE(imx8mn_gpu_shader_sels));
+ clks[IMX8MN_CLK_A53_CG] = imx_clk_gate3("arm_a53_cg", "arm_a53_src", base + 0x8000, 28);
+ clks[IMX8MN_CLK_GPU_CORE_CG] = imx_clk_gate3("gpu_core_cg", "gpu_core_src", base + 0x8180, 28);
+ clks[IMX8MN_CLK_GPU_SHADER_CG] = imx_clk_gate3("gpu_shader_cg", "gpu_shader_src", base + 0x8200, 28);
+
+ clks[IMX8MN_CLK_A53_DIV] = imx_clk_divider2("arm_a53_div", "arm_a53_cg", base + 0x8000, 0, 3);
+ clks[IMX8MN_CLK_GPU_CORE_DIV] = imx_clk_divider2("gpu_core_div", "gpu_core_cg", base + 0x8180, 0, 3);
+ clks[IMX8MN_CLK_GPU_SHADER_DIV] = imx_clk_divider2("gpu_shader_div", "gpu_shader_cg", base + 0x8200, 0, 3);
+
+ /* BUS */
+ clks[IMX8MN_CLK_MAIN_AXI] = imx8m_clk_composite_critical("main_axi", imx8mn_main_axi_sels, base + 0x8800);
+ clks[IMX8MN_CLK_ENET_AXI] = imx8m_clk_composite("enet_axi", imx8mn_enet_axi_sels, base + 0x8880);
+ clks[IMX8MN_CLK_NAND_USDHC_BUS] = imx8m_clk_composite("nand_usdhc_bus", imx8mn_nand_usdhc_sels, base + 0x8900);
+ clks[IMX8MN_CLK_DISP_AXI] = imx8m_clk_composite("disp_axi", imx8mn_disp_axi_sels, base + 0x8a00);
+ clks[IMX8MN_CLK_DISP_APB] = imx8m_clk_composite("disp_apb", imx8mn_disp_apb_sels, base + 0x8a80);
+ clks[IMX8MN_CLK_USB_BUS] = imx8m_clk_composite("usb_bus", imx8mn_usb_bus_sels, base + 0x8b80);
+ clks[IMX8MN_CLK_GPU_AXI] = imx8m_clk_composite("gpu_axi", imx8mn_gpu_axi_sels, base + 0x8c00);
+ clks[IMX8MN_CLK_GPU_AHB] = imx8m_clk_composite("gpu_ahb", imx8mn_gpu_ahb_sels, base + 0x8c80);
+ clks[IMX8MN_CLK_NOC] = imx8m_clk_composite_critical("noc", imx8mn_noc_sels, base + 0x8d00);
+
+ clks[IMX8MN_CLK_AHB] = imx8m_clk_composite_critical("ahb", imx8mn_ahb_sels, base + 0x9000);
+ clks[IMX8MN_CLK_AUDIO_AHB] = imx8m_clk_composite("audio_ahb", imx8mn_audio_ahb_sels, base + 0x9100);
+ clks[IMX8MN_CLK_IPG_ROOT] = imx_clk_divider2("ipg_root", "ahb", base + 0x9080, 0, 1);
+ clks[IMX8MN_CLK_IPG_AUDIO_ROOT] = imx_clk_divider2("ipg_audio_root", "audio_ahb", base + 0x9180, 0, 1);
+ clks[IMX8MN_CLK_DRAM_CORE] = imx_clk_mux2_flags("dram_core_clk", base + 0x9800, 24, 1, imx8mn_dram_core_sels, ARRAY_SIZE(imx8mn_dram_core_sels), CLK_IS_CRITICAL);
+ clks[IMX8MN_CLK_DRAM_ALT] = imx8m_clk_composite("dram_alt", imx8mn_dram_alt_sels, base + 0xa000);
+ clks[IMX8MN_CLK_DRAM_APB] = imx8m_clk_composite_critical("dram_apb", imx8mn_dram_apb_sels, base + 0xa080);
+ clks[IMX8MN_CLK_DISP_PIXEL] = imx8m_clk_composite("disp_pixel", imx8mn_disp_pixel_sels, base + 0xa500);
+ clks[IMX8MN_CLK_SAI2] = imx8m_clk_composite("sai2", imx8mn_sai2_sels, base + 0xa600);
+ clks[IMX8MN_CLK_SAI3] = imx8m_clk_composite("sai3", imx8mn_sai3_sels, base + 0xa680);
+ clks[IMX8MN_CLK_SAI5] = imx8m_clk_composite("sai5", imx8mn_sai5_sels, base + 0xa780);
+ clks[IMX8MN_CLK_SAI6] = imx8m_clk_composite("sai6", imx8mn_sai6_sels, base + 0xa800);
+ clks[IMX8MN_CLK_SPDIF1] = imx8m_clk_composite("spdif1", imx8mn_spdif1_sels, base + 0xa880);
+ clks[IMX8MN_CLK_ENET_REF] = imx8m_clk_composite("enet_ref", imx8mn_enet_ref_sels, base + 0xa980);
+ clks[IMX8MN_CLK_ENET_TIMER] = imx8m_clk_composite("enet_timer", imx8mn_enet_timer_sels, base + 0xaa00);
+ clks[IMX8MN_CLK_ENET_PHY_REF] = imx8m_clk_composite("enet_phy", imx8mn_enet_phy_sels, base + 0xaa80);
+ clks[IMX8MN_CLK_NAND] = imx8m_clk_composite("nand", imx8mn_nand_sels, base + 0xab00);
+ clks[IMX8MN_CLK_QSPI] = imx8m_clk_composite("qspi", imx8mn_qspi_sels, base + 0xab80);
+ clks[IMX8MN_CLK_USDHC1] = imx8m_clk_composite("usdhc1", imx8mn_usdhc1_sels, base + 0xac00);
+ clks[IMX8MN_CLK_USDHC2] = imx8m_clk_composite("usdhc2", imx8mn_usdhc2_sels, base + 0xac80);
+ clks[IMX8MN_CLK_I2C1] = imx8m_clk_composite("i2c1", imx8mn_i2c1_sels, base + 0xad00);
+ clks[IMX8MN_CLK_I2C2] = imx8m_clk_composite("i2c2", imx8mn_i2c2_sels, base + 0xad80);
+ clks[IMX8MN_CLK_I2C3] = imx8m_clk_composite("i2c3", imx8mn_i2c3_sels, base + 0xae00);
+ clks[IMX8MN_CLK_I2C4] = imx8m_clk_composite("i2c4", imx8mn_i2c4_sels, base + 0xae80);
+ clks[IMX8MN_CLK_UART1] = imx8m_clk_composite("uart1", imx8mn_uart1_sels, base + 0xaf00);
+ clks[IMX8MN_CLK_UART2] = imx8m_clk_composite("uart2", imx8mn_uart2_sels, base + 0xaf80);
+ clks[IMX8MN_CLK_UART3] = imx8m_clk_composite("uart3", imx8mn_uart3_sels, base + 0xb000);
+ clks[IMX8MN_CLK_UART4] = imx8m_clk_composite("uart4", imx8mn_uart4_sels, base + 0xb080);
+ clks[IMX8MN_CLK_USB_CORE_REF] = imx8m_clk_composite("usb_core_ref", imx8mn_usb_core_sels, base + 0xb100);
+ clks[IMX8MN_CLK_USB_PHY_REF] = imx8m_clk_composite("usb_phy_ref", imx8mn_usb_phy_sels, base + 0xb180);
+ clks[IMX8MN_CLK_ECSPI1] = imx8m_clk_composite("ecspi1", imx8mn_ecspi1_sels, base + 0xb280);
+ clks[IMX8MN_CLK_ECSPI2] = imx8m_clk_composite("ecspi2", imx8mn_ecspi2_sels, base + 0xb300);
+ clks[IMX8MN_CLK_PWM1] = imx8m_clk_composite("pwm1", imx8mn_pwm1_sels, base + 0xb380);
+ clks[IMX8MN_CLK_PWM2] = imx8m_clk_composite("pwm2", imx8mn_pwm2_sels, base + 0xb400);
+ clks[IMX8MN_CLK_PWM3] = imx8m_clk_composite("pwm3", imx8mn_pwm3_sels, base + 0xb480);
+ clks[IMX8MN_CLK_PWM4] = imx8m_clk_composite("pwm4", imx8mn_pwm4_sels, base + 0xb500);
+ clks[IMX8MN_CLK_WDOG] = imx8m_clk_composite("wdog", imx8mn_wdog_sels, base + 0xb900);
+ clks[IMX8MN_CLK_WRCLK] = imx8m_clk_composite("wrclk", imx8mn_wrclk_sels, base + 0xb980);
+ clks[IMX8MN_CLK_CLKO1] = imx8m_clk_composite("clko1", imx8mn_clko1_sels, base + 0xba00);
+ clks[IMX8MN_CLK_CLKO2] = imx8m_clk_composite("clko2", imx8mn_clko2_sels, base + 0xba80);
+ clks[IMX8MN_CLK_DSI_CORE] = imx8m_clk_composite("dsi_core", imx8mn_dsi_core_sels, base + 0xbb00);
+ clks[IMX8MN_CLK_DSI_PHY_REF] = imx8m_clk_composite("dsi_phy_ref", imx8mn_dsi_phy_sels, base + 0xbb80);
+ clks[IMX8MN_CLK_DSI_DBI] = imx8m_clk_composite("dsi_dbi", imx8mn_dsi_dbi_sels, base + 0xbc00);
+ clks[IMX8MN_CLK_USDHC3] = imx8m_clk_composite("usdhc3", imx8mn_usdhc3_sels, base + 0xbc80);
+ clks[IMX8MN_CLK_CAMERA_PIXEL] = imx8m_clk_composite("camera_pixel", imx8mn_camera_pixel_sels, base + 0xbd00);
+ clks[IMX8MN_CLK_CSI1_PHY_REF] = imx8m_clk_composite("csi1_phy_ref", imx8mn_csi1_phy_sels, base + 0xbd80);
+ clks[IMX8MN_CLK_CSI2_PHY_REF] = imx8m_clk_composite("csi2_phy_ref", imx8mn_csi2_phy_sels, base + 0xbf00);
+ clks[IMX8MN_CLK_CSI2_ESC] = imx8m_clk_composite("csi2_esc", imx8mn_csi2_esc_sels, base + 0xbf80);
+ clks[IMX8MN_CLK_ECSPI3] = imx8m_clk_composite("ecspi3", imx8mn_ecspi3_sels, base + 0xc180);
+ clks[IMX8MN_CLK_PDM] = imx8m_clk_composite("pdm", imx8mn_pdm_sels, base + 0xc200);
+ clks[IMX8MN_CLK_SAI7] = imx8m_clk_composite("sai7", imx8mn_sai7_sels, base + 0xc300);
+
+ clks[IMX8MN_CLK_ECSPI1_ROOT] = imx_clk_gate4("ecspi1_root_clk", "ecspi1", base + 0x4070, 0);
+ clks[IMX8MN_CLK_ECSPI2_ROOT] = imx_clk_gate4("ecspi2_root_clk", "ecspi2", base + 0x4080, 0);
+ clks[IMX8MN_CLK_ECSPI3_ROOT] = imx_clk_gate4("ecspi3_root_clk", "ecspi3", base + 0x4090, 0);
+ clks[IMX8MN_CLK_ENET1_ROOT] = imx_clk_gate4("enet1_root_clk", "enet_axi", base + 0x40a0, 0);
+ clks[IMX8MN_CLK_GPIO1_ROOT] = imx_clk_gate4("gpio1_root_clk", "ipg_root", base + 0x40b0, 0);
+ clks[IMX8MN_CLK_GPIO2_ROOT] = imx_clk_gate4("gpio2_root_clk", "ipg_root", base + 0x40c0, 0);
+ clks[IMX8MN_CLK_GPIO3_ROOT] = imx_clk_gate4("gpio3_root_clk", "ipg_root", base + 0x40d0, 0);
+ clks[IMX8MN_CLK_GPIO4_ROOT] = imx_clk_gate4("gpio4_root_clk", "ipg_root", base + 0x40e0, 0);
+ clks[IMX8MN_CLK_GPIO5_ROOT] = imx_clk_gate4("gpio5_root_clk", "ipg_root", base + 0x40f0, 0);
+ clks[IMX8MN_CLK_I2C1_ROOT] = imx_clk_gate4("i2c1_root_clk", "i2c1", base + 0x4170, 0);
+ clks[IMX8MN_CLK_I2C2_ROOT] = imx_clk_gate4("i2c2_root_clk", "i2c2", base + 0x4180, 0);
+ clks[IMX8MN_CLK_I2C3_ROOT] = imx_clk_gate4("i2c3_root_clk", "i2c3", base + 0x4190, 0);
+ clks[IMX8MN_CLK_I2C4_ROOT] = imx_clk_gate4("i2c4_root_clk", "i2c4", base + 0x41a0, 0);
+ clks[IMX8MN_CLK_MU_ROOT] = imx_clk_gate4("mu_root_clk", "ipg_root", base + 0x4210, 0);
+ clks[IMX8MN_CLK_OCOTP_ROOT] = imx_clk_gate4("ocotp_root_clk", "ipg_root", base + 0x4220, 0);
+ clks[IMX8MN_CLK_PWM1_ROOT] = imx_clk_gate4("pwm1_root_clk", "pwm1", base + 0x4280, 0);
+ clks[IMX8MN_CLK_PWM2_ROOT] = imx_clk_gate4("pwm2_root_clk", "pwm2", base + 0x4290, 0);
+ clks[IMX8MN_CLK_PWM3_ROOT] = imx_clk_gate4("pwm3_root_clk", "pwm3", base + 0x42a0, 0);
+ clks[IMX8MN_CLK_PWM4_ROOT] = imx_clk_gate4("pwm4_root_clk", "pwm4", base + 0x42b0, 0);
+ clks[IMX8MN_CLK_QSPI_ROOT] = imx_clk_gate4("qspi_root_clk", "qspi", base + 0x42f0, 0);
+ clks[IMX8MN_CLK_NAND_ROOT] = imx_clk_gate2_shared2("nand_root_clk", "nand", base + 0x4300, 0, &share_count_nand);
+ clks[IMX8MN_CLK_NAND_USDHC_BUS_RAWNAND_CLK] = imx_clk_gate2_shared2("nand_usdhc_rawnand_clk", "nand_usdhc_bus", base + 0x4300, 0, &share_count_nand);
+ clks[IMX8MN_CLK_SAI2_ROOT] = imx_clk_gate2_shared2("sai2_root_clk", "sai2", base + 0x4340, 0, &share_count_sai2);
+ clks[IMX8MN_CLK_SAI2_IPG] = imx_clk_gate2_shared2("sai2_ipg_clk", "ipg_audio_root", base + 0x4340, 0, &share_count_sai2);
+ clks[IMX8MN_CLK_SAI3_ROOT] = imx_clk_gate2_shared2("sai3_root_clk", "sai3", base + 0x4350, 0, &share_count_sai3);
+ clks[IMX8MN_CLK_SAI3_IPG] = imx_clk_gate2_shared2("sai3_ipg_clk", "ipg_audio_root", base + 0x4350, 0, &share_count_sai3);
+ clks[IMX8MN_CLK_SAI5_ROOT] = imx_clk_gate2_shared2("sai5_root_clk", "sai5", base + 0x4370, 0, &share_count_sai5);
+ clks[IMX8MN_CLK_SAI5_IPG] = imx_clk_gate2_shared2("sai5_ipg_clk", "ipg_audio_root", base + 0x4370, 0, &share_count_sai5);
+ clks[IMX8MN_CLK_SAI6_ROOT] = imx_clk_gate2_shared2("sai6_root_clk", "sai6", base + 0x4380, 0, &share_count_sai6);
+ clks[IMX8MN_CLK_SAI6_IPG] = imx_clk_gate2_shared2("sai6_ipg_clk", "ipg_audio_root", base + 0x4380, 0, &share_count_sai6);
+ clks[IMX8MN_CLK_UART1_ROOT] = imx_clk_gate4("uart1_root_clk", "uart1", base + 0x4490, 0);
+ clks[IMX8MN_CLK_UART2_ROOT] = imx_clk_gate4("uart2_root_clk", "uart2", base + 0x44a0, 0);
+ clks[IMX8MN_CLK_UART3_ROOT] = imx_clk_gate4("uart3_root_clk", "uart3", base + 0x44b0, 0);
+ clks[IMX8MN_CLK_UART4_ROOT] = imx_clk_gate4("uart4_root_clk", "uart4", base + 0x44c0, 0);
+ clks[IMX8MN_CLK_USB1_CTRL_ROOT] = imx_clk_gate4("usb1_ctrl_root_clk", "usb_core_ref", base + 0x44d0, 0);
+ clks[IMX8MN_CLK_GPU_CORE_ROOT] = imx_clk_gate4("gpu_core_root_clk", "gpu_core_div", base + 0x44f0, 0);
+ clks[IMX8MN_CLK_USDHC1_ROOT] = imx_clk_gate4("usdhc1_root_clk", "usdhc1", base + 0x4510, 0);
+ clks[IMX8MN_CLK_USDHC2_ROOT] = imx_clk_gate4("usdhc2_root_clk", "usdhc2", base + 0x4520, 0);
+ clks[IMX8MN_CLK_WDOG1_ROOT] = imx_clk_gate4("wdog1_root_clk", "wdog", base + 0x4530, 0);
+ clks[IMX8MN_CLK_WDOG2_ROOT] = imx_clk_gate4("wdog2_root_clk", "wdog", base + 0x4540, 0);
+ clks[IMX8MN_CLK_WDOG3_ROOT] = imx_clk_gate4("wdog3_root_clk", "wdog", base + 0x4550, 0);
+ clks[IMX8MN_CLK_GPU_BUS_ROOT] = imx_clk_gate4("gpu_root_clk", "gpu_axi", base + 0x4570, 0);
+ clks[IMX8MN_CLK_ASRC_ROOT] = imx_clk_gate4("asrc_root_clk", "audio_ahb", base + 0x4580, 0);
+ clks[IMX8MN_CLK_PDM_ROOT] = imx_clk_gate2_shared2("pdm_root_clk", "pdm", base + 0x45b0, 0, &share_count_pdm);
+ clks[IMX8MN_CLK_PDM_IPG] = imx_clk_gate2_shared2("pdm_ipg_clk", "ipg_audio_root", base + 0x45b0, 0, &share_count_pdm);
+ clks[IMX8MN_CLK_DISP_AXI_ROOT] = imx_clk_gate2_shared2("disp_axi_root_clk", "disp_axi", base + 0x45d0, 0, &share_count_disp);
+ clks[IMX8MN_CLK_DISP_APB_ROOT] = imx_clk_gate2_shared2("disp_apb_root_clk", "disp_apb", base + 0x45d0, 0, &share_count_disp);
+ clks[IMX8MN_CLK_CAMERA_PIXEL_ROOT] = imx_clk_gate2_shared2("camera_pixel_clk", "camera_pixel", base + 0x45d0, 0, &share_count_disp);
+ clks[IMX8MN_CLK_DISP_PIXEL_ROOT] = imx_clk_gate2_shared2("disp_pixel_clk", "disp_pixel", base + 0x45d0, 0, &share_count_disp);
+ clks[IMX8MN_CLK_USDHC3_ROOT] = imx_clk_gate4("usdhc3_root_clk", "usdhc3", base + 0x45e0, 0);
+ clks[IMX8MN_CLK_TMU_ROOT] = imx_clk_gate4("tmu_root_clk", "ipg_root", base + 0x4620, 0);
+ clks[IMX8MN_CLK_SDMA1_ROOT] = imx_clk_gate4("sdma1_clk", "ipg_root", base + 0x43a0, 0);
+ clks[IMX8MN_CLK_SDMA2_ROOT] = imx_clk_gate4("sdma2_clk", "ipg_audio_root", base + 0x43b0, 0);
+ clks[IMX8MN_CLK_SDMA3_ROOT] = imx_clk_gate4("sdma3_clk", "ipg_audio_root", base + 0x45f0, 0);
+ clks[IMX8MN_CLK_SAI7_ROOT] = imx_clk_gate2_shared2("sai7_root_clk", "sai7", base + 0x4650, 0, &share_count_sai7);
+
+ clks[IMX8MN_CLK_DRAM_ALT_ROOT] = imx_clk_fixed_factor("dram_alt_root", "dram_alt", 1, 4);
+
+ clks[IMX8MN_CLK_ARM] = imx_clk_cpu("arm", "arm_a53_div",
+ clks[IMX8MN_CLK_A53_DIV],
+ clks[IMX8MN_CLK_A53_SRC],
+ clks[IMX8MN_ARM_PLL_OUT],
+ clks[IMX8MN_CLK_24M]);
+
+ imx_check_clocks(clks, ARRAY_SIZE(clks));
+
+ clk_data.clks = clks;
+ clk_data.clk_num = ARRAY_SIZE(clks);
+ ret = of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
+ if (ret < 0) {
+ dev_err(dev, "failed to register clks for i.MX8MN\n");
+ goto unregister_clks;
+ }
+
+ return 0;
+
+unregister_clks:
+ imx_unregister_clocks(clks, ARRAY_SIZE(clks));
+
+ return ret;
+}
+
+static const struct of_device_id imx8mn_clk_of_match[] = {
+ { .compatible = "fsl,imx8mn-ccm" },
+ { /* Sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, imx8mn_clk_of_match);
+
+static struct platform_driver imx8mn_clk_driver = {
+ .probe = imx8mn_clocks_probe,
+ .driver = {
+ .name = "imx8mn-ccm",
+ .of_match_table = of_match_ptr(imx8mn_clk_of_match),
+ },
+};
+module_platform_driver(imx8mn_clk_driver);
diff --git a/drivers/clk/imx/clk-imx8mq.c b/drivers/clk/imx/clk-imx8mq.c
index daf1841b2adb..d407a07e7e6d 100644
--- a/drivers/clk/imx/clk-imx8mq.c
+++ b/drivers/clk/imx/clk-imx8mq.c
@@ -192,6 +192,9 @@ static const char * const imx8mq_usb_core_sels[] = {"osc_25m", "sys1_pll_100m",
static const char * const imx8mq_usb_phy_sels[] = {"osc_25m", "sys1_pll_100m", "sys1_pll_40m", "sys2_pll_100m",
"sys2_pll_200m", "clk_ext2", "clk_ext3", "audio_pll2_out", };
+static const char * const imx8mq_gic_sels[] = {"osc_25m", "sys2_pll_200m", "sys1_pll_40m", "sys2_pll_100m",
+ "sys2_pll_200m", "clk_ext2", "clk_ext3", "audio_pll2_out" };
+
static const char * const imx8mq_ecspi1_sels[] = {"osc_25m", "sys2_pll_200m", "sys1_pll_40m", "sys1_pll_160m",
"sys1_pll_800m", "sys3_pll2_out", "sys2_pll_250m", "audio_pll2_out", };
@@ -269,13 +272,20 @@ static const char * const imx8mq_clko2_sels[] = {"osc_25m", "sys2_pll_200m", "sy
static struct clk_onecell_data clk_data;
+static struct clk ** const uart_clks[] = {
+ &clks[IMX8MQ_CLK_UART1_ROOT],
+ &clks[IMX8MQ_CLK_UART2_ROOT],
+ &clks[IMX8MQ_CLK_UART3_ROOT],
+ &clks[IMX8MQ_CLK_UART4_ROOT],
+ NULL
+};
+
static int imx8mq_clocks_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct device_node *np = dev->of_node;
void __iomem *base;
int err;
- int i;
clks[IMX8MQ_CLK_DUMMY] = imx_clk_fixed("dummy", 0);
clks[IMX8MQ_CLK_32K] = of_clk_get_by_name(np, "ckil");
@@ -358,9 +368,9 @@ static int imx8mq_clocks_probe(struct platform_device *pdev)
clks[IMX8MQ_SYS2_PLL_1000M] = imx_clk_fixed_factor("sys2_pll_1000m", "sys2_pll_out", 1, 1);
np = dev->of_node;
- base = of_iomap(np, 0);
- if (WARN_ON(!base))
- return -ENOMEM;
+ base = devm_platform_ioremap_resource(pdev, 0);
+ if (WARN_ON(IS_ERR(base)))
+ return PTR_ERR(base);
/* CORE */
clks[IMX8MQ_CLK_A53_SRC] = imx_clk_mux2("arm_a53_src", base + 0x8000, 24, 3, imx8mq_a53_sels, ARRAY_SIZE(imx8mq_a53_sels));
@@ -442,6 +452,7 @@ static int imx8mq_clocks_probe(struct platform_device *pdev)
clks[IMX8MQ_CLK_UART4] = imx8m_clk_composite("uart4", imx8mq_uart4_sels, base + 0xb080);
clks[IMX8MQ_CLK_USB_CORE_REF] = imx8m_clk_composite("usb_core_ref", imx8mq_usb_core_sels, base + 0xb100);
clks[IMX8MQ_CLK_USB_PHY_REF] = imx8m_clk_composite("usb_phy_ref", imx8mq_usb_phy_sels, base + 0xb180);
+ clks[IMX8MQ_CLK_GIC] = imx8m_clk_composite_critical("gic", imx8mq_gic_sels, base + 0xb200);
clks[IMX8MQ_CLK_ECSPI1] = imx8m_clk_composite("ecspi1", imx8mq_ecspi1_sels, base + 0xb280);
clks[IMX8MQ_CLK_ECSPI2] = imx8m_clk_composite("ecspi2", imx8mq_ecspi2_sels, base + 0xb300);
clks[IMX8MQ_CLK_PWM1] = imx8m_clk_composite("pwm1", imx8mq_pwm1_sels, base + 0xb380);
@@ -507,6 +518,7 @@ static int imx8mq_clocks_probe(struct platform_device *pdev)
clks[IMX8MQ_CLK_SAI5_IPG] = imx_clk_gate2_shared2("sai5_ipg_clk", "ipg_audio_root", base + 0x4370, 0, &share_count_sai5);
clks[IMX8MQ_CLK_SAI6_ROOT] = imx_clk_gate2_shared2("sai6_root_clk", "sai6", base + 0x4380, 0, &share_count_sai6);
clks[IMX8MQ_CLK_SAI6_IPG] = imx_clk_gate2_shared2("sai6_ipg_clk", "ipg_audio_root", base + 0x4380, 0, &share_count_sai6);
+ clks[IMX8MQ_CLK_SNVS_ROOT] = imx_clk_gate4("snvs_root_clk", "ipg_root", base + 0x4470, 0);
clks[IMX8MQ_CLK_UART1_ROOT] = imx_clk_gate4("uart1_root_clk", "uart1", base + 0x4490, 0);
clks[IMX8MQ_CLK_UART2_ROOT] = imx_clk_gate4("uart2_root_clk", "uart2", base + 0x44a0, 0);
clks[IMX8MQ_CLK_UART3_ROOT] = imx_clk_gate4("uart3_root_clk", "uart3", base + 0x44b0, 0);
@@ -543,10 +555,7 @@ static int imx8mq_clocks_probe(struct platform_device *pdev)
clks[IMX8MQ_ARM_PLL_OUT],
clks[IMX8MQ_SYS1_PLL_800M]);
- for (i = 0; i < IMX8MQ_CLK_END; i++)
- if (IS_ERR(clks[i]))
- pr_err("i.MX8mq clk %u register failed with %ld\n",
- i, PTR_ERR(clks[i]));
+ imx_check_clocks(clks, ARRAY_SIZE(clks));
clk_data.clks = clks;
clk_data.clk_num = ARRAY_SIZE(clks);
@@ -554,6 +563,8 @@ static int imx8mq_clocks_probe(struct platform_device *pdev)
err = of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
WARN_ON(err);
+ imx_register_uart_clocks(uart_clks);
+
return err;
}
diff --git a/drivers/clk/imx/clk-imx8qxp-lpcg.c b/drivers/clk/imx/clk-imx8qxp-lpcg.c
index fb6edf1b8aa2..c0aff7ca6374 100644
--- a/drivers/clk/imx/clk-imx8qxp-lpcg.c
+++ b/drivers/clk/imx/clk-imx8qxp-lpcg.c
@@ -72,6 +72,11 @@ static const struct imx8qxp_lpcg_data imx8qxp_lpcg_adma[] = {
{ IMX_ADMA_LPCG_I2C2_CLK, "i2c2_lpcg_clk", "i2c2_clk", 0, ADMA_LPI2C_2_LPCG, 0, 0, },
{ IMX_ADMA_LPCG_I2C3_IPG_CLK, "i2c3_lpcg_ipg_clk", "dma_ipg_clk_root", 0, ADMA_LPI2C_3_LPCG, 16, 0, },
{ IMX_ADMA_LPCG_I2C3_CLK, "i2c3_lpcg_clk", "i2c3_clk", 0, ADMA_LPI2C_3_LPCG, 0, 0, },
+
+ { IMX_ADMA_LPCG_DSP_CORE_CLK, "dsp_lpcg_core_clk", "dma_ipg_clk_root", 0, ADMA_HIFI_LPCG, 28, 0, },
+ { IMX_ADMA_LPCG_DSP_IPG_CLK, "dsp_lpcg_ipg_clk", "dma_ipg_clk_root", 0, ADMA_HIFI_LPCG, 20, 0, },
+ { IMX_ADMA_LPCG_DSP_ADB_CLK, "dsp_lpcg_adb_clk", "dma_ipg_clk_root", 0, ADMA_HIFI_LPCG, 16, 0, },
+ { IMX_ADMA_LPCG_OCRAM_IPG_CLK, "ocram_lpcg_ipg_clk", "dma_ipg_clk_root", 0, ADMA_OCRAM_LPCG, 16, 0, },
};
static const struct imx8qxp_ss_lpcg imx8qxp_ss_adma = {
diff --git a/drivers/clk/imx/clk-pfd.c b/drivers/clk/imx/clk-pfd.c
index fa82ddea4996..50b7c30296f7 100644
--- a/drivers/clk/imx/clk-pfd.c
+++ b/drivers/clk/imx/clk-pfd.c
@@ -121,12 +121,13 @@ static const struct clk_ops clk_pfd_ops = {
.is_enabled = clk_pfd_is_enabled,
};
-struct clk *imx_clk_pfd(const char *name, const char *parent_name,
+struct clk_hw *imx_clk_hw_pfd(const char *name, const char *parent_name,
void __iomem *reg, u8 idx)
{
struct clk_pfd *pfd;
- struct clk *clk;
+ struct clk_hw *hw;
struct clk_init_data init;
+ int ret;
pfd = kzalloc(sizeof(*pfd), GFP_KERNEL);
if (!pfd)
@@ -142,10 +143,13 @@ struct clk *imx_clk_pfd(const char *name, const char *parent_name,
init.num_parents = 1;
pfd->hw.init = &init;
+ hw = &pfd->hw;
- clk = clk_register(NULL, &pfd->hw);
- if (IS_ERR(clk))
+ ret = clk_hw_register(NULL, hw);
+ if (ret) {
kfree(pfd);
+ return ERR_PTR(ret);
+ }
- return clk;
+ return hw;
}
diff --git a/drivers/clk/imx/clk-pllv3.c b/drivers/clk/imx/clk-pllv3.c
index 93b059608d3c..df91a8244fb4 100644
--- a/drivers/clk/imx/clk-pllv3.c
+++ b/drivers/clk/imx/clk-pllv3.c
@@ -410,14 +410,15 @@ static const struct clk_ops clk_pllv3_enet_ops = {
.recalc_rate = clk_pllv3_enet_recalc_rate,
};
-struct clk *imx_clk_pllv3(enum imx_pllv3_type type, const char *name,
+struct clk_hw *imx_clk_hw_pllv3(enum imx_pllv3_type type, const char *name,
const char *parent_name, void __iomem *base,
u32 div_mask)
{
struct clk_pllv3 *pll;
const struct clk_ops *ops;
- struct clk *clk;
+ struct clk_hw *hw;
struct clk_init_data init;
+ int ret;
pll = kzalloc(sizeof(*pll), GFP_KERNEL);
if (!pll)
@@ -478,10 +479,13 @@ struct clk *imx_clk_pllv3(enum imx_pllv3_type type, const char *name,
init.num_parents = 1;
pll->hw.init = &init;
+ hw = &pll->hw;
- clk = clk_register(NULL, &pll->hw);
- if (IS_ERR(clk))
+ ret = clk_hw_register(NULL, hw);
+ if (ret) {
kfree(pll);
+ return ERR_PTR(ret);
+ }
- return clk;
+ return hw;
}
diff --git a/drivers/clk/imx/clk.c b/drivers/clk/imx/clk.c
index 1efed86217f7..cfc05e43c343 100644
--- a/drivers/clk/imx/clk.c
+++ b/drivers/clk/imx/clk.c
@@ -1,14 +1,38 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/clk.h>
+#include <linux/clk-provider.h>
#include <linux/err.h>
+#include <linux/io.h>
#include <linux/of.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include "clk.h"
+#define CCM_CCDR 0x4
+#define CCDR_MMDC_CH0_MASK BIT(17)
+#define CCDR_MMDC_CH1_MASK BIT(16)
+
DEFINE_SPINLOCK(imx_ccm_lock);
-void __init imx_check_clocks(struct clk *clks[], unsigned int count)
+void imx_unregister_clocks(struct clk *clks[], unsigned int count)
+{
+ unsigned int i;
+
+ for (i = 0; i < count; i++)
+ clk_unregister(clks[i]);
+}
+
+void __init imx_mmdc_mask_handshake(void __iomem *ccm_base,
+ unsigned int chn)
+{
+ unsigned int reg;
+
+ reg = readl_relaxed(ccm_base + CCM_CCDR);
+ reg |= chn == 0 ? CCDR_MMDC_CH0_MASK : CCDR_MMDC_CH1_MASK;
+ writel_relaxed(reg, ccm_base + CCM_CCDR);
+}
+
+void imx_check_clocks(struct clk *clks[], unsigned int count)
{
unsigned i;
@@ -59,6 +83,17 @@ struct clk * __init imx_obtain_fixed_clock(
return clk;
}
+struct clk_hw * __init imx_obtain_fixed_clock_hw(
+ const char *name, unsigned long rate)
+{
+ struct clk *clk;
+
+ clk = imx_obtain_fixed_clock_from_dt(name);
+ if (IS_ERR(clk))
+ clk = imx_clk_fixed(name, rate);
+ return __clk_get_hw(clk);
+}
+
struct clk_hw * __init imx_obtain_fixed_clk_hw(struct device_node *np,
const char *name)
{
@@ -97,8 +132,8 @@ void imx_cscmr1_fixup(u32 *val)
return;
}
-static int imx_keep_uart_clocks __initdata;
-static struct clk ** const *imx_uart_clocks __initdata;
+static int imx_keep_uart_clocks;
+static struct clk ** const *imx_uart_clocks;
static int __init imx_keep_uart_clocks_param(char *str)
{
@@ -111,7 +146,7 @@ __setup_param("earlycon", imx_keep_uart_earlycon,
__setup_param("earlyprintk", imx_keep_uart_earlyprintk,
imx_keep_uart_clocks_param, 0);
-void __init imx_register_uart_clocks(struct clk ** const clks[])
+void imx_register_uart_clocks(struct clk ** const clks[])
{
if (imx_keep_uart_clocks) {
int i;
diff --git a/drivers/clk/imx/clk.h b/drivers/clk/imx/clk.h
index 8639a8f2153e..bb4ec1b33faf 100644
--- a/drivers/clk/imx/clk.h
+++ b/drivers/clk/imx/clk.h
@@ -10,6 +10,9 @@ extern spinlock_t imx_ccm_lock;
void imx_check_clocks(struct clk *clks[], unsigned int count);
void imx_check_clk_hws(struct clk_hw *clks[], unsigned int count);
void imx_register_uart_clocks(struct clk ** const clks[]);
+void imx_register_uart_clocks_hws(struct clk_hw ** const hws[]);
+void imx_mmdc_mask_handshake(void __iomem *ccm_base, unsigned int chn);
+void imx_unregister_clocks(struct clk *clks[], unsigned int count);
extern void imx_cscmr1_fixup(u32 *val);
@@ -48,6 +51,74 @@ struct imx_pll14xx_clk {
int flags;
};
+#define imx_clk_busy_divider(name, parent_name, reg, shift, width, busy_reg, busy_shift) \
+ imx_clk_hw_busy_divider(name, parent_name, reg, shift, width, busy_reg, busy_shift)->clk
+
+#define imx_clk_busy_mux(name, reg, shift, width, busy_reg, busy_shift, parent_names, num_parents) \
+ imx_clk_hw_busy_mux(name, reg, shift, width, busy_reg, busy_shift, parent_names, num_parents)->clk
+
+#define imx_clk_cpu(name, parent_name, div, mux, pll, step) \
+ imx_clk_hw_cpu(name, parent_name, div, mux, pll, step)->clk
+
+#define clk_register_gate2(dev, name, parent_name, flags, reg, bit_idx, \
+ cgr_val, clk_gate_flags, lock, share_count) \
+ clk_hw_register_gate2(dev, name, parent_name, flags, reg, bit_idx, \
+ cgr_val, clk_gate_flags, lock, share_count)->clk
+
+#define imx_clk_pllv3(type, name, parent_name, base, div_mask) \
+ imx_clk_hw_pllv3(type, name, parent_name, base, div_mask)->clk
+
+#define imx_clk_pfd(name, parent_name, reg, idx) \
+ imx_clk_hw_pfd(name, parent_name, reg, idx)->clk
+
+#define imx_clk_gate_exclusive(name, parent, reg, shift, exclusive_mask) \
+ imx_clk_hw_gate_exclusive(name, parent, reg, shift, exclusive_mask)->clk
+
+#define imx_clk_fixup_divider(name, parent, reg, shift, width, fixup) \
+ imx_clk_hw_fixup_divider(name, parent, reg, shift, width, fixup)->clk
+
+#define imx_clk_fixup_mux(name, reg, shift, width, parents, num_parents, fixup) \
+ imx_clk_hw_fixup_mux(name, reg, shift, width, parents, num_parents, fixup)->clk
+
+#define imx_clk_mux_ldb(name, reg, shift, width, parents, num_parents) \
+ imx_clk_hw_mux_ldb(name, reg, shift, width, parents, num_parents)->clk
+
+#define imx_clk_fixed_factor(name, parent, mult, div) \
+ imx_clk_hw_fixed_factor(name, parent, mult, div)->clk
+
+#define imx_clk_divider2(name, parent, reg, shift, width) \
+ imx_clk_hw_divider2(name, parent, reg, shift, width)->clk
+
+#define imx_clk_gate_dis(name, parent, reg, shift) \
+ imx_clk_hw_gate_dis(name, parent, reg, shift)->clk
+
+#define imx_clk_gate_dis_flags(name, parent, reg, shift, flags) \
+ imx_clk_hw_gate_dis_flags(name, parent, reg, shift, flags)->clk
+
+#define imx_clk_gate_flags(name, parent, reg, shift, flags) \
+ imx_clk_hw_gate_flags(name, parent, reg, shift, flags)->clk
+
+#define imx_clk_gate2(name, parent, reg, shift) \
+ imx_clk_hw_gate2(name, parent, reg, shift)->clk
+
+#define imx_clk_gate2_flags(name, parent, reg, shift, flags) \
+ imx_clk_hw_gate2_flags(name, parent, reg, shift, flags)->clk
+
+#define imx_clk_gate2_shared(name, parent, reg, shift, share_count) \
+ imx_clk_hw_gate2_shared(name, parent, reg, shift, share_count)->clk
+
+#define imx_clk_gate2_shared2(name, parent, reg, shift, share_count) \
+ imx_clk_hw_gate2_shared2(name, parent, reg, shift, share_count)->clk
+
+#define imx_clk_gate3(name, parent, reg, shift) \
+ imx_clk_hw_gate3(name, parent, reg, shift)->clk
+
+#define imx_clk_gate4(name, parent, reg, shift) \
+ imx_clk_hw_gate4(name, parent, reg, shift)->clk
+
+#define imx_clk_mux(name, reg, shift, width, parents, num_parents) \
+ imx_clk_hw_mux(name, reg, shift, width, parents, num_parents)->clk
+
struct clk *imx_clk_pll14xx(const char *name, const char *parent_name,
void __iomem *base, const struct imx_pll14xx_clk *pll_clk);
@@ -80,13 +151,30 @@ enum imx_pllv3_type {
IMX_PLLV3_AV_IMX7,
};
-struct clk *imx_clk_pllv3(enum imx_pllv3_type type, const char *name,
+struct clk_hw *imx_clk_hw_pllv3(enum imx_pllv3_type type, const char *name,
const char *parent_name, void __iomem *base, u32 div_mask);
+#define PLL_1416X_RATE(_rate, _m, _p, _s) \
+ { \
+ .rate = (_rate), \
+ .mdiv = (_m), \
+ .pdiv = (_p), \
+ .sdiv = (_s), \
+ }
+
+#define PLL_1443X_RATE(_rate, _m, _p, _s, _k) \
+ { \
+ .rate = (_rate), \
+ .mdiv = (_m), \
+ .pdiv = (_p), \
+ .sdiv = (_s), \
+ .kdiv = (_k), \
+ }
+
struct clk_hw *imx_clk_pllv4(const char *name, const char *parent_name,
void __iomem *base);
-struct clk *clk_register_gate2(struct device *dev, const char *name,
+struct clk_hw *clk_hw_register_gate2(struct device *dev, const char *name,
const char *parent_name, unsigned long flags,
void __iomem *reg, u8 bit_idx, u8 cgr_val,
u8 clk_gate_flags, spinlock_t *lock,
@@ -95,23 +183,26 @@ struct clk *clk_register_gate2(struct device *dev, const char *name,
struct clk * imx_obtain_fixed_clock(
const char *name, unsigned long rate);
+struct clk_hw *imx_obtain_fixed_clock_hw(
+ const char *name, unsigned long rate);
+
struct clk_hw *imx_obtain_fixed_clk_hw(struct device_node *np,
const char *name);
-struct clk *imx_clk_gate_exclusive(const char *name, const char *parent,
+struct clk_hw *imx_clk_hw_gate_exclusive(const char *name, const char *parent,
void __iomem *reg, u8 shift, u32 exclusive_mask);
-struct clk *imx_clk_pfd(const char *name, const char *parent_name,
+struct clk_hw *imx_clk_hw_pfd(const char *name, const char *parent_name,
void __iomem *reg, u8 idx);
struct clk_hw *imx_clk_pfdv2(const char *name, const char *parent_name,
void __iomem *reg, u8 idx);
-struct clk *imx_clk_busy_divider(const char *name, const char *parent_name,
+struct clk_hw *imx_clk_hw_busy_divider(const char *name, const char *parent_name,
void __iomem *reg, u8 shift, u8 width,
void __iomem *busy_reg, u8 busy_shift);
-struct clk *imx_clk_busy_mux(const char *name, void __iomem *reg, u8 shift,
+struct clk_hw *imx_clk_hw_busy_mux(const char *name, void __iomem *reg, u8 shift,
u8 width, void __iomem *busy_reg, u8 busy_shift,
const char * const *parent_names, int num_parents);
@@ -121,11 +212,11 @@ struct clk_hw *imx7ulp_clk_composite(const char *name,
bool rate_present, bool gate_present,
void __iomem *reg);
-struct clk *imx_clk_fixup_divider(const char *name, const char *parent,
+struct clk_hw *imx_clk_hw_fixup_divider(const char *name, const char *parent,
void __iomem *reg, u8 shift, u8 width,
void (*fixup)(u32 *val));
-struct clk *imx_clk_fixup_mux(const char *name, void __iomem *reg,
+struct clk_hw *imx_clk_hw_fixup_mux(const char *name, void __iomem *reg,
u8 shift, u8 width, const char * const *parents,
int num_parents, void (*fixup)(u32 *val));
@@ -139,19 +230,19 @@ static inline struct clk_hw *imx_clk_hw_fixed(const char *name, int rate)
return clk_hw_register_fixed_rate(NULL, name, NULL, 0, rate);
}
-static inline struct clk *imx_clk_mux_ldb(const char *name, void __iomem *reg,
+static inline struct clk_hw *imx_clk_hw_mux_ldb(const char *name, void __iomem *reg,
u8 shift, u8 width, const char * const *parents,
int num_parents)
{
- return clk_register_mux(NULL, name, parents, num_parents,
+ return clk_hw_register_mux(NULL, name, parents, num_parents,
CLK_SET_RATE_NO_REPARENT | CLK_SET_RATE_PARENT, reg,
shift, width, CLK_MUX_READ_ONLY, &imx_ccm_lock);
}
-static inline struct clk *imx_clk_fixed_factor(const char *name,
+static inline struct clk_hw *imx_clk_hw_fixed_factor(const char *name,
const char *parent, unsigned int mult, unsigned int div)
{
- return clk_register_fixed_factor(NULL, name, parent,
+ return clk_hw_register_fixed_factor(NULL, name, parent,
CLK_SET_RATE_PARENT, mult, div);
}
@@ -188,10 +279,10 @@ static inline struct clk_hw *imx_clk_hw_divider_flags(const char *name,
reg, shift, width, 0, &imx_ccm_lock);
}
-static inline struct clk *imx_clk_divider2(const char *name, const char *parent,
+static inline struct clk_hw *imx_clk_hw_divider2(const char *name, const char *parent,
void __iomem *reg, u8 shift, u8 width)
{
- return clk_register_divider(NULL, name, parent,
+ return clk_hw_register_divider(NULL, name, parent,
CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE,
reg, shift, width, 0, &imx_ccm_lock);
}
@@ -212,10 +303,10 @@ static inline struct clk *imx_clk_gate(const char *name, const char *parent,
shift, 0, &imx_ccm_lock);
}
-static inline struct clk *imx_clk_gate_flags(const char *name, const char *parent,
+static inline struct clk_hw *imx_clk_hw_gate_flags(const char *name, const char *parent,
void __iomem *reg, u8 shift, unsigned long flags)
{
- return clk_register_gate(NULL, name, parent, flags | CLK_SET_RATE_PARENT, reg,
+ return clk_hw_register_gate(NULL, name, parent, flags | CLK_SET_RATE_PARENT, reg,
shift, 0, &imx_ccm_lock);
}
@@ -226,47 +317,47 @@ static inline struct clk_hw *imx_clk_hw_gate(const char *name, const char *paren
shift, 0, &imx_ccm_lock);
}
-static inline struct clk *imx_clk_gate_dis(const char *name, const char *parent,
+static inline struct clk_hw *imx_clk_hw_gate_dis(const char *name, const char *parent,
void __iomem *reg, u8 shift)
{
- return clk_register_gate(NULL, name, parent, CLK_SET_RATE_PARENT, reg,
+ return clk_hw_register_gate(NULL, name, parent, CLK_SET_RATE_PARENT, reg,
shift, CLK_GATE_SET_TO_DISABLE, &imx_ccm_lock);
}
-static inline struct clk *imx_clk_gate_dis_flags(const char *name, const char *parent,
+static inline struct clk_hw *imx_clk_hw_gate_dis_flags(const char *name, const char *parent,
void __iomem *reg, u8 shift, unsigned long flags)
{
- return clk_register_gate(NULL, name, parent, flags | CLK_SET_RATE_PARENT, reg,
+ return clk_hw_register_gate(NULL, name, parent, flags | CLK_SET_RATE_PARENT, reg,
shift, CLK_GATE_SET_TO_DISABLE, &imx_ccm_lock);
}
-static inline struct clk *imx_clk_gate2(const char *name, const char *parent,
+static inline struct clk_hw *imx_clk_hw_gate2(const char *name, const char *parent,
void __iomem *reg, u8 shift)
{
- return clk_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT, reg,
+ return clk_hw_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT, reg,
shift, 0x3, 0, &imx_ccm_lock, NULL);
}
-static inline struct clk *imx_clk_gate2_flags(const char *name, const char *parent,
+static inline struct clk_hw *imx_clk_hw_gate2_flags(const char *name, const char *parent,
void __iomem *reg, u8 shift, unsigned long flags)
{
- return clk_register_gate2(NULL, name, parent, flags | CLK_SET_RATE_PARENT, reg,
+ return clk_hw_register_gate2(NULL, name, parent, flags | CLK_SET_RATE_PARENT, reg,
shift, 0x3, 0, &imx_ccm_lock, NULL);
}
-static inline struct clk *imx_clk_gate2_shared(const char *name,
+static inline struct clk_hw *imx_clk_hw_gate2_shared(const char *name,
const char *parent, void __iomem *reg, u8 shift,
unsigned int *share_count)
{
- return clk_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT, reg,
+ return clk_hw_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT, reg,
shift, 0x3, 0, &imx_ccm_lock, share_count);
}
-static inline struct clk *imx_clk_gate2_shared2(const char *name,
+static inline struct clk_hw *imx_clk_hw_gate2_shared2(const char *name,
const char *parent, void __iomem *reg, u8 shift,
unsigned int *share_count)
{
- return clk_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT |
+ return clk_hw_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT |
CLK_OPS_PARENT_ENABLE, reg, shift, 0x3, 0,
&imx_ccm_lock, share_count);
}
@@ -278,10 +369,10 @@ static inline struct clk *imx_clk_gate2_cgr(const char *name,
shift, cgr_val, 0, &imx_ccm_lock, NULL);
}
-static inline struct clk *imx_clk_gate3(const char *name, const char *parent,
+static inline struct clk_hw *imx_clk_hw_gate3(const char *name, const char *parent,
void __iomem *reg, u8 shift)
{
- return clk_register_gate(NULL, name, parent,
+ return clk_hw_register_gate(NULL, name, parent,
CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE,
reg, shift, 0, &imx_ccm_lock);
}
@@ -295,10 +386,10 @@ static inline struct clk *imx_clk_gate3_flags(const char *name,
reg, shift, 0, &imx_ccm_lock);
}
-static inline struct clk *imx_clk_gate4(const char *name, const char *parent,
+static inline struct clk_hw *imx_clk_hw_gate4(const char *name, const char *parent,
void __iomem *reg, u8 shift)
{
- return clk_register_gate2(NULL, name, parent,
+ return clk_hw_register_gate2(NULL, name, parent,
CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE,
reg, shift, 0x3, 0, &imx_ccm_lock, NULL);
}
@@ -312,11 +403,11 @@ static inline struct clk *imx_clk_gate4_flags(const char *name,
reg, shift, 0x3, 0, &imx_ccm_lock, NULL);
}
-static inline struct clk *imx_clk_mux(const char *name, void __iomem *reg,
+static inline struct clk_hw *imx_clk_hw_mux(const char *name, void __iomem *reg,
u8 shift, u8 width, const char * const *parents,
int num_parents)
{
- return clk_register_mux(NULL, name, parents, num_parents,
+ return clk_hw_register_mux(NULL, name, parents, num_parents,
CLK_SET_RATE_NO_REPARENT, reg, shift,
width, 0, &imx_ccm_lock);
}
@@ -373,7 +464,7 @@ static inline struct clk_hw *imx_clk_hw_mux_flags(const char *name,
reg, shift, width, 0, &imx_ccm_lock);
}
-struct clk *imx_clk_cpu(const char *name, const char *parent_name,
+struct clk_hw *imx_clk_hw_cpu(const char *name, const char *parent_name,
struct clk *div, struct clk *mux, struct clk *pll,
struct clk *step);
diff --git a/drivers/clk/ingenic/Makefile b/drivers/clk/ingenic/Makefile
index ab58a6a862a5..250570a809d3 100644
--- a/drivers/clk/ingenic/Makefile
+++ b/drivers/clk/ingenic/Makefile
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: GPL-2.0-only
-obj-$(CONFIG_INGENIC_CGU_COMMON) += cgu.o
+obj-$(CONFIG_INGENIC_CGU_COMMON) += cgu.o pm.o
obj-$(CONFIG_INGENIC_CGU_JZ4740) += jz4740-cgu.o
obj-$(CONFIG_INGENIC_CGU_JZ4725B) += jz4725b-cgu.o
obj-$(CONFIG_INGENIC_CGU_JZ4770) += jz4770-cgu.o
diff --git a/drivers/clk/ingenic/cgu.c b/drivers/clk/ingenic/cgu.c
index 92c331427513..6e963031cd87 100644
--- a/drivers/clk/ingenic/cgu.c
+++ b/drivers/clk/ingenic/cgu.c
@@ -375,8 +375,11 @@ ingenic_clk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
div_reg = readl(cgu->base + clk_info->div.reg);
div = (div_reg >> clk_info->div.shift) &
GENMASK(clk_info->div.bits - 1, 0);
- div += 1;
- div *= clk_info->div.div;
+
+ if (clk_info->div.div_table)
+ div = clk_info->div.div_table[div];
+ else
+ div = (div + 1) * clk_info->div.div;
rate /= div;
} else if (clk_info->type & CGU_CLK_FIXDIV) {
@@ -386,16 +389,37 @@ ingenic_clk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
return rate;
}
+static unsigned int
+ingenic_clk_calc_hw_div(const struct ingenic_cgu_clk_info *clk_info,
+ unsigned int div)
+{
+ unsigned int i;
+
+ for (i = 0; i < (1 << clk_info->div.bits)
+ && clk_info->div.div_table[i]; i++) {
+ if (clk_info->div.div_table[i] >= div)
+ return i;
+ }
+
+ return i - 1;
+}
+
static unsigned
ingenic_clk_calc_div(const struct ingenic_cgu_clk_info *clk_info,
unsigned long parent_rate, unsigned long req_rate)
{
- unsigned div;
+ unsigned int div, hw_div;
/* calculate the divide */
div = DIV_ROUND_UP(parent_rate, req_rate);
- /* and impose hardware constraints */
+ if (clk_info->div.div_table) {
+ hw_div = ingenic_clk_calc_hw_div(clk_info, div);
+
+ return clk_info->div.div_table[hw_div];
+ }
+
+ /* Impose hardware constraints */
div = min_t(unsigned, div, 1 << clk_info->div.bits);
div = max_t(unsigned, div, 1);
@@ -438,7 +462,7 @@ ingenic_clk_set_rate(struct clk_hw *hw, unsigned long req_rate,
const struct ingenic_cgu_clk_info *clk_info;
const unsigned timeout = 100;
unsigned long rate, flags;
- unsigned div, i;
+ unsigned int hw_div, div, i;
u32 reg, mask;
int ret = 0;
@@ -451,13 +475,18 @@ ingenic_clk_set_rate(struct clk_hw *hw, unsigned long req_rate,
if (rate != req_rate)
return -EINVAL;
+ if (clk_info->div.div_table)
+ hw_div = ingenic_clk_calc_hw_div(clk_info, div);
+ else
+ hw_div = ((div / clk_info->div.div) - 1);
+
spin_lock_irqsave(&cgu->lock, flags);
reg = readl(cgu->base + clk_info->div.reg);
/* update the divide */
mask = GENMASK(clk_info->div.bits - 1, 0);
reg &= ~(mask << clk_info->div.shift);
- reg |= ((div / clk_info->div.div) - 1) << clk_info->div.shift;
+ reg |= hw_div << clk_info->div.shift;
/* clear the stop bit */
if (clk_info->div.stop_bit != -1)
diff --git a/drivers/clk/ingenic/cgu.h b/drivers/clk/ingenic/cgu.h
index bfbcf6db437d..0dc8004079ee 100644
--- a/drivers/clk/ingenic/cgu.h
+++ b/drivers/clk/ingenic/cgu.h
@@ -10,6 +10,7 @@
#define __DRIVERS_CLK_INGENIC_CGU_H__
#include <linux/bitops.h>
+#include <linux/clk-provider.h>
#include <linux/of.h>
#include <linux/spinlock.h>
@@ -79,6 +80,8 @@ struct ingenic_cgu_mux_info {
* isn't one
* @busy_bit: the index of the busy bit within reg, or -1 if there isn't one
* @stop_bit: the index of the stop bit within reg, or -1 if there isn't one
+ * @div_table: optional table to map the value read from the register to the
+ * actual divider value
*/
struct ingenic_cgu_div_info {
unsigned reg;
@@ -88,6 +91,7 @@ struct ingenic_cgu_div_info {
s8 ce_bit;
s8 busy_bit;
s8 stop_bit;
+ const u8 *div_table;
};
/**
diff --git a/drivers/clk/ingenic/jz4725b-cgu.c b/drivers/clk/ingenic/jz4725b-cgu.c
index 8901ea0295b7..2642d36d1e2c 100644
--- a/drivers/clk/ingenic/jz4725b-cgu.c
+++ b/drivers/clk/ingenic/jz4725b-cgu.c
@@ -11,6 +11,7 @@
#include <linux/of.h>
#include <dt-bindings/clock/jz4725b-cgu.h>
#include "cgu.h"
+#include "pm.h"
/* CGU register offsets */
#define CGU_REG_CPCCR 0x00
@@ -33,6 +34,14 @@ static const s8 pll_od_encoding[4] = {
0x0, 0x1, -1, 0x3,
};
+static const u8 jz4725b_cgu_cpccr_div_table[] = {
+ 1, 2, 3, 4, 6, 8,
+};
+
+static const u8 jz4725b_cgu_pll_half_div_table[] = {
+ 2, 1,
+};
+
static const struct ingenic_cgu_clk_info jz4725b_cgu_clocks[] = {
/* External clocks */
@@ -66,37 +75,55 @@ static const struct ingenic_cgu_clk_info jz4725b_cgu_clocks[] = {
[JZ4725B_CLK_PLL_HALF] = {
"pll half", CGU_CLK_DIV,
.parents = { JZ4725B_CLK_PLL, -1, -1, -1 },
- .div = { CGU_REG_CPCCR, 21, 1, 1, -1, -1, -1 },
+ .div = {
+ CGU_REG_CPCCR, 21, 1, 1, -1, -1, -1,
+ jz4725b_cgu_pll_half_div_table,
+ },
},
[JZ4725B_CLK_CCLK] = {
"cclk", CGU_CLK_DIV,
.parents = { JZ4725B_CLK_PLL, -1, -1, -1 },
- .div = { CGU_REG_CPCCR, 0, 1, 4, 22, -1, -1 },
+ .div = {
+ CGU_REG_CPCCR, 0, 1, 4, 22, -1, -1,
+ jz4725b_cgu_cpccr_div_table,
+ },
},
[JZ4725B_CLK_HCLK] = {
"hclk", CGU_CLK_DIV,
.parents = { JZ4725B_CLK_PLL, -1, -1, -1 },
- .div = { CGU_REG_CPCCR, 4, 1, 4, 22, -1, -1 },
+ .div = {
+ CGU_REG_CPCCR, 4, 1, 4, 22, -1, -1,
+ jz4725b_cgu_cpccr_div_table,
+ },
},
[JZ4725B_CLK_PCLK] = {
"pclk", CGU_CLK_DIV,
.parents = { JZ4725B_CLK_PLL, -1, -1, -1 },
- .div = { CGU_REG_CPCCR, 8, 1, 4, 22, -1, -1 },
+ .div = {
+ CGU_REG_CPCCR, 8, 1, 4, 22, -1, -1,
+ jz4725b_cgu_cpccr_div_table,
+ },
},
[JZ4725B_CLK_MCLK] = {
"mclk", CGU_CLK_DIV,
.parents = { JZ4725B_CLK_PLL, -1, -1, -1 },
- .div = { CGU_REG_CPCCR, 12, 1, 4, 22, -1, -1 },
+ .div = {
+ CGU_REG_CPCCR, 12, 1, 4, 22, -1, -1,
+ jz4725b_cgu_cpccr_div_table,
+ },
},
[JZ4725B_CLK_IPU] = {
"ipu", CGU_CLK_DIV | CGU_CLK_GATE,
.parents = { JZ4725B_CLK_PLL, -1, -1, -1 },
- .div = { CGU_REG_CPCCR, 16, 1, 4, 22, -1, -1 },
+ .div = {
+ CGU_REG_CPCCR, 16, 1, 4, 22, -1, -1,
+ jz4725b_cgu_cpccr_div_table,
+ },
.gate = { CGU_REG_CLKGR, 13 },
},
@@ -227,5 +254,7 @@ static void __init jz4725b_cgu_init(struct device_node *np)
retval = ingenic_cgu_register_clocks(cgu);
if (retval)
pr_err("%s: failed to register CGU Clocks\n", __func__);
+
+ ingenic_cgu_register_syscore_ops(cgu);
}
CLK_OF_DECLARE(jz4725b_cgu, "ingenic,jz4725b-cgu", jz4725b_cgu_init);
diff --git a/drivers/clk/ingenic/jz4740-cgu.c b/drivers/clk/ingenic/jz4740-cgu.c
index c77f4e1506dc..4c0a20949c2c 100644
--- a/drivers/clk/ingenic/jz4740-cgu.c
+++ b/drivers/clk/ingenic/jz4740-cgu.c
@@ -11,8 +11,8 @@
#include <linux/io.h>
#include <linux/of.h>
#include <dt-bindings/clock/jz4740-cgu.h>
-#include <asm/mach-jz4740/clock.h>
#include "cgu.h"
+#include "pm.h"
/* CGU register offsets */
#define CGU_REG_CPCCR 0x00
@@ -49,6 +49,10 @@ static const s8 pll_od_encoding[4] = {
0x0, 0x1, -1, 0x3,
};
+static const u8 jz4740_cgu_cpccr_div_table[] = {
+ 1, 2, 3, 4, 6, 8, 12, 16, 24, 32,
+};
+
static const struct ingenic_cgu_clk_info jz4740_cgu_clocks[] = {
/* External clocks */
@@ -88,31 +92,46 @@ static const struct ingenic_cgu_clk_info jz4740_cgu_clocks[] = {
[JZ4740_CLK_CCLK] = {
"cclk", CGU_CLK_DIV,
.parents = { JZ4740_CLK_PLL, -1, -1, -1 },
- .div = { CGU_REG_CPCCR, 0, 1, 4, 22, -1, -1 },
+ .div = {
+ CGU_REG_CPCCR, 0, 1, 4, 22, -1, -1,
+ jz4740_cgu_cpccr_div_table,
+ },
},
[JZ4740_CLK_HCLK] = {
"hclk", CGU_CLK_DIV,
.parents = { JZ4740_CLK_PLL, -1, -1, -1 },
- .div = { CGU_REG_CPCCR, 4, 1, 4, 22, -1, -1 },
+ .div = {
+ CGU_REG_CPCCR, 4, 1, 4, 22, -1, -1,
+ jz4740_cgu_cpccr_div_table,
+ },
},
[JZ4740_CLK_PCLK] = {
"pclk", CGU_CLK_DIV,
.parents = { JZ4740_CLK_PLL, -1, -1, -1 },
- .div = { CGU_REG_CPCCR, 8, 1, 4, 22, -1, -1 },
+ .div = {
+ CGU_REG_CPCCR, 8, 1, 4, 22, -1, -1,
+ jz4740_cgu_cpccr_div_table,
+ },
},
[JZ4740_CLK_MCLK] = {
"mclk", CGU_CLK_DIV,
.parents = { JZ4740_CLK_PLL, -1, -1, -1 },
- .div = { CGU_REG_CPCCR, 12, 1, 4, 22, -1, -1 },
+ .div = {
+ CGU_REG_CPCCR, 12, 1, 4, 22, -1, -1,
+ jz4740_cgu_cpccr_div_table,
+ },
},
[JZ4740_CLK_LCD] = {
"lcd", CGU_CLK_DIV | CGU_CLK_GATE,
.parents = { JZ4740_CLK_PLL_HALF, -1, -1, -1 },
- .div = { CGU_REG_CPCCR, 16, 1, 5, 22, -1, -1 },
+ .div = {
+ CGU_REG_CPCCR, 16, 1, 5, 22, -1, -1,
+ jz4740_cgu_cpccr_div_table,
+ },
.gate = { CGU_REG_CLKGR, 10 },
},
@@ -219,77 +238,7 @@ static void __init jz4740_cgu_init(struct device_node *np)
retval = ingenic_cgu_register_clocks(cgu);
if (retval)
pr_err("%s: failed to register CGU Clocks\n", __func__);
-}
-CLK_OF_DECLARE(jz4740_cgu, "ingenic,jz4740-cgu", jz4740_cgu_init);
-
-void jz4740_clock_set_wait_mode(enum jz4740_wait_mode mode)
-{
- uint32_t lcr = readl(cgu->base + CGU_REG_LCR);
-
- switch (mode) {
- case JZ4740_WAIT_MODE_IDLE:
- lcr &= ~LCR_SLEEP;
- break;
-
- case JZ4740_WAIT_MODE_SLEEP:
- lcr |= LCR_SLEEP;
- break;
- }
-
- writel(lcr, cgu->base + CGU_REG_LCR);
-}
-void jz4740_clock_udc_disable_auto_suspend(void)
-{
- uint32_t clkgr = readl(cgu->base + CGU_REG_CLKGR);
-
- clkgr &= ~CLKGR_UDC;
- writel(clkgr, cgu->base + CGU_REG_CLKGR);
-}
-EXPORT_SYMBOL_GPL(jz4740_clock_udc_disable_auto_suspend);
-
-void jz4740_clock_udc_enable_auto_suspend(void)
-{
- uint32_t clkgr = readl(cgu->base + CGU_REG_CLKGR);
-
- clkgr |= CLKGR_UDC;
- writel(clkgr, cgu->base + CGU_REG_CLKGR);
-}
-EXPORT_SYMBOL_GPL(jz4740_clock_udc_enable_auto_suspend);
-
-#define JZ_CLOCK_GATE_UART0 BIT(0)
-#define JZ_CLOCK_GATE_TCU BIT(1)
-#define JZ_CLOCK_GATE_DMAC BIT(12)
-
-void jz4740_clock_suspend(void)
-{
- uint32_t clkgr, cppcr;
-
- clkgr = readl(cgu->base + CGU_REG_CLKGR);
- clkgr |= JZ_CLOCK_GATE_TCU | JZ_CLOCK_GATE_DMAC | JZ_CLOCK_GATE_UART0;
- writel(clkgr, cgu->base + CGU_REG_CLKGR);
-
- cppcr = readl(cgu->base + CGU_REG_CPPCR);
- cppcr &= ~BIT(jz4740_cgu_clocks[JZ4740_CLK_PLL].pll.enable_bit);
- writel(cppcr, cgu->base + CGU_REG_CPPCR);
-}
-
-void jz4740_clock_resume(void)
-{
- uint32_t clkgr, cppcr, stable;
-
- cppcr = readl(cgu->base + CGU_REG_CPPCR);
- cppcr |= BIT(jz4740_cgu_clocks[JZ4740_CLK_PLL].pll.enable_bit);
- writel(cppcr, cgu->base + CGU_REG_CPPCR);
-
- stable = BIT(jz4740_cgu_clocks[JZ4740_CLK_PLL].pll.stable_bit);
- do {
- cppcr = readl(cgu->base + CGU_REG_CPPCR);
- } while (!(cppcr & stable));
-
- clkgr = readl(cgu->base + CGU_REG_CLKGR);
- clkgr &= ~JZ_CLOCK_GATE_TCU;
- clkgr &= ~JZ_CLOCK_GATE_DMAC;
- clkgr &= ~JZ_CLOCK_GATE_UART0;
- writel(clkgr, cgu->base + CGU_REG_CLKGR);
+ ingenic_cgu_register_syscore_ops(cgu);
}
+CLK_OF_DECLARE(jz4740_cgu, "ingenic,jz4740-cgu", jz4740_cgu_init);
diff --git a/drivers/clk/ingenic/jz4770-cgu.c b/drivers/clk/ingenic/jz4770-cgu.c
index dfce740c25a8..eebc1bea3841 100644
--- a/drivers/clk/ingenic/jz4770-cgu.c
+++ b/drivers/clk/ingenic/jz4770-cgu.c
@@ -9,9 +9,9 @@
#include <linux/delay.h>
#include <linux/io.h>
#include <linux/of.h>
-#include <linux/syscore_ops.h>
#include <dt-bindings/clock/jz4770-cgu.h>
#include "cgu.h"
+#include "pm.h"
/*
* CPM registers offset address definition
@@ -38,9 +38,6 @@
#define CGU_REG_MSC2CDR 0xA8
#define CGU_REG_BCHCDR 0xAC
-/* bits within the LCR register */
-#define LCR_LPM BIT(0) /* Low Power Mode */
-
/* bits within the OPCR register */
#define OPCR_SPENDH BIT(5) /* UHC PHY suspend */
@@ -87,6 +84,10 @@ static const s8 pll_od_encoding[8] = {
0x0, 0x1, -1, 0x2, -1, -1, -1, 0x3,
};
+static const u8 jz4770_cgu_cpccr_div_table[] = {
+ 1, 2, 3, 4, 6, 8, 12,
+};
+
static const struct ingenic_cgu_clk_info jz4770_cgu_clocks[] = {
/* External clocks */
@@ -144,34 +145,52 @@ static const struct ingenic_cgu_clk_info jz4770_cgu_clocks[] = {
[JZ4770_CLK_CCLK] = {
"cclk", CGU_CLK_DIV,
.parents = { JZ4770_CLK_PLL0, },
- .div = { CGU_REG_CPCCR, 0, 1, 4, 22, -1, -1 },
+ .div = {
+ CGU_REG_CPCCR, 0, 1, 4, 22, -1, -1,
+ jz4770_cgu_cpccr_div_table,
+ },
},
[JZ4770_CLK_H0CLK] = {
"h0clk", CGU_CLK_DIV,
.parents = { JZ4770_CLK_PLL0, },
- .div = { CGU_REG_CPCCR, 4, 1, 4, 22, -1, -1 },
+ .div = {
+ CGU_REG_CPCCR, 4, 1, 4, 22, -1, -1,
+ jz4770_cgu_cpccr_div_table,
+ },
},
[JZ4770_CLK_H1CLK] = {
"h1clk", CGU_CLK_DIV | CGU_CLK_GATE,
.parents = { JZ4770_CLK_PLL0, },
- .div = { CGU_REG_CPCCR, 24, 1, 4, 22, -1, -1 },
+ .div = {
+ CGU_REG_CPCCR, 24, 1, 4, 22, -1, -1,
+ jz4770_cgu_cpccr_div_table,
+ },
.gate = { CGU_REG_CLKGR1, 7 },
},
[JZ4770_CLK_H2CLK] = {
"h2clk", CGU_CLK_DIV,
.parents = { JZ4770_CLK_PLL0, },
- .div = { CGU_REG_CPCCR, 16, 1, 4, 22, -1, -1 },
+ .div = {
+ CGU_REG_CPCCR, 16, 1, 4, 22, -1, -1,
+ jz4770_cgu_cpccr_div_table,
+ },
},
[JZ4770_CLK_C1CLK] = {
"c1clk", CGU_CLK_DIV | CGU_CLK_GATE,
.parents = { JZ4770_CLK_PLL0, },
- .div = { CGU_REG_CPCCR, 12, 1, 4, 22, -1, -1 },
+ .div = {
+ CGU_REG_CPCCR, 12, 1, 4, 22, -1, -1,
+ jz4770_cgu_cpccr_div_table,
+ },
.gate = { CGU_REG_OPCR, 31, true }, // disable CCLK stop on idle
},
[JZ4770_CLK_PCLK] = {
"pclk", CGU_CLK_DIV,
.parents = { JZ4770_CLK_PLL0, },
- .div = { CGU_REG_CPCCR, 8, 1, 4, 22, -1, -1 },
+ .div = {
+ CGU_REG_CPCCR, 8, 1, 4, 22, -1, -1,
+ jz4770_cgu_cpccr_div_table,
+ },
},
/* Those divided clocks can connect to PLL0 or PLL1 */
@@ -407,30 +426,6 @@ static const struct ingenic_cgu_clk_info jz4770_cgu_clocks[] = {
},
};
-#if IS_ENABLED(CONFIG_PM_SLEEP)
-static int jz4770_cgu_pm_suspend(void)
-{
- u32 val;
-
- val = readl(cgu->base + CGU_REG_LCR);
- writel(val | LCR_LPM, cgu->base + CGU_REG_LCR);
- return 0;
-}
-
-static void jz4770_cgu_pm_resume(void)
-{
- u32 val;
-
- val = readl(cgu->base + CGU_REG_LCR);
- writel(val & ~LCR_LPM, cgu->base + CGU_REG_LCR);
-}
-
-static struct syscore_ops jz4770_cgu_pm_ops = {
- .suspend = jz4770_cgu_pm_suspend,
- .resume = jz4770_cgu_pm_resume,
-};
-#endif /* CONFIG_PM_SLEEP */
-
static void __init jz4770_cgu_init(struct device_node *np)
{
int retval;
@@ -444,9 +439,7 @@ static void __init jz4770_cgu_init(struct device_node *np)
if (retval)
pr_err("%s: failed to register CGU Clocks\n", __func__);
-#if IS_ENABLED(CONFIG_PM_SLEEP)
- register_syscore_ops(&jz4770_cgu_pm_ops);
-#endif
+ ingenic_cgu_register_syscore_ops(cgu);
}
/* We only probe via devicetree, no need for a platform driver */
diff --git a/drivers/clk/ingenic/jz4780-cgu.c b/drivers/clk/ingenic/jz4780-cgu.c
index 2464fc4032af..8c67f89df25e 100644
--- a/drivers/clk/ingenic/jz4780-cgu.c
+++ b/drivers/clk/ingenic/jz4780-cgu.c
@@ -12,6 +12,7 @@
#include <linux/of.h>
#include <dt-bindings/clock/jz4780-cgu.h>
#include "cgu.h"
+#include "pm.h"
/* CGU register offsets */
#define CGU_REG_CLOCKCONTROL 0x00
@@ -721,5 +722,7 @@ static void __init jz4780_cgu_init(struct device_node *np)
pr_err("%s: failed to register CGU Clocks\n", __func__);
return;
}
+
+ ingenic_cgu_register_syscore_ops(cgu);
}
CLK_OF_DECLARE(jz4780_cgu, "ingenic,jz4780-cgu", jz4780_cgu_init);
diff --git a/drivers/clk/ingenic/pm.c b/drivers/clk/ingenic/pm.c
new file mode 100644
index 000000000000..341752b640d2
--- /dev/null
+++ b/drivers/clk/ingenic/pm.c
@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Paul Cercueil <paul@crapouillou.net>
+ */
+
+#include "cgu.h"
+#include "pm.h"
+
+#include <linux/io.h>
+#include <linux/syscore_ops.h>
+
+#define CGU_REG_LCR 0x04
+
+#define LCR_LOW_POWER_MODE BIT(0)
+
+static void __iomem * __maybe_unused ingenic_cgu_base;
+
+static int __maybe_unused ingenic_cgu_pm_suspend(void)
+{
+ u32 val = readl(ingenic_cgu_base + CGU_REG_LCR);
+
+ writel(val | LCR_LOW_POWER_MODE, ingenic_cgu_base + CGU_REG_LCR);
+
+ return 0;
+}
+
+static void __maybe_unused ingenic_cgu_pm_resume(void)
+{
+ u32 val = readl(ingenic_cgu_base + CGU_REG_LCR);
+
+ writel(val & ~LCR_LOW_POWER_MODE, ingenic_cgu_base + CGU_REG_LCR);
+}
+
+static struct syscore_ops __maybe_unused ingenic_cgu_pm_ops = {
+ .suspend = ingenic_cgu_pm_suspend,
+ .resume = ingenic_cgu_pm_resume,
+};
+
+void ingenic_cgu_register_syscore_ops(struct ingenic_cgu *cgu)
+{
+ if (IS_ENABLED(CONFIG_PM_SLEEP)) {
+ ingenic_cgu_base = cgu->base;
+ register_syscore_ops(&ingenic_cgu_pm_ops);
+ }
+}
diff --git a/drivers/clk/ingenic/pm.h b/drivers/clk/ingenic/pm.h
new file mode 100644
index 000000000000..fa7540407b6b
--- /dev/null
+++ b/drivers/clk/ingenic/pm.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2019 Paul Cercueil <paul@crapouillou.net>
+ */
+#ifndef DRIVERS_CLK_INGENIC_PM_H
+#define DRIVERS_CLK_INGENIC_PM_H
+
+struct ingenic_cgu;
+
+void ingenic_cgu_register_syscore_ops(struct ingenic_cgu *cgu);
+
+#endif /* DRIVERS_CLK_INGENIC_PM_H */
diff --git a/drivers/clk/keystone/Kconfig b/drivers/clk/keystone/Kconfig
index 0ca63014718a..38aeefb1e808 100644
--- a/drivers/clk/keystone/Kconfig
+++ b/drivers/clk/keystone/Kconfig
@@ -15,3 +15,14 @@ config TI_SCI_CLK
This adds the clock driver support over TI System Control Interface.
If you wish to use clock resources from the PMMC firmware, say Y.
Otherwise, say N.
+
+config TI_SCI_CLK_PROBE_FROM_FW
+ bool "Probe available clocks from firmware"
+ depends on TI_SCI_CLK
+ default n
+ help
+ Forces the TI SCI clock driver to probe available clocks from the
+ firmware. By default, only the used clocks are probed from DT.
+ This is mostly only useful for debugging purposes, and will
+ increase the boot time of the device. If you want the clocks probed
+ from firmware, say Y. Otherwise, say N.
diff --git a/drivers/clk/keystone/sci-clk.c b/drivers/clk/keystone/sci-clk.c
index 4cb70bed89a9..7edf8c8432b6 100644
--- a/drivers/clk/keystone/sci-clk.c
+++ b/drivers/clk/keystone/sci-clk.c
@@ -23,6 +23,7 @@
#include <linux/slab.h>
#include <linux/soc/ti/ti_sci_protocol.h>
#include <linux/bsearch.h>
+#include <linux/list_sort.h>
#define SCI_CLK_SSC_ENABLE BIT(0)
#define SCI_CLK_ALLOW_FREQ_CHANGE BIT(1)
@@ -52,14 +53,16 @@ struct sci_clk_provider {
* @num_parents: Number of parents for this clock
* @provider: Master clock provider
* @flags: Flags for the clock
+ * @node: Link for handling clocks probed via DT
*/
struct sci_clk {
struct clk_hw hw;
u16 dev_id;
- u8 clk_id;
- u8 num_parents;
+ u32 clk_id;
+ u32 num_parents;
struct sci_clk_provider *provider;
u8 flags;
+ struct list_head node;
};
#define to_sci_clk(_hw) container_of(_hw, struct sci_clk, hw)
@@ -218,11 +221,11 @@ static int sci_clk_set_rate(struct clk_hw *hw, unsigned long rate,
static u8 sci_clk_get_parent(struct clk_hw *hw)
{
struct sci_clk *clk = to_sci_clk(hw);
- u8 parent_id;
+ u32 parent_id = 0;
int ret;
ret = clk->provider->ops->get_parent(clk->provider->sci, clk->dev_id,
- clk->clk_id, &parent_id);
+ clk->clk_id, (void *)&parent_id);
if (ret) {
dev_err(clk->provider->dev,
"get-parent failed for dev=%d, clk=%d, ret=%d\n",
@@ -230,7 +233,9 @@ static u8 sci_clk_get_parent(struct clk_hw *hw)
return 0;
}
- return parent_id - clk->clk_id - 1;
+ parent_id = parent_id - clk->clk_id - 1;
+
+ return (u8)parent_id;
}
/**
@@ -280,8 +285,8 @@ static int _sci_clk_build(struct sci_clk_provider *provider,
int i;
int ret = 0;
- name = kasprintf(GFP_KERNEL, "%s:%d:%d", dev_name(provider->dev),
- sci_clk->dev_id, sci_clk->clk_id);
+ name = kasprintf(GFP_KERNEL, "clk:%d:%d", sci_clk->dev_id,
+ sci_clk->clk_id);
init.name = name;
@@ -306,8 +311,7 @@ static int _sci_clk_build(struct sci_clk_provider *provider,
for (i = 0; i < sci_clk->num_parents; i++) {
char *parent_name;
- parent_name = kasprintf(GFP_KERNEL, "%s:%d:%d",
- dev_name(provider->dev),
+ parent_name = kasprintf(GFP_KERNEL, "clk:%d:%d",
sci_clk->dev_id,
sci_clk->clk_id + 1 + i);
if (!parent_name) {
@@ -404,22 +408,9 @@ static const struct of_device_id ti_sci_clk_of_match[] = {
};
MODULE_DEVICE_TABLE(of, ti_sci_clk_of_match);
-/**
- * ti_sci_clk_probe - Probe function for the TI SCI clock driver
- * @pdev: platform device pointer to be probed
- *
- * Probes the TI SCI clock device. Allocates a new clock provider
- * and registers this to the common clock framework. Also applies
- * any required flags to the identified clocks via clock lists
- * supplied from DT. Returns 0 for success, negative error value
- * for failure.
- */
-static int ti_sci_clk_probe(struct platform_device *pdev)
+#ifdef CONFIG_TI_SCI_CLK_PROBE_FROM_FW
+static int ti_sci_scan_clocks_from_fw(struct sci_clk_provider *provider)
{
- struct device *dev = &pdev->dev;
- struct device_node *np = dev->of_node;
- struct sci_clk_provider *provider;
- const struct ti_sci_handle *handle;
int ret;
int num_clks = 0;
struct sci_clk **clks = NULL;
@@ -428,24 +419,14 @@ static int ti_sci_clk_probe(struct platform_device *pdev)
int max_clks = 0;
int clk_id = 0;
int dev_id = 0;
- u8 num_parents;
+ u32 num_parents = 0;
int gap_size = 0;
-
- handle = devm_ti_sci_get_handle(dev);
- if (IS_ERR(handle))
- return PTR_ERR(handle);
-
- provider = devm_kzalloc(dev, sizeof(*provider), GFP_KERNEL);
- if (!provider)
- return -ENOMEM;
-
- provider->sci = handle;
- provider->ops = &handle->ops.clk_ops;
- provider->dev = dev;
+ struct device *dev = provider->dev;
while (1) {
ret = provider->ops->get_num_parents(provider->sci, dev_id,
- clk_id, &num_parents);
+ clk_id,
+ (void *)&num_parents);
if (ret) {
gap_size++;
if (!clk_id) {
@@ -502,6 +483,188 @@ static int ti_sci_clk_probe(struct platform_device *pdev)
devm_kfree(dev, clks);
+ return 0;
+}
+
+#else
+
+static int _cmp_sci_clk_list(void *priv, struct list_head *a,
+ struct list_head *b)
+{
+ struct sci_clk *ca = container_of(a, struct sci_clk, node);
+ struct sci_clk *cb = container_of(b, struct sci_clk, node);
+
+ return _cmp_sci_clk(ca, &cb);
+}
+
+static int ti_sci_scan_clocks_from_dt(struct sci_clk_provider *provider)
+{
+ struct device *dev = provider->dev;
+ struct device_node *np = NULL;
+ int ret;
+ int index;
+ struct of_phandle_args args;
+ struct list_head clks;
+ struct sci_clk *sci_clk, *prev;
+ int num_clks = 0;
+ int num_parents;
+ int clk_id;
+ const char * const clk_names[] = {
+ "clocks", "assigned-clocks", "assigned-clock-parents", NULL
+ };
+ const char * const *clk_name;
+
+ INIT_LIST_HEAD(&clks);
+
+ clk_name = clk_names;
+
+ while (*clk_name) {
+ np = of_find_node_with_property(np, *clk_name);
+ if (!np) {
+ clk_name++;
+ break;
+ }
+
+ if (!of_device_is_available(np))
+ continue;
+
+ index = 0;
+
+ do {
+ ret = of_parse_phandle_with_args(np, *clk_name,
+ "#clock-cells", index,
+ &args);
+ if (ret)
+ break;
+
+ if (args.args_count == 2 && args.np == dev->of_node) {
+ sci_clk = devm_kzalloc(dev, sizeof(*sci_clk),
+ GFP_KERNEL);
+ if (!sci_clk)
+ return -ENOMEM;
+
+ sci_clk->dev_id = args.args[0];
+ sci_clk->clk_id = args.args[1];
+ sci_clk->provider = provider;
+ provider->ops->get_num_parents(provider->sci,
+ sci_clk->dev_id,
+ sci_clk->clk_id,
+ (void *)&sci_clk->num_parents);
+ list_add_tail(&sci_clk->node, &clks);
+
+ num_clks++;
+
+ num_parents = sci_clk->num_parents;
+ if (num_parents == 1)
+ num_parents = 0;
+
+ /*
+ * Linux kernel has inherent limitation
+ * of 255 clock parents at the moment.
+ * Right now, it is not expected that
+ * any mux clock from sci-clk driver
+ * would exceed that limit either, but
+ * the ABI basically provides that
+ * possibility. Print out a warning if
+ * this happens for any clock.
+ */
+ if (num_parents >= 255) {
+ dev_warn(dev, "too many parents for dev=%d, clk=%d (%d), cropping to 255.\n",
+ sci_clk->dev_id,
+ sci_clk->clk_id, num_parents);
+ num_parents = 255;
+ }
+
+ clk_id = args.args[1] + 1;
+
+ while (num_parents--) {
+ sci_clk = devm_kzalloc(dev,
+ sizeof(*sci_clk),
+ GFP_KERNEL);
+ if (!sci_clk)
+ return -ENOMEM;
+ sci_clk->dev_id = args.args[0];
+ sci_clk->clk_id = clk_id++;
+ sci_clk->provider = provider;
+ list_add_tail(&sci_clk->node, &clks);
+
+ num_clks++;
+ }
+ }
+
+ index++;
+ } while (args.np);
+ }
+
+ list_sort(NULL, &clks, _cmp_sci_clk_list);
+
+ provider->clocks = devm_kmalloc_array(dev, num_clks, sizeof(sci_clk),
+ GFP_KERNEL);
+ if (!provider->clocks)
+ return -ENOMEM;
+
+ num_clks = 0;
+ prev = NULL;
+
+ list_for_each_entry(sci_clk, &clks, node) {
+ if (prev && prev->dev_id == sci_clk->dev_id &&
+ prev->clk_id == sci_clk->clk_id)
+ continue;
+
+ provider->clocks[num_clks++] = sci_clk;
+ prev = sci_clk;
+ }
+
+ provider->num_clocks = num_clks;
+
+ return 0;
+}
+#endif
+
+/**
+ * ti_sci_clk_probe - Probe function for the TI SCI clock driver
+ * @pdev: platform device pointer to be probed
+ *
+ * Probes the TI SCI clock device. Allocates a new clock provider
+ * and registers this to the common clock framework. Also applies
+ * any required flags to the identified clocks via clock lists
+ * supplied from DT. Returns 0 for success, negative error value
+ * for failure.
+ */
+static int ti_sci_clk_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct device_node *np = dev->of_node;
+ struct sci_clk_provider *provider;
+ const struct ti_sci_handle *handle;
+ int ret;
+
+ handle = devm_ti_sci_get_handle(dev);
+ if (IS_ERR(handle))
+ return PTR_ERR(handle);
+
+ provider = devm_kzalloc(dev, sizeof(*provider), GFP_KERNEL);
+ if (!provider)
+ return -ENOMEM;
+
+ provider->sci = handle;
+ provider->ops = &handle->ops.clk_ops;
+ provider->dev = dev;
+
+#ifdef CONFIG_TI_SCI_CLK_PROBE_FROM_FW
+ ret = ti_sci_scan_clocks_from_fw(provider);
+ if (ret) {
+ dev_err(dev, "scan clocks from FW failed: %d\n", ret);
+ return ret;
+ }
+#else
+ ret = ti_sci_scan_clocks_from_dt(provider);
+ if (ret) {
+ dev_err(dev, "scan clocks from DT failed: %d\n", ret);
+ return ret;
+ }
+#endif
+
ret = ti_sci_init_clocks(provider);
if (ret) {
pr_err("ti-sci-init-clocks failed.\n");
diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig
index f797f09c6425..ce3d9b300bab 100644
--- a/drivers/clk/mediatek/Kconfig
+++ b/drivers/clk/mediatek/Kconfig
@@ -300,4 +300,10 @@ config COMMON_CLK_MT8516
help
This driver supports MediaTek MT8516 clocks.
+config COMMON_CLK_MT8516_AUDSYS
+ bool "Clock driver for MediaTek MT8516 audsys"
+ depends on COMMON_CLK_MT8516
+ help
+ This driver supports MediaTek MT8516 audsys clocks.
+
endmenu
diff --git a/drivers/clk/mediatek/Makefile b/drivers/clk/mediatek/Makefile
index f74937b35f68..672de0099eef 100644
--- a/drivers/clk/mediatek/Makefile
+++ b/drivers/clk/mediatek/Makefile
@@ -45,3 +45,4 @@ obj-$(CONFIG_COMMON_CLK_MT8183_MMSYS) += clk-mt8183-mm.o
obj-$(CONFIG_COMMON_CLK_MT8183_VDECSYS) += clk-mt8183-vdec.o
obj-$(CONFIG_COMMON_CLK_MT8183_VENCSYS) += clk-mt8183-venc.o
obj-$(CONFIG_COMMON_CLK_MT8516) += clk-mt8516.o
+obj-$(CONFIG_COMMON_CLK_MT8516_AUDSYS) += clk-mt8516-aud.o
diff --git a/drivers/clk/mediatek/clk-mt8183.c b/drivers/clk/mediatek/clk-mt8183.c
index 9d8651033ae9..73b7e238eee7 100644
--- a/drivers/clk/mediatek/clk-mt8183.c
+++ b/drivers/clk/mediatek/clk-mt8183.c
@@ -25,9 +25,11 @@ static const struct mtk_fixed_clk top_fixed_clks[] = {
FIXED_CLK(CLK_TOP_UNIVP_192M, "univpll_192m", "univpll", 192000000),
};
+static const struct mtk_fixed_factor top_early_divs[] = {
+ FACTOR(CLK_TOP_CLK13M, "clk13m", "clk26m", 1, 2),
+};
+
static const struct mtk_fixed_factor top_divs[] = {
- FACTOR(CLK_TOP_CLK13M, "clk13m", "clk26m", 1,
- 2),
FACTOR(CLK_TOP_F26M_CK_D2, "csw_f26m_ck_d2", "clk26m", 1,
2),
FACTOR(CLK_TOP_SYSPLL_CK, "syspll_ck", "mainpll", 1,
@@ -395,14 +397,6 @@ static const char * const atb_parents[] = {
"syspll_d5"
};
-static const char * const sspm_parents[] = {
- "clk26m",
- "univpll_d2_d4",
- "syspll_d2_d2",
- "univpll_d2_d2",
- "syspll_d3"
-};
-
static const char * const dpi0_parents[] = {
"clk26m",
"tvdpll_d2",
@@ -606,9 +600,6 @@ static const struct mtk_mux top_muxes[] = {
MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_ATB, "atb_sel",
atb_parents, 0xa0,
0xa4, 0xa8, 0, 2, 7, 0x004, 24),
- MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_SSPM, "sspm_sel",
- sspm_parents, 0xa0,
- 0xa4, 0xa8, 8, 3, 15, 0x004, 25),
MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_DPI0, "dpi0_sel",
dpi0_parents, 0xa0,
0xa4, 0xa8, 16, 4, 23, 0x004, 26),
@@ -947,12 +938,8 @@ static const struct mtk_gate infra_clks[] = {
"fufs_sel", 13),
GATE_INFRA2(CLK_INFRA_MD32_BCLK, "infra_md32_bclk",
"axi_sel", 14),
- GATE_INFRA2(CLK_INFRA_SSPM, "infra_sspm",
- "sspm_sel", 15),
GATE_INFRA2(CLK_INFRA_UNIPRO_MBIST, "infra_unipro_mbist",
"axi_sel", 16),
- GATE_INFRA2(CLK_INFRA_SSPM_BUS_HCLK, "infra_sspm_bus_hclk",
- "axi_sel", 17),
GATE_INFRA2(CLK_INFRA_I2C5, "infra_i2c5",
"i2c_sel", 18),
GATE_INFRA2(CLK_INFRA_I2C5_ARBITER, "infra_i2c5_arbiter",
@@ -986,10 +973,6 @@ static const struct mtk_gate infra_clks[] = {
"msdc50_0_sel", 1),
GATE_INFRA3(CLK_INFRA_MSDC2_SELF, "infra_msdc2_self",
"msdc50_0_sel", 2),
- GATE_INFRA3(CLK_INFRA_SSPM_26M_SELF, "infra_sspm_26m_self",
- "f_f26m_ck", 3),
- GATE_INFRA3(CLK_INFRA_SSPM_32K_SELF, "infra_sspm_32k_self",
- "f_f26m_ck", 4),
GATE_INFRA3(CLK_INFRA_UFS_AXI, "infra_ufs_axi",
"axi_sel", 5),
GATE_INFRA3(CLK_INFRA_I2C6, "infra_i2c6",
@@ -1167,37 +1150,57 @@ static int clk_mt8183_apmixed_probe(struct platform_device *pdev)
return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
}
+static struct clk_onecell_data *top_clk_data;
+
+static void clk_mt8183_top_init_early(struct device_node *node)
+{
+ int i;
+
+ top_clk_data = mtk_alloc_clk_data(CLK_TOP_NR_CLK);
+
+ for (i = 0; i < CLK_TOP_NR_CLK; i++)
+ top_clk_data->clks[i] = ERR_PTR(-EPROBE_DEFER);
+
+ mtk_clk_register_factors(top_early_divs, ARRAY_SIZE(top_early_divs),
+ top_clk_data);
+
+ of_clk_add_provider(node, of_clk_src_onecell_get, top_clk_data);
+}
+
+CLK_OF_DECLARE_DRIVER(mt8183_topckgen, "mediatek,mt8183-topckgen",
+ clk_mt8183_top_init_early);
+
static int clk_mt8183_top_probe(struct platform_device *pdev)
{
struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
void __iomem *base;
- struct clk_onecell_data *clk_data;
struct device_node *node = pdev->dev.of_node;
base = devm_ioremap_resource(&pdev->dev, res);
if (IS_ERR(base))
return PTR_ERR(base);
- clk_data = mtk_alloc_clk_data(CLK_TOP_NR_CLK);
-
mtk_clk_register_fixed_clks(top_fixed_clks, ARRAY_SIZE(top_fixed_clks),
- clk_data);
+ top_clk_data);
- mtk_clk_register_factors(top_divs, ARRAY_SIZE(top_divs), clk_data);
+ mtk_clk_register_factors(top_early_divs, ARRAY_SIZE(top_early_divs),
+ top_clk_data);
+
+ mtk_clk_register_factors(top_divs, ARRAY_SIZE(top_divs), top_clk_data);
mtk_clk_register_muxes(top_muxes, ARRAY_SIZE(top_muxes),
- node, &mt8183_clk_lock, clk_data);
+ node, &mt8183_clk_lock, top_clk_data);
mtk_clk_register_composites(top_aud_muxes, ARRAY_SIZE(top_aud_muxes),
- base, &mt8183_clk_lock, clk_data);
+ base, &mt8183_clk_lock, top_clk_data);
mtk_clk_register_composites(top_aud_divs, ARRAY_SIZE(top_aud_divs),
- base, &mt8183_clk_lock, clk_data);
+ base, &mt8183_clk_lock, top_clk_data);
mtk_clk_register_gates(node, top_clks, ARRAY_SIZE(top_clks),
- clk_data);
+ top_clk_data);
- return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+ return of_clk_add_provider(node, of_clk_src_onecell_get, top_clk_data);
}
static int clk_mt8183_infra_probe(struct platform_device *pdev)
diff --git a/drivers/clk/mediatek/clk-mt8516-aud.c b/drivers/clk/mediatek/clk-mt8516-aud.c
new file mode 100644
index 000000000000..6ab3a06dc9d5
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt8516-aud.c
@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2019 MediaTek Inc.
+ * Author: James Liao <jamesjj.liao@mediatek.com>
+ * Fabien Parent <fparent@baylibre.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt8516-clk.h>
+
+static const struct mtk_gate_regs aud_cg_regs = {
+ .set_ofs = 0x0,
+ .clr_ofs = 0x0,
+ .sta_ofs = 0x0,
+};
+
+#define GATE_AUD(_id, _name, _parent, _shift) { \
+ .id = _id, \
+ .name = _name, \
+ .parent_name = _parent, \
+ .regs = &aud_cg_regs, \
+ .shift = _shift, \
+ .ops = &mtk_clk_gate_ops_no_setclr, \
+ }
+
+static const struct mtk_gate aud_clks[] __initconst = {
+ GATE_AUD(CLK_AUD_AFE, "aud_afe", "clk26m_ck", 2),
+ GATE_AUD(CLK_AUD_I2S, "aud_i2s", "i2s_infra_bck", 6),
+ GATE_AUD(CLK_AUD_22M, "aud_22m", "rg_aud_engen1", 8),
+ GATE_AUD(CLK_AUD_24M, "aud_24m", "rg_aud_engen2", 9),
+ GATE_AUD(CLK_AUD_INTDIR, "aud_intdir", "rg_aud_spdif_in", 15),
+ GATE_AUD(CLK_AUD_APLL2_TUNER, "aud_apll2_tuner", "rg_aud_engen2", 18),
+ GATE_AUD(CLK_AUD_APLL_TUNER, "aud_apll_tuner", "rg_aud_engen1", 19),
+ GATE_AUD(CLK_AUD_HDMI, "aud_hdmi", "apll12_div4", 20),
+ GATE_AUD(CLK_AUD_SPDF, "aud_spdf", "apll12_div6", 21),
+ GATE_AUD(CLK_AUD_ADC, "aud_adc", "aud_afe", 24),
+ GATE_AUD(CLK_AUD_DAC, "aud_dac", "aud_afe", 25),
+ GATE_AUD(CLK_AUD_DAC_PREDIS, "aud_dac_predis", "aud_afe", 26),
+ GATE_AUD(CLK_AUD_TML, "aud_tml", "aud_afe", 27),
+};
+
+static void __init mtk_audsys_init(struct device_node *node)
+{
+ struct clk_onecell_data *clk_data;
+ int r;
+
+ clk_data = mtk_alloc_clk_data(CLK_AUD_NR_CLK);
+
+ mtk_clk_register_gates(node, aud_clks, ARRAY_SIZE(aud_clks), clk_data);
+
+ r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+ if (r)
+ pr_err("%s(): could not register clock provider: %d\n",
+ __func__, r);
+
+}
+CLK_OF_DECLARE(mtk_audsys, "mediatek,mt8516-audsys", mtk_audsys_init);
diff --git a/drivers/clk/mediatek/clk-mt8516.c b/drivers/clk/mediatek/clk-mt8516.c
index 26fe43cc9ea2..9d4261ecc760 100644
--- a/drivers/clk/mediatek/clk-mt8516.c
+++ b/drivers/clk/mediatek/clk-mt8516.c
@@ -231,11 +231,6 @@ static const char * const nfi1x_pad_parents[] __initconst = {
"nfi1x_ck"
};
-static const char * const ddrphycfg_parents[] __initconst = {
- "clk26m_ck",
- "mainpll_d16"
-};
-
static const char * const usb_78m_parents[] __initconst = {
"clk_null",
"clk26m_ck",
diff --git a/drivers/clk/meson/Kconfig b/drivers/clk/meson/Kconfig
index a6b20e123e0c..dabeb435d067 100644
--- a/drivers/clk/meson/Kconfig
+++ b/drivers/clk/meson/Kconfig
@@ -1,7 +1,4 @@
# SPDX-License-Identifier: GPL-2.0-only
-config COMMON_CLK_MESON_INPUT
- tristate
-
config COMMON_CLK_MESON_REGMAP
tristate
select REGMAP
@@ -33,13 +30,15 @@ config COMMON_CLK_MESON_VID_PLL_DIV
config COMMON_CLK_MESON_AO_CLKC
tristate
select COMMON_CLK_MESON_REGMAP
- select COMMON_CLK_MESON_INPUT
select RESET_CONTROLLER
config COMMON_CLK_MESON_EE_CLKC
tristate
select COMMON_CLK_MESON_REGMAP
- select COMMON_CLK_MESON_INPUT
+
+config COMMON_CLK_MESON_CPU_DYNDIV
+ tristate
+ select COMMON_CLK_MESON_REGMAP
config COMMON_CLK_MESON8B
bool
@@ -86,7 +85,6 @@ config COMMON_CLK_AXG
config COMMON_CLK_AXG_AUDIO
tristate "Meson AXG Audio Clock Controller Driver"
depends on ARCH_MESON
- select COMMON_CLK_MESON_INPUT
select COMMON_CLK_MESON_REGMAP
select COMMON_CLK_MESON_PHASE
select COMMON_CLK_MESON_SCLK_DIV
@@ -104,6 +102,7 @@ config COMMON_CLK_G12A
select COMMON_CLK_MESON_PLL
select COMMON_CLK_MESON_AO_CLKC
select COMMON_CLK_MESON_EE_CLKC
+ select COMMON_CLK_MESON_CPU_DYNDIV
select MFD_SYSCON
help
Support for the clock controller on Amlogic S905D2, S905X2 and S905Y2
diff --git a/drivers/clk/meson/Makefile b/drivers/clk/meson/Makefile
index bc35a4efd6b7..3939f218587a 100644
--- a/drivers/clk/meson/Makefile
+++ b/drivers/clk/meson/Makefile
@@ -2,9 +2,9 @@
# Amlogic clock drivers
obj-$(CONFIG_COMMON_CLK_MESON_AO_CLKC) += meson-aoclk.o
+obj-$(CONFIG_COMMON_CLK_MESON_CPU_DYNDIV) += clk-cpu-dyndiv.o
obj-$(CONFIG_COMMON_CLK_MESON_DUALDIV) += clk-dualdiv.o
obj-$(CONFIG_COMMON_CLK_MESON_EE_CLKC) += meson-eeclk.o
-obj-$(CONFIG_COMMON_CLK_MESON_INPUT) += clk-input.o
obj-$(CONFIG_COMMON_CLK_MESON_MPLL) += clk-mpll.o
obj-$(CONFIG_COMMON_CLK_MESON_PHASE) += clk-phase.o
obj-$(CONFIG_COMMON_CLK_MESON_PLL) += clk-pll.o
diff --git a/drivers/clk/meson/axg-aoclk.c b/drivers/clk/meson/axg-aoclk.c
index 0086f31288eb..b488b40c9d0e 100644
--- a/drivers/clk/meson/axg-aoclk.c
+++ b/drivers/clk/meson/axg-aoclk.c
@@ -18,8 +18,6 @@
#include "clk-regmap.h"
#include "clk-dualdiv.h"
-#define IN_PREFIX "ao-in-"
-
/*
* AO Configuration Clock registers offsets
* Register offsets from the data sheet must be multiplied by 4.
@@ -42,7 +40,9 @@ static struct clk_regmap axg_aoclk_##_name = { \
.hw.init = &(struct clk_init_data) { \
.name = "axg_ao_" #_name, \
.ops = &clk_regmap_gate_ops, \
- .parent_names = (const char *[]){ IN_PREFIX "mpeg-clk" }, \
+ .parent_data = &(const struct clk_parent_data) { \
+ .fw_name = "mpeg-clk", \
+ }, \
.num_parents = 1, \
.flags = CLK_IGNORE_UNUSED, \
}, \
@@ -64,7 +64,9 @@ static struct clk_regmap axg_aoclk_cts_oscin = {
.hw.init = &(struct clk_init_data){
.name = "cts_oscin",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
},
};
@@ -77,7 +79,9 @@ static struct clk_regmap axg_aoclk_32k_pre = {
.hw.init = &(struct clk_init_data){
.name = "axg_ao_32k_pre",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "cts_oscin" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_aoclk_cts_oscin.hw
+ },
.num_parents = 1,
},
};
@@ -124,7 +128,9 @@ static struct clk_regmap axg_aoclk_32k_div = {
.hw.init = &(struct clk_init_data){
.name = "axg_ao_32k_div",
.ops = &meson_clk_dualdiv_ops,
- .parent_names = (const char *[]){ "axg_ao_32k_pre" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_aoclk_32k_pre.hw
+ },
.num_parents = 1,
},
};
@@ -139,8 +145,10 @@ static struct clk_regmap axg_aoclk_32k_sel = {
.hw.init = &(struct clk_init_data){
.name = "axg_ao_32k_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "axg_ao_32k_div",
- "axg_ao_32k_pre" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_aoclk_32k_div.hw,
+ &axg_aoclk_32k_pre.hw,
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_PARENT,
},
@@ -154,7 +162,9 @@ static struct clk_regmap axg_aoclk_32k = {
.hw.init = &(struct clk_init_data){
.name = "axg_ao_32k",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "axg_ao_32k_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_aoclk_32k_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -170,8 +180,10 @@ static struct clk_regmap axg_aoclk_cts_rtc_oscin = {
.hw.init = &(struct clk_init_data){
.name = "axg_ao_cts_rtc_oscin",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "axg_ao_32k",
- IN_PREFIX "ext_32k-0" },
+ .parent_data = (const struct clk_parent_data []) {
+ { .hw = &axg_aoclk_32k.hw },
+ { .fw_name = "ext_32k-0", },
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_PARENT,
},
@@ -187,8 +199,10 @@ static struct clk_regmap axg_aoclk_clk81 = {
.hw.init = &(struct clk_init_data){
.name = "axg_ao_clk81",
.ops = &clk_regmap_mux_ro_ops,
- .parent_names = (const char *[]){ IN_PREFIX "mpeg-clk",
- "axg_ao_cts_rtc_oscin"},
+ .parent_data = (const struct clk_parent_data []) {
+ { .fw_name = "mpeg-clk", },
+ { .hw = &axg_aoclk_cts_rtc_oscin.hw },
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_PARENT,
},
@@ -203,8 +217,10 @@ static struct clk_regmap axg_aoclk_saradc_mux = {
.hw.init = &(struct clk_init_data){
.name = "axg_ao_saradc_mux",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal",
- "axg_ao_clk81" },
+ .parent_data = (const struct clk_parent_data []) {
+ { .fw_name = "xtal", },
+ { .hw = &axg_aoclk_clk81.hw },
+ },
.num_parents = 2,
},
};
@@ -218,7 +234,9 @@ static struct clk_regmap axg_aoclk_saradc_div = {
.hw.init = &(struct clk_init_data){
.name = "axg_ao_saradc_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "axg_ao_saradc_mux" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_aoclk_saradc_mux.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -232,7 +250,9 @@ static struct clk_regmap axg_aoclk_saradc_gate = {
.hw.init = &(struct clk_init_data){
.name = "axg_ao_saradc_gate",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "axg_ao_saradc_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_aoclk_saradc_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -290,12 +310,6 @@ static const struct clk_hw_onecell_data axg_aoclk_onecell_data = {
.num = NR_CLKS,
};
-static const struct meson_aoclk_input axg_aoclk_inputs[] = {
- { .name = "xtal", .required = true },
- { .name = "mpeg-clk", .required = true },
- { .name = "ext-32k-0", .required = false },
-};
-
static const struct meson_aoclk_data axg_aoclkc_data = {
.reset_reg = AO_RTI_GEN_CNTL_REG0,
.num_reset = ARRAY_SIZE(axg_aoclk_reset),
@@ -303,9 +317,6 @@ static const struct meson_aoclk_data axg_aoclkc_data = {
.num_clks = ARRAY_SIZE(axg_aoclk_regmap),
.clks = axg_aoclk_regmap,
.hw_data = &axg_aoclk_onecell_data,
- .inputs = axg_aoclk_inputs,
- .num_inputs = ARRAY_SIZE(axg_aoclk_inputs),
- .input_prefix = IN_PREFIX,
};
static const struct of_device_id axg_aoclkc_match_table[] = {
diff --git a/drivers/clk/meson/axg-audio.c b/drivers/clk/meson/axg-audio.c
index 8028ff6f6610..741df7e955ca 100644
--- a/drivers/clk/meson/axg-audio.c
+++ b/drivers/clk/meson/axg-audio.c
@@ -15,7 +15,6 @@
#include <linux/slab.h>
#include "axg-audio.h"
-#include "clk-input.h"
#include "clk-regmap.h"
#include "clk-phase.h"
#include "sclk-div.h"
@@ -24,7 +23,7 @@
#define AUD_SLV_SCLK_COUNT 10
#define AUD_SLV_LRCLK_COUNT 10
-#define AUD_GATE(_name, _reg, _bit, _pname, _iflags) \
+#define AUD_GATE(_name, _reg, _bit, _phws, _iflags) \
struct clk_regmap aud_##_name = { \
.data = &(struct clk_regmap_gate_data){ \
.offset = (_reg), \
@@ -33,13 +32,13 @@ struct clk_regmap aud_##_name = { \
.hw.init = &(struct clk_init_data) { \
.name = "aud_"#_name, \
.ops = &clk_regmap_gate_ops, \
- .parent_names = (const char *[]){ _pname }, \
+ .parent_hws = (const struct clk_hw *[]) { &_phws.hw }, \
.num_parents = 1, \
.flags = CLK_DUTY_CYCLE_PARENT | (_iflags), \
}, \
}
-#define AUD_MUX(_name, _reg, _mask, _shift, _dflags, _pnames, _iflags) \
+#define AUD_MUX(_name, _reg, _mask, _shift, _dflags, _pdata, _iflags) \
struct clk_regmap aud_##_name = { \
.data = &(struct clk_regmap_mux_data){ \
.offset = (_reg), \
@@ -50,13 +49,13 @@ struct clk_regmap aud_##_name = { \
.hw.init = &(struct clk_init_data){ \
.name = "aud_"#_name, \
.ops = &clk_regmap_mux_ops, \
- .parent_names = (_pnames), \
- .num_parents = ARRAY_SIZE(_pnames), \
+ .parent_data = _pdata, \
+ .num_parents = ARRAY_SIZE(_pdata), \
.flags = CLK_DUTY_CYCLE_PARENT | (_iflags), \
}, \
}
-#define AUD_DIV(_name, _reg, _shift, _width, _dflags, _pname, _iflags) \
+#define AUD_DIV(_name, _reg, _shift, _width, _dflags, _phws, _iflags) \
struct clk_regmap aud_##_name = { \
.data = &(struct clk_regmap_div_data){ \
.offset = (_reg), \
@@ -67,15 +66,27 @@ struct clk_regmap aud_##_name = { \
.hw.init = &(struct clk_init_data){ \
.name = "aud_"#_name, \
.ops = &clk_regmap_divider_ops, \
- .parent_names = (const char *[]) { _pname }, \
+ .parent_hws = (const struct clk_hw *[]) { &_phws.hw }, \
.num_parents = 1, \
.flags = (_iflags), \
}, \
}
#define AUD_PCLK_GATE(_name, _bit) \
- AUD_GATE(_name, AUDIO_CLK_GATE_EN, _bit, "audio_pclk", 0)
-
+struct clk_regmap aud_##_name = { \
+ .data = &(struct clk_regmap_gate_data){ \
+ .offset = (AUDIO_CLK_GATE_EN), \
+ .bit_idx = (_bit), \
+ }, \
+ .hw.init = &(struct clk_init_data) { \
+ .name = "aud_"#_name, \
+ .ops = &clk_regmap_gate_ops, \
+ .parent_data = &(const struct clk_parent_data) { \
+ .fw_name = "pclk", \
+ }, \
+ .num_parents = 1, \
+ }, \
+}
/* Audio peripheral clocks */
static AUD_PCLK_GATE(ddr_arb, 0);
static AUD_PCLK_GATE(pdm, 1);
@@ -100,14 +111,20 @@ static AUD_PCLK_GATE(power_detect, 19);
static AUD_PCLK_GATE(spdifout_b, 21);
/* Audio Master Clocks */
-static const char * const mst_mux_parent_names[] = {
- "aud_mst_in0", "aud_mst_in1", "aud_mst_in2", "aud_mst_in3",
- "aud_mst_in4", "aud_mst_in5", "aud_mst_in6", "aud_mst_in7",
+static const struct clk_parent_data mst_mux_parent_data[] = {
+ { .fw_name = "mst_in0", },
+ { .fw_name = "mst_in1", },
+ { .fw_name = "mst_in2", },
+ { .fw_name = "mst_in3", },
+ { .fw_name = "mst_in4", },
+ { .fw_name = "mst_in5", },
+ { .fw_name = "mst_in6", },
+ { .fw_name = "mst_in7", },
};
#define AUD_MST_MUX(_name, _reg, _flag) \
AUD_MUX(_name##_sel, _reg, 0x7, 24, _flag, \
- mst_mux_parent_names, CLK_SET_RATE_PARENT)
+ mst_mux_parent_data, 0)
#define AUD_MST_MCLK_MUX(_name, _reg) \
AUD_MST_MUX(_name, _reg, CLK_MUX_ROUND_CLOSEST)
@@ -129,7 +146,7 @@ static AUD_MST_MCLK_MUX(spdifout_b_clk, AUDIO_CLK_SPDIFOUT_B_CTRL);
#define AUD_MST_DIV(_name, _reg, _flag) \
AUD_DIV(_name##_div, _reg, 0, 16, _flag, \
- "aud_"#_name"_sel", CLK_SET_RATE_PARENT) \
+ aud_##_name##_sel, CLK_SET_RATE_PARENT) \
#define AUD_MST_MCLK_DIV(_name, _reg) \
AUD_MST_DIV(_name, _reg, CLK_DIVIDER_ROUND_CLOSEST)
@@ -150,7 +167,7 @@ static AUD_MST_SYS_DIV(pdm_sysclk, AUDIO_CLK_PDMIN_CTRL1);
static AUD_MST_MCLK_DIV(spdifout_b_clk, AUDIO_CLK_SPDIFOUT_B_CTRL);
#define AUD_MST_MCLK_GATE(_name, _reg) \
- AUD_GATE(_name, _reg, 31, "aud_"#_name"_div", \
+ AUD_GATE(_name, _reg, 31, aud_##_name##_div, \
CLK_SET_RATE_PARENT)
static AUD_MST_MCLK_GATE(mst_a_mclk, AUDIO_MCLK_A_CTRL);
@@ -168,7 +185,7 @@ static AUD_MST_MCLK_GATE(spdifout_b_clk, AUDIO_CLK_SPDIFOUT_B_CTRL);
/* Sample Clocks */
#define AUD_MST_SCLK_PRE_EN(_name, _reg) \
AUD_GATE(mst_##_name##_sclk_pre_en, _reg, 31, \
- "aud_mst_"#_name"_mclk", 0)
+ aud_mst_##_name##_mclk, 0)
static AUD_MST_SCLK_PRE_EN(a, AUDIO_MST_A_SCLK_CTRL0);
static AUD_MST_SCLK_PRE_EN(b, AUDIO_MST_B_SCLK_CTRL0);
@@ -178,7 +195,7 @@ static AUD_MST_SCLK_PRE_EN(e, AUDIO_MST_E_SCLK_CTRL0);
static AUD_MST_SCLK_PRE_EN(f, AUDIO_MST_F_SCLK_CTRL0);
#define AUD_SCLK_DIV(_name, _reg, _div_shift, _div_width, \
- _hi_shift, _hi_width, _pname, _iflags) \
+ _hi_shift, _hi_width, _phws, _iflags) \
struct clk_regmap aud_##_name = { \
.data = &(struct meson_sclk_div_data) { \
.div = { \
@@ -195,7 +212,7 @@ struct clk_regmap aud_##_name = { \
.hw.init = &(struct clk_init_data) { \
.name = "aud_"#_name, \
.ops = &meson_sclk_div_ops, \
- .parent_names = (const char *[]) { _pname }, \
+ .parent_hws = (const struct clk_hw *[]) { &_phws.hw }, \
.num_parents = 1, \
.flags = (_iflags), \
}, \
@@ -203,7 +220,7 @@ struct clk_regmap aud_##_name = { \
#define AUD_MST_SCLK_DIV(_name, _reg) \
AUD_SCLK_DIV(mst_##_name##_sclk_div, _reg, 20, 10, 0, 0, \
- "aud_mst_"#_name"_sclk_pre_en", \
+ aud_mst_##_name##_sclk_pre_en, \
CLK_SET_RATE_PARENT)
static AUD_MST_SCLK_DIV(a, AUDIO_MST_A_SCLK_CTRL0);
@@ -214,8 +231,8 @@ static AUD_MST_SCLK_DIV(e, AUDIO_MST_E_SCLK_CTRL0);
static AUD_MST_SCLK_DIV(f, AUDIO_MST_F_SCLK_CTRL0);
#define AUD_MST_SCLK_POST_EN(_name, _reg) \
- AUD_GATE(mst_##_name##_sclk_post_en, _reg, 30, \
- "aud_mst_"#_name"_sclk_div", CLK_SET_RATE_PARENT)
+ AUD_GATE(mst_##_name##_sclk_post_en, _reg, 30, \
+ aud_mst_##_name##_sclk_div, CLK_SET_RATE_PARENT)
static AUD_MST_SCLK_POST_EN(a, AUDIO_MST_A_SCLK_CTRL0);
static AUD_MST_SCLK_POST_EN(b, AUDIO_MST_B_SCLK_CTRL0);
@@ -224,8 +241,8 @@ static AUD_MST_SCLK_POST_EN(d, AUDIO_MST_D_SCLK_CTRL0);
static AUD_MST_SCLK_POST_EN(e, AUDIO_MST_E_SCLK_CTRL0);
static AUD_MST_SCLK_POST_EN(f, AUDIO_MST_F_SCLK_CTRL0);
-#define AUD_TRIPHASE(_name, _reg, _width, _shift0, _shift1, _shift2, \
- _pname, _iflags) \
+#define AUD_TRIPHASE(_name, _reg, _width, _shift0, _shift1, _shift2, \
+ _phws, _iflags) \
struct clk_regmap aud_##_name = { \
.data = &(struct meson_clk_triphase_data) { \
.ph0 = { \
@@ -247,7 +264,7 @@ struct clk_regmap aud_##_name = { \
.hw.init = &(struct clk_init_data) { \
.name = "aud_"#_name, \
.ops = &meson_clk_triphase_ops, \
- .parent_names = (const char *[]) { _pname }, \
+ .parent_hws = (const struct clk_hw *[]) { &_phws.hw }, \
.num_parents = 1, \
.flags = CLK_DUTY_CYCLE_PARENT | (_iflags), \
}, \
@@ -255,7 +272,7 @@ struct clk_regmap aud_##_name = { \
#define AUD_MST_SCLK(_name, _reg) \
AUD_TRIPHASE(mst_##_name##_sclk, _reg, 1, 0, 2, 4, \
- "aud_mst_"#_name"_sclk_post_en", CLK_SET_RATE_PARENT)
+ aud_mst_##_name##_sclk_post_en, CLK_SET_RATE_PARENT)
static AUD_MST_SCLK(a, AUDIO_MST_A_SCLK_CTRL1);
static AUD_MST_SCLK(b, AUDIO_MST_B_SCLK_CTRL1);
@@ -266,7 +283,7 @@ static AUD_MST_SCLK(f, AUDIO_MST_F_SCLK_CTRL1);
#define AUD_MST_LRCLK_DIV(_name, _reg) \
AUD_SCLK_DIV(mst_##_name##_lrclk_div, _reg, 0, 10, 10, 10, \
- "aud_mst_"#_name"_sclk_post_en", 0) \
+ aud_mst_##_name##_sclk_post_en, 0) \
static AUD_MST_LRCLK_DIV(a, AUDIO_MST_A_SCLK_CTRL0);
static AUD_MST_LRCLK_DIV(b, AUDIO_MST_B_SCLK_CTRL0);
@@ -277,7 +294,7 @@ static AUD_MST_LRCLK_DIV(f, AUDIO_MST_F_SCLK_CTRL0);
#define AUD_MST_LRCLK(_name, _reg) \
AUD_TRIPHASE(mst_##_name##_lrclk, _reg, 1, 1, 3, 5, \
- "aud_mst_"#_name"_lrclk_div", CLK_SET_RATE_PARENT)
+ aud_mst_##_name##_lrclk_div, CLK_SET_RATE_PARENT)
static AUD_MST_LRCLK(a, AUDIO_MST_A_SCLK_CTRL1);
static AUD_MST_LRCLK(b, AUDIO_MST_B_SCLK_CTRL1);
@@ -286,19 +303,29 @@ static AUD_MST_LRCLK(d, AUDIO_MST_D_SCLK_CTRL1);
static AUD_MST_LRCLK(e, AUDIO_MST_E_SCLK_CTRL1);
static AUD_MST_LRCLK(f, AUDIO_MST_F_SCLK_CTRL1);
-static const char * const tdm_sclk_parent_names[] = {
- "aud_mst_a_sclk", "aud_mst_b_sclk", "aud_mst_c_sclk",
- "aud_mst_d_sclk", "aud_mst_e_sclk", "aud_mst_f_sclk",
- "aud_slv_sclk0", "aud_slv_sclk1", "aud_slv_sclk2",
- "aud_slv_sclk3", "aud_slv_sclk4", "aud_slv_sclk5",
- "aud_slv_sclk6", "aud_slv_sclk7", "aud_slv_sclk8",
- "aud_slv_sclk9"
+static const struct clk_parent_data tdm_sclk_parent_data[] = {
+ { .hw = &aud_mst_a_sclk.hw, },
+ { .hw = &aud_mst_b_sclk.hw, },
+ { .hw = &aud_mst_c_sclk.hw, },
+ { .hw = &aud_mst_d_sclk.hw, },
+ { .hw = &aud_mst_e_sclk.hw, },
+ { .hw = &aud_mst_f_sclk.hw, },
+ { .fw_name = "slv_sclk0", },
+ { .fw_name = "slv_sclk1", },
+ { .fw_name = "slv_sclk2", },
+ { .fw_name = "slv_sclk3", },
+ { .fw_name = "slv_sclk4", },
+ { .fw_name = "slv_sclk5", },
+ { .fw_name = "slv_sclk6", },
+ { .fw_name = "slv_sclk7", },
+ { .fw_name = "slv_sclk8", },
+ { .fw_name = "slv_sclk9", },
};
#define AUD_TDM_SCLK_MUX(_name, _reg) \
AUD_MUX(tdm##_name##_sclk_sel, _reg, 0xf, 24, \
CLK_MUX_ROUND_CLOSEST, \
- tdm_sclk_parent_names, 0)
+ tdm_sclk_parent_data, 0)
static AUD_TDM_SCLK_MUX(in_a, AUDIO_CLK_TDMIN_A_CTRL);
static AUD_TDM_SCLK_MUX(in_b, AUDIO_CLK_TDMIN_B_CTRL);
@@ -310,7 +337,7 @@ static AUD_TDM_SCLK_MUX(out_c, AUDIO_CLK_TDMOUT_C_CTRL);
#define AUD_TDM_SCLK_PRE_EN(_name, _reg) \
AUD_GATE(tdm##_name##_sclk_pre_en, _reg, 31, \
- "aud_tdm"#_name"_sclk_sel", CLK_SET_RATE_PARENT)
+ aud_tdm##_name##_sclk_sel, CLK_SET_RATE_PARENT)
static AUD_TDM_SCLK_PRE_EN(in_a, AUDIO_CLK_TDMIN_A_CTRL);
static AUD_TDM_SCLK_PRE_EN(in_b, AUDIO_CLK_TDMIN_B_CTRL);
@@ -322,7 +349,7 @@ static AUD_TDM_SCLK_PRE_EN(out_c, AUDIO_CLK_TDMOUT_C_CTRL);
#define AUD_TDM_SCLK_POST_EN(_name, _reg) \
AUD_GATE(tdm##_name##_sclk_post_en, _reg, 30, \
- "aud_tdm"#_name"_sclk_pre_en", CLK_SET_RATE_PARENT)
+ aud_tdm##_name##_sclk_pre_en, CLK_SET_RATE_PARENT)
static AUD_TDM_SCLK_POST_EN(in_a, AUDIO_CLK_TDMIN_A_CTRL);
static AUD_TDM_SCLK_POST_EN(in_b, AUDIO_CLK_TDMIN_B_CTRL);
@@ -344,8 +371,9 @@ static AUD_TDM_SCLK_POST_EN(out_c, AUDIO_CLK_TDMOUT_C_CTRL);
.hw.init = &(struct clk_init_data) { \
.name = "aud_tdm"#_name"_sclk", \
.ops = &meson_clk_phase_ops, \
- .parent_names = (const char *[]) \
- { "aud_tdm"#_name"_sclk_post_en" }, \
+ .parent_hws = (const struct clk_hw *[]) { \
+ &aud_tdm##_name##_sclk_post_en.hw \
+ }, \
.num_parents = 1, \
.flags = CLK_DUTY_CYCLE_PARENT | CLK_SET_RATE_PARENT, \
}, \
@@ -359,19 +387,29 @@ static AUD_TDM_SCLK(out_a, AUDIO_CLK_TDMOUT_A_CTRL);
static AUD_TDM_SCLK(out_b, AUDIO_CLK_TDMOUT_B_CTRL);
static AUD_TDM_SCLK(out_c, AUDIO_CLK_TDMOUT_C_CTRL);
-static const char * const tdm_lrclk_parent_names[] = {
- "aud_mst_a_lrclk", "aud_mst_b_lrclk", "aud_mst_c_lrclk",
- "aud_mst_d_lrclk", "aud_mst_e_lrclk", "aud_mst_f_lrclk",
- "aud_slv_lrclk0", "aud_slv_lrclk1", "aud_slv_lrclk2",
- "aud_slv_lrclk3", "aud_slv_lrclk4", "aud_slv_lrclk5",
- "aud_slv_lrclk6", "aud_slv_lrclk7", "aud_slv_lrclk8",
- "aud_slv_lrclk9"
+static const struct clk_parent_data tdm_lrclk_parent_data[] = {
+ { .hw = &aud_mst_a_lrclk.hw, },
+ { .hw = &aud_mst_b_lrclk.hw, },
+ { .hw = &aud_mst_c_lrclk.hw, },
+ { .hw = &aud_mst_d_lrclk.hw, },
+ { .hw = &aud_mst_e_lrclk.hw, },
+ { .hw = &aud_mst_f_lrclk.hw, },
+ { .fw_name = "slv_lrclk0", },
+ { .fw_name = "slv_lrclk1", },
+ { .fw_name = "slv_lrclk2", },
+ { .fw_name = "slv_lrclk3", },
+ { .fw_name = "slv_lrclk4", },
+ { .fw_name = "slv_lrclk5", },
+ { .fw_name = "slv_lrclk6", },
+ { .fw_name = "slv_lrclk7", },
+ { .fw_name = "slv_lrclk8", },
+ { .fw_name = "slv_lrclk9", },
};
-#define AUD_TDM_LRLCK(_name, _reg) \
- AUD_MUX(tdm##_name##_lrclk, _reg, 0xf, 20, \
- CLK_MUX_ROUND_CLOSEST, \
- tdm_lrclk_parent_names, 0)
+#define AUD_TDM_LRLCK(_name, _reg) \
+ AUD_MUX(tdm##_name##_lrclk, _reg, 0xf, 20, \
+ CLK_MUX_ROUND_CLOSEST, \
+ tdm_lrclk_parent_data, 0)
static AUD_TDM_LRLCK(in_a, AUDIO_CLK_TDMIN_A_CTRL);
static AUD_TDM_LRLCK(in_b, AUDIO_CLK_TDMIN_B_CTRL);
@@ -386,39 +424,51 @@ static AUD_TDM_LRLCK(out_c, AUDIO_CLK_TDMOUT_C_CTRL);
AUD_MUX(tdm_##_name, _reg, 0x7, _shift, 0, _parents, \
CLK_SET_RATE_NO_REPARENT)
-static const char * const mclk_pad_ctrl_parent_names[] = {
- "aud_mst_a_mclk", "aud_mst_b_mclk", "aud_mst_c_mclk",
- "aud_mst_d_mclk", "aud_mst_e_mclk", "aud_mst_f_mclk",
+static const struct clk_parent_data mclk_pad_ctrl_parent_data[] = {
+ { .hw = &aud_mst_a_mclk.hw },
+ { .hw = &aud_mst_b_mclk.hw },
+ { .hw = &aud_mst_c_mclk.hw },
+ { .hw = &aud_mst_d_mclk.hw },
+ { .hw = &aud_mst_e_mclk.hw },
+ { .hw = &aud_mst_f_mclk.hw },
};
static AUD_TDM_PAD_CTRL(mclk_pad_0, AUDIO_MST_PAD_CTRL0, 0,
- mclk_pad_ctrl_parent_names);
+ mclk_pad_ctrl_parent_data);
static AUD_TDM_PAD_CTRL(mclk_pad_1, AUDIO_MST_PAD_CTRL0, 4,
- mclk_pad_ctrl_parent_names);
-
-static const char * const lrclk_pad_ctrl_parent_names[] = {
- "aud_mst_a_lrclk", "aud_mst_b_lrclk", "aud_mst_c_lrclk",
- "aud_mst_d_lrclk", "aud_mst_e_lrclk", "aud_mst_f_lrclk",
+ mclk_pad_ctrl_parent_data);
+
+static const struct clk_parent_data lrclk_pad_ctrl_parent_data[] = {
+ { .hw = &aud_mst_a_lrclk.hw },
+ { .hw = &aud_mst_b_lrclk.hw },
+ { .hw = &aud_mst_c_lrclk.hw },
+ { .hw = &aud_mst_d_lrclk.hw },
+ { .hw = &aud_mst_e_lrclk.hw },
+ { .hw = &aud_mst_f_lrclk.hw },
};
static AUD_TDM_PAD_CTRL(lrclk_pad_0, AUDIO_MST_PAD_CTRL1, 16,
- lrclk_pad_ctrl_parent_names);
+ lrclk_pad_ctrl_parent_data);
static AUD_TDM_PAD_CTRL(lrclk_pad_1, AUDIO_MST_PAD_CTRL1, 20,
- lrclk_pad_ctrl_parent_names);
+ lrclk_pad_ctrl_parent_data);
static AUD_TDM_PAD_CTRL(lrclk_pad_2, AUDIO_MST_PAD_CTRL1, 24,
- lrclk_pad_ctrl_parent_names);
-
-static const char * const sclk_pad_ctrl_parent_names[] = {
- "aud_mst_a_sclk", "aud_mst_b_sclk", "aud_mst_c_sclk",
- "aud_mst_d_sclk", "aud_mst_e_sclk", "aud_mst_f_sclk",
+ lrclk_pad_ctrl_parent_data);
+
+static const struct clk_parent_data sclk_pad_ctrl_parent_data[] = {
+ { .hw = &aud_mst_a_sclk.hw },
+ { .hw = &aud_mst_b_sclk.hw },
+ { .hw = &aud_mst_c_sclk.hw },
+ { .hw = &aud_mst_d_sclk.hw },
+ { .hw = &aud_mst_e_sclk.hw },
+ { .hw = &aud_mst_f_sclk.hw },
};
static AUD_TDM_PAD_CTRL(sclk_pad_0, AUDIO_MST_PAD_CTRL1, 0,
- sclk_pad_ctrl_parent_names);
+ sclk_pad_ctrl_parent_data);
static AUD_TDM_PAD_CTRL(sclk_pad_1, AUDIO_MST_PAD_CTRL1, 4,
- sclk_pad_ctrl_parent_names);
+ sclk_pad_ctrl_parent_data);
static AUD_TDM_PAD_CTRL(sclk_pad_2, AUDIO_MST_PAD_CTRL1, 8,
- sclk_pad_ctrl_parent_names);
+ sclk_pad_ctrl_parent_data);
/*
* Array of all clocks provided by this provider
@@ -868,54 +918,6 @@ static int devm_clk_get_enable(struct device *dev, char *id)
return 0;
}
-static int axg_register_clk_hw_input(struct device *dev,
- const char *name)
-{
- char *clk_name;
- struct clk_hw *hw;
- int err = 0;
-
- clk_name = kasprintf(GFP_KERNEL, "aud_%s", name);
- if (!clk_name)
- return -ENOMEM;
-
- hw = meson_clk_hw_register_input(dev, name, clk_name, 0);
- if (IS_ERR(hw)) {
- /* It is ok if an input clock is missing */
- if (PTR_ERR(hw) == -ENOENT) {
- dev_dbg(dev, "%s not provided", name);
- } else {
- err = PTR_ERR(hw);
- if (err != -EPROBE_DEFER)
- dev_err(dev, "failed to get %s clock", name);
- }
- }
-
- kfree(clk_name);
- return err;
-}
-
-static int axg_register_clk_hw_inputs(struct device *dev,
- const char *basename,
- unsigned int count)
-{
- char *name;
- int i, ret;
-
- for (i = 0; i < count; i++) {
- name = kasprintf(GFP_KERNEL, "%s%d", basename, i);
- if (!name)
- return -ENOMEM;
-
- ret = axg_register_clk_hw_input(dev, name);
- kfree(name);
- if (ret)
- return ret;
- }
-
- return 0;
-}
-
static const struct regmap_config axg_audio_regmap_cfg = {
.reg_bits = 32,
.val_bits = 32,
@@ -963,29 +965,6 @@ static int axg_audio_clkc_probe(struct platform_device *pdev)
return ret;
}
- /* Register the peripheral input clock */
- hw = meson_clk_hw_register_input(dev, "pclk", "audio_pclk", 0);
- if (IS_ERR(hw))
- return PTR_ERR(hw);
-
- /* Register optional input master clocks */
- ret = axg_register_clk_hw_inputs(dev, "mst_in",
- AUD_MST_IN_COUNT);
- if (ret)
- return ret;
-
- /* Register optional input slave sclks */
- ret = axg_register_clk_hw_inputs(dev, "slv_sclk",
- AUD_SLV_SCLK_COUNT);
- if (ret)
- return ret;
-
- /* Register optional input slave lrclks */
- ret = axg_register_clk_hw_inputs(dev, "slv_lrclk",
- AUD_SLV_LRCLK_COUNT);
- if (ret)
- return ret;
-
/* Populate regmap for the regmap backed clocks */
for (i = 0; i < ARRAY_SIZE(aud_clk_regmaps); i++)
aud_clk_regmaps[i]->map = map;
diff --git a/drivers/clk/meson/axg.c b/drivers/clk/meson/axg.c
index 7a8ef80e5f2c..13fc0006f63d 100644
--- a/drivers/clk/meson/axg.c
+++ b/drivers/clk/meson/axg.c
@@ -14,7 +14,6 @@
#include <linux/of_device.h>
#include <linux/platform_device.h>
-#include "clk-input.h"
#include "clk-regmap.h"
#include "clk-pll.h"
#include "clk-mpll.h"
@@ -59,7 +58,9 @@ static struct clk_regmap axg_fixed_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "fixed_pll_dco",
.ops = &meson_clk_pll_ro_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
},
};
@@ -74,7 +75,9 @@ static struct clk_regmap axg_fixed_pll = {
.hw.init = &(struct clk_init_data){
.name = "fixed_pll",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "fixed_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_fixed_pll_dco.hw
+ },
.num_parents = 1,
/*
* This clock won't ever change at runtime so
@@ -114,7 +117,9 @@ static struct clk_regmap axg_sys_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "sys_pll_dco",
.ops = &meson_clk_pll_ro_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
},
};
@@ -129,7 +134,9 @@ static struct clk_regmap axg_sys_pll = {
.hw.init = &(struct clk_init_data){
.name = "sys_pll",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "sys_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_sys_pll_dco.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -215,7 +222,9 @@ static struct clk_regmap axg_gp0_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "gp0_pll_dco",
.ops = &meson_clk_pll_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
},
};
@@ -230,7 +239,9 @@ static struct clk_regmap axg_gp0_pll = {
.hw.init = &(struct clk_init_data){
.name = "gp0_pll",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "gp0_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_gp0_pll_dco.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -284,7 +295,9 @@ static struct clk_regmap axg_hifi_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "hifi_pll_dco",
.ops = &meson_clk_pll_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
},
};
@@ -299,7 +312,9 @@ static struct clk_regmap axg_hifi_pll = {
.hw.init = &(struct clk_init_data){
.name = "hifi_pll",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "hifi_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_hifi_pll_dco.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -311,7 +326,7 @@ static struct clk_fixed_factor axg_fclk_div2_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div2_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) { &axg_fixed_pll.hw },
.num_parents = 1,
},
};
@@ -324,7 +339,9 @@ static struct clk_regmap axg_fclk_div2 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div2",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div2_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_fclk_div2_div.hw
+ },
.num_parents = 1,
.flags = CLK_IS_CRITICAL,
},
@@ -336,7 +353,7 @@ static struct clk_fixed_factor axg_fclk_div3_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div3_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) { &axg_fixed_pll.hw },
.num_parents = 1,
},
};
@@ -349,7 +366,9 @@ static struct clk_regmap axg_fclk_div3 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div3",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div3_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_fclk_div3_div.hw
+ },
.num_parents = 1,
/*
* FIXME:
@@ -372,7 +391,7 @@ static struct clk_fixed_factor axg_fclk_div4_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div4_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) { &axg_fixed_pll.hw },
.num_parents = 1,
},
};
@@ -385,7 +404,9 @@ static struct clk_regmap axg_fclk_div4 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div4",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div4_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_fclk_div4_div.hw
+ },
.num_parents = 1,
},
};
@@ -396,7 +417,7 @@ static struct clk_fixed_factor axg_fclk_div5_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div5_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) { &axg_fixed_pll.hw },
.num_parents = 1,
},
};
@@ -409,7 +430,9 @@ static struct clk_regmap axg_fclk_div5 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div5",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div5_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_fclk_div5_div.hw
+ },
.num_parents = 1,
},
};
@@ -420,7 +443,9 @@ static struct clk_fixed_factor axg_fclk_div7_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div7_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_fixed_pll.hw
+ },
.num_parents = 1,
},
};
@@ -433,7 +458,9 @@ static struct clk_regmap axg_fclk_div7 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div7",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div7_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_fclk_div7_div.hw
+ },
.num_parents = 1,
},
};
@@ -447,7 +474,9 @@ static struct clk_regmap axg_mpll_prediv = {
.hw.init = &(struct clk_init_data){
.name = "mpll_prediv",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_fixed_pll.hw
+ },
.num_parents = 1,
},
};
@@ -469,11 +498,6 @@ static struct clk_regmap axg_mpll0_div = {
.shift = 16,
.width = 9,
},
- .ssen = {
- .reg_off = HHI_MPLL_CNTL,
- .shift = 25,
- .width = 1,
- },
.misc = {
.reg_off = HHI_PLL_TOP_MISC,
.shift = 0,
@@ -485,7 +509,9 @@ static struct clk_regmap axg_mpll0_div = {
.hw.init = &(struct clk_init_data){
.name = "mpll0_div",
.ops = &meson_clk_mpll_ops,
- .parent_names = (const char *[]){ "mpll_prediv" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_mpll_prediv.hw
+ },
.num_parents = 1,
},
};
@@ -498,7 +524,9 @@ static struct clk_regmap axg_mpll0 = {
.hw.init = &(struct clk_init_data){
.name = "mpll0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpll0_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_mpll0_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -532,7 +560,9 @@ static struct clk_regmap axg_mpll1_div = {
.hw.init = &(struct clk_init_data){
.name = "mpll1_div",
.ops = &meson_clk_mpll_ops,
- .parent_names = (const char *[]){ "mpll_prediv" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_mpll_prediv.hw
+ },
.num_parents = 1,
},
};
@@ -545,7 +575,9 @@ static struct clk_regmap axg_mpll1 = {
.hw.init = &(struct clk_init_data){
.name = "mpll1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpll1_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_mpll1_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -568,6 +600,11 @@ static struct clk_regmap axg_mpll2_div = {
.shift = 16,
.width = 9,
},
+ .ssen = {
+ .reg_off = HHI_MPLL_CNTL,
+ .shift = 25,
+ .width = 1,
+ },
.misc = {
.reg_off = HHI_PLL_TOP_MISC,
.shift = 2,
@@ -579,7 +616,9 @@ static struct clk_regmap axg_mpll2_div = {
.hw.init = &(struct clk_init_data){
.name = "mpll2_div",
.ops = &meson_clk_mpll_ops,
- .parent_names = (const char *[]){ "mpll_prediv" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_mpll_prediv.hw
+ },
.num_parents = 1,
},
};
@@ -592,7 +631,9 @@ static struct clk_regmap axg_mpll2 = {
.hw.init = &(struct clk_init_data){
.name = "mpll2",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpll2_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_mpll2_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -626,7 +667,9 @@ static struct clk_regmap axg_mpll3_div = {
.hw.init = &(struct clk_init_data){
.name = "mpll3_div",
.ops = &meson_clk_mpll_ops,
- .parent_names = (const char *[]){ "mpll_prediv" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_mpll_prediv.hw
+ },
.num_parents = 1,
},
};
@@ -639,7 +682,9 @@ static struct clk_regmap axg_mpll3 = {
.hw.init = &(struct clk_init_data){
.name = "mpll3",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpll3_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_mpll3_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -702,7 +747,9 @@ static struct clk_regmap axg_pcie_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "pcie_pll_dco",
.ops = &meson_clk_pll_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
},
};
@@ -717,7 +764,9 @@ static struct clk_regmap axg_pcie_pll_od = {
.hw.init = &(struct clk_init_data){
.name = "pcie_pll_od",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "pcie_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_pcie_pll_dco.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -733,7 +782,9 @@ static struct clk_regmap axg_pcie_pll = {
.hw.init = &(struct clk_init_data){
.name = "pcie_pll",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "pcie_pll_od" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_pcie_pll_od.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -750,7 +801,7 @@ static struct clk_regmap axg_pcie_mux = {
.hw.init = &(struct clk_init_data){
.name = "pcie_mux",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "pcie_pll" },
+ .parent_hws = (const struct clk_hw *[]) { &axg_pcie_pll.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -767,7 +818,7 @@ static struct clk_regmap axg_pcie_ref = {
.hw.init = &(struct clk_init_data){
.name = "pcie_ref",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "pcie_mux" },
+ .parent_hws = (const struct clk_hw *[]) { &axg_pcie_mux.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -781,7 +832,7 @@ static struct clk_regmap axg_pcie_cml_en0 = {
.hw.init = &(struct clk_init_data) {
.name = "pcie_cml_en0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "pcie_ref" },
+ .parent_hws = (const struct clk_hw *[]) { &axg_pcie_ref.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
@@ -796,16 +847,21 @@ static struct clk_regmap axg_pcie_cml_en1 = {
.hw.init = &(struct clk_init_data) {
.name = "pcie_cml_en1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "pcie_ref" },
+ .parent_hws = (const struct clk_hw *[]) { &axg_pcie_ref.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
};
static u32 mux_table_clk81[] = { 0, 2, 3, 4, 5, 6, 7 };
-static const char * const clk81_parent_names[] = {
- IN_PREFIX "xtal", "fclk_div7", "mpll1", "mpll2", "fclk_div4",
- "fclk_div3", "fclk_div5"
+static const struct clk_parent_data clk81_parent_data[] = {
+ { .fw_name = "xtal", },
+ { .hw = &axg_fclk_div7.hw },
+ { .hw = &axg_mpll1.hw },
+ { .hw = &axg_mpll2.hw },
+ { .hw = &axg_fclk_div4.hw },
+ { .hw = &axg_fclk_div3.hw },
+ { .hw = &axg_fclk_div5.hw },
};
static struct clk_regmap axg_mpeg_clk_sel = {
@@ -818,8 +874,8 @@ static struct clk_regmap axg_mpeg_clk_sel = {
.hw.init = &(struct clk_init_data){
.name = "mpeg_clk_sel",
.ops = &clk_regmap_mux_ro_ops,
- .parent_names = clk81_parent_names,
- .num_parents = ARRAY_SIZE(clk81_parent_names),
+ .parent_data = clk81_parent_data,
+ .num_parents = ARRAY_SIZE(clk81_parent_data),
},
};
@@ -832,7 +888,9 @@ static struct clk_regmap axg_mpeg_clk_div = {
.hw.init = &(struct clk_init_data){
.name = "mpeg_clk_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "mpeg_clk_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_mpeg_clk_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -846,15 +904,20 @@ static struct clk_regmap axg_clk81 = {
.hw.init = &(struct clk_init_data){
.name = "clk81",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpeg_clk_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_mpeg_clk_div.hw
+ },
.num_parents = 1,
.flags = (CLK_SET_RATE_PARENT | CLK_IS_CRITICAL),
},
};
-static const char * const axg_sd_emmc_clk0_parent_names[] = {
- IN_PREFIX "xtal", "fclk_div2", "fclk_div3", "fclk_div5", "fclk_div7",
-
+static const struct clk_parent_data axg_sd_emmc_clk0_parent_data[] = {
+ { .fw_name = "xtal", },
+ { .hw = &axg_fclk_div2.hw },
+ { .hw = &axg_fclk_div3.hw },
+ { .hw = &axg_fclk_div5.hw },
+ { .hw = &axg_fclk_div7.hw },
/*
* Following these parent clocks, we should also have had mpll2, mpll3
* and gp0_pll but these clocks are too precious to be used here. All
@@ -873,8 +936,8 @@ static struct clk_regmap axg_sd_emmc_b_clk0_sel = {
.hw.init = &(struct clk_init_data) {
.name = "sd_emmc_b_clk0_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = axg_sd_emmc_clk0_parent_names,
- .num_parents = ARRAY_SIZE(axg_sd_emmc_clk0_parent_names),
+ .parent_data = axg_sd_emmc_clk0_parent_data,
+ .num_parents = ARRAY_SIZE(axg_sd_emmc_clk0_parent_data),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -889,7 +952,9 @@ static struct clk_regmap axg_sd_emmc_b_clk0_div = {
.hw.init = &(struct clk_init_data) {
.name = "sd_emmc_b_clk0_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "sd_emmc_b_clk0_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_sd_emmc_b_clk0_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -903,7 +968,9 @@ static struct clk_regmap axg_sd_emmc_b_clk0 = {
.hw.init = &(struct clk_init_data){
.name = "sd_emmc_b_clk0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "sd_emmc_b_clk0_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_sd_emmc_b_clk0_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -919,8 +986,8 @@ static struct clk_regmap axg_sd_emmc_c_clk0_sel = {
.hw.init = &(struct clk_init_data) {
.name = "sd_emmc_c_clk0_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = axg_sd_emmc_clk0_parent_names,
- .num_parents = ARRAY_SIZE(axg_sd_emmc_clk0_parent_names),
+ .parent_data = axg_sd_emmc_clk0_parent_data,
+ .num_parents = ARRAY_SIZE(axg_sd_emmc_clk0_parent_data),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -935,7 +1002,9 @@ static struct clk_regmap axg_sd_emmc_c_clk0_div = {
.hw.init = &(struct clk_init_data) {
.name = "sd_emmc_c_clk0_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "sd_emmc_c_clk0_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_sd_emmc_c_clk0_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -949,7 +1018,9 @@ static struct clk_regmap axg_sd_emmc_c_clk0 = {
.hw.init = &(struct clk_init_data){
.name = "sd_emmc_c_clk0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "sd_emmc_c_clk0_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_sd_emmc_c_clk0_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -957,9 +1028,18 @@ static struct clk_regmap axg_sd_emmc_c_clk0 = {
static u32 mux_table_gen_clk[] = { 0, 4, 5, 6, 7, 8,
9, 10, 11, 13, 14, };
-static const char * const gen_clk_parent_names[] = {
- IN_PREFIX "xtal", "hifi_pll", "mpll0", "mpll1", "mpll2", "mpll3",
- "fclk_div4", "fclk_div3", "fclk_div5", "fclk_div7", "gp0_pll",
+static const struct clk_parent_data gen_clk_parent_data[] = {
+ { .fw_name = "xtal", },
+ { .hw = &axg_hifi_pll.hw },
+ { .hw = &axg_mpll0.hw },
+ { .hw = &axg_mpll1.hw },
+ { .hw = &axg_mpll2.hw },
+ { .hw = &axg_mpll3.hw },
+ { .hw = &axg_fclk_div4.hw },
+ { .hw = &axg_fclk_div3.hw },
+ { .hw = &axg_fclk_div5.hw },
+ { .hw = &axg_fclk_div7.hw },
+ { .hw = &axg_gp0_pll.hw },
};
static struct clk_regmap axg_gen_clk_sel = {
@@ -978,8 +1058,8 @@ static struct clk_regmap axg_gen_clk_sel = {
* hifi_pll, mpll0, mpll1, mpll2, mpll3, fdiv4,
* fdiv3, fdiv5, [cts_msr_clk], fdiv7, gp0_pll
*/
- .parent_names = gen_clk_parent_names,
- .num_parents = ARRAY_SIZE(gen_clk_parent_names),
+ .parent_data = gen_clk_parent_data,
+ .num_parents = ARRAY_SIZE(gen_clk_parent_data),
},
};
@@ -992,7 +1072,9 @@ static struct clk_regmap axg_gen_clk_div = {
.hw.init = &(struct clk_init_data){
.name = "gen_clk_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "gen_clk_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_gen_clk_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1006,12 +1088,17 @@ static struct clk_regmap axg_gen_clk = {
.hw.init = &(struct clk_init_data){
.name = "gen_clk",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "gen_clk_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &axg_gen_clk_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
};
+#define MESON_GATE(_name, _reg, _bit) \
+ MESON_PCLK(_name, _reg, _bit, &axg_clk81.hw)
+
/* Everything Else (EE) domain gates */
static MESON_GATE(axg_ddr, HHI_GCLK_MPEG0, 0);
static MESON_GATE(axg_audio_locker, HHI_GCLK_MPEG0, 2);
diff --git a/drivers/clk/meson/clk-cpu-dyndiv.c b/drivers/clk/meson/clk-cpu-dyndiv.c
new file mode 100644
index 000000000000..36976927fe82
--- /dev/null
+++ b/drivers/clk/meson/clk-cpu-dyndiv.c
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (c) 2019 BayLibre, SAS.
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/module.h>
+
+#include "clk-regmap.h"
+#include "clk-cpu-dyndiv.h"
+
+static inline struct meson_clk_cpu_dyndiv_data *
+meson_clk_cpu_dyndiv_data(struct clk_regmap *clk)
+{
+ return (struct meson_clk_cpu_dyndiv_data *)clk->data;
+}
+
+static unsigned long meson_clk_cpu_dyndiv_recalc_rate(struct clk_hw *hw,
+ unsigned long prate)
+{
+ struct clk_regmap *clk = to_clk_regmap(hw);
+ struct meson_clk_cpu_dyndiv_data *data = meson_clk_cpu_dyndiv_data(clk);
+
+ return divider_recalc_rate(hw, prate,
+ meson_parm_read(clk->map, &data->div),
+ NULL, 0, data->div.width);
+}
+
+static long meson_clk_cpu_dyndiv_round_rate(struct clk_hw *hw,
+ unsigned long rate,
+ unsigned long *prate)
+{
+ struct clk_regmap *clk = to_clk_regmap(hw);
+ struct meson_clk_cpu_dyndiv_data *data = meson_clk_cpu_dyndiv_data(clk);
+
+ return divider_round_rate(hw, rate, prate, NULL, data->div.width, 0);
+}
+
+static int meson_clk_cpu_dyndiv_set_rate(struct clk_hw *hw, unsigned long rate,
+ unsigned long parent_rate)
+{
+ struct clk_regmap *clk = to_clk_regmap(hw);
+ struct meson_clk_cpu_dyndiv_data *data = meson_clk_cpu_dyndiv_data(clk);
+ unsigned int val;
+ int ret;
+
+ ret = divider_get_val(rate, parent_rate, NULL, data->div.width, 0);
+ if (ret < 0)
+ return ret;
+
+ val = (unsigned int)ret << data->div.shift;
+
+ /* Write the SYS_CPU_DYN_ENABLE bit before changing the divider */
+ meson_parm_write(clk->map, &data->dyn, 1);
+
+ /* Update the divider while removing the SYS_CPU_DYN_ENABLE bit */
+ return regmap_update_bits(clk->map, data->div.reg_off,
+ SETPMASK(data->div.width, data->div.shift) |
+ SETPMASK(data->dyn.width, data->dyn.shift),
+ val);
+};
+
+const struct clk_ops meson_clk_cpu_dyndiv_ops = {
+ .recalc_rate = meson_clk_cpu_dyndiv_recalc_rate,
+ .round_rate = meson_clk_cpu_dyndiv_round_rate,
+ .set_rate = meson_clk_cpu_dyndiv_set_rate,
+};
+EXPORT_SYMBOL_GPL(meson_clk_cpu_dyndiv_ops);
+
+MODULE_DESCRIPTION("Amlogic CPU Dynamic Clock divider");
+MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/meson/clk-cpu-dyndiv.h b/drivers/clk/meson/clk-cpu-dyndiv.h
new file mode 100644
index 000000000000..f4908404792e
--- /dev/null
+++ b/drivers/clk/meson/clk-cpu-dyndiv.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2019 BayLibre, SAS.
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ */
+
+#ifndef __MESON_CLK_CPU_DYNDIV_H
+#define __MESON_CLK_CPU_DYNDIV_H
+
+#include <linux/clk-provider.h>
+#include "parm.h"
+
+struct meson_clk_cpu_dyndiv_data {
+ struct parm div;
+ struct parm dyn;
+};
+
+extern const struct clk_ops meson_clk_cpu_dyndiv_ops;
+
+#endif /* __MESON_CLK_CPU_DYNDIV_H */
diff --git a/drivers/clk/meson/clk-input.c b/drivers/clk/meson/clk-input.c
deleted file mode 100644
index 086226e9dba6..000000000000
--- a/drivers/clk/meson/clk-input.c
+++ /dev/null
@@ -1,49 +0,0 @@
-// SPDX-License-Identifier: (GPL-2.0 OR MIT)
-/*
- * Copyright (c) 2018 BayLibre, SAS.
- * Author: Jerome Brunet <jbrunet@baylibre.com>
- */
-
-#include <linux/clk.h>
-#include <linux/clk-provider.h>
-#include <linux/device.h>
-#include <linux/module.h>
-#include "clk-input.h"
-
-static const struct clk_ops meson_clk_no_ops = {};
-
-struct clk_hw *meson_clk_hw_register_input(struct device *dev,
- const char *of_name,
- const char *clk_name,
- unsigned long flags)
-{
- struct clk *parent_clk = devm_clk_get(dev, of_name);
- struct clk_init_data init;
- const char *parent_name;
- struct clk_hw *hw;
- int ret;
-
- if (IS_ERR(parent_clk))
- return (struct clk_hw *)parent_clk;
-
- hw = devm_kzalloc(dev, sizeof(*hw), GFP_KERNEL);
- if (!hw)
- return ERR_PTR(-ENOMEM);
-
- parent_name = __clk_get_name(parent_clk);
- init.name = clk_name;
- init.ops = &meson_clk_no_ops;
- init.flags = flags;
- init.parent_names = &parent_name;
- init.num_parents = 1;
- hw->init = &init;
-
- ret = devm_clk_hw_register(dev, hw);
-
- return ret ? ERR_PTR(ret) : hw;
-}
-EXPORT_SYMBOL_GPL(meson_clk_hw_register_input);
-
-MODULE_DESCRIPTION("Amlogic clock input helper");
-MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/meson/clk-input.h b/drivers/clk/meson/clk-input.h
deleted file mode 100644
index 4a541b9685a6..000000000000
--- a/drivers/clk/meson/clk-input.h
+++ /dev/null
@@ -1,19 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (c) 2019 BayLibre, SAS.
- * Author: Jerome Brunet <jbrunet@baylibre.com>
- */
-
-#ifndef __MESON_CLK_INPUT_H
-#define __MESON_CLK_INPUT_H
-
-#include <linux/clk-provider.h>
-
-struct device;
-
-struct clk_hw *meson_clk_hw_register_input(struct device *dev,
- const char *of_name,
- const char *clk_name,
- unsigned long flags);
-
-#endif /* __MESON_CLK_INPUT_H */
diff --git a/drivers/clk/meson/clk-mpll.c b/drivers/clk/meson/clk-mpll.c
index f76850d99e59..2d39a8bc367c 100644
--- a/drivers/clk/meson/clk-mpll.c
+++ b/drivers/clk/meson/clk-mpll.c
@@ -115,21 +115,12 @@ static int mpll_set_rate(struct clk_hw *hw,
else
__acquire(mpll->lock);
- /* Enable and set the fractional part */
+ /* Set the fractional part */
meson_parm_write(clk->map, &mpll->sdm, sdm);
- meson_parm_write(clk->map, &mpll->sdm_en, 1);
-
- /* Set additional fractional part enable if required */
- if (MESON_PARM_APPLICABLE(&mpll->ssen))
- meson_parm_write(clk->map, &mpll->ssen, 1);
/* Set the integer divider part */
meson_parm_write(clk->map, &mpll->n2, n2);
- /* Set the magic misc bit if required */
- if (MESON_PARM_APPLICABLE(&mpll->misc))
- meson_parm_write(clk->map, &mpll->misc, 1);
-
if (mpll->lock)
spin_unlock_irqrestore(mpll->lock, flags);
else
@@ -138,6 +129,30 @@ static int mpll_set_rate(struct clk_hw *hw,
return 0;
}
+static void mpll_init(struct clk_hw *hw)
+{
+ struct clk_regmap *clk = to_clk_regmap(hw);
+ struct meson_clk_mpll_data *mpll = meson_clk_mpll_data(clk);
+
+ if (mpll->init_count)
+ regmap_multi_reg_write(clk->map, mpll->init_regs,
+ mpll->init_count);
+
+ /* Enable the fractional part */
+ meson_parm_write(clk->map, &mpll->sdm_en, 1);
+
+ /* Set spread spectrum if possible */
+ if (MESON_PARM_APPLICABLE(&mpll->ssen)) {
+ unsigned int ss =
+ mpll->flags & CLK_MESON_MPLL_SPREAD_SPECTRUM ? 1 : 0;
+ meson_parm_write(clk->map, &mpll->ssen, ss);
+ }
+
+ /* Set the magic misc bit if required */
+ if (MESON_PARM_APPLICABLE(&mpll->misc))
+ meson_parm_write(clk->map, &mpll->misc, 1);
+}
+
const struct clk_ops meson_clk_mpll_ro_ops = {
.recalc_rate = mpll_recalc_rate,
.round_rate = mpll_round_rate,
@@ -148,6 +163,7 @@ const struct clk_ops meson_clk_mpll_ops = {
.recalc_rate = mpll_recalc_rate,
.round_rate = mpll_round_rate,
.set_rate = mpll_set_rate,
+ .init = mpll_init,
};
EXPORT_SYMBOL_GPL(meson_clk_mpll_ops);
diff --git a/drivers/clk/meson/clk-mpll.h b/drivers/clk/meson/clk-mpll.h
index cf79340006dd..a991d568c43a 100644
--- a/drivers/clk/meson/clk-mpll.h
+++ b/drivers/clk/meson/clk-mpll.h
@@ -18,11 +18,14 @@ struct meson_clk_mpll_data {
struct parm n2;
struct parm ssen;
struct parm misc;
+ const struct reg_sequence *init_regs;
+ unsigned int init_count;
spinlock_t *lock;
u8 flags;
};
#define CLK_MESON_MPLL_ROUND_CLOSEST BIT(0)
+#define CLK_MESON_MPLL_SPREAD_SPECTRUM BIT(1)
extern const struct clk_ops meson_clk_mpll_ro_ops;
extern const struct clk_ops meson_clk_mpll_ops;
diff --git a/drivers/clk/meson/clk-regmap.h b/drivers/clk/meson/clk-regmap.h
index 1dd0abe3ba91..c4a39604cffd 100644
--- a/drivers/clk/meson/clk-regmap.h
+++ b/drivers/clk/meson/clk-regmap.h
@@ -111,7 +111,7 @@ clk_get_regmap_mux_data(struct clk_regmap *clk)
extern const struct clk_ops clk_regmap_mux_ops;
extern const struct clk_ops clk_regmap_mux_ro_ops;
-#define __MESON_GATE(_name, _reg, _bit, _ops) \
+#define __MESON_PCLK(_name, _reg, _bit, _ops, _pname) \
struct clk_regmap _name = { \
.data = &(struct clk_regmap_gate_data){ \
.offset = (_reg), \
@@ -120,15 +120,15 @@ struct clk_regmap _name = { \
.hw.init = &(struct clk_init_data) { \
.name = #_name, \
.ops = _ops, \
- .parent_names = (const char *[]){ "clk81" }, \
+ .parent_hws = (const struct clk_hw *[]) { _pname }, \
.num_parents = 1, \
.flags = (CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED), \
}, \
}
-#define MESON_GATE(_name, _reg, _bit) \
- __MESON_GATE(_name, _reg, _bit, &clk_regmap_gate_ops)
+#define MESON_PCLK(_name, _reg, _bit, _pname) \
+ __MESON_PCLK(_name, _reg, _bit, &clk_regmap_gate_ops, _pname)
-#define MESON_GATE_RO(_name, _reg, _bit) \
- __MESON_GATE(_name, _reg, _bit, &clk_regmap_gate_ro_ops)
+#define MESON_PCLK_RO(_name, _reg, _bit, _pname) \
+ __MESON_PCLK(_name, _reg, _bit, &clk_regmap_gate_ro_ops, _pname)
#endif /* __CLK_REGMAP_H */
diff --git a/drivers/clk/meson/g12a-aoclk.c b/drivers/clk/meson/g12a-aoclk.c
index 1994e735396b..62499563e4f5 100644
--- a/drivers/clk/meson/g12a-aoclk.c
+++ b/drivers/clk/meson/g12a-aoclk.c
@@ -18,8 +18,6 @@
#include "clk-regmap.h"
#include "clk-dualdiv.h"
-#define IN_PREFIX "ao-in-"
-
/*
* AO Configuration Clock registers offsets
* Register offsets from the data sheet must be multiplied by 4.
@@ -51,7 +49,9 @@ static struct clk_regmap g12a_aoclk_##_name = { \
.hw.init = &(struct clk_init_data) { \
.name = "g12a_ao_" #_name, \
.ops = &clk_regmap_gate_ops, \
- .parent_names = (const char *[]){ IN_PREFIX "mpeg-clk" }, \
+ .parent_data = &(const struct clk_parent_data) { \
+ .fw_name = "mpeg-clk", \
+ }, \
.num_parents = 1, \
.flags = CLK_IGNORE_UNUSED, \
}, \
@@ -81,7 +81,9 @@ static struct clk_regmap g12a_aoclk_cts_oscin = {
.hw.init = &(struct clk_init_data){
.name = "cts_oscin",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
},
};
@@ -106,7 +108,9 @@ static struct clk_regmap g12a_aoclk_32k_by_oscin_pre = {
.hw.init = &(struct clk_init_data){
.name = "g12a_ao_32k_by_oscin_pre",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "cts_oscin" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_aoclk_cts_oscin.hw
+ },
.num_parents = 1,
},
};
@@ -143,7 +147,9 @@ static struct clk_regmap g12a_aoclk_32k_by_oscin_div = {
.hw.init = &(struct clk_init_data){
.name = "g12a_ao_32k_by_oscin_div",
.ops = &meson_clk_dualdiv_ops,
- .parent_names = (const char *[]){ "g12a_ao_32k_by_oscin_pre" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_aoclk_32k_by_oscin_pre.hw
+ },
.num_parents = 1,
},
};
@@ -158,8 +164,10 @@ static struct clk_regmap g12a_aoclk_32k_by_oscin_sel = {
.hw.init = &(struct clk_init_data){
.name = "g12a_ao_32k_by_oscin_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "g12a_ao_32k_by_oscin_div",
- "g12a_ao_32k_by_oscin_pre" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_aoclk_32k_by_oscin_div.hw,
+ &g12a_aoclk_32k_by_oscin_pre.hw,
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_PARENT,
},
@@ -173,7 +181,9 @@ static struct clk_regmap g12a_aoclk_32k_by_oscin = {
.hw.init = &(struct clk_init_data){
.name = "g12a_ao_32k_by_oscin",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "g12a_ao_32k_by_oscin_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_aoclk_32k_by_oscin_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -189,7 +199,9 @@ static struct clk_regmap g12a_aoclk_cec_pre = {
.hw.init = &(struct clk_init_data){
.name = "g12a_ao_cec_pre",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "cts_oscin" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_aoclk_cts_oscin.hw
+ },
.num_parents = 1,
},
};
@@ -226,7 +238,9 @@ static struct clk_regmap g12a_aoclk_cec_div = {
.hw.init = &(struct clk_init_data){
.name = "g12a_ao_cec_div",
.ops = &meson_clk_dualdiv_ops,
- .parent_names = (const char *[]){ "g12a_ao_cec_pre" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_aoclk_cec_pre.hw
+ },
.num_parents = 1,
},
};
@@ -241,8 +255,10 @@ static struct clk_regmap g12a_aoclk_cec_sel = {
.hw.init = &(struct clk_init_data){
.name = "g12a_ao_cec_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "g12a_ao_cec_div",
- "g12a_ao_cec_pre" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_aoclk_cec_div.hw,
+ &g12a_aoclk_cec_pre.hw,
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_PARENT,
},
@@ -256,7 +272,9 @@ static struct clk_regmap g12a_aoclk_cec = {
.hw.init = &(struct clk_init_data){
.name = "g12a_ao_cec",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "g12a_ao_cec_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_aoclk_cec_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -272,8 +290,10 @@ static struct clk_regmap g12a_aoclk_cts_rtc_oscin = {
.hw.init = &(struct clk_init_data){
.name = "g12a_ao_cts_rtc_oscin",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "g12a_ao_32k_by_oscin",
- IN_PREFIX "ext_32k-0" },
+ .parent_data = (const struct clk_parent_data []) {
+ { .hw = &g12a_aoclk_32k_by_oscin.hw },
+ { .fw_name = "ext-32k-0", },
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_PARENT,
},
@@ -289,8 +309,10 @@ static struct clk_regmap g12a_aoclk_clk81 = {
.hw.init = &(struct clk_init_data){
.name = "g12a_ao_clk81",
.ops = &clk_regmap_mux_ro_ops,
- .parent_names = (const char *[]){ IN_PREFIX "mpeg-clk",
- "g12a_ao_cts_rtc_oscin"},
+ .parent_data = (const struct clk_parent_data []) {
+ { .fw_name = "mpeg-clk", },
+ { .hw = &g12a_aoclk_cts_rtc_oscin.hw },
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_PARENT,
},
@@ -305,8 +327,10 @@ static struct clk_regmap g12a_aoclk_saradc_mux = {
.hw.init = &(struct clk_init_data){
.name = "g12a_ao_saradc_mux",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal",
- "g12a_ao_clk81" },
+ .parent_data = (const struct clk_parent_data []) {
+ { .fw_name = "xtal", },
+ { .hw = &g12a_aoclk_clk81.hw },
+ },
.num_parents = 2,
},
};
@@ -320,7 +344,9 @@ static struct clk_regmap g12a_aoclk_saradc_div = {
.hw.init = &(struct clk_init_data){
.name = "g12a_ao_saradc_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "g12a_ao_saradc_mux" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_aoclk_saradc_mux.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -334,7 +360,9 @@ static struct clk_regmap g12a_aoclk_saradc_gate = {
.hw.init = &(struct clk_init_data){
.name = "g12a_ao_saradc_gate",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "g12a_ao_saradc_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_aoclk_saradc_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -417,12 +445,6 @@ static const struct clk_hw_onecell_data g12a_aoclk_onecell_data = {
.num = NR_CLKS,
};
-static const struct meson_aoclk_input g12a_aoclk_inputs[] = {
- { .name = "xtal", .required = true },
- { .name = "mpeg-clk", .required = true },
- { .name = "ext-32k-0", .required = false },
-};
-
static const struct meson_aoclk_data g12a_aoclkc_data = {
.reset_reg = AO_RTI_GEN_CNTL_REG0,
.num_reset = ARRAY_SIZE(g12a_aoclk_reset),
@@ -430,9 +452,6 @@ static const struct meson_aoclk_data g12a_aoclkc_data = {
.num_clks = ARRAY_SIZE(g12a_aoclk_regmap),
.clks = g12a_aoclk_regmap,
.hw_data = &g12a_aoclk_onecell_data,
- .inputs = g12a_aoclk_inputs,
- .num_inputs = ARRAY_SIZE(g12a_aoclk_inputs),
- .input_prefix = IN_PREFIX,
};
static const struct of_device_id g12a_aoclkc_match_table[] = {
diff --git a/drivers/clk/meson/g12a.c b/drivers/clk/meson/g12a.c
index 206fafd299ea..c3f0ffc3280d 100644
--- a/drivers/clk/meson/g12a.c
+++ b/drivers/clk/meson/g12a.c
@@ -14,11 +14,12 @@
#include <linux/init.h>
#include <linux/of_device.h>
#include <linux/platform_device.h>
+#include <linux/clk.h>
-#include "clk-input.h"
#include "clk-mpll.h"
#include "clk-pll.h"
#include "clk-regmap.h"
+#include "clk-cpu-dyndiv.h"
#include "vid-pll-div.h"
#include "meson-eeclk.h"
#include "g12a.h"
@@ -61,7 +62,9 @@ static struct clk_regmap g12a_fixed_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "fixed_pll_dco",
.ops = &meson_clk_pll_ro_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
},
};
@@ -76,7 +79,9 @@ static struct clk_regmap g12a_fixed_pll = {
.hw.init = &(struct clk_init_data){
.name = "fixed_pll",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "fixed_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_fixed_pll_dco.hw
+ },
.num_parents = 1,
/*
* This clock won't ever change at runtime so
@@ -85,16 +90,9 @@ static struct clk_regmap g12a_fixed_pll = {
},
};
-/*
- * Internal sys pll emulation configuration parameters
- */
-static const struct reg_sequence g12a_sys_init_regs[] = {
- { .reg = HHI_SYS_PLL_CNTL1, .def = 0x00000000 },
- { .reg = HHI_SYS_PLL_CNTL2, .def = 0x00000000 },
- { .reg = HHI_SYS_PLL_CNTL3, .def = 0x48681c00 },
- { .reg = HHI_SYS_PLL_CNTL4, .def = 0x88770290 },
- { .reg = HHI_SYS_PLL_CNTL5, .def = 0x39272000 },
- { .reg = HHI_SYS_PLL_CNTL6, .def = 0x56540000 },
+static const struct pll_mult_range g12a_sys_pll_mult_range = {
+ .min = 128,
+ .max = 250,
};
static struct clk_regmap g12a_sys_pll_dco = {
@@ -124,14 +122,17 @@ static struct clk_regmap g12a_sys_pll_dco = {
.shift = 29,
.width = 1,
},
- .init_regs = g12a_sys_init_regs,
- .init_count = ARRAY_SIZE(g12a_sys_init_regs),
+ .range = &g12a_sys_pll_mult_range,
},
.hw.init = &(struct clk_init_data){
.name = "sys_pll_dco",
- .ops = &meson_clk_pll_ro_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .ops = &meson_clk_pll_ops,
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
+ /* This clock feeds the CPU, avoid disabling it */
+ .flags = CLK_IS_CRITICAL,
},
};
@@ -144,9 +145,71 @@ static struct clk_regmap g12a_sys_pll = {
},
.hw.init = &(struct clk_init_data){
.name = "sys_pll",
- .ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "sys_pll_dco" },
+ .ops = &clk_regmap_divider_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_sys_pll_dco.hw
+ },
+ .num_parents = 1,
+ .flags = CLK_SET_RATE_PARENT,
+ },
+};
+
+static struct clk_regmap g12b_sys1_pll_dco = {
+ .data = &(struct meson_clk_pll_data){
+ .en = {
+ .reg_off = HHI_SYS1_PLL_CNTL0,
+ .shift = 28,
+ .width = 1,
+ },
+ .m = {
+ .reg_off = HHI_SYS1_PLL_CNTL0,
+ .shift = 0,
+ .width = 8,
+ },
+ .n = {
+ .reg_off = HHI_SYS1_PLL_CNTL0,
+ .shift = 10,
+ .width = 5,
+ },
+ .l = {
+ .reg_off = HHI_SYS1_PLL_CNTL0,
+ .shift = 31,
+ .width = 1,
+ },
+ .rst = {
+ .reg_off = HHI_SYS1_PLL_CNTL0,
+ .shift = 29,
+ .width = 1,
+ },
+ .range = &g12a_sys_pll_mult_range,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "sys1_pll_dco",
+ .ops = &meson_clk_pll_ops,
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
+ .num_parents = 1,
+ /* This clock feeds the CPU, avoid disabling it */
+ .flags = CLK_IS_CRITICAL,
+ },
+};
+
+static struct clk_regmap g12b_sys1_pll = {
+ .data = &(struct clk_regmap_div_data){
+ .offset = HHI_SYS1_PLL_CNTL0,
+ .shift = 16,
+ .width = 3,
+ .flags = CLK_DIVIDER_POWER_OF_TWO,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "sys1_pll",
+ .ops = &clk_regmap_divider_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_sys1_pll_dco.hw
+ },
.num_parents = 1,
+ .flags = CLK_SET_RATE_PARENT,
},
};
@@ -158,7 +221,26 @@ static struct clk_regmap g12a_sys_pll_div16_en = {
.hw.init = &(struct clk_init_data) {
.name = "sys_pll_div16_en",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "sys_pll" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_sys_pll.hw },
+ .num_parents = 1,
+ /*
+ * This clock is used to debug the sys_pll range
+ * Linux should not change it at runtime
+ */
+ },
+};
+
+static struct clk_regmap g12b_sys1_pll_div16_en = {
+ .data = &(struct clk_regmap_gate_data){
+ .offset = HHI_SYS_CPUB_CLK_CNTL1,
+ .bit_idx = 24,
+ },
+ .hw.init = &(struct clk_init_data) {
+ .name = "sys1_pll_div16_en",
+ .ops = &clk_regmap_gate_ro_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_sys1_pll.hw
+ },
.num_parents = 1,
/*
* This clock is used to debug the sys_pll range
@@ -173,11 +255,88 @@ static struct clk_fixed_factor g12a_sys_pll_div16 = {
.hw.init = &(struct clk_init_data){
.name = "sys_pll_div16",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "sys_pll_div16_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_sys_pll_div16_en.hw
+ },
+ .num_parents = 1,
+ },
+};
+
+static struct clk_fixed_factor g12b_sys1_pll_div16 = {
+ .mult = 1,
+ .div = 16,
+ .hw.init = &(struct clk_init_data){
+ .name = "sys1_pll_div16",
+ .ops = &clk_fixed_factor_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_sys1_pll_div16_en.hw
+ },
+ .num_parents = 1,
+ },
+};
+
+static struct clk_fixed_factor g12a_fclk_div2_div = {
+ .mult = 1,
+ .div = 2,
+ .hw.init = &(struct clk_init_data){
+ .name = "fclk_div2_div",
+ .ops = &clk_fixed_factor_ops,
+ .parent_hws = (const struct clk_hw *[]) { &g12a_fixed_pll.hw },
.num_parents = 1,
},
};
+static struct clk_regmap g12a_fclk_div2 = {
+ .data = &(struct clk_regmap_gate_data){
+ .offset = HHI_FIX_PLL_CNTL1,
+ .bit_idx = 24,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "fclk_div2",
+ .ops = &clk_regmap_gate_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_fclk_div2_div.hw
+ },
+ .num_parents = 1,
+ },
+};
+
+static struct clk_fixed_factor g12a_fclk_div3_div = {
+ .mult = 1,
+ .div = 3,
+ .hw.init = &(struct clk_init_data){
+ .name = "fclk_div3_div",
+ .ops = &clk_fixed_factor_ops,
+ .parent_hws = (const struct clk_hw *[]) { &g12a_fixed_pll.hw },
+ .num_parents = 1,
+ },
+};
+
+static struct clk_regmap g12a_fclk_div3 = {
+ .data = &(struct clk_regmap_gate_data){
+ .offset = HHI_FIX_PLL_CNTL1,
+ .bit_idx = 20,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "fclk_div3",
+ .ops = &clk_regmap_gate_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_fclk_div3_div.hw
+ },
+ .num_parents = 1,
+ /*
+ * This clock is used by the resident firmware and is required
+ * by the platform to operate correctly.
+ * Until the following condition are met, we need this clock to
+ * be marked as critical:
+ * a) Mark the clock used by a firmware resource, if possible
+ * b) CCF has a clock hand-off mechanism to make the sure the
+ * clock stays on until the proper driver comes along
+ */
+ .flags = CLK_IS_CRITICAL,
+ },
+};
+
/* Datasheet names this field as "premux0" */
static struct clk_regmap g12a_cpu_clk_premux0 = {
.data = &(struct clk_regmap_mux_data){
@@ -187,26 +346,61 @@ static struct clk_regmap g12a_cpu_clk_premux0 = {
},
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_dyn0_sel",
- .ops = &clk_regmap_mux_ro_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal",
- "fclk_div2",
- "fclk_div3" },
+ .ops = &clk_regmap_mux_ops,
+ .parent_data = (const struct clk_parent_data []) {
+ { .fw_name = "xtal", },
+ { .hw = &g12a_fclk_div2.hw },
+ { .hw = &g12a_fclk_div3.hw },
+ },
+ .num_parents = 3,
+ /* This sub-tree is used a parking clock */
+ .flags = CLK_SET_RATE_NO_REPARENT,
+ },
+};
+
+/* Datasheet names this field as "premux1" */
+static struct clk_regmap g12a_cpu_clk_premux1 = {
+ .data = &(struct clk_regmap_mux_data){
+ .offset = HHI_SYS_CPU_CLK_CNTL0,
+ .mask = 0x3,
+ .shift = 16,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cpu_clk_dyn1_sel",
+ .ops = &clk_regmap_mux_ops,
+ .parent_data = (const struct clk_parent_data []) {
+ { .fw_name = "xtal", },
+ { .hw = &g12a_fclk_div2.hw },
+ { .hw = &g12a_fclk_div3.hw },
+ },
.num_parents = 3,
+ /* This sub-tree is used a parking clock */
+ .flags = CLK_SET_RATE_NO_REPARENT
},
};
/* Datasheet names this field as "mux0_divn_tcnt" */
static struct clk_regmap g12a_cpu_clk_mux0_div = {
- .data = &(struct clk_regmap_div_data){
- .offset = HHI_SYS_CPU_CLK_CNTL0,
- .shift = 4,
- .width = 6,
+ .data = &(struct meson_clk_cpu_dyndiv_data){
+ .div = {
+ .reg_off = HHI_SYS_CPU_CLK_CNTL0,
+ .shift = 4,
+ .width = 6,
+ },
+ .dyn = {
+ .reg_off = HHI_SYS_CPU_CLK_CNTL0,
+ .shift = 26,
+ .width = 1,
+ },
},
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_dyn0_div",
- .ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "cpu_clk_dyn0_sel" },
+ .ops = &meson_clk_cpu_dyndiv_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_cpu_clk_premux0.hw
+ },
.num_parents = 1,
+ .flags = CLK_SET_RATE_PARENT,
},
};
@@ -219,27 +413,13 @@ static struct clk_regmap g12a_cpu_clk_postmux0 = {
},
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_dyn0",
- .ops = &clk_regmap_mux_ro_ops,
- .parent_names = (const char *[]){ "cpu_clk_dyn0_sel",
- "cpu_clk_dyn0_div" },
+ .ops = &clk_regmap_mux_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_cpu_clk_premux0.hw,
+ &g12a_cpu_clk_mux0_div.hw,
+ },
.num_parents = 2,
- },
-};
-
-/* Datasheet names this field as "premux1" */
-static struct clk_regmap g12a_cpu_clk_premux1 = {
- .data = &(struct clk_regmap_mux_data){
- .offset = HHI_SYS_CPU_CLK_CNTL0,
- .mask = 0x3,
- .shift = 16,
- },
- .hw.init = &(struct clk_init_data){
- .name = "cpu_clk_dyn1_sel",
- .ops = &clk_regmap_mux_ro_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal",
- "fclk_div2",
- "fclk_div3" },
- .num_parents = 3,
+ .flags = CLK_SET_RATE_PARENT,
},
};
@@ -253,7 +433,9 @@ static struct clk_regmap g12a_cpu_clk_mux1_div = {
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_dyn1_div",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "cpu_clk_dyn1_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_cpu_clk_premux1.hw
+ },
.num_parents = 1,
},
};
@@ -267,10 +449,14 @@ static struct clk_regmap g12a_cpu_clk_postmux1 = {
},
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_dyn1",
- .ops = &clk_regmap_mux_ro_ops,
- .parent_names = (const char *[]){ "cpu_clk_dyn1_sel",
- "cpu_clk_dyn1_div" },
+ .ops = &clk_regmap_mux_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_cpu_clk_premux1.hw,
+ &g12a_cpu_clk_mux1_div.hw,
+ },
.num_parents = 2,
+ /* This sub-tree is used a parking clock */
+ .flags = CLK_SET_RATE_NO_REPARENT,
},
};
@@ -283,10 +469,13 @@ static struct clk_regmap g12a_cpu_clk_dyn = {
},
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_dyn",
- .ops = &clk_regmap_mux_ro_ops,
- .parent_names = (const char *[]){ "cpu_clk_dyn0",
- "cpu_clk_dyn1" },
+ .ops = &clk_regmap_mux_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_cpu_clk_postmux0.hw,
+ &g12a_cpu_clk_postmux1.hw,
+ },
.num_parents = 2,
+ .flags = CLK_SET_RATE_PARENT,
},
};
@@ -299,13 +488,405 @@ static struct clk_regmap g12a_cpu_clk = {
},
.hw.init = &(struct clk_init_data){
.name = "cpu_clk",
- .ops = &clk_regmap_mux_ro_ops,
- .parent_names = (const char *[]){ "cpu_clk_dyn",
- "sys_pll" },
+ .ops = &clk_regmap_mux_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_cpu_clk_dyn.hw,
+ &g12a_sys_pll.hw,
+ },
+ .num_parents = 2,
+ .flags = CLK_SET_RATE_PARENT,
+ },
+};
+
+/* Datasheet names this field as "Final_mux_sel" */
+static struct clk_regmap g12b_cpu_clk = {
+ .data = &(struct clk_regmap_mux_data){
+ .offset = HHI_SYS_CPU_CLK_CNTL0,
+ .mask = 0x1,
+ .shift = 11,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cpu_clk",
+ .ops = &clk_regmap_mux_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_cpu_clk_dyn.hw,
+ &g12b_sys1_pll.hw
+ },
.num_parents = 2,
+ .flags = CLK_SET_RATE_PARENT,
+ },
+};
+
+/* Datasheet names this field as "premux0" */
+static struct clk_regmap g12b_cpub_clk_premux0 = {
+ .data = &(struct clk_regmap_mux_data){
+ .offset = HHI_SYS_CPUB_CLK_CNTL,
+ .mask = 0x3,
+ .shift = 0,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_dyn0_sel",
+ .ops = &clk_regmap_mux_ops,
+ .parent_data = (const struct clk_parent_data []) {
+ { .fw_name = "xtal", },
+ { .hw = &g12a_fclk_div2.hw },
+ { .hw = &g12a_fclk_div3.hw },
+ },
+ .num_parents = 3,
+ },
+};
+
+/* Datasheet names this field as "mux0_divn_tcnt" */
+static struct clk_regmap g12b_cpub_clk_mux0_div = {
+ .data = &(struct meson_clk_cpu_dyndiv_data){
+ .div = {
+ .reg_off = HHI_SYS_CPUB_CLK_CNTL,
+ .shift = 4,
+ .width = 6,
+ },
+ .dyn = {
+ .reg_off = HHI_SYS_CPUB_CLK_CNTL,
+ .shift = 26,
+ .width = 1,
+ },
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_dyn0_div",
+ .ops = &meson_clk_cpu_dyndiv_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk_premux0.hw
+ },
+ .num_parents = 1,
+ .flags = CLK_SET_RATE_PARENT,
},
};
+/* Datasheet names this field as "postmux0" */
+static struct clk_regmap g12b_cpub_clk_postmux0 = {
+ .data = &(struct clk_regmap_mux_data){
+ .offset = HHI_SYS_CPUB_CLK_CNTL,
+ .mask = 0x1,
+ .shift = 2,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_dyn0",
+ .ops = &clk_regmap_mux_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk_premux0.hw,
+ &g12b_cpub_clk_mux0_div.hw
+ },
+ .num_parents = 2,
+ .flags = CLK_SET_RATE_PARENT,
+ },
+};
+
+/* Datasheet names this field as "premux1" */
+static struct clk_regmap g12b_cpub_clk_premux1 = {
+ .data = &(struct clk_regmap_mux_data){
+ .offset = HHI_SYS_CPUB_CLK_CNTL,
+ .mask = 0x3,
+ .shift = 16,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_dyn1_sel",
+ .ops = &clk_regmap_mux_ops,
+ .parent_data = (const struct clk_parent_data []) {
+ { .fw_name = "xtal", },
+ { .hw = &g12a_fclk_div2.hw },
+ { .hw = &g12a_fclk_div3.hw },
+ },
+ .num_parents = 3,
+ /* This sub-tree is used a parking clock */
+ .flags = CLK_SET_RATE_NO_REPARENT,
+ },
+};
+
+/* Datasheet names this field as "Mux1_divn_tcnt" */
+static struct clk_regmap g12b_cpub_clk_mux1_div = {
+ .data = &(struct clk_regmap_div_data){
+ .offset = HHI_SYS_CPUB_CLK_CNTL,
+ .shift = 20,
+ .width = 6,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_dyn1_div",
+ .ops = &clk_regmap_divider_ro_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk_premux1.hw
+ },
+ .num_parents = 1,
+ },
+};
+
+/* Datasheet names this field as "postmux1" */
+static struct clk_regmap g12b_cpub_clk_postmux1 = {
+ .data = &(struct clk_regmap_mux_data){
+ .offset = HHI_SYS_CPUB_CLK_CNTL,
+ .mask = 0x1,
+ .shift = 18,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_dyn1",
+ .ops = &clk_regmap_mux_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk_premux1.hw,
+ &g12b_cpub_clk_mux1_div.hw
+ },
+ .num_parents = 2,
+ /* This sub-tree is used a parking clock */
+ .flags = CLK_SET_RATE_NO_REPARENT,
+ },
+};
+
+/* Datasheet names this field as "Final_dyn_mux_sel" */
+static struct clk_regmap g12b_cpub_clk_dyn = {
+ .data = &(struct clk_regmap_mux_data){
+ .offset = HHI_SYS_CPUB_CLK_CNTL,
+ .mask = 0x1,
+ .shift = 10,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_dyn",
+ .ops = &clk_regmap_mux_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk_postmux0.hw,
+ &g12b_cpub_clk_postmux1.hw
+ },
+ .num_parents = 2,
+ .flags = CLK_SET_RATE_PARENT,
+ },
+};
+
+/* Datasheet names this field as "Final_mux_sel" */
+static struct clk_regmap g12b_cpub_clk = {
+ .data = &(struct clk_regmap_mux_data){
+ .offset = HHI_SYS_CPUB_CLK_CNTL,
+ .mask = 0x1,
+ .shift = 11,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk",
+ .ops = &clk_regmap_mux_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk_dyn.hw,
+ &g12a_sys_pll.hw
+ },
+ .num_parents = 2,
+ .flags = CLK_SET_RATE_PARENT,
+ },
+};
+
+static int g12a_cpu_clk_mux_notifier_cb(struct notifier_block *nb,
+ unsigned long event, void *data)
+{
+ if (event == POST_RATE_CHANGE || event == PRE_RATE_CHANGE) {
+ /* Wait for clock propagation before/after changing the mux */
+ udelay(100);
+ return NOTIFY_OK;
+ }
+
+ return NOTIFY_DONE;
+}
+
+static struct notifier_block g12a_cpu_clk_mux_nb = {
+ .notifier_call = g12a_cpu_clk_mux_notifier_cb,
+};
+
+struct g12a_cpu_clk_postmux_nb_data {
+ struct notifier_block nb;
+ struct clk_hw *xtal;
+ struct clk_hw *cpu_clk_dyn;
+ struct clk_hw *cpu_clk_postmux0;
+ struct clk_hw *cpu_clk_postmux1;
+ struct clk_hw *cpu_clk_premux1;
+};
+
+static int g12a_cpu_clk_postmux_notifier_cb(struct notifier_block *nb,
+ unsigned long event, void *data)
+{
+ struct g12a_cpu_clk_postmux_nb_data *nb_data =
+ container_of(nb, struct g12a_cpu_clk_postmux_nb_data, nb);
+
+ switch (event) {
+ case PRE_RATE_CHANGE:
+ /*
+ * This notifier means cpu_clk_postmux0 clock will be changed
+ * to feed cpu_clk, this is the current path :
+ * cpu_clk
+ * \- cpu_clk_dyn
+ * \- cpu_clk_postmux0
+ * \- cpu_clk_muxX_div
+ * \- cpu_clk_premux0
+ * \- fclk_div3 or fclk_div2
+ * OR
+ * \- cpu_clk_premux0
+ * \- fclk_div3 or fclk_div2
+ */
+
+ /* Setup cpu_clk_premux1 to xtal */
+ clk_hw_set_parent(nb_data->cpu_clk_premux1,
+ nb_data->xtal);
+
+ /* Setup cpu_clk_postmux1 to bypass divider */
+ clk_hw_set_parent(nb_data->cpu_clk_postmux1,
+ nb_data->cpu_clk_premux1);
+
+ /* Switch to parking clk on cpu_clk_postmux1 */
+ clk_hw_set_parent(nb_data->cpu_clk_dyn,
+ nb_data->cpu_clk_postmux1);
+
+ /*
+ * Now, cpu_clk is 24MHz in the current path :
+ * cpu_clk
+ * \- cpu_clk_dyn
+ * \- cpu_clk_postmux1
+ * \- cpu_clk_premux1
+ * \- xtal
+ */
+
+ udelay(100);
+
+ return NOTIFY_OK;
+
+ case POST_RATE_CHANGE:
+ /*
+ * The cpu_clk_postmux0 has ben updated, now switch back
+ * cpu_clk_dyn to cpu_clk_postmux0 and take the changes
+ * in account.
+ */
+
+ /* Configure cpu_clk_dyn back to cpu_clk_postmux0 */
+ clk_hw_set_parent(nb_data->cpu_clk_dyn,
+ nb_data->cpu_clk_postmux0);
+
+ /*
+ * new path :
+ * cpu_clk
+ * \- cpu_clk_dyn
+ * \- cpu_clk_postmux0
+ * \- cpu_clk_muxX_div
+ * \- cpu_clk_premux0
+ * \- fclk_div3 or fclk_div2
+ * OR
+ * \- cpu_clk_premux0
+ * \- fclk_div3 or fclk_div2
+ */
+
+ udelay(100);
+
+ return NOTIFY_OK;
+
+ default:
+ return NOTIFY_DONE;
+ }
+}
+
+static struct g12a_cpu_clk_postmux_nb_data g12a_cpu_clk_postmux0_nb_data = {
+ .cpu_clk_dyn = &g12a_cpu_clk_dyn.hw,
+ .cpu_clk_postmux0 = &g12a_cpu_clk_postmux0.hw,
+ .cpu_clk_postmux1 = &g12a_cpu_clk_postmux1.hw,
+ .cpu_clk_premux1 = &g12a_cpu_clk_premux1.hw,
+ .nb.notifier_call = g12a_cpu_clk_postmux_notifier_cb,
+};
+
+static struct g12a_cpu_clk_postmux_nb_data g12b_cpub_clk_postmux0_nb_data = {
+ .cpu_clk_dyn = &g12b_cpub_clk_dyn.hw,
+ .cpu_clk_postmux0 = &g12b_cpub_clk_postmux0.hw,
+ .cpu_clk_postmux1 = &g12b_cpub_clk_postmux1.hw,
+ .cpu_clk_premux1 = &g12b_cpub_clk_premux1.hw,
+ .nb.notifier_call = g12a_cpu_clk_postmux_notifier_cb,
+};
+
+struct g12a_sys_pll_nb_data {
+ struct notifier_block nb;
+ struct clk_hw *sys_pll;
+ struct clk_hw *cpu_clk;
+ struct clk_hw *cpu_clk_dyn;
+};
+
+static int g12a_sys_pll_notifier_cb(struct notifier_block *nb,
+ unsigned long event, void *data)
+{
+ struct g12a_sys_pll_nb_data *nb_data =
+ container_of(nb, struct g12a_sys_pll_nb_data, nb);
+
+ switch (event) {
+ case PRE_RATE_CHANGE:
+ /*
+ * This notifier means sys_pll clock will be changed
+ * to feed cpu_clk, this the current path :
+ * cpu_clk
+ * \- sys_pll
+ * \- sys_pll_dco
+ */
+
+ /* Configure cpu_clk to use cpu_clk_dyn */
+ clk_hw_set_parent(nb_data->cpu_clk,
+ nb_data->cpu_clk_dyn);
+
+ /*
+ * Now, cpu_clk uses the dyn path
+ * cpu_clk
+ * \- cpu_clk_dyn
+ * \- cpu_clk_dynX
+ * \- cpu_clk_dynX_sel
+ * \- cpu_clk_dynX_div
+ * \- xtal/fclk_div2/fclk_div3
+ * \- xtal/fclk_div2/fclk_div3
+ */
+
+ udelay(100);
+
+ return NOTIFY_OK;
+
+ case POST_RATE_CHANGE:
+ /*
+ * The sys_pll has ben updated, now switch back cpu_clk to
+ * sys_pll
+ */
+
+ /* Configure cpu_clk to use sys_pll */
+ clk_hw_set_parent(nb_data->cpu_clk,
+ nb_data->sys_pll);
+
+ udelay(100);
+
+ /* new path :
+ * cpu_clk
+ * \- sys_pll
+ * \- sys_pll_dco
+ */
+
+ return NOTIFY_OK;
+
+ default:
+ return NOTIFY_DONE;
+ }
+}
+
+static struct g12a_sys_pll_nb_data g12a_sys_pll_nb_data = {
+ .sys_pll = &g12a_sys_pll.hw,
+ .cpu_clk = &g12a_cpu_clk.hw,
+ .cpu_clk_dyn = &g12a_cpu_clk_dyn.hw,
+ .nb.notifier_call = g12a_sys_pll_notifier_cb,
+};
+
+/* G12B first CPU cluster uses sys1_pll */
+static struct g12a_sys_pll_nb_data g12b_cpu_clk_sys1_pll_nb_data = {
+ .sys_pll = &g12b_sys1_pll.hw,
+ .cpu_clk = &g12b_cpu_clk.hw,
+ .cpu_clk_dyn = &g12a_cpu_clk_dyn.hw,
+ .nb.notifier_call = g12a_sys_pll_notifier_cb,
+};
+
+/* G12B second CPU cluster uses sys_pll */
+static struct g12a_sys_pll_nb_data g12b_cpub_clk_sys_pll_nb_data = {
+ .sys_pll = &g12a_sys_pll.hw,
+ .cpu_clk = &g12b_cpub_clk.hw,
+ .cpu_clk_dyn = &g12b_cpub_clk_dyn.hw,
+ .nb.notifier_call = g12a_sys_pll_notifier_cb,
+};
+
static struct clk_regmap g12a_cpu_clk_div16_en = {
.data = &(struct clk_regmap_gate_data){
.offset = HHI_SYS_CPU_CLK_CNTL1,
@@ -314,7 +895,28 @@ static struct clk_regmap g12a_cpu_clk_div16_en = {
.hw.init = &(struct clk_init_data) {
.name = "cpu_clk_div16_en",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "cpu_clk" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_cpu_clk.hw
+ },
+ .num_parents = 1,
+ /*
+ * This clock is used to debug the cpu_clk range
+ * Linux should not change it at runtime
+ */
+ },
+};
+
+static struct clk_regmap g12b_cpub_clk_div16_en = {
+ .data = &(struct clk_regmap_gate_data){
+ .offset = HHI_SYS_CPUB_CLK_CNTL1,
+ .bit_idx = 1,
+ },
+ .hw.init = &(struct clk_init_data) {
+ .name = "cpub_clk_div16_en",
+ .ops = &clk_regmap_gate_ro_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk.hw
+ },
.num_parents = 1,
/*
* This clock is used to debug the cpu_clk range
@@ -329,7 +931,22 @@ static struct clk_fixed_factor g12a_cpu_clk_div16 = {
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_div16",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "cpu_clk_div16_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_cpu_clk_div16_en.hw
+ },
+ .num_parents = 1,
+ },
+};
+
+static struct clk_fixed_factor g12b_cpub_clk_div16 = {
+ .mult = 1,
+ .div = 16,
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_div16",
+ .ops = &clk_fixed_factor_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk_div16_en.hw
+ },
.num_parents = 1,
},
};
@@ -344,7 +961,7 @@ static struct clk_regmap g12a_cpu_clk_apb_div = {
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_apb_div",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "cpu_clk" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_cpu_clk.hw },
.num_parents = 1,
},
};
@@ -357,7 +974,9 @@ static struct clk_regmap g12a_cpu_clk_apb = {
.hw.init = &(struct clk_init_data) {
.name = "cpu_clk_apb",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "cpu_clk_apb_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_cpu_clk_apb_div.hw
+ },
.num_parents = 1,
/*
* This clock is set by the ROM monitor code,
@@ -376,7 +995,7 @@ static struct clk_regmap g12a_cpu_clk_atb_div = {
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_atb_div",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "cpu_clk" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_cpu_clk.hw },
.num_parents = 1,
},
};
@@ -389,7 +1008,9 @@ static struct clk_regmap g12a_cpu_clk_atb = {
.hw.init = &(struct clk_init_data) {
.name = "cpu_clk_atb",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "cpu_clk_atb_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_cpu_clk_atb_div.hw
+ },
.num_parents = 1,
/*
* This clock is set by the ROM monitor code,
@@ -408,7 +1029,7 @@ static struct clk_regmap g12a_cpu_clk_axi_div = {
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_axi_div",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "cpu_clk" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_cpu_clk.hw },
.num_parents = 1,
},
};
@@ -421,7 +1042,9 @@ static struct clk_regmap g12a_cpu_clk_axi = {
.hw.init = &(struct clk_init_data) {
.name = "cpu_clk_axi",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "cpu_clk_axi_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_cpu_clk_axi_div.hw
+ },
.num_parents = 1,
/*
* This clock is set by the ROM monitor code,
@@ -440,7 +1063,17 @@ static struct clk_regmap g12a_cpu_clk_trace_div = {
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_trace_div",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "cpu_clk" },
+ .parent_data = &(const struct clk_parent_data) {
+ /*
+ * Note:
+ * G12A and G12B have different cpu_clks (with
+ * different struct clk_hw). We fallback to the global
+ * naming string mechanism so cpu_clk_trace_div picks
+ * up the appropriate one.
+ */
+ .name = "cpu_clk",
+ .index = -1,
+ },
.num_parents = 1,
},
};
@@ -453,7 +1086,273 @@ static struct clk_regmap g12a_cpu_clk_trace = {
.hw.init = &(struct clk_init_data) {
.name = "cpu_clk_trace",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "cpu_clk_trace_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_cpu_clk_trace_div.hw
+ },
+ .num_parents = 1,
+ /*
+ * This clock is set by the ROM monitor code,
+ * Linux should not change it at runtime
+ */
+ },
+};
+
+static struct clk_fixed_factor g12b_cpub_clk_div2 = {
+ .mult = 1,
+ .div = 2,
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_div2",
+ .ops = &clk_fixed_factor_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk.hw
+ },
+ .num_parents = 1,
+ },
+};
+
+static struct clk_fixed_factor g12b_cpub_clk_div3 = {
+ .mult = 1,
+ .div = 3,
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_div3",
+ .ops = &clk_fixed_factor_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk.hw
+ },
+ .num_parents = 1,
+ },
+};
+
+static struct clk_fixed_factor g12b_cpub_clk_div4 = {
+ .mult = 1,
+ .div = 4,
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_div4",
+ .ops = &clk_fixed_factor_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk.hw
+ },
+ .num_parents = 1,
+ },
+};
+
+static struct clk_fixed_factor g12b_cpub_clk_div5 = {
+ .mult = 1,
+ .div = 5,
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_div5",
+ .ops = &clk_fixed_factor_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk.hw
+ },
+ .num_parents = 1,
+ },
+};
+
+static struct clk_fixed_factor g12b_cpub_clk_div6 = {
+ .mult = 1,
+ .div = 6,
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_div6",
+ .ops = &clk_fixed_factor_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk.hw
+ },
+ .num_parents = 1,
+ },
+};
+
+static struct clk_fixed_factor g12b_cpub_clk_div7 = {
+ .mult = 1,
+ .div = 7,
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_div7",
+ .ops = &clk_fixed_factor_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk.hw
+ },
+ .num_parents = 1,
+ },
+};
+
+static struct clk_fixed_factor g12b_cpub_clk_div8 = {
+ .mult = 1,
+ .div = 8,
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_div8",
+ .ops = &clk_fixed_factor_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk.hw
+ },
+ .num_parents = 1,
+ },
+};
+
+static u32 mux_table_cpub[] = { 1, 2, 3, 4, 5, 6, 7 };
+static struct clk_regmap g12b_cpub_clk_apb_sel = {
+ .data = &(struct clk_regmap_mux_data){
+ .offset = HHI_SYS_CPUB_CLK_CNTL1,
+ .mask = 7,
+ .shift = 3,
+ .table = mux_table_cpub,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_apb_sel",
+ .ops = &clk_regmap_mux_ro_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk_div2.hw,
+ &g12b_cpub_clk_div3.hw,
+ &g12b_cpub_clk_div4.hw,
+ &g12b_cpub_clk_div5.hw,
+ &g12b_cpub_clk_div6.hw,
+ &g12b_cpub_clk_div7.hw,
+ &g12b_cpub_clk_div8.hw
+ },
+ .num_parents = 7,
+ },
+};
+
+static struct clk_regmap g12b_cpub_clk_apb = {
+ .data = &(struct clk_regmap_gate_data){
+ .offset = HHI_SYS_CPUB_CLK_CNTL1,
+ .bit_idx = 16,
+ .flags = CLK_GATE_SET_TO_DISABLE,
+ },
+ .hw.init = &(struct clk_init_data) {
+ .name = "cpub_clk_apb",
+ .ops = &clk_regmap_gate_ro_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk_apb_sel.hw
+ },
+ .num_parents = 1,
+ /*
+ * This clock is set by the ROM monitor code,
+ * Linux should not change it at runtime
+ */
+ },
+};
+
+static struct clk_regmap g12b_cpub_clk_atb_sel = {
+ .data = &(struct clk_regmap_mux_data){
+ .offset = HHI_SYS_CPUB_CLK_CNTL1,
+ .mask = 7,
+ .shift = 6,
+ .table = mux_table_cpub,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_atb_sel",
+ .ops = &clk_regmap_mux_ro_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk_div2.hw,
+ &g12b_cpub_clk_div3.hw,
+ &g12b_cpub_clk_div4.hw,
+ &g12b_cpub_clk_div5.hw,
+ &g12b_cpub_clk_div6.hw,
+ &g12b_cpub_clk_div7.hw,
+ &g12b_cpub_clk_div8.hw
+ },
+ .num_parents = 7,
+ },
+};
+
+static struct clk_regmap g12b_cpub_clk_atb = {
+ .data = &(struct clk_regmap_gate_data){
+ .offset = HHI_SYS_CPUB_CLK_CNTL1,
+ .bit_idx = 17,
+ .flags = CLK_GATE_SET_TO_DISABLE,
+ },
+ .hw.init = &(struct clk_init_data) {
+ .name = "cpub_clk_atb",
+ .ops = &clk_regmap_gate_ro_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk_atb_sel.hw
+ },
+ .num_parents = 1,
+ /*
+ * This clock is set by the ROM monitor code,
+ * Linux should not change it at runtime
+ */
+ },
+};
+
+static struct clk_regmap g12b_cpub_clk_axi_sel = {
+ .data = &(struct clk_regmap_mux_data){
+ .offset = HHI_SYS_CPUB_CLK_CNTL1,
+ .mask = 7,
+ .shift = 9,
+ .table = mux_table_cpub,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_axi_sel",
+ .ops = &clk_regmap_mux_ro_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk_div2.hw,
+ &g12b_cpub_clk_div3.hw,
+ &g12b_cpub_clk_div4.hw,
+ &g12b_cpub_clk_div5.hw,
+ &g12b_cpub_clk_div6.hw,
+ &g12b_cpub_clk_div7.hw,
+ &g12b_cpub_clk_div8.hw
+ },
+ .num_parents = 7,
+ },
+};
+
+static struct clk_regmap g12b_cpub_clk_axi = {
+ .data = &(struct clk_regmap_gate_data){
+ .offset = HHI_SYS_CPUB_CLK_CNTL1,
+ .bit_idx = 18,
+ .flags = CLK_GATE_SET_TO_DISABLE,
+ },
+ .hw.init = &(struct clk_init_data) {
+ .name = "cpub_clk_axi",
+ .ops = &clk_regmap_gate_ro_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk_axi_sel.hw
+ },
+ .num_parents = 1,
+ /*
+ * This clock is set by the ROM monitor code,
+ * Linux should not change it at runtime
+ */
+ },
+};
+
+static struct clk_regmap g12b_cpub_clk_trace_sel = {
+ .data = &(struct clk_regmap_mux_data){
+ .offset = HHI_SYS_CPUB_CLK_CNTL1,
+ .mask = 7,
+ .shift = 20,
+ .table = mux_table_cpub,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cpub_clk_trace_sel",
+ .ops = &clk_regmap_mux_ro_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk_div2.hw,
+ &g12b_cpub_clk_div3.hw,
+ &g12b_cpub_clk_div4.hw,
+ &g12b_cpub_clk_div5.hw,
+ &g12b_cpub_clk_div6.hw,
+ &g12b_cpub_clk_div7.hw,
+ &g12b_cpub_clk_div8.hw
+ },
+ .num_parents = 7,
+ },
+};
+
+static struct clk_regmap g12b_cpub_clk_trace = {
+ .data = &(struct clk_regmap_gate_data){
+ .offset = HHI_SYS_CPUB_CLK_CNTL1,
+ .bit_idx = 23,
+ .flags = CLK_GATE_SET_TO_DISABLE,
+ },
+ .hw.init = &(struct clk_init_data) {
+ .name = "cpub_clk_trace",
+ .ops = &clk_regmap_gate_ro_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12b_cpub_clk_trace_sel.hw
+ },
.num_parents = 1,
/*
* This clock is set by the ROM monitor code,
@@ -518,7 +1417,9 @@ static struct clk_regmap g12a_gp0_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "gp0_pll_dco",
.ops = &meson_clk_pll_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
},
};
@@ -534,7 +1435,9 @@ static struct clk_regmap g12a_gp0_pll = {
.hw.init = &(struct clk_init_data){
.name = "gp0_pll",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "gp0_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_gp0_pll_dco.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -592,7 +1495,9 @@ static struct clk_regmap g12a_hifi_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "hifi_pll_dco",
.ops = &meson_clk_pll_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
},
};
@@ -608,7 +1513,9 @@ static struct clk_regmap g12a_hifi_pll = {
.hw.init = &(struct clk_init_data){
.name = "hifi_pll",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "hifi_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_hifi_pll_dco.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -679,7 +1586,9 @@ static struct clk_regmap g12a_pcie_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "pcie_pll_dco",
.ops = &meson_clk_pcie_pll_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
},
};
@@ -690,7 +1599,9 @@ static struct clk_fixed_factor g12a_pcie_pll_dco_div2 = {
.hw.init = &(struct clk_init_data){
.name = "pcie_pll_dco_div2",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "pcie_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_pcie_pll_dco.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -708,7 +1619,9 @@ static struct clk_regmap g12a_pcie_pll_od = {
.hw.init = &(struct clk_init_data){
.name = "pcie_pll_od",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "pcie_pll_dco_div2" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_pcie_pll_dco_div2.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -720,7 +1633,9 @@ static struct clk_fixed_factor g12a_pcie_pll = {
.hw.init = &(struct clk_init_data){
.name = "pcie_pll_pll",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "pcie_pll_od" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_pcie_pll_od.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -762,7 +1677,9 @@ static struct clk_regmap g12a_hdmi_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_pll_dco",
.ops = &meson_clk_pll_ro_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
/*
* Display directly handle hdmi pll registers ATM, we need
@@ -782,7 +1699,9 @@ static struct clk_regmap g12a_hdmi_pll_od = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_pll_od",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "hdmi_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_hdmi_pll_dco.hw
+ },
.num_parents = 1,
.flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT,
},
@@ -798,7 +1717,9 @@ static struct clk_regmap g12a_hdmi_pll_od2 = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_pll_od2",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "hdmi_pll_od" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_hdmi_pll_od.hw
+ },
.num_parents = 1,
.flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT,
},
@@ -814,67 +1735,21 @@ static struct clk_regmap g12a_hdmi_pll = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_pll",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "hdmi_pll_od2" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_hdmi_pll_od2.hw
+ },
.num_parents = 1,
.flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT,
},
};
-static struct clk_fixed_factor g12a_fclk_div2_div = {
- .mult = 1,
- .div = 2,
- .hw.init = &(struct clk_init_data){
- .name = "fclk_div2_div",
- .ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
- .num_parents = 1,
- },
-};
-
-static struct clk_regmap g12a_fclk_div2 = {
- .data = &(struct clk_regmap_gate_data){
- .offset = HHI_FIX_PLL_CNTL1,
- .bit_idx = 24,
- },
- .hw.init = &(struct clk_init_data){
- .name = "fclk_div2",
- .ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div2_div" },
- .num_parents = 1,
- },
-};
-
-static struct clk_fixed_factor g12a_fclk_div3_div = {
- .mult = 1,
- .div = 3,
- .hw.init = &(struct clk_init_data){
- .name = "fclk_div3_div",
- .ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
- .num_parents = 1,
- },
-};
-
-static struct clk_regmap g12a_fclk_div3 = {
- .data = &(struct clk_regmap_gate_data){
- .offset = HHI_FIX_PLL_CNTL1,
- .bit_idx = 20,
- },
- .hw.init = &(struct clk_init_data){
- .name = "fclk_div3",
- .ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div3_div" },
- .num_parents = 1,
- },
-};
-
static struct clk_fixed_factor g12a_fclk_div4_div = {
.mult = 1,
.div = 4,
.hw.init = &(struct clk_init_data){
.name = "fclk_div4_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_fixed_pll.hw },
.num_parents = 1,
},
};
@@ -887,7 +1762,9 @@ static struct clk_regmap g12a_fclk_div4 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div4",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div4_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_fclk_div4_div.hw
+ },
.num_parents = 1,
},
};
@@ -898,7 +1775,7 @@ static struct clk_fixed_factor g12a_fclk_div5_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div5_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_fixed_pll.hw },
.num_parents = 1,
},
};
@@ -911,7 +1788,9 @@ static struct clk_regmap g12a_fclk_div5 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div5",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div5_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_fclk_div5_div.hw
+ },
.num_parents = 1,
},
};
@@ -922,7 +1801,7 @@ static struct clk_fixed_factor g12a_fclk_div7_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div7_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_fixed_pll.hw },
.num_parents = 1,
},
};
@@ -935,7 +1814,9 @@ static struct clk_regmap g12a_fclk_div7 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div7",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div7_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_fclk_div7_div.hw
+ },
.num_parents = 1,
},
};
@@ -946,7 +1827,9 @@ static struct clk_fixed_factor g12a_fclk_div2p5_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div2p5_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_fixed_pll_dco.hw
+ },
.num_parents = 1,
},
};
@@ -959,7 +1842,9 @@ static struct clk_regmap g12a_fclk_div2p5 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div2p5",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div2p5_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_fclk_div2p5_div.hw
+ },
.num_parents = 1,
},
};
@@ -970,7 +1855,9 @@ static struct clk_fixed_factor g12a_mpll_50m_div = {
.hw.init = &(struct clk_init_data){
.name = "mpll_50m_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_fixed_pll_dco.hw
+ },
.num_parents = 1,
},
};
@@ -984,8 +1871,10 @@ static struct clk_regmap g12a_mpll_50m = {
.hw.init = &(struct clk_init_data){
.name = "mpll_50m",
.ops = &clk_regmap_mux_ro_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal",
- "mpll_50m_div" },
+ .parent_data = (const struct clk_parent_data []) {
+ { .fw_name = "xtal", },
+ { .hw = &g12a_mpll_50m_div.hw },
+ },
.num_parents = 2,
},
};
@@ -996,11 +1885,17 @@ static struct clk_fixed_factor g12a_mpll_prediv = {
.hw.init = &(struct clk_init_data){
.name = "mpll_prediv",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_fixed_pll_dco.hw
+ },
.num_parents = 1,
},
};
+static const struct reg_sequence g12a_mpll0_init_regs[] = {
+ { .reg = HHI_MPLL_CNTL2, .def = 0x40000033 },
+};
+
static struct clk_regmap g12a_mpll0_div = {
.data = &(struct meson_clk_mpll_data){
.sdm = {
@@ -1024,11 +1919,15 @@ static struct clk_regmap g12a_mpll0_div = {
.width = 1,
},
.lock = &meson_clk_lock,
+ .init_regs = g12a_mpll0_init_regs,
+ .init_count = ARRAY_SIZE(g12a_mpll0_init_regs),
},
.hw.init = &(struct clk_init_data){
.name = "mpll0_div",
.ops = &meson_clk_mpll_ops,
- .parent_names = (const char *[]){ "mpll_prediv" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_mpll_prediv.hw
+ },
.num_parents = 1,
},
};
@@ -1041,12 +1940,16 @@ static struct clk_regmap g12a_mpll0 = {
.hw.init = &(struct clk_init_data){
.name = "mpll0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpll0_div" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_mpll0_div.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
};
+static const struct reg_sequence g12a_mpll1_init_regs[] = {
+ { .reg = HHI_MPLL_CNTL4, .def = 0x40000033 },
+};
+
static struct clk_regmap g12a_mpll1_div = {
.data = &(struct meson_clk_mpll_data){
.sdm = {
@@ -1070,11 +1973,15 @@ static struct clk_regmap g12a_mpll1_div = {
.width = 1,
},
.lock = &meson_clk_lock,
+ .init_regs = g12a_mpll1_init_regs,
+ .init_count = ARRAY_SIZE(g12a_mpll1_init_regs),
},
.hw.init = &(struct clk_init_data){
.name = "mpll1_div",
.ops = &meson_clk_mpll_ops,
- .parent_names = (const char *[]){ "mpll_prediv" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_mpll_prediv.hw
+ },
.num_parents = 1,
},
};
@@ -1087,12 +1994,16 @@ static struct clk_regmap g12a_mpll1 = {
.hw.init = &(struct clk_init_data){
.name = "mpll1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpll1_div" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_mpll1_div.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
};
+static const struct reg_sequence g12a_mpll2_init_regs[] = {
+ { .reg = HHI_MPLL_CNTL6, .def = 0x40000033 },
+};
+
static struct clk_regmap g12a_mpll2_div = {
.data = &(struct meson_clk_mpll_data){
.sdm = {
@@ -1116,11 +2027,15 @@ static struct clk_regmap g12a_mpll2_div = {
.width = 1,
},
.lock = &meson_clk_lock,
+ .init_regs = g12a_mpll2_init_regs,
+ .init_count = ARRAY_SIZE(g12a_mpll2_init_regs),
},
.hw.init = &(struct clk_init_data){
.name = "mpll2_div",
.ops = &meson_clk_mpll_ops,
- .parent_names = (const char *[]){ "mpll_prediv" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_mpll_prediv.hw
+ },
.num_parents = 1,
},
};
@@ -1133,12 +2048,16 @@ static struct clk_regmap g12a_mpll2 = {
.hw.init = &(struct clk_init_data){
.name = "mpll2",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpll2_div" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_mpll2_div.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
};
+static const struct reg_sequence g12a_mpll3_init_regs[] = {
+ { .reg = HHI_MPLL_CNTL8, .def = 0x40000033 },
+};
+
static struct clk_regmap g12a_mpll3_div = {
.data = &(struct meson_clk_mpll_data){
.sdm = {
@@ -1162,11 +2081,15 @@ static struct clk_regmap g12a_mpll3_div = {
.width = 1,
},
.lock = &meson_clk_lock,
+ .init_regs = g12a_mpll3_init_regs,
+ .init_count = ARRAY_SIZE(g12a_mpll3_init_regs),
},
.hw.init = &(struct clk_init_data){
.name = "mpll3_div",
.ops = &meson_clk_mpll_ops,
- .parent_names = (const char *[]){ "mpll_prediv" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_mpll_prediv.hw
+ },
.num_parents = 1,
},
};
@@ -1179,16 +2102,21 @@ static struct clk_regmap g12a_mpll3 = {
.hw.init = &(struct clk_init_data){
.name = "mpll3",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpll3_div" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_mpll3_div.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
};
static u32 mux_table_clk81[] = { 0, 2, 3, 4, 5, 6, 7 };
-static const char * const clk81_parent_names[] = {
- IN_PREFIX "xtal", "fclk_div7", "mpll1", "mpll2", "fclk_div4",
- "fclk_div3", "fclk_div5"
+static const struct clk_parent_data clk81_parent_data[] = {
+ { .fw_name = "xtal", },
+ { .hw = &g12a_fclk_div7.hw },
+ { .hw = &g12a_mpll1.hw },
+ { .hw = &g12a_mpll2.hw },
+ { .hw = &g12a_fclk_div4.hw },
+ { .hw = &g12a_fclk_div3.hw },
+ { .hw = &g12a_fclk_div5.hw },
};
static struct clk_regmap g12a_mpeg_clk_sel = {
@@ -1201,8 +2129,8 @@ static struct clk_regmap g12a_mpeg_clk_sel = {
.hw.init = &(struct clk_init_data){
.name = "mpeg_clk_sel",
.ops = &clk_regmap_mux_ro_ops,
- .parent_names = clk81_parent_names,
- .num_parents = ARRAY_SIZE(clk81_parent_names),
+ .parent_data = clk81_parent_data,
+ .num_parents = ARRAY_SIZE(clk81_parent_data),
},
};
@@ -1215,7 +2143,9 @@ static struct clk_regmap g12a_mpeg_clk_div = {
.hw.init = &(struct clk_init_data){
.name = "mpeg_clk_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "mpeg_clk_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_mpeg_clk_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1229,15 +2159,20 @@ static struct clk_regmap g12a_clk81 = {
.hw.init = &(struct clk_init_data){
.name = "clk81",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpeg_clk_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_mpeg_clk_div.hw
+ },
.num_parents = 1,
.flags = (CLK_SET_RATE_PARENT | CLK_IS_CRITICAL),
},
};
-static const char * const g12a_sd_emmc_clk0_parent_names[] = {
- IN_PREFIX "xtal", "fclk_div2", "fclk_div3", "fclk_div5", "fclk_div7",
-
+static const struct clk_parent_data g12a_sd_emmc_clk0_parent_data[] = {
+ { .fw_name = "xtal", },
+ { .hw = &g12a_fclk_div2.hw },
+ { .hw = &g12a_fclk_div3.hw },
+ { .hw = &g12a_fclk_div5.hw },
+ { .hw = &g12a_fclk_div7.hw },
/*
* Following these parent clocks, we should also have had mpll2, mpll3
* and gp0_pll but these clocks are too precious to be used here. All
@@ -1256,8 +2191,8 @@ static struct clk_regmap g12a_sd_emmc_a_clk0_sel = {
.hw.init = &(struct clk_init_data) {
.name = "sd_emmc_a_clk0_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_sd_emmc_clk0_parent_names,
- .num_parents = ARRAY_SIZE(g12a_sd_emmc_clk0_parent_names),
+ .parent_data = g12a_sd_emmc_clk0_parent_data,
+ .num_parents = ARRAY_SIZE(g12a_sd_emmc_clk0_parent_data),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1271,7 +2206,9 @@ static struct clk_regmap g12a_sd_emmc_a_clk0_div = {
.hw.init = &(struct clk_init_data) {
.name = "sd_emmc_a_clk0_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "sd_emmc_a_clk0_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_sd_emmc_a_clk0_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1285,7 +2222,9 @@ static struct clk_regmap g12a_sd_emmc_a_clk0 = {
.hw.init = &(struct clk_init_data){
.name = "sd_emmc_a_clk0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "sd_emmc_a_clk0_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_sd_emmc_a_clk0_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1301,8 +2240,8 @@ static struct clk_regmap g12a_sd_emmc_b_clk0_sel = {
.hw.init = &(struct clk_init_data) {
.name = "sd_emmc_b_clk0_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_sd_emmc_clk0_parent_names,
- .num_parents = ARRAY_SIZE(g12a_sd_emmc_clk0_parent_names),
+ .parent_data = g12a_sd_emmc_clk0_parent_data,
+ .num_parents = ARRAY_SIZE(g12a_sd_emmc_clk0_parent_data),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1316,7 +2255,9 @@ static struct clk_regmap g12a_sd_emmc_b_clk0_div = {
.hw.init = &(struct clk_init_data) {
.name = "sd_emmc_b_clk0_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "sd_emmc_b_clk0_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_sd_emmc_b_clk0_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1330,7 +2271,9 @@ static struct clk_regmap g12a_sd_emmc_b_clk0 = {
.hw.init = &(struct clk_init_data){
.name = "sd_emmc_b_clk0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "sd_emmc_b_clk0_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_sd_emmc_b_clk0_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1346,8 +2289,8 @@ static struct clk_regmap g12a_sd_emmc_c_clk0_sel = {
.hw.init = &(struct clk_init_data) {
.name = "sd_emmc_c_clk0_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_sd_emmc_clk0_parent_names,
- .num_parents = ARRAY_SIZE(g12a_sd_emmc_clk0_parent_names),
+ .parent_data = g12a_sd_emmc_clk0_parent_data,
+ .num_parents = ARRAY_SIZE(g12a_sd_emmc_clk0_parent_data),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1361,7 +2304,9 @@ static struct clk_regmap g12a_sd_emmc_c_clk0_div = {
.hw.init = &(struct clk_init_data) {
.name = "sd_emmc_c_clk0_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "sd_emmc_c_clk0_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_sd_emmc_c_clk0_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1375,17 +2320,89 @@ static struct clk_regmap g12a_sd_emmc_c_clk0 = {
.hw.init = &(struct clk_init_data){
.name = "sd_emmc_c_clk0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "sd_emmc_c_clk0_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_sd_emmc_c_clk0_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
};
+/* Video Clocks */
+
+static struct clk_regmap g12a_vid_pll_div = {
+ .data = &(struct meson_vid_pll_div_data){
+ .val = {
+ .reg_off = HHI_VID_PLL_CLK_DIV,
+ .shift = 0,
+ .width = 15,
+ },
+ .sel = {
+ .reg_off = HHI_VID_PLL_CLK_DIV,
+ .shift = 16,
+ .width = 2,
+ },
+ },
+ .hw.init = &(struct clk_init_data) {
+ .name = "vid_pll_div",
+ .ops = &meson_vid_pll_div_ro_ops,
+ .parent_hws = (const struct clk_hw *[]) { &g12a_hdmi_pll.hw },
+ .num_parents = 1,
+ .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE,
+ },
+};
+
+static const struct clk_hw *g12a_vid_pll_parent_hws[] = {
+ &g12a_vid_pll_div.hw,
+ &g12a_hdmi_pll.hw,
+};
+
+static struct clk_regmap g12a_vid_pll_sel = {
+ .data = &(struct clk_regmap_mux_data){
+ .offset = HHI_VID_PLL_CLK_DIV,
+ .mask = 0x1,
+ .shift = 18,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "vid_pll_sel",
+ .ops = &clk_regmap_mux_ops,
+ /*
+ * bit 18 selects from 2 possible parents:
+ * vid_pll_div or hdmi_pll
+ */
+ .parent_hws = g12a_vid_pll_parent_hws,
+ .num_parents = ARRAY_SIZE(g12a_vid_pll_parent_hws),
+ .flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE,
+ },
+};
+
+static struct clk_regmap g12a_vid_pll = {
+ .data = &(struct clk_regmap_gate_data){
+ .offset = HHI_VID_PLL_CLK_DIV,
+ .bit_idx = 19,
+ },
+ .hw.init = &(struct clk_init_data) {
+ .name = "vid_pll",
+ .ops = &clk_regmap_gate_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vid_pll_sel.hw
+ },
+ .num_parents = 1,
+ .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
+ },
+};
+
/* VPU Clock */
-static const char * const g12a_vpu_parent_names[] = {
- "fclk_div3", "fclk_div4", "fclk_div5", "fclk_div7",
- "mpll1", "vid_pll", "hifi_pll", "gp0_pll",
+static const struct clk_hw *g12a_vpu_parent_hws[] = {
+ &g12a_fclk_div3.hw,
+ &g12a_fclk_div4.hw,
+ &g12a_fclk_div5.hw,
+ &g12a_fclk_div7.hw,
+ &g12a_mpll1.hw,
+ &g12a_vid_pll.hw,
+ &g12a_hifi_pll.hw,
+ &g12a_gp0_pll.hw,
};
static struct clk_regmap g12a_vpu_0_sel = {
@@ -1397,8 +2414,8 @@ static struct clk_regmap g12a_vpu_0_sel = {
.hw.init = &(struct clk_init_data){
.name = "vpu_0_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_vpu_parent_names,
- .num_parents = ARRAY_SIZE(g12a_vpu_parent_names),
+ .parent_hws = g12a_vpu_parent_hws,
+ .num_parents = ARRAY_SIZE(g12a_vpu_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT,
},
};
@@ -1412,7 +2429,7 @@ static struct clk_regmap g12a_vpu_0_div = {
.hw.init = &(struct clk_init_data){
.name = "vpu_0_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vpu_0_sel" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vpu_0_sel.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1426,7 +2443,7 @@ static struct clk_regmap g12a_vpu_0 = {
.hw.init = &(struct clk_init_data) {
.name = "vpu_0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vpu_0_div" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vpu_0_div.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1441,8 +2458,8 @@ static struct clk_regmap g12a_vpu_1_sel = {
.hw.init = &(struct clk_init_data){
.name = "vpu_1_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_vpu_parent_names,
- .num_parents = ARRAY_SIZE(g12a_vpu_parent_names),
+ .parent_hws = g12a_vpu_parent_hws,
+ .num_parents = ARRAY_SIZE(g12a_vpu_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT,
},
};
@@ -1456,7 +2473,7 @@ static struct clk_regmap g12a_vpu_1_div = {
.hw.init = &(struct clk_init_data){
.name = "vpu_1_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vpu_1_sel" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vpu_1_sel.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1470,7 +2487,7 @@ static struct clk_regmap g12a_vpu_1 = {
.hw.init = &(struct clk_init_data) {
.name = "vpu_1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vpu_1_div" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vpu_1_div.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1489,7 +2506,10 @@ static struct clk_regmap g12a_vpu = {
* bit 31 selects from 2 possible parents:
* vpu_0 or vpu_1
*/
- .parent_names = (const char *[]){ "vpu_0", "vpu_1" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vpu_0.hw,
+ &g12a_vpu_1.hw,
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_NO_REPARENT,
},
@@ -1497,9 +2517,14 @@ static struct clk_regmap g12a_vpu = {
/* VDEC clocks */
-static const char * const g12a_vdec_parent_names[] = {
- "fclk_div2p5", "fclk_div3", "fclk_div4", "fclk_div5", "fclk_div7",
- "hifi_pll", "gp0_pll",
+static const struct clk_hw *g12a_vdec_parent_hws[] = {
+ &g12a_fclk_div2p5.hw,
+ &g12a_fclk_div3.hw,
+ &g12a_fclk_div4.hw,
+ &g12a_fclk_div5.hw,
+ &g12a_fclk_div7.hw,
+ &g12a_hifi_pll.hw,
+ &g12a_gp0_pll.hw,
};
static struct clk_regmap g12a_vdec_1_sel = {
@@ -1512,8 +2537,8 @@ static struct clk_regmap g12a_vdec_1_sel = {
.hw.init = &(struct clk_init_data){
.name = "vdec_1_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_vdec_parent_names,
- .num_parents = ARRAY_SIZE(g12a_vdec_parent_names),
+ .parent_hws = g12a_vdec_parent_hws,
+ .num_parents = ARRAY_SIZE(g12a_vdec_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1528,7 +2553,9 @@ static struct clk_regmap g12a_vdec_1_div = {
.hw.init = &(struct clk_init_data){
.name = "vdec_1_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vdec_1_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vdec_1_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1542,7 +2569,9 @@ static struct clk_regmap g12a_vdec_1 = {
.hw.init = &(struct clk_init_data) {
.name = "vdec_1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vdec_1_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vdec_1_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1558,8 +2587,8 @@ static struct clk_regmap g12a_vdec_hevcf_sel = {
.hw.init = &(struct clk_init_data){
.name = "vdec_hevcf_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_vdec_parent_names,
- .num_parents = ARRAY_SIZE(g12a_vdec_parent_names),
+ .parent_hws = g12a_vdec_parent_hws,
+ .num_parents = ARRAY_SIZE(g12a_vdec_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1574,7 +2603,9 @@ static struct clk_regmap g12a_vdec_hevcf_div = {
.hw.init = &(struct clk_init_data){
.name = "vdec_hevcf_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vdec_hevcf_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vdec_hevcf_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1588,7 +2619,9 @@ static struct clk_regmap g12a_vdec_hevcf = {
.hw.init = &(struct clk_init_data) {
.name = "vdec_hevcf",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vdec_hevcf_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vdec_hevcf_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1604,8 +2637,8 @@ static struct clk_regmap g12a_vdec_hevc_sel = {
.hw.init = &(struct clk_init_data){
.name = "vdec_hevc_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_vdec_parent_names,
- .num_parents = ARRAY_SIZE(g12a_vdec_parent_names),
+ .parent_hws = g12a_vdec_parent_hws,
+ .num_parents = ARRAY_SIZE(g12a_vdec_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1620,7 +2653,9 @@ static struct clk_regmap g12a_vdec_hevc_div = {
.hw.init = &(struct clk_init_data){
.name = "vdec_hevc_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vdec_hevc_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vdec_hevc_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1634,7 +2669,9 @@ static struct clk_regmap g12a_vdec_hevc = {
.hw.init = &(struct clk_init_data) {
.name = "vdec_hevc",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vdec_hevc_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vdec_hevc_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1642,9 +2679,15 @@ static struct clk_regmap g12a_vdec_hevc = {
/* VAPB Clock */
-static const char * const g12a_vapb_parent_names[] = {
- "fclk_div4", "fclk_div3", "fclk_div5", "fclk_div7",
- "mpll1", "vid_pll", "mpll2", "fclk_div2p5",
+static const struct clk_hw *g12a_vapb_parent_hws[] = {
+ &g12a_fclk_div4.hw,
+ &g12a_fclk_div3.hw,
+ &g12a_fclk_div5.hw,
+ &g12a_fclk_div7.hw,
+ &g12a_mpll1.hw,
+ &g12a_vid_pll.hw,
+ &g12a_mpll2.hw,
+ &g12a_fclk_div2p5.hw,
};
static struct clk_regmap g12a_vapb_0_sel = {
@@ -1656,8 +2699,8 @@ static struct clk_regmap g12a_vapb_0_sel = {
.hw.init = &(struct clk_init_data){
.name = "vapb_0_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_vapb_parent_names,
- .num_parents = ARRAY_SIZE(g12a_vapb_parent_names),
+ .parent_hws = g12a_vapb_parent_hws,
+ .num_parents = ARRAY_SIZE(g12a_vapb_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT,
},
};
@@ -1671,7 +2714,9 @@ static struct clk_regmap g12a_vapb_0_div = {
.hw.init = &(struct clk_init_data){
.name = "vapb_0_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vapb_0_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vapb_0_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1685,7 +2730,9 @@ static struct clk_regmap g12a_vapb_0 = {
.hw.init = &(struct clk_init_data) {
.name = "vapb_0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vapb_0_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vapb_0_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1700,8 +2747,8 @@ static struct clk_regmap g12a_vapb_1_sel = {
.hw.init = &(struct clk_init_data){
.name = "vapb_1_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_vapb_parent_names,
- .num_parents = ARRAY_SIZE(g12a_vapb_parent_names),
+ .parent_hws = g12a_vapb_parent_hws,
+ .num_parents = ARRAY_SIZE(g12a_vapb_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT,
},
};
@@ -1715,7 +2762,9 @@ static struct clk_regmap g12a_vapb_1_div = {
.hw.init = &(struct clk_init_data){
.name = "vapb_1_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vapb_1_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vapb_1_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1729,7 +2778,9 @@ static struct clk_regmap g12a_vapb_1 = {
.hw.init = &(struct clk_init_data) {
.name = "vapb_1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vapb_1_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vapb_1_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1748,7 +2799,10 @@ static struct clk_regmap g12a_vapb_sel = {
* bit 31 selects from 2 possible parents:
* vapb_0 or vapb_1
*/
- .parent_names = (const char *[]){ "vapb_0", "vapb_1" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vapb_0.hw,
+ &g12a_vapb_1.hw,
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_NO_REPARENT,
},
@@ -1762,75 +2816,21 @@ static struct clk_regmap g12a_vapb = {
.hw.init = &(struct clk_init_data) {
.name = "vapb",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vapb_sel" },
- .num_parents = 1,
- .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
- },
-};
-
-/* Video Clocks */
-
-static struct clk_regmap g12a_vid_pll_div = {
- .data = &(struct meson_vid_pll_div_data){
- .val = {
- .reg_off = HHI_VID_PLL_CLK_DIV,
- .shift = 0,
- .width = 15,
- },
- .sel = {
- .reg_off = HHI_VID_PLL_CLK_DIV,
- .shift = 16,
- .width = 2,
- },
- },
- .hw.init = &(struct clk_init_data) {
- .name = "vid_pll_div",
- .ops = &meson_vid_pll_div_ro_ops,
- .parent_names = (const char *[]){ "hdmi_pll" },
- .num_parents = 1,
- .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE,
- },
-};
-
-static const char * const g12a_vid_pll_parent_names[] = { "vid_pll_div",
- "hdmi_pll" };
-
-static struct clk_regmap g12a_vid_pll_sel = {
- .data = &(struct clk_regmap_mux_data){
- .offset = HHI_VID_PLL_CLK_DIV,
- .mask = 0x1,
- .shift = 18,
- },
- .hw.init = &(struct clk_init_data){
- .name = "vid_pll_sel",
- .ops = &clk_regmap_mux_ops,
- /*
- * bit 18 selects from 2 possible parents:
- * vid_pll_div or hdmi_pll
- */
- .parent_names = g12a_vid_pll_parent_names,
- .num_parents = ARRAY_SIZE(g12a_vid_pll_parent_names),
- .flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE,
- },
-};
-
-static struct clk_regmap g12a_vid_pll = {
- .data = &(struct clk_regmap_gate_data){
- .offset = HHI_VID_PLL_CLK_DIV,
- .bit_idx = 19,
- },
- .hw.init = &(struct clk_init_data) {
- .name = "vid_pll",
- .ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vid_pll_sel" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vapb_sel.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
};
-static const char * const g12a_vclk_parent_names[] = {
- "vid_pll", "gp0_pll", "hifi_pll", "mpll1", "fclk_div3", "fclk_div4",
- "fclk_div5", "fclk_div7"
+static const struct clk_hw *g12a_vclk_parent_hws[] = {
+ &g12a_vid_pll.hw,
+ &g12a_gp0_pll.hw,
+ &g12a_hifi_pll.hw,
+ &g12a_mpll1.hw,
+ &g12a_fclk_div3.hw,
+ &g12a_fclk_div4.hw,
+ &g12a_fclk_div5.hw,
+ &g12a_fclk_div7.hw,
};
static struct clk_regmap g12a_vclk_sel = {
@@ -1842,8 +2842,8 @@ static struct clk_regmap g12a_vclk_sel = {
.hw.init = &(struct clk_init_data){
.name = "vclk_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_vclk_parent_names,
- .num_parents = ARRAY_SIZE(g12a_vclk_parent_names),
+ .parent_hws = g12a_vclk_parent_hws,
+ .num_parents = ARRAY_SIZE(g12a_vclk_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE,
},
};
@@ -1857,8 +2857,8 @@ static struct clk_regmap g12a_vclk2_sel = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_vclk_parent_names,
- .num_parents = ARRAY_SIZE(g12a_vclk_parent_names),
+ .parent_hws = g12a_vclk_parent_hws,
+ .num_parents = ARRAY_SIZE(g12a_vclk_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE,
},
};
@@ -1871,7 +2871,7 @@ static struct clk_regmap g12a_vclk_input = {
.hw.init = &(struct clk_init_data) {
.name = "vclk_input",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk_sel" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vclk_sel.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1885,7 +2885,7 @@ static struct clk_regmap g12a_vclk2_input = {
.hw.init = &(struct clk_init_data) {
.name = "vclk2_input",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk2_sel" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2_sel.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1900,7 +2900,9 @@ static struct clk_regmap g12a_vclk_div = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vclk_input" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vclk_input.hw
+ },
.num_parents = 1,
.flags = CLK_GET_RATE_NOCACHE,
},
@@ -1915,7 +2917,9 @@ static struct clk_regmap g12a_vclk2_div = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vclk2_input" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vclk2_input.hw
+ },
.num_parents = 1,
.flags = CLK_GET_RATE_NOCACHE,
},
@@ -1929,7 +2933,7 @@ static struct clk_regmap g12a_vclk = {
.hw.init = &(struct clk_init_data) {
.name = "vclk",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk_div" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vclk_div.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1943,7 +2947,7 @@ static struct clk_regmap g12a_vclk2 = {
.hw.init = &(struct clk_init_data) {
.name = "vclk2",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk2_div" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2_div.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1957,7 +2961,7 @@ static struct clk_regmap g12a_vclk_div1 = {
.hw.init = &(struct clk_init_data) {
.name = "vclk_div1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vclk.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1971,7 +2975,7 @@ static struct clk_regmap g12a_vclk_div2_en = {
.hw.init = &(struct clk_init_data) {
.name = "vclk_div2_en",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vclk.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1985,7 +2989,7 @@ static struct clk_regmap g12a_vclk_div4_en = {
.hw.init = &(struct clk_init_data) {
.name = "vclk_div4_en",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vclk.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1999,7 +3003,7 @@ static struct clk_regmap g12a_vclk_div6_en = {
.hw.init = &(struct clk_init_data) {
.name = "vclk_div6_en",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vclk.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -2013,7 +3017,7 @@ static struct clk_regmap g12a_vclk_div12_en = {
.hw.init = &(struct clk_init_data) {
.name = "vclk_div12_en",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vclk.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -2027,7 +3031,7 @@ static struct clk_regmap g12a_vclk2_div1 = {
.hw.init = &(struct clk_init_data) {
.name = "vclk2_div1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk2" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -2041,7 +3045,7 @@ static struct clk_regmap g12a_vclk2_div2_en = {
.hw.init = &(struct clk_init_data) {
.name = "vclk2_div2_en",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk2" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -2055,7 +3059,7 @@ static struct clk_regmap g12a_vclk2_div4_en = {
.hw.init = &(struct clk_init_data) {
.name = "vclk2_div4_en",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk2" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -2069,7 +3073,7 @@ static struct clk_regmap g12a_vclk2_div6_en = {
.hw.init = &(struct clk_init_data) {
.name = "vclk2_div6_en",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk2" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -2083,7 +3087,7 @@ static struct clk_regmap g12a_vclk2_div12_en = {
.hw.init = &(struct clk_init_data) {
.name = "vclk2_div12_en",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk2" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -2095,7 +3099,9 @@ static struct clk_fixed_factor g12a_vclk_div2 = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div2",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk_div2_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vclk_div2_en.hw
+ },
.num_parents = 1,
},
};
@@ -2106,7 +3112,9 @@ static struct clk_fixed_factor g12a_vclk_div4 = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div4",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk_div4_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vclk_div4_en.hw
+ },
.num_parents = 1,
},
};
@@ -2117,7 +3125,9 @@ static struct clk_fixed_factor g12a_vclk_div6 = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div6",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk_div6_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vclk_div6_en.hw
+ },
.num_parents = 1,
},
};
@@ -2128,7 +3138,9 @@ static struct clk_fixed_factor g12a_vclk_div12 = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div12",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk_div12_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vclk_div12_en.hw
+ },
.num_parents = 1,
},
};
@@ -2139,7 +3151,9 @@ static struct clk_fixed_factor g12a_vclk2_div2 = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div2",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk2_div2_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vclk2_div2_en.hw
+ },
.num_parents = 1,
},
};
@@ -2150,7 +3164,9 @@ static struct clk_fixed_factor g12a_vclk2_div4 = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div4",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk2_div4_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vclk2_div4_en.hw
+ },
.num_parents = 1,
},
};
@@ -2161,7 +3177,9 @@ static struct clk_fixed_factor g12a_vclk2_div6 = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div6",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk2_div6_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vclk2_div6_en.hw
+ },
.num_parents = 1,
},
};
@@ -2172,16 +3190,25 @@ static struct clk_fixed_factor g12a_vclk2_div12 = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div12",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk2_div12_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_vclk2_div12_en.hw
+ },
.num_parents = 1,
},
};
static u32 mux_table_cts_sel[] = { 0, 1, 2, 3, 4, 8, 9, 10, 11, 12 };
-static const char * const g12a_cts_parent_names[] = {
- "vclk_div1", "vclk_div2", "vclk_div4", "vclk_div6",
- "vclk_div12", "vclk2_div1", "vclk2_div2", "vclk2_div4",
- "vclk2_div6", "vclk2_div12"
+static const struct clk_hw *g12a_cts_parent_hws[] = {
+ &g12a_vclk_div1.hw,
+ &g12a_vclk_div2.hw,
+ &g12a_vclk_div4.hw,
+ &g12a_vclk_div6.hw,
+ &g12a_vclk_div12.hw,
+ &g12a_vclk2_div1.hw,
+ &g12a_vclk2_div2.hw,
+ &g12a_vclk2_div4.hw,
+ &g12a_vclk2_div6.hw,
+ &g12a_vclk2_div12.hw,
};
static struct clk_regmap g12a_cts_enci_sel = {
@@ -2194,8 +3221,8 @@ static struct clk_regmap g12a_cts_enci_sel = {
.hw.init = &(struct clk_init_data){
.name = "cts_enci_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_cts_parent_names,
- .num_parents = ARRAY_SIZE(g12a_cts_parent_names),
+ .parent_hws = g12a_cts_parent_hws,
+ .num_parents = ARRAY_SIZE(g12a_cts_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE,
},
};
@@ -2210,8 +3237,8 @@ static struct clk_regmap g12a_cts_encp_sel = {
.hw.init = &(struct clk_init_data){
.name = "cts_encp_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_cts_parent_names,
- .num_parents = ARRAY_SIZE(g12a_cts_parent_names),
+ .parent_hws = g12a_cts_parent_hws,
+ .num_parents = ARRAY_SIZE(g12a_cts_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE,
},
};
@@ -2226,18 +3253,25 @@ static struct clk_regmap g12a_cts_vdac_sel = {
.hw.init = &(struct clk_init_data){
.name = "cts_vdac_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_cts_parent_names,
- .num_parents = ARRAY_SIZE(g12a_cts_parent_names),
+ .parent_hws = g12a_cts_parent_hws,
+ .num_parents = ARRAY_SIZE(g12a_cts_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE,
},
};
/* TOFIX: add support for cts_tcon */
static u32 mux_table_hdmi_tx_sel[] = { 0, 1, 2, 3, 4, 8, 9, 10, 11, 12 };
-static const char * const g12a_cts_hdmi_tx_parent_names[] = {
- "vclk_div1", "vclk_div2", "vclk_div4", "vclk_div6",
- "vclk_div12", "vclk2_div1", "vclk2_div2", "vclk2_div4",
- "vclk2_div6", "vclk2_div12"
+static const struct clk_hw *g12a_cts_hdmi_tx_parent_hws[] = {
+ &g12a_vclk_div1.hw,
+ &g12a_vclk_div2.hw,
+ &g12a_vclk_div4.hw,
+ &g12a_vclk_div6.hw,
+ &g12a_vclk_div12.hw,
+ &g12a_vclk2_div1.hw,
+ &g12a_vclk2_div2.hw,
+ &g12a_vclk2_div4.hw,
+ &g12a_vclk2_div6.hw,
+ &g12a_vclk2_div12.hw,
};
static struct clk_regmap g12a_hdmi_tx_sel = {
@@ -2250,8 +3284,8 @@ static struct clk_regmap g12a_hdmi_tx_sel = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_tx_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_cts_hdmi_tx_parent_names,
- .num_parents = ARRAY_SIZE(g12a_cts_hdmi_tx_parent_names),
+ .parent_hws = g12a_cts_hdmi_tx_parent_hws,
+ .num_parents = ARRAY_SIZE(g12a_cts_hdmi_tx_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE,
},
};
@@ -2264,7 +3298,9 @@ static struct clk_regmap g12a_cts_enci = {
.hw.init = &(struct clk_init_data) {
.name = "cts_enci",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "cts_enci_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_cts_enci_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -2278,7 +3314,9 @@ static struct clk_regmap g12a_cts_encp = {
.hw.init = &(struct clk_init_data) {
.name = "cts_encp",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "cts_encp_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_cts_encp_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -2292,7 +3330,9 @@ static struct clk_regmap g12a_cts_vdac = {
.hw.init = &(struct clk_init_data) {
.name = "cts_vdac",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "cts_vdac_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_cts_vdac_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -2306,7 +3346,9 @@ static struct clk_regmap g12a_hdmi_tx = {
.hw.init = &(struct clk_init_data) {
.name = "hdmi_tx",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "hdmi_tx_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_hdmi_tx_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -2314,8 +3356,11 @@ static struct clk_regmap g12a_hdmi_tx = {
/* HDMI Clocks */
-static const char * const g12a_hdmi_parent_names[] = {
- IN_PREFIX "xtal", "fclk_div4", "fclk_div3", "fclk_div5"
+static const struct clk_parent_data g12a_hdmi_parent_data[] = {
+ { .fw_name = "xtal", },
+ { .hw = &g12a_fclk_div4.hw },
+ { .hw = &g12a_fclk_div3.hw },
+ { .hw = &g12a_fclk_div5.hw },
};
static struct clk_regmap g12a_hdmi_sel = {
@@ -2328,8 +3373,8 @@ static struct clk_regmap g12a_hdmi_sel = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_hdmi_parent_names,
- .num_parents = ARRAY_SIZE(g12a_hdmi_parent_names),
+ .parent_data = g12a_hdmi_parent_data,
+ .num_parents = ARRAY_SIZE(g12a_hdmi_parent_data),
.flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE,
},
};
@@ -2343,7 +3388,7 @@ static struct clk_regmap g12a_hdmi_div = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "hdmi_sel" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_hdmi_sel.hw },
.num_parents = 1,
.flags = CLK_GET_RATE_NOCACHE,
},
@@ -2357,7 +3402,7 @@ static struct clk_regmap g12a_hdmi = {
.hw.init = &(struct clk_init_data) {
.name = "hdmi",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "hdmi_div" },
+ .parent_hws = (const struct clk_hw *[]) { &g12a_hdmi_div.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -2367,10 +3412,15 @@ static struct clk_regmap g12a_hdmi = {
* The MALI IP is clocked by two identical clocks (mali_0 and mali_1)
* muxed by a glitch-free switch.
*/
-
-static const char * const g12a_mali_0_1_parent_names[] = {
- IN_PREFIX "xtal", "gp0_pll", "hihi_pll", "fclk_div2p5",
- "fclk_div3", "fclk_div4", "fclk_div5", "fclk_div7"
+static const struct clk_parent_data g12a_mali_0_1_parent_data[] = {
+ { .fw_name = "xtal", },
+ { .hw = &g12a_gp0_pll.hw },
+ { .hw = &g12a_hifi_pll.hw },
+ { .hw = &g12a_fclk_div2p5.hw },
+ { .hw = &g12a_fclk_div3.hw },
+ { .hw = &g12a_fclk_div4.hw },
+ { .hw = &g12a_fclk_div5.hw },
+ { .hw = &g12a_fclk_div7.hw },
};
static struct clk_regmap g12a_mali_0_sel = {
@@ -2382,7 +3432,7 @@ static struct clk_regmap g12a_mali_0_sel = {
.hw.init = &(struct clk_init_data){
.name = "mali_0_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_mali_0_1_parent_names,
+ .parent_data = g12a_mali_0_1_parent_data,
.num_parents = 8,
.flags = CLK_SET_RATE_NO_REPARENT,
},
@@ -2397,7 +3447,9 @@ static struct clk_regmap g12a_mali_0_div = {
.hw.init = &(struct clk_init_data){
.name = "mali_0_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "mali_0_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_mali_0_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_NO_REPARENT,
},
@@ -2411,7 +3463,9 @@ static struct clk_regmap g12a_mali_0 = {
.hw.init = &(struct clk_init_data){
.name = "mali_0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mali_0_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_mali_0_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -2426,7 +3480,7 @@ static struct clk_regmap g12a_mali_1_sel = {
.hw.init = &(struct clk_init_data){
.name = "mali_1_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_mali_0_1_parent_names,
+ .parent_data = g12a_mali_0_1_parent_data,
.num_parents = 8,
.flags = CLK_SET_RATE_NO_REPARENT,
},
@@ -2441,7 +3495,9 @@ static struct clk_regmap g12a_mali_1_div = {
.hw.init = &(struct clk_init_data){
.name = "mali_1_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "mali_1_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_mali_1_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_NO_REPARENT,
},
@@ -2455,14 +3511,17 @@ static struct clk_regmap g12a_mali_1 = {
.hw.init = &(struct clk_init_data){
.name = "mali_1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mali_1_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_mali_1_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
};
-static const char * const g12a_mali_parent_names[] = {
- "mali_0", "mali_1"
+static const struct clk_hw *g12a_mali_parent_hws[] = {
+ &g12a_mali_0.hw,
+ &g12a_mali_1.hw,
};
static struct clk_regmap g12a_mali = {
@@ -2474,12 +3533,49 @@ static struct clk_regmap g12a_mali = {
.hw.init = &(struct clk_init_data){
.name = "mali",
.ops = &clk_regmap_mux_ops,
- .parent_names = g12a_mali_parent_names,
+ .parent_hws = g12a_mali_parent_hws,
.num_parents = 2,
.flags = CLK_SET_RATE_NO_REPARENT,
},
};
+static struct clk_regmap g12a_ts_div = {
+ .data = &(struct clk_regmap_div_data){
+ .offset = HHI_TS_CLK_CNTL,
+ .shift = 0,
+ .width = 8,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "ts_div",
+ .ops = &clk_regmap_divider_ro_ops,
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
+ .num_parents = 1,
+ },
+};
+
+static struct clk_regmap g12a_ts = {
+ .data = &(struct clk_regmap_gate_data){
+ .offset = HHI_TS_CLK_CNTL,
+ .bit_idx = 8,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "ts",
+ .ops = &clk_regmap_gate_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &g12a_ts_div.hw
+ },
+ .num_parents = 1,
+ },
+};
+
+#define MESON_GATE(_name, _reg, _bit) \
+ MESON_PCLK(_name, _reg, _bit, &g12a_clk81.hw)
+
+#define MESON_GATE_RO(_name, _reg, _bit) \
+ MESON_PCLK_RO(_name, _reg, _bit, &g12a_clk81.hw)
+
/* Everything Else (EE) domain gates */
static MESON_GATE(g12a_ddr, HHI_GCLK_MPEG0, 0);
static MESON_GATE(g12a_dos, HHI_GCLK_MPEG0, 1);
@@ -2769,6 +3865,257 @@ static struct clk_hw_onecell_data g12a_hw_onecell_data = {
[CLKID_VDEC_HEVCF_SEL] = &g12a_vdec_hevcf_sel.hw,
[CLKID_VDEC_HEVCF_DIV] = &g12a_vdec_hevcf_div.hw,
[CLKID_VDEC_HEVCF] = &g12a_vdec_hevcf.hw,
+ [CLKID_TS_DIV] = &g12a_ts_div.hw,
+ [CLKID_TS] = &g12a_ts.hw,
+ [NR_CLKS] = NULL,
+ },
+ .num = NR_CLKS,
+};
+
+static struct clk_hw_onecell_data g12b_hw_onecell_data = {
+ .hws = {
+ [CLKID_SYS_PLL] = &g12a_sys_pll.hw,
+ [CLKID_FIXED_PLL] = &g12a_fixed_pll.hw,
+ [CLKID_FCLK_DIV2] = &g12a_fclk_div2.hw,
+ [CLKID_FCLK_DIV3] = &g12a_fclk_div3.hw,
+ [CLKID_FCLK_DIV4] = &g12a_fclk_div4.hw,
+ [CLKID_FCLK_DIV5] = &g12a_fclk_div5.hw,
+ [CLKID_FCLK_DIV7] = &g12a_fclk_div7.hw,
+ [CLKID_FCLK_DIV2P5] = &g12a_fclk_div2p5.hw,
+ [CLKID_GP0_PLL] = &g12a_gp0_pll.hw,
+ [CLKID_MPEG_SEL] = &g12a_mpeg_clk_sel.hw,
+ [CLKID_MPEG_DIV] = &g12a_mpeg_clk_div.hw,
+ [CLKID_CLK81] = &g12a_clk81.hw,
+ [CLKID_MPLL0] = &g12a_mpll0.hw,
+ [CLKID_MPLL1] = &g12a_mpll1.hw,
+ [CLKID_MPLL2] = &g12a_mpll2.hw,
+ [CLKID_MPLL3] = &g12a_mpll3.hw,
+ [CLKID_DDR] = &g12a_ddr.hw,
+ [CLKID_DOS] = &g12a_dos.hw,
+ [CLKID_AUDIO_LOCKER] = &g12a_audio_locker.hw,
+ [CLKID_MIPI_DSI_HOST] = &g12a_mipi_dsi_host.hw,
+ [CLKID_ETH_PHY] = &g12a_eth_phy.hw,
+ [CLKID_ISA] = &g12a_isa.hw,
+ [CLKID_PL301] = &g12a_pl301.hw,
+ [CLKID_PERIPHS] = &g12a_periphs.hw,
+ [CLKID_SPICC0] = &g12a_spicc_0.hw,
+ [CLKID_I2C] = &g12a_i2c.hw,
+ [CLKID_SANA] = &g12a_sana.hw,
+ [CLKID_SD] = &g12a_sd.hw,
+ [CLKID_RNG0] = &g12a_rng0.hw,
+ [CLKID_UART0] = &g12a_uart0.hw,
+ [CLKID_SPICC1] = &g12a_spicc_1.hw,
+ [CLKID_HIU_IFACE] = &g12a_hiu_reg.hw,
+ [CLKID_MIPI_DSI_PHY] = &g12a_mipi_dsi_phy.hw,
+ [CLKID_ASSIST_MISC] = &g12a_assist_misc.hw,
+ [CLKID_SD_EMMC_A] = &g12a_emmc_a.hw,
+ [CLKID_SD_EMMC_B] = &g12a_emmc_b.hw,
+ [CLKID_SD_EMMC_C] = &g12a_emmc_c.hw,
+ [CLKID_AUDIO_CODEC] = &g12a_audio_codec.hw,
+ [CLKID_AUDIO] = &g12a_audio.hw,
+ [CLKID_ETH] = &g12a_eth_core.hw,
+ [CLKID_DEMUX] = &g12a_demux.hw,
+ [CLKID_AUDIO_IFIFO] = &g12a_audio_ififo.hw,
+ [CLKID_ADC] = &g12a_adc.hw,
+ [CLKID_UART1] = &g12a_uart1.hw,
+ [CLKID_G2D] = &g12a_g2d.hw,
+ [CLKID_RESET] = &g12a_reset.hw,
+ [CLKID_PCIE_COMB] = &g12a_pcie_comb.hw,
+ [CLKID_PARSER] = &g12a_parser.hw,
+ [CLKID_USB] = &g12a_usb_general.hw,
+ [CLKID_PCIE_PHY] = &g12a_pcie_phy.hw,
+ [CLKID_AHB_ARB0] = &g12a_ahb_arb0.hw,
+ [CLKID_AHB_DATA_BUS] = &g12a_ahb_data_bus.hw,
+ [CLKID_AHB_CTRL_BUS] = &g12a_ahb_ctrl_bus.hw,
+ [CLKID_HTX_HDCP22] = &g12a_htx_hdcp22.hw,
+ [CLKID_HTX_PCLK] = &g12a_htx_pclk.hw,
+ [CLKID_BT656] = &g12a_bt656.hw,
+ [CLKID_USB1_DDR_BRIDGE] = &g12a_usb1_to_ddr.hw,
+ [CLKID_MMC_PCLK] = &g12a_mmc_pclk.hw,
+ [CLKID_UART2] = &g12a_uart2.hw,
+ [CLKID_VPU_INTR] = &g12a_vpu_intr.hw,
+ [CLKID_GIC] = &g12a_gic.hw,
+ [CLKID_SD_EMMC_A_CLK0_SEL] = &g12a_sd_emmc_a_clk0_sel.hw,
+ [CLKID_SD_EMMC_A_CLK0_DIV] = &g12a_sd_emmc_a_clk0_div.hw,
+ [CLKID_SD_EMMC_A_CLK0] = &g12a_sd_emmc_a_clk0.hw,
+ [CLKID_SD_EMMC_B_CLK0_SEL] = &g12a_sd_emmc_b_clk0_sel.hw,
+ [CLKID_SD_EMMC_B_CLK0_DIV] = &g12a_sd_emmc_b_clk0_div.hw,
+ [CLKID_SD_EMMC_B_CLK0] = &g12a_sd_emmc_b_clk0.hw,
+ [CLKID_SD_EMMC_C_CLK0_SEL] = &g12a_sd_emmc_c_clk0_sel.hw,
+ [CLKID_SD_EMMC_C_CLK0_DIV] = &g12a_sd_emmc_c_clk0_div.hw,
+ [CLKID_SD_EMMC_C_CLK0] = &g12a_sd_emmc_c_clk0.hw,
+ [CLKID_MPLL0_DIV] = &g12a_mpll0_div.hw,
+ [CLKID_MPLL1_DIV] = &g12a_mpll1_div.hw,
+ [CLKID_MPLL2_DIV] = &g12a_mpll2_div.hw,
+ [CLKID_MPLL3_DIV] = &g12a_mpll3_div.hw,
+ [CLKID_FCLK_DIV2_DIV] = &g12a_fclk_div2_div.hw,
+ [CLKID_FCLK_DIV3_DIV] = &g12a_fclk_div3_div.hw,
+ [CLKID_FCLK_DIV4_DIV] = &g12a_fclk_div4_div.hw,
+ [CLKID_FCLK_DIV5_DIV] = &g12a_fclk_div5_div.hw,
+ [CLKID_FCLK_DIV7_DIV] = &g12a_fclk_div7_div.hw,
+ [CLKID_FCLK_DIV2P5_DIV] = &g12a_fclk_div2p5_div.hw,
+ [CLKID_HIFI_PLL] = &g12a_hifi_pll.hw,
+ [CLKID_VCLK2_VENCI0] = &g12a_vclk2_venci0.hw,
+ [CLKID_VCLK2_VENCI1] = &g12a_vclk2_venci1.hw,
+ [CLKID_VCLK2_VENCP0] = &g12a_vclk2_vencp0.hw,
+ [CLKID_VCLK2_VENCP1] = &g12a_vclk2_vencp1.hw,
+ [CLKID_VCLK2_VENCT0] = &g12a_vclk2_venct0.hw,
+ [CLKID_VCLK2_VENCT1] = &g12a_vclk2_venct1.hw,
+ [CLKID_VCLK2_OTHER] = &g12a_vclk2_other.hw,
+ [CLKID_VCLK2_ENCI] = &g12a_vclk2_enci.hw,
+ [CLKID_VCLK2_ENCP] = &g12a_vclk2_encp.hw,
+ [CLKID_DAC_CLK] = &g12a_dac_clk.hw,
+ [CLKID_AOCLK] = &g12a_aoclk_gate.hw,
+ [CLKID_IEC958] = &g12a_iec958_gate.hw,
+ [CLKID_ENC480P] = &g12a_enc480p.hw,
+ [CLKID_RNG1] = &g12a_rng1.hw,
+ [CLKID_VCLK2_ENCT] = &g12a_vclk2_enct.hw,
+ [CLKID_VCLK2_ENCL] = &g12a_vclk2_encl.hw,
+ [CLKID_VCLK2_VENCLMMC] = &g12a_vclk2_venclmmc.hw,
+ [CLKID_VCLK2_VENCL] = &g12a_vclk2_vencl.hw,
+ [CLKID_VCLK2_OTHER1] = &g12a_vclk2_other1.hw,
+ [CLKID_FIXED_PLL_DCO] = &g12a_fixed_pll_dco.hw,
+ [CLKID_SYS_PLL_DCO] = &g12a_sys_pll_dco.hw,
+ [CLKID_GP0_PLL_DCO] = &g12a_gp0_pll_dco.hw,
+ [CLKID_HIFI_PLL_DCO] = &g12a_hifi_pll_dco.hw,
+ [CLKID_DMA] = &g12a_dma.hw,
+ [CLKID_EFUSE] = &g12a_efuse.hw,
+ [CLKID_ROM_BOOT] = &g12a_rom_boot.hw,
+ [CLKID_RESET_SEC] = &g12a_reset_sec.hw,
+ [CLKID_SEC_AHB_APB3] = &g12a_sec_ahb_apb3.hw,
+ [CLKID_MPLL_PREDIV] = &g12a_mpll_prediv.hw,
+ [CLKID_VPU_0_SEL] = &g12a_vpu_0_sel.hw,
+ [CLKID_VPU_0_DIV] = &g12a_vpu_0_div.hw,
+ [CLKID_VPU_0] = &g12a_vpu_0.hw,
+ [CLKID_VPU_1_SEL] = &g12a_vpu_1_sel.hw,
+ [CLKID_VPU_1_DIV] = &g12a_vpu_1_div.hw,
+ [CLKID_VPU_1] = &g12a_vpu_1.hw,
+ [CLKID_VPU] = &g12a_vpu.hw,
+ [CLKID_VAPB_0_SEL] = &g12a_vapb_0_sel.hw,
+ [CLKID_VAPB_0_DIV] = &g12a_vapb_0_div.hw,
+ [CLKID_VAPB_0] = &g12a_vapb_0.hw,
+ [CLKID_VAPB_1_SEL] = &g12a_vapb_1_sel.hw,
+ [CLKID_VAPB_1_DIV] = &g12a_vapb_1_div.hw,
+ [CLKID_VAPB_1] = &g12a_vapb_1.hw,
+ [CLKID_VAPB_SEL] = &g12a_vapb_sel.hw,
+ [CLKID_VAPB] = &g12a_vapb.hw,
+ [CLKID_HDMI_PLL_DCO] = &g12a_hdmi_pll_dco.hw,
+ [CLKID_HDMI_PLL_OD] = &g12a_hdmi_pll_od.hw,
+ [CLKID_HDMI_PLL_OD2] = &g12a_hdmi_pll_od2.hw,
+ [CLKID_HDMI_PLL] = &g12a_hdmi_pll.hw,
+ [CLKID_VID_PLL] = &g12a_vid_pll_div.hw,
+ [CLKID_VID_PLL_SEL] = &g12a_vid_pll_sel.hw,
+ [CLKID_VID_PLL_DIV] = &g12a_vid_pll.hw,
+ [CLKID_VCLK_SEL] = &g12a_vclk_sel.hw,
+ [CLKID_VCLK2_SEL] = &g12a_vclk2_sel.hw,
+ [CLKID_VCLK_INPUT] = &g12a_vclk_input.hw,
+ [CLKID_VCLK2_INPUT] = &g12a_vclk2_input.hw,
+ [CLKID_VCLK_DIV] = &g12a_vclk_div.hw,
+ [CLKID_VCLK2_DIV] = &g12a_vclk2_div.hw,
+ [CLKID_VCLK] = &g12a_vclk.hw,
+ [CLKID_VCLK2] = &g12a_vclk2.hw,
+ [CLKID_VCLK_DIV1] = &g12a_vclk_div1.hw,
+ [CLKID_VCLK_DIV2_EN] = &g12a_vclk_div2_en.hw,
+ [CLKID_VCLK_DIV4_EN] = &g12a_vclk_div4_en.hw,
+ [CLKID_VCLK_DIV6_EN] = &g12a_vclk_div6_en.hw,
+ [CLKID_VCLK_DIV12_EN] = &g12a_vclk_div12_en.hw,
+ [CLKID_VCLK2_DIV1] = &g12a_vclk2_div1.hw,
+ [CLKID_VCLK2_DIV2_EN] = &g12a_vclk2_div2_en.hw,
+ [CLKID_VCLK2_DIV4_EN] = &g12a_vclk2_div4_en.hw,
+ [CLKID_VCLK2_DIV6_EN] = &g12a_vclk2_div6_en.hw,
+ [CLKID_VCLK2_DIV12_EN] = &g12a_vclk2_div12_en.hw,
+ [CLKID_VCLK_DIV2] = &g12a_vclk_div2.hw,
+ [CLKID_VCLK_DIV4] = &g12a_vclk_div4.hw,
+ [CLKID_VCLK_DIV6] = &g12a_vclk_div6.hw,
+ [CLKID_VCLK_DIV12] = &g12a_vclk_div12.hw,
+ [CLKID_VCLK2_DIV2] = &g12a_vclk2_div2.hw,
+ [CLKID_VCLK2_DIV4] = &g12a_vclk2_div4.hw,
+ [CLKID_VCLK2_DIV6] = &g12a_vclk2_div6.hw,
+ [CLKID_VCLK2_DIV12] = &g12a_vclk2_div12.hw,
+ [CLKID_CTS_ENCI_SEL] = &g12a_cts_enci_sel.hw,
+ [CLKID_CTS_ENCP_SEL] = &g12a_cts_encp_sel.hw,
+ [CLKID_CTS_VDAC_SEL] = &g12a_cts_vdac_sel.hw,
+ [CLKID_HDMI_TX_SEL] = &g12a_hdmi_tx_sel.hw,
+ [CLKID_CTS_ENCI] = &g12a_cts_enci.hw,
+ [CLKID_CTS_ENCP] = &g12a_cts_encp.hw,
+ [CLKID_CTS_VDAC] = &g12a_cts_vdac.hw,
+ [CLKID_HDMI_TX] = &g12a_hdmi_tx.hw,
+ [CLKID_HDMI_SEL] = &g12a_hdmi_sel.hw,
+ [CLKID_HDMI_DIV] = &g12a_hdmi_div.hw,
+ [CLKID_HDMI] = &g12a_hdmi.hw,
+ [CLKID_MALI_0_SEL] = &g12a_mali_0_sel.hw,
+ [CLKID_MALI_0_DIV] = &g12a_mali_0_div.hw,
+ [CLKID_MALI_0] = &g12a_mali_0.hw,
+ [CLKID_MALI_1_SEL] = &g12a_mali_1_sel.hw,
+ [CLKID_MALI_1_DIV] = &g12a_mali_1_div.hw,
+ [CLKID_MALI_1] = &g12a_mali_1.hw,
+ [CLKID_MALI] = &g12a_mali.hw,
+ [CLKID_MPLL_50M_DIV] = &g12a_mpll_50m_div.hw,
+ [CLKID_MPLL_50M] = &g12a_mpll_50m.hw,
+ [CLKID_SYS_PLL_DIV16_EN] = &g12a_sys_pll_div16_en.hw,
+ [CLKID_SYS_PLL_DIV16] = &g12a_sys_pll_div16.hw,
+ [CLKID_CPU_CLK_DYN0_SEL] = &g12a_cpu_clk_premux0.hw,
+ [CLKID_CPU_CLK_DYN0_DIV] = &g12a_cpu_clk_mux0_div.hw,
+ [CLKID_CPU_CLK_DYN0] = &g12a_cpu_clk_postmux0.hw,
+ [CLKID_CPU_CLK_DYN1_SEL] = &g12a_cpu_clk_premux1.hw,
+ [CLKID_CPU_CLK_DYN1_DIV] = &g12a_cpu_clk_mux1_div.hw,
+ [CLKID_CPU_CLK_DYN1] = &g12a_cpu_clk_postmux1.hw,
+ [CLKID_CPU_CLK_DYN] = &g12a_cpu_clk_dyn.hw,
+ [CLKID_CPU_CLK] = &g12b_cpu_clk.hw,
+ [CLKID_CPU_CLK_DIV16_EN] = &g12a_cpu_clk_div16_en.hw,
+ [CLKID_CPU_CLK_DIV16] = &g12a_cpu_clk_div16.hw,
+ [CLKID_CPU_CLK_APB_DIV] = &g12a_cpu_clk_apb_div.hw,
+ [CLKID_CPU_CLK_APB] = &g12a_cpu_clk_apb.hw,
+ [CLKID_CPU_CLK_ATB_DIV] = &g12a_cpu_clk_atb_div.hw,
+ [CLKID_CPU_CLK_ATB] = &g12a_cpu_clk_atb.hw,
+ [CLKID_CPU_CLK_AXI_DIV] = &g12a_cpu_clk_axi_div.hw,
+ [CLKID_CPU_CLK_AXI] = &g12a_cpu_clk_axi.hw,
+ [CLKID_CPU_CLK_TRACE_DIV] = &g12a_cpu_clk_trace_div.hw,
+ [CLKID_CPU_CLK_TRACE] = &g12a_cpu_clk_trace.hw,
+ [CLKID_PCIE_PLL_DCO] = &g12a_pcie_pll_dco.hw,
+ [CLKID_PCIE_PLL_DCO_DIV2] = &g12a_pcie_pll_dco_div2.hw,
+ [CLKID_PCIE_PLL_OD] = &g12a_pcie_pll_od.hw,
+ [CLKID_PCIE_PLL] = &g12a_pcie_pll.hw,
+ [CLKID_VDEC_1_SEL] = &g12a_vdec_1_sel.hw,
+ [CLKID_VDEC_1_DIV] = &g12a_vdec_1_div.hw,
+ [CLKID_VDEC_1] = &g12a_vdec_1.hw,
+ [CLKID_VDEC_HEVC_SEL] = &g12a_vdec_hevc_sel.hw,
+ [CLKID_VDEC_HEVC_DIV] = &g12a_vdec_hevc_div.hw,
+ [CLKID_VDEC_HEVC] = &g12a_vdec_hevc.hw,
+ [CLKID_VDEC_HEVCF_SEL] = &g12a_vdec_hevcf_sel.hw,
+ [CLKID_VDEC_HEVCF_DIV] = &g12a_vdec_hevcf_div.hw,
+ [CLKID_VDEC_HEVCF] = &g12a_vdec_hevcf.hw,
+ [CLKID_TS_DIV] = &g12a_ts_div.hw,
+ [CLKID_TS] = &g12a_ts.hw,
+ [CLKID_SYS1_PLL_DCO] = &g12b_sys1_pll_dco.hw,
+ [CLKID_SYS1_PLL] = &g12b_sys1_pll.hw,
+ [CLKID_SYS1_PLL_DIV16_EN] = &g12b_sys1_pll_div16_en.hw,
+ [CLKID_SYS1_PLL_DIV16] = &g12b_sys1_pll_div16.hw,
+ [CLKID_CPUB_CLK_DYN0_SEL] = &g12b_cpub_clk_premux0.hw,
+ [CLKID_CPUB_CLK_DYN0_DIV] = &g12b_cpub_clk_mux0_div.hw,
+ [CLKID_CPUB_CLK_DYN0] = &g12b_cpub_clk_postmux0.hw,
+ [CLKID_CPUB_CLK_DYN1_SEL] = &g12b_cpub_clk_premux1.hw,
+ [CLKID_CPUB_CLK_DYN1_DIV] = &g12b_cpub_clk_mux1_div.hw,
+ [CLKID_CPUB_CLK_DYN1] = &g12b_cpub_clk_postmux1.hw,
+ [CLKID_CPUB_CLK_DYN] = &g12b_cpub_clk_dyn.hw,
+ [CLKID_CPUB_CLK] = &g12b_cpub_clk.hw,
+ [CLKID_CPUB_CLK_DIV16_EN] = &g12b_cpub_clk_div16_en.hw,
+ [CLKID_CPUB_CLK_DIV16] = &g12b_cpub_clk_div16.hw,
+ [CLKID_CPUB_CLK_DIV2] = &g12b_cpub_clk_div2.hw,
+ [CLKID_CPUB_CLK_DIV3] = &g12b_cpub_clk_div3.hw,
+ [CLKID_CPUB_CLK_DIV4] = &g12b_cpub_clk_div4.hw,
+ [CLKID_CPUB_CLK_DIV5] = &g12b_cpub_clk_div5.hw,
+ [CLKID_CPUB_CLK_DIV6] = &g12b_cpub_clk_div6.hw,
+ [CLKID_CPUB_CLK_DIV7] = &g12b_cpub_clk_div7.hw,
+ [CLKID_CPUB_CLK_DIV8] = &g12b_cpub_clk_div8.hw,
+ [CLKID_CPUB_CLK_APB_SEL] = &g12b_cpub_clk_apb_sel.hw,
+ [CLKID_CPUB_CLK_APB] = &g12b_cpub_clk_apb.hw,
+ [CLKID_CPUB_CLK_ATB_SEL] = &g12b_cpub_clk_atb_sel.hw,
+ [CLKID_CPUB_CLK_ATB] = &g12b_cpub_clk_atb.hw,
+ [CLKID_CPUB_CLK_AXI_SEL] = &g12b_cpub_clk_axi_sel.hw,
+ [CLKID_CPUB_CLK_AXI] = &g12b_cpub_clk_axi.hw,
+ [CLKID_CPUB_CLK_TRACE_SEL] = &g12b_cpub_clk_trace_sel.hw,
+ [CLKID_CPUB_CLK_TRACE] = &g12b_cpub_clk_trace.hw,
[NR_CLKS] = NULL,
},
.num = NR_CLKS,
@@ -2966,21 +4313,239 @@ static struct clk_regmap *const g12a_clk_regmaps[] = {
&g12a_vdec_hevcf_sel,
&g12a_vdec_hevcf_div,
&g12a_vdec_hevcf,
-};
-
-static const struct meson_eeclkc_data g12a_clkc_data = {
- .regmap_clks = g12a_clk_regmaps,
- .regmap_clk_num = ARRAY_SIZE(g12a_clk_regmaps),
- .hw_onecell_data = &g12a_hw_onecell_data
+ &g12a_ts_div,
+ &g12a_ts,
+ &g12b_cpu_clk,
+ &g12b_sys1_pll_dco,
+ &g12b_sys1_pll,
+ &g12b_sys1_pll_div16_en,
+ &g12b_cpub_clk_premux0,
+ &g12b_cpub_clk_mux0_div,
+ &g12b_cpub_clk_postmux0,
+ &g12b_cpub_clk_premux1,
+ &g12b_cpub_clk_mux1_div,
+ &g12b_cpub_clk_postmux1,
+ &g12b_cpub_clk_dyn,
+ &g12b_cpub_clk,
+ &g12b_cpub_clk_div16_en,
+ &g12b_cpub_clk_apb_sel,
+ &g12b_cpub_clk_apb,
+ &g12b_cpub_clk_atb_sel,
+ &g12b_cpub_clk_atb,
+ &g12b_cpub_clk_axi_sel,
+ &g12b_cpub_clk_axi,
+ &g12b_cpub_clk_trace_sel,
+ &g12b_cpub_clk_trace,
+};
+
+static const struct reg_sequence g12a_init_regs[] = {
+ { .reg = HHI_MPLL_CNTL0, .def = 0x00000543 },
+};
+
+static int meson_g12a_dvfs_setup_common(struct platform_device *pdev,
+ struct clk_hw **hws)
+{
+ const char *notifier_clk_name;
+ struct clk *notifier_clk;
+ struct clk_hw *xtal;
+ int ret;
+
+ xtal = clk_hw_get_parent_by_index(hws[CLKID_CPU_CLK_DYN1_SEL], 0);
+
+ /* Setup clock notifier for cpu_clk_postmux0 */
+ g12a_cpu_clk_postmux0_nb_data.xtal = xtal;
+ notifier_clk_name = clk_hw_get_name(&g12a_cpu_clk_postmux0.hw);
+ notifier_clk = __clk_lookup(notifier_clk_name);
+ ret = clk_notifier_register(notifier_clk,
+ &g12a_cpu_clk_postmux0_nb_data.nb);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to register the cpu_clk_postmux0 notifier\n");
+ return ret;
+ }
+
+ /* Setup clock notifier for cpu_clk_dyn mux */
+ notifier_clk_name = clk_hw_get_name(&g12a_cpu_clk_dyn.hw);
+ notifier_clk = __clk_lookup(notifier_clk_name);
+ ret = clk_notifier_register(notifier_clk, &g12a_cpu_clk_mux_nb);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to register the cpu_clk_dyn notifier\n");
+ return ret;
+ }
+
+ return 0;
+}
+
+static int meson_g12b_dvfs_setup(struct platform_device *pdev)
+{
+ struct clk_hw **hws = g12b_hw_onecell_data.hws;
+ const char *notifier_clk_name;
+ struct clk *notifier_clk;
+ struct clk_hw *xtal;
+ int ret;
+
+ ret = meson_g12a_dvfs_setup_common(pdev, hws);
+ if (ret)
+ return ret;
+
+ xtal = clk_hw_get_parent_by_index(hws[CLKID_CPU_CLK_DYN1_SEL], 0);
+
+ /* Setup clock notifier for cpu_clk mux */
+ notifier_clk_name = clk_hw_get_name(&g12b_cpu_clk.hw);
+ notifier_clk = __clk_lookup(notifier_clk_name);
+ ret = clk_notifier_register(notifier_clk, &g12a_cpu_clk_mux_nb);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to register the cpu_clk notifier\n");
+ return ret;
+ }
+
+ /* Setup clock notifier for sys1_pll */
+ notifier_clk_name = clk_hw_get_name(&g12b_sys1_pll.hw);
+ notifier_clk = __clk_lookup(notifier_clk_name);
+ ret = clk_notifier_register(notifier_clk,
+ &g12b_cpu_clk_sys1_pll_nb_data.nb);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to register the sys1_pll notifier\n");
+ return ret;
+ }
+
+ /* Add notifiers for the second CPU cluster */
+
+ /* Setup clock notifier for cpub_clk_postmux0 */
+ g12b_cpub_clk_postmux0_nb_data.xtal = xtal;
+ notifier_clk_name = clk_hw_get_name(&g12b_cpub_clk_postmux0.hw);
+ notifier_clk = __clk_lookup(notifier_clk_name);
+ ret = clk_notifier_register(notifier_clk,
+ &g12b_cpub_clk_postmux0_nb_data.nb);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to register the cpub_clk_postmux0 notifier\n");
+ return ret;
+ }
+
+ /* Setup clock notifier for cpub_clk_dyn mux */
+ notifier_clk_name = clk_hw_get_name(&g12b_cpub_clk_dyn.hw);
+ notifier_clk = __clk_lookup(notifier_clk_name);
+ ret = clk_notifier_register(notifier_clk, &g12a_cpu_clk_mux_nb);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to register the cpub_clk_dyn notifier\n");
+ return ret;
+ }
+
+ /* Setup clock notifier for cpub_clk mux */
+ notifier_clk_name = clk_hw_get_name(&g12b_cpub_clk.hw);
+ notifier_clk = __clk_lookup(notifier_clk_name);
+ ret = clk_notifier_register(notifier_clk, &g12a_cpu_clk_mux_nb);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to register the cpub_clk notifier\n");
+ return ret;
+ }
+
+ /* Setup clock notifier for sys_pll */
+ notifier_clk_name = clk_hw_get_name(&g12a_sys_pll.hw);
+ notifier_clk = __clk_lookup(notifier_clk_name);
+ ret = clk_notifier_register(notifier_clk,
+ &g12b_cpub_clk_sys_pll_nb_data.nb);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to register the sys_pll notifier\n");
+ return ret;
+ }
+
+ return 0;
+}
+
+static int meson_g12a_dvfs_setup(struct platform_device *pdev)
+{
+ struct clk_hw **hws = g12a_hw_onecell_data.hws;
+ const char *notifier_clk_name;
+ struct clk *notifier_clk;
+ int ret;
+
+ ret = meson_g12a_dvfs_setup_common(pdev, hws);
+ if (ret)
+ return ret;
+
+ /* Setup clock notifier for cpu_clk mux */
+ notifier_clk_name = clk_hw_get_name(&g12a_cpu_clk.hw);
+ notifier_clk = __clk_lookup(notifier_clk_name);
+ ret = clk_notifier_register(notifier_clk, &g12a_cpu_clk_mux_nb);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to register the cpu_clk notifier\n");
+ return ret;
+ }
+
+ /* Setup clock notifier for sys_pll */
+ notifier_clk_name = clk_hw_get_name(&g12a_sys_pll.hw);
+ notifier_clk = __clk_lookup(notifier_clk_name);
+ ret = clk_notifier_register(notifier_clk, &g12a_sys_pll_nb_data.nb);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to register the sys_pll notifier\n");
+ return ret;
+ }
+
+ return 0;
+}
+
+struct meson_g12a_data {
+ const struct meson_eeclkc_data eeclkc_data;
+ int (*dvfs_setup)(struct platform_device *pdev);
+};
+
+static int meson_g12a_probe(struct platform_device *pdev)
+{
+ const struct meson_eeclkc_data *eeclkc_data;
+ const struct meson_g12a_data *g12a_data;
+ int ret;
+
+ eeclkc_data = of_device_get_match_data(&pdev->dev);
+ if (!eeclkc_data)
+ return -EINVAL;
+
+ ret = meson_eeclkc_probe(pdev);
+ if (ret)
+ return ret;
+
+ g12a_data = container_of(eeclkc_data, struct meson_g12a_data,
+ eeclkc_data);
+
+ if (g12a_data->dvfs_setup)
+ return g12a_data->dvfs_setup(pdev);
+
+ return 0;
+}
+
+static const struct meson_g12a_data g12a_clkc_data = {
+ .eeclkc_data = {
+ .regmap_clks = g12a_clk_regmaps,
+ .regmap_clk_num = ARRAY_SIZE(g12a_clk_regmaps),
+ .hw_onecell_data = &g12a_hw_onecell_data,
+ .init_regs = g12a_init_regs,
+ .init_count = ARRAY_SIZE(g12a_init_regs),
+ },
+ .dvfs_setup = meson_g12a_dvfs_setup,
+};
+
+static const struct meson_g12a_data g12b_clkc_data = {
+ .eeclkc_data = {
+ .regmap_clks = g12a_clk_regmaps,
+ .regmap_clk_num = ARRAY_SIZE(g12a_clk_regmaps),
+ .hw_onecell_data = &g12b_hw_onecell_data,
+ },
+ .dvfs_setup = meson_g12b_dvfs_setup,
};
static const struct of_device_id clkc_match_table[] = {
- { .compatible = "amlogic,g12a-clkc", .data = &g12a_clkc_data },
+ {
+ .compatible = "amlogic,g12a-clkc",
+ .data = &g12a_clkc_data.eeclkc_data
+ },
+ {
+ .compatible = "amlogic,g12b-clkc",
+ .data = &g12b_clkc_data.eeclkc_data
+ },
{}
};
static struct platform_driver g12a_driver = {
- .probe = meson_eeclkc_probe,
+ .probe = meson_g12a_probe,
.driver = {
.name = "g12a-clkc",
.of_match_table = clkc_match_table,
diff --git a/drivers/clk/meson/g12a.h b/drivers/clk/meson/g12a.h
index bcc05cd9882f..559a34cfdfeb 100644
--- a/drivers/clk/meson/g12a.h
+++ b/drivers/clk/meson/g12a.h
@@ -69,6 +69,8 @@
#define HHI_VDEC4_CLK_CNTL 0x1EC
#define HHI_HDCP22_CLK_CNTL 0x1F0
#define HHI_VAPBCLK_CNTL 0x1F4
+#define HHI_SYS_CPUB_CLK_CNTL1 0x200
+#define HHI_SYS_CPUB_CLK_CNTL 0x208
#define HHI_VPU_CLKB_CNTL 0x20C
#define HHI_GEN_CLK_CNTL 0x228
#define HHI_VDIN_MEAS_CLK_CNTL 0x250
@@ -102,6 +104,13 @@
#define HHI_HDMI_PLL_CNTL5 0x334
#define HHI_HDMI_PLL_CNTL6 0x338
#define HHI_SPICC_CLK_CNTL 0x3dc
+#define HHI_SYS1_PLL_CNTL0 0x380
+#define HHI_SYS1_PLL_CNTL1 0x384
+#define HHI_SYS1_PLL_CNTL2 0x388
+#define HHI_SYS1_PLL_CNTL3 0x38c
+#define HHI_SYS1_PLL_CNTL4 0x390
+#define HHI_SYS1_PLL_CNTL5 0x394
+#define HHI_SYS1_PLL_CNTL6 0x398
/*
* CLKID index values
@@ -195,8 +204,37 @@
#define CLKID_VDEC_HEVC_DIV 206
#define CLKID_VDEC_HEVCF_SEL 208
#define CLKID_VDEC_HEVCF_DIV 209
+#define CLKID_TS_DIV 211
+#define CLKID_SYS1_PLL_DCO 213
+#define CLKID_SYS1_PLL 214
+#define CLKID_SYS1_PLL_DIV16_EN 215
+#define CLKID_SYS1_PLL_DIV16 216
+#define CLKID_CPUB_CLK_DYN0_SEL 217
+#define CLKID_CPUB_CLK_DYN0_DIV 218
+#define CLKID_CPUB_CLK_DYN0 219
+#define CLKID_CPUB_CLK_DYN1_SEL 220
+#define CLKID_CPUB_CLK_DYN1_DIV 221
+#define CLKID_CPUB_CLK_DYN1 222
+#define CLKID_CPUB_CLK_DYN 223
+#define CLKID_CPUB_CLK_DIV16_EN 225
+#define CLKID_CPUB_CLK_DIV16 226
+#define CLKID_CPUB_CLK_DIV2 227
+#define CLKID_CPUB_CLK_DIV3 228
+#define CLKID_CPUB_CLK_DIV4 229
+#define CLKID_CPUB_CLK_DIV5 230
+#define CLKID_CPUB_CLK_DIV6 231
+#define CLKID_CPUB_CLK_DIV7 232
+#define CLKID_CPUB_CLK_DIV8 233
+#define CLKID_CPUB_CLK_APB_SEL 234
+#define CLKID_CPUB_CLK_APB 235
+#define CLKID_CPUB_CLK_ATB_SEL 236
+#define CLKID_CPUB_CLK_ATB 237
+#define CLKID_CPUB_CLK_AXI_SEL 238
+#define CLKID_CPUB_CLK_AXI 239
+#define CLKID_CPUB_CLK_TRACE_SEL 240
+#define CLKID_CPUB_CLK_TRACE 241
-#define NR_CLKS 211
+#define NR_CLKS 242
/* include the CLKIDs that have been made part of the DT binding */
#include <dt-bindings/clock/g12a-clkc.h>
diff --git a/drivers/clk/meson/gxbb-aoclk.c b/drivers/clk/meson/gxbb-aoclk.c
index 449f6ac189d8..e940861a396b 100644
--- a/drivers/clk/meson/gxbb-aoclk.c
+++ b/drivers/clk/meson/gxbb-aoclk.c
@@ -11,8 +11,6 @@
#include "clk-regmap.h"
#include "clk-dualdiv.h"
-#define IN_PREFIX "ao-in-"
-
/* AO Configuration Clock registers offsets */
#define AO_RTI_PWR_CNTL_REG1 0x0c
#define AO_RTI_PWR_CNTL_REG0 0x10
@@ -31,7 +29,9 @@ static struct clk_regmap _name##_ao = { \
.hw.init = &(struct clk_init_data) { \
.name = #_name "_ao", \
.ops = &clk_regmap_gate_ops, \
- .parent_names = (const char *[]){ IN_PREFIX "mpeg-clk" }, \
+ .parent_data = &(const struct clk_parent_data) { \
+ .fw_name = "mpeg-clk", \
+ }, \
.num_parents = 1, \
.flags = CLK_IGNORE_UNUSED, \
}, \
@@ -52,7 +52,9 @@ static struct clk_regmap ao_cts_oscin = {
.hw.init = &(struct clk_init_data){
.name = "ao_cts_oscin",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
},
};
@@ -65,7 +67,7 @@ static struct clk_regmap ao_32k_pre = {
.hw.init = &(struct clk_init_data){
.name = "ao_32k_pre",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "ao_cts_oscin" },
+ .parent_hws = (const struct clk_hw *[]) { &ao_cts_oscin.hw },
.num_parents = 1,
},
};
@@ -112,7 +114,7 @@ static struct clk_regmap ao_32k_div = {
.hw.init = &(struct clk_init_data){
.name = "ao_32k_div",
.ops = &meson_clk_dualdiv_ops,
- .parent_names = (const char *[]){ "ao_32k_pre" },
+ .parent_hws = (const struct clk_hw *[]) { &ao_32k_pre.hw },
.num_parents = 1,
},
};
@@ -127,8 +129,10 @@ static struct clk_regmap ao_32k_sel = {
.hw.init = &(struct clk_init_data){
.name = "ao_32k_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "ao_32k_div",
- "ao_32k_pre" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &ao_32k_div.hw,
+ &ao_32k_pre.hw
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_PARENT,
},
@@ -142,7 +146,7 @@ static struct clk_regmap ao_32k = {
.hw.init = &(struct clk_init_data){
.name = "ao_32k",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "ao_32k_sel" },
+ .parent_hws = (const struct clk_hw *[]) { &ao_32k_sel.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -159,10 +163,12 @@ static struct clk_regmap ao_cts_rtc_oscin = {
.hw.init = &(struct clk_init_data){
.name = "ao_cts_rtc_oscin",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ IN_PREFIX "ext-32k-0",
- IN_PREFIX "ext-32k-1",
- IN_PREFIX "ext-32k-2",
- "ao_32k" },
+ .parent_data = (const struct clk_parent_data []) {
+ { .fw_name = "ext-32k-0", },
+ { .fw_name = "ext-32k-1", },
+ { .fw_name = "ext-32k-2", },
+ { .hw = &ao_32k.hw },
+ },
.num_parents = 4,
.flags = CLK_SET_RATE_PARENT,
},
@@ -178,8 +184,10 @@ static struct clk_regmap ao_clk81 = {
.hw.init = &(struct clk_init_data){
.name = "ao_clk81",
.ops = &clk_regmap_mux_ro_ops,
- .parent_names = (const char *[]){ IN_PREFIX "mpeg-clk",
- "ao_cts_rtc_oscin" },
+ .parent_data = (const struct clk_parent_data []) {
+ { .fw_name = "mpeg-clk", },
+ { .hw = &ao_cts_rtc_oscin.hw },
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_PARENT,
},
@@ -208,8 +216,10 @@ static struct clk_regmap ao_cts_cec = {
* Until CCF gets fixed, adding this fake parent that won't
* ever be registered should work around the problem
*/
- .parent_names = (const char *[]){ "fixme",
- "ao_cts_rtc_oscin" },
+ .parent_data = (const struct clk_parent_data []) {
+ { .name = "fixme", .index = -1, },
+ { .hw = &ao_cts_rtc_oscin.hw },
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_PARENT,
},
@@ -261,14 +271,6 @@ static const struct clk_hw_onecell_data gxbb_aoclk_onecell_data = {
.num = NR_CLKS,
};
-static const struct meson_aoclk_input gxbb_aoclk_inputs[] = {
- { .name = "xtal", .required = true, },
- { .name = "mpeg-clk", .required = true, },
- {. name = "ext-32k-0", .required = false, },
- {. name = "ext-32k-1", .required = false, },
- {. name = "ext-32k-2", .required = false, },
-};
-
static const struct meson_aoclk_data gxbb_aoclkc_data = {
.reset_reg = AO_RTI_GEN_CNTL_REG0,
.num_reset = ARRAY_SIZE(gxbb_aoclk_reset),
@@ -276,9 +278,6 @@ static const struct meson_aoclk_data gxbb_aoclkc_data = {
.num_clks = ARRAY_SIZE(gxbb_aoclk),
.clks = gxbb_aoclk,
.hw_data = &gxbb_aoclk_onecell_data,
- .inputs = gxbb_aoclk_inputs,
- .num_inputs = ARRAY_SIZE(gxbb_aoclk_inputs),
- .input_prefix = IN_PREFIX,
};
static const struct of_device_id gxbb_aoclkc_match_table[] = {
diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c
index 29ffb4fde714..7cfb998eeb3e 100644
--- a/drivers/clk/meson/gxbb.c
+++ b/drivers/clk/meson/gxbb.c
@@ -10,15 +10,12 @@
#include <linux/platform_device.h>
#include "gxbb.h"
-#include "clk-input.h"
#include "clk-regmap.h"
#include "clk-pll.h"
#include "clk-mpll.h"
#include "meson-eeclk.h"
#include "vid-pll-div.h"
-#define IN_PREFIX "ee-in-"
-
static DEFINE_SPINLOCK(meson_clk_lock);
static const struct pll_params_table gxbb_gp0_pll_params_table[] = {
@@ -121,7 +118,9 @@ static struct clk_regmap gxbb_fixed_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "fixed_pll_dco",
.ops = &meson_clk_pll_ro_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
},
};
@@ -136,7 +135,9 @@ static struct clk_regmap gxbb_fixed_pll = {
.hw.init = &(struct clk_init_data){
.name = "fixed_pll",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "fixed_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_fixed_pll_dco.hw
+ },
.num_parents = 1,
/*
* This clock won't ever change at runtime so
@@ -151,7 +152,9 @@ static struct clk_fixed_factor gxbb_hdmi_pll_pre_mult = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_pll_pre_mult",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
},
};
@@ -192,7 +195,9 @@ static struct clk_regmap gxbb_hdmi_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_pll_dco",
.ops = &meson_clk_pll_ro_ops,
- .parent_names = (const char *[]){ "hdmi_pll_pre_mult" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_hdmi_pll_pre_mult.hw
+ },
.num_parents = 1,
/*
* Display directly handle hdmi pll registers ATM, we need
@@ -244,7 +249,9 @@ static struct clk_regmap gxl_hdmi_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_pll_dco",
.ops = &meson_clk_pll_ro_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
/*
* Display directly handle hdmi pll registers ATM, we need
@@ -264,7 +271,9 @@ static struct clk_regmap gxbb_hdmi_pll_od = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_pll_od",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "hdmi_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_hdmi_pll_dco.hw
+ },
.num_parents = 1,
.flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT,
},
@@ -280,7 +289,9 @@ static struct clk_regmap gxbb_hdmi_pll_od2 = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_pll_od2",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "hdmi_pll_od" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_hdmi_pll_od.hw
+ },
.num_parents = 1,
.flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT,
},
@@ -296,7 +307,9 @@ static struct clk_regmap gxbb_hdmi_pll = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_pll",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "hdmi_pll_od2" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_hdmi_pll_od2.hw
+ },
.num_parents = 1,
.flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT,
},
@@ -312,7 +325,9 @@ static struct clk_regmap gxl_hdmi_pll_od = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_pll_od",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "hdmi_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxl_hdmi_pll_dco.hw
+ },
.num_parents = 1,
.flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT,
},
@@ -328,7 +343,9 @@ static struct clk_regmap gxl_hdmi_pll_od2 = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_pll_od2",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "hdmi_pll_od" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxl_hdmi_pll_od.hw
+ },
.num_parents = 1,
.flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT,
},
@@ -344,7 +361,9 @@ static struct clk_regmap gxl_hdmi_pll = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_pll",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "hdmi_pll_od2" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxl_hdmi_pll_od2.hw
+ },
.num_parents = 1,
.flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT,
},
@@ -381,7 +400,9 @@ static struct clk_regmap gxbb_sys_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "sys_pll_dco",
.ops = &meson_clk_pll_ro_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
},
};
@@ -396,7 +417,9 @@ static struct clk_regmap gxbb_sys_pll = {
.hw.init = &(struct clk_init_data){
.name = "sys_pll",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "sys_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_sys_pll_dco.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -442,7 +465,9 @@ static struct clk_regmap gxbb_gp0_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "gp0_pll_dco",
.ops = &meson_clk_pll_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
},
};
@@ -494,7 +519,9 @@ static struct clk_regmap gxl_gp0_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "gp0_pll_dco",
.ops = &meson_clk_pll_ops,
- .parent_names = (const char *[]){ IN_PREFIX "xtal" },
+ .parent_data = &(const struct clk_parent_data) {
+ .fw_name = "xtal",
+ },
.num_parents = 1,
},
};
@@ -509,7 +536,17 @@ static struct clk_regmap gxbb_gp0_pll = {
.hw.init = &(struct clk_init_data){
.name = "gp0_pll",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "gp0_pll_dco" },
+ .parent_data = &(const struct clk_parent_data) {
+ /*
+ * Note:
+ * GXL and GXBB have different gp0_pll_dco (with
+ * different struct clk_hw). We fallback to the global
+ * naming string mechanism so gp0_pll picks up the
+ * appropriate one.
+ */
+ .name = "gp0_pll_dco",
+ .index = -1,
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -521,7 +558,9 @@ static struct clk_fixed_factor gxbb_fclk_div2_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div2_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_fixed_pll.hw
+ },
.num_parents = 1,
},
};
@@ -534,7 +573,9 @@ static struct clk_regmap gxbb_fclk_div2 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div2",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div2_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_fclk_div2_div.hw
+ },
.num_parents = 1,
.flags = CLK_IS_CRITICAL,
},
@@ -546,7 +587,7 @@ static struct clk_fixed_factor gxbb_fclk_div3_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div3_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_fixed_pll.hw },
.num_parents = 1,
},
};
@@ -559,7 +600,9 @@ static struct clk_regmap gxbb_fclk_div3 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div3",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div3_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_fclk_div3_div.hw
+ },
.num_parents = 1,
/*
* FIXME:
@@ -582,7 +625,7 @@ static struct clk_fixed_factor gxbb_fclk_div4_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div4_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_fixed_pll.hw },
.num_parents = 1,
},
};
@@ -595,7 +638,9 @@ static struct clk_regmap gxbb_fclk_div4 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div4",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div4_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_fclk_div4_div.hw
+ },
.num_parents = 1,
},
};
@@ -606,7 +651,7 @@ static struct clk_fixed_factor gxbb_fclk_div5_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div5_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_fixed_pll.hw },
.num_parents = 1,
},
};
@@ -619,7 +664,9 @@ static struct clk_regmap gxbb_fclk_div5 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div5",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div5_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_fclk_div5_div.hw
+ },
.num_parents = 1,
},
};
@@ -630,7 +677,7 @@ static struct clk_fixed_factor gxbb_fclk_div7_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div7_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_fixed_pll.hw },
.num_parents = 1,
},
};
@@ -643,7 +690,9 @@ static struct clk_regmap gxbb_fclk_div7 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div7",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div7_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_fclk_div7_div.hw
+ },
.num_parents = 1,
},
};
@@ -657,7 +706,7 @@ static struct clk_regmap gxbb_mpll_prediv = {
.hw.init = &(struct clk_init_data){
.name = "mpll_prediv",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_fixed_pll.hw },
.num_parents = 1,
},
};
@@ -679,17 +728,14 @@ static struct clk_regmap gxbb_mpll0_div = {
.shift = 16,
.width = 9,
},
- .ssen = {
- .reg_off = HHI_MPLL_CNTL,
- .shift = 25,
- .width = 1,
- },
.lock = &meson_clk_lock,
},
.hw.init = &(struct clk_init_data){
.name = "mpll0_div",
.ops = &meson_clk_mpll_ops,
- .parent_names = (const char *[]){ "mpll_prediv" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_mpll_prediv.hw
+ },
.num_parents = 1,
},
};
@@ -702,7 +748,7 @@ static struct clk_regmap gxbb_mpll0 = {
.hw.init = &(struct clk_init_data){
.name = "mpll0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpll0_div" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_mpll0_div.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -730,7 +776,9 @@ static struct clk_regmap gxbb_mpll1_div = {
.hw.init = &(struct clk_init_data){
.name = "mpll1_div",
.ops = &meson_clk_mpll_ops,
- .parent_names = (const char *[]){ "mpll_prediv" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_mpll_prediv.hw
+ },
.num_parents = 1,
},
};
@@ -743,7 +791,7 @@ static struct clk_regmap gxbb_mpll1 = {
.hw.init = &(struct clk_init_data){
.name = "mpll1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpll1_div" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_mpll1_div.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -771,7 +819,9 @@ static struct clk_regmap gxbb_mpll2_div = {
.hw.init = &(struct clk_init_data){
.name = "mpll2_div",
.ops = &meson_clk_mpll_ops,
- .parent_names = (const char *[]){ "mpll_prediv" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_mpll_prediv.hw
+ },
.num_parents = 1,
},
};
@@ -784,16 +834,21 @@ static struct clk_regmap gxbb_mpll2 = {
.hw.init = &(struct clk_init_data){
.name = "mpll2",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpll2_div" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_mpll2_div.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
};
static u32 mux_table_clk81[] = { 0, 2, 3, 4, 5, 6, 7 };
-static const char * const clk81_parent_names[] = {
- IN_PREFIX "xtal", "fclk_div7", "mpll1", "mpll2", "fclk_div4",
- "fclk_div3", "fclk_div5"
+static const struct clk_parent_data clk81_parent_data[] = {
+ { .fw_name = "xtal", },
+ { .hw = &gxbb_fclk_div7.hw },
+ { .hw = &gxbb_mpll1.hw },
+ { .hw = &gxbb_mpll2.hw },
+ { .hw = &gxbb_fclk_div4.hw },
+ { .hw = &gxbb_fclk_div3.hw },
+ { .hw = &gxbb_fclk_div5.hw },
};
static struct clk_regmap gxbb_mpeg_clk_sel = {
@@ -811,8 +866,8 @@ static struct clk_regmap gxbb_mpeg_clk_sel = {
* xtal, 1'b0 (wtf), fclk_div7, mpll_clkout1, mpll_clkout2,
* fclk_div4, fclk_div3, fclk_div5
*/
- .parent_names = clk81_parent_names,
- .num_parents = ARRAY_SIZE(clk81_parent_names),
+ .parent_data = clk81_parent_data,
+ .num_parents = ARRAY_SIZE(clk81_parent_data),
},
};
@@ -825,7 +880,9 @@ static struct clk_regmap gxbb_mpeg_clk_div = {
.hw.init = &(struct clk_init_data){
.name = "mpeg_clk_div",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "mpeg_clk_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_mpeg_clk_sel.hw
+ },
.num_parents = 1,
},
};
@@ -839,7 +896,9 @@ static struct clk_regmap gxbb_clk81 = {
.hw.init = &(struct clk_init_data){
.name = "clk81",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpeg_clk_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_mpeg_clk_div.hw
+ },
.num_parents = 1,
.flags = CLK_IS_CRITICAL,
},
@@ -855,7 +914,10 @@ static struct clk_regmap gxbb_sar_adc_clk_sel = {
.name = "sar_adc_clk_sel",
.ops = &clk_regmap_mux_ops,
/* NOTE: The datasheet doesn't list the parents for bit 10 */
- .parent_names = (const char *[]){ IN_PREFIX "xtal", "clk81", },
+ .parent_data = (const struct clk_parent_data []) {
+ { .fw_name = "xtal", },
+ { .hw = &gxbb_clk81.hw },
+ },
.num_parents = 2,
},
};
@@ -869,7 +931,9 @@ static struct clk_regmap gxbb_sar_adc_clk_div = {
.hw.init = &(struct clk_init_data){
.name = "sar_adc_clk_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "sar_adc_clk_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_sar_adc_clk_sel.hw
+ },
.num_parents = 1,
},
};
@@ -882,7 +946,9 @@ static struct clk_regmap gxbb_sar_adc_clk = {
.hw.init = &(struct clk_init_data){
.name = "sar_adc_clk",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "sar_adc_clk_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_sar_adc_clk_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -893,9 +959,15 @@ static struct clk_regmap gxbb_sar_adc_clk = {
* muxed by a glitch-free switch.
*/
-static const char * const gxbb_mali_0_1_parent_names[] = {
- IN_PREFIX "xtal", "gp0_pll", "mpll2", "mpll1", "fclk_div7",
- "fclk_div4", "fclk_div3", "fclk_div5"
+static const struct clk_parent_data gxbb_mali_0_1_parent_data[] = {
+ { .fw_name = "xtal", },
+ { .hw = &gxbb_gp0_pll.hw },
+ { .hw = &gxbb_mpll2.hw },
+ { .hw = &gxbb_mpll1.hw },
+ { .hw = &gxbb_fclk_div7.hw },
+ { .hw = &gxbb_fclk_div4.hw },
+ { .hw = &gxbb_fclk_div3.hw },
+ { .hw = &gxbb_fclk_div5.hw },
};
static struct clk_regmap gxbb_mali_0_sel = {
@@ -912,7 +984,7 @@ static struct clk_regmap gxbb_mali_0_sel = {
* xtal, gp0_pll, mpll2, mpll1, fclk_div7,
* fclk_div4, fclk_div3, fclk_div5
*/
- .parent_names = gxbb_mali_0_1_parent_names,
+ .parent_data = gxbb_mali_0_1_parent_data,
.num_parents = 8,
.flags = CLK_SET_RATE_NO_REPARENT,
},
@@ -927,7 +999,9 @@ static struct clk_regmap gxbb_mali_0_div = {
.hw.init = &(struct clk_init_data){
.name = "mali_0_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "mali_0_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_mali_0_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_NO_REPARENT,
},
@@ -941,7 +1015,9 @@ static struct clk_regmap gxbb_mali_0 = {
.hw.init = &(struct clk_init_data){
.name = "mali_0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mali_0_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_mali_0_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -961,7 +1037,7 @@ static struct clk_regmap gxbb_mali_1_sel = {
* xtal, gp0_pll, mpll2, mpll1, fclk_div7,
* fclk_div4, fclk_div3, fclk_div5
*/
- .parent_names = gxbb_mali_0_1_parent_names,
+ .parent_data = gxbb_mali_0_1_parent_data,
.num_parents = 8,
.flags = CLK_SET_RATE_NO_REPARENT,
},
@@ -976,7 +1052,9 @@ static struct clk_regmap gxbb_mali_1_div = {
.hw.init = &(struct clk_init_data){
.name = "mali_1_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "mali_1_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_mali_1_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_NO_REPARENT,
},
@@ -990,14 +1068,17 @@ static struct clk_regmap gxbb_mali_1 = {
.hw.init = &(struct clk_init_data){
.name = "mali_1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mali_1_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_mali_1_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
};
-static const char * const gxbb_mali_parent_names[] = {
- "mali_0", "mali_1"
+static const struct clk_hw *gxbb_mali_parent_hws[] = {
+ &gxbb_mali_0.hw,
+ &gxbb_mali_1.hw,
};
static struct clk_regmap gxbb_mali = {
@@ -1009,7 +1090,7 @@ static struct clk_regmap gxbb_mali = {
.hw.init = &(struct clk_init_data){
.name = "mali",
.ops = &clk_regmap_mux_ops,
- .parent_names = gxbb_mali_parent_names,
+ .parent_hws = gxbb_mali_parent_hws,
.num_parents = 2,
.flags = CLK_SET_RATE_NO_REPARENT,
},
@@ -1026,7 +1107,11 @@ static struct clk_regmap gxbb_cts_amclk_sel = {
.hw.init = &(struct clk_init_data){
.name = "cts_amclk_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "mpll0", "mpll1", "mpll2" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_mpll0.hw,
+ &gxbb_mpll1.hw,
+ &gxbb_mpll2.hw,
+ },
.num_parents = 3,
},
};
@@ -1041,7 +1126,9 @@ static struct clk_regmap gxbb_cts_amclk_div = {
.hw.init = &(struct clk_init_data){
.name = "cts_amclk_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "cts_amclk_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_cts_amclk_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1055,7 +1142,9 @@ static struct clk_regmap gxbb_cts_amclk = {
.hw.init = &(struct clk_init_data){
.name = "cts_amclk",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "cts_amclk_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_cts_amclk_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1072,7 +1161,11 @@ static struct clk_regmap gxbb_cts_mclk_i958_sel = {
.hw.init = &(struct clk_init_data) {
.name = "cts_mclk_i958_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "mpll0", "mpll1", "mpll2" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_mpll0.hw,
+ &gxbb_mpll1.hw,
+ &gxbb_mpll2.hw,
+ },
.num_parents = 3,
},
};
@@ -1087,7 +1180,9 @@ static struct clk_regmap gxbb_cts_mclk_i958_div = {
.hw.init = &(struct clk_init_data) {
.name = "cts_mclk_i958_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "cts_mclk_i958_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_cts_mclk_i958_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1101,7 +1196,9 @@ static struct clk_regmap gxbb_cts_mclk_i958 = {
.hw.init = &(struct clk_init_data){
.name = "cts_mclk_i958",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "cts_mclk_i958_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_cts_mclk_i958_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1116,7 +1213,10 @@ static struct clk_regmap gxbb_cts_i958 = {
.hw.init = &(struct clk_init_data){
.name = "cts_i958",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "cts_amclk", "cts_mclk_i958" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_cts_amclk.hw,
+ &gxbb_cts_mclk_i958.hw
+ },
.num_parents = 2,
/*
*The parent is specific to origin of the audio data. Let the
@@ -1126,6 +1226,33 @@ static struct clk_regmap gxbb_cts_i958 = {
},
};
+static const struct clk_parent_data gxbb_32k_clk_parent_data[] = {
+ { .fw_name = "xtal", },
+ /*
+ * FIXME: This clock is provided by the ao clock controller but the
+ * clock is not yet part of the binding of this controller, so string
+ * name must be use to set this parent.
+ */
+ { .name = "cts_slow_oscin", .index = -1 },
+ { .hw = &gxbb_fclk_div3.hw },
+ { .hw = &gxbb_fclk_div5.hw },
+};
+
+static struct clk_regmap gxbb_32k_clk_sel = {
+ .data = &(struct clk_regmap_mux_data){
+ .offset = HHI_32K_CLK_CNTL,
+ .mask = 0x3,
+ .shift = 16,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "32k_clk_sel",
+ .ops = &clk_regmap_mux_ops,
+ .parent_data = gxbb_32k_clk_parent_data,
+ .num_parents = 4,
+ .flags = CLK_SET_RATE_PARENT,
+ },
+};
+
static struct clk_regmap gxbb_32k_clk_div = {
.data = &(struct clk_regmap_div_data){
.offset = HHI_32K_CLK_CNTL,
@@ -1135,7 +1262,9 @@ static struct clk_regmap gxbb_32k_clk_div = {
.hw.init = &(struct clk_init_data){
.name = "32k_clk_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "32k_clk_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_32k_clk_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_DIVIDER_ROUND_CLOSEST,
},
@@ -1149,34 +1278,20 @@ static struct clk_regmap gxbb_32k_clk = {
.hw.init = &(struct clk_init_data){
.name = "32k_clk",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "32k_clk_div" },
- .num_parents = 1,
- .flags = CLK_SET_RATE_PARENT,
- },
-};
-
-static const char * const gxbb_32k_clk_parent_names[] = {
- IN_PREFIX "xtal", "cts_slow_oscin", "fclk_div3", "fclk_div5"
-};
-
-static struct clk_regmap gxbb_32k_clk_sel = {
- .data = &(struct clk_regmap_mux_data){
- .offset = HHI_32K_CLK_CNTL,
- .mask = 0x3,
- .shift = 16,
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_32k_clk_div.hw
},
- .hw.init = &(struct clk_init_data){
- .name = "32k_clk_sel",
- .ops = &clk_regmap_mux_ops,
- .parent_names = gxbb_32k_clk_parent_names,
- .num_parents = 4,
+ .num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
};
-static const char * const gxbb_sd_emmc_clk0_parent_names[] = {
- IN_PREFIX "xtal", "fclk_div2", "fclk_div3", "fclk_div5", "fclk_div7",
-
+static const struct clk_parent_data gxbb_sd_emmc_clk0_parent_data[] = {
+ { .fw_name = "xtal", },
+ { .hw = &gxbb_fclk_div2.hw },
+ { .hw = &gxbb_fclk_div3.hw },
+ { .hw = &gxbb_fclk_div5.hw },
+ { .hw = &gxbb_fclk_div7.hw },
/*
* Following these parent clocks, we should also have had mpll2, mpll3
* and gp0_pll but these clocks are too precious to be used here. All
@@ -1195,8 +1310,8 @@ static struct clk_regmap gxbb_sd_emmc_a_clk0_sel = {
.hw.init = &(struct clk_init_data) {
.name = "sd_emmc_a_clk0_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = gxbb_sd_emmc_clk0_parent_names,
- .num_parents = ARRAY_SIZE(gxbb_sd_emmc_clk0_parent_names),
+ .parent_data = gxbb_sd_emmc_clk0_parent_data,
+ .num_parents = ARRAY_SIZE(gxbb_sd_emmc_clk0_parent_data),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1211,7 +1326,9 @@ static struct clk_regmap gxbb_sd_emmc_a_clk0_div = {
.hw.init = &(struct clk_init_data) {
.name = "sd_emmc_a_clk0_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "sd_emmc_a_clk0_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_sd_emmc_a_clk0_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1225,7 +1342,9 @@ static struct clk_regmap gxbb_sd_emmc_a_clk0 = {
.hw.init = &(struct clk_init_data){
.name = "sd_emmc_a_clk0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "sd_emmc_a_clk0_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_sd_emmc_a_clk0_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1241,8 +1360,8 @@ static struct clk_regmap gxbb_sd_emmc_b_clk0_sel = {
.hw.init = &(struct clk_init_data) {
.name = "sd_emmc_b_clk0_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = gxbb_sd_emmc_clk0_parent_names,
- .num_parents = ARRAY_SIZE(gxbb_sd_emmc_clk0_parent_names),
+ .parent_data = gxbb_sd_emmc_clk0_parent_data,
+ .num_parents = ARRAY_SIZE(gxbb_sd_emmc_clk0_parent_data),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1257,7 +1376,9 @@ static struct clk_regmap gxbb_sd_emmc_b_clk0_div = {
.hw.init = &(struct clk_init_data) {
.name = "sd_emmc_b_clk0_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "sd_emmc_b_clk0_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_sd_emmc_b_clk0_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1271,7 +1392,9 @@ static struct clk_regmap gxbb_sd_emmc_b_clk0 = {
.hw.init = &(struct clk_init_data){
.name = "sd_emmc_b_clk0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "sd_emmc_b_clk0_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_sd_emmc_b_clk0_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1287,8 +1410,8 @@ static struct clk_regmap gxbb_sd_emmc_c_clk0_sel = {
.hw.init = &(struct clk_init_data) {
.name = "sd_emmc_c_clk0_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = gxbb_sd_emmc_clk0_parent_names,
- .num_parents = ARRAY_SIZE(gxbb_sd_emmc_clk0_parent_names),
+ .parent_data = gxbb_sd_emmc_clk0_parent_data,
+ .num_parents = ARRAY_SIZE(gxbb_sd_emmc_clk0_parent_data),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1303,7 +1426,9 @@ static struct clk_regmap gxbb_sd_emmc_c_clk0_div = {
.hw.init = &(struct clk_init_data) {
.name = "sd_emmc_c_clk0_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "sd_emmc_c_clk0_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_sd_emmc_c_clk0_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1317,7 +1442,9 @@ static struct clk_regmap gxbb_sd_emmc_c_clk0 = {
.hw.init = &(struct clk_init_data){
.name = "sd_emmc_c_clk0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "sd_emmc_c_clk0_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_sd_emmc_c_clk0_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1325,8 +1452,11 @@ static struct clk_regmap gxbb_sd_emmc_c_clk0 = {
/* VPU Clock */
-static const char * const gxbb_vpu_parent_names[] = {
- "fclk_div4", "fclk_div3", "fclk_div5", "fclk_div7"
+static const struct clk_hw *gxbb_vpu_parent_hws[] = {
+ &gxbb_fclk_div4.hw,
+ &gxbb_fclk_div3.hw,
+ &gxbb_fclk_div5.hw,
+ &gxbb_fclk_div7.hw,
};
static struct clk_regmap gxbb_vpu_0_sel = {
@@ -1342,8 +1472,8 @@ static struct clk_regmap gxbb_vpu_0_sel = {
* bits 9:10 selects from 4 possible parents:
* fclk_div4, fclk_div3, fclk_div5, fclk_div7,
*/
- .parent_names = gxbb_vpu_parent_names,
- .num_parents = ARRAY_SIZE(gxbb_vpu_parent_names),
+ .parent_hws = gxbb_vpu_parent_hws,
+ .num_parents = ARRAY_SIZE(gxbb_vpu_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT,
},
};
@@ -1357,7 +1487,7 @@ static struct clk_regmap gxbb_vpu_0_div = {
.hw.init = &(struct clk_init_data){
.name = "vpu_0_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vpu_0_sel" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vpu_0_sel.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1371,7 +1501,7 @@ static struct clk_regmap gxbb_vpu_0 = {
.hw.init = &(struct clk_init_data) {
.name = "vpu_0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vpu_0_div" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vpu_0_div.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1390,8 +1520,8 @@ static struct clk_regmap gxbb_vpu_1_sel = {
* bits 25:26 selects from 4 possible parents:
* fclk_div4, fclk_div3, fclk_div5, fclk_div7,
*/
- .parent_names = gxbb_vpu_parent_names,
- .num_parents = ARRAY_SIZE(gxbb_vpu_parent_names),
+ .parent_hws = gxbb_vpu_parent_hws,
+ .num_parents = ARRAY_SIZE(gxbb_vpu_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT,
},
};
@@ -1405,7 +1535,7 @@ static struct clk_regmap gxbb_vpu_1_div = {
.hw.init = &(struct clk_init_data){
.name = "vpu_1_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vpu_1_sel" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vpu_1_sel.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1419,7 +1549,7 @@ static struct clk_regmap gxbb_vpu_1 = {
.hw.init = &(struct clk_init_data) {
.name = "vpu_1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vpu_1_div" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vpu_1_div.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1438,7 +1568,10 @@ static struct clk_regmap gxbb_vpu = {
* bit 31 selects from 2 possible parents:
* vpu_0 or vpu_1
*/
- .parent_names = (const char *[]){ "vpu_0", "vpu_1" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vpu_0.hw,
+ &gxbb_vpu_1.hw
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_NO_REPARENT,
},
@@ -1446,8 +1579,11 @@ static struct clk_regmap gxbb_vpu = {
/* VAPB Clock */
-static const char * const gxbb_vapb_parent_names[] = {
- "fclk_div4", "fclk_div3", "fclk_div5", "fclk_div7"
+static const struct clk_hw *gxbb_vapb_parent_hws[] = {
+ &gxbb_fclk_div4.hw,
+ &gxbb_fclk_div3.hw,
+ &gxbb_fclk_div5.hw,
+ &gxbb_fclk_div7.hw,
};
static struct clk_regmap gxbb_vapb_0_sel = {
@@ -1463,8 +1599,8 @@ static struct clk_regmap gxbb_vapb_0_sel = {
* bits 9:10 selects from 4 possible parents:
* fclk_div4, fclk_div3, fclk_div5, fclk_div7,
*/
- .parent_names = gxbb_vapb_parent_names,
- .num_parents = ARRAY_SIZE(gxbb_vapb_parent_names),
+ .parent_hws = gxbb_vapb_parent_hws,
+ .num_parents = ARRAY_SIZE(gxbb_vapb_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT,
},
};
@@ -1478,7 +1614,9 @@ static struct clk_regmap gxbb_vapb_0_div = {
.hw.init = &(struct clk_init_data){
.name = "vapb_0_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vapb_0_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vapb_0_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1492,7 +1630,9 @@ static struct clk_regmap gxbb_vapb_0 = {
.hw.init = &(struct clk_init_data) {
.name = "vapb_0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vapb_0_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vapb_0_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1511,8 +1651,8 @@ static struct clk_regmap gxbb_vapb_1_sel = {
* bits 25:26 selects from 4 possible parents:
* fclk_div4, fclk_div3, fclk_div5, fclk_div7,
*/
- .parent_names = gxbb_vapb_parent_names,
- .num_parents = ARRAY_SIZE(gxbb_vapb_parent_names),
+ .parent_hws = gxbb_vapb_parent_hws,
+ .num_parents = ARRAY_SIZE(gxbb_vapb_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT,
},
};
@@ -1526,7 +1666,9 @@ static struct clk_regmap gxbb_vapb_1_div = {
.hw.init = &(struct clk_init_data){
.name = "vapb_1_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vapb_1_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vapb_1_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1540,7 +1682,9 @@ static struct clk_regmap gxbb_vapb_1 = {
.hw.init = &(struct clk_init_data) {
.name = "vapb_1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vapb_1_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vapb_1_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1559,7 +1703,10 @@ static struct clk_regmap gxbb_vapb_sel = {
* bit 31 selects from 2 possible parents:
* vapb_0 or vapb_1
*/
- .parent_names = (const char *[]){ "vapb_0", "vapb_1" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vapb_0.hw,
+ &gxbb_vapb_1.hw
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_NO_REPARENT,
},
@@ -1573,7 +1720,7 @@ static struct clk_regmap gxbb_vapb = {
.hw.init = &(struct clk_init_data) {
.name = "vapb",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vapb_sel" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vapb_sel.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1597,13 +1744,33 @@ static struct clk_regmap gxbb_vid_pll_div = {
.hw.init = &(struct clk_init_data) {
.name = "vid_pll_div",
.ops = &meson_vid_pll_div_ro_ops,
- .parent_names = (const char *[]){ "hdmi_pll" },
+ .parent_data = &(const struct clk_parent_data) {
+ /*
+ * Note:
+ * GXL and GXBB have different hdmi_plls (with
+ * different struct clk_hw). We fallback to the global
+ * naming string mechanism so vid_pll_div picks up the
+ * appropriate one.
+ */
+ .name = "hdmi_pll",
+ .index = -1,
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE,
},
};
-static const char * const gxbb_vid_pll_parent_names[] = { "vid_pll_div", "hdmi_pll" };
+static const struct clk_parent_data gxbb_vid_pll_parent_data[] = {
+ { .hw = &gxbb_vid_pll_div.hw },
+ /*
+ * Note:
+ * GXL and GXBB have different hdmi_plls (with
+ * different struct clk_hw). We fallback to the global
+ * naming string mechanism so vid_pll_div picks up the
+ * appropriate one.
+ */
+ { .name = "hdmi_pll", .index = -1 },
+};
static struct clk_regmap gxbb_vid_pll_sel = {
.data = &(struct clk_regmap_mux_data){
@@ -1618,8 +1785,8 @@ static struct clk_regmap gxbb_vid_pll_sel = {
* bit 18 selects from 2 possible parents:
* vid_pll_div or hdmi_pll
*/
- .parent_names = gxbb_vid_pll_parent_names,
- .num_parents = ARRAY_SIZE(gxbb_vid_pll_parent_names),
+ .parent_data = gxbb_vid_pll_parent_data,
+ .num_parents = ARRAY_SIZE(gxbb_vid_pll_parent_data),
.flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE,
},
};
@@ -1632,15 +1799,22 @@ static struct clk_regmap gxbb_vid_pll = {
.hw.init = &(struct clk_init_data) {
.name = "vid_pll",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vid_pll_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vid_pll_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
};
-static const char * const gxbb_vclk_parent_names[] = {
- "vid_pll", "fclk_div4", "fclk_div3", "fclk_div5", "vid_pll",
- "fclk_div7", "mpll1",
+static const struct clk_hw *gxbb_vclk_parent_hws[] = {
+ &gxbb_vid_pll.hw,
+ &gxbb_fclk_div4.hw,
+ &gxbb_fclk_div3.hw,
+ &gxbb_fclk_div5.hw,
+ &gxbb_vid_pll.hw,
+ &gxbb_fclk_div7.hw,
+ &gxbb_mpll1.hw,
};
static struct clk_regmap gxbb_vclk_sel = {
@@ -1657,8 +1831,8 @@ static struct clk_regmap gxbb_vclk_sel = {
* vid_pll, fclk_div4, fclk_div3, fclk_div5,
* vid_pll, fclk_div7, mp1
*/
- .parent_names = gxbb_vclk_parent_names,
- .num_parents = ARRAY_SIZE(gxbb_vclk_parent_names),
+ .parent_hws = gxbb_vclk_parent_hws,
+ .num_parents = ARRAY_SIZE(gxbb_vclk_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE,
},
};
@@ -1677,8 +1851,8 @@ static struct clk_regmap gxbb_vclk2_sel = {
* vid_pll, fclk_div4, fclk_div3, fclk_div5,
* vid_pll, fclk_div7, mp1
*/
- .parent_names = gxbb_vclk_parent_names,
- .num_parents = ARRAY_SIZE(gxbb_vclk_parent_names),
+ .parent_hws = gxbb_vclk_parent_hws,
+ .num_parents = ARRAY_SIZE(gxbb_vclk_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE,
},
};
@@ -1691,7 +1865,7 @@ static struct clk_regmap gxbb_vclk_input = {
.hw.init = &(struct clk_init_data) {
.name = "vclk_input",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk_sel" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vclk_sel.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1705,7 +1879,7 @@ static struct clk_regmap gxbb_vclk2_input = {
.hw.init = &(struct clk_init_data) {
.name = "vclk2_input",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk2_sel" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vclk2_sel.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1720,7 +1894,9 @@ static struct clk_regmap gxbb_vclk_div = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vclk_input" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vclk_input.hw
+ },
.num_parents = 1,
.flags = CLK_GET_RATE_NOCACHE,
},
@@ -1735,7 +1911,9 @@ static struct clk_regmap gxbb_vclk2_div = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vclk2_input" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vclk2_input.hw
+ },
.num_parents = 1,
.flags = CLK_GET_RATE_NOCACHE,
},
@@ -1749,7 +1927,7 @@ static struct clk_regmap gxbb_vclk = {
.hw.init = &(struct clk_init_data) {
.name = "vclk",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk_div" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vclk_div.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1763,7 +1941,7 @@ static struct clk_regmap gxbb_vclk2 = {
.hw.init = &(struct clk_init_data) {
.name = "vclk2",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk2_div" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vclk2_div.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1777,7 +1955,7 @@ static struct clk_regmap gxbb_vclk_div1 = {
.hw.init = &(struct clk_init_data) {
.name = "vclk_div1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vclk.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1791,7 +1969,7 @@ static struct clk_regmap gxbb_vclk_div2_en = {
.hw.init = &(struct clk_init_data) {
.name = "vclk_div2_en",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vclk.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1805,7 +1983,7 @@ static struct clk_regmap gxbb_vclk_div4_en = {
.hw.init = &(struct clk_init_data) {
.name = "vclk_div4_en",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vclk.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1819,7 +1997,7 @@ static struct clk_regmap gxbb_vclk_div6_en = {
.hw.init = &(struct clk_init_data) {
.name = "vclk_div6_en",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vclk.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1833,7 +2011,7 @@ static struct clk_regmap gxbb_vclk_div12_en = {
.hw.init = &(struct clk_init_data) {
.name = "vclk_div12_en",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vclk.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1847,7 +2025,7 @@ static struct clk_regmap gxbb_vclk2_div1 = {
.hw.init = &(struct clk_init_data) {
.name = "vclk2_div1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk2" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vclk2.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1861,7 +2039,7 @@ static struct clk_regmap gxbb_vclk2_div2_en = {
.hw.init = &(struct clk_init_data) {
.name = "vclk2_div2_en",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk2" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vclk2.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1875,7 +2053,7 @@ static struct clk_regmap gxbb_vclk2_div4_en = {
.hw.init = &(struct clk_init_data) {
.name = "vclk2_div4_en",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk2" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vclk2.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1889,7 +2067,7 @@ static struct clk_regmap gxbb_vclk2_div6_en = {
.hw.init = &(struct clk_init_data) {
.name = "vclk2_div6_en",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk2" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vclk2.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1903,7 +2081,7 @@ static struct clk_regmap gxbb_vclk2_div12_en = {
.hw.init = &(struct clk_init_data) {
.name = "vclk2_div12_en",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vclk2" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_vclk2.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -1915,7 +2093,9 @@ static struct clk_fixed_factor gxbb_vclk_div2 = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div2",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk_div2_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vclk_div2_en.hw
+ },
.num_parents = 1,
},
};
@@ -1926,7 +2106,9 @@ static struct clk_fixed_factor gxbb_vclk_div4 = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div4",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk_div4_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vclk_div4_en.hw
+ },
.num_parents = 1,
},
};
@@ -1937,7 +2119,9 @@ static struct clk_fixed_factor gxbb_vclk_div6 = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div6",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk_div6_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vclk_div6_en.hw
+ },
.num_parents = 1,
},
};
@@ -1948,7 +2132,9 @@ static struct clk_fixed_factor gxbb_vclk_div12 = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div12",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk_div12_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vclk_div12_en.hw
+ },
.num_parents = 1,
},
};
@@ -1959,7 +2145,9 @@ static struct clk_fixed_factor gxbb_vclk2_div2 = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div2",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk2_div2_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vclk2_div2_en.hw
+ },
.num_parents = 1,
},
};
@@ -1970,7 +2158,9 @@ static struct clk_fixed_factor gxbb_vclk2_div4 = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div4",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk2_div4_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vclk2_div4_en.hw
+ },
.num_parents = 1,
},
};
@@ -1981,7 +2171,9 @@ static struct clk_fixed_factor gxbb_vclk2_div6 = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div6",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk2_div6_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vclk2_div6_en.hw
+ },
.num_parents = 1,
},
};
@@ -1992,16 +2184,25 @@ static struct clk_fixed_factor gxbb_vclk2_div12 = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div12",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk2_div12_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vclk2_div12_en.hw
+ },
.num_parents = 1,
},
};
static u32 mux_table_cts_sel[] = { 0, 1, 2, 3, 4, 8, 9, 10, 11, 12 };
-static const char * const gxbb_cts_parent_names[] = {
- "vclk_div1", "vclk_div2", "vclk_div4", "vclk_div6",
- "vclk_div12", "vclk2_div1", "vclk2_div2", "vclk2_div4",
- "vclk2_div6", "vclk2_div12"
+static const struct clk_hw *gxbb_cts_parent_hws[] = {
+ &gxbb_vclk_div1.hw,
+ &gxbb_vclk_div2.hw,
+ &gxbb_vclk_div4.hw,
+ &gxbb_vclk_div6.hw,
+ &gxbb_vclk_div12.hw,
+ &gxbb_vclk2_div1.hw,
+ &gxbb_vclk2_div2.hw,
+ &gxbb_vclk2_div4.hw,
+ &gxbb_vclk2_div6.hw,
+ &gxbb_vclk2_div12.hw,
};
static struct clk_regmap gxbb_cts_enci_sel = {
@@ -2014,8 +2215,8 @@ static struct clk_regmap gxbb_cts_enci_sel = {
.hw.init = &(struct clk_init_data){
.name = "cts_enci_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = gxbb_cts_parent_names,
- .num_parents = ARRAY_SIZE(gxbb_cts_parent_names),
+ .parent_hws = gxbb_cts_parent_hws,
+ .num_parents = ARRAY_SIZE(gxbb_cts_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE,
},
};
@@ -2030,8 +2231,8 @@ static struct clk_regmap gxbb_cts_encp_sel = {
.hw.init = &(struct clk_init_data){
.name = "cts_encp_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = gxbb_cts_parent_names,
- .num_parents = ARRAY_SIZE(gxbb_cts_parent_names),
+ .parent_hws = gxbb_cts_parent_hws,
+ .num_parents = ARRAY_SIZE(gxbb_cts_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE,
},
};
@@ -2046,18 +2247,25 @@ static struct clk_regmap gxbb_cts_vdac_sel = {
.hw.init = &(struct clk_init_data){
.name = "cts_vdac_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = gxbb_cts_parent_names,
- .num_parents = ARRAY_SIZE(gxbb_cts_parent_names),
+ .parent_hws = gxbb_cts_parent_hws,
+ .num_parents = ARRAY_SIZE(gxbb_cts_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE,
},
};
/* TOFIX: add support for cts_tcon */
static u32 mux_table_hdmi_tx_sel[] = { 0, 1, 2, 3, 4, 8, 9, 10, 11, 12 };
-static const char * const gxbb_cts_hdmi_tx_parent_names[] = {
- "vclk_div1", "vclk_div2", "vclk_div4", "vclk_div6",
- "vclk_div12", "vclk2_div1", "vclk2_div2", "vclk2_div4",
- "vclk2_div6", "vclk2_div12"
+static const struct clk_hw *gxbb_cts_hdmi_tx_parent_hws[] = {
+ &gxbb_vclk_div1.hw,
+ &gxbb_vclk_div2.hw,
+ &gxbb_vclk_div4.hw,
+ &gxbb_vclk_div6.hw,
+ &gxbb_vclk_div12.hw,
+ &gxbb_vclk2_div1.hw,
+ &gxbb_vclk2_div2.hw,
+ &gxbb_vclk2_div4.hw,
+ &gxbb_vclk2_div6.hw,
+ &gxbb_vclk2_div12.hw,
};
static struct clk_regmap gxbb_hdmi_tx_sel = {
@@ -2076,8 +2284,8 @@ static struct clk_regmap gxbb_hdmi_tx_sel = {
* vclk2_div1, vclk2_div2, vclk2_div4, vclk2_div6, vclk2_div12,
* cts_tcon
*/
- .parent_names = gxbb_cts_hdmi_tx_parent_names,
- .num_parents = ARRAY_SIZE(gxbb_cts_hdmi_tx_parent_names),
+ .parent_hws = gxbb_cts_hdmi_tx_parent_hws,
+ .num_parents = ARRAY_SIZE(gxbb_cts_hdmi_tx_parent_hws),
.flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE,
},
};
@@ -2090,7 +2298,9 @@ static struct clk_regmap gxbb_cts_enci = {
.hw.init = &(struct clk_init_data) {
.name = "cts_enci",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "cts_enci_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_cts_enci_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -2104,7 +2314,9 @@ static struct clk_regmap gxbb_cts_encp = {
.hw.init = &(struct clk_init_data) {
.name = "cts_encp",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "cts_encp_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_cts_encp_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -2118,7 +2330,9 @@ static struct clk_regmap gxbb_cts_vdac = {
.hw.init = &(struct clk_init_data) {
.name = "cts_vdac",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "cts_vdac_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_cts_vdac_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -2132,7 +2346,9 @@ static struct clk_regmap gxbb_hdmi_tx = {
.hw.init = &(struct clk_init_data) {
.name = "hdmi_tx",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "hdmi_tx_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_hdmi_tx_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -2140,8 +2356,11 @@ static struct clk_regmap gxbb_hdmi_tx = {
/* HDMI Clocks */
-static const char * const gxbb_hdmi_parent_names[] = {
- IN_PREFIX "xtal", "fclk_div4", "fclk_div3", "fclk_div5"
+static const struct clk_parent_data gxbb_hdmi_parent_data[] = {
+ { .fw_name = "xtal", },
+ { .hw = &gxbb_fclk_div4.hw },
+ { .hw = &gxbb_fclk_div3.hw },
+ { .hw = &gxbb_fclk_div5.hw },
};
static struct clk_regmap gxbb_hdmi_sel = {
@@ -2154,8 +2373,8 @@ static struct clk_regmap gxbb_hdmi_sel = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = gxbb_hdmi_parent_names,
- .num_parents = ARRAY_SIZE(gxbb_hdmi_parent_names),
+ .parent_data = gxbb_hdmi_parent_data,
+ .num_parents = ARRAY_SIZE(gxbb_hdmi_parent_data),
.flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE,
},
};
@@ -2169,7 +2388,7 @@ static struct clk_regmap gxbb_hdmi_div = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "hdmi_sel" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_hdmi_sel.hw },
.num_parents = 1,
.flags = CLK_GET_RATE_NOCACHE,
},
@@ -2183,7 +2402,7 @@ static struct clk_regmap gxbb_hdmi = {
.hw.init = &(struct clk_init_data) {
.name = "hdmi",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "hdmi_div" },
+ .parent_hws = (const struct clk_hw *[]) { &gxbb_hdmi_div.hw },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
},
@@ -2191,8 +2410,11 @@ static struct clk_regmap gxbb_hdmi = {
/* VDEC clocks */
-static const char * const gxbb_vdec_parent_names[] = {
- "fclk_div4", "fclk_div3", "fclk_div5", "fclk_div7"
+static const struct clk_hw *gxbb_vdec_parent_hws[] = {
+ &gxbb_fclk_div4.hw,
+ &gxbb_fclk_div3.hw,
+ &gxbb_fclk_div5.hw,
+ &gxbb_fclk_div7.hw,
};
static struct clk_regmap gxbb_vdec_1_sel = {
@@ -2205,8 +2427,8 @@ static struct clk_regmap gxbb_vdec_1_sel = {
.hw.init = &(struct clk_init_data){
.name = "vdec_1_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = gxbb_vdec_parent_names,
- .num_parents = ARRAY_SIZE(gxbb_vdec_parent_names),
+ .parent_hws = gxbb_vdec_parent_hws,
+ .num_parents = ARRAY_SIZE(gxbb_vdec_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -2221,7 +2443,9 @@ static struct clk_regmap gxbb_vdec_1_div = {
.hw.init = &(struct clk_init_data){
.name = "vdec_1_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vdec_1_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vdec_1_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -2235,7 +2459,9 @@ static struct clk_regmap gxbb_vdec_1 = {
.hw.init = &(struct clk_init_data) {
.name = "vdec_1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vdec_1_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vdec_1_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -2251,8 +2477,8 @@ static struct clk_regmap gxbb_vdec_hevc_sel = {
.hw.init = &(struct clk_init_data){
.name = "vdec_hevc_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = gxbb_vdec_parent_names,
- .num_parents = ARRAY_SIZE(gxbb_vdec_parent_names),
+ .parent_hws = gxbb_vdec_parent_hws,
+ .num_parents = ARRAY_SIZE(gxbb_vdec_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -2267,7 +2493,9 @@ static struct clk_regmap gxbb_vdec_hevc_div = {
.hw.init = &(struct clk_init_data){
.name = "vdec_hevc_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vdec_hevc_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vdec_hevc_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -2281,7 +2509,9 @@ static struct clk_regmap gxbb_vdec_hevc = {
.hw.init = &(struct clk_init_data) {
.name = "vdec_hevc",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vdec_hevc_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_vdec_hevc_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -2289,9 +2519,18 @@ static struct clk_regmap gxbb_vdec_hevc = {
static u32 mux_table_gen_clk[] = { 0, 4, 5, 6, 7, 8,
9, 10, 11, 13, 14, };
-static const char * const gen_clk_parent_names[] = {
- IN_PREFIX "xtal", "vdec_1", "vdec_hevc", "mpll0", "mpll1", "mpll2",
- "fclk_div4", "fclk_div3", "fclk_div5", "fclk_div7", "gp0_pll",
+static const struct clk_parent_data gen_clk_parent_data[] = {
+ { .fw_name = "xtal", },
+ { .hw = &gxbb_vdec_1.hw },
+ { .hw = &gxbb_vdec_hevc.hw },
+ { .hw = &gxbb_mpll0.hw },
+ { .hw = &gxbb_mpll1.hw },
+ { .hw = &gxbb_mpll2.hw },
+ { .hw = &gxbb_fclk_div4.hw },
+ { .hw = &gxbb_fclk_div3.hw },
+ { .hw = &gxbb_fclk_div5.hw },
+ { .hw = &gxbb_fclk_div7.hw },
+ { .hw = &gxbb_gp0_pll.hw },
};
static struct clk_regmap gxbb_gen_clk_sel = {
@@ -2310,8 +2549,8 @@ static struct clk_regmap gxbb_gen_clk_sel = {
* vid_pll, vid2_pll (hevc), mpll0, mpll1, mpll2, fdiv4,
* fdiv3, fdiv5, [cts_msr_clk], fdiv7, gp0_pll
*/
- .parent_names = gen_clk_parent_names,
- .num_parents = ARRAY_SIZE(gen_clk_parent_names),
+ .parent_data = gen_clk_parent_data,
+ .num_parents = ARRAY_SIZE(gen_clk_parent_data),
},
};
@@ -2324,7 +2563,9 @@ static struct clk_regmap gxbb_gen_clk_div = {
.hw.init = &(struct clk_init_data){
.name = "gen_clk_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "gen_clk_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_gen_clk_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -2338,12 +2579,17 @@ static struct clk_regmap gxbb_gen_clk = {
.hw.init = &(struct clk_init_data){
.name = "gen_clk",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "gen_clk_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &gxbb_gen_clk_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
};
+#define MESON_GATE(_name, _reg, _bit) \
+ MESON_PCLK(_name, _reg, _bit, &gxbb_clk81.hw)
+
/* Everything Else (EE) domain gates */
static MESON_GATE(gxbb_ddr, HHI_GCLK_MPEG0, 0);
static MESON_GATE(gxbb_dos, HHI_GCLK_MPEG0, 1);
diff --git a/drivers/clk/meson/meson-aoclk.c b/drivers/clk/meson/meson-aoclk.c
index b67951909e04..bf8bea675d24 100644
--- a/drivers/clk/meson/meson-aoclk.c
+++ b/drivers/clk/meson/meson-aoclk.c
@@ -17,8 +17,6 @@
#include <linux/slab.h>
#include "meson-aoclk.h"
-#include "clk-input.h"
-
static int meson_aoclk_do_reset(struct reset_controller_dev *rcdev,
unsigned long id)
{
@@ -33,37 +31,6 @@ static const struct reset_control_ops meson_aoclk_reset_ops = {
.reset = meson_aoclk_do_reset,
};
-static int meson_aoclkc_register_inputs(struct device *dev,
- struct meson_aoclk_data *data)
-{
- struct clk_hw *hw;
- char *str;
- int i;
-
- for (i = 0; i < data->num_inputs; i++) {
- const struct meson_aoclk_input *in = &data->inputs[i];
-
- str = kasprintf(GFP_KERNEL, "%s%s", data->input_prefix,
- in->name);
- if (!str)
- return -ENOMEM;
-
- hw = meson_clk_hw_register_input(dev, in->name, str, 0);
- kfree(str);
-
- if (IS_ERR(hw)) {
- if (!in->required && PTR_ERR(hw) == -ENOENT)
- continue;
- else if (PTR_ERR(hw) != -EPROBE_DEFER)
- dev_err(dev, "failed to register input %s\n",
- in->name);
- return PTR_ERR(hw);
- }
- }
-
- return 0;
-}
-
int meson_aoclkc_probe(struct platform_device *pdev)
{
struct meson_aoclk_reset_controller *rstc;
@@ -86,10 +53,6 @@ int meson_aoclkc_probe(struct platform_device *pdev)
return PTR_ERR(regmap);
}
- ret = meson_aoclkc_register_inputs(dev, data);
- if (ret)
- return ret;
-
/* Reset Controller */
rstc->data = data;
rstc->regmap = regmap;
diff --git a/drivers/clk/meson/meson-aoclk.h b/drivers/clk/meson/meson-aoclk.h
index 999cde3868f7..605b43855a69 100644
--- a/drivers/clk/meson/meson-aoclk.h
+++ b/drivers/clk/meson/meson-aoclk.h
@@ -18,20 +18,12 @@
#include "clk-regmap.h"
-struct meson_aoclk_input {
- const char *name;
- bool required;
-};
-
struct meson_aoclk_data {
const unsigned int reset_reg;
const int num_reset;
const unsigned int *reset;
const int num_clks;
struct clk_regmap **clks;
- const int num_inputs;
- const struct meson_aoclk_input *inputs;
- const char *input_prefix;
const struct clk_hw_onecell_data *hw_data;
};
diff --git a/drivers/clk/meson/meson-eeclk.c b/drivers/clk/meson/meson-eeclk.c
index 37a34c9c3885..a7cb1e7aedc4 100644
--- a/drivers/clk/meson/meson-eeclk.c
+++ b/drivers/clk/meson/meson-eeclk.c
@@ -10,7 +10,6 @@
#include <linux/mfd/syscon.h>
#include <linux/regmap.h>
-#include "clk-input.h"
#include "clk-regmap.h"
#include "meson-eeclk.h"
@@ -18,7 +17,6 @@ int meson_eeclkc_probe(struct platform_device *pdev)
{
const struct meson_eeclkc_data *data;
struct device *dev = &pdev->dev;
- struct clk_hw *input;
struct regmap *map;
int ret, i;
@@ -34,13 +32,8 @@ int meson_eeclkc_probe(struct platform_device *pdev)
return PTR_ERR(map);
}
- input = meson_clk_hw_register_input(dev, "xtal", IN_PREFIX "xtal", 0);
- if (IS_ERR(input)) {
- ret = PTR_ERR(input);
- if (ret != -EPROBE_DEFER)
- dev_err(dev, "failed to get input clock");
- return ret;
- }
+ if (data->init_count)
+ regmap_multi_reg_write(map, data->init_regs, data->init_count);
/* Populate regmap for the regmap backed clocks */
for (i = 0; i < data->regmap_clk_num; i++)
diff --git a/drivers/clk/meson/meson-eeclk.h b/drivers/clk/meson/meson-eeclk.h
index 1b809b1419fe..77316207bde1 100644
--- a/drivers/clk/meson/meson-eeclk.h
+++ b/drivers/clk/meson/meson-eeclk.h
@@ -10,13 +10,13 @@
#include <linux/clk-provider.h>
#include "clk-regmap.h"
-#define IN_PREFIX "ee-in-"
-
struct platform_device;
struct meson_eeclkc_data {
struct clk_regmap *const *regmap_clks;
unsigned int regmap_clk_num;
+ const struct reg_sequence *init_regs;
+ unsigned int init_count;
struct clk_hw_onecell_data *hw_onecell_data;
};
diff --git a/drivers/clk/meson/meson8b.c b/drivers/clk/meson/meson8b.c
index 62cd3a7f1f65..67e6691e080c 100644
--- a/drivers/clk/meson/meson8b.c
+++ b/drivers/clk/meson/meson8b.c
@@ -97,7 +97,9 @@ static struct clk_regmap meson8b_fixed_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "fixed_pll_dco",
.ops = &meson_clk_pll_ro_ops,
- .parent_names = (const char *[]){ "xtal" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_xtal.hw
+ },
.num_parents = 1,
},
};
@@ -112,7 +114,9 @@ static struct clk_regmap meson8b_fixed_pll = {
.hw.init = &(struct clk_init_data){
.name = "fixed_pll",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "fixed_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_fixed_pll_dco.hw
+ },
.num_parents = 1,
/*
* This clock won't ever change at runtime so
@@ -158,7 +162,9 @@ static struct clk_regmap meson8b_hdmi_pll_dco = {
/* sometimes also called "HPLL" or "HPLL PLL" */
.name = "hdmi_pll_dco",
.ops = &meson_clk_pll_ro_ops,
- .parent_names = (const char *[]){ "xtal" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_xtal.hw
+ },
.num_parents = 1,
},
};
@@ -173,7 +179,9 @@ static struct clk_regmap meson8b_hdmi_pll_lvds_out = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_pll_lvds_out",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "hdmi_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_hdmi_pll_dco.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -189,7 +197,9 @@ static struct clk_regmap meson8b_hdmi_pll_hdmi_out = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_pll_hdmi_out",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "hdmi_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_hdmi_pll_dco.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -227,7 +237,9 @@ static struct clk_regmap meson8b_sys_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "sys_pll_dco",
.ops = &meson_clk_pll_ops,
- .parent_names = (const char *[]){ "xtal" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_xtal.hw
+ },
.num_parents = 1,
},
};
@@ -242,7 +254,9 @@ static struct clk_regmap meson8b_sys_pll = {
.hw.init = &(struct clk_init_data){
.name = "sys_pll",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "sys_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_sys_pll_dco.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -254,7 +268,9 @@ static struct clk_fixed_factor meson8b_fclk_div2_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div2_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_fixed_pll.hw
+ },
.num_parents = 1,
},
};
@@ -267,7 +283,9 @@ static struct clk_regmap meson8b_fclk_div2 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div2",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div2_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_fclk_div2_div.hw
+ },
.num_parents = 1,
/*
* FIXME: Ethernet with a RGMII PHYs is not working if
@@ -285,7 +303,9 @@ static struct clk_fixed_factor meson8b_fclk_div3_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div3_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_fixed_pll.hw
+ },
.num_parents = 1,
},
};
@@ -298,7 +318,9 @@ static struct clk_regmap meson8b_fclk_div3 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div3",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div3_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_fclk_div3_div.hw
+ },
.num_parents = 1,
},
};
@@ -309,7 +331,9 @@ static struct clk_fixed_factor meson8b_fclk_div4_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div4_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_fixed_pll.hw
+ },
.num_parents = 1,
},
};
@@ -322,7 +346,9 @@ static struct clk_regmap meson8b_fclk_div4 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div4",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div4_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_fclk_div4_div.hw
+ },
.num_parents = 1,
},
};
@@ -333,7 +359,9 @@ static struct clk_fixed_factor meson8b_fclk_div5_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div5_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_fixed_pll.hw
+ },
.num_parents = 1,
},
};
@@ -346,7 +374,9 @@ static struct clk_regmap meson8b_fclk_div5 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div5",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div5_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_fclk_div5_div.hw
+ },
.num_parents = 1,
},
};
@@ -357,7 +387,9 @@ static struct clk_fixed_factor meson8b_fclk_div7_div = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div7_div",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_fixed_pll.hw
+ },
.num_parents = 1,
},
};
@@ -370,7 +402,9 @@ static struct clk_regmap meson8b_fclk_div7 = {
.hw.init = &(struct clk_init_data){
.name = "fclk_div7",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "fclk_div7_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_fclk_div7_div.hw
+ },
.num_parents = 1,
},
};
@@ -384,7 +418,9 @@ static struct clk_regmap meson8b_mpll_prediv = {
.hw.init = &(struct clk_init_data){
.name = "mpll_prediv",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "fixed_pll" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_fixed_pll.hw
+ },
.num_parents = 1,
},
};
@@ -416,7 +452,9 @@ static struct clk_regmap meson8b_mpll0_div = {
.hw.init = &(struct clk_init_data){
.name = "mpll0_div",
.ops = &meson_clk_mpll_ops,
- .parent_names = (const char *[]){ "mpll_prediv" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_mpll_prediv.hw
+ },
.num_parents = 1,
},
};
@@ -429,7 +467,9 @@ static struct clk_regmap meson8b_mpll0 = {
.hw.init = &(struct clk_init_data){
.name = "mpll0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpll0_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_mpll0_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -457,7 +497,9 @@ static struct clk_regmap meson8b_mpll1_div = {
.hw.init = &(struct clk_init_data){
.name = "mpll1_div",
.ops = &meson_clk_mpll_ops,
- .parent_names = (const char *[]){ "mpll_prediv" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_mpll_prediv.hw
+ },
.num_parents = 1,
},
};
@@ -470,7 +512,9 @@ static struct clk_regmap meson8b_mpll1 = {
.hw.init = &(struct clk_init_data){
.name = "mpll1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpll1_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_mpll1_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -498,7 +542,9 @@ static struct clk_regmap meson8b_mpll2_div = {
.hw.init = &(struct clk_init_data){
.name = "mpll2_div",
.ops = &meson_clk_mpll_ops,
- .parent_names = (const char *[]){ "mpll_prediv" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_mpll_prediv.hw
+ },
.num_parents = 1,
},
};
@@ -511,7 +557,9 @@ static struct clk_regmap meson8b_mpll2 = {
.hw.init = &(struct clk_init_data){
.name = "mpll2",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpll2_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_mpll2_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -533,8 +581,11 @@ static struct clk_regmap meson8b_mpeg_clk_sel = {
* xtal, 1'b0 (wtf), fclk_div7, mpll_clkout1, mpll_clkout2,
* fclk_div4, fclk_div3, fclk_div5
*/
- .parent_names = (const char *[]){ "fclk_div3", "fclk_div4",
- "fclk_div5" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_fclk_div3.hw,
+ &meson8b_fclk_div4.hw,
+ &meson8b_fclk_div5.hw,
+ },
.num_parents = 3,
},
};
@@ -548,7 +599,9 @@ static struct clk_regmap meson8b_mpeg_clk_div = {
.hw.init = &(struct clk_init_data){
.name = "mpeg_clk_div",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "mpeg_clk_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_mpeg_clk_sel.hw
+ },
.num_parents = 1,
},
};
@@ -561,7 +614,9 @@ static struct clk_regmap meson8b_clk81 = {
.hw.init = &(struct clk_init_data){
.name = "clk81",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mpeg_clk_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_mpeg_clk_div.hw
+ },
.num_parents = 1,
.flags = CLK_IS_CRITICAL,
},
@@ -576,7 +631,10 @@ static struct clk_regmap meson8b_cpu_in_sel = {
.hw.init = &(struct clk_init_data){
.name = "cpu_in_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "xtal", "sys_pll" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_xtal.hw,
+ &meson8b_sys_pll.hw,
+ },
.num_parents = 2,
.flags = (CLK_SET_RATE_PARENT |
CLK_SET_RATE_NO_REPARENT),
@@ -589,7 +647,9 @@ static struct clk_fixed_factor meson8b_cpu_in_div2 = {
.hw.init = &(struct clk_init_data){
.name = "cpu_in_div2",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "cpu_in_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cpu_in_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -601,7 +661,9 @@ static struct clk_fixed_factor meson8b_cpu_in_div3 = {
.hw.init = &(struct clk_init_data){
.name = "cpu_in_div3",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "cpu_in_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cpu_in_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -630,7 +692,9 @@ static struct clk_regmap meson8b_cpu_scale_div = {
.hw.init = &(struct clk_init_data){
.name = "cpu_scale_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "cpu_in_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cpu_in_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -649,13 +713,15 @@ static struct clk_regmap meson8b_cpu_scale_out_sel = {
.ops = &clk_regmap_mux_ops,
/*
* NOTE: We are skipping the parent with value 0x2 (which is
- * "cpu_in_div3") because it results in a duty cycle of 33%
- * which makes the system unstable and can result in a lockup
- * of the whole system.
+ * meson8b_cpu_in_div3) because it results in a duty cycle of
+ * 33% which makes the system unstable and can result in a
+ * lockup of the whole system.
*/
- .parent_names = (const char *[]) { "cpu_in_sel",
- "cpu_in_div2",
- "cpu_scale_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cpu_in_sel.hw,
+ &meson8b_cpu_in_div2.hw,
+ &meson8b_cpu_scale_div.hw,
+ },
.num_parents = 3,
.flags = CLK_SET_RATE_PARENT,
},
@@ -670,8 +736,10 @@ static struct clk_regmap meson8b_cpu_clk = {
.hw.init = &(struct clk_init_data){
.name = "cpu_clk",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "xtal",
- "cpu_scale_out_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_xtal.hw,
+ &meson8b_cpu_scale_out_sel.hw,
+ },
.num_parents = 2,
.flags = (CLK_SET_RATE_PARENT |
CLK_SET_RATE_NO_REPARENT |
@@ -690,8 +758,13 @@ static struct clk_regmap meson8b_nand_clk_sel = {
.name = "nand_clk_sel",
.ops = &clk_regmap_mux_ops,
/* FIXME all other parents are unknown: */
- .parent_names = (const char *[]){ "fclk_div4", "fclk_div3",
- "fclk_div5", "fclk_div7", "xtal" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_fclk_div4.hw,
+ &meson8b_fclk_div3.hw,
+ &meson8b_fclk_div5.hw,
+ &meson8b_fclk_div7.hw,
+ &meson8b_xtal.hw,
+ },
.num_parents = 5,
.flags = CLK_SET_RATE_PARENT,
},
@@ -707,7 +780,9 @@ static struct clk_regmap meson8b_nand_clk_div = {
.hw.init = &(struct clk_init_data){
.name = "nand_clk_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "nand_clk_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_nand_clk_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -721,7 +796,9 @@ static struct clk_regmap meson8b_nand_clk_gate = {
.hw.init = &(struct clk_init_data){
.name = "nand_clk_gate",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "nand_clk_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_nand_clk_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -733,7 +810,9 @@ static struct clk_fixed_factor meson8b_cpu_clk_div2 = {
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_div2",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "cpu_clk" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cpu_clk.hw
+ },
.num_parents = 1,
},
};
@@ -744,7 +823,9 @@ static struct clk_fixed_factor meson8b_cpu_clk_div3 = {
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_div3",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "cpu_clk" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cpu_clk.hw
+ },
.num_parents = 1,
},
};
@@ -755,7 +836,9 @@ static struct clk_fixed_factor meson8b_cpu_clk_div4 = {
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_div4",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "cpu_clk" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cpu_clk.hw
+ },
.num_parents = 1,
},
};
@@ -766,7 +849,9 @@ static struct clk_fixed_factor meson8b_cpu_clk_div5 = {
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_div5",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "cpu_clk" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cpu_clk.hw
+ },
.num_parents = 1,
},
};
@@ -777,7 +862,9 @@ static struct clk_fixed_factor meson8b_cpu_clk_div6 = {
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_div6",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "cpu_clk" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cpu_clk.hw
+ },
.num_parents = 1,
},
};
@@ -788,7 +875,9 @@ static struct clk_fixed_factor meson8b_cpu_clk_div7 = {
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_div7",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "cpu_clk" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cpu_clk.hw
+ },
.num_parents = 1,
},
};
@@ -799,7 +888,9 @@ static struct clk_fixed_factor meson8b_cpu_clk_div8 = {
.hw.init = &(struct clk_init_data){
.name = "cpu_clk_div8",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "cpu_clk" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cpu_clk.hw
+ },
.num_parents = 1,
},
};
@@ -815,13 +906,15 @@ static struct clk_regmap meson8b_apb_clk_sel = {
.hw.init = &(struct clk_init_data){
.name = "apb_clk_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "cpu_clk_div2",
- "cpu_clk_div3",
- "cpu_clk_div4",
- "cpu_clk_div5",
- "cpu_clk_div6",
- "cpu_clk_div7",
- "cpu_clk_div8", },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cpu_clk_div2.hw,
+ &meson8b_cpu_clk_div3.hw,
+ &meson8b_cpu_clk_div4.hw,
+ &meson8b_cpu_clk_div5.hw,
+ &meson8b_cpu_clk_div6.hw,
+ &meson8b_cpu_clk_div7.hw,
+ &meson8b_cpu_clk_div8.hw,
+ },
.num_parents = 7,
},
};
@@ -835,7 +928,9 @@ static struct clk_regmap meson8b_apb_clk_gate = {
.hw.init = &(struct clk_init_data){
.name = "apb_clk_dis",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "apb_clk_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_apb_clk_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -850,13 +945,15 @@ static struct clk_regmap meson8b_periph_clk_sel = {
.hw.init = &(struct clk_init_data){
.name = "periph_clk_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "cpu_clk_div2",
- "cpu_clk_div3",
- "cpu_clk_div4",
- "cpu_clk_div5",
- "cpu_clk_div6",
- "cpu_clk_div7",
- "cpu_clk_div8", },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cpu_clk_div2.hw,
+ &meson8b_cpu_clk_div3.hw,
+ &meson8b_cpu_clk_div4.hw,
+ &meson8b_cpu_clk_div5.hw,
+ &meson8b_cpu_clk_div6.hw,
+ &meson8b_cpu_clk_div7.hw,
+ &meson8b_cpu_clk_div8.hw,
+ },
.num_parents = 7,
},
};
@@ -870,7 +967,9 @@ static struct clk_regmap meson8b_periph_clk_gate = {
.hw.init = &(struct clk_init_data){
.name = "periph_clk_dis",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "periph_clk_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_periph_clk_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -887,13 +986,15 @@ static struct clk_regmap meson8b_axi_clk_sel = {
.hw.init = &(struct clk_init_data){
.name = "axi_clk_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "cpu_clk_div2",
- "cpu_clk_div3",
- "cpu_clk_div4",
- "cpu_clk_div5",
- "cpu_clk_div6",
- "cpu_clk_div7",
- "cpu_clk_div8", },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cpu_clk_div2.hw,
+ &meson8b_cpu_clk_div3.hw,
+ &meson8b_cpu_clk_div4.hw,
+ &meson8b_cpu_clk_div5.hw,
+ &meson8b_cpu_clk_div6.hw,
+ &meson8b_cpu_clk_div7.hw,
+ &meson8b_cpu_clk_div8.hw,
+ },
.num_parents = 7,
},
};
@@ -907,7 +1008,9 @@ static struct clk_regmap meson8b_axi_clk_gate = {
.hw.init = &(struct clk_init_data){
.name = "axi_clk_dis",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "axi_clk_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_axi_clk_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -922,13 +1025,15 @@ static struct clk_regmap meson8b_l2_dram_clk_sel = {
.hw.init = &(struct clk_init_data){
.name = "l2_dram_clk_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "cpu_clk_div2",
- "cpu_clk_div3",
- "cpu_clk_div4",
- "cpu_clk_div5",
- "cpu_clk_div6",
- "cpu_clk_div7",
- "cpu_clk_div8", },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cpu_clk_div2.hw,
+ &meson8b_cpu_clk_div3.hw,
+ &meson8b_cpu_clk_div4.hw,
+ &meson8b_cpu_clk_div5.hw,
+ &meson8b_cpu_clk_div6.hw,
+ &meson8b_cpu_clk_div7.hw,
+ &meson8b_cpu_clk_div8.hw,
+ },
.num_parents = 7,
},
};
@@ -942,7 +1047,9 @@ static struct clk_regmap meson8b_l2_dram_clk_gate = {
.hw.init = &(struct clk_init_data){
.name = "l2_dram_clk_dis",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "l2_dram_clk_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_l2_dram_clk_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -963,7 +1070,9 @@ static struct clk_regmap meson8b_vid_pll_in_sel = {
* Meson8b: hdmi_pll_dco
* Meson8m2: vid2_pll
*/
- .parent_names = (const char *[]){ "hdmi_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_hdmi_pll_dco.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -977,7 +1086,9 @@ static struct clk_regmap meson8b_vid_pll_in_en = {
.hw.init = &(struct clk_init_data){
.name = "vid_pll_in_en",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "vid_pll_in_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vid_pll_in_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -992,7 +1103,9 @@ static struct clk_regmap meson8b_vid_pll_pre_div = {
.hw.init = &(struct clk_init_data){
.name = "vid_pll_pre_div",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "vid_pll_in_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vid_pll_in_en.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1007,7 +1120,9 @@ static struct clk_regmap meson8b_vid_pll_post_div = {
.hw.init = &(struct clk_init_data){
.name = "vid_pll_post_div",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "vid_pll_pre_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vid_pll_pre_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1023,8 +1138,10 @@ static struct clk_regmap meson8b_vid_pll = {
.name = "vid_pll",
.ops = &clk_regmap_mux_ro_ops,
/* TODO: parent 0x2 is vid_pll_pre_div_mult7_div2 */
- .parent_names = (const char *[]){ "vid_pll_pre_div",
- "vid_pll_post_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vid_pll_pre_div.hw,
+ &meson8b_vid_pll_post_div.hw,
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1039,15 +1156,22 @@ static struct clk_regmap meson8b_vid_pll_final_div = {
.hw.init = &(struct clk_init_data){
.name = "vid_pll_final_div",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "vid_pll" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vid_pll.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
};
-static const char * const meson8b_vclk_mux_parents[] = {
- "vid_pll_final_div", "fclk_div4", "fclk_div3", "fclk_div5",
- "vid_pll_final_div", "fclk_div7", "mpll1"
+static const struct clk_hw *meson8b_vclk_mux_parent_hws[] = {
+ &meson8b_vid_pll_final_div.hw,
+ &meson8b_fclk_div4.hw,
+ &meson8b_fclk_div3.hw,
+ &meson8b_fclk_div5.hw,
+ &meson8b_vid_pll_final_div.hw,
+ &meson8b_fclk_div7.hw,
+ &meson8b_mpll1.hw,
};
static struct clk_regmap meson8b_vclk_in_sel = {
@@ -1059,8 +1183,8 @@ static struct clk_regmap meson8b_vclk_in_sel = {
.hw.init = &(struct clk_init_data){
.name = "vclk_in_sel",
.ops = &clk_regmap_mux_ro_ops,
- .parent_names = meson8b_vclk_mux_parents,
- .num_parents = ARRAY_SIZE(meson8b_vclk_mux_parents),
+ .parent_hws = meson8b_vclk_mux_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_vclk_mux_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1073,7 +1197,9 @@ static struct clk_regmap meson8b_vclk_in_en = {
.hw.init = &(struct clk_init_data){
.name = "vclk_in_en",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "vclk_in_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk_in_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1087,7 +1213,9 @@ static struct clk_regmap meson8b_vclk_div1_gate = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div1_en",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "vclk_in_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk_in_en.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1099,7 +1227,9 @@ static struct clk_fixed_factor meson8b_vclk_div2_div = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div2",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk_in_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk_in_en.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
}
@@ -1113,7 +1243,9 @@ static struct clk_regmap meson8b_vclk_div2_div_gate = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div2_en",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "vclk_div2" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk_div2_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1125,7 +1257,9 @@ static struct clk_fixed_factor meson8b_vclk_div4_div = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div4",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk_in_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk_in_en.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
}
@@ -1139,7 +1273,9 @@ static struct clk_regmap meson8b_vclk_div4_div_gate = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div4_en",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "vclk_div4" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk_div4_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1151,7 +1287,9 @@ static struct clk_fixed_factor meson8b_vclk_div6_div = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div6",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk_in_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk_in_en.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
}
@@ -1165,7 +1303,9 @@ static struct clk_regmap meson8b_vclk_div6_div_gate = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div6_en",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "vclk_div6" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk_div6_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1177,7 +1317,9 @@ static struct clk_fixed_factor meson8b_vclk_div12_div = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div12",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk_in_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk_in_en.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
}
@@ -1191,7 +1333,9 @@ static struct clk_regmap meson8b_vclk_div12_div_gate = {
.hw.init = &(struct clk_init_data){
.name = "vclk_div12_en",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "vclk_div12" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk_div12_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1206,8 +1350,8 @@ static struct clk_regmap meson8b_vclk2_in_sel = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_in_sel",
.ops = &clk_regmap_mux_ro_ops,
- .parent_names = meson8b_vclk_mux_parents,
- .num_parents = ARRAY_SIZE(meson8b_vclk_mux_parents),
+ .parent_hws = meson8b_vclk_mux_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_vclk_mux_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1220,7 +1364,9 @@ static struct clk_regmap meson8b_vclk2_clk_in_en = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_in_en",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "vclk2_in_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk2_in_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1234,7 +1380,9 @@ static struct clk_regmap meson8b_vclk2_div1_gate = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div1_en",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "vclk2_in_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk2_clk_in_en.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1246,7 +1394,9 @@ static struct clk_fixed_factor meson8b_vclk2_div2_div = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div2",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk2_in_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk2_clk_in_en.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
}
@@ -1260,7 +1410,9 @@ static struct clk_regmap meson8b_vclk2_div2_div_gate = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div2_en",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "vclk2_div2" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk2_div2_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1272,7 +1424,9 @@ static struct clk_fixed_factor meson8b_vclk2_div4_div = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div4",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk2_in_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk2_clk_in_en.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
}
@@ -1286,7 +1440,9 @@ static struct clk_regmap meson8b_vclk2_div4_div_gate = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div4_en",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "vclk2_div4" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk2_div4_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1298,7 +1454,9 @@ static struct clk_fixed_factor meson8b_vclk2_div6_div = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div6",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk2_in_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk2_clk_in_en.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
}
@@ -1312,7 +1470,9 @@ static struct clk_regmap meson8b_vclk2_div6_div_gate = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div6_en",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "vclk2_div6" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk2_div6_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1324,7 +1484,9 @@ static struct clk_fixed_factor meson8b_vclk2_div12_div = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div12",
.ops = &clk_fixed_factor_ops,
- .parent_names = (const char *[]){ "vclk2_in_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk2_clk_in_en.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
}
@@ -1338,15 +1500,20 @@ static struct clk_regmap meson8b_vclk2_div12_div_gate = {
.hw.init = &(struct clk_init_data){
.name = "vclk2_div12_en",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "vclk2_div12" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vclk2_div12_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
};
-static const char * const meson8b_vclk_enc_mux_parents[] = {
- "vclk_div1_en", "vclk_div2_en", "vclk_div4_en", "vclk_div6_en",
- "vclk_div12_en",
+static const struct clk_hw *meson8b_vclk_enc_mux_parent_hws[] = {
+ &meson8b_vclk_div1_gate.hw,
+ &meson8b_vclk_div2_div_gate.hw,
+ &meson8b_vclk_div4_div_gate.hw,
+ &meson8b_vclk_div6_div_gate.hw,
+ &meson8b_vclk_div12_div_gate.hw,
};
static struct clk_regmap meson8b_cts_enct_sel = {
@@ -1358,8 +1525,8 @@ static struct clk_regmap meson8b_cts_enct_sel = {
.hw.init = &(struct clk_init_data){
.name = "cts_enct_sel",
.ops = &clk_regmap_mux_ro_ops,
- .parent_names = meson8b_vclk_enc_mux_parents,
- .num_parents = ARRAY_SIZE(meson8b_vclk_enc_mux_parents),
+ .parent_hws = meson8b_vclk_enc_mux_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_vclk_enc_mux_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1372,7 +1539,9 @@ static struct clk_regmap meson8b_cts_enct = {
.hw.init = &(struct clk_init_data){
.name = "cts_enct",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "cts_enct_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cts_enct_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1387,8 +1556,8 @@ static struct clk_regmap meson8b_cts_encp_sel = {
.hw.init = &(struct clk_init_data){
.name = "cts_encp_sel",
.ops = &clk_regmap_mux_ro_ops,
- .parent_names = meson8b_vclk_enc_mux_parents,
- .num_parents = ARRAY_SIZE(meson8b_vclk_enc_mux_parents),
+ .parent_hws = meson8b_vclk_enc_mux_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_vclk_enc_mux_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1401,7 +1570,9 @@ static struct clk_regmap meson8b_cts_encp = {
.hw.init = &(struct clk_init_data){
.name = "cts_encp",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "cts_encp_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cts_encp_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1416,8 +1587,8 @@ static struct clk_regmap meson8b_cts_enci_sel = {
.hw.init = &(struct clk_init_data){
.name = "cts_enci_sel",
.ops = &clk_regmap_mux_ro_ops,
- .parent_names = meson8b_vclk_enc_mux_parents,
- .num_parents = ARRAY_SIZE(meson8b_vclk_enc_mux_parents),
+ .parent_hws = meson8b_vclk_enc_mux_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_vclk_enc_mux_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1430,7 +1601,9 @@ static struct clk_regmap meson8b_cts_enci = {
.hw.init = &(struct clk_init_data){
.name = "cts_enci",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "cts_enci_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cts_enci_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1445,8 +1618,8 @@ static struct clk_regmap meson8b_hdmi_tx_pixel_sel = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_tx_pixel_sel",
.ops = &clk_regmap_mux_ro_ops,
- .parent_names = meson8b_vclk_enc_mux_parents,
- .num_parents = ARRAY_SIZE(meson8b_vclk_enc_mux_parents),
+ .parent_hws = meson8b_vclk_enc_mux_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_vclk_enc_mux_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1459,15 +1632,20 @@ static struct clk_regmap meson8b_hdmi_tx_pixel = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_tx_pixel",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "hdmi_tx_pixel_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_hdmi_tx_pixel_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
};
-static const char * const meson8b_vclk2_enc_mux_parents[] = {
- "vclk2_div1_en", "vclk2_div2_en", "vclk2_div4_en", "vclk2_div6_en",
- "vclk2_div12_en",
+static const struct clk_hw *meson8b_vclk2_enc_mux_parent_hws[] = {
+ &meson8b_vclk2_div1_gate.hw,
+ &meson8b_vclk2_div2_div_gate.hw,
+ &meson8b_vclk2_div4_div_gate.hw,
+ &meson8b_vclk2_div6_div_gate.hw,
+ &meson8b_vclk2_div12_div_gate.hw,
};
static struct clk_regmap meson8b_cts_encl_sel = {
@@ -1479,8 +1657,8 @@ static struct clk_regmap meson8b_cts_encl_sel = {
.hw.init = &(struct clk_init_data){
.name = "cts_encl_sel",
.ops = &clk_regmap_mux_ro_ops,
- .parent_names = meson8b_vclk2_enc_mux_parents,
- .num_parents = ARRAY_SIZE(meson8b_vclk2_enc_mux_parents),
+ .parent_hws = meson8b_vclk2_enc_mux_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_vclk2_enc_mux_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1493,7 +1671,9 @@ static struct clk_regmap meson8b_cts_encl = {
.hw.init = &(struct clk_init_data){
.name = "cts_encl",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "cts_encl_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cts_encl_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1508,8 +1688,8 @@ static struct clk_regmap meson8b_cts_vdac0_sel = {
.hw.init = &(struct clk_init_data){
.name = "cts_vdac0_sel",
.ops = &clk_regmap_mux_ro_ops,
- .parent_names = meson8b_vclk2_enc_mux_parents,
- .num_parents = ARRAY_SIZE(meson8b_vclk2_enc_mux_parents),
+ .parent_hws = meson8b_vclk2_enc_mux_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_vclk2_enc_mux_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1522,7 +1702,9 @@ static struct clk_regmap meson8b_cts_vdac0 = {
.hw.init = &(struct clk_init_data){
.name = "cts_vdac0",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "cts_vdac0_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cts_vdac0_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1539,7 +1721,9 @@ static struct clk_regmap meson8b_hdmi_sys_sel = {
.name = "hdmi_sys_sel",
.ops = &clk_regmap_mux_ro_ops,
/* FIXME: all other parents are unknown */
- .parent_names = (const char *[]){ "xtal" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_xtal.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_NO_REPARENT,
},
@@ -1554,7 +1738,9 @@ static struct clk_regmap meson8b_hdmi_sys_div = {
.hw.init = &(struct clk_init_data){
.name = "hdmi_sys_div",
.ops = &clk_regmap_divider_ro_ops,
- .parent_names = (const char *[]){ "hdmi_sys_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_hdmi_sys_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1568,7 +1754,9 @@ static struct clk_regmap meson8b_hdmi_sys = {
.hw.init = &(struct clk_init_data) {
.name = "hdmi_sys",
.ops = &clk_regmap_gate_ro_ops,
- .parent_names = (const char *[]){ "hdmi_sys_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_hdmi_sys_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1579,9 +1767,14 @@ static struct clk_regmap meson8b_hdmi_sys = {
* muxed by a glitch-free switch on Meson8b and Meson8m2. Meson8 only
* has mali_0 and no glitch-free mux.
*/
-static const char * const meson8b_mali_0_1_parent_names[] = {
- "xtal", "mpll2", "mpll1", "fclk_div7", "fclk_div4", "fclk_div3",
- "fclk_div5"
+static const struct clk_hw *meson8b_mali_0_1_parent_hws[] = {
+ &meson8b_xtal.hw,
+ &meson8b_mpll2.hw,
+ &meson8b_mpll1.hw,
+ &meson8b_fclk_div7.hw,
+ &meson8b_fclk_div4.hw,
+ &meson8b_fclk_div3.hw,
+ &meson8b_fclk_div5.hw,
};
static u32 meson8b_mali_0_1_mux_table[] = { 0, 2, 3, 4, 5, 6, 7 };
@@ -1596,8 +1789,8 @@ static struct clk_regmap meson8b_mali_0_sel = {
.hw.init = &(struct clk_init_data){
.name = "mali_0_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = meson8b_mali_0_1_parent_names,
- .num_parents = ARRAY_SIZE(meson8b_mali_0_1_parent_names),
+ .parent_hws = meson8b_mali_0_1_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_mali_0_1_parent_hws),
/*
* Don't propagate rate changes up because the only changeable
* parents are mpll1 and mpll2 but we need those for audio and
@@ -1617,7 +1810,9 @@ static struct clk_regmap meson8b_mali_0_div = {
.hw.init = &(struct clk_init_data){
.name = "mali_0_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "mali_0_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_mali_0_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1631,7 +1826,9 @@ static struct clk_regmap meson8b_mali_0 = {
.hw.init = &(struct clk_init_data){
.name = "mali_0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mali_0_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_mali_0_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1647,8 +1844,8 @@ static struct clk_regmap meson8b_mali_1_sel = {
.hw.init = &(struct clk_init_data){
.name = "mali_1_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = meson8b_mali_0_1_parent_names,
- .num_parents = ARRAY_SIZE(meson8b_mali_0_1_parent_names),
+ .parent_hws = meson8b_mali_0_1_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_mali_0_1_parent_hws),
/*
* Don't propagate rate changes up because the only changeable
* parents are mpll1 and mpll2 but we need those for audio and
@@ -1668,7 +1865,9 @@ static struct clk_regmap meson8b_mali_1_div = {
.hw.init = &(struct clk_init_data){
.name = "mali_1_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "mali_1_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_mali_1_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1682,7 +1881,9 @@ static struct clk_regmap meson8b_mali_1 = {
.hw.init = &(struct clk_init_data){
.name = "mali_1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "mali_1_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_mali_1_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1697,7 +1898,10 @@ static struct clk_regmap meson8b_mali = {
.hw.init = &(struct clk_init_data){
.name = "mali",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "mali_0", "mali_1" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_mali_0.hw,
+ &meson8b_mali_1.hw,
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1740,7 +1944,9 @@ static struct clk_regmap meson8m2_gp_pll_dco = {
.hw.init = &(struct clk_init_data){
.name = "gp_pll_dco",
.ops = &meson_clk_pll_ops,
- .parent_names = (const char *[]){ "xtal" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_xtal.hw
+ },
.num_parents = 1,
},
};
@@ -1755,18 +1961,26 @@ static struct clk_regmap meson8m2_gp_pll = {
.hw.init = &(struct clk_init_data){
.name = "gp_pll",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "gp_pll_dco" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8m2_gp_pll_dco.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
};
-static const char * const meson8b_vpu_0_1_parent_names[] = {
- "fclk_div4", "fclk_div3", "fclk_div5", "fclk_div7"
+static const struct clk_hw *meson8b_vpu_0_1_parent_hws[] = {
+ &meson8b_fclk_div4.hw,
+ &meson8b_fclk_div3.hw,
+ &meson8b_fclk_div5.hw,
+ &meson8b_fclk_div7.hw,
};
-static const char * const mmeson8m2_vpu_0_1_parent_names[] = {
- "fclk_div4", "fclk_div3", "fclk_div5", "gp_pll"
+static const struct clk_hw *mmeson8m2_vpu_0_1_parent_hws[] = {
+ &meson8b_fclk_div4.hw,
+ &meson8b_fclk_div3.hw,
+ &meson8b_fclk_div5.hw,
+ &meson8m2_gp_pll.hw,
};
static struct clk_regmap meson8b_vpu_0_sel = {
@@ -1778,8 +1992,8 @@ static struct clk_regmap meson8b_vpu_0_sel = {
.hw.init = &(struct clk_init_data){
.name = "vpu_0_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = meson8b_vpu_0_1_parent_names,
- .num_parents = ARRAY_SIZE(meson8b_vpu_0_1_parent_names),
+ .parent_hws = meson8b_vpu_0_1_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_vpu_0_1_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1793,8 +2007,8 @@ static struct clk_regmap meson8m2_vpu_0_sel = {
.hw.init = &(struct clk_init_data){
.name = "vpu_0_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = mmeson8m2_vpu_0_1_parent_names,
- .num_parents = ARRAY_SIZE(mmeson8m2_vpu_0_1_parent_names),
+ .parent_hws = mmeson8m2_vpu_0_1_parent_hws,
+ .num_parents = ARRAY_SIZE(mmeson8m2_vpu_0_1_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1808,7 +2022,17 @@ static struct clk_regmap meson8b_vpu_0_div = {
.hw.init = &(struct clk_init_data){
.name = "vpu_0_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vpu_0_sel" },
+ .parent_data = &(const struct clk_parent_data) {
+ /*
+ * Note:
+ * meson8b and meson8m2 have different vpu_0_sels (with
+ * different struct clk_hw). We fallback to the global
+ * naming string mechanism so vpu_0_div picks up the
+ * appropriate one.
+ */
+ .name = "vpu_0_sel",
+ .index = -1,
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1822,7 +2046,9 @@ static struct clk_regmap meson8b_vpu_0 = {
.hw.init = &(struct clk_init_data) {
.name = "vpu_0",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vpu_0_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vpu_0_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1837,8 +2063,8 @@ static struct clk_regmap meson8b_vpu_1_sel = {
.hw.init = &(struct clk_init_data){
.name = "vpu_1_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = meson8b_vpu_0_1_parent_names,
- .num_parents = ARRAY_SIZE(meson8b_vpu_0_1_parent_names),
+ .parent_hws = meson8b_vpu_0_1_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_vpu_0_1_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1852,8 +2078,8 @@ static struct clk_regmap meson8m2_vpu_1_sel = {
.hw.init = &(struct clk_init_data){
.name = "vpu_1_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = mmeson8m2_vpu_0_1_parent_names,
- .num_parents = ARRAY_SIZE(mmeson8m2_vpu_0_1_parent_names),
+ .parent_hws = mmeson8m2_vpu_0_1_parent_hws,
+ .num_parents = ARRAY_SIZE(mmeson8m2_vpu_0_1_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1867,7 +2093,17 @@ static struct clk_regmap meson8b_vpu_1_div = {
.hw.init = &(struct clk_init_data){
.name = "vpu_1_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vpu_1_sel" },
+ .parent_data = &(const struct clk_parent_data) {
+ /*
+ * Note:
+ * meson8b and meson8m2 have different vpu_1_sels (with
+ * different struct clk_hw). We fallback to the global
+ * naming string mechanism so vpu_1_div picks up the
+ * appropriate one.
+ */
+ .name = "vpu_1_sel",
+ .index = -1,
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1881,7 +2117,9 @@ static struct clk_regmap meson8b_vpu_1 = {
.hw.init = &(struct clk_init_data) {
.name = "vpu_1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vpu_1_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vpu_1_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1896,14 +2134,22 @@ static struct clk_regmap meson8b_vpu = {
.hw.init = &(struct clk_init_data){
.name = "vpu",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "vpu_0", "vpu_1" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vpu_0.hw,
+ &meson8b_vpu_1.hw,
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_NO_REPARENT,
},
};
-static const char * const meson8b_vdec_parent_names[] = {
- "fclk_div4", "fclk_div3", "fclk_div5", "fclk_div7", "mpll2", "mpll1"
+static const struct clk_hw *meson8b_vdec_parent_hws[] = {
+ &meson8b_fclk_div4.hw,
+ &meson8b_fclk_div3.hw,
+ &meson8b_fclk_div5.hw,
+ &meson8b_fclk_div7.hw,
+ &meson8b_mpll2.hw,
+ &meson8b_mpll1.hw,
};
static struct clk_regmap meson8b_vdec_1_sel = {
@@ -1916,8 +2162,8 @@ static struct clk_regmap meson8b_vdec_1_sel = {
.hw.init = &(struct clk_init_data){
.name = "vdec_1_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = meson8b_vdec_parent_names,
- .num_parents = ARRAY_SIZE(meson8b_vdec_parent_names),
+ .parent_hws = meson8b_vdec_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_vdec_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -1932,7 +2178,9 @@ static struct clk_regmap meson8b_vdec_1_1_div = {
.hw.init = &(struct clk_init_data){
.name = "vdec_1_1_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vdec_1_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vdec_1_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1946,7 +2194,9 @@ static struct clk_regmap meson8b_vdec_1_1 = {
.hw.init = &(struct clk_init_data) {
.name = "vdec_1_1",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vdec_1_1_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vdec_1_1_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1962,7 +2212,9 @@ static struct clk_regmap meson8b_vdec_1_2_div = {
.hw.init = &(struct clk_init_data){
.name = "vdec_1_2_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vdec_1_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vdec_1_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1976,7 +2228,9 @@ static struct clk_regmap meson8b_vdec_1_2 = {
.hw.init = &(struct clk_init_data) {
.name = "vdec_1_2",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vdec_1_2_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vdec_1_2_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -1992,7 +2246,10 @@ static struct clk_regmap meson8b_vdec_1 = {
.hw.init = &(struct clk_init_data){
.name = "vdec_1",
.ops = &clk_regmap_mux_ops,
- .parent_names = (const char *[]){ "vdec_1_1", "vdec_1_2" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vdec_1_1.hw,
+ &meson8b_vdec_1_2.hw,
+ },
.num_parents = 2,
.flags = CLK_SET_RATE_PARENT,
},
@@ -2008,8 +2265,8 @@ static struct clk_regmap meson8b_vdec_hcodec_sel = {
.hw.init = &(struct clk_init_data){
.name = "vdec_hcodec_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = meson8b_vdec_parent_names,
- .num_parents = ARRAY_SIZE(meson8b_vdec_parent_names),
+ .parent_hws = meson8b_vdec_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_vdec_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -2024,7 +2281,9 @@ static struct clk_regmap meson8b_vdec_hcodec_div = {
.hw.init = &(struct clk_init_data){
.name = "vdec_hcodec_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vdec_hcodec_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vdec_hcodec_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -2038,7 +2297,9 @@ static struct clk_regmap meson8b_vdec_hcodec = {
.hw.init = &(struct clk_init_data) {
.name = "vdec_hcodec",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vdec_hcodec_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vdec_hcodec_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -2054,8 +2315,8 @@ static struct clk_regmap meson8b_vdec_2_sel = {
.hw.init = &(struct clk_init_data){
.name = "vdec_2_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = meson8b_vdec_parent_names,
- .num_parents = ARRAY_SIZE(meson8b_vdec_parent_names),
+ .parent_hws = meson8b_vdec_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_vdec_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -2070,7 +2331,9 @@ static struct clk_regmap meson8b_vdec_2_div = {
.hw.init = &(struct clk_init_data){
.name = "vdec_2_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vdec_2_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vdec_2_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -2084,7 +2347,9 @@ static struct clk_regmap meson8b_vdec_2 = {
.hw.init = &(struct clk_init_data) {
.name = "vdec_2",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vdec_2_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vdec_2_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -2100,8 +2365,8 @@ static struct clk_regmap meson8b_vdec_hevc_sel = {
.hw.init = &(struct clk_init_data){
.name = "vdec_hevc_sel",
.ops = &clk_regmap_mux_ops,
- .parent_names = meson8b_vdec_parent_names,
- .num_parents = ARRAY_SIZE(meson8b_vdec_parent_names),
+ .parent_hws = meson8b_vdec_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_vdec_parent_hws),
.flags = CLK_SET_RATE_PARENT,
},
};
@@ -2116,7 +2381,9 @@ static struct clk_regmap meson8b_vdec_hevc_div = {
.hw.init = &(struct clk_init_data){
.name = "vdec_hevc_div",
.ops = &clk_regmap_divider_ops,
- .parent_names = (const char *[]){ "vdec_hevc_sel" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vdec_hevc_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -2130,7 +2397,9 @@ static struct clk_regmap meson8b_vdec_hevc_en = {
.hw.init = &(struct clk_init_data) {
.name = "vdec_hevc_en",
.ops = &clk_regmap_gate_ops,
- .parent_names = (const char *[]){ "vdec_hevc_div" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vdec_hevc_div.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
@@ -2147,12 +2416,157 @@ static struct clk_regmap meson8b_vdec_hevc = {
.name = "vdec_hevc",
.ops = &clk_regmap_mux_ops,
/* TODO: The second parent is currently unknown */
- .parent_names = (const char *[]){ "vdec_hevc_en" },
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_vdec_hevc_en.hw
+ },
+ .num_parents = 1,
+ .flags = CLK_SET_RATE_PARENT,
+ },
+};
+
+/* TODO: the clock at index 0 is "DDR_PLL" which we don't support yet */
+static const struct clk_hw *meson8b_cts_amclk_parent_hws[] = {
+ &meson8b_mpll0.hw,
+ &meson8b_mpll1.hw,
+ &meson8b_mpll2.hw
+};
+
+static u32 meson8b_cts_amclk_mux_table[] = { 1, 2, 3 };
+
+static struct clk_regmap meson8b_cts_amclk_sel = {
+ .data = &(struct clk_regmap_mux_data){
+ .offset = HHI_AUD_CLK_CNTL,
+ .mask = 0x3,
+ .shift = 9,
+ .table = meson8b_cts_amclk_mux_table,
+ .flags = CLK_MUX_ROUND_CLOSEST,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cts_amclk_sel",
+ .ops = &clk_regmap_mux_ops,
+ .parent_hws = meson8b_cts_amclk_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_cts_amclk_parent_hws),
+ },
+};
+
+static struct clk_regmap meson8b_cts_amclk_div = {
+ .data = &(struct clk_regmap_div_data) {
+ .offset = HHI_AUD_CLK_CNTL,
+ .shift = 0,
+ .width = 8,
+ .flags = CLK_DIVIDER_ROUND_CLOSEST,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cts_amclk_div",
+ .ops = &clk_regmap_divider_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cts_amclk_sel.hw
+ },
+ .num_parents = 1,
+ .flags = CLK_SET_RATE_PARENT,
+ },
+};
+
+static struct clk_regmap meson8b_cts_amclk = {
+ .data = &(struct clk_regmap_gate_data){
+ .offset = HHI_AUD_CLK_CNTL,
+ .bit_idx = 8,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cts_amclk",
+ .ops = &clk_regmap_gate_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cts_amclk_div.hw
+ },
+ .num_parents = 1,
+ .flags = CLK_SET_RATE_PARENT,
+ },
+};
+
+/* TODO: the clock at index 0 is "DDR_PLL" which we don't support yet */
+static const struct clk_hw *meson8b_cts_mclk_i958_parent_hws[] = {
+ &meson8b_mpll0.hw,
+ &meson8b_mpll1.hw,
+ &meson8b_mpll2.hw
+};
+
+static u32 meson8b_cts_mclk_i958_mux_table[] = { 1, 2, 3 };
+
+static struct clk_regmap meson8b_cts_mclk_i958_sel = {
+ .data = &(struct clk_regmap_mux_data){
+ .offset = HHI_AUD_CLK_CNTL2,
+ .mask = 0x3,
+ .shift = 25,
+ .table = meson8b_cts_mclk_i958_mux_table,
+ .flags = CLK_MUX_ROUND_CLOSEST,
+ },
+ .hw.init = &(struct clk_init_data) {
+ .name = "cts_mclk_i958_sel",
+ .ops = &clk_regmap_mux_ops,
+ .parent_hws = meson8b_cts_mclk_i958_parent_hws,
+ .num_parents = ARRAY_SIZE(meson8b_cts_mclk_i958_parent_hws),
+ },
+};
+
+static struct clk_regmap meson8b_cts_mclk_i958_div = {
+ .data = &(struct clk_regmap_div_data){
+ .offset = HHI_AUD_CLK_CNTL2,
+ .shift = 16,
+ .width = 8,
+ .flags = CLK_DIVIDER_ROUND_CLOSEST,
+ },
+ .hw.init = &(struct clk_init_data) {
+ .name = "cts_mclk_i958_div",
+ .ops = &clk_regmap_divider_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cts_mclk_i958_sel.hw
+ },
.num_parents = 1,
.flags = CLK_SET_RATE_PARENT,
},
};
+static struct clk_regmap meson8b_cts_mclk_i958 = {
+ .data = &(struct clk_regmap_gate_data){
+ .offset = HHI_AUD_CLK_CNTL2,
+ .bit_idx = 24,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cts_mclk_i958",
+ .ops = &clk_regmap_gate_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cts_mclk_i958_div.hw
+ },
+ .num_parents = 1,
+ .flags = CLK_SET_RATE_PARENT,
+ },
+};
+
+static struct clk_regmap meson8b_cts_i958 = {
+ .data = &(struct clk_regmap_mux_data){
+ .offset = HHI_AUD_CLK_CNTL2,
+ .mask = 0x1,
+ .shift = 27,
+ },
+ .hw.init = &(struct clk_init_data){
+ .name = "cts_i958",
+ .ops = &clk_regmap_mux_ops,
+ .parent_hws = (const struct clk_hw *[]) {
+ &meson8b_cts_amclk.hw,
+ &meson8b_cts_mclk_i958.hw
+ },
+ .num_parents = 2,
+ /*
+ * The parent is specific to origin of the audio data. Let the
+ * consumer choose the appropriate parent.
+ */
+ .flags = CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT,
+ },
+};
+
+#define MESON_GATE(_name, _reg, _bit) \
+ MESON_PCLK(_name, _reg, _bit, &meson8b_clk81.hw)
+
/* Everything Else (EE) domain gates */
static MESON_GATE(meson8b_ddr, HHI_GCLK_MPEG0, 0);
@@ -2432,6 +2846,13 @@ static struct clk_hw_onecell_data meson8_hw_onecell_data = {
[CLKID_VDEC_HEVC_DIV] = &meson8b_vdec_hevc_div.hw,
[CLKID_VDEC_HEVC_EN] = &meson8b_vdec_hevc_en.hw,
[CLKID_VDEC_HEVC] = &meson8b_vdec_hevc.hw,
+ [CLKID_CTS_AMCLK_SEL] = &meson8b_cts_amclk_sel.hw,
+ [CLKID_CTS_AMCLK_DIV] = &meson8b_cts_amclk_div.hw,
+ [CLKID_CTS_AMCLK] = &meson8b_cts_amclk.hw,
+ [CLKID_CTS_MCLK_I958_SEL] = &meson8b_cts_mclk_i958_sel.hw,
+ [CLKID_CTS_MCLK_I958_DIV] = &meson8b_cts_mclk_i958_div.hw,
+ [CLKID_CTS_MCLK_I958] = &meson8b_cts_mclk_i958.hw,
+ [CLKID_CTS_I958] = &meson8b_cts_i958.hw,
[CLK_NR_CLKS] = NULL,
},
.num = CLK_NR_CLKS,
@@ -2641,6 +3062,13 @@ static struct clk_hw_onecell_data meson8b_hw_onecell_data = {
[CLKID_VDEC_HEVC_DIV] = &meson8b_vdec_hevc_div.hw,
[CLKID_VDEC_HEVC_EN] = &meson8b_vdec_hevc_en.hw,
[CLKID_VDEC_HEVC] = &meson8b_vdec_hevc.hw,
+ [CLKID_CTS_AMCLK_SEL] = &meson8b_cts_amclk_sel.hw,
+ [CLKID_CTS_AMCLK_DIV] = &meson8b_cts_amclk_div.hw,
+ [CLKID_CTS_AMCLK] = &meson8b_cts_amclk.hw,
+ [CLKID_CTS_MCLK_I958_SEL] = &meson8b_cts_mclk_i958_sel.hw,
+ [CLKID_CTS_MCLK_I958_DIV] = &meson8b_cts_mclk_i958_div.hw,
+ [CLKID_CTS_MCLK_I958] = &meson8b_cts_mclk_i958.hw,
+ [CLKID_CTS_I958] = &meson8b_cts_i958.hw,
[CLK_NR_CLKS] = NULL,
},
.num = CLK_NR_CLKS,
@@ -2852,6 +3280,13 @@ static struct clk_hw_onecell_data meson8m2_hw_onecell_data = {
[CLKID_VDEC_HEVC_DIV] = &meson8b_vdec_hevc_div.hw,
[CLKID_VDEC_HEVC_EN] = &meson8b_vdec_hevc_en.hw,
[CLKID_VDEC_HEVC] = &meson8b_vdec_hevc.hw,
+ [CLKID_CTS_AMCLK_SEL] = &meson8b_cts_amclk_sel.hw,
+ [CLKID_CTS_AMCLK_DIV] = &meson8b_cts_amclk_div.hw,
+ [CLKID_CTS_AMCLK] = &meson8b_cts_amclk.hw,
+ [CLKID_CTS_MCLK_I958_SEL] = &meson8b_cts_mclk_i958_sel.hw,
+ [CLKID_CTS_MCLK_I958_DIV] = &meson8b_cts_mclk_i958_div.hw,
+ [CLKID_CTS_MCLK_I958] = &meson8b_cts_mclk_i958.hw,
+ [CLKID_CTS_I958] = &meson8b_cts_i958.hw,
[CLK_NR_CLKS] = NULL,
},
.num = CLK_NR_CLKS,
@@ -3041,6 +3476,13 @@ static struct clk_regmap *const meson8b_clk_regmaps[] = {
&meson8b_vdec_hevc_div,
&meson8b_vdec_hevc_en,
&meson8b_vdec_hevc,
+ &meson8b_cts_amclk,
+ &meson8b_cts_amclk_sel,
+ &meson8b_cts_amclk_div,
+ &meson8b_cts_mclk_i958_sel,
+ &meson8b_cts_mclk_i958_div,
+ &meson8b_cts_mclk_i958,
+ &meson8b_cts_i958,
};
static const struct meson8b_clk_reset_line {
diff --git a/drivers/clk/meson/meson8b.h b/drivers/clk/meson/meson8b.h
index ed37196187e6..c889fbeec30f 100644
--- a/drivers/clk/meson/meson8b.h
+++ b/drivers/clk/meson/meson8b.h
@@ -30,7 +30,9 @@
#define HHI_SYS_CPU_CLK_CNTL1 0x15c /* 0x57 offset in data sheet */
#define HHI_VID_CLK_DIV 0x164 /* 0x59 offset in data sheet */
#define HHI_MPEG_CLK_CNTL 0x174 /* 0x5d offset in data sheet */
+#define HHI_AUD_CLK_CNTL 0x178 /* 0x5e offset in data sheet */
#define HHI_VID_CLK_CNTL 0x17c /* 0x5f offset in data sheet */
+#define HHI_AUD_CLK_CNTL2 0x190 /* 0x64 offset in data sheet */
#define HHI_VID_CLK_CNTL2 0x194 /* 0x65 offset in data sheet */
#define HHI_VID_DIVIDER_CNTL 0x198 /* 0x66 offset in data sheet */
#define HHI_SYS_CPU_CLK_CNTL0 0x19c /* 0x67 offset in data sheet */
@@ -171,8 +173,12 @@
#define CLKID_VDEC_HEVC_SEL 203
#define CLKID_VDEC_HEVC_DIV 204
#define CLKID_VDEC_HEVC_EN 205
+#define CLKID_CTS_AMCLK_SEL 207
+#define CLKID_CTS_AMCLK_DIV 208
+#define CLKID_CTS_MCLK_I958_SEL 210
+#define CLKID_CTS_MCLK_I958_DIV 211
-#define CLK_NR_CLKS 207
+#define CLK_NR_CLKS 214
/*
* include the CLKID and RESETID that have
diff --git a/drivers/clk/mmp/clk-frac.c b/drivers/clk/mmp/clk-frac.c
index cb43d54735b0..90bf181f191a 100644
--- a/drivers/clk/mmp/clk-frac.c
+++ b/drivers/clk/mmp/clk-frac.c
@@ -78,11 +78,10 @@ static int clk_factor_set_rate(struct clk_hw *hw, unsigned long drate,
struct mmp_clk_factor_masks *masks = factor->masks;
int i;
unsigned long val;
- unsigned long prev_rate, rate = 0;
+ unsigned long rate = 0;
unsigned long flags = 0;
for (i = 0; i < factor->ftbl_cnt; i++) {
- prev_rate = rate;
rate = (((prate / 10000) * factor->ftbl[i].den) /
(factor->ftbl[i].num * factor->masks->factor)) * 10000;
if (rate > drate)
diff --git a/drivers/clk/mvebu/kirkwood.c b/drivers/clk/mvebu/kirkwood.c
index 35af3aa18f1c..47680237d0be 100644
--- a/drivers/clk/mvebu/kirkwood.c
+++ b/drivers/clk/mvebu/kirkwood.c
@@ -185,6 +185,11 @@ static void __init mv88f6180_get_clk_ratio(
}
}
+static u32 __init mv98dx1135_get_tclk_freq(void __iomem *sar)
+{
+ return 166666667;
+}
+
static const struct coreclk_soc_desc kirkwood_coreclks = {
.get_tclk_freq = kirkwood_get_tclk_freq,
.get_cpu_freq = kirkwood_get_cpu_freq,
@@ -201,6 +206,14 @@ static const struct coreclk_soc_desc mv88f6180_coreclks = {
.num_ratios = ARRAY_SIZE(kirkwood_coreclk_ratios),
};
+static const struct coreclk_soc_desc mv98dx1135_coreclks = {
+ .get_tclk_freq = mv98dx1135_get_tclk_freq,
+ .get_cpu_freq = kirkwood_get_cpu_freq,
+ .get_clk_ratio = kirkwood_get_clk_ratio,
+ .ratios = kirkwood_coreclk_ratios,
+ .num_ratios = ARRAY_SIZE(kirkwood_coreclk_ratios),
+};
+
/*
* Clock Gating Control
*/
@@ -325,6 +338,8 @@ static void __init kirkwood_clk_init(struct device_node *np)
if (of_device_is_compatible(np, "marvell,mv88f6180-core-clock"))
mvebu_coreclk_setup(np, &mv88f6180_coreclks);
+ else if (of_device_is_compatible(np, "marvell,mv98dx1135-core-clock"))
+ mvebu_coreclk_setup(np, &mv98dx1135_coreclks);
else
mvebu_coreclk_setup(np, &kirkwood_coreclks);
@@ -339,3 +354,5 @@ CLK_OF_DECLARE(kirkwood_clk, "marvell,kirkwood-core-clock",
kirkwood_clk_init);
CLK_OF_DECLARE(mv88f6180_clk, "marvell,mv88f6180-core-clock",
kirkwood_clk_init);
+CLK_OF_DECLARE(98dx1135_clk, "marvell,mv98dx1135-core-clock",
+ kirkwood_clk_init);
diff --git a/drivers/clk/qcom/clk-rpmh.c b/drivers/clk/qcom/clk-rpmh.c
index c3fd632af119..a32bfaeb7e61 100644
--- a/drivers/clk/qcom/clk-rpmh.c
+++ b/drivers/clk/qcom/clk-rpmh.c
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0
/*
- * Copyright (c) 2018, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
*/
#include <linux/clk-provider.h>
@@ -12,23 +12,13 @@
#include <linux/platform_device.h>
#include <soc/qcom/cmd-db.h>
#include <soc/qcom/rpmh.h>
+#include <soc/qcom/tcs.h>
#include <dt-bindings/clock/qcom,rpmh.h>
#define CLK_RPMH_ARC_EN_OFFSET 0
#define CLK_RPMH_VRM_EN_OFFSET 4
-#define BCM_TCS_CMD_COMMIT_MASK 0x40000000
-#define BCM_TCS_CMD_VALID_SHIFT 29
-#define BCM_TCS_CMD_VOTE_MASK 0x3fff
-#define BCM_TCS_CMD_VOTE_SHIFT 0
-
-#define BCM_TCS_CMD(valid, vote) \
- (BCM_TCS_CMD_COMMIT_MASK | \
- ((valid) << BCM_TCS_CMD_VALID_SHIFT) | \
- ((vote & BCM_TCS_CMD_VOTE_MASK) \
- << BCM_TCS_CMD_VOTE_SHIFT))
-
/**
* struct bcm_db - Auxiliary data pertaining to each Bus Clock Manager(BCM)
* @unit: divisor used to convert Hz value to an RPMh msg
@@ -269,7 +259,7 @@ static int clk_rpmh_bcm_send_cmd(struct clk_rpmh *c, bool enable)
}
cmd.addr = c->res_addr;
- cmd.data = BCM_TCS_CMD(enable, cmd_state);
+ cmd.data = BCM_TCS_CMD(1, enable, 0, cmd_state);
ret = rpmh_write_async(c->dev, RPMH_ACTIVE_ONLY_STATE, &cmd, 1);
if (ret) {
diff --git a/drivers/clk/qcom/gcc-msm8996.c b/drivers/clk/qcom/gcc-msm8996.c
index d2f39a972cad..d004cdaa0e39 100644
--- a/drivers/clk/qcom/gcc-msm8996.c
+++ b/drivers/clk/qcom/gcc-msm8996.c
@@ -130,22 +130,6 @@ static const char * const gcc_xo_gpll0_gpll4_gpll0_early_div[] = {
"gpll0_early_div"
};
-static const struct parent_map gcc_xo_gpll0_gpll2_gpll3_gpll0_early_div_map[] = {
- { P_XO, 0 },
- { P_GPLL0, 1 },
- { P_GPLL2, 2 },
- { P_GPLL3, 3 },
- { P_GPLL0_EARLY_DIV, 6 }
-};
-
-static const char * const gcc_xo_gpll0_gpll2_gpll3_gpll0_early_div[] = {
- "xo",
- "gpll0",
- "gpll2",
- "gpll3",
- "gpll0_early_div"
-};
-
static const struct parent_map gcc_xo_gpll0_gpll1_early_div_gpll1_gpll4_gpll0_early_div_map[] = {
{ P_XO, 0 },
{ P_GPLL0, 1 },
@@ -184,26 +168,6 @@ static const char * const gcc_xo_gpll0_gpll2_gpll3_gpll1_gpll2_early_gpll0_early
"gpll0_early_div"
};
-static const struct parent_map gcc_xo_gpll0_gpll2_gpll3_gpll1_gpll4_gpll0_early_div_map[] = {
- { P_XO, 0 },
- { P_GPLL0, 1 },
- { P_GPLL2, 2 },
- { P_GPLL3, 3 },
- { P_GPLL1, 4 },
- { P_GPLL4, 5 },
- { P_GPLL0_EARLY_DIV, 6 }
-};
-
-static const char * const gcc_xo_gpll0_gpll2_gpll3_gpll1_gpll4_gpll0_early_div[] = {
- "xo",
- "gpll0",
- "gpll2",
- "gpll3",
- "gpll1",
- "gpll4",
- "gpll0_early_div"
-};
-
static struct clk_fixed_factor xo = {
.mult = 1,
.div = 1,
diff --git a/drivers/clk/qcom/gcc-qcs404.c b/drivers/clk/qcom/gcc-qcs404.c
index a54807eb3b28..29cf464dd2c8 100644
--- a/drivers/clk/qcom/gcc-qcs404.c
+++ b/drivers/clk/qcom/gcc-qcs404.c
@@ -2766,6 +2766,13 @@ static const struct qcom_reset_map gcc_qcs404_resets[] = {
[GCC_PCIE_0_PHY_BCR] = { 0x3e004 },
[GCC_PCIE_0_LINK_DOWN_BCR] = { 0x3e038 },
[GCC_PCIEPHY_0_PHY_BCR] = { 0x3e03c },
+ [GCC_PCIE_0_AXI_MASTER_STICKY_ARES] = { 0x3e040, 6},
+ [GCC_PCIE_0_AHB_ARES] = { 0x3e040, 5 },
+ [GCC_PCIE_0_AXI_SLAVE_ARES] = { 0x3e040, 4 },
+ [GCC_PCIE_0_AXI_MASTER_ARES] = { 0x3e040, 3 },
+ [GCC_PCIE_0_CORE_STICKY_ARES] = { 0x3e040, 2 },
+ [GCC_PCIE_0_SLEEP_ARES] = { 0x3e040, 1 },
+ [GCC_PCIE_0_PIPE_ARES] = { 0x3e040, 0 },
[GCC_EMAC_BCR] = { 0x4e000 },
};
diff --git a/drivers/clk/qcom/gdsc.c b/drivers/clk/qcom/gdsc.c
index 679bc7d8950a..a250f59708d8 100644
--- a/drivers/clk/qcom/gdsc.c
+++ b/drivers/clk/qcom/gdsc.c
@@ -141,7 +141,9 @@ static int gdsc_toggle_logic(struct gdsc *sc, enum gdsc_status status)
udelay(1);
}
- return gdsc_poll_status(sc, status);
+ ret = gdsc_poll_status(sc, status);
+ WARN(ret, "%s status stuck at 'o%s'", sc->pd.name, status ? "ff" : "n");
+ return ret;
}
static inline int gdsc_deassert_reset(struct gdsc *sc)
diff --git a/drivers/clk/renesas/clk-div6.c b/drivers/clk/renesas/clk-div6.c
index e98a9f5b3c90..5ca183e70166 100644
--- a/drivers/clk/renesas/clk-div6.c
+++ b/drivers/clk/renesas/clk-div6.c
@@ -30,8 +30,8 @@
* @div: divisor value (1-64)
* @src_shift: Shift to access the register bits to select the parent clock
* @src_width: Number of register bits to select the parent clock (may be 0)
- * @parents: Array to map from valid parent clocks indices to hardware indices
* @nb: Notifier block to save/restore clock state for system resume
+ * @parents: Array to map from valid parent clocks indices to hardware indices
*/
struct div6_clock {
struct clk_hw hw;
@@ -39,8 +39,8 @@ struct div6_clock {
unsigned int div;
u32 src_shift;
u32 src_width;
- u8 *parents;
struct notifier_block nb;
+ u8 parents[];
};
#define to_div6_clock(_hw) container_of(_hw, struct div6_clock, hw)
@@ -221,17 +221,10 @@ struct clk * __init cpg_div6_register(const char *name,
struct clk *clk;
unsigned int i;
- clock = kzalloc(sizeof(*clock), GFP_KERNEL);
+ clock = kzalloc(struct_size(clock, parents, num_parents), GFP_KERNEL);
if (!clock)
return ERR_PTR(-ENOMEM);
- clock->parents = kmalloc_array(num_parents, sizeof(*clock->parents),
- GFP_KERNEL);
- if (!clock->parents) {
- clk = ERR_PTR(-ENOMEM);
- goto free_clock;
- }
-
clock->reg = reg;
/*
@@ -259,7 +252,7 @@ struct clk * __init cpg_div6_register(const char *name,
pr_err("%s: invalid number of parents for DIV6 clock %s\n",
__func__, name);
clk = ERR_PTR(-EINVAL);
- goto free_parents;
+ goto free_clock;
}
/* Filter out invalid parents */
@@ -282,7 +275,7 @@ struct clk * __init cpg_div6_register(const char *name,
clk = clk_register(NULL, &clock->hw);
if (IS_ERR(clk))
- goto free_parents;
+ goto free_clock;
if (notifiers) {
clock->nb.notifier_call = cpg_div6_clock_notifier_call;
@@ -291,8 +284,6 @@ struct clk * __init cpg_div6_register(const char *name,
return clk;
-free_parents:
- kfree(clock->parents);
free_clock:
kfree(clock);
return clk;
diff --git a/drivers/clk/renesas/clk-mstp.c b/drivers/clk/renesas/clk-mstp.c
index 92ece221b0d4..2db9093546c6 100644
--- a/drivers/clk/renesas/clk-mstp.c
+++ b/drivers/clk/renesas/clk-mstp.c
@@ -30,11 +30,12 @@
/**
* struct mstp_clock_group - MSTP gating clocks group
*
- * @data: clocks in this group
+ * @data: clock specifier translation for clocks in this group
* @smstpcr: module stop control register
* @mstpsr: module stop status register (optional)
* @lock: protects writes to SMSTPCR
* @width_8bit: registers are 8-bit, not 32-bit
+ * @clks: clocks in this group
*/
struct mstp_clock_group {
struct clk_onecell_data data;
@@ -42,6 +43,7 @@ struct mstp_clock_group {
void __iomem *mstpsr;
spinlock_t lock;
bool width_8bit;
+ struct clk *clks[];
};
/**
@@ -186,14 +188,13 @@ static void __init cpg_mstp_clocks_init(struct device_node *np)
struct clk **clks;
unsigned int i;
- group = kzalloc(sizeof(*group), GFP_KERNEL);
- clks = kmalloc_array(MSTP_MAX_CLOCKS, sizeof(*clks), GFP_KERNEL);
- if (group == NULL || clks == NULL) {
+ group = kzalloc(struct_size(group, clks, MSTP_MAX_CLOCKS), GFP_KERNEL);
+ if (group == NULL) {
kfree(group);
- kfree(clks);
return;
}
+ clks = group->clks;
spin_lock_init(&group->lock);
group->data.clks = clks;
@@ -203,7 +204,6 @@ static void __init cpg_mstp_clocks_init(struct device_node *np)
if (group->smstpcr == NULL) {
pr_err("%s: failed to remap SMSTPCR\n", __func__);
kfree(group);
- kfree(clks);
return;
}
@@ -297,16 +297,12 @@ found:
return PTR_ERR(clk);
error = pm_clk_create(dev);
- if (error) {
- dev_err(dev, "pm_clk_create failed %d\n", error);
+ if (error)
goto fail_put;
- }
error = pm_clk_add_clk(dev, clk);
- if (error) {
- dev_err(dev, "pm_clk_add_clk %pC failed %d\n", clk, error);
+ if (error)
goto fail_destroy;
- }
return 0;
diff --git a/drivers/clk/renesas/r8a77470-cpg-mssr.c b/drivers/clk/renesas/r8a77470-cpg-mssr.c
index ab0fb10b6bf0..d81ae65f0d18 100644
--- a/drivers/clk/renesas/r8a77470-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a77470-cpg-mssr.c
@@ -175,7 +175,7 @@ static const unsigned int r8a77470_crit_mod_clks[] __initconst = {
*---------------------------------------------------
* 0 0 20 x80 x78 x50
* 0 1 26 x60 x60 x56
- * 1 0 Prohibitted setting
+ * 1 0 Prohibited setting
* 1 1 30 x52 x52 x50
*
* *1 : Table 7.4 indicates VCO output (PLL0 = VCO)
diff --git a/drivers/clk/renesas/r8a774a1-cpg-mssr.c b/drivers/clk/renesas/r8a774a1-cpg-mssr.c
index 76ed7d1bae36..e05bfa200480 100644
--- a/drivers/clk/renesas/r8a774a1-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a774a1-cpg-mssr.c
@@ -113,6 +113,11 @@ static const struct cpg_core_clk r8a774a1_core_clks[] __initconst = {
};
static const struct mssr_mod_clk r8a774a1_mod_clks[] __initconst = {
+ DEF_MOD("tmu4", 121, R8A774A1_CLK_S0D6),
+ DEF_MOD("tmu3", 122, R8A774A1_CLK_S3D2),
+ DEF_MOD("tmu2", 123, R8A774A1_CLK_S3D2),
+ DEF_MOD("tmu1", 124, R8A774A1_CLK_S3D2),
+ DEF_MOD("tmu0", 125, R8A774A1_CLK_CP),
DEF_MOD("fdp1-0", 119, R8A774A1_CLK_S0D1),
DEF_MOD("scif5", 202, R8A774A1_CLK_S3D4),
DEF_MOD("scif4", 203, R8A774A1_CLK_S3D4),
diff --git a/drivers/clk/renesas/r8a7795-cpg-mssr.c b/drivers/clk/renesas/r8a7795-cpg-mssr.c
index 9e9a6f2c31e8..fbc8c75f4314 100644
--- a/drivers/clk/renesas/r8a7795-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a7795-cpg-mssr.c
@@ -138,6 +138,7 @@ static struct mssr_mod_clk r8a7795_mod_clks[] __initdata = {
DEF_MOD("cmt2", 301, R8A7795_CLK_R),
DEF_MOD("cmt1", 302, R8A7795_CLK_R),
DEF_MOD("cmt0", 303, R8A7795_CLK_R),
+ DEF_MOD("tpu0", 304, R8A7795_CLK_S3D4),
DEF_MOD("scif2", 310, R8A7795_CLK_S3D4),
DEF_MOD("sdif3", 311, R8A7795_CLK_SD3),
DEF_MOD("sdif2", 312, R8A7795_CLK_SD2),
@@ -201,6 +202,10 @@ static struct mssr_mod_clk r8a7795_mod_clks[] __initdata = {
DEF_MOD("ehci0", 703, R8A7795_CLK_S3D2),
DEF_MOD("hsusb", 704, R8A7795_CLK_S3D2),
DEF_MOD("hsusb3", 705, R8A7795_CLK_S3D2),
+ DEF_MOD("cmm3", 708, R8A7795_CLK_S2D1),
+ DEF_MOD("cmm2", 709, R8A7795_CLK_S2D1),
+ DEF_MOD("cmm1", 710, R8A7795_CLK_S2D1),
+ DEF_MOD("cmm0", 711, R8A7795_CLK_S2D1),
DEF_MOD("csi21", 713, R8A7795_CLK_CSI0), /* ES1.x */
DEF_MOD("csi20", 714, R8A7795_CLK_CSI0),
DEF_MOD("csi41", 715, R8A7795_CLK_CSI0),
diff --git a/drivers/clk/renesas/r8a7796-cpg-mssr.c b/drivers/clk/renesas/r8a7796-cpg-mssr.c
index d8e9af5d9ae9..90cc6a102602 100644
--- a/drivers/clk/renesas/r8a7796-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a7796-cpg-mssr.c
@@ -134,6 +134,7 @@ static const struct mssr_mod_clk r8a7796_mod_clks[] __initconst = {
DEF_MOD("cmt2", 301, R8A7796_CLK_R),
DEF_MOD("cmt1", 302, R8A7796_CLK_R),
DEF_MOD("cmt0", 303, R8A7796_CLK_R),
+ DEF_MOD("tpu0", 304, R8A7796_CLK_S3D4),
DEF_MOD("scif2", 310, R8A7796_CLK_S3D4),
DEF_MOD("sdif3", 311, R8A7796_CLK_SD3),
DEF_MOD("sdif2", 312, R8A7796_CLK_SD2),
@@ -180,6 +181,9 @@ static const struct mssr_mod_clk r8a7796_mod_clks[] __initconst = {
DEF_MOD("ehci1", 702, R8A7796_CLK_S3D2),
DEF_MOD("ehci0", 703, R8A7796_CLK_S3D2),
DEF_MOD("hsusb", 704, R8A7796_CLK_S3D2),
+ DEF_MOD("cmm2", 709, R8A7796_CLK_S2D1),
+ DEF_MOD("cmm1", 710, R8A7796_CLK_S2D1),
+ DEF_MOD("cmm0", 711, R8A7796_CLK_S2D1),
DEF_MOD("csi20", 714, R8A7796_CLK_CSI0),
DEF_MOD("csi40", 716, R8A7796_CLK_CSI0),
DEF_MOD("du2", 722, R8A7796_CLK_S2D1),
diff --git a/drivers/clk/renesas/r8a77965-cpg-mssr.c b/drivers/clk/renesas/r8a77965-cpg-mssr.c
index 8f87e314d949..b4e8c5b7d515 100644
--- a/drivers/clk/renesas/r8a77965-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a77965-cpg-mssr.c
@@ -132,6 +132,7 @@ static const struct mssr_mod_clk r8a77965_mod_clks[] __initconst = {
DEF_MOD("cmt2", 301, R8A77965_CLK_R),
DEF_MOD("cmt1", 302, R8A77965_CLK_R),
DEF_MOD("cmt0", 303, R8A77965_CLK_R),
+ DEF_MOD("tpu0", 304, R8A77965_CLK_S3D4),
DEF_MOD("scif2", 310, R8A77965_CLK_S3D4),
DEF_MOD("sdif3", 311, R8A77965_CLK_SD3),
DEF_MOD("sdif2", 312, R8A77965_CLK_SD2),
@@ -179,6 +180,9 @@ static const struct mssr_mod_clk r8a77965_mod_clks[] __initconst = {
DEF_MOD("ehci1", 702, R8A77965_CLK_S3D2),
DEF_MOD("ehci0", 703, R8A77965_CLK_S3D2),
DEF_MOD("hsusb", 704, R8A77965_CLK_S3D2),
+ DEF_MOD("cmm3", 708, R8A77965_CLK_S2D1),
+ DEF_MOD("cmm1", 710, R8A77965_CLK_S2D1),
+ DEF_MOD("cmm0", 711, R8A77965_CLK_S2D1),
DEF_MOD("csi20", 714, R8A77965_CLK_CSI0),
DEF_MOD("csi40", 716, R8A77965_CLK_CSI0),
DEF_MOD("du3", 721, R8A77965_CLK_S2D1),
diff --git a/drivers/clk/renesas/r8a77990-cpg-mssr.c b/drivers/clk/renesas/r8a77990-cpg-mssr.c
index 9570404baa58..ceabf55c21c2 100644
--- a/drivers/clk/renesas/r8a77990-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a77990-cpg-mssr.c
@@ -183,6 +183,8 @@ static const struct mssr_mod_clk r8a77990_mod_clks[] __initconst = {
DEF_MOD("ehci0", 703, R8A77990_CLK_S3D2),
DEF_MOD("hsusb", 704, R8A77990_CLK_S3D2),
+ DEF_MOD("cmm1", 710, R8A77990_CLK_S1D1),
+ DEF_MOD("cmm0", 711, R8A77990_CLK_S1D1),
DEF_MOD("csi40", 716, R8A77990_CLK_CSI0),
DEF_MOD("du1", 723, R8A77990_CLK_S1D1),
DEF_MOD("du0", 724, R8A77990_CLK_S1D1),
diff --git a/drivers/clk/renesas/r8a77995-cpg-mssr.c b/drivers/clk/renesas/r8a77995-cpg-mssr.c
index 68707277b17b..962bb337f2e7 100644
--- a/drivers/clk/renesas/r8a77995-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a77995-cpg-mssr.c
@@ -146,6 +146,8 @@ static const struct mssr_mod_clk r8a77995_mod_clks[] __initconst = {
DEF_MOD("vspbs", 627, R8A77995_CLK_S0D1),
DEF_MOD("ehci0", 703, R8A77995_CLK_S3D2),
DEF_MOD("hsusb", 704, R8A77995_CLK_S3D2),
+ DEF_MOD("cmm1", 710, R8A77995_CLK_S1D1),
+ DEF_MOD("cmm0", 711, R8A77995_CLK_S1D1),
DEF_MOD("du1", 723, R8A77995_CLK_S1D1),
DEF_MOD("du0", 724, R8A77995_CLK_S1D1),
DEF_MOD("lvds", 727, R8A77995_CLK_S2D1),
diff --git a/drivers/clk/renesas/r9a06g032-clocks.c b/drivers/clk/renesas/r9a06g032-clocks.c
index 7d042183aa37..b33e1383efe3 100644
--- a/drivers/clk/renesas/r9a06g032-clocks.c
+++ b/drivers/clk/renesas/r9a06g032-clocks.c
@@ -17,6 +17,8 @@
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/platform_device.h>
+#include <linux/pm_clock.h>
+#include <linux/pm_domain.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <dt-bindings/clock/r9a06g032-sysctrl.h>
@@ -29,6 +31,7 @@ struct r9a06g032_gate {
/* This is used to describe a clock for instantiation */
struct r9a06g032_clkdesc {
const char *name;
+ uint32_t managed: 1;
uint32_t type: 3;
uint32_t index: 8;
uint32_t source : 8; /* source index + 1 (0 == none) */
@@ -61,7 +64,11 @@ struct r9a06g032_clkdesc {
#define D_GATE(_idx, _n, _src, ...) \
{ .type = K_GATE, .index = R9A06G032_##_idx, \
.source = 1 + R9A06G032_##_src, .name = _n, \
- .gate = I_GATE(__VA_ARGS__), }
+ .gate = I_GATE(__VA_ARGS__) }
+#define D_MODULE(_idx, _n, _src, ...) \
+ { .type = K_GATE, .index = R9A06G032_##_idx, \
+ .source = 1 + R9A06G032_##_src, .name = _n, \
+ .managed = 1, .gate = I_GATE(__VA_ARGS__) }
#define D_ROOT(_idx, _n, _mul, _div) \
{ .type = K_FFC, .index = R9A06G032_##_idx, .name = _n, \
.div = _div, .mul = _mul }
@@ -122,7 +129,7 @@ enum { K_GATE = 0, K_FFC, K_DIV, K_BITSEL, K_DUALGATE };
#define R9A06G032_CLOCK_COUNT (R9A06G032_UART_GROUP_34567 + 1)
-static const struct r9a06g032_clkdesc r9a06g032_clocks[] __initconst = {
+static const struct r9a06g032_clkdesc r9a06g032_clocks[] = {
D_ROOT(CLKOUT, "clkout", 25, 1),
D_ROOT(CLK_PLL_USB, "clk_pll_usb", 12, 10),
D_FFC(CLKOUT_D10, "clkout_d10", CLKOUT, 10),
@@ -171,7 +178,7 @@ static const struct r9a06g032_clkdesc r9a06g032_clocks[] __initconst = {
D_GATE(CLK_P6_PG2, "clk_p6_pg2", DIV_P6_PG, 0x8a3, 0x8a4, 0x8a5, 0, 0xb61, 0, 0),
D_GATE(CLK_P6_PG3, "clk_p6_pg3", DIV_P6_PG, 0x8a6, 0x8a7, 0x8a8, 0, 0xb62, 0, 0),
D_GATE(CLK_P6_PG4, "clk_p6_pg4", DIV_P6_PG, 0x8a9, 0x8aa, 0x8ab, 0, 0xb63, 0, 0),
- D_GATE(CLK_PCI_USB, "clk_pci_usb", CLKOUT_D40, 0xe6, 0, 0, 0, 0, 0, 0),
+ D_MODULE(CLK_PCI_USB, "clk_pci_usb", CLKOUT_D40, 0xe6, 0, 0, 0, 0, 0, 0),
D_GATE(CLK_QSPI0, "clk_qspi0", DIV_QSPI0, 0x2a4, 0x2a5, 0, 0, 0, 0, 0),
D_GATE(CLK_QSPI1, "clk_qspi1", DIV_QSPI1, 0x484, 0x485, 0, 0, 0, 0, 0),
D_GATE(CLK_RGMII_REF, "clk_rgmii_ref", CLKOUT_D8, 0x340, 0, 0, 0, 0, 0, 0),
@@ -188,17 +195,17 @@ static const struct r9a06g032_clkdesc r9a06g032_clocks[] __initconst = {
D_GATE(CLK_SPI5, "clk_spi5", DIV_P4_PG, 0x822, 0x823, 0, 0, 0, 0, 0),
D_GATE(CLK_SWITCH, "clk_switch", DIV_SWITCH, 0x982, 0x983, 0, 0, 0, 0, 0),
D_DIV(DIV_MOTOR, "div_motor", CLKOUT_D5, 84, 2, 8),
- D_GATE(HCLK_ECAT125, "hclk_ecat125", CLKOUT_D8, 0x400, 0x401, 0, 0x402, 0, 0x440, 0x441),
- D_GATE(HCLK_PINCONFIG, "hclk_pinconfig", CLKOUT_D40, 0x740, 0x741, 0x742, 0, 0xae0, 0, 0),
- D_GATE(HCLK_SERCOS, "hclk_sercos", CLKOUT_D10, 0x420, 0x422, 0, 0x421, 0, 0x460, 0x461),
- D_GATE(HCLK_SGPIO2, "hclk_sgpio2", DIV_P5_PG, 0x8c3, 0x8c4, 0x8c5, 0, 0xb41, 0, 0),
- D_GATE(HCLK_SGPIO3, "hclk_sgpio3", DIV_P5_PG, 0x8c6, 0x8c7, 0x8c8, 0, 0xb42, 0, 0),
- D_GATE(HCLK_SGPIO4, "hclk_sgpio4", DIV_P5_PG, 0x8c9, 0x8ca, 0x8cb, 0, 0xb43, 0, 0),
- D_GATE(HCLK_TIMER0, "hclk_timer0", CLKOUT_D40, 0x743, 0x744, 0x745, 0, 0xae1, 0, 0),
- D_GATE(HCLK_TIMER1, "hclk_timer1", CLKOUT_D40, 0x746, 0x747, 0x748, 0, 0xae2, 0, 0),
- D_GATE(HCLK_USBF, "hclk_usbf", CLKOUT_D8, 0xe3, 0, 0, 0xe4, 0, 0x102, 0x103),
- D_GATE(HCLK_USBH, "hclk_usbh", CLKOUT_D8, 0xe0, 0xe1, 0, 0xe2, 0, 0x100, 0x101),
- D_GATE(HCLK_USBPM, "hclk_usbpm", CLKOUT_D8, 0xe5, 0, 0, 0, 0, 0, 0),
+ D_MODULE(HCLK_ECAT125, "hclk_ecat125", CLKOUT_D8, 0x400, 0x401, 0, 0x402, 0, 0x440, 0x441),
+ D_MODULE(HCLK_PINCONFIG, "hclk_pinconfig", CLKOUT_D40, 0x740, 0x741, 0x742, 0, 0xae0, 0, 0),
+ D_MODULE(HCLK_SERCOS, "hclk_sercos", CLKOUT_D10, 0x420, 0x422, 0, 0x421, 0, 0x460, 0x461),
+ D_MODULE(HCLK_SGPIO2, "hclk_sgpio2", DIV_P5_PG, 0x8c3, 0x8c4, 0x8c5, 0, 0xb41, 0, 0),
+ D_MODULE(HCLK_SGPIO3, "hclk_sgpio3", DIV_P5_PG, 0x8c6, 0x8c7, 0x8c8, 0, 0xb42, 0, 0),
+ D_MODULE(HCLK_SGPIO4, "hclk_sgpio4", DIV_P5_PG, 0x8c9, 0x8ca, 0x8cb, 0, 0xb43, 0, 0),
+ D_MODULE(HCLK_TIMER0, "hclk_timer0", CLKOUT_D40, 0x743, 0x744, 0x745, 0, 0xae1, 0, 0),
+ D_MODULE(HCLK_TIMER1, "hclk_timer1", CLKOUT_D40, 0x746, 0x747, 0x748, 0, 0xae2, 0, 0),
+ D_MODULE(HCLK_USBF, "hclk_usbf", CLKOUT_D8, 0xe3, 0, 0, 0xe4, 0, 0x102, 0x103),
+ D_MODULE(HCLK_USBH, "hclk_usbh", CLKOUT_D8, 0xe0, 0xe1, 0, 0xe2, 0, 0x100, 0x101),
+ D_MODULE(HCLK_USBPM, "hclk_usbpm", CLKOUT_D8, 0xe5, 0, 0, 0, 0, 0, 0),
D_GATE(CLK_48_PG_F, "clk_48_pg_f", CLK_48, 0x78c, 0x78d, 0, 0x78e, 0, 0xb04, 0xb05),
D_GATE(CLK_48_PG4, "clk_48_pg4", CLK_48, 0x789, 0x78a, 0x78b, 0, 0xb03, 0, 0),
D_FFC(CLK_DDRPHY_PLLCLK_D4, "clk_ddrphy_pllclk_d4", CLK_DDRPHY_PLLCLK, 4),
@@ -208,13 +215,13 @@ static const struct r9a06g032_clkdesc r9a06g032_clocks[] __initconst = {
D_FFC(CLK_REF_SYNC_D8, "clk_ref_sync_d8", CLK_REF_SYNC, 8),
D_FFC(CLK_SERCOS100_D2, "clk_sercos100_d2", CLK_SERCOS100, 2),
D_DIV(DIV_CA7, "div_ca7", CLK_REF_SYNC, 57, 1, 4, 1, 2, 4),
- D_GATE(HCLK_CAN0, "hclk_can0", CLK_48, 0x783, 0x784, 0x785, 0, 0xb01, 0, 0),
- D_GATE(HCLK_CAN1, "hclk_can1", CLK_48, 0x786, 0x787, 0x788, 0, 0xb02, 0, 0),
- D_GATE(HCLK_DELTASIGMA, "hclk_deltasigma", DIV_MOTOR, 0x1ef, 0x1f0, 0x1f1, 0, 0, 0, 0),
- D_GATE(HCLK_PWMPTO, "hclk_pwmpto", DIV_MOTOR, 0x1ec, 0x1ed, 0x1ee, 0, 0, 0, 0),
- D_GATE(HCLK_RSV, "hclk_rsv", CLK_48, 0x780, 0x781, 0x782, 0, 0xb00, 0, 0),
- D_GATE(HCLK_SGPIO0, "hclk_sgpio0", DIV_MOTOR, 0x1e0, 0x1e1, 0x1e2, 0, 0, 0, 0),
- D_GATE(HCLK_SGPIO1, "hclk_sgpio1", DIV_MOTOR, 0x1e3, 0x1e4, 0x1e5, 0, 0, 0, 0),
+ D_MODULE(HCLK_CAN0, "hclk_can0", CLK_48, 0x783, 0x784, 0x785, 0, 0xb01, 0, 0),
+ D_MODULE(HCLK_CAN1, "hclk_can1", CLK_48, 0x786, 0x787, 0x788, 0, 0xb02, 0, 0),
+ D_MODULE(HCLK_DELTASIGMA, "hclk_deltasigma", DIV_MOTOR, 0x1ef, 0x1f0, 0x1f1, 0, 0, 0, 0),
+ D_MODULE(HCLK_PWMPTO, "hclk_pwmpto", DIV_MOTOR, 0x1ec, 0x1ed, 0x1ee, 0, 0, 0, 0),
+ D_MODULE(HCLK_RSV, "hclk_rsv", CLK_48, 0x780, 0x781, 0x782, 0, 0xb00, 0, 0),
+ D_MODULE(HCLK_SGPIO0, "hclk_sgpio0", DIV_MOTOR, 0x1e0, 0x1e1, 0x1e2, 0, 0, 0, 0),
+ D_MODULE(HCLK_SGPIO1, "hclk_sgpio1", DIV_MOTOR, 0x1e3, 0x1e4, 0x1e5, 0, 0, 0, 0),
D_DIV(RTOS_MDC, "rtos_mdc", CLK_REF_SYNC, 100, 80, 640, 80, 160, 320, 640),
D_GATE(CLK_CM3, "clk_cm3", CLK_REF_SYNC_D4, 0xba0, 0xba1, 0, 0xba2, 0, 0xbc0, 0xbc1),
D_GATE(CLK_DDRC, "clk_ddrc", CLK_DDRPHY_PLLCLK_D4, 0x323, 0x324, 0, 0, 0, 0, 0),
@@ -222,53 +229,53 @@ static const struct r9a06g032_clkdesc r9a06g032_clocks[] __initconst = {
D_GATE(CLK_HSR50, "clk_hsr50", CLK_HSR100_D2, 0x484, 0x485, 0, 0, 0, 0, 0),
D_GATE(CLK_HW_RTOS, "clk_hw_rtos", CLK_REF_SYNC_D4, 0xc60, 0xc61, 0, 0, 0, 0, 0),
D_GATE(CLK_SERCOS50, "clk_sercos50", CLK_SERCOS100_D2, 0x424, 0x423, 0, 0, 0, 0, 0),
- D_GATE(HCLK_ADC, "hclk_adc", CLK_REF_SYNC_D8, 0x1af, 0x1b0, 0x1b1, 0, 0, 0, 0),
- D_GATE(HCLK_CM3, "hclk_cm3", CLK_REF_SYNC_D4, 0xc20, 0xc21, 0xc22, 0, 0, 0, 0),
- D_GATE(HCLK_CRYPTO_EIP150, "hclk_crypto_eip150", CLK_REF_SYNC_D4, 0x123, 0x124, 0x125, 0, 0x142, 0, 0),
- D_GATE(HCLK_CRYPTO_EIP93, "hclk_crypto_eip93", CLK_REF_SYNC_D4, 0x120, 0x121, 0, 0x122, 0, 0x140, 0x141),
- D_GATE(HCLK_DDRC, "hclk_ddrc", CLK_REF_SYNC_D4, 0x320, 0x322, 0, 0x321, 0, 0x3a0, 0x3a1),
- D_GATE(HCLK_DMA0, "hclk_dma0", CLK_REF_SYNC_D4, 0x260, 0x261, 0x262, 0x263, 0x2c0, 0x2c1, 0x2c2),
- D_GATE(HCLK_DMA1, "hclk_dma1", CLK_REF_SYNC_D4, 0x264, 0x265, 0x266, 0x267, 0x2c3, 0x2c4, 0x2c5),
- D_GATE(HCLK_GMAC0, "hclk_gmac0", CLK_REF_SYNC_D4, 0x360, 0x361, 0x362, 0x363, 0x3c0, 0x3c1, 0x3c2),
- D_GATE(HCLK_GMAC1, "hclk_gmac1", CLK_REF_SYNC_D4, 0x380, 0x381, 0x382, 0x383, 0x3e0, 0x3e1, 0x3e2),
- D_GATE(HCLK_GPIO0, "hclk_gpio0", CLK_REF_SYNC_D4, 0x212, 0x213, 0x214, 0, 0, 0, 0),
- D_GATE(HCLK_GPIO1, "hclk_gpio1", CLK_REF_SYNC_D4, 0x215, 0x216, 0x217, 0, 0, 0, 0),
- D_GATE(HCLK_GPIO2, "hclk_gpio2", CLK_REF_SYNC_D4, 0x229, 0x22a, 0x22b, 0, 0, 0, 0),
- D_GATE(HCLK_HSR, "hclk_hsr", CLK_HSR100_D2, 0x480, 0x482, 0, 0x481, 0, 0x4c0, 0x4c1),
- D_GATE(HCLK_I2C0, "hclk_i2c0", CLK_REF_SYNC_D8, 0x1a9, 0x1aa, 0x1ab, 0, 0, 0, 0),
- D_GATE(HCLK_I2C1, "hclk_i2c1", CLK_REF_SYNC_D8, 0x1ac, 0x1ad, 0x1ae, 0, 0, 0, 0),
- D_GATE(HCLK_LCD, "hclk_lcd", CLK_REF_SYNC_D4, 0x7a0, 0x7a1, 0x7a2, 0, 0xb20, 0, 0),
- D_GATE(HCLK_MSEBI_M, "hclk_msebi_m", CLK_REF_SYNC_D4, 0x164, 0x165, 0x166, 0, 0x183, 0, 0),
- D_GATE(HCLK_MSEBI_S, "hclk_msebi_s", CLK_REF_SYNC_D4, 0x160, 0x161, 0x162, 0x163, 0x180, 0x181, 0x182),
- D_GATE(HCLK_NAND, "hclk_nand", CLK_REF_SYNC_D4, 0x280, 0x281, 0x282, 0x283, 0x2e0, 0x2e1, 0x2e2),
- D_GATE(HCLK_PG_I, "hclk_pg_i", CLK_REF_SYNC_D4, 0x7ac, 0x7ad, 0, 0x7ae, 0, 0xb24, 0xb25),
- D_GATE(HCLK_PG19, "hclk_pg19", CLK_REF_SYNC_D4, 0x22c, 0x22d, 0x22e, 0, 0, 0, 0),
- D_GATE(HCLK_PG20, "hclk_pg20", CLK_REF_SYNC_D4, 0x22f, 0x230, 0x231, 0, 0, 0, 0),
- D_GATE(HCLK_PG3, "hclk_pg3", CLK_REF_SYNC_D4, 0x7a6, 0x7a7, 0x7a8, 0, 0xb22, 0, 0),
- D_GATE(HCLK_PG4, "hclk_pg4", CLK_REF_SYNC_D4, 0x7a9, 0x7aa, 0x7ab, 0, 0xb23, 0, 0),
- D_GATE(HCLK_QSPI0, "hclk_qspi0", CLK_REF_SYNC_D4, 0x2a0, 0x2a1, 0x2a2, 0x2a3, 0x300, 0x301, 0x302),
- D_GATE(HCLK_QSPI1, "hclk_qspi1", CLK_REF_SYNC_D4, 0x480, 0x481, 0x482, 0x483, 0x4c0, 0x4c1, 0x4c2),
- D_GATE(HCLK_ROM, "hclk_rom", CLK_REF_SYNC_D4, 0xaa0, 0xaa1, 0xaa2, 0, 0xb80, 0, 0),
- D_GATE(HCLK_RTC, "hclk_rtc", CLK_REF_SYNC_D8, 0xa00, 0, 0, 0, 0, 0, 0),
- D_GATE(HCLK_SDIO0, "hclk_sdio0", CLK_REF_SYNC_D4, 0x60, 0x61, 0x62, 0x63, 0x80, 0x81, 0x82),
- D_GATE(HCLK_SDIO1, "hclk_sdio1", CLK_REF_SYNC_D4, 0x640, 0x641, 0x642, 0x643, 0x660, 0x661, 0x662),
- D_GATE(HCLK_SEMAP, "hclk_semap", CLK_REF_SYNC_D4, 0x7a3, 0x7a4, 0x7a5, 0, 0xb21, 0, 0),
- D_GATE(HCLK_SPI0, "hclk_spi0", CLK_REF_SYNC_D4, 0x200, 0x201, 0x202, 0, 0, 0, 0),
- D_GATE(HCLK_SPI1, "hclk_spi1", CLK_REF_SYNC_D4, 0x203, 0x204, 0x205, 0, 0, 0, 0),
- D_GATE(HCLK_SPI2, "hclk_spi2", CLK_REF_SYNC_D4, 0x206, 0x207, 0x208, 0, 0, 0, 0),
- D_GATE(HCLK_SPI3, "hclk_spi3", CLK_REF_SYNC_D4, 0x209, 0x20a, 0x20b, 0, 0, 0, 0),
- D_GATE(HCLK_SPI4, "hclk_spi4", CLK_REF_SYNC_D4, 0x20c, 0x20d, 0x20e, 0, 0, 0, 0),
- D_GATE(HCLK_SPI5, "hclk_spi5", CLK_REF_SYNC_D4, 0x20f, 0x210, 0x211, 0, 0, 0, 0),
- D_GATE(HCLK_SWITCH, "hclk_switch", CLK_REF_SYNC_D4, 0x980, 0, 0x981, 0, 0, 0, 0),
- D_GATE(HCLK_SWITCH_RG, "hclk_switch_rg", CLK_REF_SYNC_D4, 0xc40, 0xc41, 0xc42, 0, 0, 0, 0),
- D_GATE(HCLK_UART0, "hclk_uart0", CLK_REF_SYNC_D8, 0x1a0, 0x1a1, 0x1a2, 0, 0, 0, 0),
- D_GATE(HCLK_UART1, "hclk_uart1", CLK_REF_SYNC_D8, 0x1a3, 0x1a4, 0x1a5, 0, 0, 0, 0),
- D_GATE(HCLK_UART2, "hclk_uart2", CLK_REF_SYNC_D8, 0x1a6, 0x1a7, 0x1a8, 0, 0, 0, 0),
- D_GATE(HCLK_UART3, "hclk_uart3", CLK_REF_SYNC_D4, 0x218, 0x219, 0x21a, 0, 0, 0, 0),
- D_GATE(HCLK_UART4, "hclk_uart4", CLK_REF_SYNC_D4, 0x21b, 0x21c, 0x21d, 0, 0, 0, 0),
- D_GATE(HCLK_UART5, "hclk_uart5", CLK_REF_SYNC_D4, 0x220, 0x221, 0x222, 0, 0, 0, 0),
- D_GATE(HCLK_UART6, "hclk_uart6", CLK_REF_SYNC_D4, 0x223, 0x224, 0x225, 0, 0, 0, 0),
- D_GATE(HCLK_UART7, "hclk_uart7", CLK_REF_SYNC_D4, 0x226, 0x227, 0x228, 0, 0, 0, 0),
+ D_MODULE(HCLK_ADC, "hclk_adc", CLK_REF_SYNC_D8, 0x1af, 0x1b0, 0x1b1, 0, 0, 0, 0),
+ D_MODULE(HCLK_CM3, "hclk_cm3", CLK_REF_SYNC_D4, 0xc20, 0xc21, 0xc22, 0, 0, 0, 0),
+ D_MODULE(HCLK_CRYPTO_EIP150, "hclk_crypto_eip150", CLK_REF_SYNC_D4, 0x123, 0x124, 0x125, 0, 0x142, 0, 0),
+ D_MODULE(HCLK_CRYPTO_EIP93, "hclk_crypto_eip93", CLK_REF_SYNC_D4, 0x120, 0x121, 0, 0x122, 0, 0x140, 0x141),
+ D_MODULE(HCLK_DDRC, "hclk_ddrc", CLK_REF_SYNC_D4, 0x320, 0x322, 0, 0x321, 0, 0x3a0, 0x3a1),
+ D_MODULE(HCLK_DMA0, "hclk_dma0", CLK_REF_SYNC_D4, 0x260, 0x261, 0x262, 0x263, 0x2c0, 0x2c1, 0x2c2),
+ D_MODULE(HCLK_DMA1, "hclk_dma1", CLK_REF_SYNC_D4, 0x264, 0x265, 0x266, 0x267, 0x2c3, 0x2c4, 0x2c5),
+ D_MODULE(HCLK_GMAC0, "hclk_gmac0", CLK_REF_SYNC_D4, 0x360, 0x361, 0x362, 0x363, 0x3c0, 0x3c1, 0x3c2),
+ D_MODULE(HCLK_GMAC1, "hclk_gmac1", CLK_REF_SYNC_D4, 0x380, 0x381, 0x382, 0x383, 0x3e0, 0x3e1, 0x3e2),
+ D_MODULE(HCLK_GPIO0, "hclk_gpio0", CLK_REF_SYNC_D4, 0x212, 0x213, 0x214, 0, 0, 0, 0),
+ D_MODULE(HCLK_GPIO1, "hclk_gpio1", CLK_REF_SYNC_D4, 0x215, 0x216, 0x217, 0, 0, 0, 0),
+ D_MODULE(HCLK_GPIO2, "hclk_gpio2", CLK_REF_SYNC_D4, 0x229, 0x22a, 0x22b, 0, 0, 0, 0),
+ D_MODULE(HCLK_HSR, "hclk_hsr", CLK_HSR100_D2, 0x480, 0x482, 0, 0x481, 0, 0x4c0, 0x4c1),
+ D_MODULE(HCLK_I2C0, "hclk_i2c0", CLK_REF_SYNC_D8, 0x1a9, 0x1aa, 0x1ab, 0, 0, 0, 0),
+ D_MODULE(HCLK_I2C1, "hclk_i2c1", CLK_REF_SYNC_D8, 0x1ac, 0x1ad, 0x1ae, 0, 0, 0, 0),
+ D_MODULE(HCLK_LCD, "hclk_lcd", CLK_REF_SYNC_D4, 0x7a0, 0x7a1, 0x7a2, 0, 0xb20, 0, 0),
+ D_MODULE(HCLK_MSEBI_M, "hclk_msebi_m", CLK_REF_SYNC_D4, 0x164, 0x165, 0x166, 0, 0x183, 0, 0),
+ D_MODULE(HCLK_MSEBI_S, "hclk_msebi_s", CLK_REF_SYNC_D4, 0x160, 0x161, 0x162, 0x163, 0x180, 0x181, 0x182),
+ D_MODULE(HCLK_NAND, "hclk_nand", CLK_REF_SYNC_D4, 0x280, 0x281, 0x282, 0x283, 0x2e0, 0x2e1, 0x2e2),
+ D_MODULE(HCLK_PG_I, "hclk_pg_i", CLK_REF_SYNC_D4, 0x7ac, 0x7ad, 0, 0x7ae, 0, 0xb24, 0xb25),
+ D_MODULE(HCLK_PG19, "hclk_pg19", CLK_REF_SYNC_D4, 0x22c, 0x22d, 0x22e, 0, 0, 0, 0),
+ D_MODULE(HCLK_PG20, "hclk_pg20", CLK_REF_SYNC_D4, 0x22f, 0x230, 0x231, 0, 0, 0, 0),
+ D_MODULE(HCLK_PG3, "hclk_pg3", CLK_REF_SYNC_D4, 0x7a6, 0x7a7, 0x7a8, 0, 0xb22, 0, 0),
+ D_MODULE(HCLK_PG4, "hclk_pg4", CLK_REF_SYNC_D4, 0x7a9, 0x7aa, 0x7ab, 0, 0xb23, 0, 0),
+ D_MODULE(HCLK_QSPI0, "hclk_qspi0", CLK_REF_SYNC_D4, 0x2a0, 0x2a1, 0x2a2, 0x2a3, 0x300, 0x301, 0x302),
+ D_MODULE(HCLK_QSPI1, "hclk_qspi1", CLK_REF_SYNC_D4, 0x480, 0x481, 0x482, 0x483, 0x4c0, 0x4c1, 0x4c2),
+ D_MODULE(HCLK_ROM, "hclk_rom", CLK_REF_SYNC_D4, 0xaa0, 0xaa1, 0xaa2, 0, 0xb80, 0, 0),
+ D_MODULE(HCLK_RTC, "hclk_rtc", CLK_REF_SYNC_D8, 0xa00, 0, 0, 0, 0, 0, 0),
+ D_MODULE(HCLK_SDIO0, "hclk_sdio0", CLK_REF_SYNC_D4, 0x60, 0x61, 0x62, 0x63, 0x80, 0x81, 0x82),
+ D_MODULE(HCLK_SDIO1, "hclk_sdio1", CLK_REF_SYNC_D4, 0x640, 0x641, 0x642, 0x643, 0x660, 0x661, 0x662),
+ D_MODULE(HCLK_SEMAP, "hclk_semap", CLK_REF_SYNC_D4, 0x7a3, 0x7a4, 0x7a5, 0, 0xb21, 0, 0),
+ D_MODULE(HCLK_SPI0, "hclk_spi0", CLK_REF_SYNC_D4, 0x200, 0x201, 0x202, 0, 0, 0, 0),
+ D_MODULE(HCLK_SPI1, "hclk_spi1", CLK_REF_SYNC_D4, 0x203, 0x204, 0x205, 0, 0, 0, 0),
+ D_MODULE(HCLK_SPI2, "hclk_spi2", CLK_REF_SYNC_D4, 0x206, 0x207, 0x208, 0, 0, 0, 0),
+ D_MODULE(HCLK_SPI3, "hclk_spi3", CLK_REF_SYNC_D4, 0x209, 0x20a, 0x20b, 0, 0, 0, 0),
+ D_MODULE(HCLK_SPI4, "hclk_spi4", CLK_REF_SYNC_D4, 0x20c, 0x20d, 0x20e, 0, 0, 0, 0),
+ D_MODULE(HCLK_SPI5, "hclk_spi5", CLK_REF_SYNC_D4, 0x20f, 0x210, 0x211, 0, 0, 0, 0),
+ D_MODULE(HCLK_SWITCH, "hclk_switch", CLK_REF_SYNC_D4, 0x980, 0, 0x981, 0, 0, 0, 0),
+ D_MODULE(HCLK_SWITCH_RG, "hclk_switch_rg", CLK_REF_SYNC_D4, 0xc40, 0xc41, 0xc42, 0, 0, 0, 0),
+ D_MODULE(HCLK_UART0, "hclk_uart0", CLK_REF_SYNC_D8, 0x1a0, 0x1a1, 0x1a2, 0, 0, 0, 0),
+ D_MODULE(HCLK_UART1, "hclk_uart1", CLK_REF_SYNC_D8, 0x1a3, 0x1a4, 0x1a5, 0, 0, 0, 0),
+ D_MODULE(HCLK_UART2, "hclk_uart2", CLK_REF_SYNC_D8, 0x1a6, 0x1a7, 0x1a8, 0, 0, 0, 0),
+ D_MODULE(HCLK_UART3, "hclk_uart3", CLK_REF_SYNC_D4, 0x218, 0x219, 0x21a, 0, 0, 0, 0),
+ D_MODULE(HCLK_UART4, "hclk_uart4", CLK_REF_SYNC_D4, 0x21b, 0x21c, 0x21d, 0, 0, 0, 0),
+ D_MODULE(HCLK_UART5, "hclk_uart5", CLK_REF_SYNC_D4, 0x220, 0x221, 0x222, 0, 0, 0, 0),
+ D_MODULE(HCLK_UART6, "hclk_uart6", CLK_REF_SYNC_D4, 0x223, 0x224, 0x225, 0, 0, 0, 0),
+ D_MODULE(HCLK_UART7, "hclk_uart7", CLK_REF_SYNC_D4, 0x226, 0x227, 0x228, 0, 0, 0, 0),
/*
* These are not hardware clocks, but are needed to handle the special
* case where we have a 'selector bit' that doesn't just change the
@@ -345,6 +352,84 @@ struct r9a06g032_clk_gate {
#define to_r9a06g032_gate(_hw) container_of(_hw, struct r9a06g032_clk_gate, hw)
+static int create_add_module_clock(struct of_phandle_args *clkspec,
+ struct device *dev)
+{
+ struct clk *clk;
+ int error;
+
+ clk = of_clk_get_from_provider(clkspec);
+ if (IS_ERR(clk))
+ return PTR_ERR(clk);
+
+ error = pm_clk_create(dev);
+ if (error) {
+ clk_put(clk);
+ return error;
+ }
+
+ error = pm_clk_add_clk(dev, clk);
+ if (error) {
+ pm_clk_destroy(dev);
+ clk_put(clk);
+ }
+
+ return error;
+}
+
+static int r9a06g032_attach_dev(struct generic_pm_domain *pd,
+ struct device *dev)
+{
+ struct device_node *np = dev->of_node;
+ struct of_phandle_args clkspec;
+ int i = 0;
+ int error;
+ int index;
+
+ while (!of_parse_phandle_with_args(np, "clocks", "#clock-cells", i,
+ &clkspec)) {
+ if (clkspec.np != pd->dev.of_node)
+ continue;
+
+ index = clkspec.args[0];
+ if (index < R9A06G032_CLOCK_COUNT &&
+ r9a06g032_clocks[index].managed) {
+ error = create_add_module_clock(&clkspec, dev);
+ of_node_put(clkspec.np);
+ if (error)
+ return error;
+ }
+ i++;
+ }
+
+ return 0;
+}
+
+static void r9a06g032_detach_dev(struct generic_pm_domain *unused, struct device *dev)
+{
+ if (!pm_clk_no_clocks(dev))
+ pm_clk_destroy(dev);
+}
+
+static int r9a06g032_add_clk_domain(struct device *dev)
+{
+ struct device_node *np = dev->of_node;
+ struct generic_pm_domain *pd;
+
+ pd = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL);
+ if (!pd)
+ return -ENOMEM;
+
+ pd->name = np->name;
+ pd->flags = GENPD_FLAG_PM_CLK | GENPD_FLAG_ACTIVE_WAKEUP;
+ pd->attach_dev = r9a06g032_attach_dev;
+ pd->detach_dev = r9a06g032_detach_dev;
+ pm_genpd_init(pd, &pm_domain_always_on_gov, false);
+
+ of_genpd_add_provider_simple(np, pd);
+ return 0;
+}
+
static void
r9a06g032_clk_gate_set(struct r9a06g032_priv *clocks,
struct r9a06g032_gate *g, int on)
@@ -871,8 +956,12 @@ static int __init r9a06g032_clocks_probe(struct platform_device *pdev)
if (error)
return error;
- return devm_add_action_or_reset(dev,
+ error = devm_add_action_or_reset(dev,
r9a06g032_clocks_del_clk_provider, np);
+ if (error)
+ return error;
+
+ return r9a06g032_add_clk_domain(dev);
}
static const struct of_device_id r9a06g032_match[] = {
diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c
index 0201809bbd37..d4075b130674 100644
--- a/drivers/clk/renesas/renesas-cpg-mssr.c
+++ b/drivers/clk/renesas/renesas-cpg-mssr.c
@@ -112,14 +112,15 @@ static const u16 srcr[] = {
* @dev: CPG/MSSR device
* @base: CPG/MSSR register block base address
* @rmw_lock: protects RMW register accesses
- * @clks: Array containing all Core and Module Clocks
+ * @np: Device node in DT for this CPG/MSSR module
* @num_core_clks: Number of Core Clocks in clks[]
* @num_mod_clks: Number of Module Clocks in clks[]
* @last_dt_core_clk: ID of the last Core Clock exported to DT
+ * @stbyctrl: This device has Standby Control Registers
* @notifiers: Notifier chain to save/restore clock state for system resume
* @smstpcr_saved[].mask: Mask of SMSTPCR[] bits under our control
* @smstpcr_saved[].val: Saved values of SMSTPCR[]
- * @stbyctrl: This device has Standby Control Registers
+ * @clks: Array containing all Core and Module Clocks
*/
struct cpg_mssr_priv {
#ifdef CONFIG_RESET_CONTROLLER
@@ -130,7 +131,6 @@ struct cpg_mssr_priv {
spinlock_t rmw_lock;
struct device_node *np;
- struct clk **clks;
unsigned int num_core_clks;
unsigned int num_mod_clks;
unsigned int last_dt_core_clk;
@@ -141,6 +141,8 @@ struct cpg_mssr_priv {
u32 mask;
u32 val;
} smstpcr_saved[ARRAY_SIZE(smstpcr)];
+
+ struct clk *clks[];
};
static struct cpg_mssr_priv *cpg_mssr_priv;
@@ -447,9 +449,8 @@ fail:
struct cpg_mssr_clk_domain {
struct generic_pm_domain genpd;
- struct device_node *np;
unsigned int num_core_pm_clks;
- unsigned int core_pm_clks[0];
+ unsigned int core_pm_clks[];
};
static struct cpg_mssr_clk_domain *cpg_mssr_clk_domain;
@@ -459,7 +460,7 @@ static bool cpg_mssr_is_pm_clk(const struct of_phandle_args *clkspec,
{
unsigned int i;
- if (clkspec->np != pd->np || clkspec->args_count != 2)
+ if (clkspec->np != pd->genpd.dev.of_node || clkspec->args_count != 2)
return false;
switch (clkspec->args[0]) {
@@ -510,16 +511,12 @@ found:
return PTR_ERR(clk);
error = pm_clk_create(dev);
- if (error) {
- dev_err(dev, "pm_clk_create failed %d\n", error);
+ if (error)
goto fail_put;
- }
error = pm_clk_add_clk(dev, clk);
- if (error) {
- dev_err(dev, "pm_clk_add_clk %pC failed %d\n", clk, error);
+ if (error)
goto fail_destroy;
- }
return 0;
@@ -549,7 +546,6 @@ static int __init cpg_mssr_add_clk_domain(struct device *dev,
if (!pd)
return -ENOMEM;
- pd->np = np;
pd->num_core_pm_clks = num_core_pm_clks;
memcpy(pd->core_pm_clks, core_pm_clks, pm_size);
@@ -576,17 +572,11 @@ static int cpg_mssr_reset(struct reset_controller_dev *rcdev,
unsigned int reg = id / 32;
unsigned int bit = id % 32;
u32 bitmask = BIT(bit);
- unsigned long flags;
- u32 value;
dev_dbg(priv->dev, "reset %u%02u\n", reg, bit);
/* Reset module */
- spin_lock_irqsave(&priv->rmw_lock, flags);
- value = readl(priv->base + SRCR(reg));
- value |= bitmask;
- writel(value, priv->base + SRCR(reg));
- spin_unlock_irqrestore(&priv->rmw_lock, flags);
+ writel(bitmask, priv->base + SRCR(reg));
/* Wait for at least one cycle of the RCLK clock (@ ca. 32 kHz) */
udelay(35);
@@ -603,16 +593,10 @@ static int cpg_mssr_assert(struct reset_controller_dev *rcdev, unsigned long id)
unsigned int reg = id / 32;
unsigned int bit = id % 32;
u32 bitmask = BIT(bit);
- unsigned long flags;
- u32 value;
dev_dbg(priv->dev, "assert %u%02u\n", reg, bit);
- spin_lock_irqsave(&priv->rmw_lock, flags);
- value = readl(priv->base + SRCR(reg));
- value |= bitmask;
- writel(value, priv->base + SRCR(reg));
- spin_unlock_irqrestore(&priv->rmw_lock, flags);
+ writel(bitmask, priv->base + SRCR(reg));
return 0;
}
@@ -896,7 +880,6 @@ static int __init cpg_mssr_common_init(struct device *dev,
const struct cpg_mssr_info *info)
{
struct cpg_mssr_priv *priv;
- struct clk **clks = NULL;
unsigned int nclks, i;
int error;
@@ -906,7 +889,8 @@ static int __init cpg_mssr_common_init(struct device *dev,
return error;
}
- priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+ nclks = info->num_total_core_clks + info->num_hw_mod_clks;
+ priv = kzalloc(struct_size(priv, clks, nclks), GFP_KERNEL);
if (!priv)
return -ENOMEM;
@@ -920,15 +904,7 @@ static int __init cpg_mssr_common_init(struct device *dev,
goto out_err;
}
- nclks = info->num_total_core_clks + info->num_hw_mod_clks;
- clks = kmalloc_array(nclks, sizeof(*clks), GFP_KERNEL);
- if (!clks) {
- error = -ENOMEM;
- goto out_err;
- }
-
cpg_mssr_priv = priv;
- priv->clks = clks;
priv->num_core_clks = info->num_total_core_clks;
priv->num_mod_clks = info->num_hw_mod_clks;
priv->last_dt_core_clk = info->last_dt_core_clk;
@@ -936,7 +912,7 @@ static int __init cpg_mssr_common_init(struct device *dev,
priv->stbyctrl = info->stbyctrl;
for (i = 0; i < nclks; i++)
- clks[i] = ERR_PTR(-ENOENT);
+ priv->clks[i] = ERR_PTR(-ENOENT);
error = of_clk_add_provider(np, cpg_mssr_clk_src_twocell_get, priv);
if (error)
@@ -945,7 +921,6 @@ static int __init cpg_mssr_common_init(struct device *dev,
return 0;
out_err:
- kfree(clks);
if (priv->base)
iounmap(priv->base);
kfree(priv);
diff --git a/drivers/clk/rockchip/clk-mmc-phase.c b/drivers/clk/rockchip/clk-mmc-phase.c
index c61f4d3e52e2..4abe7ff31f53 100644
--- a/drivers/clk/rockchip/clk-mmc-phase.c
+++ b/drivers/clk/rockchip/clk-mmc-phase.c
@@ -46,29 +46,27 @@ static unsigned long rockchip_mmc_recalc(struct clk_hw *hw,
static int rockchip_mmc_get_phase(struct clk_hw *hw)
{
struct rockchip_mmc_clock *mmc_clock = to_mmc_clock(hw);
- unsigned long rate = clk_get_rate(hw->clk);
+ unsigned long rate = clk_hw_get_rate(hw);
u32 raw_value;
u16 degrees;
u32 delay_num = 0;
/* See the comment for rockchip_mmc_set_phase below */
- if (!rate) {
- pr_err("%s: invalid clk rate\n", __func__);
+ if (!rate)
return -EINVAL;
- }
raw_value = readl(mmc_clock->reg) >> (mmc_clock->shift);
degrees = (raw_value & ROCKCHIP_MMC_DEGREE_MASK) * 90;
if (raw_value & ROCKCHIP_MMC_DELAY_SEL) {
- /* degrees/delaynum * 10000 */
+ /* degrees/delaynum * 1000000 */
unsigned long factor = (ROCKCHIP_MMC_DELAY_ELEMENT_PSEC / 10) *
- 36 * (rate / 1000000);
+ 36 * (rate / 10000);
delay_num = (raw_value & ROCKCHIP_MMC_DELAYNUM_MASK);
delay_num >>= ROCKCHIP_MMC_DELAYNUM_OFFSET;
- degrees += DIV_ROUND_CLOSEST(delay_num * factor, 10000);
+ degrees += DIV_ROUND_CLOSEST(delay_num * factor, 1000000);
}
return degrees % 360;
@@ -77,7 +75,7 @@ static int rockchip_mmc_get_phase(struct clk_hw *hw)
static int rockchip_mmc_set_phase(struct clk_hw *hw, int degrees)
{
struct rockchip_mmc_clock *mmc_clock = to_mmc_clock(hw);
- unsigned long rate = clk_get_rate(hw->clk);
+ unsigned long rate = clk_hw_get_rate(hw);
u8 nineties, remainder;
u8 delay_num;
u32 raw_value;
diff --git a/drivers/clk/rockchip/clk-px30.c b/drivers/clk/rockchip/clk-px30.c
index bb39a799bdb5..3a501896b280 100644
--- a/drivers/clk/rockchip/clk-px30.c
+++ b/drivers/clk/rockchip/clk-px30.c
@@ -794,6 +794,9 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = {
GATE(ACLK_GIC, "aclk_gic", "aclk_bus_pre", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(13), 12, GFLAGS),
GATE(ACLK_DCF, "aclk_dcf", "aclk_bus_pre", 0, PX30_CLKGATE_CON(13), 15, GFLAGS),
+ /* aclk_dmac is controlled by sgrf_soc_con1[11]. */
+ SGRF_GATE(ACLK_DMAC, "aclk_dmac", "aclk_bus_pre"),
+
GATE(0, "hclk_bus_niu", "hclk_bus_pre", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(13), 9, GFLAGS),
GATE(0, "hclk_rom", "hclk_bus_pre", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(13), 14, GFLAGS),
GATE(HCLK_PDM, "hclk_pdm", "hclk_bus_pre", 0, PX30_CLKGATE_CON(14), 1, GFLAGS),
@@ -957,7 +960,6 @@ static void __init px30_clk_init(struct device_node *np)
{
struct rockchip_clk_provider *ctx;
void __iomem *reg_base;
- struct clk *clk;
reg_base = of_iomap(np, 0);
if (!reg_base) {
@@ -972,14 +974,6 @@ static void __init px30_clk_init(struct device_node *np)
return;
}
- /* aclk_dmac is controlled by sgrf_soc_con1[11]. */
- clk = clk_register_fixed_factor(NULL, "aclk_dmac", "aclk_bus_pre", 0, 1, 1);
- if (IS_ERR(clk))
- pr_warn("%s: could not register clock aclk_dmac: %ld\n",
- __func__, PTR_ERR(clk));
- else
- rockchip_clk_add_lookup(ctx, clk, ACLK_DMAC);
-
rockchip_clk_register_plls(ctx, px30_pll_clks,
ARRAY_SIZE(px30_pll_clks),
PX30_GRF_SOC_STATUS0);
diff --git a/drivers/clk/rockchip/clk-rk3228.c b/drivers/clk/rockchip/clk-rk3228.c
index bdb126321e56..d17cfb7a3ff4 100644
--- a/drivers/clk/rockchip/clk-rk3228.c
+++ b/drivers/clk/rockchip/clk-rk3228.c
@@ -101,6 +101,7 @@ static struct rockchip_cpuclk_rate_table rk3228_cpuclk_rates[] __initdata = {
RK3228_CPUCLK_RATE(1608000000, 1, 7),
RK3228_CPUCLK_RATE(1512000000, 1, 7),
RK3228_CPUCLK_RATE(1488000000, 1, 5),
+ RK3228_CPUCLK_RATE(1464000000, 1, 5),
RK3228_CPUCLK_RATE(1416000000, 1, 5),
RK3228_CPUCLK_RATE(1392000000, 1, 5),
RK3228_CPUCLK_RATE(1296000000, 1, 5),
@@ -246,7 +247,7 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = {
RK2928_CLKGATE_CON(4), 0, GFLAGS),
/* PD_MISC */
- MUX(0, "hdmiphy", mux_hdmiphy_p, CLK_SET_RATE_PARENT,
+ MUX(SCLK_HDMI_PHY, "hdmiphy", mux_hdmiphy_p, CLK_SET_RATE_PARENT,
RK2928_MISC_CON, 13, 1, MFLAGS),
MUX(0, "usb480m_phy", mux_usb480m_phy_p, CLK_SET_RATE_PARENT,
RK2928_MISC_CON, 14, 1, MFLAGS),
diff --git a/drivers/clk/rockchip/clk-rk3288.c b/drivers/clk/rockchip/clk-rk3288.c
index 057629685ea1..cc2a177bbdbf 100644
--- a/drivers/clk/rockchip/clk-rk3288.c
+++ b/drivers/clk/rockchip/clk-rk3288.c
@@ -113,7 +113,6 @@ static struct rockchip_pll_rate_table rk3288_pll_rates[] = {
RK3066_PLL_RATE( 160000000, 1, 80, 12),
RK3066_PLL_RATE( 157500000, 1, 105, 16),
RK3066_PLL_RATE( 126000000, 1, 84, 16),
- RK3066_PLL_RATE( 48000000, 1, 64, 32),
{ /* sentinel */ },
};
@@ -767,6 +766,9 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = {
GATE(PCLK_GRF, "pclk_grf", "pclk_pd_alive", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(14), 11, GFLAGS),
GATE(0, "pclk_alive_niu", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 12, GFLAGS),
+ /* Watchdog pclk is controlled by RK3288_SGRF_SOC_CON0[1]. */
+ SGRF_GATE(PCLK_WDT, "pclk_wdt", "pclk_pd_alive"),
+
/* pclk_pd_pmu gates */
GATE(PCLK_PMU, "pclk_pmu", "pclk_pd_pmu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(17), 0, GFLAGS),
GATE(0, "pclk_intmem1", "pclk_pd_pmu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(17), 1, GFLAGS),
@@ -915,7 +917,6 @@ static struct syscore_ops rk3288_clk_syscore_ops = {
static void __init rk3288_clk_init(struct device_node *np)
{
struct rockchip_clk_provider *ctx;
- struct clk *clk;
rk3288_cru_base = of_iomap(np, 0);
if (!rk3288_cru_base) {
@@ -930,14 +931,6 @@ static void __init rk3288_clk_init(struct device_node *np)
return;
}
- /* Watchdog pclk is controlled by RK3288_SGRF_SOC_CON0[1]. */
- clk = clk_register_fixed_factor(NULL, "pclk_wdt", "pclk_pd_alive", 0, 1, 1);
- if (IS_ERR(clk))
- pr_warn("%s: could not register clock pclk_wdt: %ld\n",
- __func__, PTR_ERR(clk));
- else
- rockchip_clk_add_lookup(ctx, clk, PCLK_WDT);
-
rockchip_clk_register_plls(ctx, rk3288_pll_clks,
ARRAY_SIZE(rk3288_pll_clks),
RK3288_GRF_SOC_STATUS1);
diff --git a/drivers/clk/rockchip/clk-rk3328.c b/drivers/clk/rockchip/clk-rk3328.c
index 076b9777a955..c186a1985bf4 100644
--- a/drivers/clk/rockchip/clk-rk3328.c
+++ b/drivers/clk/rockchip/clk-rk3328.c
@@ -791,6 +791,9 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = {
GATE(PCLK_SARADC, "pclk_saradc", "pclk_bus", 0, RK3328_CLKGATE_CON(17), 15, GFLAGS),
GATE(0, "pclk_pmu", "pclk_bus", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(28), 3, GFLAGS),
+ /* Watchdog pclk is controlled from the secure GRF */
+ SGRF_GATE(PCLK_WDT, "pclk_wdt", "pclk_bus"),
+
GATE(PCLK_USB3PHY_OTG, "pclk_usb3phy_otg", "pclk_phy_pre", 0, RK3328_CLKGATE_CON(28), 1, GFLAGS),
GATE(PCLK_USB3PHY_PIPE, "pclk_usb3phy_pipe", "pclk_phy_pre", 0, RK3328_CLKGATE_CON(28), 2, GFLAGS),
GATE(PCLK_USB3_GRF, "pclk_usb3_grf", "pclk_phy_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 2, GFLAGS),
diff --git a/drivers/clk/rockchip/clk-rk3368.c b/drivers/clk/rockchip/clk-rk3368.c
index 43b022d9393b..55443349439b 100644
--- a/drivers/clk/rockchip/clk-rk3368.c
+++ b/drivers/clk/rockchip/clk-rk3368.c
@@ -811,6 +811,9 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = {
GATE(PCLK_GPIO2, "pclk_gpio2", "pclk_pd_alive", 0, RK3368_CLKGATE_CON(22), 2, GFLAGS),
GATE(PCLK_GPIO1, "pclk_gpio1", "pclk_pd_alive", 0, RK3368_CLKGATE_CON(22), 1, GFLAGS),
+ /* Watchdog pclk is controlled by sgrf_soc_con3[7]. */
+ SGRF_GATE(PCLK_WDT, "pclk_wdt", "pclk_pd_alive"),
+
/*
* pclk_vio gates
* pclk_vio comes from the exactly same source as hclk_vio
@@ -862,7 +865,6 @@ static void __init rk3368_clk_init(struct device_node *np)
{
struct rockchip_clk_provider *ctx;
void __iomem *reg_base;
- struct clk *clk;
reg_base = of_iomap(np, 0);
if (!reg_base) {
@@ -877,14 +879,6 @@ static void __init rk3368_clk_init(struct device_node *np)
return;
}
- /* Watchdog pclk is controlled by sgrf_soc_con3[7]. */
- clk = clk_register_fixed_factor(NULL, "pclk_wdt", "pclk_pd_alive", 0, 1, 1);
- if (IS_ERR(clk))
- pr_warn("%s: could not register clock pclk_wdt: %ld\n",
- __func__, PTR_ERR(clk));
- else
- rockchip_clk_add_lookup(ctx, clk, PCLK_WDT);
-
rockchip_clk_register_plls(ctx, rk3368_pll_clks,
ARRAY_SIZE(rk3368_pll_clks),
RK3368_GRF_SOC_STATUS0);
diff --git a/drivers/clk/rockchip/clk-rk3399.c b/drivers/clk/rockchip/clk-rk3399.c
index a7ff71313278..ce1d2446f142 100644
--- a/drivers/clk/rockchip/clk-rk3399.c
+++ b/drivers/clk/rockchip/clk-rk3399.c
@@ -1295,6 +1295,9 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
GATE(PCLK_PMU_INTR_ARB, "pclk_pmu_intr_arb", "pclk_alive", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(31), 9, GFLAGS),
GATE(PCLK_SGRF, "pclk_sgrf", "pclk_alive", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(31), 10, GFLAGS),
+ /* Watchdog pclk is controlled by RK3399 SECURE_GRF_SOC_CON3[8]. */
+ SGRF_GATE(PCLK_WDT, "pclk_wdt", "pclk_alive"),
+
GATE(SCLK_MIPIDPHY_REF, "clk_mipidphy_ref", "xin24m", 0, RK3399_CLKGATE_CON(11), 14, GFLAGS),
GATE(SCLK_DPHY_PLL, "clk_dphy_pll", "clk_mipidphy_ref", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(21), 0, GFLAGS),
@@ -1522,7 +1525,6 @@ static void __init rk3399_clk_init(struct device_node *np)
{
struct rockchip_clk_provider *ctx;
void __iomem *reg_base;
- struct clk *clk;
reg_base = of_iomap(np, 0);
if (!reg_base) {
@@ -1537,14 +1539,6 @@ static void __init rk3399_clk_init(struct device_node *np)
return;
}
- /* Watchdog pclk is controlled by RK3399 SECURE_GRF_SOC_CON3[8]. */
- clk = clk_register_fixed_factor(NULL, "pclk_wdt", "pclk_alive", 0, 1, 1);
- if (IS_ERR(clk))
- pr_warn("%s: could not register clock pclk_wdt: %ld\n",
- __func__, PTR_ERR(clk));
- else
- rockchip_clk_add_lookup(ctx, clk, PCLK_WDT);
-
rockchip_clk_register_plls(ctx, rk3399_pll_clks,
ARRAY_SIZE(rk3399_pll_clks), -1);
diff --git a/drivers/clk/rockchip/clk.h b/drivers/clk/rockchip/clk.h
index adb66cc94929..b811597a3d38 100644
--- a/drivers/clk/rockchip/clk.h
+++ b/drivers/clk/rockchip/clk.h
@@ -811,6 +811,10 @@ struct rockchip_clk_branch {
.gate_offset = -1, \
}
+/* SGRF clocks are only accessible from secure mode, so not controllable */
+#define SGRF_GATE(_id, cname, pname) \
+ FACTOR(_id, cname, pname, 0, 1, 1)
+
struct rockchip_clk_provider *rockchip_clk_init(struct device_node *np,
void __iomem *base, unsigned long nr_clks);
void rockchip_clk_of_add_provider(struct device_node *np,
diff --git a/drivers/clk/samsung/clk-exynos4.c b/drivers/clk/samsung/clk-exynos4.c
index 982eb02bafda..51564fc23c63 100644
--- a/drivers/clk/samsung/clk-exynos4.c
+++ b/drivers/clk/samsung/clk-exynos4.c
@@ -958,6 +958,7 @@ static const struct samsung_gate_clock exynos4210_gate_clks[] __initconst = {
/* list of gate clocks supported in exynos4x12 soc */
static const struct samsung_gate_clock exynos4x12_gate_clks[] __initconst = {
+ GATE(CLK_ASYNC_G3D, "async_g3d", "aclk200", GATE_IP_LEFTBUS, 6, 0, 0),
GATE(CLK_AUDSS, "audss", "sclk_epll", E4X12_GATE_IP_MAUDIO, 0, 0, 0),
GATE(CLK_MDNIE0, "mdnie0", "aclk160", GATE_IP_LCD0, 2, 0, 0),
GATE(CLK_ROTATOR, "rotator", "aclk200", E4X12_GATE_IP_IMAGE, 1, 0, 0),
diff --git a/drivers/clk/samsung/clk-exynos5-subcmu.c b/drivers/clk/samsung/clk-exynos5-subcmu.c
index 91db7894125d..65c82d922b05 100644
--- a/drivers/clk/samsung/clk-exynos5-subcmu.c
+++ b/drivers/clk/samsung/clk-exynos5-subcmu.c
@@ -14,7 +14,7 @@
#include "clk-exynos5-subcmu.h"
static struct samsung_clk_provider *ctx;
-static const struct exynos5_subcmu_info *cmu;
+static const struct exynos5_subcmu_info **cmu;
static int nr_cmus;
static void exynos5_subcmu_clk_save(void __iomem *base,
@@ -56,17 +56,17 @@ static void exynos5_subcmu_defer_gate(struct samsung_clk_provider *ctx,
* when OF-core populates all device-tree nodes.
*/
void exynos5_subcmus_init(struct samsung_clk_provider *_ctx, int _nr_cmus,
- const struct exynos5_subcmu_info *_cmu)
+ const struct exynos5_subcmu_info **_cmu)
{
ctx = _ctx;
cmu = _cmu;
nr_cmus = _nr_cmus;
for (; _nr_cmus--; _cmu++) {
- exynos5_subcmu_defer_gate(ctx, _cmu->gate_clks,
- _cmu->nr_gate_clks);
- exynos5_subcmu_clk_save(ctx->reg_base, _cmu->suspend_regs,
- _cmu->nr_suspend_regs);
+ exynos5_subcmu_defer_gate(ctx, (*_cmu)->gate_clks,
+ (*_cmu)->nr_gate_clks);
+ exynos5_subcmu_clk_save(ctx->reg_base, (*_cmu)->suspend_regs,
+ (*_cmu)->nr_suspend_regs);
}
}
@@ -163,9 +163,9 @@ static int __init exynos5_clk_probe(struct platform_device *pdev)
if (of_property_read_string(np, "label", &name) < 0)
continue;
for (i = 0; i < nr_cmus; i++)
- if (strcmp(cmu[i].pd_name, name) == 0)
+ if (strcmp(cmu[i]->pd_name, name) == 0)
exynos5_clk_register_subcmu(&pdev->dev,
- &cmu[i], np);
+ cmu[i], np);
}
return 0;
}
diff --git a/drivers/clk/samsung/clk-exynos5-subcmu.h b/drivers/clk/samsung/clk-exynos5-subcmu.h
index 755ee8aaa3de..9ae5356f25aa 100644
--- a/drivers/clk/samsung/clk-exynos5-subcmu.h
+++ b/drivers/clk/samsung/clk-exynos5-subcmu.h
@@ -21,6 +21,6 @@ struct exynos5_subcmu_info {
};
void exynos5_subcmus_init(struct samsung_clk_provider *ctx, int nr_cmus,
- const struct exynos5_subcmu_info *cmu);
+ const struct exynos5_subcmu_info **cmu);
#endif
diff --git a/drivers/clk/samsung/clk-exynos5250.c b/drivers/clk/samsung/clk-exynos5250.c
index f2b896881768..931c70a4da19 100644
--- a/drivers/clk/samsung/clk-exynos5250.c
+++ b/drivers/clk/samsung/clk-exynos5250.c
@@ -681,6 +681,10 @@ static const struct exynos5_subcmu_info exynos5250_disp_subcmu = {
.pd_name = "DISP1",
};
+static const struct exynos5_subcmu_info *exynos5250_subcmus[] = {
+ &exynos5250_disp_subcmu,
+};
+
static const struct samsung_pll_rate_table vpll_24mhz_tbl[] __initconst = {
/* sorted in descending order */
/* PLL_36XX_RATE(rate, m, p, s, k) */
@@ -843,7 +847,8 @@ static void __init exynos5250_clk_init(struct device_node *np)
samsung_clk_sleep_init(reg_base, exynos5250_clk_regs,
ARRAY_SIZE(exynos5250_clk_regs));
- exynos5_subcmus_init(ctx, 1, &exynos5250_disp_subcmu);
+ exynos5_subcmus_init(ctx, ARRAY_SIZE(exynos5250_subcmus),
+ exynos5250_subcmus);
samsung_clk_of_add_provider(np, ctx);
diff --git a/drivers/clk/samsung/clk-exynos5420.c b/drivers/clk/samsung/clk-exynos5420.c
index 12d800fd9528..7670cc596c74 100644
--- a/drivers/clk/samsung/clk-exynos5420.c
+++ b/drivers/clk/samsung/clk-exynos5420.c
@@ -131,6 +131,8 @@
#define SRC_CDREX 0x20200
#define DIV_CDREX0 0x20500
#define DIV_CDREX1 0x20504
+#define GATE_BUS_CDREX0 0x20700
+#define GATE_BUS_CDREX1 0x20704
#define KPLL_LOCK 0x28000
#define KPLL_CON0 0x28100
#define SRC_KFC 0x28200
@@ -245,6 +247,8 @@ static const unsigned long exynos5x_clk_regs[] __initconst = {
DIV_CDREX1,
SRC_KFC,
DIV_KFC0,
+ GATE_BUS_CDREX0,
+ GATE_BUS_CDREX1,
};
static const unsigned long exynos5800_clk_regs[] __initconst = {
@@ -422,6 +426,9 @@ PNAME(mout_group13_5800_p) = { "dout_osc_div", "mout_sw_aclkfl1_550_cam" };
PNAME(mout_group14_5800_p) = { "dout_aclk550_cam", "dout_sclk_sw" };
PNAME(mout_group15_5800_p) = { "dout_osc_div", "mout_sw_aclk550_cam" };
PNAME(mout_group16_5800_p) = { "dout_osc_div", "mout_mau_epll_clk" };
+PNAME(mout_mx_mspll_ccore_phy_p) = { "sclk_bpll", "mout_sclk_dpll",
+ "mout_sclk_mpll", "ff_dout_spll2",
+ "mout_sclk_spll", "mout_sclk_epll"};
/* fixed rate clocks generated outside the soc */
static struct samsung_fixed_rate_clock
@@ -447,7 +454,7 @@ static const struct samsung_fixed_factor_clock
static const struct samsung_fixed_factor_clock
exynos5800_fixed_factor_clks[] __initconst = {
FFACTOR(0, "ff_dout_epll2", "mout_sclk_epll", 1, 2, 0),
- FFACTOR(0, "ff_dout_spll2", "mout_sclk_spll", 1, 2, 0),
+ FFACTOR(CLK_FF_DOUT_SPLL2, "ff_dout_spll2", "mout_sclk_spll", 1, 2, 0),
};
static const struct samsung_mux_clock exynos5800_mux_clks[] __initconst = {
@@ -469,11 +476,14 @@ static const struct samsung_mux_clock exynos5800_mux_clks[] __initconst = {
MUX(0, "mout_aclk300_disp1", mout_group5_5800_p, SRC_TOP2, 24, 2),
MUX(0, "mout_aclk300_gscl", mout_group5_5800_p, SRC_TOP2, 28, 2),
+ MUX(CLK_MOUT_MX_MSPLL_CCORE_PHY, "mout_mx_mspll_ccore_phy",
+ mout_mx_mspll_ccore_phy_p, SRC_TOP7, 0, 3),
+
MUX(CLK_MOUT_MX_MSPLL_CCORE, "mout_mx_mspll_ccore",
- mout_mx_mspll_ccore_p, SRC_TOP7, 16, 2),
+ mout_mx_mspll_ccore_p, SRC_TOP7, 16, 3),
MUX_F(CLK_MOUT_MAU_EPLL, "mout_mau_epll_clk", mout_mau_epll_clk_5800_p,
SRC_TOP7, 20, 2, CLK_SET_RATE_PARENT, 0),
- MUX(0, "sclk_bpll", mout_bpll_p, SRC_TOP7, 24, 1),
+ MUX(CLK_SCLK_BPLL, "sclk_bpll", mout_bpll_p, SRC_TOP7, 24, 1),
MUX(0, "mout_epll2", mout_epll2_5800_p, SRC_TOP7, 28, 1),
MUX(0, "mout_aclk550_cam", mout_group3_5800_p, SRC_TOP8, 16, 3),
@@ -524,8 +534,6 @@ static const struct samsung_gate_clock exynos5800_gate_clks[] __initconst = {
GATE_BUS_TOP, 24, 0, 0),
GATE(CLK_ACLK432_SCALER, "aclk432_scaler", "mout_user_aclk432_scaler",
GATE_BUS_TOP, 27, CLK_IS_CRITICAL, 0),
- GATE(CLK_MAU_EPLL, "mau_epll", "mout_user_mau_epll",
- SRC_MASK_TOP7, 20, CLK_SET_RATE_PARENT, 0),
};
static const struct samsung_mux_clock exynos5420_mux_clks[] __initconst = {
@@ -567,8 +575,13 @@ static const struct samsung_div_clock exynos5420_div_clks[] __initconst = {
static const struct samsung_gate_clock exynos5420_gate_clks[] __initconst = {
GATE(CLK_SECKEY, "seckey", "aclk66_psgen", GATE_BUS_PERIS1, 1, 0, 0),
+ /* Maudio Block */
GATE(CLK_MAU_EPLL, "mau_epll", "mout_mau_epll_clk",
SRC_MASK_TOP7, 20, CLK_SET_RATE_PARENT, 0),
+ GATE(CLK_SCLK_MAUDIO0, "sclk_maudio0", "dout_maudio0",
+ GATE_TOP_SCLK_MAU, 0, CLK_SET_RATE_PARENT, 0),
+ GATE(CLK_SCLK_MAUPCM0, "sclk_maupcm0", "dout_maupcm0",
+ GATE_TOP_SCLK_MAU, 1, CLK_SET_RATE_PARENT, 0),
};
static const struct samsung_mux_clock exynos5x_mux_clks[] __initconst = {
@@ -645,7 +658,7 @@ static const struct samsung_mux_clock exynos5x_mux_clks[] __initconst = {
MUX(0, "mout_sclk_mpll", mout_mpll_p, SRC_TOP6, 0, 1),
MUX(CLK_MOUT_VPLL, "mout_sclk_vpll", mout_vpll_p, SRC_TOP6, 4, 1),
- MUX(0, "mout_sclk_spll", mout_spll_p, SRC_TOP6, 8, 1),
+ MUX(CLK_MOUT_SCLK_SPLL, "mout_sclk_spll", mout_spll_p, SRC_TOP6, 8, 1),
MUX(0, "mout_sclk_ipll", mout_ipll_p, SRC_TOP6, 12, 1),
MUX(0, "mout_sclk_rpll", mout_rpll_p, SRC_TOP6, 16, 1),
MUX_F(CLK_MOUT_EPLL, "mout_sclk_epll", mout_epll_p, SRC_TOP6, 20, 1,
@@ -803,8 +816,21 @@ static const struct samsung_div_clock exynos5x_div_clks[] __initconst = {
"mout_aclk400_disp1", DIV_TOP2, 4, 3),
/* CDREX Block */
- DIV(CLK_DOUT_PCLK_CDREX, "dout_pclk_cdrex", "dout_aclk_cdrex1",
- DIV_CDREX0, 28, 3),
+ /*
+ * The three clocks below are controlled using the same register and
+ * bits. They are put into one because there is a need of
+ * synchronization between the BUS and DREXs (two external memory
+ * interfaces).
+ * They are put here to show this HW assumption and for clock
+ * information summary completeness.
+ */
+ DIV_F(CLK_DOUT_PCLK_CDREX, "dout_pclk_cdrex", "dout_aclk_cdrex1",
+ DIV_CDREX0, 28, 3, CLK_GET_RATE_NOCACHE, 0),
+ DIV_F(CLK_DOUT_PCLK_DREX0, "dout_pclk_drex0", "dout_cclk_drex0",
+ DIV_CDREX0, 28, 3, CLK_GET_RATE_NOCACHE, 0),
+ DIV_F(CLK_DOUT_PCLK_DREX1, "dout_pclk_drex1", "dout_cclk_drex0",
+ DIV_CDREX0, 28, 3, CLK_GET_RATE_NOCACHE, 0),
+
DIV_F(CLK_DOUT_SCLK_CDREX, "dout_sclk_cdrex", "mout_mclk_cdrex",
DIV_CDREX0, 24, 3, CLK_SET_RATE_PARENT, 0),
DIV(CLK_DOUT_ACLK_CDREX1, "dout_aclk_cdrex1", "dout_clk2x_phy0",
@@ -867,9 +893,6 @@ static const struct samsung_div_clock exynos5x_div_clks[] __initconst = {
/* GSCL Block */
DIV(0, "dout_gscl_blk_333", "aclk333_432_gscl", DIV2_RATIO0, 6, 2),
- /* MSCL Block */
- DIV(0, "dout_mscl_blk", "aclk400_mscl", DIV2_RATIO0, 28, 2),
-
/* PSGEN */
DIV(0, "dout_gen_blk", "mout_user_aclk266", DIV2_RATIO0, 8, 1),
DIV(0, "dout_jpg_blk", "aclk166", DIV2_RATIO0, 20, 1),
@@ -994,12 +1017,6 @@ static const struct samsung_gate_clock exynos5x_gate_clks[] __initconst = {
GATE(CLK_SCLK_DP1, "sclk_dp1", "dout_dp1",
GATE_TOP_SCLK_DISP1, 20, CLK_SET_RATE_PARENT, 0),
- /* Maudio Block */
- GATE(CLK_SCLK_MAUDIO0, "sclk_maudio0", "dout_maudio0",
- GATE_TOP_SCLK_MAU, 0, CLK_SET_RATE_PARENT, 0),
- GATE(CLK_SCLK_MAUPCM0, "sclk_maupcm0", "dout_maupcm0",
- GATE_TOP_SCLK_MAU, 1, CLK_SET_RATE_PARENT, 0),
-
/* FSYS Block */
GATE(CLK_TSI, "tsi", "aclk200_fsys", GATE_BUS_FSYS0, 0, 0, 0),
GATE(CLK_PDMA0, "pdma0", "aclk200_fsys", GATE_BUS_FSYS0, 1, 0, 0),
@@ -1139,17 +1156,6 @@ static const struct samsung_gate_clock exynos5x_gate_clks[] __initconst = {
GATE(CLK_FIMC_LITE3, "fimc_lite3", "aclk333_432_gscl",
GATE_IP_GSCL1, 17, 0, 0),
- /* MSCL Block */
- GATE(CLK_MSCL0, "mscl0", "aclk400_mscl", GATE_IP_MSCL, 0, 0, 0),
- GATE(CLK_MSCL1, "mscl1", "aclk400_mscl", GATE_IP_MSCL, 1, 0, 0),
- GATE(CLK_MSCL2, "mscl2", "aclk400_mscl", GATE_IP_MSCL, 2, 0, 0),
- GATE(CLK_SMMU_MSCL0, "smmu_mscl0", "dout_mscl_blk",
- GATE_IP_MSCL, 8, 0, 0),
- GATE(CLK_SMMU_MSCL1, "smmu_mscl1", "dout_mscl_blk",
- GATE_IP_MSCL, 9, 0, 0),
- GATE(CLK_SMMU_MSCL2, "smmu_mscl2", "dout_mscl_blk",
- GATE_IP_MSCL, 10, 0, 0),
-
/* ISP */
GATE(CLK_SCLK_UART_ISP, "sclk_uart_isp", "dout_uart_isp",
GATE_TOP_SCLK_ISP, 0, CLK_SET_RATE_PARENT, 0),
@@ -1167,6 +1173,32 @@ static const struct samsung_gate_clock exynos5x_gate_clks[] __initconst = {
GATE_TOP_SCLK_ISP, 12, CLK_SET_RATE_PARENT, 0),
GATE(CLK_G3D, "g3d", "mout_user_aclk_g3d", GATE_IP_G3D, 9, 0, 0),
+
+ /* CDREX */
+ GATE(CLK_CLKM_PHY0, "clkm_phy0", "dout_sclk_cdrex",
+ GATE_BUS_CDREX0, 0, 0, 0),
+ GATE(CLK_CLKM_PHY1, "clkm_phy1", "dout_sclk_cdrex",
+ GATE_BUS_CDREX0, 1, 0, 0),
+ GATE(0, "mx_mspll_ccore_phy", "mout_mx_mspll_ccore_phy",
+ SRC_MASK_TOP7, 0, CLK_IGNORE_UNUSED, 0),
+
+ GATE(CLK_ACLK_PPMU_DREX1_1, "aclk_ppmu_drex1_1", "dout_aclk_cdrex1",
+ GATE_BUS_CDREX1, 12, CLK_IGNORE_UNUSED, 0),
+ GATE(CLK_ACLK_PPMU_DREX1_0, "aclk_ppmu_drex1_0", "dout_aclk_cdrex1",
+ GATE_BUS_CDREX1, 13, CLK_IGNORE_UNUSED, 0),
+ GATE(CLK_ACLK_PPMU_DREX0_1, "aclk_ppmu_drex0_1", "dout_aclk_cdrex1",
+ GATE_BUS_CDREX1, 14, CLK_IGNORE_UNUSED, 0),
+ GATE(CLK_ACLK_PPMU_DREX0_0, "aclk_ppmu_drex0_0", "dout_aclk_cdrex1",
+ GATE_BUS_CDREX1, 15, CLK_IGNORE_UNUSED, 0),
+
+ GATE(CLK_PCLK_PPMU_DREX1_1, "pclk_ppmu_drex1_1", "dout_pclk_cdrex",
+ GATE_BUS_CDREX1, 26, CLK_IGNORE_UNUSED, 0),
+ GATE(CLK_PCLK_PPMU_DREX1_0, "pclk_ppmu_drex1_0", "dout_pclk_cdrex",
+ GATE_BUS_CDREX1, 27, CLK_IGNORE_UNUSED, 0),
+ GATE(CLK_PCLK_PPMU_DREX0_1, "pclk_ppmu_drex0_1", "dout_pclk_cdrex",
+ GATE_BUS_CDREX1, 28, CLK_IGNORE_UNUSED, 0),
+ GATE(CLK_PCLK_PPMU_DREX0_0, "pclk_ppmu_drex0_0", "dout_pclk_cdrex",
+ GATE_BUS_CDREX1, 29, CLK_IGNORE_UNUSED, 0),
};
static const struct samsung_div_clock exynos5x_disp_div_clks[] __initconst = {
@@ -1232,32 +1264,103 @@ static struct exynos5_subcmu_reg_dump exynos5x_mfc_suspend_regs[] = {
{ DIV4_RATIO, 0, 0x3 }, /* DIV dout_mfc_blk */
};
-static const struct exynos5_subcmu_info exynos5x_subcmus[] = {
- {
- .div_clks = exynos5x_disp_div_clks,
- .nr_div_clks = ARRAY_SIZE(exynos5x_disp_div_clks),
- .gate_clks = exynos5x_disp_gate_clks,
- .nr_gate_clks = ARRAY_SIZE(exynos5x_disp_gate_clks),
- .suspend_regs = exynos5x_disp_suspend_regs,
- .nr_suspend_regs = ARRAY_SIZE(exynos5x_disp_suspend_regs),
- .pd_name = "DISP",
- }, {
- .div_clks = exynos5x_gsc_div_clks,
- .nr_div_clks = ARRAY_SIZE(exynos5x_gsc_div_clks),
- .gate_clks = exynos5x_gsc_gate_clks,
- .nr_gate_clks = ARRAY_SIZE(exynos5x_gsc_gate_clks),
- .suspend_regs = exynos5x_gsc_suspend_regs,
- .nr_suspend_regs = ARRAY_SIZE(exynos5x_gsc_suspend_regs),
- .pd_name = "GSC",
- }, {
- .div_clks = exynos5x_mfc_div_clks,
- .nr_div_clks = ARRAY_SIZE(exynos5x_mfc_div_clks),
- .gate_clks = exynos5x_mfc_gate_clks,
- .nr_gate_clks = ARRAY_SIZE(exynos5x_mfc_gate_clks),
- .suspend_regs = exynos5x_mfc_suspend_regs,
- .nr_suspend_regs = ARRAY_SIZE(exynos5x_mfc_suspend_regs),
- .pd_name = "MFC",
- },
+static const struct samsung_gate_clock exynos5x_mscl_gate_clks[] __initconst = {
+ /* MSCL Block */
+ GATE(CLK_MSCL0, "mscl0", "aclk400_mscl", GATE_IP_MSCL, 0, 0, 0),
+ GATE(CLK_MSCL1, "mscl1", "aclk400_mscl", GATE_IP_MSCL, 1, 0, 0),
+ GATE(CLK_MSCL2, "mscl2", "aclk400_mscl", GATE_IP_MSCL, 2, 0, 0),
+ GATE(CLK_SMMU_MSCL0, "smmu_mscl0", "dout_mscl_blk",
+ GATE_IP_MSCL, 8, 0, 0),
+ GATE(CLK_SMMU_MSCL1, "smmu_mscl1", "dout_mscl_blk",
+ GATE_IP_MSCL, 9, 0, 0),
+ GATE(CLK_SMMU_MSCL2, "smmu_mscl2", "dout_mscl_blk",
+ GATE_IP_MSCL, 10, 0, 0),
+};
+
+static const struct samsung_div_clock exynos5x_mscl_div_clks[] __initconst = {
+ DIV(0, "dout_mscl_blk", "aclk400_mscl", DIV2_RATIO0, 28, 2),
+};
+
+static struct exynos5_subcmu_reg_dump exynos5x_mscl_suspend_regs[] = {
+ { GATE_IP_MSCL, 0xffffffff, 0xffffffff }, /* MSCL gates */
+ { SRC_TOP3, 0, BIT(4) }, /* MUX mout_user_aclk400_mscl */
+ { DIV2_RATIO0, 0, 0x30000000 }, /* DIV dout_mscl_blk */
+};
+
+static const struct samsung_gate_clock exynos5800_mau_gate_clks[] __initconst = {
+ GATE(CLK_MAU_EPLL, "mau_epll", "mout_user_mau_epll",
+ SRC_MASK_TOP7, 20, CLK_SET_RATE_PARENT, 0),
+ GATE(CLK_SCLK_MAUDIO0, "sclk_maudio0", "dout_maudio0",
+ GATE_TOP_SCLK_MAU, 0, CLK_SET_RATE_PARENT, 0),
+ GATE(CLK_SCLK_MAUPCM0, "sclk_maupcm0", "dout_maupcm0",
+ GATE_TOP_SCLK_MAU, 1, CLK_SET_RATE_PARENT, 0),
+};
+
+static struct exynos5_subcmu_reg_dump exynos5800_mau_suspend_regs[] = {
+ { SRC_TOP9, 0, BIT(8) }, /* MUX mout_user_mau_epll */
+};
+
+static const struct exynos5_subcmu_info exynos5x_disp_subcmu = {
+ .div_clks = exynos5x_disp_div_clks,
+ .nr_div_clks = ARRAY_SIZE(exynos5x_disp_div_clks),
+ .gate_clks = exynos5x_disp_gate_clks,
+ .nr_gate_clks = ARRAY_SIZE(exynos5x_disp_gate_clks),
+ .suspend_regs = exynos5x_disp_suspend_regs,
+ .nr_suspend_regs = ARRAY_SIZE(exynos5x_disp_suspend_regs),
+ .pd_name = "DISP",
+};
+
+static const struct exynos5_subcmu_info exynos5x_gsc_subcmu = {
+ .div_clks = exynos5x_gsc_div_clks,
+ .nr_div_clks = ARRAY_SIZE(exynos5x_gsc_div_clks),
+ .gate_clks = exynos5x_gsc_gate_clks,
+ .nr_gate_clks = ARRAY_SIZE(exynos5x_gsc_gate_clks),
+ .suspend_regs = exynos5x_gsc_suspend_regs,
+ .nr_suspend_regs = ARRAY_SIZE(exynos5x_gsc_suspend_regs),
+ .pd_name = "GSC",
+};
+
+static const struct exynos5_subcmu_info exynos5x_mfc_subcmu = {
+ .div_clks = exynos5x_mfc_div_clks,
+ .nr_div_clks = ARRAY_SIZE(exynos5x_mfc_div_clks),
+ .gate_clks = exynos5x_mfc_gate_clks,
+ .nr_gate_clks = ARRAY_SIZE(exynos5x_mfc_gate_clks),
+ .suspend_regs = exynos5x_mfc_suspend_regs,
+ .nr_suspend_regs = ARRAY_SIZE(exynos5x_mfc_suspend_regs),
+ .pd_name = "MFC",
+};
+
+static const struct exynos5_subcmu_info exynos5x_mscl_subcmu = {
+ .div_clks = exynos5x_mscl_div_clks,
+ .nr_div_clks = ARRAY_SIZE(exynos5x_mscl_div_clks),
+ .gate_clks = exynos5x_mscl_gate_clks,
+ .nr_gate_clks = ARRAY_SIZE(exynos5x_mscl_gate_clks),
+ .suspend_regs = exynos5x_mscl_suspend_regs,
+ .nr_suspend_regs = ARRAY_SIZE(exynos5x_mscl_suspend_regs),
+ .pd_name = "MSC",
+};
+
+static const struct exynos5_subcmu_info exynos5800_mau_subcmu = {
+ .gate_clks = exynos5800_mau_gate_clks,
+ .nr_gate_clks = ARRAY_SIZE(exynos5800_mau_gate_clks),
+ .suspend_regs = exynos5800_mau_suspend_regs,
+ .nr_suspend_regs = ARRAY_SIZE(exynos5800_mau_suspend_regs),
+ .pd_name = "MAU",
+};
+
+static const struct exynos5_subcmu_info *exynos5x_subcmus[] = {
+ &exynos5x_disp_subcmu,
+ &exynos5x_gsc_subcmu,
+ &exynos5x_mfc_subcmu,
+ &exynos5x_mscl_subcmu,
+};
+
+static const struct exynos5_subcmu_info *exynos5800_subcmus[] = {
+ &exynos5x_disp_subcmu,
+ &exynos5x_gsc_subcmu,
+ &exynos5x_mfc_subcmu,
+ &exynos5x_mscl_subcmu,
+ &exynos5800_mau_subcmu,
};
static const struct samsung_pll_rate_table exynos5420_pll2550x_24mhz_tbl[] __initconst = {
@@ -1282,6 +1385,17 @@ static const struct samsung_pll_rate_table exynos5420_pll2550x_24mhz_tbl[] __ini
PLL_35XX_RATE(24 * MHZ, 200000000, 200, 3, 3),
};
+static const struct samsung_pll_rate_table exynos5422_bpll_rate_table[] = {
+ PLL_35XX_RATE(24 * MHZ, 825000000, 275, 4, 1),
+ PLL_35XX_RATE(24 * MHZ, 728000000, 182, 3, 1),
+ PLL_35XX_RATE(24 * MHZ, 633000000, 211, 4, 1),
+ PLL_35XX_RATE(24 * MHZ, 543000000, 181, 2, 2),
+ PLL_35XX_RATE(24 * MHZ, 413000000, 413, 6, 2),
+ PLL_35XX_RATE(24 * MHZ, 275000000, 275, 3, 3),
+ PLL_35XX_RATE(24 * MHZ, 206000000, 206, 3, 3),
+ PLL_35XX_RATE(24 * MHZ, 165000000, 110, 2, 3),
+};
+
static const struct samsung_pll_rate_table exynos5420_epll_24mhz_tbl[] = {
PLL_36XX_RATE(24 * MHZ, 600000000U, 100, 2, 1, 0),
PLL_36XX_RATE(24 * MHZ, 400000000U, 200, 3, 2, 0),
@@ -1424,9 +1538,13 @@ static void __init exynos5x_clk_init(struct device_node *np,
exynos5x_plls[apll].rate_table = exynos5420_pll2550x_24mhz_tbl;
exynos5x_plls[epll].rate_table = exynos5420_epll_24mhz_tbl;
exynos5x_plls[kpll].rate_table = exynos5420_pll2550x_24mhz_tbl;
- exynos5x_plls[bpll].rate_table = exynos5420_pll2550x_24mhz_tbl;
}
+ if (soc == EXYNOS5420)
+ exynos5x_plls[bpll].rate_table = exynos5420_pll2550x_24mhz_tbl;
+ else
+ exynos5x_plls[bpll].rate_table = exynos5422_bpll_rate_table;
+
samsung_clk_register_pll(ctx, exynos5x_plls, ARRAY_SIZE(exynos5x_plls),
reg_base);
samsung_clk_register_fixed_rate(ctx, exynos5x_fixed_rate_clks,
@@ -1475,11 +1593,17 @@ static void __init exynos5x_clk_init(struct device_node *np,
samsung_clk_extended_sleep_init(reg_base,
exynos5x_clk_regs, ARRAY_SIZE(exynos5x_clk_regs),
exynos5420_set_clksrc, ARRAY_SIZE(exynos5420_set_clksrc));
- if (soc == EXYNOS5800)
+
+ if (soc == EXYNOS5800) {
samsung_clk_sleep_init(reg_base, exynos5800_clk_regs,
ARRAY_SIZE(exynos5800_clk_regs));
- exynos5_subcmus_init(ctx, ARRAY_SIZE(exynos5x_subcmus),
- exynos5x_subcmus);
+
+ exynos5_subcmus_init(ctx, ARRAY_SIZE(exynos5800_subcmus),
+ exynos5800_subcmus);
+ } else {
+ exynos5_subcmus_init(ctx, ARRAY_SIZE(exynos5x_subcmus),
+ exynos5x_subcmus);
+ }
samsung_clk_of_add_provider(np, ctx);
}
diff --git a/drivers/clk/samsung/clk-exynos5433.c b/drivers/clk/samsung/clk-exynos5433.c
index 945d5f2ad733..7824c2ba3d8e 100644
--- a/drivers/clk/samsung/clk-exynos5433.c
+++ b/drivers/clk/samsung/clk-exynos5433.c
@@ -5587,8 +5587,8 @@ static int __init exynos5433_cmu_probe(struct platform_device *pdev)
data->nr_clk_save = info->nr_clk_regs;
data->clk_suspend = info->suspend_regs;
data->nr_clk_suspend = info->nr_suspend_regs;
- data->nr_pclks = of_count_phandle_with_args(dev->of_node, "clocks",
- "#clock-cells");
+ data->nr_pclks = of_clk_get_parent_count(dev->of_node);
+
if (data->nr_pclks > 0) {
data->pclks = devm_kcalloc(dev, sizeof(struct clk *),
data->nr_pclks, GFP_KERNEL);
diff --git a/drivers/clk/socfpga/clk-periph-s10.c b/drivers/clk/socfpga/clk-periph-s10.c
index 5c50e723ecae..1a191eeeebba 100644
--- a/drivers/clk/socfpga/clk-periph-s10.c
+++ b/drivers/clk/socfpga/clk-periph-s10.c
@@ -38,7 +38,7 @@ static unsigned long clk_peri_cnt_clk_recalc_rate(struct clk_hw *hwclk,
if (socfpgaclk->fixed_div) {
div = socfpgaclk->fixed_div;
} else {
- if (!socfpgaclk->bypass_reg)
+ if (socfpgaclk->hw.reg)
div = ((readl(socfpgaclk->hw.reg) & 0x7ff) + 1);
}
diff --git a/drivers/clk/socfpga/clk-s10.c b/drivers/clk/socfpga/clk-s10.c
index 5bed36e12951..993f3a73c71e 100644
--- a/drivers/clk/socfpga/clk-s10.c
+++ b/drivers/clk/socfpga/clk-s10.c
@@ -161,8 +161,12 @@ static const struct stratix10_gate_clock s10_gate_clks[] = {
8, 0, 0, 0, 0, 0, 0},
{ STRATIX10_SPI_M_CLK, "spi_m_clk", "l4_mp_clk", NULL, 1, 0, 0xA4,
9, 0, 0, 0, 0, 0, 0},
- { STRATIX10_NAND_CLK, "nand_clk", "l4_main_clk", NULL, 1, 0, 0xA4,
+ { STRATIX10_NAND_X_CLK, "nand_x_clk", "l4_mp_clk", NULL, 1, 0, 0xA4,
10, 0, 0, 0, 0, 0, 0},
+ { STRATIX10_NAND_CLK, "nand_clk", "nand_x_clk", NULL, 1, 0, 0xA4,
+ 10, 0, 0, 0, 0, 0, 4},
+ { STRATIX10_NAND_ECC_CLK, "nand_ecc_clk", "nand_x_clk", NULL, 1, 0, 0xA4,
+ 10, 0, 0, 0, 0, 0, 4},
};
static int s10_clk_register_c_perip(const struct stratix10_perip_c_clock *clks,
diff --git a/drivers/clk/sprd/Kconfig b/drivers/clk/sprd/Kconfig
index 91d3d721c801..3c219af25100 100644
--- a/drivers/clk/sprd/Kconfig
+++ b/drivers/clk/sprd/Kconfig
@@ -3,6 +3,7 @@ config SPRD_COMMON_CLK
tristate "Clock support for Spreadtrum SoCs"
depends on ARCH_SPRD || COMPILE_TEST
default ARCH_SPRD
+ select REGMAP_MMIO
if SPRD_COMMON_CLK
diff --git a/drivers/clk/sprd/common.c b/drivers/clk/sprd/common.c
index e038b0447206..a5bdca1de5d0 100644
--- a/drivers/clk/sprd/common.c
+++ b/drivers/clk/sprd/common.c
@@ -42,6 +42,7 @@ int sprd_clk_regmap_init(struct platform_device *pdev,
void __iomem *base;
struct device_node *node = pdev->dev.of_node;
struct regmap *regmap;
+ struct resource *res;
if (of_find_property(node, "sprd,syscon", NULL)) {
regmap = syscon_regmap_lookup_by_phandle(node, "sprd,syscon");
@@ -50,10 +51,14 @@ int sprd_clk_regmap_init(struct platform_device *pdev,
return PTR_ERR(regmap);
}
} else {
- base = of_iomap(node, 0);
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ base = devm_ioremap_resource(&pdev->dev, res);
+ if (IS_ERR(base))
+ return PTR_ERR(base);
+
regmap = devm_regmap_init_mmio(&pdev->dev, base,
&sprdclk_regmap_config);
- if (IS_ERR_OR_NULL(regmap)) {
+ if (IS_ERR(regmap)) {
pr_err("failed to init regmap\n");
return PTR_ERR(regmap);
}
diff --git a/drivers/clk/sprd/sc9860-clk.c b/drivers/clk/sprd/sc9860-clk.c
index 9980ab55271b..f76305b4bc8d 100644
--- a/drivers/clk/sprd/sc9860-clk.c
+++ b/drivers/clk/sprd/sc9860-clk.c
@@ -2023,6 +2023,7 @@ static int sc9860_clk_probe(struct platform_device *pdev)
{
const struct of_device_id *match;
const struct sprd_clk_desc *desc;
+ int ret;
match = of_match_node(sprd_sc9860_clk_ids, pdev->dev.of_node);
if (!match) {
@@ -2031,7 +2032,9 @@ static int sc9860_clk_probe(struct platform_device *pdev)
}
desc = match->data;
- sprd_clk_regmap_init(pdev, desc);
+ ret = sprd_clk_regmap_init(pdev, desc);
+ if (ret)
+ return ret;
return sprd_clk_probe(&pdev->dev, desc->hw_clks);
}
diff --git a/drivers/clk/sunxi-ng/ccu-sun4i-a10.c b/drivers/clk/sunxi-ng/ccu-sun4i-a10.c
index df43952e403e..f32366d9336e 100644
--- a/drivers/clk/sunxi-ng/ccu-sun4i-a10.c
+++ b/drivers/clk/sunxi-ng/ccu-sun4i-a10.c
@@ -160,8 +160,9 @@ static struct ccu_nk pll_periph_base_clk = {
},
};
-static CLK_FIXED_FACTOR(pll_periph_clk, "pll-periph", "pll-periph-base",
- 2, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HW(pll_periph_clk, "pll-periph",
+ &pll_periph_base_clk.common.hw,
+ 2, 1, CLK_SET_RATE_PARENT);
/* Not documented on A10 */
static struct ccu_div pll_periph_sata_clk = {
@@ -1028,19 +1029,29 @@ static struct ccu_common *sun4i_sun7i_ccu_clks[] = {
&out_b_clk.common
};
+static const struct clk_hw *clk_parent_pll_audio[] = {
+ &pll_audio_base_clk.common.hw
+};
+
/* Post-divider for pll-audio is hardcoded to 1 */
-static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio",
- "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x",
- "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x",
- "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x",
- "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_video0_2x_clk, "pll-video0-2x",
- "pll-video0", 1, 2, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_video1_2x_clk, "pll-video1-2x",
- "pll-video1", 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio",
+ clk_parent_pll_audio,
+ 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x",
+ clk_parent_pll_audio,
+ 2, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x",
+ clk_parent_pll_audio,
+ 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x",
+ clk_parent_pll_audio,
+ 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HW(pll_video0_2x_clk, "pll-video0-2x",
+ &pll_video0_clk.common.hw,
+ 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HW(pll_video1_2x_clk, "pll-video1-2x",
+ &pll_video1_clk.common.hw,
+ 1, 2, CLK_SET_RATE_PARENT);
static struct clk_hw_onecell_data sun4i_a10_hw_clks = {
diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c
index 1786ee8fe8bb..49bd7a4c015c 100644
--- a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c
+++ b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c
@@ -597,23 +597,34 @@ static SUNXI_CCU_M_WITH_GATE(gpu_clk, "gpu", "pll-gpu",
0x1a0, 0, 3, BIT(31), CLK_SET_RATE_PARENT);
/* Fixed Factor clocks */
-static CLK_FIXED_FACTOR(osc12M_clk, "osc12M", "osc24M", 2, 1, 0);
+static CLK_FIXED_FACTOR_FW_NAME(osc12M_clk, "osc12M", "hosc", 2, 1, 0);
+
+static const struct clk_hw *clk_parent_pll_audio[] = {
+ &pll_audio_base_clk.common.hw
+};
/* We hardcode the divider to 1 for now */
-static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio",
- "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x",
- "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x",
- "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x",
- "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_periph0_2x_clk, "pll-periph0-2x",
- "pll-periph0", 1, 2, 0);
-static CLK_FIXED_FACTOR(pll_periph1_2x_clk, "pll-periph1-2x",
- "pll-periph1", 1, 2, 0);
-static CLK_FIXED_FACTOR(pll_video0_2x_clk, "pll-video0-2x",
- "pll-video0", 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio",
+ clk_parent_pll_audio,
+ 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x",
+ clk_parent_pll_audio,
+ 2, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x",
+ clk_parent_pll_audio,
+ 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x",
+ clk_parent_pll_audio,
+ 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HW(pll_periph0_2x_clk, "pll-periph0-2x",
+ &pll_periph0_clk.common.hw,
+ 1, 2, 0);
+static CLK_FIXED_FACTOR_HW(pll_periph1_2x_clk, "pll-periph1-2x",
+ &pll_periph1_clk.common.hw,
+ 1, 2, 0);
+static CLK_FIXED_FACTOR_HW(pll_video0_2x_clk, "pll-video0-2x",
+ &pll_video0_clk.common.hw,
+ 1, 2, CLK_SET_RATE_PARENT);
static struct ccu_common *sun50i_a64_ccu_clks[] = {
&pll_cpux_clk.common,
diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c
index 27554eaf6929..45a1ed3fe674 100644
--- a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c
+++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c
@@ -49,7 +49,7 @@ static struct ccu_div ar100_clk = {
},
};
-static CLK_FIXED_FACTOR(r_ahb_clk, "r-ahb", "ar100", 1, 1, 0);
+static CLK_FIXED_FACTOR_HW(r_ahb_clk, "r-ahb", &ar100_clk.common.hw, 1, 1, 0);
static struct ccu_div r_apb1_clk = {
.div = _SUNXI_CCU_DIV(0, 2),
@@ -104,7 +104,7 @@ static SUNXI_CCU_GATE(r_apb2_i2c_clk, "r-apb2-i2c", "r-apb2",
static SUNXI_CCU_GATE(r_apb1_ir_clk, "r-apb1-ir", "r-apb1",
0x1cc, BIT(0), 0);
static SUNXI_CCU_GATE(r_apb1_w1_clk, "r-apb1-w1", "r-apb1",
- 0x1cc, BIT(0), 0);
+ 0x1ec, BIT(0), 0);
/* Information of IR(RX) mod clock is gathered from BSP source code */
static const char * const r_mod0_default_parents[] = { "osc32k", "osc24M" };
diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c
index 9d3f98962779..aebef4af9861 100644
--- a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c
+++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c
@@ -622,8 +622,9 @@ static SUNXI_CCU_GATE(bus_xhci_clk, "bus-xhci", "ahb3", 0xa8c, BIT(5), 0);
static SUNXI_CCU_GATE(bus_ehci3_clk, "bus-ehci3", "ahb3", 0xa8c, BIT(7), 0);
static SUNXI_CCU_GATE(bus_otg_clk, "bus-otg", "ahb3", 0xa8c, BIT(8), 0);
-static CLK_FIXED_FACTOR(pcie_ref_100m_clk, "pcie-ref-100M",
- "pll-periph0-4x", 24, 1, 0);
+static struct clk_fixed_factor pll_periph0_4x_clk;
+static CLK_FIXED_FACTOR_HW(pcie_ref_100m_clk, "pcie-ref-100M",
+ &pll_periph0_4x_clk.hw, 24, 1, 0);
static SUNXI_CCU_GATE(pcie_ref_clk, "pcie-ref", "pcie-ref-100M",
0xab0, BIT(31), 0);
static SUNXI_CCU_GATE(pcie_ref_out_clk, "pcie-ref-out", "pcie-ref",
@@ -745,34 +746,52 @@ static SUNXI_CCU_M_WITH_MUX_GATE(hdcp_clk, "hdcp", hdcp_parents, 0xc40,
static SUNXI_CCU_GATE(bus_hdcp_clk, "bus-hdcp", "ahb3", 0xc4c, BIT(0), 0);
/* Fixed factor clocks */
-static CLK_FIXED_FACTOR(osc12M_clk, "osc12M", "osc24M", 2, 1, 0);
+static CLK_FIXED_FACTOR_FW_NAME(osc12M_clk, "osc12M", "hosc", 2, 1, 0);
+
+static const struct clk_hw *clk_parent_pll_audio[] = {
+ &pll_audio_base_clk.common.hw
+};
/*
* The divider of pll-audio is fixed to 8 now, as pll-audio-4x has a
* fixed post-divider 2.
*/
-static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio",
- "pll-audio-base", 8, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x",
- "pll-audio-base", 4, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x",
- "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT);
-
-static CLK_FIXED_FACTOR(pll_periph0_4x_clk, "pll-periph0-4x",
- "pll-periph0", 1, 4, 0);
-static CLK_FIXED_FACTOR(pll_periph0_2x_clk, "pll-periph0-2x",
- "pll-periph0", 1, 2, 0);
-
-static CLK_FIXED_FACTOR(pll_periph1_4x_clk, "pll-periph1-4x",
- "pll-periph1", 1, 4, 0);
-static CLK_FIXED_FACTOR(pll_periph1_2x_clk, "pll-periph1-2x",
- "pll-periph1", 1, 2, 0);
-
-static CLK_FIXED_FACTOR(pll_video0_4x_clk, "pll-video0-4x",
- "pll-video0", 1, 4, CLK_SET_RATE_PARENT);
-
-static CLK_FIXED_FACTOR(pll_video1_4x_clk, "pll-video1-4x",
- "pll-video1", 1, 4, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio",
+ clk_parent_pll_audio,
+ 8, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x",
+ clk_parent_pll_audio,
+ 4, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x",
+ clk_parent_pll_audio,
+ 2, 1, CLK_SET_RATE_PARENT);
+
+static const struct clk_hw *pll_periph0_parents[] = {
+ &pll_periph0_clk.common.hw
+};
+static CLK_FIXED_FACTOR_HWS(pll_periph0_4x_clk, "pll-periph0-4x",
+ pll_periph0_parents,
+ 1, 4, 0);
+static CLK_FIXED_FACTOR_HWS(pll_periph0_2x_clk, "pll-periph0-2x",
+ pll_periph0_parents,
+ 1, 2, 0);
+
+static const struct clk_hw *pll_periph1_parents[] = {
+ &pll_periph1_clk.common.hw
+};
+static CLK_FIXED_FACTOR_HWS(pll_periph1_4x_clk, "pll-periph1-4x",
+ pll_periph1_parents,
+ 1, 4, 0);
+static CLK_FIXED_FACTOR_HWS(pll_periph1_2x_clk, "pll-periph1-2x",
+ pll_periph1_parents,
+ 1, 2, 0);
+
+static CLK_FIXED_FACTOR_HW(pll_video0_4x_clk, "pll-video0-4x",
+ &pll_video0_clk.common.hw,
+ 1, 4, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HW(pll_video1_4x_clk, "pll-video1-4x",
+ &pll_video1_clk.common.hw,
+ 1, 4, CLK_SET_RATE_PARENT);
static struct ccu_common *sun50i_h6_ccu_clks[] = {
&pll_cpux_clk.common,
diff --git a/drivers/clk/sunxi-ng/ccu-sun5i.c b/drivers/clk/sunxi-ng/ccu-sun5i.c
index b71ed0f6f785..b78e9b507c1c 100644
--- a/drivers/clk/sunxi-ng/ccu-sun5i.c
+++ b/drivers/clk/sunxi-ng/ccu-sun5i.c
@@ -603,19 +603,29 @@ static struct ccu_common *sun5i_a10s_ccu_clks[] = {
&iep_clk.common,
};
+static const struct clk_hw *clk_parent_pll_audio[] = {
+ &pll_audio_base_clk.common.hw
+};
+
/* We hardcode the divider to 1 for now */
-static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio",
- "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x",
- "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x",
- "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x",
- "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_video0_2x_clk, "pll-video0-2x",
- "pll-video0", 1, 2, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_video1_2x_clk, "pll-video1-2x",
- "pll-video1", 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio",
+ clk_parent_pll_audio,
+ 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x",
+ clk_parent_pll_audio,
+ 2, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x",
+ clk_parent_pll_audio,
+ 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x",
+ clk_parent_pll_audio,
+ 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HW(pll_video0_2x_clk, "pll-video0-2x",
+ &pll_video0_clk.common.hw,
+ 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HW(pll_video1_2x_clk, "pll-video1-2x",
+ &pll_video1_clk.common.hw,
+ 1, 2, CLK_SET_RATE_PARENT);
static struct clk_hw_onecell_data sun5i_a10s_hw_clks = {
.hws = {
diff --git a/drivers/clk/sunxi-ng/ccu-sun6i-a31.c b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c
index 2ff7b082df28..9b40d53266a3 100644
--- a/drivers/clk/sunxi-ng/ccu-sun6i-a31.c
+++ b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c
@@ -955,21 +955,32 @@ static struct ccu_common *sun6i_a31_ccu_clks[] = {
&out_c_clk.common,
};
+static const struct clk_hw *clk_parent_pll_audio[] = {
+ &pll_audio_base_clk.common.hw
+};
+
/* We hardcode the divider to 1 for now */
-static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio",
- "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x",
- "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x",
- "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x",
- "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_periph_2x_clk, "pll-periph-2x",
- "pll-periph", 1, 2, 0);
-static CLK_FIXED_FACTOR(pll_video0_2x_clk, "pll-video0-2x",
- "pll-video0", 1, 2, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_video1_2x_clk, "pll-video1-2x",
- "pll-video1", 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio",
+ clk_parent_pll_audio,
+ 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x",
+ clk_parent_pll_audio,
+ 2, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x",
+ clk_parent_pll_audio,
+ 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x",
+ clk_parent_pll_audio,
+ 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HW(pll_periph_2x_clk, "pll-periph-2x",
+ &pll_periph_clk.common.hw,
+ 1, 2, 0);
+static CLK_FIXED_FACTOR_HW(pll_video0_2x_clk, "pll-video0-2x",
+ &pll_video0_clk.common.hw,
+ 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HW(pll_video1_2x_clk, "pll-video1-2x",
+ &pll_video1_clk.common.hw,
+ 1, 2, CLK_SET_RATE_PARENT);
static struct clk_hw_onecell_data sun6i_a31_hw_clks = {
.hws = {
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c
index 14ced502788a..103aa504f6c8 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c
@@ -543,19 +543,29 @@ static struct ccu_common *sun8i_a23_ccu_clks[] = {
&ats_clk.common,
};
+static const struct clk_hw *clk_parent_pll_audio[] = {
+ &pll_audio_base_clk.common.hw
+};
+
/* We hardcode the divider to 1 for now */
-static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio",
- "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x",
- "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x",
- "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x",
- "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_periph_2x_clk, "pll-periph-2x",
- "pll-periph", 1, 2, 0);
-static CLK_FIXED_FACTOR(pll_video_2x_clk, "pll-video-2x",
- "pll-video", 1, 2, 0);
+static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio",
+ clk_parent_pll_audio,
+ 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x",
+ clk_parent_pll_audio,
+ 2, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x",
+ clk_parent_pll_audio,
+ 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x",
+ clk_parent_pll_audio,
+ 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HW(pll_periph_2x_clk, "pll-periph-2x",
+ &pll_periph_clk.common.hw,
+ 1, 2, 0);
+static CLK_FIXED_FACTOR_HW(pll_video_2x_clk, "pll-video-2x",
+ &pll_video_clk.common.hw,
+ 1, 2, 0);
static struct clk_hw_onecell_data sun8i_a23_hw_clks = {
.hws = {
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a33.c b/drivers/clk/sunxi-ng/ccu-sun8i-a33.c
index 61fb41f4903c..91838cd11037 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-a33.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-a33.c
@@ -580,19 +580,29 @@ static struct ccu_common *sun8i_a33_ccu_clks[] = {
&ats_clk.common,
};
+static const struct clk_hw *clk_parent_pll_audio[] = {
+ &pll_audio_base_clk.common.hw
+};
+
/* We hardcode the divider to 1 for now */
-static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio",
- "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x",
- "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x",
- "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x",
- "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_periph_2x_clk, "pll-periph-2x",
- "pll-periph", 1, 2, 0);
-static CLK_FIXED_FACTOR(pll_video_2x_clk, "pll-video-2x",
- "pll-video", 1, 2, 0);
+static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio",
+ clk_parent_pll_audio,
+ 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x",
+ clk_parent_pll_audio,
+ 2, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x",
+ clk_parent_pll_audio,
+ 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x",
+ clk_parent_pll_audio,
+ 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HW(pll_periph_2x_clk, "pll-periph-2x",
+ &pll_periph_clk.common.hw,
+ 1, 2, 0);
+static CLK_FIXED_FACTOR_HW(pll_video_2x_clk, "pll-video-2x",
+ &pll_video_clk.common.hw,
+ 1, 2, 0);
static struct clk_hw_onecell_data sun8i_a33_hw_clks = {
.hws = {
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c
index 9601504905b2..6b636362379e 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c
@@ -717,17 +717,26 @@ static struct ccu_common *sun50i_h5_ccu_clks[] = {
&gpu_clk.common,
};
+static const struct clk_hw *clk_parent_pll_audio[] = {
+ &pll_audio_base_clk.common.hw
+};
+
/* We hardcode the divider to 1 for now */
-static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio",
- "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x",
- "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x",
- "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x",
- "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_periph0_2x_clk, "pll-periph0-2x",
- "pll-periph0", 1, 2, 0);
+static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio",
+ clk_parent_pll_audio,
+ 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x",
+ clk_parent_pll_audio,
+ 2, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x",
+ clk_parent_pll_audio,
+ 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x",
+ clk_parent_pll_audio,
+ 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HW(pll_periph0_2x_clk, "pll-periph0-2x",
+ &pll_periph0_clk.common.hw,
+ 1, 2, 0);
static struct clk_hw_onecell_data sun8i_h3_hw_clks = {
.hws = {
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r.c b/drivers/clk/sunxi-ng/ccu-sun8i-r.c
index b5be11e5de0d..4646fdc61053 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-r.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-r.c
@@ -17,10 +17,13 @@
#include "ccu-sun8i-r.h"
-static const char * const ar100_parents[] = { "osc32k", "osc24M",
- "pll-periph0", "iosc" };
-static const char * const a83t_ar100_parents[] = { "osc16M-d512", "osc24M",
- "pll-periph0", "iosc" };
+static const struct clk_parent_data ar100_parents[] = {
+ { .fw_name = "losc" },
+ { .fw_name = "hosc" },
+ { .fw_name = "pll-periph" },
+ { .fw_name = "iosc" },
+};
+
static const struct ccu_mux_var_prediv ar100_predivs[] = {
{ .index = 2, .shift = 8, .width = 5 },
};
@@ -39,64 +42,49 @@ static struct ccu_div ar100_clk = {
.common = {
.reg = 0x00,
.features = CCU_FEATURE_VARIABLE_PREDIV,
- .hw.init = CLK_HW_INIT_PARENTS("ar100",
- ar100_parents,
- &ccu_div_ops,
- 0),
- },
-};
-
-static struct ccu_div a83t_ar100_clk = {
- .div = _SUNXI_CCU_DIV_FLAGS(4, 2, CLK_DIVIDER_POWER_OF_TWO),
-
- .mux = {
- .shift = 16,
- .width = 2,
-
- .var_predivs = ar100_predivs,
- .n_var_predivs = ARRAY_SIZE(ar100_predivs),
- },
-
- .common = {
- .reg = 0x00,
- .features = CCU_FEATURE_VARIABLE_PREDIV,
- .hw.init = CLK_HW_INIT_PARENTS("ar100",
- a83t_ar100_parents,
- &ccu_div_ops,
- 0),
+ .hw.init = CLK_HW_INIT_PARENTS_DATA("ar100",
+ ar100_parents,
+ &ccu_div_ops,
+ 0),
},
};
-static CLK_FIXED_FACTOR(ahb0_clk, "ahb0", "ar100", 1, 1, 0);
+static CLK_FIXED_FACTOR_HW(ahb0_clk, "ahb0", &ar100_clk.common.hw, 1, 1, 0);
static struct ccu_div apb0_clk = {
.div = _SUNXI_CCU_DIV_FLAGS(0, 2, CLK_DIVIDER_POWER_OF_TWO),
.common = {
.reg = 0x0c,
- .hw.init = CLK_HW_INIT("apb0",
- "ahb0",
- &ccu_div_ops,
- 0),
+ .hw.init = CLK_HW_INIT_HW("apb0",
+ &ahb0_clk.hw,
+ &ccu_div_ops,
+ 0),
},
};
static SUNXI_CCU_M(a83t_apb0_clk, "apb0", "ahb0", 0x0c, 0, 2, 0);
-static SUNXI_CCU_GATE(apb0_pio_clk, "apb0-pio", "apb0",
- 0x28, BIT(0), 0);
-static SUNXI_CCU_GATE(apb0_ir_clk, "apb0-ir", "apb0",
- 0x28, BIT(1), 0);
-static SUNXI_CCU_GATE(apb0_timer_clk, "apb0-timer", "apb0",
- 0x28, BIT(2), 0);
-static SUNXI_CCU_GATE(apb0_rsb_clk, "apb0-rsb", "apb0",
- 0x28, BIT(3), 0);
-static SUNXI_CCU_GATE(apb0_uart_clk, "apb0-uart", "apb0",
- 0x28, BIT(4), 0);
-static SUNXI_CCU_GATE(apb0_i2c_clk, "apb0-i2c", "apb0",
- 0x28, BIT(6), 0);
-static SUNXI_CCU_GATE(apb0_twd_clk, "apb0-twd", "apb0",
- 0x28, BIT(7), 0);
+/*
+ * Define the parent as an array that can be reused to save space
+ * instead of having compound literals for each gate. Also have it
+ * non-const so we can change it on the A83T.
+ */
+static const struct clk_hw *apb0_gate_parent[] = { &apb0_clk.common.hw };
+static SUNXI_CCU_GATE_HWS(apb0_pio_clk, "apb0-pio",
+ apb0_gate_parent, 0x28, BIT(0), 0);
+static SUNXI_CCU_GATE_HWS(apb0_ir_clk, "apb0-ir",
+ apb0_gate_parent, 0x28, BIT(1), 0);
+static SUNXI_CCU_GATE_HWS(apb0_timer_clk, "apb0-timer",
+ apb0_gate_parent, 0x28, BIT(2), 0);
+static SUNXI_CCU_GATE_HWS(apb0_rsb_clk, "apb0-rsb",
+ apb0_gate_parent, 0x28, BIT(3), 0);
+static SUNXI_CCU_GATE_HWS(apb0_uart_clk, "apb0-uart",
+ apb0_gate_parent, 0x28, BIT(4), 0);
+static SUNXI_CCU_GATE_HWS(apb0_i2c_clk, "apb0-i2c",
+ apb0_gate_parent, 0x28, BIT(6), 0);
+static SUNXI_CCU_GATE_HWS(apb0_twd_clk, "apb0-twd",
+ apb0_gate_parent, 0x28, BIT(7), 0);
static const char * const r_mod0_default_parents[] = { "osc32k", "osc24M" };
static SUNXI_CCU_MP_WITH_MUX_GATE(ir_clk, "ir",
@@ -107,7 +95,10 @@ static SUNXI_CCU_MP_WITH_MUX_GATE(ir_clk, "ir",
BIT(31), /* gate */
0);
-static const char *const a83t_r_mod0_parents[] = { "osc16M", "osc24M" };
+static const struct clk_parent_data a83t_r_mod0_parents[] = {
+ { .fw_name = "iosc" },
+ { .fw_name = "hosc" },
+};
static const struct ccu_mux_fixed_prediv a83t_ir_predivs[] = {
{ .index = 0, .div = 16 },
};
@@ -127,15 +118,15 @@ static struct ccu_mp a83t_ir_clk = {
.common = {
.reg = 0x54,
.features = CCU_FEATURE_VARIABLE_PREDIV,
- .hw.init = CLK_HW_INIT_PARENTS("ir",
- a83t_r_mod0_parents,
- &ccu_mp_ops,
- 0),
+ .hw.init = CLK_HW_INIT_PARENTS_DATA("ir",
+ a83t_r_mod0_parents,
+ &ccu_mp_ops,
+ 0),
},
};
static struct ccu_common *sun8i_a83t_r_ccu_clks[] = {
- &a83t_ar100_clk.common,
+ &ar100_clk.common,
&a83t_apb0_clk.common,
&apb0_pio_clk.common,
&apb0_ir_clk.common,
@@ -174,7 +165,7 @@ static struct ccu_common *sun50i_a64_r_ccu_clks[] = {
static struct clk_hw_onecell_data sun8i_a83t_r_hw_clks = {
.hws = {
- [CLK_AR100] = &a83t_ar100_clk.common.hw,
+ [CLK_AR100] = &ar100_clk.common.hw,
[CLK_AHB0] = &ahb0_clk.hw,
[CLK_APB0] = &a83t_apb0_clk.common.hw,
[CLK_APB0_PIO] = &apb0_pio_clk.common.hw,
@@ -291,6 +282,9 @@ static void __init sunxi_r_ccu_init(struct device_node *node,
static void __init sun8i_a83t_r_ccu_setup(struct device_node *node)
{
+ /* Fix apb0 bus gate parents here */
+ apb0_gate_parent[0] = &a83t_apb0_clk.common.hw;
+
sunxi_r_ccu_init(node, &sun8i_a83t_r_ccu_desc);
}
CLK_OF_DECLARE(sun8i_a83t_r_ccu, "allwinner,sun8i-a83t-r-ccu",
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r40.c b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c
index 540f5f7454fc..897490800102 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-r40.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c
@@ -944,25 +944,37 @@ static struct ccu_common *sun8i_r40_ccu_clks[] = {
};
/* Fixed Factor clocks */
-static CLK_FIXED_FACTOR(osc12M_clk, "osc12M", "osc24M", 2, 1, 0);
+static CLK_FIXED_FACTOR_FW_NAME(osc12M_clk, "osc12M", "hosc", 2, 1, 0);
+
+static const struct clk_hw *clk_parent_pll_audio[] = {
+ &pll_audio_base_clk.common.hw
+};
/* We hardcode the divider to 4 for now */
-static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio",
- "pll-audio-base", 4, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x",
- "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x",
- "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x",
- "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_periph0_2x_clk, "pll-periph0-2x",
- "pll-periph0", 1, 2, 0);
-static CLK_FIXED_FACTOR(pll_periph1_2x_clk, "pll-periph1-2x",
- "pll-periph1", 1, 2, 0);
-static CLK_FIXED_FACTOR(pll_video0_2x_clk, "pll-video0-2x",
- "pll-video0", 1, 2, 0);
-static CLK_FIXED_FACTOR(pll_video1_2x_clk, "pll-video1-2x",
- "pll-video1", 1, 2, 0);
+static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio",
+ clk_parent_pll_audio,
+ 4, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x",
+ clk_parent_pll_audio,
+ 2, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x",
+ clk_parent_pll_audio,
+ 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x",
+ clk_parent_pll_audio,
+ 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HW(pll_periph0_2x_clk, "pll-periph0-2x",
+ &pll_periph0_clk.common.hw,
+ 1, 2, 0);
+static CLK_FIXED_FACTOR_HW(pll_periph1_2x_clk, "pll-periph1-2x",
+ &pll_periph1_clk.common.hw,
+ 1, 2, 0);
+static CLK_FIXED_FACTOR_HW(pll_video0_2x_clk, "pll-video0-2x",
+ &pll_video0_clk.common.hw,
+ 1, 2, 0);
+static CLK_FIXED_FACTOR_HW(pll_video1_2x_clk, "pll-video1-2x",
+ &pll_video1_clk.common.hw,
+ 1, 2, 0);
static struct clk_hw_onecell_data sun8i_r40_hw_clks = {
.hws = {
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c
index cbbf06d42c2c..9b3939fc7faa 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c
@@ -429,17 +429,26 @@ static struct ccu_common *sun8i_v3s_ccu_clks[] = {
&mipi_csi_clk.common,
};
+static const struct clk_hw *clk_parent_pll_audio[] = {
+ &pll_audio_base_clk.common.hw
+};
+
/* We hardcode the divider to 4 for now */
-static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio",
- "pll-audio-base", 4, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x",
- "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x",
- "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x",
- "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_periph0_2x_clk, "pll-periph0-2x",
- "pll-periph0", 1, 2, 0);
+static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio",
+ clk_parent_pll_audio,
+ 4, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x",
+ clk_parent_pll_audio,
+ 2, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x",
+ clk_parent_pll_audio,
+ 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x",
+ clk_parent_pll_audio,
+ 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HW(pll_periph0_2x_clk, "pll-periph0-2x",
+ &pll_periph0_clk.common.hw,
+ 1, 2, 0);
static struct clk_hw_onecell_data sun8i_v3s_hw_clks = {
.hws = {
diff --git a/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c b/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c
index 2f82cd855b0f..4b4a507d04ed 100644
--- a/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c
+++ b/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c
@@ -14,18 +14,26 @@
#include "ccu-sun9i-a80-usb.h"
-static SUNXI_CCU_GATE(bus_hci0_clk, "bus-hci0", "bus-usb", 0x0, BIT(1), 0);
-static SUNXI_CCU_GATE(usb_ohci0_clk, "usb-ohci0", "osc24M", 0x0, BIT(2), 0);
-static SUNXI_CCU_GATE(bus_hci1_clk, "bus-hci1", "bus-usb", 0x0, BIT(3), 0);
-static SUNXI_CCU_GATE(bus_hci2_clk, "bus-hci2", "bus-usb", 0x0, BIT(5), 0);
-static SUNXI_CCU_GATE(usb_ohci2_clk, "usb-ohci2", "osc24M", 0x0, BIT(6), 0);
-
-static SUNXI_CCU_GATE(usb0_phy_clk, "usb0-phy", "osc24M", 0x4, BIT(1), 0);
-static SUNXI_CCU_GATE(usb1_hsic_clk, "usb1-hsic", "osc24M", 0x4, BIT(2), 0);
-static SUNXI_CCU_GATE(usb1_phy_clk, "usb1-phy", "osc24M", 0x4, BIT(3), 0);
-static SUNXI_CCU_GATE(usb2_hsic_clk, "usb2-hsic", "osc24M", 0x4, BIT(4), 0);
-static SUNXI_CCU_GATE(usb2_phy_clk, "usb2-phy", "osc24M", 0x4, BIT(5), 0);
-static SUNXI_CCU_GATE(usb_hsic_clk, "usb-hsic", "osc24M", 0x4, BIT(10), 0);
+static const struct clk_parent_data clk_parent_hosc[] = {
+ { .fw_name = "hosc" },
+};
+
+static const struct clk_parent_data clk_parent_bus[] = {
+ { .fw_name = "bus" },
+};
+
+static SUNXI_CCU_GATE_DATA(bus_hci0_clk, "bus-hci0", clk_parent_bus, 0x0, BIT(1), 0);
+static SUNXI_CCU_GATE_DATA(usb_ohci0_clk, "usb-ohci0", clk_parent_hosc, 0x0, BIT(2), 0);
+static SUNXI_CCU_GATE_DATA(bus_hci1_clk, "bus-hci1", clk_parent_bus, 0x0, BIT(3), 0);
+static SUNXI_CCU_GATE_DATA(bus_hci2_clk, "bus-hci2", clk_parent_bus, 0x0, BIT(5), 0);
+static SUNXI_CCU_GATE_DATA(usb_ohci2_clk, "usb-ohci2", clk_parent_hosc, 0x0, BIT(6), 0);
+
+static SUNXI_CCU_GATE_DATA(usb0_phy_clk, "usb0-phy", clk_parent_hosc, 0x4, BIT(1), 0);
+static SUNXI_CCU_GATE_DATA(usb1_hsic_clk, "usb1-hsic", clk_parent_hosc, 0x4, BIT(2), 0);
+static SUNXI_CCU_GATE_DATA(usb1_phy_clk, "usb1-phy", clk_parent_hosc, 0x4, BIT(3), 0);
+static SUNXI_CCU_GATE_DATA(usb2_hsic_clk, "usb2-hsic", clk_parent_hosc, 0x4, BIT(4), 0);
+static SUNXI_CCU_GATE_DATA(usb2_phy_clk, "usb2-phy", clk_parent_hosc, 0x4, BIT(5), 0);
+static SUNXI_CCU_GATE_DATA(usb_hsic_clk, "usb-hsic", clk_parent_hosc, 0x4, BIT(10), 0);
static struct ccu_common *sun9i_a80_usb_clks[] = {
&bus_hci0_clk.common,
diff --git a/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c b/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c
index e748b8a6f3c5..7ecc3a5a5b5e 100644
--- a/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c
+++ b/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c
@@ -374,16 +374,25 @@ static struct ccu_common *suniv_ccu_clks[] = {
&avs_clk.common,
};
-static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio",
- "pll-audio-base", 4, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x",
- "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x",
- "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x",
- "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT);
-static CLK_FIXED_FACTOR(pll_video_2x_clk, "pll-video-2x",
- "pll-video", 1, 2, 0);
+static const struct clk_hw *clk_parent_pll_audio[] = {
+ &pll_audio_base_clk.common.hw
+};
+
+static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio",
+ clk_parent_pll_audio,
+ 4, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x",
+ clk_parent_pll_audio,
+ 2, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x",
+ clk_parent_pll_audio,
+ 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x",
+ clk_parent_pll_audio,
+ 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR_HW(pll_video_2x_clk, "pll-video-2x",
+ &pll_video_clk.common.hw,
+ 1, 2, 0);
static struct clk_hw_onecell_data suniv_hw_clks = {
.hws = {
diff --git a/drivers/clk/sunxi-ng/ccu_common.c b/drivers/clk/sunxi-ng/ccu_common.c
index c173778c8a78..7fe3ac980e5f 100644
--- a/drivers/clk/sunxi-ng/ccu_common.c
+++ b/drivers/clk/sunxi-ng/ccu_common.c
@@ -101,7 +101,7 @@ int sunxi_ccu_probe(struct device_node *node, void __iomem *reg,
if (!hw)
continue;
- ret = clk_hw_register(NULL, hw);
+ ret = of_clk_hw_register(node, hw);
if (ret) {
pr_err("Couldn't register clock %d - %s\n",
i, clk_hw_get_name(hw));
diff --git a/drivers/clk/sunxi-ng/ccu_gate.h b/drivers/clk/sunxi-ng/ccu_gate.h
index da8100e8846d..c386689a952b 100644
--- a/drivers/clk/sunxi-ng/ccu_gate.h
+++ b/drivers/clk/sunxi-ng/ccu_gate.h
@@ -28,6 +28,59 @@ struct ccu_gate {
} \
}
+#define SUNXI_CCU_GATE_HW(_struct, _name, _parent, _reg, _gate, _flags) \
+ struct ccu_gate _struct = { \
+ .enable = _gate, \
+ .common = { \
+ .reg = _reg, \
+ .hw.init = CLK_HW_INIT_HW(_name, \
+ _parent, \
+ &ccu_gate_ops, \
+ _flags), \
+ } \
+ }
+
+#define SUNXI_CCU_GATE_FW(_struct, _name, _parent, _reg, _gate, _flags) \
+ struct ccu_gate _struct = { \
+ .enable = _gate, \
+ .common = { \
+ .reg = _reg, \
+ .hw.init = CLK_HW_INIT_FW_NAME(_name, \
+ _parent, \
+ &ccu_gate_ops, \
+ _flags), \
+ } \
+ }
+
+/*
+ * The following two macros allow the re-use of the data structure
+ * holding the parent info.
+ */
+#define SUNXI_CCU_GATE_HWS(_struct, _name, _parent, _reg, _gate, _flags) \
+ struct ccu_gate _struct = { \
+ .enable = _gate, \
+ .common = { \
+ .reg = _reg, \
+ .hw.init = CLK_HW_INIT_HWS(_name, \
+ _parent, \
+ &ccu_gate_ops, \
+ _flags), \
+ } \
+ }
+
+#define SUNXI_CCU_GATE_DATA(_struct, _name, _data, _reg, _gate, _flags) \
+ struct ccu_gate _struct = { \
+ .enable = _gate, \
+ .common = { \
+ .reg = _reg, \
+ .hw.init = \
+ CLK_HW_INIT_PARENTS_DATA(_name, \
+ _data, \
+ &ccu_gate_ops, \
+ _flags), \
+ } \
+ }
+
static inline struct ccu_gate *hw_to_ccu_gate(struct clk_hw *hw)
{
struct ccu_common *common = hw_to_ccu_common(hw);
diff --git a/drivers/clk/sunxi/clk-sunxi.c b/drivers/clk/sunxi/clk-sunxi.c
index 623fda5e911f..d3a43381a792 100644
--- a/drivers/clk/sunxi/clk-sunxi.c
+++ b/drivers/clk/sunxi/clk-sunxi.c
@@ -980,6 +980,8 @@ static struct clk ** __init sunxi_divs_clk_setup(struct device_node *node,
if (endp) {
derived_name = kstrndup(clk_name, endp - clk_name,
GFP_KERNEL);
+ if (!derived_name)
+ return NULL;
factors.name = derived_name;
} else {
factors.name = clk_name;
diff --git a/drivers/clk/tegra/clk-tegra210.c b/drivers/clk/tegra/clk-tegra210.c
index ac1d27a8c650..df172d5772d7 100644
--- a/drivers/clk/tegra/clk-tegra210.c
+++ b/drivers/clk/tegra/clk-tegra210.c
@@ -984,8 +984,6 @@ static void tegra210_pllre_set_defaults(struct tegra_clk_pll *pllre)
pllre->params->defaults_set = true;
if (val & PLL_ENABLE) {
- pr_warn("PLL_RE already enabled. Postponing set full defaults\n");
-
/*
* PLL is ON: check if defaults already set, then set those
* that can be updated in flight.
@@ -1005,13 +1003,20 @@ static void tegra210_pllre_set_defaults(struct tegra_clk_pll *pllre)
_pll_misc_chk_default(clk_base, pllre->params, 0, val,
~mask & PLLRE_MISC0_WRITE_MASK);
- /* Enable lock detect */
+ /* The PLL doesn't work if it's in IDDQ. */
val = readl_relaxed(clk_base + pllre->params->ext_misc_reg[0]);
+ if (val & PLLRE_MISC0_IDDQ)
+ pr_warn("unexpected IDDQ bit set for enabled clock\n");
+
+ /* Enable lock detect */
val &= ~mask;
val |= PLLRE_MISC0_DEFAULT_VALUE & mask;
writel_relaxed(val, clk_base + pllre->params->ext_misc_reg[0]);
udelay(1);
+ if (!pllre->params->defaults_set)
+ pr_warn("PLL_RE already enabled. Postponing set full defaults\n");
+
return;
}
@@ -2204,9 +2209,9 @@ static struct div_nmp pllu_nmp = {
};
static struct tegra_clk_pll_freq_table pll_u_freq_table[] = {
- { 12000000, 480000000, 40, 1, 0, 0 },
- { 13000000, 480000000, 36, 1, 0, 0 }, /* actual: 468.0 MHz */
- { 38400000, 480000000, 25, 2, 0, 0 },
+ { 12000000, 480000000, 40, 1, 1, 0 },
+ { 13000000, 480000000, 36, 1, 1, 0 }, /* actual: 468.0 MHz */
+ { 38400000, 480000000, 25, 2, 1, 0 },
{ 0, 0, 0, 0, 0, 0 },
};
@@ -3332,7 +3337,7 @@ static struct tegra_clk_init_table init_table[] __initdata = {
{ TEGRA210_CLK_DFLL_SOC, TEGRA210_CLK_PLL_P, 51000000, 1 },
{ TEGRA210_CLK_DFLL_REF, TEGRA210_CLK_PLL_P, 51000000, 1 },
{ TEGRA210_CLK_SBC4, TEGRA210_CLK_PLL_P, 12000000, 1 },
- { TEGRA210_CLK_PLL_RE_VCO, TEGRA210_CLK_CLK_MAX, 672000000, 1 },
+ { TEGRA210_CLK_PLL_U_OUT1, TEGRA210_CLK_CLK_MAX, 48000000, 1 },
{ TEGRA210_CLK_XUSB_GATE, TEGRA210_CLK_CLK_MAX, 0, 1 },
{ TEGRA210_CLK_XUSB_SS_SRC, TEGRA210_CLK_PLL_U_480M, 120000000, 0 },
{ TEGRA210_CLK_XUSB_FS_SRC, TEGRA210_CLK_PLL_U_48M, 48000000, 0 },
@@ -3357,7 +3362,6 @@ static struct tegra_clk_init_table init_table[] __initdata = {
{ TEGRA210_CLK_PLL_DP, TEGRA210_CLK_CLK_MAX, 270000000, 0 },
{ TEGRA210_CLK_SOC_THERM, TEGRA210_CLK_PLL_P, 51000000, 0 },
{ TEGRA210_CLK_CCLK_G, TEGRA210_CLK_CLK_MAX, 0, 1 },
- { TEGRA210_CLK_PLL_U_OUT1, TEGRA210_CLK_CLK_MAX, 48000000, 1 },
{ TEGRA210_CLK_PLL_U_OUT2, TEGRA210_CLK_CLK_MAX, 60000000, 1 },
{ TEGRA210_CLK_SPDIF_IN_SYNC, TEGRA210_CLK_CLK_MAX, 24576000, 0 },
{ TEGRA210_CLK_I2S0_SYNC, TEGRA210_CLK_CLK_MAX, 24576000, 0 },
diff --git a/drivers/clk/ti/divider.c b/drivers/clk/ti/divider.c
index 4786e0ebc2e8..6cb863c13648 100644
--- a/drivers/clk/ti/divider.c
+++ b/drivers/clk/ti/divider.c
@@ -425,91 +425,6 @@ int ti_clk_parse_divider_data(int *div_table, int num_dividers, int max_div,
return 0;
}
-static const struct clk_div_table *
-_get_div_table_from_setup(struct ti_clk_divider *setup, u8 *width)
-{
- const struct clk_div_table *table = NULL;
-
- ti_clk_parse_divider_data(setup->dividers, setup->num_dividers,
- setup->max_div, setup->flags, width,
- &table);
-
- return table;
-}
-
-struct clk_hw *ti_clk_build_component_div(struct ti_clk_divider *setup)
-{
- struct clk_omap_divider *div;
- struct clk_omap_reg *reg;
- int ret;
-
- if (!setup)
- return NULL;
-
- div = kzalloc(sizeof(*div), GFP_KERNEL);
- if (!div)
- return ERR_PTR(-ENOMEM);
-
- reg = (struct clk_omap_reg *)&div->reg;
- reg->index = setup->module;
- reg->offset = setup->reg;
-
- if (setup->flags & CLKF_INDEX_STARTS_AT_ONE)
- div->flags |= CLK_DIVIDER_ONE_BASED;
-
- if (setup->flags & CLKF_INDEX_POWER_OF_TWO)
- div->flags |= CLK_DIVIDER_POWER_OF_TWO;
-
- div->table = _get_div_table_from_setup(setup, &div->width);
- if (IS_ERR(div->table)) {
- ret = PTR_ERR(div->table);
- kfree(div);
- return ERR_PTR(ret);
- }
-
-
- div->shift = setup->bit_shift;
- div->latch = -EINVAL;
-
- return &div->hw;
-}
-
-struct clk *ti_clk_register_divider(struct ti_clk *setup)
-{
- struct ti_clk_divider *div = setup->data;
- struct clk_omap_reg reg = {
- .index = div->module,
- .offset = div->reg,
- };
- u8 width;
- u32 flags = 0;
- u8 div_flags = 0;
- const struct clk_div_table *table;
- struct clk *clk;
-
- if (div->flags & CLKF_INDEX_STARTS_AT_ONE)
- div_flags |= CLK_DIVIDER_ONE_BASED;
-
- if (div->flags & CLKF_INDEX_POWER_OF_TWO)
- div_flags |= CLK_DIVIDER_POWER_OF_TWO;
-
- if (div->flags & CLKF_SET_RATE_PARENT)
- flags |= CLK_SET_RATE_PARENT;
-
- table = _get_div_table_from_setup(div, &width);
- if (IS_ERR(table))
- return (struct clk *)table;
-
- clk = _register_divider(NULL, setup->name, div->parent,
- flags, &reg, div->bit_shift,
- width, -EINVAL, div_flags, table);
-
- if (IS_ERR(clk))
- kfree(table);
-
- return clk;
-}
-
static struct clk_div_table *
__init ti_clk_get_div_table(struct device_node *node)
{
diff --git a/drivers/clk/ti/gate.c b/drivers/clk/ti/gate.c
index 504c0e91cdc7..42389558418c 100644
--- a/drivers/clk/ti/gate.c
+++ b/drivers/clk/ti/gate.c
@@ -131,36 +131,6 @@ static struct clk *_register_gate(struct device *dev, const char *name,
return clk;
}
-struct clk_hw *ti_clk_build_component_gate(struct ti_clk_gate *setup)
-{
- struct clk_hw_omap *gate;
- struct clk_omap_reg *reg;
- const struct clk_hw_omap_ops *ops = &clkhwops_wait;
-
- if (!setup)
- return NULL;
-
- gate = kzalloc(sizeof(*gate), GFP_KERNEL);
- if (!gate)
- return ERR_PTR(-ENOMEM);
-
- reg = (struct clk_omap_reg *)&gate->enable_reg;
- reg->index = setup->module;
- reg->offset = setup->reg;
-
- gate->enable_bit = setup->bit_shift;
-
- if (setup->flags & CLKF_NO_WAIT)
- ops = NULL;
-
- if (setup->flags & CLKF_INTERFACE)
- ops = &clkhwops_iclk_wait;
-
- gate->ops = ops;
-
- return &gate->hw;
-}
-
static void __init _of_ti_gate_clk_setup(struct device_node *node,
const struct clk_ops *ops,
const struct clk_hw_omap_ops *hw_ops)
diff --git a/drivers/clk/ti/mux.c b/drivers/clk/ti/mux.c
index b7f9a4f068bf..0069e7cf3ebc 100644
--- a/drivers/clk/ti/mux.c
+++ b/drivers/clk/ti/mux.c
@@ -164,37 +164,6 @@ static struct clk *_register_mux(struct device *dev, const char *name,
return clk;
}
-struct clk *ti_clk_register_mux(struct ti_clk *setup)
-{
- struct ti_clk_mux *mux;
- u32 flags;
- u8 mux_flags = 0;
- struct clk_omap_reg reg;
- u32 mask;
-
- mux = setup->data;
- flags = CLK_SET_RATE_NO_REPARENT;
-
- mask = mux->num_parents;
- if (!(mux->flags & CLKF_INDEX_STARTS_AT_ONE))
- mask--;
-
- mask = (1 << fls(mask)) - 1;
- reg.index = mux->module;
- reg.offset = mux->reg;
- reg.ptr = NULL;
-
- if (mux->flags & CLKF_INDEX_STARTS_AT_ONE)
- mux_flags |= CLK_MUX_INDEX_ONE;
-
- if (mux->flags & CLKF_SET_RATE_PARENT)
- flags |= CLK_SET_RATE_PARENT;
-
- return _register_mux(NULL, setup->name, mux->parents, mux->num_parents,
- flags, &reg, mux->bit_shift, mask, -EINVAL,
- mux_flags, NULL);
-}
-
/**
* of_mux_clk_setup - Setup function for simple mux rate clock
* @node: DT node for the clock
diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig
index 3300739edce4..a642c23b2fba 100644
--- a/drivers/clocksource/Kconfig
+++ b/drivers/clocksource/Kconfig
@@ -43,6 +43,11 @@ config BCM_KONA_TIMER
help
Enables the support for the BCM Kona mobile timer driver.
+config DAVINCI_TIMER
+ bool "Texas Instruments DaVinci timer driver" if COMPILE_TEST
+ help
+ Enables the support for the TI DaVinci timer driver.
+
config DIGICOLOR_TIMER
bool "Digicolor timer driver" if COMPILE_TEST
select CLKSRC_MMIO
@@ -140,7 +145,7 @@ config TEGRA_TIMER
bool "Tegra timer driver" if COMPILE_TEST
select CLKSRC_MMIO
select TIMER_OF
- depends on ARM || ARM64
+ depends on ARCH_TEGRA || COMPILE_TEST
help
Enables support for the Tegra driver.
@@ -424,7 +429,7 @@ config ATMEL_ST
config ATMEL_TCB_CLKSRC
bool "Atmel TC Block timer driver" if COMPILE_TEST
- depends on HAS_IOMEM
+ depends on ARM && HAS_IOMEM
select TIMER_OF if OF
help
Support for Timer Counter Blocks on Atmel SoCs.
@@ -617,6 +622,13 @@ config CLKSRC_IMX_TPM
Enable this option to use IMX Timer/PWM Module (TPM) timer as
clocksource.
+config TIMER_IMX_SYS_CTR
+ bool "i.MX system counter timer" if COMPILE_TEST
+ select TIMER_OF
+ help
+ Enable this option to use i.MX system counter timer as a
+ clockevent.
+
config CLKSRC_ST_LPC
bool "Low power clocksource found in the LPC" if COMPILE_TEST
select TIMER_OF if OF
diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile
index 236858fa7fbf..2e7936e7833f 100644
--- a/drivers/clocksource/Makefile
+++ b/drivers/clocksource/Makefile
@@ -15,6 +15,7 @@ obj-$(CONFIG_SH_TIMER_TMU) += sh_tmu.o
obj-$(CONFIG_EM_TIMER_STI) += em_sti.o
obj-$(CONFIG_CLKBLD_I8253) += i8253.o
obj-$(CONFIG_CLKSRC_MMIO) += mmio.o
+obj-$(CONFIG_DAVINCI_TIMER) += timer-davinci.o
obj-$(CONFIG_DIGICOLOR_TIMER) += timer-digicolor.o
obj-$(CONFIG_OMAP_DM_TIMER) += timer-ti-dm.o
obj-$(CONFIG_DW_APB_TIMER) += dw_apb_timer.o
@@ -36,7 +37,7 @@ obj-$(CONFIG_U300_TIMER) += timer-u300.o
obj-$(CONFIG_SUN4I_TIMER) += timer-sun4i.o
obj-$(CONFIG_SUN5I_HSTIMER) += timer-sun5i.o
obj-$(CONFIG_MESON6_TIMER) += timer-meson6.o
-obj-$(CONFIG_TEGRA_TIMER) += timer-tegra20.o
+obj-$(CONFIG_TEGRA_TIMER) += timer-tegra.o
obj-$(CONFIG_VT8500_TIMER) += timer-vt8500.o
obj-$(CONFIG_NSPIRE_TIMER) += timer-zevio.o
obj-$(CONFIG_BCM_KONA_TIMER) += bcm_kona_timer.o
@@ -74,6 +75,7 @@ obj-$(CONFIG_CLKSRC_MIPS_GIC) += mips-gic-timer.o
obj-$(CONFIG_CLKSRC_TANGO_XTAL) += timer-tango-xtal.o
obj-$(CONFIG_CLKSRC_IMX_GPT) += timer-imx-gpt.o
obj-$(CONFIG_CLKSRC_IMX_TPM) += timer-imx-tpm.o
+obj-$(CONFIG_TIMER_IMX_SYS_CTR) += timer-imx-sysctr.o
obj-$(CONFIG_ASM9260_TIMER) += asm9260_timer.o
obj-$(CONFIG_H8300_TMR8) += h8300_timer8.o
obj-$(CONFIG_H8300_TMR16) += h8300_timer16.o
@@ -84,3 +86,4 @@ obj-$(CONFIG_ATCPIT100_TIMER) += timer-atcpit100.o
obj-$(CONFIG_RISCV_TIMER) += timer-riscv.o
obj-$(CONFIG_CSKY_MP_TIMER) += timer-mp-csky.o
obj-$(CONFIG_GX6605S_TIMER) += timer-gx6605s.o
+obj-$(CONFIG_HYPERV_TIMER) += hyperv_timer.o
diff --git a/drivers/clocksource/arc_timer.c b/drivers/clocksource/arc_timer.c
index ebfbccefc7b3..b29b5a75333e 100644
--- a/drivers/clocksource/arc_timer.c
+++ b/drivers/clocksource/arc_timer.c
@@ -13,6 +13,7 @@
*/
#include <linux/interrupt.h>
+#include <linux/bits.h>
#include <linux/clk.h>
#include <linux/clk-provider.h>
#include <linux/clocksource.h>
@@ -139,7 +140,7 @@ static u64 arc_read_rtc(struct clocksource *cs)
l = read_aux_reg(AUX_RTC_LOW);
h = read_aux_reg(AUX_RTC_HIGH);
status = read_aux_reg(AUX_RTC_CTRL);
- } while (!(status & _BITUL(31)));
+ } while (!(status & BIT(31)));
return (((u64)h) << 32) | l;
}
diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c
index 07e57a49d1e8..9a5464c625b4 100644
--- a/drivers/clocksource/arm_arch_timer.c
+++ b/drivers/clocksource/arm_arch_timer.c
@@ -801,14 +801,7 @@ static void arch_timer_evtstrm_enable(int divider)
cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
| ARCH_TIMER_VIRT_EVT_EN;
arch_timer_set_cntkctl(cntkctl);
-#ifdef CONFIG_ARM64
- cpu_set_named_feature(EVTSTRM);
-#else
- elf_hwcap |= HWCAP_EVTSTRM;
-#endif
-#ifdef CONFIG_COMPAT
- compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
-#endif
+ arch_timer_set_evtstrm_feature();
cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
}
@@ -1037,11 +1030,7 @@ static int arch_timer_cpu_pm_notify(struct notifier_block *self,
} else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) {
arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl));
-#ifdef CONFIG_ARM64
- if (cpu_have_named_feature(EVTSTRM))
-#else
- if (elf_hwcap & HWCAP_EVTSTRM)
-#endif
+ if (arch_timer_have_evtstrm_feature())
cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
}
return NOTIFY_OK;
diff --git a/drivers/clocksource/em_sti.c b/drivers/clocksource/em_sti.c
index 8e12b11e81b0..9039df4f90e2 100644
--- a/drivers/clocksource/em_sti.c
+++ b/drivers/clocksource/em_sti.c
@@ -291,10 +291,8 @@ static int em_sti_probe(struct platform_device *pdev)
platform_set_drvdata(pdev, p);
irq = platform_get_irq(pdev, 0);
- if (irq < 0) {
- dev_err(&pdev->dev, "failed to get irq\n");
+ if (irq < 0)
return irq;
- }
/* map memory, let base point to the STI instance */
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
diff --git a/drivers/clocksource/exynos_mct.c b/drivers/clocksource/exynos_mct.c
index e8eab16b154b..74cb299f5089 100644
--- a/drivers/clocksource/exynos_mct.c
+++ b/drivers/clocksource/exynos_mct.c
@@ -206,7 +206,7 @@ static void exynos4_frc_resume(struct clocksource *cs)
static struct clocksource mct_frc = {
.name = "mct-frc",
- .rating = 400,
+ .rating = 450, /* use value higher than ARM arch timer */
.read = exynos4_frc_read,
.mask = CLOCKSOURCE_MASK(32),
.flags = CLOCK_SOURCE_IS_CONTINUOUS,
@@ -461,7 +461,7 @@ static int exynos4_mct_starting_cpu(unsigned int cpu)
evt->set_state_oneshot_stopped = set_state_shutdown;
evt->tick_resume = set_state_shutdown;
evt->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT;
- evt->rating = 450;
+ evt->rating = 500; /* use value higher than ARM arch timer */
exynos4_mct_write(TICK_BASE_CNT, mevt->base + MCT_L_TCNTB_OFFSET);
diff --git a/drivers/clocksource/hyperv_timer.c b/drivers/clocksource/hyperv_timer.c
new file mode 100644
index 000000000000..2317d4e3daaf
--- /dev/null
+++ b/drivers/clocksource/hyperv_timer.c
@@ -0,0 +1,326 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Clocksource driver for the synthetic counter and timers
+ * provided by the Hyper-V hypervisor to guest VMs, as described
+ * in the Hyper-V Top Level Functional Spec (TLFS). This driver
+ * is instruction set architecture independent.
+ *
+ * Copyright (C) 2019, Microsoft, Inc.
+ *
+ * Author: Michael Kelley <mikelley@microsoft.com>
+ */
+
+#include <linux/percpu.h>
+#include <linux/cpumask.h>
+#include <linux/clockchips.h>
+#include <linux/clocksource.h>
+#include <linux/sched_clock.h>
+#include <linux/mm.h>
+#include <clocksource/hyperv_timer.h>
+#include <asm/hyperv-tlfs.h>
+#include <asm/mshyperv.h>
+
+static struct clock_event_device __percpu *hv_clock_event;
+static u64 hv_sched_clock_offset __ro_after_init;
+
+/*
+ * If false, we're using the old mechanism for stimer0 interrupts
+ * where it sends a VMbus message when it expires. The old
+ * mechanism is used when running on older versions of Hyper-V
+ * that don't support Direct Mode. While Hyper-V provides
+ * four stimer's per CPU, Linux uses only stimer0.
+ */
+static bool direct_mode_enabled;
+
+static int stimer0_irq;
+static int stimer0_vector;
+static int stimer0_message_sint;
+
+/*
+ * ISR for when stimer0 is operating in Direct Mode. Direct Mode
+ * does not use VMbus or any VMbus messages, so process here and not
+ * in the VMbus driver code.
+ */
+void hv_stimer0_isr(void)
+{
+ struct clock_event_device *ce;
+
+ ce = this_cpu_ptr(hv_clock_event);
+ ce->event_handler(ce);
+}
+EXPORT_SYMBOL_GPL(hv_stimer0_isr);
+
+static int hv_ce_set_next_event(unsigned long delta,
+ struct clock_event_device *evt)
+{
+ u64 current_tick;
+
+ current_tick = hyperv_cs->read(NULL);
+ current_tick += delta;
+ hv_init_timer(0, current_tick);
+ return 0;
+}
+
+static int hv_ce_shutdown(struct clock_event_device *evt)
+{
+ hv_init_timer(0, 0);
+ hv_init_timer_config(0, 0);
+ if (direct_mode_enabled)
+ hv_disable_stimer0_percpu_irq(stimer0_irq);
+
+ return 0;
+}
+
+static int hv_ce_set_oneshot(struct clock_event_device *evt)
+{
+ union hv_stimer_config timer_cfg;
+
+ timer_cfg.as_uint64 = 0;
+ timer_cfg.enable = 1;
+ timer_cfg.auto_enable = 1;
+ if (direct_mode_enabled) {
+ /*
+ * When it expires, the timer will directly interrupt
+ * on the specified hardware vector/IRQ.
+ */
+ timer_cfg.direct_mode = 1;
+ timer_cfg.apic_vector = stimer0_vector;
+ hv_enable_stimer0_percpu_irq(stimer0_irq);
+ } else {
+ /*
+ * When it expires, the timer will generate a VMbus message,
+ * to be handled by the normal VMbus interrupt handler.
+ */
+ timer_cfg.direct_mode = 0;
+ timer_cfg.sintx = stimer0_message_sint;
+ }
+ hv_init_timer_config(0, timer_cfg.as_uint64);
+ return 0;
+}
+
+/*
+ * hv_stimer_init - Per-cpu initialization of the clockevent
+ */
+void hv_stimer_init(unsigned int cpu)
+{
+ struct clock_event_device *ce;
+
+ /*
+ * Synthetic timers are always available except on old versions of
+ * Hyper-V on x86. In that case, just return as Linux will use a
+ * clocksource based on emulated PIT or LAPIC timer hardware.
+ */
+ if (!(ms_hyperv.features & HV_MSR_SYNTIMER_AVAILABLE))
+ return;
+
+ ce = per_cpu_ptr(hv_clock_event, cpu);
+ ce->name = "Hyper-V clockevent";
+ ce->features = CLOCK_EVT_FEAT_ONESHOT;
+ ce->cpumask = cpumask_of(cpu);
+ ce->rating = 1000;
+ ce->set_state_shutdown = hv_ce_shutdown;
+ ce->set_state_oneshot = hv_ce_set_oneshot;
+ ce->set_next_event = hv_ce_set_next_event;
+
+ clockevents_config_and_register(ce,
+ HV_CLOCK_HZ,
+ HV_MIN_DELTA_TICKS,
+ HV_MAX_MAX_DELTA_TICKS);
+}
+EXPORT_SYMBOL_GPL(hv_stimer_init);
+
+/*
+ * hv_stimer_cleanup - Per-cpu cleanup of the clockevent
+ */
+void hv_stimer_cleanup(unsigned int cpu)
+{
+ struct clock_event_device *ce;
+
+ /* Turn off clockevent device */
+ if (ms_hyperv.features & HV_MSR_SYNTIMER_AVAILABLE) {
+ ce = per_cpu_ptr(hv_clock_event, cpu);
+ hv_ce_shutdown(ce);
+ }
+}
+EXPORT_SYMBOL_GPL(hv_stimer_cleanup);
+
+/* hv_stimer_alloc - Global initialization of the clockevent and stimer0 */
+int hv_stimer_alloc(int sint)
+{
+ int ret;
+
+ hv_clock_event = alloc_percpu(struct clock_event_device);
+ if (!hv_clock_event)
+ return -ENOMEM;
+
+ direct_mode_enabled = ms_hyperv.misc_features &
+ HV_STIMER_DIRECT_MODE_AVAILABLE;
+ if (direct_mode_enabled) {
+ ret = hv_setup_stimer0_irq(&stimer0_irq, &stimer0_vector,
+ hv_stimer0_isr);
+ if (ret) {
+ free_percpu(hv_clock_event);
+ hv_clock_event = NULL;
+ return ret;
+ }
+ }
+
+ stimer0_message_sint = sint;
+ return 0;
+}
+EXPORT_SYMBOL_GPL(hv_stimer_alloc);
+
+/* hv_stimer_free - Free global resources allocated by hv_stimer_alloc() */
+void hv_stimer_free(void)
+{
+ if (direct_mode_enabled && (stimer0_irq != 0)) {
+ hv_remove_stimer0_irq(stimer0_irq);
+ stimer0_irq = 0;
+ }
+ free_percpu(hv_clock_event);
+ hv_clock_event = NULL;
+}
+EXPORT_SYMBOL_GPL(hv_stimer_free);
+
+/*
+ * Do a global cleanup of clockevents for the cases of kexec and
+ * vmbus exit
+ */
+void hv_stimer_global_cleanup(void)
+{
+ int cpu;
+ struct clock_event_device *ce;
+
+ if (ms_hyperv.features & HV_MSR_SYNTIMER_AVAILABLE) {
+ for_each_present_cpu(cpu) {
+ ce = per_cpu_ptr(hv_clock_event, cpu);
+ clockevents_unbind_device(ce, cpu);
+ }
+ }
+ hv_stimer_free();
+}
+EXPORT_SYMBOL_GPL(hv_stimer_global_cleanup);
+
+/*
+ * Code and definitions for the Hyper-V clocksources. Two
+ * clocksources are defined: one that reads the Hyper-V defined MSR, and
+ * the other that uses the TSC reference page feature as defined in the
+ * TLFS. The MSR version is for compatibility with old versions of
+ * Hyper-V and 32-bit x86. The TSC reference page version is preferred.
+ */
+
+struct clocksource *hyperv_cs;
+EXPORT_SYMBOL_GPL(hyperv_cs);
+
+static struct ms_hyperv_tsc_page tsc_pg __aligned(PAGE_SIZE);
+
+struct ms_hyperv_tsc_page *hv_get_tsc_page(void)
+{
+ return &tsc_pg;
+}
+EXPORT_SYMBOL_GPL(hv_get_tsc_page);
+
+static u64 notrace read_hv_clock_tsc(struct clocksource *arg)
+{
+ u64 current_tick = hv_read_tsc_page(&tsc_pg);
+
+ if (current_tick == U64_MAX)
+ hv_get_time_ref_count(current_tick);
+
+ return current_tick;
+}
+
+static u64 read_hv_sched_clock_tsc(void)
+{
+ return read_hv_clock_tsc(NULL) - hv_sched_clock_offset;
+}
+
+static struct clocksource hyperv_cs_tsc = {
+ .name = "hyperv_clocksource_tsc_page",
+ .rating = 400,
+ .read = read_hv_clock_tsc,
+ .mask = CLOCKSOURCE_MASK(64),
+ .flags = CLOCK_SOURCE_IS_CONTINUOUS,
+};
+
+static u64 notrace read_hv_clock_msr(struct clocksource *arg)
+{
+ u64 current_tick;
+ /*
+ * Read the partition counter to get the current tick count. This count
+ * is set to 0 when the partition is created and is incremented in
+ * 100 nanosecond units.
+ */
+ hv_get_time_ref_count(current_tick);
+ return current_tick;
+}
+
+static u64 read_hv_sched_clock_msr(void)
+{
+ return read_hv_clock_msr(NULL) - hv_sched_clock_offset;
+}
+
+static struct clocksource hyperv_cs_msr = {
+ .name = "hyperv_clocksource_msr",
+ .rating = 400,
+ .read = read_hv_clock_msr,
+ .mask = CLOCKSOURCE_MASK(64),
+ .flags = CLOCK_SOURCE_IS_CONTINUOUS,
+};
+
+static bool __init hv_init_tsc_clocksource(void)
+{
+ u64 tsc_msr;
+ phys_addr_t phys_addr;
+
+ if (!(ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE))
+ return false;
+
+ hyperv_cs = &hyperv_cs_tsc;
+ phys_addr = virt_to_phys(&tsc_pg);
+
+ /*
+ * The Hyper-V TLFS specifies to preserve the value of reserved
+ * bits in registers. So read the existing value, preserve the
+ * low order 12 bits, and add in the guest physical address
+ * (which already has at least the low 12 bits set to zero since
+ * it is page aligned). Also set the "enable" bit, which is bit 0.
+ */
+ hv_get_reference_tsc(tsc_msr);
+ tsc_msr &= GENMASK_ULL(11, 0);
+ tsc_msr = tsc_msr | 0x1 | (u64)phys_addr;
+ hv_set_reference_tsc(tsc_msr);
+
+ hv_set_clocksource_vdso(hyperv_cs_tsc);
+ clocksource_register_hz(&hyperv_cs_tsc, NSEC_PER_SEC/100);
+
+ hv_sched_clock_offset = hyperv_cs->read(hyperv_cs);
+ hv_setup_sched_clock(read_hv_sched_clock_tsc);
+
+ return true;
+}
+
+void __init hv_init_clocksource(void)
+{
+ /*
+ * Try to set up the TSC page clocksource. If it succeeds, we're
+ * done. Otherwise, set up the MSR clocksoruce. At least one of
+ * these will always be available except on very old versions of
+ * Hyper-V on x86. In that case we won't have a Hyper-V
+ * clocksource, but Linux will still run with a clocksource based
+ * on the emulated PIT or LAPIC timer.
+ */
+ if (hv_init_tsc_clocksource())
+ return;
+
+ if (!(ms_hyperv.features & HV_MSR_TIME_REF_COUNT_AVAILABLE))
+ return;
+
+ hyperv_cs = &hyperv_cs_msr;
+ clocksource_register_hz(&hyperv_cs_msr, NSEC_PER_SEC/100);
+
+ hv_sched_clock_offset = hyperv_cs->read(hyperv_cs);
+ hv_setup_sched_clock(read_hv_sched_clock_msr);
+}
+EXPORT_SYMBOL_GPL(hv_init_clocksource);
diff --git a/drivers/clocksource/renesas-ostm.c b/drivers/clocksource/renesas-ostm.c
index 61d5f3b539ce..37c39b901bb1 100644
--- a/drivers/clocksource/renesas-ostm.c
+++ b/drivers/clocksource/renesas-ostm.c
@@ -221,7 +221,7 @@ static int __init ostm_init(struct device_node *np)
}
rate = clk_get_rate(ostm_clk);
- ostm->ticks_per_jiffy = (rate + HZ / 2) / HZ;
+ ostm->ticks_per_jiffy = DIV_ROUND_CLOSEST(rate, HZ);
/*
* First probed device will be used as system clocksource. Any
diff --git a/drivers/clocksource/sh_cmt.c b/drivers/clocksource/sh_cmt.c
index 55d3e03f2cd4..ef773db080e9 100644
--- a/drivers/clocksource/sh_cmt.c
+++ b/drivers/clocksource/sh_cmt.c
@@ -776,11 +776,8 @@ static int sh_cmt_register_clockevent(struct sh_cmt_channel *ch,
int ret;
irq = platform_get_irq(ch->cmt->pdev, ch->index);
- if (irq < 0) {
- dev_err(&ch->cmt->pdev->dev, "ch%u: failed to get irq\n",
- ch->index);
+ if (irq < 0)
return irq;
- }
ret = request_irq(irq, sh_cmt_interrupt,
IRQF_TIMER | IRQF_IRQPOLL | IRQF_NOBALANCING,
@@ -921,13 +918,25 @@ static const struct platform_device_id sh_cmt_id_table[] = {
MODULE_DEVICE_TABLE(platform, sh_cmt_id_table);
static const struct of_device_id sh_cmt_of_table[] __maybe_unused = {
- { .compatible = "renesas,cmt-48", .data = &sh_cmt_info[SH_CMT_48BIT] },
+ {
+ /* deprecated, preserved for backward compatibility */
+ .compatible = "renesas,cmt-48",
+ .data = &sh_cmt_info[SH_CMT_48BIT]
+ },
{
/* deprecated, preserved for backward compatibility */
.compatible = "renesas,cmt-48-gen2",
.data = &sh_cmt_info[SH_CMT0_RCAR_GEN2]
},
{
+ .compatible = "renesas,r8a7740-cmt1",
+ .data = &sh_cmt_info[SH_CMT_48BIT]
+ },
+ {
+ .compatible = "renesas,sh73a0-cmt1",
+ .data = &sh_cmt_info[SH_CMT_48BIT]
+ },
+ {
.compatible = "renesas,rcar-gen2-cmt0",
.data = &sh_cmt_info[SH_CMT0_RCAR_GEN2]
},
diff --git a/drivers/clocksource/sh_tmu.c b/drivers/clocksource/sh_tmu.c
index 49f1c805fc95..8c4f3753b36e 100644
--- a/drivers/clocksource/sh_tmu.c
+++ b/drivers/clocksource/sh_tmu.c
@@ -462,11 +462,8 @@ static int sh_tmu_channel_setup(struct sh_tmu_channel *ch, unsigned int index,
ch->base = tmu->mapbase + 8 + ch->index * 12;
ch->irq = platform_get_irq(tmu->pdev, index);
- if (ch->irq < 0) {
- dev_err(&tmu->pdev->dev, "ch%u: failed to get irq\n",
- ch->index);
+ if (ch->irq < 0)
return ch->irq;
- }
ch->cs_enabled = false;
ch->enable_count = 0;
diff --git a/drivers/clocksource/timer-atmel-tcb.c b/drivers/clocksource/timer-atmel-tcb.c
index 6ed31f9def7e..7427b07495a8 100644
--- a/drivers/clocksource/timer-atmel-tcb.c
+++ b/drivers/clocksource/timer-atmel-tcb.c
@@ -6,6 +6,7 @@
#include <linux/irq.h>
#include <linux/clk.h>
+#include <linux/delay.h>
#include <linux/err.h>
#include <linux/ioport.h>
#include <linux/io.h>
@@ -125,6 +126,18 @@ static u64 notrace tc_sched_clock_read32(void)
return tc_get_cycles32(&clksrc);
}
+static struct delay_timer tc_delay_timer;
+
+static unsigned long tc_delay_timer_read(void)
+{
+ return tc_get_cycles(&clksrc);
+}
+
+static unsigned long notrace tc_delay_timer_read32(void)
+{
+ return tc_get_cycles32(&clksrc);
+}
+
#ifdef CONFIG_GENERIC_CLOCKEVENTS
struct tc_clkevt_device {
@@ -432,6 +445,7 @@ static int __init tcb_clksrc_init(struct device_node *node)
/* setup ony channel 0 */
tcb_setup_single_chan(&tc, best_divisor_idx);
tc_sched_clock = tc_sched_clock_read32;
+ tc_delay_timer.read_current_timer = tc_delay_timer_read32;
} else {
/* we have three clocks no matter what the
* underlying platform supports.
@@ -444,6 +458,7 @@ static int __init tcb_clksrc_init(struct device_node *node)
/* setup both channel 0 & 1 */
tcb_setup_dual_chan(&tc, best_divisor_idx);
tc_sched_clock = tc_sched_clock_read;
+ tc_delay_timer.read_current_timer = tc_delay_timer_read;
}
/* and away we go! */
@@ -458,6 +473,9 @@ static int __init tcb_clksrc_init(struct device_node *node)
sched_clock_register(tc_sched_clock, 32, divided_rate);
+ tc_delay_timer.freq = divided_rate;
+ register_current_timer_delay(&tc_delay_timer);
+
return 0;
err_unregister_clksrc:
diff --git a/drivers/clocksource/timer-davinci.c b/drivers/clocksource/timer-davinci.c
new file mode 100644
index 000000000000..62745c962049
--- /dev/null
+++ b/drivers/clocksource/timer-davinci.c
@@ -0,0 +1,369 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * TI DaVinci clocksource driver
+ *
+ * Copyright (C) 2019 Texas Instruments
+ * Author: Bartosz Golaszewski <bgolaszewski@baylibre.com>
+ * (with tiny parts adopted from code by Kevin Hilman <khilman@baylibre.com>)
+ */
+
+#include <linux/clk.h>
+#include <linux/clockchips.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/sched_clock.h>
+
+#include <clocksource/timer-davinci.h>
+
+#undef pr_fmt
+#define pr_fmt(fmt) "%s: " fmt "\n", __func__
+
+#define DAVINCI_TIMER_REG_TIM12 0x10
+#define DAVINCI_TIMER_REG_TIM34 0x14
+#define DAVINCI_TIMER_REG_PRD12 0x18
+#define DAVINCI_TIMER_REG_PRD34 0x1c
+#define DAVINCI_TIMER_REG_TCR 0x20
+#define DAVINCI_TIMER_REG_TGCR 0x24
+
+#define DAVINCI_TIMER_TIMMODE_MASK GENMASK(3, 2)
+#define DAVINCI_TIMER_RESET_MASK GENMASK(1, 0)
+#define DAVINCI_TIMER_TIMMODE_32BIT_UNCHAINED BIT(2)
+#define DAVINCI_TIMER_UNRESET GENMASK(1, 0)
+
+#define DAVINCI_TIMER_ENAMODE_MASK GENMASK(1, 0)
+#define DAVINCI_TIMER_ENAMODE_DISABLED 0x00
+#define DAVINCI_TIMER_ENAMODE_ONESHOT BIT(0)
+#define DAVINCI_TIMER_ENAMODE_PERIODIC BIT(1)
+
+#define DAVINCI_TIMER_ENAMODE_SHIFT_TIM12 6
+#define DAVINCI_TIMER_ENAMODE_SHIFT_TIM34 22
+
+#define DAVINCI_TIMER_MIN_DELTA 0x01
+#define DAVINCI_TIMER_MAX_DELTA 0xfffffffe
+
+#define DAVINCI_TIMER_CLKSRC_BITS 32
+
+#define DAVINCI_TIMER_TGCR_DEFAULT \
+ (DAVINCI_TIMER_TIMMODE_32BIT_UNCHAINED | DAVINCI_TIMER_UNRESET)
+
+struct davinci_clockevent {
+ struct clock_event_device dev;
+ void __iomem *base;
+ unsigned int cmp_off;
+};
+
+/*
+ * This must be globally accessible by davinci_timer_read_sched_clock(), so
+ * let's keep it here.
+ */
+static struct {
+ struct clocksource dev;
+ void __iomem *base;
+ unsigned int tim_off;
+} davinci_clocksource;
+
+static struct davinci_clockevent *
+to_davinci_clockevent(struct clock_event_device *clockevent)
+{
+ return container_of(clockevent, struct davinci_clockevent, dev);
+}
+
+static unsigned int
+davinci_clockevent_read(struct davinci_clockevent *clockevent,
+ unsigned int reg)
+{
+ return readl_relaxed(clockevent->base + reg);
+}
+
+static void davinci_clockevent_write(struct davinci_clockevent *clockevent,
+ unsigned int reg, unsigned int val)
+{
+ writel_relaxed(val, clockevent->base + reg);
+}
+
+static void davinci_tim12_shutdown(void __iomem *base)
+{
+ unsigned int tcr;
+
+ tcr = DAVINCI_TIMER_ENAMODE_DISABLED <<
+ DAVINCI_TIMER_ENAMODE_SHIFT_TIM12;
+ /*
+ * This function is only ever called if we're using both timer
+ * halves. In this case TIM34 runs in periodic mode and we must
+ * not modify it.
+ */
+ tcr |= DAVINCI_TIMER_ENAMODE_PERIODIC <<
+ DAVINCI_TIMER_ENAMODE_SHIFT_TIM34;
+
+ writel_relaxed(tcr, base + DAVINCI_TIMER_REG_TCR);
+}
+
+static void davinci_tim12_set_oneshot(void __iomem *base)
+{
+ unsigned int tcr;
+
+ tcr = DAVINCI_TIMER_ENAMODE_ONESHOT <<
+ DAVINCI_TIMER_ENAMODE_SHIFT_TIM12;
+ /* Same as above. */
+ tcr |= DAVINCI_TIMER_ENAMODE_PERIODIC <<
+ DAVINCI_TIMER_ENAMODE_SHIFT_TIM34;
+
+ writel_relaxed(tcr, base + DAVINCI_TIMER_REG_TCR);
+}
+
+static int davinci_clockevent_shutdown(struct clock_event_device *dev)
+{
+ struct davinci_clockevent *clockevent;
+
+ clockevent = to_davinci_clockevent(dev);
+
+ davinci_tim12_shutdown(clockevent->base);
+
+ return 0;
+}
+
+static int davinci_clockevent_set_oneshot(struct clock_event_device *dev)
+{
+ struct davinci_clockevent *clockevent = to_davinci_clockevent(dev);
+
+ davinci_clockevent_write(clockevent, DAVINCI_TIMER_REG_TIM12, 0x0);
+
+ davinci_tim12_set_oneshot(clockevent->base);
+
+ return 0;
+}
+
+static int
+davinci_clockevent_set_next_event_std(unsigned long cycles,
+ struct clock_event_device *dev)
+{
+ struct davinci_clockevent *clockevent = to_davinci_clockevent(dev);
+
+ davinci_clockevent_shutdown(dev);
+
+ davinci_clockevent_write(clockevent, DAVINCI_TIMER_REG_TIM12, 0x0);
+ davinci_clockevent_write(clockevent, DAVINCI_TIMER_REG_PRD12, cycles);
+
+ davinci_clockevent_set_oneshot(dev);
+
+ return 0;
+}
+
+static int
+davinci_clockevent_set_next_event_cmp(unsigned long cycles,
+ struct clock_event_device *dev)
+{
+ struct davinci_clockevent *clockevent = to_davinci_clockevent(dev);
+ unsigned int curr_time;
+
+ curr_time = davinci_clockevent_read(clockevent,
+ DAVINCI_TIMER_REG_TIM12);
+ davinci_clockevent_write(clockevent,
+ clockevent->cmp_off, curr_time + cycles);
+
+ return 0;
+}
+
+static irqreturn_t davinci_timer_irq_timer(int irq, void *data)
+{
+ struct davinci_clockevent *clockevent = data;
+
+ if (!clockevent_state_oneshot(&clockevent->dev))
+ davinci_tim12_shutdown(clockevent->base);
+
+ clockevent->dev.event_handler(&clockevent->dev);
+
+ return IRQ_HANDLED;
+}
+
+static u64 notrace davinci_timer_read_sched_clock(void)
+{
+ return readl_relaxed(davinci_clocksource.base +
+ davinci_clocksource.tim_off);
+}
+
+static u64 davinci_clocksource_read(struct clocksource *dev)
+{
+ return davinci_timer_read_sched_clock();
+}
+
+/*
+ * Standard use-case: we're using tim12 for clockevent and tim34 for
+ * clocksource. The default is making the former run in oneshot mode
+ * and the latter in periodic mode.
+ */
+static void davinci_clocksource_init_tim34(void __iomem *base)
+{
+ int tcr;
+
+ tcr = DAVINCI_TIMER_ENAMODE_PERIODIC <<
+ DAVINCI_TIMER_ENAMODE_SHIFT_TIM34;
+ tcr |= DAVINCI_TIMER_ENAMODE_ONESHOT <<
+ DAVINCI_TIMER_ENAMODE_SHIFT_TIM12;
+
+ writel_relaxed(0x0, base + DAVINCI_TIMER_REG_TIM34);
+ writel_relaxed(UINT_MAX, base + DAVINCI_TIMER_REG_PRD34);
+ writel_relaxed(tcr, base + DAVINCI_TIMER_REG_TCR);
+}
+
+/*
+ * Special use-case on da830: the DSP may use tim34. We're using tim12 for
+ * both clocksource and clockevent. We set tim12 to periodic and don't touch
+ * tim34.
+ */
+static void davinci_clocksource_init_tim12(void __iomem *base)
+{
+ unsigned int tcr;
+
+ tcr = DAVINCI_TIMER_ENAMODE_PERIODIC <<
+ DAVINCI_TIMER_ENAMODE_SHIFT_TIM12;
+
+ writel_relaxed(0x0, base + DAVINCI_TIMER_REG_TIM12);
+ writel_relaxed(UINT_MAX, base + DAVINCI_TIMER_REG_PRD12);
+ writel_relaxed(tcr, base + DAVINCI_TIMER_REG_TCR);
+}
+
+static void davinci_timer_init(void __iomem *base)
+{
+ /* Set clock to internal mode and disable it. */
+ writel_relaxed(0x0, base + DAVINCI_TIMER_REG_TCR);
+ /*
+ * Reset both 32-bit timers, set no prescaler for timer 34, set the
+ * timer to dual 32-bit unchained mode, unreset both 32-bit timers.
+ */
+ writel_relaxed(DAVINCI_TIMER_TGCR_DEFAULT,
+ base + DAVINCI_TIMER_REG_TGCR);
+ /* Init both counters to zero. */
+ writel_relaxed(0x0, base + DAVINCI_TIMER_REG_TIM12);
+ writel_relaxed(0x0, base + DAVINCI_TIMER_REG_TIM34);
+}
+
+int __init davinci_timer_register(struct clk *clk,
+ const struct davinci_timer_cfg *timer_cfg)
+{
+ struct davinci_clockevent *clockevent;
+ unsigned int tick_rate;
+ void __iomem *base;
+ int rv;
+
+ rv = clk_prepare_enable(clk);
+ if (rv) {
+ pr_err("Unable to prepare and enable the timer clock");
+ return rv;
+ }
+
+ if (!request_mem_region(timer_cfg->reg.start,
+ resource_size(&timer_cfg->reg),
+ "davinci-timer")) {
+ pr_err("Unable to request memory region");
+ return -EBUSY;
+ }
+
+ base = ioremap(timer_cfg->reg.start, resource_size(&timer_cfg->reg));
+ if (!base) {
+ pr_err("Unable to map the register range");
+ return -ENOMEM;
+ }
+
+ davinci_timer_init(base);
+ tick_rate = clk_get_rate(clk);
+
+ clockevent = kzalloc(sizeof(*clockevent), GFP_KERNEL | __GFP_NOFAIL);
+ if (!clockevent) {
+ pr_err("Error allocating memory for clockevent data");
+ return -ENOMEM;
+ }
+
+ clockevent->dev.name = "tim12";
+ clockevent->dev.features = CLOCK_EVT_FEAT_ONESHOT;
+ clockevent->dev.cpumask = cpumask_of(0);
+ clockevent->base = base;
+
+ if (timer_cfg->cmp_off) {
+ clockevent->cmp_off = timer_cfg->cmp_off;
+ clockevent->dev.set_next_event =
+ davinci_clockevent_set_next_event_cmp;
+ } else {
+ clockevent->dev.set_next_event =
+ davinci_clockevent_set_next_event_std;
+ clockevent->dev.set_state_oneshot =
+ davinci_clockevent_set_oneshot;
+ clockevent->dev.set_state_shutdown =
+ davinci_clockevent_shutdown;
+ }
+
+ rv = request_irq(timer_cfg->irq[DAVINCI_TIMER_CLOCKEVENT_IRQ].start,
+ davinci_timer_irq_timer, IRQF_TIMER,
+ "clockevent/tim12", clockevent);
+ if (rv) {
+ pr_err("Unable to request the clockevent interrupt");
+ return rv;
+ }
+
+ clockevents_config_and_register(&clockevent->dev, tick_rate,
+ DAVINCI_TIMER_MIN_DELTA,
+ DAVINCI_TIMER_MAX_DELTA);
+
+ davinci_clocksource.dev.rating = 300;
+ davinci_clocksource.dev.read = davinci_clocksource_read;
+ davinci_clocksource.dev.mask =
+ CLOCKSOURCE_MASK(DAVINCI_TIMER_CLKSRC_BITS);
+ davinci_clocksource.dev.flags = CLOCK_SOURCE_IS_CONTINUOUS;
+ davinci_clocksource.base = base;
+
+ if (timer_cfg->cmp_off) {
+ davinci_clocksource.dev.name = "tim12";
+ davinci_clocksource.tim_off = DAVINCI_TIMER_REG_TIM12;
+ davinci_clocksource_init_tim12(base);
+ } else {
+ davinci_clocksource.dev.name = "tim34";
+ davinci_clocksource.tim_off = DAVINCI_TIMER_REG_TIM34;
+ davinci_clocksource_init_tim34(base);
+ }
+
+ rv = clocksource_register_hz(&davinci_clocksource.dev, tick_rate);
+ if (rv) {
+ pr_err("Unable to register clocksource");
+ return rv;
+ }
+
+ sched_clock_register(davinci_timer_read_sched_clock,
+ DAVINCI_TIMER_CLKSRC_BITS, tick_rate);
+
+ return 0;
+}
+
+static int __init of_davinci_timer_register(struct device_node *np)
+{
+ struct davinci_timer_cfg timer_cfg = { };
+ struct clk *clk;
+ int rv;
+
+ rv = of_address_to_resource(np, 0, &timer_cfg.reg);
+ if (rv) {
+ pr_err("Unable to get the register range for timer");
+ return rv;
+ }
+
+ rv = of_irq_to_resource_table(np, timer_cfg.irq,
+ DAVINCI_TIMER_NUM_IRQS);
+ if (rv != DAVINCI_TIMER_NUM_IRQS) {
+ pr_err("Unable to get the interrupts for timer");
+ return rv;
+ }
+
+ clk = of_clk_get(np, 0);
+ if (IS_ERR(clk)) {
+ pr_err("Unable to get the timer clock");
+ return PTR_ERR(clk);
+ }
+
+ rv = davinci_timer_register(clk, &timer_cfg);
+ if (rv)
+ clk_put(clk);
+
+ return rv;
+}
+TIMER_OF_DECLARE(davinci_timer, "ti,da830-timer", of_davinci_timer_register);
diff --git a/drivers/clocksource/timer-imx-sysctr.c b/drivers/clocksource/timer-imx-sysctr.c
new file mode 100644
index 000000000000..b7c80a368a1b
--- /dev/null
+++ b/drivers/clocksource/timer-imx-sysctr.c
@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2017-2019 NXP
+
+#include <linux/interrupt.h>
+#include <linux/clockchips.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+
+#include "timer-of.h"
+
+#define CMP_OFFSET 0x10000
+
+#define CNTCV_LO 0x8
+#define CNTCV_HI 0xc
+#define CMPCV_LO (CMP_OFFSET + 0x20)
+#define CMPCV_HI (CMP_OFFSET + 0x24)
+#define CMPCR (CMP_OFFSET + 0x2c)
+
+#define SYS_CTR_EN 0x1
+#define SYS_CTR_IRQ_MASK 0x2
+
+#define SYS_CTR_CLK_DIV 0x3
+
+static void __iomem *sys_ctr_base;
+static u32 cmpcr;
+
+static void sysctr_timer_enable(bool enable)
+{
+ writel(enable ? cmpcr | SYS_CTR_EN : cmpcr, sys_ctr_base + CMPCR);
+}
+
+static void sysctr_irq_acknowledge(void)
+{
+ /*
+ * clear the enable bit(EN =0) will clear
+ * the status bit(ISTAT = 0), then the interrupt
+ * signal will be negated(acknowledged).
+ */
+ sysctr_timer_enable(false);
+}
+
+static inline u64 sysctr_read_counter(void)
+{
+ u32 cnt_hi, tmp_hi, cnt_lo;
+
+ do {
+ cnt_hi = readl_relaxed(sys_ctr_base + CNTCV_HI);
+ cnt_lo = readl_relaxed(sys_ctr_base + CNTCV_LO);
+ tmp_hi = readl_relaxed(sys_ctr_base + CNTCV_HI);
+ } while (tmp_hi != cnt_hi);
+
+ return ((u64) cnt_hi << 32) | cnt_lo;
+}
+
+static int sysctr_set_next_event(unsigned long delta,
+ struct clock_event_device *evt)
+{
+ u32 cmp_hi, cmp_lo;
+ u64 next;
+
+ sysctr_timer_enable(false);
+
+ next = sysctr_read_counter();
+
+ next += delta;
+
+ cmp_hi = (next >> 32) & 0x00fffff;
+ cmp_lo = next & 0xffffffff;
+
+ writel_relaxed(cmp_hi, sys_ctr_base + CMPCV_HI);
+ writel_relaxed(cmp_lo, sys_ctr_base + CMPCV_LO);
+
+ sysctr_timer_enable(true);
+
+ return 0;
+}
+
+static int sysctr_set_state_oneshot(struct clock_event_device *evt)
+{
+ return 0;
+}
+
+static int sysctr_set_state_shutdown(struct clock_event_device *evt)
+{
+ sysctr_timer_enable(false);
+
+ return 0;
+}
+
+static irqreturn_t sysctr_timer_interrupt(int irq, void *dev_id)
+{
+ struct clock_event_device *evt = dev_id;
+
+ sysctr_irq_acknowledge();
+
+ evt->event_handler(evt);
+
+ return IRQ_HANDLED;
+}
+
+static struct timer_of to_sysctr = {
+ .flags = TIMER_OF_IRQ | TIMER_OF_CLOCK | TIMER_OF_BASE,
+ .clkevt = {
+ .name = "i.MX system counter timer",
+ .features = CLOCK_EVT_FEAT_ONESHOT |
+ CLOCK_EVT_FEAT_DYNIRQ,
+ .set_state_oneshot = sysctr_set_state_oneshot,
+ .set_next_event = sysctr_set_next_event,
+ .set_state_shutdown = sysctr_set_state_shutdown,
+ .rating = 200,
+ },
+ .of_irq = {
+ .handler = sysctr_timer_interrupt,
+ .flags = IRQF_TIMER | IRQF_IRQPOLL,
+ },
+ .of_clk = {
+ .name = "per",
+ },
+};
+
+static void __init sysctr_clockevent_init(void)
+{
+ to_sysctr.clkevt.cpumask = cpumask_of(0);
+
+ clockevents_config_and_register(&to_sysctr.clkevt,
+ timer_of_rate(&to_sysctr),
+ 0xff, 0x7fffffff);
+}
+
+static int __init sysctr_timer_init(struct device_node *np)
+{
+ int ret = 0;
+
+ ret = timer_of_init(np, &to_sysctr);
+ if (ret)
+ return ret;
+
+ /* system counter clock is divided by 3 internally */
+ to_sysctr.of_clk.rate /= SYS_CTR_CLK_DIV;
+
+ sys_ctr_base = timer_of_base(&to_sysctr);
+ cmpcr = readl(sys_ctr_base + CMPCR);
+ cmpcr &= ~SYS_CTR_EN;
+
+ sysctr_clockevent_init();
+
+ return 0;
+}
+TIMER_OF_DECLARE(sysctr_timer, "nxp,sysctr-timer", sysctr_timer_init);
diff --git a/drivers/clocksource/timer-ixp4xx.c b/drivers/clocksource/timer-ixp4xx.c
index 5c2190b654cd..9396745e1c17 100644
--- a/drivers/clocksource/timer-ixp4xx.c
+++ b/drivers/clocksource/timer-ixp4xx.c
@@ -75,14 +75,19 @@ to_ixp4xx_timer(struct clock_event_device *evt)
return container_of(evt, struct ixp4xx_timer, clkevt);
}
-static u64 notrace ixp4xx_read_sched_clock(void)
+static unsigned long ixp4xx_read_timer(void)
{
return __raw_readl(local_ixp4xx_timer->base + IXP4XX_OSTS_OFFSET);
}
+static u64 notrace ixp4xx_read_sched_clock(void)
+{
+ return ixp4xx_read_timer();
+}
+
static u64 ixp4xx_clocksource_read(struct clocksource *c)
{
- return __raw_readl(local_ixp4xx_timer->base + IXP4XX_OSTS_OFFSET);
+ return ixp4xx_read_timer();
}
static irqreturn_t ixp4xx_timer_interrupt(int irq, void *dev_id)
@@ -224,6 +229,13 @@ static __init int ixp4xx_timer_register(void __iomem *base,
sched_clock_register(ixp4xx_read_sched_clock, 32, timer_freq);
+#ifdef CONFIG_ARM
+ /* Also use this timer for delays */
+ tmr->delay_timer.read_current_timer = ixp4xx_read_timer;
+ tmr->delay_timer.freq = timer_freq;
+ register_current_timer_delay(&tmr->delay_timer);
+#endif
+
return 0;
}
diff --git a/drivers/clocksource/timer-meson6.c b/drivers/clocksource/timer-meson6.c
index 84bd9479c3f8..9e8b467c71da 100644
--- a/drivers/clocksource/timer-meson6.c
+++ b/drivers/clocksource/timer-meson6.c
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* Amlogic Meson6 SoCs timer handling.
*
* Copyright (C) 2014 Carlo Caione <carlo@caione.org>
*
* Based on code from Amlogic, Inc
- *
- * This file is licensed under the terms of the GNU General Public
- * License version 2. This program is licensed "as is" without any
- * warranty of any kind, whether express or implied.
*/
#include <linux/bitfield.h>
diff --git a/drivers/clocksource/timer-npcm7xx.c b/drivers/clocksource/timer-npcm7xx.c
index 7a9bb5532d99..9780ffd8010e 100644
--- a/drivers/clocksource/timer-npcm7xx.c
+++ b/drivers/clocksource/timer-npcm7xx.c
@@ -32,7 +32,7 @@
#define NPCM7XX_Tx_INTEN BIT(29)
#define NPCM7XX_Tx_COUNTEN BIT(30)
#define NPCM7XX_Tx_ONESHOT 0x0
-#define NPCM7XX_Tx_OPER GENMASK(3, 27)
+#define NPCM7XX_Tx_OPER GENMASK(28, 27)
#define NPCM7XX_Tx_MIN_PRESCALE 0x1
#define NPCM7XX_Tx_TDR_MASK_BITS 24
#define NPCM7XX_Tx_MAX_CNT 0xFFFFFF
@@ -84,8 +84,6 @@ static int npcm7xx_timer_oneshot(struct clock_event_device *evt)
val = readl(timer_of_base(to) + NPCM7XX_REG_TCSR0);
val &= ~NPCM7XX_Tx_OPER;
-
- val = readl(timer_of_base(to) + NPCM7XX_REG_TCSR0);
val |= NPCM7XX_START_ONESHOT_Tx;
writel(val, timer_of_base(to) + NPCM7XX_REG_TCSR0);
@@ -97,12 +95,11 @@ static int npcm7xx_timer_periodic(struct clock_event_device *evt)
struct timer_of *to = to_timer_of(evt);
u32 val;
+ writel(timer_of_period(to), timer_of_base(to) + NPCM7XX_REG_TICR0);
+
val = readl(timer_of_base(to) + NPCM7XX_REG_TCSR0);
val &= ~NPCM7XX_Tx_OPER;
-
- writel(timer_of_period(to), timer_of_base(to) + NPCM7XX_REG_TICR0);
val |= NPCM7XX_START_PERIODIC_Tx;
-
writel(val, timer_of_base(to) + NPCM7XX_REG_TCSR0);
return 0;
diff --git a/drivers/clocksource/timer-of.c b/drivers/clocksource/timer-of.c
index 80542289fae7..d8c2bd4391d0 100644
--- a/drivers/clocksource/timer-of.c
+++ b/drivers/clocksource/timer-of.c
@@ -113,8 +113,10 @@ static __init int timer_of_clk_init(struct device_node *np,
of_clk->clk = of_clk->name ? of_clk_get_by_name(np, of_clk->name) :
of_clk_get(np, of_clk->index);
if (IS_ERR(of_clk->clk)) {
- pr_err("Failed to get clock for %pOF\n", np);
- return PTR_ERR(of_clk->clk);
+ ret = PTR_ERR(of_clk->clk);
+ if (ret != -EPROBE_DEFER)
+ pr_err("Failed to get clock for %pOF\n", np);
+ goto out;
}
ret = clk_prepare_enable(of_clk->clk);
diff --git a/drivers/clocksource/timer-probe.c b/drivers/clocksource/timer-probe.c
index dda1946e84dd..ee9574da53c0 100644
--- a/drivers/clocksource/timer-probe.c
+++ b/drivers/clocksource/timer-probe.c
@@ -29,7 +29,9 @@ void __init timer_probe(void)
ret = init_func_ret(np);
if (ret) {
- pr_err("Failed to initialize '%pOF': %d\n", np, ret);
+ if (ret != -EPROBE_DEFER)
+ pr_err("Failed to initialize '%pOF': %d\n", np,
+ ret);
continue;
}
diff --git a/drivers/clocksource/timer-riscv.c b/drivers/clocksource/timer-riscv.c
index 5e6038fbf115..470c7ef02ea4 100644
--- a/drivers/clocksource/timer-riscv.c
+++ b/drivers/clocksource/timer-riscv.c
@@ -2,6 +2,10 @@
/*
* Copyright (C) 2012 Regents of the University of California
* Copyright (C) 2017 SiFive
+ *
+ * All RISC-V systems have a timer attached to every hart. These timers can be
+ * read from the "time" and "timeh" CSRs, and can use the SBI to setup
+ * events.
*/
#include <linux/clocksource.h>
#include <linux/clockchips.h>
@@ -12,19 +16,6 @@
#include <asm/smp.h>
#include <asm/sbi.h>
-/*
- * All RISC-V systems have a timer attached to every hart. These timers can be
- * read by the 'rdcycle' pseudo instruction, and can use the SBI to setup
- * events. In order to abstract the architecture-specific timer reading and
- * setting functions away from the clock event insertion code, we provide
- * function pointers to the clockevent subsystem that perform two basic
- * operations: rdtime() reads the timer on the current CPU, and
- * next_event(delta) sets the next timer event to 'delta' cycles in the future.
- * As the timers are inherently a per-cpu resource, these callbacks perform
- * operations on the current hart. There is guaranteed to be exactly one timer
- * per hart on all RISC-V systems.
- */
-
static int riscv_clock_next_event(unsigned long delta,
struct clock_event_device *ce)
{
@@ -55,7 +46,7 @@ static u64 riscv_sched_clock(void)
return get_cycles64();
}
-static DEFINE_PER_CPU(struct clocksource, riscv_clocksource) = {
+static struct clocksource riscv_clocksource = {
.name = "riscv_clocksource",
.rating = 300,
.mask = CLOCKSOURCE_MASK(64),
@@ -92,7 +83,6 @@ void riscv_timer_interrupt(void)
static int __init riscv_timer_init_dt(struct device_node *n)
{
int cpuid, hartid, error;
- struct clocksource *cs;
hartid = riscv_of_processor_hartid(n);
if (hartid < 0) {
@@ -112,8 +102,7 @@ static int __init riscv_timer_init_dt(struct device_node *n)
pr_info("%s: Registering clocksource cpuid [%d] hartid [%d]\n",
__func__, cpuid, hartid);
- cs = per_cpu_ptr(&riscv_clocksource, cpuid);
- error = clocksource_register_hz(cs, riscv_timebase);
+ error = clocksource_register_hz(&riscv_clocksource, riscv_timebase);
if (error) {
pr_err("RISCV timer register failed [%d] for cpu = [%d]\n",
error, cpuid);
diff --git a/drivers/clocksource/timer-sun4i.c b/drivers/clocksource/timer-sun4i.c
index 65f38f6ca714..0ba8155b8287 100644
--- a/drivers/clocksource/timer-sun4i.c
+++ b/drivers/clocksource/timer-sun4i.c
@@ -219,5 +219,9 @@ static int __init sun4i_timer_init(struct device_node *node)
}
TIMER_OF_DECLARE(sun4i, "allwinner,sun4i-a10-timer",
sun4i_timer_init);
+TIMER_OF_DECLARE(sun8i_a23, "allwinner,sun8i-a23-timer",
+ sun4i_timer_init);
+TIMER_OF_DECLARE(sun8i_v3s, "allwinner,sun8i-v3s-timer",
+ sun4i_timer_init);
TIMER_OF_DECLARE(suniv, "allwinner,suniv-f1c100s-timer",
sun4i_timer_init);
diff --git a/drivers/clocksource/timer-tegra.c b/drivers/clocksource/timer-tegra.c
new file mode 100644
index 000000000000..e9635c25eef4
--- /dev/null
+++ b/drivers/clocksource/timer-tegra.c
@@ -0,0 +1,416 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2010 Google, Inc.
+ *
+ * Author:
+ * Colin Cross <ccross@google.com>
+ */
+
+#define pr_fmt(fmt) "tegra-timer: " fmt
+
+#include <linux/clk.h>
+#include <linux/clockchips.h>
+#include <linux/cpu.h>
+#include <linux/cpumask.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/percpu.h>
+#include <linux/sched_clock.h>
+#include <linux/time.h>
+
+#include "timer-of.h"
+
+#define RTC_SECONDS 0x08
+#define RTC_SHADOW_SECONDS 0x0c
+#define RTC_MILLISECONDS 0x10
+
+#define TIMERUS_CNTR_1US 0x10
+#define TIMERUS_USEC_CFG 0x14
+#define TIMERUS_CNTR_FREEZE 0x4c
+
+#define TIMER_PTV 0x0
+#define TIMER_PTV_EN BIT(31)
+#define TIMER_PTV_PER BIT(30)
+#define TIMER_PCR 0x4
+#define TIMER_PCR_INTR_CLR BIT(30)
+
+#define TIMER1_BASE 0x00
+#define TIMER2_BASE 0x08
+#define TIMER3_BASE 0x50
+#define TIMER4_BASE 0x58
+#define TIMER10_BASE 0x90
+
+#define TIMER1_IRQ_IDX 0
+#define TIMER10_IRQ_IDX 10
+
+#define TIMER_1MHz 1000000
+
+static u32 usec_config;
+static void __iomem *timer_reg_base;
+
+static int tegra_timer_set_next_event(unsigned long cycles,
+ struct clock_event_device *evt)
+{
+ void __iomem *reg_base = timer_of_base(to_timer_of(evt));
+
+ /*
+ * Tegra's timer uses n+1 scheme for the counter, i.e. timer will
+ * fire after one tick if 0 is loaded.
+ *
+ * The minimum and maximum numbers of oneshot ticks are defined
+ * by clockevents_config_and_register(1, 0x1fffffff + 1) invocation
+ * below in the code. Hence the cycles (ticks) can't be outside of
+ * a range supportable by hardware.
+ */
+ writel_relaxed(TIMER_PTV_EN | (cycles - 1), reg_base + TIMER_PTV);
+
+ return 0;
+}
+
+static int tegra_timer_shutdown(struct clock_event_device *evt)
+{
+ void __iomem *reg_base = timer_of_base(to_timer_of(evt));
+
+ writel_relaxed(0, reg_base + TIMER_PTV);
+
+ return 0;
+}
+
+static int tegra_timer_set_periodic(struct clock_event_device *evt)
+{
+ void __iomem *reg_base = timer_of_base(to_timer_of(evt));
+ unsigned long period = timer_of_period(to_timer_of(evt));
+
+ writel_relaxed(TIMER_PTV_EN | TIMER_PTV_PER | (period - 1),
+ reg_base + TIMER_PTV);
+
+ return 0;
+}
+
+static irqreturn_t tegra_timer_isr(int irq, void *dev_id)
+{
+ struct clock_event_device *evt = dev_id;
+ void __iomem *reg_base = timer_of_base(to_timer_of(evt));
+
+ writel_relaxed(TIMER_PCR_INTR_CLR, reg_base + TIMER_PCR);
+ evt->event_handler(evt);
+
+ return IRQ_HANDLED;
+}
+
+static void tegra_timer_suspend(struct clock_event_device *evt)
+{
+ void __iomem *reg_base = timer_of_base(to_timer_of(evt));
+
+ writel_relaxed(TIMER_PCR_INTR_CLR, reg_base + TIMER_PCR);
+}
+
+static void tegra_timer_resume(struct clock_event_device *evt)
+{
+ writel_relaxed(usec_config, timer_reg_base + TIMERUS_USEC_CFG);
+}
+
+static DEFINE_PER_CPU(struct timer_of, tegra_to) = {
+ .flags = TIMER_OF_CLOCK | TIMER_OF_BASE,
+
+ .clkevt = {
+ .name = "tegra_timer",
+ .features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PERIODIC,
+ .set_next_event = tegra_timer_set_next_event,
+ .set_state_shutdown = tegra_timer_shutdown,
+ .set_state_periodic = tegra_timer_set_periodic,
+ .set_state_oneshot = tegra_timer_shutdown,
+ .tick_resume = tegra_timer_shutdown,
+ .suspend = tegra_timer_suspend,
+ .resume = tegra_timer_resume,
+ },
+};
+
+static int tegra_timer_setup(unsigned int cpu)
+{
+ struct timer_of *to = per_cpu_ptr(&tegra_to, cpu);
+
+ writel_relaxed(0, timer_of_base(to) + TIMER_PTV);
+ writel_relaxed(TIMER_PCR_INTR_CLR, timer_of_base(to) + TIMER_PCR);
+
+ irq_force_affinity(to->clkevt.irq, cpumask_of(cpu));
+ enable_irq(to->clkevt.irq);
+
+ /*
+ * Tegra's timer uses n+1 scheme for the counter, i.e. timer will
+ * fire after one tick if 0 is loaded and thus minimum number of
+ * ticks is 1. In result both of the clocksource's tick limits are
+ * higher than a minimum and maximum that hardware register can
+ * take by 1, this is then taken into account by set_next_event
+ * callback.
+ */
+ clockevents_config_and_register(&to->clkevt, timer_of_rate(to),
+ 1, /* min */
+ 0x1fffffff + 1); /* max 29 bits + 1 */
+
+ return 0;
+}
+
+static int tegra_timer_stop(unsigned int cpu)
+{
+ struct timer_of *to = per_cpu_ptr(&tegra_to, cpu);
+
+ to->clkevt.set_state_shutdown(&to->clkevt);
+ disable_irq_nosync(to->clkevt.irq);
+
+ return 0;
+}
+
+static u64 notrace tegra_read_sched_clock(void)
+{
+ return readl_relaxed(timer_reg_base + TIMERUS_CNTR_1US);
+}
+
+#ifdef CONFIG_ARM
+static unsigned long tegra_delay_timer_read_counter_long(void)
+{
+ return readl_relaxed(timer_reg_base + TIMERUS_CNTR_1US);
+}
+
+static struct delay_timer tegra_delay_timer = {
+ .read_current_timer = tegra_delay_timer_read_counter_long,
+ .freq = TIMER_1MHz,
+};
+#endif
+
+static struct timer_of suspend_rtc_to = {
+ .flags = TIMER_OF_BASE | TIMER_OF_CLOCK,
+};
+
+/*
+ * tegra_rtc_read - Reads the Tegra RTC registers
+ * Care must be taken that this function is not called while the
+ * tegra_rtc driver could be executing to avoid race conditions
+ * on the RTC shadow register
+ */
+static u64 tegra_rtc_read_ms(struct clocksource *cs)
+{
+ void __iomem *reg_base = timer_of_base(&suspend_rtc_to);
+
+ u32 ms = readl_relaxed(reg_base + RTC_MILLISECONDS);
+ u32 s = readl_relaxed(reg_base + RTC_SHADOW_SECONDS);
+
+ return (u64)s * MSEC_PER_SEC + ms;
+}
+
+static struct clocksource suspend_rtc_clocksource = {
+ .name = "tegra_suspend_timer",
+ .rating = 200,
+ .read = tegra_rtc_read_ms,
+ .mask = CLOCKSOURCE_MASK(32),
+ .flags = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_SUSPEND_NONSTOP,
+};
+
+static inline unsigned int tegra_base_for_cpu(int cpu, bool tegra20)
+{
+ if (tegra20) {
+ switch (cpu) {
+ case 0:
+ return TIMER1_BASE;
+ case 1:
+ return TIMER2_BASE;
+ case 2:
+ return TIMER3_BASE;
+ default:
+ return TIMER4_BASE;
+ }
+ }
+
+ return TIMER10_BASE + cpu * 8;
+}
+
+static inline unsigned int tegra_irq_idx_for_cpu(int cpu, bool tegra20)
+{
+ if (tegra20)
+ return TIMER1_IRQ_IDX + cpu;
+
+ return TIMER10_IRQ_IDX + cpu;
+}
+
+static inline unsigned long tegra_rate_for_timer(struct timer_of *to,
+ bool tegra20)
+{
+ /*
+ * TIMER1-9 are fixed to 1MHz, TIMER10-13 are running off the
+ * parent clock.
+ */
+ if (tegra20)
+ return TIMER_1MHz;
+
+ return timer_of_rate(to);
+}
+
+static int __init tegra_init_timer(struct device_node *np, bool tegra20,
+ int rating)
+{
+ struct timer_of *to;
+ int cpu, ret;
+
+ to = this_cpu_ptr(&tegra_to);
+ ret = timer_of_init(np, to);
+ if (ret)
+ goto out;
+
+ timer_reg_base = timer_of_base(to);
+
+ /*
+ * Configure microsecond timers to have 1MHz clock
+ * Config register is 0xqqww, where qq is "dividend", ww is "divisor"
+ * Uses n+1 scheme
+ */
+ switch (timer_of_rate(to)) {
+ case 12000000:
+ usec_config = 0x000b; /* (11+1)/(0+1) */
+ break;
+ case 12800000:
+ usec_config = 0x043f; /* (63+1)/(4+1) */
+ break;
+ case 13000000:
+ usec_config = 0x000c; /* (12+1)/(0+1) */
+ break;
+ case 16800000:
+ usec_config = 0x0453; /* (83+1)/(4+1) */
+ break;
+ case 19200000:
+ usec_config = 0x045f; /* (95+1)/(4+1) */
+ break;
+ case 26000000:
+ usec_config = 0x0019; /* (25+1)/(0+1) */
+ break;
+ case 38400000:
+ usec_config = 0x04bf; /* (191+1)/(4+1) */
+ break;
+ case 48000000:
+ usec_config = 0x002f; /* (47+1)/(0+1) */
+ break;
+ default:
+ ret = -EINVAL;
+ goto out;
+ }
+
+ writel_relaxed(usec_config, timer_reg_base + TIMERUS_USEC_CFG);
+
+ for_each_possible_cpu(cpu) {
+ struct timer_of *cpu_to = per_cpu_ptr(&tegra_to, cpu);
+ unsigned long flags = IRQF_TIMER | IRQF_NOBALANCING;
+ unsigned long rate = tegra_rate_for_timer(to, tegra20);
+ unsigned int base = tegra_base_for_cpu(cpu, tegra20);
+ unsigned int idx = tegra_irq_idx_for_cpu(cpu, tegra20);
+ unsigned int irq = irq_of_parse_and_map(np, idx);
+
+ if (!irq) {
+ pr_err("failed to map irq for cpu%d\n", cpu);
+ ret = -EINVAL;
+ goto out_irq;
+ }
+
+ cpu_to->clkevt.irq = irq;
+ cpu_to->clkevt.rating = rating;
+ cpu_to->clkevt.cpumask = cpumask_of(cpu);
+ cpu_to->of_base.base = timer_reg_base + base;
+ cpu_to->of_clk.period = rate / HZ;
+ cpu_to->of_clk.rate = rate;
+
+ irq_set_status_flags(cpu_to->clkevt.irq, IRQ_NOAUTOEN);
+
+ ret = request_irq(cpu_to->clkevt.irq, tegra_timer_isr, flags,
+ cpu_to->clkevt.name, &cpu_to->clkevt);
+ if (ret) {
+ pr_err("failed to set up irq for cpu%d: %d\n",
+ cpu, ret);
+ irq_dispose_mapping(cpu_to->clkevt.irq);
+ cpu_to->clkevt.irq = 0;
+ goto out_irq;
+ }
+ }
+
+ sched_clock_register(tegra_read_sched_clock, 32, TIMER_1MHz);
+
+ ret = clocksource_mmio_init(timer_reg_base + TIMERUS_CNTR_1US,
+ "timer_us", TIMER_1MHz, 300, 32,
+ clocksource_mmio_readl_up);
+ if (ret)
+ pr_err("failed to register clocksource: %d\n", ret);
+
+#ifdef CONFIG_ARM
+ register_current_timer_delay(&tegra_delay_timer);
+#endif
+
+ ret = cpuhp_setup_state(CPUHP_AP_TEGRA_TIMER_STARTING,
+ "AP_TEGRA_TIMER_STARTING", tegra_timer_setup,
+ tegra_timer_stop);
+ if (ret)
+ pr_err("failed to set up cpu hp state: %d\n", ret);
+
+ return ret;
+
+out_irq:
+ for_each_possible_cpu(cpu) {
+ struct timer_of *cpu_to;
+
+ cpu_to = per_cpu_ptr(&tegra_to, cpu);
+ if (cpu_to->clkevt.irq) {
+ free_irq(cpu_to->clkevt.irq, &cpu_to->clkevt);
+ irq_dispose_mapping(cpu_to->clkevt.irq);
+ }
+ }
+
+ to->of_base.base = timer_reg_base;
+out:
+ timer_of_cleanup(to);
+
+ return ret;
+}
+
+static int __init tegra210_init_timer(struct device_node *np)
+{
+ /*
+ * Arch-timer can't survive across power cycle of CPU core and
+ * after CPUPORESET signal due to a system design shortcoming,
+ * hence tegra-timer is more preferable on Tegra210.
+ */
+ return tegra_init_timer(np, false, 460);
+}
+TIMER_OF_DECLARE(tegra210_timer, "nvidia,tegra210-timer", tegra210_init_timer);
+
+static int __init tegra20_init_timer(struct device_node *np)
+{
+ int rating;
+
+ /*
+ * Tegra20 and Tegra30 have Cortex A9 CPU that has a TWD timer,
+ * that timer runs off the CPU clock and hence is subjected to
+ * a jitter caused by DVFS clock rate changes. Tegra-timer is
+ * more preferable for older Tegra's, while later SoC generations
+ * have arch-timer as a main per-CPU timer and it is not affected
+ * by DVFS changes.
+ */
+ if (of_machine_is_compatible("nvidia,tegra20") ||
+ of_machine_is_compatible("nvidia,tegra30"))
+ rating = 460;
+ else
+ rating = 330;
+
+ return tegra_init_timer(np, true, rating);
+}
+TIMER_OF_DECLARE(tegra20_timer, "nvidia,tegra20-timer", tegra20_init_timer);
+
+static int __init tegra20_init_rtc(struct device_node *np)
+{
+ int ret;
+
+ ret = timer_of_init(np, &suspend_rtc_to);
+ if (ret)
+ return ret;
+
+ return clocksource_register_hz(&suspend_rtc_clocksource, 1000);
+}
+TIMER_OF_DECLARE(tegra20_rtc, "nvidia,tegra20-rtc", tegra20_init_rtc);
diff --git a/drivers/clocksource/timer-tegra20.c b/drivers/clocksource/timer-tegra20.c
deleted file mode 100644
index 1e7ece279730..000000000000
--- a/drivers/clocksource/timer-tegra20.c
+++ /dev/null
@@ -1,379 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2010 Google, Inc.
- *
- * Author:
- * Colin Cross <ccross@google.com>
- */
-
-#include <linux/clk.h>
-#include <linux/clockchips.h>
-#include <linux/cpu.h>
-#include <linux/cpumask.h>
-#include <linux/delay.h>
-#include <linux/err.h>
-#include <linux/interrupt.h>
-#include <linux/of_address.h>
-#include <linux/of_irq.h>
-#include <linux/percpu.h>
-#include <linux/sched_clock.h>
-#include <linux/time.h>
-
-#include "timer-of.h"
-
-#ifdef CONFIG_ARM
-#include <asm/mach/time.h>
-#endif
-
-#define RTC_SECONDS 0x08
-#define RTC_SHADOW_SECONDS 0x0c
-#define RTC_MILLISECONDS 0x10
-
-#define TIMERUS_CNTR_1US 0x10
-#define TIMERUS_USEC_CFG 0x14
-#define TIMERUS_CNTR_FREEZE 0x4c
-
-#define TIMER_PTV 0x0
-#define TIMER_PTV_EN BIT(31)
-#define TIMER_PTV_PER BIT(30)
-#define TIMER_PCR 0x4
-#define TIMER_PCR_INTR_CLR BIT(30)
-
-#ifdef CONFIG_ARM
-#define TIMER_CPU0 0x50 /* TIMER3 */
-#else
-#define TIMER_CPU0 0x90 /* TIMER10 */
-#define TIMER10_IRQ_IDX 10
-#define IRQ_IDX_FOR_CPU(cpu) (TIMER10_IRQ_IDX + cpu)
-#endif
-#define TIMER_BASE_FOR_CPU(cpu) (TIMER_CPU0 + (cpu) * 8)
-
-static u32 usec_config;
-static void __iomem *timer_reg_base;
-#ifdef CONFIG_ARM
-static struct delay_timer tegra_delay_timer;
-#endif
-
-static int tegra_timer_set_next_event(unsigned long cycles,
- struct clock_event_device *evt)
-{
- void __iomem *reg_base = timer_of_base(to_timer_of(evt));
-
- writel(TIMER_PTV_EN |
- ((cycles > 1) ? (cycles - 1) : 0), /* n+1 scheme */
- reg_base + TIMER_PTV);
-
- return 0;
-}
-
-static int tegra_timer_shutdown(struct clock_event_device *evt)
-{
- void __iomem *reg_base = timer_of_base(to_timer_of(evt));
-
- writel(0, reg_base + TIMER_PTV);
-
- return 0;
-}
-
-static int tegra_timer_set_periodic(struct clock_event_device *evt)
-{
- void __iomem *reg_base = timer_of_base(to_timer_of(evt));
-
- writel(TIMER_PTV_EN | TIMER_PTV_PER |
- ((timer_of_rate(to_timer_of(evt)) / HZ) - 1),
- reg_base + TIMER_PTV);
-
- return 0;
-}
-
-static irqreturn_t tegra_timer_isr(int irq, void *dev_id)
-{
- struct clock_event_device *evt = (struct clock_event_device *)dev_id;
- void __iomem *reg_base = timer_of_base(to_timer_of(evt));
-
- writel(TIMER_PCR_INTR_CLR, reg_base + TIMER_PCR);
- evt->event_handler(evt);
-
- return IRQ_HANDLED;
-}
-
-static void tegra_timer_suspend(struct clock_event_device *evt)
-{
- void __iomem *reg_base = timer_of_base(to_timer_of(evt));
-
- writel(TIMER_PCR_INTR_CLR, reg_base + TIMER_PCR);
-}
-
-static void tegra_timer_resume(struct clock_event_device *evt)
-{
- writel(usec_config, timer_reg_base + TIMERUS_USEC_CFG);
-}
-
-#ifdef CONFIG_ARM64
-static DEFINE_PER_CPU(struct timer_of, tegra_to) = {
- .flags = TIMER_OF_CLOCK | TIMER_OF_BASE,
-
- .clkevt = {
- .name = "tegra_timer",
- .rating = 460,
- .features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PERIODIC,
- .set_next_event = tegra_timer_set_next_event,
- .set_state_shutdown = tegra_timer_shutdown,
- .set_state_periodic = tegra_timer_set_periodic,
- .set_state_oneshot = tegra_timer_shutdown,
- .tick_resume = tegra_timer_shutdown,
- .suspend = tegra_timer_suspend,
- .resume = tegra_timer_resume,
- },
-};
-
-static int tegra_timer_setup(unsigned int cpu)
-{
- struct timer_of *to = per_cpu_ptr(&tegra_to, cpu);
-
- irq_force_affinity(to->clkevt.irq, cpumask_of(cpu));
- enable_irq(to->clkevt.irq);
-
- clockevents_config_and_register(&to->clkevt, timer_of_rate(to),
- 1, /* min */
- 0x1fffffff); /* 29 bits */
-
- return 0;
-}
-
-static int tegra_timer_stop(unsigned int cpu)
-{
- struct timer_of *to = per_cpu_ptr(&tegra_to, cpu);
-
- to->clkevt.set_state_shutdown(&to->clkevt);
- disable_irq_nosync(to->clkevt.irq);
-
- return 0;
-}
-#else /* CONFIG_ARM */
-static struct timer_of tegra_to = {
- .flags = TIMER_OF_CLOCK | TIMER_OF_BASE | TIMER_OF_IRQ,
-
- .clkevt = {
- .name = "tegra_timer",
- .rating = 300,
- .features = CLOCK_EVT_FEAT_ONESHOT |
- CLOCK_EVT_FEAT_PERIODIC |
- CLOCK_EVT_FEAT_DYNIRQ,
- .set_next_event = tegra_timer_set_next_event,
- .set_state_shutdown = tegra_timer_shutdown,
- .set_state_periodic = tegra_timer_set_periodic,
- .set_state_oneshot = tegra_timer_shutdown,
- .tick_resume = tegra_timer_shutdown,
- .suspend = tegra_timer_suspend,
- .resume = tegra_timer_resume,
- .cpumask = cpu_possible_mask,
- },
-
- .of_irq = {
- .index = 2,
- .flags = IRQF_TIMER | IRQF_TRIGGER_HIGH,
- .handler = tegra_timer_isr,
- },
-};
-
-static u64 notrace tegra_read_sched_clock(void)
-{
- return readl(timer_reg_base + TIMERUS_CNTR_1US);
-}
-
-static unsigned long tegra_delay_timer_read_counter_long(void)
-{
- return readl(timer_reg_base + TIMERUS_CNTR_1US);
-}
-
-static struct timer_of suspend_rtc_to = {
- .flags = TIMER_OF_BASE | TIMER_OF_CLOCK,
-};
-
-/*
- * tegra_rtc_read - Reads the Tegra RTC registers
- * Care must be taken that this funciton is not called while the
- * tegra_rtc driver could be executing to avoid race conditions
- * on the RTC shadow register
- */
-static u64 tegra_rtc_read_ms(struct clocksource *cs)
-{
- u32 ms = readl(timer_of_base(&suspend_rtc_to) + RTC_MILLISECONDS);
- u32 s = readl(timer_of_base(&suspend_rtc_to) + RTC_SHADOW_SECONDS);
- return (u64)s * MSEC_PER_SEC + ms;
-}
-
-static struct clocksource suspend_rtc_clocksource = {
- .name = "tegra_suspend_timer",
- .rating = 200,
- .read = tegra_rtc_read_ms,
- .mask = CLOCKSOURCE_MASK(32),
- .flags = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_SUSPEND_NONSTOP,
-};
-#endif
-
-static int tegra_timer_common_init(struct device_node *np, struct timer_of *to)
-{
- int ret = 0;
-
- ret = timer_of_init(np, to);
- if (ret < 0)
- goto out;
-
- timer_reg_base = timer_of_base(to);
-
- /*
- * Configure microsecond timers to have 1MHz clock
- * Config register is 0xqqww, where qq is "dividend", ww is "divisor"
- * Uses n+1 scheme
- */
- switch (timer_of_rate(to)) {
- case 12000000:
- usec_config = 0x000b; /* (11+1)/(0+1) */
- break;
- case 12800000:
- usec_config = 0x043f; /* (63+1)/(4+1) */
- break;
- case 13000000:
- usec_config = 0x000c; /* (12+1)/(0+1) */
- break;
- case 16800000:
- usec_config = 0x0453; /* (83+1)/(4+1) */
- break;
- case 19200000:
- usec_config = 0x045f; /* (95+1)/(4+1) */
- break;
- case 26000000:
- usec_config = 0x0019; /* (25+1)/(0+1) */
- break;
- case 38400000:
- usec_config = 0x04bf; /* (191+1)/(4+1) */
- break;
- case 48000000:
- usec_config = 0x002f; /* (47+1)/(0+1) */
- break;
- default:
- ret = -EINVAL;
- goto out;
- }
-
- writel(usec_config, timer_of_base(to) + TIMERUS_USEC_CFG);
-
-out:
- return ret;
-}
-
-#ifdef CONFIG_ARM64
-static int __init tegra_init_timer(struct device_node *np)
-{
- int cpu, ret = 0;
- struct timer_of *to;
-
- to = this_cpu_ptr(&tegra_to);
- ret = tegra_timer_common_init(np, to);
- if (ret < 0)
- goto out;
-
- for_each_possible_cpu(cpu) {
- struct timer_of *cpu_to;
-
- cpu_to = per_cpu_ptr(&tegra_to, cpu);
- cpu_to->of_base.base = timer_reg_base + TIMER_BASE_FOR_CPU(cpu);
- cpu_to->of_clk.rate = timer_of_rate(to);
- cpu_to->clkevt.cpumask = cpumask_of(cpu);
- cpu_to->clkevt.irq =
- irq_of_parse_and_map(np, IRQ_IDX_FOR_CPU(cpu));
- if (!cpu_to->clkevt.irq) {
- pr_err("%s: can't map IRQ for CPU%d\n",
- __func__, cpu);
- ret = -EINVAL;
- goto out;
- }
-
- irq_set_status_flags(cpu_to->clkevt.irq, IRQ_NOAUTOEN);
- ret = request_irq(cpu_to->clkevt.irq, tegra_timer_isr,
- IRQF_TIMER | IRQF_NOBALANCING,
- cpu_to->clkevt.name, &cpu_to->clkevt);
- if (ret) {
- pr_err("%s: cannot setup irq %d for CPU%d\n",
- __func__, cpu_to->clkevt.irq, cpu);
- ret = -EINVAL;
- goto out_irq;
- }
- }
-
- cpuhp_setup_state(CPUHP_AP_TEGRA_TIMER_STARTING,
- "AP_TEGRA_TIMER_STARTING", tegra_timer_setup,
- tegra_timer_stop);
-
- return ret;
-out_irq:
- for_each_possible_cpu(cpu) {
- struct timer_of *cpu_to;
-
- cpu_to = per_cpu_ptr(&tegra_to, cpu);
- if (cpu_to->clkevt.irq) {
- free_irq(cpu_to->clkevt.irq, &cpu_to->clkevt);
- irq_dispose_mapping(cpu_to->clkevt.irq);
- }
- }
-out:
- timer_of_cleanup(to);
- return ret;
-}
-#else /* CONFIG_ARM */
-static int __init tegra_init_timer(struct device_node *np)
-{
- int ret = 0;
-
- ret = tegra_timer_common_init(np, &tegra_to);
- if (ret < 0)
- goto out;
-
- tegra_to.of_base.base = timer_reg_base + TIMER_BASE_FOR_CPU(0);
- tegra_to.of_clk.rate = 1000000; /* microsecond timer */
-
- sched_clock_register(tegra_read_sched_clock, 32,
- timer_of_rate(&tegra_to));
- ret = clocksource_mmio_init(timer_reg_base + TIMERUS_CNTR_1US,
- "timer_us", timer_of_rate(&tegra_to),
- 300, 32, clocksource_mmio_readl_up);
- if (ret) {
- pr_err("Failed to register clocksource\n");
- goto out;
- }
-
- tegra_delay_timer.read_current_timer =
- tegra_delay_timer_read_counter_long;
- tegra_delay_timer.freq = timer_of_rate(&tegra_to);
- register_current_timer_delay(&tegra_delay_timer);
-
- clockevents_config_and_register(&tegra_to.clkevt,
- timer_of_rate(&tegra_to),
- 0x1,
- 0x1fffffff);
-
- return ret;
-out:
- timer_of_cleanup(&tegra_to);
-
- return ret;
-}
-
-static int __init tegra20_init_rtc(struct device_node *np)
-{
- int ret;
-
- ret = timer_of_init(np, &suspend_rtc_to);
- if (ret)
- return ret;
-
- clocksource_register_hz(&suspend_rtc_clocksource, 1000);
-
- return 0;
-}
-TIMER_OF_DECLARE(tegra20_rtc, "nvidia,tegra20-rtc", tegra20_init_rtc);
-#endif
-TIMER_OF_DECLARE(tegra210_timer, "nvidia,tegra210-timer", tegra_init_timer);
-TIMER_OF_DECLARE(tegra20_timer, "nvidia,tegra20-timer", tegra_init_timer);
diff --git a/drivers/connector/connector.c b/drivers/connector/connector.c
index 23553ed6b548..2d22d6bf52f2 100644
--- a/drivers/connector/connector.c
+++ b/drivers/connector/connector.c
@@ -248,16 +248,12 @@ static int __maybe_unused cn_proc_show(struct seq_file *m, void *v)
return 0;
}
-static struct cn_dev cdev = {
- .input = cn_rx_skb,
-};
-
static int cn_init(void)
{
struct cn_dev *dev = &cdev;
struct netlink_kernel_cfg cfg = {
.groups = CN_NETLINK_USERS + 0xf,
- .input = dev->input,
+ .input = cn_rx_skb,
};
dev->nls = netlink_kernel_create(&init_net, NETLINK_CONNECTOR, &cfg);
diff --git a/drivers/counter/104-quad-8.c b/drivers/counter/104-quad-8.c
index 4fa2931dcb7b..00b113f4b958 100644
--- a/drivers/counter/104-quad-8.c
+++ b/drivers/counter/104-quad-8.c
@@ -833,7 +833,7 @@ static int quad8_action_get(struct counter_device *counter,
return 0;
}
-const struct counter_ops quad8_ops = {
+static const struct counter_ops quad8_ops = {
.signal_read = quad8_signal_read,
.count_read = quad8_count_read,
.count_write = quad8_count_write,
diff --git a/drivers/counter/ftm-quaddec.c b/drivers/counter/ftm-quaddec.c
index c83c8875bf82..4046aa9f9234 100644
--- a/drivers/counter/ftm-quaddec.c
+++ b/drivers/counter/ftm-quaddec.c
@@ -100,16 +100,18 @@ static void ftm_quaddec_init(struct ftm_quaddec *ftm)
ftm_set_write_protection(ftm);
}
-static void ftm_quaddec_disable(struct ftm_quaddec *ftm)
+static void ftm_quaddec_disable(void *ftm)
{
- ftm_clear_write_protection(ftm);
- ftm_write(ftm, FTM_MODE, 0);
- ftm_write(ftm, FTM_QDCTRL, 0);
+ struct ftm_quaddec *ftm_qua = ftm;
+
+ ftm_clear_write_protection(ftm_qua);
+ ftm_write(ftm_qua, FTM_MODE, 0);
+ ftm_write(ftm_qua, FTM_QDCTRL, 0);
/*
* This is enough to disable the counter. No clock has been
* selected by writing to FTM_SC in init()
*/
- ftm_set_write_protection(ftm);
+ ftm_set_write_protection(ftm_qua);
}
static int ftm_quaddec_get_prescaler(struct counter_device *counter,
@@ -317,20 +319,13 @@ static int ftm_quaddec_probe(struct platform_device *pdev)
ftm_quaddec_init(ftm);
- ret = counter_register(&ftm->counter);
+ ret = devm_add_action_or_reset(&pdev->dev, ftm_quaddec_disable, ftm);
if (ret)
- ftm_quaddec_disable(ftm);
-
- return ret;
-}
+ return ret;
-static int ftm_quaddec_remove(struct platform_device *pdev)
-{
- struct ftm_quaddec *ftm = platform_get_drvdata(pdev);
-
- counter_unregister(&ftm->counter);
-
- ftm_quaddec_disable(ftm);
+ ret = devm_counter_register(&pdev->dev, &ftm->counter);
+ if (ret)
+ return ret;
return 0;
}
@@ -346,11 +341,10 @@ static struct platform_driver ftm_quaddec_driver = {
.of_match_table = ftm_quaddec_match,
},
.probe = ftm_quaddec_probe,
- .remove = ftm_quaddec_remove,
};
module_platform_driver(ftm_quaddec_driver);
MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Kjeld Flarup <kfa@deif.com");
-MODULE_AUTHOR("Patrick Havelange <patrick.havelange@essensium.com");
+MODULE_AUTHOR("Kjeld Flarup <kfa@deif.com>");
+MODULE_AUTHOR("Patrick Havelange <patrick.havelange@essensium.com>");
diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm
index f8129edc145e..a905796f7f85 100644
--- a/drivers/cpufreq/Kconfig.arm
+++ b/drivers/cpufreq/Kconfig.arm
@@ -19,6 +19,18 @@ config ACPI_CPPC_CPUFREQ
If in doubt, say N.
+config ARM_ALLWINNER_SUN50I_CPUFREQ_NVMEM
+ tristate "Allwinner nvmem based SUN50I CPUFreq driver"
+ depends on ARCH_SUNXI
+ depends on NVMEM_SUNXI_SID
+ select PM_OPP
+ help
+ This adds the nvmem based CPUFreq driver for Allwinner
+ h6 SoC.
+
+ To compile this driver as a module, choose M here: the
+ module will be called sun50i-cpufreq-nvmem.
+
config ARM_ARMADA_37XX_CPUFREQ
tristate "Armada 37xx CPUFreq support"
depends on ARCH_MVEBU && CPUFREQ_DT
@@ -93,6 +105,15 @@ config ARM_IMX6Q_CPUFREQ
If in doubt, say N.
+config ARM_IMX_CPUFREQ_DT
+ tristate "Freescale i.MX8M cpufreq support"
+ depends on ARCH_MXC && CPUFREQ_DT
+ help
+ This adds cpufreq driver support for Freescale i.MX8M series SoCs,
+ based on cpufreq-dt.
+
+ If in doubt, say N.
+
config ARM_KIRKWOOD_CPUFREQ
def_bool MACH_KIRKWOOD
help
@@ -111,8 +132,8 @@ config ARM_OMAP2PLUS_CPUFREQ
depends on ARCH_OMAP2PLUS
default ARCH_OMAP2PLUS
-config ARM_QCOM_CPUFREQ_KRYO
- tristate "Qualcomm Kryo based CPUFreq"
+config ARM_QCOM_CPUFREQ_NVMEM
+ tristate "Qualcomm nvmem based CPUFreq"
depends on ARM64
depends on QCOM_QFPROM
depends on QCOM_SMEM
@@ -133,6 +154,14 @@ config ARM_QCOM_CPUFREQ_HW
The driver implements the cpufreq interface for this HW engine.
Say Y if you want to support CPUFreq HW.
+config ARM_RASPBERRYPI_CPUFREQ
+ tristate "Raspberry Pi cpufreq support"
+ depends on CLK_RASPBERRYPI || COMPILE_TEST
+ help
+ This adds the CPUFreq driver for Raspberry Pi
+
+ If in doubt, say N.
+
config ARM_S3C_CPUFREQ
bool
help
diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile
index 689b26c6f949..9a9f5ccd13d9 100644
--- a/drivers/cpufreq/Makefile
+++ b/drivers/cpufreq/Makefile
@@ -56,6 +56,7 @@ obj-$(CONFIG_ACPI_CPPC_CPUFREQ) += cppc_cpufreq.o
obj-$(CONFIG_ARCH_DAVINCI) += davinci-cpufreq.o
obj-$(CONFIG_ARM_HIGHBANK_CPUFREQ) += highbank-cpufreq.o
obj-$(CONFIG_ARM_IMX6Q_CPUFREQ) += imx6q-cpufreq.o
+obj-$(CONFIG_ARM_IMX_CPUFREQ_DT) += imx-cpufreq-dt.o
obj-$(CONFIG_ARM_KIRKWOOD_CPUFREQ) += kirkwood-cpufreq.o
obj-$(CONFIG_ARM_MEDIATEK_CPUFREQ) += mediatek-cpufreq.o
obj-$(CONFIG_MACH_MVEBU_V7) += mvebu-cpufreq.o
@@ -63,7 +64,8 @@ obj-$(CONFIG_ARM_OMAP2PLUS_CPUFREQ) += omap-cpufreq.o
obj-$(CONFIG_ARM_PXA2xx_CPUFREQ) += pxa2xx-cpufreq.o
obj-$(CONFIG_PXA3xx) += pxa3xx-cpufreq.o
obj-$(CONFIG_ARM_QCOM_CPUFREQ_HW) += qcom-cpufreq-hw.o
-obj-$(CONFIG_ARM_QCOM_CPUFREQ_KRYO) += qcom-cpufreq-kryo.o
+obj-$(CONFIG_ARM_QCOM_CPUFREQ_NVMEM) += qcom-cpufreq-nvmem.o
+obj-$(CONFIG_ARM_RASPBERRYPI_CPUFREQ) += raspberrypi-cpufreq.o
obj-$(CONFIG_ARM_S3C2410_CPUFREQ) += s3c2410-cpufreq.o
obj-$(CONFIG_ARM_S3C2412_CPUFREQ) += s3c2412-cpufreq.o
obj-$(CONFIG_ARM_S3C2416_CPUFREQ) += s3c2416-cpufreq.o
@@ -78,6 +80,7 @@ obj-$(CONFIG_ARM_SCMI_CPUFREQ) += scmi-cpufreq.o
obj-$(CONFIG_ARM_SCPI_CPUFREQ) += scpi-cpufreq.o
obj-$(CONFIG_ARM_SPEAR_CPUFREQ) += spear-cpufreq.o
obj-$(CONFIG_ARM_STI_CPUFREQ) += sti-cpufreq.o
+obj-$(CONFIG_ARM_ALLWINNER_SUN50I_CPUFREQ_NVMEM) += sun50i-cpufreq-nvmem.o
obj-$(CONFIG_ARM_TANGO_CPUFREQ) += tango-cpufreq.o
obj-$(CONFIG_ARM_TEGRA20_CPUFREQ) += tegra20-cpufreq.o
obj-$(CONFIG_ARM_TEGRA124_CPUFREQ) += tegra124-cpufreq.o
diff --git a/drivers/cpufreq/armada-37xx-cpufreq.c b/drivers/cpufreq/armada-37xx-cpufreq.c
index 0df16eb1eb3c..aa0f06dec959 100644
--- a/drivers/cpufreq/armada-37xx-cpufreq.c
+++ b/drivers/cpufreq/armada-37xx-cpufreq.c
@@ -257,7 +257,7 @@ static void __init armada37xx_cpufreq_avs_configure(struct regmap *base,
static void __init armada37xx_cpufreq_avs_setup(struct regmap *base,
struct armada_37xx_dvfs *dvfs)
{
- unsigned int avs_val = 0, freq;
+ unsigned int avs_val = 0;
int load_level = 0;
if (base == NULL)
@@ -275,8 +275,6 @@ static void __init armada37xx_cpufreq_avs_setup(struct regmap *base,
for (load_level = 1; load_level < LOAD_LEVEL_NR; load_level++) {
- freq = dvfs->cpu_freq_max / dvfs->divider[load_level];
-
avs_val = dvfs->avs[load_level];
regmap_update_bits(base, ARMADA_37XX_AVS_VSET(load_level-1),
ARMADA_37XX_AVS_VDD_MASK << ARMADA_37XX_AVS_HIGH_VDD_LIMIT |
diff --git a/drivers/cpufreq/armada-8k-cpufreq.c b/drivers/cpufreq/armada-8k-cpufreq.c
index 988ebc326bdb..39e34f5066d3 100644
--- a/drivers/cpufreq/armada-8k-cpufreq.c
+++ b/drivers/cpufreq/armada-8k-cpufreq.c
@@ -136,6 +136,8 @@ static int __init armada_8k_cpufreq_init(void)
nb_cpus = num_possible_cpus();
freq_tables = kcalloc(nb_cpus, sizeof(*freq_tables), GFP_KERNEL);
+ if (!freq_tables)
+ return -ENOMEM;
cpumask_copy(&cpus, cpu_possible_mask);
/*
diff --git a/drivers/cpufreq/bmips-cpufreq.c b/drivers/cpufreq/bmips-cpufreq.c
index 56a4ebbf00e0..f7c23fa468f0 100644
--- a/drivers/cpufreq/bmips-cpufreq.c
+++ b/drivers/cpufreq/bmips-cpufreq.c
@@ -131,23 +131,18 @@ static int bmips_cpufreq_exit(struct cpufreq_policy *policy)
static int bmips_cpufreq_init(struct cpufreq_policy *policy)
{
struct cpufreq_frequency_table *freq_table;
- int ret;
freq_table = bmips_cpufreq_get_freq_table(policy);
if (IS_ERR(freq_table)) {
- ret = PTR_ERR(freq_table);
- pr_err("%s: couldn't determine frequency table (%d).\n",
- BMIPS_CPUFREQ_NAME, ret);
- return ret;
+ pr_err("%s: couldn't determine frequency table (%ld).\n",
+ BMIPS_CPUFREQ_NAME, PTR_ERR(freq_table));
+ return PTR_ERR(freq_table);
}
- ret = cpufreq_generic_init(policy, freq_table, TRANSITION_LATENCY);
- if (ret)
- bmips_cpufreq_exit(policy);
- else
- pr_info("%s: registered\n", BMIPS_CPUFREQ_NAME);
+ cpufreq_generic_init(policy, freq_table, TRANSITION_LATENCY);
+ pr_info("%s: registered\n", BMIPS_CPUFREQ_NAME);
- return ret;
+ return 0;
}
static struct cpufreq_driver bmips_cpufreq_driver = {
diff --git a/drivers/cpufreq/brcmstb-avs-cpufreq.c b/drivers/cpufreq/brcmstb-avs-cpufreq.c
index e6f9cbe5835f..77b0e5d0fb13 100644
--- a/drivers/cpufreq/brcmstb-avs-cpufreq.c
+++ b/drivers/cpufreq/brcmstb-avs-cpufreq.c
@@ -384,12 +384,12 @@ static int brcm_avs_set_pstate(struct private_data *priv, unsigned int pstate)
return __issue_avs_command(priv, AVS_CMD_SET_PSTATE, true, args);
}
-static unsigned long brcm_avs_get_voltage(void __iomem *base)
+static u32 brcm_avs_get_voltage(void __iomem *base)
{
return readl(base + AVS_MBOX_VOLTAGE1);
}
-static unsigned long brcm_avs_get_frequency(void __iomem *base)
+static u32 brcm_avs_get_frequency(void __iomem *base)
{
return readl(base + AVS_MBOX_FREQUENCY) * 1000; /* in kHz */
}
@@ -446,8 +446,8 @@ static bool brcm_avs_is_firmware_loaded(struct private_data *priv)
rc = brcm_avs_get_pmap(priv, NULL);
magic = readl(priv->base + AVS_MBOX_MAGIC);
- return (magic == AVS_FIRMWARE_MAGIC) && (rc != -ENOTSUPP) &&
- (rc != -EINVAL);
+ return (magic == AVS_FIRMWARE_MAGIC) && ((rc != -ENOTSUPP) ||
+ (rc != -EINVAL));
}
static unsigned int brcm_avs_cpufreq_get(unsigned int cpu)
@@ -653,14 +653,14 @@ static ssize_t show_brcm_avs_voltage(struct cpufreq_policy *policy, char *buf)
{
struct private_data *priv = policy->driver_data;
- return sprintf(buf, "0x%08lx\n", brcm_avs_get_voltage(priv->base));
+ return sprintf(buf, "0x%08x\n", brcm_avs_get_voltage(priv->base));
}
static ssize_t show_brcm_avs_frequency(struct cpufreq_policy *policy, char *buf)
{
struct private_data *priv = policy->driver_data;
- return sprintf(buf, "0x%08lx\n", brcm_avs_get_frequency(priv->base));
+ return sprintf(buf, "0x%08x\n", brcm_avs_get_frequency(priv->base));
}
cpufreq_freq_attr_ro(brcm_avs_pstate);
diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c
index 88e00683eaeb..bca8d1f47fd2 100644
--- a/drivers/cpufreq/cpufreq-dt-platdev.c
+++ b/drivers/cpufreq/cpufreq-dt-platdev.c
@@ -37,7 +37,6 @@ static const struct of_device_id whitelist[] __initconst = {
{ .compatible = "fsl,imx27", },
{ .compatible = "fsl,imx51", },
{ .compatible = "fsl,imx53", },
- { .compatible = "fsl,imx7d", },
{ .compatible = "marvell,berlin", },
{ .compatible = "marvell,pxa250", },
@@ -102,9 +101,16 @@ static const struct of_device_id whitelist[] __initconst = {
* platforms using "operating-points-v2" property.
*/
static const struct of_device_id blacklist[] __initconst = {
+ { .compatible = "allwinner,sun50i-h6", },
+
{ .compatible = "calxeda,highbank", },
{ .compatible = "calxeda,ecx-2000", },
+ { .compatible = "fsl,imx7d", },
+ { .compatible = "fsl,imx8mq", },
+ { .compatible = "fsl,imx8mm", },
+ { .compatible = "fsl,imx8mn", },
+
{ .compatible = "marvell,armadaxp", },
{ .compatible = "mediatek,mt2701", },
@@ -114,12 +120,14 @@ static const struct of_device_id blacklist[] __initconst = {
{ .compatible = "mediatek,mt817x", },
{ .compatible = "mediatek,mt8173", },
{ .compatible = "mediatek,mt8176", },
+ { .compatible = "mediatek,mt8183", },
{ .compatible = "nvidia,tegra124", },
{ .compatible = "nvidia,tegra210", },
{ .compatible = "qcom,apq8096", },
{ .compatible = "qcom,msm8996", },
+ { .compatible = "qcom,qcs404", },
{ .compatible = "st,stih407", },
{ .compatible = "st,stih410", },
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index e84bf0eb7239..c52d6fa32aac 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -23,6 +23,7 @@
#include <linux/kernel_stat.h>
#include <linux/module.h>
#include <linux/mutex.h>
+#include <linux/pm_qos.h>
#include <linux/slab.h>
#include <linux/suspend.h>
#include <linux/syscore_ops.h>
@@ -158,7 +159,7 @@ EXPORT_SYMBOL_GPL(arch_set_freq_scale);
* - set policies transition latency
* - policy->cpus with all possible CPUs
*/
-int cpufreq_generic_init(struct cpufreq_policy *policy,
+void cpufreq_generic_init(struct cpufreq_policy *policy,
struct cpufreq_frequency_table *table,
unsigned int transition_latency)
{
@@ -170,8 +171,6 @@ int cpufreq_generic_init(struct cpufreq_policy *policy,
* share the clock and voltage and clock.
*/
cpumask_setall(policy->cpus);
-
- return 0;
}
EXPORT_SYMBOL_GPL(cpufreq_generic_init);
@@ -356,12 +355,10 @@ static void cpufreq_notify_transition(struct cpufreq_policy *policy,
* which is not equal to what the cpufreq core thinks is
* "old frequency".
*/
- if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
- if (policy->cur && (policy->cur != freqs->old)) {
- pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
- freqs->old, policy->cur);
- freqs->old = policy->cur;
- }
+ if (policy->cur && policy->cur != freqs->old) {
+ pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
+ freqs->old, policy->cur);
+ freqs->old = policy->cur;
}
srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
@@ -631,7 +628,7 @@ static int cpufreq_parse_policy(char *str_governor,
}
/**
- * cpufreq_parse_governor - parse a governor string only for !setpolicy
+ * cpufreq_parse_governor - parse a governor string only for has_target()
*/
static int cpufreq_parse_governor(char *str_governor,
struct cpufreq_policy *policy)
@@ -716,23 +713,15 @@ static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
static ssize_t store_##file_name \
(struct cpufreq_policy *policy, const char *buf, size_t count) \
{ \
- int ret, temp; \
- struct cpufreq_policy new_policy; \
- \
- memcpy(&new_policy, policy, sizeof(*policy)); \
- new_policy.min = policy->user_policy.min; \
- new_policy.max = policy->user_policy.max; \
+ unsigned long val; \
+ int ret; \
\
- ret = sscanf(buf, "%u", &new_policy.object); \
+ ret = sscanf(buf, "%lu", &val); \
if (ret != 1) \
return -EINVAL; \
\
- temp = new_policy.object; \
- ret = cpufreq_set_policy(policy, &new_policy); \
- if (!ret) \
- policy->user_policy.object = temp; \
- \
- return ret ? ret : count; \
+ ret = dev_pm_qos_update_request(policy->object##_freq_req, val);\
+ return ret >= 0 ? count : ret; \
}
store_one(scaling_min_freq, min);
@@ -998,7 +987,7 @@ static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu)
{
struct device *dev = get_cpu_device(cpu);
- if (!dev)
+ if (unlikely(!dev))
return;
if (cpumask_test_and_set_cpu(cpu, policy->real_cpus))
@@ -1114,20 +1103,79 @@ static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cp
return ret;
}
+void refresh_frequency_limits(struct cpufreq_policy *policy)
+{
+ struct cpufreq_policy new_policy;
+
+ if (!policy_is_inactive(policy)) {
+ new_policy = *policy;
+ pr_debug("updating policy for CPU %u\n", policy->cpu);
+
+ cpufreq_set_policy(policy, &new_policy);
+ }
+}
+EXPORT_SYMBOL(refresh_frequency_limits);
+
static void handle_update(struct work_struct *work)
{
struct cpufreq_policy *policy =
container_of(work, struct cpufreq_policy, update);
- unsigned int cpu = policy->cpu;
- pr_debug("handle_update for cpu %u called\n", cpu);
- cpufreq_update_policy(cpu);
+
+ pr_debug("handle_update for cpu %u called\n", policy->cpu);
+ down_write(&policy->rwsem);
+ refresh_frequency_limits(policy);
+ up_write(&policy->rwsem);
+}
+
+static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq,
+ void *data)
+{
+ struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min);
+
+ schedule_work(&policy->update);
+ return 0;
+}
+
+static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq,
+ void *data)
+{
+ struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max);
+
+ schedule_work(&policy->update);
+ return 0;
+}
+
+static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
+{
+ struct kobject *kobj;
+ struct completion *cmp;
+
+ down_write(&policy->rwsem);
+ cpufreq_stats_free_table(policy);
+ kobj = &policy->kobj;
+ cmp = &policy->kobj_unregister;
+ up_write(&policy->rwsem);
+ kobject_put(kobj);
+
+ /*
+ * We need to make sure that the underlying kobj is
+ * actually not referenced anymore by anybody before we
+ * proceed with unloading.
+ */
+ pr_debug("waiting for dropping of refcount\n");
+ wait_for_completion(cmp);
+ pr_debug("wait complete\n");
}
static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
{
struct cpufreq_policy *policy;
+ struct device *dev = get_cpu_device(cpu);
int ret;
+ if (!dev)
+ return NULL;
+
policy = kzalloc(sizeof(*policy), GFP_KERNEL);
if (!policy)
return NULL;
@@ -1144,7 +1192,7 @@ static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
cpufreq_global_kobject, "policy%u", cpu);
if (ret) {
- pr_err("%s: failed to init policy->kobj: %d\n", __func__, ret);
+ dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret);
/*
* The entire policy object will be freed below, but the extra
* memory allocated for the kobject name needs to be freed by
@@ -1154,6 +1202,25 @@ static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
goto err_free_real_cpus;
}
+ policy->nb_min.notifier_call = cpufreq_notifier_min;
+ policy->nb_max.notifier_call = cpufreq_notifier_max;
+
+ ret = dev_pm_qos_add_notifier(dev, &policy->nb_min,
+ DEV_PM_QOS_MIN_FREQUENCY);
+ if (ret) {
+ dev_err(dev, "Failed to register MIN QoS notifier: %d (%*pbl)\n",
+ ret, cpumask_pr_args(policy->cpus));
+ goto err_kobj_remove;
+ }
+
+ ret = dev_pm_qos_add_notifier(dev, &policy->nb_max,
+ DEV_PM_QOS_MAX_FREQUENCY);
+ if (ret) {
+ dev_err(dev, "Failed to register MAX QoS notifier: %d (%*pbl)\n",
+ ret, cpumask_pr_args(policy->cpus));
+ goto err_min_qos_notifier;
+ }
+
INIT_LIST_HEAD(&policy->policy_list);
init_rwsem(&policy->rwsem);
spin_lock_init(&policy->transition_lock);
@@ -1164,6 +1231,11 @@ static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
policy->cpu = cpu;
return policy;
+err_min_qos_notifier:
+ dev_pm_qos_remove_notifier(dev, &policy->nb_min,
+ DEV_PM_QOS_MIN_FREQUENCY);
+err_kobj_remove:
+ cpufreq_policy_put_kobj(policy);
err_free_real_cpus:
free_cpumask_var(policy->real_cpus);
err_free_rcpumask:
@@ -1176,30 +1248,9 @@ err_free_policy:
return NULL;
}
-static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
-{
- struct kobject *kobj;
- struct completion *cmp;
-
- down_write(&policy->rwsem);
- cpufreq_stats_free_table(policy);
- kobj = &policy->kobj;
- cmp = &policy->kobj_unregister;
- up_write(&policy->rwsem);
- kobject_put(kobj);
-
- /*
- * We need to make sure that the underlying kobj is
- * actually not referenced anymore by anybody before we
- * proceed with unloading.
- */
- pr_debug("waiting for dropping of refcount\n");
- wait_for_completion(cmp);
- pr_debug("wait complete\n");
-}
-
static void cpufreq_policy_free(struct cpufreq_policy *policy)
{
+ struct device *dev = get_cpu_device(policy->cpu);
unsigned long flags;
int cpu;
@@ -1211,6 +1262,24 @@ static void cpufreq_policy_free(struct cpufreq_policy *policy)
per_cpu(cpufreq_cpu_data, cpu) = NULL;
write_unlock_irqrestore(&cpufreq_driver_lock, flags);
+ dev_pm_qos_remove_notifier(dev, &policy->nb_max,
+ DEV_PM_QOS_MAX_FREQUENCY);
+ dev_pm_qos_remove_notifier(dev, &policy->nb_min,
+ DEV_PM_QOS_MIN_FREQUENCY);
+
+ if (policy->max_freq_req) {
+ /*
+ * CPUFREQ_CREATE_POLICY notification is sent only after
+ * successfully adding max_freq_req request.
+ */
+ blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
+ CPUFREQ_REMOVE_POLICY, policy);
+ dev_pm_qos_remove_request(policy->max_freq_req);
+ }
+
+ dev_pm_qos_remove_request(policy->min_freq_req);
+ kfree(policy->min_freq_req);
+
cpufreq_policy_put_kobj(policy);
free_cpumask_var(policy->real_cpus);
free_cpumask_var(policy->related_cpus);
@@ -1288,19 +1357,56 @@ static int cpufreq_online(unsigned int cpu)
cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
if (new_policy) {
- policy->user_policy.min = policy->min;
- policy->user_policy.max = policy->max;
+ struct device *dev = get_cpu_device(cpu);
for_each_cpu(j, policy->related_cpus) {
per_cpu(cpufreq_cpu_data, j) = policy;
add_cpu_dev_symlink(policy, j);
}
- } else {
- policy->min = policy->user_policy.min;
- policy->max = policy->user_policy.max;
+
+ policy->min_freq_req = kzalloc(2 * sizeof(*policy->min_freq_req),
+ GFP_KERNEL);
+ if (!policy->min_freq_req)
+ goto out_destroy_policy;
+
+ ret = dev_pm_qos_add_request(dev, policy->min_freq_req,
+ DEV_PM_QOS_MIN_FREQUENCY,
+ policy->min);
+ if (ret < 0) {
+ /*
+ * So we don't call dev_pm_qos_remove_request() for an
+ * uninitialized request.
+ */
+ kfree(policy->min_freq_req);
+ policy->min_freq_req = NULL;
+
+ dev_err(dev, "Failed to add min-freq constraint (%d)\n",
+ ret);
+ goto out_destroy_policy;
+ }
+
+ /*
+ * This must be initialized right here to avoid calling
+ * dev_pm_qos_remove_request() on uninitialized request in case
+ * of errors.
+ */
+ policy->max_freq_req = policy->min_freq_req + 1;
+
+ ret = dev_pm_qos_add_request(dev, policy->max_freq_req,
+ DEV_PM_QOS_MAX_FREQUENCY,
+ policy->max);
+ if (ret < 0) {
+ policy->max_freq_req = NULL;
+ dev_err(dev, "Failed to add max-freq constraint (%d)\n",
+ ret);
+ goto out_destroy_policy;
+ }
+
+ blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
+ CPUFREQ_CREATE_POLICY, policy);
}
- if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
+ if (cpufreq_driver->get && has_target()) {
policy->cur = cpufreq_driver->get(policy->cpu);
if (!policy->cur) {
pr_err("%s: ->get() failed\n", __func__);
@@ -1375,8 +1481,7 @@ static int cpufreq_online(unsigned int cpu)
if (cpufreq_driver->ready)
cpufreq_driver->ready(policy);
- if (IS_ENABLED(CONFIG_CPU_THERMAL) &&
- cpufreq_driver->flags & CPUFREQ_IS_COOLING_DEV)
+ if (cpufreq_thermal_control_enabled(cpufreq_driver))
policy->cdev = of_cpufreq_cooling_register(policy);
pr_debug("initialization complete\n");
@@ -1466,8 +1571,7 @@ static int cpufreq_offline(unsigned int cpu)
goto unlock;
}
- if (IS_ENABLED(CONFIG_CPU_THERMAL) &&
- cpufreq_driver->flags & CPUFREQ_IS_COOLING_DEV) {
+ if (cpufreq_thermal_control_enabled(cpufreq_driver)) {
cpufreq_cooling_unregister(policy->cdev);
policy->cdev = NULL;
}
@@ -1546,6 +1650,30 @@ static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
cpufreq_freq_transition_end(policy, &freqs, 0);
}
+static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update)
+{
+ unsigned int new_freq;
+
+ new_freq = cpufreq_driver->get(policy->cpu);
+ if (!new_freq)
+ return 0;
+
+ /*
+ * If fast frequency switching is used with the given policy, the check
+ * against policy->cur is pointless, so skip it in that case.
+ */
+ if (policy->fast_switch_enabled || !has_target())
+ return new_freq;
+
+ if (policy->cur != new_freq) {
+ cpufreq_out_of_sync(policy, new_freq);
+ if (update)
+ schedule_work(&policy->update);
+ }
+
+ return new_freq;
+}
+
/**
* cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
* @cpu: CPU number
@@ -1601,31 +1729,10 @@ EXPORT_SYMBOL(cpufreq_quick_get_max);
static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
{
- unsigned int ret_freq = 0;
-
if (unlikely(policy_is_inactive(policy)))
- return ret_freq;
-
- ret_freq = cpufreq_driver->get(policy->cpu);
-
- /*
- * If fast frequency switching is used with the given policy, the check
- * against policy->cur is pointless, so skip it in that case too.
- */
- if (policy->fast_switch_enabled)
- return ret_freq;
-
- if (ret_freq && policy->cur &&
- !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
- /* verify no discrepancy between actual and
- saved value exists */
- if (unlikely(ret_freq != policy->cur)) {
- cpufreq_out_of_sync(policy, ret_freq);
- schedule_work(&policy->update);
- }
- }
+ return 0;
- return ret_freq;
+ return cpufreq_verify_current_freq(policy, true);
}
/**
@@ -1652,24 +1759,6 @@ unsigned int cpufreq_get(unsigned int cpu)
}
EXPORT_SYMBOL(cpufreq_get);
-static unsigned int cpufreq_update_current_freq(struct cpufreq_policy *policy)
-{
- unsigned int new_freq;
-
- new_freq = cpufreq_driver->get(policy->cpu);
- if (!new_freq)
- return 0;
-
- if (!policy->cur) {
- pr_debug("cpufreq: Driver did not initialize current freq\n");
- policy->cur = new_freq;
- } else if (policy->cur != new_freq && has_target()) {
- cpufreq_out_of_sync(policy, new_freq);
- }
-
- return new_freq;
-}
-
static struct subsys_interface cpufreq_interface = {
.name = "cpufreq",
.subsys = &cpu_subsys,
@@ -1731,8 +1820,8 @@ void cpufreq_suspend(void)
}
if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy))
- pr_err("%s: Failed to suspend driver: %p\n", __func__,
- policy);
+ pr_err("%s: Failed to suspend driver: %s\n", __func__,
+ cpufreq_driver->name);
}
suspend:
@@ -2064,7 +2153,7 @@ int cpufreq_driver_target(struct cpufreq_policy *policy,
unsigned int target_freq,
unsigned int relation)
{
- int ret = -EINVAL;
+ int ret;
down_write(&policy->rwsem);
@@ -2150,8 +2239,8 @@ static int cpufreq_start_governor(struct cpufreq_policy *policy)
pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
- if (cpufreq_driver->get && !cpufreq_driver->setpolicy)
- cpufreq_update_current_freq(policy);
+ if (cpufreq_driver->get)
+ cpufreq_verify_current_freq(policy, false);
if (policy->governor->start) {
ret = policy->governor->start(policy);
@@ -2271,15 +2360,13 @@ EXPORT_SYMBOL(cpufreq_get_policy);
* @policy: Policy object to modify.
* @new_policy: New policy data.
*
- * Pass @new_policy to the cpufreq driver's ->verify() callback, run the
- * installed policy notifiers for it with the CPUFREQ_ADJUST value, pass it to
- * the driver's ->verify() callback again and run the notifiers for it again
- * with the CPUFREQ_NOTIFY value. Next, copy the min and max parameters
- * of @new_policy to @policy and either invoke the driver's ->setpolicy()
- * callback (if present) or carry out a governor update for @policy. That is,
- * run the current governor's ->limits() callback (if the governor field in
- * @new_policy points to the same object as the one in @policy) or replace the
- * governor for @policy with the new one stored in @new_policy.
+ * Pass @new_policy to the cpufreq driver's ->verify() callback. Next, copy the
+ * min and max parameters of @new_policy to @policy and either invoke the
+ * driver's ->setpolicy() callback (if present) or carry out a governor update
+ * for @policy. That is, run the current governor's ->limits() callback (if the
+ * governor field in @new_policy points to the same object as the one in
+ * @policy) or replace the governor for @policy with the new one stored in
+ * @new_policy.
*
* The cpuinfo part of @policy is not updated by this function.
*/
@@ -2287,6 +2374,7 @@ int cpufreq_set_policy(struct cpufreq_policy *policy,
struct cpufreq_policy *new_policy)
{
struct cpufreq_governor *old_gov;
+ struct device *cpu_dev = get_cpu_device(policy->cpu);
int ret;
pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
@@ -2295,33 +2383,17 @@ int cpufreq_set_policy(struct cpufreq_policy *policy,
memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
/*
- * This check works well when we store new min/max freq attributes,
- * because new_policy is a copy of policy with one field updated.
- */
- if (new_policy->min > new_policy->max)
- return -EINVAL;
+ * PM QoS framework collects all the requests from users and provide us
+ * the final aggregated value here.
+ */
+ new_policy->min = dev_pm_qos_read_value(cpu_dev, DEV_PM_QOS_MIN_FREQUENCY);
+ new_policy->max = dev_pm_qos_read_value(cpu_dev, DEV_PM_QOS_MAX_FREQUENCY);
/* verify the cpu speed can be set within this limit */
ret = cpufreq_driver->verify(new_policy);
if (ret)
return ret;
- /* adjust if necessary - all reasons */
- blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
- CPUFREQ_ADJUST, new_policy);
-
- /*
- * verify the cpu speed can be set within this limit, which might be
- * different to the first one
- */
- ret = cpufreq_driver->verify(new_policy);
- if (ret)
- return ret;
-
- /* notification of the new policy */
- blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
- CPUFREQ_NOTIFY, new_policy);
-
policy->min = new_policy->min;
policy->max = new_policy->max;
trace_cpu_frequency_limits(policy);
@@ -2384,15 +2456,13 @@ int cpufreq_set_policy(struct cpufreq_policy *policy,
* @cpu: CPU to re-evaluate the policy for.
*
* Update the current frequency for the cpufreq policy of @cpu and use
- * cpufreq_set_policy() to re-apply the min and max limits saved in the
- * user_policy sub-structure of that policy, which triggers the evaluation
- * of policy notifiers and the cpufreq driver's ->verify() callback for the
- * policy in question, among other things.
+ * cpufreq_set_policy() to re-apply the min and max limits, which triggers the
+ * evaluation of policy notifiers and the cpufreq driver's ->verify() callback
+ * for the policy in question, among other things.
*/
void cpufreq_update_policy(unsigned int cpu)
{
struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu);
- struct cpufreq_policy new_policy;
if (!policy)
return;
@@ -2401,16 +2471,11 @@ void cpufreq_update_policy(unsigned int cpu)
* BIOS might change freq behind our back
* -> ask driver for current freq and notify governors about a change
*/
- if (cpufreq_driver->get && !cpufreq_driver->setpolicy &&
- (cpufreq_suspended || WARN_ON(!cpufreq_update_current_freq(policy))))
+ if (cpufreq_driver->get && has_target() &&
+ (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false))))
goto unlock;
- pr_debug("updating policy for CPU %u\n", cpu);
- memcpy(&new_policy, policy, sizeof(*policy));
- new_policy.min = policy->user_policy.min;
- new_policy.max = policy->user_policy.max;
-
- cpufreq_set_policy(policy, &new_policy);
+ refresh_frequency_limits(policy);
unlock:
cpufreq_cpu_release(policy);
@@ -2453,10 +2518,9 @@ static int cpufreq_boost_set_sw(int state)
break;
}
- down_write(&policy->rwsem);
- policy->user_policy.max = policy->max;
- cpufreq_governor_limits(policy);
- up_write(&policy->rwsem);
+ ret = dev_pm_qos_update_request(policy->max_freq_req, policy->max);
+ if (ret < 0)
+ break;
}
return ret;
diff --git a/drivers/cpufreq/davinci-cpufreq.c b/drivers/cpufreq/davinci-cpufreq.c
index 3de48ae60c29..297d23cad8b5 100644
--- a/drivers/cpufreq/davinci-cpufreq.c
+++ b/drivers/cpufreq/davinci-cpufreq.c
@@ -90,7 +90,8 @@ static int davinci_cpu_init(struct cpufreq_policy *policy)
* Setting the latency to 2000 us to accommodate addition of drivers
* to pre/post change notification list.
*/
- return cpufreq_generic_init(policy, freq_table, 2000 * 1000);
+ cpufreq_generic_init(policy, freq_table, 2000 * 1000);
+ return 0;
}
static struct cpufreq_driver davinci_driver = {
diff --git a/drivers/cpufreq/imx-cpufreq-dt.c b/drivers/cpufreq/imx-cpufreq-dt.c
new file mode 100644
index 000000000000..35db14cf3102
--- /dev/null
+++ b/drivers/cpufreq/imx-cpufreq-dt.c
@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2019 NXP
+ */
+
+#include <linux/cpu.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/nvmem-consumer.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/slab.h>
+
+#define OCOTP_CFG3_SPEED_GRADE_SHIFT 8
+#define OCOTP_CFG3_SPEED_GRADE_MASK (0x3 << 8)
+#define IMX8MN_OCOTP_CFG3_SPEED_GRADE_MASK (0xf << 8)
+#define OCOTP_CFG3_MKT_SEGMENT_SHIFT 6
+#define OCOTP_CFG3_MKT_SEGMENT_MASK (0x3 << 6)
+
+/* cpufreq-dt device registered by imx-cpufreq-dt */
+static struct platform_device *cpufreq_dt_pdev;
+static struct opp_table *cpufreq_opp_table;
+
+static int imx_cpufreq_dt_probe(struct platform_device *pdev)
+{
+ struct device *cpu_dev = get_cpu_device(0);
+ u32 cell_value, supported_hw[2];
+ int speed_grade, mkt_segment;
+ int ret;
+
+ ret = nvmem_cell_read_u32(cpu_dev, "speed_grade", &cell_value);
+ if (ret)
+ return ret;
+
+ if (of_machine_is_compatible("fsl,imx8mn"))
+ speed_grade = (cell_value & IMX8MN_OCOTP_CFG3_SPEED_GRADE_MASK)
+ >> OCOTP_CFG3_SPEED_GRADE_SHIFT;
+ else
+ speed_grade = (cell_value & OCOTP_CFG3_SPEED_GRADE_MASK)
+ >> OCOTP_CFG3_SPEED_GRADE_SHIFT;
+ mkt_segment = (cell_value & OCOTP_CFG3_MKT_SEGMENT_MASK) >> OCOTP_CFG3_MKT_SEGMENT_SHIFT;
+
+ /*
+ * Early samples without fuses written report "0 0" which means
+ * consumer segment and minimum speed grading.
+ *
+ * According to datasheet minimum speed grading is not supported for
+ * consumer parts so clamp to 1 to avoid warning for "no OPPs"
+ *
+ * Applies to i.MX8M series SoCs.
+ */
+ if (mkt_segment == 0 && speed_grade == 0 && (
+ of_machine_is_compatible("fsl,imx8mm") ||
+ of_machine_is_compatible("fsl,imx8mn") ||
+ of_machine_is_compatible("fsl,imx8mq")))
+ speed_grade = 1;
+
+ supported_hw[0] = BIT(speed_grade);
+ supported_hw[1] = BIT(mkt_segment);
+ dev_info(&pdev->dev, "cpu speed grade %d mkt segment %d supported-hw %#x %#x\n",
+ speed_grade, mkt_segment, supported_hw[0], supported_hw[1]);
+
+ cpufreq_opp_table = dev_pm_opp_set_supported_hw(cpu_dev, supported_hw, 2);
+ if (IS_ERR(cpufreq_opp_table)) {
+ ret = PTR_ERR(cpufreq_opp_table);
+ dev_err(&pdev->dev, "Failed to set supported opp: %d\n", ret);
+ return ret;
+ }
+
+ cpufreq_dt_pdev = platform_device_register_data(
+ &pdev->dev, "cpufreq-dt", -1, NULL, 0);
+ if (IS_ERR(cpufreq_dt_pdev)) {
+ dev_pm_opp_put_supported_hw(cpufreq_opp_table);
+ ret = PTR_ERR(cpufreq_dt_pdev);
+ dev_err(&pdev->dev, "Failed to register cpufreq-dt: %d\n", ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+static int imx_cpufreq_dt_remove(struct platform_device *pdev)
+{
+ platform_device_unregister(cpufreq_dt_pdev);
+ dev_pm_opp_put_supported_hw(cpufreq_opp_table);
+
+ return 0;
+}
+
+static struct platform_driver imx_cpufreq_dt_driver = {
+ .probe = imx_cpufreq_dt_probe,
+ .remove = imx_cpufreq_dt_remove,
+ .driver = {
+ .name = "imx-cpufreq-dt",
+ },
+};
+module_platform_driver(imx_cpufreq_dt_driver);
+
+MODULE_ALIAS("platform:imx-cpufreq-dt");
+MODULE_DESCRIPTION("Freescale i.MX cpufreq speed grading driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c
index 47ccfa6b17b7..648a09a1778a 100644
--- a/drivers/cpufreq/imx6q-cpufreq.c
+++ b/drivers/cpufreq/imx6q-cpufreq.c
@@ -190,14 +190,12 @@ static int imx6q_set_target(struct cpufreq_policy *policy, unsigned int index)
static int imx6q_cpufreq_init(struct cpufreq_policy *policy)
{
- int ret;
-
policy->clk = clks[ARM].clk;
- ret = cpufreq_generic_init(policy, freq_table, transition_latency);
+ cpufreq_generic_init(policy, freq_table, transition_latency);
policy->suspend_freq = max_freq;
dev_pm_opp_of_register_em(policy->cpus);
- return ret;
+ return 0;
}
static struct cpufreq_driver imx6q_cpufreq_driver = {
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index f2ff5de988c1..9f02de9a1b47 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -24,6 +24,7 @@
#include <linux/fs.h>
#include <linux/acpi.h>
#include <linux/vmalloc.h>
+#include <linux/pm_qos.h>
#include <trace/events/power.h>
#include <asm/div64.h>
@@ -898,7 +899,6 @@ static void intel_pstate_update_policies(void)
static void intel_pstate_update_max_freq(unsigned int cpu)
{
struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu);
- struct cpufreq_policy new_policy;
struct cpudata *cpudata;
if (!policy)
@@ -908,11 +908,7 @@ static void intel_pstate_update_max_freq(unsigned int cpu)
policy->cpuinfo.max_freq = global.turbo_disabled_mf ?
cpudata->pstate.max_freq : cpudata->pstate.turbo_freq;
- memcpy(&new_policy, policy, sizeof(*policy));
- new_policy.max = min(policy->user_policy.max, policy->cpuinfo.max_freq);
- new_policy.min = min(policy->user_policy.min, new_policy.max);
-
- cpufreq_set_policy(policy, &new_policy);
+ refresh_frequency_limits(policy);
cpufreq_cpu_release(policy);
}
@@ -1090,6 +1086,47 @@ static ssize_t store_no_turbo(struct kobject *a, struct kobj_attribute *b,
return count;
}
+static struct cpufreq_driver intel_pstate;
+
+static void update_qos_request(enum dev_pm_qos_req_type type)
+{
+ int max_state, turbo_max, freq, i, perf_pct;
+ struct dev_pm_qos_request *req;
+ struct cpufreq_policy *policy;
+
+ for_each_possible_cpu(i) {
+ struct cpudata *cpu = all_cpu_data[i];
+
+ policy = cpufreq_cpu_get(i);
+ if (!policy)
+ continue;
+
+ req = policy->driver_data;
+ cpufreq_cpu_put(policy);
+
+ if (!req)
+ continue;
+
+ if (hwp_active)
+ intel_pstate_get_hwp_max(i, &turbo_max, &max_state);
+ else
+ turbo_max = cpu->pstate.turbo_pstate;
+
+ if (type == DEV_PM_QOS_MIN_FREQUENCY) {
+ perf_pct = global.min_perf_pct;
+ } else {
+ req++;
+ perf_pct = global.max_perf_pct;
+ }
+
+ freq = DIV_ROUND_UP(turbo_max * perf_pct, 100);
+ freq *= cpu->pstate.scaling;
+
+ if (dev_pm_qos_update_request(req, freq) < 0)
+ pr_warn("Failed to update freq constraint: CPU%d\n", i);
+ }
+}
+
static ssize_t store_max_perf_pct(struct kobject *a, struct kobj_attribute *b,
const char *buf, size_t count)
{
@@ -1113,7 +1150,10 @@ static ssize_t store_max_perf_pct(struct kobject *a, struct kobj_attribute *b,
mutex_unlock(&intel_pstate_limits_lock);
- intel_pstate_update_policies();
+ if (intel_pstate_driver == &intel_pstate)
+ intel_pstate_update_policies();
+ else
+ update_qos_request(DEV_PM_QOS_MAX_FREQUENCY);
mutex_unlock(&intel_pstate_driver_lock);
@@ -1144,7 +1184,10 @@ static ssize_t store_min_perf_pct(struct kobject *a, struct kobj_attribute *b,
mutex_unlock(&intel_pstate_limits_lock);
- intel_pstate_update_policies();
+ if (intel_pstate_driver == &intel_pstate)
+ intel_pstate_update_policies();
+ else
+ update_qos_request(DEV_PM_QOS_MIN_FREQUENCY);
mutex_unlock(&intel_pstate_driver_lock);
@@ -1872,22 +1915,22 @@ static const struct pstate_funcs knl_funcs = {
(unsigned long)&policy }
static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
- ICPU(INTEL_FAM6_SANDYBRIDGE, core_funcs),
+ ICPU(INTEL_FAM6_SANDYBRIDGE, core_funcs),
ICPU(INTEL_FAM6_SANDYBRIDGE_X, core_funcs),
ICPU(INTEL_FAM6_ATOM_SILVERMONT, silvermont_funcs),
ICPU(INTEL_FAM6_IVYBRIDGE, core_funcs),
- ICPU(INTEL_FAM6_HASWELL_CORE, core_funcs),
- ICPU(INTEL_FAM6_BROADWELL_CORE, core_funcs),
+ ICPU(INTEL_FAM6_HASWELL, core_funcs),
+ ICPU(INTEL_FAM6_BROADWELL, core_funcs),
ICPU(INTEL_FAM6_IVYBRIDGE_X, core_funcs),
ICPU(INTEL_FAM6_HASWELL_X, core_funcs),
- ICPU(INTEL_FAM6_HASWELL_ULT, core_funcs),
- ICPU(INTEL_FAM6_HASWELL_GT3E, core_funcs),
- ICPU(INTEL_FAM6_BROADWELL_GT3E, core_funcs),
+ ICPU(INTEL_FAM6_HASWELL_L, core_funcs),
+ ICPU(INTEL_FAM6_HASWELL_G, core_funcs),
+ ICPU(INTEL_FAM6_BROADWELL_G, core_funcs),
ICPU(INTEL_FAM6_ATOM_AIRMONT, airmont_funcs),
- ICPU(INTEL_FAM6_SKYLAKE_MOBILE, core_funcs),
+ ICPU(INTEL_FAM6_SKYLAKE_L, core_funcs),
ICPU(INTEL_FAM6_BROADWELL_X, core_funcs),
- ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_funcs),
- ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_funcs),
+ ICPU(INTEL_FAM6_SKYLAKE, core_funcs),
+ ICPU(INTEL_FAM6_BROADWELL_D, core_funcs),
ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_funcs),
ICPU(INTEL_FAM6_XEON_PHI_KNM, knl_funcs),
ICPU(INTEL_FAM6_ATOM_GOLDMONT, core_funcs),
@@ -1898,20 +1941,20 @@ static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = {
- ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_funcs),
+ ICPU(INTEL_FAM6_BROADWELL_D, core_funcs),
ICPU(INTEL_FAM6_BROADWELL_X, core_funcs),
ICPU(INTEL_FAM6_SKYLAKE_X, core_funcs),
{}
};
static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = {
- ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, core_funcs),
+ ICPU(INTEL_FAM6_KABYLAKE, core_funcs),
{}
};
static const struct x86_cpu_id intel_pstate_hwp_boost_ids[] = {
ICPU(INTEL_FAM6_SKYLAKE_X, core_funcs),
- ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_funcs),
+ ICPU(INTEL_FAM6_SKYLAKE, core_funcs),
{}
};
@@ -2337,8 +2380,16 @@ static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy,
static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy)
{
- int ret = __intel_pstate_cpu_init(policy);
+ int max_state, turbo_max, min_freq, max_freq, ret;
+ struct dev_pm_qos_request *req;
+ struct cpudata *cpu;
+ struct device *dev;
+ dev = get_cpu_device(policy->cpu);
+ if (!dev)
+ return -ENODEV;
+
+ ret = __intel_pstate_cpu_init(policy);
if (ret)
return ret;
@@ -2347,7 +2398,63 @@ static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy)
/* This reflects the intel_pstate_get_cpu_pstates() setting. */
policy->cur = policy->cpuinfo.min_freq;
+ req = kcalloc(2, sizeof(*req), GFP_KERNEL);
+ if (!req) {
+ ret = -ENOMEM;
+ goto pstate_exit;
+ }
+
+ cpu = all_cpu_data[policy->cpu];
+
+ if (hwp_active)
+ intel_pstate_get_hwp_max(policy->cpu, &turbo_max, &max_state);
+ else
+ turbo_max = cpu->pstate.turbo_pstate;
+
+ min_freq = DIV_ROUND_UP(turbo_max * global.min_perf_pct, 100);
+ min_freq *= cpu->pstate.scaling;
+ max_freq = DIV_ROUND_UP(turbo_max * global.max_perf_pct, 100);
+ max_freq *= cpu->pstate.scaling;
+
+ ret = dev_pm_qos_add_request(dev, req, DEV_PM_QOS_MIN_FREQUENCY,
+ min_freq);
+ if (ret < 0) {
+ dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret);
+ goto free_req;
+ }
+
+ ret = dev_pm_qos_add_request(dev, req + 1, DEV_PM_QOS_MAX_FREQUENCY,
+ max_freq);
+ if (ret < 0) {
+ dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret);
+ goto remove_min_req;
+ }
+
+ policy->driver_data = req;
+
return 0;
+
+remove_min_req:
+ dev_pm_qos_remove_request(req);
+free_req:
+ kfree(req);
+pstate_exit:
+ intel_pstate_exit_perf_limits(policy);
+
+ return ret;
+}
+
+static int intel_cpufreq_cpu_exit(struct cpufreq_policy *policy)
+{
+ struct dev_pm_qos_request *req;
+
+ req = policy->driver_data;
+
+ dev_pm_qos_remove_request(req + 1);
+ dev_pm_qos_remove_request(req);
+ kfree(req);
+
+ return intel_pstate_cpu_exit(policy);
}
static struct cpufreq_driver intel_cpufreq = {
@@ -2356,7 +2463,7 @@ static struct cpufreq_driver intel_cpufreq = {
.target = intel_cpufreq_target,
.fast_switch = intel_cpufreq_fast_switch,
.init = intel_cpufreq_cpu_init,
- .exit = intel_pstate_cpu_exit,
+ .exit = intel_cpufreq_cpu_exit,
.stop_cpu = intel_cpufreq_stop_cpu,
.update_limits = intel_pstate_update_limits,
.name = "intel_cpufreq",
@@ -2629,7 +2736,7 @@ static inline void intel_pstate_request_control_from_smm(void) {}
static const struct x86_cpu_id hwp_support_ids[] __initconst = {
ICPU_HWP(INTEL_FAM6_BROADWELL_X, INTEL_PSTATE_HWP_BROADWELL),
- ICPU_HWP(INTEL_FAM6_BROADWELL_XEON_D, INTEL_PSTATE_HWP_BROADWELL),
+ ICPU_HWP(INTEL_FAM6_BROADWELL_D, INTEL_PSTATE_HWP_BROADWELL),
ICPU_HWP(X86_MODEL_ANY, 0),
{}
};
diff --git a/drivers/cpufreq/kirkwood-cpufreq.c b/drivers/cpufreq/kirkwood-cpufreq.c
index 7ab564c1f7ae..cb74bdc5baaa 100644
--- a/drivers/cpufreq/kirkwood-cpufreq.c
+++ b/drivers/cpufreq/kirkwood-cpufreq.c
@@ -85,7 +85,8 @@ static int kirkwood_cpufreq_target(struct cpufreq_policy *policy,
/* Module init and exit code */
static int kirkwood_cpufreq_cpu_init(struct cpufreq_policy *policy)
{
- return cpufreq_generic_init(policy, kirkwood_freq_table, 5000);
+ cpufreq_generic_init(policy, kirkwood_freq_table, 5000);
+ return 0;
}
static struct cpufreq_driver kirkwood_cpufreq_driver = {
diff --git a/drivers/cpufreq/loongson1-cpufreq.c b/drivers/cpufreq/loongson1-cpufreq.c
index 21c9ce8526c0..0ea88778882a 100644
--- a/drivers/cpufreq/loongson1-cpufreq.c
+++ b/drivers/cpufreq/loongson1-cpufreq.c
@@ -81,7 +81,7 @@ static int ls1x_cpufreq_init(struct cpufreq_policy *policy)
struct device *cpu_dev = get_cpu_device(policy->cpu);
struct cpufreq_frequency_table *freq_tbl;
unsigned int pll_freq, freq;
- int steps, i, ret;
+ int steps, i;
pll_freq = clk_get_rate(cpufreq->pll_clk) / 1000;
@@ -103,11 +103,9 @@ static int ls1x_cpufreq_init(struct cpufreq_policy *policy)
freq_tbl[i].frequency = CPUFREQ_TABLE_END;
policy->clk = cpufreq->clk;
- ret = cpufreq_generic_init(policy, freq_tbl, 0);
- if (ret)
- kfree(freq_tbl);
+ cpufreq_generic_init(policy, freq_tbl, 0);
- return ret;
+ return 0;
}
static int ls1x_cpufreq_exit(struct cpufreq_policy *policy)
diff --git a/drivers/cpufreq/loongson2_cpufreq.c b/drivers/cpufreq/loongson2_cpufreq.c
index da344696beed..890813e0bb76 100644
--- a/drivers/cpufreq/loongson2_cpufreq.c
+++ b/drivers/cpufreq/loongson2_cpufreq.c
@@ -95,7 +95,8 @@ static int loongson2_cpufreq_cpu_init(struct cpufreq_policy *policy)
}
policy->clk = cpuclk;
- return cpufreq_generic_init(policy, &loongson2_clockmod_table[0], 0);
+ cpufreq_generic_init(policy, &loongson2_clockmod_table[0], 0);
+ return 0;
}
static int loongson2_cpufreq_exit(struct cpufreq_policy *policy)
diff --git a/drivers/cpufreq/maple-cpufreq.c b/drivers/cpufreq/maple-cpufreq.c
index f5220b3d4ec5..28d346062166 100644
--- a/drivers/cpufreq/maple-cpufreq.c
+++ b/drivers/cpufreq/maple-cpufreq.c
@@ -140,7 +140,8 @@ static unsigned int maple_cpufreq_get_speed(unsigned int cpu)
static int maple_cpufreq_cpu_init(struct cpufreq_policy *policy)
{
- return cpufreq_generic_init(policy, maple_cpu_freqs, 12000);
+ cpufreq_generic_init(policy, maple_cpu_freqs, 12000);
+ return 0;
}
static struct cpufreq_driver maple_cpufreq_driver = {
diff --git a/drivers/cpufreq/mediatek-cpufreq.c b/drivers/cpufreq/mediatek-cpufreq.c
index f14f3a85f2f7..0c98dd08273d 100644
--- a/drivers/cpufreq/mediatek-cpufreq.c
+++ b/drivers/cpufreq/mediatek-cpufreq.c
@@ -338,7 +338,7 @@ static int mtk_cpu_dvfs_info_init(struct mtk_cpu_dvfs_info *info, int cpu)
goto out_free_resources;
}
- proc_reg = regulator_get_exclusive(cpu_dev, "proc");
+ proc_reg = regulator_get_optional(cpu_dev, "proc");
if (IS_ERR(proc_reg)) {
if (PTR_ERR(proc_reg) == -EPROBE_DEFER)
pr_warn("proc regulator for cpu%d not ready, retry.\n",
@@ -535,6 +535,8 @@ static const struct of_device_id mtk_cpufreq_machines[] __initconst = {
{ .compatible = "mediatek,mt817x", },
{ .compatible = "mediatek,mt8173", },
{ .compatible = "mediatek,mt8176", },
+ { .compatible = "mediatek,mt8183", },
+ { .compatible = "mediatek,mt8516", },
{ }
};
diff --git a/drivers/cpufreq/omap-cpufreq.c b/drivers/cpufreq/omap-cpufreq.c
index 29643f06a3c3..8d14b42a8c6f 100644
--- a/drivers/cpufreq/omap-cpufreq.c
+++ b/drivers/cpufreq/omap-cpufreq.c
@@ -122,23 +122,18 @@ static int omap_cpu_init(struct cpufreq_policy *policy)
dev_err(mpu_dev,
"%s: cpu%d: failed creating freq table[%d]\n",
__func__, policy->cpu, result);
- goto fail;
+ clk_put(policy->clk);
+ return result;
}
}
atomic_inc_return(&freq_table_users);
/* FIXME: what's the actual transition time? */
- result = cpufreq_generic_init(policy, freq_table, 300 * 1000);
- if (!result) {
- dev_pm_opp_of_register_em(policy->cpus);
- return 0;
- }
+ cpufreq_generic_init(policy, freq_table, 300 * 1000);
+ dev_pm_opp_of_register_em(policy->cpus);
- freq_table_free();
-fail:
- clk_put(policy->clk);
- return result;
+ return 0;
}
static int omap_cpu_exit(struct cpufreq_policy *policy)
diff --git a/drivers/cpufreq/pasemi-cpufreq.c b/drivers/cpufreq/pasemi-cpufreq.c
index 6b1e4abe3248..c66f566a854c 100644
--- a/drivers/cpufreq/pasemi-cpufreq.c
+++ b/drivers/cpufreq/pasemi-cpufreq.c
@@ -131,10 +131,18 @@ static int pas_cpufreq_cpu_init(struct cpufreq_policy *policy)
int err = -ENODEV;
cpu = of_get_cpu_node(policy->cpu, NULL);
+ if (!cpu)
+ goto out;
+ max_freqp = of_get_property(cpu, "clock-frequency", NULL);
of_node_put(cpu);
- if (!cpu)
+ if (!max_freqp) {
+ err = -EINVAL;
goto out;
+ }
+
+ /* we need the freq in kHz */
+ max_freq = *max_freqp / 1000;
dn = of_find_compatible_node(NULL, NULL, "1682m-sdc");
if (!dn)
@@ -171,16 +179,6 @@ static int pas_cpufreq_cpu_init(struct cpufreq_policy *policy)
}
pr_debug("init cpufreq on CPU %d\n", policy->cpu);
-
- max_freqp = of_get_property(cpu, "clock-frequency", NULL);
- if (!max_freqp) {
- err = -EINVAL;
- goto out_unmap_sdcpwr;
- }
-
- /* we need the freq in kHz */
- max_freq = *max_freqp / 1000;
-
pr_debug("max clock-frequency is at %u kHz\n", max_freq);
pr_debug("initializing frequency table\n");
@@ -196,10 +194,8 @@ static int pas_cpufreq_cpu_init(struct cpufreq_policy *policy)
policy->cur = pas_freqs[cur_astate].frequency;
ppc_proc_freq = policy->cur * 1000ul;
- return cpufreq_generic_init(policy, pas_freqs, get_gizmo_latency());
-
-out_unmap_sdcpwr:
- iounmap(sdcpwr_mapbase);
+ cpufreq_generic_init(policy, pas_freqs, get_gizmo_latency());
+ return 0;
out_unmap_sdcasr:
iounmap(sdcasr_mapbase);
diff --git a/drivers/cpufreq/pcc-cpufreq.c b/drivers/cpufreq/pcc-cpufreq.c
index 1e5e64643c3a..fdc767fdbe6a 100644
--- a/drivers/cpufreq/pcc-cpufreq.c
+++ b/drivers/cpufreq/pcc-cpufreq.c
@@ -582,10 +582,10 @@ static int __init pcc_cpufreq_init(void)
/* Skip initialization if another cpufreq driver is there. */
if (cpufreq_get_current_driver())
- return 0;
+ return -EEXIST;
if (acpi_disabled)
- return 0;
+ return -ENODEV;
ret = pcc_cpufreq_probe();
if (ret) {
diff --git a/drivers/cpufreq/pmac32-cpufreq.c b/drivers/cpufreq/pmac32-cpufreq.c
index 650104d729f3..73621bc11976 100644
--- a/drivers/cpufreq/pmac32-cpufreq.c
+++ b/drivers/cpufreq/pmac32-cpufreq.c
@@ -372,7 +372,8 @@ static int pmac_cpufreq_target( struct cpufreq_policy *policy,
static int pmac_cpufreq_cpu_init(struct cpufreq_policy *policy)
{
- return cpufreq_generic_init(policy, pmac_cpu_freqs, transition_latency);
+ cpufreq_generic_init(policy, pmac_cpu_freqs, transition_latency);
+ return 0;
}
static u32 read_gpio(struct device_node *np)
diff --git a/drivers/cpufreq/pmac64-cpufreq.c b/drivers/cpufreq/pmac64-cpufreq.c
index 1af3492a000d..d7542a106e6b 100644
--- a/drivers/cpufreq/pmac64-cpufreq.c
+++ b/drivers/cpufreq/pmac64-cpufreq.c
@@ -321,7 +321,8 @@ static unsigned int g5_cpufreq_get_speed(unsigned int cpu)
static int g5_cpufreq_cpu_init(struct cpufreq_policy *policy)
{
- return cpufreq_generic_init(policy, g5_cpu_freqs, transition_latency);
+ cpufreq_generic_init(policy, g5_cpu_freqs, transition_latency);
+ return 0;
}
static struct cpufreq_driver g5_cpufreq_driver = {
diff --git a/drivers/cpufreq/ppc_cbe_cpufreq.c b/drivers/cpufreq/ppc_cbe_cpufreq.c
index b83f36febf03..c58abb4cca3a 100644
--- a/drivers/cpufreq/ppc_cbe_cpufreq.c
+++ b/drivers/cpufreq/ppc_cbe_cpufreq.c
@@ -110,6 +110,13 @@ static int cbe_cpufreq_cpu_init(struct cpufreq_policy *policy)
#endif
policy->freq_table = cbe_freqs;
+ cbe_cpufreq_pmi_policy_init(policy);
+ return 0;
+}
+
+static int cbe_cpufreq_cpu_exit(struct cpufreq_policy *policy)
+{
+ cbe_cpufreq_pmi_policy_exit(policy);
return 0;
}
@@ -129,6 +136,7 @@ static struct cpufreq_driver cbe_cpufreq_driver = {
.verify = cpufreq_generic_frequency_table_verify,
.target_index = cbe_cpufreq_target,
.init = cbe_cpufreq_cpu_init,
+ .exit = cbe_cpufreq_cpu_exit,
.name = "cbe-cpufreq",
.flags = CPUFREQ_CONST_LOOPS,
};
@@ -139,15 +147,24 @@ static struct cpufreq_driver cbe_cpufreq_driver = {
static int __init cbe_cpufreq_init(void)
{
+ int ret;
+
if (!machine_is(cell))
return -ENODEV;
- return cpufreq_register_driver(&cbe_cpufreq_driver);
+ cbe_cpufreq_pmi_init();
+
+ ret = cpufreq_register_driver(&cbe_cpufreq_driver);
+ if (ret)
+ cbe_cpufreq_pmi_exit();
+
+ return ret;
}
static void __exit cbe_cpufreq_exit(void)
{
cpufreq_unregister_driver(&cbe_cpufreq_driver);
+ cbe_cpufreq_pmi_exit();
}
module_init(cbe_cpufreq_init);
diff --git a/drivers/cpufreq/ppc_cbe_cpufreq.h b/drivers/cpufreq/ppc_cbe_cpufreq.h
index 9d973519d669..00cd8633b0d9 100644
--- a/drivers/cpufreq/ppc_cbe_cpufreq.h
+++ b/drivers/cpufreq/ppc_cbe_cpufreq.h
@@ -20,6 +20,14 @@ int cbe_cpufreq_set_pmode_pmi(int cpu, unsigned int pmode);
#if IS_ENABLED(CONFIG_CPU_FREQ_CBE_PMI)
extern bool cbe_cpufreq_has_pmi;
+void cbe_cpufreq_pmi_policy_init(struct cpufreq_policy *policy);
+void cbe_cpufreq_pmi_policy_exit(struct cpufreq_policy *policy);
+void cbe_cpufreq_pmi_init(void);
+void cbe_cpufreq_pmi_exit(void);
#else
#define cbe_cpufreq_has_pmi (0)
+static inline void cbe_cpufreq_pmi_policy_init(struct cpufreq_policy *policy) {}
+static inline void cbe_cpufreq_pmi_policy_exit(struct cpufreq_policy *policy) {}
+static inline void cbe_cpufreq_pmi_init(void) {}
+static inline void cbe_cpufreq_pmi_exit(void) {}
#endif
diff --git a/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c b/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c
index 97c8ee4614b7..bc9dd30395c4 100644
--- a/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c
+++ b/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c
@@ -12,6 +12,7 @@
#include <linux/timer.h>
#include <linux/init.h>
#include <linux/of_platform.h>
+#include <linux/pm_qos.h>
#include <asm/processor.h>
#include <asm/prom.h>
@@ -24,8 +25,6 @@
#include "ppc_cbe_cpufreq.h"
-static u8 pmi_slow_mode_limit[MAX_CBE];
-
bool cbe_cpufreq_has_pmi = false;
EXPORT_SYMBOL_GPL(cbe_cpufreq_has_pmi);
@@ -65,64 +64,89 @@ EXPORT_SYMBOL_GPL(cbe_cpufreq_set_pmode_pmi);
static void cbe_cpufreq_handle_pmi(pmi_message_t pmi_msg)
{
+ struct cpufreq_policy *policy;
+ struct dev_pm_qos_request *req;
u8 node, slow_mode;
+ int cpu, ret;
BUG_ON(pmi_msg.type != PMI_TYPE_FREQ_CHANGE);
node = pmi_msg.data1;
slow_mode = pmi_msg.data2;
- pmi_slow_mode_limit[node] = slow_mode;
+ cpu = cbe_node_to_cpu(node);
pr_debug("cbe_handle_pmi: node: %d max_freq: %d\n", node, slow_mode);
-}
-
-static int pmi_notifier(struct notifier_block *nb,
- unsigned long event, void *data)
-{
- struct cpufreq_policy *policy = data;
- struct cpufreq_frequency_table *cbe_freqs = policy->freq_table;
- u8 node;
-
- /* Should this really be called for CPUFREQ_ADJUST and CPUFREQ_NOTIFY
- * policy events?)
- */
- node = cbe_cpu_to_node(policy->cpu);
-
- pr_debug("got notified, event=%lu, node=%u\n", event, node);
- if (pmi_slow_mode_limit[node] != 0) {
- pr_debug("limiting node %d to slow mode %d\n",
- node, pmi_slow_mode_limit[node]);
+ policy = cpufreq_cpu_get(cpu);
+ if (!policy) {
+ pr_warn("cpufreq policy not found cpu%d\n", cpu);
+ return;
+ }
- cpufreq_verify_within_limits(policy, 0,
+ req = policy->driver_data;
- cbe_freqs[pmi_slow_mode_limit[node]].frequency);
- }
+ ret = dev_pm_qos_update_request(req,
+ policy->freq_table[slow_mode].frequency);
+ if (ret < 0)
+ pr_warn("Failed to update freq constraint: %d\n", ret);
+ else
+ pr_debug("limiting node %d to slow mode %d\n", node, slow_mode);
- return 0;
+ cpufreq_cpu_put(policy);
}
-static struct notifier_block pmi_notifier_block = {
- .notifier_call = pmi_notifier,
-};
-
static struct pmi_handler cbe_pmi_handler = {
.type = PMI_TYPE_FREQ_CHANGE,
.handle_pmi_message = cbe_cpufreq_handle_pmi,
};
+void cbe_cpufreq_pmi_policy_init(struct cpufreq_policy *policy)
+{
+ struct dev_pm_qos_request *req;
+ int ret;
+
+ if (!cbe_cpufreq_has_pmi)
+ return;
+
+ req = kzalloc(sizeof(*req), GFP_KERNEL);
+ if (!req)
+ return;
+
+ ret = dev_pm_qos_add_request(get_cpu_device(policy->cpu), req,
+ DEV_PM_QOS_MAX_FREQUENCY,
+ policy->freq_table[0].frequency);
+ if (ret < 0) {
+ pr_err("Failed to add freq constraint (%d)\n", ret);
+ kfree(req);
+ return;
+ }
+ policy->driver_data = req;
+}
+EXPORT_SYMBOL_GPL(cbe_cpufreq_pmi_policy_init);
-static int __init cbe_cpufreq_pmi_init(void)
+void cbe_cpufreq_pmi_policy_exit(struct cpufreq_policy *policy)
{
- cbe_cpufreq_has_pmi = pmi_register_handler(&cbe_pmi_handler) == 0;
+ struct dev_pm_qos_request *req = policy->driver_data;
- if (!cbe_cpufreq_has_pmi)
- return -ENODEV;
+ if (cbe_cpufreq_has_pmi) {
+ dev_pm_qos_remove_request(req);
+ kfree(req);
+ }
+}
+EXPORT_SYMBOL_GPL(cbe_cpufreq_pmi_policy_exit);
- cpufreq_register_notifier(&pmi_notifier_block, CPUFREQ_POLICY_NOTIFIER);
+void cbe_cpufreq_pmi_init(void)
+{
+ if (!pmi_register_handler(&cbe_pmi_handler))
+ cbe_cpufreq_has_pmi = true;
+}
+EXPORT_SYMBOL_GPL(cbe_cpufreq_pmi_init);
- return 0;
+void cbe_cpufreq_pmi_exit(void)
+{
+ pmi_unregister_handler(&cbe_pmi_handler);
+ cbe_cpufreq_has_pmi = false;
}
-device_initcall(cbe_cpufreq_pmi_init);
+EXPORT_SYMBOL_GPL(cbe_cpufreq_pmi_exit);
diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c
index 4b0b50403901..a9ae2f84a4ef 100644
--- a/drivers/cpufreq/qcom-cpufreq-hw.c
+++ b/drivers/cpufreq/qcom-cpufreq-hw.c
@@ -20,6 +20,7 @@
#define LUT_VOLT GENMASK(11, 0)
#define LUT_ROW_SIZE 32
#define CLK_HW_DIV 2
+#define LUT_TURBO_IND 1
/* Register offsets */
#define REG_ENABLE 0x0
@@ -34,9 +35,12 @@ static int qcom_cpufreq_hw_target_index(struct cpufreq_policy *policy,
unsigned int index)
{
void __iomem *perf_state_reg = policy->driver_data;
+ unsigned long freq = policy->freq_table[index].frequency;
writel_relaxed(index, perf_state_reg);
+ arch_set_freq_scale(policy->related_cpus, freq,
+ policy->cpuinfo.max_freq);
return 0;
}
@@ -63,6 +67,7 @@ static unsigned int qcom_cpufreq_hw_fast_switch(struct cpufreq_policy *policy,
{
void __iomem *perf_state_reg = policy->driver_data;
int index;
+ unsigned long freq;
index = policy->cached_resolved_idx;
if (index < 0)
@@ -70,16 +75,19 @@ static unsigned int qcom_cpufreq_hw_fast_switch(struct cpufreq_policy *policy,
writel_relaxed(index, perf_state_reg);
- return policy->freq_table[index].frequency;
+ freq = policy->freq_table[index].frequency;
+ arch_set_freq_scale(policy->related_cpus, freq,
+ policy->cpuinfo.max_freq);
+
+ return freq;
}
static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev,
struct cpufreq_policy *policy,
void __iomem *base)
{
- u32 data, src, lval, i, core_count, prev_cc = 0, prev_freq = 0, freq;
+ u32 data, src, lval, i, core_count, prev_freq = 0, freq;
u32 volt;
- unsigned int max_cores = cpumask_weight(policy->cpus);
struct cpufreq_frequency_table *table;
table = kcalloc(LUT_MAX_ENTRIES + 1, sizeof(*table), GFP_KERNEL);
@@ -102,12 +110,12 @@ static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev,
else
freq = cpu_hw_rate / 1000;
- if (freq != prev_freq && core_count == max_cores) {
+ if (freq != prev_freq && core_count != LUT_TURBO_IND) {
table[i].frequency = freq;
dev_pm_opp_add(cpu_dev, freq * 1000, volt);
dev_dbg(cpu_dev, "index=%d freq=%d, core_count %d\n", i,
freq, core_count);
- } else {
+ } else if (core_count == LUT_TURBO_IND) {
table[i].frequency = CPUFREQ_ENTRY_INVALID;
}
@@ -115,14 +123,14 @@ static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev,
* Two of the same frequencies with the same core counts means
* end of table
*/
- if (i > 0 && prev_freq == freq && prev_cc == core_count) {
+ if (i > 0 && prev_freq == freq) {
struct cpufreq_frequency_table *prev = &table[i - 1];
/*
* Only treat the last frequency that might be a boost
* as the boost frequency
*/
- if (prev_cc != max_cores) {
+ if (prev->frequency == CPUFREQ_ENTRY_INVALID) {
prev->frequency = prev_freq;
prev->flags = CPUFREQ_BOOST_FREQ;
dev_pm_opp_add(cpu_dev, prev_freq * 1000, volt);
@@ -131,7 +139,6 @@ static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev,
break;
}
- prev_cc = core_count;
prev_freq = freq;
}
diff --git a/drivers/cpufreq/qcom-cpufreq-kryo.c b/drivers/cpufreq/qcom-cpufreq-kryo.c
deleted file mode 100644
index dd64dcf89c74..000000000000
--- a/drivers/cpufreq/qcom-cpufreq-kryo.c
+++ /dev/null
@@ -1,249 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (c) 2018, The Linux Foundation. All rights reserved.
- */
-
-/*
- * In Certain QCOM SoCs like apq8096 and msm8996 that have KRYO processors,
- * the CPU frequency subset and voltage value of each OPP varies
- * based on the silicon variant in use. Qualcomm Process Voltage Scaling Tables
- * defines the voltage and frequency value based on the msm-id in SMEM
- * and speedbin blown in the efuse combination.
- * The qcom-cpufreq-kryo driver reads the msm-id and efuse value from the SoC
- * to provide the OPP framework with required information.
- * This is used to determine the voltage and frequency value for each OPP of
- * operating-points-v2 table when it is parsed by the OPP framework.
- */
-
-#include <linux/cpu.h>
-#include <linux/err.h>
-#include <linux/init.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/nvmem-consumer.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
-#include <linux/pm_opp.h>
-#include <linux/slab.h>
-#include <linux/soc/qcom/smem.h>
-
-#define MSM_ID_SMEM 137
-
-enum _msm_id {
- MSM8996V3 = 0xF6ul,
- APQ8096V3 = 0x123ul,
- MSM8996SG = 0x131ul,
- APQ8096SG = 0x138ul,
-};
-
-enum _msm8996_version {
- MSM8996_V3,
- MSM8996_SG,
- NUM_OF_MSM8996_VERSIONS,
-};
-
-static struct platform_device *cpufreq_dt_pdev, *kryo_cpufreq_pdev;
-
-static enum _msm8996_version qcom_cpufreq_kryo_get_msm_id(void)
-{
- size_t len;
- u32 *msm_id;
- enum _msm8996_version version;
-
- msm_id = qcom_smem_get(QCOM_SMEM_HOST_ANY, MSM_ID_SMEM, &len);
- if (IS_ERR(msm_id))
- return NUM_OF_MSM8996_VERSIONS;
-
- /* The first 4 bytes are format, next to them is the actual msm-id */
- msm_id++;
-
- switch ((enum _msm_id)*msm_id) {
- case MSM8996V3:
- case APQ8096V3:
- version = MSM8996_V3;
- break;
- case MSM8996SG:
- case APQ8096SG:
- version = MSM8996_SG;
- break;
- default:
- version = NUM_OF_MSM8996_VERSIONS;
- }
-
- return version;
-}
-
-static int qcom_cpufreq_kryo_probe(struct platform_device *pdev)
-{
- struct opp_table **opp_tables;
- enum _msm8996_version msm8996_version;
- struct nvmem_cell *speedbin_nvmem;
- struct device_node *np;
- struct device *cpu_dev;
- unsigned cpu;
- u8 *speedbin;
- u32 versions;
- size_t len;
- int ret;
-
- cpu_dev = get_cpu_device(0);
- if (!cpu_dev)
- return -ENODEV;
-
- msm8996_version = qcom_cpufreq_kryo_get_msm_id();
- if (NUM_OF_MSM8996_VERSIONS == msm8996_version) {
- dev_err(cpu_dev, "Not Snapdragon 820/821!");
- return -ENODEV;
- }
-
- np = dev_pm_opp_of_get_opp_desc_node(cpu_dev);
- if (!np)
- return -ENOENT;
-
- ret = of_device_is_compatible(np, "operating-points-v2-kryo-cpu");
- if (!ret) {
- of_node_put(np);
- return -ENOENT;
- }
-
- speedbin_nvmem = of_nvmem_cell_get(np, NULL);
- of_node_put(np);
- if (IS_ERR(speedbin_nvmem)) {
- if (PTR_ERR(speedbin_nvmem) != -EPROBE_DEFER)
- dev_err(cpu_dev, "Could not get nvmem cell: %ld\n",
- PTR_ERR(speedbin_nvmem));
- return PTR_ERR(speedbin_nvmem);
- }
-
- speedbin = nvmem_cell_read(speedbin_nvmem, &len);
- nvmem_cell_put(speedbin_nvmem);
- if (IS_ERR(speedbin))
- return PTR_ERR(speedbin);
-
- switch (msm8996_version) {
- case MSM8996_V3:
- versions = 1 << (unsigned int)(*speedbin);
- break;
- case MSM8996_SG:
- versions = 1 << ((unsigned int)(*speedbin) + 4);
- break;
- default:
- BUG();
- break;
- }
- kfree(speedbin);
-
- opp_tables = kcalloc(num_possible_cpus(), sizeof(*opp_tables), GFP_KERNEL);
- if (!opp_tables)
- return -ENOMEM;
-
- for_each_possible_cpu(cpu) {
- cpu_dev = get_cpu_device(cpu);
- if (NULL == cpu_dev) {
- ret = -ENODEV;
- goto free_opp;
- }
-
- opp_tables[cpu] = dev_pm_opp_set_supported_hw(cpu_dev,
- &versions, 1);
- if (IS_ERR(opp_tables[cpu])) {
- ret = PTR_ERR(opp_tables[cpu]);
- dev_err(cpu_dev, "Failed to set supported hardware\n");
- goto free_opp;
- }
- }
-
- cpufreq_dt_pdev = platform_device_register_simple("cpufreq-dt", -1,
- NULL, 0);
- if (!IS_ERR(cpufreq_dt_pdev)) {
- platform_set_drvdata(pdev, opp_tables);
- return 0;
- }
-
- ret = PTR_ERR(cpufreq_dt_pdev);
- dev_err(cpu_dev, "Failed to register platform device\n");
-
-free_opp:
- for_each_possible_cpu(cpu) {
- if (IS_ERR_OR_NULL(opp_tables[cpu]))
- break;
- dev_pm_opp_put_supported_hw(opp_tables[cpu]);
- }
- kfree(opp_tables);
-
- return ret;
-}
-
-static int qcom_cpufreq_kryo_remove(struct platform_device *pdev)
-{
- struct opp_table **opp_tables = platform_get_drvdata(pdev);
- unsigned int cpu;
-
- platform_device_unregister(cpufreq_dt_pdev);
-
- for_each_possible_cpu(cpu)
- dev_pm_opp_put_supported_hw(opp_tables[cpu]);
-
- kfree(opp_tables);
-
- return 0;
-}
-
-static struct platform_driver qcom_cpufreq_kryo_driver = {
- .probe = qcom_cpufreq_kryo_probe,
- .remove = qcom_cpufreq_kryo_remove,
- .driver = {
- .name = "qcom-cpufreq-kryo",
- },
-};
-
-static const struct of_device_id qcom_cpufreq_kryo_match_list[] __initconst = {
- { .compatible = "qcom,apq8096", },
- { .compatible = "qcom,msm8996", },
- {}
-};
-
-/*
- * Since the driver depends on smem and nvmem drivers, which may
- * return EPROBE_DEFER, all the real activity is done in the probe,
- * which may be defered as well. The init here is only registering
- * the driver and the platform device.
- */
-static int __init qcom_cpufreq_kryo_init(void)
-{
- struct device_node *np = of_find_node_by_path("/");
- const struct of_device_id *match;
- int ret;
-
- if (!np)
- return -ENODEV;
-
- match = of_match_node(qcom_cpufreq_kryo_match_list, np);
- of_node_put(np);
- if (!match)
- return -ENODEV;
-
- ret = platform_driver_register(&qcom_cpufreq_kryo_driver);
- if (unlikely(ret < 0))
- return ret;
-
- kryo_cpufreq_pdev = platform_device_register_simple(
- "qcom-cpufreq-kryo", -1, NULL, 0);
- ret = PTR_ERR_OR_ZERO(kryo_cpufreq_pdev);
- if (0 == ret)
- return 0;
-
- platform_driver_unregister(&qcom_cpufreq_kryo_driver);
- return ret;
-}
-module_init(qcom_cpufreq_kryo_init);
-
-static void __exit qcom_cpufreq_kryo_exit(void)
-{
- platform_device_unregister(kryo_cpufreq_pdev);
- platform_driver_unregister(&qcom_cpufreq_kryo_driver);
-}
-module_exit(qcom_cpufreq_kryo_exit);
-
-MODULE_DESCRIPTION("Qualcomm Technologies, Inc. Kryo CPUfreq driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/qcom-cpufreq-nvmem.c b/drivers/cpufreq/qcom-cpufreq-nvmem.c
new file mode 100644
index 000000000000..f0d2d5035413
--- /dev/null
+++ b/drivers/cpufreq/qcom-cpufreq-nvmem.c
@@ -0,0 +1,352 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
+ */
+
+/*
+ * In Certain QCOM SoCs like apq8096 and msm8996 that have KRYO processors,
+ * the CPU frequency subset and voltage value of each OPP varies
+ * based on the silicon variant in use. Qualcomm Process Voltage Scaling Tables
+ * defines the voltage and frequency value based on the msm-id in SMEM
+ * and speedbin blown in the efuse combination.
+ * The qcom-cpufreq-nvmem driver reads the msm-id and efuse value from the SoC
+ * to provide the OPP framework with required information.
+ * This is used to determine the voltage and frequency value for each OPP of
+ * operating-points-v2 table when it is parsed by the OPP framework.
+ */
+
+#include <linux/cpu.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/nvmem-consumer.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_domain.h>
+#include <linux/pm_opp.h>
+#include <linux/slab.h>
+#include <linux/soc/qcom/smem.h>
+
+#define MSM_ID_SMEM 137
+
+enum _msm_id {
+ MSM8996V3 = 0xF6ul,
+ APQ8096V3 = 0x123ul,
+ MSM8996SG = 0x131ul,
+ APQ8096SG = 0x138ul,
+};
+
+enum _msm8996_version {
+ MSM8996_V3,
+ MSM8996_SG,
+ NUM_OF_MSM8996_VERSIONS,
+};
+
+struct qcom_cpufreq_drv;
+
+struct qcom_cpufreq_match_data {
+ int (*get_version)(struct device *cpu_dev,
+ struct nvmem_cell *speedbin_nvmem,
+ struct qcom_cpufreq_drv *drv);
+ const char **genpd_names;
+};
+
+struct qcom_cpufreq_drv {
+ struct opp_table **opp_tables;
+ struct opp_table **genpd_opp_tables;
+ u32 versions;
+ const struct qcom_cpufreq_match_data *data;
+};
+
+static struct platform_device *cpufreq_dt_pdev, *cpufreq_pdev;
+
+static enum _msm8996_version qcom_cpufreq_get_msm_id(void)
+{
+ size_t len;
+ u32 *msm_id;
+ enum _msm8996_version version;
+
+ msm_id = qcom_smem_get(QCOM_SMEM_HOST_ANY, MSM_ID_SMEM, &len);
+ if (IS_ERR(msm_id))
+ return NUM_OF_MSM8996_VERSIONS;
+
+ /* The first 4 bytes are format, next to them is the actual msm-id */
+ msm_id++;
+
+ switch ((enum _msm_id)*msm_id) {
+ case MSM8996V3:
+ case APQ8096V3:
+ version = MSM8996_V3;
+ break;
+ case MSM8996SG:
+ case APQ8096SG:
+ version = MSM8996_SG;
+ break;
+ default:
+ version = NUM_OF_MSM8996_VERSIONS;
+ }
+
+ return version;
+}
+
+static int qcom_cpufreq_kryo_name_version(struct device *cpu_dev,
+ struct nvmem_cell *speedbin_nvmem,
+ struct qcom_cpufreq_drv *drv)
+{
+ size_t len;
+ u8 *speedbin;
+ enum _msm8996_version msm8996_version;
+
+ msm8996_version = qcom_cpufreq_get_msm_id();
+ if (NUM_OF_MSM8996_VERSIONS == msm8996_version) {
+ dev_err(cpu_dev, "Not Snapdragon 820/821!");
+ return -ENODEV;
+ }
+
+ speedbin = nvmem_cell_read(speedbin_nvmem, &len);
+ if (IS_ERR(speedbin))
+ return PTR_ERR(speedbin);
+
+ switch (msm8996_version) {
+ case MSM8996_V3:
+ drv->versions = 1 << (unsigned int)(*speedbin);
+ break;
+ case MSM8996_SG:
+ drv->versions = 1 << ((unsigned int)(*speedbin) + 4);
+ break;
+ default:
+ BUG();
+ break;
+ }
+
+ kfree(speedbin);
+ return 0;
+}
+
+static const struct qcom_cpufreq_match_data match_data_kryo = {
+ .get_version = qcom_cpufreq_kryo_name_version,
+};
+
+static const char *qcs404_genpd_names[] = { "cpr", NULL };
+
+static const struct qcom_cpufreq_match_data match_data_qcs404 = {
+ .genpd_names = qcs404_genpd_names,
+};
+
+static int qcom_cpufreq_probe(struct platform_device *pdev)
+{
+ struct qcom_cpufreq_drv *drv;
+ struct nvmem_cell *speedbin_nvmem;
+ struct device_node *np;
+ struct device *cpu_dev;
+ unsigned cpu;
+ const struct of_device_id *match;
+ int ret;
+
+ cpu_dev = get_cpu_device(0);
+ if (!cpu_dev)
+ return -ENODEV;
+
+ np = dev_pm_opp_of_get_opp_desc_node(cpu_dev);
+ if (!np)
+ return -ENOENT;
+
+ ret = of_device_is_compatible(np, "operating-points-v2-kryo-cpu");
+ if (!ret) {
+ of_node_put(np);
+ return -ENOENT;
+ }
+
+ drv = kzalloc(sizeof(*drv), GFP_KERNEL);
+ if (!drv)
+ return -ENOMEM;
+
+ match = pdev->dev.platform_data;
+ drv->data = match->data;
+ if (!drv->data) {
+ ret = -ENODEV;
+ goto free_drv;
+ }
+
+ if (drv->data->get_version) {
+ speedbin_nvmem = of_nvmem_cell_get(np, NULL);
+ if (IS_ERR(speedbin_nvmem)) {
+ if (PTR_ERR(speedbin_nvmem) != -EPROBE_DEFER)
+ dev_err(cpu_dev,
+ "Could not get nvmem cell: %ld\n",
+ PTR_ERR(speedbin_nvmem));
+ ret = PTR_ERR(speedbin_nvmem);
+ goto free_drv;
+ }
+
+ ret = drv->data->get_version(cpu_dev, speedbin_nvmem, drv);
+ if (ret) {
+ nvmem_cell_put(speedbin_nvmem);
+ goto free_drv;
+ }
+ nvmem_cell_put(speedbin_nvmem);
+ }
+ of_node_put(np);
+
+ drv->opp_tables = kcalloc(num_possible_cpus(), sizeof(*drv->opp_tables),
+ GFP_KERNEL);
+ if (!drv->opp_tables) {
+ ret = -ENOMEM;
+ goto free_drv;
+ }
+
+ drv->genpd_opp_tables = kcalloc(num_possible_cpus(),
+ sizeof(*drv->genpd_opp_tables),
+ GFP_KERNEL);
+ if (!drv->genpd_opp_tables) {
+ ret = -ENOMEM;
+ goto free_opp;
+ }
+
+ for_each_possible_cpu(cpu) {
+ cpu_dev = get_cpu_device(cpu);
+ if (NULL == cpu_dev) {
+ ret = -ENODEV;
+ goto free_genpd_opp;
+ }
+
+ if (drv->data->get_version) {
+ drv->opp_tables[cpu] =
+ dev_pm_opp_set_supported_hw(cpu_dev,
+ &drv->versions, 1);
+ if (IS_ERR(drv->opp_tables[cpu])) {
+ ret = PTR_ERR(drv->opp_tables[cpu]);
+ dev_err(cpu_dev,
+ "Failed to set supported hardware\n");
+ goto free_genpd_opp;
+ }
+ }
+
+ if (drv->data->genpd_names) {
+ drv->genpd_opp_tables[cpu] =
+ dev_pm_opp_attach_genpd(cpu_dev,
+ drv->data->genpd_names,
+ NULL);
+ if (IS_ERR(drv->genpd_opp_tables[cpu])) {
+ ret = PTR_ERR(drv->genpd_opp_tables[cpu]);
+ if (ret != -EPROBE_DEFER)
+ dev_err(cpu_dev,
+ "Could not attach to pm_domain: %d\n",
+ ret);
+ goto free_genpd_opp;
+ }
+ }
+ }
+
+ cpufreq_dt_pdev = platform_device_register_simple("cpufreq-dt", -1,
+ NULL, 0);
+ if (!IS_ERR(cpufreq_dt_pdev)) {
+ platform_set_drvdata(pdev, drv);
+ return 0;
+ }
+
+ ret = PTR_ERR(cpufreq_dt_pdev);
+ dev_err(cpu_dev, "Failed to register platform device\n");
+
+free_genpd_opp:
+ for_each_possible_cpu(cpu) {
+ if (IS_ERR_OR_NULL(drv->genpd_opp_tables[cpu]))
+ break;
+ dev_pm_opp_detach_genpd(drv->genpd_opp_tables[cpu]);
+ }
+ kfree(drv->genpd_opp_tables);
+free_opp:
+ for_each_possible_cpu(cpu) {
+ if (IS_ERR_OR_NULL(drv->opp_tables[cpu]))
+ break;
+ dev_pm_opp_put_supported_hw(drv->opp_tables[cpu]);
+ }
+ kfree(drv->opp_tables);
+free_drv:
+ kfree(drv);
+
+ return ret;
+}
+
+static int qcom_cpufreq_remove(struct platform_device *pdev)
+{
+ struct qcom_cpufreq_drv *drv = platform_get_drvdata(pdev);
+ unsigned int cpu;
+
+ platform_device_unregister(cpufreq_dt_pdev);
+
+ for_each_possible_cpu(cpu) {
+ if (drv->opp_tables[cpu])
+ dev_pm_opp_put_supported_hw(drv->opp_tables[cpu]);
+ if (drv->genpd_opp_tables[cpu])
+ dev_pm_opp_detach_genpd(drv->genpd_opp_tables[cpu]);
+ }
+
+ kfree(drv->opp_tables);
+ kfree(drv->genpd_opp_tables);
+ kfree(drv);
+
+ return 0;
+}
+
+static struct platform_driver qcom_cpufreq_driver = {
+ .probe = qcom_cpufreq_probe,
+ .remove = qcom_cpufreq_remove,
+ .driver = {
+ .name = "qcom-cpufreq-nvmem",
+ },
+};
+
+static const struct of_device_id qcom_cpufreq_match_list[] __initconst = {
+ { .compatible = "qcom,apq8096", .data = &match_data_kryo },
+ { .compatible = "qcom,msm8996", .data = &match_data_kryo },
+ { .compatible = "qcom,qcs404", .data = &match_data_qcs404 },
+ {},
+};
+
+/*
+ * Since the driver depends on smem and nvmem drivers, which may
+ * return EPROBE_DEFER, all the real activity is done in the probe,
+ * which may be defered as well. The init here is only registering
+ * the driver and the platform device.
+ */
+static int __init qcom_cpufreq_init(void)
+{
+ struct device_node *np = of_find_node_by_path("/");
+ const struct of_device_id *match;
+ int ret;
+
+ if (!np)
+ return -ENODEV;
+
+ match = of_match_node(qcom_cpufreq_match_list, np);
+ of_node_put(np);
+ if (!match)
+ return -ENODEV;
+
+ ret = platform_driver_register(&qcom_cpufreq_driver);
+ if (unlikely(ret < 0))
+ return ret;
+
+ cpufreq_pdev = platform_device_register_data(NULL, "qcom-cpufreq-nvmem",
+ -1, match, sizeof(*match));
+ ret = PTR_ERR_OR_ZERO(cpufreq_pdev);
+ if (0 == ret)
+ return 0;
+
+ platform_driver_unregister(&qcom_cpufreq_driver);
+ return ret;
+}
+module_init(qcom_cpufreq_init);
+
+static void __exit qcom_cpufreq_exit(void)
+{
+ platform_device_unregister(cpufreq_pdev);
+ platform_driver_unregister(&qcom_cpufreq_driver);
+}
+module_exit(qcom_cpufreq_exit);
+
+MODULE_DESCRIPTION("Qualcomm Technologies, Inc. CPUfreq driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/raspberrypi-cpufreq.c b/drivers/cpufreq/raspberrypi-cpufreq.c
new file mode 100644
index 000000000000..2bc7d9734272
--- /dev/null
+++ b/drivers/cpufreq/raspberrypi-cpufreq.c
@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Raspberry Pi cpufreq driver
+ *
+ * Copyright (C) 2019, Nicolas Saenz Julienne <nsaenzjulienne@suse.de>
+ */
+
+#include <linux/clk.h>
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+
+#define RASPBERRYPI_FREQ_INTERVAL 100000000
+
+static struct platform_device *cpufreq_dt;
+
+static int raspberrypi_cpufreq_probe(struct platform_device *pdev)
+{
+ struct device *cpu_dev;
+ unsigned long min, max;
+ unsigned long rate;
+ struct clk *clk;
+ int ret;
+
+ cpu_dev = get_cpu_device(0);
+ if (!cpu_dev) {
+ pr_err("Cannot get CPU for cpufreq driver\n");
+ return -ENODEV;
+ }
+
+ clk = clk_get(cpu_dev, NULL);
+ if (IS_ERR(clk)) {
+ dev_err(cpu_dev, "Cannot get clock for CPU0\n");
+ return PTR_ERR(clk);
+ }
+
+ /*
+ * The max and min frequencies are configurable in the Raspberry Pi
+ * firmware, so we query them at runtime.
+ */
+ min = roundup(clk_round_rate(clk, 0), RASPBERRYPI_FREQ_INTERVAL);
+ max = roundup(clk_round_rate(clk, ULONG_MAX), RASPBERRYPI_FREQ_INTERVAL);
+ clk_put(clk);
+
+ for (rate = min; rate <= max; rate += RASPBERRYPI_FREQ_INTERVAL) {
+ ret = dev_pm_opp_add(cpu_dev, rate, 0);
+ if (ret)
+ goto remove_opp;
+ }
+
+ cpufreq_dt = platform_device_register_simple("cpufreq-dt", -1, NULL, 0);
+ ret = PTR_ERR_OR_ZERO(cpufreq_dt);
+ if (ret) {
+ dev_err(cpu_dev, "Failed to create platform device, %d\n", ret);
+ goto remove_opp;
+ }
+
+ return 0;
+
+remove_opp:
+ dev_pm_opp_remove_all_dynamic(cpu_dev);
+
+ return ret;
+}
+
+static int raspberrypi_cpufreq_remove(struct platform_device *pdev)
+{
+ struct device *cpu_dev;
+
+ cpu_dev = get_cpu_device(0);
+ if (cpu_dev)
+ dev_pm_opp_remove_all_dynamic(cpu_dev);
+
+ platform_device_unregister(cpufreq_dt);
+
+ return 0;
+}
+
+/*
+ * Since the driver depends on clk-raspberrypi, which may return EPROBE_DEFER,
+ * all the activity is performed in the probe, which may be defered as well.
+ */
+static struct platform_driver raspberrypi_cpufreq_driver = {
+ .driver = {
+ .name = "raspberrypi-cpufreq",
+ },
+ .probe = raspberrypi_cpufreq_probe,
+ .remove = raspberrypi_cpufreq_remove,
+};
+module_platform_driver(raspberrypi_cpufreq_driver);
+
+MODULE_AUTHOR("Nicolas Saenz Julienne <nsaenzjulienne@suse.de");
+MODULE_DESCRIPTION("Raspberry Pi cpufreq driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:raspberrypi-cpufreq");
diff --git a/drivers/cpufreq/s3c2416-cpufreq.c b/drivers/cpufreq/s3c2416-cpufreq.c
index f7ff1ed7fef1..106910351c41 100644
--- a/drivers/cpufreq/s3c2416-cpufreq.c
+++ b/drivers/cpufreq/s3c2416-cpufreq.c
@@ -447,21 +447,16 @@ static int s3c2416_cpufreq_driver_init(struct cpufreq_policy *policy)
/* Datasheet says PLL stabalisation time must be at least 300us,
* so but add some fudge. (reference in LOCKCON0 register description)
*/
- ret = cpufreq_generic_init(policy, s3c_freq->freq_table,
+ cpufreq_generic_init(policy, s3c_freq->freq_table,
(500 * 1000) + s3c_freq->regulator_latency);
- if (ret)
- goto err_freq_table;
-
register_reboot_notifier(&s3c2416_cpufreq_reboot_notifier);
return 0;
-err_freq_table:
#ifdef CONFIG_ARM_S3C2416_CPUFREQ_VCORESCALE
- regulator_put(s3c_freq->vddarm);
err_vddarm:
-#endif
clk_put(s3c_freq->armclk);
+#endif
err_armclk:
clk_put(s3c_freq->hclk);
err_hclk:
diff --git a/drivers/cpufreq/s3c64xx-cpufreq.c b/drivers/cpufreq/s3c64xx-cpufreq.c
index 37df2d892eb0..af0c00dabb22 100644
--- a/drivers/cpufreq/s3c64xx-cpufreq.c
+++ b/drivers/cpufreq/s3c64xx-cpufreq.c
@@ -144,7 +144,6 @@ out:
static int s3c64xx_cpufreq_driver_init(struct cpufreq_policy *policy)
{
- int ret;
struct cpufreq_frequency_table *freq;
if (policy->cpu != 0)
@@ -165,8 +164,7 @@ static int s3c64xx_cpufreq_driver_init(struct cpufreq_policy *policy)
#ifdef CONFIG_REGULATOR
vddarm = regulator_get(NULL, "vddarm");
if (IS_ERR(vddarm)) {
- ret = PTR_ERR(vddarm);
- pr_err("Failed to obtain VDDARM: %d\n", ret);
+ pr_err("Failed to obtain VDDARM: %ld\n", PTR_ERR(vddarm));
pr_err("Only frequency scaling available\n");
vddarm = NULL;
} else {
@@ -196,16 +194,9 @@ static int s3c64xx_cpufreq_driver_init(struct cpufreq_policy *policy)
* the PLLs, which we don't currently) is ~300us worst case,
* but add some fudge.
*/
- ret = cpufreq_generic_init(policy, s3c64xx_freq_table,
+ cpufreq_generic_init(policy, s3c64xx_freq_table,
(500 * 1000) + regulator_latency);
- if (ret != 0) {
- pr_err("Failed to configure frequency table: %d\n",
- ret);
- regulator_put(vddarm);
- clk_put(policy->clk);
- }
-
- return ret;
+ return 0;
}
static struct cpufreq_driver s3c64xx_cpufreq_driver = {
diff --git a/drivers/cpufreq/s5pv210-cpufreq.c b/drivers/cpufreq/s5pv210-cpufreq.c
index 57e5374592bd..5d10030f2560 100644
--- a/drivers/cpufreq/s5pv210-cpufreq.c
+++ b/drivers/cpufreq/s5pv210-cpufreq.c
@@ -478,7 +478,7 @@ static int s5pv210_target(struct cpufreq_policy *policy, unsigned int index)
arm_volt, arm_volt_max);
}
- printk(KERN_DEBUG "Perf changed[L%d]\n", index);
+ pr_debug("Perf changed[L%d]\n", index);
exit:
mutex_unlock(&set_freq_lock);
@@ -541,7 +541,8 @@ static int s5pv210_cpu_init(struct cpufreq_policy *policy)
s5pv210_dram_conf[1].freq = clk_get_rate(dmc1_clk);
policy->suspend_freq = SLEEP_FREQ;
- return cpufreq_generic_init(policy, s5pv210_freq_table, 40000);
+ cpufreq_generic_init(policy, s5pv210_freq_table, 40000);
+ return 0;
out_dmc1:
clk_put(dmc0_clk);
diff --git a/drivers/cpufreq/sa1100-cpufreq.c b/drivers/cpufreq/sa1100-cpufreq.c
index ab5cab93e638..5c075ef6adc0 100644
--- a/drivers/cpufreq/sa1100-cpufreq.c
+++ b/drivers/cpufreq/sa1100-cpufreq.c
@@ -181,7 +181,8 @@ static int sa1100_target(struct cpufreq_policy *policy, unsigned int ppcr)
static int __init sa1100_cpu_init(struct cpufreq_policy *policy)
{
- return cpufreq_generic_init(policy, sa11x0_freq_table, 0);
+ cpufreq_generic_init(policy, sa11x0_freq_table, 0);
+ return 0;
}
static struct cpufreq_driver sa1100_driver __refdata = {
diff --git a/drivers/cpufreq/sa1110-cpufreq.c b/drivers/cpufreq/sa1110-cpufreq.c
index dab54e051c0e..d9d04d935b3a 100644
--- a/drivers/cpufreq/sa1110-cpufreq.c
+++ b/drivers/cpufreq/sa1110-cpufreq.c
@@ -303,7 +303,8 @@ static int sa1110_target(struct cpufreq_policy *policy, unsigned int ppcr)
static int __init sa1110_cpu_init(struct cpufreq_policy *policy)
{
- return cpufreq_generic_init(policy, sa11x0_freq_table, 0);
+ cpufreq_generic_init(policy, sa11x0_freq_table, 0);
+ return 0;
}
/* sa1110_driver needs __refdata because it must remain after init registers
diff --git a/drivers/cpufreq/spear-cpufreq.c b/drivers/cpufreq/spear-cpufreq.c
index 4074e2615522..73bd8dc47074 100644
--- a/drivers/cpufreq/spear-cpufreq.c
+++ b/drivers/cpufreq/spear-cpufreq.c
@@ -153,8 +153,9 @@ static int spear_cpufreq_target(struct cpufreq_policy *policy,
static int spear_cpufreq_init(struct cpufreq_policy *policy)
{
policy->clk = spear_cpufreq.clk;
- return cpufreq_generic_init(policy, spear_cpufreq.freq_tbl,
+ cpufreq_generic_init(policy, spear_cpufreq.freq_tbl,
spear_cpufreq.transition_latency);
+ return 0;
}
static struct cpufreq_driver spear_cpufreq_driver = {
diff --git a/drivers/cpufreq/sun50i-cpufreq-nvmem.c b/drivers/cpufreq/sun50i-cpufreq-nvmem.c
new file mode 100644
index 000000000000..eca32e443716
--- /dev/null
+++ b/drivers/cpufreq/sun50i-cpufreq-nvmem.c
@@ -0,0 +1,226 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Allwinner CPUFreq nvmem based driver
+ *
+ * The sun50i-cpufreq-nvmem driver reads the efuse value from the SoC to
+ * provide the OPP framework with required information.
+ *
+ * Copyright (C) 2019 Yangtao Li <tiny.windzz@gmail.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/module.h>
+#include <linux/nvmem-consumer.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/slab.h>
+
+#define MAX_NAME_LEN 7
+
+#define NVMEM_MASK 0x7
+#define NVMEM_SHIFT 5
+
+static struct platform_device *cpufreq_dt_pdev, *sun50i_cpufreq_pdev;
+
+/**
+ * sun50i_cpufreq_get_efuse() - Parse and return efuse value present on SoC
+ * @versions: Set to the value parsed from efuse
+ *
+ * Returns 0 if success.
+ */
+static int sun50i_cpufreq_get_efuse(u32 *versions)
+{
+ struct nvmem_cell *speedbin_nvmem;
+ struct device_node *np;
+ struct device *cpu_dev;
+ u32 *speedbin, efuse_value;
+ size_t len;
+ int ret;
+
+ cpu_dev = get_cpu_device(0);
+ if (!cpu_dev)
+ return -ENODEV;
+
+ np = dev_pm_opp_of_get_opp_desc_node(cpu_dev);
+ if (!np)
+ return -ENOENT;
+
+ ret = of_device_is_compatible(np,
+ "allwinner,sun50i-h6-operating-points");
+ if (!ret) {
+ of_node_put(np);
+ return -ENOENT;
+ }
+
+ speedbin_nvmem = of_nvmem_cell_get(np, NULL);
+ of_node_put(np);
+ if (IS_ERR(speedbin_nvmem)) {
+ if (PTR_ERR(speedbin_nvmem) != -EPROBE_DEFER)
+ pr_err("Could not get nvmem cell: %ld\n",
+ PTR_ERR(speedbin_nvmem));
+ return PTR_ERR(speedbin_nvmem);
+ }
+
+ speedbin = nvmem_cell_read(speedbin_nvmem, &len);
+ nvmem_cell_put(speedbin_nvmem);
+ if (IS_ERR(speedbin))
+ return PTR_ERR(speedbin);
+
+ efuse_value = (*speedbin >> NVMEM_SHIFT) & NVMEM_MASK;
+ switch (efuse_value) {
+ case 0b0001:
+ *versions = 1;
+ break;
+ case 0b0011:
+ *versions = 2;
+ break;
+ default:
+ /*
+ * For other situations, we treat it as bin0.
+ * This vf table can be run for any good cpu.
+ */
+ *versions = 0;
+ break;
+ }
+
+ kfree(speedbin);
+ return 0;
+};
+
+static int sun50i_cpufreq_nvmem_probe(struct platform_device *pdev)
+{
+ struct opp_table **opp_tables;
+ char name[MAX_NAME_LEN];
+ unsigned int cpu;
+ u32 speed = 0;
+ int ret;
+
+ opp_tables = kcalloc(num_possible_cpus(), sizeof(*opp_tables),
+ GFP_KERNEL);
+ if (!opp_tables)
+ return -ENOMEM;
+
+ ret = sun50i_cpufreq_get_efuse(&speed);
+ if (ret)
+ return ret;
+
+ snprintf(name, MAX_NAME_LEN, "speed%d", speed);
+
+ for_each_possible_cpu(cpu) {
+ struct device *cpu_dev = get_cpu_device(cpu);
+
+ if (!cpu_dev) {
+ ret = -ENODEV;
+ goto free_opp;
+ }
+
+ opp_tables[cpu] = dev_pm_opp_set_prop_name(cpu_dev, name);
+ if (IS_ERR(opp_tables[cpu])) {
+ ret = PTR_ERR(opp_tables[cpu]);
+ pr_err("Failed to set prop name\n");
+ goto free_opp;
+ }
+ }
+
+ cpufreq_dt_pdev = platform_device_register_simple("cpufreq-dt", -1,
+ NULL, 0);
+ if (!IS_ERR(cpufreq_dt_pdev)) {
+ platform_set_drvdata(pdev, opp_tables);
+ return 0;
+ }
+
+ ret = PTR_ERR(cpufreq_dt_pdev);
+ pr_err("Failed to register platform device\n");
+
+free_opp:
+ for_each_possible_cpu(cpu) {
+ if (IS_ERR_OR_NULL(opp_tables[cpu]))
+ break;
+ dev_pm_opp_put_prop_name(opp_tables[cpu]);
+ }
+ kfree(opp_tables);
+
+ return ret;
+}
+
+static int sun50i_cpufreq_nvmem_remove(struct platform_device *pdev)
+{
+ struct opp_table **opp_tables = platform_get_drvdata(pdev);
+ unsigned int cpu;
+
+ platform_device_unregister(cpufreq_dt_pdev);
+
+ for_each_possible_cpu(cpu)
+ dev_pm_opp_put_prop_name(opp_tables[cpu]);
+
+ kfree(opp_tables);
+
+ return 0;
+}
+
+static struct platform_driver sun50i_cpufreq_driver = {
+ .probe = sun50i_cpufreq_nvmem_probe,
+ .remove = sun50i_cpufreq_nvmem_remove,
+ .driver = {
+ .name = "sun50i-cpufreq-nvmem",
+ },
+};
+
+static const struct of_device_id sun50i_cpufreq_match_list[] = {
+ { .compatible = "allwinner,sun50i-h6" },
+ {}
+};
+
+static const struct of_device_id *sun50i_cpufreq_match_node(void)
+{
+ const struct of_device_id *match;
+ struct device_node *np;
+
+ np = of_find_node_by_path("/");
+ match = of_match_node(sun50i_cpufreq_match_list, np);
+ of_node_put(np);
+
+ return match;
+}
+
+/*
+ * Since the driver depends on nvmem drivers, which may return EPROBE_DEFER,
+ * all the real activity is done in the probe, which may be defered as well.
+ * The init here is only registering the driver and the platform device.
+ */
+static int __init sun50i_cpufreq_init(void)
+{
+ const struct of_device_id *match;
+ int ret;
+
+ match = sun50i_cpufreq_match_node();
+ if (!match)
+ return -ENODEV;
+
+ ret = platform_driver_register(&sun50i_cpufreq_driver);
+ if (unlikely(ret < 0))
+ return ret;
+
+ sun50i_cpufreq_pdev =
+ platform_device_register_simple("sun50i-cpufreq-nvmem",
+ -1, NULL, 0);
+ ret = PTR_ERR_OR_ZERO(sun50i_cpufreq_pdev);
+ if (ret == 0)
+ return 0;
+
+ platform_driver_unregister(&sun50i_cpufreq_driver);
+ return ret;
+}
+module_init(sun50i_cpufreq_init);
+
+static void __exit sun50i_cpufreq_exit(void)
+{
+ platform_device_unregister(sun50i_cpufreq_pdev);
+ platform_driver_unregister(&sun50i_cpufreq_driver);
+}
+module_exit(sun50i_cpufreq_exit);
+
+MODULE_DESCRIPTION("Sun50i-h6 cpufreq driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpufreq/tegra20-cpufreq.c b/drivers/cpufreq/tegra20-cpufreq.c
index 3c32cc7b0671..f84ecd22f488 100644
--- a/drivers/cpufreq/tegra20-cpufreq.c
+++ b/drivers/cpufreq/tegra20-cpufreq.c
@@ -118,17 +118,11 @@ static int tegra_target(struct cpufreq_policy *policy, unsigned int index)
static int tegra_cpu_init(struct cpufreq_policy *policy)
{
struct tegra20_cpufreq *cpufreq = cpufreq_get_driver_data();
- int ret;
clk_prepare_enable(cpufreq->cpu_clk);
/* FIXME: what's the actual transition time? */
- ret = cpufreq_generic_init(policy, freq_table, 300 * 1000);
- if (ret) {
- clk_disable_unprepare(cpufreq->cpu_clk);
- return ret;
- }
-
+ cpufreq_generic_init(policy, freq_table, 300 * 1000);
policy->clk = cpufreq->cpu_clk;
policy->suspend_freq = freq_table[0].frequency;
return 0;
diff --git a/drivers/cpufreq/ti-cpufreq.c b/drivers/cpufreq/ti-cpufreq.c
index 2ad1ae17932d..aeaa883a8c9d 100644
--- a/drivers/cpufreq/ti-cpufreq.c
+++ b/drivers/cpufreq/ti-cpufreq.c
@@ -77,6 +77,7 @@ static unsigned long dra7_efuse_xlate(struct ti_cpufreq_data *opp_data,
case DRA7_EFUSE_HAS_ALL_MPU_OPP:
case DRA7_EFUSE_HAS_HIGH_MPU_OPP:
calculated_efuse |= DRA7_EFUSE_HIGH_MPU_OPP;
+ /* Fall through */
case DRA7_EFUSE_HAS_OD_MPU_OPP:
calculated_efuse |= DRA7_EFUSE_OD_MPU_OPP;
}
diff --git a/drivers/cpuidle/Kconfig b/drivers/cpuidle/Kconfig
index a4ac31e4a58c..88727b7c0d59 100644
--- a/drivers/cpuidle/Kconfig
+++ b/drivers/cpuidle/Kconfig
@@ -33,6 +33,17 @@ config CPU_IDLE_GOV_TEO
Some workloads benefit from using it and it generally should be safe
to use. Say Y here if you are not happy with the alternatives.
+config CPU_IDLE_GOV_HALTPOLL
+ bool "Haltpoll governor (for virtualized systems)"
+ depends on KVM_GUEST
+ help
+ This governor implements haltpoll idle state selection, to be
+ used in conjunction with the haltpoll cpuidle driver, allowing
+ for polling for a certain amount of time before entering idle
+ state.
+
+ Some virtualized workloads benefit from using it.
+
config DT_IDLE_STATES
bool
@@ -51,6 +62,15 @@ depends on PPC
source "drivers/cpuidle/Kconfig.powerpc"
endmenu
+config HALTPOLL_CPUIDLE
+ tristate "Halt poll cpuidle driver"
+ depends on X86 && KVM_GUEST
+ default y
+ help
+ This option enables halt poll cpuidle driver, which allows to poll
+ before halting in the guest (more efficient than polling in the
+ host via halt_poll_ns for some scenarios).
+
endif
config ARCH_NEEDS_CPU_IDLE_COUPLED
diff --git a/drivers/cpuidle/Kconfig.arm b/drivers/cpuidle/Kconfig.arm
index 48cb3d4bb7d1..d8530475493c 100644
--- a/drivers/cpuidle/Kconfig.arm
+++ b/drivers/cpuidle/Kconfig.arm
@@ -13,6 +13,16 @@ config ARM_CPUIDLE
initialized by calling the CPU operations init idle hook
provided by architecture code.
+config ARM_PSCI_CPUIDLE
+ bool "PSCI CPU idle Driver"
+ depends on ARM_PSCI_FW
+ select DT_IDLE_STATES
+ select CPU_IDLE_MULTIPLE_DRIVERS
+ help
+ Select this to enable PSCI firmware based CPUidle driver for ARM.
+ It provides an idle driver that is capable of detecting and
+ managing idle states through the PSCI firmware interface.
+
config ARM_BIG_LITTLE_CPUIDLE
bool "Support for ARM big.LITTLE processors"
depends on ARCH_VEXPRESS_TC2_PM || ARCH_EXYNOS
diff --git a/drivers/cpuidle/Makefile b/drivers/cpuidle/Makefile
index 9d7176cee3d3..ee70d5cc5b99 100644
--- a/drivers/cpuidle/Makefile
+++ b/drivers/cpuidle/Makefile
@@ -7,6 +7,7 @@ obj-y += cpuidle.o driver.o governor.o sysfs.o governors/
obj-$(CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED) += coupled.o
obj-$(CONFIG_DT_IDLE_STATES) += dt_idle_states.o
obj-$(CONFIG_ARCH_HAS_CPU_RELAX) += poll_state.o
+obj-$(CONFIG_HALTPOLL_CPUIDLE) += cpuidle-haltpoll.o
##################################################################################
# ARM SoC drivers
@@ -20,6 +21,7 @@ obj-$(CONFIG_ARM_U8500_CPUIDLE) += cpuidle-ux500.o
obj-$(CONFIG_ARM_AT91_CPUIDLE) += cpuidle-at91.o
obj-$(CONFIG_ARM_EXYNOS_CPUIDLE) += cpuidle-exynos.o
obj-$(CONFIG_ARM_CPUIDLE) += cpuidle-arm.o
+obj-$(CONFIG_ARM_PSCI_CPUIDLE) += cpuidle-psci.o
###############################################################################
# MIPS drivers
diff --git a/drivers/cpuidle/cpuidle-arm.c b/drivers/cpuidle/cpuidle-arm.c
index 5bcd82c35dcf..9e5156d39627 100644
--- a/drivers/cpuidle/cpuidle-arm.c
+++ b/drivers/cpuidle/cpuidle-arm.c
@@ -15,7 +15,6 @@
#include <linux/module.h>
#include <linux/of.h>
#include <linux/slab.h>
-#include <linux/topology.h>
#include <asm/cpuidle.h>
@@ -106,11 +105,17 @@ static int __init arm_idle_init_cpu(int cpu)
ret = arm_cpuidle_init(cpu);
/*
- * Allow the initialization to continue for other CPUs, if the reported
- * failure is a HW misconfiguration/breakage (-ENXIO).
+ * Allow the initialization to continue for other CPUs, if the
+ * reported failure is a HW misconfiguration/breakage (-ENXIO).
+ *
+ * Some platforms do not support idle operations
+ * (arm_cpuidle_init() returning -EOPNOTSUPP), we should
+ * not flag this case as an error, it is a valid
+ * configuration.
*/
if (ret) {
- pr_err("CPU %d failed to init idle CPU ops\n", cpu);
+ if (ret != -EOPNOTSUPP)
+ pr_err("CPU %d failed to init idle CPU ops\n", cpu);
ret = ret == -ENXIO ? 0 : ret;
goto out_kfree_drv;
}
diff --git a/drivers/cpuidle/cpuidle-haltpoll.c b/drivers/cpuidle/cpuidle-haltpoll.c
new file mode 100644
index 000000000000..932390b028f1
--- /dev/null
+++ b/drivers/cpuidle/cpuidle-haltpoll.c
@@ -0,0 +1,134 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * cpuidle driver for haltpoll governor.
+ *
+ * Copyright 2019 Red Hat, Inc. and/or its affiliates.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ *
+ * Authors: Marcelo Tosatti <mtosatti@redhat.com>
+ */
+
+#include <linux/init.h>
+#include <linux/cpu.h>
+#include <linux/cpuidle.h>
+#include <linux/module.h>
+#include <linux/sched/idle.h>
+#include <linux/kvm_para.h>
+#include <linux/cpuidle_haltpoll.h>
+
+static struct cpuidle_device __percpu *haltpoll_cpuidle_devices;
+static enum cpuhp_state haltpoll_hp_state;
+
+static int default_enter_idle(struct cpuidle_device *dev,
+ struct cpuidle_driver *drv, int index)
+{
+ if (current_clr_polling_and_test()) {
+ local_irq_enable();
+ return index;
+ }
+ default_idle();
+ return index;
+}
+
+static struct cpuidle_driver haltpoll_driver = {
+ .name = "haltpoll",
+ .governor = "haltpoll",
+ .states = {
+ { /* entry 0 is for polling */ },
+ {
+ .enter = default_enter_idle,
+ .exit_latency = 1,
+ .target_residency = 1,
+ .power_usage = -1,
+ .name = "haltpoll idle",
+ .desc = "default architecture idle",
+ },
+ },
+ .safe_state_index = 0,
+ .state_count = 2,
+};
+
+static int haltpoll_cpu_online(unsigned int cpu)
+{
+ struct cpuidle_device *dev;
+
+ dev = per_cpu_ptr(haltpoll_cpuidle_devices, cpu);
+ if (!dev->registered) {
+ dev->cpu = cpu;
+ if (cpuidle_register_device(dev)) {
+ pr_notice("cpuidle_register_device %d failed!\n", cpu);
+ return -EIO;
+ }
+ arch_haltpoll_enable(cpu);
+ }
+
+ return 0;
+}
+
+static int haltpoll_cpu_offline(unsigned int cpu)
+{
+ struct cpuidle_device *dev;
+
+ dev = per_cpu_ptr(haltpoll_cpuidle_devices, cpu);
+ if (dev->registered) {
+ arch_haltpoll_disable(cpu);
+ cpuidle_unregister_device(dev);
+ }
+
+ return 0;
+}
+
+static void haltpoll_uninit(void)
+{
+ if (haltpoll_hp_state)
+ cpuhp_remove_state(haltpoll_hp_state);
+ cpuidle_unregister_driver(&haltpoll_driver);
+
+ free_percpu(haltpoll_cpuidle_devices);
+ haltpoll_cpuidle_devices = NULL;
+}
+
+static int __init haltpoll_init(void)
+{
+ int ret;
+ struct cpuidle_driver *drv = &haltpoll_driver;
+
+ cpuidle_poll_state_init(drv);
+
+ if (!kvm_para_available() ||
+ !kvm_para_has_hint(KVM_HINTS_REALTIME))
+ return -ENODEV;
+
+ ret = cpuidle_register_driver(drv);
+ if (ret < 0)
+ return ret;
+
+ haltpoll_cpuidle_devices = alloc_percpu(struct cpuidle_device);
+ if (haltpoll_cpuidle_devices == NULL) {
+ cpuidle_unregister_driver(drv);
+ return -ENOMEM;
+ }
+
+ ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "cpuidle/haltpoll:online",
+ haltpoll_cpu_online, haltpoll_cpu_offline);
+ if (ret < 0) {
+ haltpoll_uninit();
+ } else {
+ haltpoll_hp_state = ret;
+ ret = 0;
+ }
+
+ return ret;
+}
+
+static void __exit haltpoll_exit(void)
+{
+ haltpoll_uninit();
+}
+
+module_init(haltpoll_init);
+module_exit(haltpoll_exit);
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Marcelo Tosatti <mtosatti@redhat.com>");
diff --git a/drivers/cpuidle/cpuidle-psci.c b/drivers/cpuidle/cpuidle-psci.c
new file mode 100644
index 000000000000..f3c1a2396f98
--- /dev/null
+++ b/drivers/cpuidle/cpuidle-psci.c
@@ -0,0 +1,236 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * PSCI CPU idle driver.
+ *
+ * Copyright (C) 2019 ARM Ltd.
+ * Author: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
+ */
+
+#define pr_fmt(fmt) "CPUidle PSCI: " fmt
+
+#include <linux/cpuidle.h>
+#include <linux/cpumask.h>
+#include <linux/cpu_pm.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/psci.h>
+#include <linux/slab.h>
+
+#include <asm/cpuidle.h>
+
+#include "dt_idle_states.h"
+
+static DEFINE_PER_CPU_READ_MOSTLY(u32 *, psci_power_state);
+
+static int psci_enter_idle_state(struct cpuidle_device *dev,
+ struct cpuidle_driver *drv, int idx)
+{
+ u32 *state = __this_cpu_read(psci_power_state);
+
+ return CPU_PM_CPU_IDLE_ENTER_PARAM(psci_cpu_suspend_enter,
+ idx, state[idx - 1]);
+}
+
+static struct cpuidle_driver psci_idle_driver __initdata = {
+ .name = "psci_idle",
+ .owner = THIS_MODULE,
+ /*
+ * PSCI idle states relies on architectural WFI to
+ * be represented as state index 0.
+ */
+ .states[0] = {
+ .enter = psci_enter_idle_state,
+ .exit_latency = 1,
+ .target_residency = 1,
+ .power_usage = UINT_MAX,
+ .name = "WFI",
+ .desc = "ARM WFI",
+ }
+};
+
+static const struct of_device_id psci_idle_state_match[] __initconst = {
+ { .compatible = "arm,idle-state",
+ .data = psci_enter_idle_state },
+ { },
+};
+
+static int __init psci_dt_parse_state_node(struct device_node *np, u32 *state)
+{
+ int err = of_property_read_u32(np, "arm,psci-suspend-param", state);
+
+ if (err) {
+ pr_warn("%pOF missing arm,psci-suspend-param property\n", np);
+ return err;
+ }
+
+ if (!psci_power_state_is_valid(*state)) {
+ pr_warn("Invalid PSCI power state %#x\n", *state);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int __init psci_dt_cpu_init_idle(struct device_node *cpu_node, int cpu)
+{
+ int i, ret = 0, count = 0;
+ u32 *psci_states;
+ struct device_node *state_node;
+
+ /* Count idle states */
+ while ((state_node = of_parse_phandle(cpu_node, "cpu-idle-states",
+ count))) {
+ count++;
+ of_node_put(state_node);
+ }
+
+ if (!count)
+ return -ENODEV;
+
+ psci_states = kcalloc(count, sizeof(*psci_states), GFP_KERNEL);
+ if (!psci_states)
+ return -ENOMEM;
+
+ for (i = 0; i < count; i++) {
+ state_node = of_parse_phandle(cpu_node, "cpu-idle-states", i);
+ ret = psci_dt_parse_state_node(state_node, &psci_states[i]);
+ of_node_put(state_node);
+
+ if (ret)
+ goto free_mem;
+
+ pr_debug("psci-power-state %#x index %d\n", psci_states[i], i);
+ }
+
+ /* Idle states parsed correctly, initialize per-cpu pointer */
+ per_cpu(psci_power_state, cpu) = psci_states;
+ return 0;
+
+free_mem:
+ kfree(psci_states);
+ return ret;
+}
+
+static __init int psci_cpu_init_idle(unsigned int cpu)
+{
+ struct device_node *cpu_node;
+ int ret;
+
+ /*
+ * If the PSCI cpu_suspend function hook has not been initialized
+ * idle states must not be enabled, so bail out
+ */
+ if (!psci_ops.cpu_suspend)
+ return -EOPNOTSUPP;
+
+ cpu_node = of_cpu_device_node_get(cpu);
+ if (!cpu_node)
+ return -ENODEV;
+
+ ret = psci_dt_cpu_init_idle(cpu_node, cpu);
+
+ of_node_put(cpu_node);
+
+ return ret;
+}
+
+static int __init psci_idle_init_cpu(int cpu)
+{
+ struct cpuidle_driver *drv;
+ struct device_node *cpu_node;
+ const char *enable_method;
+ int ret = 0;
+
+ cpu_node = of_cpu_device_node_get(cpu);
+ if (!cpu_node)
+ return -ENODEV;
+
+ /*
+ * Check whether the enable-method for the cpu is PSCI, fail
+ * if it is not.
+ */
+ enable_method = of_get_property(cpu_node, "enable-method", NULL);
+ if (!enable_method || (strcmp(enable_method, "psci")))
+ ret = -ENODEV;
+
+ of_node_put(cpu_node);
+ if (ret)
+ return ret;
+
+ drv = kmemdup(&psci_idle_driver, sizeof(*drv), GFP_KERNEL);
+ if (!drv)
+ return -ENOMEM;
+
+ drv->cpumask = (struct cpumask *)cpumask_of(cpu);
+
+ /*
+ * Initialize idle states data, starting at index 1, since
+ * by default idle state 0 is the quiescent state reached
+ * by the cpu by executing the wfi instruction.
+ *
+ * If no DT idle states are detected (ret == 0) let the driver
+ * initialization fail accordingly since there is no reason to
+ * initialize the idle driver if only wfi is supported, the
+ * default archictectural back-end already executes wfi
+ * on idle entry.
+ */
+ ret = dt_init_idle_driver(drv, psci_idle_state_match, 1);
+ if (ret <= 0) {
+ ret = ret ? : -ENODEV;
+ goto out_kfree_drv;
+ }
+
+ /*
+ * Initialize PSCI idle states.
+ */
+ ret = psci_cpu_init_idle(cpu);
+ if (ret) {
+ pr_err("CPU %d failed to PSCI idle\n", cpu);
+ goto out_kfree_drv;
+ }
+
+ ret = cpuidle_register(drv, NULL);
+ if (ret)
+ goto out_kfree_drv;
+
+ return 0;
+
+out_kfree_drv:
+ kfree(drv);
+ return ret;
+}
+
+/*
+ * psci_idle_init - Initializes PSCI cpuidle driver
+ *
+ * Initializes PSCI cpuidle driver for all CPUs, if any CPU fails
+ * to register cpuidle driver then rollback to cancel all CPUs
+ * registration.
+ */
+static int __init psci_idle_init(void)
+{
+ int cpu, ret;
+ struct cpuidle_driver *drv;
+ struct cpuidle_device *dev;
+
+ for_each_possible_cpu(cpu) {
+ ret = psci_idle_init_cpu(cpu);
+ if (ret)
+ goto out_fail;
+ }
+
+ return 0;
+
+out_fail:
+ while (--cpu >= 0) {
+ dev = per_cpu(cpuidle_devices, cpu);
+ drv = cpuidle_get_cpu_driver(dev);
+ cpuidle_unregister(drv);
+ kfree(drv);
+ }
+
+ return ret;
+}
+device_initcall(psci_idle_init);
diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c
index 0f4b7c45df3e..0895b988fa92 100644
--- a/drivers/cpuidle/cpuidle.c
+++ b/drivers/cpuidle/cpuidle.c
@@ -362,6 +362,36 @@ void cpuidle_reflect(struct cpuidle_device *dev, int index)
}
/**
+ * cpuidle_poll_time - return amount of time to poll for,
+ * governors can override dev->poll_limit_ns if necessary
+ *
+ * @drv: the cpuidle driver tied with the cpu
+ * @dev: the cpuidle device
+ *
+ */
+u64 cpuidle_poll_time(struct cpuidle_driver *drv,
+ struct cpuidle_device *dev)
+{
+ int i;
+ u64 limit_ns;
+
+ if (dev->poll_limit_ns)
+ return dev->poll_limit_ns;
+
+ limit_ns = TICK_NSEC;
+ for (i = 1; i < drv->state_count; i++) {
+ if (drv->states[i].disabled || dev->states_usage[i].disable)
+ continue;
+
+ limit_ns = (u64)drv->states[i].target_residency * NSEC_PER_USEC;
+ }
+
+ dev->poll_limit_ns = limit_ns;
+
+ return dev->poll_limit_ns;
+}
+
+/**
* cpuidle_install_idle_handler - installs the cpuidle idle loop handler
*/
void cpuidle_install_idle_handler(void)
diff --git a/drivers/cpuidle/cpuidle.h b/drivers/cpuidle/cpuidle.h
index d6613101af92..9f336af17fa6 100644
--- a/drivers/cpuidle/cpuidle.h
+++ b/drivers/cpuidle/cpuidle.h
@@ -9,6 +9,7 @@
/* For internal use only */
extern char param_governor[];
extern struct cpuidle_governor *cpuidle_curr_governor;
+extern struct cpuidle_governor *cpuidle_prev_governor;
extern struct list_head cpuidle_governors;
extern struct list_head cpuidle_detected_devices;
extern struct mutex cpuidle_lock;
@@ -22,6 +23,7 @@ extern void cpuidle_install_idle_handler(void);
extern void cpuidle_uninstall_idle_handler(void);
/* governors */
+extern struct cpuidle_governor *cpuidle_find_governor(const char *str);
extern int cpuidle_switch_governor(struct cpuidle_governor *gov);
/* sysfs */
diff --git a/drivers/cpuidle/driver.c b/drivers/cpuidle/driver.c
index dc32f34e68d9..80c1a830d991 100644
--- a/drivers/cpuidle/driver.c
+++ b/drivers/cpuidle/driver.c
@@ -254,12 +254,25 @@ static void __cpuidle_unregister_driver(struct cpuidle_driver *drv)
*/
int cpuidle_register_driver(struct cpuidle_driver *drv)
{
+ struct cpuidle_governor *gov;
int ret;
spin_lock(&cpuidle_driver_lock);
ret = __cpuidle_register_driver(drv);
spin_unlock(&cpuidle_driver_lock);
+ if (!ret && !strlen(param_governor) && drv->governor &&
+ (cpuidle_get_driver() == drv)) {
+ mutex_lock(&cpuidle_lock);
+ gov = cpuidle_find_governor(drv->governor);
+ if (gov) {
+ cpuidle_prev_governor = cpuidle_curr_governor;
+ if (cpuidle_switch_governor(gov) < 0)
+ cpuidle_prev_governor = NULL;
+ }
+ mutex_unlock(&cpuidle_lock);
+ }
+
return ret;
}
EXPORT_SYMBOL_GPL(cpuidle_register_driver);
@@ -274,9 +287,21 @@ EXPORT_SYMBOL_GPL(cpuidle_register_driver);
*/
void cpuidle_unregister_driver(struct cpuidle_driver *drv)
{
+ bool enabled = (cpuidle_get_driver() == drv);
+
spin_lock(&cpuidle_driver_lock);
__cpuidle_unregister_driver(drv);
spin_unlock(&cpuidle_driver_lock);
+
+ if (!enabled)
+ return;
+
+ mutex_lock(&cpuidle_lock);
+ if (cpuidle_prev_governor) {
+ if (!cpuidle_switch_governor(cpuidle_prev_governor))
+ cpuidle_prev_governor = NULL;
+ }
+ mutex_unlock(&cpuidle_lock);
}
EXPORT_SYMBOL_GPL(cpuidle_unregister_driver);
diff --git a/drivers/cpuidle/governor.c b/drivers/cpuidle/governor.c
index 9fddf828a76f..e9801f26c732 100644
--- a/drivers/cpuidle/governor.c
+++ b/drivers/cpuidle/governor.c
@@ -20,14 +20,15 @@ char param_governor[CPUIDLE_NAME_LEN];
LIST_HEAD(cpuidle_governors);
struct cpuidle_governor *cpuidle_curr_governor;
+struct cpuidle_governor *cpuidle_prev_governor;
/**
- * __cpuidle_find_governor - finds a governor of the specified name
+ * cpuidle_find_governor - finds a governor of the specified name
* @str: the name
*
* Must be called with cpuidle_lock acquired.
*/
-static struct cpuidle_governor * __cpuidle_find_governor(const char *str)
+struct cpuidle_governor *cpuidle_find_governor(const char *str)
{
struct cpuidle_governor *gov;
@@ -87,7 +88,7 @@ int cpuidle_register_governor(struct cpuidle_governor *gov)
return -ENODEV;
mutex_lock(&cpuidle_lock);
- if (__cpuidle_find_governor(gov->name) == NULL) {
+ if (cpuidle_find_governor(gov->name) == NULL) {
ret = 0;
list_add_tail(&gov->governor_list, &cpuidle_governors);
if (!cpuidle_curr_governor ||
@@ -110,7 +111,7 @@ int cpuidle_governor_latency_req(unsigned int cpu)
{
int global_req = pm_qos_request(PM_QOS_CPU_DMA_LATENCY);
struct device *device = get_cpu_device(cpu);
- int device_req = dev_pm_qos_raw_read_value(device);
+ int device_req = dev_pm_qos_raw_resume_latency(device);
return device_req < global_req ? device_req : global_req;
}
diff --git a/drivers/cpuidle/governors/Makefile b/drivers/cpuidle/governors/Makefile
index 42f44cc610dd..63abb5393a4d 100644
--- a/drivers/cpuidle/governors/Makefile
+++ b/drivers/cpuidle/governors/Makefile
@@ -6,3 +6,4 @@
obj-$(CONFIG_CPU_IDLE_GOV_LADDER) += ladder.o
obj-$(CONFIG_CPU_IDLE_GOV_MENU) += menu.o
obj-$(CONFIG_CPU_IDLE_GOV_TEO) += teo.o
+obj-$(CONFIG_CPU_IDLE_GOV_HALTPOLL) += haltpoll.o
diff --git a/drivers/cpuidle/governors/haltpoll.c b/drivers/cpuidle/governors/haltpoll.c
new file mode 100644
index 000000000000..7a703d2e0064
--- /dev/null
+++ b/drivers/cpuidle/governors/haltpoll.c
@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * haltpoll.c - haltpoll idle governor
+ *
+ * Copyright 2019 Red Hat, Inc. and/or its affiliates.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ *
+ * Authors: Marcelo Tosatti <mtosatti@redhat.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/cpuidle.h>
+#include <linux/time.h>
+#include <linux/ktime.h>
+#include <linux/hrtimer.h>
+#include <linux/tick.h>
+#include <linux/sched.h>
+#include <linux/module.h>
+#include <linux/kvm_para.h>
+
+static unsigned int guest_halt_poll_ns __read_mostly = 200000;
+module_param(guest_halt_poll_ns, uint, 0644);
+
+/* division factor to shrink halt_poll_ns */
+static unsigned int guest_halt_poll_shrink __read_mostly = 2;
+module_param(guest_halt_poll_shrink, uint, 0644);
+
+/* multiplication factor to grow per-cpu poll_limit_ns */
+static unsigned int guest_halt_poll_grow __read_mostly = 2;
+module_param(guest_halt_poll_grow, uint, 0644);
+
+/* value in us to start growing per-cpu halt_poll_ns */
+static unsigned int guest_halt_poll_grow_start __read_mostly = 50000;
+module_param(guest_halt_poll_grow_start, uint, 0644);
+
+/* allow shrinking guest halt poll */
+static bool guest_halt_poll_allow_shrink __read_mostly = true;
+module_param(guest_halt_poll_allow_shrink, bool, 0644);
+
+/**
+ * haltpoll_select - selects the next idle state to enter
+ * @drv: cpuidle driver containing state data
+ * @dev: the CPU
+ * @stop_tick: indication on whether or not to stop the tick
+ */
+static int haltpoll_select(struct cpuidle_driver *drv,
+ struct cpuidle_device *dev,
+ bool *stop_tick)
+{
+ int latency_req = cpuidle_governor_latency_req(dev->cpu);
+
+ if (!drv->state_count || latency_req == 0) {
+ *stop_tick = false;
+ return 0;
+ }
+
+ if (dev->poll_limit_ns == 0)
+ return 1;
+
+ /* Last state was poll? */
+ if (dev->last_state_idx == 0) {
+ /* Halt if no event occurred on poll window */
+ if (dev->poll_time_limit == true)
+ return 1;
+
+ *stop_tick = false;
+ /* Otherwise, poll again */
+ return 0;
+ }
+
+ *stop_tick = false;
+ /* Last state was halt: poll */
+ return 0;
+}
+
+static void adjust_poll_limit(struct cpuidle_device *dev, unsigned int block_us)
+{
+ unsigned int val;
+ u64 block_ns = block_us*NSEC_PER_USEC;
+
+ /* Grow cpu_halt_poll_us if
+ * cpu_halt_poll_us < block_ns < guest_halt_poll_us
+ */
+ if (block_ns > dev->poll_limit_ns && block_ns <= guest_halt_poll_ns) {
+ val = dev->poll_limit_ns * guest_halt_poll_grow;
+
+ if (val < guest_halt_poll_grow_start)
+ val = guest_halt_poll_grow_start;
+ if (val > guest_halt_poll_ns)
+ val = guest_halt_poll_ns;
+
+ dev->poll_limit_ns = val;
+ } else if (block_ns > guest_halt_poll_ns &&
+ guest_halt_poll_allow_shrink) {
+ unsigned int shrink = guest_halt_poll_shrink;
+
+ val = dev->poll_limit_ns;
+ if (shrink == 0)
+ val = 0;
+ else
+ val /= shrink;
+ dev->poll_limit_ns = val;
+ }
+}
+
+/**
+ * haltpoll_reflect - update variables and update poll time
+ * @dev: the CPU
+ * @index: the index of actual entered state
+ */
+static void haltpoll_reflect(struct cpuidle_device *dev, int index)
+{
+ dev->last_state_idx = index;
+
+ if (index != 0)
+ adjust_poll_limit(dev, dev->last_residency);
+}
+
+/**
+ * haltpoll_enable_device - scans a CPU's states and does setup
+ * @drv: cpuidle driver
+ * @dev: the CPU
+ */
+static int haltpoll_enable_device(struct cpuidle_driver *drv,
+ struct cpuidle_device *dev)
+{
+ dev->poll_limit_ns = 0;
+
+ return 0;
+}
+
+static struct cpuidle_governor haltpoll_governor = {
+ .name = "haltpoll",
+ .rating = 9,
+ .enable = haltpoll_enable_device,
+ .select = haltpoll_select,
+ .reflect = haltpoll_reflect,
+};
+
+static int __init init_haltpoll(void)
+{
+ if (kvm_para_available())
+ return cpuidle_register_governor(&haltpoll_governor);
+
+ return 0;
+}
+
+postcore_initcall(init_haltpoll);
diff --git a/drivers/cpuidle/governors/ladder.c b/drivers/cpuidle/governors/ladder.c
index f0dddc66af26..428eeb832fe7 100644
--- a/drivers/cpuidle/governors/ladder.c
+++ b/drivers/cpuidle/governors/ladder.c
@@ -38,7 +38,6 @@ struct ladder_device_state {
struct ladder_device {
struct ladder_device_state states[CPUIDLE_STATE_MAX];
- int last_state_idx;
};
static DEFINE_PER_CPU(struct ladder_device, ladder_devices);
@@ -49,12 +48,13 @@ static DEFINE_PER_CPU(struct ladder_device, ladder_devices);
* @old_idx: the current state index
* @new_idx: the new target state index
*/
-static inline void ladder_do_selection(struct ladder_device *ldev,
+static inline void ladder_do_selection(struct cpuidle_device *dev,
+ struct ladder_device *ldev,
int old_idx, int new_idx)
{
ldev->states[old_idx].stats.promotion_count = 0;
ldev->states[old_idx].stats.demotion_count = 0;
- ldev->last_state_idx = new_idx;
+ dev->last_state_idx = new_idx;
}
/**
@@ -68,13 +68,13 @@ static int ladder_select_state(struct cpuidle_driver *drv,
{
struct ladder_device *ldev = this_cpu_ptr(&ladder_devices);
struct ladder_device_state *last_state;
- int last_residency, last_idx = ldev->last_state_idx;
+ int last_residency, last_idx = dev->last_state_idx;
int first_idx = drv->states[0].flags & CPUIDLE_FLAG_POLLING ? 1 : 0;
int latency_req = cpuidle_governor_latency_req(dev->cpu);
/* Special case when user has set very strict latency requirement */
if (unlikely(latency_req == 0)) {
- ladder_do_selection(ldev, last_idx, 0);
+ ladder_do_selection(dev, ldev, last_idx, 0);
return 0;
}
@@ -91,7 +91,7 @@ static int ladder_select_state(struct cpuidle_driver *drv,
last_state->stats.promotion_count++;
last_state->stats.demotion_count = 0;
if (last_state->stats.promotion_count >= last_state->threshold.promotion_count) {
- ladder_do_selection(ldev, last_idx, last_idx + 1);
+ ladder_do_selection(dev, ldev, last_idx, last_idx + 1);
return last_idx + 1;
}
}
@@ -107,7 +107,7 @@ static int ladder_select_state(struct cpuidle_driver *drv,
if (drv->states[i].exit_latency <= latency_req)
break;
}
- ladder_do_selection(ldev, last_idx, i);
+ ladder_do_selection(dev, ldev, last_idx, i);
return i;
}
@@ -116,7 +116,7 @@ static int ladder_select_state(struct cpuidle_driver *drv,
last_state->stats.demotion_count++;
last_state->stats.promotion_count = 0;
if (last_state->stats.demotion_count >= last_state->threshold.demotion_count) {
- ladder_do_selection(ldev, last_idx, last_idx - 1);
+ ladder_do_selection(dev, ldev, last_idx, last_idx - 1);
return last_idx - 1;
}
}
@@ -139,7 +139,7 @@ static int ladder_enable_device(struct cpuidle_driver *drv,
struct ladder_device_state *lstate;
struct cpuidle_state *state;
- ldev->last_state_idx = first_idx;
+ dev->last_state_idx = first_idx;
for (i = first_idx; i < drv->state_count; i++) {
state = &drv->states[i];
@@ -167,9 +167,8 @@ static int ladder_enable_device(struct cpuidle_driver *drv,
*/
static void ladder_reflect(struct cpuidle_device *dev, int index)
{
- struct ladder_device *ldev = this_cpu_ptr(&ladder_devices);
if (index > 0)
- ldev->last_state_idx = index;
+ dev->last_state_idx = index;
}
static struct cpuidle_governor ladder_governor = {
diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
index e9a28c7846d6..e5a5d0c8d66b 100644
--- a/drivers/cpuidle/governors/menu.c
+++ b/drivers/cpuidle/governors/menu.c
@@ -117,7 +117,6 @@
*/
struct menu_device {
- int last_state_idx;
int needs_update;
int tick_wakeup;
@@ -302,9 +301,10 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
!drv->states[0].disabled && !dev->states_usage[0].disable)) {
/*
* In this case state[0] will be used no matter what, so return
- * it right away and keep the tick running.
+ * it right away and keep the tick running if state[0] is a
+ * polling one.
*/
- *stop_tick = false;
+ *stop_tick = !(drv->states[0].flags & CPUIDLE_FLAG_POLLING);
return 0;
}
@@ -395,16 +395,9 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
return idx;
}
- if (s->exit_latency > latency_req) {
- /*
- * If we break out of the loop for latency reasons, use
- * the target residency of the selected state as the
- * expected idle duration so that the tick is retained
- * as long as that target residency is low enough.
- */
- predicted_us = drv->states[idx].target_residency;
+ if (s->exit_latency > latency_req)
break;
- }
+
idx = i;
}
@@ -455,7 +448,7 @@ static void menu_reflect(struct cpuidle_device *dev, int index)
{
struct menu_device *data = this_cpu_ptr(&menu_devices);
- data->last_state_idx = index;
+ dev->last_state_idx = index;
data->needs_update = 1;
data->tick_wakeup = tick_nohz_idle_got_tick();
}
@@ -468,7 +461,7 @@ static void menu_reflect(struct cpuidle_device *dev, int index)
static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev)
{
struct menu_device *data = this_cpu_ptr(&menu_devices);
- int last_idx = data->last_state_idx;
+ int last_idx = dev->last_state_idx;
struct cpuidle_state *target = &drv->states[last_idx];
unsigned int measured_us;
unsigned int new_factor;
diff --git a/drivers/cpuidle/governors/teo.c b/drivers/cpuidle/governors/teo.c
index 7d05efdbd3c6..b5a0e498f798 100644
--- a/drivers/cpuidle/governors/teo.c
+++ b/drivers/cpuidle/governors/teo.c
@@ -96,7 +96,6 @@ struct teo_idle_state {
* @time_span_ns: Time between idle state selection and post-wakeup update.
* @sleep_length_ns: Time till the closest timer event (at the selection time).
* @states: Idle states data corresponding to this CPU.
- * @last_state: Idle state entered by the CPU last time.
* @interval_idx: Index of the most recent saved idle interval.
* @intervals: Saved idle duration values.
*/
@@ -104,7 +103,6 @@ struct teo_cpu {
u64 time_span_ns;
u64 sleep_length_ns;
struct teo_idle_state states[CPUIDLE_STATE_MAX];
- int last_state;
int interval_idx;
unsigned int intervals[INTERVALS];
};
@@ -125,12 +123,15 @@ static void teo_update(struct cpuidle_driver *drv, struct cpuidle_device *dev)
if (cpu_data->time_span_ns >= cpu_data->sleep_length_ns) {
/*
- * One of the safety nets has triggered or this was a timer
- * wakeup (or equivalent).
+ * One of the safety nets has triggered or the wakeup was close
+ * enough to the closest timer event expected at the idle state
+ * selection time to be discarded.
*/
- measured_us = sleep_length_us;
+ measured_us = UINT_MAX;
} else {
- unsigned int lat = drv->states[cpu_data->last_state].exit_latency;
+ unsigned int lat;
+
+ lat = drv->states[dev->last_state_idx].exit_latency;
measured_us = ktime_to_us(cpu_data->time_span_ns);
/*
@@ -189,15 +190,6 @@ static void teo_update(struct cpuidle_driver *drv, struct cpuidle_device *dev)
}
/*
- * If the total time span between idle state selection and the "reflect"
- * callback is greater than or equal to the sleep length determined at
- * the idle state selection time, the wakeup is likely to be due to a
- * timer event.
- */
- if (cpu_data->time_span_ns >= cpu_data->sleep_length_ns)
- measured_us = UINT_MAX;
-
- /*
* Save idle duration values corresponding to non-timer wakeups for
* pattern detection.
*/
@@ -242,12 +234,12 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);
int latency_req = cpuidle_governor_latency_req(dev->cpu);
unsigned int duration_us, count;
- int max_early_idx, idx, i;
+ int max_early_idx, constraint_idx, idx, i;
ktime_t delta_tick;
- if (cpu_data->last_state >= 0) {
+ if (dev->last_state_idx >= 0) {
teo_update(drv, dev);
- cpu_data->last_state = -1;
+ dev->last_state_idx = -1;
}
cpu_data->time_span_ns = local_clock();
@@ -257,6 +249,7 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
count = 0;
max_early_idx = -1;
+ constraint_idx = drv->state_count;
idx = -1;
for (i = 0; i < drv->state_count; i++) {
@@ -286,16 +279,8 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
if (s->target_residency > duration_us)
break;
- if (s->exit_latency > latency_req) {
- /*
- * If we break out of the loop for latency reasons, use
- * the target residency of the selected state as the
- * expected idle duration to avoid stopping the tick
- * as long as that target residency is low enough.
- */
- duration_us = drv->states[idx].target_residency;
- goto refine;
- }
+ if (s->exit_latency > latency_req && constraint_idx > i)
+ constraint_idx = i;
idx = i;
@@ -321,7 +306,13 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
duration_us = drv->states[idx].target_residency;
}
-refine:
+ /*
+ * If there is a latency constraint, it may be necessary to use a
+ * shallower idle state than the one selected so far.
+ */
+ if (constraint_idx < idx)
+ idx = constraint_idx;
+
if (idx < 0) {
idx = 0; /* No states enabled. Must use 0. */
} else if (idx > 0) {
@@ -331,13 +322,12 @@ refine:
/*
* Count and sum the most recent idle duration values less than
- * the target residency of the state selected so far, find the
- * max.
+ * the current expected idle duration value.
*/
for (i = 0; i < INTERVALS; i++) {
unsigned int val = cpu_data->intervals[i];
- if (val >= drv->states[idx].target_residency)
+ if (val >= duration_us)
continue;
count++;
@@ -356,8 +346,10 @@ refine:
* would be too shallow.
*/
if (!(tick_nohz_tick_stopped() && avg_us < TICK_USEC)) {
- idx = teo_find_shallower_state(drv, dev, idx, avg_us);
duration_us = avg_us;
+ if (drv->states[idx].target_residency > avg_us)
+ idx = teo_find_shallower_state(drv, dev,
+ idx, avg_us);
}
}
}
@@ -394,7 +386,7 @@ static void teo_reflect(struct cpuidle_device *dev, int state)
{
struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);
- cpu_data->last_state = state;
+ dev->last_state_idx = state;
/*
* If the wakeup was not "natural", but triggered by one of the safety
* nets, assume that the CPU might have been idle for the entire sleep
diff --git a/drivers/cpuidle/poll_state.c b/drivers/cpuidle/poll_state.c
index 02b9315a9e96..c8fa5f41dfc4 100644
--- a/drivers/cpuidle/poll_state.c
+++ b/drivers/cpuidle/poll_state.c
@@ -20,16 +20,9 @@ static int __cpuidle poll_idle(struct cpuidle_device *dev,
local_irq_enable();
if (!current_set_polling_and_test()) {
unsigned int loop_count = 0;
- u64 limit = TICK_NSEC;
- int i;
+ u64 limit;
- for (i = 1; i < drv->state_count; i++) {
- if (drv->states[i].disabled || dev->states_usage[i].disable)
- continue;
-
- limit = (u64)drv->states[i].target_residency * NSEC_PER_USEC;
- break;
- }
+ limit = cpuidle_poll_time(drv, dev);
while (!need_resched()) {
cpu_relax();
diff --git a/drivers/cpuidle/sysfs.c b/drivers/cpuidle/sysfs.c
index eb20adb5de23..2bb2683b493c 100644
--- a/drivers/cpuidle/sysfs.c
+++ b/drivers/cpuidle/sysfs.c
@@ -334,6 +334,7 @@ struct cpuidle_state_kobj {
struct cpuidle_state_usage *state_usage;
struct completion kobj_unregister;
struct kobject kobj;
+ struct cpuidle_device *device;
};
#ifdef CONFIG_SUSPEND
@@ -391,6 +392,7 @@ static inline void cpuidle_remove_s2idle_attr_group(struct cpuidle_state_kobj *k
#define kobj_to_state_obj(k) container_of(k, struct cpuidle_state_kobj, kobj)
#define kobj_to_state(k) (kobj_to_state_obj(k)->state)
#define kobj_to_state_usage(k) (kobj_to_state_obj(k)->state_usage)
+#define kobj_to_device(k) (kobj_to_state_obj(k)->device)
#define attr_to_stateattr(a) container_of(a, struct cpuidle_state_attr, attr)
static ssize_t cpuidle_state_show(struct kobject *kobj, struct attribute *attr,
@@ -414,10 +416,14 @@ static ssize_t cpuidle_state_store(struct kobject *kobj, struct attribute *attr,
struct cpuidle_state *state = kobj_to_state(kobj);
struct cpuidle_state_usage *state_usage = kobj_to_state_usage(kobj);
struct cpuidle_state_attr *cattr = attr_to_stateattr(attr);
+ struct cpuidle_device *dev = kobj_to_device(kobj);
if (cattr->store)
ret = cattr->store(state, state_usage, buf, size);
+ /* reset poll time cache */
+ dev->poll_limit_ns = 0;
+
return ret;
}
@@ -468,6 +474,7 @@ static int cpuidle_add_state_sysfs(struct cpuidle_device *device)
}
kobj->state = &drv->states[i];
kobj->state_usage = &device->states_usage[i];
+ kobj->device = device;
init_completion(&kobj->kobj_unregister);
ret = kobject_init_and_add(&kobj->kobj, &ktype_state_cpuidle,
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 0af08081e305..1fb622f2a87d 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -27,7 +27,7 @@ config CRYPTO_DEV_PADLOCK_AES
tristate "PadLock driver for AES algorithm"
depends on CRYPTO_DEV_PADLOCK
select CRYPTO_BLKCIPHER
- select CRYPTO_AES
+ select CRYPTO_LIB_AES
help
Use VIA PadLock for AES algorithm.
@@ -145,12 +145,32 @@ config CRYPTO_SHA512_S390
It is available as of z10.
+config CRYPTO_SHA3_256_S390
+ tristate "SHA3_224 and SHA3_256 digest algorithm"
+ depends on S390
+ select CRYPTO_HASH
+ help
+ This is the s390 hardware accelerated implementation of the
+ SHA3_256 secure hash standard.
+
+ It is available as of z14.
+
+config CRYPTO_SHA3_512_S390
+ tristate "SHA3_384 and SHA3_512 digest algorithm"
+ depends on S390
+ select CRYPTO_HASH
+ help
+ This is the s390 hardware accelerated implementation of the
+ SHA3_512 secure hash standard.
+
+ It is available as of z14.
+
config CRYPTO_DES_S390
tristate "DES and Triple DES cipher algorithms"
depends on S390
select CRYPTO_ALGAPI
select CRYPTO_BLKCIPHER
- select CRYPTO_DES
+ select CRYPTO_LIB_DES
help
This is the s390 hardware accelerated implementation of the
DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3).
@@ -189,12 +209,12 @@ config S390_PRNG
It is available as of z9.
config CRYPTO_GHASH_S390
- tristate "GHASH digest algorithm"
+ tristate "GHASH hash function"
depends on S390
select CRYPTO_HASH
help
- This is the s390 hardware accelerated implementation of the
- GHASH message digest algorithm for GCM (Galois/Counter Mode).
+ This is the s390 hardware accelerated implementation of GHASH,
+ the hash function used in GCM (Galois/Counter mode).
It is available as of z196.
@@ -214,8 +234,8 @@ config CRYPTO_CRC32_S390
config CRYPTO_DEV_MARVELL_CESA
tristate "Marvell's Cryptographic Engine driver"
depends on PLAT_ORION || ARCH_MVEBU
- select CRYPTO_AES
- select CRYPTO_DES
+ select CRYPTO_LIB_AES
+ select CRYPTO_LIB_DES
select CRYPTO_BLKCIPHER
select CRYPTO_HASH
select SRAM
@@ -227,7 +247,7 @@ config CRYPTO_DEV_MARVELL_CESA
config CRYPTO_DEV_NIAGARA2
tristate "Niagara2 Stream Processing Unit driver"
- select CRYPTO_DES
+ select CRYPTO_LIB_DES
select CRYPTO_BLKCIPHER
select CRYPTO_HASH
select CRYPTO_MD5
@@ -244,7 +264,7 @@ config CRYPTO_DEV_NIAGARA2
config CRYPTO_DEV_HIFN_795X
tristate "Driver HIFN 795x crypto accelerator chips"
- select CRYPTO_DES
+ select CRYPTO_LIB_DES
select CRYPTO_BLKCIPHER
select HW_RANDOM if CRYPTO_DEV_HIFN_795X_RNG
depends on PCI
@@ -300,7 +320,7 @@ config CRYPTO_DEV_TALITOS2
config CRYPTO_DEV_IXP4XX
tristate "Driver for IXP4xx crypto hardware acceleration"
depends on ARCH_IXP4XX && IXP4XX_QMGR && IXP4XX_NPE
- select CRYPTO_DES
+ select CRYPTO_LIB_DES
select CRYPTO_AEAD
select CRYPTO_AUTHENC
select CRYPTO_BLKCIPHER
@@ -312,7 +332,7 @@ config CRYPTO_DEV_PPC4XX
depends on PPC && 4xx
select CRYPTO_HASH
select CRYPTO_AEAD
- select CRYPTO_AES
+ select CRYPTO_LIB_AES
select CRYPTO_CCM
select CRYPTO_CTR
select CRYPTO_GCM
@@ -366,7 +386,7 @@ config CRYPTO_DEV_OMAP_AES
config CRYPTO_DEV_OMAP_DES
tristate "Support for OMAP DES/3DES hw engine"
depends on ARCH_OMAP2PLUS
- select CRYPTO_DES
+ select CRYPTO_LIB_DES
select CRYPTO_BLKCIPHER
select CRYPTO_ENGINE
help
@@ -384,7 +404,7 @@ config CRYPTO_DEV_PICOXCELL
select CRYPTO_AES
select CRYPTO_AUTHENC
select CRYPTO_BLKCIPHER
- select CRYPTO_DES
+ select CRYPTO_LIB_DES
select CRYPTO_CBC
select CRYPTO_ECB
select CRYPTO_SEQIV
@@ -393,7 +413,7 @@ config CRYPTO_DEV_PICOXCELL
Picochip picoXcell SoC devices. Select this for IPSEC ESP offload
and for 3gpp Layer 2 ciphering support.
- Saying m here will build a module named pipcoxcell_crypto.
+ Saying m here will build a module named picoxcell_crypto.
config CRYPTO_DEV_SAHARA
tristate "Support for SAHARA crypto accelerator"
@@ -497,7 +517,7 @@ config CRYPTO_DEV_ATMEL_AES
config CRYPTO_DEV_ATMEL_TDES
tristate "Support for Atmel DES/TDES hw accelerator"
depends on ARCH_AT91 || COMPILE_TEST
- select CRYPTO_DES
+ select CRYPTO_LIB_DES
select CRYPTO_BLKCIPHER
help
Some Atmel processors have DES/TDES hw accelerator.
@@ -520,10 +540,13 @@ config CRYPTO_DEV_ATMEL_SHA
To compile this driver as a module, choose M here: the module
will be called atmel-sha.
+config CRYPTO_DEV_ATMEL_I2C
+ tristate
+
config CRYPTO_DEV_ATMEL_ECC
tristate "Support for Microchip / Atmel ECC hw accelerator"
- depends on ARCH_AT91 || COMPILE_TEST
depends on I2C
+ select CRYPTO_DEV_ATMEL_I2C
select CRYPTO_ECDH
select CRC16
help
@@ -534,6 +557,21 @@ config CRYPTO_DEV_ATMEL_ECC
To compile this driver as a module, choose M here: the module
will be called atmel-ecc.
+config CRYPTO_DEV_ATMEL_SHA204A
+ tristate "Support for Microchip / Atmel SHA accelerator and RNG"
+ depends on I2C
+ select CRYPTO_DEV_ATMEL_I2C
+ select HW_RANDOM
+ select CRC16
+ help
+ Microhip / Atmel SHA accelerator and RNG.
+ Select this if you want to use the Microchip / Atmel SHA204A
+ module as a random number generator. (Other functions of the
+ chip are currently not exposed by this driver)
+
+ To compile this driver as a module, choose M here: the module
+ will be called atmel-sha204a.
+
config CRYPTO_DEV_CCP
bool "Support for AMD Secure Processor"
depends on ((X86 && PCI) || (ARM64 && (OF_ADDRESS || ACPI))) && HAS_IOMEM
@@ -577,7 +615,7 @@ config CRYPTO_DEV_QCE
depends on ARCH_QCOM || COMPILE_TEST
depends on HAS_IOMEM
select CRYPTO_AES
- select CRYPTO_DES
+ select CRYPTO_LIB_DES
select CRYPTO_ECB
select CRYPTO_CBC
select CRYPTO_XTS
@@ -625,7 +663,7 @@ config CRYPTO_DEV_SUN4I_SS
select CRYPTO_MD5
select CRYPTO_SHA1
select CRYPTO_AES
- select CRYPTO_DES
+ select CRYPTO_LIB_DES
select CRYPTO_BLKCIPHER
help
Some Allwinner SoC have a crypto accelerator named
@@ -648,7 +686,7 @@ config CRYPTO_DEV_ROCKCHIP
tristate "Rockchip's Cryptographic Engine driver"
depends on OF && ARCH_ROCKCHIP
select CRYPTO_AES
- select CRYPTO_DES
+ select CRYPTO_LIB_DES
select CRYPTO_MD5
select CRYPTO_SHA1
select CRYPTO_SHA256
@@ -685,7 +723,7 @@ config CRYPTO_DEV_BCM_SPU
depends on MAILBOX
default m
select CRYPTO_AUTHENC
- select CRYPTO_DES
+ select CRYPTO_LIB_DES
select CRYPTO_MD5
select CRYPTO_SHA1
select CRYPTO_SHA256
@@ -699,12 +737,11 @@ source "drivers/crypto/stm32/Kconfig"
config CRYPTO_DEV_SAFEXCEL
tristate "Inside Secure's SafeXcel cryptographic engine driver"
- depends on OF
- depends on (ARM64 && ARCH_MVEBU) || (COMPILE_TEST && 64BIT)
- select CRYPTO_AES
+ depends on OF || PCI || COMPILE_TEST
+ select CRYPTO_LIB_AES
select CRYPTO_AUTHENC
select CRYPTO_BLKCIPHER
- select CRYPTO_DES
+ select CRYPTO_LIB_DES
select CRYPTO_HASH
select CRYPTO_HMAC
select CRYPTO_MD5
@@ -712,10 +749,11 @@ config CRYPTO_DEV_SAFEXCEL
select CRYPTO_SHA256
select CRYPTO_SHA512
help
- This driver interfaces with the SafeXcel EIP-197 cryptographic engine
- designed by Inside Secure. Select this if you want to use CBC/ECB
- chain mode, AES cipher mode and SHA1/SHA224/SHA256/SHA512 hash
- algorithms.
+ This driver interfaces with the SafeXcel EIP-97 and EIP-197 cryptographic
+ engines designed by Inside Secure. It currently accelerates DES, 3DES and
+ AES block ciphers in ECB and CBC mode, as well as SHA1, SHA224, SHA256,
+ SHA384 and SHA512 hash algorithms for both basic hash and HMAC.
+ Additionally, it accelerates combined AES-CBC/HMAC-SHA AEAD operations.
config CRYPTO_DEV_ARTPEC6
tristate "Support for Axis ARTPEC-6/7 hardware crypto acceleration."
@@ -742,7 +780,7 @@ config CRYPTO_DEV_CCREE
default n
select CRYPTO_HASH
select CRYPTO_BLKCIPHER
- select CRYPTO_DES
+ select CRYPTO_LIB_DES
select CRYPTO_AEAD
select CRYPTO_AUTHENC
select CRYPTO_SHA1
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index a23a7197fcd7..afc4753b5d28 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -2,7 +2,9 @@
obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o
obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o
obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o
+obj-$(CONFIG_CRYPTO_DEV_ATMEL_I2C) += atmel-i2c.o
obj-$(CONFIG_CRYPTO_DEV_ATMEL_ECC) += atmel-ecc.o
+obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA204A) += atmel-sha204a.o
obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/
obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/
obj-$(CONFIG_CRYPTO_DEV_CCREE) += ccree/
diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c
index 49f3e0ce242c..a42f8619589d 100644
--- a/drivers/crypto/amcc/crypto4xx_alg.c
+++ b/drivers/crypto/amcc/crypto4xx_alg.c
@@ -67,12 +67,16 @@ static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm,
}
static inline int crypto4xx_crypt(struct skcipher_request *req,
- const unsigned int ivlen, bool decrypt)
+ const unsigned int ivlen, bool decrypt,
+ bool check_blocksize)
{
struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
__le32 iv[AES_IV_SIZE];
+ if (check_blocksize && !IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE))
+ return -EINVAL;
+
if (ivlen)
crypto4xx_memcpy_to_le32(iv, req->iv, ivlen);
@@ -81,24 +85,34 @@ static inline int crypto4xx_crypt(struct skcipher_request *req,
ctx->sa_len, 0, NULL);
}
-int crypto4xx_encrypt_noiv(struct skcipher_request *req)
+int crypto4xx_encrypt_noiv_block(struct skcipher_request *req)
+{
+ return crypto4xx_crypt(req, 0, false, true);
+}
+
+int crypto4xx_encrypt_iv_stream(struct skcipher_request *req)
+{
+ return crypto4xx_crypt(req, AES_IV_SIZE, false, false);
+}
+
+int crypto4xx_decrypt_noiv_block(struct skcipher_request *req)
{
- return crypto4xx_crypt(req, 0, false);
+ return crypto4xx_crypt(req, 0, true, true);
}
-int crypto4xx_encrypt_iv(struct skcipher_request *req)
+int crypto4xx_decrypt_iv_stream(struct skcipher_request *req)
{
- return crypto4xx_crypt(req, AES_IV_SIZE, false);
+ return crypto4xx_crypt(req, AES_IV_SIZE, true, false);
}
-int crypto4xx_decrypt_noiv(struct skcipher_request *req)
+int crypto4xx_encrypt_iv_block(struct skcipher_request *req)
{
- return crypto4xx_crypt(req, 0, true);
+ return crypto4xx_crypt(req, AES_IV_SIZE, false, true);
}
-int crypto4xx_decrypt_iv(struct skcipher_request *req)
+int crypto4xx_decrypt_iv_block(struct skcipher_request *req)
{
- return crypto4xx_crypt(req, AES_IV_SIZE, true);
+ return crypto4xx_crypt(req, AES_IV_SIZE, true, true);
}
/**
@@ -269,8 +283,8 @@ crypto4xx_ctr_crypt(struct skcipher_request *req, bool encrypt)
return ret;
}
- return encrypt ? crypto4xx_encrypt_iv(req)
- : crypto4xx_decrypt_iv(req);
+ return encrypt ? crypto4xx_encrypt_iv_stream(req)
+ : crypto4xx_decrypt_iv_stream(req);
}
static int crypto4xx_sk_setup_fallback(struct crypto4xx_ctx *ctx,
@@ -513,28 +527,20 @@ static int crypto4xx_aes_gcm_validate_keylen(unsigned int keylen)
static int crypto4xx_compute_gcm_hash_key_sw(__le32 *hash_start, const u8 *key,
unsigned int keylen)
{
- struct crypto_cipher *aes_tfm = NULL;
+ struct crypto_aes_ctx ctx;
uint8_t src[16] = { 0 };
- int rc = 0;
-
- aes_tfm = crypto_alloc_cipher("aes", 0, CRYPTO_ALG_NEED_FALLBACK);
- if (IS_ERR(aes_tfm)) {
- rc = PTR_ERR(aes_tfm);
- pr_warn("could not load aes cipher driver: %d\n", rc);
- return rc;
- }
+ int rc;
- rc = crypto_cipher_setkey(aes_tfm, key, keylen);
+ rc = aes_expandkey(&ctx, key, keylen);
if (rc) {
- pr_err("setkey() failed: %d\n", rc);
- goto out;
+ pr_err("aes_expandkey() failed: %d\n", rc);
+ return rc;
}
- crypto_cipher_encrypt_one(aes_tfm, src, src);
+ aes_encrypt(&ctx, src, src);
crypto4xx_memcpy_to_le32(hash_start, src, 16);
-out:
- crypto_free_cipher(aes_tfm);
- return rc;
+ memzero_explicit(&ctx, sizeof(ctx));
+ return 0;
}
int crypto4xx_setkey_aes_gcm(struct crypto_aead *cipher,
diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c
index 16d911aaa508..de5e9352e920 100644
--- a/drivers/crypto/amcc/crypto4xx_core.c
+++ b/drivers/crypto/amcc/crypto4xx_core.c
@@ -182,7 +182,6 @@ static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev)
dev->pdr_pa);
return -ENOMEM;
}
- memset(dev->pdr, 0, sizeof(struct ce_pd) * PPC4XX_NUM_PD);
dev->shadow_sa_pool = dma_alloc_coherent(dev->core_dev->device,
sizeof(union shadow_sa_buf) * PPC4XX_NUM_PD,
&dev->shadow_sa_pool_pa,
@@ -1210,8 +1209,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_IV_SIZE,
.setkey = crypto4xx_setkey_aes_cbc,
- .encrypt = crypto4xx_encrypt_iv,
- .decrypt = crypto4xx_decrypt_iv,
+ .encrypt = crypto4xx_encrypt_iv_block,
+ .decrypt = crypto4xx_decrypt_iv_block,
.init = crypto4xx_sk_init,
.exit = crypto4xx_sk_exit,
} },
@@ -1222,7 +1221,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
- .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_blocksize = 1,
.cra_ctxsize = sizeof(struct crypto4xx_ctx),
.cra_module = THIS_MODULE,
},
@@ -1230,8 +1229,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_IV_SIZE,
.setkey = crypto4xx_setkey_aes_cfb,
- .encrypt = crypto4xx_encrypt_iv,
- .decrypt = crypto4xx_decrypt_iv,
+ .encrypt = crypto4xx_encrypt_iv_stream,
+ .decrypt = crypto4xx_decrypt_iv_stream,
.init = crypto4xx_sk_init,
.exit = crypto4xx_sk_exit,
} },
@@ -1243,7 +1242,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
- .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_blocksize = 1,
.cra_ctxsize = sizeof(struct crypto4xx_ctx),
.cra_module = THIS_MODULE,
},
@@ -1263,7 +1262,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
- .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_blocksize = 1,
.cra_ctxsize = sizeof(struct crypto4xx_ctx),
.cra_module = THIS_MODULE,
},
@@ -1290,8 +1289,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.setkey = crypto4xx_setkey_aes_ecb,
- .encrypt = crypto4xx_encrypt_noiv,
- .decrypt = crypto4xx_decrypt_noiv,
+ .encrypt = crypto4xx_encrypt_noiv_block,
+ .decrypt = crypto4xx_decrypt_noiv_block,
.init = crypto4xx_sk_init,
.exit = crypto4xx_sk_exit,
} },
@@ -1302,7 +1301,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
- .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_blocksize = 1,
.cra_ctxsize = sizeof(struct crypto4xx_ctx),
.cra_module = THIS_MODULE,
},
@@ -1310,8 +1309,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_IV_SIZE,
.setkey = crypto4xx_setkey_aes_ofb,
- .encrypt = crypto4xx_encrypt_iv,
- .decrypt = crypto4xx_decrypt_iv,
+ .encrypt = crypto4xx_encrypt_iv_stream,
+ .decrypt = crypto4xx_decrypt_iv_stream,
.init = crypto4xx_sk_init,
.exit = crypto4xx_sk_exit,
} },
diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h
index ca1c25c40c23..6b6841359190 100644
--- a/drivers/crypto/amcc/crypto4xx_core.h
+++ b/drivers/crypto/amcc/crypto4xx_core.h
@@ -173,10 +173,12 @@ int crypto4xx_setkey_rfc3686(struct crypto_skcipher *cipher,
const u8 *key, unsigned int keylen);
int crypto4xx_encrypt_ctr(struct skcipher_request *req);
int crypto4xx_decrypt_ctr(struct skcipher_request *req);
-int crypto4xx_encrypt_iv(struct skcipher_request *req);
-int crypto4xx_decrypt_iv(struct skcipher_request *req);
-int crypto4xx_encrypt_noiv(struct skcipher_request *req);
-int crypto4xx_decrypt_noiv(struct skcipher_request *req);
+int crypto4xx_encrypt_iv_stream(struct skcipher_request *req);
+int crypto4xx_decrypt_iv_stream(struct skcipher_request *req);
+int crypto4xx_encrypt_iv_block(struct skcipher_request *req);
+int crypto4xx_decrypt_iv_block(struct skcipher_request *req);
+int crypto4xx_encrypt_noiv_block(struct skcipher_request *req);
+int crypto4xx_decrypt_noiv_block(struct skcipher_request *req);
int crypto4xx_rfc3686_encrypt(struct skcipher_request *req);
int crypto4xx_rfc3686_decrypt(struct skcipher_request *req);
int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm);
diff --git a/drivers/crypto/amcc/crypto4xx_trng.c b/drivers/crypto/amcc/crypto4xx_trng.c
index 02a6bed3b062..f10a87e541ed 100644
--- a/drivers/crypto/amcc/crypto4xx_trng.c
+++ b/drivers/crypto/amcc/crypto4xx_trng.c
@@ -108,7 +108,6 @@ void ppc4xx_trng_probe(struct crypto4xx_core_device *core_dev)
return;
err_out:
- of_node_put(trng);
iounmap(dev->trng_base);
kfree(rng);
dev->trng_base = NULL;
diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c
index 2b7af44c7b85..026f193556f9 100644
--- a/drivers/crypto/atmel-aes.c
+++ b/drivers/crypto/atmel-aes.c
@@ -2673,7 +2673,6 @@ static int atmel_aes_probe(struct platform_device *pdev)
/* Get the IRQ */
aes_dd->irq = platform_get_irq(pdev, 0);
if (aes_dd->irq < 0) {
- dev_err(dev, "no IRQ resource info\n");
err = aes_dd->irq;
goto res_err;
}
diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c
index ba00e4563ca0..ff02cc05affb 100644
--- a/drivers/crypto/atmel-ecc.c
+++ b/drivers/crypto/atmel-ecc.c
@@ -6,8 +6,6 @@
* Author: Tudor Ambarus <tudor.ambarus@microchip.com>
*/
-#include <linux/bitrev.h>
-#include <linux/crc16.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/err.h>
@@ -23,42 +21,11 @@
#include <crypto/internal/kpp.h>
#include <crypto/ecdh.h>
#include <crypto/kpp.h>
-#include "atmel-ecc.h"
-
-/* Used for binding tfm objects to i2c clients. */
-struct atmel_ecc_driver_data {
- struct list_head i2c_client_list;
- spinlock_t i2c_list_lock;
-} ____cacheline_aligned;
+#include "atmel-i2c.h"
static struct atmel_ecc_driver_data driver_data;
/**
- * atmel_ecc_i2c_client_priv - i2c_client private data
- * @client : pointer to i2c client device
- * @i2c_client_list_node: part of i2c_client_list
- * @lock : lock for sending i2c commands
- * @wake_token : wake token array of zeros
- * @wake_token_sz : size in bytes of the wake_token
- * @tfm_count : number of active crypto transformations on i2c client
- *
- * Reads and writes from/to the i2c client are sequential. The first byte
- * transmitted to the device is treated as the byte size. Any attempt to send
- * more than this number of bytes will cause the device to not ACK those bytes.
- * After the host writes a single command byte to the input buffer, reads are
- * prohibited until after the device completes command execution. Use a mutex
- * when sending i2c commands.
- */
-struct atmel_ecc_i2c_client_priv {
- struct i2c_client *client;
- struct list_head i2c_client_list_node;
- struct mutex lock;
- u8 wake_token[WAKE_TOKEN_MAX_SIZE];
- size_t wake_token_sz;
- atomic_t tfm_count ____cacheline_aligned;
-};
-
-/**
* atmel_ecdh_ctx - transformation context
* @client : pointer to i2c client device
* @fallback : used for unsupported curves or when user wants to use its own
@@ -80,188 +47,12 @@ struct atmel_ecdh_ctx {
bool do_fallback;
};
-/**
- * atmel_ecc_work_data - data structure representing the work
- * @ctx : transformation context.
- * @cbk : pointer to a callback function to be invoked upon completion of this
- * request. This has the form:
- * callback(struct atmel_ecc_work_data *work_data, void *areq, u8 status)
- * where:
- * @work_data: data structure representing the work
- * @areq : optional pointer to an argument passed with the original
- * request.
- * @status : status returned from the i2c client device or i2c error.
- * @areq: optional pointer to a user argument for use at callback time.
- * @work: describes the task to be executed.
- * @cmd : structure used for communicating with the device.
- */
-struct atmel_ecc_work_data {
- struct atmel_ecdh_ctx *ctx;
- void (*cbk)(struct atmel_ecc_work_data *work_data, void *areq,
- int status);
- void *areq;
- struct work_struct work;
- struct atmel_ecc_cmd cmd;
-};
-
-static u16 atmel_ecc_crc16(u16 crc, const u8 *buffer, size_t len)
-{
- return cpu_to_le16(bitrev16(crc16(crc, buffer, len)));
-}
-
-/**
- * atmel_ecc_checksum() - Generate 16-bit CRC as required by ATMEL ECC.
- * CRC16 verification of the count, opcode, param1, param2 and data bytes.
- * The checksum is saved in little-endian format in the least significant
- * two bytes of the command. CRC polynomial is 0x8005 and the initial register
- * value should be zero.
- *
- * @cmd : structure used for communicating with the device.
- */
-static void atmel_ecc_checksum(struct atmel_ecc_cmd *cmd)
-{
- u8 *data = &cmd->count;
- size_t len = cmd->count - CRC_SIZE;
- u16 *crc16 = (u16 *)(data + len);
-
- *crc16 = atmel_ecc_crc16(0, data, len);
-}
-
-static void atmel_ecc_init_read_cmd(struct atmel_ecc_cmd *cmd)
-{
- cmd->word_addr = COMMAND;
- cmd->opcode = OPCODE_READ;
- /*
- * Read the word from Configuration zone that contains the lock bytes
- * (UserExtra, Selector, LockValue, LockConfig).
- */
- cmd->param1 = CONFIG_ZONE;
- cmd->param2 = DEVICE_LOCK_ADDR;
- cmd->count = READ_COUNT;
-
- atmel_ecc_checksum(cmd);
-
- cmd->msecs = MAX_EXEC_TIME_READ;
- cmd->rxsize = READ_RSP_SIZE;
-}
-
-static void atmel_ecc_init_genkey_cmd(struct atmel_ecc_cmd *cmd, u16 keyid)
-{
- cmd->word_addr = COMMAND;
- cmd->count = GENKEY_COUNT;
- cmd->opcode = OPCODE_GENKEY;
- cmd->param1 = GENKEY_MODE_PRIVATE;
- /* a random private key will be generated and stored in slot keyID */
- cmd->param2 = cpu_to_le16(keyid);
-
- atmel_ecc_checksum(cmd);
-
- cmd->msecs = MAX_EXEC_TIME_GENKEY;
- cmd->rxsize = GENKEY_RSP_SIZE;
-}
-
-static int atmel_ecc_init_ecdh_cmd(struct atmel_ecc_cmd *cmd,
- struct scatterlist *pubkey)
-{
- size_t copied;
-
- cmd->word_addr = COMMAND;
- cmd->count = ECDH_COUNT;
- cmd->opcode = OPCODE_ECDH;
- cmd->param1 = ECDH_PREFIX_MODE;
- /* private key slot */
- cmd->param2 = cpu_to_le16(DATA_SLOT_2);
-
- /*
- * The device only supports NIST P256 ECC keys. The public key size will
- * always be the same. Use a macro for the key size to avoid unnecessary
- * computations.
- */
- copied = sg_copy_to_buffer(pubkey,
- sg_nents_for_len(pubkey,
- ATMEL_ECC_PUBKEY_SIZE),
- cmd->data, ATMEL_ECC_PUBKEY_SIZE);
- if (copied != ATMEL_ECC_PUBKEY_SIZE)
- return -EINVAL;
-
- atmel_ecc_checksum(cmd);
-
- cmd->msecs = MAX_EXEC_TIME_ECDH;
- cmd->rxsize = ECDH_RSP_SIZE;
-
- return 0;
-}
-
-/*
- * After wake and after execution of a command, there will be error, status, or
- * result bytes in the device's output register that can be retrieved by the
- * system. When the length of that group is four bytes, the codes returned are
- * detailed in error_list.
- */
-static int atmel_ecc_status(struct device *dev, u8 *status)
-{
- size_t err_list_len = ARRAY_SIZE(error_list);
- int i;
- u8 err_id = status[1];
-
- if (*status != STATUS_SIZE)
- return 0;
-
- if (err_id == STATUS_WAKE_SUCCESSFUL || err_id == STATUS_NOERR)
- return 0;
-
- for (i = 0; i < err_list_len; i++)
- if (error_list[i].value == err_id)
- break;
-
- /* if err_id is not in the error_list then ignore it */
- if (i != err_list_len) {
- dev_err(dev, "%02x: %s:\n", err_id, error_list[i].error_text);
- return err_id;
- }
-
- return 0;
-}
-
-static int atmel_ecc_wakeup(struct i2c_client *client)
-{
- struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
- u8 status[STATUS_RSP_SIZE];
- int ret;
-
- /*
- * The device ignores any levels or transitions on the SCL pin when the
- * device is idle, asleep or during waking up. Don't check for error
- * when waking up the device.
- */
- i2c_master_send(client, i2c_priv->wake_token, i2c_priv->wake_token_sz);
-
- /*
- * Wait to wake the device. Typical execution times for ecdh and genkey
- * are around tens of milliseconds. Delta is chosen to 50 microseconds.
- */
- usleep_range(TWHI_MIN, TWHI_MAX);
-
- ret = i2c_master_recv(client, status, STATUS_SIZE);
- if (ret < 0)
- return ret;
-
- return atmel_ecc_status(&client->dev, status);
-}
-
-static int atmel_ecc_sleep(struct i2c_client *client)
-{
- u8 sleep = SLEEP_TOKEN;
-
- return i2c_master_send(client, &sleep, 1);
-}
-
-static void atmel_ecdh_done(struct atmel_ecc_work_data *work_data, void *areq,
+static void atmel_ecdh_done(struct atmel_i2c_work_data *work_data, void *areq,
int status)
{
struct kpp_request *req = areq;
struct atmel_ecdh_ctx *ctx = work_data->ctx;
- struct atmel_ecc_cmd *cmd = &work_data->cmd;
+ struct atmel_i2c_cmd *cmd = &work_data->cmd;
size_t copied, n_sz;
if (status)
@@ -282,82 +73,6 @@ free_work_data:
kpp_request_complete(req, status);
}
-/*
- * atmel_ecc_send_receive() - send a command to the device and receive its
- * response.
- * @client: i2c client device
- * @cmd : structure used to communicate with the device
- *
- * After the device receives a Wake token, a watchdog counter starts within the
- * device. After the watchdog timer expires, the device enters sleep mode
- * regardless of whether some I/O transmission or command execution is in
- * progress. If a command is attempted when insufficient time remains prior to
- * watchdog timer execution, the device will return the watchdog timeout error
- * code without attempting to execute the command. There is no way to reset the
- * counter other than to put the device into sleep or idle mode and then
- * wake it up again.
- */
-static int atmel_ecc_send_receive(struct i2c_client *client,
- struct atmel_ecc_cmd *cmd)
-{
- struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
- int ret;
-
- mutex_lock(&i2c_priv->lock);
-
- ret = atmel_ecc_wakeup(client);
- if (ret)
- goto err;
-
- /* send the command */
- ret = i2c_master_send(client, (u8 *)cmd, cmd->count + WORD_ADDR_SIZE);
- if (ret < 0)
- goto err;
-
- /* delay the appropriate amount of time for command to execute */
- msleep(cmd->msecs);
-
- /* receive the response */
- ret = i2c_master_recv(client, cmd->data, cmd->rxsize);
- if (ret < 0)
- goto err;
-
- /* put the device into low-power mode */
- ret = atmel_ecc_sleep(client);
- if (ret < 0)
- goto err;
-
- mutex_unlock(&i2c_priv->lock);
- return atmel_ecc_status(&client->dev, cmd->data);
-err:
- mutex_unlock(&i2c_priv->lock);
- return ret;
-}
-
-static void atmel_ecc_work_handler(struct work_struct *work)
-{
- struct atmel_ecc_work_data *work_data =
- container_of(work, struct atmel_ecc_work_data, work);
- struct atmel_ecc_cmd *cmd = &work_data->cmd;
- struct i2c_client *client = work_data->ctx->client;
- int status;
-
- status = atmel_ecc_send_receive(client, cmd);
- work_data->cbk(work_data, work_data->areq, status);
-}
-
-static void atmel_ecc_enqueue(struct atmel_ecc_work_data *work_data,
- void (*cbk)(struct atmel_ecc_work_data *work_data,
- void *areq, int status),
- void *areq)
-{
- work_data->cbk = (void *)cbk;
- work_data->areq = areq;
-
- INIT_WORK(&work_data->work, atmel_ecc_work_handler);
- schedule_work(&work_data->work);
-}
-
static unsigned int atmel_ecdh_supported_curve(unsigned int curve_id)
{
if (curve_id == ECC_CURVE_NIST_P256)
@@ -374,7 +89,7 @@ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
unsigned int len)
{
struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
- struct atmel_ecc_cmd *cmd;
+ struct atmel_i2c_cmd *cmd;
void *public_key;
struct ecdh params;
int ret = -ENOMEM;
@@ -412,9 +127,9 @@ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
ctx->do_fallback = false;
ctx->curve_id = params.curve_id;
- atmel_ecc_init_genkey_cmd(cmd, DATA_SLOT_2);
+ atmel_i2c_init_genkey_cmd(cmd, DATA_SLOT_2);
- ret = atmel_ecc_send_receive(ctx->client, cmd);
+ ret = atmel_i2c_send_receive(ctx->client, cmd);
if (ret)
goto free_public_key;
@@ -444,6 +159,9 @@ static int atmel_ecdh_generate_public_key(struct kpp_request *req)
return crypto_kpp_generate_public_key(req);
}
+ if (!ctx->public_key)
+ return -EINVAL;
+
/* might want less than we've got */
nbytes = min_t(size_t, ATMEL_ECC_PUBKEY_SIZE, req->dst_len);
@@ -461,7 +179,7 @@ static int atmel_ecdh_compute_shared_secret(struct kpp_request *req)
{
struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
- struct atmel_ecc_work_data *work_data;
+ struct atmel_i2c_work_data *work_data;
gfp_t gfp;
int ret;
@@ -482,12 +200,13 @@ static int atmel_ecdh_compute_shared_secret(struct kpp_request *req)
return -ENOMEM;
work_data->ctx = ctx;
+ work_data->client = ctx->client;
- ret = atmel_ecc_init_ecdh_cmd(&work_data->cmd, req->src);
+ ret = atmel_i2c_init_ecdh_cmd(&work_data->cmd, req->src);
if (ret)
goto free_work_data;
- atmel_ecc_enqueue(work_data, atmel_ecdh_done, req);
+ atmel_i2c_enqueue(work_data, atmel_ecdh_done, req);
return -EINPROGRESS;
@@ -498,7 +217,7 @@ free_work_data:
static struct i2c_client *atmel_ecc_i2c_client_alloc(void)
{
- struct atmel_ecc_i2c_client_priv *i2c_priv, *min_i2c_priv = NULL;
+ struct atmel_i2c_client_priv *i2c_priv, *min_i2c_priv = NULL;
struct i2c_client *client = ERR_PTR(-ENODEV);
int min_tfm_cnt = INT_MAX;
int tfm_cnt;
@@ -533,7 +252,7 @@ static struct i2c_client *atmel_ecc_i2c_client_alloc(void)
static void atmel_ecc_i2c_client_free(struct i2c_client *client)
{
- struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
+ struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
atomic_dec(&i2c_priv->tfm_count);
}
@@ -604,96 +323,18 @@ static struct kpp_alg atmel_ecdh = {
},
};
-static inline size_t atmel_ecc_wake_token_sz(u32 bus_clk_rate)
-{
- u32 no_of_bits = DIV_ROUND_UP(TWLO_USEC * bus_clk_rate, USEC_PER_SEC);
-
- /* return the size of the wake_token in bytes */
- return DIV_ROUND_UP(no_of_bits, 8);
-}
-
-static int device_sanity_check(struct i2c_client *client)
-{
- struct atmel_ecc_cmd *cmd;
- int ret;
-
- cmd = kmalloc(sizeof(*cmd), GFP_KERNEL);
- if (!cmd)
- return -ENOMEM;
-
- atmel_ecc_init_read_cmd(cmd);
-
- ret = atmel_ecc_send_receive(client, cmd);
- if (ret)
- goto free_cmd;
-
- /*
- * It is vital that the Configuration, Data and OTP zones be locked
- * prior to release into the field of the system containing the device.
- * Failure to lock these zones may permit modification of any secret
- * keys and may lead to other security problems.
- */
- if (cmd->data[LOCK_CONFIG_IDX] || cmd->data[LOCK_VALUE_IDX]) {
- dev_err(&client->dev, "Configuration or Data and OTP zones are unlocked!\n");
- ret = -ENOTSUPP;
- }
-
- /* fall through */
-free_cmd:
- kfree(cmd);
- return ret;
-}
-
static int atmel_ecc_probe(struct i2c_client *client,
const struct i2c_device_id *id)
{
- struct atmel_ecc_i2c_client_priv *i2c_priv;
- struct device *dev = &client->dev;
+ struct atmel_i2c_client_priv *i2c_priv;
int ret;
- u32 bus_clk_rate;
-
- if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
- dev_err(dev, "I2C_FUNC_I2C not supported\n");
- return -ENODEV;
- }
- ret = of_property_read_u32(client->adapter->dev.of_node,
- "clock-frequency", &bus_clk_rate);
- if (ret) {
- dev_err(dev, "of: failed to read clock-frequency property\n");
- return ret;
- }
-
- if (bus_clk_rate > 1000000L) {
- dev_err(dev, "%d exceeds maximum supported clock frequency (1MHz)\n",
- bus_clk_rate);
- return -EINVAL;
- }
-
- i2c_priv = devm_kmalloc(dev, sizeof(*i2c_priv), GFP_KERNEL);
- if (!i2c_priv)
- return -ENOMEM;
-
- i2c_priv->client = client;
- mutex_init(&i2c_priv->lock);
-
- /*
- * WAKE_TOKEN_MAX_SIZE was calculated for the maximum bus_clk_rate -
- * 1MHz. The previous bus_clk_rate check ensures us that wake_token_sz
- * will always be smaller than or equal to WAKE_TOKEN_MAX_SIZE.
- */
- i2c_priv->wake_token_sz = atmel_ecc_wake_token_sz(bus_clk_rate);
-
- memset(i2c_priv->wake_token, 0, sizeof(i2c_priv->wake_token));
-
- atomic_set(&i2c_priv->tfm_count, 0);
-
- i2c_set_clientdata(client, i2c_priv);
-
- ret = device_sanity_check(client);
+ ret = atmel_i2c_probe(client, id);
if (ret)
return ret;
+ i2c_priv = i2c_get_clientdata(client);
+
spin_lock(&driver_data.i2c_list_lock);
list_add_tail(&i2c_priv->i2c_client_list_node,
&driver_data.i2c_client_list);
@@ -705,10 +346,10 @@ static int atmel_ecc_probe(struct i2c_client *client,
list_del(&i2c_priv->i2c_client_list_node);
spin_unlock(&driver_data.i2c_list_lock);
- dev_err(dev, "%s alg registration failed\n",
+ dev_err(&client->dev, "%s alg registration failed\n",
atmel_ecdh.base.cra_driver_name);
} else {
- dev_info(dev, "atmel ecc algorithms registered in /proc/crypto\n");
+ dev_info(&client->dev, "atmel ecc algorithms registered in /proc/crypto\n");
}
return ret;
@@ -716,7 +357,7 @@ static int atmel_ecc_probe(struct i2c_client *client,
static int atmel_ecc_remove(struct i2c_client *client)
{
- struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
+ struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
/* Return EBUSY if i2c client already allocated. */
if (atomic_read(&i2c_priv->tfm_count)) {
diff --git a/drivers/crypto/atmel-ecc.h b/drivers/crypto/atmel-ecc.h
deleted file mode 100644
index 643a3b947338..000000000000
--- a/drivers/crypto/atmel-ecc.h
+++ /dev/null
@@ -1,116 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (c) 2017, Microchip Technology Inc.
- * Author: Tudor Ambarus <tudor.ambarus@microchip.com>
- */
-
-#ifndef __ATMEL_ECC_H__
-#define __ATMEL_ECC_H__
-
-#define ATMEL_ECC_PRIORITY 300
-
-#define COMMAND 0x03 /* packet function */
-#define SLEEP_TOKEN 0x01
-#define WAKE_TOKEN_MAX_SIZE 8
-
-/* Definitions of Data and Command sizes */
-#define WORD_ADDR_SIZE 1
-#define COUNT_SIZE 1
-#define CRC_SIZE 2
-#define CMD_OVERHEAD_SIZE (COUNT_SIZE + CRC_SIZE)
-
-/* size in bytes of the n prime */
-#define ATMEL_ECC_NIST_P256_N_SIZE 32
-#define ATMEL_ECC_PUBKEY_SIZE (2 * ATMEL_ECC_NIST_P256_N_SIZE)
-
-#define STATUS_RSP_SIZE 4
-#define ECDH_RSP_SIZE (32 + CMD_OVERHEAD_SIZE)
-#define GENKEY_RSP_SIZE (ATMEL_ECC_PUBKEY_SIZE + \
- CMD_OVERHEAD_SIZE)
-#define READ_RSP_SIZE (4 + CMD_OVERHEAD_SIZE)
-#define MAX_RSP_SIZE GENKEY_RSP_SIZE
-
-/**
- * atmel_ecc_cmd - structure used for communicating with the device.
- * @word_addr: indicates the function of the packet sent to the device. This
- * byte should have a value of COMMAND for normal operation.
- * @count : number of bytes to be transferred to (or from) the device.
- * @opcode : the command code.
- * @param1 : the first parameter; always present.
- * @param2 : the second parameter; always present.
- * @data : optional remaining input data. Includes a 2-byte CRC.
- * @rxsize : size of the data received from i2c client.
- * @msecs : command execution time in milliseconds
- */
-struct atmel_ecc_cmd {
- u8 word_addr;
- u8 count;
- u8 opcode;
- u8 param1;
- u16 param2;
- u8 data[MAX_RSP_SIZE];
- u8 msecs;
- u16 rxsize;
-} __packed;
-
-/* Status/Error codes */
-#define STATUS_SIZE 0x04
-#define STATUS_NOERR 0x00
-#define STATUS_WAKE_SUCCESSFUL 0x11
-
-static const struct {
- u8 value;
- const char *error_text;
-} error_list[] = {
- { 0x01, "CheckMac or Verify miscompare" },
- { 0x03, "Parse Error" },
- { 0x05, "ECC Fault" },
- { 0x0F, "Execution Error" },
- { 0xEE, "Watchdog about to expire" },
- { 0xFF, "CRC or other communication error" },
-};
-
-/* Definitions for eeprom organization */
-#define CONFIG_ZONE 0
-
-/* Definitions for Indexes common to all commands */
-#define RSP_DATA_IDX 1 /* buffer index of data in response */
-#define DATA_SLOT_2 2 /* used for ECDH private key */
-
-/* Definitions for the device lock state */
-#define DEVICE_LOCK_ADDR 0x15
-#define LOCK_VALUE_IDX (RSP_DATA_IDX + 2)
-#define LOCK_CONFIG_IDX (RSP_DATA_IDX + 3)
-
-/*
- * Wake High delay to data communication (microseconds). SDA should be stable
- * high for this entire duration.
- */
-#define TWHI_MIN 1500
-#define TWHI_MAX 1550
-
-/* Wake Low duration */
-#define TWLO_USEC 60
-
-/* Command execution time (milliseconds) */
-#define MAX_EXEC_TIME_ECDH 58
-#define MAX_EXEC_TIME_GENKEY 115
-#define MAX_EXEC_TIME_READ 1
-
-/* Command opcode */
-#define OPCODE_ECDH 0x43
-#define OPCODE_GENKEY 0x40
-#define OPCODE_READ 0x02
-
-/* Definitions for the READ Command */
-#define READ_COUNT 7
-
-/* Definitions for the GenKey Command */
-#define GENKEY_COUNT 7
-#define GENKEY_MODE_PRIVATE 0x04
-
-/* Definitions for the ECDH Command */
-#define ECDH_COUNT 71
-#define ECDH_PREFIX_MODE 0x00
-
-#endif /* __ATMEL_ECC_H__ */
diff --git a/drivers/crypto/atmel-i2c.c b/drivers/crypto/atmel-i2c.c
new file mode 100644
index 000000000000..1d3355913b40
--- /dev/null
+++ b/drivers/crypto/atmel-i2c.c
@@ -0,0 +1,376 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Microchip / Atmel ECC (I2C) driver.
+ *
+ * Copyright (c) 2017, Microchip Technology Inc.
+ * Author: Tudor Ambarus <tudor.ambarus@microchip.com>
+ */
+
+#include <linux/bitrev.h>
+#include <linux/crc16.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/workqueue.h>
+#include "atmel-i2c.h"
+
+static const struct {
+ u8 value;
+ const char *error_text;
+} error_list[] = {
+ { 0x01, "CheckMac or Verify miscompare" },
+ { 0x03, "Parse Error" },
+ { 0x05, "ECC Fault" },
+ { 0x0F, "Execution Error" },
+ { 0xEE, "Watchdog about to expire" },
+ { 0xFF, "CRC or other communication error" },
+};
+
+/**
+ * atmel_i2c_checksum() - Generate 16-bit CRC as required by ATMEL ECC.
+ * CRC16 verification of the count, opcode, param1, param2 and data bytes.
+ * The checksum is saved in little-endian format in the least significant
+ * two bytes of the command. CRC polynomial is 0x8005 and the initial register
+ * value should be zero.
+ *
+ * @cmd : structure used for communicating with the device.
+ */
+static void atmel_i2c_checksum(struct atmel_i2c_cmd *cmd)
+{
+ u8 *data = &cmd->count;
+ size_t len = cmd->count - CRC_SIZE;
+ __le16 *__crc16 = (__le16 *)(data + len);
+
+ *__crc16 = cpu_to_le16(bitrev16(crc16(0, data, len)));
+}
+
+void atmel_i2c_init_read_cmd(struct atmel_i2c_cmd *cmd)
+{
+ cmd->word_addr = COMMAND;
+ cmd->opcode = OPCODE_READ;
+ /*
+ * Read the word from Configuration zone that contains the lock bytes
+ * (UserExtra, Selector, LockValue, LockConfig).
+ */
+ cmd->param1 = CONFIG_ZONE;
+ cmd->param2 = cpu_to_le16(DEVICE_LOCK_ADDR);
+ cmd->count = READ_COUNT;
+
+ atmel_i2c_checksum(cmd);
+
+ cmd->msecs = MAX_EXEC_TIME_READ;
+ cmd->rxsize = READ_RSP_SIZE;
+}
+EXPORT_SYMBOL(atmel_i2c_init_read_cmd);
+
+void atmel_i2c_init_random_cmd(struct atmel_i2c_cmd *cmd)
+{
+ cmd->word_addr = COMMAND;
+ cmd->opcode = OPCODE_RANDOM;
+ cmd->param1 = 0;
+ cmd->param2 = 0;
+ cmd->count = RANDOM_COUNT;
+
+ atmel_i2c_checksum(cmd);
+
+ cmd->msecs = MAX_EXEC_TIME_RANDOM;
+ cmd->rxsize = RANDOM_RSP_SIZE;
+}
+EXPORT_SYMBOL(atmel_i2c_init_random_cmd);
+
+void atmel_i2c_init_genkey_cmd(struct atmel_i2c_cmd *cmd, u16 keyid)
+{
+ cmd->word_addr = COMMAND;
+ cmd->count = GENKEY_COUNT;
+ cmd->opcode = OPCODE_GENKEY;
+ cmd->param1 = GENKEY_MODE_PRIVATE;
+ /* a random private key will be generated and stored in slot keyID */
+ cmd->param2 = cpu_to_le16(keyid);
+
+ atmel_i2c_checksum(cmd);
+
+ cmd->msecs = MAX_EXEC_TIME_GENKEY;
+ cmd->rxsize = GENKEY_RSP_SIZE;
+}
+EXPORT_SYMBOL(atmel_i2c_init_genkey_cmd);
+
+int atmel_i2c_init_ecdh_cmd(struct atmel_i2c_cmd *cmd,
+ struct scatterlist *pubkey)
+{
+ size_t copied;
+
+ cmd->word_addr = COMMAND;
+ cmd->count = ECDH_COUNT;
+ cmd->opcode = OPCODE_ECDH;
+ cmd->param1 = ECDH_PREFIX_MODE;
+ /* private key slot */
+ cmd->param2 = cpu_to_le16(DATA_SLOT_2);
+
+ /*
+ * The device only supports NIST P256 ECC keys. The public key size will
+ * always be the same. Use a macro for the key size to avoid unnecessary
+ * computations.
+ */
+ copied = sg_copy_to_buffer(pubkey,
+ sg_nents_for_len(pubkey,
+ ATMEL_ECC_PUBKEY_SIZE),
+ cmd->data, ATMEL_ECC_PUBKEY_SIZE);
+ if (copied != ATMEL_ECC_PUBKEY_SIZE)
+ return -EINVAL;
+
+ atmel_i2c_checksum(cmd);
+
+ cmd->msecs = MAX_EXEC_TIME_ECDH;
+ cmd->rxsize = ECDH_RSP_SIZE;
+
+ return 0;
+}
+EXPORT_SYMBOL(atmel_i2c_init_ecdh_cmd);
+
+/*
+ * After wake and after execution of a command, there will be error, status, or
+ * result bytes in the device's output register that can be retrieved by the
+ * system. When the length of that group is four bytes, the codes returned are
+ * detailed in error_list.
+ */
+static int atmel_i2c_status(struct device *dev, u8 *status)
+{
+ size_t err_list_len = ARRAY_SIZE(error_list);
+ int i;
+ u8 err_id = status[1];
+
+ if (*status != STATUS_SIZE)
+ return 0;
+
+ if (err_id == STATUS_WAKE_SUCCESSFUL || err_id == STATUS_NOERR)
+ return 0;
+
+ for (i = 0; i < err_list_len; i++)
+ if (error_list[i].value == err_id)
+ break;
+
+ /* if err_id is not in the error_list then ignore it */
+ if (i != err_list_len) {
+ dev_err(dev, "%02x: %s:\n", err_id, error_list[i].error_text);
+ return err_id;
+ }
+
+ return 0;
+}
+
+static int atmel_i2c_wakeup(struct i2c_client *client)
+{
+ struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
+ u8 status[STATUS_RSP_SIZE];
+ int ret;
+
+ /*
+ * The device ignores any levels or transitions on the SCL pin when the
+ * device is idle, asleep or during waking up. Don't check for error
+ * when waking up the device.
+ */
+ i2c_master_send(client, i2c_priv->wake_token, i2c_priv->wake_token_sz);
+
+ /*
+ * Wait to wake the device. Typical execution times for ecdh and genkey
+ * are around tens of milliseconds. Delta is chosen to 50 microseconds.
+ */
+ usleep_range(TWHI_MIN, TWHI_MAX);
+
+ ret = i2c_master_recv(client, status, STATUS_SIZE);
+ if (ret < 0)
+ return ret;
+
+ return atmel_i2c_status(&client->dev, status);
+}
+
+static int atmel_i2c_sleep(struct i2c_client *client)
+{
+ u8 sleep = SLEEP_TOKEN;
+
+ return i2c_master_send(client, &sleep, 1);
+}
+
+/*
+ * atmel_i2c_send_receive() - send a command to the device and receive its
+ * response.
+ * @client: i2c client device
+ * @cmd : structure used to communicate with the device
+ *
+ * After the device receives a Wake token, a watchdog counter starts within the
+ * device. After the watchdog timer expires, the device enters sleep mode
+ * regardless of whether some I/O transmission or command execution is in
+ * progress. If a command is attempted when insufficient time remains prior to
+ * watchdog timer execution, the device will return the watchdog timeout error
+ * code without attempting to execute the command. There is no way to reset the
+ * counter other than to put the device into sleep or idle mode and then
+ * wake it up again.
+ */
+int atmel_i2c_send_receive(struct i2c_client *client, struct atmel_i2c_cmd *cmd)
+{
+ struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
+ int ret;
+
+ mutex_lock(&i2c_priv->lock);
+
+ ret = atmel_i2c_wakeup(client);
+ if (ret)
+ goto err;
+
+ /* send the command */
+ ret = i2c_master_send(client, (u8 *)cmd, cmd->count + WORD_ADDR_SIZE);
+ if (ret < 0)
+ goto err;
+
+ /* delay the appropriate amount of time for command to execute */
+ msleep(cmd->msecs);
+
+ /* receive the response */
+ ret = i2c_master_recv(client, cmd->data, cmd->rxsize);
+ if (ret < 0)
+ goto err;
+
+ /* put the device into low-power mode */
+ ret = atmel_i2c_sleep(client);
+ if (ret < 0)
+ goto err;
+
+ mutex_unlock(&i2c_priv->lock);
+ return atmel_i2c_status(&client->dev, cmd->data);
+err:
+ mutex_unlock(&i2c_priv->lock);
+ return ret;
+}
+EXPORT_SYMBOL(atmel_i2c_send_receive);
+
+static void atmel_i2c_work_handler(struct work_struct *work)
+{
+ struct atmel_i2c_work_data *work_data =
+ container_of(work, struct atmel_i2c_work_data, work);
+ struct atmel_i2c_cmd *cmd = &work_data->cmd;
+ struct i2c_client *client = work_data->client;
+ int status;
+
+ status = atmel_i2c_send_receive(client, cmd);
+ work_data->cbk(work_data, work_data->areq, status);
+}
+
+void atmel_i2c_enqueue(struct atmel_i2c_work_data *work_data,
+ void (*cbk)(struct atmel_i2c_work_data *work_data,
+ void *areq, int status),
+ void *areq)
+{
+ work_data->cbk = (void *)cbk;
+ work_data->areq = areq;
+
+ INIT_WORK(&work_data->work, atmel_i2c_work_handler);
+ schedule_work(&work_data->work);
+}
+EXPORT_SYMBOL(atmel_i2c_enqueue);
+
+static inline size_t atmel_i2c_wake_token_sz(u32 bus_clk_rate)
+{
+ u32 no_of_bits = DIV_ROUND_UP(TWLO_USEC * bus_clk_rate, USEC_PER_SEC);
+
+ /* return the size of the wake_token in bytes */
+ return DIV_ROUND_UP(no_of_bits, 8);
+}
+
+static int device_sanity_check(struct i2c_client *client)
+{
+ struct atmel_i2c_cmd *cmd;
+ int ret;
+
+ cmd = kmalloc(sizeof(*cmd), GFP_KERNEL);
+ if (!cmd)
+ return -ENOMEM;
+
+ atmel_i2c_init_read_cmd(cmd);
+
+ ret = atmel_i2c_send_receive(client, cmd);
+ if (ret)
+ goto free_cmd;
+
+ /*
+ * It is vital that the Configuration, Data and OTP zones be locked
+ * prior to release into the field of the system containing the device.
+ * Failure to lock these zones may permit modification of any secret
+ * keys and may lead to other security problems.
+ */
+ if (cmd->data[LOCK_CONFIG_IDX] || cmd->data[LOCK_VALUE_IDX]) {
+ dev_err(&client->dev, "Configuration or Data and OTP zones are unlocked!\n");
+ ret = -ENOTSUPP;
+ }
+
+ /* fall through */
+free_cmd:
+ kfree(cmd);
+ return ret;
+}
+
+int atmel_i2c_probe(struct i2c_client *client, const struct i2c_device_id *id)
+{
+ struct atmel_i2c_client_priv *i2c_priv;
+ struct device *dev = &client->dev;
+ int ret;
+ u32 bus_clk_rate;
+
+ if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
+ dev_err(dev, "I2C_FUNC_I2C not supported\n");
+ return -ENODEV;
+ }
+
+ bus_clk_rate = i2c_acpi_find_bus_speed(&client->adapter->dev);
+ if (!bus_clk_rate) {
+ ret = device_property_read_u32(&client->adapter->dev,
+ "clock-frequency", &bus_clk_rate);
+ if (ret) {
+ dev_err(dev, "failed to read clock-frequency property\n");
+ return ret;
+ }
+ }
+
+ if (bus_clk_rate > 1000000L) {
+ dev_err(dev, "%d exceeds maximum supported clock frequency (1MHz)\n",
+ bus_clk_rate);
+ return -EINVAL;
+ }
+
+ i2c_priv = devm_kmalloc(dev, sizeof(*i2c_priv), GFP_KERNEL);
+ if (!i2c_priv)
+ return -ENOMEM;
+
+ i2c_priv->client = client;
+ mutex_init(&i2c_priv->lock);
+
+ /*
+ * WAKE_TOKEN_MAX_SIZE was calculated for the maximum bus_clk_rate -
+ * 1MHz. The previous bus_clk_rate check ensures us that wake_token_sz
+ * will always be smaller than or equal to WAKE_TOKEN_MAX_SIZE.
+ */
+ i2c_priv->wake_token_sz = atmel_i2c_wake_token_sz(bus_clk_rate);
+
+ memset(i2c_priv->wake_token, 0, sizeof(i2c_priv->wake_token));
+
+ atomic_set(&i2c_priv->tfm_count, 0);
+
+ i2c_set_clientdata(client, i2c_priv);
+
+ ret = device_sanity_check(client);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+EXPORT_SYMBOL(atmel_i2c_probe);
+
+MODULE_AUTHOR("Tudor Ambarus <tudor.ambarus@microchip.com>");
+MODULE_DESCRIPTION("Microchip / Atmel ECC (I2C) driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/crypto/atmel-i2c.h b/drivers/crypto/atmel-i2c.h
new file mode 100644
index 000000000000..63b97b104f16
--- /dev/null
+++ b/drivers/crypto/atmel-i2c.h
@@ -0,0 +1,185 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2017, Microchip Technology Inc.
+ * Author: Tudor Ambarus <tudor.ambarus@microchip.com>
+ */
+
+#ifndef __ATMEL_I2C_H__
+#define __ATMEL_I2C_H__
+
+#include <linux/hw_random.h>
+#include <linux/types.h>
+
+#define ATMEL_ECC_PRIORITY 300
+
+#define COMMAND 0x03 /* packet function */
+#define SLEEP_TOKEN 0x01
+#define WAKE_TOKEN_MAX_SIZE 8
+
+/* Definitions of Data and Command sizes */
+#define WORD_ADDR_SIZE 1
+#define COUNT_SIZE 1
+#define CRC_SIZE 2
+#define CMD_OVERHEAD_SIZE (COUNT_SIZE + CRC_SIZE)
+
+/* size in bytes of the n prime */
+#define ATMEL_ECC_NIST_P256_N_SIZE 32
+#define ATMEL_ECC_PUBKEY_SIZE (2 * ATMEL_ECC_NIST_P256_N_SIZE)
+
+#define STATUS_RSP_SIZE 4
+#define ECDH_RSP_SIZE (32 + CMD_OVERHEAD_SIZE)
+#define GENKEY_RSP_SIZE (ATMEL_ECC_PUBKEY_SIZE + \
+ CMD_OVERHEAD_SIZE)
+#define READ_RSP_SIZE (4 + CMD_OVERHEAD_SIZE)
+#define RANDOM_RSP_SIZE (32 + CMD_OVERHEAD_SIZE)
+#define MAX_RSP_SIZE GENKEY_RSP_SIZE
+
+/**
+ * atmel_i2c_cmd - structure used for communicating with the device.
+ * @word_addr: indicates the function of the packet sent to the device. This
+ * byte should have a value of COMMAND for normal operation.
+ * @count : number of bytes to be transferred to (or from) the device.
+ * @opcode : the command code.
+ * @param1 : the first parameter; always present.
+ * @param2 : the second parameter; always present.
+ * @data : optional remaining input data. Includes a 2-byte CRC.
+ * @rxsize : size of the data received from i2c client.
+ * @msecs : command execution time in milliseconds
+ */
+struct atmel_i2c_cmd {
+ u8 word_addr;
+ u8 count;
+ u8 opcode;
+ u8 param1;
+ __le16 param2;
+ u8 data[MAX_RSP_SIZE];
+ u8 msecs;
+ u16 rxsize;
+} __packed;
+
+/* Status/Error codes */
+#define STATUS_SIZE 0x04
+#define STATUS_NOERR 0x00
+#define STATUS_WAKE_SUCCESSFUL 0x11
+
+/* Definitions for eeprom organization */
+#define CONFIG_ZONE 0
+
+/* Definitions for Indexes common to all commands */
+#define RSP_DATA_IDX 1 /* buffer index of data in response */
+#define DATA_SLOT_2 2 /* used for ECDH private key */
+
+/* Definitions for the device lock state */
+#define DEVICE_LOCK_ADDR 0x15
+#define LOCK_VALUE_IDX (RSP_DATA_IDX + 2)
+#define LOCK_CONFIG_IDX (RSP_DATA_IDX + 3)
+
+/*
+ * Wake High delay to data communication (microseconds). SDA should be stable
+ * high for this entire duration.
+ */
+#define TWHI_MIN 1500
+#define TWHI_MAX 1550
+
+/* Wake Low duration */
+#define TWLO_USEC 60
+
+/* Command execution time (milliseconds) */
+#define MAX_EXEC_TIME_ECDH 58
+#define MAX_EXEC_TIME_GENKEY 115
+#define MAX_EXEC_TIME_READ 1
+#define MAX_EXEC_TIME_RANDOM 50
+
+/* Command opcode */
+#define OPCODE_ECDH 0x43
+#define OPCODE_GENKEY 0x40
+#define OPCODE_READ 0x02
+#define OPCODE_RANDOM 0x1b
+
+/* Definitions for the READ Command */
+#define READ_COUNT 7
+
+/* Definitions for the RANDOM Command */
+#define RANDOM_COUNT 7
+
+/* Definitions for the GenKey Command */
+#define GENKEY_COUNT 7
+#define GENKEY_MODE_PRIVATE 0x04
+
+/* Definitions for the ECDH Command */
+#define ECDH_COUNT 71
+#define ECDH_PREFIX_MODE 0x00
+
+/* Used for binding tfm objects to i2c clients. */
+struct atmel_ecc_driver_data {
+ struct list_head i2c_client_list;
+ spinlock_t i2c_list_lock;
+} ____cacheline_aligned;
+
+/**
+ * atmel_i2c_client_priv - i2c_client private data
+ * @client : pointer to i2c client device
+ * @i2c_client_list_node: part of i2c_client_list
+ * @lock : lock for sending i2c commands
+ * @wake_token : wake token array of zeros
+ * @wake_token_sz : size in bytes of the wake_token
+ * @tfm_count : number of active crypto transformations on i2c client
+ *
+ * Reads and writes from/to the i2c client are sequential. The first byte
+ * transmitted to the device is treated as the byte size. Any attempt to send
+ * more than this number of bytes will cause the device to not ACK those bytes.
+ * After the host writes a single command byte to the input buffer, reads are
+ * prohibited until after the device completes command execution. Use a mutex
+ * when sending i2c commands.
+ */
+struct atmel_i2c_client_priv {
+ struct i2c_client *client;
+ struct list_head i2c_client_list_node;
+ struct mutex lock;
+ u8 wake_token[WAKE_TOKEN_MAX_SIZE];
+ size_t wake_token_sz;
+ atomic_t tfm_count ____cacheline_aligned;
+ struct hwrng hwrng;
+};
+
+/**
+ * atmel_i2c_work_data - data structure representing the work
+ * @ctx : transformation context.
+ * @cbk : pointer to a callback function to be invoked upon completion of this
+ * request. This has the form:
+ * callback(struct atmel_i2c_work_data *work_data, void *areq, u8 status)
+ * where:
+ * @work_data: data structure representing the work
+ * @areq : optional pointer to an argument passed with the original
+ * request.
+ * @status : status returned from the i2c client device or i2c error.
+ * @areq: optional pointer to a user argument for use at callback time.
+ * @work: describes the task to be executed.
+ * @cmd : structure used for communicating with the device.
+ */
+struct atmel_i2c_work_data {
+ void *ctx;
+ struct i2c_client *client;
+ void (*cbk)(struct atmel_i2c_work_data *work_data, void *areq,
+ int status);
+ void *areq;
+ struct work_struct work;
+ struct atmel_i2c_cmd cmd;
+};
+
+int atmel_i2c_probe(struct i2c_client *client, const struct i2c_device_id *id);
+
+void atmel_i2c_enqueue(struct atmel_i2c_work_data *work_data,
+ void (*cbk)(struct atmel_i2c_work_data *work_data,
+ void *areq, int status),
+ void *areq);
+
+int atmel_i2c_send_receive(struct i2c_client *client, struct atmel_i2c_cmd *cmd);
+
+void atmel_i2c_init_read_cmd(struct atmel_i2c_cmd *cmd);
+void atmel_i2c_init_random_cmd(struct atmel_i2c_cmd *cmd);
+void atmel_i2c_init_genkey_cmd(struct atmel_i2c_cmd *cmd, u16 keyid);
+int atmel_i2c_init_ecdh_cmd(struct atmel_i2c_cmd *cmd,
+ struct scatterlist *pubkey);
+
+#endif /* __ATMEL_I2C_H__ */
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index ab0cfe748931..84cb8748a795 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -2779,7 +2779,6 @@ static int atmel_sha_probe(struct platform_device *pdev)
/* Get the IRQ */
sha_dd->irq = platform_get_irq(pdev, 0);
if (sha_dd->irq < 0) {
- dev_err(dev, "no IRQ resource info\n");
err = sha_dd->irq;
goto res_err;
}
diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c
new file mode 100644
index 000000000000..c96c14e7dab1
--- /dev/null
+++ b/drivers/crypto/atmel-sha204a.c
@@ -0,0 +1,170 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Microchip / Atmel SHA204A (I2C) driver.
+ *
+ * Copyright (c) 2019 Linaro, Ltd. <ard.biesheuvel@linaro.org>
+ */
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/workqueue.h>
+#include "atmel-i2c.h"
+
+static void atmel_sha204a_rng_done(struct atmel_i2c_work_data *work_data,
+ void *areq, int status)
+{
+ struct atmel_i2c_client_priv *i2c_priv = work_data->ctx;
+ struct hwrng *rng = areq;
+
+ if (status)
+ dev_warn_ratelimited(&i2c_priv->client->dev,
+ "i2c transaction failed (%d)\n",
+ status);
+
+ rng->priv = (unsigned long)work_data;
+ atomic_dec(&i2c_priv->tfm_count);
+}
+
+static int atmel_sha204a_rng_read_nonblocking(struct hwrng *rng, void *data,
+ size_t max)
+{
+ struct atmel_i2c_client_priv *i2c_priv;
+ struct atmel_i2c_work_data *work_data;
+
+ i2c_priv = container_of(rng, struct atmel_i2c_client_priv, hwrng);
+
+ /* keep maximum 1 asynchronous read in flight at any time */
+ if (!atomic_add_unless(&i2c_priv->tfm_count, 1, 1))
+ return 0;
+
+ if (rng->priv) {
+ work_data = (struct atmel_i2c_work_data *)rng->priv;
+ max = min(sizeof(work_data->cmd.data), max);
+ memcpy(data, &work_data->cmd.data, max);
+ rng->priv = 0;
+ } else {
+ work_data = kmalloc(sizeof(*work_data), GFP_ATOMIC);
+ if (!work_data)
+ return -ENOMEM;
+
+ work_data->ctx = i2c_priv;
+ work_data->client = i2c_priv->client;
+
+ max = 0;
+ }
+
+ atmel_i2c_init_random_cmd(&work_data->cmd);
+ atmel_i2c_enqueue(work_data, atmel_sha204a_rng_done, rng);
+
+ return max;
+}
+
+static int atmel_sha204a_rng_read(struct hwrng *rng, void *data, size_t max,
+ bool wait)
+{
+ struct atmel_i2c_client_priv *i2c_priv;
+ struct atmel_i2c_cmd cmd;
+ int ret;
+
+ if (!wait)
+ return atmel_sha204a_rng_read_nonblocking(rng, data, max);
+
+ i2c_priv = container_of(rng, struct atmel_i2c_client_priv, hwrng);
+
+ atmel_i2c_init_random_cmd(&cmd);
+
+ ret = atmel_i2c_send_receive(i2c_priv->client, &cmd);
+ if (ret)
+ return ret;
+
+ max = min(sizeof(cmd.data), max);
+ memcpy(data, cmd.data, max);
+
+ return max;
+}
+
+static int atmel_sha204a_probe(struct i2c_client *client,
+ const struct i2c_device_id *id)
+{
+ struct atmel_i2c_client_priv *i2c_priv;
+ int ret;
+
+ ret = atmel_i2c_probe(client, id);
+ if (ret)
+ return ret;
+
+ i2c_priv = i2c_get_clientdata(client);
+
+ memset(&i2c_priv->hwrng, 0, sizeof(i2c_priv->hwrng));
+
+ i2c_priv->hwrng.name = dev_name(&client->dev);
+ i2c_priv->hwrng.read = atmel_sha204a_rng_read;
+ i2c_priv->hwrng.quality = 1024;
+
+ ret = devm_hwrng_register(&client->dev, &i2c_priv->hwrng);
+ if (ret)
+ dev_warn(&client->dev, "failed to register RNG (%d)\n", ret);
+
+ return ret;
+}
+
+static int atmel_sha204a_remove(struct i2c_client *client)
+{
+ struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
+
+ if (atomic_read(&i2c_priv->tfm_count)) {
+ dev_err(&client->dev, "Device is busy\n");
+ return -EBUSY;
+ }
+
+ if (i2c_priv->hwrng.priv)
+ kfree((void *)i2c_priv->hwrng.priv);
+
+ return 0;
+}
+
+static const struct of_device_id atmel_sha204a_dt_ids[] = {
+ { .compatible = "atmel,atsha204a", },
+ { /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, atmel_sha204a_dt_ids);
+
+static const struct i2c_device_id atmel_sha204a_id[] = {
+ { "atsha204a", 0 },
+ { /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(i2c, atmel_sha204a_id);
+
+static struct i2c_driver atmel_sha204a_driver = {
+ .probe = atmel_sha204a_probe,
+ .remove = atmel_sha204a_remove,
+ .id_table = atmel_sha204a_id,
+
+ .driver.name = "atmel-sha204a",
+ .driver.of_match_table = of_match_ptr(atmel_sha204a_dt_ids),
+};
+
+static int __init atmel_sha204a_init(void)
+{
+ return i2c_add_driver(&atmel_sha204a_driver);
+}
+
+static void __exit atmel_sha204a_exit(void)
+{
+ flush_scheduled_work();
+ i2c_del_driver(&atmel_sha204a_driver);
+}
+
+module_init(atmel_sha204a_init);
+module_exit(atmel_sha204a_exit);
+
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c
index fa76620281e8..1a6c86ae6148 100644
--- a/drivers/crypto/atmel-tdes.c
+++ b/drivers/crypto/atmel-tdes.c
@@ -33,7 +33,7 @@
#include <linux/cryptohash.h>
#include <crypto/scatterwalk.h>
#include <crypto/algapi.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <crypto/hash.h>
#include <crypto/internal/hash.h>
#include <linux/platform_data/crypto-atmel.h>
@@ -773,22 +773,12 @@ static void atmel_tdes_dma_cleanup(struct atmel_tdes_dev *dd)
static int atmel_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
unsigned int keylen)
{
- u32 tmp[DES_EXPKEY_WORDS];
- int err;
- struct crypto_tfm *ctfm = crypto_ablkcipher_tfm(tfm);
-
struct atmel_tdes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+ int err;
- if (keylen != DES_KEY_SIZE) {
- crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
- return -EINVAL;
- }
-
- err = des_ekey(tmp, key);
- if (err == 0 && (ctfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
- ctfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
- return -EINVAL;
- }
+ err = verify_ablkcipher_des_key(tfm, key);
+ if (err)
+ return err;
memcpy(ctx->key, key, keylen);
ctx->keylen = keylen;
@@ -800,15 +790,11 @@ static int atmel_tdes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
unsigned int keylen)
{
struct atmel_tdes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
- u32 flags;
int err;
- flags = crypto_ablkcipher_get_flags(tfm);
- err = __des3_verify_key(&flags, key);
- if (unlikely(err)) {
- crypto_ablkcipher_set_flags(tfm, flags);
+ err = verify_ablkcipher_des3_key(tfm, key);
+ if (err)
return err;
- }
memcpy(ctx->key, key, keylen);
ctx->keylen = keylen;
@@ -1281,7 +1267,6 @@ static int atmel_tdes_probe(struct platform_device *pdev)
/* Get the IRQ */
tdes_dd->irq = platform_get_irq(pdev, 0);
if (tdes_dd->irq < 0) {
- dev_err(dev, "no IRQ resource info\n");
err = tdes_dd->irq;
goto res_err;
}
diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c
index 80fa04ef215f..4b20606983a4 100644
--- a/drivers/crypto/axis/artpec6_crypto.c
+++ b/drivers/crypto/axis/artpec6_crypto.c
@@ -2854,7 +2854,6 @@ static int artpec6_crypto_probe(struct platform_device *pdev)
struct artpec6_crypto *ac;
struct device *dev = &pdev->dev;
void __iomem *base;
- struct resource *res;
int irq;
int err;
@@ -2867,8 +2866,7 @@ static int artpec6_crypto_probe(struct platform_device *pdev)
variant = (enum artpec6_crypto_variant)match->data;
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- base = devm_ioremap_resource(&pdev->dev, res);
+ base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(base))
return PTR_ERR(base);
diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c
index 18410c9e7b29..f85356a48e7e 100644
--- a/drivers/crypto/bcm/cipher.c
+++ b/drivers/crypto/bcm/cipher.c
@@ -24,7 +24,7 @@
#include <crypto/aead.h>
#include <crypto/internal/aead.h>
#include <crypto/aes.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <crypto/hmac.h>
#include <crypto/sha.h>
#include <crypto/md5.h>
@@ -85,7 +85,7 @@ MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos");
* 0x70 - ring 2
* 0x78 - ring 3
*/
-char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 };
+static char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 };
/*
* Some SPU hw does not use BCM header on SPU messages. So BCM_HDR_LEN
* is set dynamically after reading SPU type from device tree.
@@ -1802,24 +1802,13 @@ static int des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
unsigned int keylen)
{
struct iproc_ctx_s *ctx = crypto_ablkcipher_ctx(cipher);
- u32 tmp[DES_EXPKEY_WORDS];
-
- if (keylen == DES_KEY_SIZE) {
- if (des_ekey(tmp, key) == 0) {
- if (crypto_ablkcipher_get_flags(cipher) &
- CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) {
- u32 flags = CRYPTO_TFM_RES_WEAK_KEY;
+ int err;
- crypto_ablkcipher_set_flags(cipher, flags);
- return -EINVAL;
- }
- }
+ err = verify_ablkcipher_des_key(cipher, key);
+ if (err)
+ return err;
- ctx->cipher_type = CIPHER_TYPE_DES;
- } else {
- crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
- return -EINVAL;
- }
+ ctx->cipher_type = CIPHER_TYPE_DES;
return 0;
}
@@ -1827,23 +1816,13 @@ static int threedes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
unsigned int keylen)
{
struct iproc_ctx_s *ctx = crypto_ablkcipher_ctx(cipher);
+ int err;
- if (keylen == (DES_KEY_SIZE * 3)) {
- u32 flags;
- int ret;
-
- flags = crypto_ablkcipher_get_flags(cipher);
- ret = __des3_verify_key(&flags, key);
- if (unlikely(ret)) {
- crypto_ablkcipher_set_flags(cipher, flags);
- return ret;
- }
+ err = verify_ablkcipher_des3_key(cipher, key);
+ if (err)
+ return err;
- ctx->cipher_type = CIPHER_TYPE_3DES;
- } else {
- crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
- return -EINVAL;
- }
+ ctx->cipher_type = CIPHER_TYPE_3DES;
return 0;
}
@@ -2083,7 +2062,7 @@ static int __ahash_init(struct ahash_request *req)
* Return: true if incremental hashing is not supported
* false otherwise
*/
-bool spu_no_incr_hash(struct iproc_ctx_s *ctx)
+static bool spu_no_incr_hash(struct iproc_ctx_s *ctx)
{
struct spu_hw *spu = &iproc_priv.spu;
@@ -2629,6 +2608,19 @@ static int aead_need_fallback(struct aead_request *req)
return 1;
}
+ /*
+ * RFC4106 and RFC4543 cannot handle the case where AAD is other than
+ * 16 or 20 bytes long. So use fallback in this case.
+ */
+ if (ctx->cipher.mode == CIPHER_MODE_GCM &&
+ ctx->cipher.alg == CIPHER_ALG_AES &&
+ rctx->iv_ctr_len == GCM_RFC4106_IV_SIZE &&
+ req->assoclen != 16 && req->assoclen != 20) {
+ flow_log("RFC4106/RFC4543 needs fallback for assoclen"
+ " other than 16 or 20 bytes\n");
+ return 1;
+ }
+
payload_len = req->cryptlen;
if (spu->spu_type == SPU_TYPE_SPUM)
payload_len += req->assoclen;
@@ -2855,40 +2847,16 @@ static int aead_authenc_setkey(struct crypto_aead *cipher,
switch (ctx->alg->cipher_info.alg) {
case CIPHER_ALG_DES:
- if (ctx->enckeylen == DES_KEY_SIZE) {
- u32 tmp[DES_EXPKEY_WORDS];
- u32 flags = CRYPTO_TFM_RES_WEAK_KEY;
-
- if (des_ekey(tmp, keys.enckey) == 0) {
- if (crypto_aead_get_flags(cipher) &
- CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) {
- crypto_aead_set_flags(cipher, flags);
- return -EINVAL;
- }
- }
+ if (verify_aead_des_key(cipher, keys.enckey, keys.enckeylen))
+ return -EINVAL;
- ctx->cipher_type = CIPHER_TYPE_DES;
- } else {
- goto badkey;
- }
+ ctx->cipher_type = CIPHER_TYPE_DES;
break;
case CIPHER_ALG_3DES:
- if (ctx->enckeylen == (DES_KEY_SIZE * 3)) {
- u32 flags;
-
- flags = crypto_aead_get_flags(cipher);
- ret = __des3_verify_key(&flags, keys.enckey);
- if (unlikely(ret)) {
- crypto_aead_set_flags(cipher, flags);
- return ret;
- }
-
- ctx->cipher_type = CIPHER_TYPE_3DES;
- } else {
- crypto_aead_set_flags(cipher,
- CRYPTO_TFM_RES_BAD_KEY_LEN);
+ if (verify_aead_des3_key(cipher, keys.enckey, keys.enckeylen))
return -EINVAL;
- }
+
+ ctx->cipher_type = CIPHER_TYPE_3DES;
break;
case CIPHER_ALG_AES:
switch (ctx->enckeylen) {
@@ -4809,7 +4777,7 @@ static int spu_dt_read(struct platform_device *pdev)
return 0;
}
-int bcm_spu_probe(struct platform_device *pdev)
+static int bcm_spu_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct spu_hw *spu = &iproc_priv.spu;
@@ -4853,7 +4821,7 @@ failure:
return err;
}
-int bcm_spu_remove(struct platform_device *pdev)
+static int bcm_spu_remove(struct platform_device *pdev)
{
int i;
struct device *dev = &pdev->dev;
diff --git a/drivers/crypto/bcm/spu2.c b/drivers/crypto/bcm/spu2.c
index cb477259a2e2..2add51024575 100644
--- a/drivers/crypto/bcm/spu2.c
+++ b/drivers/crypto/bcm/spu2.c
@@ -38,21 +38,21 @@ enum spu2_proto_sel {
SPU2_DTLS_AEAD = 10
};
-char *spu2_cipher_type_names[] = { "None", "AES128", "AES192", "AES256",
+static char *spu2_cipher_type_names[] = { "None", "AES128", "AES192", "AES256",
"DES", "3DES"
};
-char *spu2_cipher_mode_names[] = { "ECB", "CBC", "CTR", "CFB", "OFB", "XTS",
- "CCM", "GCM"
+static char *spu2_cipher_mode_names[] = { "ECB", "CBC", "CTR", "CFB", "OFB",
+ "XTS", "CCM", "GCM"
};
-char *spu2_hash_type_names[] = { "None", "AES128", "AES192", "AES256",
+static char *spu2_hash_type_names[] = { "None", "AES128", "AES192", "AES256",
"Reserved", "Reserved", "MD5", "SHA1", "SHA224", "SHA256", "SHA384",
"SHA512", "SHA512/224", "SHA512/256", "SHA3-224", "SHA3-256",
"SHA3-384", "SHA3-512"
};
-char *spu2_hash_mode_names[] = { "CMAC", "CBC-MAC", "XCBC-MAC", "HMAC",
+static char *spu2_hash_mode_names[] = { "CMAC", "CBC-MAC", "XCBC-MAC", "HMAC",
"Rabin", "CCM", "GCM", "Reserved"
};
diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig
index 577c9844b322..137ed3df0c74 100644
--- a/drivers/crypto/caam/Kconfig
+++ b/drivers/crypto/caam/Kconfig
@@ -2,6 +2,12 @@
config CRYPTO_DEV_FSL_CAAM_COMMON
tristate
+config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
+ tristate
+
+config CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC
+ tristate
+
config CRYPTO_DEV_FSL_CAAM
tristate "Freescale CAAM-Multicore platform driver backend"
depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE
@@ -25,7 +31,7 @@ config CRYPTO_DEV_FSL_CAAM_DEBUG
Selecting this will enable printing of various debug
information in the CAAM driver.
-config CRYPTO_DEV_FSL_CAAM_JR
+menuconfig CRYPTO_DEV_FSL_CAAM_JR
tristate "Freescale CAAM Job Ring driver backend"
default y
help
@@ -86,26 +92,26 @@ config CRYPTO_DEV_FSL_CAAM_INTC_TIME_THLD
threshold. Range is 1-65535.
config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
- tristate "Register algorithm implementations with the Crypto API"
+ bool "Register algorithm implementations with the Crypto API"
default y
+ select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
select CRYPTO_AEAD
select CRYPTO_AUTHENC
select CRYPTO_BLKCIPHER
- select CRYPTO_DES
+ select CRYPTO_LIB_DES
help
Selecting this will offload crypto for users of the
scatterlist crypto API (such as the linux native IPSec
stack) to the SEC4 via job ring.
- To compile this as a module, choose M here: the module
- will be called caamalg.
-
config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI
- tristate "Queue Interface as Crypto API backend"
+ bool "Queue Interface as Crypto API backend"
depends on FSL_DPAA && NET
default y
+ select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
select CRYPTO_AUTHENC
select CRYPTO_BLKCIPHER
+ select CRYPTO_DES
help
Selecting this will use CAAM Queue Interface (QI) for sending
& receiving crypto jobs to/from CAAM. This gives better performance
@@ -114,33 +120,26 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI
assigned to the kernel should also be more than the number of
job rings.
- To compile this as a module, choose M here: the module
- will be called caamalg_qi.
-
config CRYPTO_DEV_FSL_CAAM_AHASH_API
- tristate "Register hash algorithm implementations with Crypto API"
+ bool "Register hash algorithm implementations with Crypto API"
default y
+ select CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC
select CRYPTO_HASH
help
Selecting this will offload ahash for users of the
scatterlist crypto API to the SEC4 via job ring.
- To compile this as a module, choose M here: the module
- will be called caamhash.
-
config CRYPTO_DEV_FSL_CAAM_PKC_API
- tristate "Register public key cryptography implementations with Crypto API"
+ bool "Register public key cryptography implementations with Crypto API"
default y
select CRYPTO_RSA
help
Selecting this will allow SEC Public key support for RSA.
Supported cryptographic primitives: encryption, decryption,
signature and verification.
- To compile this as a module, choose M here: the module
- will be called caam_pkc.
config CRYPTO_DEV_FSL_CAAM_RNG_API
- tristate "Register caam device for hwrng API"
+ bool "Register caam device for hwrng API"
default y
select CRYPTO_RNG
select HW_RANDOM
@@ -148,9 +147,6 @@ config CRYPTO_DEV_FSL_CAAM_RNG_API
Selecting this will register the SEC4 hardware rng to
the hw_random API for suppying the kernel entropy pool.
- To compile this as a module, choose M here: the module
- will be called caamrng.
-
endif # CRYPTO_DEV_FSL_CAAM_JR
endif # CRYPTO_DEV_FSL_CAAM
@@ -160,10 +156,13 @@ config CRYPTO_DEV_FSL_DPAA2_CAAM
depends on FSL_MC_DPIO
depends on NETDEVICES
select CRYPTO_DEV_FSL_CAAM_COMMON
+ select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
+ select CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC
select CRYPTO_BLKCIPHER
select CRYPTO_AUTHENC
select CRYPTO_AEAD
select CRYPTO_HASH
+ select CRYPTO_DES
help
CAAM driver for QorIQ Data Path Acceleration Architecture 2.
It handles DPSECI DPAA2 objects that sit on the Management Complex
@@ -171,12 +170,3 @@ config CRYPTO_DEV_FSL_DPAA2_CAAM
To compile this as a module, choose M here: the module
will be called dpaa2_caam.
-
-config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
- def_tristate (CRYPTO_DEV_FSL_CAAM_CRYPTO_API || \
- CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI || \
- CRYPTO_DEV_FSL_DPAA2_CAAM)
-
-config CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC
- def_tristate (CRYPTO_DEV_FSL_CAAM_AHASH_API || \
- CRYPTO_DEV_FSL_DPAA2_CAAM)
diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile
index 7bbfd06a11ff..68d5cc0f28e2 100644
--- a/drivers/crypto/caam/Makefile
+++ b/drivers/crypto/caam/Makefile
@@ -11,22 +11,23 @@ ccflags-y += -DVERSION=\"\"
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += error.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o
-obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o
-obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC) += caamalg_desc.o
-obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC) += caamhash_desc.o
-obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o
-obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caam_pkc.o
-caam-objs := ctrl.o
-caam_jr-objs := jr.o key_gen.o
-caam_pkc-y := caampkc.o pkc_desc.o
+caam-y := ctrl.o
+caam_jr-y := jr.o key_gen.o
+caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o
+caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o
+caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o
+caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o
+caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caampkc.o pkc_desc.o
+
+caam-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += qi.o
ifneq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI),)
ccflags-y += -DCONFIG_CAAM_QI
- caam-objs += qi.o
endif
obj-$(CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM) += dpaa2_caam.o
dpaa2_caam-y := caamalg_qi2.o dpseci.o
+dpaa2_caam-$(CONFIG_DEBUG_FS) += dpseci-debugfs.o
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index c0ece44f303b..2912006b946b 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -74,16 +74,9 @@
#define CHACHAPOLY_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + CAAM_CMD_SZ * 6)
-#define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN)
+#define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN)
#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
-#ifdef DEBUG
-/* for print_hex_dumps with line references */
-#define debug(format, arg...) printk(format, arg)
-#else
-#define debug(format, arg...)
-#endif
-
struct caam_alg_entry {
int class1_alg_type;
int class2_alg_type;
@@ -212,6 +205,18 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
}
+ /*
+ * In case |user key| > |derived key|, using DKP<imm,imm>
+ * would result in invalid opcodes (last bytes of user key) in
+ * the resulting descriptor. Use DKP<ptr,imm> instead => both
+ * virtual and dma key addresses are needed.
+ */
+ ctx->adata.key_virt = ctx->key;
+ ctx->adata.key_dma = ctx->key_dma;
+
+ ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
+ ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
+
data_len[0] = ctx->adata.keylen_pad;
data_len[1] = ctx->cdata.keylen;
@@ -228,16 +233,6 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
ARRAY_SIZE(data_len)) < 0)
return -EINVAL;
- if (inl_mask & 1)
- ctx->adata.key_virt = ctx->key;
- else
- ctx->adata.key_dma = ctx->key_dma;
-
- if (inl_mask & 2)
- ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
- else
- ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
-
ctx->adata.key_inline = !!(inl_mask & 1);
ctx->cdata.key_inline = !!(inl_mask & 2);
@@ -260,16 +255,6 @@ skip_enc:
ARRAY_SIZE(data_len)) < 0)
return -EINVAL;
- if (inl_mask & 1)
- ctx->adata.key_virt = ctx->key;
- else
- ctx->adata.key_dma = ctx->key_dma;
-
- if (inl_mask & 2)
- ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
- else
- ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
-
ctx->adata.key_inline = !!(inl_mask & 1);
ctx->cdata.key_inline = !!(inl_mask & 2);
@@ -294,16 +279,6 @@ skip_enc:
ARRAY_SIZE(data_len)) < 0)
return -EINVAL;
- if (inl_mask & 1)
- ctx->adata.key_virt = ctx->key;
- else
- ctx->adata.key_dma = ctx->key_dma;
-
- if (inl_mask & 2)
- ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
- else
- ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
-
ctx->adata.key_inline = !!(inl_mask & 1);
ctx->cdata.key_inline = !!(inl_mask & 2);
@@ -383,6 +358,11 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+ int err;
+
+ err = crypto_gcm_check_authsize(authsize);
+ if (err)
+ return err;
ctx->authsize = authsize;
gcm_set_sh_desc(authenc);
@@ -446,6 +426,11 @@ static int rfc4106_setauthsize(struct crypto_aead *authenc,
unsigned int authsize)
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+ int err;
+
+ err = crypto_rfc4106_check_authsize(authsize);
+ if (err)
+ return err;
ctx->authsize = authsize;
rfc4106_set_sh_desc(authenc);
@@ -510,6 +495,9 @@ static int rfc4543_setauthsize(struct crypto_aead *authenc,
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+ if (authsize != 16)
+ return -EINVAL;
+
ctx->authsize = authsize;
rfc4543_set_sh_desc(authenc);
@@ -583,13 +571,11 @@ static int aead_setkey(struct crypto_aead *aead,
if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
goto badkey;
-#ifdef DEBUG
- printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
+ dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
keys.authkeylen + keys.enckeylen, keys.enckeylen,
keys.authkeylen);
- print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+ print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
/*
* If DKP is supported, use it in the shared descriptor to generate
@@ -623,11 +609,10 @@ static int aead_setkey(struct crypto_aead *aead,
memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
keys.enckeylen, ctx->dir);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
- ctx->adata.keylen_pad + keys.enckeylen, 1);
-#endif
+
+ print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
+ ctx->adata.keylen_pad + keys.enckeylen, 1);
skip_split_key:
ctx->cdata.keylen = keys.enckeylen;
@@ -643,33 +628,17 @@ static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
unsigned int keylen)
{
struct crypto_authenc_keys keys;
- u32 flags;
int err;
err = crypto_authenc_extractkeys(&keys, key, keylen);
if (unlikely(err))
- goto badkey;
+ return err;
- err = -EINVAL;
- if (keys.enckeylen != DES3_EDE_KEY_SIZE)
- goto badkey;
-
- flags = crypto_aead_get_flags(aead);
- err = __des3_verify_key(&flags, keys.enckey);
- if (unlikely(err)) {
- crypto_aead_set_flags(aead, flags);
- goto out;
- }
+ err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?:
+ aead_setkey(aead, key, keylen);
- err = aead_setkey(aead, key, keylen);
-
-out:
memzero_explicit(&keys, sizeof(keys));
return err;
-
-badkey:
- crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
- goto out;
}
static int gcm_setkey(struct crypto_aead *aead,
@@ -677,11 +646,16 @@ static int gcm_setkey(struct crypto_aead *aead,
{
struct caam_ctx *ctx = crypto_aead_ctx(aead);
struct device *jrdev = ctx->jrdev;
+ int err;
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+ err = aes_check_keylen(keylen);
+ if (err) {
+ crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return err;
+ }
+
+ print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
memcpy(ctx->key, key, keylen);
dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir);
@@ -695,14 +669,16 @@ static int rfc4106_setkey(struct crypto_aead *aead,
{
struct caam_ctx *ctx = crypto_aead_ctx(aead);
struct device *jrdev = ctx->jrdev;
+ int err;
- if (keylen < 4)
- return -EINVAL;
+ err = aes_check_keylen(keylen - 4);
+ if (err) {
+ crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return err;
+ }
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+ print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
memcpy(ctx->key, key, keylen);
@@ -721,14 +697,16 @@ static int rfc4543_setkey(struct crypto_aead *aead,
{
struct caam_ctx *ctx = crypto_aead_ctx(aead);
struct device *jrdev = ctx->jrdev;
+ int err;
- if (keylen < 4)
- return -EINVAL;
+ err = aes_check_keylen(keylen - 4);
+ if (err) {
+ crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return err;
+ }
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+ print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
memcpy(ctx->key, key, keylen);
@@ -743,7 +721,7 @@ static int rfc4543_setkey(struct crypto_aead *aead,
}
static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
- unsigned int keylen)
+ unsigned int keylen, const u32 ctx1_iv_off)
{
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
struct caam_skcipher_alg *alg =
@@ -752,32 +730,10 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
struct device *jrdev = ctx->jrdev;
unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
u32 *desc;
- u32 ctx1_iv_off = 0;
- const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
- OP_ALG_AAI_CTR_MOD128);
const bool is_rfc3686 = alg->caam.rfc3686;
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
- /*
- * AES-CTR needs to load IV in CONTEXT1 reg
- * at an offset of 128bits (16bytes)
- * CONTEXT1[255:128] = IV
- */
- if (ctr_mode)
- ctx1_iv_off = 16;
-
- /*
- * RFC3686 specific:
- * | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
- * | *key = {KEY, NONCE}
- */
- if (is_rfc3686) {
- ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
- keylen -= CTR_RFC3686_NONCE_SIZE;
- }
+ print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
ctx->cdata.keylen = keylen;
ctx->cdata.key_virt = key;
@@ -800,25 +756,86 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
return 0;
}
-static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
+static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
- u32 tmp[DES3_EDE_EXPKEY_WORDS];
- struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
+ int err;
- if (keylen == DES3_EDE_KEY_SIZE &&
- __des3_ede_setkey(tmp, &tfm->crt_flags, key, DES3_EDE_KEY_SIZE)) {
- return -EINVAL;
+ err = aes_check_keylen(keylen);
+ if (err) {
+ crypto_skcipher_set_flags(skcipher,
+ CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return err;
}
- if (!des_ekey(tmp, key) && (crypto_skcipher_get_flags(skcipher) &
- CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
+ return skcipher_setkey(skcipher, key, keylen, 0);
+}
+
+static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
+ const u8 *key, unsigned int keylen)
+{
+ u32 ctx1_iv_off;
+ int err;
+
+ /*
+ * RFC3686 specific:
+ * | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
+ * | *key = {KEY, NONCE}
+ */
+ ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
+ keylen -= CTR_RFC3686_NONCE_SIZE;
+
+ err = aes_check_keylen(keylen);
+ if (err) {
crypto_skcipher_set_flags(skcipher,
- CRYPTO_TFM_RES_WEAK_KEY);
- return -EINVAL;
+ CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return err;
+ }
+
+ return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
+}
+
+static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
+ const u8 *key, unsigned int keylen)
+{
+ u32 ctx1_iv_off;
+ int err;
+
+ /*
+ * AES-CTR needs to load IV in CONTEXT1 reg
+ * at an offset of 128bits (16bytes)
+ * CONTEXT1[255:128] = IV
+ */
+ ctx1_iv_off = 16;
+
+ err = aes_check_keylen(keylen);
+ if (err) {
+ crypto_skcipher_set_flags(skcipher,
+ CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return err;
}
- return skcipher_setkey(skcipher, key, keylen);
+ return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
+}
+
+static int arc4_skcipher_setkey(struct crypto_skcipher *skcipher,
+ const u8 *key, unsigned int keylen)
+{
+ return skcipher_setkey(skcipher, key, keylen, 0);
+}
+
+static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
+ const u8 *key, unsigned int keylen)
+{
+ return verify_skcipher_des_key(skcipher, key) ?:
+ skcipher_setkey(skcipher, key, keylen, 0);
+}
+
+static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
+ const u8 *key, unsigned int keylen)
+{
+ return verify_skcipher_des3_key(skcipher, key) ?:
+ skcipher_setkey(skcipher, key, keylen, 0);
}
static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
@@ -916,7 +933,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src,
}
if (iv_dma)
- dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
+ dma_unmap_single(dev, iv_dma, ivsize, DMA_BIDIRECTIONAL);
if (sec4_sg_bytes)
dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
DMA_TO_DEVICE);
@@ -948,21 +965,20 @@ static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
{
struct aead_request *req = context;
struct aead_edesc *edesc;
+ int ecode = 0;
-#ifdef DEBUG
- dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+ dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
if (err)
- caam_jr_strstatus(jrdev, err);
+ ecode = caam_jr_strstatus(jrdev, err);
aead_unmap(jrdev, edesc, req);
kfree(edesc);
- aead_request_complete(req, err);
+ aead_request_complete(req, ecode);
}
static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
@@ -970,27 +986,20 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
{
struct aead_request *req = context;
struct aead_edesc *edesc;
+ int ecode = 0;
-#ifdef DEBUG
- dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+ dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
if (err)
- caam_jr_strstatus(jrdev, err);
+ ecode = caam_jr_strstatus(jrdev, err);
aead_unmap(jrdev, edesc, req);
- /*
- * verify hw auth check passed else return -EBADMSG
- */
- if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
- err = -EBADMSG;
-
kfree(edesc);
- aead_request_complete(req, err);
+ aead_request_complete(req, ecode);
}
static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
@@ -1000,38 +1009,37 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
struct skcipher_edesc *edesc;
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
int ivsize = crypto_skcipher_ivsize(skcipher);
+ int ecode = 0;
-#ifdef DEBUG
- dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+ dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]);
if (err)
- caam_jr_strstatus(jrdev, err);
-
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
- edesc->src_nents > 1 ? 100 : ivsize, 1);
-#endif
- caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
- edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
+ ecode = caam_jr_strstatus(jrdev, err);
skcipher_unmap(jrdev, edesc, req);
/*
* The crypto API expects us to set the IV (req->iv) to the last
- * ciphertext block. This is used e.g. by the CTS mode.
+ * ciphertext block (CBC mode) or last counter (CTR mode).
+ * This is used e.g. by the CTS mode.
*/
- if (ivsize)
- scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen -
- ivsize, ivsize, 0);
+ if (ivsize && !ecode) {
+ memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,
+ ivsize);
+ print_hex_dump_debug("dstiv @"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
+ edesc->src_nents > 1 ? 100 : ivsize, 1);
+ }
+
+ caam_dump_sg("dst @" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
+ edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
kfree(edesc);
- skcipher_request_complete(req, err);
+ skcipher_request_complete(req, ecode);
}
static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
@@ -1039,29 +1047,39 @@ static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
{
struct skcipher_request *req = context;
struct skcipher_edesc *edesc;
-#ifdef DEBUG
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
int ivsize = crypto_skcipher_ivsize(skcipher);
+ int ecode = 0;
- dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+ dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]);
if (err)
- caam_jr_strstatus(jrdev, err);
+ ecode = caam_jr_strstatus(jrdev, err);
+
+ skcipher_unmap(jrdev, edesc, req);
+
+ /*
+ * The crypto API expects us to set the IV (req->iv) to the last
+ * ciphertext block (CBC mode) or last counter (CTR mode).
+ * This is used e.g. by the CTS mode.
+ */
+ if (ivsize && !ecode) {
+ memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,
+ ivsize);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
-#endif
- caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ",
+ print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
+ ivsize, 1);
+ }
+
+ caam_dump_sg("dst @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
- skcipher_unmap(jrdev, edesc, req);
kfree(edesc);
- skcipher_request_complete(req, err);
+ skcipher_request_complete(req, ecode);
}
/*
@@ -1106,6 +1124,7 @@ static void init_aead_job(struct aead_request *req,
if (unlikely(req->src != req->dst)) {
if (!edesc->mapped_dst_nents) {
dst_dma = 0;
+ out_options = 0;
} else if (edesc->mapped_dst_nents == 1) {
dst_dma = sg_dma_address(req->dst);
out_options = 0;
@@ -1249,6 +1268,7 @@ static void init_skcipher_job(struct skcipher_request *req,
{
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
+ struct device *jrdev = ctx->jrdev;
int ivsize = crypto_skcipher_ivsize(skcipher);
u32 *desc = edesc->hw_desc;
u32 *sh_desc;
@@ -1256,13 +1276,12 @@ static void init_skcipher_job(struct skcipher_request *req,
dma_addr_t src_dma, dst_dma, ptr;
int len, sec4_sg_index = 0;
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
- pr_err("asked=%d, cryptlen%d\n",
+ print_hex_dump_debug("presciv@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
+ dev_dbg(jrdev, "asked=%d, cryptlen%d\n",
(int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen);
-#endif
- caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__)": ",
+
+ caam_dump_sg("src @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->src,
edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
@@ -1285,7 +1304,7 @@ static void init_skcipher_job(struct skcipher_request *req,
if (likely(req->src == req->dst)) {
dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry);
out_options = in_options;
- } else if (edesc->mapped_dst_nents == 1) {
+ } else if (!ivsize && edesc->mapped_dst_nents == 1) {
dst_dma = sg_dma_address(req->dst);
} else {
dst_dma = edesc->sec4_sg_dma + sec4_sg_index *
@@ -1293,7 +1312,7 @@ static void init_skcipher_job(struct skcipher_request *req,
out_options = LDST_SGF;
}
- append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options);
+ append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options);
}
/*
@@ -1309,37 +1328,36 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
+ int src_len, dst_len = 0;
struct aead_edesc *edesc;
int sec4_sg_index, sec4_sg_len, sec4_sg_bytes;
unsigned int authsize = ctx->authsize;
if (unlikely(req->dst != req->src)) {
- src_nents = sg_nents_for_len(req->src, req->assoclen +
- req->cryptlen);
+ src_len = req->assoclen + req->cryptlen;
+ dst_len = src_len + (encrypt ? authsize : (-authsize));
+
+ src_nents = sg_nents_for_len(req->src, src_len);
if (unlikely(src_nents < 0)) {
dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
- req->assoclen + req->cryptlen);
+ src_len);
return ERR_PTR(src_nents);
}
- dst_nents = sg_nents_for_len(req->dst, req->assoclen +
- req->cryptlen +
- (encrypt ? authsize :
- (-authsize)));
+ dst_nents = sg_nents_for_len(req->dst, dst_len);
if (unlikely(dst_nents < 0)) {
dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
- req->assoclen + req->cryptlen +
- (encrypt ? authsize : (-authsize)));
+ dst_len);
return ERR_PTR(dst_nents);
}
} else {
- src_nents = sg_nents_for_len(req->src, req->assoclen +
- req->cryptlen +
- (encrypt ? authsize : 0));
+ src_len = req->assoclen + req->cryptlen +
+ (encrypt ? authsize : 0);
+
+ src_nents = sg_nents_for_len(req->src, src_len);
if (unlikely(src_nents < 0)) {
dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
- req->assoclen + req->cryptlen +
- (encrypt ? authsize : 0));
+ src_len);
return ERR_PTR(src_nents);
}
}
@@ -1380,8 +1398,16 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
}
}
+ /*
+ * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
+ * the end of the table by allocating more S/G entries.
+ */
sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0;
- sec4_sg_len += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
+ if (mapped_dst_nents > 1)
+ sec4_sg_len += pad_sg_nents(mapped_dst_nents);
+ else
+ sec4_sg_len = pad_sg_nents(sec4_sg_len);
+
sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
/* allocate space for base edesc and hw desc commands, link tables */
@@ -1403,12 +1429,12 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
sec4_sg_index = 0;
if (mapped_src_nents > 1) {
- sg_to_sec4_sg_last(req->src, mapped_src_nents,
+ sg_to_sec4_sg_last(req->src, src_len,
edesc->sec4_sg + sec4_sg_index, 0);
sec4_sg_index += mapped_src_nents;
}
if (mapped_dst_nents > 1) {
- sg_to_sec4_sg_last(req->dst, mapped_dst_nents,
+ sg_to_sec4_sg_last(req->dst, dst_len,
edesc->sec4_sg + sec4_sg_index, 0);
}
@@ -1446,11 +1472,10 @@ static int gcm_encrypt(struct aead_request *req)
/* Create and submit job descriptor */
init_gcm_job(req, edesc, all_contig, true);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
- desc_bytes(edesc->hw_desc), 1);
-#endif
+
+ print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
+ desc_bytes(edesc->hw_desc), 1);
desc = edesc->hw_desc;
ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
@@ -1532,10 +1557,7 @@ static int chachapoly_decrypt(struct aead_request *req)
static int ipsec_gcm_encrypt(struct aead_request *req)
{
- if (req->assoclen < 8)
- return -EINVAL;
-
- return gcm_encrypt(req);
+ return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_encrypt(req);
}
static int aead_encrypt(struct aead_request *req)
@@ -1556,11 +1578,10 @@ static int aead_encrypt(struct aead_request *req)
/* Create and submit job descriptor */
init_authenc_job(req, edesc, all_contig, true);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
- desc_bytes(edesc->hw_desc), 1);
-#endif
+
+ print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
+ desc_bytes(edesc->hw_desc), 1);
desc = edesc->hw_desc;
ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
@@ -1591,11 +1612,10 @@ static int gcm_decrypt(struct aead_request *req)
/* Create and submit job descriptor*/
init_gcm_job(req, edesc, all_contig, false);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
- desc_bytes(edesc->hw_desc), 1);
-#endif
+
+ print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
+ desc_bytes(edesc->hw_desc), 1);
desc = edesc->hw_desc;
ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
@@ -1611,10 +1631,7 @@ static int gcm_decrypt(struct aead_request *req)
static int ipsec_gcm_decrypt(struct aead_request *req)
{
- if (req->assoclen < 8)
- return -EINVAL;
-
- return gcm_decrypt(req);
+ return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_decrypt(req);
}
static int aead_decrypt(struct aead_request *req)
@@ -1627,7 +1644,7 @@ static int aead_decrypt(struct aead_request *req)
u32 *desc;
int ret = 0;
- caam_dump_sg(KERN_ERR, "dec src@" __stringify(__LINE__)": ",
+ caam_dump_sg("dec src@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->src,
req->assoclen + req->cryptlen, 1);
@@ -1639,11 +1656,10 @@ static int aead_decrypt(struct aead_request *req)
/* Create and submit job descriptor*/
init_authenc_job(req, edesc, all_contig, false);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
- desc_bytes(edesc->hw_desc), 1);
-#endif
+
+ print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
+ desc_bytes(edesc->hw_desc), 1);
desc = edesc->hw_desc;
ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
@@ -1719,7 +1735,29 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
else
sec4_sg_ents = mapped_src_nents + !!ivsize;
dst_sg_idx = sec4_sg_ents;
- sec4_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
+
+ /*
+ * Input, output HW S/G tables: [IV, src][dst, IV]
+ * IV entries point to the same buffer
+ * If src == dst, S/G entries are reused (S/G tables overlap)
+ *
+ * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
+ * the end of the table by allocating more S/G entries. Logic:
+ * if (output S/G)
+ * pad output S/G, if needed
+ * else if (input S/G) ...
+ * pad input S/G, if needed
+ */
+ if (ivsize || mapped_dst_nents > 1) {
+ if (req->src == req->dst)
+ sec4_sg_ents = !!ivsize + pad_sg_nents(sec4_sg_ents);
+ else
+ sec4_sg_ents += pad_sg_nents(mapped_dst_nents +
+ !!ivsize);
+ } else {
+ sec4_sg_ents = pad_sg_nents(sec4_sg_ents);
+ }
+
sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
/*
@@ -1744,10 +1782,10 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
/* Make sure IV is located in a DMAable area */
if (ivsize) {
- iv = (u8 *)edesc->hw_desc + desc_bytes + sec4_sg_bytes;
+ iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes;
memcpy(iv, req->iv, ivsize);
- iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_TO_DEVICE);
+ iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL);
if (dma_mapping_error(jrdev, iv_dma)) {
dev_err(jrdev, "unable to map IV\n");
caam_unmap(jrdev, req->src, req->dst, src_nents,
@@ -1759,13 +1797,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
}
if (dst_sg_idx)
- sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg +
- !!ivsize, 0);
+ sg_to_sec4_sg(req->src, req->cryptlen, edesc->sec4_sg +
+ !!ivsize, 0);
- if (mapped_dst_nents > 1) {
- sg_to_sec4_sg_last(req->dst, mapped_dst_nents,
- edesc->sec4_sg + dst_sg_idx, 0);
- }
+ if (req->src != req->dst && (ivsize || mapped_dst_nents > 1))
+ sg_to_sec4_sg(req->dst, req->cryptlen, edesc->sec4_sg +
+ dst_sg_idx, 0);
+
+ if (ivsize)
+ dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx +
+ mapped_dst_nents, iv_dma, ivsize, 0);
+
+ if (ivsize || mapped_dst_nents > 1)
+ sg_to_sec4_set_last(edesc->sec4_sg + dst_sg_idx +
+ mapped_dst_nents);
if (sec4_sg_bytes) {
edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
@@ -1782,11 +1827,9 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
edesc->iv_dma = iv_dma;
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "skcipher sec4_sg@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
- sec4_sg_bytes, 1);
-#endif
+ print_hex_dump_debug("skcipher sec4_sg@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
+ sec4_sg_bytes, 1);
return edesc;
}
@@ -1800,6 +1843,9 @@ static int skcipher_encrypt(struct skcipher_request *req)
u32 *desc;
int ret = 0;
+ if (!req->cryptlen)
+ return 0;
+
/* allocate extended descriptor */
edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
if (IS_ERR(edesc))
@@ -1807,11 +1853,11 @@ static int skcipher_encrypt(struct skcipher_request *req)
/* Create and submit job descriptor*/
init_skcipher_job(req, edesc, true);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "skcipher jobdesc@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
- desc_bytes(edesc->hw_desc), 1);
-#endif
+
+ print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
+ desc_bytes(edesc->hw_desc), 1);
+
desc = edesc->hw_desc;
ret = caam_jr_enqueue(jrdev, desc, skcipher_encrypt_done, req);
@@ -1830,32 +1876,25 @@ static int skcipher_decrypt(struct skcipher_request *req)
struct skcipher_edesc *edesc;
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
- int ivsize = crypto_skcipher_ivsize(skcipher);
struct device *jrdev = ctx->jrdev;
u32 *desc;
int ret = 0;
+ if (!req->cryptlen)
+ return 0;
+
/* allocate extended descriptor */
edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
if (IS_ERR(edesc))
return PTR_ERR(edesc);
- /*
- * The crypto API expects us to set the IV (req->iv) to the last
- * ciphertext block.
- */
- if (ivsize)
- scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen -
- ivsize, ivsize, 0);
-
/* Create and submit job descriptor*/
init_skcipher_job(req, edesc, false);
desc = edesc->hw_desc;
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "skcipher jobdesc@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
- desc_bytes(edesc->hw_desc), 1);
-#endif
+
+ print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
+ desc_bytes(edesc->hw_desc), 1);
ret = caam_jr_enqueue(jrdev, desc, skcipher_decrypt_done, req);
if (!ret) {
@@ -1876,7 +1915,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "cbc-aes-caam",
.cra_blocksize = AES_BLOCK_SIZE,
},
- .setkey = skcipher_setkey,
+ .setkey = aes_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE,
@@ -1892,7 +1931,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "cbc-3des-caam",
.cra_blocksize = DES3_EDE_BLOCK_SIZE,
},
- .setkey = des_skcipher_setkey,
+ .setkey = des3_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = DES3_EDE_KEY_SIZE,
@@ -1924,7 +1963,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "ctr-aes-caam",
.cra_blocksize = 1,
},
- .setkey = skcipher_setkey,
+ .setkey = ctr_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE,
@@ -1942,7 +1981,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "rfc3686-ctr-aes-caam",
.cra_blocksize = 1,
},
- .setkey = skcipher_setkey,
+ .setkey = rfc3686_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE +
@@ -1996,7 +2035,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "ecb-aes-caam",
.cra_blocksize = AES_BLOCK_SIZE,
},
- .setkey = skcipher_setkey,
+ .setkey = aes_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE,
@@ -2011,7 +2050,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "ecb-des3-caam",
.cra_blocksize = DES3_EDE_BLOCK_SIZE,
},
- .setkey = des_skcipher_setkey,
+ .setkey = des3_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = DES3_EDE_KEY_SIZE,
@@ -2026,7 +2065,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "ecb-arc4-caam",
.cra_blocksize = ARC4_BLOCK_SIZE,
},
- .setkey = skcipher_setkey,
+ .setkey = arc4_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = ARC4_MIN_KEY_SIZE,
@@ -3444,7 +3483,7 @@ static void caam_aead_exit(struct crypto_aead *tfm)
caam_exit_common(crypto_aead_ctx(tfm));
}
-static void __exit caam_algapi_exit(void)
+void caam_algapi_exit(void)
{
int i;
@@ -3489,43 +3528,15 @@ static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
alg->exit = caam_aead_exit;
}
-static int __init caam_algapi_init(void)
+int caam_algapi_init(struct device *ctrldev)
{
- struct device_node *dev_node;
- struct platform_device *pdev;
- struct caam_drv_private *priv;
+ struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
int i = 0, err = 0;
u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst;
u32 arc4_inst;
unsigned int md_limit = SHA512_DIGEST_SIZE;
bool registered = false, gcm_support;
- dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
- if (!dev_node) {
- dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
- if (!dev_node)
- return -ENODEV;
- }
-
- pdev = of_find_device_by_node(dev_node);
- if (!pdev) {
- of_node_put(dev_node);
- return -ENODEV;
- }
-
- priv = dev_get_drvdata(&pdev->dev);
- of_node_put(dev_node);
-
- /*
- * If priv is NULL, it's probably because the caam driver wasn't
- * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
- */
- if (!priv) {
- err = -ENODEV;
- goto out_put_dev;
- }
-
-
/*
* Register crypto algorithms the device supports.
* First, detect presence and attributes of DES, AES, and MD blocks.
@@ -3668,14 +3679,5 @@ static int __init caam_algapi_init(void)
if (registered)
pr_info("caam algorithms registered in /proc/crypto\n");
-out_put_dev:
- put_device(&pdev->dev);
return err;
}
-
-module_init(caam_algapi_init);
-module_exit(caam_algapi_exit);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("FSL CAAM support for crypto API");
-MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c
index 1e1a376edc2f..aa9ccca67045 100644
--- a/drivers/crypto/caam/caamalg_desc.c
+++ b/drivers/crypto/caam/caamalg_desc.c
@@ -33,12 +33,11 @@ static inline void append_dec_op1(u32 *desc, u32 type)
}
jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
- append_operation(desc, type | OP_ALG_AS_INITFINAL |
- OP_ALG_DECRYPT);
+ append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT);
uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
set_jump_tgt_here(desc, jump_cmd);
- append_operation(desc, type | OP_ALG_AS_INITFINAL |
- OP_ALG_DECRYPT | OP_ALG_AAI_DK);
+ append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT |
+ OP_ALG_AAI_DK);
set_jump_tgt_here(desc, uncond_jump_cmd);
}
@@ -115,11 +114,9 @@ void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
LDST_SRCDST_BYTE_CONTEXT);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR,
- "aead null enc shdesc@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
}
EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
@@ -204,11 +201,9 @@ void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR,
- "aead null dec shdesc@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
}
EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
@@ -358,10 +353,9 @@ void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
LDST_SRCDST_BYTE_CONTEXT);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
}
EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
@@ -475,10 +469,9 @@ void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
}
EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
@@ -510,6 +503,7 @@ void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
const bool is_qi, int era)
{
u32 geniv, moveiv;
+ u32 *wait_cmd;
/* Note: Context registers are saved. */
init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
@@ -605,6 +599,14 @@ copy_iv:
/* Will read cryptlen */
append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+
+ /*
+ * Wait for IV transfer (ofifo -> class2) to finish before starting
+ * ciphertext transfer (ofifo -> external memory).
+ */
+ wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NIFP);
+ set_jump_tgt_here(desc, wait_cmd);
+
append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
@@ -613,11 +615,9 @@ copy_iv:
append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
LDST_SRCDST_BYTE_CONTEXT);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR,
- "aead givenc shdesc@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
}
EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
@@ -742,10 +742,9 @@ void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
LDST_SRCDST_BYTE_CONTEXT);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
}
EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
@@ -838,10 +837,9 @@ void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
}
EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
@@ -854,13 +852,16 @@ EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
* @ivsize: initialization vector size
* @icvsize: integrity check value (ICV) size (truncated or full)
* @is_qi: true when called from caam/qi
+ *
+ * Input sequence: AAD | PTXT
+ * Output sequence: AAD | CTXT | ICV
+ * AAD length (assoclen), which includes the IV length, is available in Math3.
*/
void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
unsigned int ivsize, unsigned int icvsize,
const bool is_qi)
{
- u32 *key_jump_cmd;
-
+ u32 *key_jump_cmd, *zero_cryptlen_jump_cmd, *skip_instructions;
init_sh_desc(desc, HDR_SHARE_SERIAL);
/* Skip key loading if it is loaded due to sharing */
@@ -903,24 +904,26 @@ void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
- /* Read assoc data */
- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
- FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
+ /* Skip AAD */
+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
- /* Skip IV */
- append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
+ /* Read cryptlen and set this value into VARSEQOUTLEN */
+ append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
- /* Will read cryptlen bytes */
- append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+ /* If cryptlen is ZERO jump to AAD command */
+ zero_cryptlen_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
+ JUMP_COND_MATH_Z);
- /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
+ /* Read AAD data */
+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
+ FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
- /* Skip assoc data */
- append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
+ /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA);
- /* cryptlen = seqoutlen - assoclen */
- append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ);
+ /* Skip IV */
+ append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
+ append_math_add(desc, VARSEQINLEN, VARSEQOUTLEN, REG0, CAAM_CMD_SZ);
/* Write encrypted data */
append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
@@ -929,15 +932,25 @@ void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
+ /* Jump instructions to avoid double reading of AAD */
+ skip_instructions = append_jump(desc, JUMP_TEST_ALL);
+
+ /* There is no input data, cryptlen = 0 */
+ set_jump_tgt_here(desc, zero_cryptlen_jump_cmd);
+
+ /* Read AAD */
+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
+ FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
+
+ set_jump_tgt_here(desc, skip_instructions);
+
/* Write ICV */
append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
LDST_SRCDST_BYTE_CONTEXT);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR,
- "rfc4106 enc shdesc@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
}
EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
@@ -1030,11 +1043,9 @@ void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR,
- "rfc4106 dec shdesc@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
}
EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
@@ -1115,11 +1126,9 @@ void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
LDST_SRCDST_BYTE_CONTEXT);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR,
- "rfc4543 enc shdesc@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
}
EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
@@ -1205,11 +1214,9 @@ void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR,
- "rfc4543 dec shdesc@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
}
EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
@@ -1410,17 +1417,21 @@ void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
LDST_OFFSET_SHIFT));
/* Load operation */
- append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
+ append_operation(desc, cdata->algtype | OP_ALG_AS_INIT |
OP_ALG_ENCRYPT);
/* Perform operation */
skcipher_append_src_dst(desc);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR,
- "skcipher enc shdesc@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ /* Store IV */
+ if (ivsize)
+ append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
+ LDST_CLASS_1_CCB | (ctx1_iv_off <<
+ LDST_OFFSET_SHIFT));
+
+ print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
}
EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap);
@@ -1479,7 +1490,7 @@ void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
/* Choose operation */
if (ctx1_iv_off)
- append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
+ append_operation(desc, cdata->algtype | OP_ALG_AS_INIT |
OP_ALG_DECRYPT);
else
append_dec_op1(desc, cdata->algtype);
@@ -1487,11 +1498,15 @@ void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
/* Perform operation */
skcipher_append_src_dst(desc);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR,
- "skcipher dec shdesc@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ /* Store IV */
+ if (ivsize)
+ append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
+ LDST_CLASS_1_CCB | (ctx1_iv_off <<
+ LDST_OFFSET_SHIFT));
+
+ print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
}
EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap);
@@ -1538,11 +1553,13 @@ void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata)
/* Perform operation */
skcipher_append_src_dst(desc);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR,
- "xts skcipher enc shdesc@" __stringify(__LINE__) ": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ /* Store upper 8B of IV */
+ append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
+ (0x20 << LDST_OFFSET_SHIFT));
+
+ print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__)
+ ": ", DUMP_PREFIX_ADDRESS, 16, 4,
+ desc, desc_bytes(desc), 1);
}
EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap);
@@ -1588,11 +1605,13 @@ void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata)
/* Perform operation */
skcipher_append_src_dst(desc);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR,
- "xts skcipher dec shdesc@" __stringify(__LINE__) ": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ /* Store upper 8B of IV */
+ append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
+ (0x20 << LDST_OFFSET_SHIFT));
+
+ print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__)
+ ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
}
EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap);
diff --git a/drivers/crypto/caam/caamalg_desc.h b/drivers/crypto/caam/caamalg_desc.h
index d5ca42ff961a..f2893393ba5e 100644
--- a/drivers/crypto/caam/caamalg_desc.h
+++ b/drivers/crypto/caam/caamalg_desc.h
@@ -12,7 +12,7 @@
#define DESC_AEAD_BASE (4 * CAAM_CMD_SZ)
#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ)
#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ)
-#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ)
+#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 8 * CAAM_CMD_SZ)
#define DESC_QI_AEAD_ENC_LEN (DESC_AEAD_ENC_LEN + 3 * CAAM_CMD_SZ)
#define DESC_QI_AEAD_DEC_LEN (DESC_AEAD_DEC_LEN + 3 * CAAM_CMD_SZ)
#define DESC_QI_AEAD_GIVENC_LEN (DESC_AEAD_GIVENC_LEN + 3 * CAAM_CMD_SZ)
@@ -31,7 +31,7 @@
#define DESC_QI_GCM_DEC_LEN (DESC_GCM_DEC_LEN + 3 * CAAM_CMD_SZ)
#define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ)
-#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
+#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 16 * CAAM_CMD_SZ)
#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
#define DESC_QI_RFC4106_ENC_LEN (DESC_RFC4106_ENC_LEN + 5 * CAAM_CMD_SZ)
#define DESC_QI_RFC4106_DEC_LEN (DESC_RFC4106_DEC_LEN + 5 * CAAM_CMD_SZ)
@@ -44,9 +44,9 @@
#define DESC_SKCIPHER_BASE (3 * CAAM_CMD_SZ)
#define DESC_SKCIPHER_ENC_LEN (DESC_SKCIPHER_BASE + \
- 20 * CAAM_CMD_SZ)
+ 21 * CAAM_CMD_SZ)
#define DESC_SKCIPHER_DEC_LEN (DESC_SKCIPHER_BASE + \
- 15 * CAAM_CMD_SZ)
+ 16 * CAAM_CMD_SZ)
void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
unsigned int icvsize, int era);
diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c
index d290d6b41825..8e3449670d2f 100644
--- a/drivers/crypto/caam/caamalg_qi.c
+++ b/drivers/crypto/caam/caamalg_qi.c
@@ -4,7 +4,7 @@
* Based on caamalg.c
*
* Copyright 2013-2016 Freescale Semiconductor, Inc.
- * Copyright 2016-2018 NXP
+ * Copyright 2016-2019 NXP
*/
#include "compat.h"
@@ -105,6 +105,18 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
}
+ /*
+ * In case |user key| > |derived key|, using DKP<imm,imm> would result
+ * in invalid opcodes (last bytes of user key) in the resulting
+ * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
+ * addresses are needed.
+ */
+ ctx->adata.key_virt = ctx->key;
+ ctx->adata.key_dma = ctx->key_dma;
+
+ ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
+ ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
+
data_len[0] = ctx->adata.keylen_pad;
data_len[1] = ctx->cdata.keylen;
@@ -118,16 +130,6 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
ARRAY_SIZE(data_len)) < 0)
return -EINVAL;
- if (inl_mask & 1)
- ctx->adata.key_virt = ctx->key;
- else
- ctx->adata.key_dma = ctx->key_dma;
-
- if (inl_mask & 2)
- ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
- else
- ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
-
ctx->adata.key_inline = !!(inl_mask & 1);
ctx->cdata.key_inline = !!(inl_mask & 2);
@@ -143,16 +145,6 @@ skip_enc:
ARRAY_SIZE(data_len)) < 0)
return -EINVAL;
- if (inl_mask & 1)
- ctx->adata.key_virt = ctx->key;
- else
- ctx->adata.key_dma = ctx->key_dma;
-
- if (inl_mask & 2)
- ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
- else
- ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
-
ctx->adata.key_inline = !!(inl_mask & 1);
ctx->cdata.key_inline = !!(inl_mask & 2);
@@ -171,16 +163,6 @@ skip_enc:
ARRAY_SIZE(data_len)) < 0)
return -EINVAL;
- if (inl_mask & 1)
- ctx->adata.key_virt = ctx->key;
- else
- ctx->adata.key_dma = ctx->key_dma;
-
- if (inl_mask & 2)
- ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
- else
- ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
-
ctx->adata.key_inline = !!(inl_mask & 1);
ctx->cdata.key_inline = !!(inl_mask & 2);
@@ -214,13 +196,11 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key,
if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
goto badkey;
-#ifdef DEBUG
- dev_err(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
+ dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
keys.authkeylen + keys.enckeylen, keys.enckeylen,
keys.authkeylen);
- print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+ print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
/*
* If DKP is supported, use it in the shared descriptor to generate
@@ -237,7 +217,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key,
memcpy(ctx->key, keys.authkey, keys.authkeylen);
memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
keys.enckeylen);
- dma_sync_single_for_device(jrdev, ctx->key_dma,
+ dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
ctx->adata.keylen_pad +
keys.enckeylen, ctx->dir);
goto skip_split_key;
@@ -251,13 +231,13 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key,
/* postpend encryption key to auth split key */
memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
- dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
- keys.enckeylen, ctx->dir);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
- ctx->adata.keylen_pad + keys.enckeylen, 1);
-#endif
+ dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
+ ctx->adata.keylen_pad + keys.enckeylen,
+ ctx->dir);
+
+ print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
+ ctx->adata.keylen_pad + keys.enckeylen, 1);
skip_split_key:
ctx->cdata.keylen = keys.enckeylen;
@@ -297,33 +277,17 @@ static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
unsigned int keylen)
{
struct crypto_authenc_keys keys;
- u32 flags;
int err;
err = crypto_authenc_extractkeys(&keys, key, keylen);
if (unlikely(err))
- goto badkey;
-
- err = -EINVAL;
- if (keys.enckeylen != DES3_EDE_KEY_SIZE)
- goto badkey;
-
- flags = crypto_aead_get_flags(aead);
- err = __des3_verify_key(&flags, keys.enckey);
- if (unlikely(err)) {
- crypto_aead_set_flags(aead, flags);
- goto out;
- }
+ return err;
- err = aead_setkey(aead, key, keylen);
+ err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?:
+ aead_setkey(aead, key, keylen);
-out:
memzero_explicit(&keys, sizeof(keys));
return err;
-
-badkey:
- crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
- goto out;
}
static int gcm_set_sh_desc(struct crypto_aead *aead)
@@ -372,6 +336,11 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+ int err;
+
+ err = crypto_gcm_check_authsize(authsize);
+ if (err)
+ return err;
ctx->authsize = authsize;
gcm_set_sh_desc(authenc);
@@ -386,13 +355,18 @@ static int gcm_setkey(struct crypto_aead *aead,
struct device *jrdev = ctx->jrdev;
int ret;
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+ ret = aes_check_keylen(keylen);
+ if (ret) {
+ crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return ret;
+ }
+
+ print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
memcpy(ctx->key, key, keylen);
- dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir);
+ dma_sync_single_for_device(jrdev->parent, ctx->key_dma, keylen,
+ ctx->dir);
ctx->cdata.keylen = keylen;
ret = gcm_set_sh_desc(aead);
@@ -468,6 +442,11 @@ static int rfc4106_setauthsize(struct crypto_aead *authenc,
unsigned int authsize)
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+ int err;
+
+ err = crypto_rfc4106_check_authsize(authsize);
+ if (err)
+ return err;
ctx->authsize = authsize;
rfc4106_set_sh_desc(authenc);
@@ -482,13 +461,14 @@ static int rfc4106_setkey(struct crypto_aead *aead,
struct device *jrdev = ctx->jrdev;
int ret;
- if (keylen < 4)
- return -EINVAL;
+ ret = aes_check_keylen(keylen - 4);
+ if (ret) {
+ crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return ret;
+ }
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+ print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
memcpy(ctx->key, key, keylen);
/*
@@ -496,8 +476,8 @@ static int rfc4106_setkey(struct crypto_aead *aead,
* in the nonce. Update the AES key length.
*/
ctx->cdata.keylen = keylen - 4;
- dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
- ctx->dir);
+ dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
+ ctx->cdata.keylen, ctx->dir);
ret = rfc4106_set_sh_desc(aead);
if (ret)
@@ -573,6 +553,9 @@ static int rfc4543_setauthsize(struct crypto_aead *authenc,
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+ if (authsize != 16)
+ return -EINVAL;
+
ctx->authsize = authsize;
rfc4543_set_sh_desc(authenc);
@@ -586,13 +569,14 @@ static int rfc4543_setkey(struct crypto_aead *aead,
struct device *jrdev = ctx->jrdev;
int ret;
- if (keylen < 4)
- return -EINVAL;
+ ret = aes_check_keylen(keylen - 4);
+ if (ret) {
+ crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return ret;
+ }
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+ print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
memcpy(ctx->key, key, keylen);
/*
@@ -600,8 +584,8 @@ static int rfc4543_setkey(struct crypto_aead *aead,
* in the nonce. Update the AES key length.
*/
ctx->cdata.keylen = keylen - 4;
- dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
- ctx->dir);
+ dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
+ ctx->cdata.keylen, ctx->dir);
ret = rfc4543_set_sh_desc(aead);
if (ret)
@@ -630,7 +614,7 @@ static int rfc4543_setkey(struct crypto_aead *aead,
}
static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
- unsigned int keylen)
+ unsigned int keylen, const u32 ctx1_iv_off)
{
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
struct caam_skcipher_alg *alg =
@@ -638,33 +622,11 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
skcipher);
struct device *jrdev = ctx->jrdev;
unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
- u32 ctx1_iv_off = 0;
- const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
- OP_ALG_AAI_CTR_MOD128);
const bool is_rfc3686 = alg->caam.rfc3686;
int ret = 0;
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
- /*
- * AES-CTR needs to load IV in CONTEXT1 reg
- * at an offset of 128bits (16bytes)
- * CONTEXT1[255:128] = IV
- */
- if (ctr_mode)
- ctx1_iv_off = 16;
-
- /*
- * RFC3686 specific:
- * | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
- * | *key = {KEY, NONCE}
- */
- if (is_rfc3686) {
- ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
- keylen -= CTR_RFC3686_NONCE_SIZE;
- }
+ print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
ctx->cdata.keylen = keylen;
ctx->cdata.key_virt = key;
@@ -701,11 +663,80 @@ badkey:
return -EINVAL;
}
+static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
+ const u8 *key, unsigned int keylen)
+{
+ int err;
+
+ err = aes_check_keylen(keylen);
+ if (err) {
+ crypto_skcipher_set_flags(skcipher,
+ CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return err;
+ }
+
+ return skcipher_setkey(skcipher, key, keylen, 0);
+}
+
+static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
+ const u8 *key, unsigned int keylen)
+{
+ u32 ctx1_iv_off;
+ int err;
+
+ /*
+ * RFC3686 specific:
+ * | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
+ * | *key = {KEY, NONCE}
+ */
+ ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
+ keylen -= CTR_RFC3686_NONCE_SIZE;
+
+ err = aes_check_keylen(keylen);
+ if (err) {
+ crypto_skcipher_set_flags(skcipher,
+ CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return err;
+ }
+
+ return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
+}
+
+static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
+ const u8 *key, unsigned int keylen)
+{
+ u32 ctx1_iv_off;
+ int err;
+
+ /*
+ * AES-CTR needs to load IV in CONTEXT1 reg
+ * at an offset of 128bits (16bytes)
+ * CONTEXT1[255:128] = IV
+ */
+ ctx1_iv_off = 16;
+
+ err = aes_check_keylen(keylen);
+ if (err) {
+ crypto_skcipher_set_flags(skcipher,
+ CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return err;
+ }
+
+ return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
+}
+
static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
- return unlikely(des3_verify_key(skcipher, key)) ?:
- skcipher_setkey(skcipher, key, keylen);
+ return verify_skcipher_des3_key(skcipher, key) ?:
+ skcipher_setkey(skcipher, key, keylen, 0);
+}
+
+static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
+ const u8 *key, unsigned int keylen)
+{
+ return verify_skcipher_des_key(skcipher, key) ?:
+ skcipher_setkey(skcipher, key, keylen, 0);
}
static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
@@ -838,7 +869,8 @@ static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx,
static void caam_unmap(struct device *dev, struct scatterlist *src,
struct scatterlist *dst, int src_nents,
int dst_nents, dma_addr_t iv_dma, int ivsize,
- dma_addr_t qm_sg_dma, int qm_sg_bytes)
+ enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
+ int qm_sg_bytes)
{
if (dst != src) {
if (src_nents)
@@ -850,7 +882,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src,
}
if (iv_dma)
- dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
+ dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
if (qm_sg_bytes)
dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
}
@@ -863,7 +895,8 @@ static void aead_unmap(struct device *dev,
int ivsize = crypto_aead_ivsize(aead);
caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
- edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
+ edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
+ edesc->qm_sg_bytes);
dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
}
@@ -874,7 +907,8 @@ static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
int ivsize = crypto_skcipher_ivsize(skcipher);
caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
- edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
+ edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
+ edesc->qm_sg_bytes);
}
static void aead_done(struct caam_drv_req *drv_req, u32 status)
@@ -888,20 +922,8 @@ static void aead_done(struct caam_drv_req *drv_req, u32 status)
qidev = caam_ctx->qidev;
- if (unlikely(status)) {
- u32 ssrc = status & JRSTA_SSRC_MASK;
- u8 err_id = status & JRSTA_CCBERR_ERRID_MASK;
-
- caam_jr_strstatus(qidev, status);
- /*
- * verify hw auth check passed else return -EBADMSG
- */
- if (ssrc == JRSTA_SSRC_CCB_ERROR &&
- err_id == JRSTA_CCBERR_ERRID_ICVCHK)
- ecode = -EBADMSG;
- else
- ecode = -EIO;
- }
+ if (unlikely(status))
+ ecode = caam_jr_strstatus(qidev, status);
edesc = container_of(drv_req, typeof(*edesc), drv_req);
aead_unmap(qidev, edesc, aead_req);
@@ -924,6 +946,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
+ int src_len, dst_len = 0;
struct aead_edesc *edesc;
dma_addr_t qm_sg_dma, iv_dma = 0;
int ivsize = 0;
@@ -945,13 +968,13 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
}
if (likely(req->src == req->dst)) {
- src_nents = sg_nents_for_len(req->src, req->assoclen +
- req->cryptlen +
- (encrypt ? authsize : 0));
+ src_len = req->assoclen + req->cryptlen +
+ (encrypt ? authsize : 0);
+
+ src_nents = sg_nents_for_len(req->src, src_len);
if (unlikely(src_nents < 0)) {
dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
- req->assoclen + req->cryptlen +
- (encrypt ? authsize : 0));
+ src_len);
qi_cache_free(edesc);
return ERR_PTR(src_nents);
}
@@ -964,23 +987,21 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
return ERR_PTR(-ENOMEM);
}
} else {
- src_nents = sg_nents_for_len(req->src, req->assoclen +
- req->cryptlen);
+ src_len = req->assoclen + req->cryptlen;
+ dst_len = src_len + (encrypt ? authsize : (-authsize));
+
+ src_nents = sg_nents_for_len(req->src, src_len);
if (unlikely(src_nents < 0)) {
dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
- req->assoclen + req->cryptlen);
+ src_len);
qi_cache_free(edesc);
return ERR_PTR(src_nents);
}
- dst_nents = sg_nents_for_len(req->dst, req->assoclen +
- req->cryptlen +
- (encrypt ? authsize :
- (-authsize)));
+ dst_nents = sg_nents_for_len(req->dst, dst_len);
if (unlikely(dst_nents < 0)) {
dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",
- req->assoclen + req->cryptlen +
- (encrypt ? authsize : (-authsize)));
+ dst_len);
qi_cache_free(edesc);
return ERR_PTR(dst_nents);
}
@@ -1019,9 +1040,24 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
/*
* Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
* Input is not contiguous.
+ * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
+ * the end of the table by allocating more S/G entries. Logic:
+ * if (src != dst && output S/G)
+ * pad output S/G, if needed
+ * else if (src == dst && S/G)
+ * overlapping S/Gs; pad one of them
+ * else if (input S/G) ...
+ * pad input S/G, if needed
*/
- qm_sg_ents = 1 + !!ivsize + mapped_src_nents +
- (mapped_dst_nents > 1 ? mapped_dst_nents : 0);
+ qm_sg_ents = 1 + !!ivsize + mapped_src_nents;
+ if (mapped_dst_nents > 1)
+ qm_sg_ents += pad_sg_nents(mapped_dst_nents);
+ else if ((req->src == req->dst) && (mapped_src_nents > 1))
+ qm_sg_ents = max(pad_sg_nents(qm_sg_ents),
+ 1 + !!ivsize + pad_sg_nents(mapped_src_nents));
+ else
+ qm_sg_ents = pad_sg_nents(qm_sg_ents);
+
sg_table = &edesc->sgt[0];
qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
@@ -1029,7 +1065,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
qm_sg_ents, ivsize);
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
- 0, 0, 0);
+ 0, DMA_NONE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@@ -1044,7 +1080,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
if (dma_mapping_error(qidev, iv_dma)) {
dev_err(qidev, "unable to map IV\n");
caam_unmap(qidev, req->src, req->dst, src_nents,
- dst_nents, 0, 0, 0, 0);
+ dst_nents, 0, 0, DMA_NONE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@@ -1063,7 +1099,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
if (dma_mapping_error(qidev, edesc->assoclen_dma)) {
dev_err(qidev, "unable to map assoclen\n");
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
- iv_dma, ivsize, 0, 0);
+ iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@@ -1074,19 +1110,18 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
qm_sg_index++;
}
- sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0);
+ sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
qm_sg_index += mapped_src_nents;
if (mapped_dst_nents > 1)
- sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
- qm_sg_index, 0);
+ sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
if (dma_mapping_error(qidev, qm_sg_dma)) {
dev_err(qidev, "unable to map S/G table\n");
dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
- iv_dma, ivsize, 0, 0);
+ iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@@ -1109,7 +1144,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma +
(1 + !!ivsize) * sizeof(*sg_table),
out_len, 0);
- } else if (mapped_dst_nents == 1) {
+ } else if (mapped_dst_nents <= 1) {
dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len,
0);
} else {
@@ -1159,18 +1194,14 @@ static int aead_decrypt(struct aead_request *req)
static int ipsec_gcm_encrypt(struct aead_request *req)
{
- if (req->assoclen < 8)
- return -EINVAL;
-
- return aead_crypt(req, true);
+ return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req,
+ true);
}
static int ipsec_gcm_decrypt(struct aead_request *req)
{
- if (req->assoclen < 8)
- return -EINVAL;
-
- return aead_crypt(req, false);
+ return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req,
+ false);
}
static void skcipher_done(struct caam_drv_req *drv_req, u32 status)
@@ -1181,37 +1212,35 @@ static void skcipher_done(struct caam_drv_req *drv_req, u32 status)
struct caam_ctx *caam_ctx = crypto_skcipher_ctx(skcipher);
struct device *qidev = caam_ctx->qidev;
int ivsize = crypto_skcipher_ivsize(skcipher);
+ int ecode = 0;
-#ifdef DEBUG
- dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status);
-#endif
+ dev_dbg(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status);
edesc = container_of(drv_req, typeof(*edesc), drv_req);
if (status)
- caam_jr_strstatus(qidev, status);
+ ecode = caam_jr_strstatus(qidev, status);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
- edesc->src_nents > 1 ? 100 : ivsize, 1);
- caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ",
+ print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
+ edesc->src_nents > 1 ? 100 : ivsize, 1);
+ caam_dump_sg("dst @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
-#endif
skcipher_unmap(qidev, edesc, req);
/*
* The crypto API expects us to set the IV (req->iv) to the last
- * ciphertext block. This is used e.g. by the CTS mode.
+ * ciphertext block (CBC mode) or last counter (CTR mode).
+ * This is used e.g. by the CTS mode.
*/
- if (edesc->drv_req.drv_ctx->op_type == ENCRYPT)
- scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen -
- ivsize, ivsize, 0);
+ if (!ecode)
+ memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
+ ivsize);
qi_cache_free(edesc);
- skcipher_request_complete(req, status);
+ skcipher_request_complete(req, ecode);
}
static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
@@ -1276,14 +1305,26 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
qm_sg_ents = 1 + mapped_src_nents;
dst_sg_idx = qm_sg_ents;
- qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
+ /*
+ * Input, output HW S/G tables: [IV, src][dst, IV]
+ * IV entries point to the same buffer
+ * If src == dst, S/G entries are reused (S/G tables overlap)
+ *
+ * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
+ * the end of the table by allocating more S/G entries.
+ */
+ if (req->src != req->dst)
+ qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
+ else
+ qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
+
qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
ivsize > CAAM_QI_MEMCACHE_SIZE)) {
dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
qm_sg_ents, ivsize);
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
- 0, 0, 0);
+ 0, DMA_NONE, 0, 0);
return ERR_PTR(-ENOMEM);
}
@@ -1292,7 +1333,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
if (unlikely(!edesc)) {
dev_err(qidev, "could not allocate extended descriptor\n");
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
- 0, 0, 0);
+ 0, DMA_NONE, 0, 0);
return ERR_PTR(-ENOMEM);
}
@@ -1301,11 +1342,11 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
iv = (u8 *)(sg_table + qm_sg_ents);
memcpy(iv, req->iv, ivsize);
- iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE);
+ iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL);
if (dma_mapping_error(qidev, iv_dma)) {
dev_err(qidev, "unable to map IV\n");
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
- 0, 0, 0);
+ 0, DMA_NONE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@@ -1319,18 +1360,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
edesc->drv_req.drv_ctx = drv_ctx;
dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
- sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0);
+ sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
- if (mapped_dst_nents > 1)
- sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
- dst_sg_idx, 0);
+ if (req->src != req->dst)
+ sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
+
+ dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
+ ivsize, 0);
edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes,
DMA_TO_DEVICE);
if (dma_mapping_error(qidev, edesc->qm_sg_dma)) {
dev_err(qidev, "unable to map S/G table\n");
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
- iv_dma, ivsize, 0, 0);
+ iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@@ -1340,16 +1383,14 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma,
ivsize + req->cryptlen, 0);
- if (req->src == req->dst) {
+ if (req->src == req->dst)
dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma +
- sizeof(*sg_table), req->cryptlen, 0);
- } else if (mapped_dst_nents > 1) {
+ sizeof(*sg_table), req->cryptlen + ivsize,
+ 0);
+ else
dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx *
- sizeof(*sg_table), req->cryptlen, 0);
- } else {
- dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst),
- req->cryptlen, 0);
- }
+ sizeof(*sg_table), req->cryptlen + ivsize,
+ 0);
return edesc;
}
@@ -1359,9 +1400,11 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
struct skcipher_edesc *edesc;
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
- int ivsize = crypto_skcipher_ivsize(skcipher);
int ret;
+ if (!req->cryptlen)
+ return 0;
+
if (unlikely(caam_congested))
return -EAGAIN;
@@ -1370,14 +1413,6 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
if (IS_ERR(edesc))
return PTR_ERR(edesc);
- /*
- * The crypto API expects us to set the IV (req->iv) to the last
- * ciphertext block.
- */
- if (!encrypt)
- scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen -
- ivsize, ivsize, 0);
-
ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);
if (!ret) {
ret = -EINPROGRESS;
@@ -1407,7 +1442,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "cbc-aes-caam-qi",
.cra_blocksize = AES_BLOCK_SIZE,
},
- .setkey = skcipher_setkey,
+ .setkey = aes_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE,
@@ -1439,7 +1474,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "cbc-des-caam-qi",
.cra_blocksize = DES_BLOCK_SIZE,
},
- .setkey = skcipher_setkey,
+ .setkey = des_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = DES_KEY_SIZE,
@@ -1455,7 +1490,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "ctr-aes-caam-qi",
.cra_blocksize = 1,
},
- .setkey = skcipher_setkey,
+ .setkey = ctr_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE,
@@ -1473,7 +1508,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "rfc3686-ctr-aes-caam-qi",
.cra_blocksize = 1,
},
- .setkey = skcipher_setkey,
+ .setkey = rfc3686_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE +
@@ -2382,6 +2417,7 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
bool uses_dkp)
{
struct caam_drv_private *priv;
+ struct device *dev;
/*
* distribute tfms across job rings to ensure in-order
@@ -2393,16 +2429,17 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
return PTR_ERR(ctx->jrdev);
}
- priv = dev_get_drvdata(ctx->jrdev->parent);
+ dev = ctx->jrdev->parent;
+ priv = dev_get_drvdata(dev);
if (priv->era >= 6 && uses_dkp)
ctx->dir = DMA_BIDIRECTIONAL;
else
ctx->dir = DMA_TO_DEVICE;
- ctx->key_dma = dma_map_single(ctx->jrdev, ctx->key, sizeof(ctx->key),
+ ctx->key_dma = dma_map_single(dev, ctx->key, sizeof(ctx->key),
ctx->dir);
- if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) {
- dev_err(ctx->jrdev, "unable to map key\n");
+ if (dma_mapping_error(dev, ctx->key_dma)) {
+ dev_err(dev, "unable to map key\n");
caam_jr_free(ctx->jrdev);
return -ENOMEM;
}
@@ -2411,7 +2448,7 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
- ctx->qidev = priv->qidev;
+ ctx->qidev = dev;
spin_lock_init(&ctx->lock);
ctx->drv_ctx[ENCRYPT] = NULL;
@@ -2445,7 +2482,8 @@ static void caam_exit_common(struct caam_ctx *ctx)
caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]);
caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]);
- dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), ctx->dir);
+ dma_unmap_single(ctx->jrdev->parent, ctx->key_dma, sizeof(ctx->key),
+ ctx->dir);
caam_jr_free(ctx->jrdev);
}
@@ -2460,7 +2498,7 @@ static void caam_aead_exit(struct crypto_aead *tfm)
caam_exit_common(crypto_aead_ctx(tfm));
}
-static void __exit caam_qi_algapi_exit(void)
+void caam_qi_algapi_exit(void)
{
int i;
@@ -2505,46 +2543,17 @@ static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
alg->exit = caam_aead_exit;
}
-static int __init caam_qi_algapi_init(void)
+int caam_qi_algapi_init(struct device *ctrldev)
{
- struct device_node *dev_node;
- struct platform_device *pdev;
- struct device *ctrldev;
- struct caam_drv_private *priv;
+ struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
int i = 0, err = 0;
u32 aes_vid, aes_inst, des_inst, md_vid, md_inst;
unsigned int md_limit = SHA512_DIGEST_SIZE;
bool registered = false;
- dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
- if (!dev_node) {
- dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
- if (!dev_node)
- return -ENODEV;
- }
-
- pdev = of_find_device_by_node(dev_node);
- of_node_put(dev_node);
- if (!pdev)
- return -ENODEV;
-
- ctrldev = &pdev->dev;
- priv = dev_get_drvdata(ctrldev);
-
- /*
- * If priv is NULL, it's probably because the caam driver wasn't
- * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
- */
- if (!priv || !priv->qi_present) {
- err = -ENODEV;
- goto out_put_dev;
- }
-
- if (caam_dpaa2) {
- dev_info(ctrldev, "caam/qi frontend driver not suitable for DPAA 2.x, aborting...\n");
- err = -ENODEV;
- goto out_put_dev;
- }
+ /* Make sure this runs only on (DPAA 1.x) QI */
+ if (!priv->qi_present || caam_dpaa2)
+ return 0;
/*
* Register crypto algorithms the device supports.
@@ -2598,7 +2607,7 @@ static int __init caam_qi_algapi_init(void)
err = crypto_register_skcipher(&t_alg->skcipher);
if (err) {
- dev_warn(priv->qidev, "%s alg registration failed\n",
+ dev_warn(ctrldev, "%s alg registration failed\n",
t_alg->skcipher.base.cra_driver_name);
continue;
}
@@ -2654,16 +2663,7 @@ static int __init caam_qi_algapi_init(void)
}
if (registered)
- dev_info(priv->qidev, "algorithms registered in /proc/crypto\n");
+ dev_info(ctrldev, "algorithms registered in /proc/crypto\n");
-out_put_dev:
- put_device(ctrldev);
return err;
}
-
-module_init(caam_qi_algapi_init);
-module_exit(caam_qi_algapi_exit);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Support for crypto API using CAAM-QI backend");
-MODULE_AUTHOR("Freescale Semiconductor");
diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c
index 2b2980a8a9b9..3443f6d6dd83 100644
--- a/drivers/crypto/caam/caamalg_qi2.c
+++ b/drivers/crypto/caam/caamalg_qi2.c
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
/*
* Copyright 2015-2016 Freescale Semiconductor Inc.
- * Copyright 2017-2018 NXP
+ * Copyright 2017-2019 NXP
*/
#include "compat.h"
@@ -15,6 +15,7 @@
#include "key_gen.h"
#include "caamalg_desc.h"
#include "caamhash_desc.h"
+#include "dpseci-debugfs.h"
#include <linux/fsl/mc.h>
#include <soc/fsl/dpaa2-io.h>
#include <soc/fsl/dpaa2-fd.h>
@@ -140,7 +141,8 @@ static struct caam_request *to_caam_req(struct crypto_async_request *areq)
static void caam_unmap(struct device *dev, struct scatterlist *src,
struct scatterlist *dst, int src_nents,
int dst_nents, dma_addr_t iv_dma, int ivsize,
- dma_addr_t qm_sg_dma, int qm_sg_bytes)
+ enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
+ int qm_sg_bytes)
{
if (dst != src) {
if (src_nents)
@@ -152,7 +154,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src,
}
if (iv_dma)
- dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
+ dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
if (qm_sg_bytes)
dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
@@ -197,6 +199,18 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
}
+ /*
+ * In case |user key| > |derived key|, using DKP<imm,imm> would result
+ * in invalid opcodes (last bytes of user key) in the resulting
+ * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
+ * addresses are needed.
+ */
+ ctx->adata.key_virt = ctx->key;
+ ctx->adata.key_dma = ctx->key_dma;
+
+ ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
+ ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
+
data_len[0] = ctx->adata.keylen_pad;
data_len[1] = ctx->cdata.keylen;
@@ -208,16 +222,6 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
ARRAY_SIZE(data_len)) < 0)
return -EINVAL;
- if (inl_mask & 1)
- ctx->adata.key_virt = ctx->key;
- else
- ctx->adata.key_dma = ctx->key_dma;
-
- if (inl_mask & 2)
- ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
- else
- ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
-
ctx->adata.key_inline = !!(inl_mask & 1);
ctx->cdata.key_inline = !!(inl_mask & 2);
@@ -246,16 +250,6 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
ARRAY_SIZE(data_len)) < 0)
return -EINVAL;
- if (inl_mask & 1)
- ctx->adata.key_virt = ctx->key;
- else
- ctx->adata.key_dma = ctx->key_dma;
-
- if (inl_mask & 2)
- ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
- else
- ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
-
ctx->adata.key_inline = !!(inl_mask & 1);
ctx->cdata.key_inline = !!(inl_mask & 2);
@@ -328,7 +322,6 @@ static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
unsigned int keylen)
{
struct crypto_authenc_keys keys;
- u32 flags;
int err;
err = crypto_authenc_extractkeys(&keys, key, keylen);
@@ -339,14 +332,8 @@ static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
if (keys.enckeylen != DES3_EDE_KEY_SIZE)
goto badkey;
- flags = crypto_aead_get_flags(aead);
- err = __des3_verify_key(&flags, keys.enckey);
- if (unlikely(err)) {
- crypto_aead_set_flags(aead, flags);
- goto out;
- }
-
- err = aead_setkey(aead, key, keylen);
+ err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?:
+ aead_setkey(aead, key, keylen);
out:
memzero_explicit(&keys, sizeof(keys));
@@ -371,6 +358,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
+ int src_len, dst_len = 0;
struct aead_edesc *edesc;
dma_addr_t qm_sg_dma, iv_dma = 0;
int ivsize = 0;
@@ -387,23 +375,21 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
}
if (unlikely(req->dst != req->src)) {
- src_nents = sg_nents_for_len(req->src, req->assoclen +
- req->cryptlen);
+ src_len = req->assoclen + req->cryptlen;
+ dst_len = src_len + (encrypt ? authsize : (-authsize));
+
+ src_nents = sg_nents_for_len(req->src, src_len);
if (unlikely(src_nents < 0)) {
dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
- req->assoclen + req->cryptlen);
+ src_len);
qi_cache_free(edesc);
return ERR_PTR(src_nents);
}
- dst_nents = sg_nents_for_len(req->dst, req->assoclen +
- req->cryptlen +
- (encrypt ? authsize :
- (-authsize)));
+ dst_nents = sg_nents_for_len(req->dst, dst_len);
if (unlikely(dst_nents < 0)) {
dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
- req->assoclen + req->cryptlen +
- (encrypt ? authsize : (-authsize)));
+ dst_len);
qi_cache_free(edesc);
return ERR_PTR(dst_nents);
}
@@ -434,13 +420,13 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
mapped_dst_nents = 0;
}
} else {
- src_nents = sg_nents_for_len(req->src, req->assoclen +
- req->cryptlen +
- (encrypt ? authsize : 0));
+ src_len = req->assoclen + req->cryptlen +
+ (encrypt ? authsize : 0);
+
+ src_nents = sg_nents_for_len(req->src, src_len);
if (unlikely(src_nents < 0)) {
dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
- req->assoclen + req->cryptlen +
- (encrypt ? authsize : 0));
+ src_len);
qi_cache_free(edesc);
return ERR_PTR(src_nents);
}
@@ -460,9 +446,25 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
/*
* Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
* Input is not contiguous.
+ * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
+ * the end of the table by allocating more S/G entries. Logic:
+ * if (src != dst && output S/G)
+ * pad output S/G, if needed
+ * else if (src == dst && S/G)
+ * overlapping S/Gs; pad one of them
+ * else if (input S/G) ...
+ * pad input S/G, if needed
*/
- qm_sg_nents = 1 + !!ivsize + mapped_src_nents +
- (mapped_dst_nents > 1 ? mapped_dst_nents : 0);
+ qm_sg_nents = 1 + !!ivsize + mapped_src_nents;
+ if (mapped_dst_nents > 1)
+ qm_sg_nents += pad_sg_nents(mapped_dst_nents);
+ else if ((req->src == req->dst) && (mapped_src_nents > 1))
+ qm_sg_nents = max(pad_sg_nents(qm_sg_nents),
+ 1 + !!ivsize +
+ pad_sg_nents(mapped_src_nents));
+ else
+ qm_sg_nents = pad_sg_nents(qm_sg_nents);
+
sg_table = &edesc->sgt[0];
qm_sg_bytes = qm_sg_nents * sizeof(*sg_table);
if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
@@ -470,7 +472,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
qm_sg_nents, ivsize);
caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
- 0, 0, 0);
+ 0, DMA_NONE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@@ -485,7 +487,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
if (dma_mapping_error(dev, iv_dma)) {
dev_err(dev, "unable to map IV\n");
caam_unmap(dev, req->src, req->dst, src_nents,
- dst_nents, 0, 0, 0, 0);
+ dst_nents, 0, 0, DMA_NONE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@@ -509,7 +511,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
if (dma_mapping_error(dev, edesc->assoclen_dma)) {
dev_err(dev, "unable to map assoclen\n");
caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
- iv_dma, ivsize, 0, 0);
+ iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@@ -520,19 +522,18 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
qm_sg_index++;
}
- sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0);
+ sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
qm_sg_index += mapped_src_nents;
if (mapped_dst_nents > 1)
- sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
- qm_sg_index, 0);
+ sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
if (dma_mapping_error(dev, qm_sg_dma)) {
dev_err(dev, "unable to map S/G table\n");
dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
- iv_dma, ivsize, 0, 0);
+ iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@@ -559,6 +560,14 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dpaa2_fl_set_addr(out_fle, qm_sg_dma +
(1 + !!ivsize) * sizeof(*sg_table));
}
+ } else if (!mapped_dst_nents) {
+ /*
+ * crypto engine requires the output entry to be present when
+ * "frame list" FD is used.
+ * Since engine does not support FMT=2'b11 (unused entry type),
+ * leaving out_fle zeroized is the best option.
+ */
+ goto skip_out_fle;
} else if (mapped_dst_nents == 1) {
dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
@@ -570,6 +579,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dpaa2_fl_set_len(out_fle, out_len);
+skip_out_fle:
return edesc;
}
@@ -695,6 +705,11 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+ int err;
+
+ err = crypto_gcm_check_authsize(authsize);
+ if (err)
+ return err;
ctx->authsize = authsize;
gcm_set_sh_desc(authenc);
@@ -707,7 +722,13 @@ static int gcm_setkey(struct crypto_aead *aead,
{
struct caam_ctx *ctx = crypto_aead_ctx(aead);
struct device *dev = ctx->dev;
+ int ret;
+ ret = aes_check_keylen(keylen);
+ if (ret) {
+ crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return ret;
+ }
print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
@@ -781,6 +802,11 @@ static int rfc4106_setauthsize(struct crypto_aead *authenc,
unsigned int authsize)
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+ int err;
+
+ err = crypto_rfc4106_check_authsize(authsize);
+ if (err)
+ return err;
ctx->authsize = authsize;
rfc4106_set_sh_desc(authenc);
@@ -793,9 +819,13 @@ static int rfc4106_setkey(struct crypto_aead *aead,
{
struct caam_ctx *ctx = crypto_aead_ctx(aead);
struct device *dev = ctx->dev;
+ int ret;
- if (keylen < 4)
- return -EINVAL;
+ ret = aes_check_keylen(keylen - 4);
+ if (ret) {
+ crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return ret;
+ }
print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
@@ -876,6 +906,9 @@ static int rfc4543_setauthsize(struct crypto_aead *authenc,
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+ if (authsize != 16)
+ return -EINVAL;
+
ctx->authsize = authsize;
rfc4543_set_sh_desc(authenc);
@@ -887,9 +920,13 @@ static int rfc4543_setkey(struct crypto_aead *aead,
{
struct caam_ctx *ctx = crypto_aead_ctx(aead);
struct device *dev = ctx->dev;
+ int ret;
- if (keylen < 4)
- return -EINVAL;
+ ret = aes_check_keylen(keylen - 4);
+ if (ret) {
+ crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return ret;
+ }
print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
@@ -907,7 +944,7 @@ static int rfc4543_setkey(struct crypto_aead *aead,
}
static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
- unsigned int keylen)
+ unsigned int keylen, const u32 ctx1_iv_off)
{
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
struct caam_skcipher_alg *alg =
@@ -917,34 +954,11 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
struct caam_flc *flc;
unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
u32 *desc;
- u32 ctx1_iv_off = 0;
- const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
- OP_ALG_AAI_CTR_MOD128) &&
- ((ctx->cdata.algtype & OP_ALG_ALGSEL_MASK) !=
- OP_ALG_ALGSEL_CHACHA20);
const bool is_rfc3686 = alg->caam.rfc3686;
print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
- /*
- * AES-CTR needs to load IV in CONTEXT1 reg
- * at an offset of 128bits (16bytes)
- * CONTEXT1[255:128] = IV
- */
- if (ctr_mode)
- ctx1_iv_off = 16;
-
- /*
- * RFC3686 specific:
- * | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
- * | *key = {KEY, NONCE}
- */
- if (is_rfc3686) {
- ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
- keylen -= CTR_RFC3686_NONCE_SIZE;
- }
-
ctx->cdata.keylen = keylen;
ctx->cdata.key_virt = key;
ctx->cdata.key_inline = true;
@@ -972,11 +986,92 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
return 0;
}
+static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
+ const u8 *key, unsigned int keylen)
+{
+ int err;
+
+ err = aes_check_keylen(keylen);
+ if (err) {
+ crypto_skcipher_set_flags(skcipher,
+ CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return err;
+ }
+
+ return skcipher_setkey(skcipher, key, keylen, 0);
+}
+
+static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
+ const u8 *key, unsigned int keylen)
+{
+ u32 ctx1_iv_off;
+ int err;
+
+ /*
+ * RFC3686 specific:
+ * | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
+ * | *key = {KEY, NONCE}
+ */
+ ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
+ keylen -= CTR_RFC3686_NONCE_SIZE;
+
+ err = aes_check_keylen(keylen);
+ if (err) {
+ crypto_skcipher_set_flags(skcipher,
+ CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return err;
+ }
+
+ return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
+}
+
+static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
+ const u8 *key, unsigned int keylen)
+{
+ u32 ctx1_iv_off;
+ int err;
+
+ /*
+ * AES-CTR needs to load IV in CONTEXT1 reg
+ * at an offset of 128bits (16bytes)
+ * CONTEXT1[255:128] = IV
+ */
+ ctx1_iv_off = 16;
+
+ err = aes_check_keylen(keylen);
+ if (err) {
+ crypto_skcipher_set_flags(skcipher,
+ CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return err;
+ }
+
+ return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
+}
+
+static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher,
+ const u8 *key, unsigned int keylen)
+{
+ if (keylen != CHACHA_KEY_SIZE) {
+ crypto_skcipher_set_flags(skcipher,
+ CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+ }
+
+ return skcipher_setkey(skcipher, key, keylen, 0);
+}
+
+static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
+ const u8 *key, unsigned int keylen)
+{
+ return verify_skcipher_des_key(skcipher, key) ?:
+ skcipher_setkey(skcipher, key, keylen, 0);
+}
+
static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
- const u8 *key, unsigned int keylen)
+ const u8 *key, unsigned int keylen)
{
- return unlikely(des3_verify_key(skcipher, key)) ?:
- skcipher_setkey(skcipher, key, keylen);
+ return verify_skcipher_des3_key(skcipher, key) ?:
+ skcipher_setkey(skcipher, key, keylen, 0);
}
static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
@@ -1077,14 +1172,26 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
qm_sg_ents = 1 + mapped_src_nents;
dst_sg_idx = qm_sg_ents;
- qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
+ /*
+ * Input, output HW S/G tables: [IV, src][dst, IV]
+ * IV entries point to the same buffer
+ * If src == dst, S/G entries are reused (S/G tables overlap)
+ *
+ * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
+ * the end of the table by allocating more S/G entries.
+ */
+ if (req->src != req->dst)
+ qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
+ else
+ qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
+
qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry);
if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
ivsize > CAAM_QI_MEMCACHE_SIZE)) {
dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
qm_sg_ents, ivsize);
caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
- 0, 0, 0);
+ 0, DMA_NONE, 0, 0);
return ERR_PTR(-ENOMEM);
}
@@ -1093,7 +1200,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
if (unlikely(!edesc)) {
dev_err(dev, "could not allocate extended descriptor\n");
caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
- 0, 0, 0);
+ 0, DMA_NONE, 0, 0);
return ERR_PTR(-ENOMEM);
}
@@ -1102,11 +1209,11 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
iv = (u8 *)(sg_table + qm_sg_ents);
memcpy(iv, req->iv, ivsize);
- iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
+ iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL);
if (dma_mapping_error(dev, iv_dma)) {
dev_err(dev, "unable to map IV\n");
caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
- 0, 0, 0);
+ 0, DMA_NONE, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@@ -1117,18 +1224,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
edesc->qm_sg_bytes = qm_sg_bytes;
dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
- sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0);
+ sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
- if (mapped_dst_nents > 1)
- sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
- dst_sg_idx, 0);
+ if (req->src != req->dst)
+ sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
+
+ dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
+ ivsize, 0);
edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes,
DMA_TO_DEVICE);
if (dma_mapping_error(dev, edesc->qm_sg_dma)) {
dev_err(dev, "unable to map S/G table\n");
caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
- iv_dma, ivsize, 0, 0);
+ iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
@@ -1136,23 +1245,19 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
dpaa2_fl_set_final(in_fle, true);
dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize);
- dpaa2_fl_set_len(out_fle, req->cryptlen);
+ dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize);
dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
- if (req->src == req->dst) {
- dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
+ dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
+
+ if (req->src == req->dst)
dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma +
sizeof(*sg_table));
- } else if (mapped_dst_nents > 1) {
- dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
+ else
dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx *
sizeof(*sg_table));
- } else {
- dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
- dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
- }
return edesc;
}
@@ -1164,7 +1269,8 @@ static void aead_unmap(struct device *dev, struct aead_edesc *edesc,
int ivsize = crypto_aead_ivsize(aead);
caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
- edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
+ edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
+ edesc->qm_sg_bytes);
dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
}
@@ -1175,7 +1281,8 @@ static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
int ivsize = crypto_skcipher_ivsize(skcipher);
caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
- edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
+ edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
+ edesc->qm_sg_bytes);
}
static void aead_encrypt_done(void *cbk_ctx, u32 status)
@@ -1191,10 +1298,8 @@ static void aead_encrypt_done(void *cbk_ctx, u32 status)
dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
- if (unlikely(status)) {
- caam_qi2_strstatus(ctx->dev, status);
- ecode = -EIO;
- }
+ if (unlikely(status))
+ ecode = caam_qi2_strstatus(ctx->dev, status);
aead_unmap(ctx->dev, edesc, req);
qi_cache_free(edesc);
@@ -1214,17 +1319,8 @@ static void aead_decrypt_done(void *cbk_ctx, u32 status)
dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
- if (unlikely(status)) {
- caam_qi2_strstatus(ctx->dev, status);
- /*
- * verify hw auth check passed else return -EBADMSG
- */
- if ((status & JRSTA_CCBERR_ERRID_MASK) ==
- JRSTA_CCBERR_ERRID_ICVCHK)
- ecode = -EBADMSG;
- else
- ecode = -EIO;
- }
+ if (unlikely(status))
+ ecode = caam_qi2_strstatus(ctx->dev, status);
aead_unmap(ctx->dev, edesc, req);
qi_cache_free(edesc);
@@ -1289,18 +1385,12 @@ static int aead_decrypt(struct aead_request *req)
static int ipsec_gcm_encrypt(struct aead_request *req)
{
- if (req->assoclen < 8)
- return -EINVAL;
-
- return aead_encrypt(req);
+ return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req);
}
static int ipsec_gcm_decrypt(struct aead_request *req)
{
- if (req->assoclen < 8)
- return -EINVAL;
-
- return aead_decrypt(req);
+ return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req);
}
static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
@@ -1316,15 +1406,13 @@ static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
- if (unlikely(status)) {
- caam_qi2_strstatus(ctx->dev, status);
- ecode = -EIO;
- }
+ if (unlikely(status))
+ ecode = caam_qi2_strstatus(ctx->dev, status);
print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
edesc->src_nents > 1 ? 100 : ivsize, 1);
- caam_dump_sg(KERN_DEBUG, "dst @" __stringify(__LINE__)": ",
+ caam_dump_sg("dst @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
@@ -1332,10 +1420,12 @@ static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
/*
* The crypto API expects us to set the IV (req->iv) to the last
- * ciphertext block. This is used e.g. by the CTS mode.
+ * ciphertext block (CBC mode) or last counter (CTR mode).
+ * This is used e.g. by the CTS mode.
*/
- scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - ivsize,
- ivsize, 0);
+ if (!ecode)
+ memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
+ ivsize);
qi_cache_free(edesc);
skcipher_request_complete(req, ecode);
@@ -1354,19 +1444,27 @@ static void skcipher_decrypt_done(void *cbk_ctx, u32 status)
dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
- if (unlikely(status)) {
- caam_qi2_strstatus(ctx->dev, status);
- ecode = -EIO;
- }
+ if (unlikely(status))
+ ecode = caam_qi2_strstatus(ctx->dev, status);
print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
edesc->src_nents > 1 ? 100 : ivsize, 1);
- caam_dump_sg(KERN_DEBUG, "dst @" __stringify(__LINE__)": ",
+ caam_dump_sg("dst @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
skcipher_unmap(ctx->dev, edesc, req);
+
+ /*
+ * The crypto API expects us to set the IV (req->iv) to the last
+ * ciphertext block (CBC mode) or last counter (CTR mode).
+ * This is used e.g. by the CTS mode.
+ */
+ if (!ecode)
+ memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
+ ivsize);
+
qi_cache_free(edesc);
skcipher_request_complete(req, ecode);
}
@@ -1379,6 +1477,9 @@ static int skcipher_encrypt(struct skcipher_request *req)
struct caam_request *caam_req = skcipher_request_ctx(req);
int ret;
+ if (!req->cryptlen)
+ return 0;
+
/* allocate extended descriptor */
edesc = skcipher_edesc_alloc(req);
if (IS_ERR(edesc))
@@ -1405,21 +1506,15 @@ static int skcipher_decrypt(struct skcipher_request *req)
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
struct caam_request *caam_req = skcipher_request_ctx(req);
- int ivsize = crypto_skcipher_ivsize(skcipher);
int ret;
+ if (!req->cryptlen)
+ return 0;
/* allocate extended descriptor */
edesc = skcipher_edesc_alloc(req);
if (IS_ERR(edesc))
return PTR_ERR(edesc);
- /*
- * The crypto API expects us to set the IV (req->iv) to the last
- * ciphertext block.
- */
- scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - ivsize,
- ivsize, 0);
-
caam_req->flc = &ctx->flc[DECRYPT];
caam_req->flc_dma = ctx->flc_dma[DECRYPT];
caam_req->cbk = skcipher_decrypt_done;
@@ -1509,7 +1604,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "cbc-aes-caam-qi2",
.cra_blocksize = AES_BLOCK_SIZE,
},
- .setkey = skcipher_setkey,
+ .setkey = aes_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE,
@@ -1541,7 +1636,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "cbc-des-caam-qi2",
.cra_blocksize = DES_BLOCK_SIZE,
},
- .setkey = skcipher_setkey,
+ .setkey = des_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = DES_KEY_SIZE,
@@ -1557,7 +1652,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "ctr-aes-caam-qi2",
.cra_blocksize = 1,
},
- .setkey = skcipher_setkey,
+ .setkey = ctr_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE,
@@ -1575,7 +1670,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "rfc3686-ctr-aes-caam-qi2",
.cra_blocksize = 1,
},
- .setkey = skcipher_setkey,
+ .setkey = rfc3686_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE +
@@ -1614,7 +1709,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "chacha20-caam-qi2",
.cra_blocksize = 1,
},
- .setkey = skcipher_setkey,
+ .setkey = chacha20_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = CHACHA_KEY_SIZE,
@@ -2882,6 +2977,7 @@ enum hash_optype {
/**
* caam_hash_ctx - ahash per-session context
* @flc: Flow Contexts array
+ * @key: authentication key
* @flc_dma: I/O virtual addresses of the Flow Contexts
* @dev: dpseci device
* @ctx_len: size of Context Register
@@ -2889,6 +2985,7 @@ enum hash_optype {
*/
struct caam_hash_ctx {
struct caam_flc flc[HASH_NUM_OP];
+ u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
dma_addr_t flc_dma[HASH_NUM_OP];
struct device *dev;
int ctx_len;
@@ -3058,10 +3155,7 @@ static void split_key_sh_done(void *cbk_ctx, u32 err)
dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
- if (err)
- caam_qi2_strstatus(res->dev, err);
-
- res->err = err;
+ res->err = err ? caam_qi2_strstatus(res->dev, err) : 0;
complete(&res->completion);
}
@@ -3192,6 +3286,19 @@ static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
ctx->adata.key_virt = key;
ctx->adata.key_inline = true;
+ /*
+ * In case |user key| > |derived key|, using DKP<imm,imm> would result
+ * in invalid opcodes (last bytes of user key) in the resulting
+ * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
+ * addresses are needed.
+ */
+ if (keylen > ctx->adata.keylen_pad) {
+ memcpy(ctx->key, key, keylen);
+ dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma,
+ ctx->adata.keylen_pad,
+ DMA_TO_DEVICE);
+ }
+
ret = ahash_set_sh_desc(ahash);
kfree(hashed_key);
return ret;
@@ -3246,10 +3353,8 @@ static void ahash_done(void *cbk_ctx, u32 status)
dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
- if (unlikely(status)) {
- caam_qi2_strstatus(ctx->dev, status);
- ecode = -EIO;
- }
+ if (unlikely(status))
+ ecode = caam_qi2_strstatus(ctx->dev, status);
ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
memcpy(req->result, state->caam_ctx, digestsize);
@@ -3274,10 +3379,8 @@ static void ahash_done_bi(void *cbk_ctx, u32 status)
dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
- if (unlikely(status)) {
- caam_qi2_strstatus(ctx->dev, status);
- ecode = -EIO;
- }
+ if (unlikely(status))
+ ecode = caam_qi2_strstatus(ctx->dev, status);
ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
switch_buf(state);
@@ -3307,10 +3410,8 @@ static void ahash_done_ctx_src(void *cbk_ctx, u32 status)
dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
- if (unlikely(status)) {
- caam_qi2_strstatus(ctx->dev, status);
- ecode = -EIO;
- }
+ if (unlikely(status))
+ ecode = caam_qi2_strstatus(ctx->dev, status);
ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
memcpy(req->result, state->caam_ctx, digestsize);
@@ -3335,10 +3436,8 @@ static void ahash_done_ctx_dst(void *cbk_ctx, u32 status)
dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
- if (unlikely(status)) {
- caam_qi2_strstatus(ctx->dev, status);
- ecode = -EIO;
- }
+ if (unlikely(status))
+ ecode = caam_qi2_strstatus(ctx->dev, status);
ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
switch_buf(state);
@@ -3380,9 +3479,9 @@ static int ahash_update_ctx(struct ahash_request *req)
if (to_hash) {
struct dpaa2_sg_entry *sg_table;
+ int src_len = req->nbytes - *next_buflen;
- src_nents = sg_nents_for_len(req->src,
- req->nbytes - (*next_buflen));
+ src_nents = sg_nents_for_len(req->src, src_len);
if (src_nents < 0) {
dev_err(ctx->dev, "Invalid number of src SG.\n");
return src_nents;
@@ -3409,7 +3508,7 @@ static int ahash_update_ctx(struct ahash_request *req)
edesc->src_nents = src_nents;
qm_sg_src_index = 1 + (*buflen ? 1 : 0);
- qm_sg_bytes = (qm_sg_src_index + mapped_nents) *
+ qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
sizeof(*sg_table);
sg_table = &edesc->sgt[0];
@@ -3423,7 +3522,7 @@ static int ahash_update_ctx(struct ahash_request *req)
goto unmap_ctx;
if (mapped_nents) {
- sg_to_qm_sg_last(req->src, mapped_nents,
+ sg_to_qm_sg_last(req->src, src_len,
sg_table + qm_sg_src_index, 0);
if (*next_buflen)
scatterwalk_map_and_copy(next_buf, req->src,
@@ -3494,7 +3593,7 @@ static int ahash_final_ctx(struct ahash_request *req)
gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
int buflen = *current_buflen(state);
- int qm_sg_bytes, qm_sg_src_index;
+ int qm_sg_bytes;
int digestsize = crypto_ahash_digestsize(ahash);
struct ahash_edesc *edesc;
struct dpaa2_sg_entry *sg_table;
@@ -3505,8 +3604,7 @@ static int ahash_final_ctx(struct ahash_request *req)
if (!edesc)
return -ENOMEM;
- qm_sg_src_index = 1 + (buflen ? 1 : 0);
- qm_sg_bytes = qm_sg_src_index * sizeof(*sg_table);
+ qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table);
sg_table = &edesc->sgt[0];
ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
@@ -3518,7 +3616,7 @@ static int ahash_final_ctx(struct ahash_request *req)
if (ret)
goto unmap_ctx;
- dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, true);
+ dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true);
edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
DMA_TO_DEVICE);
@@ -3599,7 +3697,8 @@ static int ahash_finup_ctx(struct ahash_request *req)
edesc->src_nents = src_nents;
qm_sg_src_index = 1 + (buflen ? 1 : 0);
- qm_sg_bytes = (qm_sg_src_index + mapped_nents) * sizeof(*sg_table);
+ qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
+ sizeof(*sg_table);
sg_table = &edesc->sgt[0];
ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
@@ -3611,7 +3710,7 @@ static int ahash_finup_ctx(struct ahash_request *req)
if (ret)
goto unmap_ctx;
- sg_to_qm_sg_last(req->src, mapped_nents, sg_table + qm_sg_src_index, 0);
+ sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0);
edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
DMA_TO_DEVICE);
@@ -3696,8 +3795,8 @@ static int ahash_digest(struct ahash_request *req)
int qm_sg_bytes;
struct dpaa2_sg_entry *sg_table = &edesc->sgt[0];
- qm_sg_bytes = mapped_nents * sizeof(*sg_table);
- sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0);
+ qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table);
+ sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0);
edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
qm_sg_bytes, DMA_TO_DEVICE);
if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
@@ -3840,9 +3939,9 @@ static int ahash_update_no_ctx(struct ahash_request *req)
if (to_hash) {
struct dpaa2_sg_entry *sg_table;
+ int src_len = req->nbytes - *next_buflen;
- src_nents = sg_nents_for_len(req->src,
- req->nbytes - *next_buflen);
+ src_nents = sg_nents_for_len(req->src, src_len);
if (src_nents < 0) {
dev_err(ctx->dev, "Invalid number of src SG.\n");
return src_nents;
@@ -3868,14 +3967,15 @@ static int ahash_update_no_ctx(struct ahash_request *req)
}
edesc->src_nents = src_nents;
- qm_sg_bytes = (1 + mapped_nents) * sizeof(*sg_table);
+ qm_sg_bytes = pad_sg_nents(1 + mapped_nents) *
+ sizeof(*sg_table);
sg_table = &edesc->sgt[0];
ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
if (ret)
goto unmap_ctx;
- sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0);
+ sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0);
if (*next_buflen)
scatterwalk_map_and_copy(next_buf, req->src,
@@ -3987,14 +4087,14 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
}
edesc->src_nents = src_nents;
- qm_sg_bytes = (2 + mapped_nents) * sizeof(*sg_table);
+ qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table);
sg_table = &edesc->sgt[0];
ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
if (ret)
goto unmap;
- sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0);
+ sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0);
edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
DMA_TO_DEVICE);
@@ -4064,9 +4164,9 @@ static int ahash_update_first(struct ahash_request *req)
if (to_hash) {
struct dpaa2_sg_entry *sg_table;
+ int src_len = req->nbytes - *next_buflen;
- src_nents = sg_nents_for_len(req->src,
- req->nbytes - (*next_buflen));
+ src_nents = sg_nents_for_len(req->src, src_len);
if (src_nents < 0) {
dev_err(ctx->dev, "Invalid number of src SG.\n");
return src_nents;
@@ -4101,8 +4201,9 @@ static int ahash_update_first(struct ahash_request *req)
if (mapped_nents > 1) {
int qm_sg_bytes;
- sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0);
- qm_sg_bytes = mapped_nents * sizeof(*sg_table);
+ sg_to_qm_sg_last(req->src, src_len, sg_table, 0);
+ qm_sg_bytes = pad_sg_nents(mapped_nents) *
+ sizeof(*sg_table);
edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
qm_sg_bytes,
DMA_TO_DEVICE);
@@ -4428,11 +4529,27 @@ static int caam_hash_cra_init(struct crypto_tfm *tfm)
ctx->dev = caam_hash->dev;
+ if (alg->setkey) {
+ ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key,
+ ARRAY_SIZE(ctx->key),
+ DMA_TO_DEVICE,
+ DMA_ATTR_SKIP_CPU_SYNC);
+ if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) {
+ dev_err(ctx->dev, "unable to map key\n");
+ return -ENOMEM;
+ }
+ }
+
dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc),
DMA_BIDIRECTIONAL,
DMA_ATTR_SKIP_CPU_SYNC);
if (dma_mapping_error(ctx->dev, dma_addr)) {
dev_err(ctx->dev, "unable to map shared descriptors\n");
+ if (ctx->adata.key_dma)
+ dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
+ ARRAY_SIZE(ctx->key),
+ DMA_TO_DEVICE,
+ DMA_ATTR_SKIP_CPU_SYNC);
return -ENOMEM;
}
@@ -4458,6 +4575,10 @@ static void caam_hash_cra_exit(struct crypto_tfm *tfm)
dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc),
DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
+ if (ctx->adata.key_dma)
+ dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
+ ARRAY_SIZE(ctx->key), DMA_TO_DEVICE,
+ DMA_ATTR_SKIP_CPU_SYNC);
}
static struct caam_hash_alg *caam_hash_alloc(struct device *dev,
@@ -4662,7 +4783,7 @@ static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv,
fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK;
if (unlikely(fd_err))
- dev_err(priv->dev, "FD error: %08x\n", fd_err);
+ dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err);
/*
* FD[ADDR] is guaranteed to be valid, irrespective of errors reported
@@ -5060,6 +5181,8 @@ static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev)
goto err_bind;
}
+ dpaa2_dpseci_debugfs_init(priv);
+
/* register crypto algorithms the device supports */
for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
struct caam_skcipher_alg *t_alg = driver_algs + i;
@@ -5227,6 +5350,8 @@ static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev)
dev = &ls_dev->dev;
priv = dev_get_drvdata(dev);
+ dpaa2_dpseci_debugfs_exit(priv);
+
for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
struct caam_aead_alg *t_alg = driver_aeads + i;
diff --git a/drivers/crypto/caam/caamalg_qi2.h b/drivers/crypto/caam/caamalg_qi2.h
index be5085451053..706736776b47 100644
--- a/drivers/crypto/caam/caamalg_qi2.h
+++ b/drivers/crypto/caam/caamalg_qi2.h
@@ -10,12 +10,13 @@
#include <soc/fsl/dpaa2-io.h>
#include <soc/fsl/dpaa2-fd.h>
#include <linux/threads.h>
+#include <linux/netdevice.h>
#include "dpseci.h"
#include "desc_constr.h"
#define DPAA2_CAAM_STORE_SIZE 16
/* NAPI weight *must* be a multiple of the store size. */
-#define DPAA2_CAAM_NAPI_WEIGHT 64
+#define DPAA2_CAAM_NAPI_WEIGHT 512
/* The congestion entrance threshold was chosen so that on LS2088
* we support the maximum throughput for the available memory
@@ -64,6 +65,7 @@ struct dpaa2_caam_priv {
struct iommu_domain *domain;
struct dpaa2_caam_priv_per_cpu __percpu *ppriv;
+ struct dentry *dfs_root;
};
/**
@@ -90,33 +92,6 @@ struct dpaa2_caam_priv_per_cpu {
struct dpaa2_io *dpio;
};
-/*
- * The CAAM QI hardware constructs a job descriptor which points
- * to shared descriptor (as pointed by context_a of FQ to CAAM).
- * When the job descriptor is executed by deco, the whole job
- * descriptor together with shared descriptor gets loaded in
- * deco buffer which is 64 words long (each 32-bit).
- *
- * The job descriptor constructed by QI hardware has layout:
- *
- * HEADER (1 word)
- * Shdesc ptr (1 or 2 words)
- * SEQ_OUT_PTR (1 word)
- * Out ptr (1 or 2 words)
- * Out length (1 word)
- * SEQ_IN_PTR (1 word)
- * In ptr (1 or 2 words)
- * In length (1 word)
- *
- * The shdesc ptr is used to fetch shared descriptor contents
- * into deco buffer.
- *
- * Apart from shdesc contents, the total number of words that
- * get loaded in deco buffer are '8' or '11'. The remaining words
- * in deco buffer can be used for storing shared descriptor.
- */
-#define MAX_SDLEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / CAAM_CMD_SZ)
-
/* Length of a single buffer in the QI driver memory cache */
#define CAAM_QI_MEMCACHE_SIZE 512
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index 7205d9f4029e..65399cb2a770 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -82,14 +82,6 @@
#define HASH_MSG_LEN 8
#define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
-#ifdef DEBUG
-/* for print_hex_dumps with line references */
-#define debug(format, arg...) printk(format, arg)
-#else
-#define debug(format, arg...)
-#endif
-
-
static struct list_head hash_list;
/* ahash per-session context */
@@ -103,8 +95,8 @@ struct caam_hash_ctx {
dma_addr_t sh_desc_update_first_dma;
dma_addr_t sh_desc_fin_dma;
dma_addr_t sh_desc_digest_dma;
- dma_addr_t key_dma;
enum dma_data_direction dir;
+ enum dma_data_direction key_dir;
struct device *jrdev;
int ctx_len;
struct alginfo adata;
@@ -243,11 +235,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
ctx->ctx_len, true, ctrlpriv->era);
dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
desc_bytes(desc), ctx->dir);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR,
- "ahash update shdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+
+ print_hex_dump_debug("ahash update shdesc@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
/* ahash_update_first shared descriptor */
desc = ctx->sh_desc_update_first;
@@ -255,11 +246,9 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
ctx->ctx_len, false, ctrlpriv->era);
dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
desc_bytes(desc), ctx->dir);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR,
- "ahash update first shdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("ahash update first shdesc@"__stringify(__LINE__)
+ ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
/* ahash_final shared descriptor */
desc = ctx->sh_desc_fin;
@@ -267,11 +256,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
ctx->ctx_len, true, ctrlpriv->era);
dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
desc_bytes(desc), ctx->dir);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc,
- desc_bytes(desc), 1);
-#endif
+
+ print_hex_dump_debug("ahash final shdesc@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
/* ahash_digest shared descriptor */
desc = ctx->sh_desc_digest;
@@ -279,12 +267,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
ctx->ctx_len, false, ctrlpriv->era);
dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
desc_bytes(desc), ctx->dir);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR,
- "ahash digest shdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc,
- desc_bytes(desc), 1);
-#endif
+
+ print_hex_dump_debug("ahash digest shdesc@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
return 0;
}
@@ -296,13 +282,10 @@ static int axcbc_set_sh_desc(struct crypto_ahash *ahash)
struct device *jrdev = ctx->jrdev;
u32 *desc;
- /* key is loaded from memory for UPDATE and FINALIZE states */
- ctx->adata.key_dma = ctx->key_dma;
-
/* shared descriptor for ahash_update */
desc = ctx->sh_desc_update;
cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_UPDATE,
- ctx->ctx_len, ctx->ctx_len, 0);
+ ctx->ctx_len, ctx->ctx_len);
dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
desc_bytes(desc), ctx->dir);
print_hex_dump_debug("axcbc update shdesc@" __stringify(__LINE__)" : ",
@@ -312,7 +295,7 @@ static int axcbc_set_sh_desc(struct crypto_ahash *ahash)
/* shared descriptor for ahash_{final,finup} */
desc = ctx->sh_desc_fin;
cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_FINALIZE,
- digestsize, ctx->ctx_len, 0);
+ digestsize, ctx->ctx_len);
dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
desc_bytes(desc), ctx->dir);
print_hex_dump_debug("axcbc finup shdesc@" __stringify(__LINE__)" : ",
@@ -325,17 +308,17 @@ static int axcbc_set_sh_desc(struct crypto_ahash *ahash)
/* shared descriptor for first invocation of ahash_update */
desc = ctx->sh_desc_update_first;
cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
- ctx->ctx_len, ctx->key_dma);
+ ctx->ctx_len);
dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
desc_bytes(desc), ctx->dir);
- print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__)" : ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
- 1);
+ print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__)
+ " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
/* shared descriptor for ahash_digest */
desc = ctx->sh_desc_digest;
cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INITFINAL,
- digestsize, ctx->ctx_len, 0);
+ digestsize, ctx->ctx_len);
dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
desc_bytes(desc), ctx->dir);
print_hex_dump_debug("axcbc digest shdesc@" __stringify(__LINE__)" : ",
@@ -354,7 +337,7 @@ static int acmac_set_sh_desc(struct crypto_ahash *ahash)
/* shared descriptor for ahash_update */
desc = ctx->sh_desc_update;
cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_UPDATE,
- ctx->ctx_len, ctx->ctx_len, 0);
+ ctx->ctx_len, ctx->ctx_len);
dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
desc_bytes(desc), ctx->dir);
print_hex_dump_debug("acmac update shdesc@" __stringify(__LINE__)" : ",
@@ -364,7 +347,7 @@ static int acmac_set_sh_desc(struct crypto_ahash *ahash)
/* shared descriptor for ahash_{final,finup} */
desc = ctx->sh_desc_fin;
cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_FINALIZE,
- digestsize, ctx->ctx_len, 0);
+ digestsize, ctx->ctx_len);
dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
desc_bytes(desc), ctx->dir);
print_hex_dump_debug("acmac finup shdesc@" __stringify(__LINE__)" : ",
@@ -374,17 +357,17 @@ static int acmac_set_sh_desc(struct crypto_ahash *ahash)
/* shared descriptor for first invocation of ahash_update */
desc = ctx->sh_desc_update_first;
cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
- ctx->ctx_len, 0);
+ ctx->ctx_len);
dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
desc_bytes(desc), ctx->dir);
- print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__)" : ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__)
+ " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
/* shared descriptor for ahash_digest */
desc = ctx->sh_desc_digest;
cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INITFINAL,
- digestsize, ctx->ctx_len, 0);
+ digestsize, ctx->ctx_len);
dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
desc_bytes(desc), ctx->dir);
print_hex_dump_debug("acmac digest shdesc@" __stringify(__LINE__)" : ",
@@ -429,12 +412,11 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
LDST_SRCDST_BYTE_CONTEXT);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("key_in@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
+ print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
result.err = 0;
init_completion(&result.completion);
@@ -444,11 +426,10 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
/* in progress */
wait_for_completion(&result.completion);
ret = result.err;
-#ifdef DEBUG
- print_hex_dump(KERN_ERR,
- "digested key@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key, digestsize, 1);
-#endif
+
+ print_hex_dump_debug("digested key@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key,
+ digestsize, 1);
}
dma_unmap_single(jrdev, key_dma, *keylen, DMA_BIDIRECTIONAL);
@@ -463,15 +444,14 @@ static int ahash_setkey(struct crypto_ahash *ahash,
const u8 *key, unsigned int keylen)
{
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+ struct device *jrdev = ctx->jrdev;
int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
int digestsize = crypto_ahash_digestsize(ahash);
struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
int ret;
u8 *hashed_key = NULL;
-#ifdef DEBUG
- printk(KERN_ERR "keylen %d\n", keylen);
-#endif
+ dev_dbg(jrdev, "keylen %d\n", keylen);
if (keylen > blocksize) {
hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
@@ -497,6 +477,18 @@ static int ahash_setkey(struct crypto_ahash *ahash,
goto bad_free_key;
memcpy(ctx->key, key, keylen);
+
+ /*
+ * In case |user key| > |derived key|, using DKP<imm,imm>
+ * would result in invalid opcodes (last bytes of user key) in
+ * the resulting descriptor. Use DKP<ptr,imm> instead => both
+ * virtual and dma key addresses are needed.
+ */
+ if (keylen > ctx->adata.keylen_pad)
+ dma_sync_single_for_device(ctx->jrdev,
+ ctx->adata.key_dma,
+ ctx->adata.keylen_pad,
+ DMA_TO_DEVICE);
} else {
ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key,
keylen, CAAM_MAX_HASH_KEY_SIZE);
@@ -518,8 +510,14 @@ static int axcbc_setkey(struct crypto_ahash *ahash, const u8 *key,
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
struct device *jrdev = ctx->jrdev;
+ if (keylen != AES_KEYSIZE_128) {
+ crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+ }
+
memcpy(ctx->key, key, keylen);
- dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE);
+ dma_sync_single_for_device(jrdev, ctx->adata.key_dma, keylen,
+ DMA_TO_DEVICE);
ctx->adata.keylen = keylen;
print_hex_dump_debug("axcbc ctx.key@" __stringify(__LINE__)" : ",
@@ -532,6 +530,13 @@ static int acmac_setkey(struct crypto_ahash *ahash, const u8 *key,
unsigned int keylen)
{
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+ int err;
+
+ err = aes_check_keylen(keylen);
+ if (err) {
+ crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return err;
+ }
/* key is immediate data for all cmac shared descriptors */
ctx->adata.key_virt = key;
@@ -555,7 +560,7 @@ struct ahash_edesc {
dma_addr_t sec4_sg_dma;
int src_nents;
int sec4_sg_bytes;
- u32 hw_desc[DESC_JOB_IO_LEN / sizeof(u32)] ____cacheline_aligned;
+ u32 hw_desc[DESC_JOB_IO_LEN_MAX / sizeof(u32)] ____cacheline_aligned;
struct sec4_sg_entry sec4_sg[0];
};
@@ -600,27 +605,24 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
int digestsize = crypto_ahash_digestsize(ahash);
struct caam_hash_state *state = ahash_request_ctx(req);
-#ifdef DEBUG
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+ int ecode = 0;
- dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+ dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
if (err)
- caam_jr_strstatus(jrdev, err);
+ ecode = caam_jr_strstatus(jrdev, err);
ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
memcpy(req->result, state->caam_ctx, digestsize);
kfree(edesc);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
- ctx->ctx_len, 1);
-#endif
+ print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
+ ctx->ctx_len, 1);
- req->base.complete(&req->base, err);
+ req->base.complete(&req->base, ecode);
}
static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
@@ -631,31 +633,28 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
struct caam_hash_state *state = ahash_request_ctx(req);
-#ifdef DEBUG
int digestsize = crypto_ahash_digestsize(ahash);
+ int ecode = 0;
- dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+ dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
if (err)
- caam_jr_strstatus(jrdev, err);
+ ecode = caam_jr_strstatus(jrdev, err);
ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
switch_buf(state);
kfree(edesc);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
- ctx->ctx_len, 1);
+ print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
+ ctx->ctx_len, 1);
if (req->result)
- print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, req->result,
- digestsize, 1);
-#endif
+ print_hex_dump_debug("result@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, req->result,
+ digestsize, 1);
- req->base.complete(&req->base, err);
+ req->base.complete(&req->base, ecode);
}
static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
@@ -666,27 +665,24 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
int digestsize = crypto_ahash_digestsize(ahash);
struct caam_hash_state *state = ahash_request_ctx(req);
-#ifdef DEBUG
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+ int ecode = 0;
- dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+ dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
if (err)
- caam_jr_strstatus(jrdev, err);
+ ecode = caam_jr_strstatus(jrdev, err);
ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL);
memcpy(req->result, state->caam_ctx, digestsize);
kfree(edesc);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
- ctx->ctx_len, 1);
-#endif
+ print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
+ ctx->ctx_len, 1);
- req->base.complete(&req->base, err);
+ req->base.complete(&req->base, ecode);
}
static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
@@ -697,31 +693,28 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
struct caam_hash_state *state = ahash_request_ctx(req);
-#ifdef DEBUG
int digestsize = crypto_ahash_digestsize(ahash);
+ int ecode = 0;
- dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+ dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
if (err)
- caam_jr_strstatus(jrdev, err);
+ ecode = caam_jr_strstatus(jrdev, err);
ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE);
switch_buf(state);
kfree(edesc);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
- ctx->ctx_len, 1);
+ print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
+ ctx->ctx_len, 1);
if (req->result)
- print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, req->result,
- digestsize, 1);
-#endif
+ print_hex_dump_debug("result@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, req->result,
+ digestsize, 1);
- req->base.complete(&req->base, err);
+ req->base.complete(&req->base, ecode);
}
/*
@@ -759,9 +752,10 @@ static int ahash_edesc_add_src(struct caam_hash_ctx *ctx,
if (nents > 1 || first_sg) {
struct sec4_sg_entry *sg = edesc->sec4_sg;
- unsigned int sgsize = sizeof(*sg) * (first_sg + nents);
+ unsigned int sgsize = sizeof(*sg) *
+ pad_sg_nents(first_sg + nents);
- sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0);
+ sg_to_sec4_sg_last(req->src, to_hash, sg + first_sg, 0);
src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE);
if (dma_mapping_error(ctx->jrdev, src_dma)) {
@@ -819,8 +813,10 @@ static int ahash_update_ctx(struct ahash_request *req)
}
if (to_hash) {
- src_nents = sg_nents_for_len(req->src,
- req->nbytes - (*next_buflen));
+ int pad_nents;
+ int src_len = req->nbytes - *next_buflen;
+
+ src_nents = sg_nents_for_len(req->src, src_len);
if (src_nents < 0) {
dev_err(jrdev, "Invalid number of src SG.\n");
return src_nents;
@@ -838,15 +834,14 @@ static int ahash_update_ctx(struct ahash_request *req)
}
sec4_sg_src_index = 1 + (*buflen ? 1 : 0);
- sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
- sizeof(struct sec4_sg_entry);
+ pad_nents = pad_sg_nents(sec4_sg_src_index + mapped_nents);
+ sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry);
/*
* allocate space for base edesc and hw desc commands,
* link tables
*/
- edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
- ctx->sh_desc_update,
+ edesc = ahash_edesc_alloc(ctx, pad_nents, ctx->sh_desc_update,
ctx->sh_desc_update_dma, flags);
if (!edesc) {
dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
@@ -866,7 +861,7 @@ static int ahash_update_ctx(struct ahash_request *req)
goto unmap_ctx;
if (mapped_nents)
- sg_to_sec4_sg_last(req->src, mapped_nents,
+ sg_to_sec4_sg_last(req->src, src_len,
edesc->sec4_sg + sec4_sg_src_index,
0);
else
@@ -893,11 +888,9 @@ static int ahash_update_ctx(struct ahash_request *req)
append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc,
- desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req);
if (ret)
@@ -910,13 +903,12 @@ static int ahash_update_ctx(struct ahash_request *req)
*buflen = *next_buflen;
*next_buflen = last_buflen;
}
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
- print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
- *next_buflen, 1);
-#endif
+
+ print_hex_dump_debug("buf@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
+ print_hex_dump_debug("next buf@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
+ *next_buflen, 1);
return ret;
unmap_ctx:
@@ -935,18 +927,17 @@ static int ahash_final_ctx(struct ahash_request *req)
GFP_KERNEL : GFP_ATOMIC;
int buflen = *current_buflen(state);
u32 *desc;
- int sec4_sg_bytes, sec4_sg_src_index;
+ int sec4_sg_bytes;
int digestsize = crypto_ahash_digestsize(ahash);
struct ahash_edesc *edesc;
int ret;
- sec4_sg_src_index = 1 + (buflen ? 1 : 0);
- sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry);
+ sec4_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) *
+ sizeof(struct sec4_sg_entry);
/* allocate space for base edesc and hw desc commands, link tables */
- edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index,
- ctx->sh_desc_fin, ctx->sh_desc_fin_dma,
- flags);
+ edesc = ahash_edesc_alloc(ctx, 4, ctx->sh_desc_fin,
+ ctx->sh_desc_fin_dma, flags);
if (!edesc)
return -ENOMEM;
@@ -963,7 +954,7 @@ static int ahash_final_ctx(struct ahash_request *req)
if (ret)
goto unmap_ctx;
- sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 1);
+ sg_to_sec4_set_last(edesc->sec4_sg + (buflen ? 1 : 0));
edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
sec4_sg_bytes, DMA_TO_DEVICE);
@@ -977,10 +968,9 @@ static int ahash_final_ctx(struct ahash_request *req)
LDST_SGF);
append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
if (ret)
@@ -1058,10 +1048,9 @@ static int ahash_finup_ctx(struct ahash_request *req)
append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
if (ret)
@@ -1135,10 +1124,9 @@ static int ahash_digest(struct ahash_request *req)
return -ENOMEM;
}
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
if (!ret) {
@@ -1190,10 +1178,9 @@ static int ahash_final_no_ctx(struct ahash_request *req)
if (ret)
goto unmap;
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
if (!ret) {
@@ -1246,8 +1233,10 @@ static int ahash_update_no_ctx(struct ahash_request *req)
}
if (to_hash) {
- src_nents = sg_nents_for_len(req->src,
- req->nbytes - *next_buflen);
+ int pad_nents;
+ int src_len = req->nbytes - *next_buflen;
+
+ src_nents = sg_nents_for_len(req->src, src_len);
if (src_nents < 0) {
dev_err(jrdev, "Invalid number of src SG.\n");
return src_nents;
@@ -1264,14 +1253,14 @@ static int ahash_update_no_ctx(struct ahash_request *req)
mapped_nents = 0;
}
- sec4_sg_bytes = (1 + mapped_nents) *
- sizeof(struct sec4_sg_entry);
+ pad_nents = pad_sg_nents(1 + mapped_nents);
+ sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry);
/*
* allocate space for base edesc and hw desc commands,
* link tables
*/
- edesc = ahash_edesc_alloc(ctx, 1 + mapped_nents,
+ edesc = ahash_edesc_alloc(ctx, pad_nents,
ctx->sh_desc_update_first,
ctx->sh_desc_update_first_dma,
flags);
@@ -1287,8 +1276,7 @@ static int ahash_update_no_ctx(struct ahash_request *req)
if (ret)
goto unmap_ctx;
- sg_to_sec4_sg_last(req->src, mapped_nents,
- edesc->sec4_sg + 1, 0);
+ sg_to_sec4_sg_last(req->src, src_len, edesc->sec4_sg + 1, 0);
if (*next_buflen) {
scatterwalk_map_and_copy(next_buf, req->src,
@@ -1313,11 +1301,9 @@ static int ahash_update_no_ctx(struct ahash_request *req)
if (ret)
goto unmap_ctx;
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc,
- desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
if (ret)
@@ -1333,13 +1319,12 @@ static int ahash_update_no_ctx(struct ahash_request *req)
*buflen = *next_buflen;
*next_buflen = 0;
}
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
- print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
- *next_buflen, 1);
-#endif
+
+ print_hex_dump_debug("buf@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
+ print_hex_dump_debug("next buf@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen,
+ 1);
return ret;
unmap_ctx:
@@ -1414,10 +1399,9 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
if (ret)
goto unmap;
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
if (!ret) {
@@ -1517,11 +1501,9 @@ static int ahash_update_first(struct ahash_request *req)
if (ret)
goto unmap_ctx;
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc,
- desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
if (ret)
@@ -1539,11 +1521,10 @@ static int ahash_update_first(struct ahash_request *req)
req->nbytes, 0);
switch_buf(state);
}
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
- *next_buflen, 1);
-#endif
+
+ print_hex_dump_debug("next buf@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen,
+ 1);
return ret;
unmap_ctx:
@@ -1857,40 +1838,50 @@ static int caam_hash_cra_init(struct crypto_tfm *tfm)
if (is_xcbc_aes(caam_hash->alg_type)) {
ctx->dir = DMA_TO_DEVICE;
+ ctx->key_dir = DMA_BIDIRECTIONAL;
ctx->adata.algtype = OP_TYPE_CLASS1_ALG | caam_hash->alg_type;
ctx->ctx_len = 48;
-
- ctx->key_dma = dma_map_single_attrs(ctx->jrdev, ctx->key,
- ARRAY_SIZE(ctx->key),
- DMA_BIDIRECTIONAL,
- DMA_ATTR_SKIP_CPU_SYNC);
- if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) {
- dev_err(ctx->jrdev, "unable to map key\n");
- caam_jr_free(ctx->jrdev);
- return -ENOMEM;
- }
} else if (is_cmac_aes(caam_hash->alg_type)) {
ctx->dir = DMA_TO_DEVICE;
+ ctx->key_dir = DMA_NONE;
ctx->adata.algtype = OP_TYPE_CLASS1_ALG | caam_hash->alg_type;
ctx->ctx_len = 32;
} else {
- ctx->dir = priv->era >= 6 ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+ if (priv->era >= 6) {
+ ctx->dir = DMA_BIDIRECTIONAL;
+ ctx->key_dir = alg->setkey ? DMA_TO_DEVICE : DMA_NONE;
+ } else {
+ ctx->dir = DMA_TO_DEVICE;
+ ctx->key_dir = DMA_NONE;
+ }
ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
ctx->ctx_len = runninglen[(ctx->adata.algtype &
OP_ALG_ALGSEL_SUBMASK) >>
OP_ALG_ALGSEL_SHIFT];
}
+ if (ctx->key_dir != DMA_NONE) {
+ ctx->adata.key_dma = dma_map_single_attrs(ctx->jrdev, ctx->key,
+ ARRAY_SIZE(ctx->key),
+ ctx->key_dir,
+ DMA_ATTR_SKIP_CPU_SYNC);
+ if (dma_mapping_error(ctx->jrdev, ctx->adata.key_dma)) {
+ dev_err(ctx->jrdev, "unable to map key\n");
+ caam_jr_free(ctx->jrdev);
+ return -ENOMEM;
+ }
+ }
+
dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update,
offsetof(struct caam_hash_ctx, key),
ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
if (dma_mapping_error(ctx->jrdev, dma_addr)) {
dev_err(ctx->jrdev, "unable to map shared descriptors\n");
- if (is_xcbc_aes(caam_hash->alg_type))
- dma_unmap_single_attrs(ctx->jrdev, ctx->key_dma,
+ if (ctx->key_dir != DMA_NONE)
+ dma_unmap_single_attrs(ctx->jrdev, ctx->adata.key_dma,
ARRAY_SIZE(ctx->key),
- DMA_BIDIRECTIONAL,
+ ctx->key_dir,
DMA_ATTR_SKIP_CPU_SYNC);
caam_jr_free(ctx->jrdev);
@@ -1923,14 +1914,14 @@ static void caam_hash_cra_exit(struct crypto_tfm *tfm)
dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma,
offsetof(struct caam_hash_ctx, key),
ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
- if (is_xcbc_aes(ctx->adata.algtype))
- dma_unmap_single_attrs(ctx->jrdev, ctx->key_dma,
- ARRAY_SIZE(ctx->key), DMA_BIDIRECTIONAL,
+ if (ctx->key_dir != DMA_NONE)
+ dma_unmap_single_attrs(ctx->jrdev, ctx->adata.key_dma,
+ ARRAY_SIZE(ctx->key), ctx->key_dir,
DMA_ATTR_SKIP_CPU_SYNC);
caam_jr_free(ctx->jrdev);
}
-static void __exit caam_algapi_hash_exit(void)
+void caam_algapi_hash_exit(void)
{
struct caam_hash_alg *t_alg, *n;
@@ -1988,40 +1979,13 @@ caam_hash_alloc(struct caam_hash_template *template,
return t_alg;
}
-static int __init caam_algapi_hash_init(void)
+int caam_algapi_hash_init(struct device *ctrldev)
{
- struct device_node *dev_node;
- struct platform_device *pdev;
int i = 0, err = 0;
- struct caam_drv_private *priv;
+ struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
unsigned int md_limit = SHA512_DIGEST_SIZE;
u32 md_inst, md_vid;
- dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
- if (!dev_node) {
- dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
- if (!dev_node)
- return -ENODEV;
- }
-
- pdev = of_find_device_by_node(dev_node);
- if (!pdev) {
- of_node_put(dev_node);
- return -ENODEV;
- }
-
- priv = dev_get_drvdata(&pdev->dev);
- of_node_put(dev_node);
-
- /*
- * If priv is NULL, it's probably because the caam driver wasn't
- * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
- */
- if (!priv) {
- err = -ENODEV;
- goto out_put_dev;
- }
-
/*
* Register crypto algorithms the device supports. First, identify
* presence and attributes of MD block.
@@ -2042,10 +2006,8 @@ static int __init caam_algapi_hash_init(void)
* Skip registration of any hashing algorithms if MD block
* is not present.
*/
- if (!md_inst) {
- err = -ENODEV;
- goto out_put_dev;
- }
+ if (!md_inst)
+ return 0;
/* Limit digest size based on LP256 */
if (md_vid == CHA_VER_VID_MD_LP256)
@@ -2102,14 +2064,5 @@ static int __init caam_algapi_hash_init(void)
list_add_tail(&t_alg->entry, &hash_list);
}
-out_put_dev:
- put_device(&pdev->dev);
return err;
}
-
-module_init(caam_algapi_hash_init);
-module_exit(caam_algapi_hash_exit);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API");
-MODULE_AUTHOR("Freescale Semiconductor - NMG");
diff --git a/drivers/crypto/caam/caamhash_desc.c b/drivers/crypto/caam/caamhash_desc.c
index 71d018343ee4..78383d77da99 100644
--- a/drivers/crypto/caam/caamhash_desc.c
+++ b/drivers/crypto/caam/caamhash_desc.c
@@ -83,10 +83,9 @@ EXPORT_SYMBOL(cnstr_shdsc_ahash);
* @state: algorithm state OP_ALG_AS_{INIT, FINALIZE, INITFINALIZE, UPDATE}
* @digestsize: algorithm's digest size
* @ctx_len: size of Context Register
- * @key_dma: I/O Virtual Address of the key
*/
void cnstr_shdsc_sk_hash(u32 * const desc, struct alginfo *adata, u32 state,
- int digestsize, int ctx_len, dma_addr_t key_dma)
+ int digestsize, int ctx_len)
{
u32 *skip_key_load;
@@ -136,7 +135,7 @@ void cnstr_shdsc_sk_hash(u32 * const desc, struct alginfo *adata, u32 state,
LDST_SRCDST_BYTE_CONTEXT);
if (is_xcbc_aes(adata->algtype) && state == OP_ALG_AS_INIT)
/* Save K1 */
- append_fifo_store(desc, key_dma, adata->keylen,
+ append_fifo_store(desc, adata->key_dma, adata->keylen,
LDST_CLASS_1_CCB | FIFOST_TYPE_KEY_KEK);
}
EXPORT_SYMBOL(cnstr_shdsc_sk_hash);
diff --git a/drivers/crypto/caam/caamhash_desc.h b/drivers/crypto/caam/caamhash_desc.h
index 6947ee1f200c..4f369b8cb6ae 100644
--- a/drivers/crypto/caam/caamhash_desc.h
+++ b/drivers/crypto/caam/caamhash_desc.h
@@ -25,5 +25,5 @@ void cnstr_shdsc_ahash(u32 * const desc, struct alginfo *adata, u32 state,
int digestsize, int ctx_len, bool import_ctx, int era);
void cnstr_shdsc_sk_hash(u32 * const desc, struct alginfo *adata, u32 state,
- int digestsize, int ctx_len, dma_addr_t key_dma);
+ int digestsize, int ctx_len);
#endif /* _CAAMHASH_DESC_H_ */
diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c
index fe24485274e1..83f96d4f86e0 100644
--- a/drivers/crypto/caam/caampkc.c
+++ b/drivers/crypto/caam/caampkc.c
@@ -3,7 +3,7 @@
* caam - Freescale FSL CAAM support for Public Key Cryptography
*
* Copyright 2016 Freescale Semiconductor, Inc.
- * Copyright 2018 NXP
+ * Copyright 2018-2019 NXP
*
* There is no Shared Descriptor for PKC so that the Job Descriptor must carry
* all the desired key parameters, input and output pointers.
@@ -17,19 +17,36 @@
#include "sg_sw_sec4.h"
#include "caampkc.h"
-#define DESC_RSA_PUB_LEN (2 * CAAM_CMD_SZ + sizeof(struct rsa_pub_pdb))
+#define DESC_RSA_PUB_LEN (2 * CAAM_CMD_SZ + SIZEOF_RSA_PUB_PDB)
#define DESC_RSA_PRIV_F1_LEN (2 * CAAM_CMD_SZ + \
- sizeof(struct rsa_priv_f1_pdb))
+ SIZEOF_RSA_PRIV_F1_PDB)
#define DESC_RSA_PRIV_F2_LEN (2 * CAAM_CMD_SZ + \
- sizeof(struct rsa_priv_f2_pdb))
+ SIZEOF_RSA_PRIV_F2_PDB)
#define DESC_RSA_PRIV_F3_LEN (2 * CAAM_CMD_SZ + \
- sizeof(struct rsa_priv_f3_pdb))
+ SIZEOF_RSA_PRIV_F3_PDB)
+#define CAAM_RSA_MAX_INPUT_SIZE 512 /* for a 4096-bit modulus */
+
+/* buffer filled with zeros, used for padding */
+static u8 *zero_buffer;
+
+/*
+ * variable used to avoid double free of resources in case
+ * algorithm registration was unsuccessful
+ */
+static bool init_done;
+
+struct caam_akcipher_alg {
+ struct akcipher_alg akcipher;
+ bool registered;
+};
static void rsa_io_unmap(struct device *dev, struct rsa_edesc *edesc,
struct akcipher_request *req)
{
+ struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
+
dma_unmap_sg(dev, req->dst, edesc->dst_nents, DMA_FROM_DEVICE);
- dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
+ dma_unmap_sg(dev, req_ctx->fixup_src, edesc->src_nents, DMA_TO_DEVICE);
if (edesc->sec4_sg_bytes)
dma_unmap_single(dev, edesc->sec4_sg_dma, edesc->sec4_sg_bytes,
@@ -101,9 +118,10 @@ static void rsa_pub_done(struct device *dev, u32 *desc, u32 err, void *context)
{
struct akcipher_request *req = context;
struct rsa_edesc *edesc;
+ int ecode = 0;
if (err)
- caam_jr_strstatus(dev, err);
+ ecode = caam_jr_strstatus(dev, err);
edesc = container_of(desc, struct rsa_edesc, hw_desc[0]);
@@ -111,7 +129,7 @@ static void rsa_pub_done(struct device *dev, u32 *desc, u32 err, void *context)
rsa_io_unmap(dev, edesc, req);
kfree(edesc);
- akcipher_request_complete(req, err);
+ akcipher_request_complete(req, ecode);
}
static void rsa_priv_f1_done(struct device *dev, u32 *desc, u32 err,
@@ -119,9 +137,10 @@ static void rsa_priv_f1_done(struct device *dev, u32 *desc, u32 err,
{
struct akcipher_request *req = context;
struct rsa_edesc *edesc;
+ int ecode = 0;
if (err)
- caam_jr_strstatus(dev, err);
+ ecode = caam_jr_strstatus(dev, err);
edesc = container_of(desc, struct rsa_edesc, hw_desc[0]);
@@ -129,7 +148,7 @@ static void rsa_priv_f1_done(struct device *dev, u32 *desc, u32 err,
rsa_io_unmap(dev, edesc, req);
kfree(edesc);
- akcipher_request_complete(req, err);
+ akcipher_request_complete(req, ecode);
}
static void rsa_priv_f2_done(struct device *dev, u32 *desc, u32 err,
@@ -137,9 +156,10 @@ static void rsa_priv_f2_done(struct device *dev, u32 *desc, u32 err,
{
struct akcipher_request *req = context;
struct rsa_edesc *edesc;
+ int ecode = 0;
if (err)
- caam_jr_strstatus(dev, err);
+ ecode = caam_jr_strstatus(dev, err);
edesc = container_of(desc, struct rsa_edesc, hw_desc[0]);
@@ -147,7 +167,7 @@ static void rsa_priv_f2_done(struct device *dev, u32 *desc, u32 err,
rsa_io_unmap(dev, edesc, req);
kfree(edesc);
- akcipher_request_complete(req, err);
+ akcipher_request_complete(req, ecode);
}
static void rsa_priv_f3_done(struct device *dev, u32 *desc, u32 err,
@@ -155,9 +175,10 @@ static void rsa_priv_f3_done(struct device *dev, u32 *desc, u32 err,
{
struct akcipher_request *req = context;
struct rsa_edesc *edesc;
+ int ecode = 0;
if (err)
- caam_jr_strstatus(dev, err);
+ ecode = caam_jr_strstatus(dev, err);
edesc = container_of(desc, struct rsa_edesc, hw_desc[0]);
@@ -165,9 +186,16 @@ static void rsa_priv_f3_done(struct device *dev, u32 *desc, u32 err,
rsa_io_unmap(dev, edesc, req);
kfree(edesc);
- akcipher_request_complete(req, err);
+ akcipher_request_complete(req, ecode);
}
+/**
+ * Count leading zeros, need it to strip, from a given scatterlist
+ *
+ * @sgl : scatterlist to count zeros from
+ * @nbytes: number of zeros, in bytes, to strip
+ * @flags : operation flags
+ */
static int caam_rsa_count_leading_zeros(struct scatterlist *sgl,
unsigned int nbytes,
unsigned int flags)
@@ -187,7 +215,8 @@ static int caam_rsa_count_leading_zeros(struct scatterlist *sgl,
lzeros = 0;
len = 0;
while (nbytes > 0) {
- while (len && !*buff) {
+ /* do not strip more than given bytes */
+ while (len && !*buff && lzeros < nbytes) {
lzeros++;
len--;
buff++;
@@ -218,6 +247,7 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
struct device *dev = ctx->dev;
struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
+ struct caam_rsa_key *key = &ctx->key;
struct rsa_edesc *edesc;
gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
@@ -225,22 +255,45 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
int sgc;
int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
int src_nents, dst_nents;
+ unsigned int diff_size = 0;
int lzeros;
- lzeros = caam_rsa_count_leading_zeros(req->src, req->src_len, sg_flags);
- if (lzeros < 0)
- return ERR_PTR(lzeros);
-
- req->src_len -= lzeros;
- req->src = scatterwalk_ffwd(req_ctx->src, req->src, lzeros);
+ if (req->src_len > key->n_sz) {
+ /*
+ * strip leading zeros and
+ * return the number of zeros to skip
+ */
+ lzeros = caam_rsa_count_leading_zeros(req->src, req->src_len -
+ key->n_sz, sg_flags);
+ if (lzeros < 0)
+ return ERR_PTR(lzeros);
+
+ req_ctx->fixup_src = scatterwalk_ffwd(req_ctx->src, req->src,
+ lzeros);
+ req_ctx->fixup_src_len = req->src_len - lzeros;
+ } else {
+ /*
+ * input src is less then n key modulus,
+ * so there will be zero padding
+ */
+ diff_size = key->n_sz - req->src_len;
+ req_ctx->fixup_src = req->src;
+ req_ctx->fixup_src_len = req->src_len;
+ }
- src_nents = sg_nents_for_len(req->src, req->src_len);
+ src_nents = sg_nents_for_len(req_ctx->fixup_src,
+ req_ctx->fixup_src_len);
dst_nents = sg_nents_for_len(req->dst, req->dst_len);
- if (src_nents > 1)
- sec4_sg_len = src_nents;
+ if (!diff_size && src_nents == 1)
+ sec4_sg_len = 0; /* no need for an input hw s/g table */
+ else
+ sec4_sg_len = src_nents + !!diff_size;
+ sec4_sg_index = sec4_sg_len;
if (dst_nents > 1)
- sec4_sg_len += dst_nents;
+ sec4_sg_len += pad_sg_nents(dst_nents);
+ else
+ sec4_sg_len = pad_sg_nents(sec4_sg_len);
sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
@@ -250,7 +303,7 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
if (!edesc)
return ERR_PTR(-ENOMEM);
- sgc = dma_map_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
+ sgc = dma_map_sg(dev, req_ctx->fixup_src, src_nents, DMA_TO_DEVICE);
if (unlikely(!sgc)) {
dev_err(dev, "unable to map source\n");
goto src_fail;
@@ -263,14 +316,16 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
}
edesc->sec4_sg = (void *)edesc + sizeof(*edesc) + desclen;
+ if (diff_size)
+ dma_to_sec4_sg_one(edesc->sec4_sg, ctx->padding_dma, diff_size,
+ 0);
+
+ if (sec4_sg_index)
+ sg_to_sec4_sg_last(req_ctx->fixup_src, req_ctx->fixup_src_len,
+ edesc->sec4_sg + !!diff_size, 0);
- sec4_sg_index = 0;
- if (src_nents > 1) {
- sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);
- sec4_sg_index += src_nents;
- }
if (dst_nents > 1)
- sg_to_sec4_sg_last(req->dst, dst_nents,
+ sg_to_sec4_sg_last(req->dst, req->dst_len,
edesc->sec4_sg + sec4_sg_index, 0);
/* Save nents for later use in Job Descriptor */
@@ -289,12 +344,16 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
edesc->sec4_sg_bytes = sec4_sg_bytes;
+ print_hex_dump_debug("caampkc sec4_sg@" __stringify(__LINE__) ": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
+ edesc->sec4_sg_bytes, 1);
+
return edesc;
sec4_sg_fail:
dma_unmap_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE);
dst_fail:
- dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
+ dma_unmap_sg(dev, req_ctx->fixup_src, src_nents, DMA_TO_DEVICE);
src_fail:
kfree(edesc);
return ERR_PTR(-ENOMEM);
@@ -304,6 +363,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req,
struct rsa_edesc *edesc)
{
struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+ struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
struct caam_rsa_key *key = &ctx->key;
struct device *dev = ctx->dev;
@@ -328,7 +388,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req,
pdb->f_dma = edesc->sec4_sg_dma;
sec4_sg_index += edesc->src_nents;
} else {
- pdb->f_dma = sg_dma_address(req->src);
+ pdb->f_dma = sg_dma_address(req_ctx->fixup_src);
}
if (edesc->dst_nents > 1) {
@@ -340,7 +400,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req,
}
pdb->sgf |= (key->e_sz << RSA_PDB_E_SHIFT) | key->n_sz;
- pdb->f_len = req->src_len;
+ pdb->f_len = req_ctx->fixup_src_len;
return 0;
}
@@ -373,7 +433,9 @@ static int set_rsa_priv_f1_pdb(struct akcipher_request *req,
pdb->g_dma = edesc->sec4_sg_dma;
sec4_sg_index += edesc->src_nents;
} else {
- pdb->g_dma = sg_dma_address(req->src);
+ struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
+
+ pdb->g_dma = sg_dma_address(req_ctx->fixup_src);
}
if (edesc->dst_nents > 1) {
@@ -436,7 +498,9 @@ static int set_rsa_priv_f2_pdb(struct akcipher_request *req,
pdb->g_dma = edesc->sec4_sg_dma;
sec4_sg_index += edesc->src_nents;
} else {
- pdb->g_dma = sg_dma_address(req->src);
+ struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
+
+ pdb->g_dma = sg_dma_address(req_ctx->fixup_src);
}
if (edesc->dst_nents > 1) {
@@ -523,7 +587,9 @@ static int set_rsa_priv_f3_pdb(struct akcipher_request *req,
pdb->g_dma = edesc->sec4_sg_dma;
sec4_sg_index += edesc->src_nents;
} else {
- pdb->g_dma = sg_dma_address(req->src);
+ struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
+
+ pdb->g_dma = sg_dma_address(req_ctx->fixup_src);
}
if (edesc->dst_nents > 1) {
@@ -816,7 +882,7 @@ static int caam_rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key,
return ret;
/* Copy key in DMA zone */
- rsa_key->e = kzalloc(raw_key.e_sz, GFP_DMA | GFP_KERNEL);
+ rsa_key->e = kmemdup(raw_key.e, raw_key.e_sz, GFP_DMA | GFP_KERNEL);
if (!rsa_key->e)
goto err;
@@ -838,8 +904,6 @@ static int caam_rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key,
rsa_key->e_sz = raw_key.e_sz;
rsa_key->n_sz = raw_key.n_sz;
- memcpy(rsa_key->e, raw_key.e, raw_key.e_sz);
-
return 0;
err:
caam_rsa_free_key(rsa_key);
@@ -920,11 +984,11 @@ static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key,
return ret;
/* Copy key in DMA zone */
- rsa_key->d = kzalloc(raw_key.d_sz, GFP_DMA | GFP_KERNEL);
+ rsa_key->d = kmemdup(raw_key.d, raw_key.d_sz, GFP_DMA | GFP_KERNEL);
if (!rsa_key->d)
goto err;
- rsa_key->e = kzalloc(raw_key.e_sz, GFP_DMA | GFP_KERNEL);
+ rsa_key->e = kmemdup(raw_key.e, raw_key.e_sz, GFP_DMA | GFP_KERNEL);
if (!rsa_key->e)
goto err;
@@ -947,9 +1011,6 @@ static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key,
rsa_key->e_sz = raw_key.e_sz;
rsa_key->n_sz = raw_key.n_sz;
- memcpy(rsa_key->d, raw_key.d, raw_key.d_sz);
- memcpy(rsa_key->e, raw_key.e, raw_key.e_sz);
-
caam_rsa_set_priv_key_form(ctx, &raw_key);
return 0;
@@ -978,6 +1039,15 @@ static int caam_rsa_init_tfm(struct crypto_akcipher *tfm)
return PTR_ERR(ctx->dev);
}
+ ctx->padding_dma = dma_map_single(ctx->dev, zero_buffer,
+ CAAM_RSA_MAX_INPUT_SIZE - 1,
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(ctx->dev, ctx->padding_dma)) {
+ dev_err(ctx->dev, "unable to map padding\n");
+ caam_jr_free(ctx->dev);
+ return -ENOMEM;
+ }
+
return 0;
}
@@ -987,63 +1057,39 @@ static void caam_rsa_exit_tfm(struct crypto_akcipher *tfm)
struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
struct caam_rsa_key *key = &ctx->key;
+ dma_unmap_single(ctx->dev, ctx->padding_dma, CAAM_RSA_MAX_INPUT_SIZE -
+ 1, DMA_TO_DEVICE);
caam_rsa_free_key(key);
caam_jr_free(ctx->dev);
}
-static struct akcipher_alg caam_rsa = {
- .encrypt = caam_rsa_enc,
- .decrypt = caam_rsa_dec,
- .set_pub_key = caam_rsa_set_pub_key,
- .set_priv_key = caam_rsa_set_priv_key,
- .max_size = caam_rsa_max_size,
- .init = caam_rsa_init_tfm,
- .exit = caam_rsa_exit_tfm,
- .reqsize = sizeof(struct caam_rsa_req_ctx),
- .base = {
- .cra_name = "rsa",
- .cra_driver_name = "rsa-caam",
- .cra_priority = 3000,
- .cra_module = THIS_MODULE,
- .cra_ctxsize = sizeof(struct caam_rsa_ctx),
- },
+static struct caam_akcipher_alg caam_rsa = {
+ .akcipher = {
+ .encrypt = caam_rsa_enc,
+ .decrypt = caam_rsa_dec,
+ .set_pub_key = caam_rsa_set_pub_key,
+ .set_priv_key = caam_rsa_set_priv_key,
+ .max_size = caam_rsa_max_size,
+ .init = caam_rsa_init_tfm,
+ .exit = caam_rsa_exit_tfm,
+ .reqsize = sizeof(struct caam_rsa_req_ctx),
+ .base = {
+ .cra_name = "rsa",
+ .cra_driver_name = "rsa-caam",
+ .cra_priority = 3000,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct caam_rsa_ctx),
+ },
+ }
};
/* Public Key Cryptography module initialization handler */
-static int __init caam_pkc_init(void)
+int caam_pkc_init(struct device *ctrldev)
{
- struct device_node *dev_node;
- struct platform_device *pdev;
- struct device *ctrldev;
- struct caam_drv_private *priv;
+ struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
u32 pk_inst;
int err;
-
- dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
- if (!dev_node) {
- dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
- if (!dev_node)
- return -ENODEV;
- }
-
- pdev = of_find_device_by_node(dev_node);
- if (!pdev) {
- of_node_put(dev_node);
- return -ENODEV;
- }
-
- ctrldev = &pdev->dev;
- priv = dev_get_drvdata(ctrldev);
- of_node_put(dev_node);
-
- /*
- * If priv is NULL, it's probably because the caam driver wasn't
- * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
- */
- if (!priv) {
- err = -ENODEV;
- goto out_put_dev;
- }
+ init_done = false;
/* Determine public key hardware accelerator presence. */
if (priv->era < 10)
@@ -1053,31 +1099,37 @@ static int __init caam_pkc_init(void)
pk_inst = rd_reg32(&priv->ctrl->vreg.pkha) & CHA_VER_NUM_MASK;
/* Do not register algorithms if PKHA is not present. */
- if (!pk_inst) {
- err = -ENODEV;
- goto out_put_dev;
- }
+ if (!pk_inst)
+ return 0;
- err = crypto_register_akcipher(&caam_rsa);
- if (err)
+ /* allocate zero buffer, used for padding input */
+ zero_buffer = kzalloc(CAAM_RSA_MAX_INPUT_SIZE - 1, GFP_DMA |
+ GFP_KERNEL);
+ if (!zero_buffer)
+ return -ENOMEM;
+
+ err = crypto_register_akcipher(&caam_rsa.akcipher);
+
+ if (err) {
+ kfree(zero_buffer);
dev_warn(ctrldev, "%s alg registration failed\n",
- caam_rsa.base.cra_driver_name);
- else
+ caam_rsa.akcipher.base.cra_driver_name);
+ } else {
+ init_done = true;
+ caam_rsa.registered = true;
dev_info(ctrldev, "caam pkc algorithms registered in /proc/crypto\n");
+ }
-out_put_dev:
- put_device(ctrldev);
return err;
}
-static void __exit caam_pkc_exit(void)
+void caam_pkc_exit(void)
{
- crypto_unregister_akcipher(&caam_rsa);
-}
+ if (!init_done)
+ return;
-module_init(caam_pkc_init);
-module_exit(caam_pkc_exit);
+ if (caam_rsa.registered)
+ crypto_unregister_akcipher(&caam_rsa.akcipher);
-MODULE_LICENSE("Dual BSD/GPL");
-MODULE_DESCRIPTION("FSL CAAM support for PKC functions of crypto API");
-MODULE_AUTHOR("Freescale Semiconductor");
+ kfree(zero_buffer);
+}
diff --git a/drivers/crypto/caam/caampkc.h b/drivers/crypto/caam/caampkc.h
index 82645bcf8b27..2c488c9a3812 100644
--- a/drivers/crypto/caam/caampkc.h
+++ b/drivers/crypto/caam/caampkc.h
@@ -89,18 +89,25 @@ struct caam_rsa_key {
* caam_rsa_ctx - per session context.
* @key : RSA key in DMA zone
* @dev : device structure
+ * @padding_dma : dma address of padding, for adding it to the input
*/
struct caam_rsa_ctx {
struct caam_rsa_key key;
struct device *dev;
+ dma_addr_t padding_dma;
+
};
/**
* caam_rsa_req_ctx - per request context.
- * @src: input scatterlist (stripped of leading zeros)
+ * @src : input scatterlist (stripped of leading zeros)
+ * @fixup_src : input scatterlist (that might be stripped of leading zeros)
+ * @fixup_src_len : length of the fixup_src input scatterlist
*/
struct caam_rsa_req_ctx {
struct scatterlist src[2];
+ struct scatterlist *fixup_src;
+ unsigned int fixup_src_len;
};
/**
diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c
index 95eb5402c59f..e8baacaabe07 100644
--- a/drivers/crypto/caam/caamrng.c
+++ b/drivers/crypto/caam/caamrng.c
@@ -3,7 +3,7 @@
* caam - Freescale FSL CAAM support for hw_random
*
* Copyright 2011 Freescale Semiconductor, Inc.
- * Copyright 2018 NXP
+ * Copyright 2018-2019 NXP
*
* Based on caamalg.c crypto API driver.
*
@@ -53,7 +53,7 @@
L1_CACHE_BYTES)
/* length of descriptors */
-#define DESC_JOB_O_LEN (CAAM_CMD_SZ * 2 + CAAM_PTR_SZ * 2)
+#define DESC_JOB_O_LEN (CAAM_CMD_SZ * 2 + CAAM_PTR_SZ_MAX * 2)
#define DESC_RNG_LEN (3 * CAAM_CMD_SZ)
/* Buffer, its dma address and lock */
@@ -80,6 +80,12 @@ struct caam_rng_ctx {
static struct caam_rng_ctx *rng_ctx;
+/*
+ * Variable used to avoid double free of resources in case
+ * algorithm registration was unsuccessful
+ */
+static bool init_done;
+
static inline void rng_unmap_buf(struct device *jrdev, struct buf_data *bd)
{
if (bd->addr)
@@ -113,10 +119,8 @@ static void rng_done(struct device *jrdev, u32 *desc, u32 err, void *context)
/* Buffer refilled, invalidate cache */
dma_sync_single_for_cpu(jrdev, bd->addr, RN_BUF_SIZE, DMA_FROM_DEVICE);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "rng refreshed buf@: ",
- DUMP_PREFIX_ADDRESS, 16, 4, bd->buf, RN_BUF_SIZE, 1);
-#endif
+ print_hex_dump_debug("rng refreshed buf@: ", DUMP_PREFIX_ADDRESS, 16, 4,
+ bd->buf, RN_BUF_SIZE, 1);
}
static inline int submit_job(struct caam_rng_ctx *ctx, int to_current)
@@ -209,10 +213,10 @@ static inline int rng_create_sh_desc(struct caam_rng_ctx *ctx)
dev_err(jrdev, "unable to map shared descriptor\n");
return -ENOMEM;
}
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "rng shdesc@: ", DUMP_PREFIX_ADDRESS, 16, 4,
- desc, desc_bytes(desc), 1);
-#endif
+
+ print_hex_dump_debug("rng shdesc@: ", DUMP_PREFIX_ADDRESS, 16, 4,
+ desc, desc_bytes(desc), 1);
+
return 0;
}
@@ -233,10 +237,10 @@ static inline int rng_create_job_desc(struct caam_rng_ctx *ctx, int buf_id)
}
append_seq_out_ptr_intlen(desc, bd->addr, RN_BUF_SIZE, 0);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "rng job desc@: ", DUMP_PREFIX_ADDRESS, 16, 4,
- desc, desc_bytes(desc), 1);
-#endif
+
+ print_hex_dump_debug("rng job desc@: ", DUMP_PREFIX_ADDRESS, 16, 4,
+ desc, desc_bytes(desc), 1);
+
return 0;
}
@@ -296,46 +300,23 @@ static struct hwrng caam_rng = {
.read = caam_read,
};
-static void __exit caam_rng_exit(void)
+void caam_rng_exit(void)
{
+ if (!init_done)
+ return;
+
caam_jr_free(rng_ctx->jrdev);
hwrng_unregister(&caam_rng);
kfree(rng_ctx);
}
-static int __init caam_rng_init(void)
+int caam_rng_init(struct device *ctrldev)
{
struct device *dev;
- struct device_node *dev_node;
- struct platform_device *pdev;
- struct caam_drv_private *priv;
u32 rng_inst;
+ struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
int err;
-
- dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
- if (!dev_node) {
- dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
- if (!dev_node)
- return -ENODEV;
- }
-
- pdev = of_find_device_by_node(dev_node);
- if (!pdev) {
- of_node_put(dev_node);
- return -ENODEV;
- }
-
- priv = dev_get_drvdata(&pdev->dev);
- of_node_put(dev_node);
-
- /*
- * If priv is NULL, it's probably because the caam driver wasn't
- * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
- */
- if (!priv) {
- err = -ENODEV;
- goto out_put_dev;
- }
+ init_done = false;
/* Check for an instantiated RNG before registration */
if (priv->era < 10)
@@ -344,16 +325,13 @@ static int __init caam_rng_init(void)
else
rng_inst = rd_reg32(&priv->ctrl->vreg.rng) & CHA_VER_NUM_MASK;
- if (!rng_inst) {
- err = -ENODEV;
- goto out_put_dev;
- }
+ if (!rng_inst)
+ return 0;
dev = caam_jr_alloc();
if (IS_ERR(dev)) {
pr_err("Job Ring Device allocation for transform failed\n");
- err = PTR_ERR(dev);
- goto out_put_dev;
+ return PTR_ERR(dev);
}
rng_ctx = kmalloc(sizeof(*rng_ctx), GFP_DMA | GFP_KERNEL);
if (!rng_ctx) {
@@ -364,22 +342,17 @@ static int __init caam_rng_init(void)
if (err)
goto free_rng_ctx;
- put_device(&pdev->dev);
dev_info(dev, "registering rng-caam\n");
- return hwrng_register(&caam_rng);
+
+ err = hwrng_register(&caam_rng);
+ if (!err) {
+ init_done = true;
+ return err;
+ }
free_rng_ctx:
kfree(rng_ctx);
free_caam_alloc:
caam_jr_free(dev);
-out_put_dev:
- put_device(&pdev->dev);
return err;
}
-
-module_init(caam_rng_init);
-module_exit(caam_rng_exit);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("FSL CAAM support for hw_random API");
-MODULE_AUTHOR("Freescale Semiconductor - NMG");
diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h
index 8639b2df0371..60e2a54c19f1 100644
--- a/drivers/crypto/caam/compat.h
+++ b/drivers/crypto/caam/compat.h
@@ -32,7 +32,7 @@
#include <crypto/null.h>
#include <crypto/aes.h>
#include <crypto/ctr.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <crypto/gcm.h>
#include <crypto/sha.h>
#include <crypto/md5.h>
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index fec39c35c877..db22777d59b4 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -3,7 +3,7 @@
* Controller-level driver, kernel property detection, initialization
*
* Copyright 2008-2012 Freescale Semiconductor, Inc.
- * Copyright 2018 NXP
+ * Copyright 2018-2019 NXP
*/
#include <linux/device.h>
@@ -26,16 +26,6 @@ EXPORT_SYMBOL(caam_dpaa2);
#endif
/*
- * i.MX targets tend to have clock control subsystems that can
- * enable/disable clocking to our device.
- */
-static inline struct clk *caam_drv_identify_clk(struct device *dev,
- char *clk_name)
-{
- return caam_imx ? devm_clk_get(dev, clk_name) : NULL;
-}
-
-/*
* Descriptor to instantiate RNG State Handle 0 in normal mode and
* load the JDKEK, TDKEK and TDSK registers
*/
@@ -107,7 +97,12 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc,
int i;
- if (ctrlpriv->virt_en == 1) {
+ if (ctrlpriv->virt_en == 1 ||
+ /*
+ * Apparently on i.MX8MQ it doesn't matter if virt_en == 1
+ * and the following steps should be performed regardless
+ */
+ of_machine_is_compatible("fsl,imx8mq")) {
clrsetbits_32(&ctrl->deco_rsr, 0, DECORSR_JR0);
while (!(rd_reg32(&ctrl->deco_rsr) & DECORSR_VALID) &&
@@ -323,8 +318,8 @@ static int caam_remove(struct platform_device *pdev)
of_platform_depopulate(ctrldev);
#ifdef CONFIG_CAAM_QI
- if (ctrlpriv->qidev)
- caam_qi_shutdown(ctrlpriv->qidev);
+ if (ctrlpriv->qi_init)
+ caam_qi_shutdown(ctrldev);
#endif
/*
@@ -342,13 +337,6 @@ static int caam_remove(struct platform_device *pdev)
/* Unmap controller region */
iounmap(ctrl);
- /* shut clocks off before finalizing shutdown */
- clk_disable_unprepare(ctrlpriv->caam_ipg);
- if (ctrlpriv->caam_mem)
- clk_disable_unprepare(ctrlpriv->caam_mem);
- clk_disable_unprepare(ctrlpriv->caam_aclk);
- if (ctrlpriv->caam_emi_slow)
- clk_disable_unprepare(ctrlpriv->caam_emi_slow);
return 0;
}
@@ -497,20 +485,99 @@ static const struct of_device_id caam_match[] = {
};
MODULE_DEVICE_TABLE(of, caam_match);
+struct caam_imx_data {
+ const struct clk_bulk_data *clks;
+ int num_clks;
+};
+
+static const struct clk_bulk_data caam_imx6_clks[] = {
+ { .id = "ipg" },
+ { .id = "mem" },
+ { .id = "aclk" },
+ { .id = "emi_slow" },
+};
+
+static const struct caam_imx_data caam_imx6_data = {
+ .clks = caam_imx6_clks,
+ .num_clks = ARRAY_SIZE(caam_imx6_clks),
+};
+
+static const struct clk_bulk_data caam_imx7_clks[] = {
+ { .id = "ipg" },
+ { .id = "aclk" },
+};
+
+static const struct caam_imx_data caam_imx7_data = {
+ .clks = caam_imx7_clks,
+ .num_clks = ARRAY_SIZE(caam_imx7_clks),
+};
+
+static const struct clk_bulk_data caam_imx6ul_clks[] = {
+ { .id = "ipg" },
+ { .id = "mem" },
+ { .id = "aclk" },
+};
+
+static const struct caam_imx_data caam_imx6ul_data = {
+ .clks = caam_imx6ul_clks,
+ .num_clks = ARRAY_SIZE(caam_imx6ul_clks),
+};
+
+static const struct soc_device_attribute caam_imx_soc_table[] = {
+ { .soc_id = "i.MX6UL", .data = &caam_imx6ul_data },
+ { .soc_id = "i.MX6*", .data = &caam_imx6_data },
+ { .soc_id = "i.MX7*", .data = &caam_imx7_data },
+ { .soc_id = "i.MX8MQ", .data = &caam_imx7_data },
+ { .family = "Freescale i.MX" },
+ { /* sentinel */ }
+};
+
+static void disable_clocks(void *data)
+{
+ struct caam_drv_private *ctrlpriv = data;
+
+ clk_bulk_disable_unprepare(ctrlpriv->num_clks, ctrlpriv->clks);
+}
+
+static int init_clocks(struct device *dev, const struct caam_imx_data *data)
+{
+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev);
+ int ret;
+
+ ctrlpriv->num_clks = data->num_clks;
+ ctrlpriv->clks = devm_kmemdup(dev, data->clks,
+ data->num_clks * sizeof(data->clks[0]),
+ GFP_KERNEL);
+ if (!ctrlpriv->clks)
+ return -ENOMEM;
+
+ ret = devm_clk_bulk_get(dev, ctrlpriv->num_clks, ctrlpriv->clks);
+ if (ret) {
+ dev_err(dev,
+ "Failed to request all necessary clocks\n");
+ return ret;
+ }
+
+ ret = clk_bulk_prepare_enable(ctrlpriv->num_clks, ctrlpriv->clks);
+ if (ret) {
+ dev_err(dev,
+ "Failed to prepare/enable all necessary clocks\n");
+ return ret;
+ }
+
+ return devm_add_action_or_reset(dev, disable_clocks, ctrlpriv);
+}
+
/* Probe routine for CAAM top (controller) level */
static int caam_probe(struct platform_device *pdev)
{
int ret, ring, gen_sk, ent_delay = RTSDCTL_ENT_DLY_MIN;
u64 caam_id;
- static const struct soc_device_attribute imx_soc[] = {
- {.family = "Freescale i.MX"},
- {},
- };
+ const struct soc_device_attribute *imx_soc_match;
struct device *dev;
struct device_node *nprop, *np;
struct caam_ctrl __iomem *ctrl;
struct caam_drv_private *ctrlpriv;
- struct clk *clk;
#ifdef CONFIG_DEBUG_FS
struct caam_perfmon *perfmon;
#endif
@@ -527,101 +594,68 @@ static int caam_probe(struct platform_device *pdev)
dev_set_drvdata(dev, ctrlpriv);
nprop = pdev->dev.of_node;
- caam_imx = (bool)soc_device_match(imx_soc);
+ imx_soc_match = soc_device_match(caam_imx_soc_table);
+ caam_imx = (bool)imx_soc_match;
- /* Enable clocking */
- clk = caam_drv_identify_clk(&pdev->dev, "ipg");
- if (IS_ERR(clk)) {
- ret = PTR_ERR(clk);
- dev_err(&pdev->dev,
- "can't identify CAAM ipg clk: %d\n", ret);
- return ret;
- }
- ctrlpriv->caam_ipg = clk;
-
- if (!of_machine_is_compatible("fsl,imx7d") &&
- !of_machine_is_compatible("fsl,imx7s")) {
- clk = caam_drv_identify_clk(&pdev->dev, "mem");
- if (IS_ERR(clk)) {
- ret = PTR_ERR(clk);
- dev_err(&pdev->dev,
- "can't identify CAAM mem clk: %d\n", ret);
- return ret;
+ if (imx_soc_match) {
+ if (!imx_soc_match->data) {
+ dev_err(dev, "No clock data provided for i.MX SoC");
+ return -EINVAL;
}
- ctrlpriv->caam_mem = clk;
- }
- clk = caam_drv_identify_clk(&pdev->dev, "aclk");
- if (IS_ERR(clk)) {
- ret = PTR_ERR(clk);
- dev_err(&pdev->dev,
- "can't identify CAAM aclk clk: %d\n", ret);
- return ret;
- }
- ctrlpriv->caam_aclk = clk;
-
- if (!of_machine_is_compatible("fsl,imx6ul") &&
- !of_machine_is_compatible("fsl,imx7d") &&
- !of_machine_is_compatible("fsl,imx7s")) {
- clk = caam_drv_identify_clk(&pdev->dev, "emi_slow");
- if (IS_ERR(clk)) {
- ret = PTR_ERR(clk);
- dev_err(&pdev->dev,
- "can't identify CAAM emi_slow clk: %d\n", ret);
+ ret = init_clocks(dev, imx_soc_match->data);
+ if (ret)
return ret;
- }
- ctrlpriv->caam_emi_slow = clk;
}
- ret = clk_prepare_enable(ctrlpriv->caam_ipg);
- if (ret < 0) {
- dev_err(&pdev->dev, "can't enable CAAM ipg clock: %d\n", ret);
- return ret;
- }
-
- if (ctrlpriv->caam_mem) {
- ret = clk_prepare_enable(ctrlpriv->caam_mem);
- if (ret < 0) {
- dev_err(&pdev->dev, "can't enable CAAM secure mem clock: %d\n",
- ret);
- goto disable_caam_ipg;
- }
- }
-
- ret = clk_prepare_enable(ctrlpriv->caam_aclk);
- if (ret < 0) {
- dev_err(&pdev->dev, "can't enable CAAM aclk clock: %d\n", ret);
- goto disable_caam_mem;
- }
-
- if (ctrlpriv->caam_emi_slow) {
- ret = clk_prepare_enable(ctrlpriv->caam_emi_slow);
- if (ret < 0) {
- dev_err(&pdev->dev, "can't enable CAAM emi slow clock: %d\n",
- ret);
- goto disable_caam_aclk;
- }
- }
/* Get configuration properties from device tree */
/* First, get register page */
ctrl = of_iomap(nprop, 0);
- if (ctrl == NULL) {
+ if (!ctrl) {
dev_err(dev, "caam: of_iomap() failed\n");
- ret = -ENOMEM;
- goto disable_caam_emi_slow;
+ return -ENOMEM;
}
caam_little_end = !(bool)(rd_reg32(&ctrl->perfmon.status) &
(CSTA_PLEND | CSTA_ALT_PLEND));
-
- /* Finding the page size for using the CTPR_MS register */
comp_params = rd_reg32(&ctrl->perfmon.comp_parms_ms);
- pg_size = (comp_params & CTPR_MS_PG_SZ_MASK) >> CTPR_MS_PG_SZ_SHIFT;
+ if (comp_params & CTPR_MS_PS && rd_reg32(&ctrl->mcr) & MCFGR_LONG_PTR)
+ caam_ptr_sz = sizeof(u64);
+ else
+ caam_ptr_sz = sizeof(u32);
+ caam_dpaa2 = !!(comp_params & CTPR_MS_DPAA2);
+ ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK);
+
+#ifdef CONFIG_CAAM_QI
+ /* If (DPAA 1.x) QI present, check whether dependencies are available */
+ if (ctrlpriv->qi_present && !caam_dpaa2) {
+ ret = qman_is_probed();
+ if (!ret) {
+ ret = -EPROBE_DEFER;
+ goto iounmap_ctrl;
+ } else if (ret < 0) {
+ dev_err(dev, "failing probe due to qman probe error\n");
+ ret = -ENODEV;
+ goto iounmap_ctrl;
+ }
+
+ ret = qman_portals_probed();
+ if (!ret) {
+ ret = -EPROBE_DEFER;
+ goto iounmap_ctrl;
+ } else if (ret < 0) {
+ dev_err(dev, "failing probe due to qman portals probe error\n");
+ ret = -ENODEV;
+ goto iounmap_ctrl;
+ }
+ }
+#endif
/* Allocating the BLOCK_OFFSET based on the supported page size on
* the platform
*/
+ pg_size = (comp_params & CTPR_MS_PG_SZ_MASK) >> CTPR_MS_PG_SZ_SHIFT;
if (pg_size == 0)
BLOCK_OFFSET = PG_SIZE_4K;
else
@@ -646,7 +680,6 @@ static int caam_probe(struct platform_device *pdev)
* In case of SoCs with Management Complex, MC f/w performs
* the configuration.
*/
- caam_dpaa2 = !!(comp_params & CTPR_MS_DPAA2);
np = of_find_compatible_node(NULL, NULL, "fsl,qoriq-mc");
ctrlpriv->mc_en = !!np;
of_node_put(np);
@@ -686,28 +719,14 @@ static int caam_probe(struct platform_device *pdev)
JRSTART_JR1_START | JRSTART_JR2_START |
JRSTART_JR3_START);
- if (sizeof(dma_addr_t) == sizeof(u64)) {
- if (caam_dpaa2)
- ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49));
- else if (of_device_is_compatible(nprop, "fsl,sec-v5.0"))
- ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));
- else
- ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36));
- } else {
- ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
- }
+ ret = dma_set_mask_and_coherent(dev, caam_get_dma_mask(dev));
if (ret) {
dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret);
goto iounmap_ctrl;
}
ctrlpriv->era = caam_get_era(ctrl);
-
- ret = of_platform_populate(nprop, caam_match, NULL, dev);
- if (ret) {
- dev_err(dev, "JR platform devices creation error\n");
- goto iounmap_ctrl;
- }
+ ctrlpriv->domain = iommu_get_domain_for_dev(dev);
#ifdef CONFIG_DEBUG_FS
/*
@@ -721,21 +740,7 @@ static int caam_probe(struct platform_device *pdev)
ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root);
#endif
- ring = 0;
- for_each_available_child_of_node(nprop, np)
- if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") ||
- of_device_is_compatible(np, "fsl,sec4.0-job-ring")) {
- ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *)
- ((__force uint8_t *)ctrl +
- (ring + JR_BLOCK_NUMBER) *
- BLOCK_OFFSET
- );
- ctrlpriv->total_jobrs++;
- ring++;
- }
-
/* Check to see if (DPAA 1.x) QI present. If so, enable */
- ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK);
if (ctrlpriv->qi_present && !caam_dpaa2) {
ctrlpriv->qi = (struct caam_queue_if __iomem __force *)
((__force uint8_t *)ctrl +
@@ -752,6 +757,25 @@ static int caam_probe(struct platform_device *pdev)
#endif
}
+ ret = of_platform_populate(nprop, caam_match, NULL, dev);
+ if (ret) {
+ dev_err(dev, "JR platform devices creation error\n");
+ goto shutdown_qi;
+ }
+
+ ring = 0;
+ for_each_available_child_of_node(nprop, np)
+ if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") ||
+ of_device_is_compatible(np, "fsl,sec4.0-job-ring")) {
+ ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *)
+ ((__force uint8_t *)ctrl +
+ (ring + JR_BLOCK_NUMBER) *
+ BLOCK_OFFSET
+ );
+ ctrlpriv->total_jobrs++;
+ ring++;
+ }
+
/* If no QI and no rings specified, quit and go home */
if ((!ctrlpriv->qi_present) && (!ctrlpriv->total_jobrs)) {
dev_err(dev, "no queues configured, terminating\n");
@@ -898,18 +922,13 @@ caam_remove:
caam_remove(pdev);
return ret;
+shutdown_qi:
+#ifdef CONFIG_CAAM_QI
+ if (ctrlpriv->qi_init)
+ caam_qi_shutdown(dev);
+#endif
iounmap_ctrl:
iounmap(ctrl);
-disable_caam_emi_slow:
- if (ctrlpriv->caam_emi_slow)
- clk_disable_unprepare(ctrlpriv->caam_emi_slow);
-disable_caam_aclk:
- clk_disable_unprepare(ctrlpriv->caam_aclk);
-disable_caam_mem:
- if (ctrlpriv->caam_mem)
- clk_disable_unprepare(ctrlpriv->caam_mem);
-disable_caam_ipg:
- clk_disable_unprepare(ctrlpriv->caam_ipg);
return ret;
}
diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h
index 2980b8ef1fb1..62ce6421bb3f 100644
--- a/drivers/crypto/caam/desc_constr.h
+++ b/drivers/crypto/caam/desc_constr.h
@@ -3,6 +3,7 @@
* caam descriptor construction helper functions
*
* Copyright 2008-2012 Freescale Semiconductor, Inc.
+ * Copyright 2019 NXP
*/
#ifndef DESC_CONSTR_H
@@ -13,9 +14,41 @@
#define IMMEDIATE (1 << 23)
#define CAAM_CMD_SZ sizeof(u32)
-#define CAAM_PTR_SZ sizeof(dma_addr_t)
+#define CAAM_PTR_SZ caam_ptr_sz
+#define CAAM_PTR_SZ_MAX sizeof(dma_addr_t)
+#define CAAM_PTR_SZ_MIN sizeof(u32)
#define CAAM_DESC_BYTES_MAX (CAAM_CMD_SZ * MAX_CAAM_DESCSIZE)
-#define DESC_JOB_IO_LEN (CAAM_CMD_SZ * 5 + CAAM_PTR_SZ * 3)
+#define __DESC_JOB_IO_LEN(n) (CAAM_CMD_SZ * 5 + (n) * 3)
+#define DESC_JOB_IO_LEN __DESC_JOB_IO_LEN(CAAM_PTR_SZ)
+#define DESC_JOB_IO_LEN_MAX __DESC_JOB_IO_LEN(CAAM_PTR_SZ_MAX)
+#define DESC_JOB_IO_LEN_MIN __DESC_JOB_IO_LEN(CAAM_PTR_SZ_MIN)
+
+/*
+ * The CAAM QI hardware constructs a job descriptor which points
+ * to shared descriptor (as pointed by context_a of FQ to CAAM).
+ * When the job descriptor is executed by deco, the whole job
+ * descriptor together with shared descriptor gets loaded in
+ * deco buffer which is 64 words long (each 32-bit).
+ *
+ * The job descriptor constructed by QI hardware has layout:
+ *
+ * HEADER (1 word)
+ * Shdesc ptr (1 or 2 words)
+ * SEQ_OUT_PTR (1 word)
+ * Out ptr (1 or 2 words)
+ * Out length (1 word)
+ * SEQ_IN_PTR (1 word)
+ * In ptr (1 or 2 words)
+ * In length (1 word)
+ *
+ * The shdesc ptr is used to fetch shared descriptor contents
+ * into deco buffer.
+ *
+ * Apart from shdesc contents, the total number of words that
+ * get loaded in deco buffer are '8' or '11'. The remaining words
+ * in deco buffer can be used for storing shared descriptor.
+ */
+#define MAX_SDLEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN) / CAAM_CMD_SZ)
#ifdef DEBUG
#define PRINT_POS do { printk(KERN_DEBUG "%02d: %s\n", desc_len(desc),\
@@ -36,6 +69,17 @@
(LDOFF_ENABLE_AUTO_NFIFO << LDST_OFFSET_SHIFT))
extern bool caam_little_end;
+extern size_t caam_ptr_sz;
+
+/*
+ * HW fetches 4 S/G table entries at a time, irrespective of how many entries
+ * are in the table. It's SW's responsibility to make sure these accesses
+ * do not have side effects.
+ */
+static inline int pad_sg_nents(int sg_nents)
+{
+ return ALIGN(sg_nents, 4);
+}
static inline int desc_len(u32 * const desc)
{
@@ -92,9 +136,15 @@ static inline void init_job_desc_pdb(u32 * const desc, u32 options,
static inline void append_ptr(u32 * const desc, dma_addr_t ptr)
{
- dma_addr_t *offset = (dma_addr_t *)desc_end(desc);
+ if (caam_ptr_sz == sizeof(dma_addr_t)) {
+ dma_addr_t *offset = (dma_addr_t *)desc_end(desc);
- *offset = cpu_to_caam_dma(ptr);
+ *offset = cpu_to_caam_dma(ptr);
+ } else {
+ u32 *offset = (u32 *)desc_end(desc);
+
+ *offset = cpu_to_caam_dma(ptr);
+ }
(*desc) = cpu_to_caam32(caam32_to_cpu(*desc) +
CAAM_PTR_SZ / CAAM_CMD_SZ);
@@ -446,8 +496,8 @@ do { \
* functions where it is used.
* @keylen: length of the provided algorithm key, in bytes
* @keylen_pad: padded length of the provided algorithm key, in bytes
- * @key: address where algorithm key resides; virtual address if key_inline
- * is true, dma (bus) address if key_inline is false.
+ * @key_dma: dma (bus) address where algorithm key resides
+ * @key_virt: virtual address where algorithm key resides
* @key_inline: true - key can be inlined in the descriptor; false - key is
* referenced by the descriptor
*/
@@ -455,10 +505,8 @@ struct alginfo {
u32 algtype;
unsigned int keylen;
unsigned int keylen_pad;
- union {
- dma_addr_t key_dma;
- const void *key_virt;
- };
+ dma_addr_t key_dma;
+ const void *key_virt;
bool key_inline;
};
@@ -524,14 +572,26 @@ static inline void append_proto_dkp(u32 * const desc, struct alginfo *adata)
if (adata->key_inline) {
int words;
- append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid |
- OP_PCL_DKP_SRC_IMM | OP_PCL_DKP_DST_IMM |
- adata->keylen);
- append_data(desc, adata->key_virt, adata->keylen);
+ if (adata->keylen > adata->keylen_pad) {
+ append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid |
+ OP_PCL_DKP_SRC_PTR |
+ OP_PCL_DKP_DST_IMM | adata->keylen);
+ append_ptr(desc, adata->key_dma);
+
+ words = (ALIGN(adata->keylen_pad, CAAM_CMD_SZ) -
+ CAAM_PTR_SZ) / CAAM_CMD_SZ;
+ } else {
+ append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid |
+ OP_PCL_DKP_SRC_IMM |
+ OP_PCL_DKP_DST_IMM | adata->keylen);
+ append_data(desc, adata->key_virt, adata->keylen);
+
+ words = (ALIGN(adata->keylen_pad, CAAM_CMD_SZ) -
+ ALIGN(adata->keylen, CAAM_CMD_SZ)) /
+ CAAM_CMD_SZ;
+ }
/* Reserve space in descriptor buffer for the derived key */
- words = (ALIGN(adata->keylen_pad, CAAM_CMD_SZ) -
- ALIGN(adata->keylen, CAAM_CMD_SZ)) / CAAM_CMD_SZ;
if (words)
(*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + words);
} else {
diff --git a/drivers/crypto/caam/dpseci-debugfs.c b/drivers/crypto/caam/dpseci-debugfs.c
new file mode 100644
index 000000000000..c5bfc923abd8
--- /dev/null
+++ b/drivers/crypto/caam/dpseci-debugfs.c
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/* Copyright 2019 NXP */
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/debugfs.h>
+#include "dpseci-debugfs.h"
+
+static int dpseci_dbg_fqs_show(struct seq_file *file, void *offset)
+{
+ struct dpaa2_caam_priv *priv = (struct dpaa2_caam_priv *)file->private;
+ u32 fqid, fcnt, bcnt;
+ int i, err;
+
+ seq_printf(file, "FQ stats for %s:\n", dev_name(priv->dev));
+ seq_printf(file, "%s%16s%16s\n",
+ "Rx-VFQID",
+ "Pending frames",
+ "Pending bytes");
+
+ for (i = 0; i < priv->num_pairs; i++) {
+ fqid = priv->rx_queue_attr[i].fqid;
+ err = dpaa2_io_query_fq_count(NULL, fqid, &fcnt, &bcnt);
+ if (err)
+ continue;
+
+ seq_printf(file, "%5d%16u%16u\n", fqid, fcnt, bcnt);
+ }
+
+ seq_printf(file, "%s%16s%16s\n",
+ "Tx-VFQID",
+ "Pending frames",
+ "Pending bytes");
+
+ for (i = 0; i < priv->num_pairs; i++) {
+ fqid = priv->tx_queue_attr[i].fqid;
+ err = dpaa2_io_query_fq_count(NULL, fqid, &fcnt, &bcnt);
+ if (err)
+ continue;
+
+ seq_printf(file, "%5d%16u%16u\n", fqid, fcnt, bcnt);
+ }
+
+ return 0;
+}
+
+static int dpseci_dbg_fqs_open(struct inode *inode, struct file *file)
+{
+ int err;
+ struct dpaa2_caam_priv *priv;
+
+ priv = (struct dpaa2_caam_priv *)inode->i_private;
+
+ err = single_open(file, dpseci_dbg_fqs_show, priv);
+ if (err < 0)
+ dev_err(priv->dev, "single_open() failed\n");
+
+ return err;
+}
+
+static const struct file_operations dpseci_dbg_fq_ops = {
+ .open = dpseci_dbg_fqs_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+
+void dpaa2_dpseci_debugfs_init(struct dpaa2_caam_priv *priv)
+{
+ priv->dfs_root = debugfs_create_dir(dev_name(priv->dev), NULL);
+
+ debugfs_create_file("fq_stats", 0444, priv->dfs_root, priv,
+ &dpseci_dbg_fq_ops);
+}
+
+void dpaa2_dpseci_debugfs_exit(struct dpaa2_caam_priv *priv)
+{
+ debugfs_remove_recursive(priv->dfs_root);
+}
diff --git a/drivers/crypto/caam/dpseci-debugfs.h b/drivers/crypto/caam/dpseci-debugfs.h
new file mode 100644
index 000000000000..bc22af7bec37
--- /dev/null
+++ b/drivers/crypto/caam/dpseci-debugfs.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
+/* Copyright 2019 NXP */
+
+#ifndef DPSECI_DEBUGFS_H
+#define DPSECI_DEBUGFS_H
+
+#include <linux/dcache.h>
+#include "caamalg_qi2.h"
+
+#ifdef CONFIG_DEBUG_FS
+void dpaa2_dpseci_debugfs_init(struct dpaa2_caam_priv *priv);
+void dpaa2_dpseci_debugfs_exit(struct dpaa2_caam_priv *priv);
+#else
+static inline void dpaa2_dpseci_debugfs_init(struct dpaa2_caam_priv *priv) {}
+static inline void dpaa2_dpseci_debugfs_exit(struct dpaa2_caam_priv *priv) {}
+#endif /* CONFIG_DEBUG_FS */
+
+#endif /* DPSECI_DEBUGFS_H */
diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c
index 4da844e4b61d..17c6108b6d41 100644
--- a/drivers/crypto/caam/error.c
+++ b/drivers/crypto/caam/error.c
@@ -13,7 +13,7 @@
#ifdef DEBUG
#include <linux/highmem.h>
-void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
+void caam_dump_sg(const char *prefix_str, int prefix_type,
int rowsize, int groupsize, struct scatterlist *sg,
size_t tlen, bool ascii)
{
@@ -35,15 +35,15 @@ void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
buf = it_page + it->offset;
len = min_t(size_t, tlen, it->length);
- print_hex_dump(level, prefix_str, prefix_type, rowsize,
- groupsize, buf, len, ascii);
+ print_hex_dump_debug(prefix_str, prefix_type, rowsize,
+ groupsize, buf, len, ascii);
tlen -= len;
kunmap_atomic(it_page);
}
}
#else
-void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
+void caam_dump_sg(const char *prefix_str, int prefix_type,
int rowsize, int groupsize, struct scatterlist *sg,
size_t tlen, bool ascii)
{}
@@ -56,6 +56,9 @@ EXPORT_SYMBOL(caam_little_end);
bool caam_imx;
EXPORT_SYMBOL(caam_imx);
+size_t caam_ptr_sz;
+EXPORT_SYMBOL(caam_ptr_sz);
+
static const struct {
u8 value;
const char *error_text;
@@ -118,6 +121,7 @@ static const struct {
u8 value;
const char *error_text;
} qi_error_list[] = {
+ { 0x00, "No error" },
{ 0x1F, "Job terminated by FQ or ICID flush" },
{ 0x20, "FD format error"},
{ 0x21, "FD command format error"},
@@ -210,8 +214,8 @@ static const char * const rng_err_id_list[] = {
"Secure key generation",
};
-static void report_ccb_status(struct device *jrdev, const u32 status,
- const char *error)
+static int report_ccb_status(struct device *jrdev, const u32 status,
+ const char *error)
{
u8 cha_id = (status & JRSTA_CCBERR_CHAID_MASK) >>
JRSTA_CCBERR_CHAID_SHIFT;
@@ -247,22 +251,27 @@ static void report_ccb_status(struct device *jrdev, const u32 status,
* CCB ICV check failures are part of normal operation life;
* we leave the upper layers to do what they want with them.
*/
- if (err_id != JRSTA_CCBERR_ERRID_ICVCHK)
- dev_err(jrdev, "%08x: %s: %s %d: %s%s: %s%s\n",
- status, error, idx_str, idx,
- cha_str, cha_err_code,
- err_str, err_err_code);
+ if (err_id == JRSTA_CCBERR_ERRID_ICVCHK)
+ return -EBADMSG;
+
+ dev_err_ratelimited(jrdev, "%08x: %s: %s %d: %s%s: %s%s\n", status,
+ error, idx_str, idx, cha_str, cha_err_code,
+ err_str, err_err_code);
+
+ return -EINVAL;
}
-static void report_jump_status(struct device *jrdev, const u32 status,
- const char *error)
+static int report_jump_status(struct device *jrdev, const u32 status,
+ const char *error)
{
dev_err(jrdev, "%08x: %s: %s() not implemented\n",
status, error, __func__);
+
+ return -EINVAL;
}
-static void report_deco_status(struct device *jrdev, const u32 status,
- const char *error)
+static int report_deco_status(struct device *jrdev, const u32 status,
+ const char *error)
{
u8 err_id = status & JRSTA_DECOERR_ERROR_MASK;
u8 idx = (status & JRSTA_DECOERR_INDEX_MASK) >>
@@ -288,10 +297,12 @@ static void report_deco_status(struct device *jrdev, const u32 status,
dev_err(jrdev, "%08x: %s: %s %d: %s%s\n",
status, error, idx_str, idx, err_str, err_err_code);
+
+ return -EINVAL;
}
-static void report_qi_status(struct device *qidev, const u32 status,
- const char *error)
+static int report_qi_status(struct device *qidev, const u32 status,
+ const char *error)
{
u8 err_id = status & JRSTA_QIERR_ERROR_MASK;
const char *err_str = "unidentified error value 0x";
@@ -309,27 +320,33 @@ static void report_qi_status(struct device *qidev, const u32 status,
dev_err(qidev, "%08x: %s: %s%s\n",
status, error, err_str, err_err_code);
+
+ return -EINVAL;
}
-static void report_jr_status(struct device *jrdev, const u32 status,
- const char *error)
+static int report_jr_status(struct device *jrdev, const u32 status,
+ const char *error)
{
dev_err(jrdev, "%08x: %s: %s() not implemented\n",
status, error, __func__);
+
+ return -EINVAL;
}
-static void report_cond_code_status(struct device *jrdev, const u32 status,
- const char *error)
+static int report_cond_code_status(struct device *jrdev, const u32 status,
+ const char *error)
{
dev_err(jrdev, "%08x: %s: %s() not implemented\n",
status, error, __func__);
+
+ return -EINVAL;
}
-void caam_strstatus(struct device *jrdev, u32 status, bool qi_v2)
+int caam_strstatus(struct device *jrdev, u32 status, bool qi_v2)
{
static const struct stat_src {
- void (*report_ssed)(struct device *jrdev, const u32 status,
- const char *error);
+ int (*report_ssed)(struct device *jrdev, const u32 status,
+ const char *error);
const char *error;
} status_src[16] = {
{ NULL, "No error" },
@@ -357,11 +374,14 @@ void caam_strstatus(struct device *jrdev, u32 status, bool qi_v2)
* Otherwise print the error source name.
*/
if (status_src[ssrc].report_ssed)
- status_src[ssrc].report_ssed(jrdev, status, error);
- else if (error)
+ return status_src[ssrc].report_ssed(jrdev, status, error);
+
+ if (error)
dev_err(jrdev, "%d: %s\n", ssrc, error);
else
dev_err(jrdev, "%d: unknown error source\n", ssrc);
+
+ return -EINVAL;
}
EXPORT_SYMBOL(caam_strstatus);
diff --git a/drivers/crypto/caam/error.h b/drivers/crypto/caam/error.h
index 8c6b83e02a70..16809fa8fec7 100644
--- a/drivers/crypto/caam/error.h
+++ b/drivers/crypto/caam/error.h
@@ -12,12 +12,12 @@
#define CAAM_ERROR_STR_MAX 302
-void caam_strstatus(struct device *dev, u32 status, bool qi_v2);
+int caam_strstatus(struct device *dev, u32 status, bool qi_v2);
#define caam_jr_strstatus(jrdev, status) caam_strstatus(jrdev, status, false)
#define caam_qi2_strstatus(qidev, status) caam_strstatus(qidev, status, true)
-void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
+void caam_dump_sg(const char *prefix_str, int prefix_type,
int rowsize, int groupsize, struct scatterlist *sg,
size_t tlen, bool ascii);
diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h
index 3392615dc91b..731b06becd9c 100644
--- a/drivers/crypto/caam/intern.h
+++ b/drivers/crypto/caam/intern.h
@@ -4,12 +4,14 @@
* Private/internal definitions between modules
*
* Copyright 2008-2011 Freescale Semiconductor, Inc.
- *
+ * Copyright 2019 NXP
*/
#ifndef INTERN_H
#define INTERN_H
+#include "ctrl.h"
+
/* Currently comes from Kconfig param as a ^2 (driver-required) */
#define JOBR_DEPTH (1 << CONFIG_CRYPTO_DEV_FSL_CAAM_RINGSIZE)
@@ -53,20 +55,17 @@ struct caam_drv_private_jr {
spinlock_t inplock ____cacheline_aligned; /* Input ring index lock */
u32 inpring_avail; /* Number of free entries in input ring */
int head; /* entinfo (s/w ring) head index */
- dma_addr_t *inpring; /* Base of input ring, alloc DMA-safe */
+ void *inpring; /* Base of input ring, alloc
+ * DMA-safe */
int out_ring_read_index; /* Output index "tail" */
int tail; /* entinfo (s/w ring) tail index */
- struct jr_outentry *outring; /* Base of output ring, DMA-safe */
+ void *outring; /* Base of output ring, DMA-safe */
};
/*
* Driver-private storage for a single CAAM block instance
*/
struct caam_drv_private {
-#ifdef CONFIG_CAAM_QI
- struct device *qidev;
-#endif
-
/* Physical-presence section */
struct caam_ctrl __iomem *ctrl; /* controller region */
struct caam_deco __iomem *deco; /* DECO/CCB views */
@@ -74,12 +73,17 @@ struct caam_drv_private {
struct caam_queue_if __iomem *qi; /* QI control region */
struct caam_job_ring __iomem *jr[4]; /* JobR's register space */
+ struct iommu_domain *domain;
+
/*
* Detected geometry block. Filled in from device tree if powerpc,
* or from register-based version detection code
*/
u8 total_jobrs; /* Total Job Rings in device */
u8 qi_present; /* Nonzero if QI present in device */
+#ifdef CONFIG_CAAM_QI
+ u8 qi_init; /* Nonzero if QI has been initialized */
+#endif
u8 mc_en; /* Nonzero if MC f/w is active */
int secvio_irq; /* Security violation interrupt number */
int virt_en; /* Virtualization enabled in CAAM */
@@ -91,11 +95,8 @@ struct caam_drv_private {
Handles of the RNG4 block are initialized
by this driver */
- struct clk *caam_ipg;
- struct clk *caam_mem;
- struct clk *caam_aclk;
- struct clk *caam_emi_slow;
-
+ struct clk_bulk_data *clks;
+ int num_clks;
/*
* debugfs entries for developer view into driver/device
* variables at runtime.
@@ -107,8 +108,95 @@ struct caam_drv_private {
#endif
};
-void caam_jr_algapi_init(struct device *dev);
-void caam_jr_algapi_remove(struct device *dev);
+#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API
+
+int caam_algapi_init(struct device *dev);
+void caam_algapi_exit(void);
+
+#else
+
+static inline int caam_algapi_init(struct device *dev)
+{
+ return 0;
+}
+
+static inline void caam_algapi_exit(void)
+{
+}
+
+#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API */
+
+#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API
+
+int caam_algapi_hash_init(struct device *dev);
+void caam_algapi_hash_exit(void);
+
+#else
+
+static inline int caam_algapi_hash_init(struct device *dev)
+{
+ return 0;
+}
+
+static inline void caam_algapi_hash_exit(void)
+{
+}
+
+#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API */
+
+#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API
+
+int caam_pkc_init(struct device *dev);
+void caam_pkc_exit(void);
+
+#else
+
+static inline int caam_pkc_init(struct device *dev)
+{
+ return 0;
+}
+
+static inline void caam_pkc_exit(void)
+{
+}
+
+#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API */
+
+#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API
+
+int caam_rng_init(struct device *dev);
+void caam_rng_exit(void);
+
+#else
+
+static inline int caam_rng_init(struct device *dev)
+{
+ return 0;
+}
+
+static inline void caam_rng_exit(void)
+{
+}
+
+#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API */
+
+#ifdef CONFIG_CAAM_QI
+
+int caam_qi_algapi_init(struct device *dev);
+void caam_qi_algapi_exit(void);
+
+#else
+
+static inline int caam_qi_algapi_init(struct device *dev)
+{
+ return 0;
+}
+
+static inline void caam_qi_algapi_exit(void)
+{
+}
+
+#endif /* CONFIG_CAAM_QI */
#ifdef CONFIG_DEBUG_FS
static int caam_debugfs_u64_get(void *data, u64 *val)
@@ -127,4 +215,22 @@ DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n");
DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n");
#endif
+static inline u64 caam_get_dma_mask(struct device *dev)
+{
+ struct device_node *nprop = dev->of_node;
+
+ if (caam_ptr_sz != sizeof(u64))
+ return DMA_BIT_MASK(32);
+
+ if (caam_dpaa2)
+ return DMA_BIT_MASK(49);
+
+ if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring") ||
+ of_device_is_compatible(nprop, "fsl,sec-v5.0"))
+ return DMA_BIT_MASK(40);
+
+ return DMA_BIT_MASK(36);
+}
+
+
#endif /* INTERN_H */
diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c
index 1de2562d0982..fc97cde27059 100644
--- a/drivers/crypto/caam/jr.c
+++ b/drivers/crypto/caam/jr.c
@@ -4,6 +4,7 @@
* JobR backend functionality
*
* Copyright 2008-2012 Freescale Semiconductor, Inc.
+ * Copyright 2019 NXP
*/
#include <linux/of_irq.h>
@@ -23,6 +24,43 @@ struct jr_driver_data {
} ____cacheline_aligned;
static struct jr_driver_data driver_data;
+static DEFINE_MUTEX(algs_lock);
+static unsigned int active_devs;
+
+static void register_algs(struct device *dev)
+{
+ mutex_lock(&algs_lock);
+
+ if (++active_devs != 1)
+ goto algs_unlock;
+
+ caam_algapi_init(dev);
+ caam_algapi_hash_init(dev);
+ caam_pkc_init(dev);
+ caam_rng_init(dev);
+ caam_qi_algapi_init(dev);
+
+algs_unlock:
+ mutex_unlock(&algs_lock);
+}
+
+static void unregister_algs(void)
+{
+ mutex_lock(&algs_lock);
+
+ if (--active_devs != 0)
+ goto algs_unlock;
+
+ caam_qi_algapi_exit();
+
+ caam_rng_exit();
+ caam_pkc_exit();
+ caam_algapi_hash_exit();
+ caam_algapi_exit();
+
+algs_unlock:
+ mutex_unlock(&algs_lock);
+}
static int caam_reset_hw_jr(struct device *dev)
{
@@ -70,25 +108,12 @@ static int caam_reset_hw_jr(struct device *dev)
static int caam_jr_shutdown(struct device *dev)
{
struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
- dma_addr_t inpbusaddr, outbusaddr;
int ret;
ret = caam_reset_hw_jr(dev);
tasklet_kill(&jrp->irqtask);
- /* Release interrupt */
- free_irq(jrp->irq, dev);
-
- /* Free rings */
- inpbusaddr = rd_reg64(&jrp->rregs->inpring_base);
- outbusaddr = rd_reg64(&jrp->rregs->outring_base);
- dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
- jrp->inpring, inpbusaddr);
- dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
- jrp->outring, outbusaddr);
- kfree(jrp->entinfo);
-
return ret;
}
@@ -109,6 +134,9 @@ static int caam_jr_remove(struct platform_device *pdev)
return -EBUSY;
}
+ /* Unregister JR-based RNG & crypto algorithms */
+ unregister_algs();
+
/* Remove the node from Physical JobR list maintained by driver */
spin_lock(&driver_data.jr_alloc_lock);
list_del(&jrpriv->list_node);
@@ -118,7 +146,6 @@ static int caam_jr_remove(struct platform_device *pdev)
ret = caam_jr_shutdown(jrdev);
if (ret)
dev_err(jrdev, "Failed to shut down job ring\n");
- irq_dispose_mapping(jrpriv->irq);
return ret;
}
@@ -183,7 +210,7 @@ static void caam_jr_dequeue(unsigned long devarg)
for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) {
sw_idx = (tail + i) & (JOBR_DEPTH - 1);
- if (jrp->outring[hw_idx].desc ==
+ if (jr_outentry_desc(jrp->outring, hw_idx) ==
caam_dma_to_cpu(jrp->entinfo[sw_idx].desc_addr_dma))
break; /* found */
}
@@ -192,7 +219,8 @@ static void caam_jr_dequeue(unsigned long devarg)
/* Unmap just-run descriptor so we can post-process */
dma_unmap_single(dev,
- caam_dma_to_cpu(jrp->outring[hw_idx].desc),
+ caam_dma_to_cpu(jr_outentry_desc(jrp->outring,
+ hw_idx)),
jrp->entinfo[sw_idx].desc_size,
DMA_TO_DEVICE);
@@ -203,7 +231,8 @@ static void caam_jr_dequeue(unsigned long devarg)
usercall = jrp->entinfo[sw_idx].callbk;
userarg = jrp->entinfo[sw_idx].cbkarg;
userdesc = jrp->entinfo[sw_idx].desc_addr_virt;
- userstatus = caam32_to_cpu(jrp->outring[hw_idx].jrstatus);
+ userstatus = caam32_to_cpu(jr_outentry_jrstatus(jrp->outring,
+ hw_idx));
/*
* Make sure all information from the job has been obtained
@@ -358,7 +387,7 @@ int caam_jr_enqueue(struct device *dev, u32 *desc,
head_entry->cbkarg = areq;
head_entry->desc_addr_dma = desc_dma;
- jrp->inpring[head] = cpu_to_caam_dma(desc_dma);
+ jr_inpentry_set(jrp->inpring, head, cpu_to_caam_dma(desc_dma));
/*
* Guarantee that the descriptor's DMA address has been written to
@@ -400,35 +429,26 @@ static int caam_jr_init(struct device *dev)
jrp = dev_get_drvdata(dev);
- tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
-
- /* Connect job ring interrupt handler. */
- error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED,
- dev_name(dev), dev);
- if (error) {
- dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
- jrp->ridx, jrp->irq);
- goto out_kill_deq;
- }
-
error = caam_reset_hw_jr(dev);
if (error)
- goto out_free_irq;
+ return error;
- error = -ENOMEM;
- jrp->inpring = dma_alloc_coherent(dev, sizeof(*jrp->inpring) *
- JOBR_DEPTH, &inpbusaddr, GFP_KERNEL);
+ jrp->inpring = dmam_alloc_coherent(dev, SIZEOF_JR_INPENTRY *
+ JOBR_DEPTH, &inpbusaddr,
+ GFP_KERNEL);
if (!jrp->inpring)
- goto out_free_irq;
+ return -ENOMEM;
- jrp->outring = dma_alloc_coherent(dev, sizeof(*jrp->outring) *
- JOBR_DEPTH, &outbusaddr, GFP_KERNEL);
+ jrp->outring = dmam_alloc_coherent(dev, SIZEOF_JR_OUTENTRY *
+ JOBR_DEPTH, &outbusaddr,
+ GFP_KERNEL);
if (!jrp->outring)
- goto out_free_inpring;
+ return -ENOMEM;
- jrp->entinfo = kcalloc(JOBR_DEPTH, sizeof(*jrp->entinfo), GFP_KERNEL);
+ jrp->entinfo = devm_kcalloc(dev, JOBR_DEPTH, sizeof(*jrp->entinfo),
+ GFP_KERNEL);
if (!jrp->entinfo)
- goto out_free_outring;
+ return -ENOMEM;
for (i = 0; i < JOBR_DEPTH; i++)
jrp->entinfo[i].desc_addr_dma = !0;
@@ -452,22 +472,24 @@ static int caam_jr_init(struct device *dev)
(JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
(JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
- return 0;
+ tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
+
+ /* Connect job ring interrupt handler. */
+ error = devm_request_irq(dev, jrp->irq, caam_jr_interrupt, IRQF_SHARED,
+ dev_name(dev), dev);
+ if (error) {
+ dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
+ jrp->ridx, jrp->irq);
+ tasklet_kill(&jrp->irqtask);
+ }
-out_free_outring:
- dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
- jrp->outring, outbusaddr);
-out_free_inpring:
- dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
- jrp->inpring, inpbusaddr);
- dev_err(dev, "can't allocate job rings for %d\n", jrp->ridx);
-out_free_irq:
- free_irq(jrp->irq, dev);
-out_kill_deq:
- tasklet_kill(&jrp->irqtask);
return error;
}
+static void caam_jr_irq_dispose_mapping(void *data)
+{
+ irq_dispose_mapping((unsigned long)data);
+}
/*
* Probe routine for each detected JobR subsystem.
@@ -479,6 +501,7 @@ static int caam_jr_probe(struct platform_device *pdev)
struct caam_job_ring __iomem *ctrl;
struct caam_drv_private_jr *jrpriv;
static int total_jobrs;
+ struct resource *r;
int error;
jrdev = &pdev->dev;
@@ -494,45 +517,43 @@ static int caam_jr_probe(struct platform_device *pdev)
nprop = pdev->dev.of_node;
/* Get configuration properties from device tree */
/* First, get register page */
- ctrl = of_iomap(nprop, 0);
+ r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!r) {
+ dev_err(jrdev, "platform_get_resource() failed\n");
+ return -ENOMEM;
+ }
+
+ ctrl = devm_ioremap(jrdev, r->start, resource_size(r));
if (!ctrl) {
- dev_err(jrdev, "of_iomap() failed\n");
+ dev_err(jrdev, "devm_ioremap() failed\n");
return -ENOMEM;
}
jrpriv->rregs = (struct caam_job_ring __iomem __force *)ctrl;
- if (sizeof(dma_addr_t) == sizeof(u64)) {
- if (caam_dpaa2)
- error = dma_set_mask_and_coherent(jrdev,
- DMA_BIT_MASK(49));
- else if (of_device_is_compatible(nprop,
- "fsl,sec-v5.0-job-ring"))
- error = dma_set_mask_and_coherent(jrdev,
- DMA_BIT_MASK(40));
- else
- error = dma_set_mask_and_coherent(jrdev,
- DMA_BIT_MASK(36));
- } else {
- error = dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(32));
- }
+ error = dma_set_mask_and_coherent(jrdev, caam_get_dma_mask(jrdev));
if (error) {
dev_err(jrdev, "dma_set_mask_and_coherent failed (%d)\n",
error);
- iounmap(ctrl);
return error;
}
/* Identify the interrupt */
jrpriv->irq = irq_of_parse_and_map(nprop, 0);
+ if (!jrpriv->irq) {
+ dev_err(jrdev, "irq_of_parse_and_map failed\n");
+ return -EINVAL;
+ }
+
+ error = devm_add_action_or_reset(jrdev, caam_jr_irq_dispose_mapping,
+ (void *)(unsigned long)jrpriv->irq);
+ if (error)
+ return error;
/* Now do the platform independent part */
error = caam_jr_init(jrdev); /* now turn on hardware */
- if (error) {
- irq_dispose_mapping(jrpriv->irq);
- iounmap(ctrl);
+ if (error)
return error;
- }
jrpriv->dev = jrdev;
spin_lock(&driver_data.jr_alloc_lock);
@@ -541,6 +562,8 @@ static int caam_jr_probe(struct platform_device *pdev)
atomic_set(&jrpriv->tfm_count, 0);
+ register_algs(jrdev->parent);
+
return 0;
}
diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c
index 8d0713fae6ac..5a851ddc48fb 100644
--- a/drivers/crypto/caam/key_gen.c
+++ b/drivers/crypto/caam/key_gen.c
@@ -15,15 +15,14 @@ void split_key_done(struct device *dev, u32 *desc, u32 err,
void *context)
{
struct split_key_result *res = context;
+ int ecode = 0;
-#ifdef DEBUG
- dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+ dev_dbg(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
if (err)
- caam_jr_strstatus(dev, err);
+ ecode = caam_jr_strstatus(dev, err);
- res->err = err;
+ res->err = ecode;
complete(&res->completion);
}
@@ -49,20 +48,20 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
u32 *desc;
struct split_key_result result;
dma_addr_t dma_addr;
+ unsigned int local_max;
int ret = -ENOMEM;
adata->keylen = split_key_len(adata->algtype & OP_ALG_ALGSEL_MASK);
adata->keylen_pad = split_key_pad_len(adata->algtype &
OP_ALG_ALGSEL_MASK);
+ local_max = max(keylen, adata->keylen_pad);
-#ifdef DEBUG
- dev_err(jrdev, "split keylen %d split keylen padded %d\n",
+ dev_dbg(jrdev, "split keylen %d split keylen padded %d\n",
adata->keylen, adata->keylen_pad);
- print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1);
-#endif
+ print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1);
- if (adata->keylen_pad > max_keylen)
+ if (local_max > max_keylen)
return -EINVAL;
desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
@@ -73,8 +72,7 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
memcpy(key_out, key_in, keylen);
- dma_addr = dma_map_single(jrdev, key_out, adata->keylen_pad,
- DMA_BIDIRECTIONAL);
+ dma_addr = dma_map_single(jrdev, key_out, local_max, DMA_BIDIRECTIONAL);
if (dma_mapping_error(jrdev, dma_addr)) {
dev_err(jrdev, "unable to map key memory\n");
goto out_free;
@@ -102,10 +100,9 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
append_fifo_store(desc, dma_addr, adata->keylen,
LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK);
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+ print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
result.err = 0;
init_completion(&result.completion);
@@ -115,14 +112,13 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
/* in progress */
wait_for_completion(&result.completion);
ret = result.err;
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key_out,
- adata->keylen_pad, 1);
-#endif
+
+ print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key_out,
+ adata->keylen_pad, 1);
}
- dma_unmap_single(jrdev, dma_addr, adata->keylen_pad, DMA_BIDIRECTIONAL);
+ dma_unmap_single(jrdev, dma_addr, local_max, DMA_BIDIRECTIONAL);
out_free:
kfree(desc);
return ret;
diff --git a/drivers/crypto/caam/pdb.h b/drivers/crypto/caam/pdb.h
index 810f0bef0652..68c1fd5dee5d 100644
--- a/drivers/crypto/caam/pdb.h
+++ b/drivers/crypto/caam/pdb.h
@@ -512,7 +512,9 @@ struct rsa_pub_pdb {
dma_addr_t n_dma;
dma_addr_t e_dma;
u32 f_len;
-} __packed;
+};
+
+#define SIZEOF_RSA_PUB_PDB (2 * sizeof(u32) + 4 * caam_ptr_sz)
/**
* RSA Decrypt PDB - Private Key Form #1
@@ -528,7 +530,9 @@ struct rsa_priv_f1_pdb {
dma_addr_t f_dma;
dma_addr_t n_dma;
dma_addr_t d_dma;
-} __packed;
+};
+
+#define SIZEOF_RSA_PRIV_F1_PDB (sizeof(u32) + 4 * caam_ptr_sz)
/**
* RSA Decrypt PDB - Private Key Form #2
@@ -554,7 +558,9 @@ struct rsa_priv_f2_pdb {
dma_addr_t tmp1_dma;
dma_addr_t tmp2_dma;
u32 p_q_len;
-} __packed;
+};
+
+#define SIZEOF_RSA_PRIV_F2_PDB (2 * sizeof(u32) + 7 * caam_ptr_sz)
/**
* RSA Decrypt PDB - Private Key Form #3
@@ -586,6 +592,8 @@ struct rsa_priv_f3_pdb {
dma_addr_t tmp1_dma;
dma_addr_t tmp2_dma;
u32 p_q_len;
-} __packed;
+};
+
+#define SIZEOF_RSA_PRIV_F3_PDB (2 * sizeof(u32) + 9 * caam_ptr_sz)
#endif
diff --git a/drivers/crypto/caam/pkc_desc.c b/drivers/crypto/caam/pkc_desc.c
index 2a8d87ea94bf..0d5ee762e036 100644
--- a/drivers/crypto/caam/pkc_desc.c
+++ b/drivers/crypto/caam/pkc_desc.c
@@ -13,7 +13,7 @@
/* Descriptor for RSA Public operation */
void init_rsa_pub_desc(u32 *desc, struct rsa_pub_pdb *pdb)
{
- init_job_desc_pdb(desc, 0, sizeof(*pdb));
+ init_job_desc_pdb(desc, 0, SIZEOF_RSA_PUB_PDB);
append_cmd(desc, pdb->sgf);
append_ptr(desc, pdb->f_dma);
append_ptr(desc, pdb->g_dma);
@@ -26,7 +26,7 @@ void init_rsa_pub_desc(u32 *desc, struct rsa_pub_pdb *pdb)
/* Descriptor for RSA Private operation - Private Key Form #1 */
void init_rsa_priv_f1_desc(u32 *desc, struct rsa_priv_f1_pdb *pdb)
{
- init_job_desc_pdb(desc, 0, sizeof(*pdb));
+ init_job_desc_pdb(desc, 0, SIZEOF_RSA_PRIV_F1_PDB);
append_cmd(desc, pdb->sgf);
append_ptr(desc, pdb->g_dma);
append_ptr(desc, pdb->f_dma);
@@ -39,7 +39,7 @@ void init_rsa_priv_f1_desc(u32 *desc, struct rsa_priv_f1_pdb *pdb)
/* Descriptor for RSA Private operation - Private Key Form #2 */
void init_rsa_priv_f2_desc(u32 *desc, struct rsa_priv_f2_pdb *pdb)
{
- init_job_desc_pdb(desc, 0, sizeof(*pdb));
+ init_job_desc_pdb(desc, 0, SIZEOF_RSA_PRIV_F2_PDB);
append_cmd(desc, pdb->sgf);
append_ptr(desc, pdb->g_dma);
append_ptr(desc, pdb->f_dma);
@@ -56,7 +56,7 @@ void init_rsa_priv_f2_desc(u32 *desc, struct rsa_priv_f2_pdb *pdb)
/* Descriptor for RSA Private operation - Private Key Form #3 */
void init_rsa_priv_f3_desc(u32 *desc, struct rsa_priv_f3_pdb *pdb)
{
- init_job_desc_pdb(desc, 0, sizeof(*pdb));
+ init_job_desc_pdb(desc, 0, SIZEOF_RSA_PRIV_F3_PDB);
append_cmd(desc, pdb->sgf);
append_ptr(desc, pdb->g_dma);
append_ptr(desc, pdb->f_dma);
diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c
index 9f08f84cca59..378f627e1d64 100644
--- a/drivers/crypto/caam/qi.c
+++ b/drivers/crypto/caam/qi.c
@@ -4,7 +4,7 @@
* Queue Interface backend functionality
*
* Copyright 2013-2016 Freescale Semiconductor, Inc.
- * Copyright 2016-2017 NXP
+ * Copyright 2016-2017, 2019 NXP
*/
#include <linux/cpumask.h>
@@ -18,6 +18,7 @@
#include "desc_constr.h"
#define PREHDR_RSLS_SHIFT 31
+#define PREHDR_ABS BIT(25)
/*
* Use a reasonable backlog of frames (per CPU) as congestion threshold,
@@ -58,11 +59,9 @@ static DEFINE_PER_CPU(int, last_cpu);
/*
* caam_qi_priv - CAAM QI backend private params
* @cgr: QMan congestion group
- * @qi_pdev: platform device for QI backend
*/
struct caam_qi_priv {
struct qman_cgr cgr;
- struct platform_device *qi_pdev;
};
static struct caam_qi_priv qipriv ____cacheline_aligned;
@@ -95,6 +94,16 @@ static u64 times_congested;
*/
static struct kmem_cache *qi_cache;
+static void *caam_iova_to_virt(struct iommu_domain *domain,
+ dma_addr_t iova_addr)
+{
+ phys_addr_t phys_addr;
+
+ phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr;
+
+ return phys_to_virt(phys_addr);
+}
+
int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req)
{
struct qm_fd fd;
@@ -135,6 +144,7 @@ static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq,
const struct qm_fd *fd;
struct caam_drv_req *drv_req;
struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
+ struct caam_drv_private *priv = dev_get_drvdata(qidev);
fd = &msg->ern.fd;
@@ -143,7 +153,7 @@ static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq,
return;
}
- drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd));
+ drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd));
if (!drv_req) {
dev_err(qidev,
"Can't find original request for CAAM response\n");
@@ -153,7 +163,10 @@ static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq,
dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd),
sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL);
- drv_req->cbk(drv_req, -EIO);
+ if (fd->status)
+ drv_req->cbk(drv_req, be32_to_cpu(fd->status));
+ else
+ drv_req->cbk(drv_req, JRSTA_SSRC_QI);
}
static struct qman_fq *create_caam_req_fq(struct device *qidev,
@@ -346,6 +359,7 @@ int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc)
*/
drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
num_words);
+ drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS);
memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
dma_sync_single_for_device(qidev, drv_ctx->context_a,
sizeof(drv_ctx->sh_desc) +
@@ -401,6 +415,7 @@ struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev,
*/
drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
num_words);
+ drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS);
memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc);
hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size,
@@ -488,7 +503,7 @@ EXPORT_SYMBOL(caam_drv_ctx_rel);
void caam_qi_shutdown(struct device *qidev)
{
int i;
- struct caam_qi_priv *priv = dev_get_drvdata(qidev);
+ struct caam_qi_priv *priv = &qipriv;
const cpumask_t *cpus = qman_affine_cpus();
for_each_cpu(i, cpus) {
@@ -506,8 +521,6 @@ void caam_qi_shutdown(struct device *qidev)
qman_release_cgrid(priv->cgr.cgrid);
kmem_cache_destroy(qi_cache);
-
- platform_device_unregister(priv->qi_pdev);
}
static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested)
@@ -550,6 +563,7 @@ static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p,
struct caam_drv_req *drv_req;
const struct qm_fd *fd;
struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
+ struct caam_drv_private *priv = dev_get_drvdata(qidev);
u32 status;
if (caam_qi_napi_schedule(p, caam_napi))
@@ -563,8 +577,9 @@ static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p,
if (ssrc != JRSTA_SSRC_CCB_ERROR ||
err_id != JRSTA_CCBERR_ERRID_ICVCHK)
- dev_err(qidev, "Error: %#x in CAAM response FD\n",
- status);
+ dev_err_ratelimited(qidev,
+ "Error: %#x in CAAM response FD\n",
+ status);
}
if (unlikely(qm_fd_get_format(fd) != qm_fd_compound)) {
@@ -572,7 +587,7 @@ static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p,
return qman_cb_dqrr_consume;
}
- drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd));
+ drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd));
if (unlikely(!drv_req)) {
dev_err(qidev,
"Can't find original request for caam response\n");
@@ -692,33 +707,17 @@ static void free_rsp_fqs(void)
int caam_qi_init(struct platform_device *caam_pdev)
{
int err, i;
- struct platform_device *qi_pdev;
struct device *ctrldev = &caam_pdev->dev, *qidev;
struct caam_drv_private *ctrlpriv;
const cpumask_t *cpus = qman_affine_cpus();
- static struct platform_device_info qi_pdev_info = {
- .name = "caam_qi",
- .id = PLATFORM_DEVID_NONE
- };
-
- qi_pdev_info.parent = ctrldev;
- qi_pdev_info.dma_mask = dma_get_mask(ctrldev);
- qi_pdev = platform_device_register_full(&qi_pdev_info);
- if (IS_ERR(qi_pdev))
- return PTR_ERR(qi_pdev);
- set_dma_ops(&qi_pdev->dev, get_dma_ops(ctrldev));
ctrlpriv = dev_get_drvdata(ctrldev);
- qidev = &qi_pdev->dev;
-
- qipriv.qi_pdev = qi_pdev;
- dev_set_drvdata(qidev, &qipriv);
+ qidev = ctrldev;
/* Initialize the congestion detection */
err = init_cgr(qidev);
if (err) {
dev_err(qidev, "CGR initialization failed: %d\n", err);
- platform_device_unregister(qi_pdev);
return err;
}
@@ -727,7 +726,6 @@ int caam_qi_init(struct platform_device *caam_pdev)
if (err) {
dev_err(qidev, "Can't allocate CAAM response FQs: %d\n", err);
free_rsp_fqs();
- platform_device_unregister(qi_pdev);
return err;
}
@@ -750,15 +748,11 @@ int caam_qi_init(struct platform_device *caam_pdev)
napi_enable(irqtask);
}
- /* Hook up QI device to parent controlling caam device */
- ctrlpriv->qidev = qidev;
-
qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0,
SLAB_CACHE_DMA, NULL);
if (!qi_cache) {
dev_err(qidev, "Can't allocate CAAM cache\n");
free_rsp_fqs();
- platform_device_unregister(qi_pdev);
return -ENOMEM;
}
@@ -766,6 +760,8 @@ int caam_qi_init(struct platform_device *caam_pdev)
debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl,
&times_congested, &caam_fops_u64_ro);
#endif
+
+ ctrlpriv->qi_init = 1;
dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n");
return 0;
}
diff --git a/drivers/crypto/caam/qi.h b/drivers/crypto/caam/qi.h
index f93c9c7ed430..db0549549e3b 100644
--- a/drivers/crypto/caam/qi.h
+++ b/drivers/crypto/caam/qi.h
@@ -14,32 +14,6 @@
#include "desc.h"
#include "desc_constr.h"
-/*
- * CAAM hardware constructs a job descriptor which points to a shared descriptor
- * (as pointed by context_a of to-CAAM FQ).
- * When the job descriptor is executed by DECO, the whole job descriptor
- * together with shared descriptor gets loaded in DECO buffer, which is
- * 64 words (each 32-bit) long.
- *
- * The job descriptor constructed by CAAM hardware has the following layout:
- *
- * HEADER (1 word)
- * Shdesc ptr (1 or 2 words)
- * SEQ_OUT_PTR (1 word)
- * Out ptr (1 or 2 words)
- * Out length (1 word)
- * SEQ_IN_PTR (1 word)
- * In ptr (1 or 2 words)
- * In length (1 word)
- *
- * The shdesc ptr is used to fetch shared descriptor contents into DECO buffer.
- *
- * Apart from shdesc contents, the total number of words that get loaded in DECO
- * buffer are '8' or '11'. The remaining words in DECO buffer can be used for
- * storing shared descriptor.
- */
-#define MAX_SDLEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / CAAM_CMD_SZ)
-
/* Length of a single buffer in the QI driver memory cache */
#define CAAM_QI_MEMCACHE_SIZE 768
diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h
index 8591914d5c51..05127b70527d 100644
--- a/drivers/crypto/caam/regs.h
+++ b/drivers/crypto/caam/regs.h
@@ -12,6 +12,7 @@
#include <linux/types.h>
#include <linux/bitops.h>
#include <linux/io.h>
+#include <linux/io-64-nonatomic-hi-lo.h>
/*
* Architecture-specific register access methods
@@ -70,6 +71,7 @@
extern bool caam_little_end;
extern bool caam_imx;
+extern size_t caam_ptr_sz;
#define caam_to_cpu(len) \
static inline u##len caam##len ## _to_cpu(u##len val) \
@@ -137,46 +139,38 @@ static inline void clrsetbits_32(void __iomem *reg, u32 clear, u32 set)
* base + 0x0000 : least-significant 32 bits
* base + 0x0004 : most-significant 32 bits
*/
-#ifdef CONFIG_64BIT
static inline void wr_reg64(void __iomem *reg, u64 data)
{
- if (caam_little_end)
- iowrite64(data, reg);
- else
+ if (caam_little_end) {
+ if (caam_imx) {
+ iowrite32(data >> 32, (u32 __iomem *)(reg));
+ iowrite32(data, (u32 __iomem *)(reg) + 1);
+ } else {
+ iowrite64(data, reg);
+ }
+ } else {
iowrite64be(data, reg);
+ }
}
static inline u64 rd_reg64(void __iomem *reg)
{
- if (caam_little_end)
- return ioread64(reg);
- else
- return ioread64be(reg);
-}
+ if (caam_little_end) {
+ if (caam_imx) {
+ u32 low, high;
-#else /* CONFIG_64BIT */
-static inline void wr_reg64(void __iomem *reg, u64 data)
-{
- if (!caam_imx && caam_little_end) {
- wr_reg32((u32 __iomem *)(reg) + 1, data >> 32);
- wr_reg32((u32 __iomem *)(reg), data);
+ high = ioread32(reg);
+ low = ioread32(reg + sizeof(u32));
+
+ return low + ((u64)high << 32);
+ } else {
+ return ioread64(reg);
+ }
} else {
- wr_reg32((u32 __iomem *)(reg), data >> 32);
- wr_reg32((u32 __iomem *)(reg) + 1, data);
+ return ioread64be(reg);
}
}
-static inline u64 rd_reg64(void __iomem *reg)
-{
- if (!caam_imx && caam_little_end)
- return ((u64)rd_reg32((u32 __iomem *)(reg) + 1) << 32 |
- (u64)rd_reg32((u32 __iomem *)(reg)));
-
- return ((u64)rd_reg32((u32 __iomem *)(reg)) << 32 |
- (u64)rd_reg32((u32 __iomem *)(reg) + 1));
-}
-#endif /* CONFIG_64BIT */
-
static inline u64 cpu_to_caam_dma64(dma_addr_t value)
{
if (caam_imx)
@@ -195,22 +189,89 @@ static inline u64 caam_dma64_to_cpu(u64 value)
return caam64_to_cpu(value);
}
-#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
-#define cpu_to_caam_dma(value) cpu_to_caam_dma64(value)
-#define caam_dma_to_cpu(value) caam_dma64_to_cpu(value)
-#else
-#define cpu_to_caam_dma(value) cpu_to_caam32(value)
-#define caam_dma_to_cpu(value) caam32_to_cpu(value)
-#endif /* CONFIG_ARCH_DMA_ADDR_T_64BIT */
+static inline u64 cpu_to_caam_dma(u64 value)
+{
+ if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) &&
+ caam_ptr_sz == sizeof(u64))
+ return cpu_to_caam_dma64(value);
+ else
+ return cpu_to_caam32(value);
+}
+
+static inline u64 caam_dma_to_cpu(u64 value)
+{
+ if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) &&
+ caam_ptr_sz == sizeof(u64))
+ return caam_dma64_to_cpu(value);
+ else
+ return caam32_to_cpu(value);
+}
/*
* jr_outentry
* Represents each entry in a JobR output ring
*/
-struct jr_outentry {
- dma_addr_t desc;/* Pointer to completed descriptor */
- u32 jrstatus; /* Status for completed descriptor */
-} __packed;
+
+static inline void jr_outentry_get(void *outring, int hw_idx, dma_addr_t *desc,
+ u32 *jrstatus)
+{
+
+ if (caam_ptr_sz == sizeof(u32)) {
+ struct {
+ u32 desc;
+ u32 jrstatus;
+ } __packed *outentry = outring;
+
+ *desc = outentry[hw_idx].desc;
+ *jrstatus = outentry[hw_idx].jrstatus;
+ } else {
+ struct {
+ dma_addr_t desc;/* Pointer to completed descriptor */
+ u32 jrstatus; /* Status for completed descriptor */
+ } __packed *outentry = outring;
+
+ *desc = outentry[hw_idx].desc;
+ *jrstatus = outentry[hw_idx].jrstatus;
+ }
+}
+
+#define SIZEOF_JR_OUTENTRY (caam_ptr_sz + sizeof(u32))
+
+static inline dma_addr_t jr_outentry_desc(void *outring, int hw_idx)
+{
+ dma_addr_t desc;
+ u32 unused;
+
+ jr_outentry_get(outring, hw_idx, &desc, &unused);
+
+ return desc;
+}
+
+static inline u32 jr_outentry_jrstatus(void *outring, int hw_idx)
+{
+ dma_addr_t unused;
+ u32 jrstatus;
+
+ jr_outentry_get(outring, hw_idx, &unused, &jrstatus);
+
+ return jrstatus;
+}
+
+static inline void jr_inpentry_set(void *inpring, int hw_idx, dma_addr_t val)
+{
+ if (caam_ptr_sz == sizeof(u32)) {
+ u32 *inpentry = inpring;
+
+ inpentry[hw_idx] = val;
+ } else {
+ dma_addr_t *inpentry = inpring;
+
+ inpentry[hw_idx] = val;
+ }
+}
+
+#define SIZEOF_JR_INPENTRY caam_ptr_sz
+
/* Version registers (Era 10+) e80-eff */
struct version_regs {
@@ -338,6 +399,7 @@ struct caam_perfmon {
u32 cha_rev_ls; /* CRNR - CHA Rev No. Least significant half*/
#define CTPR_MS_QI_SHIFT 25
#define CTPR_MS_QI_MASK (0x1ull << CTPR_MS_QI_SHIFT)
+#define CTPR_MS_PS BIT(17)
#define CTPR_MS_DPAA2 BIT(13)
#define CTPR_MS_VIRT_EN_INCL 0x00000001
#define CTPR_MS_VIRT_EN_POR 0x00000002
@@ -641,6 +703,7 @@ struct caam_job_ring {
#define JRSTA_SSRC_CCB_ERROR 0x20000000
#define JRSTA_SSRC_JUMP_HALT_USER 0x30000000
#define JRSTA_SSRC_DECO 0x40000000
+#define JRSTA_SSRC_QI 0x50000000
#define JRSTA_SSRC_JRERROR 0x60000000
#define JRSTA_SSRC_JUMP_HALT_CC 0x70000000
diff --git a/drivers/crypto/caam/sg_sw_qm.h b/drivers/crypto/caam/sg_sw_qm.h
index b3e1aaaeffea..d56cc7efbc13 100644
--- a/drivers/crypto/caam/sg_sw_qm.h
+++ b/drivers/crypto/caam/sg_sw_qm.h
@@ -54,15 +54,19 @@ static inline void dma_to_qm_sg_one_last_ext(struct qm_sg_entry *qm_sg_ptr,
* but does not have final bit; instead, returns last entry
*/
static inline struct qm_sg_entry *
-sg_to_qm_sg(struct scatterlist *sg, int sg_count,
+sg_to_qm_sg(struct scatterlist *sg, int len,
struct qm_sg_entry *qm_sg_ptr, u16 offset)
{
- while (sg_count && sg) {
- dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg),
- sg_dma_len(sg), offset);
+ int ent_len;
+
+ while (len) {
+ ent_len = min_t(int, sg_dma_len(sg), len);
+
+ dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), ent_len,
+ offset);
qm_sg_ptr++;
sg = sg_next(sg);
- sg_count--;
+ len -= ent_len;
}
return qm_sg_ptr - 1;
}
@@ -71,10 +75,10 @@ sg_to_qm_sg(struct scatterlist *sg, int sg_count,
* convert scatterlist to h/w link table format
* scatterlist must have been previously dma mapped
*/
-static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count,
+static inline void sg_to_qm_sg_last(struct scatterlist *sg, int len,
struct qm_sg_entry *qm_sg_ptr, u16 offset)
{
- qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset);
+ qm_sg_ptr = sg_to_qm_sg(sg, len, qm_sg_ptr, offset);
qm_sg_entry_set_f(qm_sg_ptr, qm_sg_entry_get_len(qm_sg_ptr));
}
diff --git a/drivers/crypto/caam/sg_sw_qm2.h b/drivers/crypto/caam/sg_sw_qm2.h
index c9378402a5f8..b8b737d2b0ea 100644
--- a/drivers/crypto/caam/sg_sw_qm2.h
+++ b/drivers/crypto/caam/sg_sw_qm2.h
@@ -25,15 +25,19 @@ static inline void dma_to_qm_sg_one(struct dpaa2_sg_entry *qm_sg_ptr,
* but does not have final bit; instead, returns last entry
*/
static inline struct dpaa2_sg_entry *
-sg_to_qm_sg(struct scatterlist *sg, int sg_count,
+sg_to_qm_sg(struct scatterlist *sg, int len,
struct dpaa2_sg_entry *qm_sg_ptr, u16 offset)
{
- while (sg_count && sg) {
- dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg),
- sg_dma_len(sg), offset);
+ int ent_len;
+
+ while (len) {
+ ent_len = min_t(int, sg_dma_len(sg), len);
+
+ dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), ent_len,
+ offset);
qm_sg_ptr++;
sg = sg_next(sg);
- sg_count--;
+ len -= ent_len;
}
return qm_sg_ptr - 1;
}
@@ -42,11 +46,11 @@ sg_to_qm_sg(struct scatterlist *sg, int sg_count,
* convert scatterlist to h/w link table format
* scatterlist must have been previously dma mapped
*/
-static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count,
+static inline void sg_to_qm_sg_last(struct scatterlist *sg, int len,
struct dpaa2_sg_entry *qm_sg_ptr,
u16 offset)
{
- qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset);
+ qm_sg_ptr = sg_to_qm_sg(sg, len, qm_sg_ptr, offset);
dpaa2_sg_set_final(qm_sg_ptr, true);
}
diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h
index dbfa9fce33e0..07e1ee99273b 100644
--- a/drivers/crypto/caam/sg_sw_sec4.h
+++ b/drivers/crypto/caam/sg_sw_sec4.h
@@ -35,11 +35,9 @@ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr,
sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset &
SEC4_SG_OFFSET_MASK);
}
-#ifdef DEBUG
- print_hex_dump(KERN_ERR, "sec4_sg_ptr@: ",
- DUMP_PREFIX_ADDRESS, 16, 4, sec4_sg_ptr,
- sizeof(struct sec4_sg_entry), 1);
-#endif
+
+ print_hex_dump_debug("sec4_sg_ptr@: ", DUMP_PREFIX_ADDRESS, 16, 4,
+ sec4_sg_ptr, sizeof(struct sec4_sg_entry), 1);
}
/*
@@ -47,15 +45,19 @@ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr,
* but does not have final bit; instead, returns last entry
*/
static inline struct sec4_sg_entry *
-sg_to_sec4_sg(struct scatterlist *sg, int sg_count,
+sg_to_sec4_sg(struct scatterlist *sg, int len,
struct sec4_sg_entry *sec4_sg_ptr, u16 offset)
{
- while (sg_count) {
- dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg),
- sg_dma_len(sg), offset);
+ int ent_len;
+
+ while (len) {
+ ent_len = min_t(int, sg_dma_len(sg), len);
+
+ dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), ent_len,
+ offset);
sec4_sg_ptr++;
sg = sg_next(sg);
- sg_count--;
+ len -= ent_len;
}
return sec4_sg_ptr - 1;
}
@@ -72,11 +74,11 @@ static inline void sg_to_sec4_set_last(struct sec4_sg_entry *sec4_sg_ptr)
* convert scatterlist to h/w link table format
* scatterlist must have been previously dma mapped
*/
-static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int sg_count,
+static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int len,
struct sec4_sg_entry *sec4_sg_ptr,
u16 offset)
{
- sec4_sg_ptr = sg_to_sec4_sg(sg, sg_count, sec4_sg_ptr, offset);
+ sec4_sg_ptr = sg_to_sec4_sg(sg, len, sec4_sg_ptr, offset);
sg_to_sec4_set_last(sec4_sg_ptr);
}
diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.c b/drivers/crypto/cavium/cpt/cptvf_algs.c
index e9f4704494fb..596ce28b957d 100644
--- a/drivers/crypto/cavium/cpt/cptvf_algs.c
+++ b/drivers/crypto/cavium/cpt/cptvf_algs.c
@@ -7,8 +7,7 @@
#include <crypto/aes.h>
#include <crypto/algapi.h>
#include <crypto/authenc.h>
-#include <crypto/crypto_wq.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <crypto/xts.h>
#include <linux/crypto.h>
#include <linux/err.h>
@@ -323,31 +322,15 @@ static int cvm_cfb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
static int cvm_cbc_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
u32 keylen)
{
- u32 flags = crypto_ablkcipher_get_flags(cipher);
- int err;
-
- err = __des3_verify_key(&flags, key);
- if (unlikely(err)) {
- crypto_ablkcipher_set_flags(cipher, flags);
- return err;
- }
-
- return cvm_setkey(cipher, key, keylen, DES3_CBC);
+ return verify_ablkcipher_des3_key(cipher, key) ?:
+ cvm_setkey(cipher, key, keylen, DES3_CBC);
}
static int cvm_ecb_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
u32 keylen)
{
- u32 flags = crypto_ablkcipher_get_flags(cipher);
- int err;
-
- err = __des3_verify_key(&flags, key);
- if (unlikely(err)) {
- crypto_ablkcipher_set_flags(cipher, flags);
- return err;
- }
-
- return cvm_setkey(cipher, key, keylen, DES3_ECB);
+ return verify_ablkcipher_des3_key(cipher, key) ?:
+ cvm_setkey(cipher, key, keylen, DES3_ECB);
}
static int cvm_enc_dec_init(struct crypto_tfm *tfm)
diff --git a/drivers/crypto/cavium/nitrox/Kconfig b/drivers/crypto/cavium/nitrox/Kconfig
index dab162af41b8..7b1e751bb9cd 100644
--- a/drivers/crypto/cavium/nitrox/Kconfig
+++ b/drivers/crypto/cavium/nitrox/Kconfig
@@ -6,7 +6,7 @@ config CRYPTO_DEV_NITROX
tristate
select CRYPTO_BLKCIPHER
select CRYPTO_AES
- select CRYPTO_DES
+ select CRYPTO_LIB_DES
select FW_LOADER
config CRYPTO_DEV_NITROX_CNN55XX
diff --git a/drivers/crypto/cavium/nitrox/nitrox_csr.h b/drivers/crypto/cavium/nitrox/nitrox_csr.h
index a2a452642b38..1c8715ae0488 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_csr.h
+++ b/drivers/crypto/cavium/nitrox/nitrox_csr.h
@@ -40,9 +40,77 @@
#define EMU_FUSE_MAPX(_i) (0x1402708 + ((_i) * 0x40000))
/* UCD registers */
+#define UCD_SE_EID_UCODE_BLOCK_NUMX(_i) (0x12C0000 + ((_i) * 0x1000))
+#define UCD_AE_EID_UCODE_BLOCK_NUMX(_i) (0x12C0008 + ((_i) * 0x800))
#define UCD_UCODE_LOAD_BLOCK_NUM 0x12C0010
#define UCD_UCODE_LOAD_IDX_DATAX(_i) (0x12C0018 + ((_i) * 0x20))
-#define UCD_SE_EID_UCODE_BLOCK_NUMX(_i) (0x12C0000 + ((_i) * 0x1000))
+#define UCD_SE_CNTX(_i) (0x12C0040 + ((_i) * 0x1000))
+#define UCD_AE_CNTX(_i) (0x12C0048 + ((_i) * 0x800))
+
+/* AQM registers */
+#define AQM_CTL 0x1300000
+#define AQM_INT 0x1300008
+#define AQM_DBELL_OVF_LO 0x1300010
+#define AQM_DBELL_OVF_HI 0x1300018
+#define AQM_DBELL_OVF_LO_W1S 0x1300020
+#define AQM_DBELL_OVF_LO_ENA_W1C 0x1300028
+#define AQM_DBELL_OVF_LO_ENA_W1S 0x1300030
+#define AQM_DBELL_OVF_HI_W1S 0x1300038
+#define AQM_DBELL_OVF_HI_ENA_W1C 0x1300040
+#define AQM_DBELL_OVF_HI_ENA_W1S 0x1300048
+#define AQM_DMA_RD_ERR_LO 0x1300050
+#define AQM_DMA_RD_ERR_HI 0x1300058
+#define AQM_DMA_RD_ERR_LO_W1S 0x1300060
+#define AQM_DMA_RD_ERR_LO_ENA_W1C 0x1300068
+#define AQM_DMA_RD_ERR_LO_ENA_W1S 0x1300070
+#define AQM_DMA_RD_ERR_HI_W1S 0x1300078
+#define AQM_DMA_RD_ERR_HI_ENA_W1C 0x1300080
+#define AQM_DMA_RD_ERR_HI_ENA_W1S 0x1300088
+#define AQM_EXEC_NA_LO 0x1300090
+#define AQM_EXEC_NA_HI 0x1300098
+#define AQM_EXEC_NA_LO_W1S 0x13000A0
+#define AQM_EXEC_NA_LO_ENA_W1C 0x13000A8
+#define AQM_EXEC_NA_LO_ENA_W1S 0x13000B0
+#define AQM_EXEC_NA_HI_W1S 0x13000B8
+#define AQM_EXEC_NA_HI_ENA_W1C 0x13000C0
+#define AQM_EXEC_NA_HI_ENA_W1S 0x13000C8
+#define AQM_EXEC_ERR_LO 0x13000D0
+#define AQM_EXEC_ERR_HI 0x13000D8
+#define AQM_EXEC_ERR_LO_W1S 0x13000E0
+#define AQM_EXEC_ERR_LO_ENA_W1C 0x13000E8
+#define AQM_EXEC_ERR_LO_ENA_W1S 0x13000F0
+#define AQM_EXEC_ERR_HI_W1S 0x13000F8
+#define AQM_EXEC_ERR_HI_ENA_W1C 0x1300100
+#define AQM_EXEC_ERR_HI_ENA_W1S 0x1300108
+#define AQM_ECC_INT 0x1300110
+#define AQM_ECC_INT_W1S 0x1300118
+#define AQM_ECC_INT_ENA_W1C 0x1300120
+#define AQM_ECC_INT_ENA_W1S 0x1300128
+#define AQM_ECC_CTL 0x1300130
+#define AQM_BIST_STATUS 0x1300138
+#define AQM_CMD_INF_THRX(x) (0x1300400 + ((x) * 0x8))
+#define AQM_CMD_INFX(x) (0x1300800 + ((x) * 0x8))
+#define AQM_GRP_EXECMSK_LOX(x) (0x1300C00 + ((x) * 0x10))
+#define AQM_GRP_EXECMSK_HIX(x) (0x1300C08 + ((x) * 0x10))
+#define AQM_ACTIVITY_STAT_LO 0x1300C80
+#define AQM_ACTIVITY_STAT_HI 0x1300C88
+#define AQM_Q_CMD_PROCX(x) (0x1301000 + ((x) * 0x8))
+#define AQM_PERF_CTL_LO 0x1301400
+#define AQM_PERF_CTL_HI 0x1301408
+#define AQM_PERF_CNT 0x1301410
+
+#define AQMQ_DRBLX(x) (0x20000 + ((x) * 0x40000))
+#define AQMQ_QSZX(x) (0x20008 + ((x) * 0x40000))
+#define AQMQ_BADRX(x) (0x20010 + ((x) * 0x40000))
+#define AQMQ_NXT_CMDX(x) (0x20018 + ((x) * 0x40000))
+#define AQMQ_CMD_CNTX(x) (0x20020 + ((x) * 0x40000))
+#define AQMQ_CMP_THRX(x) (0x20028 + ((x) * 0x40000))
+#define AQMQ_CMP_CNTX(x) (0x20030 + ((x) * 0x40000))
+#define AQMQ_TIM_LDX(x) (0x20038 + ((x) * 0x40000))
+#define AQMQ_TIMERX(x) (0x20040 + ((x) * 0x40000))
+#define AQMQ_ENX(x) (0x20048 + ((x) * 0x40000))
+#define AQMQ_ACTIVITY_STATX(x) (0x20050 + ((x) * 0x40000))
+#define AQM_VF_CMP_STATX(x) (0x28000 + ((x) * 0x40000))
/* NPS core registers */
#define NPS_CORE_GBL_VFCFG 0x1000000
@@ -135,6 +203,171 @@
#define PEM0_INT 0x1080428
/**
+ * struct ucd_core_eid_ucode_block_num - Core Eid to Ucode Blk Mapping Registers
+ * @ucode_len: Ucode length identifier 32KB or 64KB
+ * @ucode_blk: Ucode Block Number
+ */
+union ucd_core_eid_ucode_block_num {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_4_63 : 60;
+ u64 ucode_len : 1;
+ u64 ucode_blk : 3;
+#else
+ u64 ucode_blk : 3;
+ u64 ucode_len : 1;
+ u64 raz_4_63 : 60;
+#endif
+ };
+};
+
+/**
+ * struct aqm_grp_execmsk_lo - Available AE engines for the group
+ * @exec_0_to_39: AE engines 0 to 39 status
+ */
+union aqm_grp_execmsk_lo {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_40_63 : 24;
+ u64 exec_0_to_39 : 40;
+#else
+ u64 exec_0_to_39 : 40;
+ u64 raz_40_63 : 24;
+#endif
+ };
+};
+
+/**
+ * struct aqm_grp_execmsk_hi - Available AE engines for the group
+ * @exec_40_to_79: AE engines 40 to 79 status
+ */
+union aqm_grp_execmsk_hi {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_40_63 : 24;
+ u64 exec_40_to_79 : 40;
+#else
+ u64 exec_40_to_79 : 40;
+ u64 raz_40_63 : 24;
+#endif
+ };
+};
+
+/**
+ * struct aqmq_drbl - AQM Queue Doorbell Counter Registers
+ * @dbell_count: Doorbell Counter
+ */
+union aqmq_drbl {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_32_63 : 32;
+ u64 dbell_count : 32;
+#else
+ u64 dbell_count : 32;
+ u64 raz_32_63 : 32;
+#endif
+ };
+};
+
+/**
+ * struct aqmq_qsz - AQM Queue Host Queue Size Registers
+ * @host_queue_size: Size, in numbers of 'aqmq_command_s' command
+ * of the Host Ring.
+ */
+union aqmq_qsz {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_32_63 : 32;
+ u64 host_queue_size : 32;
+#else
+ u64 host_queue_size : 32;
+ u64 raz_32_63 : 32;
+#endif
+ };
+};
+
+/**
+ * struct aqmq_cmp_thr - AQM Queue Commands Completed Threshold Registers
+ * @commands_completed_threshold: Count of 'aqmq_command_s' commands executed
+ * by AE engines for which completion interrupt is asserted.
+ */
+union aqmq_cmp_thr {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_32_63 : 32;
+ u64 commands_completed_threshold : 32;
+#else
+ u64 commands_completed_threshold : 32;
+ u64 raz_32_63 : 32;
+#endif
+ };
+};
+
+/**
+ * struct aqmq_cmp_cnt - AQM Queue Commands Completed Count Registers
+ * @resend: Bit to request completion interrupt Resend.
+ * @completion_status: Command completion status of the ring.
+ * @commands_completed_count: Count of 'aqmq_command_s' commands executed by
+ * AE engines.
+ */
+union aqmq_cmp_cnt {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_34_63 : 30;
+ u64 resend : 1;
+ u64 completion_status : 1;
+ u64 commands_completed_count : 32;
+#else
+ u64 commands_completed_count : 32;
+ u64 completion_status : 1;
+ u64 resend : 1;
+ u64 raz_34_63 : 30;
+#endif
+ };
+};
+
+/**
+ * struct aqmq_en - AQM Queue Enable Registers
+ * @queue_status: 1 = AQMQ is enabled, 0 = AQMQ is disabled
+ */
+union aqmq_en {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_1_63 : 63;
+ u64 queue_enable : 1;
+#else
+ u64 queue_enable : 1;
+ u64 raz_1_63 : 63;
+#endif
+ };
+};
+
+/**
+ * struct aqmq_activity_stat - AQM Queue Activity Status Registers
+ * @queue_active: 1 = AQMQ is active, 0 = AQMQ is quiescent
+ */
+union aqmq_activity_stat {
+ u64 value;
+ struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+ u64 raz_1_63 : 63;
+ u64 queue_active : 1;
+#else
+ u64 queue_active : 1;
+ u64 raz_1_63 : 63;
+#endif
+ };
+};
+
+/**
* struct emu_fuse_map - EMU Fuse Map Registers
* @ae_fuse: Fuse settings for AE 19..0
* @se_fuse: Fuse settings for SE 15..0
diff --git a/drivers/crypto/cavium/nitrox/nitrox_debugfs.c b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c
index 848ec93d4333..16f7d0bd1303 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_debugfs.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c
@@ -9,7 +9,8 @@ static int firmware_show(struct seq_file *s, void *v)
{
struct nitrox_device *ndev = s->private;
- seq_printf(s, "Version: %s\n", ndev->hw.fw_name);
+ seq_printf(s, "Version: %s\n", ndev->hw.fw_name[0]);
+ seq_printf(s, "Version: %s\n", ndev->hw.fw_name[1]);
return 0;
}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_debugfs.h b/drivers/crypto/cavium/nitrox/nitrox_debugfs.h
index f177b79bbab0..09c4cf2513fb 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_debugfs.h
+++ b/drivers/crypto/cavium/nitrox/nitrox_debugfs.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NITROX_DEBUGFS_H
#define __NITROX_DEBUGFS_H
diff --git a/drivers/crypto/cavium/nitrox/nitrox_dev.h b/drivers/crypto/cavium/nitrox/nitrox_dev.h
index 0338877b828f..2217a2736c8e 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_dev.h
+++ b/drivers/crypto/cavium/nitrox/nitrox_dev.h
@@ -10,6 +10,10 @@
#define VERSION_LEN 32
/* Maximum queues in PF mode */
#define MAX_PF_QUEUES 64
+/* Maximum device queues */
+#define MAX_DEV_QUEUES (MAX_PF_QUEUES)
+/* Maximum UCD Blocks */
+#define CNN55XX_MAX_UCD_BLOCKS 8
/**
* struct nitrox_cmdq - NITROX command queue
@@ -74,7 +78,7 @@ struct nitrox_cmdq {
*/
struct nitrox_hw {
char partname[IFNAMSIZ * 2];
- char fw_name[VERSION_LEN];
+ char fw_name[CNN55XX_MAX_UCD_BLOCKS][VERSION_LEN];
int freq;
u16 vendor_id;
@@ -206,6 +210,7 @@ enum vf_mode {
* @mode: Device mode PF/VF
* @ctx_pool: DMA pool for crypto context
* @pkt_inq: Packet input rings
+ * @aqmq: AQM command queues
* @qvec: MSI-X queue vectors information
* @iov: SR-IOV informatin
* @num_vecs: number of MSI-X vectors
@@ -232,6 +237,7 @@ struct nitrox_device {
struct dma_pool *ctx_pool;
struct nitrox_cmdq *pkt_inq;
+ struct nitrox_cmdq *aqmq[MAX_DEV_QUEUES] ____cacheline_aligned_in_smp;
struct nitrox_q_vector *qvec;
struct nitrox_iov iov;
diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.c b/drivers/crypto/cavium/nitrox/nitrox_hal.c
index 3f0df60267a9..34a2f4f30a7e 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_hal.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_hal.c
@@ -241,12 +241,12 @@ void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev)
}
/**
- * enable_nps_interrupts - enable NPS interrutps
+ * enable_nps_core_interrupts - enable NPS core interrutps
* @ndev: NITROX device.
*
- * This includes NPS core, packet in and slc interrupts.
+ * This includes NPS core interrupts.
*/
-static void enable_nps_interrupts(struct nitrox_device *ndev)
+static void enable_nps_core_interrupts(struct nitrox_device *ndev)
{
union nps_core_int_ena_w1s core_int;
@@ -258,18 +258,9 @@ static void enable_nps_interrupts(struct nitrox_device *ndev)
core_int.s.npco_dma_malform = 1;
core_int.s.host_nps_wr_err = 1;
nitrox_write_csr(ndev, NPS_CORE_INT_ENA_W1S, core_int.value);
-
- /* NPS packet in ring interrupts */
- nitrox_write_csr(ndev, NPS_PKT_IN_RERR_LO_ENA_W1S, (~0ULL));
- nitrox_write_csr(ndev, NPS_PKT_IN_RERR_HI_ENA_W1S, (~0ULL));
- nitrox_write_csr(ndev, NPS_PKT_IN_ERR_TYPE_ENA_W1S, (~0ULL));
- /* NPS packet slc port interrupts */
- nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_HI_ENA_W1S, (~0ULL));
- nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_LO_ENA_W1S, (~0ULL));
- nitrox_write_csr(ndev, NPS_PKT_SLC_ERR_TYPE_ENA_W1S, (~0uLL));
}
-void nitrox_config_nps_unit(struct nitrox_device *ndev)
+void nitrox_config_nps_core_unit(struct nitrox_device *ndev)
{
union nps_core_gbl_vfcfg core_gbl_vfcfg;
@@ -281,12 +272,149 @@ void nitrox_config_nps_unit(struct nitrox_device *ndev)
core_gbl_vfcfg.s.ilk_disable = 1;
core_gbl_vfcfg.s.cfg = __NDEV_MODE_PF;
nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, core_gbl_vfcfg.value);
+
+ /* enable nps core interrupts */
+ enable_nps_core_interrupts(ndev);
+}
+
+/**
+ * enable_nps_pkt_interrupts - enable NPS packet interrutps
+ * @ndev: NITROX device.
+ *
+ * This includes NPS packet in and slc interrupts.
+ */
+static void enable_nps_pkt_interrupts(struct nitrox_device *ndev)
+{
+ /* NPS packet in ring interrupts */
+ nitrox_write_csr(ndev, NPS_PKT_IN_RERR_LO_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, NPS_PKT_IN_RERR_HI_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, NPS_PKT_IN_ERR_TYPE_ENA_W1S, (~0ULL));
+ /* NPS packet slc port interrupts */
+ nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_HI_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_LO_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, NPS_PKT_SLC_ERR_TYPE_ENA_W1S, (~0uLL));
+}
+
+void nitrox_config_nps_pkt_unit(struct nitrox_device *ndev)
+{
/* config input and solicit ports */
nitrox_config_pkt_input_rings(ndev);
nitrox_config_pkt_solicit_ports(ndev);
- /* enable interrupts */
- enable_nps_interrupts(ndev);
+ /* enable nps packet interrupts */
+ enable_nps_pkt_interrupts(ndev);
+}
+
+static void reset_aqm_ring(struct nitrox_device *ndev, int ring)
+{
+ union aqmq_en aqmq_en_reg;
+ union aqmq_activity_stat activity_stat;
+ union aqmq_cmp_cnt cmp_cnt;
+ int max_retries = MAX_CSR_RETRIES;
+ u64 offset;
+
+ /* step 1: disable the queue */
+ offset = AQMQ_ENX(ring);
+ aqmq_en_reg.value = 0;
+ aqmq_en_reg.queue_enable = 0;
+ nitrox_write_csr(ndev, offset, aqmq_en_reg.value);
+
+ /* step 2: wait for AQMQ_ACTIVITY_STATX[QUEUE_ACTIVE] to clear */
+ usleep_range(100, 150);
+ offset = AQMQ_ACTIVITY_STATX(ring);
+ do {
+ activity_stat.value = nitrox_read_csr(ndev, offset);
+ if (!activity_stat.queue_active)
+ break;
+ udelay(50);
+ } while (max_retries--);
+
+ /* step 3: clear commands completed count */
+ offset = AQMQ_CMP_CNTX(ring);
+ cmp_cnt.value = nitrox_read_csr(ndev, offset);
+ nitrox_write_csr(ndev, offset, cmp_cnt.value);
+ usleep_range(50, 100);
+}
+
+void enable_aqm_ring(struct nitrox_device *ndev, int ring)
+{
+ union aqmq_en aqmq_en_reg;
+ u64 offset;
+
+ offset = AQMQ_ENX(ring);
+ aqmq_en_reg.value = 0;
+ aqmq_en_reg.queue_enable = 1;
+ nitrox_write_csr(ndev, offset, aqmq_en_reg.value);
+ usleep_range(50, 100);
+}
+
+void nitrox_config_aqm_rings(struct nitrox_device *ndev)
+{
+ int ring;
+
+ for (ring = 0; ring < ndev->nr_queues; ring++) {
+ struct nitrox_cmdq *cmdq = ndev->aqmq[ring];
+ union aqmq_drbl drbl;
+ union aqmq_qsz qsize;
+ union aqmq_cmp_thr cmp_thr;
+ u64 offset;
+
+ /* steps 1 - 3 */
+ reset_aqm_ring(ndev, ring);
+
+ /* step 4: clear doorbell count of ring */
+ offset = AQMQ_DRBLX(ring);
+ drbl.value = 0;
+ drbl.dbell_count = 0xFFFFFFFF;
+ nitrox_write_csr(ndev, offset, drbl.value);
+
+ /* step 5: configure host ring details */
+
+ /* set host address for next command of ring */
+ offset = AQMQ_NXT_CMDX(ring);
+ nitrox_write_csr(ndev, offset, 0ULL);
+
+ /* set host address of ring base */
+ offset = AQMQ_BADRX(ring);
+ nitrox_write_csr(ndev, offset, cmdq->dma);
+
+ /* set ring size */
+ offset = AQMQ_QSZX(ring);
+ qsize.value = 0;
+ qsize.host_queue_size = ndev->qlen;
+ nitrox_write_csr(ndev, offset, qsize.value);
+
+ /* set command completion threshold */
+ offset = AQMQ_CMP_THRX(ring);
+ cmp_thr.value = 0;
+ cmp_thr.commands_completed_threshold = 1;
+ nitrox_write_csr(ndev, offset, cmp_thr.value);
+
+ /* step 6: enable the queue */
+ enable_aqm_ring(ndev, ring);
+ }
+}
+
+static void enable_aqm_interrupts(struct nitrox_device *ndev)
+{
+ /* clear interrupt enable bits */
+ nitrox_write_csr(ndev, AQM_DBELL_OVF_LO_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, AQM_DBELL_OVF_HI_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, AQM_DMA_RD_ERR_LO_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, AQM_DMA_RD_ERR_HI_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, AQM_EXEC_NA_LO_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, AQM_EXEC_NA_HI_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, AQM_EXEC_ERR_LO_ENA_W1S, (~0ULL));
+ nitrox_write_csr(ndev, AQM_EXEC_ERR_HI_ENA_W1S, (~0ULL));
+}
+
+void nitrox_config_aqm_unit(struct nitrox_device *ndev)
+{
+ /* config aqm command queues */
+ nitrox_config_aqm_rings(ndev);
+
+ /* enable aqm interrupts */
+ enable_aqm_interrupts(ndev);
}
void nitrox_config_pom_unit(struct nitrox_device *ndev)
diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.h b/drivers/crypto/cavium/nitrox/nitrox_hal.h
index d6606418ba38..48b0af039099 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_hal.h
+++ b/drivers/crypto/cavium/nitrox/nitrox_hal.h
@@ -4,10 +4,13 @@
#include "nitrox_dev.h"
+void nitrox_config_aqm_rings(struct nitrox_device *ndev);
+void nitrox_config_aqm_unit(struct nitrox_device *ndev);
void nitrox_config_emu_unit(struct nitrox_device *ndev);
void nitrox_config_pkt_input_rings(struct nitrox_device *ndev);
void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev);
-void nitrox_config_nps_unit(struct nitrox_device *ndev);
+void nitrox_config_nps_core_unit(struct nitrox_device *ndev);
+void nitrox_config_nps_pkt_unit(struct nitrox_device *ndev);
void nitrox_config_pom_unit(struct nitrox_device *ndev);
void nitrox_config_rand_unit(struct nitrox_device *ndev);
void nitrox_config_efl_unit(struct nitrox_device *ndev);
@@ -15,6 +18,7 @@ void nitrox_config_bmi_unit(struct nitrox_device *ndev);
void nitrox_config_bmo_unit(struct nitrox_device *ndev);
void nitrox_config_lbc_unit(struct nitrox_device *ndev);
void invalidate_lbc(struct nitrox_device *ndev);
+void enable_aqm_ring(struct nitrox_device *ndev, int qno);
void enable_pkt_input_ring(struct nitrox_device *ndev, int ring);
void enable_pkt_solicit_port(struct nitrox_device *ndev, int port);
void config_nps_core_vfcfg_mode(struct nitrox_device *ndev, enum vf_mode mode);
diff --git a/drivers/crypto/cavium/nitrox/nitrox_lib.c b/drivers/crypto/cavium/nitrox/nitrox_lib.c
index 4ace9bcd603a..5cbc64b851b9 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_lib.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_lib.c
@@ -19,6 +19,8 @@
/* packet inuput ring alignments */
#define PKTIN_Q_ALIGN_BYTES 16
+/* AQM Queue input alignments */
+#define AQM_Q_ALIGN_BYTES 32
static int nitrox_cmdq_init(struct nitrox_cmdq *cmdq, int align_bytes)
{
@@ -57,11 +59,15 @@ static void nitrox_cmdq_reset(struct nitrox_cmdq *cmdq)
static void nitrox_cmdq_cleanup(struct nitrox_cmdq *cmdq)
{
- struct nitrox_device *ndev = cmdq->ndev;
+ struct nitrox_device *ndev;
+
+ if (!cmdq)
+ return;
if (!cmdq->unalign_base)
return;
+ ndev = cmdq->ndev;
cancel_work_sync(&cmdq->backlog_qflush);
dma_free_coherent(DEV(ndev), cmdq->qsize,
@@ -78,6 +84,57 @@ static void nitrox_cmdq_cleanup(struct nitrox_cmdq *cmdq)
cmdq->instr_size = 0;
}
+static void nitrox_free_aqm_queues(struct nitrox_device *ndev)
+{
+ int i;
+
+ for (i = 0; i < ndev->nr_queues; i++) {
+ nitrox_cmdq_cleanup(ndev->aqmq[i]);
+ kzfree(ndev->aqmq[i]);
+ ndev->aqmq[i] = NULL;
+ }
+}
+
+static int nitrox_alloc_aqm_queues(struct nitrox_device *ndev)
+{
+ int i, err;
+
+ for (i = 0; i < ndev->nr_queues; i++) {
+ struct nitrox_cmdq *cmdq;
+ u64 offset;
+
+ cmdq = kzalloc_node(sizeof(*cmdq), GFP_KERNEL, ndev->node);
+ if (!cmdq) {
+ err = -ENOMEM;
+ goto aqmq_fail;
+ }
+
+ cmdq->ndev = ndev;
+ cmdq->qno = i;
+ cmdq->instr_size = sizeof(struct aqmq_command_s);
+
+ /* AQM Queue Doorbell Counter Register Address */
+ offset = AQMQ_DRBLX(i);
+ cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset);
+ /* AQM Queue Commands Completed Count Register Address */
+ offset = AQMQ_CMD_CNTX(i);
+ cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset);
+
+ err = nitrox_cmdq_init(cmdq, AQM_Q_ALIGN_BYTES);
+ if (err) {
+ kzfree(cmdq);
+ goto aqmq_fail;
+ }
+ ndev->aqmq[i] = cmdq;
+ }
+
+ return 0;
+
+aqmq_fail:
+ nitrox_free_aqm_queues(ndev);
+ return err;
+}
+
static void nitrox_free_pktin_queues(struct nitrox_device *ndev)
{
int i;
@@ -222,6 +279,12 @@ int nitrox_common_sw_init(struct nitrox_device *ndev)
if (err)
destroy_crypto_dma_pool(ndev);
+ err = nitrox_alloc_aqm_queues(ndev);
+ if (err) {
+ nitrox_free_pktin_queues(ndev);
+ destroy_crypto_dma_pool(ndev);
+ }
+
return err;
}
@@ -231,6 +294,7 @@ int nitrox_common_sw_init(struct nitrox_device *ndev)
*/
void nitrox_common_sw_cleanup(struct nitrox_device *ndev)
{
+ nitrox_free_aqm_queues(ndev);
nitrox_free_pktin_queues(ndev);
destroy_crypto_dma_pool(ndev);
}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c
index fe825d0ef9ca..bc924980e10c 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_main.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_main.c
@@ -17,12 +17,17 @@
#define CNN55XX_DEV_ID 0x12
#define UCODE_HLEN 48
-#define SE_GROUP 0
+#define DEFAULT_SE_GROUP 0
+#define DEFAULT_AE_GROUP 0
-#define DRIVER_VERSION "1.1"
+#define DRIVER_VERSION "1.2"
+#define CNN55XX_UCD_BLOCK_SIZE 32768
+#define CNN55XX_MAX_UCODE_SIZE (CNN55XX_UCD_BLOCK_SIZE * 2)
#define FW_DIR "cavium/"
/* SE microcode */
#define SE_FW FW_DIR "cnn55xx_se.fw"
+/* AE microcode */
+#define AE_FW FW_DIR "cnn55xx_ae.fw"
static const char nitrox_driver_name[] = "CNN55XX";
@@ -72,10 +77,10 @@ struct ucode {
/**
* write_to_ucd_unit - Write Firmware to NITROX UCD unit
*/
-static void write_to_ucd_unit(struct nitrox_device *ndev,
- struct ucode *ucode)
+static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size,
+ u64 *ucode_data, int block_num)
{
- u32 code_size = be32_to_cpu(ucode->code_size) * 2;
+ u32 code_size;
u64 offset, data;
int i = 0;
@@ -96,11 +101,12 @@ static void write_to_ucd_unit(struct nitrox_device *ndev,
/* set the block number */
offset = UCD_UCODE_LOAD_BLOCK_NUM;
- nitrox_write_csr(ndev, offset, 0);
+ nitrox_write_csr(ndev, offset, block_num);
+ code_size = ucode_size;
code_size = roundup(code_size, 8);
while (code_size) {
- data = ucode->code[i];
+ data = ucode_data[i];
/* write 8 bytes at a time */
offset = UCD_UCODE_LOAD_IDX_DATAX(i);
nitrox_write_csr(ndev, offset, data);
@@ -108,29 +114,74 @@ static void write_to_ucd_unit(struct nitrox_device *ndev,
i++;
}
- /* put all SE cores in group 0 */
- offset = POM_GRP_EXECMASKX(SE_GROUP);
- nitrox_write_csr(ndev, offset, (~0ULL));
-
- for (i = 0; i < ndev->hw.se_cores; i++) {
- /*
- * write block number and firware length
- * bit:<2:0> block number
- * bit:3 is set SE uses 32KB microcode
- * bit:3 is clear SE uses 64KB microcode
- */
- offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
- nitrox_write_csr(ndev, offset, 0x8);
- }
usleep_range(300, 400);
}
-static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
+static int nitrox_load_fw(struct nitrox_device *ndev)
{
const struct firmware *fw;
+ const char *fw_name;
struct ucode *ucode;
- int ret;
+ u64 *ucode_data;
+ u64 offset;
+ union ucd_core_eid_ucode_block_num core_2_eid_val;
+ union aqm_grp_execmsk_lo aqm_grp_execmask_lo;
+ union aqm_grp_execmsk_hi aqm_grp_execmask_hi;
+ u32 ucode_size;
+ int ret, i = 0;
+
+ fw_name = SE_FW;
+ dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
+
+ ret = request_firmware(&fw, fw_name, DEV(ndev));
+ if (ret < 0) {
+ dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
+ return ret;
+ }
+
+ ucode = (struct ucode *)fw->data;
+
+ ucode_size = be32_to_cpu(ucode->code_size) * 2;
+ if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
+ dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
+ ucode_size, fw_name);
+ release_firmware(fw);
+ return -EINVAL;
+ }
+ ucode_data = ucode->code;
+
+ /* copy the firmware version */
+ memcpy(&ndev->hw.fw_name[0][0], ucode->version, (VERSION_LEN - 2));
+ ndev->hw.fw_name[0][VERSION_LEN - 1] = '\0';
+
+ /* Load SE Firmware on UCD Block 0 */
+ write_to_ucd_unit(ndev, ucode_size, ucode_data, 0);
+ release_firmware(fw);
+
+ /* put all SE cores in DEFAULT_SE_GROUP */
+ offset = POM_GRP_EXECMASKX(DEFAULT_SE_GROUP);
+ nitrox_write_csr(ndev, offset, (~0ULL));
+
+ /* write block number and firmware length
+ * bit:<2:0> block number
+ * bit:3 is set SE uses 32KB microcode
+ * bit:3 is clear SE uses 64KB microcode
+ */
+ core_2_eid_val.value = 0ULL;
+ core_2_eid_val.ucode_blk = 0;
+ if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
+ core_2_eid_val.ucode_len = 1;
+ else
+ core_2_eid_val.ucode_len = 0;
+
+ for (i = 0; i < ndev->hw.se_cores; i++) {
+ offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
+ nitrox_write_csr(ndev, offset, core_2_eid_val.value);
+ }
+
+
+ fw_name = AE_FW;
dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
ret = request_firmware(&fw, fw_name, DEV(ndev));
@@ -140,13 +191,50 @@ static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
}
ucode = (struct ucode *)fw->data;
+
+ ucode_size = be32_to_cpu(ucode->code_size) * 2;
+ if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
+ dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
+ ucode_size, fw_name);
+ release_firmware(fw);
+ return -EINVAL;
+ }
+ ucode_data = ucode->code;
+
/* copy the firmware version */
- memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
- ndev->hw.fw_name[VERSION_LEN - 1] = '\0';
+ memcpy(&ndev->hw.fw_name[1][0], ucode->version, (VERSION_LEN - 2));
+ ndev->hw.fw_name[1][VERSION_LEN - 1] = '\0';
+
+ /* Load AE Firmware on UCD Block 2 */
+ write_to_ucd_unit(ndev, ucode_size, ucode_data, 2);
- write_to_ucd_unit(ndev, ucode);
release_firmware(fw);
+ /* put all AE cores in DEFAULT_AE_GROUP */
+ offset = AQM_GRP_EXECMSK_LOX(DEFAULT_AE_GROUP);
+ aqm_grp_execmask_lo.exec_0_to_39 = 0xFFFFFFFFFFULL;
+ nitrox_write_csr(ndev, offset, aqm_grp_execmask_lo.value);
+ offset = AQM_GRP_EXECMSK_HIX(DEFAULT_AE_GROUP);
+ aqm_grp_execmask_hi.exec_40_to_79 = 0xFFFFFFFFFFULL;
+ nitrox_write_csr(ndev, offset, aqm_grp_execmask_hi.value);
+
+ /* write block number and firmware length
+ * bit:<2:0> block number
+ * bit:3 is set SE uses 32KB microcode
+ * bit:3 is clear SE uses 64KB microcode
+ */
+ core_2_eid_val.value = 0ULL;
+ core_2_eid_val.ucode_blk = 0;
+ if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
+ core_2_eid_val.ucode_len = 1;
+ else
+ core_2_eid_val.ucode_len = 0;
+
+ for (i = 0; i < ndev->hw.ae_cores; i++) {
+ offset = UCD_AE_EID_UCODE_BLOCK_NUMX(i);
+ nitrox_write_csr(ndev, offset, core_2_eid_val.value);
+ }
+
return 0;
}
@@ -299,7 +387,9 @@ static int nitrox_pf_hw_init(struct nitrox_device *ndev)
/* get cores information */
nitrox_get_hwinfo(ndev);
- nitrox_config_nps_unit(ndev);
+ nitrox_config_nps_core_unit(ndev);
+ nitrox_config_aqm_unit(ndev);
+ nitrox_config_nps_pkt_unit(ndev);
nitrox_config_pom_unit(ndev);
nitrox_config_efl_unit(ndev);
/* configure IO units */
@@ -309,8 +399,8 @@ static int nitrox_pf_hw_init(struct nitrox_device *ndev)
nitrox_config_lbc_unit(ndev);
nitrox_config_rand_unit(ndev);
- /* load firmware on SE cores */
- err = nitrox_load_fw(ndev, SE_FW);
+ /* load firmware on cores */
+ err = nitrox_load_fw(ndev);
if (err)
return err;
diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.h b/drivers/crypto/cavium/nitrox/nitrox_mbx.h
index 5008399775a9..7c93d0282174 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_mbx.h
+++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NITROX_MBX_H
#define __NITROX_MBX_H
diff --git a/drivers/crypto/cavium/nitrox/nitrox_req.h b/drivers/crypto/cavium/nitrox/nitrox_req.h
index efdbd0fc3e3b..f69ba02c4d25 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_req.h
+++ b/drivers/crypto/cavium/nitrox/nitrox_req.h
@@ -400,6 +400,36 @@ struct nps_pkt_instr {
};
/**
+ * struct aqmq_command_s - The 32 byte command for AE processing.
+ * @opcode: Request opcode
+ * @param1: Request control parameter 1
+ * @param2: Request control parameter 2
+ * @dlen: Input length
+ * @dptr: Input pointer points to buffer in remote host
+ * @rptr: Result pointer points to buffer in remote host
+ * @grp: AQM Group (0..7)
+ * @cptr: Context pointer
+ */
+struct aqmq_command_s {
+ __be16 opcode;
+ __be16 param1;
+ __be16 param2;
+ __be16 dlen;
+ __be64 dptr;
+ __be64 rptr;
+ union {
+ __be64 word3;
+#if defined(__BIG_ENDIAN_BITFIELD)
+ u64 grp : 3;
+ u64 cptr : 61;
+#else
+ u64 cptr : 61;
+ u64 grp : 3;
+#endif
+ };
+};
+
+/**
* struct ctx_hdr - Book keeping data about the crypto context
* @pool: Pool used to allocate crypto context
* @dma: Base DMA address of the cypto context
diff --git a/drivers/crypto/cavium/nitrox/nitrox_skcipher.c b/drivers/crypto/cavium/nitrox/nitrox_skcipher.c
index 7e4a5e69085e..3cdce1f0f257 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_skcipher.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_skcipher.c
@@ -7,7 +7,7 @@
#include <crypto/aes.h>
#include <crypto/skcipher.h>
#include <crypto/ctr.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <crypto/xts.h>
#include "nitrox_dev.h"
@@ -257,7 +257,7 @@ static int nitrox_aes_decrypt(struct skcipher_request *skreq)
static int nitrox_3des_setkey(struct crypto_skcipher *cipher,
const u8 *key, unsigned int keylen)
{
- return unlikely(des3_verify_key(cipher, key)) ?:
+ return verify_skcipher_des3_key(cipher, key) ?:
nitrox_skcipher_setkey(cipher, 0, key, keylen);
}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_sriov.c b/drivers/crypto/cavium/nitrox/nitrox_sriov.c
index bf439d8256ba..43287f8471d1 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_sriov.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_sriov.c
@@ -109,6 +109,9 @@ static int nitrox_pf_reinit(struct nitrox_device *ndev)
return err;
}
+ /* configure the AQM queues */
+ nitrox_config_aqm_rings(ndev);
+
/* configure the packet queues */
nitrox_config_pkt_input_rings(ndev);
nitrox_config_pkt_solicit_ports(ndev);
diff --git a/drivers/crypto/cavium/zip/zip_main.c b/drivers/crypto/cavium/zip/zip_main.c
index a8447a3cf366..194624b4855b 100644
--- a/drivers/crypto/cavium/zip/zip_main.c
+++ b/drivers/crypto/cavium/zip/zip_main.c
@@ -593,6 +593,7 @@ static const struct file_operations zip_stats_fops = {
.owner = THIS_MODULE,
.open = zip_stats_open,
.read = seq_read,
+ .release = single_release,
};
static int zip_clear_open(struct inode *inode, struct file *file)
@@ -604,6 +605,7 @@ static const struct file_operations zip_clear_fops = {
.owner = THIS_MODULE,
.open = zip_clear_open,
.read = seq_read,
+ .release = single_release,
};
static int zip_regs_open(struct inode *inode, struct file *file)
@@ -615,6 +617,7 @@ static const struct file_operations zip_regs_fops = {
.owner = THIS_MODULE,
.open = zip_regs_open,
.read = seq_read,
+ .release = single_release,
};
/* Root directory for thunderx_zip debugfs entry */
diff --git a/drivers/crypto/ccp/Kconfig b/drivers/crypto/ccp/Kconfig
index 48f3edc1e3fb..8fec733f567f 100644
--- a/drivers/crypto/ccp/Kconfig
+++ b/drivers/crypto/ccp/Kconfig
@@ -30,6 +30,7 @@ config CRYPTO_DEV_CCP_CRYPTO
select CRYPTO_BLKCIPHER
select CRYPTO_AUTHENC
select CRYPTO_RSA
+ select CRYPTO_LIB_AES
help
Support for using the cryptographic API with the AMD Cryptographic
Coprocessor. This module supports offload of SHA and AES algorithms.
@@ -45,3 +46,11 @@ config CRYPTO_DEV_SP_PSP
management commands in Secure Encrypted Virtualization (SEV) mode,
along with software-based Trusted Execution Environment (TEE) to
enable third-party trusted applications.
+
+config CRYPTO_DEV_CCP_DEBUGFS
+ bool "Enable CCP Internals in DebugFS"
+ default n
+ depends on CRYPTO_DEV_SP_CCP
+ help
+ Expose CCP device information such as operation statistics, feature
+ information, and descriptor queue contents.
diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile
index 51d1c0cf66c7..6b86f1e6d634 100644
--- a/drivers/crypto/ccp/Makefile
+++ b/drivers/crypto/ccp/Makefile
@@ -5,8 +5,8 @@ ccp-$(CONFIG_CRYPTO_DEV_SP_CCP) += ccp-dev.o \
ccp-ops.o \
ccp-dev-v3.o \
ccp-dev-v5.o \
- ccp-dmaengine.o \
- ccp-debugfs.o
+ ccp-dmaengine.o
+ccp-$(CONFIG_CRYPTO_DEV_CCP_DEBUGFS) += ccp-debugfs.o
ccp-$(CONFIG_PCI) += sp-pci.o
ccp-$(CONFIG_CRYPTO_DEV_SP_PSP) += psp-dev.o
diff --git a/drivers/crypto/ccp/ccp-crypto-aes-cmac.c b/drivers/crypto/ccp/ccp-crypto-aes-cmac.c
index bb7219d36b2c..32f19f402073 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes-cmac.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes-cmac.c
@@ -261,6 +261,7 @@ static int ccp_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
ccp_crypto_ahash_alg(crypto_ahash_tfm(tfm));
u64 k0_hi, k0_lo, k1_hi, k1_lo, k2_hi, k2_lo;
u64 rb_hi = 0x00, rb_lo = 0x87;
+ struct crypto_aes_ctx aes;
__be64 *gk;
int ret;
@@ -284,14 +285,14 @@ static int ccp_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
ctx->u.aes.key_len = 0;
/* Set the key for the AES cipher used to generate the keys */
- ret = crypto_cipher_setkey(ctx->u.aes.tfm_cipher, key, key_len);
+ ret = aes_expandkey(&aes, key, key_len);
if (ret)
return ret;
/* Encrypt a block of zeroes - use key area in context */
memset(ctx->u.aes.key, 0, sizeof(ctx->u.aes.key));
- crypto_cipher_encrypt_one(ctx->u.aes.tfm_cipher, ctx->u.aes.key,
- ctx->u.aes.key);
+ aes_encrypt(&aes, ctx->u.aes.key, ctx->u.aes.key);
+ memzero_explicit(&aes, sizeof(aes));
/* Generate K1 and K2 */
k0_hi = be64_to_cpu(*((__be64 *)ctx->u.aes.key));
@@ -336,32 +337,15 @@ static int ccp_aes_cmac_cra_init(struct crypto_tfm *tfm)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
- struct crypto_cipher *cipher_tfm;
ctx->complete = ccp_aes_cmac_complete;
ctx->u.aes.key_len = 0;
crypto_ahash_set_reqsize(ahash, sizeof(struct ccp_aes_cmac_req_ctx));
- cipher_tfm = crypto_alloc_cipher("aes", 0, CRYPTO_ALG_NEED_FALLBACK);
- if (IS_ERR(cipher_tfm)) {
- pr_warn("could not load aes cipher driver\n");
- return PTR_ERR(cipher_tfm);
- }
- ctx->u.aes.tfm_cipher = cipher_tfm;
-
return 0;
}
-static void ccp_aes_cmac_cra_exit(struct crypto_tfm *tfm)
-{
- struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
-
- if (ctx->u.aes.tfm_cipher)
- crypto_free_cipher(ctx->u.aes.tfm_cipher);
- ctx->u.aes.tfm_cipher = NULL;
-}
-
int ccp_register_aes_cmac_algs(struct list_head *head)
{
struct ccp_crypto_ahash_alg *ccp_alg;
@@ -401,7 +385,6 @@ int ccp_register_aes_cmac_algs(struct list_head *head)
base->cra_ctxsize = sizeof(struct ccp_ctx);
base->cra_priority = CCP_CRA_PRIORITY;
base->cra_init = ccp_aes_cmac_cra_init;
- base->cra_exit = ccp_aes_cmac_cra_exit;
base->cra_module = THIS_MODULE;
ret = crypto_register_ahash(alg);
diff --git a/drivers/crypto/ccp/ccp-crypto-aes-galois.c b/drivers/crypto/ccp/ccp-crypto-aes-galois.c
index f9fec2ddf56a..94c1ad7eeddf 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes-galois.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes-galois.c
@@ -58,6 +58,19 @@ static int ccp_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
static int ccp_aes_gcm_setauthsize(struct crypto_aead *tfm,
unsigned int authsize)
{
+ switch (authsize) {
+ case 16:
+ case 15:
+ case 14:
+ case 13:
+ case 12:
+ case 8:
+ case 4:
+ break;
+ default:
+ return -EINVAL;
+ }
+
return 0;
}
@@ -104,6 +117,7 @@ static int ccp_aes_gcm_crypt(struct aead_request *req, bool encrypt)
memset(&rctx->cmd, 0, sizeof(rctx->cmd));
INIT_LIST_HEAD(&rctx->cmd.entry);
rctx->cmd.engine = CCP_ENGINE_AES;
+ rctx->cmd.u.aes.authsize = crypto_aead_authsize(tfm);
rctx->cmd.u.aes.type = ctx->u.aes.type;
rctx->cmd.u.aes.mode = ctx->u.aes.mode;
rctx->cmd.u.aes.action = encrypt;
diff --git a/drivers/crypto/ccp/ccp-crypto-aes-xts.c b/drivers/crypto/ccp/ccp-crypto-aes-xts.c
index 783ba75e0618..8e4a531f4f70 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes-xts.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes-xts.c
@@ -116,9 +116,6 @@ static int ccp_aes_xts_crypt(struct ablkcipher_request *req,
if (!ctx->u.aes.key_len)
return -EINVAL;
- if (req->nbytes & (AES_BLOCK_SIZE - 1))
- return -EINVAL;
-
if (!req->info)
return -EINVAL;
diff --git a/drivers/crypto/ccp/ccp-crypto-aes.c b/drivers/crypto/ccp/ccp-crypto-aes.c
index ea3d6de55ff6..58c6dddfc5e1 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes.c
@@ -2,7 +2,7 @@
/*
* AMD Cryptographic Coprocessor (CCP) AES crypto API support
*
- * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
+ * Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
*
* Author: Tom Lendacky <thomas.lendacky@amd.com>
*/
@@ -76,8 +76,7 @@ static int ccp_aes_crypt(struct ablkcipher_request *req, bool encrypt)
return -EINVAL;
if (((ctx->u.aes.mode == CCP_AES_MODE_ECB) ||
- (ctx->u.aes.mode == CCP_AES_MODE_CBC) ||
- (ctx->u.aes.mode == CCP_AES_MODE_CFB)) &&
+ (ctx->u.aes.mode == CCP_AES_MODE_CBC)) &&
(req->nbytes & (AES_BLOCK_SIZE - 1)))
return -EINVAL;
@@ -288,7 +287,7 @@ static struct ccp_aes_def aes_algs[] = {
.version = CCP_VERSION(3, 0),
.name = "cfb(aes)",
.driver_name = "cfb-aes-ccp",
- .blocksize = AES_BLOCK_SIZE,
+ .blocksize = 1,
.ivsize = AES_BLOCK_SIZE,
.alg_defaults = &ccp_aes_defaults,
},
diff --git a/drivers/crypto/ccp/ccp-crypto-des3.c b/drivers/crypto/ccp/ccp-crypto-des3.c
index 5f05f834c7cd..d2c49b2f0323 100644
--- a/drivers/crypto/ccp/ccp-crypto-des3.c
+++ b/drivers/crypto/ccp/ccp-crypto-des3.c
@@ -14,7 +14,7 @@
#include <linux/crypto.h>
#include <crypto/algapi.h>
#include <crypto/scatterwalk.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include "ccp-crypto.h"
@@ -39,11 +39,10 @@ static int ccp_des3_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ablkcipher_tfm(tfm));
struct ccp_crypto_ablkcipher_alg *alg =
ccp_crypto_ablkcipher_alg(crypto_ablkcipher_tfm(tfm));
- u32 *flags = &tfm->base.crt_flags;
int err;
- err = __des3_verify_key(flags, key);
- if (unlikely(err))
+ err = verify_ablkcipher_des3_key(tfm, key);
+ if (err)
return err;
/* It's not clear that there is any support for a keysize of 112.
diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c
index 818096490829..8ee4cb45a3f3 100644
--- a/drivers/crypto/ccp/ccp-crypto-main.c
+++ b/drivers/crypto/ccp/ccp-crypto-main.c
@@ -405,8 +405,10 @@ static int ccp_crypto_init(void)
int ret;
ret = ccp_present();
- if (ret)
+ if (ret) {
+ pr_err("Cannot load: there are no available CCPs\n");
return ret;
+ }
spin_lock_init(&req_queue_lock);
INIT_LIST_HEAD(&req_queue.cmds);
diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h
index 622b34c17643..9015b5da6ba3 100644
--- a/drivers/crypto/ccp/ccp-crypto.h
+++ b/drivers/crypto/ccp/ccp-crypto.h
@@ -12,7 +12,6 @@
#include <linux/list.h>
#include <linux/wait.h>
-#include <linux/pci.h>
#include <linux/ccp.h>
#include <crypto/algapi.h>
#include <crypto/aes.h>
@@ -24,6 +23,10 @@
#include <crypto/akcipher.h>
#include <crypto/internal/rsa.h>
+/* We want the module name in front of our messages */
+#undef pr_fmt
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
#define CCP_LOG_LEVEL KERN_INFO
#define CCP_CRA_PRIORITY 300
@@ -87,9 +90,6 @@ struct ccp_aes_ctx {
/* Fallback cipher for XTS with unsupported unit sizes */
struct crypto_sync_skcipher *tfm_skcipher;
- /* Cipher used to generate CMAC K1/K2 keys */
- struct crypto_cipher *tfm_cipher;
-
enum ccp_engine engine;
enum ccp_aes_type type;
enum ccp_aes_mode mode;
diff --git a/drivers/crypto/ccp/ccp-dev-v3.c b/drivers/crypto/ccp/ccp-dev-v3.c
index 2b7d47ed5c74..0186b3df4c87 100644
--- a/drivers/crypto/ccp/ccp-dev-v3.c
+++ b/drivers/crypto/ccp/ccp-dev-v3.c
@@ -10,7 +10,6 @@
#include <linux/module.h>
#include <linux/kernel.h>
-#include <linux/pci.h>
#include <linux/kthread.h>
#include <linux/interrupt.h>
#include <linux/ccp.h>
@@ -379,7 +378,7 @@ static int ccp_init(struct ccp_device *ccp)
/* Find available queues */
ccp->qim = 0;
qmr = ioread32(ccp->io_regs + Q_MASK_REG);
- for (i = 0; i < MAX_HW_QUEUES; i++) {
+ for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) {
if (!(qmr & (1 << i)))
continue;
diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c
index 217e41bbadaf..57eb53b8ac21 100644
--- a/drivers/crypto/ccp/ccp-dev-v5.c
+++ b/drivers/crypto/ccp/ccp-dev-v5.c
@@ -2,16 +2,13 @@
/*
* AMD Cryptographic Coprocessor (CCP) driver
*
- * Copyright (C) 2016,2017 Advanced Micro Devices, Inc.
+ * Copyright (C) 2016,2019 Advanced Micro Devices, Inc.
*
* Author: Gary R Hook <gary.hook@amd.com>
*/
-#include <linux/module.h>
#include <linux/kernel.h>
-#include <linux/pci.h>
#include <linux/kthread.h>
-#include <linux/debugfs.h>
#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <linux/compiler.h>
@@ -792,8 +789,7 @@ static int ccp5_init(struct ccp_device *ccp)
/* Find available queues */
qmr = ioread32(ccp->io_regs + Q_MASK_REG);
- for (i = 0; i < MAX_HW_QUEUES; i++) {
-
+ for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) {
if (!(qmr & (1 << i)))
continue;
@@ -806,6 +802,7 @@ static int ccp5_init(struct ccp_device *ccp)
if (!dma_pool) {
dev_err(dev, "unable to allocate dma pool\n");
ret = -ENOMEM;
+ goto e_pool;
}
cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
@@ -819,9 +816,9 @@ static int ccp5_init(struct ccp_device *ccp)
/* Page alignment satisfies our needs for N <= 128 */
BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128);
cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
- cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize,
- &cmd_q->qbase_dma,
- GFP_KERNEL);
+ cmd_q->qbase = dmam_alloc_coherent(dev, cmd_q->qsize,
+ &cmd_q->qbase_dma,
+ GFP_KERNEL);
if (!cmd_q->qbase) {
dev_err(dev, "unable to allocate command queue\n");
ret = -ENOMEM;
@@ -970,8 +967,10 @@ static int ccp5_init(struct ccp_device *ccp)
if (ret)
goto e_hwrng;
+#ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS
/* Set up debugfs entries */
ccp5_debugfs_setup(ccp);
+#endif
return 0;
@@ -995,7 +994,6 @@ e_pool:
static void ccp5_destroy(struct ccp_device *ccp)
{
- struct device *dev = ccp->dev;
struct ccp_cmd_queue *cmd_q;
struct ccp_cmd *cmd;
unsigned int i;
@@ -1009,11 +1007,13 @@ static void ccp5_destroy(struct ccp_device *ccp)
/* Remove this device from the list of available units first */
ccp_del_device(ccp);
+#ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS
/* We're in the process of tearing down the entire driver;
* when all the devices are gone clean up debugfs
*/
if (ccp_present())
ccp5_debugfs_destroy();
+#endif
/* Disable and clear interrupts */
ccp5_disable_queue_interrupts(ccp);
@@ -1036,12 +1036,6 @@ static void ccp5_destroy(struct ccp_device *ccp)
sp_free_ccp_irq(ccp->sp, ccp);
- for (i = 0; i < ccp->cmd_q_count; i++) {
- cmd_q = &ccp->cmd_q[i];
- dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,
- cmd_q->qbase_dma);
- }
-
/* Flush the cmd and backlog queue */
while (!list_empty(&ccp->cmd)) {
/* Invoke the callback directly with an error code */
diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c
index cc3e96c4f5fb..73acf0fdb793 100644
--- a/drivers/crypto/ccp/ccp-dev.c
+++ b/drivers/crypto/ccp/ccp-dev.c
@@ -2,12 +2,13 @@
/*
* AMD Cryptographic Coprocessor (CCP) driver
*
- * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
+ * Copyright (C) 2013,2019 Advanced Micro Devices, Inc.
*
* Author: Tom Lendacky <thomas.lendacky@amd.com>
* Author: Gary R Hook <gary.hook@amd.com>
*/
+#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/kthread.h>
#include <linux/sched.h>
@@ -19,6 +20,7 @@
#include <linux/delay.h>
#include <linux/hw_random.h>
#include <linux/cpu.h>
+#include <linux/atomic.h>
#ifdef CONFIG_X86
#include <asm/cpu_device_id.h>
#endif
@@ -26,62 +28,81 @@
#include "ccp-dev.h"
+#define MAX_CCPS 32
+
+/* Limit CCP use to a specifed number of queues per device */
+static unsigned int nqueues = 0;
+module_param(nqueues, uint, 0444);
+MODULE_PARM_DESC(nqueues, "Number of queues per CCP (minimum 1; default: all available)");
+
+/* Limit the maximum number of configured CCPs */
+static atomic_t dev_count = ATOMIC_INIT(0);
+static unsigned int max_devs = MAX_CCPS;
+module_param(max_devs, uint, 0444);
+MODULE_PARM_DESC(max_devs, "Maximum number of CCPs to enable (default: all; 0 disables all CCPs)");
+
struct ccp_tasklet_data {
struct completion completion;
struct ccp_cmd *cmd;
};
/* Human-readable error strings */
+#define CCP_MAX_ERROR_CODE 64
static char *ccp_error_codes[] = {
"",
- "ERR 01: ILLEGAL_ENGINE",
- "ERR 02: ILLEGAL_KEY_ID",
- "ERR 03: ILLEGAL_FUNCTION_TYPE",
- "ERR 04: ILLEGAL_FUNCTION_MODE",
- "ERR 05: ILLEGAL_FUNCTION_ENCRYPT",
- "ERR 06: ILLEGAL_FUNCTION_SIZE",
- "ERR 07: Zlib_MISSING_INIT_EOM",
- "ERR 08: ILLEGAL_FUNCTION_RSVD",
- "ERR 09: ILLEGAL_BUFFER_LENGTH",
- "ERR 10: VLSB_FAULT",
- "ERR 11: ILLEGAL_MEM_ADDR",
- "ERR 12: ILLEGAL_MEM_SEL",
- "ERR 13: ILLEGAL_CONTEXT_ID",
- "ERR 14: ILLEGAL_KEY_ADDR",
- "ERR 15: 0xF Reserved",
- "ERR 16: Zlib_ILLEGAL_MULTI_QUEUE",
- "ERR 17: Zlib_ILLEGAL_JOBID_CHANGE",
- "ERR 18: CMD_TIMEOUT",
- "ERR 19: IDMA0_AXI_SLVERR",
- "ERR 20: IDMA0_AXI_DECERR",
- "ERR 21: 0x15 Reserved",
- "ERR 22: IDMA1_AXI_SLAVE_FAULT",
- "ERR 23: IDMA1_AIXI_DECERR",
- "ERR 24: 0x18 Reserved",
- "ERR 25: ZLIBVHB_AXI_SLVERR",
- "ERR 26: ZLIBVHB_AXI_DECERR",
- "ERR 27: 0x1B Reserved",
- "ERR 27: ZLIB_UNEXPECTED_EOM",
- "ERR 27: ZLIB_EXTRA_DATA",
- "ERR 30: ZLIB_BTYPE",
- "ERR 31: ZLIB_UNDEFINED_SYMBOL",
- "ERR 32: ZLIB_UNDEFINED_DISTANCE_S",
- "ERR 33: ZLIB_CODE_LENGTH_SYMBOL",
- "ERR 34: ZLIB _VHB_ILLEGAL_FETCH",
- "ERR 35: ZLIB_UNCOMPRESSED_LEN",
- "ERR 36: ZLIB_LIMIT_REACHED",
- "ERR 37: ZLIB_CHECKSUM_MISMATCH0",
- "ERR 38: ODMA0_AXI_SLVERR",
- "ERR 39: ODMA0_AXI_DECERR",
- "ERR 40: 0x28 Reserved",
- "ERR 41: ODMA1_AXI_SLVERR",
- "ERR 42: ODMA1_AXI_DECERR",
- "ERR 43: LSB_PARITY_ERR",
+ "ILLEGAL_ENGINE",
+ "ILLEGAL_KEY_ID",
+ "ILLEGAL_FUNCTION_TYPE",
+ "ILLEGAL_FUNCTION_MODE",
+ "ILLEGAL_FUNCTION_ENCRYPT",
+ "ILLEGAL_FUNCTION_SIZE",
+ "Zlib_MISSING_INIT_EOM",
+ "ILLEGAL_FUNCTION_RSVD",
+ "ILLEGAL_BUFFER_LENGTH",
+ "VLSB_FAULT",
+ "ILLEGAL_MEM_ADDR",
+ "ILLEGAL_MEM_SEL",
+ "ILLEGAL_CONTEXT_ID",
+ "ILLEGAL_KEY_ADDR",
+ "0xF Reserved",
+ "Zlib_ILLEGAL_MULTI_QUEUE",
+ "Zlib_ILLEGAL_JOBID_CHANGE",
+ "CMD_TIMEOUT",
+ "IDMA0_AXI_SLVERR",
+ "IDMA0_AXI_DECERR",
+ "0x15 Reserved",
+ "IDMA1_AXI_SLAVE_FAULT",
+ "IDMA1_AIXI_DECERR",
+ "0x18 Reserved",
+ "ZLIBVHB_AXI_SLVERR",
+ "ZLIBVHB_AXI_DECERR",
+ "0x1B Reserved",
+ "ZLIB_UNEXPECTED_EOM",
+ "ZLIB_EXTRA_DATA",
+ "ZLIB_BTYPE",
+ "ZLIB_UNDEFINED_SYMBOL",
+ "ZLIB_UNDEFINED_DISTANCE_S",
+ "ZLIB_CODE_LENGTH_SYMBOL",
+ "ZLIB _VHB_ILLEGAL_FETCH",
+ "ZLIB_UNCOMPRESSED_LEN",
+ "ZLIB_LIMIT_REACHED",
+ "ZLIB_CHECKSUM_MISMATCH0",
+ "ODMA0_AXI_SLVERR",
+ "ODMA0_AXI_DECERR",
+ "0x28 Reserved",
+ "ODMA1_AXI_SLVERR",
+ "ODMA1_AXI_DECERR",
};
-void ccp_log_error(struct ccp_device *d, int e)
+void ccp_log_error(struct ccp_device *d, unsigned int e)
{
- dev_err(d->dev, "CCP error: %s (0x%x)\n", ccp_error_codes[e], e);
+ if (WARN_ON(e >= CCP_MAX_ERROR_CODE))
+ return;
+
+ if (e < ARRAY_SIZE(ccp_error_codes))
+ dev_err(d->dev, "CCP error %d: %s\n", e, ccp_error_codes[e]);
+ else
+ dev_err(d->dev, "CCP error %d: Unknown Error\n", e);
}
/* List of CCPs, CCP count, read-write access lock, and access functions
@@ -534,6 +555,10 @@ int ccp_dev_suspend(struct sp_device *sp, pm_message_t state)
unsigned long flags;
unsigned int i;
+ /* If there's no device there's nothing to do */
+ if (!ccp)
+ return 0;
+
spin_lock_irqsave(&ccp->cmd_lock, flags);
ccp->suspending = 1;
@@ -558,6 +583,10 @@ int ccp_dev_resume(struct sp_device *sp)
unsigned long flags;
unsigned int i;
+ /* If there's no device there's nothing to do */
+ if (!ccp)
+ return 0;
+
spin_lock_irqsave(&ccp->cmd_lock, flags);
ccp->suspending = 0;
@@ -580,12 +609,24 @@ int ccp_dev_init(struct sp_device *sp)
struct ccp_device *ccp;
int ret;
+ /*
+ * Check how many we have so far, and stop after reaching
+ * that number
+ */
+ if (atomic_inc_return(&dev_count) > max_devs)
+ return 0; /* don't fail the load */
+
ret = -ENOMEM;
ccp = ccp_alloc_struct(sp);
if (!ccp)
goto e_err;
sp->ccp_data = ccp;
+ if (!nqueues || (nqueues > MAX_HW_QUEUES))
+ ccp->max_q_count = MAX_HW_QUEUES;
+ else
+ ccp->max_q_count = nqueues;
+
ccp->vdata = (struct ccp_vdata *)sp->dev_vdata->ccp_vdata;
if (!ccp->vdata || !ccp->vdata->version) {
ret = -ENODEV;
diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h
index 90523a069bff..3f68262d9ab4 100644
--- a/drivers/crypto/ccp/ccp-dev.h
+++ b/drivers/crypto/ccp/ccp-dev.h
@@ -12,11 +12,11 @@
#define __CCP_DEV_H__
#include <linux/device.h>
-#include <linux/pci.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
#include <linux/list.h>
#include <linux/wait.h>
+#include <linux/dma-direction.h>
#include <linux/dmapool.h>
#include <linux/hw_random.h>
#include <linux/bitops.h>
@@ -379,6 +379,7 @@ struct ccp_device {
*/
struct ccp_cmd_queue cmd_q[MAX_HW_QUEUES];
unsigned int cmd_q_count;
+ unsigned int max_q_count;
/* Support for the CCP True RNG
*/
@@ -629,7 +630,7 @@ struct ccp5_desc {
void ccp_add_device(struct ccp_device *ccp);
void ccp_del_device(struct ccp_device *ccp);
-extern void ccp_log_error(struct ccp_device *, int);
+extern void ccp_log_error(struct ccp_device *, unsigned int);
struct ccp_device *ccp_alloc_struct(struct sp_device *sp);
bool ccp_queues_suspended(struct ccp_device *ccp);
diff --git a/drivers/crypto/ccp/ccp-dmaengine.c b/drivers/crypto/ccp/ccp-dmaengine.c
index 7f22a45bbc11..a54f9367a580 100644
--- a/drivers/crypto/ccp/ccp-dmaengine.c
+++ b/drivers/crypto/ccp/ccp-dmaengine.c
@@ -2,13 +2,14 @@
/*
* AMD Cryptographic Coprocessor (CCP) driver
*
- * Copyright (C) 2016,2017 Advanced Micro Devices, Inc.
+ * Copyright (C) 2016,2019 Advanced Micro Devices, Inc.
*
* Author: Gary R Hook <gary.hook@amd.com>
*/
#include <linux/module.h>
#include <linux/kernel.h>
+#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
@@ -35,6 +36,10 @@ static unsigned int dma_chan_attr = CCP_DMA_DFLT;
module_param(dma_chan_attr, uint, 0444);
MODULE_PARM_DESC(dma_chan_attr, "Set DMA channel visibility: 0 (default) = device defaults, 1 = make private, 2 = make public");
+static unsigned int dmaengine = 1;
+module_param(dmaengine, uint, 0444);
+MODULE_PARM_DESC(dmaengine, "Register services with the DMA subsystem (any non-zero value, default: 1)");
+
static unsigned int ccp_get_dma_chan_attr(struct ccp_device *ccp)
{
switch (dma_chan_attr) {
@@ -637,6 +642,9 @@ int ccp_dmaengine_register(struct ccp_device *ccp)
unsigned int i;
int ret;
+ if (!dmaengine)
+ return 0;
+
ccp->ccp_dma_chan = devm_kcalloc(ccp->dev, ccp->cmd_q_count,
sizeof(*(ccp->ccp_dma_chan)),
GFP_KERNEL);
@@ -740,6 +748,9 @@ void ccp_dmaengine_unregister(struct ccp_device *ccp)
{
struct dma_device *dma_dev = &ccp->dma_dev;
+ if (!dmaengine)
+ return;
+
dma_async_device_unregister(dma_dev);
kmem_cache_destroy(ccp->dma_desc_cache);
diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c
index db8de89d990f..c8da8eb160da 100644
--- a/drivers/crypto/ccp/ccp-ops.c
+++ b/drivers/crypto/ccp/ccp-ops.c
@@ -2,7 +2,7 @@
/*
* AMD Cryptographic Coprocessor (CCP) driver
*
- * Copyright (C) 2013,2018 Advanced Micro Devices, Inc.
+ * Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
*
* Author: Tom Lendacky <thomas.lendacky@amd.com>
* Author: Gary R Hook <gary.hook@amd.com>
@@ -10,7 +10,6 @@
#include <linux/module.h>
#include <linux/kernel.h>
-#include <linux/pci.h>
#include <linux/interrupt.h>
#include <crypto/scatterwalk.h>
#include <crypto/des.h>
@@ -150,14 +149,13 @@ static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
if (len <= CCP_DMAPOOL_MAX_SIZE) {
wa->dma_pool = cmd_q->dma_pool;
- wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
+ wa->address = dma_pool_zalloc(wa->dma_pool, GFP_KERNEL,
&wa->dma.address);
if (!wa->address)
return -ENOMEM;
wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
- memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
} else {
wa->address = kzalloc(len, GFP_KERNEL);
if (!wa->address)
@@ -455,8 +453,8 @@ static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
}
-static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
- struct ccp_cmd *cmd)
+static noinline_for_stack int
+ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_aes_engine *aes = &cmd->u.aes;
struct ccp_dm_workarea key, ctx;
@@ -611,8 +609,8 @@ e_key:
return ret;
}
-static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q,
- struct ccp_cmd *cmd)
+static noinline_for_stack int
+ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_aes_engine *aes = &cmd->u.aes;
struct ccp_dm_workarea key, ctx, final_wa, tag;
@@ -622,6 +620,8 @@ static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q,
unsigned long long *final;
unsigned int dm_offset;
+ unsigned int authsize;
+ unsigned int jobid;
unsigned int ilen;
bool in_place = true; /* Default value */
int ret;
@@ -642,6 +642,21 @@ static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q,
if (!aes->key) /* Gotta have a key SGL */
return -EINVAL;
+ /* Zero defaults to 16 bytes, the maximum size */
+ authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
+ switch (authsize) {
+ case 16:
+ case 15:
+ case 14:
+ case 13:
+ case 12:
+ case 8:
+ case 4:
+ break;
+ default:
+ return -EINVAL;
+ }
+
/* First, decompose the source buffer into AAD & PT,
* and the destination buffer into AAD, CT & tag, or
* the input into CT & tag.
@@ -656,13 +671,15 @@ static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q,
p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
} else {
/* Input length for decryption includes tag */
- ilen = aes->src_len - AES_BLOCK_SIZE;
+ ilen = aes->src_len - authsize;
p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
}
+ jobid = CCP_NEW_JOBID(cmd_q->ccp);
+
memset(&op, 0, sizeof(op));
op.cmd_q = cmd_q;
- op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
+ op.jobid = jobid;
op.sb_key = cmd_q->sb_key; /* Pre-allocated */
op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
op.init = 1;
@@ -763,8 +780,7 @@ static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q,
while (src.sg_wa.bytes_left) {
ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
if (!src.sg_wa.bytes_left) {
- unsigned int nbytes = aes->src_len
- % AES_BLOCK_SIZE;
+ unsigned int nbytes = ilen % AES_BLOCK_SIZE;
if (nbytes) {
op.eom = 1;
@@ -813,6 +829,13 @@ static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q,
final[0] = cpu_to_be64(aes->aad_len * 8);
final[1] = cpu_to_be64(ilen * 8);
+ memset(&op, 0, sizeof(op));
+ op.cmd_q = cmd_q;
+ op.jobid = jobid;
+ op.sb_key = cmd_q->sb_key; /* Pre-allocated */
+ op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
+ op.init = 1;
+ op.u.aes.type = aes->type;
op.u.aes.mode = CCP_AES_MODE_GHASH;
op.u.aes.action = CCP_AES_GHASHFINAL;
op.src.type = CCP_MEMTYPE_SYSTEM;
@@ -829,18 +852,19 @@ static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q,
if (aes->action == CCP_AES_ACTION_ENCRYPT) {
/* Put the ciphered tag after the ciphertext. */
- ccp_get_dm_area(&final_wa, 0, p_tag, 0, AES_BLOCK_SIZE);
+ ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
} else {
/* Does this ciphered tag match the input? */
- ret = ccp_init_dm_workarea(&tag, cmd_q, AES_BLOCK_SIZE,
+ ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
DMA_BIDIRECTIONAL);
if (ret)
goto e_tag;
- ret = ccp_set_dm_area(&tag, 0, p_tag, 0, AES_BLOCK_SIZE);
+ ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
if (ret)
goto e_tag;
- ret = memcmp(tag.address, final_wa.address, AES_BLOCK_SIZE);
+ ret = crypto_memneq(tag.address, final_wa.address,
+ authsize) ? -EBADMSG : 0;
ccp_dm_free(&tag);
}
@@ -848,11 +872,11 @@ e_tag:
ccp_dm_free(&final_wa);
e_dst:
- if (aes->src_len && !in_place)
+ if (ilen > 0 && !in_place)
ccp_free_data(&dst, cmd_q);
e_src:
- if (aes->src_len)
+ if (ilen > 0)
ccp_free_data(&src, cmd_q);
e_aad:
@@ -868,7 +892,8 @@ e_key:
return ret;
}
-static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
+static noinline_for_stack int
+ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_aes_engine *aes = &cmd->u.aes;
struct ccp_dm_workarea key, ctx;
@@ -878,20 +903,13 @@ static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
bool in_place = false;
int ret;
- if (aes->mode == CCP_AES_MODE_CMAC)
- return ccp_run_aes_cmac_cmd(cmd_q, cmd);
-
- if (aes->mode == CCP_AES_MODE_GCM)
- return ccp_run_aes_gcm_cmd(cmd_q, cmd);
-
if (!((aes->key_len == AES_KEYSIZE_128) ||
(aes->key_len == AES_KEYSIZE_192) ||
(aes->key_len == AES_KEYSIZE_256)))
return -EINVAL;
if (((aes->mode == CCP_AES_MODE_ECB) ||
- (aes->mode == CCP_AES_MODE_CBC) ||
- (aes->mode == CCP_AES_MODE_CFB)) &&
+ (aes->mode == CCP_AES_MODE_CBC)) &&
(aes->src_len & (AES_BLOCK_SIZE - 1)))
return -EINVAL;
@@ -1051,8 +1069,8 @@ e_key:
return ret;
}
-static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
- struct ccp_cmd *cmd)
+static noinline_for_stack int
+ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_xts_aes_engine *xts = &cmd->u.xts;
struct ccp_dm_workarea key, ctx;
@@ -1251,7 +1269,8 @@ e_key:
return ret;
}
-static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
+static noinline_for_stack int
+ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_des3_engine *des3 = &cmd->u.des3;
@@ -1264,6 +1283,9 @@ static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
int ret;
/* Error checks */
+ if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
+ return -EINVAL;
+
if (!cmd_q->ccp->vdata->perform->des3)
return -EINVAL;
@@ -1346,8 +1368,6 @@ static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
* passthru option to convert from big endian to little endian.
*/
if (des3->mode != CCP_DES3_MODE_ECB) {
- u32 load_mode;
-
op.sb_ctx = cmd_q->sb_ctx;
ret = ccp_init_dm_workarea(&ctx, cmd_q,
@@ -1363,12 +1383,8 @@ static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
if (ret)
goto e_ctx;
- if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
- load_mode = CCP_PASSTHRU_BYTESWAP_NOOP;
- else
- load_mode = CCP_PASSTHRU_BYTESWAP_256BIT;
ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
- load_mode);
+ CCP_PASSTHRU_BYTESWAP_256BIT);
if (ret) {
cmd->engine_error = cmd_q->cmd_error;
goto e_ctx;
@@ -1430,10 +1446,6 @@ static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
}
/* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
- if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
- dm_offset = CCP_SB_BYTES - des3->iv_len;
- else
- dm_offset = 0;
ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
DES3_EDE_BLOCK_SIZE);
}
@@ -1454,7 +1466,8 @@ e_key:
return ret;
}
-static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
+static noinline_for_stack int
+ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_sha_engine *sha = &cmd->u.sha;
struct ccp_dm_workarea ctx;
@@ -1798,7 +1811,8 @@ e_ctx:
return ret;
}
-static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
+static noinline_for_stack int
+ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_rsa_engine *rsa = &cmd->u.rsa;
struct ccp_dm_workarea exp, src, dst;
@@ -1929,8 +1943,8 @@ e_sb:
return ret;
}
-static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
- struct ccp_cmd *cmd)
+static noinline_for_stack int
+ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_passthru_engine *pt = &cmd->u.passthru;
struct ccp_dm_workarea mask;
@@ -2061,7 +2075,8 @@ e_mask:
return ret;
}
-static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
+static noinline_for_stack int
+ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
struct ccp_cmd *cmd)
{
struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
@@ -2402,7 +2417,8 @@ e_src:
return ret;
}
-static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
+static noinline_for_stack int
+ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_ecc_engine *ecc = &cmd->u.ecc;
@@ -2439,7 +2455,17 @@ int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
switch (cmd->engine) {
case CCP_ENGINE_AES:
- ret = ccp_run_aes_cmd(cmd_q, cmd);
+ switch (cmd->u.aes.mode) {
+ case CCP_AES_MODE_CMAC:
+ ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
+ break;
+ case CCP_AES_MODE_GCM:
+ ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
+ break;
+ default:
+ ret = ccp_run_aes_cmd(cmd_q, cmd);
+ break;
+ }
break;
case CCP_ENGINE_XTS_AES_128:
ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
diff --git a/drivers/crypto/ccp/psp-dev.c b/drivers/crypto/ccp/psp-dev.c
index de5a8ca70d3d..6b17d179ef8a 100644
--- a/drivers/crypto/ccp/psp-dev.c
+++ b/drivers/crypto/ccp/psp-dev.c
@@ -24,10 +24,6 @@
#include "sp-dev.h"
#include "psp-dev.h"
-#define SEV_VERSION_GREATER_OR_EQUAL(_maj, _min) \
- ((psp_master->api_major) >= _maj && \
- (psp_master->api_minor) >= _min)
-
#define DEVICE_NAME "sev"
#define SEV_FW_FILE "amd/sev.fw"
#define SEV_FW_NAME_SIZE 64
@@ -47,6 +43,15 @@ MODULE_PARM_DESC(psp_probe_timeout, " default timeout value, in seconds, during
static bool psp_dead;
static int psp_timeout;
+static inline bool sev_version_greater_or_equal(u8 maj, u8 min)
+{
+ if (psp_master->api_major > maj)
+ return true;
+ if (psp_master->api_major == maj && psp_master->api_minor >= min)
+ return true;
+ return false;
+}
+
static struct psp_device *psp_alloc_struct(struct sp_device *sp)
{
struct device *dev = sp->dev;
@@ -588,7 +593,7 @@ static int sev_ioctl_do_get_id2(struct sev_issue_cmd *argp)
int ret;
/* SEV GET_ID is available from SEV API v0.16 and up */
- if (!SEV_VERSION_GREATER_OR_EQUAL(0, 16))
+ if (!sev_version_greater_or_equal(0, 16))
return -ENOTSUPP;
if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
@@ -651,7 +656,7 @@ static int sev_ioctl_do_get_id(struct sev_issue_cmd *argp)
int ret;
/* SEV GET_ID available from SEV API v0.16 and up */
- if (!SEV_VERSION_GREATER_OR_EQUAL(0, 16))
+ if (!sev_version_greater_or_equal(0, 16))
return -ENOTSUPP;
/* SEV FW expects the buffer it fills with the ID to be
@@ -1053,7 +1058,7 @@ void psp_pci_init(void)
psp_master->sev_state = SEV_STATE_UNINIT;
}
- if (SEV_VERSION_GREATER_OR_EQUAL(0, 15) &&
+ if (sev_version_greater_or_equal(0, 15) &&
sev_update_firmware(psp_master->dev) == 0)
sev_get_api_version();
diff --git a/drivers/crypto/ccp/psp-dev.h b/drivers/crypto/ccp/psp-dev.h
index c5e06c92d40e..82a084f02990 100644
--- a/drivers/crypto/ccp/psp-dev.h
+++ b/drivers/crypto/ccp/psp-dev.h
@@ -11,7 +11,6 @@
#define __PSP_DEV_H__
#include <linux/device.h>
-#include <linux/pci.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
#include <linux/list.h>
diff --git a/drivers/crypto/ccp/sp-dev.h b/drivers/crypto/ccp/sp-dev.h
index 8abe9ea7e76f..53c12562d31e 100644
--- a/drivers/crypto/ccp/sp-dev.h
+++ b/drivers/crypto/ccp/sp-dev.h
@@ -13,7 +13,6 @@
#define __SP_DEV_H__
#include <linux/device.h>
-#include <linux/pci.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
#include <linux/list.h>
diff --git a/drivers/crypto/ccp/sp-platform.c b/drivers/crypto/ccp/sp-platform.c
index 1b45236e3716..831aac1393a2 100644
--- a/drivers/crypto/ccp/sp-platform.c
+++ b/drivers/crypto/ccp/sp-platform.c
@@ -125,7 +125,6 @@ static int sp_platform_probe(struct platform_device *pdev)
struct sp_platform *sp_platform;
struct device *dev = &pdev->dev;
enum dev_dma_attr attr;
- struct resource *ior;
int ret;
ret = -ENOMEM;
@@ -146,8 +145,7 @@ static int sp_platform_probe(struct platform_device *pdev)
goto e_err;
}
- ior = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- sp->io_map = devm_ioremap_resource(dev, ior);
+ sp->io_map = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(sp->io_map)) {
ret = PTR_ERR(sp->io_map);
goto e_err;
diff --git a/drivers/crypto/ccree/Makefile b/drivers/crypto/ccree/Makefile
index 145e50bdbf16..5cfda508ee41 100644
--- a/drivers/crypto/ccree/Makefile
+++ b/drivers/crypto/ccree/Makefile
@@ -2,7 +2,7 @@
# Copyright (C) 2012-2019 ARM Limited (or its affiliates).
obj-$(CONFIG_CRYPTO_DEV_CCREE) := ccree.o
-ccree-y := cc_driver.o cc_buffer_mgr.o cc_request_mgr.o cc_cipher.o cc_hash.o cc_aead.o cc_ivgen.o cc_sram_mgr.o
+ccree-y := cc_driver.o cc_buffer_mgr.o cc_request_mgr.o cc_cipher.o cc_hash.o cc_aead.o cc_sram_mgr.o
ccree-$(CONFIG_CRYPTO_FIPS) += cc_fips.o
ccree-$(CONFIG_DEBUG_FS) += cc_debugfs.o
ccree-$(CONFIG_PM) += cc_pm.o
diff --git a/drivers/crypto/ccree/cc_aead.c b/drivers/crypto/ccree/cc_aead.c
index 7aa4cbe19a86..d3e8faa03f15 100644
--- a/drivers/crypto/ccree/cc_aead.c
+++ b/drivers/crypto/ccree/cc_aead.c
@@ -6,7 +6,7 @@
#include <crypto/algapi.h>
#include <crypto/internal/aead.h>
#include <crypto/authenc.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <linux/rtnetlink.h>
#include "cc_driver.h"
#include "cc_buffer_mgr.h"
@@ -236,31 +236,17 @@ static void cc_aead_complete(struct device *dev, void *cc_req, int err)
/* In case of payload authentication failure, MUST NOT
* revealed the decrypted message --> zero its memory.
*/
- cc_zero_sgl(areq->dst, areq_ctx->cryptlen);
+ sg_zero_buffer(areq->dst, sg_nents(areq->dst),
+ areq->cryptlen, 0);
err = -EBADMSG;
}
- } else { /*ENCRYPT*/
- if (areq_ctx->is_icv_fragmented) {
- u32 skip = areq->cryptlen + areq_ctx->dst_offset;
-
- cc_copy_sg_portion(dev, areq_ctx->mac_buf,
- areq_ctx->dst_sgl, skip,
- (skip + ctx->authsize),
- CC_SG_FROM_BUF);
- }
+ /*ENCRYPT*/
+ } else if (areq_ctx->is_icv_fragmented) {
+ u32 skip = areq->cryptlen + areq_ctx->dst_offset;
- /* If an IV was generated, copy it back to the user provided
- * buffer.
- */
- if (areq_ctx->backup_giv) {
- if (ctx->cipher_mode == DRV_CIPHER_CTR)
- memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv +
- CTR_RFC3686_NONCE_SIZE,
- CTR_RFC3686_IV_SIZE);
- else if (ctx->cipher_mode == DRV_CIPHER_CCM)
- memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv +
- CCM_BLOCK_IV_OFFSET, CCM_BLOCK_IV_SIZE);
- }
+ cc_copy_sg_portion(dev, areq_ctx->mac_buf, areq_ctx->dst_sgl,
+ skip, (skip + ctx->authsize),
+ CC_SG_FROM_BUF);
}
done:
aead_request_complete(areq, err);
@@ -663,33 +649,17 @@ static int cc_des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
unsigned int keylen)
{
struct crypto_authenc_keys keys;
- u32 flags;
int err;
err = crypto_authenc_extractkeys(&keys, key, keylen);
if (unlikely(err))
- goto badkey;
-
- err = -EINVAL;
- if (keys.enckeylen != DES3_EDE_KEY_SIZE)
- goto badkey;
+ return err;
- flags = crypto_aead_get_flags(aead);
- err = __des3_verify_key(&flags, keys.enckey);
- if (unlikely(err)) {
- crypto_aead_set_flags(aead, flags);
- goto out;
- }
-
- err = cc_aead_setkey(aead, key, keylen);
+ err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?:
+ cc_aead_setkey(aead, key, keylen);
-out:
memzero_explicit(&keys, sizeof(keys));
return err;
-
-badkey:
- crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
- goto out;
}
static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key,
@@ -1975,9 +1945,8 @@ static int cc_proc_aead(struct aead_request *req,
*/
memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce,
CTR_RFC3686_NONCE_SIZE);
- if (!areq_ctx->backup_giv) /*User none-generated IV*/
- memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE,
- req->iv, CTR_RFC3686_IV_SIZE);
+ memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE, req->iv,
+ CTR_RFC3686_IV_SIZE);
/* Initialize counter portion of counter block */
*(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE +
CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
@@ -2023,40 +1992,6 @@ static int cc_proc_aead(struct aead_request *req,
goto exit;
}
- /* do we need to generate IV? */
- if (areq_ctx->backup_giv) {
- /* set the DMA mapped IV address*/
- if (ctx->cipher_mode == DRV_CIPHER_CTR) {
- cc_req.ivgen_dma_addr[0] =
- areq_ctx->gen_ctx.iv_dma_addr +
- CTR_RFC3686_NONCE_SIZE;
- cc_req.ivgen_dma_addr_len = 1;
- } else if (ctx->cipher_mode == DRV_CIPHER_CCM) {
- /* In ccm, the IV needs to exist both inside B0 and
- * inside the counter.It is also copied to iv_dma_addr
- * for other reasons (like returning it to the user).
- * So, using 3 (identical) IV outputs.
- */
- cc_req.ivgen_dma_addr[0] =
- areq_ctx->gen_ctx.iv_dma_addr +
- CCM_BLOCK_IV_OFFSET;
- cc_req.ivgen_dma_addr[1] =
- sg_dma_address(&areq_ctx->ccm_adata_sg) +
- CCM_B0_OFFSET + CCM_BLOCK_IV_OFFSET;
- cc_req.ivgen_dma_addr[2] =
- sg_dma_address(&areq_ctx->ccm_adata_sg) +
- CCM_CTR_COUNT_0_OFFSET + CCM_BLOCK_IV_OFFSET;
- cc_req.ivgen_dma_addr_len = 3;
- } else {
- cc_req.ivgen_dma_addr[0] =
- areq_ctx->gen_ctx.iv_dma_addr;
- cc_req.ivgen_dma_addr_len = 1;
- }
-
- /* set the IV size (8/16 B long)*/
- cc_req.ivgen_size = crypto_aead_ivsize(tfm);
- }
-
/* STAT_PHASE_2: Create sequence */
/* Load MLLI tables to SRAM if necessary */
@@ -2107,7 +2042,6 @@ static int cc_aead_encrypt(struct aead_request *req)
/* No generated IV required */
areq_ctx->backup_iv = req->iv;
areq_ctx->assoclen = req->assoclen;
- areq_ctx->backup_giv = NULL;
areq_ctx->is_gcm4543 = false;
areq_ctx->plaintext_authenticate_only = false;
@@ -2139,7 +2073,6 @@ static int cc_rfc4309_ccm_encrypt(struct aead_request *req)
/* No generated IV required */
areq_ctx->backup_iv = req->iv;
areq_ctx->assoclen = req->assoclen;
- areq_ctx->backup_giv = NULL;
areq_ctx->is_gcm4543 = true;
cc_proc_rfc4309_ccm(req);
@@ -2161,7 +2094,6 @@ static int cc_aead_decrypt(struct aead_request *req)
/* No generated IV required */
areq_ctx->backup_iv = req->iv;
areq_ctx->assoclen = req->assoclen;
- areq_ctx->backup_giv = NULL;
areq_ctx->is_gcm4543 = false;
areq_ctx->plaintext_authenticate_only = false;
@@ -2191,7 +2123,6 @@ static int cc_rfc4309_ccm_decrypt(struct aead_request *req)
/* No generated IV required */
areq_ctx->backup_iv = req->iv;
areq_ctx->assoclen = req->assoclen;
- areq_ctx->backup_giv = NULL;
areq_ctx->is_gcm4543 = true;
cc_proc_rfc4309_ccm(req);
@@ -2311,8 +2242,6 @@ static int cc_rfc4106_gcm_encrypt(struct aead_request *req)
/* No generated IV required */
areq_ctx->backup_iv = req->iv;
areq_ctx->assoclen = req->assoclen;
- areq_ctx->backup_giv = NULL;
-
areq_ctx->plaintext_authenticate_only = false;
cc_proc_rfc4_gcm(req);
@@ -2328,9 +2257,16 @@ out:
static int cc_rfc4543_gcm_encrypt(struct aead_request *req)
{
/* Very similar to cc_aead_encrypt() above. */
-
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
+ struct device *dev = drvdata_to_dev(ctx->drvdata);
struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
- int rc;
+ int rc = -EINVAL;
+
+ if (!valid_assoclen(req)) {
+ dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
+ goto out;
+ }
memset(areq_ctx, 0, sizeof(*areq_ctx));
@@ -2340,7 +2276,6 @@ static int cc_rfc4543_gcm_encrypt(struct aead_request *req)
/* No generated IV required */
areq_ctx->backup_iv = req->iv;
areq_ctx->assoclen = req->assoclen;
- areq_ctx->backup_giv = NULL;
cc_proc_rfc4_gcm(req);
areq_ctx->is_gcm4543 = true;
@@ -2348,7 +2283,7 @@ static int cc_rfc4543_gcm_encrypt(struct aead_request *req)
rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
if (rc != -EINPROGRESS && rc != -EBUSY)
req->iv = areq_ctx->backup_iv;
-
+out:
return rc;
}
@@ -2372,8 +2307,6 @@ static int cc_rfc4106_gcm_decrypt(struct aead_request *req)
/* No generated IV required */
areq_ctx->backup_iv = req->iv;
areq_ctx->assoclen = req->assoclen;
- areq_ctx->backup_giv = NULL;
-
areq_ctx->plaintext_authenticate_only = false;
cc_proc_rfc4_gcm(req);
@@ -2389,9 +2322,16 @@ out:
static int cc_rfc4543_gcm_decrypt(struct aead_request *req)
{
/* Very similar to cc_aead_decrypt() above. */
-
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
+ struct device *dev = drvdata_to_dev(ctx->drvdata);
struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
- int rc;
+ int rc = -EINVAL;
+
+ if (!valid_assoclen(req)) {
+ dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
+ goto out;
+ }
memset(areq_ctx, 0, sizeof(*areq_ctx));
@@ -2401,7 +2341,6 @@ static int cc_rfc4543_gcm_decrypt(struct aead_request *req)
/* No generated IV required */
areq_ctx->backup_iv = req->iv;
areq_ctx->assoclen = req->assoclen;
- areq_ctx->backup_giv = NULL;
cc_proc_rfc4_gcm(req);
areq_ctx->is_gcm4543 = true;
@@ -2409,7 +2348,7 @@ static int cc_rfc4543_gcm_decrypt(struct aead_request *req)
rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
if (rc != -EINPROGRESS && rc != -EBUSY)
req->iv = areq_ctx->backup_iv;
-
+out:
return rc;
}
diff --git a/drivers/crypto/ccree/cc_aead.h b/drivers/crypto/ccree/cc_aead.h
index e51724b96c56..f12169b57f9d 100644
--- a/drivers/crypto/ccree/cc_aead.h
+++ b/drivers/crypto/ccree/cc_aead.h
@@ -65,8 +65,7 @@ struct aead_req_ctx {
unsigned int hw_iv_size ____cacheline_aligned;
/* used to prevent cache coherence problem */
u8 backup_mac[MAX_MAC_SIZE];
- u8 *backup_iv; /*store iv for generated IV flow*/
- u8 *backup_giv; /*store iv for rfc3686(ctr) flow*/
+ u8 *backup_iv; /* store orig iv */
u32 assoclen; /* internal assoclen */
dma_addr_t mac_buf_dma_addr; /* internal ICV DMA buffer */
/* buffer for internal ccm configurations */
diff --git a/drivers/crypto/ccree/cc_buffer_mgr.c b/drivers/crypto/ccree/cc_buffer_mgr.c
index c81ad33f9115..a72586eccd81 100644
--- a/drivers/crypto/ccree/cc_buffer_mgr.c
+++ b/drivers/crypto/ccree/cc_buffer_mgr.c
@@ -100,27 +100,6 @@ static unsigned int cc_get_sgl_nents(struct device *dev,
}
/**
- * cc_zero_sgl() - Zero scatter scatter list data.
- *
- * @sgl:
- */
-void cc_zero_sgl(struct scatterlist *sgl, u32 data_len)
-{
- struct scatterlist *current_sg = sgl;
- int sg_index = 0;
-
- while (sg_index <= data_len) {
- if (!current_sg) {
- /* reached the end of the sgl --> just return back */
- return;
- }
- memset(sg_virt(current_sg), 0, current_sg->length);
- sg_index += current_sg->length;
- current_sg = sg_next(current_sg);
- }
-}
-
-/**
* cc_copy_sg_portion() - Copy scatter list data,
* from to_skip to end, to dest and vice versa
*
diff --git a/drivers/crypto/ccree/cc_buffer_mgr.h b/drivers/crypto/ccree/cc_buffer_mgr.h
index a726016bdbc1..af434872c6ff 100644
--- a/drivers/crypto/ccree/cc_buffer_mgr.h
+++ b/drivers/crypto/ccree/cc_buffer_mgr.h
@@ -66,6 +66,4 @@ void cc_unmap_hash_request(struct device *dev, void *ctx,
void cc_copy_sg_portion(struct device *dev, u8 *dest, struct scatterlist *sg,
u32 to_skip, u32 end, enum cc_sg_cpy_direct direct);
-void cc_zero_sgl(struct scatterlist *sgl, u32 data_len);
-
#endif /*__BUFFER_MGR_H__*/
diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c
index 5b58226ea24d..254b48797799 100644
--- a/drivers/crypto/ccree/cc_cipher.c
+++ b/drivers/crypto/ccree/cc_cipher.c
@@ -5,7 +5,7 @@
#include <linux/module.h>
#include <crypto/algapi.h>
#include <crypto/internal/skcipher.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <crypto/xts.h>
#include <crypto/sm4.h>
#include <crypto/scatterwalk.h>
@@ -116,10 +116,6 @@ static int validate_data_size(struct cc_cipher_ctx *ctx_p,
case S_DIN_to_AES:
switch (ctx_p->cipher_mode) {
case DRV_CIPHER_XTS:
- if (size >= AES_BLOCK_SIZE &&
- IS_ALIGNED(size, AES_BLOCK_SIZE))
- return 0;
- break;
case DRV_CIPHER_CBC_CTS:
if (size >= AES_BLOCK_SIZE)
return 0;
@@ -411,16 +407,9 @@ static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key,
* HW does the expansion on its own.
*/
if (ctx_p->flow_mode == S_DIN_to_DES) {
- u32 tmp[DES3_EDE_EXPKEY_WORDS];
- if (keylen == DES3_EDE_KEY_SIZE &&
- __des3_ede_setkey(tmp, &tfm->crt_flags, key,
- DES3_EDE_KEY_SIZE)) {
- dev_dbg(dev, "weak 3DES key");
- return -EINVAL;
- } else if (!des_ekey(tmp, key) &&
- (crypto_tfm_get_flags(tfm) &
- CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
- tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
+ if ((keylen == DES3_EDE_KEY_SIZE &&
+ verify_skcipher_des3_key(sktfm, key)) ||
+ verify_skcipher_des_key(sktfm, key)) {
dev_dbg(dev, "weak DES key");
return -EINVAL;
}
@@ -945,7 +934,7 @@ static const struct cc_alg_template skcipher_algs[] = {
{
.name = "xts(paes)",
.driver_name = "xts-paes-ccree",
- .blocksize = AES_BLOCK_SIZE,
+ .blocksize = 1,
.template_skcipher = {
.setkey = cc_cipher_sethkey,
.encrypt = cc_cipher_encrypt,
@@ -963,7 +952,7 @@ static const struct cc_alg_template skcipher_algs[] = {
{
.name = "xts512(paes)",
.driver_name = "xts-paes-du512-ccree",
- .blocksize = AES_BLOCK_SIZE,
+ .blocksize = 1,
.template_skcipher = {
.setkey = cc_cipher_sethkey,
.encrypt = cc_cipher_encrypt,
@@ -982,7 +971,7 @@ static const struct cc_alg_template skcipher_algs[] = {
{
.name = "xts4096(paes)",
.driver_name = "xts-paes-du4096-ccree",
- .blocksize = AES_BLOCK_SIZE,
+ .blocksize = 1,
.template_skcipher = {
.setkey = cc_cipher_sethkey,
.encrypt = cc_cipher_encrypt,
@@ -1203,7 +1192,7 @@ static const struct cc_alg_template skcipher_algs[] = {
{
.name = "xts(aes)",
.driver_name = "xts-aes-ccree",
- .blocksize = AES_BLOCK_SIZE,
+ .blocksize = 1,
.template_skcipher = {
.setkey = cc_cipher_setkey,
.encrypt = cc_cipher_encrypt,
@@ -1220,7 +1209,7 @@ static const struct cc_alg_template skcipher_algs[] = {
{
.name = "xts512(aes)",
.driver_name = "xts-aes-du512-ccree",
- .blocksize = AES_BLOCK_SIZE,
+ .blocksize = 1,
.template_skcipher = {
.setkey = cc_cipher_setkey,
.encrypt = cc_cipher_encrypt,
@@ -1238,7 +1227,7 @@ static const struct cc_alg_template skcipher_algs[] = {
{
.name = "xts4096(aes)",
.driver_name = "xts-aes-du4096-ccree",
- .blocksize = AES_BLOCK_SIZE,
+ .blocksize = 1,
.template_skcipher = {
.setkey = cc_cipher_setkey,
.encrypt = cc_cipher_encrypt,
diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c
index 86ac7b443355..8b8eee513c27 100644
--- a/drivers/crypto/ccree/cc_driver.c
+++ b/drivers/crypto/ccree/cc_driver.c
@@ -22,7 +22,6 @@
#include "cc_cipher.h"
#include "cc_aead.h"
#include "cc_hash.h"
-#include "cc_ivgen.h"
#include "cc_sram_mgr.h"
#include "cc_pm.h"
#include "cc_fips.h"
@@ -48,6 +47,7 @@ struct cc_hw_data {
};
#define CC_NUM_IDRS 4
+#define CC_HW_RESET_LOOP_COUNT 10
/* Note: PIDR3 holds CMOD/Rev so ignored for HW identification purposes */
static const u32 pidr_0124_offsets[CC_NUM_IDRS] = {
@@ -133,6 +133,9 @@ static irqreturn_t cc_isr(int irq, void *dev_id)
u32 imr;
/* STAT_OP_TYPE_GENERIC STAT_PHASE_0: Interrupt */
+ /* if driver suspended return, probebly shared interrupt */
+ if (cc_pm_is_dev_suspended(dev))
+ return IRQ_NONE;
/* read the interrupt status */
irr = cc_ioread(drvdata, CC_REG(HOST_IRR));
@@ -188,6 +191,31 @@ static irqreturn_t cc_isr(int irq, void *dev_id)
return IRQ_HANDLED;
}
+bool cc_wait_for_reset_completion(struct cc_drvdata *drvdata)
+{
+ unsigned int val;
+ unsigned int i;
+
+ /* 712/710/63 has no reset completion indication, always return true */
+ if (drvdata->hw_rev <= CC_HW_REV_712)
+ return true;
+
+ for (i = 0; i < CC_HW_RESET_LOOP_COUNT; i++) {
+ /* in cc7x3 NVM_IS_IDLE indicates that CC reset is
+ * completed and device is fully functional
+ */
+ val = cc_ioread(drvdata, CC_REG(NVM_IS_IDLE));
+ if (val & CC_NVM_IS_IDLE_MASK) {
+ /* hw indicate reset completed */
+ return true;
+ }
+ /* allow scheduling other process on the processor */
+ schedule();
+ }
+ /* reset not completed */
+ return false;
+}
+
int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe)
{
unsigned int val, cache_params;
@@ -310,19 +338,8 @@ static int init_cc_resources(struct platform_device *plat_dev)
/* Then IRQ */
new_drvdata->irq = platform_get_irq(plat_dev, 0);
- if (new_drvdata->irq < 0) {
- dev_err(dev, "Failed getting IRQ resource\n");
+ if (new_drvdata->irq < 0)
return new_drvdata->irq;
- }
-
- rc = devm_request_irq(dev, new_drvdata->irq, cc_isr,
- IRQF_SHARED, "ccree", new_drvdata);
- if (rc) {
- dev_err(dev, "Could not register to interrupt %d\n",
- new_drvdata->irq);
- return rc;
- }
- dev_dbg(dev, "Registered to IRQ: %d\n", new_drvdata->irq);
init_completion(&new_drvdata->hw_queue_avail);
@@ -352,6 +369,11 @@ static int init_cc_resources(struct platform_device *plat_dev)
new_drvdata->sec_disabled = cc_sec_disable;
+ /* wait for Crytpcell reset completion */
+ if (!cc_wait_for_reset_completion(new_drvdata)) {
+ dev_err(dev, "Cryptocell reset not completed");
+ }
+
if (hw_rev->rev <= CC_HW_REV_712) {
/* Verify correct mapping */
val = cc_ioread(new_drvdata, new_drvdata->sig_offset);
@@ -383,6 +405,24 @@ static int init_cc_resources(struct platform_device *plat_dev)
}
sig_cidr = val;
+ /* Check HW engine configuration */
+ val = cc_ioread(new_drvdata, CC_REG(HOST_REMOVE_INPUT_PINS));
+ switch (val) {
+ case CC_PINS_FULL:
+ /* This is fine */
+ break;
+ case CC_PINS_SLIM:
+ if (new_drvdata->std_bodies & CC_STD_NIST) {
+ dev_warn(dev, "703 mode forced due to HW configuration.\n");
+ new_drvdata->std_bodies = CC_STD_OSCCA;
+ }
+ break;
+ default:
+ dev_err(dev, "Unsupported engines configuration.\n");
+ rc = -EINVAL;
+ goto post_clk_err;
+ }
+
/* Check security disable state */
val = cc_ioread(new_drvdata, CC_REG(SECURITY_DISABLED));
val &= CC_SECURITY_DISABLED_MASK;
@@ -401,6 +441,15 @@ static int init_cc_resources(struct platform_device *plat_dev)
/* Display HW versions */
dev_info(dev, "ARM CryptoCell %s Driver: HW version 0x%08X/0x%8X, Driver version %s\n",
hw_rev->name, hw_rev_pidr, sig_cidr, DRV_MODULE_VERSION);
+ /* register the driver isr function */
+ rc = devm_request_irq(dev, new_drvdata->irq, cc_isr,
+ IRQF_SHARED, "ccree", new_drvdata);
+ if (rc) {
+ dev_err(dev, "Could not register to interrupt %d\n",
+ new_drvdata->irq);
+ goto post_clk_err;
+ }
+ dev_dbg(dev, "Registered to IRQ: %d\n", new_drvdata->irq);
rc = init_cc_regs(new_drvdata, true);
if (rc) {
@@ -451,17 +500,11 @@ static int init_cc_resources(struct platform_device *plat_dev)
goto post_buf_mgr_err;
}
- rc = cc_ivgen_init(new_drvdata);
- if (rc) {
- dev_err(dev, "cc_ivgen_init failed\n");
- goto post_buf_mgr_err;
- }
-
/* Allocate crypto algs */
rc = cc_cipher_alloc(new_drvdata);
if (rc) {
dev_err(dev, "cc_cipher_alloc failed\n");
- goto post_ivgen_err;
+ goto post_buf_mgr_err;
}
/* hash must be allocated before aead since hash exports APIs */
@@ -492,8 +535,6 @@ post_hash_err:
cc_hash_free(new_drvdata);
post_cipher_err:
cc_cipher_free(new_drvdata);
-post_ivgen_err:
- cc_ivgen_fini(new_drvdata);
post_buf_mgr_err:
cc_buffer_mgr_fini(new_drvdata);
post_req_mgr_err:
@@ -525,7 +566,6 @@ static void cleanup_cc_resources(struct platform_device *plat_dev)
cc_aead_free(drvdata);
cc_hash_free(drvdata);
cc_cipher_free(drvdata);
- cc_ivgen_fini(drvdata);
cc_pm_fini(drvdata);
cc_buffer_mgr_fini(drvdata);
cc_req_mgr_fini(drvdata);
diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h
index b76181335c08..ab31d4a68c80 100644
--- a/drivers/crypto/ccree/cc_driver.h
+++ b/drivers/crypto/ccree/cc_driver.h
@@ -53,6 +53,9 @@ enum cc_std_body {
#define CC_COHERENT_CACHE_PARAMS 0xEEE
+#define CC_PINS_FULL 0x0
+#define CC_PINS_SLIM 0x9F
+
/* Maximum DMA mask supported by IP */
#define DMA_BIT_MASK_LEN 48
@@ -67,6 +70,8 @@ enum cc_std_body {
#define CC_SECURITY_DISABLED_MASK BIT(CC_SECURITY_DISABLED_VALUE_BIT_SHIFT)
+#define CC_NVM_IS_IDLE_MASK BIT(CC_NVM_IS_IDLE_VALUE_BIT_SHIFT)
+
#define AXIM_MON_COMP_VALUE GENMASK(CC_AXIM_MON_COMP_VALUE_BIT_SIZE + \
CC_AXIM_MON_COMP_VALUE_BIT_SHIFT, \
CC_AXIM_MON_COMP_VALUE_BIT_SHIFT)
@@ -121,15 +126,6 @@ struct cc_cpp_req {
struct cc_crypto_req {
void (*user_cb)(struct device *dev, void *req, int err);
void *user_arg;
- dma_addr_t ivgen_dma_addr[CC_MAX_IVGEN_DMA_ADDRESSES];
- /* For the first 'ivgen_dma_addr_len' addresses of this array,
- * generated IV would be placed in it by send_request().
- * Same generated IV for all addresses!
- */
- /* Amount of 'ivgen_dma_addr' elements to be filled. */
- unsigned int ivgen_dma_addr_len;
- /* The generated IV size required, 8/16 B allowed. */
- unsigned int ivgen_size;
struct completion seq_compl; /* request completion */
struct cc_cpp_req cpp;
};
@@ -153,7 +149,6 @@ struct cc_drvdata {
void *aead_handle;
void *request_mgr_handle;
void *fips_handle;
- void *ivgen_handle;
void *sram_mgr_handle;
void *debugfs;
struct clk *clk;
@@ -216,6 +211,7 @@ static inline void dump_byte_array(const char *name, const u8 *the_array,
__dump_byte_array(name, the_array, size);
}
+bool cc_wait_for_reset_completion(struct cc_drvdata *drvdata);
int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe);
void fini_cc_regs(struct cc_drvdata *drvdata);
int cc_clk_on(struct cc_drvdata *drvdata);
diff --git a/drivers/crypto/ccree/cc_fips.c b/drivers/crypto/ccree/cc_fips.c
index 5ad3ffb7acaa..4c8bce33abcf 100644
--- a/drivers/crypto/ccree/cc_fips.c
+++ b/drivers/crypto/ccree/cc_fips.c
@@ -3,6 +3,7 @@
#include <linux/kernel.h>
#include <linux/fips.h>
+#include <linux/notifier.h>
#include "cc_driver.h"
#include "cc_fips.h"
@@ -11,6 +12,8 @@ static void fips_dsr(unsigned long devarg);
struct cc_fips_handle {
struct tasklet_struct tasklet;
+ struct notifier_block nb;
+ struct cc_drvdata *drvdata;
};
/* The function called once at driver entry point to check
@@ -21,7 +24,13 @@ static bool cc_get_tee_fips_status(struct cc_drvdata *drvdata)
u32 reg;
reg = cc_ioread(drvdata, CC_REG(GPR_HOST));
- return (reg == (CC_FIPS_SYNC_TEE_STATUS | CC_FIPS_SYNC_MODULE_OK));
+ /* Did the TEE report status? */
+ if (reg & CC_FIPS_SYNC_TEE_STATUS)
+ /* Yes. Is it OK? */
+ return (reg & CC_FIPS_SYNC_MODULE_OK);
+
+ /* No. It's either not in use or will be reported later */
+ return true;
}
/*
@@ -40,6 +49,21 @@ void cc_set_ree_fips_status(struct cc_drvdata *drvdata, bool status)
cc_iowrite(drvdata, CC_REG(HOST_GPR0), val);
}
+/* Push REE side FIPS test failure to TEE side */
+static int cc_ree_fips_failure(struct notifier_block *nb, unsigned long unused1,
+ void *unused2)
+{
+ struct cc_fips_handle *fips_h =
+ container_of(nb, struct cc_fips_handle, nb);
+ struct cc_drvdata *drvdata = fips_h->drvdata;
+ struct device *dev = drvdata_to_dev(drvdata);
+
+ cc_set_ree_fips_status(drvdata, false);
+ dev_info(dev, "Notifying TEE of FIPS test failure...\n");
+
+ return NOTIFY_OK;
+}
+
void cc_fips_fini(struct cc_drvdata *drvdata)
{
struct cc_fips_handle *fips_h = drvdata->fips_handle;
@@ -47,6 +71,8 @@ void cc_fips_fini(struct cc_drvdata *drvdata)
if (drvdata->hw_rev < CC_HW_REV_712 || !fips_h)
return;
+ atomic_notifier_chain_unregister(&fips_fail_notif_chain, &fips_h->nb);
+
/* Kill tasklet */
tasklet_kill(&fips_h->tasklet);
drvdata->fips_handle = NULL;
@@ -118,6 +144,9 @@ int cc_fips_init(struct cc_drvdata *p_drvdata)
dev_dbg(dev, "Initializing fips tasklet\n");
tasklet_init(&fips_h->tasklet, fips_dsr, (unsigned long)p_drvdata);
+ fips_h->drvdata = p_drvdata;
+ fips_h->nb.notifier_call = cc_ree_fips_failure;
+ atomic_notifier_chain_register(&fips_fail_notif_chain, &fips_h->nb);
cc_tee_handle_fips_error(p_drvdata);
diff --git a/drivers/crypto/ccree/cc_hash.c b/drivers/crypto/ccree/cc_hash.c
index a6abe4e3bb0e..bc71bdf44a9f 100644
--- a/drivers/crypto/ccree/cc_hash.c
+++ b/drivers/crypto/ccree/cc_hash.c
@@ -25,27 +25,27 @@ struct cc_hash_handle {
struct list_head hash_list;
};
-static const u32 digest_len_init[] = {
+static const u32 cc_digest_len_init[] = {
0x00000040, 0x00000000, 0x00000000, 0x00000000 };
-static const u32 md5_init[] = {
+static const u32 cc_md5_init[] = {
SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 };
-static const u32 sha1_init[] = {
+static const u32 cc_sha1_init[] = {
SHA1_H4, SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 };
-static const u32 sha224_init[] = {
+static const u32 cc_sha224_init[] = {
SHA224_H7, SHA224_H6, SHA224_H5, SHA224_H4,
SHA224_H3, SHA224_H2, SHA224_H1, SHA224_H0 };
-static const u32 sha256_init[] = {
+static const u32 cc_sha256_init[] = {
SHA256_H7, SHA256_H6, SHA256_H5, SHA256_H4,
SHA256_H3, SHA256_H2, SHA256_H1, SHA256_H0 };
-static const u32 digest_len_sha512_init[] = {
+static const u32 cc_digest_len_sha512_init[] = {
0x00000080, 0x00000000, 0x00000000, 0x00000000 };
-static u64 sha384_init[] = {
+static u64 cc_sha384_init[] = {
SHA384_H7, SHA384_H6, SHA384_H5, SHA384_H4,
SHA384_H3, SHA384_H2, SHA384_H1, SHA384_H0 };
-static u64 sha512_init[] = {
+static u64 cc_sha512_init[] = {
SHA512_H7, SHA512_H6, SHA512_H5, SHA512_H4,
SHA512_H3, SHA512_H2, SHA512_H1, SHA512_H0 };
-static const u32 sm3_init[] = {
+static const u32 cc_sm3_init[] = {
SM3_IVH, SM3_IVG, SM3_IVF, SM3_IVE,
SM3_IVD, SM3_IVC, SM3_IVB, SM3_IVA };
@@ -144,10 +144,11 @@ static void cc_init_req(struct device *dev, struct ahash_req_ctx *state,
if (ctx->hash_mode == DRV_HASH_SHA512 ||
ctx->hash_mode == DRV_HASH_SHA384)
memcpy(state->digest_bytes_len,
- digest_len_sha512_init,
+ cc_digest_len_sha512_init,
ctx->hash_len);
else
- memcpy(state->digest_bytes_len, digest_len_init,
+ memcpy(state->digest_bytes_len,
+ cc_digest_len_init,
ctx->hash_len);
}
@@ -1873,26 +1874,26 @@ int cc_init_hash_sram(struct cc_drvdata *drvdata)
int rc = 0;
/* Copy-to-sram digest-len */
- cc_set_sram_desc(digest_len_init, sram_buff_ofs,
- ARRAY_SIZE(digest_len_init), larval_seq,
+ cc_set_sram_desc(cc_digest_len_init, sram_buff_ofs,
+ ARRAY_SIZE(cc_digest_len_init), larval_seq,
&larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
goto init_digest_const_err;
- sram_buff_ofs += sizeof(digest_len_init);
+ sram_buff_ofs += sizeof(cc_digest_len_init);
larval_seq_len = 0;
if (large_sha_supported) {
/* Copy-to-sram digest-len for sha384/512 */
- cc_set_sram_desc(digest_len_sha512_init, sram_buff_ofs,
- ARRAY_SIZE(digest_len_sha512_init),
+ cc_set_sram_desc(cc_digest_len_sha512_init, sram_buff_ofs,
+ ARRAY_SIZE(cc_digest_len_sha512_init),
larval_seq, &larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
goto init_digest_const_err;
- sram_buff_ofs += sizeof(digest_len_sha512_init);
+ sram_buff_ofs += sizeof(cc_digest_len_sha512_init);
larval_seq_len = 0;
}
@@ -1900,64 +1901,64 @@ int cc_init_hash_sram(struct cc_drvdata *drvdata)
hash_handle->larval_digest_sram_addr = sram_buff_ofs;
/* Copy-to-sram initial SHA* digests */
- cc_set_sram_desc(md5_init, sram_buff_ofs, ARRAY_SIZE(md5_init),
+ cc_set_sram_desc(cc_md5_init, sram_buff_ofs, ARRAY_SIZE(cc_md5_init),
larval_seq, &larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
goto init_digest_const_err;
- sram_buff_ofs += sizeof(md5_init);
+ sram_buff_ofs += sizeof(cc_md5_init);
larval_seq_len = 0;
- cc_set_sram_desc(sha1_init, sram_buff_ofs,
- ARRAY_SIZE(sha1_init), larval_seq,
+ cc_set_sram_desc(cc_sha1_init, sram_buff_ofs,
+ ARRAY_SIZE(cc_sha1_init), larval_seq,
&larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
goto init_digest_const_err;
- sram_buff_ofs += sizeof(sha1_init);
+ sram_buff_ofs += sizeof(cc_sha1_init);
larval_seq_len = 0;
- cc_set_sram_desc(sha224_init, sram_buff_ofs,
- ARRAY_SIZE(sha224_init), larval_seq,
+ cc_set_sram_desc(cc_sha224_init, sram_buff_ofs,
+ ARRAY_SIZE(cc_sha224_init), larval_seq,
&larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
goto init_digest_const_err;
- sram_buff_ofs += sizeof(sha224_init);
+ sram_buff_ofs += sizeof(cc_sha224_init);
larval_seq_len = 0;
- cc_set_sram_desc(sha256_init, sram_buff_ofs,
- ARRAY_SIZE(sha256_init), larval_seq,
+ cc_set_sram_desc(cc_sha256_init, sram_buff_ofs,
+ ARRAY_SIZE(cc_sha256_init), larval_seq,
&larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
goto init_digest_const_err;
- sram_buff_ofs += sizeof(sha256_init);
+ sram_buff_ofs += sizeof(cc_sha256_init);
larval_seq_len = 0;
if (sm3_supported) {
- cc_set_sram_desc(sm3_init, sram_buff_ofs,
- ARRAY_SIZE(sm3_init), larval_seq,
+ cc_set_sram_desc(cc_sm3_init, sram_buff_ofs,
+ ARRAY_SIZE(cc_sm3_init), larval_seq,
&larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
goto init_digest_const_err;
- sram_buff_ofs += sizeof(sm3_init);
+ sram_buff_ofs += sizeof(cc_sm3_init);
larval_seq_len = 0;
}
if (large_sha_supported) {
- cc_set_sram_desc((u32 *)sha384_init, sram_buff_ofs,
- (ARRAY_SIZE(sha384_init) * 2), larval_seq,
+ cc_set_sram_desc((u32 *)cc_sha384_init, sram_buff_ofs,
+ (ARRAY_SIZE(cc_sha384_init) * 2), larval_seq,
&larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
goto init_digest_const_err;
- sram_buff_ofs += sizeof(sha384_init);
+ sram_buff_ofs += sizeof(cc_sha384_init);
larval_seq_len = 0;
- cc_set_sram_desc((u32 *)sha512_init, sram_buff_ofs,
- (ARRAY_SIZE(sha512_init) * 2), larval_seq,
+ cc_set_sram_desc((u32 *)cc_sha512_init, sram_buff_ofs,
+ (ARRAY_SIZE(cc_sha512_init) * 2), larval_seq,
&larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
@@ -1986,8 +1987,8 @@ static void __init cc_swap_dwords(u32 *buf, unsigned long size)
*/
void __init cc_hash_global_init(void)
{
- cc_swap_dwords((u32 *)&sha384_init, (ARRAY_SIZE(sha384_init) * 2));
- cc_swap_dwords((u32 *)&sha512_init, (ARRAY_SIZE(sha512_init) * 2));
+ cc_swap_dwords((u32 *)&cc_sha384_init, (ARRAY_SIZE(cc_sha384_init) * 2));
+ cc_swap_dwords((u32 *)&cc_sha512_init, (ARRAY_SIZE(cc_sha512_init) * 2));
}
int cc_hash_alloc(struct cc_drvdata *drvdata)
@@ -2006,18 +2007,18 @@ int cc_hash_alloc(struct cc_drvdata *drvdata)
INIT_LIST_HEAD(&hash_handle->hash_list);
drvdata->hash_handle = hash_handle;
- sram_size_to_alloc = sizeof(digest_len_init) +
- sizeof(md5_init) +
- sizeof(sha1_init) +
- sizeof(sha224_init) +
- sizeof(sha256_init);
+ sram_size_to_alloc = sizeof(cc_digest_len_init) +
+ sizeof(cc_md5_init) +
+ sizeof(cc_sha1_init) +
+ sizeof(cc_sha224_init) +
+ sizeof(cc_sha256_init);
if (drvdata->hw_rev >= CC_HW_REV_713)
- sram_size_to_alloc += sizeof(sm3_init);
+ sram_size_to_alloc += sizeof(cc_sm3_init);
if (drvdata->hw_rev >= CC_HW_REV_712)
- sram_size_to_alloc += sizeof(digest_len_sha512_init) +
- sizeof(sha384_init) + sizeof(sha512_init);
+ sram_size_to_alloc += sizeof(cc_digest_len_sha512_init) +
+ sizeof(cc_sha384_init) + sizeof(cc_sha512_init);
sram_buff = cc_sram_alloc(drvdata, sram_size_to_alloc);
if (sram_buff == NULL_SRAM_ADDR) {
@@ -2258,22 +2259,22 @@ static const void *cc_larval_digest(struct device *dev, u32 mode)
{
switch (mode) {
case DRV_HASH_MD5:
- return md5_init;
+ return cc_md5_init;
case DRV_HASH_SHA1:
- return sha1_init;
+ return cc_sha1_init;
case DRV_HASH_SHA224:
- return sha224_init;
+ return cc_sha224_init;
case DRV_HASH_SHA256:
- return sha256_init;
+ return cc_sha256_init;
case DRV_HASH_SHA384:
- return sha384_init;
+ return cc_sha384_init;
case DRV_HASH_SHA512:
- return sha512_init;
+ return cc_sha512_init;
case DRV_HASH_SM3:
- return sm3_init;
+ return cc_sm3_init;
default:
dev_err(dev, "Invalid hash mode (%d)\n", mode);
- return md5_init;
+ return cc_md5_init;
}
}
@@ -2301,40 +2302,40 @@ cc_sram_addr_t cc_larval_digest_addr(void *drvdata, u32 mode)
return (hash_handle->larval_digest_sram_addr);
case DRV_HASH_SHA1:
return (hash_handle->larval_digest_sram_addr +
- sizeof(md5_init));
+ sizeof(cc_md5_init));
case DRV_HASH_SHA224:
return (hash_handle->larval_digest_sram_addr +
- sizeof(md5_init) +
- sizeof(sha1_init));
+ sizeof(cc_md5_init) +
+ sizeof(cc_sha1_init));
case DRV_HASH_SHA256:
return (hash_handle->larval_digest_sram_addr +
- sizeof(md5_init) +
- sizeof(sha1_init) +
- sizeof(sha224_init));
+ sizeof(cc_md5_init) +
+ sizeof(cc_sha1_init) +
+ sizeof(cc_sha224_init));
case DRV_HASH_SM3:
return (hash_handle->larval_digest_sram_addr +
- sizeof(md5_init) +
- sizeof(sha1_init) +
- sizeof(sha224_init) +
- sizeof(sha256_init));
+ sizeof(cc_md5_init) +
+ sizeof(cc_sha1_init) +
+ sizeof(cc_sha224_init) +
+ sizeof(cc_sha256_init));
case DRV_HASH_SHA384:
addr = (hash_handle->larval_digest_sram_addr +
- sizeof(md5_init) +
- sizeof(sha1_init) +
- sizeof(sha224_init) +
- sizeof(sha256_init));
+ sizeof(cc_md5_init) +
+ sizeof(cc_sha1_init) +
+ sizeof(cc_sha224_init) +
+ sizeof(cc_sha256_init));
if (sm3_supported)
- addr += sizeof(sm3_init);
+ addr += sizeof(cc_sm3_init);
return addr;
case DRV_HASH_SHA512:
addr = (hash_handle->larval_digest_sram_addr +
- sizeof(md5_init) +
- sizeof(sha1_init) +
- sizeof(sha224_init) +
- sizeof(sha256_init) +
- sizeof(sha384_init));
+ sizeof(cc_md5_init) +
+ sizeof(cc_sha1_init) +
+ sizeof(cc_sha224_init) +
+ sizeof(cc_sha256_init) +
+ sizeof(cc_sha384_init));
if (sm3_supported)
- addr += sizeof(sm3_init);
+ addr += sizeof(cc_sm3_init);
return addr;
default:
dev_err(dev, "Invalid hash mode (%d)\n", mode);
@@ -2360,7 +2361,7 @@ cc_digest_len_addr(void *drvdata, u32 mode)
#if (CC_DEV_SHA_MAX > 256)
case DRV_HASH_SHA384:
case DRV_HASH_SHA512:
- return digest_len_addr + sizeof(digest_len_init);
+ return digest_len_addr + sizeof(cc_digest_len_init);
#endif
default:
return digest_len_addr; /*to avoid kernel crash*/
diff --git a/drivers/crypto/ccree/cc_host_regs.h b/drivers/crypto/ccree/cc_host_regs.h
index d0764147573f..efe3e1d8b87b 100644
--- a/drivers/crypto/ccree/cc_host_regs.h
+++ b/drivers/crypto/ccree/cc_host_regs.h
@@ -114,6 +114,9 @@
#define CC_HOST_ICR_DSCRPTR_WATERMARK_QUEUE0_CLEAR_BIT_SIZE 0x1UL
#define CC_HOST_ICR_AXIM_COMP_INT_CLEAR_BIT_SHIFT 0x17UL
#define CC_HOST_ICR_AXIM_COMP_INT_CLEAR_BIT_SIZE 0x1UL
+#define CC_NVM_IS_IDLE_REG_OFFSET 0x0A10UL
+#define CC_NVM_IS_IDLE_VALUE_BIT_SHIFT 0x0UL
+#define CC_NVM_IS_IDLE_VALUE_BIT_SIZE 0x1UL
#define CC_SECURITY_DISABLED_REG_OFFSET 0x0A1CUL
#define CC_SECURITY_DISABLED_VALUE_BIT_SHIFT 0x0UL
#define CC_SECURITY_DISABLED_VALUE_BIT_SIZE 0x1UL
@@ -203,6 +206,23 @@
#define CC_HOST_POWER_DOWN_EN_REG_OFFSET 0xA78UL
#define CC_HOST_POWER_DOWN_EN_VALUE_BIT_SHIFT 0x0UL
#define CC_HOST_POWER_DOWN_EN_VALUE_BIT_SIZE 0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_REG_OFFSET 0x0A7CUL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_ENGINE_BIT_SHIFT 0x0UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_ENGINE_BIT_SIZE 0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_MAC_ENGINE_BIT_SHIFT 0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_MAC_ENGINE_BIT_SIZE 0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_GHASH_ENGINE_BIT_SHIFT 0x2UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_GHASH_ENGINE_BIT_SIZE 0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_DES_ENGINE_BIT_SHIFT 0x3UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_DES_ENGINE_BIT_SIZE 0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_HASH_ENGINE_BIT_SHIFT 0x4UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_HASH_ENGINE_BIT_SIZE 0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM3_ENGINE_BIT_SHIFT 0x5UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM3_ENGINE_BIT_SIZE 0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM4_ENGINE_BIT_SHIFT 0x6UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM4_ENGINE_BIT_SIZE 0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_OTP_DISCONNECTED_BIT_SHIFT 0x7UL
+#define CC_HOST_REMOVE_INPUT_PINS_OTP_DISCONNECTED_BIT_SIZE 0x1UL
// --------------------------------------
// BLOCK: ID_REGISTERS
// --------------------------------------
diff --git a/drivers/crypto/ccree/cc_ivgen.c b/drivers/crypto/ccree/cc_ivgen.c
deleted file mode 100644
index 99dc69383e20..000000000000
--- a/drivers/crypto/ccree/cc_ivgen.c
+++ /dev/null
@@ -1,276 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */
-
-#include <crypto/ctr.h>
-#include "cc_driver.h"
-#include "cc_ivgen.h"
-#include "cc_request_mgr.h"
-#include "cc_sram_mgr.h"
-#include "cc_buffer_mgr.h"
-
-/* The max. size of pool *MUST* be <= SRAM total size */
-#define CC_IVPOOL_SIZE 1024
-/* The first 32B fraction of pool are dedicated to the
- * next encryption "key" & "IV" for pool regeneration
- */
-#define CC_IVPOOL_META_SIZE (CC_AES_IV_SIZE + AES_KEYSIZE_128)
-#define CC_IVPOOL_GEN_SEQ_LEN 4
-
-/**
- * struct cc_ivgen_ctx -IV pool generation context
- * @pool: the start address of the iv-pool resides in internal RAM
- * @ctr_key_dma: address of pool's encryption key material in internal RAM
- * @ctr_iv_dma: address of pool's counter iv in internal RAM
- * @next_iv_ofs: the offset to the next available IV in pool
- * @pool_meta: virt. address of the initial enc. key/IV
- * @pool_meta_dma: phys. address of the initial enc. key/IV
- */
-struct cc_ivgen_ctx {
- cc_sram_addr_t pool;
- cc_sram_addr_t ctr_key;
- cc_sram_addr_t ctr_iv;
- u32 next_iv_ofs;
- u8 *pool_meta;
- dma_addr_t pool_meta_dma;
-};
-
-/*!
- * Generates CC_IVPOOL_SIZE of random bytes by
- * encrypting 0's using AES128-CTR.
- *
- * \param ivgen iv-pool context
- * \param iv_seq IN/OUT array to the descriptors sequence
- * \param iv_seq_len IN/OUT pointer to the sequence length
- */
-static int cc_gen_iv_pool(struct cc_ivgen_ctx *ivgen_ctx,
- struct cc_hw_desc iv_seq[], unsigned int *iv_seq_len)
-{
- unsigned int idx = *iv_seq_len;
-
- if ((*iv_seq_len + CC_IVPOOL_GEN_SEQ_LEN) > CC_IVPOOL_SEQ_LEN) {
- /* The sequence will be longer than allowed */
- return -EINVAL;
- }
- /* Setup key */
- hw_desc_init(&iv_seq[idx]);
- set_din_sram(&iv_seq[idx], ivgen_ctx->ctr_key, AES_KEYSIZE_128);
- set_setup_mode(&iv_seq[idx], SETUP_LOAD_KEY0);
- set_cipher_config0(&iv_seq[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
- set_flow_mode(&iv_seq[idx], S_DIN_to_AES);
- set_key_size_aes(&iv_seq[idx], CC_AES_128_BIT_KEY_SIZE);
- set_cipher_mode(&iv_seq[idx], DRV_CIPHER_CTR);
- idx++;
-
- /* Setup cipher state */
- hw_desc_init(&iv_seq[idx]);
- set_din_sram(&iv_seq[idx], ivgen_ctx->ctr_iv, CC_AES_IV_SIZE);
- set_cipher_config0(&iv_seq[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
- set_flow_mode(&iv_seq[idx], S_DIN_to_AES);
- set_setup_mode(&iv_seq[idx], SETUP_LOAD_STATE1);
- set_key_size_aes(&iv_seq[idx], CC_AES_128_BIT_KEY_SIZE);
- set_cipher_mode(&iv_seq[idx], DRV_CIPHER_CTR);
- idx++;
-
- /* Perform dummy encrypt to skip first block */
- hw_desc_init(&iv_seq[idx]);
- set_din_const(&iv_seq[idx], 0, CC_AES_IV_SIZE);
- set_dout_sram(&iv_seq[idx], ivgen_ctx->pool, CC_AES_IV_SIZE);
- set_flow_mode(&iv_seq[idx], DIN_AES_DOUT);
- idx++;
-
- /* Generate IV pool */
- hw_desc_init(&iv_seq[idx]);
- set_din_const(&iv_seq[idx], 0, CC_IVPOOL_SIZE);
- set_dout_sram(&iv_seq[idx], ivgen_ctx->pool, CC_IVPOOL_SIZE);
- set_flow_mode(&iv_seq[idx], DIN_AES_DOUT);
- idx++;
-
- *iv_seq_len = idx; /* Update sequence length */
-
- /* queue ordering assures pool readiness */
- ivgen_ctx->next_iv_ofs = CC_IVPOOL_META_SIZE;
-
- return 0;
-}
-
-/*!
- * Generates the initial pool in SRAM.
- * This function should be invoked when resuming driver.
- *
- * \param drvdata
- *
- * \return int Zero for success, negative value otherwise.
- */
-int cc_init_iv_sram(struct cc_drvdata *drvdata)
-{
- struct cc_ivgen_ctx *ivgen_ctx = drvdata->ivgen_handle;
- struct cc_hw_desc iv_seq[CC_IVPOOL_SEQ_LEN];
- unsigned int iv_seq_len = 0;
- int rc;
-
- /* Generate initial enc. key/iv */
- get_random_bytes(ivgen_ctx->pool_meta, CC_IVPOOL_META_SIZE);
-
- /* The first 32B reserved for the enc. Key/IV */
- ivgen_ctx->ctr_key = ivgen_ctx->pool;
- ivgen_ctx->ctr_iv = ivgen_ctx->pool + AES_KEYSIZE_128;
-
- /* Copy initial enc. key and IV to SRAM at a single descriptor */
- hw_desc_init(&iv_seq[iv_seq_len]);
- set_din_type(&iv_seq[iv_seq_len], DMA_DLLI, ivgen_ctx->pool_meta_dma,
- CC_IVPOOL_META_SIZE, NS_BIT);
- set_dout_sram(&iv_seq[iv_seq_len], ivgen_ctx->pool,
- CC_IVPOOL_META_SIZE);
- set_flow_mode(&iv_seq[iv_seq_len], BYPASS);
- iv_seq_len++;
-
- /* Generate initial pool */
- rc = cc_gen_iv_pool(ivgen_ctx, iv_seq, &iv_seq_len);
- if (rc)
- return rc;
-
- /* Fire-and-forget */
- return send_request_init(drvdata, iv_seq, iv_seq_len);
-}
-
-/*!
- * Free iv-pool and ivgen context.
- *
- * \param drvdata
- */
-void cc_ivgen_fini(struct cc_drvdata *drvdata)
-{
- struct cc_ivgen_ctx *ivgen_ctx = drvdata->ivgen_handle;
- struct device *device = &drvdata->plat_dev->dev;
-
- if (!ivgen_ctx)
- return;
-
- if (ivgen_ctx->pool_meta) {
- memset(ivgen_ctx->pool_meta, 0, CC_IVPOOL_META_SIZE);
- dma_free_coherent(device, CC_IVPOOL_META_SIZE,
- ivgen_ctx->pool_meta,
- ivgen_ctx->pool_meta_dma);
- }
-
- ivgen_ctx->pool = NULL_SRAM_ADDR;
-}
-
-/*!
- * Allocates iv-pool and maps resources.
- * This function generates the first IV pool.
- *
- * \param drvdata Driver's private context
- *
- * \return int Zero for success, negative value otherwise.
- */
-int cc_ivgen_init(struct cc_drvdata *drvdata)
-{
- struct cc_ivgen_ctx *ivgen_ctx;
- struct device *device = &drvdata->plat_dev->dev;
- int rc;
-
- /* Allocate "this" context */
- ivgen_ctx = devm_kzalloc(device, sizeof(*ivgen_ctx), GFP_KERNEL);
- if (!ivgen_ctx)
- return -ENOMEM;
-
- drvdata->ivgen_handle = ivgen_ctx;
-
- /* Allocate pool's header for initial enc. key/IV */
- ivgen_ctx->pool_meta = dma_alloc_coherent(device, CC_IVPOOL_META_SIZE,
- &ivgen_ctx->pool_meta_dma,
- GFP_KERNEL);
- if (!ivgen_ctx->pool_meta) {
- dev_err(device, "Not enough memory to allocate DMA of pool_meta (%u B)\n",
- CC_IVPOOL_META_SIZE);
- rc = -ENOMEM;
- goto out;
- }
- /* Allocate IV pool in SRAM */
- ivgen_ctx->pool = cc_sram_alloc(drvdata, CC_IVPOOL_SIZE);
- if (ivgen_ctx->pool == NULL_SRAM_ADDR) {
- dev_err(device, "SRAM pool exhausted\n");
- rc = -ENOMEM;
- goto out;
- }
-
- return cc_init_iv_sram(drvdata);
-
-out:
- cc_ivgen_fini(drvdata);
- return rc;
-}
-
-/*!
- * Acquires 16 Bytes IV from the iv-pool
- *
- * \param drvdata Driver private context
- * \param iv_out_dma Array of physical IV out addresses
- * \param iv_out_dma_len Length of iv_out_dma array (additional elements
- * of iv_out_dma array are ignore)
- * \param iv_out_size May be 8 or 16 bytes long
- * \param iv_seq IN/OUT array to the descriptors sequence
- * \param iv_seq_len IN/OUT pointer to the sequence length
- *
- * \return int Zero for success, negative value otherwise.
- */
-int cc_get_iv(struct cc_drvdata *drvdata, dma_addr_t iv_out_dma[],
- unsigned int iv_out_dma_len, unsigned int iv_out_size,
- struct cc_hw_desc iv_seq[], unsigned int *iv_seq_len)
-{
- struct cc_ivgen_ctx *ivgen_ctx = drvdata->ivgen_handle;
- unsigned int idx = *iv_seq_len;
- struct device *dev = drvdata_to_dev(drvdata);
- unsigned int t;
-
- if (iv_out_size != CC_AES_IV_SIZE &&
- iv_out_size != CTR_RFC3686_IV_SIZE) {
- return -EINVAL;
- }
- if ((iv_out_dma_len + 1) > CC_IVPOOL_SEQ_LEN) {
- /* The sequence will be longer than allowed */
- return -EINVAL;
- }
-
- /* check that number of generated IV is limited to max dma address
- * iv buffer size
- */
- if (iv_out_dma_len > CC_MAX_IVGEN_DMA_ADDRESSES) {
- /* The sequence will be longer than allowed */
- return -EINVAL;
- }
-
- for (t = 0; t < iv_out_dma_len; t++) {
- /* Acquire IV from pool */
- hw_desc_init(&iv_seq[idx]);
- set_din_sram(&iv_seq[idx], (ivgen_ctx->pool +
- ivgen_ctx->next_iv_ofs),
- iv_out_size);
- set_dout_dlli(&iv_seq[idx], iv_out_dma[t], iv_out_size,
- NS_BIT, 0);
- set_flow_mode(&iv_seq[idx], BYPASS);
- idx++;
- }
-
- /* Bypass operation is proceeded by crypto sequence, hence must
- * assure bypass-write-transaction by a memory barrier
- */
- hw_desc_init(&iv_seq[idx]);
- set_din_no_dma(&iv_seq[idx], 0, 0xfffff0);
- set_dout_no_dma(&iv_seq[idx], 0, 0, 1);
- idx++;
-
- *iv_seq_len = idx; /* update seq length */
-
- /* Update iv index */
- ivgen_ctx->next_iv_ofs += iv_out_size;
-
- if ((CC_IVPOOL_SIZE - ivgen_ctx->next_iv_ofs) < CC_AES_IV_SIZE) {
- dev_dbg(dev, "Pool exhausted, regenerating iv-pool\n");
- /* pool is drained -regenerate it! */
- return cc_gen_iv_pool(ivgen_ctx, iv_seq, iv_seq_len);
- }
-
- return 0;
-}
diff --git a/drivers/crypto/ccree/cc_ivgen.h b/drivers/crypto/ccree/cc_ivgen.h
deleted file mode 100644
index a9f5e8bba4f1..000000000000
--- a/drivers/crypto/ccree/cc_ivgen.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */
-
-#ifndef __CC_IVGEN_H__
-#define __CC_IVGEN_H__
-
-#include "cc_hw_queue_defs.h"
-
-#define CC_IVPOOL_SEQ_LEN 8
-
-/*!
- * Allocates iv-pool and maps resources.
- * This function generates the first IV pool.
- *
- * \param drvdata Driver's private context
- *
- * \return int Zero for success, negative value otherwise.
- */
-int cc_ivgen_init(struct cc_drvdata *drvdata);
-
-/*!
- * Free iv-pool and ivgen context.
- *
- * \param drvdata
- */
-void cc_ivgen_fini(struct cc_drvdata *drvdata);
-
-/*!
- * Generates the initial pool in SRAM.
- * This function should be invoked when resuming DX driver.
- *
- * \param drvdata
- *
- * \return int Zero for success, negative value otherwise.
- */
-int cc_init_iv_sram(struct cc_drvdata *drvdata);
-
-/*!
- * Acquires 16 Bytes IV from the iv-pool
- *
- * \param drvdata Driver private context
- * \param iv_out_dma Array of physical IV out addresses
- * \param iv_out_dma_len Length of iv_out_dma array (additional elements of
- * iv_out_dma array are ignore)
- * \param iv_out_size May be 8 or 16 bytes long
- * \param iv_seq IN/OUT array to the descriptors sequence
- * \param iv_seq_len IN/OUT pointer to the sequence length
- *
- * \return int Zero for success, negative value otherwise.
- */
-int cc_get_iv(struct cc_drvdata *drvdata, dma_addr_t iv_out_dma[],
- unsigned int iv_out_dma_len, unsigned int iv_out_size,
- struct cc_hw_desc iv_seq[], unsigned int *iv_seq_len);
-
-#endif /*__CC_IVGEN_H__*/
diff --git a/drivers/crypto/ccree/cc_pm.c b/drivers/crypto/ccree/cc_pm.c
index 2dad9c9543c6..dbc508fb719b 100644
--- a/drivers/crypto/ccree/cc_pm.c
+++ b/drivers/crypto/ccree/cc_pm.c
@@ -8,7 +8,6 @@
#include "cc_buffer_mgr.h"
#include "cc_request_mgr.h"
#include "cc_sram_mgr.h"
-#include "cc_ivgen.h"
#include "cc_hash.h"
#include "cc_pm.h"
#include "cc_fips.h"
@@ -49,6 +48,11 @@ int cc_pm_resume(struct device *dev)
dev_err(dev, "failed getting clock back on. We're toast.\n");
return rc;
}
+ /* wait for Crytpcell reset completion */
+ if (!cc_wait_for_reset_completion(drvdata)) {
+ dev_err(dev, "Cryptocell reset not completed");
+ return -EBUSY;
+ }
cc_iowrite(drvdata, CC_REG(HOST_POWER_DOWN_EN), POWER_DOWN_DISABLE);
rc = init_cc_regs(drvdata, false);
@@ -68,7 +72,6 @@ int cc_pm_resume(struct device *dev)
/* must be after the queue resuming as it uses the HW queue*/
cc_init_hash_sram(drvdata);
- cc_init_iv_sram(drvdata);
return 0;
}
@@ -101,6 +104,12 @@ int cc_pm_put_suspend(struct device *dev)
return rc;
}
+bool cc_pm_is_dev_suspended(struct device *dev)
+{
+ /* check device state using runtime api */
+ return pm_runtime_suspended(dev);
+}
+
int cc_pm_init(struct cc_drvdata *drvdata)
{
struct device *dev = drvdata_to_dev(drvdata);
diff --git a/drivers/crypto/ccree/cc_pm.h b/drivers/crypto/ccree/cc_pm.h
index 6190cdba5dad..a7d98a5da2e1 100644
--- a/drivers/crypto/ccree/cc_pm.h
+++ b/drivers/crypto/ccree/cc_pm.h
@@ -22,6 +22,7 @@ int cc_pm_suspend(struct device *dev);
int cc_pm_resume(struct device *dev);
int cc_pm_get(struct device *dev);
int cc_pm_put_suspend(struct device *dev);
+bool cc_pm_is_dev_suspended(struct device *dev);
#else
@@ -54,6 +55,12 @@ static inline int cc_pm_put_suspend(struct device *dev)
return 0;
}
+static inline bool cc_pm_is_dev_suspended(struct device *dev)
+{
+ /* if PM not supported device is never suspend */
+ return false;
+}
+
#endif
#endif /*__POWER_MGR_H__*/
diff --git a/drivers/crypto/ccree/cc_request_mgr.c b/drivers/crypto/ccree/cc_request_mgr.c
index 0bc6ccb0b899..a947d5a2cf35 100644
--- a/drivers/crypto/ccree/cc_request_mgr.c
+++ b/drivers/crypto/ccree/cc_request_mgr.c
@@ -6,7 +6,6 @@
#include "cc_driver.h"
#include "cc_buffer_mgr.h"
#include "cc_request_mgr.h"
-#include "cc_ivgen.h"
#include "cc_pm.h"
#define CC_MAX_POLL_ITER 10
@@ -281,36 +280,12 @@ static int cc_queues_status(struct cc_drvdata *drvdata,
static int cc_do_send_request(struct cc_drvdata *drvdata,
struct cc_crypto_req *cc_req,
struct cc_hw_desc *desc, unsigned int len,
- bool add_comp, bool ivgen)
+ bool add_comp)
{
struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
unsigned int used_sw_slots;
- unsigned int iv_seq_len = 0;
unsigned int total_seq_len = len; /*initial sequence length*/
- struct cc_hw_desc iv_seq[CC_IVPOOL_SEQ_LEN];
struct device *dev = drvdata_to_dev(drvdata);
- int rc;
-
- if (ivgen) {
- dev_dbg(dev, "Acquire IV from pool into %d DMA addresses %pad, %pad, %pad, IV-size=%u\n",
- cc_req->ivgen_dma_addr_len,
- &cc_req->ivgen_dma_addr[0],
- &cc_req->ivgen_dma_addr[1],
- &cc_req->ivgen_dma_addr[2],
- cc_req->ivgen_size);
-
- /* Acquire IV from pool */
- rc = cc_get_iv(drvdata, cc_req->ivgen_dma_addr,
- cc_req->ivgen_dma_addr_len,
- cc_req->ivgen_size, iv_seq, &iv_seq_len);
-
- if (rc) {
- dev_err(dev, "Failed to generate IV (rc=%d)\n", rc);
- return rc;
- }
-
- total_seq_len += iv_seq_len;
- }
used_sw_slots = ((req_mgr_h->req_queue_head -
req_mgr_h->req_queue_tail) &
@@ -334,8 +309,6 @@ static int cc_do_send_request(struct cc_drvdata *drvdata,
wmb();
/* STAT_PHASE_4: Push sequence */
- if (ivgen)
- enqueue_seq(drvdata, iv_seq, iv_seq_len);
enqueue_seq(drvdata, desc, len);
@@ -380,8 +353,6 @@ static void cc_proc_backlog(struct cc_drvdata *drvdata)
struct cc_bl_item *bli;
struct cc_crypto_req *creq;
void *req;
- bool ivgen;
- unsigned int total_len;
struct device *dev = drvdata_to_dev(drvdata);
int rc;
@@ -406,12 +377,9 @@ static void cc_proc_backlog(struct cc_drvdata *drvdata)
bli->notif = true;
}
- ivgen = !!creq->ivgen_dma_addr_len;
- total_len = bli->len + (ivgen ? CC_IVPOOL_SEQ_LEN : 0);
-
spin_lock(&mgr->hw_lock);
- rc = cc_queues_status(drvdata, mgr, total_len);
+ rc = cc_queues_status(drvdata, mgr, bli->len);
if (rc) {
/*
* There is still not room in the FIFO for
@@ -423,7 +391,7 @@ static void cc_proc_backlog(struct cc_drvdata *drvdata)
}
rc = cc_do_send_request(drvdata, &bli->creq, bli->desc,
- bli->len, false, ivgen);
+ bli->len, false);
spin_unlock(&mgr->hw_lock);
@@ -447,8 +415,6 @@ int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req,
{
int rc;
struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
- bool ivgen = !!cc_req->ivgen_dma_addr_len;
- unsigned int total_len = len + (ivgen ? CC_IVPOOL_SEQ_LEN : 0);
struct device *dev = drvdata_to_dev(drvdata);
bool backlog_ok = req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG;
gfp_t flags = cc_gfp_flags(req);
@@ -461,7 +427,7 @@ int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req,
}
spin_lock_bh(&mgr->hw_lock);
- rc = cc_queues_status(drvdata, mgr, total_len);
+ rc = cc_queues_status(drvdata, mgr, len);
#ifdef CC_DEBUG_FORCE_BACKLOG
if (backlog_ok)
@@ -486,8 +452,7 @@ int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req,
}
if (!rc)
- rc = cc_do_send_request(drvdata, cc_req, desc, len, false,
- ivgen);
+ rc = cc_do_send_request(drvdata, cc_req, desc, len, false);
spin_unlock_bh(&mgr->hw_lock);
return rc;
@@ -527,7 +492,7 @@ int cc_send_sync_request(struct cc_drvdata *drvdata,
reinit_completion(&drvdata->hw_queue_avail);
}
- rc = cc_do_send_request(drvdata, cc_req, desc, len, true, false);
+ rc = cc_do_send_request(drvdata, cc_req, desc, len, true);
spin_unlock_bh(&mgr->hw_lock);
if (rc != -EINPROGRESS) {
diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig
index 4b9b37a130d3..250150560e68 100644
--- a/drivers/crypto/chelsio/Kconfig
+++ b/drivers/crypto/chelsio/Kconfig
@@ -2,6 +2,7 @@
config CRYPTO_DEV_CHELSIO
tristate "Chelsio Crypto Co-processor Driver"
depends on CHELSIO_T4
+ select CRYPTO_LIB_AES
select CRYPTO_SHA1
select CRYPTO_SHA256
select CRYPTO_SHA512
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index 177f572b9589..38ee38b37ae6 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -1023,22 +1023,21 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv,
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
- struct crypto_cipher *cipher;
+ struct crypto_aes_ctx aes;
int ret, i;
u8 *key;
unsigned int keylen;
int round = reqctx->last_req_len / AES_BLOCK_SIZE;
int round8 = round / 8;
- cipher = ablkctx->aes_generic;
memcpy(iv, reqctx->iv, AES_BLOCK_SIZE);
keylen = ablkctx->enckey_len / 2;
key = ablkctx->key + keylen;
- ret = crypto_cipher_setkey(cipher, key, keylen);
+ ret = aes_expandkey(&aes, key, keylen);
if (ret)
- goto out;
- crypto_cipher_encrypt_one(cipher, iv, iv);
+ return ret;
+ aes_encrypt(&aes, iv, iv);
for (i = 0; i < round8; i++)
gf128mul_x8_ble((le128 *)iv, (le128 *)iv);
@@ -1046,9 +1045,10 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv,
gf128mul_x_ble((le128 *)iv, (le128 *)iv);
if (!isfinal)
- crypto_cipher_decrypt_one(cipher, iv, iv);
-out:
- return ret;
+ aes_decrypt(&aes, iv, iv);
+
+ memzero_explicit(&aes, sizeof(aes));
+ return 0;
}
static int chcr_update_cipher_iv(struct ablkcipher_request *req,
@@ -1411,16 +1411,6 @@ static int chcr_cra_init(struct crypto_tfm *tfm)
return PTR_ERR(ablkctx->sw_cipher);
}
- if (get_cryptoalg_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_XTS) {
- /* To update tweak*/
- ablkctx->aes_generic = crypto_alloc_cipher("aes-generic", 0, 0);
- if (IS_ERR(ablkctx->aes_generic)) {
- pr_err("failed to allocate aes cipher for tweak\n");
- return PTR_ERR(ablkctx->aes_generic);
- }
- } else
- ablkctx->aes_generic = NULL;
-
tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx);
return chcr_device_init(crypto_tfm_ctx(tfm));
}
@@ -1451,8 +1441,6 @@ static void chcr_cra_exit(struct crypto_tfm *tfm)
struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
crypto_free_sync_skcipher(ablkctx->sw_cipher);
- if (ablkctx->aes_generic)
- crypto_free_cipher(ablkctx->aes_generic);
}
static int get_alg_config(struct algo_param *params,
@@ -3364,9 +3352,9 @@ static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
{
struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
- struct crypto_cipher *cipher;
unsigned int ck_size;
int ret = 0, key_ctx_size = 0;
+ struct crypto_aes_ctx aes;
aeadctx->enckey_len = 0;
crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
@@ -3409,23 +3397,15 @@ static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
/* Calculate the H = CIPH(K, 0 repeated 16 times).
* It will go in key context
*/
- cipher = crypto_alloc_cipher("aes-generic", 0, 0);
- if (IS_ERR(cipher)) {
- aeadctx->enckey_len = 0;
- ret = -ENOMEM;
- goto out;
- }
-
- ret = crypto_cipher_setkey(cipher, key, keylen);
+ ret = aes_expandkey(&aes, key, keylen);
if (ret) {
aeadctx->enckey_len = 0;
- goto out1;
+ goto out;
}
memset(gctx->ghash_h, 0, AEAD_H_SIZE);
- crypto_cipher_encrypt_one(cipher, gctx->ghash_h, gctx->ghash_h);
+ aes_encrypt(&aes, gctx->ghash_h, gctx->ghash_h);
+ memzero_explicit(&aes, sizeof(aes));
-out1:
- crypto_free_cipher(cipher);
out:
return ret;
}
diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h
index ee20dd899e83..d1e6b51df0ce 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -333,26 +333,26 @@ struct phys_sge_pairs {
};
-static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = {
+static const u32 chcr_sha1_init[SHA1_DIGEST_SIZE / 4] = {
SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,
};
-static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = {
+static const u32 chcr_sha224_init[SHA256_DIGEST_SIZE / 4] = {
SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,
SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,
};
-static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = {
+static const u32 chcr_sha256_init[SHA256_DIGEST_SIZE / 4] = {
SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,
};
-static const u64 sha384_init[SHA512_DIGEST_SIZE / 8] = {
+static const u64 chcr_sha384_init[SHA512_DIGEST_SIZE / 8] = {
SHA384_H0, SHA384_H1, SHA384_H2, SHA384_H3,
SHA384_H4, SHA384_H5, SHA384_H6, SHA384_H7,
};
-static const u64 sha512_init[SHA512_DIGEST_SIZE / 8] = {
+static const u64 chcr_sha512_init[SHA512_DIGEST_SIZE / 8] = {
SHA512_H0, SHA512_H1, SHA512_H2, SHA512_H3,
SHA512_H4, SHA512_H5, SHA512_H6, SHA512_H7,
};
@@ -362,21 +362,21 @@ static inline void copy_hash_init_values(char *key, int digestsize)
u8 i;
__be32 *dkey = (__be32 *)key;
u64 *ldkey = (u64 *)key;
- __be64 *sha384 = (__be64 *)sha384_init;
- __be64 *sha512 = (__be64 *)sha512_init;
+ __be64 *sha384 = (__be64 *)chcr_sha384_init;
+ __be64 *sha512 = (__be64 *)chcr_sha512_init;
switch (digestsize) {
case SHA1_DIGEST_SIZE:
for (i = 0; i < SHA1_INIT_STATE; i++)
- dkey[i] = cpu_to_be32(sha1_init[i]);
+ dkey[i] = cpu_to_be32(chcr_sha1_init[i]);
break;
case SHA224_DIGEST_SIZE:
for (i = 0; i < SHA224_INIT_STATE; i++)
- dkey[i] = cpu_to_be32(sha224_init[i]);
+ dkey[i] = cpu_to_be32(chcr_sha224_init[i]);
break;
case SHA256_DIGEST_SIZE:
for (i = 0; i < SHA256_INIT_STATE; i++)
- dkey[i] = cpu_to_be32(sha256_init[i]);
+ dkey[i] = cpu_to_be32(chcr_sha256_init[i]);
break;
case SHA384_DIGEST_SIZE:
for (i = 0; i < SHA384_INIT_STATE; i++)
diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h
index 655606f2e4d0..993c97e70565 100644
--- a/drivers/crypto/chelsio/chcr_crypto.h
+++ b/drivers/crypto/chelsio/chcr_crypto.h
@@ -172,7 +172,6 @@ static inline struct chcr_context *h_ctx(struct crypto_ahash *tfm)
struct ablk_ctx {
struct crypto_sync_skcipher *sw_cipher;
- struct crypto_cipher *aes_generic;
__be32 key_ctx_hdr;
unsigned int enckey_len;
unsigned char ciph_mode;
diff --git a/drivers/crypto/chelsio/chcr_ipsec.c b/drivers/crypto/chelsio/chcr_ipsec.c
index f429aae72542..24355680f30a 100644
--- a/drivers/crypto/chelsio/chcr_ipsec.c
+++ b/drivers/crypto/chelsio/chcr_ipsec.c
@@ -132,11 +132,11 @@ static inline int chcr_ipsec_setauthsize(struct xfrm_state *x,
static inline int chcr_ipsec_setkey(struct xfrm_state *x,
struct ipsec_sa_entry *sa_entry)
{
- struct crypto_cipher *cipher;
int keylen = (x->aead->alg_key_len + 7) / 8;
unsigned char *key = x->aead->alg_key;
int ck_size, key_ctx_size = 0;
unsigned char ghash_h[AEAD_H_SIZE];
+ struct crypto_aes_ctx aes;
int ret = 0;
if (keylen > 3) {
@@ -170,26 +170,19 @@ static inline int chcr_ipsec_setkey(struct xfrm_state *x,
/* Calculate the H = CIPH(K, 0 repeated 16 times).
* It will go in key context
*/
- cipher = crypto_alloc_cipher("aes-generic", 0, 0);
- if (IS_ERR(cipher)) {
- sa_entry->enckey_len = 0;
- ret = -ENOMEM;
- goto out;
- }
-
- ret = crypto_cipher_setkey(cipher, key, keylen);
+ ret = aes_expandkey(&aes, key, keylen);
if (ret) {
sa_entry->enckey_len = 0;
- goto out1;
+ goto out;
}
memset(ghash_h, 0, AEAD_H_SIZE);
- crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h);
+ aes_encrypt(&aes, ghash_h, ghash_h);
+ memzero_explicit(&aes, sizeof(aes));
+
memcpy(sa_entry->key + (DIV_ROUND_UP(sa_entry->enckey_len, 16) *
16), ghash_h, AEAD_H_SIZE);
sa_entry->kctx_len = ((DIV_ROUND_UP(sa_entry->enckey_len, 16)) << 4) +
AEAD_H_SIZE;
-out1:
- crypto_free_cipher(cipher);
out:
return ret;
}
diff --git a/drivers/crypto/chelsio/chtls/chtls_hw.c b/drivers/crypto/chelsio/chtls/chtls_hw.c
index f2424f4c5f78..2a34035d3cfb 100644
--- a/drivers/crypto/chelsio/chtls/chtls_hw.c
+++ b/drivers/crypto/chelsio/chtls/chtls_hw.c
@@ -213,8 +213,8 @@ static int chtls_key_info(struct chtls_sock *csk,
unsigned char key[AES_KEYSIZE_128];
struct tls12_crypto_info_aes_gcm_128 *gcm_ctx;
unsigned char ghash_h[AEAD_H_SIZE];
- struct crypto_cipher *cipher;
int ck_size, key_ctx_size;
+ struct crypto_aes_ctx aes;
int ret;
gcm_ctx = (struct tls12_crypto_info_aes_gcm_128 *)
@@ -234,18 +234,13 @@ static int chtls_key_info(struct chtls_sock *csk,
/* Calculate the H = CIPH(K, 0 repeated 16 times).
* It will go in key context
*/
- cipher = crypto_alloc_cipher("aes", 0, 0);
- if (IS_ERR(cipher)) {
- ret = -ENOMEM;
- goto out;
- }
-
- ret = crypto_cipher_setkey(cipher, key, keylen);
+ ret = aes_expandkey(&aes, key, keylen);
if (ret)
- goto out1;
+ return ret;
memset(ghash_h, 0, AEAD_H_SIZE);
- crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h);
+ aes_encrypt(&aes, ghash_h, ghash_h);
+ memzero_explicit(&aes, sizeof(aes));
csk->tlshws.keylen = key_ctx_size;
/* Copy the Key context */
@@ -269,10 +264,7 @@ static int chtls_key_info(struct chtls_sock *csk,
/* erase key info from driver */
memset(gcm_ctx->key, 0, keylen);
-out1:
- crypto_free_cipher(cipher);
-out:
- return ret;
+ return 0;
}
static void chtls_set_scmd(struct chtls_sock *csk)
diff --git a/drivers/crypto/chelsio/chtls/chtls_io.c b/drivers/crypto/chelsio/chtls/chtls_io.c
index 551bca6fef24..c70cb5f272cf 100644
--- a/drivers/crypto/chelsio/chtls/chtls_io.c
+++ b/drivers/crypto/chelsio/chtls/chtls_io.c
@@ -1134,7 +1134,9 @@ copy:
}
/* Update the skb. */
if (merge) {
- skb_shinfo(skb)->frags[i - 1].size += copy;
+ skb_frag_size_add(
+ &skb_shinfo(skb)->frags[i - 1],
+ copy);
} else {
skb_fill_page_desc(skb, i, page, off, copy);
if (off + copy < pg_size) {
@@ -1247,7 +1249,7 @@ new_buf:
i = skb_shinfo(skb)->nr_frags;
if (skb_can_coalesce(skb, i, page, offset)) {
- skb_shinfo(skb)->frags[i - 1].size += copy;
+ skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
} else if (i < MAX_SKB_FRAGS) {
get_page(page);
skb_fill_page_desc(skb, i, page, offset, copy);
diff --git a/drivers/crypto/chelsio/chtls/chtls_main.c b/drivers/crypto/chelsio/chtls/chtls_main.c
index 635bb4b447fb..e6df5b95ed47 100644
--- a/drivers/crypto/chelsio/chtls/chtls_main.c
+++ b/drivers/crypto/chelsio/chtls/chtls_main.c
@@ -474,7 +474,8 @@ static int chtls_getsockopt(struct sock *sk, int level, int optname,
struct tls_context *ctx = tls_get_ctx(sk);
if (level != SOL_TLS)
- return ctx->getsockopt(sk, level, optname, optval, optlen);
+ return ctx->sk_proto->getsockopt(sk, level,
+ optname, optval, optlen);
return do_chtls_getsockopt(sk, optval, optlen);
}
@@ -541,7 +542,8 @@ static int chtls_setsockopt(struct sock *sk, int level, int optname,
struct tls_context *ctx = tls_get_ctx(sk);
if (level != SOL_TLS)
- return ctx->setsockopt(sk, level, optname, optval, optlen);
+ return ctx->sk_proto->setsockopt(sk, level,
+ optname, optval, optlen);
return do_chtls_setsockopt(sk, optname, optval, optlen);
}
diff --git a/drivers/crypto/exynos-rng.c b/drivers/crypto/exynos-rng.c
index 2cfabb99cb6e..cbd8ca6e52ee 100644
--- a/drivers/crypto/exynos-rng.c
+++ b/drivers/crypto/exynos-rng.c
@@ -268,7 +268,6 @@ static struct rng_alg exynos_rng_alg = {
static int exynos_rng_probe(struct platform_device *pdev)
{
struct exynos_rng_dev *rng;
- struct resource *res;
int ret;
if (exynos_rng_dev)
@@ -289,8 +288,7 @@ static int exynos_rng_probe(struct platform_device *pdev)
return PTR_ERR(rng->clk);
}
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- rng->mem = devm_ioremap_resource(&pdev->dev, res);
+ rng->mem = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(rng->mem))
return PTR_ERR(rng->mem);
diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c
index 5c3f02e4aece..a18e62df68d9 100644
--- a/drivers/crypto/hifn_795x.c
+++ b/drivers/crypto/hifn_795x.c
@@ -21,7 +21,7 @@
#include <linux/ktime.h>
#include <crypto/algapi.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
static char hifn_pll_ref[sizeof("extNNN")] = "ext";
module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444);
@@ -1939,25 +1939,13 @@ static void hifn_flush(struct hifn_device *dev)
static int hifn_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
unsigned int len)
{
- struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
- struct hifn_context *ctx = crypto_tfm_ctx(tfm);
+ struct hifn_context *ctx = crypto_ablkcipher_ctx(cipher);
struct hifn_device *dev = ctx->dev;
+ int err;
- if (len > HIFN_MAX_CRYPT_KEY_LENGTH) {
- crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
- return -1;
- }
-
- if (len == HIFN_DES_KEY_LENGTH) {
- u32 tmp[DES_EXPKEY_WORDS];
- int ret = des_ekey(tmp, key);
-
- if (unlikely(ret == 0) &&
- (tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
- tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
- return -EINVAL;
- }
- }
+ err = verify_ablkcipher_des_key(cipher, key);
+ if (err)
+ return err;
dev->flags &= ~HIFN_FLAG_OLD_KEY;
@@ -1972,15 +1960,11 @@ static int hifn_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
{
struct hifn_context *ctx = crypto_ablkcipher_ctx(cipher);
struct hifn_device *dev = ctx->dev;
- u32 flags;
int err;
- flags = crypto_ablkcipher_get_flags(cipher);
- err = __des3_verify_key(&flags, key);
- if (unlikely(err)) {
- crypto_ablkcipher_set_flags(cipher, flags);
+ err = verify_ablkcipher_des3_key(cipher, key);
+ if (err)
return err;
- }
dev->flags &= ~HIFN_FLAG_OLD_KEY;
diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig
index 8ca9c503bcb0..ebaf91e0146d 100644
--- a/drivers/crypto/hisilicon/Kconfig
+++ b/drivers/crypto/hisilicon/Kconfig
@@ -4,6 +4,7 @@ config CRYPTO_DEV_HISI_SEC
tristate "Support for Hisilicon SEC crypto block cipher accelerator"
select CRYPTO_BLKCIPHER
select CRYPTO_ALGAPI
+ select CRYPTO_LIB_DES
select SG_SPLIT
depends on ARM64 || COMPILE_TEST
depends on HAS_IOMEM
@@ -12,3 +13,27 @@ config CRYPTO_DEV_HISI_SEC
To compile this as a module, choose M here: the module
will be called hisi_sec.
+
+config CRYPTO_DEV_HISI_QM
+ tristate
+ depends on ARM64 && PCI && PCI_MSI
+ help
+ HiSilicon accelerator engines use a common queue management
+ interface. Specific engine driver may use this module.
+
+config CRYPTO_HISI_SGL
+ tristate
+ depends on ARM64
+ help
+ HiSilicon accelerator engines use a common hardware scatterlist
+ interface for data format. Specific engine driver may use this
+ module.
+
+config CRYPTO_DEV_HISI_ZIP
+ tristate "Support for HiSilicon ZIP accelerator"
+ depends on ARM64 && PCI && PCI_MSI
+ select CRYPTO_DEV_HISI_QM
+ select CRYPTO_HISI_SGL
+ select SG_SPLIT
+ help
+ Support for HiSilicon ZIP Driver
diff --git a/drivers/crypto/hisilicon/Makefile b/drivers/crypto/hisilicon/Makefile
index 463f46ace182..45a279741126 100644
--- a/drivers/crypto/hisilicon/Makefile
+++ b/drivers/crypto/hisilicon/Makefile
@@ -1,2 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_CRYPTO_DEV_HISI_SEC) += sec/
+obj-$(CONFIG_CRYPTO_DEV_HISI_QM) += qm.o
+obj-$(CONFIG_CRYPTO_HISI_SGL) += sgl.o
+obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += zip/
diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c
new file mode 100644
index 000000000000..f975c393a603
--- /dev/null
+++ b/drivers/crypto/hisilicon/qm.c
@@ -0,0 +1,1913 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2019 HiSilicon Limited. */
+#include <asm/page.h>
+#include <linux/bitmap.h>
+#include <linux/debugfs.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/irqreturn.h>
+#include <linux/log2.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
+#include "qm.h"
+
+/* eq/aeq irq enable */
+#define QM_VF_AEQ_INT_SOURCE 0x0
+#define QM_VF_AEQ_INT_MASK 0x4
+#define QM_VF_EQ_INT_SOURCE 0x8
+#define QM_VF_EQ_INT_MASK 0xc
+#define QM_IRQ_NUM_V1 1
+#define QM_IRQ_NUM_PF_V2 4
+#define QM_IRQ_NUM_VF_V2 2
+
+#define QM_EQ_EVENT_IRQ_VECTOR 0
+#define QM_AEQ_EVENT_IRQ_VECTOR 1
+#define QM_ABNORMAL_EVENT_IRQ_VECTOR 3
+
+/* mailbox */
+#define QM_MB_CMD_SQC 0x0
+#define QM_MB_CMD_CQC 0x1
+#define QM_MB_CMD_EQC 0x2
+#define QM_MB_CMD_AEQC 0x3
+#define QM_MB_CMD_SQC_BT 0x4
+#define QM_MB_CMD_CQC_BT 0x5
+#define QM_MB_CMD_SQC_VFT_V2 0x6
+
+#define QM_MB_CMD_SEND_BASE 0x300
+#define QM_MB_EVENT_SHIFT 8
+#define QM_MB_BUSY_SHIFT 13
+#define QM_MB_OP_SHIFT 14
+#define QM_MB_CMD_DATA_ADDR_L 0x304
+#define QM_MB_CMD_DATA_ADDR_H 0x308
+
+/* sqc shift */
+#define QM_SQ_HOP_NUM_SHIFT 0
+#define QM_SQ_PAGE_SIZE_SHIFT 4
+#define QM_SQ_BUF_SIZE_SHIFT 8
+#define QM_SQ_SQE_SIZE_SHIFT 12
+#define QM_SQ_PRIORITY_SHIFT 0
+#define QM_SQ_ORDERS_SHIFT 4
+#define QM_SQ_TYPE_SHIFT 8
+
+#define QM_SQ_TYPE_MASK GENMASK(3, 0)
+
+/* cqc shift */
+#define QM_CQ_HOP_NUM_SHIFT 0
+#define QM_CQ_PAGE_SIZE_SHIFT 4
+#define QM_CQ_BUF_SIZE_SHIFT 8
+#define QM_CQ_CQE_SIZE_SHIFT 12
+#define QM_CQ_PHASE_SHIFT 0
+#define QM_CQ_FLAG_SHIFT 1
+
+#define QM_CQE_PHASE(cqe) ((cqe)->w7 & 0x1)
+#define QM_QC_CQE_SIZE 4
+
+/* eqc shift */
+#define QM_EQE_AEQE_SIZE (2UL << 12)
+#define QM_EQC_PHASE_SHIFT 16
+
+#define QM_EQE_PHASE(eqe) (((eqe)->dw0 >> 16) & 0x1)
+#define QM_EQE_CQN_MASK GENMASK(15, 0)
+
+#define QM_AEQE_PHASE(aeqe) (((aeqe)->dw0 >> 16) & 0x1)
+#define QM_AEQE_TYPE_SHIFT 17
+
+#define QM_DOORBELL_CMD_SQ 0
+#define QM_DOORBELL_CMD_CQ 1
+#define QM_DOORBELL_CMD_EQ 2
+#define QM_DOORBELL_CMD_AEQ 3
+
+#define QM_DOORBELL_BASE_V1 0x340
+#define QM_DB_CMD_SHIFT_V1 16
+#define QM_DB_INDEX_SHIFT_V1 32
+#define QM_DB_PRIORITY_SHIFT_V1 48
+#define QM_DOORBELL_SQ_CQ_BASE_V2 0x1000
+#define QM_DOORBELL_EQ_AEQ_BASE_V2 0x2000
+#define QM_DB_CMD_SHIFT_V2 12
+#define QM_DB_RAND_SHIFT_V2 16
+#define QM_DB_INDEX_SHIFT_V2 32
+#define QM_DB_PRIORITY_SHIFT_V2 48
+
+#define QM_MEM_START_INIT 0x100040
+#define QM_MEM_INIT_DONE 0x100044
+#define QM_VFT_CFG_RDY 0x10006c
+#define QM_VFT_CFG_OP_WR 0x100058
+#define QM_VFT_CFG_TYPE 0x10005c
+#define QM_SQC_VFT 0x0
+#define QM_CQC_VFT 0x1
+#define QM_VFT_CFG 0x100060
+#define QM_VFT_CFG_OP_ENABLE 0x100054
+
+#define QM_VFT_CFG_DATA_L 0x100064
+#define QM_VFT_CFG_DATA_H 0x100068
+#define QM_SQC_VFT_BUF_SIZE (7ULL << 8)
+#define QM_SQC_VFT_SQC_SIZE (5ULL << 12)
+#define QM_SQC_VFT_INDEX_NUMBER (1ULL << 16)
+#define QM_SQC_VFT_START_SQN_SHIFT 28
+#define QM_SQC_VFT_VALID (1ULL << 44)
+#define QM_SQC_VFT_SQN_SHIFT 45
+#define QM_CQC_VFT_BUF_SIZE (7ULL << 8)
+#define QM_CQC_VFT_SQC_SIZE (5ULL << 12)
+#define QM_CQC_VFT_INDEX_NUMBER (1ULL << 16)
+#define QM_CQC_VFT_VALID (1ULL << 28)
+
+#define QM_SQC_VFT_BASE_SHIFT_V2 28
+#define QM_SQC_VFT_BASE_MASK_V2 GENMASK(5, 0)
+#define QM_SQC_VFT_NUM_SHIFT_V2 45
+#define QM_SQC_VFT_NUM_MASK_v2 GENMASK(9, 0)
+
+#define QM_DFX_CNT_CLR_CE 0x100118
+
+#define QM_ABNORMAL_INT_SOURCE 0x100000
+#define QM_ABNORMAL_INT_MASK 0x100004
+#define QM_ABNORMAL_INT_MASK_VALUE 0x1fff
+#define QM_ABNORMAL_INT_STATUS 0x100008
+#define QM_ABNORMAL_INF00 0x100010
+#define QM_FIFO_OVERFLOW_TYPE 0xc0
+#define QM_FIFO_OVERFLOW_TYPE_SHIFT 6
+#define QM_FIFO_OVERFLOW_VF 0x3f
+#define QM_ABNORMAL_INF01 0x100014
+#define QM_DB_TIMEOUT_TYPE 0xc0
+#define QM_DB_TIMEOUT_TYPE_SHIFT 6
+#define QM_DB_TIMEOUT_VF 0x3f
+#define QM_RAS_CE_ENABLE 0x1000ec
+#define QM_RAS_FE_ENABLE 0x1000f0
+#define QM_RAS_NFE_ENABLE 0x1000f4
+#define QM_RAS_CE_THRESHOLD 0x1000f8
+#define QM_RAS_CE_TIMES_PER_IRQ 1
+#define QM_RAS_MSI_INT_SEL 0x1040f4
+
+#define QM_CACHE_WB_START 0x204
+#define QM_CACHE_WB_DONE 0x208
+
+#define PCI_BAR_2 2
+#define QM_SQE_DATA_ALIGN_MASK GENMASK(6, 0)
+#define QMC_ALIGN(sz) ALIGN(sz, 32)
+
+#define QM_DBG_TMP_BUF_LEN 22
+
+#define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \
+ (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \
+ ((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT) | \
+ ((buf_sz) << QM_CQ_BUF_SIZE_SHIFT) | \
+ ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
+
+#define QM_MK_CQC_DW3_V2(cqe_sz) \
+ ((QM_Q_DEPTH - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
+
+#define QM_MK_SQC_W13(priority, orders, alg_type) \
+ (((priority) << QM_SQ_PRIORITY_SHIFT) | \
+ ((orders) << QM_SQ_ORDERS_SHIFT) | \
+ (((alg_type) & QM_SQ_TYPE_MASK) << QM_SQ_TYPE_SHIFT))
+
+#define QM_MK_SQC_DW3_V1(hop_num, pg_sz, buf_sz, sqe_sz) \
+ (((hop_num) << QM_SQ_HOP_NUM_SHIFT) | \
+ ((pg_sz) << QM_SQ_PAGE_SIZE_SHIFT) | \
+ ((buf_sz) << QM_SQ_BUF_SIZE_SHIFT) | \
+ ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
+
+#define QM_MK_SQC_DW3_V2(sqe_sz) \
+ ((QM_Q_DEPTH - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
+
+#define INIT_QC_COMMON(qc, base, pasid) do { \
+ (qc)->head = 0; \
+ (qc)->tail = 0; \
+ (qc)->base_l = lower_32_bits(base); \
+ (qc)->base_h = upper_32_bits(base); \
+ (qc)->dw3 = 0; \
+ (qc)->w8 = 0; \
+ (qc)->rsvd0 = 0; \
+ (qc)->pasid = pasid; \
+ (qc)->w11 = 0; \
+ (qc)->rsvd1 = 0; \
+} while (0)
+
+enum vft_type {
+ SQC_VFT = 0,
+ CQC_VFT,
+};
+
+struct qm_cqe {
+ __le32 rsvd0;
+ __le16 cmd_id;
+ __le16 rsvd1;
+ __le16 sq_head;
+ __le16 sq_num;
+ __le16 rsvd2;
+ __le16 w7;
+};
+
+struct qm_eqe {
+ __le32 dw0;
+};
+
+struct qm_aeqe {
+ __le32 dw0;
+};
+
+struct qm_sqc {
+ __le16 head;
+ __le16 tail;
+ __le32 base_l;
+ __le32 base_h;
+ __le32 dw3;
+ __le16 w8;
+ __le16 rsvd0;
+ __le16 pasid;
+ __le16 w11;
+ __le16 cq_num;
+ __le16 w13;
+ __le32 rsvd1;
+};
+
+struct qm_cqc {
+ __le16 head;
+ __le16 tail;
+ __le32 base_l;
+ __le32 base_h;
+ __le32 dw3;
+ __le16 w8;
+ __le16 rsvd0;
+ __le16 pasid;
+ __le16 w11;
+ __le32 dw6;
+ __le32 rsvd1;
+};
+
+struct qm_eqc {
+ __le16 head;
+ __le16 tail;
+ __le32 base_l;
+ __le32 base_h;
+ __le32 dw3;
+ __le32 rsvd[2];
+ __le32 dw6;
+};
+
+struct qm_aeqc {
+ __le16 head;
+ __le16 tail;
+ __le32 base_l;
+ __le32 base_h;
+ __le32 dw3;
+ __le32 rsvd[2];
+ __le32 dw6;
+};
+
+struct qm_mailbox {
+ __le16 w0;
+ __le16 queue_num;
+ __le32 base_l;
+ __le32 base_h;
+ __le32 rsvd;
+};
+
+struct qm_doorbell {
+ __le16 queue_num;
+ __le16 cmd;
+ __le16 index;
+ __le16 priority;
+};
+
+struct hisi_qm_hw_ops {
+ int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number);
+ void (*qm_db)(struct hisi_qm *qm, u16 qn,
+ u8 cmd, u16 index, u8 priority);
+ u32 (*get_irq_num)(struct hisi_qm *qm);
+ int (*debug_init)(struct hisi_qm *qm);
+ void (*hw_error_init)(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe,
+ u32 msi);
+ pci_ers_result_t (*hw_error_handle)(struct hisi_qm *qm);
+};
+
+static const char * const qm_debug_file_name[] = {
+ [CURRENT_Q] = "current_q",
+ [CLEAR_ENABLE] = "clear_enable",
+};
+
+struct hisi_qm_hw_error {
+ u32 int_msk;
+ const char *msg;
+};
+
+static const struct hisi_qm_hw_error qm_hw_error[] = {
+ { .int_msk = BIT(0), .msg = "qm_axi_rresp" },
+ { .int_msk = BIT(1), .msg = "qm_axi_bresp" },
+ { .int_msk = BIT(2), .msg = "qm_ecc_mbit" },
+ { .int_msk = BIT(3), .msg = "qm_ecc_1bit" },
+ { .int_msk = BIT(4), .msg = "qm_acc_get_task_timeout" },
+ { .int_msk = BIT(5), .msg = "qm_acc_do_task_timeout" },
+ { .int_msk = BIT(6), .msg = "qm_acc_wb_not_ready_timeout" },
+ { .int_msk = BIT(7), .msg = "qm_sq_cq_vf_invalid" },
+ { .int_msk = BIT(8), .msg = "qm_cq_vf_invalid" },
+ { .int_msk = BIT(9), .msg = "qm_sq_vf_invalid" },
+ { .int_msk = BIT(10), .msg = "qm_db_timeout" },
+ { .int_msk = BIT(11), .msg = "qm_of_fifo_of" },
+ { .int_msk = BIT(12), .msg = "qm_db_random_invalid" },
+ { /* sentinel */ }
+};
+
+static const char * const qm_db_timeout[] = {
+ "sq", "cq", "eq", "aeq",
+};
+
+static const char * const qm_fifo_overflow[] = {
+ "cq", "eq", "aeq",
+};
+
+/* return 0 mailbox ready, -ETIMEDOUT hardware timeout */
+static int qm_wait_mb_ready(struct hisi_qm *qm)
+{
+ u32 val;
+
+ return readl_relaxed_poll_timeout(qm->io_base + QM_MB_CMD_SEND_BASE,
+ val, !((val >> QM_MB_BUSY_SHIFT) &
+ 0x1), 10, 1000);
+}
+
+/* 128 bit should be written to hardware at one time to trigger a mailbox */
+static void qm_mb_write(struct hisi_qm *qm, const void *src)
+{
+ void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE;
+ unsigned long tmp0 = 0, tmp1 = 0;
+
+ asm volatile("ldp %0, %1, %3\n"
+ "stp %0, %1, %2\n"
+ "dsb sy\n"
+ : "=&r" (tmp0),
+ "=&r" (tmp1),
+ "+Q" (*((char *)fun_base))
+ : "Q" (*((char *)src))
+ : "memory");
+}
+
+static int qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue,
+ bool op)
+{
+ struct qm_mailbox mailbox;
+ int ret = 0;
+
+ dev_dbg(&qm->pdev->dev, "QM mailbox request to q%u: %u-%llx\n",
+ queue, cmd, (unsigned long long)dma_addr);
+
+ mailbox.w0 = cmd |
+ (op ? 0x1 << QM_MB_OP_SHIFT : 0) |
+ (0x1 << QM_MB_BUSY_SHIFT);
+ mailbox.queue_num = queue;
+ mailbox.base_l = lower_32_bits(dma_addr);
+ mailbox.base_h = upper_32_bits(dma_addr);
+ mailbox.rsvd = 0;
+
+ mutex_lock(&qm->mailbox_lock);
+
+ if (unlikely(qm_wait_mb_ready(qm))) {
+ ret = -EBUSY;
+ dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n");
+ goto busy_unlock;
+ }
+
+ qm_mb_write(qm, &mailbox);
+
+ if (unlikely(qm_wait_mb_ready(qm))) {
+ ret = -EBUSY;
+ dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n");
+ goto busy_unlock;
+ }
+
+busy_unlock:
+ mutex_unlock(&qm->mailbox_lock);
+
+ return ret;
+}
+
+static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
+{
+ u64 doorbell;
+
+ doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V1) |
+ ((u64)index << QM_DB_INDEX_SHIFT_V1) |
+ ((u64)priority << QM_DB_PRIORITY_SHIFT_V1);
+
+ writeq(doorbell, qm->io_base + QM_DOORBELL_BASE_V1);
+}
+
+static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
+{
+ u64 doorbell;
+ u64 dbase;
+ u16 randata = 0;
+
+ if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ)
+ dbase = QM_DOORBELL_SQ_CQ_BASE_V2;
+ else
+ dbase = QM_DOORBELL_EQ_AEQ_BASE_V2;
+
+ doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) |
+ ((u64)randata << QM_DB_RAND_SHIFT_V2) |
+ ((u64)index << QM_DB_INDEX_SHIFT_V2) |
+ ((u64)priority << QM_DB_PRIORITY_SHIFT_V2);
+
+ writeq(doorbell, qm->io_base + dbase);
+}
+
+static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
+{
+ dev_dbg(&qm->pdev->dev, "QM doorbell request: qn=%u, cmd=%u, index=%u\n",
+ qn, cmd, index);
+
+ qm->ops->qm_db(qm, qn, cmd, index, priority);
+}
+
+static int qm_dev_mem_reset(struct hisi_qm *qm)
+{
+ u32 val;
+
+ writel(0x1, qm->io_base + QM_MEM_START_INIT);
+ return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val,
+ val & BIT(0), 10, 1000);
+}
+
+static u32 qm_get_irq_num_v1(struct hisi_qm *qm)
+{
+ return QM_IRQ_NUM_V1;
+}
+
+static u32 qm_get_irq_num_v2(struct hisi_qm *qm)
+{
+ if (qm->fun_type == QM_HW_PF)
+ return QM_IRQ_NUM_PF_V2;
+ else
+ return QM_IRQ_NUM_VF_V2;
+}
+
+static struct hisi_qp *qm_to_hisi_qp(struct hisi_qm *qm, struct qm_eqe *eqe)
+{
+ u16 cqn = eqe->dw0 & QM_EQE_CQN_MASK;
+
+ return qm->qp_array[cqn];
+}
+
+static void qm_cq_head_update(struct hisi_qp *qp)
+{
+ if (qp->qp_status.cq_head == QM_Q_DEPTH - 1) {
+ qp->qp_status.cqc_phase = !qp->qp_status.cqc_phase;
+ qp->qp_status.cq_head = 0;
+ } else {
+ qp->qp_status.cq_head++;
+ }
+}
+
+static void qm_poll_qp(struct hisi_qp *qp, struct hisi_qm *qm)
+{
+ struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head;
+
+ if (qp->req_cb) {
+ while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) {
+ dma_rmb();
+ qp->req_cb(qp, qp->sqe + qm->sqe_size * cqe->sq_head);
+ qm_cq_head_update(qp);
+ cqe = qp->cqe + qp->qp_status.cq_head;
+ qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ,
+ qp->qp_status.cq_head, 0);
+ atomic_dec(&qp->qp_status.used);
+ }
+
+ /* set c_flag */
+ qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ,
+ qp->qp_status.cq_head, 1);
+ }
+}
+
+static void qm_qp_work_func(struct work_struct *work)
+{
+ struct hisi_qp *qp;
+
+ qp = container_of(work, struct hisi_qp, work);
+ qm_poll_qp(qp, qp->qm);
+}
+
+static irqreturn_t qm_irq_handler(int irq, void *data)
+{
+ struct hisi_qm *qm = data;
+ struct qm_eqe *eqe = qm->eqe + qm->status.eq_head;
+ struct hisi_qp *qp;
+ int eqe_num = 0;
+
+ while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) {
+ eqe_num++;
+ qp = qm_to_hisi_qp(qm, eqe);
+ if (qp)
+ queue_work(qp->wq, &qp->work);
+
+ if (qm->status.eq_head == QM_Q_DEPTH - 1) {
+ qm->status.eqc_phase = !qm->status.eqc_phase;
+ eqe = qm->eqe;
+ qm->status.eq_head = 0;
+ } else {
+ eqe++;
+ qm->status.eq_head++;
+ }
+
+ if (eqe_num == QM_Q_DEPTH / 2 - 1) {
+ eqe_num = 0;
+ qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
+ }
+ }
+
+ qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
+
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t qm_irq(int irq, void *data)
+{
+ struct hisi_qm *qm = data;
+
+ if (readl(qm->io_base + QM_VF_EQ_INT_SOURCE))
+ return qm_irq_handler(irq, data);
+
+ dev_err(&qm->pdev->dev, "invalid int source\n");
+ qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
+
+ return IRQ_NONE;
+}
+
+static irqreturn_t qm_aeq_irq(int irq, void *data)
+{
+ struct hisi_qm *qm = data;
+ struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head;
+ u32 type;
+
+ if (!readl(qm->io_base + QM_VF_AEQ_INT_SOURCE))
+ return IRQ_NONE;
+
+ while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) {
+ type = aeqe->dw0 >> QM_AEQE_TYPE_SHIFT;
+ if (type < ARRAY_SIZE(qm_fifo_overflow))
+ dev_err(&qm->pdev->dev, "%s overflow\n",
+ qm_fifo_overflow[type]);
+ else
+ dev_err(&qm->pdev->dev, "unknown error type %d\n",
+ type);
+
+ if (qm->status.aeq_head == QM_Q_DEPTH - 1) {
+ qm->status.aeqc_phase = !qm->status.aeqc_phase;
+ aeqe = qm->aeqe;
+ qm->status.aeq_head = 0;
+ } else {
+ aeqe++;
+ qm->status.aeq_head++;
+ }
+
+ qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
+ }
+
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t qm_abnormal_irq(int irq, void *data)
+{
+ const struct hisi_qm_hw_error *err = qm_hw_error;
+ struct hisi_qm *qm = data;
+ struct device *dev = &qm->pdev->dev;
+ u32 error_status, tmp;
+
+ /* read err sts */
+ tmp = readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
+ error_status = qm->msi_mask & tmp;
+
+ while (err->msg) {
+ if (err->int_msk & error_status)
+ dev_err(dev, "%s [error status=0x%x] found\n",
+ err->msg, err->int_msk);
+
+ err++;
+ }
+
+ /* clear err sts */
+ writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE);
+
+ return IRQ_HANDLED;
+}
+
+static int qm_irq_register(struct hisi_qm *qm)
+{
+ struct pci_dev *pdev = qm->pdev;
+ int ret;
+
+ ret = request_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR),
+ qm_irq, IRQF_SHARED, qm->dev_name, qm);
+ if (ret)
+ return ret;
+
+ if (qm->ver == QM_HW_V2) {
+ ret = request_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR),
+ qm_aeq_irq, IRQF_SHARED, qm->dev_name, qm);
+ if (ret)
+ goto err_aeq_irq;
+
+ if (qm->fun_type == QM_HW_PF) {
+ ret = request_irq(pci_irq_vector(pdev,
+ QM_ABNORMAL_EVENT_IRQ_VECTOR),
+ qm_abnormal_irq, IRQF_SHARED,
+ qm->dev_name, qm);
+ if (ret)
+ goto err_abonormal_irq;
+ }
+ }
+
+ return 0;
+
+err_abonormal_irq:
+ free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm);
+err_aeq_irq:
+ free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm);
+ return ret;
+}
+
+static void qm_irq_unregister(struct hisi_qm *qm)
+{
+ struct pci_dev *pdev = qm->pdev;
+
+ free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm);
+
+ if (qm->ver == QM_HW_V2) {
+ free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm);
+
+ if (qm->fun_type == QM_HW_PF)
+ free_irq(pci_irq_vector(pdev,
+ QM_ABNORMAL_EVENT_IRQ_VECTOR), qm);
+ }
+}
+
+static void qm_init_qp_status(struct hisi_qp *qp)
+{
+ struct hisi_qp_status *qp_status = &qp->qp_status;
+
+ qp_status->sq_tail = 0;
+ qp_status->cq_head = 0;
+ qp_status->cqc_phase = 1;
+ qp_status->flags = 0;
+}
+
+static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base,
+ u32 number)
+{
+ u64 tmp = 0;
+
+ if (number > 0) {
+ switch (type) {
+ case SQC_VFT:
+ switch (qm->ver) {
+ case QM_HW_V1:
+ tmp = QM_SQC_VFT_BUF_SIZE |
+ QM_SQC_VFT_SQC_SIZE |
+ QM_SQC_VFT_INDEX_NUMBER |
+ QM_SQC_VFT_VALID |
+ (u64)base << QM_SQC_VFT_START_SQN_SHIFT;
+ break;
+ case QM_HW_V2:
+ tmp = (u64)base << QM_SQC_VFT_START_SQN_SHIFT |
+ QM_SQC_VFT_VALID |
+ (u64)(number - 1) << QM_SQC_VFT_SQN_SHIFT;
+ break;
+ case QM_HW_UNKNOWN:
+ break;
+ }
+ break;
+ case CQC_VFT:
+ switch (qm->ver) {
+ case QM_HW_V1:
+ tmp = QM_CQC_VFT_BUF_SIZE |
+ QM_CQC_VFT_SQC_SIZE |
+ QM_CQC_VFT_INDEX_NUMBER |
+ QM_CQC_VFT_VALID;
+ break;
+ case QM_HW_V2:
+ tmp = QM_CQC_VFT_VALID;
+ break;
+ case QM_HW_UNKNOWN:
+ break;
+ }
+ break;
+ }
+ }
+
+ writel(lower_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_L);
+ writel(upper_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_H);
+}
+
+static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type,
+ u32 fun_num, u32 base, u32 number)
+{
+ unsigned int val;
+ int ret;
+
+ ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
+ val & BIT(0), 10, 1000);
+ if (ret)
+ return ret;
+
+ writel(0x0, qm->io_base + QM_VFT_CFG_OP_WR);
+ writel(type, qm->io_base + QM_VFT_CFG_TYPE);
+ writel(fun_num, qm->io_base + QM_VFT_CFG);
+
+ qm_vft_data_cfg(qm, type, base, number);
+
+ writel(0x0, qm->io_base + QM_VFT_CFG_RDY);
+ writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);
+
+ return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
+ val & BIT(0), 10, 1000);
+}
+
+/* The config should be conducted after qm_dev_mem_reset() */
+static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
+ u32 number)
+{
+ int ret, i;
+
+ for (i = SQC_VFT; i <= CQC_VFT; i++) {
+ ret = qm_set_vft_common(qm, i, fun_num, base, number);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number)
+{
+ u64 sqc_vft;
+ int ret;
+
+ ret = qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1);
+ if (ret)
+ return ret;
+
+ sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
+ ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32);
+ *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
+ *number = (QM_SQC_VFT_NUM_MASK_v2 &
+ (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
+
+ return 0;
+}
+
+static struct hisi_qm *file_to_qm(struct debugfs_file *file)
+{
+ struct qm_debug *debug = file->debug;
+
+ return container_of(debug, struct hisi_qm, debug);
+}
+
+static u32 current_q_read(struct debugfs_file *file)
+{
+ struct hisi_qm *qm = file_to_qm(file);
+
+ return readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) >> QM_DFX_QN_SHIFT;
+}
+
+static int current_q_write(struct debugfs_file *file, u32 val)
+{
+ struct hisi_qm *qm = file_to_qm(file);
+ u32 tmp;
+
+ if (val >= qm->debug.curr_qm_qp_num)
+ return -EINVAL;
+
+ tmp = val << QM_DFX_QN_SHIFT |
+ (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_FUN_MASK);
+ writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
+
+ tmp = val << QM_DFX_QN_SHIFT |
+ (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_FUN_MASK);
+ writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
+
+ return 0;
+}
+
+static u32 clear_enable_read(struct debugfs_file *file)
+{
+ struct hisi_qm *qm = file_to_qm(file);
+
+ return readl(qm->io_base + QM_DFX_CNT_CLR_CE);
+}
+
+/* rd_clr_ctrl 1 enable read clear, otherwise 0 disable it */
+static int clear_enable_write(struct debugfs_file *file, u32 rd_clr_ctrl)
+{
+ struct hisi_qm *qm = file_to_qm(file);
+
+ if (rd_clr_ctrl > 1)
+ return -EINVAL;
+
+ writel(rd_clr_ctrl, qm->io_base + QM_DFX_CNT_CLR_CE);
+
+ return 0;
+}
+
+static ssize_t qm_debug_read(struct file *filp, char __user *buf,
+ size_t count, loff_t *pos)
+{
+ struct debugfs_file *file = filp->private_data;
+ enum qm_debug_file index = file->index;
+ char tbuf[QM_DBG_TMP_BUF_LEN];
+ u32 val;
+ int ret;
+
+ mutex_lock(&file->lock);
+ switch (index) {
+ case CURRENT_Q:
+ val = current_q_read(file);
+ break;
+ case CLEAR_ENABLE:
+ val = clear_enable_read(file);
+ break;
+ default:
+ mutex_unlock(&file->lock);
+ return -EINVAL;
+ }
+ mutex_unlock(&file->lock);
+ ret = sprintf(tbuf, "%u\n", val);
+ return simple_read_from_buffer(buf, count, pos, tbuf, ret);
+}
+
+static ssize_t qm_debug_write(struct file *filp, const char __user *buf,
+ size_t count, loff_t *pos)
+{
+ struct debugfs_file *file = filp->private_data;
+ enum qm_debug_file index = file->index;
+ unsigned long val;
+ char tbuf[QM_DBG_TMP_BUF_LEN];
+ int len, ret;
+
+ if (*pos != 0)
+ return 0;
+
+ if (count >= QM_DBG_TMP_BUF_LEN)
+ return -ENOSPC;
+
+ len = simple_write_to_buffer(tbuf, QM_DBG_TMP_BUF_LEN - 1, pos, buf,
+ count);
+ if (len < 0)
+ return len;
+
+ tbuf[len] = '\0';
+ if (kstrtoul(tbuf, 0, &val))
+ return -EFAULT;
+
+ mutex_lock(&file->lock);
+ switch (index) {
+ case CURRENT_Q:
+ ret = current_q_write(file, val);
+ if (ret)
+ goto err_input;
+ break;
+ case CLEAR_ENABLE:
+ ret = clear_enable_write(file, val);
+ if (ret)
+ goto err_input;
+ break;
+ default:
+ ret = -EINVAL;
+ goto err_input;
+ }
+ mutex_unlock(&file->lock);
+
+ return count;
+
+err_input:
+ mutex_unlock(&file->lock);
+ return ret;
+}
+
+static const struct file_operations qm_debug_fops = {
+ .owner = THIS_MODULE,
+ .open = simple_open,
+ .read = qm_debug_read,
+ .write = qm_debug_write,
+};
+
+struct qm_dfx_registers {
+ char *reg_name;
+ u64 reg_offset;
+};
+
+#define CNT_CYC_REGS_NUM 10
+static struct qm_dfx_registers qm_dfx_regs[] = {
+ /* XXX_CNT are reading clear register */
+ {"QM_ECC_1BIT_CNT ", 0x104000ull},
+ {"QM_ECC_MBIT_CNT ", 0x104008ull},
+ {"QM_DFX_MB_CNT ", 0x104018ull},
+ {"QM_DFX_DB_CNT ", 0x104028ull},
+ {"QM_DFX_SQE_CNT ", 0x104038ull},
+ {"QM_DFX_CQE_CNT ", 0x104048ull},
+ {"QM_DFX_SEND_SQE_TO_ACC_CNT ", 0x104050ull},
+ {"QM_DFX_WB_SQE_FROM_ACC_CNT ", 0x104058ull},
+ {"QM_DFX_ACC_FINISH_CNT ", 0x104060ull},
+ {"QM_DFX_CQE_ERR_CNT ", 0x1040b4ull},
+ {"QM_DFX_FUNS_ACTIVE_ST ", 0x200ull},
+ {"QM_ECC_1BIT_INF ", 0x104004ull},
+ {"QM_ECC_MBIT_INF ", 0x10400cull},
+ {"QM_DFX_ACC_RDY_VLD0 ", 0x1040a0ull},
+ {"QM_DFX_ACC_RDY_VLD1 ", 0x1040a4ull},
+ {"QM_DFX_AXI_RDY_VLD ", 0x1040a8ull},
+ {"QM_DFX_FF_ST0 ", 0x1040c8ull},
+ {"QM_DFX_FF_ST1 ", 0x1040ccull},
+ {"QM_DFX_FF_ST2 ", 0x1040d0ull},
+ {"QM_DFX_FF_ST3 ", 0x1040d4ull},
+ {"QM_DFX_FF_ST4 ", 0x1040d8ull},
+ {"QM_DFX_FF_ST5 ", 0x1040dcull},
+ {"QM_DFX_FF_ST6 ", 0x1040e0ull},
+ {"QM_IN_IDLE_ST ", 0x1040e4ull},
+ { NULL, 0}
+};
+
+static struct qm_dfx_registers qm_vf_dfx_regs[] = {
+ {"QM_DFX_FUNS_ACTIVE_ST ", 0x200ull},
+ { NULL, 0}
+};
+
+static int qm_regs_show(struct seq_file *s, void *unused)
+{
+ struct hisi_qm *qm = s->private;
+ struct qm_dfx_registers *regs;
+ u32 val;
+
+ if (qm->fun_type == QM_HW_PF)
+ regs = qm_dfx_regs;
+ else
+ regs = qm_vf_dfx_regs;
+
+ while (regs->reg_name) {
+ val = readl(qm->io_base + regs->reg_offset);
+ seq_printf(s, "%s= 0x%08x\n", regs->reg_name, val);
+ regs++;
+ }
+
+ return 0;
+}
+
+static int qm_regs_open(struct inode *inode, struct file *file)
+{
+ return single_open(file, qm_regs_show, inode->i_private);
+}
+
+static const struct file_operations qm_regs_fops = {
+ .owner = THIS_MODULE,
+ .open = qm_regs_open,
+ .read = seq_read,
+ .release = single_release,
+};
+
+static int qm_create_debugfs_file(struct hisi_qm *qm, enum qm_debug_file index)
+{
+ struct dentry *qm_d = qm->debug.qm_d, *tmp;
+ struct debugfs_file *file = qm->debug.files + index;
+
+ tmp = debugfs_create_file(qm_debug_file_name[index], 0600, qm_d, file,
+ &qm_debug_fops);
+ if (IS_ERR(tmp))
+ return -ENOENT;
+
+ file->index = index;
+ mutex_init(&file->lock);
+ file->debug = &qm->debug;
+
+ return 0;
+}
+
+static void qm_hw_error_init_v1(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe,
+ u32 msi)
+{
+ dev_info(&qm->pdev->dev,
+ "QM v%d does not support hw error handle\n", qm->ver);
+
+ writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK);
+}
+
+static void qm_hw_error_init_v2(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe,
+ u32 msi)
+{
+ u32 irq_enable = ce | nfe | fe | msi;
+ u32 irq_unmask = ~irq_enable;
+
+ qm->error_mask = ce | nfe | fe;
+ qm->msi_mask = msi;
+
+ /* configure error type */
+ writel(ce, qm->io_base + QM_RAS_CE_ENABLE);
+ writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD);
+ writel(nfe, qm->io_base + QM_RAS_NFE_ENABLE);
+ writel(fe, qm->io_base + QM_RAS_FE_ENABLE);
+
+ /* use RAS irq default, so only set QM_RAS_MSI_INT_SEL for MSI */
+ writel(msi, qm->io_base + QM_RAS_MSI_INT_SEL);
+
+ irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
+ writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK);
+}
+
+static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status)
+{
+ const struct hisi_qm_hw_error *err = qm_hw_error;
+ struct device *dev = &qm->pdev->dev;
+ u32 reg_val, type, vf_num;
+
+ while (err->msg) {
+ if (err->int_msk & error_status) {
+ dev_err(dev, "%s [error status=0x%x] found\n",
+ err->msg, err->int_msk);
+
+ if (error_status & QM_DB_TIMEOUT) {
+ reg_val = readl(qm->io_base +
+ QM_ABNORMAL_INF01);
+ type = (reg_val & QM_DB_TIMEOUT_TYPE) >>
+ QM_DB_TIMEOUT_TYPE_SHIFT;
+ vf_num = reg_val & QM_DB_TIMEOUT_VF;
+ dev_err(dev, "qm %s doorbell timeout in function %u\n",
+ qm_db_timeout[type], vf_num);
+ }
+
+ if (error_status & QM_OF_FIFO_OF) {
+ reg_val = readl(qm->io_base +
+ QM_ABNORMAL_INF00);
+ type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >>
+ QM_FIFO_OVERFLOW_TYPE_SHIFT;
+ vf_num = reg_val & QM_FIFO_OVERFLOW_VF;
+
+ if (type < ARRAY_SIZE(qm_fifo_overflow))
+ dev_err(dev, "qm %s fifo overflow in function %u\n",
+ qm_fifo_overflow[type],
+ vf_num);
+ else
+ dev_err(dev, "unknown error type\n");
+ }
+ }
+ err++;
+ }
+}
+
+static pci_ers_result_t qm_hw_error_handle_v2(struct hisi_qm *qm)
+{
+ u32 error_status, tmp;
+
+ /* read err sts */
+ tmp = readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
+ error_status = qm->error_mask & tmp;
+
+ if (error_status) {
+ qm_log_hw_error(qm, error_status);
+
+ /* clear err sts */
+ writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE);
+
+ return PCI_ERS_RESULT_NEED_RESET;
+ }
+
+ return PCI_ERS_RESULT_RECOVERED;
+}
+
+static const struct hisi_qm_hw_ops qm_hw_ops_v1 = {
+ .qm_db = qm_db_v1,
+ .get_irq_num = qm_get_irq_num_v1,
+ .hw_error_init = qm_hw_error_init_v1,
+};
+
+static const struct hisi_qm_hw_ops qm_hw_ops_v2 = {
+ .get_vft = qm_get_vft_v2,
+ .qm_db = qm_db_v2,
+ .get_irq_num = qm_get_irq_num_v2,
+ .hw_error_init = qm_hw_error_init_v2,
+ .hw_error_handle = qm_hw_error_handle_v2,
+};
+
+static void *qm_get_avail_sqe(struct hisi_qp *qp)
+{
+ struct hisi_qp_status *qp_status = &qp->qp_status;
+ u16 sq_tail = qp_status->sq_tail;
+
+ if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH))
+ return NULL;
+
+ return qp->sqe + sq_tail * qp->qm->sqe_size;
+}
+
+/**
+ * hisi_qm_create_qp() - Create a queue pair from qm.
+ * @qm: The qm we create a qp from.
+ * @alg_type: Accelerator specific algorithm type in sqc.
+ *
+ * return created qp, -EBUSY if all qps in qm allocated, -ENOMEM if allocating
+ * qp memory fails.
+ */
+struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type)
+{
+ struct device *dev = &qm->pdev->dev;
+ struct hisi_qp *qp;
+ int qp_id, ret;
+
+ qp = kzalloc(sizeof(*qp), GFP_KERNEL);
+ if (!qp)
+ return ERR_PTR(-ENOMEM);
+
+ write_lock(&qm->qps_lock);
+
+ qp_id = find_first_zero_bit(qm->qp_bitmap, qm->qp_num);
+ if (qp_id >= qm->qp_num) {
+ write_unlock(&qm->qps_lock);
+ dev_info(&qm->pdev->dev, "QM all queues are busy!\n");
+ ret = -EBUSY;
+ goto err_free_qp;
+ }
+ set_bit(qp_id, qm->qp_bitmap);
+ qm->qp_array[qp_id] = qp;
+
+ write_unlock(&qm->qps_lock);
+
+ qp->qm = qm;
+
+ if (qm->use_dma_api) {
+ qp->qdma.size = qm->sqe_size * QM_Q_DEPTH +
+ sizeof(struct qm_cqe) * QM_Q_DEPTH;
+ qp->qdma.va = dma_alloc_coherent(dev, qp->qdma.size,
+ &qp->qdma.dma, GFP_KERNEL);
+ if (!qp->qdma.va) {
+ ret = -ENOMEM;
+ goto err_clear_bit;
+ }
+
+ dev_dbg(dev, "allocate qp dma buf(va=%pK, dma=%pad, size=%zx)\n",
+ qp->qdma.va, &qp->qdma.dma, qp->qdma.size);
+ }
+
+ qp->qp_id = qp_id;
+ qp->alg_type = alg_type;
+ INIT_WORK(&qp->work, qm_qp_work_func);
+ qp->wq = alloc_workqueue("hisi_qm", WQ_UNBOUND | WQ_HIGHPRI |
+ WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0);
+ if (!qp->wq) {
+ ret = -EFAULT;
+ goto err_free_qp_mem;
+ }
+
+ return qp;
+
+err_free_qp_mem:
+ if (qm->use_dma_api)
+ dma_free_coherent(dev, qp->qdma.size, qp->qdma.va,
+ qp->qdma.dma);
+err_clear_bit:
+ write_lock(&qm->qps_lock);
+ qm->qp_array[qp_id] = NULL;
+ clear_bit(qp_id, qm->qp_bitmap);
+ write_unlock(&qm->qps_lock);
+err_free_qp:
+ kfree(qp);
+ return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(hisi_qm_create_qp);
+
+/**
+ * hisi_qm_release_qp() - Release a qp back to its qm.
+ * @qp: The qp we want to release.
+ *
+ * This function releases the resource of a qp.
+ */
+void hisi_qm_release_qp(struct hisi_qp *qp)
+{
+ struct hisi_qm *qm = qp->qm;
+ struct qm_dma *qdma = &qp->qdma;
+ struct device *dev = &qm->pdev->dev;
+
+ if (qm->use_dma_api && qdma->va)
+ dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma);
+
+ write_lock(&qm->qps_lock);
+ qm->qp_array[qp->qp_id] = NULL;
+ clear_bit(qp->qp_id, qm->qp_bitmap);
+ write_unlock(&qm->qps_lock);
+
+ kfree(qp);
+}
+EXPORT_SYMBOL_GPL(hisi_qm_release_qp);
+
+static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid)
+{
+ struct hisi_qm *qm = qp->qm;
+ struct device *dev = &qm->pdev->dev;
+ enum qm_hw_ver ver = qm->ver;
+ struct qm_sqc *sqc;
+ struct qm_cqc *cqc;
+ dma_addr_t sqc_dma;
+ dma_addr_t cqc_dma;
+ int ret;
+
+ qm_init_qp_status(qp);
+
+ sqc = kzalloc(sizeof(struct qm_sqc), GFP_KERNEL);
+ if (!sqc)
+ return -ENOMEM;
+ sqc_dma = dma_map_single(dev, sqc, sizeof(struct qm_sqc),
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(dev, sqc_dma)) {
+ kfree(sqc);
+ return -ENOMEM;
+ }
+
+ INIT_QC_COMMON(sqc, qp->sqe_dma, pasid);
+ if (ver == QM_HW_V1) {
+ sqc->dw3 = QM_MK_SQC_DW3_V1(0, 0, 0, qm->sqe_size);
+ sqc->w8 = QM_Q_DEPTH - 1;
+ } else if (ver == QM_HW_V2) {
+ sqc->dw3 = QM_MK_SQC_DW3_V2(qm->sqe_size);
+ sqc->w8 = 0; /* rand_qc */
+ }
+ sqc->cq_num = qp_id;
+ sqc->w13 = QM_MK_SQC_W13(0, 1, qp->alg_type);
+
+ ret = qm_mb(qm, QM_MB_CMD_SQC, sqc_dma, qp_id, 0);
+ dma_unmap_single(dev, sqc_dma, sizeof(struct qm_sqc), DMA_TO_DEVICE);
+ kfree(sqc);
+ if (ret)
+ return ret;
+
+ cqc = kzalloc(sizeof(struct qm_cqc), GFP_KERNEL);
+ if (!cqc)
+ return -ENOMEM;
+ cqc_dma = dma_map_single(dev, cqc, sizeof(struct qm_cqc),
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(dev, cqc_dma)) {
+ kfree(cqc);
+ return -ENOMEM;
+ }
+
+ INIT_QC_COMMON(cqc, qp->cqe_dma, pasid);
+ if (ver == QM_HW_V1) {
+ cqc->dw3 = QM_MK_CQC_DW3_V1(0, 0, 0, 4);
+ cqc->w8 = QM_Q_DEPTH - 1;
+ } else if (ver == QM_HW_V2) {
+ cqc->dw3 = QM_MK_CQC_DW3_V2(4);
+ cqc->w8 = 0;
+ }
+ cqc->dw6 = 1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT;
+
+ ret = qm_mb(qm, QM_MB_CMD_CQC, cqc_dma, qp_id, 0);
+ dma_unmap_single(dev, cqc_dma, sizeof(struct qm_cqc), DMA_TO_DEVICE);
+ kfree(cqc);
+
+ return ret;
+}
+
+/**
+ * hisi_qm_start_qp() - Start a qp into running.
+ * @qp: The qp we want to start to run.
+ * @arg: Accelerator specific argument.
+ *
+ * After this function, qp can receive request from user. Return qp_id if
+ * successful, Return -EBUSY if failed.
+ */
+int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg)
+{
+ struct hisi_qm *qm = qp->qm;
+ struct device *dev = &qm->pdev->dev;
+ enum qm_hw_ver ver = qm->ver;
+ int qp_id = qp->qp_id;
+ int pasid = arg;
+ size_t off = 0;
+ int ret;
+
+#define QP_INIT_BUF(qp, type, size) do { \
+ (qp)->type = ((qp)->qdma.va + (off)); \
+ (qp)->type##_dma = (qp)->qdma.dma + (off); \
+ off += (size); \
+} while (0)
+
+ if (!qp->qdma.dma) {
+ dev_err(dev, "cannot get qm dma buffer\n");
+ return -EINVAL;
+ }
+
+ /* sq need 128 bytes alignment */
+ if (qp->qdma.dma & QM_SQE_DATA_ALIGN_MASK) {
+ dev_err(dev, "qm sq is not aligned to 128 byte\n");
+ return -EINVAL;
+ }
+
+ QP_INIT_BUF(qp, sqe, qm->sqe_size * QM_Q_DEPTH);
+ QP_INIT_BUF(qp, cqe, sizeof(struct qm_cqe) * QM_Q_DEPTH);
+
+ dev_dbg(dev, "init qp buffer(v%d):\n"
+ " sqe (%pK, %lx)\n"
+ " cqe (%pK, %lx)\n",
+ ver, qp->sqe, (unsigned long)qp->sqe_dma,
+ qp->cqe, (unsigned long)qp->cqe_dma);
+
+ ret = qm_qp_ctx_cfg(qp, qp_id, pasid);
+ if (ret)
+ return ret;
+
+ dev_dbg(dev, "queue %d started\n", qp_id);
+
+ return qp_id;
+}
+EXPORT_SYMBOL_GPL(hisi_qm_start_qp);
+
+/**
+ * hisi_qm_stop_qp() - Stop a qp in qm.
+ * @qp: The qp we want to stop.
+ *
+ * This function is reverse of hisi_qm_start_qp. Return 0 if successful.
+ */
+int hisi_qm_stop_qp(struct hisi_qp *qp)
+{
+ struct device *dev = &qp->qm->pdev->dev;
+ int i = 0;
+
+ /* it is stopped */
+ if (test_bit(QP_STOP, &qp->qp_status.flags))
+ return 0;
+
+ while (atomic_read(&qp->qp_status.used)) {
+ i++;
+ msleep(20);
+ if (i == 10) {
+ dev_err(dev, "Cannot drain out data for stopping, Force to stop!\n");
+ return 0;
+ }
+ }
+
+ set_bit(QP_STOP, &qp->qp_status.flags);
+
+ dev_dbg(dev, "stop queue %u!", qp->qp_id);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(hisi_qm_stop_qp);
+
+/**
+ * hisi_qp_send() - Queue up a task in the hardware queue.
+ * @qp: The qp in which to put the message.
+ * @msg: The message.
+ *
+ * This function will return -EBUSY if qp is currently full, and -EAGAIN
+ * if qp related qm is resetting.
+ */
+int hisi_qp_send(struct hisi_qp *qp, const void *msg)
+{
+ struct hisi_qp_status *qp_status = &qp->qp_status;
+ u16 sq_tail = qp_status->sq_tail;
+ u16 sq_tail_next = (sq_tail + 1) % QM_Q_DEPTH;
+ void *sqe = qm_get_avail_sqe(qp);
+
+ if (unlikely(test_bit(QP_STOP, &qp->qp_status.flags))) {
+ dev_info(&qp->qm->pdev->dev, "QP is stopped or resetting\n");
+ return -EAGAIN;
+ }
+
+ if (!sqe)
+ return -EBUSY;
+
+ memcpy(sqe, msg, qp->qm->sqe_size);
+
+ qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0);
+ atomic_inc(&qp->qp_status.used);
+ qp_status->sq_tail = sq_tail_next;
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(hisi_qp_send);
+
+static void hisi_qm_cache_wb(struct hisi_qm *qm)
+{
+ unsigned int val;
+
+ if (qm->ver == QM_HW_V2) {
+ writel(0x1, qm->io_base + QM_CACHE_WB_START);
+ if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE,
+ val, val & BIT(0), 10, 1000))
+ dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n");
+ }
+}
+
+/**
+ * hisi_qm_init() - Initialize configures about qm.
+ * @qm: The qm needing init.
+ *
+ * This function init qm, then we can call hisi_qm_start to put qm into work.
+ */
+int hisi_qm_init(struct hisi_qm *qm)
+{
+ struct pci_dev *pdev = qm->pdev;
+ struct device *dev = &pdev->dev;
+ unsigned int num_vec;
+ int ret;
+
+ switch (qm->ver) {
+ case QM_HW_V1:
+ qm->ops = &qm_hw_ops_v1;
+ break;
+ case QM_HW_V2:
+ qm->ops = &qm_hw_ops_v2;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ ret = pci_enable_device_mem(pdev);
+ if (ret < 0) {
+ dev_err(&pdev->dev, "Failed to enable device mem!\n");
+ return ret;
+ }
+
+ ret = pci_request_mem_regions(pdev, qm->dev_name);
+ if (ret < 0) {
+ dev_err(&pdev->dev, "Failed to request mem regions!\n");
+ goto err_disable_pcidev;
+ }
+
+ qm->io_base = ioremap(pci_resource_start(pdev, PCI_BAR_2),
+ pci_resource_len(qm->pdev, PCI_BAR_2));
+ if (!qm->io_base) {
+ ret = -EIO;
+ goto err_release_mem_regions;
+ }
+
+ ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
+ if (ret < 0)
+ goto err_iounmap;
+ pci_set_master(pdev);
+
+ if (!qm->ops->get_irq_num) {
+ ret = -EOPNOTSUPP;
+ goto err_iounmap;
+ }
+ num_vec = qm->ops->get_irq_num(qm);
+ ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI);
+ if (ret < 0) {
+ dev_err(dev, "Failed to enable MSI vectors!\n");
+ goto err_iounmap;
+ }
+
+ ret = qm_irq_register(qm);
+ if (ret)
+ goto err_free_irq_vectors;
+
+ mutex_init(&qm->mailbox_lock);
+ rwlock_init(&qm->qps_lock);
+
+ dev_dbg(dev, "init qm %s with %s\n", pdev->is_physfn ? "pf" : "vf",
+ qm->use_dma_api ? "dma api" : "iommu api");
+
+ return 0;
+
+err_free_irq_vectors:
+ pci_free_irq_vectors(pdev);
+err_iounmap:
+ iounmap(qm->io_base);
+err_release_mem_regions:
+ pci_release_mem_regions(pdev);
+err_disable_pcidev:
+ pci_disable_device(pdev);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(hisi_qm_init);
+
+/**
+ * hisi_qm_uninit() - Uninitialize qm.
+ * @qm: The qm needed uninit.
+ *
+ * This function uninits qm related device resources.
+ */
+void hisi_qm_uninit(struct hisi_qm *qm)
+{
+ struct pci_dev *pdev = qm->pdev;
+ struct device *dev = &pdev->dev;
+
+ if (qm->use_dma_api && qm->qdma.va) {
+ hisi_qm_cache_wb(qm);
+ dma_free_coherent(dev, qm->qdma.size,
+ qm->qdma.va, qm->qdma.dma);
+ memset(&qm->qdma, 0, sizeof(qm->qdma));
+ }
+
+ qm_irq_unregister(qm);
+ pci_free_irq_vectors(pdev);
+ iounmap(qm->io_base);
+ pci_release_mem_regions(pdev);
+ pci_disable_device(pdev);
+}
+EXPORT_SYMBOL_GPL(hisi_qm_uninit);
+
+/**
+ * hisi_qm_get_vft() - Get vft from a qm.
+ * @qm: The qm we want to get its vft.
+ * @base: The base number of queue in vft.
+ * @number: The number of queues in vft.
+ *
+ * We can allocate multiple queues to a qm by configuring virtual function
+ * table. We get related configures by this function. Normally, we call this
+ * function in VF driver to get the queue information.
+ *
+ * qm hw v1 does not support this interface.
+ */
+int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number)
+{
+ if (!base || !number)
+ return -EINVAL;
+
+ if (!qm->ops->get_vft) {
+ dev_err(&qm->pdev->dev, "Don't support vft read!\n");
+ return -EINVAL;
+ }
+
+ return qm->ops->get_vft(qm, base, number);
+}
+EXPORT_SYMBOL_GPL(hisi_qm_get_vft);
+
+/**
+ * hisi_qm_set_vft() - Set "virtual function table" for a qm.
+ * @fun_num: Number of operated function.
+ * @qm: The qm in which to set vft, alway in a PF.
+ * @base: The base number of queue in vft.
+ * @number: The number of queues in vft. 0 means invalid vft.
+ *
+ * This function is alway called in PF driver, it is used to assign queues
+ * among PF and VFs.
+ *
+ * Assign queues A~B to PF: hisi_qm_set_vft(qm, 0, A, B - A + 1)
+ * Assign queues A~B to VF: hisi_qm_set_vft(qm, 2, A, B - A + 1)
+ * (VF function number 0x2)
+ */
+int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
+ u32 number)
+{
+ u32 max_q_num = qm->ctrl_qp_num;
+
+ if (base >= max_q_num || number > max_q_num ||
+ (base + number) > max_q_num)
+ return -EINVAL;
+
+ return qm_set_sqc_cqc_vft(qm, fun_num, base, number);
+}
+EXPORT_SYMBOL_GPL(hisi_qm_set_vft);
+
+static void qm_init_eq_aeq_status(struct hisi_qm *qm)
+{
+ struct hisi_qm_status *status = &qm->status;
+
+ status->eq_head = 0;
+ status->aeq_head = 0;
+ status->eqc_phase = 1;
+ status->aeqc_phase = 1;
+}
+
+static int qm_eq_ctx_cfg(struct hisi_qm *qm)
+{
+ struct device *dev = &qm->pdev->dev;
+ struct qm_eqc *eqc;
+ struct qm_aeqc *aeqc;
+ dma_addr_t eqc_dma;
+ dma_addr_t aeqc_dma;
+ int ret;
+
+ qm_init_eq_aeq_status(qm);
+
+ eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL);
+ if (!eqc)
+ return -ENOMEM;
+ eqc_dma = dma_map_single(dev, eqc, sizeof(struct qm_eqc),
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(dev, eqc_dma)) {
+ kfree(eqc);
+ return -ENOMEM;
+ }
+
+ eqc->base_l = lower_32_bits(qm->eqe_dma);
+ eqc->base_h = upper_32_bits(qm->eqe_dma);
+ if (qm->ver == QM_HW_V1)
+ eqc->dw3 = QM_EQE_AEQE_SIZE;
+ eqc->dw6 = (QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT);
+ ret = qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0);
+ dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE);
+ kfree(eqc);
+ if (ret)
+ return ret;
+
+ aeqc = kzalloc(sizeof(struct qm_aeqc), GFP_KERNEL);
+ if (!aeqc)
+ return -ENOMEM;
+ aeqc_dma = dma_map_single(dev, aeqc, sizeof(struct qm_aeqc),
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(dev, aeqc_dma)) {
+ kfree(aeqc);
+ return -ENOMEM;
+ }
+
+ aeqc->base_l = lower_32_bits(qm->aeqe_dma);
+ aeqc->base_h = upper_32_bits(qm->aeqe_dma);
+ aeqc->dw6 = (QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT);
+
+ ret = qm_mb(qm, QM_MB_CMD_AEQC, aeqc_dma, 0, 0);
+ dma_unmap_single(dev, aeqc_dma, sizeof(struct qm_aeqc), DMA_TO_DEVICE);
+ kfree(aeqc);
+
+ return ret;
+}
+
+static int __hisi_qm_start(struct hisi_qm *qm)
+{
+ struct pci_dev *pdev = qm->pdev;
+ struct device *dev = &pdev->dev;
+ size_t off = 0;
+ int ret;
+
+#define QM_INIT_BUF(qm, type, num) do { \
+ (qm)->type = ((qm)->qdma.va + (off)); \
+ (qm)->type##_dma = (qm)->qdma.dma + (off); \
+ off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \
+} while (0)
+
+ WARN_ON(!qm->qdma.dma);
+
+ if (qm->qp_num == 0)
+ return -EINVAL;
+
+ if (qm->fun_type == QM_HW_PF) {
+ ret = qm_dev_mem_reset(qm);
+ if (ret)
+ return ret;
+
+ ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num);
+ if (ret)
+ return ret;
+ }
+
+ QM_INIT_BUF(qm, eqe, QM_Q_DEPTH);
+ QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH);
+ QM_INIT_BUF(qm, sqc, qm->qp_num);
+ QM_INIT_BUF(qm, cqc, qm->qp_num);
+
+ dev_dbg(dev, "init qm buffer:\n"
+ " eqe (%pK, %lx)\n"
+ " aeqe (%pK, %lx)\n"
+ " sqc (%pK, %lx)\n"
+ " cqc (%pK, %lx)\n",
+ qm->eqe, (unsigned long)qm->eqe_dma,
+ qm->aeqe, (unsigned long)qm->aeqe_dma,
+ qm->sqc, (unsigned long)qm->sqc_dma,
+ qm->cqc, (unsigned long)qm->cqc_dma);
+
+ ret = qm_eq_ctx_cfg(qm);
+ if (ret)
+ return ret;
+
+ ret = qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);
+ if (ret)
+ return ret;
+
+ ret = qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0);
+ if (ret)
+ return ret;
+
+ writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK);
+ writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK);
+
+ return 0;
+}
+
+/**
+ * hisi_qm_start() - start qm
+ * @qm: The qm to be started.
+ *
+ * This function starts a qm, then we can allocate qp from this qm.
+ */
+int hisi_qm_start(struct hisi_qm *qm)
+{
+ struct device *dev = &qm->pdev->dev;
+
+ dev_dbg(dev, "qm start with %d queue pairs\n", qm->qp_num);
+
+ if (!qm->qp_num) {
+ dev_err(dev, "qp_num should not be 0\n");
+ return -EINVAL;
+ }
+
+ if (!qm->qp_bitmap) {
+ qm->qp_bitmap = devm_kcalloc(dev, BITS_TO_LONGS(qm->qp_num),
+ sizeof(long), GFP_KERNEL);
+ qm->qp_array = devm_kcalloc(dev, qm->qp_num,
+ sizeof(struct hisi_qp *),
+ GFP_KERNEL);
+ if (!qm->qp_bitmap || !qm->qp_array)
+ return -ENOMEM;
+ }
+
+ if (!qm->use_dma_api) {
+ dev_dbg(&qm->pdev->dev, "qm delay start\n");
+ return 0;
+ } else if (!qm->qdma.va) {
+ qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_Q_DEPTH) +
+ QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) +
+ QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) +
+ QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num);
+ qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size,
+ &qm->qdma.dma, GFP_KERNEL);
+ dev_dbg(dev, "allocate qm dma buf(va=%pK, dma=%pad, size=%zx)\n",
+ qm->qdma.va, &qm->qdma.dma, qm->qdma.size);
+ if (!qm->qdma.va)
+ return -ENOMEM;
+ }
+
+ return __hisi_qm_start(qm);
+}
+EXPORT_SYMBOL_GPL(hisi_qm_start);
+
+/**
+ * hisi_qm_stop() - Stop a qm.
+ * @qm: The qm which will be stopped.
+ *
+ * This function stops qm and its qps, then qm can not accept request.
+ * Related resources are not released at this state, we can use hisi_qm_start
+ * to let qm start again.
+ */
+int hisi_qm_stop(struct hisi_qm *qm)
+{
+ struct device *dev;
+ struct hisi_qp *qp;
+ int ret = 0, i;
+
+ if (!qm || !qm->pdev) {
+ WARN_ON(1);
+ return -EINVAL;
+ }
+
+ dev = &qm->pdev->dev;
+
+ /* Mask eq and aeq irq */
+ writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK);
+ writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK);
+
+ /* Stop all qps belong to this qm */
+ for (i = 0; i < qm->qp_num; i++) {
+ qp = qm->qp_array[i];
+ if (qp) {
+ ret = hisi_qm_stop_qp(qp);
+ if (ret < 0) {
+ dev_err(dev, "Failed to stop qp%d!\n", i);
+ return -EBUSY;
+ }
+ }
+ }
+
+ if (qm->fun_type == QM_HW_PF) {
+ ret = hisi_qm_set_vft(qm, 0, 0, 0);
+ if (ret < 0)
+ dev_err(dev, "Failed to set vft!\n");
+ }
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(hisi_qm_stop);
+
+/**
+ * hisi_qm_debug_init() - Initialize qm related debugfs files.
+ * @qm: The qm for which we want to add debugfs files.
+ *
+ * Create qm related debugfs files.
+ */
+int hisi_qm_debug_init(struct hisi_qm *qm)
+{
+ struct dentry *qm_d, *qm_regs;
+ int i, ret;
+
+ qm_d = debugfs_create_dir("qm", qm->debug.debug_root);
+ if (IS_ERR(qm_d))
+ return -ENOENT;
+ qm->debug.qm_d = qm_d;
+
+ /* only show this in PF */
+ if (qm->fun_type == QM_HW_PF)
+ for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++)
+ if (qm_create_debugfs_file(qm, i)) {
+ ret = -ENOENT;
+ goto failed_to_create;
+ }
+
+ qm_regs = debugfs_create_file("qm_regs", 0444, qm->debug.qm_d, qm,
+ &qm_regs_fops);
+ if (IS_ERR(qm_regs)) {
+ ret = -ENOENT;
+ goto failed_to_create;
+ }
+
+ return 0;
+
+failed_to_create:
+ debugfs_remove_recursive(qm_d);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(hisi_qm_debug_init);
+
+/**
+ * hisi_qm_debug_regs_clear() - clear qm debug related registers.
+ * @qm: The qm for which we want to clear its debug registers.
+ */
+void hisi_qm_debug_regs_clear(struct hisi_qm *qm)
+{
+ struct qm_dfx_registers *regs;
+ int i;
+
+ /* clear current_q */
+ writel(0x0, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
+ writel(0x0, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
+
+ /*
+ * these registers are reading and clearing, so clear them after
+ * reading them.
+ */
+ writel(0x1, qm->io_base + QM_DFX_CNT_CLR_CE);
+
+ regs = qm_dfx_regs;
+ for (i = 0; i < CNT_CYC_REGS_NUM; i++) {
+ readl(qm->io_base + regs->reg_offset);
+ regs++;
+ }
+
+ writel(0x0, qm->io_base + QM_DFX_CNT_CLR_CE);
+}
+EXPORT_SYMBOL_GPL(hisi_qm_debug_regs_clear);
+
+/**
+ * hisi_qm_hw_error_init() - Configure qm hardware error report method.
+ * @qm: The qm which we want to configure.
+ * @ce: Bit mask of correctable error configure.
+ * @nfe: Bit mask of non-fatal error configure.
+ * @fe: Bit mask of fatal error configure.
+ * @msi: Bit mask of error reported by message signal interrupt.
+ *
+ * Hardware errors of qm can be reported either by RAS interrupts which will
+ * be handled by UEFI and then PCIe AER or by device MSI. User can configure
+ * each error to use either of above two methods. For RAS interrupts, we can
+ * configure an error as one of correctable error, non-fatal error or
+ * fatal error.
+ *
+ * Bits indicating errors can be configured to ce, nfe, fe and msi to enable
+ * related report methods. Error report will be masked if related error bit
+ * does not configure.
+ */
+void hisi_qm_hw_error_init(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe,
+ u32 msi)
+{
+ if (!qm->ops->hw_error_init) {
+ dev_err(&qm->pdev->dev, "QM version %d doesn't support hw error handling!\n",
+ qm->ver);
+ return;
+ }
+
+ qm->ops->hw_error_init(qm, ce, nfe, fe, msi);
+}
+EXPORT_SYMBOL_GPL(hisi_qm_hw_error_init);
+
+/**
+ * hisi_qm_hw_error_handle() - Handle qm non-fatal hardware errors.
+ * @qm: The qm which has non-fatal hardware errors.
+ *
+ * Accelerators use this function to handle qm non-fatal hardware errors.
+ */
+int hisi_qm_hw_error_handle(struct hisi_qm *qm)
+{
+ if (!qm->ops->hw_error_handle) {
+ dev_err(&qm->pdev->dev, "QM version %d doesn't support hw error report!\n",
+ qm->ver);
+ return PCI_ERS_RESULT_NONE;
+ }
+
+ return qm->ops->hw_error_handle(qm);
+}
+EXPORT_SYMBOL_GPL(hisi_qm_hw_error_handle);
+
+/**
+ * hisi_qm_get_hw_version() - Get hardware version of a qm.
+ * @pdev: The device which hardware version we want to get.
+ *
+ * This function gets the hardware version of a qm. Return QM_HW_UNKNOWN
+ * if the hardware version is not supported.
+ */
+enum qm_hw_ver hisi_qm_get_hw_version(struct pci_dev *pdev)
+{
+ switch (pdev->revision) {
+ case QM_HW_V1:
+ case QM_HW_V2:
+ return pdev->revision;
+ default:
+ return QM_HW_UNKNOWN;
+ }
+}
+EXPORT_SYMBOL_GPL(hisi_qm_get_hw_version);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
+MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver");
diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h
new file mode 100644
index 000000000000..70e672ae86bf
--- /dev/null
+++ b/drivers/crypto/hisilicon/qm.h
@@ -0,0 +1,215 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2019 HiSilicon Limited. */
+#ifndef HISI_ACC_QM_H
+#define HISI_ACC_QM_H
+
+#include <linux/bitfield.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
+/* qm user domain */
+#define QM_ARUSER_M_CFG_1 0x100088
+#define AXUSER_SNOOP_ENABLE BIT(30)
+#define AXUSER_CMD_TYPE GENMASK(14, 12)
+#define AXUSER_CMD_SMMU_NORMAL 1
+#define AXUSER_NS BIT(6)
+#define AXUSER_NO BIT(5)
+#define AXUSER_FP BIT(4)
+#define AXUSER_SSV BIT(0)
+#define AXUSER_BASE (AXUSER_SNOOP_ENABLE | \
+ FIELD_PREP(AXUSER_CMD_TYPE, \
+ AXUSER_CMD_SMMU_NORMAL) | \
+ AXUSER_NS | AXUSER_NO | AXUSER_FP)
+#define QM_ARUSER_M_CFG_ENABLE 0x100090
+#define ARUSER_M_CFG_ENABLE 0xfffffffe
+#define QM_AWUSER_M_CFG_1 0x100098
+#define QM_AWUSER_M_CFG_ENABLE 0x1000a0
+#define AWUSER_M_CFG_ENABLE 0xfffffffe
+#define QM_WUSER_M_CFG_ENABLE 0x1000a8
+#define WUSER_M_CFG_ENABLE 0xffffffff
+
+/* qm cache */
+#define QM_CACHE_CTL 0x100050
+#define SQC_CACHE_ENABLE BIT(0)
+#define CQC_CACHE_ENABLE BIT(1)
+#define SQC_CACHE_WB_ENABLE BIT(4)
+#define SQC_CACHE_WB_THRD GENMASK(10, 5)
+#define CQC_CACHE_WB_ENABLE BIT(11)
+#define CQC_CACHE_WB_THRD GENMASK(17, 12)
+#define QM_AXI_M_CFG 0x1000ac
+#define AXI_M_CFG 0xffff
+#define QM_AXI_M_CFG_ENABLE 0x1000b0
+#define AXI_M_CFG_ENABLE 0xffffffff
+#define QM_PEH_AXUSER_CFG 0x1000cc
+#define QM_PEH_AXUSER_CFG_ENABLE 0x1000d0
+#define PEH_AXUSER_CFG 0x401001
+#define PEH_AXUSER_CFG_ENABLE 0xffffffff
+
+#define QM_DFX_MB_CNT_VF 0x104010
+#define QM_DFX_DB_CNT_VF 0x104020
+#define QM_DFX_SQE_CNT_VF_SQN 0x104030
+#define QM_DFX_CQE_CNT_VF_CQN 0x104040
+#define QM_DFX_QN_SHIFT 16
+#define CURRENT_FUN_MASK GENMASK(5, 0)
+#define CURRENT_Q_MASK GENMASK(31, 16)
+
+#define QM_AXI_RRESP BIT(0)
+#define QM_AXI_BRESP BIT(1)
+#define QM_ECC_MBIT BIT(2)
+#define QM_ECC_1BIT BIT(3)
+#define QM_ACC_GET_TASK_TIMEOUT BIT(4)
+#define QM_ACC_DO_TASK_TIMEOUT BIT(5)
+#define QM_ACC_WB_NOT_READY_TIMEOUT BIT(6)
+#define QM_SQ_CQ_VF_INVALID BIT(7)
+#define QM_CQ_VF_INVALID BIT(8)
+#define QM_SQ_VF_INVALID BIT(9)
+#define QM_DB_TIMEOUT BIT(10)
+#define QM_OF_FIFO_OF BIT(11)
+#define QM_DB_RANDOM_INVALID BIT(12)
+
+#define QM_BASE_NFE (QM_AXI_RRESP | QM_AXI_BRESP | QM_ECC_MBIT | \
+ QM_ACC_GET_TASK_TIMEOUT | QM_DB_TIMEOUT | \
+ QM_OF_FIFO_OF)
+#define QM_BASE_CE QM_ECC_1BIT
+
+#define QM_Q_DEPTH 1024
+
+enum qp_state {
+ QP_STOP,
+};
+
+enum qm_hw_ver {
+ QM_HW_UNKNOWN = -1,
+ QM_HW_V1 = 0x20,
+ QM_HW_V2 = 0x21,
+};
+
+enum qm_fun_type {
+ QM_HW_PF,
+ QM_HW_VF,
+};
+
+enum qm_debug_file {
+ CURRENT_Q,
+ CLEAR_ENABLE,
+ DEBUG_FILE_NUM,
+};
+
+struct debugfs_file {
+ enum qm_debug_file index;
+ struct mutex lock;
+ struct qm_debug *debug;
+};
+
+struct qm_debug {
+ u32 curr_qm_qp_num;
+ struct dentry *debug_root;
+ struct dentry *qm_d;
+ struct debugfs_file files[DEBUG_FILE_NUM];
+};
+
+struct qm_dma {
+ void *va;
+ dma_addr_t dma;
+ size_t size;
+};
+
+struct hisi_qm_status {
+ u32 eq_head;
+ bool eqc_phase;
+ u32 aeq_head;
+ bool aeqc_phase;
+ unsigned long flags;
+};
+
+struct hisi_qm {
+ enum qm_hw_ver ver;
+ enum qm_fun_type fun_type;
+ const char *dev_name;
+ struct pci_dev *pdev;
+ void __iomem *io_base;
+ u32 sqe_size;
+ u32 qp_base;
+ u32 qp_num;
+ u32 ctrl_qp_num;
+
+ struct qm_dma qdma;
+ struct qm_sqc *sqc;
+ struct qm_cqc *cqc;
+ struct qm_eqe *eqe;
+ struct qm_aeqe *aeqe;
+ dma_addr_t sqc_dma;
+ dma_addr_t cqc_dma;
+ dma_addr_t eqe_dma;
+ dma_addr_t aeqe_dma;
+
+ struct hisi_qm_status status;
+
+ rwlock_t qps_lock;
+ unsigned long *qp_bitmap;
+ struct hisi_qp **qp_array;
+
+ struct mutex mailbox_lock;
+
+ const struct hisi_qm_hw_ops *ops;
+
+ struct qm_debug debug;
+
+ u32 error_mask;
+ u32 msi_mask;
+
+ bool use_dma_api;
+};
+
+struct hisi_qp_status {
+ atomic_t used;
+ u16 sq_tail;
+ u16 cq_head;
+ bool cqc_phase;
+ unsigned long flags;
+};
+
+struct hisi_qp_ops {
+ int (*fill_sqe)(void *sqe, void *q_parm, void *d_parm);
+};
+
+struct hisi_qp {
+ u32 qp_id;
+ u8 alg_type;
+ u8 req_type;
+
+ struct qm_dma qdma;
+ void *sqe;
+ struct qm_cqe *cqe;
+ dma_addr_t sqe_dma;
+ dma_addr_t cqe_dma;
+
+ struct hisi_qp_status qp_status;
+ struct hisi_qp_ops *hw_ops;
+ void *qp_ctx;
+ void (*req_cb)(struct hisi_qp *qp, void *data);
+ struct work_struct work;
+ struct workqueue_struct *wq;
+
+ struct hisi_qm *qm;
+};
+
+int hisi_qm_init(struct hisi_qm *qm);
+void hisi_qm_uninit(struct hisi_qm *qm);
+int hisi_qm_start(struct hisi_qm *qm);
+int hisi_qm_stop(struct hisi_qm *qm);
+struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type);
+int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg);
+int hisi_qm_stop_qp(struct hisi_qp *qp);
+void hisi_qm_release_qp(struct hisi_qp *qp);
+int hisi_qp_send(struct hisi_qp *qp, const void *msg);
+int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number);
+int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base, u32 number);
+int hisi_qm_debug_init(struct hisi_qm *qm);
+void hisi_qm_hw_error_init(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe,
+ u32 msi);
+int hisi_qm_hw_error_handle(struct hisi_qm *qm);
+enum qm_hw_ver hisi_qm_get_hw_version(struct pci_dev *pdev);
+void hisi_qm_debug_regs_clear(struct hisi_qm *qm);
+#endif
diff --git a/drivers/crypto/hisilicon/sec/sec_algs.c b/drivers/crypto/hisilicon/sec/sec_algs.c
index 02768af0dccd..e0508ea160f1 100644
--- a/drivers/crypto/hisilicon/sec/sec_algs.c
+++ b/drivers/crypto/hisilicon/sec/sec_algs.c
@@ -9,7 +9,7 @@
#include <crypto/aes.h>
#include <crypto/algapi.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <crypto/skcipher.h>
#include <crypto/xts.h>
#include <crypto/internal/skcipher.h>
@@ -347,25 +347,21 @@ static int sec_alg_skcipher_setkey_aes_xts(struct crypto_skcipher *tfm,
static int sec_alg_skcipher_setkey_des_ecb(struct crypto_skcipher *tfm,
const u8 *key, unsigned int keylen)
{
- if (keylen != DES_KEY_SIZE)
- return -EINVAL;
-
- return sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_ECB_64);
+ return verify_skcipher_des_key(tfm, key) ?:
+ sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_ECB_64);
}
static int sec_alg_skcipher_setkey_des_cbc(struct crypto_skcipher *tfm,
const u8 *key, unsigned int keylen)
{
- if (keylen != DES_KEY_SIZE)
- return -EINVAL;
-
- return sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_CBC_64);
+ return verify_skcipher_des_key(tfm, key) ?:
+ sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_CBC_64);
}
static int sec_alg_skcipher_setkey_3des_ecb(struct crypto_skcipher *tfm,
const u8 *key, unsigned int keylen)
{
- return unlikely(des3_verify_key(tfm, key)) ?:
+ return verify_skcipher_des3_key(tfm, key) ?:
sec_alg_skcipher_setkey(tfm, key, keylen,
SEC_C_3DES_ECB_192_3KEY);
}
@@ -373,7 +369,7 @@ static int sec_alg_skcipher_setkey_3des_ecb(struct crypto_skcipher *tfm,
static int sec_alg_skcipher_setkey_3des_cbc(struct crypto_skcipher *tfm,
const u8 *key, unsigned int keylen)
{
- return unlikely(des3_verify_key(tfm, key)) ?:
+ return verify_skcipher_des3_key(tfm, key) ?:
sec_alg_skcipher_setkey(tfm, key, keylen,
SEC_C_3DES_CBC_192_3KEY);
}
diff --git a/drivers/crypto/hisilicon/sec/sec_drv.h b/drivers/crypto/hisilicon/sec/sec_drv.h
index 2d2f186674ba..4d9063a8b10b 100644
--- a/drivers/crypto/hisilicon/sec/sec_drv.h
+++ b/drivers/crypto/hisilicon/sec/sec_drv.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2016-2017 Hisilicon Limited. */
#ifndef _SEC_DRV_H_
diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c
new file mode 100644
index 000000000000..e083d172b618
--- /dev/null
+++ b/drivers/crypto/hisilicon/sgl.c
@@ -0,0 +1,214 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2019 HiSilicon Limited. */
+#include <linux/dma-mapping.h>
+#include <linux/module.h>
+#include "./sgl.h"
+
+#define HISI_ACC_SGL_SGE_NR_MIN 1
+#define HISI_ACC_SGL_SGE_NR_MAX 255
+#define HISI_ACC_SGL_SGE_NR_DEF 10
+#define HISI_ACC_SGL_NR_MAX 256
+#define HISI_ACC_SGL_ALIGN_SIZE 64
+
+static int acc_sgl_sge_set(const char *val, const struct kernel_param *kp)
+{
+ int ret;
+ u32 n;
+
+ if (!val)
+ return -EINVAL;
+
+ ret = kstrtou32(val, 10, &n);
+ if (ret != 0 || n > HISI_ACC_SGL_SGE_NR_MAX || n == 0)
+ return -EINVAL;
+
+ return param_set_int(val, kp);
+}
+
+static const struct kernel_param_ops acc_sgl_sge_ops = {
+ .set = acc_sgl_sge_set,
+ .get = param_get_int,
+};
+
+static u32 acc_sgl_sge_nr = HISI_ACC_SGL_SGE_NR_DEF;
+module_param_cb(acc_sgl_sge_nr, &acc_sgl_sge_ops, &acc_sgl_sge_nr, 0444);
+MODULE_PARM_DESC(acc_sgl_sge_nr, "Number of sge in sgl(1-255)");
+
+struct acc_hw_sge {
+ dma_addr_t buf;
+ void *page_ctrl;
+ __le32 len;
+ __le32 pad;
+ __le32 pad0;
+ __le32 pad1;
+};
+
+/* use default sgl head size 64B */
+struct hisi_acc_hw_sgl {
+ dma_addr_t next_dma;
+ __le16 entry_sum_in_chain;
+ __le16 entry_sum_in_sgl;
+ __le16 entry_length_in_sgl;
+ __le16 pad0;
+ __le64 pad1[5];
+ struct hisi_acc_hw_sgl *next;
+ struct acc_hw_sge sge_entries[];
+} __aligned(1);
+
+/**
+ * hisi_acc_create_sgl_pool() - Create a hw sgl pool.
+ * @dev: The device which hw sgl pool belongs to.
+ * @pool: Pointer of pool.
+ * @count: Count of hisi_acc_hw_sgl in pool.
+ *
+ * This function creates a hw sgl pool, after this user can get hw sgl memory
+ * from it.
+ */
+int hisi_acc_create_sgl_pool(struct device *dev,
+ struct hisi_acc_sgl_pool *pool, u32 count)
+{
+ u32 sgl_size;
+ u32 size;
+
+ if (!dev || !pool || !count)
+ return -EINVAL;
+
+ sgl_size = sizeof(struct acc_hw_sge) * acc_sgl_sge_nr +
+ sizeof(struct hisi_acc_hw_sgl);
+ size = sgl_size * count;
+
+ pool->sgl = dma_alloc_coherent(dev, size, &pool->sgl_dma, GFP_KERNEL);
+ if (!pool->sgl)
+ return -ENOMEM;
+
+ pool->size = size;
+ pool->count = count;
+ pool->sgl_size = sgl_size;
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(hisi_acc_create_sgl_pool);
+
+/**
+ * hisi_acc_free_sgl_pool() - Free a hw sgl pool.
+ * @dev: The device which hw sgl pool belongs to.
+ * @pool: Pointer of pool.
+ *
+ * This function frees memory of a hw sgl pool.
+ */
+void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool)
+{
+ dma_free_coherent(dev, pool->size, pool->sgl, pool->sgl_dma);
+ memset(pool, 0, sizeof(struct hisi_acc_sgl_pool));
+}
+EXPORT_SYMBOL_GPL(hisi_acc_free_sgl_pool);
+
+struct hisi_acc_hw_sgl *acc_get_sgl(struct hisi_acc_sgl_pool *pool, u32 index,
+ dma_addr_t *hw_sgl_dma)
+{
+ if (!pool || !hw_sgl_dma || index >= pool->count || !pool->sgl)
+ return ERR_PTR(-EINVAL);
+
+ *hw_sgl_dma = pool->sgl_dma + pool->sgl_size * index;
+ return (void *)pool->sgl + pool->sgl_size * index;
+}
+
+void acc_put_sgl(struct hisi_acc_sgl_pool *pool, u32 index) {}
+
+static void sg_map_to_hw_sg(struct scatterlist *sgl,
+ struct acc_hw_sge *hw_sge)
+{
+ hw_sge->buf = sgl->dma_address;
+ hw_sge->len = sgl->dma_length;
+}
+
+static void inc_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl)
+{
+ hw_sgl->entry_sum_in_sgl++;
+}
+
+static void update_hw_sgl_sum_sge(struct hisi_acc_hw_sgl *hw_sgl, u16 sum)
+{
+ hw_sgl->entry_sum_in_chain = sum;
+}
+
+/**
+ * hisi_acc_sg_buf_map_to_hw_sgl - Map a scatterlist to a hw sgl.
+ * @dev: The device which hw sgl belongs to.
+ * @sgl: Scatterlist which will be mapped to hw sgl.
+ * @pool: Pool which hw sgl memory will be allocated in.
+ * @index: Index of hisi_acc_hw_sgl in pool.
+ * @hw_sgl_dma: The dma address of allocated hw sgl.
+ *
+ * This function builds hw sgl according input sgl, user can use hw_sgl_dma
+ * as src/dst in its BD. Only support single hw sgl currently.
+ */
+struct hisi_acc_hw_sgl *
+hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev,
+ struct scatterlist *sgl,
+ struct hisi_acc_sgl_pool *pool,
+ u32 index, dma_addr_t *hw_sgl_dma)
+{
+ struct hisi_acc_hw_sgl *curr_hw_sgl;
+ dma_addr_t curr_sgl_dma = 0;
+ struct acc_hw_sge *curr_hw_sge;
+ struct scatterlist *sg;
+ int sg_n = sg_nents(sgl);
+ int i, ret;
+
+ if (!dev || !sgl || !pool || !hw_sgl_dma || sg_n > acc_sgl_sge_nr)
+ return ERR_PTR(-EINVAL);
+
+ ret = dma_map_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);
+ if (!ret)
+ return ERR_PTR(-EINVAL);
+
+ curr_hw_sgl = acc_get_sgl(pool, index, &curr_sgl_dma);
+ if (!curr_hw_sgl) {
+ ret = -ENOMEM;
+ goto err_unmap_sg;
+ }
+ curr_hw_sgl->entry_length_in_sgl = acc_sgl_sge_nr;
+ curr_hw_sge = curr_hw_sgl->sge_entries;
+
+ for_each_sg(sgl, sg, sg_n, i) {
+ sg_map_to_hw_sg(sg, curr_hw_sge);
+ inc_hw_sgl_sge(curr_hw_sgl);
+ curr_hw_sge++;
+ }
+
+ update_hw_sgl_sum_sge(curr_hw_sgl, acc_sgl_sge_nr);
+ *hw_sgl_dma = curr_sgl_dma;
+
+ return curr_hw_sgl;
+
+err_unmap_sg:
+ dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);
+ return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_map_to_hw_sgl);
+
+/**
+ * hisi_acc_sg_buf_unmap() - Unmap allocated hw sgl.
+ * @dev: The device which hw sgl belongs to.
+ * @sgl: Related scatterlist.
+ * @hw_sgl: Virtual address of hw sgl.
+ * @hw_sgl_dma: DMA address of hw sgl.
+ * @pool: Pool which hw sgl is allocated in.
+ *
+ * This function unmaps allocated hw sgl.
+ */
+void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl,
+ struct hisi_acc_hw_sgl *hw_sgl)
+{
+ dma_unmap_sg(dev, sgl, sg_nents(sgl), DMA_BIDIRECTIONAL);
+
+ hw_sgl->entry_sum_in_chain = 0;
+ hw_sgl->entry_sum_in_sgl = 0;
+ hw_sgl->entry_length_in_sgl = 0;
+}
+EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_unmap);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
+MODULE_DESCRIPTION("HiSilicon Accelerator SGL support");
diff --git a/drivers/crypto/hisilicon/sgl.h b/drivers/crypto/hisilicon/sgl.h
new file mode 100644
index 000000000000..3ac8871c7acf
--- /dev/null
+++ b/drivers/crypto/hisilicon/sgl.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2019 HiSilicon Limited. */
+#ifndef HISI_ACC_SGL_H
+#define HISI_ACC_SGL_H
+
+struct hisi_acc_sgl_pool {
+ struct hisi_acc_hw_sgl *sgl;
+ dma_addr_t sgl_dma;
+ size_t size;
+ u32 count;
+ size_t sgl_size;
+};
+
+struct hisi_acc_hw_sgl *
+hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev,
+ struct scatterlist *sgl,
+ struct hisi_acc_sgl_pool *pool,
+ u32 index, dma_addr_t *hw_sgl_dma);
+void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl,
+ struct hisi_acc_hw_sgl *hw_sgl);
+int hisi_acc_create_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool,
+ u32 count);
+void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool);
+#endif
diff --git a/drivers/crypto/hisilicon/zip/Makefile b/drivers/crypto/hisilicon/zip/Makefile
new file mode 100644
index 000000000000..a936f099ee22
--- /dev/null
+++ b/drivers/crypto/hisilicon/zip/Makefile
@@ -0,0 +1,2 @@
+obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += hisi_zip.o
+hisi_zip-objs = zip_main.o zip_crypto.o
diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h
new file mode 100644
index 000000000000..ffb00d987d02
--- /dev/null
+++ b/drivers/crypto/hisilicon/zip/zip.h
@@ -0,0 +1,71 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2019 HiSilicon Limited. */
+#ifndef HISI_ZIP_H
+#define HISI_ZIP_H
+
+#undef pr_fmt
+#define pr_fmt(fmt) "hisi_zip: " fmt
+
+#include <linux/list.h>
+#include "../qm.h"
+#include "../sgl.h"
+
+/* hisi_zip_sqe dw3 */
+#define HZIP_BD_STATUS_M GENMASK(7, 0)
+/* hisi_zip_sqe dw9 */
+#define HZIP_REQ_TYPE_M GENMASK(7, 0)
+#define HZIP_ALG_TYPE_ZLIB 0x02
+#define HZIP_ALG_TYPE_GZIP 0x03
+#define HZIP_BUF_TYPE_M GENMASK(11, 8)
+#define HZIP_PBUFFER 0x0
+#define HZIP_SGL 0x1
+
+enum hisi_zip_error_type {
+ /* negative compression */
+ HZIP_NC_ERR = 0x0d,
+};
+
+struct hisi_zip_ctrl;
+
+struct hisi_zip {
+ struct hisi_qm qm;
+ struct list_head list;
+ struct hisi_zip_ctrl *ctrl;
+};
+
+struct hisi_zip_sqe {
+ u32 consumed;
+ u32 produced;
+ u32 comp_data_length;
+ u32 dw3;
+ u32 input_data_length;
+ u32 lba_l;
+ u32 lba_h;
+ u32 dw7;
+ u32 dw8;
+ u32 dw9;
+ u32 dw10;
+ u32 priv_info;
+ u32 dw12;
+ u32 tag;
+ u32 dest_avail_out;
+ u32 rsvd0;
+ u32 comp_head_addr_l;
+ u32 comp_head_addr_h;
+ u32 source_addr_l;
+ u32 source_addr_h;
+ u32 dest_addr_l;
+ u32 dest_addr_h;
+ u32 stream_ctx_addr_l;
+ u32 stream_ctx_addr_h;
+ u32 cipher_key1_addr_l;
+ u32 cipher_key1_addr_h;
+ u32 cipher_key2_addr_l;
+ u32 cipher_key2_addr_h;
+ u32 rsvd1[4];
+};
+
+struct hisi_zip *find_zip_device(int node);
+int hisi_zip_register_to_crypto(void);
+void hisi_zip_unregister_from_crypto(void);
+#endif
diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c
new file mode 100644
index 000000000000..5a3f84dcdcde
--- /dev/null
+++ b/drivers/crypto/hisilicon/zip/zip_crypto.c
@@ -0,0 +1,653 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2019 HiSilicon Limited. */
+#include <crypto/internal/acompress.h>
+#include <linux/bitfield.h>
+#include <linux/dma-mapping.h>
+#include <linux/scatterlist.h>
+#include "zip.h"
+
+#define HZIP_ZLIB_HEAD_SIZE 2
+#define HZIP_GZIP_HEAD_SIZE 10
+
+#define GZIP_HEAD_FHCRC_BIT BIT(1)
+#define GZIP_HEAD_FEXTRA_BIT BIT(2)
+#define GZIP_HEAD_FNAME_BIT BIT(3)
+#define GZIP_HEAD_FCOMMENT_BIT BIT(4)
+
+#define GZIP_HEAD_FLG_SHIFT 3
+#define GZIP_HEAD_FEXTRA_SHIFT 10
+#define GZIP_HEAD_FEXTRA_XLEN 2
+#define GZIP_HEAD_FHCRC_SIZE 2
+
+#define HZIP_CTX_Q_NUM 2
+#define HZIP_GZIP_HEAD_BUF 256
+#define HZIP_ALG_PRIORITY 300
+
+static const u8 zlib_head[HZIP_ZLIB_HEAD_SIZE] = {0x78, 0x9c};
+static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = {0x1f, 0x8b, 0x08, 0x0, 0x0,
+ 0x0, 0x0, 0x0, 0x0, 0x03};
+enum hisi_zip_alg_type {
+ HZIP_ALG_TYPE_COMP = 0,
+ HZIP_ALG_TYPE_DECOMP = 1,
+};
+
+#define COMP_NAME_TO_TYPE(alg_name) \
+ (!strcmp((alg_name), "zlib-deflate") ? HZIP_ALG_TYPE_ZLIB : \
+ !strcmp((alg_name), "gzip") ? HZIP_ALG_TYPE_GZIP : 0) \
+
+#define TO_HEAD_SIZE(req_type) \
+ (((req_type) == HZIP_ALG_TYPE_ZLIB) ? sizeof(zlib_head) : \
+ ((req_type) == HZIP_ALG_TYPE_GZIP) ? sizeof(gzip_head) : 0) \
+
+#define TO_HEAD(req_type) \
+ (((req_type) == HZIP_ALG_TYPE_ZLIB) ? zlib_head : \
+ ((req_type) == HZIP_ALG_TYPE_GZIP) ? gzip_head : 0) \
+
+struct hisi_zip_req {
+ struct acomp_req *req;
+ struct scatterlist *src;
+ struct scatterlist *dst;
+ size_t slen;
+ size_t dlen;
+ struct hisi_acc_hw_sgl *hw_src;
+ struct hisi_acc_hw_sgl *hw_dst;
+ dma_addr_t dma_src;
+ dma_addr_t dma_dst;
+ int req_id;
+};
+
+struct hisi_zip_req_q {
+ struct hisi_zip_req *q;
+ unsigned long *req_bitmap;
+ rwlock_t req_lock;
+ u16 size;
+};
+
+struct hisi_zip_qp_ctx {
+ struct hisi_qp *qp;
+ struct hisi_zip_sqe zip_sqe;
+ struct hisi_zip_req_q req_q;
+ struct hisi_acc_sgl_pool sgl_pool;
+ struct hisi_zip *zip_dev;
+ struct hisi_zip_ctx *ctx;
+};
+
+struct hisi_zip_ctx {
+#define QPC_COMP 0
+#define QPC_DECOMP 1
+ struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM];
+};
+
+static void hisi_zip_config_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type)
+{
+ u32 val;
+
+ val = (sqe->dw9) & ~HZIP_BUF_TYPE_M;
+ val |= FIELD_PREP(HZIP_BUF_TYPE_M, buf_type);
+ sqe->dw9 = val;
+}
+
+static void hisi_zip_config_tag(struct hisi_zip_sqe *sqe, u32 tag)
+{
+ sqe->tag = tag;
+}
+
+static void hisi_zip_fill_sqe(struct hisi_zip_sqe *sqe, u8 req_type,
+ dma_addr_t s_addr, dma_addr_t d_addr, u32 slen,
+ u32 dlen)
+{
+ memset(sqe, 0, sizeof(struct hisi_zip_sqe));
+
+ sqe->input_data_length = slen;
+ sqe->dw9 = FIELD_PREP(HZIP_REQ_TYPE_M, req_type);
+ sqe->dest_avail_out = dlen;
+ sqe->source_addr_l = lower_32_bits(s_addr);
+ sqe->source_addr_h = upper_32_bits(s_addr);
+ sqe->dest_addr_l = lower_32_bits(d_addr);
+ sqe->dest_addr_h = upper_32_bits(d_addr);
+}
+
+static int hisi_zip_create_qp(struct hisi_qm *qm, struct hisi_zip_qp_ctx *ctx,
+ int alg_type, int req_type)
+{
+ struct hisi_qp *qp;
+ int ret;
+
+ qp = hisi_qm_create_qp(qm, alg_type);
+ if (IS_ERR(qp))
+ return PTR_ERR(qp);
+
+ qp->req_type = req_type;
+ qp->qp_ctx = ctx;
+ ctx->qp = qp;
+
+ ret = hisi_qm_start_qp(qp, 0);
+ if (ret < 0)
+ goto err_release_qp;
+
+ return 0;
+
+err_release_qp:
+ hisi_qm_release_qp(qp);
+ return ret;
+}
+
+static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *ctx)
+{
+ hisi_qm_stop_qp(ctx->qp);
+ hisi_qm_release_qp(ctx->qp);
+}
+
+static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type)
+{
+ struct hisi_zip *hisi_zip;
+ struct hisi_qm *qm;
+ int ret, i, j;
+
+ /* find the proper zip device */
+ hisi_zip = find_zip_device(cpu_to_node(smp_processor_id()));
+ if (!hisi_zip) {
+ pr_err("Failed to find a proper ZIP device!\n");
+ return -ENODEV;
+ }
+ qm = &hisi_zip->qm;
+
+ for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
+ /* alg_type = 0 for compress, 1 for decompress in hw sqe */
+ ret = hisi_zip_create_qp(qm, &hisi_zip_ctx->qp_ctx[i], i,
+ req_type);
+ if (ret)
+ goto err;
+
+ hisi_zip_ctx->qp_ctx[i].zip_dev = hisi_zip;
+ }
+
+ return 0;
+err:
+ for (j = i - 1; j >= 0; j--)
+ hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[j]);
+
+ return ret;
+}
+
+static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx)
+{
+ int i;
+
+ for (i = 1; i >= 0; i--)
+ hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]);
+}
+
+static u16 get_extra_field_size(const u8 *start)
+{
+ return *((u16 *)start) + GZIP_HEAD_FEXTRA_XLEN;
+}
+
+static u32 get_name_field_size(const u8 *start)
+{
+ return strlen(start) + 1;
+}
+
+static u32 get_comment_field_size(const u8 *start)
+{
+ return strlen(start) + 1;
+}
+
+static u32 __get_gzip_head_size(const u8 *src)
+{
+ u8 head_flg = *(src + GZIP_HEAD_FLG_SHIFT);
+ u32 size = GZIP_HEAD_FEXTRA_SHIFT;
+
+ if (head_flg & GZIP_HEAD_FEXTRA_BIT)
+ size += get_extra_field_size(src + size);
+ if (head_flg & GZIP_HEAD_FNAME_BIT)
+ size += get_name_field_size(src + size);
+ if (head_flg & GZIP_HEAD_FCOMMENT_BIT)
+ size += get_comment_field_size(src + size);
+ if (head_flg & GZIP_HEAD_FHCRC_BIT)
+ size += GZIP_HEAD_FHCRC_SIZE;
+
+ return size;
+}
+
+static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx)
+{
+ struct hisi_zip_req_q *req_q;
+ int i, ret;
+
+ for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
+ req_q = &ctx->qp_ctx[i].req_q;
+ req_q->size = QM_Q_DEPTH;
+
+ req_q->req_bitmap = kcalloc(BITS_TO_LONGS(req_q->size),
+ sizeof(long), GFP_KERNEL);
+ if (!req_q->req_bitmap) {
+ ret = -ENOMEM;
+ if (i == 0)
+ return ret;
+
+ goto err_free_loop0;
+ }
+ rwlock_init(&req_q->req_lock);
+
+ req_q->q = kcalloc(req_q->size, sizeof(struct hisi_zip_req),
+ GFP_KERNEL);
+ if (!req_q->q) {
+ ret = -ENOMEM;
+ if (i == 0)
+ goto err_free_bitmap;
+ else
+ goto err_free_loop1;
+ }
+ }
+
+ return 0;
+
+err_free_loop1:
+ kfree(ctx->qp_ctx[QPC_DECOMP].req_q.req_bitmap);
+err_free_loop0:
+ kfree(ctx->qp_ctx[QPC_COMP].req_q.q);
+err_free_bitmap:
+ kfree(ctx->qp_ctx[QPC_COMP].req_q.req_bitmap);
+ return ret;
+}
+
+static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx)
+{
+ int i;
+
+ for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
+ kfree(ctx->qp_ctx[i].req_q.q);
+ kfree(ctx->qp_ctx[i].req_q.req_bitmap);
+ }
+}
+
+static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx)
+{
+ struct hisi_zip_qp_ctx *tmp;
+ int i, ret;
+
+ for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
+ tmp = &ctx->qp_ctx[i];
+ ret = hisi_acc_create_sgl_pool(&tmp->qp->qm->pdev->dev,
+ &tmp->sgl_pool,
+ QM_Q_DEPTH << 1);
+ if (ret < 0) {
+ if (i == 1)
+ goto err_free_sgl_pool0;
+ return -ENOMEM;
+ }
+ }
+
+ return 0;
+
+err_free_sgl_pool0:
+ hisi_acc_free_sgl_pool(&ctx->qp_ctx[QPC_COMP].qp->qm->pdev->dev,
+ &ctx->qp_ctx[QPC_COMP].sgl_pool);
+ return -ENOMEM;
+}
+
+static void hisi_zip_release_sgl_pool(struct hisi_zip_ctx *ctx)
+{
+ int i;
+
+ for (i = 0; i < HZIP_CTX_Q_NUM; i++)
+ hisi_acc_free_sgl_pool(&ctx->qp_ctx[i].qp->qm->pdev->dev,
+ &ctx->qp_ctx[i].sgl_pool);
+}
+
+static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx,
+ struct hisi_zip_req *req)
+{
+ struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
+
+ if (qp_ctx->qp->alg_type == HZIP_ALG_TYPE_COMP)
+ kfree(req->dst);
+ else
+ kfree(req->src);
+
+ write_lock(&req_q->req_lock);
+ clear_bit(req->req_id, req_q->req_bitmap);
+ memset(req, 0, sizeof(struct hisi_zip_req));
+ write_unlock(&req_q->req_lock);
+}
+
+static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data)
+{
+ struct hisi_zip_sqe *sqe = data;
+ struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx;
+ struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
+ struct hisi_zip_req *req = req_q->q + sqe->tag;
+ struct acomp_req *acomp_req = req->req;
+ struct device *dev = &qp->qm->pdev->dev;
+ u32 status, dlen, head_size;
+ int err = 0;
+
+ status = sqe->dw3 & HZIP_BD_STATUS_M;
+
+ if (status != 0 && status != HZIP_NC_ERR) {
+ dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n",
+ (qp->alg_type == 0) ? "" : "de", qp->qp_id, status,
+ sqe->produced);
+ err = -EIO;
+ }
+ dlen = sqe->produced;
+
+ hisi_acc_sg_buf_unmap(dev, req->src, req->hw_src);
+ hisi_acc_sg_buf_unmap(dev, req->dst, req->hw_dst);
+
+ head_size = (qp->alg_type == 0) ? TO_HEAD_SIZE(qp->req_type) : 0;
+ acomp_req->dlen = dlen + head_size;
+
+ if (acomp_req->base.complete)
+ acomp_request_complete(acomp_req, err);
+
+ hisi_zip_remove_req(qp_ctx, req);
+}
+
+static void hisi_zip_set_acomp_cb(struct hisi_zip_ctx *ctx,
+ void (*fn)(struct hisi_qp *, void *))
+{
+ int i;
+
+ for (i = 0; i < HZIP_CTX_Q_NUM; i++)
+ ctx->qp_ctx[i].qp->req_cb = fn;
+}
+
+static int hisi_zip_acomp_init(struct crypto_acomp *tfm)
+{
+ const char *alg_name = crypto_tfm_alg_name(&tfm->base);
+ struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base);
+ int ret;
+
+ ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name));
+ if (ret)
+ return ret;
+
+ ret = hisi_zip_create_req_q(ctx);
+ if (ret)
+ goto err_ctx_exit;
+
+ ret = hisi_zip_create_sgl_pool(ctx);
+ if (ret)
+ goto err_release_req_q;
+
+ hisi_zip_set_acomp_cb(ctx, hisi_zip_acomp_cb);
+
+ return 0;
+
+err_release_req_q:
+ hisi_zip_release_req_q(ctx);
+err_ctx_exit:
+ hisi_zip_ctx_exit(ctx);
+ return ret;
+}
+
+static void hisi_zip_acomp_exit(struct crypto_acomp *tfm)
+{
+ struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base);
+
+ hisi_zip_set_acomp_cb(ctx, NULL);
+ hisi_zip_release_sgl_pool(ctx);
+ hisi_zip_release_req_q(ctx);
+ hisi_zip_ctx_exit(ctx);
+}
+
+static int add_comp_head(struct scatterlist *dst, u8 req_type)
+{
+ int head_size = TO_HEAD_SIZE(req_type);
+ const u8 *head = TO_HEAD(req_type);
+ int ret;
+
+ ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size);
+ if (ret != head_size)
+ return -ENOMEM;
+
+ return head_size;
+}
+
+static size_t get_gzip_head_size(struct scatterlist *sgl)
+{
+ char buf[HZIP_GZIP_HEAD_BUF];
+
+ sg_copy_to_buffer(sgl, sg_nents(sgl), buf, sizeof(buf));
+
+ return __get_gzip_head_size(buf);
+}
+
+static size_t get_comp_head_size(struct scatterlist *src, u8 req_type)
+{
+ switch (req_type) {
+ case HZIP_ALG_TYPE_ZLIB:
+ return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB);
+ case HZIP_ALG_TYPE_GZIP:
+ return get_gzip_head_size(src);
+ default:
+ pr_err("request type does not support!\n");
+ return -EINVAL;
+ }
+}
+
+static int get_sg_skip_bytes(struct scatterlist *sgl, size_t bytes,
+ size_t remains, struct scatterlist **out)
+{
+#define SPLIT_NUM 2
+ size_t split_sizes[SPLIT_NUM];
+ int out_mapped_nents[SPLIT_NUM];
+
+ split_sizes[0] = bytes;
+ split_sizes[1] = remains;
+
+ return sg_split(sgl, 0, 0, SPLIT_NUM, split_sizes, out,
+ out_mapped_nents, GFP_KERNEL);
+}
+
+static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req,
+ struct hisi_zip_qp_ctx *qp_ctx,
+ size_t head_size, bool is_comp)
+{
+ struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
+ struct hisi_zip_req *q = req_q->q;
+ struct hisi_zip_req *req_cache;
+ struct scatterlist *out[2];
+ struct scatterlist *sgl;
+ size_t len;
+ int ret, req_id;
+
+ /*
+ * remove/add zlib/gzip head, as hardware operations do not include
+ * comp head. so split req->src to get sgl without heads in acomp, or
+ * add comp head to req->dst ahead of that hardware output compressed
+ * data in sgl splited from req->dst without comp head.
+ */
+ if (is_comp) {
+ sgl = req->dst;
+ len = req->dlen - head_size;
+ } else {
+ sgl = req->src;
+ len = req->slen - head_size;
+ }
+
+ ret = get_sg_skip_bytes(sgl, head_size, len, out);
+ if (ret)
+ return ERR_PTR(ret);
+
+ /* sgl for comp head is useless, so free it now */
+ kfree(out[0]);
+
+ write_lock(&req_q->req_lock);
+
+ req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size);
+ if (req_id >= req_q->size) {
+ write_unlock(&req_q->req_lock);
+ dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n");
+ kfree(out[1]);
+ return ERR_PTR(-EBUSY);
+ }
+ set_bit(req_id, req_q->req_bitmap);
+
+ req_cache = q + req_id;
+ req_cache->req_id = req_id;
+ req_cache->req = req;
+ if (is_comp) {
+ req_cache->src = req->src;
+ req_cache->dst = out[1];
+ req_cache->slen = req->slen;
+ req_cache->dlen = req->dlen - head_size;
+ } else {
+ req_cache->src = out[1];
+ req_cache->dst = req->dst;
+ req_cache->slen = req->slen - head_size;
+ req_cache->dlen = req->dlen;
+ }
+
+ write_unlock(&req_q->req_lock);
+
+ return req_cache;
+}
+
+static int hisi_zip_do_work(struct hisi_zip_req *req,
+ struct hisi_zip_qp_ctx *qp_ctx)
+{
+ struct hisi_zip_sqe *zip_sqe = &qp_ctx->zip_sqe;
+ struct hisi_qp *qp = qp_ctx->qp;
+ struct device *dev = &qp->qm->pdev->dev;
+ struct hisi_acc_sgl_pool *pool = &qp_ctx->sgl_pool;
+ dma_addr_t input;
+ dma_addr_t output;
+ int ret;
+
+ if (!req->src || !req->slen || !req->dst || !req->dlen)
+ return -EINVAL;
+
+ req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, req->src, pool,
+ req->req_id << 1, &input);
+ if (IS_ERR(req->hw_src))
+ return PTR_ERR(req->hw_src);
+ req->dma_src = input;
+
+ req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, req->dst, pool,
+ (req->req_id << 1) + 1,
+ &output);
+ if (IS_ERR(req->hw_dst)) {
+ ret = PTR_ERR(req->hw_dst);
+ goto err_unmap_input;
+ }
+ req->dma_dst = output;
+
+ hisi_zip_fill_sqe(zip_sqe, qp->req_type, input, output, req->slen,
+ req->dlen);
+ hisi_zip_config_buf_type(zip_sqe, HZIP_SGL);
+ hisi_zip_config_tag(zip_sqe, req->req_id);
+
+ /* send command to start a task */
+ ret = hisi_qp_send(qp, zip_sqe);
+ if (ret < 0)
+ goto err_unmap_output;
+
+ return -EINPROGRESS;
+
+err_unmap_output:
+ hisi_acc_sg_buf_unmap(dev, req->dst, req->hw_dst);
+err_unmap_input:
+ hisi_acc_sg_buf_unmap(dev, req->src, req->hw_src);
+ return ret;
+}
+
+static int hisi_zip_acompress(struct acomp_req *acomp_req)
+{
+ struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
+ struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_COMP];
+ struct hisi_zip_req *req;
+ size_t head_size;
+ int ret;
+
+ /* let's output compression head now */
+ head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type);
+ if (head_size < 0)
+ return -ENOMEM;
+
+ req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, true);
+ if (IS_ERR(req))
+ return PTR_ERR(req);
+
+ ret = hisi_zip_do_work(req, qp_ctx);
+ if (ret != -EINPROGRESS)
+ hisi_zip_remove_req(qp_ctx, req);
+
+ return ret;
+}
+
+static int hisi_zip_adecompress(struct acomp_req *acomp_req)
+{
+ struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
+ struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_DECOMP];
+ struct hisi_zip_req *req;
+ size_t head_size;
+ int ret;
+
+ head_size = get_comp_head_size(acomp_req->src, qp_ctx->qp->req_type);
+
+ req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false);
+ if (IS_ERR(req))
+ return PTR_ERR(req);
+
+ ret = hisi_zip_do_work(req, qp_ctx);
+ if (ret != -EINPROGRESS)
+ hisi_zip_remove_req(qp_ctx, req);
+
+ return ret;
+}
+
+static struct acomp_alg hisi_zip_acomp_zlib = {
+ .init = hisi_zip_acomp_init,
+ .exit = hisi_zip_acomp_exit,
+ .compress = hisi_zip_acompress,
+ .decompress = hisi_zip_adecompress,
+ .base = {
+ .cra_name = "zlib-deflate",
+ .cra_driver_name = "hisi-zlib-acomp",
+ .cra_module = THIS_MODULE,
+ .cra_priority = HZIP_ALG_PRIORITY,
+ .cra_ctxsize = sizeof(struct hisi_zip_ctx),
+ }
+};
+
+static struct acomp_alg hisi_zip_acomp_gzip = {
+ .init = hisi_zip_acomp_init,
+ .exit = hisi_zip_acomp_exit,
+ .compress = hisi_zip_acompress,
+ .decompress = hisi_zip_adecompress,
+ .base = {
+ .cra_name = "gzip",
+ .cra_driver_name = "hisi-gzip-acomp",
+ .cra_module = THIS_MODULE,
+ .cra_priority = HZIP_ALG_PRIORITY,
+ .cra_ctxsize = sizeof(struct hisi_zip_ctx),
+ }
+};
+
+int hisi_zip_register_to_crypto(void)
+{
+ int ret = 0;
+
+ ret = crypto_register_acomp(&hisi_zip_acomp_zlib);
+ if (ret) {
+ pr_err("Zlib acomp algorithm registration failed\n");
+ return ret;
+ }
+
+ ret = crypto_register_acomp(&hisi_zip_acomp_gzip);
+ if (ret) {
+ pr_err("Gzip acomp algorithm registration failed\n");
+ crypto_unregister_acomp(&hisi_zip_acomp_zlib);
+ }
+
+ return ret;
+}
+
+void hisi_zip_unregister_from_crypto(void)
+{
+ crypto_unregister_acomp(&hisi_zip_acomp_gzip);
+ crypto_unregister_acomp(&hisi_zip_acomp_zlib);
+}
diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c
new file mode 100644
index 000000000000..6e0ca75585d4
--- /dev/null
+++ b/drivers/crypto/hisilicon/zip/zip_main.c
@@ -0,0 +1,1013 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2019 HiSilicon Limited. */
+#include <linux/acpi.h>
+#include <linux/aer.h>
+#include <linux/bitops.h>
+#include <linux/debugfs.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/seq_file.h>
+#include <linux/topology.h>
+#include "zip.h"
+
+#define PCI_DEVICE_ID_ZIP_PF 0xa250
+#define PCI_DEVICE_ID_ZIP_VF 0xa251
+
+#define HZIP_VF_NUM 63
+#define HZIP_QUEUE_NUM_V1 4096
+#define HZIP_QUEUE_NUM_V2 1024
+
+#define HZIP_CLOCK_GATE_CTRL 0x301004
+#define COMP0_ENABLE BIT(0)
+#define COMP1_ENABLE BIT(1)
+#define DECOMP0_ENABLE BIT(2)
+#define DECOMP1_ENABLE BIT(3)
+#define DECOMP2_ENABLE BIT(4)
+#define DECOMP3_ENABLE BIT(5)
+#define DECOMP4_ENABLE BIT(6)
+#define DECOMP5_ENABLE BIT(7)
+#define ALL_COMP_DECOMP_EN (COMP0_ENABLE | COMP1_ENABLE | \
+ DECOMP0_ENABLE | DECOMP1_ENABLE | \
+ DECOMP2_ENABLE | DECOMP3_ENABLE | \
+ DECOMP4_ENABLE | DECOMP5_ENABLE)
+#define DECOMP_CHECK_ENABLE BIT(16)
+#define HZIP_FSM_MAX_CNT 0x301008
+
+#define HZIP_PORT_ARCA_CHE_0 0x301040
+#define HZIP_PORT_ARCA_CHE_1 0x301044
+#define HZIP_PORT_AWCA_CHE_0 0x301060
+#define HZIP_PORT_AWCA_CHE_1 0x301064
+#define CACHE_ALL_EN 0xffffffff
+
+#define HZIP_BD_RUSER_32_63 0x301110
+#define HZIP_SGL_RUSER_32_63 0x30111c
+#define HZIP_DATA_RUSER_32_63 0x301128
+#define HZIP_DATA_WUSER_32_63 0x301134
+#define HZIP_BD_WUSER_32_63 0x301140
+
+#define HZIP_QM_IDEL_STATUS 0x3040e4
+
+#define HZIP_CORE_DEBUG_COMP_0 0x302000
+#define HZIP_CORE_DEBUG_COMP_1 0x303000
+#define HZIP_CORE_DEBUG_DECOMP_0 0x304000
+#define HZIP_CORE_DEBUG_DECOMP_1 0x305000
+#define HZIP_CORE_DEBUG_DECOMP_2 0x306000
+#define HZIP_CORE_DEBUG_DECOMP_3 0x307000
+#define HZIP_CORE_DEBUG_DECOMP_4 0x308000
+#define HZIP_CORE_DEBUG_DECOMP_5 0x309000
+
+#define HZIP_CORE_INT_SOURCE 0x3010A0
+#define HZIP_CORE_INT_MASK 0x3010A4
+#define HZIP_CORE_INT_STATUS 0x3010AC
+#define HZIP_CORE_INT_STATUS_M_ECC BIT(1)
+#define HZIP_CORE_SRAM_ECC_ERR_INFO 0x301148
+#define SRAM_ECC_ERR_NUM_SHIFT 16
+#define SRAM_ECC_ERR_ADDR_SHIFT 24
+#define HZIP_CORE_INT_DISABLE 0x000007FF
+#define HZIP_COMP_CORE_NUM 2
+#define HZIP_DECOMP_CORE_NUM 6
+#define HZIP_CORE_NUM (HZIP_COMP_CORE_NUM + \
+ HZIP_DECOMP_CORE_NUM)
+#define HZIP_SQE_SIZE 128
+#define HZIP_SQ_SIZE (HZIP_SQE_SIZE * QM_Q_DEPTH)
+#define HZIP_PF_DEF_Q_NUM 64
+#define HZIP_PF_DEF_Q_BASE 0
+
+#define HZIP_SOFT_CTRL_CNT_CLR_CE 0x301000
+#define SOFT_CTRL_CNT_CLR_CE_BIT BIT(0)
+
+#define HZIP_NUMA_DISTANCE 100
+#define HZIP_BUF_SIZE 22
+
+static const char hisi_zip_name[] = "hisi_zip";
+static struct dentry *hzip_debugfs_root;
+LIST_HEAD(hisi_zip_list);
+DEFINE_MUTEX(hisi_zip_list_lock);
+
+#ifdef CONFIG_NUMA
+static struct hisi_zip *find_zip_device_numa(int node)
+{
+ struct hisi_zip *zip = NULL;
+ struct hisi_zip *hisi_zip;
+ int min_distance = HZIP_NUMA_DISTANCE;
+ struct device *dev;
+
+ list_for_each_entry(hisi_zip, &hisi_zip_list, list) {
+ dev = &hisi_zip->qm.pdev->dev;
+ if (node_distance(dev->numa_node, node) < min_distance) {
+ zip = hisi_zip;
+ min_distance = node_distance(dev->numa_node, node);
+ }
+ }
+
+ return zip;
+}
+#endif
+
+struct hisi_zip *find_zip_device(int node)
+{
+ struct hisi_zip *zip = NULL;
+
+ mutex_lock(&hisi_zip_list_lock);
+#ifdef CONFIG_NUMA
+ zip = find_zip_device_numa(node);
+#else
+ zip = list_first_entry(&hisi_zip_list, struct hisi_zip, list);
+#endif
+ mutex_unlock(&hisi_zip_list_lock);
+
+ return zip;
+}
+
+struct hisi_zip_hw_error {
+ u32 int_msk;
+ const char *msg;
+};
+
+static const struct hisi_zip_hw_error zip_hw_error[] = {
+ { .int_msk = BIT(0), .msg = "zip_ecc_1bitt_err" },
+ { .int_msk = BIT(1), .msg = "zip_ecc_2bit_err" },
+ { .int_msk = BIT(2), .msg = "zip_axi_rresp_err" },
+ { .int_msk = BIT(3), .msg = "zip_axi_bresp_err" },
+ { .int_msk = BIT(4), .msg = "zip_src_addr_parse_err" },
+ { .int_msk = BIT(5), .msg = "zip_dst_addr_parse_err" },
+ { .int_msk = BIT(6), .msg = "zip_pre_in_addr_err" },
+ { .int_msk = BIT(7), .msg = "zip_pre_in_data_err" },
+ { .int_msk = BIT(8), .msg = "zip_com_inf_err" },
+ { .int_msk = BIT(9), .msg = "zip_enc_inf_err" },
+ { .int_msk = BIT(10), .msg = "zip_pre_out_err" },
+ { /* sentinel */ }
+};
+
+enum ctrl_debug_file_index {
+ HZIP_CURRENT_QM,
+ HZIP_CLEAR_ENABLE,
+ HZIP_DEBUG_FILE_NUM,
+};
+
+static const char * const ctrl_debug_file_name[] = {
+ [HZIP_CURRENT_QM] = "current_qm",
+ [HZIP_CLEAR_ENABLE] = "clear_enable",
+};
+
+struct ctrl_debug_file {
+ enum ctrl_debug_file_index index;
+ spinlock_t lock;
+ struct hisi_zip_ctrl *ctrl;
+};
+
+/*
+ * One ZIP controller has one PF and multiple VFs, some global configurations
+ * which PF has need this structure.
+ *
+ * Just relevant for PF.
+ */
+struct hisi_zip_ctrl {
+ u32 num_vfs;
+ struct hisi_zip *hisi_zip;
+ struct dentry *debug_root;
+ struct ctrl_debug_file files[HZIP_DEBUG_FILE_NUM];
+};
+
+enum {
+ HZIP_COMP_CORE0,
+ HZIP_COMP_CORE1,
+ HZIP_DECOMP_CORE0,
+ HZIP_DECOMP_CORE1,
+ HZIP_DECOMP_CORE2,
+ HZIP_DECOMP_CORE3,
+ HZIP_DECOMP_CORE4,
+ HZIP_DECOMP_CORE5,
+};
+
+static const u64 core_offsets[] = {
+ [HZIP_COMP_CORE0] = 0x302000,
+ [HZIP_COMP_CORE1] = 0x303000,
+ [HZIP_DECOMP_CORE0] = 0x304000,
+ [HZIP_DECOMP_CORE1] = 0x305000,
+ [HZIP_DECOMP_CORE2] = 0x306000,
+ [HZIP_DECOMP_CORE3] = 0x307000,
+ [HZIP_DECOMP_CORE4] = 0x308000,
+ [HZIP_DECOMP_CORE5] = 0x309000,
+};
+
+static struct debugfs_reg32 hzip_dfx_regs[] = {
+ {"HZIP_GET_BD_NUM ", 0x00ull},
+ {"HZIP_GET_RIGHT_BD ", 0x04ull},
+ {"HZIP_GET_ERROR_BD ", 0x08ull},
+ {"HZIP_DONE_BD_NUM ", 0x0cull},
+ {"HZIP_WORK_CYCLE ", 0x10ull},
+ {"HZIP_IDLE_CYCLE ", 0x18ull},
+ {"HZIP_MAX_DELAY ", 0x20ull},
+ {"HZIP_MIN_DELAY ", 0x24ull},
+ {"HZIP_AVG_DELAY ", 0x28ull},
+ {"HZIP_MEM_VISIBLE_DATA ", 0x30ull},
+ {"HZIP_MEM_VISIBLE_ADDR ", 0x34ull},
+ {"HZIP_COMSUMED_BYTE ", 0x38ull},
+ {"HZIP_PRODUCED_BYTE ", 0x40ull},
+ {"HZIP_COMP_INF ", 0x70ull},
+ {"HZIP_PRE_OUT ", 0x78ull},
+ {"HZIP_BD_RD ", 0x7cull},
+ {"HZIP_BD_WR ", 0x80ull},
+ {"HZIP_GET_BD_AXI_ERR_NUM ", 0x84ull},
+ {"HZIP_GET_BD_PARSE_ERR_NUM ", 0x88ull},
+ {"HZIP_ADD_BD_AXI_ERR_NUM ", 0x8cull},
+ {"HZIP_DECOMP_STF_RELOAD_CURR_ST ", 0x94ull},
+ {"HZIP_DECOMP_LZ77_CURR_ST ", 0x9cull},
+};
+
+static int pf_q_num_set(const char *val, const struct kernel_param *kp)
+{
+ struct pci_dev *pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI,
+ PCI_DEVICE_ID_ZIP_PF, NULL);
+ u32 n, q_num;
+ u8 rev_id;
+ int ret;
+
+ if (!val)
+ return -EINVAL;
+
+ if (!pdev) {
+ q_num = min_t(u32, HZIP_QUEUE_NUM_V1, HZIP_QUEUE_NUM_V2);
+ pr_info("No device found currently, suppose queue number is %d\n",
+ q_num);
+ } else {
+ rev_id = pdev->revision;
+ switch (rev_id) {
+ case QM_HW_V1:
+ q_num = HZIP_QUEUE_NUM_V1;
+ break;
+ case QM_HW_V2:
+ q_num = HZIP_QUEUE_NUM_V2;
+ break;
+ default:
+ return -EINVAL;
+ }
+ }
+
+ ret = kstrtou32(val, 10, &n);
+ if (ret != 0 || n > q_num || n == 0)
+ return -EINVAL;
+
+ return param_set_int(val, kp);
+}
+
+static const struct kernel_param_ops pf_q_num_ops = {
+ .set = pf_q_num_set,
+ .get = param_get_int,
+};
+
+static u32 pf_q_num = HZIP_PF_DEF_Q_NUM;
+module_param_cb(pf_q_num, &pf_q_num_ops, &pf_q_num, 0444);
+MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 1-4096, v2 1-1024)");
+
+static int uacce_mode;
+module_param(uacce_mode, int, 0);
+
+static const struct pci_device_id hisi_zip_dev_ids[] = {
+ { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_ZIP_PF) },
+ { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_ZIP_VF) },
+ { 0, }
+};
+MODULE_DEVICE_TABLE(pci, hisi_zip_dev_ids);
+
+static inline void hisi_zip_add_to_list(struct hisi_zip *hisi_zip)
+{
+ mutex_lock(&hisi_zip_list_lock);
+ list_add_tail(&hisi_zip->list, &hisi_zip_list);
+ mutex_unlock(&hisi_zip_list_lock);
+}
+
+static inline void hisi_zip_remove_from_list(struct hisi_zip *hisi_zip)
+{
+ mutex_lock(&hisi_zip_list_lock);
+ list_del(&hisi_zip->list);
+ mutex_unlock(&hisi_zip_list_lock);
+}
+
+static void hisi_zip_set_user_domain_and_cache(struct hisi_zip *hisi_zip)
+{
+ void __iomem *base = hisi_zip->qm.io_base;
+
+ /* qm user domain */
+ writel(AXUSER_BASE, base + QM_ARUSER_M_CFG_1);
+ writel(ARUSER_M_CFG_ENABLE, base + QM_ARUSER_M_CFG_ENABLE);
+ writel(AXUSER_BASE, base + QM_AWUSER_M_CFG_1);
+ writel(AWUSER_M_CFG_ENABLE, base + QM_AWUSER_M_CFG_ENABLE);
+ writel(WUSER_M_CFG_ENABLE, base + QM_WUSER_M_CFG_ENABLE);
+
+ /* qm cache */
+ writel(AXI_M_CFG, base + QM_AXI_M_CFG);
+ writel(AXI_M_CFG_ENABLE, base + QM_AXI_M_CFG_ENABLE);
+ /* disable FLR triggered by BME(bus master enable) */
+ writel(PEH_AXUSER_CFG, base + QM_PEH_AXUSER_CFG);
+ writel(PEH_AXUSER_CFG_ENABLE, base + QM_PEH_AXUSER_CFG_ENABLE);
+
+ /* cache */
+ writel(CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_0);
+ writel(CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_1);
+ writel(CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_0);
+ writel(CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_1);
+
+ /* user domain configurations */
+ writel(AXUSER_BASE, base + HZIP_BD_RUSER_32_63);
+ writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63);
+ writel(AXUSER_BASE, base + HZIP_BD_WUSER_32_63);
+ writel(AXUSER_BASE, base + HZIP_DATA_RUSER_32_63);
+ writel(AXUSER_BASE, base + HZIP_DATA_WUSER_32_63);
+
+ /* let's open all compression/decompression cores */
+ writel(DECOMP_CHECK_ENABLE | ALL_COMP_DECOMP_EN,
+ base + HZIP_CLOCK_GATE_CTRL);
+
+ /* enable sqc writeback */
+ writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE |
+ CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) |
+ FIELD_PREP(CQC_CACHE_WB_THRD, 1), base + QM_CACHE_CTL);
+}
+
+static void hisi_zip_hw_error_set_state(struct hisi_zip *hisi_zip, bool state)
+{
+ struct hisi_qm *qm = &hisi_zip->qm;
+
+ if (qm->ver == QM_HW_V1) {
+ writel(HZIP_CORE_INT_DISABLE, qm->io_base + HZIP_CORE_INT_MASK);
+ dev_info(&qm->pdev->dev, "ZIP v%d does not support hw error handle\n",
+ qm->ver);
+ return;
+ }
+
+ if (state) {
+ /* clear ZIP hw error source if having */
+ writel(HZIP_CORE_INT_DISABLE, hisi_zip->qm.io_base +
+ HZIP_CORE_INT_SOURCE);
+ /* enable ZIP hw error interrupts */
+ writel(0, hisi_zip->qm.io_base + HZIP_CORE_INT_MASK);
+ } else {
+ /* disable ZIP hw error interrupts */
+ writel(HZIP_CORE_INT_DISABLE,
+ hisi_zip->qm.io_base + HZIP_CORE_INT_MASK);
+ }
+}
+
+static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file)
+{
+ struct hisi_zip *hisi_zip = file->ctrl->hisi_zip;
+
+ return &hisi_zip->qm;
+}
+
+static u32 current_qm_read(struct ctrl_debug_file *file)
+{
+ struct hisi_qm *qm = file_to_qm(file);
+
+ return readl(qm->io_base + QM_DFX_MB_CNT_VF);
+}
+
+static int current_qm_write(struct ctrl_debug_file *file, u32 val)
+{
+ struct hisi_qm *qm = file_to_qm(file);
+ struct hisi_zip_ctrl *ctrl = file->ctrl;
+ u32 vfq_num;
+ u32 tmp;
+
+ if (val > ctrl->num_vfs)
+ return -EINVAL;
+
+ /* Calculate curr_qm_qp_num and store */
+ if (val == 0) {
+ qm->debug.curr_qm_qp_num = qm->qp_num;
+ } else {
+ vfq_num = (qm->ctrl_qp_num - qm->qp_num) / ctrl->num_vfs;
+ if (val == ctrl->num_vfs)
+ qm->debug.curr_qm_qp_num = qm->ctrl_qp_num -
+ qm->qp_num - (ctrl->num_vfs - 1) * vfq_num;
+ else
+ qm->debug.curr_qm_qp_num = vfq_num;
+ }
+
+ writel(val, qm->io_base + QM_DFX_MB_CNT_VF);
+ writel(val, qm->io_base + QM_DFX_DB_CNT_VF);
+
+ tmp = val |
+ (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK);
+ writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
+
+ tmp = val |
+ (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK);
+ writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
+
+ return 0;
+}
+
+static u32 clear_enable_read(struct ctrl_debug_file *file)
+{
+ struct hisi_qm *qm = file_to_qm(file);
+
+ return readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) &
+ SOFT_CTRL_CNT_CLR_CE_BIT;
+}
+
+static int clear_enable_write(struct ctrl_debug_file *file, u32 val)
+{
+ struct hisi_qm *qm = file_to_qm(file);
+ u32 tmp;
+
+ if (val != 1 && val != 0)
+ return -EINVAL;
+
+ tmp = (readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) &
+ ~SOFT_CTRL_CNT_CLR_CE_BIT) | val;
+ writel(tmp, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);
+
+ return 0;
+}
+
+static ssize_t ctrl_debug_read(struct file *filp, char __user *buf,
+ size_t count, loff_t *pos)
+{
+ struct ctrl_debug_file *file = filp->private_data;
+ char tbuf[HZIP_BUF_SIZE];
+ u32 val;
+ int ret;
+
+ spin_lock_irq(&file->lock);
+ switch (file->index) {
+ case HZIP_CURRENT_QM:
+ val = current_qm_read(file);
+ break;
+ case HZIP_CLEAR_ENABLE:
+ val = clear_enable_read(file);
+ break;
+ default:
+ spin_unlock_irq(&file->lock);
+ return -EINVAL;
+ }
+ spin_unlock_irq(&file->lock);
+ ret = sprintf(tbuf, "%u\n", val);
+ return simple_read_from_buffer(buf, count, pos, tbuf, ret);
+}
+
+static ssize_t ctrl_debug_write(struct file *filp, const char __user *buf,
+ size_t count, loff_t *pos)
+{
+ struct ctrl_debug_file *file = filp->private_data;
+ char tbuf[HZIP_BUF_SIZE];
+ unsigned long val;
+ int len, ret;
+
+ if (*pos != 0)
+ return 0;
+
+ if (count >= HZIP_BUF_SIZE)
+ return -ENOSPC;
+
+ len = simple_write_to_buffer(tbuf, HZIP_BUF_SIZE - 1, pos, buf, count);
+ if (len < 0)
+ return len;
+
+ tbuf[len] = '\0';
+ if (kstrtoul(tbuf, 0, &val))
+ return -EFAULT;
+
+ spin_lock_irq(&file->lock);
+ switch (file->index) {
+ case HZIP_CURRENT_QM:
+ ret = current_qm_write(file, val);
+ if (ret)
+ goto err_input;
+ break;
+ case HZIP_CLEAR_ENABLE:
+ ret = clear_enable_write(file, val);
+ if (ret)
+ goto err_input;
+ break;
+ default:
+ ret = -EINVAL;
+ goto err_input;
+ }
+ spin_unlock_irq(&file->lock);
+
+ return count;
+
+err_input:
+ spin_unlock_irq(&file->lock);
+ return ret;
+}
+
+static const struct file_operations ctrl_debug_fops = {
+ .owner = THIS_MODULE,
+ .open = simple_open,
+ .read = ctrl_debug_read,
+ .write = ctrl_debug_write,
+};
+
+static int hisi_zip_core_debug_init(struct hisi_zip_ctrl *ctrl)
+{
+ struct hisi_zip *hisi_zip = ctrl->hisi_zip;
+ struct hisi_qm *qm = &hisi_zip->qm;
+ struct device *dev = &qm->pdev->dev;
+ struct debugfs_regset32 *regset;
+ struct dentry *tmp_d, *tmp;
+ char buf[HZIP_BUF_SIZE];
+ int i;
+
+ for (i = 0; i < HZIP_CORE_NUM; i++) {
+ if (i < HZIP_COMP_CORE_NUM)
+ sprintf(buf, "comp_core%d", i);
+ else
+ sprintf(buf, "decomp_core%d", i - HZIP_COMP_CORE_NUM);
+
+ tmp_d = debugfs_create_dir(buf, ctrl->debug_root);
+ if (!tmp_d)
+ return -ENOENT;
+
+ regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
+ if (!regset)
+ return -ENOENT;
+
+ regset->regs = hzip_dfx_regs;
+ regset->nregs = ARRAY_SIZE(hzip_dfx_regs);
+ regset->base = qm->io_base + core_offsets[i];
+
+ tmp = debugfs_create_regset32("regs", 0444, tmp_d, regset);
+ if (!tmp)
+ return -ENOENT;
+ }
+
+ return 0;
+}
+
+static int hisi_zip_ctrl_debug_init(struct hisi_zip_ctrl *ctrl)
+{
+ struct dentry *tmp;
+ int i;
+
+ for (i = HZIP_CURRENT_QM; i < HZIP_DEBUG_FILE_NUM; i++) {
+ spin_lock_init(&ctrl->files[i].lock);
+ ctrl->files[i].ctrl = ctrl;
+ ctrl->files[i].index = i;
+
+ tmp = debugfs_create_file(ctrl_debug_file_name[i], 0600,
+ ctrl->debug_root, ctrl->files + i,
+ &ctrl_debug_fops);
+ if (!tmp)
+ return -ENOENT;
+ }
+
+ return hisi_zip_core_debug_init(ctrl);
+}
+
+static int hisi_zip_debugfs_init(struct hisi_zip *hisi_zip)
+{
+ struct hisi_qm *qm = &hisi_zip->qm;
+ struct device *dev = &qm->pdev->dev;
+ struct dentry *dev_d;
+ int ret;
+
+ dev_d = debugfs_create_dir(dev_name(dev), hzip_debugfs_root);
+ if (!dev_d)
+ return -ENOENT;
+
+ qm->debug.debug_root = dev_d;
+ ret = hisi_qm_debug_init(qm);
+ if (ret)
+ goto failed_to_create;
+
+ if (qm->fun_type == QM_HW_PF) {
+ hisi_zip->ctrl->debug_root = dev_d;
+ ret = hisi_zip_ctrl_debug_init(hisi_zip->ctrl);
+ if (ret)
+ goto failed_to_create;
+ }
+
+ return 0;
+
+failed_to_create:
+ debugfs_remove_recursive(hzip_debugfs_root);
+ return ret;
+}
+
+static void hisi_zip_debug_regs_clear(struct hisi_zip *hisi_zip)
+{
+ struct hisi_qm *qm = &hisi_zip->qm;
+
+ writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF);
+ writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF);
+ writel(0x0, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);
+
+ hisi_qm_debug_regs_clear(qm);
+}
+
+static void hisi_zip_debugfs_exit(struct hisi_zip *hisi_zip)
+{
+ struct hisi_qm *qm = &hisi_zip->qm;
+
+ debugfs_remove_recursive(qm->debug.debug_root);
+
+ if (qm->fun_type == QM_HW_PF)
+ hisi_zip_debug_regs_clear(hisi_zip);
+}
+
+static void hisi_zip_hw_error_init(struct hisi_zip *hisi_zip)
+{
+ hisi_qm_hw_error_init(&hisi_zip->qm, QM_BASE_CE,
+ QM_BASE_NFE | QM_ACC_WB_NOT_READY_TIMEOUT, 0,
+ QM_DB_RANDOM_INVALID);
+ hisi_zip_hw_error_set_state(hisi_zip, true);
+}
+
+static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip)
+{
+ struct hisi_qm *qm = &hisi_zip->qm;
+ struct hisi_zip_ctrl *ctrl;
+
+ ctrl = devm_kzalloc(&qm->pdev->dev, sizeof(*ctrl), GFP_KERNEL);
+ if (!ctrl)
+ return -ENOMEM;
+
+ hisi_zip->ctrl = ctrl;
+ ctrl->hisi_zip = hisi_zip;
+
+ switch (qm->ver) {
+ case QM_HW_V1:
+ qm->ctrl_qp_num = HZIP_QUEUE_NUM_V1;
+ break;
+
+ case QM_HW_V2:
+ qm->ctrl_qp_num = HZIP_QUEUE_NUM_V2;
+ break;
+
+ default:
+ return -EINVAL;
+ }
+
+ hisi_zip_set_user_domain_and_cache(hisi_zip);
+ hisi_zip_hw_error_init(hisi_zip);
+ hisi_zip_debug_regs_clear(hisi_zip);
+
+ return 0;
+}
+
+static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+ struct hisi_zip *hisi_zip;
+ enum qm_hw_ver rev_id;
+ struct hisi_qm *qm;
+ int ret;
+
+ rev_id = hisi_qm_get_hw_version(pdev);
+ if (rev_id == QM_HW_UNKNOWN)
+ return -EINVAL;
+
+ hisi_zip = devm_kzalloc(&pdev->dev, sizeof(*hisi_zip), GFP_KERNEL);
+ if (!hisi_zip)
+ return -ENOMEM;
+ pci_set_drvdata(pdev, hisi_zip);
+
+ qm = &hisi_zip->qm;
+ qm->pdev = pdev;
+ qm->ver = rev_id;
+
+ qm->sqe_size = HZIP_SQE_SIZE;
+ qm->dev_name = hisi_zip_name;
+ qm->fun_type = (pdev->device == PCI_DEVICE_ID_ZIP_PF) ? QM_HW_PF :
+ QM_HW_VF;
+ switch (uacce_mode) {
+ case 0:
+ qm->use_dma_api = true;
+ break;
+ case 1:
+ qm->use_dma_api = false;
+ break;
+ case 2:
+ qm->use_dma_api = true;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ ret = hisi_qm_init(qm);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed to init qm!\n");
+ return ret;
+ }
+
+ if (qm->fun_type == QM_HW_PF) {
+ ret = hisi_zip_pf_probe_init(hisi_zip);
+ if (ret)
+ return ret;
+
+ qm->qp_base = HZIP_PF_DEF_Q_BASE;
+ qm->qp_num = pf_q_num;
+ } else if (qm->fun_type == QM_HW_VF) {
+ /*
+ * have no way to get qm configure in VM in v1 hardware,
+ * so currently force PF to uses HZIP_PF_DEF_Q_NUM, and force
+ * to trigger only one VF in v1 hardware.
+ *
+ * v2 hardware has no such problem.
+ */
+ if (qm->ver == QM_HW_V1) {
+ qm->qp_base = HZIP_PF_DEF_Q_NUM;
+ qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM;
+ } else if (qm->ver == QM_HW_V2)
+ /* v2 starts to support get vft by mailbox */
+ hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num);
+ }
+
+ ret = hisi_qm_start(qm);
+ if (ret)
+ goto err_qm_uninit;
+
+ ret = hisi_zip_debugfs_init(hisi_zip);
+ if (ret)
+ dev_err(&pdev->dev, "Failed to init debugfs (%d)!\n", ret);
+
+ hisi_zip_add_to_list(hisi_zip);
+
+ return 0;
+
+err_qm_uninit:
+ hisi_qm_uninit(qm);
+ return ret;
+}
+
+/* Currently we only support equal assignment */
+static int hisi_zip_vf_q_assign(struct hisi_zip *hisi_zip, int num_vfs)
+{
+ struct hisi_qm *qm = &hisi_zip->qm;
+ u32 qp_num = qm->qp_num;
+ u32 q_base = qp_num;
+ u32 q_num, remain_q_num, i;
+ int ret;
+
+ if (!num_vfs)
+ return -EINVAL;
+
+ remain_q_num = qm->ctrl_qp_num - qp_num;
+ if (remain_q_num < num_vfs)
+ return -EINVAL;
+
+ q_num = remain_q_num / num_vfs;
+ for (i = 1; i <= num_vfs; i++) {
+ if (i == num_vfs)
+ q_num += remain_q_num % num_vfs;
+ ret = hisi_qm_set_vft(qm, i, q_base, q_num);
+ if (ret)
+ return ret;
+ q_base += q_num;
+ }
+
+ return 0;
+}
+
+static int hisi_zip_clear_vft_config(struct hisi_zip *hisi_zip)
+{
+ struct hisi_zip_ctrl *ctrl = hisi_zip->ctrl;
+ struct hisi_qm *qm = &hisi_zip->qm;
+ u32 i, num_vfs = ctrl->num_vfs;
+ int ret;
+
+ for (i = 1; i <= num_vfs; i++) {
+ ret = hisi_qm_set_vft(qm, i, 0, 0);
+ if (ret)
+ return ret;
+ }
+
+ ctrl->num_vfs = 0;
+
+ return 0;
+}
+
+static int hisi_zip_sriov_enable(struct pci_dev *pdev, int max_vfs)
+{
+#ifdef CONFIG_PCI_IOV
+ struct hisi_zip *hisi_zip = pci_get_drvdata(pdev);
+ int pre_existing_vfs, num_vfs, ret;
+
+ pre_existing_vfs = pci_num_vf(pdev);
+
+ if (pre_existing_vfs) {
+ dev_err(&pdev->dev,
+ "Can't enable VF. Please disable pre-enabled VFs!\n");
+ return 0;
+ }
+
+ num_vfs = min_t(int, max_vfs, HZIP_VF_NUM);
+
+ ret = hisi_zip_vf_q_assign(hisi_zip, num_vfs);
+ if (ret) {
+ dev_err(&pdev->dev, "Can't assign queues for VF!\n");
+ return ret;
+ }
+
+ hisi_zip->ctrl->num_vfs = num_vfs;
+
+ ret = pci_enable_sriov(pdev, num_vfs);
+ if (ret) {
+ dev_err(&pdev->dev, "Can't enable VF!\n");
+ hisi_zip_clear_vft_config(hisi_zip);
+ return ret;
+ }
+
+ return num_vfs;
+#else
+ return 0;
+#endif
+}
+
+static int hisi_zip_sriov_disable(struct pci_dev *pdev)
+{
+ struct hisi_zip *hisi_zip = pci_get_drvdata(pdev);
+
+ if (pci_vfs_assigned(pdev)) {
+ dev_err(&pdev->dev,
+ "Can't disable VFs while VFs are assigned!\n");
+ return -EPERM;
+ }
+
+ /* remove in hisi_zip_pci_driver will be called to free VF resources */
+ pci_disable_sriov(pdev);
+
+ return hisi_zip_clear_vft_config(hisi_zip);
+}
+
+static int hisi_zip_sriov_configure(struct pci_dev *pdev, int num_vfs)
+{
+ if (num_vfs == 0)
+ return hisi_zip_sriov_disable(pdev);
+ else
+ return hisi_zip_sriov_enable(pdev, num_vfs);
+}
+
+static void hisi_zip_remove(struct pci_dev *pdev)
+{
+ struct hisi_zip *hisi_zip = pci_get_drvdata(pdev);
+ struct hisi_qm *qm = &hisi_zip->qm;
+
+ if (qm->fun_type == QM_HW_PF && hisi_zip->ctrl->num_vfs != 0)
+ hisi_zip_sriov_disable(pdev);
+
+ hisi_zip_debugfs_exit(hisi_zip);
+ hisi_qm_stop(qm);
+
+ if (qm->fun_type == QM_HW_PF)
+ hisi_zip_hw_error_set_state(hisi_zip, false);
+
+ hisi_qm_uninit(qm);
+ hisi_zip_remove_from_list(hisi_zip);
+}
+
+static void hisi_zip_log_hw_error(struct hisi_zip *hisi_zip, u32 err_sts)
+{
+ const struct hisi_zip_hw_error *err = zip_hw_error;
+ struct device *dev = &hisi_zip->qm.pdev->dev;
+ u32 err_val;
+
+ while (err->msg) {
+ if (err->int_msk & err_sts) {
+ dev_warn(dev, "%s [error status=0x%x] found\n",
+ err->msg, err->int_msk);
+
+ if (HZIP_CORE_INT_STATUS_M_ECC & err->int_msk) {
+ err_val = readl(hisi_zip->qm.io_base +
+ HZIP_CORE_SRAM_ECC_ERR_INFO);
+ dev_warn(dev, "hisi-zip multi ecc sram num=0x%x\n",
+ ((err_val >> SRAM_ECC_ERR_NUM_SHIFT) &
+ 0xFF));
+ dev_warn(dev, "hisi-zip multi ecc sram addr=0x%x\n",
+ (err_val >> SRAM_ECC_ERR_ADDR_SHIFT));
+ }
+ }
+ err++;
+ }
+}
+
+static pci_ers_result_t hisi_zip_hw_error_handle(struct hisi_zip *hisi_zip)
+{
+ u32 err_sts;
+
+ /* read err sts */
+ err_sts = readl(hisi_zip->qm.io_base + HZIP_CORE_INT_STATUS);
+
+ if (err_sts) {
+ hisi_zip_log_hw_error(hisi_zip, err_sts);
+ /* clear error interrupts */
+ writel(err_sts, hisi_zip->qm.io_base + HZIP_CORE_INT_SOURCE);
+
+ return PCI_ERS_RESULT_NEED_RESET;
+ }
+
+ return PCI_ERS_RESULT_RECOVERED;
+}
+
+static pci_ers_result_t hisi_zip_process_hw_error(struct pci_dev *pdev)
+{
+ struct hisi_zip *hisi_zip = pci_get_drvdata(pdev);
+ struct device *dev = &pdev->dev;
+ pci_ers_result_t qm_ret, zip_ret;
+
+ if (!hisi_zip) {
+ dev_err(dev,
+ "Can't recover ZIP-error occurred during device init\n");
+ return PCI_ERS_RESULT_NONE;
+ }
+
+ qm_ret = hisi_qm_hw_error_handle(&hisi_zip->qm);
+
+ zip_ret = hisi_zip_hw_error_handle(hisi_zip);
+
+ return (qm_ret == PCI_ERS_RESULT_NEED_RESET ||
+ zip_ret == PCI_ERS_RESULT_NEED_RESET) ?
+ PCI_ERS_RESULT_NEED_RESET : PCI_ERS_RESULT_RECOVERED;
+}
+
+static pci_ers_result_t hisi_zip_error_detected(struct pci_dev *pdev,
+ pci_channel_state_t state)
+{
+ if (pdev->is_virtfn)
+ return PCI_ERS_RESULT_NONE;
+
+ dev_info(&pdev->dev, "PCI error detected, state(=%d)!!\n", state);
+ if (state == pci_channel_io_perm_failure)
+ return PCI_ERS_RESULT_DISCONNECT;
+
+ return hisi_zip_process_hw_error(pdev);
+}
+
+static const struct pci_error_handlers hisi_zip_err_handler = {
+ .error_detected = hisi_zip_error_detected,
+};
+
+static struct pci_driver hisi_zip_pci_driver = {
+ .name = "hisi_zip",
+ .id_table = hisi_zip_dev_ids,
+ .probe = hisi_zip_probe,
+ .remove = hisi_zip_remove,
+ .sriov_configure = hisi_zip_sriov_configure,
+ .err_handler = &hisi_zip_err_handler,
+};
+
+static void hisi_zip_register_debugfs(void)
+{
+ if (!debugfs_initialized())
+ return;
+
+ hzip_debugfs_root = debugfs_create_dir("hisi_zip", NULL);
+ if (IS_ERR_OR_NULL(hzip_debugfs_root))
+ hzip_debugfs_root = NULL;
+}
+
+static void hisi_zip_unregister_debugfs(void)
+{
+ debugfs_remove_recursive(hzip_debugfs_root);
+}
+
+static int __init hisi_zip_init(void)
+{
+ int ret;
+
+ hisi_zip_register_debugfs();
+
+ ret = pci_register_driver(&hisi_zip_pci_driver);
+ if (ret < 0) {
+ pr_err("Failed to register pci driver.\n");
+ goto err_pci;
+ }
+
+ if (uacce_mode == 0 || uacce_mode == 2) {
+ ret = hisi_zip_register_to_crypto();
+ if (ret < 0) {
+ pr_err("Failed to register driver to crypto.\n");
+ goto err_crypto;
+ }
+ }
+
+ return 0;
+
+err_crypto:
+ pci_unregister_driver(&hisi_zip_pci_driver);
+err_pci:
+ hisi_zip_unregister_debugfs();
+
+ return ret;
+}
+
+static void __exit hisi_zip_exit(void)
+{
+ if (uacce_mode == 0 || uacce_mode == 2)
+ hisi_zip_unregister_from_crypto();
+ pci_unregister_driver(&hisi_zip_pci_driver);
+ hisi_zip_unregister_debugfs();
+}
+
+module_init(hisi_zip_init);
+module_exit(hisi_zip_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
+MODULE_DESCRIPTION("Driver for HiSilicon ZIP accelerator");
diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c
index d27c812c3d8d..fe4cc8babe1c 100644
--- a/drivers/crypto/img-hash.c
+++ b/drivers/crypto/img-hash.c
@@ -958,9 +958,7 @@ static int img_hash_probe(struct platform_device *pdev)
crypto_init_queue(&hdev->queue, IMG_HASH_QUEUE_LENGTH);
/* Register bank */
- hash_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-
- hdev->io_base = devm_ioremap_resource(dev, hash_res);
+ hdev->io_base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(hdev->io_base)) {
err = PTR_ERR(hdev->io_base);
dev_err(dev, "can't ioremap, returned %d\n", err);
@@ -980,7 +978,6 @@ static int img_hash_probe(struct platform_device *pdev)
irq = platform_get_irq(pdev, 0);
if (irq < 0) {
- dev_err(dev, "no IRQ resource info\n");
err = irq;
goto res_err;
}
diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c
index 86c699c14f84..b456b85f46d3 100644
--- a/drivers/crypto/inside-secure/safexcel.c
+++ b/drivers/crypto/inside-secure/safexcel.c
@@ -14,6 +14,7 @@
#include <linux/module.h>
#include <linux/of_platform.h>
#include <linux/of_irq.h>
+#include <linux/pci.h>
#include <linux/platform_device.h>
#include <linux/workqueue.h>
@@ -27,62 +28,205 @@ static u32 max_rings = EIP197_MAX_RINGS;
module_param(max_rings, uint, 0644);
MODULE_PARM_DESC(max_rings, "Maximum number of rings to use.");
-static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
+static void eip197_trc_cache_setupvirt(struct safexcel_crypto_priv *priv)
{
- u32 val, htable_offset;
- int i, cs_rc_max, cs_ht_wc, cs_trc_rec_wc, cs_trc_lg_rec_wc;
-
- if (priv->version == EIP197B) {
- cs_rc_max = EIP197B_CS_RC_MAX;
- cs_ht_wc = EIP197B_CS_HT_WC;
- cs_trc_rec_wc = EIP197B_CS_TRC_REC_WC;
- cs_trc_lg_rec_wc = EIP197B_CS_TRC_LG_REC_WC;
- } else {
- cs_rc_max = EIP197D_CS_RC_MAX;
- cs_ht_wc = EIP197D_CS_HT_WC;
- cs_trc_rec_wc = EIP197D_CS_TRC_REC_WC;
- cs_trc_lg_rec_wc = EIP197D_CS_TRC_LG_REC_WC;
+ int i;
+
+ /*
+ * Map all interfaces/rings to register index 0
+ * so they can share contexts. Without this, the EIP197 will
+ * assume each interface/ring to be in its own memory domain
+ * i.e. have its own subset of UNIQUE memory addresses.
+ * Which would cause records with the SAME memory address to
+ * use DIFFERENT cache buffers, causing both poor cache utilization
+ * AND serious coherence/invalidation issues.
+ */
+ for (i = 0; i < 4; i++)
+ writel(0, priv->base + EIP197_FLUE_IFC_LUT(i));
+
+ /*
+ * Initialize other virtualization regs for cache
+ * These may not be in their reset state ...
+ */
+ for (i = 0; i < priv->config.rings; i++) {
+ writel(0, priv->base + EIP197_FLUE_CACHEBASE_LO(i));
+ writel(0, priv->base + EIP197_FLUE_CACHEBASE_HI(i));
+ writel(EIP197_FLUE_CONFIG_MAGIC,
+ priv->base + EIP197_FLUE_CONFIG(i));
}
+ writel(0, priv->base + EIP197_FLUE_OFFSETS);
+ writel(0, priv->base + EIP197_FLUE_ARC4_OFFSET);
+}
- /* Enable the record cache memory access */
- val = readl(priv->base + EIP197_CS_RAM_CTRL);
- val &= ~EIP197_TRC_ENABLE_MASK;
- val |= EIP197_TRC_ENABLE_0;
- writel(val, priv->base + EIP197_CS_RAM_CTRL);
+static void eip197_trc_cache_banksel(struct safexcel_crypto_priv *priv,
+ u32 addrmid, int *actbank)
+{
+ u32 val;
+ int curbank;
+
+ curbank = addrmid >> 16;
+ if (curbank != *actbank) {
+ val = readl(priv->base + EIP197_CS_RAM_CTRL);
+ val = (val & ~EIP197_CS_BANKSEL_MASK) |
+ (curbank << EIP197_CS_BANKSEL_OFS);
+ writel(val, priv->base + EIP197_CS_RAM_CTRL);
+ *actbank = curbank;
+ }
+}
- /* Clear all ECC errors */
- writel(0, priv->base + EIP197_TRC_ECCCTRL);
+static u32 eip197_trc_cache_probe(struct safexcel_crypto_priv *priv,
+ int maxbanks, u32 probemask)
+{
+ u32 val, addrhi, addrlo, addrmid;
+ int actbank;
/*
- * Make sure the cache memory is accessible by taking record cache into
- * reset.
+ * And probe the actual size of the physically attached cache data RAM
+ * Using a binary subdivision algorithm downto 32 byte cache lines.
*/
- val = readl(priv->base + EIP197_TRC_PARAMS);
- val |= EIP197_TRC_PARAMS_SW_RESET;
- val &= ~EIP197_TRC_PARAMS_DATA_ACCESS;
- writel(val, priv->base + EIP197_TRC_PARAMS);
+ addrhi = 1 << (16 + maxbanks);
+ addrlo = 0;
+ actbank = min(maxbanks - 1, 0);
+ while ((addrhi - addrlo) > 32) {
+ /* write marker to lowest address in top half */
+ addrmid = (addrhi + addrlo) >> 1;
+ eip197_trc_cache_banksel(priv, addrmid, &actbank);
+ writel((addrmid | (addrlo << 16)) & probemask,
+ priv->base + EIP197_CLASSIFICATION_RAMS +
+ (addrmid & 0xffff));
+
+ /* write marker to lowest address in bottom half */
+ eip197_trc_cache_banksel(priv, addrlo, &actbank);
+ writel((addrlo | (addrhi << 16)) & probemask,
+ priv->base + EIP197_CLASSIFICATION_RAMS +
+ (addrlo & 0xffff));
+
+ /* read back marker from top half */
+ eip197_trc_cache_banksel(priv, addrmid, &actbank);
+ val = readl(priv->base + EIP197_CLASSIFICATION_RAMS +
+ (addrmid & 0xffff));
+
+ if (val == ((addrmid | (addrlo << 16)) & probemask)) {
+ /* read back correct, continue with top half */
+ addrlo = addrmid;
+ } else {
+ /* not read back correct, continue with bottom half */
+ addrhi = addrmid;
+ }
+ }
+ return addrhi;
+}
+
+static void eip197_trc_cache_clear(struct safexcel_crypto_priv *priv,
+ int cs_rc_max, int cs_ht_wc)
+{
+ int i;
+ u32 htable_offset, val, offset;
- /* Clear all records */
+ /* Clear all records in administration RAM */
for (i = 0; i < cs_rc_max; i++) {
- u32 val, offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE;
+ offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE;
writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) |
EIP197_CS_RC_PREV(EIP197_RC_NULL),
priv->base + offset);
- val = EIP197_CS_RC_NEXT(i+1) | EIP197_CS_RC_PREV(i-1);
+ val = EIP197_CS_RC_NEXT(i + 1) | EIP197_CS_RC_PREV(i - 1);
if (i == 0)
val |= EIP197_CS_RC_PREV(EIP197_RC_NULL);
else if (i == cs_rc_max - 1)
val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL);
- writel(val, priv->base + offset + sizeof(u32));
+ writel(val, priv->base + offset + 4);
+ /* must also initialize the address key due to ECC! */
+ writel(0, priv->base + offset + 8);
+ writel(0, priv->base + offset + 12);
}
/* Clear the hash table entries */
htable_offset = cs_rc_max * EIP197_CS_RC_SIZE;
for (i = 0; i < cs_ht_wc; i++)
writel(GENMASK(29, 0),
- priv->base + EIP197_CLASSIFICATION_RAMS + htable_offset + i * sizeof(u32));
+ priv->base + EIP197_CLASSIFICATION_RAMS +
+ htable_offset + i * sizeof(u32));
+}
+
+static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
+{
+ u32 val, dsize, asize;
+ int cs_rc_max, cs_ht_wc, cs_trc_rec_wc, cs_trc_lg_rec_wc;
+ int cs_rc_abs_max, cs_ht_sz;
+ int maxbanks;
+
+ /* Setup (dummy) virtualization for cache */
+ eip197_trc_cache_setupvirt(priv);
+
+ /*
+ * Enable the record cache memory access and
+ * probe the bank select width
+ */
+ val = readl(priv->base + EIP197_CS_RAM_CTRL);
+ val &= ~EIP197_TRC_ENABLE_MASK;
+ val |= EIP197_TRC_ENABLE_0 | EIP197_CS_BANKSEL_MASK;
+ writel(val, priv->base + EIP197_CS_RAM_CTRL);
+ val = readl(priv->base + EIP197_CS_RAM_CTRL);
+ maxbanks = ((val&EIP197_CS_BANKSEL_MASK)>>EIP197_CS_BANKSEL_OFS) + 1;
+
+ /* Clear all ECC errors */
+ writel(0, priv->base + EIP197_TRC_ECCCTRL);
+
+ /*
+ * Make sure the cache memory is accessible by taking record cache into
+ * reset. Need data memory access here, not admin access.
+ */
+ val = readl(priv->base + EIP197_TRC_PARAMS);
+ val |= EIP197_TRC_PARAMS_SW_RESET | EIP197_TRC_PARAMS_DATA_ACCESS;
+ writel(val, priv->base + EIP197_TRC_PARAMS);
+
+ /* Probed data RAM size in bytes */
+ dsize = eip197_trc_cache_probe(priv, maxbanks, 0xffffffff);
+
+ /*
+ * Now probe the administration RAM size pretty much the same way
+ * Except that only the lower 30 bits are writable and we don't need
+ * bank selects
+ */
+ val = readl(priv->base + EIP197_TRC_PARAMS);
+ /* admin access now */
+ val &= ~(EIP197_TRC_PARAMS_DATA_ACCESS | EIP197_CS_BANKSEL_MASK);
+ writel(val, priv->base + EIP197_TRC_PARAMS);
+
+ /* Probed admin RAM size in admin words */
+ asize = eip197_trc_cache_probe(priv, 0, 0xbfffffff) >> 4;
+
+ /* Clear any ECC errors detected while probing! */
+ writel(0, priv->base + EIP197_TRC_ECCCTRL);
+
+ /*
+ * Determine optimal configuration from RAM sizes
+ * Note that we assume that the physical RAM configuration is sane
+ * Therefore, we don't do any parameter error checking here ...
+ */
+
+ /* For now, just use a single record format covering everything */
+ cs_trc_rec_wc = EIP197_CS_TRC_REC_WC;
+ cs_trc_lg_rec_wc = EIP197_CS_TRC_REC_WC;
+
+ /*
+ * Step #1: How many records will physically fit?
+ * Hard upper limit is 1023!
+ */
+ cs_rc_abs_max = min_t(uint, ((dsize >> 2) / cs_trc_lg_rec_wc), 1023);
+ /* Step #2: Need at least 2 words in the admin RAM per record */
+ cs_rc_max = min_t(uint, cs_rc_abs_max, (asize >> 1));
+ /* Step #3: Determine log2 of hash table size */
+ cs_ht_sz = __fls(asize - cs_rc_max) - 2;
+ /* Step #4: determine current size of hash table in dwords */
+ cs_ht_wc = 16<<cs_ht_sz; /* dwords, not admin words */
+ /* Step #5: add back excess words and see if we can fit more records */
+ cs_rc_max = min_t(uint, cs_rc_abs_max, asize - (cs_ht_wc >> 4));
+
+ /* Clear the cache RAMs */
+ eip197_trc_cache_clear(priv, cs_rc_max, cs_ht_wc);
/* Disable the record cache memory access */
val = readl(priv->base + EIP197_CS_RAM_CTRL);
@@ -102,119 +246,231 @@ static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
/* Configure the record cache #2 */
val = EIP197_TRC_PARAMS_RC_SZ_LARGE(cs_trc_lg_rec_wc) |
EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) |
- EIP197_TRC_PARAMS_HTABLE_SZ(2);
+ EIP197_TRC_PARAMS_HTABLE_SZ(cs_ht_sz);
writel(val, priv->base + EIP197_TRC_PARAMS);
+
+ dev_info(priv->dev, "TRC init: %dd,%da (%dr,%dh)\n",
+ dsize, asize, cs_rc_max, cs_ht_wc + cs_ht_wc);
}
-static void eip197_write_firmware(struct safexcel_crypto_priv *priv,
- const struct firmware *fw, int pe, u32 ctrl,
- u32 prog_en)
+static void eip197_init_firmware(struct safexcel_crypto_priv *priv)
{
- const u32 *data = (const u32 *)fw->data;
+ int pe, i;
u32 val;
- int i;
- /* Reset the engine to make its program memory accessible */
- writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
- EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
- EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
- EIP197_PE(priv) + ctrl);
+ for (pe = 0; pe < priv->config.pes; pe++) {
+ /* Configure the token FIFO's */
+ writel(3, EIP197_PE(priv) + EIP197_PE_ICE_PUTF_CTRL(pe));
+ writel(0, EIP197_PE(priv) + EIP197_PE_ICE_PPTF_CTRL(pe));
- /* Enable access to the program memory */
- writel(prog_en, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
+ /* Clear the ICE scratchpad memory */
+ val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
+ val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |
+ EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |
+ EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |
+ EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
+ writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
+
+ /* clear the scratchpad RAM using 32 bit writes only */
+ for (i = 0; i < EIP197_NUM_OF_SCRATCH_BLOCKS; i++)
+ writel(0, EIP197_PE(priv) +
+ EIP197_PE_ICE_SCRATCH_RAM(pe) + (i << 2));
+
+ /* Reset the IFPP engine to make its program mem accessible */
+ writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
+ EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
+ EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
+ EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));
+
+ /* Reset the IPUE engine to make its program mem accessible */
+ writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
+ EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
+ EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
+ EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));
+
+ /* Enable access to all IFPP program memories */
+ writel(EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN,
+ EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
+ }
+
+}
+
+static int eip197_write_firmware(struct safexcel_crypto_priv *priv,
+ const struct firmware *fw)
+{
+ const u32 *data = (const u32 *)fw->data;
+ int i;
/* Write the firmware */
for (i = 0; i < fw->size / sizeof(u32); i++)
writel(be32_to_cpu(data[i]),
priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32));
- /* Disable access to the program memory */
- writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
+ /* Exclude final 2 NOPs from size */
+ return i - EIP197_FW_TERMINAL_NOPS;
+}
+
+/*
+ * If FW is actual production firmware, then poll for its initialization
+ * to complete and check if it is good for the HW, otherwise just return OK.
+ */
+static bool poll_fw_ready(struct safexcel_crypto_priv *priv, int fpp)
+{
+ int pe, pollcnt;
+ u32 base, pollofs;
+
+ if (fpp)
+ pollofs = EIP197_FW_FPP_READY;
+ else
+ pollofs = EIP197_FW_PUE_READY;
+
+ for (pe = 0; pe < priv->config.pes; pe++) {
+ base = EIP197_PE_ICE_SCRATCH_RAM(pe);
+ pollcnt = EIP197_FW_START_POLLCNT;
+ while (pollcnt &&
+ (readl_relaxed(EIP197_PE(priv) + base +
+ pollofs) != 1)) {
+ pollcnt--;
+ }
+ if (!pollcnt) {
+ dev_err(priv->dev, "FW(%d) for PE %d failed to start\n",
+ fpp, pe);
+ return false;
+ }
+ }
+ return true;
+}
+
+static bool eip197_start_firmware(struct safexcel_crypto_priv *priv,
+ int ipuesz, int ifppsz, int minifw)
+{
+ int pe;
+ u32 val;
+
+ for (pe = 0; pe < priv->config.pes; pe++) {
+ /* Disable access to all program memory */
+ writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
+
+ /* Start IFPP microengines */
+ if (minifw)
+ val = 0;
+ else
+ val = EIP197_PE_ICE_UENG_START_OFFSET((ifppsz - 1) &
+ EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |
+ EIP197_PE_ICE_UENG_DEBUG_RESET;
+ writel(val, EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));
+
+ /* Start IPUE microengines */
+ if (minifw)
+ val = 0;
+ else
+ val = EIP197_PE_ICE_UENG_START_OFFSET((ipuesz - 1) &
+ EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |
+ EIP197_PE_ICE_UENG_DEBUG_RESET;
+ writel(val, EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));
+ }
+
+ /* For miniFW startup, there is no initialization, so always succeed */
+ if (minifw)
+ return true;
- /* Release engine from reset */
- val = readl(EIP197_PE(priv) + ctrl);
- val &= ~EIP197_PE_ICE_x_CTRL_SW_RESET;
- writel(val, EIP197_PE(priv) + ctrl);
+ /* Wait until all the firmwares have properly started up */
+ if (!poll_fw_ready(priv, 1))
+ return false;
+ if (!poll_fw_ready(priv, 0))
+ return false;
+
+ return true;
}
static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)
{
const char *fw_name[] = {"ifpp.bin", "ipue.bin"};
const struct firmware *fw[FW_NB];
- char fw_path[31], *dir = NULL;
+ char fw_path[37], *dir = NULL;
int i, j, ret = 0, pe;
- u32 val;
+ int ipuesz, ifppsz, minifw = 0;
- switch (priv->version) {
- case EIP197B:
- dir = "eip197b";
- break;
- case EIP197D:
+ if (priv->version == EIP197D_MRVL)
dir = "eip197d";
- break;
- default:
- /* No firmware is required */
- return 0;
- }
+ else if (priv->version == EIP197B_MRVL ||
+ priv->version == EIP197_DEVBRD)
+ dir = "eip197b";
+ else
+ return -ENODEV;
+retry_fw:
for (i = 0; i < FW_NB; i++) {
- snprintf(fw_path, 31, "inside-secure/%s/%s", dir, fw_name[i]);
- ret = request_firmware(&fw[i], fw_path, priv->dev);
+ snprintf(fw_path, 37, "inside-secure/%s/%s", dir, fw_name[i]);
+ ret = firmware_request_nowarn(&fw[i], fw_path, priv->dev);
if (ret) {
- if (priv->version != EIP197B)
+ if (minifw || priv->version != EIP197B_MRVL)
goto release_fw;
/* Fallback to the old firmware location for the
* EIP197b.
*/
- ret = request_firmware(&fw[i], fw_name[i], priv->dev);
- if (ret) {
- dev_err(priv->dev,
- "Failed to request firmware %s (%d)\n",
- fw_name[i], ret);
+ ret = firmware_request_nowarn(&fw[i], fw_name[i],
+ priv->dev);
+ if (ret)
goto release_fw;
- }
}
}
- for (pe = 0; pe < priv->config.pes; pe++) {
- /* Clear the scratchpad memory */
- val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
- val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |
- EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |
- EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |
- EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
- writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
+ eip197_init_firmware(priv);
+
+ ifppsz = eip197_write_firmware(priv, fw[FW_IFPP]);
- memset_io(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_RAM(pe), 0,
- EIP197_NUM_OF_SCRATCH_BLOCKS * sizeof(u32));
+ /* Enable access to IPUE program memories */
+ for (pe = 0; pe < priv->config.pes; pe++)
+ writel(EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN,
+ EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
- eip197_write_firmware(priv, fw[FW_IFPP], pe,
- EIP197_PE_ICE_FPP_CTRL(pe),
- EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN);
+ ipuesz = eip197_write_firmware(priv, fw[FW_IPUE]);
- eip197_write_firmware(priv, fw[FW_IPUE], pe,
- EIP197_PE_ICE_PUE_CTRL(pe),
- EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN);
+ if (eip197_start_firmware(priv, ipuesz, ifppsz, minifw)) {
+ dev_dbg(priv->dev, "Firmware loaded successfully\n");
+ return 0;
}
+ ret = -ENODEV;
+
release_fw:
for (j = 0; j < i; j++)
release_firmware(fw[j]);
+ if (!minifw) {
+ /* Retry with minifw path */
+ dev_dbg(priv->dev, "Firmware set not (fully) present or init failed, falling back to BCLA mode\n");
+ dir = "eip197_minifw";
+ minifw = 1;
+ goto retry_fw;
+ }
+
+ dev_dbg(priv->dev, "Firmware load failed.\n");
+
return ret;
}
static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)
{
- u32 hdw, cd_size_rnd, val;
- int i;
-
- hdw = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
- hdw &= GENMASK(27, 25);
- hdw >>= 25;
-
- cd_size_rnd = (priv->config.cd_size + (BIT(hdw) - 1)) >> hdw;
+ u32 cd_size_rnd, val;
+ int i, cd_fetch_cnt;
+
+ cd_size_rnd = (priv->config.cd_size +
+ (BIT(priv->hwconfig.hwdataw) - 1)) >>
+ priv->hwconfig.hwdataw;
+ /* determine number of CD's we can fetch into the CD FIFO as 1 block */
+ if (priv->flags & SAFEXCEL_HW_EIP197) {
+ /* EIP197: try to fetch enough in 1 go to keep all pipes busy */
+ cd_fetch_cnt = (1 << priv->hwconfig.hwcfsize) / cd_size_rnd;
+ cd_fetch_cnt = min_t(uint, cd_fetch_cnt,
+ (priv->config.pes * EIP197_FETCH_DEPTH));
+ } else {
+ /* for the EIP97, just fetch all that fits minus 1 */
+ cd_fetch_cnt = ((1 << priv->hwconfig.hwcfsize) /
+ cd_size_rnd) - 1;
+ }
for (i = 0; i < priv->config.rings; i++) {
/* ring base address */
@@ -226,8 +482,9 @@ static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)
writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 16) |
priv->config.cd_size,
EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
- writel(((EIP197_FETCH_COUNT * (cd_size_rnd << hdw)) << 16) |
- (EIP197_FETCH_COUNT * priv->config.cd_offset),
+ writel(((cd_fetch_cnt *
+ (cd_size_rnd << priv->hwconfig.hwdataw)) << 16) |
+ (cd_fetch_cnt * priv->config.cd_offset),
EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
/* Configure DMA tx control */
@@ -245,14 +502,23 @@ static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)
static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
{
- u32 hdw, rd_size_rnd, val;
- int i;
-
- hdw = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
- hdw &= GENMASK(27, 25);
- hdw >>= 25;
-
- rd_size_rnd = (priv->config.rd_size + (BIT(hdw) - 1)) >> hdw;
+ u32 rd_size_rnd, val;
+ int i, rd_fetch_cnt;
+
+ /* determine number of RD's we can fetch into the FIFO as one block */
+ rd_size_rnd = (EIP197_RD64_FETCH_SIZE +
+ (BIT(priv->hwconfig.hwdataw) - 1)) >>
+ priv->hwconfig.hwdataw;
+ if (priv->flags & SAFEXCEL_HW_EIP197) {
+ /* EIP197: try to fetch enough in 1 go to keep all pipes busy */
+ rd_fetch_cnt = (1 << priv->hwconfig.hwrfsize) / rd_size_rnd;
+ rd_fetch_cnt = min_t(uint, rd_fetch_cnt,
+ (priv->config.pes * EIP197_FETCH_DEPTH));
+ } else {
+ /* for the EIP97, just fetch all that fits minus 1 */
+ rd_fetch_cnt = ((1 << priv->hwconfig.hwrfsize) /
+ rd_size_rnd) - 1;
+ }
for (i = 0; i < priv->config.rings; i++) {
/* ring base address */
@@ -265,8 +531,9 @@ static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
priv->config.rd_size,
EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
- writel(((EIP197_FETCH_COUNT * (rd_size_rnd << hdw)) << 16) |
- (EIP197_FETCH_COUNT * priv->config.rd_offset),
+ writel(((rd_fetch_cnt *
+ (rd_size_rnd << priv->hwconfig.hwdataw)) << 16) |
+ (rd_fetch_cnt * priv->config.rd_offset),
EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
/* Configure DMA tx control */
@@ -291,23 +558,21 @@ static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
{
- u32 version, val;
+ u32 val;
int i, ret, pe;
- /* Determine endianess and configure byte swap */
- version = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_VERSION);
- val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
+ dev_dbg(priv->dev, "HW init: using %d pipe(s) and %d ring(s)\n",
+ priv->config.pes, priv->config.rings);
- if ((version & 0xffff) == EIP197_HIA_VERSION_BE)
- val |= EIP197_MST_CTRL_BYTE_SWAP;
- else if (((version >> 16) & 0xffff) == EIP197_HIA_VERSION_LE)
- val |= (EIP197_MST_CTRL_NO_BYTE_SWAP >> 24);
-
- /* For EIP197 set maximum number of TX commands to 2^5 = 32 */
- if (priv->version == EIP197B || priv->version == EIP197D)
+ /*
+ * For EIP197's only set maximum number of TX commands to 2^5 = 32
+ * Skip for the EIP97 as it does not have this field.
+ */
+ if (priv->flags & SAFEXCEL_HW_EIP197) {
+ val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
val |= EIP197_MST_CTRL_TX_MAX_CMD(5);
-
- writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
+ writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
+ }
/* Configure wr/rd cache values */
writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) |
@@ -330,11 +595,10 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
writel(EIP197_DxE_THR_CTRL_RESET_PE,
EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
- if (priv->version == EIP197B || priv->version == EIP197D) {
- /* Reset HIA input interface arbiter */
+ if (priv->flags & SAFEXCEL_HW_EIP197)
+ /* Reset HIA input interface arbiter (EIP197 only) */
writel(EIP197_HIA_RA_PE_CTRL_RESET,
EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
- }
/* DMA transfer size to use */
val = EIP197_HIA_DFE_CFG_DIS_DEBUG;
@@ -357,12 +621,11 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
EIP197_PE_IN_xBUF_THRES_MAX(7),
EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES(pe));
- if (priv->version == EIP197B || priv->version == EIP197D) {
+ if (priv->flags & SAFEXCEL_HW_EIP197)
/* enable HIA input interface arbiter and rings */
writel(EIP197_HIA_RA_PE_CTRL_EN |
GENMASK(priv->config.rings - 1, 0),
EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
- }
/* Data Store Engine configuration */
@@ -381,10 +644,10 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8);
val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);
val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE;
- /* FIXME: instability issues can occur for EIP97 but disabling it impact
- * performances.
+ /* FIXME: instability issues can occur for EIP97 but disabling
+ * it impacts performance.
*/
- if (priv->version == EIP197B || priv->version == EIP197D)
+ if (priv->flags & SAFEXCEL_HW_EIP197)
val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;
writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG(pe));
@@ -398,17 +661,17 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
/* Processing Engine configuration */
- /* H/W capabilities selection */
- val = EIP197_FUNCTION_RSVD;
- val |= EIP197_PROTOCOL_ENCRYPT_ONLY | EIP197_PROTOCOL_HASH_ONLY;
- val |= EIP197_PROTOCOL_ENCRYPT_HASH | EIP197_PROTOCOL_HASH_DECRYPT;
- val |= EIP197_ALG_DES_ECB | EIP197_ALG_DES_CBC;
- val |= EIP197_ALG_3DES_ECB | EIP197_ALG_3DES_CBC;
- val |= EIP197_ALG_AES_ECB | EIP197_ALG_AES_CBC;
- val |= EIP197_ALG_MD5 | EIP197_ALG_HMAC_MD5;
- val |= EIP197_ALG_SHA1 | EIP197_ALG_HMAC_SHA1;
- val |= EIP197_ALG_SHA2 | EIP197_ALG_HMAC_SHA2;
- writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN(pe));
+ /* Token & context configuration */
+ val = EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES |
+ EIP197_PE_EIP96_TOKEN_CTRL_NO_TOKEN_WAIT |
+ EIP197_PE_EIP96_TOKEN_CTRL_ENABLE_TIMEOUT;
+ writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL(pe));
+
+ /* H/W capabilities selection: just enable everything */
+ writel(EIP197_FUNCTION_ALL,
+ EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN(pe));
+ writel(EIP197_FUNCTION_ALL,
+ EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION2_EN(pe));
}
/* Command Descriptor Rings prepare */
@@ -473,8 +736,9 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
/* Clear any HIA interrupt */
writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
- if (priv->version == EIP197B || priv->version == EIP197D) {
+ if (priv->flags & SAFEXCEL_HW_EIP197) {
eip197_trc_cache_init(priv);
+ priv->flags |= EIP197_TRC_CACHE;
ret = eip197_load_firmwares(priv);
if (ret)
@@ -583,16 +847,32 @@ finalize:
inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,
struct safexcel_result_desc *rdesc)
{
- if (likely(!rdesc->result_data.error_code))
+ if (likely((!rdesc->descriptor_overflow) &&
+ (!rdesc->buffer_overflow) &&
+ (!rdesc->result_data.error_code)))
return 0;
- if (rdesc->result_data.error_code & 0x407f) {
- /* Fatal error (bits 0-7, 14) */
+ if (rdesc->descriptor_overflow)
+ dev_err(priv->dev, "Descriptor overflow detected");
+
+ if (rdesc->buffer_overflow)
+ dev_err(priv->dev, "Buffer overflow detected");
+
+ if (rdesc->result_data.error_code & 0x4066) {
+ /* Fatal error (bits 1,2,5,6 & 14) */
dev_err(priv->dev,
- "cipher: result: result descriptor error (%d)\n",
+ "result descriptor error (%x)",
rdesc->result_data.error_code);
return -EIO;
- } else if (rdesc->result_data.error_code == BIT(9)) {
+ } else if (rdesc->result_data.error_code &
+ (BIT(7) | BIT(4) | BIT(3) | BIT(0))) {
+ /*
+ * Give priority over authentication fails:
+ * Blocksize, length & overflow errors,
+ * something wrong with the input!
+ */
+ return -EINVAL;
+ } else if (rdesc->result_data.error_code & BIT(9)) {
/* Authentication failed */
return -EBADMSG;
}
@@ -705,7 +985,8 @@ handle_results:
ndesc = ctx->handle_result(priv, ring, req,
&should_complete, &ret);
if (ndesc < 0) {
- dev_err(priv->dev, "failed to handle result (%d)", ndesc);
+ dev_err(priv->dev, "failed to handle result (%d)\n",
+ ndesc);
goto acknowledge;
}
@@ -720,11 +1001,10 @@ handle_results:
}
acknowledge:
- if (i) {
+ if (i)
writel(EIP197_xDR_PROC_xD_PKT(i) |
EIP197_xDR_PROC_xD_COUNT(tot_descs * priv->config.rd_offset),
EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
- }
/* If the number of requests overflowed the counter, try to proceed more
* requests.
@@ -778,7 +1058,7 @@ static irqreturn_t safexcel_irq_ring(int irq, void *data)
* reinitialized. This should not happen under
* normal circumstances.
*/
- dev_err(priv->dev, "RDR: fatal error.");
+ dev_err(priv->dev, "RDR: fatal error.\n");
} else if (likely(stat & EIP197_xDR_THRESH)) {
rc = IRQ_WAKE_THREAD;
}
@@ -808,23 +1088,45 @@ static irqreturn_t safexcel_irq_ring_thread(int irq, void *data)
return IRQ_HANDLED;
}
-static int safexcel_request_ring_irq(struct platform_device *pdev, const char *name,
+static int safexcel_request_ring_irq(void *pdev, int irqid,
+ int is_pci_dev,
irq_handler_t handler,
irq_handler_t threaded_handler,
struct safexcel_ring_irq_data *ring_irq_priv)
{
- int ret, irq = platform_get_irq_byname(pdev, name);
+ int ret, irq;
+ struct device *dev;
+
+ if (IS_ENABLED(CONFIG_PCI) && is_pci_dev) {
+ struct pci_dev *pci_pdev = pdev;
+
+ dev = &pci_pdev->dev;
+ irq = pci_irq_vector(pci_pdev, irqid);
+ if (irq < 0) {
+ dev_err(dev, "unable to get device MSI IRQ %d (err %d)\n",
+ irqid, irq);
+ return irq;
+ }
+ } else if (IS_ENABLED(CONFIG_OF)) {
+ struct platform_device *plf_pdev = pdev;
+ char irq_name[6] = {0}; /* "ringX\0" */
+
+ snprintf(irq_name, 6, "ring%d", irqid);
+ dev = &plf_pdev->dev;
+ irq = platform_get_irq_byname(plf_pdev, irq_name);
- if (irq < 0) {
- dev_err(&pdev->dev, "unable to get IRQ '%s'\n", name);
- return irq;
+ if (irq < 0) {
+ dev_err(dev, "unable to get IRQ '%s' (err %d)\n",
+ irq_name, irq);
+ return irq;
+ }
}
- ret = devm_request_threaded_irq(&pdev->dev, irq, handler,
+ ret = devm_request_threaded_irq(dev, irq, handler,
threaded_handler, IRQF_ONESHOT,
- dev_name(&pdev->dev), ring_irq_priv);
+ dev_name(dev), ring_irq_priv);
if (ret) {
- dev_err(&pdev->dev, "unable to request IRQ %d\n", irq);
+ dev_err(dev, "unable to request IRQ %d\n", irq);
return ret;
}
@@ -838,6 +1140,9 @@ static struct safexcel_alg_template *safexcel_algs[] = {
&safexcel_alg_cbc_des3_ede,
&safexcel_alg_ecb_aes,
&safexcel_alg_cbc_aes,
+ &safexcel_alg_cfb_aes,
+ &safexcel_alg_ofb_aes,
+ &safexcel_alg_ctr_aes,
&safexcel_alg_md5,
&safexcel_alg_sha1,
&safexcel_alg_sha224,
@@ -855,6 +1160,15 @@ static struct safexcel_alg_template *safexcel_algs[] = {
&safexcel_alg_authenc_hmac_sha256_cbc_aes,
&safexcel_alg_authenc_hmac_sha384_cbc_aes,
&safexcel_alg_authenc_hmac_sha512_cbc_aes,
+ &safexcel_alg_authenc_hmac_sha1_cbc_des3_ede,
+ &safexcel_alg_authenc_hmac_sha1_ctr_aes,
+ &safexcel_alg_authenc_hmac_sha224_ctr_aes,
+ &safexcel_alg_authenc_hmac_sha256_ctr_aes,
+ &safexcel_alg_authenc_hmac_sha384_ctr_aes,
+ &safexcel_alg_authenc_hmac_sha512_ctr_aes,
+ &safexcel_alg_xts_aes,
+ &safexcel_alg_gcm,
+ &safexcel_alg_ccm,
};
static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
@@ -864,7 +1178,10 @@ static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
safexcel_algs[i]->priv = priv;
- if (!(safexcel_algs[i]->engines & priv->version))
+ /* Do we have all required base algorithms available? */
+ if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=
+ safexcel_algs[i]->algo_mask)
+ /* No, so don't register this ciphersuite */
continue;
if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
@@ -882,7 +1199,10 @@ static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
fail:
for (j = 0; j < i; j++) {
- if (!(safexcel_algs[j]->engines & priv->version))
+ /* Do we have all required base algorithms available? */
+ if ((safexcel_algs[j]->algo_mask & priv->hwconfig.algo_flags) !=
+ safexcel_algs[j]->algo_mask)
+ /* No, so don't unregister this ciphersuite */
continue;
if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
@@ -901,7 +1221,10 @@ static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv)
int i;
for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
- if (!(safexcel_algs[i]->engines & priv->version))
+ /* Do we have all required base algorithms available? */
+ if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=
+ safexcel_algs[i]->algo_mask)
+ /* No, so don't unregister this ciphersuite */
continue;
if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
@@ -920,22 +1243,20 @@ static void safexcel_configure(struct safexcel_crypto_priv *priv)
val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
/* Read number of PEs from the engine */
- switch (priv->version) {
- case EIP197B:
- case EIP197D:
+ if (priv->flags & SAFEXCEL_HW_EIP197)
+ /* Wider field width for all EIP197 type engines */
mask = EIP197_N_PES_MASK;
- break;
- default:
+ else
+ /* Narrow field width for EIP97 type engine */
mask = EIP97_N_PES_MASK;
- }
+
priv->config.pes = (val >> EIP197_N_PES_OFFSET) & mask;
+ priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings);
+
val = (val & GENMASK(27, 25)) >> 25;
mask = BIT(val) - 1;
- val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
- priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings);
-
priv->config.cd_size = (sizeof(struct safexcel_command_desc) / sizeof(u32));
priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask;
@@ -947,9 +1268,7 @@ static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv)
{
struct safexcel_register_offsets *offsets = &priv->offsets;
- switch (priv->version) {
- case EIP197B:
- case EIP197D:
+ if (priv->flags & SAFEXCEL_HW_EIP197) {
offsets->hia_aic = EIP197_HIA_AIC_BASE;
offsets->hia_aic_g = EIP197_HIA_AIC_G_BASE;
offsets->hia_aic_r = EIP197_HIA_AIC_R_BASE;
@@ -960,8 +1279,8 @@ static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv)
offsets->hia_dse_thr = EIP197_HIA_DSE_THR_BASE;
offsets->hia_gen_cfg = EIP197_HIA_GEN_CFG_BASE;
offsets->pe = EIP197_PE_BASE;
- break;
- case EIP97IES:
+ offsets->global = EIP197_GLOBAL_BASE;
+ } else {
offsets->hia_aic = EIP97_HIA_AIC_BASE;
offsets->hia_aic_g = EIP97_HIA_AIC_G_BASE;
offsets->hia_aic_r = EIP97_HIA_AIC_R_BASE;
@@ -972,135 +1291,213 @@ static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv)
offsets->hia_dse_thr = EIP97_HIA_DSE_THR_BASE;
offsets->hia_gen_cfg = EIP97_HIA_GEN_CFG_BASE;
offsets->pe = EIP97_PE_BASE;
- break;
+ offsets->global = EIP97_GLOBAL_BASE;
}
}
-static int safexcel_probe(struct platform_device *pdev)
+/*
+ * Generic part of probe routine, shared by platform and PCI driver
+ *
+ * Assumes IO resources have been mapped, private data mem has been allocated,
+ * clocks have been enabled, device pointer has been assigned etc.
+ *
+ */
+static int safexcel_probe_generic(void *pdev,
+ struct safexcel_crypto_priv *priv,
+ int is_pci_dev)
{
- struct device *dev = &pdev->dev;
- struct resource *res;
- struct safexcel_crypto_priv *priv;
- int i, ret;
+ struct device *dev = priv->dev;
+ u32 peid, version, mask, val, hiaopt;
+ int i, ret, hwctg;
- priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
- if (!priv)
+ priv->context_pool = dmam_pool_create("safexcel-context", dev,
+ sizeof(struct safexcel_context_record),
+ 1, 0);
+ if (!priv->context_pool)
return -ENOMEM;
- priv->dev = dev;
- priv->version = (enum safexcel_eip_version)of_device_get_match_data(dev);
-
- if (priv->version == EIP197B || priv->version == EIP197D)
- priv->flags |= EIP197_TRC_CACHE;
+ /*
+ * First try the EIP97 HIA version regs
+ * For the EIP197, this is guaranteed to NOT return any of the test
+ * values
+ */
+ version = readl(priv->base + EIP97_HIA_AIC_BASE + EIP197_HIA_VERSION);
+
+ mask = 0; /* do not swap */
+ if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {
+ priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);
+ } else if (EIP197_REG_HI16(version) == EIP197_HIA_VERSION_BE) {
+ /* read back byte-swapped, so complement byte swap bits */
+ mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;
+ priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);
+ } else {
+ /* So it wasn't an EIP97 ... maybe it's an EIP197? */
+ version = readl(priv->base + EIP197_HIA_AIC_BASE +
+ EIP197_HIA_VERSION);
+ if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {
+ priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);
+ priv->flags |= SAFEXCEL_HW_EIP197;
+ } else if (EIP197_REG_HI16(version) ==
+ EIP197_HIA_VERSION_BE) {
+ /* read back byte-swapped, so complement swap bits */
+ mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;
+ priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);
+ priv->flags |= SAFEXCEL_HW_EIP197;
+ } else {
+ return -ENODEV;
+ }
+ }
+ /* Now initialize the reg offsets based on the probing info so far */
safexcel_init_register_offsets(priv);
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- priv->base = devm_ioremap_resource(dev, res);
- if (IS_ERR(priv->base)) {
- dev_err(dev, "failed to get resource\n");
- return PTR_ERR(priv->base);
+ /*
+ * If the version was read byte-swapped, we need to flip the device
+ * swapping Keep in mind here, though, that what we write will also be
+ * byte-swapped ...
+ */
+ if (mask) {
+ val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
+ val = val ^ (mask >> 24); /* toggle byte swap bits */
+ writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
}
- priv->clk = devm_clk_get(&pdev->dev, NULL);
- ret = PTR_ERR_OR_ZERO(priv->clk);
- /* The clock isn't mandatory */
- if (ret != -ENOENT) {
- if (ret)
- return ret;
-
- ret = clk_prepare_enable(priv->clk);
- if (ret) {
- dev_err(dev, "unable to enable clk (%d)\n", ret);
- return ret;
- }
+ /*
+ * We're not done probing yet! We may fall through to here if no HIA
+ * was found at all. So, with the endianness presumably correct now and
+ * the offsets setup, *really* probe for the EIP97/EIP197.
+ */
+ version = readl(EIP197_GLOBAL(priv) + EIP197_VERSION);
+ if (((priv->flags & SAFEXCEL_HW_EIP197) &&
+ (EIP197_REG_LO16(version) != EIP197_VERSION_LE)) ||
+ ((!(priv->flags & SAFEXCEL_HW_EIP197) &&
+ (EIP197_REG_LO16(version) != EIP97_VERSION_LE)))) {
+ /*
+ * We did not find the device that matched our initial probing
+ * (or our initial probing failed) Report appropriate error.
+ */
+ return -ENODEV;
}
- priv->reg_clk = devm_clk_get(&pdev->dev, "reg");
- ret = PTR_ERR_OR_ZERO(priv->reg_clk);
- /* The clock isn't mandatory */
- if (ret != -ENOENT) {
- if (ret)
- goto err_core_clk;
+ priv->hwconfig.hwver = EIP197_VERSION_MASK(version);
+ hwctg = version >> 28;
+ peid = version & 255;
- ret = clk_prepare_enable(priv->reg_clk);
- if (ret) {
- dev_err(dev, "unable to enable reg clk (%d)\n", ret);
- goto err_core_clk;
- }
+ /* Detect EIP96 packet engine and version */
+ version = readl(EIP197_PE(priv) + EIP197_PE_EIP96_VERSION(0));
+ if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) {
+ dev_err(dev, "EIP%d: EIP96 not detected.\n", peid);
+ return -ENODEV;
+ }
+ priv->hwconfig.pever = EIP197_VERSION_MASK(version);
+
+ hiaopt = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_OPTIONS);
+
+ if (priv->flags & SAFEXCEL_HW_EIP197) {
+ /* EIP197 */
+ priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) &
+ EIP197_HWDATAW_MASK;
+ priv->hwconfig.hwcfsize = ((hiaopt >> EIP197_CFSIZE_OFFSET) &
+ EIP197_CFSIZE_MASK) +
+ EIP197_CFSIZE_ADJUST;
+ priv->hwconfig.hwrfsize = ((hiaopt >> EIP197_RFSIZE_OFFSET) &
+ EIP197_RFSIZE_MASK) +
+ EIP197_RFSIZE_ADJUST;
+ } else {
+ /* EIP97 */
+ priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) &
+ EIP97_HWDATAW_MASK;
+ priv->hwconfig.hwcfsize = (hiaopt >> EIP97_CFSIZE_OFFSET) &
+ EIP97_CFSIZE_MASK;
+ priv->hwconfig.hwrfsize = (hiaopt >> EIP97_RFSIZE_OFFSET) &
+ EIP97_RFSIZE_MASK;
}
- ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
- if (ret)
- goto err_reg_clk;
+ /* Get supported algorithms from EIP96 transform engine */
+ priv->hwconfig.algo_flags = readl(EIP197_PE(priv) +
+ EIP197_PE_EIP96_OPTIONS(0));
- priv->context_pool = dmam_pool_create("safexcel-context", dev,
- sizeof(struct safexcel_context_record),
- 1, 0);
- if (!priv->context_pool) {
- ret = -ENOMEM;
- goto err_reg_clk;
- }
+ /* Print single info line describing what we just detected */
+ dev_info(priv->dev, "EIP%d:%x(%d)-HIA:%x(%d,%d,%d),PE:%x,alg:%08x\n",
+ peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hiaver,
+ priv->hwconfig.hwdataw, priv->hwconfig.hwcfsize,
+ priv->hwconfig.hwrfsize, priv->hwconfig.pever,
+ priv->hwconfig.algo_flags);
safexcel_configure(priv);
+ if (IS_ENABLED(CONFIG_PCI) && priv->version == EIP197_DEVBRD) {
+ /*
+ * Request MSI vectors for global + 1 per ring -
+ * or just 1 for older dev images
+ */
+ struct pci_dev *pci_pdev = pdev;
+
+ ret = pci_alloc_irq_vectors(pci_pdev,
+ priv->config.rings + 1,
+ priv->config.rings + 1,
+ PCI_IRQ_MSI | PCI_IRQ_MSIX);
+ if (ret < 0) {
+ dev_err(dev, "Failed to allocate PCI MSI interrupts\n");
+ return ret;
+ }
+ }
+
+ /* Register the ring IRQ handlers and configure the rings */
priv->ring = devm_kcalloc(dev, priv->config.rings,
sizeof(*priv->ring),
GFP_KERNEL);
- if (!priv->ring) {
- ret = -ENOMEM;
- goto err_reg_clk;
- }
+ if (!priv->ring)
+ return -ENOMEM;
for (i = 0; i < priv->config.rings; i++) {
- char irq_name[6] = {0}; /* "ringX\0" */
- char wq_name[9] = {0}; /* "wq_ringX\0" */
+ char wq_name[9] = {0};
int irq;
struct safexcel_ring_irq_data *ring_irq;
ret = safexcel_init_ring_descriptors(priv,
&priv->ring[i].cdr,
&priv->ring[i].rdr);
- if (ret)
- goto err_reg_clk;
+ if (ret) {
+ dev_err(dev, "Failed to initialize rings\n");
+ return ret;
+ }
priv->ring[i].rdr_req = devm_kcalloc(dev,
EIP197_DEFAULT_RING_SIZE,
sizeof(priv->ring[i].rdr_req),
GFP_KERNEL);
- if (!priv->ring[i].rdr_req) {
- ret = -ENOMEM;
- goto err_reg_clk;
- }
+ if (!priv->ring[i].rdr_req)
+ return -ENOMEM;
ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);
- if (!ring_irq) {
- ret = -ENOMEM;
- goto err_reg_clk;
- }
+ if (!ring_irq)
+ return -ENOMEM;
ring_irq->priv = priv;
ring_irq->ring = i;
- snprintf(irq_name, 6, "ring%d", i);
- irq = safexcel_request_ring_irq(pdev, irq_name, safexcel_irq_ring,
+ irq = safexcel_request_ring_irq(pdev,
+ EIP197_IRQ_NUMBER(i, is_pci_dev),
+ is_pci_dev,
+ safexcel_irq_ring,
safexcel_irq_ring_thread,
ring_irq);
if (irq < 0) {
- ret = irq;
- goto err_reg_clk;
+ dev_err(dev, "Failed to get IRQ ID for ring %d\n", i);
+ return irq;
}
priv->ring[i].work_data.priv = priv;
priv->ring[i].work_data.ring = i;
- INIT_WORK(&priv->ring[i].work_data.work, safexcel_dequeue_work);
+ INIT_WORK(&priv->ring[i].work_data.work,
+ safexcel_dequeue_work);
snprintf(wq_name, 9, "wq_ring%d", i);
- priv->ring[i].workqueue = create_singlethread_workqueue(wq_name);
- if (!priv->ring[i].workqueue) {
- ret = -ENOMEM;
- goto err_reg_clk;
- }
+ priv->ring[i].workqueue =
+ create_singlethread_workqueue(wq_name);
+ if (!priv->ring[i].workqueue)
+ return -ENOMEM;
priv->ring[i].requests = 0;
priv->ring[i].busy = false;
@@ -1112,28 +1509,21 @@ static int safexcel_probe(struct platform_device *pdev)
spin_lock_init(&priv->ring[i].queue_lock);
}
- platform_set_drvdata(pdev, priv);
atomic_set(&priv->ring_used, 0);
ret = safexcel_hw_init(priv);
if (ret) {
- dev_err(dev, "EIP h/w init failed (%d)\n", ret);
- goto err_reg_clk;
+ dev_err(dev, "HW init failed (%d)\n", ret);
+ return ret;
}
ret = safexcel_register_algorithms(priv);
if (ret) {
dev_err(dev, "Failed to register algorithms (%d)\n", ret);
- goto err_reg_clk;
+ return ret;
}
return 0;
-
-err_reg_clk:
- clk_disable_unprepare(priv->reg_clk);
-err_core_clk:
- clk_disable_unprepare(priv->clk);
- return ret;
}
static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv)
@@ -1155,6 +1545,76 @@ static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv)
}
}
+#if IS_ENABLED(CONFIG_OF)
+/* for Device Tree platform driver */
+
+static int safexcel_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct safexcel_crypto_priv *priv;
+ int ret;
+
+ priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ priv->dev = dev;
+ priv->version = (enum safexcel_eip_version)of_device_get_match_data(dev);
+
+ platform_set_drvdata(pdev, priv);
+
+ priv->base = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(priv->base)) {
+ dev_err(dev, "failed to get resource\n");
+ return PTR_ERR(priv->base);
+ }
+
+ priv->clk = devm_clk_get(&pdev->dev, NULL);
+ ret = PTR_ERR_OR_ZERO(priv->clk);
+ /* The clock isn't mandatory */
+ if (ret != -ENOENT) {
+ if (ret)
+ return ret;
+
+ ret = clk_prepare_enable(priv->clk);
+ if (ret) {
+ dev_err(dev, "unable to enable clk (%d)\n", ret);
+ return ret;
+ }
+ }
+
+ priv->reg_clk = devm_clk_get(&pdev->dev, "reg");
+ ret = PTR_ERR_OR_ZERO(priv->reg_clk);
+ /* The clock isn't mandatory */
+ if (ret != -ENOENT) {
+ if (ret)
+ goto err_core_clk;
+
+ ret = clk_prepare_enable(priv->reg_clk);
+ if (ret) {
+ dev_err(dev, "unable to enable reg clk (%d)\n", ret);
+ goto err_core_clk;
+ }
+ }
+
+ ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
+ if (ret)
+ goto err_reg_clk;
+
+ /* Generic EIP97/EIP197 device probing */
+ ret = safexcel_probe_generic(pdev, priv, 0);
+ if (ret)
+ goto err_reg_clk;
+
+ return 0;
+
+err_reg_clk:
+ clk_disable_unprepare(priv->reg_clk);
+err_core_clk:
+ clk_disable_unprepare(priv->clk);
+ return ret;
+}
+
static int safexcel_remove(struct platform_device *pdev)
{
struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev);
@@ -1174,30 +1634,28 @@ static int safexcel_remove(struct platform_device *pdev)
static const struct of_device_id safexcel_of_match_table[] = {
{
.compatible = "inside-secure,safexcel-eip97ies",
- .data = (void *)EIP97IES,
+ .data = (void *)EIP97IES_MRVL,
},
{
.compatible = "inside-secure,safexcel-eip197b",
- .data = (void *)EIP197B,
+ .data = (void *)EIP197B_MRVL,
},
{
.compatible = "inside-secure,safexcel-eip197d",
- .data = (void *)EIP197D,
+ .data = (void *)EIP197D_MRVL,
},
+ /* For backward compatibility and intended for generic use */
{
- /* Deprecated. Kept for backward compatibility. */
.compatible = "inside-secure,safexcel-eip97",
- .data = (void *)EIP97IES,
+ .data = (void *)EIP97IES_MRVL,
},
{
- /* Deprecated. Kept for backward compatibility. */
.compatible = "inside-secure,safexcel-eip197",
- .data = (void *)EIP197B,
+ .data = (void *)EIP197B_MRVL,
},
{},
};
-
static struct platform_driver crypto_safexcel = {
.probe = safexcel_probe,
.remove = safexcel_remove,
@@ -1206,10 +1664,166 @@ static struct platform_driver crypto_safexcel = {
.of_match_table = safexcel_of_match_table,
},
};
-module_platform_driver(crypto_safexcel);
+#endif
+
+#if IS_ENABLED(CONFIG_PCI)
+/* PCIE devices - i.e. Inside Secure development boards */
+
+static int safexcel_pci_probe(struct pci_dev *pdev,
+ const struct pci_device_id *ent)
+{
+ struct device *dev = &pdev->dev;
+ struct safexcel_crypto_priv *priv;
+ void __iomem *pciebase;
+ int rc;
+ u32 val;
+
+ dev_dbg(dev, "Probing PCIE device: vendor %04x, device %04x, subv %04x, subdev %04x, ctxt %lx\n",
+ ent->vendor, ent->device, ent->subvendor,
+ ent->subdevice, ent->driver_data);
+
+ priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ priv->dev = dev;
+ priv->version = (enum safexcel_eip_version)ent->driver_data;
+
+ pci_set_drvdata(pdev, priv);
+
+ /* enable the device */
+ rc = pcim_enable_device(pdev);
+ if (rc) {
+ dev_err(dev, "Failed to enable PCI device\n");
+ return rc;
+ }
+
+ /* take ownership of PCI BAR0 */
+ rc = pcim_iomap_regions(pdev, 1, "crypto_safexcel");
+ if (rc) {
+ dev_err(dev, "Failed to map IO region for BAR0\n");
+ return rc;
+ }
+ priv->base = pcim_iomap_table(pdev)[0];
+
+ if (priv->version == EIP197_DEVBRD) {
+ dev_dbg(dev, "Device identified as FPGA based development board - applying HW reset\n");
+
+ rc = pcim_iomap_regions(pdev, 4, "crypto_safexcel");
+ if (rc) {
+ dev_err(dev, "Failed to map IO region for BAR4\n");
+ return rc;
+ }
+
+ pciebase = pcim_iomap_table(pdev)[2];
+ val = readl(pciebase + EIP197_XLX_IRQ_BLOCK_ID_ADDR);
+ if ((val >> 16) == EIP197_XLX_IRQ_BLOCK_ID_VALUE) {
+ dev_dbg(dev, "Detected Xilinx PCIE IRQ block version %d, multiple MSI support enabled\n",
+ (val & 0xff));
+
+ /* Setup MSI identity map mapping */
+ writel(EIP197_XLX_USER_VECT_LUT0_IDENT,
+ pciebase + EIP197_XLX_USER_VECT_LUT0_ADDR);
+ writel(EIP197_XLX_USER_VECT_LUT1_IDENT,
+ pciebase + EIP197_XLX_USER_VECT_LUT1_ADDR);
+ writel(EIP197_XLX_USER_VECT_LUT2_IDENT,
+ pciebase + EIP197_XLX_USER_VECT_LUT2_ADDR);
+ writel(EIP197_XLX_USER_VECT_LUT3_IDENT,
+ pciebase + EIP197_XLX_USER_VECT_LUT3_ADDR);
+
+ /* Enable all device interrupts */
+ writel(GENMASK(31, 0),
+ pciebase + EIP197_XLX_USER_INT_ENB_MSK);
+ } else {
+ dev_err(dev, "Unrecognised IRQ block identifier %x\n",
+ val);
+ return -ENODEV;
+ }
+
+ /* HW reset FPGA dev board */
+ /* assert reset */
+ writel(1, priv->base + EIP197_XLX_GPIO_BASE);
+ wmb(); /* maintain strict ordering for accesses here */
+ /* deassert reset */
+ writel(0, priv->base + EIP197_XLX_GPIO_BASE);
+ wmb(); /* maintain strict ordering for accesses here */
+ }
+
+ /* enable bus mastering */
+ pci_set_master(pdev);
+
+ /* Generic EIP97/EIP197 device probing */
+ rc = safexcel_probe_generic(pdev, priv, 1);
+ return rc;
+}
+
+void safexcel_pci_remove(struct pci_dev *pdev)
+{
+ struct safexcel_crypto_priv *priv = pci_get_drvdata(pdev);
+ int i;
+
+ safexcel_unregister_algorithms(priv);
+
+ for (i = 0; i < priv->config.rings; i++)
+ destroy_workqueue(priv->ring[i].workqueue);
+
+ safexcel_hw_reset_rings(priv);
+}
+
+static const struct pci_device_id safexcel_pci_ids[] = {
+ {
+ PCI_DEVICE_SUB(PCI_VENDOR_ID_XILINX, 0x9038,
+ 0x16ae, 0xc522),
+ .driver_data = EIP197_DEVBRD,
+ },
+ {},
+};
+
+MODULE_DEVICE_TABLE(pci, safexcel_pci_ids);
+
+static struct pci_driver safexcel_pci_driver = {
+ .name = "crypto-safexcel",
+ .id_table = safexcel_pci_ids,
+ .probe = safexcel_pci_probe,
+ .remove = safexcel_pci_remove,
+};
+#endif
+
+static int __init safexcel_init(void)
+{
+ int rc;
+
+#if IS_ENABLED(CONFIG_OF)
+ /* Register platform driver */
+ platform_driver_register(&crypto_safexcel);
+#endif
+
+#if IS_ENABLED(CONFIG_PCI)
+ /* Register PCI driver */
+ rc = pci_register_driver(&safexcel_pci_driver);
+#endif
+
+ return 0;
+}
+
+static void __exit safexcel_exit(void)
+{
+#if IS_ENABLED(CONFIG_OF)
+ /* Unregister platform driver */
+ platform_driver_unregister(&crypto_safexcel);
+#endif
+
+#if IS_ENABLED(CONFIG_PCI)
+ /* Unregister PCI driver if successfully registered before */
+ pci_unregister_driver(&safexcel_pci_driver);
+#endif
+}
+
+module_init(safexcel_init);
+module_exit(safexcel_exit);
MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>");
MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>");
MODULE_AUTHOR("Igal Liberman <igall@marvell.com>");
-MODULE_DESCRIPTION("Support for SafeXcel cryptographic engine EIP197");
+MODULE_DESCRIPTION("Support for SafeXcel cryptographic engines: EIP97 & EIP197");
MODULE_LICENSE("GPL v2");
diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h
index 65624a81f0fd..930cc48a6f85 100644
--- a/drivers/crypto/inside-secure/safexcel.h
+++ b/drivers/crypto/inside-secure/safexcel.h
@@ -14,14 +14,23 @@
#include <crypto/sha.h>
#include <crypto/skcipher.h>
-#define EIP197_HIA_VERSION_LE 0xca35
-#define EIP197_HIA_VERSION_BE 0x35ca
+#define EIP197_HIA_VERSION_BE 0xca35
+#define EIP197_HIA_VERSION_LE 0x35ca
+#define EIP97_VERSION_LE 0x9e61
+#define EIP197_VERSION_LE 0x3ac5
+#define EIP96_VERSION_LE 0x9f60
+#define EIP197_REG_LO16(reg) (reg & 0xffff)
+#define EIP197_REG_HI16(reg) ((reg >> 16) & 0xffff)
+#define EIP197_VERSION_MASK(reg) ((reg >> 16) & 0xfff)
+#define EIP197_VERSION_SWAP(reg) (((reg & 0xf0) << 4) | \
+ ((reg >> 4) & 0xf0) | \
+ ((reg >> 12) & 0xf))
/* Static configuration */
#define EIP197_DEFAULT_RING_SIZE 400
-#define EIP197_MAX_TOKENS 8
+#define EIP197_MAX_TOKENS 18
#define EIP197_MAX_RINGS 4
-#define EIP197_FETCH_COUNT 1
+#define EIP197_FETCH_DEPTH 2
#define EIP197_MAX_BATCH_SZ 64
#define EIP197_GFP_FLAGS(base) ((base).flags & CRYPTO_TFM_REQ_MAY_SLEEP ? \
@@ -38,6 +47,27 @@
char __##name##_desc[size] CRYPTO_MINALIGN_ATTR; \
struct type##_request *name = (void *)__##name##_desc
+/* Xilinx dev board base offsets */
+#define EIP197_XLX_GPIO_BASE 0x200000
+#define EIP197_XLX_IRQ_BLOCK_ID_ADDR 0x2000
+#define EIP197_XLX_IRQ_BLOCK_ID_VALUE 0x1fc2
+#define EIP197_XLX_USER_INT_ENB_MSK 0x2004
+#define EIP197_XLX_USER_INT_ENB_SET 0x2008
+#define EIP197_XLX_USER_INT_ENB_CLEAR 0x200c
+#define EIP197_XLX_USER_INT_BLOCK 0x2040
+#define EIP197_XLX_USER_INT_PEND 0x2048
+#define EIP197_XLX_USER_VECT_LUT0_ADDR 0x2080
+#define EIP197_XLX_USER_VECT_LUT0_IDENT 0x03020100
+#define EIP197_XLX_USER_VECT_LUT1_ADDR 0x2084
+#define EIP197_XLX_USER_VECT_LUT1_IDENT 0x07060504
+#define EIP197_XLX_USER_VECT_LUT2_ADDR 0x2088
+#define EIP197_XLX_USER_VECT_LUT2_IDENT 0x0b0a0908
+#define EIP197_XLX_USER_VECT_LUT3_ADDR 0x208c
+#define EIP197_XLX_USER_VECT_LUT3_IDENT 0x0f0e0d0c
+
+/* Helper defines for probe function */
+#define EIP197_IRQ_NUMBER(i, is_pci) (i + is_pci)
+
/* Register base offsets */
#define EIP197_HIA_AIC(priv) ((priv)->base + (priv)->offsets.hia_aic)
#define EIP197_HIA_AIC_G(priv) ((priv)->base + (priv)->offsets.hia_aic_g)
@@ -49,6 +79,7 @@
#define EIP197_HIA_DSE_THR(priv) ((priv)->base + (priv)->offsets.hia_dse_thr)
#define EIP197_HIA_GEN_CFG(priv) ((priv)->base + (priv)->offsets.hia_gen_cfg)
#define EIP197_PE(priv) ((priv)->base + (priv)->offsets.pe)
+#define EIP197_GLOBAL(priv) ((priv)->base + (priv)->offsets.global)
/* EIP197 base offsets */
#define EIP197_HIA_AIC_BASE 0x90000
@@ -61,6 +92,7 @@
#define EIP197_HIA_DSE_THR_BASE 0x8d040
#define EIP197_HIA_GEN_CFG_BASE 0xf0000
#define EIP197_PE_BASE 0xa0000
+#define EIP197_GLOBAL_BASE 0xf0000
/* EIP97 base offsets */
#define EIP97_HIA_AIC_BASE 0x0
@@ -73,6 +105,7 @@
#define EIP97_HIA_DSE_THR_BASE 0xf600
#define EIP97_HIA_GEN_CFG_BASE 0x10000
#define EIP97_PE_BASE 0x10000
+#define EIP97_GLOBAL_BASE 0x10000
/* CDR/RDR register offsets */
#define EIP197_HIA_xDR_OFF(priv, r) (EIP197_HIA_AIC_xDR(priv) + (r) * 0x1000)
@@ -115,15 +148,22 @@
#define EIP197_PE_IN_TBUF_THRES(n) (0x0100 + (0x2000 * (n)))
#define EIP197_PE_ICE_SCRATCH_RAM(n) (0x0800 + (0x2000 * (n)))
#define EIP197_PE_ICE_PUE_CTRL(n) (0x0c80 + (0x2000 * (n)))
+#define EIP197_PE_ICE_PUTF_CTRL(n) (0x0d00 + (0x2000 * (n)))
#define EIP197_PE_ICE_SCRATCH_CTRL(n) (0x0d04 + (0x2000 * (n)))
#define EIP197_PE_ICE_FPP_CTRL(n) (0x0d80 + (0x2000 * (n)))
+#define EIP197_PE_ICE_PPTF_CTRL(n) (0x0e00 + (0x2000 * (n)))
#define EIP197_PE_ICE_RAM_CTRL(n) (0x0ff0 + (0x2000 * (n)))
+#define EIP197_PE_EIP96_TOKEN_CTRL(n) (0x1000 + (0x2000 * (n)))
#define EIP197_PE_EIP96_FUNCTION_EN(n) (0x1004 + (0x2000 * (n)))
#define EIP197_PE_EIP96_CONTEXT_CTRL(n) (0x1008 + (0x2000 * (n)))
#define EIP197_PE_EIP96_CONTEXT_STAT(n) (0x100c + (0x2000 * (n)))
+#define EIP197_PE_EIP96_FUNCTION2_EN(n) (0x1030 + (0x2000 * (n)))
+#define EIP197_PE_EIP96_OPTIONS(n) (0x13f8 + (0x2000 * (n)))
+#define EIP197_PE_EIP96_VERSION(n) (0x13fc + (0x2000 * (n)))
#define EIP197_PE_OUT_DBUF_THRES(n) (0x1c00 + (0x2000 * (n)))
#define EIP197_PE_OUT_TBUF_THRES(n) (0x1d00 + (0x2000 * (n)))
#define EIP197_MST_CTRL 0xfff4
+#define EIP197_VERSION 0xfffc
/* EIP197-specific registers, no indirection */
#define EIP197_CLASSIFICATION_RAMS 0xe0000
@@ -138,6 +178,12 @@
#define EIP197_TRC_ECCADMINSTAT 0xf0838
#define EIP197_TRC_ECCDATASTAT 0xf083c
#define EIP197_TRC_ECCDATA 0xf0840
+#define EIP197_FLUE_CACHEBASE_LO(n) (0xf6000 + (32 * (n)))
+#define EIP197_FLUE_CACHEBASE_HI(n) (0xf6004 + (32 * (n)))
+#define EIP197_FLUE_CONFIG(n) (0xf6010 + (32 * (n)))
+#define EIP197_FLUE_OFFSETS 0xf6808
+#define EIP197_FLUE_ARC4_OFFSET 0xf680c
+#define EIP197_FLUE_IFC_LUT(n) (0xf6820 + (4 * (n)))
#define EIP197_CS_RAM_CTRL 0xf7ff0
/* EIP197_HIA_xDR_DESC_SIZE */
@@ -185,6 +231,19 @@
#define EIP197_N_PES_OFFSET 4
#define EIP197_N_PES_MASK GENMASK(4, 0)
#define EIP97_N_PES_MASK GENMASK(2, 0)
+#define EIP197_HWDATAW_OFFSET 25
+#define EIP197_HWDATAW_MASK GENMASK(3, 0)
+#define EIP97_HWDATAW_MASK GENMASK(2, 0)
+#define EIP197_CFSIZE_OFFSET 9
+#define EIP197_CFSIZE_ADJUST 4
+#define EIP97_CFSIZE_OFFSET 8
+#define EIP197_CFSIZE_MASK GENMASK(3, 0)
+#define EIP97_CFSIZE_MASK GENMASK(4, 0)
+#define EIP197_RFSIZE_OFFSET 12
+#define EIP197_RFSIZE_ADJUST 4
+#define EIP97_RFSIZE_OFFSET 12
+#define EIP197_RFSIZE_MASK GENMASK(3, 0)
+#define EIP97_RFSIZE_MASK GENMASK(4, 0)
/* EIP197_HIA_AIC_R_ENABLE_CTRL */
#define EIP197_CDR_IRQ(n) BIT((n) * 2)
@@ -206,6 +265,11 @@
#define EIP197_DxE_THR_CTRL_EN BIT(30)
#define EIP197_DxE_THR_CTRL_RESET_PE BIT(31)
+/* EIP197_PE_ICE_PUE/FPP_CTRL */
+#define EIP197_PE_ICE_UENG_START_OFFSET(n) ((n) << 16)
+#define EIP197_PE_ICE_UENG_INIT_ALIGN_MASK 0x7ff0
+#define EIP197_PE_ICE_UENG_DEBUG_RESET BIT(3)
+
/* EIP197_HIA_AIC_G_ENABLED_STAT */
#define EIP197_G_IRQ_DFE(n) BIT((n) << 1)
#define EIP197_G_IRQ_DSE(n) BIT(((n) << 1) + 1)
@@ -222,6 +286,7 @@
#define EIP197_MST_CTRL_TX_MAX_CMD(n) (((n) & 0xf) << 20)
#define EIP197_MST_CTRL_BYTE_SWAP BIT(24)
#define EIP197_MST_CTRL_NO_BYTE_SWAP BIT(25)
+#define EIP197_MST_CTRL_BYTE_SWAP_BITS GENMASK(25, 24)
/* EIP197_PE_IN_DBUF/TBUF_THRES */
#define EIP197_PE_IN_xBUF_THRES_MIN(n) ((n) << 8)
@@ -249,42 +314,22 @@
#define EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN BIT(0)
#define EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN BIT(1)
+/* EIP197_PE_EIP96_TOKEN_CTRL */
+#define EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES BIT(16)
+#define EIP197_PE_EIP96_TOKEN_CTRL_NO_TOKEN_WAIT BIT(17)
+#define EIP197_PE_EIP96_TOKEN_CTRL_ENABLE_TIMEOUT BIT(22)
+
/* EIP197_PE_EIP96_FUNCTION_EN */
-#define EIP197_FUNCTION_RSVD (BIT(6) | BIT(15) | BIT(20) | BIT(23))
-#define EIP197_PROTOCOL_HASH_ONLY BIT(0)
-#define EIP197_PROTOCOL_ENCRYPT_ONLY BIT(1)
-#define EIP197_PROTOCOL_HASH_ENCRYPT BIT(2)
-#define EIP197_PROTOCOL_HASH_DECRYPT BIT(3)
-#define EIP197_PROTOCOL_ENCRYPT_HASH BIT(4)
-#define EIP197_PROTOCOL_DECRYPT_HASH BIT(5)
-#define EIP197_ALG_ARC4 BIT(7)
-#define EIP197_ALG_AES_ECB BIT(8)
-#define EIP197_ALG_AES_CBC BIT(9)
-#define EIP197_ALG_AES_CTR_ICM BIT(10)
-#define EIP197_ALG_AES_OFB BIT(11)
-#define EIP197_ALG_AES_CFB BIT(12)
-#define EIP197_ALG_DES_ECB BIT(13)
-#define EIP197_ALG_DES_CBC BIT(14)
-#define EIP197_ALG_DES_OFB BIT(16)
-#define EIP197_ALG_DES_CFB BIT(17)
-#define EIP197_ALG_3DES_ECB BIT(18)
-#define EIP197_ALG_3DES_CBC BIT(19)
-#define EIP197_ALG_3DES_OFB BIT(21)
-#define EIP197_ALG_3DES_CFB BIT(22)
-#define EIP197_ALG_MD5 BIT(24)
-#define EIP197_ALG_HMAC_MD5 BIT(25)
-#define EIP197_ALG_SHA1 BIT(26)
-#define EIP197_ALG_HMAC_SHA1 BIT(27)
-#define EIP197_ALG_SHA2 BIT(28)
-#define EIP197_ALG_HMAC_SHA2 BIT(29)
-#define EIP197_ALG_AES_XCBC_MAC BIT(30)
-#define EIP197_ALG_GCM_HASH BIT(31)
+#define EIP197_FUNCTION_ALL 0xffffffff
/* EIP197_PE_EIP96_CONTEXT_CTRL */
#define EIP197_CONTEXT_SIZE(n) (n)
#define EIP197_ADDRESS_MODE BIT(8)
#define EIP197_CONTROL_MODE BIT(9)
+/* EIP197_FLUE_CONFIG */
+#define EIP197_FLUE_CONFIG_MAGIC 0xc7000004
+
/* Context Control */
struct safexcel_context_record {
u32 control0;
@@ -314,6 +359,7 @@ struct safexcel_context_record {
#define CONTEXT_CONTROL_CRYPTO_ALG_AES192 (0x6 << 17)
#define CONTEXT_CONTROL_CRYPTO_ALG_AES256 (0x7 << 17)
#define CONTEXT_CONTROL_DIGEST_PRECOMPUTED (0x1 << 21)
+#define CONTEXT_CONTROL_DIGEST_XCM (0x2 << 21)
#define CONTEXT_CONTROL_DIGEST_HMAC (0x3 << 21)
#define CONTEXT_CONTROL_CRYPTO_ALG_MD5 (0x0 << 23)
#define CONTEXT_CONTROL_CRYPTO_ALG_SHA1 (0x2 << 23)
@@ -321,20 +367,33 @@ struct safexcel_context_record {
#define CONTEXT_CONTROL_CRYPTO_ALG_SHA256 (0x3 << 23)
#define CONTEXT_CONTROL_CRYPTO_ALG_SHA384 (0x6 << 23)
#define CONTEXT_CONTROL_CRYPTO_ALG_SHA512 (0x5 << 23)
+#define CONTEXT_CONTROL_CRYPTO_ALG_GHASH (0x4 << 23)
+#define CONTEXT_CONTROL_CRYPTO_ALG_XCBC128 (0x1 << 23)
+#define CONTEXT_CONTROL_CRYPTO_ALG_XCBC192 (0x2 << 23)
+#define CONTEXT_CONTROL_CRYPTO_ALG_XCBC256 (0x3 << 23)
#define CONTEXT_CONTROL_INV_FR (0x5 << 24)
#define CONTEXT_CONTROL_INV_TR (0x6 << 24)
/* control1 */
#define CONTEXT_CONTROL_CRYPTO_MODE_ECB (0 << 0)
#define CONTEXT_CONTROL_CRYPTO_MODE_CBC (1 << 0)
+#define CONTEXT_CONTROL_CRYPTO_MODE_OFB (4 << 0)
+#define CONTEXT_CONTROL_CRYPTO_MODE_CFB (5 << 0)
+#define CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD (6 << 0)
+#define CONTEXT_CONTROL_CRYPTO_MODE_XTS (7 << 0)
+#define CONTEXT_CONTROL_CRYPTO_MODE_XCM ((6 << 0) | BIT(17))
#define CONTEXT_CONTROL_IV0 BIT(5)
#define CONTEXT_CONTROL_IV1 BIT(6)
#define CONTEXT_CONTROL_IV2 BIT(7)
#define CONTEXT_CONTROL_IV3 BIT(8)
#define CONTEXT_CONTROL_DIGEST_CNT BIT(9)
#define CONTEXT_CONTROL_COUNTER_MODE BIT(10)
+#define CONTEXT_CONTROL_CRYPTO_STORE BIT(12)
#define CONTEXT_CONTROL_HASH_STORE BIT(19)
+#define EIP197_XCM_MODE_GCM 1
+#define EIP197_XCM_MODE_CCM 2
+
/* The hash counter given to the engine in the context has a granularity of
* 64 bits.
*/
@@ -345,6 +404,8 @@ struct safexcel_context_record {
#define EIP197_TRC_ENABLE_1 BIT(5)
#define EIP197_TRC_ENABLE_2 BIT(6)
#define EIP197_TRC_ENABLE_MASK GENMASK(6, 4)
+#define EIP197_CS_BANKSEL_MASK GENMASK(14, 12)
+#define EIP197_CS_BANKSEL_OFS 12
/* EIP197_TRC_PARAMS */
#define EIP197_TRC_PARAMS_SW_RESET BIT(0)
@@ -362,19 +423,11 @@ struct safexcel_context_record {
#define EIP197_TRC_PARAMS2_RC_SZ_SMALL(n) ((n) << 18)
/* Cache helpers */
-#define EIP197B_CS_RC_MAX 52
-#define EIP197D_CS_RC_MAX 96
+#define EIP197_CS_TRC_REC_WC 64
#define EIP197_CS_RC_SIZE (4 * sizeof(u32))
#define EIP197_CS_RC_NEXT(x) (x)
#define EIP197_CS_RC_PREV(x) ((x) << 10)
#define EIP197_RC_NULL 0x3ff
-#define EIP197B_CS_TRC_REC_WC 59
-#define EIP197D_CS_TRC_REC_WC 64
-#define EIP197B_CS_TRC_LG_REC_WC 73
-#define EIP197D_CS_TRC_LG_REC_WC 80
-#define EIP197B_CS_HT_WC 64
-#define EIP197D_CS_HT_WC 256
-
/* Result data */
struct result_data_desc {
@@ -416,6 +469,14 @@ struct safexcel_result_desc {
struct result_data_desc result_data;
} __packed;
+/*
+ * The EIP(1)97 only needs to fetch the descriptor part of
+ * the result descriptor, not the result token part!
+ */
+#define EIP197_RD64_FETCH_SIZE ((sizeof(struct safexcel_result_desc) -\
+ sizeof(struct result_data_desc)) /\
+ sizeof(u32))
+
struct safexcel_token {
u32 packet_length:17;
u8 stat:2;
@@ -425,13 +486,19 @@ struct safexcel_token {
#define EIP197_TOKEN_HASH_RESULT_VERIFY BIT(16)
+#define EIP197_TOKEN_CTX_OFFSET(x) (x)
+#define EIP197_TOKEN_DIRECTION_EXTERNAL BIT(11)
+#define EIP197_TOKEN_EXEC_IF_SUCCESSFUL (0x1 << 12)
+
#define EIP197_TOKEN_STAT_LAST_HASH BIT(0)
#define EIP197_TOKEN_STAT_LAST_PACKET BIT(1)
#define EIP197_TOKEN_OPCODE_DIRECTION 0x0
#define EIP197_TOKEN_OPCODE_INSERT 0x2
#define EIP197_TOKEN_OPCODE_NOOP EIP197_TOKEN_OPCODE_INSERT
#define EIP197_TOKEN_OPCODE_RETRIEVE 0x4
+#define EIP197_TOKEN_OPCODE_INSERT_REMRES 0xa
#define EIP197_TOKEN_OPCODE_VERIFY 0xd
+#define EIP197_TOKEN_OPCODE_CTX_ACCESS 0xe
#define EIP197_TOKEN_OPCODE_BYPASS GENMASK(3, 0)
static inline void eip197_noop_token(struct safexcel_token *token)
@@ -442,9 +509,12 @@ static inline void eip197_noop_token(struct safexcel_token *token)
/* Instructions */
#define EIP197_TOKEN_INS_INSERT_HASH_DIGEST 0x1c
+#define EIP197_TOKEN_INS_ORIGIN_IV0 0x14
+#define EIP197_TOKEN_INS_ORIGIN_TOKEN 0x1b
+#define EIP197_TOKEN_INS_ORIGIN_LEN(x) ((x) << 5)
#define EIP197_TOKEN_INS_TYPE_OUTPUT BIT(5)
#define EIP197_TOKEN_INS_TYPE_HASH BIT(6)
-#define EIP197_TOKEN_INS_TYPE_CRYTO BIT(7)
+#define EIP197_TOKEN_INS_TYPE_CRYPTO BIT(7)
#define EIP197_TOKEN_INS_LAST BIT(8)
/* Processing Engine Control Data */
@@ -468,6 +538,7 @@ struct safexcel_control_data_desc {
#define EIP197_OPTION_MAGIC_VALUE BIT(0)
#define EIP197_OPTION_64BIT_CTX BIT(1)
+#define EIP197_OPTION_RC_AUTO (0x2 << 3)
#define EIP197_OPTION_CTX_CTRL_IN_CMD BIT(8)
#define EIP197_OPTION_2_TOKEN_IV_CMD GENMASK(11, 10)
#define EIP197_OPTION_4_TOKEN_IV_CMD GENMASK(11, 9)
@@ -494,6 +565,11 @@ struct safexcel_command_desc {
* Internal structures & functions
*/
+#define EIP197_FW_TERMINAL_NOPS 2
+#define EIP197_FW_START_POLLCNT 16
+#define EIP197_FW_PUE_READY 0x14
+#define EIP197_FW_FPP_READY 0x18
+
enum eip197_fw {
FW_IFPP = 0,
FW_IPUE,
@@ -566,10 +642,42 @@ struct safexcel_ring {
struct crypto_async_request *backlog;
};
+/* EIP integration context flags */
enum safexcel_eip_version {
- EIP97IES = BIT(0),
- EIP197B = BIT(1),
- EIP197D = BIT(2),
+ /* Platform (EIP integration context) specifier */
+ EIP97IES_MRVL,
+ EIP197B_MRVL,
+ EIP197D_MRVL,
+ EIP197_DEVBRD
+};
+
+/* Priority we use for advertising our algorithms */
+#define SAFEXCEL_CRA_PRIORITY 300
+
+/* EIP algorithm presence flags */
+enum safexcel_eip_algorithms {
+ SAFEXCEL_ALG_BC0 = BIT(5),
+ SAFEXCEL_ALG_SM4 = BIT(6),
+ SAFEXCEL_ALG_SM3 = BIT(7),
+ SAFEXCEL_ALG_CHACHA20 = BIT(8),
+ SAFEXCEL_ALG_POLY1305 = BIT(9),
+ SAFEXCEL_SEQMASK_256 = BIT(10),
+ SAFEXCEL_SEQMASK_384 = BIT(11),
+ SAFEXCEL_ALG_AES = BIT(12),
+ SAFEXCEL_ALG_AES_XFB = BIT(13),
+ SAFEXCEL_ALG_DES = BIT(15),
+ SAFEXCEL_ALG_DES_XFB = BIT(16),
+ SAFEXCEL_ALG_ARC4 = BIT(18),
+ SAFEXCEL_ALG_AES_XTS = BIT(20),
+ SAFEXCEL_ALG_WIRELESS = BIT(21),
+ SAFEXCEL_ALG_MD5 = BIT(22),
+ SAFEXCEL_ALG_SHA1 = BIT(23),
+ SAFEXCEL_ALG_SHA2_256 = BIT(25),
+ SAFEXCEL_ALG_SHA2_512 = BIT(26),
+ SAFEXCEL_ALG_XCBC_MAC = BIT(27),
+ SAFEXCEL_ALG_CBC_MAC_ALL = BIT(29),
+ SAFEXCEL_ALG_GHASH = BIT(30),
+ SAFEXCEL_ALG_SHA3 = BIT(31),
};
struct safexcel_register_offsets {
@@ -583,10 +691,22 @@ struct safexcel_register_offsets {
u32 hia_dse_thr;
u32 hia_gen_cfg;
u32 pe;
+ u32 global;
};
enum safexcel_flags {
- EIP197_TRC_CACHE = BIT(0),
+ EIP197_TRC_CACHE = BIT(0),
+ SAFEXCEL_HW_EIP197 = BIT(1),
+};
+
+struct safexcel_hwconfig {
+ enum safexcel_eip_algorithms algo_flags;
+ int hwver;
+ int hiaver;
+ int pever;
+ int hwdataw;
+ int hwcfsize;
+ int hwrfsize;
};
struct safexcel_crypto_priv {
@@ -598,6 +718,7 @@ struct safexcel_crypto_priv {
enum safexcel_eip_version version;
struct safexcel_register_offsets offsets;
+ struct safexcel_hwconfig hwconfig;
u32 flags;
/* context DMA pool */
@@ -622,14 +743,16 @@ struct safexcel_context {
bool exit_inv;
};
+#define HASH_CACHE_SIZE SHA512_BLOCK_SIZE
+
struct safexcel_ahash_export_state {
- u64 len[2];
- u64 processed[2];
+ u64 len;
+ u64 processed;
u32 digest;
u32 state[SHA512_DIGEST_SIZE / sizeof(u32)];
- u8 cache[SHA512_BLOCK_SIZE];
+ u8 cache[HASH_CACHE_SIZE];
};
/*
@@ -640,7 +763,7 @@ struct safexcel_ahash_export_state {
struct safexcel_alg_template {
struct safexcel_crypto_priv *priv;
enum safexcel_alg_type type;
- u32 engines;
+ enum safexcel_eip_algorithms algo_mask;
union {
struct skcipher_alg skcipher;
struct aead_alg aead;
@@ -701,6 +824,9 @@ extern struct safexcel_alg_template safexcel_alg_ecb_des3_ede;
extern struct safexcel_alg_template safexcel_alg_cbc_des3_ede;
extern struct safexcel_alg_template safexcel_alg_ecb_aes;
extern struct safexcel_alg_template safexcel_alg_cbc_aes;
+extern struct safexcel_alg_template safexcel_alg_cfb_aes;
+extern struct safexcel_alg_template safexcel_alg_ofb_aes;
+extern struct safexcel_alg_template safexcel_alg_ctr_aes;
extern struct safexcel_alg_template safexcel_alg_md5;
extern struct safexcel_alg_template safexcel_alg_sha1;
extern struct safexcel_alg_template safexcel_alg_sha224;
@@ -718,5 +844,14 @@ extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes;
extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes;
extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes;
extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes;
+extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede;
+extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_aes;
+extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_ctr_aes;
+extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_ctr_aes;
+extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_ctr_aes;
+extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_ctr_aes;
+extern struct safexcel_alg_template safexcel_alg_xts_aes;
+extern struct safexcel_alg_template safexcel_alg_gcm;
+extern struct safexcel_alg_template safexcel_alg_ccm;
#endif
diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c
index de4be10b172f..ef51f8c2b473 100644
--- a/drivers/crypto/inside-secure/safexcel_cipher.c
+++ b/drivers/crypto/inside-secure/safexcel_cipher.c
@@ -12,8 +12,12 @@
#include <crypto/aead.h>
#include <crypto/aes.h>
#include <crypto/authenc.h>
-#include <crypto/des.h>
+#include <crypto/ctr.h>
+#include <crypto/internal/des.h>
+#include <crypto/gcm.h>
+#include <crypto/ghash.h>
#include <crypto/sha.h>
+#include <crypto/xts.h>
#include <crypto/skcipher.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
@@ -38,58 +42,102 @@ struct safexcel_cipher_ctx {
u32 mode;
enum safexcel_cipher_alg alg;
bool aead;
+ int xcm; /* 0=authenc, 1=GCM, 2 reserved for CCM */
- __le32 key[8];
- unsigned int key_len;
+ __le32 key[16];
+ u32 nonce;
+ unsigned int key_len, xts;
/* All the below is AEAD specific */
u32 hash_alg;
u32 state_sz;
u32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)];
u32 opad[SHA512_DIGEST_SIZE / sizeof(u32)];
+
+ struct crypto_cipher *hkaes;
};
struct safexcel_cipher_req {
enum safexcel_cipher_direction direction;
+ /* Number of result descriptors associated to the request */
+ unsigned int rdescs;
bool needs_inv;
+ int nr_src, nr_dst;
};
-static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
- struct safexcel_command_desc *cdesc,
- u32 length)
+static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
+ struct safexcel_command_desc *cdesc)
{
- struct safexcel_token *token;
- unsigned offset = 0;
+ u32 block_sz = 0;
+
+ if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) {
+ cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
+
+ /* 32 bit nonce */
+ cdesc->control_data.token[0] = ctx->nonce;
+ /* 64 bit IV part */
+ memcpy(&cdesc->control_data.token[1], iv, 8);
+ /* 32 bit counter, start at 1 (big endian!) */
+ cdesc->control_data.token[3] = cpu_to_be32(1);
+
+ return;
+ } else if (ctx->xcm == EIP197_XCM_MODE_GCM) {
+ cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
+
+ /* 96 bit IV part */
+ memcpy(&cdesc->control_data.token[0], iv, 12);
+ /* 32 bit counter, start at 1 (big endian!) */
+ cdesc->control_data.token[3] = cpu_to_be32(1);
+
+ return;
+ } else if (ctx->xcm == EIP197_XCM_MODE_CCM) {
+ cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
- if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) {
+ /* Variable length IV part */
+ memcpy(&cdesc->control_data.token[0], iv, 15 - iv[0]);
+ /* Start variable length counter at 0 */
+ memset((u8 *)&cdesc->control_data.token[0] + 15 - iv[0],
+ 0, iv[0] + 1);
+
+ return;
+ }
+
+ if (ctx->mode != CONTEXT_CONTROL_CRYPTO_MODE_ECB) {
switch (ctx->alg) {
case SAFEXCEL_DES:
- offset = DES_BLOCK_SIZE / sizeof(u32);
- memcpy(cdesc->control_data.token, iv, DES_BLOCK_SIZE);
+ block_sz = DES_BLOCK_SIZE;
cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD;
break;
case SAFEXCEL_3DES:
- offset = DES3_EDE_BLOCK_SIZE / sizeof(u32);
- memcpy(cdesc->control_data.token, iv, DES3_EDE_BLOCK_SIZE);
+ block_sz = DES3_EDE_BLOCK_SIZE;
cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD;
break;
-
case SAFEXCEL_AES:
- offset = AES_BLOCK_SIZE / sizeof(u32);
- memcpy(cdesc->control_data.token, iv, AES_BLOCK_SIZE);
+ block_sz = AES_BLOCK_SIZE;
cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
break;
}
+ memcpy(cdesc->control_data.token, iv, block_sz);
}
+}
+
+static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
+ struct safexcel_command_desc *cdesc,
+ u32 length)
+{
+ struct safexcel_token *token;
+
+ safexcel_cipher_token(ctx, iv, cdesc);
- token = (struct safexcel_token *)(cdesc->control_data.token + offset);
+ /* skip over worst case IV of 4 dwords, no need to be exact */
+ token = (struct safexcel_token *)(cdesc->control_data.token + 4);
token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
token[0].packet_length = length;
token[0].stat = EIP197_TOKEN_STAT_LAST_PACKET |
EIP197_TOKEN_STAT_LAST_HASH;
token[0].instructions = EIP197_TOKEN_INS_LAST |
- EIP197_TOKEN_INS_TYPE_CRYTO |
+ EIP197_TOKEN_INS_TYPE_CRYPTO |
EIP197_TOKEN_INS_TYPE_OUTPUT;
}
@@ -99,53 +147,123 @@ static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
u32 cryptlen, u32 assoclen, u32 digestsize)
{
struct safexcel_token *token;
- unsigned offset = 0;
- if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) {
- offset = AES_BLOCK_SIZE / sizeof(u32);
- memcpy(cdesc->control_data.token, iv, AES_BLOCK_SIZE);
+ safexcel_cipher_token(ctx, iv, cdesc);
- cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
+ if (direction == SAFEXCEL_ENCRYPT) {
+ /* align end of instruction sequence to end of token */
+ token = (struct safexcel_token *)(cdesc->control_data.token +
+ EIP197_MAX_TOKENS - 13);
+
+ token[12].opcode = EIP197_TOKEN_OPCODE_INSERT;
+ token[12].packet_length = digestsize;
+ token[12].stat = EIP197_TOKEN_STAT_LAST_HASH |
+ EIP197_TOKEN_STAT_LAST_PACKET;
+ token[12].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
+ EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
+ } else {
+ cryptlen -= digestsize;
+
+ /* align end of instruction sequence to end of token */
+ token = (struct safexcel_token *)(cdesc->control_data.token +
+ EIP197_MAX_TOKENS - 14);
+
+ token[12].opcode = EIP197_TOKEN_OPCODE_RETRIEVE;
+ token[12].packet_length = digestsize;
+ token[12].stat = EIP197_TOKEN_STAT_LAST_HASH |
+ EIP197_TOKEN_STAT_LAST_PACKET;
+ token[12].instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
+
+ token[13].opcode = EIP197_TOKEN_OPCODE_VERIFY;
+ token[13].packet_length = digestsize |
+ EIP197_TOKEN_HASH_RESULT_VERIFY;
+ token[13].stat = EIP197_TOKEN_STAT_LAST_HASH |
+ EIP197_TOKEN_STAT_LAST_PACKET;
+ token[13].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT;
}
- token = (struct safexcel_token *)(cdesc->control_data.token + offset);
+ token[6].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
+ token[6].packet_length = assoclen;
+
+ if (likely(cryptlen)) {
+ token[6].instructions = EIP197_TOKEN_INS_TYPE_HASH;
+
+ token[10].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
+ token[10].packet_length = cryptlen;
+ token[10].stat = EIP197_TOKEN_STAT_LAST_HASH;
+ token[10].instructions = EIP197_TOKEN_INS_LAST |
+ EIP197_TOKEN_INS_TYPE_CRYPTO |
+ EIP197_TOKEN_INS_TYPE_HASH |
+ EIP197_TOKEN_INS_TYPE_OUTPUT;
+ } else if (ctx->xcm != EIP197_XCM_MODE_CCM) {
+ token[6].stat = EIP197_TOKEN_STAT_LAST_HASH;
+ token[6].instructions = EIP197_TOKEN_INS_LAST |
+ EIP197_TOKEN_INS_TYPE_HASH;
+ }
- if (direction == SAFEXCEL_DECRYPT)
- cryptlen -= digestsize;
+ if (!ctx->xcm)
+ return;
- token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
- token[0].packet_length = assoclen;
- token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH |
- EIP197_TOKEN_INS_TYPE_OUTPUT;
+ token[8].opcode = EIP197_TOKEN_OPCODE_INSERT_REMRES;
+ token[8].packet_length = 0;
+ token[8].instructions = AES_BLOCK_SIZE;
- token[1].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
- token[1].packet_length = cryptlen;
- token[1].stat = EIP197_TOKEN_STAT_LAST_HASH;
- token[1].instructions = EIP197_TOKEN_INS_LAST |
- EIP197_TOKEN_INS_TYPE_CRYTO |
- EIP197_TOKEN_INS_TYPE_HASH |
- EIP197_TOKEN_INS_TYPE_OUTPUT;
+ token[9].opcode = EIP197_TOKEN_OPCODE_INSERT;
+ token[9].packet_length = AES_BLOCK_SIZE;
+ token[9].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
+ EIP197_TOKEN_INS_TYPE_CRYPTO;
- if (direction == SAFEXCEL_ENCRYPT) {
- token[2].opcode = EIP197_TOKEN_OPCODE_INSERT;
- token[2].packet_length = digestsize;
- token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
- EIP197_TOKEN_STAT_LAST_PACKET;
- token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
- EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
+ if (ctx->xcm == EIP197_XCM_MODE_GCM) {
+ token[6].instructions = EIP197_TOKEN_INS_LAST |
+ EIP197_TOKEN_INS_TYPE_HASH;
} else {
- token[2].opcode = EIP197_TOKEN_OPCODE_RETRIEVE;
- token[2].packet_length = digestsize;
- token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
- EIP197_TOKEN_STAT_LAST_PACKET;
- token[2].instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
-
- token[3].opcode = EIP197_TOKEN_OPCODE_VERIFY;
- token[3].packet_length = digestsize |
- EIP197_TOKEN_HASH_RESULT_VERIFY;
- token[3].stat = EIP197_TOKEN_STAT_LAST_HASH |
- EIP197_TOKEN_STAT_LAST_PACKET;
- token[3].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT;
+ u8 *cbcmaciv = (u8 *)&token[1];
+ u32 *aadlen = (u32 *)&token[5];
+
+ /* Construct IV block B0 for the CBC-MAC */
+ token[0].opcode = EIP197_TOKEN_OPCODE_INSERT;
+ token[0].packet_length = AES_BLOCK_SIZE +
+ ((assoclen > 0) << 1);
+ token[0].instructions = EIP197_TOKEN_INS_ORIGIN_TOKEN |
+ EIP197_TOKEN_INS_TYPE_HASH;
+ /* Variable length IV part */
+ memcpy(cbcmaciv, iv, 15 - iv[0]);
+ /* fixup flags byte */
+ cbcmaciv[0] |= ((assoclen > 0) << 6) | ((digestsize - 2) << 2);
+ /* Clear upper bytes of variable message length to 0 */
+ memset(cbcmaciv + 15 - iv[0], 0, iv[0] - 1);
+ /* insert lower 2 bytes of message length */
+ cbcmaciv[14] = cryptlen >> 8;
+ cbcmaciv[15] = cryptlen & 255;
+
+ if (assoclen) {
+ *aadlen = cpu_to_le32(cpu_to_be16(assoclen));
+ assoclen += 2;
+ }
+
+ token[6].instructions = EIP197_TOKEN_INS_TYPE_HASH;
+
+ /* Align AAD data towards hash engine */
+ token[7].opcode = EIP197_TOKEN_OPCODE_INSERT;
+ assoclen &= 15;
+ token[7].packet_length = assoclen ? 16 - assoclen : 0;
+
+ if (likely(cryptlen)) {
+ token[7].instructions = EIP197_TOKEN_INS_TYPE_HASH;
+
+ /* Align crypto data towards hash engine */
+ token[10].stat = 0;
+
+ token[11].opcode = EIP197_TOKEN_OPCODE_INSERT;
+ cryptlen &= 15;
+ token[11].packet_length = cryptlen ? 16 - cryptlen : 0;
+ token[11].stat = EIP197_TOKEN_STAT_LAST_HASH;
+ token[11].instructions = EIP197_TOKEN_INS_TYPE_HASH;
+ } else {
+ token[7].stat = EIP197_TOKEN_STAT_LAST_HASH;
+ token[7].instructions = EIP197_TOKEN_INS_LAST |
+ EIP197_TOKEN_INS_TYPE_HASH;
+ }
}
}
@@ -158,7 +276,7 @@ static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm,
struct crypto_aes_ctx aes;
int ret, i;
- ret = crypto_aes_expand_key(&aes, key, len);
+ ret = aes_expandkey(&aes, key, len);
if (ret) {
crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return ret;
@@ -182,22 +300,49 @@ static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm,
return 0;
}
-static int safexcel_aead_aes_setkey(struct crypto_aead *ctfm, const u8 *key,
- unsigned int len)
+static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key,
+ unsigned int len)
{
struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
struct safexcel_ahash_export_state istate, ostate;
struct safexcel_crypto_priv *priv = ctx->priv;
struct crypto_authenc_keys keys;
+ struct crypto_aes_ctx aes;
+ int err = -EINVAL;
if (crypto_authenc_extractkeys(&keys, key, len) != 0)
goto badkey;
- if (keys.enckeylen > sizeof(ctx->key))
- goto badkey;
+ if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) {
+ /* Minimum keysize is minimum AES key size + nonce size */
+ if (keys.enckeylen < (AES_MIN_KEY_SIZE +
+ CTR_RFC3686_NONCE_SIZE))
+ goto badkey;
+ /* last 4 bytes of key are the nonce! */
+ ctx->nonce = *(u32 *)(keys.enckey + keys.enckeylen -
+ CTR_RFC3686_NONCE_SIZE);
+ /* exclude the nonce here */
+ keys.enckeylen -= CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD;
+ }
/* Encryption key */
+ switch (ctx->alg) {
+ case SAFEXCEL_3DES:
+ err = verify_aead_des3_key(ctfm, keys.enckey, keys.enckeylen);
+ if (unlikely(err))
+ goto badkey_expflags;
+ break;
+ case SAFEXCEL_AES:
+ err = aes_expandkey(&aes, keys.enckey, keys.enckeylen);
+ if (unlikely(err))
+ goto badkey;
+ break;
+ default:
+ dev_err(priv->dev, "aead: unsupported cipher algorithm\n");
+ goto badkey;
+ }
+
if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma &&
memcmp(ctx->key, keys.enckey, keys.enckeylen))
ctx->base.needs_inv = true;
@@ -254,8 +399,9 @@ static int safexcel_aead_aes_setkey(struct crypto_aead *ctfm, const u8 *key,
badkey:
crypto_aead_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+badkey_expflags:
memzero_explicit(&keys, sizeof(keys));
- return -EINVAL;
+ return err;
}
static int safexcel_context_control(struct safexcel_cipher_ctx *ctx,
@@ -264,59 +410,78 @@ static int safexcel_context_control(struct safexcel_cipher_ctx *ctx,
struct safexcel_command_desc *cdesc)
{
struct safexcel_crypto_priv *priv = ctx->priv;
- int ctrl_size;
+ int ctrl_size = ctx->key_len / sizeof(u32);
+
+ cdesc->control_data.control1 = ctx->mode;
if (ctx->aead) {
+ /* Take in account the ipad+opad digests */
+ if (ctx->xcm) {
+ ctrl_size += ctx->state_sz / sizeof(u32);
+ cdesc->control_data.control0 =
+ CONTEXT_CONTROL_KEY_EN |
+ CONTEXT_CONTROL_DIGEST_XCM |
+ ctx->hash_alg |
+ CONTEXT_CONTROL_SIZE(ctrl_size);
+ } else {
+ ctrl_size += ctx->state_sz / sizeof(u32) * 2;
+ cdesc->control_data.control0 =
+ CONTEXT_CONTROL_KEY_EN |
+ CONTEXT_CONTROL_DIGEST_HMAC |
+ ctx->hash_alg |
+ CONTEXT_CONTROL_SIZE(ctrl_size);
+ }
if (sreq->direction == SAFEXCEL_ENCRYPT)
- cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT;
+ cdesc->control_data.control0 |=
+ (ctx->xcm == EIP197_XCM_MODE_CCM) ?
+ CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT :
+ CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT;
+
else
- cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN;
+ cdesc->control_data.control0 |=
+ (ctx->xcm == EIP197_XCM_MODE_CCM) ?
+ CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN :
+ CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN;
} else {
- cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_CRYPTO_OUT;
-
- /* The decryption control type is a combination of the
- * encryption type and CONTEXT_CONTROL_TYPE_NULL_IN, for all
- * types.
- */
- if (sreq->direction == SAFEXCEL_DECRYPT)
- cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_NULL_IN;
+ if (sreq->direction == SAFEXCEL_ENCRYPT)
+ cdesc->control_data.control0 =
+ CONTEXT_CONTROL_TYPE_CRYPTO_OUT |
+ CONTEXT_CONTROL_KEY_EN |
+ CONTEXT_CONTROL_SIZE(ctrl_size);
+ else
+ cdesc->control_data.control0 =
+ CONTEXT_CONTROL_TYPE_CRYPTO_IN |
+ CONTEXT_CONTROL_KEY_EN |
+ CONTEXT_CONTROL_SIZE(ctrl_size);
}
- cdesc->control_data.control0 |= CONTEXT_CONTROL_KEY_EN;
- cdesc->control_data.control1 |= ctx->mode;
-
- if (ctx->aead)
- cdesc->control_data.control0 |= CONTEXT_CONTROL_DIGEST_HMAC |
- ctx->hash_alg;
-
if (ctx->alg == SAFEXCEL_DES) {
- cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_DES;
+ cdesc->control_data.control0 |=
+ CONTEXT_CONTROL_CRYPTO_ALG_DES;
} else if (ctx->alg == SAFEXCEL_3DES) {
- cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_3DES;
+ cdesc->control_data.control0 |=
+ CONTEXT_CONTROL_CRYPTO_ALG_3DES;
} else if (ctx->alg == SAFEXCEL_AES) {
- switch (ctx->key_len) {
+ switch (ctx->key_len >> ctx->xts) {
case AES_KEYSIZE_128:
- cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES128;
+ cdesc->control_data.control0 |=
+ CONTEXT_CONTROL_CRYPTO_ALG_AES128;
break;
case AES_KEYSIZE_192:
- cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES192;
+ cdesc->control_data.control0 |=
+ CONTEXT_CONTROL_CRYPTO_ALG_AES192;
break;
case AES_KEYSIZE_256:
- cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES256;
+ cdesc->control_data.control0 |=
+ CONTEXT_CONTROL_CRYPTO_ALG_AES256;
break;
default:
dev_err(priv->dev, "aes keysize not supported: %u\n",
- ctx->key_len);
+ ctx->key_len >> ctx->xts);
return -EINVAL;
}
}
- ctrl_size = ctx->key_len / sizeof(u32);
- if (ctx->aead)
- /* Take in account the ipad+opad digests */
- ctrl_size += ctx->state_sz / sizeof(u32) * 2;
- cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(ctrl_size);
-
return 0;
}
@@ -328,12 +493,18 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
struct safexcel_cipher_req *sreq,
bool *should_complete, int *ret)
{
+ struct skcipher_request *areq = skcipher_request_cast(async);
+ struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq);
+ struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(skcipher);
struct safexcel_result_desc *rdesc;
int ndesc = 0;
*ret = 0;
- do {
+ if (unlikely(!sreq->rdescs))
+ return 0;
+
+ while (sreq->rdescs--) {
rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
if (IS_ERR(rdesc)) {
dev_err(priv->dev,
@@ -346,21 +517,27 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
*ret = safexcel_rdesc_check_errors(priv, rdesc);
ndesc++;
- } while (!rdesc->last_seg);
+ }
safexcel_complete(priv, ring);
if (src == dst) {
- dma_unmap_sg(priv->dev, src,
- sg_nents_for_len(src, cryptlen),
- DMA_BIDIRECTIONAL);
+ dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL);
} else {
- dma_unmap_sg(priv->dev, src,
- sg_nents_for_len(src, cryptlen),
- DMA_TO_DEVICE);
- dma_unmap_sg(priv->dev, dst,
- sg_nents_for_len(dst, cryptlen),
- DMA_FROM_DEVICE);
+ dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE);
+ dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE);
+ }
+
+ /*
+ * Update IV in req from last crypto output word for CBC modes
+ */
+ if ((!ctx->aead) && (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) &&
+ (sreq->direction == SAFEXCEL_ENCRYPT)) {
+ /* For encrypt take the last output word */
+ sg_pcopy_to_buffer(dst, sreq->nr_dst, areq->iv,
+ crypto_skcipher_ivsize(skcipher),
+ (cryptlen -
+ crypto_skcipher_ivsize(skcipher)));
}
*should_complete = true;
@@ -375,58 +552,105 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring,
unsigned int digestsize, u8 *iv, int *commands,
int *results)
{
+ struct skcipher_request *areq = skcipher_request_cast(base);
+ struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq);
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
struct safexcel_crypto_priv *priv = ctx->priv;
struct safexcel_command_desc *cdesc;
+ struct safexcel_command_desc *first_cdesc = NULL;
struct safexcel_result_desc *rdesc, *first_rdesc = NULL;
struct scatterlist *sg;
- unsigned int totlen = cryptlen + assoclen;
- int nr_src, nr_dst, n_cdesc = 0, n_rdesc = 0, queued = totlen;
- int i, ret = 0;
+ unsigned int totlen;
+ unsigned int totlen_src = cryptlen + assoclen;
+ unsigned int totlen_dst = totlen_src;
+ int n_cdesc = 0, n_rdesc = 0;
+ int queued, i, ret = 0;
+ bool first = true;
+
+ sreq->nr_src = sg_nents_for_len(src, totlen_src);
+
+ if (ctx->aead) {
+ /*
+ * AEAD has auth tag appended to output for encrypt and
+ * removed from the output for decrypt!
+ */
+ if (sreq->direction == SAFEXCEL_DECRYPT)
+ totlen_dst -= digestsize;
+ else
+ totlen_dst += digestsize;
+
+ memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32),
+ ctx->ipad, ctx->state_sz);
+ if (!ctx->xcm)
+ memcpy(ctx->base.ctxr->data + (ctx->key_len +
+ ctx->state_sz) / sizeof(u32), ctx->opad,
+ ctx->state_sz);
+ } else if ((ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) &&
+ (sreq->direction == SAFEXCEL_DECRYPT)) {
+ /*
+ * Save IV from last crypto input word for CBC modes in decrypt
+ * direction. Need to do this first in case of inplace operation
+ * as it will be overwritten.
+ */
+ sg_pcopy_to_buffer(src, sreq->nr_src, areq->iv,
+ crypto_skcipher_ivsize(skcipher),
+ (totlen_src -
+ crypto_skcipher_ivsize(skcipher)));
+ }
+
+ sreq->nr_dst = sg_nents_for_len(dst, totlen_dst);
+
+ /*
+ * Remember actual input length, source buffer length may be
+ * updated in case of inline operation below.
+ */
+ totlen = totlen_src;
+ queued = totlen_src;
if (src == dst) {
- nr_src = dma_map_sg(priv->dev, src,
- sg_nents_for_len(src, totlen),
- DMA_BIDIRECTIONAL);
- nr_dst = nr_src;
- if (!nr_src)
+ sreq->nr_src = max(sreq->nr_src, sreq->nr_dst);
+ sreq->nr_dst = sreq->nr_src;
+ if (unlikely((totlen_src || totlen_dst) &&
+ (sreq->nr_src <= 0))) {
+ dev_err(priv->dev, "In-place buffer not large enough (need %d bytes)!",
+ max(totlen_src, totlen_dst));
return -EINVAL;
+ }
+ dma_map_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL);
} else {
- nr_src = dma_map_sg(priv->dev, src,
- sg_nents_for_len(src, totlen),
- DMA_TO_DEVICE);
- if (!nr_src)
+ if (unlikely(totlen_src && (sreq->nr_src <= 0))) {
+ dev_err(priv->dev, "Source buffer not large enough (need %d bytes)!",
+ totlen_src);
return -EINVAL;
+ }
+ dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE);
- nr_dst = dma_map_sg(priv->dev, dst,
- sg_nents_for_len(dst, totlen),
- DMA_FROM_DEVICE);
- if (!nr_dst) {
- dma_unmap_sg(priv->dev, src,
- sg_nents_for_len(src, totlen),
+ if (unlikely(totlen_dst && (sreq->nr_dst <= 0))) {
+ dev_err(priv->dev, "Dest buffer not large enough (need %d bytes)!",
+ totlen_dst);
+ dma_unmap_sg(priv->dev, src, sreq->nr_src,
DMA_TO_DEVICE);
return -EINVAL;
}
+ dma_map_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE);
}
memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len);
- if (ctx->aead) {
- memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32),
- ctx->ipad, ctx->state_sz);
- memcpy(ctx->base.ctxr->data + (ctx->key_len + ctx->state_sz) / sizeof(u32),
- ctx->opad, ctx->state_sz);
- }
+ /* The EIP cannot deal with zero length input packets! */
+ if (totlen == 0)
+ totlen = 1;
/* command descriptors */
- for_each_sg(src, sg, nr_src, i) {
+ for_each_sg(src, sg, sreq->nr_src, i) {
int len = sg_dma_len(sg);
/* Do not overflow the request */
if (queued - len < 0)
len = queued;
- cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, !(queued - len),
+ cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
+ !(queued - len),
sg_dma_address(sg), len, totlen,
ctx->base.ctxr_dma);
if (IS_ERR(cdesc)) {
@@ -437,14 +661,7 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring,
n_cdesc++;
if (n_cdesc == 1) {
- safexcel_context_control(ctx, base, sreq, cdesc);
- if (ctx->aead)
- safexcel_aead_token(ctx, iv, cdesc,
- sreq->direction, cryptlen,
- assoclen, digestsize);
- else
- safexcel_skcipher_token(ctx, iv, cdesc,
- cryptlen);
+ first_cdesc = cdesc;
}
queued -= len;
@@ -452,23 +669,83 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring,
break;
}
+ if (unlikely(!n_cdesc)) {
+ /*
+ * Special case: zero length input buffer.
+ * The engine always needs the 1st command descriptor, however!
+ */
+ first_cdesc = safexcel_add_cdesc(priv, ring, 1, 1, 0, 0, totlen,
+ ctx->base.ctxr_dma);
+ n_cdesc = 1;
+ }
+
+ /* Add context control words and token to first command descriptor */
+ safexcel_context_control(ctx, base, sreq, first_cdesc);
+ if (ctx->aead)
+ safexcel_aead_token(ctx, iv, first_cdesc,
+ sreq->direction, cryptlen,
+ assoclen, digestsize);
+ else
+ safexcel_skcipher_token(ctx, iv, first_cdesc,
+ cryptlen);
+
/* result descriptors */
- for_each_sg(dst, sg, nr_dst, i) {
- bool first = !i, last = (i == nr_dst - 1);
+ for_each_sg(dst, sg, sreq->nr_dst, i) {
+ bool last = (i == sreq->nr_dst - 1);
u32 len = sg_dma_len(sg);
- rdesc = safexcel_add_rdesc(priv, ring, first, last,
- sg_dma_address(sg), len);
+ /* only allow the part of the buffer we know we need */
+ if (len > totlen_dst)
+ len = totlen_dst;
+ if (unlikely(!len))
+ break;
+ totlen_dst -= len;
+
+ /* skip over AAD space in buffer - not written */
+ if (assoclen) {
+ if (assoclen >= len) {
+ assoclen -= len;
+ continue;
+ }
+ rdesc = safexcel_add_rdesc(priv, ring, first, last,
+ sg_dma_address(sg) +
+ assoclen,
+ len - assoclen);
+ assoclen = 0;
+ } else {
+ rdesc = safexcel_add_rdesc(priv, ring, first, last,
+ sg_dma_address(sg),
+ len);
+ }
if (IS_ERR(rdesc)) {
/* No space left in the result descriptor ring */
ret = PTR_ERR(rdesc);
goto rdesc_rollback;
}
- if (first)
+ if (first) {
first_rdesc = rdesc;
+ first = false;
+ }
n_rdesc++;
}
+ if (unlikely(first)) {
+ /*
+ * Special case: AEAD decrypt with only AAD data.
+ * In this case there is NO output data from the engine,
+ * but the engine still needs a result descriptor!
+ * Create a dummy one just for catching the result token.
+ */
+ rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);
+ if (IS_ERR(rdesc)) {
+ /* No space left in the result descriptor ring */
+ ret = PTR_ERR(rdesc);
+ goto rdesc_rollback;
+ }
+ first_rdesc = rdesc;
+ n_rdesc = 1;
+ }
+
safexcel_rdr_req_set(priv, ring, first_rdesc, base);
*commands = n_cdesc;
@@ -483,16 +760,10 @@ cdesc_rollback:
safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
if (src == dst) {
- dma_unmap_sg(priv->dev, src,
- sg_nents_for_len(src, totlen),
- DMA_BIDIRECTIONAL);
+ dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL);
} else {
- dma_unmap_sg(priv->dev, src,
- sg_nents_for_len(src, totlen),
- DMA_TO_DEVICE);
- dma_unmap_sg(priv->dev, dst,
- sg_nents_for_len(dst, totlen),
- DMA_FROM_DEVICE);
+ dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE);
+ dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE);
}
return ret;
@@ -501,6 +772,7 @@ cdesc_rollback:
static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
int ring,
struct crypto_async_request *base,
+ struct safexcel_cipher_req *sreq,
bool *should_complete, int *ret)
{
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
@@ -509,7 +781,10 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
*ret = 0;
- do {
+ if (unlikely(!sreq->rdescs))
+ return 0;
+
+ while (sreq->rdescs--) {
rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
if (IS_ERR(rdesc)) {
dev_err(priv->dev,
@@ -522,7 +797,7 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
*ret = safexcel_rdesc_check_errors(priv, rdesc);
ndesc++;
- } while (!rdesc->last_seg);
+ }
safexcel_complete(priv, ring);
@@ -564,7 +839,7 @@ static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv,
if (sreq->needs_inv) {
sreq->needs_inv = false;
- err = safexcel_handle_inv_result(priv, ring, async,
+ err = safexcel_handle_inv_result(priv, ring, async, sreq,
should_complete, ret);
} else {
err = safexcel_handle_req_result(priv, ring, async, req->src,
@@ -587,7 +862,7 @@ static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv,
if (sreq->needs_inv) {
sreq->needs_inv = false;
- err = safexcel_handle_inv_result(priv, ring, async,
+ err = safexcel_handle_inv_result(priv, ring, async, sreq,
should_complete, ret);
} else {
err = safexcel_handle_req_result(priv, ring, async, req->src,
@@ -627,12 +902,24 @@ static int safexcel_skcipher_send(struct crypto_async_request *async, int ring,
BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv);
- if (sreq->needs_inv)
+ if (sreq->needs_inv) {
ret = safexcel_cipher_send_inv(async, ring, commands, results);
- else
+ } else {
+ struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
+ u8 input_iv[AES_BLOCK_SIZE];
+
+ /*
+ * Save input IV in case of CBC decrypt mode
+ * Will be overwritten with output IV prior to use!
+ */
+ memcpy(input_iv, req->iv, crypto_skcipher_ivsize(skcipher));
+
ret = safexcel_send_req(async, ring, sreq, req->src,
- req->dst, req->cryptlen, 0, 0, req->iv,
+ req->dst, req->cryptlen, 0, 0, input_iv,
commands, results);
+ }
+
+ sreq->rdescs = *results;
return ret;
}
@@ -655,6 +942,7 @@ static int safexcel_aead_send(struct crypto_async_request *async, int ring,
req->cryptlen, req->assoclen,
crypto_aead_authsize(tfm), req->iv,
commands, results);
+ sreq->rdescs = *results;
return ret;
}
@@ -724,8 +1012,7 @@ static int safexcel_aead_exit_inv(struct crypto_tfm *tfm)
static int safexcel_queue_req(struct crypto_async_request *base,
struct safexcel_cipher_req *sreq,
- enum safexcel_cipher_direction dir, u32 mode,
- enum safexcel_cipher_alg alg)
+ enum safexcel_cipher_direction dir)
{
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
struct safexcel_crypto_priv *priv = ctx->priv;
@@ -733,8 +1020,6 @@ static int safexcel_queue_req(struct crypto_async_request *base,
sreq->needs_inv = false;
sreq->direction = dir;
- ctx->alg = alg;
- ctx->mode = mode;
if (ctx->base.ctxr) {
if (priv->flags & EIP197_TRC_CACHE && ctx->base.needs_inv) {
@@ -762,18 +1047,16 @@ static int safexcel_queue_req(struct crypto_async_request *base,
return ret;
}
-static int safexcel_ecb_aes_encrypt(struct skcipher_request *req)
+static int safexcel_encrypt(struct skcipher_request *req)
{
return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
- SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
- SAFEXCEL_AES);
+ SAFEXCEL_ENCRYPT);
}
-static int safexcel_ecb_aes_decrypt(struct skcipher_request *req)
+static int safexcel_decrypt(struct skcipher_request *req)
{
return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
- SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
- SAFEXCEL_AES);
+ SAFEXCEL_DECRYPT);
}
static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm)
@@ -847,104 +1130,234 @@ static void safexcel_aead_cra_exit(struct crypto_tfm *tfm)
}
}
+static int safexcel_skcipher_aes_ecb_cra_init(struct crypto_tfm *tfm)
+{
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ safexcel_skcipher_cra_init(tfm);
+ ctx->alg = SAFEXCEL_AES;
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
+ return 0;
+}
+
struct safexcel_alg_template safexcel_alg_ecb_aes = {
.type = SAFEXCEL_ALG_TYPE_SKCIPHER,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_AES,
.alg.skcipher = {
.setkey = safexcel_skcipher_aes_setkey,
- .encrypt = safexcel_ecb_aes_encrypt,
- .decrypt = safexcel_ecb_aes_decrypt,
+ .encrypt = safexcel_encrypt,
+ .decrypt = safexcel_decrypt,
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.base = {
.cra_name = "ecb(aes)",
.cra_driver_name = "safexcel-ecb-aes",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
.cra_alignmask = 0,
- .cra_init = safexcel_skcipher_cra_init,
+ .cra_init = safexcel_skcipher_aes_ecb_cra_init,
.cra_exit = safexcel_skcipher_cra_exit,
.cra_module = THIS_MODULE,
},
},
};
-static int safexcel_cbc_aes_encrypt(struct skcipher_request *req)
+static int safexcel_skcipher_aes_cbc_cra_init(struct crypto_tfm *tfm)
{
- return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
- SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
- SAFEXCEL_AES);
-}
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
-static int safexcel_cbc_aes_decrypt(struct skcipher_request *req)
-{
- return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
- SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
- SAFEXCEL_AES);
+ safexcel_skcipher_cra_init(tfm);
+ ctx->alg = SAFEXCEL_AES;
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
+ return 0;
}
struct safexcel_alg_template safexcel_alg_cbc_aes = {
.type = SAFEXCEL_ALG_TYPE_SKCIPHER,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_AES,
.alg.skcipher = {
.setkey = safexcel_skcipher_aes_setkey,
- .encrypt = safexcel_cbc_aes_encrypt,
- .decrypt = safexcel_cbc_aes_decrypt,
+ .encrypt = safexcel_encrypt,
+ .decrypt = safexcel_decrypt,
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
.base = {
.cra_name = "cbc(aes)",
.cra_driver_name = "safexcel-cbc-aes",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
.cra_alignmask = 0,
- .cra_init = safexcel_skcipher_cra_init,
+ .cra_init = safexcel_skcipher_aes_cbc_cra_init,
.cra_exit = safexcel_skcipher_cra_exit,
.cra_module = THIS_MODULE,
},
},
};
-static int safexcel_cbc_des_encrypt(struct skcipher_request *req)
+static int safexcel_skcipher_aes_cfb_cra_init(struct crypto_tfm *tfm)
{
- return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
- SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
- SAFEXCEL_DES);
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ safexcel_skcipher_cra_init(tfm);
+ ctx->alg = SAFEXCEL_AES;
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB;
+ return 0;
}
-static int safexcel_cbc_des_decrypt(struct skcipher_request *req)
+struct safexcel_alg_template safexcel_alg_cfb_aes = {
+ .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
+ .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB,
+ .alg.skcipher = {
+ .setkey = safexcel_skcipher_aes_setkey,
+ .encrypt = safexcel_encrypt,
+ .decrypt = safexcel_decrypt,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .base = {
+ .cra_name = "cfb(aes)",
+ .cra_driver_name = "safexcel-cfb-aes",
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+ .cra_alignmask = 0,
+ .cra_init = safexcel_skcipher_aes_cfb_cra_init,
+ .cra_exit = safexcel_skcipher_cra_exit,
+ .cra_module = THIS_MODULE,
+ },
+ },
+};
+
+static int safexcel_skcipher_aes_ofb_cra_init(struct crypto_tfm *tfm)
{
- return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
- SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
- SAFEXCEL_DES);
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ safexcel_skcipher_cra_init(tfm);
+ ctx->alg = SAFEXCEL_AES;
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB;
+ return 0;
}
-static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key,
- unsigned int len)
+struct safexcel_alg_template safexcel_alg_ofb_aes = {
+ .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
+ .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB,
+ .alg.skcipher = {
+ .setkey = safexcel_skcipher_aes_setkey,
+ .encrypt = safexcel_encrypt,
+ .decrypt = safexcel_decrypt,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .base = {
+ .cra_name = "ofb(aes)",
+ .cra_driver_name = "safexcel-ofb-aes",
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+ .cra_alignmask = 0,
+ .cra_init = safexcel_skcipher_aes_ofb_cra_init,
+ .cra_exit = safexcel_skcipher_cra_exit,
+ .cra_module = THIS_MODULE,
+ },
+ },
+};
+
+static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm,
+ const u8 *key, unsigned int len)
{
struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
- u32 tmp[DES_EXPKEY_WORDS];
- int ret;
+ struct safexcel_crypto_priv *priv = ctx->priv;
+ struct crypto_aes_ctx aes;
+ int ret, i;
+ unsigned int keylen;
- if (len != DES_KEY_SIZE) {
+ /* last 4 bytes of key are the nonce! */
+ ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE);
+ /* exclude the nonce here */
+ keylen = len - CTR_RFC3686_NONCE_SIZE;
+ ret = aes_expandkey(&aes, key, keylen);
+ if (ret) {
crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
- return -EINVAL;
+ return ret;
}
- ret = des_ekey(tmp, key);
- if (!ret && (tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
- tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
- return -EINVAL;
+ if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
+ for (i = 0; i < keylen / sizeof(u32); i++) {
+ if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) {
+ ctx->base.needs_inv = true;
+ break;
+ }
+ }
}
+ for (i = 0; i < keylen / sizeof(u32); i++)
+ ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
+
+ ctx->key_len = keylen;
+
+ memzero_explicit(&aes, sizeof(aes));
+ return 0;
+}
+
+static int safexcel_skcipher_aes_ctr_cra_init(struct crypto_tfm *tfm)
+{
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ safexcel_skcipher_cra_init(tfm);
+ ctx->alg = SAFEXCEL_AES;
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD;
+ return 0;
+}
+
+struct safexcel_alg_template safexcel_alg_ctr_aes = {
+ .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
+ .algo_mask = SAFEXCEL_ALG_AES,
+ .alg.skcipher = {
+ .setkey = safexcel_skcipher_aesctr_setkey,
+ .encrypt = safexcel_encrypt,
+ .decrypt = safexcel_decrypt,
+ /* Add nonce size */
+ .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
+ .ivsize = CTR_RFC3686_IV_SIZE,
+ .base = {
+ .cra_name = "rfc3686(ctr(aes))",
+ .cra_driver_name = "safexcel-ctr-aes",
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+ .cra_alignmask = 0,
+ .cra_init = safexcel_skcipher_aes_ctr_cra_init,
+ .cra_exit = safexcel_skcipher_cra_exit,
+ .cra_module = THIS_MODULE,
+ },
+ },
+};
+
+static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key,
+ unsigned int len)
+{
+ struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm);
+ int ret;
+
+ ret = verify_skcipher_des_key(ctfm, key);
+ if (ret)
+ return ret;
+
/* if context exits and key changed, need to invalidate it */
if (ctx->base.ctxr_dma)
if (memcmp(ctx->key, key, len))
@@ -956,94 +1369,85 @@ static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key,
return 0;
}
+static int safexcel_skcipher_des_cbc_cra_init(struct crypto_tfm *tfm)
+{
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ safexcel_skcipher_cra_init(tfm);
+ ctx->alg = SAFEXCEL_DES;
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
+ return 0;
+}
+
struct safexcel_alg_template safexcel_alg_cbc_des = {
.type = SAFEXCEL_ALG_TYPE_SKCIPHER,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_DES,
.alg.skcipher = {
.setkey = safexcel_des_setkey,
- .encrypt = safexcel_cbc_des_encrypt,
- .decrypt = safexcel_cbc_des_decrypt,
+ .encrypt = safexcel_encrypt,
+ .decrypt = safexcel_decrypt,
.min_keysize = DES_KEY_SIZE,
.max_keysize = DES_KEY_SIZE,
.ivsize = DES_BLOCK_SIZE,
.base = {
.cra_name = "cbc(des)",
.cra_driver_name = "safexcel-cbc-des",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = DES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
.cra_alignmask = 0,
- .cra_init = safexcel_skcipher_cra_init,
+ .cra_init = safexcel_skcipher_des_cbc_cra_init,
.cra_exit = safexcel_skcipher_cra_exit,
.cra_module = THIS_MODULE,
},
},
};
-static int safexcel_ecb_des_encrypt(struct skcipher_request *req)
+static int safexcel_skcipher_des_ecb_cra_init(struct crypto_tfm *tfm)
{
- return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
- SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
- SAFEXCEL_DES);
-}
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
-static int safexcel_ecb_des_decrypt(struct skcipher_request *req)
-{
- return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
- SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
- SAFEXCEL_DES);
+ safexcel_skcipher_cra_init(tfm);
+ ctx->alg = SAFEXCEL_DES;
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
+ return 0;
}
struct safexcel_alg_template safexcel_alg_ecb_des = {
.type = SAFEXCEL_ALG_TYPE_SKCIPHER,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_DES,
.alg.skcipher = {
.setkey = safexcel_des_setkey,
- .encrypt = safexcel_ecb_des_encrypt,
- .decrypt = safexcel_ecb_des_decrypt,
+ .encrypt = safexcel_encrypt,
+ .decrypt = safexcel_decrypt,
.min_keysize = DES_KEY_SIZE,
.max_keysize = DES_KEY_SIZE,
- .ivsize = DES_BLOCK_SIZE,
.base = {
.cra_name = "ecb(des)",
.cra_driver_name = "safexcel-ecb-des",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = DES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
.cra_alignmask = 0,
- .cra_init = safexcel_skcipher_cra_init,
+ .cra_init = safexcel_skcipher_des_ecb_cra_init,
.cra_exit = safexcel_skcipher_cra_exit,
.cra_module = THIS_MODULE,
},
},
};
-static int safexcel_cbc_des3_ede_encrypt(struct skcipher_request *req)
-{
- return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
- SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
- SAFEXCEL_3DES);
-}
-
-static int safexcel_cbc_des3_ede_decrypt(struct skcipher_request *req)
-{
- return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
- SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
- SAFEXCEL_3DES);
-}
-
static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm,
const u8 *key, unsigned int len)
{
struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm);
int err;
- err = des3_verify_key(ctfm, key);
- if (unlikely(err))
+ err = verify_skcipher_des3_key(ctfm, key);
+ if (err)
return err;
/* if context exits and key changed, need to invalidate it */
@@ -1059,66 +1463,71 @@ static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm,
return 0;
}
+static int safexcel_skcipher_des3_cbc_cra_init(struct crypto_tfm *tfm)
+{
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ safexcel_skcipher_cra_init(tfm);
+ ctx->alg = SAFEXCEL_3DES;
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
+ return 0;
+}
+
struct safexcel_alg_template safexcel_alg_cbc_des3_ede = {
.type = SAFEXCEL_ALG_TYPE_SKCIPHER,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_DES,
.alg.skcipher = {
.setkey = safexcel_des3_ede_setkey,
- .encrypt = safexcel_cbc_des3_ede_encrypt,
- .decrypt = safexcel_cbc_des3_ede_decrypt,
+ .encrypt = safexcel_encrypt,
+ .decrypt = safexcel_decrypt,
.min_keysize = DES3_EDE_KEY_SIZE,
.max_keysize = DES3_EDE_KEY_SIZE,
.ivsize = DES3_EDE_BLOCK_SIZE,
.base = {
.cra_name = "cbc(des3_ede)",
.cra_driver_name = "safexcel-cbc-des3_ede",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = DES3_EDE_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
.cra_alignmask = 0,
- .cra_init = safexcel_skcipher_cra_init,
+ .cra_init = safexcel_skcipher_des3_cbc_cra_init,
.cra_exit = safexcel_skcipher_cra_exit,
.cra_module = THIS_MODULE,
},
},
};
-static int safexcel_ecb_des3_ede_encrypt(struct skcipher_request *req)
+static int safexcel_skcipher_des3_ecb_cra_init(struct crypto_tfm *tfm)
{
- return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
- SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
- SAFEXCEL_3DES);
-}
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
-static int safexcel_ecb_des3_ede_decrypt(struct skcipher_request *req)
-{
- return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
- SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
- SAFEXCEL_3DES);
+ safexcel_skcipher_cra_init(tfm);
+ ctx->alg = SAFEXCEL_3DES;
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
+ return 0;
}
struct safexcel_alg_template safexcel_alg_ecb_des3_ede = {
.type = SAFEXCEL_ALG_TYPE_SKCIPHER,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_DES,
.alg.skcipher = {
.setkey = safexcel_des3_ede_setkey,
- .encrypt = safexcel_ecb_des3_ede_encrypt,
- .decrypt = safexcel_ecb_des3_ede_decrypt,
+ .encrypt = safexcel_encrypt,
+ .decrypt = safexcel_decrypt,
.min_keysize = DES3_EDE_KEY_SIZE,
.max_keysize = DES3_EDE_KEY_SIZE,
- .ivsize = DES3_EDE_BLOCK_SIZE,
.base = {
.cra_name = "ecb(des3_ede)",
.cra_driver_name = "safexcel-ecb-des3_ede",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = DES3_EDE_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
.cra_alignmask = 0,
- .cra_init = safexcel_skcipher_cra_init,
+ .cra_init = safexcel_skcipher_des3_ecb_cra_init,
.cra_exit = safexcel_skcipher_cra_exit,
.cra_module = THIS_MODULE,
},
@@ -1129,16 +1538,14 @@ static int safexcel_aead_encrypt(struct aead_request *req)
{
struct safexcel_cipher_req *creq = aead_request_ctx(req);
- return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT,
- CONTEXT_CONTROL_CRYPTO_MODE_CBC, SAFEXCEL_AES);
+ return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT);
}
static int safexcel_aead_decrypt(struct aead_request *req)
{
struct safexcel_cipher_req *creq = aead_request_ctx(req);
- return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT,
- CONTEXT_CONTROL_CRYPTO_MODE_CBC, SAFEXCEL_AES);
+ return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT);
}
static int safexcel_aead_cra_init(struct crypto_tfm *tfm)
@@ -1153,6 +1560,8 @@ static int safexcel_aead_cra_init(struct crypto_tfm *tfm)
ctx->priv = tmpl->priv;
+ ctx->alg = SAFEXCEL_AES; /* default */
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; /* default */
ctx->aead = true;
ctx->base.send = safexcel_aead_send;
ctx->base.handle_result = safexcel_aead_handle_result;
@@ -1171,9 +1580,9 @@ static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm)
struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = {
.type = SAFEXCEL_ALG_TYPE_AEAD,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1,
.alg.aead = {
- .setkey = safexcel_aead_aes_setkey,
+ .setkey = safexcel_aead_setkey,
.encrypt = safexcel_aead_encrypt,
.decrypt = safexcel_aead_decrypt,
.ivsize = AES_BLOCK_SIZE,
@@ -1181,7 +1590,7 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = {
.base = {
.cra_name = "authenc(hmac(sha1),cbc(aes))",
.cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
@@ -1206,9 +1615,9 @@ static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm)
struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = {
.type = SAFEXCEL_ALG_TYPE_AEAD,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
.alg.aead = {
- .setkey = safexcel_aead_aes_setkey,
+ .setkey = safexcel_aead_setkey,
.encrypt = safexcel_aead_encrypt,
.decrypt = safexcel_aead_decrypt,
.ivsize = AES_BLOCK_SIZE,
@@ -1216,7 +1625,7 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = {
.base = {
.cra_name = "authenc(hmac(sha256),cbc(aes))",
.cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
@@ -1241,9 +1650,9 @@ static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm)
struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = {
.type = SAFEXCEL_ALG_TYPE_AEAD,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
.alg.aead = {
- .setkey = safexcel_aead_aes_setkey,
+ .setkey = safexcel_aead_setkey,
.encrypt = safexcel_aead_encrypt,
.decrypt = safexcel_aead_decrypt,
.ivsize = AES_BLOCK_SIZE,
@@ -1251,7 +1660,7 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = {
.base = {
.cra_name = "authenc(hmac(sha224),cbc(aes))",
.cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
@@ -1276,9 +1685,9 @@ static int safexcel_aead_sha512_cra_init(struct crypto_tfm *tfm)
struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = {
.type = SAFEXCEL_ALG_TYPE_AEAD,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
.alg.aead = {
- .setkey = safexcel_aead_aes_setkey,
+ .setkey = safexcel_aead_setkey,
.encrypt = safexcel_aead_encrypt,
.decrypt = safexcel_aead_decrypt,
.ivsize = AES_BLOCK_SIZE,
@@ -1286,7 +1695,7 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = {
.base = {
.cra_name = "authenc(hmac(sha512),cbc(aes))",
.cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-aes",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
@@ -1311,9 +1720,9 @@ static int safexcel_aead_sha384_cra_init(struct crypto_tfm *tfm)
struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = {
.type = SAFEXCEL_ALG_TYPE_AEAD,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
.alg.aead = {
- .setkey = safexcel_aead_aes_setkey,
+ .setkey = safexcel_aead_setkey,
.encrypt = safexcel_aead_encrypt,
.decrypt = safexcel_aead_decrypt,
.ivsize = AES_BLOCK_SIZE,
@@ -1321,7 +1730,7 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = {
.base = {
.cra_name = "authenc(hmac(sha384),cbc(aes))",
.cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-aes",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
@@ -1333,3 +1742,564 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = {
},
},
};
+
+static int safexcel_aead_sha1_des3_cra_init(struct crypto_tfm *tfm)
+{
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ safexcel_aead_sha1_cra_init(tfm);
+ ctx->alg = SAFEXCEL_3DES; /* override default */
+ return 0;
+}
+
+struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede = {
+ .type = SAFEXCEL_ALG_TYPE_AEAD,
+ .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1,
+ .alg.aead = {
+ .setkey = safexcel_aead_setkey,
+ .encrypt = safexcel_aead_encrypt,
+ .decrypt = safexcel_aead_decrypt,
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .maxauthsize = SHA1_DIGEST_SIZE,
+ .base = {
+ .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
+ .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des3_ede",
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+ .cra_alignmask = 0,
+ .cra_init = safexcel_aead_sha1_des3_cra_init,
+ .cra_exit = safexcel_aead_cra_exit,
+ .cra_module = THIS_MODULE,
+ },
+ },
+};
+
+static int safexcel_aead_sha1_ctr_cra_init(struct crypto_tfm *tfm)
+{
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ safexcel_aead_sha1_cra_init(tfm);
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
+ return 0;
+}
+
+struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_aes = {
+ .type = SAFEXCEL_ALG_TYPE_AEAD,
+ .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1,
+ .alg.aead = {
+ .setkey = safexcel_aead_setkey,
+ .encrypt = safexcel_aead_encrypt,
+ .decrypt = safexcel_aead_decrypt,
+ .ivsize = CTR_RFC3686_IV_SIZE,
+ .maxauthsize = SHA1_DIGEST_SIZE,
+ .base = {
+ .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
+ .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-aes",
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+ .cra_alignmask = 0,
+ .cra_init = safexcel_aead_sha1_ctr_cra_init,
+ .cra_exit = safexcel_aead_cra_exit,
+ .cra_module = THIS_MODULE,
+ },
+ },
+};
+
+static int safexcel_aead_sha256_ctr_cra_init(struct crypto_tfm *tfm)
+{
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ safexcel_aead_sha256_cra_init(tfm);
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
+ return 0;
+}
+
+struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_ctr_aes = {
+ .type = SAFEXCEL_ALG_TYPE_AEAD,
+ .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
+ .alg.aead = {
+ .setkey = safexcel_aead_setkey,
+ .encrypt = safexcel_aead_encrypt,
+ .decrypt = safexcel_aead_decrypt,
+ .ivsize = CTR_RFC3686_IV_SIZE,
+ .maxauthsize = SHA256_DIGEST_SIZE,
+ .base = {
+ .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
+ .cra_driver_name = "safexcel-authenc-hmac-sha256-ctr-aes",
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+ .cra_alignmask = 0,
+ .cra_init = safexcel_aead_sha256_ctr_cra_init,
+ .cra_exit = safexcel_aead_cra_exit,
+ .cra_module = THIS_MODULE,
+ },
+ },
+};
+
+static int safexcel_aead_sha224_ctr_cra_init(struct crypto_tfm *tfm)
+{
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ safexcel_aead_sha224_cra_init(tfm);
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
+ return 0;
+}
+
+struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_ctr_aes = {
+ .type = SAFEXCEL_ALG_TYPE_AEAD,
+ .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
+ .alg.aead = {
+ .setkey = safexcel_aead_setkey,
+ .encrypt = safexcel_aead_encrypt,
+ .decrypt = safexcel_aead_decrypt,
+ .ivsize = CTR_RFC3686_IV_SIZE,
+ .maxauthsize = SHA224_DIGEST_SIZE,
+ .base = {
+ .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
+ .cra_driver_name = "safexcel-authenc-hmac-sha224-ctr-aes",
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+ .cra_alignmask = 0,
+ .cra_init = safexcel_aead_sha224_ctr_cra_init,
+ .cra_exit = safexcel_aead_cra_exit,
+ .cra_module = THIS_MODULE,
+ },
+ },
+};
+
+static int safexcel_aead_sha512_ctr_cra_init(struct crypto_tfm *tfm)
+{
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ safexcel_aead_sha512_cra_init(tfm);
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
+ return 0;
+}
+
+struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_ctr_aes = {
+ .type = SAFEXCEL_ALG_TYPE_AEAD,
+ .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
+ .alg.aead = {
+ .setkey = safexcel_aead_setkey,
+ .encrypt = safexcel_aead_encrypt,
+ .decrypt = safexcel_aead_decrypt,
+ .ivsize = CTR_RFC3686_IV_SIZE,
+ .maxauthsize = SHA512_DIGEST_SIZE,
+ .base = {
+ .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
+ .cra_driver_name = "safexcel-authenc-hmac-sha512-ctr-aes",
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+ .cra_alignmask = 0,
+ .cra_init = safexcel_aead_sha512_ctr_cra_init,
+ .cra_exit = safexcel_aead_cra_exit,
+ .cra_module = THIS_MODULE,
+ },
+ },
+};
+
+static int safexcel_aead_sha384_ctr_cra_init(struct crypto_tfm *tfm)
+{
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ safexcel_aead_sha384_cra_init(tfm);
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
+ return 0;
+}
+
+struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_ctr_aes = {
+ .type = SAFEXCEL_ALG_TYPE_AEAD,
+ .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
+ .alg.aead = {
+ .setkey = safexcel_aead_setkey,
+ .encrypt = safexcel_aead_encrypt,
+ .decrypt = safexcel_aead_decrypt,
+ .ivsize = CTR_RFC3686_IV_SIZE,
+ .maxauthsize = SHA384_DIGEST_SIZE,
+ .base = {
+ .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
+ .cra_driver_name = "safexcel-authenc-hmac-sha384-ctr-aes",
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+ .cra_alignmask = 0,
+ .cra_init = safexcel_aead_sha384_ctr_cra_init,
+ .cra_exit = safexcel_aead_cra_exit,
+ .cra_module = THIS_MODULE,
+ },
+ },
+};
+
+static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm,
+ const u8 *key, unsigned int len)
+{
+ struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct safexcel_crypto_priv *priv = ctx->priv;
+ struct crypto_aes_ctx aes;
+ int ret, i;
+ unsigned int keylen;
+
+ /* Check for illegal XTS keys */
+ ret = xts_verify_key(ctfm, key, len);
+ if (ret)
+ return ret;
+
+ /* Only half of the key data is cipher key */
+ keylen = (len >> 1);
+ ret = aes_expandkey(&aes, key, keylen);
+ if (ret) {
+ crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return ret;
+ }
+
+ if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
+ for (i = 0; i < keylen / sizeof(u32); i++) {
+ if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) {
+ ctx->base.needs_inv = true;
+ break;
+ }
+ }
+ }
+
+ for (i = 0; i < keylen / sizeof(u32); i++)
+ ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
+
+ /* The other half is the tweak key */
+ ret = aes_expandkey(&aes, (u8 *)(key + keylen), keylen);
+ if (ret) {
+ crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return ret;
+ }
+
+ if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
+ for (i = 0; i < keylen / sizeof(u32); i++) {
+ if (ctx->key[i + keylen / sizeof(u32)] !=
+ cpu_to_le32(aes.key_enc[i])) {
+ ctx->base.needs_inv = true;
+ break;
+ }
+ }
+ }
+
+ for (i = 0; i < keylen / sizeof(u32); i++)
+ ctx->key[i + keylen / sizeof(u32)] =
+ cpu_to_le32(aes.key_enc[i]);
+
+ ctx->key_len = keylen << 1;
+
+ memzero_explicit(&aes, sizeof(aes));
+ return 0;
+}
+
+static int safexcel_skcipher_aes_xts_cra_init(struct crypto_tfm *tfm)
+{
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ safexcel_skcipher_cra_init(tfm);
+ ctx->alg = SAFEXCEL_AES;
+ ctx->xts = 1;
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XTS;
+ return 0;
+}
+
+static int safexcel_encrypt_xts(struct skcipher_request *req)
+{
+ if (req->cryptlen < XTS_BLOCK_SIZE)
+ return -EINVAL;
+ return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
+ SAFEXCEL_ENCRYPT);
+}
+
+static int safexcel_decrypt_xts(struct skcipher_request *req)
+{
+ if (req->cryptlen < XTS_BLOCK_SIZE)
+ return -EINVAL;
+ return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
+ SAFEXCEL_DECRYPT);
+}
+
+struct safexcel_alg_template safexcel_alg_xts_aes = {
+ .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
+ .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XTS,
+ .alg.skcipher = {
+ .setkey = safexcel_skcipher_aesxts_setkey,
+ .encrypt = safexcel_encrypt_xts,
+ .decrypt = safexcel_decrypt_xts,
+ /* XTS actually uses 2 AES keys glued together */
+ .min_keysize = AES_MIN_KEY_SIZE * 2,
+ .max_keysize = AES_MAX_KEY_SIZE * 2,
+ .ivsize = XTS_BLOCK_SIZE,
+ .base = {
+ .cra_name = "xts(aes)",
+ .cra_driver_name = "safexcel-xts-aes",
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = XTS_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+ .cra_alignmask = 0,
+ .cra_init = safexcel_skcipher_aes_xts_cra_init,
+ .cra_exit = safexcel_skcipher_cra_exit,
+ .cra_module = THIS_MODULE,
+ },
+ },
+};
+
+static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key,
+ unsigned int len)
+{
+ struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct safexcel_crypto_priv *priv = ctx->priv;
+ struct crypto_aes_ctx aes;
+ u32 hashkey[AES_BLOCK_SIZE >> 2];
+ int ret, i;
+
+ ret = aes_expandkey(&aes, key, len);
+ if (ret) {
+ crypto_aead_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ memzero_explicit(&aes, sizeof(aes));
+ return ret;
+ }
+
+ if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
+ for (i = 0; i < len / sizeof(u32); i++) {
+ if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) {
+ ctx->base.needs_inv = true;
+ break;
+ }
+ }
+ }
+
+ for (i = 0; i < len / sizeof(u32); i++)
+ ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
+
+ ctx->key_len = len;
+
+ /* Compute hash key by encrypting zeroes with cipher key */
+ crypto_cipher_clear_flags(ctx->hkaes, CRYPTO_TFM_REQ_MASK);
+ crypto_cipher_set_flags(ctx->hkaes, crypto_aead_get_flags(ctfm) &
+ CRYPTO_TFM_REQ_MASK);
+ ret = crypto_cipher_setkey(ctx->hkaes, key, len);
+ crypto_aead_set_flags(ctfm, crypto_cipher_get_flags(ctx->hkaes) &
+ CRYPTO_TFM_RES_MASK);
+ if (ret)
+ return ret;
+
+ memset(hashkey, 0, AES_BLOCK_SIZE);
+ crypto_cipher_encrypt_one(ctx->hkaes, (u8 *)hashkey, (u8 *)hashkey);
+
+ if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
+ for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) {
+ if (ctx->ipad[i] != cpu_to_be32(hashkey[i])) {
+ ctx->base.needs_inv = true;
+ break;
+ }
+ }
+ }
+
+ for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++)
+ ctx->ipad[i] = cpu_to_be32(hashkey[i]);
+
+ memzero_explicit(hashkey, AES_BLOCK_SIZE);
+ memzero_explicit(&aes, sizeof(aes));
+ return 0;
+}
+
+static int safexcel_aead_gcm_cra_init(struct crypto_tfm *tfm)
+{
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ safexcel_aead_cra_init(tfm);
+ ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_GHASH;
+ ctx->state_sz = GHASH_BLOCK_SIZE;
+ ctx->xcm = EIP197_XCM_MODE_GCM;
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */
+
+ ctx->hkaes = crypto_alloc_cipher("aes", 0, 0);
+ if (IS_ERR(ctx->hkaes))
+ return PTR_ERR(ctx->hkaes);
+
+ return 0;
+}
+
+static void safexcel_aead_gcm_cra_exit(struct crypto_tfm *tfm)
+{
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ crypto_free_cipher(ctx->hkaes);
+ safexcel_aead_cra_exit(tfm);
+}
+
+static int safexcel_aead_gcm_setauthsize(struct crypto_aead *tfm,
+ unsigned int authsize)
+{
+ return crypto_gcm_check_authsize(authsize);
+}
+
+struct safexcel_alg_template safexcel_alg_gcm = {
+ .type = SAFEXCEL_ALG_TYPE_AEAD,
+ .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH,
+ .alg.aead = {
+ .setkey = safexcel_aead_gcm_setkey,
+ .setauthsize = safexcel_aead_gcm_setauthsize,
+ .encrypt = safexcel_aead_encrypt,
+ .decrypt = safexcel_aead_decrypt,
+ .ivsize = GCM_AES_IV_SIZE,
+ .maxauthsize = GHASH_DIGEST_SIZE,
+ .base = {
+ .cra_name = "gcm(aes)",
+ .cra_driver_name = "safexcel-gcm-aes",
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+ .cra_alignmask = 0,
+ .cra_init = safexcel_aead_gcm_cra_init,
+ .cra_exit = safexcel_aead_gcm_cra_exit,
+ .cra_module = THIS_MODULE,
+ },
+ },
+};
+
+static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key,
+ unsigned int len)
+{
+ struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct safexcel_crypto_priv *priv = ctx->priv;
+ struct crypto_aes_ctx aes;
+ int ret, i;
+
+ ret = aes_expandkey(&aes, key, len);
+ if (ret) {
+ crypto_aead_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ memzero_explicit(&aes, sizeof(aes));
+ return ret;
+ }
+
+ if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
+ for (i = 0; i < len / sizeof(u32); i++) {
+ if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) {
+ ctx->base.needs_inv = true;
+ break;
+ }
+ }
+ }
+
+ for (i = 0; i < len / sizeof(u32); i++) {
+ ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
+ ctx->ipad[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] =
+ cpu_to_be32(aes.key_enc[i]);
+ }
+
+ ctx->key_len = len;
+ ctx->state_sz = 2 * AES_BLOCK_SIZE + len;
+
+ if (len == AES_KEYSIZE_192)
+ ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
+ else if (len == AES_KEYSIZE_256)
+ ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
+ else
+ ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
+
+ memzero_explicit(&aes, sizeof(aes));
+ return 0;
+}
+
+static int safexcel_aead_ccm_cra_init(struct crypto_tfm *tfm)
+{
+ struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ safexcel_aead_cra_init(tfm);
+ ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
+ ctx->state_sz = 3 * AES_BLOCK_SIZE;
+ ctx->xcm = EIP197_XCM_MODE_CCM;
+ ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */
+ return 0;
+}
+
+static int safexcel_aead_ccm_setauthsize(struct crypto_aead *tfm,
+ unsigned int authsize)
+{
+ /* Borrowed from crypto/ccm.c */
+ switch (authsize) {
+ case 4:
+ case 6:
+ case 8:
+ case 10:
+ case 12:
+ case 14:
+ case 16:
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int safexcel_ccm_encrypt(struct aead_request *req)
+{
+ struct safexcel_cipher_req *creq = aead_request_ctx(req);
+
+ if (req->iv[0] < 1 || req->iv[0] > 7)
+ return -EINVAL;
+
+ return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT);
+}
+
+static int safexcel_ccm_decrypt(struct aead_request *req)
+{
+ struct safexcel_cipher_req *creq = aead_request_ctx(req);
+
+ if (req->iv[0] < 1 || req->iv[0] > 7)
+ return -EINVAL;
+
+ return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT);
+}
+
+struct safexcel_alg_template safexcel_alg_ccm = {
+ .type = SAFEXCEL_ALG_TYPE_AEAD,
+ .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL,
+ .alg.aead = {
+ .setkey = safexcel_aead_ccm_setkey,
+ .setauthsize = safexcel_aead_ccm_setauthsize,
+ .encrypt = safexcel_ccm_encrypt,
+ .decrypt = safexcel_ccm_decrypt,
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = AES_BLOCK_SIZE,
+ .base = {
+ .cra_name = "ccm(aes)",
+ .cra_driver_name = "safexcel-ccm-aes",
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+ .cra_alignmask = 0,
+ .cra_init = safexcel_aead_ccm_cra_init,
+ .cra_exit = safexcel_aead_cra_exit,
+ .cra_module = THIS_MODULE,
+ },
+ },
+};
diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c
index ac9282c1a5ec..2effb6d21e8b 100644
--- a/drivers/crypto/inside-secure/safexcel_hash.c
+++ b/drivers/crypto/inside-secure/safexcel_hash.c
@@ -29,31 +29,31 @@ struct safexcel_ahash_req {
bool finish;
bool hmac;
bool needs_inv;
+ bool hmac_zlen;
+ bool len_is_le;
int nents;
dma_addr_t result_dma;
u32 digest;
- u8 state_sz; /* expected sate size, only set once */
+ u8 state_sz; /* expected state size, only set once */
+ u8 block_sz; /* block size, only set once */
u32 state[SHA512_DIGEST_SIZE / sizeof(u32)] __aligned(sizeof(u32));
- u64 len[2];
- u64 processed[2];
+ u64 len;
+ u64 processed;
- u8 cache[SHA512_BLOCK_SIZE] __aligned(sizeof(u32));
+ u8 cache[HASH_CACHE_SIZE] __aligned(sizeof(u32));
dma_addr_t cache_dma;
unsigned int cache_sz;
- u8 cache_next[SHA512_BLOCK_SIZE] __aligned(sizeof(u32));
+ u8 cache_next[HASH_CACHE_SIZE] __aligned(sizeof(u32));
};
static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req)
{
- if (req->len[1] > req->processed[1])
- return 0xffffffff - (req->len[0] - req->processed[0]);
-
- return req->len[0] - req->processed[0];
+ return req->len - req->processed;
}
static void safexcel_hash_token(struct safexcel_command_desc *cdesc,
@@ -77,75 +77,104 @@ static void safexcel_hash_token(struct safexcel_command_desc *cdesc,
static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
struct safexcel_ahash_req *req,
- struct safexcel_command_desc *cdesc,
- unsigned int digestsize)
+ struct safexcel_command_desc *cdesc)
{
struct safexcel_crypto_priv *priv = ctx->priv;
- int i;
+ u64 count = 0;
- cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_HASH_OUT;
cdesc->control_data.control0 |= ctx->alg;
- cdesc->control_data.control0 |= req->digest;
-
- if (req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) {
- if (req->processed[0] || req->processed[1]) {
- if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5)
- cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(5);
- else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
- cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(6);
- else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224 ||
- ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256)
- cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(9);
- else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384 ||
- ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512)
- cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(17);
-
- cdesc->control_data.control1 |= CONTEXT_CONTROL_DIGEST_CNT;
+
+ /*
+ * Copy the input digest if needed, and setup the context
+ * fields. Do this now as we need it to setup the first command
+ * descriptor.
+ */
+ if (!req->processed) {
+ /* First - and possibly only - block of basic hash only */
+ if (req->finish) {
+ cdesc->control_data.control0 |=
+ CONTEXT_CONTROL_TYPE_HASH_OUT |
+ CONTEXT_CONTROL_RESTART_HASH |
+ /* ensure its not 0! */
+ CONTEXT_CONTROL_SIZE(1);
} else {
- cdesc->control_data.control0 |= CONTEXT_CONTROL_RESTART_HASH;
+ cdesc->control_data.control0 |=
+ CONTEXT_CONTROL_TYPE_HASH_OUT |
+ CONTEXT_CONTROL_RESTART_HASH |
+ CONTEXT_CONTROL_NO_FINISH_HASH |
+ /* ensure its not 0! */
+ CONTEXT_CONTROL_SIZE(1);
}
+ return;
+ }
- if (!req->finish)
- cdesc->control_data.control0 |= CONTEXT_CONTROL_NO_FINISH_HASH;
+ /* Hash continuation or HMAC, setup (inner) digest from state */
+ memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
- /*
- * Copy the input digest if needed, and setup the context
- * fields. Do this now as we need it to setup the first command
- * descriptor.
- */
- if (req->processed[0] || req->processed[1]) {
- for (i = 0; i < digestsize / sizeof(u32); i++)
- ctx->base.ctxr->data[i] = cpu_to_le32(req->state[i]);
-
- if (req->finish) {
- u64 count = req->processed[0] / EIP197_COUNTER_BLOCK_SIZE;
- count += ((0xffffffff / EIP197_COUNTER_BLOCK_SIZE) *
- req->processed[1]);
-
- /* This is a haredware limitation, as the
- * counter must fit into an u32. This represents
- * a farily big amount of input data, so we
- * shouldn't see this.
- */
- if (unlikely(count & 0xffff0000)) {
- dev_warn(priv->dev,
- "Input data is too big\n");
- return;
- }
-
- ctx->base.ctxr->data[i] = cpu_to_le32(count);
+ if (req->finish) {
+ /* Compute digest count for hash/HMAC finish operations */
+ if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
+ req->hmac_zlen || (req->processed != req->block_sz)) {
+ count = req->processed / EIP197_COUNTER_BLOCK_SIZE;
+
+ /* This is a hardware limitation, as the
+ * counter must fit into an u32. This represents
+ * a fairly big amount of input data, so we
+ * shouldn't see this.
+ */
+ if (unlikely(count & 0xffffffff00000000ULL)) {
+ dev_warn(priv->dev,
+ "Input data is too big\n");
+ return;
}
}
- } else if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) {
- cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(2 * req->state_sz / sizeof(u32));
- memcpy(ctx->base.ctxr->data, ctx->ipad, req->state_sz);
- memcpy(ctx->base.ctxr->data + req->state_sz / sizeof(u32),
- ctx->opad, req->state_sz);
+ if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
+ /* Special case: zero length HMAC */
+ req->hmac_zlen ||
+ /* PE HW < 4.4 cannot do HMAC continue, fake using hash */
+ (req->processed != req->block_sz)) {
+ /* Basic hash continue operation, need digest + cnt */
+ cdesc->control_data.control0 |=
+ CONTEXT_CONTROL_SIZE((req->state_sz >> 2) + 1) |
+ CONTEXT_CONTROL_TYPE_HASH_OUT |
+ CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
+ /* For zero-len HMAC, don't finalize, already padded! */
+ if (req->hmac_zlen)
+ cdesc->control_data.control0 |=
+ CONTEXT_CONTROL_NO_FINISH_HASH;
+ cdesc->control_data.control1 |=
+ CONTEXT_CONTROL_DIGEST_CNT;
+ ctx->base.ctxr->data[req->state_sz >> 2] =
+ cpu_to_le32(count);
+ req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
+
+ /* Clear zero-length HMAC flag for next operation! */
+ req->hmac_zlen = false;
+ } else { /* HMAC */
+ /* Need outer digest for HMAC finalization */
+ memcpy(ctx->base.ctxr->data + (req->state_sz >> 2),
+ ctx->opad, req->state_sz);
+
+ /* Single pass HMAC - no digest count */
+ cdesc->control_data.control0 |=
+ CONTEXT_CONTROL_SIZE(req->state_sz >> 1) |
+ CONTEXT_CONTROL_TYPE_HASH_OUT |
+ CONTEXT_CONTROL_DIGEST_HMAC;
+ }
+ } else { /* Hash continuation, do not finish yet */
+ cdesc->control_data.control0 |=
+ CONTEXT_CONTROL_SIZE(req->state_sz >> 2) |
+ CONTEXT_CONTROL_DIGEST_PRECOMPUTED |
+ CONTEXT_CONTROL_TYPE_HASH_OUT |
+ CONTEXT_CONTROL_NO_FINISH_HASH;
}
}
-static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring,
+static int safexcel_ahash_enqueue(struct ahash_request *areq);
+
+static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv,
+ int ring,
struct crypto_async_request *async,
bool *should_complete, int *ret)
{
@@ -153,6 +182,7 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
struct ahash_request *areq = ahash_request_cast(async);
struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
struct safexcel_ahash_req *sreq = ahash_request_ctx(areq);
+ struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
u64 cache_len;
*ret = 0;
@@ -183,11 +213,34 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz,
DMA_TO_DEVICE);
sreq->cache_dma = 0;
+ sreq->cache_sz = 0;
}
- if (sreq->finish)
+ if (sreq->finish) {
+ if (sreq->hmac &&
+ (sreq->digest != CONTEXT_CONTROL_DIGEST_HMAC)) {
+ /* Faking HMAC using hash - need to do outer hash */
+ memcpy(sreq->cache, sreq->state,
+ crypto_ahash_digestsize(ahash));
+
+ memcpy(sreq->state, ctx->opad, sreq->state_sz);
+
+ sreq->len = sreq->block_sz +
+ crypto_ahash_digestsize(ahash);
+ sreq->processed = sreq->block_sz;
+ sreq->hmac = 0;
+
+ ctx->base.needs_inv = true;
+ areq->nbytes = 0;
+ safexcel_ahash_enqueue(areq);
+
+ *should_complete = false; /* Not done yet */
+ return 1;
+ }
+
memcpy(areq->result, sreq->state,
crypto_ahash_digestsize(ahash));
+ }
cache_len = safexcel_queued_len(sreq);
if (cache_len)
@@ -202,46 +255,44 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
int *commands, int *results)
{
struct ahash_request *areq = ahash_request_cast(async);
- struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
struct safexcel_crypto_priv *priv = ctx->priv;
struct safexcel_command_desc *cdesc, *first_cdesc = NULL;
struct safexcel_result_desc *rdesc;
struct scatterlist *sg;
- int i, extra, n_cdesc = 0, ret = 0;
+ int i, extra = 0, n_cdesc = 0, ret = 0;
u64 queued, len, cache_len;
queued = len = safexcel_queued_len(req);
- if (queued <= crypto_ahash_blocksize(ahash))
+ if (queued <= HASH_CACHE_SIZE)
cache_len = queued;
else
cache_len = queued - areq->nbytes;
- if (!req->last_req) {
+ if (!req->finish && !req->last_req) {
/* If this is not the last request and the queued data does not
- * fit into full blocks, cache it for the next send() call.
+ * fit into full cache blocks, cache it for the next send call.
+ */
+ extra = queued & (HASH_CACHE_SIZE - 1);
+
+ /* If this is not the last request and the queued data
+ * is a multiple of a block, cache the last one for now.
*/
- extra = queued & (crypto_ahash_blocksize(ahash) - 1);
if (!extra)
- /* If this is not the last request and the queued data
- * is a multiple of a block, cache the last one for now.
- */
- extra = crypto_ahash_blocksize(ahash);
+ extra = HASH_CACHE_SIZE;
- if (extra) {
- sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
- req->cache_next, extra,
- areq->nbytes - extra);
+ sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
+ req->cache_next, extra,
+ areq->nbytes - extra);
- queued -= extra;
- len -= extra;
+ queued -= extra;
+ len -= extra;
- if (!queued) {
- *commands = 0;
- *results = 0;
- return 0;
- }
+ if (!queued) {
+ *commands = 0;
+ *results = 0;
+ return 0;
}
}
@@ -268,9 +319,14 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
goto send_command;
}
+ /* Skip descriptor generation for zero-length requests */
+ if (!areq->nbytes)
+ goto send_command;
+
/* Now handle the current ahash request buffer(s) */
req->nents = dma_map_sg(priv->dev, areq->src,
- sg_nents_for_len(areq->src, areq->nbytes),
+ sg_nents_for_len(areq->src,
+ areq->nbytes),
DMA_TO_DEVICE);
if (!req->nents) {
ret = -ENOMEM;
@@ -285,7 +341,8 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
sglen = queued;
cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
- !(queued - sglen), sg_dma_address(sg),
+ !(queued - sglen),
+ sg_dma_address(sg),
sglen, len, ctx->base.ctxr_dma);
if (IS_ERR(cdesc)) {
ret = PTR_ERR(cdesc);
@@ -303,7 +360,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
send_command:
/* Setup the context options */
- safexcel_context_control(ctx, req, first_cdesc, req->state_sz);
+ safexcel_context_control(ctx, req, first_cdesc);
/* Add the token */
safexcel_hash_token(first_cdesc, len, req->state_sz);
@@ -325,9 +382,7 @@ send_command:
safexcel_rdr_req_set(priv, ring, rdesc, &areq->base);
- req->processed[0] += len;
- if (req->processed[0] < len)
- req->processed[1]++;
+ req->processed += len;
*commands = n_cdesc;
*results = 1;
@@ -345,33 +400,13 @@ unmap_cache:
if (req->cache_dma) {
dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz,
DMA_TO_DEVICE);
+ req->cache_dma = 0;
req->cache_sz = 0;
}
return ret;
}
-static inline bool safexcel_ahash_needs_inv_get(struct ahash_request *areq)
-{
- struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_ahash_req *req = ahash_request_ctx(areq);
- unsigned int state_w_sz = req->state_sz / sizeof(u32);
- u64 processed;
- int i;
-
- processed = req->processed[0] / EIP197_COUNTER_BLOCK_SIZE;
- processed += (0xffffffff / EIP197_COUNTER_BLOCK_SIZE) * req->processed[1];
-
- for (i = 0; i < state_w_sz; i++)
- if (ctx->base.ctxr->data[i] != cpu_to_le32(req->state[i]))
- return true;
-
- if (ctx->base.ctxr->data[state_w_sz] != cpu_to_le32(processed))
- return true;
-
- return false;
-}
-
static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
int ring,
struct crypto_async_request *async,
@@ -486,7 +521,7 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
struct safexcel_inv_result result = {};
int ring = ctx->base.ring;
- memset(req, 0, sizeof(struct ahash_request));
+ memset(req, 0, EIP197_AHASH_REQ_SIZE);
/* create invalidation request */
init_completion(&result.completion);
@@ -522,28 +557,22 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
static int safexcel_ahash_cache(struct ahash_request *areq)
{
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
- struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
- u64 queued, cache_len;
+ u64 cache_len;
- /* queued: everything accepted by the driver which will be handled by
- * the next send() calls.
- * tot sz handled by update() - tot sz handled by send()
- */
- queued = safexcel_queued_len(req);
/* cache_len: everything accepted by the driver but not sent yet,
* tot sz handled by update() - last req sz - tot sz handled by send()
*/
- cache_len = queued - areq->nbytes;
+ cache_len = safexcel_queued_len(req);
/*
* In case there isn't enough bytes to proceed (less than a
* block size), cache the data until we have enough.
*/
- if (cache_len + areq->nbytes <= crypto_ahash_blocksize(ahash)) {
+ if (cache_len + areq->nbytes <= HASH_CACHE_SIZE) {
sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
req->cache + cache_len,
areq->nbytes, 0);
- return areq->nbytes;
+ return 0;
}
/* We couldn't cache all the data */
@@ -561,14 +590,25 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq)
if (ctx->base.ctxr) {
if (priv->flags & EIP197_TRC_CACHE && !ctx->base.needs_inv &&
- (req->processed[0] || req->processed[1]) &&
- req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED)
- /* We're still setting needs_inv here, even though it is
+ req->processed &&
+ (/* invalidate for basic hash continuation finish */
+ (req->finish &&
+ (req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED)) ||
+ /* invalidate if (i)digest changed */
+ memcmp(ctx->base.ctxr->data, req->state, req->state_sz) ||
+ /* invalidate for HMAC continuation finish */
+ (req->finish && (req->processed != req->block_sz)) ||
+ /* invalidate for HMAC finish with odigest changed */
+ (req->finish &&
+ memcmp(ctx->base.ctxr->data + (req->state_sz>>2),
+ ctx->opad, req->state_sz))))
+ /*
+ * We're still setting needs_inv here, even though it is
* cleared right away, because the needs_inv flag can be
* set in other functions and we want to keep the same
* logic.
*/
- ctx->base.needs_inv = safexcel_ahash_needs_inv_get(areq);
+ ctx->base.needs_inv = true;
if (ctx->base.needs_inv) {
ctx->base.needs_inv = false;
@@ -598,30 +638,23 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq)
static int safexcel_ahash_update(struct ahash_request *areq)
{
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
- struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
+ int ret;
/* If the request is 0 length, do nothing */
if (!areq->nbytes)
return 0;
- req->len[0] += areq->nbytes;
- if (req->len[0] < areq->nbytes)
- req->len[1]++;
+ /* Add request to the cache if it fits */
+ ret = safexcel_ahash_cache(areq);
- safexcel_ahash_cache(areq);
+ /* Update total request length */
+ req->len += areq->nbytes;
- /*
- * We're not doing partial updates when performing an hmac request.
- * Everything will be handled by the final() call.
+ /* If not all data could fit into the cache, go process the excess.
+ * Also go process immediately for an HMAC IV precompute, which
+ * will never be finished at all, but needs to be processed anyway.
*/
- if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC)
- return 0;
-
- if (req->hmac)
- return safexcel_ahash_enqueue(areq);
-
- if (!req->last_req &&
- safexcel_queued_len(req) > crypto_ahash_blocksize(ahash))
+ if ((ret && !req->finish) || req->last_req)
return safexcel_ahash_enqueue(areq);
return 0;
@@ -632,11 +665,14 @@ static int safexcel_ahash_final(struct ahash_request *areq)
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- req->last_req = true;
req->finish = true;
- /* If we have an overall 0 length request */
- if (!req->len[0] && !req->len[1] && !areq->nbytes) {
+ if (unlikely(!req->len && !areq->nbytes)) {
+ /*
+ * If we have an overall 0 length *hash* request:
+ * The HW cannot do 0 length hash, so we provide the correct
+ * result directly here.
+ */
if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5)
memcpy(areq->result, md5_zero_message_hash,
MD5_DIGEST_SIZE);
@@ -657,6 +693,43 @@ static int safexcel_ahash_final(struct ahash_request *areq)
SHA512_DIGEST_SIZE);
return 0;
+ } else if (unlikely(req->hmac &&
+ (req->len == req->block_sz) &&
+ !areq->nbytes)) {
+ /*
+ * If we have an overall 0 length *HMAC* request:
+ * For HMAC, we need to finalize the inner digest
+ * and then perform the outer hash.
+ */
+
+ /* generate pad block in the cache */
+ /* start with a hash block of all zeroes */
+ memset(req->cache, 0, req->block_sz);
+ /* set the first byte to 0x80 to 'append a 1 bit' */
+ req->cache[0] = 0x80;
+ /* add the length in bits in the last 2 bytes */
+ if (req->len_is_le) {
+ /* Little endian length word (e.g. MD5) */
+ req->cache[req->block_sz-8] = (req->block_sz << 3) &
+ 255;
+ req->cache[req->block_sz-7] = (req->block_sz >> 5);
+ } else {
+ /* Big endian length word (e.g. any SHA) */
+ req->cache[req->block_sz-2] = (req->block_sz >> 5);
+ req->cache[req->block_sz-1] = (req->block_sz << 3) &
+ 255;
+ }
+
+ req->len += req->block_sz; /* plus 1 hash block */
+
+ /* Set special zero-length HMAC flag */
+ req->hmac_zlen = true;
+
+ /* Finalize HMAC */
+ req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
+ } else if (req->hmac) {
+ /* Finalize HMAC */
+ req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
}
return safexcel_ahash_enqueue(areq);
@@ -666,7 +739,6 @@ static int safexcel_ahash_finup(struct ahash_request *areq)
{
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
- req->last_req = true;
req->finish = true;
safexcel_ahash_update(areq);
@@ -675,26 +747,22 @@ static int safexcel_ahash_finup(struct ahash_request *areq)
static int safexcel_ahash_export(struct ahash_request *areq, void *out)
{
- struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
struct safexcel_ahash_export_state *export = out;
- export->len[0] = req->len[0];
- export->len[1] = req->len[1];
- export->processed[0] = req->processed[0];
- export->processed[1] = req->processed[1];
+ export->len = req->len;
+ export->processed = req->processed;
export->digest = req->digest;
memcpy(export->state, req->state, req->state_sz);
- memcpy(export->cache, req->cache, crypto_ahash_blocksize(ahash));
+ memcpy(export->cache, req->cache, HASH_CACHE_SIZE);
return 0;
}
static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
{
- struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
const struct safexcel_ahash_export_state *export = in;
int ret;
@@ -703,14 +771,12 @@ static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
if (ret)
return ret;
- req->len[0] = export->len[0];
- req->len[1] = export->len[1];
- req->processed[0] = export->processed[0];
- req->processed[1] = export->processed[1];
+ req->len = export->len;
+ req->processed = export->processed;
req->digest = export->digest;
- memcpy(req->cache, export->cache, crypto_ahash_blocksize(ahash));
+ memcpy(req->cache, export->cache, HASH_CACHE_SIZE);
memcpy(req->state, export->state, req->state_sz);
return 0;
@@ -739,15 +805,10 @@ static int safexcel_sha1_init(struct ahash_request *areq)
memset(req, 0, sizeof(*req));
- req->state[0] = SHA1_H0;
- req->state[1] = SHA1_H1;
- req->state[2] = SHA1_H2;
- req->state[3] = SHA1_H3;
- req->state[4] = SHA1_H4;
-
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
req->state_sz = SHA1_DIGEST_SIZE;
+ req->block_sz = SHA1_BLOCK_SIZE;
return 0;
}
@@ -784,7 +845,7 @@ static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm)
struct safexcel_alg_template safexcel_alg_sha1 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_SHA1,
.alg.ahash = {
.init = safexcel_sha1_init,
.update = safexcel_ahash_update,
@@ -799,7 +860,7 @@ struct safexcel_alg_template safexcel_alg_sha1 = {
.base = {
.cra_name = "sha1",
.cra_driver_name = "safexcel-sha1",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA1_BLOCK_SIZE,
@@ -814,10 +875,23 @@ struct safexcel_alg_template safexcel_alg_sha1 = {
static int safexcel_hmac_sha1_init(struct ahash_request *areq)
{
+ struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
- safexcel_sha1_init(areq);
- req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
+ memset(req, 0, sizeof(*req));
+
+ /* Start from ipad precompute */
+ memcpy(req->state, ctx->ipad, SHA1_DIGEST_SIZE);
+ /* Already processed the key^ipad part now! */
+ req->len = SHA1_BLOCK_SIZE;
+ req->processed = SHA1_BLOCK_SIZE;
+
+ ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
+ req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
+ req->state_sz = SHA1_DIGEST_SIZE;
+ req->block_sz = SHA1_BLOCK_SIZE;
+ req->hmac = true;
+
return 0;
}
@@ -986,21 +1060,16 @@ static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key,
struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
struct safexcel_crypto_priv *priv = ctx->priv;
struct safexcel_ahash_export_state istate, ostate;
- int ret, i;
+ int ret;
ret = safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate);
if (ret)
return ret;
- if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr) {
- for (i = 0; i < state_sz / sizeof(u32); i++) {
- if (ctx->ipad[i] != le32_to_cpu(istate.state[i]) ||
- ctx->opad[i] != le32_to_cpu(ostate.state[i])) {
- ctx->base.needs_inv = true;
- break;
- }
- }
- }
+ if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr &&
+ (memcmp(ctx->ipad, istate.state, state_sz) ||
+ memcmp(ctx->opad, ostate.state, state_sz)))
+ ctx->base.needs_inv = true;
memcpy(ctx->ipad, &istate.state, state_sz);
memcpy(ctx->opad, &ostate.state, state_sz);
@@ -1017,7 +1086,7 @@ static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
struct safexcel_alg_template safexcel_alg_hmac_sha1 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_SHA1,
.alg.ahash = {
.init = safexcel_hmac_sha1_init,
.update = safexcel_ahash_update,
@@ -1033,7 +1102,7 @@ struct safexcel_alg_template safexcel_alg_hmac_sha1 = {
.base = {
.cra_name = "hmac(sha1)",
.cra_driver_name = "safexcel-hmac-sha1",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA1_BLOCK_SIZE,
@@ -1053,18 +1122,10 @@ static int safexcel_sha256_init(struct ahash_request *areq)
memset(req, 0, sizeof(*req));
- req->state[0] = SHA256_H0;
- req->state[1] = SHA256_H1;
- req->state[2] = SHA256_H2;
- req->state[3] = SHA256_H3;
- req->state[4] = SHA256_H4;
- req->state[5] = SHA256_H5;
- req->state[6] = SHA256_H6;
- req->state[7] = SHA256_H7;
-
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
req->state_sz = SHA256_DIGEST_SIZE;
+ req->block_sz = SHA256_BLOCK_SIZE;
return 0;
}
@@ -1081,7 +1142,7 @@ static int safexcel_sha256_digest(struct ahash_request *areq)
struct safexcel_alg_template safexcel_alg_sha256 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_SHA2_256,
.alg.ahash = {
.init = safexcel_sha256_init,
.update = safexcel_ahash_update,
@@ -1096,7 +1157,7 @@ struct safexcel_alg_template safexcel_alg_sha256 = {
.base = {
.cra_name = "sha256",
.cra_driver_name = "safexcel-sha256",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA256_BLOCK_SIZE,
@@ -1116,18 +1177,10 @@ static int safexcel_sha224_init(struct ahash_request *areq)
memset(req, 0, sizeof(*req));
- req->state[0] = SHA224_H0;
- req->state[1] = SHA224_H1;
- req->state[2] = SHA224_H2;
- req->state[3] = SHA224_H3;
- req->state[4] = SHA224_H4;
- req->state[5] = SHA224_H5;
- req->state[6] = SHA224_H6;
- req->state[7] = SHA224_H7;
-
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
req->state_sz = SHA256_DIGEST_SIZE;
+ req->block_sz = SHA256_BLOCK_SIZE;
return 0;
}
@@ -1144,7 +1197,7 @@ static int safexcel_sha224_digest(struct ahash_request *areq)
struct safexcel_alg_template safexcel_alg_sha224 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_SHA2_256,
.alg.ahash = {
.init = safexcel_sha224_init,
.update = safexcel_ahash_update,
@@ -1159,7 +1212,7 @@ struct safexcel_alg_template safexcel_alg_sha224 = {
.base = {
.cra_name = "sha224",
.cra_driver_name = "safexcel-sha224",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA224_BLOCK_SIZE,
@@ -1181,10 +1234,23 @@ static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key,
static int safexcel_hmac_sha224_init(struct ahash_request *areq)
{
+ struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
- safexcel_sha224_init(areq);
- req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
+ memset(req, 0, sizeof(*req));
+
+ /* Start from ipad precompute */
+ memcpy(req->state, ctx->ipad, SHA256_DIGEST_SIZE);
+ /* Already processed the key^ipad part now! */
+ req->len = SHA256_BLOCK_SIZE;
+ req->processed = SHA256_BLOCK_SIZE;
+
+ ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
+ req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
+ req->state_sz = SHA256_DIGEST_SIZE;
+ req->block_sz = SHA256_BLOCK_SIZE;
+ req->hmac = true;
+
return 0;
}
@@ -1200,7 +1266,7 @@ static int safexcel_hmac_sha224_digest(struct ahash_request *areq)
struct safexcel_alg_template safexcel_alg_hmac_sha224 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_SHA2_256,
.alg.ahash = {
.init = safexcel_hmac_sha224_init,
.update = safexcel_ahash_update,
@@ -1216,7 +1282,7 @@ struct safexcel_alg_template safexcel_alg_hmac_sha224 = {
.base = {
.cra_name = "hmac(sha224)",
.cra_driver_name = "safexcel-hmac-sha224",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA224_BLOCK_SIZE,
@@ -1238,10 +1304,23 @@ static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key,
static int safexcel_hmac_sha256_init(struct ahash_request *areq)
{
+ struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
- safexcel_sha256_init(areq);
- req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
+ memset(req, 0, sizeof(*req));
+
+ /* Start from ipad precompute */
+ memcpy(req->state, ctx->ipad, SHA256_DIGEST_SIZE);
+ /* Already processed the key^ipad part now! */
+ req->len = SHA256_BLOCK_SIZE;
+ req->processed = SHA256_BLOCK_SIZE;
+
+ ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
+ req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
+ req->state_sz = SHA256_DIGEST_SIZE;
+ req->block_sz = SHA256_BLOCK_SIZE;
+ req->hmac = true;
+
return 0;
}
@@ -1257,7 +1336,7 @@ static int safexcel_hmac_sha256_digest(struct ahash_request *areq)
struct safexcel_alg_template safexcel_alg_hmac_sha256 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_SHA2_256,
.alg.ahash = {
.init = safexcel_hmac_sha256_init,
.update = safexcel_ahash_update,
@@ -1273,7 +1352,7 @@ struct safexcel_alg_template safexcel_alg_hmac_sha256 = {
.base = {
.cra_name = "hmac(sha256)",
.cra_driver_name = "safexcel-hmac-sha256",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA256_BLOCK_SIZE,
@@ -1293,26 +1372,10 @@ static int safexcel_sha512_init(struct ahash_request *areq)
memset(req, 0, sizeof(*req));
- req->state[0] = lower_32_bits(SHA512_H0);
- req->state[1] = upper_32_bits(SHA512_H0);
- req->state[2] = lower_32_bits(SHA512_H1);
- req->state[3] = upper_32_bits(SHA512_H1);
- req->state[4] = lower_32_bits(SHA512_H2);
- req->state[5] = upper_32_bits(SHA512_H2);
- req->state[6] = lower_32_bits(SHA512_H3);
- req->state[7] = upper_32_bits(SHA512_H3);
- req->state[8] = lower_32_bits(SHA512_H4);
- req->state[9] = upper_32_bits(SHA512_H4);
- req->state[10] = lower_32_bits(SHA512_H5);
- req->state[11] = upper_32_bits(SHA512_H5);
- req->state[12] = lower_32_bits(SHA512_H6);
- req->state[13] = upper_32_bits(SHA512_H6);
- req->state[14] = lower_32_bits(SHA512_H7);
- req->state[15] = upper_32_bits(SHA512_H7);
-
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
req->state_sz = SHA512_DIGEST_SIZE;
+ req->block_sz = SHA512_BLOCK_SIZE;
return 0;
}
@@ -1329,7 +1392,7 @@ static int safexcel_sha512_digest(struct ahash_request *areq)
struct safexcel_alg_template safexcel_alg_sha512 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_SHA2_512,
.alg.ahash = {
.init = safexcel_sha512_init,
.update = safexcel_ahash_update,
@@ -1344,7 +1407,7 @@ struct safexcel_alg_template safexcel_alg_sha512 = {
.base = {
.cra_name = "sha512",
.cra_driver_name = "safexcel-sha512",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA512_BLOCK_SIZE,
@@ -1364,26 +1427,10 @@ static int safexcel_sha384_init(struct ahash_request *areq)
memset(req, 0, sizeof(*req));
- req->state[0] = lower_32_bits(SHA384_H0);
- req->state[1] = upper_32_bits(SHA384_H0);
- req->state[2] = lower_32_bits(SHA384_H1);
- req->state[3] = upper_32_bits(SHA384_H1);
- req->state[4] = lower_32_bits(SHA384_H2);
- req->state[5] = upper_32_bits(SHA384_H2);
- req->state[6] = lower_32_bits(SHA384_H3);
- req->state[7] = upper_32_bits(SHA384_H3);
- req->state[8] = lower_32_bits(SHA384_H4);
- req->state[9] = upper_32_bits(SHA384_H4);
- req->state[10] = lower_32_bits(SHA384_H5);
- req->state[11] = upper_32_bits(SHA384_H5);
- req->state[12] = lower_32_bits(SHA384_H6);
- req->state[13] = upper_32_bits(SHA384_H6);
- req->state[14] = lower_32_bits(SHA384_H7);
- req->state[15] = upper_32_bits(SHA384_H7);
-
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
req->state_sz = SHA512_DIGEST_SIZE;
+ req->block_sz = SHA512_BLOCK_SIZE;
return 0;
}
@@ -1400,7 +1447,7 @@ static int safexcel_sha384_digest(struct ahash_request *areq)
struct safexcel_alg_template safexcel_alg_sha384 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_SHA2_512,
.alg.ahash = {
.init = safexcel_sha384_init,
.update = safexcel_ahash_update,
@@ -1415,7 +1462,7 @@ struct safexcel_alg_template safexcel_alg_sha384 = {
.base = {
.cra_name = "sha384",
.cra_driver_name = "safexcel-sha384",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA384_BLOCK_SIZE,
@@ -1437,10 +1484,23 @@ static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key,
static int safexcel_hmac_sha512_init(struct ahash_request *areq)
{
+ struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
- safexcel_sha512_init(areq);
- req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
+ memset(req, 0, sizeof(*req));
+
+ /* Start from ipad precompute */
+ memcpy(req->state, ctx->ipad, SHA512_DIGEST_SIZE);
+ /* Already processed the key^ipad part now! */
+ req->len = SHA512_BLOCK_SIZE;
+ req->processed = SHA512_BLOCK_SIZE;
+
+ ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
+ req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
+ req->state_sz = SHA512_DIGEST_SIZE;
+ req->block_sz = SHA512_BLOCK_SIZE;
+ req->hmac = true;
+
return 0;
}
@@ -1456,7 +1516,7 @@ static int safexcel_hmac_sha512_digest(struct ahash_request *areq)
struct safexcel_alg_template safexcel_alg_hmac_sha512 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_SHA2_512,
.alg.ahash = {
.init = safexcel_hmac_sha512_init,
.update = safexcel_ahash_update,
@@ -1472,7 +1532,7 @@ struct safexcel_alg_template safexcel_alg_hmac_sha512 = {
.base = {
.cra_name = "hmac(sha512)",
.cra_driver_name = "safexcel-hmac-sha512",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA512_BLOCK_SIZE,
@@ -1494,10 +1554,23 @@ static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key,
static int safexcel_hmac_sha384_init(struct ahash_request *areq)
{
+ struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
- safexcel_sha384_init(areq);
- req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
+ memset(req, 0, sizeof(*req));
+
+ /* Start from ipad precompute */
+ memcpy(req->state, ctx->ipad, SHA512_DIGEST_SIZE);
+ /* Already processed the key^ipad part now! */
+ req->len = SHA512_BLOCK_SIZE;
+ req->processed = SHA512_BLOCK_SIZE;
+
+ ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
+ req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
+ req->state_sz = SHA512_DIGEST_SIZE;
+ req->block_sz = SHA512_BLOCK_SIZE;
+ req->hmac = true;
+
return 0;
}
@@ -1513,7 +1586,7 @@ static int safexcel_hmac_sha384_digest(struct ahash_request *areq)
struct safexcel_alg_template safexcel_alg_hmac_sha384 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_SHA2_512,
.alg.ahash = {
.init = safexcel_hmac_sha384_init,
.update = safexcel_ahash_update,
@@ -1529,7 +1602,7 @@ struct safexcel_alg_template safexcel_alg_hmac_sha384 = {
.base = {
.cra_name = "hmac(sha384)",
.cra_driver_name = "safexcel-hmac-sha384",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA384_BLOCK_SIZE,
@@ -1549,14 +1622,10 @@ static int safexcel_md5_init(struct ahash_request *areq)
memset(req, 0, sizeof(*req));
- req->state[0] = MD5_H0;
- req->state[1] = MD5_H1;
- req->state[2] = MD5_H2;
- req->state[3] = MD5_H3;
-
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
req->state_sz = MD5_DIGEST_SIZE;
+ req->block_sz = MD5_HMAC_BLOCK_SIZE;
return 0;
}
@@ -1573,7 +1642,7 @@ static int safexcel_md5_digest(struct ahash_request *areq)
struct safexcel_alg_template safexcel_alg_md5 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_MD5,
.alg.ahash = {
.init = safexcel_md5_init,
.update = safexcel_ahash_update,
@@ -1588,7 +1657,7 @@ struct safexcel_alg_template safexcel_alg_md5 = {
.base = {
.cra_name = "md5",
.cra_driver_name = "safexcel-md5",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
@@ -1603,10 +1672,24 @@ struct safexcel_alg_template safexcel_alg_md5 = {
static int safexcel_hmac_md5_init(struct ahash_request *areq)
{
+ struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
- safexcel_md5_init(areq);
- req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
+ memset(req, 0, sizeof(*req));
+
+ /* Start from ipad precompute */
+ memcpy(req->state, ctx->ipad, MD5_DIGEST_SIZE);
+ /* Already processed the key^ipad part now! */
+ req->len = MD5_HMAC_BLOCK_SIZE;
+ req->processed = MD5_HMAC_BLOCK_SIZE;
+
+ ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
+ req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
+ req->state_sz = MD5_DIGEST_SIZE;
+ req->block_sz = MD5_HMAC_BLOCK_SIZE;
+ req->len_is_le = true; /* MD5 is little endian! ... */
+ req->hmac = true;
+
return 0;
}
@@ -1629,7 +1712,7 @@ static int safexcel_hmac_md5_digest(struct ahash_request *areq)
struct safexcel_alg_template safexcel_alg_hmac_md5 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
- .engines = EIP97IES | EIP197B | EIP197D,
+ .algo_mask = SAFEXCEL_ALG_MD5,
.alg.ahash = {
.init = safexcel_hmac_md5_init,
.update = safexcel_ahash_update,
@@ -1645,7 +1728,7 @@ struct safexcel_alg_template safexcel_alg_hmac_md5 = {
.base = {
.cra_name = "hmac(md5)",
.cra_driver_name = "safexcel-hmac-md5",
- .cra_priority = 300,
+ .cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
diff --git a/drivers/crypto/inside-secure/safexcel_ring.c b/drivers/crypto/inside-secure/safexcel_ring.c
index eb75fa684876..0f269b89cfd4 100644
--- a/drivers/crypto/inside-secure/safexcel_ring.c
+++ b/drivers/crypto/inside-secure/safexcel_ring.c
@@ -137,7 +137,13 @@ struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *pr
struct safexcel_token *token =
(struct safexcel_token *)cdesc->control_data.token;
- cdesc->control_data.packet_length = full_data_len;
+ /*
+ * Note that the length here MUST be >0 or else the EIP(1)97
+ * may hang. Newer EIP197 firmware actually incorporates this
+ * fix already, but that doesn't help the EIP97 and we may
+ * also be running older firmware.
+ */
+ cdesc->control_data.packet_length = full_data_len ?: 1;
cdesc->control_data.options = EIP197_OPTION_MAGIC_VALUE |
EIP197_OPTION_64BIT_CTX |
EIP197_OPTION_CTX_CTRL_IN_CMD;
@@ -145,6 +151,10 @@ struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *pr
(lower_32_bits(context) & GENMASK(31, 2)) >> 2;
cdesc->control_data.context_hi = upper_32_bits(context);
+ if (priv->version == EIP197B_MRVL ||
+ priv->version == EIP197D_MRVL)
+ cdesc->control_data.options |= EIP197_OPTION_RC_AUTO;
+
/* TODO: large xform HMAC with SHA-384/512 uses refresh = 3 */
cdesc->control_data.refresh = 2;
diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c
index e5cf3a59c420..9181523ba760 100644
--- a/drivers/crypto/ixp4xx_crypto.c
+++ b/drivers/crypto/ixp4xx_crypto.c
@@ -17,7 +17,7 @@
#include <linux/module.h>
#include <crypto/ctr.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <crypto/aes.h>
#include <crypto/hmac.h>
#include <crypto/sha.h>
@@ -100,7 +100,7 @@ struct buffer_desc {
u16 pkt_len;
u16 buf_len;
#endif
- u32 phys_addr;
+ dma_addr_t phys_addr;
u32 __reserved[4];
struct buffer_desc *next;
enum dma_data_direction dir;
@@ -117,9 +117,9 @@ struct crypt_ctl {
u8 mode; /* NPE_OP_* operation mode */
#endif
u8 iv[MAX_IVLEN]; /* IV for CBC mode or CTR IV for CTR mode */
- u32 icv_rev_aes; /* icv or rev aes */
- u32 src_buf;
- u32 dst_buf;
+ dma_addr_t icv_rev_aes; /* icv or rev aes */
+ dma_addr_t src_buf;
+ dma_addr_t dst_buf;
#ifdef __ARMEB__
u16 auth_offs; /* Authentication start offset */
u16 auth_len; /* Authentication data length */
@@ -320,7 +320,8 @@ static struct crypt_ctl *get_crypt_desc_emerg(void)
}
}
-static void free_buf_chain(struct device *dev, struct buffer_desc *buf,u32 phys)
+static void free_buf_chain(struct device *dev, struct buffer_desc *buf,
+ dma_addr_t phys)
{
while (buf) {
struct buffer_desc *buf1;
@@ -602,7 +603,7 @@ static int register_chain_var(struct crypto_tfm *tfm, u8 xpad, u32 target,
struct buffer_desc *buf;
int i;
u8 *pad;
- u32 pad_phys, buf_phys;
+ dma_addr_t pad_phys, buf_phys;
BUILD_BUG_ON(NPE_CTX_LEN < HMAC_PAD_BLOCKLEN);
pad = dma_pool_alloc(ctx_pool, GFP_KERNEL, &pad_phys);
@@ -755,10 +756,7 @@ static int setup_cipher(struct crypto_tfm *tfm, int encrypt,
}
cipher_cfg |= keylen_cfg;
} else {
- u32 tmp[DES_EXPKEY_WORDS];
- if (des_ekey(tmp, key) == 0) {
- *flags |= CRYPTO_TFM_RES_WEAK_KEY;
- }
+ crypto_des_verify_key(tfm, key);
}
/* write cfg word to cryptinfo */
*(u32*)cinfo = cpu_to_be32(cipher_cfg);
@@ -787,7 +785,7 @@ static struct buffer_desc *chainup_buffers(struct device *dev,
for (; nbytes > 0; sg = sg_next(sg)) {
unsigned len = min(nbytes, sg->length);
struct buffer_desc *next_buf;
- u32 next_buf_phys;
+ dma_addr_t next_buf_phys;
void *ptr;
nbytes -= len;
@@ -850,14 +848,8 @@ out:
static int ablk_des3_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
unsigned int key_len)
{
- u32 flags = crypto_ablkcipher_get_flags(tfm);
- int err;
-
- err = __des3_verify_key(&flags, key);
- if (unlikely(err))
- crypto_ablkcipher_set_flags(tfm, flags);
-
- return ablk_setkey(tfm, key, key_len);
+ return verify_ablkcipher_des3_key(tfm, key) ?:
+ ablk_setkey(tfm, key, key_len);
}
static int ablk_rfc3686_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
@@ -1180,7 +1172,6 @@ static int des3_aead_setkey(struct crypto_aead *tfm, const u8 *key,
unsigned int keylen)
{
struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
- u32 flags = CRYPTO_TFM_RES_BAD_KEY_LEN;
struct crypto_authenc_keys keys;
int err;
@@ -1192,12 +1183,8 @@ static int des3_aead_setkey(struct crypto_aead *tfm, const u8 *key,
if (keys.authkeylen > sizeof(ctx->authkey))
goto badkey;
- if (keys.enckeylen != DES3_EDE_KEY_SIZE)
- goto badkey;
-
- flags = crypto_aead_get_flags(tfm);
- err = __des3_verify_key(&flags, keys.enckey);
- if (unlikely(err))
+ err = verify_aead_des3_key(tfm, keys.enckey, keys.enckeylen);
+ if (err)
goto badkey;
memcpy(ctx->authkey, keys.authkey, keys.authkeylen);
@@ -1208,7 +1195,6 @@ static int des3_aead_setkey(struct crypto_aead *tfm, const u8 *key,
memzero_explicit(&keys, sizeof(keys));
return aead_setup(tfm, crypto_aead_authsize(tfm));
badkey:
- crypto_aead_set_flags(tfm, flags);
memzero_explicit(&keys, sizeof(keys));
return err;
}
diff --git a/drivers/crypto/marvell/cipher.c b/drivers/crypto/marvell/cipher.c
index f4321f3c0777..84ceddfee76b 100644
--- a/drivers/crypto/marvell/cipher.c
+++ b/drivers/crypto/marvell/cipher.c
@@ -10,7 +10,7 @@
*/
#include <crypto/aes.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include "cesa.h"
@@ -254,7 +254,7 @@ static int mv_cesa_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
int ret;
int i;
- ret = crypto_aes_expand_key(&ctx->aes, key, len);
+ ret = aes_expandkey(&ctx->aes, key, len);
if (ret) {
crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
return ret;
@@ -272,21 +272,12 @@ static int mv_cesa_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
static int mv_cesa_des_setkey(struct crypto_skcipher *cipher, const u8 *key,
unsigned int len)
{
- struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
- struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(tfm);
- u32 tmp[DES_EXPKEY_WORDS];
- int ret;
-
- if (len != DES_KEY_SIZE) {
- crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
- return -EINVAL;
- }
+ struct mv_cesa_des_ctx *ctx = crypto_skcipher_ctx(cipher);
+ int err;
- ret = des_ekey(tmp, key);
- if (!ret && (tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
- tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
- return -EINVAL;
- }
+ err = verify_skcipher_des_key(cipher, key);
+ if (err)
+ return err;
memcpy(ctx->key, key, DES_KEY_SIZE);
@@ -299,8 +290,8 @@ static int mv_cesa_des3_ede_setkey(struct crypto_skcipher *cipher,
struct mv_cesa_des_ctx *ctx = crypto_skcipher_ctx(cipher);
int err;
- err = des3_verify_key(cipher, key);
- if (unlikely(err))
+ err = verify_skcipher_des3_key(cipher, key);
+ if (err)
return err;
memcpy(ctx->key, key, DES3_EDE_KEY_SIZE);
diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c
index 0f0ac851f4eb..a2b35fb0fb89 100644
--- a/drivers/crypto/marvell/hash.c
+++ b/drivers/crypto/marvell/hash.c
@@ -1148,8 +1148,7 @@ static int mv_cesa_ahmac_pad_init(struct ahash_request *req,
}
/* Set the memory region to 0 to avoid any leak. */
- memset(keydup, 0, keylen);
- kfree(keydup);
+ kzfree(keydup);
if (ret)
return ret;
diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c
index b7477ee32ca0..90c9644fb8a8 100644
--- a/drivers/crypto/mediatek/mtk-aes.c
+++ b/drivers/crypto/mediatek/mtk-aes.c
@@ -23,7 +23,7 @@
#define AES_CT_CTRL_HDR cpu_to_le32(0x00220000)
-/* AES-CBC/ECB/CTR command token */
+/* AES-CBC/ECB/CTR/OFB/CFB command token */
#define AES_CMD0 cpu_to_le32(0x05000000)
#define AES_CMD1 cpu_to_le32(0x2d060000)
#define AES_CMD2 cpu_to_le32(0xe4a63806)
@@ -50,6 +50,8 @@
/* AES transform information word 1 fields */
#define AES_TFM_ECB cpu_to_le32(0x0 << 0)
#define AES_TFM_CBC cpu_to_le32(0x1 << 0)
+#define AES_TFM_OFB cpu_to_le32(0x4 << 0)
+#define AES_TFM_CFB128 cpu_to_le32(0x5 << 0)
#define AES_TFM_CTR_INIT cpu_to_le32(0x2 << 0) /* init counter to 1 */
#define AES_TFM_CTR_LOAD cpu_to_le32(0x6 << 0) /* load/reuse counter */
#define AES_TFM_3IV cpu_to_le32(0x7 << 5) /* using IV 0-2 */
@@ -58,13 +60,15 @@
#define AES_TFM_ENC_HASH cpu_to_le32(0x1 << 17)
/* AES flags */
-#define AES_FLAGS_CIPHER_MSK GENMASK(2, 0)
+#define AES_FLAGS_CIPHER_MSK GENMASK(4, 0)
#define AES_FLAGS_ECB BIT(0)
#define AES_FLAGS_CBC BIT(1)
#define AES_FLAGS_CTR BIT(2)
-#define AES_FLAGS_GCM BIT(3)
-#define AES_FLAGS_ENCRYPT BIT(4)
-#define AES_FLAGS_BUSY BIT(5)
+#define AES_FLAGS_OFB BIT(3)
+#define AES_FLAGS_CFB128 BIT(4)
+#define AES_FLAGS_GCM BIT(5)
+#define AES_FLAGS_ENCRYPT BIT(6)
+#define AES_FLAGS_BUSY BIT(7)
#define AES_AUTH_TAG_ERR cpu_to_le32(BIT(26))
@@ -101,6 +105,7 @@ struct mtk_aes_reqctx {
struct mtk_aes_base_ctx {
struct mtk_cryp *cryp;
u32 keylen;
+ __le32 key[12];
__le32 keymode;
mtk_aes_fn start;
@@ -405,7 +410,7 @@ exit:
return mtk_aes_complete(cryp, aes, -EINVAL);
}
-/* Initialize transform information of CBC/ECB/CTR mode */
+/* Initialize transform information of CBC/ECB/CTR/OFB/CFB mode */
static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
size_t len)
{
@@ -434,7 +439,12 @@ static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
case AES_FLAGS_CTR:
info->tfm[1] = AES_TFM_CTR_LOAD;
goto ctr;
-
+ case AES_FLAGS_OFB:
+ info->tfm[1] = AES_TFM_OFB;
+ break;
+ case AES_FLAGS_CFB128:
+ info->tfm[1] = AES_TFM_CFB128;
+ break;
default:
/* Should not happen... */
return;
@@ -525,6 +535,8 @@ static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id,
backlog->complete(backlog, -EINPROGRESS);
ctx = crypto_tfm_ctx(areq->tfm);
+ /* Write key into state buffer */
+ memcpy(ctx->info.state, ctx->key, sizeof(ctx->key));
aes->areq = areq;
aes->ctx = ctx;
@@ -644,21 +656,26 @@ static int mtk_aes_setkey(struct crypto_ablkcipher *tfm,
}
ctx->keylen = SIZE_IN_WORDS(keylen);
- mtk_aes_write_state_le(ctx->info.state, (const u32 *)key, keylen);
+ mtk_aes_write_state_le(ctx->key, (const u32 *)key, keylen);
return 0;
}
static int mtk_aes_crypt(struct ablkcipher_request *req, u64 mode)
{
- struct mtk_aes_base_ctx *ctx;
+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+ struct mtk_aes_base_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
struct mtk_aes_reqctx *rctx;
+ struct mtk_cryp *cryp;
+
+ cryp = mtk_aes_find_dev(ctx);
+ if (!cryp)
+ return -ENODEV;
- ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
rctx = ablkcipher_request_ctx(req);
rctx->mode = mode;
- return mtk_aes_handle_queue(ctx->cryp, !(mode & AES_FLAGS_ENCRYPT),
+ return mtk_aes_handle_queue(cryp, !(mode & AES_FLAGS_ENCRYPT),
&req->base);
}
@@ -692,16 +709,29 @@ static int mtk_aes_ctr_decrypt(struct ablkcipher_request *req)
return mtk_aes_crypt(req, AES_FLAGS_CTR);
}
+static int mtk_aes_ofb_encrypt(struct ablkcipher_request *req)
+{
+ return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_OFB);
+}
+
+static int mtk_aes_ofb_decrypt(struct ablkcipher_request *req)
+{
+ return mtk_aes_crypt(req, AES_FLAGS_OFB);
+}
+
+static int mtk_aes_cfb_encrypt(struct ablkcipher_request *req)
+{
+ return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CFB128);
+}
+
+static int mtk_aes_cfb_decrypt(struct ablkcipher_request *req)
+{
+ return mtk_aes_crypt(req, AES_FLAGS_CFB128);
+}
+
static int mtk_aes_cra_init(struct crypto_tfm *tfm)
{
struct mtk_aes_ctx *ctx = crypto_tfm_ctx(tfm);
- struct mtk_cryp *cryp = NULL;
-
- cryp = mtk_aes_find_dev(&ctx->base);
- if (!cryp) {
- pr_err("can't find crypto device\n");
- return -ENODEV;
- }
tfm->crt_ablkcipher.reqsize = sizeof(struct mtk_aes_reqctx);
ctx->base.start = mtk_aes_start;
@@ -711,13 +741,6 @@ static int mtk_aes_cra_init(struct crypto_tfm *tfm)
static int mtk_aes_ctr_cra_init(struct crypto_tfm *tfm)
{
struct mtk_aes_ctx *ctx = crypto_tfm_ctx(tfm);
- struct mtk_cryp *cryp = NULL;
-
- cryp = mtk_aes_find_dev(&ctx->base);
- if (!cryp) {
- pr_err("can't find crypto device\n");
- return -ENODEV;
- }
tfm->crt_ablkcipher.reqsize = sizeof(struct mtk_aes_reqctx);
ctx->base.start = mtk_aes_ctr_start;
@@ -787,6 +810,48 @@ static struct crypto_alg aes_algs[] = {
.decrypt = mtk_aes_ctr_decrypt,
}
},
+{
+ .cra_name = "ofb(aes)",
+ .cra_driver_name = "ofb-aes-mtk",
+ .cra_priority = 400,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_ASYNC,
+ .cra_init = mtk_aes_cra_init,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct mtk_aes_ctx),
+ .cra_alignmask = 0xf,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_u.ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = mtk_aes_setkey,
+ .encrypt = mtk_aes_ofb_encrypt,
+ .decrypt = mtk_aes_ofb_decrypt,
+ }
+},
+{
+ .cra_name = "cfb(aes)",
+ .cra_driver_name = "cfb-aes-mtk",
+ .cra_priority = 400,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_ASYNC,
+ .cra_init = mtk_aes_cra_init,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct mtk_aes_ctx),
+ .cra_alignmask = 0xf,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_u.ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = mtk_aes_setkey,
+ .encrypt = mtk_aes_cfb_encrypt,
+ .decrypt = mtk_aes_cfb_decrypt,
+ }
+},
};
static inline struct mtk_aes_gcm_ctx *
@@ -905,14 +970,11 @@ static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
aes->resume = mtk_aes_transfer_complete;
/* Compute total process length. */
aes->total = len + gctx->authsize;
- /* Compute text length. */
- gctx->textlen = req->cryptlen;
/* Hardware will append authenticated tag to output buffer */
scatterwalk_map_and_copy(tag, req->dst, len, gctx->authsize, 1);
} else {
aes->resume = mtk_aes_gcm_tag_verify;
aes->total = len;
- gctx->textlen = req->cryptlen - gctx->authsize;
}
return mtk_aes_gcm_dma(cryp, aes, req->src, req->dst, len);
@@ -923,6 +985,15 @@ static int mtk_aes_gcm_crypt(struct aead_request *req, u64 mode)
struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx);
struct mtk_aes_reqctx *rctx = aead_request_ctx(req);
+ struct mtk_cryp *cryp;
+ bool enc = !!(mode & AES_FLAGS_ENCRYPT);
+
+ cryp = mtk_aes_find_dev(ctx);
+ if (!cryp)
+ return -ENODEV;
+
+ /* Compute text length. */
+ gctx->textlen = req->cryptlen - (enc ? 0 : gctx->authsize);
/* Empty messages are not supported yet */
if (!gctx->textlen && !req->assoclen)
@@ -930,8 +1001,7 @@ static int mtk_aes_gcm_crypt(struct aead_request *req, u64 mode)
rctx->mode = AES_FLAGS_GCM | mode;
- return mtk_aes_handle_queue(ctx->cryp, !!(mode & AES_FLAGS_ENCRYPT),
- &req->base);
+ return mtk_aes_handle_queue(cryp, enc, &req->base);
}
/*
@@ -1003,10 +1073,8 @@ static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
if (err)
goto out;
- /* Write key into state buffer */
- mtk_aes_write_state_le(ctx->info.state, (const u32 *)key, keylen);
- /* Write key(H) into state buffer */
- mtk_aes_write_state_be(ctx->info.state + ctx->keylen, data->hash,
+ mtk_aes_write_state_le(ctx->key, (const u32 *)key, keylen);
+ mtk_aes_write_state_be(ctx->key + ctx->keylen, data->hash,
AES_BLOCK_SIZE);
out:
kzfree(data);
@@ -1046,13 +1114,6 @@ static int mtk_aes_gcm_decrypt(struct aead_request *req)
static int mtk_aes_gcm_init(struct crypto_aead *aead)
{
struct mtk_aes_gcm_ctx *ctx = crypto_aead_ctx(aead);
- struct mtk_cryp *cryp = NULL;
-
- cryp = mtk_aes_find_dev(&ctx->base);
- if (!cryp) {
- pr_err("can't find crypto device\n");
- return -ENODEV;
- }
ctx->ctr = crypto_alloc_skcipher("ctr(aes)", 0,
CRYPTO_ALG_ASYNC);
diff --git a/drivers/crypto/mediatek/mtk-platform.c b/drivers/crypto/mediatek/mtk-platform.c
index 125318a88cd4..7e3ad085b5bd 100644
--- a/drivers/crypto/mediatek/mtk-platform.c
+++ b/drivers/crypto/mediatek/mtk-platform.c
@@ -481,7 +481,6 @@ err_cleanup:
static int mtk_crypto_probe(struct platform_device *pdev)
{
- struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
struct mtk_cryp *cryp;
int i, err;
@@ -489,16 +488,14 @@ static int mtk_crypto_probe(struct platform_device *pdev)
if (!cryp)
return -ENOMEM;
- cryp->base = devm_ioremap_resource(&pdev->dev, res);
+ cryp->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(cryp->base))
return PTR_ERR(cryp->base);
for (i = 0; i < MTK_IRQ_NUM; i++) {
cryp->irq[i] = platform_get_irq(pdev, i);
- if (cryp->irq[i] < 0) {
- dev_err(cryp->dev, "no IRQ:%d resource info\n", i);
+ if (cryp->irq[i] < 0)
return cryp->irq[i];
- }
}
cryp->clk_cryp = devm_clk_get(&pdev->dev, "cryp");
diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c
index f03b0f06fb2f..9e9f48bb7f85 100644
--- a/drivers/crypto/mediatek/mtk-sha.c
+++ b/drivers/crypto/mediatek/mtk-sha.c
@@ -778,7 +778,9 @@ static int mtk_sha_finup(struct ahash_request *req)
ctx->flags |= SHA_FLAGS_FINUP;
err1 = mtk_sha_update(req);
- if (err1 == -EINPROGRESS || err1 == -EBUSY)
+ if (err1 == -EINPROGRESS ||
+ (err1 == -EBUSY && (ahash_request_flags(req) &
+ CRYPTO_TFM_REQ_MAY_BACKLOG)))
return err1;
/*
* final() has to be always called to cleanup resources
diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c
index bdc4c42d3ac8..bf8d2197bc11 100644
--- a/drivers/crypto/mxs-dcp.c
+++ b/drivers/crypto/mxs-dcp.c
@@ -986,8 +986,6 @@ static int mxs_dcp_probe(struct platform_device *pdev)
struct device *dev = &pdev->dev;
struct dcp *sdcp = NULL;
int i, ret;
-
- struct resource *iores;
int dcp_vmi_irq, dcp_irq;
if (global_sdcp) {
@@ -995,25 +993,20 @@ static int mxs_dcp_probe(struct platform_device *pdev)
return -ENODEV;
}
- iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
dcp_vmi_irq = platform_get_irq(pdev, 0);
- if (dcp_vmi_irq < 0) {
- dev_err(dev, "Failed to get IRQ: (%d)!\n", dcp_vmi_irq);
+ if (dcp_vmi_irq < 0)
return dcp_vmi_irq;
- }
dcp_irq = platform_get_irq(pdev, 1);
- if (dcp_irq < 0) {
- dev_err(dev, "Failed to get IRQ: (%d)!\n", dcp_irq);
+ if (dcp_irq < 0)
return dcp_irq;
- }
sdcp = devm_kzalloc(dev, sizeof(*sdcp), GFP_KERNEL);
if (!sdcp)
return -ENOMEM;
sdcp->dev = dev;
- sdcp->base = devm_ioremap_resource(dev, iores);
+ sdcp->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(sdcp->base))
return PTR_ERR(sdcp->base);
diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c
index 760e72a5893b..dc15b06e96ab 100644
--- a/drivers/crypto/n2_core.c
+++ b/drivers/crypto/n2_core.c
@@ -17,7 +17,7 @@
#include <crypto/md5.h>
#include <crypto/sha.h>
#include <crypto/aes.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <linux/mutex.h>
#include <linux/delay.h>
#include <linux/sched.h>
@@ -760,21 +760,13 @@ static int n2_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
- u32 tmp[DES_EXPKEY_WORDS];
int err;
- ctx->enc_type = n2alg->enc_type;
-
- if (keylen != DES_KEY_SIZE) {
- crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
- return -EINVAL;
- }
+ err = verify_ablkcipher_des_key(cipher, key);
+ if (err)
+ return err;
- err = des_ekey(tmp, key);
- if (err == 0 && (tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
- tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
- return -EINVAL;
- }
+ ctx->enc_type = n2alg->enc_type;
ctx->key_len = keylen;
memcpy(ctx->key.des, key, keylen);
@@ -787,15 +779,11 @@ static int n2_3des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
- u32 flags;
int err;
- flags = crypto_ablkcipher_get_flags(cipher);
- err = __des3_verify_key(&flags, key);
- if (unlikely(err)) {
- crypto_ablkcipher_set_flags(cipher, flags);
+ err = verify_ablkcipher_des3_key(cipher, key);
+ if (err)
return err;
- }
ctx->enc_type = n2alg->enc_type;
@@ -1295,20 +1283,20 @@ struct n2_hash_tmpl {
u8 hmac_type;
};
-static const u32 md5_init[MD5_HASH_WORDS] = {
+static const u32 n2_md5_init[MD5_HASH_WORDS] = {
cpu_to_le32(MD5_H0),
cpu_to_le32(MD5_H1),
cpu_to_le32(MD5_H2),
cpu_to_le32(MD5_H3),
};
-static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = {
+static const u32 n2_sha1_init[SHA1_DIGEST_SIZE / 4] = {
SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,
};
-static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = {
+static const u32 n2_sha256_init[SHA256_DIGEST_SIZE / 4] = {
SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,
};
-static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = {
+static const u32 n2_sha224_init[SHA256_DIGEST_SIZE / 4] = {
SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,
SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,
};
@@ -1316,7 +1304,7 @@ static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = {
static const struct n2_hash_tmpl hash_tmpls[] = {
{ .name = "md5",
.hash_zero = md5_zero_message_hash,
- .hash_init = md5_init,
+ .hash_init = n2_md5_init,
.auth_type = AUTH_TYPE_MD5,
.hmac_type = AUTH_TYPE_HMAC_MD5,
.hw_op_hashsz = MD5_DIGEST_SIZE,
@@ -1324,7 +1312,7 @@ static const struct n2_hash_tmpl hash_tmpls[] = {
.block_size = MD5_HMAC_BLOCK_SIZE },
{ .name = "sha1",
.hash_zero = sha1_zero_message_hash,
- .hash_init = sha1_init,
+ .hash_init = n2_sha1_init,
.auth_type = AUTH_TYPE_SHA1,
.hmac_type = AUTH_TYPE_HMAC_SHA1,
.hw_op_hashsz = SHA1_DIGEST_SIZE,
@@ -1332,7 +1320,7 @@ static const struct n2_hash_tmpl hash_tmpls[] = {
.block_size = SHA1_BLOCK_SIZE },
{ .name = "sha256",
.hash_zero = sha256_zero_message_hash,
- .hash_init = sha256_init,
+ .hash_init = n2_sha256_init,
.auth_type = AUTH_TYPE_SHA256,
.hmac_type = AUTH_TYPE_HMAC_SHA256,
.hw_op_hashsz = SHA256_DIGEST_SIZE,
@@ -1340,7 +1328,7 @@ static const struct n2_hash_tmpl hash_tmpls[] = {
.block_size = SHA256_BLOCK_SIZE },
{ .name = "sha224",
.hash_zero = sha224_zero_message_hash,
- .hash_init = sha224_init,
+ .hash_init = n2_sha224_init,
.auth_type = AUTH_TYPE_SHA256,
.hmac_type = AUTH_TYPE_RESERVED,
.hw_op_hashsz = SHA256_DIGEST_SIZE,
diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c
index 4acbc47973e9..c037a2403b82 100644
--- a/drivers/crypto/nx/nx-842-powernv.c
+++ b/drivers/crypto/nx/nx-842-powernv.c
@@ -27,8 +27,6 @@ MODULE_ALIAS_CRYPTO("842-nx");
#define WORKMEM_ALIGN (CRB_ALIGN)
#define CSB_WAIT_MAX (5000) /* ms */
#define VAS_RETRIES (10)
-/* # of requests allowed per RxFIFO at a time. 0 for unlimited */
-#define MAX_CREDITS_PER_RXFIFO (1024)
struct nx842_workmem {
/* Below fields must be properly aligned */
@@ -812,7 +810,11 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
rxattr.lnotify_lpid = lpid;
rxattr.lnotify_pid = pid;
rxattr.lnotify_tid = tid;
- rxattr.wcreds_max = MAX_CREDITS_PER_RXFIFO;
+ /*
+ * Maximum RX window credits can not be more than #CRBs in
+ * RxFIFO. Otherwise, can get checkstop if RxFIFO overruns.
+ */
+ rxattr.wcreds_max = fifo_size / CRB_SIZE;
/*
* Open a VAS receice window which is used to configure RxFIFO
@@ -1018,6 +1020,7 @@ static __init int nx842_powernv_init(void)
ret = nx842_powernv_probe_vas(dn);
if (ret) {
nx842_delete_coprocs();
+ of_node_put(dn);
return ret;
}
}
diff --git a/drivers/crypto/nx/nx-842-pseries.c b/drivers/crypto/nx/nx-842-pseries.c
index 5c4aa606208c..2de5e3672e42 100644
--- a/drivers/crypto/nx/nx-842-pseries.c
+++ b/drivers/crypto/nx/nx-842-pseries.c
@@ -856,7 +856,7 @@ static ssize_t nx842_##_name##_show(struct device *dev, \
rcu_read_lock(); \
local_devdata = rcu_dereference(devdata); \
if (local_devdata) \
- p = snprintf(buf, PAGE_SIZE, "%ld\n", \
+ p = snprintf(buf, PAGE_SIZE, "%lld\n", \
atomic64_read(&local_devdata->counters->_name)); \
rcu_read_unlock(); \
return p; \
@@ -909,7 +909,7 @@ static ssize_t nx842_timehist_show(struct device *dev,
}
for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {
- bytes = snprintf(p, bytes_remain, "%u-%uus:\t%ld\n",
+ bytes = snprintf(p, bytes_remain, "%u-%uus:\t%lld\n",
i ? (2<<(i-1)) : 0, (2<<i)-1,
atomic64_read(&times[i]));
bytes_remain -= bytes;
@@ -917,7 +917,7 @@ static ssize_t nx842_timehist_show(struct device *dev,
}
/* The last bucket holds everything over
* 2<<(NX842_HIST_SLOTS - 2) us */
- bytes = snprintf(p, bytes_remain, "%uus - :\t%ld\n",
+ bytes = snprintf(p, bytes_remain, "%uus - :\t%lld\n",
2<<(NX842_HIST_SLOTS - 2),
atomic64_read(&times[(NX842_HIST_SLOTS - 1)]));
p += bytes;
diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c
index 428c273a1ab6..28817880c76d 100644
--- a/drivers/crypto/nx/nx.c
+++ b/drivers/crypto/nx/nx.c
@@ -569,9 +569,7 @@ static int nx_register_algs(void)
memset(&nx_driver.stats, 0, sizeof(struct nx_stats));
- rc = NX_DEBUGFS_INIT(&nx_driver);
- if (rc)
- goto out;
+ NX_DEBUGFS_INIT(&nx_driver);
nx_driver.of.status = NX_OKAY;
diff --git a/drivers/crypto/nx/nx.h b/drivers/crypto/nx/nx.h
index c3e54af18645..7ecca168f8c4 100644
--- a/drivers/crypto/nx/nx.h
+++ b/drivers/crypto/nx/nx.h
@@ -9,9 +9,6 @@
#define NX_STRING "IBM Power7+ Nest Accelerator Crypto Driver"
#define NX_VERSION "1.0"
-static const char nx_driver_string[] = NX_STRING;
-static const char nx_driver_version[] = NX_VERSION;
-
/* a scatterlist in the format PHYP is expecting */
struct nx_sg {
u64 addr;
@@ -76,20 +73,12 @@ struct nx_stats {
atomic_t last_error_pid;
};
-struct nx_debugfs {
- struct dentry *dfs_root;
- struct dentry *dfs_aes_ops, *dfs_aes_bytes;
- struct dentry *dfs_sha256_ops, *dfs_sha256_bytes;
- struct dentry *dfs_sha512_ops, *dfs_sha512_bytes;
- struct dentry *dfs_errors, *dfs_last_error, *dfs_last_error_pid;
-};
-
struct nx_crypto_driver {
struct nx_stats stats;
struct nx_of of;
struct vio_dev *viodev;
struct vio_driver viodriver;
- struct nx_debugfs dfs;
+ struct dentry *dfs_root;
};
#define NX_GCM4106_NONCE_LEN (4)
@@ -177,7 +166,7 @@ struct nx_sg *nx_walk_and_build(struct nx_sg *, unsigned int,
#define NX_DEBUGFS_INIT(drv) nx_debugfs_init(drv)
#define NX_DEBUGFS_FINI(drv) nx_debugfs_fini(drv)
-int nx_debugfs_init(struct nx_crypto_driver *);
+void nx_debugfs_init(struct nx_crypto_driver *);
void nx_debugfs_fini(struct nx_crypto_driver *);
#else
#define NX_DEBUGFS_INIT(drv) (0)
diff --git a/drivers/crypto/nx/nx_debugfs.c b/drivers/crypto/nx/nx_debugfs.c
index 03e4f0363c6a..e0d44a5512ab 100644
--- a/drivers/crypto/nx/nx_debugfs.c
+++ b/drivers/crypto/nx/nx_debugfs.c
@@ -30,62 +30,37 @@
* Documentation/ABI/testing/debugfs-pfo-nx-crypto
*/
-int nx_debugfs_init(struct nx_crypto_driver *drv)
+void nx_debugfs_init(struct nx_crypto_driver *drv)
{
- struct nx_debugfs *dfs = &drv->dfs;
+ struct dentry *root;
- dfs->dfs_root = debugfs_create_dir(NX_NAME, NULL);
+ root = debugfs_create_dir(NX_NAME, NULL);
+ drv->dfs_root = root;
- dfs->dfs_aes_ops =
- debugfs_create_u32("aes_ops",
- S_IRUSR | S_IRGRP | S_IROTH,
- dfs->dfs_root, (u32 *)&drv->stats.aes_ops);
- dfs->dfs_sha256_ops =
- debugfs_create_u32("sha256_ops",
- S_IRUSR | S_IRGRP | S_IROTH,
- dfs->dfs_root,
- (u32 *)&drv->stats.sha256_ops);
- dfs->dfs_sha512_ops =
- debugfs_create_u32("sha512_ops",
- S_IRUSR | S_IRGRP | S_IROTH,
- dfs->dfs_root,
- (u32 *)&drv->stats.sha512_ops);
- dfs->dfs_aes_bytes =
- debugfs_create_u64("aes_bytes",
- S_IRUSR | S_IRGRP | S_IROTH,
- dfs->dfs_root,
- (u64 *)&drv->stats.aes_bytes);
- dfs->dfs_sha256_bytes =
- debugfs_create_u64("sha256_bytes",
- S_IRUSR | S_IRGRP | S_IROTH,
- dfs->dfs_root,
- (u64 *)&drv->stats.sha256_bytes);
- dfs->dfs_sha512_bytes =
- debugfs_create_u64("sha512_bytes",
- S_IRUSR | S_IRGRP | S_IROTH,
- dfs->dfs_root,
- (u64 *)&drv->stats.sha512_bytes);
- dfs->dfs_errors =
- debugfs_create_u32("errors",
- S_IRUSR | S_IRGRP | S_IROTH,
- dfs->dfs_root, (u32 *)&drv->stats.errors);
- dfs->dfs_last_error =
- debugfs_create_u32("last_error",
- S_IRUSR | S_IRGRP | S_IROTH,
- dfs->dfs_root,
- (u32 *)&drv->stats.last_error);
- dfs->dfs_last_error_pid =
- debugfs_create_u32("last_error_pid",
- S_IRUSR | S_IRGRP | S_IROTH,
- dfs->dfs_root,
- (u32 *)&drv->stats.last_error_pid);
- return 0;
+ debugfs_create_u32("aes_ops", S_IRUSR | S_IRGRP | S_IROTH,
+ root, (u32 *)&drv->stats.aes_ops);
+ debugfs_create_u32("sha256_ops", S_IRUSR | S_IRGRP | S_IROTH,
+ root, (u32 *)&drv->stats.sha256_ops);
+ debugfs_create_u32("sha512_ops", S_IRUSR | S_IRGRP | S_IROTH,
+ root, (u32 *)&drv->stats.sha512_ops);
+ debugfs_create_u64("aes_bytes", S_IRUSR | S_IRGRP | S_IROTH,
+ root, (u64 *)&drv->stats.aes_bytes);
+ debugfs_create_u64("sha256_bytes", S_IRUSR | S_IRGRP | S_IROTH,
+ root, (u64 *)&drv->stats.sha256_bytes);
+ debugfs_create_u64("sha512_bytes", S_IRUSR | S_IRGRP | S_IROTH,
+ root, (u64 *)&drv->stats.sha512_bytes);
+ debugfs_create_u32("errors", S_IRUSR | S_IRGRP | S_IROTH,
+ root, (u32 *)&drv->stats.errors);
+ debugfs_create_u32("last_error", S_IRUSR | S_IRGRP | S_IROTH,
+ root, (u32 *)&drv->stats.last_error);
+ debugfs_create_u32("last_error_pid", S_IRUSR | S_IRGRP | S_IROTH,
+ root, (u32 *)&drv->stats.last_error_pid);
}
void
nx_debugfs_fini(struct nx_crypto_driver *drv)
{
- debugfs_remove_recursive(drv->dfs.dfs_root);
+ debugfs_remove_recursive(drv->dfs_root);
}
#endif
diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c
index 45a4647f7030..2f53fbb74100 100644
--- a/drivers/crypto/omap-aes.c
+++ b/drivers/crypto/omap-aes.c
@@ -1180,7 +1180,6 @@ static int omap_aes_probe(struct platform_device *pdev)
irq = platform_get_irq(pdev, 0);
if (irq < 0) {
- dev_err(dev, "can't get IRQ resource\n");
err = irq;
goto err_irq;
}
diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c
index 1ee69a979677..b19d7e5d55ec 100644
--- a/drivers/crypto/omap-des.c
+++ b/drivers/crypto/omap-des.c
@@ -33,7 +33,7 @@
#include <linux/crypto.h>
#include <linux/interrupt.h>
#include <crypto/scatterwalk.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <crypto/algapi.h>
#include <crypto/engine.h>
@@ -650,20 +650,13 @@ static int omap_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
unsigned int keylen)
{
struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(cipher);
- struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+ int err;
pr_debug("enter, keylen: %d\n", keylen);
- /* Do we need to test against weak key? */
- if (tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) {
- u32 tmp[DES_EXPKEY_WORDS];
- int ret = des_ekey(tmp, key);
-
- if (!ret) {
- tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
- return -EINVAL;
- }
- }
+ err = verify_ablkcipher_des_key(cipher, key);
+ if (err)
+ return err;
memcpy(ctx->key, key, keylen);
ctx->keylen = keylen;
@@ -672,20 +665,16 @@ static int omap_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
}
static int omap_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
- unsigned int keylen)
+ unsigned int keylen)
{
struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(cipher);
- u32 flags;
int err;
pr_debug("enter, keylen: %d\n", keylen);
- flags = crypto_ablkcipher_get_flags(cipher);
- err = __des3_verify_key(&flags, key);
- if (unlikely(err)) {
- crypto_ablkcipher_set_flags(cipher, flags);
+ err = verify_ablkcipher_des3_key(cipher, key);
+ if (err)
return err;
- }
memcpy(ctx->key, key, keylen);
ctx->keylen = keylen;
@@ -1049,7 +1038,6 @@ static int omap_des_probe(struct platform_device *pdev)
irq = platform_get_irq(pdev, 0);
if (irq < 0) {
- dev_err(dev, "can't get IRQ resource: %d\n", irq);
err = irq;
goto err_irq;
}
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index e8e2907bd9f4..ac80bc6af093 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -1989,7 +1989,6 @@ static int omap_sham_get_res_pdev(struct omap_sham_dev *dd,
/* Get the IRQ */
dd->irq = platform_get_irq(pdev, 0);
if (dd->irq < 0) {
- dev_err(dev, "no IRQ resource info\n");
err = dd->irq;
goto err;
}
diff --git a/drivers/crypto/padlock-aes.c b/drivers/crypto/padlock-aes.c
index ad020133da19..8a0661250078 100644
--- a/drivers/crypto/padlock-aes.c
+++ b/drivers/crypto/padlock-aes.c
@@ -145,7 +145,7 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
ctx->cword.encrypt.keygen = 1;
ctx->cword.decrypt.keygen = 1;
- if (crypto_aes_expand_key(&gen_aes, in_key, key_len)) {
+ if (aes_expandkey(&gen_aes, in_key, key_len)) {
*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
return -EINVAL;
}
@@ -300,7 +300,7 @@ static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key,
return iv;
}
-static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+static void padlock_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
struct aes_ctx *ctx = aes_ctx(tfm);
@@ -309,7 +309,7 @@ static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
padlock_store_cword(&ctx->cword.encrypt);
}
-static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+static void padlock_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
struct aes_ctx *ctx = aes_ctx(tfm);
@@ -332,8 +332,8 @@ static struct crypto_alg aes_alg = {
.cia_min_keysize = AES_MIN_KEY_SIZE,
.cia_max_keysize = AES_MAX_KEY_SIZE,
.cia_setkey = aes_set_key,
- .cia_encrypt = aes_encrypt,
- .cia_decrypt = aes_decrypt,
+ .cia_encrypt = padlock_aes_encrypt,
+ .cia_decrypt = padlock_aes_decrypt,
}
}
};
diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c
index b985cb85c9bc..3cbefb41b099 100644
--- a/drivers/crypto/picoxcell_crypto.c
+++ b/drivers/crypto/picoxcell_crypto.c
@@ -6,7 +6,7 @@
#include <crypto/aes.h>
#include <crypto/algapi.h>
#include <crypto/authenc.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <crypto/md5.h>
#include <crypto/sha.h>
#include <crypto/internal/skcipher.h>
@@ -736,16 +736,12 @@ static void spacc_aead_cra_exit(struct crypto_aead *tfm)
static int spacc_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
unsigned int len)
{
- struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
- struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
- u32 tmp[DES_EXPKEY_WORDS];
+ struct spacc_ablk_ctx *ctx = crypto_ablkcipher_ctx(cipher);
+ int err;
- if (unlikely(!des_ekey(tmp, key)) &&
- (crypto_ablkcipher_get_flags(cipher) &
- CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
- tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
- return -EINVAL;
- }
+ err = verify_ablkcipher_des_key(cipher, key);
+ if (err)
+ return err;
memcpy(ctx->key, key, len);
ctx->key_len = len;
@@ -761,15 +757,11 @@ static int spacc_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
unsigned int len)
{
struct spacc_ablk_ctx *ctx = crypto_ablkcipher_ctx(cipher);
- u32 flags;
int err;
- flags = crypto_ablkcipher_get_flags(cipher);
- err = __des3_verify_key(&flags, key);
- if (unlikely(err)) {
- crypto_ablkcipher_set_flags(cipher, flags);
+ err = verify_ablkcipher_des3_key(cipher, key);
+ if (err)
return err;
- }
memcpy(ctx->key, key, len);
ctx->key_len = len;
@@ -1624,7 +1616,7 @@ MODULE_DEVICE_TABLE(of, spacc_of_id_table);
static int spacc_probe(struct platform_device *pdev)
{
int i, err, ret;
- struct resource *mem, *irq;
+ struct resource *irq;
struct device_node *np = pdev->dev.of_node;
struct spacc_engine *engine = devm_kzalloc(&pdev->dev, sizeof(*engine),
GFP_KERNEL);
@@ -1653,8 +1645,7 @@ static int spacc_probe(struct platform_device *pdev)
engine->name = dev_name(&pdev->dev);
- mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- engine->regs = devm_ioremap_resource(&pdev->dev, mem);
+ engine->regs = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(engine->regs))
return PTR_ERR(engine->regs);
diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h
index 5c4c0a253129..d78f8d5c89c3 100644
--- a/drivers/crypto/qat/qat_common/adf_common_drv.h
+++ b/drivers/crypto/qat/qat_common/adf_common_drv.h
@@ -95,7 +95,7 @@ struct service_hndl {
static inline int get_current_node(void)
{
- return topology_physical_package_id(smp_processor_id());
+ return topology_physical_package_id(raw_smp_processor_id());
}
int adf_service_register(struct service_hndl *service);
diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c
index c8d401646902..b50eb55f8f57 100644
--- a/drivers/crypto/qat/qat_common/qat_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_algs.c
@@ -131,7 +131,6 @@ struct qat_alg_ablkcipher_ctx {
struct icp_qat_fw_la_bulk_req dec_fw_req;
struct qat_crypto_instance *inst;
struct crypto_tfm *tfm;
- spinlock_t lock; /* protects qat_alg_ablkcipher_ctx struct */
};
static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg)
@@ -223,6 +222,9 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
return -EFAULT;
offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8);
+ if (offset < 0)
+ return -EFAULT;
+
hash_state_out = (__be32 *)(hash->sha.state1 + offset);
hash512_state_out = (__be64 *)hash_state_out;
@@ -253,7 +255,24 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
return 0;
}
-static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)
+static void qat_alg_init_hdr_iv_updt(struct icp_qat_fw_comn_req_hdr *header)
+{
+ ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
+ ICP_QAT_FW_CIPH_IV_64BIT_PTR);
+ ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
+ ICP_QAT_FW_LA_UPDATE_STATE);
+}
+
+static void qat_alg_init_hdr_no_iv_updt(struct icp_qat_fw_comn_req_hdr *header)
+{
+ ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
+ ICP_QAT_FW_CIPH_IV_16BYTE_DATA);
+ ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
+ ICP_QAT_FW_LA_NO_UPDATE_STATE);
+}
+
+static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header,
+ int aead)
{
header->hdr_flags =
ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
@@ -263,12 +282,12 @@ static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)
QAT_COMN_PTR_TYPE_SGL);
ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags,
ICP_QAT_FW_LA_PARTIAL_NONE);
- ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
- ICP_QAT_FW_CIPH_IV_16BYTE_DATA);
+ if (aead)
+ qat_alg_init_hdr_no_iv_updt(header);
+ else
+ qat_alg_init_hdr_iv_updt(header);
ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
ICP_QAT_FW_LA_NO_PROTO);
- ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
- ICP_QAT_FW_LA_NO_UPDATE_STATE);
}
static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm,
@@ -303,7 +322,7 @@ static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm,
return -EFAULT;
/* Request setup */
- qat_alg_init_common_hdr(header);
+ qat_alg_init_common_hdr(header, 1);
header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
@@ -390,7 +409,7 @@ static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm,
return -EFAULT;
/* Request setup */
- qat_alg_init_common_hdr(header);
+ qat_alg_init_common_hdr(header, 1);
header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
@@ -454,7 +473,7 @@ static void qat_alg_ablkcipher_init_com(struct qat_alg_ablkcipher_ctx *ctx,
struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl;
memcpy(cd->aes.key, key, keylen);
- qat_alg_init_common_hdr(header);
+ qat_alg_init_common_hdr(header, 0);
header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;
cd_pars->u.s.content_desc_params_sz =
sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3;
@@ -576,45 +595,52 @@ bad_key:
return -EINVAL;
}
-static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key,
+static int qat_alg_aead_rekey(struct crypto_aead *tfm, const uint8_t *key,
+ unsigned int keylen)
+{
+ struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
+
+ memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
+ memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
+ memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
+ memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
+
+ return qat_alg_aead_init_sessions(tfm, key, keylen,
+ ICP_QAT_HW_CIPHER_CBC_MODE);
+}
+
+static int qat_alg_aead_newkey(struct crypto_aead *tfm, const uint8_t *key,
unsigned int keylen)
{
struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
+ struct qat_crypto_instance *inst = NULL;
+ int node = get_current_node();
struct device *dev;
+ int ret;
- if (ctx->enc_cd) {
- /* rekeying */
- dev = &GET_DEV(ctx->inst->accel_dev);
- memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
- memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
- memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
- memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
- } else {
- /* new key */
- int node = get_current_node();
- struct qat_crypto_instance *inst =
- qat_crypto_get_instance_node(node);
- if (!inst) {
- return -EINVAL;
- }
-
- dev = &GET_DEV(inst->accel_dev);
- ctx->inst = inst;
- ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
- &ctx->enc_cd_paddr,
- GFP_ATOMIC);
- if (!ctx->enc_cd) {
- return -ENOMEM;
- }
- ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
- &ctx->dec_cd_paddr,
- GFP_ATOMIC);
- if (!ctx->dec_cd) {
- goto out_free_enc;
- }
+ inst = qat_crypto_get_instance_node(node);
+ if (!inst)
+ return -EINVAL;
+ dev = &GET_DEV(inst->accel_dev);
+ ctx->inst = inst;
+ ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
+ &ctx->enc_cd_paddr,
+ GFP_ATOMIC);
+ if (!ctx->enc_cd) {
+ ret = -ENOMEM;
+ goto out_free_inst;
+ }
+ ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
+ &ctx->dec_cd_paddr,
+ GFP_ATOMIC);
+ if (!ctx->dec_cd) {
+ ret = -ENOMEM;
+ goto out_free_enc;
}
- if (qat_alg_aead_init_sessions(tfm, key, keylen,
- ICP_QAT_HW_CIPHER_CBC_MODE))
+
+ ret = qat_alg_aead_init_sessions(tfm, key, keylen,
+ ICP_QAT_HW_CIPHER_CBC_MODE);
+ if (ret)
goto out_free_all;
return 0;
@@ -629,7 +655,21 @@ out_free_enc:
dma_free_coherent(dev, sizeof(struct qat_alg_cd),
ctx->enc_cd, ctx->enc_cd_paddr);
ctx->enc_cd = NULL;
- return -ENOMEM;
+out_free_inst:
+ ctx->inst = NULL;
+ qat_crypto_put_instance(inst);
+ return ret;
+}
+
+static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key,
+ unsigned int keylen)
+{
+ struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
+
+ if (ctx->enc_cd)
+ return qat_alg_aead_rekey(tfm, key, keylen);
+ else
+ return qat_alg_aead_newkey(tfm, key, keylen);
}
static void qat_alg_free_bufl(struct qat_crypto_instance *inst,
@@ -677,8 +717,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
dma_addr_t blp;
dma_addr_t bloutp = 0;
struct scatterlist *sg;
- size_t sz_out, sz = sizeof(struct qat_alg_buf_list) +
- ((1 + n) * sizeof(struct qat_alg_buf));
+ size_t sz_out, sz = struct_size(bufl, bufers, n + 1);
if (unlikely(!n))
return -EINVAL;
@@ -715,8 +754,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
struct qat_alg_buf *bufers;
n = sg_nents(sglout);
- sz_out = sizeof(struct qat_alg_buf_list) +
- ((1 + n) * sizeof(struct qat_alg_buf));
+ sz_out = struct_size(buflout, bufers, n + 1);
sg_nctr = 0;
buflout = kzalloc_node(sz_out, GFP_ATOMIC,
dev_to_node(&GET_DEV(inst->accel_dev)));
@@ -801,11 +839,17 @@ static void qat_ablkcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
struct qat_crypto_instance *inst = ctx->inst;
struct ablkcipher_request *areq = qat_req->ablkcipher_req;
uint8_t stat_filed = qat_resp->comn_resp.comn_status;
+ struct device *dev = &GET_DEV(ctx->inst->accel_dev);
int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
qat_alg_free_bufl(inst, qat_req);
if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
res = -EINVAL;
+
+ memcpy(areq->info, qat_req->iv, AES_BLOCK_SIZE);
+ dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv,
+ qat_req->iv_paddr);
+
areq->base.complete(&areq->base, res);
}
@@ -905,50 +949,49 @@ static int qat_alg_aead_enc(struct aead_request *areq)
return -EINPROGRESS;
}
-static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
+static int qat_alg_ablkcipher_rekey(struct qat_alg_ablkcipher_ctx *ctx,
+ const u8 *key, unsigned int keylen,
+ int mode)
+{
+ memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
+ memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
+ memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
+ memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
+
+ return qat_alg_ablkcipher_init_sessions(ctx, key, keylen, mode);
+}
+
+static int qat_alg_ablkcipher_newkey(struct qat_alg_ablkcipher_ctx *ctx,
const u8 *key, unsigned int keylen,
int mode)
{
- struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+ struct qat_crypto_instance *inst = NULL;
struct device *dev;
+ int node = get_current_node();
+ int ret;
- spin_lock(&ctx->lock);
- if (ctx->enc_cd) {
- /* rekeying */
- dev = &GET_DEV(ctx->inst->accel_dev);
- memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
- memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
- memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
- memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
- } else {
- /* new key */
- int node = get_current_node();
- struct qat_crypto_instance *inst =
- qat_crypto_get_instance_node(node);
- if (!inst) {
- spin_unlock(&ctx->lock);
- return -EINVAL;
- }
-
- dev = &GET_DEV(inst->accel_dev);
- ctx->inst = inst;
- ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
- &ctx->enc_cd_paddr,
- GFP_ATOMIC);
- if (!ctx->enc_cd) {
- spin_unlock(&ctx->lock);
- return -ENOMEM;
- }
- ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
- &ctx->dec_cd_paddr,
- GFP_ATOMIC);
- if (!ctx->dec_cd) {
- spin_unlock(&ctx->lock);
- goto out_free_enc;
- }
+ inst = qat_crypto_get_instance_node(node);
+ if (!inst)
+ return -EINVAL;
+ dev = &GET_DEV(inst->accel_dev);
+ ctx->inst = inst;
+ ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
+ &ctx->enc_cd_paddr,
+ GFP_ATOMIC);
+ if (!ctx->enc_cd) {
+ ret = -ENOMEM;
+ goto out_free_instance;
+ }
+ ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
+ &ctx->dec_cd_paddr,
+ GFP_ATOMIC);
+ if (!ctx->dec_cd) {
+ ret = -ENOMEM;
+ goto out_free_enc;
}
- spin_unlock(&ctx->lock);
- if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen, mode))
+
+ ret = qat_alg_ablkcipher_init_sessions(ctx, key, keylen, mode);
+ if (ret)
goto out_free_all;
return 0;
@@ -963,7 +1006,22 @@ out_free_enc:
dma_free_coherent(dev, sizeof(*ctx->enc_cd),
ctx->enc_cd, ctx->enc_cd_paddr);
ctx->enc_cd = NULL;
- return -ENOMEM;
+out_free_instance:
+ ctx->inst = NULL;
+ qat_crypto_put_instance(inst);
+ return ret;
+}
+
+static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
+ const u8 *key, unsigned int keylen,
+ int mode)
+{
+ struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+
+ if (ctx->enc_cd)
+ return qat_alg_ablkcipher_rekey(ctx, key, keylen, mode);
+ else
+ return qat_alg_ablkcipher_newkey(ctx, key, keylen, mode);
}
static int qat_alg_ablkcipher_cbc_setkey(struct crypto_ablkcipher *tfm,
@@ -995,11 +1053,23 @@ static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req)
struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req);
struct icp_qat_fw_la_cipher_req_params *cipher_param;
struct icp_qat_fw_la_bulk_req *msg;
+ struct device *dev = &GET_DEV(ctx->inst->accel_dev);
int ret, ctr = 0;
+ if (req->nbytes == 0)
+ return 0;
+
+ qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE,
+ &qat_req->iv_paddr, GFP_ATOMIC);
+ if (!qat_req->iv)
+ return -ENOMEM;
+
ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req);
- if (unlikely(ret))
+ if (unlikely(ret)) {
+ dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv,
+ qat_req->iv_paddr);
return ret;
+ }
msg = &qat_req->req;
*msg = ctx->enc_fw_req;
@@ -1012,18 +1082,29 @@ static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req)
cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
cipher_param->cipher_length = req->nbytes;
cipher_param->cipher_offset = 0;
- memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE);
+ cipher_param->u.s.cipher_IV_ptr = qat_req->iv_paddr;
+ memcpy(qat_req->iv, req->info, AES_BLOCK_SIZE);
do {
ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
} while (ret == -EAGAIN && ctr++ < 10);
if (ret == -EAGAIN) {
qat_alg_free_bufl(ctx->inst, qat_req);
+ dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv,
+ qat_req->iv_paddr);
return -EBUSY;
}
return -EINPROGRESS;
}
+static int qat_alg_ablkcipher_blk_encrypt(struct ablkcipher_request *req)
+{
+ if (req->nbytes % AES_BLOCK_SIZE != 0)
+ return -EINVAL;
+
+ return qat_alg_ablkcipher_encrypt(req);
+}
+
static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
@@ -1032,11 +1113,23 @@ static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req)
struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req);
struct icp_qat_fw_la_cipher_req_params *cipher_param;
struct icp_qat_fw_la_bulk_req *msg;
+ struct device *dev = &GET_DEV(ctx->inst->accel_dev);
int ret, ctr = 0;
+ if (req->nbytes == 0)
+ return 0;
+
+ qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE,
+ &qat_req->iv_paddr, GFP_ATOMIC);
+ if (!qat_req->iv)
+ return -ENOMEM;
+
ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req);
- if (unlikely(ret))
+ if (unlikely(ret)) {
+ dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv,
+ qat_req->iv_paddr);
return ret;
+ }
msg = &qat_req->req;
*msg = ctx->dec_fw_req;
@@ -1049,18 +1142,28 @@ static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req)
cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
cipher_param->cipher_length = req->nbytes;
cipher_param->cipher_offset = 0;
- memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE);
+ cipher_param->u.s.cipher_IV_ptr = qat_req->iv_paddr;
+ memcpy(qat_req->iv, req->info, AES_BLOCK_SIZE);
do {
ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
} while (ret == -EAGAIN && ctr++ < 10);
if (ret == -EAGAIN) {
qat_alg_free_bufl(ctx->inst, qat_req);
+ dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv,
+ qat_req->iv_paddr);
return -EBUSY;
}
return -EINPROGRESS;
}
+static int qat_alg_ablkcipher_blk_decrypt(struct ablkcipher_request *req)
+{
+ if (req->nbytes % AES_BLOCK_SIZE != 0)
+ return -EINVAL;
+
+ return qat_alg_ablkcipher_decrypt(req);
+}
static int qat_alg_aead_init(struct crypto_aead *tfm,
enum icp_qat_hw_auth_algo hash,
const char *hash_name)
@@ -1119,7 +1222,6 @@ static int qat_alg_ablkcipher_init(struct crypto_tfm *tfm)
{
struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
- spin_lock_init(&ctx->lock);
tfm->crt_ablkcipher.reqsize = sizeof(struct qat_crypto_request);
ctx->tfm = tfm;
return 0;
@@ -1221,8 +1323,8 @@ static struct crypto_alg qat_algs[] = { {
.cra_u = {
.ablkcipher = {
.setkey = qat_alg_ablkcipher_cbc_setkey,
- .decrypt = qat_alg_ablkcipher_decrypt,
- .encrypt = qat_alg_ablkcipher_encrypt,
+ .decrypt = qat_alg_ablkcipher_blk_decrypt,
+ .encrypt = qat_alg_ablkcipher_blk_encrypt,
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
@@ -1233,7 +1335,7 @@ static struct crypto_alg qat_algs[] = { {
.cra_driver_name = "qat_aes_ctr",
.cra_priority = 4001,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
- .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_blocksize = 1,
.cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx),
.cra_alignmask = 0,
.cra_type = &crypto_ablkcipher_type,
@@ -1265,8 +1367,8 @@ static struct crypto_alg qat_algs[] = { {
.cra_u = {
.ablkcipher = {
.setkey = qat_alg_ablkcipher_xts_setkey,
- .decrypt = qat_alg_ablkcipher_decrypt,
- .encrypt = qat_alg_ablkcipher_encrypt,
+ .decrypt = qat_alg_ablkcipher_blk_decrypt,
+ .encrypt = qat_alg_ablkcipher_blk_encrypt,
.min_keysize = 2 * AES_MIN_KEY_SIZE,
.max_keysize = 2 * AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h
index dc0273fe3620..c77a80020cde 100644
--- a/drivers/crypto/qat/qat_common/qat_crypto.h
+++ b/drivers/crypto/qat/qat_common/qat_crypto.h
@@ -88,6 +88,8 @@ struct qat_crypto_request {
struct qat_crypto_request_buffs buf;
void (*cb)(struct icp_qat_fw_la_resp *resp,
struct qat_crypto_request *req);
+ void *iv;
+ dma_addr_t iv_paddr;
};
#endif
diff --git a/drivers/crypto/qce/ablkcipher.c b/drivers/crypto/qce/ablkcipher.c
index a976210ba41c..7a98bf5cc967 100644
--- a/drivers/crypto/qce/ablkcipher.c
+++ b/drivers/crypto/qce/ablkcipher.c
@@ -7,7 +7,7 @@
#include <linux/interrupt.h>
#include <linux/types.h>
#include <crypto/aes.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <crypto/internal/skcipher.h>
#include "cipher.h"
@@ -154,27 +154,17 @@ static int qce_ablkcipher_setkey(struct crypto_ablkcipher *ablk, const u8 *key,
{
struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablk);
struct qce_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
- unsigned long flags = to_cipher_tmpl(tfm)->alg_flags;
int ret;
if (!key || !keylen)
return -EINVAL;
- if (IS_AES(flags)) {
- switch (keylen) {
- case AES_KEYSIZE_128:
- case AES_KEYSIZE_256:
- break;
- default:
- goto fallback;
- }
- } else if (IS_DES(flags)) {
- u32 tmp[DES_EXPKEY_WORDS];
-
- ret = des_ekey(tmp, key);
- if (!ret && (crypto_ablkcipher_get_flags(ablk) &
- CRYPTO_TFM_REQ_FORBID_WEAK_KEYS))
- goto weakkey;
+ switch (keylen) {
+ case AES_KEYSIZE_128:
+ case AES_KEYSIZE_256:
+ break;
+ default:
+ goto fallback;
}
ctx->enc_keylen = keylen;
@@ -185,24 +175,32 @@ fallback:
if (!ret)
ctx->enc_keylen = keylen;
return ret;
-weakkey:
- crypto_ablkcipher_set_flags(ablk, CRYPTO_TFM_RES_WEAK_KEY);
- return -EINVAL;
+}
+
+static int qce_des_setkey(struct crypto_ablkcipher *ablk, const u8 *key,
+ unsigned int keylen)
+{
+ struct qce_cipher_ctx *ctx = crypto_ablkcipher_ctx(ablk);
+ int err;
+
+ err = verify_ablkcipher_des_key(ablk, key);
+ if (err)
+ return err;
+
+ ctx->enc_keylen = keylen;
+ memcpy(ctx->enc_key, key, keylen);
+ return 0;
}
static int qce_des3_setkey(struct crypto_ablkcipher *ablk, const u8 *key,
unsigned int keylen)
{
struct qce_cipher_ctx *ctx = crypto_ablkcipher_ctx(ablk);
- u32 flags;
int err;
- flags = crypto_ablkcipher_get_flags(ablk);
- err = __des3_verify_key(&flags, key);
- if (unlikely(err)) {
- crypto_ablkcipher_set_flags(ablk, flags);
+ err = verify_ablkcipher_des3_key(ablk, key);
+ if (err)
return err;
- }
ctx->enc_keylen = keylen;
memcpy(ctx->enc_key, key, keylen);
@@ -374,8 +372,9 @@ static int qce_ablkcipher_register_one(const struct qce_ablkcipher_def *def,
alg->cra_ablkcipher.ivsize = def->ivsize;
alg->cra_ablkcipher.min_keysize = def->min_keysize;
alg->cra_ablkcipher.max_keysize = def->max_keysize;
- alg->cra_ablkcipher.setkey = IS_3DES(def->flags) ?
- qce_des3_setkey : qce_ablkcipher_setkey;
+ alg->cra_ablkcipher.setkey = IS_3DES(def->flags) ? qce_des3_setkey :
+ IS_DES(def->flags) ? qce_des_setkey :
+ qce_ablkcipher_setkey;
alg->cra_ablkcipher.encrypt = qce_ablkcipher_encrypt;
alg->cra_ablkcipher.decrypt = qce_ablkcipher_decrypt;
diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c
index ef1d74e8ddb2..08d4ce3bfddf 100644
--- a/drivers/crypto/qce/core.c
+++ b/drivers/crypto/qce/core.c
@@ -167,7 +167,6 @@ static int qce_crypto_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct qce_device *qce;
- struct resource *res;
int ret;
qce = devm_kzalloc(dev, sizeof(*qce), GFP_KERNEL);
@@ -177,8 +176,7 @@ static int qce_crypto_probe(struct platform_device *pdev)
qce->dev = dev;
platform_set_drvdata(pdev, qce);
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- qce->base = devm_ioremap_resource(&pdev->dev, res);
+ qce->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(qce->base))
return PTR_ERR(qce->base);
diff --git a/drivers/crypto/qcom-rng.c b/drivers/crypto/qcom-rng.c
index e54249ccc009..4730f84b646d 100644
--- a/drivers/crypto/qcom-rng.c
+++ b/drivers/crypto/qcom-rng.c
@@ -153,7 +153,6 @@ static struct rng_alg qcom_rng_alg = {
static int qcom_rng_probe(struct platform_device *pdev)
{
- struct resource *res;
struct qcom_rng *rng;
int ret;
@@ -164,8 +163,7 @@ static int qcom_rng_probe(struct platform_device *pdev)
platform_set_drvdata(pdev, rng);
mutex_init(&rng->lock);
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- rng->base = devm_ioremap_resource(&pdev->dev, res);
+ rng->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(rng->base))
return PTR_ERR(rng->base);
diff --git a/drivers/crypto/rockchip/rk3288_crypto.c b/drivers/crypto/rockchip/rk3288_crypto.c
index 8d7e2545e65a..e5714ef24bf2 100644
--- a/drivers/crypto/rockchip/rk3288_crypto.c
+++ b/drivers/crypto/rockchip/rk3288_crypto.c
@@ -311,7 +311,6 @@ MODULE_DEVICE_TABLE(of, crypto_of_id_table);
static int rk_crypto_probe(struct platform_device *pdev)
{
- struct resource *res;
struct device *dev = &pdev->dev;
struct rk_crypto_info *crypto_info;
int err = 0;
@@ -339,8 +338,7 @@ static int rk_crypto_probe(struct platform_device *pdev)
spin_lock_init(&crypto_info->lock);
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- crypto_info->reg = devm_ioremap_resource(&pdev->dev, res);
+ crypto_info->reg = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(crypto_info->reg)) {
err = PTR_ERR(crypto_info->reg);
goto err_crypto;
diff --git a/drivers/crypto/rockchip/rk3288_crypto.h b/drivers/crypto/rockchip/rk3288_crypto.h
index 54ee5b3ed9db..18e2b3f29336 100644
--- a/drivers/crypto/rockchip/rk3288_crypto.h
+++ b/drivers/crypto/rockchip/rk3288_crypto.h
@@ -3,7 +3,7 @@
#define __RK3288_CRYPTO_H__
#include <crypto/aes.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <crypto/algapi.h>
#include <linux/interrupt.h>
#include <linux/delay.h>
diff --git a/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c b/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c
index 96078aaa2098..d0f4b2d18059 100644
--- a/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c
+++ b/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c
@@ -46,15 +46,12 @@ static int rk_aes_setkey(struct crypto_ablkcipher *cipher,
static int rk_des_setkey(struct crypto_ablkcipher *cipher,
const u8 *key, unsigned int keylen)
{
- struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
- struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
- u32 tmp[DES_EXPKEY_WORDS];
+ struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(cipher);
+ int err;
- if (!des_ekey(tmp, key) &&
- (tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
- tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
- return -EINVAL;
- }
+ err = verify_ablkcipher_des_key(cipher, key);
+ if (err)
+ return err;
ctx->keylen = keylen;
memcpy_toio(ctx->dev->reg + RK_CRYPTO_TDES_KEY1_0, key, keylen);
@@ -65,15 +62,11 @@ static int rk_tdes_setkey(struct crypto_ablkcipher *cipher,
const u8 *key, unsigned int keylen)
{
struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(cipher);
- u32 flags;
int err;
- flags = crypto_ablkcipher_get_flags(cipher);
- err = __des3_verify_key(&flags, key);
- if (unlikely(err)) {
- crypto_ablkcipher_set_flags(cipher, flags);
+ err = verify_ablkcipher_des3_key(cipher, key);
+ if (err)
return err;
- }
ctx->keylen = keylen;
memcpy_toio(ctx->dev->reg + RK_CRYPTO_TDES_KEY1_0, key, keylen);
diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c
index 9ef25230c199..010f1bb20dad 100644
--- a/drivers/crypto/s5p-sss.c
+++ b/drivers/crypto/s5p-sss.c
@@ -2056,9 +2056,12 @@ static int s5p_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
struct s5p_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
struct s5p_aes_dev *dev = ctx->dev;
+ if (!req->nbytes)
+ return 0;
+
if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE) &&
((mode & FLAGS_AES_MODE_MASK) != FLAGS_AES_CTR)) {
- dev_err(dev->dev, "request size is not exact amount of AES blocks\n");
+ dev_dbg(dev->dev, "request size is not exact amount of AES blocks\n");
return -EINVAL;
}
@@ -2170,7 +2173,7 @@ static struct crypto_alg algs[] = {
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
- .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_blocksize = 1,
.cra_ctxsize = sizeof(struct s5p_aes_ctx),
.cra_alignmask = 0x0f,
.cra_type = &crypto_ablkcipher_type,
diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c
index 6b498a90181e..8ac8ec6decd5 100644
--- a/drivers/crypto/sahara.c
+++ b/drivers/crypto/sahara.c
@@ -1384,7 +1384,6 @@ MODULE_DEVICE_TABLE(of, sahara_dt_ids);
static int sahara_probe(struct platform_device *pdev)
{
struct sahara_dev *dev;
- struct resource *res;
u32 version;
int irq;
int err;
@@ -1398,17 +1397,14 @@ static int sahara_probe(struct platform_device *pdev)
platform_set_drvdata(pdev, dev);
/* Get the base address */
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- dev->regs_base = devm_ioremap_resource(&pdev->dev, res);
+ dev->regs_base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(dev->regs_base))
return PTR_ERR(dev->regs_base);
/* Get the IRQ */
irq = platform_get_irq(pdev, 0);
- if (irq < 0) {
- dev_err(&pdev->dev, "failed to get irq resource\n");
+ if (irq < 0)
return irq;
- }
err = devm_request_irq(&pdev->dev, irq, sahara_irq_handler,
0, dev_name(&pdev->dev), dev);
diff --git a/drivers/crypto/stm32/Kconfig b/drivers/crypto/stm32/Kconfig
index d6576280fc9b..1aba9372cd23 100644
--- a/drivers/crypto/stm32/Kconfig
+++ b/drivers/crypto/stm32/Kconfig
@@ -25,7 +25,7 @@ config CRYPTO_DEV_STM32_CRYP
depends on ARCH_STM32
select CRYPTO_HASH
select CRYPTO_ENGINE
- select CRYPTO_DES
+ select CRYPTO_LIB_DES
help
This enables support for the CRYP (AES/DES/TDES) hw accelerator which
can be found on STMicroelectronics STM32 SOC.
diff --git a/drivers/crypto/stm32/Makefile b/drivers/crypto/stm32/Makefile
index ce77e38c77e0..518e0e0b11a9 100644
--- a/drivers/crypto/stm32/Makefile
+++ b/drivers/crypto/stm32/Makefile
@@ -1,4 +1,4 @@
# SPDX-License-Identifier: GPL-2.0-only
-obj-$(CONFIG_CRYPTO_DEV_STM32_CRC) += stm32_crc32.o
+obj-$(CONFIG_CRYPTO_DEV_STM32_CRC) += stm32-crc32.o
obj-$(CONFIG_CRYPTO_DEV_STM32_HASH) += stm32-hash.o
obj-$(CONFIG_CRYPTO_DEV_STM32_CRYP) += stm32-cryp.o
diff --git a/drivers/crypto/stm32/stm32_crc32.c b/drivers/crypto/stm32/stm32-crc32.c
index 440c9f1bd006..9e11c3480353 100644
--- a/drivers/crypto/stm32/stm32_crc32.c
+++ b/drivers/crypto/stm32/stm32-crc32.c
@@ -255,7 +255,6 @@ static int stm32_crc_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct stm32_crc *crc;
- struct resource *res;
int ret;
crc = devm_kzalloc(dev, sizeof(*crc), GFP_KERNEL);
@@ -264,8 +263,7 @@ static int stm32_crc_probe(struct platform_device *pdev)
crc->dev = dev;
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- crc->regs = devm_ioremap_resource(dev, res);
+ crc->regs = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(crc->regs)) {
dev_err(dev, "Cannot map CRC IO\n");
return PTR_ERR(crc->regs);
diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c
index 98ae02826e8f..ba5ea6434f9c 100644
--- a/drivers/crypto/stm32/stm32-cryp.c
+++ b/drivers/crypto/stm32/stm32-cryp.c
@@ -15,7 +15,7 @@
#include <linux/reset.h>
#include <crypto/aes.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <crypto/engine.h>
#include <crypto/scatterwalk.h>
#include <crypto/internal/aead.h>
@@ -767,35 +767,15 @@ static int stm32_cryp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
static int stm32_cryp_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
unsigned int keylen)
{
- u32 tmp[DES_EXPKEY_WORDS];
-
- if (keylen != DES_KEY_SIZE)
- return -EINVAL;
-
- if ((crypto_ablkcipher_get_flags(tfm) &
- CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) &&
- unlikely(!des_ekey(tmp, key))) {
- crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY);
- return -EINVAL;
- }
-
- return stm32_cryp_setkey(tfm, key, keylen);
+ return verify_ablkcipher_des_key(tfm, key) ?:
+ stm32_cryp_setkey(tfm, key, keylen);
}
static int stm32_cryp_tdes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
unsigned int keylen)
{
- u32 flags;
- int err;
-
- flags = crypto_ablkcipher_get_flags(tfm);
- err = __des3_verify_key(&flags, key);
- if (unlikely(err)) {
- crypto_ablkcipher_set_flags(tfm, flags);
- return err;
- }
-
- return stm32_cryp_setkey(tfm, key, keylen);
+ return verify_ablkcipher_des3_key(tfm, key) ?:
+ stm32_cryp_setkey(tfm, key, keylen);
}
static int stm32_cryp_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key,
@@ -1955,7 +1935,6 @@ static int stm32_cryp_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct stm32_cryp *cryp;
- struct resource *res;
struct reset_control *rst;
int irq, ret;
@@ -1969,16 +1948,13 @@ static int stm32_cryp_probe(struct platform_device *pdev)
cryp->dev = dev;
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- cryp->regs = devm_ioremap_resource(dev, res);
+ cryp->regs = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(cryp->regs))
return PTR_ERR(cryp->regs);
irq = platform_get_irq(pdev, 0);
- if (irq < 0) {
- dev_err(dev, "Cannot get IRQ resource\n");
+ if (irq < 0)
return irq;
- }
ret = devm_request_threaded_irq(dev, irq, stm32_cryp_irq,
stm32_cryp_irq_thread, IRQF_ONESHOT,
diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c
index 29519d1c403f..cfc8e0e37bee 100644
--- a/drivers/crypto/stm32/stm32-hash.c
+++ b/drivers/crypto/stm32/stm32-hash.c
@@ -338,7 +338,7 @@ static int stm32_hash_xmit_cpu(struct stm32_hash_dev *hdev,
len32 = DIV_ROUND_UP(length, sizeof(u32));
- dev_dbg(hdev->dev, "%s: length: %d, final: %x len32 %i\n",
+ dev_dbg(hdev->dev, "%s: length: %zd, final: %x len32 %i\n",
__func__, length, final, len32);
hdev->flags |= HASH_FLAGS_CPU;
@@ -349,7 +349,7 @@ static int stm32_hash_xmit_cpu(struct stm32_hash_dev *hdev,
return -ETIMEDOUT;
if ((hdev->flags & HASH_FLAGS_HMAC) &&
- (hdev->flags & ~HASH_FLAGS_HMAC_KEY)) {
+ (!(hdev->flags & HASH_FLAGS_HMAC_KEY))) {
hdev->flags |= HASH_FLAGS_HMAC_KEY;
stm32_hash_write_key(hdev);
if (stm32_hash_wait_busy(hdev))
@@ -447,8 +447,8 @@ static int stm32_hash_xmit_dma(struct stm32_hash_dev *hdev,
dma_async_issue_pending(hdev->dma_lch);
- if (!wait_for_completion_interruptible_timeout(&hdev->dma_completion,
- msecs_to_jiffies(100)))
+ if (!wait_for_completion_timeout(&hdev->dma_completion,
+ msecs_to_jiffies(100)))
err = -ETIMEDOUT;
if (dma_async_is_tx_complete(hdev->dma_lch, cookie,
@@ -1450,10 +1450,8 @@ static int stm32_hash_probe(struct platform_device *pdev)
return ret;
irq = platform_get_irq(pdev, 0);
- if (irq < 0) {
- dev_err(dev, "Cannot get IRQ resource\n");
+ if (irq < 0)
return irq;
- }
ret = devm_request_threaded_irq(dev, irq, stm32_hash_irq_handler,
stm32_hash_irq_thread, IRQF_ONESHOT,
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c
index 7b0c42882830..6536fd4bee65 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c
+++ b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c
@@ -8,11 +8,11 @@
* keysize in CBC and ECB mode.
* Add support also for DES and 3DES in CBC and ECB mode.
*
- * You could find the datasheet in Documentation/arm/sunxi/README
+ * You could find the datasheet in Documentation/arm/sunxi.rst
*/
#include "sun4i-ss.h"
-static int sun4i_ss_opti_poll(struct skcipher_request *areq)
+static int noinline_for_stack sun4i_ss_opti_poll(struct skcipher_request *areq)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
@@ -114,6 +114,29 @@ release_ss:
return err;
}
+
+static int noinline_for_stack sun4i_ss_cipher_poll_fallback(struct skcipher_request *areq)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+ struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq);
+ SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, op->fallback_tfm);
+ int err;
+
+ skcipher_request_set_sync_tfm(subreq, op->fallback_tfm);
+ skcipher_request_set_callback(subreq, areq->base.flags, NULL,
+ NULL);
+ skcipher_request_set_crypt(subreq, areq->src, areq->dst,
+ areq->cryptlen, areq->iv);
+ if (ctx->mode & SS_DECRYPTION)
+ err = crypto_skcipher_decrypt(subreq);
+ else
+ err = crypto_skcipher_encrypt(subreq);
+ skcipher_request_zero(subreq);
+
+ return err;
+}
+
/* Generic function that support SG with size not multiple of 4 */
static int sun4i_ss_cipher_poll(struct skcipher_request *areq)
{
@@ -140,8 +163,6 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq)
unsigned int todo;
struct sg_mapping_iter mi, mo;
unsigned int oi, oo; /* offset for in and out */
- char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */
- char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */
unsigned int ob = 0; /* offset in buf */
unsigned int obo = 0; /* offset in bufo*/
unsigned int obl = 0; /* length of data in bufo */
@@ -178,20 +199,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq)
if (no_chunk == 1 && !need_fallback)
return sun4i_ss_opti_poll(areq);
- if (need_fallback) {
- SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, op->fallback_tfm);
- skcipher_request_set_sync_tfm(subreq, op->fallback_tfm);
- skcipher_request_set_callback(subreq, areq->base.flags, NULL,
- NULL);
- skcipher_request_set_crypt(subreq, areq->src, areq->dst,
- areq->cryptlen, areq->iv);
- if (ctx->mode & SS_DECRYPTION)
- err = crypto_skcipher_decrypt(subreq);
- else
- err = crypto_skcipher_encrypt(subreq);
- skcipher_request_zero(subreq);
- return err;
- }
+ if (need_fallback)
+ return sun4i_ss_cipher_poll_fallback(areq);
spin_lock_irqsave(&ss->slock, flags);
@@ -224,6 +233,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq)
while (oleft) {
if (ileft) {
+ char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */
+
/*
* todo is the number of consecutive 4byte word that we
* can read from current SG
@@ -281,6 +292,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq)
oo = 0;
}
} else {
+ char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */
+
/*
* read obl bytes in bufo, we read at maximum for
* emptying the device
@@ -529,25 +542,11 @@ int sun4i_ss_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
unsigned int keylen)
{
struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
- struct sun4i_ss_ctx *ss = op->ss;
- u32 flags;
- u32 tmp[DES_EXPKEY_WORDS];
- int ret;
-
- if (unlikely(keylen != DES_KEY_SIZE)) {
- dev_err(ss->dev, "Invalid keylen %u\n", keylen);
- crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
- return -EINVAL;
- }
-
- flags = crypto_skcipher_get_flags(tfm);
+ int err;
- ret = des_ekey(tmp, key);
- if (unlikely(!ret) && (flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
- crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY);
- dev_dbg(ss->dev, "Weak key %u\n", keylen);
- return -EINVAL;
- }
+ err = verify_skcipher_des_key(tfm, key);
+ if (err)
+ return err;
op->keylen = keylen;
memcpy(op->key, key, keylen);
@@ -565,8 +564,8 @@ int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
int err;
- err = des3_verify_key(tfm, key);
- if (unlikely(err))
+ err = verify_skcipher_des3_key(tfm, key);
+ if (err)
return err;
op->keylen = keylen;
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-core.c b/drivers/crypto/sunxi-ss/sun4i-ss-core.c
index cdcda7f059c8..9aa6fe081a27 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss-core.c
+++ b/drivers/crypto/sunxi-ss/sun4i-ss-core.c
@@ -6,7 +6,7 @@
*
* Core file which registers crypto algorithms supported by the SS.
*
- * You could find a link for the datasheet in Documentation/arm/sunxi/README
+ * You could find a link for the datasheet in Documentation/arm/sunxi.rst
*/
#include <linux/clk.h>
#include <linux/crypto.h>
@@ -225,7 +225,6 @@ static struct sun4i_ss_alg_template ss_algs[] = {
static int sun4i_ss_probe(struct platform_device *pdev)
{
- struct resource *res;
u32 v;
int err, i;
unsigned long cr;
@@ -240,8 +239,7 @@ static int sun4i_ss_probe(struct platform_device *pdev)
if (!ss)
return -ENOMEM;
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- ss->base = devm_ioremap_resource(&pdev->dev, res);
+ ss->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(ss->base)) {
dev_err(&pdev->dev, "Cannot request MMIO\n");
return PTR_ERR(ss->base);
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c
index d2b6d89aad28..fcffba5ef927 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c
+++ b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c
@@ -6,7 +6,7 @@
*
* This file add support for MD5 and SHA1.
*
- * You could find the datasheet in Documentation/arm/sunxi/README
+ * You could find the datasheet in Documentation/arm/sunxi.rst
*/
#include "sun4i-ss.h"
#include <linux/scatterlist.h>
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss.h b/drivers/crypto/sunxi-ss/sun4i-ss.h
index 68b82d1a6303..35a27a7145f8 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss.h
+++ b/drivers/crypto/sunxi-ss/sun4i-ss.h
@@ -8,7 +8,7 @@
* Support MD5 and SHA1 hash algorithms.
* Support DES and 3DES
*
- * You could find the datasheet in Documentation/arm/sunxi/README
+ * You could find the datasheet in Documentation/arm/sunxi.rst
*/
#include <linux/clk.h>
@@ -29,7 +29,7 @@
#include <crypto/internal/hash.h>
#include <crypto/internal/skcipher.h>
#include <crypto/aes.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <crypto/internal/rng.h>
#include <crypto/rng.h>
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
index fbc7bf9d7380..cb6c10b1bf36 100644
--- a/drivers/crypto/talitos.c
+++ b/drivers/crypto/talitos.c
@@ -30,7 +30,7 @@
#include <crypto/algapi.h>
#include <crypto/aes.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <crypto/sha.h>
#include <crypto/md5.h>
#include <crypto/internal/aead.h>
@@ -265,11 +265,11 @@ static int init_device(struct device *dev)
* callback must check err and feedback in descriptor header
* for device processing status.
*/
-int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
- void (*callback)(struct device *dev,
- struct talitos_desc *desc,
- void *context, int error),
- void *context)
+static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
+ void (*callback)(struct device *dev,
+ struct talitos_desc *desc,
+ void *context, int error),
+ void *context)
{
struct talitos_private *priv = dev_get_drvdata(dev);
struct talitos_request *request;
@@ -319,7 +319,21 @@ int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
return -EINPROGRESS;
}
-EXPORT_SYMBOL(talitos_submit);
+
+static __be32 get_request_hdr(struct talitos_request *request, bool is_sec1)
+{
+ struct talitos_edesc *edesc;
+
+ if (!is_sec1)
+ return request->desc->hdr;
+
+ if (!request->desc->next_desc)
+ return request->desc->hdr1;
+
+ edesc = container_of(request->desc, struct talitos_edesc, desc);
+
+ return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1;
+}
/*
* process what was done, notify callback of error if not
@@ -342,12 +356,7 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
/* descriptors with their done bits set don't get the error */
rmb();
- if (!is_sec1)
- hdr = request->desc->hdr;
- else if (request->desc->next_desc)
- hdr = (request->desc + 1)->hdr1;
- else
- hdr = request->desc->hdr1;
+ hdr = get_request_hdr(request, is_sec1);
if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
status = 0;
@@ -477,8 +486,14 @@ static u32 current_desc_hdr(struct device *dev, int ch)
}
}
- if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc)
- return (priv->chan[ch].fifo[iter].desc + 1)->hdr;
+ if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc) {
+ struct talitos_edesc *edesc;
+
+ edesc = container_of(priv->chan[ch].fifo[iter].desc,
+ struct talitos_edesc, desc);
+ return ((struct talitos_desc *)
+ (edesc->buf + edesc->dma_len))->hdr;
+ }
return priv->chan[ch].fifo[iter].desc->hdr;
}
@@ -824,7 +839,11 @@ static void talitos_unregister_rng(struct device *dev)
* HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
*/
#define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1)
+#ifdef CONFIG_CRYPTO_DEV_TALITOS2
#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE)
+#else
+#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE)
+#endif
#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
struct talitos_ctx {
@@ -906,7 +925,6 @@ static int aead_des3_setkey(struct crypto_aead *authenc,
struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
struct device *dev = ctx->dev;
struct crypto_authenc_keys keys;
- u32 flags;
int err;
err = crypto_authenc_extractkeys(&keys, key, keylen);
@@ -917,15 +935,9 @@ static int aead_des3_setkey(struct crypto_aead *authenc,
if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
goto badkey;
- if (keys.enckeylen != DES3_EDE_KEY_SIZE)
- goto badkey;
-
- flags = crypto_aead_get_flags(authenc);
- err = __des3_verify_key(&flags, keys.enckey);
- if (unlikely(err)) {
- crypto_aead_set_flags(authenc, flags);
+ err = verify_aead_des3_key(authenc, keys.enckey, keys.enckeylen);
+ if (err)
goto out;
- }
if (ctx->keylen)
dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
@@ -948,36 +960,6 @@ badkey:
goto out;
}
-/*
- * talitos_edesc - s/w-extended descriptor
- * @src_nents: number of segments in input scatterlist
- * @dst_nents: number of segments in output scatterlist
- * @icv_ool: whether ICV is out-of-line
- * @iv_dma: dma address of iv for checking continuity and link table
- * @dma_len: length of dma mapped link_tbl space
- * @dma_link_tbl: bus physical address of link_tbl/buf
- * @desc: h/w descriptor
- * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
- * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
- *
- * if decrypting (with authcheck), or either one of src_nents or dst_nents
- * is greater than 1, an integrity check value is concatenated to the end
- * of link_tbl data
- */
-struct talitos_edesc {
- int src_nents;
- int dst_nents;
- bool icv_ool;
- dma_addr_t iv_dma;
- int dma_len;
- dma_addr_t dma_link_tbl;
- struct talitos_desc desc;
- union {
- struct talitos_ptr link_tbl[0];
- u8 buf[0];
- };
-};
-
static void talitos_sg_unmap(struct device *dev,
struct talitos_edesc *edesc,
struct scatterlist *src,
@@ -1008,11 +990,13 @@ static void talitos_sg_unmap(struct device *dev,
static void ipsec_esp_unmap(struct device *dev,
struct talitos_edesc *edesc,
- struct aead_request *areq)
+ struct aead_request *areq, bool encrypt)
{
struct crypto_aead *aead = crypto_aead_reqtfm(areq);
struct talitos_ctx *ctx = crypto_aead_ctx(aead);
unsigned int ivsize = crypto_aead_ivsize(aead);
+ unsigned int authsize = crypto_aead_authsize(aead);
+ unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP;
struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3];
@@ -1021,8 +1005,8 @@ static void ipsec_esp_unmap(struct device *dev,
DMA_FROM_DEVICE);
unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE);
- talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen,
- areq->assoclen);
+ talitos_sg_unmap(dev, edesc, areq->src, areq->dst,
+ cryptlen + authsize, areq->assoclen);
if (edesc->dma_len)
dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
@@ -1032,7 +1016,7 @@ static void ipsec_esp_unmap(struct device *dev,
unsigned int dst_nents = edesc->dst_nents ? : 1;
sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize,
- areq->assoclen + areq->cryptlen - ivsize);
+ areq->assoclen + cryptlen - ivsize);
}
}
@@ -1043,31 +1027,14 @@ static void ipsec_esp_encrypt_done(struct device *dev,
struct talitos_desc *desc, void *context,
int err)
{
- struct talitos_private *priv = dev_get_drvdata(dev);
- bool is_sec1 = has_ftr_sec1(priv);
struct aead_request *areq = context;
struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
- unsigned int authsize = crypto_aead_authsize(authenc);
unsigned int ivsize = crypto_aead_ivsize(authenc);
struct talitos_edesc *edesc;
- struct scatterlist *sg;
- void *icvdata;
edesc = container_of(desc, struct talitos_edesc, desc);
- ipsec_esp_unmap(dev, edesc, areq);
-
- /* copy the generated ICV to dst */
- if (edesc->icv_ool) {
- if (is_sec1)
- icvdata = edesc->buf + areq->assoclen + areq->cryptlen;
- else
- icvdata = &edesc->link_tbl[edesc->src_nents +
- edesc->dst_nents + 2];
- sg = sg_last(areq->dst, edesc->dst_nents);
- memcpy((char *)sg_virt(sg) + sg->length - authsize,
- icvdata, authsize);
- }
+ ipsec_esp_unmap(dev, edesc, areq, true);
dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);
@@ -1084,32 +1051,16 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev,
struct crypto_aead *authenc = crypto_aead_reqtfm(req);
unsigned int authsize = crypto_aead_authsize(authenc);
struct talitos_edesc *edesc;
- struct scatterlist *sg;
char *oicv, *icv;
- struct talitos_private *priv = dev_get_drvdata(dev);
- bool is_sec1 = has_ftr_sec1(priv);
edesc = container_of(desc, struct talitos_edesc, desc);
- ipsec_esp_unmap(dev, edesc, req);
+ ipsec_esp_unmap(dev, edesc, req, false);
if (!err) {
/* auth check */
- sg = sg_last(req->dst, edesc->dst_nents ? : 1);
- icv = (char *)sg_virt(sg) + sg->length - authsize;
-
- if (edesc->dma_len) {
- if (is_sec1)
- oicv = (char *)&edesc->dma_link_tbl +
- req->assoclen + req->cryptlen;
- else
- oicv = (char *)
- &edesc->link_tbl[edesc->src_nents +
- edesc->dst_nents + 2];
- if (edesc->icv_ool)
- icv = oicv + authsize;
- } else
- oicv = (char *)&edesc->link_tbl[0];
+ oicv = edesc->buf + edesc->dma_len;
+ icv = oicv - authsize;
err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
}
@@ -1128,7 +1079,7 @@ static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
edesc = container_of(desc, struct talitos_edesc, desc);
- ipsec_esp_unmap(dev, edesc, req);
+ ipsec_esp_unmap(dev, edesc, req, false);
/* check ICV auth status */
if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
@@ -1145,11 +1096,12 @@ static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
* stop at cryptlen bytes
*/
static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
- unsigned int offset, int cryptlen,
+ unsigned int offset, int datalen, int elen,
struct talitos_ptr *link_tbl_ptr)
{
- int n_sg = sg_count;
+ int n_sg = elen ? sg_count + 1 : sg_count;
int count = 0;
+ int cryptlen = datalen + elen;
while (cryptlen && sg && n_sg--) {
unsigned int len = sg_dma_len(sg);
@@ -1164,11 +1116,20 @@ static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
if (len > cryptlen)
len = cryptlen;
+ if (datalen > 0 && len > datalen) {
+ to_talitos_ptr(link_tbl_ptr + count,
+ sg_dma_address(sg) + offset, datalen, 0);
+ to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
+ count++;
+ len -= datalen;
+ offset += datalen;
+ }
to_talitos_ptr(link_tbl_ptr + count,
sg_dma_address(sg) + offset, len, 0);
to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
count++;
cryptlen -= len;
+ datalen -= len;
offset = 0;
next:
@@ -1178,7 +1139,7 @@ next:
/* tag end of link table */
if (count > 0)
to_talitos_ptr_ext_set(link_tbl_ptr + count - 1,
- DESC_PTR_LNKTBL_RETURN, 0);
+ DESC_PTR_LNKTBL_RET, 0);
return count;
}
@@ -1186,7 +1147,8 @@ next:
static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src,
unsigned int len, struct talitos_edesc *edesc,
struct talitos_ptr *ptr, int sg_count,
- unsigned int offset, int tbl_off, int elen)
+ unsigned int offset, int tbl_off, int elen,
+ bool force)
{
struct talitos_private *priv = dev_get_drvdata(dev);
bool is_sec1 = has_ftr_sec1(priv);
@@ -1196,7 +1158,7 @@ static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src,
return 1;
}
to_talitos_ptr_ext_set(ptr, elen, is_sec1);
- if (sg_count == 1) {
+ if (sg_count == 1 && !force) {
to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1);
return sg_count;
}
@@ -1204,9 +1166,9 @@ static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src,
to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1);
return sg_count;
}
- sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len + elen,
+ sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, elen,
&edesc->link_tbl[tbl_off]);
- if (sg_count == 1) {
+ if (sg_count == 1 && !force) {
/* Only one segment now, so no link tbl needed*/
copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);
return sg_count;
@@ -1224,13 +1186,14 @@ static int talitos_sg_map(struct device *dev, struct scatterlist *src,
unsigned int offset, int tbl_off)
{
return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset,
- tbl_off, 0);
+ tbl_off, 0, false);
}
/*
* fill in and submit ipsec_esp descriptor
*/
static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
+ bool encrypt,
void (*callback)(struct device *dev,
struct talitos_desc *desc,
void *context, int error))
@@ -1240,7 +1203,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
struct talitos_ctx *ctx = crypto_aead_ctx(aead);
struct device *dev = ctx->dev;
struct talitos_desc *desc = &edesc->desc;
- unsigned int cryptlen = areq->cryptlen;
+ unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
unsigned int ivsize = crypto_aead_ivsize(aead);
int tbl_off = 0;
int sg_count, ret;
@@ -1251,6 +1214,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP;
struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3];
struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2];
+ dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize;
/* hmac key */
to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1);
@@ -1290,7 +1254,8 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
elen = authsize;
ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4],
- sg_count, areq->assoclen, tbl_off, elen);
+ sg_count, areq->assoclen, tbl_off, elen,
+ false);
if (ret > 1) {
tbl_off += ret;
@@ -1304,55 +1269,32 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
}
- ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],
- sg_count, areq->assoclen, tbl_off);
-
- if (is_ipsec_esp)
- to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1);
-
- /* ICV data */
- if (ret > 1) {
- tbl_off += ret;
- edesc->icv_ool = true;
- sync_needed = true;
-
- if (is_ipsec_esp) {
- struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
- int offset = (edesc->src_nents + edesc->dst_nents + 2) *
- sizeof(struct talitos_ptr) + authsize;
-
- /* Add an entry to the link table for ICV data */
- to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1);
- to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN,
- is_sec1);
+ if (is_ipsec_esp && encrypt)
+ elen = authsize;
+ else
+ elen = 0;
+ ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],
+ sg_count, areq->assoclen, tbl_off, elen,
+ is_ipsec_esp && !encrypt);
+ tbl_off += ret;
- /* icv data follows link tables */
- to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset,
- authsize, is_sec1);
- } else {
- dma_addr_t addr = edesc->dma_link_tbl;
+ if (!encrypt && is_ipsec_esp) {
+ struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
- if (is_sec1)
- addr += areq->assoclen + cryptlen;
- else
- addr += sizeof(struct talitos_ptr) * tbl_off;
+ /* Add an entry to the link table for ICV data */
+ to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1);
+ to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RET, is_sec1);
- to_talitos_ptr(&desc->ptr[6], addr, authsize, is_sec1);
- }
+ /* icv data follows link tables */
+ to_talitos_ptr(tbl_ptr, dma_icv, authsize, is_sec1);
+ to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1);
+ sync_needed = true;
+ } else if (!encrypt) {
+ to_talitos_ptr(&desc->ptr[6], dma_icv, authsize, is_sec1);
+ sync_needed = true;
} else if (!is_ipsec_esp) {
- ret = talitos_sg_map(dev, areq->dst, authsize, edesc,
- &desc->ptr[6], sg_count, areq->assoclen +
- cryptlen,
- tbl_off);
- if (ret > 1) {
- tbl_off += ret;
- edesc->icv_ool = true;
- sync_needed = true;
- } else {
- edesc->icv_ool = false;
- }
- } else {
- edesc->icv_ool = false;
+ talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6],
+ sg_count, areq->assoclen + cryptlen, tbl_off);
}
/* iv out */
@@ -1367,7 +1309,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
if (ret != -EINPROGRESS) {
- ipsec_esp_unmap(dev, edesc, areq);
+ ipsec_esp_unmap(dev, edesc, areq, encrypt);
kfree(edesc);
}
return ret;
@@ -1435,18 +1377,18 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
* and space for two sets of ICVs (stashed and generated)
*/
alloc_len = sizeof(struct talitos_edesc);
- if (src_nents || dst_nents) {
+ if (src_nents || dst_nents || !encrypt) {
if (is_sec1)
dma_len = (src_nents ? src_len : 0) +
- (dst_nents ? dst_len : 0);
+ (dst_nents ? dst_len : 0) + authsize;
else
dma_len = (src_nents + dst_nents + 2) *
- sizeof(struct talitos_ptr) + authsize * 2;
+ sizeof(struct talitos_ptr) + authsize;
alloc_len += dma_len;
} else {
dma_len = 0;
- alloc_len += icv_stashing ? authsize : 0;
}
+ alloc_len += icv_stashing ? authsize : 0;
/* if its a ahash, add space for a second desc next to the first one */
if (is_sec1 && !dst)
@@ -1466,15 +1408,11 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
edesc->dst_nents = dst_nents;
edesc->iv_dma = iv_dma;
edesc->dma_len = dma_len;
- if (dma_len) {
- void *addr = &edesc->link_tbl[0];
-
- if (is_sec1 && !dst)
- addr += sizeof(struct talitos_desc);
- edesc->dma_link_tbl = dma_map_single(dev, addr,
+ if (dma_len)
+ edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
edesc->dma_len,
DMA_BIDIRECTIONAL);
- }
+
return edesc;
}
@@ -1485,9 +1423,10 @@ static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
unsigned int authsize = crypto_aead_authsize(authenc);
struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
unsigned int ivsize = crypto_aead_ivsize(authenc);
+ unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
- iv, areq->assoclen, areq->cryptlen,
+ iv, areq->assoclen, cryptlen,
authsize, ivsize, icv_stashing,
areq->base.flags, encrypt);
}
@@ -1506,7 +1445,7 @@ static int aead_encrypt(struct aead_request *req)
/* set encrypt */
edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
- return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
+ return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done);
}
static int aead_decrypt(struct aead_request *req)
@@ -1516,17 +1455,15 @@ static int aead_decrypt(struct aead_request *req)
struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
struct talitos_private *priv = dev_get_drvdata(ctx->dev);
struct talitos_edesc *edesc;
- struct scatterlist *sg;
void *icvdata;
- req->cryptlen -= authsize;
-
/* allocate extended descriptor */
edesc = aead_edesc_alloc(req, req->iv, 1, false);
if (IS_ERR(edesc))
return PTR_ERR(edesc);
- if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
+ if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) &&
+ (priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
((!edesc->src_nents && !edesc->dst_nents) ||
priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
@@ -1537,24 +1474,20 @@ static int aead_decrypt(struct aead_request *req)
/* reset integrity check result bits */
- return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
+ return ipsec_esp(edesc, req, false,
+ ipsec_esp_decrypt_hwauth_done);
}
/* Have to check the ICV with software */
edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
/* stash incoming ICV for later cmp with ICV generated by the h/w */
- if (edesc->dma_len)
- icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
- edesc->dst_nents + 2];
- else
- icvdata = &edesc->link_tbl[0];
-
- sg = sg_last(req->src, edesc->src_nents ? : 1);
+ icvdata = edesc->buf + edesc->dma_len;
- memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
+ sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize,
+ req->assoclen + req->cryptlen - authsize);
- return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
+ return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done);
}
static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
@@ -1577,32 +1510,27 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
static int ablkcipher_des_setkey(struct crypto_ablkcipher *cipher,
const u8 *key, unsigned int keylen)
{
- u32 tmp[DES_EXPKEY_WORDS];
-
- if (unlikely(crypto_ablkcipher_get_flags(cipher) &
- CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) &&
- !des_ekey(tmp, key)) {
- crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_WEAK_KEY);
- return -EINVAL;
- }
-
- return ablkcipher_setkey(cipher, key, keylen);
+ return verify_ablkcipher_des_key(cipher, key) ?:
+ ablkcipher_setkey(cipher, key, keylen);
}
static int ablkcipher_des3_setkey(struct crypto_ablkcipher *cipher,
const u8 *key, unsigned int keylen)
{
- u32 flags;
- int err;
+ return verify_ablkcipher_des3_key(cipher, key) ?:
+ ablkcipher_setkey(cipher, key, keylen);
+}
- flags = crypto_ablkcipher_get_flags(cipher);
- err = __des3_verify_key(&flags, key);
- if (unlikely(err)) {
- crypto_ablkcipher_set_flags(cipher, flags);
- return err;
- }
+static int ablkcipher_aes_setkey(struct crypto_ablkcipher *cipher,
+ const u8 *key, unsigned int keylen)
+{
+ if (keylen == AES_KEYSIZE_128 || keylen == AES_KEYSIZE_192 ||
+ keylen == AES_KEYSIZE_256)
+ return ablkcipher_setkey(cipher, key, keylen);
- return ablkcipher_setkey(cipher, key, keylen);
+ crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+
+ return -EINVAL;
}
static void common_nonsnoop_unmap(struct device *dev,
@@ -1624,11 +1552,15 @@ static void ablkcipher_done(struct device *dev,
int err)
{
struct ablkcipher_request *areq = context;
+ struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
+ struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
+ unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
struct talitos_edesc *edesc;
edesc = container_of(desc, struct talitos_edesc, desc);
common_nonsnoop_unmap(dev, edesc, areq);
+ memcpy(areq->info, ctx->iv, ivsize);
kfree(edesc);
@@ -1723,6 +1655,14 @@ static int ablkcipher_encrypt(struct ablkcipher_request *areq)
struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
struct talitos_edesc *edesc;
+ unsigned int blocksize =
+ crypto_tfm_alg_blocksize(crypto_ablkcipher_tfm(cipher));
+
+ if (!areq->nbytes)
+ return 0;
+
+ if (areq->nbytes % blocksize)
+ return -EINVAL;
/* allocate extended descriptor */
edesc = ablkcipher_edesc_alloc(areq, true);
@@ -1740,6 +1680,14 @@ static int ablkcipher_decrypt(struct ablkcipher_request *areq)
struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
struct talitos_edesc *edesc;
+ unsigned int blocksize =
+ crypto_tfm_alg_blocksize(crypto_ablkcipher_tfm(cipher));
+
+ if (!areq->nbytes)
+ return 0;
+
+ if (areq->nbytes % blocksize)
+ return -EINVAL;
/* allocate extended descriptor */
edesc = ablkcipher_edesc_alloc(areq, false);
@@ -1759,14 +1707,16 @@ static void common_nonsnoop_hash_unmap(struct device *dev,
struct talitos_private *priv = dev_get_drvdata(dev);
bool is_sec1 = has_ftr_sec1(priv);
struct talitos_desc *desc = &edesc->desc;
- struct talitos_desc *desc2 = desc + 1;
+ struct talitos_desc *desc2 = (struct talitos_desc *)
+ (edesc->buf + edesc->dma_len);
unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
if (desc->next_desc &&
desc->ptr[5].ptr != desc2->ptr[5].ptr)
unmap_single_talitos_ptr(dev, &desc2->ptr[5], DMA_FROM_DEVICE);
- talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);
+ if (req_ctx->psrc)
+ talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);
/* When using hashctx-in, must unmap it. */
if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
@@ -1833,7 +1783,6 @@ static void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
static int common_nonsnoop_hash(struct talitos_edesc *edesc,
struct ahash_request *areq, unsigned int length,
- unsigned int offset,
void (*callback) (struct device *dev,
struct talitos_desc *desc,
void *context, int error))
@@ -1872,9 +1821,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc,
sg_count = edesc->src_nents ?: 1;
if (is_sec1 && sg_count > 1)
- sg_pcopy_to_buffer(req_ctx->psrc, sg_count,
- edesc->buf + sizeof(struct talitos_desc),
- length, req_ctx->nbuf);
+ sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length);
else if (length)
sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count,
DMA_TO_DEVICE);
@@ -1887,7 +1834,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc,
DMA_TO_DEVICE);
} else {
sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
- &desc->ptr[3], sg_count, offset, 0);
+ &desc->ptr[3], sg_count, 0, 0);
if (sg_count > 1)
sync_needed = true;
}
@@ -1911,7 +1858,8 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc,
talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
if (is_sec1 && req_ctx->nbuf && length) {
- struct talitos_desc *desc2 = desc + 1;
+ struct talitos_desc *desc2 = (struct talitos_desc *)
+ (edesc->buf + edesc->dma_len);
dma_addr_t next_desc;
memset(desc2, 0, sizeof(*desc2));
@@ -1932,7 +1880,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc,
DMA_TO_DEVICE);
copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1);
sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
- &desc2->ptr[3], sg_count, offset, 0);
+ &desc2->ptr[3], sg_count, 0, 0);
if (sg_count > 1)
sync_needed = true;
copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1);
@@ -2043,7 +1991,6 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
struct device *dev = ctx->dev;
struct talitos_private *priv = dev_get_drvdata(dev);
bool is_sec1 = has_ftr_sec1(priv);
- int offset = 0;
u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx];
if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
@@ -2083,6 +2030,8 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
sg_chain(req_ctx->bufsl, 2, areq->src);
req_ctx->psrc = req_ctx->bufsl;
} else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) {
+ int offset;
+
if (nbytes_to_hash > blocksize)
offset = blocksize - req_ctx->nbuf;
else
@@ -2095,7 +2044,8 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
sg_copy_to_buffer(areq->src, nents,
ctx_buf + req_ctx->nbuf, offset);
req_ctx->nbuf += offset;
- req_ctx->psrc = areq->src;
+ req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, areq->src,
+ offset);
} else
req_ctx->psrc = areq->src;
@@ -2135,8 +2085,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
if (ctx->keylen && (req_ctx->first || req_ctx->last))
edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
- return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, offset,
- ahash_done);
+ return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, ahash_done);
}
static int ahash_update(struct ahash_request *areq)
@@ -2339,7 +2288,7 @@ static struct talitos_alg_template driver_algs[] = {
.base = {
.cra_name = "authenc(hmac(sha1),cbc(aes))",
.cra_driver_name = "authenc-hmac-sha1-"
- "cbc-aes-talitos",
+ "cbc-aes-talitos-hsna",
.cra_blocksize = AES_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_ASYNC,
},
@@ -2384,7 +2333,7 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "authenc(hmac(sha1),"
"cbc(des3_ede))",
.cra_driver_name = "authenc-hmac-sha1-"
- "cbc-3des-talitos",
+ "cbc-3des-talitos-hsna",
.cra_blocksize = DES3_EDE_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_ASYNC,
},
@@ -2427,7 +2376,7 @@ static struct talitos_alg_template driver_algs[] = {
.base = {
.cra_name = "authenc(hmac(sha224),cbc(aes))",
.cra_driver_name = "authenc-hmac-sha224-"
- "cbc-aes-talitos",
+ "cbc-aes-talitos-hsna",
.cra_blocksize = AES_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_ASYNC,
},
@@ -2472,7 +2421,7 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "authenc(hmac(sha224),"
"cbc(des3_ede))",
.cra_driver_name = "authenc-hmac-sha224-"
- "cbc-3des-talitos",
+ "cbc-3des-talitos-hsna",
.cra_blocksize = DES3_EDE_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_ASYNC,
},
@@ -2515,7 +2464,7 @@ static struct talitos_alg_template driver_algs[] = {
.base = {
.cra_name = "authenc(hmac(sha256),cbc(aes))",
.cra_driver_name = "authenc-hmac-sha256-"
- "cbc-aes-talitos",
+ "cbc-aes-talitos-hsna",
.cra_blocksize = AES_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_ASYNC,
},
@@ -2560,7 +2509,7 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "authenc(hmac(sha256),"
"cbc(des3_ede))",
.cra_driver_name = "authenc-hmac-sha256-"
- "cbc-3des-talitos",
+ "cbc-3des-talitos-hsna",
.cra_blocksize = DES3_EDE_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_ASYNC,
},
@@ -2689,7 +2638,7 @@ static struct talitos_alg_template driver_algs[] = {
.base = {
.cra_name = "authenc(hmac(md5),cbc(aes))",
.cra_driver_name = "authenc-hmac-md5-"
- "cbc-aes-talitos",
+ "cbc-aes-talitos-hsna",
.cra_blocksize = AES_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_ASYNC,
},
@@ -2732,7 +2681,7 @@ static struct talitos_alg_template driver_algs[] = {
.base = {
.cra_name = "authenc(hmac(md5),cbc(des3_ede))",
.cra_driver_name = "authenc-hmac-md5-"
- "cbc-3des-talitos",
+ "cbc-3des-talitos-hsna",
.cra_blocksize = DES3_EDE_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_ASYNC,
},
@@ -2760,7 +2709,7 @@ static struct talitos_alg_template driver_algs[] = {
.cra_ablkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
+ .setkey = ablkcipher_aes_setkey,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2777,6 +2726,7 @@ static struct talitos_alg_template driver_algs[] = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
+ .setkey = ablkcipher_aes_setkey,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2787,13 +2737,14 @@ static struct talitos_alg_template driver_algs[] = {
.alg.crypto = {
.cra_name = "ctr(aes)",
.cra_driver_name = "ctr-aes-talitos",
- .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_blocksize = 1,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
CRYPTO_ALG_ASYNC,
.cra_ablkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
+ .setkey = ablkcipher_aes_setkey,
}
},
.desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP |
@@ -2810,7 +2761,6 @@ static struct talitos_alg_template driver_algs[] = {
.cra_ablkcipher = {
.min_keysize = DES_KEY_SIZE,
.max_keysize = DES_KEY_SIZE,
- .ivsize = DES_BLOCK_SIZE,
.setkey = ablkcipher_des_setkey,
}
},
@@ -2845,7 +2795,6 @@ static struct talitos_alg_template driver_algs[] = {
.cra_ablkcipher = {
.min_keysize = DES3_EDE_KEY_SIZE,
.max_keysize = DES3_EDE_KEY_SIZE,
- .ivsize = DES3_EDE_BLOCK_SIZE,
.setkey = ablkcipher_des3_setkey,
}
},
@@ -3270,7 +3219,10 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
alg->cra_priority = t_alg->algt.priority;
else
alg->cra_priority = TALITOS_CRA_PRIORITY;
- alg->cra_alignmask = 0;
+ if (has_ftr_sec1(priv))
+ alg->cra_alignmask = 3;
+ else
+ alg->cra_alignmask = 0;
alg->cra_ctxsize = sizeof(struct talitos_ctx);
alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
@@ -3418,7 +3370,7 @@ static int talitos_probe(struct platform_device *ofdev)
if (err)
goto err_out;
- if (of_device_is_compatible(np, "fsl,sec1.0")) {
+ if (has_ftr_sec1(priv)) {
if (priv->num_channels == 1)
tasklet_init(&priv->done_task[0], talitos1_done_ch0,
(unsigned long)dev);
diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h
index a65a63e0d6c1..1469b956948a 100644
--- a/drivers/crypto/talitos.h
+++ b/drivers/crypto/talitos.h
@@ -1,31 +1,8 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
/*
* Freescale SEC (talitos) device register and descriptor header defines
*
* Copyright (c) 2006-2011 Freescale Semiconductor, Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote products
- * derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
*/
#define TALITOS_TIMEOUT 100000
@@ -65,6 +42,34 @@ struct talitos_desc {
#define TALITOS_DESC_SIZE (sizeof(struct talitos_desc) - sizeof(__be32))
+/*
+ * talitos_edesc - s/w-extended descriptor
+ * @src_nents: number of segments in input scatterlist
+ * @dst_nents: number of segments in output scatterlist
+ * @iv_dma: dma address of iv for checking continuity and link table
+ * @dma_len: length of dma mapped link_tbl space
+ * @dma_link_tbl: bus physical address of link_tbl/buf
+ * @desc: h/w descriptor
+ * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
+ * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
+ *
+ * if decrypting (with authcheck), or either one of src_nents or dst_nents
+ * is greater than 1, an integrity check value is concatenated to the end
+ * of link_tbl data
+ */
+struct talitos_edesc {
+ int src_nents;
+ int dst_nents;
+ dma_addr_t iv_dma;
+ int dma_len;
+ dma_addr_t dma_link_tbl;
+ struct talitos_desc desc;
+ union {
+ struct talitos_ptr link_tbl[0];
+ u8 buf[0];
+ };
+};
+
/**
* talitos_request - descriptor submission request
* @desc: descriptor pointer (kernel virtual)
@@ -150,12 +155,6 @@ struct talitos_private {
bool rng_registered;
};
-extern int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
- void (*callback)(struct device *dev,
- struct talitos_desc *desc,
- void *context, int error),
- void *context);
-
/* .features flag */
#define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001
#define TALITOS_FTR_HW_AUTH_CHECK 0x00000002
@@ -170,13 +169,11 @@ extern int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
*/
static inline bool has_ftr_sec1(struct talitos_private *priv)
{
-#if defined(CONFIG_CRYPTO_DEV_TALITOS1) && defined(CONFIG_CRYPTO_DEV_TALITOS2)
- return priv->features & TALITOS_FTR_SEC1 ? true : false;
-#elif defined(CONFIG_CRYPTO_DEV_TALITOS1)
- return true;
-#else
- return false;
-#endif
+ if (IS_ENABLED(CONFIG_CRYPTO_DEV_TALITOS1) &&
+ IS_ENABLED(CONFIG_CRYPTO_DEV_TALITOS2))
+ return priv->features & TALITOS_FTR_SEC1;
+
+ return IS_ENABLED(CONFIG_CRYPTO_DEV_TALITOS1);
}
/*
@@ -412,5 +409,5 @@ static inline bool has_ftr_sec1(struct talitos_private *priv)
/* link table extent field bits */
#define DESC_PTR_LNKTBL_JUMP 0x80
-#define DESC_PTR_LNKTBL_RETURN 0x02
+#define DESC_PTR_LNKTBL_RET 0x02
#define DESC_PTR_LNKTBL_NEXT 0x01
diff --git a/drivers/crypto/ux500/Kconfig b/drivers/crypto/ux500/Kconfig
index 349d34eaac13..b1c6f739f77b 100644
--- a/drivers/crypto/ux500/Kconfig
+++ b/drivers/crypto/ux500/Kconfig
@@ -9,7 +9,7 @@ config CRYPTO_DEV_UX500_CRYP
depends on CRYPTO_DEV_UX500
select CRYPTO_ALGAPI
select CRYPTO_BLKCIPHER
- select CRYPTO_DES
+ select CRYPTO_LIB_DES
help
This selects the crypto driver for the UX500_CRYP hardware. It supports
AES-ECB, CBC and CTR with keys sizes of 128, 192 and 256 bit sizes.
diff --git a/drivers/crypto/ux500/cryp/cryp.c b/drivers/crypto/ux500/cryp/cryp.c
index ece83a363e11..f22f6fa612b3 100644
--- a/drivers/crypto/ux500/cryp/cryp.c
+++ b/drivers/crypto/ux500/cryp/cryp.c
@@ -314,14 +314,17 @@ void cryp_save_device_context(struct cryp_device_data *device_data,
case CRYP_KEY_SIZE_256:
ctx->key_4_l = readl_relaxed(&src_reg->key_4_l);
ctx->key_4_r = readl_relaxed(&src_reg->key_4_r);
+ /* Fall through */
case CRYP_KEY_SIZE_192:
ctx->key_3_l = readl_relaxed(&src_reg->key_3_l);
ctx->key_3_r = readl_relaxed(&src_reg->key_3_r);
+ /* Fall through */
case CRYP_KEY_SIZE_128:
ctx->key_2_l = readl_relaxed(&src_reg->key_2_l);
ctx->key_2_r = readl_relaxed(&src_reg->key_2_r);
+ /* Fall through */
default:
ctx->key_1_l = readl_relaxed(&src_reg->key_1_l);
@@ -361,14 +364,17 @@ void cryp_restore_device_context(struct cryp_device_data *device_data,
case CRYP_KEY_SIZE_256:
writel_relaxed(ctx->key_4_l, &reg->key_4_l);
writel_relaxed(ctx->key_4_r, &reg->key_4_r);
+ /* Fall through */
case CRYP_KEY_SIZE_192:
writel_relaxed(ctx->key_3_l, &reg->key_3_l);
writel_relaxed(ctx->key_3_r, &reg->key_3_r);
+ /* Fall through */
case CRYP_KEY_SIZE_128:
writel_relaxed(ctx->key_2_l, &reg->key_2_l);
writel_relaxed(ctx->key_2_r, &reg->key_2_r);
+ /* Fall through */
default:
writel_relaxed(ctx->key_1_l, &reg->key_1_l);
diff --git a/drivers/crypto/ux500/cryp/cryp.h b/drivers/crypto/ux500/cryp/cryp.h
index bd89504e8167..8da7f87b339b 100644
--- a/drivers/crypto/ux500/cryp/cryp.h
+++ b/drivers/crypto/ux500/cryp/cryp.h
@@ -241,12 +241,12 @@ struct cryp_device_data {
struct clk *clk;
struct regulator *pwr_regulator;
int power_status;
- struct spinlock ctx_lock;
+ spinlock_t ctx_lock;
struct cryp_ctx *current_ctx;
struct klist_node list_node;
struct cryp_dma dma;
bool power_state;
- struct spinlock power_state_spinlock;
+ spinlock_t power_state_spinlock;
bool restore_dev_ctx;
};
diff --git a/drivers/crypto/ux500/cryp/cryp_core.c b/drivers/crypto/ux500/cryp/cryp_core.c
index b4beb54c0dbe..1628ae7a1467 100644
--- a/drivers/crypto/ux500/cryp/cryp_core.c
+++ b/drivers/crypto/ux500/cryp/cryp_core.c
@@ -29,7 +29,7 @@
#include <crypto/aes.h>
#include <crypto/algapi.h>
#include <crypto/ctr.h>
-#include <crypto/des.h>
+#include <crypto/internal/des.h>
#include <crypto/scatterwalk.h>
#include <linux/platform_data/crypto-ux500.h>
@@ -528,9 +528,9 @@ static int cryp_set_dma_transfer(struct cryp_ctx *ctx,
dev_dbg(ctx->device->dev, "[%s]: ", __func__);
- if (unlikely(!IS_ALIGNED((u32)sg, 4))) {
+ if (unlikely(!IS_ALIGNED((unsigned long)sg, 4))) {
dev_err(ctx->device->dev, "[%s]: Data in sg list isn't "
- "aligned! Addr: 0x%08x", __func__, (u32)sg);
+ "aligned! Addr: 0x%08lx", __func__, (unsigned long)sg);
return -EFAULT;
}
@@ -763,9 +763,9 @@ static int hw_crypt_noxts(struct cryp_ctx *ctx,
ctx->outlen = ctx->datalen;
- if (unlikely(!IS_ALIGNED((u32)indata, 4))) {
+ if (unlikely(!IS_ALIGNED((unsigned long)indata, 4))) {
pr_debug(DEV_DBG_NAME " [%s]: Data isn't aligned! Addr: "
- "0x%08x", __func__, (u32)indata);
+ "0x%08lx", __func__, (unsigned long)indata);
return -EINVAL;
}
@@ -987,26 +987,13 @@ static int des_ablkcipher_setkey(struct crypto_ablkcipher *cipher,
const u8 *key, unsigned int keylen)
{
struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
- u32 *flags = &cipher->base.crt_flags;
- u32 tmp[DES_EXPKEY_WORDS];
- int ret;
+ int err;
pr_debug(DEV_DBG_NAME " [%s]", __func__);
- if (keylen != DES_KEY_SIZE) {
- *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
- pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_RES_BAD_KEY_LEN",
- __func__);
- return -EINVAL;
- }
- ret = des_ekey(tmp, key);
- if (unlikely(ret == 0) &&
- (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
- *flags |= CRYPTO_TFM_RES_WEAK_KEY;
- pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_RES_WEAK_KEY",
- __func__);
- return -EINVAL;
- }
+ err = verify_ablkcipher_des_key(cipher, key);
+ if (err)
+ return err;
memcpy(ctx->key, key, keylen);
ctx->keylen = keylen;
@@ -1019,17 +1006,13 @@ static int des3_ablkcipher_setkey(struct crypto_ablkcipher *cipher,
const u8 *key, unsigned int keylen)
{
struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
- u32 flags;
int err;
pr_debug(DEV_DBG_NAME " [%s]", __func__);
- flags = crypto_ablkcipher_get_flags(cipher);
- err = __des3_verify_key(&flags, key);
- if (unlikely(err)) {
- crypto_ablkcipher_set_flags(cipher, flags);
+ err = verify_ablkcipher_des3_key(cipher, key);
+ if (err)
return err;
- }
memcpy(ctx->key, key, keylen);
ctx->keylen = keylen;
diff --git a/drivers/crypto/ux500/hash/hash_alg.h b/drivers/crypto/ux500/hash/hash_alg.h
index ab2bd00c1c36..7c9bcc15125f 100644
--- a/drivers/crypto/ux500/hash/hash_alg.h
+++ b/drivers/crypto/ux500/hash/hash_alg.h
@@ -366,10 +366,10 @@ struct hash_device_data {
phys_addr_t phybase;
struct klist_node list_node;
struct device *dev;
- struct spinlock ctx_lock;
+ spinlock_t ctx_lock;
struct hash_ctx *current_ctx;
bool power_state;
- struct spinlock power_state_lock;
+ spinlock_t power_state_lock;
struct regulator *regulator;
struct clk *clk;
bool restore_dev_state;
diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c
index f1ebc3dfa21e..c172a6953477 100644
--- a/drivers/crypto/ux500/hash/hash_core.c
+++ b/drivers/crypto/ux500/hash/hash_core.c
@@ -806,7 +806,7 @@ static int hash_process_data(struct hash_device_data *device_data,
* HW peripheral, otherwise we first copy data
* to a local buffer
*/
- if ((0 == (((u32)data_buffer) % 4)) &&
+ if (IS_ALIGNED((unsigned long)data_buffer, 4) &&
(0 == *index))
hash_processblock(device_data,
(const u32 *)data_buffer,
@@ -864,7 +864,8 @@ static int hash_dma_final(struct ahash_request *req)
if (ret)
return ret;
- dev_dbg(device_data->dev, "%s: (ctx=0x%x)!\n", __func__, (u32) ctx);
+ dev_dbg(device_data->dev, "%s: (ctx=0x%lx)!\n", __func__,
+ (unsigned long)ctx);
if (req_ctx->updated) {
ret = hash_resume_state(device_data, &device_data->state);
@@ -969,7 +970,8 @@ static int hash_hw_final(struct ahash_request *req)
if (ret)
return ret;
- dev_dbg(device_data->dev, "%s: (ctx=0x%x)!\n", __func__, (u32) ctx);
+ dev_dbg(device_data->dev, "%s: (ctx=0x%lx)!\n", __func__,
+ (unsigned long)ctx);
if (req_ctx->updated) {
ret = hash_resume_state(device_data, &device_data->state);
@@ -1272,8 +1274,8 @@ void hash_get_digest(struct hash_device_data *device_data,
else
loop_ctr = SHA256_DIGEST_SIZE / sizeof(u32);
- dev_dbg(device_data->dev, "%s: digest array:(0x%x)\n",
- __func__, (u32) digest);
+ dev_dbg(device_data->dev, "%s: digest array:(0x%lx)\n",
+ __func__, (unsigned long)digest);
/* Copy result into digest array */
for (count = 0; count < loop_ctr; count++) {
diff --git a/drivers/crypto/virtio/virtio_crypto_algs.c b/drivers/crypto/virtio/virtio_crypto_algs.c
index 10f266d462d6..42d19205166b 100644
--- a/drivers/crypto/virtio/virtio_crypto_algs.c
+++ b/drivers/crypto/virtio/virtio_crypto_algs.c
@@ -129,13 +129,11 @@ static int virtio_crypto_alg_ablkcipher_init_session(
* Avoid to do DMA from the stack, switch to using
* dynamically-allocated for the key
*/
- uint8_t *cipher_key = kmalloc(keylen, GFP_ATOMIC);
+ uint8_t *cipher_key = kmemdup(key, keylen, GFP_ATOMIC);
if (!cipher_key)
return -ENOMEM;
- memcpy(cipher_key, key, keylen);
-
spin_lock(&vcrypto->ctrl_lock);
/* Pad ctrl header */
vcrypto->ctrl.header.opcode =
diff --git a/drivers/crypto/vmx/aes_cbc.c b/drivers/crypto/vmx/aes_cbc.c
index c7e515a1bc97..d88084447f1c 100644
--- a/drivers/crypto/vmx/aes_cbc.c
+++ b/drivers/crypto/vmx/aes_cbc.c
@@ -7,64 +7,52 @@
* Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
*/
-#include <linux/types.h>
-#include <linux/err.h>
-#include <linux/crypto.h>
-#include <linux/delay.h>
#include <asm/simd.h>
#include <asm/switch_to.h>
#include <crypto/aes.h>
#include <crypto/internal/simd.h>
-#include <crypto/scatterwalk.h>
-#include <crypto/skcipher.h>
+#include <crypto/internal/skcipher.h>
#include "aesp8-ppc.h"
struct p8_aes_cbc_ctx {
- struct crypto_sync_skcipher *fallback;
+ struct crypto_skcipher *fallback;
struct aes_key enc_key;
struct aes_key dec_key;
};
-static int p8_aes_cbc_init(struct crypto_tfm *tfm)
+static int p8_aes_cbc_init(struct crypto_skcipher *tfm)
{
- const char *alg = crypto_tfm_alg_name(tfm);
- struct crypto_sync_skcipher *fallback;
- struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
-
- fallback = crypto_alloc_sync_skcipher(alg, 0,
- CRYPTO_ALG_NEED_FALLBACK);
+ struct p8_aes_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct crypto_skcipher *fallback;
+ fallback = crypto_alloc_skcipher("cbc(aes)", 0,
+ CRYPTO_ALG_NEED_FALLBACK |
+ CRYPTO_ALG_ASYNC);
if (IS_ERR(fallback)) {
- printk(KERN_ERR
- "Failed to allocate transformation for '%s': %ld\n",
- alg, PTR_ERR(fallback));
+ pr_err("Failed to allocate cbc(aes) fallback: %ld\n",
+ PTR_ERR(fallback));
return PTR_ERR(fallback);
}
- crypto_sync_skcipher_set_flags(
- fallback,
- crypto_skcipher_get_flags((struct crypto_skcipher *)tfm));
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) +
+ crypto_skcipher_reqsize(fallback));
ctx->fallback = fallback;
-
return 0;
}
-static void p8_aes_cbc_exit(struct crypto_tfm *tfm)
+static void p8_aes_cbc_exit(struct crypto_skcipher *tfm)
{
- struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct p8_aes_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
- if (ctx->fallback) {
- crypto_free_sync_skcipher(ctx->fallback);
- ctx->fallback = NULL;
- }
+ crypto_free_skcipher(ctx->fallback);
}
-static int p8_aes_cbc_setkey(struct crypto_tfm *tfm, const u8 *key,
+static int p8_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
unsigned int keylen)
{
+ struct p8_aes_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
int ret;
- struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
preempt_disable();
pagefault_disable();
@@ -75,108 +63,71 @@ static int p8_aes_cbc_setkey(struct crypto_tfm *tfm, const u8 *key,
pagefault_enable();
preempt_enable();
- ret |= crypto_sync_skcipher_setkey(ctx->fallback, key, keylen);
+ ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen);
return ret ? -EINVAL : 0;
}
-static int p8_aes_cbc_encrypt(struct blkcipher_desc *desc,
- struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes)
+static int p8_aes_cbc_crypt(struct skcipher_request *req, int enc)
{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ const struct p8_aes_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ unsigned int nbytes;
int ret;
- struct blkcipher_walk walk;
- struct p8_aes_cbc_ctx *ctx =
- crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
if (!crypto_simd_usable()) {
- SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
- skcipher_request_set_sync_tfm(req, ctx->fallback);
- skcipher_request_set_callback(req, desc->flags, NULL, NULL);
- skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
- ret = crypto_skcipher_encrypt(req);
- skcipher_request_zero(req);
- } else {
- blkcipher_walk_init(&walk, dst, src, nbytes);
- ret = blkcipher_walk_virt(desc, &walk);
- while ((nbytes = walk.nbytes)) {
- preempt_disable();
- pagefault_disable();
- enable_kernel_vsx();
- aes_p8_cbc_encrypt(walk.src.virt.addr,
- walk.dst.virt.addr,
- nbytes & AES_BLOCK_MASK,
- &ctx->enc_key, walk.iv, 1);
- disable_kernel_vsx();
- pagefault_enable();
- preempt_enable();
-
- nbytes &= AES_BLOCK_SIZE - 1;
- ret = blkcipher_walk_done(desc, &walk, nbytes);
- }
+ struct skcipher_request *subreq = skcipher_request_ctx(req);
+
+ *subreq = *req;
+ skcipher_request_set_tfm(subreq, ctx->fallback);
+ return enc ? crypto_skcipher_encrypt(subreq) :
+ crypto_skcipher_decrypt(subreq);
}
+ ret = skcipher_walk_virt(&walk, req, false);
+ while ((nbytes = walk.nbytes) != 0) {
+ preempt_disable();
+ pagefault_disable();
+ enable_kernel_vsx();
+ aes_p8_cbc_encrypt(walk.src.virt.addr,
+ walk.dst.virt.addr,
+ round_down(nbytes, AES_BLOCK_SIZE),
+ enc ? &ctx->enc_key : &ctx->dec_key,
+ walk.iv, enc);
+ disable_kernel_vsx();
+ pagefault_enable();
+ preempt_enable();
+
+ ret = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
+ }
return ret;
}
-static int p8_aes_cbc_decrypt(struct blkcipher_desc *desc,
- struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes)
+static int p8_aes_cbc_encrypt(struct skcipher_request *req)
{
- int ret;
- struct blkcipher_walk walk;
- struct p8_aes_cbc_ctx *ctx =
- crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
-
- if (!crypto_simd_usable()) {
- SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
- skcipher_request_set_sync_tfm(req, ctx->fallback);
- skcipher_request_set_callback(req, desc->flags, NULL, NULL);
- skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
- ret = crypto_skcipher_decrypt(req);
- skcipher_request_zero(req);
- } else {
- blkcipher_walk_init(&walk, dst, src, nbytes);
- ret = blkcipher_walk_virt(desc, &walk);
- while ((nbytes = walk.nbytes)) {
- preempt_disable();
- pagefault_disable();
- enable_kernel_vsx();
- aes_p8_cbc_encrypt(walk.src.virt.addr,
- walk.dst.virt.addr,
- nbytes & AES_BLOCK_MASK,
- &ctx->dec_key, walk.iv, 0);
- disable_kernel_vsx();
- pagefault_enable();
- preempt_enable();
-
- nbytes &= AES_BLOCK_SIZE - 1;
- ret = blkcipher_walk_done(desc, &walk, nbytes);
- }
- }
-
- return ret;
+ return p8_aes_cbc_crypt(req, 1);
}
+static int p8_aes_cbc_decrypt(struct skcipher_request *req)
+{
+ return p8_aes_cbc_crypt(req, 0);
+}
-struct crypto_alg p8_aes_cbc_alg = {
- .cra_name = "cbc(aes)",
- .cra_driver_name = "p8_aes_cbc",
- .cra_module = THIS_MODULE,
- .cra_priority = 2000,
- .cra_type = &crypto_blkcipher_type,
- .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK,
- .cra_alignmask = 0,
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct p8_aes_cbc_ctx),
- .cra_init = p8_aes_cbc_init,
- .cra_exit = p8_aes_cbc_exit,
- .cra_blkcipher = {
- .ivsize = AES_BLOCK_SIZE,
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .setkey = p8_aes_cbc_setkey,
- .encrypt = p8_aes_cbc_encrypt,
- .decrypt = p8_aes_cbc_decrypt,
- },
+struct skcipher_alg p8_aes_cbc_alg = {
+ .base.cra_name = "cbc(aes)",
+ .base.cra_driver_name = "p8_aes_cbc",
+ .base.cra_module = THIS_MODULE,
+ .base.cra_priority = 2000,
+ .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct p8_aes_cbc_ctx),
+ .setkey = p8_aes_cbc_setkey,
+ .encrypt = p8_aes_cbc_encrypt,
+ .decrypt = p8_aes_cbc_decrypt,
+ .init = p8_aes_cbc_init,
+ .exit = p8_aes_cbc_exit,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
};
diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c
index dd017ef42fa9..79ba062ee1c1 100644
--- a/drivers/crypto/vmx/aes_ctr.c
+++ b/drivers/crypto/vmx/aes_ctr.c
@@ -7,62 +7,51 @@
* Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
*/
-#include <linux/types.h>
-#include <linux/err.h>
-#include <linux/crypto.h>
-#include <linux/delay.h>
#include <asm/simd.h>
#include <asm/switch_to.h>
#include <crypto/aes.h>
#include <crypto/internal/simd.h>
-#include <crypto/scatterwalk.h>
-#include <crypto/skcipher.h>
+#include <crypto/internal/skcipher.h>
#include "aesp8-ppc.h"
struct p8_aes_ctr_ctx {
- struct crypto_sync_skcipher *fallback;
+ struct crypto_skcipher *fallback;
struct aes_key enc_key;
};
-static int p8_aes_ctr_init(struct crypto_tfm *tfm)
+static int p8_aes_ctr_init(struct crypto_skcipher *tfm)
{
- const char *alg = crypto_tfm_alg_name(tfm);
- struct crypto_sync_skcipher *fallback;
- struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct crypto_skcipher *fallback;
- fallback = crypto_alloc_sync_skcipher(alg, 0,
- CRYPTO_ALG_NEED_FALLBACK);
+ fallback = crypto_alloc_skcipher("ctr(aes)", 0,
+ CRYPTO_ALG_NEED_FALLBACK |
+ CRYPTO_ALG_ASYNC);
if (IS_ERR(fallback)) {
- printk(KERN_ERR
- "Failed to allocate transformation for '%s': %ld\n",
- alg, PTR_ERR(fallback));
+ pr_err("Failed to allocate ctr(aes) fallback: %ld\n",
+ PTR_ERR(fallback));
return PTR_ERR(fallback);
}
- crypto_sync_skcipher_set_flags(
- fallback,
- crypto_skcipher_get_flags((struct crypto_skcipher *)tfm));
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) +
+ crypto_skcipher_reqsize(fallback));
ctx->fallback = fallback;
-
return 0;
}
-static void p8_aes_ctr_exit(struct crypto_tfm *tfm)
+static void p8_aes_ctr_exit(struct crypto_skcipher *tfm)
{
- struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
- if (ctx->fallback) {
- crypto_free_sync_skcipher(ctx->fallback);
- ctx->fallback = NULL;
- }
+ crypto_free_skcipher(ctx->fallback);
}
-static int p8_aes_ctr_setkey(struct crypto_tfm *tfm, const u8 *key,
+static int p8_aes_ctr_setkey(struct crypto_skcipher *tfm, const u8 *key,
unsigned int keylen)
{
+ struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
int ret;
- struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
preempt_disable();
pagefault_disable();
@@ -72,13 +61,13 @@ static int p8_aes_ctr_setkey(struct crypto_tfm *tfm, const u8 *key,
pagefault_enable();
preempt_enable();
- ret |= crypto_sync_skcipher_setkey(ctx->fallback, key, keylen);
+ ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen);
return ret ? -EINVAL : 0;
}
-static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx,
- struct blkcipher_walk *walk)
+static void p8_aes_ctr_final(const struct p8_aes_ctr_ctx *ctx,
+ struct skcipher_walk *walk)
{
u8 *ctrblk = walk->iv;
u8 keystream[AES_BLOCK_SIZE];
@@ -98,77 +87,63 @@ static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx,
crypto_inc(ctrblk, AES_BLOCK_SIZE);
}
-static int p8_aes_ctr_crypt(struct blkcipher_desc *desc,
- struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes)
+static int p8_aes_ctr_crypt(struct skcipher_request *req)
{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ const struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ unsigned int nbytes;
int ret;
- u64 inc;
- struct blkcipher_walk walk;
- struct p8_aes_ctr_ctx *ctx =
- crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
if (!crypto_simd_usable()) {
- SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
- skcipher_request_set_sync_tfm(req, ctx->fallback);
- skcipher_request_set_callback(req, desc->flags, NULL, NULL);
- skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
- ret = crypto_skcipher_encrypt(req);
- skcipher_request_zero(req);
- } else {
- blkcipher_walk_init(&walk, dst, src, nbytes);
- ret = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
- while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
- preempt_disable();
- pagefault_disable();
- enable_kernel_vsx();
- aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr,
- walk.dst.virt.addr,
- (nbytes &
- AES_BLOCK_MASK) /
- AES_BLOCK_SIZE,
- &ctx->enc_key,
- walk.iv);
- disable_kernel_vsx();
- pagefault_enable();
- preempt_enable();
-
- /* We need to update IV mostly for last bytes/round */
- inc = (nbytes & AES_BLOCK_MASK) / AES_BLOCK_SIZE;
- if (inc > 0)
- while (inc--)
- crypto_inc(walk.iv, AES_BLOCK_SIZE);
-
- nbytes &= AES_BLOCK_SIZE - 1;
- ret = blkcipher_walk_done(desc, &walk, nbytes);
- }
- if (walk.nbytes) {
- p8_aes_ctr_final(ctx, &walk);
- ret = blkcipher_walk_done(desc, &walk, 0);
- }
+ struct skcipher_request *subreq = skcipher_request_ctx(req);
+
+ *subreq = *req;
+ skcipher_request_set_tfm(subreq, ctx->fallback);
+ return crypto_skcipher_encrypt(subreq);
}
+ ret = skcipher_walk_virt(&walk, req, false);
+ while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
+ preempt_disable();
+ pagefault_disable();
+ enable_kernel_vsx();
+ aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr,
+ walk.dst.virt.addr,
+ nbytes / AES_BLOCK_SIZE,
+ &ctx->enc_key, walk.iv);
+ disable_kernel_vsx();
+ pagefault_enable();
+ preempt_enable();
+
+ do {
+ crypto_inc(walk.iv, AES_BLOCK_SIZE);
+ } while ((nbytes -= AES_BLOCK_SIZE) >= AES_BLOCK_SIZE);
+
+ ret = skcipher_walk_done(&walk, nbytes);
+ }
+ if (nbytes) {
+ p8_aes_ctr_final(ctx, &walk);
+ ret = skcipher_walk_done(&walk, 0);
+ }
return ret;
}
-struct crypto_alg p8_aes_ctr_alg = {
- .cra_name = "ctr(aes)",
- .cra_driver_name = "p8_aes_ctr",
- .cra_module = THIS_MODULE,
- .cra_priority = 2000,
- .cra_type = &crypto_blkcipher_type,
- .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK,
- .cra_alignmask = 0,
- .cra_blocksize = 1,
- .cra_ctxsize = sizeof(struct p8_aes_ctr_ctx),
- .cra_init = p8_aes_ctr_init,
- .cra_exit = p8_aes_ctr_exit,
- .cra_blkcipher = {
- .ivsize = AES_BLOCK_SIZE,
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .setkey = p8_aes_ctr_setkey,
- .encrypt = p8_aes_ctr_crypt,
- .decrypt = p8_aes_ctr_crypt,
- },
+struct skcipher_alg p8_aes_ctr_alg = {
+ .base.cra_name = "ctr(aes)",
+ .base.cra_driver_name = "p8_aes_ctr",
+ .base.cra_module = THIS_MODULE,
+ .base.cra_priority = 2000,
+ .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
+ .base.cra_blocksize = 1,
+ .base.cra_ctxsize = sizeof(struct p8_aes_ctr_ctx),
+ .setkey = p8_aes_ctr_setkey,
+ .encrypt = p8_aes_ctr_crypt,
+ .decrypt = p8_aes_ctr_crypt,
+ .init = p8_aes_ctr_init,
+ .exit = p8_aes_ctr_exit,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .chunksize = AES_BLOCK_SIZE,
};
diff --git a/drivers/crypto/vmx/aes_xts.c b/drivers/crypto/vmx/aes_xts.c
index 536167e737a0..d59e736882f6 100644
--- a/drivers/crypto/vmx/aes_xts.c
+++ b/drivers/crypto/vmx/aes_xts.c
@@ -7,67 +7,56 @@
* Author: Leonidas S. Barbosa <leosilva@linux.vnet.ibm.com>
*/
-#include <linux/types.h>
-#include <linux/err.h>
-#include <linux/crypto.h>
-#include <linux/delay.h>
#include <asm/simd.h>
#include <asm/switch_to.h>
#include <crypto/aes.h>
#include <crypto/internal/simd.h>
-#include <crypto/scatterwalk.h>
+#include <crypto/internal/skcipher.h>
#include <crypto/xts.h>
-#include <crypto/skcipher.h>
#include "aesp8-ppc.h"
struct p8_aes_xts_ctx {
- struct crypto_sync_skcipher *fallback;
+ struct crypto_skcipher *fallback;
struct aes_key enc_key;
struct aes_key dec_key;
struct aes_key tweak_key;
};
-static int p8_aes_xts_init(struct crypto_tfm *tfm)
+static int p8_aes_xts_init(struct crypto_skcipher *tfm)
{
- const char *alg = crypto_tfm_alg_name(tfm);
- struct crypto_sync_skcipher *fallback;
- struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct crypto_skcipher *fallback;
- fallback = crypto_alloc_sync_skcipher(alg, 0,
- CRYPTO_ALG_NEED_FALLBACK);
+ fallback = crypto_alloc_skcipher("xts(aes)", 0,
+ CRYPTO_ALG_NEED_FALLBACK |
+ CRYPTO_ALG_ASYNC);
if (IS_ERR(fallback)) {
- printk(KERN_ERR
- "Failed to allocate transformation for '%s': %ld\n",
- alg, PTR_ERR(fallback));
+ pr_err("Failed to allocate xts(aes) fallback: %ld\n",
+ PTR_ERR(fallback));
return PTR_ERR(fallback);
}
- crypto_sync_skcipher_set_flags(
- fallback,
- crypto_skcipher_get_flags((struct crypto_skcipher *)tfm));
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) +
+ crypto_skcipher_reqsize(fallback));
ctx->fallback = fallback;
-
return 0;
}
-static void p8_aes_xts_exit(struct crypto_tfm *tfm)
+static void p8_aes_xts_exit(struct crypto_skcipher *tfm)
{
- struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
- if (ctx->fallback) {
- crypto_free_sync_skcipher(ctx->fallback);
- ctx->fallback = NULL;
- }
+ crypto_free_skcipher(ctx->fallback);
}
-static int p8_aes_xts_setkey(struct crypto_tfm *tfm, const u8 *key,
+static int p8_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
unsigned int keylen)
{
+ struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
int ret;
- struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
- ret = xts_check_key(tfm, key, keylen);
+ ret = xts_verify_key(tfm, key, keylen);
if (ret)
return ret;
@@ -81,100 +70,90 @@ static int p8_aes_xts_setkey(struct crypto_tfm *tfm, const u8 *key,
pagefault_enable();
preempt_enable();
- ret |= crypto_sync_skcipher_setkey(ctx->fallback, key, keylen);
+ ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen);
return ret ? -EINVAL : 0;
}
-static int p8_aes_xts_crypt(struct blkcipher_desc *desc,
- struct scatterlist *dst,
- struct scatterlist *src,
- unsigned int nbytes, int enc)
+static int p8_aes_xts_crypt(struct skcipher_request *req, int enc)
{
- int ret;
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ const struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ unsigned int nbytes;
u8 tweak[AES_BLOCK_SIZE];
- u8 *iv;
- struct blkcipher_walk walk;
- struct p8_aes_xts_ctx *ctx =
- crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
-
- if (!crypto_simd_usable()) {
- SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
- skcipher_request_set_sync_tfm(req, ctx->fallback);
- skcipher_request_set_callback(req, desc->flags, NULL, NULL);
- skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
- ret = enc? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req);
- skcipher_request_zero(req);
- } else {
- blkcipher_walk_init(&walk, dst, src, nbytes);
-
- ret = blkcipher_walk_virt(desc, &walk);
+ int ret;
+
+ if (!crypto_simd_usable() || (req->cryptlen % XTS_BLOCK_SIZE) != 0) {
+ struct skcipher_request *subreq = skcipher_request_ctx(req);
+
+ *subreq = *req;
+ skcipher_request_set_tfm(subreq, ctx->fallback);
+ return enc ? crypto_skcipher_encrypt(subreq) :
+ crypto_skcipher_decrypt(subreq);
+ }
+
+ ret = skcipher_walk_virt(&walk, req, false);
+ if (ret)
+ return ret;
+
+ preempt_disable();
+ pagefault_disable();
+ enable_kernel_vsx();
+ aes_p8_encrypt(walk.iv, tweak, &ctx->tweak_key);
+
+ disable_kernel_vsx();
+ pagefault_enable();
+ preempt_enable();
+
+ while ((nbytes = walk.nbytes) != 0) {
preempt_disable();
pagefault_disable();
enable_kernel_vsx();
-
- iv = walk.iv;
- memset(tweak, 0, AES_BLOCK_SIZE);
- aes_p8_encrypt(iv, tweak, &ctx->tweak_key);
-
+ if (enc)
+ aes_p8_xts_encrypt(walk.src.virt.addr,
+ walk.dst.virt.addr,
+ round_down(nbytes, AES_BLOCK_SIZE),
+ &ctx->enc_key, NULL, tweak);
+ else
+ aes_p8_xts_decrypt(walk.src.virt.addr,
+ walk.dst.virt.addr,
+ round_down(nbytes, AES_BLOCK_SIZE),
+ &ctx->dec_key, NULL, tweak);
disable_kernel_vsx();
pagefault_enable();
preempt_enable();
- while ((nbytes = walk.nbytes)) {
- preempt_disable();
- pagefault_disable();
- enable_kernel_vsx();
- if (enc)
- aes_p8_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
- nbytes & AES_BLOCK_MASK, &ctx->enc_key, NULL, tweak);
- else
- aes_p8_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr,
- nbytes & AES_BLOCK_MASK, &ctx->dec_key, NULL, tweak);
- disable_kernel_vsx();
- pagefault_enable();
- preempt_enable();
-
- nbytes &= AES_BLOCK_SIZE - 1;
- ret = blkcipher_walk_done(desc, &walk, nbytes);
- }
+ ret = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
}
return ret;
}
-static int p8_aes_xts_encrypt(struct blkcipher_desc *desc,
- struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes)
+static int p8_aes_xts_encrypt(struct skcipher_request *req)
{
- return p8_aes_xts_crypt(desc, dst, src, nbytes, 1);
+ return p8_aes_xts_crypt(req, 1);
}
-static int p8_aes_xts_decrypt(struct blkcipher_desc *desc,
- struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes)
+static int p8_aes_xts_decrypt(struct skcipher_request *req)
{
- return p8_aes_xts_crypt(desc, dst, src, nbytes, 0);
+ return p8_aes_xts_crypt(req, 0);
}
-struct crypto_alg p8_aes_xts_alg = {
- .cra_name = "xts(aes)",
- .cra_driver_name = "p8_aes_xts",
- .cra_module = THIS_MODULE,
- .cra_priority = 2000,
- .cra_type = &crypto_blkcipher_type,
- .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK,
- .cra_alignmask = 0,
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct p8_aes_xts_ctx),
- .cra_init = p8_aes_xts_init,
- .cra_exit = p8_aes_xts_exit,
- .cra_blkcipher = {
- .ivsize = AES_BLOCK_SIZE,
- .min_keysize = 2 * AES_MIN_KEY_SIZE,
- .max_keysize = 2 * AES_MAX_KEY_SIZE,
- .setkey = p8_aes_xts_setkey,
- .encrypt = p8_aes_xts_encrypt,
- .decrypt = p8_aes_xts_decrypt,
- }
+struct skcipher_alg p8_aes_xts_alg = {
+ .base.cra_name = "xts(aes)",
+ .base.cra_driver_name = "p8_aes_xts",
+ .base.cra_module = THIS_MODULE,
+ .base.cra_priority = 2000,
+ .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct p8_aes_xts_ctx),
+ .setkey = p8_aes_xts_setkey,
+ .encrypt = p8_aes_xts_encrypt,
+ .decrypt = p8_aes_xts_decrypt,
+ .init = p8_aes_xts_init,
+ .exit = p8_aes_xts_exit,
+ .min_keysize = 2 * AES_MIN_KEY_SIZE,
+ .max_keysize = 2 * AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
};
diff --git a/drivers/crypto/vmx/aesp8-ppc.h b/drivers/crypto/vmx/aesp8-ppc.h
index 349646b73754..01774a4d26a2 100644
--- a/drivers/crypto/vmx/aesp8-ppc.h
+++ b/drivers/crypto/vmx/aesp8-ppc.h
@@ -2,8 +2,6 @@
#include <linux/types.h>
#include <crypto/aes.h>
-#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1))
-
struct aes_key {
u8 key[AES_MAX_KEYLENGTH];
int rounds;
diff --git a/drivers/crypto/vmx/aesp8-ppc.pl b/drivers/crypto/vmx/aesp8-ppc.pl
index 9c6b5c1d6a1a..db874367b602 100644
--- a/drivers/crypto/vmx/aesp8-ppc.pl
+++ b/drivers/crypto/vmx/aesp8-ppc.pl
@@ -1286,6 +1286,24 @@ ___
#########################################################################
{{{ # CTR procedure[s] #
+
+####################### WARNING: Here be dragons! #######################
+#
+# This code is written as 'ctr32', based on a 32-bit counter used
+# upstream. The kernel does *not* use a 32-bit counter. The kernel uses
+# a 128-bit counter.
+#
+# This leads to subtle changes from the upstream code: the counter
+# is incremented with vaddu_q_m rather than vaddu_w_m. This occurs in
+# both the bulk (8 blocks at a time) path, and in the individual block
+# path. Be aware of this when doing updates.
+#
+# See:
+# 1d4aa0b4c181 ("crypto: vmx - Fixing AES-CTR counter bug")
+# 009b30ac7444 ("crypto: vmx - CTR: always increment IV as quadword")
+# https://github.com/openssl/openssl/pull/8942
+#
+#########################################################################
my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10));
my ($rndkey0,$rndkey1,$inout,$tmp)= map("v$_",(0..3));
my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)=
@@ -1357,7 +1375,7 @@ Loop_ctr32_enc:
addi $idx,$idx,16
bdnz Loop_ctr32_enc
- vadduqm $ivec,$ivec,$one
+ vadduqm $ivec,$ivec,$one # Kernel change for 128-bit
vmr $dat,$inptail
lvx $inptail,0,$inp
addi $inp,$inp,16
@@ -1501,7 +1519,7 @@ Load_ctr32_enc_key:
$SHL $len,$len,4
vadduqm $out1,$ivec,$one # counter values ...
- vadduqm $out2,$ivec,$two
+ vadduqm $out2,$ivec,$two # (do all ctr adds as 128-bit)
vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0]
le?li $idx,8
vadduqm $out3,$out1,$two
diff --git a/drivers/crypto/vmx/vmx.c b/drivers/crypto/vmx/vmx.c
index 6c4c77f4e159..3e0335fb406c 100644
--- a/drivers/crypto/vmx/vmx.c
+++ b/drivers/crypto/vmx/vmx.c
@@ -15,54 +15,58 @@
#include <linux/crypto.h>
#include <asm/cputable.h>
#include <crypto/internal/hash.h>
+#include <crypto/internal/skcipher.h>
extern struct shash_alg p8_ghash_alg;
extern struct crypto_alg p8_aes_alg;
-extern struct crypto_alg p8_aes_cbc_alg;
-extern struct crypto_alg p8_aes_ctr_alg;
-extern struct crypto_alg p8_aes_xts_alg;
-static struct crypto_alg *algs[] = {
- &p8_aes_alg,
- &p8_aes_cbc_alg,
- &p8_aes_ctr_alg,
- &p8_aes_xts_alg,
- NULL,
-};
+extern struct skcipher_alg p8_aes_cbc_alg;
+extern struct skcipher_alg p8_aes_ctr_alg;
+extern struct skcipher_alg p8_aes_xts_alg;
static int __init p8_init(void)
{
- int ret = 0;
- struct crypto_alg **alg_it;
+ int ret;
- for (alg_it = algs; *alg_it; alg_it++) {
- ret = crypto_register_alg(*alg_it);
- printk(KERN_INFO "crypto_register_alg '%s' = %d\n",
- (*alg_it)->cra_name, ret);
- if (ret) {
- for (alg_it--; alg_it >= algs; alg_it--)
- crypto_unregister_alg(*alg_it);
- break;
- }
- }
+ ret = crypto_register_shash(&p8_ghash_alg);
if (ret)
- return ret;
+ goto err;
- ret = crypto_register_shash(&p8_ghash_alg);
- if (ret) {
- for (alg_it = algs; *alg_it; alg_it++)
- crypto_unregister_alg(*alg_it);
- }
+ ret = crypto_register_alg(&p8_aes_alg);
+ if (ret)
+ goto err_unregister_ghash;
+
+ ret = crypto_register_skcipher(&p8_aes_cbc_alg);
+ if (ret)
+ goto err_unregister_aes;
+
+ ret = crypto_register_skcipher(&p8_aes_ctr_alg);
+ if (ret)
+ goto err_unregister_aes_cbc;
+
+ ret = crypto_register_skcipher(&p8_aes_xts_alg);
+ if (ret)
+ goto err_unregister_aes_ctr;
+
+ return 0;
+
+err_unregister_aes_ctr:
+ crypto_unregister_skcipher(&p8_aes_ctr_alg);
+err_unregister_aes_cbc:
+ crypto_unregister_skcipher(&p8_aes_cbc_alg);
+err_unregister_aes:
+ crypto_unregister_alg(&p8_aes_alg);
+err_unregister_ghash:
+ crypto_unregister_shash(&p8_ghash_alg);
+err:
return ret;
}
static void __exit p8_exit(void)
{
- struct crypto_alg **alg_it;
-
- for (alg_it = algs; *alg_it; alg_it++) {
- printk(KERN_INFO "Removing '%s'\n", (*alg_it)->cra_name);
- crypto_unregister_alg(*alg_it);
- }
+ crypto_unregister_skcipher(&p8_aes_xts_alg);
+ crypto_unregister_skcipher(&p8_aes_ctr_alg);
+ crypto_unregister_skcipher(&p8_aes_cbc_alg);
+ crypto_unregister_alg(&p8_aes_alg);
crypto_unregister_shash(&p8_ghash_alg);
}
diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c
index 2109cfe80219..8fafbeab510a 100644
--- a/drivers/dax/bus.c
+++ b/drivers/dax/bus.c
@@ -295,6 +295,22 @@ static ssize_t target_node_show(struct device *dev,
}
static DEVICE_ATTR_RO(target_node);
+static unsigned long long dev_dax_resource(struct dev_dax *dev_dax)
+{
+ struct dax_region *dax_region = dev_dax->region;
+
+ return dax_region->res.start;
+}
+
+static ssize_t resource_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct dev_dax *dev_dax = to_dev_dax(dev);
+
+ return sprintf(buf, "%#llx\n", dev_dax_resource(dev_dax));
+}
+static DEVICE_ATTR_RO(resource);
+
static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
@@ -313,6 +329,8 @@ static umode_t dev_dax_visible(struct kobject *kobj, struct attribute *a, int n)
if (a == &dev_attr_target_node.attr && dev_dax_target_node(dev_dax) < 0)
return 0;
+ if (a == &dev_attr_resource.attr)
+ return 0400;
return a->mode;
}
@@ -320,6 +338,7 @@ static struct attribute *dev_dax_attributes[] = {
&dev_attr_modalias.attr,
&dev_attr_size.attr,
&dev_attr_target_node.attr,
+ &dev_attr_resource.attr,
NULL,
};
@@ -388,7 +407,7 @@ struct dev_dax *__devm_create_dev_dax(struct dax_region *dax_region, int id,
* No 'host' or dax_operations since there is no access to this
* device outside of mmap of the resulting character device.
*/
- dax_dev = alloc_dax(dev_dax, NULL, NULL);
+ dax_dev = alloc_dax(dev_dax, NULL, NULL, DAXDEV_F_SYNC);
if (!dax_dev)
goto err;
diff --git a/drivers/dax/dax-private.h b/drivers/dax/dax-private.h
index b4177aafbbd1..6ccca3b890d6 100644
--- a/drivers/dax/dax-private.h
+++ b/drivers/dax/dax-private.h
@@ -43,8 +43,7 @@ struct dax_region {
* @target_node: effective numa node if dev_dax memory range is onlined
* @dev - device core
* @pgmap - pgmap for memmap setup / lifetime (driver owned)
- * @ref: pgmap reference count (driver owned)
- * @cmp: @ref final put completion (driver owned)
+ * @dax_mem_res: physical address range of hotadded DAX memory
*/
struct dev_dax {
struct dax_region *region;
@@ -52,8 +51,7 @@ struct dev_dax {
int target_node;
struct device dev;
struct dev_pagemap pgmap;
- struct percpu_ref ref;
- struct completion cmp;
+ struct resource *dax_kmem_res;
};
static inline struct dev_dax *to_dev_dax(struct device *dev)
diff --git a/drivers/dax/device.c b/drivers/dax/device.c
index 8465d12fecba..1af823b2fe6b 100644
--- a/drivers/dax/device.c
+++ b/drivers/dax/device.c
@@ -14,37 +14,6 @@
#include "dax-private.h"
#include "bus.h"
-static struct dev_dax *ref_to_dev_dax(struct percpu_ref *ref)
-{
- return container_of(ref, struct dev_dax, ref);
-}
-
-static void dev_dax_percpu_release(struct percpu_ref *ref)
-{
- struct dev_dax *dev_dax = ref_to_dev_dax(ref);
-
- dev_dbg(&dev_dax->dev, "%s\n", __func__);
- complete(&dev_dax->cmp);
-}
-
-static void dev_dax_percpu_exit(struct percpu_ref *ref)
-{
- struct dev_dax *dev_dax = ref_to_dev_dax(ref);
-
- dev_dbg(&dev_dax->dev, "%s\n", __func__);
- wait_for_completion(&dev_dax->cmp);
- percpu_ref_exit(ref);
-}
-
-static void dev_dax_percpu_kill(struct percpu_ref *data)
-{
- struct percpu_ref *ref = data;
- struct dev_dax *dev_dax = ref_to_dev_dax(ref);
-
- dev_dbg(&dev_dax->dev, "%s\n", __func__);
- percpu_ref_kill(ref);
-}
-
static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma,
const char *func)
{
@@ -459,15 +428,7 @@ int dev_dax_probe(struct device *dev)
return -EBUSY;
}
- init_completion(&dev_dax->cmp);
- rc = percpu_ref_init(&dev_dax->ref, dev_dax_percpu_release, 0,
- GFP_KERNEL);
- if (rc)
- return rc;
-
- dev_dax->pgmap.ref = &dev_dax->ref;
- dev_dax->pgmap.kill = dev_dax_percpu_kill;
- dev_dax->pgmap.cleanup = dev_dax_percpu_exit;
+ dev_dax->pgmap.type = MEMORY_DEVICE_DEVDAX;
addr = devm_memremap_pages(dev, &dev_dax->pgmap);
if (IS_ERR(addr))
return PTR_ERR(addr);
diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c
index a02318c6d28a..3d0a7e702c94 100644
--- a/drivers/dax/kmem.c
+++ b/drivers/dax/kmem.c
@@ -66,23 +66,59 @@ int dev_dax_kmem_probe(struct device *dev)
new_res->name = dev_name(dev);
rc = add_memory(numa_node, new_res->start, resource_size(new_res));
- if (rc)
+ if (rc) {
+ release_resource(new_res);
+ kfree(new_res);
return rc;
+ }
+ dev_dax->dax_kmem_res = new_res;
return 0;
}
+#ifdef CONFIG_MEMORY_HOTREMOVE
+static int dev_dax_kmem_remove(struct device *dev)
+{
+ struct dev_dax *dev_dax = to_dev_dax(dev);
+ struct resource *res = dev_dax->dax_kmem_res;
+ resource_size_t kmem_start = res->start;
+ resource_size_t kmem_size = resource_size(res);
+ int rc;
+
+ /*
+ * We have one shot for removing memory, if some memory blocks were not
+ * offline prior to calling this function remove_memory() will fail, and
+ * there is no way to hotremove this memory until reboot because device
+ * unbind will succeed even if we return failure.
+ */
+ rc = remove_memory(dev_dax->target_node, kmem_start, kmem_size);
+ if (rc) {
+ dev_err(dev,
+ "DAX region %pR cannot be hotremoved until the next reboot\n",
+ res);
+ return rc;
+ }
+
+ /* Release and free dax resources */
+ release_resource(res);
+ kfree(res);
+ dev_dax->dax_kmem_res = NULL;
+
+ return 0;
+}
+#else
static int dev_dax_kmem_remove(struct device *dev)
{
/*
- * Purposely leak the request_mem_region() for the device-dax
- * range and return '0' to ->remove() attempts. The removal of
- * the device from the driver always succeeds, but the region
- * is permanently pinned as reserved by the unreleased
+ * Without hotremove purposely leak the request_mem_region() for the
+ * device-dax range and return '0' to ->remove() attempts. The removal
+ * of the device from the driver always succeeds, but the region is
+ * permanently pinned as reserved by the unreleased
* request_mem_region().
*/
return 0;
}
+#endif /* CONFIG_MEMORY_HOTREMOVE */
static struct dax_device_driver device_dax_kmem_driver = {
.drv = {
diff --git a/drivers/dax/pmem/core.c b/drivers/dax/pmem/core.c
index f9f51786d556..6eb6dfdf19bf 100644
--- a/drivers/dax/pmem/core.c
+++ b/drivers/dax/pmem/core.c
@@ -16,7 +16,7 @@ struct dev_dax *__dax_pmem_probe(struct device *dev, enum dev_dax_subsys subsys)
struct dev_dax *dev_dax;
struct nd_namespace_io *nsio;
struct dax_region *dax_region;
- struct dev_pagemap pgmap = { 0 };
+ struct dev_pagemap pgmap = { };
struct nd_namespace_common *ndns;
struct nd_dax *nd_dax = to_nd_dax(dev);
struct nd_pfn *nd_pfn = &nd_dax->nd_pfn;
diff --git a/drivers/dax/super.c b/drivers/dax/super.c
index 4e5ae7e8b557..26a654dbc69a 100644
--- a/drivers/dax/super.c
+++ b/drivers/dax/super.c
@@ -5,6 +5,7 @@
#include <linux/pagemap.h>
#include <linux/module.h>
#include <linux/mount.h>
+#include <linux/pseudo_fs.h>
#include <linux/magic.h>
#include <linux/genhd.h>
#include <linux/pfn_t.h>
@@ -195,6 +196,8 @@ enum dax_device_flags {
DAXDEV_ALIVE,
/* gate whether dax_flush() calls the low level flush routine */
DAXDEV_WRITE_CACHE,
+ /* flag to check if device supports synchronous flush */
+ DAXDEV_SYNC,
};
/**
@@ -372,6 +375,18 @@ bool dax_write_cache_enabled(struct dax_device *dax_dev)
}
EXPORT_SYMBOL_GPL(dax_write_cache_enabled);
+bool __dax_synchronous(struct dax_device *dax_dev)
+{
+ return test_bit(DAXDEV_SYNC, &dax_dev->flags);
+}
+EXPORT_SYMBOL_GPL(__dax_synchronous);
+
+void __set_dax_synchronous(struct dax_device *dax_dev)
+{
+ set_bit(DAXDEV_SYNC, &dax_dev->flags);
+}
+EXPORT_SYMBOL_GPL(__set_dax_synchronous);
+
bool dax_alive(struct dax_device *dax_dev)
{
lockdep_assert_held(&dax_srcu);
@@ -455,16 +470,19 @@ static const struct super_operations dax_sops = {
.drop_inode = generic_delete_inode,
};
-static struct dentry *dax_mount(struct file_system_type *fs_type,
- int flags, const char *dev_name, void *data)
+static int dax_init_fs_context(struct fs_context *fc)
{
- return mount_pseudo(fs_type, "dax:", &dax_sops, NULL, DAXFS_MAGIC);
+ struct pseudo_fs_context *ctx = init_pseudo(fc, DAXFS_MAGIC);
+ if (!ctx)
+ return -ENOMEM;
+ ctx->ops = &dax_sops;
+ return 0;
}
static struct file_system_type dax_fs_type = {
- .name = "dax",
- .mount = dax_mount,
- .kill_sb = kill_anon_super,
+ .name = "dax",
+ .init_fs_context = dax_init_fs_context,
+ .kill_sb = kill_anon_super,
};
static int dax_test(struct inode *inode, void *data)
@@ -526,7 +544,7 @@ static void dax_add_host(struct dax_device *dax_dev, const char *host)
}
struct dax_device *alloc_dax(void *private, const char *__host,
- const struct dax_operations *ops)
+ const struct dax_operations *ops, unsigned long flags)
{
struct dax_device *dax_dev;
const char *host;
@@ -549,6 +567,9 @@ struct dax_device *alloc_dax(void *private, const char *__host,
dax_add_host(dax_dev, host);
dax_dev->ops = ops;
dax_dev->private = private;
+ if (flags & DAXDEV_F_SYNC)
+ set_dax_synchronous(dax_dev);
+
return dax_dev;
err_dev:
@@ -648,10 +669,6 @@ static int dax_fs_init(void)
if (!dax_cache)
return -ENOMEM;
- rc = register_filesystem(&dax_fs_type);
- if (rc)
- goto err_register_fs;
-
dax_mnt = kern_mount(&dax_fs_type);
if (IS_ERR(dax_mnt)) {
rc = PTR_ERR(dax_mnt);
@@ -662,8 +679,6 @@ static int dax_fs_init(void)
return 0;
err_mount:
- unregister_filesystem(&dax_fs_type);
- err_register_fs:
kmem_cache_destroy(dax_cache);
return rc;
@@ -672,7 +687,6 @@ static int dax_fs_init(void)
static void dax_fs_exit(void)
{
kern_unmount(dax_mnt);
- unregister_filesystem(&dax_fs_type);
kmem_cache_destroy(dax_cache);
}
diff --git a/drivers/devfreq/Kconfig b/drivers/devfreq/Kconfig
index ba98a4e3ad33..defe1d438710 100644
--- a/drivers/devfreq/Kconfig
+++ b/drivers/devfreq/Kconfig
@@ -93,15 +93,28 @@ config ARM_EXYNOS_BUS_DEVFREQ
This does not yet operate with optimal voltages.
config ARM_TEGRA_DEVFREQ
- tristate "Tegra DEVFREQ Driver"
- depends on ARCH_TEGRA_124_SOC
- select DEVFREQ_GOV_SIMPLE_ONDEMAND
+ tristate "NVIDIA Tegra30/114/124/210 DEVFREQ Driver"
+ depends on ARCH_TEGRA_3x_SOC || ARCH_TEGRA_114_SOC || \
+ ARCH_TEGRA_132_SOC || ARCH_TEGRA_124_SOC || \
+ ARCH_TEGRA_210_SOC || \
+ COMPILE_TEST
select PM_OPP
help
This adds the DEVFREQ driver for the Tegra family of SoCs.
It reads ACTMON counters of memory controllers and adjusts the
operating frequencies and voltages with OPP support.
+config ARM_TEGRA20_DEVFREQ
+ tristate "NVIDIA Tegra20 DEVFREQ Driver"
+ depends on (TEGRA_MC && TEGRA20_EMC) || COMPILE_TEST
+ depends on COMMON_CLK
+ select DEVFREQ_GOV_SIMPLE_ONDEMAND
+ select PM_OPP
+ help
+ This adds the DEVFREQ driver for the Tegra20 family of SoCs.
+ It reads Memory Controller counters and adjusts the operating
+ frequencies and voltages with OPP support.
+
config ARM_RK3399_DMC_DEVFREQ
tristate "ARM RK3399 DMC DEVFREQ Driver"
depends on ARCH_ROCKCHIP
diff --git a/drivers/devfreq/Makefile b/drivers/devfreq/Makefile
index 32b8d4d3f12c..338ae8440db6 100644
--- a/drivers/devfreq/Makefile
+++ b/drivers/devfreq/Makefile
@@ -10,7 +10,8 @@ obj-$(CONFIG_DEVFREQ_GOV_PASSIVE) += governor_passive.o
# DEVFREQ Drivers
obj-$(CONFIG_ARM_EXYNOS_BUS_DEVFREQ) += exynos-bus.o
obj-$(CONFIG_ARM_RK3399_DMC_DEVFREQ) += rk3399_dmc.o
-obj-$(CONFIG_ARM_TEGRA_DEVFREQ) += tegra-devfreq.o
+obj-$(CONFIG_ARM_TEGRA_DEVFREQ) += tegra30-devfreq.o
+obj-$(CONFIG_ARM_TEGRA20_DEVFREQ) += tegra20-devfreq.o
# DEVFREQ Event Drivers
obj-$(CONFIG_PM_DEVFREQ_EVENT) += event/
diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c
index ab22bf8a12d6..446490c9d635 100644
--- a/drivers/devfreq/devfreq.c
+++ b/drivers/devfreq/devfreq.c
@@ -254,7 +254,7 @@ static struct devfreq_governor *try_then_request_governor(const char *name)
/* Restore previous state before return */
mutex_lock(&devfreq_list_lock);
if (err)
- return ERR_PTR(err);
+ return (err < 0) ? ERR_PTR(err) : ERR_PTR(-EINVAL);
governor = find_devfreq_governor(name);
}
@@ -402,7 +402,7 @@ static void devfreq_monitor(struct work_struct *work)
* devfreq_monitor_start() - Start load monitoring of devfreq instance
* @devfreq: the devfreq instance.
*
- * Helper function for starting devfreq device load monitoing. By
+ * Helper function for starting devfreq device load monitoring. By
* default delayed work based monitoring is supported. Function
* to be called from governor in response to DEVFREQ_GOV_START
* event when device is added to devfreq framework.
@@ -420,7 +420,7 @@ EXPORT_SYMBOL(devfreq_monitor_start);
* devfreq_monitor_stop() - Stop load monitoring of a devfreq instance
* @devfreq: the devfreq instance.
*
- * Helper function to stop devfreq device load monitoing. Function
+ * Helper function to stop devfreq device load monitoring. Function
* to be called from governor in response to DEVFREQ_GOV_STOP
* event when device is removed from devfreq framework.
*/
@@ -434,7 +434,7 @@ EXPORT_SYMBOL(devfreq_monitor_stop);
* devfreq_monitor_suspend() - Suspend load monitoring of a devfreq instance
* @devfreq: the devfreq instance.
*
- * Helper function to suspend devfreq device load monitoing. Function
+ * Helper function to suspend devfreq device load monitoring. Function
* to be called from governor in response to DEVFREQ_GOV_SUSPEND
* event or when polling interval is set to zero.
*
@@ -461,7 +461,7 @@ EXPORT_SYMBOL(devfreq_monitor_suspend);
* devfreq_monitor_resume() - Resume load monitoring of a devfreq instance
* @devfreq: the devfreq instance.
*
- * Helper function to resume devfreq device load monitoing. Function
+ * Helper function to resume devfreq device load monitoring. Function
* to be called from governor in response to DEVFREQ_GOV_RESUME
* event or when polling interval is set to non-zero.
*/
@@ -867,7 +867,7 @@ EXPORT_SYMBOL_GPL(devfreq_get_devfreq_by_phandle);
/**
* devm_devfreq_remove_device() - Resource-managed devfreq_remove_device()
- * @dev: the device to add devfreq feature.
+ * @dev: the device from which to remove devfreq feature.
* @devfreq: the devfreq instance to be removed
*/
void devm_devfreq_remove_device(struct device *dev, struct devfreq *devfreq)
diff --git a/drivers/devfreq/event/exynos-ppmu.c b/drivers/devfreq/event/exynos-ppmu.c
index 3ee3dd5653aa..87b42055e6bc 100644
--- a/drivers/devfreq/event/exynos-ppmu.c
+++ b/drivers/devfreq/event/exynos-ppmu.c
@@ -13,6 +13,7 @@
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/of_address.h>
+#include <linux/of_device.h>
#include <linux/platform_device.h>
#include <linux/regmap.h>
#include <linux/suspend.h>
@@ -20,6 +21,11 @@
#include "exynos-ppmu.h"
+enum exynos_ppmu_type {
+ EXYNOS_TYPE_PPMU,
+ EXYNOS_TYPE_PPMU_V2,
+};
+
struct exynos_ppmu_data {
struct clk *clk;
};
@@ -33,6 +39,7 @@ struct exynos_ppmu {
struct regmap *regmap;
struct exynos_ppmu_data ppmu;
+ enum exynos_ppmu_type ppmu_type;
};
#define PPMU_EVENT(name) \
@@ -86,6 +93,12 @@ static struct __exynos_ppmu_events {
PPMU_EVENT(d1-cpu),
PPMU_EVENT(d1-general),
PPMU_EVENT(d1-rt),
+
+ /* For Exynos5422 SoC */
+ PPMU_EVENT(dmc0_0),
+ PPMU_EVENT(dmc0_1),
+ PPMU_EVENT(dmc1_0),
+ PPMU_EVENT(dmc1_1),
};
static int exynos_ppmu_find_ppmu_id(struct devfreq_event_dev *edev)
@@ -151,9 +164,9 @@ static int exynos_ppmu_set_event(struct devfreq_event_dev *edev)
if (ret < 0)
return ret;
- /* Set the event of Read/Write data count */
+ /* Set the event of proper data type monitoring */
ret = regmap_write(info->regmap, PPMU_BEVTxSEL(id),
- PPMU_RO_DATA_CNT | PPMU_WO_DATA_CNT);
+ edev->desc->event_type);
if (ret < 0)
return ret;
@@ -365,23 +378,11 @@ static int exynos_ppmu_v2_set_event(struct devfreq_event_dev *edev)
if (ret < 0)
return ret;
- /* Set the event of Read/Write data count */
- switch (id) {
- case PPMU_PMNCNT0:
- case PPMU_PMNCNT1:
- case PPMU_PMNCNT2:
- ret = regmap_write(info->regmap, PPMU_V2_CH_EVx_TYPE(id),
- PPMU_V2_RO_DATA_CNT | PPMU_V2_WO_DATA_CNT);
- if (ret < 0)
- return ret;
- break;
- case PPMU_PMNCNT3:
- ret = regmap_write(info->regmap, PPMU_V2_CH_EVx_TYPE(id),
- PPMU_V2_EVT3_RW_DATA_CNT);
- if (ret < 0)
- return ret;
- break;
- }
+ /* Set the event of proper data type monitoring */
+ ret = regmap_write(info->regmap, PPMU_V2_CH_EVx_TYPE(id),
+ edev->desc->event_type);
+ if (ret < 0)
+ return ret;
/* Reset cycle counter/performance counter and enable PPMU */
ret = regmap_read(info->regmap, PPMU_V2_PMNC, &pmnc);
@@ -480,31 +481,24 @@ static const struct devfreq_event_ops exynos_ppmu_v2_ops = {
static const struct of_device_id exynos_ppmu_id_match[] = {
{
.compatible = "samsung,exynos-ppmu",
- .data = (void *)&exynos_ppmu_ops,
+ .data = (void *)EXYNOS_TYPE_PPMU,
}, {
.compatible = "samsung,exynos-ppmu-v2",
- .data = (void *)&exynos_ppmu_v2_ops,
+ .data = (void *)EXYNOS_TYPE_PPMU_V2,
},
{ /* sentinel */ },
};
MODULE_DEVICE_TABLE(of, exynos_ppmu_id_match);
-static struct devfreq_event_ops *exynos_bus_get_ops(struct device_node *np)
-{
- const struct of_device_id *match;
-
- match = of_match_node(exynos_ppmu_id_match, np);
- return (struct devfreq_event_ops *)match->data;
-}
-
static int of_get_devfreq_events(struct device_node *np,
struct exynos_ppmu *info)
{
struct devfreq_event_desc *desc;
- struct devfreq_event_ops *event_ops;
struct device *dev = info->dev;
struct device_node *events_np, *node;
int i, j, count;
+ const struct of_device_id *of_id;
+ int ret;
events_np = of_get_child_by_name(np, "events");
if (!events_np) {
@@ -512,7 +506,6 @@ static int of_get_devfreq_events(struct device_node *np,
"failed to get child node of devfreq-event devices\n");
return -EINVAL;
}
- event_ops = exynos_bus_get_ops(np);
count = of_get_child_count(events_np);
desc = devm_kcalloc(dev, count, sizeof(*desc), GFP_KERNEL);
@@ -520,6 +513,12 @@ static int of_get_devfreq_events(struct device_node *np,
return -ENOMEM;
info->num_events = count;
+ of_id = of_match_device(exynos_ppmu_id_match, dev);
+ if (of_id)
+ info->ppmu_type = (enum exynos_ppmu_type)of_id->data;
+ else
+ return -EINVAL;
+
j = 0;
for_each_child_of_node(events_np, node) {
for (i = 0; i < ARRAY_SIZE(ppmu_events); i++) {
@@ -537,10 +536,51 @@ static int of_get_devfreq_events(struct device_node *np,
continue;
}
- desc[j].ops = event_ops;
+ switch (info->ppmu_type) {
+ case EXYNOS_TYPE_PPMU:
+ desc[j].ops = &exynos_ppmu_ops;
+ break;
+ case EXYNOS_TYPE_PPMU_V2:
+ desc[j].ops = &exynos_ppmu_v2_ops;
+ break;
+ }
+
desc[j].driver_data = info;
of_property_read_string(node, "event-name", &desc[j].name);
+ ret = of_property_read_u32(node, "event-data-type",
+ &desc[j].event_type);
+ if (ret) {
+ /* Set the event of proper data type counting.
+ * Check if the data type has been defined in DT,
+ * use default if not.
+ */
+ if (info->ppmu_type == EXYNOS_TYPE_PPMU_V2) {
+ struct devfreq_event_dev edev;
+ int id;
+ /* Not all registers take the same value for
+ * read+write data count.
+ */
+ edev.desc = &desc[j];
+ id = exynos_ppmu_find_ppmu_id(&edev);
+
+ switch (id) {
+ case PPMU_PMNCNT0:
+ case PPMU_PMNCNT1:
+ case PPMU_PMNCNT2:
+ desc[j].event_type = PPMU_V2_RO_DATA_CNT
+ | PPMU_V2_WO_DATA_CNT;
+ break;
+ case PPMU_PMNCNT3:
+ desc[j].event_type =
+ PPMU_V2_EVT3_RW_DATA_CNT;
+ break;
+ }
+ } else {
+ desc[j].event_type = PPMU_RO_DATA_CNT |
+ PPMU_WO_DATA_CNT;
+ }
+ }
j++;
}
diff --git a/drivers/devfreq/exynos-bus.c b/drivers/devfreq/exynos-bus.c
index d9f377912c10..c832673273a2 100644
--- a/drivers/devfreq/exynos-bus.c
+++ b/drivers/devfreq/exynos-bus.c
@@ -22,7 +22,6 @@
#include <linux/slab.h>
#define DEFAULT_SATURATION_RATIO 40
-#define DEFAULT_VOLTAGE_TOLERANCE 2
struct exynos_bus {
struct device *dev;
@@ -34,9 +33,8 @@ struct exynos_bus {
unsigned long curr_freq;
- struct regulator *regulator;
+ struct opp_table *opp_table;
struct clk *clk;
- unsigned int voltage_tolerance;
unsigned int ratio;
};
@@ -90,62 +88,29 @@ static int exynos_bus_get_event(struct exynos_bus *bus,
}
/*
- * Must necessary function for devfreq simple-ondemand governor
+ * devfreq function for both simple-ondemand and passive governor
*/
static int exynos_bus_target(struct device *dev, unsigned long *freq, u32 flags)
{
struct exynos_bus *bus = dev_get_drvdata(dev);
struct dev_pm_opp *new_opp;
- unsigned long old_freq, new_freq, new_volt, tol;
int ret = 0;
- /* Get new opp-bus instance according to new bus clock */
+ /* Get correct frequency for bus. */
new_opp = devfreq_recommended_opp(dev, freq, flags);
if (IS_ERR(new_opp)) {
dev_err(dev, "failed to get recommended opp instance\n");
return PTR_ERR(new_opp);
}
- new_freq = dev_pm_opp_get_freq(new_opp);
- new_volt = dev_pm_opp_get_voltage(new_opp);
dev_pm_opp_put(new_opp);
- old_freq = bus->curr_freq;
-
- if (old_freq == new_freq)
- return 0;
- tol = new_volt * bus->voltage_tolerance / 100;
-
/* Change voltage and frequency according to new OPP level */
mutex_lock(&bus->lock);
+ ret = dev_pm_opp_set_rate(dev, *freq);
+ if (!ret)
+ bus->curr_freq = *freq;
- if (old_freq < new_freq) {
- ret = regulator_set_voltage_tol(bus->regulator, new_volt, tol);
- if (ret < 0) {
- dev_err(bus->dev, "failed to set voltage\n");
- goto out;
- }
- }
-
- ret = clk_set_rate(bus->clk, new_freq);
- if (ret < 0) {
- dev_err(dev, "failed to change clock of bus\n");
- clk_set_rate(bus->clk, old_freq);
- goto out;
- }
-
- if (old_freq > new_freq) {
- ret = regulator_set_voltage_tol(bus->regulator, new_volt, tol);
- if (ret < 0) {
- dev_err(bus->dev, "failed to set voltage\n");
- goto out;
- }
- }
- bus->curr_freq = new_freq;
-
- dev_dbg(dev, "Set the frequency of bus (%luHz -> %luHz, %luHz)\n",
- old_freq, new_freq, clk_get_rate(bus->clk));
-out:
mutex_unlock(&bus->lock);
return ret;
@@ -191,57 +156,12 @@ static void exynos_bus_exit(struct device *dev)
if (ret < 0)
dev_warn(dev, "failed to disable the devfreq-event devices\n");
- if (bus->regulator)
- regulator_disable(bus->regulator);
-
dev_pm_opp_of_remove_table(dev);
clk_disable_unprepare(bus->clk);
-}
-
-/*
- * Must necessary function for devfreq passive governor
- */
-static int exynos_bus_passive_target(struct device *dev, unsigned long *freq,
- u32 flags)
-{
- struct exynos_bus *bus = dev_get_drvdata(dev);
- struct dev_pm_opp *new_opp;
- unsigned long old_freq, new_freq;
- int ret = 0;
-
- /* Get new opp-bus instance according to new bus clock */
- new_opp = devfreq_recommended_opp(dev, freq, flags);
- if (IS_ERR(new_opp)) {
- dev_err(dev, "failed to get recommended opp instance\n");
- return PTR_ERR(new_opp);
+ if (bus->opp_table) {
+ dev_pm_opp_put_regulators(bus->opp_table);
+ bus->opp_table = NULL;
}
-
- new_freq = dev_pm_opp_get_freq(new_opp);
- dev_pm_opp_put(new_opp);
-
- old_freq = bus->curr_freq;
-
- if (old_freq == new_freq)
- return 0;
-
- /* Change the frequency according to new OPP level */
- mutex_lock(&bus->lock);
-
- ret = clk_set_rate(bus->clk, new_freq);
- if (ret < 0) {
- dev_err(dev, "failed to set the clock of bus\n");
- goto out;
- }
-
- *freq = new_freq;
- bus->curr_freq = new_freq;
-
- dev_dbg(dev, "Set the frequency of bus (%luHz -> %luHz, %luHz)\n",
- old_freq, new_freq, clk_get_rate(bus->clk));
-out:
- mutex_unlock(&bus->lock);
-
- return ret;
}
static void exynos_bus_passive_exit(struct device *dev)
@@ -256,21 +176,19 @@ static int exynos_bus_parent_parse_of(struct device_node *np,
struct exynos_bus *bus)
{
struct device *dev = bus->dev;
+ struct opp_table *opp_table;
+ const char *vdd = "vdd";
int i, ret, count, size;
- /* Get the regulator to provide each bus with the power */
- bus->regulator = devm_regulator_get(dev, "vdd");
- if (IS_ERR(bus->regulator)) {
- dev_err(dev, "failed to get VDD regulator\n");
- return PTR_ERR(bus->regulator);
- }
-
- ret = regulator_enable(bus->regulator);
- if (ret < 0) {
- dev_err(dev, "failed to enable VDD regulator\n");
+ opp_table = dev_pm_opp_set_regulators(dev, &vdd, 1);
+ if (IS_ERR(opp_table)) {
+ ret = PTR_ERR(opp_table);
+ dev_err(dev, "failed to set regulators %d\n", ret);
return ret;
}
+ bus->opp_table = opp_table;
+
/*
* Get the devfreq-event devices to get the current utilization of
* buses. This raw data will be used in devfreq ondemand governor.
@@ -311,14 +229,11 @@ static int exynos_bus_parent_parse_of(struct device_node *np,
if (of_property_read_u32(np, "exynos,saturation-ratio", &bus->ratio))
bus->ratio = DEFAULT_SATURATION_RATIO;
- if (of_property_read_u32(np, "exynos,voltage-tolerance",
- &bus->voltage_tolerance))
- bus->voltage_tolerance = DEFAULT_VOLTAGE_TOLERANCE;
-
return 0;
err_regulator:
- regulator_disable(bus->regulator);
+ dev_pm_opp_put_regulators(bus->opp_table);
+ bus->opp_table = NULL;
return ret;
}
@@ -383,6 +298,7 @@ static int exynos_bus_probe(struct platform_device *pdev)
struct exynos_bus *bus;
int ret, max_state;
unsigned long min_freq, max_freq;
+ bool passive = false;
if (!np) {
dev_err(dev, "failed to find devicetree node\n");
@@ -396,27 +312,27 @@ static int exynos_bus_probe(struct platform_device *pdev)
bus->dev = &pdev->dev;
platform_set_drvdata(pdev, bus);
- /* Parse the device-tree to get the resource information */
- ret = exynos_bus_parse_of(np, bus);
- if (ret < 0)
- return ret;
-
profile = devm_kzalloc(dev, sizeof(*profile), GFP_KERNEL);
- if (!profile) {
- ret = -ENOMEM;
- goto err;
- }
+ if (!profile)
+ return -ENOMEM;
node = of_parse_phandle(dev->of_node, "devfreq", 0);
if (node) {
of_node_put(node);
- goto passive;
+ passive = true;
} else {
ret = exynos_bus_parent_parse_of(np, bus);
+ if (ret < 0)
+ return ret;
}
+ /* Parse the device-tree to get the resource information */
+ ret = exynos_bus_parse_of(np, bus);
if (ret < 0)
- goto err;
+ goto err_reg;
+
+ if (passive)
+ goto passive;
/* Initialize the struct profile and governor data for parent device */
profile->polling_ms = 50;
@@ -468,7 +384,7 @@ static int exynos_bus_probe(struct platform_device *pdev)
goto out;
passive:
/* Initialize the struct profile and governor data for passive device */
- profile->target = exynos_bus_passive_target;
+ profile->target = exynos_bus_target;
profile->exit = exynos_bus_passive_exit;
/* Get the instance of parent devfreq device */
@@ -507,6 +423,11 @@ out:
err:
dev_pm_opp_of_remove_table(dev);
clk_disable_unprepare(bus->clk);
+err_reg:
+ if (!passive) {
+ dev_pm_opp_put_regulators(bus->opp_table);
+ bus->opp_table = NULL;
+ }
return ret;
}
diff --git a/drivers/devfreq/governor_passive.c b/drivers/devfreq/governor_passive.c
index 58308948b863..be6eeab9c814 100644
--- a/drivers/devfreq/governor_passive.c
+++ b/drivers/devfreq/governor_passive.c
@@ -149,7 +149,6 @@ static int devfreq_passive_notifier_call(struct notifier_block *nb,
static int devfreq_passive_event_handler(struct devfreq *devfreq,
unsigned int event, void *data)
{
- struct device *dev = devfreq->dev.parent;
struct devfreq_passive_data *p_data
= (struct devfreq_passive_data *)devfreq->data;
struct devfreq *parent = (struct devfreq *)p_data->parent;
@@ -165,12 +164,12 @@ static int devfreq_passive_event_handler(struct devfreq *devfreq,
p_data->this = devfreq;
nb->notifier_call = devfreq_passive_notifier_call;
- ret = devm_devfreq_register_notifier(dev, parent, nb,
+ ret = devfreq_register_notifier(parent, nb,
DEVFREQ_TRANSITION_NOTIFIER);
break;
case DEVFREQ_GOV_STOP:
- devm_devfreq_unregister_notifier(dev, parent, nb,
- DEVFREQ_TRANSITION_NOTIFIER);
+ WARN_ON(devfreq_unregister_notifier(parent, nb,
+ DEVFREQ_TRANSITION_NOTIFIER));
break;
default:
break;
diff --git a/drivers/devfreq/rk3399_dmc.c b/drivers/devfreq/rk3399_dmc.c
index 682465fa57e1..2e65d7279d79 100644
--- a/drivers/devfreq/rk3399_dmc.c
+++ b/drivers/devfreq/rk3399_dmc.c
@@ -351,7 +351,7 @@ static int rk3399_dmcfreq_probe(struct platform_device *pdev)
/*
* Get dram timing and pass it to arm trust firmware,
- * the dram drvier in arm trust firmware will get these
+ * the dram driver in arm trust firmware will get these
* timing and to do dram initial.
*/
if (!of_get_ddr_timings(&data->timing, np)) {
diff --git a/drivers/devfreq/tegra20-devfreq.c b/drivers/devfreq/tegra20-devfreq.c
new file mode 100644
index 000000000000..ff82bac9ee4e
--- /dev/null
+++ b/drivers/devfreq/tegra20-devfreq.c
@@ -0,0 +1,212 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * NVIDIA Tegra20 devfreq driver
+ *
+ * Copyright (C) 2019 GRATE-DRIVER project
+ */
+
+#include <linux/clk.h>
+#include <linux/devfreq.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/slab.h>
+
+#include <soc/tegra/mc.h>
+
+#include "governor.h"
+
+#define MC_STAT_CONTROL 0x90
+#define MC_STAT_EMC_CLOCK_LIMIT 0xa0
+#define MC_STAT_EMC_CLOCKS 0xa4
+#define MC_STAT_EMC_CONTROL 0xa8
+#define MC_STAT_EMC_COUNT 0xb8
+
+#define EMC_GATHER_CLEAR (1 << 8)
+#define EMC_GATHER_ENABLE (3 << 8)
+
+struct tegra_devfreq {
+ struct devfreq *devfreq;
+ struct clk *emc_clock;
+ void __iomem *regs;
+};
+
+static int tegra_devfreq_target(struct device *dev, unsigned long *freq,
+ u32 flags)
+{
+ struct tegra_devfreq *tegra = dev_get_drvdata(dev);
+ struct devfreq *devfreq = tegra->devfreq;
+ struct dev_pm_opp *opp;
+ unsigned long rate;
+ int err;
+
+ opp = devfreq_recommended_opp(dev, freq, flags);
+ if (IS_ERR(opp))
+ return PTR_ERR(opp);
+
+ rate = dev_pm_opp_get_freq(opp);
+ dev_pm_opp_put(opp);
+
+ err = clk_set_min_rate(tegra->emc_clock, rate);
+ if (err)
+ return err;
+
+ err = clk_set_rate(tegra->emc_clock, 0);
+ if (err)
+ goto restore_min_rate;
+
+ return 0;
+
+restore_min_rate:
+ clk_set_min_rate(tegra->emc_clock, devfreq->previous_freq);
+
+ return err;
+}
+
+static int tegra_devfreq_get_dev_status(struct device *dev,
+ struct devfreq_dev_status *stat)
+{
+ struct tegra_devfreq *tegra = dev_get_drvdata(dev);
+
+ /*
+ * EMC_COUNT returns number of memory events, that number is lower
+ * than the number of clocks. Conversion ratio of 1/8 results in a
+ * bit higher bandwidth than actually needed, it is good enough for
+ * the time being because drivers don't support requesting minimum
+ * needed memory bandwidth yet.
+ *
+ * TODO: adjust the ratio value once relevant drivers will support
+ * memory bandwidth management.
+ */
+ stat->busy_time = readl_relaxed(tegra->regs + MC_STAT_EMC_COUNT);
+ stat->total_time = readl_relaxed(tegra->regs + MC_STAT_EMC_CLOCKS) / 8;
+ stat->current_frequency = clk_get_rate(tegra->emc_clock);
+
+ writel_relaxed(EMC_GATHER_CLEAR, tegra->regs + MC_STAT_CONTROL);
+ writel_relaxed(EMC_GATHER_ENABLE, tegra->regs + MC_STAT_CONTROL);
+
+ return 0;
+}
+
+static struct devfreq_dev_profile tegra_devfreq_profile = {
+ .polling_ms = 500,
+ .target = tegra_devfreq_target,
+ .get_dev_status = tegra_devfreq_get_dev_status,
+};
+
+static struct tegra_mc *tegra_get_memory_controller(void)
+{
+ struct platform_device *pdev;
+ struct device_node *np;
+ struct tegra_mc *mc;
+
+ np = of_find_compatible_node(NULL, NULL, "nvidia,tegra20-mc-gart");
+ if (!np)
+ return ERR_PTR(-ENOENT);
+
+ pdev = of_find_device_by_node(np);
+ of_node_put(np);
+ if (!pdev)
+ return ERR_PTR(-ENODEV);
+
+ mc = platform_get_drvdata(pdev);
+ if (!mc)
+ return ERR_PTR(-EPROBE_DEFER);
+
+ return mc;
+}
+
+static int tegra_devfreq_probe(struct platform_device *pdev)
+{
+ struct tegra_devfreq *tegra;
+ struct tegra_mc *mc;
+ unsigned long max_rate;
+ unsigned long rate;
+ int err;
+
+ mc = tegra_get_memory_controller();
+ if (IS_ERR(mc)) {
+ err = PTR_ERR(mc);
+ dev_err(&pdev->dev, "failed to get memory controller: %d\n",
+ err);
+ return err;
+ }
+
+ tegra = devm_kzalloc(&pdev->dev, sizeof(*tegra), GFP_KERNEL);
+ if (!tegra)
+ return -ENOMEM;
+
+ /* EMC is a system-critical clock that is always enabled */
+ tegra->emc_clock = devm_clk_get(&pdev->dev, "emc");
+ if (IS_ERR(tegra->emc_clock)) {
+ err = PTR_ERR(tegra->emc_clock);
+ dev_err(&pdev->dev, "failed to get emc clock: %d\n", err);
+ return err;
+ }
+
+ tegra->regs = mc->regs;
+
+ max_rate = clk_round_rate(tegra->emc_clock, ULONG_MAX);
+
+ for (rate = 0; rate <= max_rate; rate++) {
+ rate = clk_round_rate(tegra->emc_clock, rate);
+
+ err = dev_pm_opp_add(&pdev->dev, rate, 0);
+ if (err) {
+ dev_err(&pdev->dev, "failed to add opp: %d\n", err);
+ goto remove_opps;
+ }
+ }
+
+ /*
+ * Reset statistic gathers state, select global bandwidth for the
+ * statistics collection mode and set clocks counter saturation
+ * limit to maximum.
+ */
+ writel_relaxed(0x00000000, tegra->regs + MC_STAT_CONTROL);
+ writel_relaxed(0x00000000, tegra->regs + MC_STAT_EMC_CONTROL);
+ writel_relaxed(0xffffffff, tegra->regs + MC_STAT_EMC_CLOCK_LIMIT);
+
+ platform_set_drvdata(pdev, tegra);
+
+ tegra->devfreq = devfreq_add_device(&pdev->dev, &tegra_devfreq_profile,
+ DEVFREQ_GOV_SIMPLE_ONDEMAND, NULL);
+ if (IS_ERR(tegra->devfreq)) {
+ err = PTR_ERR(tegra->devfreq);
+ goto remove_opps;
+ }
+
+ return 0;
+
+remove_opps:
+ dev_pm_opp_remove_all_dynamic(&pdev->dev);
+
+ return err;
+}
+
+static int tegra_devfreq_remove(struct platform_device *pdev)
+{
+ struct tegra_devfreq *tegra = platform_get_drvdata(pdev);
+
+ devfreq_remove_device(tegra->devfreq);
+ dev_pm_opp_remove_all_dynamic(&pdev->dev);
+
+ return 0;
+}
+
+static struct platform_driver tegra_devfreq_driver = {
+ .probe = tegra_devfreq_probe,
+ .remove = tegra_devfreq_remove,
+ .driver = {
+ .name = "tegra20-devfreq",
+ },
+};
+module_platform_driver(tegra_devfreq_driver);
+
+MODULE_ALIAS("platform:tegra20-devfreq");
+MODULE_AUTHOR("Dmitry Osipenko <digetx@gmail.com>");
+MODULE_DESCRIPTION("NVIDIA Tegra20 devfreq driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/devfreq/tegra-devfreq.c b/drivers/devfreq/tegra30-devfreq.c
index 35c38aad8b4f..a6ba75f4106d 100644
--- a/drivers/devfreq/tegra-devfreq.c
+++ b/drivers/devfreq/tegra30-devfreq.c
@@ -132,7 +132,6 @@ static struct tegra_devfreq_device_config actmon_device_configs[] = {
struct tegra_devfreq_device {
const struct tegra_devfreq_device_config *config;
void __iomem *regs;
- spinlock_t lock;
/* Average event count sampled in the last interrupt */
u32 avg_count;
@@ -160,6 +159,8 @@ struct tegra_devfreq {
struct notifier_block rate_change_nb;
struct tegra_devfreq_device devices[ARRAY_SIZE(actmon_device_configs)];
+
+ int irq;
};
struct tegra_actmon_emc_ratio {
@@ -179,23 +180,23 @@ static struct tegra_actmon_emc_ratio actmon_emc_ratios[] = {
static u32 actmon_readl(struct tegra_devfreq *tegra, u32 offset)
{
- return readl(tegra->regs + offset);
+ return readl_relaxed(tegra->regs + offset);
}
static void actmon_writel(struct tegra_devfreq *tegra, u32 val, u32 offset)
{
- writel(val, tegra->regs + offset);
+ writel_relaxed(val, tegra->regs + offset);
}
static u32 device_readl(struct tegra_devfreq_device *dev, u32 offset)
{
- return readl(dev->regs + offset);
+ return readl_relaxed(dev->regs + offset);
}
static void device_writel(struct tegra_devfreq_device *dev, u32 val,
u32 offset)
{
- writel(val, dev->regs + offset);
+ writel_relaxed(val, dev->regs + offset);
}
static unsigned long do_percent(unsigned long val, unsigned int pct)
@@ -231,18 +232,14 @@ static void tegra_devfreq_update_wmark(struct tegra_devfreq *tegra,
static void actmon_write_barrier(struct tegra_devfreq *tegra)
{
/* ensure the update has reached the ACTMON */
- wmb();
- actmon_readl(tegra, ACTMON_GLB_STATUS);
+ readl(tegra->regs + ACTMON_GLB_STATUS);
}
static void actmon_isr_device(struct tegra_devfreq *tegra,
struct tegra_devfreq_device *dev)
{
- unsigned long flags;
u32 intr_status, dev_ctrl;
- spin_lock_irqsave(&dev->lock, flags);
-
dev->avg_count = device_readl(dev, ACTMON_DEV_AVG_COUNT);
tegra_devfreq_update_avg_wmark(tegra, dev);
@@ -291,26 +288,6 @@ static void actmon_isr_device(struct tegra_devfreq *tegra,
device_writel(dev, ACTMON_INTR_STATUS_CLEAR, ACTMON_DEV_INTR_STATUS);
actmon_write_barrier(tegra);
-
- spin_unlock_irqrestore(&dev->lock, flags);
-}
-
-static irqreturn_t actmon_isr(int irq, void *data)
-{
- struct tegra_devfreq *tegra = data;
- bool handled = false;
- unsigned int i;
- u32 val;
-
- val = actmon_readl(tegra, ACTMON_GLB_STATUS);
- for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) {
- if (val & tegra->devices[i].config->irq_mask) {
- actmon_isr_device(tegra, tegra->devices + i);
- handled = true;
- }
- }
-
- return handled ? IRQ_WAKE_THREAD : IRQ_NONE;
}
static unsigned long actmon_cpu_to_emc_rate(struct tegra_devfreq *tegra,
@@ -337,15 +314,12 @@ static void actmon_update_target(struct tegra_devfreq *tegra,
unsigned long cpu_freq = 0;
unsigned long static_cpu_emc_freq = 0;
unsigned int avg_sustain_coef;
- unsigned long flags;
if (dev->config->avg_dependency_threshold) {
cpu_freq = cpufreq_get(0);
static_cpu_emc_freq = actmon_cpu_to_emc_rate(tegra, cpu_freq);
}
- spin_lock_irqsave(&dev->lock, flags);
-
dev->target_freq = dev->avg_count / ACTMON_SAMPLING_PERIOD;
avg_sustain_coef = 100 * 100 / dev->config->boost_up_threshold;
dev->target_freq = do_percent(dev->target_freq, avg_sustain_coef);
@@ -353,19 +327,31 @@ static void actmon_update_target(struct tegra_devfreq *tegra,
if (dev->avg_count >= dev->config->avg_dependency_threshold)
dev->target_freq = max(dev->target_freq, static_cpu_emc_freq);
-
- spin_unlock_irqrestore(&dev->lock, flags);
}
static irqreturn_t actmon_thread_isr(int irq, void *data)
{
struct tegra_devfreq *tegra = data;
+ bool handled = false;
+ unsigned int i;
+ u32 val;
mutex_lock(&tegra->devfreq->lock);
- update_devfreq(tegra->devfreq);
+
+ val = actmon_readl(tegra, ACTMON_GLB_STATUS);
+ for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) {
+ if (val & tegra->devices[i].config->irq_mask) {
+ actmon_isr_device(tegra, tegra->devices + i);
+ handled = true;
+ }
+ }
+
+ if (handled)
+ update_devfreq(tegra->devfreq);
+
mutex_unlock(&tegra->devfreq->lock);
- return IRQ_HANDLED;
+ return handled ? IRQ_HANDLED : IRQ_NONE;
}
static int tegra_actmon_rate_notify_cb(struct notifier_block *nb,
@@ -375,7 +361,6 @@ static int tegra_actmon_rate_notify_cb(struct notifier_block *nb,
struct tegra_devfreq *tegra;
struct tegra_devfreq_device *dev;
unsigned int i;
- unsigned long flags;
if (action != POST_RATE_CHANGE)
return NOTIFY_OK;
@@ -387,9 +372,7 @@ static int tegra_actmon_rate_notify_cb(struct notifier_block *nb,
for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) {
dev = &tegra->devices[i];
- spin_lock_irqsave(&dev->lock, flags);
tegra_devfreq_update_wmark(tegra, dev);
- spin_unlock_irqrestore(&dev->lock, flags);
}
actmon_write_barrier(tegra);
@@ -397,48 +380,6 @@ static int tegra_actmon_rate_notify_cb(struct notifier_block *nb,
return NOTIFY_OK;
}
-static void tegra_actmon_enable_interrupts(struct tegra_devfreq *tegra)
-{
- struct tegra_devfreq_device *dev;
- u32 val;
- unsigned int i;
-
- for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) {
- dev = &tegra->devices[i];
-
- val = device_readl(dev, ACTMON_DEV_CTRL);
- val |= ACTMON_DEV_CTRL_AVG_ABOVE_WMARK_EN;
- val |= ACTMON_DEV_CTRL_AVG_BELOW_WMARK_EN;
- val |= ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_EN;
- val |= ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN;
-
- device_writel(dev, val, ACTMON_DEV_CTRL);
- }
-
- actmon_write_barrier(tegra);
-}
-
-static void tegra_actmon_disable_interrupts(struct tegra_devfreq *tegra)
-{
- struct tegra_devfreq_device *dev;
- u32 val;
- unsigned int i;
-
- for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) {
- dev = &tegra->devices[i];
-
- val = device_readl(dev, ACTMON_DEV_CTRL);
- val &= ~ACTMON_DEV_CTRL_AVG_ABOVE_WMARK_EN;
- val &= ~ACTMON_DEV_CTRL_AVG_BELOW_WMARK_EN;
- val &= ~ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_EN;
- val &= ~ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN;
-
- device_writel(dev, val, ACTMON_DEV_CTRL);
- }
-
- actmon_write_barrier(tegra);
-}
-
static void tegra_actmon_configure_device(struct tegra_devfreq *tegra,
struct tegra_devfreq_device *dev)
{
@@ -462,34 +403,80 @@ static void tegra_actmon_configure_device(struct tegra_devfreq *tegra,
<< ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_NUM_SHIFT;
val |= (ACTMON_ABOVE_WMARK_WINDOW - 1)
<< ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_NUM_SHIFT;
+ val |= ACTMON_DEV_CTRL_AVG_ABOVE_WMARK_EN;
+ val |= ACTMON_DEV_CTRL_AVG_BELOW_WMARK_EN;
+ val |= ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_EN;
+ val |= ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN;
val |= ACTMON_DEV_CTRL_ENB;
device_writel(dev, val, ACTMON_DEV_CTRL);
+}
+
+static void tegra_actmon_start(struct tegra_devfreq *tegra)
+{
+ unsigned int i;
+
+ disable_irq(tegra->irq);
+
+ actmon_writel(tegra, ACTMON_SAMPLING_PERIOD - 1,
+ ACTMON_GLB_PERIOD_CTRL);
+
+ for (i = 0; i < ARRAY_SIZE(tegra->devices); i++)
+ tegra_actmon_configure_device(tegra, &tegra->devices[i]);
+
+ actmon_write_barrier(tegra);
+
+ enable_irq(tegra->irq);
+}
+
+static void tegra_actmon_stop(struct tegra_devfreq *tegra)
+{
+ unsigned int i;
+
+ disable_irq(tegra->irq);
+
+ for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) {
+ device_writel(&tegra->devices[i], 0x00000000, ACTMON_DEV_CTRL);
+ device_writel(&tegra->devices[i], ACTMON_INTR_STATUS_CLEAR,
+ ACTMON_DEV_INTR_STATUS);
+ }
actmon_write_barrier(tegra);
+
+ enable_irq(tegra->irq);
}
static int tegra_devfreq_target(struct device *dev, unsigned long *freq,
u32 flags)
{
struct tegra_devfreq *tegra = dev_get_drvdata(dev);
+ struct devfreq *devfreq = tegra->devfreq;
struct dev_pm_opp *opp;
- unsigned long rate = *freq * KHZ;
+ unsigned long rate;
+ int err;
- opp = devfreq_recommended_opp(dev, &rate, flags);
+ opp = devfreq_recommended_opp(dev, freq, flags);
if (IS_ERR(opp)) {
- dev_err(dev, "Failed to find opp for %lu KHz\n", *freq);
+ dev_err(dev, "Failed to find opp for %lu Hz\n", *freq);
return PTR_ERR(opp);
}
rate = dev_pm_opp_get_freq(opp);
dev_pm_opp_put(opp);
- clk_set_min_rate(tegra->emc_clock, rate);
- clk_set_rate(tegra->emc_clock, 0);
+ err = clk_set_min_rate(tegra->emc_clock, rate);
+ if (err)
+ return err;
- *freq = rate;
+ err = clk_set_rate(tegra->emc_clock, 0);
+ if (err)
+ goto restore_min_rate;
return 0;
+
+restore_min_rate:
+ clk_set_min_rate(tegra->emc_clock, devfreq->previous_freq);
+
+ return err;
}
static int tegra_devfreq_get_dev_status(struct device *dev,
@@ -497,13 +484,15 @@ static int tegra_devfreq_get_dev_status(struct device *dev,
{
struct tegra_devfreq *tegra = dev_get_drvdata(dev);
struct tegra_devfreq_device *actmon_dev;
+ unsigned long cur_freq;
- stat->current_frequency = tegra->cur_freq;
+ cur_freq = READ_ONCE(tegra->cur_freq);
/* To be used by the tegra governor */
stat->private_data = tegra;
/* The below are to be used by the other governors */
+ stat->current_frequency = cur_freq * KHZ;
actmon_dev = &tegra->devices[MCALL];
@@ -514,7 +503,7 @@ static int tegra_devfreq_get_dev_status(struct device *dev,
stat->busy_time *= 100 / BUS_SATURATION_RATIO;
/* Number of cycles in a sampling period */
- stat->total_time = ACTMON_SAMPLING_PERIOD * tegra->cur_freq;
+ stat->total_time = ACTMON_SAMPLING_PERIOD * cur_freq;
stat->busy_time = min(stat->busy_time, stat->total_time);
@@ -553,7 +542,7 @@ static int tegra_governor_get_target(struct devfreq *devfreq,
target_freq = max(target_freq, dev->target_freq);
}
- *freq = target_freq;
+ *freq = target_freq * KHZ;
return 0;
}
@@ -566,22 +555,22 @@ static int tegra_governor_event_handler(struct devfreq *devfreq,
switch (event) {
case DEVFREQ_GOV_START:
devfreq_monitor_start(devfreq);
- tegra_actmon_enable_interrupts(tegra);
+ tegra_actmon_start(tegra);
break;
case DEVFREQ_GOV_STOP:
- tegra_actmon_disable_interrupts(tegra);
+ tegra_actmon_stop(tegra);
devfreq_monitor_stop(devfreq);
break;
case DEVFREQ_GOV_SUSPEND:
- tegra_actmon_disable_interrupts(tegra);
+ tegra_actmon_stop(tegra);
devfreq_monitor_suspend(devfreq);
break;
case DEVFREQ_GOV_RESUME:
devfreq_monitor_resume(devfreq);
- tegra_actmon_enable_interrupts(tegra);
+ tegra_actmon_start(tegra);
break;
}
@@ -592,25 +581,22 @@ static struct devfreq_governor tegra_devfreq_governor = {
.name = "tegra_actmon",
.get_target_freq = tegra_governor_get_target,
.event_handler = tegra_governor_event_handler,
+ .immutable = true,
};
static int tegra_devfreq_probe(struct platform_device *pdev)
{
struct tegra_devfreq *tegra;
struct tegra_devfreq_device *dev;
- struct resource *res;
unsigned int i;
unsigned long rate;
- int irq;
int err;
tegra = devm_kzalloc(&pdev->dev, sizeof(*tegra), GFP_KERNEL);
if (!tegra)
return -ENOMEM;
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-
- tegra->regs = devm_ioremap_resource(&pdev->dev, res);
+ tegra->regs = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(tegra->regs))
return PTR_ERR(tegra->regs);
@@ -632,13 +618,10 @@ static int tegra_devfreq_probe(struct platform_device *pdev)
return PTR_ERR(tegra->emc_clock);
}
- clk_set_rate(tegra->emc_clock, ULONG_MAX);
-
- tegra->rate_change_nb.notifier_call = tegra_actmon_rate_notify_cb;
- err = clk_notifier_register(tegra->emc_clock, &tegra->rate_change_nb);
- if (err) {
- dev_err(&pdev->dev,
- "Failed to register rate change notifier\n");
+ tegra->irq = platform_get_irq(pdev, 0);
+ if (tegra->irq < 0) {
+ err = tegra->irq;
+ dev_err(&pdev->dev, "Failed to get IRQ: %d\n", err);
return err;
}
@@ -656,73 +639,94 @@ static int tegra_devfreq_probe(struct platform_device *pdev)
tegra->max_freq = clk_round_rate(tegra->emc_clock, ULONG_MAX) / KHZ;
tegra->cur_freq = clk_get_rate(tegra->emc_clock) / KHZ;
- actmon_writel(tegra, ACTMON_SAMPLING_PERIOD - 1,
- ACTMON_GLB_PERIOD_CTRL);
-
for (i = 0; i < ARRAY_SIZE(actmon_device_configs); i++) {
dev = tegra->devices + i;
dev->config = actmon_device_configs + i;
dev->regs = tegra->regs + dev->config->offset;
- spin_lock_init(&dev->lock);
-
- tegra_actmon_configure_device(tegra, dev);
}
for (rate = 0; rate <= tegra->max_freq * KHZ; rate++) {
rate = clk_round_rate(tegra->emc_clock, rate);
- dev_pm_opp_add(&pdev->dev, rate, 0);
- }
- irq = platform_get_irq(pdev, 0);
- if (irq < 0) {
- dev_err(&pdev->dev, "Failed to get IRQ: %d\n", irq);
- return irq;
+ err = dev_pm_opp_add(&pdev->dev, rate, 0);
+ if (err) {
+ dev_err(&pdev->dev, "Failed to add OPP: %d\n", err);
+ goto remove_opps;
+ }
}
platform_set_drvdata(pdev, tegra);
- err = devm_request_threaded_irq(&pdev->dev, irq, actmon_isr,
- actmon_thread_isr, IRQF_SHARED,
- "tegra-devfreq", tegra);
+ tegra->rate_change_nb.notifier_call = tegra_actmon_rate_notify_cb;
+ err = clk_notifier_register(tegra->emc_clock, &tegra->rate_change_nb);
if (err) {
- dev_err(&pdev->dev, "Interrupt request failed\n");
- return err;
+ dev_err(&pdev->dev,
+ "Failed to register rate change notifier\n");
+ goto remove_opps;
+ }
+
+ err = devfreq_add_governor(&tegra_devfreq_governor);
+ if (err) {
+ dev_err(&pdev->dev, "Failed to add governor: %d\n", err);
+ goto unreg_notifier;
}
tegra_devfreq_profile.initial_freq = clk_get_rate(tegra->emc_clock);
- tegra->devfreq = devm_devfreq_add_device(&pdev->dev,
- &tegra_devfreq_profile,
- "tegra_actmon",
- NULL);
+ tegra->devfreq = devfreq_add_device(&pdev->dev,
+ &tegra_devfreq_profile,
+ "tegra_actmon",
+ NULL);
+ if (IS_ERR(tegra->devfreq)) {
+ err = PTR_ERR(tegra->devfreq);
+ goto remove_governor;
+ }
+
+ err = devm_request_threaded_irq(&pdev->dev, tegra->irq, NULL,
+ actmon_thread_isr, IRQF_ONESHOT,
+ "tegra-devfreq", tegra);
+ if (err) {
+ dev_err(&pdev->dev, "Interrupt request failed: %d\n", err);
+ goto remove_devfreq;
+ }
return 0;
+
+remove_devfreq:
+ devfreq_remove_device(tegra->devfreq);
+
+remove_governor:
+ devfreq_remove_governor(&tegra_devfreq_governor);
+
+unreg_notifier:
+ clk_notifier_unregister(tegra->emc_clock, &tegra->rate_change_nb);
+
+remove_opps:
+ dev_pm_opp_remove_all_dynamic(&pdev->dev);
+
+ reset_control_reset(tegra->reset);
+ clk_disable_unprepare(tegra->clock);
+
+ return err;
}
static int tegra_devfreq_remove(struct platform_device *pdev)
{
struct tegra_devfreq *tegra = platform_get_drvdata(pdev);
- int irq = platform_get_irq(pdev, 0);
- u32 val;
- unsigned int i;
- for (i = 0; i < ARRAY_SIZE(actmon_device_configs); i++) {
- val = device_readl(&tegra->devices[i], ACTMON_DEV_CTRL);
- val &= ~ACTMON_DEV_CTRL_ENB;
- device_writel(&tegra->devices[i], val, ACTMON_DEV_CTRL);
- }
-
- actmon_write_barrier(tegra);
-
- devm_free_irq(&pdev->dev, irq, tegra);
+ devfreq_remove_device(tegra->devfreq);
+ devfreq_remove_governor(&tegra_devfreq_governor);
clk_notifier_unregister(tegra->emc_clock, &tegra->rate_change_nb);
+ dev_pm_opp_remove_all_dynamic(&pdev->dev);
+ reset_control_reset(tegra->reset);
clk_disable_unprepare(tegra->clock);
return 0;
}
static const struct of_device_id tegra_devfreq_of_match[] = {
+ { .compatible = "nvidia,tegra30-actmon" },
{ .compatible = "nvidia,tegra124-actmon" },
{ },
};
@@ -737,36 +741,7 @@ static struct platform_driver tegra_devfreq_driver = {
.of_match_table = tegra_devfreq_of_match,
},
};
-
-static int __init tegra_devfreq_init(void)
-{
- int ret = 0;
-
- ret = devfreq_add_governor(&tegra_devfreq_governor);
- if (ret) {
- pr_err("%s: failed to add governor: %d\n", __func__, ret);
- return ret;
- }
-
- ret = platform_driver_register(&tegra_devfreq_driver);
- if (ret)
- devfreq_remove_governor(&tegra_devfreq_governor);
-
- return ret;
-}
-module_init(tegra_devfreq_init)
-
-static void __exit tegra_devfreq_exit(void)
-{
- int ret = 0;
-
- platform_driver_unregister(&tegra_devfreq_driver);
-
- ret = devfreq_remove_governor(&tegra_devfreq_governor);
- if (ret)
- pr_err("%s: failed to remove governor: %d\n", __func__, ret);
-}
-module_exit(tegra_devfreq_exit)
+module_platform_driver(tegra_devfreq_driver);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("Tegra devfreq driver");
diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig
index d5f915830b68..b6a9c2f1bc41 100644
--- a/drivers/dma-buf/Kconfig
+++ b/drivers/dma-buf/Kconfig
@@ -15,7 +15,7 @@ config SYNC_FILE
associated with a buffer. When a job is submitted to the GPU a fence
is attached to the buffer and is transferred via userspace, using Sync
Files fds, to the DRM driver for example. More details at
- Documentation/sync_file.txt.
+ Documentation/driver-api/sync_file.rst.
config SW_SYNC
bool "Sync File Validation Framework"
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index bf4d4c80fbc6..f45bfb29ef96 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -23,8 +23,11 @@
#include <linux/poll.h>
#include <linux/reservation.h>
#include <linux/mm.h>
+#include <linux/mount.h>
+#include <linux/pseudo_fs.h>
#include <uapi/linux/dma-buf.h>
+#include <uapi/linux/magic.h>
static inline int is_dma_buf_file(struct file *);
@@ -35,6 +38,45 @@ struct dma_buf_list {
static struct dma_buf_list db_list;
+static char *dmabuffs_dname(struct dentry *dentry, char *buffer, int buflen)
+{
+ struct dma_buf *dmabuf;
+ char name[DMA_BUF_NAME_LEN];
+ size_t ret = 0;
+
+ dmabuf = dentry->d_fsdata;
+ mutex_lock(&dmabuf->lock);
+ if (dmabuf->name)
+ ret = strlcpy(name, dmabuf->name, DMA_BUF_NAME_LEN);
+ mutex_unlock(&dmabuf->lock);
+
+ return dynamic_dname(dentry, buffer, buflen, "/%s:%s",
+ dentry->d_name.name, ret > 0 ? name : "");
+}
+
+static const struct dentry_operations dma_buf_dentry_ops = {
+ .d_dname = dmabuffs_dname,
+};
+
+static struct vfsmount *dma_buf_mnt;
+
+static int dma_buf_fs_init_context(struct fs_context *fc)
+{
+ struct pseudo_fs_context *ctx;
+
+ ctx = init_pseudo(fc, DMA_BUF_MAGIC);
+ if (!ctx)
+ return -ENOMEM;
+ ctx->dops = &dma_buf_dentry_ops;
+ return 0;
+}
+
+static struct file_system_type dma_buf_fs_type = {
+ .name = "dmabuf",
+ .init_fs_context = dma_buf_fs_init_context,
+ .kill_sb = kill_anon_super,
+};
+
static int dma_buf_release(struct inode *inode, struct file *file)
{
struct dma_buf *dmabuf;
@@ -79,6 +121,10 @@ static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma)
dmabuf = file->private_data;
+ /* check if buffer supports mmap */
+ if (!dmabuf->ops->mmap)
+ return -EINVAL;
+
/* check for overflowing the buffer's size */
if (vma->vm_pgoff + vma_pages(vma) >
dmabuf->size >> PAGE_SHIFT)
@@ -265,6 +311,43 @@ out:
return events;
}
+/**
+ * dma_buf_set_name - Set a name to a specific dma_buf to track the usage.
+ * The name of the dma-buf buffer can only be set when the dma-buf is not
+ * attached to any devices. It could theoritically support changing the
+ * name of the dma-buf if the same piece of memory is used for multiple
+ * purpose between different devices.
+ *
+ * @dmabuf [in] dmabuf buffer that will be renamed.
+ * @buf: [in] A piece of userspace memory that contains the name of
+ * the dma-buf.
+ *
+ * Returns 0 on success. If the dma-buf buffer is already attached to
+ * devices, return -EBUSY.
+ *
+ */
+static long dma_buf_set_name(struct dma_buf *dmabuf, const char __user *buf)
+{
+ char *name = strndup_user(buf, DMA_BUF_NAME_LEN);
+ long ret = 0;
+
+ if (IS_ERR(name))
+ return PTR_ERR(name);
+
+ mutex_lock(&dmabuf->lock);
+ if (!list_empty(&dmabuf->attachments)) {
+ ret = -EBUSY;
+ kfree(name);
+ goto out_unlock;
+ }
+ kfree(dmabuf->name);
+ dmabuf->name = name;
+
+out_unlock:
+ mutex_unlock(&dmabuf->lock);
+ return ret;
+}
+
static long dma_buf_ioctl(struct file *file,
unsigned int cmd, unsigned long arg)
{
@@ -303,11 +386,29 @@ static long dma_buf_ioctl(struct file *file,
ret = dma_buf_begin_cpu_access(dmabuf, direction);
return ret;
+
+ case DMA_BUF_SET_NAME:
+ return dma_buf_set_name(dmabuf, (const char __user *)arg);
+
default:
return -ENOTTY;
}
}
+static void dma_buf_show_fdinfo(struct seq_file *m, struct file *file)
+{
+ struct dma_buf *dmabuf = file->private_data;
+
+ seq_printf(m, "size:\t%zu\n", dmabuf->size);
+ /* Don't count the temporary reference taken inside procfs seq_show */
+ seq_printf(m, "count:\t%ld\n", file_count(dmabuf->file) - 1);
+ seq_printf(m, "exp_name:\t%s\n", dmabuf->exp_name);
+ mutex_lock(&dmabuf->lock);
+ if (dmabuf->name)
+ seq_printf(m, "name:\t%s\n", dmabuf->name);
+ mutex_unlock(&dmabuf->lock);
+}
+
static const struct file_operations dma_buf_fops = {
.release = dma_buf_release,
.mmap = dma_buf_mmap_internal,
@@ -317,6 +418,7 @@ static const struct file_operations dma_buf_fops = {
#ifdef CONFIG_COMPAT
.compat_ioctl = dma_buf_ioctl,
#endif
+ .show_fdinfo = dma_buf_show_fdinfo,
};
/*
@@ -327,6 +429,32 @@ static inline int is_dma_buf_file(struct file *file)
return file->f_op == &dma_buf_fops;
}
+static struct file *dma_buf_getfile(struct dma_buf *dmabuf, int flags)
+{
+ struct file *file;
+ struct inode *inode = alloc_anon_inode(dma_buf_mnt->mnt_sb);
+
+ if (IS_ERR(inode))
+ return ERR_CAST(inode);
+
+ inode->i_size = dmabuf->size;
+ inode_set_bytes(inode, dmabuf->size);
+
+ file = alloc_file_pseudo(inode, dma_buf_mnt, "dmabuf",
+ flags, &dma_buf_fops);
+ if (IS_ERR(file))
+ goto err_alloc_file;
+ file->f_flags = flags & (O_ACCMODE | O_NONBLOCK);
+ file->private_data = dmabuf;
+ file->f_path.dentry->d_fsdata = dmabuf;
+
+ return file;
+
+err_alloc_file:
+ iput(inode);
+ return file;
+}
+
/**
* DOC: dma buf device access
*
@@ -393,8 +521,7 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
|| !exp_info->ops
|| !exp_info->ops->map_dma_buf
|| !exp_info->ops->unmap_dma_buf
- || !exp_info->ops->release
- || !exp_info->ops->mmap)) {
+ || !exp_info->ops->release)) {
return ERR_PTR(-EINVAL);
}
@@ -422,8 +549,7 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
}
dmabuf->resv = resv;
- file = anon_inode_getfile("dmabuf", &dma_buf_fops, dmabuf,
- exp_info->flags);
+ file = dma_buf_getfile(dmabuf, exp_info->flags);
if (IS_ERR(file)) {
ret = PTR_ERR(file);
goto err_dmabuf;
@@ -562,6 +688,7 @@ struct dma_buf_attachment *dma_buf_attach(struct dma_buf *dmabuf,
list_add(&attach->node, &dmabuf->attachments);
mutex_unlock(&dmabuf->lock);
+
return attach;
err_attach:
@@ -584,6 +711,9 @@ void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach)
if (WARN_ON(!dmabuf || !attach))
return;
+ if (attach->sgt)
+ dmabuf->ops->unmap_dma_buf(attach, attach->sgt, attach->dir);
+
mutex_lock(&dmabuf->lock);
list_del(&attach->node);
if (dmabuf->ops->detach)
@@ -619,10 +749,27 @@ struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach,
if (WARN_ON(!attach || !attach->dmabuf))
return ERR_PTR(-EINVAL);
+ if (attach->sgt) {
+ /*
+ * Two mappings with different directions for the same
+ * attachment are not allowed.
+ */
+ if (attach->dir != direction &&
+ attach->dir != DMA_BIDIRECTIONAL)
+ return ERR_PTR(-EBUSY);
+
+ return attach->sgt;
+ }
+
sg_table = attach->dmabuf->ops->map_dma_buf(attach, direction);
if (!sg_table)
sg_table = ERR_PTR(-ENOMEM);
+ if (!IS_ERR(sg_table) && attach->dmabuf->ops->cache_sgt_mapping) {
+ attach->sgt = sg_table;
+ attach->dir = direction;
+ }
+
return sg_table;
}
EXPORT_SYMBOL_GPL(dma_buf_map_attachment);
@@ -646,8 +793,10 @@ void dma_buf_unmap_attachment(struct dma_buf_attachment *attach,
if (WARN_ON(!attach || !attach->dmabuf || !sg_table))
return;
- attach->dmabuf->ops->unmap_dma_buf(attach, sg_table,
- direction);
+ if (attach->sgt == sg_table)
+ return;
+
+ attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction);
}
EXPORT_SYMBOL_GPL(dma_buf_unmap_attachment);
@@ -895,6 +1044,10 @@ int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma,
if (WARN_ON(!dmabuf || !vma))
return -EINVAL;
+ /* check if buffer supports mmap */
+ if (!dmabuf->ops->mmap)
+ return -EINVAL;
+
/* check for offset overflow */
if (pgoff + vma_pages(vma) < pgoff)
return -EOVERFLOW;
@@ -1014,8 +1167,8 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused)
return ret;
seq_puts(s, "\nDma-buf Objects:\n");
- seq_printf(s, "%-8s\t%-8s\t%-8s\t%-8s\texp_name\n",
- "size", "flags", "mode", "count");
+ seq_printf(s, "%-8s\t%-8s\t%-8s\t%-8s\texp_name\t%-8s\n",
+ "size", "flags", "mode", "count", "ino");
list_for_each_entry(buf_obj, &db_list.head, list_node) {
ret = mutex_lock_interruptible(&buf_obj->lock);
@@ -1026,11 +1179,13 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused)
continue;
}
- seq_printf(s, "%08zu\t%08x\t%08x\t%08ld\t%s\n",
+ seq_printf(s, "%08zu\t%08x\t%08x\t%08ld\t%s\t%08lu\t%s\n",
buf_obj->size,
buf_obj->file->f_flags, buf_obj->file->f_mode,
file_count(buf_obj->file),
- buf_obj->exp_name);
+ buf_obj->exp_name,
+ file_inode(buf_obj->file)->i_ino,
+ buf_obj->name ?: "");
robj = buf_obj->resv;
while (true) {
@@ -1057,6 +1212,7 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused)
fence->ops->get_driver_name(fence),
fence->ops->get_timeline_name(fence),
dma_fence_is_signaled(fence) ? "" : "un");
+ dma_fence_put(fence);
}
rcu_read_unlock();
@@ -1125,6 +1281,10 @@ static inline void dma_buf_uninit_debugfs(void)
static int __init dma_buf_init(void)
{
+ dma_buf_mnt = kern_mount(&dma_buf_fs_type);
+ if (IS_ERR(dma_buf_mnt))
+ return PTR_ERR(dma_buf_mnt);
+
mutex_init(&db_list.lock);
INIT_LIST_HEAD(&db_list.head);
dma_buf_init_debugfs();
@@ -1135,5 +1295,6 @@ subsys_initcall(dma_buf_init);
static void __exit dma_buf_deinit(void)
{
dma_buf_uninit_debugfs();
+ kern_unmount(dma_buf_mnt);
}
__exitcall(dma_buf_deinit);
diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c
index 227a19476d56..59ac96ec7ba8 100644
--- a/drivers/dma-buf/dma-fence.c
+++ b/drivers/dma-buf/dma-fence.c
@@ -248,8 +248,25 @@ void dma_fence_release(struct kref *kref)
trace_dma_fence_destroy(fence);
- /* Failed to signal before release, could be a refcounting issue */
- WARN_ON(!list_empty(&fence->cb_list));
+ if (WARN(!list_empty(&fence->cb_list),
+ "Fence %s:%s:%llx:%llx released with pending signals!\n",
+ fence->ops->get_driver_name(fence),
+ fence->ops->get_timeline_name(fence),
+ fence->context, fence->seqno)) {
+ unsigned long flags;
+
+ /*
+ * Failed to signal before release, likely a refcounting issue.
+ *
+ * This should never happen, but if it does make sure that we
+ * don't leave chains dangling. We set the error flag first
+ * so that the callbacks know this signal is due to an error.
+ */
+ spin_lock_irqsave(fence->lock, flags);
+ fence->error = -EDEADLK;
+ dma_fence_signal_locked(fence);
+ spin_unlock_irqrestore(fence->lock, flags);
+ }
if (fence->ops->release)
fence->ops->release(fence);
diff --git a/drivers/dma-buf/reservation.c b/drivers/dma-buf/reservation.c
index 4d32e2c67862..4447e13d1e89 100644
--- a/drivers/dma-buf/reservation.c
+++ b/drivers/dma-buf/reservation.c
@@ -365,6 +365,10 @@ int reservation_object_get_fences_rcu(struct reservation_object *obj,
GFP_NOWAIT | __GFP_NOWARN);
if (!nshared) {
rcu_read_unlock();
+
+ dma_fence_put(fence_excl);
+ fence_excl = NULL;
+
nshared = krealloc(shared, sz, GFP_KERNEL);
if (nshared) {
shared = nshared;
diff --git a/drivers/dma-buf/sync_debug.c b/drivers/dma-buf/sync_debug.c
index 3bb462cfb06c..101394f16930 100644
--- a/drivers/dma-buf/sync_debug.c
+++ b/drivers/dma-buf/sync_debug.c
@@ -188,29 +188,3 @@ static __init int sync_debugfs_init(void)
return 0;
}
late_initcall(sync_debugfs_init);
-
-#define DUMP_CHUNK 256
-static char sync_dump_buf[64 * 1024];
-void sync_dump(void)
-{
- struct seq_file s = {
- .buf = sync_dump_buf,
- .size = sizeof(sync_dump_buf) - 1,
- };
- int i;
-
- sync_info_debugfs_show(&s, NULL);
-
- for (i = 0; i < s.count; i += DUMP_CHUNK) {
- if ((s.count - i) > DUMP_CHUNK) {
- char c = s.buf[i + DUMP_CHUNK];
-
- s.buf[i + DUMP_CHUNK] = 0;
- pr_cont("%s", s.buf + i);
- s.buf[i + DUMP_CHUNK] = c;
- } else {
- s.buf[s.count] = 0;
- pr_cont("%s", s.buf + i);
- }
- }
-}
diff --git a/drivers/dma-buf/sync_debug.h b/drivers/dma-buf/sync_debug.h
index 05e33f937ad0..6176e52ba2d7 100644
--- a/drivers/dma-buf/sync_debug.h
+++ b/drivers/dma-buf/sync_debug.h
@@ -68,6 +68,5 @@ void sync_timeline_debug_add(struct sync_timeline *obj);
void sync_timeline_debug_remove(struct sync_timeline *obj);
void sync_file_debug_add(struct sync_file *fence);
void sync_file_debug_remove(struct sync_file *fence);
-void sync_dump(void);
#endif /* _LINUX_SYNC_H */
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 703275cc29de..7c511e3db4c8 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -103,6 +103,7 @@ config AXI_DMAC
depends on MICROBLAZE || NIOS2 || ARCH_ZYNQ || ARCH_ZYNQMP || ARCH_SOCFPGA || COMPILE_TEST
select DMA_ENGINE
select DMA_VIRTUAL_CHANNELS
+ select REGMAP_MMIO
help
Enable support for the Analog Devices AXI-DMAC peripheral. This DMA
controller is often used in Analog Device's reference designs for FPGA
@@ -293,8 +294,8 @@ config INTEL_IOATDMA
If unsure, say N.
config INTEL_IOP_ADMA
- tristate "Intel IOP ADMA support"
- depends on ARCH_IOP32X || ARCH_IOP33X || ARCH_IOP13XX
+ tristate "Intel IOP32x ADMA support"
+ depends on ARCH_IOP32X || COMPILE_TEST
select DMA_ENGINE
select ASYNC_TX_ENABLE_CHANNEL_SWITCH
help
@@ -584,7 +585,7 @@ config TEGRA20_APB_DMA
config TEGRA210_ADMA
tristate "NVIDIA Tegra210 ADMA support"
- depends on (ARCH_TEGRA_210_SOC || COMPILE_TEST) && PM_CLK
+ depends on (ARCH_TEGRA_210_SOC || COMPILE_TEST)
select DMA_ENGINE
select DMA_VIRTUAL_CHANNELS
help
@@ -666,6 +667,8 @@ source "drivers/dma/qcom/Kconfig"
source "drivers/dma/dw/Kconfig"
+source "drivers/dma/dw-edma/Kconfig"
+
source "drivers/dma/hsu/Kconfig"
source "drivers/dma/sh/Kconfig"
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index 6126e1c3a875..5bddf6f8790f 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -29,6 +29,7 @@ obj-$(CONFIG_DMA_SUN4I) += sun4i-dma.o
obj-$(CONFIG_DMA_SUN6I) += sun6i-dma.o
obj-$(CONFIG_DW_AXI_DMAC) += dw-axi-dmac/
obj-$(CONFIG_DW_DMAC_CORE) += dw/
+obj-$(CONFIG_DW_EDMA) += dw-edma/
obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o
obj-$(CONFIG_FSL_DMA) += fsldma.o
obj-$(CONFIG_FSL_EDMA) += fsl-edma.o fsl-edma-common.o
diff --git a/drivers/dma/acpi-dma.c b/drivers/dma/acpi-dma.c
index 30243f5c0710..8a05db3343d3 100644
--- a/drivers/dma/acpi-dma.c
+++ b/drivers/dma/acpi-dma.c
@@ -10,6 +10,7 @@
*/
#include <linux/device.h>
+#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/module.h>
#include <linux/kernel.h>
@@ -82,6 +83,12 @@ static int acpi_dma_parse_resource_group(const struct acpi_csrt_group *grp,
if (si->base_request_line == 0 && si->num_handshake_signals == 0)
return 0;
+ /* Set up DMA mask based on value from CSRT */
+ ret = dma_coerce_mask_and_coherent(&adev->dev,
+ DMA_BIT_MASK(si->dma_address_width));
+ if (ret)
+ return 0;
+
adma->base_request_line = si->base_request_line;
adma->end_request_line = si->base_request_line +
si->num_handshake_signals - 1;
@@ -140,7 +147,7 @@ static void acpi_dma_parse_csrt(struct acpi_device *adev, struct acpi_dma *adma)
* @dev: struct device of DMA controller
* @acpi_dma_xlate: translation function which converts a dma specifier
* into a dma_chan structure
- * @data pointer to controller specific data to be used by
+ * @data: pointer to controller specific data to be used by
* translation function
*
* Allocated memory should be freed with appropriate acpi_dma_controller_free()
@@ -224,7 +231,7 @@ static void devm_acpi_dma_release(struct device *dev, void *res)
* devm_acpi_dma_controller_register - resource managed acpi_dma_controller_register()
* @dev: device that is registering this DMA controller
* @acpi_dma_xlate: translation function
- * @data pointer to controller specific data
+ * @data: pointer to controller specific data
*
* Managed acpi_dma_controller_register(). DMA controller registered by this
* function are automatically freed on driver detach. See
@@ -257,6 +264,7 @@ EXPORT_SYMBOL_GPL(devm_acpi_dma_controller_register);
/**
* devm_acpi_dma_controller_free - resource managed acpi_dma_controller_free()
+ * @dev: device that is unregistering as DMA controller
*
* Unregister a DMA controller registered with
* devm_acpi_dma_controller_register(). Normally this function will not need to
diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c
index 464725dcad00..9adc7a2fa3d3 100644
--- a/drivers/dma/amba-pl08x.c
+++ b/drivers/dma/amba-pl08x.c
@@ -2508,9 +2508,8 @@ DEFINE_SHOW_ATTRIBUTE(pl08x_debugfs);
static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
{
/* Expose a simple debugfs interface to view all clocks */
- (void) debugfs_create_file(dev_name(&pl08x->adev->dev),
- S_IFREG | S_IRUGO, NULL, pl08x,
- &pl08x_debugfs_fops);
+ debugfs_create_file(dev_name(&pl08x->adev->dev), S_IFREG | S_IRUGO,
+ NULL, pl08x, &pl08x_debugfs_fops);
}
#else
diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c
index 627ef3e5b312..b58ac720d9a1 100644
--- a/drivers/dma/at_xdmac.c
+++ b/drivers/dma/at_xdmac.c
@@ -1568,11 +1568,14 @@ static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan)
struct at_xdmac_desc *desc;
struct dma_async_tx_descriptor *txd;
- desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node);
- txd = &desc->tx_dma_desc;
+ if (!list_empty(&atchan->xfers_list)) {
+ desc = list_first_entry(&atchan->xfers_list,
+ struct at_xdmac_desc, xfer_node);
+ txd = &desc->tx_dma_desc;
- if (txd->flags & DMA_PREP_INTERRUPT)
- dmaengine_desc_get_callback_invoke(txd, NULL);
+ if (txd->flags & DMA_PREP_INTERRUPT)
+ dmaengine_desc_get_callback_invoke(txd, NULL);
+ }
}
static void at_xdmac_handle_error(struct at_xdmac_chan *atchan)
diff --git a/drivers/dma/bcm-sba-raid.c b/drivers/dma/bcm-sba-raid.c
index fa81d0177765..275e90fa829d 100644
--- a/drivers/dma/bcm-sba-raid.c
+++ b/drivers/dma/bcm-sba-raid.c
@@ -164,7 +164,6 @@ struct sba_device {
struct list_head reqs_free_list;
/* DebugFS directory entries */
struct dentry *root;
- struct dentry *stats;
};
/* ====== Command helper routines ===== */
@@ -1716,17 +1715,11 @@ static int sba_probe(struct platform_device *pdev)
/* Create debugfs root entry */
sba->root = debugfs_create_dir(dev_name(sba->dev), NULL);
- if (IS_ERR_OR_NULL(sba->root)) {
- dev_err(sba->dev, "failed to create debugfs root entry\n");
- sba->root = NULL;
- goto skip_debugfs;
- }
/* Create debugfs stats entry */
- sba->stats = debugfs_create_devm_seqfile(sba->dev, "stats", sba->root,
- sba_debugfs_stats_show);
- if (IS_ERR_OR_NULL(sba->stats))
- dev_err(sba->dev, "failed to create debugfs stats file\n");
+ debugfs_create_devm_seqfile(sba->dev, "stats", sba->root,
+ sba_debugfs_stats_show);
+
skip_debugfs:
/* Register DMA device with Linux async framework */
diff --git a/drivers/dma/bcm2835-dma.c b/drivers/dma/bcm2835-dma.c
index 8101ff2f05c1..e4c593f48575 100644
--- a/drivers/dma/bcm2835-dma.c
+++ b/drivers/dma/bcm2835-dma.c
@@ -37,10 +37,19 @@
#define BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED 14
#define BCM2835_DMA_CHAN_NAME_SIZE 8
+/**
+ * struct bcm2835_dmadev - BCM2835 DMA controller
+ * @ddev: DMA device
+ * @base: base address of register map
+ * @dma_parms: DMA parameters (to convey 1 GByte max segment size to clients)
+ * @zero_page: bus address of zero page (to detect transactions copying from
+ * zero page and avoid accessing memory if so)
+ */
struct bcm2835_dmadev {
struct dma_device ddev;
void __iomem *base;
struct device_dma_parameters dma_parms;
+ dma_addr_t zero_page;
};
struct bcm2835_dma_cb {
@@ -687,11 +696,12 @@ static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_cyclic(
size_t period_len, enum dma_transfer_direction direction,
unsigned long flags)
{
+ struct bcm2835_dmadev *od = to_bcm2835_dma_dev(chan->device);
struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
struct bcm2835_desc *d;
dma_addr_t src, dst;
u32 info = BCM2835_DMA_WAIT_RESP;
- u32 extra = BCM2835_DMA_INT_EN;
+ u32 extra = 0;
size_t max_len = bcm2835_dma_max_frame_length(c);
size_t frames;
@@ -707,6 +717,11 @@ static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_cyclic(
return NULL;
}
+ if (flags & DMA_PREP_INTERRUPT)
+ extra |= BCM2835_DMA_INT_EN;
+ else
+ period_len = buf_len;
+
/*
* warn if buf_len is not a multiple of period_len - this may leed
* to unexpected latencies for interrupts and thus audiable clicks
@@ -732,6 +747,10 @@ static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_cyclic(
dst = c->cfg.dst_addr;
src = buf_addr;
info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC;
+
+ /* non-lite channels can write zeroes w/o accessing memory */
+ if (buf_addr == od->zero_page && !c->is_lite_channel)
+ info |= BCM2835_DMA_S_IGNORE;
}
/* calculate number of frames */
@@ -778,7 +797,10 @@ static int bcm2835_dma_terminate_all(struct dma_chan *chan)
/* stop DMA activity */
if (c->desc) {
- vchan_terminate_vdesc(&c->desc->vd);
+ if (c->desc->vd.tx.flags & DMA_PREP_INTERRUPT)
+ vchan_terminate_vdesc(&c->desc->vd);
+ else
+ vchan_vdesc_fini(&c->desc->vd);
c->desc = NULL;
bcm2835_dma_abort(c);
}
@@ -831,6 +853,9 @@ static void bcm2835_dma_free(struct bcm2835_dmadev *od)
list_del(&c->vc.chan.device_node);
tasklet_kill(&c->vc.task);
}
+
+ dma_unmap_page_attrs(od->ddev.dev, od->zero_page, PAGE_SIZE,
+ DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
}
static const struct of_device_id bcm2835_dma_of_match[] = {
@@ -871,8 +896,10 @@ static int bcm2835_dma_probe(struct platform_device *pdev)
pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask;
rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
- if (rc)
+ if (rc) {
+ dev_err(&pdev->dev, "Unable to set DMA mask\n");
return rc;
+ }
od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
if (!od)
@@ -907,11 +934,20 @@ static int bcm2835_dma_probe(struct platform_device *pdev)
od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |
BIT(DMA_MEM_TO_MEM);
od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
+ od->ddev.descriptor_reuse = true;
od->ddev.dev = &pdev->dev;
INIT_LIST_HEAD(&od->ddev.channels);
platform_set_drvdata(pdev, od);
+ od->zero_page = dma_map_page_attrs(od->ddev.dev, ZERO_PAGE(0), 0,
+ PAGE_SIZE, DMA_TO_DEVICE,
+ DMA_ATTR_SKIP_CPU_SYNC);
+ if (dma_mapping_error(od->ddev.dev, od->zero_page)) {
+ dev_err(&pdev->dev, "Failed to map zero page\n");
+ return -ENOMEM;
+ }
+
/* Request DMA channel mask from device tree */
if (of_property_read_u32(pdev->dev.of_node,
"brcm,dma-channel-mask",
diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c
index 547786ac342b..e51d836afcc7 100644
--- a/drivers/dma/coh901318.c
+++ b/drivers/dma/coh901318.c
@@ -1378,10 +1378,8 @@ static int __init init_coh901318_debugfs(void)
dma_dentry = debugfs_create_dir("dma", NULL);
- (void) debugfs_create_file("status",
- S_IFREG | S_IRUGO,
- dma_dentry, NULL,
- &coh901318_debugfs_status_operations);
+ debugfs_create_file("status", S_IFREG | S_IRUGO, dma_dentry, NULL,
+ &coh901318_debugfs_status_operations);
return 0;
}
diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c
index 8a3f1043917b..a0ee404b736e 100644
--- a/drivers/dma/dma-axi-dmac.c
+++ b/drivers/dma/dma-axi-dmac.c
@@ -2,7 +2,7 @@
/*
* Driver for the Analog Devices AXI-DMAC core
*
- * Copyright 2013-2015 Analog Devices Inc.
+ * Copyright 2013-2019 Analog Devices Inc.
* Author: Lars-Peter Clausen <lars@metafoo.de>
*/
@@ -18,7 +18,9 @@
#include <linux/of.h>
#include <linux/of_dma.h>
#include <linux/platform_device.h>
+#include <linux/regmap.h>
#include <linux/slab.h>
+#include <linux/fpga/adi-axi-common.h>
#include <dt-bindings/dma/axi-dmac.h>
@@ -62,6 +64,8 @@
#define AXI_DMAC_REG_STATUS 0x430
#define AXI_DMAC_REG_CURRENT_SRC_ADDR 0x434
#define AXI_DMAC_REG_CURRENT_DEST_ADDR 0x438
+#define AXI_DMAC_REG_PARTIAL_XFER_LEN 0x44c
+#define AXI_DMAC_REG_PARTIAL_XFER_ID 0x450
#define AXI_DMAC_CTRL_ENABLE BIT(0)
#define AXI_DMAC_CTRL_PAUSE BIT(1)
@@ -70,6 +74,10 @@
#define AXI_DMAC_IRQ_EOT BIT(1)
#define AXI_DMAC_FLAG_CYCLIC BIT(0)
+#define AXI_DMAC_FLAG_LAST BIT(1)
+#define AXI_DMAC_FLAG_PARTIAL_REPORT BIT(2)
+
+#define AXI_DMAC_FLAG_PARTIAL_XFER_DONE BIT(31)
/* The maximum ID allocated by the hardware is 31 */
#define AXI_DMAC_SG_UNUSED 32U
@@ -82,12 +90,14 @@ struct axi_dmac_sg {
unsigned int dest_stride;
unsigned int src_stride;
unsigned int id;
+ unsigned int partial_len;
bool schedule_when_free;
};
struct axi_dmac_desc {
struct virt_dma_desc vdesc;
bool cyclic;
+ bool have_partial_xfer;
unsigned int num_submitted;
unsigned int num_completed;
@@ -108,8 +118,10 @@ struct axi_dmac_chan {
unsigned int dest_type;
unsigned int max_length;
- unsigned int align_mask;
+ unsigned int address_align_mask;
+ unsigned int length_align_mask;
+ bool hw_partial_xfer;
bool hw_cyclic;
bool hw_2d;
};
@@ -167,14 +179,14 @@ static bool axi_dmac_check_len(struct axi_dmac_chan *chan, unsigned int len)
{
if (len == 0)
return false;
- if ((len & chan->align_mask) != 0) /* Not aligned */
+ if ((len & chan->length_align_mask) != 0) /* Not aligned */
return false;
return true;
}
static bool axi_dmac_check_addr(struct axi_dmac_chan *chan, dma_addr_t addr)
{
- if ((addr & chan->align_mask) != 0) /* Not aligned */
+ if ((addr & chan->address_align_mask) != 0) /* Not aligned */
return false;
return true;
}
@@ -210,11 +222,13 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan)
}
desc->num_submitted++;
- if (desc->num_submitted == desc->num_sgs) {
+ if (desc->num_submitted == desc->num_sgs ||
+ desc->have_partial_xfer) {
if (desc->cyclic)
desc->num_submitted = 0; /* Start again */
else
chan->next_desc = NULL;
+ flags |= AXI_DMAC_FLAG_LAST;
} else {
chan->next_desc = desc;
}
@@ -240,6 +254,9 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan)
desc->num_sgs == 1)
flags |= AXI_DMAC_FLAG_CYCLIC;
+ if (chan->hw_partial_xfer)
+ flags |= AXI_DMAC_FLAG_PARTIAL_REPORT;
+
axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->x_len - 1);
axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->y_len - 1);
axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags);
@@ -252,6 +269,83 @@ static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan)
struct axi_dmac_desc, vdesc.node);
}
+static inline unsigned int axi_dmac_total_sg_bytes(struct axi_dmac_chan *chan,
+ struct axi_dmac_sg *sg)
+{
+ if (chan->hw_2d)
+ return sg->x_len * sg->y_len;
+ else
+ return sg->x_len;
+}
+
+static void axi_dmac_dequeue_partial_xfers(struct axi_dmac_chan *chan)
+{
+ struct axi_dmac *dmac = chan_to_axi_dmac(chan);
+ struct axi_dmac_desc *desc;
+ struct axi_dmac_sg *sg;
+ u32 xfer_done, len, id, i;
+ bool found_sg;
+
+ do {
+ len = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_LEN);
+ id = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_ID);
+
+ found_sg = false;
+ list_for_each_entry(desc, &chan->active_descs, vdesc.node) {
+ for (i = 0; i < desc->num_sgs; i++) {
+ sg = &desc->sg[i];
+ if (sg->id == AXI_DMAC_SG_UNUSED)
+ continue;
+ if (sg->id == id) {
+ desc->have_partial_xfer = true;
+ sg->partial_len = len;
+ found_sg = true;
+ break;
+ }
+ }
+ if (found_sg)
+ break;
+ }
+
+ if (found_sg) {
+ dev_dbg(dmac->dma_dev.dev,
+ "Found partial segment id=%u, len=%u\n",
+ id, len);
+ } else {
+ dev_warn(dmac->dma_dev.dev,
+ "Not found partial segment id=%u, len=%u\n",
+ id, len);
+ }
+
+ /* Check if we have any more partial transfers */
+ xfer_done = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE);
+ xfer_done = !(xfer_done & AXI_DMAC_FLAG_PARTIAL_XFER_DONE);
+
+ } while (!xfer_done);
+}
+
+static void axi_dmac_compute_residue(struct axi_dmac_chan *chan,
+ struct axi_dmac_desc *active)
+{
+ struct dmaengine_result *rslt = &active->vdesc.tx_result;
+ unsigned int start = active->num_completed - 1;
+ struct axi_dmac_sg *sg;
+ unsigned int i, total;
+
+ rslt->result = DMA_TRANS_NOERROR;
+ rslt->residue = 0;
+
+ /*
+ * We get here if the last completed segment is partial, which
+ * means we can compute the residue from that segment onwards
+ */
+ for (i = start; i < active->num_sgs; i++) {
+ sg = &active->sg[i];
+ total = axi_dmac_total_sg_bytes(chan, sg);
+ rslt->residue += (total - sg->partial_len);
+ }
+}
+
static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan,
unsigned int completed_transfers)
{
@@ -263,6 +357,10 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan,
if (!active)
return false;
+ if (chan->hw_partial_xfer &&
+ (completed_transfers & AXI_DMAC_FLAG_PARTIAL_XFER_DONE))
+ axi_dmac_dequeue_partial_xfers(chan);
+
do {
sg = &active->sg[active->num_completed];
if (sg->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */
@@ -276,10 +374,14 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan,
start_next = true;
}
+ if (sg->partial_len)
+ axi_dmac_compute_residue(chan, active);
+
if (active->cyclic)
vchan_cyclic_callback(&active->vdesc);
- if (active->num_completed == active->num_sgs) {
+ if (active->num_completed == active->num_sgs ||
+ sg->partial_len) {
if (active->cyclic) {
active->num_completed = 0; /* wrap around */
} else {
@@ -391,7 +493,7 @@ static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan,
num_segments = DIV_ROUND_UP(period_len, chan->max_length);
segment_size = DIV_ROUND_UP(period_len, num_segments);
/* Take care of alignment */
- segment_size = ((segment_size - 1) | chan->align_mask) + 1;
+ segment_size = ((segment_size - 1) | chan->length_align_mask) + 1;
for (i = 0; i < num_periods; i++) {
len = period_len;
@@ -561,6 +663,9 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved(
desc->sg[0].y_len = 1;
}
+ if (flags & DMA_CYCLIC)
+ desc->cyclic = true;
+
return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
}
@@ -574,6 +679,44 @@ static void axi_dmac_desc_free(struct virt_dma_desc *vdesc)
kfree(container_of(vdesc, struct axi_dmac_desc, vdesc));
}
+static bool axi_dmac_regmap_rdwr(struct device *dev, unsigned int reg)
+{
+ switch (reg) {
+ case AXI_DMAC_REG_IRQ_MASK:
+ case AXI_DMAC_REG_IRQ_SOURCE:
+ case AXI_DMAC_REG_IRQ_PENDING:
+ case AXI_DMAC_REG_CTRL:
+ case AXI_DMAC_REG_TRANSFER_ID:
+ case AXI_DMAC_REG_START_TRANSFER:
+ case AXI_DMAC_REG_FLAGS:
+ case AXI_DMAC_REG_DEST_ADDRESS:
+ case AXI_DMAC_REG_SRC_ADDRESS:
+ case AXI_DMAC_REG_X_LENGTH:
+ case AXI_DMAC_REG_Y_LENGTH:
+ case AXI_DMAC_REG_DEST_STRIDE:
+ case AXI_DMAC_REG_SRC_STRIDE:
+ case AXI_DMAC_REG_TRANSFER_DONE:
+ case AXI_DMAC_REG_ACTIVE_TRANSFER_ID:
+ case AXI_DMAC_REG_STATUS:
+ case AXI_DMAC_REG_CURRENT_SRC_ADDR:
+ case AXI_DMAC_REG_CURRENT_DEST_ADDR:
+ case AXI_DMAC_REG_PARTIAL_XFER_LEN:
+ case AXI_DMAC_REG_PARTIAL_XFER_ID:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static const struct regmap_config axi_dmac_regmap_config = {
+ .reg_bits = 32,
+ .val_bits = 32,
+ .reg_stride = 4,
+ .max_register = AXI_DMAC_REG_PARTIAL_XFER_ID,
+ .readable_reg = axi_dmac_regmap_rdwr,
+ .writeable_reg = axi_dmac_regmap_rdwr,
+};
+
/*
* The configuration stored in the devicetree matches the configuration
* parameters of the peripheral instance and allows the driver to know which
@@ -617,7 +760,7 @@ static int axi_dmac_parse_chan_dt(struct device_node *of_chan,
return ret;
chan->dest_width = val / 8;
- chan->align_mask = max(chan->dest_width, chan->src_width) - 1;
+ chan->address_align_mask = max(chan->dest_width, chan->src_width) - 1;
if (axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan))
chan->direction = DMA_MEM_TO_MEM;
@@ -631,9 +774,12 @@ static int axi_dmac_parse_chan_dt(struct device_node *of_chan,
return 0;
}
-static void axi_dmac_detect_caps(struct axi_dmac *dmac)
+static int axi_dmac_detect_caps(struct axi_dmac *dmac)
{
struct axi_dmac_chan *chan = &dmac->chan;
+ unsigned int version;
+
+ version = axi_dmac_read(dmac, ADI_AXI_REG_VERSION);
axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, AXI_DMAC_FLAG_CYCLIC);
if (axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS) == AXI_DMAC_FLAG_CYCLIC)
@@ -647,6 +793,35 @@ static void axi_dmac_detect_caps(struct axi_dmac *dmac)
chan->max_length = axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH);
if (chan->max_length != UINT_MAX)
chan->max_length++;
+
+ axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, 0xffffffff);
+ if (axi_dmac_read(dmac, AXI_DMAC_REG_DEST_ADDRESS) == 0 &&
+ chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM) {
+ dev_err(dmac->dma_dev.dev,
+ "Destination memory-mapped interface not supported.");
+ return -ENODEV;
+ }
+
+ axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, 0xffffffff);
+ if (axi_dmac_read(dmac, AXI_DMAC_REG_SRC_ADDRESS) == 0 &&
+ chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM) {
+ dev_err(dmac->dma_dev.dev,
+ "Source memory-mapped interface not supported.");
+ return -ENODEV;
+ }
+
+ if (version >= ADI_AXI_PCORE_VER(4, 2, 'a'))
+ chan->hw_partial_xfer = true;
+
+ if (version >= ADI_AXI_PCORE_VER(4, 1, 'a')) {
+ axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, 0x00);
+ chan->length_align_mask =
+ axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH);
+ } else {
+ chan->length_align_mask = chan->address_align_mask;
+ }
+
+ return 0;
}
static int axi_dmac_probe(struct platform_device *pdev)
@@ -722,7 +897,11 @@ static int axi_dmac_probe(struct platform_device *pdev)
if (ret < 0)
return ret;
- axi_dmac_detect_caps(dmac);
+ ret = axi_dmac_detect_caps(dmac);
+ if (ret)
+ goto err_clk_disable;
+
+ dma_dev->copy_align = (dmac->chan.address_align_mask + 1);
axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, 0x00);
@@ -742,6 +921,8 @@ static int axi_dmac_probe(struct platform_device *pdev)
platform_set_drvdata(pdev, dmac);
+ devm_regmap_init_mmio(&pdev->dev, dmac->base, &axi_dmac_regmap_config);
+
return 0;
err_unregister_of:
diff --git a/drivers/dma/dma-jz4780.c b/drivers/dma/dma-jz4780.c
index 6b8c4c458e8a..cafb1cc065bb 100644
--- a/drivers/dma/dma-jz4780.c
+++ b/drivers/dma/dma-jz4780.c
@@ -92,6 +92,7 @@
#define JZ_SOC_DATA_PROGRAMMABLE_DMA BIT(1)
#define JZ_SOC_DATA_PER_CHAN_PM BIT(2)
#define JZ_SOC_DATA_NO_DCKES_DCKEC BIT(3)
+#define JZ_SOC_DATA_BREAK_LINKS BIT(4)
/**
* struct jz4780_dma_hwdesc - descriptor structure read by the DMA controller.
@@ -156,7 +157,6 @@ struct jz4780_dma_dev {
};
struct jz4780_dma_filter_data {
- struct device_node *of_node;
uint32_t transfer_type;
int channel;
};
@@ -356,6 +356,7 @@ static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg(
void *context)
{
struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
+ struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);
struct jz4780_dma_desc *desc;
unsigned int i;
int err;
@@ -376,7 +377,8 @@ static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg(
desc->desc[i].dcm |= JZ_DMA_DCM_TIE;
- if (i != (sg_len - 1)) {
+ if (i != (sg_len - 1) &&
+ !(jzdma->soc_data->flags & JZ_SOC_DATA_BREAK_LINKS)) {
/* Automatically proceeed to the next descriptor. */
desc->desc[i].dcm |= JZ_DMA_DCM_LINK;
@@ -665,6 +667,8 @@ static enum dma_status jz4780_dma_tx_status(struct dma_chan *chan,
static bool jz4780_dma_chan_irq(struct jz4780_dma_dev *jzdma,
struct jz4780_dma_chan *jzchan)
{
+ const unsigned int soc_flags = jzdma->soc_data->flags;
+ struct jz4780_dma_desc *desc = jzchan->desc;
uint32_t dcs;
bool ack = true;
@@ -692,8 +696,11 @@ static bool jz4780_dma_chan_irq(struct jz4780_dma_dev *jzdma,
jz4780_dma_begin(jzchan);
} else if (dcs & JZ_DMA_DCS_TT) {
- vchan_cookie_complete(&jzchan->desc->vdesc);
- jzchan->desc = NULL;
+ if (!(soc_flags & JZ_SOC_DATA_BREAK_LINKS) ||
+ (jzchan->curr_hwdesc + 1 == desc->count)) {
+ vchan_cookie_complete(&desc->vdesc);
+ jzchan->desc = NULL;
+ }
jz4780_dma_begin(jzchan);
} else {
@@ -772,8 +779,6 @@ static bool jz4780_dma_filter_fn(struct dma_chan *chan, void *param)
struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);
struct jz4780_dma_filter_data *data = param;
- if (jzdma->dma_device.dev->of_node != data->of_node)
- return false;
if (data->channel > -1) {
if (data->channel != jzchan->id)
@@ -797,7 +802,6 @@ static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec,
if (dma_spec->args_count != 2)
return NULL;
- data.of_node = ofdma->of_node;
data.transfer_type = dma_spec->args[0];
data.channel = dma_spec->args[1];
@@ -822,7 +826,8 @@ static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec,
return dma_get_slave_channel(
&jzdma->chan[data.channel].vchan.chan);
} else {
- return dma_request_channel(mask, jz4780_dma_filter_fn, &data);
+ return __dma_request_channel(&mask, jz4780_dma_filter_fn, &data,
+ ofdma->of_node);
}
}
@@ -881,10 +886,8 @@ static int jz4780_dma_probe(struct platform_device *pdev)
}
ret = platform_get_irq(pdev, 0);
- if (ret < 0) {
- dev_err(dev, "failed to get IRQ: %d\n", ret);
+ if (ret < 0)
return ret;
- }
jzdma->irq = ret;
@@ -995,6 +998,7 @@ static int jz4780_dma_remove(struct platform_device *pdev)
static const struct jz4780_dma_soc_data jz4740_dma_soc_data = {
.nb_channels = 6,
.transfer_ord_max = 5,
+ .flags = JZ_SOC_DATA_BREAK_LINKS,
};
static const struct jz4780_dma_soc_data jz4725b_dma_soc_data = {
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 58cbf9fd5a46..03ac4b96117c 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -61,7 +61,7 @@ static long dmaengine_ref_count;
/* --- sysfs implementation --- */
/**
- * dev_to_dma_chan - convert a device pointer to the its sysfs container object
+ * dev_to_dma_chan - convert a device pointer to its sysfs container object
* @dev - device node
*
* Must be called under dma_list_mutex
@@ -629,11 +629,13 @@ EXPORT_SYMBOL_GPL(dma_get_any_slave_channel);
* @mask: capabilities that the channel must satisfy
* @fn: optional callback to disposition available channels
* @fn_param: opaque parameter to pass to dma_filter_fn
+ * @np: device node to look for DMA channels
*
* Returns pointer to appropriate DMA channel on success or NULL.
*/
struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask,
- dma_filter_fn fn, void *fn_param)
+ dma_filter_fn fn, void *fn_param,
+ struct device_node *np)
{
struct dma_device *device, *_d;
struct dma_chan *chan = NULL;
@@ -641,6 +643,10 @@ struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask,
/* Find a channel */
mutex_lock(&dma_list_mutex);
list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
+ /* Finds a DMA controller with matching device node */
+ if (np && device->dev->of_node && np != device->dev->of_node)
+ continue;
+
chan = find_candidate(device, mask, fn, fn_param);
if (!IS_ERR(chan))
break;
@@ -699,7 +705,7 @@ struct dma_chan *dma_request_chan(struct device *dev, const char *name)
chan = acpi_dma_request_slave_chan_by_name(dev, name);
if (chan) {
- /* Valid channel found or requester need to be deferred */
+ /* Valid channel found or requester needs to be deferred */
if (!IS_ERR(chan) || PTR_ERR(chan) == -EPROBE_DEFER)
return chan;
}
@@ -757,7 +763,7 @@ struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask)
if (!mask)
return ERR_PTR(-ENODEV);
- chan = __dma_request_channel(mask, NULL, NULL);
+ chan = __dma_request_channel(mask, NULL, NULL, NULL);
if (!chan) {
mutex_lock(&dma_list_mutex);
if (list_empty(&dma_device_list))
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index d0ad46e916a6..a2cadfa2e6d7 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -62,7 +62,7 @@ MODULE_PARM_DESC(pq_sources,
static int timeout = 3000;
module_param(timeout, uint, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
- "Pass -1 for infinite timeout");
+ "Pass 0xFFFFFFFF (4294967295) for maximum timeout");
static bool noverify;
module_param(noverify, bool, S_IRUGO | S_IWUSR);
@@ -72,6 +72,10 @@ static bool norandom;
module_param(norandom, bool, 0644);
MODULE_PARM_DESC(norandom, "Disable random offset setup (default: random)");
+static bool polled;
+module_param(polled, bool, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(polled, "Use polling for completion instead of interrupts");
+
static bool verbose;
module_param(verbose, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
@@ -94,7 +98,7 @@ MODULE_PARM_DESC(transfer_size, "Optional custom transfer size in bytes (default
* @iterations: iterations before stopping test
* @xor_sources: number of xor source buffers
* @pq_sources: number of p+q source buffers
- * @timeout: transfer timeout in msec, -1 for infinite timeout
+ * @timeout: transfer timeout in msec, 0 - 0xFFFFFFFF (4294967295)
*/
struct dmatest_params {
unsigned int buf_size;
@@ -105,11 +109,12 @@ struct dmatest_params {
unsigned int iterations;
unsigned int xor_sources;
unsigned int pq_sources;
- int timeout;
+ unsigned int timeout;
bool noverify;
bool norandom;
int alignment;
unsigned int transfer_size;
+ bool polled;
};
/**
@@ -651,7 +656,10 @@ static int dmatest_func(void *data)
/*
* src and dst buffers are freed by ourselves below
*/
- flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
+ if (params->polled)
+ flags = DMA_CTRL_ACK;
+ else
+ flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
ktime = ktime_get();
while (!kthread_should_stop()
@@ -780,8 +788,10 @@ static int dmatest_func(void *data)
}
done->done = false;
- tx->callback = dmatest_callback;
- tx->callback_param = done;
+ if (!params->polled) {
+ tx->callback = dmatest_callback;
+ tx->callback_param = done;
+ }
cookie = tx->tx_submit(tx);
if (dma_submit_error(cookie)) {
@@ -790,12 +800,22 @@ static int dmatest_func(void *data)
msleep(100);
goto error_unmap_continue;
}
- dma_async_issue_pending(chan);
- wait_event_freezable_timeout(thread->done_wait, done->done,
- msecs_to_jiffies(params->timeout));
+ if (params->polled) {
+ status = dma_sync_wait(chan, cookie);
+ dmaengine_terminate_sync(chan);
+ if (status == DMA_COMPLETE)
+ done->done = true;
+ } else {
+ dma_async_issue_pending(chan);
+
+ wait_event_freezable_timeout(thread->done_wait,
+ done->done,
+ msecs_to_jiffies(params->timeout));
- status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
+ status = dma_async_is_tx_complete(chan, cookie, NULL,
+ NULL);
+ }
if (!done->done) {
result("test timed out", total_tests, src->off, dst->off,
@@ -1065,6 +1085,7 @@ static void add_threaded_test(struct dmatest_info *info)
params->norandom = norandom;
params->alignment = alignment;
params->transfer_size = transfer_size;
+ params->polled = polled;
request_channels(info, DMA_MEMCPY);
request_channels(info, DMA_MEMSET);
diff --git a/drivers/dma/dw-edma/Kconfig b/drivers/dma/dw-edma/Kconfig
new file mode 100644
index 000000000000..7ff17b2db6a1
--- /dev/null
+++ b/drivers/dma/dw-edma/Kconfig
@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config DW_EDMA
+ tristate "Synopsys DesignWare eDMA controller driver"
+ depends on PCI && PCI_MSI
+ select DMA_ENGINE
+ select DMA_VIRTUAL_CHANNELS
+ help
+ Support the Synopsys DesignWare eDMA controller, normally
+ implemented on endpoints SoCs.
+
+config DW_EDMA_PCIE
+ tristate "Synopsys DesignWare eDMA PCIe driver"
+ depends on PCI && PCI_MSI
+ select DW_EDMA
+ help
+ Provides a glue-logic between the Synopsys DesignWare
+ eDMA controller and an endpoint PCIe device. This also serves
+ as a reference design to whom desires to use this IP.
diff --git a/drivers/dma/dw-edma/Makefile b/drivers/dma/dw-edma/Makefile
new file mode 100644
index 000000000000..8d45c0d5689d
--- /dev/null
+++ b/drivers/dma/dw-edma/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_DW_EDMA) += dw-edma.o
+dw-edma-$(CONFIG_DEBUG_FS) := dw-edma-v0-debugfs.o
+dw-edma-objs := dw-edma-core.o \
+ dw-edma-v0-core.o $(dw-edma-y)
+obj-$(CONFIG_DW_EDMA_PCIE) += dw-edma-pcie.o
diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
new file mode 100644
index 000000000000..ff392c01bad1
--- /dev/null
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -0,0 +1,937 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ * Synopsys DesignWare eDMA core driver
+ *
+ * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
+ */
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/pm_runtime.h>
+#include <linux/dmaengine.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/dma/edma.h>
+#include <linux/pci.h>
+
+#include "dw-edma-core.h"
+#include "dw-edma-v0-core.h"
+#include "../dmaengine.h"
+#include "../virt-dma.h"
+
+static inline
+struct device *dchan2dev(struct dma_chan *dchan)
+{
+ return &dchan->dev->device;
+}
+
+static inline
+struct device *chan2dev(struct dw_edma_chan *chan)
+{
+ return &chan->vc.chan.dev->device;
+}
+
+static inline
+struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd)
+{
+ return container_of(vd, struct dw_edma_desc, vd);
+}
+
+static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk)
+{
+ struct dw_edma_burst *burst;
+
+ burst = kzalloc(sizeof(*burst), GFP_NOWAIT);
+ if (unlikely(!burst))
+ return NULL;
+
+ INIT_LIST_HEAD(&burst->list);
+ if (chunk->burst) {
+ /* Create and add new element into the linked list */
+ chunk->bursts_alloc++;
+ list_add_tail(&burst->list, &chunk->burst->list);
+ } else {
+ /* List head */
+ chunk->bursts_alloc = 0;
+ chunk->burst = burst;
+ }
+
+ return burst;
+}
+
+static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc)
+{
+ struct dw_edma_chan *chan = desc->chan;
+ struct dw_edma *dw = chan->chip->dw;
+ struct dw_edma_chunk *chunk;
+
+ chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT);
+ if (unlikely(!chunk))
+ return NULL;
+
+ INIT_LIST_HEAD(&chunk->list);
+ chunk->chan = chan;
+ /* Toggling change bit (CB) in each chunk, this is a mechanism to
+ * inform the eDMA HW block that this is a new linked list ready
+ * to be consumed.
+ * - Odd chunks originate CB equal to 0
+ * - Even chunks originate CB equal to 1
+ */
+ chunk->cb = !(desc->chunks_alloc % 2);
+ chunk->ll_region.paddr = dw->ll_region.paddr + chan->ll_off;
+ chunk->ll_region.vaddr = dw->ll_region.vaddr + chan->ll_off;
+
+ if (desc->chunk) {
+ /* Create and add new element into the linked list */
+ desc->chunks_alloc++;
+ list_add_tail(&chunk->list, &desc->chunk->list);
+ if (!dw_edma_alloc_burst(chunk)) {
+ kfree(chunk);
+ return NULL;
+ }
+ } else {
+ /* List head */
+ chunk->burst = NULL;
+ desc->chunks_alloc = 0;
+ desc->chunk = chunk;
+ }
+
+ return chunk;
+}
+
+static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan)
+{
+ struct dw_edma_desc *desc;
+
+ desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
+ if (unlikely(!desc))
+ return NULL;
+
+ desc->chan = chan;
+ if (!dw_edma_alloc_chunk(desc)) {
+ kfree(desc);
+ return NULL;
+ }
+
+ return desc;
+}
+
+static void dw_edma_free_burst(struct dw_edma_chunk *chunk)
+{
+ struct dw_edma_burst *child, *_next;
+
+ /* Remove all the list elements */
+ list_for_each_entry_safe(child, _next, &chunk->burst->list, list) {
+ list_del(&child->list);
+ kfree(child);
+ chunk->bursts_alloc--;
+ }
+
+ /* Remove the list head */
+ kfree(child);
+ chunk->burst = NULL;
+}
+
+static void dw_edma_free_chunk(struct dw_edma_desc *desc)
+{
+ struct dw_edma_chunk *child, *_next;
+
+ if (!desc->chunk)
+ return;
+
+ /* Remove all the list elements */
+ list_for_each_entry_safe(child, _next, &desc->chunk->list, list) {
+ dw_edma_free_burst(child);
+ list_del(&child->list);
+ kfree(child);
+ desc->chunks_alloc--;
+ }
+
+ /* Remove the list head */
+ kfree(child);
+ desc->chunk = NULL;
+}
+
+static void dw_edma_free_desc(struct dw_edma_desc *desc)
+{
+ dw_edma_free_chunk(desc);
+ kfree(desc);
+}
+
+static void vchan_free_desc(struct virt_dma_desc *vdesc)
+{
+ dw_edma_free_desc(vd2dw_edma_desc(vdesc));
+}
+
+static void dw_edma_start_transfer(struct dw_edma_chan *chan)
+{
+ struct dw_edma_chunk *child;
+ struct dw_edma_desc *desc;
+ struct virt_dma_desc *vd;
+
+ vd = vchan_next_desc(&chan->vc);
+ if (!vd)
+ return;
+
+ desc = vd2dw_edma_desc(vd);
+ if (!desc)
+ return;
+
+ child = list_first_entry_or_null(&desc->chunk->list,
+ struct dw_edma_chunk, list);
+ if (!child)
+ return;
+
+ dw_edma_v0_core_start(child, !desc->xfer_sz);
+ desc->xfer_sz += child->ll_region.sz;
+ dw_edma_free_burst(child);
+ list_del(&child->list);
+ kfree(child);
+ desc->chunks_alloc--;
+}
+
+static int dw_edma_device_config(struct dma_chan *dchan,
+ struct dma_slave_config *config)
+{
+ struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
+
+ memcpy(&chan->config, config, sizeof(*config));
+ chan->configured = true;
+
+ return 0;
+}
+
+static int dw_edma_device_pause(struct dma_chan *dchan)
+{
+ struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
+ int err = 0;
+
+ if (!chan->configured)
+ err = -EPERM;
+ else if (chan->status != EDMA_ST_BUSY)
+ err = -EPERM;
+ else if (chan->request != EDMA_REQ_NONE)
+ err = -EPERM;
+ else
+ chan->request = EDMA_REQ_PAUSE;
+
+ return err;
+}
+
+static int dw_edma_device_resume(struct dma_chan *dchan)
+{
+ struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
+ int err = 0;
+
+ if (!chan->configured) {
+ err = -EPERM;
+ } else if (chan->status != EDMA_ST_PAUSE) {
+ err = -EPERM;
+ } else if (chan->request != EDMA_REQ_NONE) {
+ err = -EPERM;
+ } else {
+ chan->status = EDMA_ST_BUSY;
+ dw_edma_start_transfer(chan);
+ }
+
+ return err;
+}
+
+static int dw_edma_device_terminate_all(struct dma_chan *dchan)
+{
+ struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
+ int err = 0;
+ LIST_HEAD(head);
+
+ if (!chan->configured) {
+ /* Do nothing */
+ } else if (chan->status == EDMA_ST_PAUSE) {
+ chan->status = EDMA_ST_IDLE;
+ chan->configured = false;
+ } else if (chan->status == EDMA_ST_IDLE) {
+ chan->configured = false;
+ } else if (dw_edma_v0_core_ch_status(chan) == DMA_COMPLETE) {
+ /*
+ * The channel is in a false BUSY state, probably didn't
+ * receive or lost an interrupt
+ */
+ chan->status = EDMA_ST_IDLE;
+ chan->configured = false;
+ } else if (chan->request > EDMA_REQ_PAUSE) {
+ err = -EPERM;
+ } else {
+ chan->request = EDMA_REQ_STOP;
+ }
+
+ return err;
+}
+
+static void dw_edma_device_issue_pending(struct dma_chan *dchan)
+{
+ struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
+ unsigned long flags;
+
+ spin_lock_irqsave(&chan->vc.lock, flags);
+ if (chan->configured && chan->request == EDMA_REQ_NONE &&
+ chan->status == EDMA_ST_IDLE && vchan_issue_pending(&chan->vc)) {
+ chan->status = EDMA_ST_BUSY;
+ dw_edma_start_transfer(chan);
+ }
+ spin_unlock_irqrestore(&chan->vc.lock, flags);
+}
+
+static enum dma_status
+dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
+ struct dma_tx_state *txstate)
+{
+ struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
+ struct dw_edma_desc *desc;
+ struct virt_dma_desc *vd;
+ unsigned long flags;
+ enum dma_status ret;
+ u32 residue = 0;
+
+ ret = dma_cookie_status(dchan, cookie, txstate);
+ if (ret == DMA_COMPLETE)
+ return ret;
+
+ if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE)
+ ret = DMA_PAUSED;
+
+ if (!txstate)
+ goto ret_residue;
+
+ spin_lock_irqsave(&chan->vc.lock, flags);
+ vd = vchan_find_desc(&chan->vc, cookie);
+ if (vd) {
+ desc = vd2dw_edma_desc(vd);
+ if (desc)
+ residue = desc->alloc_sz - desc->xfer_sz;
+ }
+ spin_unlock_irqrestore(&chan->vc.lock, flags);
+
+ret_residue:
+ dma_set_residue(txstate, residue);
+
+ return ret;
+}
+
+static struct dma_async_tx_descriptor *
+dw_edma_device_transfer(struct dw_edma_transfer *xfer)
+{
+ struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan);
+ enum dma_transfer_direction direction = xfer->direction;
+ phys_addr_t src_addr, dst_addr;
+ struct scatterlist *sg = NULL;
+ struct dw_edma_chunk *chunk;
+ struct dw_edma_burst *burst;
+ struct dw_edma_desc *desc;
+ u32 cnt;
+ int i;
+
+ if ((direction == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_WRITE) ||
+ (direction == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_READ))
+ return NULL;
+
+ if (xfer->cyclic) {
+ if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt)
+ return NULL;
+ } else {
+ if (xfer->xfer.sg.len < 1)
+ return NULL;
+ }
+
+ if (!chan->configured)
+ return NULL;
+
+ desc = dw_edma_alloc_desc(chan);
+ if (unlikely(!desc))
+ goto err_alloc;
+
+ chunk = dw_edma_alloc_chunk(desc);
+ if (unlikely(!chunk))
+ goto err_alloc;
+
+ src_addr = chan->config.src_addr;
+ dst_addr = chan->config.dst_addr;
+
+ if (xfer->cyclic) {
+ cnt = xfer->xfer.cyclic.cnt;
+ } else {
+ cnt = xfer->xfer.sg.len;
+ sg = xfer->xfer.sg.sgl;
+ }
+
+ for (i = 0; i < cnt; i++) {
+ if (!xfer->cyclic && !sg)
+ break;
+
+ if (chunk->bursts_alloc == chan->ll_max) {
+ chunk = dw_edma_alloc_chunk(desc);
+ if (unlikely(!chunk))
+ goto err_alloc;
+ }
+
+ burst = dw_edma_alloc_burst(chunk);
+ if (unlikely(!burst))
+ goto err_alloc;
+
+ if (xfer->cyclic)
+ burst->sz = xfer->xfer.cyclic.len;
+ else
+ burst->sz = sg_dma_len(sg);
+
+ chunk->ll_region.sz += burst->sz;
+ desc->alloc_sz += burst->sz;
+
+ if (direction == DMA_DEV_TO_MEM) {
+ burst->sar = src_addr;
+ if (xfer->cyclic) {
+ burst->dar = xfer->xfer.cyclic.paddr;
+ } else {
+ burst->dar = sg_dma_address(sg);
+ /* Unlike the typical assumption by other
+ * drivers/IPs the peripheral memory isn't
+ * a FIFO memory, in this case, it's a
+ * linear memory and that why the source
+ * and destination addresses are increased
+ * by the same portion (data length)
+ */
+ src_addr += sg_dma_len(sg);
+ }
+ } else {
+ burst->dar = dst_addr;
+ if (xfer->cyclic) {
+ burst->sar = xfer->xfer.cyclic.paddr;
+ } else {
+ burst->sar = sg_dma_address(sg);
+ /* Unlike the typical assumption by other
+ * drivers/IPs the peripheral memory isn't
+ * a FIFO memory, in this case, it's a
+ * linear memory and that why the source
+ * and destination addresses are increased
+ * by the same portion (data length)
+ */
+ dst_addr += sg_dma_len(sg);
+ }
+ }
+
+ if (!xfer->cyclic)
+ sg = sg_next(sg);
+ }
+
+ return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags);
+
+err_alloc:
+ if (desc)
+ dw_edma_free_desc(desc);
+
+ return NULL;
+}
+
+static struct dma_async_tx_descriptor *
+dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
+ unsigned int len,
+ enum dma_transfer_direction direction,
+ unsigned long flags, void *context)
+{
+ struct dw_edma_transfer xfer;
+
+ xfer.dchan = dchan;
+ xfer.direction = direction;
+ xfer.xfer.sg.sgl = sgl;
+ xfer.xfer.sg.len = len;
+ xfer.flags = flags;
+ xfer.cyclic = false;
+
+ return dw_edma_device_transfer(&xfer);
+}
+
+static struct dma_async_tx_descriptor *
+dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr,
+ size_t len, size_t count,
+ enum dma_transfer_direction direction,
+ unsigned long flags)
+{
+ struct dw_edma_transfer xfer;
+
+ xfer.dchan = dchan;
+ xfer.direction = direction;
+ xfer.xfer.cyclic.paddr = paddr;
+ xfer.xfer.cyclic.len = len;
+ xfer.xfer.cyclic.cnt = count;
+ xfer.flags = flags;
+ xfer.cyclic = true;
+
+ return dw_edma_device_transfer(&xfer);
+}
+
+static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
+{
+ struct dw_edma_desc *desc;
+ struct virt_dma_desc *vd;
+ unsigned long flags;
+
+ dw_edma_v0_core_clear_done_int(chan);
+
+ spin_lock_irqsave(&chan->vc.lock, flags);
+ vd = vchan_next_desc(&chan->vc);
+ if (vd) {
+ switch (chan->request) {
+ case EDMA_REQ_NONE:
+ desc = vd2dw_edma_desc(vd);
+ if (desc->chunks_alloc) {
+ chan->status = EDMA_ST_BUSY;
+ dw_edma_start_transfer(chan);
+ } else {
+ list_del(&vd->node);
+ vchan_cookie_complete(vd);
+ chan->status = EDMA_ST_IDLE;
+ }
+ break;
+
+ case EDMA_REQ_STOP:
+ list_del(&vd->node);
+ vchan_cookie_complete(vd);
+ chan->request = EDMA_REQ_NONE;
+ chan->status = EDMA_ST_IDLE;
+ break;
+
+ case EDMA_REQ_PAUSE:
+ chan->request = EDMA_REQ_NONE;
+ chan->status = EDMA_ST_PAUSE;
+ break;
+
+ default:
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&chan->vc.lock, flags);
+}
+
+static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
+{
+ struct virt_dma_desc *vd;
+ unsigned long flags;
+
+ dw_edma_v0_core_clear_abort_int(chan);
+
+ spin_lock_irqsave(&chan->vc.lock, flags);
+ vd = vchan_next_desc(&chan->vc);
+ if (vd) {
+ list_del(&vd->node);
+ vchan_cookie_complete(vd);
+ }
+ spin_unlock_irqrestore(&chan->vc.lock, flags);
+ chan->request = EDMA_REQ_NONE;
+ chan->status = EDMA_ST_IDLE;
+}
+
+static irqreturn_t dw_edma_interrupt(int irq, void *data, bool write)
+{
+ struct dw_edma_irq *dw_irq = data;
+ struct dw_edma *dw = dw_irq->dw;
+ unsigned long total, pos, val;
+ unsigned long off;
+ u32 mask;
+
+ if (write) {
+ total = dw->wr_ch_cnt;
+ off = 0;
+ mask = dw_irq->wr_mask;
+ } else {
+ total = dw->rd_ch_cnt;
+ off = dw->wr_ch_cnt;
+ mask = dw_irq->rd_mask;
+ }
+
+ val = dw_edma_v0_core_status_done_int(dw, write ?
+ EDMA_DIR_WRITE :
+ EDMA_DIR_READ);
+ val &= mask;
+ for_each_set_bit(pos, &val, total) {
+ struct dw_edma_chan *chan = &dw->chan[pos + off];
+
+ dw_edma_done_interrupt(chan);
+ }
+
+ val = dw_edma_v0_core_status_abort_int(dw, write ?
+ EDMA_DIR_WRITE :
+ EDMA_DIR_READ);
+ val &= mask;
+ for_each_set_bit(pos, &val, total) {
+ struct dw_edma_chan *chan = &dw->chan[pos + off];
+
+ dw_edma_abort_interrupt(chan);
+ }
+
+ return IRQ_HANDLED;
+}
+
+static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data)
+{
+ return dw_edma_interrupt(irq, data, true);
+}
+
+static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data)
+{
+ return dw_edma_interrupt(irq, data, false);
+}
+
+static irqreturn_t dw_edma_interrupt_common(int irq, void *data)
+{
+ dw_edma_interrupt(irq, data, true);
+ dw_edma_interrupt(irq, data, false);
+
+ return IRQ_HANDLED;
+}
+
+static int dw_edma_alloc_chan_resources(struct dma_chan *dchan)
+{
+ struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
+
+ if (chan->status != EDMA_ST_IDLE)
+ return -EBUSY;
+
+ pm_runtime_get(chan->chip->dev);
+
+ return 0;
+}
+
+static void dw_edma_free_chan_resources(struct dma_chan *dchan)
+{
+ unsigned long timeout = jiffies + msecs_to_jiffies(5000);
+ struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
+ int ret;
+
+ while (time_before(jiffies, timeout)) {
+ ret = dw_edma_device_terminate_all(dchan);
+ if (!ret)
+ break;
+
+ if (time_after_eq(jiffies, timeout))
+ return;
+
+ cpu_relax();
+ }
+
+ pm_runtime_put(chan->chip->dev);
+}
+
+static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write,
+ u32 wr_alloc, u32 rd_alloc)
+{
+ struct dw_edma_region *dt_region;
+ struct device *dev = chip->dev;
+ struct dw_edma *dw = chip->dw;
+ struct dw_edma_chan *chan;
+ size_t ll_chunk, dt_chunk;
+ struct dw_edma_irq *irq;
+ struct dma_device *dma;
+ u32 i, j, cnt, ch_cnt;
+ u32 alloc, off_alloc;
+ int err = 0;
+ u32 pos;
+
+ ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
+ ll_chunk = dw->ll_region.sz;
+ dt_chunk = dw->dt_region.sz;
+
+ /* Calculate linked list chunk for each channel */
+ ll_chunk /= roundup_pow_of_two(ch_cnt);
+
+ /* Calculate linked list chunk for each channel */
+ dt_chunk /= roundup_pow_of_two(ch_cnt);
+
+ if (write) {
+ i = 0;
+ cnt = dw->wr_ch_cnt;
+ dma = &dw->wr_edma;
+ alloc = wr_alloc;
+ off_alloc = 0;
+ } else {
+ i = dw->wr_ch_cnt;
+ cnt = dw->rd_ch_cnt;
+ dma = &dw->rd_edma;
+ alloc = rd_alloc;
+ off_alloc = wr_alloc;
+ }
+
+ INIT_LIST_HEAD(&dma->channels);
+ for (j = 0; (alloc || dw->nr_irqs == 1) && j < cnt; j++, i++) {
+ chan = &dw->chan[i];
+
+ dt_region = devm_kzalloc(dev, sizeof(*dt_region), GFP_KERNEL);
+ if (!dt_region)
+ return -ENOMEM;
+
+ chan->vc.chan.private = dt_region;
+
+ chan->chip = chip;
+ chan->id = j;
+ chan->dir = write ? EDMA_DIR_WRITE : EDMA_DIR_READ;
+ chan->configured = false;
+ chan->request = EDMA_REQ_NONE;
+ chan->status = EDMA_ST_IDLE;
+
+ chan->ll_off = (ll_chunk * i);
+ chan->ll_max = (ll_chunk / EDMA_LL_SZ) - 1;
+
+ chan->dt_off = (dt_chunk * i);
+
+ dev_vdbg(dev, "L. List:\tChannel %s[%u] off=0x%.8lx, max_cnt=%u\n",
+ write ? "write" : "read", j,
+ chan->ll_off, chan->ll_max);
+
+ if (dw->nr_irqs == 1)
+ pos = 0;
+ else
+ pos = off_alloc + (j % alloc);
+
+ irq = &dw->irq[pos];
+
+ if (write)
+ irq->wr_mask |= BIT(j);
+ else
+ irq->rd_mask |= BIT(j);
+
+ irq->dw = dw;
+ memcpy(&chan->msi, &irq->msi, sizeof(chan->msi));
+
+ dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n",
+ write ? "write" : "read", j,
+ chan->msi.address_hi, chan->msi.address_lo,
+ chan->msi.data);
+
+ chan->vc.desc_free = vchan_free_desc;
+ vchan_init(&chan->vc, dma);
+
+ dt_region->paddr = dw->dt_region.paddr + chan->dt_off;
+ dt_region->vaddr = dw->dt_region.vaddr + chan->dt_off;
+ dt_region->sz = dt_chunk;
+
+ dev_vdbg(dev, "Data:\tChannel %s[%u] off=0x%.8lx\n",
+ write ? "write" : "read", j, chan->dt_off);
+
+ dw_edma_v0_core_device_config(chan);
+ }
+
+ /* Set DMA channel capabilities */
+ dma_cap_zero(dma->cap_mask);
+ dma_cap_set(DMA_SLAVE, dma->cap_mask);
+ dma_cap_set(DMA_CYCLIC, dma->cap_mask);
+ dma_cap_set(DMA_PRIVATE, dma->cap_mask);
+ dma->directions = BIT(write ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV);
+ dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
+ dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
+ dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
+ dma->chancnt = cnt;
+
+ /* Set DMA channel callbacks */
+ dma->dev = chip->dev;
+ dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources;
+ dma->device_free_chan_resources = dw_edma_free_chan_resources;
+ dma->device_config = dw_edma_device_config;
+ dma->device_pause = dw_edma_device_pause;
+ dma->device_resume = dw_edma_device_resume;
+ dma->device_terminate_all = dw_edma_device_terminate_all;
+ dma->device_issue_pending = dw_edma_device_issue_pending;
+ dma->device_tx_status = dw_edma_device_tx_status;
+ dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg;
+ dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic;
+
+ dma_set_max_seg_size(dma->dev, U32_MAX);
+
+ /* Register DMA device */
+ err = dma_async_device_register(dma);
+
+ return err;
+}
+
+static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt)
+{
+ if (*nr_irqs && *alloc < cnt) {
+ (*alloc)++;
+ (*nr_irqs)--;
+ }
+}
+
+static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt)
+{
+ while (*mask * alloc < cnt)
+ (*mask)++;
+}
+
+static int dw_edma_irq_request(struct dw_edma_chip *chip,
+ u32 *wr_alloc, u32 *rd_alloc)
+{
+ struct device *dev = chip->dev;
+ struct dw_edma *dw = chip->dw;
+ u32 wr_mask = 1;
+ u32 rd_mask = 1;
+ int i, err = 0;
+ u32 ch_cnt;
+
+ ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
+
+ if (dw->nr_irqs < 1)
+ return -EINVAL;
+
+ if (dw->nr_irqs == 1) {
+ /* Common IRQ shared among all channels */
+ err = request_irq(pci_irq_vector(to_pci_dev(dev), 0),
+ dw_edma_interrupt_common,
+ IRQF_SHARED, dw->name, &dw->irq[0]);
+ if (err) {
+ dw->nr_irqs = 0;
+ return err;
+ }
+
+ get_cached_msi_msg(pci_irq_vector(to_pci_dev(dev), 0),
+ &dw->irq[0].msi);
+ } else {
+ /* Distribute IRQs equally among all channels */
+ int tmp = dw->nr_irqs;
+
+ while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) {
+ dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt);
+ dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt);
+ }
+
+ dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt);
+ dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt);
+
+ for (i = 0; i < (*wr_alloc + *rd_alloc); i++) {
+ err = request_irq(pci_irq_vector(to_pci_dev(dev), i),
+ i < *wr_alloc ?
+ dw_edma_interrupt_write :
+ dw_edma_interrupt_read,
+ IRQF_SHARED, dw->name,
+ &dw->irq[i]);
+ if (err) {
+ dw->nr_irqs = i;
+ return err;
+ }
+
+ get_cached_msi_msg(pci_irq_vector(to_pci_dev(dev), i),
+ &dw->irq[i].msi);
+ }
+
+ dw->nr_irqs = i;
+ }
+
+ return err;
+}
+
+int dw_edma_probe(struct dw_edma_chip *chip)
+{
+ struct device *dev = chip->dev;
+ struct dw_edma *dw = chip->dw;
+ u32 wr_alloc = 0;
+ u32 rd_alloc = 0;
+ int i, err;
+
+ raw_spin_lock_init(&dw->lock);
+
+ /* Find out how many write channels are supported by hardware */
+ dw->wr_ch_cnt = dw_edma_v0_core_ch_count(dw, EDMA_DIR_WRITE);
+ if (!dw->wr_ch_cnt)
+ return -EINVAL;
+
+ /* Find out how many read channels are supported by hardware */
+ dw->rd_ch_cnt = dw_edma_v0_core_ch_count(dw, EDMA_DIR_READ);
+ if (!dw->rd_ch_cnt)
+ return -EINVAL;
+
+ dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n",
+ dw->wr_ch_cnt, dw->rd_ch_cnt);
+
+ /* Allocate channels */
+ dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt,
+ sizeof(*dw->chan), GFP_KERNEL);
+ if (!dw->chan)
+ return -ENOMEM;
+
+ snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%d", chip->id);
+
+ /* Disable eDMA, only to establish the ideal initial conditions */
+ dw_edma_v0_core_off(dw);
+
+ /* Request IRQs */
+ err = dw_edma_irq_request(chip, &wr_alloc, &rd_alloc);
+ if (err)
+ return err;
+
+ /* Setup write channels */
+ err = dw_edma_channel_setup(chip, true, wr_alloc, rd_alloc);
+ if (err)
+ goto err_irq_free;
+
+ /* Setup read channels */
+ err = dw_edma_channel_setup(chip, false, wr_alloc, rd_alloc);
+ if (err)
+ goto err_irq_free;
+
+ /* Power management */
+ pm_runtime_enable(dev);
+
+ /* Turn debugfs on */
+ dw_edma_v0_core_debugfs_on(chip);
+
+ return 0;
+
+err_irq_free:
+ for (i = (dw->nr_irqs - 1); i >= 0; i--)
+ free_irq(pci_irq_vector(to_pci_dev(dev), i), &dw->irq[i]);
+
+ dw->nr_irqs = 0;
+
+ return err;
+}
+EXPORT_SYMBOL_GPL(dw_edma_probe);
+
+int dw_edma_remove(struct dw_edma_chip *chip)
+{
+ struct dw_edma_chan *chan, *_chan;
+ struct device *dev = chip->dev;
+ struct dw_edma *dw = chip->dw;
+ int i;
+
+ /* Disable eDMA */
+ dw_edma_v0_core_off(dw);
+
+ /* Free irqs */
+ for (i = (dw->nr_irqs - 1); i >= 0; i--)
+ free_irq(pci_irq_vector(to_pci_dev(dev), i), &dw->irq[i]);
+
+ /* Power management */
+ pm_runtime_disable(dev);
+
+ list_for_each_entry_safe(chan, _chan, &dw->wr_edma.channels,
+ vc.chan.device_node) {
+ list_del(&chan->vc.chan.device_node);
+ tasklet_kill(&chan->vc.task);
+ }
+
+ list_for_each_entry_safe(chan, _chan, &dw->rd_edma.channels,
+ vc.chan.device_node) {
+ list_del(&chan->vc.chan.device_node);
+ tasklet_kill(&chan->vc.task);
+ }
+
+ /* Deregister eDMA device */
+ dma_async_device_unregister(&dw->wr_edma);
+ dma_async_device_unregister(&dw->rd_edma);
+
+ /* Turn debugfs off */
+ dw_edma_v0_core_debugfs_off();
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(dw_edma_remove);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver");
+MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>");
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
new file mode 100644
index 000000000000..4e5f9f6e901b
--- /dev/null
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -0,0 +1,165 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ * Synopsys DesignWare eDMA core driver
+ *
+ * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
+ */
+
+#ifndef _DW_EDMA_CORE_H
+#define _DW_EDMA_CORE_H
+
+#include <linux/msi.h>
+#include <linux/dma/edma.h>
+
+#include "../virt-dma.h"
+
+#define EDMA_LL_SZ 24
+
+enum dw_edma_dir {
+ EDMA_DIR_WRITE = 0,
+ EDMA_DIR_READ
+};
+
+enum dw_edma_mode {
+ EDMA_MODE_LEGACY = 0,
+ EDMA_MODE_UNROLL
+};
+
+enum dw_edma_request {
+ EDMA_REQ_NONE = 0,
+ EDMA_REQ_STOP,
+ EDMA_REQ_PAUSE
+};
+
+enum dw_edma_status {
+ EDMA_ST_IDLE = 0,
+ EDMA_ST_PAUSE,
+ EDMA_ST_BUSY
+};
+
+struct dw_edma_chan;
+struct dw_edma_chunk;
+
+struct dw_edma_burst {
+ struct list_head list;
+ u64 sar;
+ u64 dar;
+ u32 sz;
+};
+
+struct dw_edma_region {
+ phys_addr_t paddr;
+ void __iomem *vaddr;
+ size_t sz;
+};
+
+struct dw_edma_chunk {
+ struct list_head list;
+ struct dw_edma_chan *chan;
+ struct dw_edma_burst *burst;
+
+ u32 bursts_alloc;
+
+ u8 cb;
+ struct dw_edma_region ll_region; /* Linked list */
+};
+
+struct dw_edma_desc {
+ struct virt_dma_desc vd;
+ struct dw_edma_chan *chan;
+ struct dw_edma_chunk *chunk;
+
+ u32 chunks_alloc;
+
+ u32 alloc_sz;
+ u32 xfer_sz;
+};
+
+struct dw_edma_chan {
+ struct virt_dma_chan vc;
+ struct dw_edma_chip *chip;
+ int id;
+ enum dw_edma_dir dir;
+
+ off_t ll_off;
+ u32 ll_max;
+
+ off_t dt_off;
+
+ struct msi_msg msi;
+
+ enum dw_edma_request request;
+ enum dw_edma_status status;
+ u8 configured;
+
+ struct dma_slave_config config;
+};
+
+struct dw_edma_irq {
+ struct msi_msg msi;
+ u32 wr_mask;
+ u32 rd_mask;
+ struct dw_edma *dw;
+};
+
+struct dw_edma {
+ char name[20];
+
+ struct dma_device wr_edma;
+ u16 wr_ch_cnt;
+
+ struct dma_device rd_edma;
+ u16 rd_ch_cnt;
+
+ struct dw_edma_region rg_region; /* Registers */
+ struct dw_edma_region ll_region; /* Linked list */
+ struct dw_edma_region dt_region; /* Data */
+
+ struct dw_edma_irq *irq;
+ int nr_irqs;
+
+ u32 version;
+ enum dw_edma_mode mode;
+
+ struct dw_edma_chan *chan;
+ const struct dw_edma_core_ops *ops;
+
+ raw_spinlock_t lock; /* Only for legacy */
+};
+
+struct dw_edma_sg {
+ struct scatterlist *sgl;
+ unsigned int len;
+};
+
+struct dw_edma_cyclic {
+ dma_addr_t paddr;
+ size_t len;
+ size_t cnt;
+};
+
+struct dw_edma_transfer {
+ struct dma_chan *dchan;
+ union dw_edma_xfer {
+ struct dw_edma_sg sg;
+ struct dw_edma_cyclic cyclic;
+ } xfer;
+ enum dma_transfer_direction direction;
+ unsigned long flags;
+ bool cyclic;
+};
+
+static inline
+struct dw_edma_chan *vc2dw_edma_chan(struct virt_dma_chan *vc)
+{
+ return container_of(vc, struct dw_edma_chan, vc);
+}
+
+static inline
+struct dw_edma_chan *dchan2dw_edma_chan(struct dma_chan *dchan)
+{
+ return vc2dw_edma_chan(to_virt_chan(dchan));
+}
+
+#endif /* _DW_EDMA_CORE_H */
diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c
new file mode 100644
index 000000000000..dc85f55e1bb8
--- /dev/null
+++ b/drivers/dma/dw-edma/dw-edma-pcie.c
@@ -0,0 +1,229 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ * Synopsys DesignWare eDMA PCIe driver
+ *
+ * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/dma/edma.h>
+#include <linux/pci-epf.h>
+#include <linux/msi.h>
+
+#include "dw-edma-core.h"
+
+struct dw_edma_pcie_data {
+ /* eDMA registers location */
+ enum pci_barno rg_bar;
+ off_t rg_off;
+ size_t rg_sz;
+ /* eDMA memory linked list location */
+ enum pci_barno ll_bar;
+ off_t ll_off;
+ size_t ll_sz;
+ /* eDMA memory data location */
+ enum pci_barno dt_bar;
+ off_t dt_off;
+ size_t dt_sz;
+ /* Other */
+ u32 version;
+ enum dw_edma_mode mode;
+ u8 irqs;
+};
+
+static const struct dw_edma_pcie_data snps_edda_data = {
+ /* eDMA registers location */
+ .rg_bar = BAR_0,
+ .rg_off = 0x00001000, /* 4 Kbytes */
+ .rg_sz = 0x00002000, /* 8 Kbytes */
+ /* eDMA memory linked list location */
+ .ll_bar = BAR_2,
+ .ll_off = 0x00000000, /* 0 Kbytes */
+ .ll_sz = 0x00800000, /* 8 Mbytes */
+ /* eDMA memory data location */
+ .dt_bar = BAR_2,
+ .dt_off = 0x00800000, /* 8 Mbytes */
+ .dt_sz = 0x03800000, /* 56 Mbytes */
+ /* Other */
+ .version = 0,
+ .mode = EDMA_MODE_UNROLL,
+ .irqs = 1,
+};
+
+static int dw_edma_pcie_probe(struct pci_dev *pdev,
+ const struct pci_device_id *pid)
+{
+ const struct dw_edma_pcie_data *pdata = (void *)pid->driver_data;
+ struct device *dev = &pdev->dev;
+ struct dw_edma_chip *chip;
+ int err, nr_irqs;
+ struct dw_edma *dw;
+
+ /* Enable PCI device */
+ err = pcim_enable_device(pdev);
+ if (err) {
+ pci_err(pdev, "enabling device failed\n");
+ return err;
+ }
+
+ /* Mapping PCI BAR regions */
+ err = pcim_iomap_regions(pdev, BIT(pdata->rg_bar) |
+ BIT(pdata->ll_bar) |
+ BIT(pdata->dt_bar),
+ pci_name(pdev));
+ if (err) {
+ pci_err(pdev, "eDMA BAR I/O remapping failed\n");
+ return err;
+ }
+
+ pci_set_master(pdev);
+
+ /* DMA configuration */
+ err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
+ if (!err) {
+ err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
+ if (err) {
+ pci_err(pdev, "consistent DMA mask 64 set failed\n");
+ return err;
+ }
+ } else {
+ pci_err(pdev, "DMA mask 64 set failed\n");
+
+ err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
+ if (err) {
+ pci_err(pdev, "DMA mask 32 set failed\n");
+ return err;
+ }
+
+ err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
+ if (err) {
+ pci_err(pdev, "consistent DMA mask 32 set failed\n");
+ return err;
+ }
+ }
+
+ /* Data structure allocation */
+ chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL);
+ if (!chip)
+ return -ENOMEM;
+
+ dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL);
+ if (!dw)
+ return -ENOMEM;
+
+ /* IRQs allocation */
+ nr_irqs = pci_alloc_irq_vectors(pdev, 1, pdata->irqs,
+ PCI_IRQ_MSI | PCI_IRQ_MSIX);
+ if (nr_irqs < 1) {
+ pci_err(pdev, "fail to alloc IRQ vector (number of IRQs=%u)\n",
+ nr_irqs);
+ return -EPERM;
+ }
+
+ /* Data structure initialization */
+ chip->dw = dw;
+ chip->dev = dev;
+ chip->id = pdev->devfn;
+ chip->irq = pdev->irq;
+
+ dw->rg_region.vaddr = pcim_iomap_table(pdev)[pdata->rg_bar];
+ dw->rg_region.vaddr += pdata->rg_off;
+ dw->rg_region.paddr = pdev->resource[pdata->rg_bar].start;
+ dw->rg_region.paddr += pdata->rg_off;
+ dw->rg_region.sz = pdata->rg_sz;
+
+ dw->ll_region.vaddr = pcim_iomap_table(pdev)[pdata->ll_bar];
+ dw->ll_region.vaddr += pdata->ll_off;
+ dw->ll_region.paddr = pdev->resource[pdata->ll_bar].start;
+ dw->ll_region.paddr += pdata->ll_off;
+ dw->ll_region.sz = pdata->ll_sz;
+
+ dw->dt_region.vaddr = pcim_iomap_table(pdev)[pdata->dt_bar];
+ dw->dt_region.vaddr += pdata->dt_off;
+ dw->dt_region.paddr = pdev->resource[pdata->dt_bar].start;
+ dw->dt_region.paddr += pdata->dt_off;
+ dw->dt_region.sz = pdata->dt_sz;
+
+ dw->version = pdata->version;
+ dw->mode = pdata->mode;
+ dw->nr_irqs = nr_irqs;
+
+ /* Debug info */
+ pci_dbg(pdev, "Version:\t%u\n", dw->version);
+
+ pci_dbg(pdev, "Mode:\t%s\n",
+ dw->mode == EDMA_MODE_LEGACY ? "Legacy" : "Unroll");
+
+ pci_dbg(pdev, "Registers:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
+ pdata->rg_bar, pdata->rg_off, pdata->rg_sz,
+ dw->rg_region.vaddr, &dw->rg_region.paddr);
+
+ pci_dbg(pdev, "L. List:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
+ pdata->ll_bar, pdata->ll_off, pdata->ll_sz,
+ dw->ll_region.vaddr, &dw->ll_region.paddr);
+
+ pci_dbg(pdev, "Data:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
+ pdata->dt_bar, pdata->dt_off, pdata->dt_sz,
+ dw->dt_region.vaddr, &dw->dt_region.paddr);
+
+ pci_dbg(pdev, "Nr. IRQs:\t%u\n", dw->nr_irqs);
+
+ /* Validating if PCI interrupts were enabled */
+ if (!pci_dev_msi_enabled(pdev)) {
+ pci_err(pdev, "enable interrupt failed\n");
+ return -EPERM;
+ }
+
+ dw->irq = devm_kcalloc(dev, nr_irqs, sizeof(*dw->irq), GFP_KERNEL);
+ if (!dw->irq)
+ return -ENOMEM;
+
+ /* Starting eDMA driver */
+ err = dw_edma_probe(chip);
+ if (err) {
+ pci_err(pdev, "eDMA probe failed\n");
+ return err;
+ }
+
+ /* Saving data structure reference */
+ pci_set_drvdata(pdev, chip);
+
+ return 0;
+}
+
+static void dw_edma_pcie_remove(struct pci_dev *pdev)
+{
+ struct dw_edma_chip *chip = pci_get_drvdata(pdev);
+ int err;
+
+ /* Stopping eDMA driver */
+ err = dw_edma_remove(chip);
+ if (err)
+ pci_warn(pdev, "can't remove device properly: %d\n", err);
+
+ /* Freeing IRQs */
+ pci_free_irq_vectors(pdev);
+}
+
+static const struct pci_device_id dw_edma_pcie_id_table[] = {
+ { PCI_DEVICE_DATA(SYNOPSYS, EDDA, &snps_edda_data) },
+ { }
+};
+MODULE_DEVICE_TABLE(pci, dw_edma_pcie_id_table);
+
+static struct pci_driver dw_edma_pcie_driver = {
+ .name = "dw-edma-pcie",
+ .id_table = dw_edma_pcie_id_table,
+ .probe = dw_edma_pcie_probe,
+ .remove = dw_edma_pcie_remove,
+};
+
+module_pci_driver(dw_edma_pcie_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Synopsys DesignWare eDMA PCIe driver");
+MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>");
diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c
new file mode 100644
index 000000000000..692de47b1670
--- /dev/null
+++ b/drivers/dma/dw-edma/dw-edma-v0-core.c
@@ -0,0 +1,350 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ * Synopsys DesignWare eDMA v0 core
+ *
+ * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
+ */
+
+#include <linux/bitfield.h>
+
+#include "dw-edma-core.h"
+#include "dw-edma-v0-core.h"
+#include "dw-edma-v0-regs.h"
+#include "dw-edma-v0-debugfs.h"
+
+enum dw_edma_control {
+ DW_EDMA_V0_CB = BIT(0),
+ DW_EDMA_V0_TCB = BIT(1),
+ DW_EDMA_V0_LLP = BIT(2),
+ DW_EDMA_V0_LIE = BIT(3),
+ DW_EDMA_V0_RIE = BIT(4),
+ DW_EDMA_V0_CCS = BIT(8),
+ DW_EDMA_V0_LLE = BIT(9),
+};
+
+static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw)
+{
+ return dw->rg_region.vaddr;
+}
+
+#define SET(dw, name, value) \
+ writel(value, &(__dw_regs(dw)->name))
+
+#define GET(dw, name) \
+ readl(&(__dw_regs(dw)->name))
+
+#define SET_RW(dw, dir, name, value) \
+ do { \
+ if ((dir) == EDMA_DIR_WRITE) \
+ SET(dw, wr_##name, value); \
+ else \
+ SET(dw, rd_##name, value); \
+ } while (0)
+
+#define GET_RW(dw, dir, name) \
+ ((dir) == EDMA_DIR_WRITE \
+ ? GET(dw, wr_##name) \
+ : GET(dw, rd_##name))
+
+#define SET_BOTH(dw, name, value) \
+ do { \
+ SET(dw, wr_##name, value); \
+ SET(dw, rd_##name, value); \
+ } while (0)
+
+static inline struct dw_edma_v0_ch_regs __iomem *
+__dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)
+{
+ if (dw->mode == EDMA_MODE_LEGACY)
+ return &(__dw_regs(dw)->type.legacy.ch);
+
+ if (dir == EDMA_DIR_WRITE)
+ return &__dw_regs(dw)->type.unroll.ch[ch].wr;
+
+ return &__dw_regs(dw)->type.unroll.ch[ch].rd;
+}
+
+static inline void writel_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
+ u32 value, void __iomem *addr)
+{
+ if (dw->mode == EDMA_MODE_LEGACY) {
+ u32 viewport_sel;
+ unsigned long flags;
+
+ raw_spin_lock_irqsave(&dw->lock, flags);
+
+ viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
+ if (dir == EDMA_DIR_READ)
+ viewport_sel |= BIT(31);
+
+ writel(viewport_sel,
+ &(__dw_regs(dw)->type.legacy.viewport_sel));
+ writel(value, addr);
+
+ raw_spin_unlock_irqrestore(&dw->lock, flags);
+ } else {
+ writel(value, addr);
+ }
+}
+
+static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
+ const void __iomem *addr)
+{
+ u32 value;
+
+ if (dw->mode == EDMA_MODE_LEGACY) {
+ u32 viewport_sel;
+ unsigned long flags;
+
+ raw_spin_lock_irqsave(&dw->lock, flags);
+
+ viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
+ if (dir == EDMA_DIR_READ)
+ viewport_sel |= BIT(31);
+
+ writel(viewport_sel,
+ &(__dw_regs(dw)->type.legacy.viewport_sel));
+ value = readl(addr);
+
+ raw_spin_unlock_irqrestore(&dw->lock, flags);
+ } else {
+ value = readl(addr);
+ }
+
+ return value;
+}
+
+#define SET_CH(dw, dir, ch, name, value) \
+ writel_ch(dw, dir, ch, value, &(__dw_ch_regs(dw, dir, ch)->name))
+
+#define GET_CH(dw, dir, ch, name) \
+ readl_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name))
+
+#define SET_LL(ll, value) \
+ writel(value, ll)
+
+/* eDMA management callbacks */
+void dw_edma_v0_core_off(struct dw_edma *dw)
+{
+ SET_BOTH(dw, int_mask, EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);
+ SET_BOTH(dw, int_clear, EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);
+ SET_BOTH(dw, engine_en, 0);
+}
+
+u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir)
+{
+ u32 num_ch;
+
+ if (dir == EDMA_DIR_WRITE)
+ num_ch = FIELD_GET(EDMA_V0_WRITE_CH_COUNT_MASK, GET(dw, ctrl));
+ else
+ num_ch = FIELD_GET(EDMA_V0_READ_CH_COUNT_MASK, GET(dw, ctrl));
+
+ if (num_ch > EDMA_V0_MAX_NR_CH)
+ num_ch = EDMA_V0_MAX_NR_CH;
+
+ return (u16)num_ch;
+}
+
+enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan)
+{
+ struct dw_edma *dw = chan->chip->dw;
+ u32 tmp;
+
+ tmp = FIELD_GET(EDMA_V0_CH_STATUS_MASK,
+ GET_CH(dw, chan->dir, chan->id, ch_control1));
+
+ if (tmp == 1)
+ return DMA_IN_PROGRESS;
+ else if (tmp == 3)
+ return DMA_COMPLETE;
+ else
+ return DMA_ERROR;
+}
+
+void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan)
+{
+ struct dw_edma *dw = chan->chip->dw;
+
+ SET_RW(dw, chan->dir, int_clear,
+ FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)));
+}
+
+void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan)
+{
+ struct dw_edma *dw = chan->chip->dw;
+
+ SET_RW(dw, chan->dir, int_clear,
+ FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)));
+}
+
+u32 dw_edma_v0_core_status_done_int(struct dw_edma *dw, enum dw_edma_dir dir)
+{
+ return FIELD_GET(EDMA_V0_DONE_INT_MASK, GET_RW(dw, dir, int_status));
+}
+
+u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir dir)
+{
+ return FIELD_GET(EDMA_V0_ABORT_INT_MASK, GET_RW(dw, dir, int_status));
+}
+
+static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
+{
+ struct dw_edma_burst *child;
+ struct dw_edma_v0_lli __iomem *lli;
+ struct dw_edma_v0_llp __iomem *llp;
+ u32 control = 0, i = 0;
+ int j;
+
+ lli = chunk->ll_region.vaddr;
+
+ if (chunk->cb)
+ control = DW_EDMA_V0_CB;
+
+ j = chunk->bursts_alloc;
+ list_for_each_entry(child, &chunk->burst->list, list) {
+ j--;
+ if (!j)
+ control |= (DW_EDMA_V0_LIE | DW_EDMA_V0_RIE);
+
+ /* Channel control */
+ SET_LL(&lli[i].control, control);
+ /* Transfer size */
+ SET_LL(&lli[i].transfer_size, child->sz);
+ /* SAR - low, high */
+ SET_LL(&lli[i].sar_low, lower_32_bits(child->sar));
+ SET_LL(&lli[i].sar_high, upper_32_bits(child->sar));
+ /* DAR - low, high */
+ SET_LL(&lli[i].dar_low, lower_32_bits(child->dar));
+ SET_LL(&lli[i].dar_high, upper_32_bits(child->dar));
+ i++;
+ }
+
+ llp = (void __iomem *)&lli[i];
+ control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB;
+ if (!chunk->cb)
+ control |= DW_EDMA_V0_CB;
+
+ /* Channel control */
+ SET_LL(&llp->control, control);
+ /* Linked list - low, high */
+ SET_LL(&llp->llp_low, lower_32_bits(chunk->ll_region.paddr));
+ SET_LL(&llp->llp_high, upper_32_bits(chunk->ll_region.paddr));
+}
+
+void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
+{
+ struct dw_edma_chan *chan = chunk->chan;
+ struct dw_edma *dw = chan->chip->dw;
+ u32 tmp;
+
+ dw_edma_v0_core_write_chunk(chunk);
+
+ if (first) {
+ /* Enable engine */
+ SET_RW(dw, chan->dir, engine_en, BIT(0));
+ /* Interrupt unmask - done, abort */
+ tmp = GET_RW(dw, chan->dir, int_mask);
+ tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id));
+ tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id));
+ SET_RW(dw, chan->dir, int_mask, tmp);
+ /* Linked list error */
+ tmp = GET_RW(dw, chan->dir, linked_list_err_en);
+ tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id));
+ SET_RW(dw, chan->dir, linked_list_err_en, tmp);
+ /* Channel control */
+ SET_CH(dw, chan->dir, chan->id, ch_control1,
+ (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE));
+ /* Linked list - low, high */
+ SET_CH(dw, chan->dir, chan->id, llp_low,
+ lower_32_bits(chunk->ll_region.paddr));
+ SET_CH(dw, chan->dir, chan->id, llp_high,
+ upper_32_bits(chunk->ll_region.paddr));
+ }
+ /* Doorbell */
+ SET_RW(dw, chan->dir, doorbell,
+ FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id));
+}
+
+int dw_edma_v0_core_device_config(struct dw_edma_chan *chan)
+{
+ struct dw_edma *dw = chan->chip->dw;
+ u32 tmp = 0;
+
+ /* MSI done addr - low, high */
+ SET_RW(dw, chan->dir, done_imwr_low, chan->msi.address_lo);
+ SET_RW(dw, chan->dir, done_imwr_high, chan->msi.address_hi);
+ /* MSI abort addr - low, high */
+ SET_RW(dw, chan->dir, abort_imwr_low, chan->msi.address_lo);
+ SET_RW(dw, chan->dir, abort_imwr_high, chan->msi.address_hi);
+ /* MSI data - low, high */
+ switch (chan->id) {
+ case 0:
+ case 1:
+ tmp = GET_RW(dw, chan->dir, ch01_imwr_data);
+ break;
+
+ case 2:
+ case 3:
+ tmp = GET_RW(dw, chan->dir, ch23_imwr_data);
+ break;
+
+ case 4:
+ case 5:
+ tmp = GET_RW(dw, chan->dir, ch45_imwr_data);
+ break;
+
+ case 6:
+ case 7:
+ tmp = GET_RW(dw, chan->dir, ch67_imwr_data);
+ break;
+ }
+
+ if (chan->id & BIT(0)) {
+ /* Channel odd {1, 3, 5, 7} */
+ tmp &= EDMA_V0_CH_EVEN_MSI_DATA_MASK;
+ tmp |= FIELD_PREP(EDMA_V0_CH_ODD_MSI_DATA_MASK,
+ chan->msi.data);
+ } else {
+ /* Channel even {0, 2, 4, 6} */
+ tmp &= EDMA_V0_CH_ODD_MSI_DATA_MASK;
+ tmp |= FIELD_PREP(EDMA_V0_CH_EVEN_MSI_DATA_MASK,
+ chan->msi.data);
+ }
+
+ switch (chan->id) {
+ case 0:
+ case 1:
+ SET_RW(dw, chan->dir, ch01_imwr_data, tmp);
+ break;
+
+ case 2:
+ case 3:
+ SET_RW(dw, chan->dir, ch23_imwr_data, tmp);
+ break;
+
+ case 4:
+ case 5:
+ SET_RW(dw, chan->dir, ch45_imwr_data, tmp);
+ break;
+
+ case 6:
+ case 7:
+ SET_RW(dw, chan->dir, ch67_imwr_data, tmp);
+ break;
+ }
+
+ return 0;
+}
+
+/* eDMA debugfs callbacks */
+void dw_edma_v0_core_debugfs_on(struct dw_edma_chip *chip)
+{
+ dw_edma_v0_debugfs_on(chip);
+}
+
+void dw_edma_v0_core_debugfs_off(void)
+{
+ dw_edma_v0_debugfs_off();
+}
diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.h b/drivers/dma/dw-edma/dw-edma-v0-core.h
new file mode 100644
index 000000000000..abae1527f1f9
--- /dev/null
+++ b/drivers/dma/dw-edma/dw-edma-v0-core.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ * Synopsys DesignWare eDMA v0 core
+ *
+ * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
+ */
+
+#ifndef _DW_EDMA_V0_CORE_H
+#define _DW_EDMA_V0_CORE_H
+
+#include <linux/dma/edma.h>
+
+/* eDMA management callbacks */
+void dw_edma_v0_core_off(struct dw_edma *chan);
+u16 dw_edma_v0_core_ch_count(struct dw_edma *chan, enum dw_edma_dir dir);
+enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan);
+void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan);
+void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan);
+u32 dw_edma_v0_core_status_done_int(struct dw_edma *chan, enum dw_edma_dir dir);
+u32 dw_edma_v0_core_status_abort_int(struct dw_edma *chan, enum dw_edma_dir dir);
+void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first);
+int dw_edma_v0_core_device_config(struct dw_edma_chan *chan);
+/* eDMA debug fs callbacks */
+void dw_edma_v0_core_debugfs_on(struct dw_edma_chip *chip);
+void dw_edma_v0_core_debugfs_off(void);
+
+#endif /* _DW_EDMA_V0_CORE_H */
diff --git a/drivers/dma/dw-edma/dw-edma-v0-debugfs.c b/drivers/dma/dw-edma/dw-edma-v0-debugfs.c
new file mode 100644
index 000000000000..42739508c0d8
--- /dev/null
+++ b/drivers/dma/dw-edma/dw-edma-v0-debugfs.c
@@ -0,0 +1,311 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ * Synopsys DesignWare eDMA v0 core
+ *
+ * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
+ */
+
+#include <linux/debugfs.h>
+#include <linux/bitfield.h>
+
+#include "dw-edma-v0-debugfs.h"
+#include "dw-edma-v0-regs.h"
+#include "dw-edma-core.h"
+
+#define REGS_ADDR(name) \
+ ((void __force *)&regs->name)
+#define REGISTER(name) \
+ { #name, REGS_ADDR(name) }
+
+#define WR_REGISTER(name) \
+ { #name, REGS_ADDR(wr_##name) }
+#define RD_REGISTER(name) \
+ { #name, REGS_ADDR(rd_##name) }
+
+#define WR_REGISTER_LEGACY(name) \
+ { #name, REGS_ADDR(type.legacy.wr_##name) }
+#define RD_REGISTER_LEGACY(name) \
+ { #name, REGS_ADDR(type.legacy.rd_##name) }
+
+#define WR_REGISTER_UNROLL(name) \
+ { #name, REGS_ADDR(type.unroll.wr_##name) }
+#define RD_REGISTER_UNROLL(name) \
+ { #name, REGS_ADDR(type.unroll.rd_##name) }
+
+#define WRITE_STR "write"
+#define READ_STR "read"
+#define CHANNEL_STR "channel"
+#define REGISTERS_STR "registers"
+
+static struct dentry *base_dir;
+static struct dw_edma *dw;
+static struct dw_edma_v0_regs __iomem *regs;
+
+static struct {
+ void __iomem *start;
+ void __iomem *end;
+} lim[2][EDMA_V0_MAX_NR_CH];
+
+struct debugfs_entries {
+ const char *name;
+ dma_addr_t *reg;
+};
+
+static int dw_edma_debugfs_u32_get(void *data, u64 *val)
+{
+ void __iomem *reg = (void __force __iomem *)data;
+ if (dw->mode == EDMA_MODE_LEGACY &&
+ reg >= (void __iomem *)&regs->type.legacy.ch) {
+ void __iomem *ptr = &regs->type.legacy.ch;
+ u32 viewport_sel = 0;
+ unsigned long flags;
+ u16 ch;
+
+ for (ch = 0; ch < dw->wr_ch_cnt; ch++)
+ if (lim[0][ch].start >= reg && reg < lim[0][ch].end) {
+ ptr += (reg - lim[0][ch].start);
+ goto legacy_sel_wr;
+ }
+
+ for (ch = 0; ch < dw->rd_ch_cnt; ch++)
+ if (lim[1][ch].start >= reg && reg < lim[1][ch].end) {
+ ptr += (reg - lim[1][ch].start);
+ goto legacy_sel_rd;
+ }
+
+ return 0;
+legacy_sel_rd:
+ viewport_sel = BIT(31);
+legacy_sel_wr:
+ viewport_sel |= FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
+
+ raw_spin_lock_irqsave(&dw->lock, flags);
+
+ writel(viewport_sel, &regs->type.legacy.viewport_sel);
+ *val = readl(ptr);
+
+ raw_spin_unlock_irqrestore(&dw->lock, flags);
+ } else {
+ *val = readl(reg);
+ }
+
+ return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(fops_x32, dw_edma_debugfs_u32_get, NULL, "0x%08llx\n");
+
+static void dw_edma_debugfs_create_x32(const struct debugfs_entries entries[],
+ int nr_entries, struct dentry *dir)
+{
+ int i;
+
+ for (i = 0; i < nr_entries; i++) {
+ if (!debugfs_create_file_unsafe(entries[i].name, 0444, dir,
+ entries[i].reg, &fops_x32))
+ break;
+ }
+}
+
+static void dw_edma_debugfs_regs_ch(struct dw_edma_v0_ch_regs __iomem *regs,
+ struct dentry *dir)
+{
+ int nr_entries;
+ const struct debugfs_entries debugfs_regs[] = {
+ REGISTER(ch_control1),
+ REGISTER(ch_control2),
+ REGISTER(transfer_size),
+ REGISTER(sar_low),
+ REGISTER(sar_high),
+ REGISTER(dar_low),
+ REGISTER(dar_high),
+ REGISTER(llp_low),
+ REGISTER(llp_high),
+ };
+
+ nr_entries = ARRAY_SIZE(debugfs_regs);
+ dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, dir);
+}
+
+static void dw_edma_debugfs_regs_wr(struct dentry *dir)
+{
+ const struct debugfs_entries debugfs_regs[] = {
+ /* eDMA global registers */
+ WR_REGISTER(engine_en),
+ WR_REGISTER(doorbell),
+ WR_REGISTER(ch_arb_weight_low),
+ WR_REGISTER(ch_arb_weight_high),
+ /* eDMA interrupts registers */
+ WR_REGISTER(int_status),
+ WR_REGISTER(int_mask),
+ WR_REGISTER(int_clear),
+ WR_REGISTER(err_status),
+ WR_REGISTER(done_imwr_low),
+ WR_REGISTER(done_imwr_high),
+ WR_REGISTER(abort_imwr_low),
+ WR_REGISTER(abort_imwr_high),
+ WR_REGISTER(ch01_imwr_data),
+ WR_REGISTER(ch23_imwr_data),
+ WR_REGISTER(ch45_imwr_data),
+ WR_REGISTER(ch67_imwr_data),
+ WR_REGISTER(linked_list_err_en),
+ };
+ const struct debugfs_entries debugfs_unroll_regs[] = {
+ /* eDMA channel context grouping */
+ WR_REGISTER_UNROLL(engine_chgroup),
+ WR_REGISTER_UNROLL(engine_hshake_cnt_low),
+ WR_REGISTER_UNROLL(engine_hshake_cnt_high),
+ WR_REGISTER_UNROLL(ch0_pwr_en),
+ WR_REGISTER_UNROLL(ch1_pwr_en),
+ WR_REGISTER_UNROLL(ch2_pwr_en),
+ WR_REGISTER_UNROLL(ch3_pwr_en),
+ WR_REGISTER_UNROLL(ch4_pwr_en),
+ WR_REGISTER_UNROLL(ch5_pwr_en),
+ WR_REGISTER_UNROLL(ch6_pwr_en),
+ WR_REGISTER_UNROLL(ch7_pwr_en),
+ };
+ struct dentry *regs_dir, *ch_dir;
+ int nr_entries, i;
+ char name[16];
+
+ regs_dir = debugfs_create_dir(WRITE_STR, dir);
+ if (!regs_dir)
+ return;
+
+ nr_entries = ARRAY_SIZE(debugfs_regs);
+ dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, regs_dir);
+
+ if (dw->mode == EDMA_MODE_UNROLL) {
+ nr_entries = ARRAY_SIZE(debugfs_unroll_regs);
+ dw_edma_debugfs_create_x32(debugfs_unroll_regs, nr_entries,
+ regs_dir);
+ }
+
+ for (i = 0; i < dw->wr_ch_cnt; i++) {
+ snprintf(name, sizeof(name), "%s:%d", CHANNEL_STR, i);
+
+ ch_dir = debugfs_create_dir(name, regs_dir);
+ if (!ch_dir)
+ return;
+
+ dw_edma_debugfs_regs_ch(&regs->type.unroll.ch[i].wr, ch_dir);
+
+ lim[0][i].start = &regs->type.unroll.ch[i].wr;
+ lim[0][i].end = &regs->type.unroll.ch[i].padding_1[0];
+ }
+}
+
+static void dw_edma_debugfs_regs_rd(struct dentry *dir)
+{
+ const struct debugfs_entries debugfs_regs[] = {
+ /* eDMA global registers */
+ RD_REGISTER(engine_en),
+ RD_REGISTER(doorbell),
+ RD_REGISTER(ch_arb_weight_low),
+ RD_REGISTER(ch_arb_weight_high),
+ /* eDMA interrupts registers */
+ RD_REGISTER(int_status),
+ RD_REGISTER(int_mask),
+ RD_REGISTER(int_clear),
+ RD_REGISTER(err_status_low),
+ RD_REGISTER(err_status_high),
+ RD_REGISTER(linked_list_err_en),
+ RD_REGISTER(done_imwr_low),
+ RD_REGISTER(done_imwr_high),
+ RD_REGISTER(abort_imwr_low),
+ RD_REGISTER(abort_imwr_high),
+ RD_REGISTER(ch01_imwr_data),
+ RD_REGISTER(ch23_imwr_data),
+ RD_REGISTER(ch45_imwr_data),
+ RD_REGISTER(ch67_imwr_data),
+ };
+ const struct debugfs_entries debugfs_unroll_regs[] = {
+ /* eDMA channel context grouping */
+ RD_REGISTER_UNROLL(engine_chgroup),
+ RD_REGISTER_UNROLL(engine_hshake_cnt_low),
+ RD_REGISTER_UNROLL(engine_hshake_cnt_high),
+ RD_REGISTER_UNROLL(ch0_pwr_en),
+ RD_REGISTER_UNROLL(ch1_pwr_en),
+ RD_REGISTER_UNROLL(ch2_pwr_en),
+ RD_REGISTER_UNROLL(ch3_pwr_en),
+ RD_REGISTER_UNROLL(ch4_pwr_en),
+ RD_REGISTER_UNROLL(ch5_pwr_en),
+ RD_REGISTER_UNROLL(ch6_pwr_en),
+ RD_REGISTER_UNROLL(ch7_pwr_en),
+ };
+ struct dentry *regs_dir, *ch_dir;
+ int nr_entries, i;
+ char name[16];
+
+ regs_dir = debugfs_create_dir(READ_STR, dir);
+ if (!regs_dir)
+ return;
+
+ nr_entries = ARRAY_SIZE(debugfs_regs);
+ dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, regs_dir);
+
+ if (dw->mode == EDMA_MODE_UNROLL) {
+ nr_entries = ARRAY_SIZE(debugfs_unroll_regs);
+ dw_edma_debugfs_create_x32(debugfs_unroll_regs, nr_entries,
+ regs_dir);
+ }
+
+ for (i = 0; i < dw->rd_ch_cnt; i++) {
+ snprintf(name, sizeof(name), "%s:%d", CHANNEL_STR, i);
+
+ ch_dir = debugfs_create_dir(name, regs_dir);
+ if (!ch_dir)
+ return;
+
+ dw_edma_debugfs_regs_ch(&regs->type.unroll.ch[i].rd, ch_dir);
+
+ lim[1][i].start = &regs->type.unroll.ch[i].rd;
+ lim[1][i].end = &regs->type.unroll.ch[i].padding_2[0];
+ }
+}
+
+static void dw_edma_debugfs_regs(void)
+{
+ const struct debugfs_entries debugfs_regs[] = {
+ REGISTER(ctrl_data_arb_prior),
+ REGISTER(ctrl),
+ };
+ struct dentry *regs_dir;
+ int nr_entries;
+
+ regs_dir = debugfs_create_dir(REGISTERS_STR, base_dir);
+ if (!regs_dir)
+ return;
+
+ nr_entries = ARRAY_SIZE(debugfs_regs);
+ dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, regs_dir);
+
+ dw_edma_debugfs_regs_wr(regs_dir);
+ dw_edma_debugfs_regs_rd(regs_dir);
+}
+
+void dw_edma_v0_debugfs_on(struct dw_edma_chip *chip)
+{
+ dw = chip->dw;
+ if (!dw)
+ return;
+
+ regs = dw->rg_region.vaddr;
+ if (!regs)
+ return;
+
+ base_dir = debugfs_create_dir(dw->name, 0);
+ if (!base_dir)
+ return;
+
+ debugfs_create_u32("version", 0444, base_dir, &dw->version);
+ debugfs_create_u32("mode", 0444, base_dir, &dw->mode);
+ debugfs_create_u16("wr_ch_cnt", 0444, base_dir, &dw->wr_ch_cnt);
+ debugfs_create_u16("rd_ch_cnt", 0444, base_dir, &dw->rd_ch_cnt);
+
+ dw_edma_debugfs_regs();
+}
+
+void dw_edma_v0_debugfs_off(void)
+{
+ debugfs_remove_recursive(base_dir);
+}
diff --git a/drivers/dma/dw-edma/dw-edma-v0-debugfs.h b/drivers/dma/dw-edma/dw-edma-v0-debugfs.h
new file mode 100644
index 000000000000..5450a0a94193
--- /dev/null
+++ b/drivers/dma/dw-edma/dw-edma-v0-debugfs.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ * Synopsys DesignWare eDMA v0 core
+ *
+ * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
+ */
+
+#ifndef _DW_EDMA_V0_DEBUG_FS_H
+#define _DW_EDMA_V0_DEBUG_FS_H
+
+#include <linux/dma/edma.h>
+
+#ifdef CONFIG_DEBUG_FS
+void dw_edma_v0_debugfs_on(struct dw_edma_chip *chip);
+void dw_edma_v0_debugfs_off(void);
+#else
+static inline void dw_edma_v0_debugfs_on(struct dw_edma_chip *chip)
+{
+}
+
+static inline void dw_edma_v0_debugfs_off(void)
+{
+}
+#endif /* CONFIG_DEBUG_FS */
+
+#endif /* _DW_EDMA_V0_DEBUG_FS_H */
diff --git a/drivers/dma/dw-edma/dw-edma-v0-regs.h b/drivers/dma/dw-edma/dw-edma-v0-regs.h
new file mode 100644
index 000000000000..cd6476884507
--- /dev/null
+++ b/drivers/dma/dw-edma/dw-edma-v0-regs.h
@@ -0,0 +1,158 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
+ * Synopsys DesignWare eDMA v0 core
+ *
+ * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
+ */
+
+#ifndef _DW_EDMA_V0_REGS_H
+#define _DW_EDMA_V0_REGS_H
+
+#include <linux/dmaengine.h>
+
+#define EDMA_V0_MAX_NR_CH 8
+#define EDMA_V0_VIEWPORT_MASK GENMASK(2, 0)
+#define EDMA_V0_DONE_INT_MASK GENMASK(7, 0)
+#define EDMA_V0_ABORT_INT_MASK GENMASK(23, 16)
+#define EDMA_V0_WRITE_CH_COUNT_MASK GENMASK(3, 0)
+#define EDMA_V0_READ_CH_COUNT_MASK GENMASK(19, 16)
+#define EDMA_V0_CH_STATUS_MASK GENMASK(6, 5)
+#define EDMA_V0_DOORBELL_CH_MASK GENMASK(2, 0)
+#define EDMA_V0_LINKED_LIST_ERR_MASK GENMASK(7, 0)
+
+#define EDMA_V0_CH_ODD_MSI_DATA_MASK GENMASK(31, 16)
+#define EDMA_V0_CH_EVEN_MSI_DATA_MASK GENMASK(15, 0)
+
+struct dw_edma_v0_ch_regs {
+ u32 ch_control1; /* 0x000 */
+ u32 ch_control2; /* 0x004 */
+ u32 transfer_size; /* 0x008 */
+ u32 sar_low; /* 0x00c */
+ u32 sar_high; /* 0x010 */
+ u32 dar_low; /* 0x014 */
+ u32 dar_high; /* 0x018 */
+ u32 llp_low; /* 0x01c */
+ u32 llp_high; /* 0x020 */
+};
+
+struct dw_edma_v0_ch {
+ struct dw_edma_v0_ch_regs wr; /* 0x200 */
+ u32 padding_1[55]; /* [0x224..0x2fc] */
+ struct dw_edma_v0_ch_regs rd; /* 0x300 */
+ u32 padding_2[55]; /* [0x224..0x2fc] */
+};
+
+struct dw_edma_v0_unroll {
+ u32 padding_1; /* 0x0f8 */
+ u32 wr_engine_chgroup; /* 0x100 */
+ u32 rd_engine_chgroup; /* 0x104 */
+ u32 wr_engine_hshake_cnt_low; /* 0x108 */
+ u32 wr_engine_hshake_cnt_high; /* 0x10c */
+ u32 padding_2[2]; /* [0x110..0x114] */
+ u32 rd_engine_hshake_cnt_low; /* 0x118 */
+ u32 rd_engine_hshake_cnt_high; /* 0x11c */
+ u32 padding_3[2]; /* [0x120..0x124] */
+ u32 wr_ch0_pwr_en; /* 0x128 */
+ u32 wr_ch1_pwr_en; /* 0x12c */
+ u32 wr_ch2_pwr_en; /* 0x130 */
+ u32 wr_ch3_pwr_en; /* 0x134 */
+ u32 wr_ch4_pwr_en; /* 0x138 */
+ u32 wr_ch5_pwr_en; /* 0x13c */
+ u32 wr_ch6_pwr_en; /* 0x140 */
+ u32 wr_ch7_pwr_en; /* 0x144 */
+ u32 padding_4[8]; /* [0x148..0x164] */
+ u32 rd_ch0_pwr_en; /* 0x168 */
+ u32 rd_ch1_pwr_en; /* 0x16c */
+ u32 rd_ch2_pwr_en; /* 0x170 */
+ u32 rd_ch3_pwr_en; /* 0x174 */
+ u32 rd_ch4_pwr_en; /* 0x178 */
+ u32 rd_ch5_pwr_en; /* 0x18c */
+ u32 rd_ch6_pwr_en; /* 0x180 */
+ u32 rd_ch7_pwr_en; /* 0x184 */
+ u32 padding_5[30]; /* [0x188..0x1fc] */
+ struct dw_edma_v0_ch ch[EDMA_V0_MAX_NR_CH]; /* [0x200..0x1120] */
+};
+
+struct dw_edma_v0_legacy {
+ u32 viewport_sel; /* 0x0f8 */
+ struct dw_edma_v0_ch_regs ch; /* [0x100..0x120] */
+};
+
+struct dw_edma_v0_regs {
+ /* eDMA global registers */
+ u32 ctrl_data_arb_prior; /* 0x000 */
+ u32 padding_1; /* 0x004 */
+ u32 ctrl; /* 0x008 */
+ u32 wr_engine_en; /* 0x00c */
+ u32 wr_doorbell; /* 0x010 */
+ u32 padding_2; /* 0x014 */
+ u32 wr_ch_arb_weight_low; /* 0x018 */
+ u32 wr_ch_arb_weight_high; /* 0x01c */
+ u32 padding_3[3]; /* [0x020..0x028] */
+ u32 rd_engine_en; /* 0x02c */
+ u32 rd_doorbell; /* 0x030 */
+ u32 padding_4; /* 0x034 */
+ u32 rd_ch_arb_weight_low; /* 0x038 */
+ u32 rd_ch_arb_weight_high; /* 0x03c */
+ u32 padding_5[3]; /* [0x040..0x048] */
+ /* eDMA interrupts registers */
+ u32 wr_int_status; /* 0x04c */
+ u32 padding_6; /* 0x050 */
+ u32 wr_int_mask; /* 0x054 */
+ u32 wr_int_clear; /* 0x058 */
+ u32 wr_err_status; /* 0x05c */
+ u32 wr_done_imwr_low; /* 0x060 */
+ u32 wr_done_imwr_high; /* 0x064 */
+ u32 wr_abort_imwr_low; /* 0x068 */
+ u32 wr_abort_imwr_high; /* 0x06c */
+ u32 wr_ch01_imwr_data; /* 0x070 */
+ u32 wr_ch23_imwr_data; /* 0x074 */
+ u32 wr_ch45_imwr_data; /* 0x078 */
+ u32 wr_ch67_imwr_data; /* 0x07c */
+ u32 padding_7[4]; /* [0x080..0x08c] */
+ u32 wr_linked_list_err_en; /* 0x090 */
+ u32 padding_8[3]; /* [0x094..0x09c] */
+ u32 rd_int_status; /* 0x0a0 */
+ u32 padding_9; /* 0x0a4 */
+ u32 rd_int_mask; /* 0x0a8 */
+ u32 rd_int_clear; /* 0x0ac */
+ u32 padding_10; /* 0x0b0 */
+ u32 rd_err_status_low; /* 0x0b4 */
+ u32 rd_err_status_high; /* 0x0b8 */
+ u32 padding_11[2]; /* [0x0bc..0x0c0] */
+ u32 rd_linked_list_err_en; /* 0x0c4 */
+ u32 padding_12; /* 0x0c8 */
+ u32 rd_done_imwr_low; /* 0x0cc */
+ u32 rd_done_imwr_high; /* 0x0d0 */
+ u32 rd_abort_imwr_low; /* 0x0d4 */
+ u32 rd_abort_imwr_high; /* 0x0d8 */
+ u32 rd_ch01_imwr_data; /* 0x0dc */
+ u32 rd_ch23_imwr_data; /* 0x0e0 */
+ u32 rd_ch45_imwr_data; /* 0x0e4 */
+ u32 rd_ch67_imwr_data; /* 0x0e8 */
+ u32 padding_13[4]; /* [0x0ec..0x0f8] */
+ /* eDMA channel context grouping */
+ union dw_edma_v0_type {
+ struct dw_edma_v0_legacy legacy; /* [0x0f8..0x120] */
+ struct dw_edma_v0_unroll unroll; /* [0x0f8..0x1120] */
+ } type;
+};
+
+struct dw_edma_v0_lli {
+ u32 control;
+ u32 transfer_size;
+ u32 sar_low;
+ u32 sar_high;
+ u32 dar_low;
+ u32 dar_high;
+};
+
+struct dw_edma_v0_llp {
+ u32 control;
+ u32 reserved;
+ u32 llp_low;
+ u32 llp_high;
+};
+
+#endif /* _DW_EDMA_V0_REGS_H */
diff --git a/drivers/dma/dw/Makefile b/drivers/dma/dw/Makefile
index 63ed895c09aa..b6f06699e91a 100644
--- a/drivers/dma/dw/Makefile
+++ b/drivers/dma/dw/Makefile
@@ -3,7 +3,9 @@ obj-$(CONFIG_DW_DMAC_CORE) += dw_dmac_core.o
dw_dmac_core-objs := core.o dw.o idma32.o
obj-$(CONFIG_DW_DMAC) += dw_dmac.o
-dw_dmac-objs := platform.o
+dw_dmac-y := platform.o
+dw_dmac-$(CONFIG_ACPI) += acpi.o
+dw_dmac-$(CONFIG_OF) += of.o
obj-$(CONFIG_DW_DMAC_PCI) += dw_dmac_pci.o
dw_dmac_pci-objs := pci.o
diff --git a/drivers/dma/dw/acpi.c b/drivers/dma/dw/acpi.c
new file mode 100644
index 000000000000..f6e8d55b4f6e
--- /dev/null
+++ b/drivers/dma/dw/acpi.c
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2013,2019 Intel Corporation
+
+#include <linux/acpi.h>
+#include <linux/acpi_dma.h>
+
+#include "internal.h"
+
+static bool dw_dma_acpi_filter(struct dma_chan *chan, void *param)
+{
+ struct acpi_dma_spec *dma_spec = param;
+ struct dw_dma_slave slave = {
+ .dma_dev = dma_spec->dev,
+ .src_id = dma_spec->slave_id,
+ .dst_id = dma_spec->slave_id,
+ .m_master = 0,
+ .p_master = 1,
+ };
+
+ return dw_dma_filter(chan, &slave);
+}
+
+void dw_dma_acpi_controller_register(struct dw_dma *dw)
+{
+ struct device *dev = dw->dma.dev;
+ struct acpi_dma_filter_info *info;
+ int ret;
+
+ if (!has_acpi_companion(dev))
+ return;
+
+ info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL);
+ if (!info)
+ return;
+
+ dma_cap_zero(info->dma_cap);
+ dma_cap_set(DMA_SLAVE, info->dma_cap);
+ info->filter_fn = dw_dma_acpi_filter;
+
+ ret = acpi_dma_controller_register(dev, acpi_dma_simple_xlate, info);
+ if (ret)
+ dev_err(dev, "could not register acpi_dma_controller\n");
+}
+
+void dw_dma_acpi_controller_free(struct dw_dma *dw)
+{
+ struct device *dev = dw->dma.dev;
+
+ if (!has_acpi_companion(dev))
+ return;
+
+ acpi_dma_controller_free(dev);
+}
diff --git a/drivers/dma/dw/internal.h b/drivers/dma/dw/internal.h
index 1dd7a4e6dd23..2e1c52eefdeb 100644
--- a/drivers/dma/dw/internal.h
+++ b/drivers/dma/dw/internal.h
@@ -23,4 +23,55 @@ int do_dw_dma_enable(struct dw_dma_chip *chip);
extern bool dw_dma_filter(struct dma_chan *chan, void *param);
+#ifdef CONFIG_ACPI
+void dw_dma_acpi_controller_register(struct dw_dma *dw);
+void dw_dma_acpi_controller_free(struct dw_dma *dw);
+#else /* !CONFIG_ACPI */
+static inline void dw_dma_acpi_controller_register(struct dw_dma *dw) {}
+static inline void dw_dma_acpi_controller_free(struct dw_dma *dw) {}
+#endif /* !CONFIG_ACPI */
+
+struct platform_device;
+
+#ifdef CONFIG_OF
+struct dw_dma_platform_data *dw_dma_parse_dt(struct platform_device *pdev);
+void dw_dma_of_controller_register(struct dw_dma *dw);
+void dw_dma_of_controller_free(struct dw_dma *dw);
+#else
+static inline struct dw_dma_platform_data *dw_dma_parse_dt(struct platform_device *pdev)
+{
+ return NULL;
+}
+static inline void dw_dma_of_controller_register(struct dw_dma *dw) {}
+static inline void dw_dma_of_controller_free(struct dw_dma *dw) {}
+#endif
+
+struct dw_dma_chip_pdata {
+ const struct dw_dma_platform_data *pdata;
+ int (*probe)(struct dw_dma_chip *chip);
+ int (*remove)(struct dw_dma_chip *chip);
+ struct dw_dma_chip *chip;
+};
+
+static __maybe_unused const struct dw_dma_chip_pdata dw_dma_chip_pdata = {
+ .probe = dw_dma_probe,
+ .remove = dw_dma_remove,
+};
+
+static const struct dw_dma_platform_data idma32_pdata = {
+ .nr_channels = 8,
+ .chan_allocation_order = CHAN_ALLOCATION_ASCENDING,
+ .chan_priority = CHAN_PRIORITY_ASCENDING,
+ .block_size = 131071,
+ .nr_masters = 1,
+ .data_width = {4},
+ .multi_block = {1, 1, 1, 1, 1, 1, 1, 1},
+};
+
+static __maybe_unused const struct dw_dma_chip_pdata idma32_chip_pdata = {
+ .pdata = &idma32_pdata,
+ .probe = idma32_dma_probe,
+ .remove = idma32_dma_remove,
+};
+
#endif /* _DMA_DW_INTERNAL_H */
diff --git a/drivers/dma/dw/of.c b/drivers/dma/dw/of.c
new file mode 100644
index 000000000000..9e27831dee32
--- /dev/null
+++ b/drivers/dma/dw/of.c
@@ -0,0 +1,131 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Platform driver for the Synopsys DesignWare DMA Controller
+ *
+ * Copyright (C) 2007-2008 Atmel Corporation
+ * Copyright (C) 2010-2011 ST Microelectronics
+ * Copyright (C) 2013 Intel Corporation
+ */
+
+#include <linux/of.h>
+#include <linux/of_dma.h>
+#include <linux/platform_device.h>
+
+#include "internal.h"
+
+static struct dma_chan *dw_dma_of_xlate(struct of_phandle_args *dma_spec,
+ struct of_dma *ofdma)
+{
+ struct dw_dma *dw = ofdma->of_dma_data;
+ struct dw_dma_slave slave = {
+ .dma_dev = dw->dma.dev,
+ };
+ dma_cap_mask_t cap;
+
+ if (dma_spec->args_count != 3)
+ return NULL;
+
+ slave.src_id = dma_spec->args[0];
+ slave.dst_id = dma_spec->args[0];
+ slave.m_master = dma_spec->args[1];
+ slave.p_master = dma_spec->args[2];
+
+ if (WARN_ON(slave.src_id >= DW_DMA_MAX_NR_REQUESTS ||
+ slave.dst_id >= DW_DMA_MAX_NR_REQUESTS ||
+ slave.m_master >= dw->pdata->nr_masters ||
+ slave.p_master >= dw->pdata->nr_masters))
+ return NULL;
+
+ dma_cap_zero(cap);
+ dma_cap_set(DMA_SLAVE, cap);
+
+ /* TODO: there should be a simpler way to do this */
+ return dma_request_channel(cap, dw_dma_filter, &slave);
+}
+
+struct dw_dma_platform_data *dw_dma_parse_dt(struct platform_device *pdev)
+{
+ struct device_node *np = pdev->dev.of_node;
+ struct dw_dma_platform_data *pdata;
+ u32 tmp, arr[DW_DMA_MAX_NR_MASTERS], mb[DW_DMA_MAX_NR_CHANNELS];
+ u32 nr_masters;
+ u32 nr_channels;
+
+ if (!np) {
+ dev_err(&pdev->dev, "Missing DT data\n");
+ return NULL;
+ }
+
+ if (of_property_read_u32(np, "dma-masters", &nr_masters))
+ return NULL;
+ if (nr_masters < 1 || nr_masters > DW_DMA_MAX_NR_MASTERS)
+ return NULL;
+
+ if (of_property_read_u32(np, "dma-channels", &nr_channels))
+ return NULL;
+ if (nr_channels > DW_DMA_MAX_NR_CHANNELS)
+ return NULL;
+
+ pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
+ if (!pdata)
+ return NULL;
+
+ pdata->nr_masters = nr_masters;
+ pdata->nr_channels = nr_channels;
+
+ if (!of_property_read_u32(np, "chan_allocation_order", &tmp))
+ pdata->chan_allocation_order = (unsigned char)tmp;
+
+ if (!of_property_read_u32(np, "chan_priority", &tmp))
+ pdata->chan_priority = tmp;
+
+ if (!of_property_read_u32(np, "block_size", &tmp))
+ pdata->block_size = tmp;
+
+ if (!of_property_read_u32_array(np, "data-width", arr, nr_masters)) {
+ for (tmp = 0; tmp < nr_masters; tmp++)
+ pdata->data_width[tmp] = arr[tmp];
+ } else if (!of_property_read_u32_array(np, "data_width", arr, nr_masters)) {
+ for (tmp = 0; tmp < nr_masters; tmp++)
+ pdata->data_width[tmp] = BIT(arr[tmp] & 0x07);
+ }
+
+ if (!of_property_read_u32_array(np, "multi-block", mb, nr_channels)) {
+ for (tmp = 0; tmp < nr_channels; tmp++)
+ pdata->multi_block[tmp] = mb[tmp];
+ } else {
+ for (tmp = 0; tmp < nr_channels; tmp++)
+ pdata->multi_block[tmp] = 1;
+ }
+
+ if (!of_property_read_u32(np, "snps,dma-protection-control", &tmp)) {
+ if (tmp > CHAN_PROTCTL_MASK)
+ return NULL;
+ pdata->protctl = tmp;
+ }
+
+ return pdata;
+}
+
+void dw_dma_of_controller_register(struct dw_dma *dw)
+{
+ struct device *dev = dw->dma.dev;
+ int ret;
+
+ if (!dev->of_node)
+ return;
+
+ ret = of_dma_controller_register(dev->of_node, dw_dma_of_xlate, dw);
+ if (ret)
+ dev_err(dev, "could not register of_dma_controller\n");
+}
+
+void dw_dma_of_controller_free(struct dw_dma *dw)
+{
+ struct device *dev = dw->dma.dev;
+
+ if (!dev->of_node)
+ return;
+
+ of_dma_controller_free(dev->of_node);
+}
diff --git a/drivers/dma/dw/pci.c b/drivers/dma/dw/pci.c
index e79a75db0852..cf6e8ec4c0ff 100644
--- a/drivers/dma/dw/pci.c
+++ b/drivers/dma/dw/pci.c
@@ -12,33 +12,10 @@
#include "internal.h"
-struct dw_dma_pci_data {
- const struct dw_dma_platform_data *pdata;
- int (*probe)(struct dw_dma_chip *chip);
-};
-
-static const struct dw_dma_pci_data dw_pci_data = {
- .probe = dw_dma_probe,
-};
-
-static const struct dw_dma_platform_data idma32_pdata = {
- .nr_channels = 8,
- .chan_allocation_order = CHAN_ALLOCATION_ASCENDING,
- .chan_priority = CHAN_PRIORITY_ASCENDING,
- .block_size = 131071,
- .nr_masters = 1,
- .data_width = {4},
- .multi_block = {1, 1, 1, 1, 1, 1, 1, 1},
-};
-
-static const struct dw_dma_pci_data idma32_pci_data = {
- .pdata = &idma32_pdata,
- .probe = idma32_dma_probe,
-};
-
static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid)
{
- const struct dw_dma_pci_data *data = (void *)pid->driver_data;
+ const struct dw_dma_chip_pdata *drv_data = (void *)pid->driver_data;
+ struct dw_dma_chip_pdata *data;
struct dw_dma_chip *chip;
int ret;
@@ -63,6 +40,10 @@ static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid)
if (ret)
return ret;
+ data = devm_kmemdup(&pdev->dev, drv_data, sizeof(*drv_data), GFP_KERNEL);
+ if (!data)
+ return -ENOMEM;
+
chip = devm_kzalloc(&pdev->dev, sizeof(*chip), GFP_KERNEL);
if (!chip)
return -ENOMEM;
@@ -73,21 +54,24 @@ static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid)
chip->irq = pdev->irq;
chip->pdata = data->pdata;
+ data->chip = chip;
+
ret = data->probe(chip);
if (ret)
return ret;
- pci_set_drvdata(pdev, chip);
+ pci_set_drvdata(pdev, data);
return 0;
}
static void dw_pci_remove(struct pci_dev *pdev)
{
- struct dw_dma_chip *chip = pci_get_drvdata(pdev);
+ struct dw_dma_chip_pdata *data = pci_get_drvdata(pdev);
+ struct dw_dma_chip *chip = data->chip;
int ret;
- ret = dw_dma_remove(chip);
+ ret = data->remove(chip);
if (ret)
dev_warn(&pdev->dev, "can't remove device properly: %d\n", ret);
}
@@ -96,16 +80,16 @@ static void dw_pci_remove(struct pci_dev *pdev)
static int dw_pci_suspend_late(struct device *dev)
{
- struct pci_dev *pci = to_pci_dev(dev);
- struct dw_dma_chip *chip = pci_get_drvdata(pci);
+ struct dw_dma_chip_pdata *data = dev_get_drvdata(dev);
+ struct dw_dma_chip *chip = data->chip;
return do_dw_dma_disable(chip);
};
static int dw_pci_resume_early(struct device *dev)
{
- struct pci_dev *pci = to_pci_dev(dev);
- struct dw_dma_chip *chip = pci_get_drvdata(pci);
+ struct dw_dma_chip_pdata *data = dev_get_drvdata(dev);
+ struct dw_dma_chip *chip = data->chip;
return do_dw_dma_enable(chip);
};
@@ -118,24 +102,29 @@ static const struct dev_pm_ops dw_pci_dev_pm_ops = {
static const struct pci_device_id dw_pci_id_table[] = {
/* Medfield (GPDMA) */
- { PCI_VDEVICE(INTEL, 0x0827), (kernel_ulong_t)&dw_pci_data },
+ { PCI_VDEVICE(INTEL, 0x0827), (kernel_ulong_t)&dw_dma_chip_pdata },
/* BayTrail */
- { PCI_VDEVICE(INTEL, 0x0f06), (kernel_ulong_t)&dw_pci_data },
- { PCI_VDEVICE(INTEL, 0x0f40), (kernel_ulong_t)&dw_pci_data },
+ { PCI_VDEVICE(INTEL, 0x0f06), (kernel_ulong_t)&dw_dma_chip_pdata },
+ { PCI_VDEVICE(INTEL, 0x0f40), (kernel_ulong_t)&dw_dma_chip_pdata },
/* Merrifield */
- { PCI_VDEVICE(INTEL, 0x11a2), (kernel_ulong_t)&idma32_pci_data },
+ { PCI_VDEVICE(INTEL, 0x11a2), (kernel_ulong_t)&idma32_chip_pdata },
/* Braswell */
- { PCI_VDEVICE(INTEL, 0x2286), (kernel_ulong_t)&dw_pci_data },
- { PCI_VDEVICE(INTEL, 0x22c0), (kernel_ulong_t)&dw_pci_data },
+ { PCI_VDEVICE(INTEL, 0x2286), (kernel_ulong_t)&dw_dma_chip_pdata },
+ { PCI_VDEVICE(INTEL, 0x22c0), (kernel_ulong_t)&dw_dma_chip_pdata },
+
+ /* Elkhart Lake iDMA 32-bit (PSE DMA) */
+ { PCI_VDEVICE(INTEL, 0x4bb4), (kernel_ulong_t)&idma32_chip_pdata },
+ { PCI_VDEVICE(INTEL, 0x4bb5), (kernel_ulong_t)&idma32_chip_pdata },
+ { PCI_VDEVICE(INTEL, 0x4bb6), (kernel_ulong_t)&idma32_chip_pdata },
/* Haswell */
- { PCI_VDEVICE(INTEL, 0x9c60), (kernel_ulong_t)&dw_pci_data },
+ { PCI_VDEVICE(INTEL, 0x9c60), (kernel_ulong_t)&dw_dma_chip_pdata },
/* Broadwell */
- { PCI_VDEVICE(INTEL, 0x9ce0), (kernel_ulong_t)&dw_pci_data },
+ { PCI_VDEVICE(INTEL, 0x9ce0), (kernel_ulong_t)&dw_dma_chip_pdata },
{ }
};
diff --git a/drivers/dma/dw/platform.c b/drivers/dma/dw/platform.c
index 382dfd9e9600..c90c798e5ec3 100644
--- a/drivers/dma/dw/platform.c
+++ b/drivers/dma/dw/platform.c
@@ -17,163 +17,28 @@
#include <linux/dmaengine.h>
#include <linux/dma-mapping.h>
#include <linux/of.h>
-#include <linux/of_dma.h>
#include <linux/acpi.h>
-#include <linux/acpi_dma.h>
#include "internal.h"
#define DRV_NAME "dw_dmac"
-static struct dma_chan *dw_dma_of_xlate(struct of_phandle_args *dma_spec,
- struct of_dma *ofdma)
-{
- struct dw_dma *dw = ofdma->of_dma_data;
- struct dw_dma_slave slave = {
- .dma_dev = dw->dma.dev,
- };
- dma_cap_mask_t cap;
-
- if (dma_spec->args_count != 3)
- return NULL;
-
- slave.src_id = dma_spec->args[0];
- slave.dst_id = dma_spec->args[0];
- slave.m_master = dma_spec->args[1];
- slave.p_master = dma_spec->args[2];
-
- if (WARN_ON(slave.src_id >= DW_DMA_MAX_NR_REQUESTS ||
- slave.dst_id >= DW_DMA_MAX_NR_REQUESTS ||
- slave.m_master >= dw->pdata->nr_masters ||
- slave.p_master >= dw->pdata->nr_masters))
- return NULL;
-
- dma_cap_zero(cap);
- dma_cap_set(DMA_SLAVE, cap);
-
- /* TODO: there should be a simpler way to do this */
- return dma_request_channel(cap, dw_dma_filter, &slave);
-}
-
-#ifdef CONFIG_ACPI
-static bool dw_dma_acpi_filter(struct dma_chan *chan, void *param)
-{
- struct acpi_dma_spec *dma_spec = param;
- struct dw_dma_slave slave = {
- .dma_dev = dma_spec->dev,
- .src_id = dma_spec->slave_id,
- .dst_id = dma_spec->slave_id,
- .m_master = 0,
- .p_master = 1,
- };
-
- return dw_dma_filter(chan, &slave);
-}
-
-static void dw_dma_acpi_controller_register(struct dw_dma *dw)
-{
- struct device *dev = dw->dma.dev;
- struct acpi_dma_filter_info *info;
- int ret;
-
- info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL);
- if (!info)
- return;
-
- dma_cap_zero(info->dma_cap);
- dma_cap_set(DMA_SLAVE, info->dma_cap);
- info->filter_fn = dw_dma_acpi_filter;
-
- ret = devm_acpi_dma_controller_register(dev, acpi_dma_simple_xlate,
- info);
- if (ret)
- dev_err(dev, "could not register acpi_dma_controller\n");
-}
-#else /* !CONFIG_ACPI */
-static inline void dw_dma_acpi_controller_register(struct dw_dma *dw) {}
-#endif /* !CONFIG_ACPI */
-
-#ifdef CONFIG_OF
-static struct dw_dma_platform_data *
-dw_dma_parse_dt(struct platform_device *pdev)
-{
- struct device_node *np = pdev->dev.of_node;
- struct dw_dma_platform_data *pdata;
- u32 tmp, arr[DW_DMA_MAX_NR_MASTERS], mb[DW_DMA_MAX_NR_CHANNELS];
- u32 nr_masters;
- u32 nr_channels;
-
- if (!np) {
- dev_err(&pdev->dev, "Missing DT data\n");
- return NULL;
- }
-
- if (of_property_read_u32(np, "dma-masters", &nr_masters))
- return NULL;
- if (nr_masters < 1 || nr_masters > DW_DMA_MAX_NR_MASTERS)
- return NULL;
-
- if (of_property_read_u32(np, "dma-channels", &nr_channels))
- return NULL;
- if (nr_channels > DW_DMA_MAX_NR_CHANNELS)
- return NULL;
-
- pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
- if (!pdata)
- return NULL;
-
- pdata->nr_masters = nr_masters;
- pdata->nr_channels = nr_channels;
-
- if (!of_property_read_u32(np, "chan_allocation_order", &tmp))
- pdata->chan_allocation_order = (unsigned char)tmp;
-
- if (!of_property_read_u32(np, "chan_priority", &tmp))
- pdata->chan_priority = tmp;
-
- if (!of_property_read_u32(np, "block_size", &tmp))
- pdata->block_size = tmp;
-
- if (!of_property_read_u32_array(np, "data-width", arr, nr_masters)) {
- for (tmp = 0; tmp < nr_masters; tmp++)
- pdata->data_width[tmp] = arr[tmp];
- } else if (!of_property_read_u32_array(np, "data_width", arr, nr_masters)) {
- for (tmp = 0; tmp < nr_masters; tmp++)
- pdata->data_width[tmp] = BIT(arr[tmp] & 0x07);
- }
-
- if (!of_property_read_u32_array(np, "multi-block", mb, nr_channels)) {
- for (tmp = 0; tmp < nr_channels; tmp++)
- pdata->multi_block[tmp] = mb[tmp];
- } else {
- for (tmp = 0; tmp < nr_channels; tmp++)
- pdata->multi_block[tmp] = 1;
- }
-
- if (!of_property_read_u32(np, "snps,dma-protection-control", &tmp)) {
- if (tmp > CHAN_PROTCTL_MASK)
- return NULL;
- pdata->protctl = tmp;
- }
-
- return pdata;
-}
-#else
-static inline struct dw_dma_platform_data *
-dw_dma_parse_dt(struct platform_device *pdev)
-{
- return NULL;
-}
-#endif
-
static int dw_probe(struct platform_device *pdev)
{
+ const struct dw_dma_chip_pdata *match;
+ struct dw_dma_chip_pdata *data;
struct dw_dma_chip *chip;
struct device *dev = &pdev->dev;
- struct resource *mem;
- const struct dw_dma_platform_data *pdata;
int err;
+ match = device_get_match_data(dev);
+ if (!match)
+ return -ENODEV;
+
+ data = devm_kmemdup(&pdev->dev, match, sizeof(*match), GFP_KERNEL);
+ if (!data)
+ return -ENOMEM;
+
chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL);
if (!chip)
return -ENOMEM;
@@ -182,8 +47,7 @@ static int dw_probe(struct platform_device *pdev)
if (chip->irq < 0)
return chip->irq;
- mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- chip->regs = devm_ioremap_resource(dev, mem);
+ chip->regs = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(chip->regs))
return PTR_ERR(chip->regs);
@@ -191,13 +55,16 @@ static int dw_probe(struct platform_device *pdev)
if (err)
return err;
- pdata = dev_get_platdata(dev);
- if (!pdata)
- pdata = dw_dma_parse_dt(pdev);
+ if (!data->pdata)
+ data->pdata = dev_get_platdata(dev);
+ if (!data->pdata)
+ data->pdata = dw_dma_parse_dt(pdev);
chip->dev = dev;
chip->id = pdev->id;
- chip->pdata = pdata;
+ chip->pdata = data->pdata;
+
+ data->chip = chip;
chip->clk = devm_clk_get(chip->dev, "hclk");
if (IS_ERR(chip->clk))
@@ -208,22 +75,15 @@ static int dw_probe(struct platform_device *pdev)
pm_runtime_enable(&pdev->dev);
- err = dw_dma_probe(chip);
+ err = data->probe(chip);
if (err)
goto err_dw_dma_probe;
- platform_set_drvdata(pdev, chip);
+ platform_set_drvdata(pdev, data);
- if (pdev->dev.of_node) {
- err = of_dma_controller_register(pdev->dev.of_node,
- dw_dma_of_xlate, chip->dw);
- if (err)
- dev_err(&pdev->dev,
- "could not register of_dma_controller\n");
- }
+ dw_dma_of_controller_register(chip->dw);
- if (ACPI_HANDLE(&pdev->dev))
- dw_dma_acpi_controller_register(chip->dw);
+ dw_dma_acpi_controller_register(chip->dw);
return 0;
@@ -235,12 +95,18 @@ err_dw_dma_probe:
static int dw_remove(struct platform_device *pdev)
{
- struct dw_dma_chip *chip = platform_get_drvdata(pdev);
+ struct dw_dma_chip_pdata *data = platform_get_drvdata(pdev);
+ struct dw_dma_chip *chip = data->chip;
+ int ret;
+
+ dw_dma_acpi_controller_free(chip->dw);
- if (pdev->dev.of_node)
- of_dma_controller_free(pdev->dev.of_node);
+ dw_dma_of_controller_free(chip->dw);
+
+ ret = data->remove(chip);
+ if (ret)
+ dev_warn(chip->dev, "can't remove device properly: %d\n", ret);
- dw_dma_remove(chip);
pm_runtime_disable(&pdev->dev);
clk_disable_unprepare(chip->clk);
@@ -249,7 +115,8 @@ static int dw_remove(struct platform_device *pdev)
static void dw_shutdown(struct platform_device *pdev)
{
- struct dw_dma_chip *chip = platform_get_drvdata(pdev);
+ struct dw_dma_chip_pdata *data = platform_get_drvdata(pdev);
+ struct dw_dma_chip *chip = data->chip;
/*
* We have to call do_dw_dma_disable() to stop any ongoing transfer. On
@@ -269,7 +136,7 @@ static void dw_shutdown(struct platform_device *pdev)
#ifdef CONFIG_OF
static const struct of_device_id dw_dma_of_id_table[] = {
- { .compatible = "snps,dma-spear1340" },
+ { .compatible = "snps,dma-spear1340", .data = &dw_dma_chip_pdata },
{}
};
MODULE_DEVICE_TABLE(of, dw_dma_of_id_table);
@@ -277,9 +144,15 @@ MODULE_DEVICE_TABLE(of, dw_dma_of_id_table);
#ifdef CONFIG_ACPI
static const struct acpi_device_id dw_dma_acpi_id_table[] = {
- { "INTL9C60", 0 },
- { "80862286", 0 },
- { "808622C0", 0 },
+ { "INTL9C60", (kernel_ulong_t)&dw_dma_chip_pdata },
+ { "80862286", (kernel_ulong_t)&dw_dma_chip_pdata },
+ { "808622C0", (kernel_ulong_t)&dw_dma_chip_pdata },
+
+ /* Elkhart Lake iDMA 32-bit (PSE DMA) */
+ { "80864BB4", (kernel_ulong_t)&idma32_chip_pdata },
+ { "80864BB5", (kernel_ulong_t)&idma32_chip_pdata },
+ { "80864BB6", (kernel_ulong_t)&idma32_chip_pdata },
+
{ }
};
MODULE_DEVICE_TABLE(acpi, dw_dma_acpi_id_table);
@@ -289,7 +162,8 @@ MODULE_DEVICE_TABLE(acpi, dw_dma_acpi_id_table);
static int dw_suspend_late(struct device *dev)
{
- struct dw_dma_chip *chip = dev_get_drvdata(dev);
+ struct dw_dma_chip_pdata *data = dev_get_drvdata(dev);
+ struct dw_dma_chip *chip = data->chip;
do_dw_dma_disable(chip);
clk_disable_unprepare(chip->clk);
@@ -299,7 +173,8 @@ static int dw_suspend_late(struct device *dev)
static int dw_resume_early(struct device *dev)
{
- struct dw_dma_chip *chip = dev_get_drvdata(dev);
+ struct dw_dma_chip_pdata *data = dev_get_drvdata(dev);
+ struct dw_dma_chip *chip = data->chip;
int ret;
ret = clk_prepare_enable(chip->clk);
diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c
index 680b2a00a953..b1a7ca91701a 100644
--- a/drivers/dma/fsl-edma-common.c
+++ b/drivers/dma/fsl-edma-common.c
@@ -47,7 +47,7 @@ static void fsl_edma_enable_request(struct fsl_edma_chan *fsl_chan)
struct edma_regs *regs = &fsl_chan->edma->regs;
u32 ch = fsl_chan->vchan.chan.chan_id;
- if (fsl_chan->edma->version == v1) {
+ if (fsl_chan->edma->drvdata->version == v1) {
edma_writeb(fsl_chan->edma, EDMA_SEEI_SEEI(ch), regs->seei);
edma_writeb(fsl_chan->edma, ch, regs->serq);
} else {
@@ -64,7 +64,7 @@ void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan)
struct edma_regs *regs = &fsl_chan->edma->regs;
u32 ch = fsl_chan->vchan.chan.chan_id;
- if (fsl_chan->edma->version == v1) {
+ if (fsl_chan->edma->drvdata->version == v1) {
edma_writeb(fsl_chan->edma, ch, regs->cerq);
edma_writeb(fsl_chan->edma, EDMA_CEEI_CEEI(ch), regs->ceei);
} else {
@@ -77,22 +77,49 @@ void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan)
}
EXPORT_SYMBOL_GPL(fsl_edma_disable_request);
+static void mux_configure8(struct fsl_edma_chan *fsl_chan, void __iomem *addr,
+ u32 off, u32 slot, bool enable)
+{
+ u8 val8;
+
+ if (enable)
+ val8 = EDMAMUX_CHCFG_ENBL | slot;
+ else
+ val8 = EDMAMUX_CHCFG_DIS;
+
+ iowrite8(val8, addr + off);
+}
+
+static void mux_configure32(struct fsl_edma_chan *fsl_chan, void __iomem *addr,
+ u32 off, u32 slot, bool enable)
+{
+ u32 val;
+
+ if (enable)
+ val = EDMAMUX_CHCFG_ENBL << 24 | slot;
+ else
+ val = EDMAMUX_CHCFG_DIS;
+
+ iowrite32(val, addr + off * 4);
+}
+
void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan,
- unsigned int slot, bool enable)
+ unsigned int slot, bool enable)
{
u32 ch = fsl_chan->vchan.chan.chan_id;
void __iomem *muxaddr;
unsigned int chans_per_mux, ch_off;
+ u32 dmamux_nr = fsl_chan->edma->drvdata->dmamuxs;
- chans_per_mux = fsl_chan->edma->n_chans / DMAMUX_NR;
+ chans_per_mux = fsl_chan->edma->n_chans / dmamux_nr;
ch_off = fsl_chan->vchan.chan.chan_id % chans_per_mux;
muxaddr = fsl_chan->edma->muxbase[ch / chans_per_mux];
slot = EDMAMUX_CHCFG_SOURCE(slot);
- if (enable)
- iowrite8(EDMAMUX_CHCFG_ENBL | slot, muxaddr + ch_off);
+ if (fsl_chan->edma->drvdata->version == v3)
+ mux_configure32(fsl_chan, muxaddr, ch_off, slot, enable);
else
- iowrite8(EDMAMUX_CHCFG_DIS, muxaddr + ch_off);
+ mux_configure8(fsl_chan, muxaddr, ch_off, slot, enable);
}
EXPORT_SYMBOL_GPL(fsl_edma_chan_mux);
@@ -647,28 +674,28 @@ void fsl_edma_setup_regs(struct fsl_edma_engine *edma)
edma->regs.erql = edma->membase + EDMA_ERQ;
edma->regs.eeil = edma->membase + EDMA_EEI;
- edma->regs.serq = edma->membase + ((edma->version == v1) ?
- EDMA_SERQ : EDMA64_SERQ);
- edma->regs.cerq = edma->membase + ((edma->version == v1) ?
- EDMA_CERQ : EDMA64_CERQ);
- edma->regs.seei = edma->membase + ((edma->version == v1) ?
- EDMA_SEEI : EDMA64_SEEI);
- edma->regs.ceei = edma->membase + ((edma->version == v1) ?
- EDMA_CEEI : EDMA64_CEEI);
- edma->regs.cint = edma->membase + ((edma->version == v1) ?
- EDMA_CINT : EDMA64_CINT);
- edma->regs.cerr = edma->membase + ((edma->version == v1) ?
- EDMA_CERR : EDMA64_CERR);
- edma->regs.ssrt = edma->membase + ((edma->version == v1) ?
- EDMA_SSRT : EDMA64_SSRT);
- edma->regs.cdne = edma->membase + ((edma->version == v1) ?
- EDMA_CDNE : EDMA64_CDNE);
- edma->regs.intl = edma->membase + ((edma->version == v1) ?
- EDMA_INTR : EDMA64_INTL);
- edma->regs.errl = edma->membase + ((edma->version == v1) ?
- EDMA_ERR : EDMA64_ERRL);
-
- if (edma->version == v2) {
+ edma->regs.serq = edma->membase + ((edma->drvdata->version == v2) ?
+ EDMA64_SERQ : EDMA_SERQ);
+ edma->regs.cerq = edma->membase + ((edma->drvdata->version == v2) ?
+ EDMA64_CERQ : EDMA_CERQ);
+ edma->regs.seei = edma->membase + ((edma->drvdata->version == v2) ?
+ EDMA64_SEEI : EDMA_SEEI);
+ edma->regs.ceei = edma->membase + ((edma->drvdata->version == v2) ?
+ EDMA64_CEEI : EDMA_CEEI);
+ edma->regs.cint = edma->membase + ((edma->drvdata->version == v2) ?
+ EDMA64_CINT : EDMA_CINT);
+ edma->regs.cerr = edma->membase + ((edma->drvdata->version == v2) ?
+ EDMA64_CERR : EDMA_CERR);
+ edma->regs.ssrt = edma->membase + ((edma->drvdata->version == v2) ?
+ EDMA64_SSRT : EDMA_SSRT);
+ edma->regs.cdne = edma->membase + ((edma->drvdata->version == v2) ?
+ EDMA64_CDNE : EDMA_CDNE);
+ edma->regs.intl = edma->membase + ((edma->drvdata->version == v2) ?
+ EDMA64_INTL : EDMA_INTR);
+ edma->regs.errl = edma->membase + ((edma->drvdata->version == v2) ?
+ EDMA64_ERRL : EDMA_ERR);
+
+ if (edma->drvdata->version == v2) {
edma->regs.erqh = edma->membase + EDMA64_ERQH;
edma->regs.eeih = edma->membase + EDMA64_EEIH;
edma->regs.errh = edma->membase + EDMA64_ERRH;
diff --git a/drivers/dma/fsl-edma-common.h b/drivers/dma/fsl-edma-common.h
index c53f76eeb4d3..5eaa2902ed39 100644
--- a/drivers/dma/fsl-edma-common.h
+++ b/drivers/dma/fsl-edma-common.h
@@ -7,6 +7,7 @@
#define _FSL_EDMA_COMMON_H_
#include <linux/dma-direction.h>
+#include <linux/platform_device.h>
#include "virt-dma.h"
#define EDMA_CR_EDBG BIT(1)
@@ -124,6 +125,7 @@ struct fsl_edma_chan {
dma_addr_t dma_dev_addr;
u32 dma_dev_size;
enum dma_data_direction dma_dir;
+ char chan_name[16];
};
struct fsl_edma_desc {
@@ -138,6 +140,15 @@ struct fsl_edma_desc {
enum edma_version {
v1, /* 32ch, Vybrid, mpc57x, etc */
v2, /* 64ch Coldfire */
+ v3, /* 32ch, i.mx7ulp */
+};
+
+struct fsl_edma_drvdata {
+ enum edma_version version;
+ u32 dmamuxs;
+ bool has_dmaclk;
+ int (*setup_irq)(struct platform_device *pdev,
+ struct fsl_edma_engine *fsl_edma);
};
struct fsl_edma_engine {
@@ -145,12 +156,13 @@ struct fsl_edma_engine {
void __iomem *membase;
void __iomem *muxbase[DMAMUX_NR];
struct clk *muxclk[DMAMUX_NR];
+ struct clk *dmaclk;
struct mutex fsl_edma_mutex;
+ const struct fsl_edma_drvdata *drvdata;
u32 n_chans;
int txirq;
int errirq;
bool big_endian;
- enum edma_version version;
struct edma_regs regs;
struct fsl_edma_chan chans[];
};
diff --git a/drivers/dma/fsl-edma.c b/drivers/dma/fsl-edma.c
index 0ddad3adb761..b626c06ac2e0 100644
--- a/drivers/dma/fsl-edma.c
+++ b/drivers/dma/fsl-edma.c
@@ -20,6 +20,13 @@
#include "fsl-edma-common.h"
+static void fsl_edma_synchronize(struct dma_chan *chan)
+{
+ struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);
+
+ vchan_synchronize(&fsl_chan->vchan);
+}
+
static irqreturn_t fsl_edma_tx_handler(int irq, void *dev_id)
{
struct fsl_edma_engine *fsl_edma = dev_id;
@@ -92,7 +99,8 @@ static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec,
struct fsl_edma_engine *fsl_edma = ofdma->of_dma_data;
struct dma_chan *chan, *_chan;
struct fsl_edma_chan *fsl_chan;
- unsigned long chans_per_mux = fsl_edma->n_chans / DMAMUX_NR;
+ u32 dmamux_nr = fsl_edma->drvdata->dmamuxs;
+ unsigned long chans_per_mux = fsl_edma->n_chans / dmamux_nr;
if (dma_spec->args_count != 2)
return NULL;
@@ -124,16 +132,12 @@ fsl_edma_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma
int ret;
fsl_edma->txirq = platform_get_irq_byname(pdev, "edma-tx");
- if (fsl_edma->txirq < 0) {
- dev_err(&pdev->dev, "Can't get edma-tx irq.\n");
+ if (fsl_edma->txirq < 0)
return fsl_edma->txirq;
- }
fsl_edma->errirq = platform_get_irq_byname(pdev, "edma-err");
- if (fsl_edma->errirq < 0) {
- dev_err(&pdev->dev, "Can't get edma-err irq.\n");
+ if (fsl_edma->errirq < 0)
return fsl_edma->errirq;
- }
if (fsl_edma->txirq == fsl_edma->errirq) {
ret = devm_request_irq(&pdev->dev, fsl_edma->txirq,
@@ -161,6 +165,49 @@ fsl_edma_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma
return 0;
}
+static int
+fsl_edma2_irq_init(struct platform_device *pdev,
+ struct fsl_edma_engine *fsl_edma)
+{
+ int i, ret, irq;
+ int count;
+
+ count = platform_irq_count(pdev);
+ dev_dbg(&pdev->dev, "%s Found %d interrupts\r\n", __func__, count);
+ if (count <= 2) {
+ dev_err(&pdev->dev, "Interrupts in DTS not correct.\n");
+ return -EINVAL;
+ }
+ /*
+ * 16 channel independent interrupts + 1 error interrupt on i.mx7ulp.
+ * 2 channel share one interrupt, for example, ch0/ch16, ch1/ch17...
+ * For now, just simply request irq without IRQF_SHARED flag, since 16
+ * channels are enough on i.mx7ulp whose M4 domain own some peripherals.
+ */
+ for (i = 0; i < count; i++) {
+ irq = platform_get_irq(pdev, i);
+ if (irq < 0)
+ return -ENXIO;
+
+ sprintf(fsl_edma->chans[i].chan_name, "eDMA2-CH%02d", i);
+
+ /* The last IRQ is for eDMA err */
+ if (i == count - 1)
+ ret = devm_request_irq(&pdev->dev, irq,
+ fsl_edma_err_handler,
+ 0, "eDMA2-ERR", fsl_edma);
+ else
+ ret = devm_request_irq(&pdev->dev, irq,
+ fsl_edma_tx_handler, 0,
+ fsl_edma->chans[i].chan_name,
+ fsl_edma);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
static void fsl_edma_irq_exit(
struct platform_device *pdev, struct fsl_edma_engine *fsl_edma)
{
@@ -180,16 +227,46 @@ static void fsl_disable_clocks(struct fsl_edma_engine *fsl_edma, int nr_clocks)
clk_disable_unprepare(fsl_edma->muxclk[i]);
}
+static struct fsl_edma_drvdata vf610_data = {
+ .version = v1,
+ .dmamuxs = DMAMUX_NR,
+ .setup_irq = fsl_edma_irq_init,
+};
+
+static struct fsl_edma_drvdata imx7ulp_data = {
+ .version = v3,
+ .dmamuxs = 1,
+ .has_dmaclk = true,
+ .setup_irq = fsl_edma2_irq_init,
+};
+
+static const struct of_device_id fsl_edma_dt_ids[] = {
+ { .compatible = "fsl,vf610-edma", .data = &vf610_data},
+ { .compatible = "fsl,imx7ulp-edma", .data = &imx7ulp_data},
+ { /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, fsl_edma_dt_ids);
+
static int fsl_edma_probe(struct platform_device *pdev)
{
+ const struct of_device_id *of_id =
+ of_match_device(fsl_edma_dt_ids, &pdev->dev);
struct device_node *np = pdev->dev.of_node;
struct fsl_edma_engine *fsl_edma;
+ const struct fsl_edma_drvdata *drvdata = NULL;
struct fsl_edma_chan *fsl_chan;
struct edma_regs *regs;
struct resource *res;
int len, chans;
int ret, i;
+ if (of_id)
+ drvdata = of_id->data;
+ if (!drvdata) {
+ dev_err(&pdev->dev, "unable to find driver data\n");
+ return -EINVAL;
+ }
+
ret = of_property_read_u32(np, "dma-channels", &chans);
if (ret) {
dev_err(&pdev->dev, "Can't get dma-channels.\n");
@@ -201,7 +278,7 @@ static int fsl_edma_probe(struct platform_device *pdev)
if (!fsl_edma)
return -ENOMEM;
- fsl_edma->version = v1;
+ fsl_edma->drvdata = drvdata;
fsl_edma->n_chans = chans;
mutex_init(&fsl_edma->fsl_edma_mutex);
@@ -213,7 +290,21 @@ static int fsl_edma_probe(struct platform_device *pdev)
fsl_edma_setup_regs(fsl_edma);
regs = &fsl_edma->regs;
- for (i = 0; i < DMAMUX_NR; i++) {
+ if (drvdata->has_dmaclk) {
+ fsl_edma->dmaclk = devm_clk_get(&pdev->dev, "dma");
+ if (IS_ERR(fsl_edma->dmaclk)) {
+ dev_err(&pdev->dev, "Missing DMA block clock.\n");
+ return PTR_ERR(fsl_edma->dmaclk);
+ }
+
+ ret = clk_prepare_enable(fsl_edma->dmaclk);
+ if (ret) {
+ dev_err(&pdev->dev, "DMA clk block failed.\n");
+ return ret;
+ }
+ }
+
+ for (i = 0; i < fsl_edma->drvdata->dmamuxs; i++) {
char clkname[32];
res = platform_get_resource(pdev, IORESOURCE_MEM, 1 + i);
@@ -259,7 +350,7 @@ static int fsl_edma_probe(struct platform_device *pdev)
}
edma_writel(fsl_edma, ~0, regs->intl);
- ret = fsl_edma_irq_init(pdev, fsl_edma);
+ ret = fsl_edma->drvdata->setup_irq(pdev, fsl_edma);
if (ret)
return ret;
@@ -279,6 +370,7 @@ static int fsl_edma_probe(struct platform_device *pdev)
fsl_edma->dma_dev.device_pause = fsl_edma_pause;
fsl_edma->dma_dev.device_resume = fsl_edma_resume;
fsl_edma->dma_dev.device_terminate_all = fsl_edma_terminate_all;
+ fsl_edma->dma_dev.device_synchronize = fsl_edma_synchronize;
fsl_edma->dma_dev.device_issue_pending = fsl_edma_issue_pending;
fsl_edma->dma_dev.src_addr_widths = FSL_EDMA_BUSWIDTHS;
@@ -291,7 +383,7 @@ static int fsl_edma_probe(struct platform_device *pdev)
if (ret) {
dev_err(&pdev->dev,
"Can't register Freescale eDMA engine. (%d)\n", ret);
- fsl_disable_clocks(fsl_edma, DMAMUX_NR);
+ fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs);
return ret;
}
@@ -300,7 +392,7 @@ static int fsl_edma_probe(struct platform_device *pdev)
dev_err(&pdev->dev,
"Can't register Freescale eDMA of_dma. (%d)\n", ret);
dma_async_device_unregister(&fsl_edma->dma_dev);
- fsl_disable_clocks(fsl_edma, DMAMUX_NR);
+ fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs);
return ret;
}
@@ -319,7 +411,7 @@ static int fsl_edma_remove(struct platform_device *pdev)
fsl_edma_cleanup_vchan(&fsl_edma->dma_dev);
of_dma_controller_free(np);
dma_async_device_unregister(&fsl_edma->dma_dev);
- fsl_disable_clocks(fsl_edma, DMAMUX_NR);
+ fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs);
return 0;
}
@@ -378,12 +470,6 @@ static const struct dev_pm_ops fsl_edma_pm_ops = {
.resume_early = fsl_edma_resume_early,
};
-static const struct of_device_id fsl_edma_dt_ids[] = {
- { .compatible = "fsl,vf610-edma", },
- { /* sentinel */ }
-};
-MODULE_DEVICE_TABLE(of, fsl_edma_dt_ids);
-
static struct platform_driver fsl_edma_driver = {
.driver = {
.name = "fsl-edma",
diff --git a/drivers/dma/fsl-qdma.c b/drivers/dma/fsl-qdma.c
index 60b062c3647b..06664fbd2d91 100644
--- a/drivers/dma/fsl-qdma.c
+++ b/drivers/dma/fsl-qdma.c
@@ -113,6 +113,7 @@
/* Field definition for Descriptor offset */
#define QDMA_CCDF_STATUS 20
#define QDMA_CCDF_OFFSET 20
+#define QDMA_SDDF_CMD(x) (((u64)(x)) << 32)
/* Field definition for safe loop count*/
#define FSL_QDMA_HALT_COUNT 1500
@@ -341,6 +342,7 @@ static void fsl_qdma_free_chan_resources(struct dma_chan *chan)
static void fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp,
dma_addr_t dst, dma_addr_t src, u32 len)
{
+ u32 cmd;
struct fsl_qdma_format *sdf, *ddf;
struct fsl_qdma_format *ccdf, *csgf_desc, *csgf_src, *csgf_dest;
@@ -369,14 +371,14 @@ static void fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp,
/* This entry is the last entry. */
qdma_csgf_set_f(csgf_dest, len);
/* Descriptor Buffer */
- sdf->data =
- cpu_to_le64(FSL_QDMA_CMD_RWTTYPE <<
- FSL_QDMA_CMD_RWTTYPE_OFFSET);
- ddf->data =
- cpu_to_le64(FSL_QDMA_CMD_RWTTYPE <<
- FSL_QDMA_CMD_RWTTYPE_OFFSET);
- ddf->data |=
- cpu_to_le64(FSL_QDMA_CMD_LWC << FSL_QDMA_CMD_LWC_OFFSET);
+ cmd = cpu_to_le32(FSL_QDMA_CMD_RWTTYPE <<
+ FSL_QDMA_CMD_RWTTYPE_OFFSET);
+ sdf->data = QDMA_SDDF_CMD(cmd);
+
+ cmd = cpu_to_le32(FSL_QDMA_CMD_RWTTYPE <<
+ FSL_QDMA_CMD_RWTTYPE_OFFSET);
+ cmd |= cpu_to_le32(FSL_QDMA_CMD_LWC << FSL_QDMA_CMD_LWC_OFFSET);
+ ddf->data = QDMA_SDDF_CMD(cmd);
}
/*
@@ -756,10 +758,8 @@ fsl_qdma_irq_init(struct platform_device *pdev,
fsl_qdma->error_irq =
platform_get_irq_byname(pdev, "qdma-error");
- if (fsl_qdma->error_irq < 0) {
- dev_err(&pdev->dev, "Can't get qdma controller irq.\n");
+ if (fsl_qdma->error_irq < 0)
return fsl_qdma->error_irq;
- }
ret = devm_request_irq(&pdev->dev, fsl_qdma->error_irq,
fsl_qdma_error_handler, 0,
@@ -774,11 +774,8 @@ fsl_qdma_irq_init(struct platform_device *pdev,
fsl_qdma->queue_irq[i] =
platform_get_irq_byname(pdev, irq_name);
- if (fsl_qdma->queue_irq[i] < 0) {
- dev_err(&pdev->dev,
- "Can't get qdma queue %d irq.\n", i);
+ if (fsl_qdma->queue_irq[i] < 0)
return fsl_qdma->queue_irq[i];
- }
ret = devm_request_irq(&pdev->dev,
fsl_qdma->queue_irq[i],
diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c
index 23e0a356f167..ad72b3f42ffa 100644
--- a/drivers/dma/fsldma.c
+++ b/drivers/dma/fsldma.c
@@ -1163,6 +1163,7 @@ static int fsl_dma_chan_probe(struct fsldma_device *fdev,
switch (chan->feature & FSL_DMA_IP_MASK) {
case FSL_DMA_IP_85XX:
chan->toggle_ext_pause = fsl_chan_toggle_ext_pause;
+ /* Fall through */
case FSL_DMA_IP_83XX:
chan->toggle_ext_start = fsl_chan_toggle_ext_start;
chan->set_src_loop_size = fsl_chan_set_src_loop_size;
diff --git a/drivers/dma/hsu/hsu.c b/drivers/dma/hsu/hsu.c
index 0c2610066ba9..025d8ad5a63c 100644
--- a/drivers/dma/hsu/hsu.c
+++ b/drivers/dma/hsu/hsu.c
@@ -61,10 +61,10 @@ static void hsu_dma_chan_start(struct hsu_dma_chan *hsuc)
if (hsuc->direction == DMA_MEM_TO_DEV) {
bsr = config->dst_maxburst;
- mtsr = config->src_addr_width;
+ mtsr = config->dst_addr_width;
} else if (hsuc->direction == DMA_DEV_TO_MEM) {
bsr = config->src_maxburst;
- mtsr = config->dst_addr_width;
+ mtsr = config->src_addr_width;
}
hsu_chan_disable(hsuc);
diff --git a/drivers/dma/imx-dma.c b/drivers/dma/imx-dma.c
index 00a089e24150..5c0fb3134825 100644
--- a/drivers/dma/imx-dma.c
+++ b/drivers/dma/imx-dma.c
@@ -556,6 +556,7 @@ static int imxdma_xfer_desc(struct imxdma_desc *d)
* We fall-through here intentionally, since a 2D transfer is
* similar to MEMCPY just adding the 2D slot configuration.
*/
+ /* Fall through */
case IMXDMA_DESC_MEMCPY:
imx_dmav1_writel(imxdma, d->src, DMA_SAR(imxdmac->channel));
imx_dmav1_writel(imxdma, d->dest, DMA_DAR(imxdmac->channel));
diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c
index 4ec84a633bd3..9ba74ab7e912 100644
--- a/drivers/dma/imx-sdma.c
+++ b/drivers/dma/imx-sdma.c
@@ -1886,10 +1886,6 @@ static int sdma_init(struct sdma_engine *sdma)
sdma->context_phys = ccb_phys +
MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control);
- /* Zero-out the CCB structures array just allocated */
- memset(sdma->channel_control, 0,
- MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control));
-
/* disable all channels */
for (i = 0; i < sdma->drvdata->num_events; i++)
writel_relaxed(0, sdma->regs + chnenbl_ofs(sdma, i));
@@ -1934,16 +1930,11 @@ disable_clk_ipg:
static bool sdma_filter_fn(struct dma_chan *chan, void *fn_param)
{
struct sdma_channel *sdmac = to_sdma_chan(chan);
- struct sdma_engine *sdma = sdmac->sdma;
struct imx_dma_data *data = fn_param;
if (!imx_dma_is_general_purpose(chan))
return false;
- /* return false if it's not the right device */
- if (sdma->dev->of_node != data->of_node)
- return false;
-
sdmac->data = *data;
chan->private = &sdmac->data;
@@ -1971,9 +1962,9 @@ static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec,
* be set to sdmac->event_id1.
*/
data.dma_request2 = 0;
- data.of_node = ofdma->of_node;
- return dma_request_channel(mask, sdma_filter_fn, &data);
+ return __dma_request_channel(&mask, sdma_filter_fn, &data,
+ ofdma->of_node);
}
static int sdma_probe(struct platform_device *pdev)
diff --git a/drivers/dma/ioat/dca.c b/drivers/dma/ioat/dca.c
index 70fd8454d002..be61c32a876f 100644
--- a/drivers/dma/ioat/dca.c
+++ b/drivers/dma/ioat/dca.c
@@ -286,8 +286,7 @@ struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
return NULL;
dca = alloc_dca_provider(&ioat_dca_ops,
- sizeof(*ioatdca)
- + (sizeof(struct ioat_dca_slot) * slots));
+ struct_size(ioatdca, req_slots, slots));
if (!dca)
return NULL;
diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c
index c6c0143670d9..a3f942a6a946 100644
--- a/drivers/dma/iop-adma.c
+++ b/drivers/dma/iop-adma.c
@@ -16,13 +16,13 @@
#include <linux/spinlock.h>
#include <linux/interrupt.h>
#include <linux/platform_device.h>
+#include <linux/prefetch.h>
#include <linux/memory.h>
#include <linux/ioport.h>
#include <linux/raid/pq.h>
#include <linux/slab.h>
-#include <mach/adma.h>
-
+#include "iop-adma.h"
#include "dmaengine.h"
#define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common)
@@ -116,9 +116,9 @@ static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
chain_node) {
pr_debug("\tcookie: %d slot: %d busy: %d "
- "this_desc: %#x next_desc: %#x ack: %d\n",
+ "this_desc: %pad next_desc: %#llx ack: %d\n",
iter->async_tx.cookie, iter->idx, busy,
- iter->async_tx.phys, iop_desc_get_next_desc(iter),
+ &iter->async_tx.phys, (u64)iop_desc_get_next_desc(iter),
async_tx_test_ack(&iter->async_tx));
prefetch(_iter);
prefetch(&_iter->async_tx);
@@ -306,9 +306,9 @@ retry:
int i;
dev_dbg(iop_chan->device->common.dev,
"allocated slot: %d "
- "(desc %p phys: %#x) slots_per_op %d\n",
+ "(desc %p phys: %#llx) slots_per_op %d\n",
iter->idx, iter->hw_desc,
- iter->async_tx.phys, slots_per_op);
+ (u64)iter->async_tx.phys, slots_per_op);
/* pre-ack all but the last descriptor */
if (num_slots != slots_per_op)
@@ -364,13 +364,11 @@ iop_adma_tx_submit(struct dma_async_tx_descriptor *tx)
struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan);
struct iop_adma_desc_slot *grp_start, *old_chain_tail;
int slot_cnt;
- int slots_per_op;
dma_cookie_t cookie;
dma_addr_t next_dma;
grp_start = sw_desc->group_head;
slot_cnt = grp_start->slot_cnt;
- slots_per_op = grp_start->slots_per_op;
spin_lock_bh(&iop_chan->lock);
cookie = dma_cookie_assign(tx);
@@ -516,7 +514,7 @@ iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
return NULL;
BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT);
- dev_dbg(iop_chan->device->common.dev, "%s len: %u\n",
+ dev_dbg(iop_chan->device->common.dev, "%s len: %zu\n",
__func__, len);
spin_lock_bh(&iop_chan->lock);
@@ -549,7 +547,7 @@ iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest,
BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
dev_dbg(iop_chan->device->common.dev,
- "%s src_cnt: %d len: %u flags: %lx\n",
+ "%s src_cnt: %d len: %zu flags: %lx\n",
__func__, src_cnt, len, flags);
spin_lock_bh(&iop_chan->lock);
@@ -582,7 +580,7 @@ iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src,
if (unlikely(!len))
return NULL;
- dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
+ dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %zu\n",
__func__, src_cnt, len);
spin_lock_bh(&iop_chan->lock);
@@ -620,7 +618,7 @@ iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
dev_dbg(iop_chan->device->common.dev,
- "%s src_cnt: %d len: %u flags: %lx\n",
+ "%s src_cnt: %d len: %zu flags: %lx\n",
__func__, src_cnt, len, flags);
if (dmaf_p_disabled_continue(flags))
@@ -683,7 +681,7 @@ iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
return NULL;
BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
- dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
+ dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %zu\n",
__func__, src_cnt, len);
spin_lock_bh(&iop_chan->lock);
diff --git a/drivers/dma/iop-adma.h b/drivers/dma/iop-adma.h
new file mode 100644
index 000000000000..c499c9578f00
--- /dev/null
+++ b/drivers/dma/iop-adma.h
@@ -0,0 +1,914 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright © 2006, Intel Corporation.
+ */
+#ifndef _ADMA_H
+#define _ADMA_H
+#include <linux/types.h>
+#include <linux/io.h>
+#include <linux/platform_data/dma-iop32x.h>
+
+/* Memory copy units */
+#define DMA_CCR(chan) (chan->mmr_base + 0x0)
+#define DMA_CSR(chan) (chan->mmr_base + 0x4)
+#define DMA_DAR(chan) (chan->mmr_base + 0xc)
+#define DMA_NDAR(chan) (chan->mmr_base + 0x10)
+#define DMA_PADR(chan) (chan->mmr_base + 0x14)
+#define DMA_PUADR(chan) (chan->mmr_base + 0x18)
+#define DMA_LADR(chan) (chan->mmr_base + 0x1c)
+#define DMA_BCR(chan) (chan->mmr_base + 0x20)
+#define DMA_DCR(chan) (chan->mmr_base + 0x24)
+
+/* Application accelerator unit */
+#define AAU_ACR(chan) (chan->mmr_base + 0x0)
+#define AAU_ASR(chan) (chan->mmr_base + 0x4)
+#define AAU_ADAR(chan) (chan->mmr_base + 0x8)
+#define AAU_ANDAR(chan) (chan->mmr_base + 0xc)
+#define AAU_SAR(src, chan) (chan->mmr_base + (0x10 + ((src) << 2)))
+#define AAU_DAR(chan) (chan->mmr_base + 0x20)
+#define AAU_ABCR(chan) (chan->mmr_base + 0x24)
+#define AAU_ADCR(chan) (chan->mmr_base + 0x28)
+#define AAU_SAR_EDCR(src_edc) (chan->mmr_base + (0x02c + ((src_edc-4) << 2)))
+#define AAU_EDCR0_IDX 8
+#define AAU_EDCR1_IDX 17
+#define AAU_EDCR2_IDX 26
+
+struct iop3xx_aau_desc_ctrl {
+ unsigned int int_en:1;
+ unsigned int blk1_cmd_ctrl:3;
+ unsigned int blk2_cmd_ctrl:3;
+ unsigned int blk3_cmd_ctrl:3;
+ unsigned int blk4_cmd_ctrl:3;
+ unsigned int blk5_cmd_ctrl:3;
+ unsigned int blk6_cmd_ctrl:3;
+ unsigned int blk7_cmd_ctrl:3;
+ unsigned int blk8_cmd_ctrl:3;
+ unsigned int blk_ctrl:2;
+ unsigned int dual_xor_en:1;
+ unsigned int tx_complete:1;
+ unsigned int zero_result_err:1;
+ unsigned int zero_result_en:1;
+ unsigned int dest_write_en:1;
+};
+
+struct iop3xx_aau_e_desc_ctrl {
+ unsigned int reserved:1;
+ unsigned int blk1_cmd_ctrl:3;
+ unsigned int blk2_cmd_ctrl:3;
+ unsigned int blk3_cmd_ctrl:3;
+ unsigned int blk4_cmd_ctrl:3;
+ unsigned int blk5_cmd_ctrl:3;
+ unsigned int blk6_cmd_ctrl:3;
+ unsigned int blk7_cmd_ctrl:3;
+ unsigned int blk8_cmd_ctrl:3;
+ unsigned int reserved2:7;
+};
+
+struct iop3xx_dma_desc_ctrl {
+ unsigned int pci_transaction:4;
+ unsigned int int_en:1;
+ unsigned int dac_cycle_en:1;
+ unsigned int mem_to_mem_en:1;
+ unsigned int crc_data_tx_en:1;
+ unsigned int crc_gen_en:1;
+ unsigned int crc_seed_dis:1;
+ unsigned int reserved:21;
+ unsigned int crc_tx_complete:1;
+};
+
+struct iop3xx_desc_dma {
+ u32 next_desc;
+ union {
+ u32 pci_src_addr;
+ u32 pci_dest_addr;
+ u32 src_addr;
+ };
+ union {
+ u32 upper_pci_src_addr;
+ u32 upper_pci_dest_addr;
+ };
+ union {
+ u32 local_pci_src_addr;
+ u32 local_pci_dest_addr;
+ u32 dest_addr;
+ };
+ u32 byte_count;
+ union {
+ u32 desc_ctrl;
+ struct iop3xx_dma_desc_ctrl desc_ctrl_field;
+ };
+ u32 crc_addr;
+};
+
+struct iop3xx_desc_aau {
+ u32 next_desc;
+ u32 src[4];
+ u32 dest_addr;
+ u32 byte_count;
+ union {
+ u32 desc_ctrl;
+ struct iop3xx_aau_desc_ctrl desc_ctrl_field;
+ };
+ union {
+ u32 src_addr;
+ u32 e_desc_ctrl;
+ struct iop3xx_aau_e_desc_ctrl e_desc_ctrl_field;
+ } src_edc[31];
+};
+
+struct iop3xx_aau_gfmr {
+ unsigned int gfmr1:8;
+ unsigned int gfmr2:8;
+ unsigned int gfmr3:8;
+ unsigned int gfmr4:8;
+};
+
+struct iop3xx_desc_pq_xor {
+ u32 next_desc;
+ u32 src[3];
+ union {
+ u32 data_mult1;
+ struct iop3xx_aau_gfmr data_mult1_field;
+ };
+ u32 dest_addr;
+ u32 byte_count;
+ union {
+ u32 desc_ctrl;
+ struct iop3xx_aau_desc_ctrl desc_ctrl_field;
+ };
+ union {
+ u32 src_addr;
+ u32 e_desc_ctrl;
+ struct iop3xx_aau_e_desc_ctrl e_desc_ctrl_field;
+ u32 data_multiplier;
+ struct iop3xx_aau_gfmr data_mult_field;
+ u32 reserved;
+ } src_edc_gfmr[19];
+};
+
+struct iop3xx_desc_dual_xor {
+ u32 next_desc;
+ u32 src0_addr;
+ u32 src1_addr;
+ u32 h_src_addr;
+ u32 d_src_addr;
+ u32 h_dest_addr;
+ u32 byte_count;
+ union {
+ u32 desc_ctrl;
+ struct iop3xx_aau_desc_ctrl desc_ctrl_field;
+ };
+ u32 d_dest_addr;
+};
+
+union iop3xx_desc {
+ struct iop3xx_desc_aau *aau;
+ struct iop3xx_desc_dma *dma;
+ struct iop3xx_desc_pq_xor *pq_xor;
+ struct iop3xx_desc_dual_xor *dual_xor;
+ void *ptr;
+};
+
+/* No support for p+q operations */
+static inline int
+iop_chan_pq_slot_count(size_t len, int src_cnt, int *slots_per_op)
+{
+ BUG();
+ return 0;
+}
+
+static inline void
+iop_desc_init_pq(struct iop_adma_desc_slot *desc, int src_cnt,
+ unsigned long flags)
+{
+ BUG();
+}
+
+static inline void
+iop_desc_set_pq_addr(struct iop_adma_desc_slot *desc, dma_addr_t *addr)
+{
+ BUG();
+}
+
+static inline void
+iop_desc_set_pq_src_addr(struct iop_adma_desc_slot *desc, int src_idx,
+ dma_addr_t addr, unsigned char coef)
+{
+ BUG();
+}
+
+static inline int
+iop_chan_pq_zero_sum_slot_count(size_t len, int src_cnt, int *slots_per_op)
+{
+ BUG();
+ return 0;
+}
+
+static inline void
+iop_desc_init_pq_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt,
+ unsigned long flags)
+{
+ BUG();
+}
+
+static inline void
+iop_desc_set_pq_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len)
+{
+ BUG();
+}
+
+#define iop_desc_set_pq_zero_sum_src_addr iop_desc_set_pq_src_addr
+
+static inline void
+iop_desc_set_pq_zero_sum_addr(struct iop_adma_desc_slot *desc, int pq_idx,
+ dma_addr_t *src)
+{
+ BUG();
+}
+
+static inline int iop_adma_get_max_xor(void)
+{
+ return 32;
+}
+
+static inline int iop_adma_get_max_pq(void)
+{
+ BUG();
+ return 0;
+}
+
+static inline u32 iop_chan_get_current_descriptor(struct iop_adma_chan *chan)
+{
+ int id = chan->device->id;
+
+ switch (id) {
+ case DMA0_ID:
+ case DMA1_ID:
+ return __raw_readl(DMA_DAR(chan));
+ case AAU_ID:
+ return __raw_readl(AAU_ADAR(chan));
+ default:
+ BUG();
+ }
+ return 0;
+}
+
+static inline void iop_chan_set_next_descriptor(struct iop_adma_chan *chan,
+ u32 next_desc_addr)
+{
+ int id = chan->device->id;
+
+ switch (id) {
+ case DMA0_ID:
+ case DMA1_ID:
+ __raw_writel(next_desc_addr, DMA_NDAR(chan));
+ break;
+ case AAU_ID:
+ __raw_writel(next_desc_addr, AAU_ANDAR(chan));
+ break;
+ }
+
+}
+
+#define IOP_ADMA_STATUS_BUSY (1 << 10)
+#define IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT (1024)
+#define IOP_ADMA_XOR_MAX_BYTE_COUNT (16 * 1024 * 1024)
+#define IOP_ADMA_MAX_BYTE_COUNT (16 * 1024 * 1024)
+
+static inline int iop_chan_is_busy(struct iop_adma_chan *chan)
+{
+ u32 status = __raw_readl(DMA_CSR(chan));
+ return (status & IOP_ADMA_STATUS_BUSY) ? 1 : 0;
+}
+
+static inline int iop_desc_is_aligned(struct iop_adma_desc_slot *desc,
+ int num_slots)
+{
+ /* num_slots will only ever be 1, 2, 4, or 8 */
+ return (desc->idx & (num_slots - 1)) ? 0 : 1;
+}
+
+/* to do: support large (i.e. > hw max) buffer sizes */
+static inline int iop_chan_memcpy_slot_count(size_t len, int *slots_per_op)
+{
+ *slots_per_op = 1;
+ return 1;
+}
+
+/* to do: support large (i.e. > hw max) buffer sizes */
+static inline int iop_chan_memset_slot_count(size_t len, int *slots_per_op)
+{
+ *slots_per_op = 1;
+ return 1;
+}
+
+static inline int iop3xx_aau_xor_slot_count(size_t len, int src_cnt,
+ int *slots_per_op)
+{
+ static const char slot_count_table[] = {
+ 1, 1, 1, 1, /* 01 - 04 */
+ 2, 2, 2, 2, /* 05 - 08 */
+ 4, 4, 4, 4, /* 09 - 12 */
+ 4, 4, 4, 4, /* 13 - 16 */
+ 8, 8, 8, 8, /* 17 - 20 */
+ 8, 8, 8, 8, /* 21 - 24 */
+ 8, 8, 8, 8, /* 25 - 28 */
+ 8, 8, 8, 8, /* 29 - 32 */
+ };
+ *slots_per_op = slot_count_table[src_cnt - 1];
+ return *slots_per_op;
+}
+
+static inline int
+iop_chan_interrupt_slot_count(int *slots_per_op, struct iop_adma_chan *chan)
+{
+ switch (chan->device->id) {
+ case DMA0_ID:
+ case DMA1_ID:
+ return iop_chan_memcpy_slot_count(0, slots_per_op);
+ case AAU_ID:
+ return iop3xx_aau_xor_slot_count(0, 2, slots_per_op);
+ default:
+ BUG();
+ }
+ return 0;
+}
+
+static inline int iop_chan_xor_slot_count(size_t len, int src_cnt,
+ int *slots_per_op)
+{
+ int slot_cnt = iop3xx_aau_xor_slot_count(len, src_cnt, slots_per_op);
+
+ if (len <= IOP_ADMA_XOR_MAX_BYTE_COUNT)
+ return slot_cnt;
+
+ len -= IOP_ADMA_XOR_MAX_BYTE_COUNT;
+ while (len > IOP_ADMA_XOR_MAX_BYTE_COUNT) {
+ len -= IOP_ADMA_XOR_MAX_BYTE_COUNT;
+ slot_cnt += *slots_per_op;
+ }
+
+ slot_cnt += *slots_per_op;
+
+ return slot_cnt;
+}
+
+/* zero sum on iop3xx is limited to 1k at a time so it requires multiple
+ * descriptors
+ */
+static inline int iop_chan_zero_sum_slot_count(size_t len, int src_cnt,
+ int *slots_per_op)
+{
+ int slot_cnt = iop3xx_aau_xor_slot_count(len, src_cnt, slots_per_op);
+
+ if (len <= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT)
+ return slot_cnt;
+
+ len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
+ while (len > IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) {
+ len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
+ slot_cnt += *slots_per_op;
+ }
+
+ slot_cnt += *slots_per_op;
+
+ return slot_cnt;
+}
+
+static inline u32 iop_desc_get_byte_count(struct iop_adma_desc_slot *desc,
+ struct iop_adma_chan *chan)
+{
+ union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
+
+ switch (chan->device->id) {
+ case DMA0_ID:
+ case DMA1_ID:
+ return hw_desc.dma->byte_count;
+ case AAU_ID:
+ return hw_desc.aau->byte_count;
+ default:
+ BUG();
+ }
+ return 0;
+}
+
+/* translate the src_idx to a descriptor word index */
+static inline int __desc_idx(int src_idx)
+{
+ static const int desc_idx_table[] = { 0, 0, 0, 0,
+ 0, 1, 2, 3,
+ 5, 6, 7, 8,
+ 9, 10, 11, 12,
+ 14, 15, 16, 17,
+ 18, 19, 20, 21,
+ 23, 24, 25, 26,
+ 27, 28, 29, 30,
+ };
+
+ return desc_idx_table[src_idx];
+}
+
+static inline u32 iop_desc_get_src_addr(struct iop_adma_desc_slot *desc,
+ struct iop_adma_chan *chan,
+ int src_idx)
+{
+ union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
+
+ switch (chan->device->id) {
+ case DMA0_ID:
+ case DMA1_ID:
+ return hw_desc.dma->src_addr;
+ case AAU_ID:
+ break;
+ default:
+ BUG();
+ }
+
+ if (src_idx < 4)
+ return hw_desc.aau->src[src_idx];
+ else
+ return hw_desc.aau->src_edc[__desc_idx(src_idx)].src_addr;
+}
+
+static inline void iop3xx_aau_desc_set_src_addr(struct iop3xx_desc_aau *hw_desc,
+ int src_idx, dma_addr_t addr)
+{
+ if (src_idx < 4)
+ hw_desc->src[src_idx] = addr;
+ else
+ hw_desc->src_edc[__desc_idx(src_idx)].src_addr = addr;
+}
+
+static inline void
+iop_desc_init_memcpy(struct iop_adma_desc_slot *desc, unsigned long flags)
+{
+ struct iop3xx_desc_dma *hw_desc = desc->hw_desc;
+ union {
+ u32 value;
+ struct iop3xx_dma_desc_ctrl field;
+ } u_desc_ctrl;
+
+ u_desc_ctrl.value = 0;
+ u_desc_ctrl.field.mem_to_mem_en = 1;
+ u_desc_ctrl.field.pci_transaction = 0xe; /* memory read block */
+ u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
+ hw_desc->desc_ctrl = u_desc_ctrl.value;
+ hw_desc->upper_pci_src_addr = 0;
+ hw_desc->crc_addr = 0;
+}
+
+static inline void
+iop_desc_init_memset(struct iop_adma_desc_slot *desc, unsigned long flags)
+{
+ struct iop3xx_desc_aau *hw_desc = desc->hw_desc;
+ union {
+ u32 value;
+ struct iop3xx_aau_desc_ctrl field;
+ } u_desc_ctrl;
+
+ u_desc_ctrl.value = 0;
+ u_desc_ctrl.field.blk1_cmd_ctrl = 0x2; /* memory block fill */
+ u_desc_ctrl.field.dest_write_en = 1;
+ u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
+ hw_desc->desc_ctrl = u_desc_ctrl.value;
+}
+
+static inline u32
+iop3xx_desc_init_xor(struct iop3xx_desc_aau *hw_desc, int src_cnt,
+ unsigned long flags)
+{
+ int i, shift;
+ u32 edcr;
+ union {
+ u32 value;
+ struct iop3xx_aau_desc_ctrl field;
+ } u_desc_ctrl;
+
+ u_desc_ctrl.value = 0;
+ switch (src_cnt) {
+ case 25 ... 32:
+ u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */
+ edcr = 0;
+ shift = 1;
+ for (i = 24; i < src_cnt; i++) {
+ edcr |= (1 << shift);
+ shift += 3;
+ }
+ hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = edcr;
+ src_cnt = 24;
+ /* fall through */
+ case 17 ... 24:
+ if (!u_desc_ctrl.field.blk_ctrl) {
+ hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0;
+ u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */
+ }
+ edcr = 0;
+ shift = 1;
+ for (i = 16; i < src_cnt; i++) {
+ edcr |= (1 << shift);
+ shift += 3;
+ }
+ hw_desc->src_edc[AAU_EDCR1_IDX].e_desc_ctrl = edcr;
+ src_cnt = 16;
+ /* fall through */
+ case 9 ... 16:
+ if (!u_desc_ctrl.field.blk_ctrl)
+ u_desc_ctrl.field.blk_ctrl = 0x2; /* use EDCR0 */
+ edcr = 0;
+ shift = 1;
+ for (i = 8; i < src_cnt; i++) {
+ edcr |= (1 << shift);
+ shift += 3;
+ }
+ hw_desc->src_edc[AAU_EDCR0_IDX].e_desc_ctrl = edcr;
+ src_cnt = 8;
+ /* fall through */
+ case 2 ... 8:
+ shift = 1;
+ for (i = 0; i < src_cnt; i++) {
+ u_desc_ctrl.value |= (1 << shift);
+ shift += 3;
+ }
+
+ if (!u_desc_ctrl.field.blk_ctrl && src_cnt > 4)
+ u_desc_ctrl.field.blk_ctrl = 0x1; /* use mini-desc */
+ }
+
+ u_desc_ctrl.field.dest_write_en = 1;
+ u_desc_ctrl.field.blk1_cmd_ctrl = 0x7; /* direct fill */
+ u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
+ hw_desc->desc_ctrl = u_desc_ctrl.value;
+
+ return u_desc_ctrl.value;
+}
+
+static inline void
+iop_desc_init_xor(struct iop_adma_desc_slot *desc, int src_cnt,
+ unsigned long flags)
+{
+ iop3xx_desc_init_xor(desc->hw_desc, src_cnt, flags);
+}
+
+/* return the number of operations */
+static inline int
+iop_desc_init_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt,
+ unsigned long flags)
+{
+ int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op;
+ struct iop3xx_desc_aau *hw_desc, *prev_hw_desc, *iter;
+ union {
+ u32 value;
+ struct iop3xx_aau_desc_ctrl field;
+ } u_desc_ctrl;
+ int i, j;
+
+ hw_desc = desc->hw_desc;
+
+ for (i = 0, j = 0; (slot_cnt -= slots_per_op) >= 0;
+ i += slots_per_op, j++) {
+ iter = iop_hw_desc_slot_idx(hw_desc, i);
+ u_desc_ctrl.value = iop3xx_desc_init_xor(iter, src_cnt, flags);
+ u_desc_ctrl.field.dest_write_en = 0;
+ u_desc_ctrl.field.zero_result_en = 1;
+ u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
+ iter->desc_ctrl = u_desc_ctrl.value;
+
+ /* for the subsequent descriptors preserve the store queue
+ * and chain them together
+ */
+ if (i) {
+ prev_hw_desc =
+ iop_hw_desc_slot_idx(hw_desc, i - slots_per_op);
+ prev_hw_desc->next_desc =
+ (u32) (desc->async_tx.phys + (i << 5));
+ }
+ }
+
+ return j;
+}
+
+static inline void
+iop_desc_init_null_xor(struct iop_adma_desc_slot *desc, int src_cnt,
+ unsigned long flags)
+{
+ struct iop3xx_desc_aau *hw_desc = desc->hw_desc;
+ union {
+ u32 value;
+ struct iop3xx_aau_desc_ctrl field;
+ } u_desc_ctrl;
+
+ u_desc_ctrl.value = 0;
+ switch (src_cnt) {
+ case 25 ... 32:
+ u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */
+ hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0;
+ /* fall through */
+ case 17 ... 24:
+ if (!u_desc_ctrl.field.blk_ctrl) {
+ hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0;
+ u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */
+ }
+ hw_desc->src_edc[AAU_EDCR1_IDX].e_desc_ctrl = 0;
+ /* fall through */
+ case 9 ... 16:
+ if (!u_desc_ctrl.field.blk_ctrl)
+ u_desc_ctrl.field.blk_ctrl = 0x2; /* use EDCR0 */
+ hw_desc->src_edc[AAU_EDCR0_IDX].e_desc_ctrl = 0;
+ /* fall through */
+ case 1 ... 8:
+ if (!u_desc_ctrl.field.blk_ctrl && src_cnt > 4)
+ u_desc_ctrl.field.blk_ctrl = 0x1; /* use mini-desc */
+ }
+
+ u_desc_ctrl.field.dest_write_en = 0;
+ u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
+ hw_desc->desc_ctrl = u_desc_ctrl.value;
+}
+
+static inline void iop_desc_set_byte_count(struct iop_adma_desc_slot *desc,
+ struct iop_adma_chan *chan,
+ u32 byte_count)
+{
+ union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
+
+ switch (chan->device->id) {
+ case DMA0_ID:
+ case DMA1_ID:
+ hw_desc.dma->byte_count = byte_count;
+ break;
+ case AAU_ID:
+ hw_desc.aau->byte_count = byte_count;
+ break;
+ default:
+ BUG();
+ }
+}
+
+static inline void
+iop_desc_init_interrupt(struct iop_adma_desc_slot *desc,
+ struct iop_adma_chan *chan)
+{
+ union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
+
+ switch (chan->device->id) {
+ case DMA0_ID:
+ case DMA1_ID:
+ iop_desc_init_memcpy(desc, 1);
+ hw_desc.dma->byte_count = 0;
+ hw_desc.dma->dest_addr = 0;
+ hw_desc.dma->src_addr = 0;
+ break;
+ case AAU_ID:
+ iop_desc_init_null_xor(desc, 2, 1);
+ hw_desc.aau->byte_count = 0;
+ hw_desc.aau->dest_addr = 0;
+ hw_desc.aau->src[0] = 0;
+ hw_desc.aau->src[1] = 0;
+ break;
+ default:
+ BUG();
+ }
+}
+
+static inline void
+iop_desc_set_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len)
+{
+ int slots_per_op = desc->slots_per_op;
+ struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter;
+ int i = 0;
+
+ if (len <= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) {
+ hw_desc->byte_count = len;
+ } else {
+ do {
+ iter = iop_hw_desc_slot_idx(hw_desc, i);
+ iter->byte_count = IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
+ len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
+ i += slots_per_op;
+ } while (len > IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT);
+
+ iter = iop_hw_desc_slot_idx(hw_desc, i);
+ iter->byte_count = len;
+ }
+}
+
+static inline void iop_desc_set_dest_addr(struct iop_adma_desc_slot *desc,
+ struct iop_adma_chan *chan,
+ dma_addr_t addr)
+{
+ union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
+
+ switch (chan->device->id) {
+ case DMA0_ID:
+ case DMA1_ID:
+ hw_desc.dma->dest_addr = addr;
+ break;
+ case AAU_ID:
+ hw_desc.aau->dest_addr = addr;
+ break;
+ default:
+ BUG();
+ }
+}
+
+static inline void iop_desc_set_memcpy_src_addr(struct iop_adma_desc_slot *desc,
+ dma_addr_t addr)
+{
+ struct iop3xx_desc_dma *hw_desc = desc->hw_desc;
+ hw_desc->src_addr = addr;
+}
+
+static inline void
+iop_desc_set_zero_sum_src_addr(struct iop_adma_desc_slot *desc, int src_idx,
+ dma_addr_t addr)
+{
+
+ struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter;
+ int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op;
+ int i;
+
+ for (i = 0; (slot_cnt -= slots_per_op) >= 0;
+ i += slots_per_op, addr += IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) {
+ iter = iop_hw_desc_slot_idx(hw_desc, i);
+ iop3xx_aau_desc_set_src_addr(iter, src_idx, addr);
+ }
+}
+
+static inline void iop_desc_set_xor_src_addr(struct iop_adma_desc_slot *desc,
+ int src_idx, dma_addr_t addr)
+{
+
+ struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter;
+ int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op;
+ int i;
+
+ for (i = 0; (slot_cnt -= slots_per_op) >= 0;
+ i += slots_per_op, addr += IOP_ADMA_XOR_MAX_BYTE_COUNT) {
+ iter = iop_hw_desc_slot_idx(hw_desc, i);
+ iop3xx_aau_desc_set_src_addr(iter, src_idx, addr);
+ }
+}
+
+static inline void iop_desc_set_next_desc(struct iop_adma_desc_slot *desc,
+ u32 next_desc_addr)
+{
+ /* hw_desc->next_desc is the same location for all channels */
+ union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
+
+ iop_paranoia(hw_desc.dma->next_desc);
+ hw_desc.dma->next_desc = next_desc_addr;
+}
+
+static inline u32 iop_desc_get_next_desc(struct iop_adma_desc_slot *desc)
+{
+ /* hw_desc->next_desc is the same location for all channels */
+ union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
+ return hw_desc.dma->next_desc;
+}
+
+static inline void iop_desc_clear_next_desc(struct iop_adma_desc_slot *desc)
+{
+ /* hw_desc->next_desc is the same location for all channels */
+ union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
+ hw_desc.dma->next_desc = 0;
+}
+
+static inline void iop_desc_set_block_fill_val(struct iop_adma_desc_slot *desc,
+ u32 val)
+{
+ struct iop3xx_desc_aau *hw_desc = desc->hw_desc;
+ hw_desc->src[0] = val;
+}
+
+static inline enum sum_check_flags
+iop_desc_get_zero_result(struct iop_adma_desc_slot *desc)
+{
+ struct iop3xx_desc_aau *hw_desc = desc->hw_desc;
+ struct iop3xx_aau_desc_ctrl desc_ctrl = hw_desc->desc_ctrl_field;
+
+ iop_paranoia(!(desc_ctrl.tx_complete && desc_ctrl.zero_result_en));
+ return desc_ctrl.zero_result_err << SUM_CHECK_P;
+}
+
+static inline void iop_chan_append(struct iop_adma_chan *chan)
+{
+ u32 dma_chan_ctrl;
+
+ dma_chan_ctrl = __raw_readl(DMA_CCR(chan));
+ dma_chan_ctrl |= 0x2;
+ __raw_writel(dma_chan_ctrl, DMA_CCR(chan));
+}
+
+static inline u32 iop_chan_get_status(struct iop_adma_chan *chan)
+{
+ return __raw_readl(DMA_CSR(chan));
+}
+
+static inline void iop_chan_disable(struct iop_adma_chan *chan)
+{
+ u32 dma_chan_ctrl = __raw_readl(DMA_CCR(chan));
+ dma_chan_ctrl &= ~1;
+ __raw_writel(dma_chan_ctrl, DMA_CCR(chan));
+}
+
+static inline void iop_chan_enable(struct iop_adma_chan *chan)
+{
+ u32 dma_chan_ctrl = __raw_readl(DMA_CCR(chan));
+
+ dma_chan_ctrl |= 1;
+ __raw_writel(dma_chan_ctrl, DMA_CCR(chan));
+}
+
+static inline void iop_adma_device_clear_eot_status(struct iop_adma_chan *chan)
+{
+ u32 status = __raw_readl(DMA_CSR(chan));
+ status &= (1 << 9);
+ __raw_writel(status, DMA_CSR(chan));
+}
+
+static inline void iop_adma_device_clear_eoc_status(struct iop_adma_chan *chan)
+{
+ u32 status = __raw_readl(DMA_CSR(chan));
+ status &= (1 << 8);
+ __raw_writel(status, DMA_CSR(chan));
+}
+
+static inline void iop_adma_device_clear_err_status(struct iop_adma_chan *chan)
+{
+ u32 status = __raw_readl(DMA_CSR(chan));
+
+ switch (chan->device->id) {
+ case DMA0_ID:
+ case DMA1_ID:
+ status &= (1 << 5) | (1 << 3) | (1 << 2) | (1 << 1);
+ break;
+ case AAU_ID:
+ status &= (1 << 5);
+ break;
+ default:
+ BUG();
+ }
+
+ __raw_writel(status, DMA_CSR(chan));
+}
+
+static inline int
+iop_is_err_int_parity(unsigned long status, struct iop_adma_chan *chan)
+{
+ return 0;
+}
+
+static inline int
+iop_is_err_mcu_abort(unsigned long status, struct iop_adma_chan *chan)
+{
+ return 0;
+}
+
+static inline int
+iop_is_err_int_tabort(unsigned long status, struct iop_adma_chan *chan)
+{
+ return 0;
+}
+
+static inline int
+iop_is_err_int_mabort(unsigned long status, struct iop_adma_chan *chan)
+{
+ return test_bit(5, &status);
+}
+
+static inline int
+iop_is_err_pci_tabort(unsigned long status, struct iop_adma_chan *chan)
+{
+ switch (chan->device->id) {
+ case DMA0_ID:
+ case DMA1_ID:
+ return test_bit(2, &status);
+ default:
+ return 0;
+ }
+}
+
+static inline int
+iop_is_err_pci_mabort(unsigned long status, struct iop_adma_chan *chan)
+{
+ switch (chan->device->id) {
+ case DMA0_ID:
+ case DMA1_ID:
+ return test_bit(3, &status);
+ default:
+ return 0;
+ }
+}
+
+static inline int
+iop_is_err_split_tx(unsigned long status, struct iop_adma_chan *chan)
+{
+ switch (chan->device->id) {
+ case DMA0_ID:
+ case DMA1_ID:
+ return test_bit(1, &status);
+ default:
+ return 0;
+ }
+}
+#endif /* _ADMA_H */
diff --git a/drivers/dma/mcf-edma.c b/drivers/dma/mcf-edma.c
index 7de54b2fafdb..e15bd15a9ef6 100644
--- a/drivers/dma/mcf-edma.c
+++ b/drivers/dma/mcf-edma.c
@@ -164,6 +164,11 @@ static void mcf_edma_irq_free(struct platform_device *pdev,
free_irq(irq, mcf_edma);
}
+static struct fsl_edma_drvdata mcf_data = {
+ .version = v2,
+ .setup_irq = mcf_edma_irq_init,
+};
+
static int mcf_edma_probe(struct platform_device *pdev)
{
struct mcf_edma_platform_data *pdata;
@@ -187,8 +192,8 @@ static int mcf_edma_probe(struct platform_device *pdev)
mcf_edma->n_chans = chans;
- /* Set up version for ColdFire edma */
- mcf_edma->version = v2;
+ /* Set up drvdata for ColdFire edma */
+ mcf_edma->drvdata = &mcf_data;
mcf_edma->big_endian = 1;
if (!mcf_edma->n_chans) {
@@ -223,7 +228,7 @@ static int mcf_edma_probe(struct platform_device *pdev)
iowrite32(~0, regs->inth);
iowrite32(~0, regs->intl);
- ret = mcf_edma_irq_init(pdev, mcf_edma);
+ ret = mcf_edma->drvdata->setup_irq(pdev, mcf_edma);
if (ret)
return ret;
diff --git a/drivers/dma/mediatek/Kconfig b/drivers/dma/mediatek/Kconfig
index 7411eb3d419e..1ad63ddc292d 100644
--- a/drivers/dma/mediatek/Kconfig
+++ b/drivers/dma/mediatek/Kconfig
@@ -25,3 +25,14 @@ config MTK_CQDMA
This controller provides the channels which is dedicated to
memory-to-memory transfer to offload from CPU.
+
+config MTK_UART_APDMA
+ tristate "MediaTek SoCs APDMA support for UART"
+ depends on OF && SERIAL_8250_MT6577
+ select DMA_ENGINE
+ select DMA_VIRTUAL_CHANNELS
+ help
+ Support for the UART DMA engine found on MediaTek MTK SoCs.
+ When SERIAL_8250_MT6577 is enabled, and if you want to use DMA,
+ you can enable the config. The DMA engine can only be used
+ with MediaTek SoCs.
diff --git a/drivers/dma/mediatek/Makefile b/drivers/dma/mediatek/Makefile
index 13b144594510..5ba39a5edc13 100644
--- a/drivers/dma/mediatek/Makefile
+++ b/drivers/dma/mediatek/Makefile
@@ -1,3 +1,4 @@
# SPDX-License-Identifier: GPL-2.0-only
+obj-$(CONFIG_MTK_UART_APDMA) += mtk-uart-apdma.o
obj-$(CONFIG_MTK_HSDMA) += mtk-hsdma.o
obj-$(CONFIG_MTK_CQDMA) += mtk-cqdma.o
diff --git a/drivers/dma/mediatek/mtk-uart-apdma.c b/drivers/dma/mediatek/mtk-uart-apdma.c
new file mode 100644
index 000000000000..f40051d6aecb
--- /dev/null
+++ b/drivers/dma/mediatek/mtk-uart-apdma.c
@@ -0,0 +1,664 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * MediaTek UART APDMA driver.
+ *
+ * Copyright (c) 2019 MediaTek Inc.
+ * Author: Long Cheng <long.cheng@mediatek.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/of_dma.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#include "../virt-dma.h"
+
+/* The default number of virtual channel */
+#define MTK_UART_APDMA_NR_VCHANS 8
+
+#define VFF_EN_B BIT(0)
+#define VFF_STOP_B BIT(0)
+#define VFF_FLUSH_B BIT(0)
+#define VFF_4G_EN_B BIT(0)
+/* rx valid size >= vff thre */
+#define VFF_RX_INT_EN_B (BIT(0) | BIT(1))
+/* tx left size >= vff thre */
+#define VFF_TX_INT_EN_B BIT(0)
+#define VFF_WARM_RST_B BIT(0)
+#define VFF_RX_INT_CLR_B (BIT(0) | BIT(1))
+#define VFF_TX_INT_CLR_B 0
+#define VFF_STOP_CLR_B 0
+#define VFF_EN_CLR_B 0
+#define VFF_INT_EN_CLR_B 0
+#define VFF_4G_SUPPORT_CLR_B 0
+
+/*
+ * interrupt trigger level for tx
+ * if threshold is n, no polling is required to start tx.
+ * otherwise need polling VFF_FLUSH.
+ */
+#define VFF_TX_THRE(n) (n)
+/* interrupt trigger level for rx */
+#define VFF_RX_THRE(n) ((n) * 3 / 4)
+
+#define VFF_RING_SIZE 0xffff
+/* invert this bit when wrap ring head again */
+#define VFF_RING_WRAP 0x10000
+
+#define VFF_INT_FLAG 0x00
+#define VFF_INT_EN 0x04
+#define VFF_EN 0x08
+#define VFF_RST 0x0c
+#define VFF_STOP 0x10
+#define VFF_FLUSH 0x14
+#define VFF_ADDR 0x1c
+#define VFF_LEN 0x24
+#define VFF_THRE 0x28
+#define VFF_WPT 0x2c
+#define VFF_RPT 0x30
+/* TX: the buffer size HW can read. RX: the buffer size SW can read. */
+#define VFF_VALID_SIZE 0x3c
+/* TX: the buffer size SW can write. RX: the buffer size HW can write. */
+#define VFF_LEFT_SIZE 0x40
+#define VFF_DEBUG_STATUS 0x50
+#define VFF_4G_SUPPORT 0x54
+
+struct mtk_uart_apdmadev {
+ struct dma_device ddev;
+ struct clk *clk;
+ bool support_33bits;
+ unsigned int dma_requests;
+};
+
+struct mtk_uart_apdma_desc {
+ struct virt_dma_desc vd;
+
+ dma_addr_t addr;
+ unsigned int avail_len;
+};
+
+struct mtk_chan {
+ struct virt_dma_chan vc;
+ struct dma_slave_config cfg;
+ struct mtk_uart_apdma_desc *desc;
+ enum dma_transfer_direction dir;
+
+ void __iomem *base;
+ unsigned int irq;
+
+ unsigned int rx_status;
+};
+
+static inline struct mtk_uart_apdmadev *
+to_mtk_uart_apdma_dev(struct dma_device *d)
+{
+ return container_of(d, struct mtk_uart_apdmadev, ddev);
+}
+
+static inline struct mtk_chan *to_mtk_uart_apdma_chan(struct dma_chan *c)
+{
+ return container_of(c, struct mtk_chan, vc.chan);
+}
+
+static inline struct mtk_uart_apdma_desc *to_mtk_uart_apdma_desc
+ (struct dma_async_tx_descriptor *t)
+{
+ return container_of(t, struct mtk_uart_apdma_desc, vd.tx);
+}
+
+static void mtk_uart_apdma_write(struct mtk_chan *c,
+ unsigned int reg, unsigned int val)
+{
+ writel(val, c->base + reg);
+}
+
+static unsigned int mtk_uart_apdma_read(struct mtk_chan *c, unsigned int reg)
+{
+ return readl(c->base + reg);
+}
+
+static void mtk_uart_apdma_desc_free(struct virt_dma_desc *vd)
+{
+ struct dma_chan *chan = vd->tx.chan;
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+
+ kfree(c->desc);
+}
+
+static void mtk_uart_apdma_start_tx(struct mtk_chan *c)
+{
+ struct mtk_uart_apdmadev *mtkd =
+ to_mtk_uart_apdma_dev(c->vc.chan.device);
+ struct mtk_uart_apdma_desc *d = c->desc;
+ unsigned int wpt, vff_sz;
+
+ vff_sz = c->cfg.dst_port_window_size;
+ if (!mtk_uart_apdma_read(c, VFF_LEN)) {
+ mtk_uart_apdma_write(c, VFF_ADDR, d->addr);
+ mtk_uart_apdma_write(c, VFF_LEN, vff_sz);
+ mtk_uart_apdma_write(c, VFF_THRE, VFF_TX_THRE(vff_sz));
+ mtk_uart_apdma_write(c, VFF_WPT, 0);
+ mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_TX_INT_CLR_B);
+
+ if (mtkd->support_33bits)
+ mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_EN_B);
+ }
+
+ mtk_uart_apdma_write(c, VFF_EN, VFF_EN_B);
+ if (mtk_uart_apdma_read(c, VFF_EN) != VFF_EN_B)
+ dev_err(c->vc.chan.device->dev, "Enable TX fail\n");
+
+ if (!mtk_uart_apdma_read(c, VFF_LEFT_SIZE)) {
+ mtk_uart_apdma_write(c, VFF_INT_EN, VFF_TX_INT_EN_B);
+ return;
+ }
+
+ wpt = mtk_uart_apdma_read(c, VFF_WPT);
+
+ wpt += c->desc->avail_len;
+ if ((wpt & VFF_RING_SIZE) == vff_sz)
+ wpt = (wpt & VFF_RING_WRAP) ^ VFF_RING_WRAP;
+
+ /* Let DMA start moving data */
+ mtk_uart_apdma_write(c, VFF_WPT, wpt);
+
+ /* HW auto set to 0 when left size >= threshold */
+ mtk_uart_apdma_write(c, VFF_INT_EN, VFF_TX_INT_EN_B);
+ if (!mtk_uart_apdma_read(c, VFF_FLUSH))
+ mtk_uart_apdma_write(c, VFF_FLUSH, VFF_FLUSH_B);
+}
+
+static void mtk_uart_apdma_start_rx(struct mtk_chan *c)
+{
+ struct mtk_uart_apdmadev *mtkd =
+ to_mtk_uart_apdma_dev(c->vc.chan.device);
+ struct mtk_uart_apdma_desc *d = c->desc;
+ unsigned int vff_sz;
+
+ vff_sz = c->cfg.src_port_window_size;
+ if (!mtk_uart_apdma_read(c, VFF_LEN)) {
+ mtk_uart_apdma_write(c, VFF_ADDR, d->addr);
+ mtk_uart_apdma_write(c, VFF_LEN, vff_sz);
+ mtk_uart_apdma_write(c, VFF_THRE, VFF_RX_THRE(vff_sz));
+ mtk_uart_apdma_write(c, VFF_RPT, 0);
+ mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_RX_INT_CLR_B);
+
+ if (mtkd->support_33bits)
+ mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_EN_B);
+ }
+
+ mtk_uart_apdma_write(c, VFF_INT_EN, VFF_RX_INT_EN_B);
+ mtk_uart_apdma_write(c, VFF_EN, VFF_EN_B);
+ if (mtk_uart_apdma_read(c, VFF_EN) != VFF_EN_B)
+ dev_err(c->vc.chan.device->dev, "Enable RX fail\n");
+}
+
+static void mtk_uart_apdma_tx_handler(struct mtk_chan *c)
+{
+ struct mtk_uart_apdma_desc *d = c->desc;
+
+ mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_TX_INT_CLR_B);
+ mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B);
+ mtk_uart_apdma_write(c, VFF_EN, VFF_EN_CLR_B);
+
+ list_del(&d->vd.node);
+ vchan_cookie_complete(&d->vd);
+}
+
+static void mtk_uart_apdma_rx_handler(struct mtk_chan *c)
+{
+ struct mtk_uart_apdma_desc *d = c->desc;
+ unsigned int len, wg, rg;
+ int cnt;
+
+ mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_RX_INT_CLR_B);
+
+ if (!mtk_uart_apdma_read(c, VFF_VALID_SIZE))
+ return;
+
+ mtk_uart_apdma_write(c, VFF_EN, VFF_EN_CLR_B);
+ mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B);
+
+ len = c->cfg.src_port_window_size;
+ rg = mtk_uart_apdma_read(c, VFF_RPT);
+ wg = mtk_uart_apdma_read(c, VFF_WPT);
+ cnt = (wg & VFF_RING_SIZE) - (rg & VFF_RING_SIZE);
+
+ /*
+ * The buffer is ring buffer. If wrap bit different,
+ * represents the start of the next cycle for WPT
+ */
+ if ((rg ^ wg) & VFF_RING_WRAP)
+ cnt += len;
+
+ c->rx_status = d->avail_len - cnt;
+ mtk_uart_apdma_write(c, VFF_RPT, wg);
+
+ list_del(&d->vd.node);
+ vchan_cookie_complete(&d->vd);
+}
+
+static irqreturn_t mtk_uart_apdma_irq_handler(int irq, void *dev_id)
+{
+ struct dma_chan *chan = (struct dma_chan *)dev_id;
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+ unsigned long flags;
+
+ spin_lock_irqsave(&c->vc.lock, flags);
+ if (c->dir == DMA_DEV_TO_MEM)
+ mtk_uart_apdma_rx_handler(c);
+ else if (c->dir == DMA_MEM_TO_DEV)
+ mtk_uart_apdma_tx_handler(c);
+ spin_unlock_irqrestore(&c->vc.lock, flags);
+
+ return IRQ_HANDLED;
+}
+
+static int mtk_uart_apdma_alloc_chan_resources(struct dma_chan *chan)
+{
+ struct mtk_uart_apdmadev *mtkd = to_mtk_uart_apdma_dev(chan->device);
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+ unsigned int status;
+ int ret;
+
+ ret = pm_runtime_get_sync(mtkd->ddev.dev);
+ if (ret < 0) {
+ pm_runtime_put_noidle(chan->device->dev);
+ return ret;
+ }
+
+ mtk_uart_apdma_write(c, VFF_ADDR, 0);
+ mtk_uart_apdma_write(c, VFF_THRE, 0);
+ mtk_uart_apdma_write(c, VFF_LEN, 0);
+ mtk_uart_apdma_write(c, VFF_RST, VFF_WARM_RST_B);
+
+ ret = readx_poll_timeout(readl, c->base + VFF_EN,
+ status, !status, 10, 100);
+ if (ret)
+ return ret;
+
+ ret = request_irq(c->irq, mtk_uart_apdma_irq_handler,
+ IRQF_TRIGGER_NONE, KBUILD_MODNAME, chan);
+ if (ret < 0) {
+ dev_err(chan->device->dev, "Can't request dma IRQ\n");
+ return -EINVAL;
+ }
+
+ if (mtkd->support_33bits)
+ mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_SUPPORT_CLR_B);
+
+ return ret;
+}
+
+static void mtk_uart_apdma_free_chan_resources(struct dma_chan *chan)
+{
+ struct mtk_uart_apdmadev *mtkd = to_mtk_uart_apdma_dev(chan->device);
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+
+ free_irq(c->irq, chan);
+
+ tasklet_kill(&c->vc.task);
+
+ vchan_free_chan_resources(&c->vc);
+
+ pm_runtime_put_sync(mtkd->ddev.dev);
+}
+
+static enum dma_status mtk_uart_apdma_tx_status(struct dma_chan *chan,
+ dma_cookie_t cookie,
+ struct dma_tx_state *txstate)
+{
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+ enum dma_status ret;
+
+ ret = dma_cookie_status(chan, cookie, txstate);
+ if (!txstate)
+ return ret;
+
+ dma_set_residue(txstate, c->rx_status);
+
+ return ret;
+}
+
+/*
+ * dmaengine_prep_slave_single will call the function. and sglen is 1.
+ * 8250 uart using one ring buffer, and deal with one sg.
+ */
+static struct dma_async_tx_descriptor *mtk_uart_apdma_prep_slave_sg
+ (struct dma_chan *chan, struct scatterlist *sgl,
+ unsigned int sglen, enum dma_transfer_direction dir,
+ unsigned long tx_flags, void *context)
+{
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+ struct mtk_uart_apdma_desc *d;
+
+ if (!is_slave_direction(dir) || sglen != 1)
+ return NULL;
+
+ /* Now allocate and setup the descriptor */
+ d = kzalloc(sizeof(*d), GFP_ATOMIC);
+ if (!d)
+ return NULL;
+
+ d->avail_len = sg_dma_len(sgl);
+ d->addr = sg_dma_address(sgl);
+ c->dir = dir;
+
+ return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
+}
+
+static void mtk_uart_apdma_issue_pending(struct dma_chan *chan)
+{
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+ struct virt_dma_desc *vd;
+ unsigned long flags;
+
+ spin_lock_irqsave(&c->vc.lock, flags);
+ if (vchan_issue_pending(&c->vc)) {
+ vd = vchan_next_desc(&c->vc);
+ c->desc = to_mtk_uart_apdma_desc(&vd->tx);
+
+ if (c->dir == DMA_DEV_TO_MEM)
+ mtk_uart_apdma_start_rx(c);
+ else if (c->dir == DMA_MEM_TO_DEV)
+ mtk_uart_apdma_start_tx(c);
+ }
+
+ spin_unlock_irqrestore(&c->vc.lock, flags);
+}
+
+static int mtk_uart_apdma_slave_config(struct dma_chan *chan,
+ struct dma_slave_config *config)
+{
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+
+ memcpy(&c->cfg, config, sizeof(*config));
+
+ return 0;
+}
+
+static int mtk_uart_apdma_terminate_all(struct dma_chan *chan)
+{
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+ unsigned long flags;
+ unsigned int status;
+ LIST_HEAD(head);
+ int ret;
+
+ mtk_uart_apdma_write(c, VFF_FLUSH, VFF_FLUSH_B);
+
+ ret = readx_poll_timeout(readl, c->base + VFF_FLUSH,
+ status, status != VFF_FLUSH_B, 10, 100);
+ if (ret)
+ dev_err(c->vc.chan.device->dev, "flush: fail, status=0x%x\n",
+ mtk_uart_apdma_read(c, VFF_DEBUG_STATUS));
+
+ /*
+ * Stop need 3 steps.
+ * 1. set stop to 1
+ * 2. wait en to 0
+ * 3. set stop as 0
+ */
+ mtk_uart_apdma_write(c, VFF_STOP, VFF_STOP_B);
+ ret = readx_poll_timeout(readl, c->base + VFF_EN,
+ status, !status, 10, 100);
+ if (ret)
+ dev_err(c->vc.chan.device->dev, "stop: fail, status=0x%x\n",
+ mtk_uart_apdma_read(c, VFF_DEBUG_STATUS));
+
+ mtk_uart_apdma_write(c, VFF_STOP, VFF_STOP_CLR_B);
+ mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B);
+
+ if (c->dir == DMA_DEV_TO_MEM)
+ mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_RX_INT_CLR_B);
+ else if (c->dir == DMA_MEM_TO_DEV)
+ mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_TX_INT_CLR_B);
+
+ synchronize_irq(c->irq);
+
+ spin_lock_irqsave(&c->vc.lock, flags);
+ vchan_get_all_descriptors(&c->vc, &head);
+ vchan_dma_desc_free_list(&c->vc, &head);
+ spin_unlock_irqrestore(&c->vc.lock, flags);
+
+ return 0;
+}
+
+static int mtk_uart_apdma_device_pause(struct dma_chan *chan)
+{
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+ unsigned long flags;
+
+ spin_lock_irqsave(&c->vc.lock, flags);
+
+ mtk_uart_apdma_write(c, VFF_EN, VFF_EN_CLR_B);
+ mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B);
+
+ synchronize_irq(c->irq);
+
+ spin_unlock_irqrestore(&c->vc.lock, flags);
+
+ return 0;
+}
+
+static void mtk_uart_apdma_free(struct mtk_uart_apdmadev *mtkd)
+{
+ while (!list_empty(&mtkd->ddev.channels)) {
+ struct mtk_chan *c = list_first_entry(&mtkd->ddev.channels,
+ struct mtk_chan, vc.chan.device_node);
+
+ list_del(&c->vc.chan.device_node);
+ tasklet_kill(&c->vc.task);
+ }
+}
+
+static const struct of_device_id mtk_uart_apdma_match[] = {
+ { .compatible = "mediatek,mt6577-uart-dma", },
+ { /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, mtk_uart_apdma_match);
+
+static int mtk_uart_apdma_probe(struct platform_device *pdev)
+{
+ struct device_node *np = pdev->dev.of_node;
+ struct mtk_uart_apdmadev *mtkd;
+ int bit_mask = 32, rc;
+ struct resource *res;
+ struct mtk_chan *c;
+ unsigned int i;
+
+ mtkd = devm_kzalloc(&pdev->dev, sizeof(*mtkd), GFP_KERNEL);
+ if (!mtkd)
+ return -ENOMEM;
+
+ mtkd->clk = devm_clk_get(&pdev->dev, NULL);
+ if (IS_ERR(mtkd->clk)) {
+ dev_err(&pdev->dev, "No clock specified\n");
+ rc = PTR_ERR(mtkd->clk);
+ return rc;
+ }
+
+ if (of_property_read_bool(np, "mediatek,dma-33bits"))
+ mtkd->support_33bits = true;
+
+ if (mtkd->support_33bits)
+ bit_mask = 33;
+
+ rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(bit_mask));
+ if (rc)
+ return rc;
+
+ dma_cap_set(DMA_SLAVE, mtkd->ddev.cap_mask);
+ mtkd->ddev.device_alloc_chan_resources =
+ mtk_uart_apdma_alloc_chan_resources;
+ mtkd->ddev.device_free_chan_resources =
+ mtk_uart_apdma_free_chan_resources;
+ mtkd->ddev.device_tx_status = mtk_uart_apdma_tx_status;
+ mtkd->ddev.device_issue_pending = mtk_uart_apdma_issue_pending;
+ mtkd->ddev.device_prep_slave_sg = mtk_uart_apdma_prep_slave_sg;
+ mtkd->ddev.device_config = mtk_uart_apdma_slave_config;
+ mtkd->ddev.device_pause = mtk_uart_apdma_device_pause;
+ mtkd->ddev.device_terminate_all = mtk_uart_apdma_terminate_all;
+ mtkd->ddev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE);
+ mtkd->ddev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE);
+ mtkd->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
+ mtkd->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
+ mtkd->ddev.dev = &pdev->dev;
+ INIT_LIST_HEAD(&mtkd->ddev.channels);
+
+ mtkd->dma_requests = MTK_UART_APDMA_NR_VCHANS;
+ if (of_property_read_u32(np, "dma-requests", &mtkd->dma_requests)) {
+ dev_info(&pdev->dev,
+ "Using %u as missing dma-requests property\n",
+ MTK_UART_APDMA_NR_VCHANS);
+ }
+
+ for (i = 0; i < mtkd->dma_requests; i++) {
+ c = devm_kzalloc(mtkd->ddev.dev, sizeof(*c), GFP_KERNEL);
+ if (!c) {
+ rc = -ENODEV;
+ goto err_no_dma;
+ }
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, i);
+ if (!res) {
+ rc = -ENODEV;
+ goto err_no_dma;
+ }
+
+ c->base = devm_ioremap_resource(&pdev->dev, res);
+ if (IS_ERR(c->base)) {
+ rc = PTR_ERR(c->base);
+ goto err_no_dma;
+ }
+ c->vc.desc_free = mtk_uart_apdma_desc_free;
+ vchan_init(&c->vc, &mtkd->ddev);
+
+ rc = platform_get_irq(pdev, i);
+ if (rc < 0)
+ goto err_no_dma;
+ c->irq = rc;
+ }
+
+ pm_runtime_enable(&pdev->dev);
+ pm_runtime_set_active(&pdev->dev);
+
+ rc = dma_async_device_register(&mtkd->ddev);
+ if (rc)
+ goto rpm_disable;
+
+ platform_set_drvdata(pdev, mtkd);
+
+ /* Device-tree DMA controller registration */
+ rc = of_dma_controller_register(np, of_dma_xlate_by_chan_id, mtkd);
+ if (rc)
+ goto dma_remove;
+
+ return rc;
+
+dma_remove:
+ dma_async_device_unregister(&mtkd->ddev);
+rpm_disable:
+ pm_runtime_disable(&pdev->dev);
+err_no_dma:
+ mtk_uart_apdma_free(mtkd);
+ return rc;
+}
+
+static int mtk_uart_apdma_remove(struct platform_device *pdev)
+{
+ struct mtk_uart_apdmadev *mtkd = platform_get_drvdata(pdev);
+
+ of_dma_controller_free(pdev->dev.of_node);
+
+ mtk_uart_apdma_free(mtkd);
+
+ dma_async_device_unregister(&mtkd->ddev);
+
+ pm_runtime_disable(&pdev->dev);
+
+ return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int mtk_uart_apdma_suspend(struct device *dev)
+{
+ struct mtk_uart_apdmadev *mtkd = dev_get_drvdata(dev);
+
+ if (!pm_runtime_suspended(dev))
+ clk_disable_unprepare(mtkd->clk);
+
+ return 0;
+}
+
+static int mtk_uart_apdma_resume(struct device *dev)
+{
+ int ret;
+ struct mtk_uart_apdmadev *mtkd = dev_get_drvdata(dev);
+
+ if (!pm_runtime_suspended(dev)) {
+ ret = clk_prepare_enable(mtkd->clk);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+#endif /* CONFIG_PM_SLEEP */
+
+#ifdef CONFIG_PM
+static int mtk_uart_apdma_runtime_suspend(struct device *dev)
+{
+ struct mtk_uart_apdmadev *mtkd = dev_get_drvdata(dev);
+
+ clk_disable_unprepare(mtkd->clk);
+
+ return 0;
+}
+
+static int mtk_uart_apdma_runtime_resume(struct device *dev)
+{
+ int ret;
+ struct mtk_uart_apdmadev *mtkd = dev_get_drvdata(dev);
+
+ ret = clk_prepare_enable(mtkd->clk);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+#endif /* CONFIG_PM */
+
+static const struct dev_pm_ops mtk_uart_apdma_pm_ops = {
+ SET_SYSTEM_SLEEP_PM_OPS(mtk_uart_apdma_suspend, mtk_uart_apdma_resume)
+ SET_RUNTIME_PM_OPS(mtk_uart_apdma_runtime_suspend,
+ mtk_uart_apdma_runtime_resume, NULL)
+};
+
+static struct platform_driver mtk_uart_apdma_driver = {
+ .probe = mtk_uart_apdma_probe,
+ .remove = mtk_uart_apdma_remove,
+ .driver = {
+ .name = KBUILD_MODNAME,
+ .pm = &mtk_uart_apdma_pm_ops,
+ .of_match_table = of_match_ptr(mtk_uart_apdma_match),
+ },
+};
+
+module_platform_driver(mtk_uart_apdma_driver);
+
+MODULE_DESCRIPTION("MediaTek UART APDMA Controller Driver");
+MODULE_AUTHOR("Long Cheng <long.cheng@mediatek.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/dma/mic_x100_dma.c b/drivers/dma/mic_x100_dma.c
index 730a18d0c6d6..fea8608a7810 100644
--- a/drivers/dma/mic_x100_dma.c
+++ b/drivers/dma/mic_x100_dma.c
@@ -717,10 +717,8 @@ static int mic_dma_driver_probe(struct mbus_device *mbdev)
if (mic_dma_dbg) {
mic_dma_dev->dbg_dir = debugfs_create_dir(dev_name(&mbdev->dev),
mic_dma_dbg);
- if (mic_dma_dev->dbg_dir)
- debugfs_create_file("mic_dma_reg", 0444,
- mic_dma_dev->dbg_dir, mic_dma_dev,
- &mic_dma_reg_fops);
+ debugfs_create_file("mic_dma_reg", 0444, mic_dma_dev->dbg_dir,
+ mic_dma_dev, &mic_dma_reg_fops);
}
return 0;
}
diff --git a/drivers/dma/mmp_tdma.c b/drivers/dma/mmp_tdma.c
index bb3ccbf90a31..e7d1e12bf464 100644
--- a/drivers/dma/mmp_tdma.c
+++ b/drivers/dma/mmp_tdma.c
@@ -582,18 +582,12 @@ static int mmp_tdma_chan_init(struct mmp_tdma_device *tdev,
}
struct mmp_tdma_filter_param {
- struct device_node *of_node;
unsigned int chan_id;
};
static bool mmp_tdma_filter_fn(struct dma_chan *chan, void *fn_param)
{
struct mmp_tdma_filter_param *param = fn_param;
- struct mmp_tdma_chan *tdmac = to_mmp_tdma_chan(chan);
- struct dma_device *pdma_device = tdmac->chan.device;
-
- if (pdma_device->dev->of_node != param->of_node)
- return false;
if (chan->chan_id != param->chan_id)
return false;
@@ -611,13 +605,13 @@ static struct dma_chan *mmp_tdma_xlate(struct of_phandle_args *dma_spec,
if (dma_spec->args_count != 1)
return NULL;
- param.of_node = ofdma->of_node;
param.chan_id = dma_spec->args[0];
if (param.chan_id >= TDMA_CHANNEL_NUM)
return NULL;
- return dma_request_channel(mask, mmp_tdma_filter_fn, &param);
+ return __dma_request_channel(&mask, mmp_tdma_filter_fn, &param,
+ ofdma->of_node);
}
static const struct of_device_id mmp_tdma_dt_ids[] = {
diff --git a/drivers/dma/mv_xor_v2.c b/drivers/dma/mv_xor_v2.c
index fa5dab481203..e3850f04f676 100644
--- a/drivers/dma/mv_xor_v2.c
+++ b/drivers/dma/mv_xor_v2.c
@@ -33,7 +33,6 @@
#define MV_XOR_V2_DMA_IMSG_CDAT_OFF 0x014
#define MV_XOR_V2_DMA_IMSG_THRD_OFF 0x018
#define MV_XOR_V2_DMA_IMSG_THRD_MASK 0x7FFF
-#define MV_XOR_V2_DMA_IMSG_THRD_SHIFT 0x0
#define MV_XOR_V2_DMA_IMSG_TIMER_EN BIT(18)
#define MV_XOR_V2_DMA_DESQ_AWATTR_OFF 0x01C
/* Same flags as MV_XOR_V2_DMA_DESQ_ARATTR_OFF */
@@ -50,7 +49,6 @@
#define MV_XOR_V2_DMA_DESQ_ADD_OFF 0x808
#define MV_XOR_V2_DMA_IMSG_TMOT 0x810
#define MV_XOR_V2_DMA_IMSG_TIMER_THRD_MASK 0x1FFF
-#define MV_XOR_V2_DMA_IMSG_TIMER_THRD_SHIFT 0
/* XOR Global registers */
#define MV_XOR_V2_GLOB_BW_CTRL 0x4
@@ -261,16 +259,15 @@ void mv_xor_v2_enable_imsg_thrd(struct mv_xor_v2_device *xor_dev)
/* Configure threshold of number of descriptors, and enable timer */
reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_THRD_OFF);
- reg &= (~MV_XOR_V2_DMA_IMSG_THRD_MASK << MV_XOR_V2_DMA_IMSG_THRD_SHIFT);
- reg |= (MV_XOR_V2_DONE_IMSG_THRD << MV_XOR_V2_DMA_IMSG_THRD_SHIFT);
+ reg &= ~MV_XOR_V2_DMA_IMSG_THRD_MASK;
+ reg |= MV_XOR_V2_DONE_IMSG_THRD;
reg |= MV_XOR_V2_DMA_IMSG_TIMER_EN;
writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_THRD_OFF);
/* Configure Timer Threshold */
reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_TMOT);
- reg &= (~MV_XOR_V2_DMA_IMSG_TIMER_THRD_MASK <<
- MV_XOR_V2_DMA_IMSG_TIMER_THRD_SHIFT);
- reg |= (MV_XOR_V2_TIMER_THRD << MV_XOR_V2_DMA_IMSG_TIMER_THRD_SHIFT);
+ reg &= ~MV_XOR_V2_DMA_IMSG_TIMER_THRD_MASK;
+ reg |= MV_XOR_V2_TIMER_THRD;
writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_TMOT);
}
diff --git a/drivers/dma/mxs-dma.c b/drivers/dma/mxs-dma.c
index 22cc7f68ef6e..3039bba0e4d5 100644
--- a/drivers/dma/mxs-dma.c
+++ b/drivers/dma/mxs-dma.c
@@ -24,6 +24,7 @@
#include <linux/of_device.h>
#include <linux/of_dma.h>
#include <linux/list.h>
+#include <linux/dma/mxs-dma.h>
#include <asm/irq.h>
@@ -77,6 +78,7 @@
#define BM_CCW_COMMAND (3 << 0)
#define CCW_CHAIN (1 << 2)
#define CCW_IRQ (1 << 3)
+#define CCW_WAIT4RDY (1 << 5)
#define CCW_DEC_SEM (1 << 6)
#define CCW_WAIT4END (1 << 7)
#define CCW_HALT_ON_TERM (1 << 8)
@@ -477,16 +479,16 @@ static void mxs_dma_free_chan_resources(struct dma_chan *chan)
* ......
* ->device_prep_slave_sg(0);
* ......
- * ->device_prep_slave_sg(DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+ * ->device_prep_slave_sg(DMA_CTRL_ACK);
* ......
* [3] If there are more than two DMA commands in the DMA chain, the code
* should be:
* ......
* ->device_prep_slave_sg(0); // First
* ......
- * ->device_prep_slave_sg(DMA_PREP_INTERRUPT [| DMA_CTRL_ACK]);
+ * ->device_prep_slave_sg(DMA_CTRL_ACK]);
* ......
- * ->device_prep_slave_sg(DMA_PREP_INTERRUPT | DMA_CTRL_ACK); // Last
+ * ->device_prep_slave_sg(DMA_CTRL_ACK); // Last
* ......
*/
static struct dma_async_tx_descriptor *mxs_dma_prep_slave_sg(
@@ -500,13 +502,12 @@ static struct dma_async_tx_descriptor *mxs_dma_prep_slave_sg(
struct scatterlist *sg;
u32 i, j;
u32 *pio;
- bool append = flags & DMA_PREP_INTERRUPT;
- int idx = append ? mxs_chan->desc_count : 0;
+ int idx = 0;
- if (mxs_chan->status == DMA_IN_PROGRESS && !append)
- return NULL;
+ if (mxs_chan->status == DMA_IN_PROGRESS)
+ idx = mxs_chan->desc_count;
- if (sg_len + (append ? idx : 0) > NUM_CCW) {
+ if (sg_len + idx > NUM_CCW) {
dev_err(mxs_dma->dma_device.dev,
"maximum number of sg exceeded: %d > %d\n",
sg_len, NUM_CCW);
@@ -520,7 +521,7 @@ static struct dma_async_tx_descriptor *mxs_dma_prep_slave_sg(
* If the sg is prepared with append flag set, the sg
* will be appended to the last prepared sg.
*/
- if (append) {
+ if (idx) {
BUG_ON(idx < 1);
ccw = &mxs_chan->ccw[idx - 1];
ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * idx;
@@ -541,12 +542,14 @@ static struct dma_async_tx_descriptor *mxs_dma_prep_slave_sg(
ccw->bits = 0;
ccw->bits |= CCW_IRQ;
ccw->bits |= CCW_DEC_SEM;
- if (flags & DMA_CTRL_ACK)
+ if (flags & MXS_DMA_CTRL_WAIT4END)
ccw->bits |= CCW_WAIT4END;
ccw->bits |= CCW_HALT_ON_TERM;
ccw->bits |= CCW_TERM_FLUSH;
ccw->bits |= BF_CCW(sg_len, PIO_NUM);
ccw->bits |= BF_CCW(MXS_DMA_CMD_NO_XFER, COMMAND);
+ if (flags & MXS_DMA_CTRL_WAIT4RDY)
+ ccw->bits |= CCW_WAIT4RDY;
} else {
for_each_sg(sgl, sg, sg_len, i) {
if (sg_dma_len(sg) > MAX_XFER_BYTES) {
@@ -573,7 +576,7 @@ static struct dma_async_tx_descriptor *mxs_dma_prep_slave_sg(
ccw->bits &= ~CCW_CHAIN;
ccw->bits |= CCW_IRQ;
ccw->bits |= CCW_DEC_SEM;
- if (flags & DMA_CTRL_ACK)
+ if (flags & MXS_DMA_CTRL_WAIT4END)
ccw->bits |= CCW_WAIT4END;
}
}
@@ -716,7 +719,6 @@ err_out:
}
struct mxs_dma_filter_param {
- struct device_node *of_node;
unsigned int chan_id;
};
@@ -727,9 +729,6 @@ static bool mxs_dma_filter_fn(struct dma_chan *chan, void *fn_param)
struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
int chan_irq;
- if (mxs_dma->dma_device.dev->of_node != param->of_node)
- return false;
-
if (chan->chan_id != param->chan_id)
return false;
@@ -752,13 +751,13 @@ static struct dma_chan *mxs_dma_xlate(struct of_phandle_args *dma_spec,
if (dma_spec->args_count != 1)
return NULL;
- param.of_node = ofdma->of_node;
param.chan_id = dma_spec->args[0];
if (param.chan_id >= mxs_dma->nr_channels)
return NULL;
- return dma_request_channel(mask, mxs_dma_filter_fn, &param);
+ return __dma_request_channel(&mask, mxs_dma_filter_fn, &param,
+ ofdma->of_node);
}
static int __init mxs_dma_probe(struct platform_device *pdev)
diff --git a/drivers/dma/of-dma.c b/drivers/dma/of-dma.c
index 1e4d9ef2aea1..c2d779daa4b5 100644
--- a/drivers/dma/of-dma.c
+++ b/drivers/dma/of-dma.c
@@ -313,8 +313,8 @@ struct dma_chan *of_dma_simple_xlate(struct of_phandle_args *dma_spec,
if (count != 1)
return NULL;
- return dma_request_channel(info->dma_cap, info->filter_fn,
- &dma_spec->args[0]);
+ return __dma_request_channel(&info->dma_cap, info->filter_fn,
+ &dma_spec->args[0], dma_spec->np);
}
EXPORT_SYMBOL_GPL(of_dma_simple_xlate);
diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c
index 56f9fabc99c4..6cce9ef61b29 100644
--- a/drivers/dma/pl330.c
+++ b/drivers/dma/pl330.c
@@ -25,6 +25,7 @@
#include <linux/err.h>
#include <linux/pm_runtime.h>
#include <linux/bug.h>
+#include <linux/reset.h>
#include "dmaengine.h"
#define PL330_MAX_CHAN 8
@@ -496,6 +497,9 @@ struct pl330_dmac {
unsigned int num_peripherals;
struct dma_pl330_chan *peripherals; /* keep at end */
int quirks;
+
+ struct reset_control *rstc;
+ struct reset_control *rstc_ocp;
};
static struct pl330_of_quirks {
@@ -1918,9 +1922,10 @@ static int dmac_alloc_resources(struct pl330_dmac *pl330)
if (ret) {
dev_err(pl330->ddma.dev, "%s:%d Can't to create channels for DMAC!\n",
__func__, __LINE__);
- dma_free_coherent(pl330->ddma.dev,
+ dma_free_attrs(pl330->ddma.dev,
chans * pl330->mcbufsz,
- pl330->mcode_cpu, pl330->mcode_bus);
+ pl330->mcode_cpu, pl330->mcode_bus,
+ DMA_ATTR_PRIVILEGED);
return ret;
}
@@ -1999,9 +2004,9 @@ static void pl330_del(struct pl330_dmac *pl330)
/* Free DMAC resources */
dmac_free_threads(pl330);
- dma_free_coherent(pl330->ddma.dev,
+ dma_free_attrs(pl330->ddma.dev,
pl330->pcfg.num_chan * pl330->mcbufsz, pl330->mcode_cpu,
- pl330->mcode_bus);
+ pl330->mcode_bus, DMA_ATTR_PRIVILEGED);
}
/* forward declaration */
@@ -3024,6 +3029,32 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id)
amba_set_drvdata(adev, pl330);
+ pl330->rstc = devm_reset_control_get_optional(&adev->dev, "dma");
+ if (IS_ERR(pl330->rstc)) {
+ if (PTR_ERR(pl330->rstc) != -EPROBE_DEFER)
+ dev_err(&adev->dev, "Failed to get reset!\n");
+ return PTR_ERR(pl330->rstc);
+ } else {
+ ret = reset_control_deassert(pl330->rstc);
+ if (ret) {
+ dev_err(&adev->dev, "Couldn't deassert the device from reset!\n");
+ return ret;
+ }
+ }
+
+ pl330->rstc_ocp = devm_reset_control_get_optional(&adev->dev, "dma-ocp");
+ if (IS_ERR(pl330->rstc_ocp)) {
+ if (PTR_ERR(pl330->rstc_ocp) != -EPROBE_DEFER)
+ dev_err(&adev->dev, "Failed to get OCP reset!\n");
+ return PTR_ERR(pl330->rstc_ocp);
+ } else {
+ ret = reset_control_deassert(pl330->rstc_ocp);
+ if (ret) {
+ dev_err(&adev->dev, "Couldn't deassert the device from OCP reset!\n");
+ return ret;
+ }
+ }
+
for (i = 0; i < AMBA_NR_IRQS; i++) {
irq = adev->irq[i];
if (irq) {
@@ -3164,6 +3195,11 @@ probe_err3:
probe_err2:
pl330_del(pl330);
+ if (pl330->rstc_ocp)
+ reset_control_assert(pl330->rstc_ocp);
+
+ if (pl330->rstc)
+ reset_control_assert(pl330->rstc);
return ret;
}
@@ -3202,6 +3238,11 @@ static int pl330_remove(struct amba_device *adev)
pl330_del(pl330);
+ if (pl330->rstc_ocp)
+ reset_control_assert(pl330->rstc_ocp);
+
+ if (pl330->rstc)
+ reset_control_assert(pl330->rstc);
return 0;
}
diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c
index 468c234cb3be..349fb312c872 100644
--- a/drivers/dma/pxa_dma.c
+++ b/drivers/dma/pxa_dma.c
@@ -129,7 +129,6 @@ struct pxad_device {
spinlock_t phy_lock; /* Phy association */
#ifdef CONFIG_DEBUG_FS
struct dentry *dbgfs_root;
- struct dentry *dbgfs_state;
struct dentry **dbgfs_chan;
#endif
};
@@ -323,31 +322,18 @@ static struct dentry *pxad_dbg_alloc_chan(struct pxad_device *pdev,
int ch, struct dentry *chandir)
{
char chan_name[11];
- struct dentry *chan, *chan_state = NULL, *chan_descr = NULL;
- struct dentry *chan_reqs = NULL;
+ struct dentry *chan;
void *dt;
scnprintf(chan_name, sizeof(chan_name), "%d", ch);
chan = debugfs_create_dir(chan_name, chandir);
dt = (void *)&pdev->phys[ch];
- if (chan)
- chan_state = debugfs_create_file("state", 0400, chan, dt,
- &chan_state_fops);
- if (chan_state)
- chan_descr = debugfs_create_file("descriptors", 0400, chan, dt,
- &descriptors_fops);
- if (chan_descr)
- chan_reqs = debugfs_create_file("requesters", 0400, chan, dt,
- &requester_chan_fops);
- if (!chan_reqs)
- goto err_state;
+ debugfs_create_file("state", 0400, chan, dt, &chan_state_fops);
+ debugfs_create_file("descriptors", 0400, chan, dt, &descriptors_fops);
+ debugfs_create_file("requesters", 0400, chan, dt, &requester_chan_fops);
return chan;
-
-err_state:
- debugfs_remove_recursive(chan);
- return NULL;
}
static void pxad_init_debugfs(struct pxad_device *pdev)
@@ -355,40 +341,20 @@ static void pxad_init_debugfs(struct pxad_device *pdev)
int i;
struct dentry *chandir;
- pdev->dbgfs_root = debugfs_create_dir(dev_name(pdev->slave.dev), NULL);
- if (IS_ERR(pdev->dbgfs_root) || !pdev->dbgfs_root)
- goto err_root;
-
- pdev->dbgfs_state = debugfs_create_file("state", 0400, pdev->dbgfs_root,
- pdev, &state_fops);
- if (!pdev->dbgfs_state)
- goto err_state;
-
pdev->dbgfs_chan =
- kmalloc_array(pdev->nr_chans, sizeof(*pdev->dbgfs_state),
+ kmalloc_array(pdev->nr_chans, sizeof(struct dentry *),
GFP_KERNEL);
if (!pdev->dbgfs_chan)
- goto err_alloc;
+ return;
+
+ pdev->dbgfs_root = debugfs_create_dir(dev_name(pdev->slave.dev), NULL);
+
+ debugfs_create_file("state", 0400, pdev->dbgfs_root, pdev, &state_fops);
chandir = debugfs_create_dir("channels", pdev->dbgfs_root);
- if (!chandir)
- goto err_chandir;
- for (i = 0; i < pdev->nr_chans; i++) {
+ for (i = 0; i < pdev->nr_chans; i++)
pdev->dbgfs_chan[i] = pxad_dbg_alloc_chan(pdev, i, chandir);
- if (!pdev->dbgfs_chan[i])
- goto err_chans;
- }
-
- return;
-err_chans:
-err_chandir:
- kfree(pdev->dbgfs_chan);
-err_alloc:
-err_state:
- debugfs_remove_recursive(pdev->dbgfs_root);
-err_root:
- pr_err("pxad: debugfs is not available\n");
}
static void pxad_cleanup_debugfs(struct pxad_device *pdev)
diff --git a/drivers/dma/qcom/hidma.h b/drivers/dma/qcom/hidma.h
index f337e2789ddc..f212466744f3 100644
--- a/drivers/dma/qcom/hidma.h
+++ b/drivers/dma/qcom/hidma.h
@@ -93,8 +93,6 @@ struct hidma_chan {
* It is used by the DMA complete notification to
* locate the descriptor that initiated the transfer.
*/
- struct dentry *debugfs;
- struct dentry *stats;
struct hidma_dev *dmadev;
struct hidma_desc *running;
@@ -126,7 +124,6 @@ struct hidma_dev {
struct dma_device ddev;
struct dentry *debugfs;
- struct dentry *stats;
/* sysfs entry for the channel id */
struct device_attribute *chid_attrs;
@@ -158,6 +155,6 @@ irqreturn_t hidma_ll_inthandler(int irq, void *arg);
irqreturn_t hidma_ll_inthandler_msi(int irq, void *arg, int cause);
void hidma_cleanup_pending_tre(struct hidma_lldev *llhndl, u8 err_info,
u8 err_code);
-int hidma_debug_init(struct hidma_dev *dmadev);
+void hidma_debug_init(struct hidma_dev *dmadev);
void hidma_debug_uninit(struct hidma_dev *dmadev);
#endif
diff --git a/drivers/dma/qcom/hidma_dbg.c b/drivers/dma/qcom/hidma_dbg.c
index 75b0691a670d..ce87c7937a0e 100644
--- a/drivers/dma/qcom/hidma_dbg.c
+++ b/drivers/dma/qcom/hidma_dbg.c
@@ -138,17 +138,13 @@ void hidma_debug_uninit(struct hidma_dev *dmadev)
debugfs_remove_recursive(dmadev->debugfs);
}
-int hidma_debug_init(struct hidma_dev *dmadev)
+void hidma_debug_init(struct hidma_dev *dmadev)
{
- int rc = 0;
int chidx = 0;
struct list_head *position = NULL;
+ struct dentry *dir;
dmadev->debugfs = debugfs_create_dir(dev_name(dmadev->ddev.dev), NULL);
- if (!dmadev->debugfs) {
- rc = -ENODEV;
- return rc;
- }
/* walk through the virtual channel list */
list_for_each(position, &dmadev->ddev.channels) {
@@ -157,32 +153,13 @@ int hidma_debug_init(struct hidma_dev *dmadev)
chan = list_entry(position, struct hidma_chan,
chan.device_node);
sprintf(chan->dbg_name, "chan%d", chidx);
- chan->debugfs = debugfs_create_dir(chan->dbg_name,
+ dir = debugfs_create_dir(chan->dbg_name,
dmadev->debugfs);
- if (!chan->debugfs) {
- rc = -ENOMEM;
- goto cleanup;
- }
- chan->stats = debugfs_create_file("stats", S_IRUGO,
- chan->debugfs, chan,
- &hidma_chan_fops);
- if (!chan->stats) {
- rc = -ENOMEM;
- goto cleanup;
- }
+ debugfs_create_file("stats", S_IRUGO, dir, chan,
+ &hidma_chan_fops);
chidx++;
}
- dmadev->stats = debugfs_create_file("stats", S_IRUGO,
- dmadev->debugfs, dmadev,
- &hidma_dma_fops);
- if (!dmadev->stats) {
- rc = -ENOMEM;
- goto cleanup;
- }
-
- return 0;
-cleanup:
- hidma_debug_uninit(dmadev);
- return rc;
+ debugfs_create_file("stats", S_IRUGO, dmadev->debugfs, dmadev,
+ &hidma_dma_fops);
}
diff --git a/drivers/dma/qcom/hidma_ll.c b/drivers/dma/qcom/hidma_ll.c
index 5bf8b145c427..bb4471e84e48 100644
--- a/drivers/dma/qcom/hidma_ll.c
+++ b/drivers/dma/qcom/hidma_ll.c
@@ -749,7 +749,6 @@ struct hidma_lldev *hidma_ll_init(struct device *dev, u32 nr_tres,
if (!lldev->tre_ring)
return NULL;
- memset(lldev->tre_ring, 0, (HIDMA_TRE_SIZE + 1) * nr_tres);
lldev->tre_ring_size = HIDMA_TRE_SIZE * nr_tres;
lldev->nr_tres = nr_tres;
@@ -769,7 +768,6 @@ struct hidma_lldev *hidma_ll_init(struct device *dev, u32 nr_tres,
if (!lldev->evre_ring)
return NULL;
- memset(lldev->evre_ring, 0, (HIDMA_EVRE_SIZE + 1) * nr_tres);
lldev->evre_ring_size = HIDMA_EVRE_SIZE * nr_tres;
/* the EVRE ring has to be EVRE_SIZE aligned */
diff --git a/drivers/dma/qcom/hidma_mgmt.c b/drivers/dma/qcom/hidma_mgmt.c
index 3022d66e7a33..806ca02c52d7 100644
--- a/drivers/dma/qcom/hidma_mgmt.c
+++ b/drivers/dma/qcom/hidma_mgmt.c
@@ -183,7 +183,6 @@ static int hidma_mgmt_probe(struct platform_device *pdev)
irq = platform_get_irq(pdev, 0);
if (irq < 0) {
- dev_err(&pdev->dev, "irq resources not found\n");
rc = irq;
goto out;
}
@@ -388,7 +387,6 @@ static int __init hidma_mgmt_of_populate_channels(struct device_node *np)
ret = PTR_ERR(new_pdev);
goto out;
}
- of_node_get(child);
new_pdev->dev.of_node = child;
of_dma_configure(&new_pdev->dev, child, true);
/*
@@ -396,9 +394,14 @@ static int __init hidma_mgmt_of_populate_channels(struct device_node *np)
* platforms with or without MSI support.
*/
of_msi_configure(&new_pdev->dev, child);
- of_node_put(child);
}
+
+ kfree(res);
+
+ return ret;
+
out:
+ of_node_put(child);
kfree(res);
return ret;
diff --git a/drivers/dma/s3c24xx-dma.c b/drivers/dma/s3c24xx-dma.c
index ad30f3d2c7f6..43da8eeb18ef 100644
--- a/drivers/dma/s3c24xx-dma.c
+++ b/drivers/dma/s3c24xx-dma.c
@@ -1237,11 +1237,8 @@ static int s3c24xx_dma_probe(struct platform_device *pdev)
phy->host = s3cdma;
phy->irq = platform_get_irq(pdev, i);
- if (phy->irq < 0) {
- dev_err(&pdev->dev, "failed to get irq %d, err %d\n",
- i, phy->irq);
+ if (phy->irq < 0)
continue;
- }
ret = devm_request_irq(&pdev->dev, phy->irq, s3c24xx_dma_irq,
0, pdev->name, phy);
diff --git a/drivers/dma/sh/Kconfig b/drivers/dma/sh/Kconfig
index 4d6b02b3b1f1..54d5d0369d3c 100644
--- a/drivers/dma/sh/Kconfig
+++ b/drivers/dma/sh/Kconfig
@@ -47,9 +47,3 @@ config RENESAS_USB_DMAC
help
This driver supports the USB-DMA controller found in the Renesas
SoCs.
-
-config SUDMAC
- tristate "Renesas SUDMAC support"
- depends on SH_DMAE_BASE
- help
- Enable support for the Renesas SUDMAC controllers.
diff --git a/drivers/dma/sh/Makefile b/drivers/dma/sh/Makefile
index 42110dd57a56..112fbd22bb3f 100644
--- a/drivers/dma/sh/Makefile
+++ b/drivers/dma/sh/Makefile
@@ -15,4 +15,3 @@ obj-$(CONFIG_SH_DMAE) += shdma.o
obj-$(CONFIG_RCAR_DMAC) += rcar-dmac.o
obj-$(CONFIG_RENESAS_USB_DMAC) += usb-dmac.o
-obj-$(CONFIG_SUDMAC) += sudmac.o
diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c
index 33ab1b607e2b..3993ab65c62c 100644
--- a/drivers/dma/sh/rcar-dmac.c
+++ b/drivers/dma/sh/rcar-dmac.c
@@ -192,6 +192,7 @@ struct rcar_dmac_chan {
* @iomem: remapped I/O memory base
* @n_channels: number of available channels
* @channels: array of DMAC channels
+ * @channels_mask: bitfield of which DMA channels are managed by this driver
* @modules: bitmask of client modules in use
*/
struct rcar_dmac {
@@ -202,6 +203,7 @@ struct rcar_dmac {
unsigned int n_channels;
struct rcar_dmac_chan *channels;
+ unsigned int channels_mask;
DECLARE_BITMAP(modules, 256);
};
@@ -438,7 +440,7 @@ static int rcar_dmac_init(struct rcar_dmac *dmac)
u16 dmaor;
/* Clear all channels and enable the DMAC globally. */
- rcar_dmac_write(dmac, RCAR_DMACHCLR, GENMASK(dmac->n_channels - 1, 0));
+ rcar_dmac_write(dmac, RCAR_DMACHCLR, dmac->channels_mask);
rcar_dmac_write(dmac, RCAR_DMAOR,
RCAR_DMAOR_PRI_FIXED | RCAR_DMAOR_DME);
@@ -814,6 +816,9 @@ static void rcar_dmac_stop_all_chan(struct rcar_dmac *dmac)
for (i = 0; i < dmac->n_channels; ++i) {
struct rcar_dmac_chan *chan = &dmac->channels[i];
+ if (!(dmac->channels_mask & BIT(i)))
+ continue;
+
/* Stop and reinitialize the channel. */
spin_lock_irq(&chan->lock);
rcar_dmac_chan_halt(chan);
@@ -1165,7 +1170,7 @@ rcar_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
/* Someone calling slave DMA on a generic channel? */
- if (rchan->mid_rid < 0 || !sg_len) {
+ if (rchan->mid_rid < 0 || !sg_len || !sg_dma_len(sgl)) {
dev_warn(chan->device->dev,
"%s: bad parameter: len=%d, id=%d\n",
__func__, sg_len, rchan->mid_rid);
@@ -1654,8 +1659,7 @@ static bool rcar_dmac_chan_filter(struct dma_chan *chan, void *arg)
* Forcing it to call dma_request_channel() and iterate through all
* channels from all controllers is just pointless.
*/
- if (chan->device->device_config != rcar_dmac_device_config ||
- dma_spec->np != chan->device->dev->of_node)
+ if (chan->device->device_config != rcar_dmac_device_config)
return false;
return !test_and_set_bit(dma_spec->args[0], dmac->modules);
@@ -1675,7 +1679,8 @@ static struct dma_chan *rcar_dmac_of_xlate(struct of_phandle_args *dma_spec,
dma_cap_zero(mask);
dma_cap_set(DMA_SLAVE, mask);
- chan = dma_request_channel(mask, rcar_dmac_chan_filter, dma_spec);
+ chan = __dma_request_channel(&mask, rcar_dmac_chan_filter, dma_spec,
+ ofdma->of_node);
if (!chan)
return NULL;
@@ -1744,10 +1749,8 @@ static int rcar_dmac_chan_probe(struct rcar_dmac *dmac,
/* Request the channel interrupt. */
sprintf(pdev_irqname, "ch%u", index);
rchan->irq = platform_get_irq_byname(pdev, pdev_irqname);
- if (rchan->irq < 0) {
- dev_err(dmac->dev, "no IRQ specified for channel %u\n", index);
+ if (rchan->irq < 0)
return -ENODEV;
- }
irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u",
dev_name(dmac->dev), index);
@@ -1776,6 +1779,8 @@ static int rcar_dmac_chan_probe(struct rcar_dmac *dmac,
return 0;
}
+#define RCAR_DMAC_MAX_CHANNELS 32
+
static int rcar_dmac_parse_of(struct device *dev, struct rcar_dmac *dmac)
{
struct device_node *np = dev->of_node;
@@ -1787,12 +1792,16 @@ static int rcar_dmac_parse_of(struct device *dev, struct rcar_dmac *dmac)
return ret;
}
- if (dmac->n_channels <= 0 || dmac->n_channels >= 100) {
+ /* The hardware and driver don't support more than 32 bits in CHCLR */
+ if (dmac->n_channels <= 0 ||
+ dmac->n_channels >= RCAR_DMAC_MAX_CHANNELS) {
dev_err(dev, "invalid number of channels %u\n",
dmac->n_channels);
return -EINVAL;
}
+ dmac->channels_mask = GENMASK(dmac->n_channels - 1, 0);
+
return 0;
}
@@ -1802,7 +1811,6 @@ static int rcar_dmac_probe(struct platform_device *pdev)
DMA_SLAVE_BUSWIDTH_2_BYTES | DMA_SLAVE_BUSWIDTH_4_BYTES |
DMA_SLAVE_BUSWIDTH_8_BYTES | DMA_SLAVE_BUSWIDTH_16_BYTES |
DMA_SLAVE_BUSWIDTH_32_BYTES | DMA_SLAVE_BUSWIDTH_64_BYTES;
- unsigned int channels_offset = 0;
struct dma_device *engine;
struct rcar_dmac *dmac;
struct resource *mem;
@@ -1831,10 +1839,8 @@ static int rcar_dmac_probe(struct platform_device *pdev)
* level we can't disable it selectively, so ignore channel 0 for now if
* the device is part of an IOMMU group.
*/
- if (device_iommu_mapped(&pdev->dev)) {
- dmac->n_channels--;
- channels_offset = 1;
- }
+ if (device_iommu_mapped(&pdev->dev))
+ dmac->channels_mask &= ~BIT(0);
dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels,
sizeof(*dmac->channels), GFP_KERNEL);
@@ -1892,8 +1898,10 @@ static int rcar_dmac_probe(struct platform_device *pdev)
INIT_LIST_HEAD(&engine->channels);
for (i = 0; i < dmac->n_channels; ++i) {
- ret = rcar_dmac_chan_probe(dmac, &dmac->channels[i],
- i + channels_offset);
+ if (!(dmac->channels_mask & BIT(i)))
+ continue;
+
+ ret = rcar_dmac_chan_probe(dmac, &dmac->channels[i], i);
if (ret < 0)
goto error;
}
diff --git a/drivers/dma/sh/sudmac.c b/drivers/dma/sh/sudmac.c
deleted file mode 100644
index 30cc3553cb8b..000000000000
--- a/drivers/dma/sh/sudmac.c
+++ /dev/null
@@ -1,414 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Renesas SUDMAC support
- *
- * Copyright (C) 2013 Renesas Solutions Corp.
- *
- * based on drivers/dma/sh/shdma.c:
- * Copyright (C) 2011-2012 Guennadi Liakhovetski <g.liakhovetski@gmx.de>
- * Copyright (C) 2009 Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>
- * Copyright (C) 2009 Renesas Solutions, Inc. All rights reserved.
- * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved.
- */
-
-#include <linux/dmaengine.h>
-#include <linux/err.h>
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-#include <linux/sudmac.h>
-
-struct sudmac_chan {
- struct shdma_chan shdma_chan;
- void __iomem *base;
- char dev_id[16]; /* unique name per DMAC of channel */
-
- u32 offset; /* for CFG, BA, BBC, CA, CBC, DEN */
- u32 cfg;
- u32 dint_end_bit;
-};
-
-struct sudmac_device {
- struct shdma_dev shdma_dev;
- struct sudmac_pdata *pdata;
- void __iomem *chan_reg;
-};
-
-struct sudmac_regs {
- u32 base_addr;
- u32 base_byte_count;
-};
-
-struct sudmac_desc {
- struct sudmac_regs hw;
- struct shdma_desc shdma_desc;
-};
-
-#define to_chan(schan) container_of(schan, struct sudmac_chan, shdma_chan)
-#define to_desc(sdesc) container_of(sdesc, struct sudmac_desc, shdma_desc)
-#define to_sdev(sc) container_of(sc->shdma_chan.dma_chan.device, \
- struct sudmac_device, shdma_dev.dma_dev)
-
-/* SUDMAC register */
-#define SUDMAC_CH0CFG 0x00
-#define SUDMAC_CH0BA 0x10
-#define SUDMAC_CH0BBC 0x18
-#define SUDMAC_CH0CA 0x20
-#define SUDMAC_CH0CBC 0x28
-#define SUDMAC_CH0DEN 0x30
-#define SUDMAC_DSTSCLR 0x38
-#define SUDMAC_DBUFCTRL 0x3C
-#define SUDMAC_DINTCTRL 0x40
-#define SUDMAC_DINTSTS 0x44
-#define SUDMAC_DINTSTSCLR 0x48
-#define SUDMAC_CH0SHCTRL 0x50
-
-/* Definitions for the sudmac_channel.config */
-#define SUDMAC_SENDBUFM 0x1000 /* b12: Transmit Buffer Mode */
-#define SUDMAC_RCVENDM 0x0100 /* b8: Receive Data Transfer End Mode */
-#define SUDMAC_LBA_WAIT 0x0030 /* b5-4: Local Bus Access Wait */
-
-/* Definitions for the sudmac_channel.dint_end_bit */
-#define SUDMAC_CH1ENDE 0x0002 /* b1: Ch1 DMA Transfer End Int Enable */
-#define SUDMAC_CH0ENDE 0x0001 /* b0: Ch0 DMA Transfer End Int Enable */
-
-#define SUDMAC_DRV_NAME "sudmac"
-
-static void sudmac_writel(struct sudmac_chan *sc, u32 data, u32 reg)
-{
- iowrite32(data, sc->base + reg);
-}
-
-static u32 sudmac_readl(struct sudmac_chan *sc, u32 reg)
-{
- return ioread32(sc->base + reg);
-}
-
-static bool sudmac_is_busy(struct sudmac_chan *sc)
-{
- u32 den = sudmac_readl(sc, SUDMAC_CH0DEN + sc->offset);
-
- if (den)
- return true; /* working */
-
- return false; /* waiting */
-}
-
-static void sudmac_set_reg(struct sudmac_chan *sc, struct sudmac_regs *hw,
- struct shdma_desc *sdesc)
-{
- sudmac_writel(sc, sc->cfg, SUDMAC_CH0CFG + sc->offset);
- sudmac_writel(sc, hw->base_addr, SUDMAC_CH0BA + sc->offset);
- sudmac_writel(sc, hw->base_byte_count, SUDMAC_CH0BBC + sc->offset);
-}
-
-static void sudmac_start(struct sudmac_chan *sc)
-{
- u32 dintctrl = sudmac_readl(sc, SUDMAC_DINTCTRL);
-
- sudmac_writel(sc, dintctrl | sc->dint_end_bit, SUDMAC_DINTCTRL);
- sudmac_writel(sc, 1, SUDMAC_CH0DEN + sc->offset);
-}
-
-static void sudmac_start_xfer(struct shdma_chan *schan,
- struct shdma_desc *sdesc)
-{
- struct sudmac_chan *sc = to_chan(schan);
- struct sudmac_desc *sd = to_desc(sdesc);
-
- sudmac_set_reg(sc, &sd->hw, sdesc);
- sudmac_start(sc);
-}
-
-static bool sudmac_channel_busy(struct shdma_chan *schan)
-{
- struct sudmac_chan *sc = to_chan(schan);
-
- return sudmac_is_busy(sc);
-}
-
-static void sudmac_setup_xfer(struct shdma_chan *schan, int slave_id)
-{
-}
-
-static const struct sudmac_slave_config *sudmac_find_slave(
- struct sudmac_chan *sc, int slave_id)
-{
- struct sudmac_device *sdev = to_sdev(sc);
- struct sudmac_pdata *pdata = sdev->pdata;
- const struct sudmac_slave_config *cfg;
- int i;
-
- for (i = 0, cfg = pdata->slave; i < pdata->slave_num; i++, cfg++)
- if (cfg->slave_id == slave_id)
- return cfg;
-
- return NULL;
-}
-
-static int sudmac_set_slave(struct shdma_chan *schan, int slave_id,
- dma_addr_t slave_addr, bool try)
-{
- struct sudmac_chan *sc = to_chan(schan);
- const struct sudmac_slave_config *cfg = sudmac_find_slave(sc, slave_id);
-
- if (!cfg)
- return -ENODEV;
-
- return 0;
-}
-
-static inline void sudmac_dma_halt(struct sudmac_chan *sc)
-{
- u32 dintctrl = sudmac_readl(sc, SUDMAC_DINTCTRL);
-
- sudmac_writel(sc, 0, SUDMAC_CH0DEN + sc->offset);
- sudmac_writel(sc, dintctrl & ~sc->dint_end_bit, SUDMAC_DINTCTRL);
- sudmac_writel(sc, sc->dint_end_bit, SUDMAC_DINTSTSCLR);
-}
-
-static int sudmac_desc_setup(struct shdma_chan *schan,
- struct shdma_desc *sdesc,
- dma_addr_t src, dma_addr_t dst, size_t *len)
-{
- struct sudmac_chan *sc = to_chan(schan);
- struct sudmac_desc *sd = to_desc(sdesc);
-
- dev_dbg(sc->shdma_chan.dev, "%s: src=%pad, dst=%pad, len=%zu\n",
- __func__, &src, &dst, *len);
-
- if (*len > schan->max_xfer_len)
- *len = schan->max_xfer_len;
-
- if (dst)
- sd->hw.base_addr = dst;
- else if (src)
- sd->hw.base_addr = src;
- sd->hw.base_byte_count = *len;
-
- return 0;
-}
-
-static void sudmac_halt(struct shdma_chan *schan)
-{
- struct sudmac_chan *sc = to_chan(schan);
-
- sudmac_dma_halt(sc);
-}
-
-static bool sudmac_chan_irq(struct shdma_chan *schan, int irq)
-{
- struct sudmac_chan *sc = to_chan(schan);
- u32 dintsts = sudmac_readl(sc, SUDMAC_DINTSTS);
-
- if (!(dintsts & sc->dint_end_bit))
- return false;
-
- /* DMA stop */
- sudmac_dma_halt(sc);
-
- return true;
-}
-
-static size_t sudmac_get_partial(struct shdma_chan *schan,
- struct shdma_desc *sdesc)
-{
- struct sudmac_chan *sc = to_chan(schan);
- struct sudmac_desc *sd = to_desc(sdesc);
- u32 current_byte_count = sudmac_readl(sc, SUDMAC_CH0CBC + sc->offset);
-
- return sd->hw.base_byte_count - current_byte_count;
-}
-
-static bool sudmac_desc_completed(struct shdma_chan *schan,
- struct shdma_desc *sdesc)
-{
- struct sudmac_chan *sc = to_chan(schan);
- struct sudmac_desc *sd = to_desc(sdesc);
- u32 current_addr = sudmac_readl(sc, SUDMAC_CH0CA + sc->offset);
-
- return sd->hw.base_addr + sd->hw.base_byte_count == current_addr;
-}
-
-static int sudmac_chan_probe(struct sudmac_device *su_dev, int id, int irq,
- unsigned long flags)
-{
- struct shdma_dev *sdev = &su_dev->shdma_dev;
- struct platform_device *pdev = to_platform_device(sdev->dma_dev.dev);
- struct sudmac_chan *sc;
- struct shdma_chan *schan;
- int err;
-
- sc = devm_kzalloc(&pdev->dev, sizeof(struct sudmac_chan), GFP_KERNEL);
- if (!sc)
- return -ENOMEM;
-
- schan = &sc->shdma_chan;
- schan->max_xfer_len = 64 * 1024 * 1024 - 1;
-
- shdma_chan_probe(sdev, schan, id);
-
- sc->base = su_dev->chan_reg;
-
- /* get platform_data */
- sc->offset = su_dev->pdata->channel->offset;
- if (su_dev->pdata->channel->config & SUDMAC_TX_BUFFER_MODE)
- sc->cfg |= SUDMAC_SENDBUFM;
- if (su_dev->pdata->channel->config & SUDMAC_RX_END_MODE)
- sc->cfg |= SUDMAC_RCVENDM;
- sc->cfg |= (su_dev->pdata->channel->wait << 4) & SUDMAC_LBA_WAIT;
-
- if (su_dev->pdata->channel->dint_end_bit & SUDMAC_DMA_BIT_CH0)
- sc->dint_end_bit |= SUDMAC_CH0ENDE;
- if (su_dev->pdata->channel->dint_end_bit & SUDMAC_DMA_BIT_CH1)
- sc->dint_end_bit |= SUDMAC_CH1ENDE;
-
- /* set up channel irq */
- if (pdev->id >= 0)
- snprintf(sc->dev_id, sizeof(sc->dev_id), "sudmac%d.%d",
- pdev->id, id);
- else
- snprintf(sc->dev_id, sizeof(sc->dev_id), "sudmac%d", id);
-
- err = shdma_request_irq(schan, irq, flags, sc->dev_id);
- if (err) {
- dev_err(sdev->dma_dev.dev,
- "DMA channel %d request_irq failed %d\n", id, err);
- goto err_no_irq;
- }
-
- return 0;
-
-err_no_irq:
- /* remove from dmaengine device node */
- shdma_chan_remove(schan);
- return err;
-}
-
-static void sudmac_chan_remove(struct sudmac_device *su_dev)
-{
- struct shdma_chan *schan;
- int i;
-
- shdma_for_each_chan(schan, &su_dev->shdma_dev, i) {
- BUG_ON(!schan);
-
- shdma_chan_remove(schan);
- }
-}
-
-static dma_addr_t sudmac_slave_addr(struct shdma_chan *schan)
-{
- /* SUDMAC doesn't need the address */
- return 0;
-}
-
-static struct shdma_desc *sudmac_embedded_desc(void *buf, int i)
-{
- return &((struct sudmac_desc *)buf)[i].shdma_desc;
-}
-
-static const struct shdma_ops sudmac_shdma_ops = {
- .desc_completed = sudmac_desc_completed,
- .halt_channel = sudmac_halt,
- .channel_busy = sudmac_channel_busy,
- .slave_addr = sudmac_slave_addr,
- .desc_setup = sudmac_desc_setup,
- .set_slave = sudmac_set_slave,
- .setup_xfer = sudmac_setup_xfer,
- .start_xfer = sudmac_start_xfer,
- .embedded_desc = sudmac_embedded_desc,
- .chan_irq = sudmac_chan_irq,
- .get_partial = sudmac_get_partial,
-};
-
-static int sudmac_probe(struct platform_device *pdev)
-{
- struct sudmac_pdata *pdata = dev_get_platdata(&pdev->dev);
- int err, i;
- struct sudmac_device *su_dev;
- struct dma_device *dma_dev;
- struct resource *chan, *irq_res;
-
- /* get platform data */
- if (!pdata)
- return -ENODEV;
-
- irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
- if (!irq_res)
- return -ENODEV;
-
- err = -ENOMEM;
- su_dev = devm_kzalloc(&pdev->dev, sizeof(struct sudmac_device),
- GFP_KERNEL);
- if (!su_dev)
- return err;
-
- dma_dev = &su_dev->shdma_dev.dma_dev;
-
- chan = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- su_dev->chan_reg = devm_ioremap_resource(&pdev->dev, chan);
- if (IS_ERR(su_dev->chan_reg))
- return PTR_ERR(su_dev->chan_reg);
-
- dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
-
- su_dev->shdma_dev.ops = &sudmac_shdma_ops;
- su_dev->shdma_dev.desc_size = sizeof(struct sudmac_desc);
- err = shdma_init(&pdev->dev, &su_dev->shdma_dev, pdata->channel_num);
- if (err < 0)
- return err;
-
- /* platform data */
- su_dev->pdata = dev_get_platdata(&pdev->dev);
-
- platform_set_drvdata(pdev, su_dev);
-
- /* Create DMA Channel */
- for (i = 0; i < pdata->channel_num; i++) {
- err = sudmac_chan_probe(su_dev, i, irq_res->start, IRQF_SHARED);
- if (err)
- goto chan_probe_err;
- }
-
- err = dma_async_device_register(&su_dev->shdma_dev.dma_dev);
- if (err < 0)
- goto chan_probe_err;
-
- return err;
-
-chan_probe_err:
- sudmac_chan_remove(su_dev);
-
- shdma_cleanup(&su_dev->shdma_dev);
-
- return err;
-}
-
-static int sudmac_remove(struct platform_device *pdev)
-{
- struct sudmac_device *su_dev = platform_get_drvdata(pdev);
- struct dma_device *dma_dev = &su_dev->shdma_dev.dma_dev;
-
- dma_async_device_unregister(dma_dev);
- sudmac_chan_remove(su_dev);
- shdma_cleanup(&su_dev->shdma_dev);
-
- return 0;
-}
-
-static struct platform_driver sudmac_driver = {
- .driver = {
- .name = SUDMAC_DRV_NAME,
- },
- .probe = sudmac_probe,
- .remove = sudmac_remove,
-};
-module_platform_driver(sudmac_driver);
-
-MODULE_AUTHOR("Yoshihiro Shimoda");
-MODULE_DESCRIPTION("Renesas SUDMAC driver");
-MODULE_LICENSE("GPL v2");
-MODULE_ALIAS("platform:" SUDMAC_DRV_NAME);
diff --git a/drivers/dma/sh/usb-dmac.c b/drivers/dma/sh/usb-dmac.c
index 59403f6d008a..b218a013c260 100644
--- a/drivers/dma/sh/usb-dmac.c
+++ b/drivers/dma/sh/usb-dmac.c
@@ -57,7 +57,7 @@ struct usb_dmac_desc {
u32 residue;
struct list_head node;
dma_cookie_t done_cookie;
- struct usb_dmac_sg sg[0];
+ struct usb_dmac_sg sg[];
};
#define to_usb_dmac_desc(vd) container_of(vd, struct usb_dmac_desc, vd)
@@ -636,9 +636,6 @@ static bool usb_dmac_chan_filter(struct dma_chan *chan, void *arg)
struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan);
struct of_phandle_args *dma_spec = arg;
- if (dma_spec->np != chan->device->dev->of_node)
- return false;
-
/* USB-DMAC should be used with fixed usb controller's FIFO */
if (uchan->index != dma_spec->args[0])
return false;
@@ -659,7 +656,8 @@ static struct dma_chan *usb_dmac_of_xlate(struct of_phandle_args *dma_spec,
dma_cap_zero(mask);
dma_cap_set(DMA_SLAVE, mask);
- chan = dma_request_channel(mask, usb_dmac_chan_filter, dma_spec);
+ chan = __dma_request_channel(&mask, usb_dmac_chan_filter, dma_spec,
+ ofdma->of_node);
if (!chan)
return NULL;
@@ -719,10 +717,8 @@ static int usb_dmac_chan_probe(struct usb_dmac *dmac,
/* Request the channel interrupt. */
sprintf(pdev_irqname, "ch%u", index);
uchan->irq = platform_get_irq_byname(pdev, pdev_irqname);
- if (uchan->irq < 0) {
- dev_err(dmac->dev, "no IRQ specified for channel %u\n", index);
+ if (uchan->irq < 0)
return -ENODEV;
- }
irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u",
dev_name(dmac->dev), index);
diff --git a/drivers/dma/sprd-dma.c b/drivers/dma/sprd-dma.c
index baac476c8622..525dc7338fe3 100644
--- a/drivers/dma/sprd-dma.c
+++ b/drivers/dma/sprd-dma.c
@@ -908,6 +908,7 @@ sprd_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
struct dma_slave_config *slave_cfg = &schan->slave_cfg;
dma_addr_t src = 0, dst = 0;
+ dma_addr_t start_src = 0, start_dst = 0;
struct sprd_dma_desc *sdesc;
struct scatterlist *sg;
u32 len = 0;
@@ -954,6 +955,11 @@ sprd_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
dst = sg_dma_address(sg);
}
+ if (!i) {
+ start_src = src;
+ start_dst = dst;
+ }
+
/*
* The link-list mode needs at least 2 link-list
* configurations. If there is only one sg, it doesn't
@@ -970,8 +976,8 @@ sprd_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
}
}
- ret = sprd_dma_fill_desc(chan, &sdesc->chn_hw, 0, 0, src, dst, len,
- dir, flags, slave_cfg);
+ ret = sprd_dma_fill_desc(chan, &sdesc->chn_hw, 0, 0, start_src,
+ start_dst, len, dir, flags, slave_cfg);
if (ret) {
kfree(sdesc);
return NULL;
diff --git a/drivers/dma/st_fdma.c b/drivers/dma/st_fdma.c
index a3ee0f6bb664..67087dbe2f9f 100644
--- a/drivers/dma/st_fdma.c
+++ b/drivers/dma/st_fdma.c
@@ -771,10 +771,8 @@ static int st_fdma_probe(struct platform_device *pdev)
platform_set_drvdata(pdev, fdev);
fdev->irq = platform_get_irq(pdev, 0);
- if (fdev->irq < 0) {
- dev_err(&pdev->dev, "Failed to get irq resource\n");
+ if (fdev->irq < 0)
return -EINVAL;
- }
ret = devm_request_irq(&pdev->dev, fdev->irq, st_fdma_irq_handler, 0,
dev_name(&pdev->dev), fdev);
diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c
index 89d710899010..de8bfd9a76e9 100644
--- a/drivers/dma/ste_dma40.c
+++ b/drivers/dma/ste_dma40.c
@@ -142,7 +142,7 @@ enum d40_events {
* when the DMA hw is powered off.
* TODO: Add save/restore of D40_DREG_GCC on dma40 v3 or later, if that works.
*/
-static u32 d40_backup_regs[] = {
+static __maybe_unused u32 d40_backup_regs[] = {
D40_DREG_LCPA,
D40_DREG_LCLA,
D40_DREG_PRMSE,
@@ -211,7 +211,7 @@ static u32 d40_backup_regs_v4b[] = {
#define BACKUP_REGS_SZ_V4B ARRAY_SIZE(d40_backup_regs_v4b)
-static u32 d40_backup_regs_chan[] = {
+static __maybe_unused u32 d40_backup_regs_chan[] = {
D40_CHAN_REG_SSCFG,
D40_CHAN_REG_SSELT,
D40_CHAN_REG_SSPTR,
diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c
index da41bab98f5b..5989b0893521 100644
--- a/drivers/dma/stm32-dma.c
+++ b/drivers/dma/stm32-dma.c
@@ -243,12 +243,6 @@ static void stm32_dma_write(struct stm32_dma_device *dmadev, u32 reg, u32 val)
writel_relaxed(val, dmadev->base + reg);
}
-static struct stm32_dma_desc *stm32_dma_alloc_desc(u32 num_sgs)
-{
- return kzalloc(sizeof(struct stm32_dma_desc) +
- sizeof(struct stm32_dma_sg_req) * num_sgs, GFP_NOWAIT);
-}
-
static int stm32_dma_get_width(struct stm32_dma_chan *chan,
enum dma_slave_buswidth width)
{
@@ -853,7 +847,7 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg(
return NULL;
}
- desc = stm32_dma_alloc_desc(sg_len);
+ desc = kzalloc(struct_size(desc, sg_req, sg_len), GFP_NOWAIT);
if (!desc)
return NULL;
@@ -954,7 +948,7 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic(
num_periods = buf_len / period_len;
- desc = stm32_dma_alloc_desc(num_periods);
+ desc = kzalloc(struct_size(desc, sg_req, num_periods), GFP_NOWAIT);
if (!desc)
return NULL;
@@ -989,7 +983,7 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_memcpy(
int i;
num_sgs = DIV_ROUND_UP(len, STM32_DMA_ALIGNED_MAX_DATA_ITEMS);
- desc = stm32_dma_alloc_desc(num_sgs);
+ desc = kzalloc(struct_size(desc, sg_req, num_sgs), GFP_NOWAIT);
if (!desc)
return NULL;
@@ -1365,14 +1359,9 @@ static int stm32_dma_probe(struct platform_device *pdev)
for (i = 0; i < STM32_DMA_MAX_CHANNELS; i++) {
chan = &dmadev->chan[i];
- chan->irq = platform_get_irq(pdev, i);
ret = platform_get_irq(pdev, i);
- if (ret < 0) {
- if (ret != -EPROBE_DEFER)
- dev_err(&pdev->dev,
- "No irq resource for chan %d\n", i);
+ if (ret < 0)
goto err_unregister;
- }
chan->irq = ret;
ret = devm_request_irq(&pdev->dev, chan->irq,
diff --git a/drivers/dma/stm32-dmamux.c b/drivers/dma/stm32-dmamux.c
index 715aad7a9192..3c89bd39e096 100644
--- a/drivers/dma/stm32-dmamux.c
+++ b/drivers/dma/stm32-dmamux.c
@@ -185,8 +185,7 @@ static int stm32_dmamux_probe(struct platform_device *pdev)
if (!node)
return -ENODEV;
- count = device_property_read_u32_array(&pdev->dev, "dma-masters",
- NULL, 0);
+ count = device_property_count_u32(&pdev->dev, "dma-masters");
if (count < 0) {
dev_err(&pdev->dev, "Can't get DMA master(s) node\n");
return -ENODEV;
@@ -295,8 +294,7 @@ static int stm32_dmamux_probe(struct platform_device *pdev)
#ifdef CONFIG_PM
static int stm32_dmamux_runtime_suspend(struct device *dev)
{
- struct platform_device *pdev =
- container_of(dev, struct platform_device, dev);
+ struct platform_device *pdev = to_platform_device(dev);
struct stm32_dmamux_data *stm32_dmamux = platform_get_drvdata(pdev);
clk_disable_unprepare(stm32_dmamux->clk);
@@ -306,8 +304,7 @@ static int stm32_dmamux_runtime_suspend(struct device *dev)
static int stm32_dmamux_runtime_resume(struct device *dev)
{
- struct platform_device *pdev =
- container_of(dev, struct platform_device, dev);
+ struct platform_device *pdev = to_platform_device(dev);
struct stm32_dmamux_data *stm32_dmamux = platform_get_drvdata(pdev);
int ret;
diff --git a/drivers/dma/stm32-mdma.c b/drivers/dma/stm32-mdma.c
index d6e919d3936a..5838311cf990 100644
--- a/drivers/dma/stm32-mdma.c
+++ b/drivers/dma/stm32-mdma.c
@@ -1366,7 +1366,7 @@ static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid)
chan = &dmadev->chan[id];
if (!chan) {
- dev_err(chan2dev(chan), "MDMA channel not initialized\n");
+ dev_dbg(mdma2dev(dmadev), "MDMA channel not initialized\n");
goto exit;
}
@@ -1555,8 +1555,7 @@ static int stm32_mdma_probe(struct platform_device *pdev)
nr_requests);
}
- count = device_property_read_u32_array(&pdev->dev, "st,ahb-addr-masks",
- NULL, 0);
+ count = device_property_count_u32(&pdev->dev, "st,ahb-addr-masks");
if (count < 0)
count = 0;
@@ -1638,10 +1637,8 @@ static int stm32_mdma_probe(struct platform_device *pdev)
}
dmadev->irq = platform_get_irq(pdev, 0);
- if (dmadev->irq < 0) {
- dev_err(&pdev->dev, "failed to get IRQ\n");
+ if (dmadev->irq < 0)
return dmadev->irq;
- }
ret = devm_request_irq(&pdev->dev, dmadev->irq, stm32_mdma_irq_handler,
0, dev_name(&pdev->dev), dmadev);
diff --git a/drivers/dma/sun4i-dma.c b/drivers/dma/sun4i-dma.c
index 1f80568b2613..e397a50058c8 100644
--- a/drivers/dma/sun4i-dma.c
+++ b/drivers/dma/sun4i-dma.c
@@ -1132,10 +1132,8 @@ static int sun4i_dma_probe(struct platform_device *pdev)
return PTR_ERR(priv->base);
priv->irq = platform_get_irq(pdev, 0);
- if (priv->irq < 0) {
- dev_err(&pdev->dev, "Cannot claim IRQ\n");
+ if (priv->irq < 0)
return priv->irq;
- }
priv->clk = devm_clk_get(&pdev->dev, NULL);
if (IS_ERR(priv->clk)) {
diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c
index e8fcc69b1de9..06cd7f867f7c 100644
--- a/drivers/dma/sun6i-dma.c
+++ b/drivers/dma/sun6i-dma.c
@@ -64,17 +64,20 @@
#define DMA_CHAN_LLI_ADDR 0x08
#define DMA_CHAN_CUR_CFG 0x0c
-#define DMA_CHAN_MAX_DRQ 0x1f
-#define DMA_CHAN_CFG_SRC_DRQ(x) ((x) & DMA_CHAN_MAX_DRQ)
-#define DMA_CHAN_CFG_SRC_IO_MODE BIT(5)
-#define DMA_CHAN_CFG_SRC_LINEAR_MODE (0 << 5)
+#define DMA_CHAN_MAX_DRQ_A31 0x1f
+#define DMA_CHAN_MAX_DRQ_H6 0x3f
+#define DMA_CHAN_CFG_SRC_DRQ_A31(x) ((x) & DMA_CHAN_MAX_DRQ_A31)
+#define DMA_CHAN_CFG_SRC_DRQ_H6(x) ((x) & DMA_CHAN_MAX_DRQ_H6)
+#define DMA_CHAN_CFG_SRC_MODE_A31(x) (((x) & 0x1) << 5)
+#define DMA_CHAN_CFG_SRC_MODE_H6(x) (((x) & 0x1) << 8)
#define DMA_CHAN_CFG_SRC_BURST_A31(x) (((x) & 0x3) << 7)
#define DMA_CHAN_CFG_SRC_BURST_H3(x) (((x) & 0x3) << 6)
#define DMA_CHAN_CFG_SRC_WIDTH(x) (((x) & 0x3) << 9)
-#define DMA_CHAN_CFG_DST_DRQ(x) (DMA_CHAN_CFG_SRC_DRQ(x) << 16)
-#define DMA_CHAN_CFG_DST_IO_MODE (DMA_CHAN_CFG_SRC_IO_MODE << 16)
-#define DMA_CHAN_CFG_DST_LINEAR_MODE (DMA_CHAN_CFG_SRC_LINEAR_MODE << 16)
+#define DMA_CHAN_CFG_DST_DRQ_A31(x) (DMA_CHAN_CFG_SRC_DRQ_A31(x) << 16)
+#define DMA_CHAN_CFG_DST_DRQ_H6(x) (DMA_CHAN_CFG_SRC_DRQ_H6(x) << 16)
+#define DMA_CHAN_CFG_DST_MODE_A31(x) (DMA_CHAN_CFG_SRC_MODE_A31(x) << 16)
+#define DMA_CHAN_CFG_DST_MODE_H6(x) (DMA_CHAN_CFG_SRC_MODE_H6(x) << 16)
#define DMA_CHAN_CFG_DST_BURST_A31(x) (DMA_CHAN_CFG_SRC_BURST_A31(x) << 16)
#define DMA_CHAN_CFG_DST_BURST_H3(x) (DMA_CHAN_CFG_SRC_BURST_H3(x) << 16)
#define DMA_CHAN_CFG_DST_WIDTH(x) (DMA_CHAN_CFG_SRC_WIDTH(x) << 16)
@@ -94,6 +97,8 @@
#define LLI_LAST_ITEM 0xfffff800
#define NORMAL_WAIT 8
#define DRQ_SDRAM 1
+#define LINEAR_MODE 0
+#define IO_MODE 1
/* forward declaration */
struct sun6i_dma_dev;
@@ -121,10 +126,13 @@ struct sun6i_dma_config {
*/
void (*clock_autogate_enable)(struct sun6i_dma_dev *);
void (*set_burst_length)(u32 *p_cfg, s8 src_burst, s8 dst_burst);
+ void (*set_drq)(u32 *p_cfg, s8 src_drq, s8 dst_drq);
+ void (*set_mode)(u32 *p_cfg, s8 src_mode, s8 dst_mode);
u32 src_burst_lengths;
u32 dst_burst_lengths;
u32 src_addr_widths;
u32 dst_addr_widths;
+ bool has_mbus_clk;
};
/*
@@ -178,6 +186,7 @@ struct sun6i_dma_dev {
struct dma_device slave;
void __iomem *base;
struct clk *clk;
+ struct clk *clk_mbus;
int irq;
spinlock_t lock;
struct reset_control *rstc;
@@ -305,6 +314,30 @@ static void sun6i_set_burst_length_h3(u32 *p_cfg, s8 src_burst, s8 dst_burst)
DMA_CHAN_CFG_DST_BURST_H3(dst_burst);
}
+static void sun6i_set_drq_a31(u32 *p_cfg, s8 src_drq, s8 dst_drq)
+{
+ *p_cfg |= DMA_CHAN_CFG_SRC_DRQ_A31(src_drq) |
+ DMA_CHAN_CFG_DST_DRQ_A31(dst_drq);
+}
+
+static void sun6i_set_drq_h6(u32 *p_cfg, s8 src_drq, s8 dst_drq)
+{
+ *p_cfg |= DMA_CHAN_CFG_SRC_DRQ_H6(src_drq) |
+ DMA_CHAN_CFG_DST_DRQ_H6(dst_drq);
+}
+
+static void sun6i_set_mode_a31(u32 *p_cfg, s8 src_mode, s8 dst_mode)
+{
+ *p_cfg |= DMA_CHAN_CFG_SRC_MODE_A31(src_mode) |
+ DMA_CHAN_CFG_DST_MODE_A31(dst_mode);
+}
+
+static void sun6i_set_mode_h6(u32 *p_cfg, s8 src_mode, s8 dst_mode)
+{
+ *p_cfg |= DMA_CHAN_CFG_SRC_MODE_H6(src_mode) |
+ DMA_CHAN_CFG_DST_MODE_H6(dst_mode);
+}
+
static size_t sun6i_get_chan_size(struct sun6i_pchan *pchan)
{
struct sun6i_desc *txd = pchan->desc;
@@ -628,14 +661,12 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy(
burst = convert_burst(8);
width = convert_buswidth(DMA_SLAVE_BUSWIDTH_4_BYTES);
- v_lli->cfg = DMA_CHAN_CFG_SRC_DRQ(DRQ_SDRAM) |
- DMA_CHAN_CFG_DST_DRQ(DRQ_SDRAM) |
- DMA_CHAN_CFG_DST_LINEAR_MODE |
- DMA_CHAN_CFG_SRC_LINEAR_MODE |
- DMA_CHAN_CFG_SRC_WIDTH(width) |
+ v_lli->cfg = DMA_CHAN_CFG_SRC_WIDTH(width) |
DMA_CHAN_CFG_DST_WIDTH(width);
sdev->cfg->set_burst_length(&v_lli->cfg, burst, burst);
+ sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, DRQ_SDRAM);
+ sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, LINEAR_MODE);
sun6i_dma_lli_add(NULL, v_lli, p_lli, txd);
@@ -687,11 +718,9 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg(
if (dir == DMA_MEM_TO_DEV) {
v_lli->src = sg_dma_address(sg);
v_lli->dst = sconfig->dst_addr;
- v_lli->cfg = lli_cfg |
- DMA_CHAN_CFG_DST_IO_MODE |
- DMA_CHAN_CFG_SRC_LINEAR_MODE |
- DMA_CHAN_CFG_SRC_DRQ(DRQ_SDRAM) |
- DMA_CHAN_CFG_DST_DRQ(vchan->port);
+ v_lli->cfg = lli_cfg;
+ sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port);
+ sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE);
dev_dbg(chan2dev(chan),
"%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n",
@@ -702,11 +731,9 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg(
} else {
v_lli->src = sconfig->src_addr;
v_lli->dst = sg_dma_address(sg);
- v_lli->cfg = lli_cfg |
- DMA_CHAN_CFG_DST_LINEAR_MODE |
- DMA_CHAN_CFG_SRC_IO_MODE |
- DMA_CHAN_CFG_DST_DRQ(DRQ_SDRAM) |
- DMA_CHAN_CFG_SRC_DRQ(vchan->port);
+ v_lli->cfg = lli_cfg;
+ sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM);
+ sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE);
dev_dbg(chan2dev(chan),
"%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n",
@@ -772,19 +799,15 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic(
if (dir == DMA_MEM_TO_DEV) {
v_lli->src = buf_addr + period_len * i;
v_lli->dst = sconfig->dst_addr;
- v_lli->cfg = lli_cfg |
- DMA_CHAN_CFG_DST_IO_MODE |
- DMA_CHAN_CFG_SRC_LINEAR_MODE |
- DMA_CHAN_CFG_SRC_DRQ(DRQ_SDRAM) |
- DMA_CHAN_CFG_DST_DRQ(vchan->port);
+ v_lli->cfg = lli_cfg;
+ sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port);
+ sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE);
} else {
v_lli->src = sconfig->src_addr;
v_lli->dst = buf_addr + period_len * i;
- v_lli->cfg = lli_cfg |
- DMA_CHAN_CFG_DST_LINEAR_MODE |
- DMA_CHAN_CFG_SRC_IO_MODE |
- DMA_CHAN_CFG_DST_DRQ(DRQ_SDRAM) |
- DMA_CHAN_CFG_SRC_DRQ(vchan->port);
+ v_lli->cfg = lli_cfg;
+ sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM);
+ sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE);
}
prev = sun6i_dma_lli_add(prev, v_lli, p_lli, txd);
@@ -1049,6 +1072,8 @@ static struct sun6i_dma_config sun6i_a31_dma_cfg = {
.nr_max_requests = 30,
.nr_max_vchans = 53,
.set_burst_length = sun6i_set_burst_length_a31,
+ .set_drq = sun6i_set_drq_a31,
+ .set_mode = sun6i_set_mode_a31,
.src_burst_lengths = BIT(1) | BIT(8),
.dst_burst_lengths = BIT(1) | BIT(8),
.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
@@ -1070,6 +1095,8 @@ static struct sun6i_dma_config sun8i_a23_dma_cfg = {
.nr_max_vchans = 37,
.clock_autogate_enable = sun6i_enable_clock_autogate_a23,
.set_burst_length = sun6i_set_burst_length_a31,
+ .set_drq = sun6i_set_drq_a31,
+ .set_mode = sun6i_set_mode_a31,
.src_burst_lengths = BIT(1) | BIT(8),
.dst_burst_lengths = BIT(1) | BIT(8),
.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
@@ -1086,6 +1113,8 @@ static struct sun6i_dma_config sun8i_a83t_dma_cfg = {
.nr_max_vchans = 39,
.clock_autogate_enable = sun6i_enable_clock_autogate_a23,
.set_burst_length = sun6i_set_burst_length_a31,
+ .set_drq = sun6i_set_drq_a31,
+ .set_mode = sun6i_set_mode_a31,
.src_burst_lengths = BIT(1) | BIT(8),
.dst_burst_lengths = BIT(1) | BIT(8),
.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
@@ -1109,6 +1138,8 @@ static struct sun6i_dma_config sun8i_h3_dma_cfg = {
.nr_max_vchans = 34,
.clock_autogate_enable = sun6i_enable_clock_autogate_h3,
.set_burst_length = sun6i_set_burst_length_h3,
+ .set_drq = sun6i_set_drq_a31,
+ .set_mode = sun6i_set_mode_a31,
.src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
.dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
@@ -1128,6 +1159,29 @@ static struct sun6i_dma_config sun8i_h3_dma_cfg = {
static struct sun6i_dma_config sun50i_a64_dma_cfg = {
.clock_autogate_enable = sun6i_enable_clock_autogate_h3,
.set_burst_length = sun6i_set_burst_length_h3,
+ .set_drq = sun6i_set_drq_a31,
+ .set_mode = sun6i_set_mode_a31,
+ .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
+ .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
+ .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
+ BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
+ BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
+ BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
+ .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
+ BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
+ BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
+ BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
+};
+
+/*
+ * The H6 binding uses the number of dma channels from the
+ * device tree node.
+ */
+static struct sun6i_dma_config sun50i_h6_dma_cfg = {
+ .clock_autogate_enable = sun6i_enable_clock_autogate_h3,
+ .set_burst_length = sun6i_set_burst_length_h3,
+ .set_drq = sun6i_set_drq_h6,
+ .set_mode = sun6i_set_mode_h6,
.src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
.dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
@@ -1138,6 +1192,7 @@ static struct sun6i_dma_config sun50i_a64_dma_cfg = {
BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
+ .has_mbus_clk = true,
};
/*
@@ -1151,6 +1206,8 @@ static struct sun6i_dma_config sun8i_v3s_dma_cfg = {
.nr_max_vchans = 24,
.clock_autogate_enable = sun6i_enable_clock_autogate_a23,
.set_burst_length = sun6i_set_burst_length_a31,
+ .set_drq = sun6i_set_drq_a31,
+ .set_mode = sun6i_set_mode_a31,
.src_burst_lengths = BIT(1) | BIT(8),
.dst_burst_lengths = BIT(1) | BIT(8),
.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
@@ -1168,6 +1225,7 @@ static const struct of_device_id sun6i_dma_match[] = {
{ .compatible = "allwinner,sun8i-h3-dma", .data = &sun8i_h3_dma_cfg },
{ .compatible = "allwinner,sun8i-v3s-dma", .data = &sun8i_v3s_dma_cfg },
{ .compatible = "allwinner,sun50i-a64-dma", .data = &sun50i_a64_dma_cfg },
+ { .compatible = "allwinner,sun50i-h6-dma", .data = &sun50i_h6_dma_cfg },
{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, sun6i_dma_match);
@@ -1193,10 +1251,8 @@ static int sun6i_dma_probe(struct platform_device *pdev)
return PTR_ERR(sdc->base);
sdc->irq = platform_get_irq(pdev, 0);
- if (sdc->irq < 0) {
- dev_err(&pdev->dev, "Cannot claim IRQ\n");
+ if (sdc->irq < 0)
return sdc->irq;
- }
sdc->clk = devm_clk_get(&pdev->dev, NULL);
if (IS_ERR(sdc->clk)) {
@@ -1204,6 +1260,14 @@ static int sun6i_dma_probe(struct platform_device *pdev)
return PTR_ERR(sdc->clk);
}
+ if (sdc->cfg->has_mbus_clk) {
+ sdc->clk_mbus = devm_clk_get(&pdev->dev, "mbus");
+ if (IS_ERR(sdc->clk_mbus)) {
+ dev_err(&pdev->dev, "No mbus clock specified\n");
+ return PTR_ERR(sdc->clk_mbus);
+ }
+ }
+
sdc->rstc = devm_reset_control_get(&pdev->dev, NULL);
if (IS_ERR(sdc->rstc)) {
dev_err(&pdev->dev, "No reset controller specified\n");
@@ -1258,8 +1322,8 @@ static int sun6i_dma_probe(struct platform_device *pdev)
ret = of_property_read_u32(np, "dma-requests", &sdc->max_request);
if (ret && !sdc->max_request) {
dev_info(&pdev->dev, "Missing dma-requests, using %u.\n",
- DMA_CHAN_MAX_DRQ);
- sdc->max_request = DMA_CHAN_MAX_DRQ;
+ DMA_CHAN_MAX_DRQ_A31);
+ sdc->max_request = DMA_CHAN_MAX_DRQ_A31;
}
/*
@@ -1308,11 +1372,19 @@ static int sun6i_dma_probe(struct platform_device *pdev)
goto err_reset_assert;
}
+ if (sdc->cfg->has_mbus_clk) {
+ ret = clk_prepare_enable(sdc->clk_mbus);
+ if (ret) {
+ dev_err(&pdev->dev, "Couldn't enable mbus clock\n");
+ goto err_clk_disable;
+ }
+ }
+
ret = devm_request_irq(&pdev->dev, sdc->irq, sun6i_dma_interrupt, 0,
dev_name(&pdev->dev), sdc);
if (ret) {
dev_err(&pdev->dev, "Cannot request IRQ\n");
- goto err_clk_disable;
+ goto err_mbus_clk_disable;
}
ret = dma_async_device_register(&sdc->slave);
@@ -1337,6 +1409,8 @@ err_dma_unregister:
dma_async_device_unregister(&sdc->slave);
err_irq_disable:
sun6i_kill_tasklet(sdc);
+err_mbus_clk_disable:
+ clk_disable_unprepare(sdc->clk_mbus);
err_clk_disable:
clk_disable_unprepare(sdc->clk);
err_reset_assert:
@@ -1355,6 +1429,7 @@ static int sun6i_dma_remove(struct platform_device *pdev)
sun6i_kill_tasklet(sdc);
+ clk_disable_unprepare(sdc->clk_mbus);
clk_disable_unprepare(sdc->clk);
reset_control_assert(sdc->rstc);
diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c
index ef317c90fbe1..3a45079d11ec 100644
--- a/drivers/dma/tegra20-apb-dma.c
+++ b/drivers/dma/tegra20-apb-dma.c
@@ -152,6 +152,7 @@ struct tegra_dma_sg_req {
bool last_sg;
struct list_head node;
struct tegra_dma_desc *dma_desc;
+ unsigned int words_xferred;
};
/*
@@ -496,6 +497,7 @@ static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc,
tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR,
nsg_req->ch_regs.csr | TEGRA_APBDMA_CSR_ENB);
nsg_req->configured = true;
+ nsg_req->words_xferred = 0;
tegra_dma_resume(tdc);
}
@@ -511,6 +513,7 @@ static void tdc_start_head_req(struct tegra_dma_channel *tdc)
typeof(*sg_req), node);
tegra_dma_start(tdc, sg_req);
sg_req->configured = true;
+ sg_req->words_xferred = 0;
tdc->busy = true;
}
@@ -638,6 +641,8 @@ static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc,
list_add_tail(&dma_desc->cb_node, &tdc->cb_desc);
dma_desc->cb_count++;
+ sgreq->words_xferred = 0;
+
/* If not last req then put at end of pending list */
if (!list_is_last(&sgreq->node, &tdc->pending_sg_req)) {
list_move_tail(&sgreq->node, &tdc->pending_sg_req);
@@ -797,6 +802,65 @@ skip_dma_stop:
return 0;
}
+static unsigned int tegra_dma_sg_bytes_xferred(struct tegra_dma_channel *tdc,
+ struct tegra_dma_sg_req *sg_req)
+{
+ unsigned long status, wcount = 0;
+
+ if (!list_is_first(&sg_req->node, &tdc->pending_sg_req))
+ return 0;
+
+ if (tdc->tdma->chip_data->support_separate_wcount_reg)
+ wcount = tdc_read(tdc, TEGRA_APBDMA_CHAN_WORD_TRANSFER);
+
+ status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
+
+ if (!tdc->tdma->chip_data->support_separate_wcount_reg)
+ wcount = status;
+
+ if (status & TEGRA_APBDMA_STATUS_ISE_EOC)
+ return sg_req->req_len;
+
+ wcount = get_current_xferred_count(tdc, sg_req, wcount);
+
+ if (!wcount) {
+ /*
+ * If wcount wasn't ever polled for this SG before, then
+ * simply assume that transfer hasn't started yet.
+ *
+ * Otherwise it's the end of the transfer.
+ *
+ * The alternative would be to poll the status register
+ * until EOC bit is set or wcount goes UP. That's so
+ * because EOC bit is getting set only after the last
+ * burst's completion and counter is less than the actual
+ * transfer size by 4 bytes. The counter value wraps around
+ * in a cyclic mode before EOC is set(!), so we can't easily
+ * distinguish start of transfer from its end.
+ */
+ if (sg_req->words_xferred)
+ wcount = sg_req->req_len - 4;
+
+ } else if (wcount < sg_req->words_xferred) {
+ /*
+ * This case will never happen for a non-cyclic transfer.
+ *
+ * For a cyclic transfer, although it is possible for the
+ * next transfer to have already started (resetting the word
+ * count), this case should still not happen because we should
+ * have detected that the EOC bit is set and hence the transfer
+ * was completed.
+ */
+ WARN_ON_ONCE(1);
+
+ wcount = sg_req->req_len - 4;
+ } else {
+ sg_req->words_xferred = wcount;
+ }
+
+ return wcount;
+}
+
static enum dma_status tegra_dma_tx_status(struct dma_chan *dc,
dma_cookie_t cookie, struct dma_tx_state *txstate)
{
@@ -806,6 +870,7 @@ static enum dma_status tegra_dma_tx_status(struct dma_chan *dc,
enum dma_status ret;
unsigned long flags;
unsigned int residual;
+ unsigned int bytes = 0;
ret = dma_cookie_status(dc, cookie, txstate);
if (ret == DMA_COMPLETE)
@@ -825,6 +890,7 @@ static enum dma_status tegra_dma_tx_status(struct dma_chan *dc,
list_for_each_entry(sg_req, &tdc->pending_sg_req, node) {
dma_desc = sg_req->dma_desc;
if (dma_desc->txd.cookie == cookie) {
+ bytes = tegra_dma_sg_bytes_xferred(tdc, sg_req);
ret = dma_desc->dma_status;
goto found;
}
@@ -836,7 +902,7 @@ static enum dma_status tegra_dma_tx_status(struct dma_chan *dc,
found:
if (dma_desc && txstate) {
residual = dma_desc->bytes_requested -
- (dma_desc->bytes_transferred %
+ ((dma_desc->bytes_transferred + bytes) %
dma_desc->bytes_requested);
dma_set_residue(txstate, residual);
}
@@ -977,8 +1043,12 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg(
csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT;
}
- if (flags & DMA_PREP_INTERRUPT)
+ if (flags & DMA_PREP_INTERRUPT) {
csr |= TEGRA_APBDMA_CSR_IE_EOC;
+ } else {
+ WARN_ON_ONCE(1);
+ return NULL;
+ }
apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1;
@@ -1120,8 +1190,12 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic(
csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT;
}
- if (flags & DMA_PREP_INTERRUPT)
+ if (flags & DMA_PREP_INTERRUPT) {
csr |= TEGRA_APBDMA_CSR_IE_EOC;
+ } else {
+ WARN_ON_ONCE(1);
+ return NULL;
+ }
apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1;
@@ -1433,12 +1507,7 @@ static int tegra_dma_probe(struct platform_device *pdev)
BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
- /*
- * XXX The hardware appears to support
- * DMA_RESIDUE_GRANULARITY_BURST-level reporting, but it's
- * only used by this driver during tegra_dma_terminate_all()
- */
- tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
+ tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
tdma->dma_dev.device_config = tegra_dma_slave_config;
tdma->dma_dev.device_terminate_all = tegra_dma_terminate_all;
tdma->dma_dev.device_tx_status = tegra_dma_tx_status;
diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c
index 2805853e963f..5f8adf5c1f20 100644
--- a/drivers/dma/tegra210-adma.c
+++ b/drivers/dma/tegra210-adma.c
@@ -42,12 +42,8 @@
#define ADMA_CH_CONFIG_MAX_BUFS 8
#define ADMA_CH_FIFO_CTRL 0x2c
-#define TEGRA210_ADMA_CH_FIFO_CTRL_OFLWTHRES(val) (((val) & 0xf) << 24)
-#define TEGRA210_ADMA_CH_FIFO_CTRL_STRVTHRES(val) (((val) & 0xf) << 16)
#define TEGRA210_ADMA_CH_FIFO_CTRL_TXSIZE(val) (((val) & 0xf) << 8)
#define TEGRA210_ADMA_CH_FIFO_CTRL_RXSIZE(val) ((val) & 0xf)
-#define TEGRA186_ADMA_CH_FIFO_CTRL_OFLWTHRES(val) (((val) & 0x1f) << 24)
-#define TEGRA186_ADMA_CH_FIFO_CTRL_STRVTHRES(val) (((val) & 0x1f) << 16)
#define TEGRA186_ADMA_CH_FIFO_CTRL_TXSIZE(val) (((val) & 0x1f) << 8)
#define TEGRA186_ADMA_CH_FIFO_CTRL_RXSIZE(val) ((val) & 0x1f)
@@ -64,14 +60,10 @@
#define TEGRA_ADMA_BURST_COMPLETE_TIME 20
-#define TEGRA210_FIFO_CTRL_DEFAULT (TEGRA210_ADMA_CH_FIFO_CTRL_OFLWTHRES(1) | \
- TEGRA210_ADMA_CH_FIFO_CTRL_STRVTHRES(1) | \
- TEGRA210_ADMA_CH_FIFO_CTRL_TXSIZE(3) | \
+#define TEGRA210_FIFO_CTRL_DEFAULT (TEGRA210_ADMA_CH_FIFO_CTRL_TXSIZE(3) | \
TEGRA210_ADMA_CH_FIFO_CTRL_RXSIZE(3))
-#define TEGRA186_FIFO_CTRL_DEFAULT (TEGRA186_ADMA_CH_FIFO_CTRL_OFLWTHRES(1) | \
- TEGRA186_ADMA_CH_FIFO_CTRL_STRVTHRES(1) | \
- TEGRA186_ADMA_CH_FIFO_CTRL_TXSIZE(3) | \
+#define TEGRA186_FIFO_CTRL_DEFAULT (TEGRA186_ADMA_CH_FIFO_CTRL_TXSIZE(3) | \
TEGRA186_ADMA_CH_FIFO_CTRL_RXSIZE(3))
#define ADMA_CH_REG_FIELD_VAL(val, mask, shift) (((val) & mask) << shift)
@@ -712,7 +704,7 @@ static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec,
return chan;
}
-static int tegra_adma_runtime_suspend(struct device *dev)
+static int __maybe_unused tegra_adma_runtime_suspend(struct device *dev)
{
struct tegra_adma *tdma = dev_get_drvdata(dev);
struct tegra_adma_chan_regs *ch_reg;
@@ -744,7 +736,7 @@ clk_disable:
return 0;
}
-static int tegra_adma_runtime_resume(struct device *dev)
+static int __maybe_unused tegra_adma_runtime_resume(struct device *dev)
{
struct tegra_adma *tdma = dev_get_drvdata(dev);
struct tegra_adma_chan_regs *ch_reg;
diff --git a/drivers/dma/ti/dma-crossbar.c b/drivers/dma/ti/dma-crossbar.c
index ad2f0a4cd6a4..f255056696ee 100644
--- a/drivers/dma/ti/dma-crossbar.c
+++ b/drivers/dma/ti/dma-crossbar.c
@@ -391,8 +391,10 @@ static int ti_dra7_xbar_probe(struct platform_device *pdev)
ret = of_property_read_u32_array(node, pname, (u32 *)rsv_events,
nelm * 2);
- if (ret)
+ if (ret) {
+ kfree(rsv_events);
return ret;
+ }
for (i = 0; i < nelm; i++) {
ti_dra7_xbar_reserve(rsv_events[i][0], rsv_events[i][1],
diff --git a/drivers/dma/ti/edma.c b/drivers/dma/ti/edma.c
index ceabdea40ae0..ba7c4f07fcd6 100644
--- a/drivers/dma/ti/edma.c
+++ b/drivers/dma/ti/edma.c
@@ -15,7 +15,7 @@
#include <linux/dmaengine.h>
#include <linux/dma-mapping.h>
-#include <linux/edma.h>
+#include <linux/bitmap.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/interrupt.h>
@@ -133,6 +133,17 @@
#define EDMA_CONT_PARAMS_FIXED_EXACT 1002
#define EDMA_CONT_PARAMS_FIXED_NOT_EXACT 1003
+/*
+ * 64bit array registers are split into two 32bit registers:
+ * reg0: channel/event 0-31
+ * reg1: channel/event 32-63
+ *
+ * bit 5 in the channel number tells the array index (0/1)
+ * bit 0-4 (0x1f) is the bit offset within the register
+ */
+#define EDMA_REG_ARRAY_INDEX(channel) ((channel) >> 5)
+#define EDMA_CHANNEL_BIT(channel) (BIT((channel) & 0x1f))
+
/* PaRAM slots are laid out like this */
struct edmacc_param {
u32 opt;
@@ -169,6 +180,7 @@ struct edma_desc {
struct list_head node;
enum dma_transfer_direction direction;
int cyclic;
+ bool polled;
int absync;
int pset_nr;
struct edma_chan *echan;
@@ -412,12 +424,6 @@ static inline void edma_param_or(struct edma_cc *ecc, int offset, int param_no,
edma_or(ecc, EDMA_PARM + offset + (param_no << 5), or);
}
-static inline void edma_set_bits(int offset, int len, unsigned long *p)
-{
- for (; len > 0; len--)
- set_bit(offset + (len - 1), p);
-}
-
static void edma_assign_priority_to_queue(struct edma_cc *ecc, int queue_no,
int priority)
{
@@ -441,15 +447,14 @@ static void edma_setup_interrupt(struct edma_chan *echan, bool enable)
{
struct edma_cc *ecc = echan->ecc;
int channel = EDMA_CHAN_SLOT(echan->ch_num);
+ int idx = EDMA_REG_ARRAY_INDEX(channel);
+ int ch_bit = EDMA_CHANNEL_BIT(channel);
if (enable) {
- edma_shadow0_write_array(ecc, SH_ICR, channel >> 5,
- BIT(channel & 0x1f));
- edma_shadow0_write_array(ecc, SH_IESR, channel >> 5,
- BIT(channel & 0x1f));
+ edma_shadow0_write_array(ecc, SH_ICR, idx, ch_bit);
+ edma_shadow0_write_array(ecc, SH_IESR, idx, ch_bit);
} else {
- edma_shadow0_write_array(ecc, SH_IECR, channel >> 5,
- BIT(channel & 0x1f));
+ edma_shadow0_write_array(ecc, SH_IECR, idx, ch_bit);
}
}
@@ -587,26 +592,26 @@ static void edma_start(struct edma_chan *echan)
{
struct edma_cc *ecc = echan->ecc;
int channel = EDMA_CHAN_SLOT(echan->ch_num);
- int j = (channel >> 5);
- unsigned int mask = BIT(channel & 0x1f);
+ int idx = EDMA_REG_ARRAY_INDEX(channel);
+ int ch_bit = EDMA_CHANNEL_BIT(channel);
if (!echan->hw_triggered) {
/* EDMA channels without event association */
- dev_dbg(ecc->dev, "ESR%d %08x\n", j,
- edma_shadow0_read_array(ecc, SH_ESR, j));
- edma_shadow0_write_array(ecc, SH_ESR, j, mask);
+ dev_dbg(ecc->dev, "ESR%d %08x\n", idx,
+ edma_shadow0_read_array(ecc, SH_ESR, idx));
+ edma_shadow0_write_array(ecc, SH_ESR, idx, ch_bit);
} else {
/* EDMA channel with event association */
- dev_dbg(ecc->dev, "ER%d %08x\n", j,
- edma_shadow0_read_array(ecc, SH_ER, j));
+ dev_dbg(ecc->dev, "ER%d %08x\n", idx,
+ edma_shadow0_read_array(ecc, SH_ER, idx));
/* Clear any pending event or error */
- edma_write_array(ecc, EDMA_ECR, j, mask);
- edma_write_array(ecc, EDMA_EMCR, j, mask);
+ edma_write_array(ecc, EDMA_ECR, idx, ch_bit);
+ edma_write_array(ecc, EDMA_EMCR, idx, ch_bit);
/* Clear any SER */
- edma_shadow0_write_array(ecc, SH_SECR, j, mask);
- edma_shadow0_write_array(ecc, SH_EESR, j, mask);
- dev_dbg(ecc->dev, "EER%d %08x\n", j,
- edma_shadow0_read_array(ecc, SH_EER, j));
+ edma_shadow0_write_array(ecc, SH_SECR, idx, ch_bit);
+ edma_shadow0_write_array(ecc, SH_EESR, idx, ch_bit);
+ dev_dbg(ecc->dev, "EER%d %08x\n", idx,
+ edma_shadow0_read_array(ecc, SH_EER, idx));
}
}
@@ -614,19 +619,19 @@ static void edma_stop(struct edma_chan *echan)
{
struct edma_cc *ecc = echan->ecc;
int channel = EDMA_CHAN_SLOT(echan->ch_num);
- int j = (channel >> 5);
- unsigned int mask = BIT(channel & 0x1f);
+ int idx = EDMA_REG_ARRAY_INDEX(channel);
+ int ch_bit = EDMA_CHANNEL_BIT(channel);
- edma_shadow0_write_array(ecc, SH_EECR, j, mask);
- edma_shadow0_write_array(ecc, SH_ECR, j, mask);
- edma_shadow0_write_array(ecc, SH_SECR, j, mask);
- edma_write_array(ecc, EDMA_EMCR, j, mask);
+ edma_shadow0_write_array(ecc, SH_EECR, idx, ch_bit);
+ edma_shadow0_write_array(ecc, SH_ECR, idx, ch_bit);
+ edma_shadow0_write_array(ecc, SH_SECR, idx, ch_bit);
+ edma_write_array(ecc, EDMA_EMCR, idx, ch_bit);
/* clear possibly pending completion interrupt */
- edma_shadow0_write_array(ecc, SH_ICR, j, mask);
+ edma_shadow0_write_array(ecc, SH_ICR, idx, ch_bit);
- dev_dbg(ecc->dev, "EER%d %08x\n", j,
- edma_shadow0_read_array(ecc, SH_EER, j));
+ dev_dbg(ecc->dev, "EER%d %08x\n", idx,
+ edma_shadow0_read_array(ecc, SH_EER, idx));
/* REVISIT: consider guarding against inappropriate event
* chaining by overwriting with dummy_paramset.
@@ -640,45 +645,49 @@ static void edma_stop(struct edma_chan *echan)
static void edma_pause(struct edma_chan *echan)
{
int channel = EDMA_CHAN_SLOT(echan->ch_num);
- unsigned int mask = BIT(channel & 0x1f);
- edma_shadow0_write_array(echan->ecc, SH_EECR, channel >> 5, mask);
+ edma_shadow0_write_array(echan->ecc, SH_EECR,
+ EDMA_REG_ARRAY_INDEX(channel),
+ EDMA_CHANNEL_BIT(channel));
}
/* Re-enable EDMA hardware events on the specified channel. */
static void edma_resume(struct edma_chan *echan)
{
int channel = EDMA_CHAN_SLOT(echan->ch_num);
- unsigned int mask = BIT(channel & 0x1f);
- edma_shadow0_write_array(echan->ecc, SH_EESR, channel >> 5, mask);
+ edma_shadow0_write_array(echan->ecc, SH_EESR,
+ EDMA_REG_ARRAY_INDEX(channel),
+ EDMA_CHANNEL_BIT(channel));
}
static void edma_trigger_channel(struct edma_chan *echan)
{
struct edma_cc *ecc = echan->ecc;
int channel = EDMA_CHAN_SLOT(echan->ch_num);
- unsigned int mask = BIT(channel & 0x1f);
+ int idx = EDMA_REG_ARRAY_INDEX(channel);
+ int ch_bit = EDMA_CHANNEL_BIT(channel);
- edma_shadow0_write_array(ecc, SH_ESR, (channel >> 5), mask);
+ edma_shadow0_write_array(ecc, SH_ESR, idx, ch_bit);
- dev_dbg(ecc->dev, "ESR%d %08x\n", (channel >> 5),
- edma_shadow0_read_array(ecc, SH_ESR, (channel >> 5)));
+ dev_dbg(ecc->dev, "ESR%d %08x\n", idx,
+ edma_shadow0_read_array(ecc, SH_ESR, idx));
}
static void edma_clean_channel(struct edma_chan *echan)
{
struct edma_cc *ecc = echan->ecc;
int channel = EDMA_CHAN_SLOT(echan->ch_num);
- int j = (channel >> 5);
- unsigned int mask = BIT(channel & 0x1f);
+ int idx = EDMA_REG_ARRAY_INDEX(channel);
+ int ch_bit = EDMA_CHANNEL_BIT(channel);
- dev_dbg(ecc->dev, "EMR%d %08x\n", j, edma_read_array(ecc, EDMA_EMR, j));
- edma_shadow0_write_array(ecc, SH_ECR, j, mask);
+ dev_dbg(ecc->dev, "EMR%d %08x\n", idx,
+ edma_read_array(ecc, EDMA_EMR, idx));
+ edma_shadow0_write_array(ecc, SH_ECR, idx, ch_bit);
/* Clear the corresponding EMR bits */
- edma_write_array(ecc, EDMA_EMCR, j, mask);
+ edma_write_array(ecc, EDMA_EMCR, idx, ch_bit);
/* Clear any SER */
- edma_shadow0_write_array(ecc, SH_SECR, j, mask);
+ edma_shadow0_write_array(ecc, SH_SECR, idx, ch_bit);
edma_write(ecc, EDMA_CCERRCLR, BIT(16) | BIT(1) | BIT(0));
}
@@ -708,7 +717,8 @@ static int edma_alloc_channel(struct edma_chan *echan,
int channel = EDMA_CHAN_SLOT(echan->ch_num);
/* ensure access through shadow region 0 */
- edma_or_array2(ecc, EDMA_DRAE, 0, channel >> 5, BIT(channel & 0x1f));
+ edma_or_array2(ecc, EDMA_DRAE, 0, EDMA_REG_ARRAY_INDEX(channel),
+ EDMA_CHANNEL_BIT(channel));
/* ensure no events are pending */
edma_stop(echan);
@@ -1011,6 +1021,7 @@ static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
src_cidx = cidx;
dst_bidx = acnt;
dst_cidx = cidx;
+ epset->addr = src_addr;
} else {
dev_err(dev, "%s: direction not implemented yet\n", __func__);
return -EINVAL;
@@ -1211,8 +1222,9 @@ static struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
edesc->pset[0].param.opt |= ITCCHEN;
if (nslots == 1) {
- /* Enable transfer complete interrupt */
- edesc->pset[0].param.opt |= TCINTEN;
+ /* Enable transfer complete interrupt if requested */
+ if (tx_flags & DMA_PREP_INTERRUPT)
+ edesc->pset[0].param.opt |= TCINTEN;
} else {
/* Enable transfer complete chaining for the first slot */
edesc->pset[0].param.opt |= TCCHEN;
@@ -1239,9 +1251,14 @@ static struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
}
edesc->pset[1].param.opt |= ITCCHEN;
- edesc->pset[1].param.opt |= TCINTEN;
+ /* Enable transfer complete interrupt if requested */
+ if (tx_flags & DMA_PREP_INTERRUPT)
+ edesc->pset[1].param.opt |= TCINTEN;
}
+ if (!(tx_flags & DMA_PREP_INTERRUPT))
+ edesc->polled = true;
+
return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
}
@@ -1721,7 +1738,11 @@ static u32 edma_residue(struct edma_desc *edesc)
int loop_count = EDMA_MAX_TR_WAIT_LOOPS;
struct edma_chan *echan = edesc->echan;
struct edma_pset *pset = edesc->pset;
- dma_addr_t done, pos;
+ dma_addr_t done, pos, pos_old;
+ int channel = EDMA_CHAN_SLOT(echan->ch_num);
+ int idx = EDMA_REG_ARRAY_INDEX(channel);
+ int ch_bit = EDMA_CHANNEL_BIT(channel);
+ int event_reg;
int i;
/*
@@ -1734,16 +1755,20 @@ static u32 edma_residue(struct edma_desc *edesc)
* "pos" may represent a transfer request that is still being
* processed by the EDMACC or EDMATC. We will busy wait until
* any one of the situations occurs:
- * 1. the DMA hardware is idle
- * 2. a new transfer request is setup
+ * 1. while and event is pending for the channel
+ * 2. a position updated
* 3. we hit the loop limit
*/
- while (edma_read(echan->ecc, EDMA_CCSTAT) & EDMA_CCSTAT_ACTV) {
- /* check if a new transfer request is setup */
- if (edma_get_position(echan->ecc,
- echan->slot[0], dst) != pos) {
+ if (is_slave_direction(edesc->direction))
+ event_reg = SH_ER;
+ else
+ event_reg = SH_ESR;
+
+ pos_old = pos;
+ while (edma_shadow0_read_array(echan->ecc, event_reg, idx) & ch_bit) {
+ pos = edma_get_position(echan->ecc, echan->slot[0], dst);
+ if (pos != pos_old)
break;
- }
if (!--loop_count) {
dev_dbg_ratelimited(echan->vchan.chan.device->dev,
@@ -1769,6 +1794,12 @@ static u32 edma_residue(struct edma_desc *edesc)
}
/*
+ * If the position is 0, then EDMA loaded the closing dummy slot, the
+ * transfer is completed
+ */
+ if (!pos)
+ return 0;
+ /*
* For SG operation we catch up with the last processed
* status.
*/
@@ -1796,19 +1827,46 @@ static enum dma_status edma_tx_status(struct dma_chan *chan,
struct dma_tx_state *txstate)
{
struct edma_chan *echan = to_edma_chan(chan);
- struct virt_dma_desc *vdesc;
+ struct dma_tx_state txstate_tmp;
enum dma_status ret;
unsigned long flags;
ret = dma_cookie_status(chan, cookie, txstate);
- if (ret == DMA_COMPLETE || !txstate)
+
+ if (ret == DMA_COMPLETE)
return ret;
+ /* Provide a dummy dma_tx_state for completion checking */
+ if (!txstate)
+ txstate = &txstate_tmp;
+
spin_lock_irqsave(&echan->vchan.lock, flags);
- if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie)
+ if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie) {
txstate->residue = edma_residue(echan->edesc);
- else if ((vdesc = vchan_find_desc(&echan->vchan, cookie)))
- txstate->residue = to_edma_desc(&vdesc->tx)->residue;
+ } else {
+ struct virt_dma_desc *vdesc = vchan_find_desc(&echan->vchan,
+ cookie);
+
+ if (vdesc)
+ txstate->residue = to_edma_desc(&vdesc->tx)->residue;
+ else
+ txstate->residue = 0;
+ }
+
+ /*
+ * Mark the cookie completed if the residue is 0 for non cyclic
+ * transfers
+ */
+ if (ret != DMA_COMPLETE && !txstate->residue &&
+ echan->edesc && echan->edesc->polled &&
+ echan->edesc->vdesc.tx.cookie == cookie) {
+ edma_stop(echan);
+ vchan_cookie_complete(&echan->edesc->vdesc);
+ echan->edesc = NULL;
+ edma_execute(echan);
+ ret = DMA_COMPLETE;
+ }
+
spin_unlock_irqrestore(&echan->vchan.lock, flags);
return ret;
@@ -2185,11 +2243,13 @@ static struct dma_chan *of_edma_xlate(struct of_phandle_args *dma_spec,
}
#endif
+static bool edma_filter_fn(struct dma_chan *chan, void *param);
+
static int edma_probe(struct platform_device *pdev)
{
struct edma_soc_info *info = pdev->dev.platform_data;
s8 (*queue_priority_mapping)[2];
- int i, off, ln;
+ int i, off;
const s16 (*rsv_slots)[2];
const s16 (*xbar_chans)[2];
int irq;
@@ -2273,21 +2333,22 @@ static int edma_probe(struct platform_device *pdev)
ecc->default_queue = info->default_queue;
- for (i = 0; i < ecc->num_slots; i++)
- edma_write_slot(ecc, i, &dummy_paramset);
-
if (info->rsv) {
/* Set the reserved slots in inuse list */
rsv_slots = info->rsv->rsv_slots;
if (rsv_slots) {
- for (i = 0; rsv_slots[i][0] != -1; i++) {
- off = rsv_slots[i][0];
- ln = rsv_slots[i][1];
- edma_set_bits(off, ln, ecc->slot_inuse);
- }
+ for (i = 0; rsv_slots[i][0] != -1; i++)
+ bitmap_set(ecc->slot_inuse, rsv_slots[i][0],
+ rsv_slots[i][1]);
}
}
+ for (i = 0; i < ecc->num_slots; i++) {
+ /* Reset only unused - not reserved - paRAM slots */
+ if (!test_bit(i, ecc->slot_inuse))
+ edma_write_slot(ecc, i, &dummy_paramset);
+ }
+
/* Clear the xbar mapped channels in unused list */
xbar_chans = info->xbar_chans;
if (xbar_chans) {
@@ -2366,11 +2427,10 @@ static int edma_probe(struct platform_device *pdev)
edma_assign_priority_to_queue(ecc, queue_priority_mapping[i][0],
queue_priority_mapping[i][1]);
- for (i = 0; i < ecc->num_region; i++) {
- edma_write_array2(ecc, EDMA_DRAE, i, 0, 0x0);
- edma_write_array2(ecc, EDMA_DRAE, i, 1, 0x0);
- edma_write_array(ecc, EDMA_QRAE, i, 0x0);
- }
+ edma_write_array2(ecc, EDMA_DRAE, 0, 0, 0x0);
+ edma_write_array2(ecc, EDMA_DRAE, 0, 1, 0x0);
+ edma_write_array(ecc, EDMA_QRAE, 0, 0x0);
+
ecc->info = info;
/* Init the dma device and channels */
@@ -2482,8 +2542,9 @@ static int edma_pm_resume(struct device *dev)
for (i = 0; i < ecc->num_channels; i++) {
if (echan[i].alloced) {
/* ensure access through shadow region 0 */
- edma_or_array2(ecc, EDMA_DRAE, 0, i >> 5,
- BIT(i & 0x1f));
+ edma_or_array2(ecc, EDMA_DRAE, 0,
+ EDMA_REG_ARRAY_INDEX(i),
+ EDMA_CHANNEL_BIT(i));
edma_setup_interrupt(&echan[i], true);
@@ -2524,7 +2585,7 @@ static struct platform_driver edma_tptc_driver = {
},
};
-bool edma_filter_fn(struct dma_chan *chan, void *param)
+static bool edma_filter_fn(struct dma_chan *chan, void *param)
{
bool match = false;
@@ -2539,7 +2600,6 @@ bool edma_filter_fn(struct dma_chan *chan, void *param)
}
return match;
}
-EXPORT_SYMBOL(edma_filter_fn);
static int edma_init(void)
{
diff --git a/drivers/dma/ti/omap-dma.c b/drivers/dma/ti/omap-dma.c
index ba2489d4ea24..6b6ba238b81a 100644
--- a/drivers/dma/ti/omap-dma.c
+++ b/drivers/dma/ti/omap-dma.c
@@ -91,6 +91,7 @@ struct omap_desc {
bool using_ll;
enum dma_transfer_direction dir;
dma_addr_t dev_addr;
+ bool polled;
int32_t fi; /* for OMAP_DMA_SYNC_PACKET / double indexing */
int16_t ei; /* for double indexing */
@@ -202,6 +203,7 @@ static const unsigned es_bytes[] = {
[CSDP_DATA_TYPE_32] = 4,
};
+static bool omap_dma_filter_fn(struct dma_chan *chan, void *param);
static struct of_dma_filter_info omap_dma_info = {
.filter_fn = omap_dma_filter_fn,
};
@@ -812,31 +814,22 @@ static enum dma_status omap_dma_tx_status(struct dma_chan *chan,
dma_cookie_t cookie, struct dma_tx_state *txstate)
{
struct omap_chan *c = to_omap_dma_chan(chan);
- struct virt_dma_desc *vd;
enum dma_status ret;
unsigned long flags;
+ struct omap_desc *d = NULL;
ret = dma_cookie_status(chan, cookie, txstate);
-
- if (!c->paused && c->running) {
- uint32_t ccr = omap_dma_chan_read(c, CCR);
- /*
- * The channel is no longer active, set the return value
- * accordingly
- */
- if (!(ccr & CCR_ENABLE))
- ret = DMA_COMPLETE;
- }
-
- if (ret == DMA_COMPLETE || !txstate)
+ if (ret == DMA_COMPLETE)
return ret;
spin_lock_irqsave(&c->vc.lock, flags);
- vd = vchan_find_desc(&c->vc, cookie);
- if (vd) {
- txstate->residue = omap_dma_desc_size(to_omap_dma_desc(&vd->tx));
- } else if (c->desc && c->desc->vd.tx.cookie == cookie) {
- struct omap_desc *d = c->desc;
+ if (c->desc && c->desc->vd.tx.cookie == cookie)
+ d = c->desc;
+
+ if (!txstate)
+ goto out;
+
+ if (d) {
dma_addr_t pos;
if (d->dir == DMA_MEM_TO_DEV)
@@ -848,10 +841,31 @@ static enum dma_status omap_dma_tx_status(struct dma_chan *chan,
txstate->residue = omap_dma_desc_size_pos(d, pos);
} else {
- txstate->residue = 0;
+ struct virt_dma_desc *vd = vchan_find_desc(&c->vc, cookie);
+
+ if (vd)
+ txstate->residue = omap_dma_desc_size(
+ to_omap_dma_desc(&vd->tx));
+ else
+ txstate->residue = 0;
}
- if (ret == DMA_IN_PROGRESS && c->paused)
+
+out:
+ if (ret == DMA_IN_PROGRESS && c->paused) {
ret = DMA_PAUSED;
+ } else if (d && d->polled && c->running) {
+ uint32_t ccr = omap_dma_chan_read(c, CCR);
+ /*
+ * The channel is no longer active, set the return value
+ * accordingly and mark it as completed
+ */
+ if (!(ccr & CCR_ENABLE)) {
+ ret = DMA_COMPLETE;
+ omap_dma_start_desc(c);
+ vchan_cookie_complete(&d->vd);
+ }
+ }
+
spin_unlock_irqrestore(&c->vc.lock, flags);
return ret;
@@ -1178,7 +1192,10 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy(
d->ccr = c->ccr;
d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC;
- d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
+ if (tx_flags & DMA_PREP_INTERRUPT)
+ d->cicr |= CICR_FRAME_IE;
+ else
+ d->polled = true;
d->csdp = data_type;
@@ -1234,7 +1251,7 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved(
if (src_icg) {
d->ccr |= CCR_SRC_AMODE_DBLIDX;
d->ei = 1;
- d->fi = src_icg;
+ d->fi = src_icg + 1;
} else if (xt->src_inc) {
d->ccr |= CCR_SRC_AMODE_POSTINC;
d->fi = 0;
@@ -1249,7 +1266,7 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved(
if (dst_icg) {
d->ccr |= CCR_DST_AMODE_DBLIDX;
sg->ei = 1;
- sg->fi = dst_icg;
+ sg->fi = dst_icg + 1;
} else if (xt->dst_inc) {
d->ccr |= CCR_DST_AMODE_POSTINC;
sg->fi = 0;
@@ -1540,8 +1557,10 @@ static int omap_dma_probe(struct platform_device *pdev)
rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq,
IRQF_SHARED, "omap-dma-engine", od);
- if (rc)
+ if (rc) {
+ omap_dma_free(od);
return rc;
+ }
}
if (omap_dma_glbl_read(od, CAPS_0) & CAPS_0_SUPPORT_LL123)
@@ -1637,7 +1656,7 @@ static struct platform_driver omap_dma_driver = {
},
};
-bool omap_dma_filter_fn(struct dma_chan *chan, void *param)
+static bool omap_dma_filter_fn(struct dma_chan *chan, void *param)
{
if (chan->device->dev->driver == &omap_dma_driver.driver) {
struct omap_dmadev *od = to_omap_dma_dev(chan->device);
@@ -1651,7 +1670,6 @@ bool omap_dma_filter_fn(struct dma_chan *chan, void *param)
}
return false;
}
-EXPORT_SYMBOL_GPL(omap_dma_filter_fn);
static int omap_dma_init(void)
{
diff --git a/drivers/dma/uniphier-mdmac.c b/drivers/dma/uniphier-mdmac.c
index ec65a7430dc4..fde54687856b 100644
--- a/drivers/dma/uniphier-mdmac.c
+++ b/drivers/dma/uniphier-mdmac.c
@@ -354,11 +354,8 @@ static int uniphier_mdmac_chan_init(struct platform_device *pdev,
int irq, ret;
irq = platform_get_irq(pdev, chan_id);
- if (irq < 0) {
- dev_err(&pdev->dev, "failed to get IRQ number for ch%d\n",
- chan_id);
+ if (irq < 0)
return irq;
- }
irq_name = devm_kasprintf(dev, GFP_KERNEL, "uniphier-mio-dmac-ch%d",
chan_id);
diff --git a/drivers/dma/virt-dma.c b/drivers/dma/virt-dma.c
index bb5390847257..ec4adf4260a0 100644
--- a/drivers/dma/virt-dma.c
+++ b/drivers/dma/virt-dma.c
@@ -98,7 +98,7 @@ static void vchan_complete(unsigned long arg)
}
spin_unlock_irq(&vc->lock);
- dmaengine_desc_callback_invoke(&cb, NULL);
+ dmaengine_desc_callback_invoke(&cb, &vd->tx_result);
list_for_each_entry_safe(vd, _vd, &head, node) {
dmaengine_desc_get_callback(&vd->tx, &cb);
@@ -106,7 +106,7 @@ static void vchan_complete(unsigned long arg)
list_del(&vd->node);
vchan_vdesc_fini(vd);
- dmaengine_desc_callback_invoke(&cb, NULL);
+ dmaengine_desc_callback_invoke(&cb, &vd->tx_result);
}
}
diff --git a/drivers/dma/virt-dma.h b/drivers/dma/virt-dma.h
index 23342ca23d4a..ab158bac03a7 100644
--- a/drivers/dma/virt-dma.h
+++ b/drivers/dma/virt-dma.h
@@ -14,6 +14,7 @@
struct virt_dma_desc {
struct dma_async_tx_descriptor tx;
+ struct dmaengine_result tx_result;
/* protected by vc.lock */
struct list_head node;
};
@@ -62,6 +63,9 @@ static inline struct dma_async_tx_descriptor *vchan_tx_prep(struct virt_dma_chan
vd->tx.tx_submit = vchan_tx_submit;
vd->tx.desc_free = vchan_tx_desc_free;
+ vd->tx_result.result = DMA_TRANS_NOERROR;
+ vd->tx_result.residue = 0;
+
spin_lock_irqsave(&vc->lock, flags);
list_add_tail(&vd->node, &vc->desc_allocated);
spin_unlock_irqrestore(&vc->lock, flags);
diff --git a/drivers/dma/xgene-dma.c b/drivers/dma/xgene-dma.c
index 957c269ce1fd..cd60fa6d6750 100644
--- a/drivers/dma/xgene-dma.c
+++ b/drivers/dma/xgene-dma.c
@@ -1678,20 +1678,16 @@ static int xgene_dma_get_resources(struct platform_device *pdev,
/* Get DMA error interrupt */
irq = platform_get_irq(pdev, 0);
- if (irq <= 0) {
- dev_err(&pdev->dev, "Failed to get Error IRQ\n");
+ if (irq <= 0)
return -ENXIO;
- }
pdma->err_irq = irq;
/* Get DMA Rx ring descriptor interrupts for all DMA channels */
for (i = 1; i <= XGENE_DMA_MAX_CHANNEL; i++) {
irq = platform_get_irq(pdev, i);
- if (irq <= 0) {
- dev_err(&pdev->dev, "Failed to get Rx IRQ\n");
+ if (irq <= 0)
return -ENXIO;
- }
pdma->chan[i - 1].rx_irq = irq;
}
diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c
index 36c092349b5b..e7dc3c4dc8e0 100644
--- a/drivers/dma/xilinx/xilinx_dma.c
+++ b/drivers/dma/xilinx/xilinx_dma.c
@@ -1095,7 +1095,7 @@ static void xilinx_dma_start(struct xilinx_dma_chan *chan)
static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan)
{
struct xilinx_vdma_config *config = &chan->config;
- struct xilinx_dma_tx_descriptor *desc, *tail_desc;
+ struct xilinx_dma_tx_descriptor *desc;
u32 reg, j;
struct xilinx_vdma_tx_segment *segment, *last = NULL;
int i = 0;
@@ -1112,8 +1112,6 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan)
desc = list_first_entry(&chan->pending_list,
struct xilinx_dma_tx_descriptor, node);
- tail_desc = list_last_entry(&chan->pending_list,
- struct xilinx_dma_tx_descriptor, node);
/* Configure the hardware using info in the config structure */
if (chan->has_vflip) {
diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig
index 5e2e0348d460..2a2603bfb918 100644
--- a/drivers/edac/Kconfig
+++ b/drivers/edac/Kconfig
@@ -460,6 +460,12 @@ config EDAC_ALTERA_SDMMC
Support for error detection and correction on the
Altera SDMMC FIFO Memory for Altera SoCs.
+config EDAC_SIFIVE
+ bool "Sifive platform EDAC driver"
+ depends on EDAC=y && RISCV
+ help
+ Support for error detection and correction on the SiFive SoCs.
+
config EDAC_SYNOPSYS
tristate "Synopsys DDR Memory Controller"
depends on ARCH_ZYNQ || ARCH_ZYNQMP
@@ -504,4 +510,11 @@ config EDAC_ASPEED
First, ECC must be configured in the bootloader. Then, this driver
will expose error counters via the EDAC kernel framework.
+config EDAC_BLUEFIELD
+ tristate "Mellanox BlueField Memory ECC"
+ depends on ARM64 && ((MELLANOX_PLATFORM && ACPI) || COMPILE_TEST)
+ help
+ Support for error detection and correction on the
+ Mellanox BlueField SoCs.
+
endif # EDAC
diff --git a/drivers/edac/Makefile b/drivers/edac/Makefile
index 89ad4a84a0f6..d265ff9311f0 100644
--- a/drivers/edac/Makefile
+++ b/drivers/edac/Makefile
@@ -79,8 +79,10 @@ obj-$(CONFIG_EDAC_OCTEON_PCI) += octeon_edac-pci.o
obj-$(CONFIG_EDAC_THUNDERX) += thunderx_edac.o
obj-$(CONFIG_EDAC_ALTERA) += altera_edac.o
+obj-$(CONFIG_EDAC_SIFIVE) += sifive_edac.o
obj-$(CONFIG_EDAC_SYNOPSYS) += synopsys_edac.o
obj-$(CONFIG_EDAC_XGENE) += xgene_edac.o
obj-$(CONFIG_EDAC_TI) += ti_edac.o
obj-$(CONFIG_EDAC_QCOM) += qcom_edac.o
obj-$(CONFIG_EDAC_ASPEED) += aspeed_edac.o
+obj-$(CONFIG_EDAC_BLUEFIELD) += bluefield_edac.o
diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c
index 8816f74a22b4..fbda4b876afd 100644
--- a/drivers/edac/altera_edac.c
+++ b/drivers/edac/altera_edac.c
@@ -222,7 +222,6 @@ static unsigned long get_total_mem(void)
static const struct of_device_id altr_sdram_ctrl_of_match[] = {
{ .compatible = "altr,sdram-edac", .data = &c5_data},
{ .compatible = "altr,sdram-edac-a10", .data = &a10_data},
- { .compatible = "altr,sdram-edac-s10", .data = &a10_data},
{},
};
MODULE_DEVICE_TABLE(of, altr_sdram_ctrl_of_match);
@@ -1170,6 +1169,24 @@ static int __init __maybe_unused altr_init_a10_ecc_device_type(char *compat)
return 0;
}
+/*********************** SDRAM EDAC Device Functions *********************/
+
+#ifdef CONFIG_EDAC_ALTERA_SDRAM
+
+static const struct edac_device_prv_data s10_sdramecc_data = {
+ .setup = altr_check_ecc_deps,
+ .ce_clear_mask = ALTR_S10_ECC_SERRPENA,
+ .ue_clear_mask = ALTR_S10_ECC_DERRPENA,
+ .ecc_enable_mask = ALTR_S10_ECC_EN,
+ .ecc_en_ofst = ALTR_S10_ECC_CTRL_SDRAM_OFST,
+ .ce_set_mask = ALTR_S10_ECC_TSERRA,
+ .ue_set_mask = ALTR_S10_ECC_TDERRA,
+ .set_err_ofst = ALTR_S10_ECC_INTTEST_OFST,
+ .ecc_irq_handler = altr_edac_a10_ecc_irq,
+ .inject_fops = &altr_edac_a10_device_inject_fops,
+};
+#endif /* CONFIG_EDAC_ALTERA_SDRAM */
+
/*********************** OCRAM EDAC Device Functions *********************/
#ifdef CONFIG_EDAC_ALTERA_OCRAM
@@ -1223,8 +1240,31 @@ static const struct edac_device_prv_data ocramecc_data = {
.inject_fops = &altr_edac_device_inject_fops,
};
+static int __maybe_unused
+altr_check_ocram_deps_init(struct altr_edac_device_dev *device)
+{
+ void __iomem *base = device->base;
+ int ret;
+
+ ret = altr_check_ecc_deps(device);
+ if (ret)
+ return ret;
+
+ /* Verify OCRAM has been initialized */
+ if (!ecc_test_bits(ALTR_A10_ECC_INITCOMPLETEA,
+ (base + ALTR_A10_ECC_INITSTAT_OFST)))
+ return -ENODEV;
+
+ /* Enable IRQ on Single Bit Error */
+ writel(ALTR_A10_ECC_SERRINTEN, (base + ALTR_A10_ECC_ERRINTENS_OFST));
+ /* Ensure all writes complete */
+ wmb();
+
+ return 0;
+}
+
static const struct edac_device_prv_data a10_ocramecc_data = {
- .setup = altr_check_ecc_deps,
+ .setup = altr_check_ocram_deps_init,
.ce_clear_mask = ALTR_A10_ECC_SERRPENA,
.ue_clear_mask = ALTR_A10_ECC_DERRPENA,
.irq_status_mask = A10_SYSMGR_ECC_INTSTAT_OCRAM,
@@ -1234,7 +1274,7 @@ static const struct edac_device_prv_data a10_ocramecc_data = {
.ue_set_mask = ALTR_A10_ECC_TDERRA,
.set_err_ofst = ALTR_A10_ECC_INTTEST_OFST,
.ecc_irq_handler = altr_edac_a10_ecc_irq,
- .inject_fops = &altr_edac_a10_device_inject_fops,
+ .inject_fops = &altr_edac_a10_device_inject2_fops,
/*
* OCRAM panic on uncorrectable error because sleep/resume
* functions and FPGA contents are stored in OCRAM. Prefer
@@ -1560,8 +1600,12 @@ static int altr_portb_setup(struct altr_edac_device_dev *device)
dci->mod_name = ecc_name;
dci->dev_name = ecc_name;
- /* Update the IRQs for PortB */
+ /* Update the PortB IRQs - A10 has 4, S10 has 2, Index accordingly */
+#ifdef CONFIG_ARCH_STRATIX10
+ altdev->sb_irq = irq_of_parse_and_map(np, 1);
+#else
altdev->sb_irq = irq_of_parse_and_map(np, 2);
+#endif
if (!altdev->sb_irq) {
edac_printk(KERN_ERR, EDAC_DEVICE, "Error PortB SBIRQ alloc\n");
rc = -ENODEV;
@@ -1576,6 +1620,15 @@ static int altr_portb_setup(struct altr_edac_device_dev *device)
goto err_release_group_1;
}
+#ifdef CONFIG_ARCH_STRATIX10
+ /* Use IRQ to determine SError origin instead of assigning IRQ */
+ rc = of_property_read_u32_index(np, "interrupts", 1, &altdev->db_irq);
+ if (rc) {
+ edac_printk(KERN_ERR, EDAC_DEVICE,
+ "Error PortB DBIRQ alloc\n");
+ goto err_release_group_1;
+ }
+#else
altdev->db_irq = irq_of_parse_and_map(np, 3);
if (!altdev->db_irq) {
edac_printk(KERN_ERR, EDAC_DEVICE, "Error PortB DBIRQ alloc\n");
@@ -1590,6 +1643,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device)
edac_printk(KERN_ERR, EDAC_DEVICE, "PortB DBERR IRQ error\n");
goto err_release_group_1;
}
+#endif
rc = edac_device_add_device(dci);
if (rc) {
@@ -1722,6 +1776,9 @@ static const struct of_device_id altr_edac_a10_device_of_match[] = {
#ifdef CONFIG_EDAC_ALTERA_SDMMC
{ .compatible = "altr,socfpga-sdmmc-ecc", .data = &a10_sdmmcecca_data },
#endif
+#ifdef CONFIG_EDAC_ALTERA_SDRAM
+ { .compatible = "altr,sdram-edac-s10", .data = &s10_sdramecc_data },
+#endif
{},
};
MODULE_DEVICE_TABLE(of, altr_edac_a10_device_of_match);
@@ -1829,6 +1886,7 @@ static void altr_edac_a10_irq_handler(struct irq_desc *desc)
struct altr_arria10_edac *edac = irq_desc_get_handler_data(desc);
struct irq_chip *chip = irq_desc_get_chip(desc);
int irq = irq_desc_get_irq(desc);
+ unsigned long bits;
dberr = (irq == edac->db_irq) ? 1 : 0;
sm_offset = dberr ? A10_SYSMGR_ECC_INTSTAT_DERR_OFST :
@@ -1838,7 +1896,8 @@ static void altr_edac_a10_irq_handler(struct irq_desc *desc)
regmap_read(edac->ecc_mgr_map, sm_offset, &irq_status);
- for_each_set_bit(bit, (unsigned long *)&irq_status, 32) {
+ bits = irq_status;
+ for_each_set_bit(bit, &bits, 32) {
irq = irq_linear_revmap(edac->domain, dberr * 32 + bit);
if (irq)
generic_handle_irq(irq);
@@ -1852,6 +1911,10 @@ static int validate_parent_available(struct device_node *np)
struct device_node *parent;
int ret = 0;
+ /* SDRAM must be present for Linux (implied parent) */
+ if (of_device_is_compatible(np, "altr,sdram-edac-s10"))
+ return 0;
+
/* Ensure parent device is enabled if parent node exists */
parent = of_parse_phandle(np, "altr,ecc-parent", 0);
if (parent && !of_device_is_available(parent))
@@ -1861,6 +1924,22 @@ static int validate_parent_available(struct device_node *np)
return ret;
}
+static int get_s10_sdram_edac_resource(struct device_node *np,
+ struct resource *res)
+{
+ struct device_node *parent;
+ int ret;
+
+ parent = of_parse_phandle(np, "altr,sdr-syscon", 0);
+ if (!parent)
+ return -ENODEV;
+
+ ret = of_address_to_resource(parent, 0, res);
+ of_node_put(parent);
+
+ return ret;
+}
+
static int altr_edac_a10_device_add(struct altr_arria10_edac *edac,
struct device_node *np)
{
@@ -1888,7 +1967,11 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac,
if (!devres_open_group(edac->dev, altr_edac_a10_device_add, GFP_KERNEL))
return -ENOMEM;
- rc = of_address_to_resource(np, 0, &res);
+ if (of_device_is_compatible(np, "altr,sdram-edac-s10"))
+ rc = get_s10_sdram_edac_resource(np, &res);
+ else
+ rc = of_address_to_resource(np, 0, &res);
+
if (rc < 0) {
edac_printk(KERN_ERR, EDAC_DEVICE,
"%s: no resource address\n", ecc_name);
@@ -2194,13 +2277,15 @@ static int altr_edac_a10_probe(struct platform_device *pdev)
of_device_is_compatible(child, "altr,socfpga-dma-ecc") ||
of_device_is_compatible(child, "altr,socfpga-usb-ecc") ||
of_device_is_compatible(child, "altr,socfpga-qspi-ecc") ||
+#ifdef CONFIG_EDAC_ALTERA_SDRAM
+ of_device_is_compatible(child, "altr,sdram-edac-s10") ||
+#endif
of_device_is_compatible(child, "altr,socfpga-sdmmc-ecc"))
altr_edac_a10_device_add(edac, child);
#ifdef CONFIG_EDAC_ALTERA_SDRAM
- else if ((of_device_is_compatible(child, "altr,sdram-edac-a10")) ||
- (of_device_is_compatible(child, "altr,sdram-edac-s10")))
+ else if (of_device_is_compatible(child, "altr,sdram-edac-a10"))
of_platform_populate(pdev->dev.of_node,
altr_sdram_ctrl_of_match,
NULL, &pdev->dev);
diff --git a/drivers/edac/altera_edac.h b/drivers/edac/altera_edac.h
index 55654cc4bcdf..3727e72c8c2e 100644
--- a/drivers/edac/altera_edac.h
+++ b/drivers/edac/altera_edac.h
@@ -289,6 +289,29 @@ struct altr_sdram_mc_data {
#define ALTR_A10_ECC_INIT_WATCHDOG_10US 10000
/************* Stratix10 Defines **************/
+#define ALTR_S10_ECC_CTRL_SDRAM_OFST 0x00
+#define ALTR_S10_ECC_EN BIT(0)
+
+#define ALTR_S10_ECC_ERRINTEN_OFST 0x10
+#define ALTR_S10_ECC_ERRINTENS_OFST 0x14
+#define ALTR_S10_ECC_ERRINTENR_OFST 0x18
+#define ALTR_S10_ECC_SERRINTEN BIT(0)
+
+#define ALTR_S10_ECC_INTMODE_OFST 0x1C
+#define ALTR_S10_ECC_INTMODE BIT(0)
+
+#define ALTR_S10_ECC_INTSTAT_OFST 0x20
+#define ALTR_S10_ECC_SERRPENA BIT(0)
+#define ALTR_S10_ECC_DERRPENA BIT(8)
+#define ALTR_S10_ECC_ERRPENA_MASK (ALTR_S10_ECC_SERRPENA | \
+ ALTR_S10_ECC_DERRPENA)
+
+#define ALTR_S10_ECC_INTTEST_OFST 0x24
+#define ALTR_S10_ECC_TSERRA BIT(0)
+#define ALTR_S10_ECC_TDERRA BIT(8)
+#define ALTR_S10_ECC_TSERRB BIT(16)
+#define ALTR_S10_ECC_TDERRB BIT(24)
+
#define ALTR_S10_DERR_ADDRA_OFST 0x2C
/* Stratix10 ECC Manager Defines */
@@ -300,7 +323,7 @@ struct altr_sdram_mc_data {
#define S10_SYSMGR_UE_ADDR_OFST 0x224
#define S10_DDR0_IRQ_MASK BIT(16)
-#define S10_DBE_IRQ_MASK 0x3FE
+#define S10_DBE_IRQ_MASK 0x3FFFE
/* Define ECC Block Offsets for peripherals */
#define ECC_BLK_ADDRESS_OFST 0x40
diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c
index 873437be86d9..c1d4536ae466 100644
--- a/drivers/edac/amd64_edac.c
+++ b/drivers/edac/amd64_edac.c
@@ -788,51 +788,45 @@ static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
(dclr & BIT(15)) ? "yes" : "no");
}
-/*
- * The Address Mask should be a contiguous set of bits in the non-interleaved
- * case. So to check for CS interleaving, find the most- and least-significant
- * bits of the mask, generate a contiguous bitmask, and compare the two.
- */
-static bool f17_cs_interleaved(struct amd64_pvt *pvt, u8 ctrl, int cs)
+#define CS_EVEN_PRIMARY BIT(0)
+#define CS_ODD_PRIMARY BIT(1)
+#define CS_EVEN_SECONDARY BIT(2)
+#define CS_ODD_SECONDARY BIT(3)
+
+#define CS_EVEN (CS_EVEN_PRIMARY | CS_EVEN_SECONDARY)
+#define CS_ODD (CS_ODD_PRIMARY | CS_ODD_SECONDARY)
+
+static int f17_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt)
{
- u32 mask = pvt->csels[ctrl].csmasks[cs >> 1];
- u32 msb = fls(mask) - 1, lsb = ffs(mask) - 1;
- u32 test_mask = GENMASK(msb, lsb);
+ int cs_mode = 0;
- edac_dbg(1, "mask=0x%08x test_mask=0x%08x\n", mask, test_mask);
+ if (csrow_enabled(2 * dimm, ctrl, pvt))
+ cs_mode |= CS_EVEN_PRIMARY;
- return mask ^ test_mask;
+ if (csrow_enabled(2 * dimm + 1, ctrl, pvt))
+ cs_mode |= CS_ODD_PRIMARY;
+
+ /* Asymmetric dual-rank DIMM support. */
+ if (csrow_sec_enabled(2 * dimm + 1, ctrl, pvt))
+ cs_mode |= CS_ODD_SECONDARY;
+
+ return cs_mode;
}
static void debug_display_dimm_sizes_df(struct amd64_pvt *pvt, u8 ctrl)
{
- int dimm, size0, size1, cs0, cs1;
+ int dimm, size0, size1, cs0, cs1, cs_mode;
edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl);
- for (dimm = 0; dimm < 4; dimm++) {
- size0 = 0;
+ for (dimm = 0; dimm < 2; dimm++) {
cs0 = dimm * 2;
-
- if (csrow_enabled(cs0, ctrl, pvt))
- size0 = pvt->ops->dbam_to_cs(pvt, ctrl, 0, cs0);
-
- size1 = 0;
cs1 = dimm * 2 + 1;
- if (csrow_enabled(cs1, ctrl, pvt)) {
- /*
- * CS interleaving is only supported if both CSes have
- * the same amount of memory. Because they are
- * interleaved, it will look like both CSes have the
- * full amount of memory. Save the size for both as
- * half the amount we found on CS0, if interleaved.
- */
- if (f17_cs_interleaved(pvt, ctrl, cs1))
- size1 = size0 = (size0 >> 1);
- else
- size1 = pvt->ops->dbam_to_cs(pvt, ctrl, 0, cs1);
- }
+ cs_mode = f17_get_cs_mode(dimm, ctrl, pvt);
+
+ size0 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs0);
+ size1 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs1);
amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
cs0, size0,
@@ -942,89 +936,119 @@ static void prep_chip_selects(struct amd64_pvt *pvt)
} else if (pvt->fam == 0x15 && pvt->model == 0x30) {
pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
+ } else if (pvt->fam >= 0x17) {
+ int umc;
+
+ for_each_umc(umc) {
+ pvt->csels[umc].b_cnt = 4;
+ pvt->csels[umc].m_cnt = 2;
+ }
+
} else {
pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
}
}
+static void read_umc_base_mask(struct amd64_pvt *pvt)
+{
+ u32 umc_base_reg, umc_base_reg_sec;
+ u32 umc_mask_reg, umc_mask_reg_sec;
+ u32 base_reg, base_reg_sec;
+ u32 mask_reg, mask_reg_sec;
+ u32 *base, *base_sec;
+ u32 *mask, *mask_sec;
+ int cs, umc;
+
+ for_each_umc(umc) {
+ umc_base_reg = get_umc_base(umc) + UMCCH_BASE_ADDR;
+ umc_base_reg_sec = get_umc_base(umc) + UMCCH_BASE_ADDR_SEC;
+
+ for_each_chip_select(cs, umc, pvt) {
+ base = &pvt->csels[umc].csbases[cs];
+ base_sec = &pvt->csels[umc].csbases_sec[cs];
+
+ base_reg = umc_base_reg + (cs * 4);
+ base_reg_sec = umc_base_reg_sec + (cs * 4);
+
+ if (!amd_smn_read(pvt->mc_node_id, base_reg, base))
+ edac_dbg(0, " DCSB%d[%d]=0x%08x reg: 0x%x\n",
+ umc, cs, *base, base_reg);
+
+ if (!amd_smn_read(pvt->mc_node_id, base_reg_sec, base_sec))
+ edac_dbg(0, " DCSB_SEC%d[%d]=0x%08x reg: 0x%x\n",
+ umc, cs, *base_sec, base_reg_sec);
+ }
+
+ umc_mask_reg = get_umc_base(umc) + UMCCH_ADDR_MASK;
+ umc_mask_reg_sec = get_umc_base(umc) + UMCCH_ADDR_MASK_SEC;
+
+ for_each_chip_select_mask(cs, umc, pvt) {
+ mask = &pvt->csels[umc].csmasks[cs];
+ mask_sec = &pvt->csels[umc].csmasks_sec[cs];
+
+ mask_reg = umc_mask_reg + (cs * 4);
+ mask_reg_sec = umc_mask_reg_sec + (cs * 4);
+
+ if (!amd_smn_read(pvt->mc_node_id, mask_reg, mask))
+ edac_dbg(0, " DCSM%d[%d]=0x%08x reg: 0x%x\n",
+ umc, cs, *mask, mask_reg);
+
+ if (!amd_smn_read(pvt->mc_node_id, mask_reg_sec, mask_sec))
+ edac_dbg(0, " DCSM_SEC%d[%d]=0x%08x reg: 0x%x\n",
+ umc, cs, *mask_sec, mask_reg_sec);
+ }
+ }
+}
+
/*
* Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
*/
static void read_dct_base_mask(struct amd64_pvt *pvt)
{
- int base_reg0, base_reg1, mask_reg0, mask_reg1, cs;
+ int cs;
prep_chip_selects(pvt);
- if (pvt->umc) {
- base_reg0 = get_umc_base(0) + UMCCH_BASE_ADDR;
- base_reg1 = get_umc_base(1) + UMCCH_BASE_ADDR;
- mask_reg0 = get_umc_base(0) + UMCCH_ADDR_MASK;
- mask_reg1 = get_umc_base(1) + UMCCH_ADDR_MASK;
- } else {
- base_reg0 = DCSB0;
- base_reg1 = DCSB1;
- mask_reg0 = DCSM0;
- mask_reg1 = DCSM1;
- }
+ if (pvt->umc)
+ return read_umc_base_mask(pvt);
for_each_chip_select(cs, 0, pvt) {
- int reg0 = base_reg0 + (cs * 4);
- int reg1 = base_reg1 + (cs * 4);
+ int reg0 = DCSB0 + (cs * 4);
+ int reg1 = DCSB1 + (cs * 4);
u32 *base0 = &pvt->csels[0].csbases[cs];
u32 *base1 = &pvt->csels[1].csbases[cs];
- if (pvt->umc) {
- if (!amd_smn_read(pvt->mc_node_id, reg0, base0))
- edac_dbg(0, " DCSB0[%d]=0x%08x reg: 0x%x\n",
- cs, *base0, reg0);
+ if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
+ edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
+ cs, *base0, reg0);
- if (!amd_smn_read(pvt->mc_node_id, reg1, base1))
- edac_dbg(0, " DCSB1[%d]=0x%08x reg: 0x%x\n",
- cs, *base1, reg1);
- } else {
- if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
- edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
- cs, *base0, reg0);
-
- if (pvt->fam == 0xf)
- continue;
+ if (pvt->fam == 0xf)
+ continue;
- if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
- edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
- cs, *base1, (pvt->fam == 0x10) ? reg1
- : reg0);
- }
+ if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
+ edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
+ cs, *base1, (pvt->fam == 0x10) ? reg1
+ : reg0);
}
for_each_chip_select_mask(cs, 0, pvt) {
- int reg0 = mask_reg0 + (cs * 4);
- int reg1 = mask_reg1 + (cs * 4);
+ int reg0 = DCSM0 + (cs * 4);
+ int reg1 = DCSM1 + (cs * 4);
u32 *mask0 = &pvt->csels[0].csmasks[cs];
u32 *mask1 = &pvt->csels[1].csmasks[cs];
- if (pvt->umc) {
- if (!amd_smn_read(pvt->mc_node_id, reg0, mask0))
- edac_dbg(0, " DCSM0[%d]=0x%08x reg: 0x%x\n",
- cs, *mask0, reg0);
-
- if (!amd_smn_read(pvt->mc_node_id, reg1, mask1))
- edac_dbg(0, " DCSM1[%d]=0x%08x reg: 0x%x\n",
- cs, *mask1, reg1);
- } else {
- if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
- edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
- cs, *mask0, reg0);
+ if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
+ edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
+ cs, *mask0, reg0);
- if (pvt->fam == 0xf)
- continue;
+ if (pvt->fam == 0xf)
+ continue;
- if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
- edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
- cs, *mask1, (pvt->fam == 0x10) ? reg1
- : reg0);
- }
+ if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
+ edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
+ cs, *mask1, (pvt->fam == 0x10) ? reg1
+ : reg0);
}
}
@@ -1556,18 +1580,58 @@ static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
return ddr3_cs_size(cs_mode, false);
}
-static int f17_base_addr_to_cs_size(struct amd64_pvt *pvt, u8 umc,
+static int f17_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc,
unsigned int cs_mode, int csrow_nr)
{
- u32 base_addr = pvt->csels[umc].csbases[csrow_nr];
+ u32 addr_mask_orig, addr_mask_deinterleaved;
+ u32 msb, weight, num_zero_bits;
+ int dimm, size = 0;
- /* Each mask is used for every two base addresses. */
- u32 addr_mask = pvt->csels[umc].csmasks[csrow_nr >> 1];
+ /* No Chip Selects are enabled. */
+ if (!cs_mode)
+ return size;
- /* Register [31:1] = Address [39:9]. Size is in kBs here. */
- u32 size = ((addr_mask >> 1) - (base_addr >> 1) + 1) >> 1;
+ /* Requested size of an even CS but none are enabled. */
+ if (!(cs_mode & CS_EVEN) && !(csrow_nr & 1))
+ return size;
- edac_dbg(1, "BaseAddr: 0x%x, AddrMask: 0x%x\n", base_addr, addr_mask);
+ /* Requested size of an odd CS but none are enabled. */
+ if (!(cs_mode & CS_ODD) && (csrow_nr & 1))
+ return size;
+
+ /*
+ * There is one mask per DIMM, and two Chip Selects per DIMM.
+ * CS0 and CS1 -> DIMM0
+ * CS2 and CS3 -> DIMM1
+ */
+ dimm = csrow_nr >> 1;
+
+ /* Asymmetric dual-rank DIMM support. */
+ if ((csrow_nr & 1) && (cs_mode & CS_ODD_SECONDARY))
+ addr_mask_orig = pvt->csels[umc].csmasks_sec[dimm];
+ else
+ addr_mask_orig = pvt->csels[umc].csmasks[dimm];
+
+ /*
+ * The number of zero bits in the mask is equal to the number of bits
+ * in a full mask minus the number of bits in the current mask.
+ *
+ * The MSB is the number of bits in the full mask because BIT[0] is
+ * always 0.
+ */
+ msb = fls(addr_mask_orig) - 1;
+ weight = hweight_long(addr_mask_orig);
+ num_zero_bits = msb - weight;
+
+ /* Take the number of zero bits off from the top of the mask. */
+ addr_mask_deinterleaved = GENMASK_ULL(msb - num_zero_bits, 1);
+
+ edac_dbg(1, "CS%d DIMM%d AddrMasks:\n", csrow_nr, dimm);
+ edac_dbg(1, " Original AddrMask: 0x%x\n", addr_mask_orig);
+ edac_dbg(1, " Deinterleaved AddrMask: 0x%x\n", addr_mask_deinterleaved);
+
+ /* Register [31:1] = Address [39:9]. Size is in kBs here. */
+ size = (addr_mask_deinterleaved >> 2) + 1;
/* Return size in MBs. */
return size >> 10;
@@ -2232,7 +2296,7 @@ static struct amd64_family_type family_types[] = {
.f6_id = PCI_DEVICE_ID_AMD_17H_DF_F6,
.ops = {
.early_channel_count = f17_early_channel_count,
- .dbam_to_cs = f17_base_addr_to_cs_size,
+ .dbam_to_cs = f17_addr_mask_to_cs_size,
}
},
[F17_M10H_CPUS] = {
@@ -2241,7 +2305,7 @@ static struct amd64_family_type family_types[] = {
.f6_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F6,
.ops = {
.early_channel_count = f17_early_channel_count,
- .dbam_to_cs = f17_base_addr_to_cs_size,
+ .dbam_to_cs = f17_addr_mask_to_cs_size,
}
},
[F17_M30H_CPUS] = {
@@ -2250,7 +2314,16 @@ static struct amd64_family_type family_types[] = {
.f6_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F6,
.ops = {
.early_channel_count = f17_early_channel_count,
- .dbam_to_cs = f17_base_addr_to_cs_size,
+ .dbam_to_cs = f17_addr_mask_to_cs_size,
+ }
+ },
+ [F17_M70H_CPUS] = {
+ .ctl_name = "F17h_M70h",
+ .f0_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F0,
+ .f6_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F6,
+ .ops = {
+ .early_channel_count = f17_early_channel_count,
+ .dbam_to_cs = f17_addr_mask_to_cs_size,
}
},
};
@@ -2537,13 +2610,6 @@ static void decode_umc_error(int node_id, struct mce *m)
err.channel = find_umc_channel(m);
- if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) {
- err.err_code = ERR_NORM_ADDR;
- goto log_error;
- }
-
- error_address_to_page_and_offset(sys_addr, &err);
-
if (!(m->status & MCI_STATUS_SYNDV)) {
err.err_code = ERR_SYND;
goto log_error;
@@ -2560,6 +2626,13 @@ static void decode_umc_error(int node_id, struct mce *m)
err.csrow = m->synd & 0x7;
+ if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) {
+ err.err_code = ERR_NORM_ADDR;
+ goto log_error;
+ }
+
+ error_address_to_page_and_offset(sys_addr, &err);
+
log_error:
__log_ecc_error(mci, &err, ecc_type);
}
@@ -2809,10 +2882,12 @@ static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr_orig)
int csrow_nr = csrow_nr_orig;
u32 cs_mode, nr_pages;
- if (!pvt->umc)
+ if (!pvt->umc) {
csrow_nr >>= 1;
-
- cs_mode = DBAM_DIMM(csrow_nr, dbam);
+ cs_mode = DBAM_DIMM(csrow_nr, dbam);
+ } else {
+ cs_mode = f17_get_cs_mode(csrow_nr >> 1, dct, pvt);
+ }
nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, csrow_nr);
nr_pages <<= 20 - PAGE_SHIFT;
@@ -2824,6 +2899,49 @@ static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr_orig)
return nr_pages;
}
+static int init_csrows_df(struct mem_ctl_info *mci)
+{
+ struct amd64_pvt *pvt = mci->pvt_info;
+ enum edac_type edac_mode = EDAC_NONE;
+ enum dev_type dev_type = DEV_UNKNOWN;
+ struct dimm_info *dimm;
+ int empty = 1;
+ u8 umc, cs;
+
+ if (mci->edac_ctl_cap & EDAC_FLAG_S16ECD16ED) {
+ edac_mode = EDAC_S16ECD16ED;
+ dev_type = DEV_X16;
+ } else if (mci->edac_ctl_cap & EDAC_FLAG_S8ECD8ED) {
+ edac_mode = EDAC_S8ECD8ED;
+ dev_type = DEV_X8;
+ } else if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED) {
+ edac_mode = EDAC_S4ECD4ED;
+ dev_type = DEV_X4;
+ } else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED) {
+ edac_mode = EDAC_SECDED;
+ }
+
+ for_each_umc(umc) {
+ for_each_chip_select(cs, umc, pvt) {
+ if (!csrow_enabled(cs, umc, pvt))
+ continue;
+
+ empty = 0;
+ dimm = mci->csrows[cs]->channels[umc]->dimm;
+
+ edac_dbg(1, "MC node: %d, csrow: %d\n",
+ pvt->mc_node_id, cs);
+
+ dimm->nr_pages = get_csrow_nr_pages(pvt, umc, cs);
+ dimm->mtype = pvt->dram_type;
+ dimm->edac_mode = edac_mode;
+ dimm->dtype = dev_type;
+ }
+ }
+
+ return empty;
+}
+
/*
* Initialize the array of csrow attribute instances, based on the values
* from pci config hardware registers.
@@ -2838,15 +2956,16 @@ static int init_csrows(struct mem_ctl_info *mci)
int nr_pages = 0;
u32 val;
- if (!pvt->umc) {
- amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
+ if (pvt->umc)
+ return init_csrows_df(mci);
- pvt->nbcfg = val;
+ amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
- edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
- pvt->mc_node_id, val,
- !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
- }
+ pvt->nbcfg = val;
+
+ edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
+ pvt->mc_node_id, val,
+ !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
/*
* We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
@@ -2883,13 +3002,7 @@ static int init_csrows(struct mem_ctl_info *mci)
edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
/* Determine DIMM ECC mode: */
- if (pvt->umc) {
- if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED)
- edac_mode = EDAC_S4ECD4ED;
- else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED)
- edac_mode = EDAC_SECDED;
-
- } else if (pvt->nbcfg & NBCFG_ECC_ENABLE) {
+ if (pvt->nbcfg & NBCFG_ECC_ENABLE) {
edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL)
? EDAC_S4ECD4ED
: EDAC_SECDED;
@@ -3137,12 +3250,15 @@ static bool ecc_enabled(struct pci_dev *F3, u16 nid)
static inline void
f17h_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
{
- u8 i, ecc_en = 1, cpk_en = 1;
+ u8 i, ecc_en = 1, cpk_en = 1, dev_x4 = 1, dev_x16 = 1;
for_each_umc(i) {
if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
ecc_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_ENABLED);
cpk_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_CHIPKILL_CAP);
+
+ dev_x4 &= !!(pvt->umc[i].dimm_cfg & BIT(6));
+ dev_x16 &= !!(pvt->umc[i].dimm_cfg & BIT(7));
}
}
@@ -3150,8 +3266,15 @@ f17h_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
if (ecc_en) {
mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
- if (cpk_en)
+ if (!cpk_en)
+ return;
+
+ if (dev_x4)
mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
+ else if (dev_x16)
+ mci->edac_ctl_cap |= EDAC_FLAG_S16ECD16ED;
+ else
+ mci->edac_ctl_cap |= EDAC_FLAG_S8ECD8ED;
}
}
@@ -3241,6 +3364,10 @@ static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
fam_type = &family_types[F17_M30H_CPUS];
pvt->ops = &family_types[F17_M30H_CPUS].ops;
break;
+ } else if (pvt->model >= 0x70 && pvt->model <= 0x7f) {
+ fam_type = &family_types[F17_M70H_CPUS];
+ pvt->ops = &family_types[F17_M70H_CPUS].ops;
+ break;
}
/* fall through */
case 0x18:
diff --git a/drivers/edac/amd64_edac.h b/drivers/edac/amd64_edac.h
index 8f66472f7adc..8c3cda81e619 100644
--- a/drivers/edac/amd64_edac.h
+++ b/drivers/edac/amd64_edac.h
@@ -96,6 +96,7 @@
/* Hardware limit on ChipSelect rows per MC and processors per system */
#define NUM_CHIPSELECTS 8
#define DRAM_RANGES 8
+#define NUM_CONTROLLERS 8
#define ON true
#define OFF false
@@ -119,6 +120,8 @@
#define PCI_DEVICE_ID_AMD_17H_M10H_DF_F6 0x15ee
#define PCI_DEVICE_ID_AMD_17H_M30H_DF_F0 0x1490
#define PCI_DEVICE_ID_AMD_17H_M30H_DF_F6 0x1496
+#define PCI_DEVICE_ID_AMD_17H_M70H_DF_F0 0x1440
+#define PCI_DEVICE_ID_AMD_17H_M70H_DF_F6 0x1446
/*
* Function 1 - Address Map
@@ -168,7 +171,8 @@
#define DCSM0 0x60
#define DCSM1 0x160
-#define csrow_enabled(i, dct, pvt) ((pvt)->csels[(dct)].csbases[(i)] & DCSB_CS_ENABLE)
+#define csrow_enabled(i, dct, pvt) ((pvt)->csels[(dct)].csbases[(i)] & DCSB_CS_ENABLE)
+#define csrow_sec_enabled(i, dct, pvt) ((pvt)->csels[(dct)].csbases_sec[(i)] & DCSB_CS_ENABLE)
#define DRAM_CONTROL 0x78
@@ -258,7 +262,9 @@
/* UMC CH register offsets */
#define UMCCH_BASE_ADDR 0x0
+#define UMCCH_BASE_ADDR_SEC 0x10
#define UMCCH_ADDR_MASK 0x20
+#define UMCCH_ADDR_MASK_SEC 0x28
#define UMCCH_ADDR_CFG 0x30
#define UMCCH_DIMM_CFG 0x80
#define UMCCH_UMC_CFG 0x100
@@ -285,6 +291,7 @@ enum amd_families {
F17_CPUS,
F17_M10H_CPUS,
F17_M30H_CPUS,
+ F17_M70H_CPUS,
NUM_FAMILIES,
};
@@ -311,9 +318,11 @@ struct dram_range {
/* A DCT chip selects collection */
struct chip_select {
u32 csbases[NUM_CHIPSELECTS];
+ u32 csbases_sec[NUM_CHIPSELECTS];
u8 b_cnt;
u32 csmasks[NUM_CHIPSELECTS];
+ u32 csmasks_sec[NUM_CHIPSELECTS];
u8 m_cnt;
};
@@ -351,8 +360,8 @@ struct amd64_pvt {
u32 dbam0; /* DRAM Base Address Mapping reg for DCT0 */
u32 dbam1; /* DRAM Base Address Mapping reg for DCT1 */
- /* one for each DCT */
- struct chip_select csels[2];
+ /* one for each DCT/UMC */
+ struct chip_select csels[NUM_CONTROLLERS];
/* DRAM base and limit pairs F1x[78,70,68,60,58,50,48,40] */
struct dram_range ranges[DRAM_RANGES];
diff --git a/drivers/edac/aspeed_edac.c b/drivers/edac/aspeed_edac.c
index 11833c0a5d07..5634437bb39d 100644
--- a/drivers/edac/aspeed_edac.c
+++ b/drivers/edac/aspeed_edac.c
@@ -281,15 +281,11 @@ static int aspeed_probe(struct platform_device *pdev)
struct device *dev = &pdev->dev;
struct edac_mc_layer layers[2];
struct mem_ctl_info *mci;
- struct device_node *np;
struct resource *res;
void __iomem *regs;
u32 reg04;
int rc;
- /* setup regmap */
- np = dev->of_node;
-
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (!res)
return -ENOENT;
diff --git a/drivers/edac/bluefield_edac.c b/drivers/edac/bluefield_edac.c
new file mode 100644
index 000000000000..e4736eb37bfb
--- /dev/null
+++ b/drivers/edac/bluefield_edac.c
@@ -0,0 +1,356 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Bluefield-specific EDAC driver.
+ *
+ * Copyright (c) 2019 Mellanox Technologies.
+ */
+
+#include <linux/acpi.h>
+#include <linux/arm-smccc.h>
+#include <linux/bitfield.h>
+#include <linux/edac.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include "edac_module.h"
+
+#define DRIVER_NAME "bluefield-edac"
+
+/*
+ * Mellanox BlueField EMI (External Memory Interface) register definitions.
+ */
+
+#define MLXBF_ECC_CNT 0x340
+#define MLXBF_ECC_CNT__SERR_CNT GENMASK(15, 0)
+#define MLXBF_ECC_CNT__DERR_CNT GENMASK(31, 16)
+
+#define MLXBF_ECC_ERR 0x348
+#define MLXBF_ECC_ERR__SECC BIT(0)
+#define MLXBF_ECC_ERR__DECC BIT(16)
+
+#define MLXBF_ECC_LATCH_SEL 0x354
+#define MLXBF_ECC_LATCH_SEL__START BIT(24)
+
+#define MLXBF_ERR_ADDR_0 0x358
+
+#define MLXBF_ERR_ADDR_1 0x37c
+
+#define MLXBF_SYNDROM 0x35c
+#define MLXBF_SYNDROM__DERR BIT(0)
+#define MLXBF_SYNDROM__SERR BIT(1)
+#define MLXBF_SYNDROM__SYN GENMASK(25, 16)
+
+#define MLXBF_ADD_INFO 0x364
+#define MLXBF_ADD_INFO__ERR_PRANK GENMASK(9, 8)
+
+#define MLXBF_EDAC_MAX_DIMM_PER_MC 2
+#define MLXBF_EDAC_ERROR_GRAIN 8
+
+/*
+ * Request MLNX_SIP_GET_DIMM_INFO
+ *
+ * Retrieve information about DIMM on a certain slot.
+ *
+ * Call register usage:
+ * a0: MLNX_SIP_GET_DIMM_INFO
+ * a1: (Memory controller index) << 16 | (Dimm index in memory controller)
+ * a2-7: not used.
+ *
+ * Return status:
+ * a0: MLXBF_DIMM_INFO defined below describing the DIMM.
+ * a1-3: not used.
+ */
+#define MLNX_SIP_GET_DIMM_INFO 0x82000008
+
+/* Format for the SMC response about the memory information */
+#define MLXBF_DIMM_INFO__SIZE_GB GENMASK_ULL(15, 0)
+#define MLXBF_DIMM_INFO__IS_RDIMM BIT(16)
+#define MLXBF_DIMM_INFO__IS_LRDIMM BIT(17)
+#define MLXBF_DIMM_INFO__IS_NVDIMM BIT(18)
+#define MLXBF_DIMM_INFO__RANKS GENMASK_ULL(23, 21)
+#define MLXBF_DIMM_INFO__PACKAGE_X GENMASK_ULL(31, 24)
+
+struct bluefield_edac_priv {
+ int dimm_ranks[MLXBF_EDAC_MAX_DIMM_PER_MC];
+ void __iomem *emi_base;
+ int dimm_per_mc;
+};
+
+static u64 smc_call1(u64 smc_op, u64 smc_arg)
+{
+ struct arm_smccc_res res;
+
+ arm_smccc_smc(smc_op, smc_arg, 0, 0, 0, 0, 0, 0, &res);
+
+ return res.a0;
+}
+
+/*
+ * Gather the ECC information from the External Memory Interface registers
+ * and report it to the edac handler.
+ */
+static void bluefield_gather_report_ecc(struct mem_ctl_info *mci,
+ int error_cnt,
+ int is_single_ecc)
+{
+ struct bluefield_edac_priv *priv = mci->pvt_info;
+ u32 dram_additional_info, err_prank, edea0, edea1;
+ u32 ecc_latch_select, dram_syndrom, serr, derr, syndrom;
+ enum hw_event_mc_err_type ecc_type;
+ u64 ecc_dimm_addr;
+ int ecc_dimm;
+
+ ecc_type = is_single_ecc ? HW_EVENT_ERR_CORRECTED :
+ HW_EVENT_ERR_UNCORRECTED;
+
+ /*
+ * Tell the External Memory Interface to populate the relevant
+ * registers with information about the last ECC error occurrence.
+ */
+ ecc_latch_select = MLXBF_ECC_LATCH_SEL__START;
+ writel(ecc_latch_select, priv->emi_base + MLXBF_ECC_LATCH_SEL);
+
+ /*
+ * Verify that the ECC reported info in the registers is of the
+ * same type as the one asked to report. If not, just report the
+ * error without the detailed information.
+ */
+ dram_syndrom = readl(priv->emi_base + MLXBF_SYNDROM);
+ serr = FIELD_GET(MLXBF_SYNDROM__SERR, dram_syndrom);
+ derr = FIELD_GET(MLXBF_SYNDROM__DERR, dram_syndrom);
+ syndrom = FIELD_GET(MLXBF_SYNDROM__SYN, dram_syndrom);
+
+ if ((is_single_ecc && !serr) || (!is_single_ecc && !derr)) {
+ edac_mc_handle_error(ecc_type, mci, error_cnt, 0, 0, 0,
+ 0, 0, -1, mci->ctl_name, "");
+ return;
+ }
+
+ dram_additional_info = readl(priv->emi_base + MLXBF_ADD_INFO);
+ err_prank = FIELD_GET(MLXBF_ADD_INFO__ERR_PRANK, dram_additional_info);
+
+ ecc_dimm = (err_prank >= 2 && priv->dimm_ranks[0] <= 2) ? 1 : 0;
+
+ edea0 = readl(priv->emi_base + MLXBF_ERR_ADDR_0);
+ edea1 = readl(priv->emi_base + MLXBF_ERR_ADDR_1);
+
+ ecc_dimm_addr = ((u64)edea1 << 32) | edea0;
+
+ edac_mc_handle_error(ecc_type, mci, error_cnt,
+ PFN_DOWN(ecc_dimm_addr),
+ offset_in_page(ecc_dimm_addr),
+ syndrom, ecc_dimm, 0, 0, mci->ctl_name, "");
+}
+
+static void bluefield_edac_check(struct mem_ctl_info *mci)
+{
+ struct bluefield_edac_priv *priv = mci->pvt_info;
+ u32 ecc_count, single_error_count, double_error_count, ecc_error = 0;
+
+ /*
+ * The memory controller might not be initialized by the firmware
+ * when there isn't memory, which may lead to bad register readings.
+ */
+ if (mci->edac_cap == EDAC_FLAG_NONE)
+ return;
+
+ ecc_count = readl(priv->emi_base + MLXBF_ECC_CNT);
+ single_error_count = FIELD_GET(MLXBF_ECC_CNT__SERR_CNT, ecc_count);
+ double_error_count = FIELD_GET(MLXBF_ECC_CNT__DERR_CNT, ecc_count);
+
+ if (single_error_count) {
+ ecc_error |= MLXBF_ECC_ERR__SECC;
+
+ bluefield_gather_report_ecc(mci, single_error_count, 1);
+ }
+
+ if (double_error_count) {
+ ecc_error |= MLXBF_ECC_ERR__DECC;
+
+ bluefield_gather_report_ecc(mci, double_error_count, 0);
+ }
+
+ /* Write to clear reported errors. */
+ if (ecc_count)
+ writel(ecc_error, priv->emi_base + MLXBF_ECC_ERR);
+}
+
+/* Initialize the DIMMs information for the given memory controller. */
+static void bluefield_edac_init_dimms(struct mem_ctl_info *mci)
+{
+ struct bluefield_edac_priv *priv = mci->pvt_info;
+ int mem_ctrl_idx = mci->mc_idx;
+ struct dimm_info *dimm;
+ u64 smc_info, smc_arg;
+ int is_empty = 1, i;
+
+ for (i = 0; i < priv->dimm_per_mc; i++) {
+ dimm = mci->dimms[i];
+
+ smc_arg = mem_ctrl_idx << 16 | i;
+ smc_info = smc_call1(MLNX_SIP_GET_DIMM_INFO, smc_arg);
+
+ if (!FIELD_GET(MLXBF_DIMM_INFO__SIZE_GB, smc_info)) {
+ dimm->mtype = MEM_EMPTY;
+ continue;
+ }
+
+ is_empty = 0;
+
+ dimm->edac_mode = EDAC_SECDED;
+
+ if (FIELD_GET(MLXBF_DIMM_INFO__IS_NVDIMM, smc_info))
+ dimm->mtype = MEM_NVDIMM;
+ else if (FIELD_GET(MLXBF_DIMM_INFO__IS_LRDIMM, smc_info))
+ dimm->mtype = MEM_LRDDR4;
+ else if (FIELD_GET(MLXBF_DIMM_INFO__IS_RDIMM, smc_info))
+ dimm->mtype = MEM_RDDR4;
+ else
+ dimm->mtype = MEM_DDR4;
+
+ dimm->nr_pages =
+ FIELD_GET(MLXBF_DIMM_INFO__SIZE_GB, smc_info) *
+ (SZ_1G / PAGE_SIZE);
+ dimm->grain = MLXBF_EDAC_ERROR_GRAIN;
+
+ /* Mem controller for BlueField only supports x4, x8 and x16 */
+ switch (FIELD_GET(MLXBF_DIMM_INFO__PACKAGE_X, smc_info)) {
+ case 4:
+ dimm->dtype = DEV_X4;
+ break;
+ case 8:
+ dimm->dtype = DEV_X8;
+ break;
+ case 16:
+ dimm->dtype = DEV_X16;
+ break;
+ default:
+ dimm->dtype = DEV_UNKNOWN;
+ }
+
+ priv->dimm_ranks[i] =
+ FIELD_GET(MLXBF_DIMM_INFO__RANKS, smc_info);
+ }
+
+ if (is_empty)
+ mci->edac_cap = EDAC_FLAG_NONE;
+ else
+ mci->edac_cap = EDAC_FLAG_SECDED;
+}
+
+static int bluefield_edac_mc_probe(struct platform_device *pdev)
+{
+ struct bluefield_edac_priv *priv;
+ struct device *dev = &pdev->dev;
+ struct edac_mc_layer layers[1];
+ struct mem_ctl_info *mci;
+ struct resource *emi_res;
+ unsigned int mc_idx, dimm_count;
+ int rc, ret;
+
+ /* Read the MSS (Memory SubSystem) index from ACPI table. */
+ if (device_property_read_u32(dev, "mss_number", &mc_idx)) {
+ dev_warn(dev, "bf_edac: MSS number unknown\n");
+ return -EINVAL;
+ }
+
+ /* Read the DIMMs per MC from ACPI table. */
+ if (device_property_read_u32(dev, "dimm_per_mc", &dimm_count)) {
+ dev_warn(dev, "bf_edac: DIMMs per MC unknown\n");
+ return -EINVAL;
+ }
+
+ if (dimm_count > MLXBF_EDAC_MAX_DIMM_PER_MC) {
+ dev_warn(dev, "bf_edac: DIMMs per MC not valid\n");
+ return -EINVAL;
+ }
+
+ emi_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!emi_res)
+ return -EINVAL;
+
+ layers[0].type = EDAC_MC_LAYER_SLOT;
+ layers[0].size = dimm_count;
+ layers[0].is_virt_csrow = true;
+
+ mci = edac_mc_alloc(mc_idx, ARRAY_SIZE(layers), layers, sizeof(*priv));
+ if (!mci)
+ return -ENOMEM;
+
+ priv = mci->pvt_info;
+
+ priv->dimm_per_mc = dimm_count;
+ priv->emi_base = devm_ioremap_resource(dev, emi_res);
+ if (IS_ERR(priv->emi_base)) {
+ dev_err(dev, "failed to map EMI IO resource\n");
+ ret = PTR_ERR(priv->emi_base);
+ goto err;
+ }
+
+ mci->pdev = dev;
+ mci->mtype_cap = MEM_FLAG_DDR4 | MEM_FLAG_RDDR4 |
+ MEM_FLAG_LRDDR4 | MEM_FLAG_NVDIMM;
+ mci->edac_ctl_cap = EDAC_FLAG_SECDED;
+
+ mci->mod_name = DRIVER_NAME;
+ mci->ctl_name = "BlueField_Memory_Controller";
+ mci->dev_name = dev_name(dev);
+ mci->edac_check = bluefield_edac_check;
+
+ /* Initialize mci with the actual populated DIMM information. */
+ bluefield_edac_init_dimms(mci);
+
+ platform_set_drvdata(pdev, mci);
+
+ /* Register with EDAC core */
+ rc = edac_mc_add_mc(mci);
+ if (rc) {
+ dev_err(dev, "failed to register with EDAC core\n");
+ ret = rc;
+ goto err;
+ }
+
+ /* Only POLL mode supported so far. */
+ edac_op_state = EDAC_OPSTATE_POLL;
+
+ return 0;
+
+err:
+ edac_mc_free(mci);
+
+ return ret;
+
+}
+
+static int bluefield_edac_mc_remove(struct platform_device *pdev)
+{
+ struct mem_ctl_info *mci = platform_get_drvdata(pdev);
+
+ edac_mc_del_mc(&pdev->dev);
+ edac_mc_free(mci);
+
+ return 0;
+}
+
+static const struct acpi_device_id bluefield_mc_acpi_ids[] = {
+ {"MLNXBF08", 0},
+ {}
+};
+
+MODULE_DEVICE_TABLE(acpi, bluefield_mc_acpi_ids);
+
+static struct platform_driver bluefield_edac_mc_driver = {
+ .driver = {
+ .name = DRIVER_NAME,
+ .acpi_match_table = bluefield_mc_acpi_ids,
+ },
+ .probe = bluefield_edac_mc_probe,
+ .remove = bluefield_edac_mc_remove,
+};
+
+module_platform_driver(bluefield_edac_mc_driver);
+
+MODULE_DESCRIPTION("Mellanox BlueField memory edac driver");
+MODULE_AUTHOR("Mellanox Technologies");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/edac/debugfs.c b/drivers/edac/debugfs.c
index 6b8e484db851..1f943599a8ac 100644
--- a/drivers/edac/debugfs.c
+++ b/drivers/edac/debugfs.c
@@ -118,23 +118,23 @@ edac_debugfs_create_file(const char *name, umode_t mode, struct dentry *parent,
EXPORT_SYMBOL_GPL(edac_debugfs_create_file);
/* Wrapper for debugfs_create_x8() */
-struct dentry *edac_debugfs_create_x8(const char *name, umode_t mode,
- struct dentry *parent, u8 *value)
+void edac_debugfs_create_x8(const char *name, umode_t mode,
+ struct dentry *parent, u8 *value)
{
if (!parent)
parent = edac_debugfs;
- return debugfs_create_x8(name, mode, parent, value);
+ debugfs_create_x8(name, mode, parent, value);
}
EXPORT_SYMBOL_GPL(edac_debugfs_create_x8);
/* Wrapper for debugfs_create_x16() */
-struct dentry *edac_debugfs_create_x16(const char *name, umode_t mode,
- struct dentry *parent, u16 *value)
+void edac_debugfs_create_x16(const char *name, umode_t mode,
+ struct dentry *parent, u16 *value)
{
if (!parent)
parent = edac_debugfs;
- return debugfs_create_x16(name, mode, parent, value);
+ debugfs_create_x16(name, mode, parent, value);
}
EXPORT_SYMBOL_GPL(edac_debugfs_create_x16);
diff --git a/drivers/edac/edac_mc.c b/drivers/edac/edac_mc.c
index 64922c8fa7e3..e6fd079783bd 100644
--- a/drivers/edac/edac_mc.c
+++ b/drivers/edac/edac_mc.c
@@ -114,8 +114,8 @@ static const struct kernel_param_ops edac_report_ops = {
module_param_cb(edac_report, &edac_report_ops, &edac_report, 0644);
-unsigned edac_dimm_info_location(struct dimm_info *dimm, char *buf,
- unsigned len)
+unsigned int edac_dimm_info_location(struct dimm_info *dimm, char *buf,
+ unsigned int len)
{
struct mem_ctl_info *mci = dimm->mci;
int i, n, count = 0;
@@ -236,9 +236,9 @@ EXPORT_SYMBOL_GPL(edac_mem_types);
* At return, the pointer 'p' will be incremented to be used on a next call
* to this function.
*/
-void *edac_align_ptr(void **p, unsigned size, int n_elems)
+void *edac_align_ptr(void **p, unsigned int size, int n_elems)
{
- unsigned align, r;
+ unsigned int align, r;
void *ptr = *p;
*p += size * n_elems;
@@ -275,38 +275,37 @@ void *edac_align_ptr(void **p, unsigned size, int n_elems)
static void _edac_mc_free(struct mem_ctl_info *mci)
{
- int i, chn, row;
struct csrow_info *csr;
- const unsigned int tot_dimms = mci->tot_dimms;
- const unsigned int tot_channels = mci->num_cschannel;
- const unsigned int tot_csrows = mci->nr_csrows;
+ int i, chn, row;
if (mci->dimms) {
- for (i = 0; i < tot_dimms; i++)
+ for (i = 0; i < mci->tot_dimms; i++)
kfree(mci->dimms[i]);
kfree(mci->dimms);
}
+
if (mci->csrows) {
- for (row = 0; row < tot_csrows; row++) {
+ for (row = 0; row < mci->nr_csrows; row++) {
csr = mci->csrows[row];
- if (csr) {
- if (csr->channels) {
- for (chn = 0; chn < tot_channels; chn++)
- kfree(csr->channels[chn]);
- kfree(csr->channels);
- }
- kfree(csr);
+ if (!csr)
+ continue;
+
+ if (csr->channels) {
+ for (chn = 0; chn < mci->num_cschannel; chn++)
+ kfree(csr->channels[chn]);
+ kfree(csr->channels);
}
+ kfree(csr);
}
kfree(mci->csrows);
}
kfree(mci);
}
-struct mem_ctl_info *edac_mc_alloc(unsigned mc_num,
- unsigned n_layers,
+struct mem_ctl_info *edac_mc_alloc(unsigned int mc_num,
+ unsigned int n_layers,
struct edac_mc_layer *layers,
- unsigned sz_pvt)
+ unsigned int sz_pvt)
{
struct mem_ctl_info *mci;
struct edac_mc_layer *layer;
@@ -314,9 +313,9 @@ struct mem_ctl_info *edac_mc_alloc(unsigned mc_num,
struct rank_info *chan;
struct dimm_info *dimm;
u32 *ce_per_layer[EDAC_MAX_LAYERS], *ue_per_layer[EDAC_MAX_LAYERS];
- unsigned pos[EDAC_MAX_LAYERS];
- unsigned size, tot_dimms = 1, count = 1;
- unsigned tot_csrows = 1, tot_channels = 1, tot_errcount = 0;
+ unsigned int pos[EDAC_MAX_LAYERS];
+ unsigned int size, tot_dimms = 1, count = 1;
+ unsigned int tot_csrows = 1, tot_channels = 1, tot_errcount = 0;
void *pvt, *p, *ptr = NULL;
int i, j, row, chn, n, len, off;
bool per_rank = false;
@@ -1235,9 +1234,13 @@ void edac_mc_handle_error(const enum hw_event_mc_err_type type,
if (p > e->location)
*(p - 1) = '\0';
- /* Report the error via the trace interface */
- grain_bits = fls_long(e->grain) + 1;
+ /* Sanity-check driver-supplied grain value. */
+ if (WARN_ON_ONCE(!e->grain))
+ e->grain = 1;
+ grain_bits = fls_long(e->grain - 1);
+
+ /* Report the error via the trace interface */
if (IS_ENABLED(CONFIG_RAS))
trace_mc_event(type, e->msg, e->label, e->error_count,
mci->mc_idx, e->top_layer, e->mid_layer,
diff --git a/drivers/edac/edac_mc.h b/drivers/edac/edac_mc.h
index 4165e15995ad..02aac5c61d00 100644
--- a/drivers/edac/edac_mc.h
+++ b/drivers/edac/edac_mc.h
@@ -122,10 +122,10 @@ do { \
* On success, return a pointer to struct mem_ctl_info pointer;
* %NULL otherwise
*/
-struct mem_ctl_info *edac_mc_alloc(unsigned mc_num,
- unsigned n_layers,
+struct mem_ctl_info *edac_mc_alloc(unsigned int mc_num,
+ unsigned int n_layers,
struct edac_mc_layer *layers,
- unsigned sz_pvt);
+ unsigned int sz_pvt);
/**
* edac_get_owner - Return the owner's mod_name of EDAC MC
diff --git a/drivers/edac/edac_mc_sysfs.c b/drivers/edac/edac_mc_sysfs.c
index 464174685589..32d016f1ecd1 100644
--- a/drivers/edac/edac_mc_sysfs.c
+++ b/drivers/edac/edac_mc_sysfs.c
@@ -26,7 +26,7 @@
static int edac_mc_log_ue = 1;
static int edac_mc_log_ce = 1;
static int edac_mc_panic_on_ue;
-static int edac_mc_poll_msec = 1000;
+static unsigned int edac_mc_poll_msec = 1000;
/* Getter functions for above */
int edac_mc_get_log_ue(void)
@@ -45,30 +45,30 @@ int edac_mc_get_panic_on_ue(void)
}
/* this is temporary */
-int edac_mc_get_poll_msec(void)
+unsigned int edac_mc_get_poll_msec(void)
{
return edac_mc_poll_msec;
}
static int edac_set_poll_msec(const char *val, const struct kernel_param *kp)
{
- unsigned long l;
+ unsigned int i;
int ret;
if (!val)
return -EINVAL;
- ret = kstrtoul(val, 0, &l);
+ ret = kstrtouint(val, 0, &i);
if (ret)
return ret;
- if (l < 1000)
+ if (i < 1000)
return -EINVAL;
- *((unsigned long *)kp->arg) = l;
+ *((unsigned int *)kp->arg) = i;
/* notify edac_mc engine to reset the poll period */
- edac_mc_reset_delay_period(l);
+ edac_mc_reset_delay_period(i);
return 0;
}
@@ -82,7 +82,7 @@ MODULE_PARM_DESC(edac_mc_log_ue,
module_param(edac_mc_log_ce, int, 0644);
MODULE_PARM_DESC(edac_mc_log_ce,
"Log correctable error to console: 0=off 1=on");
-module_param_call(edac_mc_poll_msec, edac_set_poll_msec, param_get_int,
+module_param_call(edac_mc_poll_msec, edac_set_poll_msec, param_get_uint,
&edac_mc_poll_msec, 0644);
MODULE_PARM_DESC(edac_mc_poll_msec, "Polling period in milliseconds");
@@ -131,7 +131,7 @@ static const char * const edac_caps[] = {
struct dev_ch_attribute {
struct device_attribute attr;
- int channel;
+ unsigned int channel;
};
#define DEVICE_CHANNEL(_name, _mode, _show, _store, _var) \
@@ -200,7 +200,7 @@ static ssize_t channel_dimm_label_show(struct device *dev,
char *data)
{
struct csrow_info *csrow = to_csrow(dev);
- unsigned chan = to_channel(mattr);
+ unsigned int chan = to_channel(mattr);
struct rank_info *rank = csrow->channels[chan];
/* if field has not been initialized, there is nothing to send */
@@ -216,7 +216,7 @@ static ssize_t channel_dimm_label_store(struct device *dev,
const char *data, size_t count)
{
struct csrow_info *csrow = to_csrow(dev);
- unsigned chan = to_channel(mattr);
+ unsigned int chan = to_channel(mattr);
struct rank_info *rank = csrow->channels[chan];
size_t copy_count = count;
@@ -240,7 +240,7 @@ static ssize_t channel_ce_count_show(struct device *dev,
struct device_attribute *mattr, char *data)
{
struct csrow_info *csrow = to_csrow(dev);
- unsigned chan = to_channel(mattr);
+ unsigned int chan = to_channel(mattr);
struct rank_info *rank = csrow->channels[chan];
return sprintf(data, "%u\n", rank->ce_count);
@@ -278,7 +278,7 @@ static void csrow_attr_release(struct device *dev)
{
struct csrow_info *csrow = container_of(dev, struct csrow_info, dev);
- edac_dbg(1, "Releasing csrow device %s\n", dev_name(dev));
+ edac_dbg(1, "device %s released\n", dev_name(dev));
kfree(csrow);
}
@@ -404,6 +404,8 @@ static inline int nr_pages_per_csrow(struct csrow_info *csrow)
static int edac_create_csrow_object(struct mem_ctl_info *mci,
struct csrow_info *csrow, int index)
{
+ int err;
+
csrow->dev.type = &csrow_attr_type;
csrow->dev.groups = csrow_dev_groups;
device_initialize(&csrow->dev);
@@ -412,10 +414,16 @@ static int edac_create_csrow_object(struct mem_ctl_info *mci,
dev_set_name(&csrow->dev, "csrow%d", index);
dev_set_drvdata(&csrow->dev, csrow);
- edac_dbg(0, "creating (virtual) csrow node %s\n",
- dev_name(&csrow->dev));
+ err = device_add(&csrow->dev);
+ if (err) {
+ edac_dbg(1, "failure: create device %s\n", dev_name(&csrow->dev));
+ put_device(&csrow->dev);
+ return err;
+ }
+
+ edac_dbg(0, "device %s created\n", dev_name(&csrow->dev));
- return device_add(&csrow->dev);
+ return 0;
}
/* Create a CSROW object under specifed edac_mc_device */
@@ -429,12 +437,8 @@ static int edac_create_csrow_objects(struct mem_ctl_info *mci)
if (!nr_pages_per_csrow(csrow))
continue;
err = edac_create_csrow_object(mci, mci->csrows[i], i);
- if (err < 0) {
- edac_dbg(1,
- "failure: create csrow objects for csrow %d\n",
- i);
+ if (err < 0)
goto error;
- }
}
return 0;
@@ -443,7 +447,8 @@ error:
csrow = mci->csrows[i];
if (!nr_pages_per_csrow(csrow))
continue;
- put_device(&mci->csrows[i]->dev);
+
+ device_del(&mci->csrows[i]->dev);
}
return err;
@@ -617,7 +622,7 @@ static void dimm_attr_release(struct device *dev)
{
struct dimm_info *dimm = container_of(dev, struct dimm_info, dev);
- edac_dbg(1, "Releasing dimm device %s\n", dev_name(dev));
+ edac_dbg(1, "device %s released\n", dev_name(dev));
kfree(dimm);
}
@@ -645,11 +650,22 @@ static int edac_create_dimm_object(struct mem_ctl_info *mci,
dev_set_drvdata(&dimm->dev, dimm);
pm_runtime_forbid(&mci->dev);
- err = device_add(&dimm->dev);
+ err = device_add(&dimm->dev);
+ if (err) {
+ edac_dbg(1, "failure: create device %s\n", dev_name(&dimm->dev));
+ put_device(&dimm->dev);
+ return err;
+ }
- edac_dbg(0, "creating rank/dimm device %s\n", dev_name(&dimm->dev));
+ if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
+ char location[80];
- return err;
+ edac_dimm_info_location(dimm, location, sizeof(location));
+ edac_dbg(0, "device %s created at location %s\n",
+ dev_name(&dimm->dev), location);
+ }
+
+ return 0;
}
/*
@@ -892,7 +908,7 @@ static void mci_attr_release(struct device *dev)
{
struct mem_ctl_info *mci = container_of(dev, struct mem_ctl_info, dev);
- edac_dbg(1, "Releasing csrow device %s\n", dev_name(dev));
+ edac_dbg(1, "device %s released\n", dev_name(dev));
kfree(mci);
}
@@ -924,13 +940,15 @@ int edac_create_sysfs_mci_device(struct mem_ctl_info *mci,
dev_set_drvdata(&mci->dev, mci);
pm_runtime_forbid(&mci->dev);
- edac_dbg(0, "creating device %s\n", dev_name(&mci->dev));
err = device_add(&mci->dev);
if (err < 0) {
edac_dbg(1, "failure: create device %s\n", dev_name(&mci->dev));
- goto out;
+ put_device(&mci->dev);
+ return err;
}
+ edac_dbg(0, "device %s created\n", dev_name(&mci->dev));
+
/*
* Create the dimm/rank devices
*/
@@ -940,22 +958,9 @@ int edac_create_sysfs_mci_device(struct mem_ctl_info *mci,
if (!dimm->nr_pages)
continue;
-#ifdef CONFIG_EDAC_DEBUG
- edac_dbg(1, "creating dimm%d, located at ", i);
- if (edac_debug_level >= 1) {
- int lay;
- for (lay = 0; lay < mci->n_layers; lay++)
- printk(KERN_CONT "%s %d ",
- edac_layer_name[mci->layers[lay].type],
- dimm->location[lay]);
- printk(KERN_CONT "\n");
- }
-#endif
err = edac_create_dimm_object(mci, dimm, i);
- if (err) {
- edac_dbg(1, "failure: create dimm %d obj\n", i);
+ if (err)
goto fail_unregister_dimm;
- }
}
#ifdef CONFIG_EDAC_LEGACY_SYSFS
@@ -977,7 +982,6 @@ fail_unregister_dimm:
}
device_unregister(&mci->dev);
-out:
return err;
}
@@ -1001,14 +1005,14 @@ void edac_remove_sysfs_mci_device(struct mem_ctl_info *mci)
struct dimm_info *dimm = mci->dimms[i];
if (dimm->nr_pages == 0)
continue;
- edac_dbg(0, "removing device %s\n", dev_name(&dimm->dev));
+ edac_dbg(1, "unregistering device %s\n", dev_name(&dimm->dev));
device_unregister(&dimm->dev);
}
}
void edac_unregister_sysfs(struct mem_ctl_info *mci)
{
- edac_dbg(1, "Unregistering device %s\n", dev_name(&mci->dev));
+ edac_dbg(1, "unregistering device %s\n", dev_name(&mci->dev));
device_unregister(&mci->dev);
}
@@ -1019,7 +1023,7 @@ static void mc_attr_release(struct device *dev)
* parent device, used to create the /sys/devices/mc sysfs node.
* So, there are no attributes on it.
*/
- edac_dbg(1, "Releasing device %s\n", dev_name(dev));
+ edac_dbg(1, "device %s released\n", dev_name(dev));
kfree(dev);
}
@@ -1034,10 +1038,8 @@ int __init edac_mc_sysfs_init(void)
int err;
mci_pdev = kzalloc(sizeof(*mci_pdev), GFP_KERNEL);
- if (!mci_pdev) {
- err = -ENOMEM;
- goto out;
- }
+ if (!mci_pdev)
+ return -ENOMEM;
mci_pdev->bus = edac_get_sysfs_subsys();
mci_pdev->type = &mc_attr_type;
@@ -1045,17 +1047,15 @@ int __init edac_mc_sysfs_init(void)
dev_set_name(mci_pdev, "mc");
err = device_add(mci_pdev);
- if (err < 0)
- goto out_put_device;
+ if (err < 0) {
+ edac_dbg(1, "failure: create device %s\n", dev_name(mci_pdev));
+ put_device(mci_pdev);
+ return err;
+ }
edac_dbg(0, "device %s created\n", dev_name(mci_pdev));
return 0;
-
- out_put_device:
- put_device(mci_pdev);
- out:
- return err;
}
void edac_mc_sysfs_exit(void)
diff --git a/drivers/edac/edac_module.h b/drivers/edac/edac_module.h
index dd7d0b509aa3..b2f59ee76c22 100644
--- a/drivers/edac/edac_module.h
+++ b/drivers/edac/edac_module.h
@@ -36,7 +36,7 @@ extern int edac_mc_get_log_ue(void);
extern int edac_mc_get_log_ce(void);
extern int edac_mc_get_panic_on_ue(void);
extern int edac_get_poll_msec(void);
-extern int edac_mc_get_poll_msec(void);
+extern unsigned int edac_mc_get_poll_msec(void);
unsigned edac_dimm_info_location(struct dimm_info *dimm, char *buf,
unsigned len);
@@ -78,10 +78,10 @@ edac_debugfs_create_dir_at(const char *dirname, struct dentry *parent);
struct dentry *
edac_debugfs_create_file(const char *name, umode_t mode, struct dentry *parent,
void *data, const struct file_operations *fops);
-struct dentry *
-edac_debugfs_create_x8(const char *name, umode_t mode, struct dentry *parent, u8 *value);
-struct dentry *
-edac_debugfs_create_x16(const char *name, umode_t mode, struct dentry *parent, u16 *value);
+void edac_debugfs_create_x8(const char *name, umode_t mode,
+ struct dentry *parent, u8 *value);
+void edac_debugfs_create_x16(const char *name, umode_t mode,
+ struct dentry *parent, u16 *value);
#else
static inline void edac_debugfs_init(void) { }
static inline void edac_debugfs_exit(void) { }
@@ -92,12 +92,10 @@ edac_debugfs_create_dir_at(const char *dirname, struct dentry *parent) { return
static inline struct dentry *
edac_debugfs_create_file(const char *name, umode_t mode, struct dentry *parent,
void *data, const struct file_operations *fops) { return NULL; }
-static inline struct dentry *
-edac_debugfs_create_x8(const char *name, umode_t mode,
- struct dentry *parent, u8 *value) { return NULL; }
-static inline struct dentry *
-edac_debugfs_create_x16(const char *name, umode_t mode,
- struct dentry *parent, u16 *value) { return NULL; }
+static inline void edac_debugfs_create_x8(const char *name, umode_t mode,
+ struct dentry *parent, u8 *value) { }
+static inline void edac_debugfs_create_x16(const char *name, umode_t mode,
+ struct dentry *parent, u16 *value) { }
#endif
/*
diff --git a/drivers/edac/ghes_edac.c b/drivers/edac/ghes_edac.c
index 7f19f1c672c3..d413a0bdc9ad 100644
--- a/drivers/edac/ghes_edac.c
+++ b/drivers/edac/ghes_edac.c
@@ -68,7 +68,7 @@ struct memdev_dmi_entry {
struct ghes_edac_dimm_fill {
struct mem_ctl_info *mci;
- unsigned count;
+ unsigned int count;
};
static void ghes_edac_count_dimms(const struct dmi_header *dh, void *arg)
diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c
index 6f06aec4877c..c370d5457e6b 100644
--- a/drivers/edac/i10nm_base.c
+++ b/drivers/edac/i10nm_base.c
@@ -123,7 +123,9 @@ static int i10nm_get_all_munits(void)
}
static const struct x86_cpu_id i10nm_cpuids[] = {
- { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_TREMONT_X, 0, 0 },
+ { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_TREMONT_D, 0, 0 },
+ { X86_VENDOR_INTEL, 6, INTEL_FAM6_ICELAKE_X, 0, 0 },
+ { X86_VENDOR_INTEL, 6, INTEL_FAM6_ICELAKE_D, 0, 0 },
{ }
};
MODULE_DEVICE_TABLE(x86cpu, i10nm_cpuids);
@@ -166,9 +168,9 @@ static int i10nm_get_dimm_config(struct mem_ctl_info *mci)
ndimms += skx_get_nvdimm_info(dimm, imc, i, j,
EDAC_MOD_STR);
}
- if (ndimms && !i10nm_check_ecc(imc, 0)) {
- i10nm_printk(KERN_ERR, "ECC is disabled on imc %d\n",
- imc->mc);
+ if (ndimms && !i10nm_check_ecc(imc, i)) {
+ i10nm_printk(KERN_ERR, "ECC is disabled on imc %d channel %d\n",
+ imc->mc, i);
return -ENODEV;
}
}
@@ -265,7 +267,7 @@ static int __init i10nm_init(void)
goto fail;
list_for_each_entry(d, i10nm_edac_list, list) {
- rc = skx_get_src_id(d, &src_id);
+ rc = skx_get_src_id(d, 0xf8, &src_id);
if (rc < 0)
goto fail;
diff --git a/drivers/edac/i5100_edac.c b/drivers/edac/i5100_edac.c
index b506eef6b146..251f2b692785 100644
--- a/drivers/edac/i5100_edac.c
+++ b/drivers/edac/i5100_edac.c
@@ -417,7 +417,8 @@ static const char *i5100_err_msg(unsigned err)
}
/* convert csrow index into a rank (per channel -- 0..5) */
-static int i5100_csrow_to_rank(const struct mem_ctl_info *mci, int csrow)
+static unsigned int i5100_csrow_to_rank(const struct mem_ctl_info *mci,
+ unsigned int csrow)
{
const struct i5100_priv *priv = mci->pvt_info;
@@ -425,7 +426,8 @@ static int i5100_csrow_to_rank(const struct mem_ctl_info *mci, int csrow)
}
/* convert csrow index into a channel (0..1) */
-static int i5100_csrow_to_chan(const struct mem_ctl_info *mci, int csrow)
+static unsigned int i5100_csrow_to_chan(const struct mem_ctl_info *mci,
+ unsigned int csrow)
{
const struct i5100_priv *priv = mci->pvt_info;
@@ -653,11 +655,11 @@ static struct pci_dev *pci_get_device_func(unsigned vendor,
return ret;
}
-static unsigned long i5100_npages(struct mem_ctl_info *mci, int csrow)
+static unsigned long i5100_npages(struct mem_ctl_info *mci, unsigned int csrow)
{
struct i5100_priv *priv = mci->pvt_info;
- const unsigned chan_rank = i5100_csrow_to_rank(mci, csrow);
- const unsigned chan = i5100_csrow_to_chan(mci, csrow);
+ const unsigned int chan_rank = i5100_csrow_to_rank(mci, csrow);
+ const unsigned int chan = i5100_csrow_to_chan(mci, csrow);
unsigned addr_lines;
/* dimm present? */
@@ -852,8 +854,8 @@ static void i5100_init_csrows(struct mem_ctl_info *mci)
for (i = 0; i < mci->tot_dimms; i++) {
struct dimm_info *dimm;
const unsigned long npages = i5100_npages(mci, i);
- const unsigned chan = i5100_csrow_to_chan(mci, i);
- const unsigned rank = i5100_csrow_to_rank(mci, i);
+ const unsigned int chan = i5100_csrow_to_chan(mci, i);
+ const unsigned int rank = i5100_csrow_to_rank(mci, i);
if (!npages)
continue;
diff --git a/drivers/edac/ie31200_edac.c b/drivers/edac/ie31200_edac.c
index adf60eb45bd4..d26300f9cb07 100644
--- a/drivers/edac/ie31200_edac.c
+++ b/drivers/edac/ie31200_edac.c
@@ -20,11 +20,13 @@
* 0c08: Xeon E3-1200 v3 Processor DRAM Controller
* 1918: Xeon E3-1200 v5 Skylake Host Bridge/DRAM Registers
* 5918: Xeon E3-1200 Xeon E3-1200 v6/7th Gen Core Processor Host Bridge/DRAM Registers
+ * 3e..: 8th/9th Gen Core Processor Host Bridge/DRAM Registers
*
* Based on Intel specification:
* http://www.intel.com/content/dam/www/public/us/en/documents/datasheets/xeon-e3-1200v3-vol-2-datasheet.pdf
* http://www.intel.com/content/www/us/en/processors/xeon/xeon-e3-1200-family-vol-2-datasheet.html
* http://www.intel.com/content/www/us/en/processors/core/7th-gen-core-family-mobile-h-processor-lines-datasheet-vol-2.html
+ * https://www.intel.com/content/www/us/en/products/docs/processors/core/8th-gen-core-family-datasheet-vol-2.html
*
* According to the above datasheet (p.16):
* "
@@ -61,6 +63,26 @@
#define PCI_DEVICE_ID_INTEL_IE31200_HB_8 0x1918
#define PCI_DEVICE_ID_INTEL_IE31200_HB_9 0x5918
+/* Coffee Lake-S */
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_MASK 0x3e00
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_1 0x3e0f
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_2 0x3e18
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_3 0x3e1f
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_4 0x3e30
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_5 0x3e31
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_6 0x3e32
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_7 0x3e33
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_8 0x3ec2
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_9 0x3ec6
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_10 0x3eca
+
+/* Test if HB is for Skylake or later. */
+#define DEVICE_ID_SKYLAKE_OR_LATER(did) \
+ (((did) == PCI_DEVICE_ID_INTEL_IE31200_HB_8) || \
+ ((did) == PCI_DEVICE_ID_INTEL_IE31200_HB_9) || \
+ (((did) & PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_MASK) == \
+ PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_MASK))
+
#define IE31200_DIMMS 4
#define IE31200_RANKS 8
#define IE31200_RANKS_PER_CHANNEL 4
@@ -381,10 +403,10 @@ static int ie31200_probe1(struct pci_dev *pdev, int dev_idx)
u32 addr_decode, mad_offset;
/*
- * Kaby Lake seems to work like Skylake. Please re-visit this logic
- * when adding new CPU support.
+ * Kaby Lake, Coffee Lake seem to work like Skylake. Please re-visit
+ * this logic when adding new CPU support.
*/
- bool skl = (pdev->device >= PCI_DEVICE_ID_INTEL_IE31200_HB_8);
+ bool skl = DEVICE_ID_SKYLAKE_OR_LATER(pdev->device);
edac_dbg(0, "MC:\n");
@@ -542,36 +564,26 @@ static void ie31200_remove_one(struct pci_dev *pdev)
}
static const struct pci_device_id ie31200_pci_tbl[] = {
- {
- PCI_VEND_DEV(INTEL, IE31200_HB_1), PCI_ANY_ID, PCI_ANY_ID, 0, 0,
- IE31200},
- {
- PCI_VEND_DEV(INTEL, IE31200_HB_2), PCI_ANY_ID, PCI_ANY_ID, 0, 0,
- IE31200},
- {
- PCI_VEND_DEV(INTEL, IE31200_HB_3), PCI_ANY_ID, PCI_ANY_ID, 0, 0,
- IE31200},
- {
- PCI_VEND_DEV(INTEL, IE31200_HB_4), PCI_ANY_ID, PCI_ANY_ID, 0, 0,
- IE31200},
- {
- PCI_VEND_DEV(INTEL, IE31200_HB_5), PCI_ANY_ID, PCI_ANY_ID, 0, 0,
- IE31200},
- {
- PCI_VEND_DEV(INTEL, IE31200_HB_6), PCI_ANY_ID, PCI_ANY_ID, 0, 0,
- IE31200},
- {
- PCI_VEND_DEV(INTEL, IE31200_HB_7), PCI_ANY_ID, PCI_ANY_ID, 0, 0,
- IE31200},
- {
- PCI_VEND_DEV(INTEL, IE31200_HB_8), PCI_ANY_ID, PCI_ANY_ID, 0, 0,
- IE31200},
- {
- PCI_VEND_DEV(INTEL, IE31200_HB_9), PCI_ANY_ID, PCI_ANY_ID, 0, 0,
- IE31200},
- {
- 0,
- } /* 0 terminated list. */
+ { PCI_VEND_DEV(INTEL, IE31200_HB_1), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_2), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_3), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_4), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_5), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_6), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_7), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_8), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_9), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_CFL_1), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_CFL_2), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_CFL_3), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_CFL_4), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_CFL_5), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_CFL_6), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_CFL_7), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_CFL_8), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_CFL_9), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { PCI_VEND_DEV(INTEL, IE31200_HB_CFL_10), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+ { 0, } /* 0 terminated list. */
};
MODULE_DEVICE_TABLE(pci, ie31200_pci_tbl);
diff --git a/drivers/edac/pnd2_edac.c b/drivers/edac/pnd2_edac.c
index ca25f8fe57ef..b1193be1ef1d 100644
--- a/drivers/edac/pnd2_edac.c
+++ b/drivers/edac/pnd2_edac.c
@@ -260,11 +260,14 @@ static u64 get_sideband_reg_base_addr(void)
}
}
+#define DNV_MCHBAR_SIZE 0x8000
+#define DNV_SB_PORT_SIZE 0x10000
static int dnv_rd_reg(int port, int off, int op, void *data, size_t sz, char *name)
{
struct pci_dev *pdev;
char *base;
u64 addr;
+ unsigned long size;
if (op == 4) {
pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x1980, NULL);
@@ -279,15 +282,17 @@ static int dnv_rd_reg(int port, int off, int op, void *data, size_t sz, char *na
addr = get_mem_ctrl_hub_base_addr();
if (!addr)
return -ENODEV;
+ size = DNV_MCHBAR_SIZE;
} else {
/* MMIO via sideband register base address */
addr = get_sideband_reg_base_addr();
if (!addr)
return -ENODEV;
addr += (port << 16);
+ size = DNV_SB_PORT_SIZE;
}
- base = ioremap((resource_size_t)addr, 0x10000);
+ base = ioremap((resource_size_t)addr, size);
if (!base)
return -ENODEV;
@@ -1533,7 +1538,7 @@ static struct dunit_ops dnv_ops = {
static const struct x86_cpu_id pnd2_cpuids[] = {
{ X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_GOLDMONT, 0, (kernel_ulong_t)&apl_ops },
- { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_GOLDMONT_X, 0, (kernel_ulong_t)&dnv_ops },
+ { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_GOLDMONT_D, 0, (kernel_ulong_t)&dnv_ops },
{ }
};
MODULE_DEVICE_TABLE(x86cpu, pnd2_cpuids);
diff --git a/drivers/edac/sb_edac.c b/drivers/edac/sb_edac.c
index fa700a170380..f743502ca9b7 100644
--- a/drivers/edac/sb_edac.c
+++ b/drivers/edac/sb_edac.c
@@ -1511,7 +1511,6 @@ static int knl_get_dimm_capacity(struct sbridge_pvt *pvt, u64 *mc_sizes)
sad_actual_size[mc] += tad_size;
}
}
- tad_base = tad_limit+1;
}
}
@@ -3430,7 +3429,7 @@ static const struct x86_cpu_id sbridge_cpuids[] = {
INTEL_CPU_FAM6(IVYBRIDGE_X, pci_dev_descr_ibridge_table),
INTEL_CPU_FAM6(HASWELL_X, pci_dev_descr_haswell_table),
INTEL_CPU_FAM6(BROADWELL_X, pci_dev_descr_broadwell_table),
- INTEL_CPU_FAM6(BROADWELL_XEON_D, pci_dev_descr_broadwell_table),
+ INTEL_CPU_FAM6(BROADWELL_D, pci_dev_descr_broadwell_table),
INTEL_CPU_FAM6(XEON_PHI_KNL, pci_dev_descr_knl_table),
INTEL_CPU_FAM6(XEON_PHI_KNM, pci_dev_descr_knl_table),
{ }
diff --git a/drivers/edac/sifive_edac.c b/drivers/edac/sifive_edac.c
new file mode 100644
index 000000000000..413cdb4a591d
--- /dev/null
+++ b/drivers/edac/sifive_edac.c
@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * SiFive Platform EDAC Driver
+ *
+ * Copyright (C) 2018-2019 SiFive, Inc.
+ *
+ * This driver is partially based on octeon_edac-pc.c
+ *
+ */
+#include <linux/edac.h>
+#include <linux/platform_device.h>
+#include "edac_module.h"
+#include <asm/sifive_l2_cache.h>
+
+#define DRVNAME "sifive_edac"
+
+struct sifive_edac_priv {
+ struct notifier_block notifier;
+ struct edac_device_ctl_info *dci;
+};
+
+/**
+ * EDAC error callback
+ *
+ * @event: non-zero if unrecoverable.
+ */
+static
+int ecc_err_event(struct notifier_block *this, unsigned long event, void *ptr)
+{
+ const char *msg = (char *)ptr;
+ struct sifive_edac_priv *p;
+
+ p = container_of(this, struct sifive_edac_priv, notifier);
+
+ if (event == SIFIVE_L2_ERR_TYPE_UE)
+ edac_device_handle_ue(p->dci, 0, 0, msg);
+ else if (event == SIFIVE_L2_ERR_TYPE_CE)
+ edac_device_handle_ce(p->dci, 0, 0, msg);
+
+ return NOTIFY_OK;
+}
+
+static int ecc_register(struct platform_device *pdev)
+{
+ struct sifive_edac_priv *p;
+
+ p = devm_kzalloc(&pdev->dev, sizeof(*p), GFP_KERNEL);
+ if (!p)
+ return -ENOMEM;
+
+ p->notifier.notifier_call = ecc_err_event;
+ platform_set_drvdata(pdev, p);
+
+ p->dci = edac_device_alloc_ctl_info(0, "sifive_ecc", 1, "sifive_ecc",
+ 1, 1, NULL, 0,
+ edac_device_alloc_index());
+ if (IS_ERR(p->dci))
+ return PTR_ERR(p->dci);
+
+ p->dci->dev = &pdev->dev;
+ p->dci->mod_name = "Sifive ECC Manager";
+ p->dci->ctl_name = dev_name(&pdev->dev);
+ p->dci->dev_name = dev_name(&pdev->dev);
+
+ if (edac_device_add_device(p->dci)) {
+ dev_err(p->dci->dev, "failed to register with EDAC core\n");
+ goto err;
+ }
+
+ register_sifive_l2_error_notifier(&p->notifier);
+
+ return 0;
+
+err:
+ edac_device_free_ctl_info(p->dci);
+
+ return -ENXIO;
+}
+
+static int ecc_unregister(struct platform_device *pdev)
+{
+ struct sifive_edac_priv *p = platform_get_drvdata(pdev);
+
+ unregister_sifive_l2_error_notifier(&p->notifier);
+ edac_device_del_device(&pdev->dev);
+ edac_device_free_ctl_info(p->dci);
+
+ return 0;
+}
+
+static struct platform_device *sifive_pdev;
+
+static int __init sifive_edac_init(void)
+{
+ int ret;
+
+ sifive_pdev = platform_device_register_simple(DRVNAME, 0, NULL, 0);
+ if (IS_ERR(sifive_pdev))
+ return PTR_ERR(sifive_pdev);
+
+ ret = ecc_register(sifive_pdev);
+ if (ret)
+ platform_device_unregister(sifive_pdev);
+
+ return ret;
+}
+
+static void __exit sifive_edac_exit(void)
+{
+ ecc_unregister(sifive_pdev);
+ platform_device_unregister(sifive_pdev);
+}
+
+module_init(sifive_edac_init);
+module_exit(sifive_edac_exit);
+
+MODULE_AUTHOR("SiFive Inc.");
+MODULE_DESCRIPTION("SiFive platform EDAC driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/edac/skx_base.c b/drivers/edac/skx_base.c
index a5c8fa3a249a..0fcf3785e8f3 100644
--- a/drivers/edac/skx_base.c
+++ b/drivers/edac/skx_base.c
@@ -639,7 +639,7 @@ static int __init skx_init(void)
}
list_for_each_entry(d, skx_edac_list, list) {
- rc = skx_get_src_id(d, &src_id);
+ rc = skx_get_src_id(d, 0xf0, &src_id);
if (rc < 0)
goto fail;
rc = skx_get_node_id(d, &node_id);
diff --git a/drivers/edac/skx_common.c b/drivers/edac/skx_common.c
index b0dddcfa9baa..d8ff63d91b86 100644
--- a/drivers/edac/skx_common.c
+++ b/drivers/edac/skx_common.c
@@ -136,11 +136,11 @@ void skx_set_decode(skx_decode_f decode)
skx_decode = decode;
}
-int skx_get_src_id(struct skx_dev *d, u8 *id)
+int skx_get_src_id(struct skx_dev *d, int off, u8 *id)
{
u32 reg;
- if (pci_read_config_dword(d->util_all, 0xf0, &reg)) {
+ if (pci_read_config_dword(d->util_all, off, &reg)) {
skx_printk(KERN_ERR, "Failed to read src id\n");
return -ENODEV;
}
diff --git a/drivers/edac/skx_common.h b/drivers/edac/skx_common.h
index d18fa98669af..08cc971a50ea 100644
--- a/drivers/edac/skx_common.h
+++ b/drivers/edac/skx_common.h
@@ -118,7 +118,7 @@ int __init skx_adxl_get(void);
void __exit skx_adxl_put(void);
void skx_set_decode(skx_decode_f decode);
-int skx_get_src_id(struct skx_dev *d, u8 *id);
+int skx_get_src_id(struct skx_dev *d, int off, u8 *id);
int skx_get_node_id(struct skx_dev *d, u8 *id);
int skx_get_all_bus_mappings(unsigned int did, int off, enum type,
diff --git a/drivers/extcon/Kconfig b/drivers/extcon/Kconfig
index 6f5af4196b8d..fa1804460e8c 100644
--- a/drivers/extcon/Kconfig
+++ b/drivers/extcon/Kconfig
@@ -37,6 +37,18 @@ config EXTCON_AXP288
Say Y here to enable support for USB peripheral detection
and USB MUX switching by X-Power AXP288 PMIC.
+config EXTCON_FSA9480
+ tristate "FSA9480 EXTCON Support"
+ depends on INPUT && I2C
+ select IRQ_DOMAIN
+ select REGMAP_I2C
+ help
+ If you say yes here you get support for the Fairchild Semiconductor
+ FSA9480 microUSB switch and accessory detector chip. The FSA9480 is a USB
+ port accessory detector and switch. The FSA9480 is fully controlled using
+ I2C and enables USB data, stereo and mono audio, video, microphone
+ and UART data to use a common connector port.
+
config EXTCON_GPIO
tristate "GPIO extcon support"
depends on GPIOLIB || COMPILE_TEST
diff --git a/drivers/extcon/Makefile b/drivers/extcon/Makefile
index d3941a735df3..52096fd8a216 100644
--- a/drivers/extcon/Makefile
+++ b/drivers/extcon/Makefile
@@ -8,6 +8,7 @@ extcon-core-objs += extcon.o devres.o
obj-$(CONFIG_EXTCON_ADC_JACK) += extcon-adc-jack.o
obj-$(CONFIG_EXTCON_ARIZONA) += extcon-arizona.o
obj-$(CONFIG_EXTCON_AXP288) += extcon-axp288.o
+obj-$(CONFIG_EXTCON_FSA9480) += extcon-fsa9480.o
obj-$(CONFIG_EXTCON_GPIO) += extcon-gpio.o
obj-$(CONFIG_EXTCON_INTEL_INT3496) += extcon-intel-int3496.o
obj-$(CONFIG_EXTCON_INTEL_CHT_WC) += extcon-intel-cht-wc.o
diff --git a/drivers/extcon/extcon-adc-jack.c b/drivers/extcon/extcon-adc-jack.c
index ee9b5f70bfa4..ad02dc6747a4 100644
--- a/drivers/extcon/extcon-adc-jack.c
+++ b/drivers/extcon/extcon-adc-jack.c
@@ -140,10 +140,8 @@ static int adc_jack_probe(struct platform_device *pdev)
return err;
data->irq = platform_get_irq(pdev, 0);
- if (data->irq < 0) {
- dev_err(&pdev->dev, "platform_get_irq failed\n");
+ if (data->irq < 0)
return -ENODEV;
- }
err = request_any_context_irq(data->irq, adc_jack_irq_thread,
pdata->irq_flags, pdata->name, data);
diff --git a/drivers/extcon/extcon-arizona.c b/drivers/extcon/extcon-arizona.c
index bb6434726c7a..e970134c95fa 100644
--- a/drivers/extcon/extcon-arizona.c
+++ b/drivers/extcon/extcon-arizona.c
@@ -326,10 +326,12 @@ static void arizona_start_mic(struct arizona_extcon_info *info)
arizona_extcon_pulse_micbias(info);
- regmap_update_bits_check(arizona->regmap, ARIZONA_MIC_DETECT_1,
- ARIZONA_MICD_ENA, ARIZONA_MICD_ENA,
- &change);
- if (!change) {
+ ret = regmap_update_bits_check(arizona->regmap, ARIZONA_MIC_DETECT_1,
+ ARIZONA_MICD_ENA, ARIZONA_MICD_ENA,
+ &change);
+ if (ret < 0) {
+ dev_err(arizona->dev, "Failed to enable micd: %d\n", ret);
+ } else if (!change) {
regulator_disable(info->micvdd);
pm_runtime_put_autosuspend(info->dev);
}
@@ -341,12 +343,14 @@ static void arizona_stop_mic(struct arizona_extcon_info *info)
const char *widget = arizona_extcon_get_micbias(info);
struct snd_soc_dapm_context *dapm = arizona->dapm;
struct snd_soc_component *component = snd_soc_dapm_to_component(dapm);
- bool change;
+ bool change = false;
int ret;
- regmap_update_bits_check(arizona->regmap, ARIZONA_MIC_DETECT_1,
- ARIZONA_MICD_ENA, 0,
- &change);
+ ret = regmap_update_bits_check(arizona->regmap, ARIZONA_MIC_DETECT_1,
+ ARIZONA_MICD_ENA, 0,
+ &change);
+ if (ret < 0)
+ dev_err(arizona->dev, "Failed to disable micd: %d\n", ret);
ret = snd_soc_component_disable_pin(component, widget);
if (ret != 0)
@@ -1249,7 +1253,7 @@ static int arizona_extcon_get_micd_configs(struct device *dev,
int i, j;
u32 *vals;
- nconfs = device_property_read_u32_array(arizona->dev, prop, NULL, 0);
+ nconfs = device_property_count_u32(arizona->dev, prop);
if (nconfs <= 0)
return 0;
@@ -1718,12 +1722,15 @@ static int arizona_extcon_remove(struct platform_device *pdev)
struct arizona *arizona = info->arizona;
int jack_irq_rise, jack_irq_fall;
bool change;
+ int ret;
- regmap_update_bits_check(arizona->regmap, ARIZONA_MIC_DETECT_1,
- ARIZONA_MICD_ENA, 0,
- &change);
-
- if (change) {
+ ret = regmap_update_bits_check(arizona->regmap, ARIZONA_MIC_DETECT_1,
+ ARIZONA_MICD_ENA, 0,
+ &change);
+ if (ret < 0) {
+ dev_err(&pdev->dev, "Failed to disable micd on remove: %d\n",
+ ret);
+ } else if (change) {
regulator_disable(info->micvdd);
pm_runtime_put(info->dev);
}
diff --git a/drivers/extcon/extcon-axp288.c b/drivers/extcon/extcon-axp288.c
index 7254852e6ec0..415afaf479e7 100644
--- a/drivers/extcon/extcon-axp288.c
+++ b/drivers/extcon/extcon-axp288.c
@@ -121,7 +121,6 @@ static const char * const axp288_pwr_up_down_info[] = {
"Last shutdown caused by PMIC UVLO threshold",
"Last shutdown caused by SOC initiated cold off",
"Last shutdown caused by user pressing the power button",
- NULL,
};
/*
@@ -130,18 +129,21 @@ static const char * const axp288_pwr_up_down_info[] = {
*/
static void axp288_extcon_log_rsi(struct axp288_extcon_info *info)
{
- const char * const *rsi;
unsigned int val, i, clear_mask = 0;
+ unsigned long bits;
int ret;
ret = regmap_read(info->regmap, AXP288_PS_BOOT_REASON_REG, &val);
- for (i = 0, rsi = axp288_pwr_up_down_info; *rsi; rsi++, i++) {
- if (val & BIT(i)) {
- dev_dbg(info->dev, "%s\n", *rsi);
- clear_mask |= BIT(i);
- }
+ if (ret < 0) {
+ dev_err(info->dev, "failed to read reset source indicator\n");
+ return;
}
+ bits = val & GENMASK(ARRAY_SIZE(axp288_pwr_up_down_info) - 1, 0);
+ for_each_set_bit(i, &bits, ARRAY_SIZE(axp288_pwr_up_down_info))
+ dev_dbg(info->dev, "%s\n", axp288_pwr_up_down_info[i]);
+ clear_mask = bits;
+
/* Clear the register value for next reboot (write 1 to clear bit) */
regmap_write(info->regmap, AXP288_PS_BOOT_REASON_REG, clear_mask);
}
diff --git a/drivers/extcon/extcon-fsa9480.c b/drivers/extcon/extcon-fsa9480.c
new file mode 100644
index 000000000000..8405512f5199
--- /dev/null
+++ b/drivers/extcon/extcon-fsa9480.c
@@ -0,0 +1,396 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * extcon-fsa9480.c - Fairchild Semiconductor FSA9480 extcon driver
+ *
+ * Copyright (c) 2019 Tomasz Figa <tomasz.figa@gmail.com>
+ *
+ * Loosely based on old fsa9480 misc-device driver.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/i2c.h>
+#include <linux/slab.h>
+#include <linux/bitops.h>
+#include <linux/interrupt.h>
+#include <linux/err.h>
+#include <linux/platform_device.h>
+#include <linux/kobject.h>
+#include <linux/extcon-provider.h>
+#include <linux/irqdomain.h>
+#include <linux/regmap.h>
+
+/* FSA9480 I2C registers */
+#define FSA9480_REG_DEVID 0x01
+#define FSA9480_REG_CTRL 0x02
+#define FSA9480_REG_INT1 0x03
+#define FSA9480_REG_INT2 0x04
+#define FSA9480_REG_INT1_MASK 0x05
+#define FSA9480_REG_INT2_MASK 0x06
+#define FSA9480_REG_ADC 0x07
+#define FSA9480_REG_TIMING1 0x08
+#define FSA9480_REG_TIMING2 0x09
+#define FSA9480_REG_DEV_T1 0x0a
+#define FSA9480_REG_DEV_T2 0x0b
+#define FSA9480_REG_BTN1 0x0c
+#define FSA9480_REG_BTN2 0x0d
+#define FSA9480_REG_CK 0x0e
+#define FSA9480_REG_CK_INT1 0x0f
+#define FSA9480_REG_CK_INT2 0x10
+#define FSA9480_REG_CK_INTMASK1 0x11
+#define FSA9480_REG_CK_INTMASK2 0x12
+#define FSA9480_REG_MANSW1 0x13
+#define FSA9480_REG_MANSW2 0x14
+#define FSA9480_REG_END 0x15
+
+/* Control */
+#define CON_SWITCH_OPEN (1 << 4)
+#define CON_RAW_DATA (1 << 3)
+#define CON_MANUAL_SW (1 << 2)
+#define CON_WAIT (1 << 1)
+#define CON_INT_MASK (1 << 0)
+#define CON_MASK (CON_SWITCH_OPEN | CON_RAW_DATA | \
+ CON_MANUAL_SW | CON_WAIT)
+
+/* Device Type 1 */
+#define DEV_USB_OTG 7
+#define DEV_DEDICATED_CHG 6
+#define DEV_USB_CHG 5
+#define DEV_CAR_KIT 4
+#define DEV_UART 3
+#define DEV_USB 2
+#define DEV_AUDIO_2 1
+#define DEV_AUDIO_1 0
+
+#define DEV_T1_USB_MASK (DEV_USB_OTG | DEV_USB)
+#define DEV_T1_UART_MASK (DEV_UART)
+#define DEV_T1_CHARGER_MASK (DEV_DEDICATED_CHG | DEV_USB_CHG)
+
+/* Device Type 2 */
+#define DEV_AV 14
+#define DEV_TTY 13
+#define DEV_PPD 12
+#define DEV_JIG_UART_OFF 11
+#define DEV_JIG_UART_ON 10
+#define DEV_JIG_USB_OFF 9
+#define DEV_JIG_USB_ON 8
+
+#define DEV_T2_USB_MASK (DEV_JIG_USB_OFF | DEV_JIG_USB_ON)
+#define DEV_T2_UART_MASK (DEV_JIG_UART_OFF | DEV_JIG_UART_ON)
+#define DEV_T2_JIG_MASK (DEV_JIG_USB_OFF | DEV_JIG_USB_ON | \
+ DEV_JIG_UART_OFF | DEV_JIG_UART_ON)
+
+/*
+ * Manual Switch
+ * D- [7:5] / D+ [4:2]
+ * 000: Open all / 001: USB / 010: AUDIO / 011: UART / 100: V_AUDIO
+ */
+#define SW_VAUDIO ((4 << 5) | (4 << 2))
+#define SW_UART ((3 << 5) | (3 << 2))
+#define SW_AUDIO ((2 << 5) | (2 << 2))
+#define SW_DHOST ((1 << 5) | (1 << 2))
+#define SW_AUTO ((0 << 5) | (0 << 2))
+
+/* Interrupt 1 */
+#define INT1_MASK (0xff << 0)
+#define INT_DETACH (1 << 1)
+#define INT_ATTACH (1 << 0)
+
+/* Interrupt 2 mask */
+#define INT2_MASK (0x1f << 0)
+
+/* Timing Set 1 */
+#define TIMING1_ADC_500MS (0x6 << 0)
+
+struct fsa9480_usbsw {
+ struct device *dev;
+ struct regmap *regmap;
+ struct extcon_dev *edev;
+ u16 cable;
+};
+
+static const unsigned int fsa9480_extcon_cable[] = {
+ EXTCON_USB_HOST,
+ EXTCON_USB,
+ EXTCON_CHG_USB_DCP,
+ EXTCON_CHG_USB_SDP,
+ EXTCON_CHG_USB_ACA,
+ EXTCON_JACK_LINE_OUT,
+ EXTCON_JACK_VIDEO_OUT,
+ EXTCON_JIG,
+
+ EXTCON_NONE,
+};
+
+static const u64 cable_types[] = {
+ [DEV_USB_OTG] = BIT_ULL(EXTCON_USB_HOST),
+ [DEV_DEDICATED_CHG] = BIT_ULL(EXTCON_USB) | BIT_ULL(EXTCON_CHG_USB_DCP),
+ [DEV_USB_CHG] = BIT_ULL(EXTCON_USB) | BIT_ULL(EXTCON_CHG_USB_SDP),
+ [DEV_CAR_KIT] = BIT_ULL(EXTCON_USB) | BIT_ULL(EXTCON_CHG_USB_SDP)
+ | BIT_ULL(EXTCON_JACK_LINE_OUT),
+ [DEV_UART] = BIT_ULL(EXTCON_JIG),
+ [DEV_USB] = BIT_ULL(EXTCON_USB) | BIT_ULL(EXTCON_CHG_USB_SDP),
+ [DEV_AUDIO_2] = BIT_ULL(EXTCON_JACK_LINE_OUT),
+ [DEV_AUDIO_1] = BIT_ULL(EXTCON_JACK_LINE_OUT),
+ [DEV_AV] = BIT_ULL(EXTCON_JACK_LINE_OUT)
+ | BIT_ULL(EXTCON_JACK_VIDEO_OUT),
+ [DEV_TTY] = BIT_ULL(EXTCON_JIG),
+ [DEV_PPD] = BIT_ULL(EXTCON_JACK_LINE_OUT) | BIT_ULL(EXTCON_CHG_USB_ACA),
+ [DEV_JIG_UART_OFF] = BIT_ULL(EXTCON_JIG),
+ [DEV_JIG_UART_ON] = BIT_ULL(EXTCON_JIG),
+ [DEV_JIG_USB_OFF] = BIT_ULL(EXTCON_USB) | BIT_ULL(EXTCON_JIG),
+ [DEV_JIG_USB_ON] = BIT_ULL(EXTCON_USB) | BIT_ULL(EXTCON_JIG),
+};
+
+/* Define regmap configuration of FSA9480 for I2C communication */
+static bool fsa9480_volatile_reg(struct device *dev, unsigned int reg)
+{
+ switch (reg) {
+ case FSA9480_REG_INT1_MASK:
+ return true;
+ default:
+ break;
+ }
+ return false;
+}
+
+static const struct regmap_config fsa9480_regmap_config = {
+ .reg_bits = 8,
+ .val_bits = 8,
+ .volatile_reg = fsa9480_volatile_reg,
+ .max_register = FSA9480_REG_END,
+};
+
+static int fsa9480_write_reg(struct fsa9480_usbsw *usbsw, int reg, int value)
+{
+ int ret;
+
+ ret = regmap_write(usbsw->regmap, reg, value);
+ if (ret < 0)
+ dev_err(usbsw->dev, "%s: err %d\n", __func__, ret);
+
+ return ret;
+}
+
+static int fsa9480_read_reg(struct fsa9480_usbsw *usbsw, int reg)
+{
+ int ret, val;
+
+ ret = regmap_read(usbsw->regmap, reg, &val);
+ if (ret < 0) {
+ dev_err(usbsw->dev, "%s: err %d\n", __func__, ret);
+ return ret;
+ }
+
+ return val;
+}
+
+static int fsa9480_read_irq(struct fsa9480_usbsw *usbsw, int *value)
+{
+ u8 regs[2];
+ int ret;
+
+ ret = regmap_bulk_read(usbsw->regmap, FSA9480_REG_INT1, regs, 2);
+ if (ret < 0)
+ dev_err(usbsw->dev, "%s: err %d\n", __func__, ret);
+
+ *value = regs[1] << 8 | regs[0];
+ return ret;
+}
+
+static void fsa9480_handle_change(struct fsa9480_usbsw *usbsw,
+ u16 mask, bool attached)
+{
+ while (mask) {
+ int dev = fls64(mask) - 1;
+ u64 cables = cable_types[dev];
+
+ while (cables) {
+ int cable = fls64(cables) - 1;
+
+ extcon_set_state_sync(usbsw->edev, cable, attached);
+ cables &= ~BIT_ULL(cable);
+ }
+
+ mask &= ~BIT_ULL(dev);
+ }
+}
+
+static void fsa9480_detect_dev(struct fsa9480_usbsw *usbsw)
+{
+ int val1, val2;
+ u16 val;
+
+ val1 = fsa9480_read_reg(usbsw, FSA9480_REG_DEV_T1);
+ val2 = fsa9480_read_reg(usbsw, FSA9480_REG_DEV_T2);
+ if (val1 < 0 || val2 < 0) {
+ dev_err(usbsw->dev, "%s: failed to read registers", __func__);
+ return;
+ }
+ val = val2 << 8 | val1;
+
+ dev_info(usbsw->dev, "dev1: 0x%x, dev2: 0x%x\n", val1, val2);
+
+ /* handle detached cables first */
+ fsa9480_handle_change(usbsw, usbsw->cable & ~val, false);
+
+ /* then handle attached ones */
+ fsa9480_handle_change(usbsw, val & ~usbsw->cable, true);
+
+ usbsw->cable = val;
+}
+
+static irqreturn_t fsa9480_irq_handler(int irq, void *data)
+{
+ struct fsa9480_usbsw *usbsw = data;
+ int intr = 0;
+
+ /* clear interrupt */
+ fsa9480_read_irq(usbsw, &intr);
+ if (!intr)
+ return IRQ_NONE;
+
+ /* device detection */
+ fsa9480_detect_dev(usbsw);
+
+ return IRQ_HANDLED;
+}
+
+static int fsa9480_probe(struct i2c_client *client,
+ const struct i2c_device_id *id)
+{
+ struct fsa9480_usbsw *info;
+ int ret;
+
+ if (!client->irq) {
+ dev_err(&client->dev, "no interrupt provided\n");
+ return -EINVAL;
+ }
+
+ info = devm_kzalloc(&client->dev, sizeof(*info), GFP_KERNEL);
+ if (!info)
+ return -ENOMEM;
+ info->dev = &client->dev;
+
+ i2c_set_clientdata(client, info);
+
+ /* External connector */
+ info->edev = devm_extcon_dev_allocate(info->dev,
+ fsa9480_extcon_cable);
+ if (IS_ERR(info->edev)) {
+ dev_err(info->dev, "failed to allocate memory for extcon\n");
+ ret = -ENOMEM;
+ return ret;
+ }
+
+ ret = devm_extcon_dev_register(info->dev, info->edev);
+ if (ret) {
+ dev_err(info->dev, "failed to register extcon device\n");
+ return ret;
+ }
+
+ info->regmap = devm_regmap_init_i2c(client, &fsa9480_regmap_config);
+ if (IS_ERR(info->regmap)) {
+ ret = PTR_ERR(info->regmap);
+ dev_err(info->dev, "failed to allocate register map: %d\n",
+ ret);
+ return ret;
+ }
+
+ /* ADC Detect Time: 500ms */
+ fsa9480_write_reg(info, FSA9480_REG_TIMING1, TIMING1_ADC_500MS);
+
+ /* configure automatic switching */
+ fsa9480_write_reg(info, FSA9480_REG_CTRL, CON_MASK);
+
+ /* unmask interrupt (attach/detach only) */
+ fsa9480_write_reg(info, FSA9480_REG_INT1_MASK,
+ INT1_MASK & ~(INT_ATTACH | INT_DETACH));
+ fsa9480_write_reg(info, FSA9480_REG_INT2_MASK, INT2_MASK);
+
+ ret = devm_request_threaded_irq(info->dev, client->irq, NULL,
+ fsa9480_irq_handler,
+ IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
+ "fsa9480", info);
+ if (ret) {
+ dev_err(info->dev, "failed to request IRQ\n");
+ return ret;
+ }
+
+ device_init_wakeup(info->dev, true);
+ fsa9480_detect_dev(info);
+
+ return 0;
+}
+
+static int fsa9480_remove(struct i2c_client *client)
+{
+ return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int fsa9480_suspend(struct device *dev)
+{
+ struct i2c_client *client = to_i2c_client(dev);
+
+ if (device_may_wakeup(&client->dev) && client->irq)
+ enable_irq_wake(client->irq);
+
+ return 0;
+}
+
+static int fsa9480_resume(struct device *dev)
+{
+ struct i2c_client *client = to_i2c_client(dev);
+
+ if (device_may_wakeup(&client->dev) && client->irq)
+ disable_irq_wake(client->irq);
+
+ return 0;
+}
+#endif
+
+static const struct dev_pm_ops fsa9480_pm_ops = {
+ SET_SYSTEM_SLEEP_PM_OPS(fsa9480_suspend, fsa9480_resume)
+};
+
+static const struct i2c_device_id fsa9480_id[] = {
+ { "fsa9480", 0 },
+ {}
+};
+MODULE_DEVICE_TABLE(i2c, fsa9480_id);
+
+static const struct of_device_id fsa9480_of_match[] = {
+ { .compatible = "fcs,fsa9480", },
+ { .compatible = "fcs,fsa880", },
+ { },
+};
+MODULE_DEVICE_TABLE(of, fsa9480_of_match);
+
+static struct i2c_driver fsa9480_i2c_driver = {
+ .driver = {
+ .name = "fsa9480",
+ .pm = &fsa9480_pm_ops,
+ .of_match_table = fsa9480_of_match,
+ },
+ .probe = fsa9480_probe,
+ .remove = fsa9480_remove,
+ .id_table = fsa9480_id,
+};
+
+static int __init fsa9480_module_init(void)
+{
+ return i2c_add_driver(&fsa9480_i2c_driver);
+}
+subsys_initcall(fsa9480_module_init);
+
+static void __exit fsa9480_module_exit(void)
+{
+ i2c_del_driver(&fsa9480_i2c_driver);
+}
+module_exit(fsa9480_module_exit);
+
+MODULE_DESCRIPTION("Fairchild Semiconductor FSA9480 extcon driver");
+MODULE_AUTHOR("Tomasz Figa <tomasz.figa@gmail.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/extcon/extcon-gpio.c b/drivers/extcon/extcon-gpio.c
index faddeac948db..c211222f5d0c 100644
--- a/drivers/extcon/extcon-gpio.c
+++ b/drivers/extcon/extcon-gpio.c
@@ -22,26 +22,22 @@
/**
* struct gpio_extcon_data - A simple GPIO-controlled extcon device state container.
* @edev: Extcon device.
- * @irq: Interrupt line for the external connector.
* @work: Work fired by the interrupt.
* @debounce_jiffies: Number of jiffies to wait for the GPIO to stabilize, from the debounce
* value.
* @gpiod: GPIO descriptor for this external connector.
* @extcon_id: The unique id of specific external connector.
* @debounce: Debounce time for GPIO IRQ in ms.
- * @irq_flags: IRQ Flags (e.g., IRQF_TRIGGER_LOW).
* @check_on_resume: Boolean describing whether to check the state of gpio
* while resuming from sleep.
*/
struct gpio_extcon_data {
struct extcon_dev *edev;
- int irq;
struct delayed_work work;
unsigned long debounce_jiffies;
struct gpio_desc *gpiod;
unsigned int extcon_id;
unsigned long debounce;
- unsigned long irq_flags;
bool check_on_resume;
};
@@ -69,6 +65,8 @@ static int gpio_extcon_probe(struct platform_device *pdev)
{
struct gpio_extcon_data *data;
struct device *dev = &pdev->dev;
+ unsigned long irq_flags;
+ int irq;
int ret;
data = devm_kzalloc(dev, sizeof(struct gpio_extcon_data), GFP_KERNEL);
@@ -82,15 +80,26 @@ static int gpio_extcon_probe(struct platform_device *pdev)
* developed to get the extcon id from device-tree or others.
* On later, it have to be solved.
*/
- if (!data->irq_flags || data->extcon_id > EXTCON_NONE)
+ if (data->extcon_id > EXTCON_NONE)
return -EINVAL;
data->gpiod = devm_gpiod_get(dev, "extcon", GPIOD_IN);
if (IS_ERR(data->gpiod))
return PTR_ERR(data->gpiod);
- data->irq = gpiod_to_irq(data->gpiod);
- if (data->irq <= 0)
- return data->irq;
+ irq = gpiod_to_irq(data->gpiod);
+ if (irq <= 0)
+ return irq;
+
+ /*
+ * It is unlikely that this is an acknowledged interrupt that goes
+ * away after handling, what we are looking for are falling edges
+ * if the signal is active low, and rising edges if the signal is
+ * active high.
+ */
+ if (gpiod_is_active_low(data->gpiod))
+ irq_flags = IRQF_TRIGGER_FALLING;
+ else
+ irq_flags = IRQF_TRIGGER_RISING;
/* Allocate the memory of extcon devie and register extcon device */
data->edev = devm_extcon_dev_allocate(dev, &data->extcon_id);
@@ -109,8 +118,8 @@ static int gpio_extcon_probe(struct platform_device *pdev)
* Request the interrupt of gpio to detect whether external connector
* is attached or detached.
*/
- ret = devm_request_any_context_irq(dev, data->irq,
- gpio_irq_handler, data->irq_flags,
+ ret = devm_request_any_context_irq(dev, irq,
+ gpio_irq_handler, irq_flags,
pdev->name, data);
if (ret < 0)
return ret;
diff --git a/drivers/extcon/extcon-max77843.c b/drivers/extcon/extcon-max77843.c
index a343a6ef3506..e6b50ca83008 100644
--- a/drivers/extcon/extcon-max77843.c
+++ b/drivers/extcon/extcon-max77843.c
@@ -774,12 +774,12 @@ static int max77843_init_muic_regmap(struct max77693_dev *max77843)
{
int ret;
- max77843->i2c_muic = i2c_new_dummy(max77843->i2c->adapter,
+ max77843->i2c_muic = i2c_new_dummy_device(max77843->i2c->adapter,
I2C_ADDR_MUIC);
- if (!max77843->i2c_muic) {
+ if (IS_ERR(max77843->i2c_muic)) {
dev_err(&max77843->i2c->dev,
"Cannot allocate I2C device for MUIC\n");
- return -ENOMEM;
+ return PTR_ERR(max77843->i2c_muic);
}
i2c_set_clientdata(max77843->i2c_muic, max77843);
diff --git a/drivers/extcon/extcon-sm5502.c b/drivers/extcon/extcon-sm5502.c
index 98e4f616b8f1..dc43847ad2b0 100644
--- a/drivers/extcon/extcon-sm5502.c
+++ b/drivers/extcon/extcon-sm5502.c
@@ -597,7 +597,7 @@ static int sm5022_muic_i2c_probe(struct i2c_client *i2c,
ret = devm_request_threaded_irq(info->dev, virq, NULL,
sm5502_muic_irq_handler,
- IRQF_NO_SUSPEND,
+ IRQF_NO_SUSPEND | IRQF_ONESHOT,
muic_irq->name, info);
if (ret) {
dev_err(info->dev,
diff --git a/drivers/firewire/core-device.c b/drivers/firewire/core-device.c
index 3dc1cbf849db..b785e936244f 100644
--- a/drivers/firewire/core-device.c
+++ b/drivers/firewire/core-device.c
@@ -957,7 +957,7 @@ static void set_broadcast_channel(struct fw_device *device, int generation)
device->bc_implemented = BC_IMPLEMENTED;
break;
}
- /* else fall through to case address error */
+ /* else, fall through - to case address error */
case RCODE_ADDRESS_ERROR:
device->bc_implemented = BC_UNIMPLEMENTED;
}
diff --git a/drivers/firewire/core-iso.c b/drivers/firewire/core-iso.c
index 42566b7be8f5..df8a56a979b9 100644
--- a/drivers/firewire/core-iso.c
+++ b/drivers/firewire/core-iso.c
@@ -284,7 +284,7 @@ static int manage_channel(struct fw_card *card, int irm_id, int generation,
if ((data[0] & bit) == (data[1] & bit))
continue;
- /* 1394-1995 IRM, fall through to retry. */
+ /* fall through - It's a 1394-1995 IRM, retry. */
default:
if (retry) {
retry--;
diff --git a/drivers/firewire/core-topology.c b/drivers/firewire/core-topology.c
index 46bd22dde535..94a13fca8267 100644
--- a/drivers/firewire/core-topology.c
+++ b/drivers/firewire/core-topology.c
@@ -54,6 +54,7 @@ static u32 *count_ports(u32 *sid, int *total_port_count, int *child_port_count)
switch (port_type) {
case SELFID_PORT_CHILD:
(*child_port_count)++;
+ /* fall through */
case SELFID_PORT_PARENT:
case SELFID_PORT_NCONN:
(*total_port_count)++;
diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig
index d40ccc3af9e2..e40a77bfe821 100644
--- a/drivers/firmware/Kconfig
+++ b/drivers/firmware/Kconfig
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# For a description of the syntax of this configuration file,
-# see Documentation/kbuild/kconfig-language.txt.
+# see Documentation/kbuild/kconfig-language.rst.
#
menu "Firmware Drivers"
@@ -157,7 +157,7 @@ config DMI_SCAN_MACHINE_NON_EFI_FALLBACK
config ISCSI_IBFT_FIND
bool "iSCSI Boot Firmware Table Attributes"
- depends on X86 && ACPI
+ depends on X86 && ISCSI_IBFT
default n
help
This option enables the kernel to find the region of memory
@@ -168,7 +168,8 @@ config ISCSI_IBFT_FIND
config ISCSI_IBFT
tristate "iSCSI Boot Firmware Table Attributes module"
select ISCSI_BOOT_SYSFS
- depends on ISCSI_IBFT_FIND && SCSI && SCSI_LOWLEVEL
+ select ISCSI_IBFT_FIND if X86
+ depends on ACPI && SCSI && SCSI_LOWLEVEL
default n
help
This option enables support for detection and exposing of iSCSI
@@ -215,6 +216,24 @@ config INTEL_STRATIX10_SERVICE
Say Y here if you want Stratix10 service layer support.
+config INTEL_STRATIX10_RSU
+ tristate "Intel Stratix10 Remote System Update"
+ depends on INTEL_STRATIX10_SERVICE
+ help
+ The Intel Remote System Update (RSU) driver exposes interfaces
+ access through the Intel Service Layer to user space via sysfs
+ device attribute nodes. The RSU interfaces report/control some of
+ the optional RSU features of the Stratix 10 SoC FPGA.
+
+ The RSU provides a way for customers to update the boot
+ configuration of a Stratix 10 SoC device with significantly reduced
+ risk of corrupting the bitstream storage and bricking the system.
+
+ Enable RSU support if you are using an Intel SoC FPGA with the RSU
+ feature enabled and you want Linux user space control.
+
+ Say Y here if you want Intel RSU support.
+
config QCOM_SCM
bool
depends on ARM || ARM64
@@ -270,6 +289,20 @@ config TRUSTED_FOUNDATIONS
Choose N if you don't know what this is about.
+config TURRIS_MOX_RWTM
+ tristate "Turris Mox rWTM secure firmware driver"
+ depends on ARCH_MVEBU || COMPILE_TEST
+ depends on HAS_DMA && OF
+ depends on MAILBOX
+ select HW_RANDOM
+ select ARMADA_37XX_RWTM_MBOX
+ help
+ This driver communicates with the firmware on the Cortex-M3 secure
+ processor of the Turris Mox router. Enable if you are building for
+ Turris Mox, and you will be able to read the device serial number and
+ other manufacturing data and also utilize the Entropy Bit Generator
+ for hardware random number generation.
+
config HAVE_ARM_SMCCC
bool
diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile
index 3fa0b34eb72f..3fcb91975bdc 100644
--- a/drivers/firmware/Makefile
+++ b/drivers/firmware/Makefile
@@ -11,6 +11,7 @@ obj-$(CONFIG_EDD) += edd.o
obj-$(CONFIG_EFI_PCDP) += pcdp.o
obj-$(CONFIG_DMIID) += dmi-id.o
obj-$(CONFIG_INTEL_STRATIX10_SERVICE) += stratix10-svc.o
+obj-$(CONFIG_INTEL_STRATIX10_RSU) += stratix10-rsu.o
obj-$(CONFIG_ISCSI_IBFT_FIND) += iscsi_ibft_find.o
obj-$(CONFIG_ISCSI_IBFT) += iscsi_ibft.o
obj-$(CONFIG_FIRMWARE_MEMMAP) += memmap.o
@@ -22,6 +23,7 @@ obj-$(CONFIG_QCOM_SCM_32) += qcom_scm-32.o
CFLAGS_qcom_scm-32.o :=$(call as-instr,.arch armv7-a\n.arch_extension sec,-DREQUIRES_SEC=1) -march=armv7-a
obj-$(CONFIG_TI_SCI_PROTOCOL) += ti_sci.o
obj-$(CONFIG_TRUSTED_FOUNDATIONS) += trusted_foundations.o
+obj-$(CONFIG_TURRIS_MOX_RWTM) += turris-mox-rwtm.o
obj-$(CONFIG_ARM_SCMI_PROTOCOL) += arm_scmi/
obj-y += psci/
diff --git a/drivers/firmware/arm_scmi/Makefile b/drivers/firmware/arm_scmi/Makefile
index c47d28d556b6..5f298f00a82e 100644
--- a/drivers/firmware/arm_scmi/Makefile
+++ b/drivers/firmware/arm_scmi/Makefile
@@ -2,5 +2,5 @@
obj-y = scmi-bus.o scmi-driver.o scmi-protocols.o
scmi-bus-y = bus.o
scmi-driver-y = driver.o
-scmi-protocols-y = base.o clock.o perf.o power.o sensors.o
+scmi-protocols-y = base.o clock.o perf.o power.o reset.o sensors.o
obj-$(CONFIG_ARM_SCMI_POWER_DOMAIN) += scmi_pm_domain.o
diff --git a/drivers/firmware/arm_scmi/base.c b/drivers/firmware/arm_scmi/base.c
index 204390297f4b..f804e8af6521 100644
--- a/drivers/firmware/arm_scmi/base.c
+++ b/drivers/firmware/arm_scmi/base.c
@@ -204,7 +204,7 @@ static int scmi_base_discover_agent_get(const struct scmi_handle *handle,
if (ret)
return ret;
- *(__le32 *)t->tx.buf = cpu_to_le32(id);
+ put_unaligned_le32(id, t->tx.buf);
ret = scmi_do_xfer(handle, t);
if (!ret)
diff --git a/drivers/firmware/arm_scmi/clock.c b/drivers/firmware/arm_scmi/clock.c
index 30fc04e28431..32526a793f3a 100644
--- a/drivers/firmware/arm_scmi/clock.c
+++ b/drivers/firmware/arm_scmi/clock.c
@@ -56,7 +56,7 @@ struct scmi_msg_resp_clock_describe_rates {
struct scmi_clock_set_rate {
__le32 flags;
#define CLOCK_SET_ASYNC BIT(0)
-#define CLOCK_SET_DELAYED BIT(1)
+#define CLOCK_SET_IGNORE_RESP BIT(1)
#define CLOCK_SET_ROUND_UP BIT(2)
#define CLOCK_SET_ROUND_AUTO BIT(3)
__le32 id;
@@ -67,6 +67,7 @@ struct scmi_clock_set_rate {
struct clock_info {
int num_clocks;
int max_async_req;
+ atomic_t cur_async_req;
struct scmi_clock_info *clk;
};
@@ -106,7 +107,7 @@ static int scmi_clock_attributes_get(const struct scmi_handle *handle,
if (ret)
return ret;
- *(__le32 *)t->tx.buf = cpu_to_le32(clk_id);
+ put_unaligned_le32(clk_id, t->tx.buf);
attr = t->rx.buf;
ret = scmi_do_xfer(handle, t);
@@ -185,6 +186,8 @@ scmi_clock_describe_rates_get(const struct scmi_handle *handle, u32 clk_id,
if (rate_discrete)
clk->list.num_rates = tot_rate_cnt;
+ clk->rate_discrete = rate_discrete;
+
err:
scmi_xfer_put(handle, t);
return ret;
@@ -201,39 +204,47 @@ scmi_clock_rate_get(const struct scmi_handle *handle, u32 clk_id, u64 *value)
if (ret)
return ret;
- *(__le32 *)t->tx.buf = cpu_to_le32(clk_id);
+ put_unaligned_le32(clk_id, t->tx.buf);
ret = scmi_do_xfer(handle, t);
- if (!ret) {
- __le32 *pval = t->rx.buf;
-
- *value = le32_to_cpu(*pval);
- *value |= (u64)le32_to_cpu(*(pval + 1)) << 32;
- }
+ if (!ret)
+ *value = get_unaligned_le64(t->rx.buf);
scmi_xfer_put(handle, t);
return ret;
}
static int scmi_clock_rate_set(const struct scmi_handle *handle, u32 clk_id,
- u32 config, u64 rate)
+ u64 rate)
{
int ret;
+ u32 flags = 0;
struct scmi_xfer *t;
struct scmi_clock_set_rate *cfg;
+ struct clock_info *ci = handle->clk_priv;
ret = scmi_xfer_get_init(handle, CLOCK_RATE_SET, SCMI_PROTOCOL_CLOCK,
sizeof(*cfg), 0, &t);
if (ret)
return ret;
+ if (ci->max_async_req &&
+ atomic_inc_return(&ci->cur_async_req) < ci->max_async_req)
+ flags |= CLOCK_SET_ASYNC;
+
cfg = t->tx.buf;
- cfg->flags = cpu_to_le32(config);
+ cfg->flags = cpu_to_le32(flags);
cfg->id = cpu_to_le32(clk_id);
cfg->value_low = cpu_to_le32(rate & 0xffffffff);
cfg->value_high = cpu_to_le32(rate >> 32);
- ret = scmi_do_xfer(handle, t);
+ if (flags & CLOCK_SET_ASYNC)
+ ret = scmi_do_xfer_with_response(handle, t);
+ else
+ ret = scmi_do_xfer(handle, t);
+
+ if (ci->max_async_req)
+ atomic_dec(&ci->cur_async_req);
scmi_xfer_put(handle, t);
return ret;
diff --git a/drivers/firmware/arm_scmi/common.h b/drivers/firmware/arm_scmi/common.h
index 937a930ce87d..5237c2ff79fe 100644
--- a/drivers/firmware/arm_scmi/common.h
+++ b/drivers/firmware/arm_scmi/common.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
/*
* System Control and Management Interface (SCMI) Message Protocol
* driver common header file containing some definitions, structures
@@ -15,6 +15,8 @@
#include <linux/scmi_protocol.h>
#include <linux/types.h>
+#include <asm/unaligned.h>
+
#define PROTOCOL_REV_MINOR_MASK GENMASK(15, 0)
#define PROTOCOL_REV_MAJOR_MASK GENMASK(31, 16)
#define PROTOCOL_REV_MAJOR(x) (u16)(FIELD_GET(PROTOCOL_REV_MAJOR_MASK, (x)))
@@ -48,11 +50,11 @@ struct scmi_msg_resp_prot_version {
/**
* struct scmi_msg_hdr - Message(Tx/Rx) header
*
- * @id: The identifier of the command being sent
- * @protocol_id: The identifier of the protocol used to send @id command
- * @seq: The token to identify the message. when a message/command returns,
- * the platform returns the whole message header unmodified including
- * the token
+ * @id: The identifier of the message being sent
+ * @protocol_id: The identifier of the protocol used to send @id message
+ * @seq: The token to identify the message. When a message returns, the
+ * platform returns the whole message header unmodified including the
+ * token
* @status: Status of the transfer once it's complete
* @poll_completion: Indicate if the transfer needs to be polled for
* completion or interrupt mode is used
@@ -84,17 +86,21 @@ struct scmi_msg {
* @rx: Receive message, the buffer should be pre-allocated to store
* message. If request-ACK protocol is used, we can reuse the same
* buffer for the rx path as we use for the tx path.
- * @done: completion event
+ * @done: command message transmit completion event
+ * @async: pointer to delayed response message received event completion
*/
struct scmi_xfer {
struct scmi_msg_hdr hdr;
struct scmi_msg tx;
struct scmi_msg rx;
struct completion done;
+ struct completion *async_done;
};
void scmi_xfer_put(const struct scmi_handle *h, struct scmi_xfer *xfer);
int scmi_do_xfer(const struct scmi_handle *h, struct scmi_xfer *xfer);
+int scmi_do_xfer_with_response(const struct scmi_handle *h,
+ struct scmi_xfer *xfer);
int scmi_xfer_get_init(const struct scmi_handle *h, u8 msg_id, u8 prot_id,
size_t tx_size, size_t rx_size, struct scmi_xfer **p);
int scmi_handle_put(const struct scmi_handle *handle);
diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c
index b5bc4c7a8fab..3eb0382491ce 100644
--- a/drivers/firmware/arm_scmi/driver.c
+++ b/drivers/firmware/arm_scmi/driver.c
@@ -30,8 +30,14 @@
#include "common.h"
#define MSG_ID_MASK GENMASK(7, 0)
+#define MSG_XTRACT_ID(hdr) FIELD_GET(MSG_ID_MASK, (hdr))
#define MSG_TYPE_MASK GENMASK(9, 8)
+#define MSG_XTRACT_TYPE(hdr) FIELD_GET(MSG_TYPE_MASK, (hdr))
+#define MSG_TYPE_COMMAND 0
+#define MSG_TYPE_DELAYED_RESP 2
+#define MSG_TYPE_NOTIFICATION 3
#define MSG_PROTOCOL_ID_MASK GENMASK(17, 10)
+#define MSG_XTRACT_PROT_ID(hdr) FIELD_GET(MSG_PROTOCOL_ID_MASK, (hdr))
#define MSG_TOKEN_ID_MASK GENMASK(27, 18)
#define MSG_XTRACT_TOKEN(hdr) FIELD_GET(MSG_TOKEN_ID_MASK, (hdr))
#define MSG_TOKEN_MAX (MSG_XTRACT_TOKEN(MSG_TOKEN_ID_MASK) + 1)
@@ -86,7 +92,7 @@ struct scmi_desc {
};
/**
- * struct scmi_chan_info - Structure representing a SCMI channel informfation
+ * struct scmi_chan_info - Structure representing a SCMI channel information
*
* @cl: Mailbox Client
* @chan: Transmit/Receive mailbox channel
@@ -111,8 +117,9 @@ struct scmi_chan_info {
* @handle: Instance of SCMI handle to send to clients
* @version: SCMI revision information containing protocol version,
* implementation version and (sub-)vendor identification.
- * @minfo: Message info
- * @tx_idr: IDR object to map protocol id to channel info pointer
+ * @tx_minfo: Universal Transmit Message management info
+ * @tx_idr: IDR object to map protocol id to Tx channel info pointer
+ * @rx_idr: IDR object to map protocol id to Rx channel info pointer
* @protocols_imp: List of protocols implemented, currently maximum of
* MAX_PROTOCOLS_IMP elements allocated by the base protocol
* @node: List head
@@ -123,8 +130,9 @@ struct scmi_info {
const struct scmi_desc *desc;
struct scmi_revision_info version;
struct scmi_handle handle;
- struct scmi_xfers_info minfo;
+ struct scmi_xfers_info tx_minfo;
struct idr tx_idr;
+ struct idr rx_idr;
u8 *protocols_imp;
struct list_head node;
int users;
@@ -182,7 +190,7 @@ static inline int scmi_to_linux_errno(int errno)
static inline void scmi_dump_header_dbg(struct device *dev,
struct scmi_msg_hdr *hdr)
{
- dev_dbg(dev, "Command ID: %x Sequence ID: %x Protocol: %x\n",
+ dev_dbg(dev, "Message ID: %x Sequence ID: %x Protocol: %x\n",
hdr->id, hdr->seq, hdr->protocol_id);
}
@@ -190,7 +198,7 @@ static void scmi_fetch_response(struct scmi_xfer *xfer,
struct scmi_shared_mem __iomem *mem)
{
xfer->hdr.status = ioread32(mem->msg_payload);
- /* Skip the length of header and statues in payload area i.e 8 bytes*/
+ /* Skip the length of header and status in payload area i.e 8 bytes */
xfer->rx.len = min_t(size_t, xfer->rx.len, ioread32(&mem->length) - 8);
/* Take a copy to the rx buffer.. */
@@ -198,56 +206,12 @@ static void scmi_fetch_response(struct scmi_xfer *xfer,
}
/**
- * scmi_rx_callback() - mailbox client callback for receive messages
- *
- * @cl: client pointer
- * @m: mailbox message
- *
- * Processes one received message to appropriate transfer information and
- * signals completion of the transfer.
- *
- * NOTE: This function will be invoked in IRQ context, hence should be
- * as optimal as possible.
- */
-static void scmi_rx_callback(struct mbox_client *cl, void *m)
-{
- u16 xfer_id;
- struct scmi_xfer *xfer;
- struct scmi_chan_info *cinfo = client_to_scmi_chan_info(cl);
- struct device *dev = cinfo->dev;
- struct scmi_info *info = handle_to_scmi_info(cinfo->handle);
- struct scmi_xfers_info *minfo = &info->minfo;
- struct scmi_shared_mem __iomem *mem = cinfo->payload;
-
- xfer_id = MSG_XTRACT_TOKEN(ioread32(&mem->msg_header));
-
- /* Are we even expecting this? */
- if (!test_bit(xfer_id, minfo->xfer_alloc_table)) {
- dev_err(dev, "message for %d is not expected!\n", xfer_id);
- return;
- }
-
- xfer = &minfo->xfer_block[xfer_id];
-
- scmi_dump_header_dbg(dev, &xfer->hdr);
- /* Is the message of valid length? */
- if (xfer->rx.len > info->desc->max_msg_size) {
- dev_err(dev, "unable to handle %zu xfer(max %d)\n",
- xfer->rx.len, info->desc->max_msg_size);
- return;
- }
-
- scmi_fetch_response(xfer, mem);
- complete(&xfer->done);
-}
-
-/**
* pack_scmi_header() - packs and returns 32-bit header
*
* @hdr: pointer to header containing all the information on message id,
* protocol id and sequence id.
*
- * Return: 32-bit packed command header to be sent to the platform.
+ * Return: 32-bit packed message header to be sent to the platform.
*/
static inline u32 pack_scmi_header(struct scmi_msg_hdr *hdr)
{
@@ -257,6 +221,18 @@ static inline u32 pack_scmi_header(struct scmi_msg_hdr *hdr)
}
/**
+ * unpack_scmi_header() - unpacks and records message and protocol id
+ *
+ * @msg_hdr: 32-bit packed message header sent from the platform
+ * @hdr: pointer to header to fetch message and protocol id.
+ */
+static inline void unpack_scmi_header(u32 msg_hdr, struct scmi_msg_hdr *hdr)
+{
+ hdr->id = MSG_XTRACT_ID(msg_hdr);
+ hdr->protocol_id = MSG_XTRACT_PROT_ID(msg_hdr);
+}
+
+/**
* scmi_tx_prepare() - mailbox client callback to prepare for the transfer
*
* @cl: client pointer
@@ -271,6 +247,14 @@ static void scmi_tx_prepare(struct mbox_client *cl, void *m)
struct scmi_chan_info *cinfo = client_to_scmi_chan_info(cl);
struct scmi_shared_mem __iomem *mem = cinfo->payload;
+ /*
+ * Ideally channel must be free by now unless OS timeout last
+ * request and platform continued to process the same, wait
+ * until it releases the shared memory, otherwise we may endup
+ * overwriting its response with new message payload or vice-versa
+ */
+ spin_until_cond(ioread32(&mem->channel_status) &
+ SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE);
/* Mark channel busy + clear error */
iowrite32(0x0, &mem->channel_status);
iowrite32(t->hdr.poll_completion ? 0 : SCMI_SHMEM_FLAG_INTR_ENABLED,
@@ -285,8 +269,9 @@ static void scmi_tx_prepare(struct mbox_client *cl, void *m)
* scmi_xfer_get() - Allocate one message
*
* @handle: Pointer to SCMI entity handle
+ * @minfo: Pointer to Tx/Rx Message management info based on channel type
*
- * Helper function which is used by various command functions that are
+ * Helper function which is used by various message functions that are
* exposed to clients of this driver for allocating a message traffic event.
*
* This function can sleep depending on pending requests already in the system
@@ -295,13 +280,13 @@ static void scmi_tx_prepare(struct mbox_client *cl, void *m)
*
* Return: 0 if all went fine, else corresponding error.
*/
-static struct scmi_xfer *scmi_xfer_get(const struct scmi_handle *handle)
+static struct scmi_xfer *scmi_xfer_get(const struct scmi_handle *handle,
+ struct scmi_xfers_info *minfo)
{
u16 xfer_id;
struct scmi_xfer *xfer;
unsigned long flags, bit_pos;
struct scmi_info *info = handle_to_scmi_info(handle);
- struct scmi_xfers_info *minfo = &info->minfo;
/* Keep the locked section as small as possible */
spin_lock_irqsave(&minfo->xfer_lock, flags);
@@ -324,18 +309,17 @@ static struct scmi_xfer *scmi_xfer_get(const struct scmi_handle *handle)
}
/**
- * scmi_xfer_put() - Release a message
+ * __scmi_xfer_put() - Release a message
*
- * @handle: Pointer to SCMI entity handle
+ * @minfo: Pointer to Tx/Rx Message management info based on channel type
* @xfer: message that was reserved by scmi_xfer_get
*
* This holds a spinlock to maintain integrity of internal data structures.
*/
-void scmi_xfer_put(const struct scmi_handle *handle, struct scmi_xfer *xfer)
+static void
+__scmi_xfer_put(struct scmi_xfers_info *minfo, struct scmi_xfer *xfer)
{
unsigned long flags;
- struct scmi_info *info = handle_to_scmi_info(handle);
- struct scmi_xfers_info *minfo = &info->minfo;
/*
* Keep the locked section as small as possible
@@ -347,6 +331,68 @@ void scmi_xfer_put(const struct scmi_handle *handle, struct scmi_xfer *xfer)
spin_unlock_irqrestore(&minfo->xfer_lock, flags);
}
+/**
+ * scmi_rx_callback() - mailbox client callback for receive messages
+ *
+ * @cl: client pointer
+ * @m: mailbox message
+ *
+ * Processes one received message to appropriate transfer information and
+ * signals completion of the transfer.
+ *
+ * NOTE: This function will be invoked in IRQ context, hence should be
+ * as optimal as possible.
+ */
+static void scmi_rx_callback(struct mbox_client *cl, void *m)
+{
+ u8 msg_type;
+ u32 msg_hdr;
+ u16 xfer_id;
+ struct scmi_xfer *xfer;
+ struct scmi_chan_info *cinfo = client_to_scmi_chan_info(cl);
+ struct device *dev = cinfo->dev;
+ struct scmi_info *info = handle_to_scmi_info(cinfo->handle);
+ struct scmi_xfers_info *minfo = &info->tx_minfo;
+ struct scmi_shared_mem __iomem *mem = cinfo->payload;
+
+ msg_hdr = ioread32(&mem->msg_header);
+ msg_type = MSG_XTRACT_TYPE(msg_hdr);
+ xfer_id = MSG_XTRACT_TOKEN(msg_hdr);
+
+ if (msg_type == MSG_TYPE_NOTIFICATION)
+ return; /* Notifications not yet supported */
+
+ /* Are we even expecting this? */
+ if (!test_bit(xfer_id, minfo->xfer_alloc_table)) {
+ dev_err(dev, "message for %d is not expected!\n", xfer_id);
+ return;
+ }
+
+ xfer = &minfo->xfer_block[xfer_id];
+
+ scmi_dump_header_dbg(dev, &xfer->hdr);
+
+ scmi_fetch_response(xfer, mem);
+
+ if (msg_type == MSG_TYPE_DELAYED_RESP)
+ complete(xfer->async_done);
+ else
+ complete(&xfer->done);
+}
+
+/**
+ * scmi_xfer_put() - Release a transmit message
+ *
+ * @handle: Pointer to SCMI entity handle
+ * @xfer: message that was reserved by scmi_xfer_get
+ */
+void scmi_xfer_put(const struct scmi_handle *handle, struct scmi_xfer *xfer)
+{
+ struct scmi_info *info = handle_to_scmi_info(handle);
+
+ __scmi_xfer_put(&info->tx_minfo, xfer);
+}
+
static bool
scmi_xfer_poll_done(const struct scmi_chan_info *cinfo, struct scmi_xfer *xfer)
{
@@ -435,8 +481,36 @@ int scmi_do_xfer(const struct scmi_handle *handle, struct scmi_xfer *xfer)
return ret;
}
+#define SCMI_MAX_RESPONSE_TIMEOUT (2 * MSEC_PER_SEC)
+
+/**
+ * scmi_do_xfer_with_response() - Do one transfer and wait until the delayed
+ * response is received
+ *
+ * @handle: Pointer to SCMI entity handle
+ * @xfer: Transfer to initiate and wait for response
+ *
+ * Return: -ETIMEDOUT in case of no delayed response, if transmit error,
+ * return corresponding error, else if all goes well, return 0.
+ */
+int scmi_do_xfer_with_response(const struct scmi_handle *handle,
+ struct scmi_xfer *xfer)
+{
+ int ret, timeout = msecs_to_jiffies(SCMI_MAX_RESPONSE_TIMEOUT);
+ DECLARE_COMPLETION_ONSTACK(async_response);
+
+ xfer->async_done = &async_response;
+
+ ret = scmi_do_xfer(handle, xfer);
+ if (!ret && !wait_for_completion_timeout(xfer->async_done, timeout))
+ ret = -ETIMEDOUT;
+
+ xfer->async_done = NULL;
+ return ret;
+}
+
/**
- * scmi_xfer_get_init() - Allocate and initialise one message
+ * scmi_xfer_get_init() - Allocate and initialise one message for transmit
*
* @handle: Pointer to SCMI entity handle
* @msg_id: Message identifier
@@ -457,6 +531,7 @@ int scmi_xfer_get_init(const struct scmi_handle *handle, u8 msg_id, u8 prot_id,
int ret;
struct scmi_xfer *xfer;
struct scmi_info *info = handle_to_scmi_info(handle);
+ struct scmi_xfers_info *minfo = &info->tx_minfo;
struct device *dev = info->dev;
/* Ensure we have sane transfer sizes */
@@ -464,7 +539,7 @@ int scmi_xfer_get_init(const struct scmi_handle *handle, u8 msg_id, u8 prot_id,
tx_size > info->desc->max_msg_size)
return -ERANGE;
- xfer = scmi_xfer_get(handle);
+ xfer = scmi_xfer_get(handle, minfo);
if (IS_ERR(xfer)) {
ret = PTR_ERR(xfer);
dev_err(dev, "failed to get free message slot(%d)\n", ret);
@@ -597,27 +672,13 @@ int scmi_handle_put(const struct scmi_handle *handle)
return 0;
}
-static const struct scmi_desc scmi_generic_desc = {
- .max_rx_timeout_ms = 30, /* We may increase this if required */
- .max_msg = 20, /* Limited by MBOX_TX_QUEUE_LEN */
- .max_msg_size = 128,
-};
-
-/* Each compatible listed below must have descriptor associated with it */
-static const struct of_device_id scmi_of_match[] = {
- { .compatible = "arm,scmi", .data = &scmi_generic_desc },
- { /* Sentinel */ },
-};
-
-MODULE_DEVICE_TABLE(of, scmi_of_match);
-
static int scmi_xfer_info_init(struct scmi_info *sinfo)
{
int i;
struct scmi_xfer *xfer;
struct device *dev = sinfo->dev;
const struct scmi_desc *desc = sinfo->desc;
- struct scmi_xfers_info *info = &sinfo->minfo;
+ struct scmi_xfers_info *info = &sinfo->tx_minfo;
/* Pre-allocated messages, no more than what hdr.seq can support */
if (WARN_ON(desc->max_msg >= MSG_TOKEN_MAX)) {
@@ -652,61 +713,32 @@ static int scmi_xfer_info_init(struct scmi_info *sinfo)
return 0;
}
-static int scmi_mailbox_check(struct device_node *np)
+static int scmi_mailbox_check(struct device_node *np, int idx)
{
- return of_parse_phandle_with_args(np, "mboxes", "#mbox-cells", 0, NULL);
+ return of_parse_phandle_with_args(np, "mboxes", "#mbox-cells",
+ idx, NULL);
}
-static int scmi_mbox_free_channel(int id, void *p, void *data)
+static int scmi_mbox_chan_setup(struct scmi_info *info, struct device *dev,
+ int prot_id, bool tx)
{
- struct scmi_chan_info *cinfo = p;
- struct idr *idr = data;
-
- if (!IS_ERR_OR_NULL(cinfo->chan)) {
- mbox_free_channel(cinfo->chan);
- cinfo->chan = NULL;
- }
-
- idr_remove(idr, id);
-
- return 0;
-}
-
-static int scmi_remove(struct platform_device *pdev)
-{
- int ret = 0;
- struct scmi_info *info = platform_get_drvdata(pdev);
- struct idr *idr = &info->tx_idr;
-
- mutex_lock(&scmi_list_mutex);
- if (info->users)
- ret = -EBUSY;
- else
- list_del(&info->node);
- mutex_unlock(&scmi_list_mutex);
-
- if (ret)
- return ret;
-
- /* Safe to free channels since no more users */
- ret = idr_for_each(idr, scmi_mbox_free_channel, idr);
- idr_destroy(&info->tx_idr);
-
- return ret;
-}
-
-static inline int
-scmi_mbox_chan_setup(struct scmi_info *info, struct device *dev, int prot_id)
-{
- int ret;
+ int ret, idx;
struct resource res;
resource_size_t size;
struct device_node *shmem, *np = dev->of_node;
struct scmi_chan_info *cinfo;
struct mbox_client *cl;
+ struct idr *idr;
+ const char *desc = tx ? "Tx" : "Rx";
- if (scmi_mailbox_check(np)) {
- cinfo = idr_find(&info->tx_idr, SCMI_PROTOCOL_BASE);
+ /* Transmit channel is first entry i.e. index 0 */
+ idx = tx ? 0 : 1;
+ idr = tx ? &info->tx_idr : &info->rx_idr;
+
+ if (scmi_mailbox_check(np, idx)) {
+ cinfo = idr_find(idr, SCMI_PROTOCOL_BASE);
+ if (unlikely(!cinfo)) /* Possible only if platform has no Rx */
+ return -EINVAL;
goto idr_alloc;
}
@@ -719,36 +751,36 @@ scmi_mbox_chan_setup(struct scmi_info *info, struct device *dev, int prot_id)
cl = &cinfo->cl;
cl->dev = dev;
cl->rx_callback = scmi_rx_callback;
- cl->tx_prepare = scmi_tx_prepare;
+ cl->tx_prepare = tx ? scmi_tx_prepare : NULL;
cl->tx_block = false;
- cl->knows_txdone = true;
+ cl->knows_txdone = tx;
- shmem = of_parse_phandle(np, "shmem", 0);
+ shmem = of_parse_phandle(np, "shmem", idx);
ret = of_address_to_resource(shmem, 0, &res);
of_node_put(shmem);
if (ret) {
- dev_err(dev, "failed to get SCMI Tx payload mem resource\n");
+ dev_err(dev, "failed to get SCMI %s payload memory\n", desc);
return ret;
}
size = resource_size(&res);
cinfo->payload = devm_ioremap(info->dev, res.start, size);
if (!cinfo->payload) {
- dev_err(dev, "failed to ioremap SCMI Tx payload\n");
+ dev_err(dev, "failed to ioremap SCMI %s payload\n", desc);
return -EADDRNOTAVAIL;
}
- /* Transmit channel is first entry i.e. index 0 */
- cinfo->chan = mbox_request_channel(cl, 0);
+ cinfo->chan = mbox_request_channel(cl, idx);
if (IS_ERR(cinfo->chan)) {
ret = PTR_ERR(cinfo->chan);
if (ret != -EPROBE_DEFER)
- dev_err(dev, "failed to request SCMI Tx mailbox\n");
+ dev_err(dev, "failed to request SCMI %s mailbox\n",
+ desc);
return ret;
}
idr_alloc:
- ret = idr_alloc(&info->tx_idr, cinfo, prot_id, prot_id + 1, GFP_KERNEL);
+ ret = idr_alloc(idr, cinfo, prot_id, prot_id + 1, GFP_KERNEL);
if (ret != prot_id) {
dev_err(dev, "unable to allocate SCMI idr slot err %d\n", ret);
return ret;
@@ -758,6 +790,17 @@ idr_alloc:
return 0;
}
+static inline int
+scmi_mbox_txrx_setup(struct scmi_info *info, struct device *dev, int prot_id)
+{
+ int ret = scmi_mbox_chan_setup(info, dev, prot_id, true);
+
+ if (!ret) /* Rx is optional, hence no error check */
+ scmi_mbox_chan_setup(info, dev, prot_id, false);
+
+ return ret;
+}
+
static inline void
scmi_create_protocol_device(struct device_node *np, struct scmi_info *info,
int prot_id)
@@ -771,7 +814,7 @@ scmi_create_protocol_device(struct device_node *np, struct scmi_info *info,
return;
}
- if (scmi_mbox_chan_setup(info, &sdev->dev, prot_id)) {
+ if (scmi_mbox_txrx_setup(info, &sdev->dev, prot_id)) {
dev_err(&sdev->dev, "failed to setup transport\n");
scmi_device_destroy(sdev);
return;
@@ -791,7 +834,7 @@ static int scmi_probe(struct platform_device *pdev)
struct device_node *child, *np = dev->of_node;
/* Only mailbox method supported, check for the presence of one */
- if (scmi_mailbox_check(np)) {
+ if (scmi_mailbox_check(np, 0)) {
dev_err(dev, "no mailbox found in %pOF\n", np);
return -EINVAL;
}
@@ -814,12 +857,13 @@ static int scmi_probe(struct platform_device *pdev)
platform_set_drvdata(pdev, info);
idr_init(&info->tx_idr);
+ idr_init(&info->rx_idr);
handle = &info->handle;
handle->dev = info->dev;
handle->version = &info->version;
- ret = scmi_mbox_chan_setup(info, dev, SCMI_PROTOCOL_BASE);
+ ret = scmi_mbox_txrx_setup(info, dev, SCMI_PROTOCOL_BASE);
if (ret)
return ret;
@@ -854,6 +898,62 @@ static int scmi_probe(struct platform_device *pdev)
return 0;
}
+static int scmi_mbox_free_channel(int id, void *p, void *data)
+{
+ struct scmi_chan_info *cinfo = p;
+ struct idr *idr = data;
+
+ if (!IS_ERR_OR_NULL(cinfo->chan)) {
+ mbox_free_channel(cinfo->chan);
+ cinfo->chan = NULL;
+ }
+
+ idr_remove(idr, id);
+
+ return 0;
+}
+
+static int scmi_remove(struct platform_device *pdev)
+{
+ int ret = 0;
+ struct scmi_info *info = platform_get_drvdata(pdev);
+ struct idr *idr = &info->tx_idr;
+
+ mutex_lock(&scmi_list_mutex);
+ if (info->users)
+ ret = -EBUSY;
+ else
+ list_del(&info->node);
+ mutex_unlock(&scmi_list_mutex);
+
+ if (ret)
+ return ret;
+
+ /* Safe to free channels since no more users */
+ ret = idr_for_each(idr, scmi_mbox_free_channel, idr);
+ idr_destroy(&info->tx_idr);
+
+ idr = &info->rx_idr;
+ ret = idr_for_each(idr, scmi_mbox_free_channel, idr);
+ idr_destroy(&info->rx_idr);
+
+ return ret;
+}
+
+static const struct scmi_desc scmi_generic_desc = {
+ .max_rx_timeout_ms = 30, /* We may increase this if required */
+ .max_msg = 20, /* Limited by MBOX_TX_QUEUE_LEN */
+ .max_msg_size = 128,
+};
+
+/* Each compatible listed below must have descriptor associated with it */
+static const struct of_device_id scmi_of_match[] = {
+ { .compatible = "arm,scmi", .data = &scmi_generic_desc },
+ { /* Sentinel */ },
+};
+
+MODULE_DEVICE_TABLE(of, scmi_of_match);
+
static struct platform_driver scmi_driver = {
.driver = {
.name = "arm-scmi",
diff --git a/drivers/firmware/arm_scmi/perf.c b/drivers/firmware/arm_scmi/perf.c
index 3c8ae7cc35de..4a8012e3cb8c 100644
--- a/drivers/firmware/arm_scmi/perf.c
+++ b/drivers/firmware/arm_scmi/perf.c
@@ -5,7 +5,10 @@
* Copyright (C) 2018 ARM Ltd.
*/
+#include <linux/bits.h>
#include <linux/of.h>
+#include <linux/io.h>
+#include <linux/io-64-nonatomic-hi-lo.h>
#include <linux/platform_device.h>
#include <linux/pm_opp.h>
#include <linux/sort.h>
@@ -21,6 +24,7 @@ enum scmi_performance_protocol_cmd {
PERF_LEVEL_GET = 0x8,
PERF_NOTIFY_LIMITS = 0x9,
PERF_NOTIFY_LEVEL = 0xa,
+ PERF_DESCRIBE_FASTCHANNEL = 0xb,
};
struct scmi_opp {
@@ -44,6 +48,7 @@ struct scmi_msg_resp_perf_domain_attributes {
#define SUPPORTS_SET_PERF_LVL(x) ((x) & BIT(30))
#define SUPPORTS_PERF_LIMIT_NOTIFY(x) ((x) & BIT(29))
#define SUPPORTS_PERF_LEVEL_NOTIFY(x) ((x) & BIT(28))
+#define SUPPORTS_PERF_FASTCHANNELS(x) ((x) & BIT(27))
__le32 rate_limit_us;
__le32 sustained_freq_khz;
__le32 sustained_perf_level;
@@ -87,17 +92,56 @@ struct scmi_msg_resp_perf_describe_levels {
} opp[0];
};
+struct scmi_perf_get_fc_info {
+ __le32 domain;
+ __le32 message_id;
+};
+
+struct scmi_msg_resp_perf_desc_fc {
+ __le32 attr;
+#define SUPPORTS_DOORBELL(x) ((x) & BIT(0))
+#define DOORBELL_REG_WIDTH(x) FIELD_GET(GENMASK(2, 1), (x))
+ __le32 rate_limit;
+ __le32 chan_addr_low;
+ __le32 chan_addr_high;
+ __le32 chan_size;
+ __le32 db_addr_low;
+ __le32 db_addr_high;
+ __le32 db_set_lmask;
+ __le32 db_set_hmask;
+ __le32 db_preserve_lmask;
+ __le32 db_preserve_hmask;
+};
+
+struct scmi_fc_db_info {
+ int width;
+ u64 set;
+ u64 mask;
+ void __iomem *addr;
+};
+
+struct scmi_fc_info {
+ void __iomem *level_set_addr;
+ void __iomem *limit_set_addr;
+ void __iomem *level_get_addr;
+ void __iomem *limit_get_addr;
+ struct scmi_fc_db_info *level_set_db;
+ struct scmi_fc_db_info *limit_set_db;
+};
+
struct perf_dom_info {
bool set_limits;
bool set_perf;
bool perf_limit_notify;
bool perf_level_notify;
+ bool perf_fastchannels;
u32 opp_count;
u32 sustained_freq_khz;
u32 sustained_perf_level;
u32 mult_factor;
char name[SCMI_MAX_STR_SIZE];
struct scmi_opp opp[MAX_OPPS];
+ struct scmi_fc_info *fc_info;
};
struct scmi_perf_info {
@@ -151,7 +195,7 @@ scmi_perf_domain_attributes_get(const struct scmi_handle *handle, u32 domain,
if (ret)
return ret;
- *(__le32 *)t->tx.buf = cpu_to_le32(domain);
+ put_unaligned_le32(domain, t->tx.buf);
attr = t->rx.buf;
ret = scmi_do_xfer(handle, t);
@@ -162,6 +206,7 @@ scmi_perf_domain_attributes_get(const struct scmi_handle *handle, u32 domain,
dom_info->set_perf = SUPPORTS_SET_PERF_LVL(flags);
dom_info->perf_limit_notify = SUPPORTS_PERF_LIMIT_NOTIFY(flags);
dom_info->perf_level_notify = SUPPORTS_PERF_LEVEL_NOTIFY(flags);
+ dom_info->perf_fastchannels = SUPPORTS_PERF_FASTCHANNELS(flags);
dom_info->sustained_freq_khz =
le32_to_cpu(attr->sustained_freq_khz);
dom_info->sustained_perf_level =
@@ -249,8 +294,42 @@ scmi_perf_describe_levels_get(const struct scmi_handle *handle, u32 domain,
return ret;
}
-static int scmi_perf_limits_set(const struct scmi_handle *handle, u32 domain,
- u32 max_perf, u32 min_perf)
+#define SCMI_PERF_FC_RING_DB(w) \
+do { \
+ u##w val = 0; \
+ \
+ if (db->mask) \
+ val = ioread##w(db->addr) & db->mask; \
+ iowrite##w((u##w)db->set | val, db->addr); \
+} while (0)
+
+static void scmi_perf_fc_ring_db(struct scmi_fc_db_info *db)
+{
+ if (!db || !db->addr)
+ return;
+
+ if (db->width == 1)
+ SCMI_PERF_FC_RING_DB(8);
+ else if (db->width == 2)
+ SCMI_PERF_FC_RING_DB(16);
+ else if (db->width == 4)
+ SCMI_PERF_FC_RING_DB(32);
+ else /* db->width == 8 */
+#ifdef CONFIG_64BIT
+ SCMI_PERF_FC_RING_DB(64);
+#else
+ {
+ u64 val = 0;
+
+ if (db->mask)
+ val = ioread64_hi_lo(db->addr) & db->mask;
+ iowrite64_hi_lo(db->set, db->addr);
+ }
+#endif
+}
+
+static int scmi_perf_mb_limits_set(const struct scmi_handle *handle, u32 domain,
+ u32 max_perf, u32 min_perf)
{
int ret;
struct scmi_xfer *t;
@@ -272,8 +351,24 @@ static int scmi_perf_limits_set(const struct scmi_handle *handle, u32 domain,
return ret;
}
-static int scmi_perf_limits_get(const struct scmi_handle *handle, u32 domain,
- u32 *max_perf, u32 *min_perf)
+static int scmi_perf_limits_set(const struct scmi_handle *handle, u32 domain,
+ u32 max_perf, u32 min_perf)
+{
+ struct scmi_perf_info *pi = handle->perf_priv;
+ struct perf_dom_info *dom = pi->dom_info + domain;
+
+ if (dom->fc_info && dom->fc_info->limit_set_addr) {
+ iowrite32(max_perf, dom->fc_info->limit_set_addr);
+ iowrite32(min_perf, dom->fc_info->limit_set_addr + 4);
+ scmi_perf_fc_ring_db(dom->fc_info->limit_set_db);
+ return 0;
+ }
+
+ return scmi_perf_mb_limits_set(handle, domain, max_perf, min_perf);
+}
+
+static int scmi_perf_mb_limits_get(const struct scmi_handle *handle, u32 domain,
+ u32 *max_perf, u32 *min_perf)
{
int ret;
struct scmi_xfer *t;
@@ -284,7 +379,7 @@ static int scmi_perf_limits_get(const struct scmi_handle *handle, u32 domain,
if (ret)
return ret;
- *(__le32 *)t->tx.buf = cpu_to_le32(domain);
+ put_unaligned_le32(domain, t->tx.buf);
ret = scmi_do_xfer(handle, t);
if (!ret) {
@@ -298,8 +393,23 @@ static int scmi_perf_limits_get(const struct scmi_handle *handle, u32 domain,
return ret;
}
-static int scmi_perf_level_set(const struct scmi_handle *handle, u32 domain,
- u32 level, bool poll)
+static int scmi_perf_limits_get(const struct scmi_handle *handle, u32 domain,
+ u32 *max_perf, u32 *min_perf)
+{
+ struct scmi_perf_info *pi = handle->perf_priv;
+ struct perf_dom_info *dom = pi->dom_info + domain;
+
+ if (dom->fc_info && dom->fc_info->limit_get_addr) {
+ *max_perf = ioread32(dom->fc_info->limit_get_addr);
+ *min_perf = ioread32(dom->fc_info->limit_get_addr + 4);
+ return 0;
+ }
+
+ return scmi_perf_mb_limits_get(handle, domain, max_perf, min_perf);
+}
+
+static int scmi_perf_mb_level_set(const struct scmi_handle *handle, u32 domain,
+ u32 level, bool poll)
{
int ret;
struct scmi_xfer *t;
@@ -321,8 +431,23 @@ static int scmi_perf_level_set(const struct scmi_handle *handle, u32 domain,
return ret;
}
-static int scmi_perf_level_get(const struct scmi_handle *handle, u32 domain,
- u32 *level, bool poll)
+static int scmi_perf_level_set(const struct scmi_handle *handle, u32 domain,
+ u32 level, bool poll)
+{
+ struct scmi_perf_info *pi = handle->perf_priv;
+ struct perf_dom_info *dom = pi->dom_info + domain;
+
+ if (dom->fc_info && dom->fc_info->level_set_addr) {
+ iowrite32(level, dom->fc_info->level_set_addr);
+ scmi_perf_fc_ring_db(dom->fc_info->level_set_db);
+ return 0;
+ }
+
+ return scmi_perf_mb_level_set(handle, domain, level, poll);
+}
+
+static int scmi_perf_mb_level_get(const struct scmi_handle *handle, u32 domain,
+ u32 *level, bool poll)
{
int ret;
struct scmi_xfer *t;
@@ -333,16 +458,128 @@ static int scmi_perf_level_get(const struct scmi_handle *handle, u32 domain,
return ret;
t->hdr.poll_completion = poll;
- *(__le32 *)t->tx.buf = cpu_to_le32(domain);
+ put_unaligned_le32(domain, t->tx.buf);
ret = scmi_do_xfer(handle, t);
if (!ret)
- *level = le32_to_cpu(*(__le32 *)t->rx.buf);
+ *level = get_unaligned_le32(t->rx.buf);
scmi_xfer_put(handle, t);
return ret;
}
+static int scmi_perf_level_get(const struct scmi_handle *handle, u32 domain,
+ u32 *level, bool poll)
+{
+ struct scmi_perf_info *pi = handle->perf_priv;
+ struct perf_dom_info *dom = pi->dom_info + domain;
+
+ if (dom->fc_info && dom->fc_info->level_get_addr) {
+ *level = ioread32(dom->fc_info->level_get_addr);
+ return 0;
+ }
+
+ return scmi_perf_mb_level_get(handle, domain, level, poll);
+}
+
+static bool scmi_perf_fc_size_is_valid(u32 msg, u32 size)
+{
+ if ((msg == PERF_LEVEL_GET || msg == PERF_LEVEL_SET) && size == 4)
+ return true;
+ if ((msg == PERF_LIMITS_GET || msg == PERF_LIMITS_SET) && size == 8)
+ return true;
+ return false;
+}
+
+static void
+scmi_perf_domain_desc_fc(const struct scmi_handle *handle, u32 domain,
+ u32 message_id, void __iomem **p_addr,
+ struct scmi_fc_db_info **p_db)
+{
+ int ret;
+ u32 flags;
+ u64 phys_addr;
+ u8 size;
+ void __iomem *addr;
+ struct scmi_xfer *t;
+ struct scmi_fc_db_info *db;
+ struct scmi_perf_get_fc_info *info;
+ struct scmi_msg_resp_perf_desc_fc *resp;
+
+ if (!p_addr)
+ return;
+
+ ret = scmi_xfer_get_init(handle, PERF_DESCRIBE_FASTCHANNEL,
+ SCMI_PROTOCOL_PERF,
+ sizeof(*info), sizeof(*resp), &t);
+ if (ret)
+ return;
+
+ info = t->tx.buf;
+ info->domain = cpu_to_le32(domain);
+ info->message_id = cpu_to_le32(message_id);
+
+ ret = scmi_do_xfer(handle, t);
+ if (ret)
+ goto err_xfer;
+
+ resp = t->rx.buf;
+ flags = le32_to_cpu(resp->attr);
+ size = le32_to_cpu(resp->chan_size);
+ if (!scmi_perf_fc_size_is_valid(message_id, size))
+ goto err_xfer;
+
+ phys_addr = le32_to_cpu(resp->chan_addr_low);
+ phys_addr |= (u64)le32_to_cpu(resp->chan_addr_high) << 32;
+ addr = devm_ioremap(handle->dev, phys_addr, size);
+ if (!addr)
+ goto err_xfer;
+ *p_addr = addr;
+
+ if (p_db && SUPPORTS_DOORBELL(flags)) {
+ db = devm_kzalloc(handle->dev, sizeof(*db), GFP_KERNEL);
+ if (!db)
+ goto err_xfer;
+
+ size = 1 << DOORBELL_REG_WIDTH(flags);
+ phys_addr = le32_to_cpu(resp->db_addr_low);
+ phys_addr |= (u64)le32_to_cpu(resp->db_addr_high) << 32;
+ addr = devm_ioremap(handle->dev, phys_addr, size);
+ if (!addr)
+ goto err_xfer;
+
+ db->addr = addr;
+ db->width = size;
+ db->set = le32_to_cpu(resp->db_set_lmask);
+ db->set |= (u64)le32_to_cpu(resp->db_set_hmask) << 32;
+ db->mask = le32_to_cpu(resp->db_preserve_lmask);
+ db->mask |= (u64)le32_to_cpu(resp->db_preserve_hmask) << 32;
+ *p_db = db;
+ }
+err_xfer:
+ scmi_xfer_put(handle, t);
+}
+
+static void scmi_perf_domain_init_fc(const struct scmi_handle *handle,
+ u32 domain, struct scmi_fc_info **p_fc)
+{
+ struct scmi_fc_info *fc;
+
+ fc = devm_kzalloc(handle->dev, sizeof(*fc), GFP_KERNEL);
+ if (!fc)
+ return;
+
+ scmi_perf_domain_desc_fc(handle, domain, PERF_LEVEL_SET,
+ &fc->level_set_addr, &fc->level_set_db);
+ scmi_perf_domain_desc_fc(handle, domain, PERF_LEVEL_GET,
+ &fc->level_get_addr, NULL);
+ scmi_perf_domain_desc_fc(handle, domain, PERF_LIMITS_SET,
+ &fc->limit_set_addr, &fc->limit_set_db);
+ scmi_perf_domain_desc_fc(handle, domain, PERF_LIMITS_GET,
+ &fc->limit_get_addr, NULL);
+ *p_fc = fc;
+}
+
/* Device specific ops */
static int scmi_dev_domain_id(struct device *dev)
{
@@ -494,6 +731,9 @@ static int scmi_perf_protocol_init(struct scmi_handle *handle)
scmi_perf_domain_attributes_get(handle, domain, dom);
scmi_perf_describe_levels_get(handle, domain, dom);
+
+ if (dom->perf_fastchannels)
+ scmi_perf_domain_init_fc(handle, domain, &dom->fc_info);
}
handle->perf_ops = &perf_ops;
diff --git a/drivers/firmware/arm_scmi/power.c b/drivers/firmware/arm_scmi/power.c
index 62f3401a1f01..5abef7079c0a 100644
--- a/drivers/firmware/arm_scmi/power.c
+++ b/drivers/firmware/arm_scmi/power.c
@@ -96,7 +96,7 @@ scmi_power_domain_attributes_get(const struct scmi_handle *handle, u32 domain,
if (ret)
return ret;
- *(__le32 *)t->tx.buf = cpu_to_le32(domain);
+ put_unaligned_le32(domain, t->tx.buf);
attr = t->rx.buf;
ret = scmi_do_xfer(handle, t);
@@ -147,11 +147,11 @@ scmi_power_state_get(const struct scmi_handle *handle, u32 domain, u32 *state)
if (ret)
return ret;
- *(__le32 *)t->tx.buf = cpu_to_le32(domain);
+ put_unaligned_le32(domain, t->tx.buf);
ret = scmi_do_xfer(handle, t);
if (!ret)
- *state = le32_to_cpu(*(__le32 *)t->rx.buf);
+ *state = get_unaligned_le32(t->rx.buf);
scmi_xfer_put(handle, t);
return ret;
diff --git a/drivers/firmware/arm_scmi/reset.c b/drivers/firmware/arm_scmi/reset.c
new file mode 100644
index 000000000000..64cc81915581
--- /dev/null
+++ b/drivers/firmware/arm_scmi/reset.c
@@ -0,0 +1,231 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * System Control and Management Interface (SCMI) Reset Protocol
+ *
+ * Copyright (C) 2019 ARM Ltd.
+ */
+
+#include "common.h"
+
+enum scmi_reset_protocol_cmd {
+ RESET_DOMAIN_ATTRIBUTES = 0x3,
+ RESET = 0x4,
+ RESET_NOTIFY = 0x5,
+};
+
+enum scmi_reset_protocol_notify {
+ RESET_ISSUED = 0x0,
+};
+
+#define NUM_RESET_DOMAIN_MASK 0xffff
+#define RESET_NOTIFY_ENABLE BIT(0)
+
+struct scmi_msg_resp_reset_domain_attributes {
+ __le32 attributes;
+#define SUPPORTS_ASYNC_RESET(x) ((x) & BIT(31))
+#define SUPPORTS_NOTIFY_RESET(x) ((x) & BIT(30))
+ __le32 latency;
+ u8 name[SCMI_MAX_STR_SIZE];
+};
+
+struct scmi_msg_reset_domain_reset {
+ __le32 domain_id;
+ __le32 flags;
+#define AUTONOMOUS_RESET BIT(0)
+#define EXPLICIT_RESET_ASSERT BIT(1)
+#define ASYNCHRONOUS_RESET BIT(2)
+ __le32 reset_state;
+#define ARCH_RESET_TYPE BIT(31)
+#define COLD_RESET_STATE BIT(0)
+#define ARCH_COLD_RESET (ARCH_RESET_TYPE | COLD_RESET_STATE)
+};
+
+struct reset_dom_info {
+ bool async_reset;
+ bool reset_notify;
+ u32 latency_us;
+ char name[SCMI_MAX_STR_SIZE];
+};
+
+struct scmi_reset_info {
+ int num_domains;
+ struct reset_dom_info *dom_info;
+};
+
+static int scmi_reset_attributes_get(const struct scmi_handle *handle,
+ struct scmi_reset_info *pi)
+{
+ int ret;
+ struct scmi_xfer *t;
+ u32 attr;
+
+ ret = scmi_xfer_get_init(handle, PROTOCOL_ATTRIBUTES,
+ SCMI_PROTOCOL_RESET, 0, sizeof(attr), &t);
+ if (ret)
+ return ret;
+
+ ret = scmi_do_xfer(handle, t);
+ if (!ret) {
+ attr = get_unaligned_le32(t->rx.buf);
+ pi->num_domains = attr & NUM_RESET_DOMAIN_MASK;
+ }
+
+ scmi_xfer_put(handle, t);
+ return ret;
+}
+
+static int
+scmi_reset_domain_attributes_get(const struct scmi_handle *handle, u32 domain,
+ struct reset_dom_info *dom_info)
+{
+ int ret;
+ struct scmi_xfer *t;
+ struct scmi_msg_resp_reset_domain_attributes *attr;
+
+ ret = scmi_xfer_get_init(handle, RESET_DOMAIN_ATTRIBUTES,
+ SCMI_PROTOCOL_RESET, sizeof(domain),
+ sizeof(*attr), &t);
+ if (ret)
+ return ret;
+
+ put_unaligned_le32(domain, t->tx.buf);
+ attr = t->rx.buf;
+
+ ret = scmi_do_xfer(handle, t);
+ if (!ret) {
+ u32 attributes = le32_to_cpu(attr->attributes);
+
+ dom_info->async_reset = SUPPORTS_ASYNC_RESET(attributes);
+ dom_info->reset_notify = SUPPORTS_NOTIFY_RESET(attributes);
+ dom_info->latency_us = le32_to_cpu(attr->latency);
+ if (dom_info->latency_us == U32_MAX)
+ dom_info->latency_us = 0;
+ strlcpy(dom_info->name, attr->name, SCMI_MAX_STR_SIZE);
+ }
+
+ scmi_xfer_put(handle, t);
+ return ret;
+}
+
+static int scmi_reset_num_domains_get(const struct scmi_handle *handle)
+{
+ struct scmi_reset_info *pi = handle->reset_priv;
+
+ return pi->num_domains;
+}
+
+static char *scmi_reset_name_get(const struct scmi_handle *handle, u32 domain)
+{
+ struct scmi_reset_info *pi = handle->reset_priv;
+ struct reset_dom_info *dom = pi->dom_info + domain;
+
+ return dom->name;
+}
+
+static int scmi_reset_latency_get(const struct scmi_handle *handle, u32 domain)
+{
+ struct scmi_reset_info *pi = handle->reset_priv;
+ struct reset_dom_info *dom = pi->dom_info + domain;
+
+ return dom->latency_us;
+}
+
+static int scmi_domain_reset(const struct scmi_handle *handle, u32 domain,
+ u32 flags, u32 state)
+{
+ int ret;
+ struct scmi_xfer *t;
+ struct scmi_msg_reset_domain_reset *dom;
+ struct scmi_reset_info *pi = handle->reset_priv;
+ struct reset_dom_info *rdom = pi->dom_info + domain;
+
+ if (rdom->async_reset)
+ flags |= ASYNCHRONOUS_RESET;
+
+ ret = scmi_xfer_get_init(handle, RESET, SCMI_PROTOCOL_RESET,
+ sizeof(*dom), 0, &t);
+ if (ret)
+ return ret;
+
+ dom = t->tx.buf;
+ dom->domain_id = cpu_to_le32(domain);
+ dom->flags = cpu_to_le32(flags);
+ dom->domain_id = cpu_to_le32(state);
+
+ if (rdom->async_reset)
+ ret = scmi_do_xfer_with_response(handle, t);
+ else
+ ret = scmi_do_xfer(handle, t);
+
+ scmi_xfer_put(handle, t);
+ return ret;
+}
+
+static int scmi_reset_domain_reset(const struct scmi_handle *handle, u32 domain)
+{
+ return scmi_domain_reset(handle, domain, AUTONOMOUS_RESET,
+ ARCH_COLD_RESET);
+}
+
+static int
+scmi_reset_domain_assert(const struct scmi_handle *handle, u32 domain)
+{
+ return scmi_domain_reset(handle, domain, EXPLICIT_RESET_ASSERT,
+ ARCH_COLD_RESET);
+}
+
+static int
+scmi_reset_domain_deassert(const struct scmi_handle *handle, u32 domain)
+{
+ return scmi_domain_reset(handle, domain, 0, ARCH_COLD_RESET);
+}
+
+static struct scmi_reset_ops reset_ops = {
+ .num_domains_get = scmi_reset_num_domains_get,
+ .name_get = scmi_reset_name_get,
+ .latency_get = scmi_reset_latency_get,
+ .reset = scmi_reset_domain_reset,
+ .assert = scmi_reset_domain_assert,
+ .deassert = scmi_reset_domain_deassert,
+};
+
+static int scmi_reset_protocol_init(struct scmi_handle *handle)
+{
+ int domain;
+ u32 version;
+ struct scmi_reset_info *pinfo;
+
+ scmi_version_get(handle, SCMI_PROTOCOL_RESET, &version);
+
+ dev_dbg(handle->dev, "Reset Version %d.%d\n",
+ PROTOCOL_REV_MAJOR(version), PROTOCOL_REV_MINOR(version));
+
+ pinfo = devm_kzalloc(handle->dev, sizeof(*pinfo), GFP_KERNEL);
+ if (!pinfo)
+ return -ENOMEM;
+
+ scmi_reset_attributes_get(handle, pinfo);
+
+ pinfo->dom_info = devm_kcalloc(handle->dev, pinfo->num_domains,
+ sizeof(*pinfo->dom_info), GFP_KERNEL);
+ if (!pinfo->dom_info)
+ return -ENOMEM;
+
+ for (domain = 0; domain < pinfo->num_domains; domain++) {
+ struct reset_dom_info *dom = pinfo->dom_info + domain;
+
+ scmi_reset_domain_attributes_get(handle, domain, dom);
+ }
+
+ handle->reset_ops = &reset_ops;
+ handle->reset_priv = pinfo;
+
+ return 0;
+}
+
+static int __init scmi_reset_init(void)
+{
+ return scmi_protocol_register(SCMI_PROTOCOL_RESET,
+ &scmi_reset_protocol_init);
+}
+subsys_initcall(scmi_reset_init);
diff --git a/drivers/firmware/arm_scmi/sensors.c b/drivers/firmware/arm_scmi/sensors.c
index b53d5cc9c9f6..a400ea805fc2 100644
--- a/drivers/firmware/arm_scmi/sensors.c
+++ b/drivers/firmware/arm_scmi/sensors.c
@@ -9,8 +9,8 @@
enum scmi_sensor_protocol_cmd {
SENSOR_DESCRIPTION_GET = 0x3,
- SENSOR_CONFIG_SET = 0x4,
- SENSOR_TRIP_POINT_SET = 0x5,
+ SENSOR_TRIP_POINT_NOTIFY = 0x4,
+ SENSOR_TRIP_POINT_CONFIG = 0x5,
SENSOR_READING_GET = 0x6,
};
@@ -30,19 +30,22 @@ struct scmi_msg_resp_sensor_description {
__le32 id;
__le32 attributes_low;
#define SUPPORTS_ASYNC_READ(x) ((x) & BIT(31))
-#define NUM_TRIP_POINTS(x) (((x) >> 4) & 0xff)
+#define NUM_TRIP_POINTS(x) ((x) & 0xff)
__le32 attributes_high;
#define SENSOR_TYPE(x) ((x) & 0xff)
-#define SENSOR_SCALE(x) (((x) >> 11) & 0x3f)
+#define SENSOR_SCALE(x) (((x) >> 11) & 0x1f)
+#define SENSOR_SCALE_SIGN BIT(4)
+#define SENSOR_SCALE_EXTEND GENMASK(7, 5)
#define SENSOR_UPDATE_SCALE(x) (((x) >> 22) & 0x1f)
#define SENSOR_UPDATE_BASE(x) (((x) >> 27) & 0x1f)
u8 name[SCMI_MAX_STR_SIZE];
} desc[0];
};
-struct scmi_msg_set_sensor_config {
+struct scmi_msg_sensor_trip_point_notify {
__le32 id;
__le32 event_control;
+#define SENSOR_TP_NOTIFY_ALL BIT(0)
};
struct scmi_msg_set_sensor_trip_point {
@@ -117,7 +120,7 @@ static int scmi_sensor_description_get(const struct scmi_handle *handle,
do {
/* Set the number of sensors to be skipped/already read */
- *(__le32 *)t->tx.buf = cpu_to_le32(desc_index);
+ put_unaligned_le32(desc_index, t->tx.buf);
ret = scmi_do_xfer(handle, t);
if (ret)
@@ -133,13 +136,20 @@ static int scmi_sensor_description_get(const struct scmi_handle *handle,
}
for (cnt = 0; cnt < num_returned; cnt++) {
- u32 attrh;
+ u32 attrh, attrl;
struct scmi_sensor_info *s;
+ attrl = le32_to_cpu(buf->desc[cnt].attributes_low);
attrh = le32_to_cpu(buf->desc[cnt].attributes_high);
s = &si->sensors[desc_index + cnt];
s->id = le32_to_cpu(buf->desc[cnt].id);
s->type = SENSOR_TYPE(attrh);
+ s->scale = SENSOR_SCALE(attrh);
+ /* Sign extend to a full s8 */
+ if (s->scale & SENSOR_SCALE_SIGN)
+ s->scale |= SENSOR_SCALE_EXTEND;
+ s->async = SUPPORTS_ASYNC_READ(attrl);
+ s->num_trip_points = NUM_TRIP_POINTS(attrl);
strlcpy(s->name, buf->desc[cnt].name, SCMI_MAX_STR_SIZE);
}
@@ -154,15 +164,15 @@ static int scmi_sensor_description_get(const struct scmi_handle *handle,
return ret;
}
-static int
-scmi_sensor_configuration_set(const struct scmi_handle *handle, u32 sensor_id)
+static int scmi_sensor_trip_point_notify(const struct scmi_handle *handle,
+ u32 sensor_id, bool enable)
{
int ret;
- u32 evt_cntl = BIT(0);
+ u32 evt_cntl = enable ? SENSOR_TP_NOTIFY_ALL : 0;
struct scmi_xfer *t;
- struct scmi_msg_set_sensor_config *cfg;
+ struct scmi_msg_sensor_trip_point_notify *cfg;
- ret = scmi_xfer_get_init(handle, SENSOR_CONFIG_SET,
+ ret = scmi_xfer_get_init(handle, SENSOR_TRIP_POINT_NOTIFY,
SCMI_PROTOCOL_SENSOR, sizeof(*cfg), 0, &t);
if (ret)
return ret;
@@ -177,15 +187,16 @@ scmi_sensor_configuration_set(const struct scmi_handle *handle, u32 sensor_id)
return ret;
}
-static int scmi_sensor_trip_point_set(const struct scmi_handle *handle,
- u32 sensor_id, u8 trip_id, u64 trip_value)
+static int
+scmi_sensor_trip_point_config(const struct scmi_handle *handle, u32 sensor_id,
+ u8 trip_id, u64 trip_value)
{
int ret;
u32 evt_cntl = SENSOR_TP_BOTH;
struct scmi_xfer *t;
struct scmi_msg_set_sensor_trip_point *trip;
- ret = scmi_xfer_get_init(handle, SENSOR_TRIP_POINT_SET,
+ ret = scmi_xfer_get_init(handle, SENSOR_TRIP_POINT_CONFIG,
SCMI_PROTOCOL_SENSOR, sizeof(*trip), 0, &t);
if (ret)
return ret;
@@ -203,11 +214,13 @@ static int scmi_sensor_trip_point_set(const struct scmi_handle *handle,
}
static int scmi_sensor_reading_get(const struct scmi_handle *handle,
- u32 sensor_id, bool async, u64 *value)
+ u32 sensor_id, u64 *value)
{
int ret;
struct scmi_xfer *t;
struct scmi_msg_sensor_reading_get *sensor;
+ struct sensors_info *si = handle->sensor_priv;
+ struct scmi_sensor_info *s = si->sensors + sensor_id;
ret = scmi_xfer_get_init(handle, SENSOR_READING_GET,
SCMI_PROTOCOL_SENSOR, sizeof(*sensor),
@@ -217,14 +230,18 @@ static int scmi_sensor_reading_get(const struct scmi_handle *handle,
sensor = t->tx.buf;
sensor->id = cpu_to_le32(sensor_id);
- sensor->flags = cpu_to_le32(async ? SENSOR_READ_ASYNC : 0);
- ret = scmi_do_xfer(handle, t);
- if (!ret) {
- __le32 *pval = t->rx.buf;
-
- *value = le32_to_cpu(*pval);
- *value |= (u64)le32_to_cpu(*(pval + 1)) << 32;
+ if (s->async) {
+ sensor->flags = cpu_to_le32(SENSOR_READ_ASYNC);
+ ret = scmi_do_xfer_with_response(handle, t);
+ if (!ret)
+ *value = get_unaligned_le64((void *)
+ ((__le32 *)t->rx.buf + 1));
+ } else {
+ sensor->flags = cpu_to_le32(0);
+ ret = scmi_do_xfer(handle, t);
+ if (!ret)
+ *value = get_unaligned_le64(t->rx.buf);
}
scmi_xfer_put(handle, t);
@@ -249,8 +266,8 @@ static int scmi_sensor_count_get(const struct scmi_handle *handle)
static struct scmi_sensor_ops sensor_ops = {
.count_get = scmi_sensor_count_get,
.info_get = scmi_sensor_info_get,
- .configuration_set = scmi_sensor_configuration_set,
- .trip_point_set = scmi_sensor_trip_point_set,
+ .trip_point_notify = scmi_sensor_trip_point_notify,
+ .trip_point_config = scmi_sensor_trip_point_config,
.reading_get = scmi_sensor_reading_get,
};
diff --git a/drivers/firmware/arm_scpi.c b/drivers/firmware/arm_scpi.c
index 725164b83242..a80c331c3a6e 100644
--- a/drivers/firmware/arm_scpi.c
+++ b/drivers/firmware/arm_scpi.c
@@ -1011,10 +1011,6 @@ static int scpi_probe(struct platform_device *pdev)
scpi_info->firmware_version));
scpi_info->scpi_ops = &scpi_ops;
- ret = devm_device_add_groups(dev, versions_groups);
- if (ret)
- dev_err(dev, "unable to create sysfs version group\n");
-
return devm_of_platform_populate(dev);
}
@@ -1030,6 +1026,7 @@ static struct platform_driver scpi_driver = {
.driver = {
.name = "scpi_protocol",
.of_match_table = scpi_of_match,
+ .dev_groups = versions_groups,
},
.probe = scpi_probe,
.remove = scpi_remove,
diff --git a/drivers/firmware/efi/Kconfig b/drivers/firmware/efi/Kconfig
index d4ea929e8b34..178ee8106828 100644
--- a/drivers/firmware/efi/Kconfig
+++ b/drivers/firmware/efi/Kconfig
@@ -180,6 +180,19 @@ config RESET_ATTACK_MITIGATION
have been evicted, since otherwise it will trigger even on clean
reboots.
+config EFI_RCI2_TABLE
+ bool "EFI Runtime Configuration Interface Table Version 2 Support"
+ help
+ Displays the content of the Runtime Configuration Interface
+ Table version 2 on Dell EMC PowerEdge systems as a binary
+ attribute 'rci2' under /sys/firmware/efi/tables directory.
+
+ RCI2 table contains BIOS HII in XML format and is used to populate
+ BIOS setup page in Dell EMC OpenManage Server Administrator tool.
+ The BIOS setup page contains BIOS tokens which can be configured.
+
+ Say Y here for Dell EMC PowerEdge systems.
+
endmenu
config UEFI_CPER
diff --git a/drivers/firmware/efi/Makefile b/drivers/firmware/efi/Makefile
index d2d0d2030620..4ac2de4dfa72 100644
--- a/drivers/firmware/efi/Makefile
+++ b/drivers/firmware/efi/Makefile
@@ -25,6 +25,7 @@ obj-$(CONFIG_EFI_BOOTLOADER_CONTROL) += efibc.o
obj-$(CONFIG_EFI_TEST) += test/
obj-$(CONFIG_EFI_DEV_PATH_PARSER) += dev-path-parser.o
obj-$(CONFIG_APPLE_PROPERTIES) += apple-properties.o
+obj-$(CONFIG_EFI_RCI2_TABLE) += rci2-table.o
arm-obj-$(CONFIG_EFI) := arm-init.o arm-runtime.o
obj-$(CONFIG_ARM) += $(arm-obj-y)
diff --git a/drivers/firmware/efi/cper.c b/drivers/firmware/efi/cper.c
index 8fa977c7861f..addf0749dd8b 100644
--- a/drivers/firmware/efi/cper.c
+++ b/drivers/firmware/efi/cper.c
@@ -390,6 +390,21 @@ static void cper_print_pcie(const char *pfx, const struct cper_sec_pcie *pcie,
printk(
"%s""bridge: secondary_status: 0x%04x, control: 0x%04x\n",
pfx, pcie->bridge.secondary_status, pcie->bridge.control);
+
+ /* Fatal errors call __ghes_panic() before AER handler prints this */
+ if ((pcie->validation_bits & CPER_PCIE_VALID_AER_INFO) &&
+ (gdata->error_severity & CPER_SEV_FATAL)) {
+ struct aer_capability_regs *aer;
+
+ aer = (struct aer_capability_regs *)pcie->aer_info;
+ printk("%saer_uncor_status: 0x%08x, aer_uncor_mask: 0x%08x\n",
+ pfx, aer->uncor_status, aer->uncor_mask);
+ printk("%saer_uncor_severity: 0x%08x\n",
+ pfx, aer->uncor_severity);
+ printk("%sTLP Header: %08x %08x %08x %08x\n", pfx,
+ aer->header_log.dw0, aer->header_log.dw1,
+ aer->header_log.dw2, aer->header_log.dw3);
+ }
}
static void cper_print_tstamp(const char *pfx,
diff --git a/drivers/firmware/efi/dev-path-parser.c b/drivers/firmware/efi/dev-path-parser.c
index 85ec99f97841..20123384271c 100644
--- a/drivers/firmware/efi/dev-path-parser.c
+++ b/drivers/firmware/efi/dev-path-parser.c
@@ -17,9 +17,9 @@ struct acpi_hid_uid {
char uid[11]; /* UINT_MAX + null byte */
};
-static int __init match_acpi_dev(struct device *dev, void *data)
+static int __init match_acpi_dev(struct device *dev, const void *data)
{
- struct acpi_hid_uid hid_uid = *(struct acpi_hid_uid *)data;
+ struct acpi_hid_uid hid_uid = *(const struct acpi_hid_uid *)data;
struct acpi_device *adev = to_acpi_device(dev);
if (acpi_match_device_ids(adev, hid_uid.hid))
diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c
index 4b7cf7bc0ded..8f1ab04f6743 100644
--- a/drivers/firmware/efi/efi.c
+++ b/drivers/firmware/efi/efi.c
@@ -39,11 +39,9 @@ struct efi __read_mostly efi = {
.acpi20 = EFI_INVALID_TABLE_ADDR,
.smbios = EFI_INVALID_TABLE_ADDR,
.smbios3 = EFI_INVALID_TABLE_ADDR,
- .sal_systab = EFI_INVALID_TABLE_ADDR,
.boot_info = EFI_INVALID_TABLE_ADDR,
.hcdp = EFI_INVALID_TABLE_ADDR,
.uga = EFI_INVALID_TABLE_ADDR,
- .uv_systab = EFI_INVALID_TABLE_ADDR,
.fw_vendor = EFI_INVALID_TABLE_ADDR,
.runtime = EFI_INVALID_TABLE_ADDR,
.config_table = EFI_INVALID_TABLE_ADDR,
@@ -52,29 +50,11 @@ struct efi __read_mostly efi = {
.mem_attr_table = EFI_INVALID_TABLE_ADDR,
.rng_seed = EFI_INVALID_TABLE_ADDR,
.tpm_log = EFI_INVALID_TABLE_ADDR,
+ .tpm_final_log = EFI_INVALID_TABLE_ADDR,
.mem_reserve = EFI_INVALID_TABLE_ADDR,
};
EXPORT_SYMBOL(efi);
-static unsigned long *efi_tables[] = {
- &efi.mps,
- &efi.acpi,
- &efi.acpi20,
- &efi.smbios,
- &efi.smbios3,
- &efi.sal_systab,
- &efi.boot_info,
- &efi.hcdp,
- &efi.uga,
- &efi.uv_systab,
- &efi.fw_vendor,
- &efi.runtime,
- &efi.config_table,
- &efi.esrt,
- &efi.properties_table,
- &efi.mem_attr_table,
-};
-
struct mm_struct efi_mm = {
.mm_rb = RB_ROOT,
.mm_users = ATOMIC_INIT(2),
@@ -475,7 +455,6 @@ static __initdata efi_config_table_type_t common_tables[] = {
{ACPI_TABLE_GUID, "ACPI", &efi.acpi},
{HCDP_TABLE_GUID, "HCDP", &efi.hcdp},
{MPS_TABLE_GUID, "MPS", &efi.mps},
- {SAL_SYSTEM_TABLE_GUID, "SALsystab", &efi.sal_systab},
{SMBIOS_TABLE_GUID, "SMBIOS", &efi.smbios},
{SMBIOS3_TABLE_GUID, "SMBIOS 3.0", &efi.smbios3},
{UGA_IO_PROTOCOL_GUID, "UGA", &efi.uga},
@@ -484,7 +463,11 @@ static __initdata efi_config_table_type_t common_tables[] = {
{EFI_MEMORY_ATTRIBUTES_TABLE_GUID, "MEMATTR", &efi.mem_attr_table},
{LINUX_EFI_RANDOM_SEED_TABLE_GUID, "RNG", &efi.rng_seed},
{LINUX_EFI_TPM_EVENT_LOG_GUID, "TPMEventLog", &efi.tpm_log},
+ {LINUX_EFI_TPM_FINAL_LOG_GUID, "TPMFinalLog", &efi.tpm_final_log},
{LINUX_EFI_MEMRESERVE_TABLE_GUID, "MEMRESERVE", &efi.mem_reserve},
+#ifdef CONFIG_EFI_RCI2_TABLE
+ {DELLEMC_EFI_RCI2_TABLE_GUID, NULL, &rci2_table_phys},
+#endif
{NULL_GUID, NULL, NULL},
};
@@ -962,20 +945,6 @@ int efi_status_to_err(efi_status_t status)
return err;
}
-bool efi_is_table_address(unsigned long phys_addr)
-{
- unsigned int i;
-
- if (phys_addr == EFI_INVALID_TABLE_ADDR)
- return false;
-
- for (i = 0; i < ARRAY_SIZE(efi_tables); i++)
- if (*(efi_tables[i]) == phys_addr)
- return true;
-
- return false;
-}
-
static DEFINE_SPINLOCK(efi_mem_reserve_persistent_lock);
static struct linux_efi_memreserve *efi_memreserve_root __ro_after_init;
diff --git a/drivers/firmware/efi/libstub/efi-stub-helper.c b/drivers/firmware/efi/libstub/efi-stub-helper.c
index e4610e72b78f..3caae7f2cf56 100644
--- a/drivers/firmware/efi/libstub/efi-stub-helper.c
+++ b/drivers/firmware/efi/libstub/efi-stub-helper.c
@@ -926,3 +926,34 @@ free_map:
fail:
return status;
}
+
+#define GET_EFI_CONFIG_TABLE(bits) \
+static void *get_efi_config_table##bits(efi_system_table_t *_sys_table, \
+ efi_guid_t guid) \
+{ \
+ efi_system_table_##bits##_t *sys_table; \
+ efi_config_table_##bits##_t *tables; \
+ int i; \
+ \
+ sys_table = (typeof(sys_table))_sys_table; \
+ tables = (typeof(tables))(unsigned long)sys_table->tables; \
+ \
+ for (i = 0; i < sys_table->nr_tables; i++) { \
+ if (efi_guidcmp(tables[i].guid, guid) != 0) \
+ continue; \
+ \
+ return (void *)(unsigned long)tables[i].table; \
+ } \
+ \
+ return NULL; \
+}
+GET_EFI_CONFIG_TABLE(32)
+GET_EFI_CONFIG_TABLE(64)
+
+void *get_efi_config_table(efi_system_table_t *sys_table, efi_guid_t guid)
+{
+ if (efi_is_64bit())
+ return get_efi_config_table64(sys_table, guid);
+ else
+ return get_efi_config_table32(sys_table, guid);
+}
diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h
index 1b1dfcaa6fb9..7f1556fd867d 100644
--- a/drivers/firmware/efi/libstub/efistub.h
+++ b/drivers/firmware/efi/libstub/efistub.h
@@ -65,6 +65,8 @@ efi_status_t check_platform_features(efi_system_table_t *sys_table_arg);
efi_status_t efi_random_get_seed(efi_system_table_t *sys_table_arg);
+void *get_efi_config_table(efi_system_table_t *sys_table, efi_guid_t guid);
+
/* Helper macros for the usual case of using simple C variables: */
#ifndef fdt_setprop_inplace_var
#define fdt_setprop_inplace_var(fdt, node_offset, name, var) \
diff --git a/drivers/firmware/efi/libstub/fdt.c b/drivers/firmware/efi/libstub/fdt.c
index 5440ba17a1c5..0bf0190917e0 100644
--- a/drivers/firmware/efi/libstub/fdt.c
+++ b/drivers/firmware/efi/libstub/fdt.c
@@ -363,26 +363,17 @@ fail:
void *get_fdt(efi_system_table_t *sys_table, unsigned long *fdt_size)
{
- efi_guid_t fdt_guid = DEVICE_TREE_GUID;
- efi_config_table_t *tables;
- int i;
+ void *fdt;
- tables = (efi_config_table_t *)sys_table->tables;
+ fdt = get_efi_config_table(sys_table, DEVICE_TREE_GUID);
- for (i = 0; i < sys_table->nr_tables; i++) {
- void *fdt;
+ if (!fdt)
+ return NULL;
- if (efi_guidcmp(tables[i].guid, fdt_guid) != 0)
- continue;
-
- fdt = (void *)tables[i].table;
- if (fdt_check_header(fdt) != 0) {
- pr_efi_err(sys_table, "Invalid header detected on UEFI supplied FDT, ignoring ...\n");
- return NULL;
- }
- *fdt_size = fdt_totalsize(fdt);
- return fdt;
+ if (fdt_check_header(fdt) != 0) {
+ pr_efi_err(sys_table, "Invalid header detected on UEFI supplied FDT, ignoring ...\n");
+ return NULL;
}
-
- return NULL;
+ *fdt_size = fdt_totalsize(fdt);
+ return fdt;
}
diff --git a/drivers/firmware/efi/libstub/tpm.c b/drivers/firmware/efi/libstub/tpm.c
index 5bd04f75d8d6..eb9af83e4d59 100644
--- a/drivers/firmware/efi/libstub/tpm.c
+++ b/drivers/firmware/efi/libstub/tpm.c
@@ -57,31 +57,40 @@ void efi_enable_reset_attack_mitigation(efi_system_table_t *sys_table_arg)
#endif
-static void efi_retrieve_tpm2_eventlog_1_2(efi_system_table_t *sys_table_arg)
+void efi_retrieve_tpm2_eventlog(efi_system_table_t *sys_table_arg)
{
efi_guid_t tcg2_guid = EFI_TCG2_PROTOCOL_GUID;
efi_guid_t linux_eventlog_guid = LINUX_EFI_TPM_EVENT_LOG_GUID;
efi_status_t status;
efi_physical_addr_t log_location = 0, log_last_entry = 0;
struct linux_efi_tpm_eventlog *log_tbl = NULL;
+ struct efi_tcg2_final_events_table *final_events_table;
unsigned long first_entry_addr, last_entry_addr;
size_t log_size, last_entry_size;
efi_bool_t truncated;
+ int version = EFI_TCG2_EVENT_LOG_FORMAT_TCG_2;
void *tcg2_protocol = NULL;
+ int final_events_size = 0;
status = efi_call_early(locate_protocol, &tcg2_guid, NULL,
&tcg2_protocol);
if (status != EFI_SUCCESS)
return;
- status = efi_call_proto(efi_tcg2_protocol, get_event_log, tcg2_protocol,
- EFI_TCG2_EVENT_LOG_FORMAT_TCG_1_2,
- &log_location, &log_last_entry, &truncated);
- if (status != EFI_SUCCESS)
- return;
+ status = efi_call_proto(efi_tcg2_protocol, get_event_log,
+ tcg2_protocol, version, &log_location,
+ &log_last_entry, &truncated);
+
+ if (status != EFI_SUCCESS || !log_location) {
+ version = EFI_TCG2_EVENT_LOG_FORMAT_TCG_1_2;
+ status = efi_call_proto(efi_tcg2_protocol, get_event_log,
+ tcg2_protocol, version, &log_location,
+ &log_last_entry, &truncated);
+ if (status != EFI_SUCCESS || !log_location)
+ return;
+
+ }
- if (!log_location)
- return;
first_entry_addr = (unsigned long) log_location;
/*
@@ -96,8 +105,23 @@ static void efi_retrieve_tpm2_eventlog_1_2(efi_system_table_t *sys_table_arg)
* We need to calculate its size to deduce the full size of
* the logs.
*/
- last_entry_size = sizeof(struct tcpa_event) +
- ((struct tcpa_event *) last_entry_addr)->event_size;
+ if (version == EFI_TCG2_EVENT_LOG_FORMAT_TCG_2) {
+ /*
+ * The TCG2 log format has variable length entries,
+ * and the information to decode the hash algorithms
+ * back into a size is contained in the first entry -
+ * pass a pointer to the final entry (to calculate its
+ * size) and the first entry (so we know how long each
+ * digest is)
+ */
+ last_entry_size =
+ __calc_tpm2_event_size((void *)last_entry_addr,
+ (void *)(long)log_location,
+ false);
+ } else {
+ last_entry_size = sizeof(struct tcpa_event) +
+ ((struct tcpa_event *) last_entry_addr)->event_size;
+ }
log_size = log_last_entry - log_location + last_entry_size;
}
@@ -112,9 +136,37 @@ static void efi_retrieve_tpm2_eventlog_1_2(efi_system_table_t *sys_table_arg)
return;
}
+ /*
+ * Figure out whether any events have already been logged to the
+ * final events structure, and if so how much space they take up
+ */
+ final_events_table = get_efi_config_table(sys_table_arg,
+ LINUX_EFI_TPM_FINAL_LOG_GUID);
+ if (final_events_table && final_events_table->nr_events) {
+ struct tcg_pcr_event2_head *header;
+ int offset;
+ void *data;
+ int event_size;
+ int i = final_events_table->nr_events;
+
+ data = (void *)final_events_table;
+ offset = sizeof(final_events_table->version) +
+ sizeof(final_events_table->nr_events);
+
+ while (i > 0) {
+ header = data + offset + final_events_size;
+ event_size = __calc_tpm2_event_size(header,
+ (void *)(long)log_location,
+ false);
+ final_events_size += event_size;
+ i--;
+ }
+ }
+
memset(log_tbl, 0, sizeof(*log_tbl) + log_size);
log_tbl->size = log_size;
- log_tbl->version = EFI_TCG2_EVENT_LOG_FORMAT_TCG_1_2;
+ log_tbl->final_events_preboot_size = final_events_size;
+ log_tbl->version = version;
memcpy(log_tbl->log, (void *) first_entry_addr, log_size);
status = efi_call_early(install_configuration_table,
@@ -126,9 +178,3 @@ static void efi_retrieve_tpm2_eventlog_1_2(efi_system_table_t *sys_table_arg)
err_free:
efi_call_early(free_pool, log_tbl);
}
-
-void efi_retrieve_tpm2_eventlog(efi_system_table_t *sys_table_arg)
-{
- /* Only try to retrieve the logs in 1.2 format. */
- efi_retrieve_tpm2_eventlog_1_2(sys_table_arg);
-}
diff --git a/drivers/firmware/efi/rci2-table.c b/drivers/firmware/efi/rci2-table.c
new file mode 100644
index 000000000000..3e290f96620a
--- /dev/null
+++ b/drivers/firmware/efi/rci2-table.c
@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Export Runtime Configuration Interface Table Version 2 (RCI2)
+ * to sysfs
+ *
+ * Copyright (C) 2019 Dell Inc
+ * by Narendra K <Narendra.K@dell.com>
+ *
+ * System firmware advertises the address of the RCI2 Table via
+ * an EFI Configuration Table entry. This code retrieves the RCI2
+ * table from the address and exports it to sysfs as a binary
+ * attribute 'rci2' under /sys/firmware/efi/tables directory.
+ */
+
+#include <linux/kobject.h>
+#include <linux/device.h>
+#include <linux/sysfs.h>
+#include <linux/efi.h>
+#include <linux/types.h>
+#include <linux/io.h>
+
+#define RCI_SIGNATURE "_RC_"
+
+struct rci2_table_global_hdr {
+ u16 type;
+ u16 resvd0;
+ u16 hdr_len;
+ u8 rci2_sig[4];
+ u16 resvd1;
+ u32 resvd2;
+ u32 resvd3;
+ u8 major_rev;
+ u8 minor_rev;
+ u16 num_of_structs;
+ u32 rci2_len;
+ u16 rci2_chksum;
+} __packed;
+
+static u8 *rci2_base;
+static u32 rci2_table_len;
+unsigned long rci2_table_phys __ro_after_init = EFI_INVALID_TABLE_ADDR;
+
+static ssize_t raw_table_read(struct file *file, struct kobject *kobj,
+ struct bin_attribute *attr, char *buf,
+ loff_t pos, size_t count)
+{
+ memcpy(buf, attr->private + pos, count);
+ return count;
+}
+
+static BIN_ATTR(rci2, S_IRUSR, raw_table_read, NULL, 0);
+
+static u16 checksum(void)
+{
+ u8 len_is_odd = rci2_table_len % 2;
+ u32 chksum_len = rci2_table_len;
+ u16 *base = (u16 *)rci2_base;
+ u8 buf[2] = {0};
+ u32 offset = 0;
+ u16 chksum = 0;
+
+ if (len_is_odd)
+ chksum_len -= 1;
+
+ while (offset < chksum_len) {
+ chksum += *base;
+ offset += 2;
+ base++;
+ }
+
+ if (len_is_odd) {
+ buf[0] = *(u8 *)base;
+ chksum += *(u16 *)(buf);
+ }
+
+ return chksum;
+}
+
+int __init efi_rci2_sysfs_init(void)
+{
+ struct kobject *tables_kobj;
+ int ret = -ENOMEM;
+
+ rci2_base = memremap(rci2_table_phys,
+ sizeof(struct rci2_table_global_hdr),
+ MEMREMAP_WB);
+ if (!rci2_base) {
+ pr_debug("RCI2 table init failed - could not map RCI2 table\n");
+ goto err;
+ }
+
+ if (strncmp(rci2_base +
+ offsetof(struct rci2_table_global_hdr, rci2_sig),
+ RCI_SIGNATURE, 4)) {
+ pr_debug("RCI2 table init failed - incorrect signature\n");
+ ret = -ENODEV;
+ goto err_unmap;
+ }
+
+ rci2_table_len = *(u32 *)(rci2_base +
+ offsetof(struct rci2_table_global_hdr,
+ rci2_len));
+
+ memunmap(rci2_base);
+
+ if (!rci2_table_len) {
+ pr_debug("RCI2 table init failed - incorrect table length\n");
+ goto err;
+ }
+
+ rci2_base = memremap(rci2_table_phys, rci2_table_len, MEMREMAP_WB);
+ if (!rci2_base) {
+ pr_debug("RCI2 table - could not map RCI2 table\n");
+ goto err;
+ }
+
+ if (checksum() != 0) {
+ pr_debug("RCI2 table - incorrect checksum\n");
+ ret = -ENODEV;
+ goto err_unmap;
+ }
+
+ tables_kobj = kobject_create_and_add("tables", efi_kobj);
+ if (!tables_kobj) {
+ pr_debug("RCI2 table - tables_kobj creation failed\n");
+ goto err_unmap;
+ }
+
+ bin_attr_rci2.size = rci2_table_len;
+ bin_attr_rci2.private = rci2_base;
+ ret = sysfs_create_bin_file(tables_kobj, &bin_attr_rci2);
+ if (ret != 0) {
+ pr_debug("RCI2 table - rci2 sysfs bin file creation failed\n");
+ kobject_del(tables_kobj);
+ kobject_put(tables_kobj);
+ goto err_unmap;
+ }
+
+ return 0;
+
+ err_unmap:
+ memunmap(rci2_base);
+ err:
+ pr_debug("RCI2 table - sysfs initialization failed\n");
+ return ret;
+}
+late_initcall(efi_rci2_sysfs_init);
diff --git a/drivers/firmware/efi/tpm.c b/drivers/firmware/efi/tpm.c
index 3a689b40ccc0..1d3f5ca3eaaf 100644
--- a/drivers/firmware/efi/tpm.c
+++ b/drivers/firmware/efi/tpm.c
@@ -4,11 +4,34 @@
* Thiebaud Weksteen <tweek@google.com>
*/
+#define TPM_MEMREMAP(start, size) early_memremap(start, size)
+#define TPM_MEMUNMAP(start, size) early_memunmap(start, size)
+
+#include <asm/early_ioremap.h>
#include <linux/efi.h>
#include <linux/init.h>
#include <linux/memblock.h>
+#include <linux/tpm_eventlog.h>
-#include <asm/early_ioremap.h>
+int efi_tpm_final_log_size;
+EXPORT_SYMBOL(efi_tpm_final_log_size);
+
+static int tpm2_calc_event_log_size(void *data, int count, void *size_info)
+{
+ struct tcg_pcr_event2_head *header;
+ int event_size, size = 0;
+
+ while (count > 0) {
+ header = data + size;
+ event_size = __calc_tpm2_event_size(header, size_info, true);
+ if (event_size == 0)
+ return -1;
+ size += event_size;
+ count--;
+ }
+
+ return size;
+}
/*
* Reserve the memory associated with the TPM Event Log configuration table.
@@ -16,22 +39,54 @@
int __init efi_tpm_eventlog_init(void)
{
struct linux_efi_tpm_eventlog *log_tbl;
+ struct efi_tcg2_final_events_table *final_tbl;
unsigned int tbl_size;
+ int ret = 0;
- if (efi.tpm_log == EFI_INVALID_TABLE_ADDR)
+ if (efi.tpm_log == EFI_INVALID_TABLE_ADDR) {
+ /*
+ * We can't calculate the size of the final events without the
+ * first entry in the TPM log, so bail here.
+ */
return 0;
+ }
log_tbl = early_memremap(efi.tpm_log, sizeof(*log_tbl));
if (!log_tbl) {
pr_err("Failed to map TPM Event Log table @ 0x%lx\n",
- efi.tpm_log);
+ efi.tpm_log);
efi.tpm_log = EFI_INVALID_TABLE_ADDR;
return -ENOMEM;
}
tbl_size = sizeof(*log_tbl) + log_tbl->size;
memblock_reserve(efi.tpm_log, tbl_size);
+
+ if (efi.tpm_final_log == EFI_INVALID_TABLE_ADDR)
+ goto out;
+
+ final_tbl = early_memremap(efi.tpm_final_log, sizeof(*final_tbl));
+
+ if (!final_tbl) {
+ pr_err("Failed to map TPM Final Event Log table @ 0x%lx\n",
+ efi.tpm_final_log);
+ efi.tpm_final_log = EFI_INVALID_TABLE_ADDR;
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ tbl_size = tpm2_calc_event_log_size((void *)efi.tpm_final_log
+ + sizeof(final_tbl->version)
+ + sizeof(final_tbl->nr_events),
+ final_tbl->nr_events,
+ log_tbl->log);
+ memblock_reserve((unsigned long)final_tbl,
+ tbl_size + sizeof(*final_tbl));
+ early_memunmap(final_tbl, sizeof(*final_tbl));
+ efi_tpm_final_log_size = tbl_size;
+
+out:
early_memunmap(log_tbl, sizeof(*log_tbl));
- return 0;
+ return ret;
}
diff --git a/drivers/firmware/google/coreboot_table.h b/drivers/firmware/google/coreboot_table.h
index 1f63b3034b17..7b7b4a6eedda 100644
--- a/drivers/firmware/google/coreboot_table.h
+++ b/drivers/firmware/google/coreboot_table.h
@@ -12,7 +12,7 @@
#ifndef __COREBOOT_TABLE_H
#define __COREBOOT_TABLE_H
-#include <linux/io.h>
+#include <linux/device.h>
/* Coreboot table header structure */
struct coreboot_table_header {
@@ -83,4 +83,13 @@ int coreboot_driver_register(struct coreboot_driver *driver);
/* Unregister a driver that uses the data from a coreboot table. */
void coreboot_driver_unregister(struct coreboot_driver *driver);
+/* module_coreboot_driver() - Helper macro for drivers that don't do
+ * anything special in module init/exit. This eliminates a lot of
+ * boilerplate. Each module may only use this macro once, and
+ * calling it replaces module_init() and module_exit()
+ */
+#define module_coreboot_driver(__coreboot_driver) \
+ module_driver(__coreboot_driver, coreboot_driver_register, \
+ coreboot_driver_unregister)
+
#endif /* __COREBOOT_TABLE_H */
diff --git a/drivers/firmware/google/framebuffer-coreboot.c b/drivers/firmware/google/framebuffer-coreboot.c
index 7e67b651e4ac..916f26adc595 100644
--- a/drivers/firmware/google/framebuffer-coreboot.c
+++ b/drivers/firmware/google/framebuffer-coreboot.c
@@ -89,19 +89,7 @@ static struct coreboot_driver framebuffer_driver = {
},
.tag = CB_TAG_FRAMEBUFFER,
};
-
-static int __init coreboot_framebuffer_init(void)
-{
- return coreboot_driver_register(&framebuffer_driver);
-}
-
-static void coreboot_framebuffer_exit(void)
-{
- coreboot_driver_unregister(&framebuffer_driver);
-}
-
-module_init(coreboot_framebuffer_init);
-module_exit(coreboot_framebuffer_exit);
+module_coreboot_driver(framebuffer_driver);
MODULE_AUTHOR("Samuel Holland <samuel@sholland.org>");
MODULE_LICENSE("GPL");
diff --git a/drivers/firmware/google/memconsole-coreboot.c b/drivers/firmware/google/memconsole-coreboot.c
index ac90e8536565..fd7f0fbec07e 100644
--- a/drivers/firmware/google/memconsole-coreboot.c
+++ b/drivers/firmware/google/memconsole-coreboot.c
@@ -8,6 +8,7 @@
*/
#include <linux/device.h>
+#include <linux/io.h>
#include <linux/kernel.h>
#include <linux/module.h>
@@ -26,7 +27,7 @@ struct cbmem_cons {
#define CURSOR_MASK ((1 << 28) - 1)
#define OVERFLOW (1 << 31)
-static struct cbmem_cons __iomem *cbmem_console;
+static struct cbmem_cons *cbmem_console;
static u32 cbmem_console_size;
/*
@@ -67,7 +68,7 @@ static ssize_t memconsole_coreboot_read(char *buf, loff_t pos, size_t count)
static int memconsole_probe(struct coreboot_device *dev)
{
- struct cbmem_cons __iomem *tmp_cbmc;
+ struct cbmem_cons *tmp_cbmc;
tmp_cbmc = memremap(dev->cbmem_ref.cbmem_addr,
sizeof(*tmp_cbmc), MEMREMAP_WB);
@@ -77,13 +78,13 @@ static int memconsole_probe(struct coreboot_device *dev)
/* Read size only once to prevent overrun attack through /dev/mem. */
cbmem_console_size = tmp_cbmc->size_dont_access_after_boot;
- cbmem_console = memremap(dev->cbmem_ref.cbmem_addr,
+ cbmem_console = devm_memremap(&dev->dev, dev->cbmem_ref.cbmem_addr,
cbmem_console_size + sizeof(*cbmem_console),
MEMREMAP_WB);
memunmap(tmp_cbmc);
- if (!cbmem_console)
- return -ENOMEM;
+ if (IS_ERR(cbmem_console))
+ return PTR_ERR(cbmem_console);
memconsole_setup(memconsole_coreboot_read);
@@ -94,9 +95,6 @@ static int memconsole_remove(struct coreboot_device *dev)
{
memconsole_exit();
- if (cbmem_console)
- memunmap(cbmem_console);
-
return 0;
}
@@ -108,19 +106,7 @@ static struct coreboot_driver memconsole_driver = {
},
.tag = CB_TAG_CBMEM_CONSOLE,
};
-
-static void coreboot_memconsole_exit(void)
-{
- coreboot_driver_unregister(&memconsole_driver);
-}
-
-static int __init coreboot_memconsole_init(void)
-{
- return coreboot_driver_register(&memconsole_driver);
-}
-
-module_exit(coreboot_memconsole_exit);
-module_init(coreboot_memconsole_init);
+module_coreboot_driver(memconsole_driver);
MODULE_AUTHOR("Google, Inc.");
MODULE_LICENSE("GPL");
diff --git a/drivers/firmware/google/memconsole.c b/drivers/firmware/google/memconsole.c
index fe5aa740c34d..44d314ad69e4 100644
--- a/drivers/firmware/google/memconsole.c
+++ b/drivers/firmware/google/memconsole.c
@@ -7,21 +7,22 @@
* Copyright 2017 Google Inc.
*/
-#include <linux/init.h>
#include <linux/sysfs.h>
#include <linux/kobject.h>
#include <linux/module.h>
#include "memconsole.h"
-static ssize_t (*memconsole_read_func)(char *, loff_t, size_t);
-
static ssize_t memconsole_read(struct file *filp, struct kobject *kobp,
struct bin_attribute *bin_attr, char *buf,
loff_t pos, size_t count)
{
+ ssize_t (*memconsole_read_func)(char *, loff_t, size_t);
+
+ memconsole_read_func = bin_attr->private;
if (WARN_ON_ONCE(!memconsole_read_func))
return -EIO;
+
return memconsole_read_func(buf, pos, count);
}
@@ -32,7 +33,7 @@ static struct bin_attribute memconsole_bin_attr = {
void memconsole_setup(ssize_t (*read_func)(char *, loff_t, size_t))
{
- memconsole_read_func = read_func;
+ memconsole_bin_attr.private = read_func;
}
EXPORT_SYMBOL(memconsole_setup);
diff --git a/drivers/firmware/google/vpd.c b/drivers/firmware/google/vpd.c
index fd5212c395c0..db0812263d46 100644
--- a/drivers/firmware/google/vpd.c
+++ b/drivers/firmware/google/vpd.c
@@ -92,8 +92,8 @@ static int vpd_section_check_key_name(const u8 *key, s32 key_len)
return VPD_OK;
}
-static int vpd_section_attrib_add(const u8 *key, s32 key_len,
- const u8 *value, s32 value_len,
+static int vpd_section_attrib_add(const u8 *key, u32 key_len,
+ const u8 *value, u32 value_len,
void *arg)
{
int ret;
@@ -316,19 +316,7 @@ static struct coreboot_driver vpd_driver = {
},
.tag = CB_TAG_VPD,
};
-
-static int __init coreboot_vpd_init(void)
-{
- return coreboot_driver_register(&vpd_driver);
-}
-
-static void __exit coreboot_vpd_exit(void)
-{
- coreboot_driver_unregister(&vpd_driver);
-}
-
-module_init(coreboot_vpd_init);
-module_exit(coreboot_vpd_exit);
+module_coreboot_driver(vpd_driver);
MODULE_AUTHOR("Google, Inc.");
MODULE_LICENSE("GPL");
diff --git a/drivers/firmware/google/vpd_decode.c b/drivers/firmware/google/vpd_decode.c
index c62fa7063a7c..dda525c0f968 100644
--- a/drivers/firmware/google/vpd_decode.c
+++ b/drivers/firmware/google/vpd_decode.c
@@ -7,12 +7,10 @@
* Copyright 2017 Google Inc.
*/
-#include <linux/export.h>
-
#include "vpd_decode.h"
-static int vpd_decode_len(const s32 max_len, const u8 *in,
- s32 *length, s32 *decoded_len)
+static int vpd_decode_len(const u32 max_len, const u8 *in,
+ u32 *length, u32 *decoded_len)
{
u8 more;
int i = 0;
@@ -32,18 +30,39 @@ static int vpd_decode_len(const s32 max_len, const u8 *in,
} while (more);
*decoded_len = i;
+ return VPD_OK;
+}
+static int vpd_decode_entry(const u32 max_len, const u8 *input_buf,
+ u32 *_consumed, const u8 **entry, u32 *entry_len)
+{
+ u32 decoded_len;
+ u32 consumed = *_consumed;
+
+ if (vpd_decode_len(max_len - consumed, &input_buf[consumed],
+ entry_len, &decoded_len) != VPD_OK)
+ return VPD_FAIL;
+ if (max_len - consumed < decoded_len)
+ return VPD_FAIL;
+
+ consumed += decoded_len;
+ *entry = input_buf + consumed;
+
+ /* entry_len is untrusted data and must be checked again. */
+ if (max_len - consumed < *entry_len)
+ return VPD_FAIL;
+
+ consumed += decoded_len;
+ *_consumed = consumed;
return VPD_OK;
}
-int vpd_decode_string(const s32 max_len, const u8 *input_buf, s32 *consumed,
+int vpd_decode_string(const u32 max_len, const u8 *input_buf, u32 *consumed,
vpd_decode_callback callback, void *callback_arg)
{
int type;
- int res;
- s32 key_len;
- s32 value_len;
- s32 decoded_len;
+ u32 key_len;
+ u32 value_len;
const u8 *key;
const u8 *value;
@@ -58,26 +77,14 @@ int vpd_decode_string(const s32 max_len, const u8 *input_buf, s32 *consumed,
case VPD_TYPE_STRING:
(*consumed)++;
- /* key */
- res = vpd_decode_len(max_len - *consumed, &input_buf[*consumed],
- &key_len, &decoded_len);
- if (res != VPD_OK || *consumed + decoded_len >= max_len)
+ if (vpd_decode_entry(max_len, input_buf, consumed, &key,
+ &key_len) != VPD_OK)
return VPD_FAIL;
- *consumed += decoded_len;
- key = &input_buf[*consumed];
- *consumed += key_len;
-
- /* value */
- res = vpd_decode_len(max_len - *consumed, &input_buf[*consumed],
- &value_len, &decoded_len);
- if (res != VPD_OK || *consumed + decoded_len > max_len)
+ if (vpd_decode_entry(max_len, input_buf, consumed, &value,
+ &value_len) != VPD_OK)
return VPD_FAIL;
- *consumed += decoded_len;
- value = &input_buf[*consumed];
- *consumed += value_len;
-
if (type == VPD_TYPE_STRING)
return callback(key, key_len, value, value_len,
callback_arg);
diff --git a/drivers/firmware/google/vpd_decode.h b/drivers/firmware/google/vpd_decode.h
index cf8c2ace155a..8dbe41cac599 100644
--- a/drivers/firmware/google/vpd_decode.h
+++ b/drivers/firmware/google/vpd_decode.h
@@ -25,8 +25,8 @@ enum {
};
/* Callback for vpd_decode_string to invoke. */
-typedef int vpd_decode_callback(const u8 *key, s32 key_len,
- const u8 *value, s32 value_len,
+typedef int vpd_decode_callback(const u8 *key, u32 key_len,
+ const u8 *value, u32 value_len,
void *arg);
/*
@@ -44,7 +44,7 @@ typedef int vpd_decode_callback(const u8 *key, s32 key_len,
* If one entry is successfully decoded, sends it to callback and returns the
* result.
*/
-int vpd_decode_string(const s32 max_len, const u8 *input_buf, s32 *consumed,
+int vpd_decode_string(const u32 max_len, const u8 *input_buf, u32 *consumed,
vpd_decode_callback callback, void *callback_arg);
#endif /* __VPD_DECODE_H */
diff --git a/drivers/firmware/imx/Kconfig b/drivers/firmware/imx/Kconfig
index 42b566f8903f..0dbee32da4c6 100644
--- a/drivers/firmware/imx/Kconfig
+++ b/drivers/firmware/imx/Kconfig
@@ -1,4 +1,15 @@
# SPDX-License-Identifier: GPL-2.0-only
+config IMX_DSP
+ bool "IMX DSP Protocol driver"
+ depends on IMX_MBOX
+ help
+ This enables DSP IPC protocol between host AP (Linux)
+ and the firmware running on DSP.
+ DSP exists on some i.MX8 processors (e.g i.MX8QM, i.MX8QXP).
+
+ It acts like a doorbell. Client might use shared memory to
+ exchange information with DSP side.
+
config IMX_SCU
bool "IMX SCU Protocol driver"
depends on IMX_MBOX
diff --git a/drivers/firmware/imx/Makefile b/drivers/firmware/imx/Makefile
index 802c4ad8e8f9..08bc9ddfbdfb 100644
--- a/drivers/firmware/imx/Makefile
+++ b/drivers/firmware/imx/Makefile
@@ -1,3 +1,4 @@
# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_IMX_DSP) += imx-dsp.o
obj-$(CONFIG_IMX_SCU) += imx-scu.o misc.o imx-scu-irq.o
obj-$(CONFIG_IMX_SCU_PD) += scu-pd.o
diff --git a/drivers/firmware/imx/imx-dsp.c b/drivers/firmware/imx/imx-dsp.c
new file mode 100644
index 000000000000..a43d2db5cbdb
--- /dev/null
+++ b/drivers/firmware/imx/imx-dsp.c
@@ -0,0 +1,155 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright 2019 NXP
+ * Author: Daniel Baluta <daniel.baluta@nxp.com>
+ *
+ * Implementation of the DSP IPC interface (host side)
+ */
+
+#include <linux/firmware/imx/dsp.h>
+#include <linux/kernel.h>
+#include <linux/mailbox_client.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+/*
+ * imx_dsp_ring_doorbell - triggers an interrupt on the other side (DSP)
+ *
+ * @dsp: DSP IPC handle
+ * @chan_idx: index of the channel where to trigger the interrupt
+ *
+ * Returns non-negative value for success, negative value for error
+ */
+int imx_dsp_ring_doorbell(struct imx_dsp_ipc *ipc, unsigned int idx)
+{
+ int ret;
+ struct imx_dsp_chan *dsp_chan;
+
+ if (idx >= DSP_MU_CHAN_NUM)
+ return -EINVAL;
+
+ dsp_chan = &ipc->chans[idx];
+ ret = mbox_send_message(dsp_chan->ch, NULL);
+ if (ret < 0)
+ return ret;
+
+ return 0;
+}
+EXPORT_SYMBOL(imx_dsp_ring_doorbell);
+
+/*
+ * imx_dsp_handle_rx - rx callback used by imx mailbox
+ *
+ * @c: mbox client
+ * @msg: message received
+ *
+ * Users of DSP IPC will need to privde handle_reply and handle_request
+ * callbacks.
+ */
+static void imx_dsp_handle_rx(struct mbox_client *c, void *msg)
+{
+ struct imx_dsp_chan *chan = container_of(c, struct imx_dsp_chan, cl);
+
+ if (chan->idx == 0) {
+ chan->ipc->ops->handle_reply(chan->ipc);
+ } else {
+ chan->ipc->ops->handle_request(chan->ipc);
+ imx_dsp_ring_doorbell(chan->ipc, 1);
+ }
+}
+
+static int imx_dsp_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct imx_dsp_ipc *dsp_ipc;
+ struct imx_dsp_chan *dsp_chan;
+ struct mbox_client *cl;
+ char *chan_name;
+ int ret;
+ int i, j;
+
+ device_set_of_node_from_dev(&pdev->dev, pdev->dev.parent);
+
+ dsp_ipc = devm_kzalloc(dev, sizeof(*dsp_ipc), GFP_KERNEL);
+ if (!dsp_ipc)
+ return -ENOMEM;
+
+ for (i = 0; i < DSP_MU_CHAN_NUM; i++) {
+ if (i < 2)
+ chan_name = kasprintf(GFP_KERNEL, "txdb%d", i);
+ else
+ chan_name = kasprintf(GFP_KERNEL, "rxdb%d", i - 2);
+
+ if (!chan_name)
+ return -ENOMEM;
+
+ dsp_chan = &dsp_ipc->chans[i];
+ cl = &dsp_chan->cl;
+ cl->dev = dev;
+ cl->tx_block = false;
+ cl->knows_txdone = true;
+ cl->rx_callback = imx_dsp_handle_rx;
+
+ dsp_chan->ipc = dsp_ipc;
+ dsp_chan->idx = i % 2;
+ dsp_chan->ch = mbox_request_channel_byname(cl, chan_name);
+ if (IS_ERR(dsp_chan->ch)) {
+ ret = PTR_ERR(dsp_chan->ch);
+ if (ret != -EPROBE_DEFER)
+ dev_err(dev, "Failed to request mbox chan %s ret %d\n",
+ chan_name, ret);
+ goto out;
+ }
+
+ dev_dbg(dev, "request mbox chan %s\n", chan_name);
+ /* chan_name is not used anymore by framework */
+ kfree(chan_name);
+ }
+
+ dsp_ipc->dev = dev;
+
+ dev_set_drvdata(dev, dsp_ipc);
+
+ dev_info(dev, "NXP i.MX DSP IPC initialized\n");
+
+ return devm_of_platform_populate(dev);
+out:
+ kfree(chan_name);
+ for (j = 0; j < i; j++) {
+ dsp_chan = &dsp_ipc->chans[j];
+ mbox_free_channel(dsp_chan->ch);
+ }
+
+ return ret;
+}
+
+static int imx_dsp_remove(struct platform_device *pdev)
+{
+ struct imx_dsp_chan *dsp_chan;
+ struct imx_dsp_ipc *dsp_ipc;
+ int i;
+
+ dsp_ipc = dev_get_drvdata(&pdev->dev);
+
+ for (i = 0; i < DSP_MU_CHAN_NUM; i++) {
+ dsp_chan = &dsp_ipc->chans[i];
+ mbox_free_channel(dsp_chan->ch);
+ }
+
+ return 0;
+}
+
+static struct platform_driver imx_dsp_driver = {
+ .driver = {
+ .name = "imx-dsp",
+ },
+ .probe = imx_dsp_probe,
+ .remove = imx_dsp_remove,
+};
+builtin_platform_driver(imx_dsp_driver);
+
+MODULE_AUTHOR("Daniel Baluta <daniel.baluta@nxp.com>");
+MODULE_DESCRIPTION("IMX DSP IPC protocol driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/firmware/imx/scu-pd.c b/drivers/firmware/imx/scu-pd.c
index 480cec69e2c9..b556612207e5 100644
--- a/drivers/firmware/imx/scu-pd.c
+++ b/drivers/firmware/imx/scu-pd.c
@@ -92,7 +92,8 @@ static const struct imx_sc_pd_range imx8qxp_scu_pd_ranges[] = {
{ "gpt", IMX_SC_R_GPT_0, 5, true, 0 },
{ "kpp", IMX_SC_R_KPP, 1, false, 0 },
{ "fspi", IMX_SC_R_FSPI_0, 2, true, 0 },
- { "mu", IMX_SC_R_MU_0A, 14, true, 0 },
+ { "mu_a", IMX_SC_R_MU_0A, 14, true, 0 },
+ { "mu_b", IMX_SC_R_MU_13B, 1, true, 13 },
/* CONN SS */
{ "usb", IMX_SC_R_USB_0, 2, true, 0 },
@@ -130,6 +131,7 @@ static const struct imx_sc_pd_range imx8qxp_scu_pd_ranges[] = {
{ "lcd0-pwm", IMX_SC_R_LCD_0_PWM_0, 1, true, 0 },
{ "lpuart", IMX_SC_R_UART_0, 4, true, 0 },
{ "lpspi", IMX_SC_R_SPI_0, 4, true, 0 },
+ { "irqstr_dsp", IMX_SC_R_IRQSTR_DSP, 1, false, 0 },
/* VPU SS */
{ "vpu", IMX_SC_R_VPU, 1, false, 0 },
diff --git a/drivers/firmware/iscsi_ibft.c b/drivers/firmware/iscsi_ibft.c
index ab3aa3983833..7e12cbdf957c 100644
--- a/drivers/firmware/iscsi_ibft.c
+++ b/drivers/firmware/iscsi_ibft.c
@@ -84,6 +84,10 @@ MODULE_DESCRIPTION("sysfs interface to BIOS iBFT information");
MODULE_LICENSE("GPL");
MODULE_VERSION(IBFT_ISCSI_VERSION);
+#ifndef CONFIG_ISCSI_IBFT_FIND
+struct acpi_table_ibft *ibft_addr;
+#endif
+
struct ibft_hdr {
u8 id;
u8 version;
diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c
index f82ccd39a913..84f4ff351c62 100644
--- a/drivers/firmware/psci/psci.c
+++ b/drivers/firmware/psci/psci.c
@@ -103,7 +103,7 @@ static inline bool psci_power_state_loses_context(u32 state)
return state & mask;
}
-static inline bool psci_power_state_is_valid(u32 state)
+bool psci_power_state_is_valid(u32 state)
{
const u32 valid_mask = psci_has_ext_power_state() ?
PSCI_1_0_EXT_POWER_STATE_MASK :
@@ -277,175 +277,24 @@ static int __init psci_features(u32 psci_func_id)
}
#ifdef CONFIG_CPU_IDLE
-static DEFINE_PER_CPU_READ_MOSTLY(u32 *, psci_power_state);
-
-static int psci_dt_parse_state_node(struct device_node *np, u32 *state)
-{
- int err = of_property_read_u32(np, "arm,psci-suspend-param", state);
-
- if (err) {
- pr_warn("%pOF missing arm,psci-suspend-param property\n", np);
- return err;
- }
-
- if (!psci_power_state_is_valid(*state)) {
- pr_warn("Invalid PSCI power state %#x\n", *state);
- return -EINVAL;
- }
-
- return 0;
-}
-
-static int psci_dt_cpu_init_idle(struct device_node *cpu_node, int cpu)
-{
- int i, ret = 0, count = 0;
- u32 *psci_states;
- struct device_node *state_node;
-
- /* Count idle states */
- while ((state_node = of_parse_phandle(cpu_node, "cpu-idle-states",
- count))) {
- count++;
- of_node_put(state_node);
- }
-
- if (!count)
- return -ENODEV;
-
- psci_states = kcalloc(count, sizeof(*psci_states), GFP_KERNEL);
- if (!psci_states)
- return -ENOMEM;
-
- for (i = 0; i < count; i++) {
- state_node = of_parse_phandle(cpu_node, "cpu-idle-states", i);
- ret = psci_dt_parse_state_node(state_node, &psci_states[i]);
- of_node_put(state_node);
-
- if (ret)
- goto free_mem;
-
- pr_debug("psci-power-state %#x index %d\n", psci_states[i], i);
- }
-
- /* Idle states parsed correctly, initialize per-cpu pointer */
- per_cpu(psci_power_state, cpu) = psci_states;
- return 0;
-
-free_mem:
- kfree(psci_states);
- return ret;
-}
-
-#ifdef CONFIG_ACPI
-#include <acpi/processor.h>
-
-static int __maybe_unused psci_acpi_cpu_init_idle(unsigned int cpu)
-{
- int i, count;
- u32 *psci_states;
- struct acpi_lpi_state *lpi;
- struct acpi_processor *pr = per_cpu(processors, cpu);
-
- if (unlikely(!pr || !pr->flags.has_lpi))
- return -EINVAL;
-
- count = pr->power.count - 1;
- if (count <= 0)
- return -ENODEV;
-
- psci_states = kcalloc(count, sizeof(*psci_states), GFP_KERNEL);
- if (!psci_states)
- return -ENOMEM;
-
- for (i = 0; i < count; i++) {
- u32 state;
-
- lpi = &pr->power.lpi_states[i + 1];
- /*
- * Only bits[31:0] represent a PSCI power_state while
- * bits[63:32] must be 0x0 as per ARM ACPI FFH Specification
- */
- state = lpi->address;
- if (!psci_power_state_is_valid(state)) {
- pr_warn("Invalid PSCI power state %#x\n", state);
- kfree(psci_states);
- return -EINVAL;
- }
- psci_states[i] = state;
- }
- /* Idle states parsed correctly, initialize per-cpu pointer */
- per_cpu(psci_power_state, cpu) = psci_states;
- return 0;
-}
-#else
-static int __maybe_unused psci_acpi_cpu_init_idle(unsigned int cpu)
-{
- return -EINVAL;
-}
-#endif
-
-int psci_cpu_init_idle(unsigned int cpu)
-{
- struct device_node *cpu_node;
- int ret;
-
- /*
- * If the PSCI cpu_suspend function hook has not been initialized
- * idle states must not be enabled, so bail out
- */
- if (!psci_ops.cpu_suspend)
- return -EOPNOTSUPP;
-
- if (!acpi_disabled)
- return psci_acpi_cpu_init_idle(cpu);
-
- cpu_node = of_get_cpu_node(cpu, NULL);
- if (!cpu_node)
- return -ENODEV;
-
- ret = psci_dt_cpu_init_idle(cpu_node, cpu);
-
- of_node_put(cpu_node);
-
- return ret;
-}
-
-static int psci_suspend_finisher(unsigned long index)
+static int psci_suspend_finisher(unsigned long state)
{
- u32 *state = __this_cpu_read(psci_power_state);
+ u32 power_state = state;
- return psci_ops.cpu_suspend(state[index - 1],
- __pa_symbol(cpu_resume));
+ return psci_ops.cpu_suspend(power_state, __pa_symbol(cpu_resume));
}
-int psci_cpu_suspend_enter(unsigned long index)
+int psci_cpu_suspend_enter(u32 state)
{
int ret;
- u32 *state = __this_cpu_read(psci_power_state);
- /*
- * idle state index 0 corresponds to wfi, should never be called
- * from the cpu_suspend operations
- */
- if (WARN_ON_ONCE(!index))
- return -EINVAL;
- if (!psci_power_state_loses_context(state[index - 1]))
- ret = psci_ops.cpu_suspend(state[index - 1], 0);
+ if (!psci_power_state_loses_context(state))
+ ret = psci_ops.cpu_suspend(state, 0);
else
- ret = cpu_suspend(index, psci_suspend_finisher);
+ ret = cpu_suspend(state, psci_suspend_finisher);
return ret;
}
-
-/* ARM specific CPU idle operations */
-#ifdef CONFIG_ARM
-static const struct cpuidle_ops psci_cpuidle_ops __initconst = {
- .suspend = psci_cpu_suspend_enter,
- .init = psci_dt_cpu_init_idle,
-};
-
-CPUIDLE_METHOD_OF_DECLARE(psci, "psci", &psci_cpuidle_ops);
-#endif
#endif
static int psci_system_suspend(unsigned long unused)
diff --git a/drivers/firmware/psci/psci_checker.c b/drivers/firmware/psci/psci_checker.c
index 08c85099d4d0..6a445397771c 100644
--- a/drivers/firmware/psci/psci_checker.c
+++ b/drivers/firmware/psci/psci_checker.c
@@ -228,8 +228,11 @@ out_free_cpus:
static void dummy_callback(struct timer_list *unused) {}
-static int suspend_cpu(int index, bool broadcast)
+static int suspend_cpu(struct cpuidle_device *dev,
+ struct cpuidle_driver *drv, int index)
{
+ struct cpuidle_state *state = &drv->states[index];
+ bool broadcast = state->flags & CPUIDLE_FLAG_TIMER_STOP;
int ret;
arch_cpu_idle_enter();
@@ -254,11 +257,7 @@ static int suspend_cpu(int index, bool broadcast)
}
}
- /*
- * Replicate the common ARM cpuidle enter function
- * (arm_enter_idle_state).
- */
- ret = CPU_PM_CPU_IDLE_ENTER(arm_cpuidle_suspend, index);
+ ret = state->enter(dev, drv, index);
if (broadcast)
tick_broadcast_exit();
@@ -301,9 +300,8 @@ static int suspend_test_thread(void *arg)
* doesn't use PSCI).
*/
for (index = 1; index < drv->state_count; ++index) {
- struct cpuidle_state *state = &drv->states[index];
- bool broadcast = state->flags & CPUIDLE_FLAG_TIMER_STOP;
int ret;
+ struct cpuidle_state *state = &drv->states[index];
/*
* Set the timer to wake this CPU up in some time (which
@@ -318,7 +316,7 @@ static int suspend_test_thread(void *arg)
/* IRQs must be disabled during suspend operations. */
local_irq_disable();
- ret = suspend_cpu(index, broadcast);
+ ret = suspend_cpu(dev, drv, index);
/*
* We have woken up. Re-enable IRQs to handle any
@@ -359,16 +357,16 @@ static int suspend_test_thread(void *arg)
for (;;) {
/* Needs to be set first to avoid missing a wakeup. */
set_current_state(TASK_INTERRUPTIBLE);
- if (kthread_should_stop()) {
- __set_current_state(TASK_RUNNING);
+ if (kthread_should_park())
break;
- }
schedule();
}
pr_info("CPU %d suspend test results: success %d, shallow states %d, errors %d\n",
cpu, nb_suspend, nb_shallow_sleep, nb_err);
+ kthread_parkme();
+
return nb_err;
}
@@ -433,8 +431,10 @@ static int suspend_tests(void)
/* Stop and destroy all threads, get return status. */
- for (i = 0; i < nb_threads; ++i)
+ for (i = 0; i < nb_threads; ++i) {
+ err += kthread_park(threads[i]);
err += kthread_stop(threads[i]);
+ }
out:
cpuidle_resume_and_unlock();
kfree(threads);
diff --git a/drivers/firmware/qcom_scm.c b/drivers/firmware/qcom_scm.c
index 2ddc118dba1b..4802ab170fe5 100644
--- a/drivers/firmware/qcom_scm.c
+++ b/drivers/firmware/qcom_scm.c
@@ -9,6 +9,7 @@
#include <linux/init.h>
#include <linux/cpumask.h>
#include <linux/export.h>
+#include <linux/dma-direct.h>
#include <linux/dma-mapping.h>
#include <linux/module.h>
#include <linux/types.h>
@@ -425,21 +426,23 @@ EXPORT_SYMBOL(qcom_scm_set_remote_state);
* @mem_sz: size of the region.
* @srcvm: vmid for current set of owners, each set bit in
* flag indicate a unique owner
- * @newvm: array having new owners and corrsponding permission
+ * @newvm: array having new owners and corresponding permission
* flags
* @dest_cnt: number of owners in next set.
*
- * Return negative errno on failure, 0 on success, with @srcvm updated.
+ * Return negative errno on failure or 0 on success with @srcvm updated.
*/
int qcom_scm_assign_mem(phys_addr_t mem_addr, size_t mem_sz,
unsigned int *srcvm,
- struct qcom_scm_vmperm *newvm, int dest_cnt)
+ const struct qcom_scm_vmperm *newvm,
+ unsigned int dest_cnt)
{
struct qcom_scm_current_perm_info *destvm;
struct qcom_scm_mem_map_info *mem_to_map;
phys_addr_t mem_to_map_phys;
phys_addr_t dest_phys;
phys_addr_t ptr_phys;
+ dma_addr_t ptr_dma;
size_t mem_to_map_sz;
size_t dest_sz;
size_t src_sz;
@@ -447,52 +450,50 @@ int qcom_scm_assign_mem(phys_addr_t mem_addr, size_t mem_sz,
int next_vm;
__le32 *src;
void *ptr;
- int ret;
- int len;
- int i;
+ int ret, i, b;
+ unsigned long srcvm_bits = *srcvm;
- src_sz = hweight_long(*srcvm) * sizeof(*src);
+ src_sz = hweight_long(srcvm_bits) * sizeof(*src);
mem_to_map_sz = sizeof(*mem_to_map);
dest_sz = dest_cnt * sizeof(*destvm);
ptr_sz = ALIGN(src_sz, SZ_64) + ALIGN(mem_to_map_sz, SZ_64) +
ALIGN(dest_sz, SZ_64);
- ptr = dma_alloc_coherent(__scm->dev, ptr_sz, &ptr_phys, GFP_KERNEL);
+ ptr = dma_alloc_coherent(__scm->dev, ptr_sz, &ptr_dma, GFP_KERNEL);
if (!ptr)
return -ENOMEM;
+ ptr_phys = dma_to_phys(__scm->dev, ptr_dma);
/* Fill source vmid detail */
src = ptr;
- len = hweight_long(*srcvm);
- for (i = 0; i < len; i++) {
- src[i] = cpu_to_le32(ffs(*srcvm) - 1);
- *srcvm ^= 1 << (ffs(*srcvm) - 1);
- }
+ i = 0;
+ for_each_set_bit(b, &srcvm_bits, BITS_PER_LONG)
+ src[i++] = cpu_to_le32(b);
/* Fill details of mem buff to map */
mem_to_map = ptr + ALIGN(src_sz, SZ_64);
mem_to_map_phys = ptr_phys + ALIGN(src_sz, SZ_64);
- mem_to_map[0].mem_addr = cpu_to_le64(mem_addr);
- mem_to_map[0].mem_size = cpu_to_le64(mem_sz);
+ mem_to_map->mem_addr = cpu_to_le64(mem_addr);
+ mem_to_map->mem_size = cpu_to_le64(mem_sz);
next_vm = 0;
/* Fill details of next vmid detail */
destvm = ptr + ALIGN(mem_to_map_sz, SZ_64) + ALIGN(src_sz, SZ_64);
dest_phys = ptr_phys + ALIGN(mem_to_map_sz, SZ_64) + ALIGN(src_sz, SZ_64);
- for (i = 0; i < dest_cnt; i++) {
- destvm[i].vmid = cpu_to_le32(newvm[i].vmid);
- destvm[i].perm = cpu_to_le32(newvm[i].perm);
- destvm[i].ctx = 0;
- destvm[i].ctx_size = 0;
- next_vm |= BIT(newvm[i].vmid);
+ for (i = 0; i < dest_cnt; i++, destvm++, newvm++) {
+ destvm->vmid = cpu_to_le32(newvm->vmid);
+ destvm->perm = cpu_to_le32(newvm->perm);
+ destvm->ctx = 0;
+ destvm->ctx_size = 0;
+ next_vm |= BIT(newvm->vmid);
}
ret = __qcom_scm_assign_mem(__scm->dev, mem_to_map_phys, mem_to_map_sz,
ptr_phys, src_sz, dest_phys, dest_sz);
- dma_free_coherent(__scm->dev, ALIGN(ptr_sz, SZ_64), ptr, ptr_phys);
+ dma_free_coherent(__scm->dev, ptr_sz, ptr, ptr_dma);
if (ret) {
dev_err(__scm->dev,
- "Assign memory protection call failed %d.\n", ret);
+ "Assign memory protection call failed %d\n", ret);
return -EINVAL;
}
diff --git a/drivers/firmware/raspberrypi.c b/drivers/firmware/raspberrypi.c
index 61be15d9df7d..da26a584dca0 100644
--- a/drivers/firmware/raspberrypi.c
+++ b/drivers/firmware/raspberrypi.c
@@ -20,6 +20,7 @@
#define MBOX_CHAN_PROPERTY 8
static struct platform_device *rpi_hwmon;
+static struct platform_device *rpi_clk;
struct rpi_firmware {
struct mbox_client cl;
@@ -207,6 +208,12 @@ rpi_register_hwmon_driver(struct device *dev, struct rpi_firmware *fw)
-1, NULL, 0);
}
+static void rpi_register_clk_driver(struct device *dev)
+{
+ rpi_clk = platform_device_register_data(dev, "raspberrypi-clk",
+ -1, NULL, 0);
+}
+
static int rpi_firmware_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
@@ -234,6 +241,7 @@ static int rpi_firmware_probe(struct platform_device *pdev)
rpi_firmware_print_firmware_revision(fw);
rpi_register_hwmon_driver(dev, fw);
+ rpi_register_clk_driver(dev);
return 0;
}
@@ -254,6 +262,8 @@ static int rpi_firmware_remove(struct platform_device *pdev)
platform_device_unregister(rpi_hwmon);
rpi_hwmon = NULL;
+ platform_device_unregister(rpi_clk);
+ rpi_clk = NULL;
mbox_free_channel(fw->chan);
return 0;
diff --git a/drivers/firmware/stratix10-rsu.c b/drivers/firmware/stratix10-rsu.c
new file mode 100644
index 000000000000..bb008c019920
--- /dev/null
+++ b/drivers/firmware/stratix10-rsu.c
@@ -0,0 +1,451 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018-2019, Intel Corporation
+ */
+
+#include <linux/arm-smccc.h>
+#include <linux/bitfield.h>
+#include <linux/completion.h>
+#include <linux/kobject.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/firmware/intel/stratix10-svc-client.h>
+#include <linux/string.h>
+#include <linux/sysfs.h>
+
+#define RSU_STATE_MASK GENMASK_ULL(31, 0)
+#define RSU_VERSION_MASK GENMASK_ULL(63, 32)
+#define RSU_ERROR_LOCATION_MASK GENMASK_ULL(31, 0)
+#define RSU_ERROR_DETAIL_MASK GENMASK_ULL(63, 32)
+#define RSU_FW_VERSION_MASK GENMASK_ULL(15, 0)
+
+#define RSU_TIMEOUT (msecs_to_jiffies(SVC_RSU_REQUEST_TIMEOUT_MS))
+
+#define INVALID_RETRY_COUNTER 0xFFFFFFFF
+
+typedef void (*rsu_callback)(struct stratix10_svc_client *client,
+ struct stratix10_svc_cb_data *data);
+/**
+ * struct stratix10_rsu_priv - rsu data structure
+ * @chan: pointer to the allocated service channel
+ * @client: active service client
+ * @completion: state for callback completion
+ * @lock: a mutex to protect callback completion state
+ * @status.current_image: address of image currently running in flash
+ * @status.fail_image: address of failed image in flash
+ * @status.version: the version number of RSU firmware
+ * @status.state: the state of RSU system
+ * @status.error_details: error code
+ * @status.error_location: the error offset inside the image that failed
+ * @retry_counter: the current image's retry counter
+ */
+struct stratix10_rsu_priv {
+ struct stratix10_svc_chan *chan;
+ struct stratix10_svc_client client;
+ struct completion completion;
+ struct mutex lock;
+ struct {
+ unsigned long current_image;
+ unsigned long fail_image;
+ unsigned int version;
+ unsigned int state;
+ unsigned int error_details;
+ unsigned int error_location;
+ } status;
+ unsigned int retry_counter;
+};
+
+/**
+ * rsu_status_callback() - Status callback from Intel Service Layer
+ * @client: pointer to service client
+ * @data: pointer to callback data structure
+ *
+ * Callback from Intel service layer for RSU status request. Status is
+ * only updated after a system reboot, so a get updated status call is
+ * made during driver probe.
+ */
+static void rsu_status_callback(struct stratix10_svc_client *client,
+ struct stratix10_svc_cb_data *data)
+{
+ struct stratix10_rsu_priv *priv = client->priv;
+ struct arm_smccc_res *res = (struct arm_smccc_res *)data->kaddr1;
+
+ if (data->status == BIT(SVC_STATUS_RSU_OK)) {
+ priv->status.version = FIELD_GET(RSU_VERSION_MASK,
+ res->a2);
+ priv->status.state = FIELD_GET(RSU_STATE_MASK, res->a2);
+ priv->status.fail_image = res->a1;
+ priv->status.current_image = res->a0;
+ priv->status.error_location =
+ FIELD_GET(RSU_ERROR_LOCATION_MASK, res->a3);
+ priv->status.error_details =
+ FIELD_GET(RSU_ERROR_DETAIL_MASK, res->a3);
+ } else {
+ dev_err(client->dev, "COMMAND_RSU_STATUS returned 0x%lX\n",
+ res->a0);
+ priv->status.version = 0;
+ priv->status.state = 0;
+ priv->status.fail_image = 0;
+ priv->status.current_image = 0;
+ priv->status.error_location = 0;
+ priv->status.error_details = 0;
+ }
+
+ complete(&priv->completion);
+}
+
+/**
+ * rsu_command_callback() - Update callback from Intel Service Layer
+ * @client: pointer to client
+ * @data: pointer to callback data structure
+ *
+ * Callback from Intel service layer for RSU commands.
+ */
+static void rsu_command_callback(struct stratix10_svc_client *client,
+ struct stratix10_svc_cb_data *data)
+{
+ struct stratix10_rsu_priv *priv = client->priv;
+
+ if (data->status != BIT(SVC_STATUS_RSU_OK))
+ dev_err(client->dev, "RSU returned status is %i\n",
+ data->status);
+ complete(&priv->completion);
+}
+
+/**
+ * rsu_retry_callback() - Callback from Intel service layer for getting
+ * the current image's retry counter from firmware
+ * @client: pointer to client
+ * @data: pointer to callback data structure
+ *
+ * Callback from Intel service layer for retry counter, which is used by
+ * user to know how many times the images is still allowed to reload
+ * itself before giving up and starting RSU fail-over flow.
+ */
+static void rsu_retry_callback(struct stratix10_svc_client *client,
+ struct stratix10_svc_cb_data *data)
+{
+ struct stratix10_rsu_priv *priv = client->priv;
+ unsigned int *counter = (unsigned int *)data->kaddr1;
+
+ if (data->status == BIT(SVC_STATUS_RSU_OK))
+ priv->retry_counter = *counter;
+ else
+ dev_err(client->dev, "Failed to get retry counter %i\n",
+ data->status);
+
+ complete(&priv->completion);
+}
+
+/**
+ * rsu_send_msg() - send a message to Intel service layer
+ * @priv: pointer to rsu private data
+ * @command: RSU status or update command
+ * @arg: the request argument, the bitstream address or notify status
+ * @callback: function pointer for the callback (status or update)
+ *
+ * Start an Intel service layer transaction to perform the SMC call that
+ * is necessary to get RSU boot log or set the address of bitstream to
+ * boot after reboot.
+ *
+ * Returns 0 on success or -ETIMEDOUT on error.
+ */
+static int rsu_send_msg(struct stratix10_rsu_priv *priv,
+ enum stratix10_svc_command_code command,
+ unsigned long arg,
+ rsu_callback callback)
+{
+ struct stratix10_svc_client_msg msg;
+ int ret;
+
+ mutex_lock(&priv->lock);
+ reinit_completion(&priv->completion);
+ priv->client.receive_cb = callback;
+
+ msg.command = command;
+ if (arg)
+ msg.arg[0] = arg;
+
+ ret = stratix10_svc_send(priv->chan, &msg);
+ if (ret < 0)
+ goto status_done;
+
+ ret = wait_for_completion_interruptible_timeout(&priv->completion,
+ RSU_TIMEOUT);
+ if (!ret) {
+ dev_err(priv->client.dev,
+ "timeout waiting for SMC call\n");
+ ret = -ETIMEDOUT;
+ goto status_done;
+ } else if (ret < 0) {
+ dev_err(priv->client.dev,
+ "error %d waiting for SMC call\n", ret);
+ goto status_done;
+ } else {
+ ret = 0;
+ }
+
+status_done:
+ stratix10_svc_done(priv->chan);
+ mutex_unlock(&priv->lock);
+ return ret;
+}
+
+/*
+ * This driver exposes some optional features of the Intel Stratix 10 SoC FPGA.
+ * The sysfs interfaces exposed here are FPGA Remote System Update (RSU)
+ * related. They allow user space software to query the configuration system
+ * status and to request optional reboot behavior specific to Intel FPGAs.
+ */
+
+static ssize_t current_image_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct stratix10_rsu_priv *priv = dev_get_drvdata(dev);
+
+ if (!priv)
+ return -ENODEV;
+
+ return sprintf(buf, "0x%08lx\n", priv->status.current_image);
+}
+
+static ssize_t fail_image_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct stratix10_rsu_priv *priv = dev_get_drvdata(dev);
+
+ if (!priv)
+ return -ENODEV;
+
+ return sprintf(buf, "0x%08lx\n", priv->status.fail_image);
+}
+
+static ssize_t version_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct stratix10_rsu_priv *priv = dev_get_drvdata(dev);
+
+ if (!priv)
+ return -ENODEV;
+
+ return sprintf(buf, "0x%08x\n", priv->status.version);
+}
+
+static ssize_t state_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct stratix10_rsu_priv *priv = dev_get_drvdata(dev);
+
+ if (!priv)
+ return -ENODEV;
+
+ return sprintf(buf, "0x%08x\n", priv->status.state);
+}
+
+static ssize_t error_location_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct stratix10_rsu_priv *priv = dev_get_drvdata(dev);
+
+ if (!priv)
+ return -ENODEV;
+
+ return sprintf(buf, "0x%08x\n", priv->status.error_location);
+}
+
+static ssize_t error_details_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct stratix10_rsu_priv *priv = dev_get_drvdata(dev);
+
+ if (!priv)
+ return -ENODEV;
+
+ return sprintf(buf, "0x%08x\n", priv->status.error_details);
+}
+
+static ssize_t retry_counter_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct stratix10_rsu_priv *priv = dev_get_drvdata(dev);
+
+ if (!priv)
+ return -ENODEV;
+
+ return sprintf(buf, "0x%08x\n", priv->retry_counter);
+}
+
+static ssize_t reboot_image_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct stratix10_rsu_priv *priv = dev_get_drvdata(dev);
+ unsigned long address;
+ int ret;
+
+ if (priv == 0)
+ return -ENODEV;
+
+ ret = kstrtoul(buf, 0, &address);
+ if (ret)
+ return ret;
+
+ ret = rsu_send_msg(priv, COMMAND_RSU_UPDATE,
+ address, rsu_command_callback);
+ if (ret) {
+ dev_err(dev, "Error, RSU update returned %i\n", ret);
+ return ret;
+ }
+
+ return count;
+}
+
+static ssize_t notify_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct stratix10_rsu_priv *priv = dev_get_drvdata(dev);
+ unsigned long status;
+ int ret;
+
+ if (priv == 0)
+ return -ENODEV;
+
+ ret = kstrtoul(buf, 0, &status);
+ if (ret)
+ return ret;
+
+ ret = rsu_send_msg(priv, COMMAND_RSU_NOTIFY,
+ status, rsu_command_callback);
+ if (ret) {
+ dev_err(dev, "Error, RSU notify returned %i\n", ret);
+ return ret;
+ }
+
+ /* to get the updated state */
+ ret = rsu_send_msg(priv, COMMAND_RSU_STATUS,
+ 0, rsu_status_callback);
+ if (ret) {
+ dev_err(dev, "Error, getting RSU status %i\n", ret);
+ return ret;
+ }
+
+ /* only 19.3 or late version FW supports retry counter feature */
+ if (FIELD_GET(RSU_FW_VERSION_MASK, priv->status.version)) {
+ ret = rsu_send_msg(priv, COMMAND_RSU_RETRY,
+ 0, rsu_retry_callback);
+ if (ret) {
+ dev_err(dev,
+ "Error, getting RSU retry %i\n", ret);
+ return ret;
+ }
+ }
+
+ return count;
+}
+
+static DEVICE_ATTR_RO(current_image);
+static DEVICE_ATTR_RO(fail_image);
+static DEVICE_ATTR_RO(state);
+static DEVICE_ATTR_RO(version);
+static DEVICE_ATTR_RO(error_location);
+static DEVICE_ATTR_RO(error_details);
+static DEVICE_ATTR_RO(retry_counter);
+static DEVICE_ATTR_WO(reboot_image);
+static DEVICE_ATTR_WO(notify);
+
+static struct attribute *rsu_attrs[] = {
+ &dev_attr_current_image.attr,
+ &dev_attr_fail_image.attr,
+ &dev_attr_state.attr,
+ &dev_attr_version.attr,
+ &dev_attr_error_location.attr,
+ &dev_attr_error_details.attr,
+ &dev_attr_retry_counter.attr,
+ &dev_attr_reboot_image.attr,
+ &dev_attr_notify.attr,
+ NULL
+};
+
+ATTRIBUTE_GROUPS(rsu);
+
+static int stratix10_rsu_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct stratix10_rsu_priv *priv;
+ int ret;
+
+ priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ priv->client.dev = dev;
+ priv->client.receive_cb = NULL;
+ priv->client.priv = priv;
+ priv->status.current_image = 0;
+ priv->status.fail_image = 0;
+ priv->status.error_location = 0;
+ priv->status.error_details = 0;
+ priv->status.version = 0;
+ priv->status.state = 0;
+ priv->retry_counter = INVALID_RETRY_COUNTER;
+
+ mutex_init(&priv->lock);
+ priv->chan = stratix10_svc_request_channel_byname(&priv->client,
+ SVC_CLIENT_RSU);
+ if (IS_ERR(priv->chan)) {
+ dev_err(dev, "couldn't get service channel %s\n",
+ SVC_CLIENT_RSU);
+ return PTR_ERR(priv->chan);
+ }
+
+ init_completion(&priv->completion);
+ platform_set_drvdata(pdev, priv);
+
+ /* get the initial state from firmware */
+ ret = rsu_send_msg(priv, COMMAND_RSU_STATUS,
+ 0, rsu_status_callback);
+ if (ret) {
+ dev_err(dev, "Error, getting RSU status %i\n", ret);
+ stratix10_svc_free_channel(priv->chan);
+ }
+
+ /* only 19.3 or late version FW supports retry counter feature */
+ if (FIELD_GET(RSU_FW_VERSION_MASK, priv->status.version)) {
+ ret = rsu_send_msg(priv, COMMAND_RSU_RETRY, 0,
+ rsu_retry_callback);
+ if (ret) {
+ dev_err(dev,
+ "Error, getting RSU retry %i\n", ret);
+ stratix10_svc_free_channel(priv->chan);
+ }
+ }
+
+ return ret;
+}
+
+static int stratix10_rsu_remove(struct platform_device *pdev)
+{
+ struct stratix10_rsu_priv *priv = platform_get_drvdata(pdev);
+
+ stratix10_svc_free_channel(priv->chan);
+ return 0;
+}
+
+static struct platform_driver stratix10_rsu_driver = {
+ .probe = stratix10_rsu_probe,
+ .remove = stratix10_rsu_remove,
+ .driver = {
+ .name = "stratix10-rsu",
+ .dev_groups = rsu_groups,
+ },
+};
+
+module_platform_driver(stratix10_rsu_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Intel Remote System Update Driver");
+MODULE_AUTHOR("Richard Gong <richard.gong@intel.com>");
diff --git a/drivers/firmware/stratix10-svc.c b/drivers/firmware/stratix10-svc.c
index 6e6514825ad0..b485321189e1 100644
--- a/drivers/firmware/stratix10-svc.c
+++ b/drivers/firmware/stratix10-svc.c
@@ -38,6 +38,9 @@
#define FPGA_CONFIG_DATA_CLAIM_TIMEOUT_MS 200
#define FPGA_CONFIG_STATUS_TIMEOUT_SEC 30
+/* stratix10 service layer clients */
+#define STRATIX10_RSU "stratix10-rsu"
+
typedef void (svc_invoke_fn)(unsigned long, unsigned long, unsigned long,
unsigned long, unsigned long, unsigned long,
unsigned long, unsigned long,
@@ -45,6 +48,14 @@ typedef void (svc_invoke_fn)(unsigned long, unsigned long, unsigned long,
struct stratix10_svc_chan;
/**
+ * struct stratix10_svc - svc private data
+ * @stratix10_svc_rsu: pointer to stratix10 RSU device
+ */
+struct stratix10_svc {
+ struct platform_device *stratix10_svc_rsu;
+};
+
+/**
* struct stratix10_svc_sh_memory - service shared memory structure
* @sync_complete: state for a completion
* @addr: physical address of shared memory block
@@ -296,7 +307,12 @@ static void svc_thread_recv_status_ok(struct stratix10_svc_data *p_data,
cb_data->status = BIT(SVC_STATUS_RECONFIG_COMPLETED);
break;
case COMMAND_RSU_UPDATE:
+ case COMMAND_RSU_NOTIFY:
+ cb_data->status = BIT(SVC_STATUS_RSU_OK);
+ break;
+ case COMMAND_RSU_RETRY:
cb_data->status = BIT(SVC_STATUS_RSU_OK);
+ cb_data->kaddr1 = &res.a1;
break;
default:
pr_warn("it shouldn't happen\n");
@@ -386,6 +402,16 @@ static int svc_normal_to_secure_thread(void *data)
a1 = pdata->arg[0];
a2 = 0;
break;
+ case COMMAND_RSU_NOTIFY:
+ a0 = INTEL_SIP_SMC_RSU_NOTIFY;
+ a1 = pdata->arg[0];
+ a2 = 0;
+ break;
+ case COMMAND_RSU_RETRY:
+ a0 = INTEL_SIP_SMC_RSU_RETRY_COUNTER;
+ a1 = 0;
+ a2 = 0;
+ break;
default:
pr_warn("it shouldn't happen\n");
break;
@@ -438,7 +464,28 @@ static int svc_normal_to_secure_thread(void *data)
pr_debug("%s: STATUS_REJECTED\n", __func__);
break;
case INTEL_SIP_SMC_FPGA_CONFIG_STATUS_ERROR:
+ case INTEL_SIP_SMC_RSU_ERROR:
pr_err("%s: STATUS_ERROR\n", __func__);
+ switch (pdata->command) {
+ /* for FPGA mgr */
+ case COMMAND_RECONFIG_DATA_CLAIM:
+ case COMMAND_RECONFIG:
+ case COMMAND_RECONFIG_DATA_SUBMIT:
+ case COMMAND_RECONFIG_STATUS:
+ cbdata->status =
+ BIT(SVC_STATUS_RECONFIG_ERROR);
+ break;
+
+ /* for RSU */
+ case COMMAND_RSU_STATUS:
+ case COMMAND_RSU_UPDATE:
+ case COMMAND_RSU_NOTIFY:
+ case COMMAND_RSU_RETRY:
+ cbdata->status =
+ BIT(SVC_STATUS_RSU_ERROR);
+ break;
+ }
+
cbdata->status = BIT(SVC_STATUS_RECONFIG_ERROR);
cbdata->kaddr1 = NULL;
cbdata->kaddr2 = NULL;
@@ -530,7 +577,7 @@ static int svc_get_sh_memory(struct platform_device *pdev,
if (!sh_memory->addr || !sh_memory->size) {
dev_err(dev,
- "fails to get shared memory info from secure world\n");
+ "failed to get shared memory info from secure world\n");
return -ENOMEM;
}
@@ -768,7 +815,7 @@ int stratix10_svc_send(struct stratix10_svc_chan *chan, void *msg)
"svc_smc_hvc_thread");
if (IS_ERR(chan->ctrl->task)) {
dev_err(chan->ctrl->dev,
- "fails to create svc_smc_hvc_thread\n");
+ "failed to create svc_smc_hvc_thread\n");
kfree(p_data);
return -EINVAL;
}
@@ -913,6 +960,8 @@ static int stratix10_svc_drv_probe(struct platform_device *pdev)
struct stratix10_svc_chan *chans;
struct gen_pool *genpool;
struct stratix10_svc_sh_memory *sh_memory;
+ struct stratix10_svc *svc;
+
svc_invoke_fn *invoke_fn;
size_t fifo_size;
int ret;
@@ -957,7 +1006,7 @@ static int stratix10_svc_drv_probe(struct platform_device *pdev)
fifo_size = sizeof(struct stratix10_svc_data) * SVC_NUM_DATA_IN_FIFO;
ret = kfifo_alloc(&controller->svc_fifo, fifo_size, GFP_KERNEL);
if (ret) {
- dev_err(dev, "fails to allocate FIFO\n");
+ dev_err(dev, "failed to allocate FIFO\n");
return ret;
}
spin_lock_init(&controller->svc_fifo_lock);
@@ -975,6 +1024,24 @@ static int stratix10_svc_drv_probe(struct platform_device *pdev)
list_add_tail(&controller->node, &svc_ctrl);
platform_set_drvdata(pdev, controller);
+ /* add svc client device(s) */
+ svc = devm_kzalloc(dev, sizeof(*svc), GFP_KERNEL);
+ if (!svc)
+ return -ENOMEM;
+
+ svc->stratix10_svc_rsu = platform_device_alloc(STRATIX10_RSU, 0);
+ if (!svc->stratix10_svc_rsu) {
+ dev_err(dev, "failed to allocate %s device\n", STRATIX10_RSU);
+ return -ENOMEM;
+ }
+
+ ret = platform_device_add(svc->stratix10_svc_rsu);
+ if (ret) {
+ platform_device_put(svc->stratix10_svc_rsu);
+ return ret;
+ }
+ dev_set_drvdata(dev, svc);
+
pr_info("Intel Service Layer Driver Initialized\n");
return ret;
@@ -982,8 +1049,11 @@ static int stratix10_svc_drv_probe(struct platform_device *pdev)
static int stratix10_svc_drv_remove(struct platform_device *pdev)
{
+ struct stratix10_svc *svc = dev_get_drvdata(&pdev->dev);
struct stratix10_svc_controller *ctrl = platform_get_drvdata(pdev);
+ platform_device_unregister(svc->stratix10_svc_rsu);
+
kfifo_free(&ctrl->svc_fifo);
if (ctrl->task) {
kthread_stop(ctrl->task);
diff --git a/drivers/firmware/tegra/bpmp.c b/drivers/firmware/tegra/bpmp.c
index 2418abfe1fb6..19c56133234b 100644
--- a/drivers/firmware/tegra/bpmp.c
+++ b/drivers/firmware/tegra/bpmp.c
@@ -803,7 +803,9 @@ static int __maybe_unused tegra_bpmp_resume(struct device *dev)
return 0;
}
-static SIMPLE_DEV_PM_OPS(tegra_bpmp_pm_ops, NULL, tegra_bpmp_resume);
+static const struct dev_pm_ops tegra_bpmp_pm_ops = {
+ .resume_early = tegra_bpmp_resume,
+};
#if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC)
diff --git a/drivers/firmware/ti_sci.c b/drivers/firmware/ti_sci.c
index ef93406ace1b..4126be9e3216 100644
--- a/drivers/firmware/ti_sci.c
+++ b/drivers/firmware/ti_sci.c
@@ -466,9 +466,9 @@ static int ti_sci_cmd_get_revision(struct ti_sci_info *info)
struct ti_sci_xfer *xfer;
int ret;
- /* No need to setup flags since it is expected to respond */
xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_VERSION,
- 0x0, sizeof(struct ti_sci_msg_hdr),
+ TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+ sizeof(struct ti_sci_msg_hdr),
sizeof(*rev_info));
if (IS_ERR(xfer)) {
ret = PTR_ERR(xfer);
@@ -596,9 +596,9 @@ static int ti_sci_get_device_state(const struct ti_sci_handle *handle,
info = handle_to_ti_sci_info(handle);
dev = info->dev;
- /* Response is expected, so need of any flags */
xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_GET_DEVICE_STATE,
- 0, sizeof(*req), sizeof(*resp));
+ TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+ sizeof(*req), sizeof(*resp));
if (IS_ERR(xfer)) {
ret = PTR_ERR(xfer);
dev_err(dev, "Message alloc failed(%d)\n", ret);
@@ -635,6 +635,7 @@ fail:
/**
* ti_sci_cmd_get_device() - command to request for device managed by TISCI
+ * that can be shared with other hosts.
* @handle: Pointer to TISCI handle as retrieved by *ti_sci_get_handle
* @id: Device Identifier
*
@@ -642,12 +643,30 @@ fail:
* usage count by balancing get_device with put_device. No refcounting is
* managed by driver for that purpose.
*
- * NOTE: The request is for exclusive access for the processor.
- *
* Return: 0 if all went fine, else return appropriate error.
*/
static int ti_sci_cmd_get_device(const struct ti_sci_handle *handle, u32 id)
{
+ return ti_sci_set_device_state(handle, id, 0,
+ MSG_DEVICE_SW_STATE_ON);
+}
+
+/**
+ * ti_sci_cmd_get_device_exclusive() - command to request for device managed by
+ * TISCI that is exclusively owned by the
+ * requesting host.
+ * @handle: Pointer to TISCI handle as retrieved by *ti_sci_get_handle
+ * @id: Device Identifier
+ *
+ * Request for the device - NOTE: the client MUST maintain integrity of
+ * usage count by balancing get_device with put_device. No refcounting is
+ * managed by driver for that purpose.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_get_device_exclusive(const struct ti_sci_handle *handle,
+ u32 id)
+{
return ti_sci_set_device_state(handle, id,
MSG_FLAG_DEVICE_EXCLUSIVE,
MSG_DEVICE_SW_STATE_ON);
@@ -666,6 +685,26 @@ static int ti_sci_cmd_get_device(const struct ti_sci_handle *handle, u32 id)
*/
static int ti_sci_cmd_idle_device(const struct ti_sci_handle *handle, u32 id)
{
+ return ti_sci_set_device_state(handle, id, 0,
+ MSG_DEVICE_SW_STATE_RETENTION);
+}
+
+/**
+ * ti_sci_cmd_idle_device_exclusive() - Command to idle a device managed by
+ * TISCI that is exclusively owned by
+ * requesting host.
+ * @handle: Pointer to TISCI handle as retrieved by *ti_sci_get_handle
+ * @id: Device Identifier
+ *
+ * Request for the device - NOTE: the client MUST maintain integrity of
+ * usage count by balancing get_device with put_device. No refcounting is
+ * managed by driver for that purpose.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_idle_device_exclusive(const struct ti_sci_handle *handle,
+ u32 id)
+{
return ti_sci_set_device_state(handle, id,
MSG_FLAG_DEVICE_EXCLUSIVE,
MSG_DEVICE_SW_STATE_RETENTION);
@@ -916,7 +955,7 @@ static int ti_sci_cmd_get_device_resets(const struct ti_sci_handle *handle,
* Return: 0 if all went well, else returns appropriate error value.
*/
static int ti_sci_set_clock_state(const struct ti_sci_handle *handle,
- u32 dev_id, u8 clk_id,
+ u32 dev_id, u32 clk_id,
u32 flags, u8 state)
{
struct ti_sci_info *info;
@@ -944,7 +983,12 @@ static int ti_sci_set_clock_state(const struct ti_sci_handle *handle,
}
req = (struct ti_sci_msg_req_set_clock_state *)xfer->xfer_buf;
req->dev_id = dev_id;
- req->clk_id = clk_id;
+ if (clk_id < 255) {
+ req->clk_id = clk_id;
+ } else {
+ req->clk_id = 255;
+ req->clk_id_32 = clk_id;
+ }
req->request_state = state;
ret = ti_sci_do_xfer(info, xfer);
@@ -976,7 +1020,7 @@ fail:
* Return: 0 if all went well, else returns appropriate error value.
*/
static int ti_sci_cmd_get_clock_state(const struct ti_sci_handle *handle,
- u32 dev_id, u8 clk_id,
+ u32 dev_id, u32 clk_id,
u8 *programmed_state, u8 *current_state)
{
struct ti_sci_info *info;
@@ -1007,7 +1051,12 @@ static int ti_sci_cmd_get_clock_state(const struct ti_sci_handle *handle,
}
req = (struct ti_sci_msg_req_get_clock_state *)xfer->xfer_buf;
req->dev_id = dev_id;
- req->clk_id = clk_id;
+ if (clk_id < 255) {
+ req->clk_id = clk_id;
+ } else {
+ req->clk_id = 255;
+ req->clk_id_32 = clk_id;
+ }
ret = ti_sci_do_xfer(info, xfer);
if (ret) {
@@ -1047,8 +1096,8 @@ fail:
* Return: 0 if all went well, else returns appropriate error value.
*/
static int ti_sci_cmd_get_clock(const struct ti_sci_handle *handle, u32 dev_id,
- u8 clk_id, bool needs_ssc, bool can_change_freq,
- bool enable_input_term)
+ u32 clk_id, bool needs_ssc,
+ bool can_change_freq, bool enable_input_term)
{
u32 flags = 0;
@@ -1073,7 +1122,7 @@ static int ti_sci_cmd_get_clock(const struct ti_sci_handle *handle, u32 dev_id,
* Return: 0 if all went well, else returns appropriate error value.
*/
static int ti_sci_cmd_idle_clock(const struct ti_sci_handle *handle,
- u32 dev_id, u8 clk_id)
+ u32 dev_id, u32 clk_id)
{
return ti_sci_set_clock_state(handle, dev_id, clk_id, 0,
MSG_CLOCK_SW_STATE_UNREQ);
@@ -1092,7 +1141,7 @@ static int ti_sci_cmd_idle_clock(const struct ti_sci_handle *handle,
* Return: 0 if all went well, else returns appropriate error value.
*/
static int ti_sci_cmd_put_clock(const struct ti_sci_handle *handle,
- u32 dev_id, u8 clk_id)
+ u32 dev_id, u32 clk_id)
{
return ti_sci_set_clock_state(handle, dev_id, clk_id, 0,
MSG_CLOCK_SW_STATE_AUTO);
@@ -1110,7 +1159,7 @@ static int ti_sci_cmd_put_clock(const struct ti_sci_handle *handle,
* Return: 0 if all went well, else returns appropriate error value.
*/
static int ti_sci_cmd_clk_is_auto(const struct ti_sci_handle *handle,
- u32 dev_id, u8 clk_id, bool *req_state)
+ u32 dev_id, u32 clk_id, bool *req_state)
{
u8 state = 0;
int ret;
@@ -1139,7 +1188,7 @@ static int ti_sci_cmd_clk_is_auto(const struct ti_sci_handle *handle,
* Return: 0 if all went well, else returns appropriate error value.
*/
static int ti_sci_cmd_clk_is_on(const struct ti_sci_handle *handle, u32 dev_id,
- u8 clk_id, bool *req_state, bool *curr_state)
+ u32 clk_id, bool *req_state, bool *curr_state)
{
u8 c_state = 0, r_state = 0;
int ret;
@@ -1172,7 +1221,7 @@ static int ti_sci_cmd_clk_is_on(const struct ti_sci_handle *handle, u32 dev_id,
* Return: 0 if all went well, else returns appropriate error value.
*/
static int ti_sci_cmd_clk_is_off(const struct ti_sci_handle *handle, u32 dev_id,
- u8 clk_id, bool *req_state, bool *curr_state)
+ u32 clk_id, bool *req_state, bool *curr_state)
{
u8 c_state = 0, r_state = 0;
int ret;
@@ -1204,7 +1253,7 @@ static int ti_sci_cmd_clk_is_off(const struct ti_sci_handle *handle, u32 dev_id,
* Return: 0 if all went well, else returns appropriate error value.
*/
static int ti_sci_cmd_clk_set_parent(const struct ti_sci_handle *handle,
- u32 dev_id, u8 clk_id, u8 parent_id)
+ u32 dev_id, u32 clk_id, u32 parent_id)
{
struct ti_sci_info *info;
struct ti_sci_msg_req_set_clock_parent *req;
@@ -1231,8 +1280,18 @@ static int ti_sci_cmd_clk_set_parent(const struct ti_sci_handle *handle,
}
req = (struct ti_sci_msg_req_set_clock_parent *)xfer->xfer_buf;
req->dev_id = dev_id;
- req->clk_id = clk_id;
- req->parent_id = parent_id;
+ if (clk_id < 255) {
+ req->clk_id = clk_id;
+ } else {
+ req->clk_id = 255;
+ req->clk_id_32 = clk_id;
+ }
+ if (parent_id < 255) {
+ req->parent_id = parent_id;
+ } else {
+ req->parent_id = 255;
+ req->parent_id_32 = parent_id;
+ }
ret = ti_sci_do_xfer(info, xfer);
if (ret) {
@@ -1262,7 +1321,7 @@ fail:
* Return: 0 if all went well, else returns appropriate error value.
*/
static int ti_sci_cmd_clk_get_parent(const struct ti_sci_handle *handle,
- u32 dev_id, u8 clk_id, u8 *parent_id)
+ u32 dev_id, u32 clk_id, u32 *parent_id)
{
struct ti_sci_info *info;
struct ti_sci_msg_req_get_clock_parent *req;
@@ -1289,7 +1348,12 @@ static int ti_sci_cmd_clk_get_parent(const struct ti_sci_handle *handle,
}
req = (struct ti_sci_msg_req_get_clock_parent *)xfer->xfer_buf;
req->dev_id = dev_id;
- req->clk_id = clk_id;
+ if (clk_id < 255) {
+ req->clk_id = clk_id;
+ } else {
+ req->clk_id = 255;
+ req->clk_id_32 = clk_id;
+ }
ret = ti_sci_do_xfer(info, xfer);
if (ret) {
@@ -1299,10 +1363,14 @@ static int ti_sci_cmd_clk_get_parent(const struct ti_sci_handle *handle,
resp = (struct ti_sci_msg_resp_get_clock_parent *)xfer->xfer_buf;
- if (!ti_sci_is_response_ack(resp))
+ if (!ti_sci_is_response_ack(resp)) {
ret = -ENODEV;
- else
- *parent_id = resp->parent_id;
+ } else {
+ if (resp->parent_id < 255)
+ *parent_id = resp->parent_id;
+ else
+ *parent_id = resp->parent_id_32;
+ }
fail:
ti_sci_put_one_xfer(&info->minfo, xfer);
@@ -1322,8 +1390,8 @@ fail:
* Return: 0 if all went well, else returns appropriate error value.
*/
static int ti_sci_cmd_clk_get_num_parents(const struct ti_sci_handle *handle,
- u32 dev_id, u8 clk_id,
- u8 *num_parents)
+ u32 dev_id, u32 clk_id,
+ u32 *num_parents)
{
struct ti_sci_info *info;
struct ti_sci_msg_req_get_clock_num_parents *req;
@@ -1350,7 +1418,12 @@ static int ti_sci_cmd_clk_get_num_parents(const struct ti_sci_handle *handle,
}
req = (struct ti_sci_msg_req_get_clock_num_parents *)xfer->xfer_buf;
req->dev_id = dev_id;
- req->clk_id = clk_id;
+ if (clk_id < 255) {
+ req->clk_id = clk_id;
+ } else {
+ req->clk_id = 255;
+ req->clk_id_32 = clk_id;
+ }
ret = ti_sci_do_xfer(info, xfer);
if (ret) {
@@ -1360,10 +1433,14 @@ static int ti_sci_cmd_clk_get_num_parents(const struct ti_sci_handle *handle,
resp = (struct ti_sci_msg_resp_get_clock_num_parents *)xfer->xfer_buf;
- if (!ti_sci_is_response_ack(resp))
+ if (!ti_sci_is_response_ack(resp)) {
ret = -ENODEV;
- else
- *num_parents = resp->num_parents;
+ } else {
+ if (resp->num_parents < 255)
+ *num_parents = resp->num_parents;
+ else
+ *num_parents = resp->num_parents_32;
+ }
fail:
ti_sci_put_one_xfer(&info->minfo, xfer);
@@ -1391,7 +1468,7 @@ fail:
* Return: 0 if all went well, else returns appropriate error value.
*/
static int ti_sci_cmd_clk_get_match_freq(const struct ti_sci_handle *handle,
- u32 dev_id, u8 clk_id, u64 min_freq,
+ u32 dev_id, u32 clk_id, u64 min_freq,
u64 target_freq, u64 max_freq,
u64 *match_freq)
{
@@ -1420,7 +1497,12 @@ static int ti_sci_cmd_clk_get_match_freq(const struct ti_sci_handle *handle,
}
req = (struct ti_sci_msg_req_query_clock_freq *)xfer->xfer_buf;
req->dev_id = dev_id;
- req->clk_id = clk_id;
+ if (clk_id < 255) {
+ req->clk_id = clk_id;
+ } else {
+ req->clk_id = 255;
+ req->clk_id_32 = clk_id;
+ }
req->min_freq_hz = min_freq;
req->target_freq_hz = target_freq;
req->max_freq_hz = max_freq;
@@ -1463,7 +1545,7 @@ fail:
* Return: 0 if all went well, else returns appropriate error value.
*/
static int ti_sci_cmd_clk_set_freq(const struct ti_sci_handle *handle,
- u32 dev_id, u8 clk_id, u64 min_freq,
+ u32 dev_id, u32 clk_id, u64 min_freq,
u64 target_freq, u64 max_freq)
{
struct ti_sci_info *info;
@@ -1491,7 +1573,12 @@ static int ti_sci_cmd_clk_set_freq(const struct ti_sci_handle *handle,
}
req = (struct ti_sci_msg_req_set_clock_freq *)xfer->xfer_buf;
req->dev_id = dev_id;
- req->clk_id = clk_id;
+ if (clk_id < 255) {
+ req->clk_id = clk_id;
+ } else {
+ req->clk_id = 255;
+ req->clk_id_32 = clk_id;
+ }
req->min_freq_hz = min_freq;
req->target_freq_hz = target_freq;
req->max_freq_hz = max_freq;
@@ -1524,7 +1611,7 @@ fail:
* Return: 0 if all went well, else returns appropriate error value.
*/
static int ti_sci_cmd_clk_get_freq(const struct ti_sci_handle *handle,
- u32 dev_id, u8 clk_id, u64 *freq)
+ u32 dev_id, u32 clk_id, u64 *freq)
{
struct ti_sci_info *info;
struct ti_sci_msg_req_get_clock_freq *req;
@@ -1551,7 +1638,12 @@ static int ti_sci_cmd_clk_get_freq(const struct ti_sci_handle *handle,
}
req = (struct ti_sci_msg_req_get_clock_freq *)xfer->xfer_buf;
req->dev_id = dev_id;
- req->clk_id = clk_id;
+ if (clk_id < 255) {
+ req->clk_id = clk_id;
+ } else {
+ req->clk_id = 255;
+ req->clk_id_32 = clk_id;
+ }
ret = ti_sci_do_xfer(info, xfer);
if (ret) {
@@ -2004,6 +2096,823 @@ static int ti_sci_cmd_free_event_map(const struct ti_sci_handle *handle,
ia_id, vint, global_event, vint_status_bit, 0);
}
+/**
+ * ti_sci_cmd_ring_config() - configure RA ring
+ * @handle: Pointer to TI SCI handle.
+ * @valid_params: Bitfield defining validity of ring configuration
+ * parameters
+ * @nav_id: Device ID of Navigator Subsystem from which the ring is
+ * allocated
+ * @index: Ring index
+ * @addr_lo: The ring base address lo 32 bits
+ * @addr_hi: The ring base address hi 32 bits
+ * @count: Number of ring elements
+ * @mode: The mode of the ring
+ * @size: The ring element size.
+ * @order_id: Specifies the ring's bus order ID
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ *
+ * See @ti_sci_msg_rm_ring_cfg_req for more info.
+ */
+static int ti_sci_cmd_ring_config(const struct ti_sci_handle *handle,
+ u32 valid_params, u16 nav_id, u16 index,
+ u32 addr_lo, u32 addr_hi, u32 count,
+ u8 mode, u8 size, u8 order_id)
+{
+ struct ti_sci_msg_rm_ring_cfg_req *req;
+ struct ti_sci_msg_hdr *resp;
+ struct ti_sci_xfer *xfer;
+ struct ti_sci_info *info;
+ struct device *dev;
+ int ret = 0;
+
+ if (IS_ERR_OR_NULL(handle))
+ return -EINVAL;
+
+ info = handle_to_ti_sci_info(handle);
+ dev = info->dev;
+
+ xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_RM_RING_CFG,
+ TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+ sizeof(*req), sizeof(*resp));
+ if (IS_ERR(xfer)) {
+ ret = PTR_ERR(xfer);
+ dev_err(dev, "RM_RA:Message config failed(%d)\n", ret);
+ return ret;
+ }
+ req = (struct ti_sci_msg_rm_ring_cfg_req *)xfer->xfer_buf;
+ req->valid_params = valid_params;
+ req->nav_id = nav_id;
+ req->index = index;
+ req->addr_lo = addr_lo;
+ req->addr_hi = addr_hi;
+ req->count = count;
+ req->mode = mode;
+ req->size = size;
+ req->order_id = order_id;
+
+ ret = ti_sci_do_xfer(info, xfer);
+ if (ret) {
+ dev_err(dev, "RM_RA:Mbox config send fail %d\n", ret);
+ goto fail;
+ }
+
+ resp = (struct ti_sci_msg_hdr *)xfer->xfer_buf;
+ ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+ ti_sci_put_one_xfer(&info->minfo, xfer);
+ dev_dbg(dev, "RM_RA:config ring %u ret:%d\n", index, ret);
+ return ret;
+}
+
+/**
+ * ti_sci_cmd_ring_get_config() - get RA ring configuration
+ * @handle: Pointer to TI SCI handle.
+ * @nav_id: Device ID of Navigator Subsystem from which the ring is
+ * allocated
+ * @index: Ring index
+ * @addr_lo: Returns ring's base address lo 32 bits
+ * @addr_hi: Returns ring's base address hi 32 bits
+ * @count: Returns number of ring elements
+ * @mode: Returns mode of the ring
+ * @size: Returns ring element size
+ * @order_id: Returns ring's bus order ID
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ *
+ * See @ti_sci_msg_rm_ring_get_cfg_req for more info.
+ */
+static int ti_sci_cmd_ring_get_config(const struct ti_sci_handle *handle,
+ u32 nav_id, u32 index, u8 *mode,
+ u32 *addr_lo, u32 *addr_hi,
+ u32 *count, u8 *size, u8 *order_id)
+{
+ struct ti_sci_msg_rm_ring_get_cfg_resp *resp;
+ struct ti_sci_msg_rm_ring_get_cfg_req *req;
+ struct ti_sci_xfer *xfer;
+ struct ti_sci_info *info;
+ struct device *dev;
+ int ret = 0;
+
+ if (IS_ERR_OR_NULL(handle))
+ return -EINVAL;
+
+ info = handle_to_ti_sci_info(handle);
+ dev = info->dev;
+
+ xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_RM_RING_GET_CFG,
+ TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+ sizeof(*req), sizeof(*resp));
+ if (IS_ERR(xfer)) {
+ ret = PTR_ERR(xfer);
+ dev_err(dev,
+ "RM_RA:Message get config failed(%d)\n", ret);
+ return ret;
+ }
+ req = (struct ti_sci_msg_rm_ring_get_cfg_req *)xfer->xfer_buf;
+ req->nav_id = nav_id;
+ req->index = index;
+
+ ret = ti_sci_do_xfer(info, xfer);
+ if (ret) {
+ dev_err(dev, "RM_RA:Mbox get config send fail %d\n", ret);
+ goto fail;
+ }
+
+ resp = (struct ti_sci_msg_rm_ring_get_cfg_resp *)xfer->xfer_buf;
+
+ if (!ti_sci_is_response_ack(resp)) {
+ ret = -ENODEV;
+ } else {
+ if (mode)
+ *mode = resp->mode;
+ if (addr_lo)
+ *addr_lo = resp->addr_lo;
+ if (addr_hi)
+ *addr_hi = resp->addr_hi;
+ if (count)
+ *count = resp->count;
+ if (size)
+ *size = resp->size;
+ if (order_id)
+ *order_id = resp->order_id;
+ };
+
+fail:
+ ti_sci_put_one_xfer(&info->minfo, xfer);
+ dev_dbg(dev, "RM_RA:get config ring %u ret:%d\n", index, ret);
+ return ret;
+}
+
+/**
+ * ti_sci_cmd_rm_psil_pair() - Pair PSI-L source to destination thread
+ * @handle: Pointer to TI SCI handle.
+ * @nav_id: Device ID of Navigator Subsystem which should be used for
+ * pairing
+ * @src_thread: Source PSI-L thread ID
+ * @dst_thread: Destination PSI-L thread ID
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_rm_psil_pair(const struct ti_sci_handle *handle,
+ u32 nav_id, u32 src_thread, u32 dst_thread)
+{
+ struct ti_sci_msg_psil_pair *req;
+ struct ti_sci_msg_hdr *resp;
+ struct ti_sci_xfer *xfer;
+ struct ti_sci_info *info;
+ struct device *dev;
+ int ret = 0;
+
+ if (IS_ERR(handle))
+ return PTR_ERR(handle);
+ if (!handle)
+ return -EINVAL;
+
+ info = handle_to_ti_sci_info(handle);
+ dev = info->dev;
+
+ xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_RM_PSIL_PAIR,
+ TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+ sizeof(*req), sizeof(*resp));
+ if (IS_ERR(xfer)) {
+ ret = PTR_ERR(xfer);
+ dev_err(dev, "RM_PSIL:Message reconfig failed(%d)\n", ret);
+ return ret;
+ }
+ req = (struct ti_sci_msg_psil_pair *)xfer->xfer_buf;
+ req->nav_id = nav_id;
+ req->src_thread = src_thread;
+ req->dst_thread = dst_thread;
+
+ ret = ti_sci_do_xfer(info, xfer);
+ if (ret) {
+ dev_err(dev, "RM_PSIL:Mbox send fail %d\n", ret);
+ goto fail;
+ }
+
+ resp = (struct ti_sci_msg_hdr *)xfer->xfer_buf;
+ ret = ti_sci_is_response_ack(resp) ? 0 : -EINVAL;
+
+fail:
+ ti_sci_put_one_xfer(&info->minfo, xfer);
+
+ return ret;
+}
+
+/**
+ * ti_sci_cmd_rm_psil_unpair() - Unpair PSI-L source from destination thread
+ * @handle: Pointer to TI SCI handle.
+ * @nav_id: Device ID of Navigator Subsystem which should be used for
+ * unpairing
+ * @src_thread: Source PSI-L thread ID
+ * @dst_thread: Destination PSI-L thread ID
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_rm_psil_unpair(const struct ti_sci_handle *handle,
+ u32 nav_id, u32 src_thread, u32 dst_thread)
+{
+ struct ti_sci_msg_psil_unpair *req;
+ struct ti_sci_msg_hdr *resp;
+ struct ti_sci_xfer *xfer;
+ struct ti_sci_info *info;
+ struct device *dev;
+ int ret = 0;
+
+ if (IS_ERR(handle))
+ return PTR_ERR(handle);
+ if (!handle)
+ return -EINVAL;
+
+ info = handle_to_ti_sci_info(handle);
+ dev = info->dev;
+
+ xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_RM_PSIL_UNPAIR,
+ TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+ sizeof(*req), sizeof(*resp));
+ if (IS_ERR(xfer)) {
+ ret = PTR_ERR(xfer);
+ dev_err(dev, "RM_PSIL:Message reconfig failed(%d)\n", ret);
+ return ret;
+ }
+ req = (struct ti_sci_msg_psil_unpair *)xfer->xfer_buf;
+ req->nav_id = nav_id;
+ req->src_thread = src_thread;
+ req->dst_thread = dst_thread;
+
+ ret = ti_sci_do_xfer(info, xfer);
+ if (ret) {
+ dev_err(dev, "RM_PSIL:Mbox send fail %d\n", ret);
+ goto fail;
+ }
+
+ resp = (struct ti_sci_msg_hdr *)xfer->xfer_buf;
+ ret = ti_sci_is_response_ack(resp) ? 0 : -EINVAL;
+
+fail:
+ ti_sci_put_one_xfer(&info->minfo, xfer);
+
+ return ret;
+}
+
+/**
+ * ti_sci_cmd_rm_udmap_tx_ch_cfg() - Configure a UDMAP TX channel
+ * @handle: Pointer to TI SCI handle.
+ * @params: Pointer to ti_sci_msg_rm_udmap_tx_ch_cfg TX channel config
+ * structure
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ *
+ * See @ti_sci_msg_rm_udmap_tx_ch_cfg and @ti_sci_msg_rm_udmap_tx_ch_cfg_req for
+ * more info.
+ */
+static int ti_sci_cmd_rm_udmap_tx_ch_cfg(const struct ti_sci_handle *handle,
+ const struct ti_sci_msg_rm_udmap_tx_ch_cfg *params)
+{
+ struct ti_sci_msg_rm_udmap_tx_ch_cfg_req *req;
+ struct ti_sci_msg_hdr *resp;
+ struct ti_sci_xfer *xfer;
+ struct ti_sci_info *info;
+ struct device *dev;
+ int ret = 0;
+
+ if (IS_ERR_OR_NULL(handle))
+ return -EINVAL;
+
+ info = handle_to_ti_sci_info(handle);
+ dev = info->dev;
+
+ xfer = ti_sci_get_one_xfer(info, TISCI_MSG_RM_UDMAP_TX_CH_CFG,
+ TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+ sizeof(*req), sizeof(*resp));
+ if (IS_ERR(xfer)) {
+ ret = PTR_ERR(xfer);
+ dev_err(dev, "Message TX_CH_CFG alloc failed(%d)\n", ret);
+ return ret;
+ }
+ req = (struct ti_sci_msg_rm_udmap_tx_ch_cfg_req *)xfer->xfer_buf;
+ req->valid_params = params->valid_params;
+ req->nav_id = params->nav_id;
+ req->index = params->index;
+ req->tx_pause_on_err = params->tx_pause_on_err;
+ req->tx_filt_einfo = params->tx_filt_einfo;
+ req->tx_filt_pswords = params->tx_filt_pswords;
+ req->tx_atype = params->tx_atype;
+ req->tx_chan_type = params->tx_chan_type;
+ req->tx_supr_tdpkt = params->tx_supr_tdpkt;
+ req->tx_fetch_size = params->tx_fetch_size;
+ req->tx_credit_count = params->tx_credit_count;
+ req->txcq_qnum = params->txcq_qnum;
+ req->tx_priority = params->tx_priority;
+ req->tx_qos = params->tx_qos;
+ req->tx_orderid = params->tx_orderid;
+ req->fdepth = params->fdepth;
+ req->tx_sched_priority = params->tx_sched_priority;
+ req->tx_burst_size = params->tx_burst_size;
+
+ ret = ti_sci_do_xfer(info, xfer);
+ if (ret) {
+ dev_err(dev, "Mbox send TX_CH_CFG fail %d\n", ret);
+ goto fail;
+ }
+
+ resp = (struct ti_sci_msg_hdr *)xfer->xfer_buf;
+ ret = ti_sci_is_response_ack(resp) ? 0 : -EINVAL;
+
+fail:
+ ti_sci_put_one_xfer(&info->minfo, xfer);
+ dev_dbg(dev, "TX_CH_CFG: chn %u ret:%u\n", params->index, ret);
+ return ret;
+}
+
+/**
+ * ti_sci_cmd_rm_udmap_rx_ch_cfg() - Configure a UDMAP RX channel
+ * @handle: Pointer to TI SCI handle.
+ * @params: Pointer to ti_sci_msg_rm_udmap_rx_ch_cfg RX channel config
+ * structure
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ *
+ * See @ti_sci_msg_rm_udmap_rx_ch_cfg and @ti_sci_msg_rm_udmap_rx_ch_cfg_req for
+ * more info.
+ */
+static int ti_sci_cmd_rm_udmap_rx_ch_cfg(const struct ti_sci_handle *handle,
+ const struct ti_sci_msg_rm_udmap_rx_ch_cfg *params)
+{
+ struct ti_sci_msg_rm_udmap_rx_ch_cfg_req *req;
+ struct ti_sci_msg_hdr *resp;
+ struct ti_sci_xfer *xfer;
+ struct ti_sci_info *info;
+ struct device *dev;
+ int ret = 0;
+
+ if (IS_ERR_OR_NULL(handle))
+ return -EINVAL;
+
+ info = handle_to_ti_sci_info(handle);
+ dev = info->dev;
+
+ xfer = ti_sci_get_one_xfer(info, TISCI_MSG_RM_UDMAP_RX_CH_CFG,
+ TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+ sizeof(*req), sizeof(*resp));
+ if (IS_ERR(xfer)) {
+ ret = PTR_ERR(xfer);
+ dev_err(dev, "Message RX_CH_CFG alloc failed(%d)\n", ret);
+ return ret;
+ }
+ req = (struct ti_sci_msg_rm_udmap_rx_ch_cfg_req *)xfer->xfer_buf;
+ req->valid_params = params->valid_params;
+ req->nav_id = params->nav_id;
+ req->index = params->index;
+ req->rx_fetch_size = params->rx_fetch_size;
+ req->rxcq_qnum = params->rxcq_qnum;
+ req->rx_priority = params->rx_priority;
+ req->rx_qos = params->rx_qos;
+ req->rx_orderid = params->rx_orderid;
+ req->rx_sched_priority = params->rx_sched_priority;
+ req->flowid_start = params->flowid_start;
+ req->flowid_cnt = params->flowid_cnt;
+ req->rx_pause_on_err = params->rx_pause_on_err;
+ req->rx_atype = params->rx_atype;
+ req->rx_chan_type = params->rx_chan_type;
+ req->rx_ignore_short = params->rx_ignore_short;
+ req->rx_ignore_long = params->rx_ignore_long;
+ req->rx_burst_size = params->rx_burst_size;
+
+ ret = ti_sci_do_xfer(info, xfer);
+ if (ret) {
+ dev_err(dev, "Mbox send RX_CH_CFG fail %d\n", ret);
+ goto fail;
+ }
+
+ resp = (struct ti_sci_msg_hdr *)xfer->xfer_buf;
+ ret = ti_sci_is_response_ack(resp) ? 0 : -EINVAL;
+
+fail:
+ ti_sci_put_one_xfer(&info->minfo, xfer);
+ dev_dbg(dev, "RX_CH_CFG: chn %u ret:%d\n", params->index, ret);
+ return ret;
+}
+
+/**
+ * ti_sci_cmd_rm_udmap_rx_flow_cfg() - Configure UDMAP RX FLOW
+ * @handle: Pointer to TI SCI handle.
+ * @params: Pointer to ti_sci_msg_rm_udmap_flow_cfg RX FLOW config
+ * structure
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ *
+ * See @ti_sci_msg_rm_udmap_flow_cfg and @ti_sci_msg_rm_udmap_flow_cfg_req for
+ * more info.
+ */
+static int ti_sci_cmd_rm_udmap_rx_flow_cfg(const struct ti_sci_handle *handle,
+ const struct ti_sci_msg_rm_udmap_flow_cfg *params)
+{
+ struct ti_sci_msg_rm_udmap_flow_cfg_req *req;
+ struct ti_sci_msg_hdr *resp;
+ struct ti_sci_xfer *xfer;
+ struct ti_sci_info *info;
+ struct device *dev;
+ int ret = 0;
+
+ if (IS_ERR_OR_NULL(handle))
+ return -EINVAL;
+
+ info = handle_to_ti_sci_info(handle);
+ dev = info->dev;
+
+ xfer = ti_sci_get_one_xfer(info, TISCI_MSG_RM_UDMAP_FLOW_CFG,
+ TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+ sizeof(*req), sizeof(*resp));
+ if (IS_ERR(xfer)) {
+ ret = PTR_ERR(xfer);
+ dev_err(dev, "RX_FL_CFG: Message alloc failed(%d)\n", ret);
+ return ret;
+ }
+ req = (struct ti_sci_msg_rm_udmap_flow_cfg_req *)xfer->xfer_buf;
+ req->valid_params = params->valid_params;
+ req->nav_id = params->nav_id;
+ req->flow_index = params->flow_index;
+ req->rx_einfo_present = params->rx_einfo_present;
+ req->rx_psinfo_present = params->rx_psinfo_present;
+ req->rx_error_handling = params->rx_error_handling;
+ req->rx_desc_type = params->rx_desc_type;
+ req->rx_sop_offset = params->rx_sop_offset;
+ req->rx_dest_qnum = params->rx_dest_qnum;
+ req->rx_src_tag_hi = params->rx_src_tag_hi;
+ req->rx_src_tag_lo = params->rx_src_tag_lo;
+ req->rx_dest_tag_hi = params->rx_dest_tag_hi;
+ req->rx_dest_tag_lo = params->rx_dest_tag_lo;
+ req->rx_src_tag_hi_sel = params->rx_src_tag_hi_sel;
+ req->rx_src_tag_lo_sel = params->rx_src_tag_lo_sel;
+ req->rx_dest_tag_hi_sel = params->rx_dest_tag_hi_sel;
+ req->rx_dest_tag_lo_sel = params->rx_dest_tag_lo_sel;
+ req->rx_fdq0_sz0_qnum = params->rx_fdq0_sz0_qnum;
+ req->rx_fdq1_qnum = params->rx_fdq1_qnum;
+ req->rx_fdq2_qnum = params->rx_fdq2_qnum;
+ req->rx_fdq3_qnum = params->rx_fdq3_qnum;
+ req->rx_ps_location = params->rx_ps_location;
+
+ ret = ti_sci_do_xfer(info, xfer);
+ if (ret) {
+ dev_err(dev, "RX_FL_CFG: Mbox send fail %d\n", ret);
+ goto fail;
+ }
+
+ resp = (struct ti_sci_msg_hdr *)xfer->xfer_buf;
+ ret = ti_sci_is_response_ack(resp) ? 0 : -EINVAL;
+
+fail:
+ ti_sci_put_one_xfer(&info->minfo, xfer);
+ dev_dbg(info->dev, "RX_FL_CFG: %u ret:%d\n", params->flow_index, ret);
+ return ret;
+}
+
+/**
+ * ti_sci_cmd_proc_request() - Command to request a physical processor control
+ * @handle: Pointer to TI SCI handle
+ * @proc_id: Processor ID this request is for
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_proc_request(const struct ti_sci_handle *handle,
+ u8 proc_id)
+{
+ struct ti_sci_msg_req_proc_request *req;
+ struct ti_sci_msg_hdr *resp;
+ struct ti_sci_info *info;
+ struct ti_sci_xfer *xfer;
+ struct device *dev;
+ int ret = 0;
+
+ if (!handle)
+ return -EINVAL;
+ if (IS_ERR(handle))
+ return PTR_ERR(handle);
+
+ info = handle_to_ti_sci_info(handle);
+ dev = info->dev;
+
+ xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_PROC_REQUEST,
+ TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+ sizeof(*req), sizeof(*resp));
+ if (IS_ERR(xfer)) {
+ ret = PTR_ERR(xfer);
+ dev_err(dev, "Message alloc failed(%d)\n", ret);
+ return ret;
+ }
+ req = (struct ti_sci_msg_req_proc_request *)xfer->xfer_buf;
+ req->processor_id = proc_id;
+
+ ret = ti_sci_do_xfer(info, xfer);
+ if (ret) {
+ dev_err(dev, "Mbox send fail %d\n", ret);
+ goto fail;
+ }
+
+ resp = (struct ti_sci_msg_hdr *)xfer->tx_message.buf;
+
+ ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+ ti_sci_put_one_xfer(&info->minfo, xfer);
+
+ return ret;
+}
+
+/**
+ * ti_sci_cmd_proc_release() - Command to release a physical processor control
+ * @handle: Pointer to TI SCI handle
+ * @proc_id: Processor ID this request is for
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_proc_release(const struct ti_sci_handle *handle,
+ u8 proc_id)
+{
+ struct ti_sci_msg_req_proc_release *req;
+ struct ti_sci_msg_hdr *resp;
+ struct ti_sci_info *info;
+ struct ti_sci_xfer *xfer;
+ struct device *dev;
+ int ret = 0;
+
+ if (!handle)
+ return -EINVAL;
+ if (IS_ERR(handle))
+ return PTR_ERR(handle);
+
+ info = handle_to_ti_sci_info(handle);
+ dev = info->dev;
+
+ xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_PROC_RELEASE,
+ TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+ sizeof(*req), sizeof(*resp));
+ if (IS_ERR(xfer)) {
+ ret = PTR_ERR(xfer);
+ dev_err(dev, "Message alloc failed(%d)\n", ret);
+ return ret;
+ }
+ req = (struct ti_sci_msg_req_proc_release *)xfer->xfer_buf;
+ req->processor_id = proc_id;
+
+ ret = ti_sci_do_xfer(info, xfer);
+ if (ret) {
+ dev_err(dev, "Mbox send fail %d\n", ret);
+ goto fail;
+ }
+
+ resp = (struct ti_sci_msg_hdr *)xfer->tx_message.buf;
+
+ ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+ ti_sci_put_one_xfer(&info->minfo, xfer);
+
+ return ret;
+}
+
+/**
+ * ti_sci_cmd_proc_handover() - Command to handover a physical processor
+ * control to a host in the processor's access
+ * control list.
+ * @handle: Pointer to TI SCI handle
+ * @proc_id: Processor ID this request is for
+ * @host_id: Host ID to get the control of the processor
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_proc_handover(const struct ti_sci_handle *handle,
+ u8 proc_id, u8 host_id)
+{
+ struct ti_sci_msg_req_proc_handover *req;
+ struct ti_sci_msg_hdr *resp;
+ struct ti_sci_info *info;
+ struct ti_sci_xfer *xfer;
+ struct device *dev;
+ int ret = 0;
+
+ if (!handle)
+ return -EINVAL;
+ if (IS_ERR(handle))
+ return PTR_ERR(handle);
+
+ info = handle_to_ti_sci_info(handle);
+ dev = info->dev;
+
+ xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_PROC_HANDOVER,
+ TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+ sizeof(*req), sizeof(*resp));
+ if (IS_ERR(xfer)) {
+ ret = PTR_ERR(xfer);
+ dev_err(dev, "Message alloc failed(%d)\n", ret);
+ return ret;
+ }
+ req = (struct ti_sci_msg_req_proc_handover *)xfer->xfer_buf;
+ req->processor_id = proc_id;
+ req->host_id = host_id;
+
+ ret = ti_sci_do_xfer(info, xfer);
+ if (ret) {
+ dev_err(dev, "Mbox send fail %d\n", ret);
+ goto fail;
+ }
+
+ resp = (struct ti_sci_msg_hdr *)xfer->tx_message.buf;
+
+ ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+ ti_sci_put_one_xfer(&info->minfo, xfer);
+
+ return ret;
+}
+
+/**
+ * ti_sci_cmd_proc_set_config() - Command to set the processor boot
+ * configuration flags
+ * @handle: Pointer to TI SCI handle
+ * @proc_id: Processor ID this request is for
+ * @config_flags_set: Configuration flags to be set
+ * @config_flags_clear: Configuration flags to be cleared.
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_proc_set_config(const struct ti_sci_handle *handle,
+ u8 proc_id, u64 bootvector,
+ u32 config_flags_set,
+ u32 config_flags_clear)
+{
+ struct ti_sci_msg_req_set_config *req;
+ struct ti_sci_msg_hdr *resp;
+ struct ti_sci_info *info;
+ struct ti_sci_xfer *xfer;
+ struct device *dev;
+ int ret = 0;
+
+ if (!handle)
+ return -EINVAL;
+ if (IS_ERR(handle))
+ return PTR_ERR(handle);
+
+ info = handle_to_ti_sci_info(handle);
+ dev = info->dev;
+
+ xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_SET_CONFIG,
+ TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+ sizeof(*req), sizeof(*resp));
+ if (IS_ERR(xfer)) {
+ ret = PTR_ERR(xfer);
+ dev_err(dev, "Message alloc failed(%d)\n", ret);
+ return ret;
+ }
+ req = (struct ti_sci_msg_req_set_config *)xfer->xfer_buf;
+ req->processor_id = proc_id;
+ req->bootvector_low = bootvector & TI_SCI_ADDR_LOW_MASK;
+ req->bootvector_high = (bootvector & TI_SCI_ADDR_HIGH_MASK) >>
+ TI_SCI_ADDR_HIGH_SHIFT;
+ req->config_flags_set = config_flags_set;
+ req->config_flags_clear = config_flags_clear;
+
+ ret = ti_sci_do_xfer(info, xfer);
+ if (ret) {
+ dev_err(dev, "Mbox send fail %d\n", ret);
+ goto fail;
+ }
+
+ resp = (struct ti_sci_msg_hdr *)xfer->tx_message.buf;
+
+ ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+ ti_sci_put_one_xfer(&info->minfo, xfer);
+
+ return ret;
+}
+
+/**
+ * ti_sci_cmd_proc_set_control() - Command to set the processor boot
+ * control flags
+ * @handle: Pointer to TI SCI handle
+ * @proc_id: Processor ID this request is for
+ * @control_flags_set: Control flags to be set
+ * @control_flags_clear: Control flags to be cleared
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_proc_set_control(const struct ti_sci_handle *handle,
+ u8 proc_id, u32 control_flags_set,
+ u32 control_flags_clear)
+{
+ struct ti_sci_msg_req_set_ctrl *req;
+ struct ti_sci_msg_hdr *resp;
+ struct ti_sci_info *info;
+ struct ti_sci_xfer *xfer;
+ struct device *dev;
+ int ret = 0;
+
+ if (!handle)
+ return -EINVAL;
+ if (IS_ERR(handle))
+ return PTR_ERR(handle);
+
+ info = handle_to_ti_sci_info(handle);
+ dev = info->dev;
+
+ xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_SET_CTRL,
+ TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+ sizeof(*req), sizeof(*resp));
+ if (IS_ERR(xfer)) {
+ ret = PTR_ERR(xfer);
+ dev_err(dev, "Message alloc failed(%d)\n", ret);
+ return ret;
+ }
+ req = (struct ti_sci_msg_req_set_ctrl *)xfer->xfer_buf;
+ req->processor_id = proc_id;
+ req->control_flags_set = control_flags_set;
+ req->control_flags_clear = control_flags_clear;
+
+ ret = ti_sci_do_xfer(info, xfer);
+ if (ret) {
+ dev_err(dev, "Mbox send fail %d\n", ret);
+ goto fail;
+ }
+
+ resp = (struct ti_sci_msg_hdr *)xfer->tx_message.buf;
+
+ ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+ ti_sci_put_one_xfer(&info->minfo, xfer);
+
+ return ret;
+}
+
+/**
+ * ti_sci_cmd_get_boot_status() - Command to get the processor boot status
+ * @handle: Pointer to TI SCI handle
+ * @proc_id: Processor ID this request is for
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_proc_get_status(const struct ti_sci_handle *handle,
+ u8 proc_id, u64 *bv, u32 *cfg_flags,
+ u32 *ctrl_flags, u32 *sts_flags)
+{
+ struct ti_sci_msg_resp_get_status *resp;
+ struct ti_sci_msg_req_get_status *req;
+ struct ti_sci_info *info;
+ struct ti_sci_xfer *xfer;
+ struct device *dev;
+ int ret = 0;
+
+ if (!handle)
+ return -EINVAL;
+ if (IS_ERR(handle))
+ return PTR_ERR(handle);
+
+ info = handle_to_ti_sci_info(handle);
+ dev = info->dev;
+
+ xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_GET_STATUS,
+ TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+ sizeof(*req), sizeof(*resp));
+ if (IS_ERR(xfer)) {
+ ret = PTR_ERR(xfer);
+ dev_err(dev, "Message alloc failed(%d)\n", ret);
+ return ret;
+ }
+ req = (struct ti_sci_msg_req_get_status *)xfer->xfer_buf;
+ req->processor_id = proc_id;
+
+ ret = ti_sci_do_xfer(info, xfer);
+ if (ret) {
+ dev_err(dev, "Mbox send fail %d\n", ret);
+ goto fail;
+ }
+
+ resp = (struct ti_sci_msg_resp_get_status *)xfer->tx_message.buf;
+
+ if (!ti_sci_is_response_ack(resp)) {
+ ret = -ENODEV;
+ } else {
+ *bv = (resp->bootvector_low & TI_SCI_ADDR_LOW_MASK) |
+ (((u64)resp->bootvector_high << TI_SCI_ADDR_HIGH_SHIFT) &
+ TI_SCI_ADDR_HIGH_MASK);
+ *cfg_flags = resp->config_flags;
+ *ctrl_flags = resp->control_flags;
+ *sts_flags = resp->status_flags;
+ }
+
+fail:
+ ti_sci_put_one_xfer(&info->minfo, xfer);
+
+ return ret;
+}
+
/*
* ti_sci_setup_ops() - Setup the operations structures
* @info: pointer to TISCI pointer
@@ -2016,11 +2925,17 @@ static void ti_sci_setup_ops(struct ti_sci_info *info)
struct ti_sci_clk_ops *cops = &ops->clk_ops;
struct ti_sci_rm_core_ops *rm_core_ops = &ops->rm_core_ops;
struct ti_sci_rm_irq_ops *iops = &ops->rm_irq_ops;
+ struct ti_sci_rm_ringacc_ops *rops = &ops->rm_ring_ops;
+ struct ti_sci_rm_psil_ops *psilops = &ops->rm_psil_ops;
+ struct ti_sci_rm_udmap_ops *udmap_ops = &ops->rm_udmap_ops;
+ struct ti_sci_proc_ops *pops = &ops->proc_ops;
core_ops->reboot_device = ti_sci_cmd_core_reboot;
dops->get_device = ti_sci_cmd_get_device;
+ dops->get_device_exclusive = ti_sci_cmd_get_device_exclusive;
dops->idle_device = ti_sci_cmd_idle_device;
+ dops->idle_device_exclusive = ti_sci_cmd_idle_device_exclusive;
dops->put_device = ti_sci_cmd_put_device;
dops->is_valid = ti_sci_cmd_dev_is_valid;
@@ -2055,6 +2970,23 @@ static void ti_sci_setup_ops(struct ti_sci_info *info)
iops->set_event_map = ti_sci_cmd_set_event_map;
iops->free_irq = ti_sci_cmd_free_irq;
iops->free_event_map = ti_sci_cmd_free_event_map;
+
+ rops->config = ti_sci_cmd_ring_config;
+ rops->get_config = ti_sci_cmd_ring_get_config;
+
+ psilops->pair = ti_sci_cmd_rm_psil_pair;
+ psilops->unpair = ti_sci_cmd_rm_psil_unpair;
+
+ udmap_ops->tx_ch_cfg = ti_sci_cmd_rm_udmap_tx_ch_cfg;
+ udmap_ops->rx_ch_cfg = ti_sci_cmd_rm_udmap_rx_ch_cfg;
+ udmap_ops->rx_flow_cfg = ti_sci_cmd_rm_udmap_rx_flow_cfg;
+
+ pops->request = ti_sci_cmd_proc_request;
+ pops->release = ti_sci_cmd_proc_release;
+ pops->handover = ti_sci_cmd_proc_handover;
+ pops->set_config = ti_sci_cmd_proc_set_config;
+ pops->set_control = ti_sci_cmd_proc_set_control;
+ pops->get_status = ti_sci_cmd_proc_get_status;
}
/**
@@ -2342,6 +3274,7 @@ devm_ti_sci_get_of_resource(const struct ti_sci_handle *handle,
struct device *dev, u32 dev_id, char *of_prop)
{
struct ti_sci_resource *res;
+ bool valid_set = false;
u32 resource_subtype;
int i, ret;
@@ -2349,12 +3282,13 @@ devm_ti_sci_get_of_resource(const struct ti_sci_handle *handle,
if (!res)
return ERR_PTR(-ENOMEM);
- res->sets = of_property_count_elems_of_size(dev_of_node(dev), of_prop,
- sizeof(u32));
- if (res->sets < 0) {
+ ret = of_property_count_elems_of_size(dev_of_node(dev), of_prop,
+ sizeof(u32));
+ if (ret < 0) {
dev_err(dev, "%s resource type ids not available\n", of_prop);
- return ERR_PTR(res->sets);
+ return ERR_PTR(ret);
}
+ res->sets = ret;
res->desc = devm_kcalloc(dev, res->sets, sizeof(*res->desc),
GFP_KERNEL);
@@ -2372,15 +3306,18 @@ devm_ti_sci_get_of_resource(const struct ti_sci_handle *handle,
&res->desc[i].start,
&res->desc[i].num);
if (ret) {
- dev_err(dev, "dev = %d subtype %d not allocated for this host\n",
+ dev_dbg(dev, "dev = %d subtype %d not allocated for this host\n",
dev_id, resource_subtype);
- return ERR_PTR(ret);
+ res->desc[i].start = 0;
+ res->desc[i].num = 0;
+ continue;
}
dev_dbg(dev, "dev = %d, subtype = %d, start = %d, num = %d\n",
dev_id, resource_subtype, res->desc[i].start,
res->desc[i].num);
+ valid_set = true;
res->desc[i].res_map =
devm_kzalloc(dev, BITS_TO_LONGS(res->desc[i].num) *
sizeof(*res->desc[i].res_map), GFP_KERNEL);
@@ -2389,7 +3326,10 @@ devm_ti_sci_get_of_resource(const struct ti_sci_handle *handle,
}
raw_spin_lock_init(&res->lock);
- return res;
+ if (valid_set)
+ return res;
+
+ return ERR_PTR(-EINVAL);
}
static int tisci_reboot_handler(struct notifier_block *nb, unsigned long mode,
diff --git a/drivers/firmware/ti_sci.h b/drivers/firmware/ti_sci.h
index 4983827151bf..f0d068c03944 100644
--- a/drivers/firmware/ti_sci.h
+++ b/drivers/firmware/ti_sci.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: BSD-3-Clause
+/* SPDX-License-Identifier: BSD-3-Clause */
/*
* Texas Instruments System Control Interface (TISCI) Protocol
*
@@ -42,6 +42,43 @@
#define TI_SCI_MSG_SET_IRQ 0x1000
#define TI_SCI_MSG_FREE_IRQ 0x1001
+/* NAVSS resource management */
+/* Ringacc requests */
+#define TI_SCI_MSG_RM_RING_ALLOCATE 0x1100
+#define TI_SCI_MSG_RM_RING_FREE 0x1101
+#define TI_SCI_MSG_RM_RING_RECONFIG 0x1102
+#define TI_SCI_MSG_RM_RING_RESET 0x1103
+#define TI_SCI_MSG_RM_RING_CFG 0x1110
+#define TI_SCI_MSG_RM_RING_GET_CFG 0x1111
+
+/* PSI-L requests */
+#define TI_SCI_MSG_RM_PSIL_PAIR 0x1280
+#define TI_SCI_MSG_RM_PSIL_UNPAIR 0x1281
+
+#define TI_SCI_MSG_RM_UDMAP_TX_ALLOC 0x1200
+#define TI_SCI_MSG_RM_UDMAP_TX_FREE 0x1201
+#define TI_SCI_MSG_RM_UDMAP_RX_ALLOC 0x1210
+#define TI_SCI_MSG_RM_UDMAP_RX_FREE 0x1211
+#define TI_SCI_MSG_RM_UDMAP_FLOW_CFG 0x1220
+#define TI_SCI_MSG_RM_UDMAP_OPT_FLOW_CFG 0x1221
+
+#define TISCI_MSG_RM_UDMAP_TX_CH_CFG 0x1205
+#define TISCI_MSG_RM_UDMAP_TX_CH_GET_CFG 0x1206
+#define TISCI_MSG_RM_UDMAP_RX_CH_CFG 0x1215
+#define TISCI_MSG_RM_UDMAP_RX_CH_GET_CFG 0x1216
+#define TISCI_MSG_RM_UDMAP_FLOW_CFG 0x1230
+#define TISCI_MSG_RM_UDMAP_FLOW_SIZE_THRESH_CFG 0x1231
+#define TISCI_MSG_RM_UDMAP_FLOW_GET_CFG 0x1232
+#define TISCI_MSG_RM_UDMAP_FLOW_SIZE_THRESH_GET_CFG 0x1233
+
+/* Processor Control requests */
+#define TI_SCI_MSG_PROC_REQUEST 0xc000
+#define TI_SCI_MSG_PROC_RELEASE 0xc001
+#define TI_SCI_MSG_PROC_HANDOVER 0xc005
+#define TI_SCI_MSG_SET_CONFIG 0xc100
+#define TI_SCI_MSG_SET_CTRL 0xc101
+#define TI_SCI_MSG_GET_STATUS 0xc400
+
/**
* struct ti_sci_msg_hdr - Generic Message Header for All messages and responses
* @type: Type of messages: One of TI_SCI_MSG* values
@@ -202,7 +239,8 @@ struct ti_sci_msg_req_set_device_resets {
* @dev_id: Device identifier this request is for
* @clk_id: Clock identifier for the device for this request.
* Each device has it's own set of clock inputs. This indexes
- * which clock input to modify.
+ * which clock input to modify. Set to 255 if clock ID is
+ * greater than or equal to 255.
* @request_state: Request the state for the clock to be set to.
* MSG_CLOCK_SW_STATE_UNREQ: The IP does not require this clock,
* it can be disabled, regardless of the state of the device
@@ -213,6 +251,9 @@ struct ti_sci_msg_req_set_device_resets {
* being required by the device.(default)
* MSG_CLOCK_SW_STATE_REQ: Configure the clock to be enabled,
* regardless of the state of the device.
+ * @clk_id_32: Clock identifier for the device for this request.
+ * Only to be used if the clock ID is greater than or equal to
+ * 255.
*
* Normally, all required clocks are managed by TISCI entity, this is used
* only for specific control *IF* required. Auto managed state is
@@ -234,6 +275,7 @@ struct ti_sci_msg_req_set_clock_state {
#define MSG_CLOCK_SW_STATE_AUTO 1
#define MSG_CLOCK_SW_STATE_REQ 2
u8 request_state;
+ u32 clk_id_32;
} __packed;
/**
@@ -242,7 +284,11 @@ struct ti_sci_msg_req_set_clock_state {
* @dev_id: Device identifier this request is for
* @clk_id: Clock identifier for the device for this request.
* Each device has it's own set of clock inputs. This indexes
- * which clock input to get state of.
+ * which clock input to get state of. Set to 255 if the clock
+ * ID is greater than or equal to 255.
+ * @clk_id_32: Clock identifier for the device for the request.
+ * Only to be used if the clock ID is greater than or equal to
+ * 255.
*
* Request type is TI_SCI_MSG_GET_CLOCK_STATE, response is state
* of the clock
@@ -251,6 +297,7 @@ struct ti_sci_msg_req_get_clock_state {
struct ti_sci_msg_hdr hdr;
u32 dev_id;
u8 clk_id;
+ u32 clk_id_32;
} __packed;
/**
@@ -278,9 +325,13 @@ struct ti_sci_msg_resp_get_clock_state {
* @dev_id: Device identifier this request is for
* @clk_id: Clock identifier for the device for this request.
* Each device has it's own set of clock inputs. This indexes
- * which clock input to modify.
+ * which clock input to modify. Set to 255 if clock ID is
+ * greater than or equal to 255.
* @parent_id: The new clock parent is selectable by an index via this
- * parameter.
+ * parameter. Set to 255 if clock ID is greater than or
+ * equal to 255.
+ * @clk_id_32: Clock identifier if @clk_id field is 255.
+ * @parent_id_32: Parent identifier if @parent_id is 255.
*
* Request type is TI_SCI_MSG_SET_CLOCK_PARENT, response is generic
* ACK / NACK message.
@@ -290,6 +341,8 @@ struct ti_sci_msg_req_set_clock_parent {
u32 dev_id;
u8 clk_id;
u8 parent_id;
+ u32 clk_id_32;
+ u32 parent_id_32;
} __packed;
/**
@@ -298,7 +351,10 @@ struct ti_sci_msg_req_set_clock_parent {
* @dev_id: Device identifier this request is for
* @clk_id: Clock identifier for the device for this request.
* Each device has it's own set of clock inputs. This indexes
- * which clock input to get the parent for.
+ * which clock input to get the parent for. If this field
+ * contains 255, the actual clock identifier is stored in
+ * @clk_id_32.
+ * @clk_id_32: Clock identifier if the @clk_id field contains 255.
*
* Request type is TI_SCI_MSG_GET_CLOCK_PARENT, response is parent information
*/
@@ -306,25 +362,32 @@ struct ti_sci_msg_req_get_clock_parent {
struct ti_sci_msg_hdr hdr;
u32 dev_id;
u8 clk_id;
+ u32 clk_id_32;
} __packed;
/**
* struct ti_sci_msg_resp_get_clock_parent - Response with clock parent
* @hdr: Generic Header
- * @parent_id: The current clock parent
+ * @parent_id: The current clock parent. If set to 255, the current parent
+ * ID can be found from the @parent_id_32 field.
+ * @parent_id_32: Current clock parent if @parent_id field is set to
+ * 255.
*
* Response to TI_SCI_MSG_GET_CLOCK_PARENT.
*/
struct ti_sci_msg_resp_get_clock_parent {
struct ti_sci_msg_hdr hdr;
u8 parent_id;
+ u32 parent_id_32;
} __packed;
/**
* struct ti_sci_msg_req_get_clock_num_parents - Request to get clock parents
* @hdr: Generic header
* @dev_id: Device identifier this request is for
- * @clk_id: Clock identifier for the device for this request.
+ * @clk_id: Clock identifier for the device for this request. Set to
+ * 255 if clock ID is greater than or equal to 255.
+ * @clk_id_32: Clock identifier if the @clk_id field contains 255.
*
* This request provides information about how many clock parent options
* are available for a given clock to a device. This is typically used
@@ -337,18 +400,24 @@ struct ti_sci_msg_req_get_clock_num_parents {
struct ti_sci_msg_hdr hdr;
u32 dev_id;
u8 clk_id;
+ u32 clk_id_32;
} __packed;
/**
* struct ti_sci_msg_resp_get_clock_num_parents - Response for get clk parents
* @hdr: Generic header
- * @num_parents: Number of clock parents
+ * @num_parents: Number of clock parents. If set to 255, the actual
+ * number of parents is stored into @num_parents_32
+ * field instead.
+ * @num_parents_32: Number of clock parents if @num_parents field is
+ * set to 255.
*
* Response to TI_SCI_MSG_GET_NUM_CLOCK_PARENTS
*/
struct ti_sci_msg_resp_get_clock_num_parents {
struct ti_sci_msg_hdr hdr;
u8 num_parents;
+ u32 num_parents_32;
} __packed;
/**
@@ -363,7 +432,9 @@ struct ti_sci_msg_resp_get_clock_num_parents {
* @max_freq_hz: The maximum allowable frequency in Hz. This is the maximum
* allowable programmed frequency and does not account for clock
* tolerances and jitter.
- * @clk_id: Clock identifier for the device for this request.
+ * @clk_id: Clock identifier for the device for this request. Set to
+ * 255 if clock identifier is greater than or equal to 255.
+ * @clk_id_32: Clock identifier if @clk_id is set to 255.
*
* NOTE: Normally clock frequency management is automatically done by TISCI
* entity. In case of specific requests, TISCI evaluates capability to achieve
@@ -380,6 +451,7 @@ struct ti_sci_msg_req_query_clock_freq {
u64 target_freq_hz;
u64 max_freq_hz;
u8 clk_id;
+ u32 clk_id_32;
} __packed;
/**
@@ -407,7 +479,9 @@ struct ti_sci_msg_resp_query_clock_freq {
* @max_freq_hz: The maximum allowable frequency in Hz. This is the maximum
* allowable programmed frequency and does not account for clock
* tolerances and jitter.
- * @clk_id: Clock identifier for the device for this request.
+ * @clk_id: Clock identifier for the device for this request. Set to
+ * 255 if clock ID is greater than or equal to 255.
+ * @clk_id_32: Clock identifier if @clk_id field is set to 255.
*
* NOTE: Normally clock frequency management is automatically done by TISCI
* entity. In case of specific requests, TISCI evaluates capability to achieve
@@ -436,13 +510,16 @@ struct ti_sci_msg_req_set_clock_freq {
u64 target_freq_hz;
u64 max_freq_hz;
u8 clk_id;
+ u32 clk_id_32;
} __packed;
/**
* struct ti_sci_msg_req_get_clock_freq - Request to get the clock frequency
* @hdr: Generic Header
* @dev_id: Device identifier this request is for
- * @clk_id: Clock identifier for the device for this request.
+ * @clk_id: Clock identifier for the device for this request. Set to
+ * 255 if clock ID is greater than or equal to 255.
+ * @clk_id_32: Clock identifier if @clk_id field is set to 255.
*
* NOTE: Normally clock frequency management is automatically done by TISCI
* entity. In some cases, clock frequencies are configured by host.
@@ -454,6 +531,7 @@ struct ti_sci_msg_req_get_clock_freq {
struct ti_sci_msg_hdr hdr;
u32 dev_id;
u8 clk_id;
+ u32 clk_id_32;
} __packed;
/**
@@ -563,4 +641,777 @@ struct ti_sci_msg_req_manage_irq {
u8 secondary_host;
} __packed;
+/**
+ * struct ti_sci_msg_rm_ring_cfg_req - Configure a Navigator Subsystem ring
+ *
+ * Configures the non-real-time registers of a Navigator Subsystem ring.
+ * @hdr: Generic Header
+ * @valid_params: Bitfield defining validity of ring configuration parameters.
+ * The ring configuration fields are not valid, and will not be used for
+ * ring configuration, if their corresponding valid bit is zero.
+ * Valid bit usage:
+ * 0 - Valid bit for @tisci_msg_rm_ring_cfg_req addr_lo
+ * 1 - Valid bit for @tisci_msg_rm_ring_cfg_req addr_hi
+ * 2 - Valid bit for @tisci_msg_rm_ring_cfg_req count
+ * 3 - Valid bit for @tisci_msg_rm_ring_cfg_req mode
+ * 4 - Valid bit for @tisci_msg_rm_ring_cfg_req size
+ * 5 - Valid bit for @tisci_msg_rm_ring_cfg_req order_id
+ * @nav_id: Device ID of Navigator Subsystem from which the ring is allocated
+ * @index: ring index to be configured.
+ * @addr_lo: 32 LSBs of ring base address to be programmed into the ring's
+ * RING_BA_LO register
+ * @addr_hi: 16 MSBs of ring base address to be programmed into the ring's
+ * RING_BA_HI register.
+ * @count: Number of ring elements. Must be even if mode is CREDENTIALS or QM
+ * modes.
+ * @mode: Specifies the mode the ring is to be configured.
+ * @size: Specifies encoded ring element size. To calculate the encoded size use
+ * the formula (log2(size_bytes) - 2), where size_bytes cannot be
+ * greater than 256.
+ * @order_id: Specifies the ring's bus order ID.
+ */
+struct ti_sci_msg_rm_ring_cfg_req {
+ struct ti_sci_msg_hdr hdr;
+ u32 valid_params;
+ u16 nav_id;
+ u16 index;
+ u32 addr_lo;
+ u32 addr_hi;
+ u32 count;
+ u8 mode;
+ u8 size;
+ u8 order_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_rm_ring_get_cfg_req - Get RA ring's configuration
+ *
+ * Gets the configuration of the non-real-time register fields of a ring. The
+ * host, or a supervisor of the host, who owns the ring must be the requesting
+ * host. The values of the non-real-time registers are returned in
+ * @ti_sci_msg_rm_ring_get_cfg_resp.
+ *
+ * @hdr: Generic Header
+ * @nav_id: Device ID of Navigator Subsystem from which the ring is allocated
+ * @index: ring index.
+ */
+struct ti_sci_msg_rm_ring_get_cfg_req {
+ struct ti_sci_msg_hdr hdr;
+ u16 nav_id;
+ u16 index;
+} __packed;
+
+/**
+ * struct ti_sci_msg_rm_ring_get_cfg_resp - Ring get configuration response
+ *
+ * Response received by host processor after RM has handled
+ * @ti_sci_msg_rm_ring_get_cfg_req. The response contains the ring's
+ * non-real-time register values.
+ *
+ * @hdr: Generic Header
+ * @addr_lo: Ring 32 LSBs of base address
+ * @addr_hi: Ring 16 MSBs of base address.
+ * @count: Ring number of elements.
+ * @mode: Ring mode.
+ * @size: encoded Ring element size
+ * @order_id: ing order ID.
+ */
+struct ti_sci_msg_rm_ring_get_cfg_resp {
+ struct ti_sci_msg_hdr hdr;
+ u32 addr_lo;
+ u32 addr_hi;
+ u32 count;
+ u8 mode;
+ u8 size;
+ u8 order_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_psil_pair - Pairs a PSI-L source thread to a destination
+ * thread
+ * @hdr: Generic Header
+ * @nav_id: SoC Navigator Subsystem device ID whose PSI-L config proxy is
+ * used to pair the source and destination threads.
+ * @src_thread: PSI-L source thread ID within the PSI-L System thread map.
+ *
+ * UDMAP transmit channels mapped to source threads will have their
+ * TCHAN_THRD_ID register programmed with the destination thread if the pairing
+ * is successful.
+
+ * @dst_thread: PSI-L destination thread ID within the PSI-L System thread map.
+ * PSI-L destination threads start at index 0x8000. The request is NACK'd if
+ * the destination thread is not greater than or equal to 0x8000.
+ *
+ * UDMAP receive channels mapped to destination threads will have their
+ * RCHAN_THRD_ID register programmed with the source thread if the pairing
+ * is successful.
+ *
+ * Request type is TI_SCI_MSG_RM_PSIL_PAIR, response is a generic ACK or NACK
+ * message.
+ */
+struct ti_sci_msg_psil_pair {
+ struct ti_sci_msg_hdr hdr;
+ u32 nav_id;
+ u32 src_thread;
+ u32 dst_thread;
+} __packed;
+
+/**
+ * struct ti_sci_msg_psil_unpair - Unpairs a PSI-L source thread from a
+ * destination thread
+ * @hdr: Generic Header
+ * @nav_id: SoC Navigator Subsystem device ID whose PSI-L config proxy is
+ * used to unpair the source and destination threads.
+ * @src_thread: PSI-L source thread ID within the PSI-L System thread map.
+ *
+ * UDMAP transmit channels mapped to source threads will have their
+ * TCHAN_THRD_ID register cleared if the unpairing is successful.
+ *
+ * @dst_thread: PSI-L destination thread ID within the PSI-L System thread map.
+ * PSI-L destination threads start at index 0x8000. The request is NACK'd if
+ * the destination thread is not greater than or equal to 0x8000.
+ *
+ * UDMAP receive channels mapped to destination threads will have their
+ * RCHAN_THRD_ID register cleared if the unpairing is successful.
+ *
+ * Request type is TI_SCI_MSG_RM_PSIL_UNPAIR, response is a generic ACK or NACK
+ * message.
+ */
+struct ti_sci_msg_psil_unpair {
+ struct ti_sci_msg_hdr hdr;
+ u32 nav_id;
+ u32 src_thread;
+ u32 dst_thread;
+} __packed;
+
+/**
+ * struct ti_sci_msg_udmap_rx_flow_cfg - UDMAP receive flow configuration
+ * message
+ * @hdr: Generic Header
+ * @nav_id: SoC Navigator Subsystem device ID from which the receive flow is
+ * allocated
+ * @flow_index: UDMAP receive flow index for non-optional configuration.
+ * @rx_ch_index: Specifies the index of the receive channel using the flow_index
+ * @rx_einfo_present: UDMAP receive flow extended packet info present.
+ * @rx_psinfo_present: UDMAP receive flow PS words present.
+ * @rx_error_handling: UDMAP receive flow error handling configuration. Valid
+ * values are TI_SCI_RM_UDMAP_RX_FLOW_ERR_DROP/RETRY.
+ * @rx_desc_type: UDMAP receive flow descriptor type. It can be one of
+ * TI_SCI_RM_UDMAP_RX_FLOW_DESC_HOST/MONO.
+ * @rx_sop_offset: UDMAP receive flow start of packet offset.
+ * @rx_dest_qnum: UDMAP receive flow destination queue number.
+ * @rx_ps_location: UDMAP receive flow PS words location.
+ * 0 - end of packet descriptor
+ * 1 - Beginning of the data buffer
+ * @rx_src_tag_hi: UDMAP receive flow source tag high byte constant
+ * @rx_src_tag_lo: UDMAP receive flow source tag low byte constant
+ * @rx_dest_tag_hi: UDMAP receive flow destination tag high byte constant
+ * @rx_dest_tag_lo: UDMAP receive flow destination tag low byte constant
+ * @rx_src_tag_hi_sel: UDMAP receive flow source tag high byte selector
+ * @rx_src_tag_lo_sel: UDMAP receive flow source tag low byte selector
+ * @rx_dest_tag_hi_sel: UDMAP receive flow destination tag high byte selector
+ * @rx_dest_tag_lo_sel: UDMAP receive flow destination tag low byte selector
+ * @rx_size_thresh_en: UDMAP receive flow packet size based free buffer queue
+ * enable. If enabled, the ti_sci_rm_udmap_rx_flow_opt_cfg also need to be
+ * configured and sent.
+ * @rx_fdq0_sz0_qnum: UDMAP receive flow free descriptor queue 0.
+ * @rx_fdq1_qnum: UDMAP receive flow free descriptor queue 1.
+ * @rx_fdq2_qnum: UDMAP receive flow free descriptor queue 2.
+ * @rx_fdq3_qnum: UDMAP receive flow free descriptor queue 3.
+ *
+ * For detailed information on the settings, see the UDMAP section of the TRM.
+ */
+struct ti_sci_msg_udmap_rx_flow_cfg {
+ struct ti_sci_msg_hdr hdr;
+ u32 nav_id;
+ u32 flow_index;
+ u32 rx_ch_index;
+ u8 rx_einfo_present;
+ u8 rx_psinfo_present;
+ u8 rx_error_handling;
+ u8 rx_desc_type;
+ u16 rx_sop_offset;
+ u16 rx_dest_qnum;
+ u8 rx_ps_location;
+ u8 rx_src_tag_hi;
+ u8 rx_src_tag_lo;
+ u8 rx_dest_tag_hi;
+ u8 rx_dest_tag_lo;
+ u8 rx_src_tag_hi_sel;
+ u8 rx_src_tag_lo_sel;
+ u8 rx_dest_tag_hi_sel;
+ u8 rx_dest_tag_lo_sel;
+ u8 rx_size_thresh_en;
+ u16 rx_fdq0_sz0_qnum;
+ u16 rx_fdq1_qnum;
+ u16 rx_fdq2_qnum;
+ u16 rx_fdq3_qnum;
+} __packed;
+
+/**
+ * struct rm_ti_sci_msg_udmap_rx_flow_opt_cfg - parameters for UDMAP receive
+ * flow optional configuration
+ * @hdr: Generic Header
+ * @nav_id: SoC Navigator Subsystem device ID from which the receive flow is
+ * allocated
+ * @flow_index: UDMAP receive flow index for optional configuration.
+ * @rx_ch_index: Specifies the index of the receive channel using the flow_index
+ * @rx_size_thresh0: UDMAP receive flow packet size threshold 0.
+ * @rx_size_thresh1: UDMAP receive flow packet size threshold 1.
+ * @rx_size_thresh2: UDMAP receive flow packet size threshold 2.
+ * @rx_fdq0_sz1_qnum: UDMAP receive flow free descriptor queue for size
+ * threshold 1.
+ * @rx_fdq0_sz2_qnum: UDMAP receive flow free descriptor queue for size
+ * threshold 2.
+ * @rx_fdq0_sz3_qnum: UDMAP receive flow free descriptor queue for size
+ * threshold 3.
+ *
+ * For detailed information on the settings, see the UDMAP section of the TRM.
+ */
+struct rm_ti_sci_msg_udmap_rx_flow_opt_cfg {
+ struct ti_sci_msg_hdr hdr;
+ u32 nav_id;
+ u32 flow_index;
+ u32 rx_ch_index;
+ u16 rx_size_thresh0;
+ u16 rx_size_thresh1;
+ u16 rx_size_thresh2;
+ u16 rx_fdq0_sz1_qnum;
+ u16 rx_fdq0_sz2_qnum;
+ u16 rx_fdq0_sz3_qnum;
+} __packed;
+
+/**
+ * Configures a Navigator Subsystem UDMAP transmit channel
+ *
+ * Configures the non-real-time registers of a Navigator Subsystem UDMAP
+ * transmit channel. The channel index must be assigned to the host defined
+ * in the TISCI header via the RM board configuration resource assignment
+ * range list.
+ *
+ * @hdr: Generic Header
+ *
+ * @valid_params: Bitfield defining validity of tx channel configuration
+ * parameters. The tx channel configuration fields are not valid, and will not
+ * be used for ch configuration, if their corresponding valid bit is zero.
+ * Valid bit usage:
+ * 0 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_pause_on_err
+ * 1 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_atype
+ * 2 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_chan_type
+ * 3 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_fetch_size
+ * 4 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::txcq_qnum
+ * 5 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_priority
+ * 6 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_qos
+ * 7 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_orderid
+ * 8 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_sched_priority
+ * 9 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_filt_einfo
+ * 10 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_filt_pswords
+ * 11 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_supr_tdpkt
+ * 12 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_credit_count
+ * 13 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::fdepth
+ * 14 - Valid bit for @ref ti_sci_msg_rm_udmap_tx_ch_cfg::tx_burst_size
+ *
+ * @nav_id: SoC device ID of Navigator Subsystem where tx channel is located
+ *
+ * @index: UDMAP transmit channel index.
+ *
+ * @tx_pause_on_err: UDMAP transmit channel pause on error configuration to
+ * be programmed into the tx_pause_on_err field of the channel's TCHAN_TCFG
+ * register.
+ *
+ * @tx_filt_einfo: UDMAP transmit channel extended packet information passing
+ * configuration to be programmed into the tx_filt_einfo field of the
+ * channel's TCHAN_TCFG register.
+ *
+ * @tx_filt_pswords: UDMAP transmit channel protocol specific word passing
+ * configuration to be programmed into the tx_filt_pswords field of the
+ * channel's TCHAN_TCFG register.
+ *
+ * @tx_atype: UDMAP transmit channel non Ring Accelerator access pointer
+ * interpretation configuration to be programmed into the tx_atype field of
+ * the channel's TCHAN_TCFG register.
+ *
+ * @tx_chan_type: UDMAP transmit channel functional channel type and work
+ * passing mechanism configuration to be programmed into the tx_chan_type
+ * field of the channel's TCHAN_TCFG register.
+ *
+ * @tx_supr_tdpkt: UDMAP transmit channel teardown packet generation suppression
+ * configuration to be programmed into the tx_supr_tdpkt field of the channel's
+ * TCHAN_TCFG register.
+ *
+ * @tx_fetch_size: UDMAP transmit channel number of 32-bit descriptor words to
+ * fetch configuration to be programmed into the tx_fetch_size field of the
+ * channel's TCHAN_TCFG register. The user must make sure to set the maximum
+ * word count that can pass through the channel for any allowed descriptor type.
+ *
+ * @tx_credit_count: UDMAP transmit channel transfer request credit count
+ * configuration to be programmed into the count field of the TCHAN_TCREDIT
+ * register. Specifies how many credits for complete TRs are available.
+ *
+ * @txcq_qnum: UDMAP transmit channel completion queue configuration to be
+ * programmed into the txcq_qnum field of the TCHAN_TCQ register. The specified
+ * completion queue must be assigned to the host, or a subordinate of the host,
+ * requesting configuration of the transmit channel.
+ *
+ * @tx_priority: UDMAP transmit channel transmit priority value to be programmed
+ * into the priority field of the channel's TCHAN_TPRI_CTRL register.
+ *
+ * @tx_qos: UDMAP transmit channel transmit qos value to be programmed into the
+ * qos field of the channel's TCHAN_TPRI_CTRL register.
+ *
+ * @tx_orderid: UDMAP transmit channel bus order id value to be programmed into
+ * the orderid field of the channel's TCHAN_TPRI_CTRL register.
+ *
+ * @fdepth: UDMAP transmit channel FIFO depth configuration to be programmed
+ * into the fdepth field of the TCHAN_TFIFO_DEPTH register. Sets the number of
+ * Tx FIFO bytes which are allowed to be stored for the channel. Check the UDMAP
+ * section of the TRM for restrictions regarding this parameter.
+ *
+ * @tx_sched_priority: UDMAP transmit channel tx scheduling priority
+ * configuration to be programmed into the priority field of the channel's
+ * TCHAN_TST_SCHED register.
+ *
+ * @tx_burst_size: UDMAP transmit channel burst size configuration to be
+ * programmed into the tx_burst_size field of the TCHAN_TCFG register.
+ */
+struct ti_sci_msg_rm_udmap_tx_ch_cfg_req {
+ struct ti_sci_msg_hdr hdr;
+ u32 valid_params;
+ u16 nav_id;
+ u16 index;
+ u8 tx_pause_on_err;
+ u8 tx_filt_einfo;
+ u8 tx_filt_pswords;
+ u8 tx_atype;
+ u8 tx_chan_type;
+ u8 tx_supr_tdpkt;
+ u16 tx_fetch_size;
+ u8 tx_credit_count;
+ u16 txcq_qnum;
+ u8 tx_priority;
+ u8 tx_qos;
+ u8 tx_orderid;
+ u16 fdepth;
+ u8 tx_sched_priority;
+ u8 tx_burst_size;
+} __packed;
+
+/**
+ * Configures a Navigator Subsystem UDMAP receive channel
+ *
+ * Configures the non-real-time registers of a Navigator Subsystem UDMAP
+ * receive channel. The channel index must be assigned to the host defined
+ * in the TISCI header via the RM board configuration resource assignment
+ * range list.
+ *
+ * @hdr: Generic Header
+ *
+ * @valid_params: Bitfield defining validity of rx channel configuration
+ * parameters.
+ * The rx channel configuration fields are not valid, and will not be used for
+ * ch configuration, if their corresponding valid bit is zero.
+ * Valid bit usage:
+ * 0 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_pause_on_err
+ * 1 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_atype
+ * 2 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_chan_type
+ * 3 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_fetch_size
+ * 4 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rxcq_qnum
+ * 5 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_priority
+ * 6 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_qos
+ * 7 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_orderid
+ * 8 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_sched_priority
+ * 9 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::flowid_start
+ * 10 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::flowid_cnt
+ * 11 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_ignore_short
+ * 12 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_ignore_long
+ * 14 - Valid bit for @ti_sci_msg_rm_udmap_rx_ch_cfg_req::rx_burst_size
+ *
+ * @nav_id: SoC device ID of Navigator Subsystem where rx channel is located
+ *
+ * @index: UDMAP receive channel index.
+ *
+ * @rx_fetch_size: UDMAP receive channel number of 32-bit descriptor words to
+ * fetch configuration to be programmed into the rx_fetch_size field of the
+ * channel's RCHAN_RCFG register.
+ *
+ * @rxcq_qnum: UDMAP receive channel completion queue configuration to be
+ * programmed into the rxcq_qnum field of the RCHAN_RCQ register.
+ * The specified completion queue must be assigned to the host, or a subordinate
+ * of the host, requesting configuration of the receive channel.
+ *
+ * @rx_priority: UDMAP receive channel receive priority value to be programmed
+ * into the priority field of the channel's RCHAN_RPRI_CTRL register.
+ *
+ * @rx_qos: UDMAP receive channel receive qos value to be programmed into the
+ * qos field of the channel's RCHAN_RPRI_CTRL register.
+ *
+ * @rx_orderid: UDMAP receive channel bus order id value to be programmed into
+ * the orderid field of the channel's RCHAN_RPRI_CTRL register.
+ *
+ * @rx_sched_priority: UDMAP receive channel rx scheduling priority
+ * configuration to be programmed into the priority field of the channel's
+ * RCHAN_RST_SCHED register.
+ *
+ * @flowid_start: UDMAP receive channel additional flows starting index
+ * configuration to program into the flow_start field of the RCHAN_RFLOW_RNG
+ * register. Specifies the starting index for flow IDs the receive channel is to
+ * make use of beyond the default flow. flowid_start and @ref flowid_cnt must be
+ * set as valid and configured together. The starting flow ID set by
+ * @ref flowid_cnt must be a flow index within the Navigator Subsystem's subset
+ * of flows beyond the default flows statically mapped to receive channels.
+ * The additional flows must be assigned to the host, or a subordinate of the
+ * host, requesting configuration of the receive channel.
+ *
+ * @flowid_cnt: UDMAP receive channel additional flows count configuration to
+ * program into the flowid_cnt field of the RCHAN_RFLOW_RNG register.
+ * This field specifies how many flow IDs are in the additional contiguous range
+ * of legal flow IDs for the channel. @ref flowid_start and flowid_cnt must be
+ * set as valid and configured together. Disabling the valid_params field bit
+ * for flowid_cnt indicates no flow IDs other than the default are to be
+ * allocated and used by the receive channel. @ref flowid_start plus flowid_cnt
+ * cannot be greater than the number of receive flows in the receive channel's
+ * Navigator Subsystem. The additional flows must be assigned to the host, or a
+ * subordinate of the host, requesting configuration of the receive channel.
+ *
+ * @rx_pause_on_err: UDMAP receive channel pause on error configuration to be
+ * programmed into the rx_pause_on_err field of the channel's RCHAN_RCFG
+ * register.
+ *
+ * @rx_atype: UDMAP receive channel non Ring Accelerator access pointer
+ * interpretation configuration to be programmed into the rx_atype field of the
+ * channel's RCHAN_RCFG register.
+ *
+ * @rx_chan_type: UDMAP receive channel functional channel type and work passing
+ * mechanism configuration to be programmed into the rx_chan_type field of the
+ * channel's RCHAN_RCFG register.
+ *
+ * @rx_ignore_short: UDMAP receive channel short packet treatment configuration
+ * to be programmed into the rx_ignore_short field of the RCHAN_RCFG register.
+ *
+ * @rx_ignore_long: UDMAP receive channel long packet treatment configuration to
+ * be programmed into the rx_ignore_long field of the RCHAN_RCFG register.
+ *
+ * @rx_burst_size: UDMAP receive channel burst size configuration to be
+ * programmed into the rx_burst_size field of the RCHAN_RCFG register.
+ */
+struct ti_sci_msg_rm_udmap_rx_ch_cfg_req {
+ struct ti_sci_msg_hdr hdr;
+ u32 valid_params;
+ u16 nav_id;
+ u16 index;
+ u16 rx_fetch_size;
+ u16 rxcq_qnum;
+ u8 rx_priority;
+ u8 rx_qos;
+ u8 rx_orderid;
+ u8 rx_sched_priority;
+ u16 flowid_start;
+ u16 flowid_cnt;
+ u8 rx_pause_on_err;
+ u8 rx_atype;
+ u8 rx_chan_type;
+ u8 rx_ignore_short;
+ u8 rx_ignore_long;
+ u8 rx_burst_size;
+} __packed;
+
+/**
+ * Configures a Navigator Subsystem UDMAP receive flow
+ *
+ * Configures a Navigator Subsystem UDMAP receive flow's registers.
+ * Configuration does not include the flow registers which handle size-based
+ * free descriptor queue routing.
+ *
+ * The flow index must be assigned to the host defined in the TISCI header via
+ * the RM board configuration resource assignment range list.
+ *
+ * @hdr: Standard TISCI header
+ *
+ * @valid_params
+ * Bitfield defining validity of rx flow configuration parameters. The
+ * rx flow configuration fields are not valid, and will not be used for flow
+ * configuration, if their corresponding valid bit is zero. Valid bit usage:
+ * 0 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_einfo_present
+ * 1 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_psinfo_present
+ * 2 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_error_handling
+ * 3 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_desc_type
+ * 4 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_sop_offset
+ * 5 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_dest_qnum
+ * 6 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_src_tag_hi
+ * 7 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_src_tag_lo
+ * 8 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_dest_tag_hi
+ * 9 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_dest_tag_lo
+ * 10 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_src_tag_hi_sel
+ * 11 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_src_tag_lo_sel
+ * 12 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_dest_tag_hi_sel
+ * 13 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_dest_tag_lo_sel
+ * 14 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_fdq0_sz0_qnum
+ * 15 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_fdq1_sz0_qnum
+ * 16 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_fdq2_sz0_qnum
+ * 17 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_fdq3_sz0_qnum
+ * 18 - Valid bit for @tisci_msg_rm_udmap_flow_cfg_req::rx_ps_location
+ *
+ * @nav_id: SoC device ID of Navigator Subsystem from which the receive flow is
+ * allocated
+ *
+ * @flow_index: UDMAP receive flow index for non-optional configuration.
+ *
+ * @rx_einfo_present:
+ * UDMAP receive flow extended packet info present configuration to be
+ * programmed into the rx_einfo_present field of the flow's RFLOW_RFA register.
+ *
+ * @rx_psinfo_present:
+ * UDMAP receive flow PS words present configuration to be programmed into the
+ * rx_psinfo_present field of the flow's RFLOW_RFA register.
+ *
+ * @rx_error_handling:
+ * UDMAP receive flow error handling configuration to be programmed into the
+ * rx_error_handling field of the flow's RFLOW_RFA register.
+ *
+ * @rx_desc_type:
+ * UDMAP receive flow descriptor type configuration to be programmed into the
+ * rx_desc_type field field of the flow's RFLOW_RFA register.
+ *
+ * @rx_sop_offset:
+ * UDMAP receive flow start of packet offset configuration to be programmed
+ * into the rx_sop_offset field of the RFLOW_RFA register. See the UDMAP
+ * section of the TRM for more information on this setting. Valid values for
+ * this field are 0-255 bytes.
+ *
+ * @rx_dest_qnum:
+ * UDMAP receive flow destination queue configuration to be programmed into the
+ * rx_dest_qnum field of the flow's RFLOW_RFA register. The specified
+ * destination queue must be valid within the Navigator Subsystem and must be
+ * owned by the host, or a subordinate of the host, requesting allocation and
+ * configuration of the receive flow.
+ *
+ * @rx_src_tag_hi:
+ * UDMAP receive flow source tag high byte constant configuration to be
+ * programmed into the rx_src_tag_hi field of the flow's RFLOW_RFB register.
+ * See the UDMAP section of the TRM for more information on this setting.
+ *
+ * @rx_src_tag_lo:
+ * UDMAP receive flow source tag low byte constant configuration to be
+ * programmed into the rx_src_tag_lo field of the flow's RFLOW_RFB register.
+ * See the UDMAP section of the TRM for more information on this setting.
+ *
+ * @rx_dest_tag_hi:
+ * UDMAP receive flow destination tag high byte constant configuration to be
+ * programmed into the rx_dest_tag_hi field of the flow's RFLOW_RFB register.
+ * See the UDMAP section of the TRM for more information on this setting.
+ *
+ * @rx_dest_tag_lo:
+ * UDMAP receive flow destination tag low byte constant configuration to be
+ * programmed into the rx_dest_tag_lo field of the flow's RFLOW_RFB register.
+ * See the UDMAP section of the TRM for more information on this setting.
+ *
+ * @rx_src_tag_hi_sel:
+ * UDMAP receive flow source tag high byte selector configuration to be
+ * programmed into the rx_src_tag_hi_sel field of the RFLOW_RFC register. See
+ * the UDMAP section of the TRM for more information on this setting.
+ *
+ * @rx_src_tag_lo_sel:
+ * UDMAP receive flow source tag low byte selector configuration to be
+ * programmed into the rx_src_tag_lo_sel field of the RFLOW_RFC register. See
+ * the UDMAP section of the TRM for more information on this setting.
+ *
+ * @rx_dest_tag_hi_sel:
+ * UDMAP receive flow destination tag high byte selector configuration to be
+ * programmed into the rx_dest_tag_hi_sel field of the RFLOW_RFC register. See
+ * the UDMAP section of the TRM for more information on this setting.
+ *
+ * @rx_dest_tag_lo_sel:
+ * UDMAP receive flow destination tag low byte selector configuration to be
+ * programmed into the rx_dest_tag_lo_sel field of the RFLOW_RFC register. See
+ * the UDMAP section of the TRM for more information on this setting.
+ *
+ * @rx_fdq0_sz0_qnum:
+ * UDMAP receive flow free descriptor queue 0 configuration to be programmed
+ * into the rx_fdq0_sz0_qnum field of the flow's RFLOW_RFD register. See the
+ * UDMAP section of the TRM for more information on this setting. The specified
+ * free queue must be valid within the Navigator Subsystem and must be owned
+ * by the host, or a subordinate of the host, requesting allocation and
+ * configuration of the receive flow.
+ *
+ * @rx_fdq1_qnum:
+ * UDMAP receive flow free descriptor queue 1 configuration to be programmed
+ * into the rx_fdq1_qnum field of the flow's RFLOW_RFD register. See the
+ * UDMAP section of the TRM for more information on this setting. The specified
+ * free queue must be valid within the Navigator Subsystem and must be owned
+ * by the host, or a subordinate of the host, requesting allocation and
+ * configuration of the receive flow.
+ *
+ * @rx_fdq2_qnum:
+ * UDMAP receive flow free descriptor queue 2 configuration to be programmed
+ * into the rx_fdq2_qnum field of the flow's RFLOW_RFE register. See the
+ * UDMAP section of the TRM for more information on this setting. The specified
+ * free queue must be valid within the Navigator Subsystem and must be owned
+ * by the host, or a subordinate of the host, requesting allocation and
+ * configuration of the receive flow.
+ *
+ * @rx_fdq3_qnum:
+ * UDMAP receive flow free descriptor queue 3 configuration to be programmed
+ * into the rx_fdq3_qnum field of the flow's RFLOW_RFE register. See the
+ * UDMAP section of the TRM for more information on this setting. The specified
+ * free queue must be valid within the Navigator Subsystem and must be owned
+ * by the host, or a subordinate of the host, requesting allocation and
+ * configuration of the receive flow.
+ *
+ * @rx_ps_location:
+ * UDMAP receive flow PS words location configuration to be programmed into the
+ * rx_ps_location field of the flow's RFLOW_RFA register.
+ */
+struct ti_sci_msg_rm_udmap_flow_cfg_req {
+ struct ti_sci_msg_hdr hdr;
+ u32 valid_params;
+ u16 nav_id;
+ u16 flow_index;
+ u8 rx_einfo_present;
+ u8 rx_psinfo_present;
+ u8 rx_error_handling;
+ u8 rx_desc_type;
+ u16 rx_sop_offset;
+ u16 rx_dest_qnum;
+ u8 rx_src_tag_hi;
+ u8 rx_src_tag_lo;
+ u8 rx_dest_tag_hi;
+ u8 rx_dest_tag_lo;
+ u8 rx_src_tag_hi_sel;
+ u8 rx_src_tag_lo_sel;
+ u8 rx_dest_tag_hi_sel;
+ u8 rx_dest_tag_lo_sel;
+ u16 rx_fdq0_sz0_qnum;
+ u16 rx_fdq1_qnum;
+ u16 rx_fdq2_qnum;
+ u16 rx_fdq3_qnum;
+ u8 rx_ps_location;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_proc_request - Request a processor
+ * @hdr: Generic Header
+ * @processor_id: ID of processor being requested
+ *
+ * Request type is TI_SCI_MSG_PROC_REQUEST, response is a generic ACK/NACK
+ * message.
+ */
+struct ti_sci_msg_req_proc_request {
+ struct ti_sci_msg_hdr hdr;
+ u8 processor_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_proc_release - Release a processor
+ * @hdr: Generic Header
+ * @processor_id: ID of processor being released
+ *
+ * Request type is TI_SCI_MSG_PROC_RELEASE, response is a generic ACK/NACK
+ * message.
+ */
+struct ti_sci_msg_req_proc_release {
+ struct ti_sci_msg_hdr hdr;
+ u8 processor_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_proc_handover - Handover a processor to a host
+ * @hdr: Generic Header
+ * @processor_id: ID of processor being handed over
+ * @host_id: Host ID the control needs to be transferred to
+ *
+ * Request type is TI_SCI_MSG_PROC_HANDOVER, response is a generic ACK/NACK
+ * message.
+ */
+struct ti_sci_msg_req_proc_handover {
+ struct ti_sci_msg_hdr hdr;
+ u8 processor_id;
+ u8 host_id;
+} __packed;
+
+/* Boot Vector masks */
+#define TI_SCI_ADDR_LOW_MASK GENMASK_ULL(31, 0)
+#define TI_SCI_ADDR_HIGH_MASK GENMASK_ULL(63, 32)
+#define TI_SCI_ADDR_HIGH_SHIFT 32
+
+/**
+ * struct ti_sci_msg_req_set_config - Set Processor boot configuration
+ * @hdr: Generic Header
+ * @processor_id: ID of processor being configured
+ * @bootvector_low: Lower 32 bit address (Little Endian) of boot vector
+ * @bootvector_high: Higher 32 bit address (Little Endian) of boot vector
+ * @config_flags_set: Optional Processor specific Config Flags to set.
+ * Setting a bit here implies the corresponding mode
+ * will be set
+ * @config_flags_clear: Optional Processor specific Config Flags to clear.
+ * Setting a bit here implies the corresponding mode
+ * will be cleared
+ *
+ * Request type is TI_SCI_MSG_PROC_HANDOVER, response is a generic ACK/NACK
+ * message.
+ */
+struct ti_sci_msg_req_set_config {
+ struct ti_sci_msg_hdr hdr;
+ u8 processor_id;
+ u32 bootvector_low;
+ u32 bootvector_high;
+ u32 config_flags_set;
+ u32 config_flags_clear;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_set_ctrl - Set Processor boot control flags
+ * @hdr: Generic Header
+ * @processor_id: ID of processor being configured
+ * @control_flags_set: Optional Processor specific Control Flags to set.
+ * Setting a bit here implies the corresponding mode
+ * will be set
+ * @control_flags_clear:Optional Processor specific Control Flags to clear.
+ * Setting a bit here implies the corresponding mode
+ * will be cleared
+ *
+ * Request type is TI_SCI_MSG_SET_CTRL, response is a generic ACK/NACK
+ * message.
+ */
+struct ti_sci_msg_req_set_ctrl {
+ struct ti_sci_msg_hdr hdr;
+ u8 processor_id;
+ u32 control_flags_set;
+ u32 control_flags_clear;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_get_status - Processor boot status request
+ * @hdr: Generic Header
+ * @processor_id: ID of processor whose status is being requested
+ *
+ * Request type is TI_SCI_MSG_GET_STATUS, response is an appropriate
+ * message, or NACK in case of inability to satisfy request.
+ */
+struct ti_sci_msg_req_get_status {
+ struct ti_sci_msg_hdr hdr;
+ u8 processor_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_resp_get_status - Processor boot status response
+ * @hdr: Generic Header
+ * @processor_id: ID of processor whose status is returned
+ * @bootvector_low: Lower 32 bit address (Little Endian) of boot vector
+ * @bootvector_high: Higher 32 bit address (Little Endian) of boot vector
+ * @config_flags: Optional Processor specific Config Flags set currently
+ * @control_flags: Optional Processor specific Control Flags set currently
+ * @status_flags: Optional Processor specific Status Flags set currently
+ *
+ * Response structure to a TI_SCI_MSG_GET_STATUS request.
+ */
+struct ti_sci_msg_resp_get_status {
+ struct ti_sci_msg_hdr hdr;
+ u8 processor_id;
+ u32 bootvector_low;
+ u32 bootvector_high;
+ u32 config_flags;
+ u32 control_flags;
+ u32 status_flags;
+} __packed;
+
#endif /* __TI_SCI_H */
diff --git a/drivers/firmware/turris-mox-rwtm.c b/drivers/firmware/turris-mox-rwtm.c
new file mode 100644
index 000000000000..72be58960e54
--- /dev/null
+++ b/drivers/firmware/turris-mox-rwtm.c
@@ -0,0 +1,384 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Turris Mox rWTM firmware driver
+ *
+ * Copyright (C) 2019 Marek Behun <marek.behun@nic.cz>
+ */
+
+#include <linux/armada-37xx-rwtm-mailbox.h>
+#include <linux/completion.h>
+#include <linux/dma-mapping.h>
+#include <linux/hw_random.h>
+#include <linux/mailbox_client.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#define DRIVER_NAME "turris-mox-rwtm"
+
+/*
+ * The macros and constants below come from Turris Mox's rWTM firmware code.
+ * This firmware is open source and it's sources can be found at
+ * https://gitlab.labs.nic.cz/turris/mox-boot-builder/tree/master/wtmi.
+ */
+
+#define MBOX_STS_SUCCESS (0 << 30)
+#define MBOX_STS_FAIL (1 << 30)
+#define MBOX_STS_BADCMD (2 << 30)
+#define MBOX_STS_ERROR(s) ((s) & (3 << 30))
+#define MBOX_STS_VALUE(s) (((s) >> 10) & 0xfffff)
+#define MBOX_STS_CMD(s) ((s) & 0x3ff)
+
+enum mbox_cmd {
+ MBOX_CMD_GET_RANDOM = 1,
+ MBOX_CMD_BOARD_INFO = 2,
+ MBOX_CMD_ECDSA_PUB_KEY = 3,
+ MBOX_CMD_HASH = 4,
+ MBOX_CMD_SIGN = 5,
+ MBOX_CMD_VERIFY = 6,
+
+ MBOX_CMD_OTP_READ = 7,
+ MBOX_CMD_OTP_WRITE = 8,
+};
+
+struct mox_kobject;
+
+struct mox_rwtm {
+ struct device *dev;
+ struct mbox_client mbox_client;
+ struct mbox_chan *mbox;
+ struct mox_kobject *kobj;
+ struct hwrng hwrng;
+
+ struct armada_37xx_rwtm_rx_msg reply;
+
+ void *buf;
+ dma_addr_t buf_phys;
+
+ struct mutex busy;
+ struct completion cmd_done;
+
+ /* board information */
+ int has_board_info;
+ u64 serial_number;
+ int board_version, ram_size;
+ u8 mac_address1[6], mac_address2[6];
+
+ /* public key burned in eFuse */
+ int has_pubkey;
+ u8 pubkey[135];
+};
+
+struct mox_kobject {
+ struct kobject kobj;
+ struct mox_rwtm *rwtm;
+};
+
+static inline struct kobject *rwtm_to_kobj(struct mox_rwtm *rwtm)
+{
+ return &rwtm->kobj->kobj;
+}
+
+static inline struct mox_rwtm *to_rwtm(struct kobject *kobj)
+{
+ return container_of(kobj, struct mox_kobject, kobj)->rwtm;
+}
+
+static void mox_kobj_release(struct kobject *kobj)
+{
+ kfree(to_rwtm(kobj)->kobj);
+}
+
+static struct kobj_type mox_kobj_ktype = {
+ .release = mox_kobj_release,
+ .sysfs_ops = &kobj_sysfs_ops,
+};
+
+static int mox_kobj_create(struct mox_rwtm *rwtm)
+{
+ rwtm->kobj = kzalloc(sizeof(*rwtm->kobj), GFP_KERNEL);
+ if (!rwtm->kobj)
+ return -ENOMEM;
+
+ kobject_init(rwtm_to_kobj(rwtm), &mox_kobj_ktype);
+ if (kobject_add(rwtm_to_kobj(rwtm), firmware_kobj, "turris-mox-rwtm")) {
+ kobject_put(rwtm_to_kobj(rwtm));
+ return -ENXIO;
+ }
+
+ rwtm->kobj->rwtm = rwtm;
+
+ return 0;
+}
+
+#define MOX_ATTR_RO(name, format, cat) \
+static ssize_t \
+name##_show(struct kobject *kobj, struct kobj_attribute *a, \
+ char *buf) \
+{ \
+ struct mox_rwtm *rwtm = to_rwtm(kobj); \
+ if (!rwtm->has_##cat) \
+ return -ENODATA; \
+ return sprintf(buf, format, rwtm->name); \
+} \
+static struct kobj_attribute mox_attr_##name = __ATTR_RO(name)
+
+MOX_ATTR_RO(serial_number, "%016llX\n", board_info);
+MOX_ATTR_RO(board_version, "%i\n", board_info);
+MOX_ATTR_RO(ram_size, "%i\n", board_info);
+MOX_ATTR_RO(mac_address1, "%pM\n", board_info);
+MOX_ATTR_RO(mac_address2, "%pM\n", board_info);
+MOX_ATTR_RO(pubkey, "%s\n", pubkey);
+
+static int mox_get_status(enum mbox_cmd cmd, u32 retval)
+{
+ if (MBOX_STS_CMD(retval) != cmd ||
+ MBOX_STS_ERROR(retval) != MBOX_STS_SUCCESS)
+ return -EIO;
+ else if (MBOX_STS_ERROR(retval) == MBOX_STS_FAIL)
+ return -(int)MBOX_STS_VALUE(retval);
+ else
+ return MBOX_STS_VALUE(retval);
+}
+
+static const struct attribute *mox_rwtm_attrs[] = {
+ &mox_attr_serial_number.attr,
+ &mox_attr_board_version.attr,
+ &mox_attr_ram_size.attr,
+ &mox_attr_mac_address1.attr,
+ &mox_attr_mac_address2.attr,
+ &mox_attr_pubkey.attr,
+ NULL
+};
+
+static void mox_rwtm_rx_callback(struct mbox_client *cl, void *data)
+{
+ struct mox_rwtm *rwtm = dev_get_drvdata(cl->dev);
+ struct armada_37xx_rwtm_rx_msg *msg = data;
+
+ rwtm->reply = *msg;
+ complete(&rwtm->cmd_done);
+}
+
+static void reply_to_mac_addr(u8 *mac, u32 t1, u32 t2)
+{
+ mac[0] = t1 >> 8;
+ mac[1] = t1;
+ mac[2] = t2 >> 24;
+ mac[3] = t2 >> 16;
+ mac[4] = t2 >> 8;
+ mac[5] = t2;
+}
+
+static int mox_get_board_info(struct mox_rwtm *rwtm)
+{
+ struct armada_37xx_rwtm_tx_msg msg;
+ struct armada_37xx_rwtm_rx_msg *reply = &rwtm->reply;
+ int ret;
+
+ msg.command = MBOX_CMD_BOARD_INFO;
+ ret = mbox_send_message(rwtm->mbox, &msg);
+ if (ret < 0)
+ return ret;
+
+ ret = wait_for_completion_timeout(&rwtm->cmd_done, HZ / 2);
+ if (ret < 0)
+ return ret;
+
+ ret = mox_get_status(MBOX_CMD_BOARD_INFO, reply->retval);
+ if (ret < 0 && ret != -ENODATA) {
+ return ret;
+ } else if (ret == -ENODATA) {
+ dev_warn(rwtm->dev,
+ "Board does not have manufacturing information burned!\n");
+ } else {
+ rwtm->serial_number = reply->status[1];
+ rwtm->serial_number <<= 32;
+ rwtm->serial_number |= reply->status[0];
+ rwtm->board_version = reply->status[2];
+ rwtm->ram_size = reply->status[3];
+ reply_to_mac_addr(rwtm->mac_address1, reply->status[4],
+ reply->status[5]);
+ reply_to_mac_addr(rwtm->mac_address2, reply->status[6],
+ reply->status[7]);
+ rwtm->has_board_info = 1;
+
+ pr_info("Turris Mox serial number %016llX\n",
+ rwtm->serial_number);
+ pr_info(" board version %i\n", rwtm->board_version);
+ pr_info(" burned RAM size %i MiB\n", rwtm->ram_size);
+ }
+
+ msg.command = MBOX_CMD_ECDSA_PUB_KEY;
+ ret = mbox_send_message(rwtm->mbox, &msg);
+ if (ret < 0)
+ return ret;
+
+ ret = wait_for_completion_timeout(&rwtm->cmd_done, HZ / 2);
+ if (ret < 0)
+ return ret;
+
+ ret = mox_get_status(MBOX_CMD_ECDSA_PUB_KEY, reply->retval);
+ if (ret < 0 && ret != -ENODATA) {
+ return ret;
+ } else if (ret == -ENODATA) {
+ dev_warn(rwtm->dev, "Board has no public key burned!\n");
+ } else {
+ u32 *s = reply->status;
+
+ rwtm->has_pubkey = 1;
+ sprintf(rwtm->pubkey,
+ "%06x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x",
+ ret, s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7],
+ s[8], s[9], s[10], s[11], s[12], s[13], s[14], s[15]);
+ }
+
+ return 0;
+}
+
+static int mox_hwrng_read(struct hwrng *rng, void *data, size_t max, bool wait)
+{
+ struct mox_rwtm *rwtm = (struct mox_rwtm *) rng->priv;
+ struct armada_37xx_rwtm_tx_msg msg;
+ int ret;
+
+ if (max > 4096)
+ max = 4096;
+
+ msg.command = MBOX_CMD_GET_RANDOM;
+ msg.args[0] = 1;
+ msg.args[1] = rwtm->buf_phys;
+ msg.args[2] = (max + 3) & ~3;
+
+ if (!wait) {
+ if (!mutex_trylock(&rwtm->busy))
+ return -EBUSY;
+ } else {
+ mutex_lock(&rwtm->busy);
+ }
+
+ ret = mbox_send_message(rwtm->mbox, &msg);
+ if (ret < 0)
+ goto unlock_mutex;
+
+ ret = wait_for_completion_interruptible(&rwtm->cmd_done);
+ if (ret < 0)
+ goto unlock_mutex;
+
+ ret = mox_get_status(MBOX_CMD_GET_RANDOM, rwtm->reply.retval);
+ if (ret < 0)
+ goto unlock_mutex;
+
+ memcpy(data, rwtm->buf, max);
+ ret = max;
+
+unlock_mutex:
+ mutex_unlock(&rwtm->busy);
+ return ret;
+}
+
+static int turris_mox_rwtm_probe(struct platform_device *pdev)
+{
+ struct mox_rwtm *rwtm;
+ struct device *dev = &pdev->dev;
+ int ret;
+
+ rwtm = devm_kzalloc(dev, sizeof(*rwtm), GFP_KERNEL);
+ if (!rwtm)
+ return -ENOMEM;
+
+ rwtm->dev = dev;
+ rwtm->buf = dmam_alloc_coherent(dev, PAGE_SIZE, &rwtm->buf_phys,
+ GFP_KERNEL);
+ if (!rwtm->buf)
+ return -ENOMEM;
+
+ ret = mox_kobj_create(rwtm);
+ if (ret < 0) {
+ dev_err(dev, "Cannot create turris-mox-rwtm kobject!\n");
+ return ret;
+ }
+
+ ret = sysfs_create_files(rwtm_to_kobj(rwtm), mox_rwtm_attrs);
+ if (ret < 0) {
+ dev_err(dev, "Cannot create sysfs files!\n");
+ goto put_kobj;
+ }
+
+ platform_set_drvdata(pdev, rwtm);
+
+ mutex_init(&rwtm->busy);
+
+ rwtm->mbox_client.dev = dev;
+ rwtm->mbox_client.rx_callback = mox_rwtm_rx_callback;
+
+ rwtm->mbox = mbox_request_channel(&rwtm->mbox_client, 0);
+ if (IS_ERR(rwtm->mbox)) {
+ ret = PTR_ERR(rwtm->mbox);
+ if (ret != -EPROBE_DEFER)
+ dev_err(dev, "Cannot request mailbox channel: %i\n",
+ ret);
+ goto remove_files;
+ }
+
+ init_completion(&rwtm->cmd_done);
+
+ ret = mox_get_board_info(rwtm);
+ if (ret < 0)
+ dev_warn(dev, "Cannot read board information: %i\n", ret);
+
+ rwtm->hwrng.name = DRIVER_NAME "_hwrng";
+ rwtm->hwrng.read = mox_hwrng_read;
+ rwtm->hwrng.priv = (unsigned long) rwtm;
+ rwtm->hwrng.quality = 1024;
+
+ ret = devm_hwrng_register(dev, &rwtm->hwrng);
+ if (ret < 0) {
+ dev_err(dev, "Cannot register HWRNG: %i\n", ret);
+ goto free_channel;
+ }
+
+ return 0;
+
+free_channel:
+ mbox_free_channel(rwtm->mbox);
+remove_files:
+ sysfs_remove_files(rwtm_to_kobj(rwtm), mox_rwtm_attrs);
+put_kobj:
+ kobject_put(rwtm_to_kobj(rwtm));
+ return ret;
+}
+
+static int turris_mox_rwtm_remove(struct platform_device *pdev)
+{
+ struct mox_rwtm *rwtm = platform_get_drvdata(pdev);
+
+ sysfs_remove_files(rwtm_to_kobj(rwtm), mox_rwtm_attrs);
+ kobject_put(rwtm_to_kobj(rwtm));
+ mbox_free_channel(rwtm->mbox);
+
+ return 0;
+}
+
+static const struct of_device_id turris_mox_rwtm_match[] = {
+ { .compatible = "cznic,turris-mox-rwtm", },
+ { },
+};
+
+MODULE_DEVICE_TABLE(of, turris_mox_rwtm_match);
+
+static struct platform_driver turris_mox_rwtm_driver = {
+ .probe = turris_mox_rwtm_probe,
+ .remove = turris_mox_rwtm_remove,
+ .driver = {
+ .name = DRIVER_NAME,
+ .of_match_table = turris_mox_rwtm_match,
+ },
+};
+module_platform_driver(turris_mox_rwtm_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Turris Mox rWTM firmware driver");
+MODULE_AUTHOR("Marek Behun <marek.behun@nic.cz>");
diff --git a/drivers/fmc/Kconfig b/drivers/fmc/Kconfig
deleted file mode 100644
index ae3d7f634932..000000000000
--- a/drivers/fmc/Kconfig
+++ /dev/null
@@ -1,52 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# FMC (ANSI-VITA 57.1) bus support
-#
-
-menuconfig FMC
- tristate "FMC support"
- help
-
- FMC (FPGA Mezzanine Carrier) is a mechanical and electrical
- standard for mezzanine cards that plug into a carrier board.
- This kernel subsystem supports the matching between carrier
- and mezzanine based on identifiers stored in the internal I2C
- EEPROM, as well as having carrier-independent drivers.
-
- The framework was born outside of the kernel and at this time
- the off-tree code base is more complete. Code and documentation
- is at git://ohwr.org/fmc-projects/fmc-bus.git .
-
-if FMC
-
-config FMC_FAKEDEV
- tristate "FMC fake device (software testing)"
- help
- This is a fake carrier, bringing a default EEPROM content
- that can be rewritten at run time and usef for matching
- mezzanines.
-
-config FMC_TRIVIAL
- tristate "FMC trivial mezzanine driver (software testing)"
- help
- This is a fake mezzanine driver, to show how FMC works and test it.
- The driver also handles interrupts (we used it with a real carrier
- before the mezzanines were produced)
-
-config FMC_WRITE_EEPROM
- tristate "FMC mezzanine driver to write I2C EEPROM"
- help
- This driver matches every mezzanine device and can write the
- internal EEPROM of the PCB, using the firmware loader to get
- its binary and the function carrier->reprogram to actually do it.
- It is useful when the mezzanines are produced.
-
-config FMC_CHARDEV
- tristate "FMC mezzanine driver that registers a char device"
- help
- This driver matches every mezzanine device and allows user
- space to read and write registers using a char device. It
- can be used to write user-space drivers, or just get
- acquainted with a mezzanine before writing its specific driver.
-
-endif # FMC
diff --git a/drivers/fmc/Makefile b/drivers/fmc/Makefile
deleted file mode 100644
index e3da6192cf39..000000000000
--- a/drivers/fmc/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-
-obj-$(CONFIG_FMC) += fmc.o
-
-fmc-y = fmc-core.o
-fmc-y += fmc-match.o
-fmc-y += fmc-sdb.o
-fmc-y += fru-parse.o
-fmc-y += fmc-dump.o
-fmc-y += fmc-debug.o
-
-obj-$(CONFIG_FMC_FAKEDEV) += fmc-fakedev.o
-obj-$(CONFIG_FMC_TRIVIAL) += fmc-trivial.o
-obj-$(CONFIG_FMC_WRITE_EEPROM) += fmc-write-eeprom.o
-obj-$(CONFIG_FMC_CHARDEV) += fmc-chardev.o
diff --git a/drivers/fmc/fmc-chardev.c b/drivers/fmc/fmc-chardev.c
deleted file mode 100644
index 7d2091b5e978..000000000000
--- a/drivers/fmc/fmc-chardev.c
+++ /dev/null
@@ -1,199 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Copyright (C) 2012 CERN (www.cern.ch)
- * Author: Alessandro Rubini <rubini@gnudd.com>
- *
- * This work is part of the White Rabbit project, a research effort led
- * by CERN, the European Institute for Nuclear Research.
- */
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/list.h>
-#include <linux/slab.h>
-#include <linux/fs.h>
-#include <linux/miscdevice.h>
-#include <linux/spinlock.h>
-#include <linux/fmc.h>
-#include <linux/uaccess.h>
-
-static LIST_HEAD(fc_devices);
-static DEFINE_SPINLOCK(fc_lock);
-
-struct fc_instance {
- struct list_head list;
- struct fmc_device *fmc;
- struct miscdevice misc;
-};
-
-/* at open time, we must identify our device */
-static int fc_open(struct inode *ino, struct file *f)
-{
- struct fmc_device *fmc;
- struct fc_instance *fc;
- int minor = iminor(ino);
-
- list_for_each_entry(fc, &fc_devices, list)
- if (fc->misc.minor == minor)
- break;
- if (fc->misc.minor != minor)
- return -ENODEV;
- fmc = fc->fmc;
- if (try_module_get(fmc->owner) == 0)
- return -ENODEV;
-
- f->private_data = fmc;
- return 0;
-}
-
-static int fc_release(struct inode *ino, struct file *f)
-{
- struct fmc_device *fmc = f->private_data;
- module_put(fmc->owner);
- return 0;
-}
-
-/* read and write are simple after the default llseek has been used */
-static ssize_t fc_read(struct file *f, char __user *buf, size_t count,
- loff_t *offp)
-{
- struct fmc_device *fmc = f->private_data;
- unsigned long addr;
- uint32_t val;
-
- if (count < sizeof(val))
- return -EINVAL;
- count = sizeof(val);
-
- addr = *offp;
- if (addr > fmc->memlen)
- return -ESPIPE; /* Illegal seek */
- val = fmc_readl(fmc, addr);
- if (copy_to_user(buf, &val, count))
- return -EFAULT;
- *offp += count;
- return count;
-}
-
-static ssize_t fc_write(struct file *f, const char __user *buf, size_t count,
- loff_t *offp)
-{
- struct fmc_device *fmc = f->private_data;
- unsigned long addr;
- uint32_t val;
-
- if (count < sizeof(val))
- return -EINVAL;
- count = sizeof(val);
-
- addr = *offp;
- if (addr > fmc->memlen)
- return -ESPIPE; /* Illegal seek */
- if (copy_from_user(&val, buf, count))
- return -EFAULT;
- fmc_writel(fmc, val, addr);
- *offp += count;
- return count;
-}
-
-static const struct file_operations fc_fops = {
- .owner = THIS_MODULE,
- .open = fc_open,
- .release = fc_release,
- .llseek = generic_file_llseek,
- .read = fc_read,
- .write = fc_write,
-};
-
-
-/* Device part .. */
-static int fc_probe(struct fmc_device *fmc);
-static int fc_remove(struct fmc_device *fmc);
-
-static struct fmc_driver fc_drv = {
- .version = FMC_VERSION,
- .driver.name = KBUILD_MODNAME,
- .probe = fc_probe,
- .remove = fc_remove,
- /* no table: we want to match everything */
-};
-
-/* We accept the generic busid parameter */
-FMC_PARAM_BUSID(fc_drv);
-
-/* probe and remove must allocate and release a misc device */
-static int fc_probe(struct fmc_device *fmc)
-{
- int ret;
- int index = 0;
-
- struct fc_instance *fc;
-
- index = fmc_validate(fmc, &fc_drv);
- if (index < 0)
- return -EINVAL; /* not our device: invalid */
-
- /* Create a char device: we want to create it anew */
- fc = kzalloc(sizeof(*fc), GFP_KERNEL);
- if (!fc)
- return -ENOMEM;
- fc->fmc = fmc;
- fc->misc.minor = MISC_DYNAMIC_MINOR;
- fc->misc.fops = &fc_fops;
- fc->misc.name = kstrdup(dev_name(&fmc->dev), GFP_KERNEL);
-
- ret = misc_register(&fc->misc);
- if (ret < 0)
- goto out;
- spin_lock(&fc_lock);
- list_add(&fc->list, &fc_devices);
- spin_unlock(&fc_lock);
- dev_info(&fc->fmc->dev, "Created misc device \"%s\"\n",
- fc->misc.name);
- return 0;
-
-out:
- kfree(fc->misc.name);
- kfree(fc);
- return ret;
-}
-
-static int fc_remove(struct fmc_device *fmc)
-{
- struct fc_instance *fc;
-
- list_for_each_entry(fc, &fc_devices, list)
- if (fc->fmc == fmc)
- break;
- if (fc->fmc != fmc) {
- dev_err(&fmc->dev, "remove called but not found\n");
- return -ENODEV;
- }
-
- spin_lock(&fc_lock);
- list_del(&fc->list);
- spin_unlock(&fc_lock);
- misc_deregister(&fc->misc);
- kfree(fc->misc.name);
- kfree(fc);
-
- return 0;
-}
-
-
-static int fc_init(void)
-{
- int ret;
-
- ret = fmc_driver_register(&fc_drv);
- return ret;
-}
-
-static void fc_exit(void)
-{
- fmc_driver_unregister(&fc_drv);
-}
-
-module_init(fc_init);
-module_exit(fc_exit);
-
-MODULE_LICENSE("GPL");
diff --git a/drivers/fmc/fmc-core.c b/drivers/fmc/fmc-core.c
deleted file mode 100644
index 573f5471f680..000000000000
--- a/drivers/fmc/fmc-core.c
+++ /dev/null
@@ -1,388 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Copyright (C) 2012 CERN (www.cern.ch)
- * Author: Alessandro Rubini <rubini@gnudd.com>
- *
- * This work is part of the White Rabbit project, a research effort led
- * by CERN, the European Institute for Nuclear Research.
- */
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/init.h>
-#include <linux/device.h>
-#include <linux/fmc.h>
-#include <linux/fmc-sdb.h>
-
-#include "fmc-private.h"
-
-static int fmc_check_version(unsigned long version, const char *name)
-{
- if (__FMC_MAJOR(version) != FMC_MAJOR) {
- pr_err("%s: \"%s\" has wrong major (has %li, expected %i)\n",
- __func__, name, __FMC_MAJOR(version), FMC_MAJOR);
- return -EINVAL;
- }
-
- if (__FMC_MINOR(version) != FMC_MINOR)
- pr_info("%s: \"%s\" has wrong minor (has %li, expected %i)\n",
- __func__, name, __FMC_MINOR(version), FMC_MINOR);
- return 0;
-}
-
-static int fmc_uevent(struct device *dev, struct kobj_uevent_env *env)
-{
- /* struct fmc_device *fdev = to_fmc_device(dev); */
-
- /* FIXME: The MODALIAS */
- add_uevent_var(env, "MODALIAS=%s", "fmc");
- return 0;
-}
-
-static int fmc_probe(struct device *dev)
-{
- struct fmc_driver *fdrv = to_fmc_driver(dev->driver);
- struct fmc_device *fdev = to_fmc_device(dev);
-
- return fdrv->probe(fdev);
-}
-
-static int fmc_remove(struct device *dev)
-{
- struct fmc_driver *fdrv = to_fmc_driver(dev->driver);
- struct fmc_device *fdev = to_fmc_device(dev);
-
- return fdrv->remove(fdev);
-}
-
-static void fmc_shutdown(struct device *dev)
-{
- /* not implemented but mandatory */
-}
-
-static struct bus_type fmc_bus_type = {
- .name = "fmc",
- .match = fmc_match,
- .uevent = fmc_uevent,
- .probe = fmc_probe,
- .remove = fmc_remove,
- .shutdown = fmc_shutdown,
-};
-
-static void fmc_release(struct device *dev)
-{
- struct fmc_device *fmc = container_of(dev, struct fmc_device, dev);
-
- kfree(fmc);
-}
-
-/*
- * The eeprom is exported in sysfs, through a binary attribute
- */
-
-static ssize_t fmc_read_eeprom(struct file *file, struct kobject *kobj,
- struct bin_attribute *bin_attr,
- char *buf, loff_t off, size_t count)
-{
- struct device *dev;
- struct fmc_device *fmc;
- int eelen;
-
- dev = container_of(kobj, struct device, kobj);
- fmc = container_of(dev, struct fmc_device, dev);
- eelen = fmc->eeprom_len;
- if (off > eelen)
- return -ESPIPE;
- if (off == eelen)
- return 0; /* EOF */
- if (off + count > eelen)
- count = eelen - off;
- memcpy(buf, fmc->eeprom + off, count);
- return count;
-}
-
-static ssize_t fmc_write_eeprom(struct file *file, struct kobject *kobj,
- struct bin_attribute *bin_attr,
- char *buf, loff_t off, size_t count)
-{
- struct device *dev;
- struct fmc_device *fmc;
-
- dev = container_of(kobj, struct device, kobj);
- fmc = container_of(dev, struct fmc_device, dev);
- return fmc->op->write_ee(fmc, off, buf, count);
-}
-
-static struct bin_attribute fmc_eeprom_attr = {
- .attr = { .name = "eeprom", .mode = S_IRUGO | S_IWUSR, },
- .size = 8192, /* more or less standard */
- .read = fmc_read_eeprom,
- .write = fmc_write_eeprom,
-};
-
-int fmc_irq_request(struct fmc_device *fmc, irq_handler_t h,
- char *name, int flags)
-{
- if (fmc->op->irq_request)
- return fmc->op->irq_request(fmc, h, name, flags);
- return -EPERM;
-}
-EXPORT_SYMBOL(fmc_irq_request);
-
-void fmc_irq_free(struct fmc_device *fmc)
-{
- if (fmc->op->irq_free)
- fmc->op->irq_free(fmc);
-}
-EXPORT_SYMBOL(fmc_irq_free);
-
-void fmc_irq_ack(struct fmc_device *fmc)
-{
- if (likely(fmc->op->irq_ack))
- fmc->op->irq_ack(fmc);
-}
-EXPORT_SYMBOL(fmc_irq_ack);
-
-int fmc_validate(struct fmc_device *fmc, struct fmc_driver *drv)
-{
- if (fmc->op->validate)
- return fmc->op->validate(fmc, drv);
- return -EPERM;
-}
-EXPORT_SYMBOL(fmc_validate);
-
-int fmc_gpio_config(struct fmc_device *fmc, struct fmc_gpio *gpio, int ngpio)
-{
- if (fmc->op->gpio_config)
- return fmc->op->gpio_config(fmc, gpio, ngpio);
- return -EPERM;
-}
-EXPORT_SYMBOL(fmc_gpio_config);
-
-int fmc_read_ee(struct fmc_device *fmc, int pos, void *d, int l)
-{
- if (fmc->op->read_ee)
- return fmc->op->read_ee(fmc, pos, d, l);
- return -EPERM;
-}
-EXPORT_SYMBOL(fmc_read_ee);
-
-int fmc_write_ee(struct fmc_device *fmc, int pos, const void *d, int l)
-{
- if (fmc->op->write_ee)
- return fmc->op->write_ee(fmc, pos, d, l);
- return -EPERM;
-}
-EXPORT_SYMBOL(fmc_write_ee);
-
-/*
- * Functions for client modules follow
- */
-
-int fmc_driver_register(struct fmc_driver *drv)
-{
- if (fmc_check_version(drv->version, drv->driver.name))
- return -EINVAL;
- drv->driver.bus = &fmc_bus_type;
- return driver_register(&drv->driver);
-}
-EXPORT_SYMBOL(fmc_driver_register);
-
-void fmc_driver_unregister(struct fmc_driver *drv)
-{
- driver_unregister(&drv->driver);
-}
-EXPORT_SYMBOL(fmc_driver_unregister);
-
-/*
- * When a device set is registered, all eeproms must be read
- * and all FRUs must be parsed
- */
-int fmc_device_register_n_gw(struct fmc_device **devs, int n,
- struct fmc_gateware *gw)
-{
- struct fmc_device *fmc, **devarray;
- uint32_t device_id;
- int i, ret = 0;
-
- if (n < 1)
- return 0;
-
- /* Check the version of the first data structure (function prints) */
- if (fmc_check_version(devs[0]->version, devs[0]->carrier_name))
- return -EINVAL;
-
- devarray = kmemdup(devs, n * sizeof(*devs), GFP_KERNEL);
- if (!devarray)
- return -ENOMEM;
-
- /* Make all other checks before continuing, for all devices */
- for (i = 0; i < n; i++) {
- fmc = devarray[i];
- if (!fmc->hwdev) {
- pr_err("%s: device nr. %i has no hwdev pointer\n",
- __func__, i);
- ret = -EINVAL;
- break;
- }
- if (fmc->flags & FMC_DEVICE_NO_MEZZANINE) {
- dev_info(fmc->hwdev, "absent mezzanine in slot %d\n",
- fmc->slot_id);
- continue;
- }
- if (!fmc->eeprom) {
- dev_err(fmc->hwdev, "no eeprom provided for slot %i\n",
- fmc->slot_id);
- ret = -EINVAL;
- }
- if (!fmc->eeprom_addr) {
- dev_err(fmc->hwdev, "no eeprom_addr for slot %i\n",
- fmc->slot_id);
- ret = -EINVAL;
- }
- if (!fmc->carrier_name || !fmc->carrier_data ||
- !fmc->device_id) {
- dev_err(fmc->hwdev,
- "device nr %i: carrier name, "
- "data or dev_id not set\n", i);
- ret = -EINVAL;
- }
- if (ret)
- break;
-
- }
- if (ret) {
- kfree(devarray);
- return ret;
- }
-
- /* Validation is ok. Now init and register the devices */
- for (i = 0; i < n; i++) {
- fmc = devarray[i];
-
- fmc->nr_slots = n; /* each slot must know how many are there */
- fmc->devarray = devarray;
-
- device_initialize(&fmc->dev);
- fmc->dev.release = fmc_release;
- fmc->dev.parent = fmc->hwdev;
-
- /* Fill the identification stuff (may fail) */
- fmc_fill_id_info(fmc);
-
- fmc->dev.bus = &fmc_bus_type;
-
- /* Name from mezzanine info or carrier info. Or 0,1,2.. */
- device_id = fmc->device_id;
- if (!fmc->mezzanine_name)
- dev_set_name(&fmc->dev, "fmc-%04x", device_id);
- else
- dev_set_name(&fmc->dev, "%s-%04x", fmc->mezzanine_name,
- device_id);
-
- if (gw) {
- /*
- * The carrier already know the bitstream to load
- * for this set of FMC mezzanines.
- */
- ret = fmc->op->reprogram_raw(fmc, NULL,
- gw->bitstream, gw->len);
- if (ret) {
- dev_warn(fmc->hwdev,
- "Invalid gateware for FMC mezzanine\n");
- goto out;
- }
- }
-
- ret = device_add(&fmc->dev);
- if (ret < 0) {
- dev_err(fmc->hwdev, "Slot %i: Failed in registering "
- "\"%s\"\n", fmc->slot_id, fmc->dev.kobj.name);
- goto out;
- }
- ret = sysfs_create_bin_file(&fmc->dev.kobj, &fmc_eeprom_attr);
- if (ret < 0) {
- dev_err(&fmc->dev, "Failed in registering eeprom\n");
- goto out1;
- }
- /* This device went well, give information to the user */
- fmc_dump_eeprom(fmc);
- fmc_debug_init(fmc);
- }
- return 0;
-
-out1:
- device_del(&fmc->dev);
-out:
- kfree(devarray);
- for (i--; i >= 0; i--) {
- fmc_debug_exit(devs[i]);
- sysfs_remove_bin_file(&devs[i]->dev.kobj, &fmc_eeprom_attr);
- device_del(&devs[i]->dev);
- fmc_free_id_info(devs[i]);
- put_device(&devs[i]->dev);
- }
- return ret;
-
-}
-EXPORT_SYMBOL(fmc_device_register_n_gw);
-
-int fmc_device_register_n(struct fmc_device **devs, int n)
-{
- return fmc_device_register_n_gw(devs, n, NULL);
-}
-EXPORT_SYMBOL(fmc_device_register_n);
-
-int fmc_device_register_gw(struct fmc_device *fmc, struct fmc_gateware *gw)
-{
- return fmc_device_register_n_gw(&fmc, 1, gw);
-}
-EXPORT_SYMBOL(fmc_device_register_gw);
-
-int fmc_device_register(struct fmc_device *fmc)
-{
- return fmc_device_register_n(&fmc, 1);
-}
-EXPORT_SYMBOL(fmc_device_register);
-
-void fmc_device_unregister_n(struct fmc_device **devs, int n)
-{
- int i;
-
- if (n < 1)
- return;
-
- /* Free devarray first, not used by the later loop */
- kfree(devs[0]->devarray);
-
- for (i = 0; i < n; i++) {
- fmc_debug_exit(devs[i]);
- sysfs_remove_bin_file(&devs[i]->dev.kobj, &fmc_eeprom_attr);
- device_del(&devs[i]->dev);
- fmc_free_id_info(devs[i]);
- put_device(&devs[i]->dev);
- }
-}
-EXPORT_SYMBOL(fmc_device_unregister_n);
-
-void fmc_device_unregister(struct fmc_device *fmc)
-{
- fmc_device_unregister_n(&fmc, 1);
-}
-EXPORT_SYMBOL(fmc_device_unregister);
-
-/* Init and exit are trivial */
-static int fmc_init(void)
-{
- return bus_register(&fmc_bus_type);
-}
-
-static void fmc_exit(void)
-{
- bus_unregister(&fmc_bus_type);
-}
-
-module_init(fmc_init);
-module_exit(fmc_exit);
-
-MODULE_LICENSE("GPL");
diff --git a/drivers/fmc/fmc-debug.c b/drivers/fmc/fmc-debug.c
deleted file mode 100644
index 1734c7cf0e76..000000000000
--- a/drivers/fmc/fmc-debug.c
+++ /dev/null
@@ -1,172 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Copyright (C) 2015 CERN (www.cern.ch)
- * Author: Federico Vaga <federico.vaga@cern.ch>
- */
-
-#include <linux/module.h>
-#include <linux/device.h>
-#include <linux/init.h>
-#include <linux/fs.h>
-#include <linux/debugfs.h>
-#include <linux/seq_file.h>
-#include <asm/byteorder.h>
-
-#include <linux/fmc.h>
-#include <linux/sdb.h>
-#include <linux/fmc-sdb.h>
-
-#define FMC_DBG_SDB_DUMP "dump_sdb"
-
-static char *__strip_trailing_space(char *buf, char *str, int len)
-{
- int i = len - 1;
-
- memcpy(buf, str, len);
- buf[len] = '\0';
- while (i >= 0 && buf[i] == ' ')
- buf[i--] = '\0';
- return buf;
-}
-
-#define __sdb_string(buf, field) ({ \
- BUILD_BUG_ON(sizeof(buf) < sizeof(field)); \
- __strip_trailing_space(buf, (void *)(field), sizeof(field)); \
- })
-
-/**
- * We do not check seq_printf() errors because we want to see things in any case
- */
-static void fmc_sdb_dump_recursive(struct fmc_device *fmc, struct seq_file *s,
- const struct sdb_array *arr)
-{
- unsigned long base = arr->baseaddr;
- int i, j, n = arr->len, level = arr->level;
- char tmp[64];
-
- for (i = 0; i < n; i++) {
- union sdb_record *r;
- struct sdb_product *p;
- struct sdb_component *c;
-
- r = &arr->record[i];
- c = &r->dev.sdb_component;
- p = &c->product;
-
- for (j = 0; j < level; j++)
- seq_printf(s, " ");
- switch (r->empty.record_type) {
- case sdb_type_interconnect:
- seq_printf(s, "%08llx:%08x %.19s\n",
- __be64_to_cpu(p->vendor_id),
- __be32_to_cpu(p->device_id),
- p->name);
- break;
- case sdb_type_device:
- seq_printf(s, "%08llx:%08x %.19s (%08llx-%08llx)\n",
- __be64_to_cpu(p->vendor_id),
- __be32_to_cpu(p->device_id),
- p->name,
- __be64_to_cpu(c->addr_first) + base,
- __be64_to_cpu(c->addr_last) + base);
- break;
- case sdb_type_bridge:
- seq_printf(s, "%08llx:%08x %.19s (bridge: %08llx)\n",
- __be64_to_cpu(p->vendor_id),
- __be32_to_cpu(p->device_id),
- p->name,
- __be64_to_cpu(c->addr_first) + base);
- if (IS_ERR(arr->subtree[i])) {
- seq_printf(s, "SDB: (bridge error %li)\n",
- PTR_ERR(arr->subtree[i]));
- break;
- }
- fmc_sdb_dump_recursive(fmc, s, arr->subtree[i]);
- break;
- case sdb_type_integration:
- seq_printf(s, "integration\n");
- break;
- case sdb_type_repo_url:
- seq_printf(s, "Synthesis repository: %s\n",
- __sdb_string(tmp, r->repo_url.repo_url));
- break;
- case sdb_type_synthesis:
- seq_printf(s, "Bitstream '%s' ",
- __sdb_string(tmp, r->synthesis.syn_name));
- seq_printf(s, "synthesized %08x by %s ",
- __be32_to_cpu(r->synthesis.date),
- __sdb_string(tmp, r->synthesis.user_name));
- seq_printf(s, "(%s version %x), ",
- __sdb_string(tmp, r->synthesis.tool_name),
- __be32_to_cpu(r->synthesis.tool_version));
- seq_printf(s, "commit %pm\n",
- r->synthesis.commit_id);
- break;
- case sdb_type_empty:
- seq_printf(s, "empty\n");
- break;
- default:
- seq_printf(s, "UNKNOWN TYPE 0x%02x\n",
- r->empty.record_type);
- break;
- }
- }
-}
-
-static int fmc_sdb_dump(struct seq_file *s, void *offset)
-{
- struct fmc_device *fmc = s->private;
-
- if (!fmc->sdb) {
- seq_printf(s, "no SDB information\n");
- return 0;
- }
-
- seq_printf(s, "FMC: %s (%s), slot %i, device %s\n", dev_name(fmc->hwdev),
- fmc->carrier_name, fmc->slot_id, dev_name(&fmc->dev));
- /* Dump SDB information */
- fmc_sdb_dump_recursive(fmc, s, fmc->sdb);
-
- return 0;
-}
-
-
-static int fmc_sdb_dump_open(struct inode *inode, struct file *file)
-{
- struct fmc_device *fmc = inode->i_private;
-
- return single_open(file, fmc_sdb_dump, fmc);
-}
-
-
-const struct file_operations fmc_dbgfs_sdb_dump = {
- .owner = THIS_MODULE,
- .open = fmc_sdb_dump_open,
- .read = seq_read,
- .llseek = seq_lseek,
- .release = single_release,
-};
-
-int fmc_debug_init(struct fmc_device *fmc)
-{
- fmc->dbg_dir = debugfs_create_dir(dev_name(&fmc->dev), NULL);
- if (IS_ERR_OR_NULL(fmc->dbg_dir)) {
- pr_err("FMC: Cannot create debugfs\n");
- return PTR_ERR(fmc->dbg_dir);
- }
-
- fmc->dbg_sdb_dump = debugfs_create_file(FMC_DBG_SDB_DUMP, 0444,
- fmc->dbg_dir, fmc,
- &fmc_dbgfs_sdb_dump);
- if (IS_ERR_OR_NULL(fmc->dbg_sdb_dump))
- pr_err("FMC: Cannot create debugfs file %s\n",
- FMC_DBG_SDB_DUMP);
-
- return 0;
-}
-
-void fmc_debug_exit(struct fmc_device *fmc)
-{
- if (fmc->dbg_dir)
- debugfs_remove_recursive(fmc->dbg_dir);
-}
diff --git a/drivers/fmc/fmc-dump.c b/drivers/fmc/fmc-dump.c
deleted file mode 100644
index 6c81dbde1d16..000000000000
--- a/drivers/fmc/fmc-dump.c
+++ /dev/null
@@ -1,58 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Copyright (C) 2013 CERN (www.cern.ch)
- * Author: Alessandro Rubini <rubini@gnudd.com>
- *
- * This work is part of the White Rabbit project, a research effort led
- * by CERN, the European Institute for Nuclear Research.
- */
-#include <linux/kernel.h>
-#include <linux/moduleparam.h>
-#include <linux/device.h>
-#include <linux/fmc.h>
-#include <linux/fmc-sdb.h>
-
-static int fmc_must_dump_eeprom;
-module_param_named(dump_eeprom, fmc_must_dump_eeprom, int, 0644);
-
-#define LINELEN 16
-
-/* Dumping 8k takes oh so much: avoid duplicate lines */
-static const uint8_t *dump_line(int addr, const uint8_t *line,
- const uint8_t *prev)
-{
- int i;
-
- if (!prev || memcmp(line, prev, LINELEN)) {
- pr_info("%04x: ", addr);
- for (i = 0; i < LINELEN; ) {
- printk(KERN_CONT "%02x", line[i]);
- i++;
- printk(i & 3 ? " " : i & (LINELEN - 1) ? " " : "\n");
- }
- return line;
- }
- /* repeated line */
- if (line == prev + LINELEN)
- pr_info("[...]\n");
- return prev;
-}
-
-void fmc_dump_eeprom(const struct fmc_device *fmc)
-{
- const uint8_t *line, *prev;
- int i;
-
- if (!fmc_must_dump_eeprom)
- return;
-
- pr_info("FMC: %s (%s), slot %i, device %s\n", dev_name(fmc->hwdev),
- fmc->carrier_name, fmc->slot_id, dev_name(&fmc->dev));
- pr_info("FMC: dumping eeprom 0x%x (%i) bytes\n", fmc->eeprom_len,
- fmc->eeprom_len);
-
- line = fmc->eeprom;
- prev = NULL;
- for (i = 0; i < fmc->eeprom_len; i += LINELEN, line += LINELEN)
- prev = dump_line(i, line, prev);
-}
diff --git a/drivers/fmc/fmc-fakedev.c b/drivers/fmc/fmc-fakedev.c
deleted file mode 100644
index 941d0930969a..000000000000
--- a/drivers/fmc/fmc-fakedev.c
+++ /dev/null
@@ -1,355 +0,0 @@
-/*
- * Copyright (C) 2012 CERN (www.cern.ch)
- * Author: Alessandro Rubini <rubini@gnudd.com>
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * The software is provided "as is"; the copyright holders disclaim
- * all warranties and liabilities, to the extent permitted by
- * applicable law.
- */
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/string.h>
-#include <linux/device.h>
-#include <linux/slab.h>
-#include <linux/firmware.h>
-#include <linux/workqueue.h>
-#include <linux/err.h>
-#include <linux/fmc.h>
-
-#define FF_EEPROM_SIZE 8192 /* The standard eeprom size */
-#define FF_MAX_MEZZANINES 4 /* Fakes a multi-mezzanine carrier */
-
-/* The user can pass up to 4 names of eeprom images to load */
-static char *ff_eeprom[FF_MAX_MEZZANINES];
-static int ff_nr_eeprom;
-module_param_array_named(eeprom, ff_eeprom, charp, &ff_nr_eeprom, 0444);
-
-/* The user can ask for a multi-mezzanine carrier, with the default eeprom */
-static int ff_nr_dev = 1;
-module_param_named(ndev, ff_nr_dev, int, 0444);
-
-
-/* Lazily, don't support the "standard" module parameters */
-
-/*
- * Eeprom built from these commands:
-
- ../fru-generator -v fake-vendor -n fake-design-for-testing \
- -s 01234 -p none > IPMI-FRU
-
- gensdbfs . ../fake-eeprom.bin
-*/
-static char ff_eeimg[FF_MAX_MEZZANINES][FF_EEPROM_SIZE] = {
- {
- 0x01, 0x00, 0x00, 0x01, 0x00, 0x0c, 0x00, 0xf2, 0x01, 0x0b, 0x00, 0xb2,
- 0x86, 0x87, 0xcb, 0x66, 0x61, 0x6b, 0x65, 0x2d, 0x76, 0x65, 0x6e, 0x64,
- 0x6f, 0x72, 0xd7, 0x66, 0x61, 0x6b, 0x65, 0x2d, 0x64, 0x65, 0x73, 0x69,
- 0x67, 0x6e, 0x2d, 0x66, 0x6f, 0x72, 0x2d, 0x74, 0x65, 0x73, 0x74, 0x69,
- 0x6e, 0x67, 0xc5, 0x30, 0x31, 0x32, 0x33, 0x34, 0xc4, 0x6e, 0x6f, 0x6e,
- 0x65, 0xda, 0x32, 0x30, 0x31, 0x32, 0x2d, 0x31, 0x31, 0x2d, 0x31, 0x39,
- 0x20, 0x32, 0x32, 0x3a, 0x34, 0x32, 0x3a, 0x33, 0x30, 0x2e, 0x30, 0x37,
- 0x34, 0x30, 0x35, 0x35, 0xc1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x87,
- 0x02, 0x02, 0x0d, 0xf7, 0xf8, 0x02, 0xb0, 0x04, 0x74, 0x04, 0xec, 0x04,
- 0x00, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x02, 0x02, 0x0d, 0x5c, 0x93, 0x01,
- 0x4a, 0x01, 0x39, 0x01, 0x5a, 0x01, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x0b,
- 0x02, 0x02, 0x0d, 0x63, 0x8c, 0x00, 0xfa, 0x00, 0xed, 0x00, 0x06, 0x01,
- 0x00, 0x00, 0x00, 0x00, 0xa0, 0x0f, 0x01, 0x02, 0x0d, 0xfb, 0xf5, 0x05,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x01, 0x02, 0x0d, 0xfc, 0xf4, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x0d, 0xfd, 0xf3, 0x03,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0xfa, 0x82, 0x0b, 0xea, 0x8f, 0xa2, 0x12, 0x00, 0x00, 0x1e, 0x44, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x53, 0x44, 0x42, 0x2d, 0x00, 0x03, 0x01, 0x01,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x01, 0xc4, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x61, 0x74, 0x61,
- 0x2e, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00,
- 0x2e, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
- 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xc0,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xc4, 0x46, 0x69, 0x6c, 0x65,
- 0x44, 0x61, 0x74, 0x61, 0x6e, 0x61, 0x6d, 0x65, 0x00, 0x00, 0x00, 0x01,
- 0x00, 0x00, 0x00, 0x00, 0x6e, 0x61, 0x6d, 0x65, 0x20, 0x20, 0x20, 0x20,
- 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x01,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdf,
- 0x46, 0x69, 0x6c, 0x65, 0x44, 0x61, 0x74, 0x61, 0x49, 0x50, 0x4d, 0x49,
- 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x49, 0x50, 0x4d, 0x49,
- 0x2d, 0x46, 0x52, 0x55, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
- 0x20, 0x20, 0x20, 0x01, 0x66, 0x61, 0x6b, 0x65, 0x0a,
- },
-};
-
-struct ff_dev {
- struct fmc_device *fmc[FF_MAX_MEZZANINES];
- struct device dev;
-};
-
-static struct ff_dev *ff_current_dev; /* We have 1 carrier, 1 slot */
-
-static int ff_reprogram(struct fmc_device *fmc, struct fmc_driver *drv,
- char *gw)
-{
- const struct firmware *fw;
- int ret;
-
- if (!gw) {
- /* program golden: success */
- fmc->flags &= ~FMC_DEVICE_HAS_CUSTOM;
- fmc->flags |= FMC_DEVICE_HAS_GOLDEN;
- return 0;
- }
-
- dev_info(&fmc->dev, "reprogramming with %s\n", gw);
- ret = request_firmware(&fw, gw, &fmc->dev);
- if (ret < 0) {
- dev_warn(&fmc->dev, "request firmware \"%s\": error %i\n",
- gw, ret);
- goto out;
- }
- fmc->flags &= ~FMC_DEVICE_HAS_GOLDEN;
- fmc->flags |= FMC_DEVICE_HAS_CUSTOM;
-
-out:
- release_firmware(fw);
- return ret;
-}
-
-static int ff_irq_request(struct fmc_device *fmc, irq_handler_t handler,
- char *name, int flags)
-{
- return -EOPNOTSUPP;
-}
-
-/* FIXME: should also have some fake FMC GPIO mapping */
-
-
-/*
- * This work function is called when we changed the eeprom. It removes the
- * current fmc device and registers a new one, with different identifiers.
- */
-static struct ff_dev *ff_dev_create(void); /* defined later */
-
-static void ff_work_fn(struct work_struct *work)
-{
- struct ff_dev *ff = ff_current_dev;
- int ret;
-
- fmc_device_unregister_n(ff->fmc, ff_nr_dev);
- device_unregister(&ff->dev);
- ff_current_dev = NULL;
-
- ff = ff_dev_create();
- if (IS_ERR(ff)) {
- pr_warning("%s: can't re-create FMC devices\n", __func__);
- return;
- }
- ret = fmc_device_register_n(ff->fmc, ff_nr_dev);
- if (ret < 0) {
- dev_warn(&ff->dev, "can't re-register FMC devices\n");
- device_unregister(&ff->dev);
- return;
- }
-
- ff_current_dev = ff;
-}
-
-static DECLARE_DELAYED_WORK(ff_work, ff_work_fn);
-
-
-/* low-level i2c */
-static int ff_eeprom_read(struct fmc_device *fmc, uint32_t offset,
- void *buf, size_t size)
-{
- if (offset > FF_EEPROM_SIZE)
- return -EINVAL;
- if (offset + size > FF_EEPROM_SIZE)
- size = FF_EEPROM_SIZE - offset;
- memcpy(buf, fmc->eeprom + offset, size);
- return size;
-}
-
-static int ff_eeprom_write(struct fmc_device *fmc, uint32_t offset,
- const void *buf, size_t size)
-{
- if (offset > FF_EEPROM_SIZE)
- return -EINVAL;
- if (offset + size > FF_EEPROM_SIZE)
- size = FF_EEPROM_SIZE - offset;
- dev_info(&fmc->dev, "write_eeprom: offset %i, size %zi\n",
- (int)offset, size);
- memcpy(fmc->eeprom + offset, buf, size);
- schedule_delayed_work(&ff_work, HZ * 2); /* remove, replug, in 2s */
- return size;
-}
-
-/* i2c operations for fmc */
-static int ff_read_ee(struct fmc_device *fmc, int pos, void *data, int len)
-{
- if (!(fmc->flags & FMC_DEVICE_HAS_GOLDEN))
- return -EOPNOTSUPP;
- return ff_eeprom_read(fmc, pos, data, len);
-}
-
-static int ff_write_ee(struct fmc_device *fmc, int pos,
- const void *data, int len)
-{
- if (!(fmc->flags & FMC_DEVICE_HAS_GOLDEN))
- return -EOPNOTSUPP;
- return ff_eeprom_write(fmc, pos, data, len);
-}
-
-/* readl and writel do not do anything. Don't waste RAM with "base" */
-static uint32_t ff_readl(struct fmc_device *fmc, int offset)
-{
- return 0;
-}
-
-static void ff_writel(struct fmc_device *fmc, uint32_t value, int offset)
-{
- return;
-}
-
-/* validate is useful so fmc-write-eeprom will not reprogram every 2 seconds */
-static int ff_validate(struct fmc_device *fmc, struct fmc_driver *drv)
-{
- int i;
-
- if (!drv->busid_n)
- return 0; /* everyhing is valid */
- for (i = 0; i < drv->busid_n; i++)
- if (drv->busid_val[i] == fmc->device_id)
- return i;
- return -ENOENT;
-}
-
-
-
-static struct fmc_operations ff_fmc_operations = {
- .read32 = ff_readl,
- .write32 = ff_writel,
- .reprogram = ff_reprogram,
- .irq_request = ff_irq_request,
- .read_ee = ff_read_ee,
- .write_ee = ff_write_ee,
- .validate = ff_validate,
-};
-
-/* This device is kmalloced: release it */
-static void ff_dev_release(struct device *dev)
-{
- struct ff_dev *ff = container_of(dev, struct ff_dev, dev);
- kfree(ff);
-}
-
-static struct fmc_device ff_template_fmc = {
- .version = FMC_VERSION,
- .owner = THIS_MODULE,
- .carrier_name = "fake-fmc-carrier",
- .device_id = 0xf001, /* fool */
- .eeprom_len = sizeof(ff_eeimg[0]),
- .memlen = 0x1000, /* 4k, to show something */
- .op = &ff_fmc_operations,
- .hwdev = NULL, /* filled at creation time */
- .flags = FMC_DEVICE_HAS_GOLDEN,
-};
-
-static struct ff_dev *ff_dev_create(void)
-{
- struct ff_dev *ff;
- struct fmc_device *fmc;
- int i, ret;
-
- ff = kzalloc(sizeof(*ff), GFP_KERNEL);
- if (!ff)
- return ERR_PTR(-ENOMEM);
- dev_set_name(&ff->dev, "fake-fmc-carrier");
- ff->dev.release = ff_dev_release;
-
- ret = device_register(&ff->dev);
- if (ret < 0) {
- put_device(&ff->dev);
- return ERR_PTR(ret);
- }
-
- /* Create fmc structures that refer to this new "hw" device */
- for (i = 0; i < ff_nr_dev; i++) {
- fmc = kmemdup(&ff_template_fmc, sizeof(ff_template_fmc),
- GFP_KERNEL);
- fmc->hwdev = &ff->dev;
- fmc->carrier_data = ff;
- fmc->nr_slots = ff_nr_dev;
- /* the following fields are different for each slot */
- fmc->eeprom = ff_eeimg[i];
- fmc->eeprom_addr = 0x50 + 2 * i;
- fmc->slot_id = i;
- ff->fmc[i] = fmc;
- /* increment the identifier, each must be different */
- ff_template_fmc.device_id++;
- }
- return ff;
-}
-
-/* init and exit */
-static int ff_init(void)
-{
- struct ff_dev *ff;
- const struct firmware *fw;
- int i, len, ret = 0;
-
- /* Replicate the default eeprom for the max number of mezzanines */
- for (i = 1; i < FF_MAX_MEZZANINES; i++)
- memcpy(ff_eeimg[i], ff_eeimg[0], sizeof(ff_eeimg[0]));
-
- if (ff_nr_eeprom > ff_nr_dev)
- ff_nr_dev = ff_nr_eeprom;
-
- ff = ff_dev_create();
- if (IS_ERR(ff))
- return PTR_ERR(ff);
-
- /* If the user passed "eeprom=" as a parameter, fetch them */
- for (i = 0; i < ff_nr_eeprom; i++) {
- if (!strlen(ff_eeprom[i]))
- continue;
- ret = request_firmware(&fw, ff_eeprom[i], &ff->dev);
- if (ret < 0) {
- dev_err(&ff->dev, "Mezzanine %i: can't load \"%s\" "
- "(error %i)\n", i, ff_eeprom[i], -ret);
- } else {
- len = min_t(size_t, fw->size, (size_t)FF_EEPROM_SIZE);
- memcpy(ff_eeimg[i], fw->data, len);
- release_firmware(fw);
- dev_info(&ff->dev, "Mezzanine %i: eeprom \"%s\"\n", i,
- ff_eeprom[i]);
- }
- }
-
- ret = fmc_device_register_n(ff->fmc, ff_nr_dev);
- if (ret) {
- device_unregister(&ff->dev);
- return ret;
- }
- ff_current_dev = ff;
- return ret;
-}
-
-static void ff_exit(void)
-{
- if (ff_current_dev) {
- fmc_device_unregister_n(ff_current_dev->fmc, ff_nr_dev);
- device_unregister(&ff_current_dev->dev);
- }
- cancel_delayed_work_sync(&ff_work);
-}
-
-module_init(ff_init);
-module_exit(ff_exit);
-
-MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/fmc/fmc-match.c b/drivers/fmc/fmc-match.c
deleted file mode 100644
index 995bd6041a67..000000000000
--- a/drivers/fmc/fmc-match.c
+++ /dev/null
@@ -1,113 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Copyright (C) 2012 CERN (www.cern.ch)
- * Author: Alessandro Rubini <rubini@gnudd.com>
- *
- * This work is part of the White Rabbit project, a research effort led
- * by CERN, the European Institute for Nuclear Research.
- */
-#include <linux/kernel.h>
-#include <linux/slab.h>
-#include <linux/fmc.h>
-#include <linux/ipmi-fru.h>
-
-/* The fru parser is both user and kernel capable: it needs alloc */
-void *fru_alloc(size_t size)
-{
- return kzalloc(size, GFP_KERNEL);
-}
-
-/* The actual match function */
-int fmc_match(struct device *dev, struct device_driver *drv)
-{
- struct fmc_driver *fdrv = to_fmc_driver(drv);
- struct fmc_device *fdev = to_fmc_device(dev);
- struct fmc_fru_id *fid;
- int i, matched = 0;
-
- /* This currently only matches the EEPROM (FRU id) */
- fid = fdrv->id_table.fru_id;
- if (!fid) {
- dev_warn(&fdev->dev, "Driver has no ID: matches all\n");
- matched = 1;
- } else {
- if (!fdev->id.manufacturer || !fdev->id.product_name)
- return 0; /* the device has no FRU information */
- for (i = 0; i < fdrv->id_table.fru_id_nr; i++, fid++) {
- if (fid->manufacturer &&
- strcmp(fid->manufacturer, fdev->id.manufacturer))
- continue;
- if (fid->product_name &&
- strcmp(fid->product_name, fdev->id.product_name))
- continue;
- matched = 1;
- break;
- }
- }
-
- /* FIXME: match SDB contents */
- return matched;
-}
-
-/* This function creates ID info for a newly registered device */
-int fmc_fill_id_info(struct fmc_device *fmc)
-{
- struct fru_common_header *h;
- struct fru_board_info_area *bia;
- int ret, allocated = 0;
-
- /* If we know the eeprom length, try to read it off the device */
- if (fmc->eeprom_len && !fmc->eeprom) {
- fmc->eeprom = kzalloc(fmc->eeprom_len, GFP_KERNEL);
- if (!fmc->eeprom)
- return -ENOMEM;
- allocated = 1;
- ret = fmc_read_ee(fmc, 0, fmc->eeprom, fmc->eeprom_len);
- if (ret < 0)
- goto out;
- }
-
- /* If no eeprom, continue with other matches */
- if (!fmc->eeprom)
- return 0;
-
- dev_info(fmc->hwdev, "mezzanine %i\n", fmc->slot_id); /* header */
-
- /* So we have the eeprom: parse the FRU part (if any) */
- h = (void *)fmc->eeprom;
- if (h->format != 1) {
- pr_info(" EEPROM has no FRU information\n");
- goto out;
- }
- if (!fru_header_cksum_ok(h)) {
- pr_info(" FRU: wrong header checksum\n");
- goto out;
- }
- bia = fru_get_board_area(h);
- if (!fru_bia_cksum_ok(bia)) {
- pr_info(" FRU: wrong board area checksum\n");
- goto out;
- }
- fmc->id.manufacturer = fru_get_board_manufacturer(h);
- fmc->id.product_name = fru_get_product_name(h);
- pr_info(" Manufacturer: %s\n", fmc->id.manufacturer);
- pr_info(" Product name: %s\n", fmc->id.product_name);
-
- /* Create the short name (FIXME: look in sdb as well) */
- fmc->mezzanine_name = kstrdup(fmc->id.product_name, GFP_KERNEL);
-
-out:
- if (allocated) {
- kfree(fmc->eeprom);
- fmc->eeprom = NULL;
- }
- return 0; /* no error: let other identification work */
-}
-
-/* Some ID data is allocated using fru_alloc() above, so release it */
-void fmc_free_id_info(struct fmc_device *fmc)
-{
- kfree(fmc->mezzanine_name);
- kfree(fmc->id.manufacturer);
- kfree(fmc->id.product_name);
-}
diff --git a/drivers/fmc/fmc-private.h b/drivers/fmc/fmc-private.h
deleted file mode 100644
index 93cb8030f764..000000000000
--- a/drivers/fmc/fmc-private.h
+++ /dev/null
@@ -1,8 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright (C) 2015 CERN (www.cern.ch)
- * Author: Federico Vaga <federico.vaga@cern.ch>
- */
-
-extern int fmc_debug_init(struct fmc_device *fmc);
-extern void fmc_debug_exit(struct fmc_device *fmc);
diff --git a/drivers/fmc/fmc-sdb.c b/drivers/fmc/fmc-sdb.c
deleted file mode 100644
index 14758db1a5fb..000000000000
--- a/drivers/fmc/fmc-sdb.c
+++ /dev/null
@@ -1,219 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Copyright (C) 2012 CERN (www.cern.ch)
- * Author: Alessandro Rubini <rubini@gnudd.com>
- *
- * This work is part of the White Rabbit project, a research effort led
- * by CERN, the European Institute for Nuclear Research.
- */
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/fmc.h>
-#include <linux/sdb.h>
-#include <linux/err.h>
-#include <linux/fmc-sdb.h>
-#include <asm/byteorder.h>
-
-static uint32_t __sdb_rd(struct fmc_device *fmc, unsigned long address,
- int convert)
-{
- uint32_t res = fmc_readl(fmc, address);
- if (convert)
- return __be32_to_cpu(res);
- return res;
-}
-
-static struct sdb_array *__fmc_scan_sdb_tree(struct fmc_device *fmc,
- unsigned long sdb_addr,
- unsigned long reg_base, int level)
-{
- uint32_t onew;
- int i, j, n, convert = 0;
- struct sdb_array *arr, *sub;
-
- onew = fmc_readl(fmc, sdb_addr);
- if (onew == SDB_MAGIC) {
- /* Uh! If we are little-endian, we must convert */
- if (SDB_MAGIC != __be32_to_cpu(SDB_MAGIC))
- convert = 1;
- } else if (onew == __be32_to_cpu(SDB_MAGIC)) {
- /* ok, don't convert */
- } else {
- return ERR_PTR(-ENOENT);
- }
- /* So, the magic was there: get the count from offset 4*/
- onew = __sdb_rd(fmc, sdb_addr + 4, convert);
- n = __be16_to_cpu(*(uint16_t *)&onew);
- arr = kzalloc(sizeof(*arr), GFP_KERNEL);
- if (!arr)
- return ERR_PTR(-ENOMEM);
- arr->record = kcalloc(n, sizeof(arr->record[0]), GFP_KERNEL);
- arr->subtree = kcalloc(n, sizeof(arr->subtree[0]), GFP_KERNEL);
- if (!arr->record || !arr->subtree) {
- kfree(arr->record);
- kfree(arr->subtree);
- kfree(arr);
- return ERR_PTR(-ENOMEM);
- }
-
- arr->len = n;
- arr->level = level;
- arr->fmc = fmc;
- for (i = 0; i < n; i++) {
- union sdb_record *r;
-
- for (j = 0; j < sizeof(arr->record[0]); j += 4) {
- *(uint32_t *)((void *)(arr->record + i) + j) =
- __sdb_rd(fmc, sdb_addr + (i * 64) + j, convert);
- }
- r = &arr->record[i];
- arr->subtree[i] = ERR_PTR(-ENODEV);
- if (r->empty.record_type == sdb_type_bridge) {
- struct sdb_component *c = &r->bridge.sdb_component;
- uint64_t subaddr = __be64_to_cpu(r->bridge.sdb_child);
- uint64_t newbase = __be64_to_cpu(c->addr_first);
-
- subaddr += reg_base;
- newbase += reg_base;
- sub = __fmc_scan_sdb_tree(fmc, subaddr, newbase,
- level + 1);
- arr->subtree[i] = sub; /* may be error */
- if (IS_ERR(sub))
- continue;
- sub->parent = arr;
- sub->baseaddr = newbase;
- }
- }
- return arr;
-}
-
-int fmc_scan_sdb_tree(struct fmc_device *fmc, unsigned long address)
-{
- struct sdb_array *ret;
- if (fmc->sdb)
- return -EBUSY;
- ret = __fmc_scan_sdb_tree(fmc, address, 0 /* regs */, 0);
- if (IS_ERR(ret))
- return PTR_ERR(ret);
- fmc->sdb = ret;
- return 0;
-}
-EXPORT_SYMBOL(fmc_scan_sdb_tree);
-
-static void __fmc_sdb_free(struct sdb_array *arr)
-{
- int i, n;
-
- if (!arr)
- return;
- n = arr->len;
- for (i = 0; i < n; i++) {
- if (IS_ERR(arr->subtree[i]))
- continue;
- __fmc_sdb_free(arr->subtree[i]);
- }
- kfree(arr->record);
- kfree(arr->subtree);
- kfree(arr);
-}
-
-int fmc_free_sdb_tree(struct fmc_device *fmc)
-{
- __fmc_sdb_free(fmc->sdb);
- fmc->sdb = NULL;
- return 0;
-}
-EXPORT_SYMBOL(fmc_free_sdb_tree);
-
-/* This helper calls reprogram and inizialized sdb as well */
-int fmc_reprogram_raw(struct fmc_device *fmc, struct fmc_driver *d,
- void *gw, unsigned long len, int sdb_entry)
-{
- int ret;
-
- ret = fmc->op->reprogram_raw(fmc, d, gw, len);
- if (ret < 0)
- return ret;
- if (sdb_entry < 0)
- return ret;
-
- /* We are required to find SDB at a given offset */
- ret = fmc_scan_sdb_tree(fmc, sdb_entry);
- if (ret < 0) {
- dev_err(&fmc->dev, "Can't find SDB at address 0x%x\n",
- sdb_entry);
- return -ENODEV;
- }
-
- return 0;
-}
-EXPORT_SYMBOL(fmc_reprogram_raw);
-
-/* This helper calls reprogram and inizialized sdb as well */
-int fmc_reprogram(struct fmc_device *fmc, struct fmc_driver *d, char *gw,
- int sdb_entry)
-{
- int ret;
-
- ret = fmc->op->reprogram(fmc, d, gw);
- if (ret < 0)
- return ret;
- if (sdb_entry < 0)
- return ret;
-
- /* We are required to find SDB at a given offset */
- ret = fmc_scan_sdb_tree(fmc, sdb_entry);
- if (ret < 0) {
- dev_err(&fmc->dev, "Can't find SDB at address 0x%x\n",
- sdb_entry);
- return -ENODEV;
- }
-
- return 0;
-}
-EXPORT_SYMBOL(fmc_reprogram);
-
-void fmc_show_sdb_tree(const struct fmc_device *fmc)
-{
- pr_err("%s: not supported anymore, use debugfs to dump SDB\n",
- __func__);
-}
-EXPORT_SYMBOL(fmc_show_sdb_tree);
-
-signed long fmc_find_sdb_device(struct sdb_array *tree,
- uint64_t vid, uint32_t did, unsigned long *sz)
-{
- signed long res = -ENODEV;
- union sdb_record *r;
- struct sdb_product *p;
- struct sdb_component *c;
- int i, n = tree->len;
- uint64_t last, first;
-
- /* FIXME: what if the first interconnect is not at zero? */
- for (i = 0; i < n; i++) {
- r = &tree->record[i];
- c = &r->dev.sdb_component;
- p = &c->product;
-
- if (!IS_ERR(tree->subtree[i]))
- res = fmc_find_sdb_device(tree->subtree[i],
- vid, did, sz);
- if (res >= 0)
- return res + tree->baseaddr;
- if (r->empty.record_type != sdb_type_device)
- continue;
- if (__be64_to_cpu(p->vendor_id) != vid)
- continue;
- if (__be32_to_cpu(p->device_id) != did)
- continue;
- /* found */
- last = __be64_to_cpu(c->addr_last);
- first = __be64_to_cpu(c->addr_first);
- if (sz)
- *sz = (typeof(*sz))(last + 1 - first);
- return first + tree->baseaddr;
- }
- return res;
-}
-EXPORT_SYMBOL(fmc_find_sdb_device);
diff --git a/drivers/fmc/fmc-trivial.c b/drivers/fmc/fmc-trivial.c
deleted file mode 100644
index 8defdee3e3a3..000000000000
--- a/drivers/fmc/fmc-trivial.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Copyright (C) 2012 CERN (www.cern.ch)
- * Author: Alessandro Rubini <rubini@gnudd.com>
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * The software is provided "as is"; the copyright holders disclaim
- * all warranties and liabilities, to the extent permitted by
- * applicable law.
- */
-
-/* A trivial fmc driver that can load a gateware file and reports interrupts */
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/gpio.h>
-#include <linux/fmc.h>
-
-static struct fmc_driver t_drv; /* initialized later */
-
-static irqreturn_t t_handler(int irq, void *dev_id)
-{
- struct fmc_device *fmc = dev_id;
-
- fmc_irq_ack(fmc);
- dev_info(&fmc->dev, "received irq %i\n", irq);
- return IRQ_HANDLED;
-}
-
-static struct fmc_gpio t_gpio[] = {
- {
- .gpio = FMC_GPIO_IRQ(0),
- .mode = GPIOF_DIR_IN,
- .irqmode = IRQF_TRIGGER_RISING,
- }, {
- .gpio = FMC_GPIO_IRQ(1),
- .mode = GPIOF_DIR_IN,
- .irqmode = IRQF_TRIGGER_RISING,
- }
-};
-
-static int t_probe(struct fmc_device *fmc)
-{
- int ret;
- int index = 0;
-
- index = fmc_validate(fmc, &t_drv);
- if (index < 0)
- return -EINVAL; /* not our device: invalid */
-
- ret = fmc_irq_request(fmc, t_handler, "fmc-trivial", IRQF_SHARED);
- if (ret < 0)
- return ret;
- /* ignore error code of call below, we really don't care */
- fmc_gpio_config(fmc, t_gpio, ARRAY_SIZE(t_gpio));
-
- ret = fmc_reprogram(fmc, &t_drv, "", 0);
- if (ret == -EPERM) /* programming not supported */
- ret = 0;
- if (ret < 0)
- fmc_irq_free(fmc);
-
- /* FIXME: reprogram LM32 too */
- return ret;
-}
-
-static int t_remove(struct fmc_device *fmc)
-{
- fmc_irq_free(fmc);
- return 0;
-}
-
-static struct fmc_driver t_drv = {
- .version = FMC_VERSION,
- .driver.name = KBUILD_MODNAME,
- .probe = t_probe,
- .remove = t_remove,
- /* no table, as the current match just matches everything */
-};
-
- /* We accept the generic parameters */
-FMC_PARAM_BUSID(t_drv);
-FMC_PARAM_GATEWARE(t_drv);
-
-static int t_init(void)
-{
- int ret;
-
- ret = fmc_driver_register(&t_drv);
- return ret;
-}
-
-static void t_exit(void)
-{
- fmc_driver_unregister(&t_drv);
-}
-
-module_init(t_init);
-module_exit(t_exit);
-
-MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/fmc/fmc-write-eeprom.c b/drivers/fmc/fmc-write-eeprom.c
deleted file mode 100644
index 1c7826e3f526..000000000000
--- a/drivers/fmc/fmc-write-eeprom.c
+++ /dev/null
@@ -1,175 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Copyright (C) 2012 CERN (www.cern.ch)
- * Author: Alessandro Rubini <rubini@gnudd.com>
- *
- * This work is part of the White Rabbit project, a research effort led
- * by CERN, the European Institute for Nuclear Research.
- */
-#include <linux/module.h>
-#include <linux/string.h>
-#include <linux/firmware.h>
-#include <linux/init.h>
-#include <linux/fmc.h>
-#include <asm/unaligned.h>
-
-/*
- * This module uses the firmware loader to program the whole or part
- * of the FMC eeprom. The meat is in the _run functions. However, no
- * default file name is provided, to avoid accidental mishaps. Also,
- * you must pass the busid argument
- */
-static struct fmc_driver fwe_drv;
-
-FMC_PARAM_BUSID(fwe_drv);
-
-/* The "file=" is like the generic "gateware=" used elsewhere */
-static char *fwe_file[FMC_MAX_CARDS];
-static int fwe_file_n;
-module_param_array_named(file, fwe_file, charp, &fwe_file_n, 0444);
-
-static int fwe_run_tlv(struct fmc_device *fmc, const struct firmware *fw,
- int write)
-{
- const uint8_t *p = fw->data;
- int len = fw->size;
- uint16_t thislen, thisaddr;
- int err;
-
- /* format is: 'w' addr16 len16 data... */
- while (len > 5) {
- thisaddr = get_unaligned_le16(p+1);
- thislen = get_unaligned_le16(p+3);
- if (p[0] != 'w' || thislen + 5 > len) {
- dev_err(&fmc->dev, "invalid tlv at offset %ti\n",
- p - fw->data);
- return -EINVAL;
- }
- err = 0;
- if (write) {
- dev_info(&fmc->dev, "write %i bytes at 0x%04x\n",
- thislen, thisaddr);
- err = fmc_write_ee(fmc, thisaddr, p + 5, thislen);
- }
- if (err < 0) {
- dev_err(&fmc->dev, "write failure @0x%04x\n",
- thisaddr);
- return err;
- }
- p += 5 + thislen;
- len -= 5 + thislen;
- }
- if (write)
- dev_info(&fmc->dev, "write_eeprom: success\n");
- return 0;
-}
-
-static int fwe_run_bin(struct fmc_device *fmc, const struct firmware *fw)
-{
- int ret;
-
- dev_info(&fmc->dev, "programming %zi bytes\n", fw->size);
- ret = fmc_write_ee(fmc, 0, (void *)fw->data, fw->size);
- if (ret < 0) {
- dev_info(&fmc->dev, "write_eeprom: error %i\n", ret);
- return ret;
- }
- dev_info(&fmc->dev, "write_eeprom: success\n");
- return 0;
-}
-
-static int fwe_run(struct fmc_device *fmc, const struct firmware *fw, char *s)
-{
- char *last4 = s + strlen(s) - 4;
- int err;
-
- if (!strcmp(last4, ".bin"))
- return fwe_run_bin(fmc, fw);
- if (!strcmp(last4, ".tlv")) {
- err = fwe_run_tlv(fmc, fw, 0);
- if (!err)
- err = fwe_run_tlv(fmc, fw, 1);
- return err;
- }
- dev_err(&fmc->dev, "invalid file name \"%s\"\n", s);
- return -EINVAL;
-}
-
-/*
- * Programming is done at probe time. Morever, only those listed with
- * busid= are programmed.
- * card is probed for, only one is programmed. Unfortunately, it's
- * difficult to know in advance when probing the first card if others
- * are there.
- */
-static int fwe_probe(struct fmc_device *fmc)
-{
- int err, index = 0;
- const struct firmware *fw;
- struct device *dev = &fmc->dev;
- char *s;
-
- if (!fwe_drv.busid_n) {
- dev_err(dev, "%s: no busid passed, refusing all cards\n",
- KBUILD_MODNAME);
- return -ENODEV;
- }
-
- index = fmc_validate(fmc, &fwe_drv);
- if (index < 0) {
- pr_err("%s: refusing device \"%s\"\n", KBUILD_MODNAME,
- dev_name(dev));
- return -ENODEV;
- }
- if (index >= fwe_file_n) {
- pr_err("%s: no filename for device index %i\n",
- KBUILD_MODNAME, index);
- return -ENODEV;
- }
- s = fwe_file[index];
- if (!s) {
- pr_err("%s: no filename for \"%s\" not programming\n",
- KBUILD_MODNAME, dev_name(dev));
- return -ENOENT;
- }
- err = request_firmware(&fw, s, dev);
- if (err < 0) {
- dev_err(&fmc->dev, "request firmware \"%s\": error %i\n",
- s, err);
- return err;
- }
- fwe_run(fmc, fw, s);
- release_firmware(fw);
- return 0;
-}
-
-static int fwe_remove(struct fmc_device *fmc)
-{
- return 0;
-}
-
-static struct fmc_driver fwe_drv = {
- .version = FMC_VERSION,
- .driver.name = KBUILD_MODNAME,
- .probe = fwe_probe,
- .remove = fwe_remove,
- /* no table, as the current match just matches everything */
-};
-
-static int fwe_init(void)
-{
- int ret;
-
- ret = fmc_driver_register(&fwe_drv);
- return ret;
-}
-
-static void fwe_exit(void)
-{
- fmc_driver_unregister(&fwe_drv);
-}
-
-module_init(fwe_init);
-module_exit(fwe_exit);
-
-MODULE_LICENSE("GPL");
diff --git a/drivers/fmc/fru-parse.c b/drivers/fmc/fru-parse.c
deleted file mode 100644
index f551b81f4fd9..000000000000
--- a/drivers/fmc/fru-parse.c
+++ /dev/null
@@ -1,80 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Copyright (C) 2012 CERN (www.cern.ch)
- * Author: Alessandro Rubini <rubini@gnudd.com>
- *
- * This work is part of the White Rabbit project, a research effort led
- * by CERN, the European Institute for Nuclear Research.
- */
-#include <linux/ipmi-fru.h>
-
-/* Some internal helpers */
-static struct fru_type_length *
-__fru_get_board_tl(struct fru_common_header *header, int nr)
-{
- struct fru_board_info_area *bia;
- struct fru_type_length *tl;
-
- bia = fru_get_board_area(header);
- tl = bia->tl;
- while (nr > 0 && !fru_is_eof(tl)) {
- tl = fru_next_tl(tl);
- nr--;
- }
- if (fru_is_eof(tl))
- return NULL;
- return tl;
-}
-
-static char *__fru_alloc_get_tl(struct fru_common_header *header, int nr)
-{
- struct fru_type_length *tl;
- char *res;
-
- tl = __fru_get_board_tl(header, nr);
- if (!tl)
- return NULL;
-
- res = fru_alloc(fru_strlen(tl) + 1);
- if (!res)
- return NULL;
- return fru_strcpy(res, tl);
-}
-
-/* Public checksum verifiers */
-int fru_header_cksum_ok(struct fru_common_header *header)
-{
- uint8_t *ptr = (void *)header;
- int i, sum;
-
- for (i = sum = 0; i < sizeof(*header); i++)
- sum += ptr[i];
- return (sum & 0xff) == 0;
-}
-int fru_bia_cksum_ok(struct fru_board_info_area *bia)
-{
- uint8_t *ptr = (void *)bia;
- int i, sum;
-
- for (i = sum = 0; i < 8 * bia->area_len; i++)
- sum += ptr[i];
- return (sum & 0xff) == 0;
-}
-
-/* Get various stuff, trivial */
-char *fru_get_board_manufacturer(struct fru_common_header *header)
-{
- return __fru_alloc_get_tl(header, 0);
-}
-char *fru_get_product_name(struct fru_common_header *header)
-{
- return __fru_alloc_get_tl(header, 1);
-}
-char *fru_get_serial_number(struct fru_common_header *header)
-{
- return __fru_alloc_get_tl(header, 2);
-}
-char *fru_get_part_number(struct fru_common_header *header)
-{
- return __fru_alloc_get_tl(header, 3);
-}
diff --git a/drivers/fpga/Kconfig b/drivers/fpga/Kconfig
index 8072c195d831..73c779e920ed 100644
--- a/drivers/fpga/Kconfig
+++ b/drivers/fpga/Kconfig
@@ -26,9 +26,9 @@ config FPGA_MGR_SOCFPGA_A10
FPGA manager driver support for Altera Arria10 SoCFPGA.
config ALTERA_PR_IP_CORE
- tristate "Altera Partial Reconfiguration IP Core"
- help
- Core driver support for Altera Partial Reconfiguration IP component
+ tristate "Altera Partial Reconfiguration IP Core"
+ help
+ Core driver support for Altera Partial Reconfiguration IP component
config ALTERA_PR_IP_CORE_PLAT
tristate "Platform support of Altera Partial Reconfiguration IP Core"
@@ -40,16 +40,17 @@ config ALTERA_PR_IP_CORE_PLAT
config FPGA_MGR_ALTERA_PS_SPI
tristate "Altera FPGA Passive Serial over SPI"
depends on SPI
+ select BITREVERSE
help
FPGA manager driver support for Altera Arria/Cyclone/Stratix
using the passive serial interface over SPI.
config FPGA_MGR_ALTERA_CVP
- tristate "Altera Arria-V/Cyclone-V/Stratix-V CvP FPGA Manager"
+ tristate "Altera CvP FPGA Manager"
depends on PCI
help
- FPGA manager driver support for Arria-V, Cyclone-V, Stratix-V
- and Arria 10 Altera FPGAs using the CvP interface over PCIe.
+ FPGA manager driver support for Arria-V, Cyclone-V, Stratix-V,
+ Arria 10 and Stratix10 Altera FPGAs using the CvP interface over PCIe.
config FPGA_MGR_ZYNQ_FPGA
tristate "Xilinx Zynq FPGA"
diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile
index 312b9371742f..4865b74b00a4 100644
--- a/drivers/fpga/Makefile
+++ b/drivers/fpga/Makefile
@@ -39,8 +39,9 @@ obj-$(CONFIG_FPGA_DFL_FME_BRIDGE) += dfl-fme-br.o
obj-$(CONFIG_FPGA_DFL_FME_REGION) += dfl-fme-region.o
obj-$(CONFIG_FPGA_DFL_AFU) += dfl-afu.o
-dfl-fme-objs := dfl-fme-main.o dfl-fme-pr.o
+dfl-fme-objs := dfl-fme-main.o dfl-fme-pr.o dfl-fme-error.o
dfl-afu-objs := dfl-afu-main.o dfl-afu-region.o dfl-afu-dma-region.o
+dfl-afu-objs += dfl-afu-error.o
# Drivers for FPGAs which implement DFL
obj-$(CONFIG_FPGA_DFL_PCI) += dfl-pci.o
diff --git a/drivers/fpga/altera-cvp.c b/drivers/fpga/altera-cvp.c
index 770915fb97f9..4e0edb60bfba 100644
--- a/drivers/fpga/altera-cvp.c
+++ b/drivers/fpga/altera-cvp.c
@@ -22,10 +22,10 @@
#define TIMEOUT_US 2000 /* CVP STATUS timeout for USERMODE polling */
/* Vendor Specific Extended Capability Registers */
-#define VSE_PCIE_EXT_CAP_ID 0x200
+#define VSE_PCIE_EXT_CAP_ID 0x0
#define VSE_PCIE_EXT_CAP_ID_VAL 0x000b /* 16bit */
-#define VSE_CVP_STATUS 0x21c /* 32bit */
+#define VSE_CVP_STATUS 0x1c /* 32bit */
#define VSE_CVP_STATUS_CFG_RDY BIT(18) /* CVP_CONFIG_READY */
#define VSE_CVP_STATUS_CFG_ERR BIT(19) /* CVP_CONFIG_ERROR */
#define VSE_CVP_STATUS_CVP_EN BIT(20) /* ctrl block is enabling CVP */
@@ -33,41 +33,93 @@
#define VSE_CVP_STATUS_CFG_DONE BIT(23) /* CVP_CONFIG_DONE */
#define VSE_CVP_STATUS_PLD_CLK_IN_USE BIT(24) /* PLD_CLK_IN_USE */
-#define VSE_CVP_MODE_CTRL 0x220 /* 32bit */
+#define VSE_CVP_MODE_CTRL 0x20 /* 32bit */
#define VSE_CVP_MODE_CTRL_CVP_MODE BIT(0) /* CVP (1) or normal mode (0) */
#define VSE_CVP_MODE_CTRL_HIP_CLK_SEL BIT(1) /* PMA (1) or fabric clock (0) */
#define VSE_CVP_MODE_CTRL_NUMCLKS_OFF 8 /* NUMCLKS bits offset */
#define VSE_CVP_MODE_CTRL_NUMCLKS_MASK GENMASK(15, 8)
-#define VSE_CVP_DATA 0x228 /* 32bit */
-#define VSE_CVP_PROG_CTRL 0x22c /* 32bit */
+#define VSE_CVP_DATA 0x28 /* 32bit */
+#define VSE_CVP_PROG_CTRL 0x2c /* 32bit */
#define VSE_CVP_PROG_CTRL_CONFIG BIT(0)
#define VSE_CVP_PROG_CTRL_START_XFER BIT(1)
+#define VSE_CVP_PROG_CTRL_MASK GENMASK(1, 0)
-#define VSE_UNCOR_ERR_STATUS 0x234 /* 32bit */
+#define VSE_UNCOR_ERR_STATUS 0x34 /* 32bit */
#define VSE_UNCOR_ERR_CVP_CFG_ERR BIT(5) /* CVP_CONFIG_ERROR_LATCHED */
+#define V1_VSEC_OFFSET 0x200 /* Vendor Specific Offset V1 */
+/* V2 Defines */
+#define VSE_CVP_TX_CREDITS 0x49 /* 8bit */
+
+#define V2_CREDIT_TIMEOUT_US 20000
+#define V2_CHECK_CREDIT_US 10
+#define V2_POLL_TIMEOUT_US 1000000
+#define V2_USER_TIMEOUT_US 500000
+
+#define V1_POLL_TIMEOUT_US 10
+
#define DRV_NAME "altera-cvp"
#define ALTERA_CVP_MGR_NAME "Altera CvP FPGA Manager"
+/* Write block sizes */
+#define ALTERA_CVP_V1_SIZE 4
+#define ALTERA_CVP_V2_SIZE 4096
+
/* Optional CvP config error status check for debugging */
static bool altera_cvp_chkcfg;
+struct cvp_priv;
+
struct altera_cvp_conf {
struct fpga_manager *mgr;
struct pci_dev *pci_dev;
void __iomem *map;
- void (*write_data)(struct altera_cvp_conf *, u32);
+ void (*write_data)(struct altera_cvp_conf *conf,
+ u32 data);
char mgr_name[64];
u8 numclks;
+ u32 sent_packets;
+ u32 vsec_offset;
+ const struct cvp_priv *priv;
+};
+
+struct cvp_priv {
+ void (*switch_clk)(struct altera_cvp_conf *conf);
+ int (*clear_state)(struct altera_cvp_conf *conf);
+ int (*wait_credit)(struct fpga_manager *mgr, u32 blocks);
+ size_t block_size;
+ int poll_time_us;
+ int user_time_us;
};
+static int altera_read_config_byte(struct altera_cvp_conf *conf,
+ int where, u8 *val)
+{
+ return pci_read_config_byte(conf->pci_dev, conf->vsec_offset + where,
+ val);
+}
+
+static int altera_read_config_dword(struct altera_cvp_conf *conf,
+ int where, u32 *val)
+{
+ return pci_read_config_dword(conf->pci_dev, conf->vsec_offset + where,
+ val);
+}
+
+static int altera_write_config_dword(struct altera_cvp_conf *conf,
+ int where, u32 val)
+{
+ return pci_write_config_dword(conf->pci_dev, conf->vsec_offset + where,
+ val);
+}
+
static enum fpga_mgr_states altera_cvp_state(struct fpga_manager *mgr)
{
struct altera_cvp_conf *conf = mgr->priv;
u32 status;
- pci_read_config_dword(conf->pci_dev, VSE_CVP_STATUS, &status);
+ altera_read_config_dword(conf, VSE_CVP_STATUS, &status);
if (status & VSE_CVP_STATUS_CFG_DONE)
return FPGA_MGR_STATE_OPERATING;
@@ -85,7 +137,8 @@ static void altera_cvp_write_data_iomem(struct altera_cvp_conf *conf, u32 val)
static void altera_cvp_write_data_config(struct altera_cvp_conf *conf, u32 val)
{
- pci_write_config_dword(conf->pci_dev, VSE_CVP_DATA, val);
+ pci_write_config_dword(conf->pci_dev, conf->vsec_offset + VSE_CVP_DATA,
+ val);
}
/* switches between CvP clock and internal clock */
@@ -95,10 +148,10 @@ static void altera_cvp_dummy_write(struct altera_cvp_conf *conf)
u32 val;
/* set 1 CVP clock cycle for every CVP Data Register Write */
- pci_read_config_dword(conf->pci_dev, VSE_CVP_MODE_CTRL, &val);
+ altera_read_config_dword(conf, VSE_CVP_MODE_CTRL, &val);
val &= ~VSE_CVP_MODE_CTRL_NUMCLKS_MASK;
val |= 1 << VSE_CVP_MODE_CTRL_NUMCLKS_OFF;
- pci_write_config_dword(conf->pci_dev, VSE_CVP_MODE_CTRL, val);
+ altera_write_config_dword(conf, VSE_CVP_MODE_CTRL, val);
for (i = 0; i < CVP_DUMMY_WR; i++)
conf->write_data(conf, 0); /* dummy data, could be any value */
@@ -115,7 +168,7 @@ static int altera_cvp_wait_status(struct altera_cvp_conf *conf, u32 status_mask,
retries++;
do {
- pci_read_config_dword(conf->pci_dev, VSE_CVP_STATUS, &val);
+ altera_read_config_dword(conf, VSE_CVP_STATUS, &val);
if ((val & status_mask) == status_val)
return 0;
@@ -126,32 +179,136 @@ static int altera_cvp_wait_status(struct altera_cvp_conf *conf, u32 status_mask,
return -ETIMEDOUT;
}
+static int altera_cvp_chk_error(struct fpga_manager *mgr, size_t bytes)
+{
+ struct altera_cvp_conf *conf = mgr->priv;
+ u32 val;
+ int ret;
+
+ /* STEP 10 (optional) - check CVP_CONFIG_ERROR flag */
+ ret = altera_read_config_dword(conf, VSE_CVP_STATUS, &val);
+ if (ret || (val & VSE_CVP_STATUS_CFG_ERR)) {
+ dev_err(&mgr->dev, "CVP_CONFIG_ERROR after %zu bytes!\n",
+ bytes);
+ return -EPROTO;
+ }
+ return 0;
+}
+
+/*
+ * CvP Version2 Functions
+ * Recent Intel FPGAs use a credit mechanism to throttle incoming
+ * bitstreams and a different method of clearing the state.
+ */
+
+static int altera_cvp_v2_clear_state(struct altera_cvp_conf *conf)
+{
+ u32 val;
+ int ret;
+
+ /* Clear the START_XFER and CVP_CONFIG bits */
+ ret = altera_read_config_dword(conf, VSE_CVP_PROG_CTRL, &val);
+ if (ret) {
+ dev_err(&conf->pci_dev->dev,
+ "Error reading CVP Program Control Register\n");
+ return ret;
+ }
+
+ val &= ~VSE_CVP_PROG_CTRL_MASK;
+ ret = altera_write_config_dword(conf, VSE_CVP_PROG_CTRL, val);
+ if (ret) {
+ dev_err(&conf->pci_dev->dev,
+ "Error writing CVP Program Control Register\n");
+ return ret;
+ }
+
+ return altera_cvp_wait_status(conf, VSE_CVP_STATUS_CFG_RDY, 0,
+ conf->priv->poll_time_us);
+}
+
+static int altera_cvp_v2_wait_for_credit(struct fpga_manager *mgr,
+ u32 blocks)
+{
+ u32 timeout = V2_CREDIT_TIMEOUT_US / V2_CHECK_CREDIT_US;
+ struct altera_cvp_conf *conf = mgr->priv;
+ int ret;
+ u8 val;
+
+ do {
+ ret = altera_read_config_byte(conf, VSE_CVP_TX_CREDITS, &val);
+ if (ret) {
+ dev_err(&conf->pci_dev->dev,
+ "Error reading CVP Credit Register\n");
+ return ret;
+ }
+
+ /* Return if there is space in FIFO */
+ if (val - (u8)conf->sent_packets)
+ return 0;
+
+ ret = altera_cvp_chk_error(mgr, blocks * ALTERA_CVP_V2_SIZE);
+ if (ret) {
+ dev_err(&conf->pci_dev->dev,
+ "CE Bit error credit reg[0x%x]:sent[0x%x]\n",
+ val, conf->sent_packets);
+ return -EAGAIN;
+ }
+
+ /* Limit the check credit byte traffic */
+ usleep_range(V2_CHECK_CREDIT_US, V2_CHECK_CREDIT_US + 1);
+ } while (timeout--);
+
+ dev_err(&conf->pci_dev->dev, "Timeout waiting for credit\n");
+ return -ETIMEDOUT;
+}
+
+static int altera_cvp_send_block(struct altera_cvp_conf *conf,
+ const u32 *data, size_t len)
+{
+ u32 mask, words = len / sizeof(u32);
+ int i, remainder;
+
+ for (i = 0; i < words; i++)
+ conf->write_data(conf, *data++);
+
+ /* write up to 3 trailing bytes, if any */
+ remainder = len % sizeof(u32);
+ if (remainder) {
+ mask = BIT(remainder * 8) - 1;
+ if (mask)
+ conf->write_data(conf, *data & mask);
+ }
+
+ return 0;
+}
+
static int altera_cvp_teardown(struct fpga_manager *mgr,
struct fpga_image_info *info)
{
struct altera_cvp_conf *conf = mgr->priv;
- struct pci_dev *pdev = conf->pci_dev;
int ret;
u32 val;
/* STEP 12 - reset START_XFER bit */
- pci_read_config_dword(pdev, VSE_CVP_PROG_CTRL, &val);
+ altera_read_config_dword(conf, VSE_CVP_PROG_CTRL, &val);
val &= ~VSE_CVP_PROG_CTRL_START_XFER;
- pci_write_config_dword(pdev, VSE_CVP_PROG_CTRL, val);
+ altera_write_config_dword(conf, VSE_CVP_PROG_CTRL, val);
/* STEP 13 - reset CVP_CONFIG bit */
val &= ~VSE_CVP_PROG_CTRL_CONFIG;
- pci_write_config_dword(pdev, VSE_CVP_PROG_CTRL, val);
+ altera_write_config_dword(conf, VSE_CVP_PROG_CTRL, val);
/*
* STEP 14
* - set CVP_NUMCLKS to 1 and then issue CVP_DUMMY_WR dummy
* writes to the HIP
*/
- altera_cvp_dummy_write(conf); /* from CVP clock to internal clock */
+ if (conf->priv->switch_clk)
+ conf->priv->switch_clk(conf);
/* STEP 15 - poll CVP_CONFIG_READY bit for 0 with 10us timeout */
- ret = altera_cvp_wait_status(conf, VSE_CVP_STATUS_CFG_RDY, 0, 10);
+ ret = altera_cvp_wait_status(conf, VSE_CVP_STATUS_CFG_RDY, 0,
+ conf->priv->poll_time_us);
if (ret)
dev_err(&mgr->dev, "CFG_RDY == 0 timeout\n");
@@ -163,7 +320,6 @@ static int altera_cvp_write_init(struct fpga_manager *mgr,
const char *buf, size_t count)
{
struct altera_cvp_conf *conf = mgr->priv;
- struct pci_dev *pdev = conf->pci_dev;
u32 iflags, val;
int ret;
@@ -183,7 +339,7 @@ static int altera_cvp_write_init(struct fpga_manager *mgr,
conf->numclks = 1; /* for uncompressed and unencrypted images */
/* STEP 1 - read CVP status and check CVP_EN flag */
- pci_read_config_dword(pdev, VSE_CVP_STATUS, &val);
+ altera_read_config_dword(conf, VSE_CVP_STATUS, &val);
if (!(val & VSE_CVP_STATUS_CVP_EN)) {
dev_err(&mgr->dev, "CVP mode off: 0x%04x\n", val);
return -ENODEV;
@@ -201,30 +357,42 @@ static int altera_cvp_write_init(struct fpga_manager *mgr,
* - set HIP_CLK_SEL and CVP_MODE (must be set in the order mentioned)
*/
/* switch from fabric to PMA clock */
- pci_read_config_dword(pdev, VSE_CVP_MODE_CTRL, &val);
+ altera_read_config_dword(conf, VSE_CVP_MODE_CTRL, &val);
val |= VSE_CVP_MODE_CTRL_HIP_CLK_SEL;
- pci_write_config_dword(pdev, VSE_CVP_MODE_CTRL, val);
+ altera_write_config_dword(conf, VSE_CVP_MODE_CTRL, val);
/* set CVP mode */
- pci_read_config_dword(pdev, VSE_CVP_MODE_CTRL, &val);
+ altera_read_config_dword(conf, VSE_CVP_MODE_CTRL, &val);
val |= VSE_CVP_MODE_CTRL_CVP_MODE;
- pci_write_config_dword(pdev, VSE_CVP_MODE_CTRL, val);
+ altera_write_config_dword(conf, VSE_CVP_MODE_CTRL, val);
/*
* STEP 3
* - set CVP_NUMCLKS to 1 and issue CVP_DUMMY_WR dummy writes to the HIP
*/
- altera_cvp_dummy_write(conf);
+ if (conf->priv->switch_clk)
+ conf->priv->switch_clk(conf);
+
+ if (conf->priv->clear_state) {
+ ret = conf->priv->clear_state(conf);
+ if (ret) {
+ dev_err(&mgr->dev, "Problem clearing out state\n");
+ return ret;
+ }
+ }
+
+ conf->sent_packets = 0;
/* STEP 4 - set CVP_CONFIG bit */
- pci_read_config_dword(pdev, VSE_CVP_PROG_CTRL, &val);
+ altera_read_config_dword(conf, VSE_CVP_PROG_CTRL, &val);
/* request control block to begin transfer using CVP */
val |= VSE_CVP_PROG_CTRL_CONFIG;
- pci_write_config_dword(pdev, VSE_CVP_PROG_CTRL, val);
+ altera_write_config_dword(conf, VSE_CVP_PROG_CTRL, val);
- /* STEP 5 - poll CVP_CONFIG READY for 1 with 10us timeout */
+ /* STEP 5 - poll CVP_CONFIG READY for 1 with timeout */
ret = altera_cvp_wait_status(conf, VSE_CVP_STATUS_CFG_RDY,
- VSE_CVP_STATUS_CFG_RDY, 10);
+ VSE_CVP_STATUS_CFG_RDY,
+ conf->priv->poll_time_us);
if (ret) {
dev_warn(&mgr->dev, "CFG_RDY == 1 timeout\n");
return ret;
@@ -234,33 +402,28 @@ static int altera_cvp_write_init(struct fpga_manager *mgr,
* STEP 6
* - set CVP_NUMCLKS to 1 and issue CVP_DUMMY_WR dummy writes to the HIP
*/
- altera_cvp_dummy_write(conf);
+ if (conf->priv->switch_clk)
+ conf->priv->switch_clk(conf);
+
+ if (altera_cvp_chkcfg) {
+ ret = altera_cvp_chk_error(mgr, 0);
+ if (ret) {
+ dev_warn(&mgr->dev, "CFG_RDY == 1 timeout\n");
+ return ret;
+ }
+ }
/* STEP 7 - set START_XFER */
- pci_read_config_dword(pdev, VSE_CVP_PROG_CTRL, &val);
+ altera_read_config_dword(conf, VSE_CVP_PROG_CTRL, &val);
val |= VSE_CVP_PROG_CTRL_START_XFER;
- pci_write_config_dword(pdev, VSE_CVP_PROG_CTRL, val);
+ altera_write_config_dword(conf, VSE_CVP_PROG_CTRL, val);
/* STEP 8 - start transfer (set CVP_NUMCLKS for bitstream) */
- pci_read_config_dword(pdev, VSE_CVP_MODE_CTRL, &val);
- val &= ~VSE_CVP_MODE_CTRL_NUMCLKS_MASK;
- val |= conf->numclks << VSE_CVP_MODE_CTRL_NUMCLKS_OFF;
- pci_write_config_dword(pdev, VSE_CVP_MODE_CTRL, val);
-
- return 0;
-}
-
-static inline int altera_cvp_chk_error(struct fpga_manager *mgr, size_t bytes)
-{
- struct altera_cvp_conf *conf = mgr->priv;
- u32 val;
-
- /* STEP 10 (optional) - check CVP_CONFIG_ERROR flag */
- pci_read_config_dword(conf->pci_dev, VSE_CVP_STATUS, &val);
- if (val & VSE_CVP_STATUS_CFG_ERR) {
- dev_err(&mgr->dev, "CVP_CONFIG_ERROR after %zu bytes!\n",
- bytes);
- return -EPROTO;
+ if (conf->priv->switch_clk) {
+ altera_read_config_dword(conf, VSE_CVP_MODE_CTRL, &val);
+ val &= ~VSE_CVP_MODE_CTRL_NUMCLKS_MASK;
+ val |= conf->numclks << VSE_CVP_MODE_CTRL_NUMCLKS_OFF;
+ altera_write_config_dword(conf, VSE_CVP_MODE_CTRL, val);
}
return 0;
}
@@ -269,20 +432,32 @@ static int altera_cvp_write(struct fpga_manager *mgr, const char *buf,
size_t count)
{
struct altera_cvp_conf *conf = mgr->priv;
+ size_t done, remaining, len;
const u32 *data;
- size_t done, remaining;
int status = 0;
- u32 mask;
/* STEP 9 - write 32-bit data from RBF file to CVP data register */
data = (u32 *)buf;
remaining = count;
done = 0;
- while (remaining >= 4) {
- conf->write_data(conf, *data++);
- done += 4;
- remaining -= 4;
+ while (remaining) {
+ /* Use credit throttling if available */
+ if (conf->priv->wait_credit) {
+ status = conf->priv->wait_credit(mgr, done);
+ if (status) {
+ dev_err(&conf->pci_dev->dev,
+ "Wait Credit ERR: 0x%x\n", status);
+ return status;
+ }
+ }
+
+ len = min(conf->priv->block_size, remaining);
+ altera_cvp_send_block(conf, data, len);
+ data += len / sizeof(u32);
+ done += len;
+ remaining -= len;
+ conf->sent_packets++;
/*
* STEP 10 (optional) and STEP 11
@@ -300,11 +475,6 @@ static int altera_cvp_write(struct fpga_manager *mgr, const char *buf,
}
}
- /* write up to 3 trailing bytes, if any */
- mask = BIT(remaining * 8) - 1;
- if (mask)
- conf->write_data(conf, *data & mask);
-
if (altera_cvp_chkcfg)
status = altera_cvp_chk_error(mgr, count);
@@ -315,31 +485,30 @@ static int altera_cvp_write_complete(struct fpga_manager *mgr,
struct fpga_image_info *info)
{
struct altera_cvp_conf *conf = mgr->priv;
- struct pci_dev *pdev = conf->pci_dev;
+ u32 mask, val;
int ret;
- u32 mask;
- u32 val;
ret = altera_cvp_teardown(mgr, info);
if (ret)
return ret;
/* STEP 16 - check CVP_CONFIG_ERROR_LATCHED bit */
- pci_read_config_dword(pdev, VSE_UNCOR_ERR_STATUS, &val);
+ altera_read_config_dword(conf, VSE_UNCOR_ERR_STATUS, &val);
if (val & VSE_UNCOR_ERR_CVP_CFG_ERR) {
dev_err(&mgr->dev, "detected CVP_CONFIG_ERROR_LATCHED!\n");
return -EPROTO;
}
/* STEP 17 - reset CVP_MODE and HIP_CLK_SEL bit */
- pci_read_config_dword(pdev, VSE_CVP_MODE_CTRL, &val);
+ altera_read_config_dword(conf, VSE_CVP_MODE_CTRL, &val);
val &= ~VSE_CVP_MODE_CTRL_HIP_CLK_SEL;
val &= ~VSE_CVP_MODE_CTRL_CVP_MODE;
- pci_write_config_dword(pdev, VSE_CVP_MODE_CTRL, val);
+ altera_write_config_dword(conf, VSE_CVP_MODE_CTRL, val);
/* STEP 18 - poll PLD_CLK_IN_USE and USER_MODE bits */
mask = VSE_CVP_STATUS_PLD_CLK_IN_USE | VSE_CVP_STATUS_USERMODE;
- ret = altera_cvp_wait_status(conf, mask, mask, TIMEOUT_US);
+ ret = altera_cvp_wait_status(conf, mask, mask,
+ conf->priv->user_time_us);
if (ret)
dev_err(&mgr->dev, "PLD_CLK_IN_USE|USERMODE timeout\n");
@@ -353,6 +522,21 @@ static const struct fpga_manager_ops altera_cvp_ops = {
.write_complete = altera_cvp_write_complete,
};
+static const struct cvp_priv cvp_priv_v1 = {
+ .switch_clk = altera_cvp_dummy_write,
+ .block_size = ALTERA_CVP_V1_SIZE,
+ .poll_time_us = V1_POLL_TIMEOUT_US,
+ .user_time_us = TIMEOUT_US,
+};
+
+static const struct cvp_priv cvp_priv_v2 = {
+ .clear_state = altera_cvp_v2_clear_state,
+ .wait_credit = altera_cvp_v2_wait_for_credit,
+ .block_size = ALTERA_CVP_V2_SIZE,
+ .poll_time_us = V2_POLL_TIMEOUT_US,
+ .user_time_us = V2_USER_TIMEOUT_US,
+};
+
static ssize_t chkcfg_show(struct device_driver *dev, char *buf)
{
return snprintf(buf, 3, "%d\n", altera_cvp_chkcfg);
@@ -394,22 +578,29 @@ static int altera_cvp_probe(struct pci_dev *pdev,
{
struct altera_cvp_conf *conf;
struct fpga_manager *mgr;
+ int ret, offset;
u16 cmd, val;
u32 regval;
- int ret;
+
+ /* Discover the Vendor Specific Offset for this device */
+ offset = pci_find_next_ext_capability(pdev, 0, PCI_EXT_CAP_ID_VNDR);
+ if (!offset) {
+ dev_err(&pdev->dev, "No Vendor Specific Offset.\n");
+ return -ENODEV;
+ }
/*
* First check if this is the expected FPGA device. PCI config
* space access works without enabling the PCI device, memory
* space access is enabled further down.
*/
- pci_read_config_word(pdev, VSE_PCIE_EXT_CAP_ID, &val);
+ pci_read_config_word(pdev, offset + VSE_PCIE_EXT_CAP_ID, &val);
if (val != VSE_PCIE_EXT_CAP_ID_VAL) {
dev_err(&pdev->dev, "Wrong EXT_CAP_ID value 0x%x\n", val);
return -ENODEV;
}
- pci_read_config_dword(pdev, VSE_CVP_STATUS, &regval);
+ pci_read_config_dword(pdev, offset + VSE_CVP_STATUS, &regval);
if (!(regval & VSE_CVP_STATUS_CVP_EN)) {
dev_err(&pdev->dev,
"CVP is disabled for this device: CVP_STATUS Reg 0x%x\n",
@@ -421,6 +612,8 @@ static int altera_cvp_probe(struct pci_dev *pdev,
if (!conf)
return -ENOMEM;
+ conf->vsec_offset = offset;
+
/*
* Enable memory BAR access. We cannot use pci_enable_device() here
* because it will make the driver unusable with FPGA devices that
@@ -445,6 +638,11 @@ static int altera_cvp_probe(struct pci_dev *pdev,
conf->pci_dev = pdev;
conf->write_data = altera_cvp_write_data_iomem;
+ if (conf->vsec_offset == V1_VSEC_OFFSET)
+ conf->priv = &cvp_priv_v1;
+ else
+ conf->priv = &cvp_priv_v2;
+
conf->map = pci_iomap(pdev, CVP_BAR, 0);
if (!conf->map) {
dev_warn(&pdev->dev, "Mapping CVP BAR failed\n");
diff --git a/drivers/fpga/altera-pr-ip-core-plat.c b/drivers/fpga/altera-pr-ip-core-plat.c
index b293d83143f1..99b9cc0e70f0 100644
--- a/drivers/fpga/altera-pr-ip-core-plat.c
+++ b/drivers/fpga/altera-pr-ip-core-plat.c
@@ -32,7 +32,9 @@ static int alt_pr_platform_remove(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
- return alt_pr_unregister(dev);
+ alt_pr_unregister(dev);
+
+ return 0;
}
static const struct of_device_id alt_pr_of_match[] = {
diff --git a/drivers/fpga/altera-pr-ip-core.c b/drivers/fpga/altera-pr-ip-core.c
index a7a3bf0b5202..2cf25fd5e897 100644
--- a/drivers/fpga/altera-pr-ip-core.c
+++ b/drivers/fpga/altera-pr-ip-core.c
@@ -201,15 +201,13 @@ int alt_pr_register(struct device *dev, void __iomem *reg_base)
}
EXPORT_SYMBOL_GPL(alt_pr_register);
-int alt_pr_unregister(struct device *dev)
+void alt_pr_unregister(struct device *dev)
{
struct fpga_manager *mgr = dev_get_drvdata(dev);
dev_dbg(dev, "%s\n", __func__);
fpga_mgr_unregister(mgr);
-
- return 0;
}
EXPORT_SYMBOL_GPL(alt_pr_unregister);
diff --git a/drivers/fpga/altera-ps-spi.c b/drivers/fpga/altera-ps-spi.c
index a13f224303c6..0221dee8dd4c 100644
--- a/drivers/fpga/altera-ps-spi.c
+++ b/drivers/fpga/altera-ps-spi.c
@@ -210,7 +210,7 @@ static int altera_ps_write_complete(struct fpga_manager *mgr,
return -EIO;
}
- if (!IS_ERR(conf->confd)) {
+ if (conf->confd) {
if (!gpiod_get_raw_value_cansleep(conf->confd)) {
dev_err(&mgr->dev, "CONF_DONE is inactive!\n");
return -EIO;
@@ -289,10 +289,13 @@ static int altera_ps_probe(struct spi_device *spi)
return PTR_ERR(conf->status);
}
- conf->confd = devm_gpiod_get(&spi->dev, "confd", GPIOD_IN);
+ conf->confd = devm_gpiod_get_optional(&spi->dev, "confd", GPIOD_IN);
if (IS_ERR(conf->confd)) {
- dev_warn(&spi->dev, "Not using confd gpio: %ld\n",
- PTR_ERR(conf->confd));
+ dev_err(&spi->dev, "Failed to get confd gpio: %ld\n",
+ PTR_ERR(conf->confd));
+ return PTR_ERR(conf->confd);
+ } else if (!conf->confd) {
+ dev_warn(&spi->dev, "Not using confd gpio");
}
/* Register manager with unique name */
diff --git a/drivers/fpga/dfl-afu-dma-region.c b/drivers/fpga/dfl-afu-dma-region.c
index dcd80b088c7b..62f924489db5 100644
--- a/drivers/fpga/dfl-afu-dma-region.c
+++ b/drivers/fpga/dfl-afu-dma-region.c
@@ -12,6 +12,7 @@
#include <linux/dma-mapping.h>
#include <linux/sched/signal.h>
#include <linux/uaccess.h>
+#include <linux/mm.h>
#include "dfl-afu.h"
@@ -32,52 +33,6 @@ void afu_dma_region_init(struct dfl_feature_platform_data *pdata)
}
/**
- * afu_dma_adjust_locked_vm - adjust locked memory
- * @dev: port device
- * @npages: number of pages
- * @incr: increase or decrease locked memory
- *
- * Increase or decrease the locked memory size with npages input.
- *
- * Return 0 on success.
- * Return -ENOMEM if locked memory size is over the limit and no CAP_IPC_LOCK.
- */
-static int afu_dma_adjust_locked_vm(struct device *dev, long npages, bool incr)
-{
- unsigned long locked, lock_limit;
- int ret = 0;
-
- /* the task is exiting. */
- if (!current->mm)
- return 0;
-
- down_write(&current->mm->mmap_sem);
-
- if (incr) {
- locked = current->mm->locked_vm + npages;
- lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-
- if (locked > lock_limit && !capable(CAP_IPC_LOCK))
- ret = -ENOMEM;
- else
- current->mm->locked_vm += npages;
- } else {
- if (WARN_ON_ONCE(npages > current->mm->locked_vm))
- npages = current->mm->locked_vm;
- current->mm->locked_vm -= npages;
- }
-
- dev_dbg(dev, "[%d] RLIMIT_MEMLOCK %c%ld %ld/%ld%s\n", current->pid,
- incr ? '+' : '-', npages << PAGE_SHIFT,
- current->mm->locked_vm << PAGE_SHIFT, rlimit(RLIMIT_MEMLOCK),
- ret ? "- exceeded" : "");
-
- up_write(&current->mm->mmap_sem);
-
- return ret;
-}
-
-/**
* afu_dma_pin_pages - pin pages of given dma memory region
* @pdata: feature device platform data
* @region: dma memory region to be pinned
@@ -92,7 +47,7 @@ static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata,
struct device *dev = &pdata->dev->dev;
int ret, pinned;
- ret = afu_dma_adjust_locked_vm(dev, npages, true);
+ ret = account_locked_vm(current->mm, npages, true);
if (ret)
return ret;
@@ -121,7 +76,7 @@ put_pages:
free_pages:
kfree(region->pages);
unlock_vm:
- afu_dma_adjust_locked_vm(dev, npages, false);
+ account_locked_vm(current->mm, npages, false);
return ret;
}
@@ -141,7 +96,7 @@ static void afu_dma_unpin_pages(struct dfl_feature_platform_data *pdata,
put_all_pages(region->pages, npages);
kfree(region->pages);
- afu_dma_adjust_locked_vm(dev, npages, false);
+ account_locked_vm(current->mm, npages, false);
dev_dbg(dev, "%ld pages unpinned\n", npages);
}
diff --git a/drivers/fpga/dfl-afu-error.c b/drivers/fpga/dfl-afu-error.c
new file mode 100644
index 000000000000..c1467ae1a6b6
--- /dev/null
+++ b/drivers/fpga/dfl-afu-error.c
@@ -0,0 +1,230 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for FPGA Accelerated Function Unit (AFU) Error Reporting
+ *
+ * Copyright 2019 Intel Corporation, Inc.
+ *
+ * Authors:
+ * Wu Hao <hao.wu@linux.intel.com>
+ * Xiao Guangrong <guangrong.xiao@linux.intel.com>
+ * Joseph Grecco <joe.grecco@intel.com>
+ * Enno Luebbers <enno.luebbers@intel.com>
+ * Tim Whisonant <tim.whisonant@intel.com>
+ * Ananda Ravuri <ananda.ravuri@intel.com>
+ * Mitchel Henry <henry.mitchel@intel.com>
+ */
+
+#include <linux/uaccess.h>
+
+#include "dfl-afu.h"
+
+#define PORT_ERROR_MASK 0x8
+#define PORT_ERROR 0x10
+#define PORT_FIRST_ERROR 0x18
+#define PORT_MALFORMED_REQ0 0x20
+#define PORT_MALFORMED_REQ1 0x28
+
+#define ERROR_MASK GENMASK_ULL(63, 0)
+
+/* mask or unmask port errors by the error mask register. */
+static void __afu_port_err_mask(struct device *dev, bool mask)
+{
+ void __iomem *base;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_ERROR);
+
+ writeq(mask ? ERROR_MASK : 0, base + PORT_ERROR_MASK);
+}
+
+static void afu_port_err_mask(struct device *dev, bool mask)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+
+ mutex_lock(&pdata->lock);
+ __afu_port_err_mask(dev, mask);
+ mutex_unlock(&pdata->lock);
+}
+
+/* clear port errors. */
+static int afu_port_err_clear(struct device *dev, u64 err)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ struct platform_device *pdev = to_platform_device(dev);
+ void __iomem *base_err, *base_hdr;
+ int ret = -EBUSY;
+ u64 v;
+
+ base_err = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_ERROR);
+ base_hdr = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_HEADER);
+
+ mutex_lock(&pdata->lock);
+
+ /*
+ * clear Port Errors
+ *
+ * - Check for AP6 State
+ * - Halt Port by keeping Port in reset
+ * - Set PORT Error mask to all 1 to mask errors
+ * - Clear all errors
+ * - Set Port mask to all 0 to enable errors
+ * - All errors start capturing new errors
+ * - Enable Port by pulling the port out of reset
+ */
+
+ /* if device is still in AP6 power state, can not clear any error. */
+ v = readq(base_hdr + PORT_HDR_STS);
+ if (FIELD_GET(PORT_STS_PWR_STATE, v) == PORT_STS_PWR_STATE_AP6) {
+ dev_err(dev, "Could not clear errors, device in AP6 state.\n");
+ goto done;
+ }
+
+ /* Halt Port by keeping Port in reset */
+ ret = __afu_port_disable(pdev);
+ if (ret)
+ goto done;
+
+ /* Mask all errors */
+ __afu_port_err_mask(dev, true);
+
+ /* Clear errors if err input matches with current port errors.*/
+ v = readq(base_err + PORT_ERROR);
+
+ if (v == err) {
+ writeq(v, base_err + PORT_ERROR);
+
+ v = readq(base_err + PORT_FIRST_ERROR);
+ writeq(v, base_err + PORT_FIRST_ERROR);
+ } else {
+ ret = -EINVAL;
+ }
+
+ /* Clear mask */
+ __afu_port_err_mask(dev, false);
+
+ /* Enable the Port by clear the reset */
+ __afu_port_enable(pdev);
+
+done:
+ mutex_unlock(&pdata->lock);
+ return ret;
+}
+
+static ssize_t errors_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ u64 error;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_ERROR);
+
+ mutex_lock(&pdata->lock);
+ error = readq(base + PORT_ERROR);
+ mutex_unlock(&pdata->lock);
+
+ return sprintf(buf, "0x%llx\n", (unsigned long long)error);
+}
+
+static ssize_t errors_store(struct device *dev, struct device_attribute *attr,
+ const char *buff, size_t count)
+{
+ u64 value;
+ int ret;
+
+ if (kstrtou64(buff, 0, &value))
+ return -EINVAL;
+
+ ret = afu_port_err_clear(dev, value);
+
+ return ret ? ret : count;
+}
+static DEVICE_ATTR_RW(errors);
+
+static ssize_t first_error_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ u64 error;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_ERROR);
+
+ mutex_lock(&pdata->lock);
+ error = readq(base + PORT_FIRST_ERROR);
+ mutex_unlock(&pdata->lock);
+
+ return sprintf(buf, "0x%llx\n", (unsigned long long)error);
+}
+static DEVICE_ATTR_RO(first_error);
+
+static ssize_t first_malformed_req_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ u64 req0, req1;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_ERROR);
+
+ mutex_lock(&pdata->lock);
+ req0 = readq(base + PORT_MALFORMED_REQ0);
+ req1 = readq(base + PORT_MALFORMED_REQ1);
+ mutex_unlock(&pdata->lock);
+
+ return sprintf(buf, "0x%016llx%016llx\n",
+ (unsigned long long)req1, (unsigned long long)req0);
+}
+static DEVICE_ATTR_RO(first_malformed_req);
+
+static struct attribute *port_err_attrs[] = {
+ &dev_attr_errors.attr,
+ &dev_attr_first_error.attr,
+ &dev_attr_first_malformed_req.attr,
+ NULL,
+};
+
+static umode_t port_err_attrs_visible(struct kobject *kobj,
+ struct attribute *attr, int n)
+{
+ struct device *dev = kobj_to_dev(kobj);
+
+ /*
+ * sysfs entries are visible only if related private feature is
+ * enumerated.
+ */
+ if (!dfl_get_feature_by_id(dev, PORT_FEATURE_ID_ERROR))
+ return 0;
+
+ return attr->mode;
+}
+
+const struct attribute_group port_err_group = {
+ .name = "errors",
+ .attrs = port_err_attrs,
+ .is_visible = port_err_attrs_visible,
+};
+
+static int port_err_init(struct platform_device *pdev,
+ struct dfl_feature *feature)
+{
+ afu_port_err_mask(&pdev->dev, false);
+
+ return 0;
+}
+
+static void port_err_uinit(struct platform_device *pdev,
+ struct dfl_feature *feature)
+{
+ afu_port_err_mask(&pdev->dev, true);
+}
+
+const struct dfl_feature_id port_err_id_table[] = {
+ {.id = PORT_FEATURE_ID_ERROR,},
+ {0,}
+};
+
+const struct dfl_feature_ops port_err_ops = {
+ .init = port_err_init,
+ .uinit = port_err_uinit,
+};
diff --git a/drivers/fpga/dfl-afu-main.c b/drivers/fpga/dfl-afu-main.c
index 02baa6a227c0..e4a34dc7947f 100644
--- a/drivers/fpga/dfl-afu-main.c
+++ b/drivers/fpga/dfl-afu-main.c
@@ -22,14 +22,17 @@
#include "dfl-afu.h"
/**
- * port_enable - enable a port
+ * __afu_port_enable - enable a port by clear reset
* @pdev: port platform device.
*
* Enable Port by clear the port soft reset bit, which is set by default.
* The AFU is unable to respond to any MMIO access while in reset.
- * port_enable function should only be used after port_disable function.
+ * __afu_port_enable function should only be used after __afu_port_disable
+ * function.
+ *
+ * The caller needs to hold lock for protection.
*/
-static void port_enable(struct platform_device *pdev)
+void __afu_port_enable(struct platform_device *pdev)
{
struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev->dev);
void __iomem *base;
@@ -52,13 +55,14 @@ static void port_enable(struct platform_device *pdev)
#define RST_POLL_TIMEOUT 1000 /* us */
/**
- * port_disable - disable a port
+ * __afu_port_disable - disable a port by hold reset
* @pdev: port platform device.
*
- * Disable Port by setting the port soft reset bit, it puts the port into
- * reset.
+ * Disable Port by setting the port soft reset bit, it puts the port into reset.
+ *
+ * The caller needs to hold lock for protection.
*/
-static int port_disable(struct platform_device *pdev)
+int __afu_port_disable(struct platform_device *pdev)
{
struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev->dev);
void __iomem *base;
@@ -104,9 +108,9 @@ static int __port_reset(struct platform_device *pdev)
{
int ret;
- ret = port_disable(pdev);
+ ret = __afu_port_disable(pdev);
if (!ret)
- port_enable(pdev);
+ __afu_port_enable(pdev);
return ret;
}
@@ -141,27 +145,267 @@ id_show(struct device *dev, struct device_attribute *attr, char *buf)
}
static DEVICE_ATTR_RO(id);
-static const struct attribute *port_hdr_attrs[] = {
+static ssize_t
+ltr_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ u64 v;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_HEADER);
+
+ mutex_lock(&pdata->lock);
+ v = readq(base + PORT_HDR_CTRL);
+ mutex_unlock(&pdata->lock);
+
+ return sprintf(buf, "%x\n", (u8)FIELD_GET(PORT_CTRL_LATENCY, v));
+}
+
+static ssize_t
+ltr_store(struct device *dev, struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ bool ltr;
+ u64 v;
+
+ if (kstrtobool(buf, &ltr))
+ return -EINVAL;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_HEADER);
+
+ mutex_lock(&pdata->lock);
+ v = readq(base + PORT_HDR_CTRL);
+ v &= ~PORT_CTRL_LATENCY;
+ v |= FIELD_PREP(PORT_CTRL_LATENCY, ltr ? 1 : 0);
+ writeq(v, base + PORT_HDR_CTRL);
+ mutex_unlock(&pdata->lock);
+
+ return count;
+}
+static DEVICE_ATTR_RW(ltr);
+
+static ssize_t
+ap1_event_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ u64 v;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_HEADER);
+
+ mutex_lock(&pdata->lock);
+ v = readq(base + PORT_HDR_STS);
+ mutex_unlock(&pdata->lock);
+
+ return sprintf(buf, "%x\n", (u8)FIELD_GET(PORT_STS_AP1_EVT, v));
+}
+
+static ssize_t
+ap1_event_store(struct device *dev, struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ bool clear;
+
+ if (kstrtobool(buf, &clear) || !clear)
+ return -EINVAL;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_HEADER);
+
+ mutex_lock(&pdata->lock);
+ writeq(PORT_STS_AP1_EVT, base + PORT_HDR_STS);
+ mutex_unlock(&pdata->lock);
+
+ return count;
+}
+static DEVICE_ATTR_RW(ap1_event);
+
+static ssize_t
+ap2_event_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ u64 v;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_HEADER);
+
+ mutex_lock(&pdata->lock);
+ v = readq(base + PORT_HDR_STS);
+ mutex_unlock(&pdata->lock);
+
+ return sprintf(buf, "%x\n", (u8)FIELD_GET(PORT_STS_AP2_EVT, v));
+}
+
+static ssize_t
+ap2_event_store(struct device *dev, struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ bool clear;
+
+ if (kstrtobool(buf, &clear) || !clear)
+ return -EINVAL;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_HEADER);
+
+ mutex_lock(&pdata->lock);
+ writeq(PORT_STS_AP2_EVT, base + PORT_HDR_STS);
+ mutex_unlock(&pdata->lock);
+
+ return count;
+}
+static DEVICE_ATTR_RW(ap2_event);
+
+static ssize_t
+power_state_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ u64 v;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_HEADER);
+
+ mutex_lock(&pdata->lock);
+ v = readq(base + PORT_HDR_STS);
+ mutex_unlock(&pdata->lock);
+
+ return sprintf(buf, "0x%x\n", (u8)FIELD_GET(PORT_STS_PWR_STATE, v));
+}
+static DEVICE_ATTR_RO(power_state);
+
+static ssize_t
+userclk_freqcmd_store(struct device *dev, struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ u64 userclk_freq_cmd;
+ void __iomem *base;
+
+ if (kstrtou64(buf, 0, &userclk_freq_cmd))
+ return -EINVAL;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_HEADER);
+
+ mutex_lock(&pdata->lock);
+ writeq(userclk_freq_cmd, base + PORT_HDR_USRCLK_CMD0);
+ mutex_unlock(&pdata->lock);
+
+ return count;
+}
+static DEVICE_ATTR_WO(userclk_freqcmd);
+
+static ssize_t
+userclk_freqcntrcmd_store(struct device *dev, struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ u64 userclk_freqcntr_cmd;
+ void __iomem *base;
+
+ if (kstrtou64(buf, 0, &userclk_freqcntr_cmd))
+ return -EINVAL;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_HEADER);
+
+ mutex_lock(&pdata->lock);
+ writeq(userclk_freqcntr_cmd, base + PORT_HDR_USRCLK_CMD1);
+ mutex_unlock(&pdata->lock);
+
+ return count;
+}
+static DEVICE_ATTR_WO(userclk_freqcntrcmd);
+
+static ssize_t
+userclk_freqsts_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ u64 userclk_freqsts;
+ void __iomem *base;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_HEADER);
+
+ mutex_lock(&pdata->lock);
+ userclk_freqsts = readq(base + PORT_HDR_USRCLK_STS0);
+ mutex_unlock(&pdata->lock);
+
+ return sprintf(buf, "0x%llx\n", (unsigned long long)userclk_freqsts);
+}
+static DEVICE_ATTR_RO(userclk_freqsts);
+
+static ssize_t
+userclk_freqcntrsts_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ u64 userclk_freqcntrsts;
+ void __iomem *base;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_HEADER);
+
+ mutex_lock(&pdata->lock);
+ userclk_freqcntrsts = readq(base + PORT_HDR_USRCLK_STS1);
+ mutex_unlock(&pdata->lock);
+
+ return sprintf(buf, "0x%llx\n",
+ (unsigned long long)userclk_freqcntrsts);
+}
+static DEVICE_ATTR_RO(userclk_freqcntrsts);
+
+static struct attribute *port_hdr_attrs[] = {
&dev_attr_id.attr,
+ &dev_attr_ltr.attr,
+ &dev_attr_ap1_event.attr,
+ &dev_attr_ap2_event.attr,
+ &dev_attr_power_state.attr,
+ &dev_attr_userclk_freqcmd.attr,
+ &dev_attr_userclk_freqcntrcmd.attr,
+ &dev_attr_userclk_freqsts.attr,
+ &dev_attr_userclk_freqcntrsts.attr,
NULL,
};
-static int port_hdr_init(struct platform_device *pdev,
- struct dfl_feature *feature)
+static umode_t port_hdr_attrs_visible(struct kobject *kobj,
+ struct attribute *attr, int n)
{
- dev_dbg(&pdev->dev, "PORT HDR Init.\n");
+ struct device *dev = kobj_to_dev(kobj);
+ umode_t mode = attr->mode;
+ void __iomem *base;
- port_reset(pdev);
+ base = dfl_get_feature_ioaddr_by_id(dev, PORT_FEATURE_ID_HEADER);
+
+ if (dfl_feature_revision(base) > 0) {
+ /*
+ * userclk sysfs interfaces are only visible in case port
+ * revision is 0, as hardware with revision >0 doesn't
+ * support this.
+ */
+ if (attr == &dev_attr_userclk_freqcmd.attr ||
+ attr == &dev_attr_userclk_freqcntrcmd.attr ||
+ attr == &dev_attr_userclk_freqsts.attr ||
+ attr == &dev_attr_userclk_freqcntrsts.attr)
+ mode = 0;
+ }
- return sysfs_create_files(&pdev->dev.kobj, port_hdr_attrs);
+ return mode;
}
-static void port_hdr_uinit(struct platform_device *pdev,
- struct dfl_feature *feature)
+static const struct attribute_group port_hdr_group = {
+ .attrs = port_hdr_attrs,
+ .is_visible = port_hdr_attrs_visible,
+};
+
+static int port_hdr_init(struct platform_device *pdev,
+ struct dfl_feature *feature)
{
- dev_dbg(&pdev->dev, "PORT HDR UInit.\n");
+ port_reset(pdev);
- sysfs_remove_files(&pdev->dev.kobj, port_hdr_attrs);
+ return 0;
}
static long
@@ -185,9 +429,13 @@ port_hdr_ioctl(struct platform_device *pdev, struct dfl_feature *feature,
return ret;
}
+static const struct dfl_feature_id port_hdr_id_table[] = {
+ {.id = PORT_FEATURE_ID_HEADER,},
+ {0,}
+};
+
static const struct dfl_feature_ops port_hdr_ops = {
.init = port_hdr_init,
- .uinit = port_hdr_uinit,
.ioctl = port_hdr_ioctl,
};
@@ -214,52 +462,91 @@ afu_id_show(struct device *dev, struct device_attribute *attr, char *buf)
}
static DEVICE_ATTR_RO(afu_id);
-static const struct attribute *port_afu_attrs[] = {
+static struct attribute *port_afu_attrs[] = {
&dev_attr_afu_id.attr,
NULL
};
+static umode_t port_afu_attrs_visible(struct kobject *kobj,
+ struct attribute *attr, int n)
+{
+ struct device *dev = kobj_to_dev(kobj);
+
+ /*
+ * sysfs entries are visible only if related private feature is
+ * enumerated.
+ */
+ if (!dfl_get_feature_by_id(dev, PORT_FEATURE_ID_AFU))
+ return 0;
+
+ return attr->mode;
+}
+
+static const struct attribute_group port_afu_group = {
+ .attrs = port_afu_attrs,
+ .is_visible = port_afu_attrs_visible,
+};
+
static int port_afu_init(struct platform_device *pdev,
struct dfl_feature *feature)
{
struct resource *res = &pdev->resource[feature->resource_index];
- int ret;
- dev_dbg(&pdev->dev, "PORT AFU Init.\n");
+ return afu_mmio_region_add(dev_get_platdata(&pdev->dev),
+ DFL_PORT_REGION_INDEX_AFU,
+ resource_size(res), res->start,
+ DFL_PORT_REGION_MMAP | DFL_PORT_REGION_READ |
+ DFL_PORT_REGION_WRITE);
+}
- ret = afu_mmio_region_add(dev_get_platdata(&pdev->dev),
- DFL_PORT_REGION_INDEX_AFU, resource_size(res),
- res->start, DFL_PORT_REGION_READ |
- DFL_PORT_REGION_WRITE | DFL_PORT_REGION_MMAP);
- if (ret)
- return ret;
+static const struct dfl_feature_id port_afu_id_table[] = {
+ {.id = PORT_FEATURE_ID_AFU,},
+ {0,}
+};
- return sysfs_create_files(&pdev->dev.kobj, port_afu_attrs);
-}
+static const struct dfl_feature_ops port_afu_ops = {
+ .init = port_afu_init,
+};
-static void port_afu_uinit(struct platform_device *pdev,
- struct dfl_feature *feature)
+static int port_stp_init(struct platform_device *pdev,
+ struct dfl_feature *feature)
{
- dev_dbg(&pdev->dev, "PORT AFU UInit.\n");
+ struct resource *res = &pdev->resource[feature->resource_index];
- sysfs_remove_files(&pdev->dev.kobj, port_afu_attrs);
+ return afu_mmio_region_add(dev_get_platdata(&pdev->dev),
+ DFL_PORT_REGION_INDEX_STP,
+ resource_size(res), res->start,
+ DFL_PORT_REGION_MMAP | DFL_PORT_REGION_READ |
+ DFL_PORT_REGION_WRITE);
}
-static const struct dfl_feature_ops port_afu_ops = {
- .init = port_afu_init,
- .uinit = port_afu_uinit,
+static const struct dfl_feature_id port_stp_id_table[] = {
+ {.id = PORT_FEATURE_ID_STP,},
+ {0,}
+};
+
+static const struct dfl_feature_ops port_stp_ops = {
+ .init = port_stp_init,
};
static struct dfl_feature_driver port_feature_drvs[] = {
{
- .id = PORT_FEATURE_ID_HEADER,
+ .id_table = port_hdr_id_table,
.ops = &port_hdr_ops,
},
{
- .id = PORT_FEATURE_ID_AFU,
+ .id_table = port_afu_id_table,
.ops = &port_afu_ops,
},
{
+ .id_table = port_err_id_table,
+ .ops = &port_err_ops,
+ },
+ {
+ .id_table = port_stp_id_table,
+ .ops = &port_stp_ops,
+ },
+ {
.ops = NULL,
}
};
@@ -545,9 +832,9 @@ static int port_enable_set(struct platform_device *pdev, bool enable)
mutex_lock(&pdata->lock);
if (enable)
- port_enable(pdev);
+ __afu_port_enable(pdev);
else
- ret = port_disable(pdev);
+ ret = __afu_port_disable(pdev);
mutex_unlock(&pdata->lock);
return ret;
@@ -599,9 +886,17 @@ static int afu_remove(struct platform_device *pdev)
return 0;
}
+static const struct attribute_group *afu_dev_groups[] = {
+ &port_hdr_group,
+ &port_afu_group,
+ &port_err_group,
+ NULL
+};
+
static struct platform_driver afu_driver = {
.driver = {
- .name = DFL_FPGA_FEATURE_DEV_PORT,
+ .name = DFL_FPGA_FEATURE_DEV_PORT,
+ .dev_groups = afu_dev_groups,
},
.probe = afu_probe,
.remove = afu_remove,
diff --git a/drivers/fpga/dfl-afu.h b/drivers/fpga/dfl-afu.h
index 0c7630ae3cda..576e94960086 100644
--- a/drivers/fpga/dfl-afu.h
+++ b/drivers/fpga/dfl-afu.h
@@ -79,6 +79,10 @@ struct dfl_afu {
struct dfl_feature_platform_data *pdata;
};
+/* hold pdata->lock when call __afu_port_enable/disable */
+void __afu_port_enable(struct platform_device *pdev);
+int __afu_port_disable(struct platform_device *pdev);
+
void afu_mmio_region_init(struct dfl_feature_platform_data *pdata);
int afu_mmio_region_add(struct dfl_feature_platform_data *pdata,
u32 region_index, u64 region_size, u64 phys, u32 flags);
@@ -97,4 +101,9 @@ int afu_dma_unmap_region(struct dfl_feature_platform_data *pdata, u64 iova);
struct dfl_afu_dma_region *
afu_dma_region_find(struct dfl_feature_platform_data *pdata,
u64 iova, u64 size);
+
+extern const struct dfl_feature_ops port_err_ops;
+extern const struct dfl_feature_id port_err_id_table[];
+extern const struct attribute_group port_err_group;
+
#endif /* __DFL_AFU_H */
diff --git a/drivers/fpga/dfl-fme-error.c b/drivers/fpga/dfl-fme-error.c
new file mode 100644
index 000000000000..f897d414b923
--- /dev/null
+++ b/drivers/fpga/dfl-fme-error.c
@@ -0,0 +1,359 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for FPGA Management Engine Error Management
+ *
+ * Copyright 2019 Intel Corporation, Inc.
+ *
+ * Authors:
+ * Kang Luwei <luwei.kang@intel.com>
+ * Xiao Guangrong <guangrong.xiao@linux.intel.com>
+ * Wu Hao <hao.wu@intel.com>
+ * Joseph Grecco <joe.grecco@intel.com>
+ * Enno Luebbers <enno.luebbers@intel.com>
+ * Tim Whisonant <tim.whisonant@intel.com>
+ * Ananda Ravuri <ananda.ravuri@intel.com>
+ * Mitchel, Henry <henry.mitchel@intel.com>
+ */
+
+#include <linux/uaccess.h>
+
+#include "dfl.h"
+#include "dfl-fme.h"
+
+#define FME_ERROR_MASK 0x8
+#define FME_ERROR 0x10
+#define MBP_ERROR BIT_ULL(6)
+#define PCIE0_ERROR_MASK 0x18
+#define PCIE0_ERROR 0x20
+#define PCIE1_ERROR_MASK 0x28
+#define PCIE1_ERROR 0x30
+#define FME_FIRST_ERROR 0x38
+#define FME_NEXT_ERROR 0x40
+#define RAS_NONFAT_ERROR_MASK 0x48
+#define RAS_NONFAT_ERROR 0x50
+#define RAS_CATFAT_ERROR_MASK 0x58
+#define RAS_CATFAT_ERROR 0x60
+#define RAS_ERROR_INJECT 0x68
+#define INJECT_ERROR_MASK GENMASK_ULL(2, 0)
+
+#define ERROR_MASK GENMASK_ULL(63, 0)
+
+static ssize_t pcie0_errors_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ u64 value;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, FME_FEATURE_ID_GLOBAL_ERR);
+
+ mutex_lock(&pdata->lock);
+ value = readq(base + PCIE0_ERROR);
+ mutex_unlock(&pdata->lock);
+
+ return sprintf(buf, "0x%llx\n", (unsigned long long)value);
+}
+
+static ssize_t pcie0_errors_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ int ret = 0;
+ u64 v, val;
+
+ if (kstrtou64(buf, 0, &val))
+ return -EINVAL;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, FME_FEATURE_ID_GLOBAL_ERR);
+
+ mutex_lock(&pdata->lock);
+ writeq(GENMASK_ULL(63, 0), base + PCIE0_ERROR_MASK);
+
+ v = readq(base + PCIE0_ERROR);
+ if (val == v)
+ writeq(v, base + PCIE0_ERROR);
+ else
+ ret = -EINVAL;
+
+ writeq(0ULL, base + PCIE0_ERROR_MASK);
+ mutex_unlock(&pdata->lock);
+ return ret ? ret : count;
+}
+static DEVICE_ATTR_RW(pcie0_errors);
+
+static ssize_t pcie1_errors_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ u64 value;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, FME_FEATURE_ID_GLOBAL_ERR);
+
+ mutex_lock(&pdata->lock);
+ value = readq(base + PCIE1_ERROR);
+ mutex_unlock(&pdata->lock);
+
+ return sprintf(buf, "0x%llx\n", (unsigned long long)value);
+}
+
+static ssize_t pcie1_errors_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ int ret = 0;
+ u64 v, val;
+
+ if (kstrtou64(buf, 0, &val))
+ return -EINVAL;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, FME_FEATURE_ID_GLOBAL_ERR);
+
+ mutex_lock(&pdata->lock);
+ writeq(GENMASK_ULL(63, 0), base + PCIE1_ERROR_MASK);
+
+ v = readq(base + PCIE1_ERROR);
+ if (val == v)
+ writeq(v, base + PCIE1_ERROR);
+ else
+ ret = -EINVAL;
+
+ writeq(0ULL, base + PCIE1_ERROR_MASK);
+ mutex_unlock(&pdata->lock);
+ return ret ? ret : count;
+}
+static DEVICE_ATTR_RW(pcie1_errors);
+
+static ssize_t nonfatal_errors_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ void __iomem *base;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, FME_FEATURE_ID_GLOBAL_ERR);
+
+ return sprintf(buf, "0x%llx\n",
+ (unsigned long long)readq(base + RAS_NONFAT_ERROR));
+}
+static DEVICE_ATTR_RO(nonfatal_errors);
+
+static ssize_t catfatal_errors_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ void __iomem *base;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, FME_FEATURE_ID_GLOBAL_ERR);
+
+ return sprintf(buf, "0x%llx\n",
+ (unsigned long long)readq(base + RAS_CATFAT_ERROR));
+}
+static DEVICE_ATTR_RO(catfatal_errors);
+
+static ssize_t inject_errors_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ u64 v;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, FME_FEATURE_ID_GLOBAL_ERR);
+
+ mutex_lock(&pdata->lock);
+ v = readq(base + RAS_ERROR_INJECT);
+ mutex_unlock(&pdata->lock);
+
+ return sprintf(buf, "0x%llx\n",
+ (unsigned long long)FIELD_GET(INJECT_ERROR_MASK, v));
+}
+
+static ssize_t inject_errors_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ u8 inject_error;
+ u64 v;
+
+ if (kstrtou8(buf, 0, &inject_error))
+ return -EINVAL;
+
+ if (inject_error & ~INJECT_ERROR_MASK)
+ return -EINVAL;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, FME_FEATURE_ID_GLOBAL_ERR);
+
+ mutex_lock(&pdata->lock);
+ v = readq(base + RAS_ERROR_INJECT);
+ v &= ~INJECT_ERROR_MASK;
+ v |= FIELD_PREP(INJECT_ERROR_MASK, inject_error);
+ writeq(v, base + RAS_ERROR_INJECT);
+ mutex_unlock(&pdata->lock);
+
+ return count;
+}
+static DEVICE_ATTR_RW(inject_errors);
+
+static ssize_t fme_errors_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ u64 value;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, FME_FEATURE_ID_GLOBAL_ERR);
+
+ mutex_lock(&pdata->lock);
+ value = readq(base + FME_ERROR);
+ mutex_unlock(&pdata->lock);
+
+ return sprintf(buf, "0x%llx\n", (unsigned long long)value);
+}
+
+static ssize_t fme_errors_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ u64 v, val;
+ int ret = 0;
+
+ if (kstrtou64(buf, 0, &val))
+ return -EINVAL;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, FME_FEATURE_ID_GLOBAL_ERR);
+
+ mutex_lock(&pdata->lock);
+ writeq(GENMASK_ULL(63, 0), base + FME_ERROR_MASK);
+
+ v = readq(base + FME_ERROR);
+ if (val == v)
+ writeq(v, base + FME_ERROR);
+ else
+ ret = -EINVAL;
+
+ /* Workaround: disable MBP_ERROR if feature revision is 0 */
+ writeq(dfl_feature_revision(base) ? 0ULL : MBP_ERROR,
+ base + FME_ERROR_MASK);
+ mutex_unlock(&pdata->lock);
+ return ret ? ret : count;
+}
+static DEVICE_ATTR_RW(fme_errors);
+
+static ssize_t first_error_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ u64 value;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, FME_FEATURE_ID_GLOBAL_ERR);
+
+ mutex_lock(&pdata->lock);
+ value = readq(base + FME_FIRST_ERROR);
+ mutex_unlock(&pdata->lock);
+
+ return sprintf(buf, "0x%llx\n", (unsigned long long)value);
+}
+static DEVICE_ATTR_RO(first_error);
+
+static ssize_t next_error_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+ u64 value;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, FME_FEATURE_ID_GLOBAL_ERR);
+
+ mutex_lock(&pdata->lock);
+ value = readq(base + FME_NEXT_ERROR);
+ mutex_unlock(&pdata->lock);
+
+ return sprintf(buf, "0x%llx\n", (unsigned long long)value);
+}
+static DEVICE_ATTR_RO(next_error);
+
+static struct attribute *fme_global_err_attrs[] = {
+ &dev_attr_pcie0_errors.attr,
+ &dev_attr_pcie1_errors.attr,
+ &dev_attr_nonfatal_errors.attr,
+ &dev_attr_catfatal_errors.attr,
+ &dev_attr_inject_errors.attr,
+ &dev_attr_fme_errors.attr,
+ &dev_attr_first_error.attr,
+ &dev_attr_next_error.attr,
+ NULL,
+};
+
+static umode_t fme_global_err_attrs_visible(struct kobject *kobj,
+ struct attribute *attr, int n)
+{
+ struct device *dev = kobj_to_dev(kobj);
+
+ /*
+ * sysfs entries are visible only if related private feature is
+ * enumerated.
+ */
+ if (!dfl_get_feature_by_id(dev, FME_FEATURE_ID_GLOBAL_ERR))
+ return 0;
+
+ return attr->mode;
+}
+
+const struct attribute_group fme_global_err_group = {
+ .name = "errors",
+ .attrs = fme_global_err_attrs,
+ .is_visible = fme_global_err_attrs_visible,
+};
+
+static void fme_err_mask(struct device *dev, bool mask)
+{
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(dev);
+ void __iomem *base;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, FME_FEATURE_ID_GLOBAL_ERR);
+
+ mutex_lock(&pdata->lock);
+
+ /* Workaround: keep MBP_ERROR always masked if revision is 0 */
+ if (dfl_feature_revision(base))
+ writeq(mask ? ERROR_MASK : 0, base + FME_ERROR_MASK);
+ else
+ writeq(mask ? ERROR_MASK : MBP_ERROR, base + FME_ERROR_MASK);
+
+ writeq(mask ? ERROR_MASK : 0, base + PCIE0_ERROR_MASK);
+ writeq(mask ? ERROR_MASK : 0, base + PCIE1_ERROR_MASK);
+ writeq(mask ? ERROR_MASK : 0, base + RAS_NONFAT_ERROR_MASK);
+ writeq(mask ? ERROR_MASK : 0, base + RAS_CATFAT_ERROR_MASK);
+
+ mutex_unlock(&pdata->lock);
+}
+
+static int fme_global_err_init(struct platform_device *pdev,
+ struct dfl_feature *feature)
+{
+ fme_err_mask(&pdev->dev, false);
+
+ return 0;
+}
+
+static void fme_global_err_uinit(struct platform_device *pdev,
+ struct dfl_feature *feature)
+{
+ fme_err_mask(&pdev->dev, true);
+}
+
+const struct dfl_feature_id fme_global_err_id_table[] = {
+ {.id = FME_FEATURE_ID_GLOBAL_ERR,},
+ {0,}
+};
+
+const struct dfl_feature_ops fme_global_err_ops = {
+ .init = fme_global_err_init,
+ .uinit = fme_global_err_uinit,
+};
diff --git a/drivers/fpga/dfl-fme-main.c b/drivers/fpga/dfl-fme-main.c
index 086ad2420ade..4d78e182878f 100644
--- a/drivers/fpga/dfl-fme-main.c
+++ b/drivers/fpga/dfl-fme-main.c
@@ -16,6 +16,7 @@
#include <linux/kernel.h>
#include <linux/module.h>
+#include <linux/uaccess.h>
#include <linux/fpga-dfl.h>
#include "dfl.h"
@@ -72,50 +73,126 @@ static ssize_t bitstream_metadata_show(struct device *dev,
}
static DEVICE_ATTR_RO(bitstream_metadata);
-static const struct attribute *fme_hdr_attrs[] = {
+static ssize_t cache_size_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ void __iomem *base;
+ u64 v;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, FME_FEATURE_ID_HEADER);
+
+ v = readq(base + FME_HDR_CAP);
+
+ return sprintf(buf, "%u\n",
+ (unsigned int)FIELD_GET(FME_CAP_CACHE_SIZE, v));
+}
+static DEVICE_ATTR_RO(cache_size);
+
+static ssize_t fabric_version_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ void __iomem *base;
+ u64 v;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, FME_FEATURE_ID_HEADER);
+
+ v = readq(base + FME_HDR_CAP);
+
+ return sprintf(buf, "%u\n",
+ (unsigned int)FIELD_GET(FME_CAP_FABRIC_VERID, v));
+}
+static DEVICE_ATTR_RO(fabric_version);
+
+static ssize_t socket_id_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ void __iomem *base;
+ u64 v;
+
+ base = dfl_get_feature_ioaddr_by_id(dev, FME_FEATURE_ID_HEADER);
+
+ v = readq(base + FME_HDR_CAP);
+
+ return sprintf(buf, "%u\n",
+ (unsigned int)FIELD_GET(FME_CAP_SOCKET_ID, v));
+}
+static DEVICE_ATTR_RO(socket_id);
+
+static struct attribute *fme_hdr_attrs[] = {
&dev_attr_ports_num.attr,
&dev_attr_bitstream_id.attr,
&dev_attr_bitstream_metadata.attr,
+ &dev_attr_cache_size.attr,
+ &dev_attr_fabric_version.attr,
+ &dev_attr_socket_id.attr,
NULL,
};
-static int fme_hdr_init(struct platform_device *pdev,
- struct dfl_feature *feature)
+static const struct attribute_group fme_hdr_group = {
+ .attrs = fme_hdr_attrs,
+};
+
+static long fme_hdr_ioctl_release_port(struct dfl_feature_platform_data *pdata,
+ unsigned long arg)
{
- void __iomem *base = feature->ioaddr;
- int ret;
+ struct dfl_fpga_cdev *cdev = pdata->dfl_cdev;
+ int port_id;
- dev_dbg(&pdev->dev, "FME HDR Init.\n");
- dev_dbg(&pdev->dev, "FME cap %llx.\n",
- (unsigned long long)readq(base + FME_HDR_CAP));
+ if (get_user(port_id, (int __user *)arg))
+ return -EFAULT;
- ret = sysfs_create_files(&pdev->dev.kobj, fme_hdr_attrs);
- if (ret)
- return ret;
+ return dfl_fpga_cdev_release_port(cdev, port_id);
+}
- return 0;
+static long fme_hdr_ioctl_assign_port(struct dfl_feature_platform_data *pdata,
+ unsigned long arg)
+{
+ struct dfl_fpga_cdev *cdev = pdata->dfl_cdev;
+ int port_id;
+
+ if (get_user(port_id, (int __user *)arg))
+ return -EFAULT;
+
+ return dfl_fpga_cdev_assign_port(cdev, port_id);
}
-static void fme_hdr_uinit(struct platform_device *pdev,
- struct dfl_feature *feature)
+static long fme_hdr_ioctl(struct platform_device *pdev,
+ struct dfl_feature *feature,
+ unsigned int cmd, unsigned long arg)
{
- dev_dbg(&pdev->dev, "FME HDR UInit.\n");
- sysfs_remove_files(&pdev->dev.kobj, fme_hdr_attrs);
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev->dev);
+
+ switch (cmd) {
+ case DFL_FPGA_FME_PORT_RELEASE:
+ return fme_hdr_ioctl_release_port(pdata, arg);
+ case DFL_FPGA_FME_PORT_ASSIGN:
+ return fme_hdr_ioctl_assign_port(pdata, arg);
+ }
+
+ return -ENODEV;
}
+static const struct dfl_feature_id fme_hdr_id_table[] = {
+ {.id = FME_FEATURE_ID_HEADER,},
+ {0,}
+};
+
static const struct dfl_feature_ops fme_hdr_ops = {
- .init = fme_hdr_init,
- .uinit = fme_hdr_uinit,
+ .ioctl = fme_hdr_ioctl,
};
static struct dfl_feature_driver fme_feature_drvs[] = {
{
- .id = FME_FEATURE_ID_HEADER,
+ .id_table = fme_hdr_id_table,
.ops = &fme_hdr_ops,
},
{
- .id = FME_FEATURE_ID_PR_MGMT,
- .ops = &pr_mgmt_ops,
+ .id_table = fme_pr_mgmt_id_table,
+ .ops = &fme_pr_mgmt_ops,
+ },
+ {
+ .id_table = fme_global_err_id_table,
+ .ops = &fme_global_err_ops,
},
{
.ops = NULL,
@@ -263,9 +340,16 @@ static int fme_remove(struct platform_device *pdev)
return 0;
}
+static const struct attribute_group *fme_dev_groups[] = {
+ &fme_hdr_group,
+ &fme_global_err_group,
+ NULL
+};
+
static struct platform_driver fme_driver = {
.driver = {
- .name = DFL_FPGA_FEATURE_DEV_FME,
+ .name = DFL_FPGA_FEATURE_DEV_FME,
+ .dev_groups = fme_dev_groups,
},
.probe = fme_probe,
.remove = fme_remove,
diff --git a/drivers/fpga/dfl-fme-mgr.c b/drivers/fpga/dfl-fme-mgr.c
index 76f37709dd1a..b3f7eee3c93f 100644
--- a/drivers/fpga/dfl-fme-mgr.c
+++ b/drivers/fpga/dfl-fme-mgr.c
@@ -30,8 +30,8 @@
#define FME_PR_STS 0x10
#define FME_PR_DATA 0x18
#define FME_PR_ERR 0x20
-#define FME_PR_INTFC_ID_H 0xA8
-#define FME_PR_INTFC_ID_L 0xB0
+#define FME_PR_INTFC_ID_L 0xA8
+#define FME_PR_INTFC_ID_H 0xB0
/* FME PR Control Register Bitfield */
#define FME_PR_CTRL_PR_RST BIT_ULL(0) /* Reset PR engine */
diff --git a/drivers/fpga/dfl-fme-pr.c b/drivers/fpga/dfl-fme-pr.c
index d9ca9554844a..a233a53db708 100644
--- a/drivers/fpga/dfl-fme-pr.c
+++ b/drivers/fpga/dfl-fme-pr.c
@@ -74,6 +74,7 @@ static int fme_pr(struct platform_device *pdev, unsigned long arg)
struct dfl_fme *fme;
unsigned long minsz;
void *buf = NULL;
+ size_t length;
int ret = 0;
u64 v;
@@ -85,9 +86,6 @@ static int fme_pr(struct platform_device *pdev, unsigned long arg)
if (port_pr.argsz < minsz || port_pr.flags)
return -EINVAL;
- if (!IS_ALIGNED(port_pr.buffer_size, 4))
- return -EINVAL;
-
/* get fme header region */
fme_hdr = dfl_get_feature_ioaddr_by_id(&pdev->dev,
FME_FEATURE_ID_HEADER);
@@ -103,7 +101,13 @@ static int fme_pr(struct platform_device *pdev, unsigned long arg)
port_pr.buffer_size))
return -EFAULT;
- buf = vmalloc(port_pr.buffer_size);
+ /*
+ * align PR buffer per PR bandwidth, as HW ignores the extra padding
+ * data automatically.
+ */
+ length = ALIGN(port_pr.buffer_size, 4);
+
+ buf = vmalloc(length);
if (!buf)
return -ENOMEM;
@@ -140,7 +144,7 @@ static int fme_pr(struct platform_device *pdev, unsigned long arg)
fpga_image_info_free(region->info);
info->buf = buf;
- info->count = port_pr.buffer_size;
+ info->count = length;
info->region_id = port_pr.port_id;
region->info = info;
@@ -159,9 +163,6 @@ unlock_exit:
mutex_unlock(&pdata->lock);
free_exit:
vfree(buf);
- if (copy_to_user((void __user *)arg, &port_pr, minsz))
- return -EFAULT;
-
return ret;
}
@@ -469,7 +470,12 @@ static long fme_pr_ioctl(struct platform_device *pdev,
return ret;
}
-const struct dfl_feature_ops pr_mgmt_ops = {
+const struct dfl_feature_id fme_pr_mgmt_id_table[] = {
+ {.id = FME_FEATURE_ID_PR_MGMT,},
+ {0}
+};
+
+const struct dfl_feature_ops fme_pr_mgmt_ops = {
.init = pr_mgmt_init,
.uinit = pr_mgmt_uinit,
.ioctl = fme_pr_ioctl,
diff --git a/drivers/fpga/dfl-fme.h b/drivers/fpga/dfl-fme.h
index 5394a216c5c0..6685c8ef965b 100644
--- a/drivers/fpga/dfl-fme.h
+++ b/drivers/fpga/dfl-fme.h
@@ -33,6 +33,10 @@ struct dfl_fme {
struct dfl_feature_platform_data *pdata;
};
-extern const struct dfl_feature_ops pr_mgmt_ops;
+extern const struct dfl_feature_ops fme_pr_mgmt_ops;
+extern const struct dfl_feature_id fme_pr_mgmt_id_table[];
+extern const struct dfl_feature_ops fme_global_err_ops;
+extern const struct dfl_feature_id fme_global_err_id_table[];
+extern const struct attribute_group fme_global_err_group;
#endif /* __DFL_FME_H */
diff --git a/drivers/fpga/dfl-pci.c b/drivers/fpga/dfl-pci.c
index 66b5720582bb..89ca292236ad 100644
--- a/drivers/fpga/dfl-pci.c
+++ b/drivers/fpga/dfl-pci.c
@@ -223,8 +223,43 @@ disable_error_report_exit:
return ret;
}
+static int cci_pci_sriov_configure(struct pci_dev *pcidev, int num_vfs)
+{
+ struct cci_drvdata *drvdata = pci_get_drvdata(pcidev);
+ struct dfl_fpga_cdev *cdev = drvdata->cdev;
+ int ret = 0;
+
+ if (!num_vfs) {
+ /*
+ * disable SRIOV and then put released ports back to default
+ * PF access mode.
+ */
+ pci_disable_sriov(pcidev);
+
+ dfl_fpga_cdev_config_ports_pf(cdev);
+
+ } else {
+ /*
+ * before enable SRIOV, put released ports into VF access mode
+ * first of all.
+ */
+ ret = dfl_fpga_cdev_config_ports_vf(cdev, num_vfs);
+ if (ret)
+ return ret;
+
+ ret = pci_enable_sriov(pcidev, num_vfs);
+ if (ret)
+ dfl_fpga_cdev_config_ports_pf(cdev);
+ }
+
+ return ret;
+}
+
static void cci_pci_remove(struct pci_dev *pcidev)
{
+ if (dev_is_pf(&pcidev->dev))
+ cci_pci_sriov_configure(pcidev, 0);
+
cci_remove_feature_devs(pcidev);
pci_disable_pcie_error_reporting(pcidev);
}
@@ -234,6 +269,7 @@ static struct pci_driver cci_pci_driver = {
.id_table = cci_pcie_id_tbl,
.probe = cci_pci_probe,
.remove = cci_pci_remove,
+ .sriov_configure = cci_pci_sriov_configure,
};
module_pci_driver(cci_pci_driver);
diff --git a/drivers/fpga/dfl.c b/drivers/fpga/dfl.c
index 4b66aaa32b5a..96a2b8274a33 100644
--- a/drivers/fpga/dfl.c
+++ b/drivers/fpga/dfl.c
@@ -231,16 +231,20 @@ EXPORT_SYMBOL_GPL(dfl_fpga_port_ops_del);
*/
int dfl_fpga_check_port_id(struct platform_device *pdev, void *pport_id)
{
- struct dfl_fpga_port_ops *port_ops = dfl_fpga_port_ops_get(pdev);
- int port_id;
+ struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev->dev);
+ struct dfl_fpga_port_ops *port_ops;
+
+ if (pdata->id != FEATURE_DEV_ID_UNUSED)
+ return pdata->id == *(int *)pport_id;
+ port_ops = dfl_fpga_port_ops_get(pdev);
if (!port_ops || !port_ops->get_id)
return 0;
- port_id = port_ops->get_id(pdev);
+ pdata->id = port_ops->get_id(pdev);
dfl_fpga_port_ops_put(port_ops);
- return port_id == *(int *)pport_id;
+ return pdata->id == *(int *)pport_id;
}
EXPORT_SYMBOL_GPL(dfl_fpga_check_port_id);
@@ -255,7 +259,8 @@ void dfl_fpga_dev_feature_uinit(struct platform_device *pdev)
dfl_fpga_dev_for_each_feature(pdata, feature)
if (feature->ops) {
- feature->ops->uinit(pdev, feature);
+ if (feature->ops->uinit)
+ feature->ops->uinit(pdev, feature);
feature->ops = NULL;
}
}
@@ -266,17 +271,34 @@ static int dfl_feature_instance_init(struct platform_device *pdev,
struct dfl_feature *feature,
struct dfl_feature_driver *drv)
{
- int ret;
+ int ret = 0;
- ret = drv->ops->init(pdev, feature);
- if (ret)
- return ret;
+ if (drv->ops->init) {
+ ret = drv->ops->init(pdev, feature);
+ if (ret)
+ return ret;
+ }
feature->ops = drv->ops;
return ret;
}
+static bool dfl_feature_drv_match(struct dfl_feature *feature,
+ struct dfl_feature_driver *driver)
+{
+ const struct dfl_feature_id *ids = driver->id_table;
+
+ if (ids) {
+ while (ids->id) {
+ if (ids->id == feature->id)
+ return true;
+ ids++;
+ }
+ }
+ return false;
+}
+
/**
* dfl_fpga_dev_feature_init - init for sub features of dfl feature device
* @pdev: feature device.
@@ -297,8 +319,7 @@ int dfl_fpga_dev_feature_init(struct platform_device *pdev,
while (drv->ops) {
dfl_fpga_dev_for_each_feature(pdata, feature) {
- /* match feature and drv using id */
- if (feature->id == drv->id) {
+ if (dfl_feature_drv_match(feature, drv)) {
ret = dfl_feature_instance_init(pdev, pdata,
feature, drv);
if (ret)
@@ -474,6 +495,7 @@ static int build_info_commit_dev(struct build_feature_devs_info *binfo)
pdata->dev = fdev;
pdata->num = binfo->feature_num;
pdata->dfl_cdev = binfo->cdev;
+ pdata->id = FEATURE_DEV_ID_UNUSED;
mutex_init(&pdata->lock);
lockdep_set_class_and_name(&pdata->lock, &dfl_pdata_keys[type],
dfl_pdata_key_strings[type]);
@@ -973,25 +995,27 @@ void dfl_fpga_feature_devs_remove(struct dfl_fpga_cdev *cdev)
{
struct dfl_feature_platform_data *pdata, *ptmp;
- remove_feature_devs(cdev);
-
mutex_lock(&cdev->lock);
- if (cdev->fme_dev) {
- /* the fme should be unregistered. */
- WARN_ON(device_is_registered(cdev->fme_dev));
+ if (cdev->fme_dev)
put_device(cdev->fme_dev);
- }
list_for_each_entry_safe(pdata, ptmp, &cdev->port_dev_list, node) {
struct platform_device *port_dev = pdata->dev;
- /* the port should be unregistered. */
- WARN_ON(device_is_registered(&port_dev->dev));
+ /* remove released ports */
+ if (!device_is_registered(&port_dev->dev)) {
+ dfl_id_free(feature_dev_id_type(port_dev),
+ port_dev->id);
+ platform_device_put(port_dev);
+ }
+
list_del(&pdata->node);
put_device(&port_dev->dev);
}
mutex_unlock(&cdev->lock);
+ remove_feature_devs(cdev);
+
fpga_region_unregister(cdev->region);
devm_kfree(cdev->parent, cdev);
}
@@ -1042,6 +1066,170 @@ static int __init dfl_fpga_init(void)
return ret;
}
+/**
+ * dfl_fpga_cdev_release_port - release a port platform device
+ *
+ * @cdev: parent container device.
+ * @port_id: id of the port platform device.
+ *
+ * This function allows user to release a port platform device. This is a
+ * mandatory step before turn a port from PF into VF for SRIOV support.
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+int dfl_fpga_cdev_release_port(struct dfl_fpga_cdev *cdev, int port_id)
+{
+ struct platform_device *port_pdev;
+ int ret = -ENODEV;
+
+ mutex_lock(&cdev->lock);
+ port_pdev = __dfl_fpga_cdev_find_port(cdev, &port_id,
+ dfl_fpga_check_port_id);
+ if (!port_pdev)
+ goto unlock_exit;
+
+ if (!device_is_registered(&port_pdev->dev)) {
+ ret = -EBUSY;
+ goto put_dev_exit;
+ }
+
+ ret = dfl_feature_dev_use_begin(dev_get_platdata(&port_pdev->dev));
+ if (ret)
+ goto put_dev_exit;
+
+ platform_device_del(port_pdev);
+ cdev->released_port_num++;
+put_dev_exit:
+ put_device(&port_pdev->dev);
+unlock_exit:
+ mutex_unlock(&cdev->lock);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(dfl_fpga_cdev_release_port);
+
+/**
+ * dfl_fpga_cdev_assign_port - assign a port platform device back
+ *
+ * @cdev: parent container device.
+ * @port_id: id of the port platform device.
+ *
+ * This function allows user to assign a port platform device back. This is
+ * a mandatory step after disable SRIOV support.
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+int dfl_fpga_cdev_assign_port(struct dfl_fpga_cdev *cdev, int port_id)
+{
+ struct platform_device *port_pdev;
+ int ret = -ENODEV;
+
+ mutex_lock(&cdev->lock);
+ port_pdev = __dfl_fpga_cdev_find_port(cdev, &port_id,
+ dfl_fpga_check_port_id);
+ if (!port_pdev)
+ goto unlock_exit;
+
+ if (device_is_registered(&port_pdev->dev)) {
+ ret = -EBUSY;
+ goto put_dev_exit;
+ }
+
+ ret = platform_device_add(port_pdev);
+ if (ret)
+ goto put_dev_exit;
+
+ dfl_feature_dev_use_end(dev_get_platdata(&port_pdev->dev));
+ cdev->released_port_num--;
+put_dev_exit:
+ put_device(&port_pdev->dev);
+unlock_exit:
+ mutex_unlock(&cdev->lock);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(dfl_fpga_cdev_assign_port);
+
+static void config_port_access_mode(struct device *fme_dev, int port_id,
+ bool is_vf)
+{
+ void __iomem *base;
+ u64 v;
+
+ base = dfl_get_feature_ioaddr_by_id(fme_dev, FME_FEATURE_ID_HEADER);
+
+ v = readq(base + FME_HDR_PORT_OFST(port_id));
+
+ v &= ~FME_PORT_OFST_ACC_CTRL;
+ v |= FIELD_PREP(FME_PORT_OFST_ACC_CTRL,
+ is_vf ? FME_PORT_OFST_ACC_VF : FME_PORT_OFST_ACC_PF);
+
+ writeq(v, base + FME_HDR_PORT_OFST(port_id));
+}
+
+#define config_port_vf_mode(dev, id) config_port_access_mode(dev, id, true)
+#define config_port_pf_mode(dev, id) config_port_access_mode(dev, id, false)
+
+/**
+ * dfl_fpga_cdev_config_ports_pf - configure ports to PF access mode
+ *
+ * @cdev: parent container device.
+ *
+ * This function is needed in sriov configuration routine. It could be used to
+ * configure the all released ports from VF access mode to PF.
+ */
+void dfl_fpga_cdev_config_ports_pf(struct dfl_fpga_cdev *cdev)
+{
+ struct dfl_feature_platform_data *pdata;
+
+ mutex_lock(&cdev->lock);
+ list_for_each_entry(pdata, &cdev->port_dev_list, node) {
+ if (device_is_registered(&pdata->dev->dev))
+ continue;
+
+ config_port_pf_mode(cdev->fme_dev, pdata->id);
+ }
+ mutex_unlock(&cdev->lock);
+}
+EXPORT_SYMBOL_GPL(dfl_fpga_cdev_config_ports_pf);
+
+/**
+ * dfl_fpga_cdev_config_ports_vf - configure ports to VF access mode
+ *
+ * @cdev: parent container device.
+ * @num_vfs: VF device number.
+ *
+ * This function is needed in sriov configuration routine. It could be used to
+ * configure the released ports from PF access mode to VF.
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+int dfl_fpga_cdev_config_ports_vf(struct dfl_fpga_cdev *cdev, int num_vfs)
+{
+ struct dfl_feature_platform_data *pdata;
+ int ret = 0;
+
+ mutex_lock(&cdev->lock);
+ /*
+ * can't turn multiple ports into 1 VF device, only 1 port for 1 VF
+ * device, so if released port number doesn't match VF device number,
+ * then reject the request with -EINVAL error code.
+ */
+ if (cdev->released_port_num != num_vfs) {
+ ret = -EINVAL;
+ goto done;
+ }
+
+ list_for_each_entry(pdata, &cdev->port_dev_list, node) {
+ if (device_is_registered(&pdata->dev->dev))
+ continue;
+
+ config_port_vf_mode(cdev->fme_dev, pdata->id);
+ }
+done:
+ mutex_unlock(&cdev->lock);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(dfl_fpga_cdev_config_ports_vf);
+
static void __exit dfl_fpga_exit(void)
{
dfl_chardev_uinit();
diff --git a/drivers/fpga/dfl.h b/drivers/fpga/dfl.h
index a8b869e9e5b7..9f0e656de720 100644
--- a/drivers/fpga/dfl.h
+++ b/drivers/fpga/dfl.h
@@ -30,8 +30,8 @@
/* plus one for fme device */
#define MAX_DFL_FEATURE_DEV_NUM (MAX_DFL_FPGA_PORT_NUM + 1)
-/* Reserved 0x0 for Header Group Register and 0xff for AFU */
-#define FEATURE_ID_FIU_HEADER 0x0
+/* Reserved 0xfe for Header Group Register and 0xff for AFU */
+#define FEATURE_ID_FIU_HEADER 0xfe
#define FEATURE_ID_AFU 0xff
#define FME_FEATURE_ID_HEADER FEATURE_ID_FIU_HEADER
@@ -119,6 +119,11 @@
#define PORT_HDR_NEXT_AFU NEXT_AFU
#define PORT_HDR_CAP 0x30
#define PORT_HDR_CTRL 0x38
+#define PORT_HDR_STS 0x40
+#define PORT_HDR_USRCLK_CMD0 0x50
+#define PORT_HDR_USRCLK_CMD1 0x58
+#define PORT_HDR_USRCLK_STS0 0x60
+#define PORT_HDR_USRCLK_STS1 0x68
/* Port Capability Register Bitfield */
#define PORT_CAP_PORT_NUM GENMASK_ULL(1, 0) /* ID of this port */
@@ -130,6 +135,16 @@
/* Latency tolerance reporting. '1' >= 40us, '0' < 40us.*/
#define PORT_CTRL_LATENCY BIT_ULL(2)
#define PORT_CTRL_SFTRST_ACK BIT_ULL(4) /* HW ack for reset */
+
+/* Port Status Register Bitfield */
+#define PORT_STS_AP2_EVT BIT_ULL(13) /* AP2 event detected */
+#define PORT_STS_AP1_EVT BIT_ULL(12) /* AP1 event detected */
+#define PORT_STS_PWR_STATE GENMASK_ULL(11, 8) /* AFU power states */
+#define PORT_STS_PWR_STATE_NORM 0
+#define PORT_STS_PWR_STATE_AP1 1 /* 50% throttling */
+#define PORT_STS_PWR_STATE_AP2 2 /* 90% throttling */
+#define PORT_STS_PWR_STATE_AP6 6 /* 100% throttling */
+
/**
* struct dfl_fpga_port_ops - port ops
*
@@ -154,13 +169,22 @@ void dfl_fpga_port_ops_put(struct dfl_fpga_port_ops *ops);
int dfl_fpga_check_port_id(struct platform_device *pdev, void *pport_id);
/**
- * struct dfl_feature_driver - sub feature's driver
+ * struct dfl_feature_id - dfl private feature id
*
- * @id: sub feature id.
- * @ops: ops of this sub feature.
+ * @id: unique dfl private feature id.
*/
-struct dfl_feature_driver {
+struct dfl_feature_id {
u64 id;
+};
+
+/**
+ * struct dfl_feature_driver - dfl private feature driver
+ *
+ * @id_table: id_table for dfl private features supported by this driver.
+ * @ops: ops of this dfl private feature driver.
+ */
+struct dfl_feature_driver {
+ const struct dfl_feature_id *id_table;
const struct dfl_feature_ops *ops;
};
@@ -183,6 +207,8 @@ struct dfl_feature {
#define DEV_STATUS_IN_USE 0
+#define FEATURE_DEV_ID_UNUSED (-1)
+
/**
* struct dfl_feature_platform_data - platform data for feature devices
*
@@ -191,6 +217,7 @@ struct dfl_feature {
* @cdev: cdev of feature dev.
* @dev: ptr to platform device linked with this platform data.
* @dfl_cdev: ptr to container device.
+ * @id: id used for this feature device.
* @disable_count: count for port disable.
* @num: number for sub features.
* @dev_status: dev status (e.g. DEV_STATUS_IN_USE).
@@ -203,6 +230,7 @@ struct dfl_feature_platform_data {
struct cdev cdev;
struct platform_device *dev;
struct dfl_fpga_cdev *dfl_cdev;
+ int id;
unsigned int disable_count;
unsigned long dev_status;
void *private;
@@ -331,6 +359,11 @@ static inline bool dfl_feature_is_port(void __iomem *base)
(FIELD_GET(DFH_ID, v) == DFH_ID_FIU_PORT);
}
+static inline u8 dfl_feature_revision(void __iomem *base)
+{
+ return (u8)FIELD_GET(DFH_REVISION, readq(base + DFH));
+}
+
/**
* struct dfl_fpga_enum_info - DFL FPGA enumeration information
*
@@ -373,6 +406,7 @@ void dfl_fpga_enum_info_free(struct dfl_fpga_enum_info *info);
* @fme_dev: FME feature device under this container device.
* @lock: mutex lock to protect the port device list.
* @port_dev_list: list of all port feature devices under this container device.
+ * @released_port_num: released port number under this container device.
*/
struct dfl_fpga_cdev {
struct device *parent;
@@ -380,6 +414,7 @@ struct dfl_fpga_cdev {
struct device *fme_dev;
struct mutex lock;
struct list_head port_dev_list;
+ int released_port_num;
};
struct dfl_fpga_cdev *
@@ -407,4 +442,9 @@ dfl_fpga_cdev_find_port(struct dfl_fpga_cdev *cdev, void *data,
return pdev;
}
+
+int dfl_fpga_cdev_release_port(struct dfl_fpga_cdev *cdev, int port_id);
+int dfl_fpga_cdev_assign_port(struct dfl_fpga_cdev *cdev, int port_id);
+void dfl_fpga_cdev_config_ports_pf(struct dfl_fpga_cdev *cdev);
+int dfl_fpga_cdev_config_ports_vf(struct dfl_fpga_cdev *cdev, int num_vf);
#endif /* __FPGA_DFL_H */
diff --git a/drivers/fpga/fpga-bridge.c b/drivers/fpga/fpga-bridge.c
index 80bd8f1b2aa6..4bab9028940a 100644
--- a/drivers/fpga/fpga-bridge.c
+++ b/drivers/fpga/fpga-bridge.c
@@ -19,11 +19,6 @@ static struct class *fpga_bridge_class;
/* Lock for adding/removing bridges to linked lists*/
static spinlock_t bridge_list_lock;
-static int fpga_bridge_of_node_match(struct device *dev, const void *data)
-{
- return dev->of_node == data;
-}
-
/**
* fpga_bridge_enable - Enable transactions on the bridge
*
@@ -104,8 +99,7 @@ struct fpga_bridge *of_fpga_bridge_get(struct device_node *np,
{
struct device *dev;
- dev = class_find_device(fpga_bridge_class, NULL, np,
- fpga_bridge_of_node_match);
+ dev = class_find_device_by_of_node(fpga_bridge_class, np);
if (!dev)
return ERR_PTR(-ENODEV);
diff --git a/drivers/fpga/fpga-mgr.c b/drivers/fpga/fpga-mgr.c
index c3866816456a..e05104f5e40c 100644
--- a/drivers/fpga/fpga-mgr.c
+++ b/drivers/fpga/fpga-mgr.c
@@ -482,11 +482,6 @@ struct fpga_manager *fpga_mgr_get(struct device *dev)
}
EXPORT_SYMBOL_GPL(fpga_mgr_get);
-static int fpga_mgr_of_node_match(struct device *dev, const void *data)
-{
- return dev->of_node == data;
-}
-
/**
* of_fpga_mgr_get - Given a device node, get a reference to a fpga mgr.
*
@@ -498,8 +493,7 @@ struct fpga_manager *of_fpga_mgr_get(struct device_node *node)
{
struct device *dev;
- dev = class_find_device(fpga_mgr_class, NULL, node,
- fpga_mgr_of_node_match);
+ dev = class_find_device_by_of_node(fpga_mgr_class, node);
if (!dev)
return ERR_PTR(-ENODEV);
diff --git a/drivers/fpga/of-fpga-region.c b/drivers/fpga/of-fpga-region.c
index 75f64abf9c81..e405309baadc 100644
--- a/drivers/fpga/of-fpga-region.c
+++ b/drivers/fpga/of-fpga-region.c
@@ -22,11 +22,6 @@ static const struct of_device_id fpga_region_of_match[] = {
};
MODULE_DEVICE_TABLE(of, fpga_region_of_match);
-static int fpga_region_of_node_match(struct device *dev, const void *data)
-{
- return dev->of_node == data;
-}
-
/**
* of_fpga_region_find - find FPGA region
* @np: device node of FPGA Region
@@ -37,7 +32,7 @@ static int fpga_region_of_node_match(struct device *dev, const void *data)
*/
static struct fpga_region *of_fpga_region_find(struct device_node *np)
{
- return fpga_region_class_find(NULL, np, fpga_region_of_node_match);
+ return fpga_region_class_find(NULL, np, device_match_of_node);
}
/**
diff --git a/drivers/fsi/cf-fsi-fw.h b/drivers/fsi/cf-fsi-fw.h
index 712df0461911..1118eaf7ee39 100644
--- a/drivers/fsi/cf-fsi-fw.h
+++ b/drivers/fsi/cf-fsi-fw.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
#ifndef __CF_FSI_FW_H
#define __CF_FSI_FW_H
diff --git a/drivers/fsi/fsi-core.c b/drivers/fsi/fsi-core.c
index 1d83f3ba478b..1f76740f33b6 100644
--- a/drivers/fsi/fsi-core.c
+++ b/drivers/fsi/fsi-core.c
@@ -1029,6 +1029,14 @@ static int fsi_slave_init(struct fsi_master *master, int link, uint8_t id)
}
+ rc = fsi_slave_set_smode(slave);
+ if (rc) {
+ dev_warn(&master->dev,
+ "can't set smode on slave:%02x:%02x %d\n",
+ link, id, rc);
+ goto err_free;
+ }
+
/* Allocate a minor in the FSI space */
rc = __fsi_get_new_minor(slave, fsi_dev_cfam, &slave->dev.devt,
&slave->cdev_idx);
@@ -1040,17 +1048,14 @@ static int fsi_slave_init(struct fsi_master *master, int link, uint8_t id)
rc = cdev_device_add(&slave->cdev, &slave->dev);
if (rc) {
dev_err(&slave->dev, "Error %d creating slave device\n", rc);
- goto err_free;
+ goto err_free_ida;
}
- rc = fsi_slave_set_smode(slave);
- if (rc) {
- dev_warn(&master->dev,
- "can't set smode on slave:%02x:%02x %d\n",
- link, id, rc);
- kfree(slave);
- return -ENODEV;
- }
+ /* Now that we have the cdev registered with the core, any fatal
+ * failures beyond this point will need to clean up through
+ * cdev_device_del(). Fortunately though, nothing past here is fatal.
+ */
+
if (master->link_config)
master->link_config(master, link,
slave->t_send_delay,
@@ -1067,10 +1072,13 @@ static int fsi_slave_init(struct fsi_master *master, int link, uint8_t id)
dev_dbg(&master->dev, "failed during slave scan with: %d\n",
rc);
- return rc;
+ return 0;
- err_free:
- put_device(&slave->dev);
+err_free_ida:
+ fsi_free_minor(slave->dev.devt);
+err_free:
+ of_node_put(slave->dev.of_node);
+ kfree(slave);
return rc;
}
diff --git a/drivers/fsi/fsi-occ.c b/drivers/fsi/fsi-occ.c
index a2301cea1cbb..7da9c81759ac 100644
--- a/drivers/fsi/fsi-occ.c
+++ b/drivers/fsi/fsi-occ.c
@@ -412,6 +412,7 @@ int fsi_occ_submit(struct device *dev, const void *request, size_t req_len,
msecs_to_jiffies(OCC_CMD_IN_PRG_WAIT_MS);
struct occ *occ = dev_get_drvdata(dev);
struct occ_response *resp = response;
+ u8 seq_no;
u16 resp_data_length;
unsigned long start;
int rc;
@@ -426,6 +427,8 @@ int fsi_occ_submit(struct device *dev, const void *request, size_t req_len,
mutex_lock(&occ->occ_lock);
+ /* Extract the seq_no from the command (first byte) */
+ seq_no = *(const u8 *)request;
rc = occ_putsram(occ, OCC_SRAM_CMD_ADDR, request, req_len);
if (rc)
goto done;
@@ -441,11 +444,17 @@ int fsi_occ_submit(struct device *dev, const void *request, size_t req_len,
if (rc)
goto done;
- if (resp->return_status == OCC_RESP_CMD_IN_PRG) {
+ if (resp->return_status == OCC_RESP_CMD_IN_PRG ||
+ resp->seq_no != seq_no) {
rc = -ETIMEDOUT;
- if (time_after(jiffies, start + timeout))
- break;
+ if (time_after(jiffies, start + timeout)) {
+ dev_err(occ->dev, "resp timeout status=%02x "
+ "resp seq_no=%d our seq_no=%d\n",
+ resp->return_status, resp->seq_no,
+ seq_no);
+ goto done;
+ }
set_current_state(TASK_UNINTERRUPTIBLE);
schedule_timeout(wait_time);
diff --git a/drivers/fsi/fsi-sbefifo.c b/drivers/fsi/fsi-sbefifo.c
index d92f5b87c251..f54df9ebc8b3 100644
--- a/drivers/fsi/fsi-sbefifo.c
+++ b/drivers/fsi/fsi-sbefifo.c
@@ -289,11 +289,11 @@ static int sbefifo_check_sbe_state(struct sbefifo *sbefifo)
switch ((sbm & CFAM_SBM_SBE_STATE_MASK) >> CFAM_SBM_SBE_STATE_SHIFT) {
case SBE_STATE_UNKNOWN:
return -ESHUTDOWN;
+ case SBE_STATE_DMT:
+ return -EBUSY;
case SBE_STATE_IPLING:
case SBE_STATE_ISTEP:
case SBE_STATE_MPIPL:
- case SBE_STATE_DMT:
- return -EBUSY;
case SBE_STATE_RUNTIME:
case SBE_STATE_DUMP: /* Not sure about that one */
break;
diff --git a/drivers/fsi/fsi-scom.c b/drivers/fsi/fsi-scom.c
index 343153d47e5b..004dc03ccf09 100644
--- a/drivers/fsi/fsi-scom.c
+++ b/drivers/fsi/fsi-scom.c
@@ -38,8 +38,7 @@
#define SCOM_STATUS_PIB_RESP_MASK 0x00007000
#define SCOM_STATUS_PIB_RESP_SHIFT 12
-#define SCOM_STATUS_ANY_ERR (SCOM_STATUS_ERR_SUMMARY | \
- SCOM_STATUS_PROTECTION | \
+#define SCOM_STATUS_ANY_ERR (SCOM_STATUS_PROTECTION | \
SCOM_STATUS_PARITY | \
SCOM_STATUS_PIB_ABORT | \
SCOM_STATUS_PIB_RESP_MASK)
@@ -251,11 +250,6 @@ static int handle_fsi2pib_status(struct scom_device *scom, uint32_t status)
/* Return -EBUSY on PIB abort to force a retry */
if (status & SCOM_STATUS_PIB_ABORT)
return -EBUSY;
- if (status & SCOM_STATUS_ERR_SUMMARY) {
- fsi_device_write(scom->fsi_dev, SCOM_FSI2PIB_RESET_REG, &dummy,
- sizeof(uint32_t));
- return -EIO;
- }
return 0;
}
diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
index acd40eb51c46..38e096e6925f 100644
--- a/drivers/gpio/Kconfig
+++ b/drivers/gpio/Kconfig
@@ -62,16 +62,12 @@ config GPIO_SYSFS
bool "/sys/class/gpio/... (sysfs interface)"
depends on SYSFS
help
- Say Y here to add a sysfs interface for GPIOs.
+ Say Y here to add the legacy sysfs interface for GPIOs.
- This is mostly useful to work around omissions in a system's
- kernel support. Those are common in custom and semicustom
- hardware assembled using standard kernels with a minimum of
- custom patches. In those cases, userspace code may import
- a given GPIO from the kernel, if no kernel driver requested it.
-
- Kernel drivers may also request that a particular GPIO be
- exported to userspace; this can be useful when debugging.
+ This ABI is deprecated. If you want to use GPIO from userspace,
+ use the character device /dev/gpiochipN with the appropriate
+ ioctl() operations instead. The character device is always
+ available.
config GPIO_GENERIC
depends on HAS_IOMEM # Only for IOMEM drivers
@@ -178,7 +174,7 @@ config GPIO_CLPS711X
config GPIO_DAVINCI
bool "TI Davinci/Keystone GPIO support"
default y if ARCH_DAVINCI
- depends on ARM && (ARCH_DAVINCI || ARCH_KEYSTONE)
+ depends on (ARM || ARM64) && (ARCH_DAVINCI || ARCH_KEYSTONE || ARCH_K3)
help
Say yes here to enable GPIO support for TI Davinci/Keystone SoCs.
@@ -279,7 +275,7 @@ config GPIO_ICH
config GPIO_IOP
tristate "Intel IOP GPIO"
- depends on ARCH_IOP32X || ARCH_IOP33X || COMPILE_TEST
+ depends on ARCH_IOP32X || COMPILE_TEST
select GPIO_GENERIC
help
Say yes here to support the GPIO functionality of a number of Intel
@@ -292,7 +288,7 @@ config GPIO_IXP4XX
depends on ARM # For <asm/mach-types.h>
depends on ARCH_IXP4XX
select GPIO_GENERIC
- select IRQ_DOMAIN
+ select GPIOLIB_IRQCHIP
select IRQ_DOMAIN_HIERARCHY
help
Say yes here to support the GPIO functionality of a number of Intel
@@ -315,6 +311,13 @@ config GPIO_LPC18XX
Select this option to enable GPIO driver for
NXP LPC18XX/43XX devices.
+config GPIO_LPC32XX
+ tristate "NXP LPC32XX GPIO support"
+ depends on OF_GPIO && (ARCH_LPC32XX || COMPILE_TEST)
+ help
+ Select this option to enable GPIO driver for
+ NXP LPC32XX devices.
+
config GPIO_LYNXPOINT
tristate "Intel Lynxpoint GPIO support"
depends on ACPI && X86
@@ -493,7 +496,8 @@ config GPIO_STA2X11
config GPIO_STP_XWAY
bool "XWAY STP GPIOs"
- depends on SOC_XWAY
+ depends on SOC_XWAY || COMPILE_TEST
+ depends on OF_GPIO
help
This enables support for the Serial To Parallel (STP) unit found on
XWAY SoC. The STP allows the SoC to drive a shift registers cascade,
@@ -542,6 +546,7 @@ config GPIO_THUNDERX
tristate "Cavium ThunderX/OCTEON-TX GPIO"
depends on ARCH_THUNDER || (64BIT && COMPILE_TEST)
depends on PCI_MSI
+ select GPIOLIB_IRQCHIP
select IRQ_DOMAIN_HIERARCHY
select IRQ_FASTEOI_HIERARCHY_HANDLERS
help
@@ -602,7 +607,6 @@ config GPIO_XGENE_SB
config GPIO_XILINX
tristate "Xilinx GPIO support"
- depends on OF_GPIO
help
Say yes here to support the Xilinx FPGA GPIO device
@@ -979,6 +983,17 @@ config GPIO_ARIZONA
help
Support for GPIOs on Wolfson Arizona class devices.
+config GPIO_BD70528
+ tristate "ROHM BD70528 GPIO support"
+ depends on MFD_ROHM_BD70528
+ help
+ Support for GPIOs on ROHM BD70528 PMIC. There are four GPIOs
+ available on the ROHM PMIC in total. The GPIOs can also
+ generate interrupts.
+
+ This driver can also be built as a module. If so, the module
+ will be called gpio-bd70528.
+
config GPIO_BD9571MWV
tristate "ROHM BD9571 GPIO support"
depends on MFD_BD9571MWV
@@ -1305,7 +1320,7 @@ config GPIO_BT8XX
The card needs to be physically altered for using it as a
GPIO card. For more information on how to build a GPIO card
from a BT8xx TV card, see the documentation file at
- Documentation/bt8xxgpio.txt
+ Documentation/driver-api/bt8xxgpio.rst
If unsure, say N.
@@ -1438,6 +1453,15 @@ config GPIO_XRA1403
help
GPIO driver for EXAR XRA1403 16-bit SPI-based GPIO expander.
+config GPIO_MOXTET
+ tristate "Turris Mox Moxtet bus GPIO expander"
+ depends on MOXTET
+ help
+ Say yes here if you are building for the Turris Mox router.
+ This is the driver needed for configuring the GPIOs via the Moxtet
+ bus. For example the Mox module with SFP cage needs this driver
+ so that phylink can use corresponding GPIOs.
+
endmenu
menu "USB GPIO expanders"
@@ -1458,7 +1482,6 @@ endmenu
config GPIO_MOCKUP
tristate "GPIO Testing Driver"
- depends on GPIOLIB
select IRQ_SIM
help
This enables GPIO Testing driver, which provides a way to test GPIO
diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile
index 6700eee860b7..d2fd19c15bae 100644
--- a/drivers/gpio/Makefile
+++ b/drivers/gpio/Makefile
@@ -17,154 +17,155 @@ obj-$(CONFIG_GPIO_GENERIC) += gpio-generic.o
# directly supported by gpio-generic
gpio-generic-$(CONFIG_GPIO_GENERIC) += gpio-mmio.o
-obj-$(CONFIG_GPIO_104_DIO_48E) += gpio-104-dio-48e.o
-obj-$(CONFIG_GPIO_104_IDIO_16) += gpio-104-idio-16.o
-obj-$(CONFIG_GPIO_104_IDI_48) += gpio-104-idi-48.o
-obj-$(CONFIG_GPIO_74X164) += gpio-74x164.o
-obj-$(CONFIG_GPIO_74XX_MMIO) += gpio-74xx-mmio.o
-obj-$(CONFIG_GPIO_ADNP) += gpio-adnp.o
-obj-$(CONFIG_GPIO_ADP5520) += gpio-adp5520.o
-obj-$(CONFIG_GPIO_ADP5588) += gpio-adp5588.o
-obj-$(CONFIG_GPIO_ALTERA) += gpio-altera.o
-obj-$(CONFIG_GPIO_ALTERA_A10SR) += gpio-altera-a10sr.o
-obj-$(CONFIG_GPIO_AMD_FCH) += gpio-amd-fch.o
-obj-$(CONFIG_GPIO_AMD8111) += gpio-amd8111.o
-obj-$(CONFIG_GPIO_AMDPT) += gpio-amdpt.o
-obj-$(CONFIG_GPIO_ARIZONA) += gpio-arizona.o
-obj-$(CONFIG_GPIO_ATH79) += gpio-ath79.o
-obj-$(CONFIG_GPIO_ASPEED) += gpio-aspeed.o
-obj-$(CONFIG_GPIO_RASPBERRYPI_EXP) += gpio-raspberrypi-exp.o
-obj-$(CONFIG_GPIO_BCM_KONA) += gpio-bcm-kona.o
-obj-$(CONFIG_GPIO_BD9571MWV) += gpio-bd9571mwv.o
-obj-$(CONFIG_GPIO_BRCMSTB) += gpio-brcmstb.o
-obj-$(CONFIG_GPIO_BT8XX) += gpio-bt8xx.o
-obj-$(CONFIG_GPIO_CADENCE) += gpio-cadence.o
-obj-$(CONFIG_GPIO_CLPS711X) += gpio-clps711x.o
-obj-$(CONFIG_GPIO_CS5535) += gpio-cs5535.o
-obj-$(CONFIG_GPIO_CRYSTAL_COVE) += gpio-crystalcove.o
-obj-$(CONFIG_GPIO_DA9052) += gpio-da9052.o
-obj-$(CONFIG_GPIO_DA9055) += gpio-da9055.o
-obj-$(CONFIG_GPIO_DAVINCI) += gpio-davinci.o
-obj-$(CONFIG_GPIO_DLN2) += gpio-dln2.o
-obj-$(CONFIG_GPIO_DWAPB) += gpio-dwapb.o
-obj-$(CONFIG_GPIO_EIC_SPRD) += gpio-eic-sprd.o
-obj-$(CONFIG_GPIO_EM) += gpio-em.o
-obj-$(CONFIG_GPIO_EP93XX) += gpio-ep93xx.o
-obj-$(CONFIG_GPIO_EXAR) += gpio-exar.o
-obj-$(CONFIG_GPIO_F7188X) += gpio-f7188x.o
-obj-$(CONFIG_GPIO_FTGPIO010) += gpio-ftgpio010.o
-obj-$(CONFIG_GPIO_GE_FPGA) += gpio-ge.o
-obj-$(CONFIG_GPIO_GPIO_MM) += gpio-gpio-mm.o
-obj-$(CONFIG_GPIO_GRGPIO) += gpio-grgpio.o
-obj-$(CONFIG_GPIO_GW_PLD) += gpio-gw-pld.o
-obj-$(CONFIG_GPIO_HLWD) += gpio-hlwd.o
-obj-$(CONFIG_HTC_EGPIO) += gpio-htc-egpio.o
-obj-$(CONFIG_GPIO_ICH) += gpio-ich.o
-obj-$(CONFIG_GPIO_IOP) += gpio-iop.o
-obj-$(CONFIG_GPIO_IXP4XX) += gpio-ixp4xx.o
-obj-$(CONFIG_GPIO_IT87) += gpio-it87.o
-obj-$(CONFIG_GPIO_JANZ_TTL) += gpio-janz-ttl.o
-obj-$(CONFIG_GPIO_KEMPLD) += gpio-kempld.o
-obj-$(CONFIG_ARCH_KS8695) += gpio-ks8695.o
-obj-$(CONFIG_GPIO_INTEL_MID) += gpio-intel-mid.o
-obj-$(CONFIG_GPIO_LOONGSON) += gpio-loongson.o
-obj-$(CONFIG_GPIO_LP3943) += gpio-lp3943.o
-obj-$(CONFIG_GPIO_LPC18XX) += gpio-lpc18xx.o
-obj-$(CONFIG_ARCH_LPC32XX) += gpio-lpc32xx.o
-obj-$(CONFIG_GPIO_LP873X) += gpio-lp873x.o
-obj-$(CONFIG_GPIO_LP87565) += gpio-lp87565.o
-obj-$(CONFIG_GPIO_LYNXPOINT) += gpio-lynxpoint.o
-obj-$(CONFIG_GPIO_MADERA) += gpio-madera.o
-obj-$(CONFIG_GPIO_MAX3191X) += gpio-max3191x.o
-obj-$(CONFIG_GPIO_MAX730X) += gpio-max730x.o
-obj-$(CONFIG_GPIO_MAX7300) += gpio-max7300.o
-obj-$(CONFIG_GPIO_MAX7301) += gpio-max7301.o
-obj-$(CONFIG_GPIO_MAX732X) += gpio-max732x.o
-obj-$(CONFIG_GPIO_MAX77620) += gpio-max77620.o
-obj-$(CONFIG_GPIO_MAX77650) += gpio-max77650.o
-obj-$(CONFIG_GPIO_MB86S7X) += gpio-mb86s7x.o
-obj-$(CONFIG_GPIO_MENZ127) += gpio-menz127.o
-obj-$(CONFIG_GPIO_MERRIFIELD) += gpio-merrifield.o
-obj-$(CONFIG_GPIO_MC33880) += gpio-mc33880.o
-obj-$(CONFIG_GPIO_MC9S08DZ60) += gpio-mc9s08dz60.o
-obj-$(CONFIG_GPIO_MLXBF) += gpio-mlxbf.o
-obj-$(CONFIG_GPIO_ML_IOH) += gpio-ml-ioh.o
-obj-$(CONFIG_GPIO_MM_LANTIQ) += gpio-mm-lantiq.o
-obj-$(CONFIG_GPIO_MOCKUP) += gpio-mockup.o
-obj-$(CONFIG_GPIO_MPC5200) += gpio-mpc5200.o
-obj-$(CONFIG_GPIO_MPC8XXX) += gpio-mpc8xxx.o
-obj-$(CONFIG_GPIO_MSIC) += gpio-msic.o
+obj-$(CONFIG_GPIO_104_DIO_48E) += gpio-104-dio-48e.o
+obj-$(CONFIG_GPIO_104_IDI_48) += gpio-104-idi-48.o
+obj-$(CONFIG_GPIO_104_IDIO_16) += gpio-104-idio-16.o
+obj-$(CONFIG_GPIO_74X164) += gpio-74x164.o
+obj-$(CONFIG_GPIO_74XX_MMIO) += gpio-74xx-mmio.o
+obj-$(CONFIG_GPIO_ADNP) += gpio-adnp.o
+obj-$(CONFIG_GPIO_ADP5520) += gpio-adp5520.o
+obj-$(CONFIG_GPIO_ADP5588) += gpio-adp5588.o
+obj-$(CONFIG_GPIO_ALTERA_A10SR) += gpio-altera-a10sr.o
+obj-$(CONFIG_GPIO_ALTERA) += gpio-altera.o
+obj-$(CONFIG_GPIO_AMD8111) += gpio-amd8111.o
+obj-$(CONFIG_GPIO_AMD_FCH) += gpio-amd-fch.o
+obj-$(CONFIG_GPIO_AMDPT) += gpio-amdpt.o
+obj-$(CONFIG_GPIO_ARIZONA) += gpio-arizona.o
+obj-$(CONFIG_GPIO_ASPEED) += gpio-aspeed.o
+obj-$(CONFIG_GPIO_ATH79) += gpio-ath79.o
+obj-$(CONFIG_GPIO_BCM_KONA) += gpio-bcm-kona.o
+obj-$(CONFIG_GPIO_BD70528) += gpio-bd70528.o
+obj-$(CONFIG_GPIO_BD9571MWV) += gpio-bd9571mwv.o
+obj-$(CONFIG_GPIO_BRCMSTB) += gpio-brcmstb.o
+obj-$(CONFIG_GPIO_BT8XX) += gpio-bt8xx.o
+obj-$(CONFIG_GPIO_CADENCE) += gpio-cadence.o
+obj-$(CONFIG_GPIO_CLPS711X) += gpio-clps711x.o
+obj-$(CONFIG_GPIO_SNPS_CREG) += gpio-creg-snps.o
+obj-$(CONFIG_GPIO_CRYSTAL_COVE) += gpio-crystalcove.o
+obj-$(CONFIG_GPIO_CS5535) += gpio-cs5535.o
+obj-$(CONFIG_GPIO_DA9052) += gpio-da9052.o
+obj-$(CONFIG_GPIO_DA9055) += gpio-da9055.o
+obj-$(CONFIG_GPIO_DAVINCI) += gpio-davinci.o
+obj-$(CONFIG_GPIO_DLN2) += gpio-dln2.o
+obj-$(CONFIG_GPIO_DWAPB) += gpio-dwapb.o
+obj-$(CONFIG_GPIO_EIC_SPRD) += gpio-eic-sprd.o
+obj-$(CONFIG_GPIO_EM) += gpio-em.o
+obj-$(CONFIG_GPIO_EP93XX) += gpio-ep93xx.o
+obj-$(CONFIG_GPIO_EXAR) += gpio-exar.o
+obj-$(CONFIG_GPIO_F7188X) += gpio-f7188x.o
+obj-$(CONFIG_GPIO_FTGPIO010) += gpio-ftgpio010.o
+obj-$(CONFIG_GPIO_GE_FPGA) += gpio-ge.o
+obj-$(CONFIG_GPIO_GPIO_MM) += gpio-gpio-mm.o
+obj-$(CONFIG_GPIO_GRGPIO) += gpio-grgpio.o
+obj-$(CONFIG_GPIO_GW_PLD) += gpio-gw-pld.o
+obj-$(CONFIG_GPIO_HLWD) += gpio-hlwd.o
+obj-$(CONFIG_HTC_EGPIO) += gpio-htc-egpio.o
+obj-$(CONFIG_GPIO_ICH) += gpio-ich.o
+obj-$(CONFIG_GPIO_INTEL_MID) += gpio-intel-mid.o
+obj-$(CONFIG_GPIO_IOP) += gpio-iop.o
+obj-$(CONFIG_GPIO_IT87) += gpio-it87.o
+obj-$(CONFIG_GPIO_IXP4XX) += gpio-ixp4xx.o
+obj-$(CONFIG_GPIO_JANZ_TTL) += gpio-janz-ttl.o
+obj-$(CONFIG_GPIO_KEMPLD) += gpio-kempld.o
+obj-$(CONFIG_GPIO_LOONGSON1) += gpio-loongson1.o
+obj-$(CONFIG_GPIO_LOONGSON) += gpio-loongson.o
+obj-$(CONFIG_GPIO_LP3943) += gpio-lp3943.o
+obj-$(CONFIG_GPIO_LP873X) += gpio-lp873x.o
+obj-$(CONFIG_GPIO_LP87565) += gpio-lp87565.o
+obj-$(CONFIG_GPIO_LPC18XX) += gpio-lpc18xx.o
+obj-$(CONFIG_GPIO_LPC32XX) += gpio-lpc32xx.o
+obj-$(CONFIG_GPIO_LYNXPOINT) += gpio-lynxpoint.o
+obj-$(CONFIG_GPIO_MADERA) += gpio-madera.o
+obj-$(CONFIG_GPIO_MAX3191X) += gpio-max3191x.o
+obj-$(CONFIG_GPIO_MAX7300) += gpio-max7300.o
+obj-$(CONFIG_GPIO_MAX7301) += gpio-max7301.o
+obj-$(CONFIG_GPIO_MAX730X) += gpio-max730x.o
+obj-$(CONFIG_GPIO_MAX732X) += gpio-max732x.o
+obj-$(CONFIG_GPIO_MAX77620) += gpio-max77620.o
+obj-$(CONFIG_GPIO_MAX77650) += gpio-max77650.o
+obj-$(CONFIG_GPIO_MB86S7X) += gpio-mb86s7x.o
+obj-$(CONFIG_GPIO_MC33880) += gpio-mc33880.o
+obj-$(CONFIG_GPIO_MC9S08DZ60) += gpio-mc9s08dz60.o
+obj-$(CONFIG_GPIO_MENZ127) += gpio-menz127.o
+obj-$(CONFIG_GPIO_MERRIFIELD) += gpio-merrifield.o
+obj-$(CONFIG_GPIO_ML_IOH) += gpio-ml-ioh.o
+obj-$(CONFIG_GPIO_MLXBF) += gpio-mlxbf.o
+obj-$(CONFIG_GPIO_MM_LANTIQ) += gpio-mm-lantiq.o
+obj-$(CONFIG_GPIO_MOCKUP) += gpio-mockup.o
+obj-$(CONFIG_GPIO_MOXTET) += gpio-moxtet.o
+obj-$(CONFIG_GPIO_MPC5200) += gpio-mpc5200.o
+obj-$(CONFIG_GPIO_MPC8XXX) += gpio-mpc8xxx.o
+obj-$(CONFIG_GPIO_MSIC) += gpio-msic.o
obj-$(CONFIG_GPIO_MT7621) += gpio-mt7621.o
-obj-$(CONFIG_GPIO_MVEBU) += gpio-mvebu.o
-obj-$(CONFIG_GPIO_MXC) += gpio-mxc.o
-obj-$(CONFIG_GPIO_MXS) += gpio-mxs.o
-obj-$(CONFIG_GPIO_OCTEON) += gpio-octeon.o
-obj-$(CONFIG_GPIO_OMAP) += gpio-omap.o
-obj-$(CONFIG_GPIO_PCA953X) += gpio-pca953x.o
-obj-$(CONFIG_GPIO_PCF857X) += gpio-pcf857x.o
-obj-$(CONFIG_GPIO_PCH) += gpio-pch.o
-obj-$(CONFIG_GPIO_PCI_IDIO_16) += gpio-pci-idio-16.o
-obj-$(CONFIG_GPIO_PCIE_IDIO_24) += gpio-pcie-idio-24.o
-obj-$(CONFIG_GPIO_PISOSR) += gpio-pisosr.o
-obj-$(CONFIG_GPIO_PL061) += gpio-pl061.o
+obj-$(CONFIG_GPIO_MVEBU) += gpio-mvebu.o
+obj-$(CONFIG_GPIO_MXC) += gpio-mxc.o
+obj-$(CONFIG_GPIO_MXS) += gpio-mxs.o
+obj-$(CONFIG_GPIO_OCTEON) += gpio-octeon.o
+obj-$(CONFIG_GPIO_OMAP) += gpio-omap.o
+obj-$(CONFIG_GPIO_PALMAS) += gpio-palmas.o
+obj-$(CONFIG_GPIO_PCA953X) += gpio-pca953x.o
+obj-$(CONFIG_GPIO_PCF857X) += gpio-pcf857x.o
+obj-$(CONFIG_GPIO_PCH) += gpio-pch.o
+obj-$(CONFIG_GPIO_PCIE_IDIO_24) += gpio-pcie-idio-24.o
+obj-$(CONFIG_GPIO_PCI_IDIO_16) += gpio-pci-idio-16.o
+obj-$(CONFIG_GPIO_PISOSR) += gpio-pisosr.o
+obj-$(CONFIG_GPIO_PL061) += gpio-pl061.o
obj-$(CONFIG_GPIO_PMIC_EIC_SPRD) += gpio-pmic-eic-sprd.o
-obj-$(CONFIG_GPIO_PXA) += gpio-pxa.o
-obj-$(CONFIG_GPIO_RC5T583) += gpio-rc5t583.o
-obj-$(CONFIG_GPIO_RDC321X) += gpio-rdc321x.o
-obj-$(CONFIG_GPIO_RCAR) += gpio-rcar.o
-obj-$(CONFIG_GPIO_REG) += gpio-reg.o
-obj-$(CONFIG_ARCH_SA1100) += gpio-sa1100.o
+obj-$(CONFIG_GPIO_PXA) += gpio-pxa.o
+obj-$(CONFIG_GPIO_RASPBERRYPI_EXP) += gpio-raspberrypi-exp.o
+obj-$(CONFIG_GPIO_RC5T583) += gpio-rc5t583.o
+obj-$(CONFIG_GPIO_RCAR) += gpio-rcar.o
+obj-$(CONFIG_GPIO_RDC321X) += gpio-rdc321x.o
+obj-$(CONFIG_GPIO_REG) += gpio-reg.o
+obj-$(CONFIG_ARCH_SA1100) += gpio-sa1100.o
obj-$(CONFIG_GPIO_SAMA5D2_PIOBU) += gpio-sama5d2-piobu.o
-obj-$(CONFIG_GPIO_SCH) += gpio-sch.o
-obj-$(CONFIG_GPIO_SCH311X) += gpio-sch311x.o
-obj-$(CONFIG_GPIO_SNPS_CREG) += gpio-creg-snps.o
-obj-$(CONFIG_GPIO_SODAVILLE) += gpio-sodaville.o
-obj-$(CONFIG_GPIO_SPEAR_SPICS) += gpio-spear-spics.o
-obj-$(CONFIG_GPIO_SPRD) += gpio-sprd.o
-obj-$(CONFIG_GPIO_STA2X11) += gpio-sta2x11.o
-obj-$(CONFIG_GPIO_STMPE) += gpio-stmpe.o
-obj-$(CONFIG_GPIO_STP_XWAY) += gpio-stp-xway.o
-obj-$(CONFIG_GPIO_SYSCON) += gpio-syscon.o
-obj-$(CONFIG_GPIO_TB10X) += gpio-tb10x.o
-obj-$(CONFIG_GPIO_TC3589X) += gpio-tc3589x.o
-obj-$(CONFIG_GPIO_TEGRA) += gpio-tegra.o
-obj-$(CONFIG_GPIO_TEGRA186) += gpio-tegra186.o
-obj-$(CONFIG_GPIO_THUNDERX) += gpio-thunderx.o
-obj-$(CONFIG_GPIO_TIMBERDALE) += gpio-timberdale.o
-obj-$(CONFIG_GPIO_PALMAS) += gpio-palmas.o
-obj-$(CONFIG_GPIO_SIOX) += gpio-siox.o
-obj-$(CONFIG_GPIO_TPIC2810) += gpio-tpic2810.o
-obj-$(CONFIG_GPIO_TPS65086) += gpio-tps65086.o
-obj-$(CONFIG_GPIO_TPS65218) += gpio-tps65218.o
-obj-$(CONFIG_GPIO_TPS6586X) += gpio-tps6586x.o
-obj-$(CONFIG_GPIO_TPS65910) += gpio-tps65910.o
-obj-$(CONFIG_GPIO_TPS65912) += gpio-tps65912.o
-obj-$(CONFIG_GPIO_TPS68470) += gpio-tps68470.o
-obj-$(CONFIG_GPIO_TQMX86) += gpio-tqmx86.o
-obj-$(CONFIG_GPIO_TS4800) += gpio-ts4800.o
-obj-$(CONFIG_GPIO_TS4900) += gpio-ts4900.o
-obj-$(CONFIG_GPIO_TS5500) += gpio-ts5500.o
-obj-$(CONFIG_GPIO_TWL4030) += gpio-twl4030.o
-obj-$(CONFIG_GPIO_TWL6040) += gpio-twl6040.o
-obj-$(CONFIG_GPIO_UCB1400) += gpio-ucb1400.o
-obj-$(CONFIG_GPIO_UNIPHIER) += gpio-uniphier.o
-obj-$(CONFIG_GPIO_VF610) += gpio-vf610.o
-obj-$(CONFIG_GPIO_VIPERBOARD) += gpio-viperboard.o
-obj-$(CONFIG_GPIO_VR41XX) += gpio-vr41xx.o
-obj-$(CONFIG_GPIO_VX855) += gpio-vx855.o
-obj-$(CONFIG_GPIO_WHISKEY_COVE) += gpio-wcove.o
-obj-$(CONFIG_GPIO_WINBOND) += gpio-winbond.o
-obj-$(CONFIG_GPIO_WM831X) += gpio-wm831x.o
-obj-$(CONFIG_GPIO_WM8350) += gpio-wm8350.o
-obj-$(CONFIG_GPIO_WM8994) += gpio-wm8994.o
-obj-$(CONFIG_GPIO_WS16C48) += gpio-ws16c48.o
-obj-$(CONFIG_GPIO_XGENE) += gpio-xgene.o
-obj-$(CONFIG_GPIO_XGENE_SB) += gpio-xgene-sb.o
-obj-$(CONFIG_GPIO_XILINX) += gpio-xilinx.o
-obj-$(CONFIG_GPIO_XLP) += gpio-xlp.o
-obj-$(CONFIG_GPIO_XRA1403) += gpio-xra1403.o
-obj-$(CONFIG_GPIO_XTENSA) += gpio-xtensa.o
-obj-$(CONFIG_GPIO_ZEVIO) += gpio-zevio.o
-obj-$(CONFIG_GPIO_ZYNQ) += gpio-zynq.o
-obj-$(CONFIG_GPIO_ZX) += gpio-zx.o
-obj-$(CONFIG_GPIO_LOONGSON1) += gpio-loongson1.o
+obj-$(CONFIG_GPIO_SCH311X) += gpio-sch311x.o
+obj-$(CONFIG_GPIO_SCH) += gpio-sch.o
+obj-$(CONFIG_GPIO_SIOX) += gpio-siox.o
+obj-$(CONFIG_GPIO_SODAVILLE) += gpio-sodaville.o
+obj-$(CONFIG_GPIO_SPEAR_SPICS) += gpio-spear-spics.o
+obj-$(CONFIG_GPIO_SPRD) += gpio-sprd.o
+obj-$(CONFIG_GPIO_STA2X11) += gpio-sta2x11.o
+obj-$(CONFIG_GPIO_STMPE) += gpio-stmpe.o
+obj-$(CONFIG_GPIO_STP_XWAY) += gpio-stp-xway.o
+obj-$(CONFIG_GPIO_SYSCON) += gpio-syscon.o
+obj-$(CONFIG_GPIO_TB10X) += gpio-tb10x.o
+obj-$(CONFIG_GPIO_TC3589X) += gpio-tc3589x.o
+obj-$(CONFIG_GPIO_TEGRA186) += gpio-tegra186.o
+obj-$(CONFIG_GPIO_TEGRA) += gpio-tegra.o
+obj-$(CONFIG_GPIO_THUNDERX) += gpio-thunderx.o
+obj-$(CONFIG_GPIO_TIMBERDALE) += gpio-timberdale.o
+obj-$(CONFIG_GPIO_TPIC2810) += gpio-tpic2810.o
+obj-$(CONFIG_GPIO_TPS65086) += gpio-tps65086.o
+obj-$(CONFIG_GPIO_TPS65218) += gpio-tps65218.o
+obj-$(CONFIG_GPIO_TPS6586X) += gpio-tps6586x.o
+obj-$(CONFIG_GPIO_TPS65910) += gpio-tps65910.o
+obj-$(CONFIG_GPIO_TPS65912) += gpio-tps65912.o
+obj-$(CONFIG_GPIO_TPS68470) += gpio-tps68470.o
+obj-$(CONFIG_GPIO_TQMX86) += gpio-tqmx86.o
+obj-$(CONFIG_GPIO_TS4800) += gpio-ts4800.o
+obj-$(CONFIG_GPIO_TS4900) += gpio-ts4900.o
+obj-$(CONFIG_GPIO_TS5500) += gpio-ts5500.o
+obj-$(CONFIG_GPIO_TWL4030) += gpio-twl4030.o
+obj-$(CONFIG_GPIO_TWL6040) += gpio-twl6040.o
+obj-$(CONFIG_GPIO_UCB1400) += gpio-ucb1400.o
+obj-$(CONFIG_GPIO_UNIPHIER) += gpio-uniphier.o
+obj-$(CONFIG_GPIO_VF610) += gpio-vf610.o
+obj-$(CONFIG_GPIO_VIPERBOARD) += gpio-viperboard.o
+obj-$(CONFIG_GPIO_VR41XX) += gpio-vr41xx.o
+obj-$(CONFIG_GPIO_VX855) += gpio-vx855.o
+obj-$(CONFIG_GPIO_WHISKEY_COVE) += gpio-wcove.o
+obj-$(CONFIG_GPIO_WINBOND) += gpio-winbond.o
+obj-$(CONFIG_GPIO_WM831X) += gpio-wm831x.o
+obj-$(CONFIG_GPIO_WM8350) += gpio-wm8350.o
+obj-$(CONFIG_GPIO_WM8994) += gpio-wm8994.o
+obj-$(CONFIG_GPIO_WS16C48) += gpio-ws16c48.o
+obj-$(CONFIG_GPIO_XGENE) += gpio-xgene.o
+obj-$(CONFIG_GPIO_XGENE_SB) += gpio-xgene-sb.o
+obj-$(CONFIG_GPIO_XILINX) += gpio-xilinx.o
+obj-$(CONFIG_GPIO_XLP) += gpio-xlp.o
+obj-$(CONFIG_GPIO_XRA1403) += gpio-xra1403.o
+obj-$(CONFIG_GPIO_XTENSA) += gpio-xtensa.o
+obj-$(CONFIG_GPIO_ZEVIO) += gpio-zevio.o
+obj-$(CONFIG_GPIO_ZX) += gpio-zx.o
+obj-$(CONFIG_GPIO_ZYNQ) += gpio-zynq.o
diff --git a/drivers/gpio/TODO b/drivers/gpio/TODO
index 19d27c904916..9c048f10c9ad 100644
--- a/drivers/gpio/TODO
+++ b/drivers/gpio/TODO
@@ -90,6 +90,46 @@ GPIOLIB irqchip
The GPIOLIB irqchip is a helper irqchip for "simple cases" that should
try to cover any generic kind of irqchip cascaded from a GPIO.
+- Convert all the GPIOLIB_IRQCHIP users to pass an irqchip template,
+ parent and flags before calling [devm_]gpiochip_add[_data]().
+ Currently we set up the irqchip after setting up the gpiochip
+ using gpiochip_irqchip_add() and gpiochip_set_[chained|nested]_irqchip().
+ This is too complex, so convert all users over to just set up
+ the irqchip before registering the gpio_chip, typical example:
+
+ /* Typical state container with dynamic irqchip */
+ struct my_gpio {
+ struct gpio_chip gc;
+ struct irq_chip irq;
+ };
+
+ int irq; /* from platform etc */
+ struct my_gpio *g;
+ struct gpio_irq_chip *girq
+
+ /* Set up the irqchip dynamically */
+ g->irq.name = "my_gpio_irq";
+ g->irq.irq_ack = my_gpio_ack_irq;
+ g->irq.irq_mask = my_gpio_mask_irq;
+ g->irq.irq_unmask = my_gpio_unmask_irq;
+ g->irq.irq_set_type = my_gpio_set_irq_type;
+
+ /* Get a pointer to the gpio_irq_chip */
+ girq = &g->gc.irq;
+ girq->chip = &g->irq;
+ girq->parent_handler = ftgpio_gpio_irq_handler;
+ girq->num_parents = 1;
+ girq->parents = devm_kcalloc(dev, 1, sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents)
+ return -ENOMEM;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_bad_irq;
+ girq->parents[0] = irq;
+
+ When this is done, we will delete the old APIs for instatiating
+ GPIOLIB_IRQCHIP and simplify the code.
+
- Look over and identify any remaining easily converted drivers and
dry-code conversions to gpiolib irqchip for maintainers to test
diff --git a/drivers/gpio/gpio-altera.c b/drivers/gpio/gpio-altera.c
index e088b908c2c1..9f2e6b04c361 100644
--- a/drivers/gpio/gpio-altera.c
+++ b/drivers/gpio/gpio-altera.c
@@ -30,6 +30,7 @@ struct altera_gpio_chip {
raw_spinlock_t gpio_lock;
int interrupt_trigger;
int mapped_irq;
+ struct irq_chip irq_chip;
};
static void altera_gpio_irq_unmask(struct irq_data *d)
@@ -101,15 +102,6 @@ static unsigned int altera_gpio_irq_startup(struct irq_data *d)
return 0;
}
-static struct irq_chip altera_irq_chip = {
- .name = "altera-gpio",
- .irq_mask = altera_gpio_irq_mask,
- .irq_unmask = altera_gpio_irq_unmask,
- .irq_set_type = altera_gpio_irq_set_type,
- .irq_startup = altera_gpio_irq_startup,
- .irq_shutdown = altera_gpio_irq_mask,
-};
-
static int altera_gpio_get(struct gpio_chip *gc, unsigned offset)
{
struct of_mm_gpio_chip *mm_gc;
@@ -246,6 +238,7 @@ static int altera_gpio_probe(struct platform_device *pdev)
struct device_node *node = pdev->dev.of_node;
int reg, ret;
struct altera_gpio_chip *altera_gc;
+ struct gpio_irq_chip *girq;
altera_gc = devm_kzalloc(&pdev->dev, sizeof(*altera_gc), GFP_KERNEL);
if (!altera_gc)
@@ -273,50 +266,50 @@ static int altera_gpio_probe(struct platform_device *pdev)
altera_gc->mmchip.gc.owner = THIS_MODULE;
altera_gc->mmchip.gc.parent = &pdev->dev;
- ret = of_mm_gpiochip_add_data(node, &altera_gc->mmchip, altera_gc);
- if (ret) {
- dev_err(&pdev->dev, "Failed adding memory mapped gpiochip\n");
- return ret;
- }
-
- platform_set_drvdata(pdev, altera_gc);
-
altera_gc->mapped_irq = platform_get_irq(pdev, 0);
if (altera_gc->mapped_irq < 0)
goto skip_irq;
if (of_property_read_u32(node, "altr,interrupt-type", &reg)) {
- ret = -EINVAL;
dev_err(&pdev->dev,
"altr,interrupt-type value not set in device tree\n");
- goto teardown;
+ return -EINVAL;
}
altera_gc->interrupt_trigger = reg;
- ret = gpiochip_irqchip_add(&altera_gc->mmchip.gc, &altera_irq_chip, 0,
- handle_bad_irq, IRQ_TYPE_NONE);
+ altera_gc->irq_chip.name = "altera-gpio";
+ altera_gc->irq_chip.irq_mask = altera_gpio_irq_mask;
+ altera_gc->irq_chip.irq_unmask = altera_gpio_irq_unmask;
+ altera_gc->irq_chip.irq_set_type = altera_gpio_irq_set_type;
+ altera_gc->irq_chip.irq_startup = altera_gpio_irq_startup;
+ altera_gc->irq_chip.irq_shutdown = altera_gpio_irq_mask;
+
+ girq = &altera_gc->mmchip.gc.irq;
+ girq->chip = &altera_gc->irq_chip;
+ if (altera_gc->interrupt_trigger == IRQ_TYPE_LEVEL_HIGH)
+ girq->parent_handler = altera_gpio_irq_leveL_high_handler;
+ else
+ girq->parent_handler = altera_gpio_irq_edge_handler;
+ girq->num_parents = 1;
+ girq->parents = devm_kcalloc(&pdev->dev, 1, sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents)
+ return -ENOMEM;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_bad_irq;
+ girq->parents[0] = altera_gc->mapped_irq;
+skip_irq:
+ ret = of_mm_gpiochip_add_data(node, &altera_gc->mmchip, altera_gc);
if (ret) {
- dev_err(&pdev->dev, "could not add irqchip\n");
- goto teardown;
+ dev_err(&pdev->dev, "Failed adding memory mapped gpiochip\n");
+ return ret;
}
- gpiochip_set_chained_irqchip(&altera_gc->mmchip.gc,
- &altera_irq_chip,
- altera_gc->mapped_irq,
- altera_gc->interrupt_trigger == IRQ_TYPE_LEVEL_HIGH ?
- altera_gpio_irq_leveL_high_handler :
- altera_gpio_irq_edge_handler);
+ platform_set_drvdata(pdev, altera_gc);
-skip_irq:
return 0;
-teardown:
- of_mm_gpiochip_remove(&altera_gc->mmchip);
- pr_err("%pOF: registration failed with status %d\n",
- node, ret);
-
- return ret;
}
static int altera_gpio_remove(struct platform_device *pdev)
diff --git a/drivers/gpio/gpio-amd-fch.c b/drivers/gpio/gpio-amd-fch.c
index 38c3f4a3d4aa..181df1581df5 100644
--- a/drivers/gpio/gpio-amd-fch.c
+++ b/drivers/gpio/gpio-amd-fch.c
@@ -25,14 +25,13 @@
#define AMD_FCH_GPIO_FLAG_WRITE BIT(22)
#define AMD_FCH_GPIO_FLAG_READ BIT(16)
-static struct resource amd_fch_gpio_iores =
+static const struct resource amd_fch_gpio_iores =
DEFINE_RES_MEM_NAMED(
AMD_FCH_MMIO_BASE + AMD_FCH_GPIO_BANK0_BASE,
AMD_FCH_GPIO_SIZE,
"amd-fch-gpio-iomem");
struct amd_fch_gpio_priv {
- struct platform_device *pdev;
struct gpio_chip gc;
void __iomem *base;
struct amd_fch_gpio_pdata *pdata;
@@ -153,7 +152,6 @@ static int amd_fch_gpio_probe(struct platform_device *pdev)
return -ENOMEM;
priv->pdata = pdata;
- priv->pdev = pdev;
priv->gc.owner = THIS_MODULE;
priv->gc.parent = &pdev->dev;
diff --git a/drivers/gpio/gpio-amdpt.c b/drivers/gpio/gpio-amdpt.c
index ad255ba7ece9..44398992ae15 100644
--- a/drivers/gpio/gpio-amdpt.c
+++ b/drivers/gpio/gpio-amdpt.c
@@ -88,7 +88,7 @@ static int pt_gpio_probe(struct platform_device *pdev)
pt_gpio->reg_base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(pt_gpio->reg_base)) {
- dev_err(&pdev->dev, "Failed to map MMIO resource for PT GPIO.\n");
+ dev_err(dev, "Failed to map MMIO resource for PT GPIO.\n");
return PTR_ERR(pt_gpio->reg_base);
}
@@ -98,7 +98,7 @@ static int pt_gpio_probe(struct platform_device *pdev)
pt_gpio->reg_base + PT_DIRECTION_REG, NULL,
BGPIOF_READ_OUTPUT_REG_SET);
if (ret) {
- dev_err(&pdev->dev, "bgpio_init failed\n");
+ dev_err(dev, "bgpio_init failed\n");
return ret;
}
@@ -107,11 +107,11 @@ static int pt_gpio_probe(struct platform_device *pdev)
pt_gpio->gc.free = pt_gpio_free;
pt_gpio->gc.ngpio = PT_TOTAL_GPIO;
#if defined(CONFIG_OF_GPIO)
- pt_gpio->gc.of_node = pdev->dev.of_node;
+ pt_gpio->gc.of_node = dev->of_node;
#endif
ret = gpiochip_add_data(&pt_gpio->gc, pt_gpio);
if (ret) {
- dev_err(&pdev->dev, "Failed to register GPIO lib\n");
+ dev_err(dev, "Failed to register GPIO lib\n");
return ret;
}
@@ -121,7 +121,7 @@ static int pt_gpio_probe(struct platform_device *pdev)
writel(0, pt_gpio->reg_base + PT_SYNC_REG);
writel(0, pt_gpio->reg_base + PT_CLOCKRATE_REG);
- dev_dbg(&pdev->dev, "PT GPIO driver loaded\n");
+ dev_dbg(dev, "PT GPIO driver loaded\n");
return ret;
}
diff --git a/drivers/gpio/gpio-arizona.c b/drivers/gpio/gpio-arizona.c
index c07fad975049..5640efe5e750 100644
--- a/drivers/gpio/gpio-arizona.c
+++ b/drivers/gpio/gpio-arizona.c
@@ -142,7 +142,7 @@ static const struct gpio_chip template_chip = {
static int arizona_gpio_probe(struct platform_device *pdev)
{
struct arizona *arizona = dev_get_drvdata(pdev->dev.parent);
- struct arizona_pdata *pdata = dev_get_platdata(arizona->dev);
+ struct arizona_pdata *pdata = &arizona->pdata;
struct arizona_gpio *arizona_gpio;
int ret;
@@ -177,7 +177,7 @@ static int arizona_gpio_probe(struct platform_device *pdev)
return -EINVAL;
}
- if (pdata && pdata->gpio_base)
+ if (pdata->gpio_base)
arizona_gpio->gpio_chip.base = pdata->gpio_base;
else
arizona_gpio->gpio_chip.base = -1;
diff --git a/drivers/gpio/gpio-aspeed.c b/drivers/gpio/gpio-aspeed.c
index 13d80bfbc3b6..09e53c5f3b0a 100644
--- a/drivers/gpio/gpio-aspeed.c
+++ b/drivers/gpio/gpio-aspeed.c
@@ -52,6 +52,7 @@ struct aspeed_gpio_config {
*/
struct aspeed_gpio {
struct gpio_chip chip;
+ struct irq_chip irqc;
spinlock_t lock;
void __iomem *base;
int irq;
@@ -661,12 +662,14 @@ static void aspeed_gpio_irq_handler(struct irq_desc *desc)
struct gpio_chip *gc = irq_desc_get_handler_data(desc);
struct irq_chip *ic = irq_desc_get_chip(desc);
struct aspeed_gpio *data = gpiochip_get_data(gc);
- unsigned int i, p, girq;
+ unsigned int i, p, girq, banks;
unsigned long reg;
+ struct aspeed_gpio *gpio = gpiochip_get_data(gc);
chained_irq_enter(ic, desc);
- for (i = 0; i < ARRAY_SIZE(aspeed_gpio_banks); i++) {
+ banks = DIV_ROUND_UP(gpio->chip.ngpio, 32);
+ for (i = 0; i < banks; i++) {
const struct aspeed_gpio_bank *bank = &aspeed_gpio_banks[i];
reg = ioread32(bank_reg(data, bank, reg_irq_status));
@@ -681,16 +684,11 @@ static void aspeed_gpio_irq_handler(struct irq_desc *desc)
chained_irq_exit(ic, desc);
}
-static struct irq_chip aspeed_gpio_irqchip = {
- .name = "aspeed-gpio",
- .irq_ack = aspeed_gpio_irq_ack,
- .irq_mask = aspeed_gpio_irq_mask,
- .irq_unmask = aspeed_gpio_irq_unmask,
- .irq_set_type = aspeed_gpio_set_type,
-};
-
-static void set_irq_valid_mask(struct aspeed_gpio *gpio)
+static void aspeed_init_irq_valid_mask(struct gpio_chip *gc,
+ unsigned long *valid_mask,
+ unsigned int ngpios)
{
+ struct aspeed_gpio *gpio = gpiochip_get_data(gc);
const struct aspeed_bank_props *props = gpio->config->props;
while (!is_bank_props_sentinel(props)) {
@@ -701,42 +699,16 @@ static void set_irq_valid_mask(struct aspeed_gpio *gpio)
for_each_clear_bit(offset, &input, 32) {
unsigned int i = props->bank * 32 + offset;
- if (i >= gpio->config->nr_gpios)
+ if (i >= gpio->chip.ngpio)
break;
- clear_bit(i, gpio->chip.irq.valid_mask);
+ clear_bit(i, valid_mask);
}
props++;
}
}
-static int aspeed_gpio_setup_irqs(struct aspeed_gpio *gpio,
- struct platform_device *pdev)
-{
- int rc;
-
- rc = platform_get_irq(pdev, 0);
- if (rc < 0)
- return rc;
-
- gpio->irq = rc;
-
- set_irq_valid_mask(gpio);
-
- rc = gpiochip_irqchip_add(&gpio->chip, &aspeed_gpio_irqchip,
- 0, handle_bad_irq, IRQ_TYPE_NONE);
- if (rc) {
- dev_info(&pdev->dev, "Could not add irqchip\n");
- return rc;
- }
-
- gpiochip_set_chained_irqchip(&gpio->chip, &aspeed_gpio_irqchip,
- gpio->irq, aspeed_gpio_irq_handler);
-
- return 0;
-}
-
static int aspeed_gpio_reset_tolerance(struct gpio_chip *chip,
unsigned int offset, bool enable)
{
@@ -1040,10 +1012,10 @@ int aspeed_gpio_copro_grab_gpio(struct gpio_desc *desc,
unsigned long flags;
if (!gpio->cf_copro_bankmap)
- gpio->cf_copro_bankmap = kzalloc(gpio->config->nr_gpios >> 3, GFP_KERNEL);
+ gpio->cf_copro_bankmap = kzalloc(gpio->chip.ngpio >> 3, GFP_KERNEL);
if (!gpio->cf_copro_bankmap)
return -ENOMEM;
- if (offset < 0 || offset > gpio->config->nr_gpios)
+ if (offset < 0 || offset > gpio->chip.ngpio)
return -EINVAL;
bindex = offset >> 3;
@@ -1088,7 +1060,7 @@ int aspeed_gpio_copro_release_gpio(struct gpio_desc *desc)
if (!gpio->cf_copro_bankmap)
return -ENXIO;
- if (offset < 0 || offset > gpio->config->nr_gpios)
+ if (offset < 0 || offset > gpio->chip.ngpio)
return -EINVAL;
bindex = offset >> 3;
@@ -1141,9 +1113,25 @@ static const struct aspeed_gpio_config ast2500_config =
/* 232 for simplicity, actual number is 228 (4-GPIO hole in GPIOAB) */
{ .nr_gpios = 232, .props = ast2500_bank_props, };
+static const struct aspeed_bank_props ast2600_bank_props[] = {
+ /* input output */
+ {5, 0xffffffff, 0x0000ffff}, /* U/V/W/X */
+ {6, 0xffff0000, 0x0fff0000}, /* Y/Z */
+ { },
+};
+
+static const struct aspeed_gpio_config ast2600_config =
+ /*
+ * ast2600 has two controllers one with 208 GPIOs and one with 36 GPIOs.
+ * We expect ngpio being set in the device tree and this is a fallback
+ * option.
+ */
+ { .nr_gpios = 208, .props = ast2600_bank_props, };
+
static const struct of_device_id aspeed_gpio_of_table[] = {
{ .compatible = "aspeed,ast2400-gpio", .data = &ast2400_config, },
{ .compatible = "aspeed,ast2500-gpio", .data = &ast2500_config, },
+ { .compatible = "aspeed,ast2600-gpio", .data = &ast2600_config, },
{}
};
MODULE_DEVICE_TABLE(of, aspeed_gpio_of_table);
@@ -1152,7 +1140,8 @@ static int __init aspeed_gpio_probe(struct platform_device *pdev)
{
const struct of_device_id *gpio_id;
struct aspeed_gpio *gpio;
- int rc, i, banks;
+ int rc, i, banks, err;
+ u32 ngpio;
gpio = devm_kzalloc(&pdev->dev, sizeof(*gpio), GFP_KERNEL);
if (!gpio)
@@ -1178,7 +1167,10 @@ static int __init aspeed_gpio_probe(struct platform_device *pdev)
gpio->config = gpio_id->data;
gpio->chip.parent = &pdev->dev;
- gpio->chip.ngpio = gpio->config->nr_gpios;
+ err = of_property_read_u32(pdev->dev.of_node, "ngpios", &ngpio);
+ gpio->chip.ngpio = (u16) ngpio;
+ if (err)
+ gpio->chip.ngpio = gpio->config->nr_gpios;
gpio->chip.direction_input = aspeed_gpio_dir_in;
gpio->chip.direction_output = aspeed_gpio_dir_out;
gpio->chip.get_direction = aspeed_gpio_get_direction;
@@ -1189,10 +1181,9 @@ static int __init aspeed_gpio_probe(struct platform_device *pdev)
gpio->chip.set_config = aspeed_gpio_set_config;
gpio->chip.label = dev_name(&pdev->dev);
gpio->chip.base = -1;
- gpio->chip.irq.need_valid_mask = true;
/* Allocate a cache of the output registers */
- banks = gpio->config->nr_gpios >> 5;
+ banks = DIV_ROUND_UP(gpio->chip.ngpio, 32);
gpio->dcache = devm_kcalloc(&pdev->dev,
banks, sizeof(u32), GFP_KERNEL);
if (!gpio->dcache)
@@ -1212,16 +1203,42 @@ static int __init aspeed_gpio_probe(struct platform_device *pdev)
aspeed_gpio_change_cmd_source(gpio, bank, 3, GPIO_CMDSRC_ARM);
}
- rc = devm_gpiochip_add_data(&pdev->dev, &gpio->chip, gpio);
- if (rc < 0)
- return rc;
+ /* Optionally set up an irqchip if there is an IRQ */
+ rc = platform_get_irq(pdev, 0);
+ if (rc > 0) {
+ struct gpio_irq_chip *girq;
+
+ gpio->irq = rc;
+ girq = &gpio->chip.irq;
+ girq->chip = &gpio->irqc;
+ girq->chip->name = dev_name(&pdev->dev);
+ girq->chip->irq_ack = aspeed_gpio_irq_ack;
+ girq->chip->irq_mask = aspeed_gpio_irq_mask;
+ girq->chip->irq_unmask = aspeed_gpio_irq_unmask;
+ girq->chip->irq_set_type = aspeed_gpio_set_type;
+ girq->parent_handler = aspeed_gpio_irq_handler;
+ girq->num_parents = 1;
+ girq->parents = devm_kcalloc(&pdev->dev, 1,
+ sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents)
+ return -ENOMEM;
+ girq->parents[0] = gpio->irq;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_bad_irq;
+ girq->init_valid_mask = aspeed_init_irq_valid_mask;
+ }
gpio->offset_timer =
devm_kzalloc(&pdev->dev, gpio->chip.ngpio, GFP_KERNEL);
if (!gpio->offset_timer)
return -ENOMEM;
- return aspeed_gpio_setup_irqs(gpio, pdev);
+ rc = devm_gpiochip_add_data(&pdev->dev, &gpio->chip, gpio);
+ if (rc < 0)
+ return rc;
+
+ return 0;
}
static struct platform_driver aspeed_gpio_driver = {
diff --git a/drivers/gpio/gpio-ath79.c b/drivers/gpio/gpio-ath79.c
index 6c6dcda1100c..f1a5ea9b3de2 100644
--- a/drivers/gpio/gpio-ath79.c
+++ b/drivers/gpio/gpio-ath79.c
@@ -222,14 +222,16 @@ MODULE_DEVICE_TABLE(of, ath79_gpio_of_match);
static int ath79_gpio_probe(struct platform_device *pdev)
{
struct ath79_gpio_platform_data *pdata = dev_get_platdata(&pdev->dev);
- struct device_node *np = pdev->dev.of_node;
+ struct device *dev = &pdev->dev;
+ struct device_node *np = dev->of_node;
struct ath79_gpio_ctrl *ctrl;
+ struct gpio_irq_chip *girq;
struct resource *res;
u32 ath79_gpio_count;
bool oe_inverted;
int err;
- ctrl = devm_kzalloc(&pdev->dev, sizeof(*ctrl), GFP_KERNEL);
+ ctrl = devm_kzalloc(dev, sizeof(*ctrl), GFP_KERNEL);
if (!ctrl)
return -ENOMEM;
platform_set_drvdata(pdev, ctrl);
@@ -237,7 +239,7 @@ static int ath79_gpio_probe(struct platform_device *pdev)
if (np) {
err = of_property_read_u32(np, "ngpios", &ath79_gpio_count);
if (err) {
- dev_err(&pdev->dev, "ngpios property is not valid\n");
+ dev_err(dev, "ngpios property is not valid\n");
return err;
}
oe_inverted = of_device_is_compatible(np, "qca,ar9340-gpio");
@@ -245,25 +247,24 @@ static int ath79_gpio_probe(struct platform_device *pdev)
ath79_gpio_count = pdata->ngpios;
oe_inverted = pdata->oe_inverted;
} else {
- dev_err(&pdev->dev, "No DT node or platform data found\n");
+ dev_err(dev, "No DT node or platform data found\n");
return -EINVAL;
}
if (ath79_gpio_count >= 32) {
- dev_err(&pdev->dev, "ngpios must be less than 32\n");
+ dev_err(dev, "ngpios must be less than 32\n");
return -EINVAL;
}
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (!res)
return -EINVAL;
- ctrl->base = devm_ioremap_nocache(
- &pdev->dev, res->start, resource_size(res));
+ ctrl->base = devm_ioremap_nocache(dev, res->start, resource_size(res));
if (!ctrl->base)
return -ENOMEM;
raw_spin_lock_init(&ctrl->lock);
- err = bgpio_init(&ctrl->gc, &pdev->dev, 4,
+ err = bgpio_init(&ctrl->gc, dev, 4,
ctrl->base + AR71XX_GPIO_REG_IN,
ctrl->base + AR71XX_GPIO_REG_SET,
ctrl->base + AR71XX_GPIO_REG_CLEAR,
@@ -271,45 +272,33 @@ static int ath79_gpio_probe(struct platform_device *pdev)
oe_inverted ? ctrl->base + AR71XX_GPIO_REG_OE : NULL,
0);
if (err) {
- dev_err(&pdev->dev, "bgpio_init failed\n");
+ dev_err(dev, "bgpio_init failed\n");
return err;
}
/* Use base 0 to stay compatible with legacy platforms */
ctrl->gc.base = 0;
- err = gpiochip_add_data(&ctrl->gc, ctrl);
- if (err) {
- dev_err(&pdev->dev,
- "cannot add AR71xx GPIO chip, error=%d", err);
- return err;
+ /* Optional interrupt setup */
+ if (!np || of_property_read_bool(np, "interrupt-controller")) {
+ girq = &ctrl->gc.irq;
+ girq->chip = &ath79_gpio_irqchip;
+ girq->parent_handler = ath79_gpio_irq_handler;
+ girq->num_parents = 1;
+ girq->parents = devm_kcalloc(dev, 1, sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents)
+ return -ENOMEM;
+ girq->parents[0] = platform_get_irq(pdev, 0);
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_simple_irq;
}
- if (np && !of_property_read_bool(np, "interrupt-controller"))
- return 0;
-
- err = gpiochip_irqchip_add(&ctrl->gc, &ath79_gpio_irqchip, 0,
- handle_simple_irq, IRQ_TYPE_NONE);
+ err = devm_gpiochip_add_data(dev, &ctrl->gc, ctrl);
if (err) {
- dev_err(&pdev->dev, "failed to add gpiochip_irqchip\n");
- goto gpiochip_remove;
+ dev_err(dev,
+ "cannot add AR71xx GPIO chip, error=%d", err);
+ return err;
}
-
- gpiochip_set_chained_irqchip(&ctrl->gc, &ath79_gpio_irqchip,
- platform_get_irq(pdev, 0),
- ath79_gpio_irq_handler);
-
- return 0;
-
-gpiochip_remove:
- gpiochip_remove(&ctrl->gc);
- return err;
-}
-
-static int ath79_gpio_remove(struct platform_device *pdev)
-{
- struct ath79_gpio_ctrl *ctrl = platform_get_drvdata(pdev);
-
- gpiochip_remove(&ctrl->gc);
return 0;
}
@@ -319,7 +308,6 @@ static struct platform_driver ath79_gpio_driver = {
.of_match_table = ath79_gpio_of_match,
},
.probe = ath79_gpio_probe,
- .remove = ath79_gpio_remove,
};
module_platform_driver(ath79_gpio_driver);
diff --git a/drivers/gpio/gpio-bd70528.c b/drivers/gpio/gpio-bd70528.c
new file mode 100644
index 000000000000..0c1ead12d883
--- /dev/null
+++ b/drivers/gpio/gpio-bd70528.c
@@ -0,0 +1,232 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2018 ROHM Semiconductors
+// gpio-bd70528.c ROHM BD70528MWV gpio driver
+
+#include <linux/gpio/driver.h>
+#include <linux/mfd/rohm-bd70528.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#define GPIO_IN_REG(offset) (BD70528_REG_GPIO1_IN + (offset) * 2)
+#define GPIO_OUT_REG(offset) (BD70528_REG_GPIO1_OUT + (offset) * 2)
+
+struct bd70528_gpio {
+ struct rohm_regmap_dev chip;
+ struct gpio_chip gpio;
+};
+
+static int bd70528_set_debounce(struct bd70528_gpio *bdgpio,
+ unsigned int offset, unsigned int debounce)
+{
+ u8 val;
+
+ switch (debounce) {
+ case 0:
+ val = BD70528_DEBOUNCE_DISABLE;
+ break;
+ case 1 ... 15:
+ val = BD70528_DEBOUNCE_15MS;
+ break;
+ case 16 ... 30:
+ val = BD70528_DEBOUNCE_30MS;
+ break;
+ case 31 ... 50:
+ val = BD70528_DEBOUNCE_50MS;
+ break;
+ default:
+ dev_err(bdgpio->chip.dev,
+ "Invalid debounce value %u\n", debounce);
+ return -EINVAL;
+ }
+ return regmap_update_bits(bdgpio->chip.regmap, GPIO_IN_REG(offset),
+ BD70528_DEBOUNCE_MASK, val);
+}
+
+static int bd70528_get_direction(struct gpio_chip *chip, unsigned int offset)
+{
+ struct bd70528_gpio *bdgpio = gpiochip_get_data(chip);
+ int val, ret;
+
+ /* Do we need to do something to IRQs here? */
+ ret = regmap_read(bdgpio->chip.regmap, GPIO_OUT_REG(offset), &val);
+ if (ret) {
+ dev_err(bdgpio->chip.dev, "Could not read gpio direction\n");
+ return ret;
+ }
+
+ return !(val & BD70528_GPIO_OUT_EN_MASK);
+}
+
+static int bd70528_gpio_set_config(struct gpio_chip *chip, unsigned int offset,
+ unsigned long config)
+{
+ struct bd70528_gpio *bdgpio = gpiochip_get_data(chip);
+
+ switch (pinconf_to_config_param(config)) {
+ case PIN_CONFIG_DRIVE_OPEN_DRAIN:
+ return regmap_update_bits(bdgpio->chip.regmap,
+ GPIO_OUT_REG(offset),
+ BD70528_GPIO_DRIVE_MASK,
+ BD70528_GPIO_OPEN_DRAIN);
+ break;
+ case PIN_CONFIG_DRIVE_PUSH_PULL:
+ return regmap_update_bits(bdgpio->chip.regmap,
+ GPIO_OUT_REG(offset),
+ BD70528_GPIO_DRIVE_MASK,
+ BD70528_GPIO_PUSH_PULL);
+ break;
+ case PIN_CONFIG_INPUT_DEBOUNCE:
+ return bd70528_set_debounce(bdgpio, offset,
+ pinconf_to_config_argument(config));
+ break;
+ default:
+ break;
+ }
+ return -ENOTSUPP;
+}
+
+static int bd70528_direction_input(struct gpio_chip *chip, unsigned int offset)
+{
+ struct bd70528_gpio *bdgpio = gpiochip_get_data(chip);
+
+ /* Do we need to do something to IRQs here? */
+ return regmap_update_bits(bdgpio->chip.regmap, GPIO_OUT_REG(offset),
+ BD70528_GPIO_OUT_EN_MASK,
+ BD70528_GPIO_OUT_DISABLE);
+}
+
+static void bd70528_gpio_set(struct gpio_chip *chip, unsigned int offset,
+ int value)
+{
+ int ret;
+ struct bd70528_gpio *bdgpio = gpiochip_get_data(chip);
+ u8 val = (value) ? BD70528_GPIO_OUT_HI : BD70528_GPIO_OUT_LO;
+
+ ret = regmap_update_bits(bdgpio->chip.regmap, GPIO_OUT_REG(offset),
+ BD70528_GPIO_OUT_MASK, val);
+ if (ret)
+ dev_err(bdgpio->chip.dev, "Could not set gpio to %d\n", value);
+}
+
+static int bd70528_direction_output(struct gpio_chip *chip, unsigned int offset,
+ int value)
+{
+ struct bd70528_gpio *bdgpio = gpiochip_get_data(chip);
+
+ bd70528_gpio_set(chip, offset, value);
+ return regmap_update_bits(bdgpio->chip.regmap, GPIO_OUT_REG(offset),
+ BD70528_GPIO_OUT_EN_MASK,
+ BD70528_GPIO_OUT_ENABLE);
+}
+
+#define GPIO_IN_STATE_MASK(offset) (BD70528_GPIO_IN_STATE_BASE << (offset))
+
+static int bd70528_gpio_get_o(struct bd70528_gpio *bdgpio, unsigned int offset)
+{
+ int ret;
+ unsigned int val;
+
+ ret = regmap_read(bdgpio->chip.regmap, GPIO_OUT_REG(offset), &val);
+ if (!ret)
+ ret = !!(val & BD70528_GPIO_OUT_MASK);
+ else
+ dev_err(bdgpio->chip.dev, "GPIO (out) state read failed\n");
+
+ return ret;
+}
+
+static int bd70528_gpio_get_i(struct bd70528_gpio *bdgpio, unsigned int offset)
+{
+ unsigned int val;
+ int ret;
+
+ ret = regmap_read(bdgpio->chip.regmap, BD70528_REG_GPIO_STATE, &val);
+
+ if (!ret)
+ ret = !(val & GPIO_IN_STATE_MASK(offset));
+ else
+ dev_err(bdgpio->chip.dev, "GPIO (in) state read failed\n");
+
+ return ret;
+}
+
+static int bd70528_gpio_get(struct gpio_chip *chip, unsigned int offset)
+{
+ int ret;
+ struct bd70528_gpio *bdgpio = gpiochip_get_data(chip);
+
+ /*
+ * There is a race condition where someone might be changing the
+ * GPIO direction after we get it but before we read the value. But
+ * application design where GPIO direction may be changed just when
+ * we read GPIO value would be pointless as reader could not know
+ * whether the returned high/low state is caused by input or output.
+ * Or then there must be other ways to mitigate the issue. Thus
+ * locking would make no sense.
+ */
+ ret = bd70528_get_direction(chip, offset);
+ if (ret == 0)
+ ret = bd70528_gpio_get_o(bdgpio, offset);
+ else if (ret == 1)
+ ret = bd70528_gpio_get_i(bdgpio, offset);
+ else
+ dev_err(bdgpio->chip.dev, "failed to read GPIO direction\n");
+
+ return ret;
+}
+
+static int bd70528_probe(struct platform_device *pdev)
+{
+ struct bd70528_gpio *bdgpio;
+ struct rohm_regmap_dev *bd70528;
+ int ret;
+
+ bd70528 = dev_get_drvdata(pdev->dev.parent);
+ if (!bd70528) {
+ dev_err(&pdev->dev, "No MFD driver data\n");
+ return -EINVAL;
+ }
+
+ bdgpio = devm_kzalloc(&pdev->dev, sizeof(*bdgpio),
+ GFP_KERNEL);
+ if (!bdgpio)
+ return -ENOMEM;
+ bdgpio->chip.dev = &pdev->dev;
+ bdgpio->gpio.parent = pdev->dev.parent;
+ bdgpio->gpio.label = "bd70528-gpio";
+ bdgpio->gpio.owner = THIS_MODULE;
+ bdgpio->gpio.get_direction = bd70528_get_direction;
+ bdgpio->gpio.direction_input = bd70528_direction_input;
+ bdgpio->gpio.direction_output = bd70528_direction_output;
+ bdgpio->gpio.set_config = bd70528_gpio_set_config;
+ bdgpio->gpio.can_sleep = true;
+ bdgpio->gpio.get = bd70528_gpio_get;
+ bdgpio->gpio.set = bd70528_gpio_set;
+ bdgpio->gpio.ngpio = 4;
+ bdgpio->gpio.base = -1;
+#ifdef CONFIG_OF_GPIO
+ bdgpio->gpio.of_node = pdev->dev.parent->of_node;
+#endif
+ bdgpio->chip.regmap = bd70528->regmap;
+
+ ret = devm_gpiochip_add_data(&pdev->dev, &bdgpio->gpio,
+ bdgpio);
+ if (ret)
+ dev_err(&pdev->dev, "gpio_init: Failed to add bd70528-gpio\n");
+
+ return ret;
+}
+
+static struct platform_driver bd70528_gpio = {
+ .driver = {
+ .name = "bd70528-gpio"
+ },
+ .probe = bd70528_probe,
+};
+
+module_platform_driver(bd70528_gpio);
+
+MODULE_AUTHOR("Matti Vaittinen <matti.vaittinen@fi.rohmeurope.com>");
+MODULE_DESCRIPTION("BD70528 voltage regulator driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-brcmstb.c b/drivers/gpio/gpio-brcmstb.c
index af936dcca659..05e3f99ae59c 100644
--- a/drivers/gpio/gpio-brcmstb.c
+++ b/drivers/gpio/gpio-brcmstb.c
@@ -636,10 +636,8 @@ static int brcmstb_gpio_probe(struct platform_device *pdev)
if (of_property_read_bool(np, "interrupt-controller")) {
priv->parent_irq = platform_get_irq(pdev, 0);
- if (priv->parent_irq <= 0) {
- dev_err(dev, "Couldn't get IRQ");
+ if (priv->parent_irq <= 0)
return -ENOENT;
- }
} else {
priv->parent_irq = -ENOENT;
}
diff --git a/drivers/gpio/gpio-cadence.c b/drivers/gpio/gpio-cadence.c
index 712ae212b0b4..a4d3239d2594 100644
--- a/drivers/gpio/gpio-cadence.c
+++ b/drivers/gpio/gpio-cadence.c
@@ -214,27 +214,33 @@ static int cdns_gpio_probe(struct platform_device *pdev)
goto err_revert_dir;
}
- ret = devm_gpiochip_add_data(&pdev->dev, &cgpio->gc, cgpio);
- if (ret < 0) {
- dev_err(&pdev->dev, "Could not register gpiochip, %d\n", ret);
- goto err_disable_clk;
- }
-
/*
- * irq_chip support
+ * Optional irq_chip support
*/
irq = platform_get_irq(pdev, 0);
if (irq >= 0) {
- ret = gpiochip_irqchip_add(&cgpio->gc, &cdns_gpio_irqchip,
- 0, handle_level_irq,
- IRQ_TYPE_NONE);
- if (ret) {
- dev_err(&pdev->dev, "Could not add irqchip, %d\n",
- ret);
+ struct gpio_irq_chip *girq;
+
+ girq = &cgpio->gc.irq;
+ girq->chip = &cdns_gpio_irqchip;
+ girq->parent_handler = cdns_gpio_irq_handler;
+ girq->num_parents = 1;
+ girq->parents = devm_kcalloc(&pdev->dev, 1,
+ sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents) {
+ ret = -ENOMEM;
goto err_disable_clk;
}
- gpiochip_set_chained_irqchip(&cgpio->gc, &cdns_gpio_irqchip,
- irq, cdns_gpio_irq_handler);
+ girq->parents[0] = irq;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_level_irq;
+ }
+
+ ret = devm_gpiochip_add_data(&pdev->dev, &cgpio->gc, cgpio);
+ if (ret < 0) {
+ dev_err(&pdev->dev, "Could not register gpiochip, %d\n", ret);
+ goto err_disable_clk;
}
cgpio->bypass_orig = ioread32(cgpio->regs + CDNS_GPIO_BYPASS_MODE);
diff --git a/drivers/gpio/gpio-creg-snps.c b/drivers/gpio/gpio-creg-snps.c
index 8cbc94d0d424..ff19a8ad5663 100644
--- a/drivers/gpio/gpio-creg-snps.c
+++ b/drivers/gpio/gpio-creg-snps.c
@@ -137,7 +137,6 @@ static int creg_gpio_probe(struct platform_device *pdev)
const struct of_device_id *match;
struct device *dev = &pdev->dev;
struct creg_gpio *hcg;
- struct resource *mem;
u32 ngpios;
int ret;
@@ -145,8 +144,7 @@ static int creg_gpio_probe(struct platform_device *pdev)
if (!hcg)
return -ENOMEM;
- mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- hcg->regs = devm_ioremap_resource(dev, mem);
+ hcg->regs = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(hcg->regs))
return PTR_ERR(hcg->regs);
diff --git a/drivers/gpio/gpio-cs5535.c b/drivers/gpio/gpio-cs5535.c
index 6314225dbed0..53b24e3ae7de 100644
--- a/drivers/gpio/gpio-cs5535.c
+++ b/drivers/gpio/gpio-cs5535.c
@@ -41,7 +41,7 @@ MODULE_PARM_DESC(mask, "GPIO channel mask.");
/*
* FIXME: convert this singleton driver to use the state container
- * design pattern, see Documentation/driver-model/design-patterns.txt
+ * design pattern, see Documentation/driver-api/driver-model/design-patterns.rst
*/
static struct cs5535_gpio_chip {
struct gpio_chip chip;
diff --git a/drivers/gpio/gpio-davinci.c b/drivers/gpio/gpio-davinci.c
index 3bbf5804bd11..e0b025689625 100644
--- a/drivers/gpio/gpio-davinci.c
+++ b/drivers/gpio/gpio-davinci.c
@@ -238,8 +238,9 @@ static int davinci_gpio_probe(struct platform_device *pdev)
for (i = 0; i < nirq; i++) {
chips->irqs[i] = platform_get_irq(pdev, i);
if (chips->irqs[i] < 0) {
- dev_info(dev, "IRQ not populated, err = %d\n",
- chips->irqs[i]);
+ if (chips->irqs[i] != -EPROBE_DEFER)
+ dev_info(dev, "IRQ not populated, err = %d\n",
+ chips->irqs[i]);
return chips->irqs[i];
}
}
@@ -297,7 +298,7 @@ static int davinci_gpio_probe(struct platform_device *pdev)
static void gpio_irq_disable(struct irq_data *d)
{
struct davinci_gpio_regs __iomem *g = irq2regs(d);
- u32 mask = (u32) irq_data_get_irq_handler_data(d);
+ uintptr_t mask = (uintptr_t)irq_data_get_irq_handler_data(d);
writel_relaxed(mask, &g->clr_falling);
writel_relaxed(mask, &g->clr_rising);
@@ -306,7 +307,7 @@ static void gpio_irq_disable(struct irq_data *d)
static void gpio_irq_enable(struct irq_data *d)
{
struct davinci_gpio_regs __iomem *g = irq2regs(d);
- u32 mask = (u32) irq_data_get_irq_handler_data(d);
+ uintptr_t mask = (uintptr_t)irq_data_get_irq_handler_data(d);
unsigned status = irqd_get_trigger_type(d);
status &= IRQ_TYPE_EDGE_FALLING | IRQ_TYPE_EDGE_RISING;
@@ -447,7 +448,7 @@ davinci_gpio_irq_map(struct irq_domain *d, unsigned int irq,
"davinci_gpio");
irq_set_irq_type(irq, IRQ_TYPE_NONE);
irq_set_chip_data(irq, (__force void *)g);
- irq_set_handler_data(irq, (void *)__gpio_mask(hw));
+ irq_set_handler_data(irq, (void *)(uintptr_t)__gpio_mask(hw));
return 0;
}
@@ -632,6 +633,7 @@ done:
static const struct of_device_id davinci_gpio_ids[] = {
{ .compatible = "ti,keystone-gpio", keystone_gpio_get_irq_chip},
+ { .compatible = "ti,am654-gpio", keystone_gpio_get_irq_chip},
{ .compatible = "ti,dm6441-gpio", davinci_gpio_get_irq_chip},
{ /* sentinel */ },
};
diff --git a/drivers/gpio/gpio-dwapb.c b/drivers/gpio/gpio-dwapb.c
index 3108be5e208c..92e127e74813 100644
--- a/drivers/gpio/gpio-dwapb.c
+++ b/drivers/gpio/gpio-dwapb.c
@@ -27,6 +27,7 @@
#include <linux/slab.h>
#include "gpiolib.h"
+#include "gpiolib-acpi.h"
#define GPIO_SWPORTA_DR 0x00
#define GPIO_SWPORTA_DDR 0x04
diff --git a/drivers/gpio/gpio-eic-sprd.c b/drivers/gpio/gpio-eic-sprd.c
index 77092268ee95..fe7a73f52329 100644
--- a/drivers/gpio/gpio-eic-sprd.c
+++ b/drivers/gpio/gpio-eic-sprd.c
@@ -568,7 +568,6 @@ static int sprd_eic_probe(struct platform_device *pdev)
const struct sprd_eic_variant_data *pdata;
struct gpio_irq_chip *irq;
struct sprd_eic *sprd_eic;
- struct resource *res;
int ret, i;
pdata = of_device_get_match_data(&pdev->dev);
@@ -585,10 +584,8 @@ static int sprd_eic_probe(struct platform_device *pdev)
sprd_eic->type = pdata->type;
sprd_eic->irq = platform_get_irq(pdev, 0);
- if (sprd_eic->irq < 0) {
- dev_err(&pdev->dev, "Failed to get EIC interrupt.\n");
+ if (sprd_eic->irq < 0)
return sprd_eic->irq;
- }
for (i = 0; i < SPRD_EIC_MAX_BANK; i++) {
/*
@@ -597,13 +594,9 @@ static int sprd_eic_probe(struct platform_device *pdev)
* have one bank EIC, thus base[1] and base[2] can be
* optional.
*/
- res = platform_get_resource(pdev, IORESOURCE_MEM, i);
- if (!res)
- continue;
-
- sprd_eic->base[i] = devm_ioremap_resource(&pdev->dev, res);
+ sprd_eic->base[i] = devm_platform_ioremap_resource(pdev, i);
if (IS_ERR(sprd_eic->base[i]))
- return PTR_ERR(sprd_eic->base[i]);
+ continue;
}
sprd_eic->chip.label = sprd_eic_label_name[sprd_eic->type];
diff --git a/drivers/gpio/gpio-em.c b/drivers/gpio/gpio-em.c
index 84a7375cee0a..620f25b7efb4 100644
--- a/drivers/gpio/gpio-em.c
+++ b/drivers/gpio/gpio-em.c
@@ -259,21 +259,27 @@ static const struct irq_domain_ops em_gio_irq_domain_ops = {
.xlate = irq_domain_xlate_twocell,
};
+static void em_gio_irq_domain_remove(void *data)
+{
+ struct irq_domain *domain = data;
+
+ irq_domain_remove(domain);
+}
+
static int em_gio_probe(struct platform_device *pdev)
{
struct em_gio_priv *p;
struct resource *io[2], *irq[2];
struct gpio_chip *gpio_chip;
struct irq_chip *irq_chip;
- const char *name = dev_name(&pdev->dev);
+ struct device *dev = &pdev->dev;
+ const char *name = dev_name(dev);
unsigned int ngpios;
int ret;
- p = devm_kzalloc(&pdev->dev, sizeof(*p), GFP_KERNEL);
- if (!p) {
- ret = -ENOMEM;
- goto err0;
- }
+ p = devm_kzalloc(dev, sizeof(*p), GFP_KERNEL);
+ if (!p)
+ return -ENOMEM;
p->pdev = pdev;
platform_set_drvdata(pdev, p);
@@ -285,35 +291,27 @@ static int em_gio_probe(struct platform_device *pdev)
irq[1] = platform_get_resource(pdev, IORESOURCE_IRQ, 1);
if (!io[0] || !io[1] || !irq[0] || !irq[1]) {
- dev_err(&pdev->dev, "missing IRQ or IOMEM\n");
- ret = -EINVAL;
- goto err0;
+ dev_err(dev, "missing IRQ or IOMEM\n");
+ return -EINVAL;
}
- p->base0 = devm_ioremap_nocache(&pdev->dev, io[0]->start,
+ p->base0 = devm_ioremap_nocache(dev, io[0]->start,
resource_size(io[0]));
- if (!p->base0) {
- dev_err(&pdev->dev, "failed to remap low I/O memory\n");
- ret = -ENXIO;
- goto err0;
- }
+ if (!p->base0)
+ return -ENOMEM;
- p->base1 = devm_ioremap_nocache(&pdev->dev, io[1]->start,
+ p->base1 = devm_ioremap_nocache(dev, io[1]->start,
resource_size(io[1]));
- if (!p->base1) {
- dev_err(&pdev->dev, "failed to remap high I/O memory\n");
- ret = -ENXIO;
- goto err0;
- }
+ if (!p->base1)
+ return -ENOMEM;
- if (of_property_read_u32(pdev->dev.of_node, "ngpios", &ngpios)) {
- dev_err(&pdev->dev, "Missing ngpios OF property\n");
- ret = -EINVAL;
- goto err0;
+ if (of_property_read_u32(dev->of_node, "ngpios", &ngpios)) {
+ dev_err(dev, "Missing ngpios OF property\n");
+ return -EINVAL;
}
gpio_chip = &p->gpio_chip;
- gpio_chip->of_node = pdev->dev.of_node;
+ gpio_chip->of_node = dev->of_node;
gpio_chip->direction_input = em_gio_direction_input;
gpio_chip->get = em_gio_get;
gpio_chip->direction_output = em_gio_direction_output;
@@ -322,7 +320,7 @@ static int em_gio_probe(struct platform_device *pdev)
gpio_chip->request = em_gio_request;
gpio_chip->free = em_gio_free;
gpio_chip->label = name;
- gpio_chip->parent = &pdev->dev;
+ gpio_chip->parent = dev;
gpio_chip->owner = THIS_MODULE;
gpio_chip->base = -1;
gpio_chip->ngpio = ngpios;
@@ -336,50 +334,37 @@ static int em_gio_probe(struct platform_device *pdev)
irq_chip->irq_release_resources = em_gio_irq_relres;
irq_chip->flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND;
- p->irq_domain = irq_domain_add_simple(pdev->dev.of_node, ngpios, 0,
+ p->irq_domain = irq_domain_add_simple(dev->of_node, ngpios, 0,
&em_gio_irq_domain_ops, p);
if (!p->irq_domain) {
- ret = -ENXIO;
- dev_err(&pdev->dev, "cannot initialize irq domain\n");
- goto err0;
+ dev_err(dev, "cannot initialize irq domain\n");
+ return -ENXIO;
}
- if (devm_request_irq(&pdev->dev, irq[0]->start,
+ ret = devm_add_action_or_reset(dev, em_gio_irq_domain_remove,
+ p->irq_domain);
+ if (ret)
+ return ret;
+
+ if (devm_request_irq(dev, irq[0]->start,
em_gio_irq_handler, 0, name, p)) {
- dev_err(&pdev->dev, "failed to request low IRQ\n");
- ret = -ENOENT;
- goto err1;
+ dev_err(dev, "failed to request low IRQ\n");
+ return -ENOENT;
}
- if (devm_request_irq(&pdev->dev, irq[1]->start,
+ if (devm_request_irq(dev, irq[1]->start,
em_gio_irq_handler, 0, name, p)) {
- dev_err(&pdev->dev, "failed to request high IRQ\n");
- ret = -ENOENT;
- goto err1;
+ dev_err(dev, "failed to request high IRQ\n");
+ return -ENOENT;
}
- ret = gpiochip_add_data(gpio_chip, p);
+ ret = devm_gpiochip_add_data(dev, gpio_chip, p);
if (ret) {
- dev_err(&pdev->dev, "failed to add GPIO controller\n");
- goto err1;
+ dev_err(dev, "failed to add GPIO controller\n");
+ return ret;
}
return 0;
-
-err1:
- irq_domain_remove(p->irq_domain);
-err0:
- return ret;
-}
-
-static int em_gio_remove(struct platform_device *pdev)
-{
- struct em_gio_priv *p = platform_get_drvdata(pdev);
-
- gpiochip_remove(&p->gpio_chip);
-
- irq_domain_remove(p->irq_domain);
- return 0;
}
static const struct of_device_id em_gio_dt_ids[] = {
@@ -390,7 +375,6 @@ MODULE_DEVICE_TABLE(of, em_gio_dt_ids);
static struct platform_driver em_gio_device_driver = {
.probe = em_gio_probe,
- .remove = em_gio_remove,
.driver = {
.name = "em_gio",
.of_match_table = em_gio_dt_ids,
diff --git a/drivers/gpio/gpio-ep93xx.c b/drivers/gpio/gpio-ep93xx.c
index 71728d6e0bca..226da8df6f10 100644
--- a/drivers/gpio/gpio-ep93xx.c
+++ b/drivers/gpio/gpio-ep93xx.c
@@ -269,56 +269,6 @@ static struct irq_chip ep93xx_gpio_irq_chip = {
.irq_set_type = ep93xx_gpio_irq_type,
};
-static int ep93xx_gpio_init_irq(struct platform_device *pdev,
- struct ep93xx_gpio *epg)
-{
- int ab_parent_irq = platform_get_irq(pdev, 0);
- struct device *dev = &pdev->dev;
- int gpio_irq;
- int ret;
- int i;
-
- /* The A bank */
- ret = gpiochip_irqchip_add(&epg->gc[0], &ep93xx_gpio_irq_chip,
- 64, handle_level_irq,
- IRQ_TYPE_NONE);
- if (ret) {
- dev_err(dev, "Could not add irqchip 0\n");
- return ret;
- }
- gpiochip_set_chained_irqchip(&epg->gc[0], &ep93xx_gpio_irq_chip,
- ab_parent_irq,
- ep93xx_gpio_ab_irq_handler);
-
- /* The B bank */
- ret = gpiochip_irqchip_add(&epg->gc[1], &ep93xx_gpio_irq_chip,
- 72, handle_level_irq,
- IRQ_TYPE_NONE);
- if (ret) {
- dev_err(dev, "Could not add irqchip 1\n");
- return ret;
- }
- gpiochip_set_chained_irqchip(&epg->gc[1], &ep93xx_gpio_irq_chip,
- ab_parent_irq,
- ep93xx_gpio_ab_irq_handler);
-
- /* The F bank */
- for (i = 0; i < 8; i++) {
- gpio_irq = EP93XX_GPIO_F_IRQ_BASE + i;
- irq_set_chip_data(gpio_irq, &epg->gc[5]);
- irq_set_chip_and_handler(gpio_irq, &ep93xx_gpio_irq_chip,
- handle_level_irq);
- irq_clear_status_flags(gpio_irq, IRQ_NOREQUEST);
- }
-
- for (i = 1; i <= 8; i++)
- irq_set_chained_handler_and_data(platform_get_irq(pdev, i),
- ep93xx_gpio_f_irq_handler,
- &epg->gc[i]);
- return 0;
-}
-
-
/*************************************************************************
* gpiolib interface for EP93xx on-chip GPIOs
*************************************************************************/
@@ -328,26 +278,33 @@ struct ep93xx_gpio_bank {
int dir;
int base;
bool has_irq;
+ bool has_hierarchical_irq;
+ unsigned int irq_base;
};
-#define EP93XX_GPIO_BANK(_label, _data, _dir, _base, _has_irq) \
+#define EP93XX_GPIO_BANK(_label, _data, _dir, _base, _has_irq, _has_hier, _irq_base) \
{ \
.label = _label, \
.data = _data, \
.dir = _dir, \
.base = _base, \
.has_irq = _has_irq, \
+ .has_hierarchical_irq = _has_hier, \
+ .irq_base = _irq_base, \
}
static struct ep93xx_gpio_bank ep93xx_gpio_banks[] = {
- EP93XX_GPIO_BANK("A", 0x00, 0x10, 0, true), /* Bank A has 8 IRQs */
- EP93XX_GPIO_BANK("B", 0x04, 0x14, 8, true), /* Bank B has 8 IRQs */
- EP93XX_GPIO_BANK("C", 0x08, 0x18, 40, false),
- EP93XX_GPIO_BANK("D", 0x0c, 0x1c, 24, false),
- EP93XX_GPIO_BANK("E", 0x20, 0x24, 32, false),
- EP93XX_GPIO_BANK("F", 0x30, 0x34, 16, true), /* Bank F has 8 IRQs */
- EP93XX_GPIO_BANK("G", 0x38, 0x3c, 48, false),
- EP93XX_GPIO_BANK("H", 0x40, 0x44, 56, false),
+ /* Bank A has 8 IRQs */
+ EP93XX_GPIO_BANK("A", 0x00, 0x10, 0, true, false, 64),
+ /* Bank B has 8 IRQs */
+ EP93XX_GPIO_BANK("B", 0x04, 0x14, 8, true, false, 72),
+ EP93XX_GPIO_BANK("C", 0x08, 0x18, 40, false, false, 0),
+ EP93XX_GPIO_BANK("D", 0x0c, 0x1c, 24, false, false, 0),
+ EP93XX_GPIO_BANK("E", 0x20, 0x24, 32, false, false, 0),
+ /* Bank F has 8 IRQs */
+ EP93XX_GPIO_BANK("F", 0x30, 0x34, 16, false, true, 0),
+ EP93XX_GPIO_BANK("G", 0x38, 0x3c, 48, false, false, 0),
+ EP93XX_GPIO_BANK("H", 0x40, 0x44, 56, false, false, 0),
};
static int ep93xx_gpio_set_config(struct gpio_chip *gc, unsigned offset,
@@ -369,12 +326,15 @@ static int ep93xx_gpio_f_to_irq(struct gpio_chip *gc, unsigned offset)
return EP93XX_GPIO_F_IRQ_BASE + offset;
}
-static int ep93xx_gpio_add_bank(struct gpio_chip *gc, struct device *dev,
+static int ep93xx_gpio_add_bank(struct gpio_chip *gc,
+ struct platform_device *pdev,
struct ep93xx_gpio *epg,
struct ep93xx_gpio_bank *bank)
{
void __iomem *data = epg->base + bank->data;
void __iomem *dir = epg->base + bank->dir;
+ struct device *dev = &pdev->dev;
+ struct gpio_irq_chip *girq;
int err;
err = bgpio_init(gc, dev, 1, data, NULL, NULL, dir, NULL, 0);
@@ -384,8 +344,59 @@ static int ep93xx_gpio_add_bank(struct gpio_chip *gc, struct device *dev,
gc->label = bank->label;
gc->base = bank->base;
- if (bank->has_irq)
+ girq = &gc->irq;
+ if (bank->has_irq || bank->has_hierarchical_irq) {
gc->set_config = ep93xx_gpio_set_config;
+ girq->chip = &ep93xx_gpio_irq_chip;
+ }
+
+ if (bank->has_irq) {
+ int ab_parent_irq = platform_get_irq(pdev, 0);
+
+ girq->parent_handler = ep93xx_gpio_ab_irq_handler;
+ girq->num_parents = 1;
+ girq->parents = devm_kcalloc(dev, 1,
+ sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents)
+ return -ENOMEM;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_level_irq;
+ girq->parents[0] = ab_parent_irq;
+ girq->first = bank->irq_base;
+ }
+
+ /* Only bank F has especially funky IRQ handling */
+ if (bank->has_hierarchical_irq) {
+ int gpio_irq;
+ int i;
+
+ /*
+ * FIXME: convert this to use hierarchical IRQ support!
+ * this requires fixing the root irqchip to be hierarchial.
+ */
+ girq->parent_handler = ep93xx_gpio_f_irq_handler;
+ girq->num_parents = 8;
+ girq->parents = devm_kcalloc(dev, 8,
+ sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents)
+ return -ENOMEM;
+ /* Pick resources 1..8 for these IRQs */
+ for (i = 1; i <= 8; i++)
+ girq->parents[i - 1] = platform_get_irq(pdev, i);
+ for (i = 0; i < 8; i++) {
+ gpio_irq = EP93XX_GPIO_F_IRQ_BASE + i;
+ irq_set_chip_data(gpio_irq, &epg->gc[5]);
+ irq_set_chip_and_handler(gpio_irq,
+ &ep93xx_gpio_irq_chip,
+ handle_level_irq);
+ irq_clear_status_flags(gpio_irq, IRQ_NOREQUEST);
+ }
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_level_irq;
+ gc->to_irq = ep93xx_gpio_f_to_irq;
+ }
return devm_gpiochip_add_data(dev, gc, epg);
}
@@ -393,16 +404,13 @@ static int ep93xx_gpio_add_bank(struct gpio_chip *gc, struct device *dev,
static int ep93xx_gpio_probe(struct platform_device *pdev)
{
struct ep93xx_gpio *epg;
- struct resource *res;
int i;
- struct device *dev = &pdev->dev;
- epg = devm_kzalloc(dev, sizeof(*epg), GFP_KERNEL);
+ epg = devm_kzalloc(&pdev->dev, sizeof(*epg), GFP_KERNEL);
if (!epg)
return -ENOMEM;
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- epg->base = devm_ioremap_resource(dev, res);
+ epg->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(epg->base))
return PTR_ERR(epg->base);
@@ -410,16 +418,11 @@ static int ep93xx_gpio_probe(struct platform_device *pdev)
struct gpio_chip *gc = &epg->gc[i];
struct ep93xx_gpio_bank *bank = &ep93xx_gpio_banks[i];
- if (ep93xx_gpio_add_bank(gc, &pdev->dev, epg, bank))
+ if (ep93xx_gpio_add_bank(gc, pdev, epg, bank))
dev_warn(&pdev->dev, "Unable to add gpio bank %s\n",
bank->label);
- /* Only bank F has especially funky IRQ handling */
- if (i == 5)
- gc->to_irq = ep93xx_gpio_f_to_irq;
}
- ep93xx_gpio_init_irq(pdev, epg);
-
return 0;
}
diff --git a/drivers/gpio/gpio-ftgpio010.c b/drivers/gpio/gpio-ftgpio010.c
index 8ff8ce2970d9..fbddb1662428 100644
--- a/drivers/gpio/gpio-ftgpio010.c
+++ b/drivers/gpio/gpio-ftgpio010.c
@@ -226,6 +226,7 @@ static int ftgpio_gpio_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct ftgpio_gpio *g;
+ struct gpio_irq_chip *girq;
int irq;
int ret;
@@ -277,9 +278,25 @@ static int ftgpio_gpio_probe(struct platform_device *pdev)
if (!IS_ERR(g->clk))
g->gc.set_config = ftgpio_gpio_set_config;
- ret = devm_gpiochip_add_data(dev, &g->gc, g);
- if (ret)
+ g->irq.name = "FTGPIO010";
+ g->irq.irq_ack = ftgpio_gpio_ack_irq;
+ g->irq.irq_mask = ftgpio_gpio_mask_irq;
+ g->irq.irq_unmask = ftgpio_gpio_unmask_irq;
+ g->irq.irq_set_type = ftgpio_gpio_set_irq_type;
+
+ girq = &g->gc.irq;
+ girq->chip = &g->irq;
+ girq->parent_handler = ftgpio_gpio_irq_handler;
+ girq->num_parents = 1;
+ girq->parents = devm_kcalloc(dev, 1, sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents) {
+ ret = -ENOMEM;
goto dis_clk;
+ }
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_bad_irq;
+ girq->parents[0] = irq;
/* Disable, unmask and clear all interrupts */
writel(0x0, g->base + GPIO_INT_EN);
@@ -289,21 +306,9 @@ static int ftgpio_gpio_probe(struct platform_device *pdev)
/* Clear any use of debounce */
writel(0x0, g->base + GPIO_DEBOUNCE_EN);
- g->irq.name = "FTGPIO010";
- g->irq.irq_ack = ftgpio_gpio_ack_irq;
- g->irq.irq_mask = ftgpio_gpio_mask_irq;
- g->irq.irq_unmask = ftgpio_gpio_unmask_irq;
- g->irq.irq_set_type = ftgpio_gpio_set_irq_type;
-
- ret = gpiochip_irqchip_add(&g->gc, &g->irq,
- 0, handle_bad_irq,
- IRQ_TYPE_NONE);
- if (ret) {
- dev_info(dev, "could not add irqchip\n");
+ ret = devm_gpiochip_add_data(dev, &g->gc, g);
+ if (ret)
goto dis_clk;
- }
- gpiochip_set_chained_irqchip(&g->gc, &g->irq,
- irq, ftgpio_gpio_irq_handler);
platform_set_drvdata(pdev, g);
dev_info(dev, "FTGPIO010 @%p registered\n", g->base);
diff --git a/drivers/gpio/gpio-grgpio.c b/drivers/gpio/gpio-grgpio.c
index 7df48e76baea..08234e64993a 100644
--- a/drivers/gpio/gpio-grgpio.c
+++ b/drivers/gpio/gpio-grgpio.c
@@ -329,7 +329,6 @@ static int grgpio_probe(struct platform_device *ofdev)
void __iomem *regs;
struct gpio_chip *gc;
struct grgpio_priv *priv;
- struct resource *res;
int err;
u32 prop;
s32 *irqmap;
@@ -340,8 +339,7 @@ static int grgpio_probe(struct platform_device *ofdev)
if (!priv)
return -ENOMEM;
- res = platform_get_resource(ofdev, IORESOURCE_MEM, 0);
- regs = devm_ioremap_resource(&ofdev->dev, res);
+ regs = devm_platform_ioremap_resource(ofdev, 0);
if (IS_ERR(regs))
return PTR_ERR(regs);
@@ -410,8 +408,6 @@ static int grgpio_probe(struct platform_device *ofdev)
* Continue without irq functionality for that
* gpio line
*/
- dev_err(priv->dev,
- "Failed to get irq for offset %d\n", i);
continue;
}
priv->uirqs[lirq->index].uirq = ret;
diff --git a/drivers/gpio/gpio-hlwd.c b/drivers/gpio/gpio-hlwd.c
index e5fa00f8145f..4a17599f6d44 100644
--- a/drivers/gpio/gpio-hlwd.c
+++ b/drivers/gpio/gpio-hlwd.c
@@ -244,43 +244,45 @@ static int hlwd_gpio_probe(struct platform_device *pdev)
ngpios = 32;
hlwd->gpioc.ngpio = ngpios;
- res = devm_gpiochip_add_data(&pdev->dev, &hlwd->gpioc, hlwd);
- if (res)
- return res;
-
/* Mask and ack all interrupts */
iowrite32be(0, hlwd->regs + HW_GPIOB_INTMASK);
iowrite32be(0xffffffff, hlwd->regs + HW_GPIOB_INTFLAG);
/*
* If this GPIO controller is not marked as an interrupt controller in
- * the DT, return.
+ * the DT, skip interrupt support.
*/
- if (!of_property_read_bool(pdev->dev.of_node, "interrupt-controller"))
- return 0;
-
- hlwd->irq = platform_get_irq(pdev, 0);
- if (hlwd->irq < 0) {
- dev_info(&pdev->dev, "platform_get_irq returned %d\n",
- hlwd->irq);
- return hlwd->irq;
+ if (of_property_read_bool(pdev->dev.of_node, "interrupt-controller")) {
+ struct gpio_irq_chip *girq;
+
+ hlwd->irq = platform_get_irq(pdev, 0);
+ if (hlwd->irq < 0) {
+ dev_info(&pdev->dev, "platform_get_irq returned %d\n",
+ hlwd->irq);
+ return hlwd->irq;
+ }
+
+ hlwd->irqc.name = dev_name(&pdev->dev);
+ hlwd->irqc.irq_mask = hlwd_gpio_irq_mask;
+ hlwd->irqc.irq_unmask = hlwd_gpio_irq_unmask;
+ hlwd->irqc.irq_enable = hlwd_gpio_irq_enable;
+ hlwd->irqc.irq_set_type = hlwd_gpio_irq_set_type;
+
+ girq = &hlwd->gpioc.irq;
+ girq->chip = &hlwd->irqc;
+ girq->parent_handler = hlwd_gpio_irqhandler;
+ girq->num_parents = 1;
+ girq->parents = devm_kcalloc(&pdev->dev, 1,
+ sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents)
+ return -ENOMEM;
+ girq->parents[0] = hlwd->irq;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_level_irq;
}
- hlwd->irqc.name = dev_name(&pdev->dev);
- hlwd->irqc.irq_mask = hlwd_gpio_irq_mask;
- hlwd->irqc.irq_unmask = hlwd_gpio_irq_unmask;
- hlwd->irqc.irq_enable = hlwd_gpio_irq_enable;
- hlwd->irqc.irq_set_type = hlwd_gpio_irq_set_type;
-
- res = gpiochip_irqchip_add(&hlwd->gpioc, &hlwd->irqc, 0,
- handle_level_irq, IRQ_TYPE_NONE);
- if (res)
- return res;
-
- gpiochip_set_chained_irqchip(&hlwd->gpioc, &hlwd->irqc,
- hlwd->irq, hlwd_gpio_irqhandler);
-
- return 0;
+ return devm_gpiochip_add_data(&pdev->dev, &hlwd->gpioc, hlwd);
}
static const struct of_device_id hlwd_gpio_match[] = {
diff --git a/drivers/gpio/gpio-htc-egpio.c b/drivers/gpio/gpio-htc-egpio.c
index 9d3ac51a765c..6eb56f7ab9c9 100644
--- a/drivers/gpio/gpio-htc-egpio.c
+++ b/drivers/gpio/gpio-htc-egpio.c
@@ -118,20 +118,6 @@ static void egpio_handler(struct irq_desc *desc)
}
}
-int htc_egpio_get_wakeup_irq(struct device *dev)
-{
- struct egpio_info *ei = dev_get_drvdata(dev);
-
- /* Read current pins. */
- u16 readval = egpio_readw(ei, ei->ack_register);
- /* Ack/unmask interrupts. */
- ack_irqs(ei);
- /* Return first set pin. */
- readval &= ei->irqs_enabled;
- return ei->irq_start + ffs(readval) - 1;
-}
-EXPORT_SYMBOL(htc_egpio_get_wakeup_irq);
-
static inline int egpio_pos(struct egpio_info *ei, int bit)
{
return bit >> ei->reg_shift;
diff --git a/drivers/gpio/gpio-intel-mid.c b/drivers/gpio/gpio-intel-mid.c
index 4e803baf980e..4d835f9089df 100644
--- a/drivers/gpio/gpio-intel-mid.c
+++ b/drivers/gpio/gpio-intel-mid.c
@@ -329,6 +329,7 @@ static int intel_gpio_probe(struct pci_dev *pdev,
u32 gpio_base;
u32 irq_base;
int retval;
+ struct gpio_irq_chip *girq;
struct intel_mid_gpio_ddata *ddata =
(struct intel_mid_gpio_ddata *)id->driver_data;
@@ -369,6 +370,22 @@ static int intel_gpio_probe(struct pci_dev *pdev,
spin_lock_init(&priv->lock);
+ girq = &priv->chip.irq;
+ girq->chip = &intel_mid_irqchip;
+ girq->parent_handler = intel_mid_irq_handler;
+ girq->num_parents = 1;
+ girq->parents = devm_kcalloc(&pdev->dev, girq->num_parents,
+ sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents)
+ return -ENOMEM;
+ girq->parents[0] = pdev->irq;
+ girq->first = irq_base;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_simple_irq;
+
+ intel_mid_irq_init_hw(priv);
+
pci_set_drvdata(pdev, priv);
retval = devm_gpiochip_add_data(&pdev->dev, &priv->chip, priv);
if (retval) {
@@ -376,24 +393,6 @@ static int intel_gpio_probe(struct pci_dev *pdev,
return retval;
}
- retval = gpiochip_irqchip_add(&priv->chip,
- &intel_mid_irqchip,
- irq_base,
- handle_simple_irq,
- IRQ_TYPE_NONE);
- if (retval) {
- dev_err(&pdev->dev,
- "could not connect irqchip to gpiochip\n");
- return retval;
- }
-
- intel_mid_irq_init_hw(priv);
-
- gpiochip_set_chained_irqchip(&priv->chip,
- &intel_mid_irqchip,
- pdev->irq,
- intel_mid_irq_handler);
-
pm_runtime_put_noidle(&pdev->dev);
pm_runtime_allow(&pdev->dev);
diff --git a/drivers/gpio/gpio-iop.c b/drivers/gpio/gpio-iop.c
index e355c5961eb9..7390b5ca09e3 100644
--- a/drivers/gpio/gpio-iop.c
+++ b/drivers/gpio/gpio-iop.c
@@ -36,6 +36,7 @@ static int iop3xx_gpio_probe(struct platform_device *pdev)
gc->base = 0;
gc->owner = THIS_MODULE;
+ gc->label = "gpio-iop";
return devm_gpiochip_add_data(&pdev->dev, gc, NULL);
}
diff --git a/drivers/gpio/gpio-ixp4xx.c b/drivers/gpio/gpio-ixp4xx.c
index 4b1cf7ea858d..b3b050604e0b 100644
--- a/drivers/gpio/gpio-ixp4xx.c
+++ b/drivers/gpio/gpio-ixp4xx.c
@@ -47,7 +47,6 @@
* @dev: containing device for this instance
* @fwnode: the fwnode for this GPIO chip
* @gc: gpiochip for this instance
- * @domain: irqdomain for this chip instance
* @base: remapped I/O-memory base
* @irq_edge: Each bit represents an IRQ: 1: edge-triggered,
* 0: level triggered
@@ -56,48 +55,22 @@ struct ixp4xx_gpio {
struct device *dev;
struct fwnode_handle *fwnode;
struct gpio_chip gc;
- struct irq_domain *domain;
void __iomem *base;
unsigned long long irq_edge;
};
-/**
- * struct ixp4xx_gpio_map - IXP4 GPIO to parent IRQ map
- * @gpio_offset: offset of the IXP4 GPIO line
- * @parent_hwirq: hwirq on the parent IRQ controller
- */
-struct ixp4xx_gpio_map {
- int gpio_offset;
- int parent_hwirq;
-};
-
-/* GPIO lines 0..12 have corresponding IRQs, GPIOs 13..15 have no IRQs */
-const struct ixp4xx_gpio_map ixp4xx_gpiomap[] = {
- { .gpio_offset = 0, .parent_hwirq = 6 },
- { .gpio_offset = 1, .parent_hwirq = 7 },
- { .gpio_offset = 2, .parent_hwirq = 19 },
- { .gpio_offset = 3, .parent_hwirq = 20 },
- { .gpio_offset = 4, .parent_hwirq = 21 },
- { .gpio_offset = 5, .parent_hwirq = 22 },
- { .gpio_offset = 6, .parent_hwirq = 23 },
- { .gpio_offset = 7, .parent_hwirq = 24 },
- { .gpio_offset = 8, .parent_hwirq = 25 },
- { .gpio_offset = 9, .parent_hwirq = 26 },
- { .gpio_offset = 10, .parent_hwirq = 27 },
- { .gpio_offset = 11, .parent_hwirq = 28 },
- { .gpio_offset = 12, .parent_hwirq = 29 },
-};
-
static void ixp4xx_gpio_irq_ack(struct irq_data *d)
{
- struct ixp4xx_gpio *g = irq_data_get_irq_chip_data(d);
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct ixp4xx_gpio *g = gpiochip_get_data(gc);
__raw_writel(BIT(d->hwirq), g->base + IXP4XX_REG_GPIS);
}
static void ixp4xx_gpio_irq_unmask(struct irq_data *d)
{
- struct ixp4xx_gpio *g = irq_data_get_irq_chip_data(d);
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct ixp4xx_gpio *g = gpiochip_get_data(gc);
/* ACK when unmasking if not edge-triggered */
if (!(g->irq_edge & BIT(d->hwirq)))
@@ -108,7 +81,8 @@ static void ixp4xx_gpio_irq_unmask(struct irq_data *d)
static int ixp4xx_gpio_irq_set_type(struct irq_data *d, unsigned int type)
{
- struct ixp4xx_gpio *g = irq_data_get_irq_chip_data(d);
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct ixp4xx_gpio *g = gpiochip_get_data(gc);
int line = d->hwirq;
unsigned long flags;
u32 int_style;
@@ -187,122 +161,31 @@ static struct irq_chip ixp4xx_gpio_irqchip = {
.irq_set_type = ixp4xx_gpio_irq_set_type,
};
-static int ixp4xx_gpio_to_irq(struct gpio_chip *gc, unsigned int offset)
+static int ixp4xx_gpio_child_to_parent_hwirq(struct gpio_chip *gc,
+ unsigned int child,
+ unsigned int child_type,
+ unsigned int *parent,
+ unsigned int *parent_type)
{
- struct ixp4xx_gpio *g = gpiochip_get_data(gc);
- struct irq_fwspec fwspec;
-
- fwspec.fwnode = g->fwnode;
- fwspec.param_count = 2;
- fwspec.param[0] = offset;
- fwspec.param[1] = IRQ_TYPE_NONE;
-
- return irq_create_fwspec_mapping(&fwspec);
-}
+ /* All these interrupts are level high in the CPU */
+ *parent_type = IRQ_TYPE_LEVEL_HIGH;
-static int ixp4xx_gpio_irq_domain_translate(struct irq_domain *domain,
- struct irq_fwspec *fwspec,
- unsigned long *hwirq,
- unsigned int *type)
-{
-
- /* We support standard DT translation */
- if (is_of_node(fwspec->fwnode) && fwspec->param_count == 2) {
- *hwirq = fwspec->param[0];
- *type = fwspec->param[1];
+ /* GPIO lines 0..12 have dedicated IRQs */
+ if (child == 0) {
+ *parent = 6;
return 0;
}
-
- /* This goes away when we transition to DT */
- if (is_fwnode_irqchip(fwspec->fwnode)) {
- if (fwspec->param_count != 2)
- return -EINVAL;
- *hwirq = fwspec->param[0];
- *type = fwspec->param[1];
- WARN_ON(*type == IRQ_TYPE_NONE);
+ if (child == 1) {
+ *parent = 7;
return 0;
}
- return -EINVAL;
-}
-
-static int ixp4xx_gpio_irq_domain_alloc(struct irq_domain *d,
- unsigned int irq, unsigned int nr_irqs,
- void *data)
-{
- struct ixp4xx_gpio *g = d->host_data;
- irq_hw_number_t hwirq;
- unsigned int type = IRQ_TYPE_NONE;
- struct irq_fwspec *fwspec = data;
- int ret;
- int i;
-
- ret = ixp4xx_gpio_irq_domain_translate(d, fwspec, &hwirq, &type);
- if (ret)
- return ret;
-
- dev_dbg(g->dev, "allocate IRQ %d..%d, hwirq %lu..%lu\n",
- irq, irq + nr_irqs - 1,
- hwirq, hwirq + nr_irqs - 1);
-
- for (i = 0; i < nr_irqs; i++) {
- struct irq_fwspec parent_fwspec;
- const struct ixp4xx_gpio_map *map;
- int j;
-
- /* Not all lines support IRQs */
- for (j = 0; j < ARRAY_SIZE(ixp4xx_gpiomap); j++) {
- map = &ixp4xx_gpiomap[j];
- if (map->gpio_offset == hwirq)
- break;
- }
- if (j == ARRAY_SIZE(ixp4xx_gpiomap)) {
- dev_err(g->dev, "can't look up hwirq %lu\n", hwirq);
- return -EINVAL;
- }
- dev_dbg(g->dev, "found parent hwirq %u\n", map->parent_hwirq);
-
- /*
- * We set handle_bad_irq because the .set_type() should
- * always be invoked and set the right type of handler.
- */
- irq_domain_set_info(d,
- irq + i,
- hwirq + i,
- &ixp4xx_gpio_irqchip,
- g,
- handle_bad_irq,
- NULL, NULL);
- irq_set_probe(irq + i);
-
- /*
- * Create a IRQ fwspec to send up to the parent irqdomain:
- * specify the hwirq we address on the parent and tie it
- * all together up the chain.
- */
- parent_fwspec.fwnode = d->parent->fwnode;
- parent_fwspec.param_count = 2;
- parent_fwspec.param[0] = map->parent_hwirq;
- /* This parent only handles asserted level IRQs */
- parent_fwspec.param[1] = IRQ_TYPE_LEVEL_HIGH;
- dev_dbg(g->dev, "alloc_irqs_parent for %d parent hwirq %d\n",
- irq + i, map->parent_hwirq);
- ret = irq_domain_alloc_irqs_parent(d, irq + i, 1,
- &parent_fwspec);
- if (ret)
- dev_err(g->dev,
- "failed to allocate parent hwirq %d for hwirq %lu\n",
- map->parent_hwirq, hwirq);
+ if (child >= 2 && child <= 12) {
+ *parent = child + 17;
+ return 0;
}
-
- return 0;
+ return -EINVAL;
}
-static const struct irq_domain_ops ixp4xx_gpio_irqdomain_ops = {
- .translate = ixp4xx_gpio_irq_domain_translate,
- .alloc = ixp4xx_gpio_irq_domain_alloc,
- .free = irq_domain_free_irqs_common,
-};
-
static int ixp4xx_gpio_probe(struct platform_device *pdev)
{
unsigned long flags;
@@ -311,8 +194,8 @@ static int ixp4xx_gpio_probe(struct platform_device *pdev)
struct irq_domain *parent;
struct resource *res;
struct ixp4xx_gpio *g;
+ struct gpio_irq_chip *girq;
int ret;
- int i;
g = devm_kzalloc(dev, sizeof(*g), GFP_KERNEL);
if (!g)
@@ -321,9 +204,36 @@ static int ixp4xx_gpio_probe(struct platform_device *pdev)
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
g->base = devm_ioremap_resource(dev, res);
- if (IS_ERR(g->base)) {
- dev_err(dev, "ioremap error\n");
+ if (IS_ERR(g->base))
return PTR_ERR(g->base);
+
+ /*
+ * When we convert to device tree we will simply look up the
+ * parent irqdomain using irq_find_host(parent) as parent comes
+ * from IRQCHIP_DECLARE(), then use of_node_to_fwnode() to get
+ * the fwnode. For now we need this boardfile style code.
+ */
+ if (np) {
+ struct device_node *irq_parent;
+
+ irq_parent = of_irq_find_parent(np);
+ if (!irq_parent) {
+ dev_err(dev, "no IRQ parent node\n");
+ return -ENODEV;
+ }
+ parent = irq_find_host(irq_parent);
+ if (!parent) {
+ dev_err(dev, "no IRQ parent domain\n");
+ return -ENODEV;
+ }
+ g->fwnode = of_node_to_fwnode(np);
+ } else {
+ parent = ixp4xx_get_irq_domain();
+ g->fwnode = irq_domain_alloc_fwnode(&res->start);
+ if (!g->fwnode) {
+ dev_err(dev, "no domain base\n");
+ return -ENODEV;
+ }
}
/*
@@ -360,7 +270,6 @@ static int ixp4xx_gpio_probe(struct platform_device *pdev)
dev_err(dev, "unable to init generic GPIO\n");
return ret;
}
- g->gc.to_irq = ixp4xx_gpio_to_irq;
g->gc.ngpio = 16;
g->gc.label = "IXP4XX_GPIO_CHIP";
/*
@@ -372,86 +281,22 @@ static int ixp4xx_gpio_probe(struct platform_device *pdev)
g->gc.parent = &pdev->dev;
g->gc.owner = THIS_MODULE;
+ girq = &g->gc.irq;
+ girq->chip = &ixp4xx_gpio_irqchip;
+ girq->fwnode = g->fwnode;
+ girq->parent_domain = parent;
+ girq->child_to_parent_hwirq = ixp4xx_gpio_child_to_parent_hwirq;
+ girq->handler = handle_bad_irq;
+ girq->default_type = IRQ_TYPE_NONE;
+
ret = devm_gpiochip_add_data(dev, &g->gc, g);
if (ret) {
dev_err(dev, "failed to add SoC gpiochip\n");
return ret;
}
- /*
- * When we convert to device tree we will simply look up the
- * parent irqdomain using irq_find_host(parent) as parent comes
- * from IRQCHIP_DECLARE(), then use of_node_to_fwnode() to get
- * the fwnode. For now we need this boardfile style code.
- */
- if (np) {
- struct device_node *irq_parent;
-
- irq_parent = of_irq_find_parent(np);
- if (!irq_parent) {
- dev_err(dev, "no IRQ parent node\n");
- return -ENODEV;
- }
- parent = irq_find_host(irq_parent);
- if (!parent) {
- dev_err(dev, "no IRQ parent domain\n");
- return -ENODEV;
- }
- g->fwnode = of_node_to_fwnode(np);
- } else {
- parent = ixp4xx_get_irq_domain();
- g->fwnode = irq_domain_alloc_fwnode(g->base);
- if (!g->fwnode) {
- dev_err(dev, "no domain base\n");
- return -ENODEV;
- }
- }
- g->domain = irq_domain_create_hierarchy(parent,
- IRQ_DOMAIN_FLAG_HIERARCHY,
- ARRAY_SIZE(ixp4xx_gpiomap),
- g->fwnode,
- &ixp4xx_gpio_irqdomain_ops,
- g);
- if (!g->domain) {
- irq_domain_free_fwnode(g->fwnode);
- dev_err(dev, "no hierarchical irq domain\n");
- return ret;
- }
-
- /*
- * After adding OF support, this is no longer needed: irqs
- * will be allocated for the respective fwnodes.
- */
- if (!np) {
- for (i = 0; i < ARRAY_SIZE(ixp4xx_gpiomap); i++) {
- const struct ixp4xx_gpio_map *map = &ixp4xx_gpiomap[i];
- struct irq_fwspec fwspec;
-
- fwspec.fwnode = g->fwnode;
- /* This is the hwirq for the GPIO line side of things */
- fwspec.param[0] = map->gpio_offset;
- fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
- fwspec.param_count = 2;
- ret = __irq_domain_alloc_irqs(g->domain,
- -1, /* just pick something */
- 1,
- NUMA_NO_NODE,
- &fwspec,
- false,
- NULL);
- if (ret < 0) {
- irq_domain_free_fwnode(g->fwnode);
- dev_err(dev,
- "can not allocate irq for GPIO line %d parent hwirq %d in hierarchy domain: %d\n",
- map->gpio_offset, map->parent_hwirq,
- ret);
- return ret;
- }
- }
- }
-
platform_set_drvdata(pdev, g);
- dev_info(dev, "IXP4 GPIO @%p registered\n", g->base);
+ dev_info(dev, "IXP4 GPIO registered\n");
return 0;
}
diff --git a/drivers/gpio/gpio-janz-ttl.c b/drivers/gpio/gpio-janz-ttl.c
index 6b5b5a8b9173..cdf50e4ea165 100644
--- a/drivers/gpio/gpio-janz-ttl.c
+++ b/drivers/gpio/gpio-janz-ttl.c
@@ -140,18 +140,17 @@ static void ttl_setup_device(struct ttl_module *mod)
static int ttl_probe(struct platform_device *pdev)
{
struct janz_platform_data *pdata;
- struct device *dev = &pdev->dev;
struct ttl_module *mod;
struct gpio_chip *gpio;
int ret;
pdata = dev_get_platdata(&pdev->dev);
if (!pdata) {
- dev_err(dev, "no platform data\n");
+ dev_err(&pdev->dev, "no platform data\n");
return -ENXIO;
}
- mod = devm_kzalloc(dev, sizeof(*mod), GFP_KERNEL);
+ mod = devm_kzalloc(&pdev->dev, sizeof(*mod), GFP_KERNEL);
if (!mod)
return -ENOMEM;
@@ -177,9 +176,9 @@ static int ttl_probe(struct platform_device *pdev)
gpio->base = -1;
gpio->ngpio = 20;
- ret = devm_gpiochip_add_data(dev, gpio, NULL);
+ ret = devm_gpiochip_add_data(&pdev->dev, gpio, NULL);
if (ret) {
- dev_err(dev, "unable to add GPIO chip\n");
+ dev_err(&pdev->dev, "unable to add GPIO chip\n");
return ret;
}
diff --git a/drivers/gpio/gpio-ks8695.c b/drivers/gpio/gpio-ks8695.c
deleted file mode 100644
index a0f87c124894..000000000000
--- a/drivers/gpio/gpio-ks8695.c
+++ /dev/null
@@ -1,284 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * arch/arm/mach-ks8695/gpio.c
- *
- * Copyright (C) 2006 Andrew Victor
- * Updated to GPIOLIB, Copyright 2008 Simtec Electronics
- * Daniel Silverstone <dsilvers@simtec.co.uk>
- */
-#include <linux/gpio/driver.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/init.h>
-#include <linux/debugfs.h>
-#include <linux/seq_file.h>
-#include <linux/module.h>
-#include <linux/io.h>
-
-#include <mach/hardware.h>
-#include <asm/mach/irq.h>
-
-#include <mach/regs-gpio.h>
-#include <mach/gpio-ks8695.h>
-
-/*
- * Configure a GPIO line for either GPIO function, or its internal
- * function (Interrupt, Timer, etc).
- */
-static void ks8695_gpio_mode(unsigned int pin, short gpio)
-{
- unsigned int enable[] = { IOPC_IOEINT0EN, IOPC_IOEINT1EN, IOPC_IOEINT2EN, IOPC_IOEINT3EN, IOPC_IOTIM0EN, IOPC_IOTIM1EN };
- unsigned long x, flags;
-
- if (pin > KS8695_GPIO_5) /* only GPIO 0..5 have internal functions */
- return;
-
- local_irq_save(flags);
-
- x = __raw_readl(KS8695_GPIO_VA + KS8695_IOPC);
- if (gpio) /* GPIO: set bit to 0 */
- x &= ~enable[pin];
- else /* Internal function: set bit to 1 */
- x |= enable[pin];
- __raw_writel(x, KS8695_GPIO_VA + KS8695_IOPC);
-
- local_irq_restore(flags);
-}
-
-
-static unsigned short gpio_irq[] = { KS8695_IRQ_EXTERN0, KS8695_IRQ_EXTERN1, KS8695_IRQ_EXTERN2, KS8695_IRQ_EXTERN3 };
-
-/*
- * Configure GPIO pin as external interrupt source.
- */
-int ks8695_gpio_interrupt(unsigned int pin, unsigned int type)
-{
- unsigned long x, flags;
-
- if (pin > KS8695_GPIO_3) /* only GPIO 0..3 can generate IRQ */
- return -EINVAL;
-
- local_irq_save(flags);
-
- /* set pin as input */
- x = __raw_readl(KS8695_GPIO_VA + KS8695_IOPM);
- x &= ~IOPM(pin);
- __raw_writel(x, KS8695_GPIO_VA + KS8695_IOPM);
-
- local_irq_restore(flags);
-
- /* Set IRQ triggering type */
- irq_set_irq_type(gpio_irq[pin], type);
-
- /* enable interrupt mode */
- ks8695_gpio_mode(pin, 0);
-
- return 0;
-}
-EXPORT_SYMBOL(ks8695_gpio_interrupt);
-
-
-
-/* .... Generic GPIO interface .............................................. */
-
-/*
- * Configure the GPIO line as an input.
- */
-static int ks8695_gpio_direction_input(struct gpio_chip *gc, unsigned int pin)
-{
- unsigned long x, flags;
-
- if (pin > KS8695_GPIO_15)
- return -EINVAL;
-
- /* set pin to GPIO mode */
- ks8695_gpio_mode(pin, 1);
-
- local_irq_save(flags);
-
- /* set pin as input */
- x = __raw_readl(KS8695_GPIO_VA + KS8695_IOPM);
- x &= ~IOPM(pin);
- __raw_writel(x, KS8695_GPIO_VA + KS8695_IOPM);
-
- local_irq_restore(flags);
-
- return 0;
-}
-
-
-/*
- * Configure the GPIO line as an output, with default state.
- */
-static int ks8695_gpio_direction_output(struct gpio_chip *gc,
- unsigned int pin, int state)
-{
- unsigned long x, flags;
-
- if (pin > KS8695_GPIO_15)
- return -EINVAL;
-
- /* set pin to GPIO mode */
- ks8695_gpio_mode(pin, 1);
-
- local_irq_save(flags);
-
- /* set line state */
- x = __raw_readl(KS8695_GPIO_VA + KS8695_IOPD);
- if (state)
- x |= IOPD(pin);
- else
- x &= ~IOPD(pin);
- __raw_writel(x, KS8695_GPIO_VA + KS8695_IOPD);
-
- /* set pin as output */
- x = __raw_readl(KS8695_GPIO_VA + KS8695_IOPM);
- x |= IOPM(pin);
- __raw_writel(x, KS8695_GPIO_VA + KS8695_IOPM);
-
- local_irq_restore(flags);
-
- return 0;
-}
-
-
-/*
- * Set the state of an output GPIO line.
- */
-static void ks8695_gpio_set_value(struct gpio_chip *gc,
- unsigned int pin, int state)
-{
- unsigned long x, flags;
-
- if (pin > KS8695_GPIO_15)
- return;
-
- local_irq_save(flags);
-
- /* set output line state */
- x = __raw_readl(KS8695_GPIO_VA + KS8695_IOPD);
- if (state)
- x |= IOPD(pin);
- else
- x &= ~IOPD(pin);
- __raw_writel(x, KS8695_GPIO_VA + KS8695_IOPD);
-
- local_irq_restore(flags);
-}
-
-
-/*
- * Read the state of a GPIO line.
- */
-static int ks8695_gpio_get_value(struct gpio_chip *gc, unsigned int pin)
-{
- unsigned long x;
-
- if (pin > KS8695_GPIO_15)
- return -EINVAL;
-
- x = __raw_readl(KS8695_GPIO_VA + KS8695_IOPD);
- return (x & IOPD(pin)) != 0;
-}
-
-
-/*
- * Map GPIO line to IRQ number.
- */
-static int ks8695_gpio_to_irq(struct gpio_chip *gc, unsigned int pin)
-{
- if (pin > KS8695_GPIO_3) /* only GPIO 0..3 can generate IRQ */
- return -EINVAL;
-
- return gpio_irq[pin];
-}
-
-/* GPIOLIB interface */
-
-static struct gpio_chip ks8695_gpio_chip = {
- .label = "KS8695",
- .direction_input = ks8695_gpio_direction_input,
- .direction_output = ks8695_gpio_direction_output,
- .get = ks8695_gpio_get_value,
- .set = ks8695_gpio_set_value,
- .to_irq = ks8695_gpio_to_irq,
- .base = 0,
- .ngpio = 16,
- .can_sleep = false,
-};
-
-/* Register the GPIOs */
-void ks8695_register_gpios(void)
-{
- if (gpiochip_add_data(&ks8695_gpio_chip, NULL))
- printk(KERN_ERR "Unable to register core GPIOs\n");
-}
-
-/* .... Debug interface ..................................................... */
-
-#ifdef CONFIG_DEBUG_FS
-
-static int ks8695_gpio_show(struct seq_file *s, void *unused)
-{
- unsigned int enable[] = { IOPC_IOEINT0EN, IOPC_IOEINT1EN, IOPC_IOEINT2EN, IOPC_IOEINT3EN, IOPC_IOTIM0EN, IOPC_IOTIM1EN };
- unsigned int intmask[] = { IOPC_IOEINT0TM, IOPC_IOEINT1TM, IOPC_IOEINT2TM, IOPC_IOEINT3TM };
- unsigned long mode, ctrl, data;
- int i;
-
- mode = __raw_readl(KS8695_GPIO_VA + KS8695_IOPM);
- ctrl = __raw_readl(KS8695_GPIO_VA + KS8695_IOPC);
- data = __raw_readl(KS8695_GPIO_VA + KS8695_IOPD);
-
- seq_printf(s, "Pin\tI/O\tFunction\tState\n\n");
-
- for (i = KS8695_GPIO_0; i <= KS8695_GPIO_15 ; i++) {
- seq_printf(s, "%i:\t", i);
-
- seq_printf(s, "%s\t", (mode & IOPM(i)) ? "Output" : "Input");
-
- if (i <= KS8695_GPIO_3) {
- if (ctrl & enable[i]) {
- seq_printf(s, "EXT%i ", i);
-
- switch ((ctrl & intmask[i]) >> (4 * i)) {
- case IOPC_TM_LOW:
- seq_printf(s, "(Low)"); break;
- case IOPC_TM_HIGH:
- seq_printf(s, "(High)"); break;
- case IOPC_TM_RISING:
- seq_printf(s, "(Rising)"); break;
- case IOPC_TM_FALLING:
- seq_printf(s, "(Falling)"); break;
- case IOPC_TM_EDGE:
- seq_printf(s, "(Edges)"); break;
- }
- } else
- seq_printf(s, "GPIO\t");
- } else if (i <= KS8695_GPIO_5) {
- if (ctrl & enable[i])
- seq_printf(s, "TOUT%i\t", i - KS8695_GPIO_4);
- else
- seq_printf(s, "GPIO\t");
- } else {
- seq_printf(s, "GPIO\t");
- }
-
- seq_printf(s, "\t");
-
- seq_printf(s, "%i\n", (data & IOPD(i)) ? 1 : 0);
- }
- return 0;
-}
-
-DEFINE_SHOW_ATTRIBUTE(ks8695_gpio);
-
-static int __init ks8695_gpio_debugfs_init(void)
-{
- /* /sys/kernel/debug/ks8695_gpio */
- debugfs_create_file("ks8695_gpio", S_IFREG | S_IRUGO, NULL, NULL,
- &ks8695_gpio_fops);
- return 0;
-}
-postcore_initcall(ks8695_gpio_debugfs_init);
-
-#endif
diff --git a/drivers/gpio/gpio-lpc32xx.c b/drivers/gpio/gpio-lpc32xx.c
index 24885b3db3d5..4e626c4235c2 100644
--- a/drivers/gpio/gpio-lpc32xx.c
+++ b/drivers/gpio/gpio-lpc32xx.c
@@ -16,36 +16,33 @@
#include <linux/platform_device.h>
#include <linux/module.h>
-#include <mach/hardware.h>
-#include <mach/platform.h>
-
-#define LPC32XX_GPIO_P3_INP_STATE _GPREG(0x000)
-#define LPC32XX_GPIO_P3_OUTP_SET _GPREG(0x004)
-#define LPC32XX_GPIO_P3_OUTP_CLR _GPREG(0x008)
-#define LPC32XX_GPIO_P3_OUTP_STATE _GPREG(0x00C)
-#define LPC32XX_GPIO_P2_DIR_SET _GPREG(0x010)
-#define LPC32XX_GPIO_P2_DIR_CLR _GPREG(0x014)
-#define LPC32XX_GPIO_P2_DIR_STATE _GPREG(0x018)
-#define LPC32XX_GPIO_P2_INP_STATE _GPREG(0x01C)
-#define LPC32XX_GPIO_P2_OUTP_SET _GPREG(0x020)
-#define LPC32XX_GPIO_P2_OUTP_CLR _GPREG(0x024)
-#define LPC32XX_GPIO_P2_MUX_SET _GPREG(0x028)
-#define LPC32XX_GPIO_P2_MUX_CLR _GPREG(0x02C)
-#define LPC32XX_GPIO_P2_MUX_STATE _GPREG(0x030)
-#define LPC32XX_GPIO_P0_INP_STATE _GPREG(0x040)
-#define LPC32XX_GPIO_P0_OUTP_SET _GPREG(0x044)
-#define LPC32XX_GPIO_P0_OUTP_CLR _GPREG(0x048)
-#define LPC32XX_GPIO_P0_OUTP_STATE _GPREG(0x04C)
-#define LPC32XX_GPIO_P0_DIR_SET _GPREG(0x050)
-#define LPC32XX_GPIO_P0_DIR_CLR _GPREG(0x054)
-#define LPC32XX_GPIO_P0_DIR_STATE _GPREG(0x058)
-#define LPC32XX_GPIO_P1_INP_STATE _GPREG(0x060)
-#define LPC32XX_GPIO_P1_OUTP_SET _GPREG(0x064)
-#define LPC32XX_GPIO_P1_OUTP_CLR _GPREG(0x068)
-#define LPC32XX_GPIO_P1_OUTP_STATE _GPREG(0x06C)
-#define LPC32XX_GPIO_P1_DIR_SET _GPREG(0x070)
-#define LPC32XX_GPIO_P1_DIR_CLR _GPREG(0x074)
-#define LPC32XX_GPIO_P1_DIR_STATE _GPREG(0x078)
+#define LPC32XX_GPIO_P3_INP_STATE (0x000)
+#define LPC32XX_GPIO_P3_OUTP_SET (0x004)
+#define LPC32XX_GPIO_P3_OUTP_CLR (0x008)
+#define LPC32XX_GPIO_P3_OUTP_STATE (0x00C)
+#define LPC32XX_GPIO_P2_DIR_SET (0x010)
+#define LPC32XX_GPIO_P2_DIR_CLR (0x014)
+#define LPC32XX_GPIO_P2_DIR_STATE (0x018)
+#define LPC32XX_GPIO_P2_INP_STATE (0x01C)
+#define LPC32XX_GPIO_P2_OUTP_SET (0x020)
+#define LPC32XX_GPIO_P2_OUTP_CLR (0x024)
+#define LPC32XX_GPIO_P2_MUX_SET (0x028)
+#define LPC32XX_GPIO_P2_MUX_CLR (0x02C)
+#define LPC32XX_GPIO_P2_MUX_STATE (0x030)
+#define LPC32XX_GPIO_P0_INP_STATE (0x040)
+#define LPC32XX_GPIO_P0_OUTP_SET (0x044)
+#define LPC32XX_GPIO_P0_OUTP_CLR (0x048)
+#define LPC32XX_GPIO_P0_OUTP_STATE (0x04C)
+#define LPC32XX_GPIO_P0_DIR_SET (0x050)
+#define LPC32XX_GPIO_P0_DIR_CLR (0x054)
+#define LPC32XX_GPIO_P0_DIR_STATE (0x058)
+#define LPC32XX_GPIO_P1_INP_STATE (0x060)
+#define LPC32XX_GPIO_P1_OUTP_SET (0x064)
+#define LPC32XX_GPIO_P1_OUTP_CLR (0x068)
+#define LPC32XX_GPIO_P1_OUTP_STATE (0x06C)
+#define LPC32XX_GPIO_P1_DIR_SET (0x070)
+#define LPC32XX_GPIO_P1_DIR_CLR (0x074)
+#define LPC32XX_GPIO_P1_DIR_STATE (0x078)
#define GPIO012_PIN_TO_BIT(x) (1 << (x))
#define GPIO3_PIN_TO_BIT(x) (1 << ((x) + 25))
@@ -72,12 +69,12 @@
#define LPC32XX_GPO_P3_GRP (LPC32XX_GPI_P3_GRP + LPC32XX_GPI_P3_MAX)
struct gpio_regs {
- void __iomem *inp_state;
- void __iomem *outp_state;
- void __iomem *outp_set;
- void __iomem *outp_clr;
- void __iomem *dir_set;
- void __iomem *dir_clr;
+ unsigned long inp_state;
+ unsigned long outp_state;
+ unsigned long outp_set;
+ unsigned long outp_clr;
+ unsigned long dir_set;
+ unsigned long dir_clr;
};
/*
@@ -165,16 +162,27 @@ static struct gpio_regs gpio_grp_regs_p3 = {
struct lpc32xx_gpio_chip {
struct gpio_chip chip;
struct gpio_regs *gpio_grp;
+ void __iomem *reg_base;
};
+static inline u32 gpreg_read(struct lpc32xx_gpio_chip *group, unsigned long offset)
+{
+ return __raw_readl(group->reg_base + offset);
+}
+
+static inline void gpreg_write(struct lpc32xx_gpio_chip *group, u32 val, unsigned long offset)
+{
+ __raw_writel(val, group->reg_base + offset);
+}
+
static void __set_gpio_dir_p012(struct lpc32xx_gpio_chip *group,
unsigned pin, int input)
{
if (input)
- __raw_writel(GPIO012_PIN_TO_BIT(pin),
+ gpreg_write(group, GPIO012_PIN_TO_BIT(pin),
group->gpio_grp->dir_clr);
else
- __raw_writel(GPIO012_PIN_TO_BIT(pin),
+ gpreg_write(group, GPIO012_PIN_TO_BIT(pin),
group->gpio_grp->dir_set);
}
@@ -184,19 +192,19 @@ static void __set_gpio_dir_p3(struct lpc32xx_gpio_chip *group,
u32 u = GPIO3_PIN_TO_BIT(pin);
if (input)
- __raw_writel(u, group->gpio_grp->dir_clr);
+ gpreg_write(group, u, group->gpio_grp->dir_clr);
else
- __raw_writel(u, group->gpio_grp->dir_set);
+ gpreg_write(group, u, group->gpio_grp->dir_set);
}
static void __set_gpio_level_p012(struct lpc32xx_gpio_chip *group,
unsigned pin, int high)
{
if (high)
- __raw_writel(GPIO012_PIN_TO_BIT(pin),
+ gpreg_write(group, GPIO012_PIN_TO_BIT(pin),
group->gpio_grp->outp_set);
else
- __raw_writel(GPIO012_PIN_TO_BIT(pin),
+ gpreg_write(group, GPIO012_PIN_TO_BIT(pin),
group->gpio_grp->outp_clr);
}
@@ -206,31 +214,31 @@ static void __set_gpio_level_p3(struct lpc32xx_gpio_chip *group,
u32 u = GPIO3_PIN_TO_BIT(pin);
if (high)
- __raw_writel(u, group->gpio_grp->outp_set);
+ gpreg_write(group, u, group->gpio_grp->outp_set);
else
- __raw_writel(u, group->gpio_grp->outp_clr);
+ gpreg_write(group, u, group->gpio_grp->outp_clr);
}
static void __set_gpo_level_p3(struct lpc32xx_gpio_chip *group,
unsigned pin, int high)
{
if (high)
- __raw_writel(GPO3_PIN_TO_BIT(pin), group->gpio_grp->outp_set);
+ gpreg_write(group, GPO3_PIN_TO_BIT(pin), group->gpio_grp->outp_set);
else
- __raw_writel(GPO3_PIN_TO_BIT(pin), group->gpio_grp->outp_clr);
+ gpreg_write(group, GPO3_PIN_TO_BIT(pin), group->gpio_grp->outp_clr);
}
static int __get_gpio_state_p012(struct lpc32xx_gpio_chip *group,
unsigned pin)
{
- return GPIO012_PIN_IN_SEL(__raw_readl(group->gpio_grp->inp_state),
+ return GPIO012_PIN_IN_SEL(gpreg_read(group, group->gpio_grp->inp_state),
pin);
}
static int __get_gpio_state_p3(struct lpc32xx_gpio_chip *group,
unsigned pin)
{
- int state = __raw_readl(group->gpio_grp->inp_state);
+ int state = gpreg_read(group, group->gpio_grp->inp_state);
/*
* P3 GPIO pin input mapping is not contiguous, GPIOP3-0..4 is mapped
@@ -242,13 +250,13 @@ static int __get_gpio_state_p3(struct lpc32xx_gpio_chip *group,
static int __get_gpi_state_p3(struct lpc32xx_gpio_chip *group,
unsigned pin)
{
- return GPI3_PIN_IN_SEL(__raw_readl(group->gpio_grp->inp_state), pin);
+ return GPI3_PIN_IN_SEL(gpreg_read(group, group->gpio_grp->inp_state), pin);
}
static int __get_gpo_state_p3(struct lpc32xx_gpio_chip *group,
unsigned pin)
{
- return GPO3_PIN_IN_SEL(__raw_readl(group->gpio_grp->outp_state), pin);
+ return GPO3_PIN_IN_SEL(gpreg_read(group, group->gpio_grp->outp_state), pin);
}
/*
@@ -497,12 +505,18 @@ static int lpc32xx_of_xlate(struct gpio_chip *gc,
static int lpc32xx_gpio_probe(struct platform_device *pdev)
{
int i;
+ void __iomem *reg_base;
+
+ reg_base = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(reg_base))
+ return PTR_ERR(reg_base);
for (i = 0; i < ARRAY_SIZE(lpc32xx_gpiochip); i++) {
if (pdev->dev.of_node) {
lpc32xx_gpiochip[i].chip.of_xlate = lpc32xx_of_xlate;
lpc32xx_gpiochip[i].chip.of_gpio_n_cells = 3;
lpc32xx_gpiochip[i].chip.of_node = pdev->dev.of_node;
+ lpc32xx_gpiochip[i].reg_base = reg_base;
}
devm_gpiochip_add_data(&pdev->dev, &lpc32xx_gpiochip[i].chip,
&lpc32xx_gpiochip[i]);
@@ -527,3 +541,7 @@ static struct platform_driver lpc32xx_gpio_driver = {
};
module_platform_driver(lpc32xx_gpio_driver);
+
+MODULE_AUTHOR("Kevin Wells <kevin.wells@nxp.com>");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("GPIO driver for LPC32xx SoC");
diff --git a/drivers/gpio/gpio-lynxpoint.c b/drivers/gpio/gpio-lynxpoint.c
index 31b4a091ab60..6bb9741ad036 100644
--- a/drivers/gpio/gpio-lynxpoint.c
+++ b/drivers/gpio/gpio-lynxpoint.c
@@ -358,25 +358,30 @@ static int lp_gpio_probe(struct platform_device *pdev)
gc->can_sleep = false;
gc->parent = dev;
- ret = devm_gpiochip_add_data(dev, gc, lg);
- if (ret) {
- dev_err(dev, "failed adding lp-gpio chip\n");
- return ret;
- }
-
/* set up interrupts */
if (irq_rc && irq_rc->start) {
+ struct gpio_irq_chip *girq;
+
+ girq = &gc->irq;
+ girq->chip = &lp_irqchip;
+ girq->parent_handler = lp_gpio_irq_handler;
+ girq->num_parents = 1;
+ girq->parents = devm_kcalloc(&pdev->dev, girq->num_parents,
+ sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents)
+ return -ENOMEM;
+ girq->parents[0] = (unsigned)irq_rc->start;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_simple_irq;
+
lp_gpio_irq_init_hw(lg);
- ret = gpiochip_irqchip_add(gc, &lp_irqchip, 0,
- handle_simple_irq, IRQ_TYPE_NONE);
- if (ret) {
- dev_err(dev, "failed to add irqchip\n");
- return ret;
- }
+ }
- gpiochip_set_chained_irqchip(gc, &lp_irqchip,
- (unsigned)irq_rc->start,
- lp_gpio_irq_handler);
+ ret = devm_gpiochip_add_data(dev, gc, lg);
+ if (ret) {
+ dev_err(dev, "failed adding lp-gpio chip\n");
+ return ret;
}
pm_runtime_enable(dev);
diff --git a/drivers/gpio/gpio-madera.c b/drivers/gpio/gpio-madera.c
index c9dad0543672..7086f8b5388f 100644
--- a/drivers/gpio/gpio-madera.c
+++ b/drivers/gpio/gpio-madera.c
@@ -1,12 +1,8 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0-only
/*
* GPIO support for Cirrus Logic Madera codecs
*
* Copyright (C) 2015-2018 Cirrus Logic
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by the
- * Free Software Foundation; version 2.
*/
#include <linux/gpio/driver.h>
@@ -124,7 +120,7 @@ static const struct gpio_chip madera_gpio_chip = {
static int madera_gpio_probe(struct platform_device *pdev)
{
struct madera *madera = dev_get_drvdata(pdev->dev.parent);
- struct madera_pdata *pdata = dev_get_platdata(madera->dev);
+ struct madera_pdata *pdata = &madera->pdata;
struct madera_gpio *madera_gpio;
int ret;
@@ -140,6 +136,9 @@ static int madera_gpio_probe(struct platform_device *pdev)
madera_gpio->gpio_chip.parent = pdev->dev.parent;
switch (madera->type) {
+ case CS47L15:
+ madera_gpio->gpio_chip.ngpio = CS47L15_NUM_GPIOS;
+ break;
case CS47L35:
madera_gpio->gpio_chip.ngpio = CS47L35_NUM_GPIOS;
break;
@@ -151,13 +150,18 @@ static int madera_gpio_probe(struct platform_device *pdev)
case CS47L91:
madera_gpio->gpio_chip.ngpio = CS47L90_NUM_GPIOS;
break;
+ case CS42L92:
+ case CS47L92:
+ case CS47L93:
+ madera_gpio->gpio_chip.ngpio = CS47L92_NUM_GPIOS;
+ break;
default:
dev_err(&pdev->dev, "Unknown chip variant %d\n", madera->type);
return -EINVAL;
}
/* We want to be usable on systems that don't use devicetree or acpi */
- if (pdata && pdata->gpio_base)
+ if (pdata->gpio_base)
madera_gpio->gpio_chip.base = pdata->gpio_base;
else
madera_gpio->gpio_chip.base = -1;
diff --git a/drivers/gpio/gpio-max732x.c b/drivers/gpio/gpio-max732x.c
index 5e4102e7b1f9..5fb0bcf31142 100644
--- a/drivers/gpio/gpio-max732x.c
+++ b/drivers/gpio/gpio-max732x.c
@@ -649,12 +649,12 @@ static int max732x_probe(struct i2c_client *client,
case 0x60:
chip->client_group_a = client;
if (nr_port > 8) {
- c = i2c_new_dummy(client->adapter, addr_b);
- if (!c) {
+ c = devm_i2c_new_dummy_device(&client->dev,
+ client->adapter, addr_b);
+ if (IS_ERR(c)) {
dev_err(&client->dev,
"Failed to allocate I2C device\n");
- ret = -ENODEV;
- goto out_failed;
+ return PTR_ERR(c);
}
chip->client_group_b = chip->client_dummy = c;
}
@@ -662,12 +662,12 @@ static int max732x_probe(struct i2c_client *client,
case 0x50:
chip->client_group_b = client;
if (nr_port > 8) {
- c = i2c_new_dummy(client->adapter, addr_a);
- if (!c) {
+ c = devm_i2c_new_dummy_device(&client->dev,
+ client->adapter, addr_a);
+ if (IS_ERR(c)) {
dev_err(&client->dev,
"Failed to allocate I2C device\n");
- ret = -ENODEV;
- goto out_failed;
+ return PTR_ERR(c);
}
chip->client_group_a = chip->client_dummy = c;
}
@@ -675,37 +675,33 @@ static int max732x_probe(struct i2c_client *client,
default:
dev_err(&client->dev, "invalid I2C address specified %02x\n",
client->addr);
- ret = -EINVAL;
- goto out_failed;
+ return -EINVAL;
}
if (nr_port > 8 && !chip->client_dummy) {
dev_err(&client->dev,
"Failed to allocate second group I2C device\n");
- ret = -ENODEV;
- goto out_failed;
+ return -ENODEV;
}
mutex_init(&chip->lock);
ret = max732x_readb(chip, is_group_a(chip, 0), &chip->reg_out[0]);
if (ret)
- goto out_failed;
+ return ret;
if (nr_port > 8) {
ret = max732x_readb(chip, is_group_a(chip, 8), &chip->reg_out[1]);
if (ret)
- goto out_failed;
+ return ret;
}
- ret = gpiochip_add_data(&chip->gpio_chip, chip);
+ ret = devm_gpiochip_add_data(&client->dev, &chip->gpio_chip, chip);
if (ret)
- goto out_failed;
+ return ret;
ret = max732x_irq_setup(chip, id);
- if (ret) {
- gpiochip_remove(&chip->gpio_chip);
- goto out_failed;
- }
+ if (ret)
+ return ret;
if (pdata && pdata->setup) {
ret = pdata->setup(client, chip->gpio_chip.base,
@@ -716,10 +712,6 @@ static int max732x_probe(struct i2c_client *client,
i2c_set_clientdata(client, chip);
return 0;
-
-out_failed:
- i2c_unregister_device(chip->client_dummy);
- return ret;
}
static int max732x_remove(struct i2c_client *client)
@@ -739,11 +731,6 @@ static int max732x_remove(struct i2c_client *client)
}
}
- gpiochip_remove(&chip->gpio_chip);
-
- /* unregister any dummy i2c_client */
- i2c_unregister_device(chip->client_dummy);
-
return 0;
}
diff --git a/drivers/gpio/gpio-max77620.c b/drivers/gpio/gpio-max77620.c
index b7d89e30131e..47d05e357e61 100644
--- a/drivers/gpio/gpio-max77620.c
+++ b/drivers/gpio/gpio-max77620.c
@@ -270,10 +270,8 @@ static int max77620_gpio_probe(struct platform_device *pdev)
int ret;
gpio_irq = platform_get_irq(pdev, 0);
- if (gpio_irq <= 0) {
- dev_err(&pdev->dev, "GPIO irq not available %d\n", gpio_irq);
+ if (gpio_irq <= 0)
return -ENODEV;
- }
mgpio = devm_kzalloc(&pdev->dev, sizeof(*mgpio), GFP_KERNEL);
if (!mgpio)
diff --git a/drivers/gpio/gpio-max77650.c b/drivers/gpio/gpio-max77650.c
index 3f03f4e8956c..3075f2513c6f 100644
--- a/drivers/gpio/gpio-max77650.c
+++ b/drivers/gpio/gpio-max77650.c
@@ -188,3 +188,4 @@ module_platform_driver(max77650_gpio_driver);
MODULE_DESCRIPTION("MAXIM 77650/77651 GPIO driver");
MODULE_AUTHOR("Bartosz Golaszewski <bgolaszewski@baylibre.com>");
MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:max77650-gpio");
diff --git a/drivers/gpio/gpio-mb86s7x.c b/drivers/gpio/gpio-mb86s7x.c
index 9bfff171f9fe..501e89548f53 100644
--- a/drivers/gpio/gpio-mb86s7x.c
+++ b/drivers/gpio/gpio-mb86s7x.c
@@ -6,6 +6,7 @@
* Copyright (C) 2015 Linaro Ltd.
*/
+#include <linux/acpi.h>
#include <linux/io.h>
#include <linux/init.h>
#include <linux/clk.h>
@@ -19,6 +20,9 @@
#include <linux/spinlock.h>
#include <linux/slab.h>
+#include "gpiolib.h"
+#include "gpiolib-acpi.h"
+
/*
* Only first 8bits of a register correspond to each pin,
* so there are 4 registers for 32 pins.
@@ -135,6 +139,20 @@ static void mb86s70_gpio_set(struct gpio_chip *gc, unsigned gpio, int value)
spin_unlock_irqrestore(&gchip->lock, flags);
}
+static int mb86s70_gpio_to_irq(struct gpio_chip *gc, unsigned int offset)
+{
+ int irq, index;
+
+ for (index = 0;; index++) {
+ irq = platform_get_irq(to_platform_device(gc->parent), index);
+ if (irq <= 0)
+ break;
+ if (irq_get_irq_data(irq)->hwirq == offset)
+ return irq;
+ }
+ return -EINVAL;
+}
+
static int mb86s70_gpio_probe(struct platform_device *pdev)
{
struct mb86s70_gpio_chip *gchip;
@@ -150,13 +168,15 @@ static int mb86s70_gpio_probe(struct platform_device *pdev)
if (IS_ERR(gchip->base))
return PTR_ERR(gchip->base);
- gchip->clk = devm_clk_get(&pdev->dev, NULL);
- if (IS_ERR(gchip->clk))
- return PTR_ERR(gchip->clk);
+ if (!has_acpi_companion(&pdev->dev)) {
+ gchip->clk = devm_clk_get(&pdev->dev, NULL);
+ if (IS_ERR(gchip->clk))
+ return PTR_ERR(gchip->clk);
- ret = clk_prepare_enable(gchip->clk);
- if (ret)
- return ret;
+ ret = clk_prepare_enable(gchip->clk);
+ if (ret)
+ return ret;
+ }
spin_lock_init(&gchip->lock);
@@ -172,19 +192,28 @@ static int mb86s70_gpio_probe(struct platform_device *pdev)
gchip->gc.parent = &pdev->dev;
gchip->gc.base = -1;
+ if (has_acpi_companion(&pdev->dev))
+ gchip->gc.to_irq = mb86s70_gpio_to_irq;
+
ret = gpiochip_add_data(&gchip->gc, gchip);
if (ret) {
dev_err(&pdev->dev, "couldn't register gpio driver\n");
clk_disable_unprepare(gchip->clk);
+ return ret;
}
- return ret;
+ if (has_acpi_companion(&pdev->dev))
+ acpi_gpiochip_request_interrupts(&gchip->gc);
+
+ return 0;
}
static int mb86s70_gpio_remove(struct platform_device *pdev)
{
struct mb86s70_gpio_chip *gchip = platform_get_drvdata(pdev);
+ if (has_acpi_companion(&pdev->dev))
+ acpi_gpiochip_free_interrupts(&gchip->gc);
gpiochip_remove(&gchip->gc);
clk_disable_unprepare(gchip->clk);
@@ -197,10 +226,19 @@ static const struct of_device_id mb86s70_gpio_dt_ids[] = {
};
MODULE_DEVICE_TABLE(of, mb86s70_gpio_dt_ids);
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id mb86s70_gpio_acpi_ids[] = {
+ { "SCX0007" },
+ { /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(acpi, mb86s70_gpio_acpi_ids);
+#endif
+
static struct platform_driver mb86s70_gpio_driver = {
.driver = {
.name = "mb86s70-gpio",
.of_match_table = mb86s70_gpio_dt_ids,
+ .acpi_match_table = ACPI_PTR(mb86s70_gpio_acpi_ids),
},
.probe = mb86s70_gpio_probe,
.remove = mb86s70_gpio_remove,
diff --git a/drivers/gpio/gpio-merrifield.c b/drivers/gpio/gpio-merrifield.c
index 3302125e5265..4f27ddfe1e2f 100644
--- a/drivers/gpio/gpio-merrifield.c
+++ b/drivers/gpio/gpio-merrifield.c
@@ -397,6 +397,7 @@ static int mrfld_gpio_probe(struct pci_dev *pdev, const struct pci_device_id *id
{
const struct mrfld_gpio_pinrange *range;
const char *pinctrl_dev_name;
+ struct gpio_irq_chip *girq;
struct mrfld_gpio *priv;
u32 gpio_base, irq_base;
void __iomem *base;
@@ -444,6 +445,21 @@ static int mrfld_gpio_probe(struct pci_dev *pdev, const struct pci_device_id *id
raw_spin_lock_init(&priv->lock);
+ girq = &priv->chip.irq;
+ girq->chip = &mrfld_irqchip;
+ girq->parent_handler = mrfld_irq_handler;
+ girq->num_parents = 1;
+ girq->parents = devm_kcalloc(&pdev->dev, girq->num_parents,
+ sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents)
+ return -ENOMEM;
+ girq->parents[0] = pdev->irq;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_bad_irq;
+
+ mrfld_irq_init_hw(priv);
+
pci_set_drvdata(pdev, priv);
retval = devm_gpiochip_add_data(&pdev->dev, &priv->chip, priv);
if (retval) {
@@ -465,18 +481,6 @@ static int mrfld_gpio_probe(struct pci_dev *pdev, const struct pci_device_id *id
}
}
- retval = gpiochip_irqchip_add(&priv->chip, &mrfld_irqchip, irq_base,
- handle_bad_irq, IRQ_TYPE_NONE);
- if (retval) {
- dev_err(&pdev->dev, "could not connect irqchip to gpiochip\n");
- return retval;
- }
-
- mrfld_irq_init_hw(priv);
-
- gpiochip_set_chained_irqchip(&priv->chip, &mrfld_irqchip, pdev->irq,
- mrfld_irq_handler);
-
return 0;
}
diff --git a/drivers/gpio/gpio-mockup.c b/drivers/gpio/gpio-mockup.c
index b6a4efce7c92..213aedc97dc2 100644
--- a/drivers/gpio/gpio-mockup.c
+++ b/drivers/gpio/gpio-mockup.c
@@ -309,13 +309,13 @@ static const struct file_operations gpio_mockup_debugfs_ops = {
.read = gpio_mockup_debugfs_read,
.write = gpio_mockup_debugfs_write,
.llseek = no_llseek,
+ .release = single_release,
};
static void gpio_mockup_debugfs_setup(struct device *dev,
struct gpio_mockup_chip *chip)
{
struct gpio_mockup_dbgfs_private *priv;
- struct dentry *evfile;
struct gpio_chip *gc;
const char *devname;
char *name;
@@ -325,32 +325,25 @@ static void gpio_mockup_debugfs_setup(struct device *dev,
devname = dev_name(&gc->gpiodev->dev);
chip->dbg_dir = debugfs_create_dir(devname, gpio_mockup_dbg_dir);
- if (IS_ERR_OR_NULL(chip->dbg_dir))
- goto err;
for (i = 0; i < gc->ngpio; i++) {
name = devm_kasprintf(dev, GFP_KERNEL, "%d", i);
if (!name)
- goto err;
+ return;
priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
if (!priv)
- goto err;
+ return;
priv->chip = chip;
priv->offset = i;
priv->desc = &gc->gpiodev->descs[i];
- evfile = debugfs_create_file(name, 0200, chip->dbg_dir, priv,
- &gpio_mockup_debugfs_ops);
- if (IS_ERR_OR_NULL(evfile))
- goto err;
+ debugfs_create_file(name, 0200, chip->dbg_dir, priv,
+ &gpio_mockup_debugfs_ops);
}
return;
-
-err:
- dev_err(dev, "error creating debugfs files\n");
}
static int gpio_mockup_name_lines(struct device *dev,
@@ -447,8 +440,7 @@ static int gpio_mockup_probe(struct platform_device *pdev)
if (rv)
return rv;
- if (!IS_ERR_OR_NULL(gpio_mockup_dbg_dir))
- gpio_mockup_debugfs_setup(dev, chip);
+ gpio_mockup_debugfs_setup(dev, chip);
return 0;
}
@@ -501,8 +493,6 @@ static int __init gpio_mockup_init(void)
}
gpio_mockup_dbg_dir = debugfs_create_dir("gpio-mockup", NULL);
- if (IS_ERR_OR_NULL(gpio_mockup_dbg_dir))
- gpio_mockup_err("error creating debugfs directory\n");
err = platform_driver_register(&gpio_mockup_driver);
if (err) {
diff --git a/drivers/gpio/gpio-moxtet.c b/drivers/gpio/gpio-moxtet.c
new file mode 100644
index 000000000000..3fd729994a38
--- /dev/null
+++ b/drivers/gpio/gpio-moxtet.c
@@ -0,0 +1,179 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Turris Mox Moxtet GPIO expander
+ *
+ * Copyright (C) 2018 Marek Behun <marek.behun@nic.cz>
+ */
+
+#include <linux/bitops.h>
+#include <linux/gpio/driver.h>
+#include <linux/moxtet.h>
+#include <linux/module.h>
+
+#define MOXTET_GPIO_NGPIOS 12
+#define MOXTET_GPIO_INPUTS 4
+
+struct moxtet_gpio_desc {
+ u16 in_mask;
+ u16 out_mask;
+};
+
+static const struct moxtet_gpio_desc descs[] = {
+ [TURRIS_MOX_MODULE_SFP] = {
+ .in_mask = GENMASK(2, 0),
+ .out_mask = GENMASK(5, 4),
+ },
+};
+
+struct moxtet_gpio_chip {
+ struct device *dev;
+ struct gpio_chip gpio_chip;
+ const struct moxtet_gpio_desc *desc;
+};
+
+static int moxtet_gpio_get_value(struct gpio_chip *gc, unsigned int offset)
+{
+ struct moxtet_gpio_chip *chip = gpiochip_get_data(gc);
+ int ret;
+
+ if (chip->desc->in_mask & BIT(offset)) {
+ ret = moxtet_device_read(chip->dev);
+ } else if (chip->desc->out_mask & BIT(offset)) {
+ ret = moxtet_device_written(chip->dev);
+ if (ret >= 0)
+ ret <<= MOXTET_GPIO_INPUTS;
+ } else {
+ return -EINVAL;
+ }
+
+ if (ret < 0)
+ return ret;
+
+ return !!(ret & BIT(offset));
+}
+
+static void moxtet_gpio_set_value(struct gpio_chip *gc, unsigned int offset,
+ int val)
+{
+ struct moxtet_gpio_chip *chip = gpiochip_get_data(gc);
+ int state;
+
+ state = moxtet_device_written(chip->dev);
+ if (state < 0)
+ return;
+
+ offset -= MOXTET_GPIO_INPUTS;
+
+ if (val)
+ state |= BIT(offset);
+ else
+ state &= ~BIT(offset);
+
+ moxtet_device_write(chip->dev, state);
+}
+
+static int moxtet_gpio_get_direction(struct gpio_chip *gc, unsigned int offset)
+{
+ struct moxtet_gpio_chip *chip = gpiochip_get_data(gc);
+
+ /* All lines are hard wired to be either input or output, not both. */
+ if (chip->desc->in_mask & BIT(offset))
+ return 1;
+ else if (chip->desc->out_mask & BIT(offset))
+ return 0;
+ else
+ return -EINVAL;
+}
+
+static int moxtet_gpio_direction_input(struct gpio_chip *gc,
+ unsigned int offset)
+{
+ struct moxtet_gpio_chip *chip = gpiochip_get_data(gc);
+
+ if (chip->desc->in_mask & BIT(offset))
+ return 0;
+ else if (chip->desc->out_mask & BIT(offset))
+ return -ENOTSUPP;
+ else
+ return -EINVAL;
+}
+
+static int moxtet_gpio_direction_output(struct gpio_chip *gc,
+ unsigned int offset, int val)
+{
+ struct moxtet_gpio_chip *chip = gpiochip_get_data(gc);
+
+ if (chip->desc->out_mask & BIT(offset))
+ moxtet_gpio_set_value(gc, offset, val);
+ else if (chip->desc->in_mask & BIT(offset))
+ return -ENOTSUPP;
+ else
+ return -EINVAL;
+
+ return 0;
+}
+
+static int moxtet_gpio_probe(struct device *dev)
+{
+ struct moxtet_gpio_chip *chip;
+ struct device_node *nc = dev->of_node;
+ int id;
+
+ id = to_moxtet_device(dev)->id;
+
+ if (id >= ARRAY_SIZE(descs)) {
+ dev_err(dev, "%pOF Moxtet device id 0x%x is not supported by gpio-moxtet driver\n",
+ nc, id);
+ return -ENOTSUPP;
+ }
+
+ chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL);
+ if (!chip)
+ return -ENOMEM;
+
+ chip->dev = dev;
+ chip->gpio_chip.parent = dev;
+ chip->desc = &descs[id];
+
+ dev_set_drvdata(dev, chip);
+
+ chip->gpio_chip.label = dev_name(dev);
+ chip->gpio_chip.get_direction = moxtet_gpio_get_direction;
+ chip->gpio_chip.direction_input = moxtet_gpio_direction_input;
+ chip->gpio_chip.direction_output = moxtet_gpio_direction_output;
+ chip->gpio_chip.get = moxtet_gpio_get_value;
+ chip->gpio_chip.set = moxtet_gpio_set_value;
+ chip->gpio_chip.base = -1;
+
+ chip->gpio_chip.ngpio = MOXTET_GPIO_NGPIOS;
+
+ chip->gpio_chip.can_sleep = true;
+ chip->gpio_chip.owner = THIS_MODULE;
+
+ return devm_gpiochip_add_data(dev, &chip->gpio_chip, chip);
+}
+
+static const struct of_device_id moxtet_gpio_dt_ids[] = {
+ { .compatible = "cznic,moxtet-gpio", },
+ {},
+};
+MODULE_DEVICE_TABLE(of, moxtet_gpio_dt_ids);
+
+static const enum turris_mox_module_id moxtet_gpio_module_table[] = {
+ TURRIS_MOX_MODULE_SFP,
+ 0,
+};
+
+static struct moxtet_driver moxtet_gpio_driver = {
+ .driver = {
+ .name = "moxtet-gpio",
+ .of_match_table = moxtet_gpio_dt_ids,
+ .probe = moxtet_gpio_probe,
+ },
+ .id_table = moxtet_gpio_module_table,
+};
+module_moxtet_driver(moxtet_gpio_driver);
+
+MODULE_AUTHOR("Marek Behun <marek.behun@nic.cz>");
+MODULE_DESCRIPTION("Turris Mox Moxtet GPIO expander");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpio/gpio-mpc8xxx.c b/drivers/gpio/gpio-mpc8xxx.c
index c8673a5d9412..16a47de29c94 100644
--- a/drivers/gpio/gpio-mpc8xxx.c
+++ b/drivers/gpio/gpio-mpc8xxx.c
@@ -32,6 +32,7 @@
#define GPIO_IMR 0x10
#define GPIO_ICR 0x14
#define GPIO_ICR2 0x18
+#define GPIO_IBE 0x18
struct mpc8xxx_gpio_chip {
struct gpio_chip gc;
@@ -45,6 +46,27 @@ struct mpc8xxx_gpio_chip {
unsigned int irqn;
};
+/* The GPIO Input Buffer Enable register(GPIO_IBE) is used to
+ * control the input enable of each individual GPIO port.
+ * When an individual GPIO port’s direction is set to
+ * input (GPIO_GPDIR[DRn=0]), the associated input enable must be
+ * set (GPIOxGPIE[IEn]=1) to propagate the port value to the GPIO
+ * Data Register.
+ */
+static int ls1028a_gpio_dir_in_init(struct gpio_chip *gc)
+{
+ unsigned long flags;
+ struct mpc8xxx_gpio_chip *mpc8xxx_gc = gpiochip_get_data(gc);
+
+ spin_lock_irqsave(&gc->bgpio_lock, flags);
+
+ gc->write_reg(mpc8xxx_gc->regs + GPIO_IBE, 0xffffffff);
+
+ spin_unlock_irqrestore(&gc->bgpio_lock, flags);
+
+ return 0;
+}
+
/*
* This hardware has a big endian bit assignment such that GPIO line 0 is
* connected to bit 31, line 1 to bit 30 ... line 31 to bit 0.
@@ -261,6 +283,7 @@ static const struct irq_domain_ops mpc8xxx_gpio_irq_ops = {
};
struct mpc8xxx_gpio_devtype {
+ int (*gpio_dir_in_init)(struct gpio_chip *chip);
int (*gpio_dir_out)(struct gpio_chip *, unsigned int, int);
int (*gpio_get)(struct gpio_chip *, unsigned int);
int (*irq_set_type)(struct irq_data *, unsigned int);
@@ -271,6 +294,10 @@ static const struct mpc8xxx_gpio_devtype mpc512x_gpio_devtype = {
.irq_set_type = mpc512x_irq_set_type,
};
+static const struct mpc8xxx_gpio_devtype ls1028a_gpio_devtype = {
+ .gpio_dir_in_init = ls1028a_gpio_dir_in_init,
+};
+
static const struct mpc8xxx_gpio_devtype mpc5125_gpio_devtype = {
.gpio_dir_out = mpc5125_gpio_dir_out,
.irq_set_type = mpc512x_irq_set_type,
@@ -291,6 +318,8 @@ static const struct of_device_id mpc8xxx_gpio_ids[] = {
{ .compatible = "fsl,mpc5121-gpio", .data = &mpc512x_gpio_devtype, },
{ .compatible = "fsl,mpc5125-gpio", .data = &mpc5125_gpio_devtype, },
{ .compatible = "fsl,pq3-gpio", },
+ { .compatible = "fsl,ls1028a-gpio", .data = &ls1028a_gpio_devtype, },
+ { .compatible = "fsl,ls1088a-gpio", .data = &ls1028a_gpio_devtype, },
{ .compatible = "fsl,qoriq-gpio", },
{}
};
@@ -376,6 +405,9 @@ static int mpc8xxx_probe(struct platform_device *pdev)
/* ack and mask all irqs */
gc->write_reg(mpc8xxx_gc->regs + GPIO_IER, 0xffffffff);
gc->write_reg(mpc8xxx_gc->regs + GPIO_IMR, 0);
+ /* enable input buffer */
+ if (devtype->gpio_dir_in_init)
+ devtype->gpio_dir_in_init(gc);
irq_set_chained_handler_and_data(mpc8xxx_gc->irqn,
mpc8xxx_gpio_irq_cascade, mpc8xxx_gc);
diff --git a/drivers/gpio/gpio-mt7621.c b/drivers/gpio/gpio-mt7621.c
index 79654fb2e50f..d1d785f983a7 100644
--- a/drivers/gpio/gpio-mt7621.c
+++ b/drivers/gpio/gpio-mt7621.c
@@ -241,13 +241,6 @@ mediatek_gpio_bank_probe(struct device *dev,
if (!rg->chip.label)
return -ENOMEM;
- ret = devm_gpiochip_add_data(dev, &rg->chip, mtk);
- if (ret < 0) {
- dev_err(dev, "Could not register gpio %d, ret=%d\n",
- rg->chip.ngpio, ret);
- return ret;
- }
-
rg->irq_chip.name = dev_name(dev);
rg->irq_chip.parent_device = dev;
rg->irq_chip.irq_unmask = mediatek_gpio_irq_unmask;
@@ -256,8 +249,10 @@ mediatek_gpio_bank_probe(struct device *dev,
rg->irq_chip.irq_set_type = mediatek_gpio_irq_type;
if (mtk->gpio_irq) {
+ struct gpio_irq_chip *girq;
+
/*
- * Manually request the irq here instead of passing
+ * Directly request the irq here instead of passing
* a flow-handler to gpiochip_set_chained_irqchip,
* because the irq is shared.
*/
@@ -271,15 +266,21 @@ mediatek_gpio_bank_probe(struct device *dev,
return ret;
}
- ret = gpiochip_irqchip_add(&rg->chip, &rg->irq_chip,
- 0, handle_simple_irq, IRQ_TYPE_NONE);
- if (ret) {
- dev_err(dev, "failed to add gpiochip_irqchip\n");
- return ret;
- }
+ girq = &rg->chip.irq;
+ girq->chip = &rg->irq_chip;
+ /* This will let us handle the parent IRQ in the driver */
+ girq->parent_handler = NULL;
+ girq->num_parents = 0;
+ girq->parents = NULL;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_simple_irq;
+ }
- gpiochip_set_chained_irqchip(&rg->chip, &rg->irq_chip,
- mtk->gpio_irq, NULL);
+ ret = devm_gpiochip_add_data(dev, &rg->chip, mtk);
+ if (ret < 0) {
+ dev_err(dev, "Could not register gpio %d, ret=%d\n",
+ rg->chip.ngpio, ret);
+ return ret;
}
/* set polarity to low for all gpios */
diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c
index 059094ac44cb..869d47f89599 100644
--- a/drivers/gpio/gpio-mvebu.c
+++ b/drivers/gpio/gpio-mvebu.c
@@ -38,6 +38,7 @@
#include <linux/err.h>
#include <linux/gpio/driver.h>
#include <linux/gpio/consumer.h>
+#include <linux/gpio/machine.h>
#include <linux/init.h>
#include <linux/io.h>
#include <linux/irq.h>
@@ -618,18 +619,14 @@ static int mvebu_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm)
ret = -EBUSY;
} else {
desc = gpiochip_request_own_desc(&mvchip->chip,
- pwm->hwpwm, "mvebu-pwm", 0);
+ pwm->hwpwm, "mvebu-pwm",
+ GPIO_ACTIVE_HIGH,
+ GPIOD_OUT_LOW);
if (IS_ERR(desc)) {
ret = PTR_ERR(desc);
goto out;
}
- ret = gpiod_direction_output(desc, 0);
- if (ret) {
- gpiochip_free_own_desc(desc);
- goto out;
- }
-
mvpwm->gpiod = desc;
}
out:
diff --git a/drivers/gpio/gpio-mxc.c b/drivers/gpio/gpio-mxc.c
index b2813580c582..7907a8755866 100644
--- a/drivers/gpio/gpio-mxc.c
+++ b/drivers/gpio/gpio-mxc.c
@@ -435,12 +435,9 @@ static int mxc_gpio_probe(struct platform_device *pdev)
return port->irq;
/* the controller clock is optional */
- port->clk = devm_clk_get(&pdev->dev, NULL);
- if (IS_ERR(port->clk)) {
- if (PTR_ERR(port->clk) == -EPROBE_DEFER)
- return -EPROBE_DEFER;
- port->clk = NULL;
- }
+ port->clk = devm_clk_get_optional(&pdev->dev, NULL);
+ if (IS_ERR(port->clk))
+ return PTR_ERR(port->clk);
err = clk_prepare_enable(port->clk);
if (err) {
diff --git a/drivers/gpio/gpio-omap.c b/drivers/gpio/gpio-omap.c
index 9276ef616430..d0f27084a942 100644
--- a/drivers/gpio/gpio-omap.c
+++ b/drivers/gpio/gpio-omap.c
@@ -44,8 +44,9 @@ struct gpio_regs {
};
struct gpio_bank {
- struct list_head node;
void __iomem *base;
+ const struct omap_gpio_reg_offs *regs;
+
int irq;
u32 non_wakeup_gpios;
u32 enabled_non_wakeup_gpios;
@@ -72,11 +73,7 @@ struct gpio_bank {
int context_loss_count;
void (*set_dataout)(struct gpio_bank *bank, unsigned gpio, int enable);
- void (*set_dataout_multiple)(struct gpio_bank *bank,
- unsigned long *mask, unsigned long *bits);
int (*get_context_loss_count)(struct device *dev);
-
- struct omap_gpio_reg_offs *regs;
};
#define GPIO_MOD_CTRL_BIT BIT(0)
@@ -92,20 +89,25 @@ static inline struct gpio_bank *omap_irq_data_get_bank(struct irq_data *d)
return gpiochip_get_data(chip);
}
-static void omap_set_gpio_direction(struct gpio_bank *bank, int gpio,
- int is_input)
+static inline u32 omap_gpio_rmw(void __iomem *reg, u32 mask, bool set)
{
- void __iomem *reg = bank->base;
- u32 l;
+ u32 val = readl_relaxed(reg);
- reg += bank->regs->direction;
- l = readl_relaxed(reg);
- if (is_input)
- l |= BIT(gpio);
+ if (set)
+ val |= mask;
else
- l &= ~(BIT(gpio));
- writel_relaxed(l, reg);
- bank->context.oe = l;
+ val &= ~mask;
+
+ writel_relaxed(val, reg);
+
+ return val;
+}
+
+static void omap_set_gpio_direction(struct gpio_bank *bank, int gpio,
+ int is_input)
+{
+ bank->context.oe = omap_gpio_rmw(bank->base + bank->regs->direction,
+ BIT(gpio), is_input);
}
@@ -131,88 +133,8 @@ static void omap_set_gpio_dataout_reg(struct gpio_bank *bank, unsigned offset,
static void omap_set_gpio_dataout_mask(struct gpio_bank *bank, unsigned offset,
int enable)
{
- void __iomem *reg = bank->base + bank->regs->dataout;
- u32 gpio_bit = BIT(offset);
- u32 l;
-
- l = readl_relaxed(reg);
- if (enable)
- l |= gpio_bit;
- else
- l &= ~gpio_bit;
- writel_relaxed(l, reg);
- bank->context.dataout = l;
-}
-
-static int omap_get_gpio_datain(struct gpio_bank *bank, int offset)
-{
- void __iomem *reg = bank->base + bank->regs->datain;
-
- return (readl_relaxed(reg) & (BIT(offset))) != 0;
-}
-
-static int omap_get_gpio_dataout(struct gpio_bank *bank, int offset)
-{
- void __iomem *reg = bank->base + bank->regs->dataout;
-
- return (readl_relaxed(reg) & (BIT(offset))) != 0;
-}
-
-/* set multiple data out values using dedicate set/clear register */
-static void omap_set_gpio_dataout_reg_multiple(struct gpio_bank *bank,
- unsigned long *mask,
- unsigned long *bits)
-{
- void __iomem *reg = bank->base;
- u32 l;
-
- l = *bits & *mask;
- writel_relaxed(l, reg + bank->regs->set_dataout);
- bank->context.dataout |= l;
-
- l = ~*bits & *mask;
- writel_relaxed(l, reg + bank->regs->clr_dataout);
- bank->context.dataout &= ~l;
-}
-
-/* set multiple data out values using mask register */
-static void omap_set_gpio_dataout_mask_multiple(struct gpio_bank *bank,
- unsigned long *mask,
- unsigned long *bits)
-{
- void __iomem *reg = bank->base + bank->regs->dataout;
- u32 l = (readl_relaxed(reg) & ~*mask) | (*bits & *mask);
-
- writel_relaxed(l, reg);
- bank->context.dataout = l;
-}
-
-static unsigned long omap_get_gpio_datain_multiple(struct gpio_bank *bank,
- unsigned long *mask)
-{
- void __iomem *reg = bank->base + bank->regs->datain;
-
- return readl_relaxed(reg) & *mask;
-}
-
-static unsigned long omap_get_gpio_dataout_multiple(struct gpio_bank *bank,
- unsigned long *mask)
-{
- void __iomem *reg = bank->base + bank->regs->dataout;
-
- return readl_relaxed(reg) & *mask;
-}
-
-static inline void omap_gpio_rmw(void __iomem *base, u32 reg, u32 mask, bool set)
-{
- int l = readl_relaxed(base + reg);
-
- if (set)
- l |= mask;
- else
- l &= ~mask;
-
- writel_relaxed(l, base + reg);
+ bank->context.dataout = omap_gpio_rmw(bank->base + bank->regs->dataout,
+ BIT(offset), enable);
}
static inline void omap_gpio_dbck_enable(struct gpio_bank *bank)
@@ -256,7 +178,6 @@ static inline void omap_gpio_dbck_disable(struct gpio_bank *bank)
static int omap2_set_gpio_debounce(struct gpio_bank *bank, unsigned offset,
unsigned debounce)
{
- void __iomem *reg;
u32 val;
u32 l;
bool enable = !!debounce;
@@ -273,19 +194,11 @@ static int omap2_set_gpio_debounce(struct gpio_bank *bank, unsigned offset,
l = BIT(offset);
clk_enable(bank->dbck);
- reg = bank->base + bank->regs->debounce;
- writel_relaxed(debounce, reg);
+ writel_relaxed(debounce, bank->base + bank->regs->debounce);
- reg = bank->base + bank->regs->debounce_en;
- val = readl_relaxed(reg);
-
- if (enable)
- val |= l;
- else
- val &= ~l;
+ val = omap_gpio_rmw(bank->base + bank->regs->debounce_en, l, enable);
bank->dbck_enable_mask = val;
- writel_relaxed(val, reg);
clk_disable(bank->dbck);
/*
* Enable debounce clock per module.
@@ -360,9 +273,9 @@ static inline void omap_set_gpio_trigger(struct gpio_bank *bank, int gpio,
void __iomem *base = bank->base;
u32 gpio_bit = BIT(gpio);
- omap_gpio_rmw(base, bank->regs->leveldetect0, gpio_bit,
+ omap_gpio_rmw(base + bank->regs->leveldetect0, gpio_bit,
trigger & IRQ_TYPE_LEVEL_LOW);
- omap_gpio_rmw(base, bank->regs->leveldetect1, gpio_bit,
+ omap_gpio_rmw(base + bank->regs->leveldetect1, gpio_bit,
trigger & IRQ_TYPE_LEVEL_HIGH);
/*
@@ -370,9 +283,9 @@ static inline void omap_set_gpio_trigger(struct gpio_bank *bank, int gpio,
* to be woken from idle state. Set the appropriate edge detection
* in addition to the level detection.
*/
- omap_gpio_rmw(base, bank->regs->risingdetect, gpio_bit,
+ omap_gpio_rmw(base + bank->regs->risingdetect, gpio_bit,
trigger & (IRQ_TYPE_EDGE_RISING | IRQ_TYPE_LEVEL_HIGH));
- omap_gpio_rmw(base, bank->regs->fallingdetect, gpio_bit,
+ omap_gpio_rmw(base + bank->regs->fallingdetect, gpio_bit,
trigger & (IRQ_TYPE_EDGE_FALLING | IRQ_TYPE_LEVEL_LOW));
bank->context.leveldetect0 =
@@ -384,11 +297,8 @@ static inline void omap_set_gpio_trigger(struct gpio_bank *bank, int gpio,
bank->context.fallingdetect =
readl_relaxed(bank->base + bank->regs->fallingdetect);
- if (likely(!(bank->non_wakeup_gpios & gpio_bit))) {
- omap_gpio_rmw(base, bank->regs->wkup_en, gpio_bit, trigger != 0);
- bank->context.wake_en =
- readl_relaxed(bank->base + bank->regs->wkup_en);
- }
+ bank->level_mask = bank->context.leveldetect0 |
+ bank->context.leveldetect1;
/* This part needs to be executed always for OMAP{34xx, 44xx} */
if (!bank->regs->irqctrl && !omap_gpio_is_off_wakeup_capable(bank, gpio)) {
@@ -403,44 +313,25 @@ static inline void omap_set_gpio_trigger(struct gpio_bank *bank, int gpio,
else
bank->enabled_non_wakeup_gpios &= ~gpio_bit;
}
-
- bank->level_mask =
- readl_relaxed(bank->base + bank->regs->leveldetect0) |
- readl_relaxed(bank->base + bank->regs->leveldetect1);
}
-#ifdef CONFIG_ARCH_OMAP1
/*
* This only applies to chips that can't do both rising and falling edge
* detection at once. For all other chips, this function is a noop.
*/
static void omap_toggle_gpio_edge_triggering(struct gpio_bank *bank, int gpio)
{
- void __iomem *reg = bank->base;
- u32 l = 0;
-
- if (!bank->regs->irqctrl)
- return;
+ if (IS_ENABLED(CONFIG_ARCH_OMAP1) && bank->regs->irqctrl) {
+ void __iomem *reg = bank->base + bank->regs->irqctrl;
- reg += bank->regs->irqctrl;
-
- l = readl_relaxed(reg);
- if ((l >> gpio) & 1)
- l &= ~(BIT(gpio));
- else
- l |= BIT(gpio);
-
- writel_relaxed(l, reg);
+ writel_relaxed(readl_relaxed(reg) ^ BIT(gpio), reg);
+ }
}
-#else
-static void omap_toggle_gpio_edge_triggering(struct gpio_bank *bank, int gpio) {}
-#endif
static int omap_set_gpio_triggering(struct gpio_bank *bank, int gpio,
unsigned trigger)
{
void __iomem *reg = bank->base;
- void __iomem *base = bank->base;
u32 l = 0;
if (bank->regs->leveldetect0 && bank->regs->wkup_en) {
@@ -472,11 +363,6 @@ static int omap_set_gpio_triggering(struct gpio_bank *bank, int gpio,
l |= 2 << (gpio << 1);
if (trigger & IRQ_TYPE_EDGE_FALLING)
l |= BIT(gpio << 1);
-
- /* Enable wake-up during idle for dynamic tick */
- omap_gpio_rmw(base, bank->regs->wkup_en, BIT(gpio), trigger);
- bank->context.wake_en =
- readl_relaxed(bank->base + bank->regs->wkup_en);
writel_relaxed(l, reg);
}
return 0;
@@ -505,17 +391,6 @@ static void omap_enable_gpio_module(struct gpio_bank *bank, unsigned offset)
static void omap_disable_gpio_module(struct gpio_bank *bank, unsigned offset)
{
- void __iomem *base = bank->base;
-
- if (bank->regs->wkup_en &&
- !LINE_USED(bank->mod_usage, offset) &&
- !LINE_USED(bank->irq_usage, offset)) {
- /* Disable wake-up during idle for dynamic tick */
- omap_gpio_rmw(base, bank->regs->wkup_en, BIT(offset), 0);
- bank->context.wake_en =
- readl_relaxed(bank->base + bank->regs->wkup_en);
- }
-
if (bank->regs->ctrl && !BANK_USED(bank)) {
void __iomem *reg = bank->base + bank->regs->ctrl;
u32 ctrl;
@@ -626,57 +501,39 @@ static u32 omap_get_gpio_irqbank_mask(struct gpio_bank *bank)
return l;
}
-static void omap_enable_gpio_irqbank(struct gpio_bank *bank, int gpio_mask)
+static inline void omap_set_gpio_irqenable(struct gpio_bank *bank,
+ unsigned offset, int enable)
{
void __iomem *reg = bank->base;
- u32 l;
+ u32 gpio_mask = BIT(offset);
- if (bank->regs->set_irqenable) {
- reg += bank->regs->set_irqenable;
- l = gpio_mask;
- bank->context.irqenable1 |= gpio_mask;
+ if (bank->regs->set_irqenable && bank->regs->clr_irqenable) {
+ if (enable) {
+ reg += bank->regs->set_irqenable;
+ bank->context.irqenable1 |= gpio_mask;
+ } else {
+ reg += bank->regs->clr_irqenable;
+ bank->context.irqenable1 &= ~gpio_mask;
+ }
+ writel_relaxed(gpio_mask, reg);
} else {
- reg += bank->regs->irqenable;
- l = readl_relaxed(reg);
- if (bank->regs->irqenable_inv)
- l &= ~gpio_mask;
- else
- l |= gpio_mask;
- bank->context.irqenable1 = l;
+ bank->context.irqenable1 =
+ omap_gpio_rmw(reg + bank->regs->irqenable, gpio_mask,
+ enable ^ bank->regs->irqenable_inv);
}
- writel_relaxed(l, reg);
-}
-
-static void omap_disable_gpio_irqbank(struct gpio_bank *bank, int gpio_mask)
-{
- void __iomem *reg = bank->base;
- u32 l;
-
- if (bank->regs->clr_irqenable) {
- reg += bank->regs->clr_irqenable;
- l = gpio_mask;
- bank->context.irqenable1 &= ~gpio_mask;
- } else {
- reg += bank->regs->irqenable;
- l = readl_relaxed(reg);
- if (bank->regs->irqenable_inv)
- l |= gpio_mask;
- else
- l &= ~gpio_mask;
- bank->context.irqenable1 = l;
+ /*
+ * Program GPIO wakeup along with IRQ enable to satisfy OMAP4430 TRM
+ * note requiring correlation between the IRQ enable registers and
+ * the wakeup registers. In any case, we want wakeup from idle
+ * enabled for the GPIOs which support this feature.
+ */
+ if (bank->regs->wkup_en &&
+ (bank->regs->edgectrl1 || !(bank->non_wakeup_gpios & gpio_mask))) {
+ bank->context.wake_en =
+ omap_gpio_rmw(bank->base + bank->regs->wkup_en,
+ gpio_mask, enable);
}
-
- writel_relaxed(l, reg);
-}
-
-static inline void omap_set_gpio_irqenable(struct gpio_bank *bank,
- unsigned offset, int enable)
-{
- if (enable)
- omap_enable_gpio_irqbank(bank, BIT(offset));
- else
- omap_disable_gpio_irqbank(bank, BIT(offset));
}
/* Use disable_irq_wake() and enable_irq_wake() functions from drivers */
@@ -687,38 +544,6 @@ static int omap_gpio_wake_enable(struct irq_data *d, unsigned int enable)
return irq_set_irq_wake(bank->irq, enable);
}
-static int omap_gpio_request(struct gpio_chip *chip, unsigned offset)
-{
- struct gpio_bank *bank = gpiochip_get_data(chip);
- unsigned long flags;
-
- pm_runtime_get_sync(chip->parent);
-
- raw_spin_lock_irqsave(&bank->lock, flags);
- omap_enable_gpio_module(bank, offset);
- bank->mod_usage |= BIT(offset);
- raw_spin_unlock_irqrestore(&bank->lock, flags);
-
- return 0;
-}
-
-static void omap_gpio_free(struct gpio_chip *chip, unsigned offset)
-{
- struct gpio_bank *bank = gpiochip_get_data(chip);
- unsigned long flags;
-
- raw_spin_lock_irqsave(&bank->lock, flags);
- bank->mod_usage &= ~(BIT(offset));
- if (!LINE_USED(bank->irq_usage, offset)) {
- omap_set_gpio_direction(bank, offset, 1);
- omap_clear_gpio_debounce(bank, offset);
- }
- omap_disable_gpio_module(bank, offset);
- raw_spin_unlock_irqrestore(&bank->lock, flags);
-
- pm_runtime_put(chip->parent);
-}
-
/*
* We need to unmask the GPIO bank interrupt as soon as possible to
* avoid missing GPIO interrupts for other lines in the bank.
@@ -731,7 +556,7 @@ static void omap_gpio_free(struct gpio_chip *chip, unsigned offset)
static irqreturn_t omap_gpio_irq_handler(int irq, void *gpiobank)
{
void __iomem *isr_reg = NULL;
- u32 enabled, isr, level_mask;
+ u32 enabled, isr, edge;
unsigned int bit;
struct gpio_bank *bank = gpiobank;
unsigned long wa_lock_flags;
@@ -751,16 +576,14 @@ static irqreturn_t omap_gpio_irq_handler(int irq, void *gpiobank)
enabled = omap_get_gpio_irqbank_mask(bank);
isr = readl_relaxed(isr_reg) & enabled;
- if (bank->level_mask)
- level_mask = bank->level_mask & enabled;
- else
- level_mask = 0;
-
- /* clear edge sensitive interrupts before handler(s) are
- called so that we don't miss any interrupt occurred while
- executing them */
- if (isr & ~level_mask)
- omap_clear_gpio_irqbank(bank, isr & ~level_mask);
+ /*
+ * Clear edge sensitive interrupts before calling handler(s)
+ * so subsequent edge transitions are not missed while the
+ * handlers are running.
+ */
+ edge = isr & ~bank->level_mask;
+ if (edge)
+ omap_clear_gpio_irqbank(bank, edge);
raw_spin_unlock_irqrestore(&bank->lock, lock_flags);
@@ -807,8 +630,6 @@ static unsigned int omap_gpio_irq_startup(struct irq_data *d)
if (!LINE_USED(bank->mod_usage, offset))
omap_set_gpio_direction(bank, offset, 1);
- else if (!omap_gpio_is_input(bank, offset))
- goto err;
omap_enable_gpio_module(bank, offset);
bank->irq_usage |= BIT(offset);
@@ -816,9 +637,6 @@ static unsigned int omap_gpio_irq_startup(struct irq_data *d)
omap_gpio_unmask_irq(d);
return 0;
-err:
- raw_spin_unlock_irqrestore(&bank->lock, flags);
- return -EINVAL;
}
static void omap_gpio_irq_shutdown(struct irq_data *d)
@@ -829,9 +647,9 @@ static void omap_gpio_irq_shutdown(struct irq_data *d)
raw_spin_lock_irqsave(&bank->lock, flags);
bank->irq_usage &= ~(BIT(offset));
- omap_set_gpio_irqenable(bank, offset, 0);
- omap_clear_gpio_irqstatus(bank, offset);
omap_set_gpio_triggering(bank, offset, IRQ_TYPE_NONE);
+ omap_clear_gpio_irqstatus(bank, offset);
+ omap_set_gpio_irqenable(bank, offset, 0);
if (!LINE_USED(bank->mod_usage, offset))
omap_clear_gpio_debounce(bank, offset);
omap_disable_gpio_module(bank, offset);
@@ -852,14 +670,6 @@ static void gpio_irq_bus_sync_unlock(struct irq_data *data)
pm_runtime_put(bank->chip.parent);
}
-static void omap_gpio_ack_irq(struct irq_data *d)
-{
- struct gpio_bank *bank = omap_irq_data_get_bank(d);
- unsigned offset = d->hwirq;
-
- omap_clear_gpio_irqstatus(bank, offset);
-}
-
static void omap_gpio_mask_irq(struct irq_data *d)
{
struct gpio_bank *bank = omap_irq_data_get_bank(d);
@@ -867,8 +677,8 @@ static void omap_gpio_mask_irq(struct irq_data *d)
unsigned long flags;
raw_spin_lock_irqsave(&bank->lock, flags);
- omap_set_gpio_irqenable(bank, offset, 0);
omap_set_gpio_triggering(bank, offset, IRQ_TYPE_NONE);
+ omap_set_gpio_irqenable(bank, offset, 0);
raw_spin_unlock_irqrestore(&bank->lock, flags);
}
@@ -880,9 +690,6 @@ static void omap_gpio_unmask_irq(struct irq_data *d)
unsigned long flags;
raw_spin_lock_irqsave(&bank->lock, flags);
- if (trigger)
- omap_set_gpio_triggering(bank, offset, trigger);
-
omap_set_gpio_irqenable(bank, offset, 1);
/*
@@ -890,9 +697,13 @@ static void omap_gpio_unmask_irq(struct irq_data *d)
* is cleared, thus after the handler has run. OMAP4 needs this done
* after enabing the interrupt to clear the wakeup status.
*/
- if (bank->level_mask & BIT(offset))
+ if (bank->regs->leveldetect0 && bank->regs->wkup_en &&
+ trigger & (IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_LEVEL_LOW))
omap_clear_gpio_irqstatus(bank, offset);
+ if (trigger)
+ omap_set_gpio_triggering(bank, offset, trigger);
+
raw_spin_unlock_irqrestore(&bank->lock, flags);
}
@@ -958,19 +769,44 @@ static inline void omap_mpuio_init(struct gpio_bank *bank)
/*---------------------------------------------------------------------*/
-static int omap_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
+static int omap_gpio_request(struct gpio_chip *chip, unsigned offset)
{
- struct gpio_bank *bank;
+ struct gpio_bank *bank = gpiochip_get_data(chip);
+ unsigned long flags;
+
+ pm_runtime_get_sync(chip->parent);
+
+ raw_spin_lock_irqsave(&bank->lock, flags);
+ omap_enable_gpio_module(bank, offset);
+ bank->mod_usage |= BIT(offset);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
+
+ return 0;
+}
+
+static void omap_gpio_free(struct gpio_chip *chip, unsigned offset)
+{
+ struct gpio_bank *bank = gpiochip_get_data(chip);
unsigned long flags;
- void __iomem *reg;
- int dir;
- bank = gpiochip_get_data(chip);
- reg = bank->base + bank->regs->direction;
raw_spin_lock_irqsave(&bank->lock, flags);
- dir = !!(readl_relaxed(reg) & BIT(offset));
+ bank->mod_usage &= ~(BIT(offset));
+ if (!LINE_USED(bank->irq_usage, offset)) {
+ omap_set_gpio_direction(bank, offset, 1);
+ omap_clear_gpio_debounce(bank, offset);
+ }
+ omap_disable_gpio_module(bank, offset);
raw_spin_unlock_irqrestore(&bank->lock, flags);
- return dir;
+
+ pm_runtime_put(chip->parent);
+}
+
+static int omap_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
+{
+ struct gpio_bank *bank = gpiochip_get_data(chip);
+
+ return !!(readl_relaxed(bank->base + bank->regs->direction) &
+ BIT(offset));
}
static int omap_gpio_input(struct gpio_chip *chip, unsigned offset)
@@ -987,14 +823,15 @@ static int omap_gpio_input(struct gpio_chip *chip, unsigned offset)
static int omap_gpio_get(struct gpio_chip *chip, unsigned offset)
{
- struct gpio_bank *bank;
-
- bank = gpiochip_get_data(chip);
+ struct gpio_bank *bank = gpiochip_get_data(chip);
+ void __iomem *reg;
if (omap_gpio_is_input(bank, offset))
- return omap_get_gpio_datain(bank, offset);
+ reg = bank->base + bank->regs->datain;
else
- return omap_get_gpio_dataout(bank, offset);
+ reg = bank->base + bank->regs->dataout;
+
+ return (readl_relaxed(reg) & BIT(offset)) != 0;
}
static int omap_gpio_output(struct gpio_chip *chip, unsigned offset, int value)
@@ -1014,18 +851,20 @@ static int omap_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask,
unsigned long *bits)
{
struct gpio_bank *bank = gpiochip_get_data(chip);
- void __iomem *reg = bank->base + bank->regs->direction;
- unsigned long in = readl_relaxed(reg), l;
+ void __iomem *base = bank->base;
+ u32 direction, m, val = 0;
- *bits = 0;
+ direction = readl_relaxed(base + bank->regs->direction);
- l = in & *mask;
- if (l)
- *bits |= omap_get_gpio_datain_multiple(bank, &l);
+ m = direction & *mask;
+ if (m)
+ val |= readl_relaxed(base + bank->regs->datain) & m;
- l = ~in & *mask;
- if (l)
- *bits |= omap_get_gpio_dataout_multiple(bank, &l);
+ m = ~direction & *mask;
+ if (m)
+ val |= readl_relaxed(base + bank->regs->dataout) & m;
+
+ *bits = val;
return 0;
}
@@ -1078,10 +917,14 @@ static void omap_gpio_set_multiple(struct gpio_chip *chip, unsigned long *mask,
unsigned long *bits)
{
struct gpio_bank *bank = gpiochip_get_data(chip);
+ void __iomem *reg = bank->base + bank->regs->dataout;
unsigned long flags;
+ u32 l;
raw_spin_lock_irqsave(&bank->lock, flags);
- bank->set_dataout_multiple(bank, mask, bits);
+ l = (readl_relaxed(reg) & ~*mask) | (*bits & *mask);
+ writel_relaxed(l, reg);
+ bank->context.dataout = l;
raw_spin_unlock_irqrestore(&bank->lock, flags);
}
@@ -1115,9 +958,9 @@ static void omap_gpio_mod_init(struct gpio_bank *bank)
return;
}
- omap_gpio_rmw(base, bank->regs->irqenable, l,
+ omap_gpio_rmw(base + bank->regs->irqenable, l,
bank->regs->irqenable_inv);
- omap_gpio_rmw(base, bank->regs->irqstatus, l,
+ omap_gpio_rmw(base + bank->regs->irqstatus, l,
!bank->regs->irqenable_inv);
if (bank->regs->debounce_en)
writel_relaxed(0, base + bank->regs->debounce_en);
@@ -1180,11 +1023,8 @@ static int omap_gpio_chip_init(struct gpio_bank *bank, struct irq_chip *irqc)
#endif
/* MPUIO is a bit different, reading IRQ status clears it */
- if (bank->is_mpuio) {
- irqc->irq_ack = dummy_irq_chip.irq_ack;
- if (!bank->regs->wkup_en)
- irqc->irq_set_wake = NULL;
- }
+ if (bank->is_mpuio && !bank->regs->wkup_en)
+ irqc->irq_set_wake = NULL;
irq = &bank->chip.irq;
irq->chip = irqc;
@@ -1215,7 +1055,7 @@ static int omap_gpio_chip_init(struct gpio_bank *bank, struct irq_chip *irqc)
static void omap_gpio_init_context(struct gpio_bank *p)
{
- struct omap_gpio_reg_offs *regs = p->regs;
+ const struct omap_gpio_reg_offs *regs = p->regs;
void __iomem *base = p->base;
p->context.ctrl = readl_relaxed(base + regs->ctrl);
@@ -1227,60 +1067,56 @@ static void omap_gpio_init_context(struct gpio_bank *p)
p->context.fallingdetect = readl_relaxed(base + regs->fallingdetect);
p->context.irqenable1 = readl_relaxed(base + regs->irqenable);
p->context.irqenable2 = readl_relaxed(base + regs->irqenable2);
-
- if (regs->set_dataout && p->regs->clr_dataout)
- p->context.dataout = readl_relaxed(base + regs->set_dataout);
- else
- p->context.dataout = readl_relaxed(base + regs->dataout);
+ p->context.dataout = readl_relaxed(base + regs->dataout);
p->context_valid = true;
}
static void omap_gpio_restore_context(struct gpio_bank *bank)
{
- writel_relaxed(bank->context.wake_en,
- bank->base + bank->regs->wkup_en);
- writel_relaxed(bank->context.ctrl, bank->base + bank->regs->ctrl);
- writel_relaxed(bank->context.leveldetect0,
- bank->base + bank->regs->leveldetect0);
- writel_relaxed(bank->context.leveldetect1,
- bank->base + bank->regs->leveldetect1);
- writel_relaxed(bank->context.risingdetect,
- bank->base + bank->regs->risingdetect);
- writel_relaxed(bank->context.fallingdetect,
- bank->base + bank->regs->fallingdetect);
- if (bank->regs->set_dataout && bank->regs->clr_dataout)
- writel_relaxed(bank->context.dataout,
- bank->base + bank->regs->set_dataout);
- else
- writel_relaxed(bank->context.dataout,
- bank->base + bank->regs->dataout);
- writel_relaxed(bank->context.oe, bank->base + bank->regs->direction);
+ const struct omap_gpio_reg_offs *regs = bank->regs;
+ void __iomem *base = bank->base;
+
+ writel_relaxed(bank->context.wake_en, base + regs->wkup_en);
+ writel_relaxed(bank->context.ctrl, base + regs->ctrl);
+ writel_relaxed(bank->context.leveldetect0, base + regs->leveldetect0);
+ writel_relaxed(bank->context.leveldetect1, base + regs->leveldetect1);
+ writel_relaxed(bank->context.risingdetect, base + regs->risingdetect);
+ writel_relaxed(bank->context.fallingdetect, base + regs->fallingdetect);
+ writel_relaxed(bank->context.dataout, base + regs->dataout);
+ writel_relaxed(bank->context.oe, base + regs->direction);
if (bank->dbck_enable_mask) {
- writel_relaxed(bank->context.debounce, bank->base +
- bank->regs->debounce);
+ writel_relaxed(bank->context.debounce, base + regs->debounce);
writel_relaxed(bank->context.debounce_en,
- bank->base + bank->regs->debounce_en);
+ base + regs->debounce_en);
}
- writel_relaxed(bank->context.irqenable1,
- bank->base + bank->regs->irqenable);
- writel_relaxed(bank->context.irqenable2,
- bank->base + bank->regs->irqenable2);
+ writel_relaxed(bank->context.irqenable1, base + regs->irqenable);
+ writel_relaxed(bank->context.irqenable2, base + regs->irqenable2);
}
static void omap_gpio_idle(struct gpio_bank *bank, bool may_lose_context)
{
struct device *dev = bank->chip.parent;
void __iomem *base = bank->base;
- u32 nowake;
+ u32 mask, nowake;
bank->saved_datain = readl_relaxed(base + bank->regs->datain);
if (!bank->enabled_non_wakeup_gpios)
goto update_gpio_context_count;
+ /* Check for pending EDGE_FALLING, ignore EDGE_BOTH */
+ mask = bank->enabled_non_wakeup_gpios & bank->context.fallingdetect;
+ mask &= ~bank->context.risingdetect;
+ bank->saved_datain |= mask;
+
+ /* Check for pending EDGE_RISING, ignore EDGE_BOTH */
+ mask = bank->enabled_non_wakeup_gpios & bank->context.risingdetect;
+ mask &= ~bank->context.fallingdetect;
+ bank->saved_datain &= ~mask;
+
if (!may_lose_context)
goto update_gpio_context_count;
@@ -1291,8 +1127,8 @@ static void omap_gpio_idle(struct gpio_bank *bank, bool may_lose_context)
*/
if (!bank->loses_context && bank->enabled_non_wakeup_gpios) {
nowake = bank->enabled_non_wakeup_gpios;
- omap_gpio_rmw(base, bank->regs->fallingdetect, nowake, ~nowake);
- omap_gpio_rmw(base, bank->regs->risingdetect, nowake, ~nowake);
+ omap_gpio_rmw(base + bank->regs->fallingdetect, nowake, ~nowake);
+ omap_gpio_rmw(base + bank->regs->risingdetect, nowake, ~nowake);
}
update_gpio_context_count:
@@ -1421,7 +1257,7 @@ static int gpio_omap_cpu_notifier(struct notifier_block *nb,
return NOTIFY_OK;
}
-static struct omap_gpio_reg_offs omap2_gpio_regs = {
+static const struct omap_gpio_reg_offs omap2_gpio_regs = {
.revision = OMAP24XX_GPIO_REVISION,
.direction = OMAP24XX_GPIO_OE,
.datain = OMAP24XX_GPIO_DATAIN,
@@ -1444,7 +1280,7 @@ static struct omap_gpio_reg_offs omap2_gpio_regs = {
.fallingdetect = OMAP24XX_GPIO_FALLINGDETECT,
};
-static struct omap_gpio_reg_offs omap4_gpio_regs = {
+static const struct omap_gpio_reg_offs omap4_gpio_regs = {
.revision = OMAP4_GPIO_REVISION,
.direction = OMAP4_GPIO_OE,
.datain = OMAP4_GPIO_DATAIN,
@@ -1453,6 +1289,8 @@ static struct omap_gpio_reg_offs omap4_gpio_regs = {
.clr_dataout = OMAP4_GPIO_CLEARDATAOUT,
.irqstatus = OMAP4_GPIO_IRQSTATUS0,
.irqstatus2 = OMAP4_GPIO_IRQSTATUS1,
+ .irqstatus_raw0 = OMAP4_GPIO_IRQSTATUSRAW0,
+ .irqstatus_raw1 = OMAP4_GPIO_IRQSTATUSRAW1,
.irqenable = OMAP4_GPIO_IRQSTATUSSET0,
.irqenable2 = OMAP4_GPIO_IRQSTATUSSET1,
.set_irqenable = OMAP4_GPIO_IRQSTATUSSET0,
@@ -1528,7 +1366,7 @@ static int omap_gpio_probe(struct platform_device *pdev)
irqc->irq_startup = omap_gpio_irq_startup,
irqc->irq_shutdown = omap_gpio_irq_shutdown,
- irqc->irq_ack = omap_gpio_ack_irq,
+ irqc->irq_ack = dummy_irq_chip.irq_ack,
irqc->irq_mask = omap_gpio_mask_irq,
irqc->irq_unmask = omap_gpio_unmask_irq,
irqc->irq_set_type = omap_gpio_irq_type,
@@ -1572,14 +1410,10 @@ static int omap_gpio_probe(struct platform_device *pdev)
pdata->get_context_loss_count;
}
- if (bank->regs->set_dataout && bank->regs->clr_dataout) {
+ if (bank->regs->set_dataout && bank->regs->clr_dataout)
bank->set_dataout = omap_set_gpio_dataout_reg;
- bank->set_dataout_multiple = omap_set_gpio_dataout_reg_multiple;
- } else {
+ else
bank->set_dataout = omap_set_gpio_dataout_mask;
- bank->set_dataout_multiple =
- omap_set_gpio_dataout_mask_multiple;
- }
raw_spin_lock_init(&bank->lock);
raw_spin_lock_init(&bank->wa_lock);
@@ -1635,7 +1469,6 @@ static int omap_gpio_remove(struct platform_device *pdev)
struct gpio_bank *bank = platform_get_drvdata(pdev);
cpu_pm_unregister_notifier(&bank->nb);
- list_del(&bank->node);
gpiochip_remove(&bank->chip);
pm_runtime_disable(&pdev->dev);
if (bank->dbck_flag)
diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c
index cfe827cefad8..de5d1383f28d 100644
--- a/drivers/gpio/gpio-pca953x.c
+++ b/drivers/gpio/gpio-pca953x.c
@@ -9,6 +9,7 @@
*/
#include <linux/acpi.h>
+#include <linux/bits.h>
#include <linux/gpio/driver.h>
#include <linux/gpio/consumer.h>
#include <linux/i2c.h>
@@ -28,9 +29,9 @@
#define PCA953X_INVERT 0x02
#define PCA953X_DIRECTION 0x03
-#define REG_ADDR_MASK 0x3f
-#define REG_ADDR_EXT 0x40
-#define REG_ADDR_AI 0x80
+#define REG_ADDR_MASK GENMASK(5, 0)
+#define REG_ADDR_EXT BIT(6)
+#define REG_ADDR_AI BIT(7)
#define PCA957X_IN 0x00
#define PCA957X_INVRT 0x01
@@ -55,17 +56,17 @@
#define PCAL6524_OUT_INDCONF 0x2c
#define PCAL6524_DEBOUNCE 0x2d
-#define PCA_GPIO_MASK 0x00FF
+#define PCA_GPIO_MASK GENMASK(7, 0)
-#define PCAL_GPIO_MASK 0x1f
-#define PCAL_PINCTRL_MASK 0x60
+#define PCAL_GPIO_MASK GENMASK(4, 0)
+#define PCAL_PINCTRL_MASK GENMASK(6, 5)
-#define PCA_INT 0x0100
-#define PCA_PCAL 0x0200
+#define PCA_INT BIT(8)
+#define PCA_PCAL BIT(9)
#define PCA_LATCH_INT (PCA_PCAL | PCA_INT)
-#define PCA953X_TYPE 0x1000
-#define PCA957X_TYPE 0x2000
-#define PCA_TYPE_MASK 0xF000
+#define PCA953X_TYPE BIT(12)
+#define PCA957X_TYPE BIT(13)
+#define PCA_TYPE_MASK GENMASK(15, 12)
#define PCA_CHIP_TYPE(x) ((x) & PCA_TYPE_MASK)
@@ -565,7 +566,7 @@ static void pca953x_irq_mask(struct irq_data *d)
struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
struct pca953x_chip *chip = gpiochip_get_data(gc);
- chip->irq_mask[d->hwirq / BANK_SZ] &= ~(1 << (d->hwirq % BANK_SZ));
+ chip->irq_mask[d->hwirq / BANK_SZ] &= ~BIT(d->hwirq % BANK_SZ);
}
static void pca953x_irq_unmask(struct irq_data *d)
@@ -573,7 +574,7 @@ static void pca953x_irq_unmask(struct irq_data *d)
struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
struct pca953x_chip *chip = gpiochip_get_data(gc);
- chip->irq_mask[d->hwirq / BANK_SZ] |= 1 << (d->hwirq % BANK_SZ);
+ chip->irq_mask[d->hwirq / BANK_SZ] |= BIT(d->hwirq % BANK_SZ);
}
static int pca953x_irq_set_wake(struct irq_data *d, unsigned int on)
@@ -604,10 +605,9 @@ static void pca953x_irq_bus_sync_unlock(struct irq_data *d)
u8 new_irqs;
int level, i;
u8 invert_irq_mask[MAX_BANK];
- int reg_direction[MAX_BANK];
+ u8 reg_direction[MAX_BANK];
- regmap_bulk_read(chip->regmap, chip->regs->direction, reg_direction,
- NBANK(chip));
+ pca953x_read_regs(chip, chip->regs->direction, reg_direction);
if (chip->driver_data & PCA_PCAL) {
/* Enable latch on interrupt-enabled inputs */
@@ -641,7 +641,7 @@ static int pca953x_irq_set_type(struct irq_data *d, unsigned int type)
struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
struct pca953x_chip *chip = gpiochip_get_data(gc);
int bank_nb = d->hwirq / BANK_SZ;
- u8 mask = 1 << (d->hwirq % BANK_SZ);
+ u8 mask = BIT(d->hwirq % BANK_SZ);
if (!(type & IRQ_TYPE_EDGE_BOTH)) {
dev_err(&chip->client->dev, "irq %d: unsupported type %d\n",
@@ -666,7 +666,7 @@ static void pca953x_irq_shutdown(struct irq_data *d)
{
struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
struct pca953x_chip *chip = gpiochip_get_data(gc);
- u8 mask = 1 << (d->hwirq % BANK_SZ);
+ u8 mask = BIT(d->hwirq % BANK_SZ);
chip->irq_trig_raise[d->hwirq / BANK_SZ] &= ~mask;
chip->irq_trig_fall[d->hwirq / BANK_SZ] &= ~mask;
@@ -679,7 +679,7 @@ static bool pca953x_irq_pending(struct pca953x_chip *chip, u8 *pending)
bool pending_seen = false;
bool trigger_seen = false;
u8 trigger[MAX_BANK];
- int reg_direction[MAX_BANK];
+ u8 reg_direction[MAX_BANK];
int ret, i;
if (chip->driver_data & PCA_PCAL) {
@@ -710,8 +710,7 @@ static bool pca953x_irq_pending(struct pca953x_chip *chip, u8 *pending)
return false;
/* Remove output pins from the equation */
- regmap_bulk_read(chip->regmap, chip->regs->direction, reg_direction,
- NBANK(chip));
+ pca953x_read_regs(chip, chip->regs->direction, reg_direction);
for (i = 0; i < NBANK(chip); i++)
cur_stat[i] &= reg_direction[i];
@@ -768,7 +767,7 @@ static int pca953x_irq_setup(struct pca953x_chip *chip,
{
struct i2c_client *client = chip->client;
struct irq_chip *irq_chip = &chip->irq_chip;
- int reg_direction[MAX_BANK];
+ u8 reg_direction[MAX_BANK];
int ret, i;
if (!client->irq)
@@ -789,8 +788,7 @@ static int pca953x_irq_setup(struct pca953x_chip *chip,
* interrupt. We have to rely on the previous read for
* this purpose.
*/
- regmap_bulk_read(chip->regmap, chip->regs->direction, reg_direction,
- NBANK(chip));
+ pca953x_read_regs(chip, chip->regs->direction, reg_direction);
for (i = 0; i < NBANK(chip); i++)
chip->irq_stat[i] &= reg_direction[i];
mutex_init(&chip->irq_lock);
@@ -849,12 +847,12 @@ static int device_pca95xx_init(struct pca953x_chip *chip, u32 invert)
ret = regcache_sync_region(chip->regmap, chip->regs->output,
chip->regs->output + NBANK(chip));
- if (ret != 0)
+ if (ret)
goto out;
ret = regcache_sync_region(chip->regmap, chip->regs->direction,
chip->regs->direction + NBANK(chip));
- if (ret != 0)
+ if (ret)
goto out;
/* set platform specific polarity inversion */
@@ -949,19 +947,15 @@ static int pca953x_probe(struct i2c_client *client,
if (i2c_id) {
chip->driver_data = i2c_id->driver_data;
} else {
- const struct acpi_device_id *acpi_id;
- struct device *dev = &client->dev;
-
- chip->driver_data = (uintptr_t)of_device_get_match_data(dev);
- if (!chip->driver_data) {
- acpi_id = acpi_match_device(pca953x_acpi_ids, dev);
- if (!acpi_id) {
- ret = -ENODEV;
- goto err_exit;
- }
-
- chip->driver_data = acpi_id->driver_data;
+ const void *match;
+
+ match = device_get_match_data(&client->dev);
+ if (!match) {
+ ret = -ENODEV;
+ goto err_exit;
}
+
+ chip->driver_data = (uintptr_t)match;
}
i2c_set_clientdata(client, chip);
@@ -1041,8 +1035,7 @@ static int pca953x_remove(struct i2c_client *client)
ret = pdata->teardown(client, chip->gpio_chip.base,
chip->gpio_chip.ngpio, pdata->context);
if (ret < 0)
- dev_err(&client->dev, "%s failed, %d\n",
- "teardown", ret);
+ dev_err(&client->dev, "teardown failed, %d\n", ret);
} else {
ret = 0;
}
@@ -1064,14 +1057,14 @@ static int pca953x_regcache_sync(struct device *dev)
*/
ret = regcache_sync_region(chip->regmap, chip->regs->direction,
chip->regs->direction + NBANK(chip));
- if (ret != 0) {
+ if (ret) {
dev_err(dev, "Failed to sync GPIO dir registers: %d\n", ret);
return ret;
}
ret = regcache_sync_region(chip->regmap, chip->regs->output,
chip->regs->output + NBANK(chip));
- if (ret != 0) {
+ if (ret) {
dev_err(dev, "Failed to sync GPIO out registers: %d\n", ret);
return ret;
}
@@ -1080,7 +1073,7 @@ static int pca953x_regcache_sync(struct device *dev)
if (chip->driver_data & PCA_PCAL) {
ret = regcache_sync_region(chip->regmap, PCAL953X_IN_LATCH,
PCAL953X_IN_LATCH + NBANK(chip));
- if (ret != 0) {
+ if (ret) {
dev_err(dev, "Failed to sync INT latch registers: %d\n",
ret);
return ret;
@@ -1088,7 +1081,7 @@ static int pca953x_regcache_sync(struct device *dev)
ret = regcache_sync_region(chip->regmap, PCAL953X_INT_MASK,
PCAL953X_INT_MASK + NBANK(chip));
- if (ret != 0) {
+ if (ret) {
dev_err(dev, "Failed to sync INT mask registers: %d\n",
ret);
return ret;
@@ -1120,7 +1113,7 @@ static int pca953x_resume(struct device *dev)
if (!atomic_read(&chip->wakeup_path)) {
ret = regulator_enable(chip->regulator);
- if (ret != 0) {
+ if (ret) {
dev_err(dev, "Failed to enable regulator: %d\n", ret);
return 0;
}
@@ -1133,7 +1126,7 @@ static int pca953x_resume(struct device *dev)
return ret;
ret = regcache_sync(chip->regmap);
- if (ret != 0) {
+ if (ret) {
dev_err(dev, "Failed to restore register map: %d\n", ret);
return ret;
}
@@ -1178,6 +1171,7 @@ static const struct of_device_id pca953x_dt_ids[] = {
{ .compatible = "ti,tca6408", .data = OF_953X( 8, PCA_INT), },
{ .compatible = "ti,tca6416", .data = OF_953X(16, PCA_INT), },
{ .compatible = "ti,tca6424", .data = OF_953X(24, PCA_INT), },
+ { .compatible = "ti,tca9539", .data = OF_953X(16, PCA_INT), },
{ .compatible = "onnn,cat9554", .data = OF_953X( 8, PCA_INT), },
{ .compatible = "onnn,pca9654", .data = OF_953X( 8, PCA_INT), },
diff --git a/drivers/gpio/gpio-pch.c b/drivers/gpio/gpio-pch.c
index 1d99293096f2..3f3d9a94b709 100644
--- a/drivers/gpio/gpio-pch.c
+++ b/drivers/gpio/gpio-pch.c
@@ -409,8 +409,7 @@ static int pch_gpio_probe(struct pci_dev *pdev,
static int __maybe_unused pch_gpio_suspend(struct device *dev)
{
- struct pci_dev *pdev = to_pci_dev(dev);
- struct pch_gpio *chip = pci_get_drvdata(pdev);
+ struct pch_gpio *chip = dev_get_drvdata(dev);
unsigned long flags;
spin_lock_irqsave(&chip->spinlock, flags);
@@ -422,8 +421,7 @@ static int __maybe_unused pch_gpio_suspend(struct device *dev)
static int __maybe_unused pch_gpio_resume(struct device *dev)
{
- struct pci_dev *pdev = to_pci_dev(dev);
- struct pch_gpio *chip = pci_get_drvdata(pdev);
+ struct pch_gpio *chip = dev_get_drvdata(dev);
unsigned long flags;
spin_lock_irqsave(&chip->spinlock, flags);
diff --git a/drivers/gpio/gpio-pl061.c b/drivers/gpio/gpio-pl061.c
index 9aad32206e84..722ce5cf861e 100644
--- a/drivers/gpio/gpio-pl061.c
+++ b/drivers/gpio/gpio-pl061.c
@@ -283,6 +283,7 @@ static int pl061_probe(struct amba_device *adev, const struct amba_id *id)
{
struct device *dev = &adev->dev;
struct pl061 *pl061;
+ struct gpio_irq_chip *girq;
int ret, irq;
pl061 = devm_kzalloc(dev, sizeof(*pl061), GFP_KERNEL);
@@ -310,10 +311,6 @@ static int pl061_probe(struct amba_device *adev, const struct amba_id *id)
pl061->gc.parent = dev;
pl061->gc.owner = THIS_MODULE;
- ret = gpiochip_add_data(&pl061->gc, pl061);
- if (ret)
- return ret;
-
/*
* irq_chip support
*/
@@ -332,19 +329,24 @@ static int pl061_probe(struct amba_device *adev, const struct amba_id *id)
}
pl061->parent_irq = irq;
- ret = gpiochip_irqchip_add(&pl061->gc, &pl061->irq_chip,
- 0, handle_bad_irq,
- IRQ_TYPE_NONE);
- if (ret) {
- dev_info(&adev->dev, "could not add irqchip\n");
+ girq = &pl061->gc.irq;
+ girq->chip = &pl061->irq_chip;
+ girq->parent_handler = pl061_irq_handler;
+ girq->num_parents = 1;
+ girq->parents = devm_kcalloc(dev, 1, sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents)
+ return -ENOMEM;
+ girq->parents[0] = irq;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_bad_irq;
+
+ ret = devm_gpiochip_add_data(dev, &pl061->gc, pl061);
+ if (ret)
return ret;
- }
- gpiochip_set_chained_irqchip(&pl061->gc, &pl061->irq_chip,
- irq, pl061_irq_handler);
amba_set_drvdata(adev, pl061);
- dev_info(&adev->dev, "PL061 GPIO chip @%pa registered\n",
- &adev->res.start);
+ dev_info(dev, "PL061 GPIO chip registered\n");
return 0;
}
diff --git a/drivers/gpio/gpio-pmic-eic-sprd.c b/drivers/gpio/gpio-pmic-eic-sprd.c
index 24228cf79afc..05000cace9b2 100644
--- a/drivers/gpio/gpio-pmic-eic-sprd.c
+++ b/drivers/gpio/gpio-pmic-eic-sprd.c
@@ -305,10 +305,8 @@ static int sprd_pmic_eic_probe(struct platform_device *pdev)
mutex_init(&pmic_eic->buslock);
pmic_eic->irq = platform_get_irq(pdev, 0);
- if (pmic_eic->irq < 0) {
- dev_err(&pdev->dev, "Failed to get PMIC EIC interrupt.\n");
+ if (pmic_eic->irq < 0)
return pmic_eic->irq;
- }
pmic_eic->map = dev_get_regmap(pdev->dev.parent, NULL);
if (!pmic_eic->map)
diff --git a/drivers/gpio/gpio-rcar.c b/drivers/gpio/gpio-rcar.c
index 70e95fc4779f..187984d26f47 100644
--- a/drivers/gpio/gpio-rcar.c
+++ b/drivers/gpio/gpio-rcar.c
@@ -489,7 +489,7 @@ static int gpio_rcar_probe(struct platform_device *pdev)
irq_chip->irq_unmask = gpio_rcar_irq_enable;
irq_chip->irq_set_type = gpio_rcar_irq_set_type;
irq_chip->irq_set_wake = gpio_rcar_irq_set_wake;
- irq_chip->flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_MASK_ON_SUSPEND;
+ irq_chip->flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_MASK_ON_SUSPEND;
ret = gpiochip_add_data(gpio_chip, p);
if (ret) {
diff --git a/drivers/gpio/gpio-siox.c b/drivers/gpio/gpio-siox.c
index 571b2a81c6de..006a7e6a75f2 100644
--- a/drivers/gpio/gpio-siox.c
+++ b/drivers/gpio/gpio-siox.c
@@ -211,20 +211,22 @@ static int gpio_siox_get_direction(struct gpio_chip *chip, unsigned int offset)
static int gpio_siox_probe(struct siox_device *sdevice)
{
struct gpio_siox_ddata *ddata;
+ struct gpio_irq_chip *girq;
+ struct device *dev = &sdevice->dev;
int ret;
- ddata = devm_kzalloc(&sdevice->dev, sizeof(*ddata), GFP_KERNEL);
+ ddata = devm_kzalloc(dev, sizeof(*ddata), GFP_KERNEL);
if (!ddata)
return -ENOMEM;
- dev_set_drvdata(&sdevice->dev, ddata);
+ dev_set_drvdata(dev, ddata);
mutex_init(&ddata->lock);
spin_lock_init(&ddata->irqlock);
ddata->gchip.base = -1;
ddata->gchip.can_sleep = 1;
- ddata->gchip.parent = &sdevice->dev;
+ ddata->gchip.parent = dev;
ddata->gchip.owner = THIS_MODULE;
ddata->gchip.get = gpio_siox_get;
ddata->gchip.set = gpio_siox_set;
@@ -239,54 +241,27 @@ static int gpio_siox_probe(struct siox_device *sdevice)
ddata->ichip.irq_unmask = gpio_siox_irq_unmask;
ddata->ichip.irq_set_type = gpio_siox_irq_set_type;
- ret = gpiochip_add(&ddata->gchip);
- if (ret) {
- dev_err(&sdevice->dev,
- "Failed to register gpio chip (%d)\n", ret);
- goto err_gpiochip;
- }
+ girq = &ddata->gchip.irq;
+ girq->chip = &ddata->ichip;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_level_irq;
- ret = gpiochip_irqchip_add(&ddata->gchip, &ddata->ichip,
- 0, handle_level_irq, IRQ_TYPE_EDGE_RISING);
- if (ret) {
- dev_err(&sdevice->dev,
- "Failed to register irq chip (%d)\n", ret);
-err_gpiochip:
- gpiochip_remove(&ddata->gchip);
- }
+ ret = devm_gpiochip_add_data(dev, &ddata->gchip, NULL);
+ if (ret)
+ dev_err(dev, "Failed to register gpio chip (%d)\n", ret);
return ret;
}
-static int gpio_siox_remove(struct siox_device *sdevice)
-{
- struct gpio_siox_ddata *ddata = dev_get_drvdata(&sdevice->dev);
-
- gpiochip_remove(&ddata->gchip);
- return 0;
-}
-
static struct siox_driver gpio_siox_driver = {
.probe = gpio_siox_probe,
- .remove = gpio_siox_remove,
.set_data = gpio_siox_set_data,
.get_data = gpio_siox_get_data,
.driver = {
.name = "gpio-siox",
},
};
-
-static int __init gpio_siox_init(void)
-{
- return siox_driver_register(&gpio_siox_driver);
-}
-module_init(gpio_siox_init);
-
-static void __exit gpio_siox_exit(void)
-{
- siox_driver_unregister(&gpio_siox_driver);
-}
-module_exit(gpio_siox_exit);
+module_siox_driver(gpio_siox_driver);
MODULE_AUTHOR("Uwe Kleine-Koenig <u.kleine-koenig@pengutronix.de>");
MODULE_DESCRIPTION("SIOX gpio driver");
diff --git a/drivers/gpio/gpio-sprd.c b/drivers/gpio/gpio-sprd.c
index f5c8b3a351d5..d7314d39ab65 100644
--- a/drivers/gpio/gpio-sprd.c
+++ b/drivers/gpio/gpio-sprd.c
@@ -226,10 +226,8 @@ static int sprd_gpio_probe(struct platform_device *pdev)
return -ENOMEM;
sprd_gpio->irq = platform_get_irq(pdev, 0);
- if (sprd_gpio->irq < 0) {
- dev_err(&pdev->dev, "Failed to get GPIO interrupt.\n");
+ if (sprd_gpio->irq < 0)
return sprd_gpio->irq;
- }
sprd_gpio->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(sprd_gpio->base))
diff --git a/drivers/gpio/gpio-stmpe.c b/drivers/gpio/gpio-stmpe.c
index dbf9cbe36b2b..994d542daf53 100644
--- a/drivers/gpio/gpio-stmpe.c
+++ b/drivers/gpio/gpio-stmpe.c
@@ -429,6 +429,23 @@ static irqreturn_t stmpe_gpio_irq(int irq, void *dev)
return IRQ_HANDLED;
}
+static void stmpe_init_irq_valid_mask(struct gpio_chip *gc,
+ unsigned long *valid_mask,
+ unsigned int ngpios)
+{
+ struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(gc);
+ int i;
+
+ if (!stmpe_gpio->norequest_mask)
+ return;
+
+ /* Forbid unused lines to be mapped as IRQs */
+ for (i = 0; i < sizeof(u32); i++) {
+ if (stmpe_gpio->norequest_mask & BIT(i))
+ clear_bit(i, valid_mask);
+ }
+}
+
static int stmpe_gpio_probe(struct platform_device *pdev)
{
struct stmpe *stmpe = dev_get_drvdata(pdev->dev.parent);
@@ -454,14 +471,21 @@ static int stmpe_gpio_probe(struct platform_device *pdev)
stmpe_gpio->chip.parent = &pdev->dev;
stmpe_gpio->chip.of_node = np;
stmpe_gpio->chip.base = -1;
+ /*
+ * REVISIT: this makes sure the valid mask gets allocated and
+ * filled in when adding the gpio_chip, but the rest of the
+ * gpio_irqchip is still filled in using the old method
+ * in gpiochip_irqchip_add_nested() so clean this up once we
+ * get the gpio_irqchip to initialize while adding the
+ * gpio_chip also for threaded irqchips.
+ */
+ stmpe_gpio->chip.irq.init_valid_mask = stmpe_init_irq_valid_mask;
if (IS_ENABLED(CONFIG_DEBUG_FS))
stmpe_gpio->chip.dbg_show = stmpe_dbg_show;
of_property_read_u32(np, "st,norequest-mask",
&stmpe_gpio->norequest_mask);
- if (stmpe_gpio->norequest_mask)
- stmpe_gpio->chip.irq.need_valid_mask = true;
irq = platform_get_irq(pdev, 0);
if (irq < 0)
@@ -487,14 +511,6 @@ static int stmpe_gpio_probe(struct platform_device *pdev)
dev_err(&pdev->dev, "unable to get irq: %d\n", ret);
goto out_disable;
}
- if (stmpe_gpio->norequest_mask) {
- int i;
-
- /* Forbid unused lines to be mapped as IRQs */
- for (i = 0; i < sizeof(u32); i++)
- if (stmpe_gpio->norequest_mask & BIT(i))
- clear_bit(i, stmpe_gpio->chip.irq.valid_mask);
- }
ret = gpiochip_irqchip_add_nested(&stmpe_gpio->chip,
&stmpe_gpio_irq_chip,
0,
diff --git a/drivers/gpio/gpio-stp-xway.c b/drivers/gpio/gpio-stp-xway.c
index 24c478392394..9e23a5ae8108 100644
--- a/drivers/gpio/gpio-stp-xway.c
+++ b/drivers/gpio/gpio-stp-xway.c
@@ -15,8 +15,6 @@
#include <linux/clk.h>
#include <linux/err.h>
-#include <lantiq_soc.h>
-
/*
* The Serial To Parallel (STP) is found on MIPS based Lantiq socs. It is a
* peripheral controller used to drive external shift register cascades. At most
@@ -71,8 +69,7 @@
#define xway_stp_r32(m, reg) __raw_readl(m + reg)
#define xway_stp_w32(m, val, reg) __raw_writel(val, m + reg)
#define xway_stp_w32_mask(m, clear, set, reg) \
- ltq_w32((ltq_r32(m + reg) & ~(clear)) | (set), \
- m + reg)
+ xway_stp_w32(m, (xway_stp_r32(m, reg) & ~(clear)) | (set), reg)
struct xway_stp {
struct gpio_chip gc;
@@ -156,9 +153,9 @@ static int xway_stp_request(struct gpio_chip *gc, unsigned gpio)
/**
* xway_stp_hw_init() - Configure the STP unit and enable the clock gate
- * @virt: pointer to the remapped register range
+ * @chip: Pointer to the xway_stp chip structure
*/
-static int xway_stp_hw_init(struct xway_stp *chip)
+static void xway_stp_hw_init(struct xway_stp *chip)
{
/* sane defaults */
xway_stp_w32(chip->virt, 0, XWAY_STP_AR);
@@ -201,8 +198,6 @@ static int xway_stp_hw_init(struct xway_stp *chip)
if (chip->reserved)
xway_stp_w32_mask(chip->virt, XWAY_STP_UPD_MASK,
XWAY_STP_UPD_FPI, XWAY_STP_CON1);
-
- return 0;
}
static int xway_stp_probe(struct platform_device *pdev)
@@ -258,21 +253,27 @@ static int xway_stp_probe(struct platform_device *pdev)
if (!of_find_property(pdev->dev.of_node, "lantiq,rising", NULL))
chip->edge = XWAY_STP_FALLING;
- clk = clk_get(&pdev->dev, NULL);
+ clk = devm_clk_get(&pdev->dev, NULL);
if (IS_ERR(clk)) {
dev_err(&pdev->dev, "Failed to get clock\n");
return PTR_ERR(clk);
}
- clk_enable(clk);
- ret = xway_stp_hw_init(chip);
- if (!ret)
- ret = devm_gpiochip_add_data(&pdev->dev, &chip->gc, chip);
+ ret = clk_prepare_enable(clk);
+ if (ret)
+ return ret;
- if (!ret)
- dev_info(&pdev->dev, "Init done\n");
+ xway_stp_hw_init(chip);
- return ret;
+ ret = devm_gpiochip_add_data(&pdev->dev, &chip->gc, chip);
+ if (ret) {
+ clk_disable_unprepare(clk);
+ return ret;
+ }
+
+ dev_info(&pdev->dev, "Init done\n");
+
+ return 0;
}
static const struct of_device_id xway_stp_match[] = {
diff --git a/drivers/gpio/gpio-tb10x.c b/drivers/gpio/gpio-tb10x.c
index bd1f3f775ce9..5e375186f90e 100644
--- a/drivers/gpio/gpio-tb10x.c
+++ b/drivers/gpio/gpio-tb10x.c
@@ -171,10 +171,8 @@ static int tb10x_gpio_probe(struct platform_device *pdev)
struct irq_chip_generic *gc;
ret = platform_get_irq(pdev, 0);
- if (ret < 0) {
- dev_err(dev, "No interrupt specified.\n");
+ if (ret < 0)
return ret;
- }
tb10x_gpio->gc.to_irq = tb10x_gpio_to_irq;
tb10x_gpio->irq = ret;
diff --git a/drivers/gpio/gpio-tegra.c b/drivers/gpio/gpio-tegra.c
index f57bfc07ae22..8a01d3694b28 100644
--- a/drivers/gpio/gpio-tegra.c
+++ b/drivers/gpio/gpio-tegra.c
@@ -541,8 +541,8 @@ DEFINE_SHOW_ATTRIBUTE(tegra_dbg_gpio);
static void tegra_gpio_debuginit(struct tegra_gpio_info *tgi)
{
- (void) debugfs_create_file("tegra_gpio", 0444,
- NULL, tgi, &tegra_dbg_gpio_fops);
+ debugfs_create_file("tegra_gpio", 0444, NULL, tgi,
+ &tegra_dbg_gpio_fops);
}
#else
@@ -624,10 +624,8 @@ static int tegra_gpio_probe(struct platform_device *pdev)
for (i = 0; i < tgi->bank_count; i++) {
ret = platform_get_irq(pdev, i);
- if (ret < 0) {
- dev_err(&pdev->dev, "Missing IRQ resource: %d\n", ret);
+ if (ret < 0)
return ret;
- }
bank = &tgi->bank_info[i];
bank->bank = i;
diff --git a/drivers/gpio/gpio-thunderx.c b/drivers/gpio/gpio-thunderx.c
index 715371b5102a..ddad5c7ea617 100644
--- a/drivers/gpio/gpio-thunderx.c
+++ b/drivers/gpio/gpio-thunderx.c
@@ -53,7 +53,6 @@ struct thunderx_line {
struct thunderx_gpio {
struct gpio_chip chip;
u8 __iomem *register_base;
- struct irq_domain *irqd;
struct msix_entry *msix_entries; /* per line MSI-X */
struct thunderx_line *line_entries; /* per line irq info */
raw_spinlock_t lock;
@@ -283,54 +282,60 @@ static void thunderx_gpio_set_multiple(struct gpio_chip *chip,
}
}
-static void thunderx_gpio_irq_ack(struct irq_data *data)
+static void thunderx_gpio_irq_ack(struct irq_data *d)
{
- struct thunderx_line *txline = irq_data_get_irq_chip_data(data);
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct thunderx_gpio *txgpio = gpiochip_get_data(gc);
writeq(GPIO_INTR_INTR,
- txline->txgpio->register_base + intr_reg(txline->line));
+ txgpio->register_base + intr_reg(irqd_to_hwirq(d)));
}
-static void thunderx_gpio_irq_mask(struct irq_data *data)
+static void thunderx_gpio_irq_mask(struct irq_data *d)
{
- struct thunderx_line *txline = irq_data_get_irq_chip_data(data);
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct thunderx_gpio *txgpio = gpiochip_get_data(gc);
writeq(GPIO_INTR_ENA_W1C,
- txline->txgpio->register_base + intr_reg(txline->line));
+ txgpio->register_base + intr_reg(irqd_to_hwirq(d)));
}
-static void thunderx_gpio_irq_mask_ack(struct irq_data *data)
+static void thunderx_gpio_irq_mask_ack(struct irq_data *d)
{
- struct thunderx_line *txline = irq_data_get_irq_chip_data(data);
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct thunderx_gpio *txgpio = gpiochip_get_data(gc);
writeq(GPIO_INTR_ENA_W1C | GPIO_INTR_INTR,
- txline->txgpio->register_base + intr_reg(txline->line));
+ txgpio->register_base + intr_reg(irqd_to_hwirq(d)));
}
-static void thunderx_gpio_irq_unmask(struct irq_data *data)
+static void thunderx_gpio_irq_unmask(struct irq_data *d)
{
- struct thunderx_line *txline = irq_data_get_irq_chip_data(data);
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct thunderx_gpio *txgpio = gpiochip_get_data(gc);
writeq(GPIO_INTR_ENA_W1S,
- txline->txgpio->register_base + intr_reg(txline->line));
+ txgpio->register_base + intr_reg(irqd_to_hwirq(d)));
}
-static int thunderx_gpio_irq_set_type(struct irq_data *data,
+static int thunderx_gpio_irq_set_type(struct irq_data *d,
unsigned int flow_type)
{
- struct thunderx_line *txline = irq_data_get_irq_chip_data(data);
- struct thunderx_gpio *txgpio = txline->txgpio;
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct thunderx_gpio *txgpio = gpiochip_get_data(gc);
+ struct thunderx_line *txline =
+ &txgpio->line_entries[irqd_to_hwirq(d)];
u64 bit_cfg;
- irqd_set_trigger_type(data, flow_type);
+ irqd_set_trigger_type(d, flow_type);
bit_cfg = txline->fil_bits | GPIO_BIT_CFG_INT_EN;
if (flow_type & IRQ_TYPE_EDGE_BOTH) {
- irq_set_handler_locked(data, handle_fasteoi_ack_irq);
+ irq_set_handler_locked(d, handle_fasteoi_ack_irq);
bit_cfg |= GPIO_BIT_CFG_INT_TYPE;
} else {
- irq_set_handler_locked(data, handle_fasteoi_mask_irq);
+ irq_set_handler_locked(d, handle_fasteoi_mask_irq);
}
raw_spin_lock(&txgpio->lock);
@@ -359,33 +364,6 @@ static void thunderx_gpio_irq_disable(struct irq_data *data)
irq_chip_disable_parent(data);
}
-static int thunderx_gpio_irq_request_resources(struct irq_data *data)
-{
- struct thunderx_line *txline = irq_data_get_irq_chip_data(data);
- struct thunderx_gpio *txgpio = txline->txgpio;
- int r;
-
- r = gpiochip_lock_as_irq(&txgpio->chip, txline->line);
- if (r)
- return r;
-
- r = irq_chip_request_resources_parent(data);
- if (r)
- gpiochip_unlock_as_irq(&txgpio->chip, txline->line);
-
- return r;
-}
-
-static void thunderx_gpio_irq_release_resources(struct irq_data *data)
-{
- struct thunderx_line *txline = irq_data_get_irq_chip_data(data);
- struct thunderx_gpio *txgpio = txline->txgpio;
-
- irq_chip_release_resources_parent(data);
-
- gpiochip_unlock_as_irq(&txgpio->chip, txline->line);
-}
-
/*
* Interrupts are chained from underlying MSI-X vectors. We have
* these irq_chip functions to be able to handle level triggering
@@ -402,48 +380,22 @@ static struct irq_chip thunderx_gpio_irq_chip = {
.irq_unmask = thunderx_gpio_irq_unmask,
.irq_eoi = irq_chip_eoi_parent,
.irq_set_affinity = irq_chip_set_affinity_parent,
- .irq_request_resources = thunderx_gpio_irq_request_resources,
- .irq_release_resources = thunderx_gpio_irq_release_resources,
.irq_set_type = thunderx_gpio_irq_set_type,
.flags = IRQCHIP_SET_TYPE_MASKED
};
-static int thunderx_gpio_irq_translate(struct irq_domain *d,
- struct irq_fwspec *fwspec,
- irq_hw_number_t *hwirq,
- unsigned int *type)
+static int thunderx_gpio_child_to_parent_hwirq(struct gpio_chip *gc,
+ unsigned int child,
+ unsigned int child_type,
+ unsigned int *parent,
+ unsigned int *parent_type)
{
- struct thunderx_gpio *txgpio = d->host_data;
-
- if (WARN_ON(fwspec->param_count < 2))
- return -EINVAL;
- if (fwspec->param[0] >= txgpio->chip.ngpio)
- return -EINVAL;
- *hwirq = fwspec->param[0];
- *type = fwspec->param[1] & IRQ_TYPE_SENSE_MASK;
- return 0;
-}
-
-static int thunderx_gpio_irq_alloc(struct irq_domain *d, unsigned int virq,
- unsigned int nr_irqs, void *arg)
-{
- struct thunderx_line *txline = arg;
+ struct thunderx_gpio *txgpio = gpiochip_get_data(gc);
- return irq_domain_set_hwirq_and_chip(d, virq, txline->line,
- &thunderx_gpio_irq_chip, txline);
-}
-
-static const struct irq_domain_ops thunderx_gpio_irqd_ops = {
- .alloc = thunderx_gpio_irq_alloc,
- .translate = thunderx_gpio_irq_translate
-};
-
-static int thunderx_gpio_to_irq(struct gpio_chip *chip, unsigned int offset)
-{
- struct thunderx_gpio *txgpio = gpiochip_get_data(chip);
-
- return irq_find_mapping(txgpio->irqd, offset);
+ *parent = txgpio->base_msi + (2 * child);
+ *parent_type = IRQ_TYPE_LEVEL_HIGH;
+ return 0;
}
static int thunderx_gpio_probe(struct pci_dev *pdev,
@@ -453,6 +405,7 @@ static int thunderx_gpio_probe(struct pci_dev *pdev,
struct device *dev = &pdev->dev;
struct thunderx_gpio *txgpio;
struct gpio_chip *chip;
+ struct gpio_irq_chip *girq;
int ngpio, i;
int err = 0;
@@ -497,8 +450,8 @@ static int thunderx_gpio_probe(struct pci_dev *pdev,
}
txgpio->msix_entries = devm_kcalloc(dev,
- ngpio, sizeof(struct msix_entry),
- GFP_KERNEL);
+ ngpio, sizeof(struct msix_entry),
+ GFP_KERNEL);
if (!txgpio->msix_entries) {
err = -ENOMEM;
goto out;
@@ -539,27 +492,6 @@ static int thunderx_gpio_probe(struct pci_dev *pdev,
if (err < 0)
goto out;
- /*
- * Push GPIO specific irqdomain on hierarchy created as a side
- * effect of the pci_enable_msix()
- */
- txgpio->irqd = irq_domain_create_hierarchy(irq_get_irq_data(txgpio->msix_entries[0].vector)->domain,
- 0, 0, of_node_to_fwnode(dev->of_node),
- &thunderx_gpio_irqd_ops, txgpio);
- if (!txgpio->irqd) {
- err = -ENOMEM;
- goto out;
- }
-
- /* Push on irq_data and the domain for each line. */
- for (i = 0; i < ngpio; i++) {
- err = irq_domain_push_irq(txgpio->irqd,
- txgpio->msix_entries[i].vector,
- &txgpio->line_entries[i]);
- if (err < 0)
- dev_err(dev, "irq_domain_push_irq: %d\n", err);
- }
-
chip->label = KBUILD_MODNAME;
chip->parent = dev;
chip->owner = THIS_MODULE;
@@ -574,11 +506,28 @@ static int thunderx_gpio_probe(struct pci_dev *pdev,
chip->set = thunderx_gpio_set;
chip->set_multiple = thunderx_gpio_set_multiple;
chip->set_config = thunderx_gpio_set_config;
- chip->to_irq = thunderx_gpio_to_irq;
+ girq = &chip->irq;
+ girq->chip = &thunderx_gpio_irq_chip;
+ girq->fwnode = of_node_to_fwnode(dev->of_node);
+ girq->parent_domain =
+ irq_get_irq_data(txgpio->msix_entries[0].vector)->domain;
+ girq->child_to_parent_hwirq = thunderx_gpio_child_to_parent_hwirq;
+ girq->handler = handle_bad_irq;
+ girq->default_type = IRQ_TYPE_NONE;
+
err = devm_gpiochip_add_data(dev, chip, txgpio);
if (err)
goto out;
+ /* Push on irq_data and the domain for each line. */
+ for (i = 0; i < ngpio; i++) {
+ err = irq_domain_push_irq(chip->irq.domain,
+ txgpio->msix_entries[i].vector,
+ chip);
+ if (err < 0)
+ dev_err(dev, "irq_domain_push_irq: %d\n", err);
+ }
+
dev_info(dev, "ThunderX GPIO: %d lines with base %d.\n",
ngpio, chip->base);
return 0;
@@ -593,10 +542,10 @@ static void thunderx_gpio_remove(struct pci_dev *pdev)
struct thunderx_gpio *txgpio = pci_get_drvdata(pdev);
for (i = 0; i < txgpio->chip.ngpio; i++)
- irq_domain_pop_irq(txgpio->irqd,
+ irq_domain_pop_irq(txgpio->chip.irq.domain,
txgpio->msix_entries[i].vector);
- irq_domain_remove(txgpio->irqd);
+ irq_domain_remove(txgpio->chip.irq.domain);
pci_set_drvdata(pdev, NULL);
}
diff --git a/drivers/gpio/gpio-tqmx86.c b/drivers/gpio/gpio-tqmx86.c
index d5880db7f9d4..a3109bcaa0ac 100644
--- a/drivers/gpio/gpio-tqmx86.c
+++ b/drivers/gpio/gpio-tqmx86.c
@@ -214,11 +214,23 @@ static const struct dev_pm_ops tqmx86_gpio_dev_pm_ops = {
tqmx86_gpio_runtime_resume, NULL)
};
+static void tqmx86_init_irq_valid_mask(struct gpio_chip *chip,
+ unsigned long *valid_mask,
+ unsigned int ngpios)
+{
+ /* Only GPIOs 4-7 are valid for interrupts. Clear the others */
+ clear_bit(0, valid_mask);
+ clear_bit(1, valid_mask);
+ clear_bit(2, valid_mask);
+ clear_bit(3, valid_mask);
+}
+
static int tqmx86_gpio_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct tqmx86_gpio_data *gpio;
struct gpio_chip *chip;
+ struct gpio_irq_chip *girq;
void __iomem *io_base;
struct resource *res;
int ret, irq;
@@ -259,17 +271,10 @@ static int tqmx86_gpio_probe(struct platform_device *pdev)
chip->get = tqmx86_gpio_get;
chip->set = tqmx86_gpio_set;
chip->ngpio = TQMX86_NGPIO;
- chip->irq.need_valid_mask = true;
chip->parent = pdev->dev.parent;
pm_runtime_enable(&pdev->dev);
- ret = devm_gpiochip_add_data(dev, chip, gpio);
- if (ret) {
- dev_err(dev, "Could not register GPIO chip\n");
- goto out_pm_dis;
- }
-
if (irq) {
struct irq_chip *irq_chip = &gpio->irq_chip;
u8 irq_status;
@@ -287,23 +292,28 @@ static int tqmx86_gpio_probe(struct platform_device *pdev)
irq_status = tqmx86_gpio_read(gpio, TQMX86_GPIIS);
tqmx86_gpio_write(gpio, irq_status, TQMX86_GPIIS);
- ret = gpiochip_irqchip_add(chip, irq_chip,
- 0, handle_simple_irq,
- IRQ_TYPE_EDGE_BOTH);
- if (ret) {
- dev_err(dev, "Could not add irq chip\n");
+ girq = &chip->irq;
+ girq->chip = irq_chip;
+ girq->parent_handler = tqmx86_gpio_irq_handler;
+ girq->num_parents = 1;
+ girq->parents = devm_kcalloc(&pdev->dev, 1,
+ sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents) {
+ ret = -ENOMEM;
goto out_pm_dis;
}
-
- gpiochip_set_chained_irqchip(chip, irq_chip,
- irq, tqmx86_gpio_irq_handler);
+ girq->parents[0] = irq;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_simple_irq;
+ girq->init_valid_mask = tqmx86_init_irq_valid_mask;
}
- /* Only GPIOs 4-7 are valid for interrupts. Clear the others */
- clear_bit(0, chip->irq.valid_mask);
- clear_bit(1, chip->irq.valid_mask);
- clear_bit(2, chip->irq.valid_mask);
- clear_bit(3, chip->irq.valid_mask);
+ ret = devm_gpiochip_add_data(dev, chip, gpio);
+ if (ret) {
+ dev_err(dev, "Could not register GPIO chip\n");
+ goto out_pm_dis;
+ }
dev_info(dev, "GPIO functionality initialized with %d pins\n",
chip->ngpio);
diff --git a/drivers/gpio/gpio-vf610.c b/drivers/gpio/gpio-vf610.c
index 30aef41e3b7e..58776f2d69ff 100644
--- a/drivers/gpio/gpio-vf610.c
+++ b/drivers/gpio/gpio-vf610.c
@@ -243,6 +243,7 @@ static int vf610_gpio_probe(struct platform_device *pdev)
struct device_node *np = dev->of_node;
struct vf610_gpio_port *port;
struct gpio_chip *gc;
+ struct gpio_irq_chip *girq;
struct irq_chip *ic;
int i;
int ret;
@@ -265,7 +266,8 @@ static int vf610_gpio_probe(struct platform_device *pdev)
return port->irq;
port->clk_port = devm_clk_get(dev, "port");
- if (!IS_ERR(port->clk_port)) {
+ ret = PTR_ERR_OR_ZERO(port->clk_port);
+ if (!ret) {
ret = clk_prepare_enable(port->clk_port);
if (ret)
return ret;
@@ -273,16 +275,17 @@ static int vf610_gpio_probe(struct platform_device *pdev)
port->clk_port);
if (ret)
return ret;
- } else if (port->clk_port == ERR_PTR(-EPROBE_DEFER)) {
+ } else if (ret == -EPROBE_DEFER) {
/*
* Percolate deferrals, for anything else,
* just live without the clocking.
*/
- return PTR_ERR(port->clk_port);
+ return ret;
}
port->clk_gpio = devm_clk_get(dev, "gpio");
- if (!IS_ERR(port->clk_gpio)) {
+ ret = PTR_ERR_OR_ZERO(port->clk_gpio);
+ if (!ret) {
ret = clk_prepare_enable(port->clk_gpio);
if (ret)
return ret;
@@ -290,8 +293,8 @@ static int vf610_gpio_probe(struct platform_device *pdev)
port->clk_gpio);
if (ret)
return ret;
- } else if (port->clk_gpio == ERR_PTR(-EPROBE_DEFER)) {
- return PTR_ERR(port->clk_gpio);
+ } else if (ret == -EPROBE_DEFER) {
+ return ret;
}
gc = &port->gc;
@@ -316,10 +319,6 @@ static int vf610_gpio_probe(struct platform_device *pdev)
ic->irq_set_type = vf610_gpio_irq_set_type;
ic->irq_set_wake = vf610_gpio_irq_set_wake;
- ret = devm_gpiochip_add_data(dev, gc, port);
- if (ret < 0)
- return ret;
-
/* Mask all GPIO interrupts */
for (i = 0; i < gc->ngpio; i++)
vf610_gpio_writel(0, port->base + PORT_PCR(i));
@@ -327,15 +326,20 @@ static int vf610_gpio_probe(struct platform_device *pdev)
/* Clear the interrupt status register for all GPIO's */
vf610_gpio_writel(~0, port->base + PORT_ISFR);
- ret = gpiochip_irqchip_add(gc, ic, 0, handle_edge_irq, IRQ_TYPE_NONE);
- if (ret) {
- dev_err(dev, "failed to add irqchip\n");
- return ret;
- }
- gpiochip_set_chained_irqchip(gc, ic, port->irq,
- vf610_gpio_irq_handler);
+ girq = &gc->irq;
+ girq->chip = ic;
+ girq->parent_handler = vf610_gpio_irq_handler;
+ girq->num_parents = 1;
+ girq->parents = devm_kcalloc(&pdev->dev, 1,
+ sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents)
+ return -ENOMEM;
+ girq->parents[0] = port->irq;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_edge_irq;
- return 0;
+ return devm_gpiochip_add_data(dev, gc, port);
}
static struct platform_driver vf610_gpio_driver = {
diff --git a/drivers/gpio/gpio-viperboard.c b/drivers/gpio/gpio-viperboard.c
index 9b604f13e302..c301c1d56dd2 100644
--- a/drivers/gpio/gpio-viperboard.c
+++ b/drivers/gpio/gpio-viperboard.c
@@ -79,7 +79,7 @@ MODULE_PARM_DESC(gpioa_freq,
/* ----- begin of gipo a chip -------------------------------------------- */
static int vprbrd_gpioa_get(struct gpio_chip *chip,
- unsigned offset)
+ unsigned int offset)
{
int ret, answer, error = 0;
struct vprbrd_gpio *gpio = gpiochip_get_data(chip);
@@ -129,7 +129,7 @@ static int vprbrd_gpioa_get(struct gpio_chip *chip,
}
static void vprbrd_gpioa_set(struct gpio_chip *chip,
- unsigned offset, int value)
+ unsigned int offset, int value)
{
int ret;
struct vprbrd_gpio *gpio = gpiochip_get_data(chip);
@@ -170,7 +170,7 @@ static void vprbrd_gpioa_set(struct gpio_chip *chip,
}
static int vprbrd_gpioa_direction_input(struct gpio_chip *chip,
- unsigned offset)
+ unsigned int offset)
{
int ret;
struct vprbrd_gpio *gpio = gpiochip_get_data(chip);
@@ -207,7 +207,7 @@ static int vprbrd_gpioa_direction_input(struct gpio_chip *chip,
}
static int vprbrd_gpioa_direction_output(struct gpio_chip *chip,
- unsigned offset, int value)
+ unsigned int offset, int value)
{
int ret;
struct vprbrd_gpio *gpio = gpiochip_get_data(chip);
@@ -251,8 +251,8 @@ static int vprbrd_gpioa_direction_output(struct gpio_chip *chip,
/* ----- begin of gipo b chip -------------------------------------------- */
-static int vprbrd_gpiob_setdir(struct vprbrd *vb, unsigned offset,
- unsigned dir)
+static int vprbrd_gpiob_setdir(struct vprbrd *vb, unsigned int offset,
+ unsigned int dir)
{
struct vprbrd_gpiob_msg *gbmsg = (struct vprbrd_gpiob_msg *)vb->buf;
int ret;
@@ -273,7 +273,7 @@ static int vprbrd_gpiob_setdir(struct vprbrd *vb, unsigned offset,
}
static int vprbrd_gpiob_get(struct gpio_chip *chip,
- unsigned offset)
+ unsigned int offset)
{
int ret;
u16 val;
@@ -305,7 +305,7 @@ static int vprbrd_gpiob_get(struct gpio_chip *chip,
}
static void vprbrd_gpiob_set(struct gpio_chip *chip,
- unsigned offset, int value)
+ unsigned int offset, int value)
{
int ret;
struct vprbrd_gpio *gpio = gpiochip_get_data(chip);
@@ -338,7 +338,7 @@ static void vprbrd_gpiob_set(struct gpio_chip *chip,
}
static int vprbrd_gpiob_direction_input(struct gpio_chip *chip,
- unsigned offset)
+ unsigned int offset)
{
int ret;
struct vprbrd_gpio *gpio = gpiochip_get_data(chip);
@@ -359,7 +359,7 @@ static int vprbrd_gpiob_direction_input(struct gpio_chip *chip,
}
static int vprbrd_gpiob_direction_output(struct gpio_chip *chip,
- unsigned offset, int value)
+ unsigned int offset, int value)
{
int ret;
struct vprbrd_gpio *gpio = gpiochip_get_data(chip);
diff --git a/drivers/gpio/gpio-vr41xx.c b/drivers/gpio/gpio-vr41xx.c
index b13a49c89cc1..98cd715ccc33 100644
--- a/drivers/gpio/gpio-vr41xx.c
+++ b/drivers/gpio/gpio-vr41xx.c
@@ -467,10 +467,9 @@ static struct gpio_chip vr41xx_gpio_chip = {
static int giu_probe(struct platform_device *pdev)
{
- struct resource *res;
unsigned int trigger, i, pin;
struct irq_chip *chip;
- int irq, ret;
+ int irq;
switch (pdev->id) {
case GPIO_50PINS_PULLUPDOWN:
@@ -489,21 +488,14 @@ static int giu_probe(struct platform_device *pdev)
return -ENODEV;
}
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- if (!res)
- return -EBUSY;
-
- giu_base = ioremap(res->start, resource_size(res));
- if (!giu_base)
- return -ENOMEM;
+ giu_base = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(giu_base))
+ return PTR_ERR(giu_base);
vr41xx_gpio_chip.parent = &pdev->dev;
- ret = gpiochip_add_data(&vr41xx_gpio_chip, NULL);
- if (!ret) {
- iounmap(giu_base);
+ if (gpiochip_add_data(&vr41xx_gpio_chip, NULL))
return -ENODEV;
- }
giu_write(GIUINTENL, 0);
giu_write(GIUINTENH, 0);
@@ -534,7 +526,6 @@ static int giu_probe(struct platform_device *pdev)
static int giu_remove(struct platform_device *pdev)
{
if (giu_base) {
- iounmap(giu_base);
giu_base = NULL;
}
diff --git a/drivers/gpio/gpio-xgene-sb.c b/drivers/gpio/gpio-xgene-sb.c
index 38c01912c7b2..25d86441666e 100644
--- a/drivers/gpio/gpio-xgene-sb.c
+++ b/drivers/gpio/gpio-xgene-sb.c
@@ -16,6 +16,7 @@
#include <linux/acpi.h>
#include "gpiolib.h"
+#include "gpiolib-acpi.h"
/* Common property names */
#define XGENE_NIRQ_PROPERTY "apm,nr-irqs"
diff --git a/drivers/gpio/gpio-xilinx.c b/drivers/gpio/gpio-xilinx.c
index 32944eb886c1..a9748b5198e6 100644
--- a/drivers/gpio/gpio-xilinx.c
+++ b/drivers/gpio/gpio-xilinx.c
@@ -11,7 +11,6 @@
#include <linux/module.h>
#include <linux/of_device.h>
#include <linux/of_platform.h>
-#include <linux/of_gpio.h>
#include <linux/io.h>
#include <linux/gpio/driver.h>
#include <linux/slab.h>
@@ -33,14 +32,16 @@
/**
* struct xgpio_instance - Stores information about GPIO device
- * @mmchip: OF GPIO chip for memory mapped banks
+ * @gc: GPIO chip
+ * @regs: register block
* @gpio_width: GPIO width for every channel
* @gpio_state: GPIO state shadow register
* @gpio_dir: GPIO direction shadow register
* @gpio_lock: Lock used for synchronization
*/
struct xgpio_instance {
- struct of_mm_gpio_chip mmchip;
+ struct gpio_chip gc;
+ void __iomem *regs;
unsigned int gpio_width[2];
u32 gpio_state[2];
u32 gpio_dir[2];
@@ -84,11 +85,10 @@ static inline int xgpio_offset(struct xgpio_instance *chip, int gpio)
*/
static int xgpio_get(struct gpio_chip *gc, unsigned int gpio)
{
- struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
struct xgpio_instance *chip = gpiochip_get_data(gc);
u32 val;
- val = xgpio_readreg(mm_gc->regs + XGPIO_DATA_OFFSET +
+ val = xgpio_readreg(chip->regs + XGPIO_DATA_OFFSET +
xgpio_regoffset(chip, gpio));
return !!(val & BIT(xgpio_offset(chip, gpio)));
@@ -106,7 +106,6 @@ static int xgpio_get(struct gpio_chip *gc, unsigned int gpio)
static void xgpio_set(struct gpio_chip *gc, unsigned int gpio, int val)
{
unsigned long flags;
- struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
struct xgpio_instance *chip = gpiochip_get_data(gc);
int index = xgpio_index(chip, gpio);
int offset = xgpio_offset(chip, gpio);
@@ -119,7 +118,7 @@ static void xgpio_set(struct gpio_chip *gc, unsigned int gpio, int val)
else
chip->gpio_state[index] &= ~BIT(offset);
- xgpio_writereg(mm_gc->regs + XGPIO_DATA_OFFSET +
+ xgpio_writereg(chip->regs + XGPIO_DATA_OFFSET +
xgpio_regoffset(chip, gpio), chip->gpio_state[index]);
spin_unlock_irqrestore(&chip->gpio_lock[index], flags);
@@ -138,7 +137,6 @@ static void xgpio_set_multiple(struct gpio_chip *gc, unsigned long *mask,
unsigned long *bits)
{
unsigned long flags;
- struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
struct xgpio_instance *chip = gpiochip_get_data(gc);
int index = xgpio_index(chip, 0);
int offset, i;
@@ -150,7 +148,7 @@ static void xgpio_set_multiple(struct gpio_chip *gc, unsigned long *mask,
if (*mask == 0)
break;
if (index != xgpio_index(chip, i)) {
- xgpio_writereg(mm_gc->regs + XGPIO_DATA_OFFSET +
+ xgpio_writereg(chip->regs + XGPIO_DATA_OFFSET +
xgpio_regoffset(chip, i),
chip->gpio_state[index]);
spin_unlock_irqrestore(&chip->gpio_lock[index], flags);
@@ -166,7 +164,7 @@ static void xgpio_set_multiple(struct gpio_chip *gc, unsigned long *mask,
}
}
- xgpio_writereg(mm_gc->regs + XGPIO_DATA_OFFSET +
+ xgpio_writereg(chip->regs + XGPIO_DATA_OFFSET +
xgpio_regoffset(chip, i), chip->gpio_state[index]);
spin_unlock_irqrestore(&chip->gpio_lock[index], flags);
@@ -184,7 +182,6 @@ static void xgpio_set_multiple(struct gpio_chip *gc, unsigned long *mask,
static int xgpio_dir_in(struct gpio_chip *gc, unsigned int gpio)
{
unsigned long flags;
- struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
struct xgpio_instance *chip = gpiochip_get_data(gc);
int index = xgpio_index(chip, gpio);
int offset = xgpio_offset(chip, gpio);
@@ -193,7 +190,7 @@ static int xgpio_dir_in(struct gpio_chip *gc, unsigned int gpio)
/* Set the GPIO bit in shadow register and set direction as input */
chip->gpio_dir[index] |= BIT(offset);
- xgpio_writereg(mm_gc->regs + XGPIO_TRI_OFFSET +
+ xgpio_writereg(chip->regs + XGPIO_TRI_OFFSET +
xgpio_regoffset(chip, gpio), chip->gpio_dir[index]);
spin_unlock_irqrestore(&chip->gpio_lock[index], flags);
@@ -216,7 +213,6 @@ static int xgpio_dir_in(struct gpio_chip *gc, unsigned int gpio)
static int xgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val)
{
unsigned long flags;
- struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc);
struct xgpio_instance *chip = gpiochip_get_data(gc);
int index = xgpio_index(chip, gpio);
int offset = xgpio_offset(chip, gpio);
@@ -228,12 +224,12 @@ static int xgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val)
chip->gpio_state[index] |= BIT(offset);
else
chip->gpio_state[index] &= ~BIT(offset);
- xgpio_writereg(mm_gc->regs + XGPIO_DATA_OFFSET +
+ xgpio_writereg(chip->regs + XGPIO_DATA_OFFSET +
xgpio_regoffset(chip, gpio), chip->gpio_state[index]);
/* Clear the GPIO bit in shadow register and set direction as output */
chip->gpio_dir[index] &= ~BIT(offset);
- xgpio_writereg(mm_gc->regs + XGPIO_TRI_OFFSET +
+ xgpio_writereg(chip->regs + XGPIO_TRI_OFFSET +
xgpio_regoffset(chip, gpio), chip->gpio_dir[index]);
spin_unlock_irqrestore(&chip->gpio_lock[index], flags);
@@ -243,43 +239,23 @@ static int xgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val)
/**
* xgpio_save_regs - Set initial values of GPIO pins
- * @mm_gc: Pointer to memory mapped GPIO chip structure
+ * @chip: Pointer to GPIO instance
*/
-static void xgpio_save_regs(struct of_mm_gpio_chip *mm_gc)
+static void xgpio_save_regs(struct xgpio_instance *chip)
{
- struct xgpio_instance *chip =
- container_of(mm_gc, struct xgpio_instance, mmchip);
-
- xgpio_writereg(mm_gc->regs + XGPIO_DATA_OFFSET, chip->gpio_state[0]);
- xgpio_writereg(mm_gc->regs + XGPIO_TRI_OFFSET, chip->gpio_dir[0]);
+ xgpio_writereg(chip->regs + XGPIO_DATA_OFFSET, chip->gpio_state[0]);
+ xgpio_writereg(chip->regs + XGPIO_TRI_OFFSET, chip->gpio_dir[0]);
if (!chip->gpio_width[1])
return;
- xgpio_writereg(mm_gc->regs + XGPIO_DATA_OFFSET + XGPIO_CHANNEL_OFFSET,
+ xgpio_writereg(chip->regs + XGPIO_DATA_OFFSET + XGPIO_CHANNEL_OFFSET,
chip->gpio_state[1]);
- xgpio_writereg(mm_gc->regs + XGPIO_TRI_OFFSET + XGPIO_CHANNEL_OFFSET,
+ xgpio_writereg(chip->regs + XGPIO_TRI_OFFSET + XGPIO_CHANNEL_OFFSET,
chip->gpio_dir[1]);
}
/**
- * xgpio_remove - Remove method for the GPIO device.
- * @pdev: pointer to the platform device
- *
- * This function remove gpiochips and frees all the allocated resources.
- *
- * Return: 0 always
- */
-static int xgpio_remove(struct platform_device *pdev)
-{
- struct xgpio_instance *chip = platform_get_drvdata(pdev);
-
- of_mm_gpiochip_remove(&chip->mmchip);
-
- return 0;
-}
-
-/**
* xgpio_of_probe - Probe method for the GPIO device.
* @pdev: pointer to the platform device
*
@@ -340,21 +316,28 @@ static int xgpio_probe(struct platform_device *pdev)
spin_lock_init(&chip->gpio_lock[1]);
}
- chip->mmchip.gc.ngpio = chip->gpio_width[0] + chip->gpio_width[1];
- chip->mmchip.gc.parent = &pdev->dev;
- chip->mmchip.gc.direction_input = xgpio_dir_in;
- chip->mmchip.gc.direction_output = xgpio_dir_out;
- chip->mmchip.gc.get = xgpio_get;
- chip->mmchip.gc.set = xgpio_set;
- chip->mmchip.gc.set_multiple = xgpio_set_multiple;
+ chip->gc.base = -1;
+ chip->gc.ngpio = chip->gpio_width[0] + chip->gpio_width[1];
+ chip->gc.parent = &pdev->dev;
+ chip->gc.direction_input = xgpio_dir_in;
+ chip->gc.direction_output = xgpio_dir_out;
+ chip->gc.get = xgpio_get;
+ chip->gc.set = xgpio_set;
+ chip->gc.set_multiple = xgpio_set_multiple;
+
+ chip->gc.label = dev_name(&pdev->dev);
+
+ chip->regs = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(chip->regs)) {
+ dev_err(&pdev->dev, "failed to ioremap memory resource\n");
+ return PTR_ERR(chip->regs);
+ }
- chip->mmchip.save_regs = xgpio_save_regs;
+ xgpio_save_regs(chip);
- /* Call the OF gpio helper to setup and register the GPIO device */
- status = of_mm_gpiochip_add_data(np, &chip->mmchip, chip);
+ status = devm_gpiochip_add_data(&pdev->dev, &chip->gc, chip);
if (status) {
- pr_err("%pOF: error in probe function with status %d\n",
- np, status);
+ dev_err(&pdev->dev, "failed to add GPIO chip\n");
return status;
}
@@ -370,7 +353,6 @@ MODULE_DEVICE_TABLE(of, xgpio_of_match);
static struct platform_driver xgpio_plat_driver = {
.probe = xgpio_probe,
- .remove = xgpio_remove,
.driver = {
.name = "gpio-xilinx",
.of_match_table = xgpio_of_match,
diff --git a/drivers/gpio/gpio-xlp.c b/drivers/gpio/gpio-xlp.c
index 54d3359444f3..d7b16bb9e4e4 100644
--- a/drivers/gpio/gpio-xlp.c
+++ b/drivers/gpio/gpio-xlp.c
@@ -290,6 +290,7 @@ MODULE_DEVICE_TABLE(of, xlp_gpio_of_ids);
static int xlp_gpio_probe(struct platform_device *pdev)
{
struct gpio_chip *gc;
+ struct gpio_irq_chip *girq;
struct xlp_gpio_priv *priv;
void __iomem *gpio_base;
int irq_base, irq, err;
@@ -395,27 +396,27 @@ static int xlp_gpio_probe(struct platform_device *pdev)
irq_base = 0;
}
+ girq = &gc->irq;
+ girq->chip = &xlp_gpio_irq_chip;
+ girq->parent_handler = xlp_gpio_generic_handler;
+ girq->num_parents = 1;
+ girq->parents = devm_kcalloc(&pdev->dev, 1,
+ sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents)
+ return -ENOMEM;
+ girq->parents[0] = irq;
+ girq->first = irq_base;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_level_irq;
+
err = gpiochip_add_data(gc, priv);
if (err < 0)
return err;
- err = gpiochip_irqchip_add(gc, &xlp_gpio_irq_chip, irq_base,
- handle_level_irq, IRQ_TYPE_NONE);
- if (err) {
- dev_err(&pdev->dev, "Could not connect irqchip to gpiochip!\n");
- goto out_gpio_remove;
- }
-
- gpiochip_set_chained_irqchip(gc, &xlp_gpio_irq_chip, irq,
- xlp_gpio_generic_handler);
-
dev_info(&pdev->dev, "registered %d GPIOs\n", gc->ngpio);
return 0;
-
-out_gpio_remove:
- gpiochip_remove(gc);
- return err;
}
#ifdef CONFIG_ACPI
diff --git a/drivers/gpio/gpio-zx.c b/drivers/gpio/gpio-zx.c
index 8637adb6bc20..98cbaf0e415e 100644
--- a/drivers/gpio/gpio-zx.c
+++ b/drivers/gpio/gpio-zx.c
@@ -215,6 +215,7 @@ static int zx_gpio_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct zx_gpio *chip;
+ struct gpio_irq_chip *girq;
int irq, id, ret;
chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL);
@@ -242,32 +243,30 @@ static int zx_gpio_probe(struct platform_device *pdev)
chip->gc.parent = dev;
chip->gc.owner = THIS_MODULE;
- ret = gpiochip_add_data(&chip->gc, chip);
- if (ret)
- return ret;
-
/*
* irq_chip support
*/
writew_relaxed(0xffff, chip->base + ZX_GPIO_IM);
writew_relaxed(0, chip->base + ZX_GPIO_IE);
irq = platform_get_irq(pdev, 0);
- if (irq < 0) {
- dev_err(dev, "invalid IRQ\n");
- gpiochip_remove(&chip->gc);
- return -ENODEV;
- }
+ if (irq < 0)
+ return irq;
+ girq = &chip->gc.irq;
+ girq->chip = &zx_irqchip;
+ girq->parent_handler = zx_irq_handler;
+ girq->num_parents = 1;
+ girq->parents = devm_kcalloc(&pdev->dev, 1,
+ sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents)
+ return -ENOMEM;
+ girq->parents[0] = irq;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_simple_irq;
- ret = gpiochip_irqchip_add(&chip->gc, &zx_irqchip,
- 0, handle_simple_irq,
- IRQ_TYPE_NONE);
- if (ret) {
- dev_err(dev, "could not add irqchip\n");
- gpiochip_remove(&chip->gc);
+ ret = gpiochip_add_data(&chip->gc, chip);
+ if (ret)
return ret;
- }
- gpiochip_set_chained_irqchip(&chip->gc, &zx_irqchip,
- irq, zx_irq_handler);
platform_set_drvdata(pdev, chip);
dev_info(dev, "ZX GPIO chip registered\n");
diff --git a/drivers/gpio/gpio-zynq.c b/drivers/gpio/gpio-zynq.c
index f241b6c13dbe..cd475ff4bcad 100644
--- a/drivers/gpio/gpio-zynq.c
+++ b/drivers/gpio/gpio-zynq.c
@@ -830,6 +830,7 @@ static int zynq_gpio_probe(struct platform_device *pdev)
int ret, bank_num;
struct zynq_gpio *gpio;
struct gpio_chip *chip;
+ struct gpio_irq_chip *girq;
const struct of_device_id *match;
gpio = devm_kzalloc(&pdev->dev, sizeof(*gpio), GFP_KERNEL);
@@ -849,10 +850,8 @@ static int zynq_gpio_probe(struct platform_device *pdev)
return PTR_ERR(gpio->base_addr);
gpio->irq = platform_get_irq(pdev, 0);
- if (gpio->irq < 0) {
- dev_err(&pdev->dev, "invalid IRQ\n");
+ if (gpio->irq < 0)
return gpio->irq;
- }
/* configure the gpio chip */
chip = &gpio->chip;
@@ -887,34 +886,38 @@ static int zynq_gpio_probe(struct platform_device *pdev)
if (ret < 0)
goto err_pm_dis;
- /* report a bug if gpio chip registration fails */
- ret = gpiochip_add_data(chip, gpio);
- if (ret) {
- dev_err(&pdev->dev, "Failed to add gpio chip\n");
- goto err_pm_put;
- }
-
/* disable interrupts for all banks */
for (bank_num = 0; bank_num < gpio->p_data->max_bank; bank_num++)
writel_relaxed(ZYNQ_GPIO_IXR_DISABLE_ALL, gpio->base_addr +
ZYNQ_GPIO_INTDIS_OFFSET(bank_num));
- ret = gpiochip_irqchip_add(chip, &zynq_gpio_edge_irqchip, 0,
- handle_level_irq, IRQ_TYPE_NONE);
- if (ret) {
- dev_err(&pdev->dev, "Failed to add irq chip\n");
- goto err_rm_gpiochip;
+ /* Set up the GPIO irqchip */
+ girq = &chip->irq;
+ girq->chip = &zynq_gpio_edge_irqchip;
+ girq->parent_handler = zynq_gpio_irqhandler;
+ girq->num_parents = 1;
+ girq->parents = devm_kcalloc(&pdev->dev, 1,
+ sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents) {
+ ret = -ENOMEM;
+ goto err_pm_put;
}
+ girq->parents[0] = gpio->irq;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_level_irq;
- gpiochip_set_chained_irqchip(chip, &zynq_gpio_edge_irqchip, gpio->irq,
- zynq_gpio_irqhandler);
+ /* report a bug if gpio chip registration fails */
+ ret = gpiochip_add_data(chip, gpio);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed to add gpio chip\n");
+ goto err_pm_put;
+ }
pm_runtime_put(&pdev->dev);
return 0;
-err_rm_gpiochip:
- gpiochip_remove(chip);
err_pm_put:
pm_runtime_put(&pdev->dev);
err_pm_dis:
diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c
index c9fc9e232aaf..609ed16ae933 100644
--- a/drivers/gpio/gpiolib-acpi.c
+++ b/drivers/gpio/gpiolib-acpi.c
@@ -7,6 +7,7 @@
* Mika Westerberg <mika.westerberg@linux.intel.com>
*/
+#include <linux/dmi.h>
#include <linux/errno.h>
#include <linux/gpio/consumer.h>
#include <linux/gpio/driver.h>
@@ -18,6 +19,12 @@
#include <linux/pinctrl/pinctrl.h>
#include "gpiolib.h"
+#include "gpiolib-acpi.h"
+
+static int run_edge_events_on_boot = -1;
+module_param(run_edge_events_on_boot, int, 0444);
+MODULE_PARM_DESC(run_edge_events_on_boot,
+ "Run edge _AEI event-handlers at boot: 0=no, 1=yes, -1=auto");
/**
* struct acpi_gpio_event - ACPI GPIO event handler data
@@ -170,10 +177,13 @@ static void acpi_gpiochip_request_irq(struct acpi_gpio_chip *acpi_gpio,
event->irq_requested = true;
/* Make sure we trigger the initial state of edge-triggered IRQs */
- value = gpiod_get_raw_value_cansleep(event->desc);
- if (((event->irqflags & IRQF_TRIGGER_RISING) && value == 1) ||
- ((event->irqflags & IRQF_TRIGGER_FALLING) && value == 0))
- event->handler(event->irq, event);
+ if (run_edge_events_on_boot &&
+ (event->irqflags & (IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING))) {
+ value = gpiod_get_raw_value_cansleep(event->desc);
+ if (((event->irqflags & IRQF_TRIGGER_RISING) && value == 1) ||
+ ((event->irqflags & IRQF_TRIGGER_FALLING) && value == 0))
+ event->handler(event->irq, event);
+ }
}
static void acpi_gpiochip_request_irqs(struct acpi_gpio_chip *acpi_gpio)
@@ -217,14 +227,13 @@ static acpi_status acpi_gpiochip_alloc_event(struct acpi_resource *ares,
if (!handler)
return AE_OK;
- desc = gpiochip_request_own_desc(chip, pin, "ACPI:Event", 0);
+ desc = gpiochip_request_own_desc(chip, pin, "ACPI:Event",
+ GPIO_ACTIVE_HIGH, GPIOD_IN);
if (IS_ERR(desc)) {
dev_err(chip->parent, "Failed to request GPIO\n");
return AE_ERROR;
}
- gpiod_direction_input(desc);
-
ret = gpiochip_lock_as_irq(chip, pin);
if (ret) {
dev_err(chip->parent, "Failed to lock GPIO as interrupt\n");
@@ -383,6 +392,13 @@ int acpi_dev_add_driver_gpios(struct acpi_device *adev,
}
EXPORT_SYMBOL_GPL(acpi_dev_add_driver_gpios);
+void acpi_dev_remove_driver_gpios(struct acpi_device *adev)
+{
+ if (adev)
+ adev->driver_gpios = NULL;
+}
+EXPORT_SYMBOL_GPL(acpi_dev_remove_driver_gpios);
+
static void devm_acpi_dev_release_driver_gpios(struct device *dev, void *res)
{
acpi_dev_remove_driver_gpios(ACPI_COMPANION(dev));
@@ -721,6 +737,16 @@ static struct gpio_desc *acpi_get_gpiod_by_index(struct acpi_device *adev,
return ret ? ERR_PTR(ret) : lookup.desc;
}
+static bool acpi_can_fallback_to_crs(struct acpi_device *adev,
+ const char *con_id)
+{
+ /* Never allow fallback if the device has properties */
+ if (acpi_dev_has_props(adev) || adev->driver_gpios)
+ return false;
+
+ return con_id == NULL;
+}
+
struct gpio_desc *acpi_find_gpio(struct device *dev,
const char *con_id,
unsigned int idx,
@@ -951,6 +977,7 @@ acpi_gpio_adr_space_handler(u32 function, acpi_physical_address address,
const char *label = "ACPI:OpRegion";
desc = gpiochip_request_own_desc(chip, pin, label,
+ GPIO_ACTIVE_HIGH,
flags);
if (IS_ERR(desc)) {
status = AE_ERROR;
@@ -1256,15 +1283,6 @@ int acpi_gpio_count(struct device *dev, const char *con_id)
return count ? count : -ENOENT;
}
-bool acpi_can_fallback_to_crs(struct acpi_device *adev, const char *con_id)
-{
- /* Never allow fallback if the device has properties */
- if (acpi_dev_has_props(adev) || adev->driver_gpios)
- return false;
-
- return con_id == NULL;
-}
-
/* Run deferred acpi_gpiochip_request_irqs() */
static int acpi_gpio_handle_deferred_request_irqs(void)
{
@@ -1283,3 +1301,28 @@ static int acpi_gpio_handle_deferred_request_irqs(void)
}
/* We must use _sync so that this runs after the first deferred_probe run */
late_initcall_sync(acpi_gpio_handle_deferred_request_irqs);
+
+static const struct dmi_system_id run_edge_events_on_boot_blacklist[] = {
+ {
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "MINIX"),
+ DMI_MATCH(DMI_PRODUCT_NAME, "Z83-4"),
+ }
+ },
+ {} /* Terminating entry */
+};
+
+static int acpi_gpio_setup_params(void)
+{
+ if (run_edge_events_on_boot < 0) {
+ if (dmi_check_system(run_edge_events_on_boot_blacklist))
+ run_edge_events_on_boot = 0;
+ else
+ run_edge_events_on_boot = 1;
+ }
+
+ return 0;
+}
+
+/* Directly after dmi_setup() which runs as core_initcall() */
+postcore_initcall(acpi_gpio_setup_params);
diff --git a/drivers/gpio/gpiolib-acpi.h b/drivers/gpio/gpiolib-acpi.h
new file mode 100644
index 000000000000..1c6d65cf0629
--- /dev/null
+++ b/drivers/gpio/gpiolib-acpi.h
@@ -0,0 +1,96 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * ACPI helpers for GPIO API
+ *
+ * Copyright (C) 2012,2019 Intel Corporation
+ */
+
+#ifndef GPIOLIB_ACPI_H
+#define GPIOLIB_ACPI_H
+
+struct acpi_device;
+
+/**
+ * struct acpi_gpio_info - ACPI GPIO specific information
+ * @adev: reference to ACPI device which consumes GPIO resource
+ * @flags: GPIO initialization flags
+ * @gpioint: if %true this GPIO is of type GpioInt otherwise type is GpioIo
+ * @pin_config: pin bias as provided by ACPI
+ * @polarity: interrupt polarity as provided by ACPI
+ * @triggering: triggering type as provided by ACPI
+ * @quirks: Linux specific quirks as provided by struct acpi_gpio_mapping
+ */
+struct acpi_gpio_info {
+ struct acpi_device *adev;
+ enum gpiod_flags flags;
+ bool gpioint;
+ int pin_config;
+ int polarity;
+ int triggering;
+ unsigned int quirks;
+};
+
+#ifdef CONFIG_ACPI
+void acpi_gpiochip_add(struct gpio_chip *chip);
+void acpi_gpiochip_remove(struct gpio_chip *chip);
+
+void acpi_gpiochip_request_interrupts(struct gpio_chip *chip);
+void acpi_gpiochip_free_interrupts(struct gpio_chip *chip);
+
+int acpi_gpio_update_gpiod_flags(enum gpiod_flags *flags,
+ struct acpi_gpio_info *info);
+int acpi_gpio_update_gpiod_lookup_flags(unsigned long *lookupflags,
+ struct acpi_gpio_info *info);
+
+struct gpio_desc *acpi_find_gpio(struct device *dev,
+ const char *con_id,
+ unsigned int idx,
+ enum gpiod_flags *dflags,
+ unsigned long *lookupflags);
+struct gpio_desc *acpi_node_get_gpiod(struct fwnode_handle *fwnode,
+ const char *propname, int index,
+ struct acpi_gpio_info *info);
+
+int acpi_gpio_count(struct device *dev, const char *con_id);
+#else
+static inline void acpi_gpiochip_add(struct gpio_chip *chip) { }
+static inline void acpi_gpiochip_remove(struct gpio_chip *chip) { }
+
+static inline void
+acpi_gpiochip_request_interrupts(struct gpio_chip *chip) { }
+
+static inline void
+acpi_gpiochip_free_interrupts(struct gpio_chip *chip) { }
+
+static inline int
+acpi_gpio_update_gpiod_flags(enum gpiod_flags *flags, struct acpi_gpio_info *info)
+{
+ return 0;
+}
+static inline int
+acpi_gpio_update_gpiod_lookup_flags(unsigned long *lookupflags,
+ struct acpi_gpio_info *info)
+{
+ return 0;
+}
+
+static inline struct gpio_desc *
+acpi_find_gpio(struct device *dev, const char *con_id,
+ unsigned int idx, enum gpiod_flags *dflags,
+ unsigned long *lookupflags)
+{
+ return ERR_PTR(-ENOENT);
+}
+static inline struct gpio_desc *
+acpi_node_get_gpiod(struct fwnode_handle *fwnode, const char *propname,
+ int index, struct acpi_gpio_info *info)
+{
+ return ERR_PTR(-ENXIO);
+}
+static inline int acpi_gpio_count(struct device *dev, const char *con_id)
+{
+ return -ENODEV;
+}
+#endif
+
+#endif /* GPIOLIB_ACPI_H */
diff --git a/drivers/gpio/gpiolib-devres.c b/drivers/gpio/gpiolib-devres.c
index 0acc2cc6e868..98e3c20d9730 100644
--- a/drivers/gpio/gpiolib-devres.c
+++ b/drivers/gpio/gpiolib-devres.c
@@ -59,7 +59,7 @@ struct gpio_desc *__must_check devm_gpiod_get(struct device *dev,
{
return devm_gpiod_get_index(dev, con_id, 0, flags);
}
-EXPORT_SYMBOL(devm_gpiod_get);
+EXPORT_SYMBOL_GPL(devm_gpiod_get);
/**
* devm_gpiod_get_optional - Resource-managed gpiod_get_optional()
@@ -77,7 +77,7 @@ struct gpio_desc *__must_check devm_gpiod_get_optional(struct device *dev,
{
return devm_gpiod_get_index_optional(dev, con_id, 0, flags);
}
-EXPORT_SYMBOL(devm_gpiod_get_optional);
+EXPORT_SYMBOL_GPL(devm_gpiod_get_optional);
/**
* devm_gpiod_get_index - Resource-managed gpiod_get_index()
@@ -127,7 +127,7 @@ struct gpio_desc *__must_check devm_gpiod_get_index(struct device *dev,
return desc;
}
-EXPORT_SYMBOL(devm_gpiod_get_index);
+EXPORT_SYMBOL_GPL(devm_gpiod_get_index);
/**
* devm_gpiod_get_from_of_node() - obtain a GPIO from an OF node
@@ -182,7 +182,7 @@ struct gpio_desc *devm_gpiod_get_from_of_node(struct device *dev,
return desc;
}
-EXPORT_SYMBOL(devm_gpiod_get_from_of_node);
+EXPORT_SYMBOL_GPL(devm_gpiod_get_from_of_node);
/**
* devm_fwnode_get_index_gpiod_from_child - get a GPIO descriptor from a
@@ -239,7 +239,7 @@ struct gpio_desc *devm_fwnode_get_index_gpiod_from_child(struct device *dev,
return desc;
}
-EXPORT_SYMBOL(devm_fwnode_get_index_gpiod_from_child);
+EXPORT_SYMBOL_GPL(devm_fwnode_get_index_gpiod_from_child);
/**
* devm_gpiod_get_index_optional - Resource-managed gpiod_get_index_optional()
@@ -268,7 +268,7 @@ struct gpio_desc *__must_check devm_gpiod_get_index_optional(struct device *dev,
return desc;
}
-EXPORT_SYMBOL(devm_gpiod_get_index_optional);
+EXPORT_SYMBOL_GPL(devm_gpiod_get_index_optional);
/**
* devm_gpiod_get_array - Resource-managed gpiod_get_array()
@@ -303,7 +303,7 @@ struct gpio_descs *__must_check devm_gpiod_get_array(struct device *dev,
return descs;
}
-EXPORT_SYMBOL(devm_gpiod_get_array);
+EXPORT_SYMBOL_GPL(devm_gpiod_get_array);
/**
* devm_gpiod_get_array_optional - Resource-managed gpiod_get_array_optional()
@@ -328,7 +328,7 @@ devm_gpiod_get_array_optional(struct device *dev, const char *con_id,
return descs;
}
-EXPORT_SYMBOL(devm_gpiod_get_array_optional);
+EXPORT_SYMBOL_GPL(devm_gpiod_get_array_optional);
/**
* devm_gpiod_put - Resource-managed gpiod_put()
@@ -344,7 +344,7 @@ void devm_gpiod_put(struct device *dev, struct gpio_desc *desc)
WARN_ON(devres_release(dev, devm_gpiod_release, devm_gpiod_match,
&desc));
}
-EXPORT_SYMBOL(devm_gpiod_put);
+EXPORT_SYMBOL_GPL(devm_gpiod_put);
/**
* devm_gpiod_unhinge - Remove resource management from a gpio descriptor
@@ -374,7 +374,7 @@ void devm_gpiod_unhinge(struct device *dev, struct gpio_desc *desc)
/* Anything else we should warn about */
WARN_ON(ret);
}
-EXPORT_SYMBOL(devm_gpiod_unhinge);
+EXPORT_SYMBOL_GPL(devm_gpiod_unhinge);
/**
* devm_gpiod_put_array - Resource-managed gpiod_put_array()
@@ -390,7 +390,7 @@ void devm_gpiod_put_array(struct device *dev, struct gpio_descs *descs)
WARN_ON(devres_release(dev, devm_gpiod_release_array,
devm_gpiod_match_array, &descs));
}
-EXPORT_SYMBOL(devm_gpiod_put_array);
+EXPORT_SYMBOL_GPL(devm_gpiod_put_array);
@@ -444,7 +444,7 @@ int devm_gpio_request(struct device *dev, unsigned gpio, const char *label)
return 0;
}
-EXPORT_SYMBOL(devm_gpio_request);
+EXPORT_SYMBOL_GPL(devm_gpio_request);
/**
* devm_gpio_request_one - request a single GPIO with initial setup
@@ -474,7 +474,7 @@ int devm_gpio_request_one(struct device *dev, unsigned gpio,
return 0;
}
-EXPORT_SYMBOL(devm_gpio_request_one);
+EXPORT_SYMBOL_GPL(devm_gpio_request_one);
/**
* devm_gpio_free - free a GPIO
@@ -492,4 +492,4 @@ void devm_gpio_free(struct device *dev, unsigned int gpio)
WARN_ON(devres_release(dev, devm_gpio_release, devm_gpio_match,
&gpio));
}
-EXPORT_SYMBOL(devm_gpio_free);
+EXPORT_SYMBOL_GPL(devm_gpio_free);
diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c
index 9c9b965d7d6d..1eea2c6c2e1d 100644
--- a/drivers/gpio/gpiolib-of.c
+++ b/drivers/gpio/gpiolib-of.c
@@ -21,6 +21,34 @@
#include <linux/gpio/machine.h>
#include "gpiolib.h"
+#include "gpiolib-of.h"
+
+/*
+ * This is used by external users of of_gpio_count() from <linux/of_gpio.h>
+ *
+ * FIXME: get rid of those external users by converting them to GPIO
+ * descriptors and let them all use gpiod_get_count()
+ */
+int of_gpio_get_count(struct device *dev, const char *con_id)
+{
+ int ret;
+ char propname[32];
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(gpio_suffixes); i++) {
+ if (con_id)
+ snprintf(propname, sizeof(propname), "%s-%s",
+ con_id, gpio_suffixes[i]);
+ else
+ snprintf(propname, sizeof(propname), "%s",
+ gpio_suffixes[i]);
+
+ ret = of_gpio_named_count(dev->of_node, propname);
+ if (ret > 0)
+ break;
+ }
+ return ret ? ret : -ENOENT;
+}
static int of_gpiochip_match_node_and_xlate(struct gpio_chip *chip, void *data)
{
@@ -53,6 +81,23 @@ static struct gpio_desc *of_xlate_and_get_gpiod_flags(struct gpio_chip *chip,
return gpiochip_get_desc(chip, ret);
}
+/**
+ * of_gpio_need_valid_mask() - figure out if the OF GPIO driver needs
+ * to set the .valid_mask
+ * @dev: the device for the GPIO provider
+ * @return: true if the valid mask needs to be set
+ */
+bool of_gpio_need_valid_mask(const struct gpio_chip *gc)
+{
+ int size;
+ struct device_node *np = gc->of_node;
+
+ size = of_property_count_u32_elems(np, "gpio-reserved-ranges");
+ if (size > 0 && size % 2 == 0)
+ return true;
+ return false;
+}
+
static void of_gpio_flags_quirks(struct device_node *np,
const char *propname,
enum of_gpio_flags *flags,
@@ -118,15 +163,8 @@ static void of_gpio_flags_quirks(struct device_node *np,
* Legacy handling of SPI active high chip select. If we have a
* property named "cs-gpios" we need to inspect the child node
* to determine if the flags should have inverted semantics.
- *
- * This does not apply to an SPI device named "spi-gpio", because
- * these have traditionally obtained their own GPIOs by parsing
- * the device tree directly and did not respect any "spi-cs-high"
- * property on the SPI bus children.
*/
- if (IS_ENABLED(CONFIG_SPI_MASTER) &&
- !strcmp(propname, "cs-gpios") &&
- !of_device_is_compatible(np, "spi-gpio") &&
+ if (IS_ENABLED(CONFIG_SPI_MASTER) && !strcmp(propname, "cs-gpios") &&
of_property_read_bool(np, "cs-gpios")) {
struct device_node *child;
u32 cs;
@@ -161,10 +199,17 @@ static void of_gpio_flags_quirks(struct device_node *np,
of_node_full_name(child));
*flags |= OF_GPIO_ACTIVE_LOW;
}
+ of_node_put(child);
break;
}
}
}
+
+ /* Legacy handling of stmmac's active-low PHY reset line */
+ if (IS_ENABLED(CONFIG_STMMAC_ETH) &&
+ !strcmp(propname, "snps,reset-gpio") &&
+ of_property_read_bool(np, "snps,reset-active-low"))
+ *flags |= OF_GPIO_ACTIVE_LOW;
}
/**
@@ -178,7 +223,7 @@ static void of_gpio_flags_quirks(struct device_node *np,
* value on the error condition. If @flags is not NULL the function also fills
* in flags for the GPIO.
*/
-struct gpio_desc *of_get_named_gpiod_flags(struct device_node *np,
+static struct gpio_desc *of_get_named_gpiod_flags(struct device_node *np,
const char *propname, int index, enum of_gpio_flags *flags)
{
struct of_phandle_args gpiospec;
@@ -229,7 +274,76 @@ int of_get_named_gpio_flags(struct device_node *np, const char *list_name,
else
return desc_to_gpio(desc);
}
-EXPORT_SYMBOL(of_get_named_gpio_flags);
+EXPORT_SYMBOL_GPL(of_get_named_gpio_flags);
+
+/**
+ * gpiod_get_from_of_node() - obtain a GPIO from an OF node
+ * @node: handle of the OF node
+ * @propname: name of the DT property representing the GPIO
+ * @index: index of the GPIO to obtain for the consumer
+ * @dflags: GPIO initialization flags
+ * @label: label to attach to the requested GPIO
+ *
+ * Returns:
+ * On successful request the GPIO pin is configured in accordance with
+ * provided @dflags.
+ *
+ * In case of error an ERR_PTR() is returned.
+ */
+struct gpio_desc *gpiod_get_from_of_node(struct device_node *node,
+ const char *propname, int index,
+ enum gpiod_flags dflags,
+ const char *label)
+{
+ unsigned long lflags = GPIO_LOOKUP_FLAGS_DEFAULT;
+ struct gpio_desc *desc;
+ enum of_gpio_flags flags;
+ bool active_low = false;
+ bool single_ended = false;
+ bool open_drain = false;
+ bool transitory = false;
+ int ret;
+
+ desc = of_get_named_gpiod_flags(node, propname,
+ index, &flags);
+
+ if (!desc || IS_ERR(desc)) {
+ return desc;
+ }
+
+ active_low = flags & OF_GPIO_ACTIVE_LOW;
+ single_ended = flags & OF_GPIO_SINGLE_ENDED;
+ open_drain = flags & OF_GPIO_OPEN_DRAIN;
+ transitory = flags & OF_GPIO_TRANSITORY;
+
+ ret = gpiod_request(desc, label);
+ if (ret == -EBUSY && (flags & GPIOD_FLAGS_BIT_NONEXCLUSIVE))
+ return desc;
+ if (ret)
+ return ERR_PTR(ret);
+
+ if (active_low)
+ lflags |= GPIO_ACTIVE_LOW;
+
+ if (single_ended) {
+ if (open_drain)
+ lflags |= GPIO_OPEN_DRAIN;
+ else
+ lflags |= GPIO_OPEN_SOURCE;
+ }
+
+ if (transitory)
+ lflags |= GPIO_TRANSITORY;
+
+ ret = gpiod_configure_flags(desc, propname, lflags, dflags);
+ if (ret < 0) {
+ gpiod_put(desc);
+ return ERR_PTR(ret);
+ }
+
+ return desc;
+}
+EXPORT_SYMBOL_GPL(gpiod_get_from_of_node);
/*
* The SPI GPIO bindings happened before we managed to establish that GPIO
@@ -262,6 +376,37 @@ static struct gpio_desc *of_find_spi_gpio(struct device *dev, const char *con_id
}
/*
+ * The old Freescale bindings use simply "gpios" as name for the chip select
+ * lines rather than "cs-gpios" like all other SPI hardware. Account for this
+ * with a special quirk.
+ */
+static struct gpio_desc *of_find_spi_cs_gpio(struct device *dev,
+ const char *con_id,
+ unsigned int idx,
+ unsigned long *flags)
+{
+ struct device_node *np = dev->of_node;
+
+ if (!IS_ENABLED(CONFIG_SPI_MASTER))
+ return ERR_PTR(-ENOENT);
+
+ /* Allow this specifically for Freescale devices */
+ if (!of_device_is_compatible(np, "fsl,spi") &&
+ !of_device_is_compatible(np, "aeroflexgaisler,spictrl"))
+ return ERR_PTR(-ENOENT);
+ /* Allow only if asking for "cs-gpios" */
+ if (!con_id || strcmp(con_id, "cs"))
+ return ERR_PTR(-ENOENT);
+
+ /*
+ * While all other SPI controllers use "cs-gpios" the Freescale
+ * uses just "gpios" so translate to that when "cs-gpios" is
+ * requested.
+ */
+ return of_find_gpio(dev, NULL, idx, flags);
+}
+
+/*
* Some regulator bindings happened before we managed to establish that GPIO
* properties should be named "foo-gpios" so we have this special kludge for
* them.
@@ -293,6 +438,19 @@ static struct gpio_desc *of_find_regulator_gpio(struct device *dev, const char *
return desc;
}
+static struct gpio_desc *of_find_arizona_gpio(struct device *dev,
+ const char *con_id,
+ enum of_gpio_flags *of_flags)
+{
+ if (!IS_ENABLED(CONFIG_MFD_ARIZONA))
+ return ERR_PTR(-ENOENT);
+
+ if (!con_id || strcmp(con_id, "wlf,reset"))
+ return ERR_PTR(-ENOENT);
+
+ return of_get_named_gpiod_flags(dev->of_node, con_id, 0, of_flags);
+}
+
struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id,
unsigned int idx, unsigned long *flags)
{
@@ -312,30 +470,30 @@ struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id,
desc = of_get_named_gpiod_flags(dev->of_node, prop_name, idx,
&of_flags);
- /*
- * -EPROBE_DEFER in our case means that we found a
- * valid GPIO property, but no controller has been
- * registered so far.
- *
- * This means we don't need to look any further for
- * alternate name conventions, and we should really
- * preserve the return code for our user to be able to
- * retry probing later.
- */
- if (IS_ERR(desc) && PTR_ERR(desc) == -EPROBE_DEFER)
- return desc;
- if (!IS_ERR(desc) || (PTR_ERR(desc) != -ENOENT))
+ if (!IS_ERR(desc) || PTR_ERR(desc) != -ENOENT)
break;
}
- /* Special handling for SPI GPIOs if used */
- if (IS_ERR(desc))
+ if (IS_ERR(desc) && PTR_ERR(desc) == -ENOENT) {
+ /* Special handling for SPI GPIOs if used */
desc = of_find_spi_gpio(dev, con_id, &of_flags);
+ }
- /* Special handling for regulator GPIOs if used */
- if (IS_ERR(desc) && PTR_ERR(desc) != -EPROBE_DEFER)
+ if (IS_ERR(desc) && PTR_ERR(desc) == -ENOENT) {
+ /* This quirk looks up flags and all */
+ desc = of_find_spi_cs_gpio(dev, con_id, idx, flags);
+ if (!IS_ERR(desc))
+ return desc;
+ }
+
+ if (IS_ERR(desc) && PTR_ERR(desc) == -ENOENT) {
+ /* Special handling for regulator GPIOs if used */
desc = of_find_regulator_gpio(dev, con_id, &of_flags);
+ }
+
+ if (IS_ERR(desc) && PTR_ERR(desc) == -ENOENT)
+ desc = of_find_arizona_gpio(dev, con_id, &of_flags);
if (IS_ERR(desc))
return desc;
@@ -486,8 +644,9 @@ static int of_gpiochip_scan_gpios(struct gpio_chip *chip)
* GPIO chips. This function performs only one sanity check: whether GPIO
* is less than ngpios (that is specified in the gpio_chip).
*/
-int of_gpio_simple_xlate(struct gpio_chip *gc,
- const struct of_phandle_args *gpiospec, u32 *flags)
+static int of_gpio_simple_xlate(struct gpio_chip *gc,
+ const struct of_phandle_args *gpiospec,
+ u32 *flags)
{
/*
* We're discouraging gpio_cells < 2, since that way you'll have to
@@ -511,7 +670,6 @@ int of_gpio_simple_xlate(struct gpio_chip *gc,
return gpiospec->args[0];
}
-EXPORT_SYMBOL(of_gpio_simple_xlate);
/**
* of_mm_gpiochip_add_data - Add memory mapped GPIO chip (bank)
@@ -568,7 +726,7 @@ err0:
pr_err("%pOF: GPIO chip registration failed with status %d\n", np, ret);
return ret;
}
-EXPORT_SYMBOL(of_mm_gpiochip_add_data);
+EXPORT_SYMBOL_GPL(of_mm_gpiochip_add_data);
/**
* of_mm_gpiochip_remove - Remove memory mapped GPIO chip (bank)
@@ -585,7 +743,7 @@ void of_mm_gpiochip_remove(struct of_mm_gpio_chip *mm_gc)
iounmap(mm_gc->regs);
kfree(gc->label);
}
-EXPORT_SYMBOL(of_mm_gpiochip_remove);
+EXPORT_SYMBOL_GPL(of_mm_gpiochip_remove);
static void of_gpiochip_init_valid_mask(struct gpio_chip *chip)
{
@@ -697,7 +855,7 @@ static int of_gpiochip_add_pin_range(struct gpio_chip *chip) { return 0; }
int of_gpiochip_add(struct gpio_chip *chip)
{
- int status;
+ int ret;
if (!chip->of_node)
return 0;
@@ -712,9 +870,9 @@ int of_gpiochip_add(struct gpio_chip *chip)
of_gpiochip_init_valid_mask(chip);
- status = of_gpiochip_add_pin_range(chip);
- if (status)
- return status;
+ ret = of_gpiochip_add_pin_range(chip);
+ if (ret)
+ return ret;
/* If the chip defines names itself, these take precedence */
if (!chip->names)
@@ -723,13 +881,13 @@ int of_gpiochip_add(struct gpio_chip *chip)
of_node_get(chip->of_node);
- status = of_gpiochip_scan_gpios(chip);
- if (status) {
+ ret = of_gpiochip_scan_gpios(chip);
+ if (ret) {
of_node_put(chip->of_node);
gpiochip_remove_pin_ranges(chip);
}
- return status;
+ return ret;
}
void of_gpiochip_remove(struct gpio_chip *chip)
diff --git a/drivers/gpio/gpiolib-of.h b/drivers/gpio/gpiolib-of.h
new file mode 100644
index 000000000000..9768831b1fe2
--- /dev/null
+++ b/drivers/gpio/gpiolib-of.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef GPIOLIB_OF_H
+#define GPIOLIB_OF_H
+
+struct gpio_chip;
+enum of_gpio_flags;
+
+#ifdef CONFIG_OF_GPIO
+struct gpio_desc *of_find_gpio(struct device *dev,
+ const char *con_id,
+ unsigned int idx,
+ unsigned long *lookupflags);
+int of_gpiochip_add(struct gpio_chip *gc);
+void of_gpiochip_remove(struct gpio_chip *gc);
+int of_gpio_get_count(struct device *dev, const char *con_id);
+bool of_gpio_need_valid_mask(const struct gpio_chip *gc);
+#else
+static inline struct gpio_desc *of_find_gpio(struct device *dev,
+ const char *con_id,
+ unsigned int idx,
+ unsigned long *lookupflags)
+{
+ return ERR_PTR(-ENOENT);
+}
+static inline int of_gpiochip_add(struct gpio_chip *gc) { return 0; }
+static inline void of_gpiochip_remove(struct gpio_chip *gc) { }
+static inline int of_gpio_get_count(struct device *dev, const char *con_id)
+{
+ return 0;
+}
+static inline bool of_gpio_need_valid_mask(const struct gpio_chip *gc)
+{
+ return false;
+}
+#endif /* CONFIG_OF_GPIO */
+
+#endif /* GPIOLIB_OF_H */
diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c
index e013d417a936..822988818efc 100644
--- a/drivers/gpio/gpiolib.c
+++ b/drivers/gpio/gpiolib.c
@@ -11,7 +11,6 @@
#include <linux/debugfs.h>
#include <linux/seq_file.h>
#include <linux/gpio.h>
-#include <linux/of_gpio.h>
#include <linux/idr.h>
#include <linux/slab.h>
#include <linux/acpi.h>
@@ -30,6 +29,8 @@
#include <uapi/linux/gpio.h>
#include "gpiolib.h"
+#include "gpiolib-of.h"
+#include "gpiolib-acpi.h"
#define CREATE_TRACE_POINTS
#include <trace/events/gpio.h>
@@ -213,7 +214,7 @@ int gpiod_get_direction(struct gpio_desc *desc)
{
struct gpio_chip *chip;
unsigned offset;
- int status;
+ int ret;
chip = gpiod_to_chip(desc);
offset = gpio_chip_hwgpio(desc);
@@ -221,17 +222,17 @@ int gpiod_get_direction(struct gpio_desc *desc)
if (!chip->get_direction)
return -ENOTSUPP;
- status = chip->get_direction(chip, offset);
- if (status > 0) {
+ ret = chip->get_direction(chip, offset);
+ if (ret > 0) {
/* GPIOF_DIR_IN, or other positive */
- status = 1;
+ ret = 1;
clear_bit(FLAG_IS_OUT, &desc->flags);
}
- if (status == 0) {
+ if (ret == 0) {
/* GPIOF_DIR_OUT */
set_bit(FLAG_IS_OUT, &desc->flags);
}
- return status;
+ return ret;
}
EXPORT_SYMBOL_GPL(gpiod_get_direction);
@@ -350,7 +351,7 @@ static unsigned long *gpiochip_allocate_mask(struct gpio_chip *chip)
{
unsigned long *p;
- p = kmalloc_array(BITS_TO_LONGS(chip->ngpio), sizeof(*p), GFP_KERNEL);
+ p = bitmap_alloc(chip->ngpio, GFP_KERNEL);
if (!p)
return NULL;
@@ -360,38 +361,31 @@ static unsigned long *gpiochip_allocate_mask(struct gpio_chip *chip)
return p;
}
-static int gpiochip_alloc_valid_mask(struct gpio_chip *gpiochip)
+static int gpiochip_alloc_valid_mask(struct gpio_chip *gc)
{
-#ifdef CONFIG_OF_GPIO
- int size;
- struct device_node *np = gpiochip->of_node;
-
- size = of_property_count_u32_elems(np, "gpio-reserved-ranges");
- if (size > 0 && size % 2 == 0)
- gpiochip->need_valid_mask = true;
-#endif
-
- if (!gpiochip->need_valid_mask)
+ if (!(of_gpio_need_valid_mask(gc) || gc->init_valid_mask))
return 0;
- gpiochip->valid_mask = gpiochip_allocate_mask(gpiochip);
- if (!gpiochip->valid_mask)
+ gc->valid_mask = gpiochip_allocate_mask(gc);
+ if (!gc->valid_mask)
return -ENOMEM;
return 0;
}
-static int gpiochip_init_valid_mask(struct gpio_chip *gpiochip)
+static int gpiochip_init_valid_mask(struct gpio_chip *gc)
{
- if (gpiochip->init_valid_mask)
- return gpiochip->init_valid_mask(gpiochip);
+ if (gc->init_valid_mask)
+ return gc->init_valid_mask(gc,
+ gc->valid_mask,
+ gc->ngpio);
return 0;
}
static void gpiochip_free_valid_mask(struct gpio_chip *gpiochip)
{
- kfree(gpiochip->valid_mask);
+ bitmap_free(gpiochip->valid_mask);
gpiochip->valid_mask = NULL;
}
@@ -536,6 +530,14 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip)
return -EINVAL;
/*
+ * Do not allow both INPUT & OUTPUT flags to be set as they are
+ * contradictory.
+ */
+ if ((lflags & GPIOHANDLE_REQUEST_INPUT) &&
+ (lflags & GPIOHANDLE_REQUEST_OUTPUT))
+ return -EINVAL;
+
+ /*
* Do not allow OPEN_SOURCE & OPEN_DRAIN flags in a single request. If
* the hardware actually supports enabling both at the same time the
* electrical result would be disastrous.
@@ -857,7 +859,7 @@ static irqreturn_t lineevent_irq_thread(int irq, void *p)
}
ret = kfifo_put(&le->events, ge);
- if (ret != 0)
+ if (ret)
wake_up_poll(&le->wait, EPOLLIN);
return IRQ_HANDLED;
@@ -926,7 +928,9 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip)
}
/* This is just wrong: we don't look for events on output lines */
- if (lflags & GPIOHANDLE_REQUEST_OUTPUT) {
+ if ((lflags & GPIOHANDLE_REQUEST_OUTPUT) ||
+ (lflags & GPIOHANDLE_REQUEST_OPEN_DRAIN) ||
+ (lflags & GPIOHANDLE_REQUEST_OPEN_SOURCE)) {
ret = -EINVAL;
goto out_free_label;
}
@@ -940,10 +944,6 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip)
if (lflags & GPIOHANDLE_REQUEST_ACTIVE_LOW)
set_bit(FLAG_ACTIVE_LOW, &desc->flags);
- if (lflags & GPIOHANDLE_REQUEST_OPEN_DRAIN)
- set_bit(FLAG_OPEN_DRAIN, &desc->flags);
- if (lflags & GPIOHANDLE_REQUEST_OPEN_SOURCE)
- set_bit(FLAG_OPEN_SOURCE, &desc->flags);
ret = gpiod_direction_input(desc);
if (ret)
@@ -956,9 +956,11 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip)
}
if (eflags & GPIOEVENT_REQUEST_RISING_EDGE)
- irqflags |= IRQF_TRIGGER_RISING;
+ irqflags |= test_bit(FLAG_ACTIVE_LOW, &desc->flags) ?
+ IRQF_TRIGGER_FALLING : IRQF_TRIGGER_RISING;
if (eflags & GPIOEVENT_REQUEST_FALLING_EDGE)
- irqflags |= IRQF_TRIGGER_FALLING;
+ irqflags |= test_bit(FLAG_ACTIVE_LOW, &desc->flags) ?
+ IRQF_TRIGGER_RISING : IRQF_TRIGGER_FALLING;
irqflags |= IRQF_ONESHOT;
INIT_KFIFO(le->events);
@@ -1089,9 +1091,11 @@ static long gpio_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
if (test_bit(FLAG_ACTIVE_LOW, &desc->flags))
lineinfo.flags |= GPIOLINE_FLAG_ACTIVE_LOW;
if (test_bit(FLAG_OPEN_DRAIN, &desc->flags))
- lineinfo.flags |= GPIOLINE_FLAG_OPEN_DRAIN;
+ lineinfo.flags |= (GPIOLINE_FLAG_OPEN_DRAIN |
+ GPIOLINE_FLAG_IS_OUT);
if (test_bit(FLAG_OPEN_SOURCE, &desc->flags))
- lineinfo.flags |= GPIOLINE_FLAG_OPEN_SOURCE;
+ lineinfo.flags |= (GPIOLINE_FLAG_OPEN_SOURCE |
+ GPIOLINE_FLAG_IS_OUT);
if (copy_to_user(ip, &lineinfo, sizeof(lineinfo)))
return -EFAULT;
@@ -1172,21 +1176,21 @@ static void gpiodevice_release(struct device *dev)
static int gpiochip_setup_dev(struct gpio_device *gdev)
{
- int status;
+ int ret;
cdev_init(&gdev->chrdev, &gpio_fileops);
gdev->chrdev.owner = THIS_MODULE;
gdev->dev.devt = MKDEV(MAJOR(gpio_devt), gdev->id);
- status = cdev_device_add(&gdev->chrdev, &gdev->dev);
- if (status)
- return status;
+ ret = cdev_device_add(&gdev->chrdev, &gdev->dev);
+ if (ret)
+ return ret;
chip_dbg(gdev->chip, "added GPIO chardev (%d:%d)\n",
MAJOR(gpio_devt), gdev->id);
- status = gpiochip_sysfs_register(gdev);
- if (status)
+ ret = gpiochip_sysfs_register(gdev);
+ if (ret)
goto err_remove_device;
/* From this point, the .release() function cleans up gpio_device */
@@ -1199,7 +1203,7 @@ static int gpiochip_setup_dev(struct gpio_device *gdev)
err_remove_device:
cdev_device_del(&gdev->chrdev, &gdev->dev);
- return status;
+ return ret;
}
static void gpiochip_machine_hog(struct gpio_chip *chip, struct gpiod_hog *hog)
@@ -1240,13 +1244,13 @@ static void machine_gpiochip_add(struct gpio_chip *chip)
static void gpiochip_setup_devs(void)
{
struct gpio_device *gdev;
- int err;
+ int ret;
list_for_each_entry(gdev, &gpio_devices, list) {
- err = gpiochip_setup_dev(gdev);
- if (err)
+ ret = gpiochip_setup_dev(gdev);
+ if (ret)
pr_err("%s: Failed to initialize gpio device (%d)\n",
- dev_name(&gdev->dev), err);
+ dev_name(&gdev->dev), ret);
}
}
@@ -1255,7 +1259,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *chip, void *data,
struct lock_class_key *request_key)
{
unsigned long flags;
- int status = 0;
+ int ret = 0;
unsigned i;
int base = chip->base;
struct gpio_device *gdev;
@@ -1285,7 +1289,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *chip, void *data,
gdev->id = ida_simple_get(&gpio_ida, 0, 0, GFP_KERNEL);
if (gdev->id < 0) {
- status = gdev->id;
+ ret = gdev->id;
goto err_free_gdev;
}
dev_set_name(&gdev->dev, "gpiochip%d", gdev->id);
@@ -1301,13 +1305,13 @@ int gpiochip_add_data_with_key(struct gpio_chip *chip, void *data,
gdev->descs = kcalloc(chip->ngpio, sizeof(gdev->descs[0]), GFP_KERNEL);
if (!gdev->descs) {
- status = -ENOMEM;
+ ret = -ENOMEM;
goto err_free_ida;
}
if (chip->ngpio == 0) {
chip_err(chip, "tried to insert a GPIO chip with zero lines\n");
- status = -EINVAL;
+ ret = -EINVAL;
goto err_free_descs;
}
@@ -1317,7 +1321,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *chip, void *data,
gdev->label = kstrdup_const(chip->label ?: "unknown", GFP_KERNEL);
if (!gdev->label) {
- status = -ENOMEM;
+ ret = -ENOMEM;
goto err_free_descs;
}
@@ -1336,7 +1340,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *chip, void *data,
if (base < 0) {
base = gpiochip_find_base(chip->ngpio);
if (base < 0) {
- status = base;
+ ret = base;
spin_unlock_irqrestore(&gpio_lock, flags);
goto err_free_label;
}
@@ -1350,8 +1354,8 @@ int gpiochip_add_data_with_key(struct gpio_chip *chip, void *data,
}
gdev->base = base;
- status = gpiodev_add_to_list(gdev);
- if (status) {
+ ret = gpiodev_add_to_list(gdev);
+ if (ret) {
spin_unlock_irqrestore(&gpio_lock, flags);
goto err_free_label;
}
@@ -1365,45 +1369,50 @@ int gpiochip_add_data_with_key(struct gpio_chip *chip, void *data,
INIT_LIST_HEAD(&gdev->pin_ranges);
#endif
- status = gpiochip_set_desc_names(chip);
- if (status)
+ ret = gpiochip_set_desc_names(chip);
+ if (ret)
goto err_remove_from_list;
- status = gpiochip_irqchip_init_valid_mask(chip);
- if (status)
+ ret = gpiochip_alloc_valid_mask(chip);
+ if (ret)
goto err_remove_from_list;
- status = gpiochip_alloc_valid_mask(chip);
- if (status)
- goto err_remove_irqchip_mask;
-
- status = gpiochip_add_irqchip(chip, lock_key, request_key);
- if (status)
+ ret = of_gpiochip_add(chip);
+ if (ret)
goto err_free_gpiochip_mask;
- status = of_gpiochip_add(chip);
- if (status)
- goto err_remove_chip;
-
- status = gpiochip_init_valid_mask(chip);
- if (status)
+ ret = gpiochip_init_valid_mask(chip);
+ if (ret)
goto err_remove_of_chip;
for (i = 0; i < chip->ngpio; i++) {
struct gpio_desc *desc = &gdev->descs[i];
- if (chip->get_direction && gpiochip_line_is_valid(chip, i))
- desc->flags = !chip->get_direction(chip, i) ?
- (1 << FLAG_IS_OUT) : 0;
- else
- desc->flags = !chip->direction_input ?
- (1 << FLAG_IS_OUT) : 0;
+ if (chip->get_direction && gpiochip_line_is_valid(chip, i)) {
+ if (!chip->get_direction(chip, i))
+ set_bit(FLAG_IS_OUT, &desc->flags);
+ else
+ clear_bit(FLAG_IS_OUT, &desc->flags);
+ } else {
+ if (!chip->direction_input)
+ set_bit(FLAG_IS_OUT, &desc->flags);
+ else
+ clear_bit(FLAG_IS_OUT, &desc->flags);
+ }
}
acpi_gpiochip_add(chip);
machine_gpiochip_add(chip);
+ ret = gpiochip_irqchip_init_valid_mask(chip);
+ if (ret)
+ goto err_remove_acpi_chip;
+
+ ret = gpiochip_add_irqchip(chip, lock_key, request_key);
+ if (ret)
+ goto err_remove_irqchip_mask;
+
/*
* By first adding the chardev, and then adding the device,
* we get a device node entry in sysfs under
@@ -1413,23 +1422,23 @@ int gpiochip_add_data_with_key(struct gpio_chip *chip, void *data,
* Otherwise, defer until later.
*/
if (gpiolib_initialized) {
- status = gpiochip_setup_dev(gdev);
- if (status)
- goto err_remove_acpi_chip;
+ ret = gpiochip_setup_dev(gdev);
+ if (ret)
+ goto err_remove_irqchip;
}
return 0;
+err_remove_irqchip:
+ gpiochip_irqchip_remove(chip);
+err_remove_irqchip_mask:
+ gpiochip_irqchip_free_valid_mask(chip);
err_remove_acpi_chip:
acpi_gpiochip_remove(chip);
err_remove_of_chip:
gpiochip_free_hogs(chip);
of_gpiochip_remove(chip);
-err_remove_chip:
- gpiochip_irqchip_remove(chip);
err_free_gpiochip_mask:
gpiochip_free_valid_mask(chip);
-err_remove_irqchip_mask:
- gpiochip_irqchip_free_valid_mask(chip);
err_remove_from_list:
spin_lock_irqsave(&gpio_lock, flags);
list_del(&gdev->list);
@@ -1444,9 +1453,9 @@ err_free_gdev:
/* failures here can mean systems won't boot... */
pr_err("%s: GPIOs %d..%d (%s) failed to register, %d\n", __func__,
gdev->base, gdev->base + gdev->ngpio - 1,
- chip->label ? : "generic", status);
+ chip->label ? : "generic", ret);
kfree(gdev);
- return status;
+ return ret;
}
EXPORT_SYMBOL_GPL(gpiochip_add_data_with_key);
@@ -1612,21 +1621,25 @@ static struct gpio_chip *find_chip_by_name(const char *name)
* The following is irqchip helper code for gpiochips.
*/
-static int gpiochip_irqchip_init_valid_mask(struct gpio_chip *gpiochip)
+static int gpiochip_irqchip_init_valid_mask(struct gpio_chip *gc)
{
- if (!gpiochip->irq.need_valid_mask)
+ struct gpio_irq_chip *girq = &gc->irq;
+
+ if (!girq->init_valid_mask)
return 0;
- gpiochip->irq.valid_mask = gpiochip_allocate_mask(gpiochip);
- if (!gpiochip->irq.valid_mask)
+ girq->valid_mask = gpiochip_allocate_mask(gc);
+ if (!girq->valid_mask)
return -ENOMEM;
+ girq->init_valid_mask(gc, girq->valid_mask, gc->ngpio);
+
return 0;
}
static void gpiochip_irqchip_free_valid_mask(struct gpio_chip *gpiochip)
{
- kfree(gpiochip->irq.valid_mask);
+ bitmap_free(gpiochip->irq.valid_mask);
gpiochip->irq.valid_mask = NULL;
}
@@ -1644,39 +1657,47 @@ EXPORT_SYMBOL_GPL(gpiochip_irqchip_irq_valid);
/**
* gpiochip_set_cascaded_irqchip() - connects a cascaded irqchip to a gpiochip
- * @gpiochip: the gpiochip to set the irqchip chain to
+ * @gc: the gpiochip to set the irqchip chain to
* @parent_irq: the irq number corresponding to the parent IRQ for this
* chained irqchip
* @parent_handler: the parent interrupt handler for the accumulated IRQ
* coming out of the gpiochip. If the interrupt is nested rather than
* cascaded, pass NULL in this handler argument
*/
-static void gpiochip_set_cascaded_irqchip(struct gpio_chip *gpiochip,
+static void gpiochip_set_cascaded_irqchip(struct gpio_chip *gc,
unsigned int parent_irq,
irq_flow_handler_t parent_handler)
{
- if (!gpiochip->irq.domain) {
- chip_err(gpiochip, "called %s before setting up irqchip\n",
+ struct gpio_irq_chip *girq = &gc->irq;
+ struct device *dev = &gc->gpiodev->dev;
+
+ if (!girq->domain) {
+ chip_err(gc, "called %s before setting up irqchip\n",
__func__);
return;
}
if (parent_handler) {
- if (gpiochip->can_sleep) {
- chip_err(gpiochip,
+ if (gc->can_sleep) {
+ chip_err(gc,
"you cannot have chained interrupts on a chip that may sleep\n");
return;
}
+ girq->parents = devm_kcalloc(dev, 1,
+ sizeof(*girq->parents),
+ GFP_KERNEL);
+ if (!girq->parents) {
+ chip_err(gc, "out of memory allocating parent IRQ\n");
+ return;
+ }
+ girq->parents[0] = parent_irq;
+ girq->num_parents = 1;
/*
* The parent irqchip is already using the chip_data for this
* irqchip, so our callbacks simply use the handler_data.
*/
irq_set_chained_handler_and_data(parent_irq, parent_handler,
- gpiochip);
-
- gpiochip->irq.parent_irq = parent_irq;
- gpiochip->irq.parents = &gpiochip->irq.parent_irq;
- gpiochip->irq.num_parents = 1;
+ gc);
}
}
@@ -1718,6 +1739,273 @@ void gpiochip_set_nested_irqchip(struct gpio_chip *gpiochip,
}
EXPORT_SYMBOL_GPL(gpiochip_set_nested_irqchip);
+#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
+
+/**
+ * gpiochip_set_hierarchical_irqchip() - connects a hierarchical irqchip
+ * to a gpiochip
+ * @gc: the gpiochip to set the irqchip hierarchical handler to
+ * @irqchip: the irqchip to handle this level of the hierarchy, the interrupt
+ * will then percolate up to the parent
+ */
+static void gpiochip_set_hierarchical_irqchip(struct gpio_chip *gc,
+ struct irq_chip *irqchip)
+{
+ /* DT will deal with mapping each IRQ as we go along */
+ if (is_of_node(gc->irq.fwnode))
+ return;
+
+ /*
+ * This is for legacy and boardfile "irqchip" fwnodes: allocate
+ * irqs upfront instead of dynamically since we don't have the
+ * dynamic type of allocation that hardware description languages
+ * provide. Once all GPIO drivers using board files are gone from
+ * the kernel we can delete this code, but for a transitional period
+ * it is necessary to keep this around.
+ */
+ if (is_fwnode_irqchip(gc->irq.fwnode)) {
+ int i;
+ int ret;
+
+ for (i = 0; i < gc->ngpio; i++) {
+ struct irq_fwspec fwspec;
+ unsigned int parent_hwirq;
+ unsigned int parent_type;
+ struct gpio_irq_chip *girq = &gc->irq;
+
+ /*
+ * We call the child to parent translation function
+ * only to check if the child IRQ is valid or not.
+ * Just pick the rising edge type here as that is what
+ * we likely need to support.
+ */
+ ret = girq->child_to_parent_hwirq(gc, i,
+ IRQ_TYPE_EDGE_RISING,
+ &parent_hwirq,
+ &parent_type);
+ if (ret) {
+ chip_err(gc, "skip set-up on hwirq %d\n",
+ i);
+ continue;
+ }
+
+ fwspec.fwnode = gc->irq.fwnode;
+ /* This is the hwirq for the GPIO line side of things */
+ fwspec.param[0] = girq->child_offset_to_irq(gc, i);
+ /* Just pick something */
+ fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
+ fwspec.param_count = 2;
+ ret = __irq_domain_alloc_irqs(gc->irq.domain,
+ /* just pick something */
+ -1,
+ 1,
+ NUMA_NO_NODE,
+ &fwspec,
+ false,
+ NULL);
+ if (ret < 0) {
+ chip_err(gc,
+ "can not allocate irq for GPIO line %d parent hwirq %d in hierarchy domain: %d\n",
+ i, parent_hwirq,
+ ret);
+ }
+ }
+ }
+
+ chip_err(gc, "%s unknown fwnode type proceed anyway\n", __func__);
+
+ return;
+}
+
+static int gpiochip_hierarchy_irq_domain_translate(struct irq_domain *d,
+ struct irq_fwspec *fwspec,
+ unsigned long *hwirq,
+ unsigned int *type)
+{
+ /* We support standard DT translation */
+ if (is_of_node(fwspec->fwnode) && fwspec->param_count == 2) {
+ return irq_domain_translate_twocell(d, fwspec, hwirq, type);
+ }
+
+ /* This is for board files and others not using DT */
+ if (is_fwnode_irqchip(fwspec->fwnode)) {
+ int ret;
+
+ ret = irq_domain_translate_twocell(d, fwspec, hwirq, type);
+ if (ret)
+ return ret;
+ WARN_ON(*type == IRQ_TYPE_NONE);
+ return 0;
+ }
+ return -EINVAL;
+}
+
+static int gpiochip_hierarchy_irq_domain_alloc(struct irq_domain *d,
+ unsigned int irq,
+ unsigned int nr_irqs,
+ void *data)
+{
+ struct gpio_chip *gc = d->host_data;
+ irq_hw_number_t hwirq;
+ unsigned int type = IRQ_TYPE_NONE;
+ struct irq_fwspec *fwspec = data;
+ struct irq_fwspec parent_fwspec;
+ unsigned int parent_hwirq;
+ unsigned int parent_type;
+ struct gpio_irq_chip *girq = &gc->irq;
+ int ret;
+
+ /*
+ * The nr_irqs parameter is always one except for PCI multi-MSI
+ * so this should not happen.
+ */
+ WARN_ON(nr_irqs != 1);
+
+ ret = gc->irq.child_irq_domain_ops.translate(d, fwspec, &hwirq, &type);
+ if (ret)
+ return ret;
+
+ chip_info(gc, "allocate IRQ %d, hwirq %lu\n", irq, hwirq);
+
+ ret = girq->child_to_parent_hwirq(gc, hwirq, type,
+ &parent_hwirq, &parent_type);
+ if (ret) {
+ chip_err(gc, "can't look up hwirq %lu\n", hwirq);
+ return ret;
+ }
+ chip_info(gc, "found parent hwirq %u\n", parent_hwirq);
+
+ /*
+ * We set handle_bad_irq because the .set_type() should
+ * always be invoked and set the right type of handler.
+ */
+ irq_domain_set_info(d,
+ irq,
+ hwirq,
+ gc->irq.chip,
+ gc,
+ girq->handler,
+ NULL, NULL);
+ irq_set_probe(irq);
+
+ /*
+ * Create a IRQ fwspec to send up to the parent irqdomain:
+ * specify the hwirq we address on the parent and tie it
+ * all together up the chain.
+ */
+ parent_fwspec.fwnode = d->parent->fwnode;
+ /* This parent only handles asserted level IRQs */
+ girq->populate_parent_fwspec(gc, &parent_fwspec, parent_hwirq,
+ parent_type);
+ chip_info(gc, "alloc_irqs_parent for %d parent hwirq %d\n",
+ irq, parent_hwirq);
+ ret = irq_domain_alloc_irqs_parent(d, irq, 1, &parent_fwspec);
+ if (ret)
+ chip_err(gc,
+ "failed to allocate parent hwirq %d for hwirq %lu\n",
+ parent_hwirq, hwirq);
+
+ return ret;
+}
+
+static unsigned int gpiochip_child_offset_to_irq_noop(struct gpio_chip *chip,
+ unsigned int offset)
+{
+ return offset;
+}
+
+static void gpiochip_hierarchy_setup_domain_ops(struct irq_domain_ops *ops)
+{
+ ops->activate = gpiochip_irq_domain_activate;
+ ops->deactivate = gpiochip_irq_domain_deactivate;
+ ops->alloc = gpiochip_hierarchy_irq_domain_alloc;
+ ops->free = irq_domain_free_irqs_common;
+
+ /*
+ * We only allow overriding the translate() function for
+ * hierarchical chips, and this should only be done if the user
+ * really need something other than 1:1 translation.
+ */
+ if (!ops->translate)
+ ops->translate = gpiochip_hierarchy_irq_domain_translate;
+}
+
+static int gpiochip_hierarchy_add_domain(struct gpio_chip *gc)
+{
+ if (!gc->irq.child_to_parent_hwirq ||
+ !gc->irq.fwnode) {
+ chip_err(gc, "missing irqdomain vital data\n");
+ return -EINVAL;
+ }
+
+ if (!gc->irq.child_offset_to_irq)
+ gc->irq.child_offset_to_irq = gpiochip_child_offset_to_irq_noop;
+
+ if (!gc->irq.populate_parent_fwspec)
+ gc->irq.populate_parent_fwspec =
+ gpiochip_populate_parent_fwspec_twocell;
+
+ gpiochip_hierarchy_setup_domain_ops(&gc->irq.child_irq_domain_ops);
+
+ gc->irq.domain = irq_domain_create_hierarchy(
+ gc->irq.parent_domain,
+ 0,
+ gc->ngpio,
+ gc->irq.fwnode,
+ &gc->irq.child_irq_domain_ops,
+ gc);
+
+ if (!gc->irq.domain)
+ return -ENOMEM;
+
+ gpiochip_set_hierarchical_irqchip(gc, gc->irq.chip);
+
+ return 0;
+}
+
+static bool gpiochip_hierarchy_is_hierarchical(struct gpio_chip *gc)
+{
+ return !!gc->irq.parent_domain;
+}
+
+void gpiochip_populate_parent_fwspec_twocell(struct gpio_chip *chip,
+ struct irq_fwspec *fwspec,
+ unsigned int parent_hwirq,
+ unsigned int parent_type)
+{
+ fwspec->param_count = 2;
+ fwspec->param[0] = parent_hwirq;
+ fwspec->param[1] = parent_type;
+}
+EXPORT_SYMBOL_GPL(gpiochip_populate_parent_fwspec_twocell);
+
+void gpiochip_populate_parent_fwspec_fourcell(struct gpio_chip *chip,
+ struct irq_fwspec *fwspec,
+ unsigned int parent_hwirq,
+ unsigned int parent_type)
+{
+ fwspec->param_count = 4;
+ fwspec->param[0] = 0;
+ fwspec->param[1] = parent_hwirq;
+ fwspec->param[2] = 0;
+ fwspec->param[3] = parent_type;
+}
+EXPORT_SYMBOL_GPL(gpiochip_populate_parent_fwspec_fourcell);
+
+#else
+
+static int gpiochip_hierarchy_add_domain(struct gpio_chip *gc)
+{
+ return -EINVAL;
+}
+
+static bool gpiochip_hierarchy_is_hierarchical(struct gpio_chip *gc)
+{
+ return false;
+}
+
+#endif /* CONFIG_IRQ_DOMAIN_HIERARCHY */
+
/**
* gpiochip_irq_map() - maps an IRQ into a GPIO irqchip
* @d: the irqdomain used by this irqchip
@@ -1732,7 +2020,7 @@ int gpiochip_irq_map(struct irq_domain *d, unsigned int irq,
irq_hw_number_t hwirq)
{
struct gpio_chip *chip = d->host_data;
- int err = 0;
+ int ret = 0;
if (!gpiochip_irqchip_irq_valid(chip, hwirq))
return -ENXIO;
@@ -1750,12 +2038,12 @@ int gpiochip_irq_map(struct irq_domain *d, unsigned int irq,
irq_set_noprobe(irq);
if (chip->irq.num_parents == 1)
- err = irq_set_parent(irq, chip->irq.parents[0]);
+ ret = irq_set_parent(irq, chip->irq.parents[0]);
else if (chip->irq.map)
- err = irq_set_parent(irq, chip->irq.map[hwirq]);
+ ret = irq_set_parent(irq, chip->irq.map[hwirq]);
- if (err < 0)
- return err;
+ if (ret < 0)
+ return ret;
/*
* No set-up of the hardware will happen if IRQ_TYPE_NONE
@@ -1786,6 +2074,11 @@ static const struct irq_domain_ops gpiochip_domain_ops = {
.xlate = irq_domain_xlate_twocell,
};
+/*
+ * TODO: move these activate/deactivate in under the hierarchicial
+ * irqchip implementation as static once SPMI and SSBI (all external
+ * users) are phased over.
+ */
/**
* gpiochip_irq_domain_activate() - Lock a GPIO to be used as an IRQ
* @domain: The IRQ domain used by this IRQ chip
@@ -1825,10 +2118,25 @@ EXPORT_SYMBOL_GPL(gpiochip_irq_domain_deactivate);
static int gpiochip_to_irq(struct gpio_chip *chip, unsigned offset)
{
+ struct irq_domain *domain = chip->irq.domain;
+
if (!gpiochip_irqchip_irq_valid(chip, offset))
return -ENXIO;
- return irq_create_mapping(chip->irq.domain, offset);
+#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
+ if (irq_domain_is_hierarchy(domain)) {
+ struct irq_fwspec spec;
+
+ spec.fwnode = domain->fwnode;
+ spec.param_count = 2;
+ spec.param[0] = chip->irq.child_offset_to_irq(chip, offset);
+ spec.param[1] = IRQ_TYPE_NONE;
+
+ return irq_create_fwspec_mapping(&spec);
+ }
+#endif
+
+ return irq_create_mapping(domain, offset);
}
static int gpiochip_irq_reqres(struct irq_data *d)
@@ -1905,7 +2213,7 @@ static int gpiochip_add_irqchip(struct gpio_chip *gpiochip,
struct lock_class_key *request_key)
{
struct irq_chip *irqchip = gpiochip->irq.chip;
- const struct irq_domain_ops *ops;
+ const struct irq_domain_ops *ops = NULL;
struct device_node *np;
unsigned int type;
unsigned int i;
@@ -1941,16 +2249,25 @@ static int gpiochip_add_irqchip(struct gpio_chip *gpiochip,
gpiochip->irq.lock_key = lock_key;
gpiochip->irq.request_key = request_key;
- if (gpiochip->irq.domain_ops)
- ops = gpiochip->irq.domain_ops;
- else
- ops = &gpiochip_domain_ops;
-
- gpiochip->irq.domain = irq_domain_add_simple(np, gpiochip->ngpio,
- gpiochip->irq.first,
- ops, gpiochip);
- if (!gpiochip->irq.domain)
- return -EINVAL;
+ /* If a parent irqdomain is provided, let's build a hierarchy */
+ if (gpiochip_hierarchy_is_hierarchical(gpiochip)) {
+ int ret = gpiochip_hierarchy_add_domain(gpiochip);
+ if (ret)
+ return ret;
+ } else {
+ /* Some drivers provide custom irqdomain ops */
+ if (gpiochip->irq.domain_ops)
+ ops = gpiochip->irq.domain_ops;
+
+ if (!ops)
+ ops = &gpiochip_domain_ops;
+ gpiochip->irq.domain = irq_domain_add_simple(np,
+ gpiochip->ngpio,
+ gpiochip->irq.first,
+ ops, gpiochip);
+ if (!gpiochip->irq.domain)
+ return -EINVAL;
+ }
if (gpiochip->irq.parent_handler) {
void *data = gpiochip->irq.parent_handler_data ?: gpiochip;
@@ -2313,7 +2630,7 @@ EXPORT_SYMBOL_GPL(gpiochip_remove_pin_ranges);
static int gpiod_request_commit(struct gpio_desc *desc, const char *label)
{
struct gpio_chip *chip = desc->gdev->chip;
- int status;
+ int ret;
unsigned long flags;
unsigned offset;
@@ -2331,10 +2648,10 @@ static int gpiod_request_commit(struct gpio_desc *desc, const char *label)
if (test_and_set_bit(FLAG_REQUESTED, &desc->flags) == 0) {
desc_set_label(desc, label ? : "?");
- status = 0;
+ ret = 0;
} else {
kfree_const(label);
- status = -EBUSY;
+ ret = -EBUSY;
goto done;
}
@@ -2343,12 +2660,12 @@ static int gpiod_request_commit(struct gpio_desc *desc, const char *label)
spin_unlock_irqrestore(&gpio_lock, flags);
offset = gpio_chip_hwgpio(desc);
if (gpiochip_line_is_valid(chip, offset))
- status = chip->request(chip, offset);
+ ret = chip->request(chip, offset);
else
- status = -EINVAL;
+ ret = -EINVAL;
spin_lock_irqsave(&gpio_lock, flags);
- if (status < 0) {
+ if (ret < 0) {
desc_set_label(desc, NULL);
kfree_const(label);
clear_bit(FLAG_REQUESTED, &desc->flags);
@@ -2363,7 +2680,7 @@ static int gpiod_request_commit(struct gpio_desc *desc, const char *label)
}
done:
spin_unlock_irqrestore(&gpio_lock, flags);
- return status;
+ return ret;
}
/*
@@ -2406,24 +2723,24 @@ static int validate_desc(const struct gpio_desc *desc, const char *func)
int gpiod_request(struct gpio_desc *desc, const char *label)
{
- int status = -EPROBE_DEFER;
+ int ret = -EPROBE_DEFER;
struct gpio_device *gdev;
VALIDATE_DESC(desc);
gdev = desc->gdev;
if (try_module_get(gdev->owner)) {
- status = gpiod_request_commit(desc, label);
- if (status < 0)
+ ret = gpiod_request_commit(desc, label);
+ if (ret < 0)
module_put(gdev->owner);
else
get_device(&gdev->dev);
}
- if (status)
- gpiod_dbg(desc, "%s: status %d\n", __func__, status);
+ if (ret)
+ gpiod_dbg(desc, "%s: status %d\n", __func__, ret);
- return status;
+ return ret;
}
static bool gpiod_free_commit(struct gpio_desc *desc)
@@ -2503,7 +2820,11 @@ EXPORT_SYMBOL_GPL(gpiochip_is_requested);
* @chip: GPIO chip
* @hwnum: hardware number of the GPIO for which to request the descriptor
* @label: label for the GPIO
- * @flags: flags for this GPIO or 0 if default
+ * @lflags: lookup flags for this GPIO or 0 if default, this can be used to
+ * specify things like line inversion semantics with the machine flags
+ * such as GPIO_OUT_LOW
+ * @dflags: descriptor request flags for this GPIO or 0 if default, this
+ * can be used to specify consumer semantics such as open drain
*
* Function allows GPIO chip drivers to request and use their own GPIO
* descriptors via gpiolib API. Difference to gpiod_request() is that this
@@ -2517,26 +2838,26 @@ EXPORT_SYMBOL_GPL(gpiochip_is_requested);
*/
struct gpio_desc *gpiochip_request_own_desc(struct gpio_chip *chip, u16 hwnum,
const char *label,
- enum gpiod_flags flags)
+ enum gpio_lookup_flags lflags,
+ enum gpiod_flags dflags)
{
- unsigned long lflags = GPIO_LOOKUP_FLAGS_DEFAULT;
struct gpio_desc *desc = gpiochip_get_desc(chip, hwnum);
- int err;
+ int ret;
if (IS_ERR(desc)) {
chip_err(chip, "failed to get GPIO descriptor\n");
return desc;
}
- err = gpiod_request_commit(desc, label);
- if (err < 0)
- return ERR_PTR(err);
+ ret = gpiod_request_commit(desc, label);
+ if (ret < 0)
+ return ERR_PTR(ret);
- err = gpiod_configure_flags(desc, label, lflags, flags);
- if (err) {
+ ret = gpiod_configure_flags(desc, label, lflags, dflags);
+ if (ret) {
chip_err(chip, "setup of own GPIO %s failed\n", label);
gpiod_free_commit(desc);
- return ERR_PTR(err);
+ return ERR_PTR(ret);
}
return desc;
@@ -2599,7 +2920,7 @@ static int gpio_set_config(struct gpio_chip *gc, unsigned offset,
int gpiod_direction_input(struct gpio_desc *desc)
{
struct gpio_chip *chip;
- int status = 0;
+ int ret = 0;
VALIDATE_DESC(desc);
chip = desc->gdev->chip;
@@ -2623,7 +2944,7 @@ int gpiod_direction_input(struct gpio_desc *desc)
* assume we are in input mode after this.
*/
if (chip->direction_input) {
- status = chip->direction_input(chip, gpio_chip_hwgpio(desc));
+ ret = chip->direction_input(chip, gpio_chip_hwgpio(desc));
} else if (chip->get_direction &&
(chip->get_direction(chip, gpio_chip_hwgpio(desc)) != 1)) {
gpiod_warn(desc,
@@ -2631,7 +2952,7 @@ int gpiod_direction_input(struct gpio_desc *desc)
__func__);
return -EIO;
}
- if (status == 0)
+ if (ret == 0)
clear_bit(FLAG_IS_OUT, &desc->flags);
if (test_bit(FLAG_PULL_UP, &desc->flags))
@@ -2641,9 +2962,9 @@ int gpiod_direction_input(struct gpio_desc *desc)
gpio_set_config(chip, gpio_chip_hwgpio(desc),
PIN_CONFIG_BIAS_PULL_DOWN);
- trace_gpio_direction(desc_to_gpio(desc), 1, status);
+ trace_gpio_direction(desc_to_gpio(desc), 1, ret);
- return status;
+ return ret;
}
EXPORT_SYMBOL_GPL(gpiod_direction_input);
@@ -2915,7 +3236,7 @@ int gpiod_get_array_value_complex(bool raw, bool can_sleep,
struct gpio_array *array_info,
unsigned long *value_bitmap)
{
- int err, i = 0;
+ int ret, i = 0;
/*
* Validate array_info against desc_array and its size.
@@ -2928,11 +3249,11 @@ int gpiod_get_array_value_complex(bool raw, bool can_sleep,
if (!can_sleep)
WARN_ON(array_info->chip->can_sleep);
- err = gpio_chip_get_multiple(array_info->chip,
+ ret = gpio_chip_get_multiple(array_info->chip,
array_info->get_mask,
value_bitmap);
- if (err)
- return err;
+ if (ret)
+ return ret;
if (!raw && !bitmap_empty(array_info->invert_mask, array_size))
bitmap_xor(value_bitmap, value_bitmap,
@@ -3019,13 +3340,13 @@ int gpiod_get_array_value_complex(bool raw, bool can_sleep,
* Return the GPIO's raw value, i.e. the value of the physical line disregarding
* its ACTIVE_LOW status, or negative errno on failure.
*
- * This function should be called from contexts where we cannot sleep, and will
+ * This function can be called from contexts where we cannot sleep, and will
* complain if the GPIO chip functions potentially sleep.
*/
int gpiod_get_raw_value(const struct gpio_desc *desc)
{
VALIDATE_DESC(desc);
- /* Should be using gpio_get_value_cansleep() */
+ /* Should be using gpiod_get_raw_value_cansleep() */
WARN_ON(desc->gdev->chip->can_sleep);
return gpiod_get_raw_value_commit(desc);
}
@@ -3038,7 +3359,7 @@ EXPORT_SYMBOL_GPL(gpiod_get_raw_value);
* Return the GPIO's logical value, i.e. taking the ACTIVE_LOW status into
* account, or negative errno on failure.
*
- * This function should be called from contexts where we cannot sleep, and will
+ * This function can be called from contexts where we cannot sleep, and will
* complain if the GPIO chip functions potentially sleep.
*/
int gpiod_get_value(const struct gpio_desc *desc)
@@ -3046,7 +3367,7 @@ int gpiod_get_value(const struct gpio_desc *desc)
int value;
VALIDATE_DESC(desc);
- /* Should be using gpio_get_value_cansleep() */
+ /* Should be using gpiod_get_value_cansleep() */
WARN_ON(desc->gdev->chip->can_sleep);
value = gpiod_get_raw_value_commit(desc);
@@ -3071,7 +3392,7 @@ EXPORT_SYMBOL_GPL(gpiod_get_value);
* without regard for their ACTIVE_LOW status. Return 0 in case of success,
* else an error code.
*
- * This function should be called from contexts where we cannot sleep,
+ * This function can be called from contexts where we cannot sleep,
* and it will complain if the GPIO chip functions potentially sleep.
*/
int gpiod_get_raw_array_value(unsigned int array_size,
@@ -3097,7 +3418,7 @@ EXPORT_SYMBOL_GPL(gpiod_get_raw_array_value);
* Read the logical values of the GPIOs, i.e. taking their ACTIVE_LOW status
* into account. Return 0 in case of success, else an error code.
*
- * This function should be called from contexts where we cannot sleep,
+ * This function can be called from contexts where we cannot sleep,
* and it will complain if the GPIO chip functions potentially sleep.
*/
int gpiod_get_array_value(unsigned int array_size,
@@ -3120,24 +3441,24 @@ EXPORT_SYMBOL_GPL(gpiod_get_array_value);
*/
static void gpio_set_open_drain_value_commit(struct gpio_desc *desc, bool value)
{
- int err = 0;
+ int ret = 0;
struct gpio_chip *chip = desc->gdev->chip;
int offset = gpio_chip_hwgpio(desc);
if (value) {
- err = chip->direction_input(chip, offset);
- if (!err)
+ ret = chip->direction_input(chip, offset);
+ if (!ret)
clear_bit(FLAG_IS_OUT, &desc->flags);
} else {
- err = chip->direction_output(chip, offset, 0);
- if (!err)
+ ret = chip->direction_output(chip, offset, 0);
+ if (!ret)
set_bit(FLAG_IS_OUT, &desc->flags);
}
- trace_gpio_direction(desc_to_gpio(desc), value, err);
- if (err < 0)
+ trace_gpio_direction(desc_to_gpio(desc), value, ret);
+ if (ret < 0)
gpiod_err(desc,
"%s: Error in set_value for open drain err %d\n",
- __func__, err);
+ __func__, ret);
}
/*
@@ -3147,24 +3468,24 @@ static void gpio_set_open_drain_value_commit(struct gpio_desc *desc, bool value)
*/
static void gpio_set_open_source_value_commit(struct gpio_desc *desc, bool value)
{
- int err = 0;
+ int ret = 0;
struct gpio_chip *chip = desc->gdev->chip;
int offset = gpio_chip_hwgpio(desc);
if (value) {
- err = chip->direction_output(chip, offset, 1);
- if (!err)
+ ret = chip->direction_output(chip, offset, 1);
+ if (!ret)
set_bit(FLAG_IS_OUT, &desc->flags);
} else {
- err = chip->direction_input(chip, offset);
- if (!err)
+ ret = chip->direction_input(chip, offset);
+ if (!ret)
clear_bit(FLAG_IS_OUT, &desc->flags);
}
- trace_gpio_direction(desc_to_gpio(desc), !value, err);
- if (err < 0)
+ trace_gpio_direction(desc_to_gpio(desc), !value, ret);
+ if (ret < 0)
gpiod_err(desc,
"%s: Error in set_value for open source err %d\n",
- __func__, err);
+ __func__, ret);
}
static void gpiod_set_raw_value_commit(struct gpio_desc *desc, bool value)
@@ -3311,13 +3632,13 @@ int gpiod_set_array_value_complex(bool raw, bool can_sleep,
* Set the raw value of the GPIO, i.e. the value of its physical line without
* regard for its ACTIVE_LOW status.
*
- * This function should be called from contexts where we cannot sleep, and will
+ * This function can be called from contexts where we cannot sleep, and will
* complain if the GPIO chip functions potentially sleep.
*/
void gpiod_set_raw_value(struct gpio_desc *desc, int value)
{
VALIDATE_DESC_VOID(desc);
- /* Should be using gpiod_set_value_cansleep() */
+ /* Should be using gpiod_set_raw_value_cansleep() */
WARN_ON(desc->gdev->chip->can_sleep);
gpiod_set_raw_value_commit(desc, value);
}
@@ -3352,12 +3673,13 @@ static void gpiod_set_value_nocheck(struct gpio_desc *desc, int value)
* Set the logical value of the GPIO, i.e. taking its ACTIVE_LOW,
* OPEN_DRAIN and OPEN_SOURCE flags into account.
*
- * This function should be called from contexts where we cannot sleep, and will
+ * This function can be called from contexts where we cannot sleep, and will
* complain if the GPIO chip functions potentially sleep.
*/
void gpiod_set_value(struct gpio_desc *desc, int value)
{
VALIDATE_DESC_VOID(desc);
+ /* Should be using gpiod_set_value_cansleep() */
WARN_ON(desc->gdev->chip->can_sleep);
gpiod_set_value_nocheck(desc, value);
}
@@ -3373,7 +3695,7 @@ EXPORT_SYMBOL_GPL(gpiod_set_value);
* Set the raw values of the GPIOs, i.e. the values of the physical lines
* without regard for their ACTIVE_LOW status.
*
- * This function should be called from contexts where we cannot sleep, and will
+ * This function can be called from contexts where we cannot sleep, and will
* complain if the GPIO chip functions potentially sleep.
*/
int gpiod_set_raw_array_value(unsigned int array_size,
@@ -3398,7 +3720,7 @@ EXPORT_SYMBOL_GPL(gpiod_set_raw_array_value);
* Set the logical values of the GPIOs, i.e. taking their ACTIVE_LOW status
* into account.
*
- * This function should be called from contexts where we cannot sleep, and will
+ * This function can be called from contexts where we cannot sleep, and will
* complain if the GPIO chip functions potentially sleep.
*/
int gpiod_set_array_value(unsigned int array_size,
@@ -3980,27 +4302,6 @@ static struct gpio_desc *gpiod_find(struct device *dev, const char *con_id,
return desc;
}
-static int dt_gpio_count(struct device *dev, const char *con_id)
-{
- int ret;
- char propname[32];
- unsigned int i;
-
- for (i = 0; i < ARRAY_SIZE(gpio_suffixes); i++) {
- if (con_id)
- snprintf(propname, sizeof(propname), "%s-%s",
- con_id, gpio_suffixes[i]);
- else
- snprintf(propname, sizeof(propname), "%s",
- gpio_suffixes[i]);
-
- ret = of_gpio_named_count(dev->of_node, propname);
- if (ret > 0)
- break;
- }
- return ret ? ret : -ENOENT;
-}
-
static int platform_gpio_count(struct device *dev, const char *con_id)
{
struct gpiod_lookup_table *table;
@@ -4033,7 +4334,7 @@ int gpiod_count(struct device *dev, const char *con_id)
int count = -ENOENT;
if (IS_ENABLED(CONFIG_OF) && dev && dev->of_node)
- count = dt_gpio_count(dev, con_id);
+ count = of_gpio_get_count(dev, con_id);
else if (IS_ENABLED(CONFIG_ACPI) && dev && ACPI_HANDLE(dev))
count = acpi_gpio_count(dev, con_id);
@@ -4095,7 +4396,7 @@ EXPORT_SYMBOL_GPL(gpiod_get_optional);
int gpiod_configure_flags(struct gpio_desc *desc, const char *con_id,
unsigned long lflags, enum gpiod_flags dflags)
{
- int status;
+ int ret;
if (lflags & GPIO_ACTIVE_LOW)
set_bit(FLAG_ACTIVE_LOW, &desc->flags);
@@ -4128,9 +4429,9 @@ int gpiod_configure_flags(struct gpio_desc *desc, const char *con_id,
else if (lflags & GPIO_PULL_DOWN)
set_bit(FLAG_PULL_DOWN, &desc->flags);
- status = gpiod_set_transitory(desc, (lflags & GPIO_TRANSITORY));
- if (status < 0)
- return status;
+ ret = gpiod_set_transitory(desc, (lflags & GPIO_TRANSITORY));
+ if (ret < 0)
+ return ret;
/* No particular flag request, return here... */
if (!(dflags & GPIOD_FLAGS_BIT_DIR_SET)) {
@@ -4140,12 +4441,12 @@ int gpiod_configure_flags(struct gpio_desc *desc, const char *con_id,
/* Process flags */
if (dflags & GPIOD_FLAGS_BIT_DIR_OUT)
- status = gpiod_direction_output(desc,
+ ret = gpiod_direction_output(desc,
!!(dflags & GPIOD_FLAGS_BIT_DIR_VAL));
else
- status = gpiod_direction_input(desc);
+ ret = gpiod_direction_input(desc);
- return status;
+ return ret;
}
/**
@@ -4169,7 +4470,7 @@ struct gpio_desc *__must_check gpiod_get_index(struct device *dev,
{
unsigned long lookupflags = GPIO_LOOKUP_FLAGS_DEFAULT;
struct gpio_desc *desc = NULL;
- int status;
+ int ret;
/* Maybe we have a device name, maybe not */
const char *devname = dev ? dev_name(dev) : "?";
@@ -4204,9 +4505,9 @@ struct gpio_desc *__must_check gpiod_get_index(struct device *dev,
* If a connection label was passed use that, else attempt to use
* the device name as label
*/
- status = gpiod_request(desc, con_id ? con_id : devname);
- if (status < 0) {
- if (status == -EBUSY && flags & GPIOD_FLAGS_BIT_NONEXCLUSIVE) {
+ ret = gpiod_request(desc, con_id ? con_id : devname);
+ if (ret < 0) {
+ if (ret == -EBUSY && flags & GPIOD_FLAGS_BIT_NONEXCLUSIVE) {
/*
* This happens when there are several consumers for
* the same GPIO line: we just return here without
@@ -4219,93 +4520,20 @@ struct gpio_desc *__must_check gpiod_get_index(struct device *dev,
con_id ? con_id : devname);
return desc;
} else {
- return ERR_PTR(status);
+ return ERR_PTR(ret);
}
}
- status = gpiod_configure_flags(desc, con_id, lookupflags, flags);
- if (status < 0) {
- dev_dbg(dev, "setup of GPIO %s failed\n", con_id);
- gpiod_put(desc);
- return ERR_PTR(status);
- }
-
- return desc;
-}
-EXPORT_SYMBOL_GPL(gpiod_get_index);
-
-/**
- * gpiod_get_from_of_node() - obtain a GPIO from an OF node
- * @node: handle of the OF node
- * @propname: name of the DT property representing the GPIO
- * @index: index of the GPIO to obtain for the consumer
- * @dflags: GPIO initialization flags
- * @label: label to attach to the requested GPIO
- *
- * Returns:
- * On successful request the GPIO pin is configured in accordance with
- * provided @dflags. If the node does not have the requested GPIO
- * property, NULL is returned.
- *
- * In case of error an ERR_PTR() is returned.
- */
-struct gpio_desc *gpiod_get_from_of_node(struct device_node *node,
- const char *propname, int index,
- enum gpiod_flags dflags,
- const char *label)
-{
- unsigned long lflags = GPIO_LOOKUP_FLAGS_DEFAULT;
- struct gpio_desc *desc;
- enum of_gpio_flags flags;
- bool active_low = false;
- bool single_ended = false;
- bool open_drain = false;
- bool transitory = false;
- int ret;
-
- desc = of_get_named_gpiod_flags(node, propname,
- index, &flags);
-
- if (!desc || IS_ERR(desc)) {
- /* If it is not there, just return NULL */
- if (PTR_ERR(desc) == -ENOENT)
- return NULL;
- return desc;
- }
-
- active_low = flags & OF_GPIO_ACTIVE_LOW;
- single_ended = flags & OF_GPIO_SINGLE_ENDED;
- open_drain = flags & OF_GPIO_OPEN_DRAIN;
- transitory = flags & OF_GPIO_TRANSITORY;
-
- ret = gpiod_request(desc, label);
- if (ret == -EBUSY && (flags & GPIOD_FLAGS_BIT_NONEXCLUSIVE))
- return desc;
- if (ret)
- return ERR_PTR(ret);
-
- if (active_low)
- lflags |= GPIO_ACTIVE_LOW;
-
- if (single_ended) {
- if (open_drain)
- lflags |= GPIO_OPEN_DRAIN;
- else
- lflags |= GPIO_OPEN_SOURCE;
- }
-
- if (transitory)
- lflags |= GPIO_TRANSITORY;
-
- ret = gpiod_configure_flags(desc, propname, lflags, dflags);
+ ret = gpiod_configure_flags(desc, con_id, lookupflags, flags);
if (ret < 0) {
+ dev_dbg(dev, "setup of GPIO %s failed\n", con_id);
gpiod_put(desc);
return ERR_PTR(ret);
}
return desc;
}
-EXPORT_SYMBOL(gpiod_get_from_of_node);
+EXPORT_SYMBOL_GPL(gpiod_get_index);
/**
* fwnode_get_named_gpiod - obtain a GPIO from firmware node
@@ -4415,33 +4643,18 @@ int gpiod_hog(struct gpio_desc *desc, const char *name,
struct gpio_chip *chip;
struct gpio_desc *local_desc;
int hwnum;
- int status;
+ int ret;
chip = gpiod_to_chip(desc);
hwnum = gpio_chip_hwgpio(desc);
- /*
- * FIXME: not very elegant that we call gpiod_configure_flags()
- * twice here (once inside gpiochip_request_own_desc() and
- * again here), but the gpiochip_request_own_desc() is external
- * and cannot really pass the lflags so this is the lesser evil
- * at the moment. Pass zero as dflags on this first call so we
- * don't screw anything up.
- */
- local_desc = gpiochip_request_own_desc(chip, hwnum, name, 0);
+ local_desc = gpiochip_request_own_desc(chip, hwnum, name,
+ lflags, dflags);
if (IS_ERR(local_desc)) {
- status = PTR_ERR(local_desc);
+ ret = PTR_ERR(local_desc);
pr_err("requesting hog GPIO %s (chip %s, offset %d) failed, %d\n",
- name, chip->label, hwnum, status);
- return status;
- }
-
- status = gpiod_configure_flags(desc, name, lflags, dflags);
- if (status < 0) {
- pr_err("setup of hog GPIO %s (chip %s, offset %d) failed, %d\n",
- name, chip->label, hwnum, status);
- gpiochip_free_own_desc(desc);
- return status;
+ name, chip->label, hwnum, ret);
+ return ret;
}
/* Mark GPIO as hogged so it can be identified and removed later */
@@ -4805,8 +5018,8 @@ static const struct file_operations gpiolib_operations = {
static int __init gpiolib_debugfs_init(void)
{
/* /sys/kernel/debug/gpio */
- (void) debugfs_create_file("gpio", S_IFREG | S_IRUGO,
- NULL, NULL, &gpiolib_operations);
+ debugfs_create_file("gpio", S_IFREG | S_IRUGO, NULL, NULL,
+ &gpiolib_operations);
return 0;
}
subsys_initcall(gpiolib_debugfs_init);
diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h
index 7a65dad43932..b8b10a409c7b 100644
--- a/drivers/gpio/gpiolib.h
+++ b/drivers/gpio/gpiolib.h
@@ -16,9 +16,6 @@
#include <linux/module.h>
#include <linux/cdev.h>
-enum of_gpio_flags;
-struct acpi_device;
-
/**
* struct gpio_device - internal state container for GPIO devices
* @id: numerical ID number for the GPIO chip
@@ -69,126 +66,9 @@ struct gpio_device {
#endif
};
-/**
- * struct acpi_gpio_info - ACPI GPIO specific information
- * @adev: reference to ACPI device which consumes GPIO resource
- * @flags: GPIO initialization flags
- * @gpioint: if %true this GPIO is of type GpioInt otherwise type is GpioIo
- * @pin_config: pin bias as provided by ACPI
- * @polarity: interrupt polarity as provided by ACPI
- * @triggering: triggering type as provided by ACPI
- * @quirks: Linux specific quirks as provided by struct acpi_gpio_mapping
- */
-struct acpi_gpio_info {
- struct acpi_device *adev;
- enum gpiod_flags flags;
- bool gpioint;
- int pin_config;
- int polarity;
- int triggering;
- unsigned int quirks;
-};
-
/* gpio suffixes used for ACPI and device tree lookup */
static __maybe_unused const char * const gpio_suffixes[] = { "gpios", "gpio" };
-#ifdef CONFIG_OF_GPIO
-struct gpio_desc *of_find_gpio(struct device *dev,
- const char *con_id,
- unsigned int idx,
- unsigned long *lookupflags);
-struct gpio_desc *of_get_named_gpiod_flags(struct device_node *np,
- const char *list_name, int index, enum of_gpio_flags *flags);
-int of_gpiochip_add(struct gpio_chip *gc);
-void of_gpiochip_remove(struct gpio_chip *gc);
-#else
-static inline struct gpio_desc *of_find_gpio(struct device *dev,
- const char *con_id,
- unsigned int idx,
- unsigned long *lookupflags)
-{
- return ERR_PTR(-ENOENT);
-}
-static inline struct gpio_desc *of_get_named_gpiod_flags(struct device_node *np,
- const char *list_name, int index, enum of_gpio_flags *flags)
-{
- return ERR_PTR(-ENOENT);
-}
-static inline int of_gpiochip_add(struct gpio_chip *gc) { return 0; }
-static inline void of_gpiochip_remove(struct gpio_chip *gc) { }
-#endif /* CONFIG_OF_GPIO */
-
-#ifdef CONFIG_ACPI
-void acpi_gpiochip_add(struct gpio_chip *chip);
-void acpi_gpiochip_remove(struct gpio_chip *chip);
-
-void acpi_gpiochip_request_interrupts(struct gpio_chip *chip);
-void acpi_gpiochip_free_interrupts(struct gpio_chip *chip);
-
-int acpi_gpio_update_gpiod_flags(enum gpiod_flags *flags,
- struct acpi_gpio_info *info);
-int acpi_gpio_update_gpiod_lookup_flags(unsigned long *lookupflags,
- struct acpi_gpio_info *info);
-
-struct gpio_desc *acpi_find_gpio(struct device *dev,
- const char *con_id,
- unsigned int idx,
- enum gpiod_flags *dflags,
- unsigned long *lookupflags);
-struct gpio_desc *acpi_node_get_gpiod(struct fwnode_handle *fwnode,
- const char *propname, int index,
- struct acpi_gpio_info *info);
-
-int acpi_gpio_count(struct device *dev, const char *con_id);
-
-bool acpi_can_fallback_to_crs(struct acpi_device *adev, const char *con_id);
-#else
-static inline void acpi_gpiochip_add(struct gpio_chip *chip) { }
-static inline void acpi_gpiochip_remove(struct gpio_chip *chip) { }
-
-static inline void
-acpi_gpiochip_request_interrupts(struct gpio_chip *chip) { }
-
-static inline void
-acpi_gpiochip_free_interrupts(struct gpio_chip *chip) { }
-
-static inline int
-acpi_gpio_update_gpiod_flags(enum gpiod_flags *flags, struct acpi_gpio_info *info)
-{
- return 0;
-}
-static inline int
-acpi_gpio_update_gpiod_lookup_flags(unsigned long *lookupflags,
- struct acpi_gpio_info *info)
-{
- return 0;
-}
-
-static inline struct gpio_desc *
-acpi_find_gpio(struct device *dev, const char *con_id,
- unsigned int idx, enum gpiod_flags *dflags,
- unsigned long *lookupflags)
-{
- return ERR_PTR(-ENOENT);
-}
-static inline struct gpio_desc *
-acpi_node_get_gpiod(struct fwnode_handle *fwnode, const char *propname,
- int index, struct acpi_gpio_info *info)
-{
- return ERR_PTR(-ENXIO);
-}
-static inline int acpi_gpio_count(struct device *dev, const char *con_id)
-{
- return -ENODEV;
-}
-
-static inline bool acpi_can_fallback_to_crs(struct acpi_device *adev,
- const char *con_id)
-{
- return false;
-}
-#endif
-
struct gpio_array {
struct gpio_desc **desc;
unsigned int size;
@@ -210,7 +90,7 @@ int gpiod_set_array_value_complex(bool raw, bool can_sleep,
struct gpio_array *array_info,
unsigned long *value_bitmap);
-extern struct spinlock gpio_lock;
+extern spinlock_t gpio_lock;
extern struct list_head gpio_devices;
struct gpio_desc {
diff --git a/drivers/gpio/sgpio-aspeed.c b/drivers/gpio/sgpio-aspeed.c
new file mode 100644
index 000000000000..7e99860ca447
--- /dev/null
+++ b/drivers/gpio/sgpio-aspeed.c
@@ -0,0 +1,533 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2019 American Megatrends International LLC.
+ *
+ * Author: Karthikeyan Mani <karthikeyanm@amiindia.co.in>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/gpio/driver.h>
+#include <linux/hashtable.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+
+#define MAX_NR_SGPIO 80
+
+#define ASPEED_SGPIO_CTRL 0x54
+
+#define ASPEED_SGPIO_PINS_MASK GENMASK(9, 6)
+#define ASPEED_SGPIO_CLK_DIV_MASK GENMASK(31, 16)
+#define ASPEED_SGPIO_ENABLE BIT(0)
+
+struct aspeed_sgpio {
+ struct gpio_chip chip;
+ struct clk *pclk;
+ spinlock_t lock;
+ void __iomem *base;
+ uint32_t dir_in[3];
+ int irq;
+};
+
+struct aspeed_sgpio_bank {
+ uint16_t val_regs;
+ uint16_t rdata_reg;
+ uint16_t irq_regs;
+ const char names[4][3];
+};
+
+/*
+ * Note: The "value" register returns the input value when the GPIO is
+ * configured as an input.
+ *
+ * The "rdata" register returns the output value when the GPIO is
+ * configured as an output.
+ */
+static const struct aspeed_sgpio_bank aspeed_sgpio_banks[] = {
+ {
+ .val_regs = 0x0000,
+ .rdata_reg = 0x0070,
+ .irq_regs = 0x0004,
+ .names = { "A", "B", "C", "D" },
+ },
+ {
+ .val_regs = 0x001C,
+ .rdata_reg = 0x0074,
+ .irq_regs = 0x0020,
+ .names = { "E", "F", "G", "H" },
+ },
+ {
+ .val_regs = 0x0038,
+ .rdata_reg = 0x0078,
+ .irq_regs = 0x003C,
+ .names = { "I", "J" },
+ },
+};
+
+enum aspeed_sgpio_reg {
+ reg_val,
+ reg_rdata,
+ reg_irq_enable,
+ reg_irq_type0,
+ reg_irq_type1,
+ reg_irq_type2,
+ reg_irq_status,
+};
+
+#define GPIO_VAL_VALUE 0x00
+#define GPIO_IRQ_ENABLE 0x00
+#define GPIO_IRQ_TYPE0 0x04
+#define GPIO_IRQ_TYPE1 0x08
+#define GPIO_IRQ_TYPE2 0x0C
+#define GPIO_IRQ_STATUS 0x10
+
+static void __iomem *bank_reg(struct aspeed_sgpio *gpio,
+ const struct aspeed_sgpio_bank *bank,
+ const enum aspeed_sgpio_reg reg)
+{
+ switch (reg) {
+ case reg_val:
+ return gpio->base + bank->val_regs + GPIO_VAL_VALUE;
+ case reg_rdata:
+ return gpio->base + bank->rdata_reg;
+ case reg_irq_enable:
+ return gpio->base + bank->irq_regs + GPIO_IRQ_ENABLE;
+ case reg_irq_type0:
+ return gpio->base + bank->irq_regs + GPIO_IRQ_TYPE0;
+ case reg_irq_type1:
+ return gpio->base + bank->irq_regs + GPIO_IRQ_TYPE1;
+ case reg_irq_type2:
+ return gpio->base + bank->irq_regs + GPIO_IRQ_TYPE2;
+ case reg_irq_status:
+ return gpio->base + bank->irq_regs + GPIO_IRQ_STATUS;
+ default:
+ /* acturally if code runs to here, it's an error case */
+ BUG_ON(1);
+ }
+}
+
+#define GPIO_BANK(x) ((x) >> 5)
+#define GPIO_OFFSET(x) ((x) & 0x1f)
+#define GPIO_BIT(x) BIT(GPIO_OFFSET(x))
+
+static const struct aspeed_sgpio_bank *to_bank(unsigned int offset)
+{
+ unsigned int bank = GPIO_BANK(offset);
+
+ WARN_ON(bank >= ARRAY_SIZE(aspeed_sgpio_banks));
+ return &aspeed_sgpio_banks[bank];
+}
+
+static int aspeed_sgpio_get(struct gpio_chip *gc, unsigned int offset)
+{
+ struct aspeed_sgpio *gpio = gpiochip_get_data(gc);
+ const struct aspeed_sgpio_bank *bank = to_bank(offset);
+ unsigned long flags;
+ enum aspeed_sgpio_reg reg;
+ bool is_input;
+ int rc = 0;
+
+ spin_lock_irqsave(&gpio->lock, flags);
+
+ is_input = gpio->dir_in[GPIO_BANK(offset)] & GPIO_BIT(offset);
+ reg = is_input ? reg_val : reg_rdata;
+ rc = !!(ioread32(bank_reg(gpio, bank, reg)) & GPIO_BIT(offset));
+
+ spin_unlock_irqrestore(&gpio->lock, flags);
+
+ return rc;
+}
+
+static void sgpio_set_value(struct gpio_chip *gc, unsigned int offset, int val)
+{
+ struct aspeed_sgpio *gpio = gpiochip_get_data(gc);
+ const struct aspeed_sgpio_bank *bank = to_bank(offset);
+ void __iomem *addr;
+ u32 reg = 0;
+
+ addr = bank_reg(gpio, bank, reg_val);
+ reg = ioread32(addr);
+
+ if (val)
+ reg |= GPIO_BIT(offset);
+ else
+ reg &= ~GPIO_BIT(offset);
+
+ iowrite32(reg, addr);
+}
+
+static void aspeed_sgpio_set(struct gpio_chip *gc, unsigned int offset, int val)
+{
+ struct aspeed_sgpio *gpio = gpiochip_get_data(gc);
+ unsigned long flags;
+
+ spin_lock_irqsave(&gpio->lock, flags);
+
+ sgpio_set_value(gc, offset, val);
+
+ spin_unlock_irqrestore(&gpio->lock, flags);
+}
+
+static int aspeed_sgpio_dir_in(struct gpio_chip *gc, unsigned int offset)
+{
+ struct aspeed_sgpio *gpio = gpiochip_get_data(gc);
+ unsigned long flags;
+
+ spin_lock_irqsave(&gpio->lock, flags);
+ gpio->dir_in[GPIO_BANK(offset)] |= GPIO_BIT(offset);
+ spin_unlock_irqrestore(&gpio->lock, flags);
+
+ return 0;
+}
+
+static int aspeed_sgpio_dir_out(struct gpio_chip *gc, unsigned int offset, int val)
+{
+ struct aspeed_sgpio *gpio = gpiochip_get_data(gc);
+ unsigned long flags;
+
+ spin_lock_irqsave(&gpio->lock, flags);
+
+ gpio->dir_in[GPIO_BANK(offset)] &= ~GPIO_BIT(offset);
+ sgpio_set_value(gc, offset, val);
+
+ spin_unlock_irqrestore(&gpio->lock, flags);
+
+ return 0;
+}
+
+static int aspeed_sgpio_get_direction(struct gpio_chip *gc, unsigned int offset)
+{
+ int dir_status;
+ struct aspeed_sgpio *gpio = gpiochip_get_data(gc);
+ unsigned long flags;
+
+ spin_lock_irqsave(&gpio->lock, flags);
+ dir_status = gpio->dir_in[GPIO_BANK(offset)] & GPIO_BIT(offset);
+ spin_unlock_irqrestore(&gpio->lock, flags);
+
+ return dir_status;
+
+}
+
+static void irqd_to_aspeed_sgpio_data(struct irq_data *d,
+ struct aspeed_sgpio **gpio,
+ const struct aspeed_sgpio_bank **bank,
+ u32 *bit, int *offset)
+{
+ struct aspeed_sgpio *internal;
+
+ *offset = irqd_to_hwirq(d);
+ internal = irq_data_get_irq_chip_data(d);
+ WARN_ON(!internal);
+
+ *gpio = internal;
+ *bank = to_bank(*offset);
+ *bit = GPIO_BIT(*offset);
+}
+
+static void aspeed_sgpio_irq_ack(struct irq_data *d)
+{
+ const struct aspeed_sgpio_bank *bank;
+ struct aspeed_sgpio *gpio;
+ unsigned long flags;
+ void __iomem *status_addr;
+ int offset;
+ u32 bit;
+
+ irqd_to_aspeed_sgpio_data(d, &gpio, &bank, &bit, &offset);
+
+ status_addr = bank_reg(gpio, bank, reg_irq_status);
+
+ spin_lock_irqsave(&gpio->lock, flags);
+
+ iowrite32(bit, status_addr);
+
+ spin_unlock_irqrestore(&gpio->lock, flags);
+}
+
+static void aspeed_sgpio_irq_set_mask(struct irq_data *d, bool set)
+{
+ const struct aspeed_sgpio_bank *bank;
+ struct aspeed_sgpio *gpio;
+ unsigned long flags;
+ u32 reg, bit;
+ void __iomem *addr;
+ int offset;
+
+ irqd_to_aspeed_sgpio_data(d, &gpio, &bank, &bit, &offset);
+ addr = bank_reg(gpio, bank, reg_irq_enable);
+
+ spin_lock_irqsave(&gpio->lock, flags);
+
+ reg = ioread32(addr);
+ if (set)
+ reg |= bit;
+ else
+ reg &= ~bit;
+
+ iowrite32(reg, addr);
+
+ spin_unlock_irqrestore(&gpio->lock, flags);
+}
+
+static void aspeed_sgpio_irq_mask(struct irq_data *d)
+{
+ aspeed_sgpio_irq_set_mask(d, false);
+}
+
+static void aspeed_sgpio_irq_unmask(struct irq_data *d)
+{
+ aspeed_sgpio_irq_set_mask(d, true);
+}
+
+static int aspeed_sgpio_set_type(struct irq_data *d, unsigned int type)
+{
+ u32 type0 = 0;
+ u32 type1 = 0;
+ u32 type2 = 0;
+ u32 bit, reg;
+ const struct aspeed_sgpio_bank *bank;
+ irq_flow_handler_t handler;
+ struct aspeed_sgpio *gpio;
+ unsigned long flags;
+ void __iomem *addr;
+ int offset;
+
+ irqd_to_aspeed_sgpio_data(d, &gpio, &bank, &bit, &offset);
+
+ switch (type & IRQ_TYPE_SENSE_MASK) {
+ case IRQ_TYPE_EDGE_BOTH:
+ type2 |= bit;
+ /* fall through */
+ case IRQ_TYPE_EDGE_RISING:
+ type0 |= bit;
+ /* fall through */
+ case IRQ_TYPE_EDGE_FALLING:
+ handler = handle_edge_irq;
+ break;
+ case IRQ_TYPE_LEVEL_HIGH:
+ type0 |= bit;
+ /* fall through */
+ case IRQ_TYPE_LEVEL_LOW:
+ type1 |= bit;
+ handler = handle_level_irq;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ spin_lock_irqsave(&gpio->lock, flags);
+
+ addr = bank_reg(gpio, bank, reg_irq_type0);
+ reg = ioread32(addr);
+ reg = (reg & ~bit) | type0;
+ iowrite32(reg, addr);
+
+ addr = bank_reg(gpio, bank, reg_irq_type1);
+ reg = ioread32(addr);
+ reg = (reg & ~bit) | type1;
+ iowrite32(reg, addr);
+
+ addr = bank_reg(gpio, bank, reg_irq_type2);
+ reg = ioread32(addr);
+ reg = (reg & ~bit) | type2;
+ iowrite32(reg, addr);
+
+ spin_unlock_irqrestore(&gpio->lock, flags);
+
+ irq_set_handler_locked(d, handler);
+
+ return 0;
+}
+
+static void aspeed_sgpio_irq_handler(struct irq_desc *desc)
+{
+ struct gpio_chip *gc = irq_desc_get_handler_data(desc);
+ struct irq_chip *ic = irq_desc_get_chip(desc);
+ struct aspeed_sgpio *data = gpiochip_get_data(gc);
+ unsigned int i, p, girq;
+ unsigned long reg;
+
+ chained_irq_enter(ic, desc);
+
+ for (i = 0; i < ARRAY_SIZE(aspeed_sgpio_banks); i++) {
+ const struct aspeed_sgpio_bank *bank = &aspeed_sgpio_banks[i];
+
+ reg = ioread32(bank_reg(data, bank, reg_irq_status));
+
+ for_each_set_bit(p, &reg, 32) {
+ girq = irq_find_mapping(gc->irq.domain, i * 32 + p);
+ generic_handle_irq(girq);
+ }
+
+ }
+
+ chained_irq_exit(ic, desc);
+}
+
+static struct irq_chip aspeed_sgpio_irqchip = {
+ .name = "aspeed-sgpio",
+ .irq_ack = aspeed_sgpio_irq_ack,
+ .irq_mask = aspeed_sgpio_irq_mask,
+ .irq_unmask = aspeed_sgpio_irq_unmask,
+ .irq_set_type = aspeed_sgpio_set_type,
+};
+
+static int aspeed_sgpio_setup_irqs(struct aspeed_sgpio *gpio,
+ struct platform_device *pdev)
+{
+ int rc, i;
+ const struct aspeed_sgpio_bank *bank;
+ struct gpio_irq_chip *irq;
+
+ rc = platform_get_irq(pdev, 0);
+ if (rc < 0)
+ return rc;
+
+ gpio->irq = rc;
+
+ /* Disable IRQ and clear Interrupt status registers for all SPGIO Pins. */
+ for (i = 0; i < ARRAY_SIZE(aspeed_sgpio_banks); i++) {
+ bank = &aspeed_sgpio_banks[i];
+ /* disable irq enable bits */
+ iowrite32(0x00000000, bank_reg(gpio, bank, reg_irq_enable));
+ /* clear status bits */
+ iowrite32(0xffffffff, bank_reg(gpio, bank, reg_irq_status));
+ }
+
+ irq = &gpio->chip.irq;
+ irq->chip = &aspeed_sgpio_irqchip;
+ irq->handler = handle_bad_irq;
+ irq->default_type = IRQ_TYPE_NONE;
+ irq->parent_handler = aspeed_sgpio_irq_handler;
+ irq->parent_handler_data = gpio;
+ irq->parents = &gpio->irq;
+ irq->num_parents = 1;
+
+ /* set IRQ settings and Enable Interrupt */
+ for (i = 0; i < ARRAY_SIZE(aspeed_sgpio_banks); i++) {
+ bank = &aspeed_sgpio_banks[i];
+ /* set falling or level-low irq */
+ iowrite32(0x00000000, bank_reg(gpio, bank, reg_irq_type0));
+ /* trigger type is edge */
+ iowrite32(0x00000000, bank_reg(gpio, bank, reg_irq_type1));
+ /* dual edge trigger mode. */
+ iowrite32(0xffffffff, bank_reg(gpio, bank, reg_irq_type2));
+ /* enable irq */
+ iowrite32(0xffffffff, bank_reg(gpio, bank, reg_irq_enable));
+ }
+
+ return 0;
+}
+
+static const struct of_device_id aspeed_sgpio_of_table[] = {
+ { .compatible = "aspeed,ast2400-sgpio" },
+ { .compatible = "aspeed,ast2500-sgpio" },
+ {}
+};
+
+MODULE_DEVICE_TABLE(of, aspeed_sgpio_of_table);
+
+static int __init aspeed_sgpio_probe(struct platform_device *pdev)
+{
+ struct aspeed_sgpio *gpio;
+ u32 nr_gpios, sgpio_freq, sgpio_clk_div;
+ int rc;
+ unsigned long apb_freq;
+
+ gpio = devm_kzalloc(&pdev->dev, sizeof(*gpio), GFP_KERNEL);
+ if (!gpio)
+ return -ENOMEM;
+
+ gpio->base = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(gpio->base))
+ return PTR_ERR(gpio->base);
+
+ rc = of_property_read_u32(pdev->dev.of_node, "ngpios", &nr_gpios);
+ if (rc < 0) {
+ dev_err(&pdev->dev, "Could not read ngpios property\n");
+ return -EINVAL;
+ } else if (nr_gpios > MAX_NR_SGPIO) {
+ dev_err(&pdev->dev, "Number of GPIOs exceeds the maximum of %d: %d\n",
+ MAX_NR_SGPIO, nr_gpios);
+ return -EINVAL;
+ }
+
+ rc = of_property_read_u32(pdev->dev.of_node, "bus-frequency", &sgpio_freq);
+ if (rc < 0) {
+ dev_err(&pdev->dev, "Could not read bus-frequency property\n");
+ return -EINVAL;
+ }
+
+ gpio->pclk = devm_clk_get(&pdev->dev, NULL);
+ if (IS_ERR(gpio->pclk)) {
+ dev_err(&pdev->dev, "devm_clk_get failed\n");
+ return PTR_ERR(gpio->pclk);
+ }
+
+ apb_freq = clk_get_rate(gpio->pclk);
+
+ /*
+ * From the datasheet,
+ * SGPIO period = 1/PCLK * 2 * (GPIO254[31:16] + 1)
+ * period = 2 * (GPIO254[31:16] + 1) / PCLK
+ * frequency = 1 / (2 * (GPIO254[31:16] + 1) / PCLK)
+ * frequency = PCLK / (2 * (GPIO254[31:16] + 1))
+ * frequency * 2 * (GPIO254[31:16] + 1) = PCLK
+ * GPIO254[31:16] = PCLK / (frequency * 2) - 1
+ */
+ if (sgpio_freq == 0)
+ return -EINVAL;
+
+ sgpio_clk_div = (apb_freq / (sgpio_freq * 2)) - 1;
+
+ if (sgpio_clk_div > (1 << 16) - 1)
+ return -EINVAL;
+
+ iowrite32(FIELD_PREP(ASPEED_SGPIO_CLK_DIV_MASK, sgpio_clk_div) |
+ FIELD_PREP(ASPEED_SGPIO_PINS_MASK, (nr_gpios / 8)) |
+ ASPEED_SGPIO_ENABLE,
+ gpio->base + ASPEED_SGPIO_CTRL);
+
+ spin_lock_init(&gpio->lock);
+
+ gpio->chip.parent = &pdev->dev;
+ gpio->chip.ngpio = nr_gpios;
+ gpio->chip.direction_input = aspeed_sgpio_dir_in;
+ gpio->chip.direction_output = aspeed_sgpio_dir_out;
+ gpio->chip.get_direction = aspeed_sgpio_get_direction;
+ gpio->chip.request = NULL;
+ gpio->chip.free = NULL;
+ gpio->chip.get = aspeed_sgpio_get;
+ gpio->chip.set = aspeed_sgpio_set;
+ gpio->chip.set_config = NULL;
+ gpio->chip.label = dev_name(&pdev->dev);
+ gpio->chip.base = -1;
+
+ /* set all SGPIO pins as input (1). */
+ memset(gpio->dir_in, 0xff, sizeof(gpio->dir_in));
+
+ aspeed_sgpio_setup_irqs(gpio, pdev);
+
+ rc = devm_gpiochip_add_data(&pdev->dev, &gpio->chip, gpio);
+ if (rc < 0)
+ return rc;
+
+ return 0;
+}
+
+static struct platform_driver aspeed_sgpio_driver = {
+ .driver = {
+ .name = KBUILD_MODNAME,
+ .of_match_table = aspeed_sgpio_of_table,
+ },
+};
+
+module_platform_driver_probe(aspeed_sgpio_driver, aspeed_sgpio_probe);
+MODULE_DESCRIPTION("Aspeed Serial GPIO Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index 36f900d63979..3c88420e3497 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -141,7 +141,7 @@ config DRM_LOAD_EDID_FIRMWARE
monitor are unable to provide appropriate EDID data. Since this
feature is provided as a workaround for broken hardware, the
default case is N. Details and instructions how to build your own
- EDID data are given in Documentation/EDID/HOWTO.txt.
+ EDID data are given in Documentation/driver-api/edid.rst.
config DRM_DP_CEC
bool "Enable DisplayPort CEC-Tunneling-over-AUX HDMI support"
@@ -161,6 +161,13 @@ config DRM_TTM
GPU memory types. Will be enabled automatically if a device driver
uses it.
+config DRM_VRAM_HELPER
+ tristate
+ depends on DRM
+ select DRM_TTM
+ help
+ Helpers for VRAM memory management
+
config DRM_GEM_CMA_HELPER
bool
depends on DRM
@@ -309,6 +316,8 @@ source "drivers/gpu/drm/sti/Kconfig"
source "drivers/gpu/drm/imx/Kconfig"
+source "drivers/gpu/drm/ingenic/Kconfig"
+
source "drivers/gpu/drm/v3d/Kconfig"
source "drivers/gpu/drm/vc4/Kconfig"
@@ -343,6 +352,8 @@ source "drivers/gpu/drm/panfrost/Kconfig"
source "drivers/gpu/drm/aspeed/Kconfig"
+source "drivers/gpu/drm/mcde/Kconfig"
+
# Keep legacy drivers last
menuconfig DRM_LEGACY
@@ -383,7 +394,7 @@ config DRM_R128
config DRM_I810
tristate "Intel I810"
# !PREEMPT because of missing ioctl locking
- depends on DRM && AGP && AGP_INTEL && (!PREEMPT || BROKEN)
+ depends on DRM && AGP && AGP_INTEL && (!PREEMPTION || BROKEN)
help
Choose this option if you have an Intel I810 graphics card. If M is
selected, the module will be called i810. AGP support is required
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index 72f5036d9bfa..9f0d2ee35794 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -17,7 +17,7 @@ drm-y := drm_auth.o drm_cache.o \
drm_plane.o drm_color_mgmt.o drm_print.o \
drm_dumb_buffers.o drm_mode_config.o drm_vblank.o \
drm_syncobj.o drm_lease.o drm_writeback.o drm_client.o \
- drm_atomic_uapi.o
+ drm_client_modeset.o drm_atomic_uapi.o drm_hdcp.o
drm-$(CONFIG_DRM_LEGACY) += drm_legacy_misc.o drm_bufs.o drm_context.o drm_dma.o drm_scatter.o drm_lock.o
drm-$(CONFIG_DRM_LIB_RANDOM) += lib/drm_random.o
@@ -32,13 +32,18 @@ drm-$(CONFIG_AGP) += drm_agpsupport.o
drm-$(CONFIG_DEBUG_FS) += drm_debugfs.o drm_debugfs_crc.o
drm-$(CONFIG_DRM_LOAD_EDID_FIRMWARE) += drm_edid_load.o
+drm_vram_helper-y := drm_gem_vram_helper.o \
+ drm_vram_helper_common.o \
+ drm_vram_mm_helper.o
+obj-$(CONFIG_DRM_VRAM_HELPER) += drm_vram_helper.o
+
drm_kms_helper-y := drm_crtc_helper.o drm_dp_helper.o drm_dsc.o drm_probe_helper.o \
drm_plane_helper.o drm_dp_mst_topology.o drm_atomic_helper.o \
drm_kms_helper_common.o drm_dp_dual_mode_helper.o \
drm_simple_kms_helper.o drm_modeset_helper.o \
drm_scdc_helper.o drm_gem_framebuffer_helper.o \
drm_atomic_state_helper.o drm_damage_helper.o \
- drm_format_helper.o
+ drm_format_helper.o drm_self_refresh_helper.o
drm_kms_helper-$(CONFIG_DRM_PANEL_BRIDGE) += bridge/panel.o
drm_kms_helper-$(CONFIG_DRM_FBDEV_EMULATION) += drm_fb_helper.o
@@ -94,6 +99,7 @@ obj-$(CONFIG_DRM_TEGRA) += tegra/
obj-$(CONFIG_DRM_STM) += stm/
obj-$(CONFIG_DRM_STI) += sti/
obj-$(CONFIG_DRM_IMX) += imx/
+obj-$(CONFIG_DRM_INGENIC) += ingenic/
obj-$(CONFIG_DRM_MEDIATEK) += mediatek/
obj-$(CONFIG_DRM_MESON) += meson/
obj-y += i2c/
@@ -113,3 +119,4 @@ obj-$(CONFIG_DRM_VBOXVIDEO) += vboxvideo/
obj-$(CONFIG_DRM_LIMA) += lima/
obj-$(CONFIG_DRM_PANFROST) += panfrost/
obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/
+obj-$(CONFIG_DRM_MCDE) += mcde/
diff --git a/drivers/gpu/drm/amd/amdgpu/Kconfig b/drivers/gpu/drm/amd/amdgpu/Kconfig
index 844f0a162981..f6e5c0282fc1 100644
--- a/drivers/gpu/drm/amd/amdgpu/Kconfig
+++ b/drivers/gpu/drm/amd/amdgpu/Kconfig
@@ -27,10 +27,10 @@ config DRM_AMDGPU_CIK
config DRM_AMDGPU_USERPTR
bool "Always enable userptr write support"
depends on DRM_AMDGPU
- select MMU_NOTIFIER
+ depends on HMM_MIRROR
help
- This option selects CONFIG_MMU_NOTIFIER if it isn't already
- selected to enabled full userptr support.
+ This option selects CONFIG_HMM and CONFIG_HMM_MIRROR if it
+ isn't already selected to enabled full userptr support.
config DRM_AMDGPU_GART_DEBUGFS
bool "Allow GART access through debugfs"
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index fdd0ca4b0f0b..56e084367b93 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -49,12 +49,14 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
amdgpu_cs.o amdgpu_bios.o amdgpu_benchmark.o amdgpu_test.o \
amdgpu_pm.o atombios_dp.o amdgpu_afmt.o amdgpu_trace_points.o \
atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
- amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
+ amdgpu_dma_buf.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o amdgpu_atomfirmware.o \
amdgpu_vf_error.o amdgpu_sched.o amdgpu_debugfs.o amdgpu_ids.o \
amdgpu_gmc.o amdgpu_xgmi.o amdgpu_csa.o amdgpu_ras.o amdgpu_vm_cpu.o \
- amdgpu_vm_sdma.o
+ amdgpu_vm_sdma.o amdgpu_discovery.o
+
+amdgpu-$(CONFIG_PERF_EVENTS) += amdgpu_pmu.o
# add asic specific block
amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
@@ -64,7 +66,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce
amdgpu-y += \
vi.o mxgpu_vi.o nbio_v6_1.o soc15.o emu_soc.o mxgpu_ai.o nbio_v7_0.o vega10_reg_init.o \
- vega20_reg_init.o nbio_v7_4.o
+ vega20_reg_init.o nbio_v7_4.o nbio_v2_3.o nv.o navi10_reg_init.o
# add DF block
amdgpu-y += \
@@ -75,7 +77,8 @@ amdgpu-y += \
amdgpu-y += \
gmc_v7_0.o \
gmc_v8_0.o \
- gfxhub_v1_0.o mmhub_v1_0.o gmc_v9_0.o gfxhub_v1_1.o
+ gfxhub_v1_0.o mmhub_v1_0.o gmc_v9_0.o gfxhub_v1_1.o \
+ gfxhub_v2_0.o mmhub_v2_0.o gmc_v10_0.o
# add IH block
amdgpu-y += \
@@ -84,7 +87,8 @@ amdgpu-y += \
iceland_ih.o \
tonga_ih.o \
cz_ih.o \
- vega10_ih.o
+ vega10_ih.o \
+ navi10_ih.o
# add PSP block
amdgpu-y += \
@@ -108,14 +112,20 @@ amdgpu-y += \
amdgpu_gfx.o \
amdgpu_rlc.o \
gfx_v8_0.o \
- gfx_v9_0.o
+ gfx_v9_0.o \
+ gfx_v10_0.o
# add async DMA block
amdgpu-y += \
amdgpu_sdma.o \
sdma_v2_4.o \
sdma_v3_0.o \
- sdma_v4_0.o
+ sdma_v4_0.o \
+ sdma_v5_0.o
+
+# add MES block
+amdgpu-y += \
+ mes_v10_1.o
# add UVD block
amdgpu-y += \
@@ -133,7 +143,12 @@ amdgpu-y += \
# add VCN block
amdgpu-y += \
amdgpu_vcn.o \
- vcn_v1_0.o
+ vcn_v1_0.o \
+ vcn_v2_0.o
+
+# add ATHUB block
+amdgpu-y += \
+ athub_v2_0.o
# add amdkfd interfaces
amdgpu-y += amdgpu_amdkfd.o
@@ -146,7 +161,8 @@ amdgpu-y += \
amdgpu_amdkfd_fence.o \
amdgpu_amdkfd_gpuvm.o \
amdgpu_amdkfd_gfx_v8.o \
- amdgpu_amdkfd_gfx_v9.o
+ amdgpu_amdkfd_gfx_v9.o \
+ amdgpu_amdkfd_gfx_v10.o
ifneq ($(CONFIG_DRM_AMDGPU_CIK),)
amdgpu-y += amdgpu_amdkfd_gfx_v7.o
@@ -173,7 +189,7 @@ endif
amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o
amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o
amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o
-amdgpu-$(CONFIG_MMU_NOTIFIER) += amdgpu_mn.o
+amdgpu-$(CONFIG_HMM_MIRROR) += amdgpu_mn.o
include $(FULL_AMD_PATH)/powerplay/Makefile
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 14398f55f602..8199d201b43a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -44,9 +44,9 @@
#include <drm/ttm/ttm_module.h>
#include <drm/ttm/ttm_execbuf_util.h>
-#include <drm/drmP.h>
-#include <drm/drm_gem.h>
#include <drm/amdgpu_drm.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_ioctl.h>
#include <drm/gpu_scheduler.h>
#include <kgd_kfd_interface.h>
@@ -84,6 +84,8 @@
#include "amdgpu_doorbell.h"
#include "amdgpu_amdkfd.h"
#include "amdgpu_smu.h"
+#include "amdgpu_discovery.h"
+#include "amdgpu_mes.h"
#define MAX_GPU_INSTANCE 16
@@ -118,7 +120,6 @@ extern int amdgpu_disp_priority;
extern int amdgpu_hw_i2c;
extern int amdgpu_pcie_gen2;
extern int amdgpu_msi;
-extern int amdgpu_lockup_timeout;
extern int amdgpu_dpm;
extern int amdgpu_fw_load_type;
extern int amdgpu_aspm;
@@ -143,7 +144,6 @@ extern uint amdgpu_sdma_phase_quantum;
extern char *amdgpu_disable_cu;
extern char *amdgpu_virtual_display;
extern uint amdgpu_pp_feature_mask;
-extern int amdgpu_vram_page_split;
extern int amdgpu_ngg;
extern int amdgpu_prim_buf_per_se;
extern int amdgpu_pos_buf_per_se;
@@ -156,9 +156,15 @@ extern int amdgpu_gpu_recovery;
extern int amdgpu_emu_mode;
extern uint amdgpu_smu_memory_pool_size;
extern uint amdgpu_dc_feature_mask;
+extern uint amdgpu_dm_abm_level;
extern struct amdgpu_mgpu_info mgpu_info;
extern int amdgpu_ras_enable;
extern uint amdgpu_ras_mask;
+extern int amdgpu_async_gfx_ring;
+extern int amdgpu_mcbp;
+extern int amdgpu_discovery;
+extern int amdgpu_mes;
+extern int amdgpu_noretry;
#ifdef CONFIG_DRM_AMDGPU_SI
extern int amdgpu_si_support;
@@ -211,9 +217,11 @@ struct amdgpu_irq_src;
struct amdgpu_fpriv;
struct amdgpu_bo_va_mapping;
struct amdgpu_atif;
+struct kfd_vm_fault_info;
enum amdgpu_cp_irq {
- AMDGPU_CP_IRQ_GFX_EOP = 0,
+ AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP = 0,
+ AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP,
AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP,
AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP,
AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP,
@@ -415,6 +423,7 @@ struct amdgpu_fpriv {
};
int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv);
+int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev);
int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm,
unsigned size, struct amdgpu_ib *ib);
@@ -558,6 +567,8 @@ struct amdgpu_asic_funcs {
uint64_t *count1);
/* do we need to reset the asic at init time (e.g., kexec) */
bool (*need_reset_on_init)(struct amdgpu_device *adev);
+ /* PCIe replay counter */
+ uint64_t (*get_pcie_replay_count)(struct amdgpu_device *adev);
};
/*
@@ -639,6 +650,11 @@ struct nbio_hdp_flush_reg {
u32 ref_and_mask_sdma1;
};
+struct amdgpu_mmio_remap {
+ u32 reg_offset;
+ resource_size_t bus_addr;
+};
+
struct amdgpu_nbio_funcs {
const struct nbio_hdp_flush_reg *hdp_flush_reg;
u32 (*get_hdp_flush_req_offset)(struct amdgpu_device *adev);
@@ -651,6 +667,8 @@ struct amdgpu_nbio_funcs {
u32 (*get_memsize)(struct amdgpu_device *adev);
void (*sdma_doorbell_range)(struct amdgpu_device *adev, int instance,
bool use_doorbell, int doorbell_index, int doorbell_size);
+ void (*vcn_doorbell_range)(struct amdgpu_device *adev, bool use_doorbell,
+ int doorbell_index);
void (*enable_doorbell_aperture)(struct amdgpu_device *adev,
bool enable);
void (*enable_doorbell_selfring_aperture)(struct amdgpu_device *adev,
@@ -666,10 +684,11 @@ struct amdgpu_nbio_funcs {
void (*ih_control)(struct amdgpu_device *adev);
void (*init_registers)(struct amdgpu_device *adev);
void (*detect_hw_virt)(struct amdgpu_device *adev);
+ void (*remap_hdp_registers)(struct amdgpu_device *adev);
};
struct amdgpu_df_funcs {
- void (*init)(struct amdgpu_device *adev);
+ void (*sw_init)(struct amdgpu_device *adev);
void (*enable_broadcast_mode)(struct amdgpu_device *adev,
bool enable);
u32 (*get_fb_channel_number)(struct amdgpu_device *adev);
@@ -680,6 +699,12 @@ struct amdgpu_df_funcs {
u32 *flags);
void (*enable_ecc_force_par_wr_rmw)(struct amdgpu_device *adev,
bool enable);
+ int (*pmc_start)(struct amdgpu_device *adev, uint64_t config,
+ int is_enable);
+ int (*pmc_stop)(struct amdgpu_device *adev, uint64_t config,
+ int is_disable);
+ void (*pmc_get_count)(struct amdgpu_device *adev, uint64_t config,
+ uint64_t *count);
};
/* Define the HW IP blocks will be used in driver , add more if necessary */
enum amd_hw_ip_block_type {
@@ -714,6 +739,7 @@ struct amd_powerplay {
};
#define AMDGPU_RESET_MAGIC_NUM 64
+#define AMDGPU_MAX_DF_PERFMONS 4
struct amdgpu_device {
struct device *dev;
struct drm_device *ddev;
@@ -740,6 +766,7 @@ struct amdgpu_device {
struct amdgpu_debugfs debugfs[AMDGPU_DEBUGFS_MAX_COMPONENTS];
unsigned debugfs_count;
#if defined(CONFIG_DEBUG_FS)
+ struct dentry *debugfs_preempt;
struct dentry *debugfs_regs[AMDGPU_DEBUGFS_MAX_COMPONENTS];
#endif
struct amdgpu_atif *atif;
@@ -749,6 +776,7 @@ struct amdgpu_device {
struct mutex grbm_idx_mutex;
struct dev_pm_domain vga_pm_domain;
bool have_disp_power_ref;
+ bool have_atomics_support;
/* BIOS */
bool is_atom_fw;
@@ -764,6 +792,7 @@ struct amdgpu_device {
void __iomem *rmmio;
/* protects concurrent MM_INDEX/DATA based register access */
spinlock_t mmio_idx_lock;
+ struct amdgpu_mmio_remap rmmio_remap;
/* protects concurrent SMC based register access */
spinlock_t smc_idx_lock;
amdgpu_rreg_t smc_rreg;
@@ -889,6 +918,13 @@ struct amdgpu_device {
/* display related functionality */
struct amdgpu_display_manager dm;
+ /* discovery */
+ uint8_t *discovery;
+
+ /* mes */
+ bool enable_mes;
+ struct amdgpu_mes mes;
+
struct amdgpu_ip_block ip_blocks[AMDGPU_MAX_IP_NUM];
int num_ip_blocks;
struct mutex mn_lock;
@@ -906,7 +942,7 @@ struct amdgpu_device {
const struct amdgpu_df_funcs *df_funcs;
/* delayed work_func for deferring clockgating during resume */
- struct delayed_work late_init_work;
+ struct delayed_work delayed_init_work;
struct amdgpu_virt virt;
/* firmware VRAM reservation */
@@ -936,6 +972,14 @@ struct amdgpu_device {
struct work_struct xgmi_reset_work;
bool in_baco_reset;
+
+ long gfx_timeout;
+ long sdma_timeout;
+ long video_timeout;
+ long compute_timeout;
+
+ uint64_t unique_id;
+ uint64_t df_perfmon_config_assign_mask[AMDGPU_MAX_DF_PERFMONS];
};
static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
@@ -1065,6 +1109,7 @@ int emu_soc_asic_init(struct amdgpu_device *adev);
#define amdgpu_asic_init_doorbell_index(adev) (adev)->asic_funcs->init_doorbell_index((adev))
#define amdgpu_asic_get_pcie_usage(adev, cnt0, cnt1) ((adev)->asic_funcs->get_pcie_usage((adev), (cnt0), (cnt1)))
#define amdgpu_asic_need_reset_on_init(adev) (adev)->asic_funcs->need_reset_on_init((adev))
+#define amdgpu_asic_get_pcie_replay_count(adev) ((adev)->asic_funcs->get_pcie_replay_count((adev)))
/* Common functions */
bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev);
@@ -1081,6 +1126,9 @@ void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
const u32 array_size);
bool amdgpu_device_is_px(struct drm_device *dev);
+bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
+ struct amdgpu_device *peer_adev);
+
/* atpx handler */
#if defined(CONFIG_VGA_SWITCHEROO)
void amdgpu_register_atpx_handler(void);
@@ -1170,5 +1218,24 @@ int amdgpu_dm_display_resume(struct amdgpu_device *adev );
static inline int amdgpu_dm_display_resume(struct amdgpu_device *adev) { return 0; }
#endif
+
+void amdgpu_register_gpu_instance(struct amdgpu_device *adev);
+void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev);
+
#include "amdgpu_object.h"
+
+/* used by df_v3_6.c and amdgpu_pmu.c */
+#define AMDGPU_PMU_ATTR(_name, _object) \
+static ssize_t \
+_name##_show(struct device *dev, \
+ struct device_attribute *attr, \
+ char *page) \
+{ \
+ BUILD_BUG_ON(sizeof(_object) >= PAGE_SIZE - 1); \
+ return sprintf(page, _object "\n"); \
+} \
+ \
+static struct device_attribute pmu_attr_##_name = __ATTR_RO(_name)
+
#endif
+
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
index 0a4fba196b84..eba42c752bca 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
@@ -24,6 +24,7 @@
*/
#include <linux/irqdomain.h>
+#include <linux/pci.h>
#include <linux/pm_domain.h>
#include <linux/platform_device.h>
#include <sound/designware_i2s.h>
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
index 56f8ca2a3bb4..1e41367ef74e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
@@ -27,7 +27,7 @@
#include <linux/power_supply.h>
#include <linux/pm_runtime.h>
#include <acpi/video.h>
-#include <drm/drmP.h>
+
#include <drm/drm_crtc_helper.h>
#include "amdgpu.h"
#include "amdgpu_pm.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_afmt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_afmt.c
index 3889486f71fe..a4d65973bf7c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_afmt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_afmt.c
@@ -25,7 +25,7 @@
*/
#include <linux/hdmi.h>
#include <linux/gcd.h>
-#include <drm/drmP.h>
+
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index aeead072fa79..9fa4f25a3745 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -22,11 +22,13 @@
#include "amdgpu_amdkfd.h"
#include "amd_shared.h"
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_gfx.h"
+#include "amdgpu_dma_buf.h"
#include <linux/module.h>
#include <linux/dma-buf.h>
+#include "amdgpu_xgmi.h"
static const unsigned int compute_vmid_bitmap = 0xFF00;
@@ -76,6 +78,7 @@ void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev)
case CHIP_POLARIS10:
case CHIP_POLARIS11:
case CHIP_POLARIS12:
+ case CHIP_VEGAM:
kfd2kgd = amdgpu_amdkfd_gfx_8_0_get_functions();
break;
case CHIP_VEGA10:
@@ -84,6 +87,9 @@ void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev)
case CHIP_RAVEN:
kfd2kgd = amdgpu_amdkfd_gfx_9_0_get_functions();
break;
+ case CHIP_NAVI10:
+ kfd2kgd = amdgpu_amdkfd_gfx_10_0_get_functions();
+ break;
default:
dev_info(adev->dev, "kfd not supported on this ASIC\n");
return;
@@ -148,21 +154,23 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev)
};
/* this is going to have a few of the MSBs set that we need to
- * clear */
+ * clear
+ */
bitmap_complement(gpu_resources.queue_bitmap,
adev->gfx.mec.queue_bitmap,
KGD_MAX_QUEUES);
/* remove the KIQ bit as well */
if (adev->gfx.kiq.ring.sched.ready)
- clear_bit(amdgpu_gfx_queue_to_bit(adev,
+ clear_bit(amdgpu_gfx_mec_queue_to_bit(adev,
adev->gfx.kiq.ring.me - 1,
adev->gfx.kiq.ring.pipe,
adev->gfx.kiq.ring.queue),
gpu_resources.queue_bitmap);
/* According to linux/bitmap.h we shouldn't use bitmap_clear if
- * nbits is not compile time constant */
+ * nbits is not compile time constant
+ */
last_valid_bit = 1 /* only first MEC can have compute queues */
* adev->gfx.mec.num_pipe_per_mec
* adev->gfx.mec.num_queue_per_pipe;
@@ -335,6 +343,40 @@ void amdgpu_amdkfd_free_gtt_mem(struct kgd_dev *kgd, void *mem_obj)
amdgpu_bo_unref(&(bo));
}
+int amdgpu_amdkfd_alloc_gws(struct kgd_dev *kgd, size_t size,
+ void **mem_obj)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+ struct amdgpu_bo *bo = NULL;
+ struct amdgpu_bo_param bp;
+ int r;
+
+ memset(&bp, 0, sizeof(bp));
+ bp.size = size;
+ bp.byte_align = 1;
+ bp.domain = AMDGPU_GEM_DOMAIN_GWS;
+ bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
+ bp.type = ttm_bo_type_device;
+ bp.resv = NULL;
+
+ r = amdgpu_bo_create(adev, &bp, &bo);
+ if (r) {
+ dev_err(adev->dev,
+ "failed to allocate gws BO for amdkfd (%d)\n", r);
+ return r;
+ }
+
+ *mem_obj = bo;
+ return 0;
+}
+
+void amdgpu_amdkfd_free_gws(struct kgd_dev *kgd, void *mem_obj)
+{
+ struct amdgpu_bo *bo = (struct amdgpu_bo *)mem_obj;
+
+ amdgpu_bo_unref(&bo);
+}
+
uint32_t amdgpu_amdkfd_get_fw_version(struct kgd_dev *kgd,
enum kgd_engine_type type)
{
@@ -398,9 +440,12 @@ void amdgpu_amdkfd_get_local_mem_info(struct kgd_dev *kgd,
if (amdgpu_sriov_vf(adev))
mem_info->mem_clk_max = adev->clock.default_mclk / 100;
- else if (adev->powerplay.pp_funcs)
- mem_info->mem_clk_max = amdgpu_dpm_get_mclk(adev, false) / 100;
- else
+ else if (adev->powerplay.pp_funcs) {
+ if (amdgpu_emu_mode == 1)
+ mem_info->mem_clk_max = 0;
+ else
+ mem_info->mem_clk_max = amdgpu_dpm_get_mclk(adev, false) / 100;
+ } else
mem_info->mem_clk_max = 100;
}
@@ -518,6 +563,34 @@ uint64_t amdgpu_amdkfd_get_hive_id(struct kgd_dev *kgd)
return adev->gmc.xgmi.hive_id;
}
+uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src)
+{
+ struct amdgpu_device *peer_adev = (struct amdgpu_device *)src;
+ struct amdgpu_device *adev = (struct amdgpu_device *)dst;
+ int ret = amdgpu_xgmi_get_hops_count(adev, peer_adev);
+
+ if (ret < 0) {
+ DRM_ERROR("amdgpu: failed to get xgmi hops count between node %d and %d. ret = %d\n",
+ adev->gmc.xgmi.physical_node_id,
+ peer_adev->gmc.xgmi.physical_node_id, ret);
+ ret = 0;
+ }
+ return (uint8_t)ret;
+}
+
+uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+
+ return adev->rmmio_remap.bus_addr;
+}
+
+uint32_t amdgpu_amdkfd_get_num_gws(struct kgd_dev *kgd)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+
+ return adev->gds.gws_size;
+}
int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
uint32_t vmid, uint64_t gpu_addr,
@@ -595,6 +668,13 @@ bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid)
return false;
}
+bool amdgpu_amdkfd_have_atomics_support(struct kgd_dev *kgd)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+
+ return adev->have_atomics_support;
+}
+
#ifndef CONFIG_HSA_AMD
bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
{
@@ -635,6 +715,11 @@ struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void)
return NULL;
}
+struct kfd2kgd_calls *amdgpu_amdkfd_gfx_10_0_get_functions(void)
+{
+ return NULL;
+}
+
struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev,
const struct kfd2kgd_calls *f2g)
{
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 4e37fa7e85b1..b6076d19e442 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -61,7 +61,6 @@ struct kgd_mem {
atomic_t invalid;
struct amdkfd_process_info *process_info;
- struct page **user_pages;
struct amdgpu_sync sync;
@@ -136,10 +135,12 @@ int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
uint32_t vmid, uint64_t gpu_addr,
uint32_t *ib_cmd, uint32_t ib_len);
void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle);
+bool amdgpu_amdkfd_have_atomics_support(struct kgd_dev *kgd);
struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void);
struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void);
struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void);
+struct kfd2kgd_calls *amdgpu_amdkfd_gfx_10_0_get_functions(void);
bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);
@@ -154,6 +155,10 @@ int amdgpu_amdkfd_alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
void **mem_obj, uint64_t *gpu_addr,
void **cpu_ptr, bool mqd_gfx9);
void amdgpu_amdkfd_free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
+int amdgpu_amdkfd_alloc_gws(struct kgd_dev *kgd, size_t size, void **mem_obj);
+void amdgpu_amdkfd_free_gws(struct kgd_dev *kgd, void *mem_obj);
+int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem);
+int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem);
uint32_t amdgpu_amdkfd_get_fw_version(struct kgd_dev *kgd,
enum kgd_engine_type type);
void amdgpu_amdkfd_get_local_mem_info(struct kgd_dev *kgd,
@@ -169,6 +174,9 @@ int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd,
uint32_t *flags);
uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd);
uint64_t amdgpu_amdkfd_get_hive_id(struct kgd_dev *kgd);
+uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd);
+uint32_t amdgpu_amdkfd_get_num_gws(struct kgd_dev *kgd);
+uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src);
#define read_user_wptr(mmptr, wptr, dst) \
({ \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c
new file mode 100644
index 000000000000..0723f800e815
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c
@@ -0,0 +1,975 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#undef pr_fmt
+#define pr_fmt(fmt) "kfd2kgd: " fmt
+
+#include <linux/module.h>
+#include <linux/fdtable.h>
+#include <linux/uaccess.h>
+#include <linux/firmware.h>
+#include <linux/mmu_context.h>
+#include <drm/drmP.h>
+#include "amdgpu.h"
+#include "amdgpu_amdkfd.h"
+#include "amdgpu_ucode.h"
+#include "soc15_hw_ip.h"
+#include "gc/gc_10_1_0_offset.h"
+#include "gc/gc_10_1_0_sh_mask.h"
+#include "navi10_enum.h"
+#include "athub/athub_2_0_0_offset.h"
+#include "athub/athub_2_0_0_sh_mask.h"
+#include "oss/osssys_5_0_0_offset.h"
+#include "oss/osssys_5_0_0_sh_mask.h"
+#include "soc15_common.h"
+#include "v10_structs.h"
+#include "nv.h"
+#include "nvd.h"
+
+enum hqd_dequeue_request_type {
+ NO_ACTION = 0,
+ DRAIN_PIPE,
+ RESET_WAVES,
+ SAVE_WAVES
+};
+
+/*
+ * Register access functions
+ */
+
+static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
+ uint32_t sh_mem_config,
+ uint32_t sh_mem_ape1_base, uint32_t sh_mem_ape1_limit,
+ uint32_t sh_mem_bases);
+static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
+ unsigned int vmid);
+static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
+static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+ uint32_t queue_id, uint32_t __user *wptr,
+ uint32_t wptr_shift, uint32_t wptr_mask,
+ struct mm_struct *mm);
+static int kgd_hqd_dump(struct kgd_dev *kgd,
+ uint32_t pipe_id, uint32_t queue_id,
+ uint32_t (**dump)[2], uint32_t *n_regs);
+static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
+ uint32_t __user *wptr, struct mm_struct *mm);
+static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
+ uint32_t engine_id, uint32_t queue_id,
+ uint32_t (**dump)[2], uint32_t *n_regs);
+static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
+ uint32_t pipe_id, uint32_t queue_id);
+static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
+static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+ enum kfd_preempt_type reset_type,
+ unsigned int utimeout, uint32_t pipe_id,
+ uint32_t queue_id);
+static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+ unsigned int utimeout);
+#if 0
+static uint32_t get_watch_base_addr(struct amdgpu_device *adev);
+#endif
+static int kgd_address_watch_disable(struct kgd_dev *kgd);
+static int kgd_address_watch_execute(struct kgd_dev *kgd,
+ unsigned int watch_point_id,
+ uint32_t cntl_val,
+ uint32_t addr_hi,
+ uint32_t addr_lo);
+static int kgd_wave_control_execute(struct kgd_dev *kgd,
+ uint32_t gfx_index_val,
+ uint32_t sq_cmd);
+static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
+ unsigned int watch_point_id,
+ unsigned int reg_offset);
+
+static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd,
+ uint8_t vmid);
+static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
+ uint8_t vmid);
+static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
+ uint64_t page_table_base);
+static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid);
+static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid);
+
+/* Because of REG_GET_FIELD() being used, we put this function in the
+ * asic specific file.
+ */
+static int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd,
+ struct tile_config *config)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+
+ config->gb_addr_config = adev->gfx.config.gb_addr_config;
+#if 0
+/* TODO - confirm REG_GET_FIELD x2, should be OK as is... but
+ * MC_ARB_RAMCFG register doesn't exist on Vega10 - initial amdgpu
+ * changes commented out related code, doing the same here for now but
+ * need to sync with Ken et al
+ */
+ config->num_banks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
+ MC_ARB_RAMCFG, NOOFBANK);
+ config->num_ranks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
+ MC_ARB_RAMCFG, NOOFRANKS);
+#endif
+
+ config->tile_config_ptr = adev->gfx.config.tile_mode_array;
+ config->num_tile_configs =
+ ARRAY_SIZE(adev->gfx.config.tile_mode_array);
+ config->macro_tile_config_ptr =
+ adev->gfx.config.macrotile_mode_array;
+ config->num_macro_tile_configs =
+ ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
+
+ return 0;
+}
+
+static const struct kfd2kgd_calls kfd2kgd = {
+ .program_sh_mem_settings = kgd_program_sh_mem_settings,
+ .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
+ .init_interrupts = kgd_init_interrupts,
+ .hqd_load = kgd_hqd_load,
+ .hqd_sdma_load = kgd_hqd_sdma_load,
+ .hqd_dump = kgd_hqd_dump,
+ .hqd_sdma_dump = kgd_hqd_sdma_dump,
+ .hqd_is_occupied = kgd_hqd_is_occupied,
+ .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied,
+ .hqd_destroy = kgd_hqd_destroy,
+ .hqd_sdma_destroy = kgd_hqd_sdma_destroy,
+ .address_watch_disable = kgd_address_watch_disable,
+ .address_watch_execute = kgd_address_watch_execute,
+ .wave_control_execute = kgd_wave_control_execute,
+ .address_watch_get_offset = kgd_address_watch_get_offset,
+ .get_atc_vmid_pasid_mapping_pasid =
+ get_atc_vmid_pasid_mapping_pasid,
+ .get_atc_vmid_pasid_mapping_valid =
+ get_atc_vmid_pasid_mapping_valid,
+ .invalidate_tlbs = invalidate_tlbs,
+ .invalidate_tlbs_vmid = invalidate_tlbs_vmid,
+ .set_vm_context_page_table_base = set_vm_context_page_table_base,
+ .get_tile_config = amdgpu_amdkfd_get_tile_config,
+};
+
+struct kfd2kgd_calls *amdgpu_amdkfd_gfx_10_0_get_functions()
+{
+ return (struct kfd2kgd_calls *)&kfd2kgd;
+}
+
+static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
+{
+ return (struct amdgpu_device *)kgd;
+}
+
+static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
+ uint32_t queue, uint32_t vmid)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+ mutex_lock(&adev->srbm_mutex);
+ nv_grbm_select(adev, mec, pipe, queue, vmid);
+}
+
+static void unlock_srbm(struct kgd_dev *kgd)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+ nv_grbm_select(adev, 0, 0, 0, 0);
+ mutex_unlock(&adev->srbm_mutex);
+}
+
+static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
+ uint32_t queue_id)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+ uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+ uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
+
+ lock_srbm(kgd, mec, pipe, queue_id, 0);
+}
+
+static uint32_t get_queue_mask(struct amdgpu_device *adev,
+ uint32_t pipe_id, uint32_t queue_id)
+{
+ unsigned int bit = (pipe_id * adev->gfx.mec.num_queue_per_pipe +
+ queue_id) & 31;
+
+ return ((uint32_t)1) << bit;
+}
+
+static void release_queue(struct kgd_dev *kgd)
+{
+ unlock_srbm(kgd);
+}
+
+static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
+ uint32_t sh_mem_config,
+ uint32_t sh_mem_ape1_base,
+ uint32_t sh_mem_ape1_limit,
+ uint32_t sh_mem_bases)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+ lock_srbm(kgd, 0, 0, 0, vmid);
+
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG), sh_mem_config);
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_BASES), sh_mem_bases);
+ /* APE1 no longer exists on GFX9 */
+
+ unlock_srbm(kgd);
+}
+
+static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
+ unsigned int vmid)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+ /*
+ * We have to assume that there is no outstanding mapping.
+ * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because
+ * a mapping is in progress or because a mapping finished
+ * and the SW cleared it.
+ * So the protocol is to always wait & clear.
+ */
+ uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid |
+ ATC_VMID0_PASID_MAPPING__VALID_MASK;
+
+ pr_debug("pasid 0x%x vmid %d, reg value %x\n", pasid, vmid, pasid_mapping);
+ /*
+ * need to do this twice, once for gfx and once for mmhub
+ * for ATC add 16 to VMID for mmhub, for IH different registers.
+ * ATC_VMID0..15 registers are separate from ATC_VMID16..31.
+ */
+
+ pr_debug("ATHUB, reg %x\n", SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + vmid);
+ WREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + vmid,
+ pasid_mapping);
+
+#if 0
+ /* TODO: uncomment this code when the hardware support is ready. */
+ while (!(RREG32(SOC15_REG_OFFSET(
+ ATHUB, 0,
+ mmATC_VMID_PASID_MAPPING_UPDATE_STATUS)) &
+ (1U << vmid)))
+ cpu_relax();
+
+ pr_debug("ATHUB mapping update finished\n");
+ WREG32(SOC15_REG_OFFSET(ATHUB, 0,
+ mmATC_VMID_PASID_MAPPING_UPDATE_STATUS),
+ 1U << vmid);
+#endif
+
+ /* Mapping vmid to pasid also for IH block */
+ pr_debug("update mapping for IH block and mmhub");
+ WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid,
+ pasid_mapping);
+
+ return 0;
+}
+
+/* TODO - RING0 form of field is obsolete, seems to date back to SI
+ * but still works
+ */
+
+static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t mec;
+ uint32_t pipe;
+
+ mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+ pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
+
+ lock_srbm(kgd, mec, pipe, 0, 0);
+
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmCPC_INT_CNTL),
+ CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK |
+ CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK);
+
+ unlock_srbm(kgd);
+
+ return 0;
+}
+
+static uint32_t get_sdma_base_addr(struct amdgpu_device *adev,
+ unsigned int engine_id,
+ unsigned int queue_id)
+{
+ uint32_t base[2] = {
+ SOC15_REG_OFFSET(SDMA0, 0,
+ mmSDMA0_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL,
+ /* On gfx10, mmSDMA1_xxx registers are defined NOT based
+ * on SDMA1 base address (dw 0x1860) but based on SDMA0
+ * base address (dw 0x1260). Therefore use mmSDMA0_RLC0_RB_CNTL
+ * instead of mmSDMA1_RLC0_RB_CNTL for the base address calc
+ * below
+ */
+ SOC15_REG_OFFSET(SDMA1, 0,
+ mmSDMA1_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL
+ };
+ uint32_t retval;
+
+ retval = base[engine_id] + queue_id * (mmSDMA0_RLC1_RB_CNTL -
+ mmSDMA0_RLC0_RB_CNTL);
+
+ pr_debug("sdma base address: 0x%x\n", retval);
+
+ return retval;
+}
+
+#if 0
+static uint32_t get_watch_base_addr(struct amdgpu_device *adev)
+{
+ uint32_t retval = SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_ADDR_H) -
+ mmTCP_WATCH0_ADDR_H;
+
+ pr_debug("kfd: reg watch base address: 0x%x\n", retval);
+
+ return retval;
+}
+#endif
+
+static inline struct v10_compute_mqd *get_mqd(void *mqd)
+{
+ return (struct v10_compute_mqd *)mqd;
+}
+
+static inline struct v10_sdma_mqd *get_sdma_mqd(void *mqd)
+{
+ return (struct v10_sdma_mqd *)mqd;
+}
+
+static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+ uint32_t queue_id, uint32_t __user *wptr,
+ uint32_t wptr_shift, uint32_t wptr_mask,
+ struct mm_struct *mm)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ struct v10_compute_mqd *m;
+ uint32_t *mqd_hqd;
+ uint32_t reg, hqd_base, data;
+
+ m = get_mqd(mqd);
+
+ pr_debug("Load hqd of pipe %d queue %d\n", pipe_id, queue_id);
+ acquire_queue(kgd, pipe_id, queue_id);
+
+ /* HIQ is set during driver init period with vmid set to 0*/
+ if (m->cp_hqd_vmid == 0) {
+ uint32_t value, mec, pipe;
+
+ mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+ pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
+
+ pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n",
+ mec, pipe, queue_id);
+ value = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS));
+ value = REG_SET_FIELD(value, RLC_CP_SCHEDULERS, scheduler1,
+ ((mec << 5) | (pipe << 3) | queue_id | 0x80));
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS), value);
+ }
+
+ /* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */
+ mqd_hqd = &m->cp_mqd_base_addr_lo;
+ hqd_base = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR);
+
+ for (reg = hqd_base;
+ reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++)
+ WREG32(reg, mqd_hqd[reg - hqd_base]);
+
+
+ /* Activate doorbell logic before triggering WPTR poll. */
+ data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
+ CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), data);
+
+ if (wptr) {
+ /* Don't read wptr with get_user because the user
+ * context may not be accessible (if this function
+ * runs in a work queue). Instead trigger a one-shot
+ * polling read from memory in the CP. This assumes
+ * that wptr is GPU-accessible in the queue's VMID via
+ * ATC or SVM. WPTR==RPTR before starting the poll so
+ * the CP starts fetching new commands from the right
+ * place.
+ *
+ * Guessing a 64-bit WPTR from a 32-bit RPTR is a bit
+ * tricky. Assume that the queue didn't overflow. The
+ * number of valid bits in the 32-bit RPTR depends on
+ * the queue size. The remaining bits are taken from
+ * the saved 64-bit WPTR. If the WPTR wrapped, add the
+ * queue size.
+ */
+ uint32_t queue_size =
+ 2 << REG_GET_FIELD(m->cp_hqd_pq_control,
+ CP_HQD_PQ_CONTROL, QUEUE_SIZE);
+ uint64_t guessed_wptr = m->cp_hqd_pq_rptr & (queue_size - 1);
+
+ if ((m->cp_hqd_pq_wptr_lo & (queue_size - 1)) < guessed_wptr)
+ guessed_wptr += queue_size;
+ guessed_wptr += m->cp_hqd_pq_wptr_lo & ~(queue_size - 1);
+ guessed_wptr += (uint64_t)m->cp_hqd_pq_wptr_hi << 32;
+
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO),
+ lower_32_bits(guessed_wptr));
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI),
+ upper_32_bits(guessed_wptr));
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),
+ lower_32_bits((uint64_t)wptr));
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),
+ upper_32_bits((uint64_t)wptr));
+ pr_debug("%s setting CP_PQ_WPTR_POLL_CNTL1 to %x\n", __func__, get_queue_mask(adev, pipe_id, queue_id));
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_WPTR_POLL_CNTL1),
+ get_queue_mask(adev, pipe_id, queue_id));
+ }
+
+ /* Start the EOP fetcher */
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_RPTR),
+ REG_SET_FIELD(m->cp_hqd_eop_rptr,
+ CP_HQD_EOP_RPTR, INIT_FETCHER, 1));
+
+ data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE), data);
+
+ release_queue(kgd);
+
+ return 0;
+}
+
+static int kgd_hqd_dump(struct kgd_dev *kgd,
+ uint32_t pipe_id, uint32_t queue_id,
+ uint32_t (**dump)[2], uint32_t *n_regs)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t i = 0, reg;
+#define HQD_N_REGS 56
+#define DUMP_REG(addr) do { \
+ if (WARN_ON_ONCE(i >= HQD_N_REGS)) \
+ break; \
+ (*dump)[i][0] = (addr) << 2; \
+ (*dump)[i++][1] = RREG32(addr); \
+ } while (0)
+
+ *dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
+ if (*dump == NULL)
+ return -ENOMEM;
+
+ acquire_queue(kgd, pipe_id, queue_id);
+
+ for (reg = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR);
+ reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++)
+ DUMP_REG(reg);
+
+ release_queue(kgd);
+
+ WARN_ON_ONCE(i != HQD_N_REGS);
+ *n_regs = i;
+
+ return 0;
+}
+
+static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
+ uint32_t __user *wptr, struct mm_struct *mm)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ struct v10_sdma_mqd *m;
+ uint32_t sdma_base_addr, sdmax_gfx_context_cntl;
+ unsigned long end_jiffies;
+ uint32_t data;
+ uint64_t data64;
+ uint64_t __user *wptr64 = (uint64_t __user *)wptr;
+
+ m = get_sdma_mqd(mqd);
+ sdma_base_addr = get_sdma_base_addr(adev, m->sdma_engine_id,
+ m->sdma_queue_id);
+ pr_debug("sdma load base addr %x for engine %d, queue %d\n", sdma_base_addr, m->sdma_engine_id, m->sdma_queue_id);
+ sdmax_gfx_context_cntl = m->sdma_engine_id ?
+ SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_GFX_CONTEXT_CNTL) :
+ SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_CONTEXT_CNTL);
+
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL,
+ m->sdmax_rlcx_rb_cntl & (~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK));
+
+ end_jiffies = msecs_to_jiffies(2000) + jiffies;
+ while (true) {
+ data = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
+ if (data & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
+ break;
+ if (time_after(jiffies, end_jiffies))
+ return -ETIME;
+ usleep_range(500, 1000);
+ }
+ data = RREG32(sdmax_gfx_context_cntl);
+ data = REG_SET_FIELD(data, SDMA0_GFX_CONTEXT_CNTL,
+ RESUME_CTX, 0);
+ WREG32(sdmax_gfx_context_cntl, data);
+
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL_OFFSET,
+ m->sdmax_rlcx_doorbell_offset);
+
+ data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_RLC0_DOORBELL,
+ ENABLE, 1);
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, data);
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, m->sdmax_rlcx_rb_rptr);
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_HI,
+ m->sdmax_rlcx_rb_rptr_hi);
+
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 1);
+ if (read_user_wptr(mm, wptr64, data64)) {
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR,
+ lower_32_bits(data64));
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR_HI,
+ upper_32_bits(data64));
+ } else {
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR,
+ m->sdmax_rlcx_rb_rptr);
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR_HI,
+ m->sdmax_rlcx_rb_rptr_hi);
+ }
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 0);
+
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, m->sdmax_rlcx_rb_base);
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE_HI,
+ m->sdmax_rlcx_rb_base_hi);
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_LO,
+ m->sdmax_rlcx_rb_rptr_addr_lo);
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI,
+ m->sdmax_rlcx_rb_rptr_addr_hi);
+
+ data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_RLC0_RB_CNTL,
+ RB_ENABLE, 1);
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, data);
+
+ return 0;
+}
+
+static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
+ uint32_t engine_id, uint32_t queue_id,
+ uint32_t (**dump)[2], uint32_t *n_regs)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t sdma_base_addr = get_sdma_base_addr(adev, engine_id, queue_id);
+ uint32_t i = 0, reg;
+#undef HQD_N_REGS
+#define HQD_N_REGS (19+6+7+10)
+
+ pr_debug("sdma dump engine id %d queue_id %d\n", engine_id, queue_id);
+ pr_debug("sdma base addr %x\n", sdma_base_addr);
+
+ *dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
+ if (*dump == NULL)
+ return -ENOMEM;
+
+ for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++)
+ DUMP_REG(sdma_base_addr + reg);
+ for (reg = mmSDMA0_RLC0_STATUS; reg <= mmSDMA0_RLC0_CSA_ADDR_HI; reg++)
+ DUMP_REG(sdma_base_addr + reg);
+ for (reg = mmSDMA0_RLC0_IB_SUB_REMAIN;
+ reg <= mmSDMA0_RLC0_MINOR_PTR_UPDATE; reg++)
+ DUMP_REG(sdma_base_addr + reg);
+ for (reg = mmSDMA0_RLC0_MIDCMD_DATA0;
+ reg <= mmSDMA0_RLC0_MIDCMD_CNTL; reg++)
+ DUMP_REG(sdma_base_addr + reg);
+
+ WARN_ON_ONCE(i != HQD_N_REGS);
+ *n_regs = i;
+
+ return 0;
+}
+
+static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
+ uint32_t pipe_id, uint32_t queue_id)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t act;
+ bool retval = false;
+ uint32_t low, high;
+
+ acquire_queue(kgd, pipe_id, queue_id);
+ act = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE));
+ if (act) {
+ low = lower_32_bits(queue_address >> 8);
+ high = upper_32_bits(queue_address >> 8);
+
+ if (low == RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE)) &&
+ high == RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE_HI)))
+ retval = true;
+ }
+ release_queue(kgd);
+ return retval;
+}
+
+static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ struct v10_sdma_mqd *m;
+ uint32_t sdma_base_addr;
+ uint32_t sdma_rlc_rb_cntl;
+
+ m = get_sdma_mqd(mqd);
+ sdma_base_addr = get_sdma_base_addr(adev, m->sdma_engine_id,
+ m->sdma_queue_id);
+
+ sdma_rlc_rb_cntl = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL);
+
+ if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK)
+ return true;
+
+ return false;
+}
+
+static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+ enum kfd_preempt_type reset_type,
+ unsigned int utimeout, uint32_t pipe_id,
+ uint32_t queue_id)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ enum hqd_dequeue_request_type type;
+ unsigned long end_jiffies;
+ uint32_t temp;
+ struct v10_compute_mqd *m = get_mqd(mqd);
+
+#if 0
+ unsigned long flags;
+ int retry;
+#endif
+
+ acquire_queue(kgd, pipe_id, queue_id);
+
+ if (m->cp_hqd_vmid == 0)
+ WREG32_FIELD15(GC, 0, RLC_CP_SCHEDULERS, scheduler1, 0);
+
+ switch (reset_type) {
+ case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN:
+ type = DRAIN_PIPE;
+ break;
+ case KFD_PREEMPT_TYPE_WAVEFRONT_RESET:
+ type = RESET_WAVES;
+ break;
+ default:
+ type = DRAIN_PIPE;
+ break;
+ }
+
+#if 0 /* Is this still needed? */
+ /* Workaround: If IQ timer is active and the wait time is close to or
+ * equal to 0, dequeueing is not safe. Wait until either the wait time
+ * is larger or timer is cleared. Also, ensure that IQ_REQ_PEND is
+ * cleared before continuing. Also, ensure wait times are set to at
+ * least 0x3.
+ */
+ local_irq_save(flags);
+ preempt_disable();
+ retry = 5000; /* wait for 500 usecs at maximum */
+ while (true) {
+ temp = RREG32(mmCP_HQD_IQ_TIMER);
+ if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, PROCESSING_IQ)) {
+ pr_debug("HW is processing IQ\n");
+ goto loop;
+ }
+ if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, ACTIVE)) {
+ if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, RETRY_TYPE)
+ == 3) /* SEM-rearm is safe */
+ break;
+ /* Wait time 3 is safe for CP, but our MMIO read/write
+ * time is close to 1 microsecond, so check for 10 to
+ * leave more buffer room
+ */
+ if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, WAIT_TIME)
+ >= 10)
+ break;
+ pr_debug("IQ timer is active\n");
+ } else
+ break;
+loop:
+ if (!retry) {
+ pr_err("CP HQD IQ timer status time out\n");
+ break;
+ }
+ ndelay(100);
+ --retry;
+ }
+ retry = 1000;
+ while (true) {
+ temp = RREG32(mmCP_HQD_DEQUEUE_REQUEST);
+ if (!(temp & CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK))
+ break;
+ pr_debug("Dequeue request is pending\n");
+
+ if (!retry) {
+ pr_err("CP HQD dequeue request time out\n");
+ break;
+ }
+ ndelay(100);
+ --retry;
+ }
+ local_irq_restore(flags);
+ preempt_enable();
+#endif
+
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), type);
+
+ end_jiffies = (utimeout * HZ / 1000) + jiffies;
+ while (true) {
+ temp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE));
+ if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK))
+ break;
+ if (time_after(jiffies, end_jiffies)) {
+ pr_err("cp queue preemption time out.\n");
+ release_queue(kgd);
+ return -ETIME;
+ }
+ usleep_range(500, 1000);
+ }
+
+ release_queue(kgd);
+ return 0;
+}
+
+static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+ unsigned int utimeout)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ struct v10_sdma_mqd *m;
+ uint32_t sdma_base_addr;
+ uint32_t temp;
+ unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies;
+
+ m = get_sdma_mqd(mqd);
+ sdma_base_addr = get_sdma_base_addr(adev, m->sdma_engine_id,
+ m->sdma_queue_id);
+
+ temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL);
+ temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK;
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, temp);
+
+ while (true) {
+ temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
+ if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
+ break;
+ if (time_after(jiffies, end_jiffies))
+ return -ETIME;
+ usleep_range(500, 1000);
+ }
+
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 0);
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL,
+ RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL) |
+ SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK);
+
+ m->sdmax_rlcx_rb_rptr = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR);
+ m->sdmax_rlcx_rb_rptr_hi =
+ RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_HI);
+
+ return 0;
+}
+
+static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd,
+ uint8_t vmid)
+{
+ uint32_t reg;
+ struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+ reg = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
+ + vmid);
+ return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK;
+}
+
+static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
+ uint8_t vmid)
+{
+ uint32_t reg;
+ struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+ reg = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
+ + vmid);
+ return reg & ATC_VMID0_PASID_MAPPING__PASID_MASK;
+}
+
+static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+ uint32_t req = (1 << vmid) |
+ (0 << GCVM_INVALIDATE_ENG0_REQ__FLUSH_TYPE__SHIFT) |/* legacy */
+ GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PTES_MASK |
+ GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE0_MASK |
+ GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE1_MASK |
+ GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE2_MASK |
+ GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L1_PTES_MASK;
+
+ mutex_lock(&adev->srbm_mutex);
+
+ /* Use light weight invalidation.
+ *
+ * TODO 1: agree on the right set of invalidation registers for
+ * KFD use. Use the last one for now. Invalidate only GCHUB as
+ * SDMA is now moved to GCHUB
+ *
+ * TODO 2: support range-based invalidation, requires kfg2kgd
+ * interface change
+ */
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32),
+ 0xffffffff);
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32),
+ 0x0000001f);
+
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_REQ), req);
+
+ while (!(RREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ACK)) &
+ (1 << vmid)))
+ cpu_relax();
+
+ mutex_unlock(&adev->srbm_mutex);
+}
+
+static int invalidate_tlbs_with_kiq(struct amdgpu_device *adev, uint16_t pasid)
+{
+ signed long r;
+ uint32_t seq;
+ struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
+
+ spin_lock(&adev->gfx.kiq.ring_lock);
+ amdgpu_ring_alloc(ring, 12); /* fence + invalidate_tlbs package*/
+ amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
+ amdgpu_ring_write(ring,
+ PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
+ PACKET3_INVALIDATE_TLBS_PASID(pasid));
+ amdgpu_fence_emit_polling(ring, &seq);
+ amdgpu_ring_commit(ring);
+ spin_unlock(&adev->gfx.kiq.ring_lock);
+
+ r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout);
+ if (r < 1) {
+ DRM_ERROR("wait for kiq fence error: %ld.\n", r);
+ return -ETIME;
+ }
+
+ return 0;
+}
+
+static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+ int vmid;
+ struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
+
+ if (amdgpu_emu_mode == 0 && ring->sched.ready)
+ return invalidate_tlbs_with_kiq(adev, pasid);
+
+ for (vmid = 0; vmid < 16; vmid++) {
+ if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid))
+ continue;
+ if (get_atc_vmid_pasid_mapping_valid(kgd, vmid)) {
+ if (get_atc_vmid_pasid_mapping_pasid(kgd, vmid)
+ == pasid) {
+ write_vmid_invalidate_request(kgd, vmid);
+ break;
+ }
+ }
+ }
+
+ return 0;
+}
+
+static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+ if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) {
+ pr_err("non kfd vmid %d\n", vmid);
+ return 0;
+ }
+
+ write_vmid_invalidate_request(kgd, vmid);
+ return 0;
+}
+
+static int kgd_address_watch_disable(struct kgd_dev *kgd)
+{
+ return 0;
+}
+
+static int kgd_address_watch_execute(struct kgd_dev *kgd,
+ unsigned int watch_point_id,
+ uint32_t cntl_val,
+ uint32_t addr_hi,
+ uint32_t addr_lo)
+{
+ return 0;
+}
+
+static int kgd_wave_control_execute(struct kgd_dev *kgd,
+ uint32_t gfx_index_val,
+ uint32_t sq_cmd)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t data = 0;
+
+ mutex_lock(&adev->grbm_idx_mutex);
+
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), gfx_index_val);
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_CMD), sq_cmd);
+
+ data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
+ INSTANCE_BROADCAST_WRITES, 1);
+ data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
+ SA_BROADCAST_WRITES, 1);
+ data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
+ SE_BROADCAST_WRITES, 1);
+
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), data);
+ mutex_unlock(&adev->grbm_idx_mutex);
+
+ return 0;
+}
+
+static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
+ unsigned int watch_point_id,
+ unsigned int reg_offset)
+{
+ return 0;
+}
+
+static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
+ uint64_t page_table_base)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint64_t base = page_table_base | AMDGPU_PTE_VALID;
+
+ if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) {
+ pr_err("trying to set page table base for wrong VMID %u\n",
+ vmid);
+ return;
+ }
+
+ /* TODO: take advantage of per-process address space size. For
+ * now, all processes share the same address space size, like
+ * on GFX8 and older.
+ */
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32) + (vmid*2), 0);
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32) + (vmid*2), 0);
+
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32) + (vmid*2),
+ lower_32_bits(adev->vm_manager.max_pfn - 1));
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32) + (vmid*2),
+ upper_32_bits(adev->vm_manager.max_pfn - 1));
+
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32) + (vmid*2), lower_32_bits(base));
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32) + (vmid*2), upper_32_bits(base));
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
index fa09e11a600c..5f459bf5f622 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
@@ -23,7 +23,7 @@
#include <linux/fdtable.h>
#include <linux/uaccess.h>
#include <linux/mmu_context.h>
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_amdkfd.h"
#include "cikd.h"
@@ -310,7 +310,7 @@ static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m)
retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET +
m->sdma_queue_id * KFD_CIK_SDMA_QUEUE_OFFSET;
- pr_debug("kfd: sdma base address: 0x%x\n", retval);
+ pr_debug("sdma base address: 0x%x\n", retval);
return retval;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
index fec3a6aa1de6..6d2f61449606 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
@@ -24,7 +24,7 @@
#include <linux/fdtable.h>
#include <linux/uaccess.h>
#include <linux/mmu_context.h>
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_amdkfd.h"
#include "gfx_v8_0.h"
@@ -266,7 +266,7 @@ static inline uint32_t get_sdma_base_addr(struct vi_sdma_mqd *m)
retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET +
m->sdma_queue_id * KFD_VI_SDMA_QUEUE_OFFSET;
- pr_debug("kfd: sdma base address: 0x%x\n", retval);
+ pr_debug("sdma base address: 0x%x\n", retval);
return retval;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
index ef3d93b995b2..85395f2d83a6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
@@ -26,7 +26,7 @@
#include <linux/fdtable.h>
#include <linux/uaccess.h>
#include <linux/mmu_context.h>
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_amdkfd.h"
#include "soc15_hw_ip.h"
@@ -225,8 +225,8 @@ static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
lock_srbm(kgd, 0, 0, 0, vmid);
- WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG), sh_mem_config);
- WREG32(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_BASES), sh_mem_bases);
+ WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG), sh_mem_config);
+ WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_BASES), sh_mem_bases);
/* APE1 no longer exists on GFX9 */
unlock_srbm(kgd);
@@ -369,7 +369,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
value = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS));
value = REG_SET_FIELD(value, RLC_CP_SCHEDULERS, scheduler1,
((mec << 5) | (pipe << 3) | queue_id | 0x80));
- WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS), value);
+ WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CP_SCHEDULERS), value);
}
/* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */
@@ -378,13 +378,13 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
for (reg = hqd_base;
reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++)
- WREG32(reg, mqd_hqd[reg - hqd_base]);
+ WREG32_RLC(reg, mqd_hqd[reg - hqd_base]);
/* Activate doorbell logic before triggering WPTR poll. */
data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
- WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), data);
+ WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), data);
if (wptr) {
/* Don't read wptr with get_user because the user
@@ -413,25 +413,25 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
guessed_wptr += m->cp_hqd_pq_wptr_lo & ~(queue_size - 1);
guessed_wptr += (uint64_t)m->cp_hqd_pq_wptr_hi << 32;
- WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO),
+ WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO),
lower_32_bits(guessed_wptr));
- WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI),
+ WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI),
upper_32_bits(guessed_wptr));
- WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),
+ WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),
lower_32_bits((uintptr_t)wptr));
- WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),
+ WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),
upper_32_bits((uintptr_t)wptr));
WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_WPTR_POLL_CNTL1),
get_queue_mask(adev, pipe_id, queue_id));
}
/* Start the EOP fetcher */
- WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_RPTR),
+ WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_RPTR),
REG_SET_FIELD(m->cp_hqd_eop_rptr,
CP_HQD_EOP_RPTR, INIT_FETCHER, 1));
data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
- WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE), data);
+ WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE), data);
release_queue(kgd);
@@ -633,7 +633,7 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
acquire_queue(kgd, pipe_id, queue_id);
if (m->cp_hqd_vmid == 0)
- WREG32_FIELD15(GC, 0, RLC_CP_SCHEDULERS, scheduler1, 0);
+ WREG32_FIELD15_RLC(GC, 0, RLC_CP_SCHEDULERS, scheduler1, 0);
switch (reset_type) {
case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN:
@@ -647,7 +647,7 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
break;
}
- WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), type);
+ WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), type);
end_jiffies = (utimeout * HZ / 1000) + jiffies;
while (true) {
@@ -726,29 +726,8 @@ static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
return reg & ATC_VMID0_PASID_MAPPING__PASID_MASK;
}
-static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid)
-{
- struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
-
- /* Use legacy mode tlb invalidation.
- *
- * Currently on Raven the code below is broken for anything but
- * legacy mode due to a MMHUB power gating problem. A workaround
- * is for MMHUB to wait until the condition PER_VMID_INVALIDATE_REQ
- * == PER_VMID_INVALIDATE_ACK instead of simply waiting for the ack
- * bit.
- *
- * TODO 1: agree on the right set of invalidation registers for
- * KFD use. Use the last one for now. Invalidate both GC and
- * MMHUB.
- *
- * TODO 2: support range-based invalidation, requires kfg2kgd
- * interface change
- */
- amdgpu_gmc_flush_gpu_tlb(adev, vmid, 0);
-}
-
-static int invalidate_tlbs_with_kiq(struct amdgpu_device *adev, uint16_t pasid)
+static int invalidate_tlbs_with_kiq(struct amdgpu_device *adev, uint16_t pasid,
+ uint32_t flush_type)
{
signed long r;
uint32_t seq;
@@ -761,7 +740,7 @@ static int invalidate_tlbs_with_kiq(struct amdgpu_device *adev, uint16_t pasid)
PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
PACKET3_INVALIDATE_TLBS_ALL_HUB(1) |
PACKET3_INVALIDATE_TLBS_PASID(pasid) |
- PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(0)); /* legacy */
+ PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
amdgpu_fence_emit_polling(ring, &seq);
amdgpu_ring_commit(ring);
spin_unlock(&adev->gfx.kiq.ring_lock);
@@ -780,12 +759,16 @@ static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid)
struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
int vmid;
struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
+ uint32_t flush_type = 0;
if (adev->in_gpu_reset)
return -EIO;
+ if (adev->gmc.xgmi.num_physical_nodes &&
+ adev->asic_type == CHIP_VEGA20)
+ flush_type = 2;
if (ring->sched.ready)
- return invalidate_tlbs_with_kiq(adev, pasid);
+ return invalidate_tlbs_with_kiq(adev, pasid, flush_type);
for (vmid = 0; vmid < 16; vmid++) {
if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid))
@@ -793,7 +776,8 @@ static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid)
if (get_atc_vmid_pasid_mapping_valid(kgd, vmid)) {
if (get_atc_vmid_pasid_mapping_pasid(kgd, vmid)
== pasid) {
- write_vmid_invalidate_request(kgd, vmid);
+ amdgpu_gmc_flush_gpu_tlb(adev, vmid,
+ flush_type);
break;
}
}
@@ -811,7 +795,22 @@ static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid)
return 0;
}
- write_vmid_invalidate_request(kgd, vmid);
+ /* Use legacy mode tlb invalidation.
+ *
+ * Currently on Raven the code below is broken for anything but
+ * legacy mode due to a MMHUB power gating problem. A workaround
+ * is for MMHUB to wait until the condition PER_VMID_INVALIDATE_REQ
+ * == PER_VMID_INVALIDATE_ACK instead of simply waiting for the ack
+ * bit.
+ *
+ * TODO 1: agree on the right set of invalidation registers for
+ * KFD use. Use the last one for now. Invalidate both GC and
+ * MMHUB.
+ *
+ * TODO 2: support range-based invalidation, requires kfg2kgd
+ * interface change
+ */
+ amdgpu_gmc_flush_gpu_tlb(adev, vmid, 0);
return 0;
}
@@ -838,7 +837,7 @@ static int kgd_wave_control_execute(struct kgd_dev *kgd,
mutex_lock(&adev->grbm_idx_mutex);
- WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), gfx_index_val);
+ WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, gfx_index_val);
WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_CMD), sq_cmd);
data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
@@ -848,7 +847,7 @@ static int kgd_wave_control_execute(struct kgd_dev *kgd,
data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
SE_BROADCAST_WRITES, 1);
- WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), data);
+ WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
mutex_unlock(&adev->grbm_idx_mutex);
return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index a6e5184d436c..6a5c96e519b1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -22,14 +22,16 @@
#define pr_fmt(fmt) "kfd2kgd: " fmt
+#include <linux/dma-buf.h>
#include <linux/list.h>
#include <linux/pagemap.h>
#include <linux/sched/mm.h>
-#include <linux/dma-buf.h>
-#include <drm/drmP.h>
+#include <linux/sched/task.h>
+
#include "amdgpu_object.h"
#include "amdgpu_vm.h"
#include "amdgpu_amdkfd.h"
+#include "amdgpu_dma_buf.h"
/* Special VM and GART address alignment needed for VI pre-Fiji due to
* a HW bug.
@@ -456,6 +458,17 @@ static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,
mutex_unlock(&process_info->lock);
}
+static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem,
+ struct amdkfd_process_info *process_info)
+{
+ struct ttm_validate_buffer *bo_list_entry;
+
+ bo_list_entry = &mem->validate_list;
+ mutex_lock(&process_info->lock);
+ list_del(&bo_list_entry->head);
+ mutex_unlock(&process_info->lock);
+}
+
/* Initializes user pages. It registers the MMU notifier and validates
* the userptr BO in the GTT domain.
*
@@ -491,28 +504,12 @@ static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm,
goto out;
}
- /* If no restore worker is running concurrently, user_pages
- * should not be allocated
- */
- WARN(mem->user_pages, "Leaking user_pages array");
-
- mem->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages,
- sizeof(struct page *),
- GFP_KERNEL | __GFP_ZERO);
- if (!mem->user_pages) {
- pr_err("%s: Failed to allocate pages array\n", __func__);
- ret = -ENOMEM;
- goto unregister_out;
- }
-
- ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm, mem->user_pages);
+ ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages);
if (ret) {
pr_err("%s: Failed to get user pages: %d\n", __func__, ret);
- goto free_out;
+ goto unregister_out;
}
- amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, mem->user_pages);
-
ret = amdgpu_bo_reserve(bo, true);
if (ret) {
pr_err("%s: Failed to reserve BO\n", __func__);
@@ -525,11 +522,7 @@ static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm,
amdgpu_bo_unreserve(bo);
release_out:
- if (ret)
- release_pages(mem->user_pages, bo->tbo.ttm->num_pages);
-free_out:
- kvfree(mem->user_pages);
- mem->user_pages = NULL;
+ amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
unregister_out:
if (ret)
amdgpu_mn_unregister(bo);
@@ -588,13 +581,12 @@ static int reserve_bo_and_vm(struct kgd_mem *mem,
ctx->kfd_bo.priority = 0;
ctx->kfd_bo.tv.bo = &bo->tbo;
ctx->kfd_bo.tv.num_shared = 1;
- ctx->kfd_bo.user_pages = NULL;
list_add(&ctx->kfd_bo.tv.head, &ctx->list);
amdgpu_vm_get_pd_bo(vm, &ctx->list, &ctx->vm_pd[0]);
ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list,
- false, &ctx->duplicates);
+ false, &ctx->duplicates, true);
if (!ret)
ctx->reserved = true;
else {
@@ -652,7 +644,6 @@ static int reserve_bo_and_cond_vms(struct kgd_mem *mem,
ctx->kfd_bo.priority = 0;
ctx->kfd_bo.tv.bo = &bo->tbo;
ctx->kfd_bo.tv.num_shared = 1;
- ctx->kfd_bo.user_pages = NULL;
list_add(&ctx->kfd_bo.tv.head, &ctx->list);
i = 0;
@@ -668,7 +659,7 @@ static int reserve_bo_and_cond_vms(struct kgd_mem *mem,
}
ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list,
- false, &ctx->duplicates);
+ false, &ctx->duplicates, true);
if (!ret)
ctx->reserved = true;
else
@@ -822,7 +813,7 @@ static int process_sync_pds_resv(struct amdkfd_process_info *process_info,
ret = amdgpu_sync_resv(NULL,
sync, pd->tbo.resv,
- AMDGPU_FENCE_OWNER_UNDEFINED, false);
+ AMDGPU_FENCE_OWNER_KFD, false);
if (ret)
return ret;
}
@@ -896,6 +887,9 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
AMDGPU_FENCE_OWNER_KFD, false);
if (ret)
goto wait_pd_fail;
+ ret = reservation_object_reserve_shared(vm->root.base.bo->tbo.resv, 1);
+ if (ret)
+ goto reserve_shared_fail;
amdgpu_bo_fence(vm->root.base.bo,
&vm->process_info->eviction_fence->base, true);
amdgpu_bo_unreserve(vm->root.base.bo);
@@ -909,6 +903,7 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
return 0;
+reserve_shared_fail:
wait_pd_fail:
validate_pd_fail:
amdgpu_bo_unreserve(vm->root.base.bo);
@@ -1109,7 +1104,8 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
if (!offset || !*offset)
return -EINVAL;
user_addr = *offset;
- } else if (flags & ALLOC_MEM_FLAGS_DOORBELL) {
+ } else if (flags & (ALLOC_MEM_FLAGS_DOORBELL |
+ ALLOC_MEM_FLAGS_MMIO_REMAP)) {
domain = AMDGPU_GEM_DOMAIN_GTT;
alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
bo_type = ttm_bo_type_sg;
@@ -1144,7 +1140,8 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
adev->asic_type != CHIP_FIJI &&
adev->asic_type != CHIP_POLARIS10 &&
adev->asic_type != CHIP_POLARIS11 &&
- adev->asic_type != CHIP_POLARIS12) ?
+ adev->asic_type != CHIP_POLARIS12 &&
+ adev->asic_type != CHIP_VEGAM) ?
VI_BO_SIZE_ALIGN : 1;
mapping_flags = AMDGPU_VM_PAGE_READABLE;
@@ -1199,12 +1196,8 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
if (user_addr) {
ret = init_user_pages(*mem, current->mm, user_addr);
- if (ret) {
- mutex_lock(&avm->process_info->lock);
- list_del(&(*mem)->validate_list.head);
- mutex_unlock(&avm->process_info->lock);
+ if (ret)
goto allocate_init_user_pages_failed;
- }
}
if (offset)
@@ -1213,6 +1206,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
return 0;
allocate_init_user_pages_failed:
+ remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
amdgpu_bo_unref(&bo);
/* Don't unreserve system mem limit twice */
goto err_reserve_limit;
@@ -1262,15 +1256,6 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
list_del(&bo_list_entry->head);
mutex_unlock(&process_info->lock);
- /* Free user pages if necessary */
- if (mem->user_pages) {
- pr_debug("%s: Freeing user_pages array\n", __func__);
- if (mem->user_pages[0])
- release_pages(mem->user_pages,
- mem->bo->tbo.ttm->num_pages);
- kvfree(mem->user_pages);
- }
-
ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx);
if (unlikely(ret))
return ret;
@@ -1294,8 +1279,8 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
/* Free the sync object */
amdgpu_sync_free(&mem->sync);
- /* If the SG is not NULL, it's one we created for a doorbell
- * BO. We need to free it.
+ /* If the SG is not NULL, it's one we created for a doorbell or mmio
+ * remap BO. We need to free it.
*/
if (mem->bo->tbo.sg) {
sg_free_table(mem->bo->tbo.sg);
@@ -1409,7 +1394,7 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
ret = map_bo_to_gpuvm(adev, entry, ctx.sync,
is_invalid_userptr);
if (ret) {
- pr_err("Failed to map radeon bo to gpuvm\n");
+ pr_err("Failed to map bo to gpuvm\n");
goto map_bo_to_gpuvm_failed;
}
@@ -1744,36 +1729,20 @@ static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
bo = mem->bo;
- if (!mem->user_pages) {
- mem->user_pages =
- kvmalloc_array(bo->tbo.ttm->num_pages,
- sizeof(struct page *),
- GFP_KERNEL | __GFP_ZERO);
- if (!mem->user_pages) {
- pr_err("%s: Failed to allocate pages array\n",
- __func__);
- return -ENOMEM;
- }
- } else if (mem->user_pages[0]) {
- release_pages(mem->user_pages, bo->tbo.ttm->num_pages);
- }
-
/* Get updated user pages */
- ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm,
- mem->user_pages);
+ ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages);
if (ret) {
- mem->user_pages[0] = NULL;
- pr_info("%s: Failed to get user pages: %d\n",
+ pr_debug("%s: Failed to get user pages: %d\n",
__func__, ret);
- /* Pretend it succeeded. It will fail later
- * with a VM fault if the GPU tries to access
- * it. Better than hanging indefinitely with
- * stalled user mode queues.
- */
+
+ /* Return error -EBUSY or -ENOMEM, retry restore */
+ return ret;
}
+ amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
+
/* Mark the BO as valid unless it was invalidated
- * again concurrently
+ * again concurrently.
*/
if (atomic_cmpxchg(&mem->invalid, invalid, 0) != invalid)
return -EAGAIN;
@@ -1806,7 +1775,8 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
GFP_KERNEL);
if (!pd_bo_list_entries) {
pr_err("%s: Failed to allocate PD BO list entries\n", __func__);
- return -ENOMEM;
+ ret = -ENOMEM;
+ goto out_no_mem;
}
INIT_LIST_HEAD(&resv_list);
@@ -1827,10 +1797,11 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
}
/* Reserve all BOs and page tables for validation */
- ret = ttm_eu_reserve_buffers(&ticket, &resv_list, false, &duplicates);
+ ret = ttm_eu_reserve_buffers(&ticket, &resv_list, false, &duplicates,
+ true);
WARN(!list_empty(&duplicates), "Duplicates should be empty");
if (ret)
- goto out;
+ goto out_free;
amdgpu_sync_create(&sync);
@@ -1846,10 +1817,8 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
bo = mem->bo;
- /* Copy pages array and validate the BO if we got user pages */
- if (mem->user_pages[0]) {
- amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm,
- mem->user_pages);
+ /* Validate the BO if we got user pages */
+ if (bo->tbo.ttm->pages[0]) {
amdgpu_bo_placement_from_domain(bo, mem->domain);
ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
if (ret) {
@@ -1858,13 +1827,6 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
}
}
- /* Validate succeeded, now the BO owns the pages, free
- * our copy of the pointer array. Put this BO back on
- * the userptr_valid_list. If we need to revalidate
- * it, we need to start from scratch.
- */
- kvfree(mem->user_pages);
- mem->user_pages = NULL;
list_move_tail(&mem->validate_list.head,
&process_info->userptr_valid_list);
@@ -1897,8 +1859,9 @@ unreserve_out:
ttm_eu_backoff_reservation(&ticket, &resv_list);
amdgpu_sync_wait(&sync, false);
amdgpu_sync_free(&sync);
-out:
+out_free:
kfree(pd_bo_list_entries);
+out_no_mem:
return ret;
}
@@ -1963,6 +1926,7 @@ static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work)
* hanging. No point trying again.
*/
}
+
unlock_out:
mutex_unlock(&process_info->lock);
mmput(mm);
@@ -2032,7 +1996,7 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef)
}
ret = ttm_eu_reserve_buffers(&ctx.ticket, &ctx.list,
- false, &duplicate_save);
+ false, &duplicate_save, true);
if (ret) {
pr_debug("Memory eviction: TTM Reserve Failed. Try again\n");
goto ttm_reserve_fail;
@@ -2130,3 +2094,92 @@ ttm_reserve_fail:
kfree(pd_bo_list);
return ret;
}
+
+int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem)
+{
+ struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
+ struct amdgpu_bo *gws_bo = (struct amdgpu_bo *)gws;
+ int ret;
+
+ if (!info || !gws)
+ return -EINVAL;
+
+ *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
+ if (!*mem)
+ return -ENOMEM;
+
+ mutex_init(&(*mem)->lock);
+ (*mem)->bo = amdgpu_bo_ref(gws_bo);
+ (*mem)->domain = AMDGPU_GEM_DOMAIN_GWS;
+ (*mem)->process_info = process_info;
+ add_kgd_mem_to_kfd_bo_list(*mem, process_info, false);
+ amdgpu_sync_create(&(*mem)->sync);
+
+
+ /* Validate gws bo the first time it is added to process */
+ mutex_lock(&(*mem)->process_info->lock);
+ ret = amdgpu_bo_reserve(gws_bo, false);
+ if (unlikely(ret)) {
+ pr_err("Reserve gws bo failed %d\n", ret);
+ goto bo_reservation_failure;
+ }
+
+ ret = amdgpu_amdkfd_bo_validate(gws_bo, AMDGPU_GEM_DOMAIN_GWS, true);
+ if (ret) {
+ pr_err("GWS BO validate failed %d\n", ret);
+ goto bo_validation_failure;
+ }
+ /* GWS resource is shared b/t amdgpu and amdkfd
+ * Add process eviction fence to bo so they can
+ * evict each other.
+ */
+ ret = reservation_object_reserve_shared(gws_bo->tbo.resv, 1);
+ if (ret)
+ goto reserve_shared_fail;
+ amdgpu_bo_fence(gws_bo, &process_info->eviction_fence->base, true);
+ amdgpu_bo_unreserve(gws_bo);
+ mutex_unlock(&(*mem)->process_info->lock);
+
+ return ret;
+
+reserve_shared_fail:
+bo_validation_failure:
+ amdgpu_bo_unreserve(gws_bo);
+bo_reservation_failure:
+ mutex_unlock(&(*mem)->process_info->lock);
+ amdgpu_sync_free(&(*mem)->sync);
+ remove_kgd_mem_from_kfd_bo_list(*mem, process_info);
+ amdgpu_bo_unref(&gws_bo);
+ mutex_destroy(&(*mem)->lock);
+ kfree(*mem);
+ *mem = NULL;
+ return ret;
+}
+
+int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem)
+{
+ int ret;
+ struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
+ struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
+ struct amdgpu_bo *gws_bo = kgd_mem->bo;
+
+ /* Remove BO from process's validate list so restore worker won't touch
+ * it anymore
+ */
+ remove_kgd_mem_from_kfd_bo_list(kgd_mem, process_info);
+
+ ret = amdgpu_bo_reserve(gws_bo, false);
+ if (unlikely(ret)) {
+ pr_err("Reserve gws bo failed %d\n", ret);
+ //TODO add BO back to validate_list?
+ return ret;
+ }
+ amdgpu_amdkfd_remove_eviction_fence(gws_bo,
+ process_info->eviction_fence);
+ amdgpu_bo_unreserve(gws_bo);
+ amdgpu_sync_free(&kgd_mem->sync);
+ amdgpu_bo_unref(&gws_bo);
+ mutex_destroy(&kgd_mem->lock);
+ kfree(mem);
+ return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
index e02781b37e73..1c9d40f97a9b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
@@ -23,7 +23,7 @@
* Authors: Dave Airlie
* Alex Deucher
*/
-#include <drm/drmP.h>
+
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
#include "amdgpu_atombios.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index f96d75c6e099..daf687428cdb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -20,7 +20,7 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
-#include <drm/drmP.h>
+
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
#include "atomfirmware.h"
@@ -118,6 +118,7 @@ union umc_info {
union vram_info {
struct atom_vram_info_header_v2_3 v23;
+ struct atom_vram_info_header_v2_4 v24;
};
/*
* Return vram width from integrated system info table, if available,
@@ -126,22 +127,50 @@ union vram_info {
int amdgpu_atomfirmware_get_vram_width(struct amdgpu_device *adev)
{
struct amdgpu_mode_info *mode_info = &adev->mode_info;
- int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
- integratedsysteminfo);
+ int index;
u16 data_offset, size;
union igp_info *igp_info;
+ union vram_info *vram_info;
+ u32 mem_channel_number;
+ u32 mem_channel_width;
u8 frev, crev;
+ if (adev->flags & AMD_IS_APU)
+ index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
+ integratedsysteminfo);
+ else
+ index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
+ vram_info);
+
/* get any igp specific overrides */
if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, &size,
&frev, &crev, &data_offset)) {
- igp_info = (union igp_info *)
- (mode_info->atom_context->bios + data_offset);
- switch (crev) {
- case 11:
- return igp_info->v11.umachannelnumber * 64;
- default:
- return 0;
+ if (adev->flags & AMD_IS_APU) {
+ igp_info = (union igp_info *)
+ (mode_info->atom_context->bios + data_offset);
+ switch (crev) {
+ case 11:
+ mem_channel_number = igp_info->v11.umachannelnumber;
+ /* channel width is 64 */
+ return mem_channel_number * 64;
+ default:
+ return 0;
+ }
+ } else {
+ vram_info = (union vram_info *)
+ (mode_info->atom_context->bios + data_offset);
+ switch (crev) {
+ case 3:
+ mem_channel_number = vram_info->v23.vram_module[0].channel_num;
+ mem_channel_width = vram_info->v23.vram_module[0].channel_width;
+ return mem_channel_number * (1 << mem_channel_width);
+ case 4:
+ mem_channel_number = vram_info->v24.vram_module[0].channel_num;
+ mem_channel_width = vram_info->v24.vram_module[0].channel_width;
+ return mem_channel_number * (1 << mem_channel_width);
+ default:
+ return 0;
+ }
}
}
@@ -179,6 +208,9 @@ static int convert_atom_mem_type_to_vram_type (struct amdgpu_device *adev,
case ATOM_DGPU_VRAM_TYPE_HBM2:
vram_type = AMDGPU_VRAM_TYPE_HBM;
break;
+ case ATOM_DGPU_VRAM_TYPE_GDDR6:
+ vram_type = AMDGPU_VRAM_TYPE_GDDR6;
+ break;
default:
vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
break;
@@ -227,6 +259,9 @@ int amdgpu_atomfirmware_get_vram_type(struct amdgpu_device *adev)
case 3:
mem_type = vram_info->v23.vram_module[0].memory_type;
return convert_atom_mem_type_to_vram_type(adev, mem_type);
+ case 4:
+ mem_type = vram_info->v24.vram_module[0].memory_type;
+ return convert_atom_mem_type_to_vram_type(adev, mem_type);
default:
return 0;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
index 9b384a94d2f3..3e35a8f2c5e5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
@@ -574,6 +574,7 @@ static const struct amdgpu_px_quirk amdgpu_px_quirk_list[] = {
{ 0x1002, 0x6900, 0x1002, 0x0124, AMDGPU_PX_QUIRK_FORCE_ATPX },
{ 0x1002, 0x6900, 0x1028, 0x0812, AMDGPU_PX_QUIRK_FORCE_ATPX },
{ 0x1002, 0x6900, 0x1028, 0x0813, AMDGPU_PX_QUIRK_FORCE_ATPX },
+ { 0x1002, 0x699f, 0x1028, 0x0814, AMDGPU_PX_QUIRK_FORCE_ATPX },
{ 0x1002, 0x6900, 0x1025, 0x125A, AMDGPU_PX_QUIRK_FORCE_ATPX },
{ 0x1002, 0x6900, 0x17AA, 0x3806, AMDGPU_PX_QUIRK_FORCE_ATPX },
{ 0, 0, 0, 0, 0 },
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
index 3079ea8523c5..649e68c4479b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
@@ -21,7 +21,7 @@
*
* Authors: Jerome Glisse
*/
-#include <drm/drmP.h>
+
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
index a5df80d50d44..50dff69a0f6e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
@@ -25,10 +25,11 @@
* Alex Deucher
* Jerome Glisse
*/
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "atom.h"
+#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/acpi.h>
/*
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
index 5c79da8e1150..7bcf86c61999 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
@@ -28,7 +28,8 @@
* Christian König <deathsimple@vodafone.de>
*/
-#include <drm/drmP.h>
+#include <linux/uaccess.h>
+
#include "amdgpu.h"
#include "amdgpu_trace.h"
@@ -81,9 +82,9 @@ int amdgpu_bo_list_create(struct amdgpu_device *adev, struct drm_file *filp,
return -ENOMEM;
kref_init(&list->refcount);
- list->gds_obj = adev->gds.gds_gfx_bo;
- list->gws_obj = adev->gds.gws_gfx_bo;
- list->oa_obj = adev->gds.oa_gfx_bo;
+ list->gds_obj = NULL;
+ list->gws_obj = NULL;
+ list->oa_obj = NULL;
array = amdgpu_bo_list_array_entry(list, 0);
memset(array, 0, num_entries * sizeof(struct amdgpu_bo_list_entry));
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h
index 7c5f5d1601e6..a130e766cbdb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h
@@ -36,7 +36,7 @@ struct amdgpu_bo_list_entry {
struct amdgpu_bo_va *bo_va;
uint32_t priority;
struct page **user_pages;
- int user_invalidated;
+ bool user_invalidated;
};
struct amdgpu_bo_list {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index 387f1cf1dc20..031b094607bd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -22,8 +22,9 @@
*
*/
#include <linux/list.h>
+#include <linux/pci.h>
#include <linux/slab.h>
-#include <drm/drmP.h>
+
#include <linux/firmware.h>
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
index bf04c12bd324..73b2ede773d3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
@@ -23,7 +23,7 @@
* Authors: Dave Airlie
* Alex Deucher
*/
-#include <drm/drmP.h>
+
#include <drm/drm_edid.h>
#include <drm/drm_fb_helper.h>
#include <drm/drm_probe_helper.h>
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 2f6239b6be6f..8b26c970a3cb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -24,9 +24,11 @@
* Authors:
* Jerome Glisse <glisse@freedesktop.org>
*/
+
+#include <linux/file.h>
#include <linux/pagemap.h>
#include <linux/sync_file.h>
-#include <drm/drmP.h>
+
#include <drm/amdgpu_drm.h>
#include <drm/drm_syncobj.h>
#include "amdgpu.h"
@@ -52,7 +54,6 @@ static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
p->uf_entry.tv.bo = &bo->tbo;
/* One for TTM and one for the CS job */
p->uf_entry.tv.num_shared = 2;
- p->uf_entry.user_pages = NULL;
drm_gem_object_put_unlocked(gobj);
@@ -542,14 +543,14 @@ static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
if (usermm && usermm != current->mm)
return -EPERM;
- /* Check if we have user pages and nobody bound the BO already */
- if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm) &&
- lobj->user_pages) {
+ if (amdgpu_ttm_tt_is_userptr(bo->tbo.ttm) &&
+ lobj->user_invalidated && lobj->user_pages) {
amdgpu_bo_placement_from_domain(bo,
AMDGPU_GEM_DOMAIN_CPU);
r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
if (r)
return r;
+
amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm,
lobj->user_pages);
binding_userptr = true;
@@ -580,7 +581,6 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
struct amdgpu_bo *gds;
struct amdgpu_bo *gws;
struct amdgpu_bo *oa;
- unsigned tries = 10;
int r;
INIT_LIST_HEAD(&p->validated);
@@ -616,79 +616,45 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
if (p->uf_entry.tv.bo && !ttm_to_amdgpu_bo(p->uf_entry.tv.bo)->parent)
list_add(&p->uf_entry.tv.head, &p->validated);
- while (1) {
- struct list_head need_pages;
-
- r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true,
- &duplicates);
- if (unlikely(r != 0)) {
- if (r != -ERESTARTSYS)
- DRM_ERROR("ttm_eu_reserve_buffers failed.\n");
- goto error_free_pages;
- }
-
- INIT_LIST_HEAD(&need_pages);
- amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
- struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
-
- if (amdgpu_ttm_tt_userptr_invalidated(bo->tbo.ttm,
- &e->user_invalidated) && e->user_pages) {
-
- /* We acquired a page array, but somebody
- * invalidated it. Free it and try again
- */
- release_pages(e->user_pages,
- bo->tbo.ttm->num_pages);
- kvfree(e->user_pages);
- e->user_pages = NULL;
- }
-
- if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm) &&
- !e->user_pages) {
- list_del(&e->tv.head);
- list_add(&e->tv.head, &need_pages);
-
- amdgpu_bo_unreserve(bo);
- }
+ /* Get userptr backing pages. If pages are updated after registered
+ * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do
+ * amdgpu_ttm_backend_bind() to flush and invalidate new pages
+ */
+ amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
+ struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
+ bool userpage_invalidated = false;
+ int i;
+
+ e->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages,
+ sizeof(struct page *),
+ GFP_KERNEL | __GFP_ZERO);
+ if (!e->user_pages) {
+ DRM_ERROR("calloc failure\n");
+ return -ENOMEM;
}
- if (list_empty(&need_pages))
- break;
-
- /* Unreserve everything again. */
- ttm_eu_backoff_reservation(&p->ticket, &p->validated);
-
- /* We tried too many times, just abort */
- if (!--tries) {
- r = -EDEADLK;
- DRM_ERROR("deadlock in %s\n", __func__);
- goto error_free_pages;
+ r = amdgpu_ttm_tt_get_user_pages(bo, e->user_pages);
+ if (r) {
+ kvfree(e->user_pages);
+ e->user_pages = NULL;
+ return r;
}
- /* Fill the page arrays for all userptrs. */
- list_for_each_entry(e, &need_pages, tv.head) {
- struct ttm_tt *ttm = e->tv.bo->ttm;
-
- e->user_pages = kvmalloc_array(ttm->num_pages,
- sizeof(struct page*),
- GFP_KERNEL | __GFP_ZERO);
- if (!e->user_pages) {
- r = -ENOMEM;
- DRM_ERROR("calloc failure in %s\n", __func__);
- goto error_free_pages;
- }
-
- r = amdgpu_ttm_tt_get_user_pages(ttm, e->user_pages);
- if (r) {
- DRM_ERROR("amdgpu_ttm_tt_get_user_pages failed.\n");
- kvfree(e->user_pages);
- e->user_pages = NULL;
- goto error_free_pages;
+ for (i = 0; i < bo->tbo.ttm->num_pages; i++) {
+ if (bo->tbo.ttm->pages[i] != e->user_pages[i]) {
+ userpage_invalidated = true;
+ break;
}
}
+ e->user_invalidated = userpage_invalidated;
+ }
- /* And try again. */
- list_splice(&need_pages, &p->validated);
+ r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true,
+ &duplicates, false);
+ if (unlikely(r != 0)) {
+ if (r != -ERESTARTSYS)
+ DRM_ERROR("ttm_eu_reserve_buffers failed.\n");
+ goto out;
}
amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold,
@@ -707,16 +673,12 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
}
r = amdgpu_cs_list_validate(p, &duplicates);
- if (r) {
- DRM_ERROR("amdgpu_cs_list_validate(duplicates) failed.\n");
+ if (r)
goto error_validate;
- }
r = amdgpu_cs_list_validate(p, &p->validated);
- if (r) {
- DRM_ERROR("amdgpu_cs_list_validate(validated) failed.\n");
+ if (r)
goto error_validate;
- }
amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
p->bytes_moved_vis);
@@ -757,17 +719,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
error_validate:
if (r)
ttm_eu_backoff_reservation(&p->ticket, &p->validated);
-
-error_free_pages:
-
- amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
- if (!e->user_pages)
- continue;
-
- release_pages(e->user_pages, e->tv.bo->ttm->num_pages);
- kvfree(e->user_pages);
- }
-
+out:
return r;
}
@@ -922,7 +874,7 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
if (r)
return r;
- if (amdgpu_sriov_vf(adev)) {
+ if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) {
struct dma_fence *f;
bo_va = fpriv->csa_va;
@@ -1011,7 +963,8 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
continue;
- if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX && amdgpu_sriov_vf(adev)) {
+ if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX &&
+ (amdgpu_mcbp || amdgpu_sriov_vf(adev))) {
if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
if (chunk_ib->flags & AMDGPU_IB_FLAG_CE)
ce_preempt++;
@@ -1054,11 +1007,9 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
j++;
}
- /* UVD & VCE fw doesn't support user fences */
+ /* MM engine doesn't support user fences */
ring = to_amdgpu_ring(parser->entity->rq->sched);
- if (parser->job->uf_addr && (
- ring->funcs->type == AMDGPU_RING_TYPE_UVD ||
- ring->funcs->type == AMDGPU_RING_TYPE_VCE))
+ if (parser->job->uf_addr && ring->funcs->no_user_fence)
return -EINVAL;
return amdgpu_ctx_wait_prev_fence(parser->ctx, parser->entity);
@@ -1093,29 +1044,27 @@ static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p,
return r;
}
- fence = amdgpu_ctx_get_fence(ctx, entity,
- deps[i].handle);
+ fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle);
+ amdgpu_ctx_put(ctx);
+
+ if (IS_ERR(fence))
+ return PTR_ERR(fence);
+ else if (!fence)
+ continue;
if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) {
- struct drm_sched_fence *s_fence = to_drm_sched_fence(fence);
+ struct drm_sched_fence *s_fence;
struct dma_fence *old = fence;
+ s_fence = to_drm_sched_fence(fence);
fence = dma_fence_get(&s_fence->scheduled);
dma_fence_put(old);
}
- if (IS_ERR(fence)) {
- r = PTR_ERR(fence);
- amdgpu_ctx_put(ctx);
+ r = amdgpu_sync_fence(p->adev, &p->job->sync, fence, true);
+ dma_fence_put(fence);
+ if (r)
return r;
- } else if (fence) {
- r = amdgpu_sync_fence(p->adev, &p->job->sync, fence,
- true);
- dma_fence_put(fence);
- amdgpu_ctx_put(ctx);
- if (r)
- return r;
- }
}
return 0;
}
@@ -1194,6 +1143,9 @@ static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser *p,
num_deps = chunk->length_dw * 4 /
sizeof(struct drm_amdgpu_cs_chunk_sem);
+ if (p->post_deps)
+ return -EINVAL;
+
p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),
GFP_KERNEL);
p->num_post_deps = 0;
@@ -1217,8 +1169,7 @@ static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser *p,
static int amdgpu_cs_process_syncobj_timeline_out_dep(struct amdgpu_cs_parser *p,
- struct amdgpu_cs_chunk
- *chunk)
+ struct amdgpu_cs_chunk *chunk)
{
struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps;
unsigned num_deps;
@@ -1228,6 +1179,9 @@ static int amdgpu_cs_process_syncobj_timeline_out_dep(struct amdgpu_cs_parser *p
num_deps = chunk->length_dw * 4 /
sizeof(struct drm_amdgpu_cs_chunk_syncobj);
+ if (p->post_deps)
+ return -EINVAL;
+
p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),
GFP_KERNEL);
p->num_post_deps = 0;
@@ -1328,7 +1282,6 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
struct amdgpu_bo_list_entry *e;
struct amdgpu_job *job;
uint64_t seq;
-
int r;
job = p->job;
@@ -1338,15 +1291,23 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
if (r)
goto error_unlock;
- /* No memory allocation is allowed while holding the mn lock */
+ /* No memory allocation is allowed while holding the mn lock.
+ * p->mn is hold until amdgpu_cs_submit is finished and fence is added
+ * to BOs.
+ */
amdgpu_mn_lock(p->mn);
+
+ /* If userptr are invalidated after amdgpu_cs_parser_bos(), return
+ * -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl.
+ */
amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
- if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm)) {
- r = -ERESTARTSYS;
- goto error_abort;
- }
+ r |= !amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
+ }
+ if (r) {
+ r = -EAGAIN;
+ goto error_abort;
}
job->owner = p->filp;
@@ -1424,7 +1385,7 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
if (r) {
if (r == -ENOMEM)
DRM_ERROR("Not enough memory for command submission!\n");
- else if (r != -ERESTARTSYS)
+ else if (r != -ERESTARTSYS && r != -EAGAIN)
DRM_ERROR("Failed to process the buffer list %d!\n", r);
goto out;
}
@@ -1442,6 +1403,7 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
out:
amdgpu_cs_parser_fini(&parser, r, reserved_buffers);
+
return r;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
index 54dd02a898b9..35a8d3c96fc9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
@@ -47,6 +47,7 @@ int amdgpu_allocate_static_csa(struct amdgpu_device *adev, struct amdgpu_bo **bo
return -ENOMEM;
memset(ptr, 0, size);
+ adev->virt.csa_cpu_addr = ptr;
return 0;
}
@@ -79,7 +80,7 @@ int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm,
list_add(&csa_tv.head, &list);
amdgpu_vm_get_pd_bo(vm, &list, &pd);
- r = ttm_eu_reserve_buffers(&ticket, &list, true, NULL);
+ r = ttm_eu_reserve_buffers(&ticket, &list, true, NULL, false);
if (r) {
DRM_ERROR("failed to reserve CSA,PD BOs: err=%d\n", r);
return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index a28a3d722ba2..7398b4850649 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -22,7 +22,6 @@
* Authors: monk liu <monk.liu@amd.com>
*/
-#include <drm/drmP.h>
#include <drm/drm_auth.h>
#include "amdgpu.h"
#include "amdgpu_sched.h"
@@ -535,21 +534,24 @@ int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
struct drm_sched_entity *entity)
{
struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
- unsigned idx = centity->sequence & (amdgpu_sched_jobs - 1);
- struct dma_fence *other = centity->fences[idx];
+ struct dma_fence *other;
+ unsigned idx;
+ long r;
- if (other) {
- signed long r;
- r = dma_fence_wait(other, true);
- if (r < 0) {
- if (r != -ERESTARTSYS)
- DRM_ERROR("Error (%ld) waiting for fence!\n", r);
+ spin_lock(&ctx->ring_lock);
+ idx = centity->sequence & (amdgpu_sched_jobs - 1);
+ other = dma_fence_get(centity->fences[idx]);
+ spin_unlock(&ctx->ring_lock);
- return r;
- }
- }
+ if (!other)
+ return 0;
- return 0;
+ r = dma_fence_wait(other, true);
+ if (r < 0 && r != -ERESTARTSYS)
+ DRM_ERROR("Error (%ld) waiting for fence!\n", r);
+
+ dma_fence_put(other);
+ return r;
}
void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
index 8930d66f2204..5652cc72ed3a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
@@ -24,8 +24,11 @@
*/
#include <linux/kthread.h>
-#include <drm/drmP.h>
-#include <linux/debugfs.h>
+#include <linux/pci.h>
+#include <linux/uaccess.h>
+
+#include <drm/drm_debugfs.h>
+
#include "amdgpu.h"
/**
@@ -103,10 +106,10 @@ static int amdgpu_debugfs_process_reg_op(bool read, struct file *f,
ssize_t result = 0;
int r;
bool pm_pg_lock, use_bank, use_ring;
- unsigned instance_bank, sh_bank, se_bank, me, pipe, queue;
+ unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
pm_pg_lock = use_bank = use_ring = false;
- instance_bank = sh_bank = se_bank = me = pipe = queue = 0;
+ instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
if (size & 0x3 || *pos & 0x3 ||
((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
@@ -132,6 +135,7 @@ static int amdgpu_debugfs_process_reg_op(bool read, struct file *f,
me = (*pos & GENMASK_ULL(33, 24)) >> 24;
pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
+ vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
use_ring = 1;
} else {
@@ -149,7 +153,7 @@ static int amdgpu_debugfs_process_reg_op(bool read, struct file *f,
sh_bank, instance_bank);
} else if (use_ring) {
mutex_lock(&adev->srbm_mutex);
- amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue);
+ amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid);
}
if (pm_pg_lock)
@@ -182,7 +186,7 @@ end:
amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
mutex_unlock(&adev->grbm_idx_mutex);
} else if (use_ring) {
- amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0);
+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
mutex_unlock(&adev->srbm_mutex);
}
@@ -703,7 +707,7 @@ static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
- data = kmalloc_array(1024, sizeof(*data), GFP_KERNEL);
+ data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
if (!data)
return -ENOMEM;
@@ -920,17 +924,195 @@ static const struct drm_info_list amdgpu_debugfs_list[] = {
{"amdgpu_evict_gtt", &amdgpu_debugfs_evict_gtt},
};
+static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
+ struct dma_fence **fences)
+{
+ struct amdgpu_fence_driver *drv = &ring->fence_drv;
+ uint32_t sync_seq, last_seq;
+
+ last_seq = atomic_read(&ring->fence_drv.last_seq);
+ sync_seq = ring->fence_drv.sync_seq;
+
+ last_seq &= drv->num_fences_mask;
+ sync_seq &= drv->num_fences_mask;
+
+ do {
+ struct dma_fence *fence, **ptr;
+
+ ++last_seq;
+ last_seq &= drv->num_fences_mask;
+ ptr = &drv->fences[last_seq];
+
+ fence = rcu_dereference_protected(*ptr, 1);
+ RCU_INIT_POINTER(*ptr, NULL);
+
+ if (!fence)
+ continue;
+
+ fences[last_seq] = fence;
+
+ } while (last_seq != sync_seq);
+}
+
+static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
+ int length)
+{
+ int i;
+ struct dma_fence *fence;
+
+ for (i = 0; i < length; i++) {
+ fence = fences[i];
+ if (!fence)
+ continue;
+ dma_fence_signal(fence);
+ dma_fence_put(fence);
+ }
+}
+
+static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
+{
+ struct drm_sched_job *s_job;
+ struct dma_fence *fence;
+
+ spin_lock(&sched->job_list_lock);
+ list_for_each_entry(s_job, &sched->ring_mirror_list, node) {
+ fence = sched->ops->run_job(s_job);
+ dma_fence_put(fence);
+ }
+ spin_unlock(&sched->job_list_lock);
+}
+
+static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
+{
+ struct amdgpu_job *job;
+ struct drm_sched_job *s_job;
+ uint32_t preempt_seq;
+ struct dma_fence *fence, **ptr;
+ struct amdgpu_fence_driver *drv = &ring->fence_drv;
+ struct drm_gpu_scheduler *sched = &ring->sched;
+
+ if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
+ return;
+
+ preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
+ if (preempt_seq <= atomic_read(&drv->last_seq))
+ return;
+
+ preempt_seq &= drv->num_fences_mask;
+ ptr = &drv->fences[preempt_seq];
+ fence = rcu_dereference_protected(*ptr, 1);
+
+ spin_lock(&sched->job_list_lock);
+ list_for_each_entry(s_job, &sched->ring_mirror_list, node) {
+ job = to_amdgpu_job(s_job);
+ if (job->fence == fence)
+ /* mark the job as preempted */
+ job->preemption_status |= AMDGPU_IB_PREEMPTED;
+ }
+ spin_unlock(&sched->job_list_lock);
+}
+
+static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
+{
+ int r, resched, length;
+ struct amdgpu_ring *ring;
+ struct dma_fence **fences = NULL;
+ struct amdgpu_device *adev = (struct amdgpu_device *)data;
+
+ if (val >= AMDGPU_MAX_RINGS)
+ return -EINVAL;
+
+ ring = adev->rings[val];
+
+ if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
+ return -EINVAL;
+
+ /* the last preemption failed */
+ if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
+ return -EBUSY;
+
+ length = ring->fence_drv.num_fences_mask + 1;
+ fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
+ if (!fences)
+ return -ENOMEM;
+
+ /* stop the scheduler */
+ kthread_park(ring->sched.thread);
+
+ resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
+
+ /* preempt the IB */
+ r = amdgpu_ring_preempt_ib(ring);
+ if (r) {
+ DRM_WARN("failed to preempt ring %d\n", ring->idx);
+ goto failure;
+ }
+
+ amdgpu_fence_process(ring);
+
+ if (atomic_read(&ring->fence_drv.last_seq) !=
+ ring->fence_drv.sync_seq) {
+ DRM_INFO("ring %d was preempted\n", ring->idx);
+
+ amdgpu_ib_preempt_mark_partial_job(ring);
+
+ /* swap out the old fences */
+ amdgpu_ib_preempt_fences_swap(ring, fences);
+
+ amdgpu_fence_driver_force_completion(ring);
+
+ /* resubmit unfinished jobs */
+ amdgpu_ib_preempt_job_recovery(&ring->sched);
+
+ /* wait for jobs finished */
+ amdgpu_fence_wait_empty(ring);
+
+ /* signal the old fences */
+ amdgpu_ib_preempt_signal_fences(fences, length);
+ }
+
+failure:
+ /* restart the scheduler */
+ kthread_unpark(ring->sched.thread);
+
+ ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
+
+ if (fences)
+ kfree(fences);
+
+ return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(fops_ib_preempt, NULL,
+ amdgpu_debugfs_ib_preempt, "%llu\n");
+
int amdgpu_debugfs_init(struct amdgpu_device *adev)
{
+ adev->debugfs_preempt =
+ debugfs_create_file("amdgpu_preempt_ib", 0600,
+ adev->ddev->primary->debugfs_root,
+ (void *)adev, &fops_ib_preempt);
+ if (!(adev->debugfs_preempt)) {
+ DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
+ return -EIO;
+ }
+
return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list,
ARRAY_SIZE(amdgpu_debugfs_list));
}
+void amdgpu_debugfs_preempt_cleanup(struct amdgpu_device *adev)
+{
+ if (adev->debugfs_preempt)
+ debugfs_remove(adev->debugfs_preempt);
+}
+
#else
int amdgpu_debugfs_init(struct amdgpu_device *adev)
{
return 0;
}
+void amdgpu_debugfs_preempt_cleanup(struct amdgpu_device *adev) { }
int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
{
return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
index 8260d8073c26..f289d28ad6b2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
@@ -34,6 +34,7 @@ struct amdgpu_debugfs {
int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev);
int amdgpu_debugfs_init(struct amdgpu_device *adev);
+void amdgpu_debugfs_preempt_cleanup(struct amdgpu_device *adev);
int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
const struct drm_info_list *files,
unsigned nfiles);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index f4ac632a87b2..5a7f893cf724 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -27,9 +27,10 @@
*/
#include <linux/power_supply.h>
#include <linux/kthread.h>
+#include <linux/module.h>
#include <linux/console.h>
#include <linux/slab.h>
-#include <drm/drmP.h>
+
#include <drm/drm_atomic_helper.h>
#include <drm/drm_probe_helper.h>
#include <drm/amdgpu_drm.h>
@@ -51,6 +52,7 @@
#endif
#include "vi.h"
#include "soc15.h"
+#include "nv.h"
#include "bif/bif_4_1_d.h"
#include <linux/pci.h>
#include <linux/firmware.h>
@@ -61,12 +63,14 @@
#include "amdgpu_xgmi.h"
#include "amdgpu_ras.h"
+#include "amdgpu_pmu.h"
MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
+MODULE_FIRMWARE("amdgpu/navi10_gpu_info.bin");
#define AMDGPU_RESUME_MS 2000
@@ -94,9 +98,32 @@ static const char *amdgpu_asic_name[] = {
"VEGA12",
"VEGA20",
"RAVEN",
+ "NAVI10",
"LAST",
};
+/**
+ * DOC: pcie_replay_count
+ *
+ * The amdgpu driver provides a sysfs API for reporting the total number
+ * of PCIe replays (NAKs)
+ * The file pcie_replay_count is used for this and returns the total
+ * number of replays as a sum of the NAKs generated and NAKs received
+ */
+
+static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct drm_device *ddev = dev_get_drvdata(dev);
+ struct amdgpu_device *adev = ddev->dev_private;
+ uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
+
+ return snprintf(buf, PAGE_SIZE, "%llu\n", cnt);
+}
+
+static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
+ amdgpu_device_get_pcie_replay_count, NULL);
+
static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
/**
@@ -484,7 +511,10 @@ void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
} else {
tmp = RREG32(reg);
tmp &= ~and_mask;
- tmp |= or_mask;
+ if (adev->family >= AMDGPU_FAMILY_AI)
+ tmp |= (or_mask & and_mask);
+ else
+ tmp |= or_mask;
}
WREG32(reg, tmp);
}
@@ -910,8 +940,10 @@ def_value:
* Validates certain module parameters and updates
* the associated values used by the driver (all asics).
*/
-static void amdgpu_device_check_arguments(struct amdgpu_device *adev)
+static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
{
+ int ret = 0;
+
if (amdgpu_sched_jobs < 4) {
dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
amdgpu_sched_jobs);
@@ -949,19 +981,15 @@ static void amdgpu_device_check_arguments(struct amdgpu_device *adev)
amdgpu_device_check_block_size(adev);
- if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
- !is_power_of_2(amdgpu_vram_page_split))) {
- dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
- amdgpu_vram_page_split);
- amdgpu_vram_page_split = 1024;
- }
-
- if (amdgpu_lockup_timeout == 0) {
- dev_warn(adev->dev, "lockup_timeout msut be > 0, adjusting to 10000\n");
- amdgpu_lockup_timeout = 10000;
+ ret = amdgpu_device_get_job_timeout_settings(adev);
+ if (ret) {
+ dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
+ return ret;
}
adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
+
+ return ret;
}
/**
@@ -1356,6 +1384,9 @@ static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
else
chip_name = "raven";
break;
+ case CHIP_NAVI10:
+ chip_name = "navi10";
+ break;
}
snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
@@ -1402,6 +1433,23 @@ static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
adev->gfx.cu_info.max_scratch_slots_per_cu =
le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
+ if (hdr->version_minor >= 1) {
+ const struct gpu_info_firmware_v1_1 *gpu_info_fw =
+ (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
+ le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+ adev->gfx.config.num_sc_per_sh =
+ le32_to_cpu(gpu_info_fw->num_sc_per_sh);
+ adev->gfx.config.num_packer_per_sc =
+ le32_to_cpu(gpu_info_fw->num_packer_per_sc);
+ }
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ if (hdr->version_minor == 2) {
+ const struct gpu_info_firmware_v1_2 *gpu_info_fw =
+ (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
+ le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+ adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
+ }
+#endif
break;
}
default:
@@ -1490,6 +1538,13 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
if (r)
return r;
break;
+ case CHIP_NAVI10:
+ adev->family = AMDGPU_FAMILY_NV;
+
+ r = nv_set_ip_blocks(adev);
+ if (r)
+ return r;
+ break;
default:
/* FIXME: not supported yet */
return -EINVAL;
@@ -1505,6 +1560,9 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
r = amdgpu_virt_request_full_gpu(adev, true);
if (r)
return -EAGAIN;
+
+ /* query the reg access mode at the very beginning */
+ amdgpu_virt_init_reg_access_mode(adev);
}
adev->pm.pp_feature = amdgpu_pp_feature_mask;
@@ -1532,6 +1590,19 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
adev->ip_blocks[i].status.valid = true;
}
}
+ /* get the vbios after the asic_funcs are set up */
+ if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
+ /* Read BIOS */
+ if (!amdgpu_get_bios(adev))
+ return -EINVAL;
+
+ r = amdgpu_atombios_init(adev);
+ if (r) {
+ dev_err(adev->dev, "amdgpu_atombios_init failed\n");
+ amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
+ return r;
+ }
+ }
}
adev->cg_flags &= amdgpu_cg_mask;
@@ -1550,6 +1621,7 @@ static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
if (adev->ip_blocks[i].status.hw)
continue;
if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
+ (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
r = adev->ip_blocks[i].version->funcs->hw_init(adev);
if (r) {
@@ -1670,7 +1742,7 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
adev->ip_blocks[i].status.hw = true;
/* right after GMC hw init, we create CSA */
- if (amdgpu_sriov_vf(adev)) {
+ if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) {
r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
AMDGPU_GEM_DOMAIN_VRAM,
AMDGPU_CSA_SIZE);
@@ -1821,6 +1893,43 @@ static int amdgpu_device_set_pg_state(struct amdgpu_device *adev, enum amd_power
return 0;
}
+static int amdgpu_device_enable_mgpu_fan_boost(void)
+{
+ struct amdgpu_gpu_instance *gpu_ins;
+ struct amdgpu_device *adev;
+ int i, ret = 0;
+
+ mutex_lock(&mgpu_info.mutex);
+
+ /*
+ * MGPU fan boost feature should be enabled
+ * only when there are two or more dGPUs in
+ * the system
+ */
+ if (mgpu_info.num_dgpu < 2)
+ goto out;
+
+ for (i = 0; i < mgpu_info.num_dgpu; i++) {
+ gpu_ins = &(mgpu_info.gpu_ins[i]);
+ adev = gpu_ins->adev;
+ if (!(adev->flags & AMD_IS_APU) &&
+ !gpu_ins->mgpu_fan_enabled &&
+ adev->powerplay.pp_funcs &&
+ adev->powerplay.pp_funcs->enable_mgpu_fan_boost) {
+ ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
+ if (ret)
+ break;
+
+ gpu_ins->mgpu_fan_enabled = 1;
+ }
+ }
+
+out:
+ mutex_unlock(&mgpu_info.mutex);
+
+ return ret;
+}
+
/**
* amdgpu_device_ip_late_init - run late init for hardware IPs
*
@@ -1854,11 +1963,15 @@ static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
- queue_delayed_work(system_wq, &adev->late_init_work,
- msecs_to_jiffies(AMDGPU_RESUME_MS));
-
amdgpu_device_fill_reset_magic(adev);
+ r = amdgpu_device_enable_mgpu_fan_boost();
+ if (r)
+ DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
+
+ /* set to low pstate by default */
+ amdgpu_xgmi_set_pstate(adev, 0);
+
return 0;
}
@@ -1957,65 +2070,20 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
return 0;
}
-static int amdgpu_device_enable_mgpu_fan_boost(void)
-{
- struct amdgpu_gpu_instance *gpu_ins;
- struct amdgpu_device *adev;
- int i, ret = 0;
-
- mutex_lock(&mgpu_info.mutex);
-
- /*
- * MGPU fan boost feature should be enabled
- * only when there are two or more dGPUs in
- * the system
- */
- if (mgpu_info.num_dgpu < 2)
- goto out;
-
- for (i = 0; i < mgpu_info.num_dgpu; i++) {
- gpu_ins = &(mgpu_info.gpu_ins[i]);
- adev = gpu_ins->adev;
- if (!(adev->flags & AMD_IS_APU) &&
- !gpu_ins->mgpu_fan_enabled &&
- adev->powerplay.pp_funcs &&
- adev->powerplay.pp_funcs->enable_mgpu_fan_boost) {
- ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
- if (ret)
- break;
-
- gpu_ins->mgpu_fan_enabled = 1;
- }
- }
-
-out:
- mutex_unlock(&mgpu_info.mutex);
-
- return ret;
-}
-
/**
- * amdgpu_device_ip_late_init_func_handler - work handler for ib test
+ * amdgpu_device_delayed_init_work_handler - work handler for IB tests
*
* @work: work_struct.
*/
-static void amdgpu_device_ip_late_init_func_handler(struct work_struct *work)
+static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
{
struct amdgpu_device *adev =
- container_of(work, struct amdgpu_device, late_init_work.work);
+ container_of(work, struct amdgpu_device, delayed_init_work.work);
int r;
r = amdgpu_ib_ring_tests(adev);
if (r)
DRM_ERROR("ib ring test failed (%d).\n", r);
-
- r = amdgpu_device_enable_mgpu_fan_boost();
- if (r)
- DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
-
- /*set to low pstate by default */
- amdgpu_xgmi_set_pstate(adev, 0);
-
}
static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
@@ -2356,6 +2424,9 @@ bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
case CHIP_RAVEN:
#endif
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ case CHIP_NAVI10:
+#endif
return amdgpu_dc != 0;
#endif
default:
@@ -2466,8 +2537,11 @@ int amdgpu_device_init(struct amdgpu_device *adev,
hash_init(adev->mn_hash);
mutex_init(&adev->lock_reset);
mutex_init(&adev->virt.dpm_mutex);
+ mutex_init(&adev->psp.mutex);
- amdgpu_device_check_arguments(adev);
+ r = amdgpu_device_check_arguments(adev);
+ if (r)
+ return r;
spin_lock_init(&adev->mmio_idx_lock);
spin_lock_init(&adev->smc_idx_lock);
@@ -2485,8 +2559,8 @@ int amdgpu_device_init(struct amdgpu_device *adev,
INIT_LIST_HEAD(&adev->ring_lru_list);
spin_lock_init(&adev->ring_lru_list_lock);
- INIT_DELAYED_WORK(&adev->late_init_work,
- amdgpu_device_ip_late_init_func_handler);
+ INIT_DELAYED_WORK(&adev->delayed_init_work,
+ amdgpu_device_delayed_init_work_handler);
INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
amdgpu_device_delay_enable_gfx_off);
@@ -2523,8 +2597,33 @@ int amdgpu_device_init(struct amdgpu_device *adev,
if (adev->rio_mem == NULL)
DRM_INFO("PCI I/O BAR is not found.\n");
+ /* enable PCIE atomic ops */
+ r = pci_enable_atomic_ops_to_root(adev->pdev,
+ PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
+ PCI_EXP_DEVCAP2_ATOMIC_COMP64);
+ if (r) {
+ adev->have_atomics_support = false;
+ DRM_INFO("PCIE atomic ops is not supported\n");
+ } else {
+ adev->have_atomics_support = true;
+ }
+
amdgpu_device_get_pcie_info(adev);
+ if (amdgpu_mcbp)
+ DRM_INFO("MCBP is enabled\n");
+
+ if (amdgpu_mes && adev->asic_type >= CHIP_NAVI10)
+ adev->enable_mes = true;
+
+ if (amdgpu_discovery && adev->asic_type >= CHIP_NAVI10) {
+ r = amdgpu_discovery_init(adev);
+ if (r) {
+ dev_err(adev->dev, "amdgpu_discovery_init failed\n");
+ return r;
+ }
+ }
+
/* early init functions */
r = amdgpu_device_ip_early_init(adev);
if (r)
@@ -2552,19 +2651,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
goto fence_driver_init;
}
- /* Read BIOS */
- if (!amdgpu_get_bios(adev)) {
- r = -EINVAL;
- goto failed;
- }
-
- r = amdgpu_atombios_init(adev);
- if (r) {
- dev_err(adev->dev, "amdgpu_atombios_init failed\n");
- amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
- goto failed;
- }
-
/* detect if we are with an SRIOV vbios */
amdgpu_device_detect_sriov_bios(adev);
@@ -2662,10 +2748,17 @@ fence_driver_init:
amdgpu_fbdev_init(adev);
+ if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev))
+ amdgpu_pm_virt_sysfs_init(adev);
+
r = amdgpu_pm_sysfs_init(adev);
if (r)
DRM_ERROR("registering pm debugfs failed (%d).\n", r);
+ r = amdgpu_ucode_sysfs_init(adev);
+ if (r)
+ DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
+
r = amdgpu_debugfs_gem_init(adev);
if (r)
DRM_ERROR("registering gem debugfs failed (%d).\n", r);
@@ -2706,7 +2799,21 @@ fence_driver_init:
}
/* must succeed. */
- amdgpu_ras_post_init(adev);
+ amdgpu_ras_resume(adev);
+
+ queue_delayed_work(system_wq, &adev->delayed_init_work,
+ msecs_to_jiffies(AMDGPU_RESUME_MS));
+
+ r = device_create_file(adev->dev, &dev_attr_pcie_replay_count);
+ if (r) {
+ dev_err(adev->dev, "Could not create pcie_replay_count");
+ return r;
+ }
+
+ if (IS_ENABLED(CONFIG_PERF_EVENTS))
+ r = amdgpu_pmu_init(adev);
+ if (r)
+ dev_err(adev->dev, "amdgpu_pmu_init failed\n");
return 0;
@@ -2749,7 +2856,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
adev->firmware.gpu_info_fw = NULL;
}
adev->accel_working = false;
- cancel_delayed_work_sync(&adev->late_init_work);
+ cancel_delayed_work_sync(&adev->delayed_init_work);
/* free i2c buses */
if (!amdgpu_device_has_dc_support(adev))
amdgpu_i2c_fini(adev);
@@ -2770,7 +2877,17 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
iounmap(adev->rmmio);
adev->rmmio = NULL;
amdgpu_device_doorbell_fini(adev);
+ if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev))
+ amdgpu_pm_virt_sysfs_fini(adev);
+
amdgpu_debugfs_regs_cleanup(adev);
+ device_remove_file(adev->dev, &dev_attr_pcie_replay_count);
+ amdgpu_ucode_sysfs_fini(adev);
+ if (IS_ENABLED(CONFIG_PERF_EVENTS))
+ amdgpu_pmu_fini(adev);
+ amdgpu_debugfs_preempt_cleanup(adev);
+ if (amdgpu_discovery && adev->asic_type >= CHIP_NAVI10)
+ amdgpu_discovery_fini(adev);
}
@@ -2810,7 +2927,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
if (fbcon)
amdgpu_fbdev_set_suspend(adev, 1);
- cancel_delayed_work_sync(&adev->late_init_work);
+ cancel_delayed_work_sync(&adev->delayed_init_work);
if (!amdgpu_device_has_dc_support(adev)) {
/* turn off display hw */
@@ -2851,6 +2968,8 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
amdgpu_amdkfd_suspend(adev);
+ amdgpu_ras_suspend(adev);
+
r = amdgpu_device_ip_suspend_phase1(adev);
/* evict vram memory */
@@ -2928,6 +3047,9 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
if (r)
return r;
+ queue_delayed_work(system_wq, &adev->delayed_init_work,
+ msecs_to_jiffies(AMDGPU_RESUME_MS));
+
if (!amdgpu_device_has_dc_support(adev)) {
/* pin cursors */
list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
@@ -2951,7 +3073,7 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
return r;
/* Make sure IB tests flushed */
- flush_delayed_work(&adev->late_init_work);
+ flush_delayed_work(&adev->delayed_init_work);
/* blat the mode back in */
if (fbcon) {
@@ -2971,6 +3093,8 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
drm_kms_helper_poll_enable(dev);
+ amdgpu_ras_resume(adev);
+
/*
* Most of the connector probing functions try to acquire runtime pm
* refs to ensure that the GPU is powered on when connector polling is
@@ -3335,8 +3459,6 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
if (!ring || !ring->sched.thread)
continue;
- drm_sched_stop(&ring->sched);
-
/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
amdgpu_fence_driver_force_completion(ring);
}
@@ -3344,8 +3466,7 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
if(job)
drm_sched_increase_karma(&job->base);
-
-
+ /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
if (!amdgpu_sriov_vf(adev)) {
if (!need_full_reset)
@@ -3452,6 +3573,19 @@ static int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive,
if (vram_lost)
amdgpu_device_fill_reset_magic(tmp_adev);
+ /*
+ * Add this ASIC as tracked as reset was already
+ * complete successfully.
+ */
+ amdgpu_register_gpu_instance(tmp_adev);
+
+ r = amdgpu_device_ip_late_init(tmp_adev);
+ if (r)
+ goto out;
+
+ /* must succeed. */
+ amdgpu_ras_resume(tmp_adev);
+
/* Update PSP FW topology after reset */
if (hive && tmp_adev->gmc.xgmi.num_physical_nodes > 1)
r = amdgpu_xgmi_update_topology(hive, tmp_adev);
@@ -3483,38 +3617,21 @@ end:
return r;
}
-static void amdgpu_device_post_asic_reset(struct amdgpu_device *adev,
- struct amdgpu_job *job)
+static bool amdgpu_device_lock_adev(struct amdgpu_device *adev, bool trylock)
{
- int i;
+ if (trylock) {
+ if (!mutex_trylock(&adev->lock_reset))
+ return false;
+ } else
+ mutex_lock(&adev->lock_reset);
- for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
- struct amdgpu_ring *ring = adev->rings[i];
-
- if (!ring || !ring->sched.thread)
- continue;
-
- if (!adev->asic_reset_res)
- drm_sched_resubmit_jobs(&ring->sched);
-
- drm_sched_start(&ring->sched, !adev->asic_reset_res);
- }
-
- if (!amdgpu_device_has_dc_support(adev)) {
- drm_helper_resume_force_mode(adev->ddev);
- }
-
- adev->asic_reset_res = 0;
-}
-
-static void amdgpu_device_lock_adev(struct amdgpu_device *adev)
-{
- mutex_lock(&adev->lock_reset);
atomic_inc(&adev->gpu_reset_counter);
adev->in_gpu_reset = 1;
/* Block kfd: SRIOV would do it separately */
if (!amdgpu_sriov_vf(adev))
amdgpu_amdkfd_pre_reset(adev);
+
+ return true;
}
static void amdgpu_device_unlock_adev(struct amdgpu_device *adev)
@@ -3542,40 +3659,44 @@ static void amdgpu_device_unlock_adev(struct amdgpu_device *adev)
int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
struct amdgpu_job *job)
{
- int r;
+ struct list_head device_list, *device_list_handle = NULL;
+ bool need_full_reset, job_signaled;
struct amdgpu_hive_info *hive = NULL;
- bool need_full_reset = false;
struct amdgpu_device *tmp_adev = NULL;
- struct list_head device_list, *device_list_handle = NULL;
+ int i, r = 0;
+ need_full_reset = job_signaled = false;
INIT_LIST_HEAD(&device_list);
dev_info(adev->dev, "GPU reset begin!\n");
+ cancel_delayed_work_sync(&adev->delayed_init_work);
+
+ hive = amdgpu_get_xgmi_hive(adev, false);
+
/*
- * In case of XGMI hive disallow concurrent resets to be triggered
- * by different nodes. No point also since the one node already executing
- * reset will also reset all the other nodes in the hive.
+ * Here we trylock to avoid chain of resets executing from
+ * either trigger by jobs on different adevs in XGMI hive or jobs on
+ * different schedulers for same device while this TO handler is running.
+ * We always reset all schedulers for device and all devices for XGMI
+ * hive so that should take care of them too.
*/
- hive = amdgpu_get_xgmi_hive(adev, 0);
- if (hive && adev->gmc.xgmi.num_physical_nodes > 1 &&
- !mutex_trylock(&hive->reset_lock))
+
+ if (hive && !mutex_trylock(&hive->reset_lock)) {
+ DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress",
+ job->base.id, hive->hive_id);
return 0;
+ }
/* Start with adev pre asic reset first for soft reset check.*/
- amdgpu_device_lock_adev(adev);
- r = amdgpu_device_pre_asic_reset(adev,
- job,
- &need_full_reset);
- if (r) {
- /*TODO Should we stop ?*/
- DRM_ERROR("GPU pre asic reset failed with err, %d for drm dev, %s ",
- r, adev->ddev->unique);
- adev->asic_reset_res = r;
+ if (!amdgpu_device_lock_adev(adev, !hive)) {
+ DRM_INFO("Bailing on TDR for s_job:%llx, as another already in progress",
+ job->base.id);
+ return 0;
}
/* Build list of devices to reset */
- if (need_full_reset && adev->gmc.xgmi.num_physical_nodes > 1) {
+ if (adev->gmc.xgmi.num_physical_nodes > 1) {
if (!hive) {
amdgpu_device_unlock_adev(adev);
return -ENODEV;
@@ -3592,13 +3713,67 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
device_list_handle = &device_list;
}
+ /*
+ * Mark these ASICs to be reseted as untracked first
+ * And add them back after reset completed
+ */
+ list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head)
+ amdgpu_unregister_gpu_instance(tmp_adev);
+
+ /* block all schedulers and reset given job's ring */
+ list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
+ /* disable ras on ALL IPs */
+ if (amdgpu_device_ip_need_full_reset(tmp_adev))
+ amdgpu_ras_suspend(tmp_adev);
+
+ for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+ struct amdgpu_ring *ring = tmp_adev->rings[i];
+
+ if (!ring || !ring->sched.thread)
+ continue;
+
+ drm_sched_stop(&ring->sched, &job->base);
+ }
+ }
+
+
+ /*
+ * Must check guilty signal here since after this point all old
+ * HW fences are force signaled.
+ *
+ * job->base holds a reference to parent fence
+ */
+ if (job && job->base.s_fence->parent &&
+ dma_fence_is_signaled(job->base.s_fence->parent))
+ job_signaled = true;
+
+ if (!amdgpu_device_ip_need_full_reset(adev))
+ device_list_handle = &device_list;
+
+ if (job_signaled) {
+ dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
+ goto skip_hw_reset;
+ }
+
+
+ /* Guilty job will be freed after this*/
+ r = amdgpu_device_pre_asic_reset(adev,
+ job,
+ &need_full_reset);
+ if (r) {
+ /*TODO Should we stop ?*/
+ DRM_ERROR("GPU pre asic reset failed with err, %d for drm dev, %s ",
+ r, adev->ddev->unique);
+ adev->asic_reset_res = r;
+ }
+
retry: /* Rest of adevs pre asic reset from XGMI hive. */
list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
if (tmp_adev == adev)
continue;
- amdgpu_device_lock_adev(tmp_adev);
+ amdgpu_device_lock_adev(tmp_adev, false);
r = amdgpu_device_pre_asic_reset(tmp_adev,
NULL,
&need_full_reset);
@@ -3622,9 +3797,28 @@ retry: /* Rest of adevs pre asic reset from XGMI hive. */
goto retry;
}
+skip_hw_reset:
+
/* Post ASIC reset for all devs .*/
list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
- amdgpu_device_post_asic_reset(tmp_adev, tmp_adev == adev ? job : NULL);
+ for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
+ struct amdgpu_ring *ring = tmp_adev->rings[i];
+
+ if (!ring || !ring->sched.thread)
+ continue;
+
+ /* No point to resubmit jobs if we didn't HW reset*/
+ if (!tmp_adev->asic_reset_res && !job_signaled)
+ drm_sched_resubmit_jobs(&ring->sched);
+
+ drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res);
+ }
+
+ if (!amdgpu_device_has_dc_support(tmp_adev) && !job_signaled) {
+ drm_helper_resume_force_mode(tmp_adev->ddev);
+ }
+
+ tmp_adev->asic_reset_res = 0;
if (r) {
/* bad news, how to tell it to userspace ? */
@@ -3637,7 +3831,7 @@ retry: /* Rest of adevs pre asic reset from XGMI hive. */
amdgpu_device_unlock_adev(tmp_adev);
}
- if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
+ if (hive)
mutex_unlock(&hive->reset_lock);
if (r)
@@ -3645,43 +3839,6 @@ retry: /* Rest of adevs pre asic reset from XGMI hive. */
return r;
}
-static void amdgpu_device_get_min_pci_speed_width(struct amdgpu_device *adev,
- enum pci_bus_speed *speed,
- enum pcie_link_width *width)
-{
- struct pci_dev *pdev = adev->pdev;
- enum pci_bus_speed cur_speed;
- enum pcie_link_width cur_width;
- u32 ret = 1;
-
- *speed = PCI_SPEED_UNKNOWN;
- *width = PCIE_LNK_WIDTH_UNKNOWN;
-
- while (pdev) {
- cur_speed = pcie_get_speed_cap(pdev);
- cur_width = pcie_get_width_cap(pdev);
- ret = pcie_bandwidth_available(adev->pdev, NULL,
- NULL, &cur_width);
- if (!ret)
- cur_width = PCIE_LNK_WIDTH_RESRV;
-
- if (cur_speed != PCI_SPEED_UNKNOWN) {
- if (*speed == PCI_SPEED_UNKNOWN)
- *speed = cur_speed;
- else if (cur_speed < *speed)
- *speed = cur_speed;
- }
-
- if (cur_width != PCIE_LNK_WIDTH_UNKNOWN) {
- if (*width == PCIE_LNK_WIDTH_UNKNOWN)
- *width = cur_width;
- else if (cur_width < *width)
- *width = cur_width;
- }
- pdev = pci_upstream_bridge(pdev);
- }
-}
-
/**
* amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
*
@@ -3715,8 +3872,8 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
return;
- amdgpu_device_get_min_pci_speed_width(adev, &platform_speed_cap,
- &platform_link_width);
+ pcie_bandwidth_available(adev->pdev, NULL,
+ &platform_speed_cap, &platform_link_width);
if (adev->pm.pcie_gen_mask == 0) {
/* asic caps */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
new file mode 100644
index 000000000000..1481899f86c1
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
@@ -0,0 +1,415 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_discovery.h"
+#include "soc15_common.h"
+#include "soc15_hw_ip.h"
+#include "nbio/nbio_2_3_offset.h"
+#include "discovery.h"
+
+#define mmRCC_CONFIG_MEMSIZE 0xde3
+#define mmMM_INDEX 0x0
+#define mmMM_INDEX_HI 0x6
+#define mmMM_DATA 0x1
+#define HW_ID_MAX 300
+
+const char *hw_id_names[HW_ID_MAX] = {
+ [MP1_HWID] = "MP1",
+ [MP2_HWID] = "MP2",
+ [THM_HWID] = "THM",
+ [SMUIO_HWID] = "SMUIO",
+ [FUSE_HWID] = "FUSE",
+ [CLKA_HWID] = "CLKA",
+ [PWR_HWID] = "PWR",
+ [GC_HWID] = "GC",
+ [UVD_HWID] = "UVD",
+ [AUDIO_AZ_HWID] = "AUDIO_AZ",
+ [ACP_HWID] = "ACP",
+ [DCI_HWID] = "DCI",
+ [DMU_HWID] = "DMU",
+ [DCO_HWID] = "DCO",
+ [DIO_HWID] = "DIO",
+ [XDMA_HWID] = "XDMA",
+ [DCEAZ_HWID] = "DCEAZ",
+ [DAZ_HWID] = "DAZ",
+ [SDPMUX_HWID] = "SDPMUX",
+ [NTB_HWID] = "NTB",
+ [IOHC_HWID] = "IOHC",
+ [L2IMU_HWID] = "L2IMU",
+ [VCE_HWID] = "VCE",
+ [MMHUB_HWID] = "MMHUB",
+ [ATHUB_HWID] = "ATHUB",
+ [DBGU_NBIO_HWID] = "DBGU_NBIO",
+ [DFX_HWID] = "DFX",
+ [DBGU0_HWID] = "DBGU0",
+ [DBGU1_HWID] = "DBGU1",
+ [OSSSYS_HWID] = "OSSSYS",
+ [HDP_HWID] = "HDP",
+ [SDMA0_HWID] = "SDMA0",
+ [SDMA1_HWID] = "SDMA1",
+ [ISP_HWID] = "ISP",
+ [DBGU_IO_HWID] = "DBGU_IO",
+ [DF_HWID] = "DF",
+ [CLKB_HWID] = "CLKB",
+ [FCH_HWID] = "FCH",
+ [DFX_DAP_HWID] = "DFX_DAP",
+ [L1IMU_PCIE_HWID] = "L1IMU_PCIE",
+ [L1IMU_NBIF_HWID] = "L1IMU_NBIF",
+ [L1IMU_IOAGR_HWID] = "L1IMU_IOAGR",
+ [L1IMU3_HWID] = "L1IMU3",
+ [L1IMU4_HWID] = "L1IMU4",
+ [L1IMU5_HWID] = "L1IMU5",
+ [L1IMU6_HWID] = "L1IMU6",
+ [L1IMU7_HWID] = "L1IMU7",
+ [L1IMU8_HWID] = "L1IMU8",
+ [L1IMU9_HWID] = "L1IMU9",
+ [L1IMU10_HWID] = "L1IMU10",
+ [L1IMU11_HWID] = "L1IMU11",
+ [L1IMU12_HWID] = "L1IMU12",
+ [L1IMU13_HWID] = "L1IMU13",
+ [L1IMU14_HWID] = "L1IMU14",
+ [L1IMU15_HWID] = "L1IMU15",
+ [WAFLC_HWID] = "WAFLC",
+ [FCH_USB_PD_HWID] = "FCH_USB_PD",
+ [PCIE_HWID] = "PCIE",
+ [PCS_HWID] = "PCS",
+ [DDCL_HWID] = "DDCL",
+ [SST_HWID] = "SST",
+ [IOAGR_HWID] = "IOAGR",
+ [NBIF_HWID] = "NBIF",
+ [IOAPIC_HWID] = "IOAPIC",
+ [SYSTEMHUB_HWID] = "SYSTEMHUB",
+ [NTBCCP_HWID] = "NTBCCP",
+ [UMC_HWID] = "UMC",
+ [SATA_HWID] = "SATA",
+ [USB_HWID] = "USB",
+ [CCXSEC_HWID] = "CCXSEC",
+ [XGMI_HWID] = "XGMI",
+ [XGBE_HWID] = "XGBE",
+ [MP0_HWID] = "MP0",
+};
+
+static int hw_id_map[MAX_HWIP] = {
+ [GC_HWIP] = GC_HWID,
+ [HDP_HWIP] = HDP_HWID,
+ [SDMA0_HWIP] = SDMA0_HWID,
+ [SDMA1_HWIP] = SDMA1_HWID,
+ [MMHUB_HWIP] = MMHUB_HWID,
+ [ATHUB_HWIP] = ATHUB_HWID,
+ [NBIO_HWIP] = NBIF_HWID,
+ [MP0_HWIP] = MP0_HWID,
+ [MP1_HWIP] = MP1_HWID,
+ [UVD_HWIP] = UVD_HWID,
+ [VCE_HWIP] = VCE_HWID,
+ [DF_HWIP] = DF_HWID,
+ [DCE_HWIP] = DMU_HWID,
+ [OSSSYS_HWIP] = OSSSYS_HWID,
+ [SMUIO_HWIP] = SMUIO_HWID,
+ [PWR_HWIP] = PWR_HWID,
+ [NBIF_HWIP] = NBIF_HWID,
+ [THM_HWIP] = THM_HWID,
+ [CLK_HWIP] = CLKA_HWID,
+};
+
+static int amdgpu_discovery_read_binary(struct amdgpu_device *adev, uint8_t *binary)
+{
+ uint32_t *p = (uint32_t *)binary;
+ uint64_t vram_size = (uint64_t)RREG32(mmRCC_CONFIG_MEMSIZE) << 20;
+ uint64_t pos = vram_size - BINARY_MAX_SIZE;
+ unsigned long flags;
+
+ while (pos < vram_size) {
+ spin_lock_irqsave(&adev->mmio_idx_lock, flags);
+ WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
+ WREG32_NO_KIQ(mmMM_INDEX_HI, pos >> 31);
+ *p++ = RREG32_NO_KIQ(mmMM_DATA);
+ spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
+ pos += 4;
+ }
+
+ return 0;
+}
+
+static uint16_t amdgpu_discovery_calculate_checksum(uint8_t *data, uint32_t size)
+{
+ uint16_t checksum = 0;
+ int i;
+
+ for (i = 0; i < size; i++)
+ checksum += data[i];
+
+ return checksum;
+}
+
+static inline bool amdgpu_discovery_verify_checksum(uint8_t *data, uint32_t size,
+ uint16_t expected)
+{
+ return !!(amdgpu_discovery_calculate_checksum(data, size) == expected);
+}
+
+int amdgpu_discovery_init(struct amdgpu_device *adev)
+{
+ struct table_info *info;
+ struct binary_header *bhdr;
+ struct ip_discovery_header *ihdr;
+ struct gpu_info_header *ghdr;
+ uint16_t offset;
+ uint16_t size;
+ uint16_t checksum;
+ int r;
+
+ adev->discovery = kzalloc(BINARY_MAX_SIZE, GFP_KERNEL);
+ if (!adev->discovery)
+ return -ENOMEM;
+
+ r = amdgpu_discovery_read_binary(adev, adev->discovery);
+ if (r) {
+ DRM_ERROR("failed to read ip discovery binary\n");
+ goto out;
+ }
+
+ bhdr = (struct binary_header *)adev->discovery;
+
+ if (le32_to_cpu(bhdr->binary_signature) != BINARY_SIGNATURE) {
+ DRM_ERROR("invalid ip discovery binary signature\n");
+ r = -EINVAL;
+ goto out;
+ }
+
+ offset = offsetof(struct binary_header, binary_checksum) +
+ sizeof(bhdr->binary_checksum);
+ size = bhdr->binary_size - offset;
+ checksum = bhdr->binary_checksum;
+
+ if (!amdgpu_discovery_verify_checksum(adev->discovery + offset,
+ size, checksum)) {
+ DRM_ERROR("invalid ip discovery binary checksum\n");
+ r = -EINVAL;
+ goto out;
+ }
+
+ info = &bhdr->table_list[IP_DISCOVERY];
+ offset = le16_to_cpu(info->offset);
+ checksum = le16_to_cpu(info->checksum);
+ ihdr = (struct ip_discovery_header *)(adev->discovery + offset);
+
+ if (le32_to_cpu(ihdr->signature) != DISCOVERY_TABLE_SIGNATURE) {
+ DRM_ERROR("invalid ip discovery data table signature\n");
+ r = -EINVAL;
+ goto out;
+ }
+
+ if (!amdgpu_discovery_verify_checksum(adev->discovery + offset,
+ ihdr->size, checksum)) {
+ DRM_ERROR("invalid ip discovery data table checksum\n");
+ r = -EINVAL;
+ goto out;
+ }
+
+ info = &bhdr->table_list[GC];
+ offset = le16_to_cpu(info->offset);
+ checksum = le16_to_cpu(info->checksum);
+ ghdr = (struct gpu_info_header *)(adev->discovery + offset);
+
+ if (!amdgpu_discovery_verify_checksum(adev->discovery + offset,
+ ghdr->size, checksum)) {
+ DRM_ERROR("invalid gc data table checksum\n");
+ r = -EINVAL;
+ goto out;
+ }
+
+ return 0;
+
+out:
+ kfree(adev->discovery);
+ adev->discovery = NULL;
+
+ return r;
+}
+
+void amdgpu_discovery_fini(struct amdgpu_device *adev)
+{
+ kfree(adev->discovery);
+ adev->discovery = NULL;
+}
+
+int amdgpu_discovery_reg_base_init(struct amdgpu_device *adev)
+{
+ struct binary_header *bhdr;
+ struct ip_discovery_header *ihdr;
+ struct die_header *dhdr;
+ struct ip *ip;
+ uint16_t die_offset;
+ uint16_t ip_offset;
+ uint16_t num_dies;
+ uint16_t num_ips;
+ uint8_t num_base_address;
+ int hw_ip;
+ int i, j, k;
+
+ if (!adev->discovery) {
+ DRM_ERROR("ip discovery uninitialized\n");
+ return -EINVAL;
+ }
+
+ bhdr = (struct binary_header *)adev->discovery;
+ ihdr = (struct ip_discovery_header *)(adev->discovery +
+ le16_to_cpu(bhdr->table_list[IP_DISCOVERY].offset));
+ num_dies = le16_to_cpu(ihdr->num_dies);
+
+ DRM_DEBUG("number of dies: %d\n", num_dies);
+
+ for (i = 0; i < num_dies; i++) {
+ die_offset = le16_to_cpu(ihdr->die_info[i].die_offset);
+ dhdr = (struct die_header *)(adev->discovery + die_offset);
+ num_ips = le16_to_cpu(dhdr->num_ips);
+ ip_offset = die_offset + sizeof(*dhdr);
+
+ if (le16_to_cpu(dhdr->die_id) != i) {
+ DRM_ERROR("invalid die id %d, expected %d\n",
+ le16_to_cpu(dhdr->die_id), i);
+ return -EINVAL;
+ }
+
+ DRM_DEBUG("number of hardware IPs on die%d: %d\n",
+ le16_to_cpu(dhdr->die_id), num_ips);
+
+ for (j = 0; j < num_ips; j++) {
+ ip = (struct ip *)(adev->discovery + ip_offset);
+ num_base_address = ip->num_base_address;
+
+ DRM_DEBUG("%s(%d) #%d v%d.%d.%d:\n",
+ hw_id_names[le16_to_cpu(ip->hw_id)],
+ le16_to_cpu(ip->hw_id),
+ ip->number_instance,
+ ip->major, ip->minor,
+ ip->revision);
+
+ for (k = 0; k < num_base_address; k++) {
+ /*
+ * convert the endianness of base addresses in place,
+ * so that we don't need to convert them when accessing adev->reg_offset.
+ */
+ ip->base_address[k] = le32_to_cpu(ip->base_address[k]);
+ DRM_DEBUG("\t0x%08x\n", ip->base_address[k]);
+ }
+
+ for (hw_ip = 0; hw_ip < MAX_HWIP; hw_ip++) {
+ if (hw_id_map[hw_ip] == le16_to_cpu(ip->hw_id)) {
+ DRM_INFO("set register base offset for %s\n",
+ hw_id_names[le16_to_cpu(ip->hw_id)]);
+ adev->reg_offset[hw_ip][ip->number_instance] =
+ ip->base_address;
+ }
+
+ }
+
+ ip_offset += sizeof(*ip) + 4 * (ip->num_base_address - 1);
+ }
+ }
+
+ return 0;
+}
+
+int amdgpu_discovery_get_ip_version(struct amdgpu_device *adev, int hw_id,
+ int *major, int *minor)
+{
+ struct binary_header *bhdr;
+ struct ip_discovery_header *ihdr;
+ struct die_header *dhdr;
+ struct ip *ip;
+ uint16_t die_offset;
+ uint16_t ip_offset;
+ uint16_t num_dies;
+ uint16_t num_ips;
+ int i, j;
+
+ if (!adev->discovery) {
+ DRM_ERROR("ip discovery uninitialized\n");
+ return -EINVAL;
+ }
+
+ bhdr = (struct binary_header *)adev->discovery;
+ ihdr = (struct ip_discovery_header *)(adev->discovery +
+ le16_to_cpu(bhdr->table_list[IP_DISCOVERY].offset));
+ num_dies = le16_to_cpu(ihdr->num_dies);
+
+ for (i = 0; i < num_dies; i++) {
+ die_offset = le16_to_cpu(ihdr->die_info[i].die_offset);
+ dhdr = (struct die_header *)(adev->discovery + die_offset);
+ num_ips = le16_to_cpu(dhdr->num_ips);
+ ip_offset = die_offset + sizeof(*dhdr);
+
+ for (j = 0; j < num_ips; j++) {
+ ip = (struct ip *)(adev->discovery + ip_offset);
+
+ if (le16_to_cpu(ip->hw_id) == hw_id) {
+ if (major)
+ *major = ip->major;
+ if (minor)
+ *minor = ip->minor;
+ return 0;
+ }
+ ip_offset += sizeof(*ip) + 4 * (ip->num_base_address - 1);
+ }
+ }
+
+ return -EINVAL;
+}
+
+int amdgpu_discovery_get_gfx_info(struct amdgpu_device *adev)
+{
+ struct binary_header *bhdr;
+ struct gc_info_v1_0 *gc_info;
+
+ if (!adev->discovery) {
+ DRM_ERROR("ip discovery uninitialized\n");
+ return -EINVAL;
+ }
+
+ bhdr = (struct binary_header *)adev->discovery;
+ gc_info = (struct gc_info_v1_0 *)(adev->discovery +
+ le16_to_cpu(bhdr->table_list[GC].offset));
+
+ adev->gfx.config.max_shader_engines = le32_to_cpu(gc_info->gc_num_se);
+ adev->gfx.config.max_cu_per_sh = 2 * (le32_to_cpu(gc_info->gc_num_wgp0_per_sa) +
+ le32_to_cpu(gc_info->gc_num_wgp1_per_sa));
+ adev->gfx.config.max_sh_per_se = le32_to_cpu(gc_info->gc_num_sa_per_se);
+ adev->gfx.config.max_backends_per_se = le32_to_cpu(gc_info->gc_num_rb_per_se);
+ adev->gfx.config.max_texture_channel_caches = le32_to_cpu(gc_info->gc_num_gl2c);
+ adev->gfx.config.max_gprs = le32_to_cpu(gc_info->gc_num_gprs);
+ adev->gfx.config.max_gs_threads = le32_to_cpu(gc_info->gc_num_max_gs_thds);
+ adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gc_info->gc_gs_table_depth);
+ adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gc_info->gc_gsprim_buff_depth);
+ adev->gfx.config.double_offchip_lds_buf = le32_to_cpu(gc_info->gc_double_offchip_lds_buffer);
+ adev->gfx.cu_info.wave_front_size = le32_to_cpu(gc_info->gc_wave_size);
+ adev->gfx.cu_info.max_waves_per_simd = le32_to_cpu(gc_info->gc_max_waves_per_simd);
+ adev->gfx.cu_info.max_scratch_slots_per_cu = le32_to_cpu(gc_info->gc_max_scratch_slots_per_cu);
+ adev->gfx.cu_info.lds_size = le32_to_cpu(gc_info->gc_lds_size);
+ adev->gfx.config.num_sc_per_sh = le32_to_cpu(gc_info->gc_num_sc_per_se) /
+ le32_to_cpu(gc_info->gc_num_sa_per_se);
+ adev->gfx.config.num_packer_per_sc = le32_to_cpu(gc_info->gc_num_packer_per_sc);
+
+ return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
new file mode 100644
index 000000000000..85b8c4d4d576
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __AMDGPU_DISCOVERY__
+#define __AMDGPU_DISCOVERY__
+
+int amdgpu_discovery_init(struct amdgpu_device *adev);
+void amdgpu_discovery_fini(struct amdgpu_device *adev);
+int amdgpu_discovery_reg_base_init(struct amdgpu_device *adev);
+int amdgpu_discovery_get_ip_version(struct amdgpu_device *adev, int hw_id,
+ int *major, int *minor);
+int amdgpu_discovery_get_gfx_info(struct amdgpu_device *adev);
+
+#endif /* __AMDGPU_DISCOVERY__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
index b083b219b1a9..535650967b1a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
@@ -23,7 +23,7 @@
* Authors: Dave Airlie
* Alex Deucher
*/
-#include <drm/drmP.h>
+
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
#include "amdgpu_i2c.h"
@@ -32,11 +32,13 @@
#include "amdgpu_display.h"
#include <asm/div64.h>
+#include <linux/pci.h>
#include <linux/pm_runtime.h>
#include <drm/drm_crtc_helper.h>
#include <drm/drm_edid.h>
#include <drm/drm_gem_framebuffer_helper.h>
#include <drm/drm_fb_helper.h>
+#include <drm/drm_vblank.h>
static void amdgpu_display_flip_callback(struct dma_fence *f,
struct dma_fence_cb *cb)
@@ -631,10 +633,6 @@ int amdgpu_display_modeset_create_props(struct amdgpu_device *adev)
amdgpu_dither_enum_list, sz);
if (amdgpu_device_has_dc_support(adev)) {
- adev->mode_info.max_bpc_property =
- drm_property_create_range(adev->ddev, 0, "max bpc", 8, 16);
- if (!adev->mode_info.max_bpc_property)
- return -ENOMEM;
adev->mode_info.abm_level_property =
drm_property_create_range(adev->ddev, 0,
"abm level", 0, 4);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index a38e0fb4a6fe..489041df1f45 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -1,5 +1,5 @@
/*
- * Copyright 2012 Advanced Micro Devices, Inc.
+ * Copyright 2019 Advanced Micro Devices, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -31,8 +31,6 @@
* objects between different devices via PRIME <prime_buffer_sharing>`.
*/
-#include <drm/drmP.h>
-
#include "amdgpu.h"
#include "amdgpu_display.h"
#include "amdgpu_gem.h"
@@ -103,7 +101,8 @@ void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
* Returns:
* 0 on success or a negative error code on failure.
*/
-int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+int amdgpu_gem_prime_mmap(struct drm_gem_object *obj,
+ struct vm_area_struct *vma)
{
struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
@@ -137,57 +136,6 @@ int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma
return ret;
}
-/**
- * amdgpu_gem_prime_import_sg_table - &drm_driver.gem_prime_import_sg_table
- * implementation
- * @dev: DRM device
- * @attach: DMA-buf attachment
- * @sg: Scatter/gather table
- *
- * Imports shared DMA buffer memory exported by another device.
- *
- * Returns:
- * A new GEM BO of the given DRM device, representing the memory
- * described by the given DMA-buf attachment and scatter/gather table.
- */
-struct drm_gem_object *
-amdgpu_gem_prime_import_sg_table(struct drm_device *dev,
- struct dma_buf_attachment *attach,
- struct sg_table *sg)
-{
- struct reservation_object *resv = attach->dmabuf->resv;
- struct amdgpu_device *adev = dev->dev_private;
- struct amdgpu_bo *bo;
- struct amdgpu_bo_param bp;
- int ret;
-
- memset(&bp, 0, sizeof(bp));
- bp.size = attach->dmabuf->size;
- bp.byte_align = PAGE_SIZE;
- bp.domain = AMDGPU_GEM_DOMAIN_CPU;
- bp.flags = 0;
- bp.type = ttm_bo_type_sg;
- bp.resv = resv;
- ww_mutex_lock(&resv->lock, NULL);
- ret = amdgpu_bo_create(adev, &bp, &bo);
- if (ret)
- goto error;
-
- bo->tbo.sg = sg;
- bo->tbo.ttm->sg = sg;
- bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
- bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
- if (attach->dmabuf->ops != &amdgpu_dmabuf_ops)
- bo->prime_shared_count = 1;
-
- ww_mutex_unlock(&resv->lock);
- return &bo->gem_base;
-
-error:
- ww_mutex_unlock(&resv->lock);
- return ERR_PTR(ret);
-}
-
static int
__reservation_object_make_exclusive(struct reservation_object *obj)
{
@@ -231,7 +179,7 @@ err_fences_put:
}
/**
- * amdgpu_gem_map_attach - &dma_buf_ops.attach implementation
+ * amdgpu_dma_buf_map_attach - &dma_buf_ops.attach implementation
* @dma_buf: Shared DMA buffer
* @attach: DMA-buf attachment
*
@@ -242,8 +190,8 @@ err_fences_put:
* Returns:
* 0 on success or a negative error code on failure.
*/
-static int amdgpu_gem_map_attach(struct dma_buf *dma_buf,
- struct dma_buf_attachment *attach)
+static int amdgpu_dma_buf_map_attach(struct dma_buf *dma_buf,
+ struct dma_buf_attachment *attach)
{
struct drm_gem_object *obj = dma_buf->priv;
struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
@@ -291,15 +239,15 @@ error_detach:
}
/**
- * amdgpu_gem_map_detach - &dma_buf_ops.detach implementation
+ * amdgpu_dma_buf_map_detach - &dma_buf_ops.detach implementation
* @dma_buf: Shared DMA buffer
* @attach: DMA-buf attachment
*
* This is called when a shared DMA buffer no longer needs to be accessible by
* another device. For now, simply unpins the buffer from GTT.
*/
-static void amdgpu_gem_map_detach(struct dma_buf *dma_buf,
- struct dma_buf_attachment *attach)
+static void amdgpu_dma_buf_map_detach(struct dma_buf *dma_buf,
+ struct dma_buf_attachment *attach)
{
struct drm_gem_object *obj = dma_buf->priv;
struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
@@ -334,7 +282,7 @@ struct reservation_object *amdgpu_gem_prime_res_obj(struct drm_gem_object *obj)
}
/**
- * amdgpu_gem_begin_cpu_access - &dma_buf_ops.begin_cpu_access implementation
+ * amdgpu_dma_buf_begin_cpu_access - &dma_buf_ops.begin_cpu_access implementation
* @dma_buf: Shared DMA buffer
* @direction: Direction of DMA transfer
*
@@ -345,8 +293,8 @@ struct reservation_object *amdgpu_gem_prime_res_obj(struct drm_gem_object *obj)
* Returns:
* 0 on success or a negative error code on failure.
*/
-static int amdgpu_gem_begin_cpu_access(struct dma_buf *dma_buf,
- enum dma_data_direction direction)
+static int amdgpu_dma_buf_begin_cpu_access(struct dma_buf *dma_buf,
+ enum dma_data_direction direction)
{
struct amdgpu_bo *bo = gem_to_amdgpu_bo(dma_buf->priv);
struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
@@ -374,12 +322,12 @@ static int amdgpu_gem_begin_cpu_access(struct dma_buf *dma_buf,
}
const struct dma_buf_ops amdgpu_dmabuf_ops = {
- .attach = amdgpu_gem_map_attach,
- .detach = amdgpu_gem_map_detach,
+ .attach = amdgpu_dma_buf_map_attach,
+ .detach = amdgpu_dma_buf_map_detach,
.map_dma_buf = drm_gem_map_dma_buf,
.unmap_dma_buf = drm_gem_unmap_dma_buf,
.release = drm_gem_dmabuf_release,
- .begin_cpu_access = amdgpu_gem_begin_cpu_access,
+ .begin_cpu_access = amdgpu_dma_buf_begin_cpu_access,
.mmap = drm_gem_dmabuf_mmap,
.vmap = drm_gem_dmabuf_vmap,
.vunmap = drm_gem_dmabuf_vunmap,
@@ -418,6 +366,57 @@ struct dma_buf *amdgpu_gem_prime_export(struct drm_device *dev,
}
/**
+ * amdgpu_gem_prime_import_sg_table - &drm_driver.gem_prime_import_sg_table
+ * implementation
+ * @dev: DRM device
+ * @attach: DMA-buf attachment
+ * @sg: Scatter/gather table
+ *
+ * Imports shared DMA buffer memory exported by another device.
+ *
+ * Returns:
+ * A new GEM BO of the given DRM device, representing the memory
+ * described by the given DMA-buf attachment and scatter/gather table.
+ */
+struct drm_gem_object *
+amdgpu_gem_prime_import_sg_table(struct drm_device *dev,
+ struct dma_buf_attachment *attach,
+ struct sg_table *sg)
+{
+ struct reservation_object *resv = attach->dmabuf->resv;
+ struct amdgpu_device *adev = dev->dev_private;
+ struct amdgpu_bo *bo;
+ struct amdgpu_bo_param bp;
+ int ret;
+
+ memset(&bp, 0, sizeof(bp));
+ bp.size = attach->dmabuf->size;
+ bp.byte_align = PAGE_SIZE;
+ bp.domain = AMDGPU_GEM_DOMAIN_CPU;
+ bp.flags = 0;
+ bp.type = ttm_bo_type_sg;
+ bp.resv = resv;
+ ww_mutex_lock(&resv->lock, NULL);
+ ret = amdgpu_bo_create(adev, &bp, &bo);
+ if (ret)
+ goto error;
+
+ bo->tbo.sg = sg;
+ bo->tbo.ttm->sg = sg;
+ bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
+ bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
+ if (attach->dmabuf->ops != &amdgpu_dmabuf_ops)
+ bo->prime_shared_count = 1;
+
+ ww_mutex_unlock(&resv->lock);
+ return &bo->gem_base;
+
+error:
+ ww_mutex_unlock(&resv->lock);
+ return ERR_PTR(ret);
+}
+
+/**
* amdgpu_gem_prime_import - &drm_driver.gem_prime_import implementation
* @dev: DRM device
* @dma_buf: Shared DMA buffer
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.h
new file mode 100644
index 000000000000..c7056cbe8685
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __AMDGPU_DMA_BUF_H__
+#define __AMDGPU_DMA_BUF_H__
+
+#include <drm/drm_gem.h>
+
+struct sg_table *amdgpu_gem_prime_get_sg_table(struct drm_gem_object *obj);
+struct drm_gem_object *
+amdgpu_gem_prime_import_sg_table(struct drm_device *dev,
+ struct dma_buf_attachment *attach,
+ struct sg_table *sg);
+struct dma_buf *amdgpu_gem_prime_export(struct drm_device *dev,
+ struct drm_gem_object *gobj,
+ int flags);
+struct drm_gem_object *amdgpu_gem_prime_import(struct drm_device *dev,
+ struct dma_buf *dma_buf);
+struct reservation_object *amdgpu_gem_prime_res_obj(struct drm_gem_object *);
+void *amdgpu_gem_prime_vmap(struct drm_gem_object *obj);
+void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
+int amdgpu_gem_prime_mmap(struct drm_gem_object *obj,
+ struct vm_area_struct *vma);
+
+extern const struct dma_buf_ops amdgpu_dmabuf_ops;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
index 68959b923f89..790263dcc064 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
@@ -51,6 +51,7 @@ struct amdgpu_doorbell_index {
uint32_t userqueue_start;
uint32_t userqueue_end;
uint32_t gfx_ring0;
+ uint32_t gfx_ring1;
uint32_t sdma_engine[8];
uint32_t ih;
union {
@@ -153,6 +154,45 @@ typedef enum _AMDGPU_VEGA20_DOORBELL_ASSIGNMENT
AMDGPU_VEGA20_DOORBELL_INVALID = 0xFFFF
} AMDGPU_VEGA20_DOORBELL_ASSIGNMENT;
+typedef enum _AMDGPU_NAVI10_DOORBELL_ASSIGNMENT
+{
+ /* Compute + GFX: 0~255 */
+ AMDGPU_NAVI10_DOORBELL_KIQ = 0x000,
+ AMDGPU_NAVI10_DOORBELL_HIQ = 0x001,
+ AMDGPU_NAVI10_DOORBELL_DIQ = 0x002,
+ AMDGPU_NAVI10_DOORBELL_MEC_RING0 = 0x003,
+ AMDGPU_NAVI10_DOORBELL_MEC_RING1 = 0x004,
+ AMDGPU_NAVI10_DOORBELL_MEC_RING2 = 0x005,
+ AMDGPU_NAVI10_DOORBELL_MEC_RING3 = 0x006,
+ AMDGPU_NAVI10_DOORBELL_MEC_RING4 = 0x007,
+ AMDGPU_NAVI10_DOORBELL_MEC_RING5 = 0x008,
+ AMDGPU_NAVI10_DOORBELL_MEC_RING6 = 0x009,
+ AMDGPU_NAVI10_DOORBELL_MEC_RING7 = 0x00A,
+ AMDGPU_NAVI10_DOORBELL_USERQUEUE_START = 0x00B,
+ AMDGPU_NAVI10_DOORBELL_USERQUEUE_END = 0x08A,
+ AMDGPU_NAVI10_DOORBELL_GFX_RING0 = 0x08B,
+ AMDGPU_NAVI10_DOORBELL_GFX_RING1 = 0x08C,
+ /* SDMA:256~335*/
+ AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE0 = 0x100,
+ AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE1 = 0x10A,
+ /* IH: 376~391 */
+ AMDGPU_NAVI10_DOORBELL_IH = 0x178,
+ /* MMSCH: 392~407
+ * overlap the doorbell assignment with VCN as they are mutually exclusive
+ * VCE engine's doorbell is 32 bit and two VCE ring share one QWORD
+ */
+ AMDGPU_NAVI10_DOORBELL64_VCN0_1 = 0x188, /* lower 32 bits for VNC0 and upper 32 bits for VNC1 */
+ AMDGPU_NAVI10_DOORBELL64_VCN2_3 = 0x189,
+ AMDGPU_NAVI10_DOORBELL64_VCN4_5 = 0x18A,
+ AMDGPU_NAVI10_DOORBELL64_VCN6_7 = 0x18B,
+
+ AMDGPU_NAVI10_DOORBELL64_FIRST_NON_CP = AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE0,
+ AMDGPU_NAVI10_DOORBELL64_LAST_NON_CP = AMDGPU_NAVI10_DOORBELL64_VCN6_7,
+
+ AMDGPU_NAVI10_DOORBELL_MAX_ASSIGNMENT = 0x18F,
+ AMDGPU_NAVI10_DOORBELL_INVALID = 0xFFFF
+} AMDGPU_NAVI10_DOORBELL_ASSIGNMENT;
+
/*
* 64bit doorbell, offset are in QWORD, occupy 2KB doorbell space
*/
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
index 523b8ab6b04e..61bd10310604 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
@@ -22,7 +22,6 @@
* Authors: Alex Deucher
*/
-#include <drm/drmP.h>
#include "amdgpu.h"
#include "amdgpu_atombios.h"
#include "amdgpu_i2c.h"
@@ -907,16 +906,63 @@ amdgpu_get_vce_clock_state(void *handle, u32 idx)
int amdgpu_dpm_get_sclk(struct amdgpu_device *adev, bool low)
{
- if (is_support_sw_smu(adev))
- return smu_get_sclk(&adev->smu, low);
- else
+ uint32_t clk_freq;
+ int ret = 0;
+ if (is_support_sw_smu(adev)) {
+ ret = smu_get_dpm_freq_range(&adev->smu, SMU_GFXCLK,
+ low ? &clk_freq : NULL,
+ !low ? &clk_freq : NULL);
+ if (ret)
+ return 0;
+ return clk_freq * 100;
+
+ } else {
return (adev)->powerplay.pp_funcs->get_sclk((adev)->powerplay.pp_handle, (low));
+ }
}
int amdgpu_dpm_get_mclk(struct amdgpu_device *adev, bool low)
{
- if (is_support_sw_smu(adev))
- return smu_get_mclk(&adev->smu, low);
- else
+ uint32_t clk_freq;
+ int ret = 0;
+ if (is_support_sw_smu(adev)) {
+ ret = smu_get_dpm_freq_range(&adev->smu, SMU_UCLK,
+ low ? &clk_freq : NULL,
+ !low ? &clk_freq : NULL);
+ if (ret)
+ return 0;
+ return clk_freq * 100;
+
+ } else {
return (adev)->powerplay.pp_funcs->get_mclk((adev)->powerplay.pp_handle, (low));
+ }
+}
+
+int amdgpu_dpm_set_powergating_by_smu(struct amdgpu_device *adev, uint32_t block_type, bool gate)
+{
+ int ret = 0;
+ bool swsmu = is_support_sw_smu(adev);
+
+ switch (block_type) {
+ case AMD_IP_BLOCK_TYPE_GFX:
+ case AMD_IP_BLOCK_TYPE_UVD:
+ case AMD_IP_BLOCK_TYPE_VCN:
+ case AMD_IP_BLOCK_TYPE_VCE:
+ if (swsmu)
+ ret = smu_dpm_set_power_gate(&adev->smu, block_type, gate);
+ else
+ ret = ((adev)->powerplay.pp_funcs->set_powergating_by_smu(
+ (adev)->powerplay.pp_handle, block_type, gate));
+ break;
+ case AMD_IP_BLOCK_TYPE_GMC:
+ case AMD_IP_BLOCK_TYPE_ACP:
+ case AMD_IP_BLOCK_TYPE_SDMA:
+ ret = ((adev)->powerplay.pp_funcs->set_powergating_by_smu(
+ (adev)->powerplay.pp_handle, block_type, gate));
+ break;
+ default:
+ break;
+ }
+
+ return ret;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
index dca35407879d..1c5c0fd76dbf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
@@ -75,6 +75,20 @@ struct amdgpu_dpm_thermal {
int min_temp;
/* high temperature threshold */
int max_temp;
+ /* edge max emergency(shutdown) temp */
+ int max_edge_emergency_temp;
+ /* hotspot low temperature threshold */
+ int min_hotspot_temp;
+ /* hotspot high temperature critical threshold */
+ int max_hotspot_crit_temp;
+ /* hotspot max emergency(shutdown) temp */
+ int max_hotspot_emergency_temp;
+ /* memory low temperature threshold */
+ int min_mem_temp;
+ /* memory high temperature critical threshold */
+ int max_mem_crit_temp;
+ /* memory max emergency(shutdown) temp */
+ int max_mem_emergency_temp;
/* was last interrupt low to high or high to low */
bool high_to_low;
/* interrupt source */
@@ -341,10 +355,6 @@ enum amdgpu_pcie_gen {
((adev)->powerplay.pp_funcs->set_clockgating_by_smu(\
(adev)->powerplay.pp_handle, msg_id))
-#define amdgpu_dpm_set_powergating_by_smu(adev, block_type, gate) \
- ((adev)->powerplay.pp_funcs->set_powergating_by_smu(\
- (adev)->powerplay.pp_handle, block_type, gate))
-
#define amdgpu_dpm_get_power_profile_mode(adev, buf) \
((adev)->powerplay.pp_funcs->get_power_profile_mode(\
(adev)->powerplay.pp_handle, buf))
@@ -506,6 +516,9 @@ enum amdgpu_pcie_gen amdgpu_get_pcie_gen_support(struct amdgpu_device *adev,
struct amd_vce_state*
amdgpu_get_vce_clock_state(void *handle, u32 idx);
+int amdgpu_dpm_set_powergating_by_smu(struct amdgpu_device *adev,
+ uint32_t block_type, bool gate);
+
extern int amdgpu_dpm_get_sclk(struct amdgpu_device *adev, bool low);
extern int amdgpu_dpm_get_mclk(struct amdgpu_device *adev, bool low);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 1e2cc9d68a05..5376328d3fd0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -22,21 +22,23 @@
* OTHER DEALINGS IN THE SOFTWARE.
*/
-#include <drm/drmP.h>
#include <drm/amdgpu_drm.h>
+#include <drm/drm_drv.h>
#include <drm/drm_gem.h>
+#include <drm/drm_vblank.h>
#include "amdgpu_drv.h"
#include <drm/drm_pciids.h>
#include <linux/console.h>
#include <linux/module.h>
+#include <linux/pci.h>
#include <linux/pm_runtime.h>
#include <linux/vga_switcheroo.h>
#include <drm/drm_probe_helper.h>
#include "amdgpu.h"
#include "amdgpu_irq.h"
-#include "amdgpu_gem.h"
+#include "amdgpu_dma_buf.h"
#include "amdgpu_amdkfd.h"
@@ -76,11 +78,14 @@
* - 3.30.0 - Add AMDGPU_SCHED_OP_CONTEXT_PRIORITY_OVERRIDE.
* - 3.31.0 - Add support for per-flip tiling attribute changes with DC
* - 3.32.0 - Add syncobj timeline support to AMDGPU_CS.
+ * - 3.33.0 - Fixes for GDS ENOMEM failures in AMDGPU_CS.
*/
#define KMS_DRIVER_MAJOR 3
-#define KMS_DRIVER_MINOR 32
+#define KMS_DRIVER_MINOR 33
#define KMS_DRIVER_PATCHLEVEL 0
+#define AMDGPU_MAX_TIMEOUT_PARAM_LENTH 256
+
int amdgpu_vram_limit = 0;
int amdgpu_vis_vram_limit = 0;
int amdgpu_gart_size = -1; /* auto */
@@ -93,7 +98,7 @@ int amdgpu_disp_priority = 0;
int amdgpu_hw_i2c = 0;
int amdgpu_pcie_gen2 = -1;
int amdgpu_msi = -1;
-int amdgpu_lockup_timeout = 10000;
+char amdgpu_lockup_timeout[AMDGPU_MAX_TIMEOUT_PARAM_LENTH];
int amdgpu_dpm = -1;
int amdgpu_fw_load_type = -1;
int amdgpu_aspm = -1;
@@ -106,7 +111,6 @@ int amdgpu_vm_fragment_size = -1;
int amdgpu_vm_block_size = -1;
int amdgpu_vm_fault_stop = 0;
int amdgpu_vm_debug = 0;
-int amdgpu_vram_page_split = 512;
int amdgpu_vm_update_mode = -1;
int amdgpu_exp_hw_support = 0;
int amdgpu_dc = -1;
@@ -134,12 +138,17 @@ int amdgpu_emu_mode = 0;
uint amdgpu_smu_memory_pool_size = 0;
/* FBC (bit 0) disabled by default*/
uint amdgpu_dc_feature_mask = 0;
+int amdgpu_async_gfx_ring = 1;
+int amdgpu_mcbp = 0;
+int amdgpu_discovery = -1;
+int amdgpu_mes = 0;
+int amdgpu_noretry;
struct amdgpu_mgpu_info mgpu_info = {
.mutex = __MUTEX_INITIALIZER(mgpu_info.mutex),
};
int amdgpu_ras_enable = -1;
-uint amdgpu_ras_mask = 0xffffffff;
+uint amdgpu_ras_mask = 0xfffffffb;
/**
* DOC: vramlimit (int)
@@ -227,16 +236,28 @@ MODULE_PARM_DESC(msi, "MSI support (1 = enable, 0 = disable, -1 = auto)");
module_param_named(msi, amdgpu_msi, int, 0444);
/**
- * DOC: lockup_timeout (int)
- * Set GPU scheduler timeout value in ms. Value 0 is invalidated, will be adjusted to 10000.
- * Negative values mean 'infinite timeout' (MAX_JIFFY_OFFSET). The default is 10000.
- */
-MODULE_PARM_DESC(lockup_timeout, "GPU lockup timeout in ms > 0 (default 10000)");
-module_param_named(lockup_timeout, amdgpu_lockup_timeout, int, 0444);
+ * DOC: lockup_timeout (string)
+ * Set GPU scheduler timeout value in ms.
+ *
+ * The format can be [Non-Compute] or [GFX,Compute,SDMA,Video]. That is there can be one or
+ * multiple values specified. 0 and negative values are invalidated. They will be adjusted
+ * to default timeout.
+ * - With one value specified, the setting will apply to all non-compute jobs.
+ * - With multiple values specified, the first one will be for GFX. The second one is for Compute.
+ * And the third and fourth ones are for SDMA and Video.
+ * By default(with no lockup_timeout settings), the timeout for all non-compute(GFX, SDMA and Video)
+ * jobs is 10000. And there is no timeout enforced on compute jobs.
+ */
+MODULE_PARM_DESC(lockup_timeout, "GPU lockup timeout in ms (default: 10000 for non-compute jobs and infinity timeout for compute jobs."
+ " 0: keep default value. negative: infinity timeout), "
+ "format is [Non-Compute] or [GFX,Compute,SDMA,Video]");
+module_param_string(lockup_timeout, amdgpu_lockup_timeout, sizeof(amdgpu_lockup_timeout), 0444);
/**
* DOC: dpm (int)
- * Override for dynamic power management setting (1 = enable, 0 = disable). The default is -1 (auto).
+ * Override for dynamic power management setting
+ * (0 = disable, 1 = enable, 2 = enable sw smu driver for vega20)
+ * The default is -1 (auto).
*/
MODULE_PARM_DESC(dpm, "DPM support (1 = enable, 0 = disable, -1 = auto)");
module_param_named(dpm, amdgpu_dpm, int, 0444);
@@ -332,13 +353,6 @@ MODULE_PARM_DESC(vm_update_mode, "VM update using CPU (0 = never (default except
module_param_named(vm_update_mode, amdgpu_vm_update_mode, int, 0444);
/**
- * DOC: vram_page_split (int)
- * Override the number of pages after we split VRAM allocations (default 512, -1 = disable). The default is 512.
- */
-MODULE_PARM_DESC(vram_page_split, "Number of pages after we split VRAM allocations (default 512, -1 = disable)");
-module_param_named(vram_page_split, amdgpu_vram_page_split, int, 0444);
-
-/**
* DOC: exp_hw_support (int)
* Enable experimental hw support (1 = enable). The default is 0 (disabled).
*/
@@ -561,6 +575,44 @@ MODULE_PARM_DESC(smu_memory_pool_size,
"0x1 = 256Mbyte, 0x2 = 512Mbyte, 0x4 = 1 Gbyte, 0x8 = 2GByte");
module_param_named(smu_memory_pool_size, amdgpu_smu_memory_pool_size, uint, 0444);
+/**
+ * DOC: async_gfx_ring (int)
+ * It is used to enable gfx rings that could be configured with different prioritites or equal priorities
+ */
+MODULE_PARM_DESC(async_gfx_ring,
+ "Asynchronous GFX rings that could be configured with either different priorities (HP3D ring and LP3D ring), or equal priorities (0 = disabled, 1 = enabled (default))");
+module_param_named(async_gfx_ring, amdgpu_async_gfx_ring, int, 0444);
+
+/**
+ * DOC: mcbp (int)
+ * It is used to enable mid command buffer preemption. (0 = disabled (default), 1 = enabled)
+ */
+MODULE_PARM_DESC(mcbp,
+ "Enable Mid-command buffer preemption (0 = disabled (default), 1 = enabled)");
+module_param_named(mcbp, amdgpu_mcbp, int, 0444);
+
+/**
+ * DOC: discovery (int)
+ * Allow driver to discover hardware IP information from IP Discovery table at the top of VRAM.
+ * (-1 = auto (default), 0 = disabled, 1 = enabled)
+ */
+MODULE_PARM_DESC(discovery,
+ "Allow driver to discover hardware IPs from IP Discovery table at the top of VRAM");
+module_param_named(discovery, amdgpu_discovery, int, 0444);
+
+/**
+ * DOC: mes (int)
+ * Enable Micro Engine Scheduler. This is a new hw scheduling engine for gfx, sdma, and compute.
+ * (0 = disabled (default), 1 = enabled)
+ */
+MODULE_PARM_DESC(mes,
+ "Enable Micro Engine Scheduler (0 = disabled (default), 1 = enabled)");
+module_param_named(mes, amdgpu_mes, int, 0444);
+
+MODULE_PARM_DESC(noretry,
+ "Disable retry faults (0 = retry enabled (default), 1 = retry disabled)");
+module_param_named(noretry, amdgpu_noretry, int, 0644);
+
#ifdef CONFIG_HSA_AMD
/**
* DOC: sched_policy (int)
@@ -637,17 +689,6 @@ MODULE_PARM_DESC(ignore_crat,
"Ignore CRAT table during KFD initialization (0 = use CRAT (default), 1 = ignore CRAT)");
/**
- * DOC: noretry (int)
- * This parameter sets sh_mem_config.retry_disable. Default value, 0, enables retry.
- * Setting 1 disables retry.
- * Retry is needed for recoverable page faults.
- */
-int noretry;
-module_param(noretry, int, 0644);
-MODULE_PARM_DESC(noretry,
- "Set sh_mem_config.retry_disable on Vega10 (0 = retry enabled (default), 1 = retry disabled)");
-
-/**
* DOC: halt_if_hws_hang (int)
* Halt if HWS hang is detected. Default value, 0, disables the halt on hang.
* Setting 1 enables halt on hang.
@@ -655,6 +696,24 @@ MODULE_PARM_DESC(noretry,
int halt_if_hws_hang;
module_param(halt_if_hws_hang, int, 0644);
MODULE_PARM_DESC(halt_if_hws_hang, "Halt if HWS hang is detected (0 = off (default), 1 = on)");
+
+/**
+ * DOC: hws_gws_support(bool)
+ * Whether HWS support gws barriers. Default value: false (not supported)
+ * This will be replaced with a MEC firmware version check once firmware
+ * is ready
+ */
+bool hws_gws_support;
+module_param(hws_gws_support, bool, 0444);
+MODULE_PARM_DESC(hws_gws_support, "MEC FW support gws barriers (false = not supported (Default), true = supported)");
+
+/**
+ * DOC: queue_preemption_timeout_ms (int)
+ * queue preemption timeout in ms (1 = Minimum, 9000 = default)
+ */
+int queue_preemption_timeout_ms = 9000;
+module_param(queue_preemption_timeout_ms, int, 0644);
+MODULE_PARM_DESC(queue_preemption_timeout_ms, "queue preemption timeout in ms (1 = Minimum, 9000 = default)");
#endif
/**
@@ -665,6 +724,22 @@ MODULE_PARM_DESC(halt_if_hws_hang, "Halt if HWS hang is detected (0 = off (defau
MODULE_PARM_DESC(dcfeaturemask, "all stable DC features enabled (default))");
module_param_named(dcfeaturemask, amdgpu_dc_feature_mask, uint, 0444);
+/**
+ * DOC: abmlevel (uint)
+ * Override the default ABM (Adaptive Backlight Management) level used for DC
+ * enabled hardware. Requires DMCU to be supported and loaded.
+ * Valid levels are 0-4. A value of 0 indicates that ABM should be disabled by
+ * default. Values 1-4 control the maximum allowable brightness reduction via
+ * the ABM algorithm, with 1 being the least reduction and 4 being the most
+ * reduction.
+ *
+ * Defaults to 0, or disabled. Userspace can still override this level later
+ * after boot.
+ */
+uint amdgpu_dm_abm_level = 0;
+MODULE_PARM_DESC(abmlevel, "ABM level (0 = off (default), 1-4 = backlight reduction level) ");
+module_param_named(abmlevel, amdgpu_dm_abm_level, uint, 0444);
+
static const struct pci_device_id pciidlist[] = {
#ifdef CONFIG_DRM_AMDGPU_SI
{0x1002, 0x6780, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
@@ -921,6 +996,14 @@ static const struct pci_device_id pciidlist[] = {
/* Raven */
{0x1002, 0x15dd, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RAVEN|AMD_IS_APU},
{0x1002, 0x15d8, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RAVEN|AMD_IS_APU},
+ /* Navi10 */
+ {0x1002, 0x7310, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVI10},
+ {0x1002, 0x7312, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVI10},
+ {0x1002, 0x7318, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVI10},
+ {0x1002, 0x7319, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVI10},
+ {0x1002, 0x731A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVI10},
+ {0x1002, 0x731B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVI10},
+ {0x1002, 0x731F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVI10},
{0, 0, 0}
};
@@ -1216,6 +1299,66 @@ int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv)
return 0;
}
+int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
+{
+ char *input = amdgpu_lockup_timeout;
+ char *timeout_setting = NULL;
+ int index = 0;
+ long timeout;
+ int ret = 0;
+
+ /*
+ * By default timeout for non compute jobs is 10000.
+ * And there is no timeout enforced on compute jobs.
+ */
+ adev->gfx_timeout = msecs_to_jiffies(10000);
+ adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
+ adev->compute_timeout = MAX_SCHEDULE_TIMEOUT;
+
+ if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENTH)) {
+ while ((timeout_setting = strsep(&input, ",")) &&
+ strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENTH)) {
+ ret = kstrtol(timeout_setting, 0, &timeout);
+ if (ret)
+ return ret;
+
+ if (timeout == 0) {
+ index++;
+ continue;
+ } else if (timeout < 0) {
+ timeout = MAX_SCHEDULE_TIMEOUT;
+ } else {
+ timeout = msecs_to_jiffies(timeout);
+ }
+
+ switch (index++) {
+ case 0:
+ adev->gfx_timeout = timeout;
+ break;
+ case 1:
+ adev->compute_timeout = timeout;
+ break;
+ case 2:
+ adev->sdma_timeout = timeout;
+ break;
+ case 3:
+ adev->video_timeout = timeout;
+ break;
+ default:
+ break;
+ }
+ }
+ /*
+ * There is only one value specified and
+ * it should apply to all non-compute jobs.
+ */
+ if (index == 1)
+ adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
+ }
+
+ return ret;
+}
+
static bool
amdgpu_get_crtc_scanout_position(struct drm_device *dev, unsigned int pipe,
bool in_vblank_irq, int *vpos, int *hpos,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_encoders.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_encoders.c
index ec78e2b2015c..571a6dfb473e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_encoders.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_encoders.c
@@ -23,7 +23,7 @@
* Authors: Dave Airlie
* Alex Deucher
*/
-#include <drm/drmP.h>
+
#include <drm/drm_crtc_helper.h>
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
index e47609218839..eb3569b46c1e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
@@ -23,22 +23,22 @@
* Authors:
* David Airlie
*/
+
#include <linux/module.h>
-#include <linux/slab.h>
#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+#include <linux/vga_switcheroo.h>
-#include <drm/drmP.h>
+#include <drm/amdgpu_drm.h>
#include <drm/drm_crtc.h>
#include <drm/drm_crtc_helper.h>
-#include <drm/amdgpu_drm.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_fourcc.h>
+
#include "amdgpu.h"
#include "cikd.h"
#include "amdgpu_gem.h"
-#include <drm/drm_fb_helper.h>
-
-#include <linux/vga_switcheroo.h>
-
#include "amdgpu_display.h"
/* object hierarchy -
@@ -121,6 +121,7 @@ static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev,
struct drm_mode_fb_cmd2 *mode_cmd,
struct drm_gem_object **gobj_p)
{
+ const struct drm_format_info *info;
struct amdgpu_device *adev = rfbdev->adev;
struct drm_gem_object *gobj = NULL;
struct amdgpu_bo *abo = NULL;
@@ -131,7 +132,8 @@ static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev,
int height = mode_cmd->height;
u32 cpp;
- cpp = drm_format_plane_cpp(mode_cmd->pixel_format, 0);
+ info = drm_get_format_info(adev->ddev, mode_cmd);
+ cpp = info->cpp[0];
/* need to align pitch with crtc limits */
mode_cmd->pitches[0] = amdgpu_align_pitch(adev, mode_cmd->width, cpp,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index 4dee2326b29c..23085b352cf2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -34,7 +34,9 @@
#include <linux/kref.h>
#include <linux/slab.h>
#include <linux/firmware.h>
-#include <drm/drmP.h>
+
+#include <drm/drm_debugfs.h>
+
#include "amdgpu.h"
#include "amdgpu_trace.h"
@@ -427,9 +429,13 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,
unsigned num_hw_submission)
{
+ struct amdgpu_device *adev = ring->adev;
long timeout;
int r;
+ if (!adev)
+ return -EINVAL;
+
/* Check that num_hw_submission is a power of two */
if ((num_hw_submission & (num_hw_submission - 1)) != 0)
return -EINVAL;
@@ -451,12 +457,31 @@ int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,
/* No need to setup the GPU scheduler for KIQ ring */
if (ring->funcs->type != AMDGPU_RING_TYPE_KIQ) {
- /* for non-sriov case, no timeout enforce on compute ring */
- if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
- && !amdgpu_sriov_vf(ring->adev))
- timeout = MAX_SCHEDULE_TIMEOUT;
- else
- timeout = msecs_to_jiffies(amdgpu_lockup_timeout);
+ switch (ring->funcs->type) {
+ case AMDGPU_RING_TYPE_GFX:
+ timeout = adev->gfx_timeout;
+ break;
+ case AMDGPU_RING_TYPE_COMPUTE:
+ /*
+ * For non-sriov case, no timeout enforce
+ * on compute ring by default. Unless user
+ * specifies a timeout for compute ring.
+ *
+ * For sriov case, always use the timeout
+ * as gfx ring
+ */
+ if (!amdgpu_sriov_vf(ring->adev))
+ timeout = adev->compute_timeout;
+ else
+ timeout = adev->gfx_timeout;
+ break;
+ case AMDGPU_RING_TYPE_SDMA:
+ timeout = adev->sdma_timeout;
+ break;
+ default:
+ timeout = adev->video_timeout;
+ break;
+ }
r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,
num_hw_submission, amdgpu_job_hang_limit,
@@ -684,22 +709,30 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data)
amdgpu_fence_process(ring);
seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name);
- seq_printf(m, "Last signaled fence 0x%08x\n",
+ seq_printf(m, "Last signaled fence 0x%08x\n",
atomic_read(&ring->fence_drv.last_seq));
- seq_printf(m, "Last emitted 0x%08x\n",
+ seq_printf(m, "Last emitted 0x%08x\n",
ring->fence_drv.sync_seq);
+ if (ring->funcs->type == AMDGPU_RING_TYPE_GFX ||
+ ring->funcs->type == AMDGPU_RING_TYPE_SDMA) {
+ seq_printf(m, "Last signaled trailing fence 0x%08x\n",
+ le32_to_cpu(*ring->trail_fence_cpu_addr));
+ seq_printf(m, "Last emitted 0x%08x\n",
+ ring->trail_seq);
+ }
+
if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
continue;
/* set in CP_VMID_PREEMPT and preemption occurred */
- seq_printf(m, "Last preempted 0x%08x\n",
+ seq_printf(m, "Last preempted 0x%08x\n",
le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));
/* set in CP_VMID_RESET and reset occurred */
- seq_printf(m, "Last reset 0x%08x\n",
+ seq_printf(m, "Last reset 0x%08x\n",
le32_to_cpu(*(ring->fence_drv.cpu_addr + 4)));
/* Both preemption and reset occurred */
- seq_printf(m, "Last both 0x%08x\n",
+ seq_printf(m, "Last both 0x%08x\n",
le32_to_cpu(*(ring->fence_drv.cpu_addr + 6)));
}
return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
index 6d11e1721147..d79ab1da9e07 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
@@ -25,7 +25,10 @@
* Alex Deucher
* Jerome Glisse
*/
-#include <drm/drmP.h>
+
+#include <linux/pci.h>
+#include <linux/vmalloc.h>
+
#include <drm/amdgpu_drm.h>
#ifdef CONFIG_X86
#include <asm/set_memory.h>
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h
index f89f5734d985..f6ac1e9548f2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h
@@ -27,26 +27,11 @@
struct amdgpu_ring;
struct amdgpu_bo;
-struct amdgpu_gds_asic_info {
- uint32_t total_size;
- uint32_t gfx_partition_size;
- uint32_t cs_partition_size;
-};
-
struct amdgpu_gds {
- struct amdgpu_gds_asic_info mem;
- struct amdgpu_gds_asic_info gws;
- struct amdgpu_gds_asic_info oa;
- uint32_t gds_compute_max_wave_id;
-
- /* At present, GDS, GWS and OA resources for gfx (graphics)
- * is always pre-allocated and available for graphics operation.
- * Such resource is shared between all gfx clients.
- * TODO: move this operation to user space
- * */
- struct amdgpu_bo* gds_gfx_bo;
- struct amdgpu_bo* gws_gfx_bo;
- struct amdgpu_bo* oa_gfx_bo;
+ uint32_t gds_size;
+ uint32_t gws_size;
+ uint32_t oa_size;
+ uint32_t gds_compute_max_wave_id;
};
struct amdgpu_gds_reg_offset {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index d4fcf5475464..939f8305511b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -26,9 +26,13 @@
* Jerome Glisse
*/
#include <linux/ktime.h>
+#include <linux/module.h>
#include <linux/pagemap.h>
-#include <drm/drmP.h>
+#include <linux/pci.h>
+
#include <drm/amdgpu_drm.h>
+#include <drm/drm_debugfs.h>
+
#include "amdgpu.h"
#include "amdgpu_display.h"
#include "amdgpu_xgmi.h"
@@ -171,7 +175,7 @@ void amdgpu_gem_object_close(struct drm_gem_object *obj,
amdgpu_vm_get_pd_bo(vm, &list, &vm_pd);
- r = ttm_eu_reserve_buffers(&ticket, &list, false, &duplicates);
+ r = ttm_eu_reserve_buffers(&ticket, &list, false, &duplicates, false);
if (r) {
dev_err(adev->dev, "leaking bo va because "
"we fail to reserve bo (%d)\n", r);
@@ -323,33 +327,30 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
}
if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) {
- r = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm,
- bo->tbo.ttm->pages);
+ r = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages);
if (r)
goto release_object;
r = amdgpu_bo_reserve(bo, true);
if (r)
- goto free_pages;
+ goto user_pages_done;
amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
amdgpu_bo_unreserve(bo);
if (r)
- goto free_pages;
+ goto user_pages_done;
}
r = drm_gem_handle_create(filp, gobj, &handle);
- /* drop reference from allocate - handle holds it now */
- drm_gem_object_put_unlocked(gobj);
if (r)
- return r;
+ goto user_pages_done;
args->handle = handle;
- return 0;
-free_pages:
- release_pages(bo->tbo.ttm->pages, bo->tbo.ttm->num_pages);
+user_pages_done:
+ if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE)
+ amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
release_object:
drm_gem_object_put_unlocked(gobj);
@@ -610,7 +611,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
amdgpu_vm_get_pd_bo(&fpriv->vm, &list, &vm_pd);
- r = ttm_eu_reserve_buffers(&ticket, &list, true, &duplicates);
+ r = ttm_eu_reserve_buffers(&ticket, &list, true, &duplicates, false);
if (r)
goto error_unref;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h
index f1ddfc50bcc7..b8ba6e27c61f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h
@@ -39,22 +39,6 @@ int amdgpu_gem_object_open(struct drm_gem_object *obj,
void amdgpu_gem_object_close(struct drm_gem_object *obj,
struct drm_file *file_priv);
unsigned long amdgpu_gem_timeout(uint64_t timeout_ns);
-struct sg_table *amdgpu_gem_prime_get_sg_table(struct drm_gem_object *obj);
-struct drm_gem_object *
-amdgpu_gem_prime_import_sg_table(struct drm_device *dev,
- struct dma_buf_attachment *attach,
- struct sg_table *sg);
-struct dma_buf *amdgpu_gem_prime_export(struct drm_device *dev,
- struct drm_gem_object *gobj,
- int flags);
-struct drm_gem_object *amdgpu_gem_prime_import(struct drm_device *dev,
- struct dma_buf *dma_buf);
-struct reservation_object *amdgpu_gem_prime_res_obj(struct drm_gem_object *);
-void *amdgpu_gem_prime_vmap(struct drm_gem_object *obj);
-void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
-int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
-
-extern const struct dma_buf_ops amdgpu_dmabuf_ops;
/*
* GEM objects.
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 997932ebbb83..74066e1466f7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -22,7 +22,7 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_gfx.h"
#include "amdgpu_rlc.h"
@@ -34,8 +34,8 @@
* GPU GFX IP block helpers function.
*/
-int amdgpu_gfx_queue_to_bit(struct amdgpu_device *adev, int mec,
- int pipe, int queue)
+int amdgpu_gfx_mec_queue_to_bit(struct amdgpu_device *adev, int mec,
+ int pipe, int queue)
{
int bit = 0;
@@ -47,8 +47,8 @@ int amdgpu_gfx_queue_to_bit(struct amdgpu_device *adev, int mec,
return bit;
}
-void amdgpu_gfx_bit_to_queue(struct amdgpu_device *adev, int bit,
- int *mec, int *pipe, int *queue)
+void amdgpu_gfx_bit_to_mec_queue(struct amdgpu_device *adev, int bit,
+ int *mec, int *pipe, int *queue)
{
*queue = bit % adev->gfx.mec.num_queue_per_pipe;
*pipe = (bit / adev->gfx.mec.num_queue_per_pipe)
@@ -61,10 +61,40 @@ void amdgpu_gfx_bit_to_queue(struct amdgpu_device *adev, int bit,
bool amdgpu_gfx_is_mec_queue_enabled(struct amdgpu_device *adev,
int mec, int pipe, int queue)
{
- return test_bit(amdgpu_gfx_queue_to_bit(adev, mec, pipe, queue),
+ return test_bit(amdgpu_gfx_mec_queue_to_bit(adev, mec, pipe, queue),
adev->gfx.mec.queue_bitmap);
}
+int amdgpu_gfx_me_queue_to_bit(struct amdgpu_device *adev,
+ int me, int pipe, int queue)
+{
+ int bit = 0;
+
+ bit += me * adev->gfx.me.num_pipe_per_me
+ * adev->gfx.me.num_queue_per_pipe;
+ bit += pipe * adev->gfx.me.num_queue_per_pipe;
+ bit += queue;
+
+ return bit;
+}
+
+void amdgpu_gfx_bit_to_me_queue(struct amdgpu_device *adev, int bit,
+ int *me, int *pipe, int *queue)
+{
+ *queue = bit % adev->gfx.me.num_queue_per_pipe;
+ *pipe = (bit / adev->gfx.me.num_queue_per_pipe)
+ % adev->gfx.me.num_pipe_per_me;
+ *me = (bit / adev->gfx.me.num_queue_per_pipe)
+ / adev->gfx.me.num_pipe_per_me;
+}
+
+bool amdgpu_gfx_is_me_queue_enabled(struct amdgpu_device *adev,
+ int me, int pipe, int queue)
+{
+ return test_bit(amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue),
+ adev->gfx.me.queue_bitmap);
+}
+
/**
* amdgpu_gfx_scratch_get - Allocate a scratch register
*
@@ -199,6 +229,30 @@ void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev)
adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS;
}
+void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev)
+{
+ int i, queue, pipe, me;
+
+ for (i = 0; i < AMDGPU_MAX_GFX_QUEUES; ++i) {
+ queue = i % adev->gfx.me.num_queue_per_pipe;
+ pipe = (i / adev->gfx.me.num_queue_per_pipe)
+ % adev->gfx.me.num_pipe_per_me;
+ me = (i / adev->gfx.me.num_queue_per_pipe)
+ / adev->gfx.me.num_pipe_per_me;
+
+ if (me >= adev->gfx.me.num_me)
+ break;
+ /* policy: amdgpu owns the first queue per pipe at this stage
+ * will extend to mulitple queues per pipe later */
+ if (me == 0 && queue < 1)
+ set_bit(i, adev->gfx.me.queue_bitmap);
+ }
+
+ /* update the number of active graphics rings */
+ adev->gfx.num_gfx_rings =
+ bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
+}
+
static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev,
struct amdgpu_ring *ring)
{
@@ -213,7 +267,7 @@ static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev,
if (test_bit(queue_bit, adev->gfx.mec.queue_bitmap))
continue;
- amdgpu_gfx_bit_to_queue(adev, queue_bit, &mec, &pipe, &queue);
+ amdgpu_gfx_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue);
/*
* 1. Using pipes 2/3 from MEC 2 seems cause problems.
@@ -306,9 +360,9 @@ int amdgpu_gfx_kiq_init(struct amdgpu_device *adev,
return 0;
}
-/* create MQD for each compute queue */
-int amdgpu_gfx_compute_mqd_sw_init(struct amdgpu_device *adev,
- unsigned mqd_size)
+/* create MQD for each compute/gfx queue */
+int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
+ unsigned mqd_size)
{
struct amdgpu_ring *ring = NULL;
int r, i;
@@ -335,6 +389,27 @@ int amdgpu_gfx_compute_mqd_sw_init(struct amdgpu_device *adev,
dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
}
+ if (adev->asic_type == CHIP_NAVI10 && amdgpu_async_gfx_ring) {
+ /* create MQD for each KGQ */
+ for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
+ ring = &adev->gfx.gfx_ring[i];
+ if (!ring->mqd_obj) {
+ r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
+ AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
+ &ring->mqd_gpu_addr, &ring->mqd_ptr);
+ if (r) {
+ dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
+ return r;
+ }
+
+ /* prepare MQD backup */
+ adev->gfx.me.mqd_backup[i] = kmalloc(mqd_size, GFP_KERNEL);
+ if (!adev->gfx.me.mqd_backup[i])
+ dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
+ }
+ }
+ }
+
/* create MQD for each KCQ */
for (i = 0; i < adev->gfx.num_compute_rings; i++) {
ring = &adev->gfx.compute_ring[i];
@@ -343,7 +418,7 @@ int amdgpu_gfx_compute_mqd_sw_init(struct amdgpu_device *adev,
AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
&ring->mqd_gpu_addr, &ring->mqd_ptr);
if (r) {
- dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r);
+ dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
return r;
}
@@ -357,11 +432,21 @@ int amdgpu_gfx_compute_mqd_sw_init(struct amdgpu_device *adev,
return 0;
}
-void amdgpu_gfx_compute_mqd_sw_fini(struct amdgpu_device *adev)
+void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev)
{
struct amdgpu_ring *ring = NULL;
int i;
+ if (adev->asic_type == CHIP_NAVI10 && amdgpu_async_gfx_ring) {
+ for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
+ ring = &adev->gfx.gfx_ring[i];
+ kfree(adev->gfx.me.mqd_backup[i]);
+ amdgpu_bo_free_kernel(&ring->mqd_obj,
+ &ring->mqd_gpu_addr,
+ &ring->mqd_ptr);
+ }
+ }
+
for (i = 0; i < adev->gfx.num_compute_rings; i++) {
ring = &adev->gfx.compute_ring[i];
kfree(adev->gfx.mec.mqd_backup[i]);
@@ -371,12 +456,81 @@ void amdgpu_gfx_compute_mqd_sw_fini(struct amdgpu_device *adev)
}
ring = &adev->gfx.kiq.ring;
+ if (adev->asic_type == CHIP_NAVI10 && amdgpu_async_gfx_ring)
+ kfree(adev->gfx.me.mqd_backup[AMDGPU_MAX_GFX_RINGS]);
kfree(adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS]);
amdgpu_bo_free_kernel(&ring->mqd_obj,
&ring->mqd_gpu_addr,
&ring->mqd_ptr);
}
+int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev)
+{
+ struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+ struct amdgpu_ring *kiq_ring = &kiq->ring;
+ int i;
+
+ if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
+ return -EINVAL;
+
+ if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *
+ adev->gfx.num_compute_rings))
+ return -ENOMEM;
+
+ for (i = 0; i < adev->gfx.num_compute_rings; i++)
+ kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.compute_ring[i],
+ RESET_QUEUES, 0, 0);
+
+ return amdgpu_ring_test_ring(kiq_ring);
+}
+
+int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev)
+{
+ struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+ struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
+ uint64_t queue_mask = 0;
+ int r, i;
+
+ if (!kiq->pmf || !kiq->pmf->kiq_map_queues || !kiq->pmf->kiq_set_resources)
+ return -EINVAL;
+
+ for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {
+ if (!test_bit(i, adev->gfx.mec.queue_bitmap))
+ continue;
+
+ /* This situation may be hit in the future if a new HW
+ * generation exposes more than 64 queues. If so, the
+ * definition of queue_mask needs updating */
+ if (WARN_ON(i > (sizeof(queue_mask)*8))) {
+ DRM_ERROR("Invalid KCQ enabled: %d\n", i);
+ break;
+ }
+
+ queue_mask |= (1ull << i);
+ }
+
+ DRM_INFO("kiq ring mec %d pipe %d q %d\n", kiq_ring->me, kiq_ring->pipe,
+ kiq_ring->queue);
+
+ r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size *
+ adev->gfx.num_compute_rings +
+ kiq->pmf->set_resources_size);
+ if (r) {
+ DRM_ERROR("Failed to lock KIQ (%d).\n", r);
+ return r;
+ }
+
+ kiq->pmf->kiq_set_resources(kiq_ring, queue_mask);
+ for (i = 0; i < adev->gfx.num_compute_rings; i++)
+ kiq->pmf->kiq_map_queues(kiq_ring, &adev->gfx.compute_ring[i]);
+
+ r = amdgpu_ring_test_helper(kiq_ring);
+ if (r)
+ DRM_ERROR("KCQ enable failed\n");
+
+ return r;
+}
+
/* amdgpu_gfx_off_ctrl - Handle gfx off feature enable/disable
*
* @adev: amdgpu_device pointer
@@ -393,7 +547,9 @@ void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable)
if (!(adev->pm.pp_feature & PP_GFXOFF_MASK))
return;
- if (!adev->powerplay.pp_funcs || !adev->powerplay.pp_funcs->set_powergating_by_smu)
+ if (!is_support_sw_smu(adev) &&
+ (!adev->powerplay.pp_funcs ||
+ !adev->powerplay.pp_funcs->set_powergating_by_smu))
return;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index 09fc53af3d35..1199b5828b90 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -38,6 +38,7 @@
#define AMDGPU_GFX_CG_DISABLED_MODE 0x00000004L
#define AMDGPU_GFX_LBPW_DISABLED_MODE 0x00000008L
+#define AMDGPU_MAX_GFX_QUEUES KGD_MAX_QUEUES
#define AMDGPU_MAX_COMPUTE_QUEUES KGD_MAX_QUEUES
struct amdgpu_mec {
@@ -54,12 +55,41 @@ struct amdgpu_mec {
DECLARE_BITMAP(queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
};
+enum amdgpu_unmap_queues_action {
+ PREEMPT_QUEUES = 0,
+ RESET_QUEUES,
+ DISABLE_PROCESS_QUEUES,
+ PREEMPT_QUEUES_NO_UNMAP,
+};
+
+struct kiq_pm4_funcs {
+ /* Support ASIC-specific kiq pm4 packets*/
+ void (*kiq_set_resources)(struct amdgpu_ring *kiq_ring,
+ uint64_t queue_mask);
+ void (*kiq_map_queues)(struct amdgpu_ring *kiq_ring,
+ struct amdgpu_ring *ring);
+ void (*kiq_unmap_queues)(struct amdgpu_ring *kiq_ring,
+ struct amdgpu_ring *ring,
+ enum amdgpu_unmap_queues_action action,
+ u64 gpu_addr, u64 seq);
+ void (*kiq_query_status)(struct amdgpu_ring *kiq_ring,
+ struct amdgpu_ring *ring,
+ u64 addr,
+ u64 seq);
+ /* Packet sizes */
+ int set_resources_size;
+ int map_queues_size;
+ int unmap_queues_size;
+ int query_status_size;
+};
+
struct amdgpu_kiq {
u64 eop_gpu_addr;
struct amdgpu_bo *eop_obj;
spinlock_t ring_lock;
struct amdgpu_ring ring;
struct amdgpu_irq_src irq;
+ const struct kiq_pm4_funcs *pmf;
};
/*
@@ -131,6 +161,10 @@ struct amdgpu_gfx_config {
uint32_t double_offchip_lds_buf;
/* cached value of DB_DEBUG2 */
uint32_t db_debug2;
+ /* gfx10 specific config */
+ uint32_t num_sc_per_sh;
+ uint32_t num_packer_per_sc;
+ uint32_t pa_sc_tile_steering_override;
};
struct amdgpu_cu_info {
@@ -161,7 +195,7 @@ struct amdgpu_gfx_funcs {
uint32_t wave, uint32_t start, uint32_t size,
uint32_t *dst);
void (*select_me_pipe_q)(struct amdgpu_device *adev, u32 me, u32 pipe,
- u32 queue);
+ u32 queue, u32 vmid);
};
struct amdgpu_ngg_buf {
@@ -191,10 +225,38 @@ struct sq_work {
unsigned ih_data;
};
+struct amdgpu_pfp {
+ struct amdgpu_bo *pfp_fw_obj;
+ uint64_t pfp_fw_gpu_addr;
+ uint32_t *pfp_fw_ptr;
+};
+
+struct amdgpu_ce {
+ struct amdgpu_bo *ce_fw_obj;
+ uint64_t ce_fw_gpu_addr;
+ uint32_t *ce_fw_ptr;
+};
+
+struct amdgpu_me {
+ struct amdgpu_bo *me_fw_obj;
+ uint64_t me_fw_gpu_addr;
+ uint32_t *me_fw_ptr;
+ uint32_t num_me;
+ uint32_t num_pipe_per_me;
+ uint32_t num_queue_per_pipe;
+ void *mqd_backup[AMDGPU_MAX_GFX_RINGS + 1];
+
+ /* These are the resources for which amdgpu takes ownership */
+ DECLARE_BITMAP(queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
+};
+
struct amdgpu_gfx {
struct mutex gpu_clock_mutex;
struct amdgpu_gfx_config config;
struct amdgpu_rlc rlc;
+ struct amdgpu_pfp pfp;
+ struct amdgpu_ce ce;
+ struct amdgpu_me me;
struct amdgpu_mec mec;
struct amdgpu_kiq kiq;
struct amdgpu_scratch scratch;
@@ -265,7 +327,7 @@ struct amdgpu_gfx {
#define amdgpu_gfx_get_gpu_clock_counter(adev) (adev)->gfx.funcs->get_gpu_clock_counter((adev))
#define amdgpu_gfx_select_se_sh(adev, se, sh, instance) (adev)->gfx.funcs->select_se_sh((adev), (se), (sh), (instance))
-#define amdgpu_gfx_select_me_pipe_q(adev, me, pipe, q) (adev)->gfx.funcs->select_me_pipe_q((adev), (me), (pipe), (q))
+#define amdgpu_gfx_select_me_pipe_q(adev, me, pipe, q, vmid) (adev)->gfx.funcs->select_me_pipe_q((adev), (me), (pipe), (q), (vmid))
/**
* amdgpu_gfx_create_bitmask - create a bitmask
@@ -297,17 +359,27 @@ void amdgpu_gfx_kiq_fini(struct amdgpu_device *adev);
int amdgpu_gfx_kiq_init(struct amdgpu_device *adev,
unsigned hpd_size);
-int amdgpu_gfx_compute_mqd_sw_init(struct amdgpu_device *adev,
- unsigned mqd_size);
-void amdgpu_gfx_compute_mqd_sw_fini(struct amdgpu_device *adev);
+int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
+ unsigned mqd_size);
+void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev);
+int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev);
+int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev);
void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev);
-int amdgpu_gfx_queue_to_bit(struct amdgpu_device *adev, int mec,
- int pipe, int queue);
-void amdgpu_gfx_bit_to_queue(struct amdgpu_device *adev, int bit,
- int *mec, int *pipe, int *queue);
+void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev);
+
+int amdgpu_gfx_mec_queue_to_bit(struct amdgpu_device *adev, int mec,
+ int pipe, int queue);
+void amdgpu_gfx_bit_to_mec_queue(struct amdgpu_device *adev, int bit,
+ int *mec, int *pipe, int *queue);
bool amdgpu_gfx_is_mec_queue_enabled(struct amdgpu_device *adev, int mec,
int pipe, int queue);
+int amdgpu_gfx_me_queue_to_bit(struct amdgpu_device *adev, int me,
+ int pipe, int queue);
+void amdgpu_gfx_bit_to_me_queue(struct amdgpu_device *adev, int bit,
+ int *me, int *pipe, int *queue);
+bool amdgpu_gfx_is_me_queue_enabled(struct amdgpu_device *adev, int me,
+ int pipe, int queue);
void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable);
#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 250d9212cc38..924d83e711ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -24,6 +24,8 @@
*
*/
+#include <linux/io-64-nonatomic-lo-hi.h>
+
#include "amdgpu.h"
/**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
index 62591d081856..627104401e84 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
@@ -22,7 +22,6 @@
* Authors: Christian König
*/
-#include <drm/drmP.h>
#include "amdgpu.h"
struct amdgpu_gtt_mgr {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c
index f2739995c335..70dbe343f51d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c
@@ -23,9 +23,10 @@
* Authors: Dave Airlie
* Alex Deucher
*/
+
#include <linux/export.h>
+#include <linux/pci.h>
-#include <drm/drmP.h>
#include <drm/drm_edid.h>
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index fe393a46f881..7850084a05e3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -28,8 +28,10 @@
*/
#include <linux/seq_file.h>
#include <linux/slab.h>
-#include <drm/drmP.h>
+
#include <drm/amdgpu_drm.h>
+#include <drm/drm_debugfs.h>
+
#include "amdgpu.h"
#include "atom.h"
#include "amdgpu_trace.h"
@@ -209,6 +211,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
skip_preamble = ring->current_ctx == fence_ctx;
if (job && ring->funcs->emit_cntxcntl) {
status |= job->preamble_status;
+ status |= job->preemption_status;
amdgpu_ring_emit_cntxcntl(ring, status);
}
@@ -217,9 +220,10 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
/* drop preamble IBs if we don't have a context switch */
if ((ib->flags & AMDGPU_IB_FLAG_PREAMBLE) &&
- skip_preamble &&
- !(status & AMDGPU_PREAMBLE_IB_PRESENT_FIRST) &&
- !amdgpu_sriov_vf(adev)) /* for SRIOV preemption, Preamble CE ib must be inserted anyway */
+ skip_preamble &&
+ !(status & AMDGPU_PREAMBLE_IB_PRESENT_FIRST) &&
+ !amdgpu_mcbp &&
+ !amdgpu_sriov_vf(adev)) /* for SRIOV preemption, Preamble CE ib must be inserted anyway */
continue;
amdgpu_ring_emit_ib(ring, job, ib, status);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
index df9b173c3d0b..57b3d8a9bef3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
@@ -24,7 +24,7 @@
#include <linux/idr.h>
#include <linux/dma-fence-array.h>
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_trace.h"
@@ -364,8 +364,11 @@ static int amdgpu_vmid_grab_used(struct amdgpu_vm *vm,
if (updates && (!flushed || dma_fence_is_later(updates, flushed)))
needs_flush = true;
- /* Concurrent flushes are only possible starting with Vega10 */
- if (adev->asic_type < CHIP_VEGA10 && needs_flush)
+ /* Concurrent flushes are only possible starting with Vega10 and
+ * are broken on Navi10 and Navi14.
+ */
+ if (needs_flush && (adev->asic_type < CHIP_VEGA10 ||
+ adev->asic_type == CHIP_NAVI10))
continue;
/* Good, we can use this VMID. Remember this submission as
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
index 934dfdcb4e73..6d8f05511aba 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
@@ -21,7 +21,8 @@
*
*/
-#include <drm/drmP.h>
+#include <linux/dma-mapping.h>
+
#include "amdgpu.h"
#include "amdgpu_ih.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ioc32.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ioc32.c
index 26482914dc4b..5cf142e849bb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ioc32.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ioc32.c
@@ -29,8 +29,9 @@
*/
#include <linux/compat.h>
-#include <drm/drmP.h>
#include <drm/amdgpu_drm.h>
+#include <drm/drm_ioctl.h>
+
#include "amdgpu_drv.h"
long amdgpu_kms_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
index af4c3b1af322..2a3f5ec298db 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
@@ -43,8 +43,11 @@
*/
#include <linux/irq.h>
-#include <drm/drmP.h>
+#include <linux/pci.h>
+
#include <drm/drm_crtc_helper.h>
+#include <drm/drm_irq.h>
+#include <drm/drm_vblank.h>
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
#include "amdgpu_ih.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 0a17fb1af204..9d76e0923a5a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -24,7 +24,7 @@
#include <linux/kthread.h>
#include <linux/wait.h>
#include <linux/sched.h>
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_trace.h"
@@ -51,6 +51,8 @@ static void amdgpu_job_timedout(struct drm_sched_job *s_job)
if (amdgpu_device_should_recover_gpu(ring->adev))
amdgpu_device_gpu_recover(ring->adev, job);
+ else
+ drm_sched_suspend_timeout(&ring->sched);
}
int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
index e1b46a6703de..51e62504c279 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
@@ -29,6 +29,8 @@
#define AMDGPU_PREAMBLE_IB_PRESENT_FIRST (1 << 1)
/* bit set means context switch occured */
#define AMDGPU_HAVE_CTX_SWITCH (1 << 2)
+/* bit set means IB is preempted */
+#define AMDGPU_IB_PREEMPTED (1 << 3)
#define to_amdgpu_job(sched_job) \
container_of((sched_job), struct amdgpu_job, base)
@@ -45,6 +47,7 @@ struct amdgpu_job {
struct amdgpu_ib *ibs;
struct dma_fence *fence; /* the hw fence */
uint32_t preamble_status;
+ uint32_t preemption_status;
uint32_t num_ibs;
void *owner;
bool vm_needs_flush;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index b17d0545728e..0cf7e8606fd3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -25,8 +25,9 @@
* Alex Deucher
* Jerome Glisse
*/
-#include <drm/drmP.h>
+
#include "amdgpu.h"
+#include <drm/drm_debugfs.h>
#include <drm/amdgpu_drm.h>
#include "amdgpu_sched.h"
#include "amdgpu_uvd.h"
@@ -35,13 +36,15 @@
#include <linux/vga_switcheroo.h>
#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/pci.h>
#include <linux/pm_runtime.h>
#include "amdgpu_amdkfd.h"
#include "amdgpu_gem.h"
#include "amdgpu_display.h"
#include "amdgpu_ras.h"
-static void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev)
+void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev)
{
struct amdgpu_gpu_instance *gpu_instance;
int i;
@@ -102,7 +105,7 @@ done_free:
dev->dev_private = NULL;
}
-static void amdgpu_register_gpu_instance(struct amdgpu_device *adev)
+void amdgpu_register_gpu_instance(struct amdgpu_device *adev)
{
struct amdgpu_gpu_instance *gpu_instance;
@@ -590,13 +593,10 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
struct drm_amdgpu_info_gds gds_info;
memset(&gds_info, 0, sizeof(gds_info));
- gds_info.gds_gfx_partition_size = adev->gds.mem.gfx_partition_size;
- gds_info.compute_partition_size = adev->gds.mem.cs_partition_size;
- gds_info.gds_total_size = adev->gds.mem.total_size;
- gds_info.gws_per_gfx_partition = adev->gds.gws.gfx_partition_size;
- gds_info.gws_per_compute_partition = adev->gds.gws.cs_partition_size;
- gds_info.oa_per_gfx_partition = adev->gds.oa.gfx_partition_size;
- gds_info.oa_per_compute_partition = adev->gds.oa.cs_partition_size;
+ gds_info.compute_partition_size = adev->gds.gds_size;
+ gds_info.gds_total_size = adev->gds.gds_size;
+ gds_info.gws_per_compute_partition = adev->gds.gws_size;
+ gds_info.oa_per_compute_partition = adev->gds.oa_size;
return copy_to_user(out, &gds_info,
min((size_t)size, sizeof(gds_info))) ? -EFAULT : 0;
}
@@ -712,7 +712,7 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
dev_info.ids_flags = 0;
if (adev->flags & AMD_IS_APU)
dev_info.ids_flags |= AMDGPU_IDS_FLAGS_FUSION;
- if (amdgpu_sriov_vf(adev))
+ if (amdgpu_mcbp || amdgpu_sriov_vf(adev))
dev_info.ids_flags |= AMDGPU_IDS_FLAGS_PREEMPTION;
vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
@@ -765,6 +765,10 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
dev_info.gs_prim_buffer_depth = adev->gfx.config.gs_prim_buffer_depth;
dev_info.max_gs_waves_per_vgt = adev->gfx.config.max_gs_threads;
+ if (adev->family >= AMDGPU_FAMILY_NV)
+ dev_info.pa_sc_tile_steering_override =
+ adev->gfx.config.pa_sc_tile_steering_override;
+
return copy_to_user(out, &dev_info,
min((size_t)size, sizeof(dev_info))) ? -EFAULT : 0;
}
@@ -977,7 +981,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
int r, pasid;
/* Ensure IB tests are run on ring */
- flush_delayed_work(&adev->late_init_work);
+ flush_delayed_work(&adev->delayed_init_work);
file_priv->driver_priv = NULL;
@@ -1006,7 +1010,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
goto error_vm;
}
- if (amdgpu_sriov_vf(adev)) {
+ if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) {
uint64_t csa_addr = amdgpu_csa_vaddr(adev) & AMDGPU_GMC_HOLE_MASK;
r = amdgpu_map_static_csa(adev, &fpriv->vm, adev->virt.csa_obj,
@@ -1069,7 +1073,7 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
amdgpu_vm_bo_rmv(adev, fpriv->prt_va);
- if (amdgpu_sriov_vf(adev)) {
+ if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) {
/* TODO: how to handle reserve failure */
BUG_ON(amdgpu_bo_reserve(adev->virt.csa_obj, true));
amdgpu_vm_bo_rmv(adev, fpriv->csa_va);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
new file mode 100644
index 000000000000..78fe49033543
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __AMDGPU_MES_H__
+#define __AMDGPU_MES_H__
+
+struct amdgpu_mes_funcs;
+
+struct amdgpu_mes {
+ struct amdgpu_adev *adev;
+
+ const struct firmware *fw;
+
+ /* mes ucode */
+ struct amdgpu_bo *ucode_fw_obj;
+ uint64_t ucode_fw_gpu_addr;
+ uint32_t *ucode_fw_ptr;
+ uint32_t ucode_fw_version;
+ uint64_t uc_start_addr;
+
+ /* mes ucode data */
+ struct amdgpu_bo *data_fw_obj;
+ uint64_t data_fw_gpu_addr;
+ uint32_t *data_fw_ptr;
+ uint32_t data_fw_version;
+ uint64_t data_start_addr;
+
+ /* ip specific functions */
+ struct amdgpu_mes_funcs *funcs;
+};
+
+struct mes_add_queue_input {
+ uint32_t process_id;
+ uint64_t page_table_base_addr;
+ uint64_t process_va_start;
+ uint64_t process_va_end;
+ uint64_t process_quantum;
+ uint64_t process_context_addr;
+ uint64_t gang_quantum;
+ uint64_t gang_context_addr;
+ uint32_t inprocess_gang_priority;
+ uint32_t gang_global_priority_level;
+ uint32_t doorbell_offset;
+ uint64_t mqd_addr;
+ uint64_t wptr_addr;
+ uint32_t queue_type;
+ uint32_t paging;
+};
+
+struct mes_remove_queue_input {
+ uint32_t doorbell_offset;
+ uint64_t gang_context_addr;
+};
+
+struct mes_suspend_gang_input {
+ bool suspend_all_gangs;
+ uint64_t gang_context_addr;
+ uint64_t suspend_fence_addr;
+ uint32_t suspend_fence_value;
+};
+
+struct mes_resume_gang_input {
+ bool resume_all_gangs;
+ uint64_t gang_context_addr;
+};
+
+struct amdgpu_mes_funcs {
+ int (*add_hw_queue)(struct amdgpu_mes *mes,
+ struct mes_add_queue_input *input);
+
+ int (*remove_hw_queue)(struct amdgpu_mes *mes,
+ struct mes_remove_queue_input *input);
+
+ int (*suspend_gang)(struct amdgpu_mes *mes,
+ struct mes_suspend_gang_input *input);
+
+ int (*resume_gang)(struct amdgpu_mes *mes,
+ struct mes_resume_gang_input *input);
+};
+
+#endif /* __AMDGPU_MES_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
index 58ed401c5996..3971c201f320 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
@@ -45,51 +45,12 @@
#include <linux/firmware.h>
#include <linux/module.h>
-#include <linux/mmu_notifier.h>
-#include <linux/interval_tree.h>
-#include <drm/drmP.h>
#include <drm/drm.h>
#include "amdgpu.h"
#include "amdgpu_amdkfd.h"
/**
- * struct amdgpu_mn
- *
- * @adev: amdgpu device pointer
- * @mm: process address space
- * @mn: MMU notifier structure
- * @type: type of MMU notifier
- * @work: destruction work item
- * @node: hash table node to find structure by adev and mn
- * @lock: rw semaphore protecting the notifier nodes
- * @objects: interval tree containing amdgpu_mn_nodes
- * @read_lock: mutex for recursive locking of @lock
- * @recursion: depth of recursion
- *
- * Data for each amdgpu device and process address space.
- */
-struct amdgpu_mn {
- /* constant after initialisation */
- struct amdgpu_device *adev;
- struct mm_struct *mm;
- struct mmu_notifier mn;
- enum amdgpu_mn_type type;
-
- /* only used on destruction */
- struct work_struct work;
-
- /* protected by adev->mn_lock */
- struct hlist_node node;
-
- /* objects protected by lock */
- struct rw_semaphore lock;
- struct rb_root_cached objects;
- struct mutex read_lock;
- atomic_t recursion;
-};
-
-/**
* struct amdgpu_mn_node
*
* @it: interval node defining start-last of the affected address range
@@ -103,7 +64,7 @@ struct amdgpu_mn_node {
};
/**
- * amdgpu_mn_destroy - destroy the MMU notifier
+ * amdgpu_mn_destroy - destroy the HMM mirror
*
* @work: previously sheduled work item
*
@@ -129,28 +90,26 @@ static void amdgpu_mn_destroy(struct work_struct *work)
}
up_write(&amn->lock);
mutex_unlock(&adev->mn_lock);
- mmu_notifier_unregister_no_release(&amn->mn, amn->mm);
+
+ hmm_mirror_unregister(&amn->mirror);
kfree(amn);
}
/**
- * amdgpu_mn_release - callback to notify about mm destruction
+ * amdgpu_hmm_mirror_release - callback to notify about mm destruction
*
- * @mn: our notifier
- * @mm: the mm this callback is about
+ * @mirror: the HMM mirror (mm) this callback is about
*
- * Shedule a work item to lazy destroy our notifier.
+ * Shedule a work item to lazy destroy HMM mirror.
*/
-static void amdgpu_mn_release(struct mmu_notifier *mn,
- struct mm_struct *mm)
+static void amdgpu_hmm_mirror_release(struct hmm_mirror *mirror)
{
- struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
+ struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror);
INIT_WORK(&amn->work, amdgpu_mn_destroy);
schedule_work(&amn->work);
}
-
/**
* amdgpu_mn_lock - take the write side lock for this notifier
*
@@ -181,14 +140,10 @@ void amdgpu_mn_unlock(struct amdgpu_mn *mn)
static int amdgpu_mn_read_lock(struct amdgpu_mn *amn, bool blockable)
{
if (blockable)
- mutex_lock(&amn->read_lock);
- else if (!mutex_trylock(&amn->read_lock))
+ down_read(&amn->lock);
+ else if (!down_read_trylock(&amn->lock))
return -EAGAIN;
- if (atomic_inc_return(&amn->recursion) == 1)
- down_read_non_owner(&amn->lock);
- mutex_unlock(&amn->read_lock);
-
return 0;
}
@@ -199,8 +154,7 @@ static int amdgpu_mn_read_lock(struct amdgpu_mn *amn, bool blockable)
*/
static void amdgpu_mn_read_unlock(struct amdgpu_mn *amn)
{
- if (atomic_dec_return(&amn->recursion) == 0)
- up_read_non_owner(&amn->lock);
+ up_read(&amn->lock);
}
/**
@@ -229,149 +183,132 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,
true, false, MAX_SCHEDULE_TIMEOUT);
if (r <= 0)
DRM_ERROR("(%ld) failed to wait for user bo\n", r);
-
- amdgpu_ttm_tt_mark_user_pages(bo->tbo.ttm);
}
}
/**
- * amdgpu_mn_invalidate_range_start_gfx - callback to notify about mm change
+ * amdgpu_mn_sync_pagetables_gfx - callback to notify about mm change
*
- * @mn: our notifier
- * @range: mmu notifier context
+ * @mirror: the hmm_mirror (mm) is about to update
+ * @update: the update start, end address
*
* Block for operations on BOs to finish and mark pages as accessed and
* potentially dirty.
*/
-static int amdgpu_mn_invalidate_range_start_gfx(struct mmu_notifier *mn,
- const struct mmu_notifier_range *range)
+static int amdgpu_mn_sync_pagetables_gfx(struct hmm_mirror *mirror,
+ const struct hmm_update *update)
{
- struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
+ struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror);
+ unsigned long start = update->start;
+ unsigned long end = update->end;
+ bool blockable = update->blockable;
struct interval_tree_node *it;
- unsigned long end;
/* notification is exclusive, but interval is inclusive */
- end = range->end - 1;
+ end -= 1;
/* TODO we should be able to split locking for interval tree and
* amdgpu_mn_invalidate_node
*/
- if (amdgpu_mn_read_lock(amn, mmu_notifier_range_blockable(range)))
+ if (amdgpu_mn_read_lock(amn, blockable))
return -EAGAIN;
- it = interval_tree_iter_first(&amn->objects, range->start, end);
+ it = interval_tree_iter_first(&amn->objects, start, end);
while (it) {
struct amdgpu_mn_node *node;
- if (!mmu_notifier_range_blockable(range)) {
+ if (!blockable) {
amdgpu_mn_read_unlock(amn);
return -EAGAIN;
}
node = container_of(it, struct amdgpu_mn_node, it);
- it = interval_tree_iter_next(it, range->start, end);
+ it = interval_tree_iter_next(it, start, end);
- amdgpu_mn_invalidate_node(node, range->start, end);
+ amdgpu_mn_invalidate_node(node, start, end);
}
+ amdgpu_mn_read_unlock(amn);
+
return 0;
}
/**
- * amdgpu_mn_invalidate_range_start_hsa - callback to notify about mm change
+ * amdgpu_mn_sync_pagetables_hsa - callback to notify about mm change
*
- * @mn: our notifier
- * @mm: the mm this callback is about
- * @start: start of updated range
- * @end: end of updated range
+ * @mirror: the hmm_mirror (mm) is about to update
+ * @update: the update start, end address
*
* We temporarily evict all BOs between start and end. This
* necessitates evicting all user-mode queues of the process. The BOs
* are restorted in amdgpu_mn_invalidate_range_end_hsa.
*/
-static int amdgpu_mn_invalidate_range_start_hsa(struct mmu_notifier *mn,
- const struct mmu_notifier_range *range)
+static int amdgpu_mn_sync_pagetables_hsa(struct hmm_mirror *mirror,
+ const struct hmm_update *update)
{
- struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
+ struct amdgpu_mn *amn = container_of(mirror, struct amdgpu_mn, mirror);
+ unsigned long start = update->start;
+ unsigned long end = update->end;
+ bool blockable = update->blockable;
struct interval_tree_node *it;
- unsigned long end;
/* notification is exclusive, but interval is inclusive */
- end = range->end - 1;
+ end -= 1;
- if (amdgpu_mn_read_lock(amn, mmu_notifier_range_blockable(range)))
+ if (amdgpu_mn_read_lock(amn, blockable))
return -EAGAIN;
- it = interval_tree_iter_first(&amn->objects, range->start, end);
+ it = interval_tree_iter_first(&amn->objects, start, end);
while (it) {
struct amdgpu_mn_node *node;
struct amdgpu_bo *bo;
- if (!mmu_notifier_range_blockable(range)) {
+ if (!blockable) {
amdgpu_mn_read_unlock(amn);
return -EAGAIN;
}
node = container_of(it, struct amdgpu_mn_node, it);
- it = interval_tree_iter_next(it, range->start, end);
+ it = interval_tree_iter_next(it, start, end);
list_for_each_entry(bo, &node->bos, mn_list) {
struct kgd_mem *mem = bo->kfd_bo;
if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm,
- range->start,
- end))
- amdgpu_amdkfd_evict_userptr(mem, range->mm);
+ start, end))
+ amdgpu_amdkfd_evict_userptr(mem, amn->mm);
}
}
+ amdgpu_mn_read_unlock(amn);
+
return 0;
}
-/**
- * amdgpu_mn_invalidate_range_end - callback to notify about mm change
- *
- * @mn: our notifier
- * @mm: the mm this callback is about
- * @start: start of updated range
- * @end: end of updated range
- *
- * Release the lock again to allow new command submissions.
+/* Low bits of any reasonable mm pointer will be unused due to struct
+ * alignment. Use these bits to make a unique key from the mm pointer
+ * and notifier type.
*/
-static void amdgpu_mn_invalidate_range_end(struct mmu_notifier *mn,
- const struct mmu_notifier_range *range)
-{
- struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
-
- amdgpu_mn_read_unlock(amn);
-}
+#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type))
-static const struct mmu_notifier_ops amdgpu_mn_ops[] = {
+static struct hmm_mirror_ops amdgpu_hmm_mirror_ops[] = {
[AMDGPU_MN_TYPE_GFX] = {
- .release = amdgpu_mn_release,
- .invalidate_range_start = amdgpu_mn_invalidate_range_start_gfx,
- .invalidate_range_end = amdgpu_mn_invalidate_range_end,
+ .sync_cpu_device_pagetables = amdgpu_mn_sync_pagetables_gfx,
+ .release = amdgpu_hmm_mirror_release
},
[AMDGPU_MN_TYPE_HSA] = {
- .release = amdgpu_mn_release,
- .invalidate_range_start = amdgpu_mn_invalidate_range_start_hsa,
- .invalidate_range_end = amdgpu_mn_invalidate_range_end,
+ .sync_cpu_device_pagetables = amdgpu_mn_sync_pagetables_hsa,
+ .release = amdgpu_hmm_mirror_release
},
};
-/* Low bits of any reasonable mm pointer will be unused due to struct
- * alignment. Use these bits to make a unique key from the mm pointer
- * and notifier type.
- */
-#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type))
-
/**
- * amdgpu_mn_get - create notifier context
+ * amdgpu_mn_get - create HMM mirror context
*
* @adev: amdgpu device pointer
* @type: type of MMU notifier context
*
- * Creates a notifier context for current->mm.
+ * Creates a HMM mirror context for current->mm.
*/
struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
enum amdgpu_mn_type type)
@@ -401,12 +338,10 @@ struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
amn->mm = mm;
init_rwsem(&amn->lock);
amn->type = type;
- amn->mn.ops = &amdgpu_mn_ops[type];
amn->objects = RB_ROOT_CACHED;
- mutex_init(&amn->read_lock);
- atomic_set(&amn->recursion, 0);
- r = __mmu_notifier_register(&amn->mn, mm);
+ amn->mirror.ops = &amdgpu_hmm_mirror_ops[type];
+ r = hmm_mirror_register(&amn->mirror, mm);
if (r)
goto free_amn;
@@ -432,7 +367,7 @@ free_amn:
* @bo: amdgpu buffer object
* @addr: userptr addr we should monitor
*
- * Registers an MMU notifier for the given BO at the specified address.
+ * Registers an HMM mirror for the given BO at the specified address.
* Returns 0 on success, -ERRNO if anything goes wrong.
*/
int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
@@ -488,11 +423,11 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
}
/**
- * amdgpu_mn_unregister - unregister a BO for notifier updates
+ * amdgpu_mn_unregister - unregister a BO for HMM mirror updates
*
* @bo: amdgpu buffer object
*
- * Remove any registration of MMU notifier updates from the buffer object.
+ * Remove any registration of HMM mirror updates from the buffer object.
*/
void amdgpu_mn_unregister(struct amdgpu_bo *bo)
{
@@ -528,3 +463,25 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo)
mutex_unlock(&adev->mn_lock);
}
+/* flags used by HMM internal, not related to CPU/GPU PTE flags */
+static const uint64_t hmm_range_flags[HMM_PFN_FLAG_MAX] = {
+ (1 << 0), /* HMM_PFN_VALID */
+ (1 << 1), /* HMM_PFN_WRITE */
+ 0 /* HMM_PFN_DEVICE_PRIVATE */
+};
+
+static const uint64_t hmm_range_values[HMM_PFN_VALUE_MAX] = {
+ 0xfffffffffffffffeUL, /* HMM_PFN_ERROR */
+ 0, /* HMM_PFN_NONE */
+ 0xfffffffffffffffcUL /* HMM_PFN_SPECIAL */
+};
+
+void amdgpu_hmm_init_range(struct hmm_range *range)
+{
+ if (range) {
+ range->flags = hmm_range_flags;
+ range->values = hmm_range_values;
+ range->pfn_shift = PAGE_SHIFT;
+ INIT_LIST_HEAD(&range->list);
+ }
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h
index eb0f432f78fe..b8ed68943625 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h
@@ -24,23 +24,61 @@
#ifndef __AMDGPU_MN_H__
#define __AMDGPU_MN_H__
-/*
- * MMU Notifier
- */
-struct amdgpu_mn;
+#include <linux/types.h>
+#include <linux/hmm.h>
+#include <linux/rwsem.h>
+#include <linux/workqueue.h>
+#include <linux/interval_tree.h>
enum amdgpu_mn_type {
AMDGPU_MN_TYPE_GFX,
AMDGPU_MN_TYPE_HSA,
};
-#if defined(CONFIG_MMU_NOTIFIER)
+/**
+ * struct amdgpu_mn
+ *
+ * @adev: amdgpu device pointer
+ * @mm: process address space
+ * @type: type of MMU notifier
+ * @work: destruction work item
+ * @node: hash table node to find structure by adev and mn
+ * @lock: rw semaphore protecting the notifier nodes
+ * @objects: interval tree containing amdgpu_mn_nodes
+ * @mirror: HMM mirror function support
+ *
+ * Data for each amdgpu device and process address space.
+ */
+struct amdgpu_mn {
+ /* constant after initialisation */
+ struct amdgpu_device *adev;
+ struct mm_struct *mm;
+ enum amdgpu_mn_type type;
+
+ /* only used on destruction */
+ struct work_struct work;
+
+ /* protected by adev->mn_lock */
+ struct hlist_node node;
+
+ /* objects protected by lock */
+ struct rw_semaphore lock;
+ struct rb_root_cached objects;
+
+#ifdef CONFIG_HMM_MIRROR
+ /* HMM mirror */
+ struct hmm_mirror mirror;
+#endif
+};
+
+#if defined(CONFIG_HMM_MIRROR)
void amdgpu_mn_lock(struct amdgpu_mn *mn);
void amdgpu_mn_unlock(struct amdgpu_mn *mn);
struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
enum amdgpu_mn_type type);
int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);
void amdgpu_mn_unregister(struct amdgpu_bo *bo);
+void amdgpu_hmm_init_range(struct hmm_range *range);
#else
static inline void amdgpu_mn_lock(struct amdgpu_mn *mn) {}
static inline void amdgpu_mn_unlock(struct amdgpu_mn *mn) {}
@@ -51,6 +89,8 @@ static inline struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
}
static inline int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
{
+ DRM_WARN_ONCE("HMM_MIRROR kernel config option is not enabled, "
+ "add CONFIG_ZONE_DEVICE=y in config file to fix this\n");
return -ENODEV;
}
static inline void amdgpu_mn_unregister(struct amdgpu_bo *bo) {}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
index 2e9e3db778c6..eb9975f4decb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
@@ -331,8 +331,6 @@ struct amdgpu_mode_info {
struct drm_property *audio_property;
/* FMT dithering */
struct drm_property *dither_property;
- /* maximum number of bits per channel for monitor color */
- struct drm_property *max_bpc_property;
/* Adaptive Backlight Modulation (power feature) */
struct drm_property *abm_level_property;
/* hardcoded DFP edid from BIOS */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 93b2c5a48a71..bea6f298dfdc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -31,7 +31,7 @@
*/
#include <linux/list.h>
#include <linux/slab.h>
-#include <drm/drmP.h>
+
#include <drm/amdgpu_drm.h>
#include <drm/drm_cache.h>
#include "amdgpu.h"
@@ -495,7 +495,11 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev,
#endif
bo->tbo.bdev = &adev->mman.bdev;
- amdgpu_bo_placement_from_domain(bo, bp->domain);
+ if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA |
+ AMDGPU_GEM_DOMAIN_GDS))
+ amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
+ else
+ amdgpu_bo_placement_from_domain(bo, bp->domain);
if (bp->type == ttm_bo_type_kernel)
bo->tbo.priority = 1;
@@ -975,6 +979,7 @@ static const char *amdgpu_vram_names[] = {
"HBM",
"DDR3",
"DDR4",
+ "GDDR6",
};
/**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index c430e8259038..d60593cc436e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -155,7 +155,7 @@ static inline int amdgpu_bo_reserve(struct amdgpu_bo *bo, bool no_intr)
struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
int r;
- r = ttm_bo_reserve(&bo->tbo, !no_intr, false, NULL);
+ r = __ttm_bo_reserve(&bo->tbo, !no_intr, false, NULL);
if (unlikely(r != 0)) {
if (r != -ERESTARTSYS)
dev_err(adev->dev, "%p reserve failed\n", bo);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pll.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pll.c
index 8e67c1210d7c..1f2305b7bd13 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pll.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pll.c
@@ -20,7 +20,7 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
-#include <drm/drmP.h>
+
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
#include "atom.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index abeaab4bf1bc..2b546567853b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -22,7 +22,9 @@
* Authors: Rafał Miłecki <zajec5@gmail.com>
* Alex Deucher <alexdeucher@gmail.com>
*/
-#include <drm/drmP.h>
+
+#include <drm/drm_debugfs.h>
+
#include "amdgpu.h"
#include "amdgpu_drv.h"
#include "amdgpu_pm.h"
@@ -31,6 +33,7 @@
#include "amdgpu_smu.h"
#include "atom.h"
#include <linux/power_supply.h>
+#include <linux/pci.h>
#include <linux/hwmon.h>
#include <linux/hwmon-sysfs.h>
#include <linux/nospec.h>
@@ -64,9 +67,21 @@ static const struct cg_flag_name clocks[] = {
{AMD_CG_SUPPORT_DRM_LS, "Digital Right Management Light Sleep"},
{AMD_CG_SUPPORT_ROM_MGCG, "Rom Medium Grain Clock Gating"},
{AMD_CG_SUPPORT_DF_MGCG, "Data Fabric Medium Grain Clock Gating"},
+
+ {AMD_CG_SUPPORT_ATHUB_MGCG, "Address Translation Hub Medium Grain Clock Gating"},
+ {AMD_CG_SUPPORT_ATHUB_LS, "Address Translation Hub Light Sleep"},
{0, NULL},
};
+static const struct hwmon_temp_label {
+ enum PP_HWMON_TEMP channel;
+ const char *label;
+} temp_label[] = {
+ {PP_TEMP_EDGE, "edge"},
+ {PP_TEMP_JUNCTION, "junction"},
+ {PP_TEMP_MEM, "mem"},
+};
+
void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev)
{
if (adev->pm.dpm_enabled) {
@@ -144,12 +159,16 @@ static ssize_t amdgpu_get_dpm_state(struct device *dev,
struct amdgpu_device *adev = ddev->dev_private;
enum amd_pm_state_type pm;
- if (is_support_sw_smu(adev) && adev->smu.ppt_funcs->get_current_power_state)
- pm = amdgpu_smu_get_current_power_state(adev);
- else if (adev->powerplay.pp_funcs->get_current_power_state)
+ if (is_support_sw_smu(adev)) {
+ if (adev->smu.ppt_funcs->get_current_power_state)
+ pm = amdgpu_smu_get_current_power_state(adev);
+ else
+ pm = adev->pm.dpm.user_state;
+ } else if (adev->powerplay.pp_funcs->get_current_power_state) {
pm = amdgpu_dpm_get_current_power_state(adev);
- else
+ } else {
pm = adev->pm.dpm.user_state;
+ }
return snprintf(buf, PAGE_SIZE, "%s\n",
(pm == POWER_STATE_TYPE_BATTERY) ? "battery" :
@@ -176,7 +195,11 @@ static ssize_t amdgpu_set_dpm_state(struct device *dev,
goto fail;
}
- if (adev->powerplay.pp_funcs->dispatch_tasks) {
+ if (is_support_sw_smu(adev)) {
+ mutex_lock(&adev->pm.mutex);
+ adev->pm.dpm.user_state = state;
+ mutex_unlock(&adev->pm.mutex);
+ } else if (adev->powerplay.pp_funcs->dispatch_tasks) {
amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_ENABLE_USER_STATE, &state);
} else {
mutex_lock(&adev->pm.mutex);
@@ -260,8 +283,11 @@ static ssize_t amdgpu_get_dpm_forced_performance_level(struct device *dev,
struct amdgpu_device *adev = ddev->dev_private;
enum amd_dpm_forced_level level = 0xff;
- if ((adev->flags & AMD_IS_PX) &&
- (ddev->switch_power_state != DRM_SWITCH_POWER_ON))
+ if (amdgpu_sriov_vf(adev))
+ return 0;
+
+ if ((adev->flags & AMD_IS_PX) &&
+ (ddev->switch_power_state != DRM_SWITCH_POWER_ON))
return snprintf(buf, PAGE_SIZE, "off\n");
if (is_support_sw_smu(adev))
@@ -299,10 +325,12 @@ static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
(ddev->switch_power_state != DRM_SWITCH_POWER_ON))
return -EINVAL;
- if (is_support_sw_smu(adev))
- current_level = smu_get_performance_level(&adev->smu);
- else if (adev->powerplay.pp_funcs->get_performance_level)
- current_level = amdgpu_dpm_get_performance_level(adev);
+ if (!amdgpu_sriov_vf(adev)) {
+ if (is_support_sw_smu(adev))
+ current_level = smu_get_performance_level(&adev->smu);
+ else if (adev->powerplay.pp_funcs->get_performance_level)
+ current_level = amdgpu_dpm_get_performance_level(adev);
+ }
if (strncmp("low", buf, strlen("low")) == 0) {
level = AMD_DPM_FORCED_LEVEL_LOW;
@@ -353,18 +381,9 @@ static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
}
if (is_support_sw_smu(adev)) {
- mutex_lock(&adev->pm.mutex);
- if (adev->pm.dpm.thermal_active) {
- count = -EINVAL;
- mutex_unlock(&adev->pm.mutex);
- goto fail;
- }
ret = smu_force_performance_level(&adev->smu, level);
if (ret)
count = -EINVAL;
- else
- adev->pm.dpm.forced_level = level;
- mutex_unlock(&adev->pm.mutex);
} else if (adev->powerplay.pp_funcs->force_performance_level) {
mutex_lock(&adev->pm.mutex);
if (adev->pm.dpm.thermal_active) {
@@ -678,12 +697,12 @@ static ssize_t amdgpu_set_pp_od_clk_voltage(struct device *dev,
if (ret)
return -EINVAL;
} else {
- if (adev->powerplay.pp_funcs->odn_edit_dpm_table)
+ if (adev->powerplay.pp_funcs->odn_edit_dpm_table) {
ret = amdgpu_dpm_odn_edit_dpm_table(adev, type,
parameter, parameter_size);
-
- if (ret)
- return -EINVAL;
+ if (ret)
+ return -EINVAL;
+ }
if (type == PP_OD_COMMIT_DPM_TABLE) {
if (adev->powerplay.pp_funcs->dispatch_tasks) {
@@ -709,10 +728,10 @@ static ssize_t amdgpu_get_pp_od_clk_voltage(struct device *dev,
uint32_t size = 0;
if (is_support_sw_smu(adev)) {
- size = smu_print_clk_levels(&adev->smu, OD_SCLK, buf);
- size += smu_print_clk_levels(&adev->smu, OD_MCLK, buf+size);
- size += smu_print_clk_levels(&adev->smu, OD_VDDC_CURVE, buf+size);
- size += smu_print_clk_levels(&adev->smu, OD_RANGE, buf+size);
+ size = smu_print_clk_levels(&adev->smu, SMU_OD_SCLK, buf);
+ size += smu_print_clk_levels(&adev->smu, SMU_OD_MCLK, buf+size);
+ size += smu_print_clk_levels(&adev->smu, SMU_OD_VDDC_CURVE, buf+size);
+ size += smu_print_clk_levels(&adev->smu, SMU_OD_RANGE, buf+size);
return size;
} else if (adev->powerplay.pp_funcs->print_clock_levels) {
size = amdgpu_dpm_print_clock_levels(adev, OD_SCLK, buf);
@@ -758,7 +777,11 @@ static ssize_t amdgpu_set_ppfeature_status(struct device *dev,
pr_debug("featuremask = 0x%llx\n", featuremask);
- if (adev->powerplay.pp_funcs->set_ppfeature_status) {
+ if (is_support_sw_smu(adev)) {
+ ret = smu_set_ppfeature_status(&adev->smu, featuremask);
+ if (ret)
+ return -EINVAL;
+ } else if (adev->powerplay.pp_funcs->set_ppfeature_status) {
ret = amdgpu_dpm_set_ppfeature_status(adev, featuremask);
if (ret)
return -EINVAL;
@@ -774,7 +797,9 @@ static ssize_t amdgpu_get_ppfeature_status(struct device *dev,
struct drm_device *ddev = dev_get_drvdata(dev);
struct amdgpu_device *adev = ddev->dev_private;
- if (adev->powerplay.pp_funcs->get_ppfeature_status)
+ if (is_support_sw_smu(adev)) {
+ return smu_get_ppfeature_status(&adev->smu, buf);
+ } else if (adev->powerplay.pp_funcs->get_ppfeature_status)
return amdgpu_dpm_get_ppfeature_status(adev, buf);
return snprintf(buf, PAGE_SIZE, "\n");
@@ -817,7 +842,7 @@ static ssize_t amdgpu_get_pp_dpm_sclk(struct device *dev,
return adev->virt.ops->get_pp_clk(adev, PP_SCLK, buf);
if (is_support_sw_smu(adev))
- return smu_print_clk_levels(&adev->smu, PP_SCLK, buf);
+ return smu_print_clk_levels(&adev->smu, SMU_SCLK, buf);
else if (adev->powerplay.pp_funcs->print_clock_levels)
return amdgpu_dpm_print_clock_levels(adev, PP_SCLK, buf);
else
@@ -870,12 +895,15 @@ static ssize_t amdgpu_set_pp_dpm_sclk(struct device *dev,
int ret;
uint32_t mask = 0;
+ if (amdgpu_sriov_vf(adev))
+ return 0;
+
ret = amdgpu_read_mask(buf, count, &mask);
if (ret)
return ret;
if (is_support_sw_smu(adev))
- ret = smu_force_clk_levels(&adev->smu, PP_SCLK, mask);
+ ret = smu_force_clk_levels(&adev->smu, SMU_SCLK, mask);
else if (adev->powerplay.pp_funcs->force_clock_level)
ret = amdgpu_dpm_force_clock_level(adev, PP_SCLK, mask);
@@ -892,8 +920,12 @@ static ssize_t amdgpu_get_pp_dpm_mclk(struct device *dev,
struct drm_device *ddev = dev_get_drvdata(dev);
struct amdgpu_device *adev = ddev->dev_private;
+ if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev) &&
+ adev->virt.ops->get_pp_clk)
+ return adev->virt.ops->get_pp_clk(adev, PP_MCLK, buf);
+
if (is_support_sw_smu(adev))
- return smu_print_clk_levels(&adev->smu, PP_MCLK, buf);
+ return smu_print_clk_levels(&adev->smu, SMU_MCLK, buf);
else if (adev->powerplay.pp_funcs->print_clock_levels)
return amdgpu_dpm_print_clock_levels(adev, PP_MCLK, buf);
else
@@ -910,12 +942,15 @@ static ssize_t amdgpu_set_pp_dpm_mclk(struct device *dev,
int ret;
uint32_t mask = 0;
+ if (amdgpu_sriov_vf(adev))
+ return 0;
+
ret = amdgpu_read_mask(buf, count, &mask);
if (ret)
return ret;
if (is_support_sw_smu(adev))
- ret = smu_force_clk_levels(&adev->smu, PP_MCLK, mask);
+ ret = smu_force_clk_levels(&adev->smu, SMU_MCLK, mask);
else if (adev->powerplay.pp_funcs->force_clock_level)
ret = amdgpu_dpm_force_clock_level(adev, PP_MCLK, mask);
@@ -933,7 +968,7 @@ static ssize_t amdgpu_get_pp_dpm_socclk(struct device *dev,
struct amdgpu_device *adev = ddev->dev_private;
if (is_support_sw_smu(adev))
- return smu_print_clk_levels(&adev->smu, PP_SOCCLK, buf);
+ return smu_print_clk_levels(&adev->smu, SMU_SOCCLK, buf);
else if (adev->powerplay.pp_funcs->print_clock_levels)
return amdgpu_dpm_print_clock_levels(adev, PP_SOCCLK, buf);
else
@@ -955,7 +990,7 @@ static ssize_t amdgpu_set_pp_dpm_socclk(struct device *dev,
return ret;
if (is_support_sw_smu(adev))
- ret = smu_force_clk_levels(&adev->smu, PP_SOCCLK, mask);
+ ret = smu_force_clk_levels(&adev->smu, SMU_SOCCLK, mask);
else if (adev->powerplay.pp_funcs->force_clock_level)
ret = amdgpu_dpm_force_clock_level(adev, PP_SOCCLK, mask);
@@ -973,7 +1008,7 @@ static ssize_t amdgpu_get_pp_dpm_fclk(struct device *dev,
struct amdgpu_device *adev = ddev->dev_private;
if (is_support_sw_smu(adev))
- return smu_print_clk_levels(&adev->smu, PP_FCLK, buf);
+ return smu_print_clk_levels(&adev->smu, SMU_FCLK, buf);
else if (adev->powerplay.pp_funcs->print_clock_levels)
return amdgpu_dpm_print_clock_levels(adev, PP_FCLK, buf);
else
@@ -995,7 +1030,7 @@ static ssize_t amdgpu_set_pp_dpm_fclk(struct device *dev,
return ret;
if (is_support_sw_smu(adev))
- ret = smu_force_clk_levels(&adev->smu, PP_FCLK, mask);
+ ret = smu_force_clk_levels(&adev->smu, SMU_FCLK, mask);
else if (adev->powerplay.pp_funcs->force_clock_level)
ret = amdgpu_dpm_force_clock_level(adev, PP_FCLK, mask);
@@ -1013,7 +1048,7 @@ static ssize_t amdgpu_get_pp_dpm_dcefclk(struct device *dev,
struct amdgpu_device *adev = ddev->dev_private;
if (is_support_sw_smu(adev))
- return smu_print_clk_levels(&adev->smu, PP_DCEFCLK, buf);
+ return smu_print_clk_levels(&adev->smu, SMU_DCEFCLK, buf);
else if (adev->powerplay.pp_funcs->print_clock_levels)
return amdgpu_dpm_print_clock_levels(adev, PP_DCEFCLK, buf);
else
@@ -1035,7 +1070,7 @@ static ssize_t amdgpu_set_pp_dpm_dcefclk(struct device *dev,
return ret;
if (is_support_sw_smu(adev))
- ret = smu_force_clk_levels(&adev->smu, PP_DCEFCLK, mask);
+ ret = smu_force_clk_levels(&adev->smu, SMU_DCEFCLK, mask);
else if (adev->powerplay.pp_funcs->force_clock_level)
ret = amdgpu_dpm_force_clock_level(adev, PP_DCEFCLK, mask);
@@ -1053,7 +1088,7 @@ static ssize_t amdgpu_get_pp_dpm_pcie(struct device *dev,
struct amdgpu_device *adev = ddev->dev_private;
if (is_support_sw_smu(adev))
- return smu_print_clk_levels(&adev->smu, PP_PCIE, buf);
+ return smu_print_clk_levels(&adev->smu, SMU_PCIE, buf);
else if (adev->powerplay.pp_funcs->print_clock_levels)
return amdgpu_dpm_print_clock_levels(adev, PP_PCIE, buf);
else
@@ -1075,7 +1110,7 @@ static ssize_t amdgpu_set_pp_dpm_pcie(struct device *dev,
return ret;
if (is_support_sw_smu(adev))
- ret = smu_force_clk_levels(&adev->smu, PP_PCIE, mask);
+ ret = smu_force_clk_levels(&adev->smu, SMU_PCIE, mask);
else if (adev->powerplay.pp_funcs->force_clock_level)
ret = amdgpu_dpm_force_clock_level(adev, PP_PCIE, mask);
@@ -1094,7 +1129,7 @@ static ssize_t amdgpu_get_pp_sclk_od(struct device *dev,
uint32_t value = 0;
if (is_support_sw_smu(adev))
- value = smu_get_od_percentage(&(adev->smu), OD_SCLK);
+ value = smu_get_od_percentage(&(adev->smu), SMU_OD_SCLK);
else if (adev->powerplay.pp_funcs->get_sclk_od)
value = amdgpu_dpm_get_sclk_od(adev);
@@ -1119,7 +1154,7 @@ static ssize_t amdgpu_set_pp_sclk_od(struct device *dev,
}
if (is_support_sw_smu(adev)) {
- value = smu_set_od_percentage(&(adev->smu), OD_SCLK, (uint32_t)value);
+ value = smu_set_od_percentage(&(adev->smu), SMU_OD_SCLK, (uint32_t)value);
} else {
if (adev->powerplay.pp_funcs->set_sclk_od)
amdgpu_dpm_set_sclk_od(adev, (uint32_t)value);
@@ -1145,7 +1180,7 @@ static ssize_t amdgpu_get_pp_mclk_od(struct device *dev,
uint32_t value = 0;
if (is_support_sw_smu(adev))
- value = smu_get_od_percentage(&(adev->smu), OD_MCLK);
+ value = smu_get_od_percentage(&(adev->smu), SMU_OD_MCLK);
else if (adev->powerplay.pp_funcs->get_mclk_od)
value = amdgpu_dpm_get_mclk_od(adev);
@@ -1170,7 +1205,7 @@ static ssize_t amdgpu_set_pp_mclk_od(struct device *dev,
}
if (is_support_sw_smu(adev)) {
- value = smu_set_od_percentage(&(adev->smu), OD_MCLK, (uint32_t)value);
+ value = smu_set_od_percentage(&(adev->smu), SMU_OD_MCLK, (uint32_t)value);
} else {
if (adev->powerplay.pp_funcs->set_mclk_od)
amdgpu_dpm_set_mclk_od(adev, (uint32_t)value);
@@ -1303,6 +1338,32 @@ static ssize_t amdgpu_get_busy_percent(struct device *dev,
}
/**
+ * DOC: mem_busy_percent
+ *
+ * The amdgpu driver provides a sysfs API for reading how busy the VRAM
+ * is as a percentage. The file mem_busy_percent is used for this.
+ * The SMU firmware computes a percentage of load based on the
+ * aggregate activity level in the IP cores.
+ */
+static ssize_t amdgpu_get_memory_busy_percent(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct drm_device *ddev = dev_get_drvdata(dev);
+ struct amdgpu_device *adev = ddev->dev_private;
+ int r, value, size = sizeof(value);
+
+ /* read the IP busy sensor */
+ r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_MEM_LOAD,
+ (void *)&value, &size);
+
+ if (r)
+ return r;
+
+ return snprintf(buf, PAGE_SIZE, "%d\n", value);
+}
+
+/**
* DOC: pcie_bw
*
* The amdgpu driver provides a sysfs API for estimating how much data
@@ -1327,6 +1388,29 @@ static ssize_t amdgpu_get_pcie_bw(struct device *dev,
count0, count1, pcie_get_mps(adev->pdev));
}
+/**
+ * DOC: unique_id
+ *
+ * The amdgpu driver provides a sysfs API for providing a unique ID for the GPU
+ * The file unique_id is used for this.
+ * This will provide a Unique ID that will persist from machine to machine
+ *
+ * NOTE: This will only work for GFX9 and newer. This file will be absent
+ * on unsupported ASICs (GFX8 and older)
+ */
+static ssize_t amdgpu_get_unique_id(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct drm_device *ddev = dev_get_drvdata(dev);
+ struct amdgpu_device *adev = ddev->dev_private;
+
+ if (adev->unique_id)
+ return snprintf(buf, PAGE_SIZE, "%016llx\n", adev->unique_id);
+
+ return 0;
+}
+
static DEVICE_ATTR(power_dpm_state, S_IRUGO | S_IWUSR, amdgpu_get_dpm_state, amdgpu_set_dpm_state);
static DEVICE_ATTR(power_dpm_force_performance_level, S_IRUGO | S_IWUSR,
amdgpu_get_dpm_forced_performance_level,
@@ -1371,10 +1455,13 @@ static DEVICE_ATTR(pp_od_clk_voltage, S_IRUGO | S_IWUSR,
amdgpu_set_pp_od_clk_voltage);
static DEVICE_ATTR(gpu_busy_percent, S_IRUGO,
amdgpu_get_busy_percent, NULL);
+static DEVICE_ATTR(mem_busy_percent, S_IRUGO,
+ amdgpu_get_memory_busy_percent, NULL);
static DEVICE_ATTR(pcie_bw, S_IRUGO, amdgpu_get_pcie_bw, NULL);
static DEVICE_ATTR(ppfeatures, S_IRUGO | S_IWUSR,
amdgpu_get_ppfeature_status,
amdgpu_set_ppfeature_status);
+static DEVICE_ATTR(unique_id, S_IRUGO, amdgpu_get_unique_id, NULL);
static ssize_t amdgpu_hwmon_show_temp(struct device *dev,
struct device_attribute *attr,
@@ -1382,18 +1469,40 @@ static ssize_t amdgpu_hwmon_show_temp(struct device *dev,
{
struct amdgpu_device *adev = dev_get_drvdata(dev);
struct drm_device *ddev = adev->ddev;
- int r, temp, size = sizeof(temp);
+ int channel = to_sensor_dev_attr(attr)->index;
+ int r, temp = 0, size = sizeof(temp);
/* Can't get temperature when the card is off */
if ((adev->flags & AMD_IS_PX) &&
(ddev->switch_power_state != DRM_SWITCH_POWER_ON))
return -EINVAL;
- /* get the temperature */
- r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_TEMP,
- (void *)&temp, &size);
- if (r)
- return r;
+ if (channel >= PP_TEMP_MAX)
+ return -EINVAL;
+
+ switch (channel) {
+ case PP_TEMP_JUNCTION:
+ /* get current junction temperature */
+ r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_HOTSPOT_TEMP,
+ (void *)&temp, &size);
+ if (r)
+ return r;
+ break;
+ case PP_TEMP_EDGE:
+ /* get current edge temperature */
+ r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_EDGE_TEMP,
+ (void *)&temp, &size);
+ if (r)
+ return r;
+ break;
+ case PP_TEMP_MEM:
+ /* get current memory temperature */
+ r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_MEM_TEMP,
+ (void *)&temp, &size);
+ if (r)
+ return r;
+ break;
+ }
return snprintf(buf, PAGE_SIZE, "%d\n", temp);
}
@@ -1414,6 +1523,76 @@ static ssize_t amdgpu_hwmon_show_temp_thresh(struct device *dev,
return snprintf(buf, PAGE_SIZE, "%d\n", temp);
}
+static ssize_t amdgpu_hwmon_show_hotspot_temp_thresh(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct amdgpu_device *adev = dev_get_drvdata(dev);
+ int hyst = to_sensor_dev_attr(attr)->index;
+ int temp;
+
+ if (hyst)
+ temp = adev->pm.dpm.thermal.min_hotspot_temp;
+ else
+ temp = adev->pm.dpm.thermal.max_hotspot_crit_temp;
+
+ return snprintf(buf, PAGE_SIZE, "%d\n", temp);
+}
+
+static ssize_t amdgpu_hwmon_show_mem_temp_thresh(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct amdgpu_device *adev = dev_get_drvdata(dev);
+ int hyst = to_sensor_dev_attr(attr)->index;
+ int temp;
+
+ if (hyst)
+ temp = adev->pm.dpm.thermal.min_mem_temp;
+ else
+ temp = adev->pm.dpm.thermal.max_mem_crit_temp;
+
+ return snprintf(buf, PAGE_SIZE, "%d\n", temp);
+}
+
+static ssize_t amdgpu_hwmon_show_temp_label(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ int channel = to_sensor_dev_attr(attr)->index;
+
+ if (channel >= PP_TEMP_MAX)
+ return -EINVAL;
+
+ return snprintf(buf, PAGE_SIZE, "%s\n", temp_label[channel].label);
+}
+
+static ssize_t amdgpu_hwmon_show_temp_emergency(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct amdgpu_device *adev = dev_get_drvdata(dev);
+ int channel = to_sensor_dev_attr(attr)->index;
+ int temp = 0;
+
+ if (channel >= PP_TEMP_MAX)
+ return -EINVAL;
+
+ switch (channel) {
+ case PP_TEMP_JUNCTION:
+ temp = adev->pm.dpm.thermal.max_hotspot_emergency_temp;
+ break;
+ case PP_TEMP_EDGE:
+ temp = adev->pm.dpm.thermal.max_edge_emergency_temp;
+ break;
+ case PP_TEMP_MEM:
+ temp = adev->pm.dpm.thermal.max_mem_emergency_temp;
+ break;
+ }
+
+ return snprintf(buf, PAGE_SIZE, "%d\n", temp);
+}
+
static ssize_t amdgpu_hwmon_get_pwm1_enable(struct device *dev,
struct device_attribute *attr,
char *buf)
@@ -1563,7 +1742,7 @@ static ssize_t amdgpu_hwmon_get_fan1_input(struct device *dev,
return -EINVAL;
if (is_support_sw_smu(adev)) {
- err = smu_get_current_rpm(&adev->smu, &speed);
+ err = smu_get_fan_speed_rpm(&adev->smu, &speed);
if (err)
return err;
} else if (adev->powerplay.pp_funcs->get_fan_speed_rpm) {
@@ -1623,7 +1802,7 @@ static ssize_t amdgpu_hwmon_get_fan1_target(struct device *dev,
return -EINVAL;
if (is_support_sw_smu(adev)) {
- err = smu_get_current_rpm(&adev->smu, &rpm);
+ err = smu_get_fan_speed_rpm(&adev->smu, &rpm);
if (err)
return err;
} else if (adev->powerplay.pp_funcs->get_fan_speed_rpm) {
@@ -1906,11 +2085,6 @@ static ssize_t amdgpu_hwmon_show_sclk(struct device *dev,
(ddev->switch_power_state != DRM_SWITCH_POWER_ON))
return -EINVAL;
- /* sanity check PP is enabled */
- if (!(adev->powerplay.pp_funcs &&
- adev->powerplay.pp_funcs->read_sensor))
- return -EINVAL;
-
/* get the sclk */
r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GFX_SCLK,
(void *)&sclk, &size);
@@ -1941,11 +2115,6 @@ static ssize_t amdgpu_hwmon_show_mclk(struct device *dev,
(ddev->switch_power_state != DRM_SWITCH_POWER_ON))
return -EINVAL;
- /* sanity check PP is enabled */
- if (!(adev->powerplay.pp_funcs &&
- adev->powerplay.pp_funcs->read_sensor))
- return -EINVAL;
-
/* get the sclk */
r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GFX_MCLK,
(void *)&mclk, &size);
@@ -1983,11 +2152,20 @@ static ssize_t amdgpu_hwmon_show_mclk_label(struct device *dev,
*
* hwmon interfaces for GPU temperature:
*
- * - temp1_input: the on die GPU temperature in millidegrees Celsius
+ * - temp[1-3]_input: the on die GPU temperature in millidegrees Celsius
+ * - temp2_input and temp3_input are supported on SOC15 dGPUs only
+ *
+ * - temp[1-3]_label: temperature channel label
+ * - temp2_label and temp3_label are supported on SOC15 dGPUs only
+ *
+ * - temp[1-3]_crit: temperature critical max value in millidegrees Celsius
+ * - temp2_crit and temp3_crit are supported on SOC15 dGPUs only
*
- * - temp1_crit: temperature critical max value in millidegrees Celsius
+ * - temp[1-3]_crit_hyst: temperature hysteresis for critical limit in millidegrees Celsius
+ * - temp2_crit_hyst and temp3_crit_hyst are supported on SOC15 dGPUs only
*
- * - temp1_crit_hyst: temperature hysteresis for critical limit in millidegrees Celsius
+ * - temp[1-3]_emergency: temperature emergency max value(asic shutdown) in millidegrees Celsius
+ * - these are supported on SOC15 dGPUs only
*
* hwmon interfaces for GPU voltage:
*
@@ -2035,9 +2213,21 @@ static ssize_t amdgpu_hwmon_show_mclk_label(struct device *dev,
*
*/
-static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, amdgpu_hwmon_show_temp, NULL, 0);
+static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, amdgpu_hwmon_show_temp, NULL, PP_TEMP_EDGE);
static SENSOR_DEVICE_ATTR(temp1_crit, S_IRUGO, amdgpu_hwmon_show_temp_thresh, NULL, 0);
static SENSOR_DEVICE_ATTR(temp1_crit_hyst, S_IRUGO, amdgpu_hwmon_show_temp_thresh, NULL, 1);
+static SENSOR_DEVICE_ATTR(temp1_emergency, S_IRUGO, amdgpu_hwmon_show_temp_emergency, NULL, PP_TEMP_EDGE);
+static SENSOR_DEVICE_ATTR(temp2_input, S_IRUGO, amdgpu_hwmon_show_temp, NULL, PP_TEMP_JUNCTION);
+static SENSOR_DEVICE_ATTR(temp2_crit, S_IRUGO, amdgpu_hwmon_show_hotspot_temp_thresh, NULL, 0);
+static SENSOR_DEVICE_ATTR(temp2_crit_hyst, S_IRUGO, amdgpu_hwmon_show_hotspot_temp_thresh, NULL, 1);
+static SENSOR_DEVICE_ATTR(temp2_emergency, S_IRUGO, amdgpu_hwmon_show_temp_emergency, NULL, PP_TEMP_JUNCTION);
+static SENSOR_DEVICE_ATTR(temp3_input, S_IRUGO, amdgpu_hwmon_show_temp, NULL, PP_TEMP_MEM);
+static SENSOR_DEVICE_ATTR(temp3_crit, S_IRUGO, amdgpu_hwmon_show_mem_temp_thresh, NULL, 0);
+static SENSOR_DEVICE_ATTR(temp3_crit_hyst, S_IRUGO, amdgpu_hwmon_show_mem_temp_thresh, NULL, 1);
+static SENSOR_DEVICE_ATTR(temp3_emergency, S_IRUGO, amdgpu_hwmon_show_temp_emergency, NULL, PP_TEMP_MEM);
+static SENSOR_DEVICE_ATTR(temp1_label, S_IRUGO, amdgpu_hwmon_show_temp_label, NULL, PP_TEMP_EDGE);
+static SENSOR_DEVICE_ATTR(temp2_label, S_IRUGO, amdgpu_hwmon_show_temp_label, NULL, PP_TEMP_JUNCTION);
+static SENSOR_DEVICE_ATTR(temp3_label, S_IRUGO, amdgpu_hwmon_show_temp_label, NULL, PP_TEMP_MEM);
static SENSOR_DEVICE_ATTR(pwm1, S_IRUGO | S_IWUSR, amdgpu_hwmon_get_pwm1, amdgpu_hwmon_set_pwm1, 0);
static SENSOR_DEVICE_ATTR(pwm1_enable, S_IRUGO | S_IWUSR, amdgpu_hwmon_get_pwm1_enable, amdgpu_hwmon_set_pwm1_enable, 0);
static SENSOR_DEVICE_ATTR(pwm1_min, S_IRUGO, amdgpu_hwmon_get_pwm1_min, NULL, 0);
@@ -2064,6 +2254,18 @@ static struct attribute *hwmon_attributes[] = {
&sensor_dev_attr_temp1_input.dev_attr.attr,
&sensor_dev_attr_temp1_crit.dev_attr.attr,
&sensor_dev_attr_temp1_crit_hyst.dev_attr.attr,
+ &sensor_dev_attr_temp2_input.dev_attr.attr,
+ &sensor_dev_attr_temp2_crit.dev_attr.attr,
+ &sensor_dev_attr_temp2_crit_hyst.dev_attr.attr,
+ &sensor_dev_attr_temp3_input.dev_attr.attr,
+ &sensor_dev_attr_temp3_crit.dev_attr.attr,
+ &sensor_dev_attr_temp3_crit_hyst.dev_attr.attr,
+ &sensor_dev_attr_temp1_emergency.dev_attr.attr,
+ &sensor_dev_attr_temp2_emergency.dev_attr.attr,
+ &sensor_dev_attr_temp3_emergency.dev_attr.attr,
+ &sensor_dev_attr_temp1_label.dev_attr.attr,
+ &sensor_dev_attr_temp2_label.dev_attr.attr,
+ &sensor_dev_attr_temp3_label.dev_attr.attr,
&sensor_dev_attr_pwm1.dev_attr.attr,
&sensor_dev_attr_pwm1_enable.dev_attr.attr,
&sensor_dev_attr_pwm1_min.dev_attr.attr,
@@ -2186,6 +2388,22 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
attr == &sensor_dev_attr_freq2_label.dev_attr.attr))
return 0;
+ /* only SOC15 dGPUs support hotspot and mem temperatures */
+ if (((adev->flags & AMD_IS_APU) ||
+ adev->asic_type < CHIP_VEGA10) &&
+ (attr == &sensor_dev_attr_temp2_crit.dev_attr.attr ||
+ attr == &sensor_dev_attr_temp2_crit_hyst.dev_attr.attr ||
+ attr == &sensor_dev_attr_temp3_crit.dev_attr.attr ||
+ attr == &sensor_dev_attr_temp3_crit_hyst.dev_attr.attr ||
+ attr == &sensor_dev_attr_temp1_emergency.dev_attr.attr ||
+ attr == &sensor_dev_attr_temp2_emergency.dev_attr.attr ||
+ attr == &sensor_dev_attr_temp3_emergency.dev_attr.attr ||
+ attr == &sensor_dev_attr_temp2_input.dev_attr.attr ||
+ attr == &sensor_dev_attr_temp3_input.dev_attr.attr ||
+ attr == &sensor_dev_attr_temp2_label.dev_attr.attr ||
+ attr == &sensor_dev_attr_temp3_label.dev_attr.attr))
+ return 0;
+
return effective_mode;
}
@@ -2490,6 +2708,44 @@ void amdgpu_pm_print_power_states(struct amdgpu_device *adev)
}
+int amdgpu_pm_virt_sysfs_init(struct amdgpu_device *adev)
+{
+ int ret = 0;
+
+ if (!(amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev)))
+ return ret;
+
+ ret = device_create_file(adev->dev, &dev_attr_pp_dpm_sclk);
+ if (ret) {
+ DRM_ERROR("failed to create device file pp_dpm_sclk\n");
+ return ret;
+ }
+
+ ret = device_create_file(adev->dev, &dev_attr_pp_dpm_mclk);
+ if (ret) {
+ DRM_ERROR("failed to create device file pp_dpm_mclk\n");
+ return ret;
+ }
+
+ ret = device_create_file(adev->dev, &dev_attr_power_dpm_force_performance_level);
+ if (ret) {
+ DRM_ERROR("failed to create device file for dpm state\n");
+ return ret;
+ }
+
+ return ret;
+}
+
+void amdgpu_pm_virt_sysfs_fini(struct amdgpu_device *adev)
+{
+ if (!(amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev)))
+ return;
+
+ device_remove_file(adev->dev, &dev_attr_power_dpm_force_performance_level);
+ device_remove_file(adev->dev, &dev_attr_pp_dpm_sclk);
+ device_remove_file(adev->dev, &dev_attr_pp_dpm_mclk);
+}
+
int amdgpu_pm_load_smu_firmware(struct amdgpu_device *adev, uint32_t *smu_version)
{
int r;
@@ -2627,6 +2883,17 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
"gpu_busy_level\n");
return ret;
}
+ /* APU does not have its own dedicated memory */
+ if (!(adev->flags & AMD_IS_APU) &&
+ (adev->asic_type != CHIP_VEGA10)) {
+ ret = device_create_file(adev->dev,
+ &dev_attr_mem_busy_percent);
+ if (ret) {
+ DRM_ERROR("failed to create device file "
+ "mem_busy_percent\n");
+ return ret;
+ }
+ }
/* PCIe Perf counters won't work on APU nodes */
if (!(adev->flags & AMD_IS_APU)) {
ret = device_create_file(adev->dev, &dev_attr_pcie_bw);
@@ -2635,6 +2902,12 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
return ret;
}
}
+ if (adev->unique_id)
+ ret = device_create_file(adev->dev, &dev_attr_unique_id);
+ if (ret) {
+ DRM_ERROR("failed to create device file unique_id\n");
+ return ret;
+ }
ret = amdgpu_debugfs_pm_init(adev);
if (ret) {
DRM_ERROR("Failed to register debugfs file for dpm!\n");
@@ -2692,8 +2965,13 @@ void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev)
device_remove_file(adev->dev,
&dev_attr_pp_od_clk_voltage);
device_remove_file(adev->dev, &dev_attr_gpu_busy_percent);
+ if (!(adev->flags & AMD_IS_APU) &&
+ (adev->asic_type != CHIP_VEGA10))
+ device_remove_file(adev->dev, &dev_attr_mem_busy_percent);
if (!(adev->flags & AMD_IS_APU))
device_remove_file(adev->dev, &dev_attr_pcie_bw);
+ if (adev->unique_id)
+ device_remove_file(adev->dev, &dev_attr_unique_id);
if ((adev->asic_type >= CHIP_VEGA10) &&
!(adev->flags & AMD_IS_APU))
device_remove_file(adev->dev, &dev_attr_ppfeatures);
@@ -2716,13 +2994,10 @@ void amdgpu_pm_compute_clocks(struct amdgpu_device *adev)
}
if (is_support_sw_smu(adev)) {
- struct smu_context *smu = &adev->smu;
struct smu_dpm_context *smu_dpm = &adev->smu.smu_dpm;
- mutex_lock(&(smu->mutex));
smu_handle_task(&adev->smu,
smu_dpm->dpm_level,
AMD_PP_TASK_DISPLAY_CONFIG_CHANGE);
- mutex_unlock(&(smu->mutex));
} else {
if (adev->powerplay.pp_funcs->dispatch_tasks) {
if (!amdgpu_device_has_dc_support(adev)) {
@@ -2790,34 +3065,54 @@ static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *a
/* GPU Load */
if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_LOAD, (void *)&value, &size))
seq_printf(m, "GPU Load: %u %%\n", value);
+ /* MEM Load */
+ if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_MEM_LOAD, (void *)&value, &size))
+ seq_printf(m, "MEM Load: %u %%\n", value);
+
seq_printf(m, "\n");
/* SMC feature mask */
if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_ENABLED_SMC_FEATURES_MASK, (void *)&value64, &size))
seq_printf(m, "SMC Feature Mask: 0x%016llx\n", value64);
- /* UVD clocks */
- if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_UVD_POWER, (void *)&value, &size)) {
- if (!value) {
- seq_printf(m, "UVD: Disabled\n");
- } else {
- seq_printf(m, "UVD: Enabled\n");
- if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_UVD_DCLK, (void *)&value, &size))
- seq_printf(m, "\t%u MHz (DCLK)\n", value/100);
- if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_UVD_VCLK, (void *)&value, &size))
- seq_printf(m, "\t%u MHz (VCLK)\n", value/100);
+ if (adev->asic_type > CHIP_VEGA20) {
+ /* VCN clocks */
+ if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VCN_POWER_STATE, (void *)&value, &size)) {
+ if (!value) {
+ seq_printf(m, "VCN: Disabled\n");
+ } else {
+ seq_printf(m, "VCN: Enabled\n");
+ if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_UVD_DCLK, (void *)&value, &size))
+ seq_printf(m, "\t%u MHz (DCLK)\n", value/100);
+ if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_UVD_VCLK, (void *)&value, &size))
+ seq_printf(m, "\t%u MHz (VCLK)\n", value/100);
+ }
}
- }
- seq_printf(m, "\n");
+ seq_printf(m, "\n");
+ } else {
+ /* UVD clocks */
+ if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_UVD_POWER, (void *)&value, &size)) {
+ if (!value) {
+ seq_printf(m, "UVD: Disabled\n");
+ } else {
+ seq_printf(m, "UVD: Enabled\n");
+ if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_UVD_DCLK, (void *)&value, &size))
+ seq_printf(m, "\t%u MHz (DCLK)\n", value/100);
+ if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_UVD_VCLK, (void *)&value, &size))
+ seq_printf(m, "\t%u MHz (VCLK)\n", value/100);
+ }
+ }
+ seq_printf(m, "\n");
- /* VCE clocks */
- if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VCE_POWER, (void *)&value, &size)) {
- if (!value) {
- seq_printf(m, "VCE: Disabled\n");
- } else {
- seq_printf(m, "VCE: Enabled\n");
- if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VCE_ECCLK, (void *)&value, &size))
- seq_printf(m, "\t%u MHz (ECCLK)\n", value/100);
+ /* VCE clocks */
+ if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VCE_POWER, (void *)&value, &size)) {
+ if (!value) {
+ seq_printf(m, "VCE: Disabled\n");
+ } else {
+ seq_printf(m, "VCE: Enabled\n");
+ if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VCE_ECCLK, (void *)&value, &size))
+ seq_printf(m, "\t%u MHz (ECCLK)\n", value/100);
+ }
}
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.h
index 7ff0e7621fff..ef31448ee8d8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.h
@@ -32,7 +32,9 @@ struct cg_flag_name
void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev);
int amdgpu_pm_sysfs_init(struct amdgpu_device *adev);
+int amdgpu_pm_virt_sysfs_init(struct amdgpu_device *adev);
void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev);
+void amdgpu_pm_virt_sysfs_fini(struct amdgpu_device *adev);
void amdgpu_pm_print_power_states(struct amdgpu_device *adev);
int amdgpu_pm_load_smu_firmware(struct amdgpu_device *adev, uint32_t *smu_version);
void amdgpu_pm_compute_clocks(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c
new file mode 100644
index 000000000000..0e6dba9f60f0
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.c
@@ -0,0 +1,280 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Author: Jonathan Kim <jonathan.kim@amd.com>
+ *
+ */
+
+#include <linux/perf_event.h>
+#include <linux/init.h>
+#include "amdgpu.h"
+#include "amdgpu_pmu.h"
+#include "df_v3_6.h"
+
+#define PMU_NAME_SIZE 32
+
+/* record to keep track of pmu entry per pmu type per device */
+struct amdgpu_pmu_entry {
+ struct list_head entry;
+ struct amdgpu_device *adev;
+ struct pmu pmu;
+ unsigned int pmu_perf_type;
+};
+
+static LIST_HEAD(amdgpu_pmu_list);
+
+
+/* initialize perf counter */
+static int amdgpu_perf_event_init(struct perf_event *event)
+{
+ struct hw_perf_event *hwc = &event->hw;
+
+ /* test the event attr type check for PMU enumeration */
+ if (event->attr.type != event->pmu->type)
+ return -ENOENT;
+
+ /* update the hw_perf_event struct with config data */
+ hwc->conf = event->attr.config;
+
+ return 0;
+}
+
+/* start perf counter */
+static void amdgpu_perf_start(struct perf_event *event, int flags)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ struct amdgpu_pmu_entry *pe = container_of(event->pmu,
+ struct amdgpu_pmu_entry,
+ pmu);
+
+ if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
+ return;
+
+ WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
+ hwc->state = 0;
+
+ switch (pe->pmu_perf_type) {
+ case PERF_TYPE_AMDGPU_DF:
+ if (!(flags & PERF_EF_RELOAD))
+ pe->adev->df_funcs->pmc_start(pe->adev, hwc->conf, 1);
+
+ pe->adev->df_funcs->pmc_start(pe->adev, hwc->conf, 0);
+ break;
+ default:
+ break;
+ }
+
+ perf_event_update_userpage(event);
+
+}
+
+/* read perf counter */
+static void amdgpu_perf_read(struct perf_event *event)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ struct amdgpu_pmu_entry *pe = container_of(event->pmu,
+ struct amdgpu_pmu_entry,
+ pmu);
+
+ u64 count, prev;
+
+ do {
+ prev = local64_read(&hwc->prev_count);
+
+ switch (pe->pmu_perf_type) {
+ case PERF_TYPE_AMDGPU_DF:
+ pe->adev->df_funcs->pmc_get_count(pe->adev, hwc->conf,
+ &count);
+ break;
+ default:
+ count = 0;
+ break;
+ };
+ } while (local64_cmpxchg(&hwc->prev_count, prev, count) != prev);
+
+ local64_add(count - prev, &event->count);
+}
+
+/* stop perf counter */
+static void amdgpu_perf_stop(struct perf_event *event, int flags)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ struct amdgpu_pmu_entry *pe = container_of(event->pmu,
+ struct amdgpu_pmu_entry,
+ pmu);
+
+ if (hwc->state & PERF_HES_UPTODATE)
+ return;
+
+ switch (pe->pmu_perf_type) {
+ case PERF_TYPE_AMDGPU_DF:
+ pe->adev->df_funcs->pmc_stop(pe->adev, hwc->conf, 0);
+ break;
+ default:
+ break;
+ };
+
+ WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
+ hwc->state |= PERF_HES_STOPPED;
+
+ if (hwc->state & PERF_HES_UPTODATE)
+ return;
+
+ amdgpu_perf_read(event);
+ hwc->state |= PERF_HES_UPTODATE;
+}
+
+/* add perf counter */
+static int amdgpu_perf_add(struct perf_event *event, int flags)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ int retval;
+
+ struct amdgpu_pmu_entry *pe = container_of(event->pmu,
+ struct amdgpu_pmu_entry,
+ pmu);
+
+ event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
+
+ switch (pe->pmu_perf_type) {
+ case PERF_TYPE_AMDGPU_DF:
+ retval = pe->adev->df_funcs->pmc_start(pe->adev, hwc->conf, 1);
+ break;
+ default:
+ return 0;
+ };
+
+ if (retval)
+ return retval;
+
+ if (flags & PERF_EF_START)
+ amdgpu_perf_start(event, PERF_EF_RELOAD);
+
+ return retval;
+
+}
+
+/* delete perf counter */
+static void amdgpu_perf_del(struct perf_event *event, int flags)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ struct amdgpu_pmu_entry *pe = container_of(event->pmu,
+ struct amdgpu_pmu_entry,
+ pmu);
+
+ amdgpu_perf_stop(event, PERF_EF_UPDATE);
+
+ switch (pe->pmu_perf_type) {
+ case PERF_TYPE_AMDGPU_DF:
+ pe->adev->df_funcs->pmc_stop(pe->adev, hwc->conf, 1);
+ break;
+ default:
+ break;
+ };
+
+ perf_event_update_userpage(event);
+}
+
+/* vega20 pmus */
+
+/* init pmu tracking per pmu type */
+static int init_pmu_by_type(struct amdgpu_device *adev,
+ const struct attribute_group *attr_groups[],
+ char *pmu_type_name, char *pmu_file_prefix,
+ unsigned int pmu_perf_type,
+ unsigned int num_counters)
+{
+ char pmu_name[PMU_NAME_SIZE];
+ struct amdgpu_pmu_entry *pmu_entry;
+ int ret = 0;
+
+ pmu_entry = kzalloc(sizeof(struct amdgpu_pmu_entry), GFP_KERNEL);
+
+ if (!pmu_entry)
+ return -ENOMEM;
+
+ pmu_entry->adev = adev;
+ pmu_entry->pmu = (struct pmu){
+ .event_init = amdgpu_perf_event_init,
+ .add = amdgpu_perf_add,
+ .del = amdgpu_perf_del,
+ .start = amdgpu_perf_start,
+ .stop = amdgpu_perf_stop,
+ .read = amdgpu_perf_read,
+ .task_ctx_nr = perf_invalid_context,
+ };
+
+ pmu_entry->pmu.attr_groups = attr_groups;
+ pmu_entry->pmu_perf_type = pmu_perf_type;
+ snprintf(pmu_name, PMU_NAME_SIZE, "%s_%d",
+ pmu_file_prefix, adev->ddev->primary->index);
+
+ ret = perf_pmu_register(&pmu_entry->pmu, pmu_name, -1);
+
+ if (ret) {
+ kfree(pmu_entry);
+ pr_warn("Error initializing AMDGPU %s PMUs.\n", pmu_type_name);
+ return ret;
+ }
+
+ pr_info("Detected AMDGPU %s Counters. # of Counters = %d.\n",
+ pmu_type_name, num_counters);
+
+ list_add_tail(&pmu_entry->entry, &amdgpu_pmu_list);
+
+ return 0;
+}
+
+/* init amdgpu_pmu */
+int amdgpu_pmu_init(struct amdgpu_device *adev)
+{
+ int ret = 0;
+
+ switch (adev->asic_type) {
+ case CHIP_VEGA20:
+ /* init df */
+ ret = init_pmu_by_type(adev, df_v3_6_attr_groups,
+ "DF", "amdgpu_df", PERF_TYPE_AMDGPU_DF,
+ DF_V3_6_MAX_COUNTERS);
+
+ /* other pmu types go here*/
+ break;
+ default:
+ return 0;
+ }
+
+ return 0;
+}
+
+
+/* destroy all pmu data associated with target device */
+void amdgpu_pmu_fini(struct amdgpu_device *adev)
+{
+ struct amdgpu_pmu_entry *pe, *temp;
+
+ list_for_each_entry_safe(pe, temp, &amdgpu_pmu_list, entry) {
+ if (pe->adev == adev) {
+ list_del(&pe->entry);
+ perf_pmu_unregister(&pe->pmu);
+ kfree(pe);
+ }
+ }
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h
new file mode 100644
index 000000000000..7dddb7160a11
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pmu.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Author: Jonathan Kim <jonathan.kim@amd.com>
+ *
+ */
+
+#ifndef _AMDGPU_PMU_H_
+#define _AMDGPU_PMU_H_
+
+enum amdgpu_pmu_perf_type {
+ PERF_TYPE_AMDGPU_DF = 0,
+ PERF_TYPE_AMDGPU_MAX
+};
+
+int amdgpu_pmu_init(struct amdgpu_device *adev);
+void amdgpu_pmu_fini(struct amdgpu_device *adev);
+
+#endif /* _AMDGPU_PMU_H_ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 86cc24b2e0aa..c027e5e7713e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -24,7 +24,7 @@
*/
#include <linux/firmware.h>
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_psp.h"
#include "amdgpu_ucode.h"
@@ -46,12 +46,19 @@ static int psp_early_init(void *handle)
case CHIP_VEGA10:
case CHIP_VEGA12:
psp_v3_1_set_psp_funcs(psp);
+ psp->autoload_supported = false;
break;
case CHIP_RAVEN:
psp_v10_0_set_psp_funcs(psp);
+ psp->autoload_supported = false;
break;
case CHIP_VEGA20:
psp_v11_0_set_psp_funcs(psp);
+ psp->autoload_supported = false;
+ break;
+ case CHIP_NAVI10:
+ psp_v11_0_set_psp_funcs(psp);
+ psp->autoload_supported = true;
break;
default:
return -EINVAL;
@@ -123,6 +130,8 @@ psp_cmd_submit_buf(struct psp_context *psp,
int index;
int timeout = 2000;
+ mutex_lock(&psp->mutex);
+
memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
@@ -132,6 +141,7 @@ psp_cmd_submit_buf(struct psp_context *psp,
fence_mc_addr, index);
if (ret) {
atomic_dec(&psp->fence_value);
+ mutex_unlock(&psp->mutex);
return ret;
}
@@ -154,8 +164,10 @@ psp_cmd_submit_buf(struct psp_context *psp,
ucode->ucode_id);
DRM_WARN("psp command failed and response status is (%d)\n",
psp->cmd_buf_mem->resp.status);
- if (!timeout)
+ if (!timeout) {
+ mutex_unlock(&psp->mutex);
return -EINVAL;
+ }
}
/* get xGMI session id from response buffer */
@@ -165,6 +177,7 @@ psp_cmd_submit_buf(struct psp_context *psp,
ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
}
+ mutex_unlock(&psp->mutex);
return ret;
}
@@ -182,10 +195,44 @@ static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
cmd->cmd.cmd_setup_tmr.buf_size = size;
}
+static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd,
+ uint64_t pri_buf_mc, uint32_t size)
+{
+ cmd->cmd_id = GFX_CMD_ID_LOAD_TOC;
+ cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc);
+ cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc);
+ cmd->cmd.cmd_load_toc.toc_size = size;
+}
+
+/* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */
+static int psp_load_toc(struct psp_context *psp,
+ uint32_t *tmr_size)
+{
+ int ret;
+ struct psp_gfx_cmd_resp *cmd;
+
+ cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
+ if (!cmd)
+ return -ENOMEM;
+ /* Copy toc to psp firmware private buffer */
+ memset(psp->fw_pri_buf, 0, PSP_1_MEG);
+ memcpy(psp->fw_pri_buf, psp->toc_start_addr, psp->toc_bin_size);
+
+ psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc_bin_size);
+
+ ret = psp_cmd_submit_buf(psp, NULL, cmd,
+ psp->fence_buf_mc_addr);
+ if (!ret)
+ *tmr_size = psp->cmd_buf_mem->resp.tmr_size;
+ kfree(cmd);
+ return ret;
+}
+
/* Set up Trusted Memory Region */
static int psp_tmr_init(struct psp_context *psp)
{
int ret;
+ int tmr_size;
/*
* According to HW engineer, they prefer the TMR address be "naturally
@@ -194,7 +241,21 @@ static int psp_tmr_init(struct psp_context *psp)
* Note: this memory need be reserved till the driver
* uninitializes.
*/
- ret = amdgpu_bo_create_kernel(psp->adev, PSP_TMR_SIZE, PSP_TMR_SIZE,
+ tmr_size = PSP_TMR_SIZE;
+
+ /* For ASICs support RLC autoload, psp will parse the toc
+ * and calculate the total size of TMR needed */
+ if (psp->toc_start_addr &&
+ psp->toc_bin_size &&
+ psp->fw_pri_buf) {
+ ret = psp_load_toc(psp, &tmr_size);
+ if (ret) {
+ DRM_ERROR("Failed to load toc\n");
+ return ret;
+ }
+ }
+
+ ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_SIZE,
AMDGPU_GEM_DOMAIN_VRAM,
&psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
@@ -210,9 +271,10 @@ static int psp_tmr_load(struct psp_context *psp)
if (!cmd)
return -ENOMEM;
- psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr, PSP_TMR_SIZE);
- DRM_INFO("reserve 0x%x from 0x%llx for PSP TMR SIZE\n",
- PSP_TMR_SIZE, psp->tmr_mc_addr);
+ psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr,
+ amdgpu_bo_size(psp->tmr_bo));
+ DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n",
+ amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr);
ret = psp_cmd_submit_buf(psp, NULL, cmd,
psp->fence_buf_mc_addr);
@@ -289,6 +351,34 @@ static int psp_asd_load(struct psp_context *psp)
return ret;
}
+static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
+ uint32_t id, uint32_t value)
+{
+ cmd->cmd_id = GFX_CMD_ID_PROG_REG;
+ cmd->cmd.cmd_setup_reg_prog.reg_value = value;
+ cmd->cmd.cmd_setup_reg_prog.reg_id = id;
+}
+
+int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
+ uint32_t value)
+{
+ struct psp_gfx_cmd_resp *cmd = NULL;
+ int ret = 0;
+
+ if (reg >= PSP_REG_LAST)
+ return -EINVAL;
+
+ cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
+ if (!cmd)
+ return -ENOMEM;
+
+ psp_prep_reg_prog_cmd_buf(cmd, reg, value);
+ ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
+
+ kfree(cmd);
+ return ret;
+}
+
static void psp_prep_xgmi_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
uint64_t xgmi_ta_mc, uint64_t xgmi_mc_shared,
uint32_t xgmi_ta_size, uint32_t shared_size)
@@ -679,6 +769,15 @@ static int psp_hw_start(struct psp_context *psp)
int ret;
if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) {
+ if (psp->kdb_bin_size &&
+ (psp->funcs->bootloader_load_kdb != NULL)) {
+ ret = psp_bootloader_load_kdb(psp);
+ if (ret) {
+ DRM_ERROR("PSP load kdb failed!\n");
+ return ret;
+ }
+ }
+
ret = psp_bootloader_load_sysdrv(psp);
if (ret) {
DRM_ERROR("PSP load sysdrv failed!\n");
@@ -698,12 +797,24 @@ static int psp_hw_start(struct psp_context *psp)
return ret;
}
+ ret = psp_tmr_init(psp);
+ if (ret) {
+ DRM_ERROR("PSP tmr init failed!\n");
+ return ret;
+ }
+
ret = psp_tmr_load(psp);
if (ret) {
DRM_ERROR("PSP load tmr failed!\n");
return ret;
}
+ ret = psp_asd_init(psp);
+ if (ret) {
+ DRM_ERROR("PSP asd init failed!\n");
+ return ret;
+ }
+
ret = psp_asd_load(psp);
if (ret) {
DRM_ERROR("PSP load asd failed!\n");
@@ -795,6 +906,12 @@ static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
case AMDGPU_UCODE_ID_DMCU_INTV:
*type = GFX_FW_TYPE_DMCU_ISR;
break;
+ case AMDGPU_UCODE_ID_VCN0_RAM:
+ *type = GFX_FW_TYPE_VCN0_RAM;
+ break;
+ case AMDGPU_UCODE_ID_VCN1_RAM:
+ *type = GFX_FW_TYPE_VCN1_RAM;
+ break;
case AMDGPU_UCODE_ID_MAXIMUM:
default:
return -EINVAL;
@@ -823,19 +940,45 @@ static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
return ret;
}
+static int psp_execute_np_fw_load(struct psp_context *psp,
+ struct amdgpu_firmware_info *ucode)
+{
+ int ret = 0;
+
+ ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd);
+ if (ret)
+ return ret;
+
+ ret = psp_cmd_submit_buf(psp, ucode, psp->cmd,
+ psp->fence_buf_mc_addr);
+
+ return ret;
+}
+
static int psp_np_fw_load(struct psp_context *psp)
{
int i, ret;
struct amdgpu_firmware_info *ucode;
struct amdgpu_device* adev = psp->adev;
+ if (psp->autoload_supported) {
+ ucode = &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
+ if (!ucode->fw)
+ goto out;
+
+ ret = psp_execute_np_fw_load(psp, ucode);
+ if (ret)
+ return ret;
+ }
+
+out:
for (i = 0; i < adev->firmware.max_ucodes; i++) {
ucode = &adev->firmware.ucode[i];
if (!ucode->fw)
continue;
if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
- psp_smu_reload_quirk(psp))
+ (psp_smu_reload_quirk(psp) || psp->autoload_supported))
continue;
if (amdgpu_sriov_vf(adev) &&
(ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
@@ -843,16 +986,24 @@ static int psp_np_fw_load(struct psp_context *psp)
|| ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G))
/*skip ucode loading in SRIOV VF */
continue;
+ if (psp->autoload_supported &&
+ (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT ||
+ ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT))
+ /* skip mec JT when autoload is enabled */
+ continue;
- ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd);
- if (ret)
- return ret;
-
- ret = psp_cmd_submit_buf(psp, ucode, psp->cmd,
- psp->fence_buf_mc_addr);
+ ret = psp_execute_np_fw_load(psp, ucode);
if (ret)
return ret;
+ /* Start rlc autoload after psp recieved all the gfx firmware */
+ if (ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM) {
+ ret = psp_rlc_autoload(psp);
+ if (ret) {
+ DRM_ERROR("Failed to start rlc autoload\n");
+ return ret;
+ }
+ }
#if 0
/* check if firmware loaded sucessfully */
if (!amdgpu_psp_check_fw_loading_status(adev, i))
@@ -911,18 +1062,6 @@ static int psp_load_fw(struct amdgpu_device *adev)
goto failed;
}
- ret = psp_tmr_init(psp);
- if (ret) {
- DRM_ERROR("PSP tmr init failed!\n");
- goto failed;
- }
-
- ret = psp_asd_init(psp);
- if (ret) {
- DRM_ERROR("PSP asd init failed!\n");
- goto failed;
- }
-
skip_memalloc:
ret = psp_hw_start(psp);
if (ret)
@@ -1064,10 +1203,49 @@ failed:
int psp_gpu_reset(struct amdgpu_device *adev)
{
+ int ret;
+
if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
return 0;
- return psp_mode1_reset(&adev->psp);
+ mutex_lock(&adev->psp.mutex);
+ ret = psp_mode1_reset(&adev->psp);
+ mutex_unlock(&adev->psp.mutex);
+
+ return ret;
+}
+
+int psp_rlc_autoload_start(struct psp_context *psp)
+{
+ int ret;
+ struct psp_gfx_cmd_resp *cmd;
+
+ if (amdgpu_sriov_vf(psp->adev))
+ return 0;
+
+ cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
+ if (!cmd)
+ return -ENOMEM;
+
+ cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC;
+
+ ret = psp_cmd_submit_buf(psp, NULL, cmd,
+ psp->fence_buf_mc_addr);
+ kfree(cmd);
+ return ret;
+}
+
+int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx,
+ uint64_t cmd_gpu_addr, int cmd_size)
+{
+ struct amdgpu_firmware_info ucode = {0};
+
+ ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM :
+ AMDGPU_UCODE_ID_VCN0_RAM;
+ ucode.mc_addr = cmd_gpu_addr;
+ ucode.ucode_size = cmd_size;
+
+ return psp_execute_np_fw_load(&adev->psp, &ucode);
}
static bool psp_check_fw_loading_status(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
index cde113f07c96..e0fc2a790e53 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
@@ -42,6 +42,12 @@ struct psp_context;
struct psp_xgmi_node_info;
struct psp_xgmi_topology_info;
+enum psp_bootloader_cmd {
+ PSP_BL__LOAD_SYSDRV = 0x10000,
+ PSP_BL__LOAD_SOSDRV = 0x20000,
+ PSP_BL__LOAD_KEY_DATABASE = 0x80000,
+};
+
enum psp_ring_type
{
PSP_RING_TYPE__INVALID = 0,
@@ -62,9 +68,18 @@ struct psp_ring
uint32_t ring_size;
};
+/* More registers may will be supported */
+enum psp_reg_prog_id {
+ PSP_REG_IH_RB_CNTL = 0, /* register IH_RB_CNTL */
+ PSP_REG_IH_RB_CNTL_RING1 = 1, /* register IH_RB_CNTL_RING1 */
+ PSP_REG_IH_RB_CNTL_RING2 = 2, /* register IH_RB_CNTL_RING2 */
+ PSP_REG_LAST
+};
+
struct psp_funcs
{
int (*init_microcode)(struct psp_context *psp);
+ int (*bootloader_load_kdb)(struct psp_context *psp);
int (*bootloader_load_sysdrv)(struct psp_context *psp);
int (*bootloader_load_sos)(struct psp_context *psp);
int (*ring_init)(struct psp_context *psp, enum psp_ring_type ring_type);
@@ -93,6 +108,20 @@ struct psp_funcs
int (*ras_trigger_error)(struct psp_context *psp,
struct ta_ras_trigger_error_input *info);
int (*ras_cure_posion)(struct psp_context *psp, uint64_t *mode_ptr);
+ int (*rlc_autoload_start)(struct psp_context *psp);
+};
+
+#define AMDGPU_XGMI_MAX_CONNECTED_NODES 64
+struct psp_xgmi_node_info {
+ uint64_t node_id;
+ uint8_t num_hops;
+ uint8_t is_sharing_enabled;
+ enum ta_xgmi_assigned_sdma_engine sdma_engine;
+};
+
+struct psp_xgmi_topology_info {
+ uint32_t num_nodes;
+ struct psp_xgmi_node_info nodes[AMDGPU_XGMI_MAX_CONNECTED_NODES];
};
struct psp_xgmi_context {
@@ -101,6 +130,7 @@ struct psp_xgmi_context {
struct amdgpu_bo *xgmi_shared_bo;
uint64_t xgmi_shared_mc_addr;
void *xgmi_shared_buf;
+ struct psp_xgmi_topology_info top_info;
};
struct psp_ras_context {
@@ -132,8 +162,12 @@ struct psp_context
uint32_t sos_feature_version;
uint32_t sys_bin_size;
uint32_t sos_bin_size;
+ uint32_t toc_bin_size;
+ uint32_t kdb_bin_size;
uint8_t *sys_start_addr;
uint8_t *sos_start_addr;
+ uint8_t *toc_start_addr;
+ uint8_t *kdb_start_addr;
/* tmr buffer */
struct amdgpu_bo *tmr_bo;
@@ -162,6 +196,8 @@ struct psp_context
/* fence value associated with cmd buffer */
atomic_t fence_value;
+ /* flag to mark whether gfx fw autoload is supported or not */
+ bool autoload_supported;
/* xgmi ta firmware and buffer */
const struct firmware *ta_fw;
@@ -174,6 +210,7 @@ struct psp_context
uint8_t *ta_ras_start_addr;
struct psp_xgmi_context xgmi_context;
struct psp_ras_context ras;
+ struct mutex mutex;
};
struct amdgpu_psp_funcs {
@@ -181,18 +218,6 @@ struct amdgpu_psp_funcs {
enum AMDGPU_UCODE_ID);
};
-#define AMDGPU_XGMI_MAX_CONNECTED_NODES 64
-struct psp_xgmi_node_info {
- uint64_t node_id;
- uint8_t num_hops;
- uint8_t is_sharing_enabled;
- enum ta_xgmi_assigned_sdma_engine sdma_engine;
-};
-
-struct psp_xgmi_topology_info {
- uint32_t num_nodes;
- struct psp_xgmi_node_info nodes[AMDGPU_XGMI_MAX_CONNECTED_NODES];
-};
#define psp_ring_init(psp, type) (psp)->funcs->ring_init((psp), (type))
#define psp_ring_create(psp, type) (psp)->funcs->ring_create((psp), (type))
@@ -204,6 +229,8 @@ struct psp_xgmi_topology_info {
(psp)->funcs->compare_sram_data((psp), (ucode), (type))
#define psp_init_microcode(psp) \
((psp)->funcs->init_microcode ? (psp)->funcs->init_microcode((psp)) : 0)
+#define psp_bootloader_load_kdb(psp) \
+ ((psp)->funcs->bootloader_load_kdb ? (psp)->funcs->bootloader_load_kdb((psp)) : 0)
#define psp_bootloader_load_sysdrv(psp) \
((psp)->funcs->bootloader_load_sysdrv ? (psp)->funcs->bootloader_load_sysdrv((psp)) : 0)
#define psp_bootloader_load_sos(psp) \
@@ -224,6 +251,8 @@ struct psp_xgmi_topology_info {
#define psp_xgmi_set_topology_info(psp, num_device, topology) \
((psp)->funcs->xgmi_set_topology_info ? \
(psp)->funcs->xgmi_set_topology_info((psp), (num_device), (topology)) : -EINVAL)
+#define psp_rlc_autoload(psp) \
+ ((psp)->funcs->rlc_autoload_start ? (psp)->funcs->rlc_autoload_start((psp)) : 0)
#define amdgpu_psp_check_fw_loading_status(adev, i) (adev)->firmware.funcs->check_fw_loading_status((adev), (i))
@@ -243,12 +272,18 @@ extern int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
extern const struct amdgpu_ip_block_version psp_v10_0_ip_block;
int psp_gpu_reset(struct amdgpu_device *adev);
+int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx,
+ uint64_t cmd_gpu_addr, int cmd_size);
+
int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id);
int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id);
int psp_ras_enable_features(struct psp_context *psp,
union ta_ras_cmd_input *info, bool enable);
-extern const struct amdgpu_ip_block_version psp_v11_0_ip_block;
+int psp_rlc_autoload_start(struct psp_context *psp);
+extern const struct amdgpu_ip_block_version psp_v11_0_ip_block;
+int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
+ uint32_t value);
#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
index 22bd21efe6b1..fac7aa2c244f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
@@ -24,6 +24,8 @@
#include <linux/debugfs.h>
#include <linux/list.h>
#include <linux/module.h>
+#include <linux/uaccess.h>
+
#include "amdgpu.h"
#include "amdgpu_ras.h"
#include "amdgpu_atomfirmware.h"
@@ -90,6 +92,12 @@ struct ras_manager {
struct ras_err_data err_data;
};
+struct ras_badpage {
+ unsigned int bp;
+ unsigned int size;
+ unsigned int flags;
+};
+
const char *ras_error_string[] = {
"none",
"parity",
@@ -118,13 +126,15 @@ const char *ras_block_string[] = {
#define ras_err_str(i) (ras_error_string[ffs(i)])
#define ras_block_str(i) (ras_block_string[i])
-#define AMDGPU_RAS_FLAG_INIT_BY_VBIOS 1
+#define AMDGPU_RAS_FLAG_INIT_BY_VBIOS 1
+#define AMDGPU_RAS_FLAG_INIT_NEED_RESET 2
#define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS)
-static void amdgpu_ras_self_test(struct amdgpu_device *adev)
-{
- /* TODO */
-}
+static int amdgpu_ras_reserve_vram(struct amdgpu_device *adev,
+ uint64_t offset, uint64_t size,
+ struct amdgpu_bo **bo_ptr);
+static int amdgpu_ras_release_vram(struct amdgpu_device *adev,
+ struct amdgpu_bo **bo_ptr);
static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf,
size_t size, loff_t *pos)
@@ -237,8 +247,8 @@ static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f,
return 0;
}
-/*
- * DOC: ras debugfs control interface
+/**
+ * DOC: AMDGPU RAS debugfs control interface
*
* It accepts struct ras_debug_if who has two members.
*
@@ -300,6 +310,7 @@ static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, const char __user *
{
struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
struct ras_debug_if data;
+ struct amdgpu_bo *bo;
int ret = 0;
ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data);
@@ -317,7 +328,17 @@ static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, const char __user *
ret = amdgpu_ras_feature_enable(adev, &data.head, 1);
break;
case 2:
+ ret = amdgpu_ras_reserve_vram(adev,
+ data.inject.address, PAGE_SIZE, &bo);
+ if (ret) {
+ /* address was offset, now it is absolute.*/
+ data.inject.address += adev->gmc.vram_start;
+ if (data.inject.address > adev->gmc.vram_end)
+ break;
+ } else
+ data.inject.address = amdgpu_bo_gpu_offset(bo);
ret = amdgpu_ras_error_inject(adev, &data.inject);
+ amdgpu_ras_release_vram(adev, &bo);
break;
default:
ret = -EINVAL;
@@ -521,6 +542,8 @@ int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
enable ? "enable":"disable",
ras_block_str(head->block),
ret);
+ if (ret == TA_RAS_STATUS__RESET_NEEDED)
+ return -EAGAIN;
return -EINVAL;
}
@@ -541,16 +564,32 @@ int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev,
return -EINVAL;
if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
- /* If ras is enabled by vbios, we set up ras object first in
- * both case. For enable, that is all what we need do. For
- * disable, we need perform a ras TA disable cmd after that.
- */
- ret = __amdgpu_ras_feature_enable(adev, head, 1);
- if (ret)
- return ret;
+ if (enable) {
+ /* There is no harm to issue a ras TA cmd regardless of
+ * the currecnt ras state.
+ * If current state == target state, it will do nothing
+ * But sometimes it requests driver to reset and repost
+ * with error code -EAGAIN.
+ */
+ ret = amdgpu_ras_feature_enable(adev, head, 1);
+ /* With old ras TA, we might fail to enable ras.
+ * Log it and just setup the object.
+ * TODO need remove this WA in the future.
+ */
+ if (ret == -EINVAL) {
+ ret = __amdgpu_ras_feature_enable(adev, head, 1);
+ if (!ret)
+ DRM_INFO("RAS INFO: %s setup object\n",
+ ras_block_str(head->block));
+ }
+ } else {
+ /* setup the object then issue a ras TA disable cmd.*/
+ ret = __amdgpu_ras_feature_enable(adev, head, 1);
+ if (ret)
+ return ret;
- if (!enable)
ret = amdgpu_ras_feature_enable(adev, head, 0);
+ }
} else
ret = amdgpu_ras_feature_enable(adev, head, enable);
@@ -645,6 +684,12 @@ int amdgpu_ras_error_inject(struct amdgpu_device *adev,
if (!obj)
return -EINVAL;
+ if (block_info.block_id != TA_RAS_BLOCK__UMC) {
+ DRM_INFO("%s error injection is not supported yet\n",
+ ras_block_str(info->head.block));
+ return -EINVAL;
+ }
+
ret = psp_ras_trigger_error(&adev->psp, &block_info);
if (ret)
DRM_ERROR("RAS ERROR: inject %s error failed ret %d\n",
@@ -691,6 +736,77 @@ int amdgpu_ras_query_error_count(struct amdgpu_device *adev,
/* sysfs begin */
+static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
+ struct ras_badpage **bps, unsigned int *count);
+
+static char *amdgpu_ras_badpage_flags_str(unsigned int flags)
+{
+ switch (flags) {
+ case 0:
+ return "R";
+ case 1:
+ return "P";
+ case 2:
+ default:
+ return "F";
+ };
+}
+
+/*
+ * DOC: ras sysfs gpu_vram_bad_pages interface
+ *
+ * It allows user to read the bad pages of vram on the gpu through
+ * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages
+ *
+ * It outputs multiple lines, and each line stands for one gpu page.
+ *
+ * The format of one line is below,
+ * gpu pfn : gpu page size : flags
+ *
+ * gpu pfn and gpu page size are printed in hex format.
+ * flags can be one of below character,
+ * R: reserved, this gpu page is reserved and not able to use.
+ * P: pending for reserve, this gpu page is marked as bad, will be reserved
+ * in next window of page_reserve.
+ * F: unable to reserve. this gpu page can't be reserved due to some reasons.
+ *
+ * examples:
+ * 0x00000001 : 0x00001000 : R
+ * 0x00000002 : 0x00001000 : P
+ */
+
+static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f,
+ struct kobject *kobj, struct bin_attribute *attr,
+ char *buf, loff_t ppos, size_t count)
+{
+ struct amdgpu_ras *con =
+ container_of(attr, struct amdgpu_ras, badpages_attr);
+ struct amdgpu_device *adev = con->adev;
+ const unsigned int element_size =
+ sizeof("0xabcdabcd : 0x12345678 : R\n") - 1;
+ unsigned int start = div64_ul(ppos + element_size - 1, element_size);
+ unsigned int end = div64_ul(ppos + count - 1, element_size);
+ ssize_t s = 0;
+ struct ras_badpage *bps = NULL;
+ unsigned int bps_count = 0;
+
+ memset(buf, 0, count);
+
+ if (amdgpu_ras_badpages_read(adev, &bps, &bps_count))
+ return 0;
+
+ for (; start < end && start < bps_count; start++)
+ s += scnprintf(&buf[s], element_size + 1,
+ "0x%08x : 0x%08x : %1s\n",
+ bps[start].bp,
+ bps[start].size,
+ amdgpu_ras_badpage_flags_str(bps[start].flags));
+
+ kfree(bps);
+
+ return s;
+}
+
static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev,
struct device_attribute *attr, char *buf)
{
@@ -731,9 +847,14 @@ static int amdgpu_ras_sysfs_create_feature_node(struct amdgpu_device *adev)
&con->features_attr.attr,
NULL
};
+ struct bin_attribute *bin_attrs[] = {
+ &con->badpages_attr,
+ NULL
+ };
struct attribute_group group = {
.name = "ras",
.attrs = attrs,
+ .bin_attrs = bin_attrs,
};
con->features_attr = (struct device_attribute) {
@@ -743,7 +864,19 @@ static int amdgpu_ras_sysfs_create_feature_node(struct amdgpu_device *adev)
},
.show = amdgpu_ras_sysfs_features_read,
};
+
+ con->badpages_attr = (struct bin_attribute) {
+ .attr = {
+ .name = "gpu_vram_bad_pages",
+ .mode = S_IRUGO,
+ },
+ .size = 0,
+ .private = NULL,
+ .read = amdgpu_ras_sysfs_badpages_read,
+ };
+
sysfs_attr_init(attrs[0]);
+ sysfs_bin_attr_init(bin_attrs[0]);
return sysfs_create_group(&adev->dev->kobj, &group);
}
@@ -755,9 +888,14 @@ static int amdgpu_ras_sysfs_remove_feature_node(struct amdgpu_device *adev)
&con->features_attr.attr,
NULL
};
+ struct bin_attribute *bin_attrs[] = {
+ &con->badpages_attr,
+ NULL
+ };
struct attribute_group group = {
.name = "ras",
.attrs = attrs,
+ .bin_attrs = bin_attrs,
};
sysfs_remove_group(&adev->dev->kobj, &group);
@@ -833,40 +971,24 @@ static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev)
/* sysfs end */
/* debugfs begin */
-static int amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev)
+static void amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev)
{
struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
struct drm_minor *minor = adev->ddev->primary;
- struct dentry *root = minor->debugfs_root, *dir;
- struct dentry *ent;
- dir = debugfs_create_dir("ras", root);
- if (IS_ERR(dir))
- return -EINVAL;
-
- con->dir = dir;
-
- ent = debugfs_create_file("ras_ctrl",
- S_IWUGO | S_IRUGO, con->dir,
- adev, &amdgpu_ras_debugfs_ctrl_ops);
- if (IS_ERR(ent)) {
- debugfs_remove(con->dir);
- return -EINVAL;
- }
-
- con->ent = ent;
- return 0;
+ con->dir = debugfs_create_dir("ras", minor->debugfs_root);
+ con->ent = debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, con->dir,
+ adev, &amdgpu_ras_debugfs_ctrl_ops);
}
-int amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
+void amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
struct ras_fs_if *head)
{
struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head);
- struct dentry *ent;
if (!obj || obj->ent)
- return -EINVAL;
+ return;
get_obj(obj);
@@ -874,34 +996,25 @@ int amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
head->debugfs_name,
sizeof(obj->fs_data.debugfs_name));
- ent = debugfs_create_file(obj->fs_data.debugfs_name,
- S_IWUGO | S_IRUGO, con->dir,
- obj, &amdgpu_ras_debugfs_ops);
-
- if (IS_ERR(ent))
- return -EINVAL;
-
- obj->ent = ent;
-
- return 0;
+ obj->ent = debugfs_create_file(obj->fs_data.debugfs_name,
+ S_IWUGO | S_IRUGO, con->dir, obj,
+ &amdgpu_ras_debugfs_ops);
}
-int amdgpu_ras_debugfs_remove(struct amdgpu_device *adev,
+void amdgpu_ras_debugfs_remove(struct amdgpu_device *adev,
struct ras_common_if *head)
{
struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
if (!obj || !obj->ent)
- return 0;
+ return;
debugfs_remove(obj->ent);
obj->ent = NULL;
put_obj(obj);
-
- return 0;
}
-static int amdgpu_ras_debugfs_remove_all(struct amdgpu_device *adev)
+static void amdgpu_ras_debugfs_remove_all(struct amdgpu_device *adev)
{
struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
struct ras_manager *obj, *tmp;
@@ -914,8 +1027,6 @@ static int amdgpu_ras_debugfs_remove_all(struct amdgpu_device *adev)
debugfs_remove(con->dir);
con->dir = NULL;
con->ent = NULL;
-
- return 0;
}
/* debugfs end */
@@ -1089,6 +1200,53 @@ static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev)
/* ih end */
/* recovery begin */
+
+/* return 0 on success.
+ * caller need free bps.
+ */
+static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
+ struct ras_badpage **bps, unsigned int *count)
+{
+ struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+ struct ras_err_handler_data *data;
+ int i = 0;
+ int ret = 0;
+
+ if (!con || !con->eh_data || !bps || !count)
+ return -EINVAL;
+
+ mutex_lock(&con->recovery_lock);
+ data = con->eh_data;
+ if (!data || data->count == 0) {
+ *bps = NULL;
+ goto out;
+ }
+
+ *bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL);
+ if (!*bps) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ for (; i < data->count; i++) {
+ (*bps)[i] = (struct ras_badpage){
+ .bp = data->bps[i].bp,
+ .size = AMDGPU_GPU_PAGE_SIZE,
+ .flags = 0,
+ };
+
+ if (data->last_reserved <= i)
+ (*bps)[i].flags = 1;
+ else if (data->bps[i].bo == NULL)
+ (*bps)[i].flags = 2;
+ }
+
+ *count = data->count;
+out:
+ mutex_unlock(&con->recovery_lock);
+ return ret;
+}
+
static void amdgpu_ras_do_recovery(struct work_struct *work)
{
struct amdgpu_ras *ras =
@@ -1340,6 +1498,19 @@ static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
}
/* recovery end */
+/* return 0 if ras will reset gpu and repost.*/
+int amdgpu_ras_request_reset_on_boot(struct amdgpu_device *adev,
+ unsigned int block)
+{
+ struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
+
+ if (!ras)
+ return -EINVAL;
+
+ ras->flags |= AMDGPU_RAS_FLAG_INIT_NEED_RESET;
+ return 0;
+}
+
/*
* check hardware's ras ability which will be saved in hw_supported.
* if hardware does not support ras, we can skip some ras initializtion and
@@ -1387,6 +1558,12 @@ int amdgpu_ras_init(struct amdgpu_device *adev)
amdgpu_ras_check_supported(adev, &con->hw_supported,
&con->supported);
+ if (!con->hw_supported) {
+ amdgpu_ras_set_context(adev, NULL);
+ kfree(con);
+ return 0;
+ }
+
con->features = 0;
INIT_LIST_HEAD(&con->head);
/* Might need get this flag from vbios. */
@@ -1400,8 +1577,6 @@ int amdgpu_ras_init(struct amdgpu_device *adev)
if (amdgpu_ras_fs_init(adev))
goto fs_out;
- amdgpu_ras_self_test(adev);
-
DRM_INFO("RAS INFO: ras initialized successfully, "
"hardware ability[%x] ras_mask[%x]\n",
con->hw_supported, con->supported);
@@ -1415,8 +1590,10 @@ recovery_out:
return -EINVAL;
}
-/* do some init work after IP late init as dependence */
-void amdgpu_ras_post_init(struct amdgpu_device *adev)
+/* do some init work after IP late init as dependence.
+ * and it runs in resume/gpu reset/booting up cases.
+ */
+void amdgpu_ras_resume(struct amdgpu_device *adev)
{
struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
struct ras_manager *obj, *tmp;
@@ -1444,6 +1621,32 @@ void amdgpu_ras_post_init(struct amdgpu_device *adev)
}
}
}
+
+ if (con->flags & AMDGPU_RAS_FLAG_INIT_NEED_RESET) {
+ con->flags &= ~AMDGPU_RAS_FLAG_INIT_NEED_RESET;
+ /* setup ras obj state as disabled.
+ * for init_by_vbios case.
+ * if we want to enable ras, just enable it in a normal way.
+ * If we want do disable it, need setup ras obj as enabled,
+ * then issue another TA disable cmd.
+ * See feature_enable_on_boot
+ */
+ amdgpu_ras_disable_all_features(adev, 1);
+ amdgpu_ras_reset_gpu(adev, 0);
+ }
+}
+
+void amdgpu_ras_suspend(struct amdgpu_device *adev)
+{
+ struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+
+ if (!con)
+ return;
+
+ amdgpu_ras_disable_all_features(adev, 0);
+ /* Make sure all ras objects are disabled. */
+ if (con->features)
+ amdgpu_ras_disable_all_features(adev, 1);
}
/* do some fini work before IP fini as dependence */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
index 24c6e5fcda86..b2841195bd3b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
@@ -93,6 +93,7 @@ struct amdgpu_ras {
struct dentry *ent;
/* sysfs */
struct device_attribute features_attr;
+ struct bin_attribute badpages_attr;
/* block array */
struct ras_manager *objs;
@@ -177,6 +178,12 @@ static inline int amdgpu_ras_is_supported(struct amdgpu_device *adev,
return ras && (ras->supported & (1 << block));
}
+int amdgpu_ras_request_reset_on_boot(struct amdgpu_device *adev,
+ unsigned int block);
+
+void amdgpu_ras_resume(struct amdgpu_device *adev);
+void amdgpu_ras_suspend(struct amdgpu_device *adev);
+
int amdgpu_ras_query_error_count(struct amdgpu_device *adev,
bool is_ce);
@@ -189,13 +196,10 @@ int amdgpu_ras_reserve_bad_pages(struct amdgpu_device *adev);
static inline int amdgpu_ras_reset_gpu(struct amdgpu_device *adev,
bool is_baco)
{
- /* remove me when gpu reset works on vega20 A1. */
-#if 0
struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0)
schedule_work(&ras->recovery_work);
-#endif
return 0;
}
@@ -257,7 +261,6 @@ amdgpu_ras_error_to_ta(enum amdgpu_ras_error_type error) {
/* called in ip_init and ip_fini */
int amdgpu_ras_init(struct amdgpu_device *adev);
-void amdgpu_ras_post_init(struct amdgpu_device *adev);
int amdgpu_ras_fini(struct amdgpu_device *adev);
int amdgpu_ras_pre_fini(struct amdgpu_device *adev);
@@ -273,10 +276,10 @@ int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
struct ras_common_if *head);
-int amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
+void amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
struct ras_fs_if *head);
-int amdgpu_ras_debugfs_remove(struct amdgpu_device *adev,
+void amdgpu_ras_debugfs_remove(struct amdgpu_device *adev,
struct ras_common_if *head);
int amdgpu_ras_error_query(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 8f5026c123ef..e5c83e164d82 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -28,8 +28,9 @@
*/
#include <linux/seq_file.h>
#include <linux/slab.h>
+#include <linux/uaccess.h>
#include <linux/debugfs.h>
-#include <drm/drmP.h>
+
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
#include "atom.h"
@@ -281,6 +282,16 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
return r;
}
+ r = amdgpu_device_wb_get(adev, &ring->trail_fence_offs);
+ if (r) {
+ dev_err(adev->dev,
+ "(%d) ring trail_fence_offs wb alloc failed\n", r);
+ return r;
+ }
+ ring->trail_fence_gpu_addr =
+ adev->wb.gpu_addr + (ring->trail_fence_offs * 4);
+ ring->trail_fence_cpu_addr = &adev->wb.wb[ring->trail_fence_offs];
+
r = amdgpu_device_wb_get(adev, &ring->cond_exe_offs);
if (r) {
dev_err(adev->dev, "(%d) ring cond_exec_polling wb alloc failed\n", r);
@@ -399,7 +410,7 @@ bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid,
{
ktime_t deadline = ktime_add_us(ktime_get(), 10000);
- if (!ring->funcs->soft_recovery || !fence)
+ if (amdgpu_sriov_vf(ring->adev) || !ring->funcs->soft_recovery || !fence)
return false;
atomic_inc(&ring->adev->gpu_reset_counter);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index d7fae2676269..4410c97ac9b7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -29,8 +29,8 @@
#include <drm/drm_print.h>
/* max number of rings */
-#define AMDGPU_MAX_RINGS 23
-#define AMDGPU_MAX_GFX_RINGS 1
+#define AMDGPU_MAX_RINGS 24
+#define AMDGPU_MAX_GFX_RINGS 2
#define AMDGPU_MAX_COMPUTE_RINGS 8
#define AMDGPU_MAX_VCE_RINGS 3
#define AMDGPU_MAX_UVD_ENC_RINGS 2
@@ -114,6 +114,7 @@ struct amdgpu_ring_funcs {
uint32_t align_mask;
u32 nop;
bool support_64bit_ptrs;
+ bool no_user_fence;
unsigned vmhub;
unsigned extra_dw;
@@ -171,6 +172,7 @@ struct amdgpu_ring_funcs {
enum drm_sched_priority priority);
/* Try to soft recover the ring to make the fence signal */
void (*soft_recovery)(struct amdgpu_ring *ring, unsigned vmid);
+ int (*preempt_ib)(struct amdgpu_ring *ring);
};
struct amdgpu_ring {
@@ -205,6 +207,10 @@ struct amdgpu_ring {
unsigned fence_offs;
uint64_t current_ctx;
char name[16];
+ u32 trail_seq;
+ unsigned trail_fence_offs;
+ u64 trail_fence_gpu_addr;
+ volatile u32 *trail_fence_cpu_addr;
unsigned cond_exe_offs;
u64 cond_exe_gpu_addr;
volatile u32 *cond_exe_cpu_addr;
@@ -245,6 +251,7 @@ struct amdgpu_ring {
#define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib)))
#define amdgpu_ring_init_cond_exec(r) (r)->funcs->init_cond_exec((r))
#define amdgpu_ring_patch_cond_exec(r,o) (r)->funcs->patch_cond_exec((r),(o))
+#define amdgpu_ring_preempt_ib(r) (r)->funcs->preempt_ib(r)
int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw);
void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count);
@@ -265,6 +272,12 @@ void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring,
bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid,
struct dma_fence *fence);
+static inline void amdgpu_ring_set_preempt_cond_exec(struct amdgpu_ring *ring,
+ bool cond_exec)
+{
+ *ring->cond_exe_cpu_addr = cond_exec;
+}
+
static inline void amdgpu_ring_clear_ring(struct amdgpu_ring *ring)
{
int i = 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
index 49a8ab52113b..d3d4707f2168 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
@@ -26,6 +26,94 @@
#include "clearstate_defs.h"
+/* firmware ID used in rlc toc */
+typedef enum _FIRMWARE_ID_ {
+ FIRMWARE_ID_INVALID = 0,
+ FIRMWARE_ID_RLC_G_UCODE = 1,
+ FIRMWARE_ID_RLC_TOC = 2,
+ FIRMWARE_ID_RLCG_SCRATCH = 3,
+ FIRMWARE_ID_RLC_SRM_ARAM = 4,
+ FIRMWARE_ID_RLC_SRM_INDEX_ADDR = 5,
+ FIRMWARE_ID_RLC_SRM_INDEX_DATA = 6,
+ FIRMWARE_ID_RLC_P_UCODE = 7,
+ FIRMWARE_ID_RLC_V_UCODE = 8,
+ FIRMWARE_ID_RLX6_UCODE = 9,
+ FIRMWARE_ID_RLX6_DRAM_BOOT = 10,
+ FIRMWARE_ID_GLOBAL_TAP_DELAYS = 11,
+ FIRMWARE_ID_SE0_TAP_DELAYS = 12,
+ FIRMWARE_ID_SE1_TAP_DELAYS = 13,
+ FIRMWARE_ID_GLOBAL_SE0_SE1_SKEW_DELAYS = 14,
+ FIRMWARE_ID_SDMA0_UCODE = 15,
+ FIRMWARE_ID_SDMA0_JT = 16,
+ FIRMWARE_ID_SDMA1_UCODE = 17,
+ FIRMWARE_ID_SDMA1_JT = 18,
+ FIRMWARE_ID_CP_CE = 19,
+ FIRMWARE_ID_CP_PFP = 20,
+ FIRMWARE_ID_CP_ME = 21,
+ FIRMWARE_ID_CP_MEC = 22,
+ FIRMWARE_ID_CP_MES = 23,
+ FIRMWARE_ID_MES_STACK = 24,
+ FIRMWARE_ID_RLC_SRM_DRAM_SR = 25,
+ FIRMWARE_ID_RLCG_SCRATCH_SR = 26,
+ FIRMWARE_ID_RLCP_SCRATCH_SR = 27,
+ FIRMWARE_ID_RLCV_SCRATCH_SR = 28,
+ FIRMWARE_ID_RLX6_DRAM_SR = 29,
+ FIRMWARE_ID_SDMA0_PG_CONTEXT = 30,
+ FIRMWARE_ID_SDMA1_PG_CONTEXT = 31,
+ FIRMWARE_ID_GLOBAL_MUX_SELECT_RAM = 32,
+ FIRMWARE_ID_SE0_MUX_SELECT_RAM = 33,
+ FIRMWARE_ID_SE1_MUX_SELECT_RAM = 34,
+ FIRMWARE_ID_ACCUM_CTRL_RAM = 35,
+ FIRMWARE_ID_RLCP_CAM = 36,
+ FIRMWARE_ID_RLC_SPP_CAM_EXT = 37,
+ FIRMWARE_ID_MAX = 38,
+} FIRMWARE_ID;
+
+typedef struct _RLC_TABLE_OF_CONTENT {
+ union {
+ unsigned int DW0;
+ struct {
+ unsigned int offset : 25;
+ unsigned int id : 7;
+ };
+ };
+
+ union {
+ unsigned int DW1;
+ struct {
+ unsigned int load_at_boot : 1;
+ unsigned int load_at_vddgfx : 1;
+ unsigned int load_at_reset : 1;
+ unsigned int memory_destination : 2;
+ unsigned int vfflr_image_code : 4;
+ unsigned int load_mode_direct : 1;
+ unsigned int save_for_vddgfx : 1;
+ unsigned int save_for_vfflr : 1;
+ unsigned int reserved : 1;
+ unsigned int signed_source : 1;
+ unsigned int size : 18;
+ };
+ };
+
+ union {
+ unsigned int DW2;
+ struct {
+ unsigned int indirect_addr_reg : 16;
+ unsigned int index : 16;
+ };
+ };
+
+ union {
+ unsigned int DW3;
+ struct {
+ unsigned int indirect_data_reg : 16;
+ unsigned int indirect_start_offset : 16;
+ };
+ };
+} RLC_TABLE_OF_CONTENT;
+
+#define RLC_TOC_MAX_SIZE 64
+
struct amdgpu_rlc_funcs {
bool (*is_rlc_enabled)(struct amdgpu_device *adev);
void (*set_safe_mode)(struct amdgpu_device *adev);
@@ -85,6 +173,16 @@ struct amdgpu_rlc {
u8 *save_restore_list_srm;
bool is_rlc_v2_1;
+
+ /* for rlc autoload */
+ struct amdgpu_bo *rlc_autoload_bo;
+ u64 rlc_autoload_gpu_addr;
+ void *rlc_autoload_ptr;
+
+ /* rlc toc buffer */
+ struct amdgpu_bo *rlc_toc_bo;
+ uint64_t rlc_toc_gpu_addr;
+ void *rlc_toc_buf;
};
void amdgpu_gfx_rlc_enter_safe_mode(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
index bfaf5c6323be..0bd1d4ffc19e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
@@ -41,7 +41,7 @@
* If we are asked to block we wait on all the oldest fence of all
* rings. We just wait for any of those fence to complete.
*/
-#include <drm/drmP.h>
+
#include "amdgpu.h"
static void amdgpu_sa_bo_remove_locked(struct amdgpu_sa_bo *sa_bo);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
index 639297250c21..c799691dfa84 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
@@ -23,8 +23,11 @@
*/
#include <linux/fdtable.h>
+#include <linux/file.h>
#include <linux/pid.h>
+
#include <drm/amdgpu_drm.h>
+
#include "amdgpu.h"
#include "amdgpu_vm.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h
index 2a1a0c734bdd..12299fd95691 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h
@@ -25,7 +25,10 @@
#ifndef __AMDGPU_SCHED_H__
#define __AMDGPU_SCHED_H__
-#include <drm/drmP.h>
+enum drm_sched_priority;
+
+struct drm_device;
+struct drm_file;
enum drm_sched_priority amdgpu_to_sched_priority(int amdgpu_priority);
int amdgpu_sched_ioctl(struct drm_device *dev, void *data,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c
index 115bb0c99b0f..5c13c503e61f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c
@@ -20,10 +20,14 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_sdma.h"
+#define AMDGPU_CSA_SDMA_SIZE 64
+/* SDMA CSA reside in the 3rd page of CSA */
+#define AMDGPU_CSA_SDMA_OFFSET (4096 * 2)
+
/*
* GPU SDMA IP block helpers function.
*/
@@ -56,3 +60,26 @@ int amdgpu_sdma_get_index_from_ring(struct amdgpu_ring *ring, uint32_t *index)
return -EINVAL;
}
+
+uint64_t amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring *ring,
+ unsigned vmid)
+{
+ struct amdgpu_device *adev = ring->adev;
+ uint64_t csa_mc_addr;
+ uint32_t index = 0;
+ int r;
+
+ if (vmid == 0 || !amdgpu_mcbp)
+ return 0;
+
+ r = amdgpu_sdma_get_index_from_ring(ring, &index);
+
+ if (r || index > 31)
+ csa_mc_addr = 0;
+ else
+ csa_mc_addr = amdgpu_csa_vaddr(adev) +
+ AMDGPU_CSA_SDMA_OFFSET +
+ index * AMDGPU_CSA_SDMA_SIZE;
+
+ return csa_mc_addr;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
index 1ba9ba3b54f7..35dd152f9d5c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
@@ -97,5 +97,5 @@ struct amdgpu_buffer_funcs {
struct amdgpu_sdma_instance *
amdgpu_sdma_get_instance_from_ring(struct amdgpu_ring *ring);
int amdgpu_sdma_get_index_from_ring(struct amdgpu_ring *ring, uint32_t *index);
-
+uint64_t amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring *ring, unsigned vmid);
#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_socbb.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_socbb.h
new file mode 100644
index 000000000000..f4176cb01790
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_socbb.h
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __AMDGPU_SOCBB_H__
+#define __AMDGPU_SOCBB_H__
+
+struct gpu_info_voltage_scaling_v1_0 {
+ uint32_t state;
+ uint32_t dscclk_mhz;
+ uint32_t dcfclk_mhz;
+ uint32_t socclk_mhz;
+ uint32_t dram_speed_mts;
+ uint32_t fabricclk_mhz;
+ uint32_t dispclk_mhz;
+ uint32_t phyclk_mhz;
+ uint32_t dppclk_mhz;
+};
+
+struct gpu_info_soc_bounding_box_v1_0 {
+ uint32_t sr_exit_time_us;
+ uint32_t sr_enter_plus_exit_time_us;
+ uint32_t urgent_latency_us;
+ uint32_t urgent_latency_pixel_data_only_us;
+ uint32_t urgent_latency_pixel_mixed_with_vm_data_us;
+ uint32_t urgent_latency_vm_data_only_us;
+ uint32_t writeback_latency_us;
+ uint32_t ideal_dram_bw_after_urgent_percent;
+ uint32_t pct_ideal_dram_sdp_bw_after_urgent_pixel_only; // PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelDataOnly
+ uint32_t pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm;
+ uint32_t pct_ideal_dram_sdp_bw_after_urgent_vm_only;
+ uint32_t max_avg_sdp_bw_use_normal_percent;
+ uint32_t max_avg_dram_bw_use_normal_percent;
+ uint32_t max_request_size_bytes;
+ uint32_t downspread_percent;
+ uint32_t dram_page_open_time_ns;
+ uint32_t dram_rw_turnaround_time_ns;
+ uint32_t dram_return_buffer_per_channel_bytes;
+ uint32_t dram_channel_width_bytes;
+ uint32_t fabric_datapath_to_dcn_data_return_bytes;
+ uint32_t dcn_downspread_percent;
+ uint32_t dispclk_dppclk_vco_speed_mhz;
+ uint32_t dfs_vco_period_ps;
+ uint32_t urgent_out_of_order_return_per_channel_pixel_only_bytes;
+ uint32_t urgent_out_of_order_return_per_channel_pixel_and_vm_bytes;
+ uint32_t urgent_out_of_order_return_per_channel_vm_only_bytes;
+ uint32_t round_trip_ping_latency_dcfclk_cycles;
+ uint32_t urgent_out_of_order_return_per_channel_bytes;
+ uint32_t channel_interleave_bytes;
+ uint32_t num_banks;
+ uint32_t num_chans;
+ uint32_t vmm_page_size_bytes;
+ uint32_t dram_clock_change_latency_us;
+ uint32_t writeback_dram_clock_change_latency_us;
+ uint32_t return_bus_width_bytes;
+ uint32_t voltage_override;
+ uint32_t xfc_bus_transport_time_us;
+ uint32_t xfc_xbuf_latency_tolerance_us;
+ uint32_t use_urgent_burst_bw;
+ uint32_t num_states;
+ struct gpu_info_voltage_scaling_v1_0 clock_limits[8];
+};
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index 2d6f5ec77a68..9828f3c7c655 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -28,7 +28,6 @@
* Christian König <christian.koenig@amd.com>
*/
-#include <drm/drmP.h>
#include "amdgpu.h"
#include "amdgpu_trace.h"
#include "amdgpu_amdkfd.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
index 8904e62dca7a..b66d29d5ffa2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
@@ -22,7 +22,7 @@
*
* Authors: Michel Dänzer
*/
-#include <drm/drmP.h>
+
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
#include "amdgpu_uvd.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index d3ca2424b5fe..77674a7b9616 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -28,8 +28,6 @@
#include <linux/types.h>
#include <linux/tracepoint.h>
-#include <drm/drmP.h>
-
#undef TRACE_SYSTEM
#define TRACE_SYSTEM amdgpu
#define TRACE_INCLUDE_FILE amdgpu_trace
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
index f212402570a5..57c6c39ba064 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
@@ -21,7 +21,7 @@
*
* Author : Dave Airlie <airlied@redhat.com>
*/
-#include <drm/drmP.h>
+
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 0c52d1f9fe0f..e51b48ac48eb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -29,20 +29,26 @@
* Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
* Dave Airlie
*/
+
+#include <linux/dma-mapping.h>
+#include <linux/iommu.h>
+#include <linux/hmm.h>
+#include <linux/pagemap.h>
+#include <linux/sched/task.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
+#include <linux/swap.h>
+#include <linux/swiotlb.h>
+
#include <drm/ttm/ttm_bo_api.h>
#include <drm/ttm/ttm_bo_driver.h>
#include <drm/ttm/ttm_placement.h>
#include <drm/ttm/ttm_module.h>
#include <drm/ttm/ttm_page_alloc.h>
-#include <drm/drmP.h>
+
+#include <drm/drm_debugfs.h>
#include <drm/amdgpu_drm.h>
-#include <linux/seq_file.h>
-#include <linux/slab.h>
-#include <linux/swiotlb.h>
-#include <linux/swap.h>
-#include <linux/pagemap.h>
-#include <linux/debugfs.h>
-#include <linux/iommu.h>
+
#include "amdgpu.h"
#include "amdgpu_object.h"
#include "amdgpu_trace.h"
@@ -385,6 +391,7 @@ int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
src_node_start = amdgpu_mm_node_addr(src->bo, ++src_mm,
src->mem);
src_node_size = (src_mm->size << PAGE_SHIFT);
+ src_page_offset = 0;
} else {
src_node_start += cur_size;
src_page_offset = src_node_start & (PAGE_SIZE - 1);
@@ -394,6 +401,7 @@ int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
dst_node_start = amdgpu_mm_node_addr(dst->bo, ++dst_mm,
dst->mem);
dst_node_size = (dst_mm->size << PAGE_SHIFT);
+ dst_page_offset = 0;
} else {
dst_node_start += cur_size;
dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
@@ -481,6 +489,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict,
placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
if (unlikely(r)) {
+ pr_err("Failed to find GTT space for blit from VRAM\n");
return r;
}
@@ -539,6 +548,7 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict,
placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
if (unlikely(r)) {
+ pr_err("Failed to find GTT space for blit to VRAM\n");
return r;
}
@@ -559,6 +569,30 @@ out_cleanup:
}
/**
+ * amdgpu_mem_visible - Check that memory can be accessed by ttm_bo_move_memcpy
+ *
+ * Called by amdgpu_bo_move()
+ */
+static bool amdgpu_mem_visible(struct amdgpu_device *adev,
+ struct ttm_mem_reg *mem)
+{
+ struct drm_mm_node *nodes = mem->mm_node;
+
+ if (mem->mem_type == TTM_PL_SYSTEM ||
+ mem->mem_type == TTM_PL_TT)
+ return true;
+ if (mem->mem_type != TTM_PL_VRAM)
+ return false;
+
+ /* ttm_mem_reg_ioremap only supports contiguous memory */
+ if (nodes->size != mem->num_pages)
+ return false;
+
+ return ((nodes->start + nodes->size) << PAGE_SHIFT)
+ <= adev->gmc.visible_vram_size;
+}
+
+/**
* amdgpu_bo_move - Move a buffer object to a new memory location
*
* Called by ttm_bo_handle_move_mem()
@@ -602,8 +636,10 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
return 0;
}
- if (!adev->mman.buffer_funcs_enabled)
+ if (!adev->mman.buffer_funcs_enabled) {
+ r = -ENODEV;
goto memcpy;
+ }
if (old_mem->mem_type == TTM_PL_VRAM &&
new_mem->mem_type == TTM_PL_SYSTEM) {
@@ -618,10 +654,16 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
if (r) {
memcpy:
- r = ttm_bo_move_memcpy(bo, ctx, new_mem);
- if (r) {
+ /* Check that all memory is CPU accessible */
+ if (!amdgpu_mem_visible(adev, old_mem) ||
+ !amdgpu_mem_visible(adev, new_mem)) {
+ pr_err("Move buffer fallback to memcpy unavailable\n");
return r;
}
+
+ r = ttm_bo_move_memcpy(bo, ctx, new_mem);
+ if (r)
+ return r;
}
if (bo->type == ttm_bo_type_device &&
@@ -703,143 +745,183 @@ static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
/*
* TTM backend functions.
*/
-struct amdgpu_ttm_gup_task_list {
- struct list_head list;
- struct task_struct *task;
-};
-
struct amdgpu_ttm_tt {
struct ttm_dma_tt ttm;
u64 offset;
uint64_t userptr;
struct task_struct *usertask;
uint32_t userflags;
- spinlock_t guptasklock;
- struct list_head guptasks;
- atomic_t mmu_invalidations;
- uint32_t last_set_pages;
+#if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
+ struct hmm_range *range;
+#endif
};
/**
- * amdgpu_ttm_tt_get_user_pages - Pin pages of memory pointed to by a USERPTR
- * pointer to memory
+ * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user
+ * memory and start HMM tracking CPU page table update
*
- * Called by amdgpu_gem_userptr_ioctl() and amdgpu_cs_parser_bos().
- * This provides a wrapper around the get_user_pages() call to provide
- * device accessible pages that back user memory.
+ * Calling function must call amdgpu_ttm_tt_userptr_range_done() once and only
+ * once afterwards to stop HMM tracking
*/
-int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
+#if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
+
+#define MAX_RETRY_HMM_RANGE_FAULT 16
+
+int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages)
{
+ struct hmm_mirror *mirror = bo->mn ? &bo->mn->mirror : NULL;
+ struct ttm_tt *ttm = bo->tbo.ttm;
struct amdgpu_ttm_tt *gtt = (void *)ttm;
struct mm_struct *mm = gtt->usertask->mm;
- unsigned int flags = 0;
- unsigned pinned = 0;
- int r;
+ unsigned long start = gtt->userptr;
+ struct vm_area_struct *vma;
+ struct hmm_range *range;
+ unsigned long i;
+ uint64_t *pfns;
+ int retry = 0;
+ int r = 0;
if (!mm) /* Happens during process shutdown */
return -ESRCH;
- if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
- flags |= FOLL_WRITE;
+ if (unlikely(!mirror)) {
+ DRM_DEBUG_DRIVER("Failed to get hmm_mirror\n");
+ r = -EFAULT;
+ goto out;
+ }
- down_read(&mm->mmap_sem);
+ vma = find_vma(mm, start);
+ if (unlikely(!vma || start < vma->vm_start)) {
+ r = -EFAULT;
+ goto out;
+ }
+ if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) &&
+ vma->vm_file)) {
+ r = -EPERM;
+ goto out;
+ }
- if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) {
- /*
- * check that we only use anonymous memory to prevent problems
- * with writeback
- */
- unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;
- struct vm_area_struct *vma;
+ range = kzalloc(sizeof(*range), GFP_KERNEL);
+ if (unlikely(!range)) {
+ r = -ENOMEM;
+ goto out;
+ }
- vma = find_vma(mm, gtt->userptr);
- if (!vma || vma->vm_file || vma->vm_end < end) {
- up_read(&mm->mmap_sem);
- return -EPERM;
- }
+ pfns = kvmalloc_array(ttm->num_pages, sizeof(*pfns), GFP_KERNEL);
+ if (unlikely(!pfns)) {
+ r = -ENOMEM;
+ goto out_free_ranges;
}
- /* loop enough times using contiguous pages of memory */
- do {
- unsigned num_pages = ttm->num_pages - pinned;
- uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE;
- struct page **p = pages + pinned;
- struct amdgpu_ttm_gup_task_list guptask;
+ amdgpu_hmm_init_range(range);
+ range->default_flags = range->flags[HMM_PFN_VALID];
+ range->default_flags |= amdgpu_ttm_tt_is_readonly(ttm) ?
+ 0 : range->flags[HMM_PFN_WRITE];
+ range->pfn_flags_mask = 0;
+ range->pfns = pfns;
+ hmm_range_register(range, mirror, start,
+ start + ttm->num_pages * PAGE_SIZE, PAGE_SHIFT);
- guptask.task = current;
- spin_lock(&gtt->guptasklock);
- list_add(&guptask.list, &gtt->guptasks);
- spin_unlock(&gtt->guptasklock);
+retry:
+ /*
+ * Just wait for range to be valid, safe to ignore return value as we
+ * will use the return value of hmm_range_fault() below under the
+ * mmap_sem to ascertain the validity of the range.
+ */
+ hmm_range_wait_until_valid(range, HMM_RANGE_DEFAULT_TIMEOUT);
+
+ down_read(&mm->mmap_sem);
+
+ r = hmm_range_fault(range, true);
+ if (unlikely(r < 0)) {
+ if (likely(r == -EAGAIN)) {
+ /*
+ * return -EAGAIN, mmap_sem is dropped
+ */
+ if (retry++ < MAX_RETRY_HMM_RANGE_FAULT)
+ goto retry;
+ else
+ pr_err("Retry hmm fault too many times\n");
+ }
- if (mm == current->mm)
- r = get_user_pages(userptr, num_pages, flags, p, NULL);
- else
- r = get_user_pages_remote(gtt->usertask,
- mm, userptr, num_pages,
- flags, p, NULL, NULL);
+ goto out_up_read;
+ }
- spin_lock(&gtt->guptasklock);
- list_del(&guptask.list);
- spin_unlock(&gtt->guptasklock);
+ up_read(&mm->mmap_sem);
- if (r < 0)
- goto release_pages;
+ for (i = 0; i < ttm->num_pages; i++) {
+ pages[i] = hmm_device_entry_to_page(range, pfns[i]);
+ if (unlikely(!pages[i])) {
+ pr_err("Page fault failed for pfn[%lu] = 0x%llx\n",
+ i, pfns[i]);
+ r = -ENOMEM;
- pinned += r;
+ goto out_free_pfns;
+ }
+ }
- } while (pinned < ttm->num_pages);
+ gtt->range = range;
- up_read(&mm->mmap_sem);
return 0;
-release_pages:
- release_pages(pages, pinned);
- up_read(&mm->mmap_sem);
+out_up_read:
+ if (likely(r != -EAGAIN))
+ up_read(&mm->mmap_sem);
+out_free_pfns:
+ hmm_range_unregister(range);
+ kvfree(pfns);
+out_free_ranges:
+ kfree(range);
+out:
return r;
}
/**
- * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
+ * amdgpu_ttm_tt_userptr_range_done - stop HMM track the CPU page table change
+ * Check if the pages backing this ttm range have been invalidated
*
- * Called by amdgpu_cs_list_validate(). This creates the page list
- * that backs user memory and will ultimately be mapped into the device
- * address space.
+ * Returns: true if pages are still valid
*/
-void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
+bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm)
{
struct amdgpu_ttm_tt *gtt = (void *)ttm;
- unsigned i;
+ bool r = false;
- gtt->last_set_pages = atomic_read(&gtt->mmu_invalidations);
- for (i = 0; i < ttm->num_pages; ++i) {
- if (ttm->pages[i])
- put_page(ttm->pages[i]);
+ if (!gtt || !gtt->userptr)
+ return false;
- ttm->pages[i] = pages ? pages[i] : NULL;
+ DRM_DEBUG_DRIVER("user_pages_done 0x%llx pages 0x%lx\n",
+ gtt->userptr, ttm->num_pages);
+
+ WARN_ONCE(!gtt->range || !gtt->range->pfns,
+ "No user pages to check\n");
+
+ if (gtt->range) {
+ r = hmm_range_valid(gtt->range);
+ hmm_range_unregister(gtt->range);
+
+ kvfree(gtt->range->pfns);
+ kfree(gtt->range);
+ gtt->range = NULL;
}
+
+ return r;
}
+#endif
/**
- * amdgpu_ttm_tt_mark_user_page - Mark pages as dirty
+ * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
*
- * Called while unpinning userptr pages
+ * Called by amdgpu_cs_list_validate(). This creates the page list
+ * that backs user memory and will ultimately be mapped into the device
+ * address space.
*/
-void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm)
+void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
{
- struct amdgpu_ttm_tt *gtt = (void *)ttm;
- unsigned i;
-
- for (i = 0; i < ttm->num_pages; ++i) {
- struct page *page = ttm->pages[i];
-
- if (!page)
- continue;
+ unsigned long i;
- if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
- set_page_dirty(page);
-
- mark_page_accessed(page);
- }
+ for (i = 0; i < ttm->num_pages; ++i)
+ ttm->pages[i] = pages ? pages[i] : NULL;
}
/**
@@ -901,10 +983,14 @@ static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
/* unmap the pages mapped to the device */
dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
- /* mark the pages as dirty */
- amdgpu_ttm_tt_mark_user_pages(ttm);
-
sg_free_table(ttm->sg);
+
+#if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
+ if (gtt->range &&
+ ttm->pages[0] == hmm_device_entry_to_page(gtt->range,
+ gtt->range->pfns[0]))
+ WARN_ONCE(1, "Missing get_user_page_done\n");
+#endif
}
int amdgpu_ttm_gart_bind(struct amdgpu_device *adev,
@@ -925,8 +1011,8 @@ int amdgpu_ttm_gart_bind(struct amdgpu_device *adev,
goto gart_bind_fail;
/* Patch mtype of the second part BO */
- flags &= ~AMDGPU_PTE_MTYPE_MASK;
- flags |= AMDGPU_PTE_MTYPE(AMDGPU_MTYPE_NC);
+ flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
+ flags |= AMDGPU_PTE_MTYPE_VG10(AMDGPU_MTYPE_NC);
r = amdgpu_gart_bind(adev,
gtt->offset + (page_idx << PAGE_SHIFT),
@@ -1254,11 +1340,6 @@ int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
gtt->usertask = current->group_leader;
get_task_struct(gtt->usertask);
- spin_lock_init(&gtt->guptasklock);
- INIT_LIST_HEAD(&gtt->guptasks);
- atomic_set(&gtt->mmu_invalidations, 0);
- gtt->last_set_pages = 0;
-
return 0;
}
@@ -1287,7 +1368,6 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
unsigned long end)
{
struct amdgpu_ttm_tt *gtt = (void *)ttm;
- struct amdgpu_ttm_gup_task_list *entry;
unsigned long size;
if (gtt == NULL || !gtt->userptr)
@@ -1300,48 +1380,20 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
if (gtt->userptr > end || gtt->userptr + size <= start)
return false;
- /* Search the lists of tasks that hold this mapping and see
- * if current is one of them. If it is return false.
- */
- spin_lock(&gtt->guptasklock);
- list_for_each_entry(entry, &gtt->guptasks, list) {
- if (entry->task == current) {
- spin_unlock(&gtt->guptasklock);
- return false;
- }
- }
- spin_unlock(&gtt->guptasklock);
-
- atomic_inc(&gtt->mmu_invalidations);
-
return true;
}
/**
- * amdgpu_ttm_tt_userptr_invalidated - Has the ttm_tt object been invalidated?
+ * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr?
*/
-bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
- int *last_invalidated)
-{
- struct amdgpu_ttm_tt *gtt = (void *)ttm;
- int prev_invalidated = *last_invalidated;
-
- *last_invalidated = atomic_read(&gtt->mmu_invalidations);
- return prev_invalidated != *last_invalidated;
-}
-
-/**
- * amdgpu_ttm_tt_userptr_needs_pages - Have the pages backing this ttm_tt object
- * been invalidated since the last time they've been set?
- */
-bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm)
+bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm)
{
struct amdgpu_ttm_tt *gtt = (void *)ttm;
if (gtt == NULL || !gtt->userptr)
return false;
- return atomic_read(&gtt->mmu_invalidations) != gtt->last_set_pages;
+ return true;
}
/**
@@ -1753,44 +1805,26 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
/* Initialize various on-chip memory pools */
r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GDS,
- adev->gds.mem.total_size);
+ adev->gds.gds_size);
if (r) {
DRM_ERROR("Failed initializing GDS heap.\n");
return r;
}
- r = amdgpu_bo_create_kernel(adev, adev->gds.mem.gfx_partition_size,
- 4, AMDGPU_GEM_DOMAIN_GDS,
- &adev->gds.gds_gfx_bo, NULL, NULL);
- if (r)
- return r;
-
r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GWS,
- adev->gds.gws.total_size);
+ adev->gds.gws_size);
if (r) {
DRM_ERROR("Failed initializing gws heap.\n");
return r;
}
- r = amdgpu_bo_create_kernel(adev, adev->gds.gws.gfx_partition_size,
- 1, AMDGPU_GEM_DOMAIN_GWS,
- &adev->gds.gws_gfx_bo, NULL, NULL);
- if (r)
- return r;
-
r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_OA,
- adev->gds.oa.total_size);
+ adev->gds.oa_size);
if (r) {
DRM_ERROR("Failed initializing oa heap.\n");
return r;
}
- r = amdgpu_bo_create_kernel(adev, adev->gds.oa.gfx_partition_size,
- 1, AMDGPU_GEM_DOMAIN_OA,
- &adev->gds.oa_gfx_bo, NULL, NULL);
- if (r)
- return r;
-
/* Register debugfs entries for amdgpu_ttm */
r = amdgpu_ttm_debugfs_init(adev);
if (r) {
@@ -2061,9 +2095,9 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
mm_node = bo->tbo.mem.mm_node;
num_loops = 0;
while (num_pages) {
- uint32_t byte_count = mm_node->size << PAGE_SHIFT;
+ uint64_t byte_count = mm_node->size << PAGE_SHIFT;
- num_loops += DIV_ROUND_UP(byte_count, max_bytes);
+ num_loops += DIV_ROUND_UP_ULL(byte_count, max_bytes);
num_pages -= mm_node->size;
++mm_node;
}
@@ -2089,12 +2123,13 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
mm_node = bo->tbo.mem.mm_node;
while (num_pages) {
- uint32_t byte_count = mm_node->size << PAGE_SHIFT;
+ uint64_t byte_count = mm_node->size << PAGE_SHIFT;
uint64_t dst_addr;
dst_addr = amdgpu_mm_node_addr(&bo->tbo, mm_node, &bo->tbo.mem);
while (byte_count) {
- uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
+ uint32_t cur_size_in_bytes = min_t(uint64_t, byte_count,
+ max_bytes);
amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data,
dst_addr, cur_size_in_bytes);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index b5b2d101f7db..caa76c693700 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -101,9 +101,22 @@ int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma);
int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo);
int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo);
-int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages);
+#if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
+int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages);
+bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm);
+#else
+static inline int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo,
+ struct page **pages)
+{
+ return -EPERM;
+}
+static inline bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm)
+{
+ return false;
+}
+#endif
+
void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages);
-void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm);
int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
uint32_t flags);
bool amdgpu_ttm_tt_has_userptr(struct ttm_tt *ttm);
@@ -112,7 +125,7 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
unsigned long end);
bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
int *last_invalidated);
-bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm);
+bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm);
bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm);
uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_mem_reg *mem);
uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
index 7b33867036e7..bfaa0eac3213 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
@@ -24,7 +24,7 @@
#include <linux/firmware.h>
#include <linux/slab.h>
#include <linux/module.h>
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_ucode.h"
@@ -77,6 +77,14 @@ void amdgpu_ucode_print_smc_hdr(const struct common_firmware_header *hdr)
container_of(hdr, struct smc_firmware_header_v1_0, header);
DRM_DEBUG("ucode_start_addr: %u\n", le32_to_cpu(smc_hdr->ucode_start_addr));
+ } else if (version_major == 2) {
+ const struct smc_firmware_header_v1_0 *v1_hdr =
+ container_of(hdr, struct smc_firmware_header_v1_0, header);
+ const struct smc_firmware_header_v2_0 *v2_hdr =
+ container_of(v1_hdr, struct smc_firmware_header_v2_0, v1_0);
+
+ DRM_INFO("ppt_offset_bytes: %u\n", le32_to_cpu(v2_hdr->ppt_offset_bytes));
+ DRM_INFO("ppt_size_bytes: %u\n", le32_to_cpu(v2_hdr->ppt_size_bytes));
} else {
DRM_ERROR("Unknown SMC ucode version: %u.%u\n", version_major, version_minor);
}
@@ -227,6 +235,46 @@ void amdgpu_ucode_print_sdma_hdr(const struct common_firmware_header *hdr)
}
}
+void amdgpu_ucode_print_psp_hdr(const struct common_firmware_header *hdr)
+{
+ uint16_t version_major = le16_to_cpu(hdr->header_version_major);
+ uint16_t version_minor = le16_to_cpu(hdr->header_version_minor);
+
+ DRM_DEBUG("PSP\n");
+ amdgpu_ucode_print_common_hdr(hdr);
+
+ if (version_major == 1) {
+ const struct psp_firmware_header_v1_0 *psp_hdr =
+ container_of(hdr, struct psp_firmware_header_v1_0, header);
+
+ DRM_DEBUG("ucode_feature_version: %u\n",
+ le32_to_cpu(psp_hdr->ucode_feature_version));
+ DRM_DEBUG("sos_offset_bytes: %u\n",
+ le32_to_cpu(psp_hdr->sos_offset_bytes));
+ DRM_DEBUG("sos_size_bytes: %u\n",
+ le32_to_cpu(psp_hdr->sos_size_bytes));
+ if (version_minor == 1) {
+ const struct psp_firmware_header_v1_1 *psp_hdr_v1_1 =
+ container_of(psp_hdr, struct psp_firmware_header_v1_1, v1_0);
+ DRM_DEBUG("toc_header_version: %u\n",
+ le32_to_cpu(psp_hdr_v1_1->toc_header_version));
+ DRM_DEBUG("toc_offset_bytes: %u\n",
+ le32_to_cpu(psp_hdr_v1_1->toc_offset_bytes));
+ DRM_DEBUG("toc_size_bytes: %u\n",
+ le32_to_cpu(psp_hdr_v1_1->toc_size_bytes));
+ DRM_DEBUG("kdb_header_version: %u\n",
+ le32_to_cpu(psp_hdr_v1_1->kdb_header_version));
+ DRM_DEBUG("kdb_offset_bytes: %u\n",
+ le32_to_cpu(psp_hdr_v1_1->kdb_offset_bytes));
+ DRM_DEBUG("kdb_size_bytes: %u\n",
+ le32_to_cpu(psp_hdr_v1_1->kdb_size_bytes));
+ }
+ } else {
+ DRM_ERROR("Unknown PSP ucode version: %u.%u\n",
+ version_major, version_minor);
+ }
+}
+
void amdgpu_ucode_print_gpu_info_hdr(const struct common_firmware_header *hdr)
{
uint16_t version_major = le16_to_cpu(hdr->header_version_major);
@@ -302,6 +350,7 @@ amdgpu_ucode_get_load_type(struct amdgpu_device *adev, int load_type)
case CHIP_RAVEN:
case CHIP_VEGA12:
case CHIP_VEGA20:
+ case CHIP_NAVI10:
if (!load_type)
return AMDGPU_FW_LOAD_DIRECT;
else
@@ -313,6 +362,69 @@ amdgpu_ucode_get_load_type(struct amdgpu_device *adev, int load_type)
return AMDGPU_FW_LOAD_DIRECT;
}
+#define FW_VERSION_ATTR(name, mode, field) \
+static ssize_t show_##name(struct device *dev, \
+ struct device_attribute *attr, \
+ char *buf) \
+{ \
+ struct drm_device *ddev = dev_get_drvdata(dev); \
+ struct amdgpu_device *adev = ddev->dev_private; \
+ \
+ return snprintf(buf, PAGE_SIZE, "0x%08x\n", adev->field); \
+} \
+static DEVICE_ATTR(name, mode, show_##name, NULL)
+
+FW_VERSION_ATTR(vce_fw_version, 0444, vce.fw_version);
+FW_VERSION_ATTR(uvd_fw_version, 0444, uvd.fw_version);
+FW_VERSION_ATTR(mc_fw_version, 0444, gmc.fw_version);
+FW_VERSION_ATTR(me_fw_version, 0444, gfx.me_fw_version);
+FW_VERSION_ATTR(pfp_fw_version, 0444, gfx.pfp_fw_version);
+FW_VERSION_ATTR(ce_fw_version, 0444, gfx.ce_fw_version);
+FW_VERSION_ATTR(rlc_fw_version, 0444, gfx.rlc_fw_version);
+FW_VERSION_ATTR(rlc_srlc_fw_version, 0444, gfx.rlc_srlc_fw_version);
+FW_VERSION_ATTR(rlc_srlg_fw_version, 0444, gfx.rlc_srlg_fw_version);
+FW_VERSION_ATTR(rlc_srls_fw_version, 0444, gfx.rlc_srls_fw_version);
+FW_VERSION_ATTR(mec_fw_version, 0444, gfx.mec_fw_version);
+FW_VERSION_ATTR(mec2_fw_version, 0444, gfx.mec2_fw_version);
+FW_VERSION_ATTR(sos_fw_version, 0444, psp.sos_fw_version);
+FW_VERSION_ATTR(asd_fw_version, 0444, psp.asd_fw_version);
+FW_VERSION_ATTR(ta_ras_fw_version, 0444, psp.ta_fw_version);
+FW_VERSION_ATTR(ta_xgmi_fw_version, 0444, psp.ta_fw_version);
+FW_VERSION_ATTR(smc_fw_version, 0444, pm.fw_version);
+FW_VERSION_ATTR(sdma_fw_version, 0444, sdma.instance[0].fw_version);
+FW_VERSION_ATTR(sdma2_fw_version, 0444, sdma.instance[1].fw_version);
+FW_VERSION_ATTR(vcn_fw_version, 0444, vcn.fw_version);
+FW_VERSION_ATTR(dmcu_fw_version, 0444, dm.dmcu_fw_version);
+
+static struct attribute *fw_attrs[] = {
+ &dev_attr_vce_fw_version.attr, &dev_attr_uvd_fw_version.attr,
+ &dev_attr_mc_fw_version.attr, &dev_attr_me_fw_version.attr,
+ &dev_attr_pfp_fw_version.attr, &dev_attr_ce_fw_version.attr,
+ &dev_attr_rlc_fw_version.attr, &dev_attr_rlc_srlc_fw_version.attr,
+ &dev_attr_rlc_srlg_fw_version.attr, &dev_attr_rlc_srls_fw_version.attr,
+ &dev_attr_mec_fw_version.attr, &dev_attr_mec2_fw_version.attr,
+ &dev_attr_sos_fw_version.attr, &dev_attr_asd_fw_version.attr,
+ &dev_attr_ta_ras_fw_version.attr, &dev_attr_ta_xgmi_fw_version.attr,
+ &dev_attr_smc_fw_version.attr, &dev_attr_sdma_fw_version.attr,
+ &dev_attr_sdma2_fw_version.attr, &dev_attr_vcn_fw_version.attr,
+ &dev_attr_dmcu_fw_version.attr, NULL
+};
+
+static const struct attribute_group fw_attr_group = {
+ .name = "fw_version",
+ .attrs = fw_attrs
+};
+
+int amdgpu_ucode_sysfs_init(struct amdgpu_device *adev)
+{
+ return sysfs_create_group(&adev->dev->kobj, &fw_attr_group);
+}
+
+void amdgpu_ucode_sysfs_fini(struct amdgpu_device *adev)
+{
+ sysfs_remove_group(&adev->dev->kobj, &fw_attr_group);
+}
+
static int amdgpu_ucode_init_single_fw(struct amdgpu_device *adev,
struct amdgpu_firmware_info *ucode,
uint64_t mc_addr, void *kptr)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
index 7ac25a1c7853..c1fb6dc86440 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
@@ -23,6 +23,8 @@
#ifndef __AMDGPU_UCODE_H__
#define __AMDGPU_UCODE_H__
+#include "amdgpu_socbb.h"
+
struct common_firmware_header {
uint32_t size_bytes; /* size of the entire header+image(s) in bytes */
uint32_t header_size_bytes; /* size of just the header in bytes */
@@ -49,6 +51,26 @@ struct smc_firmware_header_v1_0 {
uint32_t ucode_start_addr;
};
+/* version_major=2, version_minor=0 */
+struct smc_firmware_header_v2_0 {
+ struct smc_firmware_header_v1_0 v1_0;
+ uint32_t ppt_offset_bytes; /* soft pptable offset */
+ uint32_t ppt_size_bytes; /* soft pptable size */
+};
+
+struct smc_soft_pptable_entry {
+ uint32_t id;
+ uint32_t ppt_offset_bytes;
+ uint32_t ppt_size_bytes;
+};
+
+/* version_major=2, version_minor=1 */
+struct smc_firmware_header_v2_1 {
+ struct smc_firmware_header_v1_0 v1_0;
+ uint32_t pptable_count;
+ uint32_t pptable_entry_offset;
+};
+
/* version_major=1, version_minor=0 */
struct psp_firmware_header_v1_0 {
struct common_firmware_header header;
@@ -57,6 +79,17 @@ struct psp_firmware_header_v1_0 {
uint32_t sos_size_bytes;
};
+/* version_major=1, version_minor=1 */
+struct psp_firmware_header_v1_1 {
+ struct psp_firmware_header_v1_0 v1_0;
+ uint32_t toc_header_version;
+ uint32_t toc_offset_bytes;
+ uint32_t toc_size_bytes;
+ uint32_t kdb_header_version;
+ uint32_t kdb_offset_bytes;
+ uint32_t kdb_size_bytes;
+};
+
/* version_major=1, version_minor=0 */
struct ta_firmware_header_v1_0 {
struct common_firmware_header header;
@@ -77,6 +110,21 @@ struct gfx_firmware_header_v1_0 {
};
/* version_major=1, version_minor=0 */
+struct mes_firmware_header_v1_0 {
+ struct common_firmware_header header;
+ uint32_t mes_ucode_version;
+ uint32_t mes_ucode_size_bytes;
+ uint32_t mes_ucode_offset_bytes;
+ uint32_t mes_ucode_data_version;
+ uint32_t mes_ucode_data_size_bytes;
+ uint32_t mes_ucode_data_offset_bytes;
+ uint32_t mes_uc_start_addr_lo;
+ uint32_t mes_uc_start_addr_hi;
+ uint32_t mes_data_start_addr_lo;
+ uint32_t mes_data_start_addr_hi;
+};
+
+/* version_major=1, version_minor=0 */
struct rlc_firmware_header_v1_0 {
struct common_firmware_header header;
uint32_t ucode_feature_version;
@@ -161,6 +209,19 @@ struct gpu_info_firmware_v1_0 {
uint32_t gc_lds_size;
};
+struct gpu_info_firmware_v1_1 {
+ struct gpu_info_firmware_v1_0 v1_0;
+ uint32_t num_sc_per_sh;
+ uint32_t num_packer_per_sc;
+};
+
+/* gpu info payload
+ * version_major=1, version_minor=1 */
+struct gpu_info_firmware_v1_2 {
+ struct gpu_info_firmware_v1_1 v1_1;
+ struct gpu_info_soc_bounding_box_v1_0 soc_bounding_box;
+};
+
/* version_major=1, version_minor=0 */
struct gpu_info_firmware_header_v1_0 {
struct common_firmware_header header;
@@ -180,7 +241,9 @@ union amdgpu_firmware_header {
struct common_firmware_header common;
struct mc_firmware_header_v1_0 mc;
struct smc_firmware_header_v1_0 smc;
+ struct smc_firmware_header_v2_0 smc_v2_0;
struct psp_firmware_header_v1_0 psp;
+ struct psp_firmware_header_v1_1 psp_v1_1;
struct ta_firmware_header_v1_0 ta;
struct gfx_firmware_header_v1_0 gfx;
struct rlc_firmware_header_v1_0 rlc;
@@ -206,6 +269,8 @@ enum AMDGPU_UCODE_ID {
AMDGPU_UCODE_ID_CP_MEC1_JT,
AMDGPU_UCODE_ID_CP_MEC2,
AMDGPU_UCODE_ID_CP_MEC2_JT,
+ AMDGPU_UCODE_ID_CP_MES,
+ AMDGPU_UCODE_ID_CP_MES_DATA,
AMDGPU_UCODE_ID_RLC_G,
AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL,
AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM,
@@ -218,6 +283,8 @@ enum AMDGPU_UCODE_ID {
AMDGPU_UCODE_ID_VCN,
AMDGPU_UCODE_ID_DMCU_ERAM,
AMDGPU_UCODE_ID_DMCU_INTV,
+ AMDGPU_UCODE_ID_VCN0_RAM,
+ AMDGPU_UCODE_ID_VCN1_RAM,
AMDGPU_UCODE_ID_MAXIMUM,
};
@@ -232,6 +299,7 @@ enum amdgpu_firmware_load_type {
AMDGPU_FW_LOAD_DIRECT = 0,
AMDGPU_FW_LOAD_SMU,
AMDGPU_FW_LOAD_PSP,
+ AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO,
};
/* conform to smu_ucode_xfer_cz.h */
@@ -284,6 +352,7 @@ void amdgpu_ucode_print_smc_hdr(const struct common_firmware_header *hdr);
void amdgpu_ucode_print_gfx_hdr(const struct common_firmware_header *hdr);
void amdgpu_ucode_print_rlc_hdr(const struct common_firmware_header *hdr);
void amdgpu_ucode_print_sdma_hdr(const struct common_firmware_header *hdr);
+void amdgpu_ucode_print_psp_hdr(const struct common_firmware_header *hdr);
void amdgpu_ucode_print_gpu_info_hdr(const struct common_firmware_header *hdr);
int amdgpu_ucode_validate(const struct firmware *fw);
bool amdgpu_ucode_hdr_version(union amdgpu_firmware_header *hdr,
@@ -291,7 +360,9 @@ bool amdgpu_ucode_hdr_version(union amdgpu_firmware_header *hdr,
int amdgpu_ucode_init_bo(struct amdgpu_device *adev);
int amdgpu_ucode_create_bo(struct amdgpu_device *adev);
+int amdgpu_ucode_sysfs_init(struct amdgpu_device *adev);
void amdgpu_ucode_free_bo(struct amdgpu_device *adev);
+void amdgpu_ucode_sysfs_fini(struct amdgpu_device *adev);
enum amdgpu_firmware_load_type
amdgpu_ucode_get_load_type(struct amdgpu_device *adev, int load_type);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index 4e5d13e41f6a..5b2fea3b4a2c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -30,7 +30,7 @@
#include <linux/firmware.h>
#include <linux/module.h>
-#include <drm/drmP.h>
+
#include <drm/drm.h>
#include "amdgpu.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index f7189e22f6b7..b70b3c45bb29 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -27,7 +27,7 @@
#include <linux/firmware.h>
#include <linux/module.h>
-#include <drm/drmP.h>
+
#include <drm/drm.h>
#include "amdgpu.h"
@@ -1092,7 +1092,7 @@ int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring)
for (i = 0; i < timeout; i++) {
if (amdgpu_ring_get_rptr(ring) != rptr)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= timeout)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
index e6b07ece3910..2e12eeb314a7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
@@ -26,7 +26,8 @@
#include <linux/firmware.h>
#include <linux/module.h>
-#include <drm/drmP.h>
+#include <linux/pci.h>
+
#include <drm/drm.h>
#include "amdgpu.h"
@@ -45,10 +46,12 @@
#define FIRMWARE_RAVEN "amdgpu/raven_vcn.bin"
#define FIRMWARE_PICASSO "amdgpu/picasso_vcn.bin"
#define FIRMWARE_RAVEN2 "amdgpu/raven2_vcn.bin"
+#define FIRMWARE_NAVI10 "amdgpu/navi10_vcn.bin"
MODULE_FIRMWARE(FIRMWARE_RAVEN);
MODULE_FIRMWARE(FIRMWARE_PICASSO);
MODULE_FIRMWARE(FIRMWARE_RAVEN2);
+MODULE_FIRMWARE(FIRMWARE_NAVI10);
static void amdgpu_vcn_idle_work_handler(struct work_struct *work);
@@ -71,6 +74,12 @@ int amdgpu_vcn_sw_init(struct amdgpu_device *adev)
else
fw_name = FIRMWARE_RAVEN;
break;
+ case CHIP_NAVI10:
+ fw_name = FIRMWARE_NAVI10;
+ if ((adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) &&
+ (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG))
+ adev->vcn.indirect_sram = true;
+ break;
default:
return -EINVAL;
}
@@ -132,6 +141,16 @@ int amdgpu_vcn_sw_init(struct amdgpu_device *adev)
return r;
}
+ if (adev->vcn.indirect_sram) {
+ r = amdgpu_bo_create_kernel(adev, 64 * 2 * 4, PAGE_SIZE,
+ AMDGPU_GEM_DOMAIN_VRAM, &adev->vcn.dpg_sram_bo,
+ &adev->vcn.dpg_sram_gpu_addr, &adev->vcn.dpg_sram_cpu_addr);
+ if (r) {
+ dev_err(adev->dev, "(%d) failed to allocate DPG bo\n", r);
+ return r;
+ }
+ }
+
return 0;
}
@@ -141,6 +160,12 @@ int amdgpu_vcn_sw_fini(struct amdgpu_device *adev)
kvfree(adev->vcn.saved_bo);
+ if (adev->vcn.indirect_sram) {
+ amdgpu_bo_free_kernel(&adev->vcn.dpg_sram_bo,
+ &adev->vcn.dpg_sram_gpu_addr,
+ (void **)&adev->vcn.dpg_sram_cpu_addr);
+ }
+
amdgpu_bo_free_kernel(&adev->vcn.vcpu_bo,
&adev->vcn.gpu_addr,
(void **)&adev->vcn.cpu_addr);
@@ -212,132 +237,6 @@ int amdgpu_vcn_resume(struct amdgpu_device *adev)
return 0;
}
-static int amdgpu_vcn_pause_dpg_mode(struct amdgpu_device *adev,
- struct dpg_pause_state *new_state)
-{
- int ret_code;
- uint32_t reg_data = 0;
- uint32_t reg_data2 = 0;
- struct amdgpu_ring *ring;
-
- /* pause/unpause if state is changed */
- if (adev->vcn.pause_state.fw_based != new_state->fw_based) {
- DRM_DEBUG("dpg pause state changed %d:%d -> %d:%d",
- adev->vcn.pause_state.fw_based, adev->vcn.pause_state.jpeg,
- new_state->fw_based, new_state->jpeg);
-
- reg_data = RREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE) &
- (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
-
- if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
- ret_code = 0;
-
- if (!(reg_data & UVD_DPG_PAUSE__JPEG_PAUSE_DPG_ACK_MASK))
- SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
- UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF,
- UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
-
- if (!ret_code) {
- /* pause DPG non-jpeg */
- reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
- WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
- SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_DPG_PAUSE,
- UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
- UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK, ret_code);
-
- /* Restore */
- ring = &adev->vcn.ring_enc[0];
- WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
- WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
- WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
- WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
- WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
-
- ring = &adev->vcn.ring_enc[1];
- WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
- WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
- WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
- WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
- WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
-
- ring = &adev->vcn.ring_dec;
- WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
- RREG32_SOC15(UVD, 0, mmUVD_SCRATCH2) & 0x7FFFFFFF);
- SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
- UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON,
- UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
- }
- } else {
- /* unpause dpg non-jpeg, no need to wait */
- reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
- WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
- }
- adev->vcn.pause_state.fw_based = new_state->fw_based;
- }
-
- /* pause/unpause if state is changed */
- if (adev->vcn.pause_state.jpeg != new_state->jpeg) {
- DRM_DEBUG("dpg pause state changed %d:%d -> %d:%d",
- adev->vcn.pause_state.fw_based, adev->vcn.pause_state.jpeg,
- new_state->fw_based, new_state->jpeg);
-
- reg_data = RREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE) &
- (~UVD_DPG_PAUSE__JPEG_PAUSE_DPG_ACK_MASK);
-
- if (new_state->jpeg == VCN_DPG_STATE__PAUSE) {
- ret_code = 0;
-
- if (!(reg_data & UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK))
- SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
- UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF,
- UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
-
- if (!ret_code) {
- /* Make sure JPRG Snoop is disabled before sending the pause */
- reg_data2 = RREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS);
- reg_data2 |= UVD_POWER_STATUS__JRBC_SNOOP_DIS_MASK;
- WREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS, reg_data2);
-
- /* pause DPG jpeg */
- reg_data |= UVD_DPG_PAUSE__JPEG_PAUSE_DPG_REQ_MASK;
- WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
- SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_DPG_PAUSE,
- UVD_DPG_PAUSE__JPEG_PAUSE_DPG_ACK_MASK,
- UVD_DPG_PAUSE__JPEG_PAUSE_DPG_ACK_MASK, ret_code);
-
- /* Restore */
- ring = &adev->vcn.ring_jpeg;
- WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_VMID, 0);
- WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_CNTL,
- UVD_JRBC_RB_CNTL__RB_NO_FETCH_MASK |
- UVD_JRBC_RB_CNTL__RB_RPTR_WR_EN_MASK);
- WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_LOW,
- lower_32_bits(ring->gpu_addr));
- WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
- upper_32_bits(ring->gpu_addr));
- WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_RPTR, ring->wptr);
- WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR, ring->wptr);
- WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_CNTL,
- UVD_JRBC_RB_CNTL__RB_RPTR_WR_EN_MASK);
-
- ring = &adev->vcn.ring_dec;
- WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
- RREG32_SOC15(UVD, 0, mmUVD_SCRATCH2) & 0x7FFFFFFF);
- SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
- UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON,
- UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
- }
- } else {
- /* unpause dpg jpeg, no need to wait */
- reg_data &= ~UVD_DPG_PAUSE__JPEG_PAUSE_DPG_REQ_MASK;
- WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
- }
- adev->vcn.pause_state.jpeg = new_state->jpeg;
- }
-
- return 0;
-}
-
static void amdgpu_vcn_idle_work_handler(struct work_struct *work)
{
struct amdgpu_device *adev =
@@ -362,7 +261,7 @@ static void amdgpu_vcn_idle_work_handler(struct work_struct *work)
else
new_state.jpeg = VCN_DPG_STATE__UNPAUSE;
- amdgpu_vcn_pause_dpg_mode(adev, &new_state);
+ adev->vcn.pause_dpg_mode(adev, &new_state);
}
fences += amdgpu_fence_count_emitted(&adev->vcn.ring_jpeg);
@@ -370,7 +269,7 @@ static void amdgpu_vcn_idle_work_handler(struct work_struct *work)
if (fences == 0) {
amdgpu_gfx_off_ctrl(adev, true);
- if (adev->pm.dpm_enabled)
+ if (adev->asic_type < CHIP_NAVI10 && adev->pm.dpm_enabled)
amdgpu_dpm_enable_uvd(adev, false);
else
amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCN,
@@ -387,7 +286,7 @@ void amdgpu_vcn_ring_begin_use(struct amdgpu_ring *ring)
if (set_clocks) {
amdgpu_gfx_off_ctrl(adev, false);
- if (adev->pm.dpm_enabled)
+ if (adev->asic_type < CHIP_NAVI10 && adev->pm.dpm_enabled)
amdgpu_dpm_enable_uvd(adev, true);
else
amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCN,
@@ -417,7 +316,7 @@ void amdgpu_vcn_ring_begin_use(struct amdgpu_ring *ring)
else if (ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG)
new_state.jpeg = VCN_DPG_STATE__PAUSE;
- amdgpu_vcn_pause_dpg_mode(adev, &new_state);
+ adev->vcn.pause_dpg_mode(adev, &new_state);
}
}
@@ -433,20 +332,18 @@ int amdgpu_vcn_dec_ring_test_ring(struct amdgpu_ring *ring)
unsigned i;
int r;
- WREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_SCRATCH9), 0xCAFEDEAD);
+ WREG32(adev->vcn.external.scratch9, 0xCAFEDEAD);
r = amdgpu_ring_alloc(ring, 3);
if (r)
return r;
-
- amdgpu_ring_write(ring,
- PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_SCRATCH9), 0));
+ amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.scratch9, 0));
amdgpu_ring_write(ring, 0xDEADBEEF);
amdgpu_ring_commit(ring);
for (i = 0; i < adev->usec_timeout; i++) {
- tmp = RREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_SCRATCH9));
+ tmp = RREG32(adev->vcn.external.scratch9);
if (tmp == 0xDEADBEEF)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
@@ -472,14 +369,14 @@ static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring,
ib = &job->ibs[0];
addr = amdgpu_bo_gpu_offset(bo);
- ib->ptr[0] = PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0);
+ ib->ptr[0] = PACKET0(adev->vcn.internal.data0, 0);
ib->ptr[1] = addr;
- ib->ptr[2] = PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0);
+ ib->ptr[2] = PACKET0(adev->vcn.internal.data1, 0);
ib->ptr[3] = addr >> 32;
- ib->ptr[4] = PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0);
+ ib->ptr[4] = PACKET0(adev->vcn.internal.cmd, 0);
ib->ptr[5] = 0;
for (i = 6; i < 16; i += 2) {
- ib->ptr[i] = PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP), 0);
+ ib->ptr[i] = PACKET0(adev->vcn.internal.nop, 0);
ib->ptr[i+1] = 0;
}
ib->length_dw = 16;
@@ -610,7 +507,7 @@ int amdgpu_vcn_enc_ring_test_ring(struct amdgpu_ring *ring)
for (i = 0; i < adev->usec_timeout; i++) {
if (amdgpu_ring_get_rptr(ring) != rptr)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
@@ -754,22 +651,20 @@ int amdgpu_vcn_jpeg_ring_test_ring(struct amdgpu_ring *ring)
unsigned i;
int r;
- WREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_SCRATCH9), 0xCAFEDEAD);
+ WREG32(adev->vcn.external.jpeg_pitch, 0xCAFEDEAD);
r = amdgpu_ring_alloc(ring, 3);
-
if (r)
return r;
- amdgpu_ring_write(ring,
- PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_SCRATCH9), 0, 0, 0));
+ amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.jpeg_pitch, 0));
amdgpu_ring_write(ring, 0xDEADBEEF);
amdgpu_ring_commit(ring);
for (i = 0; i < adev->usec_timeout; i++) {
- tmp = RREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_SCRATCH9));
+ tmp = RREG32(adev->vcn.external.jpeg_pitch);
if (tmp == 0xDEADBEEF)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
@@ -794,7 +689,7 @@ static int amdgpu_vcn_jpeg_set_reg(struct amdgpu_ring *ring, uint32_t handle,
ib = &job->ibs[0];
- ib->ptr[0] = PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_SCRATCH9), 0, 0, PACKETJ_TYPE0);
+ ib->ptr[0] = PACKETJ(adev->vcn.internal.jpeg_pitch, 0, 0, PACKETJ_TYPE0);
ib->ptr[1] = 0xDEADBEEF;
for (i = 2; i < 16; i += 2) {
ib->ptr[i] = PACKETJ(0, 0, 0, PACKETJ_TYPE6);
@@ -840,10 +735,10 @@ int amdgpu_vcn_jpeg_ring_test_ib(struct amdgpu_ring *ring, long timeout)
}
for (i = 0; i < adev->usec_timeout; i++) {
- tmp = RREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_SCRATCH9));
+ tmp = RREG32(adev->vcn.external.jpeg_pitch);
if (tmp == 0xDEADBEEF)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
index a0ad19af9080..19661c645703 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
@@ -25,11 +25,12 @@
#define __AMDGPU_VCN_H__
#define AMDGPU_VCN_STACK_SIZE (128*1024)
-#define AMDGPU_VCN_CONTEXT_SIZE (512*1024)
+#define AMDGPU_VCN_CONTEXT_SIZE (512*1024)
#define AMDGPU_VCN_FIRMWARE_OFFSET 256
#define AMDGPU_VCN_MAX_ENC_RINGS 3
+#define VCN_DEC_KMD_CMD 0x80000000
#define VCN_DEC_CMD_FENCE 0x00000000
#define VCN_DEC_CMD_TRAP 0x00000001
#define VCN_DEC_CMD_WRITE_REG 0x00000004
@@ -45,8 +46,81 @@
#define VCN_ENC_CMD_REG_WRITE 0x0000000b
#define VCN_ENC_CMD_REG_WAIT 0x0000000c
+#define VCN_VID_SOC_ADDRESS_2_0 0x1fa00
+#define VCN_AON_SOC_ADDRESS_2_0 0x1f800
+#define VCN_VID_IP_ADDRESS_2_0 0x0
+#define VCN_AON_IP_ADDRESS_2_0 0x30000
+
+#define RREG32_SOC15_DPG_MODE(ip, inst, reg, mask, sram_sel) \
+ ({ WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_MASK, mask); \
+ WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_CTL, \
+ UVD_DPG_LMA_CTL__MASK_EN_MASK | \
+ ((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) \
+ << UVD_DPG_LMA_CTL__READ_WRITE_ADDR__SHIFT) | \
+ (sram_sel << UVD_DPG_LMA_CTL__SRAM_SEL__SHIFT)); \
+ RREG32_SOC15(ip, inst, mmUVD_DPG_LMA_DATA); \
+ })
+
+#define WREG32_SOC15_DPG_MODE(ip, inst, reg, value, mask, sram_sel) \
+ do { \
+ WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_DATA, value); \
+ WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_MASK, mask); \
+ WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_CTL, \
+ UVD_DPG_LMA_CTL__READ_WRITE_MASK | \
+ ((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) \
+ << UVD_DPG_LMA_CTL__READ_WRITE_ADDR__SHIFT) | \
+ (sram_sel << UVD_DPG_LMA_CTL__SRAM_SEL__SHIFT)); \
+ } while (0)
+
+#define SOC15_DPG_MODE_OFFSET_2_0(ip, inst, reg) \
+ ({ \
+ uint32_t internal_reg_offset, addr; \
+ bool video_range, aon_range; \
+ \
+ addr = (adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg); \
+ addr <<= 2; \
+ video_range = ((((0xFFFFF & addr) >= (VCN_VID_SOC_ADDRESS_2_0)) && \
+ ((0xFFFFF & addr) < ((VCN_VID_SOC_ADDRESS_2_0 + 0x2600))))); \
+ aon_range = ((((0xFFFFF & addr) >= (VCN_AON_SOC_ADDRESS_2_0)) && \
+ ((0xFFFFF & addr) < ((VCN_AON_SOC_ADDRESS_2_0 + 0x600))))); \
+ if (video_range) \
+ internal_reg_offset = ((0xFFFFF & addr) - (VCN_VID_SOC_ADDRESS_2_0) + \
+ (VCN_VID_IP_ADDRESS_2_0)); \
+ else if (aon_range) \
+ internal_reg_offset = ((0xFFFFF & addr) - (VCN_AON_SOC_ADDRESS_2_0) + \
+ (VCN_AON_IP_ADDRESS_2_0)); \
+ else \
+ internal_reg_offset = (0xFFFFF & addr); \
+ \
+ internal_reg_offset >>= 2; \
+ })
+
+#define RREG32_SOC15_DPG_MODE_2_0(offset, mask_en) \
+ ({ \
+ WREG32_SOC15(VCN, 0, mmUVD_DPG_LMA_CTL, \
+ (0x0 << UVD_DPG_LMA_CTL__READ_WRITE__SHIFT | \
+ mask_en << UVD_DPG_LMA_CTL__MASK_EN__SHIFT | \
+ offset << UVD_DPG_LMA_CTL__READ_WRITE_ADDR__SHIFT)); \
+ RREG32_SOC15(VCN, 0, mmUVD_DPG_LMA_DATA); \
+ })
+
+#define WREG32_SOC15_DPG_MODE_2_0(offset, value, mask_en, indirect) \
+ do { \
+ if (!indirect) { \
+ WREG32_SOC15(VCN, 0, mmUVD_DPG_LMA_DATA, value); \
+ WREG32_SOC15(VCN, 0, mmUVD_DPG_LMA_CTL, \
+ (0x1 << UVD_DPG_LMA_CTL__READ_WRITE__SHIFT | \
+ mask_en << UVD_DPG_LMA_CTL__MASK_EN__SHIFT | \
+ offset << UVD_DPG_LMA_CTL__READ_WRITE_ADDR__SHIFT)); \
+ } else { \
+ *adev->vcn.dpg_sram_curr_addr++ = offset; \
+ *adev->vcn.dpg_sram_curr_addr++ = value; \
+ } \
+ } while (0)
+
enum engine_status_constants {
UVD_PGFSM_STATUS__UVDM_UVDU_PWR_ON = 0x2AAAA0,
+ UVD_PGFSM_STATUS__UVDM_UVDU_PWR_ON_2_0 = 0xAAAA0,
UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON = 0x00000002,
UVD_STATUS__UVD_BUSY = 0x00000004,
GB_ADDR_CONFIG_DEFAULT = 0x26010011,
@@ -54,6 +128,7 @@ enum engine_status_constants {
UVD_STATUS__BUSY = 0x5,
UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF = 0x1,
UVD_STATUS__RBC_BUSY = 0x1,
+ UVD_PGFSM_STATUS_UVDJ_PWR_ON = 0,
};
enum internal_dpg_state {
@@ -66,6 +141,15 @@ struct dpg_pause_state {
enum internal_dpg_state jpeg;
};
+struct amdgpu_vcn_reg{
+ unsigned data0;
+ unsigned data1;
+ unsigned cmd;
+ unsigned nop;
+ unsigned scratch9;
+ unsigned jpeg_pitch;
+};
+
struct amdgpu_vcn {
struct amdgpu_bo *vcpu_bo;
void *cpu_addr;
@@ -81,6 +165,15 @@ struct amdgpu_vcn {
unsigned num_enc_rings;
enum amd_powergating_state cur_state;
struct dpg_pause_state pause_state;
+ struct amdgpu_vcn_reg internal, external;
+ int (*pause_dpg_mode)(struct amdgpu_device *adev,
+ struct dpg_pause_state *new_state);
+
+ bool indirect_sram;
+ struct amdgpu_bo *dpg_sram_bo;
+ void *dpg_sram_cpu_addr;
+ uint64_t dpg_sram_gpu_addr;
+ uint32_t *dpg_sram_curr_addr;
};
int amdgpu_vcn_sw_init(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 7d484fad3909..59dd204498c5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -21,6 +21,10 @@
*
*/
+#include <linux/module.h>
+
+#include <drm/drm_drv.h>
+
#include "amdgpu.h"
bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev)
@@ -386,7 +390,8 @@ static uint32_t parse_clk(char *buf, bool min)
if (!ptr)
break;
ptr+=2;
- clk = simple_strtoul(ptr, NULL, 10);
+ if (kstrtou32(ptr, 10, &clk))
+ return 0;
} while (!min);
return clk * 100;
@@ -426,3 +431,47 @@ uint32_t amdgpu_virt_get_mclk(struct amdgpu_device *adev, bool lowest)
return clk;
}
+void amdgpu_virt_init_reg_access_mode(struct amdgpu_device *adev)
+{
+ struct amdgpu_virt *virt = &adev->virt;
+
+ if (virt->ops && virt->ops->init_reg_access_mode)
+ virt->ops->init_reg_access_mode(adev);
+}
+
+bool amdgpu_virt_support_psp_prg_ih_reg(struct amdgpu_device *adev)
+{
+ bool ret = false;
+ struct amdgpu_virt *virt = &adev->virt;
+
+ if (amdgpu_sriov_vf(adev)
+ && (virt->reg_access_mode & AMDGPU_VIRT_REG_ACCESS_PSP_PRG_IH))
+ ret = true;
+
+ return ret;
+}
+
+bool amdgpu_virt_support_rlc_prg_reg(struct amdgpu_device *adev)
+{
+ bool ret = false;
+ struct amdgpu_virt *virt = &adev->virt;
+
+ if (amdgpu_sriov_vf(adev)
+ && (virt->reg_access_mode & AMDGPU_VIRT_REG_ACCESS_RLC)
+ && !(amdgpu_sriov_runtime(adev)))
+ ret = true;
+
+ return ret;
+}
+
+bool amdgpu_virt_support_skip_setting(struct amdgpu_device *adev)
+{
+ bool ret = false;
+ struct amdgpu_virt *virt = &adev->virt;
+
+ if (amdgpu_sriov_vf(adev)
+ && (virt->reg_access_mode & AMDGPU_VIRT_REG_SKIP_SEETING))
+ ret = true;
+
+ return ret;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 584947b7ccf3..f5107731e9c4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -48,6 +48,12 @@ struct amdgpu_vf_error_buffer {
uint64_t data[AMDGPU_VF_ERROR_ENTRY_SIZE];
};
+/* According to the fw feature, some new reg access modes are supported */
+#define AMDGPU_VIRT_REG_ACCESS_LEGACY (1 << 0) /* directly mmio */
+#define AMDGPU_VIRT_REG_ACCESS_PSP_PRG_IH (1 << 1) /* by PSP */
+#define AMDGPU_VIRT_REG_ACCESS_RLC (1 << 2) /* by RLC */
+#define AMDGPU_VIRT_REG_SKIP_SEETING (1 << 3) /* Skip setting reg */
+
/**
* struct amdgpu_virt_ops - amdgpu device virt operations
*/
@@ -59,6 +65,7 @@ struct amdgpu_virt_ops {
void (*trans_msg)(struct amdgpu_device *adev, u32 req, u32 data1, u32 data2, u32 data3);
int (*get_pp_clk)(struct amdgpu_device *adev, u32 type, char *buf);
int (*force_dpm_level)(struct amdgpu_device *adev, u32 level);
+ void (*init_reg_access_mode)(struct amdgpu_device *adev);
};
/*
@@ -246,6 +253,7 @@ typedef struct amdgim_vf2pf_info_v2 amdgim_vf2pf_info ;
struct amdgpu_virt {
uint32_t caps;
struct amdgpu_bo *csa_obj;
+ void *csa_cpu_addr;
bool chained_ib_support;
uint32_t reg_val_offs;
struct amdgpu_irq_src ack_irq;
@@ -258,6 +266,7 @@ struct amdgpu_virt {
uint32_t gim_feature;
/* protect DPM events to GIM */
struct mutex dpm_mutex;
+ uint32_t reg_access_mode;
};
#define amdgpu_sriov_enabled(adev) \
@@ -307,4 +316,9 @@ void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev);
uint32_t amdgpu_virt_get_sclk(struct amdgpu_device *adev, bool lowest);
uint32_t amdgpu_virt_get_mclk(struct amdgpu_device *adev, bool lowest);
+void amdgpu_virt_init_reg_access_mode(struct amdgpu_device *adev);
+bool amdgpu_virt_support_psp_prg_ih_reg(struct amdgpu_device *adev);
+bool amdgpu_virt_support_rlc_prg_reg(struct amdgpu_device *adev);
+bool amdgpu_virt_support_skip_setting(struct amdgpu_device *adev);
+
#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 4f10f5aba00b..24c3c05e2fb7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -28,7 +28,7 @@
#include <linux/dma-fence-array.h>
#include <linux/interval_tree_generic.h>
#include <linux/idr.h>
-#include <drm/drmP.h>
+
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
#include "amdgpu_trace.h"
@@ -1574,12 +1574,22 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
flags &= ~AMDGPU_PTE_EXECUTABLE;
flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
- flags &= ~AMDGPU_PTE_MTYPE_MASK;
- flags |= (mapping->flags & AMDGPU_PTE_MTYPE_MASK);
+ if (adev->asic_type == CHIP_NAVI10) {
+ flags &= ~AMDGPU_PTE_MTYPE_NV10_MASK;
+ flags |= (mapping->flags & AMDGPU_PTE_MTYPE_NV10_MASK);
+ } else {
+ flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
+ flags |= (mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK);
+ }
if ((mapping->flags & AMDGPU_PTE_PRT) &&
(adev->asic_type >= CHIP_VEGA10)) {
flags |= AMDGPU_PTE_PRT;
+ if (adev->asic_type >= CHIP_NAVI10) {
+ flags |= AMDGPU_PTE_SNOOPED;
+ flags |= AMDGPU_PTE_LOG;
+ flags |= AMDGPU_PTE_SYSTEM;
+ }
flags &= ~AMDGPU_PTE_VALID;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 91baf95212a6..489a162ca620 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -67,6 +67,8 @@ struct amdgpu_bo_list_entry;
/* PDE is handled as PTE for VEGA10 */
#define AMDGPU_PDE_PTE (1ULL << 54)
+#define AMDGPU_PTE_LOG (1ULL << 55)
+
/* PTE is handled as PDE for VEGA10 (Translate Further) */
#define AMDGPU_PTE_TF (1ULL << 56)
@@ -75,8 +77,8 @@ struct amdgpu_bo_list_entry;
/* For GFX9 */
-#define AMDGPU_PTE_MTYPE(a) ((uint64_t)a << 57)
-#define AMDGPU_PTE_MTYPE_MASK AMDGPU_PTE_MTYPE(3ULL)
+#define AMDGPU_PTE_MTYPE_VG10(a) ((uint64_t)(a) << 57)
+#define AMDGPU_PTE_MTYPE_VG10_MASK AMDGPU_PTE_MTYPE_VG10(3ULL)
#define AMDGPU_MTYPE_NC 0
#define AMDGPU_MTYPE_CC 2
@@ -86,7 +88,11 @@ struct amdgpu_bo_list_entry;
| AMDGPU_PTE_EXECUTABLE \
| AMDGPU_PTE_READABLE \
| AMDGPU_PTE_WRITEABLE \
- | AMDGPU_PTE_MTYPE(AMDGPU_MTYPE_CC))
+ | AMDGPU_PTE_MTYPE_VG10(AMDGPU_MTYPE_CC))
+
+/* NAVI10 only */
+#define AMDGPU_PTE_MTYPE_NV10(a) ((uint64_t)(a) << 48)
+#define AMDGPU_PTE_MTYPE_NV10_MASK AMDGPU_PTE_MTYPE_NV10(7ULL)
/* How to programm VM fault handling */
#define AMDGPU_VM_FAULT_STOP_NEVER 0
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index ec9ea3fdbb4a..3a9d8c15fe9f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -22,7 +22,6 @@
* Authors: Christian König
*/
-#include <drm/drmP.h>
#include "amdgpu.h"
struct amdgpu_vram_mgr {
@@ -276,7 +275,7 @@ static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
struct drm_mm_node *nodes;
enum drm_mm_insert_mode mode;
unsigned long lpfn, num_nodes, pages_per_node, pages_left;
- uint64_t usage = 0, vis_usage = 0;
+ uint64_t vis_usage = 0, mem_bytes;
unsigned i;
int r;
@@ -284,20 +283,34 @@ static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
if (!lpfn)
lpfn = man->size;
- if (place->flags & TTM_PL_FLAG_CONTIGUOUS ||
- amdgpu_vram_page_split == -1) {
+ /* bail out quickly if there's likely not enough VRAM for this BO */
+ mem_bytes = (u64)mem->num_pages << PAGE_SHIFT;
+ if (atomic64_add_return(mem_bytes, &mgr->usage) > adev->gmc.mc_vram_size) {
+ atomic64_sub(mem_bytes, &mgr->usage);
+ mem->mm_node = NULL;
+ return 0;
+ }
+
+ if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
pages_per_node = ~0ul;
num_nodes = 1;
} else {
- pages_per_node = max((uint32_t)amdgpu_vram_page_split,
- mem->page_alignment);
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ pages_per_node = HPAGE_PMD_NR;
+#else
+ /* default to 2MB */
+ pages_per_node = (2UL << (20UL - PAGE_SHIFT));
+#endif
+ pages_per_node = max((uint32_t)pages_per_node, mem->page_alignment);
num_nodes = DIV_ROUND_UP(mem->num_pages, pages_per_node);
}
- nodes = kvmalloc_array(num_nodes, sizeof(*nodes),
+ nodes = kvmalloc_array((uint32_t)num_nodes, sizeof(*nodes),
GFP_KERNEL | __GFP_ZERO);
- if (!nodes)
+ if (!nodes) {
+ atomic64_sub(mem_bytes, &mgr->usage);
return -ENOMEM;
+ }
mode = DRM_MM_INSERT_BEST;
if (place->flags & TTM_PL_FLAG_TOPDOWN)
@@ -317,7 +330,6 @@ static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
if (unlikely(r))
break;
- usage += nodes[i].size << PAGE_SHIFT;
vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]);
amdgpu_vram_mgr_virt_start(mem, &nodes[i]);
pages_left -= pages;
@@ -337,14 +349,12 @@ static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
if (unlikely(r))
goto error;
- usage += nodes[i].size << PAGE_SHIFT;
vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]);
amdgpu_vram_mgr_virt_start(mem, &nodes[i]);
pages_left -= pages;
}
spin_unlock(&mgr->lock);
- atomic64_add(usage, &mgr->usage);
atomic64_add(vis_usage, &mgr->vis_usage);
mem->mm_node = nodes;
@@ -355,6 +365,7 @@ error:
while (i--)
drm_mm_remove_node(&nodes[i]);
spin_unlock(&mgr->lock);
+ atomic64_sub(mem->num_pages << PAGE_SHIFT, &mgr->usage);
kvfree(nodes);
return r == -ENOSPC ? 0 : r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index a48c84c51775..d11eba09eadd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
@@ -40,6 +40,34 @@ void *amdgpu_xgmi_hive_try_lock(struct amdgpu_hive_info *hive)
return &hive->device_list;
}
+/**
+ * DOC: AMDGPU XGMI Support
+ *
+ * XGMI is a high speed interconnect that joins multiple GPU cards
+ * into a homogeneous memory space that is organized by a collective
+ * hive ID and individual node IDs, both of which are 64-bit numbers.
+ *
+ * The file xgmi_device_id contains the unique per GPU device ID and
+ * is stored in the /sys/class/drm/card${cardno}/device/ directory.
+ *
+ * Inside the device directory a sub-directory 'xgmi_hive_info' is
+ * created which contains the hive ID and the list of nodes.
+ *
+ * The hive ID is stored in:
+ * /sys/class/drm/card${cardno}/device/xgmi_hive_info/xgmi_hive_id
+ *
+ * The node information is stored in numbered directories:
+ * /sys/class/drm/card${cardno}/device/xgmi_hive_info/node${nodeno}/xgmi_device_id
+ *
+ * Each device has their own xgmi_hive_info direction with a mirror
+ * set of node sub-directories.
+ *
+ * The XGMI memory space is built by contiguously adding the power of
+ * two padded VRAM space from each node to each other.
+ *
+ */
+
+
static ssize_t amdgpu_xgmi_show_hive_id(struct device *dev,
struct device_attribute *attr, char *buf)
{
@@ -238,7 +266,7 @@ int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_dev
/* Each psp need to set the latest topology */
ret = psp_xgmi_set_topology_info(&adev->psp,
hive->number_devices,
- &hive->topology_info);
+ &adev->psp.xgmi_context.top_info);
if (ret)
dev_err(adev->dev,
"XGMI: Set topology failure on device %llx, hive %llx, ret %d",
@@ -248,9 +276,22 @@ int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_dev
return ret;
}
+
+int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
+ struct amdgpu_device *peer_adev)
+{
+ struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
+ int i;
+
+ for (i = 0 ; i < top->num_nodes; ++i)
+ if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
+ return top->nodes[i].num_hops;
+ return -EINVAL;
+}
+
int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
{
- struct psp_xgmi_topology_info *hive_topology;
+ struct psp_xgmi_topology_info *top_info;
struct amdgpu_hive_info *hive;
struct amdgpu_xgmi *entry;
struct amdgpu_device *tmp_adev = NULL;
@@ -283,35 +324,46 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
goto exit;
}
- hive_topology = &hive->topology_info;
+ top_info = &adev->psp.xgmi_context.top_info;
list_add_tail(&adev->gmc.xgmi.head, &hive->device_list);
list_for_each_entry(entry, &hive->device_list, head)
- hive_topology->nodes[count++].node_id = entry->node_id;
+ top_info->nodes[count++].node_id = entry->node_id;
+ top_info->num_nodes = count;
hive->number_devices = count;
- /* Each psp need to get the latest topology */
list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
- ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count, hive_topology);
+ /* update node list for other device in the hive */
+ if (tmp_adev != adev) {
+ top_info = &tmp_adev->psp.xgmi_context.top_info;
+ top_info->nodes[count - 1].node_id = adev->gmc.xgmi.node_id;
+ top_info->num_nodes = count;
+ }
+ ret = amdgpu_xgmi_update_topology(hive, tmp_adev);
+ if (ret)
+ goto exit;
+ }
+
+ /* get latest topology info for each device from psp */
+ list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
+ ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count,
+ &tmp_adev->psp.xgmi_context.top_info);
if (ret) {
dev_err(tmp_adev->dev,
"XGMI: Get topology failure on device %llx, hive %llx, ret %d",
tmp_adev->gmc.xgmi.node_id,
tmp_adev->gmc.xgmi.hive_id, ret);
/* To do : continue with some node failed or disable the whole hive */
- break;
+ goto exit;
}
}
- list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
- ret = amdgpu_xgmi_update_topology(hive, tmp_adev);
- if (ret)
- break;
- }
-
if (!ret)
ret = amdgpu_xgmi_sysfs_add_dev_info(adev, hive);
+
+ mutex_unlock(&hive->hive_lock);
+exit:
if (!ret)
dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n",
adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id);
@@ -320,9 +372,6 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id,
ret);
-
- mutex_unlock(&hive->hive_lock);
-exit:
return ret;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
index 3e9c91e9a4bf..fbcee31788c4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
@@ -27,7 +27,6 @@
struct amdgpu_hive_info {
uint64_t hive_id;
struct list_head device_list;
- struct psp_xgmi_topology_info topology_info;
int number_devices;
struct mutex hive_lock, reset_lock;
struct kobject *kobj;
@@ -41,6 +40,8 @@ int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_dev
int amdgpu_xgmi_add_device(struct amdgpu_device *adev);
void amdgpu_xgmi_remove_device(struct amdgpu_device *adev);
int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate);
+int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
+ struct amdgpu_device *peer_adev);
static inline bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
struct amdgpu_device *bo_adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/athub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/athub_v2_0.c
new file mode 100644
index 000000000000..89b32b6b81c8
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/athub_v2_0.c
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "amdgpu.h"
+#include "athub_v2_0.h"
+
+#include "athub/athub_2_0_0_offset.h"
+#include "athub/athub_2_0_0_sh_mask.h"
+#include "athub/athub_2_0_0_default.h"
+#include "navi10_enum.h"
+
+#include "soc15_common.h"
+
+static void
+athub_v2_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
+ bool enable)
+{
+ uint32_t def, data;
+
+ def = data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL);
+
+ if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG))
+ data |= ATHUB_MISC_CNTL__CG_ENABLE_MASK;
+ else
+ data &= ~ATHUB_MISC_CNTL__CG_ENABLE_MASK;
+
+ if (def != data)
+ WREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL, data);
+}
+
+static void
+athub_v2_0_update_medium_grain_light_sleep(struct amdgpu_device *adev,
+ bool enable)
+{
+ uint32_t def, data;
+
+ def = data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL);
+
+ if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS) &&
+ (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS))
+ data |= ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK;
+ else
+ data &= ~ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK;
+
+ if (def != data)
+ WREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL, data);
+}
+
+int athub_v2_0_set_clockgating(struct amdgpu_device *adev,
+ enum amd_clockgating_state state)
+{
+ if (amdgpu_sriov_vf(adev))
+ return 0;
+
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ athub_v2_0_update_medium_grain_clock_gating(adev,
+ state == AMD_CG_STATE_GATE ? true : false);
+ athub_v2_0_update_medium_grain_light_sleep(adev,
+ state == AMD_CG_STATE_GATE ? true : false);
+ break;
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+void athub_v2_0_get_clockgating(struct amdgpu_device *adev, u32 *flags)
+{
+ int data;
+
+ /* AMD_CG_SUPPORT_ATHUB_MGCG */
+ data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL);
+ if (data & ATHUB_MISC_CNTL__CG_ENABLE_MASK)
+ *flags |= AMD_CG_SUPPORT_ATHUB_MGCG;
+
+ /* AMD_CG_SUPPORT_ATHUB_LS */
+ if (data & ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK)
+ *flags |= AMD_CG_SUPPORT_ATHUB_LS;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/athub_v2_0.h b/drivers/gpu/drm/amd/amdgpu/athub_v2_0.h
new file mode 100644
index 000000000000..02932c1c8bab
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/athub_v2_0.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __ATHUB_V2_0_H__
+#define __ATHUB_V2_0_H__
+
+int athub_v2_0_set_clockgating(struct amdgpu_device *adev,
+ enum amd_clockgating_state state);
+void athub_v2_0_get_clockgating(struct amdgpu_device *adev, u32 *flags);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/atom.h b/drivers/gpu/drm/amd/amdgpu/atom.h
index a39170991afe..4205bbe5d8d7 100644
--- a/drivers/gpu/drm/amd/amdgpu/atom.h
+++ b/drivers/gpu/drm/amd/amdgpu/atom.h
@@ -26,7 +26,8 @@
#define ATOM_H
#include <linux/types.h>
-#include <drm/drmP.h>
+
+struct drm_device;
#define ATOM_BIOS_MAGIC 0xAA55
#define ATOM_ATI_MAGIC_PTR 0x30
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c b/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c
index 8a0818b23ea4..213e62a28ba0 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c
@@ -23,7 +23,7 @@
* Authors: Dave Airlie
* Alex Deucher
*/
-#include <drm/drmP.h>
+
#include <drm/drm_crtc_helper.h>
#include <drm/amdgpu_drm.h>
#include <drm/drm_fixed.h>
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_dp.c b/drivers/gpu/drm/amd/amdgpu/atombios_dp.c
index f81068ba4cc6..6858cde9fc5d 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_dp.c
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_dp.c
@@ -24,7 +24,7 @@
* Alex Deucher
* Jerome Glisse
*/
-#include <drm/drmP.h>
+
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
index 60e2447e12c5..1e94a9b652f7 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
@@ -23,7 +23,9 @@
* Authors: Dave Airlie
* Alex Deucher
*/
-#include <drm/drmP.h>
+
+#include <linux/pci.h>
+
#include <drm/drm_crtc_helper.h>
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c b/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c
index f9b2ce9a98f3..980c363b1a0a 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c
@@ -22,7 +22,7 @@
* Authors: Alex Deucher
*
*/
-#include <drm/drmP.h>
+
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
#include "atom.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index 07c1f239e9c3..1ffbc0d3d7a1 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -24,7 +24,8 @@
#include <linux/firmware.h>
#include <linux/slab.h>
#include <linux/module.h>
-#include <drm/drmP.h>
+#include <linux/pci.h>
+
#include "amdgpu.h"
#include "amdgpu_atombios.h"
#include "amdgpu_ih.h"
@@ -1804,6 +1805,18 @@ static bool cik_need_reset_on_init(struct amdgpu_device *adev)
return false;
}
+static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
+{
+ uint64_t nak_r, nak_g;
+
+ /* Get the number of NAKs received and generated */
+ nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
+ nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
+
+ /* Add the total number of NAKs, i.e the number of replays */
+ return (nak_r + nak_g);
+}
+
static const struct amdgpu_asic_funcs cik_asic_funcs =
{
.read_disabled_bios = &cik_read_disabled_bios,
@@ -1821,6 +1834,7 @@ static const struct amdgpu_asic_funcs cik_asic_funcs =
.init_doorbell_index = &legacy_doorbell_index_init,
.get_pcie_usage = &cik_get_pcie_usage,
.need_reset_on_init = &cik_need_reset_on_init,
+ .get_pcie_replay_count = &cik_get_pcie_replay_count,
};
static int cik_common_early_init(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_ih.c b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
index 721c757156e8..401c99f0b2d0 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
@@ -20,7 +20,9 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
-#include <drm/drmP.h>
+
+#include <linux/pci.h>
+
#include "amdgpu.h"
#include "amdgpu_ih.h"
#include "cikd.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index d42808b05971..c45304f1047c 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -21,8 +21,10 @@
*
* Authors: Alex Deucher
*/
+
#include <linux/firmware.h>
-#include <drm/drmP.h>
+#include <linux/module.h>
+
#include "amdgpu.h"
#include "amdgpu_ucode.h"
#include "amdgpu_trace.h"
@@ -640,7 +642,7 @@ static int cik_sdma_ring_test_ring(struct amdgpu_ring *ring)
tmp = le32_to_cpu(adev->wb.wb[index]);
if (tmp == 0xDEADBEEF)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
diff --git a/drivers/gpu/drm/amd/amdgpu/clearstate_gfx10.h b/drivers/gpu/drm/amd/amdgpu/clearstate_gfx10.h
new file mode 100644
index 000000000000..27a2ff0a07d2
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/clearstate_gfx10.h
@@ -0,0 +1,975 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+static const unsigned int gfx10_SECT_CONTEXT_def_1[] = {
+ 0x00000000, // DB_RENDER_CONTROL
+ 0x00000000, // DB_COUNT_CONTROL
+ 0x00000000, // DB_DEPTH_VIEW
+ 0x00000000, // DB_RENDER_OVERRIDE
+ 0x00000000, // DB_RENDER_OVERRIDE2
+ 0x00000000, // DB_HTILE_DATA_BASE
+ 0x00000000, // HOLE
+ 0x00000000, // DB_DEPTH_SIZE_XY
+ 0x00000000, // DB_DEPTH_BOUNDS_MIN
+ 0x00000000, // DB_DEPTH_BOUNDS_MAX
+ 0x00000000, // DB_STENCIL_CLEAR
+ 0x00000000, // DB_DEPTH_CLEAR
+ 0x00000000, // PA_SC_SCREEN_SCISSOR_TL
+ 0x40004000, // PA_SC_SCREEN_SCISSOR_BR
+ 0x00000000, // DB_DFSM_CONTROL
+ 0x00000000, // DB_DEPTH_INFO
+ 0x00000000, // DB_Z_INFO
+ 0x00000000, // DB_STENCIL_INFO
+ 0x00000000, // DB_Z_READ_BASE
+ 0x00000000, // DB_STENCIL_READ_BASE
+ 0x00000000, // DB_Z_WRITE_BASE
+ 0x00000000, // DB_STENCIL_WRITE_BASE
+ 0x00000000, // DB_DEPTH_SIZE
+ 0x00000000, // DB_DEPTH_SLICE
+ 0x00000000, // DB_Z_INFO2
+ 0x00000000, // DB_STENCIL_INFO2
+ 0x00000000, // DB_Z_READ_BASE_HI
+ 0x00000000, // DB_STENCIL_READ_BASE_HI
+ 0x00000000, // DB_Z_WRITE_BASE_HI
+ 0x00000000, // DB_STENCIL_WRITE_BASE_HI
+ 0x00000000, // DB_HTILE_DATA_BASE_HI
+ 0x00150055, // DB_RMI_L2_CACHE_CONTROL
+ 0x00000000, // TA_BC_BASE_ADDR
+ 0x00000000, // TA_BC_BASE_ADDR_HI
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0x00000000, // COHER_DEST_BASE_HI_0
+ 0x00000000, // COHER_DEST_BASE_HI_1
+ 0x00000000, // COHER_DEST_BASE_HI_2
+ 0x00000000, // COHER_DEST_BASE_HI_3
+ 0x00000000, // COHER_DEST_BASE_2
+ 0x00000000, // COHER_DEST_BASE_3
+ 0x00000000, // PA_SC_WINDOW_OFFSET
+ 0x80000000, // PA_SC_WINDOW_SCISSOR_TL
+ 0x40004000, // PA_SC_WINDOW_SCISSOR_BR
+ 0x0000ffff, // PA_SC_CLIPRECT_RULE
+ 0x00000000, // PA_SC_CLIPRECT_0_TL
+ 0x40004000, // PA_SC_CLIPRECT_0_BR
+ 0x00000000, // PA_SC_CLIPRECT_1_TL
+ 0x40004000, // PA_SC_CLIPRECT_1_BR
+ 0x00000000, // PA_SC_CLIPRECT_2_TL
+ 0x40004000, // PA_SC_CLIPRECT_2_BR
+ 0x00000000, // PA_SC_CLIPRECT_3_TL
+ 0x40004000, // PA_SC_CLIPRECT_3_BR
+ 0xaa99aaaa, // PA_SC_EDGERULE
+ 0x00000000, // PA_SU_HARDWARE_SCREEN_OFFSET
+ 0xffffffff, // CB_TARGET_MASK
+ 0xffffffff, // CB_SHADER_MASK
+ 0x80000000, // PA_SC_GENERIC_SCISSOR_TL
+ 0x40004000, // PA_SC_GENERIC_SCISSOR_BR
+ 0x00000000, // COHER_DEST_BASE_0
+ 0x00000000, // COHER_DEST_BASE_1
+ 0x80000000, // PA_SC_VPORT_SCISSOR_0_TL
+ 0x40004000, // PA_SC_VPORT_SCISSOR_0_BR
+ 0x80000000, // PA_SC_VPORT_SCISSOR_1_TL
+ 0x40004000, // PA_SC_VPORT_SCISSOR_1_BR
+ 0x80000000, // PA_SC_VPORT_SCISSOR_2_TL
+ 0x40004000, // PA_SC_VPORT_SCISSOR_2_BR
+ 0x80000000, // PA_SC_VPORT_SCISSOR_3_TL
+ 0x40004000, // PA_SC_VPORT_SCISSOR_3_BR
+ 0x80000000, // PA_SC_VPORT_SCISSOR_4_TL
+ 0x40004000, // PA_SC_VPORT_SCISSOR_4_BR
+ 0x80000000, // PA_SC_VPORT_SCISSOR_5_TL
+ 0x40004000, // PA_SC_VPORT_SCISSOR_5_BR
+ 0x80000000, // PA_SC_VPORT_SCISSOR_6_TL
+ 0x40004000, // PA_SC_VPORT_SCISSOR_6_BR
+ 0x80000000, // PA_SC_VPORT_SCISSOR_7_TL
+ 0x40004000, // PA_SC_VPORT_SCISSOR_7_BR
+ 0x80000000, // PA_SC_VPORT_SCISSOR_8_TL
+ 0x40004000, // PA_SC_VPORT_SCISSOR_8_BR
+ 0x80000000, // PA_SC_VPORT_SCISSOR_9_TL
+ 0x40004000, // PA_SC_VPORT_SCISSOR_9_BR
+ 0x80000000, // PA_SC_VPORT_SCISSOR_10_TL
+ 0x40004000, // PA_SC_VPORT_SCISSOR_10_BR
+ 0x80000000, // PA_SC_VPORT_SCISSOR_11_TL
+ 0x40004000, // PA_SC_VPORT_SCISSOR_11_BR
+ 0x80000000, // PA_SC_VPORT_SCISSOR_12_TL
+ 0x40004000, // PA_SC_VPORT_SCISSOR_12_BR
+ 0x80000000, // PA_SC_VPORT_SCISSOR_13_TL
+ 0x40004000, // PA_SC_VPORT_SCISSOR_13_BR
+ 0x80000000, // PA_SC_VPORT_SCISSOR_14_TL
+ 0x40004000, // PA_SC_VPORT_SCISSOR_14_BR
+ 0x80000000, // PA_SC_VPORT_SCISSOR_15_TL
+ 0x40004000, // PA_SC_VPORT_SCISSOR_15_BR
+ 0x00000000, // PA_SC_VPORT_ZMIN_0
+ 0x3f800000, // PA_SC_VPORT_ZMAX_0
+ 0x00000000, // PA_SC_VPORT_ZMIN_1
+ 0x3f800000, // PA_SC_VPORT_ZMAX_1
+ 0x00000000, // PA_SC_VPORT_ZMIN_2
+ 0x3f800000, // PA_SC_VPORT_ZMAX_2
+ 0x00000000, // PA_SC_VPORT_ZMIN_3
+ 0x3f800000, // PA_SC_VPORT_ZMAX_3
+ 0x00000000, // PA_SC_VPORT_ZMIN_4
+ 0x3f800000, // PA_SC_VPORT_ZMAX_4
+ 0x00000000, // PA_SC_VPORT_ZMIN_5
+ 0x3f800000, // PA_SC_VPORT_ZMAX_5
+ 0x00000000, // PA_SC_VPORT_ZMIN_6
+ 0x3f800000, // PA_SC_VPORT_ZMAX_6
+ 0x00000000, // PA_SC_VPORT_ZMIN_7
+ 0x3f800000, // PA_SC_VPORT_ZMAX_7
+ 0x00000000, // PA_SC_VPORT_ZMIN_8
+ 0x3f800000, // PA_SC_VPORT_ZMAX_8
+ 0x00000000, // PA_SC_VPORT_ZMIN_9
+ 0x3f800000, // PA_SC_VPORT_ZMAX_9
+ 0x00000000, // PA_SC_VPORT_ZMIN_10
+ 0x3f800000, // PA_SC_VPORT_ZMAX_10
+ 0x00000000, // PA_SC_VPORT_ZMIN_11
+ 0x3f800000, // PA_SC_VPORT_ZMAX_11
+ 0x00000000, // PA_SC_VPORT_ZMIN_12
+ 0x3f800000, // PA_SC_VPORT_ZMAX_12
+ 0x00000000, // PA_SC_VPORT_ZMIN_13
+ 0x3f800000, // PA_SC_VPORT_ZMAX_13
+ 0x00000000, // PA_SC_VPORT_ZMIN_14
+ 0x3f800000, // PA_SC_VPORT_ZMAX_14
+ 0x00000000, // PA_SC_VPORT_ZMIN_15
+ 0x3f800000, // PA_SC_VPORT_ZMAX_15
+ 0x00000000, // PA_SC_RASTER_CONFIG
+ 0x00000000, // PA_SC_RASTER_CONFIG_1
+ 0x00000000, // PA_SC_SCREEN_EXTENT_CONTROL
+};
+static const unsigned int gfx10_SECT_CONTEXT_def_2[] = {
+ 0x00000000, // CP_PERFMON_CNTX_CNTL
+ 0x00000000, // CP_RINGID
+ 0x00000000, // CP_VMID
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0x00000000, // PA_SC_RIGHT_VERT_GRID
+ 0x00000000, // PA_SC_LEFT_VERT_GRID
+ 0x00000000, // PA_SC_HORIZ_GRID
+ 0x00000000, // HOLE
+ 0x00000000, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0xffffffff, // VGT_MAX_VTX_INDX
+ 0x00000000, // VGT_MIN_VTX_INDX
+ 0x00000000, // VGT_INDX_OFFSET
+ 0x00000000, // VGT_MULTI_PRIM_IB_RESET_INDX
+ 0x00550055, // CB_RMI_GL2_CACHE_CONTROL
+ 0x00000000, // CB_BLEND_RED
+ 0x00000000, // CB_BLEND_GREEN
+ 0x00000000, // CB_BLEND_BLUE
+ 0x00000000, // CB_BLEND_ALPHA
+ 0x00000000, // CB_DCC_CONTROL
+ 0x00000000, // CB_COVERAGE_OUT_CONTROL
+ 0x00000000, // DB_STENCIL_CONTROL
+ 0x01000000, // DB_STENCILREFMASK
+ 0x01000000, // DB_STENCILREFMASK_BF
+ 0, // HOLE
+ 0x00000000, // PA_CL_VPORT_XSCALE
+ 0x00000000, // PA_CL_VPORT_XOFFSET
+ 0x00000000, // PA_CL_VPORT_YSCALE
+ 0x00000000, // PA_CL_VPORT_YOFFSET
+ 0x00000000, // PA_CL_VPORT_ZSCALE
+ 0x00000000, // PA_CL_VPORT_ZOFFSET
+ 0x00000000, // PA_CL_VPORT_XSCALE_1
+ 0x00000000, // PA_CL_VPORT_XOFFSET_1
+ 0x00000000, // PA_CL_VPORT_YSCALE_1
+ 0x00000000, // PA_CL_VPORT_YOFFSET_1
+ 0x00000000, // PA_CL_VPORT_ZSCALE_1
+ 0x00000000, // PA_CL_VPORT_ZOFFSET_1
+ 0x00000000, // PA_CL_VPORT_XSCALE_2
+ 0x00000000, // PA_CL_VPORT_XOFFSET_2
+ 0x00000000, // PA_CL_VPORT_YSCALE_2
+ 0x00000000, // PA_CL_VPORT_YOFFSET_2
+ 0x00000000, // PA_CL_VPORT_ZSCALE_2
+ 0x00000000, // PA_CL_VPORT_ZOFFSET_2
+ 0x00000000, // PA_CL_VPORT_XSCALE_3
+ 0x00000000, // PA_CL_VPORT_XOFFSET_3
+ 0x00000000, // PA_CL_VPORT_YSCALE_3
+ 0x00000000, // PA_CL_VPORT_YOFFSET_3
+ 0x00000000, // PA_CL_VPORT_ZSCALE_3
+ 0x00000000, // PA_CL_VPORT_ZOFFSET_3
+ 0x00000000, // PA_CL_VPORT_XSCALE_4
+ 0x00000000, // PA_CL_VPORT_XOFFSET_4
+ 0x00000000, // PA_CL_VPORT_YSCALE_4
+ 0x00000000, // PA_CL_VPORT_YOFFSET_4
+ 0x00000000, // PA_CL_VPORT_ZSCALE_4
+ 0x00000000, // PA_CL_VPORT_ZOFFSET_4
+ 0x00000000, // PA_CL_VPORT_XSCALE_5
+ 0x00000000, // PA_CL_VPORT_XOFFSET_5
+ 0x00000000, // PA_CL_VPORT_YSCALE_5
+ 0x00000000, // PA_CL_VPORT_YOFFSET_5
+ 0x00000000, // PA_CL_VPORT_ZSCALE_5
+ 0x00000000, // PA_CL_VPORT_ZOFFSET_5
+ 0x00000000, // PA_CL_VPORT_XSCALE_6
+ 0x00000000, // PA_CL_VPORT_XOFFSET_6
+ 0x00000000, // PA_CL_VPORT_YSCALE_6
+ 0x00000000, // PA_CL_VPORT_YOFFSET_6
+ 0x00000000, // PA_CL_VPORT_ZSCALE_6
+ 0x00000000, // PA_CL_VPORT_ZOFFSET_6
+ 0x00000000, // PA_CL_VPORT_XSCALE_7
+ 0x00000000, // PA_CL_VPORT_XOFFSET_7
+ 0x00000000, // PA_CL_VPORT_YSCALE_7
+ 0x00000000, // PA_CL_VPORT_YOFFSET_7
+ 0x00000000, // PA_CL_VPORT_ZSCALE_7
+ 0x00000000, // PA_CL_VPORT_ZOFFSET_7
+ 0x00000000, // PA_CL_VPORT_XSCALE_8
+ 0x00000000, // PA_CL_VPORT_XOFFSET_8
+ 0x00000000, // PA_CL_VPORT_YSCALE_8
+ 0x00000000, // PA_CL_VPORT_YOFFSET_8
+ 0x00000000, // PA_CL_VPORT_ZSCALE_8
+ 0x00000000, // PA_CL_VPORT_ZOFFSET_8
+ 0x00000000, // PA_CL_VPORT_XSCALE_9
+ 0x00000000, // PA_CL_VPORT_XOFFSET_9
+ 0x00000000, // PA_CL_VPORT_YSCALE_9
+ 0x00000000, // PA_CL_VPORT_YOFFSET_9
+ 0x00000000, // PA_CL_VPORT_ZSCALE_9
+ 0x00000000, // PA_CL_VPORT_ZOFFSET_9
+ 0x00000000, // PA_CL_VPORT_XSCALE_10
+ 0x00000000, // PA_CL_VPORT_XOFFSET_10
+ 0x00000000, // PA_CL_VPORT_YSCALE_10
+ 0x00000000, // PA_CL_VPORT_YOFFSET_10
+ 0x00000000, // PA_CL_VPORT_ZSCALE_10
+ 0x00000000, // PA_CL_VPORT_ZOFFSET_10
+ 0x00000000, // PA_CL_VPORT_XSCALE_11
+ 0x00000000, // PA_CL_VPORT_XOFFSET_11
+ 0x00000000, // PA_CL_VPORT_YSCALE_11
+ 0x00000000, // PA_CL_VPORT_YOFFSET_11
+ 0x00000000, // PA_CL_VPORT_ZSCALE_11
+ 0x00000000, // PA_CL_VPORT_ZOFFSET_11
+ 0x00000000, // PA_CL_VPORT_XSCALE_12
+ 0x00000000, // PA_CL_VPORT_XOFFSET_12
+ 0x00000000, // PA_CL_VPORT_YSCALE_12
+ 0x00000000, // PA_CL_VPORT_YOFFSET_12
+ 0x00000000, // PA_CL_VPORT_ZSCALE_12
+ 0x00000000, // PA_CL_VPORT_ZOFFSET_12
+ 0x00000000, // PA_CL_VPORT_XSCALE_13
+ 0x00000000, // PA_CL_VPORT_XOFFSET_13
+ 0x00000000, // PA_CL_VPORT_YSCALE_13
+ 0x00000000, // PA_CL_VPORT_YOFFSET_13
+ 0x00000000, // PA_CL_VPORT_ZSCALE_13
+ 0x00000000, // PA_CL_VPORT_ZOFFSET_13
+ 0x00000000, // PA_CL_VPORT_XSCALE_14
+ 0x00000000, // PA_CL_VPORT_XOFFSET_14
+ 0x00000000, // PA_CL_VPORT_YSCALE_14
+ 0x00000000, // PA_CL_VPORT_YOFFSET_14
+ 0x00000000, // PA_CL_VPORT_ZSCALE_14
+ 0x00000000, // PA_CL_VPORT_ZOFFSET_14
+ 0x00000000, // PA_CL_VPORT_XSCALE_15
+ 0x00000000, // PA_CL_VPORT_XOFFSET_15
+ 0x00000000, // PA_CL_VPORT_YSCALE_15
+ 0x00000000, // PA_CL_VPORT_YOFFSET_15
+ 0x00000000, // PA_CL_VPORT_ZSCALE_15
+ 0x00000000, // PA_CL_VPORT_ZOFFSET_15
+ 0x00000000, // PA_CL_UCP_0_X
+ 0x00000000, // PA_CL_UCP_0_Y
+ 0x00000000, // PA_CL_UCP_0_Z
+ 0x00000000, // PA_CL_UCP_0_W
+ 0x00000000, // PA_CL_UCP_1_X
+ 0x00000000, // PA_CL_UCP_1_Y
+ 0x00000000, // PA_CL_UCP_1_Z
+ 0x00000000, // PA_CL_UCP_1_W
+ 0x00000000, // PA_CL_UCP_2_X
+ 0x00000000, // PA_CL_UCP_2_Y
+ 0x00000000, // PA_CL_UCP_2_Z
+ 0x00000000, // PA_CL_UCP_2_W
+ 0x00000000, // PA_CL_UCP_3_X
+ 0x00000000, // PA_CL_UCP_3_Y
+ 0x00000000, // PA_CL_UCP_3_Z
+ 0x00000000, // PA_CL_UCP_3_W
+ 0x00000000, // PA_CL_UCP_4_X
+ 0x00000000, // PA_CL_UCP_4_Y
+ 0x00000000, // PA_CL_UCP_4_Z
+ 0x00000000, // PA_CL_UCP_4_W
+ 0x00000000, // PA_CL_UCP_5_X
+ 0x00000000, // PA_CL_UCP_5_Y
+ 0x00000000, // PA_CL_UCP_5_Z
+ 0x00000000, // PA_CL_UCP_5_W
+ 0x00000000, // PA_CL_PROG_NEAR_CLIP_Z
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0x00000000, // SPI_PS_INPUT_CNTL_0
+ 0x00000000, // SPI_PS_INPUT_CNTL_1
+ 0x00000000, // SPI_PS_INPUT_CNTL_2
+ 0x00000000, // SPI_PS_INPUT_CNTL_3
+ 0x00000000, // SPI_PS_INPUT_CNTL_4
+ 0x00000000, // SPI_PS_INPUT_CNTL_5
+ 0x00000000, // SPI_PS_INPUT_CNTL_6
+ 0x00000000, // SPI_PS_INPUT_CNTL_7
+ 0x00000000, // SPI_PS_INPUT_CNTL_8
+ 0x00000000, // SPI_PS_INPUT_CNTL_9
+ 0x00000000, // SPI_PS_INPUT_CNTL_10
+ 0x00000000, // SPI_PS_INPUT_CNTL_11
+ 0x00000000, // SPI_PS_INPUT_CNTL_12
+ 0x00000000, // SPI_PS_INPUT_CNTL_13
+ 0x00000000, // SPI_PS_INPUT_CNTL_14
+ 0x00000000, // SPI_PS_INPUT_CNTL_15
+ 0x00000000, // SPI_PS_INPUT_CNTL_16
+ 0x00000000, // SPI_PS_INPUT_CNTL_17
+ 0x00000000, // SPI_PS_INPUT_CNTL_18
+ 0x00000000, // SPI_PS_INPUT_CNTL_19
+ 0x00000000, // SPI_PS_INPUT_CNTL_20
+ 0x00000000, // SPI_PS_INPUT_CNTL_21
+ 0x00000000, // SPI_PS_INPUT_CNTL_22
+ 0x00000000, // SPI_PS_INPUT_CNTL_23
+ 0x00000000, // SPI_PS_INPUT_CNTL_24
+ 0x00000000, // SPI_PS_INPUT_CNTL_25
+ 0x00000000, // SPI_PS_INPUT_CNTL_26
+ 0x00000000, // SPI_PS_INPUT_CNTL_27
+ 0x00000000, // SPI_PS_INPUT_CNTL_28
+ 0x00000000, // SPI_PS_INPUT_CNTL_29
+ 0x00000000, // SPI_PS_INPUT_CNTL_30
+ 0x00000000, // SPI_PS_INPUT_CNTL_31
+ 0x00000000, // SPI_VS_OUT_CONFIG
+ 0, // HOLE
+ 0x00000000, // SPI_PS_INPUT_ENA
+ 0x00000000, // SPI_PS_INPUT_ADDR
+ 0x00000000, // SPI_INTERP_CONTROL_0
+ 0x00000002, // SPI_PS_IN_CONTROL
+ 0, // HOLE
+ 0x00000000, // SPI_BARYC_CNTL
+ 0, // HOLE
+ 0x00000000, // SPI_TMPRING_SIZE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0x00000000, // SPI_SHADER_IDX_FORMAT
+ 0x00000000, // SPI_SHADER_POS_FORMAT
+ 0x00000000, // SPI_SHADER_Z_FORMAT
+ 0x00000000, // SPI_SHADER_COL_FORMAT
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0x00000000, // SX_PS_DOWNCONVERT
+ 0x00000000, // SX_BLEND_OPT_EPSILON
+ 0x00000000, // SX_BLEND_OPT_CONTROL
+ 0x00000000, // SX_MRT0_BLEND_OPT
+ 0x00000000, // SX_MRT1_BLEND_OPT
+ 0x00000000, // SX_MRT2_BLEND_OPT
+ 0x00000000, // SX_MRT3_BLEND_OPT
+ 0x00000000, // SX_MRT4_BLEND_OPT
+ 0x00000000, // SX_MRT5_BLEND_OPT
+ 0x00000000, // SX_MRT6_BLEND_OPT
+ 0x00000000, // SX_MRT7_BLEND_OPT
+ 0x00000000, // CB_BLEND0_CONTROL
+ 0x00000000, // CB_BLEND1_CONTROL
+ 0x00000000, // CB_BLEND2_CONTROL
+ 0x00000000, // CB_BLEND3_CONTROL
+ 0x00000000, // CB_BLEND4_CONTROL
+ 0x00000000, // CB_BLEND5_CONTROL
+ 0x00000000, // CB_BLEND6_CONTROL
+ 0x00000000, // CB_BLEND7_CONTROL
+};
+static const unsigned int gfx10_SECT_CONTEXT_def_3[] = {
+ 0x00000000, // PA_CL_POINT_X_RAD
+ 0x00000000, // PA_CL_POINT_Y_RAD
+ 0x00000000, // PA_CL_POINT_SIZE
+ 0x00000000, // PA_CL_POINT_CULL_RAD
+};
+static const unsigned int gfx10_SECT_CONTEXT_def_4[] = {
+ 0x00000000, // VGT_GS_MAX_PRIMS_PER_SUBGROUP
+ 0x00000000, // DB_DEPTH_CONTROL
+ 0x00000000, // DB_EQAA
+ 0x00000000, // CB_COLOR_CONTROL
+ 0x00000000, // DB_SHADER_CONTROL
+ 0x00090000, // PA_CL_CLIP_CNTL
+ 0x00000004, // PA_SU_SC_MODE_CNTL
+ 0x00000000, // PA_CL_VTE_CNTL
+ 0x00000000, // PA_CL_VS_OUT_CNTL
+ 0x00000000, // PA_CL_NANINF_CNTL
+ 0x00000000, // PA_SU_LINE_STIPPLE_CNTL
+ 0x00000000, // PA_SU_LINE_STIPPLE_SCALE
+ 0x00000000, // PA_SU_PRIM_FILTER_CNTL
+ 0x00000000, // PA_SU_SMALL_PRIM_FILTER_CNTL
+ 0x00000000, // PA_CL_OBJPRIM_ID_CNTL
+ 0x00000000, // PA_CL_NGG_CNTL
+ 0x00000000, // PA_SU_OVER_RASTERIZATION_CNTL
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0x00000000, // PA_SU_POINT_SIZE
+ 0x00000000, // PA_SU_POINT_MINMAX
+ 0x00000000, // PA_SU_LINE_CNTL
+ 0x00000000, // PA_SC_LINE_STIPPLE
+ 0x00000000, // VGT_OUTPUT_PATH_CNTL
+ 0x00000000, // VGT_HOS_CNTL
+ 0x00000000, // VGT_HOS_MAX_TESS_LEVEL
+ 0x00000000, // VGT_HOS_MIN_TESS_LEVEL
+ 0x00000000, // VGT_HOS_REUSE_DEPTH
+ 0x00000000, // VGT_GROUP_PRIM_TYPE
+ 0x00000000, // VGT_GROUP_FIRST_DECR
+ 0x00000000, // VGT_GROUP_DECR
+ 0x00000000, // VGT_GROUP_VECT_0_CNTL
+ 0x00000000, // VGT_GROUP_VECT_1_CNTL
+ 0x00000000, // VGT_GROUP_VECT_0_FMT_CNTL
+ 0x00000000, // VGT_GROUP_VECT_1_FMT_CNTL
+ 0x00000000, // VGT_GS_MODE
+ 0x00000000, // VGT_GS_ONCHIP_CNTL
+ 0x00000000, // PA_SC_MODE_CNTL_0
+ 0x00000000, // PA_SC_MODE_CNTL_1
+ 0x00000000, // VGT_ENHANCE
+ 0x00000100, // VGT_GS_PER_ES
+ 0x00000080, // VGT_ES_PER_GS
+ 0x00000002, // VGT_GS_PER_VS
+ 0x00000000, // VGT_GSVS_RING_OFFSET_1
+ 0x00000000, // VGT_GSVS_RING_OFFSET_2
+ 0x00000000, // VGT_GSVS_RING_OFFSET_3
+ 0x00000000, // VGT_GS_OUT_PRIM_TYPE
+ 0x00000000, // IA_ENHANCE
+};
+static const unsigned int gfx10_SECT_CONTEXT_def_5[] = {
+ 0x00000000, // WD_ENHANCE
+ 0x00000000, // VGT_PRIMITIVEID_EN
+};
+static const unsigned int gfx10_SECT_CONTEXT_def_6[] = {
+ 0x00000000, // VGT_PRIMITIVEID_RESET
+};
+static const unsigned int gfx10_SECT_CONTEXT_def_7[] = {
+ 0x00000000, // VGT_MULTI_PRIM_IB_RESET_EN
+ 0x00000000, // VGT_DRAW_PAYLOAD_CNTL
+ 0x00000000, // HOLE
+ 0x00000000, // VGT_INSTANCE_STEP_RATE_0
+ 0x00000000, // VGT_INSTANCE_STEP_RATE_1
+ 0x000000ff, // IA_MULTI_VGT_PARAM
+ 0x00000000, // VGT_ESGS_RING_ITEMSIZE
+ 0x00000000, // VGT_GSVS_RING_ITEMSIZE
+ 0x00000000, // VGT_REUSE_OFF
+ 0x00000000, // VGT_VTX_CNT_EN
+ 0x00000000, // DB_HTILE_SURFACE
+ 0x00000000, // DB_SRESULTS_COMPARE_STATE0
+ 0x00000000, // DB_SRESULTS_COMPARE_STATE1
+ 0x00000000, // DB_PRELOAD_CONTROL
+ 0, // HOLE
+ 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_0
+ 0x00000000, // VGT_STRMOUT_VTX_STRIDE_0
+ 0, // HOLE
+ 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_0
+ 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_1
+ 0x00000000, // VGT_STRMOUT_VTX_STRIDE_1
+ 0, // HOLE
+ 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_1
+ 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_2
+ 0x00000000, // VGT_STRMOUT_VTX_STRIDE_2
+ 0, // HOLE
+ 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_2
+ 0x00000000, // VGT_STRMOUT_BUFFER_SIZE_3
+ 0x00000000, // VGT_STRMOUT_VTX_STRIDE_3
+ 0, // HOLE
+ 0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_3
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_OFFSET
+ 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE
+ 0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE
+ 0, // HOLE
+ 0x00000000, // VGT_GS_MAX_VERT_OUT
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0, // HOLE
+ 0x00000000, // VGT_TESS_DISTRIBUTION
+ 0x00000000, // VGT_SHADER_STAGES_EN
+ 0x00000000, // VGT_LS_HS_CONFIG
+ 0x00000000, // VGT_GS_VERT_ITEMSIZE
+ 0x00000000, // VGT_GS_VERT_ITEMSIZE_1
+ 0x00000000, // VGT_GS_VERT_ITEMSIZE_2
+ 0x00000000, // VGT_GS_VERT_ITEMSIZE_3
+ 0x00000000, // VGT_TF_PARAM
+ 0x00000000, // DB_ALPHA_TO_MASK
+ 0x00000000, // VGT_DISPATCH_DRAW_INDEX
+ 0x00000000, // PA_SU_POLY_OFFSET_DB_FMT_CNTL
+ 0x00000000, // PA_SU_POLY_OFFSET_CLAMP
+ 0x00000000, // PA_SU_POLY_OFFSET_FRONT_SCALE
+ 0x00000000, // PA_SU_POLY_OFFSET_FRONT_OFFSET
+ 0x00000000, // PA_SU_POLY_OFFSET_BACK_SCALE
+ 0x00000000, // PA_SU_POLY_OFFSET_BACK_OFFSET
+ 0x00000000, // VGT_GS_INSTANCE_CNT
+ 0x00000000, // VGT_STRMOUT_CONFIG
+ 0x00000000, // VGT_STRMOUT_BUFFER_CONFIG
+};
+static const unsigned int gfx10_SECT_CONTEXT_def_8[] = {
+ 0x00000000, // PA_SC_CENTROID_PRIORITY_0
+ 0x00000000, // PA_SC_CENTROID_PRIORITY_1
+ 0x00001000, // PA_SC_LINE_CNTL
+ 0x00000000, // PA_SC_AA_CONFIG
+ 0x00000005, // PA_SU_VTX_CNTL
+ 0x3f800000, // PA_CL_GB_VERT_CLIP_ADJ
+ 0x3f800000, // PA_CL_GB_VERT_DISC_ADJ
+ 0x3f800000, // PA_CL_GB_HORZ_CLIP_ADJ
+ 0x3f800000, // PA_CL_GB_HORZ_DISC_ADJ
+ 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0
+ 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1
+ 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2
+ 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3
+ 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0
+ 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1
+ 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2
+ 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3
+ 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0
+ 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1
+ 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2
+ 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3
+ 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0
+ 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1
+ 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2
+ 0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3
+ 0xffffffff, // PA_SC_AA_MASK_X0Y0_X1Y0
+ 0xffffffff, // PA_SC_AA_MASK_X0Y1_X1Y1
+ 0x00000000, // PA_SC_SHADER_CONTROL
+ 0x00000003, // PA_SC_BINNER_CNTL_0
+ 0x00000000, // PA_SC_BINNER_CNTL_1
+ 0x00100000, // PA_SC_CONSERVATIVE_RASTERIZATION_CNTL
+ 0x00000000, // PA_SC_NGG_MODE_CNTL
+ 0, // HOLE
+ 0x0000001e, // VGT_VERTEX_REUSE_BLOCK_CNTL
+ 0x00000020, // VGT_OUT_DEALLOC_CNTL
+ 0x00000000, // CB_COLOR0_BASE
+ 0x00000000, // CB_COLOR0_PITCH
+ 0x00000000, // CB_COLOR0_SLICE
+ 0x00000000, // CB_COLOR0_VIEW
+ 0x00000000, // CB_COLOR0_INFO
+ 0x00000000, // CB_COLOR0_ATTRIB
+ 0x00000000, // CB_COLOR0_DCC_CONTROL
+ 0x00000000, // CB_COLOR0_CMASK
+ 0x00000000, // CB_COLOR0_CMASK_SLICE
+ 0x00000000, // CB_COLOR0_FMASK
+ 0x00000000, // CB_COLOR0_FMASK_SLICE
+ 0x00000000, // CB_COLOR0_CLEAR_WORD0
+ 0x00000000, // CB_COLOR0_CLEAR_WORD1
+ 0x00000000, // CB_COLOR0_DCC_BASE
+ 0, // HOLE
+ 0x00000000, // CB_COLOR1_BASE
+ 0x00000000, // CB_COLOR1_PITCH
+ 0x00000000, // CB_COLOR1_SLICE
+ 0x00000000, // CB_COLOR1_VIEW
+ 0x00000000, // CB_COLOR1_INFO
+ 0x00000000, // CB_COLOR1_ATTRIB
+ 0x00000000, // CB_COLOR1_DCC_CONTROL
+ 0x00000000, // CB_COLOR1_CMASK
+ 0x00000000, // CB_COLOR1_CMASK_SLICE
+ 0x00000000, // CB_COLOR1_FMASK
+ 0x00000000, // CB_COLOR1_FMASK_SLICE
+ 0x00000000, // CB_COLOR1_CLEAR_WORD0
+ 0x00000000, // CB_COLOR1_CLEAR_WORD1
+ 0x00000000, // CB_COLOR1_DCC_BASE
+ 0, // HOLE
+ 0x00000000, // CB_COLOR2_BASE
+ 0x00000000, // CB_COLOR2_PITCH
+ 0x00000000, // CB_COLOR2_SLICE
+ 0x00000000, // CB_COLOR2_VIEW
+ 0x00000000, // CB_COLOR2_INFO
+ 0x00000000, // CB_COLOR2_ATTRIB
+ 0x00000000, // CB_COLOR2_DCC_CONTROL
+ 0x00000000, // CB_COLOR2_CMASK
+ 0x00000000, // CB_COLOR2_CMASK_SLICE
+ 0x00000000, // CB_COLOR2_FMASK
+ 0x00000000, // CB_COLOR2_FMASK_SLICE
+ 0x00000000, // CB_COLOR2_CLEAR_WORD0
+ 0x00000000, // CB_COLOR2_CLEAR_WORD1
+ 0x00000000, // CB_COLOR2_DCC_BASE
+ 0, // HOLE
+ 0x00000000, // CB_COLOR3_BASE
+ 0x00000000, // CB_COLOR3_PITCH
+ 0x00000000, // CB_COLOR3_SLICE
+ 0x00000000, // CB_COLOR3_VIEW
+ 0x00000000, // CB_COLOR3_INFO
+ 0x00000000, // CB_COLOR3_ATTRIB
+ 0x00000000, // CB_COLOR3_DCC_CONTROL
+ 0x00000000, // CB_COLOR3_CMASK
+ 0x00000000, // CB_COLOR3_CMASK_SLICE
+ 0x00000000, // CB_COLOR3_FMASK
+ 0x00000000, // CB_COLOR3_FMASK_SLICE
+ 0x00000000, // CB_COLOR3_CLEAR_WORD0
+ 0x00000000, // CB_COLOR3_CLEAR_WORD1
+ 0x00000000, // CB_COLOR3_DCC_BASE
+ 0, // HOLE
+ 0x00000000, // CB_COLOR4_BASE
+ 0x00000000, // CB_COLOR4_PITCH
+ 0x00000000, // CB_COLOR4_SLICE
+ 0x00000000, // CB_COLOR4_VIEW
+ 0x00000000, // CB_COLOR4_INFO
+ 0x00000000, // CB_COLOR4_ATTRIB
+ 0x00000000, // CB_COLOR4_DCC_CONTROL
+ 0x00000000, // CB_COLOR4_CMASK
+ 0x00000000, // CB_COLOR4_CMASK_SLICE
+ 0x00000000, // CB_COLOR4_FMASK
+ 0x00000000, // CB_COLOR4_FMASK_SLICE
+ 0x00000000, // CB_COLOR4_CLEAR_WORD0
+ 0x00000000, // CB_COLOR4_CLEAR_WORD1
+ 0x00000000, // CB_COLOR4_DCC_BASE
+ 0, // HOLE
+ 0x00000000, // CB_COLOR5_BASE
+ 0x00000000, // CB_COLOR5_PITCH
+ 0x00000000, // CB_COLOR5_SLICE
+ 0x00000000, // CB_COLOR5_VIEW
+ 0x00000000, // CB_COLOR5_INFO
+ 0x00000000, // CB_COLOR5_ATTRIB
+ 0x00000000, // CB_COLOR5_DCC_CONTROL
+ 0x00000000, // CB_COLOR5_CMASK
+ 0x00000000, // CB_COLOR5_CMASK_SLICE
+ 0x00000000, // CB_COLOR5_FMASK
+ 0x00000000, // CB_COLOR5_FMASK_SLICE
+ 0x00000000, // CB_COLOR5_CLEAR_WORD0
+ 0x00000000, // CB_COLOR5_CLEAR_WORD1
+ 0x00000000, // CB_COLOR5_DCC_BASE
+ 0, // HOLE
+ 0x00000000, // CB_COLOR6_BASE
+ 0x00000000, // CB_COLOR6_PITCH
+ 0x00000000, // CB_COLOR6_SLICE
+ 0x00000000, // CB_COLOR6_VIEW
+ 0x00000000, // CB_COLOR6_INFO
+ 0x00000000, // CB_COLOR6_ATTRIB
+ 0x00000000, // CB_COLOR6_DCC_CONTROL
+ 0x00000000, // CB_COLOR6_CMASK
+ 0x00000000, // CB_COLOR6_CMASK_SLICE
+ 0x00000000, // CB_COLOR6_FMASK
+ 0x00000000, // CB_COLOR6_FMASK_SLICE
+ 0x00000000, // CB_COLOR6_CLEAR_WORD0
+ 0x00000000, // CB_COLOR6_CLEAR_WORD1
+ 0x00000000, // CB_COLOR6_DCC_BASE
+ 0, // HOLE
+ 0x00000000, // CB_COLOR7_BASE
+ 0x00000000, // CB_COLOR7_PITCH
+ 0x00000000, // CB_COLOR7_SLICE
+ 0x00000000, // CB_COLOR7_VIEW
+ 0x00000000, // CB_COLOR7_INFO
+ 0x00000000, // CB_COLOR7_ATTRIB
+ 0x00000000, // CB_COLOR7_DCC_CONTROL
+ 0x00000000, // CB_COLOR7_CMASK
+ 0x00000000, // CB_COLOR7_CMASK_SLICE
+ 0x00000000, // CB_COLOR7_FMASK
+ 0x00000000, // CB_COLOR7_FMASK_SLICE
+ 0x00000000, // CB_COLOR7_CLEAR_WORD0
+ 0x00000000, // CB_COLOR7_CLEAR_WORD1
+ 0x00000000, // CB_COLOR7_DCC_BASE
+ 0, // HOLE
+ 0x00000000, // CB_COLOR0_BASE_EXT
+ 0x00000000, // CB_COLOR1_BASE_EXT
+ 0x00000000, // CB_COLOR2_BASE_EXT
+ 0x00000000, // CB_COLOR3_BASE_EXT
+ 0x00000000, // CB_COLOR4_BASE_EXT
+ 0x00000000, // CB_COLOR5_BASE_EXT
+ 0x00000000, // CB_COLOR6_BASE_EXT
+ 0x00000000, // CB_COLOR7_BASE_EXT
+ 0x00000000, // CB_COLOR0_CMASK_BASE_EXT
+ 0x00000000, // CB_COLOR1_CMASK_BASE_EXT
+ 0x00000000, // CB_COLOR2_CMASK_BASE_EXT
+ 0x00000000, // CB_COLOR3_CMASK_BASE_EXT
+ 0x00000000, // CB_COLOR4_CMASK_BASE_EXT
+ 0x00000000, // CB_COLOR5_CMASK_BASE_EXT
+ 0x00000000, // CB_COLOR6_CMASK_BASE_EXT
+ 0x00000000, // CB_COLOR7_CMASK_BASE_EXT
+ 0x00000000, // CB_COLOR0_FMASK_BASE_EXT
+ 0x00000000, // CB_COLOR1_FMASK_BASE_EXT
+ 0x00000000, // CB_COLOR2_FMASK_BASE_EXT
+ 0x00000000, // CB_COLOR3_FMASK_BASE_EXT
+ 0x00000000, // CB_COLOR4_FMASK_BASE_EXT
+ 0x00000000, // CB_COLOR5_FMASK_BASE_EXT
+ 0x00000000, // CB_COLOR6_FMASK_BASE_EXT
+ 0x00000000, // CB_COLOR7_FMASK_BASE_EXT
+ 0x00000000, // CB_COLOR0_DCC_BASE_EXT
+ 0x00000000, // CB_COLOR1_DCC_BASE_EXT
+ 0x00000000, // CB_COLOR2_DCC_BASE_EXT
+ 0x00000000, // CB_COLOR3_DCC_BASE_EXT
+ 0x00000000, // CB_COLOR4_DCC_BASE_EXT
+ 0x00000000, // CB_COLOR5_DCC_BASE_EXT
+ 0x00000000, // CB_COLOR6_DCC_BASE_EXT
+ 0x00000000, // CB_COLOR7_DCC_BASE_EXT
+ 0x00000000, // CB_COLOR0_ATTRIB2
+ 0x00000000, // CB_COLOR1_ATTRIB2
+ 0x00000000, // CB_COLOR2_ATTRIB2
+ 0x00000000, // CB_COLOR3_ATTRIB2
+ 0x00000000, // CB_COLOR4_ATTRIB2
+ 0x00000000, // CB_COLOR5_ATTRIB2
+ 0x00000000, // CB_COLOR6_ATTRIB2
+ 0x00000000, // CB_COLOR7_ATTRIB2
+ 0x00000000, // CB_COLOR0_ATTRIB3
+ 0x00000000, // CB_COLOR1_ATTRIB3
+ 0x00000000, // CB_COLOR2_ATTRIB3
+ 0x00000000, // CB_COLOR3_ATTRIB3
+ 0x00000000, // CB_COLOR4_ATTRIB3
+ 0x00000000, // CB_COLOR5_ATTRIB3
+ 0x00000000, // CB_COLOR6_ATTRIB3
+ 0x00000000, // CB_COLOR7_ATTRIB3
+};
+static const struct cs_extent_def gfx10_SECT_CONTEXT_defs[] = {
+ {gfx10_SECT_CONTEXT_def_1, 0x0000a000, 215 },
+ {gfx10_SECT_CONTEXT_def_2, 0x0000a0d8, 272 },
+ {gfx10_SECT_CONTEXT_def_3, 0x0000a1f5, 4 },
+ {gfx10_SECT_CONTEXT_def_4, 0x0000a1ff, 158 },
+ {gfx10_SECT_CONTEXT_def_5, 0x0000a2a0, 2 },
+ {gfx10_SECT_CONTEXT_def_6, 0x0000a2a3, 1 },
+ {gfx10_SECT_CONTEXT_def_7, 0x0000a2a5, 66 },
+ {gfx10_SECT_CONTEXT_def_8, 0x0000a2f5, 203 },
+ { 0, 0, 0 }
+};
+static const struct cs_section_def gfx10_cs_data[] = {
+ { gfx10_SECT_CONTEXT_defs, SECT_CONTEXT },
+ { 0, SECT_NONE }
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ih.c b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
index 61024b9c7a4b..1dca0cabc326 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
@@ -20,7 +20,9 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
-#include <drm/drmP.h>
+
+#include <linux/pci.h>
+
#include "amdgpu.h"
#include "amdgpu_ih.h"
#include "vid.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index 1f0426d2fc2a..1ffd1963e765 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -20,7 +20,10 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
-#include <drm/drmP.h>
+
+#include <drm/drm_fourcc.h>
+#include <drm/drm_vblank.h>
+
#include "amdgpu.h"
#include "amdgpu_pm.h"
#include "amdgpu_i2c.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
index 2280b971d758..9e0782b54066 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
@@ -20,7 +20,10 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
-#include <drm/drmP.h>
+
+#include <drm/drm_fourcc.h>
+#include <drm/drm_vblank.h>
+
#include "amdgpu.h"
#include "amdgpu_pm.h"
#include "amdgpu_i2c.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
index bea32f076b91..4bf453e07dca 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
@@ -20,7 +20,12 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
-#include <drm/drmP.h>
+
+#include <linux/pci.h>
+
+#include <drm/drm_fourcc.h>
+#include <drm/drm_vblank.h>
+
#include "amdgpu.h"
#include "amdgpu_pm.h"
#include "amdgpu_i2c.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index 13da915991dd..b23418ca8f6a 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -20,7 +20,10 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
-#include <drm/drmP.h>
+
+#include <drm/drm_fourcc.h>
+#include <drm/drm_vblank.h>
+
#include "amdgpu.h"
#include "amdgpu_pm.h"
#include "amdgpu_i2c.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
index e4cc1d48eaab..3cc0a16649f9 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
@@ -20,7 +20,9 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
-#include <drm/drmP.h>
+
+#include <drm/drm_vblank.h>
+
#include "amdgpu.h"
#include "amdgpu_pm.h"
#include "amdgpu_i2c.h"
@@ -455,6 +457,7 @@ static int dce_virtual_hw_init(void *handle)
case CHIP_VEGA10:
case CHIP_VEGA12:
case CHIP_VEGA20:
+ case CHIP_NAVI10:
break;
default:
DRM_ERROR("Virtual display unsupported ASIC type: 0x%X\n", adev->asic_type);
diff --git a/drivers/gpu/drm/amd/amdgpu/df_v1_7.c b/drivers/gpu/drm/amd/amdgpu/df_v1_7.c
index 9935371db7ce..844c03868248 100644
--- a/drivers/gpu/drm/amd/amdgpu/df_v1_7.c
+++ b/drivers/gpu/drm/amd/amdgpu/df_v1_7.c
@@ -29,7 +29,7 @@
static u32 df_v1_7_channel_number[] = {1, 2, 0, 4, 0, 8, 0, 16, 2};
-static void df_v1_7_init (struct amdgpu_device *adev)
+static void df_v1_7_sw_init(struct amdgpu_device *adev)
{
}
@@ -110,7 +110,7 @@ static void df_v1_7_enable_ecc_force_par_wr_rmw(struct amdgpu_device *adev,
}
const struct amdgpu_df_funcs df_v1_7_funcs = {
- .init = df_v1_7_init,
+ .sw_init = df_v1_7_sw_init,
.enable_broadcast_mode = df_v1_7_enable_broadcast_mode,
.get_fb_channel_number = df_v1_7_get_fb_channel_number,
.get_hbm_channel_number = df_v1_7_get_hbm_channel_number,
diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
index d5ebe566809b..ef6e91f9f51c 100644
--- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
+++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.c
@@ -30,8 +30,104 @@
static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0,
16, 32, 0, 0, 0, 2, 4, 8};
-static void df_v3_6_init(struct amdgpu_device *adev)
+/* init df format attrs */
+AMDGPU_PMU_ATTR(event, "config:0-7");
+AMDGPU_PMU_ATTR(instance, "config:8-15");
+AMDGPU_PMU_ATTR(umask, "config:16-23");
+
+/* df format attributes */
+static struct attribute *df_v3_6_format_attrs[] = {
+ &pmu_attr_event.attr,
+ &pmu_attr_instance.attr,
+ &pmu_attr_umask.attr,
+ NULL
+};
+
+/* df format attribute group */
+static struct attribute_group df_v3_6_format_attr_group = {
+ .name = "format",
+ .attrs = df_v3_6_format_attrs,
+};
+
+/* df event attrs */
+AMDGPU_PMU_ATTR(cake0_pcsout_txdata,
+ "event=0x7,instance=0x46,umask=0x2");
+AMDGPU_PMU_ATTR(cake1_pcsout_txdata,
+ "event=0x7,instance=0x47,umask=0x2");
+AMDGPU_PMU_ATTR(cake0_pcsout_txmeta,
+ "event=0x7,instance=0x46,umask=0x4");
+AMDGPU_PMU_ATTR(cake1_pcsout_txmeta,
+ "event=0x7,instance=0x47,umask=0x4");
+AMDGPU_PMU_ATTR(cake0_ftiinstat_reqalloc,
+ "event=0xb,instance=0x46,umask=0x4");
+AMDGPU_PMU_ATTR(cake1_ftiinstat_reqalloc,
+ "event=0xb,instance=0x47,umask=0x4");
+AMDGPU_PMU_ATTR(cake0_ftiinstat_rspalloc,
+ "event=0xb,instance=0x46,umask=0x8");
+AMDGPU_PMU_ATTR(cake1_ftiinstat_rspalloc,
+ "event=0xb,instance=0x47,umask=0x8");
+
+/* df event attributes */
+static struct attribute *df_v3_6_event_attrs[] = {
+ &pmu_attr_cake0_pcsout_txdata.attr,
+ &pmu_attr_cake1_pcsout_txdata.attr,
+ &pmu_attr_cake0_pcsout_txmeta.attr,
+ &pmu_attr_cake1_pcsout_txmeta.attr,
+ &pmu_attr_cake0_ftiinstat_reqalloc.attr,
+ &pmu_attr_cake1_ftiinstat_reqalloc.attr,
+ &pmu_attr_cake0_ftiinstat_rspalloc.attr,
+ &pmu_attr_cake1_ftiinstat_rspalloc.attr,
+ NULL
+};
+
+/* df event attribute group */
+static struct attribute_group df_v3_6_event_attr_group = {
+ .name = "events",
+ .attrs = df_v3_6_event_attrs
+};
+
+/* df event attr groups */
+const struct attribute_group *df_v3_6_attr_groups[] = {
+ &df_v3_6_format_attr_group,
+ &df_v3_6_event_attr_group,
+ NULL
+};
+
+/* get the number of df counters available */
+static ssize_t df_v3_6_get_df_cntr_avail(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
{
+ struct amdgpu_device *adev;
+ struct drm_device *ddev;
+ int i, count;
+
+ ddev = dev_get_drvdata(dev);
+ adev = ddev->dev_private;
+ count = 0;
+
+ for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
+ if (adev->df_perfmon_config_assign_mask[i] == 0)
+ count++;
+ }
+
+ return snprintf(buf, PAGE_SIZE, "%i\n", count);
+}
+
+/* device attr for available perfmon counters */
+static DEVICE_ATTR(df_cntr_avail, S_IRUGO, df_v3_6_get_df_cntr_avail, NULL);
+
+/* init perfmons */
+static void df_v3_6_sw_init(struct amdgpu_device *adev)
+{
+ int i, ret;
+
+ ret = device_create_file(adev->dev, &dev_attr_df_cntr_avail);
+ if (ret)
+ DRM_ERROR("failed to create file for available df counters\n");
+
+ for (i = 0; i < AMDGPU_MAX_DF_PERFMONS; i++)
+ adev->df_perfmon_config_assign_mask[i] = 0;
}
static void df_v3_6_enable_broadcast_mode(struct amdgpu_device *adev,
@@ -105,12 +201,303 @@ static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
*flags |= AMD_CG_SUPPORT_DF_MGCG;
}
+/* get assigned df perfmon ctr as int */
+static int df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
+ uint64_t config)
+{
+ int i;
+
+ for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
+ if ((config & 0x0FFFFFFUL) ==
+ adev->df_perfmon_config_assign_mask[i])
+ return i;
+ }
+
+ return -EINVAL;
+}
+
+/* get address based on counter assignment */
+static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev,
+ uint64_t config,
+ int is_ctrl,
+ uint32_t *lo_base_addr,
+ uint32_t *hi_base_addr)
+{
+ int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
+
+ if (target_cntr < 0)
+ return;
+
+ switch (target_cntr) {
+
+ case 0:
+ *lo_base_addr = is_ctrl ? smnPerfMonCtlLo0 : smnPerfMonCtrLo0;
+ *hi_base_addr = is_ctrl ? smnPerfMonCtlHi0 : smnPerfMonCtrHi0;
+ break;
+ case 1:
+ *lo_base_addr = is_ctrl ? smnPerfMonCtlLo1 : smnPerfMonCtrLo1;
+ *hi_base_addr = is_ctrl ? smnPerfMonCtlHi1 : smnPerfMonCtrHi1;
+ break;
+ case 2:
+ *lo_base_addr = is_ctrl ? smnPerfMonCtlLo2 : smnPerfMonCtrLo2;
+ *hi_base_addr = is_ctrl ? smnPerfMonCtlHi2 : smnPerfMonCtrHi2;
+ break;
+ case 3:
+ *lo_base_addr = is_ctrl ? smnPerfMonCtlLo3 : smnPerfMonCtrLo3;
+ *hi_base_addr = is_ctrl ? smnPerfMonCtlHi3 : smnPerfMonCtrHi3;
+ break;
+
+ }
+
+}
+
+/* get read counter address */
+static void df_v3_6_pmc_get_read_settings(struct amdgpu_device *adev,
+ uint64_t config,
+ uint32_t *lo_base_addr,
+ uint32_t *hi_base_addr)
+{
+ df_v3_6_pmc_get_addr(adev, config, 0, lo_base_addr, hi_base_addr);
+}
+
+/* get control counter settings i.e. address and values to set */
+static int df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
+ uint64_t config,
+ uint32_t *lo_base_addr,
+ uint32_t *hi_base_addr,
+ uint32_t *lo_val,
+ uint32_t *hi_val)
+{
+ df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, hi_base_addr);
+
+ if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) {
+ DRM_ERROR("[DF PMC] addressing not retrieved! Lo: %x, Hi: %x",
+ *lo_base_addr, *hi_base_addr);
+ return -ENXIO;
+ }
+
+ if (lo_val && hi_val) {
+ uint32_t eventsel, instance, unitmask;
+ uint32_t instance_10, instance_5432, instance_76;
+
+ eventsel = DF_V3_6_GET_EVENT(config) & 0x3f;
+ unitmask = DF_V3_6_GET_UNITMASK(config) & 0xf;
+ instance = DF_V3_6_GET_INSTANCE(config);
+
+ instance_10 = instance & 0x3;
+ instance_5432 = (instance >> 2) & 0xf;
+ instance_76 = (instance >> 6) & 0x3;
+
+ *lo_val = (unitmask << 8) | (instance_10 << 6) | eventsel;
+ *hi_val = (instance_76 << 29) | instance_5432;
+ }
+
+ return 0;
+}
+
+/* assign df performance counters for read */
+static int df_v3_6_pmc_assign_cntr(struct amdgpu_device *adev,
+ uint64_t config,
+ int *is_assigned)
+{
+ int i, target_cntr;
+
+ *is_assigned = 0;
+
+ target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
+
+ if (target_cntr >= 0) {
+ *is_assigned = 1;
+ return 0;
+ }
+
+ for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
+ if (adev->df_perfmon_config_assign_mask[i] == 0U) {
+ adev->df_perfmon_config_assign_mask[i] =
+ config & 0x0FFFFFFUL;
+ return 0;
+ }
+ }
+
+ return -ENOSPC;
+}
+
+/* release performance counter */
+static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev,
+ uint64_t config)
+{
+ int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
+
+ if (target_cntr >= 0)
+ adev->df_perfmon_config_assign_mask[target_cntr] = 0ULL;
+}
+
+
+static void df_v3_6_reset_perfmon_cntr(struct amdgpu_device *adev,
+ uint64_t config)
+{
+ uint32_t lo_base_addr, hi_base_addr;
+
+ df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
+ &hi_base_addr);
+
+ if ((lo_base_addr == 0) || (hi_base_addr == 0))
+ return;
+
+ WREG32_PCIE(lo_base_addr, 0UL);
+ WREG32_PCIE(hi_base_addr, 0UL);
+}
+
+
+static int df_v3_6_add_perfmon_cntr(struct amdgpu_device *adev,
+ uint64_t config)
+{
+ uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
+ int ret, is_assigned;
+
+ ret = df_v3_6_pmc_assign_cntr(adev, config, &is_assigned);
+
+ if (ret || is_assigned)
+ return ret;
+
+ ret = df_v3_6_pmc_get_ctrl_settings(adev,
+ config,
+ &lo_base_addr,
+ &hi_base_addr,
+ &lo_val,
+ &hi_val);
+
+ if (ret)
+ return ret;
+
+ DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
+ config, lo_base_addr, hi_base_addr, lo_val, hi_val);
+
+ WREG32_PCIE(lo_base_addr, lo_val);
+ WREG32_PCIE(hi_base_addr, hi_val);
+
+ return ret;
+}
+
+static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
+ int is_enable)
+{
+ uint32_t lo_base_addr, hi_base_addr, lo_val;
+ int ret = 0;
+
+ switch (adev->asic_type) {
+ case CHIP_VEGA20:
+
+ df_v3_6_reset_perfmon_cntr(adev, config);
+
+ if (is_enable) {
+ ret = df_v3_6_add_perfmon_cntr(adev, config);
+ } else {
+ ret = df_v3_6_pmc_get_ctrl_settings(adev,
+ config,
+ &lo_base_addr,
+ &hi_base_addr,
+ NULL,
+ NULL);
+
+ if (ret)
+ return ret;
+
+ lo_val = RREG32_PCIE(lo_base_addr);
+
+ DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
+ config, lo_base_addr, hi_base_addr, lo_val);
+
+ WREG32_PCIE(lo_base_addr, lo_val | (1ULL << 22));
+ }
+
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+}
+
+static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
+ int is_disable)
+{
+ uint32_t lo_base_addr, hi_base_addr, lo_val;
+ int ret = 0;
+
+ switch (adev->asic_type) {
+ case CHIP_VEGA20:
+ ret = df_v3_6_pmc_get_ctrl_settings(adev,
+ config,
+ &lo_base_addr,
+ &hi_base_addr,
+ NULL,
+ NULL);
+
+ if (ret)
+ return ret;
+
+ lo_val = RREG32_PCIE(lo_base_addr);
+
+ DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x",
+ config, lo_base_addr, hi_base_addr, lo_val);
+
+ WREG32_PCIE(lo_base_addr, lo_val & ~(1ULL << 22));
+
+ if (is_disable)
+ df_v3_6_pmc_release_cntr(adev, config);
+
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+}
+
+static void df_v3_6_pmc_get_count(struct amdgpu_device *adev,
+ uint64_t config,
+ uint64_t *count)
+{
+ uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
+ *count = 0;
+
+ switch (adev->asic_type) {
+ case CHIP_VEGA20:
+
+ df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
+ &hi_base_addr);
+
+ if ((lo_base_addr == 0) || (hi_base_addr == 0))
+ return;
+
+ lo_val = RREG32_PCIE(lo_base_addr);
+ hi_val = RREG32_PCIE(hi_base_addr);
+
+ *count = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
+
+ if (*count >= DF_V3_6_PERFMON_OVERFLOW)
+ *count = 0;
+
+ DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
+ config, lo_base_addr, hi_base_addr, lo_val, hi_val);
+
+ break;
+
+ default:
+ break;
+ }
+}
+
const struct amdgpu_df_funcs df_v3_6_funcs = {
- .init = df_v3_6_init,
+ .sw_init = df_v3_6_sw_init,
.enable_broadcast_mode = df_v3_6_enable_broadcast_mode,
.get_fb_channel_number = df_v3_6_get_fb_channel_number,
.get_hbm_channel_number = df_v3_6_get_hbm_channel_number,
.update_medium_grain_clock_gating =
df_v3_6_update_medium_grain_clock_gating,
.get_clockgating_state = df_v3_6_get_clockgating_state,
+ .pmc_start = df_v3_6_pmc_start,
+ .pmc_stop = df_v3_6_pmc_stop,
+ .pmc_get_count = df_v3_6_pmc_get_count
};
diff --git a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
index e79c58e5efcb..76998541bc30 100644
--- a/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
+++ b/drivers/gpu/drm/amd/amdgpu/df_v3_6.h
@@ -35,6 +35,16 @@ enum DF_V3_6_MGCG {
DF_V3_6_MGCG_ENABLE_63_CYCLE_DELAY = 15
};
+/* Defined in global_features.h as FTI_PERFMON_VISIBLE */
+#define DF_V3_6_MAX_COUNTERS 4
+
+/* get flags from df perfmon config */
+#define DF_V3_6_GET_EVENT(x) (x & 0xFFUL)
+#define DF_V3_6_GET_INSTANCE(x) ((x >> 8) & 0xFFUL)
+#define DF_V3_6_GET_UNITMASK(x) ((x >> 16) & 0xFFUL)
+#define DF_V3_6_PERFMON_OVERFLOW 0xFFFFFFFFFFFFULL
+
+extern const struct attribute_group *df_v3_6_attr_groups[];
extern const struct amdgpu_df_funcs df_v3_6_funcs;
#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
new file mode 100644
index 000000000000..f41287f9000d
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
@@ -0,0 +1,5216 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/firmware.h>
+#include <drm/drmP.h>
+#include "amdgpu.h"
+#include "amdgpu_gfx.h"
+#include "amdgpu_psp.h"
+#include "amdgpu_smu.h"
+#include "nv.h"
+#include "nvd.h"
+
+#include "gc/gc_10_1_0_offset.h"
+#include "gc/gc_10_1_0_sh_mask.h"
+#include "navi10_enum.h"
+#include "hdp/hdp_5_0_0_offset.h"
+#include "ivsrcid/gfx/irqsrcs_gfx_10_1.h"
+
+#include "soc15.h"
+#include "soc15_common.h"
+#include "clearstate_gfx10.h"
+#include "v10_structs.h"
+#include "gfx_v10_0.h"
+#include "nbio_v2_3.h"
+
+/**
+ * Navi10 has two graphic rings to share each graphic pipe.
+ * 1. Primary ring
+ * 2. Async ring
+ *
+ * In bring-up phase, it just used primary ring so set gfx ring number as 1 at
+ * first.
+ */
+#define GFX10_NUM_GFX_RINGS 2
+#define GFX10_MEC_HPD_SIZE 2048
+
+#define F32_CE_PROGRAM_RAM_SIZE 65536
+#define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
+
+MODULE_FIRMWARE("amdgpu/navi10_ce.bin");
+MODULE_FIRMWARE("amdgpu/navi10_pfp.bin");
+MODULE_FIRMWARE("amdgpu/navi10_me.bin");
+MODULE_FIRMWARE("amdgpu/navi10_mec.bin");
+MODULE_FIRMWARE("amdgpu/navi10_mec2.bin");
+MODULE_FIRMWARE("amdgpu/navi10_rlc.bin");
+
+static const struct soc15_reg_golden golden_settings_gc_10_1[] =
+{
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_4, 0xffffffff, 0x00400014),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_CPF_CLK_CTRL, 0xfcff8fff, 0xf8000100),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_SPI_CLK_CTRL, 0xc0000000, 0xc0000100),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_SQ_CLK_CTRL, 0x60000ff0, 0x60000100),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_SQG_CLK_CTRL, 0x40000000, 0x40000100),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_VGT_CLK_CTRL, 0xffff8fff, 0xffff8100),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_WD_CLK_CTRL, 0xfeff8fff, 0xfeff8100),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmCH_PIPE_STEER, 0xffffffff, 0xe4e4e4e4),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmCH_VC5_ENABLE, 0x00000002, 0x00000000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0x000007ff, 0x000005ff),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG, 0x20000000, 0x20000000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xffffffff, 0x00000420),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x00000200, 0x00000200),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG4, 0x07900000, 0x04900000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DFSM_TILES_IN_FLIGHT, 0x0000ffff, 0x0000003f),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_LAST_OF_BURST_CONFIG, 0xffffffff, 0x03860204),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCR_GENERAL_CNTL, 0x1ff0ffff, 0x00000500),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmGE_PRIV_CONTROL, 0x000007ff, 0x000001fe),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL1_PIPE_STEER, 0xffffffff, 0xe4e4e4e4),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2_PIPE_STEER_0, 0x77777777, 0x10321032),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2_PIPE_STEER_1, 0x77777777, 0x02310231),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2A_ADDR_MATCH_MASK, 0xffffffff, 0xffffffcf),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_ADDR_MATCH_MASK, 0xffffffff, 0xffffffcf),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_CGTT_SCLK_CTRL, 0x10000000, 0x10000100),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_CTRL2, 0xffffffff, 0x1402002f),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_CTRL3, 0xffff9fff, 0x00001188),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x08000009),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00400000, 0x04440000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_SPARE, 0xffffffff, 0xffff3101),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_ALU_CLK_CTRL, 0xffffffff, 0xffffffff),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_ARB_CONFIG, 0x00000100, 0x00000130),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_LDS_CLK_CTRL, 0xffffffff, 0xffffffff),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfff7ffff, 0x01030000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CNTL, 0x60000010, 0x479c0010),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmUTCL1_CGTT_CLK_CTRL, 0xfeff0fff, 0x40000100),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmUTCL1_CTRL, 0x00800000, 0x00800000)
+};
+
+static const struct soc15_reg_golden golden_settings_gc_10_0_nv10[] =
+{
+ /* Pending on emulation bring up */
+};
+
+#define DEFAULT_SH_MEM_CONFIG \
+ ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \
+ (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \
+ (SH_MEM_RETRY_MODE_ALL << SH_MEM_CONFIG__RETRY_MODE__SHIFT) | \
+ (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))
+
+
+static void gfx_v10_0_set_ring_funcs(struct amdgpu_device *adev);
+static void gfx_v10_0_set_irq_funcs(struct amdgpu_device *adev);
+static void gfx_v10_0_set_gds_init(struct amdgpu_device *adev);
+static void gfx_v10_0_set_rlc_funcs(struct amdgpu_device *adev);
+static int gfx_v10_0_get_cu_info(struct amdgpu_device *adev,
+ struct amdgpu_cu_info *cu_info);
+static uint64_t gfx_v10_0_get_gpu_clock_counter(struct amdgpu_device *adev);
+static void gfx_v10_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
+ u32 sh_num, u32 instance);
+static u32 gfx_v10_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev);
+
+static int gfx_v10_0_rlc_backdoor_autoload_buffer_init(struct amdgpu_device *adev);
+static void gfx_v10_0_rlc_backdoor_autoload_buffer_fini(struct amdgpu_device *adev);
+static int gfx_v10_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev);
+static int gfx_v10_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
+static void gfx_v10_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume);
+static void gfx_v10_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume);
+static void gfx_v10_0_ring_emit_tmz(struct amdgpu_ring *ring, bool start);
+
+static void gfx10_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask)
+{
+ amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
+ amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
+ PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */
+ amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
+ amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
+ amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
+ amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
+ amdgpu_ring_write(kiq_ring, 0); /* oac mask */
+ amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
+}
+
+static void gfx10_kiq_map_queues(struct amdgpu_ring *kiq_ring,
+ struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = kiq_ring->adev;
+ uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
+ uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
+ uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
+
+ amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
+ /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
+ amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
+ PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
+ PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
+ PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
+ PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
+ PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
+ PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
+ PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
+ PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
+ PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
+ amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
+ amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
+ amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
+ amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
+ amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
+}
+
+static void gfx10_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
+ struct amdgpu_ring *ring,
+ enum amdgpu_unmap_queues_action action,
+ u64 gpu_addr, u64 seq)
+{
+ uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
+
+ amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
+ amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
+ PACKET3_UNMAP_QUEUES_ACTION(action) |
+ PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
+ PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
+ PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
+ amdgpu_ring_write(kiq_ring,
+ PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
+
+ if (action == PREEMPT_QUEUES_NO_UNMAP) {
+ amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
+ amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
+ amdgpu_ring_write(kiq_ring, seq);
+ } else {
+ amdgpu_ring_write(kiq_ring, 0);
+ amdgpu_ring_write(kiq_ring, 0);
+ amdgpu_ring_write(kiq_ring, 0);
+ }
+}
+
+static void gfx10_kiq_query_status(struct amdgpu_ring *kiq_ring,
+ struct amdgpu_ring *ring,
+ u64 addr,
+ u64 seq)
+{
+ uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
+
+ amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
+ amdgpu_ring_write(kiq_ring,
+ PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
+ PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
+ PACKET3_QUERY_STATUS_COMMAND(2));
+ amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
+ PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
+ PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
+ amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
+ amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
+ amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
+ amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
+}
+
+static const struct kiq_pm4_funcs gfx_v10_0_kiq_pm4_funcs = {
+ .kiq_set_resources = gfx10_kiq_set_resources,
+ .kiq_map_queues = gfx10_kiq_map_queues,
+ .kiq_unmap_queues = gfx10_kiq_unmap_queues,
+ .kiq_query_status = gfx10_kiq_query_status,
+ .set_resources_size = 8,
+ .map_queues_size = 7,
+ .unmap_queues_size = 6,
+ .query_status_size = 7,
+};
+
+static void gfx_v10_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
+{
+ adev->gfx.kiq.pmf = &gfx_v10_0_kiq_pm4_funcs;
+}
+
+static void gfx_v10_0_init_golden_registers(struct amdgpu_device *adev)
+{
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ soc15_program_register_sequence(adev,
+ golden_settings_gc_10_1,
+ (const u32)ARRAY_SIZE(golden_settings_gc_10_1));
+ soc15_program_register_sequence(adev,
+ golden_settings_gc_10_0_nv10,
+ (const u32)ARRAY_SIZE(golden_settings_gc_10_0_nv10));
+ break;
+ default:
+ break;
+ }
+}
+
+static void gfx_v10_0_scratch_init(struct amdgpu_device *adev)
+{
+ adev->gfx.scratch.num_reg = 8;
+ adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
+ adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
+}
+
+static void gfx_v10_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
+ bool wc, uint32_t reg, uint32_t val)
+{
+ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+ amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
+ WRITE_DATA_DST_SEL(0) | (wc ? WR_CONFIRM : 0));
+ amdgpu_ring_write(ring, reg);
+ amdgpu_ring_write(ring, 0);
+ amdgpu_ring_write(ring, val);
+}
+
+static void gfx_v10_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
+ int mem_space, int opt, uint32_t addr0,
+ uint32_t addr1, uint32_t ref, uint32_t mask,
+ uint32_t inv)
+{
+ amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
+ amdgpu_ring_write(ring,
+ /* memory (1) or register (0) */
+ (WAIT_REG_MEM_MEM_SPACE(mem_space) |
+ WAIT_REG_MEM_OPERATION(opt) | /* wait */
+ WAIT_REG_MEM_FUNCTION(3) | /* equal */
+ WAIT_REG_MEM_ENGINE(eng_sel)));
+
+ if (mem_space)
+ BUG_ON(addr0 & 0x3); /* Dword align */
+ amdgpu_ring_write(ring, addr0);
+ amdgpu_ring_write(ring, addr1);
+ amdgpu_ring_write(ring, ref);
+ amdgpu_ring_write(ring, mask);
+ amdgpu_ring_write(ring, inv); /* poll interval */
+}
+
+static int gfx_v10_0_ring_test_ring(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+ uint32_t scratch;
+ uint32_t tmp = 0;
+ unsigned i;
+ int r;
+
+ r = amdgpu_gfx_scratch_get(adev, &scratch);
+ if (r) {
+ DRM_ERROR("amdgpu: cp failed to get scratch reg (%d).\n", r);
+ return r;
+ }
+
+ WREG32(scratch, 0xCAFEDEAD);
+
+ r = amdgpu_ring_alloc(ring, 3);
+ if (r) {
+ DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n",
+ ring->idx, r);
+ amdgpu_gfx_scratch_free(adev, scratch);
+ return r;
+ }
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
+ amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
+ amdgpu_ring_write(ring, 0xDEADBEEF);
+ amdgpu_ring_commit(ring);
+
+ for (i = 0; i < adev->usec_timeout; i++) {
+ tmp = RREG32(scratch);
+ if (tmp == 0xDEADBEEF)
+ break;
+ if (amdgpu_emu_mode == 1)
+ msleep(1);
+ else
+ DRM_UDELAY(1);
+ }
+ if (i < adev->usec_timeout) {
+ if (amdgpu_emu_mode == 1)
+ DRM_INFO("ring test on %d succeeded in %d msecs\n",
+ ring->idx, i);
+ else
+ DRM_INFO("ring test on %d succeeded in %d usecs\n",
+ ring->idx, i);
+ } else {
+ DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
+ ring->idx, scratch, tmp);
+ r = -EINVAL;
+ }
+ amdgpu_gfx_scratch_free(adev, scratch);
+
+ return r;
+}
+
+static int gfx_v10_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
+{
+ struct amdgpu_device *adev = ring->adev;
+ struct amdgpu_ib ib;
+ struct dma_fence *f = NULL;
+ uint32_t scratch;
+ uint32_t tmp = 0;
+ long r;
+
+ r = amdgpu_gfx_scratch_get(adev, &scratch);
+ if (r) {
+ DRM_ERROR("amdgpu: failed to get scratch reg (%ld).\n", r);
+ return r;
+ }
+
+ WREG32(scratch, 0xCAFEDEAD);
+
+ memset(&ib, 0, sizeof(ib));
+ r = amdgpu_ib_get(adev, NULL, 256, &ib);
+ if (r) {
+ DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
+ goto err1;
+ }
+
+ ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
+ ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START));
+ ib.ptr[2] = 0xDEADBEEF;
+ ib.length_dw = 3;
+
+ r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
+ if (r)
+ goto err2;
+
+ r = dma_fence_wait_timeout(f, false, timeout);
+ if (r == 0) {
+ DRM_ERROR("amdgpu: IB test timed out.\n");
+ r = -ETIMEDOUT;
+ goto err2;
+ } else if (r < 0) {
+ DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
+ goto err2;
+ }
+
+ tmp = RREG32(scratch);
+ if (tmp == 0xDEADBEEF) {
+ DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+ r = 0;
+ } else {
+ DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
+ scratch, tmp);
+ r = -EINVAL;
+ }
+err2:
+ amdgpu_ib_free(adev, &ib, NULL);
+ dma_fence_put(f);
+err1:
+ amdgpu_gfx_scratch_free(adev, scratch);
+
+ return r;
+}
+
+static void gfx_v10_0_free_microcode(struct amdgpu_device *adev)
+{
+ release_firmware(adev->gfx.pfp_fw);
+ adev->gfx.pfp_fw = NULL;
+ release_firmware(adev->gfx.me_fw);
+ adev->gfx.me_fw = NULL;
+ release_firmware(adev->gfx.ce_fw);
+ adev->gfx.ce_fw = NULL;
+ release_firmware(adev->gfx.rlc_fw);
+ adev->gfx.rlc_fw = NULL;
+ release_firmware(adev->gfx.mec_fw);
+ adev->gfx.mec_fw = NULL;
+ release_firmware(adev->gfx.mec2_fw);
+ adev->gfx.mec2_fw = NULL;
+
+ kfree(adev->gfx.rlc.register_list_format);
+}
+
+static void gfx_v10_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
+{
+ const struct rlc_firmware_header_v2_1 *rlc_hdr;
+
+ rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
+ adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
+ adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
+ adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
+ adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
+ adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
+ adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
+ adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
+ adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
+ adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
+ adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
+ adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
+ adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
+ adev->gfx.rlc.reg_list_format_direct_reg_list_length =
+ le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
+}
+
+static void gfx_v10_0_check_gfxoff_flag(struct amdgpu_device *adev)
+{
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
+ break;
+ default:
+ break;
+ }
+}
+
+static int gfx_v10_0_init_microcode(struct amdgpu_device *adev)
+{
+ const char *chip_name;
+ char fw_name[30];
+ int err;
+ struct amdgpu_firmware_info *info = NULL;
+ const struct common_firmware_header *header = NULL;
+ const struct gfx_firmware_header_v1_0 *cp_hdr;
+ const struct rlc_firmware_header_v2_0 *rlc_hdr;
+ unsigned int *tmp = NULL;
+ unsigned int i = 0;
+ uint16_t version_major;
+ uint16_t version_minor;
+
+ DRM_DEBUG("\n");
+
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ chip_name = "navi10";
+ break;
+ default:
+ BUG();
+ }
+
+ snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
+ err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
+ if (err)
+ goto out;
+ err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
+ if (err)
+ goto out;
+ cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
+ adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
+ adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
+
+ snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
+ err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
+ if (err)
+ goto out;
+ err = amdgpu_ucode_validate(adev->gfx.me_fw);
+ if (err)
+ goto out;
+ cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
+ adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
+ adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
+
+ snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
+ err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
+ if (err)
+ goto out;
+ err = amdgpu_ucode_validate(adev->gfx.ce_fw);
+ if (err)
+ goto out;
+ cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
+ adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
+ adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
+
+ snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
+ err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
+ if (err)
+ goto out;
+ err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
+ rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
+ version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
+ version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
+ if (version_major == 2 && version_minor == 1)
+ adev->gfx.rlc.is_rlc_v2_1 = true;
+
+ adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
+ adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
+ adev->gfx.rlc.save_and_restore_offset =
+ le32_to_cpu(rlc_hdr->save_and_restore_offset);
+ adev->gfx.rlc.clear_state_descriptor_offset =
+ le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
+ adev->gfx.rlc.avail_scratch_ram_locations =
+ le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
+ adev->gfx.rlc.reg_restore_list_size =
+ le32_to_cpu(rlc_hdr->reg_restore_list_size);
+ adev->gfx.rlc.reg_list_format_start =
+ le32_to_cpu(rlc_hdr->reg_list_format_start);
+ adev->gfx.rlc.reg_list_format_separate_start =
+ le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
+ adev->gfx.rlc.starting_offsets_start =
+ le32_to_cpu(rlc_hdr->starting_offsets_start);
+ adev->gfx.rlc.reg_list_format_size_bytes =
+ le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
+ adev->gfx.rlc.reg_list_size_bytes =
+ le32_to_cpu(rlc_hdr->reg_list_size_bytes);
+ adev->gfx.rlc.register_list_format =
+ kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
+ adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
+ if (!adev->gfx.rlc.register_list_format) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ tmp = (unsigned int *)((uintptr_t)rlc_hdr +
+ le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
+ for (i = 0 ; i < (rlc_hdr->reg_list_format_size_bytes >> 2); i++)
+ adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
+
+ adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
+
+ tmp = (unsigned int *)((uintptr_t)rlc_hdr +
+ le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
+ for (i = 0 ; i < (rlc_hdr->reg_list_size_bytes >> 2); i++)
+ adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
+
+ if (adev->gfx.rlc.is_rlc_v2_1)
+ gfx_v10_0_init_rlc_ext_microcode(adev);
+
+ snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
+ err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
+ if (err)
+ goto out;
+ err = amdgpu_ucode_validate(adev->gfx.mec_fw);
+ if (err)
+ goto out;
+ cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
+ adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
+ adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
+
+ snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
+ err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
+ if (!err) {
+ err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
+ if (err)
+ goto out;
+ cp_hdr = (const struct gfx_firmware_header_v1_0 *)
+ adev->gfx.mec2_fw->data;
+ adev->gfx.mec2_fw_version =
+ le32_to_cpu(cp_hdr->header.ucode_version);
+ adev->gfx.mec2_feature_version =
+ le32_to_cpu(cp_hdr->ucode_feature_version);
+ } else {
+ err = 0;
+ adev->gfx.mec2_fw = NULL;
+ }
+
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+ info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
+ info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
+ info->fw = adev->gfx.pfp_fw;
+ header = (const struct common_firmware_header *)info->fw->data;
+ adev->firmware.fw_size +=
+ ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
+
+ info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
+ info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
+ info->fw = adev->gfx.me_fw;
+ header = (const struct common_firmware_header *)info->fw->data;
+ adev->firmware.fw_size +=
+ ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
+
+ info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
+ info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
+ info->fw = adev->gfx.ce_fw;
+ header = (const struct common_firmware_header *)info->fw->data;
+ adev->firmware.fw_size +=
+ ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
+
+ info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
+ info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
+ info->fw = adev->gfx.rlc_fw;
+ header = (const struct common_firmware_header *)info->fw->data;
+ adev->firmware.fw_size +=
+ ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
+
+ if (adev->gfx.rlc.is_rlc_v2_1 &&
+ adev->gfx.rlc.save_restore_list_cntl_size_bytes &&
+ adev->gfx.rlc.save_restore_list_gpm_size_bytes &&
+ adev->gfx.rlc.save_restore_list_srm_size_bytes) {
+ info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
+ info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
+ info->fw = adev->gfx.rlc_fw;
+ adev->firmware.fw_size +=
+ ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);
+
+ info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
+ info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
+ info->fw = adev->gfx.rlc_fw;
+ adev->firmware.fw_size +=
+ ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
+
+ info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
+ info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
+ info->fw = adev->gfx.rlc_fw;
+ adev->firmware.fw_size +=
+ ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
+ }
+
+ info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
+ info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
+ info->fw = adev->gfx.mec_fw;
+ header = (const struct common_firmware_header *)info->fw->data;
+ cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
+ adev->firmware.fw_size +=
+ ALIGN(le32_to_cpu(header->ucode_size_bytes) -
+ le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
+
+ info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
+ info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
+ info->fw = adev->gfx.mec_fw;
+ adev->firmware.fw_size +=
+ ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
+
+ if (adev->gfx.mec2_fw) {
+ info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
+ info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
+ info->fw = adev->gfx.mec2_fw;
+ header = (const struct common_firmware_header *)info->fw->data;
+ cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
+ adev->firmware.fw_size +=
+ ALIGN(le32_to_cpu(header->ucode_size_bytes) -
+ le32_to_cpu(cp_hdr->jt_size) * 4,
+ PAGE_SIZE);
+ info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT];
+ info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT;
+ info->fw = adev->gfx.mec2_fw;
+ adev->firmware.fw_size +=
+ ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4,
+ PAGE_SIZE);
+ }
+ }
+
+out:
+ if (err) {
+ dev_err(adev->dev,
+ "gfx10: Failed to load firmware \"%s\"\n",
+ fw_name);
+ release_firmware(adev->gfx.pfp_fw);
+ adev->gfx.pfp_fw = NULL;
+ release_firmware(adev->gfx.me_fw);
+ adev->gfx.me_fw = NULL;
+ release_firmware(adev->gfx.ce_fw);
+ adev->gfx.ce_fw = NULL;
+ release_firmware(adev->gfx.rlc_fw);
+ adev->gfx.rlc_fw = NULL;
+ release_firmware(adev->gfx.mec_fw);
+ adev->gfx.mec_fw = NULL;
+ release_firmware(adev->gfx.mec2_fw);
+ adev->gfx.mec2_fw = NULL;
+ }
+
+ gfx_v10_0_check_gfxoff_flag(adev);
+
+ return err;
+}
+
+static u32 gfx_v10_0_get_csb_size(struct amdgpu_device *adev)
+{
+ u32 count = 0;
+ const struct cs_section_def *sect = NULL;
+ const struct cs_extent_def *ext = NULL;
+
+ /* begin clear state */
+ count += 2;
+ /* context control state */
+ count += 3;
+
+ for (sect = gfx10_cs_data; sect->section != NULL; ++sect) {
+ for (ext = sect->section; ext->extent != NULL; ++ext) {
+ if (sect->id == SECT_CONTEXT)
+ count += 2 + ext->reg_count;
+ else
+ return 0;
+ }
+ }
+
+ /* set PA_SC_TILE_STEERING_OVERRIDE */
+ count += 3;
+ /* end clear state */
+ count += 2;
+ /* clear state */
+ count += 2;
+
+ return count;
+}
+
+static void gfx_v10_0_get_csb_buffer(struct amdgpu_device *adev,
+ volatile u32 *buffer)
+{
+ u32 count = 0, i;
+ const struct cs_section_def *sect = NULL;
+ const struct cs_extent_def *ext = NULL;
+ int ctx_reg_offset;
+
+ if (adev->gfx.rlc.cs_data == NULL)
+ return;
+ if (buffer == NULL)
+ return;
+
+ buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+ buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
+
+ buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
+ buffer[count++] = cpu_to_le32(0x80000000);
+ buffer[count++] = cpu_to_le32(0x80000000);
+
+ for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
+ for (ext = sect->section; ext->extent != NULL; ++ext) {
+ if (sect->id == SECT_CONTEXT) {
+ buffer[count++] =
+ cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
+ buffer[count++] = cpu_to_le32(ext->reg_index -
+ PACKET3_SET_CONTEXT_REG_START);
+ for (i = 0; i < ext->reg_count; i++)
+ buffer[count++] = cpu_to_le32(ext->extent[i]);
+ } else {
+ return;
+ }
+ }
+ }
+
+ ctx_reg_offset =
+ SOC15_REG_OFFSET(GC, 0, mmPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
+ buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
+ buffer[count++] = cpu_to_le32(ctx_reg_offset);
+ buffer[count++] = cpu_to_le32(adev->gfx.config.pa_sc_tile_steering_override);
+
+ buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+ buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
+
+ buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
+ buffer[count++] = cpu_to_le32(0);
+}
+
+static void gfx_v10_0_rlc_fini(struct amdgpu_device *adev)
+{
+ /* clear state block */
+ amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
+ &adev->gfx.rlc.clear_state_gpu_addr,
+ (void **)&adev->gfx.rlc.cs_ptr);
+
+ /* jump table block */
+ amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
+ &adev->gfx.rlc.cp_table_gpu_addr,
+ (void **)&adev->gfx.rlc.cp_table_ptr);
+}
+
+static int gfx_v10_0_rlc_init(struct amdgpu_device *adev)
+{
+ const struct cs_section_def *cs_data;
+ int r;
+
+ adev->gfx.rlc.cs_data = gfx10_cs_data;
+
+ cs_data = adev->gfx.rlc.cs_data;
+
+ if (cs_data) {
+ /* init clear state block */
+ r = amdgpu_gfx_rlc_init_csb(adev);
+ if (r)
+ return r;
+ }
+
+ return 0;
+}
+
+static int gfx_v10_0_csb_vram_pin(struct amdgpu_device *adev)
+{
+ int r;
+
+ r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);
+ if (unlikely(r != 0))
+ return r;
+
+ r = amdgpu_bo_pin(adev->gfx.rlc.clear_state_obj,
+ AMDGPU_GEM_DOMAIN_VRAM);
+ if (!r)
+ adev->gfx.rlc.clear_state_gpu_addr =
+ amdgpu_bo_gpu_offset(adev->gfx.rlc.clear_state_obj);
+
+ amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
+
+ return r;
+}
+
+static void gfx_v10_0_csb_vram_unpin(struct amdgpu_device *adev)
+{
+ int r;
+
+ if (!adev->gfx.rlc.clear_state_obj)
+ return;
+
+ r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, true);
+ if (likely(r == 0)) {
+ amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
+ amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
+ }
+}
+
+static void gfx_v10_0_mec_fini(struct amdgpu_device *adev)
+{
+ amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
+ amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
+}
+
+static int gfx_v10_0_me_init(struct amdgpu_device *adev)
+{
+ int r;
+
+ bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
+
+ amdgpu_gfx_graphics_queue_acquire(adev);
+
+ r = gfx_v10_0_init_microcode(adev);
+ if (r)
+ DRM_ERROR("Failed to load gfx firmware!\n");
+
+ return r;
+}
+
+static int gfx_v10_0_mec_init(struct amdgpu_device *adev)
+{
+ int r;
+ u32 *hpd;
+ const __le32 *fw_data = NULL;
+ unsigned fw_size;
+ u32 *fw = NULL;
+ size_t mec_hpd_size;
+
+ const struct gfx_firmware_header_v1_0 *mec_hdr = NULL;
+
+ bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
+
+ /* take ownership of the relevant compute queues */
+ amdgpu_gfx_compute_queue_acquire(adev);
+ mec_hpd_size = adev->gfx.num_compute_rings * GFX10_MEC_HPD_SIZE;
+
+ r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
+ AMDGPU_GEM_DOMAIN_GTT,
+ &adev->gfx.mec.hpd_eop_obj,
+ &adev->gfx.mec.hpd_eop_gpu_addr,
+ (void **)&hpd);
+ if (r) {
+ dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
+ gfx_v10_0_mec_fini(adev);
+ return r;
+ }
+
+ memset(hpd, 0, adev->gfx.mec.hpd_eop_obj->tbo.mem.size);
+
+ amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
+ amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
+
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+ mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
+
+ fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
+ le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
+ fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
+
+ r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
+ PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
+ &adev->gfx.mec.mec_fw_obj,
+ &adev->gfx.mec.mec_fw_gpu_addr,
+ (void **)&fw);
+ if (r) {
+ dev_err(adev->dev, "(%d) failed to create mec fw bo\n", r);
+ gfx_v10_0_mec_fini(adev);
+ return r;
+ }
+
+ memcpy(fw, fw_data, fw_size);
+
+ amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
+ amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
+ }
+
+ return 0;
+}
+
+static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address)
+{
+ WREG32_SOC15(GC, 0, mmSQ_IND_INDEX,
+ (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
+ (address << SQ_IND_INDEX__INDEX__SHIFT));
+ return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
+}
+
+static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave,
+ uint32_t thread, uint32_t regno,
+ uint32_t num, uint32_t *out)
+{
+ WREG32_SOC15(GC, 0, mmSQ_IND_INDEX,
+ (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
+ (regno << SQ_IND_INDEX__INDEX__SHIFT) |
+ (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) |
+ (SQ_IND_INDEX__AUTO_INCR_MASK));
+ while (num--)
+ *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
+}
+
+static void gfx_v10_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
+{
+ /* in gfx10 the SIMD_ID is specified as part of the INSTANCE
+ * field when performing a select_se_sh so it should be
+ * zero here */
+ WARN_ON(simd != 0);
+
+ /* type 2 wave data */
+ dst[(*no_fields)++] = 2;
+ dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS);
+ dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO);
+ dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI);
+ dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO);
+ dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI);
+ dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1);
+ dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2);
+ dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_INST_DW0);
+ dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC);
+ dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC);
+ dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAPSTS);
+ dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS);
+ dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2);
+ dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1);
+ dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0);
+}
+
+static void gfx_v10_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
+ uint32_t wave, uint32_t start,
+ uint32_t size, uint32_t *dst)
+{
+ WARN_ON(simd != 0);
+
+ wave_read_regs(
+ adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size,
+ dst);
+}
+
+static void gfx_v10_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
+ uint32_t wave, uint32_t thread,
+ uint32_t start, uint32_t size,
+ uint32_t *dst)
+{
+ wave_read_regs(
+ adev, wave, thread,
+ start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
+}
+
+static void gfx_v10_0_select_me_pipe_q(struct amdgpu_device *adev,
+ u32 me, u32 pipe, u32 q, u32 vm)
+ {
+ nv_grbm_select(adev, me, pipe, q, vm);
+ }
+
+
+static const struct amdgpu_gfx_funcs gfx_v10_0_gfx_funcs = {
+ .get_gpu_clock_counter = &gfx_v10_0_get_gpu_clock_counter,
+ .select_se_sh = &gfx_v10_0_select_se_sh,
+ .read_wave_data = &gfx_v10_0_read_wave_data,
+ .read_wave_sgprs = &gfx_v10_0_read_wave_sgprs,
+ .read_wave_vgprs = &gfx_v10_0_read_wave_vgprs,
+ .select_me_pipe_q = &gfx_v10_0_select_me_pipe_q,
+};
+
+static void gfx_v10_0_gpu_early_init(struct amdgpu_device *adev)
+{
+ u32 gb_addr_config;
+
+ adev->gfx.funcs = &gfx_v10_0_gfx_funcs;
+
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ adev->gfx.config.max_hw_contexts = 8;
+ adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
+ adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
+ adev->gfx.config.sc_hiz_tile_fifo_size = 0;
+ adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
+ gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
+ break;
+ default:
+ BUG();
+ break;
+ }
+
+ adev->gfx.config.gb_addr_config = gb_addr_config;
+
+ adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
+ REG_GET_FIELD(adev->gfx.config.gb_addr_config,
+ GB_ADDR_CONFIG, NUM_PIPES);
+
+ adev->gfx.config.max_tile_pipes =
+ adev->gfx.config.gb_addr_config_fields.num_pipes;
+
+ adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
+ REG_GET_FIELD(adev->gfx.config.gb_addr_config,
+ GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS);
+ adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
+ REG_GET_FIELD(adev->gfx.config.gb_addr_config,
+ GB_ADDR_CONFIG, NUM_RB_PER_SE);
+ adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
+ REG_GET_FIELD(adev->gfx.config.gb_addr_config,
+ GB_ADDR_CONFIG, NUM_SHADER_ENGINES);
+ adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
+ REG_GET_FIELD(adev->gfx.config.gb_addr_config,
+ GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE));
+}
+
+static int gfx_v10_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
+ int me, int pipe, int queue)
+{
+ int r;
+ struct amdgpu_ring *ring;
+ unsigned int irq_type;
+
+ ring = &adev->gfx.gfx_ring[ring_id];
+
+ ring->me = me;
+ ring->pipe = pipe;
+ ring->queue = queue;
+
+ ring->ring_obj = NULL;
+ ring->use_doorbell = true;
+
+ if (!ring_id)
+ ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
+ else
+ ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1;
+ sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue);
+
+ irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe;
+ r = amdgpu_ring_init(adev, ring, 1024,
+ &adev->gfx.eop_irq, irq_type);
+ if (r)
+ return r;
+ return 0;
+}
+
+static int gfx_v10_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
+ int mec, int pipe, int queue)
+{
+ int r;
+ unsigned irq_type;
+ struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
+
+ ring = &adev->gfx.compute_ring[ring_id];
+
+ /* mec0 is me1 */
+ ring->me = mec + 1;
+ ring->pipe = pipe;
+ ring->queue = queue;
+
+ ring->ring_obj = NULL;
+ ring->use_doorbell = true;
+ ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
+ ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
+ + (ring_id * GFX10_MEC_HPD_SIZE);
+ sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
+
+ irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
+ + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
+ + ring->pipe;
+
+ /* type-2 packets are deprecated on MEC, use type-3 instead */
+ r = amdgpu_ring_init(adev, ring, 1024,
+ &adev->gfx.eop_irq, irq_type);
+ if (r)
+ return r;
+
+ return 0;
+}
+
+static int gfx_v10_0_sw_init(void *handle)
+{
+ int i, j, k, r, ring_id = 0;
+ struct amdgpu_kiq *kiq;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ adev->gfx.me.num_me = 1;
+ adev->gfx.me.num_pipe_per_me = 2;
+ adev->gfx.me.num_queue_per_pipe = 1;
+ adev->gfx.mec.num_mec = 2;
+ adev->gfx.mec.num_pipe_per_mec = 4;
+ adev->gfx.mec.num_queue_per_pipe = 8;
+ break;
+ default:
+ adev->gfx.me.num_me = 1;
+ adev->gfx.me.num_pipe_per_me = 1;
+ adev->gfx.me.num_queue_per_pipe = 1;
+ adev->gfx.mec.num_mec = 1;
+ adev->gfx.mec.num_pipe_per_mec = 4;
+ adev->gfx.mec.num_queue_per_pipe = 8;
+ break;
+ }
+
+ /* KIQ event */
+ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP,
+ GFX_10_1__SRCID__CP_IB2_INTERRUPT_PKT,
+ &adev->gfx.kiq.irq);
+ if (r)
+ return r;
+
+ /* EOP Event */
+ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP,
+ GFX_10_1__SRCID__CP_EOP_INTERRUPT,
+ &adev->gfx.eop_irq);
+ if (r)
+ return r;
+
+ /* Privileged reg */
+ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_10_1__SRCID__CP_PRIV_REG_FAULT,
+ &adev->gfx.priv_reg_irq);
+ if (r)
+ return r;
+
+ /* Privileged inst */
+ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_10_1__SRCID__CP_PRIV_INSTR_FAULT,
+ &adev->gfx.priv_inst_irq);
+ if (r)
+ return r;
+
+ adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
+
+ gfx_v10_0_scratch_init(adev);
+
+ r = gfx_v10_0_me_init(adev);
+ if (r)
+ return r;
+
+ r = gfx_v10_0_rlc_init(adev);
+ if (r) {
+ DRM_ERROR("Failed to init rlc BOs!\n");
+ return r;
+ }
+
+ r = gfx_v10_0_mec_init(adev);
+ if (r) {
+ DRM_ERROR("Failed to init MEC BOs!\n");
+ return r;
+ }
+
+ /* set up the gfx ring */
+ for (i = 0; i < adev->gfx.me.num_me; i++) {
+ for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
+ for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
+ if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
+ continue;
+
+ r = gfx_v10_0_gfx_ring_init(adev, ring_id,
+ i, k, j);
+ if (r)
+ return r;
+ ring_id++;
+ }
+ }
+ }
+
+ ring_id = 0;
+ /* set up the compute queues - allocate horizontally across pipes */
+ for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
+ for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
+ for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
+ if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k,
+ j))
+ continue;
+
+ r = gfx_v10_0_compute_ring_init(adev, ring_id,
+ i, k, j);
+ if (r)
+ return r;
+
+ ring_id++;
+ }
+ }
+ }
+
+ r = amdgpu_gfx_kiq_init(adev, GFX10_MEC_HPD_SIZE);
+ if (r) {
+ DRM_ERROR("Failed to init KIQ BOs!\n");
+ return r;
+ }
+
+ kiq = &adev->gfx.kiq;
+ r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
+ if (r)
+ return r;
+
+ r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v10_compute_mqd));
+ if (r)
+ return r;
+
+ /* allocate visible FB for rlc auto-loading fw */
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
+ r = gfx_v10_0_rlc_backdoor_autoload_buffer_init(adev);
+ if (r)
+ return r;
+ }
+
+ adev->gfx.ce_ram_size = F32_CE_PROGRAM_RAM_SIZE;
+
+ gfx_v10_0_gpu_early_init(adev);
+
+ return 0;
+}
+
+static void gfx_v10_0_pfp_fini(struct amdgpu_device *adev)
+{
+ amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj,
+ &adev->gfx.pfp.pfp_fw_gpu_addr,
+ (void **)&adev->gfx.pfp.pfp_fw_ptr);
+}
+
+static void gfx_v10_0_ce_fini(struct amdgpu_device *adev)
+{
+ amdgpu_bo_free_kernel(&adev->gfx.ce.ce_fw_obj,
+ &adev->gfx.ce.ce_fw_gpu_addr,
+ (void **)&adev->gfx.ce.ce_fw_ptr);
+}
+
+static void gfx_v10_0_me_fini(struct amdgpu_device *adev)
+{
+ amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
+ &adev->gfx.me.me_fw_gpu_addr,
+ (void **)&adev->gfx.me.me_fw_ptr);
+}
+
+static int gfx_v10_0_sw_fini(void *handle)
+{
+ int i;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ for (i = 0; i < adev->gfx.num_gfx_rings; i++)
+ amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
+ for (i = 0; i < adev->gfx.num_compute_rings; i++)
+ amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
+
+ amdgpu_gfx_mqd_sw_fini(adev);
+ amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
+ amdgpu_gfx_kiq_fini(adev);
+
+ gfx_v10_0_pfp_fini(adev);
+ gfx_v10_0_ce_fini(adev);
+ gfx_v10_0_me_fini(adev);
+ gfx_v10_0_rlc_fini(adev);
+ gfx_v10_0_mec_fini(adev);
+
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
+ gfx_v10_0_rlc_backdoor_autoload_buffer_fini(adev);
+
+ gfx_v10_0_free_microcode(adev);
+
+ return 0;
+}
+
+
+static void gfx_v10_0_tiling_mode_table_init(struct amdgpu_device *adev)
+{
+ /* TODO */
+}
+
+static void gfx_v10_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
+ u32 sh_num, u32 instance)
+{
+ u32 data;
+
+ if (instance == 0xffffffff)
+ data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
+ INSTANCE_BROADCAST_WRITES, 1);
+ else
+ data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX,
+ instance);
+
+ if (se_num == 0xffffffff)
+ data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES,
+ 1);
+ else
+ data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
+
+ if (sh_num == 0xffffffff)
+ data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES,
+ 1);
+ else
+ data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num);
+
+ WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, data);
+}
+
+static u32 gfx_v10_0_get_rb_active_bitmap(struct amdgpu_device *adev)
+{
+ u32 data, mask;
+
+ data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
+ data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
+
+ data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
+ data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
+
+ mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
+ adev->gfx.config.max_sh_per_se);
+
+ return (~data) & mask;
+}
+
+static void gfx_v10_0_setup_rb(struct amdgpu_device *adev)
+{
+ int i, j;
+ u32 data;
+ u32 active_rbs = 0;
+ u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
+ adev->gfx.config.max_sh_per_se;
+
+ mutex_lock(&adev->grbm_idx_mutex);
+ for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
+ for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
+ gfx_v10_0_select_se_sh(adev, i, j, 0xffffffff);
+ data = gfx_v10_0_get_rb_active_bitmap(adev);
+ active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
+ rb_bitmap_width_per_sh);
+ }
+ }
+ gfx_v10_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+ mutex_unlock(&adev->grbm_idx_mutex);
+
+ adev->gfx.config.backend_enable_mask = active_rbs;
+ adev->gfx.config.num_rbs = hweight32(active_rbs);
+}
+
+static u32 gfx_v10_0_init_pa_sc_tile_steering_override(struct amdgpu_device *adev)
+{
+ uint32_t num_sc;
+ uint32_t enabled_rb_per_sh;
+ uint32_t active_rb_bitmap;
+ uint32_t num_rb_per_sc;
+ uint32_t num_packer_per_sc;
+ uint32_t pa_sc_tile_steering_override;
+
+ /* init num_sc */
+ num_sc = adev->gfx.config.max_shader_engines * adev->gfx.config.max_sh_per_se *
+ adev->gfx.config.num_sc_per_sh;
+ /* init num_rb_per_sc */
+ active_rb_bitmap = gfx_v10_0_get_rb_active_bitmap(adev);
+ enabled_rb_per_sh = hweight32(active_rb_bitmap);
+ num_rb_per_sc = enabled_rb_per_sh / adev->gfx.config.num_sc_per_sh;
+ /* init num_packer_per_sc */
+ num_packer_per_sc = adev->gfx.config.num_packer_per_sc;
+
+ pa_sc_tile_steering_override = 0;
+ pa_sc_tile_steering_override |=
+ (order_base_2(num_sc) << PA_SC_TILE_STEERING_OVERRIDE__NUM_SC__SHIFT) &
+ PA_SC_TILE_STEERING_OVERRIDE__NUM_SC_MASK;
+ pa_sc_tile_steering_override |=
+ (order_base_2(num_rb_per_sc) << PA_SC_TILE_STEERING_OVERRIDE__NUM_RB_PER_SC__SHIFT) &
+ PA_SC_TILE_STEERING_OVERRIDE__NUM_RB_PER_SC_MASK;
+ pa_sc_tile_steering_override |=
+ (order_base_2(num_packer_per_sc) << PA_SC_TILE_STEERING_OVERRIDE__NUM_PACKER_PER_SC__SHIFT) &
+ PA_SC_TILE_STEERING_OVERRIDE__NUM_PACKER_PER_SC_MASK;
+
+ return pa_sc_tile_steering_override;
+}
+
+#define DEFAULT_SH_MEM_BASES (0x6000)
+#define FIRST_COMPUTE_VMID (8)
+#define LAST_COMPUTE_VMID (16)
+
+static void gfx_v10_0_init_compute_vmid(struct amdgpu_device *adev)
+{
+ int i;
+ uint32_t sh_mem_bases;
+
+ /*
+ * Configure apertures:
+ * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
+ * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
+ * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
+ */
+ sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
+
+ mutex_lock(&adev->srbm_mutex);
+ for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
+ nv_grbm_select(adev, 0, 0, 0, i);
+ /* CP and shaders */
+ WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
+ WREG32_SOC15(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
+ }
+ nv_grbm_select(adev, 0, 0, 0, 0);
+ mutex_unlock(&adev->srbm_mutex);
+
+ /* Initialize all compute VMIDs to have no GDS, GWS, or OA
+ acccess. These should be enabled by FW for target VMIDs. */
+ for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
+ WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
+ WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
+ WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
+ WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
+ }
+}
+
+static void gfx_v10_0_tcp_harvest(struct amdgpu_device *adev)
+{
+ int i, j, k;
+ int max_wgp_per_sh = adev->gfx.config.max_cu_per_sh >> 1;
+ u32 tmp, wgp_active_bitmap = 0;
+ u32 gcrd_targets_disable_tcp = 0;
+ u32 utcl_invreq_disable = 0;
+ /*
+ * GCRD_TARGETS_DISABLE field contains
+ * for Navi10: GL1C=[18:15], SQC=[14:10], TCP=[9:0]
+ */
+ u32 gcrd_targets_disable_mask = amdgpu_gfx_create_bitmask(
+ 2 * max_wgp_per_sh + /* TCP */
+ max_wgp_per_sh + /* SQC */
+ 4); /* GL1C */
+ /*
+ * UTCL1_UTCL0_INVREQ_DISABLE field contains
+ * for Navi10: SQG=[24], RMI=[23:20], SQC=[19:10], TCP=[9:0]
+ */
+ u32 utcl_invreq_disable_mask = amdgpu_gfx_create_bitmask(
+ 2 * max_wgp_per_sh + /* TCP */
+ 2 * max_wgp_per_sh + /* SQC */
+ 4 + /* RMI */
+ 1); /* SQG */
+
+ if (adev->asic_type == CHIP_NAVI10) {
+ mutex_lock(&adev->grbm_idx_mutex);
+ for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
+ for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
+ gfx_v10_0_select_se_sh(adev, i, j, 0xffffffff);
+ wgp_active_bitmap = gfx_v10_0_get_wgp_active_bitmap_per_sh(adev);
+ /*
+ * Set corresponding TCP bits for the inactive WGPs in
+ * GCRD_SA_TARGETS_DISABLE
+ */
+ gcrd_targets_disable_tcp = 0;
+ /* Set TCP & SQC bits in UTCL1_UTCL0_INVREQ_DISABLE */
+ utcl_invreq_disable = 0;
+
+ for (k = 0; k < max_wgp_per_sh; k++) {
+ if (!(wgp_active_bitmap & (1 << k))) {
+ gcrd_targets_disable_tcp |= 3 << (2 * k);
+ utcl_invreq_disable |= (3 << (2 * k)) |
+ (3 << (2 * (max_wgp_per_sh + k)));
+ }
+ }
+
+ tmp = RREG32_SOC15(GC, 0, mmUTCL1_UTCL0_INVREQ_DISABLE);
+ /* only override TCP & SQC bits */
+ tmp &= 0xffffffff << (4 * max_wgp_per_sh);
+ tmp |= (utcl_invreq_disable & utcl_invreq_disable_mask);
+ WREG32_SOC15(GC, 0, mmUTCL1_UTCL0_INVREQ_DISABLE, tmp);
+
+ tmp = RREG32_SOC15(GC, 0, mmGCRD_SA_TARGETS_DISABLE);
+ /* only override TCP bits */
+ tmp &= 0xffffffff << (2 * max_wgp_per_sh);
+ tmp |= (gcrd_targets_disable_tcp & gcrd_targets_disable_mask);
+ WREG32_SOC15(GC, 0, mmGCRD_SA_TARGETS_DISABLE, tmp);
+ }
+ }
+
+ gfx_v10_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+ mutex_unlock(&adev->grbm_idx_mutex);
+ }
+}
+
+static void gfx_v10_0_constants_init(struct amdgpu_device *adev)
+{
+ u32 tmp;
+ int i;
+
+ WREG32_FIELD15(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
+
+ gfx_v10_0_tiling_mode_table_init(adev);
+
+ gfx_v10_0_setup_rb(adev);
+ gfx_v10_0_get_cu_info(adev, &adev->gfx.cu_info);
+ adev->gfx.config.pa_sc_tile_steering_override =
+ gfx_v10_0_init_pa_sc_tile_steering_override(adev);
+
+ /* XXX SH_MEM regs */
+ /* where to put LDS, scratch, GPUVM in FSA64 space */
+ mutex_lock(&adev->srbm_mutex);
+ for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB].num_ids; i++) {
+ nv_grbm_select(adev, 0, 0, 0, i);
+ /* CP and shaders */
+ WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
+ if (i != 0) {
+ tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
+ (adev->gmc.private_aperture_start >> 48));
+ tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
+ (adev->gmc.shared_aperture_start >> 48));
+ WREG32_SOC15(GC, 0, mmSH_MEM_BASES, tmp);
+ }
+ }
+ nv_grbm_select(adev, 0, 0, 0, 0);
+
+ mutex_unlock(&adev->srbm_mutex);
+
+ gfx_v10_0_init_compute_vmid(adev);
+
+}
+
+static void gfx_v10_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
+ bool enable)
+{
+ u32 tmp = RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
+
+ tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE,
+ enable ? 1 : 0);
+ tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE,
+ enable ? 1 : 0);
+ tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE,
+ enable ? 1 : 0);
+ tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE,
+ enable ? 1 : 0);
+
+ WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
+}
+
+static void gfx_v10_0_init_csb(struct amdgpu_device *adev)
+{
+ /* csib */
+ WREG32_SOC15(GC, 0, mmRLC_CSIB_ADDR_HI,
+ adev->gfx.rlc.clear_state_gpu_addr >> 32);
+ WREG32_SOC15(GC, 0, mmRLC_CSIB_ADDR_LO,
+ adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
+ WREG32_SOC15(GC, 0, mmRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);
+}
+
+static void gfx_v10_0_init_pg(struct amdgpu_device *adev)
+{
+ gfx_v10_0_init_csb(adev);
+
+ amdgpu_gmc_flush_gpu_tlb(adev, 0, 0);
+
+ /* TODO: init power gating */
+ return;
+}
+
+void gfx_v10_0_rlc_stop(struct amdgpu_device *adev)
+{
+ u32 tmp = RREG32_SOC15(GC, 0, mmRLC_CNTL);
+
+ tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
+ WREG32_SOC15(GC, 0, mmRLC_CNTL, tmp);
+}
+
+static void gfx_v10_0_rlc_reset(struct amdgpu_device *adev)
+{
+ WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
+ udelay(50);
+ WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
+ udelay(50);
+}
+
+static void gfx_v10_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev,
+ bool enable)
+{
+ uint32_t rlc_pg_cntl;
+
+ rlc_pg_cntl = RREG32_SOC15(GC, 0, mmRLC_PG_CNTL);
+
+ if (!enable) {
+ /* RLC_PG_CNTL[23] = 0 (default)
+ * RLC will wait for handshake acks with SMU
+ * GFXOFF will be enabled
+ * RLC_PG_CNTL[23] = 1
+ * RLC will not issue any message to SMU
+ * hence no handshake between SMU & RLC
+ * GFXOFF will be disabled
+ */
+ rlc_pg_cntl |= 0x80000;
+ } else
+ rlc_pg_cntl &= ~0x80000;
+ WREG32_SOC15(GC, 0, mmRLC_PG_CNTL, rlc_pg_cntl);
+}
+
+static void gfx_v10_0_rlc_start(struct amdgpu_device *adev)
+{
+ /* TODO: enable rlc & smu handshake until smu
+ * and gfxoff feature works as expected */
+ if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK))
+ gfx_v10_0_rlc_smu_handshake_cntl(adev, false);
+
+ WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
+ udelay(50);
+}
+
+static void gfx_v10_0_rlc_enable_srm(struct amdgpu_device *adev)
+{
+ uint32_t tmp;
+
+ /* enable Save Restore Machine */
+ tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
+ tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
+ tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
+}
+
+static int gfx_v10_0_rlc_load_microcode(struct amdgpu_device *adev)
+{
+ const struct rlc_firmware_header_v2_0 *hdr;
+ const __le32 *fw_data;
+ unsigned i, fw_size;
+
+ if (!adev->gfx.rlc_fw)
+ return -EINVAL;
+
+ hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
+ amdgpu_ucode_print_rlc_hdr(&hdr->header);
+
+ fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
+ le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+ fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
+
+ WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
+ RLCG_UCODE_LOADING_START_ADDRESS);
+
+ for (i = 0; i < fw_size; i++)
+ WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA,
+ le32_to_cpup(fw_data++));
+
+ WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
+
+ return 0;
+}
+
+static int gfx_v10_0_rlc_resume(struct amdgpu_device *adev)
+{
+ int r;
+
+ if (amdgpu_sriov_vf(adev))
+ return 0;
+
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+ r = gfx_v10_0_wait_for_rlc_autoload_complete(adev);
+ if (r)
+ return r;
+ gfx_v10_0_init_pg(adev);
+
+ /* enable RLC SRM */
+ gfx_v10_0_rlc_enable_srm(adev);
+
+ } else {
+ adev->gfx.rlc.funcs->stop(adev);
+
+ /* disable CG */
+ WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
+
+ /* disable PG */
+ WREG32_SOC15(GC, 0, mmRLC_PG_CNTL, 0);
+
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+ /* legacy rlc firmware loading */
+ r = gfx_v10_0_rlc_load_microcode(adev);
+ if (r)
+ return r;
+ } else if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
+ /* rlc backdoor autoload firmware */
+ r = gfx_v10_0_rlc_backdoor_autoload_enable(adev);
+ if (r)
+ return r;
+ }
+
+ gfx_v10_0_init_pg(adev);
+ adev->gfx.rlc.funcs->start(adev);
+
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
+ r = gfx_v10_0_wait_for_rlc_autoload_complete(adev);
+ if (r)
+ return r;
+ }
+ }
+ return 0;
+}
+
+static struct {
+ FIRMWARE_ID id;
+ unsigned int offset;
+ unsigned int size;
+} rlc_autoload_info[FIRMWARE_ID_MAX];
+
+static int gfx_v10_0_parse_rlc_toc(struct amdgpu_device *adev)
+{
+ int ret;
+ RLC_TABLE_OF_CONTENT *rlc_toc;
+
+ ret = amdgpu_bo_create_reserved(adev, adev->psp.toc_bin_size, PAGE_SIZE,
+ AMDGPU_GEM_DOMAIN_GTT,
+ &adev->gfx.rlc.rlc_toc_bo,
+ &adev->gfx.rlc.rlc_toc_gpu_addr,
+ (void **)&adev->gfx.rlc.rlc_toc_buf);
+ if (ret) {
+ dev_err(adev->dev, "(%d) failed to create rlc toc bo\n", ret);
+ return ret;
+ }
+
+ /* Copy toc from psp sos fw to rlc toc buffer */
+ memcpy(adev->gfx.rlc.rlc_toc_buf, adev->psp.toc_start_addr, adev->psp.toc_bin_size);
+
+ rlc_toc = (RLC_TABLE_OF_CONTENT *)adev->gfx.rlc.rlc_toc_buf;
+ while (rlc_toc && (rlc_toc->id > FIRMWARE_ID_INVALID) &&
+ (rlc_toc->id < FIRMWARE_ID_MAX)) {
+ if ((rlc_toc->id >= FIRMWARE_ID_CP_CE) &&
+ (rlc_toc->id <= FIRMWARE_ID_CP_MES)) {
+ /* Offset needs 4KB alignment */
+ rlc_toc->offset = ALIGN(rlc_toc->offset * 4, PAGE_SIZE);
+ }
+
+ rlc_autoload_info[rlc_toc->id].id = rlc_toc->id;
+ rlc_autoload_info[rlc_toc->id].offset = rlc_toc->offset * 4;
+ rlc_autoload_info[rlc_toc->id].size = rlc_toc->size * 4;
+
+ rlc_toc++;
+ };
+
+ return 0;
+}
+
+static uint32_t gfx_v10_0_calc_toc_total_size(struct amdgpu_device *adev)
+{
+ uint32_t total_size = 0;
+ FIRMWARE_ID id;
+ int ret;
+
+ ret = gfx_v10_0_parse_rlc_toc(adev);
+ if (ret) {
+ dev_err(adev->dev, "failed to parse rlc toc\n");
+ return 0;
+ }
+
+ for (id = FIRMWARE_ID_RLC_G_UCODE; id < FIRMWARE_ID_MAX; id++)
+ total_size += rlc_autoload_info[id].size;
+
+ /* In case the offset in rlc toc ucode is aligned */
+ if (total_size < rlc_autoload_info[FIRMWARE_ID_MAX-1].offset)
+ total_size = rlc_autoload_info[FIRMWARE_ID_MAX-1].offset +
+ rlc_autoload_info[FIRMWARE_ID_MAX-1].size;
+
+ return total_size;
+}
+
+static int gfx_v10_0_rlc_backdoor_autoload_buffer_init(struct amdgpu_device *adev)
+{
+ int r;
+ uint32_t total_size;
+
+ total_size = gfx_v10_0_calc_toc_total_size(adev);
+
+ r = amdgpu_bo_create_reserved(adev, total_size, PAGE_SIZE,
+ AMDGPU_GEM_DOMAIN_GTT,
+ &adev->gfx.rlc.rlc_autoload_bo,
+ &adev->gfx.rlc.rlc_autoload_gpu_addr,
+ (void **)&adev->gfx.rlc.rlc_autoload_ptr);
+ if (r) {
+ dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r);
+ return r;
+ }
+
+ return 0;
+}
+
+static void gfx_v10_0_rlc_backdoor_autoload_buffer_fini(struct amdgpu_device *adev)
+{
+ amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_toc_bo,
+ &adev->gfx.rlc.rlc_toc_gpu_addr,
+ (void **)&adev->gfx.rlc.rlc_toc_buf);
+ amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo,
+ &adev->gfx.rlc.rlc_autoload_gpu_addr,
+ (void **)&adev->gfx.rlc.rlc_autoload_ptr);
+}
+
+static void gfx_v10_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,
+ FIRMWARE_ID id,
+ const void *fw_data,
+ uint32_t fw_size)
+{
+ uint32_t toc_offset;
+ uint32_t toc_fw_size;
+ char *ptr = adev->gfx.rlc.rlc_autoload_ptr;
+
+ if (id <= FIRMWARE_ID_INVALID || id >= FIRMWARE_ID_MAX)
+ return;
+
+ toc_offset = rlc_autoload_info[id].offset;
+ toc_fw_size = rlc_autoload_info[id].size;
+
+ if (fw_size == 0)
+ fw_size = toc_fw_size;
+
+ if (fw_size > toc_fw_size)
+ fw_size = toc_fw_size;
+
+ memcpy(ptr + toc_offset, fw_data, fw_size);
+
+ if (fw_size < toc_fw_size)
+ memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size);
+}
+
+static void gfx_v10_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev)
+{
+ void *data;
+ uint32_t size;
+
+ data = adev->gfx.rlc.rlc_toc_buf;
+ size = rlc_autoload_info[FIRMWARE_ID_RLC_TOC].size;
+
+ gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
+ FIRMWARE_ID_RLC_TOC,
+ data, size);
+}
+
+static void gfx_v10_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev)
+{
+ const __le32 *fw_data;
+ uint32_t fw_size;
+ const struct gfx_firmware_header_v1_0 *cp_hdr;
+ const struct rlc_firmware_header_v2_0 *rlc_hdr;
+
+ /* pfp ucode */
+ cp_hdr = (const struct gfx_firmware_header_v1_0 *)
+ adev->gfx.pfp_fw->data;
+ fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
+ le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
+ fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
+ gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
+ FIRMWARE_ID_CP_PFP,
+ fw_data, fw_size);
+
+ /* ce ucode */
+ cp_hdr = (const struct gfx_firmware_header_v1_0 *)
+ adev->gfx.ce_fw->data;
+ fw_data = (const __le32 *)(adev->gfx.ce_fw->data +
+ le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
+ fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
+ gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
+ FIRMWARE_ID_CP_CE,
+ fw_data, fw_size);
+
+ /* me ucode */
+ cp_hdr = (const struct gfx_firmware_header_v1_0 *)
+ adev->gfx.me_fw->data;
+ fw_data = (const __le32 *)(adev->gfx.me_fw->data +
+ le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
+ fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
+ gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
+ FIRMWARE_ID_CP_ME,
+ fw_data, fw_size);
+
+ /* rlc ucode */
+ rlc_hdr = (const struct rlc_firmware_header_v2_0 *)
+ adev->gfx.rlc_fw->data;
+ fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
+ le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes));
+ fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes);
+ gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
+ FIRMWARE_ID_RLC_G_UCODE,
+ fw_data, fw_size);
+
+ /* mec1 ucode */
+ cp_hdr = (const struct gfx_firmware_header_v1_0 *)
+ adev->gfx.mec_fw->data;
+ fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
+ le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
+ fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
+ cp_hdr->jt_size * 4;
+ gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
+ FIRMWARE_ID_CP_MEC,
+ fw_data, fw_size);
+ /* mec2 ucode is not necessary if mec2 ucode is same as mec1 */
+}
+
+/* Temporarily put sdma part here */
+static void gfx_v10_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev)
+{
+ const __le32 *fw_data;
+ uint32_t fw_size;
+ const struct sdma_firmware_header_v1_0 *sdma_hdr;
+ int i;
+
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ sdma_hdr = (const struct sdma_firmware_header_v1_0 *)
+ adev->sdma.instance[i].fw->data;
+ fw_data = (const __le32 *) (adev->sdma.instance[i].fw->data +
+ le32_to_cpu(sdma_hdr->header.ucode_array_offset_bytes));
+ fw_size = le32_to_cpu(sdma_hdr->header.ucode_size_bytes);
+
+ if (i == 0) {
+ gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
+ FIRMWARE_ID_SDMA0_UCODE, fw_data, fw_size);
+ gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
+ FIRMWARE_ID_SDMA0_JT,
+ (uint32_t *)fw_data +
+ sdma_hdr->jt_offset,
+ sdma_hdr->jt_size * 4);
+ } else if (i == 1) {
+ gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
+ FIRMWARE_ID_SDMA1_UCODE, fw_data, fw_size);
+ gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
+ FIRMWARE_ID_SDMA1_JT,
+ (uint32_t *)fw_data +
+ sdma_hdr->jt_offset,
+ sdma_hdr->jt_size * 4);
+ }
+ }
+}
+
+static int gfx_v10_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
+{
+ uint32_t rlc_g_offset, rlc_g_size, tmp;
+ uint64_t gpu_addr;
+
+ gfx_v10_0_rlc_backdoor_autoload_copy_toc_ucode(adev);
+ gfx_v10_0_rlc_backdoor_autoload_copy_sdma_ucode(adev);
+ gfx_v10_0_rlc_backdoor_autoload_copy_gfx_ucode(adev);
+
+ rlc_g_offset = rlc_autoload_info[FIRMWARE_ID_RLC_G_UCODE].offset;
+ rlc_g_size = rlc_autoload_info[FIRMWARE_ID_RLC_G_UCODE].size;
+ gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset;
+
+ WREG32_SOC15(GC, 0, mmRLC_HYP_BOOTLOAD_ADDR_HI, upper_32_bits(gpu_addr));
+ WREG32_SOC15(GC, 0, mmRLC_HYP_BOOTLOAD_ADDR_LO, lower_32_bits(gpu_addr));
+ WREG32_SOC15(GC, 0, mmRLC_HYP_BOOTLOAD_SIZE, rlc_g_size);
+
+ tmp = RREG32_SOC15(GC, 0, mmRLC_HYP_RESET_VECTOR);
+ if (!(tmp & (RLC_HYP_RESET_VECTOR__COLD_BOOT_EXIT_MASK |
+ RLC_HYP_RESET_VECTOR__VDDGFX_EXIT_MASK))) {
+ DRM_ERROR("Neither COLD_BOOT_EXIT nor VDDGFX_EXIT is set\n");
+ return -EINVAL;
+ }
+
+ tmp = RREG32_SOC15(GC, 0, mmRLC_CNTL);
+ if (tmp & RLC_CNTL__RLC_ENABLE_F32_MASK) {
+ DRM_ERROR("RLC ROM should halt itself\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int gfx_v10_0_rlc_backdoor_autoload_config_me_cache(struct amdgpu_device *adev)
+{
+ uint32_t usec_timeout = 50000; /* wait for 50ms */
+ uint32_t tmp;
+ int i;
+ uint64_t addr;
+
+ /* Trigger an invalidation of the L1 instruction caches */
+ tmp = RREG32_SOC15(GC, 0, mmCP_ME_IC_OP_CNTL);
+ tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1);
+ WREG32_SOC15(GC, 0, mmCP_ME_IC_OP_CNTL, tmp);
+
+ /* Wait for invalidation complete */
+ for (i = 0; i < usec_timeout; i++) {
+ tmp = RREG32_SOC15(GC, 0, mmCP_ME_IC_OP_CNTL);
+ if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
+ INVALIDATE_CACHE_COMPLETE))
+ break;
+ udelay(1);
+ }
+
+ if (i >= usec_timeout) {
+ dev_err(adev->dev, "failed to invalidate instruction cache\n");
+ return -EINVAL;
+ }
+
+ /* Program me ucode address into intruction cache address register */
+ addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
+ rlc_autoload_info[FIRMWARE_ID_CP_ME].offset;
+ WREG32_SOC15(GC, 0, mmCP_ME_IC_BASE_LO,
+ lower_32_bits(addr) & 0xFFFFF000);
+ WREG32_SOC15(GC, 0, mmCP_ME_IC_BASE_HI,
+ upper_32_bits(addr));
+
+ return 0;
+}
+
+static int gfx_v10_0_rlc_backdoor_autoload_config_ce_cache(struct amdgpu_device *adev)
+{
+ uint32_t usec_timeout = 50000; /* wait for 50ms */
+ uint32_t tmp;
+ int i;
+ uint64_t addr;
+
+ /* Trigger an invalidation of the L1 instruction caches */
+ tmp = RREG32_SOC15(GC, 0, mmCP_CE_IC_OP_CNTL);
+ tmp = REG_SET_FIELD(tmp, CP_CE_IC_OP_CNTL, INVALIDATE_CACHE, 1);
+ WREG32_SOC15(GC, 0, mmCP_CE_IC_OP_CNTL, tmp);
+
+ /* Wait for invalidation complete */
+ for (i = 0; i < usec_timeout; i++) {
+ tmp = RREG32_SOC15(GC, 0, mmCP_CE_IC_OP_CNTL);
+ if (1 == REG_GET_FIELD(tmp, CP_CE_IC_OP_CNTL,
+ INVALIDATE_CACHE_COMPLETE))
+ break;
+ udelay(1);
+ }
+
+ if (i >= usec_timeout) {
+ dev_err(adev->dev, "failed to invalidate instruction cache\n");
+ return -EINVAL;
+ }
+
+ /* Program ce ucode address into intruction cache address register */
+ addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
+ rlc_autoload_info[FIRMWARE_ID_CP_CE].offset;
+ WREG32_SOC15(GC, 0, mmCP_CE_IC_BASE_LO,
+ lower_32_bits(addr) & 0xFFFFF000);
+ WREG32_SOC15(GC, 0, mmCP_CE_IC_BASE_HI,
+ upper_32_bits(addr));
+
+ return 0;
+}
+
+static int gfx_v10_0_rlc_backdoor_autoload_config_pfp_cache(struct amdgpu_device *adev)
+{
+ uint32_t usec_timeout = 50000; /* wait for 50ms */
+ uint32_t tmp;
+ int i;
+ uint64_t addr;
+
+ /* Trigger an invalidation of the L1 instruction caches */
+ tmp = RREG32_SOC15(GC, 0, mmCP_PFP_IC_OP_CNTL);
+ tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1);
+ WREG32_SOC15(GC, 0, mmCP_PFP_IC_OP_CNTL, tmp);
+
+ /* Wait for invalidation complete */
+ for (i = 0; i < usec_timeout; i++) {
+ tmp = RREG32_SOC15(GC, 0, mmCP_PFP_IC_OP_CNTL);
+ if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
+ INVALIDATE_CACHE_COMPLETE))
+ break;
+ udelay(1);
+ }
+
+ if (i >= usec_timeout) {
+ dev_err(adev->dev, "failed to invalidate instruction cache\n");
+ return -EINVAL;
+ }
+
+ /* Program pfp ucode address into intruction cache address register */
+ addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
+ rlc_autoload_info[FIRMWARE_ID_CP_PFP].offset;
+ WREG32_SOC15(GC, 0, mmCP_PFP_IC_BASE_LO,
+ lower_32_bits(addr) & 0xFFFFF000);
+ WREG32_SOC15(GC, 0, mmCP_PFP_IC_BASE_HI,
+ upper_32_bits(addr));
+
+ return 0;
+}
+
+static int gfx_v10_0_rlc_backdoor_autoload_config_mec_cache(struct amdgpu_device *adev)
+{
+ uint32_t usec_timeout = 50000; /* wait for 50ms */
+ uint32_t tmp;
+ int i;
+ uint64_t addr;
+
+ /* Trigger an invalidation of the L1 instruction caches */
+ tmp = RREG32_SOC15(GC, 0, mmCP_CPC_IC_OP_CNTL);
+ tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
+ WREG32_SOC15(GC, 0, mmCP_CPC_IC_OP_CNTL, tmp);
+
+ /* Wait for invalidation complete */
+ for (i = 0; i < usec_timeout; i++) {
+ tmp = RREG32_SOC15(GC, 0, mmCP_CPC_IC_OP_CNTL);
+ if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
+ INVALIDATE_CACHE_COMPLETE))
+ break;
+ udelay(1);
+ }
+
+ if (i >= usec_timeout) {
+ dev_err(adev->dev, "failed to invalidate instruction cache\n");
+ return -EINVAL;
+ }
+
+ /* Program mec1 ucode address into intruction cache address register */
+ addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
+ rlc_autoload_info[FIRMWARE_ID_CP_MEC].offset;
+ WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
+ lower_32_bits(addr) & 0xFFFFF000);
+ WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
+ upper_32_bits(addr));
+
+ return 0;
+}
+
+static int gfx_v10_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
+{
+ uint32_t cp_status;
+ uint32_t bootload_status;
+ int i, r;
+
+ for (i = 0; i < adev->usec_timeout; i++) {
+ cp_status = RREG32_SOC15(GC, 0, mmCP_STAT);
+ bootload_status = RREG32_SOC15(GC, 0, mmRLC_RLCS_BOOTLOAD_STATUS);
+ if ((cp_status == 0) &&
+ (REG_GET_FIELD(bootload_status,
+ RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) {
+ break;
+ }
+ udelay(1);
+ }
+
+ if (i >= adev->usec_timeout) {
+ dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n");
+ return -ETIMEDOUT;
+ }
+
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
+ r = gfx_v10_0_rlc_backdoor_autoload_config_me_cache(adev);
+ if (r)
+ return r;
+
+ r = gfx_v10_0_rlc_backdoor_autoload_config_ce_cache(adev);
+ if (r)
+ return r;
+
+ r = gfx_v10_0_rlc_backdoor_autoload_config_pfp_cache(adev);
+ if (r)
+ return r;
+
+ r = gfx_v10_0_rlc_backdoor_autoload_config_mec_cache(adev);
+ if (r)
+ return r;
+ }
+
+ return 0;
+}
+
+static void gfx_v10_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
+{
+ int i;
+ u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
+
+ tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
+ tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
+ tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
+ if (!enable) {
+ for (i = 0; i < adev->gfx.num_gfx_rings; i++)
+ adev->gfx.gfx_ring[i].sched.ready = false;
+ }
+ WREG32_SOC15(GC, 0, mmCP_ME_CNTL, tmp);
+ udelay(50);
+}
+
+static int gfx_v10_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev)
+{
+ int r;
+ const struct gfx_firmware_header_v1_0 *pfp_hdr;
+ const __le32 *fw_data;
+ unsigned i, fw_size;
+ uint32_t tmp;
+ uint32_t usec_timeout = 50000; /* wait for 50ms */
+
+ pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
+ adev->gfx.pfp_fw->data;
+
+ amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
+
+ fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
+ le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
+ fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes);
+
+ r = amdgpu_bo_create_reserved(adev, pfp_hdr->header.ucode_size_bytes,
+ PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
+ &adev->gfx.pfp.pfp_fw_obj,
+ &adev->gfx.pfp.pfp_fw_gpu_addr,
+ (void **)&adev->gfx.pfp.pfp_fw_ptr);
+ if (r) {
+ dev_err(adev->dev, "(%d) failed to create pfp fw bo\n", r);
+ gfx_v10_0_pfp_fini(adev);
+ return r;
+ }
+
+ memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_data, fw_size);
+
+ amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
+ amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
+
+ /* Trigger an invalidation of the L1 instruction caches */
+ tmp = RREG32_SOC15(GC, 0, mmCP_PFP_IC_OP_CNTL);
+ tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1);
+ WREG32_SOC15(GC, 0, mmCP_PFP_IC_OP_CNTL, tmp);
+
+ /* Wait for invalidation complete */
+ for (i = 0; i < usec_timeout; i++) {
+ tmp = RREG32_SOC15(GC, 0, mmCP_PFP_IC_OP_CNTL);
+ if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
+ INVALIDATE_CACHE_COMPLETE))
+ break;
+ udelay(1);
+ }
+
+ if (i >= usec_timeout) {
+ dev_err(adev->dev, "failed to invalidate instruction cache\n");
+ return -EINVAL;
+ }
+
+ if (amdgpu_emu_mode == 1)
+ adev->nbio_funcs->hdp_flush(adev, NULL);
+
+ tmp = RREG32_SOC15(GC, 0, mmCP_PFP_IC_BASE_CNTL);
+ tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
+ tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
+ tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
+ tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
+ WREG32_SOC15(GC, 0, mmCP_PFP_IC_BASE_CNTL, tmp);
+ WREG32_SOC15(GC, 0, mmCP_PFP_IC_BASE_LO,
+ adev->gfx.pfp.pfp_fw_gpu_addr & 0xFFFFF000);
+ WREG32_SOC15(GC, 0, mmCP_PFP_IC_BASE_HI,
+ upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
+
+ return 0;
+}
+
+static int gfx_v10_0_cp_gfx_load_ce_microcode(struct amdgpu_device *adev)
+{
+ int r;
+ const struct gfx_firmware_header_v1_0 *ce_hdr;
+ const __le32 *fw_data;
+ unsigned i, fw_size;
+ uint32_t tmp;
+ uint32_t usec_timeout = 50000; /* wait for 50ms */
+
+ ce_hdr = (const struct gfx_firmware_header_v1_0 *)
+ adev->gfx.ce_fw->data;
+
+ amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
+
+ fw_data = (const __le32 *)(adev->gfx.ce_fw->data +
+ le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
+ fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes);
+
+ r = amdgpu_bo_create_reserved(adev, ce_hdr->header.ucode_size_bytes,
+ PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
+ &adev->gfx.ce.ce_fw_obj,
+ &adev->gfx.ce.ce_fw_gpu_addr,
+ (void **)&adev->gfx.ce.ce_fw_ptr);
+ if (r) {
+ dev_err(adev->dev, "(%d) failed to create ce fw bo\n", r);
+ gfx_v10_0_ce_fini(adev);
+ return r;
+ }
+
+ memcpy(adev->gfx.ce.ce_fw_ptr, fw_data, fw_size);
+
+ amdgpu_bo_kunmap(adev->gfx.ce.ce_fw_obj);
+ amdgpu_bo_unreserve(adev->gfx.ce.ce_fw_obj);
+
+ /* Trigger an invalidation of the L1 instruction caches */
+ tmp = RREG32_SOC15(GC, 0, mmCP_CE_IC_OP_CNTL);
+ tmp = REG_SET_FIELD(tmp, CP_CE_IC_OP_CNTL, INVALIDATE_CACHE, 1);
+ WREG32_SOC15(GC, 0, mmCP_CE_IC_OP_CNTL, tmp);
+
+ /* Wait for invalidation complete */
+ for (i = 0; i < usec_timeout; i++) {
+ tmp = RREG32_SOC15(GC, 0, mmCP_CE_IC_OP_CNTL);
+ if (1 == REG_GET_FIELD(tmp, CP_CE_IC_OP_CNTL,
+ INVALIDATE_CACHE_COMPLETE))
+ break;
+ udelay(1);
+ }
+
+ if (i >= usec_timeout) {
+ dev_err(adev->dev, "failed to invalidate instruction cache\n");
+ return -EINVAL;
+ }
+
+ if (amdgpu_emu_mode == 1)
+ adev->nbio_funcs->hdp_flush(adev, NULL);
+
+ tmp = RREG32_SOC15(GC, 0, mmCP_CE_IC_BASE_CNTL);
+ tmp = REG_SET_FIELD(tmp, CP_CE_IC_BASE_CNTL, VMID, 0);
+ tmp = REG_SET_FIELD(tmp, CP_CE_IC_BASE_CNTL, CACHE_POLICY, 0);
+ tmp = REG_SET_FIELD(tmp, CP_CE_IC_BASE_CNTL, EXE_DISABLE, 0);
+ tmp = REG_SET_FIELD(tmp, CP_CE_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
+ WREG32_SOC15(GC, 0, mmCP_CE_IC_BASE_LO,
+ adev->gfx.ce.ce_fw_gpu_addr & 0xFFFFF000);
+ WREG32_SOC15(GC, 0, mmCP_CE_IC_BASE_HI,
+ upper_32_bits(adev->gfx.ce.ce_fw_gpu_addr));
+
+ return 0;
+}
+
+static int gfx_v10_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev)
+{
+ int r;
+ const struct gfx_firmware_header_v1_0 *me_hdr;
+ const __le32 *fw_data;
+ unsigned i, fw_size;
+ uint32_t tmp;
+ uint32_t usec_timeout = 50000; /* wait for 50ms */
+
+ me_hdr = (const struct gfx_firmware_header_v1_0 *)
+ adev->gfx.me_fw->data;
+
+ amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
+
+ fw_data = (const __le32 *)(adev->gfx.me_fw->data +
+ le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
+ fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes);
+
+ r = amdgpu_bo_create_reserved(adev, me_hdr->header.ucode_size_bytes,
+ PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
+ &adev->gfx.me.me_fw_obj,
+ &adev->gfx.me.me_fw_gpu_addr,
+ (void **)&adev->gfx.me.me_fw_ptr);
+ if (r) {
+ dev_err(adev->dev, "(%d) failed to create me fw bo\n", r);
+ gfx_v10_0_me_fini(adev);
+ return r;
+ }
+
+ memcpy(adev->gfx.me.me_fw_ptr, fw_data, fw_size);
+
+ amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
+ amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
+
+ /* Trigger an invalidation of the L1 instruction caches */
+ tmp = RREG32_SOC15(GC, 0, mmCP_ME_IC_OP_CNTL);
+ tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1);
+ WREG32_SOC15(GC, 0, mmCP_ME_IC_OP_CNTL, tmp);
+
+ /* Wait for invalidation complete */
+ for (i = 0; i < usec_timeout; i++) {
+ tmp = RREG32_SOC15(GC, 0, mmCP_ME_IC_OP_CNTL);
+ if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
+ INVALIDATE_CACHE_COMPLETE))
+ break;
+ udelay(1);
+ }
+
+ if (i >= usec_timeout) {
+ dev_err(adev->dev, "failed to invalidate instruction cache\n");
+ return -EINVAL;
+ }
+
+ if (amdgpu_emu_mode == 1)
+ adev->nbio_funcs->hdp_flush(adev, NULL);
+
+ tmp = RREG32_SOC15(GC, 0, mmCP_ME_IC_BASE_CNTL);
+ tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
+ tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
+ tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
+ tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
+ WREG32_SOC15(GC, 0, mmCP_ME_IC_BASE_LO,
+ adev->gfx.me.me_fw_gpu_addr & 0xFFFFF000);
+ WREG32_SOC15(GC, 0, mmCP_ME_IC_BASE_HI,
+ upper_32_bits(adev->gfx.me.me_fw_gpu_addr));
+
+ return 0;
+}
+
+static int gfx_v10_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
+{
+ int r;
+
+ if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
+ return -EINVAL;
+
+ gfx_v10_0_cp_gfx_enable(adev, false);
+
+ r = gfx_v10_0_cp_gfx_load_pfp_microcode(adev);
+ if (r) {
+ dev_err(adev->dev, "(%d) failed to load pfp fw\n", r);
+ return r;
+ }
+
+ r = gfx_v10_0_cp_gfx_load_ce_microcode(adev);
+ if (r) {
+ dev_err(adev->dev, "(%d) failed to load ce fw\n", r);
+ return r;
+ }
+
+ r = gfx_v10_0_cp_gfx_load_me_microcode(adev);
+ if (r) {
+ dev_err(adev->dev, "(%d) failed to load me fw\n", r);
+ return r;
+ }
+
+ return 0;
+}
+
+static int gfx_v10_0_cp_gfx_start(struct amdgpu_device *adev)
+{
+ struct amdgpu_ring *ring;
+ const struct cs_section_def *sect = NULL;
+ const struct cs_extent_def *ext = NULL;
+ int r, i;
+ int ctx_reg_offset;
+
+ /* init the CP */
+ WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT,
+ adev->gfx.config.max_hw_contexts - 1);
+ WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
+
+ gfx_v10_0_cp_gfx_enable(adev, true);
+
+ ring = &adev->gfx.gfx_ring[0];
+ r = amdgpu_ring_alloc(ring, gfx_v10_0_get_csb_size(adev) + 4);
+ if (r) {
+ DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
+ return r;
+ }
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+ amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
+ amdgpu_ring_write(ring, 0x80000000);
+ amdgpu_ring_write(ring, 0x80000000);
+
+ for (sect = gfx10_cs_data; sect->section != NULL; ++sect) {
+ for (ext = sect->section; ext->extent != NULL; ++ext) {
+ if (sect->id == SECT_CONTEXT) {
+ amdgpu_ring_write(ring,
+ PACKET3(PACKET3_SET_CONTEXT_REG,
+ ext->reg_count));
+ amdgpu_ring_write(ring, ext->reg_index -
+ PACKET3_SET_CONTEXT_REG_START);
+ for (i = 0; i < ext->reg_count; i++)
+ amdgpu_ring_write(ring, ext->extent[i]);
+ }
+ }
+ }
+
+ ctx_reg_offset =
+ SOC15_REG_OFFSET(GC, 0, mmPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
+ amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
+ amdgpu_ring_write(ring, ctx_reg_offset);
+ amdgpu_ring_write(ring, adev->gfx.config.pa_sc_tile_steering_override);
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+ amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
+ amdgpu_ring_write(ring, 0);
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
+ amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
+ amdgpu_ring_write(ring, 0x8000);
+ amdgpu_ring_write(ring, 0x8000);
+
+ amdgpu_ring_commit(ring);
+
+ /* submit cs packet to copy state 0 to next available state */
+ ring = &adev->gfx.gfx_ring[1];
+ r = amdgpu_ring_alloc(ring, 2);
+ if (r) {
+ DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
+ return r;
+ }
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
+ amdgpu_ring_write(ring, 0);
+
+ amdgpu_ring_commit(ring);
+
+ return 0;
+}
+
+static void gfx_v10_0_cp_gfx_switch_pipe(struct amdgpu_device *adev,
+ CP_PIPE_ID pipe)
+{
+ u32 tmp;
+
+ tmp = RREG32_SOC15(GC, 0, mmGRBM_GFX_CNTL);
+ tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe);
+
+ WREG32_SOC15(GC, 0, mmGRBM_GFX_CNTL, tmp);
+}
+
+static void gfx_v10_0_cp_gfx_set_doorbell(struct amdgpu_device *adev,
+ struct amdgpu_ring *ring)
+{
+ u32 tmp;
+
+ tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
+ if (ring->use_doorbell) {
+ tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
+ DOORBELL_OFFSET, ring->doorbell_index);
+ tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
+ DOORBELL_EN, 1);
+ } else {
+ tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
+ DOORBELL_EN, 0);
+ }
+ WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
+ tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
+ DOORBELL_RANGE_LOWER, ring->doorbell_index);
+ WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
+
+ WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
+ CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
+}
+
+static int gfx_v10_0_cp_gfx_resume(struct amdgpu_device *adev)
+{
+ struct amdgpu_ring *ring;
+ u32 tmp;
+ u32 rb_bufsz;
+ u64 rb_addr, rptr_addr, wptr_gpu_addr;
+ u32 i;
+
+ /* Set the write pointer delay */
+ WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
+
+ /* set the RB to use vmid 0 */
+ WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
+
+ /* Init gfx ring 0 for pipe 0 */
+ mutex_lock(&adev->srbm_mutex);
+ gfx_v10_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
+ mutex_unlock(&adev->srbm_mutex);
+ /* Set ring buffer size */
+ ring = &adev->gfx.gfx_ring[0];
+ rb_bufsz = order_base_2(ring->ring_size / 8);
+ tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
+ tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
+#ifdef __BIG_ENDIAN
+ tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
+#endif
+ WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
+
+ /* Initialize the ring buffer's write pointers */
+ ring->wptr = 0;
+ WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
+ WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
+
+ /* set the wb address wether it's enabled or not */
+ rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
+ WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
+ WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
+ CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
+
+ wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
+ WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO,
+ lower_32_bits(wptr_gpu_addr));
+ WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI,
+ upper_32_bits(wptr_gpu_addr));
+
+ mdelay(1);
+ WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
+
+ rb_addr = ring->gpu_addr >> 8;
+ WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
+ WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
+
+ WREG32_SOC15(GC, 0, mmCP_RB_ACTIVE, 1);
+
+ gfx_v10_0_cp_gfx_set_doorbell(adev, ring);
+
+ /* Init gfx ring 1 for pipe 1 */
+ mutex_lock(&adev->srbm_mutex);
+ gfx_v10_0_cp_gfx_switch_pipe(adev, PIPE_ID1);
+ mutex_unlock(&adev->srbm_mutex);
+ ring = &adev->gfx.gfx_ring[1];
+ rb_bufsz = order_base_2(ring->ring_size / 8);
+ tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz);
+ tmp = REG_SET_FIELD(tmp, CP_RB1_CNTL, RB_BLKSZ, rb_bufsz - 2);
+ WREG32_SOC15(GC, 0, mmCP_RB1_CNTL, tmp);
+ /* Initialize the ring buffer's write pointers */
+ ring->wptr = 0;
+ WREG32_SOC15(GC, 0, mmCP_RB1_WPTR, lower_32_bits(ring->wptr));
+ WREG32_SOC15(GC, 0, mmCP_RB1_WPTR_HI, upper_32_bits(ring->wptr));
+ /* Set the wb address wether it's enabled or not */
+ rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
+ WREG32_SOC15(GC, 0, mmCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr));
+ WREG32_SOC15(GC, 0, mmCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
+ CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
+ wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
+ WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO,
+ lower_32_bits(wptr_gpu_addr));
+ WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI,
+ upper_32_bits(wptr_gpu_addr));
+
+ mdelay(1);
+ WREG32_SOC15(GC, 0, mmCP_RB1_CNTL, tmp);
+
+ rb_addr = ring->gpu_addr >> 8;
+ WREG32_SOC15(GC, 0, mmCP_RB1_BASE, rb_addr);
+ WREG32_SOC15(GC, 0, mmCP_RB1_BASE_HI, upper_32_bits(rb_addr));
+ WREG32_SOC15(GC, 0, mmCP_RB1_ACTIVE, 1);
+
+ gfx_v10_0_cp_gfx_set_doorbell(adev, ring);
+
+ /* Switch to pipe 0 */
+ mutex_lock(&adev->srbm_mutex);
+ gfx_v10_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
+ mutex_unlock(&adev->srbm_mutex);
+
+ /* start the ring */
+ gfx_v10_0_cp_gfx_start(adev);
+
+ for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
+ ring = &adev->gfx.gfx_ring[i];
+ ring->sched.ready = true;
+ }
+
+ return 0;
+}
+
+static void gfx_v10_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
+{
+ int i;
+
+ if (enable) {
+ WREG32_SOC15(GC, 0, mmCP_MEC_CNTL, 0);
+ } else {
+ WREG32_SOC15(GC, 0, mmCP_MEC_CNTL,
+ (CP_MEC_CNTL__MEC_ME1_HALT_MASK |
+ CP_MEC_CNTL__MEC_ME2_HALT_MASK));
+ for (i = 0; i < adev->gfx.num_compute_rings; i++)
+ adev->gfx.compute_ring[i].sched.ready = false;
+ adev->gfx.kiq.ring.sched.ready = false;
+ }
+ udelay(50);
+}
+
+static int gfx_v10_0_cp_compute_load_microcode(struct amdgpu_device *adev)
+{
+ const struct gfx_firmware_header_v1_0 *mec_hdr;
+ const __le32 *fw_data;
+ unsigned i;
+ u32 tmp;
+ u32 usec_timeout = 50000; /* Wait for 50 ms */
+
+ if (!adev->gfx.mec_fw)
+ return -EINVAL;
+
+ gfx_v10_0_cp_compute_enable(adev, false);
+
+ mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
+ amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
+
+ fw_data = (const __le32 *)
+ (adev->gfx.mec_fw->data +
+ le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
+
+ /* Trigger an invalidation of the L1 instruction caches */
+ tmp = RREG32_SOC15(GC, 0, mmCP_CPC_IC_OP_CNTL);
+ tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
+ WREG32_SOC15(GC, 0, mmCP_CPC_IC_OP_CNTL, tmp);
+
+ /* Wait for invalidation complete */
+ for (i = 0; i < usec_timeout; i++) {
+ tmp = RREG32_SOC15(GC, 0, mmCP_CPC_IC_OP_CNTL);
+ if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
+ INVALIDATE_CACHE_COMPLETE))
+ break;
+ udelay(1);
+ }
+
+ if (i >= usec_timeout) {
+ dev_err(adev->dev, "failed to invalidate instruction cache\n");
+ return -EINVAL;
+ }
+
+ if (amdgpu_emu_mode == 1)
+ adev->nbio_funcs->hdp_flush(adev, NULL);
+
+ tmp = RREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL);
+ tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
+ tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
+ tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
+ WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
+
+ WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr &
+ 0xFFFFF000);
+ WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
+ upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
+
+ /* MEC1 */
+ WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 0);
+
+ for (i = 0; i < mec_hdr->jt_size; i++)
+ WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
+ le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
+
+ WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version);
+
+ /*
+ * TODO: Loading MEC2 firmware is only necessary if MEC2 should run
+ * different microcode than MEC1.
+ */
+
+ return 0;
+}
+
+static void gfx_v10_0_kiq_setting(struct amdgpu_ring *ring)
+{
+ uint32_t tmp;
+ struct amdgpu_device *adev = ring->adev;
+
+ /* tell RLC which is KIQ queue */
+ tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
+ tmp &= 0xffffff00;
+ tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
+ WREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
+ tmp |= 0x80;
+ WREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
+}
+
+static int gfx_v10_0_gfx_mqd_init(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+ struct v10_gfx_mqd *mqd = ring->mqd_ptr;
+ uint64_t hqd_gpu_addr, wb_gpu_addr;
+ uint32_t tmp;
+ uint32_t rb_bufsz;
+
+ /* set up gfx hqd wptr */
+ mqd->cp_gfx_hqd_wptr = 0;
+ mqd->cp_gfx_hqd_wptr_hi = 0;
+
+ /* set the pointer to the MQD */
+ mqd->cp_mqd_base_addr = ring->mqd_gpu_addr & 0xfffffffc;
+ mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
+
+ /* set up mqd control */
+ tmp = RREG32_SOC15(GC, 0, mmCP_GFX_MQD_CONTROL);
+ tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0);
+ tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1);
+ tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0);
+ mqd->cp_gfx_mqd_control = tmp;
+
+ /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */
+ tmp = RREG32_SOC15(GC, 0, mmCP_GFX_HQD_VMID);
+ tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0);
+ mqd->cp_gfx_hqd_vmid = 0;
+
+ /* set up default queue priority level
+ * 0x0 = low priority, 0x1 = high priority */
+ tmp = RREG32_SOC15(GC, 0, mmCP_GFX_HQD_QUEUE_PRIORITY);
+ tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, 0);
+ mqd->cp_gfx_hqd_queue_priority = tmp;
+
+ /* set up time quantum */
+ tmp = RREG32_SOC15(GC, 0, mmCP_GFX_HQD_QUANTUM);
+ tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1);
+ mqd->cp_gfx_hqd_quantum = tmp;
+
+ /* set up gfx hqd base. this is similar as CP_RB_BASE */
+ hqd_gpu_addr = ring->gpu_addr >> 8;
+ mqd->cp_gfx_hqd_base = hqd_gpu_addr;
+ mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr);
+
+ /* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */
+ wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
+ mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc;
+ mqd->cp_gfx_hqd_rptr_addr_hi =
+ upper_32_bits(wb_gpu_addr) & 0xffff;
+
+ /* set up rb_wptr_poll addr */
+ wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
+ mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
+ mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
+
+ /* set up the gfx_hqd_control, similar as CP_RB0_CNTL */
+ rb_bufsz = order_base_2(ring->ring_size / 4) - 1;
+ tmp = RREG32_SOC15(GC, 0, mmCP_GFX_HQD_CNTL);
+ tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz);
+ tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2);
+#ifdef __BIG_ENDIAN
+ tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1);
+#endif
+ mqd->cp_gfx_hqd_cntl = tmp;
+
+ /* set up cp_doorbell_control */
+ tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
+ if (ring->use_doorbell) {
+ tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
+ DOORBELL_OFFSET, ring->doorbell_index);
+ tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
+ DOORBELL_EN, 1);
+ } else
+ tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
+ DOORBELL_EN, 0);
+ mqd->cp_rb_doorbell_control = tmp;
+
+ /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
+ ring->wptr = 0;
+ mqd->cp_gfx_hqd_rptr = RREG32_SOC15(GC, 0, mmCP_GFX_HQD_RPTR);
+
+ /* active the queue */
+ mqd->cp_gfx_hqd_active = 1;
+
+ return 0;
+}
+
+#ifdef BRING_UP_DEBUG
+static int gfx_v10_0_gfx_queue_init_register(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+ struct v10_gfx_mqd *mqd = ring->mqd_ptr;
+
+ /* set mmCP_GFX_HQD_WPTR/_HI to 0 */
+ WREG32_SOC15(GC, 0, mmCP_GFX_HQD_WPTR, mqd->cp_gfx_hqd_wptr);
+ WREG32_SOC15(GC, 0, mmCP_GFX_HQD_WPTR_HI, mqd->cp_gfx_hqd_wptr_hi);
+
+ /* set GFX_MQD_BASE */
+ WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr);
+ WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
+
+ /* set GFX_MQD_CONTROL */
+ WREG32_SOC15(GC, 0, mmCP_GFX_MQD_CONTROL, mqd->cp_gfx_mqd_control);
+
+ /* set GFX_HQD_VMID to 0 */
+ WREG32_SOC15(GC, 0, mmCP_GFX_HQD_VMID, mqd->cp_gfx_hqd_vmid);
+
+ WREG32_SOC15(GC, 0, mmCP_GFX_HQD_QUEUE_PRIORITY,
+ mqd->cp_gfx_hqd_queue_priority);
+ WREG32_SOC15(GC, 0, mmCP_GFX_HQD_QUANTUM, mqd->cp_gfx_hqd_quantum);
+
+ /* set GFX_HQD_BASE, similar as CP_RB_BASE */
+ WREG32_SOC15(GC, 0, mmCP_GFX_HQD_BASE, mqd->cp_gfx_hqd_base);
+ WREG32_SOC15(GC, 0, mmCP_GFX_HQD_BASE_HI, mqd->cp_gfx_hqd_base_hi);
+
+ /* set GFX_HQD_RPTR_ADDR, similar as CP_RB_RPTR */
+ WREG32_SOC15(GC, 0, mmCP_GFX_HQD_RPTR_ADDR, mqd->cp_gfx_hqd_rptr_addr);
+ WREG32_SOC15(GC, 0, mmCP_GFX_HQD_RPTR_ADDR_HI, mqd->cp_gfx_hqd_rptr_addr_hi);
+
+ /* set GFX_HQD_CNTL, similar as CP_RB_CNTL */
+ WREG32_SOC15(GC, 0, mmCP_GFX_HQD_CNTL, mqd->cp_gfx_hqd_cntl);
+
+ /* set RB_WPTR_POLL_ADDR */
+ WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, mqd->cp_rb_wptr_poll_addr_lo);
+ WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, mqd->cp_rb_wptr_poll_addr_hi);
+
+ /* set RB_DOORBELL_CONTROL */
+ WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, mqd->cp_rb_doorbell_control);
+
+ /* active the queue */
+ WREG32_SOC15(GC, 0, mmCP_GFX_HQD_ACTIVE, mqd->cp_gfx_hqd_active);
+
+ return 0;
+}
+#endif
+
+static int gfx_v10_0_gfx_init_queue(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+ struct v10_gfx_mqd *mqd = ring->mqd_ptr;
+
+ if (!adev->in_gpu_reset && !adev->in_suspend) {
+ memset((void *)mqd, 0, sizeof(*mqd));
+ mutex_lock(&adev->srbm_mutex);
+ nv_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
+ gfx_v10_0_gfx_mqd_init(ring);
+#ifdef BRING_UP_DEBUG
+ gfx_v10_0_gfx_queue_init_register(ring);
+#endif
+ nv_grbm_select(adev, 0, 0, 0, 0);
+ mutex_unlock(&adev->srbm_mutex);
+ if (adev->gfx.me.mqd_backup[AMDGPU_MAX_GFX_RINGS])
+ memcpy(adev->gfx.me.mqd_backup[AMDGPU_MAX_GFX_RINGS], mqd, sizeof(*mqd));
+ } else if (adev->in_gpu_reset) {
+ /* reset mqd with the backup copy */
+ if (adev->gfx.me.mqd_backup[AMDGPU_MAX_GFX_RINGS])
+ memcpy(mqd, adev->gfx.me.mqd_backup[AMDGPU_MAX_GFX_RINGS], sizeof(*mqd));
+ /* reset the ring */
+ ring->wptr = 0;
+ amdgpu_ring_clear_ring(ring);
+#ifdef BRING_UP_DEBUG
+ mutex_lock(&adev->srbm_mutex);
+ nv_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
+ gfx_v10_0_gfx_queue_init_register(ring);
+ nv_grbm_select(adev, 0, 0, 0, 0);
+ mutex_unlock(&adev->srbm_mutex);
+#endif
+ } else {
+ amdgpu_ring_clear_ring(ring);
+ }
+
+ return 0;
+}
+
+#ifndef BRING_UP_DEBUG
+static int gfx_v10_0_kiq_enable_kgq(struct amdgpu_device *adev)
+{
+ struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+ struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
+ int r, i;
+
+ if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
+ return -EINVAL;
+
+ r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size *
+ adev->gfx.num_gfx_rings);
+ if (r) {
+ DRM_ERROR("Failed to lock KIQ (%d).\n", r);
+ return r;
+ }
+
+ for (i = 0; i < adev->gfx.num_gfx_rings; i++)
+ kiq->pmf->kiq_map_queues(kiq_ring, &adev->gfx.gfx_ring[i]);
+
+ r = amdgpu_ring_test_ring(kiq_ring);
+ if (r) {
+ DRM_ERROR("kfq enable failed\n");
+ kiq_ring->sched.ready = false;
+ }
+ return r;
+}
+#endif
+
+static int gfx_v10_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
+{
+ int r, i;
+ struct amdgpu_ring *ring;
+
+ for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
+ ring = &adev->gfx.gfx_ring[i];
+
+ r = amdgpu_bo_reserve(ring->mqd_obj, false);
+ if (unlikely(r != 0))
+ goto done;
+
+ r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
+ if (!r) {
+ r = gfx_v10_0_gfx_init_queue(ring);
+ amdgpu_bo_kunmap(ring->mqd_obj);
+ ring->mqd_ptr = NULL;
+ }
+ amdgpu_bo_unreserve(ring->mqd_obj);
+ if (r)
+ goto done;
+ }
+#ifndef BRING_UP_DEBUG
+ r = gfx_v10_0_kiq_enable_kgq(adev);
+ if (r)
+ goto done;
+#endif
+ r = gfx_v10_0_cp_gfx_start(adev);
+ if (r)
+ goto done;
+
+ for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
+ ring = &adev->gfx.gfx_ring[i];
+ ring->sched.ready = true;
+ }
+done:
+ return r;
+}
+
+static int gfx_v10_0_compute_mqd_init(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+ struct v10_compute_mqd *mqd = ring->mqd_ptr;
+ uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
+ uint32_t tmp;
+
+ mqd->header = 0xC0310800;
+ mqd->compute_pipelinestat_enable = 0x00000001;
+ mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
+ mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
+ mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
+ mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
+ mqd->compute_misc_reserved = 0x00000003;
+
+ eop_base_addr = ring->eop_gpu_addr >> 8;
+ mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
+ mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
+
+ /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
+ tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
+ tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
+ (order_base_2(GFX10_MEC_HPD_SIZE / 4) - 1));
+
+ mqd->cp_hqd_eop_control = tmp;
+
+ /* enable doorbell? */
+ tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
+
+ if (ring->use_doorbell) {
+ tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+ DOORBELL_OFFSET, ring->doorbell_index);
+ tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+ DOORBELL_EN, 1);
+ tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+ DOORBELL_SOURCE, 0);
+ tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+ DOORBELL_HIT, 0);
+ } else {
+ tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+ DOORBELL_EN, 0);
+ }
+
+ mqd->cp_hqd_pq_doorbell_control = tmp;
+
+ /* disable the queue if it's active */
+ ring->wptr = 0;
+ mqd->cp_hqd_dequeue_request = 0;
+ mqd->cp_hqd_pq_rptr = 0;
+ mqd->cp_hqd_pq_wptr_lo = 0;
+ mqd->cp_hqd_pq_wptr_hi = 0;
+
+ /* set the pointer to the MQD */
+ mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
+ mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
+
+ /* set MQD vmid to 0 */
+ tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
+ tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
+ mqd->cp_mqd_control = tmp;
+
+ /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
+ hqd_gpu_addr = ring->gpu_addr >> 8;
+ mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
+ mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
+
+ /* set up the HQD, this is similar to CP_RB0_CNTL */
+ tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
+ tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
+ (order_base_2(ring->ring_size / 4) - 1));
+ tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
+ ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
+#ifdef __BIG_ENDIAN
+ tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
+#endif
+ tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
+ tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
+ tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
+ tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
+ mqd->cp_hqd_pq_control = tmp;
+
+ /* set the wb address whether it's enabled or not */
+ wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
+ mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
+ mqd->cp_hqd_pq_rptr_report_addr_hi =
+ upper_32_bits(wb_gpu_addr) & 0xffff;
+
+ /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
+ wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
+ mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
+ mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
+
+ tmp = 0;
+ /* enable the doorbell if requested */
+ if (ring->use_doorbell) {
+ tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
+ tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+ DOORBELL_OFFSET, ring->doorbell_index);
+
+ tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+ DOORBELL_EN, 1);
+ tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+ DOORBELL_SOURCE, 0);
+ tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+ DOORBELL_HIT, 0);
+ }
+
+ mqd->cp_hqd_pq_doorbell_control = tmp;
+
+ /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
+ ring->wptr = 0;
+ mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
+
+ /* set the vmid for the queue */
+ mqd->cp_hqd_vmid = 0;
+
+ tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
+ tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
+ mqd->cp_hqd_persistent_state = tmp;
+
+ /* set MIN_IB_AVAIL_SIZE */
+ tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
+ tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
+ mqd->cp_hqd_ib_control = tmp;
+
+ /* activate the queue */
+ mqd->cp_hqd_active = 1;
+
+ return 0;
+}
+
+static int gfx_v10_0_kiq_init_register(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+ struct v10_compute_mqd *mqd = ring->mqd_ptr;
+ int j;
+
+ /* disable wptr polling */
+ WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
+
+ /* write the EOP addr */
+ WREG32_SOC15(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
+ mqd->cp_hqd_eop_base_addr_lo);
+ WREG32_SOC15(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
+ mqd->cp_hqd_eop_base_addr_hi);
+
+ /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
+ WREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL,
+ mqd->cp_hqd_eop_control);
+
+ /* enable doorbell? */
+ WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
+ mqd->cp_hqd_pq_doorbell_control);
+
+ /* disable the queue if it's active */
+ if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
+ WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
+ for (j = 0; j < adev->usec_timeout; j++) {
+ if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
+ break;
+ udelay(1);
+ }
+ WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
+ mqd->cp_hqd_dequeue_request);
+ WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR,
+ mqd->cp_hqd_pq_rptr);
+ WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_LO,
+ mqd->cp_hqd_pq_wptr_lo);
+ WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_HI,
+ mqd->cp_hqd_pq_wptr_hi);
+ }
+
+ /* set the pointer to the MQD */
+ WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR,
+ mqd->cp_mqd_base_addr_lo);
+ WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR_HI,
+ mqd->cp_mqd_base_addr_hi);
+
+ /* set MQD vmid to 0 */
+ WREG32_SOC15(GC, 0, mmCP_MQD_CONTROL,
+ mqd->cp_mqd_control);
+
+ /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
+ WREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE,
+ mqd->cp_hqd_pq_base_lo);
+ WREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE_HI,
+ mqd->cp_hqd_pq_base_hi);
+
+ /* set up the HQD, this is similar to CP_RB0_CNTL */
+ WREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL,
+ mqd->cp_hqd_pq_control);
+
+ /* set the wb address whether it's enabled or not */
+ WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
+ mqd->cp_hqd_pq_rptr_report_addr_lo);
+ WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
+ mqd->cp_hqd_pq_rptr_report_addr_hi);
+
+ /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
+ WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
+ mqd->cp_hqd_pq_wptr_poll_addr_lo);
+ WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
+ mqd->cp_hqd_pq_wptr_poll_addr_hi);
+
+ /* enable the doorbell if requested */
+ if (ring->use_doorbell) {
+ WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
+ (adev->doorbell_index.kiq * 2) << 2);
+ WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
+ (adev->doorbell_index.userqueue_end * 2) << 2);
+ }
+
+ WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
+ mqd->cp_hqd_pq_doorbell_control);
+
+ /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
+ WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_LO,
+ mqd->cp_hqd_pq_wptr_lo);
+ WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_HI,
+ mqd->cp_hqd_pq_wptr_hi);
+
+ /* set the vmid for the queue */
+ WREG32_SOC15(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
+
+ WREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE,
+ mqd->cp_hqd_persistent_state);
+
+ /* activate the queue */
+ WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE,
+ mqd->cp_hqd_active);
+
+ if (ring->use_doorbell)
+ WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
+
+ return 0;
+}
+
+static int gfx_v10_0_kiq_init_queue(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+ struct v10_compute_mqd *mqd = ring->mqd_ptr;
+ int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
+
+ gfx_v10_0_kiq_setting(ring);
+
+ if (adev->in_gpu_reset) { /* for GPU_RESET case */
+ /* reset MQD to a clean status */
+ if (adev->gfx.mec.mqd_backup[mqd_idx])
+ memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
+
+ /* reset ring buffer */
+ ring->wptr = 0;
+ amdgpu_ring_clear_ring(ring);
+
+ mutex_lock(&adev->srbm_mutex);
+ nv_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
+ gfx_v10_0_kiq_init_register(ring);
+ nv_grbm_select(adev, 0, 0, 0, 0);
+ mutex_unlock(&adev->srbm_mutex);
+ } else {
+ memset((void *)mqd, 0, sizeof(*mqd));
+ mutex_lock(&adev->srbm_mutex);
+ nv_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
+ gfx_v10_0_compute_mqd_init(ring);
+ gfx_v10_0_kiq_init_register(ring);
+ nv_grbm_select(adev, 0, 0, 0, 0);
+ mutex_unlock(&adev->srbm_mutex);
+
+ if (adev->gfx.mec.mqd_backup[mqd_idx])
+ memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
+ }
+
+ return 0;
+}
+
+static int gfx_v10_0_kcq_init_queue(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+ struct v10_compute_mqd *mqd = ring->mqd_ptr;
+ int mqd_idx = ring - &adev->gfx.compute_ring[0];
+
+ if (!adev->in_gpu_reset && !adev->in_suspend) {
+ memset((void *)mqd, 0, sizeof(*mqd));
+ mutex_lock(&adev->srbm_mutex);
+ nv_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
+ gfx_v10_0_compute_mqd_init(ring);
+ nv_grbm_select(adev, 0, 0, 0, 0);
+ mutex_unlock(&adev->srbm_mutex);
+
+ if (adev->gfx.mec.mqd_backup[mqd_idx])
+ memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
+ } else if (adev->in_gpu_reset) { /* for GPU_RESET case */
+ /* reset MQD to a clean status */
+ if (adev->gfx.mec.mqd_backup[mqd_idx])
+ memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
+
+ /* reset ring buffer */
+ ring->wptr = 0;
+ amdgpu_ring_clear_ring(ring);
+ } else {
+ amdgpu_ring_clear_ring(ring);
+ }
+
+ return 0;
+}
+
+static int gfx_v10_0_kiq_resume(struct amdgpu_device *adev)
+{
+ struct amdgpu_ring *ring;
+ int r;
+
+ ring = &adev->gfx.kiq.ring;
+
+ r = amdgpu_bo_reserve(ring->mqd_obj, false);
+ if (unlikely(r != 0))
+ return r;
+
+ r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
+ if (unlikely(r != 0))
+ return r;
+
+ gfx_v10_0_kiq_init_queue(ring);
+ amdgpu_bo_kunmap(ring->mqd_obj);
+ ring->mqd_ptr = NULL;
+ amdgpu_bo_unreserve(ring->mqd_obj);
+ ring->sched.ready = true;
+ return 0;
+}
+
+static int gfx_v10_0_kcq_resume(struct amdgpu_device *adev)
+{
+ struct amdgpu_ring *ring = NULL;
+ int r = 0, i;
+
+ gfx_v10_0_cp_compute_enable(adev, true);
+
+ for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+ ring = &adev->gfx.compute_ring[i];
+
+ r = amdgpu_bo_reserve(ring->mqd_obj, false);
+ if (unlikely(r != 0))
+ goto done;
+ r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
+ if (!r) {
+ r = gfx_v10_0_kcq_init_queue(ring);
+ amdgpu_bo_kunmap(ring->mqd_obj);
+ ring->mqd_ptr = NULL;
+ }
+ amdgpu_bo_unreserve(ring->mqd_obj);
+ if (r)
+ goto done;
+ }
+
+ r = amdgpu_gfx_enable_kcq(adev);
+done:
+ return r;
+}
+
+static int gfx_v10_0_cp_resume(struct amdgpu_device *adev)
+{
+ int r, i;
+ struct amdgpu_ring *ring;
+
+ if (!(adev->flags & AMD_IS_APU))
+ gfx_v10_0_enable_gui_idle_interrupt(adev, false);
+
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+ /* legacy firmware loading */
+ r = gfx_v10_0_cp_gfx_load_microcode(adev);
+ if (r)
+ return r;
+
+ r = gfx_v10_0_cp_compute_load_microcode(adev);
+ if (r)
+ return r;
+ }
+
+ r = gfx_v10_0_kiq_resume(adev);
+ if (r)
+ return r;
+
+ r = gfx_v10_0_kcq_resume(adev);
+ if (r)
+ return r;
+
+ if (!amdgpu_async_gfx_ring) {
+ r = gfx_v10_0_cp_gfx_resume(adev);
+ if (r)
+ return r;
+ } else {
+ r = gfx_v10_0_cp_async_gfx_ring_resume(adev);
+ if (r)
+ return r;
+ }
+
+ for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
+ ring = &adev->gfx.gfx_ring[i];
+ DRM_INFO("gfx %d ring me %d pipe %d q %d\n",
+ i, ring->me, ring->pipe, ring->queue);
+ r = amdgpu_ring_test_ring(ring);
+ if (r) {
+ ring->sched.ready = false;
+ return r;
+ }
+ }
+
+ for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+ ring = &adev->gfx.compute_ring[i];
+ ring->sched.ready = true;
+ DRM_INFO("compute ring %d mec %d pipe %d q %d\n",
+ i, ring->me, ring->pipe, ring->queue);
+ r = amdgpu_ring_test_ring(ring);
+ if (r)
+ ring->sched.ready = false;
+ }
+
+ return 0;
+}
+
+static void gfx_v10_0_cp_enable(struct amdgpu_device *adev, bool enable)
+{
+ gfx_v10_0_cp_gfx_enable(adev, enable);
+ gfx_v10_0_cp_compute_enable(adev, enable);
+}
+
+static bool gfx_v10_0_check_grbm_cam_remapping(struct amdgpu_device *adev)
+{
+ uint32_t data, pattern = 0xDEADBEEF;
+
+ /* check if mmVGT_ESGS_RING_SIZE_UMD
+ * has been remapped to mmVGT_ESGS_RING_SIZE */
+ data = RREG32_SOC15(GC, 0, mmVGT_ESGS_RING_SIZE);
+
+ WREG32_SOC15(GC, 0, mmVGT_ESGS_RING_SIZE, 0);
+
+ WREG32_SOC15(GC, 0, mmVGT_ESGS_RING_SIZE_UMD, pattern);
+
+ if (RREG32_SOC15(GC, 0, mmVGT_ESGS_RING_SIZE) == pattern) {
+ WREG32_SOC15(GC, 0, mmVGT_ESGS_RING_SIZE_UMD, data);
+ return true;
+ } else {
+ WREG32_SOC15(GC, 0, mmVGT_ESGS_RING_SIZE, data);
+ return false;
+ }
+}
+
+static void gfx_v10_0_setup_grbm_cam_remapping(struct amdgpu_device *adev)
+{
+ uint32_t data;
+
+ /* initialize cam_index to 0
+ * index will auto-inc after each data writting */
+ WREG32_SOC15(GC, 0, mmGRBM_CAM_INDEX, 0);
+
+ /* mmVGT_TF_RING_SIZE_UMD -> mmVGT_TF_RING_SIZE */
+ data = (SOC15_REG_OFFSET(GC, 0, mmVGT_TF_RING_SIZE_UMD) <<
+ GRBM_CAM_DATA__CAM_ADDR__SHIFT) |
+ (SOC15_REG_OFFSET(GC, 0, mmVGT_TF_RING_SIZE) <<
+ GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT);
+ WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA_UPPER, 0);
+ WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA, data);
+
+ /* mmVGT_TF_MEMORY_BASE_UMD -> mmVGT_TF_MEMORY_BASE */
+ data = (SOC15_REG_OFFSET(GC, 0, mmVGT_TF_MEMORY_BASE_UMD) <<
+ GRBM_CAM_DATA__CAM_ADDR__SHIFT) |
+ (SOC15_REG_OFFSET(GC, 0, mmVGT_TF_MEMORY_BASE) <<
+ GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT);
+ WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA_UPPER, 0);
+ WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA, data);
+
+ /* mmVGT_TF_MEMORY_BASE_HI_UMD -> mmVGT_TF_MEMORY_BASE_HI */
+ data = (SOC15_REG_OFFSET(GC, 0, mmVGT_TF_MEMORY_BASE_HI_UMD) <<
+ GRBM_CAM_DATA__CAM_ADDR__SHIFT) |
+ (SOC15_REG_OFFSET(GC, 0, mmVGT_TF_MEMORY_BASE_HI) <<
+ GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT);
+ WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA_UPPER, 0);
+ WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA, data);
+
+ /* mmVGT_HS_OFFCHIP_PARAM_UMD -> mmVGT_HS_OFFCHIP_PARAM */
+ data = (SOC15_REG_OFFSET(GC, 0, mmVGT_HS_OFFCHIP_PARAM_UMD) <<
+ GRBM_CAM_DATA__CAM_ADDR__SHIFT) |
+ (SOC15_REG_OFFSET(GC, 0, mmVGT_HS_OFFCHIP_PARAM) <<
+ GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT);
+ WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA_UPPER, 0);
+ WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA, data);
+
+ /* mmVGT_ESGS_RING_SIZE_UMD -> mmVGT_ESGS_RING_SIZE */
+ data = (SOC15_REG_OFFSET(GC, 0, mmVGT_ESGS_RING_SIZE_UMD) <<
+ GRBM_CAM_DATA__CAM_ADDR__SHIFT) |
+ (SOC15_REG_OFFSET(GC, 0, mmVGT_ESGS_RING_SIZE) <<
+ GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT);
+ WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA_UPPER, 0);
+ WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA, data);
+
+ /* mmVGT_GSVS_RING_SIZE_UMD -> mmVGT_GSVS_RING_SIZE */
+ data = (SOC15_REG_OFFSET(GC, 0, mmVGT_GSVS_RING_SIZE_UMD) <<
+ GRBM_CAM_DATA__CAM_ADDR__SHIFT) |
+ (SOC15_REG_OFFSET(GC, 0, mmVGT_GSVS_RING_SIZE) <<
+ GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT);
+ WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA_UPPER, 0);
+ WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA, data);
+
+ /* mmSPI_CONFIG_CNTL_REMAP -> mmSPI_CONFIG_CNTL */
+ data = (SOC15_REG_OFFSET(GC, 0, mmSPI_CONFIG_CNTL_REMAP) <<
+ GRBM_CAM_DATA__CAM_ADDR__SHIFT) |
+ (SOC15_REG_OFFSET(GC, 0, mmSPI_CONFIG_CNTL) <<
+ GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT);
+ WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA_UPPER, 0);
+ WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA, data);
+}
+
+static int gfx_v10_0_hw_init(void *handle)
+{
+ int r;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ r = gfx_v10_0_csb_vram_pin(adev);
+ if (r)
+ return r;
+
+ if (!amdgpu_emu_mode)
+ gfx_v10_0_init_golden_registers(adev);
+
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+ /**
+ * For gfx 10, rlc firmware loading relies on smu firmware is
+ * loaded firstly, so in direct type, it has to load smc ucode
+ * here before rlc.
+ */
+ r = smu_load_microcode(&adev->smu);
+ if (r)
+ return r;
+
+ r = smu_check_fw_status(&adev->smu);
+ if (r) {
+ pr_err("SMC firmware status is not correct\n");
+ return r;
+ }
+ }
+
+ /* if GRBM CAM not remapped, set up the remapping */
+ if (!gfx_v10_0_check_grbm_cam_remapping(adev))
+ gfx_v10_0_setup_grbm_cam_remapping(adev);
+
+ gfx_v10_0_constants_init(adev);
+
+ r = gfx_v10_0_rlc_resume(adev);
+ if (r)
+ return r;
+
+ /*
+ * init golden registers and rlc resume may override some registers,
+ * reconfig them here
+ */
+ gfx_v10_0_tcp_harvest(adev);
+
+ r = gfx_v10_0_cp_resume(adev);
+ if (r)
+ return r;
+
+ return r;
+}
+
+#ifndef BRING_UP_DEBUG
+static int gfx_v10_0_kiq_disable_kgq(struct amdgpu_device *adev)
+{
+ struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+ struct amdgpu_ring *kiq_ring = &kiq->ring;
+ int i;
+
+ if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
+ return -EINVAL;
+
+ if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *
+ adev->gfx.num_gfx_rings))
+ return -ENOMEM;
+
+ for (i = 0; i < adev->gfx.num_gfx_rings; i++)
+ kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.gfx_ring[i],
+ PREEMPT_QUEUES, 0, 0);
+
+ return amdgpu_ring_test_ring(kiq_ring);
+}
+#endif
+
+static int gfx_v10_0_hw_fini(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ int r;
+
+ amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
+ amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+#ifndef BRING_UP_DEBUG
+ if (amdgpu_async_gfx_ring) {
+ r = gfx_v10_0_kiq_disable_kgq(adev);
+ if (r)
+ DRM_ERROR("KGQ disable failed\n");
+ }
+#endif
+ if (amdgpu_gfx_disable_kcq(adev))
+ DRM_ERROR("KCQ disable failed\n");
+ if (amdgpu_sriov_vf(adev)) {
+ pr_debug("For SRIOV client, shouldn't do anything.\n");
+ return 0;
+ }
+ gfx_v10_0_cp_enable(adev, false);
+ gfx_v10_0_enable_gui_idle_interrupt(adev, false);
+ gfx_v10_0_csb_vram_unpin(adev);
+
+ return 0;
+}
+
+static int gfx_v10_0_suspend(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ adev->in_suspend = true;
+ return gfx_v10_0_hw_fini(adev);
+}
+
+static int gfx_v10_0_resume(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ int r;
+
+ r = gfx_v10_0_hw_init(adev);
+ adev->in_suspend = false;
+ return r;
+}
+
+static bool gfx_v10_0_is_idle(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
+ GRBM_STATUS, GUI_ACTIVE))
+ return false;
+ else
+ return true;
+}
+
+static int gfx_v10_0_wait_for_idle(void *handle)
+{
+ unsigned i;
+ u32 tmp;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ for (i = 0; i < adev->usec_timeout; i++) {
+ /* read MC_STATUS */
+ tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS) &
+ GRBM_STATUS__GUI_ACTIVE_MASK;
+
+ if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
+ return 0;
+ udelay(1);
+ }
+ return -ETIMEDOUT;
+}
+
+static int gfx_v10_0_soft_reset(void *handle)
+{
+ u32 grbm_soft_reset = 0;
+ u32 tmp;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ /* GRBM_STATUS */
+ tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
+ if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
+ GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
+ GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__DB_BUSY_MASK |
+ GRBM_STATUS__CB_BUSY_MASK | GRBM_STATUS__GDS_BUSY_MASK |
+ GRBM_STATUS__SPI_BUSY_MASK | GRBM_STATUS__GE_BUSY_NO_DMA_MASK
+ | GRBM_STATUS__BCI_BUSY_MASK)) {
+ grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
+ GRBM_SOFT_RESET, SOFT_RESET_CP,
+ 1);
+ grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
+ GRBM_SOFT_RESET, SOFT_RESET_GFX,
+ 1);
+ }
+
+ if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
+ grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
+ GRBM_SOFT_RESET, SOFT_RESET_CP,
+ 1);
+ }
+
+ /* GRBM_STATUS2 */
+ tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
+ if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
+ grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
+ GRBM_SOFT_RESET, SOFT_RESET_RLC,
+ 1);
+
+ if (grbm_soft_reset) {
+ /* stop the rlc */
+ gfx_v10_0_rlc_stop(adev);
+
+ /* Disable GFX parsing/prefetching */
+ gfx_v10_0_cp_gfx_enable(adev, false);
+
+ /* Disable MEC parsing/prefetching */
+ gfx_v10_0_cp_compute_enable(adev, false);
+
+ if (grbm_soft_reset) {
+ tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
+ tmp |= grbm_soft_reset;
+ dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
+ WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
+ tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
+
+ udelay(50);
+
+ tmp &= ~grbm_soft_reset;
+ WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
+ tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
+ }
+
+ /* Wait a little for things to settle down */
+ udelay(50);
+ }
+ return 0;
+}
+
+static uint64_t gfx_v10_0_get_gpu_clock_counter(struct amdgpu_device *adev)
+{
+ uint64_t clock;
+
+ mutex_lock(&adev->gfx.gpu_clock_mutex);
+ WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
+ clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
+ ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
+ mutex_unlock(&adev->gfx.gpu_clock_mutex);
+ return clock;
+}
+
+static void gfx_v10_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
+ uint32_t vmid,
+ uint32_t gds_base, uint32_t gds_size,
+ uint32_t gws_base, uint32_t gws_size,
+ uint32_t oa_base, uint32_t oa_size)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ /* GDS Base */
+ gfx_v10_0_write_data_to_reg(ring, 0, false,
+ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
+ gds_base);
+
+ /* GDS Size */
+ gfx_v10_0_write_data_to_reg(ring, 0, false,
+ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
+ gds_size);
+
+ /* GWS */
+ gfx_v10_0_write_data_to_reg(ring, 0, false,
+ SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
+ gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
+
+ /* OA */
+ gfx_v10_0_write_data_to_reg(ring, 0, false,
+ SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
+ (1 << (oa_size + oa_base)) - (1 << oa_base));
+}
+
+static int gfx_v10_0_early_init(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ adev->gfx.num_gfx_rings = GFX10_NUM_GFX_RINGS;
+ adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS;
+
+ gfx_v10_0_set_kiq_pm4_funcs(adev);
+ gfx_v10_0_set_ring_funcs(adev);
+ gfx_v10_0_set_irq_funcs(adev);
+ gfx_v10_0_set_gds_init(adev);
+ gfx_v10_0_set_rlc_funcs(adev);
+
+ return 0;
+}
+
+static int gfx_v10_0_late_init(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ int r;
+
+ r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
+ if (r)
+ return r;
+
+ r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
+ if (r)
+ return r;
+
+ return 0;
+}
+
+static bool gfx_v10_0_is_rlc_enabled(struct amdgpu_device *adev)
+{
+ uint32_t rlc_cntl;
+
+ /* if RLC is not enabled, do nothing */
+ rlc_cntl = RREG32_SOC15(GC, 0, mmRLC_CNTL);
+ return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false;
+}
+
+static void gfx_v10_0_set_safe_mode(struct amdgpu_device *adev)
+{
+ uint32_t data;
+ unsigned i;
+
+ data = RLC_SAFE_MODE__CMD_MASK;
+ data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
+ WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
+
+ /* wait for RLC_SAFE_MODE */
+ for (i = 0; i < adev->usec_timeout; i++) {
+ if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
+ break;
+ udelay(1);
+ }
+}
+
+static void gfx_v10_0_unset_safe_mode(struct amdgpu_device *adev)
+{
+ uint32_t data;
+
+ data = RLC_SAFE_MODE__CMD_MASK;
+ WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
+}
+
+static void gfx_v10_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
+ bool enable)
+{
+ uint32_t data, def;
+
+ /* It is disabled by HW by default */
+ if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
+ /* 1 - RLC_CGTT_MGCG_OVERRIDE */
+ def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
+ data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
+ RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
+ RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
+
+ /* only for Vega10 & Raven1 */
+ data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
+
+ if (def != data)
+ WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
+
+ /* MGLS is a global flag to control all MGLS in GFX */
+ if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
+ /* 2 - RLC memory Light sleep */
+ if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
+ def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
+ data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
+ if (def != data)
+ WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
+ }
+ /* 3 - CP memory Light sleep */
+ if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
+ def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
+ data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
+ if (def != data)
+ WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
+ }
+ }
+ } else {
+ /* 1 - MGCG_OVERRIDE */
+ def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
+ data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
+ RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
+ RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
+ RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
+ if (def != data)
+ WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
+
+ /* 2 - disable MGLS in RLC */
+ data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
+ if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
+ data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
+ WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
+ }
+
+ /* 3 - disable MGLS in CP */
+ data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
+ if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
+ data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
+ WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
+ }
+ }
+}
+
+static void gfx_v10_0_update_3d_clock_gating(struct amdgpu_device *adev,
+ bool enable)
+{
+ uint32_t data, def;
+
+ /* Enable 3D CGCG/CGLS */
+ if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) {
+ /* write cmd to clear cgcg/cgls ov */
+ def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
+ /* unset CGCG override */
+ data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
+ /* update CGCG and CGLS override bits */
+ if (def != data)
+ WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
+ /* enable 3Dcgcg FSM(0x0000363f) */
+ def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
+ data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
+ RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
+ if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
+ data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
+ RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
+ if (def != data)
+ WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
+
+ /* set IDLE_POLL_COUNT(0x00900100) */
+ def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
+ data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
+ (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
+ if (def != data)
+ WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
+ } else {
+ /* Disable CGCG/CGLS */
+ def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
+ /* disable cgcg, cgls should be disabled */
+ data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
+ RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
+ /* disable cgcg and cgls in FSM */
+ if (def != data)
+ WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
+ }
+}
+
+static void gfx_v10_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
+ bool enable)
+{
+ uint32_t def, data;
+
+ if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
+ def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
+ /* unset CGCG override */
+ data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
+ if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
+ data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
+ else
+ data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
+ /* update CGCG and CGLS override bits */
+ if (def != data)
+ WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
+
+ /* enable cgcg FSM(0x0000363F) */
+ def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
+ data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
+ RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
+ if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
+ data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
+ RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
+ if (def != data)
+ WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
+
+ /* set IDLE_POLL_COUNT(0x00900100) */
+ def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
+ data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
+ (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
+ if (def != data)
+ WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
+ } else {
+ def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
+ /* reset CGCG/CGLS bits */
+ data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
+ /* disable cgcg and cgls in FSM */
+ if (def != data)
+ WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
+ }
+}
+
+static int gfx_v10_0_update_gfx_clock_gating(struct amdgpu_device *adev,
+ bool enable)
+{
+ amdgpu_gfx_rlc_enter_safe_mode(adev);
+
+ if (enable) {
+ /* CGCG/CGLS should be enabled after MGCG/MGLS
+ * === MGCG + MGLS ===
+ */
+ gfx_v10_0_update_medium_grain_clock_gating(adev, enable);
+ /* === CGCG /CGLS for GFX 3D Only === */
+ gfx_v10_0_update_3d_clock_gating(adev, enable);
+ /* === CGCG + CGLS === */
+ gfx_v10_0_update_coarse_grain_clock_gating(adev, enable);
+ } else {
+ /* CGCG/CGLS should be disabled before MGCG/MGLS
+ * === CGCG + CGLS ===
+ */
+ gfx_v10_0_update_coarse_grain_clock_gating(adev, enable);
+ /* === CGCG /CGLS for GFX 3D Only === */
+ gfx_v10_0_update_3d_clock_gating(adev, enable);
+ /* === MGCG + MGLS === */
+ gfx_v10_0_update_medium_grain_clock_gating(adev, enable);
+ }
+
+ if (adev->cg_flags &
+ (AMD_CG_SUPPORT_GFX_MGCG |
+ AMD_CG_SUPPORT_GFX_CGLS |
+ AMD_CG_SUPPORT_GFX_CGCG |
+ AMD_CG_SUPPORT_GFX_CGLS |
+ AMD_CG_SUPPORT_GFX_3D_CGCG |
+ AMD_CG_SUPPORT_GFX_3D_CGLS))
+ gfx_v10_0_enable_gui_idle_interrupt(adev, enable);
+
+ amdgpu_gfx_rlc_exit_safe_mode(adev);
+
+ return 0;
+}
+
+static const struct amdgpu_rlc_funcs gfx_v10_0_rlc_funcs = {
+ .is_rlc_enabled = gfx_v10_0_is_rlc_enabled,
+ .set_safe_mode = gfx_v10_0_set_safe_mode,
+ .unset_safe_mode = gfx_v10_0_unset_safe_mode,
+ .init = gfx_v10_0_rlc_init,
+ .get_csb_size = gfx_v10_0_get_csb_size,
+ .get_csb_buffer = gfx_v10_0_get_csb_buffer,
+ .resume = gfx_v10_0_rlc_resume,
+ .stop = gfx_v10_0_rlc_stop,
+ .reset = gfx_v10_0_rlc_reset,
+ .start = gfx_v10_0_rlc_start
+};
+
+static int gfx_v10_0_set_powergating_state(void *handle,
+ enum amd_powergating_state state)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ bool enable = (state == AMD_PG_STATE_GATE) ? true : false;
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ if (!enable) {
+ amdgpu_gfx_off_ctrl(adev, false);
+ cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work);
+ } else
+ amdgpu_gfx_off_ctrl(adev, true);
+ break;
+ default:
+ break;
+ }
+ return 0;
+}
+
+static int gfx_v10_0_set_clockgating_state(void *handle,
+ enum amd_clockgating_state state)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ gfx_v10_0_update_gfx_clock_gating(adev,
+ state == AMD_CG_STATE_GATE ? true : false);
+ break;
+ default:
+ break;
+ }
+ return 0;
+}
+
+static void gfx_v10_0_get_clockgating_state(void *handle, u32 *flags)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ int data;
+
+ /* AMD_CG_SUPPORT_GFX_MGCG */
+ data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
+ if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
+ *flags |= AMD_CG_SUPPORT_GFX_MGCG;
+
+ /* AMD_CG_SUPPORT_GFX_CGCG */
+ data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
+ if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
+ *flags |= AMD_CG_SUPPORT_GFX_CGCG;
+
+ /* AMD_CG_SUPPORT_GFX_CGLS */
+ if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
+ *flags |= AMD_CG_SUPPORT_GFX_CGLS;
+
+ /* AMD_CG_SUPPORT_GFX_RLC_LS */
+ data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
+ if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
+ *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
+
+ /* AMD_CG_SUPPORT_GFX_CP_LS */
+ data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
+ if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
+ *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
+
+ /* AMD_CG_SUPPORT_GFX_3D_CGCG */
+ data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
+ if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
+ *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
+
+ /* AMD_CG_SUPPORT_GFX_3D_CGLS */
+ if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
+ *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
+}
+
+static u64 gfx_v10_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
+{
+ return ring->adev->wb.wb[ring->rptr_offs]; /* gfx10 is 32bit rptr*/
+}
+
+static u64 gfx_v10_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+ u64 wptr;
+
+ /* XXX check if swapping is necessary on BE */
+ if (ring->use_doorbell) {
+ wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]);
+ } else {
+ wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
+ wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
+ }
+
+ return wptr;
+}
+
+static void gfx_v10_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ if (ring->use_doorbell) {
+ /* XXX check if swapping is necessary on BE */
+ atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
+ WDOORBELL64(ring->doorbell_index, ring->wptr);
+ } else {
+ WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
+ WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
+ }
+}
+
+static u64 gfx_v10_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
+{
+ return ring->adev->wb.wb[ring->rptr_offs]; /* gfx10 hardware is 32bit rptr */
+}
+
+static u64 gfx_v10_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
+{
+ u64 wptr;
+
+ /* XXX check if swapping is necessary on BE */
+ if (ring->use_doorbell)
+ wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]);
+ else
+ BUG();
+ return wptr;
+}
+
+static void gfx_v10_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ /* XXX check if swapping is necessary on BE */
+ if (ring->use_doorbell) {
+ atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
+ WDOORBELL64(ring->doorbell_index, ring->wptr);
+ } else {
+ BUG(); /* only DOORBELL method supported on gfx10 now */
+ }
+}
+
+static void gfx_v10_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+ u32 ref_and_mask, reg_mem_engine;
+ const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio_funcs->hdp_flush_reg;
+
+ if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
+ switch (ring->me) {
+ case 1:
+ ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
+ break;
+ case 2:
+ ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
+ break;
+ default:
+ return;
+ }
+ reg_mem_engine = 0;
+ } else {
+ ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
+ reg_mem_engine = 1; /* pfp */
+ }
+
+ gfx_v10_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
+ adev->nbio_funcs->get_hdp_flush_req_offset(adev),
+ adev->nbio_funcs->get_hdp_flush_done_offset(adev),
+ ref_and_mask, ref_and_mask, 0x20);
+}
+
+static void gfx_v10_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
+ struct amdgpu_job *job,
+ struct amdgpu_ib *ib,
+ uint32_t flags)
+{
+ unsigned vmid = AMDGPU_JOB_GET_VMID(job);
+ u32 header, control = 0;
+
+ if (ib->flags & AMDGPU_IB_FLAG_CE)
+ header = PACKET3(PACKET3_INDIRECT_BUFFER_CNST, 2);
+ else
+ header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
+
+ control |= ib->length_dw | (vmid << 24);
+
+ if (amdgpu_mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
+ control |= INDIRECT_BUFFER_PRE_ENB(1);
+
+ if (flags & AMDGPU_IB_PREEMPTED)
+ control |= INDIRECT_BUFFER_PRE_RESUME(1);
+
+ if (!(ib->flags & AMDGPU_IB_FLAG_CE))
+ gfx_v10_0_ring_emit_de_meta(ring,
+ flags & AMDGPU_IB_PREEMPTED ? true : false);
+ }
+
+ amdgpu_ring_write(ring, header);
+ BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
+ amdgpu_ring_write(ring,
+#ifdef __BIG_ENDIAN
+ (2 << 0) |
+#endif
+ lower_32_bits(ib->gpu_addr));
+ amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
+ amdgpu_ring_write(ring, control);
+}
+
+static void gfx_v10_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
+ struct amdgpu_job *job,
+ struct amdgpu_ib *ib,
+ uint32_t flags)
+{
+ unsigned vmid = AMDGPU_JOB_GET_VMID(job);
+ u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
+
+ /* Currently, there is a high possibility to get wave ID mismatch
+ * between ME and GDS, leading to a hw deadlock, because ME generates
+ * different wave IDs than the GDS expects. This situation happens
+ * randomly when at least 5 compute pipes use GDS ordered append.
+ * The wave IDs generated by ME are also wrong after suspend/resume.
+ * Those are probably bugs somewhere else in the kernel driver.
+ *
+ * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
+ * GDS to 0 for this ring (me/pipe).
+ */
+ if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
+ amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
+ amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
+ amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
+ }
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
+ BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
+ amdgpu_ring_write(ring,
+#ifdef __BIG_ENDIAN
+ (2 << 0) |
+#endif
+ lower_32_bits(ib->gpu_addr));
+ amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
+ amdgpu_ring_write(ring, control);
+}
+
+static void gfx_v10_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
+ u64 seq, unsigned flags)
+{
+ struct amdgpu_device *adev = ring->adev;
+ bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
+ bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
+
+ /* Interrupt not work fine on GFX10.1 model yet. Use fallback instead */
+ if (adev->pdev->device == 0x50)
+ int_sel = false;
+
+ /* RELEASE_MEM - flush caches, send int */
+ amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
+ amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ |
+ PACKET3_RELEASE_MEM_GCR_GL2_WB |
+ PACKET3_RELEASE_MEM_GCR_GLM_INV | /* must be set with GLM_WB */
+ PACKET3_RELEASE_MEM_GCR_GLM_WB |
+ PACKET3_RELEASE_MEM_CACHE_POLICY(3) |
+ PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
+ PACKET3_RELEASE_MEM_EVENT_INDEX(5)));
+ amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) |
+ PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0)));
+
+ /*
+ * the address should be Qword aligned if 64bit write, Dword
+ * aligned if only send 32bit data low (discard data high)
+ */
+ if (write64bit)
+ BUG_ON(addr & 0x7);
+ else
+ BUG_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, lower_32_bits(seq));
+ amdgpu_ring_write(ring, upper_32_bits(seq));
+ amdgpu_ring_write(ring, 0);
+}
+
+static void gfx_v10_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
+{
+ int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
+ uint32_t seq = ring->fence_drv.sync_seq;
+ uint64_t addr = ring->fence_drv.gpu_addr;
+
+ gfx_v10_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr),
+ upper_32_bits(addr), seq, 0xffffffff, 4);
+}
+
+static void gfx_v10_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
+ unsigned vmid, uint64_t pd_addr)
+{
+ amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
+
+ /* compute doesn't have PFP */
+ if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
+ /* sync PFP to ME, otherwise we might get invalid PFP reads */
+ amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
+ amdgpu_ring_write(ring, 0x0);
+ }
+}
+
+static void gfx_v10_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
+ u64 seq, unsigned int flags)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ /* we only allocate 32bit for each seq wb address */
+ BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
+
+ /* write fence seq to the "addr" */
+ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+ amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+ WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, lower_32_bits(seq));
+
+ if (flags & AMDGPU_FENCE_FLAG_INT) {
+ /* set register to trigger INT */
+ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+ amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+ WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
+ amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
+ amdgpu_ring_write(ring, 0);
+ amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
+ }
+}
+
+static void gfx_v10_0_ring_emit_sb(struct amdgpu_ring *ring)
+{
+ amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
+ amdgpu_ring_write(ring, 0);
+}
+
+static void gfx_v10_0_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
+{
+ uint32_t dw2 = 0;
+
+ if (amdgpu_mcbp)
+ gfx_v10_0_ring_emit_ce_meta(ring,
+ flags & AMDGPU_IB_PREEMPTED ? true : false);
+
+ gfx_v10_0_ring_emit_tmz(ring, true);
+
+ dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
+ if (flags & AMDGPU_HAVE_CTX_SWITCH) {
+ /* set load_global_config & load_global_uconfig */
+ dw2 |= 0x8001;
+ /* set load_cs_sh_regs */
+ dw2 |= 0x01000000;
+ /* set load_per_context_state & load_gfx_sh_regs for GFX */
+ dw2 |= 0x10002;
+
+ /* set load_ce_ram if preamble presented */
+ if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
+ dw2 |= 0x10000000;
+ } else {
+ /* still load_ce_ram if this is the first time preamble presented
+ * although there is no context switch happens.
+ */
+ if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
+ dw2 |= 0x10000000;
+ }
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
+ amdgpu_ring_write(ring, dw2);
+ amdgpu_ring_write(ring, 0);
+}
+
+static unsigned gfx_v10_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
+{
+ unsigned ret;
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
+ amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
+ amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
+ amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
+ ret = ring->wptr & ring->buf_mask;
+ amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
+
+ return ret;
+}
+
+static void gfx_v10_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
+{
+ unsigned cur;
+ BUG_ON(offset > ring->buf_mask);
+ BUG_ON(ring->ring[offset] != 0x55aa55aa);
+
+ cur = (ring->wptr - 1) & ring->buf_mask;
+ if (likely(cur > offset))
+ ring->ring[offset] = cur - offset;
+ else
+ ring->ring[offset] = (ring->buf_mask + 1) - offset + cur;
+}
+
+static int gfx_v10_0_ring_preempt_ib(struct amdgpu_ring *ring)
+{
+ int i, r = 0;
+ struct amdgpu_device *adev = ring->adev;
+ struct amdgpu_kiq *kiq = &adev->gfx.kiq;
+ struct amdgpu_ring *kiq_ring = &kiq->ring;
+
+ if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
+ return -EINVAL;
+
+ if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size))
+ return -ENOMEM;
+
+ /* assert preemption condition */
+ amdgpu_ring_set_preempt_cond_exec(ring, false);
+
+ /* assert IB preemption, emit the trailing fence */
+ kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
+ ring->trail_fence_gpu_addr,
+ ++ring->trail_seq);
+ amdgpu_ring_commit(kiq_ring);
+
+ /* poll the trailing fence */
+ for (i = 0; i < adev->usec_timeout; i++) {
+ if (ring->trail_seq ==
+ le32_to_cpu(*(ring->trail_fence_cpu_addr)))
+ break;
+ DRM_UDELAY(1);
+ }
+
+ if (i >= adev->usec_timeout) {
+ r = -EINVAL;
+ DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
+ }
+
+ /* deassert preemption condition */
+ amdgpu_ring_set_preempt_cond_exec(ring, true);
+ return r;
+}
+
+static void gfx_v10_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume)
+{
+ struct amdgpu_device *adev = ring->adev;
+ struct v10_ce_ib_state ce_payload = {0};
+ uint64_t csa_addr;
+ int cnt;
+
+ cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
+ csa_addr = amdgpu_csa_vaddr(ring->adev);
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
+ amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
+ WRITE_DATA_DST_SEL(8) |
+ WR_CONFIRM) |
+ WRITE_DATA_CACHE_POLICY(0));
+ amdgpu_ring_write(ring, lower_32_bits(csa_addr +
+ offsetof(struct v10_gfx_meta_data, ce_payload)));
+ amdgpu_ring_write(ring, upper_32_bits(csa_addr +
+ offsetof(struct v10_gfx_meta_data, ce_payload)));
+
+ if (resume)
+ amdgpu_ring_write_multiple(ring, adev->virt.csa_cpu_addr +
+ offsetof(struct v10_gfx_meta_data,
+ ce_payload),
+ sizeof(ce_payload) >> 2);
+ else
+ amdgpu_ring_write_multiple(ring, (void *)&ce_payload,
+ sizeof(ce_payload) >> 2);
+}
+
+static void gfx_v10_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume)
+{
+ struct amdgpu_device *adev = ring->adev;
+ struct v10_de_ib_state de_payload = {0};
+ uint64_t csa_addr, gds_addr;
+ int cnt;
+
+ csa_addr = amdgpu_csa_vaddr(ring->adev);
+ gds_addr = ALIGN(csa_addr + AMDGPU_CSA_SIZE - adev->gds.gds_size,
+ PAGE_SIZE);
+ de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
+ de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
+
+ cnt = (sizeof(de_payload) >> 2) + 4 - 2;
+ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
+ amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
+ WRITE_DATA_DST_SEL(8) |
+ WR_CONFIRM) |
+ WRITE_DATA_CACHE_POLICY(0));
+ amdgpu_ring_write(ring, lower_32_bits(csa_addr +
+ offsetof(struct v10_gfx_meta_data, de_payload)));
+ amdgpu_ring_write(ring, upper_32_bits(csa_addr +
+ offsetof(struct v10_gfx_meta_data, de_payload)));
+
+ if (resume)
+ amdgpu_ring_write_multiple(ring, adev->virt.csa_cpu_addr +
+ offsetof(struct v10_gfx_meta_data,
+ de_payload),
+ sizeof(de_payload) >> 2);
+ else
+ amdgpu_ring_write_multiple(ring, (void *)&de_payload,
+ sizeof(de_payload) >> 2);
+}
+
+static void gfx_v10_0_ring_emit_tmz(struct amdgpu_ring *ring, bool start)
+{
+ amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
+ amdgpu_ring_write(ring, FRAME_CMD(start ? 0 : 1)); /* frame_end */
+}
+
+static void gfx_v10_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
+ amdgpu_ring_write(ring, 0 | /* src: register*/
+ (5 << 8) | /* dst: memory */
+ (1 << 20)); /* write confirm */
+ amdgpu_ring_write(ring, reg);
+ amdgpu_ring_write(ring, 0);
+ amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
+ adev->virt.reg_val_offs * 4));
+ amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
+ adev->virt.reg_val_offs * 4));
+}
+
+static void gfx_v10_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
+ uint32_t val)
+{
+ uint32_t cmd = 0;
+
+ switch (ring->funcs->type) {
+ case AMDGPU_RING_TYPE_GFX:
+ cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
+ break;
+ case AMDGPU_RING_TYPE_KIQ:
+ cmd = (1 << 16); /* no inc addr */
+ break;
+ default:
+ cmd = WR_CONFIRM;
+ break;
+ }
+ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+ amdgpu_ring_write(ring, cmd);
+ amdgpu_ring_write(ring, reg);
+ amdgpu_ring_write(ring, 0);
+ amdgpu_ring_write(ring, val);
+}
+
+static void gfx_v10_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
+ uint32_t val, uint32_t mask)
+{
+ gfx_v10_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
+}
+
+static void
+gfx_v10_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
+ uint32_t me, uint32_t pipe,
+ enum amdgpu_interrupt_state state)
+{
+ uint32_t cp_int_cntl, cp_int_cntl_reg;
+
+ if (!me) {
+ switch (pipe) {
+ case 0:
+ cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0);
+ break;
+ case 1:
+ cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING1);
+ break;
+ default:
+ DRM_DEBUG("invalid pipe %d\n", pipe);
+ return;
+ }
+ } else {
+ DRM_DEBUG("invalid me %d\n", me);
+ return;
+ }
+
+ switch (state) {
+ case AMDGPU_IRQ_STATE_DISABLE:
+ cp_int_cntl = RREG32(cp_int_cntl_reg);
+ cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
+ TIME_STAMP_INT_ENABLE, 0);
+ WREG32(cp_int_cntl_reg, cp_int_cntl);
+ break;
+ case AMDGPU_IRQ_STATE_ENABLE:
+ cp_int_cntl = RREG32(cp_int_cntl_reg);
+ cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
+ TIME_STAMP_INT_ENABLE, 1);
+ WREG32(cp_int_cntl_reg, cp_int_cntl);
+ break;
+ default:
+ break;
+ }
+}
+
+static void gfx_v10_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
+ int me, int pipe,
+ enum amdgpu_interrupt_state state)
+{
+ u32 mec_int_cntl, mec_int_cntl_reg;
+
+ /*
+ * amdgpu controls only the first MEC. That's why this function only
+ * handles the setting of interrupts for this specific MEC. All other
+ * pipes' interrupts are set by amdkfd.
+ */
+
+ if (me == 1) {
+ switch (pipe) {
+ case 0:
+ mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
+ break;
+ case 1:
+ mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
+ break;
+ case 2:
+ mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
+ break;
+ case 3:
+ mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
+ break;
+ default:
+ DRM_DEBUG("invalid pipe %d\n", pipe);
+ return;
+ }
+ } else {
+ DRM_DEBUG("invalid me %d\n", me);
+ return;
+ }
+
+ switch (state) {
+ case AMDGPU_IRQ_STATE_DISABLE:
+ mec_int_cntl = RREG32(mec_int_cntl_reg);
+ mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
+ TIME_STAMP_INT_ENABLE, 0);
+ WREG32(mec_int_cntl_reg, mec_int_cntl);
+ break;
+ case AMDGPU_IRQ_STATE_ENABLE:
+ mec_int_cntl = RREG32(mec_int_cntl_reg);
+ mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
+ TIME_STAMP_INT_ENABLE, 1);
+ WREG32(mec_int_cntl_reg, mec_int_cntl);
+ break;
+ default:
+ break;
+ }
+}
+
+static int gfx_v10_0_set_eop_interrupt_state(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *src,
+ unsigned type,
+ enum amdgpu_interrupt_state state)
+{
+ switch (type) {
+ case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
+ gfx_v10_0_set_gfx_eop_interrupt_state(adev, 0, 0, state);
+ break;
+ case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP:
+ gfx_v10_0_set_gfx_eop_interrupt_state(adev, 0, 1, state);
+ break;
+ case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
+ gfx_v10_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
+ break;
+ case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
+ gfx_v10_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
+ break;
+ case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
+ gfx_v10_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
+ break;
+ case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
+ gfx_v10_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
+ break;
+ case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
+ gfx_v10_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
+ break;
+ case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
+ gfx_v10_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
+ break;
+ case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
+ gfx_v10_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
+ break;
+ case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
+ gfx_v10_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
+ break;
+ default:
+ break;
+ }
+ return 0;
+}
+
+static int gfx_v10_0_eop_irq(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *source,
+ struct amdgpu_iv_entry *entry)
+{
+ int i;
+ u8 me_id, pipe_id, queue_id;
+ struct amdgpu_ring *ring;
+
+ DRM_DEBUG("IH: CP EOP\n");
+ me_id = (entry->ring_id & 0x0c) >> 2;
+ pipe_id = (entry->ring_id & 0x03) >> 0;
+ queue_id = (entry->ring_id & 0x70) >> 4;
+
+ switch (me_id) {
+ case 0:
+ if (pipe_id == 0)
+ amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
+ else
+ amdgpu_fence_process(&adev->gfx.gfx_ring[1]);
+ break;
+ case 1:
+ case 2:
+ for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+ ring = &adev->gfx.compute_ring[i];
+ /* Per-queue interrupt is supported for MEC starting from VI.
+ * The interrupt can only be enabled/disabled per pipe instead of per queue.
+ */
+ if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
+ amdgpu_fence_process(ring);
+ }
+ break;
+ }
+ return 0;
+}
+
+static int gfx_v10_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *source,
+ unsigned type,
+ enum amdgpu_interrupt_state state)
+{
+ switch (state) {
+ case AMDGPU_IRQ_STATE_DISABLE:
+ case AMDGPU_IRQ_STATE_ENABLE:
+ WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
+ PRIV_REG_INT_ENABLE,
+ state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
+ break;
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+static int gfx_v10_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *source,
+ unsigned type,
+ enum amdgpu_interrupt_state state)
+{
+ switch (state) {
+ case AMDGPU_IRQ_STATE_DISABLE:
+ case AMDGPU_IRQ_STATE_ENABLE:
+ WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
+ PRIV_INSTR_INT_ENABLE,
+ state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+static void gfx_v10_0_handle_priv_fault(struct amdgpu_device *adev,
+ struct amdgpu_iv_entry *entry)
+{
+ u8 me_id, pipe_id, queue_id;
+ struct amdgpu_ring *ring;
+ int i;
+
+ me_id = (entry->ring_id & 0x0c) >> 2;
+ pipe_id = (entry->ring_id & 0x03) >> 0;
+ queue_id = (entry->ring_id & 0x70) >> 4;
+
+ switch (me_id) {
+ case 0:
+ for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
+ ring = &adev->gfx.gfx_ring[i];
+ /* we only enabled 1 gfx queue per pipe for now */
+ if (ring->me == me_id && ring->pipe == pipe_id)
+ drm_sched_fault(&ring->sched);
+ }
+ break;
+ case 1:
+ case 2:
+ for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+ ring = &adev->gfx.compute_ring[i];
+ if (ring->me == me_id && ring->pipe == pipe_id &&
+ ring->queue == queue_id)
+ drm_sched_fault(&ring->sched);
+ }
+ break;
+ default:
+ BUG();
+ }
+}
+
+static int gfx_v10_0_priv_reg_irq(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *source,
+ struct amdgpu_iv_entry *entry)
+{
+ DRM_ERROR("Illegal register access in command stream\n");
+ gfx_v10_0_handle_priv_fault(adev, entry);
+ return 0;
+}
+
+static int gfx_v10_0_priv_inst_irq(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *source,
+ struct amdgpu_iv_entry *entry)
+{
+ DRM_ERROR("Illegal instruction in command stream\n");
+ gfx_v10_0_handle_priv_fault(adev, entry);
+ return 0;
+}
+
+static int gfx_v10_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *src,
+ unsigned int type,
+ enum amdgpu_interrupt_state state)
+{
+ uint32_t tmp, target;
+ struct amdgpu_ring *ring = &(adev->gfx.kiq.ring);
+
+ if (ring->me == 1)
+ target = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
+ else
+ target = SOC15_REG_OFFSET(GC, 0, mmCP_ME2_PIPE0_INT_CNTL);
+ target += ring->pipe;
+
+ switch (type) {
+ case AMDGPU_CP_KIQ_IRQ_DRIVER0:
+ if (state == AMDGPU_IRQ_STATE_DISABLE) {
+ tmp = RREG32_SOC15(GC, 0, mmCPC_INT_CNTL);
+ tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
+ GENERIC2_INT_ENABLE, 0);
+ WREG32_SOC15(GC, 0, mmCPC_INT_CNTL, tmp);
+
+ tmp = RREG32(target);
+ tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL,
+ GENERIC2_INT_ENABLE, 0);
+ WREG32(target, tmp);
+ } else {
+ tmp = RREG32_SOC15(GC, 0, mmCPC_INT_CNTL);
+ tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
+ GENERIC2_INT_ENABLE, 1);
+ WREG32_SOC15(GC, 0, mmCPC_INT_CNTL, tmp);
+
+ tmp = RREG32(target);
+ tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL,
+ GENERIC2_INT_ENABLE, 1);
+ WREG32(target, tmp);
+ }
+ break;
+ default:
+ BUG(); /* kiq only support GENERIC2_INT now */
+ break;
+ }
+ return 0;
+}
+
+static int gfx_v10_0_kiq_irq(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *source,
+ struct amdgpu_iv_entry *entry)
+{
+ u8 me_id, pipe_id, queue_id;
+ struct amdgpu_ring *ring = &(adev->gfx.kiq.ring);
+
+ me_id = (entry->ring_id & 0x0c) >> 2;
+ pipe_id = (entry->ring_id & 0x03) >> 0;
+ queue_id = (entry->ring_id & 0x70) >> 4;
+ DRM_DEBUG("IH: CPC GENERIC2_INT, me:%d, pipe:%d, queue:%d\n",
+ me_id, pipe_id, queue_id);
+
+ amdgpu_fence_process(ring);
+ return 0;
+}
+
+static const struct amd_ip_funcs gfx_v10_0_ip_funcs = {
+ .name = "gfx_v10_0",
+ .early_init = gfx_v10_0_early_init,
+ .late_init = gfx_v10_0_late_init,
+ .sw_init = gfx_v10_0_sw_init,
+ .sw_fini = gfx_v10_0_sw_fini,
+ .hw_init = gfx_v10_0_hw_init,
+ .hw_fini = gfx_v10_0_hw_fini,
+ .suspend = gfx_v10_0_suspend,
+ .resume = gfx_v10_0_resume,
+ .is_idle = gfx_v10_0_is_idle,
+ .wait_for_idle = gfx_v10_0_wait_for_idle,
+ .soft_reset = gfx_v10_0_soft_reset,
+ .set_clockgating_state = gfx_v10_0_set_clockgating_state,
+ .set_powergating_state = gfx_v10_0_set_powergating_state,
+ .get_clockgating_state = gfx_v10_0_get_clockgating_state,
+};
+
+static const struct amdgpu_ring_funcs gfx_v10_0_ring_funcs_gfx = {
+ .type = AMDGPU_RING_TYPE_GFX,
+ .align_mask = 0xff,
+ .nop = PACKET3(PACKET3_NOP, 0x3FFF),
+ .support_64bit_ptrs = true,
+ .vmhub = AMDGPU_GFXHUB,
+ .get_rptr = gfx_v10_0_ring_get_rptr_gfx,
+ .get_wptr = gfx_v10_0_ring_get_wptr_gfx,
+ .set_wptr = gfx_v10_0_ring_set_wptr_gfx,
+ .emit_frame_size = /* totally 242 maximum if 16 IBs */
+ 5 + /* COND_EXEC */
+ 7 + /* PIPELINE_SYNC */
+ SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
+ SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
+ 2 + /* VM_FLUSH */
+ 8 + /* FENCE for VM_FLUSH */
+ 20 + /* GDS switch */
+ 4 + /* double SWITCH_BUFFER,
+ * the first COND_EXEC jump to the place
+ * just prior to this double SWITCH_BUFFER
+ */
+ 5 + /* COND_EXEC */
+ 7 + /* HDP_flush */
+ 4 + /* VGT_flush */
+ 14 + /* CE_META */
+ 31 + /* DE_META */
+ 3 + /* CNTX_CTRL */
+ 5 + /* HDP_INVL */
+ 8 + 8 + /* FENCE x2 */
+ 2, /* SWITCH_BUFFER */
+ .emit_ib_size = 4, /* gfx_v10_0_ring_emit_ib_gfx */
+ .emit_ib = gfx_v10_0_ring_emit_ib_gfx,
+ .emit_fence = gfx_v10_0_ring_emit_fence,
+ .emit_pipeline_sync = gfx_v10_0_ring_emit_pipeline_sync,
+ .emit_vm_flush = gfx_v10_0_ring_emit_vm_flush,
+ .emit_gds_switch = gfx_v10_0_ring_emit_gds_switch,
+ .emit_hdp_flush = gfx_v10_0_ring_emit_hdp_flush,
+ .test_ring = gfx_v10_0_ring_test_ring,
+ .test_ib = gfx_v10_0_ring_test_ib,
+ .insert_nop = amdgpu_ring_insert_nop,
+ .pad_ib = amdgpu_ring_generic_pad_ib,
+ .emit_switch_buffer = gfx_v10_0_ring_emit_sb,
+ .emit_cntxcntl = gfx_v10_0_ring_emit_cntxcntl,
+ .init_cond_exec = gfx_v10_0_ring_emit_init_cond_exec,
+ .patch_cond_exec = gfx_v10_0_ring_emit_patch_cond_exec,
+ .preempt_ib = gfx_v10_0_ring_preempt_ib,
+ .emit_tmz = gfx_v10_0_ring_emit_tmz,
+ .emit_wreg = gfx_v10_0_ring_emit_wreg,
+ .emit_reg_wait = gfx_v10_0_ring_emit_reg_wait,
+};
+
+static const struct amdgpu_ring_funcs gfx_v10_0_ring_funcs_compute = {
+ .type = AMDGPU_RING_TYPE_COMPUTE,
+ .align_mask = 0xff,
+ .nop = PACKET3(PACKET3_NOP, 0x3FFF),
+ .support_64bit_ptrs = true,
+ .vmhub = AMDGPU_GFXHUB,
+ .get_rptr = gfx_v10_0_ring_get_rptr_compute,
+ .get_wptr = gfx_v10_0_ring_get_wptr_compute,
+ .set_wptr = gfx_v10_0_ring_set_wptr_compute,
+ .emit_frame_size =
+ 20 + /* gfx_v10_0_ring_emit_gds_switch */
+ 7 + /* gfx_v10_0_ring_emit_hdp_flush */
+ 5 + /* hdp invalidate */
+ 7 + /* gfx_v10_0_ring_emit_pipeline_sync */
+ SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
+ SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
+ 2 + /* gfx_v10_0_ring_emit_vm_flush */
+ 8 + 8 + 8, /* gfx_v10_0_ring_emit_fence x3 for user fence, vm fence */
+ .emit_ib_size = 7, /* gfx_v10_0_ring_emit_ib_compute */
+ .emit_ib = gfx_v10_0_ring_emit_ib_compute,
+ .emit_fence = gfx_v10_0_ring_emit_fence,
+ .emit_pipeline_sync = gfx_v10_0_ring_emit_pipeline_sync,
+ .emit_vm_flush = gfx_v10_0_ring_emit_vm_flush,
+ .emit_gds_switch = gfx_v10_0_ring_emit_gds_switch,
+ .emit_hdp_flush = gfx_v10_0_ring_emit_hdp_flush,
+ .test_ring = gfx_v10_0_ring_test_ring,
+ .test_ib = gfx_v10_0_ring_test_ib,
+ .insert_nop = amdgpu_ring_insert_nop,
+ .pad_ib = amdgpu_ring_generic_pad_ib,
+ .emit_wreg = gfx_v10_0_ring_emit_wreg,
+ .emit_reg_wait = gfx_v10_0_ring_emit_reg_wait,
+};
+
+static const struct amdgpu_ring_funcs gfx_v10_0_ring_funcs_kiq = {
+ .type = AMDGPU_RING_TYPE_KIQ,
+ .align_mask = 0xff,
+ .nop = PACKET3(PACKET3_NOP, 0x3FFF),
+ .support_64bit_ptrs = true,
+ .vmhub = AMDGPU_GFXHUB,
+ .get_rptr = gfx_v10_0_ring_get_rptr_compute,
+ .get_wptr = gfx_v10_0_ring_get_wptr_compute,
+ .set_wptr = gfx_v10_0_ring_set_wptr_compute,
+ .emit_frame_size =
+ 20 + /* gfx_v10_0_ring_emit_gds_switch */
+ 7 + /* gfx_v10_0_ring_emit_hdp_flush */
+ 5 + /*hdp invalidate */
+ 7 + /* gfx_v10_0_ring_emit_pipeline_sync */
+ SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
+ SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
+ 2 + /* gfx_v10_0_ring_emit_vm_flush */
+ 8 + 8 + 8, /* gfx_v10_0_ring_emit_fence_kiq x3 for user fence, vm fence */
+ .emit_ib_size = 7, /* gfx_v10_0_ring_emit_ib_compute */
+ .emit_ib = gfx_v10_0_ring_emit_ib_compute,
+ .emit_fence = gfx_v10_0_ring_emit_fence_kiq,
+ .test_ring = gfx_v10_0_ring_test_ring,
+ .test_ib = gfx_v10_0_ring_test_ib,
+ .insert_nop = amdgpu_ring_insert_nop,
+ .pad_ib = amdgpu_ring_generic_pad_ib,
+ .emit_rreg = gfx_v10_0_ring_emit_rreg,
+ .emit_wreg = gfx_v10_0_ring_emit_wreg,
+ .emit_reg_wait = gfx_v10_0_ring_emit_reg_wait,
+};
+
+static void gfx_v10_0_set_ring_funcs(struct amdgpu_device *adev)
+{
+ int i;
+
+ adev->gfx.kiq.ring.funcs = &gfx_v10_0_ring_funcs_kiq;
+
+ for (i = 0; i < adev->gfx.num_gfx_rings; i++)
+ adev->gfx.gfx_ring[i].funcs = &gfx_v10_0_ring_funcs_gfx;
+
+ for (i = 0; i < adev->gfx.num_compute_rings; i++)
+ adev->gfx.compute_ring[i].funcs = &gfx_v10_0_ring_funcs_compute;
+}
+
+static const struct amdgpu_irq_src_funcs gfx_v10_0_eop_irq_funcs = {
+ .set = gfx_v10_0_set_eop_interrupt_state,
+ .process = gfx_v10_0_eop_irq,
+};
+
+static const struct amdgpu_irq_src_funcs gfx_v10_0_priv_reg_irq_funcs = {
+ .set = gfx_v10_0_set_priv_reg_fault_state,
+ .process = gfx_v10_0_priv_reg_irq,
+};
+
+static const struct amdgpu_irq_src_funcs gfx_v10_0_priv_inst_irq_funcs = {
+ .set = gfx_v10_0_set_priv_inst_fault_state,
+ .process = gfx_v10_0_priv_inst_irq,
+};
+
+static const struct amdgpu_irq_src_funcs gfx_v10_0_kiq_irq_funcs = {
+ .set = gfx_v10_0_kiq_set_interrupt_state,
+ .process = gfx_v10_0_kiq_irq,
+};
+
+static void gfx_v10_0_set_irq_funcs(struct amdgpu_device *adev)
+{
+ adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
+ adev->gfx.eop_irq.funcs = &gfx_v10_0_eop_irq_funcs;
+
+ adev->gfx.kiq.irq.num_types = AMDGPU_CP_KIQ_IRQ_LAST;
+ adev->gfx.kiq.irq.funcs = &gfx_v10_0_kiq_irq_funcs;
+
+ adev->gfx.priv_reg_irq.num_types = 1;
+ adev->gfx.priv_reg_irq.funcs = &gfx_v10_0_priv_reg_irq_funcs;
+
+ adev->gfx.priv_inst_irq.num_types = 1;
+ adev->gfx.priv_inst_irq.funcs = &gfx_v10_0_priv_inst_irq_funcs;
+}
+
+static void gfx_v10_0_set_rlc_funcs(struct amdgpu_device *adev)
+{
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ adev->gfx.rlc.funcs = &gfx_v10_0_rlc_funcs;
+ break;
+ default:
+ break;
+ }
+}
+
+static void gfx_v10_0_set_gds_init(struct amdgpu_device *adev)
+{
+ /* init asic gds info */
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ default:
+ adev->gds.gds_size = 0x10000;
+ adev->gds.gds_compute_max_wave_id = 0x4ff;
+ break;
+ }
+
+ adev->gds.gws_size = 64;
+ adev->gds.oa_size = 16;
+}
+
+static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
+ u32 bitmap)
+{
+ u32 data;
+
+ if (!bitmap)
+ return;
+
+ data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
+ data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
+
+ WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
+}
+
+static u32 gfx_v10_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev)
+{
+ u32 data, wgp_bitmask;
+ data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
+ data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
+
+ data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
+ data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
+
+ wgp_bitmask =
+ amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1);
+
+ return (~data) & wgp_bitmask;
+}
+
+static u32 gfx_v10_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev)
+{
+ u32 wgp_idx, wgp_active_bitmap;
+ u32 cu_bitmap_per_wgp, cu_active_bitmap;
+
+ wgp_active_bitmap = gfx_v10_0_get_wgp_active_bitmap_per_sh(adev);
+ cu_active_bitmap = 0;
+
+ for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) {
+ /* if there is one WGP enabled, it means 2 CUs will be enabled */
+ cu_bitmap_per_wgp = 3 << (2 * wgp_idx);
+ if (wgp_active_bitmap & (1 << wgp_idx))
+ cu_active_bitmap |= cu_bitmap_per_wgp;
+ }
+
+ return cu_active_bitmap;
+}
+
+static int gfx_v10_0_get_cu_info(struct amdgpu_device *adev,
+ struct amdgpu_cu_info *cu_info)
+{
+ int i, j, k, counter, active_cu_number = 0;
+ u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
+ unsigned disable_masks[4 * 2];
+
+ if (!adev || !cu_info)
+ return -EINVAL;
+
+ amdgpu_gfx_parse_disable_cu(disable_masks, 4, 2);
+
+ mutex_lock(&adev->grbm_idx_mutex);
+ for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
+ for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
+ mask = 1;
+ ao_bitmap = 0;
+ counter = 0;
+ gfx_v10_0_select_se_sh(adev, i, j, 0xffffffff);
+ if (i < 4 && j < 2)
+ gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh(
+ adev, disable_masks[i * 2 + j]);
+ bitmap = gfx_v10_0_get_cu_active_bitmap_per_sh(adev);
+ cu_info->bitmap[i][j] = bitmap;
+
+ for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
+ if (bitmap & mask) {
+ if (counter < adev->gfx.config.max_cu_per_sh)
+ ao_bitmap |= mask;
+ counter++;
+ }
+ mask <<= 1;
+ }
+ active_cu_number += counter;
+ if (i < 2 && j < 2)
+ ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
+ cu_info->ao_cu_bitmap[i][j] = ao_bitmap;
+ }
+ }
+ gfx_v10_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+ mutex_unlock(&adev->grbm_idx_mutex);
+
+ cu_info->number = active_cu_number;
+ cu_info->ao_cu_mask = ao_cu_mask;
+ cu_info->simd_per_cu = NUM_SIMD_PER_CU;
+
+ return 0;
+}
+
+const struct amdgpu_ip_block_version gfx_v10_0_ip_block =
+{
+ .type = AMD_IP_BLOCK_TYPE_GFX,
+ .major = 10,
+ .minor = 0,
+ .rev = 0,
+ .funcs = &gfx_v10_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/vi_dpm.h b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.h
index c43e03fddfba..b442e50324d0 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.h
@@ -1,5 +1,5 @@
/*
- * Copyright 2014 Advanced Micro Devices, Inc.
+ * Copyright 2019 dvanced Micro Devices, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -21,12 +21,9 @@
*
*/
-#ifndef __VI_DPM_H__
-#define __VI_DPM_H__
+#ifndef __GFX_V10_0_H__
+#define __GFX_V10_0_H__
-extern const struct amd_ip_funcs cz_dpm_ip_funcs;
-int cz_smu_init(struct amdgpu_device *adev);
-int cz_smu_start(struct amdgpu_device *adev);
-int cz_smu_fini(struct amdgpu_device *adev);
+extern const struct amdgpu_ip_block_version gfx_v10_0_ip_block;
#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
index c0cb244f58cd..7f0a63628c43 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
@@ -21,6 +21,8 @@
*
*/
#include <linux/firmware.h>
+#include <linux/module.h>
+
#include "amdgpu.h"
#include "amdgpu_ih.h"
#include "amdgpu_gfx.h"
@@ -1812,7 +1814,7 @@ static int gfx_v6_0_ring_test_ring(struct amdgpu_ring *ring)
tmp = RREG32(scratch);
if (tmp == 0xDEADBEEF)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
@@ -3041,7 +3043,7 @@ static void gfx_v6_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
}
static void gfx_v6_0_select_me_pipe_q(struct amdgpu_device *adev,
- u32 me, u32 pipe, u32 q)
+ u32 me, u32 pipe, u32 q, u32 vm)
{
DRM_INFO("Not implemented\n");
}
@@ -3113,7 +3115,7 @@ static int gfx_v6_0_sw_init(void *handle)
ring->ring_obj = NULL;
sprintf(ring->name, "gfx");
r = amdgpu_ring_init(adev, ring, 1024,
- &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_EOP);
+ &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP);
if (r)
return r;
}
@@ -3348,7 +3350,7 @@ static int gfx_v6_0_set_eop_interrupt_state(struct amdgpu_device *adev,
enum amdgpu_interrupt_state state)
{
switch (type) {
- case AMDGPU_CP_IRQ_GFX_EOP:
+ case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
gfx_v6_0_set_gfx_eop_interrupt_state(adev, state);
break;
case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index a59e0fdf5a97..21187275dfd3 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -20,8 +20,10 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
+
#include <linux/firmware.h>
-#include <drm/drmP.h>
+#include <linux/module.h>
+
#include "amdgpu.h"
#include "amdgpu_ih.h"
#include "amdgpu_gfx.h"
@@ -1877,6 +1879,15 @@ static void gfx_v7_0_init_compute_vmid(struct amdgpu_device *adev)
}
cik_srbm_select(adev, 0, 0, 0, 0);
mutex_unlock(&adev->srbm_mutex);
+
+ /* Initialize all compute VMIDs to have no GDS, GWS, or OA
+ acccess. These should be enabled by FW for target VMIDs. */
+ for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
+ WREG32(amdgpu_gds_reg_offset[i].mem_base, 0);
+ WREG32(amdgpu_gds_reg_offset[i].mem_size, 0);
+ WREG32(amdgpu_gds_reg_offset[i].gws, 0);
+ WREG32(amdgpu_gds_reg_offset[i].oa, 0);
+ }
}
static void gfx_v7_0_config_init(struct amdgpu_device *adev)
@@ -2080,7 +2091,7 @@ static int gfx_v7_0_ring_test_ring(struct amdgpu_ring *ring)
tmp = RREG32(scratch);
if (tmp == 0xDEADBEEF)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
r = -ETIMEDOUT;
@@ -4167,9 +4178,9 @@ static void gfx_v7_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
}
static void gfx_v7_0_select_me_pipe_q(struct amdgpu_device *adev,
- u32 me, u32 pipe, u32 q)
+ u32 me, u32 pipe, u32 q, u32 vm)
{
- cik_srbm_select(adev, me, pipe, q, 0);
+ cik_srbm_select(adev, me, pipe, q, vm);
}
static const struct amdgpu_gfx_funcs gfx_v7_0_gfx_funcs = {
@@ -4460,7 +4471,7 @@ static int gfx_v7_0_sw_init(void *handle)
ring->ring_obj = NULL;
sprintf(ring->name, "gfx");
r = amdgpu_ring_init(adev, ring, 1024,
- &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_EOP);
+ &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP);
if (r)
return r;
}
@@ -4493,12 +4504,8 @@ static int gfx_v7_0_sw_init(void *handle)
static int gfx_v7_0_sw_fini(void *handle)
{
- int i;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
- amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL);
- amdgpu_bo_free_kernel(&adev->gds.gws_gfx_bo, NULL, NULL);
- amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL);
+ int i;
for (i = 0; i < adev->gfx.num_gfx_rings; i++)
amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
@@ -4801,7 +4808,7 @@ static int gfx_v7_0_set_eop_interrupt_state(struct amdgpu_device *adev,
enum amdgpu_interrupt_state state)
{
switch (type) {
- case AMDGPU_CP_IRQ_GFX_EOP:
+ case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
gfx_v7_0_set_gfx_eop_interrupt_state(adev, state);
break;
case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
@@ -5070,30 +5077,10 @@ static void gfx_v7_0_set_irq_funcs(struct amdgpu_device *adev)
static void gfx_v7_0_set_gds_init(struct amdgpu_device *adev)
{
/* init asci gds info */
- adev->gds.mem.total_size = RREG32(mmGDS_VMID0_SIZE);
- adev->gds.gws.total_size = 64;
- adev->gds.oa.total_size = 16;
+ adev->gds.gds_size = RREG32(mmGDS_VMID0_SIZE);
+ adev->gds.gws_size = 64;
+ adev->gds.oa_size = 16;
adev->gds.gds_compute_max_wave_id = RREG32(mmGDS_COMPUTE_MAX_WAVE_ID);
-
- if (adev->gds.mem.total_size == 64 * 1024) {
- adev->gds.mem.gfx_partition_size = 4096;
- adev->gds.mem.cs_partition_size = 4096;
-
- adev->gds.gws.gfx_partition_size = 4;
- adev->gds.gws.cs_partition_size = 4;
-
- adev->gds.oa.gfx_partition_size = 4;
- adev->gds.oa.cs_partition_size = 1;
- } else {
- adev->gds.mem.gfx_partition_size = 1024;
- adev->gds.mem.cs_partition_size = 1024;
-
- adev->gds.gws.gfx_partition_size = 16;
- adev->gds.gws.cs_partition_size = 16;
-
- adev->gds.oa.gfx_partition_size = 4;
- adev->gds.oa.cs_partition_size = 4;
- }
}
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index 02955e6e9dd9..ee1ccdcf2d30 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -20,9 +20,13 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
+
+#include <linux/delay.h>
#include <linux/kernel.h>
#include <linux/firmware.h>
-#include <drm/drmP.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
#include "amdgpu.h"
#include "amdgpu_gfx.h"
#include "vi.h"
@@ -855,7 +859,7 @@ static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring)
tmp = RREG32(scratch);
if (tmp == 0xDEADBEEF)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
@@ -1317,6 +1321,39 @@ static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
return 0;
}
+static int gfx_v8_0_csb_vram_pin(struct amdgpu_device *adev)
+{
+ int r;
+
+ r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);
+ if (unlikely(r != 0))
+ return r;
+
+ r = amdgpu_bo_pin(adev->gfx.rlc.clear_state_obj,
+ AMDGPU_GEM_DOMAIN_VRAM);
+ if (!r)
+ adev->gfx.rlc.clear_state_gpu_addr =
+ amdgpu_bo_gpu_offset(adev->gfx.rlc.clear_state_obj);
+
+ amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
+
+ return r;
+}
+
+static void gfx_v8_0_csb_vram_unpin(struct amdgpu_device *adev)
+{
+ int r;
+
+ if (!adev->gfx.rlc.clear_state_obj)
+ return;
+
+ r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, true);
+ if (likely(r == 0)) {
+ amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
+ amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
+ }
+}
+
static void gfx_v8_0_mec_fini(struct amdgpu_device *adev)
{
amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
@@ -2005,7 +2042,7 @@ static int gfx_v8_0_sw_init(void *handle)
}
r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
- AMDGPU_CP_IRQ_GFX_EOP);
+ AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP);
if (r)
return r;
}
@@ -2042,7 +2079,7 @@ static int gfx_v8_0_sw_init(void *handle)
return r;
/* create MQD for all compute queues as well as KIQ for SRIOV case */
- r = amdgpu_gfx_compute_mqd_sw_init(adev, sizeof(struct vi_mqd_allocation));
+ r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct vi_mqd_allocation));
if (r)
return r;
@@ -2057,19 +2094,15 @@ static int gfx_v8_0_sw_init(void *handle)
static int gfx_v8_0_sw_fini(void *handle)
{
- int i;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
- amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL);
- amdgpu_bo_free_kernel(&adev->gds.gws_gfx_bo, NULL, NULL);
- amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL);
+ int i;
for (i = 0; i < adev->gfx.num_gfx_rings; i++)
amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
for (i = 0; i < adev->gfx.num_compute_rings; i++)
amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
- amdgpu_gfx_compute_mqd_sw_fini(adev);
+ amdgpu_gfx_mqd_sw_fini(adev);
amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
amdgpu_gfx_kiq_fini(adev);
@@ -3436,9 +3469,9 @@ static void gfx_v8_0_select_se_sh(struct amdgpu_device *adev,
}
static void gfx_v8_0_select_me_pipe_q(struct amdgpu_device *adev,
- u32 me, u32 pipe, u32 q)
+ u32 me, u32 pipe, u32 q, u32 vm)
{
- vi_srbm_select(adev, me, pipe, q, 0);
+ vi_srbm_select(adev, me, pipe, q, vm);
}
static u32 gfx_v8_0_get_rb_active_bitmap(struct amdgpu_device *adev)
@@ -3706,6 +3739,15 @@ static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev)
}
vi_srbm_select(adev, 0, 0, 0, 0);
mutex_unlock(&adev->srbm_mutex);
+
+ /* Initialize all compute VMIDs to have no GDS, GWS, or OA
+ acccess. These should be enabled by FW for target VMIDs. */
+ for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
+ WREG32(amdgpu_gds_reg_offset[i].mem_base, 0);
+ WREG32(amdgpu_gds_reg_offset[i].mem_size, 0);
+ WREG32(amdgpu_gds_reg_offset[i].gws, 0);
+ WREG32(amdgpu_gds_reg_offset[i].oa, 0);
+ }
}
static void gfx_v8_0_config_init(struct amdgpu_device *adev)
@@ -3925,11 +3967,10 @@ static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev)
int list_size;
unsigned int *register_list_format =
- kmalloc(adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
+ kmemdup(adev->gfx.rlc.register_list_format,
+ adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
if (!register_list_format)
return -ENOMEM;
- memcpy(register_list_format, adev->gfx.rlc.register_list_format,
- adev->gfx.rlc.reg_list_format_size_bytes);
gfx_v8_0_parse_ind_reg_list(register_list_format,
RLC_FormatDirectRegListLength,
@@ -4777,6 +4818,10 @@ static int gfx_v8_0_hw_init(void *handle)
gfx_v8_0_init_golden_registers(adev);
gfx_v8_0_constants_init(adev);
+ r = gfx_v8_0_csb_vram_pin(adev);
+ if (r)
+ return r;
+
r = adev->gfx.rlc.funcs->resume(adev);
if (r)
return r;
@@ -4893,6 +4938,9 @@ static int gfx_v8_0_hw_fini(void *handle)
else
pr_err("rlc is busy, skip halt rlc\n");
amdgpu_gfx_rlc_exit_safe_mode(adev);
+
+ gfx_v8_0_csb_vram_unpin(adev);
+
return 0;
}
@@ -6217,7 +6265,7 @@ static void gfx_v8_0_pipe_reserve_resources(struct amdgpu_device *adev,
struct amdgpu_ring *iring;
mutex_lock(&adev->gfx.pipe_reserve_mutex);
- pipe = amdgpu_gfx_queue_to_bit(adev, ring->me, ring->pipe, 0);
+ pipe = amdgpu_gfx_mec_queue_to_bit(adev, ring->me, ring->pipe, 0);
if (acquire)
set_bit(pipe, adev->gfx.pipe_reserve_bitmap);
else
@@ -6236,20 +6284,20 @@ static void gfx_v8_0_pipe_reserve_resources(struct amdgpu_device *adev,
/* Lower all pipes without a current reservation */
for (i = 0; i < adev->gfx.num_gfx_rings; ++i) {
iring = &adev->gfx.gfx_ring[i];
- pipe = amdgpu_gfx_queue_to_bit(adev,
- iring->me,
- iring->pipe,
- 0);
+ pipe = amdgpu_gfx_mec_queue_to_bit(adev,
+ iring->me,
+ iring->pipe,
+ 0);
reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);
gfx_v8_0_ring_set_pipe_percent(iring, reserve);
}
for (i = 0; i < adev->gfx.num_compute_rings; ++i) {
iring = &adev->gfx.compute_ring[i];
- pipe = amdgpu_gfx_queue_to_bit(adev,
- iring->me,
- iring->pipe,
- 0);
+ pipe = amdgpu_gfx_mec_queue_to_bit(adev,
+ iring->me,
+ iring->pipe,
+ 0);
reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);
gfx_v8_0_ring_set_pipe_percent(iring, reserve);
}
@@ -6537,7 +6585,7 @@ static int gfx_v8_0_set_eop_interrupt_state(struct amdgpu_device *adev,
enum amdgpu_interrupt_state state)
{
switch (type) {
- case AMDGPU_CP_IRQ_GFX_EOP:
+ case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
gfx_v8_0_set_gfx_eop_interrupt_state(adev, state);
break;
case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
@@ -7010,30 +7058,10 @@ static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev)
static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev)
{
/* init asci gds info */
- adev->gds.mem.total_size = RREG32(mmGDS_VMID0_SIZE);
- adev->gds.gws.total_size = 64;
- adev->gds.oa.total_size = 16;
+ adev->gds.gds_size = RREG32(mmGDS_VMID0_SIZE);
+ adev->gds.gws_size = 64;
+ adev->gds.oa_size = 16;
adev->gds.gds_compute_max_wave_id = RREG32(mmGDS_COMPUTE_MAX_WAVE_ID);
-
- if (adev->gds.mem.total_size == 64 * 1024) {
- adev->gds.mem.gfx_partition_size = 4096;
- adev->gds.mem.cs_partition_size = 4096;
-
- adev->gds.gws.gfx_partition_size = 4;
- adev->gds.gws.cs_partition_size = 4;
-
- adev->gds.oa.gfx_partition_size = 4;
- adev->gds.oa.cs_partition_size = 1;
- } else {
- adev->gds.mem.gfx_partition_size = 1024;
- adev->gds.mem.cs_partition_size = 1024;
-
- adev->gds.gws.gfx_partition_size = 16;
- adev->gds.gws.cs_partition_size = 16;
-
- adev->gds.oa.gfx_partition_size = 4;
- adev->gds.oa.cs_partition_size = 4;
- }
}
static void gfx_v8_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 2f18c64d531f..c066e1d3f981 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -20,9 +20,13 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
+
+#include <linux/delay.h>
#include <linux/kernel.h>
#include <linux/firmware.h>
-#include <drm/drmP.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
#include "amdgpu.h"
#include "amdgpu_gfx.h"
#include "soc15.h"
@@ -35,6 +39,7 @@
#include "vega10_enum.h"
#include "hdp/hdp_4_0_offset.h"
+#include "soc15.h"
#include "soc15_common.h"
#include "clearstate_gfx9.h"
#include "v9_structs.h"
@@ -304,17 +309,20 @@ static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance);
static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
+static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
{
switch (adev->asic_type) {
case CHIP_VEGA10:
- soc15_program_register_sequence(adev,
- golden_settings_gc_9_0,
- ARRAY_SIZE(golden_settings_gc_9_0));
- soc15_program_register_sequence(adev,
- golden_settings_gc_9_0_vg10,
- ARRAY_SIZE(golden_settings_gc_9_0_vg10));
+ if (!amdgpu_virt_support_skip_setting(adev)) {
+ soc15_program_register_sequence(adev,
+ golden_settings_gc_9_0,
+ ARRAY_SIZE(golden_settings_gc_9_0));
+ soc15_program_register_sequence(adev,
+ golden_settings_gc_9_0_vg10,
+ ARRAY_SIZE(golden_settings_gc_9_0_vg10));
+ }
break;
case CHIP_VEGA12:
soc15_program_register_sequence(adev,
@@ -419,7 +427,7 @@ static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
tmp = RREG32(scratch);
if (tmp == 0xDEADBEEF)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
@@ -588,14 +596,18 @@ static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
case CHIP_VEGA20:
break;
case CHIP_RAVEN:
- if (adev->rev_id >= 0x8 || adev->pdev->device == 0x15d8)
- break;
- if ((adev->gfx.rlc_fw_version != 106 &&
- adev->gfx.rlc_fw_version < 531) ||
- (adev->gfx.rlc_fw_version == 53815) ||
- (adev->gfx.rlc_feature_version < 1) ||
- !adev->gfx.rlc.is_rlc_v2_1)
+ if (!(adev->rev_id >= 0x8 || adev->pdev->device == 0x15d8)
+ &&((adev->gfx.rlc_fw_version != 106 &&
+ adev->gfx.rlc_fw_version < 531) ||
+ (adev->gfx.rlc_fw_version == 53815) ||
+ (adev->gfx.rlc_feature_version < 1) ||
+ !adev->gfx.rlc.is_rlc_v2_1))
adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
+
+ if (adev->pm.pp_feature & PP_GFXOFF_MASK)
+ adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
+ AMD_PG_SUPPORT_CP |
+ AMD_PG_SUPPORT_RLC_SMU_HS;
break;
default:
break;
@@ -1305,9 +1317,9 @@ static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
}
static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
- u32 me, u32 pipe, u32 q)
+ u32 me, u32 pipe, u32 q, u32 vm)
{
- soc15_grbm_select(adev, me, pipe, q, 0);
+ soc15_grbm_select(adev, me, pipe, q, vm);
}
static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
@@ -1468,8 +1480,7 @@ static int gfx_v9_0_ngg_init(struct amdgpu_device *adev)
/* GDS reserve memory: 64 bytes alignment */
adev->gfx.ngg.gds_reserve_size = ALIGN(5 * 4, 0x40);
- adev->gds.mem.total_size -= adev->gfx.ngg.gds_reserve_size;
- adev->gds.mem.gfx_partition_size -= adev->gfx.ngg.gds_reserve_size;
+ adev->gds.gds_size -= adev->gfx.ngg.gds_reserve_size;
adev->gfx.ngg.gds_reserve_addr = RREG32_SOC15(GC, 0, mmGDS_VMID0_BASE);
adev->gfx.ngg.gds_reserve_addr += RREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE);
@@ -1577,7 +1588,7 @@ static int gfx_v9_0_ngg_en(struct amdgpu_device *adev)
gfx_v9_0_write_data_to_reg(ring, 0, false,
SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE),
- (adev->gds.mem.total_size +
+ (adev->gds.gds_size +
adev->gfx.ngg.gds_reserve_size));
amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
@@ -1718,7 +1729,7 @@ static int gfx_v9_0_sw_init(void *handle)
ring->use_doorbell = true;
ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
r = amdgpu_ring_init(adev, ring, 1024,
- &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_EOP);
+ &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP);
if (r)
return r;
}
@@ -1754,7 +1765,7 @@ static int gfx_v9_0_sw_init(void *handle)
return r;
/* create MQD for all compute queues as wel as KIQ for SRIOV case */
- r = amdgpu_gfx_compute_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation));
+ r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation));
if (r)
return r;
@@ -1791,24 +1802,18 @@ static int gfx_v9_0_sw_fini(void *handle)
kfree(ras_if);
}
- amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL);
- amdgpu_bo_free_kernel(&adev->gds.gws_gfx_bo, NULL, NULL);
- amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL);
-
for (i = 0; i < adev->gfx.num_gfx_rings; i++)
amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
for (i = 0; i < adev->gfx.num_compute_rings; i++)
amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
- amdgpu_gfx_compute_mqd_sw_fini(adev);
+ amdgpu_gfx_mqd_sw_fini(adev);
amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
amdgpu_gfx_kiq_fini(adev);
gfx_v9_0_mec_fini(adev);
gfx_v9_0_ngg_fini(adev);
- amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
- &adev->gfx.rlc.clear_state_gpu_addr,
- (void **)&adev->gfx.rlc.cs_ptr);
+ amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
if (adev->asic_type == CHIP_RAVEN) {
amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
&adev->gfx.rlc.cp_table_gpu_addr,
@@ -1844,7 +1849,7 @@ static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh
else
data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
- WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, data);
+ WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
}
static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
@@ -1912,11 +1917,20 @@ static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
soc15_grbm_select(adev, 0, 0, 0, i);
/* CP and shaders */
- WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
- WREG32_SOC15(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
+ WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
+ WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
}
soc15_grbm_select(adev, 0, 0, 0, 0);
mutex_unlock(&adev->srbm_mutex);
+
+ /* Initialize all compute VMIDs to have no GDS, GWS, or OA
+ acccess. These should be enabled by FW for target VMIDs. */
+ for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
+ WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
+ WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
+ WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
+ WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
+ }
}
static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
@@ -1924,7 +1938,7 @@ static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
u32 tmp;
int i;
- WREG32_FIELD15(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
+ WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
gfx_v9_0_tiling_mode_table_init(adev);
@@ -1941,17 +1955,21 @@ static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
if (i == 0) {
tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
SH_MEM_ALIGNMENT_MODE_UNALIGNED);
- WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, tmp);
- WREG32_SOC15(GC, 0, mmSH_MEM_BASES, 0);
+ tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
+ !!amdgpu_noretry);
+ WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
+ WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
} else {
tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
SH_MEM_ALIGNMENT_MODE_UNALIGNED);
- WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, tmp);
+ tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
+ !!amdgpu_noretry);
+ WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
(adev->gmc.private_aperture_start >> 48));
tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
(adev->gmc.shared_aperture_start >> 48));
- WREG32_SOC15(GC, 0, mmSH_MEM_BASES, tmp);
+ WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
}
}
soc15_grbm_select(adev, 0, 0, 0, 0);
@@ -2015,11 +2033,11 @@ static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
{
/* csib */
- WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
+ WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
adev->gfx.rlc.clear_state_gpu_addr >> 32);
- WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
+ WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
- WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
+ WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
adev->gfx.rlc.clear_state_size);
}
@@ -2073,11 +2091,10 @@ static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
u32 tmp = 0;
u32 *register_list_format =
- kmalloc(adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
+ kmemdup(adev->gfx.rlc.register_list_format,
+ adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
if (!register_list_format)
return -ENOMEM;
- memcpy(register_list_format, adev->gfx.rlc.register_list_format,
- adev->gfx.rlc.reg_list_format_size_bytes);
/* setup unique_indirect_regs array and indirect_start_offsets array */
unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
@@ -2489,7 +2506,7 @@ static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
for (i = 0; i < adev->gfx.num_gfx_rings; i++)
adev->gfx.gfx_ring[i].sched.ready = false;
}
- WREG32_SOC15(GC, 0, mmCP_ME_CNTL, tmp);
+ WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
udelay(50);
}
@@ -2687,9 +2704,9 @@ static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
int i;
if (enable) {
- WREG32_SOC15(GC, 0, mmCP_MEC_CNTL, 0);
+ WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
} else {
- WREG32_SOC15(GC, 0, mmCP_MEC_CNTL,
+ WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
(CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
for (i = 0; i < adev->gfx.num_compute_rings; i++)
adev->gfx.compute_ring[i].sched.ready = false;
@@ -2750,9 +2767,9 @@ static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
tmp &= 0xffffff00;
tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
- WREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
+ WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
tmp |= 0x80;
- WREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
+ WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
}
static int gfx_v9_0_kiq_kcq_enable(struct amdgpu_device *adev)
@@ -2970,67 +2987,67 @@ static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
/* disable wptr polling */
WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
- WREG32_SOC15(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
mqd->cp_hqd_eop_base_addr_lo);
- WREG32_SOC15(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
mqd->cp_hqd_eop_base_addr_hi);
/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
- WREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
mqd->cp_hqd_eop_control);
/* enable doorbell? */
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
mqd->cp_hqd_pq_doorbell_control);
/* disable the queue if it's active */
if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
- WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
for (j = 0; j < adev->usec_timeout; j++) {
if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
break;
udelay(1);
}
- WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
mqd->cp_hqd_dequeue_request);
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
mqd->cp_hqd_pq_rptr);
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_LO,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
mqd->cp_hqd_pq_wptr_lo);
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_HI,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
mqd->cp_hqd_pq_wptr_hi);
}
/* set the pointer to the MQD */
- WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR,
+ WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
mqd->cp_mqd_base_addr_lo);
- WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR_HI,
+ WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
mqd->cp_mqd_base_addr_hi);
/* set MQD vmid to 0 */
- WREG32_SOC15(GC, 0, mmCP_MQD_CONTROL,
+ WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
mqd->cp_mqd_control);
/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
mqd->cp_hqd_pq_base_lo);
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE_HI,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
mqd->cp_hqd_pq_base_hi);
/* set up the HQD, this is similar to CP_RB0_CNTL */
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
mqd->cp_hqd_pq_control);
/* set the wb address whether it's enabled or not */
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
mqd->cp_hqd_pq_rptr_report_addr_lo);
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
mqd->cp_hqd_pq_rptr_report_addr_hi);
/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
mqd->cp_hqd_pq_wptr_poll_addr_lo);
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
mqd->cp_hqd_pq_wptr_poll_addr_hi);
/* enable the doorbell if requested */
@@ -3041,23 +3058,23 @@ static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
(adev->doorbell_index.userqueue_end * 2) << 2);
}
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
mqd->cp_hqd_pq_doorbell_control);
/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_LO,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
mqd->cp_hqd_pq_wptr_lo);
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_HI,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
mqd->cp_hqd_pq_wptr_hi);
/* set the vmid for the queue */
- WREG32_SOC15(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
- WREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
mqd->cp_hqd_persistent_state);
/* activate the queue */
- WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
mqd->cp_hqd_active);
if (ring->use_doorbell)
@@ -3074,7 +3091,7 @@ static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
/* disable the queue if it's active */
if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
- WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
for (j = 0; j < adev->usec_timeout; j++) {
if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
@@ -3086,21 +3103,21 @@ static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
DRM_DEBUG("KIQ dequeue request failed.\n");
/* Manual disable if dequeue request times out */
- WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE, 0);
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
}
- WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
0);
}
- WREG32_SOC15(GC, 0, mmCP_HQD_IQ_TIMER, 0);
- WREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL, 0);
- WREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR, 0);
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
- WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
return 0;
}
@@ -3520,6 +3537,279 @@ static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
(1 << (oa_size + oa_base)) - (1 << oa_base));
}
+static const u32 vgpr_init_compute_shader[] =
+{
+ 0xb07c0000, 0xbe8000ff,
+ 0x000000f8, 0xbf110800,
+ 0x7e000280, 0x7e020280,
+ 0x7e040280, 0x7e060280,
+ 0x7e080280, 0x7e0a0280,
+ 0x7e0c0280, 0x7e0e0280,
+ 0x80808800, 0xbe803200,
+ 0xbf84fff5, 0xbf9c0000,
+ 0xd28c0001, 0x0001007f,
+ 0xd28d0001, 0x0002027e,
+ 0x10020288, 0xb8810904,
+ 0xb7814000, 0xd1196a01,
+ 0x00000301, 0xbe800087,
+ 0xbefc00c1, 0xd89c4000,
+ 0x00020201, 0xd89cc080,
+ 0x00040401, 0x320202ff,
+ 0x00000800, 0x80808100,
+ 0xbf84fff8, 0x7e020280,
+ 0xbf810000, 0x00000000,
+};
+
+static const u32 sgpr_init_compute_shader[] =
+{
+ 0xb07c0000, 0xbe8000ff,
+ 0x0000005f, 0xbee50080,
+ 0xbe812c65, 0xbe822c65,
+ 0xbe832c65, 0xbe842c65,
+ 0xbe852c65, 0xb77c0005,
+ 0x80808500, 0xbf84fff8,
+ 0xbe800080, 0xbf810000,
+};
+
+static const struct soc15_reg_entry vgpr_init_regs[] = {
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x1000000 }, /* CU_GROUP_COUNT=1 */
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 256*2 },
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 1 },
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x100007f }, /* VGPRS=15 (256 logical VGPRs, SGPRS=1 (16 SGPRs, BULKY=1 */
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */
+};
+
+static const struct soc15_reg_entry sgpr_init_regs[] = {
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x1000000 }, /* CU_GROUP_COUNT=1 */
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 256*2 },
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 1 },
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x340 }, /* SGPRS=13 (112 GPRS) */
+ { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
+};
+
+static const struct soc15_reg_entry sec_ded_counter_registers[] = {
+ { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
+ { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
+ { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
+ { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
+ { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
+ { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
+ { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
+ { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
+ { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
+ { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
+ { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
+ { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
+ { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
+ { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
+ { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
+ { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
+ { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
+ { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
+ { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
+ { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
+ { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
+ { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
+ { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
+ { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
+ { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
+ { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
+ { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
+ { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
+ { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
+ { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
+ { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
+ { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
+};
+
+static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
+{
+ struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
+ int i, r;
+
+ r = amdgpu_ring_alloc(ring, 7);
+ if (r) {
+ DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
+ ring->name, r);
+ return r;
+ }
+
+ WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
+ WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
+
+ amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
+ amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
+ PACKET3_DMA_DATA_DST_SEL(1) |
+ PACKET3_DMA_DATA_SRC_SEL(2) |
+ PACKET3_DMA_DATA_ENGINE(0)));
+ amdgpu_ring_write(ring, 0);
+ amdgpu_ring_write(ring, 0);
+ amdgpu_ring_write(ring, 0);
+ amdgpu_ring_write(ring, 0);
+ amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
+ adev->gds.gds_size);
+
+ amdgpu_ring_commit(ring);
+
+ for (i = 0; i < adev->usec_timeout; i++) {
+ if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
+ break;
+ udelay(1);
+ }
+
+ if (i >= adev->usec_timeout)
+ r = -ETIMEDOUT;
+
+ WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
+
+ return r;
+}
+
+static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
+{
+ struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
+ struct amdgpu_ib ib;
+ struct dma_fence *f = NULL;
+ int r, i, j, k;
+ unsigned total_size, vgpr_offset, sgpr_offset;
+ u64 gpu_addr;
+
+ /* only support when RAS is enabled */
+ if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
+ return 0;
+
+ /* bail if the compute ring is not ready */
+ if (!ring->sched.ready)
+ return 0;
+
+ total_size =
+ ((ARRAY_SIZE(vgpr_init_regs) * 3) + 4 + 5 + 2) * 4;
+ total_size +=
+ ((ARRAY_SIZE(sgpr_init_regs) * 3) + 4 + 5 + 2) * 4;
+ total_size = ALIGN(total_size, 256);
+ vgpr_offset = total_size;
+ total_size += ALIGN(sizeof(vgpr_init_compute_shader), 256);
+ sgpr_offset = total_size;
+ total_size += sizeof(sgpr_init_compute_shader);
+
+ /* allocate an indirect buffer to put the commands in */
+ memset(&ib, 0, sizeof(ib));
+ r = amdgpu_ib_get(adev, NULL, total_size, &ib);
+ if (r) {
+ DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
+ return r;
+ }
+
+ /* load the compute shaders */
+ for (i = 0; i < ARRAY_SIZE(vgpr_init_compute_shader); i++)
+ ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_compute_shader[i];
+
+ for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
+ ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
+
+ /* init the ib length to 0 */
+ ib.length_dw = 0;
+
+ /* VGPR */
+ /* write the register state for the compute dispatch */
+ for (i = 0; i < ARRAY_SIZE(vgpr_init_regs); i++) {
+ ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
+ ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs[i])
+ - PACKET3_SET_SH_REG_START;
+ ib.ptr[ib.length_dw++] = vgpr_init_regs[i].reg_value;
+ }
+ /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
+ gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
+ ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
+ ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
+ - PACKET3_SET_SH_REG_START;
+ ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
+ ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
+
+ /* write dispatch packet */
+ ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
+ ib.ptr[ib.length_dw++] = 128; /* x */
+ ib.ptr[ib.length_dw++] = 1; /* y */
+ ib.ptr[ib.length_dw++] = 1; /* z */
+ ib.ptr[ib.length_dw++] =
+ REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
+
+ /* write CS partial flush packet */
+ ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
+ ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
+
+ /* SGPR */
+ /* write the register state for the compute dispatch */
+ for (i = 0; i < ARRAY_SIZE(sgpr_init_regs); i++) {
+ ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
+ ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr_init_regs[i])
+ - PACKET3_SET_SH_REG_START;
+ ib.ptr[ib.length_dw++] = sgpr_init_regs[i].reg_value;
+ }
+ /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
+ gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
+ ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
+ ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
+ - PACKET3_SET_SH_REG_START;
+ ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
+ ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
+
+ /* write dispatch packet */
+ ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
+ ib.ptr[ib.length_dw++] = 128; /* x */
+ ib.ptr[ib.length_dw++] = 1; /* y */
+ ib.ptr[ib.length_dw++] = 1; /* z */
+ ib.ptr[ib.length_dw++] =
+ REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
+
+ /* write CS partial flush packet */
+ ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
+ ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
+
+ /* shedule the ib on the ring */
+ r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
+ if (r) {
+ DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
+ goto fail;
+ }
+
+ /* wait for the GPU to finish processing the IB */
+ r = dma_fence_wait(f, false);
+ if (r) {
+ DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
+ goto fail;
+ }
+
+ /* read back registers to clear the counters */
+ mutex_lock(&adev->grbm_idx_mutex);
+ for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++) {
+ for (j = 0; j < sec_ded_counter_registers[i].se_num; j++) {
+ for (k = 0; k < sec_ded_counter_registers[i].instance; k++) {
+ gfx_v9_0_select_se_sh(adev, j, 0x0, k);
+ RREG32(SOC15_REG_ENTRY_OFFSET(sec_ded_counter_registers[i]));
+ }
+ }
+ }
+ WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
+ mutex_unlock(&adev->grbm_idx_mutex);
+
+fail:
+ amdgpu_ib_free(adev, &ib, NULL);
+ dma_fence_put(f);
+
+ return r;
+}
+
static int gfx_v9_0_early_init(void *handle)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -3561,8 +3851,35 @@ static int gfx_v9_0_ecc_late_init(void *handle)
return 0;
}
- if (*ras_if)
+ r = gfx_v9_0_do_edc_gds_workarounds(adev);
+ if (r)
+ return r;
+
+ /* requires IBs so do in late init after IB pool is initialized */
+ r = gfx_v9_0_do_edc_gpr_workarounds(adev);
+ if (r)
+ return r;
+
+ /* handle resume path. */
+ if (*ras_if) {
+ /* resend ras TA enable cmd during resume.
+ * prepare to handle failure.
+ */
+ ih_info.head = **ras_if;
+ r = amdgpu_ras_feature_enable_on_boot(adev, *ras_if, 1);
+ if (r) {
+ if (r == -EAGAIN) {
+ /* request a gpu reset. will run again. */
+ amdgpu_ras_request_reset_on_boot(adev,
+ AMDGPU_RAS_BLOCK__GFX);
+ return 0;
+ }
+ /* fail to enable ras, cleanup all. */
+ goto irq;
+ }
+ /* enable successfully. continue. */
goto resume;
+ }
*ras_if = kmalloc(sizeof(**ras_if), GFP_KERNEL);
if (!*ras_if)
@@ -3571,8 +3888,14 @@ static int gfx_v9_0_ecc_late_init(void *handle)
**ras_if = ras_block;
r = amdgpu_ras_feature_enable_on_boot(adev, *ras_if, 1);
- if (r)
+ if (r) {
+ if (r == -EAGAIN) {
+ amdgpu_ras_request_reset_on_boot(adev,
+ AMDGPU_RAS_BLOCK__GFX);
+ r = 0;
+ }
goto feature;
+ }
ih_info.head = **ras_if;
fs_info.head = **ras_if;
@@ -3581,9 +3904,7 @@ static int gfx_v9_0_ecc_late_init(void *handle)
if (r)
goto interrupt;
- r = amdgpu_ras_debugfs_create(adev, &fs_info);
- if (r)
- goto debugfs;
+ amdgpu_ras_debugfs_create(adev, &fs_info);
r = amdgpu_ras_sysfs_create(adev, &fs_info);
if (r)
@@ -3598,14 +3919,13 @@ irq:
amdgpu_ras_sysfs_remove(adev, *ras_if);
sysfs:
amdgpu_ras_debugfs_remove(adev, *ras_if);
-debugfs:
amdgpu_ras_interrupt_remove_handler(adev, &ih_info);
interrupt:
amdgpu_ras_feature_enable(adev, *ras_if, 0);
feature:
kfree(*ras_if);
*ras_if = NULL;
- return -EINVAL;
+ return r;
}
static int gfx_v9_0_late_init(void *handle)
@@ -4259,7 +4579,7 @@ static void gfx_v9_0_pipe_reserve_resources(struct amdgpu_device *adev,
struct amdgpu_ring *iring;
mutex_lock(&adev->gfx.pipe_reserve_mutex);
- pipe = amdgpu_gfx_queue_to_bit(adev, ring->me, ring->pipe, 0);
+ pipe = amdgpu_gfx_mec_queue_to_bit(adev, ring->me, ring->pipe, 0);
if (acquire)
set_bit(pipe, adev->gfx.pipe_reserve_bitmap);
else
@@ -4278,20 +4598,20 @@ static void gfx_v9_0_pipe_reserve_resources(struct amdgpu_device *adev,
/* Lower all pipes without a current reservation */
for (i = 0; i < adev->gfx.num_gfx_rings; ++i) {
iring = &adev->gfx.gfx_ring[i];
- pipe = amdgpu_gfx_queue_to_bit(adev,
- iring->me,
- iring->pipe,
- 0);
+ pipe = amdgpu_gfx_mec_queue_to_bit(adev,
+ iring->me,
+ iring->pipe,
+ 0);
reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);
gfx_v9_0_ring_set_pipe_percent(iring, reserve);
}
for (i = 0; i < adev->gfx.num_compute_rings; ++i) {
iring = &adev->gfx.compute_ring[i];
- pipe = amdgpu_gfx_queue_to_bit(adev,
- iring->me,
- iring->pipe,
- 0);
+ pipe = amdgpu_gfx_mec_queue_to_bit(adev,
+ iring->me,
+ iring->pipe,
+ 0);
reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);
gfx_v9_0_ring_set_pipe_percent(iring, reserve);
}
@@ -4310,8 +4630,8 @@ static void gfx_v9_0_hqd_set_priority(struct amdgpu_device *adev,
mutex_lock(&adev->srbm_mutex);
soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
- WREG32_SOC15(GC, 0, mmCP_HQD_PIPE_PRIORITY, pipe_priority);
- WREG32_SOC15(GC, 0, mmCP_HQD_QUEUE_PRIORITY, queue_priority);
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PIPE_PRIORITY, pipe_priority);
+ WREG32_SOC15_RLC(GC, 0, mmCP_HQD_QUEUE_PRIORITY, queue_priority);
soc15_grbm_select(adev, 0, 0, 0, 0);
mutex_unlock(&adev->srbm_mutex);
@@ -4553,7 +4873,7 @@ static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
- WREG32(mmSQ_CMD, value);
+ WREG32_SOC15(GC, 0, mmSQ_CMD, value);
}
static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
@@ -4706,7 +5026,7 @@ static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
enum amdgpu_interrupt_state state)
{
switch (type) {
- case AMDGPU_CP_IRQ_GFX_EOP:
+ case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
break;
case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
@@ -5047,13 +5367,13 @@ static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
case CHIP_VEGA10:
case CHIP_VEGA12:
case CHIP_VEGA20:
- adev->gds.mem.total_size = 0x10000;
+ adev->gds.gds_size = 0x10000;
break;
case CHIP_RAVEN:
- adev->gds.mem.total_size = 0x1000;
+ adev->gds.gds_size = 0x1000;
break;
default:
- adev->gds.mem.total_size = 0x10000;
+ adev->gds.gds_size = 0x10000;
break;
}
@@ -5077,28 +5397,8 @@ static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
break;
}
- adev->gds.gws.total_size = 64;
- adev->gds.oa.total_size = 16;
-
- if (adev->gds.mem.total_size == 64 * 1024) {
- adev->gds.mem.gfx_partition_size = 4096;
- adev->gds.mem.cs_partition_size = 4096;
-
- adev->gds.gws.gfx_partition_size = 4;
- adev->gds.gws.cs_partition_size = 4;
-
- adev->gds.oa.gfx_partition_size = 4;
- adev->gds.oa.cs_partition_size = 1;
- } else {
- adev->gds.mem.gfx_partition_size = 1024;
- adev->gds.mem.cs_partition_size = 1024;
-
- adev->gds.gws.gfx_partition_size = 16;
- adev->gds.gws.cs_partition_size = 16;
-
- adev->gds.oa.gfx_partition_size = 4;
- adev->gds.oa.cs_partition_size = 4;
- }
+ adev->gds.gws_size = 64;
+ adev->gds.oa_size = 16;
}
static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
index 7bb5359d0bbd..15986748f59f 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
@@ -71,12 +71,12 @@ static void gfxhub_v1_0_init_system_aperture_regs(struct amdgpu_device *adev)
uint64_t value;
/* Program the AGP BAR */
- WREG32_SOC15(GC, 0, mmMC_VM_AGP_BASE, 0);
- WREG32_SOC15(GC, 0, mmMC_VM_AGP_BOT, adev->gmc.agp_start >> 24);
- WREG32_SOC15(GC, 0, mmMC_VM_AGP_TOP, adev->gmc.agp_end >> 24);
+ WREG32_SOC15_RLC(GC, 0, mmMC_VM_AGP_BASE, 0);
+ WREG32_SOC15_RLC(GC, 0, mmMC_VM_AGP_BOT, adev->gmc.agp_start >> 24);
+ WREG32_SOC15_RLC(GC, 0, mmMC_VM_AGP_TOP, adev->gmc.agp_end >> 24);
/* Program the system aperture low logical page number. */
- WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
+ WREG32_SOC15_RLC(GC, 0, mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18);
if (adev->asic_type == CHIP_RAVEN && adev->rev_id >= 0x8)
@@ -86,11 +86,11 @@ static void gfxhub_v1_0_init_system_aperture_regs(struct amdgpu_device *adev)
* workaround that increase system aperture high address (add 1)
* to get rid of the VM fault and hardware hang.
*/
- WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
+ WREG32_SOC15_RLC(GC, 0, mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
max((adev->gmc.fb_end >> 18) + 0x1,
adev->gmc.agp_end >> 18));
else
- WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
+ WREG32_SOC15_RLC(GC, 0, mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18);
/* Set default page address. */
@@ -129,7 +129,7 @@ static void gfxhub_v1_0_init_tlb_regs(struct amdgpu_device *adev)
MTYPE, MTYPE_UC);/* XXX for emulation. */
tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1);
- WREG32_SOC15(GC, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp);
+ WREG32_SOC15_RLC(GC, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp);
}
static void gfxhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
@@ -146,12 +146,12 @@ static void gfxhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 0);
tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
- WREG32_SOC15(GC, 0, mmVM_L2_CNTL, tmp);
+ WREG32_SOC15_RLC(GC, 0, mmVM_L2_CNTL, tmp);
tmp = RREG32_SOC15(GC, 0, mmVM_L2_CNTL2);
tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
- WREG32_SOC15(GC, 0, mmVM_L2_CNTL2, tmp);
+ WREG32_SOC15_RLC(GC, 0, mmVM_L2_CNTL2, tmp);
tmp = mmVM_L2_CNTL3_DEFAULT;
if (adev->gmc.translate_further) {
@@ -163,12 +163,12 @@ static void gfxhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3,
L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
}
- WREG32_SOC15(GC, 0, mmVM_L2_CNTL3, tmp);
+ WREG32_SOC15_RLC(GC, 0, mmVM_L2_CNTL3, tmp);
tmp = mmVM_L2_CNTL4_DEFAULT;
tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
- WREG32_SOC15(GC, 0, mmVM_L2_CNTL4, tmp);
+ WREG32_SOC15_RLC(GC, 0, mmVM_L2_CNTL4, tmp);
}
static void gfxhub_v1_0_enable_system_domain(struct amdgpu_device *adev)
@@ -236,7 +236,8 @@ static void gfxhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
block_size);
/* Send no-retry XNACK on fault to suppress VM fault storm. */
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
- RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 1);
+ RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
+ !amdgpu_noretry);
WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_CNTL, i, tmp);
WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, i*2, 0);
WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, i*2, 0);
@@ -267,9 +268,9 @@ int gfxhub_v1_0_gart_enable(struct amdgpu_device *adev)
* VF copy registers so vbios post doesn't program them, for
* SRIOV driver need to program them
*/
- WREG32_SOC15(GC, 0, mmMC_VM_FB_LOCATION_BASE,
+ WREG32_SOC15_RLC(GC, 0, mmMC_VM_FB_LOCATION_BASE,
adev->gmc.vram_start >> 24);
- WREG32_SOC15(GC, 0, mmMC_VM_FB_LOCATION_TOP,
+ WREG32_SOC15_RLC(GC, 0, mmMC_VM_FB_LOCATION_TOP,
adev->gmc.vram_end >> 24);
}
@@ -303,7 +304,7 @@ void gfxhub_v1_0_gart_disable(struct amdgpu_device *adev)
MC_VM_MX_L1_TLB_CNTL,
ENABLE_ADVANCED_DRIVER_MODEL,
0);
- WREG32_SOC15(GC, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp);
+ WREG32_SOC15_RLC(GC, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp);
/* Setup L2 cache */
WREG32_FIELD15(GC, 0, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
new file mode 100644
index 000000000000..d605b4963f8a
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
@@ -0,0 +1,354 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "amdgpu.h"
+#include "gfxhub_v2_0.h"
+
+#include "gc/gc_10_1_0_offset.h"
+#include "gc/gc_10_1_0_sh_mask.h"
+#include "gc/gc_10_1_0_default.h"
+#include "navi10_enum.h"
+
+#include "soc15_common.h"
+
+u64 gfxhub_v2_0_get_fb_location(struct amdgpu_device *adev)
+{
+ u64 base = RREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_BASE);
+
+ base &= GCMC_VM_FB_LOCATION_BASE__FB_BASE_MASK;
+ base <<= 24;
+
+ return base;
+}
+
+u64 gfxhub_v2_0_get_mc_fb_offset(struct amdgpu_device *adev)
+{
+ return (u64)RREG32_SOC15(GC, 0, mmGCMC_VM_FB_OFFSET) << 24;
+}
+
+static void gfxhub_v2_0_init_gart_pt_regs(struct amdgpu_device *adev)
+{
+ uint64_t value = amdgpu_gmc_pd_addr(adev->gart.bo);
+
+
+ WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
+ lower_32_bits(value));
+
+ WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
+ upper_32_bits(value));
+}
+
+static void gfxhub_v2_0_init_gart_aperture_regs(struct amdgpu_device *adev)
+{
+ gfxhub_v2_0_init_gart_pt_regs(adev);
+
+ WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
+ (u32)(adev->gmc.gart_start >> 12));
+ WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
+ (u32)(adev->gmc.gart_start >> 44));
+
+ WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
+ (u32)(adev->gmc.gart_end >> 12));
+ WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
+ (u32)(adev->gmc.gart_end >> 44));
+}
+
+static void gfxhub_v2_0_init_system_aperture_regs(struct amdgpu_device *adev)
+{
+ uint64_t value;
+
+ /* Disable AGP. */
+ WREG32_SOC15(GC, 0, mmGCMC_VM_AGP_BASE, 0);
+ WREG32_SOC15(GC, 0, mmGCMC_VM_AGP_TOP, 0);
+ WREG32_SOC15(GC, 0, mmGCMC_VM_AGP_BOT, 0x00FFFFFF);
+
+ /* Program the system aperture low logical page number. */
+ WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_LOW_ADDR,
+ adev->gmc.vram_start >> 18);
+ WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
+ adev->gmc.vram_end >> 18);
+
+ /* Set default page address. */
+ value = adev->vram_scratch.gpu_addr - adev->gmc.vram_start
+ + adev->vm_manager.vram_base_offset;
+ WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
+ (u32)(value >> 12));
+ WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
+ (u32)(value >> 44));
+
+ /* Program "protection fault". */
+ WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
+ (u32)(adev->dummy_page_addr >> 12));
+ WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
+ (u32)((u64)adev->dummy_page_addr >> 44));
+
+ WREG32_FIELD15(GC, 0, GCVM_L2_PROTECTION_FAULT_CNTL2,
+ ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+}
+
+
+static void gfxhub_v2_0_init_tlb_regs(struct amdgpu_device *adev)
+{
+ uint32_t tmp;
+
+ /* Setup TLB control */
+ tmp = RREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL);
+
+ tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
+ tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
+ tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
+ ENABLE_ADVANCED_DRIVER_MODEL, 1);
+ tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
+ SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
+ tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
+ tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
+ MTYPE, MTYPE_UC); /* UC, uncached */
+
+ WREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL, tmp);
+}
+
+static void gfxhub_v2_0_init_cache_regs(struct amdgpu_device *adev)
+{
+ uint32_t tmp;
+
+ /* Setup L2 cache */
+ tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_CNTL);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_CACHE, 1);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
+ ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
+ /* XXX for emulation, Refer to closed source code.*/
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
+ L2_PDE0_CACHE_TAG_GENERATION_MODE, 0);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 1);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
+ WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL, tmp);
+
+ tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_CNTL2);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
+ WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL2, tmp);
+
+ tmp = mmGCVM_L2_CNTL3_DEFAULT;
+ WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL3, tmp);
+
+ tmp = mmGCVM_L2_CNTL4_DEFAULT;
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
+ WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL4, tmp);
+}
+
+static void gfxhub_v2_0_enable_system_domain(struct amdgpu_device *adev)
+{
+ uint32_t tmp;
+
+ tmp = RREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_CNTL);
+ tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
+ tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
+ WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_CNTL, tmp);
+}
+
+static void gfxhub_v2_0_disable_identity_aperture(struct amdgpu_device *adev)
+{
+ WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
+ 0xFFFFFFFF);
+ WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
+ 0x0000000F);
+
+ WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32,
+ 0);
+ WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32,
+ 0);
+
+ WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0);
+ WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0);
+
+}
+
+static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev)
+{
+ int i;
+ uint32_t tmp;
+
+ for (i = 0; i <= 14; i++) {
+ tmp = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_CNTL, i);
+ tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
+ tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
+ adev->vm_manager.num_level);
+ tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+ RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+ tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+ DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+ tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+ PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+ tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+ VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+ tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+ READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+ tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+ WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+ tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+ EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+ tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+ PAGE_TABLE_BLOCK_SIZE,
+ adev->vm_manager.block_size - 9);
+ /* Send no-retry XNACK on fault to suppress VM fault storm. */
+ tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+ RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
+ !amdgpu_noretry);
+ WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_CNTL, i, tmp);
+ WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, i*2, 0);
+ WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, i*2, 0);
+ WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, i*2,
+ lower_32_bits(adev->vm_manager.max_pfn - 1));
+ WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, i*2,
+ upper_32_bits(adev->vm_manager.max_pfn - 1));
+ }
+}
+
+static void gfxhub_v2_0_program_invalidation(struct amdgpu_device *adev)
+{
+ unsigned i;
+
+ for (i = 0 ; i < 18; ++i) {
+ WREG32_SOC15_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
+ 2 * i, 0xffffffff);
+ WREG32_SOC15_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
+ 2 * i, 0x1f);
+ }
+}
+
+int gfxhub_v2_0_gart_enable(struct amdgpu_device *adev)
+{
+ if (amdgpu_sriov_vf(adev)) {
+ /*
+ * GCMC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are
+ * VF copy registers so vbios post doesn't program them, for
+ * SRIOV driver need to program them
+ */
+ WREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_BASE,
+ adev->gmc.vram_start >> 24);
+ WREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_TOP,
+ adev->gmc.vram_end >> 24);
+ }
+
+ /* GART Enable. */
+ gfxhub_v2_0_init_gart_aperture_regs(adev);
+ gfxhub_v2_0_init_system_aperture_regs(adev);
+ gfxhub_v2_0_init_tlb_regs(adev);
+ gfxhub_v2_0_init_cache_regs(adev);
+
+ gfxhub_v2_0_enable_system_domain(adev);
+ gfxhub_v2_0_disable_identity_aperture(adev);
+ gfxhub_v2_0_setup_vmid_config(adev);
+ gfxhub_v2_0_program_invalidation(adev);
+
+ return 0;
+}
+
+void gfxhub_v2_0_gart_disable(struct amdgpu_device *adev)
+{
+ u32 tmp;
+ u32 i;
+
+ /* Disable all tables */
+ for (i = 0; i < 16; i++)
+ WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_CNTL, i, 0);
+
+ /* Setup TLB control */
+ tmp = RREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL);
+ tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
+ tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
+ ENABLE_ADVANCED_DRIVER_MODEL, 0);
+ WREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL, tmp);
+
+ /* Setup L2 cache */
+ WREG32_FIELD15(GC, 0, GCVM_L2_CNTL, ENABLE_L2_CACHE, 0);
+ WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL3, 0);
+}
+
+/**
+ * gfxhub_v2_0_set_fault_enable_default - update GART/VM fault handling
+ *
+ * @adev: amdgpu_device pointer
+ * @value: true redirects VM faults to the default page
+ */
+void gfxhub_v2_0_set_fault_enable_default(struct amdgpu_device *adev,
+ bool value)
+{
+ u32 tmp;
+ tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+ RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+ PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+ PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+ PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+ TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
+ value);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+ NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+ DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+ VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+ READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+ WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+ EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ if (!value) {
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+ CRASH_ON_NO_RETRY_FAULT, 1);
+ tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+ CRASH_ON_RETRY_FAULT, 1);
+ }
+ WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL, tmp);
+}
+
+void gfxhub_v2_0_init(struct amdgpu_device *adev)
+{
+ struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB];
+
+ hub->ctx0_ptb_addr_lo32 =
+ SOC15_REG_OFFSET(GC, 0,
+ mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
+ hub->ctx0_ptb_addr_hi32 =
+ SOC15_REG_OFFSET(GC, 0,
+ mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
+ hub->vm_inv_eng0_req =
+ SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_REQ);
+ hub->vm_inv_eng0_ack =
+ SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ACK);
+ hub->vm_context0_cntl =
+ SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_CNTL);
+ hub->vm_l2_pro_fault_status =
+ SOC15_REG_OFFSET(GC, 0, mmGCVM_L2_PROTECTION_FAULT_STATUS);
+ hub->vm_l2_pro_fault_cntl =
+ SOC15_REG_OFFSET(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.h b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.h
new file mode 100644
index 000000000000..06807940748b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __GFXHUB_V2_0_H__
+#define __GFXHUB_V2_0_H__
+
+u64 gfxhub_v2_0_get_fb_location(struct amdgpu_device *adev);
+int gfxhub_v2_0_gart_enable(struct amdgpu_device *adev);
+void gfxhub_v2_0_gart_disable(struct amdgpu_device *adev);
+void gfxhub_v2_0_set_fault_enable_default(struct amdgpu_device *adev,
+ bool value);
+void gfxhub_v2_0_init(struct amdgpu_device *adev);
+u64 gfxhub_v2_0_get_mc_fb_offset(struct amdgpu_device *adev);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
new file mode 100644
index 000000000000..5eeb72fcc123
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
@@ -0,0 +1,918 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/firmware.h>
+#include <linux/pci.h>
+#include "amdgpu.h"
+#include "amdgpu_atomfirmware.h"
+#include "gmc_v10_0.h"
+
+#include "hdp/hdp_5_0_0_offset.h"
+#include "hdp/hdp_5_0_0_sh_mask.h"
+#include "gc/gc_10_1_0_sh_mask.h"
+#include "mmhub/mmhub_2_0_0_sh_mask.h"
+#include "dcn/dcn_2_0_0_offset.h"
+#include "dcn/dcn_2_0_0_sh_mask.h"
+#include "oss/osssys_5_0_0_offset.h"
+#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
+#include "navi10_enum.h"
+
+#include "soc15.h"
+#include "soc15_common.h"
+
+#include "nbio_v2_3.h"
+
+#include "gfxhub_v2_0.h"
+#include "mmhub_v2_0.h"
+#include "athub_v2_0.h"
+/* XXX Move this macro to navi10 header file, which is like vid.h for VI.*/
+#define AMDGPU_NUM_OF_VMIDS 8
+
+#if 0
+static const struct soc15_reg_golden golden_settings_navi10_hdp[] =
+{
+ /* TODO add golden setting for hdp */
+};
+#endif
+
+static int
+gmc_v10_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *src, unsigned type,
+ enum amdgpu_interrupt_state state)
+{
+ struct amdgpu_vmhub *hub;
+ u32 tmp, reg, bits[AMDGPU_MAX_VMHUBS], i;
+
+ bits[AMDGPU_GFXHUB] = GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+ GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+ GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+ GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+ GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+ GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+ GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
+
+ bits[AMDGPU_MMHUB] = MMVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+ MMVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+ MMVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+ MMVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+ MMVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+ MMVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+ MMVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
+
+ switch (state) {
+ case AMDGPU_IRQ_STATE_DISABLE:
+ /* MM HUB */
+ hub = &adev->vmhub[AMDGPU_MMHUB];
+ for (i = 0; i < 16; i++) {
+ reg = hub->vm_context0_cntl + i;
+ tmp = RREG32(reg);
+ tmp &= ~bits[AMDGPU_MMHUB];
+ WREG32(reg, tmp);
+ }
+
+ /* GFX HUB */
+ hub = &adev->vmhub[AMDGPU_GFXHUB];
+ for (i = 0; i < 16; i++) {
+ reg = hub->vm_context0_cntl + i;
+ tmp = RREG32(reg);
+ tmp &= ~bits[AMDGPU_GFXHUB];
+ WREG32(reg, tmp);
+ }
+ break;
+ case AMDGPU_IRQ_STATE_ENABLE:
+ /* MM HUB */
+ hub = &adev->vmhub[AMDGPU_MMHUB];
+ for (i = 0; i < 16; i++) {
+ reg = hub->vm_context0_cntl + i;
+ tmp = RREG32(reg);
+ tmp |= bits[AMDGPU_MMHUB];
+ WREG32(reg, tmp);
+ }
+
+ /* GFX HUB */
+ hub = &adev->vmhub[AMDGPU_GFXHUB];
+ for (i = 0; i < 16; i++) {
+ reg = hub->vm_context0_cntl + i;
+ tmp = RREG32(reg);
+ tmp |= bits[AMDGPU_GFXHUB];
+ WREG32(reg, tmp);
+ }
+ break;
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+static int gmc_v10_0_process_interrupt(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *source,
+ struct amdgpu_iv_entry *entry)
+{
+ struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src];
+ uint32_t status = 0;
+ u64 addr;
+
+ addr = (u64)entry->src_data[0] << 12;
+ addr |= ((u64)entry->src_data[1] & 0xf) << 44;
+
+ if (!amdgpu_sriov_vf(adev)) {
+ status = RREG32(hub->vm_l2_pro_fault_status);
+ WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
+ }
+
+ if (printk_ratelimit()) {
+ dev_err(adev->dev,
+ "[%s] VMC page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n",
+ entry->vmid_src ? "mmhub" : "gfxhub",
+ entry->src_id, entry->ring_id, entry->vmid,
+ entry->pasid);
+ dev_err(adev->dev, " at page 0x%016llx from %d\n",
+ addr, entry->client_id);
+ if (!amdgpu_sriov_vf(adev))
+ dev_err(adev->dev,
+ "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
+ status);
+ }
+
+ return 0;
+}
+
+static const struct amdgpu_irq_src_funcs gmc_v10_0_irq_funcs = {
+ .set = gmc_v10_0_vm_fault_interrupt_state,
+ .process = gmc_v10_0_process_interrupt,
+};
+
+static void gmc_v10_0_set_irq_funcs(struct amdgpu_device *adev)
+{
+ adev->gmc.vm_fault.num_types = 1;
+ adev->gmc.vm_fault.funcs = &gmc_v10_0_irq_funcs;
+}
+
+static uint32_t gmc_v10_0_get_invalidate_req(unsigned int vmid,
+ uint32_t flush_type)
+{
+ u32 req = 0;
+
+ /* invalidate using legacy mode on vmid*/
+ req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
+ PER_VMID_INVALIDATE_REQ, 1 << vmid);
+ req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
+ req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
+ req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
+ req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
+ req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
+ req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
+ req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
+ CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0);
+
+ return req;
+}
+
+/*
+ * GART
+ * VMID 0 is the physical GPU addresses as used by the kernel.
+ * VMIDs 1-15 are used for userspace clients and are handled
+ * by the amdgpu vm/hsa code.
+ */
+
+static void gmc_v10_0_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid,
+ unsigned int vmhub, uint32_t flush_type)
+{
+ struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
+ u32 tmp = gmc_v10_0_get_invalidate_req(vmid, flush_type);
+ /* Use register 17 for GART */
+ const unsigned eng = 17;
+ unsigned int i;
+
+ WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
+
+ /* Wait for ACK with a delay.*/
+ for (i = 0; i < adev->usec_timeout; i++) {
+ tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + eng);
+ tmp &= 1 << vmid;
+ if (tmp)
+ break;
+
+ udelay(1);
+ }
+
+ if (i < adev->usec_timeout)
+ return;
+
+ DRM_ERROR("Timeout waiting for VM flush ACK!\n");
+}
+
+/**
+ * gmc_v10_0_flush_gpu_tlb - gart tlb flush callback
+ *
+ * @adev: amdgpu_device pointer
+ * @vmid: vm instance to flush
+ *
+ * Flush the TLB for the requested page table.
+ */
+static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev,
+ uint32_t vmid, uint32_t flush_type)
+{
+ struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+ struct dma_fence *fence;
+ struct amdgpu_job *job;
+
+ int r;
+
+ /* flush hdp cache */
+ adev->nbio_funcs->hdp_flush(adev, NULL);
+
+ mutex_lock(&adev->mman.gtt_window_lock);
+
+ gmc_v10_0_flush_vm_hub(adev, vmid, AMDGPU_MMHUB, 0);
+ if (!adev->mman.buffer_funcs_enabled ||
+ !adev->ib_pool_ready ||
+ adev->in_gpu_reset) {
+ gmc_v10_0_flush_vm_hub(adev, vmid, AMDGPU_GFXHUB, 0);
+ mutex_unlock(&adev->mman.gtt_window_lock);
+ return;
+ }
+
+ /* The SDMA on Navi has a bug which can theoretically result in memory
+ * corruption if an invalidation happens at the same time as an VA
+ * translation. Avoid this by doing the invalidation from the SDMA
+ * itself.
+ */
+ r = amdgpu_job_alloc_with_ib(adev, 16 * 4, &job);
+ if (r)
+ goto error_alloc;
+
+ job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
+ job->vm_needs_flush = true;
+ amdgpu_ring_pad_ib(ring, &job->ibs[0]);
+ r = amdgpu_job_submit(job, &adev->mman.entity,
+ AMDGPU_FENCE_OWNER_UNDEFINED, &fence);
+ if (r)
+ goto error_submit;
+
+ mutex_unlock(&adev->mman.gtt_window_lock);
+
+ dma_fence_wait(fence, false);
+ dma_fence_put(fence);
+
+ return;
+
+error_submit:
+ amdgpu_job_free(job);
+
+error_alloc:
+ mutex_unlock(&adev->mman.gtt_window_lock);
+ DRM_ERROR("Error flushing GPU TLB using the SDMA (%d)!\n", r);
+}
+
+static uint64_t gmc_v10_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
+ unsigned vmid, uint64_t pd_addr)
+{
+ struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
+ uint32_t req = gmc_v10_0_get_invalidate_req(vmid, 0);
+ unsigned eng = ring->vm_inv_eng;
+
+ amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + (2 * vmid),
+ lower_32_bits(pd_addr));
+
+ amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + (2 * vmid),
+ upper_32_bits(pd_addr));
+
+ amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_req + eng, req);
+
+ /* wait for the invalidate to complete */
+ amdgpu_ring_emit_reg_wait(ring, hub->vm_inv_eng0_ack + eng,
+ 1 << vmid, 1 << vmid);
+
+ return pd_addr;
+}
+
+static void gmc_v10_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
+ unsigned pasid)
+{
+ struct amdgpu_device *adev = ring->adev;
+ uint32_t reg;
+
+ if (ring->funcs->vmhub == AMDGPU_GFXHUB)
+ reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
+ else
+ reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
+
+ amdgpu_ring_emit_wreg(ring, reg, pasid);
+}
+
+/*
+ * PTE format on NAVI 10:
+ * 63:59 reserved
+ * 58:57 reserved
+ * 56 F
+ * 55 L
+ * 54 reserved
+ * 53:52 SW
+ * 51 T
+ * 50:48 mtype
+ * 47:12 4k physical page base address
+ * 11:7 fragment
+ * 6 write
+ * 5 read
+ * 4 exe
+ * 3 Z
+ * 2 snooped
+ * 1 system
+ * 0 valid
+ *
+ * PDE format on NAVI 10:
+ * 63:59 block fragment size
+ * 58:55 reserved
+ * 54 P
+ * 53:48 reserved
+ * 47:6 physical base address of PD or PTE
+ * 5:3 reserved
+ * 2 C
+ * 1 system
+ * 0 valid
+ */
+static uint64_t gmc_v10_0_get_vm_pte_flags(struct amdgpu_device *adev,
+ uint32_t flags)
+{
+ uint64_t pte_flag = 0;
+
+ if (flags & AMDGPU_VM_PAGE_EXECUTABLE)
+ pte_flag |= AMDGPU_PTE_EXECUTABLE;
+ if (flags & AMDGPU_VM_PAGE_READABLE)
+ pte_flag |= AMDGPU_PTE_READABLE;
+ if (flags & AMDGPU_VM_PAGE_WRITEABLE)
+ pte_flag |= AMDGPU_PTE_WRITEABLE;
+
+ switch (flags & AMDGPU_VM_MTYPE_MASK) {
+ case AMDGPU_VM_MTYPE_DEFAULT:
+ pte_flag |= AMDGPU_PTE_MTYPE_NV10(MTYPE_NC);
+ break;
+ case AMDGPU_VM_MTYPE_NC:
+ pte_flag |= AMDGPU_PTE_MTYPE_NV10(MTYPE_NC);
+ break;
+ case AMDGPU_VM_MTYPE_WC:
+ pte_flag |= AMDGPU_PTE_MTYPE_NV10(MTYPE_WC);
+ break;
+ case AMDGPU_VM_MTYPE_CC:
+ pte_flag |= AMDGPU_PTE_MTYPE_NV10(MTYPE_CC);
+ break;
+ case AMDGPU_VM_MTYPE_UC:
+ pte_flag |= AMDGPU_PTE_MTYPE_NV10(MTYPE_UC);
+ break;
+ default:
+ pte_flag |= AMDGPU_PTE_MTYPE_NV10(MTYPE_NC);
+ break;
+ }
+
+ if (flags & AMDGPU_VM_PAGE_PRT)
+ pte_flag |= AMDGPU_PTE_PRT;
+
+ return pte_flag;
+}
+
+static void gmc_v10_0_get_vm_pde(struct amdgpu_device *adev, int level,
+ uint64_t *addr, uint64_t *flags)
+{
+ if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
+ *addr = adev->vm_manager.vram_base_offset + *addr -
+ adev->gmc.vram_start;
+ BUG_ON(*addr & 0xFFFF00000000003FULL);
+
+ if (!adev->gmc.translate_further)
+ return;
+
+ if (level == AMDGPU_VM_PDB1) {
+ /* Set the block fragment size */
+ if (!(*flags & AMDGPU_PDE_PTE))
+ *flags |= AMDGPU_PDE_BFS(0x9);
+
+ } else if (level == AMDGPU_VM_PDB0) {
+ if (*flags & AMDGPU_PDE_PTE)
+ *flags &= ~AMDGPU_PDE_PTE;
+ else
+ *flags |= AMDGPU_PTE_TF;
+ }
+}
+
+static const struct amdgpu_gmc_funcs gmc_v10_0_gmc_funcs = {
+ .flush_gpu_tlb = gmc_v10_0_flush_gpu_tlb,
+ .emit_flush_gpu_tlb = gmc_v10_0_emit_flush_gpu_tlb,
+ .emit_pasid_mapping = gmc_v10_0_emit_pasid_mapping,
+ .get_vm_pte_flags = gmc_v10_0_get_vm_pte_flags,
+ .get_vm_pde = gmc_v10_0_get_vm_pde
+};
+
+static void gmc_v10_0_set_gmc_funcs(struct amdgpu_device *adev)
+{
+ if (adev->gmc.gmc_funcs == NULL)
+ adev->gmc.gmc_funcs = &gmc_v10_0_gmc_funcs;
+}
+
+static int gmc_v10_0_early_init(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ gmc_v10_0_set_gmc_funcs(adev);
+ gmc_v10_0_set_irq_funcs(adev);
+
+ adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
+ adev->gmc.shared_aperture_end =
+ adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
+ adev->gmc.private_aperture_start = 0x1000000000000000ULL;
+ adev->gmc.private_aperture_end =
+ adev->gmc.private_aperture_start + (4ULL << 30) - 1;
+
+ return 0;
+}
+
+static int gmc_v10_0_late_init(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ unsigned vm_inv_eng[AMDGPU_MAX_VMHUBS] = { 4, 4 };
+ unsigned i;
+
+ for(i = 0; i < adev->num_rings; ++i) {
+ struct amdgpu_ring *ring = adev->rings[i];
+ unsigned vmhub = ring->funcs->vmhub;
+
+ ring->vm_inv_eng = vm_inv_eng[vmhub]++;
+ dev_info(adev->dev, "ring %u(%s) uses VM inv eng %u on hub %u\n",
+ ring->idx, ring->name, ring->vm_inv_eng,
+ ring->funcs->vmhub);
+ }
+
+ /* Engine 17 is used for GART flushes */
+ for(i = 0; i < AMDGPU_MAX_VMHUBS; ++i)
+ BUG_ON(vm_inv_eng[i] > 17);
+
+ return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
+}
+
+static void gmc_v10_0_vram_gtt_location(struct amdgpu_device *adev,
+ struct amdgpu_gmc *mc)
+{
+ u64 base = 0;
+
+ if (!amdgpu_sriov_vf(adev))
+ base = gfxhub_v2_0_get_fb_location(adev);
+
+ amdgpu_gmc_vram_location(adev, &adev->gmc, base);
+ amdgpu_gmc_gart_location(adev, mc);
+
+ /* base offset of vram pages */
+ adev->vm_manager.vram_base_offset = gfxhub_v2_0_get_mc_fb_offset(adev);
+}
+
+/**
+ * gmc_v10_0_mc_init - initialize the memory controller driver params
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Look up the amount of vram, vram width, and decide how to place
+ * vram and gart within the GPU's physical address space.
+ * Returns 0 for success.
+ */
+static int gmc_v10_0_mc_init(struct amdgpu_device *adev)
+{
+ int chansize, numchan;
+
+ if (!amdgpu_emu_mode)
+ adev->gmc.vram_width = amdgpu_atomfirmware_get_vram_width(adev);
+ else {
+ /* hard code vram_width for emulation */
+ chansize = 128;
+ numchan = 1;
+ adev->gmc.vram_width = numchan * chansize;
+ }
+
+ /* Could aper size report 0 ? */
+ adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
+ adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
+
+ /* size in MB on si */
+ adev->gmc.mc_vram_size =
+ adev->nbio_funcs->get_memsize(adev) * 1024ULL * 1024ULL;
+ adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
+ adev->gmc.visible_vram_size = adev->gmc.aper_size;
+
+ /* In case the PCI BAR is larger than the actual amount of vram */
+ if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)
+ adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
+
+ /* set the gart size */
+ if (amdgpu_gart_size == -1) {
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ default:
+ adev->gmc.gart_size = 512ULL << 20;
+ break;
+ }
+ } else
+ adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
+
+ gmc_v10_0_vram_gtt_location(adev, &adev->gmc);
+
+ return 0;
+}
+
+static int gmc_v10_0_gart_init(struct amdgpu_device *adev)
+{
+ int r;
+
+ if (adev->gart.bo) {
+ WARN(1, "NAVI10 PCIE GART already initialized\n");
+ return 0;
+ }
+
+ /* Initialize common gart structure */
+ r = amdgpu_gart_init(adev);
+ if (r)
+ return r;
+
+ adev->gart.table_size = adev->gart.num_gpu_pages * 8;
+ adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_NV10(MTYPE_UC) |
+ AMDGPU_PTE_EXECUTABLE;
+
+ return amdgpu_gart_table_vram_alloc(adev);
+}
+
+static unsigned gmc_v10_0_get_vbios_fb_size(struct amdgpu_device *adev)
+{
+ u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
+ unsigned size;
+
+ if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
+ size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */
+ } else {
+ u32 viewport;
+ u32 pitch;
+
+ viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
+ pitch = RREG32_SOC15(DCE, 0, mmHUBPREQ0_DCSURF_SURFACE_PITCH);
+ size = (REG_GET_FIELD(viewport,
+ HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
+ REG_GET_FIELD(pitch, HUBPREQ0_DCSURF_SURFACE_PITCH, PITCH) *
+ 4);
+ }
+ /* return 0 if the pre-OS buffer uses up most of vram */
+ if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) {
+ DRM_ERROR("Warning: pre-OS buffer uses most of vram, \
+ be aware of gart table overwrite\n");
+ return 0;
+ }
+
+ return size;
+}
+
+
+
+static int gmc_v10_0_sw_init(void *handle)
+{
+ int r;
+ int dma_bits;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ gfxhub_v2_0_init(adev);
+ mmhub_v2_0_init(adev);
+
+ spin_lock_init(&adev->gmc.invalidate_lock);
+
+ adev->gmc.vram_type = amdgpu_atomfirmware_get_vram_type(adev);
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ /*
+ * To fulfill 4-level page support,
+ * vm size is 256TB (48bit), maximum size of Navi10,
+ * block size 512 (9bit)
+ */
+ amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
+ break;
+ default:
+ break;
+ }
+
+ /* This interrupt is VMC page fault.*/
+ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC,
+ VMC_1_0__SRCID__VM_FAULT,
+ &adev->gmc.vm_fault);
+ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2,
+ UTCL2_1_0__SRCID__FAULT,
+ &adev->gmc.vm_fault);
+ if (r)
+ return r;
+
+ /*
+ * Set the internal MC address mask This is the max address of the GPU's
+ * internal address space.
+ */
+ adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
+
+ /*
+ * Reserve 8M stolen memory for navi10 like vega10
+ * TODO: will check if it's really needed on asic.
+ */
+ if (amdgpu_emu_mode == 1)
+ adev->gmc.stolen_size = 0;
+ else
+ adev->gmc.stolen_size = 9 * 1024 *1024;
+
+ /*
+ * Set DMA mask + need_dma32 flags.
+ * PCIE - can handle 44-bits.
+ * IGP - can handle 44-bits
+ * PCI - dma32 for legacy pci gart, 44 bits on navi10
+ */
+ adev->need_dma32 = false;
+ dma_bits = adev->need_dma32 ? 32 : 44;
+
+ r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
+ if (r) {
+ adev->need_dma32 = true;
+ dma_bits = 32;
+ printk(KERN_WARNING "amdgpu: No suitable DMA available.\n");
+ }
+
+ r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
+ if (r) {
+ pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
+ printk(KERN_WARNING "amdgpu: No coherent DMA available.\n");
+ }
+
+ r = gmc_v10_0_mc_init(adev);
+ if (r)
+ return r;
+
+ adev->gmc.stolen_size = gmc_v10_0_get_vbios_fb_size(adev);
+
+ /* Memory manager */
+ r = amdgpu_bo_init(adev);
+ if (r)
+ return r;
+
+ r = gmc_v10_0_gart_init(adev);
+ if (r)
+ return r;
+
+ /*
+ * number of VMs
+ * VMID 0 is reserved for System
+ * amdgpu graphics/compute will use VMIDs 1-7
+ * amdkfd will use VMIDs 8-15
+ */
+ adev->vm_manager.id_mgr[AMDGPU_GFXHUB].num_ids = AMDGPU_NUM_OF_VMIDS;
+ adev->vm_manager.id_mgr[AMDGPU_MMHUB].num_ids = AMDGPU_NUM_OF_VMIDS;
+
+ amdgpu_vm_manager_init(adev);
+
+ return 0;
+}
+
+/**
+ * gmc_v8_0_gart_fini - vm fini callback
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Tears down the driver GART/VM setup (CIK).
+ */
+static void gmc_v10_0_gart_fini(struct amdgpu_device *adev)
+{
+ amdgpu_gart_table_vram_free(adev);
+ amdgpu_gart_fini(adev);
+}
+
+static int gmc_v10_0_sw_fini(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ amdgpu_vm_manager_fini(adev);
+ gmc_v10_0_gart_fini(adev);
+ amdgpu_gem_force_release(adev);
+ amdgpu_bo_fini(adev);
+
+ return 0;
+}
+
+static void gmc_v10_0_init_golden_registers(struct amdgpu_device *adev)
+{
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ break;
+ default:
+ break;
+ }
+}
+
+/**
+ * gmc_v10_0_gart_enable - gart enable
+ *
+ * @adev: amdgpu_device pointer
+ */
+static int gmc_v10_0_gart_enable(struct amdgpu_device *adev)
+{
+ int r;
+ bool value;
+ u32 tmp;
+
+ if (adev->gart.bo == NULL) {
+ dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
+ return -EINVAL;
+ }
+
+ r = amdgpu_gart_table_vram_pin(adev);
+ if (r)
+ return r;
+
+ r = gfxhub_v2_0_gart_enable(adev);
+ if (r)
+ return r;
+
+ r = mmhub_v2_0_gart_enable(adev);
+ if (r)
+ return r;
+
+ tmp = RREG32_SOC15(HDP, 0, mmHDP_MISC_CNTL);
+ tmp |= HDP_MISC_CNTL__FLUSH_INVALIDATE_CACHE_MASK;
+ WREG32_SOC15(HDP, 0, mmHDP_MISC_CNTL, tmp);
+
+ tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL);
+ WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp);
+
+ /* Flush HDP after it is initialized */
+ adev->nbio_funcs->hdp_flush(adev, NULL);
+
+ value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ?
+ false : true;
+
+ gfxhub_v2_0_set_fault_enable_default(adev, value);
+ mmhub_v2_0_set_fault_enable_default(adev, value);
+ gmc_v10_0_flush_gpu_tlb(adev, 0, 0);
+
+ DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
+ (unsigned)(adev->gmc.gart_size >> 20),
+ (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
+
+ adev->gart.ready = true;
+
+ return 0;
+}
+
+static int gmc_v10_0_hw_init(void *handle)
+{
+ int r;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ /* The sequence of these two function calls matters.*/
+ gmc_v10_0_init_golden_registers(adev);
+
+ r = gmc_v10_0_gart_enable(adev);
+ if (r)
+ return r;
+
+ return 0;
+}
+
+/**
+ * gmc_v10_0_gart_disable - gart disable
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * This disables all VM page table.
+ */
+static void gmc_v10_0_gart_disable(struct amdgpu_device *adev)
+{
+ gfxhub_v2_0_gart_disable(adev);
+ mmhub_v2_0_gart_disable(adev);
+ amdgpu_gart_table_vram_unpin(adev);
+}
+
+static int gmc_v10_0_hw_fini(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ if (amdgpu_sriov_vf(adev)) {
+ /* full access mode, so don't touch any GMC register */
+ DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
+ return 0;
+ }
+
+ amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
+ gmc_v10_0_gart_disable(adev);
+
+ return 0;
+}
+
+static int gmc_v10_0_suspend(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ gmc_v10_0_hw_fini(adev);
+
+ return 0;
+}
+
+static int gmc_v10_0_resume(void *handle)
+{
+ int r;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ r = gmc_v10_0_hw_init(adev);
+ if (r)
+ return r;
+
+ amdgpu_vmid_reset_all(adev);
+
+ return 0;
+}
+
+static bool gmc_v10_0_is_idle(void *handle)
+{
+ /* MC is always ready in GMC v10.*/
+ return true;
+}
+
+static int gmc_v10_0_wait_for_idle(void *handle)
+{
+ /* There is no need to wait for MC idle in GMC v10.*/
+ return 0;
+}
+
+static int gmc_v10_0_soft_reset(void *handle)
+{
+ return 0;
+}
+
+static int gmc_v10_0_set_clockgating_state(void *handle,
+ enum amd_clockgating_state state)
+{
+ int r;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ r = mmhub_v2_0_set_clockgating(adev, state);
+ if (r)
+ return r;
+
+ return athub_v2_0_set_clockgating(adev, state);
+}
+
+static void gmc_v10_0_get_clockgating_state(void *handle, u32 *flags)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ mmhub_v2_0_get_clockgating(adev, flags);
+
+ athub_v2_0_get_clockgating(adev, flags);
+}
+
+static int gmc_v10_0_set_powergating_state(void *handle,
+ enum amd_powergating_state state)
+{
+ return 0;
+}
+
+const struct amd_ip_funcs gmc_v10_0_ip_funcs = {
+ .name = "gmc_v10_0",
+ .early_init = gmc_v10_0_early_init,
+ .late_init = gmc_v10_0_late_init,
+ .sw_init = gmc_v10_0_sw_init,
+ .sw_fini = gmc_v10_0_sw_fini,
+ .hw_init = gmc_v10_0_hw_init,
+ .hw_fini = gmc_v10_0_hw_fini,
+ .suspend = gmc_v10_0_suspend,
+ .resume = gmc_v10_0_resume,
+ .is_idle = gmc_v10_0_is_idle,
+ .wait_for_idle = gmc_v10_0_wait_for_idle,
+ .soft_reset = gmc_v10_0_soft_reset,
+ .set_clockgating_state = gmc_v10_0_set_clockgating_state,
+ .set_powergating_state = gmc_v10_0_set_powergating_state,
+ .get_clockgating_state = gmc_v10_0_get_clockgating_state,
+};
+
+const struct amdgpu_ip_block_version gmc_v10_0_ip_block =
+{
+ .type = AMD_IP_BLOCK_TYPE_GMC,
+ .major = 10,
+ .minor = 0,
+ .rev = 0,
+ .funcs = &gmc_v10_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.h b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.h
new file mode 100644
index 000000000000..7daa53d8996c
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __GMC_V10_0_H__
+#define __GMC_V10_0_H__
+
+extern const struct amd_ip_funcs gmc_v10_0_ip_funcs;
+extern const struct amdgpu_ip_block_version gmc_v10_0_ip_block;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index b06d876da2d9..ca8dbe91cc8b 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -20,8 +20,11 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
+
#include <linux/firmware.h>
-#include <drm/drmP.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
#include <drm/drm_cache.h>
#include "amdgpu.h"
#include "gmc_v6_0.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index 75aa3332aee2..57f80065d57a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -20,8 +20,11 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
+
#include <linux/firmware.h>
-#include <drm/drmP.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
#include <drm/drm_cache.h>
#include "amdgpu.h"
#include "cikd.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index 8a3b5e6fc6c9..9238280d1ff7 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -20,8 +20,11 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
+
#include <linux/firmware.h>
-#include <drm/drmP.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
#include <drm/drm_cache.h>
#include "amdgpu.h"
#include "gmc_v8_0.h"
@@ -289,7 +292,7 @@ out:
*
* @adev: amdgpu_device pointer
*
- * Load the GDDR MC ucode into the hw (CIK).
+ * Load the GDDR MC ucode into the hw (VI).
* Returns 0 on success, error on failure.
*/
static int gmc_v8_0_tonga_mc_load_microcode(struct amdgpu_device *adev)
@@ -443,7 +446,7 @@ static void gmc_v8_0_vram_gtt_location(struct amdgpu_device *adev,
* @adev: amdgpu_device pointer
*
* Set the location of vram, gart, and AGP in the GPU's
- * physical address space (CIK).
+ * physical address space (VI).
*/
static void gmc_v8_0_mc_program(struct amdgpu_device *adev)
{
@@ -515,7 +518,7 @@ static void gmc_v8_0_mc_program(struct amdgpu_device *adev)
* @adev: amdgpu_device pointer
*
* Look up the amount of vram, vram width, and decide how to place
- * vram and gart within the GPU's physical address space (CIK).
+ * vram and gart within the GPU's physical address space (VI).
* Returns 0 for success.
*/
static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
@@ -630,7 +633,7 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
* @adev: amdgpu_device pointer
* @vmid: vm instance to flush
*
- * Flush the TLB for the requested page table (CIK).
+ * Flush the TLB for the requested page table (VI).
*/
static void gmc_v8_0_flush_gpu_tlb(struct amdgpu_device *adev,
uint32_t vmid, uint32_t flush_type)
@@ -800,7 +803,7 @@ static void gmc_v8_0_set_prt(struct amdgpu_device *adev, bool enable)
* This sets up the TLBs, programs the page tables for VMID0,
* sets up the hw for VMIDs 1-15 which are allocated on
* demand, and sets up the global locations for the LDS, GDS,
- * and GPUVM for FSA64 clients (CIK).
+ * and GPUVM for FSA64 clients (VI).
* Returns 0 for success, errors for failure.
*/
static int gmc_v8_0_gart_enable(struct amdgpu_device *adev)
@@ -948,7 +951,7 @@ static int gmc_v8_0_gart_init(struct amdgpu_device *adev)
*
* @adev: amdgpu_device pointer
*
- * This disables all VM page table (CIK).
+ * This disables all VM page table (VI).
*/
static void gmc_v8_0_gart_disable(struct amdgpu_device *adev)
{
@@ -978,7 +981,7 @@ static void gmc_v8_0_gart_disable(struct amdgpu_device *adev)
* @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
* @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
*
- * Print human readable fault information (CIK).
+ * Print human readable fault information (VI).
*/
static void gmc_v8_0_vm_decode_fault(struct amdgpu_device *adev, u32 status,
u32 addr, u32 mc_client, unsigned pasid)
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 72837b8c7031..73f3b79ab131 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -20,8 +20,12 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
+
#include <linux/firmware.h>
+#include <linux/pci.h>
+
#include <drm/drm_cache.h>
+
#include "amdgpu.h"
#include "gmc_v9_0.h"
#include "amdgpu_atomfirmware.h"
@@ -531,22 +535,22 @@ static uint64_t gmc_v9_0_get_vm_pte_flags(struct amdgpu_device *adev,
switch (flags & AMDGPU_VM_MTYPE_MASK) {
case AMDGPU_VM_MTYPE_DEFAULT:
- pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC);
+ pte_flag |= AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
break;
case AMDGPU_VM_MTYPE_NC:
- pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC);
+ pte_flag |= AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
break;
case AMDGPU_VM_MTYPE_WC:
- pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_WC);
+ pte_flag |= AMDGPU_PTE_MTYPE_VG10(MTYPE_WC);
break;
case AMDGPU_VM_MTYPE_CC:
- pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_CC);
+ pte_flag |= AMDGPU_PTE_MTYPE_VG10(MTYPE_CC);
break;
case AMDGPU_VM_MTYPE_UC:
- pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_UC);
+ pte_flag |= AMDGPU_PTE_MTYPE_VG10(MTYPE_UC);
break;
default:
- pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC);
+ pte_flag |= AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
break;
}
@@ -686,8 +690,25 @@ static int gmc_v9_0_ecc_late_init(void *handle)
return 0;
}
/* handle resume path. */
- if (*ras_if)
+ if (*ras_if) {
+ /* resend ras TA enable cmd during resume.
+ * prepare to handle failure.
+ */
+ ih_info.head = **ras_if;
+ r = amdgpu_ras_feature_enable_on_boot(adev, *ras_if, 1);
+ if (r) {
+ if (r == -EAGAIN) {
+ /* request a gpu reset. will run again. */
+ amdgpu_ras_request_reset_on_boot(adev,
+ AMDGPU_RAS_BLOCK__UMC);
+ return 0;
+ }
+ /* fail to enable ras, cleanup all. */
+ goto irq;
+ }
+ /* enable successfully. continue. */
goto resume;
+ }
*ras_if = kmalloc(sizeof(**ras_if), GFP_KERNEL);
if (!*ras_if)
@@ -696,8 +717,14 @@ static int gmc_v9_0_ecc_late_init(void *handle)
**ras_if = ras_block;
r = amdgpu_ras_feature_enable_on_boot(adev, *ras_if, 1);
- if (r)
+ if (r) {
+ if (r == -EAGAIN) {
+ amdgpu_ras_request_reset_on_boot(adev,
+ AMDGPU_RAS_BLOCK__UMC);
+ r = 0;
+ }
goto feature;
+ }
ih_info.head = **ras_if;
fs_info.head = **ras_if;
@@ -706,9 +733,7 @@ static int gmc_v9_0_ecc_late_init(void *handle)
if (r)
goto interrupt;
- r = amdgpu_ras_debugfs_create(adev, &fs_info);
- if (r)
- goto debugfs;
+ amdgpu_ras_debugfs_create(adev, &fs_info);
r = amdgpu_ras_sysfs_create(adev, &fs_info);
if (r)
@@ -723,14 +748,13 @@ irq:
amdgpu_ras_sysfs_remove(adev, *ras_if);
sysfs:
amdgpu_ras_debugfs_remove(adev, *ras_if);
-debugfs:
amdgpu_ras_interrupt_remove_handler(adev, &ih_info);
interrupt:
amdgpu_ras_feature_enable(adev, *ras_if, 0);
feature:
kfree(*ras_if);
*ras_if = NULL;
- return -EINVAL;
+ return r;
}
@@ -892,7 +916,7 @@ static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
if (r)
return r;
adev->gart.table_size = adev->gart.num_gpu_pages * 8;
- adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE(MTYPE_UC) |
+ adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) |
AMDGPU_PTE_EXECUTABLE;
return amdgpu_gart_table_vram_alloc(adev);
}
@@ -1099,6 +1123,9 @@ static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
switch (adev->asic_type) {
case CHIP_VEGA10:
+ if (amdgpu_virt_support_skip_setting(adev))
+ break;
+ /* fall through */
case CHIP_VEGA20:
soc15_program_register_sequence(adev,
golden_settings_mmhub_1_0_0,
@@ -1163,6 +1190,9 @@ static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL);
WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp);
+ WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8));
+ WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40));
+
/* After HDP is initialized, flush HDP.*/
adev->nbio_funcs->hdp_flush(adev, NULL);
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
index b1626e1d2f5d..a13dd9a51149 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
@@ -20,7 +20,9 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
-#include <drm/drmP.h>
+
+#include <linux/pci.h>
+
#include "amdgpu.h"
#include "amdgpu_ih.h"
#include "vid.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
index f2e6b148ccad..4b3faaccecb9 100644
--- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
@@ -21,7 +21,6 @@
*
*/
-#include <drm/drmP.h>
#include "amdgpu.h"
#include "amdgpu_pm.h"
#include "cikd.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/kv_smc.c b/drivers/gpu/drm/amd/amdgpu/kv_smc.c
index b82e33c01571..2d9ab6b8be66 100644
--- a/drivers/gpu/drm/amd/amdgpu/kv_smc.c
+++ b/drivers/gpu/drm/amd/amdgpu/kv_smc.c
@@ -22,7 +22,6 @@
* Authors: Alex Deucher
*/
-#include <drm/drmP.h>
#include "amdgpu.h"
#include "cikd.h"
#include "kv_dpm.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c
new file mode 100644
index 000000000000..29fab7984855
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c
@@ -0,0 +1,366 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/firmware.h>
+#include <linux/module.h>
+#include "amdgpu.h"
+#include "soc15_common.h"
+#include "nv.h"
+#include "gc/gc_10_1_0_offset.h"
+#include "gc/gc_10_1_0_sh_mask.h"
+
+MODULE_FIRMWARE("amdgpu/navi10_mes.bin");
+
+static int mes_v10_1_add_hw_queue(struct amdgpu_mes *mes,
+ struct mes_add_queue_input *input)
+{
+ return 0;
+}
+
+static int mes_v10_1_remove_hw_queue(struct amdgpu_mes *mes,
+ struct mes_remove_queue_input *input)
+{
+ return 0;
+}
+
+static int mes_v10_1_suspend_gang(struct amdgpu_mes *mes,
+ struct mes_suspend_gang_input *input)
+{
+ return 0;
+}
+
+static int mes_v10_1_resume_gang(struct amdgpu_mes *mes,
+ struct mes_resume_gang_input *input)
+{
+ return 0;
+}
+
+static const struct amdgpu_mes_funcs mes_v10_1_funcs = {
+ .add_hw_queue = mes_v10_1_add_hw_queue,
+ .remove_hw_queue = mes_v10_1_remove_hw_queue,
+ .suspend_gang = mes_v10_1_suspend_gang,
+ .resume_gang = mes_v10_1_resume_gang,
+};
+
+static int mes_v10_1_init_microcode(struct amdgpu_device *adev)
+{
+ const char *chip_name;
+ char fw_name[30];
+ int err;
+ const struct mes_firmware_header_v1_0 *mes_hdr;
+
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ chip_name = "navi10";
+ break;
+ default:
+ BUG();
+ }
+
+ snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes.bin", chip_name);
+ err = request_firmware(&adev->mes.fw, fw_name, adev->dev);
+ if (err)
+ return err;
+
+ err = amdgpu_ucode_validate(adev->mes.fw);
+ if (err) {
+ release_firmware(adev->mes.fw);
+ adev->mes.fw = NULL;
+ return err;
+ }
+
+ mes_hdr = (const struct mes_firmware_header_v1_0 *)adev->mes.fw->data;
+ adev->mes.ucode_fw_version = le32_to_cpu(mes_hdr->mes_ucode_version);
+ adev->mes.ucode_fw_version =
+ le32_to_cpu(mes_hdr->mes_ucode_data_version);
+ adev->mes.uc_start_addr =
+ le32_to_cpu(mes_hdr->mes_uc_start_addr_lo) |
+ ((uint64_t)(le32_to_cpu(mes_hdr->mes_uc_start_addr_hi)) << 32);
+ adev->mes.data_start_addr =
+ le32_to_cpu(mes_hdr->mes_data_start_addr_lo) |
+ ((uint64_t)(le32_to_cpu(mes_hdr->mes_data_start_addr_hi)) << 32);
+
+ return 0;
+}
+
+static void mes_v10_1_free_microcode(struct amdgpu_device *adev)
+{
+ release_firmware(adev->mes.fw);
+ adev->mes.fw = NULL;
+}
+
+static int mes_v10_1_allocate_ucode_buffer(struct amdgpu_device *adev)
+{
+ int r;
+ const struct mes_firmware_header_v1_0 *mes_hdr;
+ const __le32 *fw_data;
+ unsigned fw_size;
+
+ mes_hdr = (const struct mes_firmware_header_v1_0 *)
+ adev->mes.fw->data;
+
+ fw_data = (const __le32 *)(adev->mes.fw->data +
+ le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
+ fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
+
+ r = amdgpu_bo_create_reserved(adev, fw_size,
+ PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
+ &adev->mes.ucode_fw_obj,
+ &adev->mes.ucode_fw_gpu_addr,
+ (void **)&adev->mes.ucode_fw_ptr);
+ if (r) {
+ dev_err(adev->dev, "(%d) failed to create mes fw bo\n", r);
+ return r;
+ }
+
+ memcpy(adev->mes.ucode_fw_ptr, fw_data, fw_size);
+
+ amdgpu_bo_kunmap(adev->mes.ucode_fw_obj);
+ amdgpu_bo_unreserve(adev->mes.ucode_fw_obj);
+
+ return 0;
+}
+
+static int mes_v10_1_allocate_ucode_data_buffer(struct amdgpu_device *adev)
+{
+ int r;
+ const struct mes_firmware_header_v1_0 *mes_hdr;
+ const __le32 *fw_data;
+ unsigned fw_size;
+
+ mes_hdr = (const struct mes_firmware_header_v1_0 *)
+ adev->mes.fw->data;
+
+ fw_data = (const __le32 *)(adev->mes.fw->data +
+ le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
+ fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
+
+ r = amdgpu_bo_create_reserved(adev, fw_size,
+ 64 * 1024, AMDGPU_GEM_DOMAIN_GTT,
+ &adev->mes.data_fw_obj,
+ &adev->mes.data_fw_gpu_addr,
+ (void **)&adev->mes.data_fw_ptr);
+ if (r) {
+ dev_err(adev->dev, "(%d) failed to create mes data fw bo\n", r);
+ return r;
+ }
+
+ memcpy(adev->mes.data_fw_ptr, fw_data, fw_size);
+
+ amdgpu_bo_kunmap(adev->mes.data_fw_obj);
+ amdgpu_bo_unreserve(adev->mes.data_fw_obj);
+
+ return 0;
+}
+
+static void mes_v10_1_free_ucode_buffers(struct amdgpu_device *adev)
+{
+ amdgpu_bo_free_kernel(&adev->mes.data_fw_obj,
+ &adev->mes.data_fw_gpu_addr,
+ (void **)&adev->mes.data_fw_ptr);
+
+ amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj,
+ &adev->mes.ucode_fw_gpu_addr,
+ (void **)&adev->mes.ucode_fw_ptr);
+}
+
+static void mes_v10_1_enable(struct amdgpu_device *adev, bool enable)
+{
+ uint32_t data = 0;
+
+ if (enable) {
+ data = RREG32_SOC15(GC, 0, mmCP_MES_CNTL);
+ data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
+ WREG32_SOC15(GC, 0, mmCP_MES_CNTL, data);
+
+ /* set ucode start address */
+ WREG32_SOC15(GC, 0, mmCP_MES_PRGRM_CNTR_START,
+ (uint32_t)(adev->mes.uc_start_addr) >> 2);
+
+ /* clear BYPASS_UNCACHED to avoid hangs after interrupt. */
+ data = RREG32_SOC15(GC, 0, mmCP_MES_DC_OP_CNTL);
+ data = REG_SET_FIELD(data, CP_MES_DC_OP_CNTL,
+ BYPASS_UNCACHED, 0);
+ WREG32_SOC15(GC, 0, mmCP_MES_DC_OP_CNTL, data);
+
+ /* unhalt MES and activate pipe0 */
+ data = REG_SET_FIELD(0, CP_MES_CNTL, MES_PIPE0_ACTIVE, 1);
+ WREG32_SOC15(GC, 0, mmCP_MES_CNTL, data);
+ } else {
+ data = RREG32_SOC15(GC, 0, mmCP_MES_CNTL);
+ data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_ACTIVE, 0);
+ data = REG_SET_FIELD(data, CP_MES_CNTL,
+ MES_INVALIDATE_ICACHE, 1);
+ data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
+ data = REG_SET_FIELD(data, CP_MES_CNTL, MES_HALT, 1);
+ WREG32_SOC15(GC, 0, mmCP_MES_CNTL, data);
+ }
+}
+
+/* This function is for backdoor MES firmware */
+static int mes_v10_1_load_microcode(struct amdgpu_device *adev)
+{
+ int r;
+ uint32_t data;
+
+ if (!adev->mes.fw)
+ return -EINVAL;
+
+ r = mes_v10_1_allocate_ucode_buffer(adev);
+ if (r)
+ return r;
+
+ r = mes_v10_1_allocate_ucode_data_buffer(adev);
+ if (r) {
+ mes_v10_1_free_ucode_buffers(adev);
+ return r;
+ }
+
+ mes_v10_1_enable(adev, false);
+
+ WREG32_SOC15(GC, 0, mmCP_MES_IC_BASE_CNTL, 0);
+
+ mutex_lock(&adev->srbm_mutex);
+ /* me=3, pipe=0, queue=0 */
+ nv_grbm_select(adev, 3, 0, 0, 0);
+
+ /* set ucode start address */
+ WREG32_SOC15(GC, 0, mmCP_MES_PRGRM_CNTR_START,
+ (uint32_t)(adev->mes.uc_start_addr) >> 2);
+
+ /* set ucode fimrware address */
+ WREG32_SOC15(GC, 0, mmCP_MES_IC_BASE_LO,
+ lower_32_bits(adev->mes.ucode_fw_gpu_addr));
+ WREG32_SOC15(GC, 0, mmCP_MES_IC_BASE_HI,
+ upper_32_bits(adev->mes.ucode_fw_gpu_addr));
+
+ /* set ucode instruction cache boundary to 2M-1 */
+ WREG32_SOC15(GC, 0, mmCP_MES_MIBOUND_LO, 0x1FFFFF);
+
+ /* set ucode data firmware address */
+ WREG32_SOC15(GC, 0, mmCP_MES_MDBASE_LO,
+ lower_32_bits(adev->mes.data_fw_gpu_addr));
+ WREG32_SOC15(GC, 0, mmCP_MES_MDBASE_HI,
+ upper_32_bits(adev->mes.data_fw_gpu_addr));
+
+ /* Set 0x3FFFF (256K-1) to CP_MES_MDBOUND_LO */
+ WREG32_SOC15(GC, 0, mmCP_MES_MDBOUND_LO, 0x3FFFF);
+
+ /* invalidate ICACHE */
+ data = RREG32_SOC15(GC, 0, mmCP_MES_IC_OP_CNTL);
+ data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 0);
+ data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, INVALIDATE_CACHE, 1);
+ WREG32_SOC15(GC, 0, mmCP_MES_IC_OP_CNTL, data);
+
+ /* prime the ICACHE. */
+ data = RREG32_SOC15(GC, 0, mmCP_MES_IC_OP_CNTL);
+ data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 1);
+ WREG32_SOC15(GC, 0, mmCP_MES_IC_OP_CNTL, data);
+
+ nv_grbm_select(adev, 0, 0, 0, 0);
+ mutex_unlock(&adev->srbm_mutex);
+
+ return 0;
+}
+
+static int mes_v10_1_sw_init(void *handle)
+{
+ int r;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ r = mes_v10_1_init_microcode(adev);
+ if (r)
+ return r;
+
+ return 0;
+}
+
+static int mes_v10_1_sw_fini(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ mes_v10_1_free_microcode(adev);
+
+ return 0;
+}
+
+static int mes_v10_1_hw_init(void *handle)
+{
+ int r;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+ r = mes_v10_1_load_microcode(adev);
+ if (r) {
+ DRM_ERROR("failed to MES fw, r=%d\n", r);
+ return r;
+ }
+ } else {
+ DRM_ERROR("only support direct fw loading on MES\n");
+ return -EINVAL;
+ }
+
+ mes_v10_1_enable(adev, true);
+
+ return 0;
+}
+
+static int mes_v10_1_hw_fini(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ mes_v10_1_enable(adev, false);
+
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT)
+ mes_v10_1_free_ucode_buffers(adev);
+
+ return 0;
+}
+
+static int mes_v10_1_suspend(void *handle)
+{
+ return 0;
+}
+
+static int mes_v10_1_resume(void *handle)
+{
+ return 0;
+}
+
+static const struct amd_ip_funcs mes_v10_1_ip_funcs = {
+ .name = "mes_v10_1",
+ .sw_init = mes_v10_1_sw_init,
+ .sw_fini = mes_v10_1_sw_fini,
+ .hw_init = mes_v10_1_hw_init,
+ .hw_fini = mes_v10_1_hw_fini,
+ .suspend = mes_v10_1_suspend,
+ .resume = mes_v10_1_resume,
+};
+
+const struct amdgpu_ip_block_version mes_v10_1_ip_block = {
+ .type = AMD_IP_BLOCK_TYPE_MES,
+ .major = 10,
+ .minor = 1,
+ .rev = 0,
+ .funcs = &mes_v10_1_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v10_1.h b/drivers/gpu/drm/amd/amdgpu/mes_v10_1.h
new file mode 100644
index 000000000000..9afd6ddb01e9
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v10_1.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __MES_V10_1_H__
+#define __MES_V10_1_H__
+
+extern const struct amdgpu_ip_block_version mes_v10_1_ip_block;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
index 41a9a5779623..dc5ce03034d3 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
@@ -111,6 +111,9 @@ static void mmhub_v1_0_init_system_aperture_regs(struct amdgpu_device *adev)
WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18);
+ if (amdgpu_virt_support_skip_setting(adev))
+ return;
+
/* Set default page address. */
value = adev->vram_scratch.gpu_addr - adev->gmc.vram_start +
adev->vm_manager.vram_base_offset;
@@ -156,6 +159,9 @@ static void mmhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
{
uint32_t tmp;
+ if (amdgpu_virt_support_skip_setting(adev))
+ return;
+
/* Setup L2 cache */
tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL);
tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
@@ -202,6 +208,9 @@ static void mmhub_v1_0_enable_system_domain(struct amdgpu_device *adev)
static void mmhub_v1_0_disable_identity_aperture(struct amdgpu_device *adev)
{
+ if (amdgpu_virt_support_skip_setting(adev))
+ return;
+
WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
0XFFFFFFFF);
WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
@@ -256,7 +265,8 @@ static void mmhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
block_size);
/* Send no-retry XNACK on fault to suppress VM fault storm. */
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
- RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 1);
+ RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
+ !amdgpu_noretry);
WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_CNTL, i, tmp);
WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, i*2, 0);
WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, i*2, 0);
@@ -338,11 +348,13 @@ void mmhub_v1_0_gart_disable(struct amdgpu_device *adev)
0);
WREG32_SOC15(MMHUB, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp);
- /* Setup L2 cache */
- tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL);
- tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
- WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL, tmp);
- WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL3, 0);
+ if (!amdgpu_virt_support_skip_setting(adev)) {
+ /* Setup L2 cache */
+ tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL);
+ tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
+ WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL, tmp);
+ WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL3, 0);
+ }
}
/**
@@ -354,6 +366,10 @@ void mmhub_v1_0_gart_disable(struct amdgpu_device *adev)
void mmhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev, bool value)
{
u32 tmp;
+
+ if (amdgpu_virt_support_skip_setting(adev))
+ return;
+
tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL);
tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c
new file mode 100644
index 000000000000..0f9549f19ade
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c
@@ -0,0 +1,445 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "amdgpu.h"
+#include "mmhub_v2_0.h"
+
+#include "mmhub/mmhub_2_0_0_offset.h"
+#include "mmhub/mmhub_2_0_0_sh_mask.h"
+#include "mmhub/mmhub_2_0_0_default.h"
+#include "navi10_enum.h"
+
+#include "soc15_common.h"
+
+static void mmhub_v2_0_init_gart_pt_regs(struct amdgpu_device *adev)
+{
+ uint64_t value = amdgpu_gmc_pd_addr(adev->gart.bo);
+
+ WREG32_SOC15(MMHUB, 0, mmMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
+ lower_32_bits(value));
+
+ WREG32_SOC15(MMHUB, 0, mmMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
+ upper_32_bits(value));
+}
+
+static void mmhub_v2_0_init_gart_aperture_regs(struct amdgpu_device *adev)
+{
+ mmhub_v2_0_init_gart_pt_regs(adev);
+
+ WREG32_SOC15(MMHUB, 0, mmMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
+ (u32)(adev->gmc.gart_start >> 12));
+ WREG32_SOC15(MMHUB, 0, mmMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
+ (u32)(adev->gmc.gart_start >> 44));
+
+ WREG32_SOC15(MMHUB, 0, mmMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
+ (u32)(adev->gmc.gart_end >> 12));
+ WREG32_SOC15(MMHUB, 0, mmMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
+ (u32)(adev->gmc.gart_end >> 44));
+}
+
+static void mmhub_v2_0_init_system_aperture_regs(struct amdgpu_device *adev)
+{
+ uint64_t value;
+ uint32_t tmp;
+
+ /* Disable AGP. */
+ WREG32_SOC15(MMHUB, 0, mmMMMC_VM_AGP_BASE, 0);
+ WREG32_SOC15(MMHUB, 0, mmMMMC_VM_AGP_TOP, 0);
+ WREG32_SOC15(MMHUB, 0, mmMMMC_VM_AGP_BOT, 0x00FFFFFF);
+
+ /* Program the system aperture low logical page number. */
+ WREG32_SOC15(MMHUB, 0, mmMMMC_VM_SYSTEM_APERTURE_LOW_ADDR,
+ adev->gmc.vram_start >> 18);
+ WREG32_SOC15(MMHUB, 0, mmMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
+ adev->gmc.vram_end >> 18);
+
+ /* Set default page address. */
+ value = adev->vram_scratch.gpu_addr - adev->gmc.vram_start +
+ adev->vm_manager.vram_base_offset;
+ WREG32_SOC15(MMHUB, 0, mmMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
+ (u32)(value >> 12));
+ WREG32_SOC15(MMHUB, 0, mmMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
+ (u32)(value >> 44));
+
+ /* Program "protection fault". */
+ WREG32_SOC15(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
+ (u32)(adev->dummy_page_addr >> 12));
+ WREG32_SOC15(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
+ (u32)((u64)adev->dummy_page_addr >> 44));
+
+ tmp = RREG32_SOC15(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_CNTL2);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL2,
+ ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+ WREG32_SOC15(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_CNTL2, tmp);
+}
+
+static void mmhub_v2_0_init_tlb_regs(struct amdgpu_device *adev)
+{
+ uint32_t tmp;
+
+ /* Setup TLB control */
+ tmp = RREG32_SOC15(MMHUB, 0, mmMMMC_VM_MX_L1_TLB_CNTL);
+
+ tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
+ tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
+ tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
+ ENABLE_ADVANCED_DRIVER_MODEL, 1);
+ tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
+ SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
+ tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
+ tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
+ MTYPE, MTYPE_UC); /* UC, uncached */
+
+ WREG32_SOC15(MMHUB, 0, mmMMMC_VM_MX_L1_TLB_CNTL, tmp);
+}
+
+static void mmhub_v2_0_init_cache_regs(struct amdgpu_device *adev)
+{
+ uint32_t tmp;
+
+ /* Setup L2 cache */
+ tmp = RREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_CACHE, 1);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL,
+ ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
+ /* XXX for emulation, Refer to closed source code.*/
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE,
+ 0);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 1);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
+ WREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL, tmp);
+
+ tmp = RREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL2);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
+ WREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL2, tmp);
+
+ tmp = mmMMVM_L2_CNTL3_DEFAULT;
+ WREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL3, tmp);
+
+ tmp = mmMMVM_L2_CNTL4_DEFAULT;
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
+ WREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL4, tmp);
+}
+
+static void mmhub_v2_0_enable_system_domain(struct amdgpu_device *adev)
+{
+ uint32_t tmp;
+
+ tmp = RREG32_SOC15(MMHUB, 0, mmMMVM_CONTEXT0_CNTL);
+ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
+ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
+ WREG32_SOC15(MMHUB, 0, mmMMVM_CONTEXT0_CNTL, tmp);
+}
+
+static void mmhub_v2_0_disable_identity_aperture(struct amdgpu_device *adev)
+{
+ WREG32_SOC15(MMHUB, 0,
+ mmMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
+ 0xFFFFFFFF);
+ WREG32_SOC15(MMHUB, 0,
+ mmMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
+ 0x0000000F);
+
+ WREG32_SOC15(MMHUB, 0,
+ mmMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 0);
+ WREG32_SOC15(MMHUB, 0,
+ mmMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 0);
+
+ WREG32_SOC15(MMHUB, 0, mmMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32,
+ 0);
+ WREG32_SOC15(MMHUB, 0, mmMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32,
+ 0);
+}
+
+static void mmhub_v2_0_setup_vmid_config(struct amdgpu_device *adev)
+{
+ int i;
+ uint32_t tmp;
+
+ for (i = 0; i <= 14; i++) {
+ tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_CONTEXT1_CNTL, i);
+ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
+ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
+ adev->vm_manager.num_level);
+ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+ RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+ DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
+ 1);
+ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+ PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+ VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+ READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+ WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+ EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+ PAGE_TABLE_BLOCK_SIZE,
+ adev->vm_manager.block_size - 9);
+ /* Send no-retry XNACK on fault to suppress VM fault storm. */
+ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+ RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
+ !amdgpu_noretry);
+ WREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_CONTEXT1_CNTL, i, tmp);
+ WREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, i*2, 0);
+ WREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, i*2, 0);
+ WREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, i*2,
+ lower_32_bits(adev->vm_manager.max_pfn - 1));
+ WREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, i*2,
+ upper_32_bits(adev->vm_manager.max_pfn - 1));
+ }
+}
+
+static void mmhub_v2_0_program_invalidation(struct amdgpu_device *adev)
+{
+ unsigned i;
+
+ for (i = 0; i < 18; ++i) {
+ WREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
+ 2 * i, 0xffffffff);
+ WREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
+ 2 * i, 0x1f);
+ }
+}
+
+int mmhub_v2_0_gart_enable(struct amdgpu_device *adev)
+{
+ if (amdgpu_sriov_vf(adev)) {
+ /*
+ * MMMC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are
+ * VF copy registers so vbios post doesn't program them, for
+ * SRIOV driver need to program them
+ */
+ WREG32_SOC15(MMHUB, 0, mmMMMC_VM_FB_LOCATION_BASE,
+ adev->gmc.vram_start >> 24);
+ WREG32_SOC15(MMHUB, 0, mmMMMC_VM_FB_LOCATION_TOP,
+ adev->gmc.vram_end >> 24);
+ }
+
+ /* GART Enable. */
+ mmhub_v2_0_init_gart_aperture_regs(adev);
+ mmhub_v2_0_init_system_aperture_regs(adev);
+ mmhub_v2_0_init_tlb_regs(adev);
+ mmhub_v2_0_init_cache_regs(adev);
+
+ mmhub_v2_0_enable_system_domain(adev);
+ mmhub_v2_0_disable_identity_aperture(adev);
+ mmhub_v2_0_setup_vmid_config(adev);
+ mmhub_v2_0_program_invalidation(adev);
+
+ return 0;
+}
+
+void mmhub_v2_0_gart_disable(struct amdgpu_device *adev)
+{
+ u32 tmp;
+ u32 i;
+
+ /* Disable all tables */
+ for (i = 0; i < 16; i++)
+ WREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_CONTEXT0_CNTL, i, 0);
+
+ /* Setup TLB control */
+ tmp = RREG32_SOC15(MMHUB, 0, mmMMMC_VM_MX_L1_TLB_CNTL);
+ tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
+ tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
+ ENABLE_ADVANCED_DRIVER_MODEL, 0);
+ WREG32_SOC15(MMHUB, 0, mmMMMC_VM_MX_L1_TLB_CNTL, tmp);
+
+ /* Setup L2 cache */
+ tmp = RREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_CACHE, 0);
+ WREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL, tmp);
+ WREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL3, 0);
+}
+
+/**
+ * mmhub_v2_0_set_fault_enable_default - update GART/VM fault handling
+ *
+ * @adev: amdgpu_device pointer
+ * @value: true redirects VM faults to the default page
+ */
+void mmhub_v2_0_set_fault_enable_default(struct amdgpu_device *adev, bool value)
+{
+ u32 tmp;
+ tmp = RREG32_SOC15(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_CNTL);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
+ RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
+ PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
+ PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
+ PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
+ TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
+ value);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
+ NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
+ DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
+ VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
+ READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
+ WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
+ EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+ if (!value) {
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
+ CRASH_ON_NO_RETRY_FAULT, 1);
+ tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
+ CRASH_ON_RETRY_FAULT, 1);
+ }
+ WREG32_SOC15(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_CNTL, tmp);
+}
+
+void mmhub_v2_0_init(struct amdgpu_device *adev)
+{
+ struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB];
+
+ hub->ctx0_ptb_addr_lo32 =
+ SOC15_REG_OFFSET(MMHUB, 0,
+ mmMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
+ hub->ctx0_ptb_addr_hi32 =
+ SOC15_REG_OFFSET(MMHUB, 0,
+ mmMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
+ hub->vm_inv_eng0_req =
+ SOC15_REG_OFFSET(MMHUB, 0, mmMMVM_INVALIDATE_ENG0_REQ);
+ hub->vm_inv_eng0_ack =
+ SOC15_REG_OFFSET(MMHUB, 0, mmMMVM_INVALIDATE_ENG0_ACK);
+ hub->vm_context0_cntl =
+ SOC15_REG_OFFSET(MMHUB, 0, mmMMVM_CONTEXT0_CNTL);
+ hub->vm_l2_pro_fault_status =
+ SOC15_REG_OFFSET(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_STATUS);
+ hub->vm_l2_pro_fault_cntl =
+ SOC15_REG_OFFSET(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_CNTL);
+
+}
+
+static void mmhub_v2_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
+ bool enable)
+{
+ uint32_t def, data, def1, data1;
+
+ def = data = RREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG);
+
+ def1 = data1 = RREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2);
+
+ if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG)) {
+ data |= MM_ATC_L2_MISC_CG__ENABLE_MASK;
+
+ data1 &= ~(DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
+ DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
+ DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
+ DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
+ DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
+ DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
+
+ } else {
+ data &= ~MM_ATC_L2_MISC_CG__ENABLE_MASK;
+
+ data1 |= (DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
+ DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
+ DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
+ DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
+ DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
+ DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
+ }
+
+ if (def != data)
+ WREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG, data);
+
+ if (def1 != data1)
+ WREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2, data1);
+}
+
+static void mmhub_v2_0_update_medium_grain_light_sleep(struct amdgpu_device *adev,
+ bool enable)
+{
+ uint32_t def, data;
+
+ def = data = RREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG);
+
+ if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS))
+ data |= MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
+ else
+ data &= ~MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
+
+ if (def != data)
+ WREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG, data);
+}
+
+int mmhub_v2_0_set_clockgating(struct amdgpu_device *adev,
+ enum amd_clockgating_state state)
+{
+ if (amdgpu_sriov_vf(adev))
+ return 0;
+
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ mmhub_v2_0_update_medium_grain_clock_gating(adev,
+ state == AMD_CG_STATE_GATE ? true : false);
+ mmhub_v2_0_update_medium_grain_light_sleep(adev,
+ state == AMD_CG_STATE_GATE ? true : false);
+ break;
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+void mmhub_v2_0_get_clockgating(struct amdgpu_device *adev, u32 *flags)
+{
+ int data, data1;
+
+ if (amdgpu_sriov_vf(adev))
+ *flags = 0;
+
+ data = RREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG);
+
+ data1 = RREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2);
+
+ /* AMD_CG_SUPPORT_MC_MGCG */
+ if ((data & MM_ATC_L2_MISC_CG__ENABLE_MASK) &&
+ !(data1 & (DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
+ DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
+ DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
+ DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
+ DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
+ DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK)))
+ *flags |= AMD_CG_SUPPORT_MC_MGCG;
+
+ /* AMD_CG_SUPPORT_MC_LS */
+ if (data & MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK)
+ *flags |= AMD_CG_SUPPORT_MC_LS;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.h b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.h
new file mode 100644
index 000000000000..db16f3ece218
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __MMHUB_V2_0_H__
+#define __MMHUB_V2_0_H__
+
+int mmhub_v2_0_gart_enable(struct amdgpu_device *adev);
+void mmhub_v2_0_gart_disable(struct amdgpu_device *adev);
+void mmhub_v2_0_set_fault_enable_default(struct amdgpu_device *adev,
+ bool value);
+void mmhub_v2_0_init(struct amdgpu_device *adev);
+int mmhub_v2_0_set_clockgating(struct amdgpu_device *adev,
+ enum amd_clockgating_state state);
+void mmhub_v2_0_get_clockgating(struct amdgpu_device *adev, u32 *flags);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
index 2471e7cf75ea..235548c0b41f 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
@@ -26,6 +26,7 @@
#include "nbio/nbio_6_1_sh_mask.h"
#include "gc/gc_9_0_offset.h"
#include "gc/gc_9_0_sh_mask.h"
+#include "mp/mp_9_0_offset.h"
#include "soc15.h"
#include "vega10_ih.h"
#include "soc15_common.h"
@@ -343,7 +344,7 @@ flr_done:
/* Trigger recovery for world switch failure if no TDR */
if (amdgpu_device_should_recover_gpu(adev)
- && amdgpu_lockup_timeout == MAX_SCHEDULE_TIMEOUT)
+ && adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT)
amdgpu_device_gpu_recover(adev, NULL);
}
@@ -448,6 +449,20 @@ void xgpu_ai_mailbox_put_irq(struct amdgpu_device *adev)
amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
}
+static void xgpu_ai_init_reg_access_mode(struct amdgpu_device *adev)
+{
+ adev->virt.reg_access_mode = AMDGPU_VIRT_REG_ACCESS_LEGACY;
+
+ /* Enable L1 security reg access mode by defaul, as non-security VF
+ * will no longer be supported.
+ */
+ adev->virt.reg_access_mode |= AMDGPU_VIRT_REG_ACCESS_RLC;
+
+ adev->virt.reg_access_mode |= AMDGPU_VIRT_REG_ACCESS_PSP_PRG_IH;
+
+ adev->virt.reg_access_mode |= AMDGPU_VIRT_REG_SKIP_SEETING;
+}
+
const struct amdgpu_virt_ops xgpu_ai_virt_ops = {
.req_full_gpu = xgpu_ai_request_full_gpu_access,
.rel_full_gpu = xgpu_ai_release_full_gpu_access,
@@ -456,4 +471,5 @@ const struct amdgpu_virt_ops xgpu_ai_virt_ops = {
.trans_msg = xgpu_ai_mailbox_trans_msg,
.get_pp_clk = xgpu_ai_get_pp_clk,
.force_dpm_level = xgpu_ai_force_dpm_level,
+ .init_reg_access_mode = xgpu_ai_init_reg_access_mode,
};
diff --git a/drivers/gpu/drm/amd/amdgpu/navi10_ih.c b/drivers/gpu/drm/amd/amdgpu/navi10_ih.c
new file mode 100644
index 000000000000..e963746be11c
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/navi10_ih.c
@@ -0,0 +1,486 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <drm/drmP.h>
+#include "amdgpu.h"
+#include "amdgpu_ih.h"
+
+#include "oss/osssys_5_0_0_offset.h"
+#include "oss/osssys_5_0_0_sh_mask.h"
+
+#include "soc15_common.h"
+#include "navi10_ih.h"
+
+
+static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
+
+/**
+ * navi10_ih_enable_interrupts - Enable the interrupt ring buffer
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Enable the interrupt ring buffer (NAVI10).
+ */
+static void navi10_ih_enable_interrupts(struct amdgpu_device *adev)
+{
+ u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
+
+ ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
+ ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
+ adev->irq.ih.enabled = true;
+}
+
+/**
+ * navi10_ih_disable_interrupts - Disable the interrupt ring buffer
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Disable the interrupt ring buffer (NAVI10).
+ */
+static void navi10_ih_disable_interrupts(struct amdgpu_device *adev)
+{
+ u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
+
+ ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
+ ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
+ /* set rptr, wptr to 0 */
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
+ adev->irq.ih.enabled = false;
+ adev->irq.ih.rptr = 0;
+}
+
+static uint32_t navi10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl)
+{
+ int rb_bufsz = order_base_2(ih->ring_size / 4);
+
+ ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
+ MC_SPACE, ih->use_bus_addr ? 1 : 4);
+ ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
+ WPTR_OVERFLOW_CLEAR, 1);
+ ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
+ WPTR_OVERFLOW_ENABLE, 1);
+ ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
+ /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register
+ * value is written to memory
+ */
+ ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
+ WPTR_WRITEBACK_ENABLE, 1);
+ ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
+ ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
+ ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);
+
+ return ih_rb_cntl;
+}
+
+/**
+ * navi10_ih_irq_init - init and enable the interrupt ring
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Allocate a ring buffer for the interrupt controller,
+ * enable the RLC, disable interrupts, enable the IH
+ * ring buffer and enable it (NAVI).
+ * Called at device load and reume.
+ * Returns 0 for success, errors for failure.
+ */
+static int navi10_ih_irq_init(struct amdgpu_device *adev)
+{
+ struct amdgpu_ih_ring *ih = &adev->irq.ih;
+ int ret = 0;
+ u32 ih_rb_cntl, ih_doorbell_rtpr, ih_chicken;
+ u32 tmp;
+
+ /* disable irqs */
+ navi10_ih_disable_interrupts(adev);
+
+ adev->nbio_funcs->ih_control(adev);
+
+ /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, ih->gpu_addr >> 8);
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (ih->gpu_addr >> 40) & 0xff);
+
+ ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
+ ih_rb_cntl = navi10_ih_rb_cntl(ih, ih_rb_cntl);
+ ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM,
+ !!adev->irq.msi_enabled);
+
+ if (unlikely(adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT)) {
+ if (ih->use_bus_addr) {
+ ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN);
+ ih_chicken = REG_SET_FIELD(ih_chicken,
+ IH_CHICKEN, MC_SPACE_GPA_ENABLE, 1);
+ WREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN, ih_chicken);
+ }
+ }
+
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
+
+ /* set the writeback address whether it's enabled or not */
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO,
+ lower_32_bits(ih->wptr_addr));
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI,
+ upper_32_bits(ih->wptr_addr) & 0xFFFF);
+
+ /* set rptr, wptr to 0 */
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
+
+ ih_doorbell_rtpr = RREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR);
+ if (ih->use_doorbell) {
+ ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
+ IH_DOORBELL_RPTR, OFFSET,
+ ih->doorbell_index);
+ ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
+ IH_DOORBELL_RPTR, ENABLE, 1);
+ } else {
+ ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
+ IH_DOORBELL_RPTR, ENABLE, 0);
+ }
+ WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, ih_doorbell_rtpr);
+
+ adev->nbio_funcs->ih_doorbell_range(adev, ih->use_doorbell,
+ ih->doorbell_index);
+
+ tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL);
+ tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
+ CLIENT18_IS_STORM_CLIENT, 1);
+ WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp);
+
+ tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL);
+ tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
+ WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp);
+
+ pci_set_master(adev->pdev);
+
+ /* enable interrupts */
+ navi10_ih_enable_interrupts(adev);
+
+ return ret;
+}
+
+/**
+ * navi10_ih_irq_disable - disable interrupts
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Disable interrupts on the hw (NAVI10).
+ */
+static void navi10_ih_irq_disable(struct amdgpu_device *adev)
+{
+ navi10_ih_disable_interrupts(adev);
+
+ /* Wait and acknowledge irq */
+ mdelay(1);
+}
+
+/**
+ * navi10_ih_get_wptr - get the IH ring buffer wptr
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Get the IH ring buffer wptr from either the register
+ * or the writeback memory buffer (NAVI10). Also check for
+ * ring buffer overflow and deal with it.
+ * Returns the value of the wptr.
+ */
+static u32 navi10_ih_get_wptr(struct amdgpu_device *adev,
+ struct amdgpu_ih_ring *ih)
+{
+ u32 wptr, reg, tmp;
+
+ wptr = le32_to_cpu(*ih->wptr_cpu);
+
+ if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
+ goto out;
+
+ reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR);
+ wptr = RREG32_NO_KIQ(reg);
+ if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
+ goto out;
+ wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
+
+ /* When a ring buffer overflow happen start parsing interrupt
+ * from the last not overwritten vector (wptr + 32). Hopefully
+ * this should allow us to catch up.
+ */
+ tmp = (wptr + 32) & ih->ptr_mask;
+ dev_warn(adev->dev, "IH ring buffer overflow "
+ "(0x%08X, 0x%08X, 0x%08X)\n",
+ wptr, ih->rptr, tmp);
+ ih->rptr = tmp;
+
+ reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL);
+ tmp = RREG32_NO_KIQ(reg);
+ tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
+ WREG32_NO_KIQ(reg, tmp);
+out:
+ return (wptr & ih->ptr_mask);
+}
+
+/**
+ * navi10_ih_decode_iv - decode an interrupt vector
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Decodes the interrupt vector at the current rptr
+ * position and also advance the position.
+ */
+static void navi10_ih_decode_iv(struct amdgpu_device *adev,
+ struct amdgpu_ih_ring *ih,
+ struct amdgpu_iv_entry *entry)
+{
+ /* wptr/rptr are in bytes! */
+ u32 ring_index = ih->rptr >> 2;
+ uint32_t dw[8];
+
+ dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);
+ dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);
+ dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);
+ dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);
+ dw[4] = le32_to_cpu(ih->ring[ring_index + 4]);
+ dw[5] = le32_to_cpu(ih->ring[ring_index + 5]);
+ dw[6] = le32_to_cpu(ih->ring[ring_index + 6]);
+ dw[7] = le32_to_cpu(ih->ring[ring_index + 7]);
+
+ entry->client_id = dw[0] & 0xff;
+ entry->src_id = (dw[0] >> 8) & 0xff;
+ entry->ring_id = (dw[0] >> 16) & 0xff;
+ entry->vmid = (dw[0] >> 24) & 0xf;
+ entry->vmid_src = (dw[0] >> 31);
+ entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32);
+ entry->timestamp_src = dw[2] >> 31;
+ entry->pasid = dw[3] & 0xffff;
+ entry->pasid_src = dw[3] >> 31;
+ entry->src_data[0] = dw[4];
+ entry->src_data[1] = dw[5];
+ entry->src_data[2] = dw[6];
+ entry->src_data[3] = dw[7];
+
+ /* wptr/rptr are in bytes! */
+ ih->rptr += 32;
+}
+
+/**
+ * navi10_ih_set_rptr - set the IH ring buffer rptr
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Set the IH ring buffer rptr.
+ */
+static void navi10_ih_set_rptr(struct amdgpu_device *adev,
+ struct amdgpu_ih_ring *ih)
+{
+ if (ih->use_doorbell) {
+ /* XXX check if swapping is necessary on BE */
+ *ih->rptr_cpu = ih->rptr;
+ WDOORBELL32(ih->doorbell_index, ih->rptr);
+ } else
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, ih->rptr);
+}
+
+static int navi10_ih_early_init(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ navi10_ih_set_interrupt_funcs(adev);
+ return 0;
+}
+
+static int navi10_ih_sw_init(void *handle)
+{
+ int r;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ bool use_bus_addr;
+
+ /* use gpu virtual address for ih ring
+ * until ih_checken is programmed to allow
+ * use bus address for ih ring by psp bl */
+ use_bus_addr =
+ (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) ? false : true;
+ r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, use_bus_addr);
+ if (r)
+ return r;
+
+ adev->irq.ih.use_doorbell = true;
+ adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1;
+
+ r = amdgpu_irq_init(adev);
+
+ return r;
+}
+
+static int navi10_ih_sw_fini(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ amdgpu_irq_fini(adev);
+ amdgpu_ih_ring_fini(adev, &adev->irq.ih);
+
+ return 0;
+}
+
+static int navi10_ih_hw_init(void *handle)
+{
+ int r;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ r = navi10_ih_irq_init(adev);
+ if (r)
+ return r;
+
+ return 0;
+}
+
+static int navi10_ih_hw_fini(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ navi10_ih_irq_disable(adev);
+
+ return 0;
+}
+
+static int navi10_ih_suspend(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ return navi10_ih_hw_fini(adev);
+}
+
+static int navi10_ih_resume(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ return navi10_ih_hw_init(adev);
+}
+
+static bool navi10_ih_is_idle(void *handle)
+{
+ /* todo */
+ return true;
+}
+
+static int navi10_ih_wait_for_idle(void *handle)
+{
+ /* todo */
+ return -ETIMEDOUT;
+}
+
+static int navi10_ih_soft_reset(void *handle)
+{
+ /* todo */
+ return 0;
+}
+
+static void navi10_ih_update_clockgating_state(struct amdgpu_device *adev,
+ bool enable)
+{
+ uint32_t data, def, field_val;
+
+ if (adev->cg_flags & AMD_CG_SUPPORT_IH_CG) {
+ def = data = RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL);
+ field_val = enable ? 0 : 1;
+ data = REG_SET_FIELD(data, IH_CLK_CTRL,
+ DBUS_MUX_CLK_SOFT_OVERRIDE, field_val);
+ data = REG_SET_FIELD(data, IH_CLK_CTRL,
+ OSSSYS_SHARE_CLK_SOFT_OVERRIDE, field_val);
+ data = REG_SET_FIELD(data, IH_CLK_CTRL,
+ LIMIT_SMN_CLK_SOFT_OVERRIDE, field_val);
+ data = REG_SET_FIELD(data, IH_CLK_CTRL,
+ DYN_CLK_SOFT_OVERRIDE, field_val);
+ data = REG_SET_FIELD(data, IH_CLK_CTRL,
+ REG_CLK_SOFT_OVERRIDE, field_val);
+ if (def != data)
+ WREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL, data);
+ }
+
+ return;
+}
+
+static int navi10_ih_set_clockgating_state(void *handle,
+ enum amd_clockgating_state state)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ navi10_ih_update_clockgating_state(adev,
+ state == AMD_CG_STATE_GATE ? true : false);
+ return 0;
+}
+
+static int navi10_ih_set_powergating_state(void *handle,
+ enum amd_powergating_state state)
+{
+ return 0;
+}
+
+static void navi10_ih_get_clockgating_state(void *handle, u32 *flags)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ if (!RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL))
+ *flags |= AMD_CG_SUPPORT_IH_CG;
+
+ return;
+}
+
+static const struct amd_ip_funcs navi10_ih_ip_funcs = {
+ .name = "navi10_ih",
+ .early_init = navi10_ih_early_init,
+ .late_init = NULL,
+ .sw_init = navi10_ih_sw_init,
+ .sw_fini = navi10_ih_sw_fini,
+ .hw_init = navi10_ih_hw_init,
+ .hw_fini = navi10_ih_hw_fini,
+ .suspend = navi10_ih_suspend,
+ .resume = navi10_ih_resume,
+ .is_idle = navi10_ih_is_idle,
+ .wait_for_idle = navi10_ih_wait_for_idle,
+ .soft_reset = navi10_ih_soft_reset,
+ .set_clockgating_state = navi10_ih_set_clockgating_state,
+ .set_powergating_state = navi10_ih_set_powergating_state,
+ .get_clockgating_state = navi10_ih_get_clockgating_state,
+};
+
+static const struct amdgpu_ih_funcs navi10_ih_funcs = {
+ .get_wptr = navi10_ih_get_wptr,
+ .decode_iv = navi10_ih_decode_iv,
+ .set_rptr = navi10_ih_set_rptr
+};
+
+static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
+{
+ if (adev->irq.ih_funcs == NULL)
+ adev->irq.ih_funcs = &navi10_ih_funcs;
+}
+
+const struct amdgpu_ip_block_version navi10_ih_ip_block =
+{
+ .type = AMD_IP_BLOCK_TYPE_IH,
+ .major = 5,
+ .minor = 0,
+ .rev = 0,
+ .funcs = &navi10_ih_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/navi10_ih.h b/drivers/gpu/drm/amd/amdgpu/navi10_ih.h
new file mode 100644
index 000000000000..140fbdaaed17
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/navi10_ih.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __NAVI10_IH_H__
+#define __NAVI10_IH_H__
+
+extern const struct amdgpu_ip_block_version navi10_ih_ip_block;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/navi10_reg_init.c b/drivers/gpu/drm/amd/amdgpu/navi10_reg_init.c
new file mode 100644
index 000000000000..55014ce8670a
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/navi10_reg_init.c
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "nv.h"
+
+#include "soc15_common.h"
+#include "soc15_hw_ip.h"
+#include "navi10_ip_offset.h"
+
+int navi10_reg_base_init(struct amdgpu_device *adev)
+{
+ int r, i;
+
+ if (amdgpu_discovery) {
+ r = amdgpu_discovery_reg_base_init(adev);
+ if (r) {
+ DRM_WARN("failed to init reg base from ip discovery table, "
+ "fallback to legacy init method\n");
+ goto legacy_init;
+ }
+
+ return 0;
+ }
+
+legacy_init:
+ for (i = 0 ; i < MAX_INSTANCE ; ++i) {
+ adev->reg_offset[GC_HWIP][i] = (uint32_t *)(&(GC_BASE.instance[i]));
+ adev->reg_offset[HDP_HWIP][i] = (uint32_t *)(&(HDP_BASE.instance[i]));
+ adev->reg_offset[MMHUB_HWIP][i] = (uint32_t *)(&(MMHUB_BASE.instance[i]));
+ adev->reg_offset[ATHUB_HWIP][i] = (uint32_t *)(&(ATHUB_BASE.instance[i]));
+ adev->reg_offset[NBIO_HWIP][i] = (uint32_t *)(&(NBIO_BASE.instance[i]));
+ adev->reg_offset[MP0_HWIP][i] = (uint32_t *)(&(MP0_BASE.instance[i]));
+ adev->reg_offset[MP1_HWIP][i] = (uint32_t *)(&(MP1_BASE.instance[i]));
+ adev->reg_offset[VCN_HWIP][i] = (uint32_t *)(&(VCN_BASE.instance[i]));
+ adev->reg_offset[DF_HWIP][i] = (uint32_t *)(&(DF_BASE.instance[i]));
+ adev->reg_offset[DCE_HWIP][i] = (uint32_t *)(&(DCN_BASE.instance[i]));
+ adev->reg_offset[OSSSYS_HWIP][i] = (uint32_t *)(&(OSSSYS_BASE.instance[i]));
+ adev->reg_offset[SDMA0_HWIP][i] = (uint32_t *)(&(GC_BASE.instance[i]));
+ adev->reg_offset[SDMA1_HWIP][i] = (uint32_t *)(&(GC_BASE.instance[i]));
+ adev->reg_offset[SMUIO_HWIP][i] = (uint32_t *)(&(SMUIO_BASE.instance[i]));
+ adev->reg_offset[THM_HWIP][i] = (uint32_t *)(&(THM_BASE.instance[i]));
+ adev->reg_offset[CLK_HWIP][i] = (uint32_t *)(&(CLK_BASE.instance[i]));
+ }
+
+ return 0;
+}
+
+
diff --git a/drivers/gpu/drm/amd/amdgpu/navi10_sdma_pkt_open.h b/drivers/gpu/drm/amd/amdgpu/navi10_sdma_pkt_open.h
new file mode 100644
index 000000000000..074a9a09c0a7
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/navi10_sdma_pkt_open.h
@@ -0,0 +1,4806 @@
+/*
+ * Copyright (C) 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __NAVI10_SDMA_PKT_OPEN_H_
+#define __NAVI10_SDMA_PKT_OPEN_H_
+
+#define SDMA_OP_NOP 0
+#define SDMA_OP_COPY 1
+#define SDMA_OP_WRITE 2
+#define SDMA_OP_INDIRECT 4
+#define SDMA_OP_FENCE 5
+#define SDMA_OP_TRAP 6
+#define SDMA_OP_SEM 7
+#define SDMA_OP_POLL_REGMEM 8
+#define SDMA_OP_COND_EXE 9
+#define SDMA_OP_ATOMIC 10
+#define SDMA_OP_CONST_FILL 11
+#define SDMA_OP_PTEPDE 12
+#define SDMA_OP_TIMESTAMP 13
+#define SDMA_OP_SRBM_WRITE 14
+#define SDMA_OP_PRE_EXE 15
+#define SDMA_OP_GPUVM_INV 16
+#define SDMA_OP_GCR_REQ 17
+#define SDMA_OP_DUMMY_TRAP 32
+#define SDMA_SUBOP_TIMESTAMP_SET 0
+#define SDMA_SUBOP_TIMESTAMP_GET 1
+#define SDMA_SUBOP_TIMESTAMP_GET_GLOBAL 2
+#define SDMA_SUBOP_COPY_LINEAR 0
+#define SDMA_SUBOP_COPY_LINEAR_SUB_WIND 4
+#define SDMA_SUBOP_COPY_TILED 1
+#define SDMA_SUBOP_COPY_TILED_SUB_WIND 5
+#define SDMA_SUBOP_COPY_T2T_SUB_WIND 6
+#define SDMA_SUBOP_COPY_SOA 3
+#define SDMA_SUBOP_COPY_DIRTY_PAGE 7
+#define SDMA_SUBOP_COPY_LINEAR_PHY 8
+#define SDMA_SUBOP_COPY_LINEAR_BC 16
+#define SDMA_SUBOP_COPY_TILED_BC 17
+#define SDMA_SUBOP_COPY_LINEAR_SUB_WIND_BC 20
+#define SDMA_SUBOP_COPY_TILED_SUB_WIND_BC 21
+#define SDMA_SUBOP_COPY_T2T_SUB_WIND_BC 22
+#define SDMA_SUBOP_WRITE_LINEAR 0
+#define SDMA_SUBOP_WRITE_TILED 1
+#define SDMA_SUBOP_WRITE_TILED_BC 17
+#define SDMA_SUBOP_PTEPDE_GEN 0
+#define SDMA_SUBOP_PTEPDE_COPY 1
+#define SDMA_SUBOP_PTEPDE_RMW 2
+#define SDMA_SUBOP_PTEPDE_COPY_BACKWARDS 3
+#define SDMA_SUBOP_DATA_FILL_MULTI 1
+#define SDMA_SUBOP_POLL_REG_WRITE_MEM 1
+#define SDMA_SUBOP_POLL_DBIT_WRITE_MEM 2
+#define SDMA_SUBOP_POLL_MEM_VERIFY 3
+#define HEADER_AGENT_DISPATCH 4
+#define HEADER_BARRIER 5
+#define SDMA_OP_AQL_COPY 0
+#define SDMA_OP_AQL_BARRIER_OR 0
+
+/*define for op field*/
+#define SDMA_PKT_HEADER_op_offset 0
+#define SDMA_PKT_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_HEADER_op_shift 0
+#define SDMA_PKT_HEADER_OP(x) (((x) & SDMA_PKT_HEADER_op_mask) << SDMA_PKT_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_HEADER_sub_op_offset 0
+#define SDMA_PKT_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_HEADER_sub_op_shift 8
+#define SDMA_PKT_HEADER_SUB_OP(x) (((x) & SDMA_PKT_HEADER_sub_op_mask) << SDMA_PKT_HEADER_sub_op_shift)
+
+/*
+** Definitions for SDMA_PKT_COPY_LINEAR packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_LINEAR_HEADER_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_COPY_LINEAR_HEADER_op_shift 0
+#define SDMA_PKT_COPY_LINEAR_HEADER_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_HEADER_op_mask) << SDMA_PKT_COPY_LINEAR_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_LINEAR_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_COPY_LINEAR_HEADER_sub_op_shift 8
+#define SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_HEADER_sub_op_mask) << SDMA_PKT_COPY_LINEAR_HEADER_sub_op_shift)
+
+/*define for encrypt field*/
+#define SDMA_PKT_COPY_LINEAR_HEADER_encrypt_offset 0
+#define SDMA_PKT_COPY_LINEAR_HEADER_encrypt_mask 0x00000001
+#define SDMA_PKT_COPY_LINEAR_HEADER_encrypt_shift 16
+#define SDMA_PKT_COPY_LINEAR_HEADER_ENCRYPT(x) (((x) & SDMA_PKT_COPY_LINEAR_HEADER_encrypt_mask) << SDMA_PKT_COPY_LINEAR_HEADER_encrypt_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_LINEAR_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_LINEAR_HEADER_tmz_mask 0x00000001
+#define SDMA_PKT_COPY_LINEAR_HEADER_tmz_shift 18
+#define SDMA_PKT_COPY_LINEAR_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_LINEAR_HEADER_tmz_mask) << SDMA_PKT_COPY_LINEAR_HEADER_tmz_shift)
+
+/*define for backwards field*/
+#define SDMA_PKT_COPY_LINEAR_HEADER_backwards_offset 0
+#define SDMA_PKT_COPY_LINEAR_HEADER_backwards_mask 0x00000001
+#define SDMA_PKT_COPY_LINEAR_HEADER_backwards_shift 25
+#define SDMA_PKT_COPY_LINEAR_HEADER_BACKWARDS(x) (((x) & SDMA_PKT_COPY_LINEAR_HEADER_backwards_mask) << SDMA_PKT_COPY_LINEAR_HEADER_backwards_shift)
+
+/*define for broadcast field*/
+#define SDMA_PKT_COPY_LINEAR_HEADER_broadcast_offset 0
+#define SDMA_PKT_COPY_LINEAR_HEADER_broadcast_mask 0x00000001
+#define SDMA_PKT_COPY_LINEAR_HEADER_broadcast_shift 27
+#define SDMA_PKT_COPY_LINEAR_HEADER_BROADCAST(x) (((x) & SDMA_PKT_COPY_LINEAR_HEADER_broadcast_mask) << SDMA_PKT_COPY_LINEAR_HEADER_broadcast_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_LINEAR_COUNT_count_offset 1
+#define SDMA_PKT_COPY_LINEAR_COUNT_count_mask 0x003FFFFF
+#define SDMA_PKT_COPY_LINEAR_COUNT_count_shift 0
+#define SDMA_PKT_COPY_LINEAR_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_LINEAR_COUNT_count_mask) << SDMA_PKT_COPY_LINEAR_COUNT_count_shift)
+
+/*define for PARAMETER word*/
+/*define for dst_sw field*/
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_dst_sw_offset 2
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_dst_sw_mask 0x00000003
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_dst_sw_shift 16
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_DST_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_PARAMETER_dst_sw_mask) << SDMA_PKT_COPY_LINEAR_PARAMETER_dst_sw_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_src_sw_offset 2
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_src_sw_mask 0x00000003
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_src_sw_shift 24
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_SRC_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_PARAMETER_src_sw_mask) << SDMA_PKT_COPY_LINEAR_PARAMETER_src_sw_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_offset 3
+#define SDMA_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_shift 0
+#define SDMA_PKT_COPY_LINEAR_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_offset 4
+#define SDMA_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_shift 0
+#define SDMA_PKT_COPY_LINEAR_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_offset 5
+#define SDMA_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_shift 0
+#define SDMA_PKT_COPY_LINEAR_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_offset 6
+#define SDMA_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_shift 0
+#define SDMA_PKT_COPY_LINEAR_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_LINEAR_BC packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_LINEAR_BC_HEADER_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_BC_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_COPY_LINEAR_BC_HEADER_op_shift 0
+#define SDMA_PKT_COPY_LINEAR_BC_HEADER_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_HEADER_op_mask) << SDMA_PKT_COPY_LINEAR_BC_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_LINEAR_BC_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_BC_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_COPY_LINEAR_BC_HEADER_sub_op_shift 8
+#define SDMA_PKT_COPY_LINEAR_BC_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_HEADER_sub_op_mask) << SDMA_PKT_COPY_LINEAR_BC_HEADER_sub_op_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_LINEAR_BC_COUNT_count_offset 1
+#define SDMA_PKT_COPY_LINEAR_BC_COUNT_count_mask 0x003FFFFF
+#define SDMA_PKT_COPY_LINEAR_BC_COUNT_count_shift 0
+#define SDMA_PKT_COPY_LINEAR_BC_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_COUNT_count_mask) << SDMA_PKT_COPY_LINEAR_BC_COUNT_count_shift)
+
+/*define for PARAMETER word*/
+/*define for dst_sw field*/
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_sw_offset 2
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_sw_mask 0x00000003
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_sw_shift 16
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_DST_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_sw_mask) << SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_sw_shift)
+
+/*define for dst_ha field*/
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_ha_offset 2
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_ha_mask 0x00000001
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_ha_shift 22
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_DST_HA(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_ha_mask) << SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_ha_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_sw_offset 2
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_sw_mask 0x00000003
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_sw_shift 24
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_SRC_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_sw_mask) << SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_sw_shift)
+
+/*define for src_ha field*/
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_ha_offset 2
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_ha_mask 0x00000001
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_ha_shift 30
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_SRC_HA(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_ha_mask) << SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_ha_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_LO_src_addr_31_0_offset 3
+#define SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_LO_src_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_LO_src_addr_31_0_shift 0
+#define SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_HI_src_addr_63_32_offset 4
+#define SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_HI_src_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_HI_src_addr_63_32_shift 0
+#define SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_LO_dst_addr_31_0_offset 5
+#define SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_LO_dst_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_LO_dst_addr_31_0_shift 0
+#define SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_HI_dst_addr_63_32_offset 6
+#define SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_HI_dst_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_HI_dst_addr_63_32_shift 0
+#define SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_HI_dst_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_DIRTY_PAGE packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_op_offset 0
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_op_shift 0
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_OP(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_HEADER_op_mask) << SDMA_PKT_COPY_DIRTY_PAGE_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_sub_op_shift 8
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_HEADER_sub_op_mask) << SDMA_PKT_COPY_DIRTY_PAGE_HEADER_sub_op_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_tmz_mask 0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_tmz_shift 18
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_HEADER_tmz_mask) << SDMA_PKT_COPY_DIRTY_PAGE_HEADER_tmz_shift)
+
+/*define for all field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_all_offset 0
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_all_mask 0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_all_shift 31
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_ALL(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_HEADER_all_mask) << SDMA_PKT_COPY_DIRTY_PAGE_HEADER_all_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_COUNT_count_offset 1
+#define SDMA_PKT_COPY_DIRTY_PAGE_COUNT_count_mask 0x003FFFFF
+#define SDMA_PKT_COPY_DIRTY_PAGE_COUNT_count_shift 0
+#define SDMA_PKT_COPY_DIRTY_PAGE_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_COUNT_count_mask) << SDMA_PKT_COPY_DIRTY_PAGE_COUNT_count_shift)
+
+/*define for PARAMETER word*/
+/*define for dst_mtype field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_mtype_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_mtype_mask 0x00000007
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_mtype_shift 3
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_DST_MTYPE(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_mtype_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_mtype_shift)
+
+/*define for dst_l2_policy field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_l2_policy_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_l2_policy_mask 0x00000003
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_l2_policy_shift 6
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_DST_L2_POLICY(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_l2_policy_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_l2_policy_shift)
+
+/*define for src_mtype field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_mtype_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_mtype_mask 0x00000007
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_mtype_shift 11
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_SRC_MTYPE(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_mtype_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_mtype_shift)
+
+/*define for src_l2_policy field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_l2_policy_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_l2_policy_mask 0x00000003
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_l2_policy_shift 14
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_SRC_L2_POLICY(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_l2_policy_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_l2_policy_shift)
+
+/*define for dst_sw field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sw_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sw_mask 0x00000003
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sw_shift 16
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_DST_SW(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sw_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sw_shift)
+
+/*define for dst_gcc field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gcc_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gcc_mask 0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gcc_shift 19
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_DST_GCC(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gcc_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gcc_shift)
+
+/*define for dst_sys field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sys_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sys_mask 0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sys_shift 20
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_DST_SYS(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sys_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sys_shift)
+
+/*define for dst_snoop field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_snoop_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_snoop_mask 0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_snoop_shift 22
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_DST_SNOOP(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_snoop_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_snoop_shift)
+
+/*define for dst_gpa field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gpa_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gpa_mask 0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gpa_shift 23
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_DST_GPA(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gpa_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gpa_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sw_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sw_mask 0x00000003
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sw_shift 24
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_SRC_SW(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sw_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sw_shift)
+
+/*define for src_sys field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sys_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sys_mask 0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sys_shift 28
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_SRC_SYS(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sys_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sys_shift)
+
+/*define for src_snoop field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_snoop_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_snoop_mask 0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_snoop_shift 30
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_SRC_SNOOP(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_snoop_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_snoop_shift)
+
+/*define for src_gpa field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_gpa_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_gpa_mask 0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_gpa_shift 31
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_SRC_GPA(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_gpa_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_gpa_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_LO_src_addr_31_0_offset 3
+#define SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_LO_src_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_LO_src_addr_31_0_shift 0
+#define SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_HI_src_addr_63_32_offset 4
+#define SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_HI_src_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_HI_src_addr_63_32_shift 0
+#define SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_LO_dst_addr_31_0_offset 5
+#define SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_LO_dst_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_LO_dst_addr_31_0_shift 0
+#define SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_HI_dst_addr_63_32_offset 6
+#define SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_HI_dst_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_HI_dst_addr_63_32_shift 0
+#define SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_HI_dst_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_PHYSICAL_LINEAR packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_op_offset 0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_op_shift 0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_OP(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_op_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_sub_op_shift 8
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_sub_op_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_sub_op_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_tmz_mask 0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_tmz_shift 18
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_tmz_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_tmz_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_count_offset 1
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_count_mask 0x003FFFFF
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_count_shift 0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_count_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_count_shift)
+
+/*define for PARAMETER word*/
+/*define for dst_mtype field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_mtype_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_mtype_mask 0x00000007
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_mtype_shift 3
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_DST_MTYPE(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_mtype_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_mtype_shift)
+
+/*define for dst_l2_policy field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_l2_policy_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_l2_policy_mask 0x00000003
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_l2_policy_shift 6
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_DST_L2_POLICY(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_l2_policy_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_l2_policy_shift)
+
+/*define for src_mtype field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_mtype_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_mtype_mask 0x00000007
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_mtype_shift 11
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_SRC_MTYPE(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_mtype_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_mtype_shift)
+
+/*define for src_l2_policy field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_l2_policy_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_l2_policy_mask 0x00000003
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_l2_policy_shift 14
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_SRC_L2_POLICY(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_l2_policy_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_l2_policy_shift)
+
+/*define for dst_sw field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sw_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sw_mask 0x00000003
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sw_shift 16
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_DST_SW(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sw_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sw_shift)
+
+/*define for dst_gcc field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gcc_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gcc_mask 0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gcc_shift 19
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_DST_GCC(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gcc_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gcc_shift)
+
+/*define for dst_sys field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sys_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sys_mask 0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sys_shift 20
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_DST_SYS(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sys_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sys_shift)
+
+/*define for dst_log field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_log_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_log_mask 0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_log_shift 21
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_DST_LOG(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_log_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_log_shift)
+
+/*define for dst_snoop field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_snoop_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_snoop_mask 0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_snoop_shift 22
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_DST_SNOOP(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_snoop_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_snoop_shift)
+
+/*define for dst_gpa field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gpa_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gpa_mask 0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gpa_shift 23
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_DST_GPA(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gpa_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gpa_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sw_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sw_mask 0x00000003
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sw_shift 24
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_SRC_SW(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sw_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sw_shift)
+
+/*define for src_gcc field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gcc_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gcc_mask 0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gcc_shift 27
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_SRC_GCC(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gcc_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gcc_shift)
+
+/*define for src_sys field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sys_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sys_mask 0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sys_shift 28
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_SRC_SYS(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sys_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sys_shift)
+
+/*define for src_snoop field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_snoop_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_snoop_mask 0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_snoop_shift 30
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_SRC_SNOOP(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_snoop_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_snoop_shift)
+
+/*define for src_gpa field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gpa_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gpa_mask 0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gpa_shift 31
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_SRC_GPA(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gpa_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gpa_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_LO_src_addr_31_0_offset 3
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_LO_src_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_LO_src_addr_31_0_shift 0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_HI_src_addr_63_32_offset 4
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_HI_src_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_HI_src_addr_63_32_shift 0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_LO_dst_addr_31_0_offset 5
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_LO_dst_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_LO_dst_addr_31_0_shift 0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_HI_dst_addr_63_32_offset 6
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_HI_dst_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_HI_dst_addr_63_32_shift 0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_HI_dst_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_BROADCAST_LINEAR packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_op_offset 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_op_shift 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_OP(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_op_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_sub_op_shift 8
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_sub_op_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_sub_op_shift)
+
+/*define for encrypt field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_encrypt_offset 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_encrypt_mask 0x00000001
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_encrypt_shift 16
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_ENCRYPT(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_encrypt_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_encrypt_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_tmz_mask 0x00000001
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_tmz_shift 18
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_tmz_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_tmz_shift)
+
+/*define for broadcast field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_broadcast_offset 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_broadcast_mask 0x00000001
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_broadcast_shift 27
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_BROADCAST(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_broadcast_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_broadcast_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_COUNT_count_offset 1
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_COUNT_count_mask 0x003FFFFF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_COUNT_count_shift 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_COUNT_count_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_COUNT_count_shift)
+
+/*define for PARAMETER word*/
+/*define for dst2_sw field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst2_sw_offset 2
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst2_sw_mask 0x00000003
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst2_sw_shift 8
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_DST2_SW(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst2_sw_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst2_sw_shift)
+
+/*define for dst1_sw field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst1_sw_offset 2
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst1_sw_mask 0x00000003
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst1_sw_shift 16
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_DST1_SW(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst1_sw_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst1_sw_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_src_sw_offset 2
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_src_sw_mask 0x00000003
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_src_sw_shift 24
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_SRC_SW(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_src_sw_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_src_sw_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_LO_src_addr_31_0_offset 3
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_LO_src_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_LO_src_addr_31_0_shift 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_HI_src_addr_63_32_offset 4
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_HI_src_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_HI_src_addr_63_32_shift 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DST1_ADDR_LO word*/
+/*define for dst1_addr_31_0 field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_LO_dst1_addr_31_0_offset 5
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_LO_dst1_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_LO_dst1_addr_31_0_shift 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_LO_DST1_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_LO_dst1_addr_31_0_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_LO_dst1_addr_31_0_shift)
+
+/*define for DST1_ADDR_HI word*/
+/*define for dst1_addr_63_32 field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_HI_dst1_addr_63_32_offset 6
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_HI_dst1_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_HI_dst1_addr_63_32_shift 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_HI_DST1_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_HI_dst1_addr_63_32_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_HI_dst1_addr_63_32_shift)
+
+/*define for DST2_ADDR_LO word*/
+/*define for dst2_addr_31_0 field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_LO_dst2_addr_31_0_offset 7
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_LO_dst2_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_LO_dst2_addr_31_0_shift 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_LO_DST2_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_LO_dst2_addr_31_0_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_LO_dst2_addr_31_0_shift)
+
+/*define for DST2_ADDR_HI word*/
+/*define for dst2_addr_63_32 field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_HI_dst2_addr_63_32_offset 8
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_HI_dst2_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_HI_dst2_addr_63_32_shift 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_HI_DST2_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_HI_dst2_addr_63_32_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_HI_dst2_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_LINEAR_SUBWIN packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_op_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_op_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_sub_op_shift 8
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_sub_op_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_sub_op_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_tmz_mask 0x00000001
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_tmz_shift 18
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_tmz_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_tmz_shift)
+
+/*define for elementsize field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_elementsize_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_elementsize_mask 0x00000007
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_elementsize_shift 29
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_ELEMENTSIZE(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_elementsize_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_elementsize_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_LO_src_addr_31_0_offset 1
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_LO_src_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_LO_src_addr_31_0_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_HI_src_addr_63_32_offset 2
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_HI_src_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_HI_src_addr_63_32_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for src_x field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_x_offset 3
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_x_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_SRC_X(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_x_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_x_shift)
+
+/*define for src_y field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_y_offset 3
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_y_shift 16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_SRC_Y(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_y_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_y_shift)
+
+/*define for DW_4 word*/
+/*define for src_z field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_z_offset 4
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_z_mask 0x00001FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_z_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_SRC_Z(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_z_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_z_shift)
+
+/*define for src_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_pitch_offset 4
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_pitch_mask 0x0007FFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_pitch_shift 13
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_SRC_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_pitch_shift)
+
+/*define for DW_5 word*/
+/*define for src_slice_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_5_src_slice_pitch_offset 5
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_5_src_slice_pitch_mask 0x0FFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_5_src_slice_pitch_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_5_SRC_SLICE_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_5_src_slice_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_5_src_slice_pitch_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_LO_dst_addr_31_0_offset 6
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_LO_dst_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_LO_dst_addr_31_0_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_HI_dst_addr_63_32_offset 7
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_HI_dst_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_HI_dst_addr_63_32_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for DW_8 word*/
+/*define for dst_x field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_x_offset 8
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_x_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_DST_X(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_x_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_x_shift)
+
+/*define for dst_y field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_y_offset 8
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_y_shift 16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_DST_Y(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_y_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_y_shift)
+
+/*define for DW_9 word*/
+/*define for dst_z field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_z_offset 9
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_z_mask 0x00001FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_z_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_DST_Z(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_z_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_z_shift)
+
+/*define for dst_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_pitch_offset 9
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_pitch_mask 0x0007FFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_pitch_shift 13
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_DST_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_pitch_shift)
+
+/*define for DW_10 word*/
+/*define for dst_slice_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_10_dst_slice_pitch_offset 10
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_10_dst_slice_pitch_mask 0x0FFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_10_dst_slice_pitch_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_10_DST_SLICE_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_10_dst_slice_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_10_dst_slice_pitch_shift)
+
+/*define for DW_11 word*/
+/*define for rect_x field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_x_offset 11
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_x_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_RECT_X(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_x_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_x_shift)
+
+/*define for rect_y field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_y_offset 11
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_y_shift 16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_RECT_Y(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_y_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_y_shift)
+
+/*define for DW_12 word*/
+/*define for rect_z field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_rect_z_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_rect_z_mask 0x00001FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_rect_z_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_RECT_Z(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_rect_z_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_rect_z_shift)
+
+/*define for dst_sw field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_dst_sw_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_dst_sw_mask 0x00000003
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_dst_sw_shift 16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_DST_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_dst_sw_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_dst_sw_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_src_sw_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_src_sw_mask 0x00000003
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_src_sw_shift 24
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_SRC_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_src_sw_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_src_sw_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_LINEAR_SUBWIN_BC packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_op_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_op_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_sub_op_shift 8
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_sub_op_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_sub_op_shift)
+
+/*define for elementsize field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_elementsize_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_elementsize_mask 0x00000007
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_elementsize_shift 29
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_ELEMENTSIZE(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_elementsize_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_elementsize_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_LO_src_addr_31_0_offset 1
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_LO_src_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_LO_src_addr_31_0_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_HI_src_addr_63_32_offset 2
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_HI_src_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_HI_src_addr_63_32_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for src_x field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_x_offset 3
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_x_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_SRC_X(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_x_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_x_shift)
+
+/*define for src_y field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_y_offset 3
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_y_shift 16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_SRC_Y(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_y_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_y_shift)
+
+/*define for DW_4 word*/
+/*define for src_z field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_z_offset 4
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_z_mask 0x000007FF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_z_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_SRC_Z(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_z_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_z_shift)
+
+/*define for src_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_pitch_offset 4
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_pitch_mask 0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_pitch_shift 13
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_SRC_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_pitch_shift)
+
+/*define for DW_5 word*/
+/*define for src_slice_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_5_src_slice_pitch_offset 5
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_5_src_slice_pitch_mask 0x0FFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_5_src_slice_pitch_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_5_SRC_SLICE_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_5_src_slice_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_5_src_slice_pitch_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_LO_dst_addr_31_0_offset 6
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_LO_dst_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_LO_dst_addr_31_0_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_HI_dst_addr_63_32_offset 7
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_HI_dst_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_HI_dst_addr_63_32_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for DW_8 word*/
+/*define for dst_x field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_x_offset 8
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_x_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_DST_X(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_x_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_x_shift)
+
+/*define for dst_y field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_y_offset 8
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_y_shift 16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_DST_Y(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_y_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_y_shift)
+
+/*define for DW_9 word*/
+/*define for dst_z field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_z_offset 9
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_z_mask 0x000007FF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_z_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_DST_Z(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_z_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_z_shift)
+
+/*define for dst_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_pitch_offset 9
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_pitch_mask 0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_pitch_shift 13
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_DST_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_pitch_shift)
+
+/*define for DW_10 word*/
+/*define for dst_slice_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_10_dst_slice_pitch_offset 10
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_10_dst_slice_pitch_mask 0x0FFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_10_dst_slice_pitch_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_10_DST_SLICE_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_10_dst_slice_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_10_dst_slice_pitch_shift)
+
+/*define for DW_11 word*/
+/*define for rect_x field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_x_offset 11
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_x_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_RECT_X(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_x_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_x_shift)
+
+/*define for rect_y field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_y_offset 11
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_y_shift 16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_RECT_Y(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_y_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_y_shift)
+
+/*define for DW_12 word*/
+/*define for rect_z field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_rect_z_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_rect_z_mask 0x000007FF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_rect_z_shift 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_RECT_Z(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_rect_z_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_rect_z_shift)
+
+/*define for dst_sw field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_sw_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_sw_mask 0x00000003
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_sw_shift 16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_DST_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_sw_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_sw_shift)
+
+/*define for dst_ha field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_ha_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_ha_mask 0x00000001
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_ha_shift 22
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_DST_HA(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_ha_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_ha_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_sw_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_sw_mask 0x00000003
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_sw_shift 24
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_SRC_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_sw_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_sw_shift)
+
+/*define for src_ha field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_ha_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_ha_mask 0x00000001
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_ha_shift 30
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_SRC_HA(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_ha_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_ha_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_TILED packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_TILED_HEADER_op_offset 0
+#define SDMA_PKT_COPY_TILED_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_COPY_TILED_HEADER_op_shift 0
+#define SDMA_PKT_COPY_TILED_HEADER_OP(x) (((x) & SDMA_PKT_COPY_TILED_HEADER_op_mask) << SDMA_PKT_COPY_TILED_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_TILED_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_TILED_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_COPY_TILED_HEADER_sub_op_shift 8
+#define SDMA_PKT_COPY_TILED_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_TILED_HEADER_sub_op_mask) << SDMA_PKT_COPY_TILED_HEADER_sub_op_shift)
+
+/*define for encrypt field*/
+#define SDMA_PKT_COPY_TILED_HEADER_encrypt_offset 0
+#define SDMA_PKT_COPY_TILED_HEADER_encrypt_mask 0x00000001
+#define SDMA_PKT_COPY_TILED_HEADER_encrypt_shift 16
+#define SDMA_PKT_COPY_TILED_HEADER_ENCRYPT(x) (((x) & SDMA_PKT_COPY_TILED_HEADER_encrypt_mask) << SDMA_PKT_COPY_TILED_HEADER_encrypt_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_TILED_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_TILED_HEADER_tmz_mask 0x00000001
+#define SDMA_PKT_COPY_TILED_HEADER_tmz_shift 18
+#define SDMA_PKT_COPY_TILED_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_TILED_HEADER_tmz_mask) << SDMA_PKT_COPY_TILED_HEADER_tmz_shift)
+
+/*define for detile field*/
+#define SDMA_PKT_COPY_TILED_HEADER_detile_offset 0
+#define SDMA_PKT_COPY_TILED_HEADER_detile_mask 0x00000001
+#define SDMA_PKT_COPY_TILED_HEADER_detile_shift 31
+#define SDMA_PKT_COPY_TILED_HEADER_DETILE(x) (((x) & SDMA_PKT_COPY_TILED_HEADER_detile_mask) << SDMA_PKT_COPY_TILED_HEADER_detile_shift)
+
+/*define for TILED_ADDR_LO word*/
+/*define for tiled_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_TILED_ADDR_LO_tiled_addr_31_0_offset 1
+#define SDMA_PKT_COPY_TILED_TILED_ADDR_LO_tiled_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_TILED_ADDR_LO_tiled_addr_31_0_shift 0
+#define SDMA_PKT_COPY_TILED_TILED_ADDR_LO_TILED_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_TILED_ADDR_LO_tiled_addr_31_0_mask) << SDMA_PKT_COPY_TILED_TILED_ADDR_LO_tiled_addr_31_0_shift)
+
+/*define for TILED_ADDR_HI word*/
+/*define for tiled_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_TILED_ADDR_HI_tiled_addr_63_32_offset 2
+#define SDMA_PKT_COPY_TILED_TILED_ADDR_HI_tiled_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_TILED_ADDR_HI_tiled_addr_63_32_shift 0
+#define SDMA_PKT_COPY_TILED_TILED_ADDR_HI_TILED_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_TILED_ADDR_HI_tiled_addr_63_32_mask) << SDMA_PKT_COPY_TILED_TILED_ADDR_HI_tiled_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for width field*/
+#define SDMA_PKT_COPY_TILED_DW_3_width_offset 3
+#define SDMA_PKT_COPY_TILED_DW_3_width_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_DW_3_width_shift 0
+#define SDMA_PKT_COPY_TILED_DW_3_WIDTH(x) (((x) & SDMA_PKT_COPY_TILED_DW_3_width_mask) << SDMA_PKT_COPY_TILED_DW_3_width_shift)
+
+/*define for DW_4 word*/
+/*define for height field*/
+#define SDMA_PKT_COPY_TILED_DW_4_height_offset 4
+#define SDMA_PKT_COPY_TILED_DW_4_height_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_DW_4_height_shift 0
+#define SDMA_PKT_COPY_TILED_DW_4_HEIGHT(x) (((x) & SDMA_PKT_COPY_TILED_DW_4_height_mask) << SDMA_PKT_COPY_TILED_DW_4_height_shift)
+
+/*define for depth field*/
+#define SDMA_PKT_COPY_TILED_DW_4_depth_offset 4
+#define SDMA_PKT_COPY_TILED_DW_4_depth_mask 0x00001FFF
+#define SDMA_PKT_COPY_TILED_DW_4_depth_shift 16
+#define SDMA_PKT_COPY_TILED_DW_4_DEPTH(x) (((x) & SDMA_PKT_COPY_TILED_DW_4_depth_mask) << SDMA_PKT_COPY_TILED_DW_4_depth_shift)
+
+/*define for DW_5 word*/
+/*define for element_size field*/
+#define SDMA_PKT_COPY_TILED_DW_5_element_size_offset 5
+#define SDMA_PKT_COPY_TILED_DW_5_element_size_mask 0x00000007
+#define SDMA_PKT_COPY_TILED_DW_5_element_size_shift 0
+#define SDMA_PKT_COPY_TILED_DW_5_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_TILED_DW_5_element_size_mask) << SDMA_PKT_COPY_TILED_DW_5_element_size_shift)
+
+/*define for swizzle_mode field*/
+#define SDMA_PKT_COPY_TILED_DW_5_swizzle_mode_offset 5
+#define SDMA_PKT_COPY_TILED_DW_5_swizzle_mode_mask 0x0000001F
+#define SDMA_PKT_COPY_TILED_DW_5_swizzle_mode_shift 3
+#define SDMA_PKT_COPY_TILED_DW_5_SWIZZLE_MODE(x) (((x) & SDMA_PKT_COPY_TILED_DW_5_swizzle_mode_mask) << SDMA_PKT_COPY_TILED_DW_5_swizzle_mode_shift)
+
+/*define for dimension field*/
+#define SDMA_PKT_COPY_TILED_DW_5_dimension_offset 5
+#define SDMA_PKT_COPY_TILED_DW_5_dimension_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_DW_5_dimension_shift 9
+#define SDMA_PKT_COPY_TILED_DW_5_DIMENSION(x) (((x) & SDMA_PKT_COPY_TILED_DW_5_dimension_mask) << SDMA_PKT_COPY_TILED_DW_5_dimension_shift)
+
+/*define for mip_max field*/
+#define SDMA_PKT_COPY_TILED_DW_5_mip_max_offset 5
+#define SDMA_PKT_COPY_TILED_DW_5_mip_max_mask 0x0000000F
+#define SDMA_PKT_COPY_TILED_DW_5_mip_max_shift 16
+#define SDMA_PKT_COPY_TILED_DW_5_MIP_MAX(x) (((x) & SDMA_PKT_COPY_TILED_DW_5_mip_max_mask) << SDMA_PKT_COPY_TILED_DW_5_mip_max_shift)
+
+/*define for DW_6 word*/
+/*define for x field*/
+#define SDMA_PKT_COPY_TILED_DW_6_x_offset 6
+#define SDMA_PKT_COPY_TILED_DW_6_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_DW_6_x_shift 0
+#define SDMA_PKT_COPY_TILED_DW_6_X(x) (((x) & SDMA_PKT_COPY_TILED_DW_6_x_mask) << SDMA_PKT_COPY_TILED_DW_6_x_shift)
+
+/*define for y field*/
+#define SDMA_PKT_COPY_TILED_DW_6_y_offset 6
+#define SDMA_PKT_COPY_TILED_DW_6_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_DW_6_y_shift 16
+#define SDMA_PKT_COPY_TILED_DW_6_Y(x) (((x) & SDMA_PKT_COPY_TILED_DW_6_y_mask) << SDMA_PKT_COPY_TILED_DW_6_y_shift)
+
+/*define for DW_7 word*/
+/*define for z field*/
+#define SDMA_PKT_COPY_TILED_DW_7_z_offset 7
+#define SDMA_PKT_COPY_TILED_DW_7_z_mask 0x00001FFF
+#define SDMA_PKT_COPY_TILED_DW_7_z_shift 0
+#define SDMA_PKT_COPY_TILED_DW_7_Z(x) (((x) & SDMA_PKT_COPY_TILED_DW_7_z_mask) << SDMA_PKT_COPY_TILED_DW_7_z_shift)
+
+/*define for linear_sw field*/
+#define SDMA_PKT_COPY_TILED_DW_7_linear_sw_offset 7
+#define SDMA_PKT_COPY_TILED_DW_7_linear_sw_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_DW_7_linear_sw_shift 16
+#define SDMA_PKT_COPY_TILED_DW_7_LINEAR_SW(x) (((x) & SDMA_PKT_COPY_TILED_DW_7_linear_sw_mask) << SDMA_PKT_COPY_TILED_DW_7_linear_sw_shift)
+
+/*define for linear_cc field*/
+#define SDMA_PKT_COPY_TILED_DW_7_linear_cc_offset 7
+#define SDMA_PKT_COPY_TILED_DW_7_linear_cc_mask 0x00000001
+#define SDMA_PKT_COPY_TILED_DW_7_linear_cc_shift 20
+#define SDMA_PKT_COPY_TILED_DW_7_LINEAR_CC(x) (((x) & SDMA_PKT_COPY_TILED_DW_7_linear_cc_mask) << SDMA_PKT_COPY_TILED_DW_7_linear_cc_shift)
+
+/*define for tile_sw field*/
+#define SDMA_PKT_COPY_TILED_DW_7_tile_sw_offset 7
+#define SDMA_PKT_COPY_TILED_DW_7_tile_sw_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_DW_7_tile_sw_shift 24
+#define SDMA_PKT_COPY_TILED_DW_7_TILE_SW(x) (((x) & SDMA_PKT_COPY_TILED_DW_7_tile_sw_mask) << SDMA_PKT_COPY_TILED_DW_7_tile_sw_shift)
+
+/*define for LINEAR_ADDR_LO word*/
+/*define for linear_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_LINEAR_ADDR_LO_linear_addr_31_0_offset 8
+#define SDMA_PKT_COPY_TILED_LINEAR_ADDR_LO_linear_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_LINEAR_ADDR_LO_linear_addr_31_0_shift 0
+#define SDMA_PKT_COPY_TILED_LINEAR_ADDR_LO_LINEAR_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_LINEAR_ADDR_LO_linear_addr_31_0_mask) << SDMA_PKT_COPY_TILED_LINEAR_ADDR_LO_linear_addr_31_0_shift)
+
+/*define for LINEAR_ADDR_HI word*/
+/*define for linear_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_LINEAR_ADDR_HI_linear_addr_63_32_offset 9
+#define SDMA_PKT_COPY_TILED_LINEAR_ADDR_HI_linear_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_LINEAR_ADDR_HI_linear_addr_63_32_shift 0
+#define SDMA_PKT_COPY_TILED_LINEAR_ADDR_HI_LINEAR_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_LINEAR_ADDR_HI_linear_addr_63_32_mask) << SDMA_PKT_COPY_TILED_LINEAR_ADDR_HI_linear_addr_63_32_shift)
+
+/*define for LINEAR_PITCH word*/
+/*define for linear_pitch field*/
+#define SDMA_PKT_COPY_TILED_LINEAR_PITCH_linear_pitch_offset 10
+#define SDMA_PKT_COPY_TILED_LINEAR_PITCH_linear_pitch_mask 0x0007FFFF
+#define SDMA_PKT_COPY_TILED_LINEAR_PITCH_linear_pitch_shift 0
+#define SDMA_PKT_COPY_TILED_LINEAR_PITCH_LINEAR_PITCH(x) (((x) & SDMA_PKT_COPY_TILED_LINEAR_PITCH_linear_pitch_mask) << SDMA_PKT_COPY_TILED_LINEAR_PITCH_linear_pitch_shift)
+
+/*define for LINEAR_SLICE_PITCH word*/
+/*define for linear_slice_pitch field*/
+#define SDMA_PKT_COPY_TILED_LINEAR_SLICE_PITCH_linear_slice_pitch_offset 11
+#define SDMA_PKT_COPY_TILED_LINEAR_SLICE_PITCH_linear_slice_pitch_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_LINEAR_SLICE_PITCH_linear_slice_pitch_shift 0
+#define SDMA_PKT_COPY_TILED_LINEAR_SLICE_PITCH_LINEAR_SLICE_PITCH(x) (((x) & SDMA_PKT_COPY_TILED_LINEAR_SLICE_PITCH_linear_slice_pitch_mask) << SDMA_PKT_COPY_TILED_LINEAR_SLICE_PITCH_linear_slice_pitch_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_TILED_COUNT_count_offset 12
+#define SDMA_PKT_COPY_TILED_COUNT_count_mask 0x003FFFFF
+#define SDMA_PKT_COPY_TILED_COUNT_count_shift 0
+#define SDMA_PKT_COPY_TILED_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_TILED_COUNT_count_mask) << SDMA_PKT_COPY_TILED_COUNT_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_TILED_BC packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_TILED_BC_HEADER_op_offset 0
+#define SDMA_PKT_COPY_TILED_BC_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_COPY_TILED_BC_HEADER_op_shift 0
+#define SDMA_PKT_COPY_TILED_BC_HEADER_OP(x) (((x) & SDMA_PKT_COPY_TILED_BC_HEADER_op_mask) << SDMA_PKT_COPY_TILED_BC_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_TILED_BC_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_TILED_BC_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_COPY_TILED_BC_HEADER_sub_op_shift 8
+#define SDMA_PKT_COPY_TILED_BC_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_TILED_BC_HEADER_sub_op_mask) << SDMA_PKT_COPY_TILED_BC_HEADER_sub_op_shift)
+
+/*define for detile field*/
+#define SDMA_PKT_COPY_TILED_BC_HEADER_detile_offset 0
+#define SDMA_PKT_COPY_TILED_BC_HEADER_detile_mask 0x00000001
+#define SDMA_PKT_COPY_TILED_BC_HEADER_detile_shift 31
+#define SDMA_PKT_COPY_TILED_BC_HEADER_DETILE(x) (((x) & SDMA_PKT_COPY_TILED_BC_HEADER_detile_mask) << SDMA_PKT_COPY_TILED_BC_HEADER_detile_shift)
+
+/*define for TILED_ADDR_LO word*/
+/*define for tiled_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_BC_TILED_ADDR_LO_tiled_addr_31_0_offset 1
+#define SDMA_PKT_COPY_TILED_BC_TILED_ADDR_LO_tiled_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_BC_TILED_ADDR_LO_tiled_addr_31_0_shift 0
+#define SDMA_PKT_COPY_TILED_BC_TILED_ADDR_LO_TILED_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_BC_TILED_ADDR_LO_tiled_addr_31_0_mask) << SDMA_PKT_COPY_TILED_BC_TILED_ADDR_LO_tiled_addr_31_0_shift)
+
+/*define for TILED_ADDR_HI word*/
+/*define for tiled_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_BC_TILED_ADDR_HI_tiled_addr_63_32_offset 2
+#define SDMA_PKT_COPY_TILED_BC_TILED_ADDR_HI_tiled_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_BC_TILED_ADDR_HI_tiled_addr_63_32_shift 0
+#define SDMA_PKT_COPY_TILED_BC_TILED_ADDR_HI_TILED_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_BC_TILED_ADDR_HI_tiled_addr_63_32_mask) << SDMA_PKT_COPY_TILED_BC_TILED_ADDR_HI_tiled_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for width field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_3_width_offset 3
+#define SDMA_PKT_COPY_TILED_BC_DW_3_width_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_BC_DW_3_width_shift 0
+#define SDMA_PKT_COPY_TILED_BC_DW_3_WIDTH(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_3_width_mask) << SDMA_PKT_COPY_TILED_BC_DW_3_width_shift)
+
+/*define for DW_4 word*/
+/*define for height field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_4_height_offset 4
+#define SDMA_PKT_COPY_TILED_BC_DW_4_height_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_BC_DW_4_height_shift 0
+#define SDMA_PKT_COPY_TILED_BC_DW_4_HEIGHT(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_4_height_mask) << SDMA_PKT_COPY_TILED_BC_DW_4_height_shift)
+
+/*define for depth field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_4_depth_offset 4
+#define SDMA_PKT_COPY_TILED_BC_DW_4_depth_mask 0x000007FF
+#define SDMA_PKT_COPY_TILED_BC_DW_4_depth_shift 16
+#define SDMA_PKT_COPY_TILED_BC_DW_4_DEPTH(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_4_depth_mask) << SDMA_PKT_COPY_TILED_BC_DW_4_depth_shift)
+
+/*define for DW_5 word*/
+/*define for element_size field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_element_size_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_element_size_mask 0x00000007
+#define SDMA_PKT_COPY_TILED_BC_DW_5_element_size_shift 0
+#define SDMA_PKT_COPY_TILED_BC_DW_5_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_element_size_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_element_size_shift)
+
+/*define for array_mode field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_array_mode_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_array_mode_mask 0x0000000F
+#define SDMA_PKT_COPY_TILED_BC_DW_5_array_mode_shift 3
+#define SDMA_PKT_COPY_TILED_BC_DW_5_ARRAY_MODE(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_array_mode_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_array_mode_shift)
+
+/*define for mit_mode field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_mit_mode_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_mit_mode_mask 0x00000007
+#define SDMA_PKT_COPY_TILED_BC_DW_5_mit_mode_shift 8
+#define SDMA_PKT_COPY_TILED_BC_DW_5_MIT_MODE(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_mit_mode_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_mit_mode_shift)
+
+/*define for tilesplit_size field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_tilesplit_size_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_tilesplit_size_mask 0x00000007
+#define SDMA_PKT_COPY_TILED_BC_DW_5_tilesplit_size_shift 11
+#define SDMA_PKT_COPY_TILED_BC_DW_5_TILESPLIT_SIZE(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_tilesplit_size_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_tilesplit_size_shift)
+
+/*define for bank_w field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_bank_w_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_bank_w_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_BC_DW_5_bank_w_shift 15
+#define SDMA_PKT_COPY_TILED_BC_DW_5_BANK_W(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_bank_w_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_bank_w_shift)
+
+/*define for bank_h field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_bank_h_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_bank_h_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_BC_DW_5_bank_h_shift 18
+#define SDMA_PKT_COPY_TILED_BC_DW_5_BANK_H(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_bank_h_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_bank_h_shift)
+
+/*define for num_bank field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_num_bank_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_num_bank_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_BC_DW_5_num_bank_shift 21
+#define SDMA_PKT_COPY_TILED_BC_DW_5_NUM_BANK(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_num_bank_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_num_bank_shift)
+
+/*define for mat_aspt field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_mat_aspt_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_mat_aspt_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_BC_DW_5_mat_aspt_shift 24
+#define SDMA_PKT_COPY_TILED_BC_DW_5_MAT_ASPT(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_mat_aspt_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_mat_aspt_shift)
+
+/*define for pipe_config field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_pipe_config_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_pipe_config_mask 0x0000001F
+#define SDMA_PKT_COPY_TILED_BC_DW_5_pipe_config_shift 26
+#define SDMA_PKT_COPY_TILED_BC_DW_5_PIPE_CONFIG(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_pipe_config_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_pipe_config_shift)
+
+/*define for DW_6 word*/
+/*define for x field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_6_x_offset 6
+#define SDMA_PKT_COPY_TILED_BC_DW_6_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_BC_DW_6_x_shift 0
+#define SDMA_PKT_COPY_TILED_BC_DW_6_X(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_6_x_mask) << SDMA_PKT_COPY_TILED_BC_DW_6_x_shift)
+
+/*define for y field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_6_y_offset 6
+#define SDMA_PKT_COPY_TILED_BC_DW_6_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_BC_DW_6_y_shift 16
+#define SDMA_PKT_COPY_TILED_BC_DW_6_Y(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_6_y_mask) << SDMA_PKT_COPY_TILED_BC_DW_6_y_shift)
+
+/*define for DW_7 word*/
+/*define for z field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_7_z_offset 7
+#define SDMA_PKT_COPY_TILED_BC_DW_7_z_mask 0x000007FF
+#define SDMA_PKT_COPY_TILED_BC_DW_7_z_shift 0
+#define SDMA_PKT_COPY_TILED_BC_DW_7_Z(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_7_z_mask) << SDMA_PKT_COPY_TILED_BC_DW_7_z_shift)
+
+/*define for linear_sw field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_7_linear_sw_offset 7
+#define SDMA_PKT_COPY_TILED_BC_DW_7_linear_sw_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_BC_DW_7_linear_sw_shift 16
+#define SDMA_PKT_COPY_TILED_BC_DW_7_LINEAR_SW(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_7_linear_sw_mask) << SDMA_PKT_COPY_TILED_BC_DW_7_linear_sw_shift)
+
+/*define for tile_sw field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_7_tile_sw_offset 7
+#define SDMA_PKT_COPY_TILED_BC_DW_7_tile_sw_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_BC_DW_7_tile_sw_shift 24
+#define SDMA_PKT_COPY_TILED_BC_DW_7_TILE_SW(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_7_tile_sw_mask) << SDMA_PKT_COPY_TILED_BC_DW_7_tile_sw_shift)
+
+/*define for LINEAR_ADDR_LO word*/
+/*define for linear_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_LO_linear_addr_31_0_offset 8
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_LO_linear_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_LO_linear_addr_31_0_shift 0
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_LO_LINEAR_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_LO_linear_addr_31_0_mask) << SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_LO_linear_addr_31_0_shift)
+
+/*define for LINEAR_ADDR_HI word*/
+/*define for linear_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_HI_linear_addr_63_32_offset 9
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_HI_linear_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_HI_linear_addr_63_32_shift 0
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_HI_LINEAR_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_HI_linear_addr_63_32_mask) << SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_HI_linear_addr_63_32_shift)
+
+/*define for LINEAR_PITCH word*/
+/*define for linear_pitch field*/
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_PITCH_linear_pitch_offset 10
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_PITCH_linear_pitch_mask 0x0007FFFF
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_PITCH_linear_pitch_shift 0
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_PITCH_LINEAR_PITCH(x) (((x) & SDMA_PKT_COPY_TILED_BC_LINEAR_PITCH_linear_pitch_mask) << SDMA_PKT_COPY_TILED_BC_LINEAR_PITCH_linear_pitch_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_TILED_BC_COUNT_count_offset 11
+#define SDMA_PKT_COPY_TILED_BC_COUNT_count_mask 0x000FFFFF
+#define SDMA_PKT_COPY_TILED_BC_COUNT_count_shift 2
+#define SDMA_PKT_COPY_TILED_BC_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_TILED_BC_COUNT_count_mask) << SDMA_PKT_COPY_TILED_BC_COUNT_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_L2T_BROADCAST packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_op_offset 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_op_shift 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_OP(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_HEADER_op_mask) << SDMA_PKT_COPY_L2T_BROADCAST_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_sub_op_shift 8
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_HEADER_sub_op_mask) << SDMA_PKT_COPY_L2T_BROADCAST_HEADER_sub_op_shift)
+
+/*define for encrypt field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_encrypt_offset 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_encrypt_mask 0x00000001
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_encrypt_shift 16
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_ENCRYPT(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_HEADER_encrypt_mask) << SDMA_PKT_COPY_L2T_BROADCAST_HEADER_encrypt_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_tmz_mask 0x00000001
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_tmz_shift 18
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_HEADER_tmz_mask) << SDMA_PKT_COPY_L2T_BROADCAST_HEADER_tmz_shift)
+
+/*define for videocopy field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_videocopy_offset 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_videocopy_mask 0x00000001
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_videocopy_shift 26
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_VIDEOCOPY(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_HEADER_videocopy_mask) << SDMA_PKT_COPY_L2T_BROADCAST_HEADER_videocopy_shift)
+
+/*define for broadcast field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_broadcast_offset 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_broadcast_mask 0x00000001
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_broadcast_shift 27
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_BROADCAST(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_HEADER_broadcast_mask) << SDMA_PKT_COPY_L2T_BROADCAST_HEADER_broadcast_shift)
+
+/*define for TILED_ADDR_LO_0 word*/
+/*define for tiled_addr0_31_0 field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_0_tiled_addr0_31_0_offset 1
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_0_tiled_addr0_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_0_tiled_addr0_31_0_shift 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_0_TILED_ADDR0_31_0(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_0_tiled_addr0_31_0_mask) << SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_0_tiled_addr0_31_0_shift)
+
+/*define for TILED_ADDR_HI_0 word*/
+/*define for tiled_addr0_63_32 field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_0_tiled_addr0_63_32_offset 2
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_0_tiled_addr0_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_0_tiled_addr0_63_32_shift 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_0_TILED_ADDR0_63_32(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_0_tiled_addr0_63_32_mask) << SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_0_tiled_addr0_63_32_shift)
+
+/*define for TILED_ADDR_LO_1 word*/
+/*define for tiled_addr1_31_0 field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_1_tiled_addr1_31_0_offset 3
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_1_tiled_addr1_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_1_tiled_addr1_31_0_shift 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_1_TILED_ADDR1_31_0(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_1_tiled_addr1_31_0_mask) << SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_1_tiled_addr1_31_0_shift)
+
+/*define for TILED_ADDR_HI_1 word*/
+/*define for tiled_addr1_63_32 field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_1_tiled_addr1_63_32_offset 4
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_1_tiled_addr1_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_1_tiled_addr1_63_32_shift 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_1_TILED_ADDR1_63_32(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_1_tiled_addr1_63_32_mask) << SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_1_tiled_addr1_63_32_shift)
+
+/*define for DW_5 word*/
+/*define for width field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_5_width_offset 5
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_5_width_mask 0x00003FFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_5_width_shift 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_5_WIDTH(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_5_width_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_5_width_shift)
+
+/*define for DW_6 word*/
+/*define for height field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_6_height_offset 6
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_6_height_mask 0x00003FFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_6_height_shift 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_6_HEIGHT(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_6_height_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_6_height_shift)
+
+/*define for depth field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_6_depth_offset 6
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_6_depth_mask 0x00001FFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_6_depth_shift 16
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_6_DEPTH(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_6_depth_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_6_depth_shift)
+
+/*define for DW_7 word*/
+/*define for element_size field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_element_size_offset 7
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_element_size_mask 0x00000007
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_element_size_shift 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_7_element_size_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_7_element_size_shift)
+
+/*define for swizzle_mode field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_swizzle_mode_offset 7
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_swizzle_mode_mask 0x0000001F
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_swizzle_mode_shift 3
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_SWIZZLE_MODE(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_7_swizzle_mode_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_7_swizzle_mode_shift)
+
+/*define for dimension field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_dimension_offset 7
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_dimension_mask 0x00000003
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_dimension_shift 9
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_DIMENSION(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_7_dimension_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_7_dimension_shift)
+
+/*define for mip_max field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_mip_max_offset 7
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_mip_max_mask 0x0000000F
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_mip_max_shift 16
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_MIP_MAX(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_7_mip_max_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_7_mip_max_shift)
+
+/*define for DW_8 word*/
+/*define for x field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_8_x_offset 8
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_8_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_8_x_shift 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_8_X(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_8_x_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_8_x_shift)
+
+/*define for y field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_8_y_offset 8
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_8_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_8_y_shift 16
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_8_Y(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_8_y_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_8_y_shift)
+
+/*define for DW_9 word*/
+/*define for z field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_9_z_offset 9
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_9_z_mask 0x00001FFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_9_z_shift 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_9_Z(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_9_z_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_9_z_shift)
+
+/*define for DW_10 word*/
+/*define for dst2_sw field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_dst2_sw_offset 10
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_dst2_sw_mask 0x00000003
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_dst2_sw_shift 8
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_DST2_SW(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_10_dst2_sw_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_10_dst2_sw_shift)
+
+/*define for linear_sw field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_linear_sw_offset 10
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_linear_sw_mask 0x00000003
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_linear_sw_shift 16
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_LINEAR_SW(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_10_linear_sw_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_10_linear_sw_shift)
+
+/*define for tile_sw field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_tile_sw_offset 10
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_tile_sw_mask 0x00000003
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_tile_sw_shift 24
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_TILE_SW(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_10_tile_sw_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_10_tile_sw_shift)
+
+/*define for LINEAR_ADDR_LO word*/
+/*define for linear_addr_31_0 field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_LO_linear_addr_31_0_offset 11
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_LO_linear_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_LO_linear_addr_31_0_shift 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_LO_LINEAR_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_LO_linear_addr_31_0_mask) << SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_LO_linear_addr_31_0_shift)
+
+/*define for LINEAR_ADDR_HI word*/
+/*define for linear_addr_63_32 field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_HI_linear_addr_63_32_offset 12
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_HI_linear_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_HI_linear_addr_63_32_shift 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_HI_LINEAR_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_HI_linear_addr_63_32_mask) << SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_HI_linear_addr_63_32_shift)
+
+/*define for LINEAR_PITCH word*/
+/*define for linear_pitch field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_PITCH_linear_pitch_offset 13
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_PITCH_linear_pitch_mask 0x0007FFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_PITCH_linear_pitch_shift 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_PITCH_LINEAR_PITCH(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_PITCH_linear_pitch_mask) << SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_PITCH_linear_pitch_shift)
+
+/*define for LINEAR_SLICE_PITCH word*/
+/*define for linear_slice_pitch field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_SLICE_PITCH_linear_slice_pitch_offset 14
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_SLICE_PITCH_linear_slice_pitch_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_SLICE_PITCH_linear_slice_pitch_shift 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_SLICE_PITCH_LINEAR_SLICE_PITCH(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_SLICE_PITCH_linear_slice_pitch_mask) << SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_SLICE_PITCH_linear_slice_pitch_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_COUNT_count_offset 15
+#define SDMA_PKT_COPY_L2T_BROADCAST_COUNT_count_mask 0x003FFFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_COUNT_count_shift 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_COUNT_count_mask) << SDMA_PKT_COPY_L2T_BROADCAST_COUNT_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_T2T packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_T2T_HEADER_op_offset 0
+#define SDMA_PKT_COPY_T2T_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_COPY_T2T_HEADER_op_shift 0
+#define SDMA_PKT_COPY_T2T_HEADER_OP(x) (((x) & SDMA_PKT_COPY_T2T_HEADER_op_mask) << SDMA_PKT_COPY_T2T_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_T2T_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_T2T_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_COPY_T2T_HEADER_sub_op_shift 8
+#define SDMA_PKT_COPY_T2T_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_T2T_HEADER_sub_op_mask) << SDMA_PKT_COPY_T2T_HEADER_sub_op_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_T2T_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_T2T_HEADER_tmz_mask 0x00000001
+#define SDMA_PKT_COPY_T2T_HEADER_tmz_shift 18
+#define SDMA_PKT_COPY_T2T_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_T2T_HEADER_tmz_mask) << SDMA_PKT_COPY_T2T_HEADER_tmz_shift)
+
+/*define for dcc field*/
+#define SDMA_PKT_COPY_T2T_HEADER_dcc_offset 0
+#define SDMA_PKT_COPY_T2T_HEADER_dcc_mask 0x00000001
+#define SDMA_PKT_COPY_T2T_HEADER_dcc_shift 19
+#define SDMA_PKT_COPY_T2T_HEADER_DCC(x) (((x) & SDMA_PKT_COPY_T2T_HEADER_dcc_mask) << SDMA_PKT_COPY_T2T_HEADER_dcc_shift)
+
+/*define for dcc_dir field*/
+#define SDMA_PKT_COPY_T2T_HEADER_dcc_dir_offset 0
+#define SDMA_PKT_COPY_T2T_HEADER_dcc_dir_mask 0x00000001
+#define SDMA_PKT_COPY_T2T_HEADER_dcc_dir_shift 31
+#define SDMA_PKT_COPY_T2T_HEADER_DCC_DIR(x) (((x) & SDMA_PKT_COPY_T2T_HEADER_dcc_dir_mask) << SDMA_PKT_COPY_T2T_HEADER_dcc_dir_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_T2T_SRC_ADDR_LO_src_addr_31_0_offset 1
+#define SDMA_PKT_COPY_T2T_SRC_ADDR_LO_src_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_SRC_ADDR_LO_src_addr_31_0_shift 0
+#define SDMA_PKT_COPY_T2T_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_T2T_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_T2T_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_T2T_SRC_ADDR_HI_src_addr_63_32_offset 2
+#define SDMA_PKT_COPY_T2T_SRC_ADDR_HI_src_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_SRC_ADDR_HI_src_addr_63_32_shift 0
+#define SDMA_PKT_COPY_T2T_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_T2T_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_T2T_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for src_x field*/
+#define SDMA_PKT_COPY_T2T_DW_3_src_x_offset 3
+#define SDMA_PKT_COPY_T2T_DW_3_src_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_3_src_x_shift 0
+#define SDMA_PKT_COPY_T2T_DW_3_SRC_X(x) (((x) & SDMA_PKT_COPY_T2T_DW_3_src_x_mask) << SDMA_PKT_COPY_T2T_DW_3_src_x_shift)
+
+/*define for src_y field*/
+#define SDMA_PKT_COPY_T2T_DW_3_src_y_offset 3
+#define SDMA_PKT_COPY_T2T_DW_3_src_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_3_src_y_shift 16
+#define SDMA_PKT_COPY_T2T_DW_3_SRC_Y(x) (((x) & SDMA_PKT_COPY_T2T_DW_3_src_y_mask) << SDMA_PKT_COPY_T2T_DW_3_src_y_shift)
+
+/*define for DW_4 word*/
+/*define for src_z field*/
+#define SDMA_PKT_COPY_T2T_DW_4_src_z_offset 4
+#define SDMA_PKT_COPY_T2T_DW_4_src_z_mask 0x00001FFF
+#define SDMA_PKT_COPY_T2T_DW_4_src_z_shift 0
+#define SDMA_PKT_COPY_T2T_DW_4_SRC_Z(x) (((x) & SDMA_PKT_COPY_T2T_DW_4_src_z_mask) << SDMA_PKT_COPY_T2T_DW_4_src_z_shift)
+
+/*define for src_width field*/
+#define SDMA_PKT_COPY_T2T_DW_4_src_width_offset 4
+#define SDMA_PKT_COPY_T2T_DW_4_src_width_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_4_src_width_shift 16
+#define SDMA_PKT_COPY_T2T_DW_4_SRC_WIDTH(x) (((x) & SDMA_PKT_COPY_T2T_DW_4_src_width_mask) << SDMA_PKT_COPY_T2T_DW_4_src_width_shift)
+
+/*define for DW_5 word*/
+/*define for src_height field*/
+#define SDMA_PKT_COPY_T2T_DW_5_src_height_offset 5
+#define SDMA_PKT_COPY_T2T_DW_5_src_height_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_5_src_height_shift 0
+#define SDMA_PKT_COPY_T2T_DW_5_SRC_HEIGHT(x) (((x) & SDMA_PKT_COPY_T2T_DW_5_src_height_mask) << SDMA_PKT_COPY_T2T_DW_5_src_height_shift)
+
+/*define for src_depth field*/
+#define SDMA_PKT_COPY_T2T_DW_5_src_depth_offset 5
+#define SDMA_PKT_COPY_T2T_DW_5_src_depth_mask 0x00001FFF
+#define SDMA_PKT_COPY_T2T_DW_5_src_depth_shift 16
+#define SDMA_PKT_COPY_T2T_DW_5_SRC_DEPTH(x) (((x) & SDMA_PKT_COPY_T2T_DW_5_src_depth_mask) << SDMA_PKT_COPY_T2T_DW_5_src_depth_shift)
+
+/*define for DW_6 word*/
+/*define for src_element_size field*/
+#define SDMA_PKT_COPY_T2T_DW_6_src_element_size_offset 6
+#define SDMA_PKT_COPY_T2T_DW_6_src_element_size_mask 0x00000007
+#define SDMA_PKT_COPY_T2T_DW_6_src_element_size_shift 0
+#define SDMA_PKT_COPY_T2T_DW_6_SRC_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_T2T_DW_6_src_element_size_mask) << SDMA_PKT_COPY_T2T_DW_6_src_element_size_shift)
+
+/*define for src_swizzle_mode field*/
+#define SDMA_PKT_COPY_T2T_DW_6_src_swizzle_mode_offset 6
+#define SDMA_PKT_COPY_T2T_DW_6_src_swizzle_mode_mask 0x0000001F
+#define SDMA_PKT_COPY_T2T_DW_6_src_swizzle_mode_shift 3
+#define SDMA_PKT_COPY_T2T_DW_6_SRC_SWIZZLE_MODE(x) (((x) & SDMA_PKT_COPY_T2T_DW_6_src_swizzle_mode_mask) << SDMA_PKT_COPY_T2T_DW_6_src_swizzle_mode_shift)
+
+/*define for src_dimension field*/
+#define SDMA_PKT_COPY_T2T_DW_6_src_dimension_offset 6
+#define SDMA_PKT_COPY_T2T_DW_6_src_dimension_mask 0x00000003
+#define SDMA_PKT_COPY_T2T_DW_6_src_dimension_shift 9
+#define SDMA_PKT_COPY_T2T_DW_6_SRC_DIMENSION(x) (((x) & SDMA_PKT_COPY_T2T_DW_6_src_dimension_mask) << SDMA_PKT_COPY_T2T_DW_6_src_dimension_shift)
+
+/*define for src_mip_max field*/
+#define SDMA_PKT_COPY_T2T_DW_6_src_mip_max_offset 6
+#define SDMA_PKT_COPY_T2T_DW_6_src_mip_max_mask 0x0000000F
+#define SDMA_PKT_COPY_T2T_DW_6_src_mip_max_shift 16
+#define SDMA_PKT_COPY_T2T_DW_6_SRC_MIP_MAX(x) (((x) & SDMA_PKT_COPY_T2T_DW_6_src_mip_max_mask) << SDMA_PKT_COPY_T2T_DW_6_src_mip_max_shift)
+
+/*define for src_mip_id field*/
+#define SDMA_PKT_COPY_T2T_DW_6_src_mip_id_offset 6
+#define SDMA_PKT_COPY_T2T_DW_6_src_mip_id_mask 0x0000000F
+#define SDMA_PKT_COPY_T2T_DW_6_src_mip_id_shift 20
+#define SDMA_PKT_COPY_T2T_DW_6_SRC_MIP_ID(x) (((x) & SDMA_PKT_COPY_T2T_DW_6_src_mip_id_mask) << SDMA_PKT_COPY_T2T_DW_6_src_mip_id_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_COPY_T2T_DST_ADDR_LO_dst_addr_31_0_offset 7
+#define SDMA_PKT_COPY_T2T_DST_ADDR_LO_dst_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_DST_ADDR_LO_dst_addr_31_0_shift 0
+#define SDMA_PKT_COPY_T2T_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_T2T_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_COPY_T2T_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_COPY_T2T_DST_ADDR_HI_dst_addr_63_32_offset 8
+#define SDMA_PKT_COPY_T2T_DST_ADDR_HI_dst_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_DST_ADDR_HI_dst_addr_63_32_shift 0
+#define SDMA_PKT_COPY_T2T_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_T2T_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_COPY_T2T_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for DW_9 word*/
+/*define for dst_x field*/
+#define SDMA_PKT_COPY_T2T_DW_9_dst_x_offset 9
+#define SDMA_PKT_COPY_T2T_DW_9_dst_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_9_dst_x_shift 0
+#define SDMA_PKT_COPY_T2T_DW_9_DST_X(x) (((x) & SDMA_PKT_COPY_T2T_DW_9_dst_x_mask) << SDMA_PKT_COPY_T2T_DW_9_dst_x_shift)
+
+/*define for dst_y field*/
+#define SDMA_PKT_COPY_T2T_DW_9_dst_y_offset 9
+#define SDMA_PKT_COPY_T2T_DW_9_dst_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_9_dst_y_shift 16
+#define SDMA_PKT_COPY_T2T_DW_9_DST_Y(x) (((x) & SDMA_PKT_COPY_T2T_DW_9_dst_y_mask) << SDMA_PKT_COPY_T2T_DW_9_dst_y_shift)
+
+/*define for DW_10 word*/
+/*define for dst_z field*/
+#define SDMA_PKT_COPY_T2T_DW_10_dst_z_offset 10
+#define SDMA_PKT_COPY_T2T_DW_10_dst_z_mask 0x00001FFF
+#define SDMA_PKT_COPY_T2T_DW_10_dst_z_shift 0
+#define SDMA_PKT_COPY_T2T_DW_10_DST_Z(x) (((x) & SDMA_PKT_COPY_T2T_DW_10_dst_z_mask) << SDMA_PKT_COPY_T2T_DW_10_dst_z_shift)
+
+/*define for dst_width field*/
+#define SDMA_PKT_COPY_T2T_DW_10_dst_width_offset 10
+#define SDMA_PKT_COPY_T2T_DW_10_dst_width_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_10_dst_width_shift 16
+#define SDMA_PKT_COPY_T2T_DW_10_DST_WIDTH(x) (((x) & SDMA_PKT_COPY_T2T_DW_10_dst_width_mask) << SDMA_PKT_COPY_T2T_DW_10_dst_width_shift)
+
+/*define for DW_11 word*/
+/*define for dst_height field*/
+#define SDMA_PKT_COPY_T2T_DW_11_dst_height_offset 11
+#define SDMA_PKT_COPY_T2T_DW_11_dst_height_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_11_dst_height_shift 0
+#define SDMA_PKT_COPY_T2T_DW_11_DST_HEIGHT(x) (((x) & SDMA_PKT_COPY_T2T_DW_11_dst_height_mask) << SDMA_PKT_COPY_T2T_DW_11_dst_height_shift)
+
+/*define for dst_depth field*/
+#define SDMA_PKT_COPY_T2T_DW_11_dst_depth_offset 11
+#define SDMA_PKT_COPY_T2T_DW_11_dst_depth_mask 0x00001FFF
+#define SDMA_PKT_COPY_T2T_DW_11_dst_depth_shift 16
+#define SDMA_PKT_COPY_T2T_DW_11_DST_DEPTH(x) (((x) & SDMA_PKT_COPY_T2T_DW_11_dst_depth_mask) << SDMA_PKT_COPY_T2T_DW_11_dst_depth_shift)
+
+/*define for DW_12 word*/
+/*define for dst_element_size field*/
+#define SDMA_PKT_COPY_T2T_DW_12_dst_element_size_offset 12
+#define SDMA_PKT_COPY_T2T_DW_12_dst_element_size_mask 0x00000007
+#define SDMA_PKT_COPY_T2T_DW_12_dst_element_size_shift 0
+#define SDMA_PKT_COPY_T2T_DW_12_DST_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_T2T_DW_12_dst_element_size_mask) << SDMA_PKT_COPY_T2T_DW_12_dst_element_size_shift)
+
+/*define for dst_swizzle_mode field*/
+#define SDMA_PKT_COPY_T2T_DW_12_dst_swizzle_mode_offset 12
+#define SDMA_PKT_COPY_T2T_DW_12_dst_swizzle_mode_mask 0x0000001F
+#define SDMA_PKT_COPY_T2T_DW_12_dst_swizzle_mode_shift 3
+#define SDMA_PKT_COPY_T2T_DW_12_DST_SWIZZLE_MODE(x) (((x) & SDMA_PKT_COPY_T2T_DW_12_dst_swizzle_mode_mask) << SDMA_PKT_COPY_T2T_DW_12_dst_swizzle_mode_shift)
+
+/*define for dst_dimension field*/
+#define SDMA_PKT_COPY_T2T_DW_12_dst_dimension_offset 12
+#define SDMA_PKT_COPY_T2T_DW_12_dst_dimension_mask 0x00000003
+#define SDMA_PKT_COPY_T2T_DW_12_dst_dimension_shift 9
+#define SDMA_PKT_COPY_T2T_DW_12_DST_DIMENSION(x) (((x) & SDMA_PKT_COPY_T2T_DW_12_dst_dimension_mask) << SDMA_PKT_COPY_T2T_DW_12_dst_dimension_shift)
+
+/*define for dst_mip_max field*/
+#define SDMA_PKT_COPY_T2T_DW_12_dst_mip_max_offset 12
+#define SDMA_PKT_COPY_T2T_DW_12_dst_mip_max_mask 0x0000000F
+#define SDMA_PKT_COPY_T2T_DW_12_dst_mip_max_shift 16
+#define SDMA_PKT_COPY_T2T_DW_12_DST_MIP_MAX(x) (((x) & SDMA_PKT_COPY_T2T_DW_12_dst_mip_max_mask) << SDMA_PKT_COPY_T2T_DW_12_dst_mip_max_shift)
+
+/*define for dst_mip_id field*/
+#define SDMA_PKT_COPY_T2T_DW_12_dst_mip_id_offset 12
+#define SDMA_PKT_COPY_T2T_DW_12_dst_mip_id_mask 0x0000000F
+#define SDMA_PKT_COPY_T2T_DW_12_dst_mip_id_shift 20
+#define SDMA_PKT_COPY_T2T_DW_12_DST_MIP_ID(x) (((x) & SDMA_PKT_COPY_T2T_DW_12_dst_mip_id_mask) << SDMA_PKT_COPY_T2T_DW_12_dst_mip_id_shift)
+
+/*define for DW_13 word*/
+/*define for rect_x field*/
+#define SDMA_PKT_COPY_T2T_DW_13_rect_x_offset 13
+#define SDMA_PKT_COPY_T2T_DW_13_rect_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_13_rect_x_shift 0
+#define SDMA_PKT_COPY_T2T_DW_13_RECT_X(x) (((x) & SDMA_PKT_COPY_T2T_DW_13_rect_x_mask) << SDMA_PKT_COPY_T2T_DW_13_rect_x_shift)
+
+/*define for rect_y field*/
+#define SDMA_PKT_COPY_T2T_DW_13_rect_y_offset 13
+#define SDMA_PKT_COPY_T2T_DW_13_rect_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_13_rect_y_shift 16
+#define SDMA_PKT_COPY_T2T_DW_13_RECT_Y(x) (((x) & SDMA_PKT_COPY_T2T_DW_13_rect_y_mask) << SDMA_PKT_COPY_T2T_DW_13_rect_y_shift)
+
+/*define for DW_14 word*/
+/*define for rect_z field*/
+#define SDMA_PKT_COPY_T2T_DW_14_rect_z_offset 14
+#define SDMA_PKT_COPY_T2T_DW_14_rect_z_mask 0x00001FFF
+#define SDMA_PKT_COPY_T2T_DW_14_rect_z_shift 0
+#define SDMA_PKT_COPY_T2T_DW_14_RECT_Z(x) (((x) & SDMA_PKT_COPY_T2T_DW_14_rect_z_mask) << SDMA_PKT_COPY_T2T_DW_14_rect_z_shift)
+
+/*define for dst_sw field*/
+#define SDMA_PKT_COPY_T2T_DW_14_dst_sw_offset 14
+#define SDMA_PKT_COPY_T2T_DW_14_dst_sw_mask 0x00000003
+#define SDMA_PKT_COPY_T2T_DW_14_dst_sw_shift 16
+#define SDMA_PKT_COPY_T2T_DW_14_DST_SW(x) (((x) & SDMA_PKT_COPY_T2T_DW_14_dst_sw_mask) << SDMA_PKT_COPY_T2T_DW_14_dst_sw_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_T2T_DW_14_src_sw_offset 14
+#define SDMA_PKT_COPY_T2T_DW_14_src_sw_mask 0x00000003
+#define SDMA_PKT_COPY_T2T_DW_14_src_sw_shift 24
+#define SDMA_PKT_COPY_T2T_DW_14_SRC_SW(x) (((x) & SDMA_PKT_COPY_T2T_DW_14_src_sw_mask) << SDMA_PKT_COPY_T2T_DW_14_src_sw_shift)
+
+/*define for META_ADDR_LO word*/
+/*define for meta_addr_31_0 field*/
+#define SDMA_PKT_COPY_T2T_META_ADDR_LO_meta_addr_31_0_offset 15
+#define SDMA_PKT_COPY_T2T_META_ADDR_LO_meta_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_META_ADDR_LO_meta_addr_31_0_shift 0
+#define SDMA_PKT_COPY_T2T_META_ADDR_LO_META_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_T2T_META_ADDR_LO_meta_addr_31_0_mask) << SDMA_PKT_COPY_T2T_META_ADDR_LO_meta_addr_31_0_shift)
+
+/*define for META_ADDR_HI word*/
+/*define for meta_addr_63_32 field*/
+#define SDMA_PKT_COPY_T2T_META_ADDR_HI_meta_addr_63_32_offset 16
+#define SDMA_PKT_COPY_T2T_META_ADDR_HI_meta_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_META_ADDR_HI_meta_addr_63_32_shift 0
+#define SDMA_PKT_COPY_T2T_META_ADDR_HI_META_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_T2T_META_ADDR_HI_meta_addr_63_32_mask) << SDMA_PKT_COPY_T2T_META_ADDR_HI_meta_addr_63_32_shift)
+
+/*define for META_CONFIG word*/
+/*define for data_format field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_data_format_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_data_format_mask 0x0000007F
+#define SDMA_PKT_COPY_T2T_META_CONFIG_data_format_shift 0
+#define SDMA_PKT_COPY_T2T_META_CONFIG_DATA_FORMAT(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_data_format_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_data_format_shift)
+
+/*define for color_transform_disable field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_color_transform_disable_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_color_transform_disable_mask 0x00000001
+#define SDMA_PKT_COPY_T2T_META_CONFIG_color_transform_disable_shift 7
+#define SDMA_PKT_COPY_T2T_META_CONFIG_COLOR_TRANSFORM_DISABLE(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_color_transform_disable_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_color_transform_disable_shift)
+
+/*define for alpha_is_on_msb field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_alpha_is_on_msb_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_alpha_is_on_msb_mask 0x00000001
+#define SDMA_PKT_COPY_T2T_META_CONFIG_alpha_is_on_msb_shift 8
+#define SDMA_PKT_COPY_T2T_META_CONFIG_ALPHA_IS_ON_MSB(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_alpha_is_on_msb_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_alpha_is_on_msb_shift)
+
+/*define for number_type field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_number_type_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_number_type_mask 0x00000007
+#define SDMA_PKT_COPY_T2T_META_CONFIG_number_type_shift 9
+#define SDMA_PKT_COPY_T2T_META_CONFIG_NUMBER_TYPE(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_number_type_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_number_type_shift)
+
+/*define for surface_type field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_surface_type_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_surface_type_mask 0x00000003
+#define SDMA_PKT_COPY_T2T_META_CONFIG_surface_type_shift 12
+#define SDMA_PKT_COPY_T2T_META_CONFIG_SURFACE_TYPE(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_surface_type_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_surface_type_shift)
+
+/*define for max_comp_block_size field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_max_comp_block_size_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_max_comp_block_size_mask 0x00000003
+#define SDMA_PKT_COPY_T2T_META_CONFIG_max_comp_block_size_shift 24
+#define SDMA_PKT_COPY_T2T_META_CONFIG_MAX_COMP_BLOCK_SIZE(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_max_comp_block_size_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_max_comp_block_size_shift)
+
+/*define for max_uncomp_block_size field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_max_uncomp_block_size_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_max_uncomp_block_size_mask 0x00000003
+#define SDMA_PKT_COPY_T2T_META_CONFIG_max_uncomp_block_size_shift 26
+#define SDMA_PKT_COPY_T2T_META_CONFIG_MAX_UNCOMP_BLOCK_SIZE(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_max_uncomp_block_size_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_max_uncomp_block_size_shift)
+
+/*define for write_compress_enable field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_write_compress_enable_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_write_compress_enable_mask 0x00000001
+#define SDMA_PKT_COPY_T2T_META_CONFIG_write_compress_enable_shift 28
+#define SDMA_PKT_COPY_T2T_META_CONFIG_WRITE_COMPRESS_ENABLE(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_write_compress_enable_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_write_compress_enable_shift)
+
+/*define for meta_tmz field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_meta_tmz_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_meta_tmz_mask 0x00000001
+#define SDMA_PKT_COPY_T2T_META_CONFIG_meta_tmz_shift 29
+#define SDMA_PKT_COPY_T2T_META_CONFIG_META_TMZ(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_meta_tmz_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_meta_tmz_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_T2T_BC packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_T2T_BC_HEADER_op_offset 0
+#define SDMA_PKT_COPY_T2T_BC_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_COPY_T2T_BC_HEADER_op_shift 0
+#define SDMA_PKT_COPY_T2T_BC_HEADER_OP(x) (((x) & SDMA_PKT_COPY_T2T_BC_HEADER_op_mask) << SDMA_PKT_COPY_T2T_BC_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_T2T_BC_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_T2T_BC_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_COPY_T2T_BC_HEADER_sub_op_shift 8
+#define SDMA_PKT_COPY_T2T_BC_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_T2T_BC_HEADER_sub_op_mask) << SDMA_PKT_COPY_T2T_BC_HEADER_sub_op_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_T2T_BC_SRC_ADDR_LO_src_addr_31_0_offset 1
+#define SDMA_PKT_COPY_T2T_BC_SRC_ADDR_LO_src_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_BC_SRC_ADDR_LO_src_addr_31_0_shift 0
+#define SDMA_PKT_COPY_T2T_BC_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_T2T_BC_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_T2T_BC_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_T2T_BC_SRC_ADDR_HI_src_addr_63_32_offset 2
+#define SDMA_PKT_COPY_T2T_BC_SRC_ADDR_HI_src_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_BC_SRC_ADDR_HI_src_addr_63_32_shift 0
+#define SDMA_PKT_COPY_T2T_BC_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_T2T_BC_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_T2T_BC_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for src_x field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_3_src_x_offset 3
+#define SDMA_PKT_COPY_T2T_BC_DW_3_src_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_3_src_x_shift 0
+#define SDMA_PKT_COPY_T2T_BC_DW_3_SRC_X(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_3_src_x_mask) << SDMA_PKT_COPY_T2T_BC_DW_3_src_x_shift)
+
+/*define for src_y field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_3_src_y_offset 3
+#define SDMA_PKT_COPY_T2T_BC_DW_3_src_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_3_src_y_shift 16
+#define SDMA_PKT_COPY_T2T_BC_DW_3_SRC_Y(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_3_src_y_mask) << SDMA_PKT_COPY_T2T_BC_DW_3_src_y_shift)
+
+/*define for DW_4 word*/
+/*define for src_z field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_4_src_z_offset 4
+#define SDMA_PKT_COPY_T2T_BC_DW_4_src_z_mask 0x000007FF
+#define SDMA_PKT_COPY_T2T_BC_DW_4_src_z_shift 0
+#define SDMA_PKT_COPY_T2T_BC_DW_4_SRC_Z(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_4_src_z_mask) << SDMA_PKT_COPY_T2T_BC_DW_4_src_z_shift)
+
+/*define for src_width field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_4_src_width_offset 4
+#define SDMA_PKT_COPY_T2T_BC_DW_4_src_width_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_4_src_width_shift 16
+#define SDMA_PKT_COPY_T2T_BC_DW_4_SRC_WIDTH(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_4_src_width_mask) << SDMA_PKT_COPY_T2T_BC_DW_4_src_width_shift)
+
+/*define for DW_5 word*/
+/*define for src_height field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_5_src_height_offset 5
+#define SDMA_PKT_COPY_T2T_BC_DW_5_src_height_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_5_src_height_shift 0
+#define SDMA_PKT_COPY_T2T_BC_DW_5_SRC_HEIGHT(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_5_src_height_mask) << SDMA_PKT_COPY_T2T_BC_DW_5_src_height_shift)
+
+/*define for src_depth field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_5_src_depth_offset 5
+#define SDMA_PKT_COPY_T2T_BC_DW_5_src_depth_mask 0x000007FF
+#define SDMA_PKT_COPY_T2T_BC_DW_5_src_depth_shift 16
+#define SDMA_PKT_COPY_T2T_BC_DW_5_SRC_DEPTH(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_5_src_depth_mask) << SDMA_PKT_COPY_T2T_BC_DW_5_src_depth_shift)
+
+/*define for DW_6 word*/
+/*define for src_element_size field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_element_size_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_element_size_mask 0x00000007
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_element_size_shift 0
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_element_size_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_element_size_shift)
+
+/*define for src_array_mode field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_array_mode_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_array_mode_mask 0x0000000F
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_array_mode_shift 3
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_ARRAY_MODE(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_array_mode_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_array_mode_shift)
+
+/*define for src_mit_mode field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_mit_mode_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_mit_mode_mask 0x00000007
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_mit_mode_shift 8
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_MIT_MODE(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_mit_mode_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_mit_mode_shift)
+
+/*define for src_tilesplit_size field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_tilesplit_size_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_tilesplit_size_mask 0x00000007
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_tilesplit_size_shift 11
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_TILESPLIT_SIZE(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_tilesplit_size_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_tilesplit_size_shift)
+
+/*define for src_bank_w field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_w_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_w_mask 0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_w_shift 15
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_BANK_W(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_w_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_w_shift)
+
+/*define for src_bank_h field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_h_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_h_mask 0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_h_shift 18
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_BANK_H(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_h_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_h_shift)
+
+/*define for src_num_bank field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_num_bank_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_num_bank_mask 0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_num_bank_shift 21
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_NUM_BANK(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_num_bank_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_num_bank_shift)
+
+/*define for src_mat_aspt field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_mat_aspt_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_mat_aspt_mask 0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_mat_aspt_shift 24
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_MAT_ASPT(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_mat_aspt_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_mat_aspt_shift)
+
+/*define for src_pipe_config field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_pipe_config_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_pipe_config_mask 0x0000001F
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_pipe_config_shift 26
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_PIPE_CONFIG(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_pipe_config_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_pipe_config_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_COPY_T2T_BC_DST_ADDR_LO_dst_addr_31_0_offset 7
+#define SDMA_PKT_COPY_T2T_BC_DST_ADDR_LO_dst_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_BC_DST_ADDR_LO_dst_addr_31_0_shift 0
+#define SDMA_PKT_COPY_T2T_BC_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_T2T_BC_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_COPY_T2T_BC_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_COPY_T2T_BC_DST_ADDR_HI_dst_addr_63_32_offset 8
+#define SDMA_PKT_COPY_T2T_BC_DST_ADDR_HI_dst_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_BC_DST_ADDR_HI_dst_addr_63_32_shift 0
+#define SDMA_PKT_COPY_T2T_BC_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_T2T_BC_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_COPY_T2T_BC_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for DW_9 word*/
+/*define for dst_x field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_9_dst_x_offset 9
+#define SDMA_PKT_COPY_T2T_BC_DW_9_dst_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_9_dst_x_shift 0
+#define SDMA_PKT_COPY_T2T_BC_DW_9_DST_X(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_9_dst_x_mask) << SDMA_PKT_COPY_T2T_BC_DW_9_dst_x_shift)
+
+/*define for dst_y field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_9_dst_y_offset 9
+#define SDMA_PKT_COPY_T2T_BC_DW_9_dst_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_9_dst_y_shift 16
+#define SDMA_PKT_COPY_T2T_BC_DW_9_DST_Y(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_9_dst_y_mask) << SDMA_PKT_COPY_T2T_BC_DW_9_dst_y_shift)
+
+/*define for DW_10 word*/
+/*define for dst_z field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_10_dst_z_offset 10
+#define SDMA_PKT_COPY_T2T_BC_DW_10_dst_z_mask 0x000007FF
+#define SDMA_PKT_COPY_T2T_BC_DW_10_dst_z_shift 0
+#define SDMA_PKT_COPY_T2T_BC_DW_10_DST_Z(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_10_dst_z_mask) << SDMA_PKT_COPY_T2T_BC_DW_10_dst_z_shift)
+
+/*define for dst_width field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_10_dst_width_offset 10
+#define SDMA_PKT_COPY_T2T_BC_DW_10_dst_width_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_10_dst_width_shift 16
+#define SDMA_PKT_COPY_T2T_BC_DW_10_DST_WIDTH(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_10_dst_width_mask) << SDMA_PKT_COPY_T2T_BC_DW_10_dst_width_shift)
+
+/*define for DW_11 word*/
+/*define for dst_height field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_11_dst_height_offset 11
+#define SDMA_PKT_COPY_T2T_BC_DW_11_dst_height_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_11_dst_height_shift 0
+#define SDMA_PKT_COPY_T2T_BC_DW_11_DST_HEIGHT(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_11_dst_height_mask) << SDMA_PKT_COPY_T2T_BC_DW_11_dst_height_shift)
+
+/*define for dst_depth field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_11_dst_depth_offset 11
+#define SDMA_PKT_COPY_T2T_BC_DW_11_dst_depth_mask 0x00000FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_11_dst_depth_shift 16
+#define SDMA_PKT_COPY_T2T_BC_DW_11_DST_DEPTH(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_11_dst_depth_mask) << SDMA_PKT_COPY_T2T_BC_DW_11_dst_depth_shift)
+
+/*define for DW_12 word*/
+/*define for dst_element_size field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_element_size_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_element_size_mask 0x00000007
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_element_size_shift 0
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_element_size_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_element_size_shift)
+
+/*define for dst_array_mode field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_array_mode_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_array_mode_mask 0x0000000F
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_array_mode_shift 3
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_ARRAY_MODE(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_array_mode_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_array_mode_shift)
+
+/*define for dst_mit_mode field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_mit_mode_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_mit_mode_mask 0x00000007
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_mit_mode_shift 8
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_MIT_MODE(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_mit_mode_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_mit_mode_shift)
+
+/*define for dst_tilesplit_size field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_tilesplit_size_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_tilesplit_size_mask 0x00000007
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_tilesplit_size_shift 11
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_TILESPLIT_SIZE(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_tilesplit_size_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_tilesplit_size_shift)
+
+/*define for dst_bank_w field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_w_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_w_mask 0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_w_shift 15
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_BANK_W(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_w_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_w_shift)
+
+/*define for dst_bank_h field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_h_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_h_mask 0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_h_shift 18
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_BANK_H(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_h_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_h_shift)
+
+/*define for dst_num_bank field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_num_bank_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_num_bank_mask 0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_num_bank_shift 21
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_NUM_BANK(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_num_bank_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_num_bank_shift)
+
+/*define for dst_mat_aspt field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_mat_aspt_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_mat_aspt_mask 0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_mat_aspt_shift 24
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_MAT_ASPT(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_mat_aspt_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_mat_aspt_shift)
+
+/*define for dst_pipe_config field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_pipe_config_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_pipe_config_mask 0x0000001F
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_pipe_config_shift 26
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_PIPE_CONFIG(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_pipe_config_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_pipe_config_shift)
+
+/*define for DW_13 word*/
+/*define for rect_x field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_13_rect_x_offset 13
+#define SDMA_PKT_COPY_T2T_BC_DW_13_rect_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_13_rect_x_shift 0
+#define SDMA_PKT_COPY_T2T_BC_DW_13_RECT_X(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_13_rect_x_mask) << SDMA_PKT_COPY_T2T_BC_DW_13_rect_x_shift)
+
+/*define for rect_y field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_13_rect_y_offset 13
+#define SDMA_PKT_COPY_T2T_BC_DW_13_rect_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_13_rect_y_shift 16
+#define SDMA_PKT_COPY_T2T_BC_DW_13_RECT_Y(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_13_rect_y_mask) << SDMA_PKT_COPY_T2T_BC_DW_13_rect_y_shift)
+
+/*define for DW_14 word*/
+/*define for rect_z field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_14_rect_z_offset 14
+#define SDMA_PKT_COPY_T2T_BC_DW_14_rect_z_mask 0x000007FF
+#define SDMA_PKT_COPY_T2T_BC_DW_14_rect_z_shift 0
+#define SDMA_PKT_COPY_T2T_BC_DW_14_RECT_Z(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_14_rect_z_mask) << SDMA_PKT_COPY_T2T_BC_DW_14_rect_z_shift)
+
+/*define for dst_sw field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_14_dst_sw_offset 14
+#define SDMA_PKT_COPY_T2T_BC_DW_14_dst_sw_mask 0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_14_dst_sw_shift 16
+#define SDMA_PKT_COPY_T2T_BC_DW_14_DST_SW(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_14_dst_sw_mask) << SDMA_PKT_COPY_T2T_BC_DW_14_dst_sw_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_14_src_sw_offset 14
+#define SDMA_PKT_COPY_T2T_BC_DW_14_src_sw_mask 0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_14_src_sw_shift 24
+#define SDMA_PKT_COPY_T2T_BC_DW_14_SRC_SW(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_14_src_sw_mask) << SDMA_PKT_COPY_T2T_BC_DW_14_src_sw_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_TILED_SUBWIN packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_op_offset 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_op_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_OP(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_HEADER_op_mask) << SDMA_PKT_COPY_TILED_SUBWIN_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_sub_op_shift 8
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_HEADER_sub_op_mask) << SDMA_PKT_COPY_TILED_SUBWIN_HEADER_sub_op_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_tmz_mask 0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_tmz_shift 18
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_HEADER_tmz_mask) << SDMA_PKT_COPY_TILED_SUBWIN_HEADER_tmz_shift)
+
+/*define for dcc field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_dcc_offset 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_dcc_mask 0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_dcc_shift 19
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_DCC(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_HEADER_dcc_mask) << SDMA_PKT_COPY_TILED_SUBWIN_HEADER_dcc_shift)
+
+/*define for detile field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_detile_offset 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_detile_mask 0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_detile_shift 31
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_DETILE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_HEADER_detile_mask) << SDMA_PKT_COPY_TILED_SUBWIN_HEADER_detile_shift)
+
+/*define for TILED_ADDR_LO word*/
+/*define for tiled_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_LO_tiled_addr_31_0_offset 1
+#define SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_LO_tiled_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_LO_tiled_addr_31_0_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_LO_TILED_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_LO_tiled_addr_31_0_mask) << SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_LO_tiled_addr_31_0_shift)
+
+/*define for TILED_ADDR_HI word*/
+/*define for tiled_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_HI_tiled_addr_63_32_offset 2
+#define SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_HI_tiled_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_HI_tiled_addr_63_32_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_HI_TILED_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_HI_tiled_addr_63_32_mask) << SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_HI_tiled_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for tiled_x field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_x_offset 3
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_x_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_3_TILED_X(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_x_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_x_shift)
+
+/*define for tiled_y field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_y_offset 3
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_y_shift 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_3_TILED_Y(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_y_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_y_shift)
+
+/*define for DW_4 word*/
+/*define for tiled_z field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_4_tiled_z_offset 4
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_4_tiled_z_mask 0x00001FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_4_tiled_z_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_4_TILED_Z(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_4_tiled_z_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_4_tiled_z_shift)
+
+/*define for width field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_4_width_offset 4
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_4_width_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_4_width_shift 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_4_WIDTH(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_4_width_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_4_width_shift)
+
+/*define for DW_5 word*/
+/*define for height field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_5_height_offset 5
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_5_height_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_5_height_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_5_HEIGHT(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_5_height_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_5_height_shift)
+
+/*define for depth field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_5_depth_offset 5
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_5_depth_mask 0x00001FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_5_depth_shift 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_5_DEPTH(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_5_depth_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_5_depth_shift)
+
+/*define for DW_6 word*/
+/*define for element_size field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_element_size_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_element_size_mask 0x00000007
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_element_size_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_6_element_size_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_6_element_size_shift)
+
+/*define for swizzle_mode field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_swizzle_mode_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_swizzle_mode_mask 0x0000001F
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_swizzle_mode_shift 3
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_SWIZZLE_MODE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_6_swizzle_mode_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_6_swizzle_mode_shift)
+
+/*define for dimension field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_dimension_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_dimension_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_dimension_shift 9
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_DIMENSION(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_6_dimension_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_6_dimension_shift)
+
+/*define for mip_max field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_max_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_max_mask 0x0000000F
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_max_shift 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_MIP_MAX(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_max_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_max_shift)
+
+/*define for mip_id field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_id_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_id_mask 0x0000000F
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_id_shift 20
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_MIP_ID(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_id_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_id_shift)
+
+/*define for LINEAR_ADDR_LO word*/
+/*define for linear_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_LO_linear_addr_31_0_offset 7
+#define SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_LO_linear_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_LO_linear_addr_31_0_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_LO_LINEAR_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_LO_linear_addr_31_0_mask) << SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_LO_linear_addr_31_0_shift)
+
+/*define for LINEAR_ADDR_HI word*/
+/*define for linear_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_HI_linear_addr_63_32_offset 8
+#define SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_HI_linear_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_HI_linear_addr_63_32_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_HI_LINEAR_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_HI_linear_addr_63_32_mask) << SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_HI_linear_addr_63_32_shift)
+
+/*define for DW_9 word*/
+/*define for linear_x field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_x_offset 9
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_x_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_9_LINEAR_X(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_x_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_x_shift)
+
+/*define for linear_y field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_y_offset 9
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_y_shift 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_9_LINEAR_Y(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_y_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_y_shift)
+
+/*define for DW_10 word*/
+/*define for linear_z field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_z_offset 10
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_z_mask 0x00001FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_z_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_10_LINEAR_Z(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_z_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_z_shift)
+
+/*define for linear_pitch field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_pitch_offset 10
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_pitch_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_pitch_shift 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_10_LINEAR_PITCH(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_pitch_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_pitch_shift)
+
+/*define for DW_11 word*/
+/*define for linear_slice_pitch field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_11_linear_slice_pitch_offset 11
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_11_linear_slice_pitch_mask 0x0FFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_11_linear_slice_pitch_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_11_LINEAR_SLICE_PITCH(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_11_linear_slice_pitch_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_11_linear_slice_pitch_shift)
+
+/*define for DW_12 word*/
+/*define for rect_x field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_x_offset 12
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_x_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_12_RECT_X(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_x_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_x_shift)
+
+/*define for rect_y field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_y_offset 12
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_y_shift 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_12_RECT_Y(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_y_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_y_shift)
+
+/*define for DW_13 word*/
+/*define for rect_z field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_rect_z_offset 13
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_rect_z_mask 0x00001FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_rect_z_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_RECT_Z(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_13_rect_z_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_13_rect_z_shift)
+
+/*define for linear_sw field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_linear_sw_offset 13
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_linear_sw_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_linear_sw_shift 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_LINEAR_SW(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_13_linear_sw_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_13_linear_sw_shift)
+
+/*define for tile_sw field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_tile_sw_offset 13
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_tile_sw_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_tile_sw_shift 24
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_TILE_SW(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_13_tile_sw_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_13_tile_sw_shift)
+
+/*define for META_ADDR_LO word*/
+/*define for meta_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_LO_meta_addr_31_0_offset 14
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_LO_meta_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_LO_meta_addr_31_0_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_LO_META_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_LO_meta_addr_31_0_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_LO_meta_addr_31_0_shift)
+
+/*define for META_ADDR_HI word*/
+/*define for meta_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_HI_meta_addr_63_32_offset 15
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_HI_meta_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_HI_meta_addr_63_32_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_HI_META_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_HI_meta_addr_63_32_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_HI_meta_addr_63_32_shift)
+
+/*define for META_CONFIG word*/
+/*define for data_format field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_data_format_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_data_format_mask 0x0000007F
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_data_format_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_DATA_FORMAT(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_data_format_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_data_format_shift)
+
+/*define for color_transform_disable field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_color_transform_disable_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_color_transform_disable_mask 0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_color_transform_disable_shift 7
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_COLOR_TRANSFORM_DISABLE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_color_transform_disable_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_color_transform_disable_shift)
+
+/*define for alpha_is_on_msb field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_alpha_is_on_msb_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_alpha_is_on_msb_mask 0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_alpha_is_on_msb_shift 8
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_ALPHA_IS_ON_MSB(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_alpha_is_on_msb_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_alpha_is_on_msb_shift)
+
+/*define for number_type field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_number_type_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_number_type_mask 0x00000007
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_number_type_shift 9
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_NUMBER_TYPE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_number_type_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_number_type_shift)
+
+/*define for surface_type field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_surface_type_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_surface_type_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_surface_type_shift 12
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_SURFACE_TYPE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_surface_type_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_surface_type_shift)
+
+/*define for max_comp_block_size field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_comp_block_size_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_comp_block_size_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_comp_block_size_shift 24
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_MAX_COMP_BLOCK_SIZE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_comp_block_size_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_comp_block_size_shift)
+
+/*define for max_uncomp_block_size field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_uncomp_block_size_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_uncomp_block_size_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_uncomp_block_size_shift 26
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_MAX_UNCOMP_BLOCK_SIZE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_uncomp_block_size_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_uncomp_block_size_shift)
+
+/*define for write_compress_enable field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_write_compress_enable_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_write_compress_enable_mask 0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_write_compress_enable_shift 28
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_WRITE_COMPRESS_ENABLE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_write_compress_enable_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_write_compress_enable_shift)
+
+/*define for meta_tmz field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_meta_tmz_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_meta_tmz_mask 0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_meta_tmz_shift 29
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_META_TMZ(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_meta_tmz_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_meta_tmz_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_TILED_SUBWIN_BC packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_op_offset 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_op_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_OP(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_op_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_sub_op_shift 8
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_sub_op_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_sub_op_shift)
+
+/*define for detile field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_detile_offset 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_detile_mask 0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_detile_shift 31
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_DETILE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_detile_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_detile_shift)
+
+/*define for TILED_ADDR_LO word*/
+/*define for tiled_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_LO_tiled_addr_31_0_offset 1
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_LO_tiled_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_LO_tiled_addr_31_0_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_LO_TILED_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_LO_tiled_addr_31_0_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_LO_tiled_addr_31_0_shift)
+
+/*define for TILED_ADDR_HI word*/
+/*define for tiled_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_HI_tiled_addr_63_32_offset 2
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_HI_tiled_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_HI_tiled_addr_63_32_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_HI_TILED_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_HI_tiled_addr_63_32_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_HI_tiled_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for tiled_x field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_x_offset 3
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_x_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_TILED_X(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_x_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_x_shift)
+
+/*define for tiled_y field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_y_offset 3
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_y_shift 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_TILED_Y(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_y_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_y_shift)
+
+/*define for DW_4 word*/
+/*define for tiled_z field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_tiled_z_offset 4
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_tiled_z_mask 0x000007FF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_tiled_z_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_TILED_Z(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_tiled_z_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_tiled_z_shift)
+
+/*define for width field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_width_offset 4
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_width_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_width_shift 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_WIDTH(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_width_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_width_shift)
+
+/*define for DW_5 word*/
+/*define for height field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_height_offset 5
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_height_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_height_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_HEIGHT(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_height_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_height_shift)
+
+/*define for depth field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_depth_offset 5
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_depth_mask 0x000007FF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_depth_shift 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_DEPTH(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_depth_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_depth_shift)
+
+/*define for DW_6 word*/
+/*define for element_size field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_element_size_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_element_size_mask 0x00000007
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_element_size_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_element_size_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_element_size_shift)
+
+/*define for array_mode field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_array_mode_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_array_mode_mask 0x0000000F
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_array_mode_shift 3
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_ARRAY_MODE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_array_mode_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_array_mode_shift)
+
+/*define for mit_mode field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mit_mode_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mit_mode_mask 0x00000007
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mit_mode_shift 8
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_MIT_MODE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mit_mode_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mit_mode_shift)
+
+/*define for tilesplit_size field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_tilesplit_size_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_tilesplit_size_mask 0x00000007
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_tilesplit_size_shift 11
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_TILESPLIT_SIZE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_tilesplit_size_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_tilesplit_size_shift)
+
+/*define for bank_w field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_w_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_w_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_w_shift 15
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_BANK_W(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_w_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_w_shift)
+
+/*define for bank_h field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_h_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_h_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_h_shift 18
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_BANK_H(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_h_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_h_shift)
+
+/*define for num_bank field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_num_bank_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_num_bank_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_num_bank_shift 21
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_NUM_BANK(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_num_bank_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_num_bank_shift)
+
+/*define for mat_aspt field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mat_aspt_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mat_aspt_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mat_aspt_shift 24
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_MAT_ASPT(x) ((x & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mat_aspt_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mat_aspt_shift)
+
+/*define for pipe_config field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_pipe_config_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_pipe_config_mask 0x0000001F
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_pipe_config_shift 26
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_PIPE_CONFIG(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_pipe_config_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_pipe_config_shift)
+
+/*define for LINEAR_ADDR_LO word*/
+/*define for linear_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_LO_linear_addr_31_0_offset 7
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_LO_linear_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_LO_linear_addr_31_0_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_LO_LINEAR_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_LO_linear_addr_31_0_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_LO_linear_addr_31_0_shift)
+
+/*define for LINEAR_ADDR_HI word*/
+/*define for linear_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_HI_linear_addr_63_32_offset 8
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_HI_linear_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_HI_linear_addr_63_32_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_HI_LINEAR_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_HI_linear_addr_63_32_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_HI_linear_addr_63_32_shift)
+
+/*define for DW_9 word*/
+/*define for linear_x field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_x_offset 9
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_x_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_LINEAR_X(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_x_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_x_shift)
+
+/*define for linear_y field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_y_offset 9
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_y_shift 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_LINEAR_Y(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_y_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_y_shift)
+
+/*define for DW_10 word*/
+/*define for linear_z field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_z_offset 10
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_z_mask 0x000007FF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_z_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_LINEAR_Z(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_z_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_z_shift)
+
+/*define for linear_pitch field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_pitch_offset 10
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_pitch_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_pitch_shift 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_LINEAR_PITCH(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_pitch_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_pitch_shift)
+
+/*define for DW_11 word*/
+/*define for linear_slice_pitch field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_11_linear_slice_pitch_offset 11
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_11_linear_slice_pitch_mask 0x0FFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_11_linear_slice_pitch_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_11_LINEAR_SLICE_PITCH(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_11_linear_slice_pitch_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_11_linear_slice_pitch_shift)
+
+/*define for DW_12 word*/
+/*define for rect_x field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_x_offset 12
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_x_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_x_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_RECT_X(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_x_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_x_shift)
+
+/*define for rect_y field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_y_offset 12
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_y_mask 0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_y_shift 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_RECT_Y(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_y_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_y_shift)
+
+/*define for DW_13 word*/
+/*define for rect_z field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_rect_z_offset 13
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_rect_z_mask 0x000007FF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_rect_z_shift 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_RECT_Z(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_rect_z_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_rect_z_shift)
+
+/*define for linear_sw field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_linear_sw_offset 13
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_linear_sw_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_linear_sw_shift 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_LINEAR_SW(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_linear_sw_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_linear_sw_shift)
+
+/*define for tile_sw field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_tile_sw_offset 13
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_tile_sw_mask 0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_tile_sw_shift 24
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_TILE_SW(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_tile_sw_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_tile_sw_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_STRUCT packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_STRUCT_HEADER_op_offset 0
+#define SDMA_PKT_COPY_STRUCT_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_COPY_STRUCT_HEADER_op_shift 0
+#define SDMA_PKT_COPY_STRUCT_HEADER_OP(x) (((x) & SDMA_PKT_COPY_STRUCT_HEADER_op_mask) << SDMA_PKT_COPY_STRUCT_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_STRUCT_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_STRUCT_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_COPY_STRUCT_HEADER_sub_op_shift 8
+#define SDMA_PKT_COPY_STRUCT_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_STRUCT_HEADER_sub_op_mask) << SDMA_PKT_COPY_STRUCT_HEADER_sub_op_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_STRUCT_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_STRUCT_HEADER_tmz_mask 0x00000001
+#define SDMA_PKT_COPY_STRUCT_HEADER_tmz_shift 18
+#define SDMA_PKT_COPY_STRUCT_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_STRUCT_HEADER_tmz_mask) << SDMA_PKT_COPY_STRUCT_HEADER_tmz_shift)
+
+/*define for detile field*/
+#define SDMA_PKT_COPY_STRUCT_HEADER_detile_offset 0
+#define SDMA_PKT_COPY_STRUCT_HEADER_detile_mask 0x00000001
+#define SDMA_PKT_COPY_STRUCT_HEADER_detile_shift 31
+#define SDMA_PKT_COPY_STRUCT_HEADER_DETILE(x) (((x) & SDMA_PKT_COPY_STRUCT_HEADER_detile_mask) << SDMA_PKT_COPY_STRUCT_HEADER_detile_shift)
+
+/*define for SB_ADDR_LO word*/
+/*define for sb_addr_31_0 field*/
+#define SDMA_PKT_COPY_STRUCT_SB_ADDR_LO_sb_addr_31_0_offset 1
+#define SDMA_PKT_COPY_STRUCT_SB_ADDR_LO_sb_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_STRUCT_SB_ADDR_LO_sb_addr_31_0_shift 0
+#define SDMA_PKT_COPY_STRUCT_SB_ADDR_LO_SB_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_STRUCT_SB_ADDR_LO_sb_addr_31_0_mask) << SDMA_PKT_COPY_STRUCT_SB_ADDR_LO_sb_addr_31_0_shift)
+
+/*define for SB_ADDR_HI word*/
+/*define for sb_addr_63_32 field*/
+#define SDMA_PKT_COPY_STRUCT_SB_ADDR_HI_sb_addr_63_32_offset 2
+#define SDMA_PKT_COPY_STRUCT_SB_ADDR_HI_sb_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_STRUCT_SB_ADDR_HI_sb_addr_63_32_shift 0
+#define SDMA_PKT_COPY_STRUCT_SB_ADDR_HI_SB_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_STRUCT_SB_ADDR_HI_sb_addr_63_32_mask) << SDMA_PKT_COPY_STRUCT_SB_ADDR_HI_sb_addr_63_32_shift)
+
+/*define for START_INDEX word*/
+/*define for start_index field*/
+#define SDMA_PKT_COPY_STRUCT_START_INDEX_start_index_offset 3
+#define SDMA_PKT_COPY_STRUCT_START_INDEX_start_index_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_STRUCT_START_INDEX_start_index_shift 0
+#define SDMA_PKT_COPY_STRUCT_START_INDEX_START_INDEX(x) (((x) & SDMA_PKT_COPY_STRUCT_START_INDEX_start_index_mask) << SDMA_PKT_COPY_STRUCT_START_INDEX_start_index_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_STRUCT_COUNT_count_offset 4
+#define SDMA_PKT_COPY_STRUCT_COUNT_count_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_STRUCT_COUNT_count_shift 0
+#define SDMA_PKT_COPY_STRUCT_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_STRUCT_COUNT_count_mask) << SDMA_PKT_COPY_STRUCT_COUNT_count_shift)
+
+/*define for DW_5 word*/
+/*define for stride field*/
+#define SDMA_PKT_COPY_STRUCT_DW_5_stride_offset 5
+#define SDMA_PKT_COPY_STRUCT_DW_5_stride_mask 0x000007FF
+#define SDMA_PKT_COPY_STRUCT_DW_5_stride_shift 0
+#define SDMA_PKT_COPY_STRUCT_DW_5_STRIDE(x) (((x) & SDMA_PKT_COPY_STRUCT_DW_5_stride_mask) << SDMA_PKT_COPY_STRUCT_DW_5_stride_shift)
+
+/*define for linear_sw field*/
+#define SDMA_PKT_COPY_STRUCT_DW_5_linear_sw_offset 5
+#define SDMA_PKT_COPY_STRUCT_DW_5_linear_sw_mask 0x00000003
+#define SDMA_PKT_COPY_STRUCT_DW_5_linear_sw_shift 16
+#define SDMA_PKT_COPY_STRUCT_DW_5_LINEAR_SW(x) (((x) & SDMA_PKT_COPY_STRUCT_DW_5_linear_sw_mask) << SDMA_PKT_COPY_STRUCT_DW_5_linear_sw_shift)
+
+/*define for struct_sw field*/
+#define SDMA_PKT_COPY_STRUCT_DW_5_struct_sw_offset 5
+#define SDMA_PKT_COPY_STRUCT_DW_5_struct_sw_mask 0x00000003
+#define SDMA_PKT_COPY_STRUCT_DW_5_struct_sw_shift 24
+#define SDMA_PKT_COPY_STRUCT_DW_5_STRUCT_SW(x) (((x) & SDMA_PKT_COPY_STRUCT_DW_5_struct_sw_mask) << SDMA_PKT_COPY_STRUCT_DW_5_struct_sw_shift)
+
+/*define for LINEAR_ADDR_LO word*/
+/*define for linear_addr_31_0 field*/
+#define SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_LO_linear_addr_31_0_offset 6
+#define SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_LO_linear_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_LO_linear_addr_31_0_shift 0
+#define SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_LO_LINEAR_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_LO_linear_addr_31_0_mask) << SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_LO_linear_addr_31_0_shift)
+
+/*define for LINEAR_ADDR_HI word*/
+/*define for linear_addr_63_32 field*/
+#define SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_HI_linear_addr_63_32_offset 7
+#define SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_HI_linear_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_HI_linear_addr_63_32_shift 0
+#define SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_HI_LINEAR_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_HI_linear_addr_63_32_mask) << SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_HI_linear_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_WRITE_UNTILED packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_WRITE_UNTILED_HEADER_op_offset 0
+#define SDMA_PKT_WRITE_UNTILED_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_WRITE_UNTILED_HEADER_op_shift 0
+#define SDMA_PKT_WRITE_UNTILED_HEADER_OP(x) (((x) & SDMA_PKT_WRITE_UNTILED_HEADER_op_mask) << SDMA_PKT_WRITE_UNTILED_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_WRITE_UNTILED_HEADER_sub_op_offset 0
+#define SDMA_PKT_WRITE_UNTILED_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_WRITE_UNTILED_HEADER_sub_op_shift 8
+#define SDMA_PKT_WRITE_UNTILED_HEADER_SUB_OP(x) (((x) & SDMA_PKT_WRITE_UNTILED_HEADER_sub_op_mask) << SDMA_PKT_WRITE_UNTILED_HEADER_sub_op_shift)
+
+/*define for encrypt field*/
+#define SDMA_PKT_WRITE_UNTILED_HEADER_encrypt_offset 0
+#define SDMA_PKT_WRITE_UNTILED_HEADER_encrypt_mask 0x00000001
+#define SDMA_PKT_WRITE_UNTILED_HEADER_encrypt_shift 16
+#define SDMA_PKT_WRITE_UNTILED_HEADER_ENCRYPT(x) (((x) & SDMA_PKT_WRITE_UNTILED_HEADER_encrypt_mask) << SDMA_PKT_WRITE_UNTILED_HEADER_encrypt_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_WRITE_UNTILED_HEADER_tmz_offset 0
+#define SDMA_PKT_WRITE_UNTILED_HEADER_tmz_mask 0x00000001
+#define SDMA_PKT_WRITE_UNTILED_HEADER_tmz_shift 18
+#define SDMA_PKT_WRITE_UNTILED_HEADER_TMZ(x) (((x) & SDMA_PKT_WRITE_UNTILED_HEADER_tmz_mask) << SDMA_PKT_WRITE_UNTILED_HEADER_tmz_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_WRITE_UNTILED_DST_ADDR_LO_dst_addr_31_0_offset 1
+#define SDMA_PKT_WRITE_UNTILED_DST_ADDR_LO_dst_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_WRITE_UNTILED_DST_ADDR_LO_dst_addr_31_0_shift 0
+#define SDMA_PKT_WRITE_UNTILED_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_WRITE_UNTILED_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_WRITE_UNTILED_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_WRITE_UNTILED_DST_ADDR_HI_dst_addr_63_32_offset 2
+#define SDMA_PKT_WRITE_UNTILED_DST_ADDR_HI_dst_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_WRITE_UNTILED_DST_ADDR_HI_dst_addr_63_32_shift 0
+#define SDMA_PKT_WRITE_UNTILED_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_WRITE_UNTILED_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_WRITE_UNTILED_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for count field*/
+#define SDMA_PKT_WRITE_UNTILED_DW_3_count_offset 3
+#define SDMA_PKT_WRITE_UNTILED_DW_3_count_mask 0x000FFFFF
+#define SDMA_PKT_WRITE_UNTILED_DW_3_count_shift 0
+#define SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(x) (((x) & SDMA_PKT_WRITE_UNTILED_DW_3_count_mask) << SDMA_PKT_WRITE_UNTILED_DW_3_count_shift)
+
+/*define for sw field*/
+#define SDMA_PKT_WRITE_UNTILED_DW_3_sw_offset 3
+#define SDMA_PKT_WRITE_UNTILED_DW_3_sw_mask 0x00000003
+#define SDMA_PKT_WRITE_UNTILED_DW_3_sw_shift 24
+#define SDMA_PKT_WRITE_UNTILED_DW_3_SW(x) (((x) & SDMA_PKT_WRITE_UNTILED_DW_3_sw_mask) << SDMA_PKT_WRITE_UNTILED_DW_3_sw_shift)
+
+/*define for DATA0 word*/
+/*define for data0 field*/
+#define SDMA_PKT_WRITE_UNTILED_DATA0_data0_offset 4
+#define SDMA_PKT_WRITE_UNTILED_DATA0_data0_mask 0xFFFFFFFF
+#define SDMA_PKT_WRITE_UNTILED_DATA0_data0_shift 0
+#define SDMA_PKT_WRITE_UNTILED_DATA0_DATA0(x) (((x) & SDMA_PKT_WRITE_UNTILED_DATA0_data0_mask) << SDMA_PKT_WRITE_UNTILED_DATA0_data0_shift)
+
+
+/*
+** Definitions for SDMA_PKT_WRITE_TILED packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_WRITE_TILED_HEADER_op_offset 0
+#define SDMA_PKT_WRITE_TILED_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_WRITE_TILED_HEADER_op_shift 0
+#define SDMA_PKT_WRITE_TILED_HEADER_OP(x) (((x) & SDMA_PKT_WRITE_TILED_HEADER_op_mask) << SDMA_PKT_WRITE_TILED_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_WRITE_TILED_HEADER_sub_op_offset 0
+#define SDMA_PKT_WRITE_TILED_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_WRITE_TILED_HEADER_sub_op_shift 8
+#define SDMA_PKT_WRITE_TILED_HEADER_SUB_OP(x) (((x) & SDMA_PKT_WRITE_TILED_HEADER_sub_op_mask) << SDMA_PKT_WRITE_TILED_HEADER_sub_op_shift)
+
+/*define for encrypt field*/
+#define SDMA_PKT_WRITE_TILED_HEADER_encrypt_offset 0
+#define SDMA_PKT_WRITE_TILED_HEADER_encrypt_mask 0x00000001
+#define SDMA_PKT_WRITE_TILED_HEADER_encrypt_shift 16
+#define SDMA_PKT_WRITE_TILED_HEADER_ENCRYPT(x) (((x) & SDMA_PKT_WRITE_TILED_HEADER_encrypt_mask) << SDMA_PKT_WRITE_TILED_HEADER_encrypt_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_WRITE_TILED_HEADER_tmz_offset 0
+#define SDMA_PKT_WRITE_TILED_HEADER_tmz_mask 0x00000001
+#define SDMA_PKT_WRITE_TILED_HEADER_tmz_shift 18
+#define SDMA_PKT_WRITE_TILED_HEADER_TMZ(x) (((x) & SDMA_PKT_WRITE_TILED_HEADER_tmz_mask) << SDMA_PKT_WRITE_TILED_HEADER_tmz_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_WRITE_TILED_DST_ADDR_LO_dst_addr_31_0_offset 1
+#define SDMA_PKT_WRITE_TILED_DST_ADDR_LO_dst_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_WRITE_TILED_DST_ADDR_LO_dst_addr_31_0_shift 0
+#define SDMA_PKT_WRITE_TILED_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_WRITE_TILED_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_WRITE_TILED_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_WRITE_TILED_DST_ADDR_HI_dst_addr_63_32_offset 2
+#define SDMA_PKT_WRITE_TILED_DST_ADDR_HI_dst_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_WRITE_TILED_DST_ADDR_HI_dst_addr_63_32_shift 0
+#define SDMA_PKT_WRITE_TILED_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_WRITE_TILED_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_WRITE_TILED_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for width field*/
+#define SDMA_PKT_WRITE_TILED_DW_3_width_offset 3
+#define SDMA_PKT_WRITE_TILED_DW_3_width_mask 0x00003FFF
+#define SDMA_PKT_WRITE_TILED_DW_3_width_shift 0
+#define SDMA_PKT_WRITE_TILED_DW_3_WIDTH(x) (((x) & SDMA_PKT_WRITE_TILED_DW_3_width_mask) << SDMA_PKT_WRITE_TILED_DW_3_width_shift)
+
+/*define for DW_4 word*/
+/*define for height field*/
+#define SDMA_PKT_WRITE_TILED_DW_4_height_offset 4
+#define SDMA_PKT_WRITE_TILED_DW_4_height_mask 0x00003FFF
+#define SDMA_PKT_WRITE_TILED_DW_4_height_shift 0
+#define SDMA_PKT_WRITE_TILED_DW_4_HEIGHT(x) (((x) & SDMA_PKT_WRITE_TILED_DW_4_height_mask) << SDMA_PKT_WRITE_TILED_DW_4_height_shift)
+
+/*define for depth field*/
+#define SDMA_PKT_WRITE_TILED_DW_4_depth_offset 4
+#define SDMA_PKT_WRITE_TILED_DW_4_depth_mask 0x00001FFF
+#define SDMA_PKT_WRITE_TILED_DW_4_depth_shift 16
+#define SDMA_PKT_WRITE_TILED_DW_4_DEPTH(x) (((x) & SDMA_PKT_WRITE_TILED_DW_4_depth_mask) << SDMA_PKT_WRITE_TILED_DW_4_depth_shift)
+
+/*define for DW_5 word*/
+/*define for element_size field*/
+#define SDMA_PKT_WRITE_TILED_DW_5_element_size_offset 5
+#define SDMA_PKT_WRITE_TILED_DW_5_element_size_mask 0x00000007
+#define SDMA_PKT_WRITE_TILED_DW_5_element_size_shift 0
+#define SDMA_PKT_WRITE_TILED_DW_5_ELEMENT_SIZE(x) (((x) & SDMA_PKT_WRITE_TILED_DW_5_element_size_mask) << SDMA_PKT_WRITE_TILED_DW_5_element_size_shift)
+
+/*define for swizzle_mode field*/
+#define SDMA_PKT_WRITE_TILED_DW_5_swizzle_mode_offset 5
+#define SDMA_PKT_WRITE_TILED_DW_5_swizzle_mode_mask 0x0000001F
+#define SDMA_PKT_WRITE_TILED_DW_5_swizzle_mode_shift 3
+#define SDMA_PKT_WRITE_TILED_DW_5_SWIZZLE_MODE(x) (((x) & SDMA_PKT_WRITE_TILED_DW_5_swizzle_mode_mask) << SDMA_PKT_WRITE_TILED_DW_5_swizzle_mode_shift)
+
+/*define for dimension field*/
+#define SDMA_PKT_WRITE_TILED_DW_5_dimension_offset 5
+#define SDMA_PKT_WRITE_TILED_DW_5_dimension_mask 0x00000003
+#define SDMA_PKT_WRITE_TILED_DW_5_dimension_shift 9
+#define SDMA_PKT_WRITE_TILED_DW_5_DIMENSION(x) (((x) & SDMA_PKT_WRITE_TILED_DW_5_dimension_mask) << SDMA_PKT_WRITE_TILED_DW_5_dimension_shift)
+
+/*define for mip_max field*/
+#define SDMA_PKT_WRITE_TILED_DW_5_mip_max_offset 5
+#define SDMA_PKT_WRITE_TILED_DW_5_mip_max_mask 0x0000000F
+#define SDMA_PKT_WRITE_TILED_DW_5_mip_max_shift 16
+#define SDMA_PKT_WRITE_TILED_DW_5_MIP_MAX(x) (((x) & SDMA_PKT_WRITE_TILED_DW_5_mip_max_mask) << SDMA_PKT_WRITE_TILED_DW_5_mip_max_shift)
+
+/*define for DW_6 word*/
+/*define for x field*/
+#define SDMA_PKT_WRITE_TILED_DW_6_x_offset 6
+#define SDMA_PKT_WRITE_TILED_DW_6_x_mask 0x00003FFF
+#define SDMA_PKT_WRITE_TILED_DW_6_x_shift 0
+#define SDMA_PKT_WRITE_TILED_DW_6_X(x) (((x) & SDMA_PKT_WRITE_TILED_DW_6_x_mask) << SDMA_PKT_WRITE_TILED_DW_6_x_shift)
+
+/*define for y field*/
+#define SDMA_PKT_WRITE_TILED_DW_6_y_offset 6
+#define SDMA_PKT_WRITE_TILED_DW_6_y_mask 0x00003FFF
+#define SDMA_PKT_WRITE_TILED_DW_6_y_shift 16
+#define SDMA_PKT_WRITE_TILED_DW_6_Y(x) (((x) & SDMA_PKT_WRITE_TILED_DW_6_y_mask) << SDMA_PKT_WRITE_TILED_DW_6_y_shift)
+
+/*define for DW_7 word*/
+/*define for z field*/
+#define SDMA_PKT_WRITE_TILED_DW_7_z_offset 7
+#define SDMA_PKT_WRITE_TILED_DW_7_z_mask 0x00001FFF
+#define SDMA_PKT_WRITE_TILED_DW_7_z_shift 0
+#define SDMA_PKT_WRITE_TILED_DW_7_Z(x) (((x) & SDMA_PKT_WRITE_TILED_DW_7_z_mask) << SDMA_PKT_WRITE_TILED_DW_7_z_shift)
+
+/*define for sw field*/
+#define SDMA_PKT_WRITE_TILED_DW_7_sw_offset 7
+#define SDMA_PKT_WRITE_TILED_DW_7_sw_mask 0x00000003
+#define SDMA_PKT_WRITE_TILED_DW_7_sw_shift 24
+#define SDMA_PKT_WRITE_TILED_DW_7_SW(x) (((x) & SDMA_PKT_WRITE_TILED_DW_7_sw_mask) << SDMA_PKT_WRITE_TILED_DW_7_sw_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_WRITE_TILED_COUNT_count_offset 8
+#define SDMA_PKT_WRITE_TILED_COUNT_count_mask 0x000FFFFF
+#define SDMA_PKT_WRITE_TILED_COUNT_count_shift 0
+#define SDMA_PKT_WRITE_TILED_COUNT_COUNT(x) (((x) & SDMA_PKT_WRITE_TILED_COUNT_count_mask) << SDMA_PKT_WRITE_TILED_COUNT_count_shift)
+
+/*define for DATA0 word*/
+/*define for data0 field*/
+#define SDMA_PKT_WRITE_TILED_DATA0_data0_offset 9
+#define SDMA_PKT_WRITE_TILED_DATA0_data0_mask 0xFFFFFFFF
+#define SDMA_PKT_WRITE_TILED_DATA0_data0_shift 0
+#define SDMA_PKT_WRITE_TILED_DATA0_DATA0(x) (((x) & SDMA_PKT_WRITE_TILED_DATA0_data0_mask) << SDMA_PKT_WRITE_TILED_DATA0_data0_shift)
+
+
+/*
+** Definitions for SDMA_PKT_WRITE_TILED_BC packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_WRITE_TILED_BC_HEADER_op_offset 0
+#define SDMA_PKT_WRITE_TILED_BC_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_WRITE_TILED_BC_HEADER_op_shift 0
+#define SDMA_PKT_WRITE_TILED_BC_HEADER_OP(x) (((x) & SDMA_PKT_WRITE_TILED_BC_HEADER_op_mask) << SDMA_PKT_WRITE_TILED_BC_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_WRITE_TILED_BC_HEADER_sub_op_offset 0
+#define SDMA_PKT_WRITE_TILED_BC_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_WRITE_TILED_BC_HEADER_sub_op_shift 8
+#define SDMA_PKT_WRITE_TILED_BC_HEADER_SUB_OP(x) (((x) & SDMA_PKT_WRITE_TILED_BC_HEADER_sub_op_mask) << SDMA_PKT_WRITE_TILED_BC_HEADER_sub_op_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_WRITE_TILED_BC_DST_ADDR_LO_dst_addr_31_0_offset 1
+#define SDMA_PKT_WRITE_TILED_BC_DST_ADDR_LO_dst_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_WRITE_TILED_BC_DST_ADDR_LO_dst_addr_31_0_shift 0
+#define SDMA_PKT_WRITE_TILED_BC_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_WRITE_TILED_BC_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_WRITE_TILED_BC_DST_ADDR_HI_dst_addr_63_32_offset 2
+#define SDMA_PKT_WRITE_TILED_BC_DST_ADDR_HI_dst_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_WRITE_TILED_BC_DST_ADDR_HI_dst_addr_63_32_shift 0
+#define SDMA_PKT_WRITE_TILED_BC_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_WRITE_TILED_BC_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for width field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_3_width_offset 3
+#define SDMA_PKT_WRITE_TILED_BC_DW_3_width_mask 0x00003FFF
+#define SDMA_PKT_WRITE_TILED_BC_DW_3_width_shift 0
+#define SDMA_PKT_WRITE_TILED_BC_DW_3_WIDTH(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_3_width_mask) << SDMA_PKT_WRITE_TILED_BC_DW_3_width_shift)
+
+/*define for DW_4 word*/
+/*define for height field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_4_height_offset 4
+#define SDMA_PKT_WRITE_TILED_BC_DW_4_height_mask 0x00003FFF
+#define SDMA_PKT_WRITE_TILED_BC_DW_4_height_shift 0
+#define SDMA_PKT_WRITE_TILED_BC_DW_4_HEIGHT(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_4_height_mask) << SDMA_PKT_WRITE_TILED_BC_DW_4_height_shift)
+
+/*define for depth field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_4_depth_offset 4
+#define SDMA_PKT_WRITE_TILED_BC_DW_4_depth_mask 0x000007FF
+#define SDMA_PKT_WRITE_TILED_BC_DW_4_depth_shift 16
+#define SDMA_PKT_WRITE_TILED_BC_DW_4_DEPTH(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_4_depth_mask) << SDMA_PKT_WRITE_TILED_BC_DW_4_depth_shift)
+
+/*define for DW_5 word*/
+/*define for element_size field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_element_size_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_element_size_mask 0x00000007
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_element_size_shift 0
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_ELEMENT_SIZE(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_element_size_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_element_size_shift)
+
+/*define for array_mode field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_array_mode_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_array_mode_mask 0x0000000F
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_array_mode_shift 3
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_ARRAY_MODE(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_array_mode_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_array_mode_shift)
+
+/*define for mit_mode field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_mit_mode_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_mit_mode_mask 0x00000007
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_mit_mode_shift 8
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_MIT_MODE(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_mit_mode_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_mit_mode_shift)
+
+/*define for tilesplit_size field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_tilesplit_size_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_tilesplit_size_mask 0x00000007
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_tilesplit_size_shift 11
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_TILESPLIT_SIZE(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_tilesplit_size_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_tilesplit_size_shift)
+
+/*define for bank_w field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_bank_w_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_bank_w_mask 0x00000003
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_bank_w_shift 15
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_BANK_W(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_bank_w_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_bank_w_shift)
+
+/*define for bank_h field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_bank_h_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_bank_h_mask 0x00000003
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_bank_h_shift 18
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_BANK_H(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_bank_h_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_bank_h_shift)
+
+/*define for num_bank field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_num_bank_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_num_bank_mask 0x00000003
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_num_bank_shift 21
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_NUM_BANK(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_num_bank_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_num_bank_shift)
+
+/*define for mat_aspt field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_mat_aspt_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_mat_aspt_mask 0x00000003
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_mat_aspt_shift 24
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_MAT_ASPT(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_mat_aspt_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_mat_aspt_shift)
+
+/*define for pipe_config field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_pipe_config_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_pipe_config_mask 0x0000001F
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_pipe_config_shift 26
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_PIPE_CONFIG(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_pipe_config_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_pipe_config_shift)
+
+/*define for DW_6 word*/
+/*define for x field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_6_x_offset 6
+#define SDMA_PKT_WRITE_TILED_BC_DW_6_x_mask 0x00003FFF
+#define SDMA_PKT_WRITE_TILED_BC_DW_6_x_shift 0
+#define SDMA_PKT_WRITE_TILED_BC_DW_6_X(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_6_x_mask) << SDMA_PKT_WRITE_TILED_BC_DW_6_x_shift)
+
+/*define for y field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_6_y_offset 6
+#define SDMA_PKT_WRITE_TILED_BC_DW_6_y_mask 0x00003FFF
+#define SDMA_PKT_WRITE_TILED_BC_DW_6_y_shift 16
+#define SDMA_PKT_WRITE_TILED_BC_DW_6_Y(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_6_y_mask) << SDMA_PKT_WRITE_TILED_BC_DW_6_y_shift)
+
+/*define for DW_7 word*/
+/*define for z field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_7_z_offset 7
+#define SDMA_PKT_WRITE_TILED_BC_DW_7_z_mask 0x000007FF
+#define SDMA_PKT_WRITE_TILED_BC_DW_7_z_shift 0
+#define SDMA_PKT_WRITE_TILED_BC_DW_7_Z(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_7_z_mask) << SDMA_PKT_WRITE_TILED_BC_DW_7_z_shift)
+
+/*define for sw field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_7_sw_offset 7
+#define SDMA_PKT_WRITE_TILED_BC_DW_7_sw_mask 0x00000003
+#define SDMA_PKT_WRITE_TILED_BC_DW_7_sw_shift 24
+#define SDMA_PKT_WRITE_TILED_BC_DW_7_SW(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_7_sw_mask) << SDMA_PKT_WRITE_TILED_BC_DW_7_sw_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_WRITE_TILED_BC_COUNT_count_offset 8
+#define SDMA_PKT_WRITE_TILED_BC_COUNT_count_mask 0x000FFFFF
+#define SDMA_PKT_WRITE_TILED_BC_COUNT_count_shift 2
+#define SDMA_PKT_WRITE_TILED_BC_COUNT_COUNT(x) (((x) & SDMA_PKT_WRITE_TILED_BC_COUNT_count_mask) << SDMA_PKT_WRITE_TILED_BC_COUNT_count_shift)
+
+/*define for DATA0 word*/
+/*define for data0 field*/
+#define SDMA_PKT_WRITE_TILED_BC_DATA0_data0_offset 9
+#define SDMA_PKT_WRITE_TILED_BC_DATA0_data0_mask 0xFFFFFFFF
+#define SDMA_PKT_WRITE_TILED_BC_DATA0_data0_shift 0
+#define SDMA_PKT_WRITE_TILED_BC_DATA0_DATA0(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DATA0_data0_mask) << SDMA_PKT_WRITE_TILED_BC_DATA0_data0_shift)
+
+
+/*
+** Definitions for SDMA_PKT_PTEPDE_COPY packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_PTEPDE_COPY_HEADER_op_offset 0
+#define SDMA_PKT_PTEPDE_COPY_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_PTEPDE_COPY_HEADER_op_shift 0
+#define SDMA_PKT_PTEPDE_COPY_HEADER_OP(x) (((x) & SDMA_PKT_PTEPDE_COPY_HEADER_op_mask) << SDMA_PKT_PTEPDE_COPY_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_PTEPDE_COPY_HEADER_sub_op_offset 0
+#define SDMA_PKT_PTEPDE_COPY_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_PTEPDE_COPY_HEADER_sub_op_shift 8
+#define SDMA_PKT_PTEPDE_COPY_HEADER_SUB_OP(x) (((x) & SDMA_PKT_PTEPDE_COPY_HEADER_sub_op_mask) << SDMA_PKT_PTEPDE_COPY_HEADER_sub_op_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_PTEPDE_COPY_HEADER_tmz_offset 0
+#define SDMA_PKT_PTEPDE_COPY_HEADER_tmz_mask 0x00000001
+#define SDMA_PKT_PTEPDE_COPY_HEADER_tmz_shift 18
+#define SDMA_PKT_PTEPDE_COPY_HEADER_TMZ(x) (((x) & SDMA_PKT_PTEPDE_COPY_HEADER_tmz_mask) << SDMA_PKT_PTEPDE_COPY_HEADER_tmz_shift)
+
+/*define for ptepde_op field*/
+#define SDMA_PKT_PTEPDE_COPY_HEADER_ptepde_op_offset 0
+#define SDMA_PKT_PTEPDE_COPY_HEADER_ptepde_op_mask 0x00000001
+#define SDMA_PKT_PTEPDE_COPY_HEADER_ptepde_op_shift 31
+#define SDMA_PKT_PTEPDE_COPY_HEADER_PTEPDE_OP(x) (((x) & SDMA_PKT_PTEPDE_COPY_HEADER_ptepde_op_mask) << SDMA_PKT_PTEPDE_COPY_HEADER_ptepde_op_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_PTEPDE_COPY_SRC_ADDR_LO_src_addr_31_0_offset 1
+#define SDMA_PKT_PTEPDE_COPY_SRC_ADDR_LO_src_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_SRC_ADDR_LO_src_addr_31_0_shift 0
+#define SDMA_PKT_PTEPDE_COPY_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_PTEPDE_COPY_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_PTEPDE_COPY_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_PTEPDE_COPY_SRC_ADDR_HI_src_addr_63_32_offset 2
+#define SDMA_PKT_PTEPDE_COPY_SRC_ADDR_HI_src_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_SRC_ADDR_HI_src_addr_63_32_shift 0
+#define SDMA_PKT_PTEPDE_COPY_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_PTEPDE_COPY_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_PTEPDE_COPY_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_PTEPDE_COPY_DST_ADDR_LO_dst_addr_31_0_offset 3
+#define SDMA_PKT_PTEPDE_COPY_DST_ADDR_LO_dst_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_DST_ADDR_LO_dst_addr_31_0_shift 0
+#define SDMA_PKT_PTEPDE_COPY_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_PTEPDE_COPY_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_PTEPDE_COPY_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_PTEPDE_COPY_DST_ADDR_HI_dst_addr_63_32_offset 4
+#define SDMA_PKT_PTEPDE_COPY_DST_ADDR_HI_dst_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_DST_ADDR_HI_dst_addr_63_32_shift 0
+#define SDMA_PKT_PTEPDE_COPY_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_PTEPDE_COPY_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_PTEPDE_COPY_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for MASK_DW0 word*/
+/*define for mask_dw0 field*/
+#define SDMA_PKT_PTEPDE_COPY_MASK_DW0_mask_dw0_offset 5
+#define SDMA_PKT_PTEPDE_COPY_MASK_DW0_mask_dw0_mask 0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_MASK_DW0_mask_dw0_shift 0
+#define SDMA_PKT_PTEPDE_COPY_MASK_DW0_MASK_DW0(x) (((x) & SDMA_PKT_PTEPDE_COPY_MASK_DW0_mask_dw0_mask) << SDMA_PKT_PTEPDE_COPY_MASK_DW0_mask_dw0_shift)
+
+/*define for MASK_DW1 word*/
+/*define for mask_dw1 field*/
+#define SDMA_PKT_PTEPDE_COPY_MASK_DW1_mask_dw1_offset 6
+#define SDMA_PKT_PTEPDE_COPY_MASK_DW1_mask_dw1_mask 0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_MASK_DW1_mask_dw1_shift 0
+#define SDMA_PKT_PTEPDE_COPY_MASK_DW1_MASK_DW1(x) (((x) & SDMA_PKT_PTEPDE_COPY_MASK_DW1_mask_dw1_mask) << SDMA_PKT_PTEPDE_COPY_MASK_DW1_mask_dw1_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_PTEPDE_COPY_COUNT_count_offset 7
+#define SDMA_PKT_PTEPDE_COPY_COUNT_count_mask 0x0007FFFF
+#define SDMA_PKT_PTEPDE_COPY_COUNT_count_shift 0
+#define SDMA_PKT_PTEPDE_COPY_COUNT_COUNT(x) (((x) & SDMA_PKT_PTEPDE_COPY_COUNT_count_mask) << SDMA_PKT_PTEPDE_COPY_COUNT_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_PTEPDE_COPY_BACKWARDS packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_op_offset 0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_op_shift 0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_OP(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_op_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_sub_op_offset 0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_sub_op_shift 8
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_SUB_OP(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_sub_op_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_sub_op_shift)
+
+/*define for pte_size field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_pte_size_offset 0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_pte_size_mask 0x00000003
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_pte_size_shift 28
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_PTE_SIZE(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_pte_size_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_pte_size_shift)
+
+/*define for direction field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_direction_offset 0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_direction_mask 0x00000001
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_direction_shift 30
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_DIRECTION(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_direction_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_direction_shift)
+
+/*define for ptepde_op field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_ptepde_op_offset 0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_ptepde_op_mask 0x00000001
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_ptepde_op_shift 31
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_PTEPDE_OP(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_ptepde_op_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_ptepde_op_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_LO_src_addr_31_0_offset 1
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_LO_src_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_LO_src_addr_31_0_shift 0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_HI_src_addr_63_32_offset 2
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_HI_src_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_HI_src_addr_63_32_shift 0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_LO_dst_addr_31_0_offset 3
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_LO_dst_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_LO_dst_addr_31_0_shift 0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_HI_dst_addr_63_32_offset 4
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_HI_dst_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_HI_dst_addr_63_32_shift 0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for MASK_BIT_FOR_DW word*/
+/*define for mask_first_xfer field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_first_xfer_offset 5
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_first_xfer_mask 0x000000FF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_first_xfer_shift 0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_MASK_FIRST_XFER(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_first_xfer_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_first_xfer_shift)
+
+/*define for mask_last_xfer field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_last_xfer_offset 5
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_last_xfer_mask 0x000000FF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_last_xfer_shift 8
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_MASK_LAST_XFER(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_last_xfer_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_last_xfer_shift)
+
+/*define for COUNT_IN_32B_XFER word*/
+/*define for count field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_COUNT_IN_32B_XFER_count_offset 6
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_COUNT_IN_32B_XFER_count_mask 0x0001FFFF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_COUNT_IN_32B_XFER_count_shift 0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_COUNT_IN_32B_XFER_COUNT(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_COUNT_IN_32B_XFER_count_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_COUNT_IN_32B_XFER_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_PTEPDE_RMW packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_op_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_PTEPDE_RMW_HEADER_op_shift 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_OP(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_op_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_sub_op_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_PTEPDE_RMW_HEADER_sub_op_shift 8
+#define SDMA_PKT_PTEPDE_RMW_HEADER_SUB_OP(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_sub_op_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_sub_op_shift)
+
+/*define for mtype field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_mtype_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_mtype_mask 0x00000007
+#define SDMA_PKT_PTEPDE_RMW_HEADER_mtype_shift 16
+#define SDMA_PKT_PTEPDE_RMW_HEADER_MTYPE(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_mtype_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_mtype_shift)
+
+/*define for gcc field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_gcc_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_gcc_mask 0x00000001
+#define SDMA_PKT_PTEPDE_RMW_HEADER_gcc_shift 19
+#define SDMA_PKT_PTEPDE_RMW_HEADER_GCC(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_gcc_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_gcc_shift)
+
+/*define for sys field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_sys_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_sys_mask 0x00000001
+#define SDMA_PKT_PTEPDE_RMW_HEADER_sys_shift 20
+#define SDMA_PKT_PTEPDE_RMW_HEADER_SYS(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_sys_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_sys_shift)
+
+/*define for snp field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_snp_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_snp_mask 0x00000001
+#define SDMA_PKT_PTEPDE_RMW_HEADER_snp_shift 22
+#define SDMA_PKT_PTEPDE_RMW_HEADER_SNP(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_snp_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_snp_shift)
+
+/*define for gpa field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_gpa_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_gpa_mask 0x00000001
+#define SDMA_PKT_PTEPDE_RMW_HEADER_gpa_shift 23
+#define SDMA_PKT_PTEPDE_RMW_HEADER_GPA(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_gpa_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_gpa_shift)
+
+/*define for l2_policy field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_l2_policy_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_l2_policy_mask 0x00000003
+#define SDMA_PKT_PTEPDE_RMW_HEADER_l2_policy_shift 24
+#define SDMA_PKT_PTEPDE_RMW_HEADER_L2_POLICY(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_l2_policy_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_l2_policy_shift)
+
+/*define for ADDR_LO word*/
+/*define for addr_31_0 field*/
+#define SDMA_PKT_PTEPDE_RMW_ADDR_LO_addr_31_0_offset 1
+#define SDMA_PKT_PTEPDE_RMW_ADDR_LO_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_RMW_ADDR_LO_addr_31_0_shift 0
+#define SDMA_PKT_PTEPDE_RMW_ADDR_LO_ADDR_31_0(x) (((x) & SDMA_PKT_PTEPDE_RMW_ADDR_LO_addr_31_0_mask) << SDMA_PKT_PTEPDE_RMW_ADDR_LO_addr_31_0_shift)
+
+/*define for ADDR_HI word*/
+/*define for addr_63_32 field*/
+#define SDMA_PKT_PTEPDE_RMW_ADDR_HI_addr_63_32_offset 2
+#define SDMA_PKT_PTEPDE_RMW_ADDR_HI_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_RMW_ADDR_HI_addr_63_32_shift 0
+#define SDMA_PKT_PTEPDE_RMW_ADDR_HI_ADDR_63_32(x) (((x) & SDMA_PKT_PTEPDE_RMW_ADDR_HI_addr_63_32_mask) << SDMA_PKT_PTEPDE_RMW_ADDR_HI_addr_63_32_shift)
+
+/*define for MASK_LO word*/
+/*define for mask_31_0 field*/
+#define SDMA_PKT_PTEPDE_RMW_MASK_LO_mask_31_0_offset 3
+#define SDMA_PKT_PTEPDE_RMW_MASK_LO_mask_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_RMW_MASK_LO_mask_31_0_shift 0
+#define SDMA_PKT_PTEPDE_RMW_MASK_LO_MASK_31_0(x) (((x) & SDMA_PKT_PTEPDE_RMW_MASK_LO_mask_31_0_mask) << SDMA_PKT_PTEPDE_RMW_MASK_LO_mask_31_0_shift)
+
+/*define for MASK_HI word*/
+/*define for mask_63_32 field*/
+#define SDMA_PKT_PTEPDE_RMW_MASK_HI_mask_63_32_offset 4
+#define SDMA_PKT_PTEPDE_RMW_MASK_HI_mask_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_RMW_MASK_HI_mask_63_32_shift 0
+#define SDMA_PKT_PTEPDE_RMW_MASK_HI_MASK_63_32(x) (((x) & SDMA_PKT_PTEPDE_RMW_MASK_HI_mask_63_32_mask) << SDMA_PKT_PTEPDE_RMW_MASK_HI_mask_63_32_shift)
+
+/*define for VALUE_LO word*/
+/*define for value_31_0 field*/
+#define SDMA_PKT_PTEPDE_RMW_VALUE_LO_value_31_0_offset 5
+#define SDMA_PKT_PTEPDE_RMW_VALUE_LO_value_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_RMW_VALUE_LO_value_31_0_shift 0
+#define SDMA_PKT_PTEPDE_RMW_VALUE_LO_VALUE_31_0(x) (((x) & SDMA_PKT_PTEPDE_RMW_VALUE_LO_value_31_0_mask) << SDMA_PKT_PTEPDE_RMW_VALUE_LO_value_31_0_shift)
+
+/*define for VALUE_HI word*/
+/*define for value_63_32 field*/
+#define SDMA_PKT_PTEPDE_RMW_VALUE_HI_value_63_32_offset 6
+#define SDMA_PKT_PTEPDE_RMW_VALUE_HI_value_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_RMW_VALUE_HI_value_63_32_shift 0
+#define SDMA_PKT_PTEPDE_RMW_VALUE_HI_VALUE_63_32(x) (((x) & SDMA_PKT_PTEPDE_RMW_VALUE_HI_value_63_32_mask) << SDMA_PKT_PTEPDE_RMW_VALUE_HI_value_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_WRITE_INCR packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_WRITE_INCR_HEADER_op_offset 0
+#define SDMA_PKT_WRITE_INCR_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_WRITE_INCR_HEADER_op_shift 0
+#define SDMA_PKT_WRITE_INCR_HEADER_OP(x) (((x) & SDMA_PKT_WRITE_INCR_HEADER_op_mask) << SDMA_PKT_WRITE_INCR_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_WRITE_INCR_HEADER_sub_op_offset 0
+#define SDMA_PKT_WRITE_INCR_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_WRITE_INCR_HEADER_sub_op_shift 8
+#define SDMA_PKT_WRITE_INCR_HEADER_SUB_OP(x) (((x) & SDMA_PKT_WRITE_INCR_HEADER_sub_op_mask) << SDMA_PKT_WRITE_INCR_HEADER_sub_op_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_WRITE_INCR_DST_ADDR_LO_dst_addr_31_0_offset 1
+#define SDMA_PKT_WRITE_INCR_DST_ADDR_LO_dst_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_WRITE_INCR_DST_ADDR_LO_dst_addr_31_0_shift 0
+#define SDMA_PKT_WRITE_INCR_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_WRITE_INCR_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_WRITE_INCR_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_WRITE_INCR_DST_ADDR_HI_dst_addr_63_32_offset 2
+#define SDMA_PKT_WRITE_INCR_DST_ADDR_HI_dst_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_WRITE_INCR_DST_ADDR_HI_dst_addr_63_32_shift 0
+#define SDMA_PKT_WRITE_INCR_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_WRITE_INCR_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_WRITE_INCR_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for MASK_DW0 word*/
+/*define for mask_dw0 field*/
+#define SDMA_PKT_WRITE_INCR_MASK_DW0_mask_dw0_offset 3
+#define SDMA_PKT_WRITE_INCR_MASK_DW0_mask_dw0_mask 0xFFFFFFFF
+#define SDMA_PKT_WRITE_INCR_MASK_DW0_mask_dw0_shift 0
+#define SDMA_PKT_WRITE_INCR_MASK_DW0_MASK_DW0(x) (((x) & SDMA_PKT_WRITE_INCR_MASK_DW0_mask_dw0_mask) << SDMA_PKT_WRITE_INCR_MASK_DW0_mask_dw0_shift)
+
+/*define for MASK_DW1 word*/
+/*define for mask_dw1 field*/
+#define SDMA_PKT_WRITE_INCR_MASK_DW1_mask_dw1_offset 4
+#define SDMA_PKT_WRITE_INCR_MASK_DW1_mask_dw1_mask 0xFFFFFFFF
+#define SDMA_PKT_WRITE_INCR_MASK_DW1_mask_dw1_shift 0
+#define SDMA_PKT_WRITE_INCR_MASK_DW1_MASK_DW1(x) (((x) & SDMA_PKT_WRITE_INCR_MASK_DW1_mask_dw1_mask) << SDMA_PKT_WRITE_INCR_MASK_DW1_mask_dw1_shift)
+
+/*define for INIT_DW0 word*/
+/*define for init_dw0 field*/
+#define SDMA_PKT_WRITE_INCR_INIT_DW0_init_dw0_offset 5
+#define SDMA_PKT_WRITE_INCR_INIT_DW0_init_dw0_mask 0xFFFFFFFF
+#define SDMA_PKT_WRITE_INCR_INIT_DW0_init_dw0_shift 0
+#define SDMA_PKT_WRITE_INCR_INIT_DW0_INIT_DW0(x) (((x) & SDMA_PKT_WRITE_INCR_INIT_DW0_init_dw0_mask) << SDMA_PKT_WRITE_INCR_INIT_DW0_init_dw0_shift)
+
+/*define for INIT_DW1 word*/
+/*define for init_dw1 field*/
+#define SDMA_PKT_WRITE_INCR_INIT_DW1_init_dw1_offset 6
+#define SDMA_PKT_WRITE_INCR_INIT_DW1_init_dw1_mask 0xFFFFFFFF
+#define SDMA_PKT_WRITE_INCR_INIT_DW1_init_dw1_shift 0
+#define SDMA_PKT_WRITE_INCR_INIT_DW1_INIT_DW1(x) (((x) & SDMA_PKT_WRITE_INCR_INIT_DW1_init_dw1_mask) << SDMA_PKT_WRITE_INCR_INIT_DW1_init_dw1_shift)
+
+/*define for INCR_DW0 word*/
+/*define for incr_dw0 field*/
+#define SDMA_PKT_WRITE_INCR_INCR_DW0_incr_dw0_offset 7
+#define SDMA_PKT_WRITE_INCR_INCR_DW0_incr_dw0_mask 0xFFFFFFFF
+#define SDMA_PKT_WRITE_INCR_INCR_DW0_incr_dw0_shift 0
+#define SDMA_PKT_WRITE_INCR_INCR_DW0_INCR_DW0(x) (((x) & SDMA_PKT_WRITE_INCR_INCR_DW0_incr_dw0_mask) << SDMA_PKT_WRITE_INCR_INCR_DW0_incr_dw0_shift)
+
+/*define for INCR_DW1 word*/
+/*define for incr_dw1 field*/
+#define SDMA_PKT_WRITE_INCR_INCR_DW1_incr_dw1_offset 8
+#define SDMA_PKT_WRITE_INCR_INCR_DW1_incr_dw1_mask 0xFFFFFFFF
+#define SDMA_PKT_WRITE_INCR_INCR_DW1_incr_dw1_shift 0
+#define SDMA_PKT_WRITE_INCR_INCR_DW1_INCR_DW1(x) (((x) & SDMA_PKT_WRITE_INCR_INCR_DW1_incr_dw1_mask) << SDMA_PKT_WRITE_INCR_INCR_DW1_incr_dw1_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_WRITE_INCR_COUNT_count_offset 9
+#define SDMA_PKT_WRITE_INCR_COUNT_count_mask 0x0007FFFF
+#define SDMA_PKT_WRITE_INCR_COUNT_count_shift 0
+#define SDMA_PKT_WRITE_INCR_COUNT_COUNT(x) (((x) & SDMA_PKT_WRITE_INCR_COUNT_count_mask) << SDMA_PKT_WRITE_INCR_COUNT_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_INDIRECT packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_INDIRECT_HEADER_op_offset 0
+#define SDMA_PKT_INDIRECT_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_INDIRECT_HEADER_op_shift 0
+#define SDMA_PKT_INDIRECT_HEADER_OP(x) (((x) & SDMA_PKT_INDIRECT_HEADER_op_mask) << SDMA_PKT_INDIRECT_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_INDIRECT_HEADER_sub_op_offset 0
+#define SDMA_PKT_INDIRECT_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_INDIRECT_HEADER_sub_op_shift 8
+#define SDMA_PKT_INDIRECT_HEADER_SUB_OP(x) (((x) & SDMA_PKT_INDIRECT_HEADER_sub_op_mask) << SDMA_PKT_INDIRECT_HEADER_sub_op_shift)
+
+/*define for vmid field*/
+#define SDMA_PKT_INDIRECT_HEADER_vmid_offset 0
+#define SDMA_PKT_INDIRECT_HEADER_vmid_mask 0x0000000F
+#define SDMA_PKT_INDIRECT_HEADER_vmid_shift 16
+#define SDMA_PKT_INDIRECT_HEADER_VMID(x) (((x) & SDMA_PKT_INDIRECT_HEADER_vmid_mask) << SDMA_PKT_INDIRECT_HEADER_vmid_shift)
+
+/*define for priv field*/
+#define SDMA_PKT_INDIRECT_HEADER_priv_offset 0
+#define SDMA_PKT_INDIRECT_HEADER_priv_mask 0x00000001
+#define SDMA_PKT_INDIRECT_HEADER_priv_shift 31
+#define SDMA_PKT_INDIRECT_HEADER_PRIV(x) (((x) & SDMA_PKT_INDIRECT_HEADER_priv_mask) << SDMA_PKT_INDIRECT_HEADER_priv_shift)
+
+/*define for BASE_LO word*/
+/*define for ib_base_31_0 field*/
+#define SDMA_PKT_INDIRECT_BASE_LO_ib_base_31_0_offset 1
+#define SDMA_PKT_INDIRECT_BASE_LO_ib_base_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_INDIRECT_BASE_LO_ib_base_31_0_shift 0
+#define SDMA_PKT_INDIRECT_BASE_LO_IB_BASE_31_0(x) (((x) & SDMA_PKT_INDIRECT_BASE_LO_ib_base_31_0_mask) << SDMA_PKT_INDIRECT_BASE_LO_ib_base_31_0_shift)
+
+/*define for BASE_HI word*/
+/*define for ib_base_63_32 field*/
+#define SDMA_PKT_INDIRECT_BASE_HI_ib_base_63_32_offset 2
+#define SDMA_PKT_INDIRECT_BASE_HI_ib_base_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_INDIRECT_BASE_HI_ib_base_63_32_shift 0
+#define SDMA_PKT_INDIRECT_BASE_HI_IB_BASE_63_32(x) (((x) & SDMA_PKT_INDIRECT_BASE_HI_ib_base_63_32_mask) << SDMA_PKT_INDIRECT_BASE_HI_ib_base_63_32_shift)
+
+/*define for IB_SIZE word*/
+/*define for ib_size field*/
+#define SDMA_PKT_INDIRECT_IB_SIZE_ib_size_offset 3
+#define SDMA_PKT_INDIRECT_IB_SIZE_ib_size_mask 0x000FFFFF
+#define SDMA_PKT_INDIRECT_IB_SIZE_ib_size_shift 0
+#define SDMA_PKT_INDIRECT_IB_SIZE_IB_SIZE(x) (((x) & SDMA_PKT_INDIRECT_IB_SIZE_ib_size_mask) << SDMA_PKT_INDIRECT_IB_SIZE_ib_size_shift)
+
+/*define for CSA_ADDR_LO word*/
+/*define for csa_addr_31_0 field*/
+#define SDMA_PKT_INDIRECT_CSA_ADDR_LO_csa_addr_31_0_offset 4
+#define SDMA_PKT_INDIRECT_CSA_ADDR_LO_csa_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_INDIRECT_CSA_ADDR_LO_csa_addr_31_0_shift 0
+#define SDMA_PKT_INDIRECT_CSA_ADDR_LO_CSA_ADDR_31_0(x) (((x) & SDMA_PKT_INDIRECT_CSA_ADDR_LO_csa_addr_31_0_mask) << SDMA_PKT_INDIRECT_CSA_ADDR_LO_csa_addr_31_0_shift)
+
+/*define for CSA_ADDR_HI word*/
+/*define for csa_addr_63_32 field*/
+#define SDMA_PKT_INDIRECT_CSA_ADDR_HI_csa_addr_63_32_offset 5
+#define SDMA_PKT_INDIRECT_CSA_ADDR_HI_csa_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_INDIRECT_CSA_ADDR_HI_csa_addr_63_32_shift 0
+#define SDMA_PKT_INDIRECT_CSA_ADDR_HI_CSA_ADDR_63_32(x) (((x) & SDMA_PKT_INDIRECT_CSA_ADDR_HI_csa_addr_63_32_mask) << SDMA_PKT_INDIRECT_CSA_ADDR_HI_csa_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_SEMAPHORE packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_SEMAPHORE_HEADER_op_offset 0
+#define SDMA_PKT_SEMAPHORE_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_SEMAPHORE_HEADER_op_shift 0
+#define SDMA_PKT_SEMAPHORE_HEADER_OP(x) (((x) & SDMA_PKT_SEMAPHORE_HEADER_op_mask) << SDMA_PKT_SEMAPHORE_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_SEMAPHORE_HEADER_sub_op_offset 0
+#define SDMA_PKT_SEMAPHORE_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_SEMAPHORE_HEADER_sub_op_shift 8
+#define SDMA_PKT_SEMAPHORE_HEADER_SUB_OP(x) (((x) & SDMA_PKT_SEMAPHORE_HEADER_sub_op_mask) << SDMA_PKT_SEMAPHORE_HEADER_sub_op_shift)
+
+/*define for write_one field*/
+#define SDMA_PKT_SEMAPHORE_HEADER_write_one_offset 0
+#define SDMA_PKT_SEMAPHORE_HEADER_write_one_mask 0x00000001
+#define SDMA_PKT_SEMAPHORE_HEADER_write_one_shift 29
+#define SDMA_PKT_SEMAPHORE_HEADER_WRITE_ONE(x) (((x) & SDMA_PKT_SEMAPHORE_HEADER_write_one_mask) << SDMA_PKT_SEMAPHORE_HEADER_write_one_shift)
+
+/*define for signal field*/
+#define SDMA_PKT_SEMAPHORE_HEADER_signal_offset 0
+#define SDMA_PKT_SEMAPHORE_HEADER_signal_mask 0x00000001
+#define SDMA_PKT_SEMAPHORE_HEADER_signal_shift 30
+#define SDMA_PKT_SEMAPHORE_HEADER_SIGNAL(x) (((x) & SDMA_PKT_SEMAPHORE_HEADER_signal_mask) << SDMA_PKT_SEMAPHORE_HEADER_signal_shift)
+
+/*define for mailbox field*/
+#define SDMA_PKT_SEMAPHORE_HEADER_mailbox_offset 0
+#define SDMA_PKT_SEMAPHORE_HEADER_mailbox_mask 0x00000001
+#define SDMA_PKT_SEMAPHORE_HEADER_mailbox_shift 31
+#define SDMA_PKT_SEMAPHORE_HEADER_MAILBOX(x) (((x) & SDMA_PKT_SEMAPHORE_HEADER_mailbox_mask) << SDMA_PKT_SEMAPHORE_HEADER_mailbox_shift)
+
+/*define for ADDR_LO word*/
+/*define for addr_31_0 field*/
+#define SDMA_PKT_SEMAPHORE_ADDR_LO_addr_31_0_offset 1
+#define SDMA_PKT_SEMAPHORE_ADDR_LO_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_SEMAPHORE_ADDR_LO_addr_31_0_shift 0
+#define SDMA_PKT_SEMAPHORE_ADDR_LO_ADDR_31_0(x) (((x) & SDMA_PKT_SEMAPHORE_ADDR_LO_addr_31_0_mask) << SDMA_PKT_SEMAPHORE_ADDR_LO_addr_31_0_shift)
+
+/*define for ADDR_HI word*/
+/*define for addr_63_32 field*/
+#define SDMA_PKT_SEMAPHORE_ADDR_HI_addr_63_32_offset 2
+#define SDMA_PKT_SEMAPHORE_ADDR_HI_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_SEMAPHORE_ADDR_HI_addr_63_32_shift 0
+#define SDMA_PKT_SEMAPHORE_ADDR_HI_ADDR_63_32(x) (((x) & SDMA_PKT_SEMAPHORE_ADDR_HI_addr_63_32_mask) << SDMA_PKT_SEMAPHORE_ADDR_HI_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_FENCE packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_FENCE_HEADER_op_offset 0
+#define SDMA_PKT_FENCE_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_FENCE_HEADER_op_shift 0
+#define SDMA_PKT_FENCE_HEADER_OP(x) (((x) & SDMA_PKT_FENCE_HEADER_op_mask) << SDMA_PKT_FENCE_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_FENCE_HEADER_sub_op_offset 0
+#define SDMA_PKT_FENCE_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_FENCE_HEADER_sub_op_shift 8
+#define SDMA_PKT_FENCE_HEADER_SUB_OP(x) (((x) & SDMA_PKT_FENCE_HEADER_sub_op_mask) << SDMA_PKT_FENCE_HEADER_sub_op_shift)
+
+/*define for mtype field*/
+#define SDMA_PKT_FENCE_HEADER_mtype_offset 0
+#define SDMA_PKT_FENCE_HEADER_mtype_mask 0x00000007
+#define SDMA_PKT_FENCE_HEADER_mtype_shift 16
+#define SDMA_PKT_FENCE_HEADER_MTYPE(x) (((x) & SDMA_PKT_FENCE_HEADER_mtype_mask) << SDMA_PKT_FENCE_HEADER_mtype_shift)
+
+/*define for gcc field*/
+#define SDMA_PKT_FENCE_HEADER_gcc_offset 0
+#define SDMA_PKT_FENCE_HEADER_gcc_mask 0x00000001
+#define SDMA_PKT_FENCE_HEADER_gcc_shift 19
+#define SDMA_PKT_FENCE_HEADER_GCC(x) (((x) & SDMA_PKT_FENCE_HEADER_gcc_mask) << SDMA_PKT_FENCE_HEADER_gcc_shift)
+
+/*define for sys field*/
+#define SDMA_PKT_FENCE_HEADER_sys_offset 0
+#define SDMA_PKT_FENCE_HEADER_sys_mask 0x00000001
+#define SDMA_PKT_FENCE_HEADER_sys_shift 20
+#define SDMA_PKT_FENCE_HEADER_SYS(x) (((x) & SDMA_PKT_FENCE_HEADER_sys_mask) << SDMA_PKT_FENCE_HEADER_sys_shift)
+
+/*define for snp field*/
+#define SDMA_PKT_FENCE_HEADER_snp_offset 0
+#define SDMA_PKT_FENCE_HEADER_snp_mask 0x00000001
+#define SDMA_PKT_FENCE_HEADER_snp_shift 22
+#define SDMA_PKT_FENCE_HEADER_SNP(x) (((x) & SDMA_PKT_FENCE_HEADER_snp_mask) << SDMA_PKT_FENCE_HEADER_snp_shift)
+
+/*define for gpa field*/
+#define SDMA_PKT_FENCE_HEADER_gpa_offset 0
+#define SDMA_PKT_FENCE_HEADER_gpa_mask 0x00000001
+#define SDMA_PKT_FENCE_HEADER_gpa_shift 23
+#define SDMA_PKT_FENCE_HEADER_GPA(x) (((x) & SDMA_PKT_FENCE_HEADER_gpa_mask) << SDMA_PKT_FENCE_HEADER_gpa_shift)
+
+/*define for l2_policy field*/
+#define SDMA_PKT_FENCE_HEADER_l2_policy_offset 0
+#define SDMA_PKT_FENCE_HEADER_l2_policy_mask 0x00000003
+#define SDMA_PKT_FENCE_HEADER_l2_policy_shift 24
+#define SDMA_PKT_FENCE_HEADER_L2_POLICY(x) (((x) & SDMA_PKT_FENCE_HEADER_l2_policy_mask) << SDMA_PKT_FENCE_HEADER_l2_policy_shift)
+
+/*define for ADDR_LO word*/
+/*define for addr_31_0 field*/
+#define SDMA_PKT_FENCE_ADDR_LO_addr_31_0_offset 1
+#define SDMA_PKT_FENCE_ADDR_LO_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_FENCE_ADDR_LO_addr_31_0_shift 0
+#define SDMA_PKT_FENCE_ADDR_LO_ADDR_31_0(x) (((x) & SDMA_PKT_FENCE_ADDR_LO_addr_31_0_mask) << SDMA_PKT_FENCE_ADDR_LO_addr_31_0_shift)
+
+/*define for ADDR_HI word*/
+/*define for addr_63_32 field*/
+#define SDMA_PKT_FENCE_ADDR_HI_addr_63_32_offset 2
+#define SDMA_PKT_FENCE_ADDR_HI_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_FENCE_ADDR_HI_addr_63_32_shift 0
+#define SDMA_PKT_FENCE_ADDR_HI_ADDR_63_32(x) (((x) & SDMA_PKT_FENCE_ADDR_HI_addr_63_32_mask) << SDMA_PKT_FENCE_ADDR_HI_addr_63_32_shift)
+
+/*define for DATA word*/
+/*define for data field*/
+#define SDMA_PKT_FENCE_DATA_data_offset 3
+#define SDMA_PKT_FENCE_DATA_data_mask 0xFFFFFFFF
+#define SDMA_PKT_FENCE_DATA_data_shift 0
+#define SDMA_PKT_FENCE_DATA_DATA(x) (((x) & SDMA_PKT_FENCE_DATA_data_mask) << SDMA_PKT_FENCE_DATA_data_shift)
+
+
+/*
+** Definitions for SDMA_PKT_SRBM_WRITE packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_SRBM_WRITE_HEADER_op_offset 0
+#define SDMA_PKT_SRBM_WRITE_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_SRBM_WRITE_HEADER_op_shift 0
+#define SDMA_PKT_SRBM_WRITE_HEADER_OP(x) (((x) & SDMA_PKT_SRBM_WRITE_HEADER_op_mask) << SDMA_PKT_SRBM_WRITE_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_SRBM_WRITE_HEADER_sub_op_offset 0
+#define SDMA_PKT_SRBM_WRITE_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_SRBM_WRITE_HEADER_sub_op_shift 8
+#define SDMA_PKT_SRBM_WRITE_HEADER_SUB_OP(x) (((x) & SDMA_PKT_SRBM_WRITE_HEADER_sub_op_mask) << SDMA_PKT_SRBM_WRITE_HEADER_sub_op_shift)
+
+/*define for byte_en field*/
+#define SDMA_PKT_SRBM_WRITE_HEADER_byte_en_offset 0
+#define SDMA_PKT_SRBM_WRITE_HEADER_byte_en_mask 0x0000000F
+#define SDMA_PKT_SRBM_WRITE_HEADER_byte_en_shift 28
+#define SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(x) (((x) & SDMA_PKT_SRBM_WRITE_HEADER_byte_en_mask) << SDMA_PKT_SRBM_WRITE_HEADER_byte_en_shift)
+
+/*define for ADDR word*/
+/*define for addr field*/
+#define SDMA_PKT_SRBM_WRITE_ADDR_addr_offset 1
+#define SDMA_PKT_SRBM_WRITE_ADDR_addr_mask 0x0003FFFF
+#define SDMA_PKT_SRBM_WRITE_ADDR_addr_shift 0
+#define SDMA_PKT_SRBM_WRITE_ADDR_ADDR(x) (((x) & SDMA_PKT_SRBM_WRITE_ADDR_addr_mask) << SDMA_PKT_SRBM_WRITE_ADDR_addr_shift)
+
+/*define for apertureid field*/
+#define SDMA_PKT_SRBM_WRITE_ADDR_apertureid_offset 1
+#define SDMA_PKT_SRBM_WRITE_ADDR_apertureid_mask 0x00000FFF
+#define SDMA_PKT_SRBM_WRITE_ADDR_apertureid_shift 20
+#define SDMA_PKT_SRBM_WRITE_ADDR_APERTUREID(x) (((x) & SDMA_PKT_SRBM_WRITE_ADDR_apertureid_mask) << SDMA_PKT_SRBM_WRITE_ADDR_apertureid_shift)
+
+/*define for DATA word*/
+/*define for data field*/
+#define SDMA_PKT_SRBM_WRITE_DATA_data_offset 2
+#define SDMA_PKT_SRBM_WRITE_DATA_data_mask 0xFFFFFFFF
+#define SDMA_PKT_SRBM_WRITE_DATA_data_shift 0
+#define SDMA_PKT_SRBM_WRITE_DATA_DATA(x) (((x) & SDMA_PKT_SRBM_WRITE_DATA_data_mask) << SDMA_PKT_SRBM_WRITE_DATA_data_shift)
+
+
+/*
+** Definitions for SDMA_PKT_PRE_EXE packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_PRE_EXE_HEADER_op_offset 0
+#define SDMA_PKT_PRE_EXE_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_PRE_EXE_HEADER_op_shift 0
+#define SDMA_PKT_PRE_EXE_HEADER_OP(x) (((x) & SDMA_PKT_PRE_EXE_HEADER_op_mask) << SDMA_PKT_PRE_EXE_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_PRE_EXE_HEADER_sub_op_offset 0
+#define SDMA_PKT_PRE_EXE_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_PRE_EXE_HEADER_sub_op_shift 8
+#define SDMA_PKT_PRE_EXE_HEADER_SUB_OP(x) (((x) & SDMA_PKT_PRE_EXE_HEADER_sub_op_mask) << SDMA_PKT_PRE_EXE_HEADER_sub_op_shift)
+
+/*define for dev_sel field*/
+#define SDMA_PKT_PRE_EXE_HEADER_dev_sel_offset 0
+#define SDMA_PKT_PRE_EXE_HEADER_dev_sel_mask 0x000000FF
+#define SDMA_PKT_PRE_EXE_HEADER_dev_sel_shift 16
+#define SDMA_PKT_PRE_EXE_HEADER_DEV_SEL(x) (((x) & SDMA_PKT_PRE_EXE_HEADER_dev_sel_mask) << SDMA_PKT_PRE_EXE_HEADER_dev_sel_shift)
+
+/*define for EXEC_COUNT word*/
+/*define for exec_count field*/
+#define SDMA_PKT_PRE_EXE_EXEC_COUNT_exec_count_offset 1
+#define SDMA_PKT_PRE_EXE_EXEC_COUNT_exec_count_mask 0x00003FFF
+#define SDMA_PKT_PRE_EXE_EXEC_COUNT_exec_count_shift 0
+#define SDMA_PKT_PRE_EXE_EXEC_COUNT_EXEC_COUNT(x) (((x) & SDMA_PKT_PRE_EXE_EXEC_COUNT_exec_count_mask) << SDMA_PKT_PRE_EXE_EXEC_COUNT_exec_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COND_EXE packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COND_EXE_HEADER_op_offset 0
+#define SDMA_PKT_COND_EXE_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_COND_EXE_HEADER_op_shift 0
+#define SDMA_PKT_COND_EXE_HEADER_OP(x) (((x) & SDMA_PKT_COND_EXE_HEADER_op_mask) << SDMA_PKT_COND_EXE_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COND_EXE_HEADER_sub_op_offset 0
+#define SDMA_PKT_COND_EXE_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_COND_EXE_HEADER_sub_op_shift 8
+#define SDMA_PKT_COND_EXE_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COND_EXE_HEADER_sub_op_mask) << SDMA_PKT_COND_EXE_HEADER_sub_op_shift)
+
+/*define for ADDR_LO word*/
+/*define for addr_31_0 field*/
+#define SDMA_PKT_COND_EXE_ADDR_LO_addr_31_0_offset 1
+#define SDMA_PKT_COND_EXE_ADDR_LO_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_COND_EXE_ADDR_LO_addr_31_0_shift 0
+#define SDMA_PKT_COND_EXE_ADDR_LO_ADDR_31_0(x) (((x) & SDMA_PKT_COND_EXE_ADDR_LO_addr_31_0_mask) << SDMA_PKT_COND_EXE_ADDR_LO_addr_31_0_shift)
+
+/*define for ADDR_HI word*/
+/*define for addr_63_32 field*/
+#define SDMA_PKT_COND_EXE_ADDR_HI_addr_63_32_offset 2
+#define SDMA_PKT_COND_EXE_ADDR_HI_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_COND_EXE_ADDR_HI_addr_63_32_shift 0
+#define SDMA_PKT_COND_EXE_ADDR_HI_ADDR_63_32(x) (((x) & SDMA_PKT_COND_EXE_ADDR_HI_addr_63_32_mask) << SDMA_PKT_COND_EXE_ADDR_HI_addr_63_32_shift)
+
+/*define for REFERENCE word*/
+/*define for reference field*/
+#define SDMA_PKT_COND_EXE_REFERENCE_reference_offset 3
+#define SDMA_PKT_COND_EXE_REFERENCE_reference_mask 0xFFFFFFFF
+#define SDMA_PKT_COND_EXE_REFERENCE_reference_shift 0
+#define SDMA_PKT_COND_EXE_REFERENCE_REFERENCE(x) (((x) & SDMA_PKT_COND_EXE_REFERENCE_reference_mask) << SDMA_PKT_COND_EXE_REFERENCE_reference_shift)
+
+/*define for EXEC_COUNT word*/
+/*define for exec_count field*/
+#define SDMA_PKT_COND_EXE_EXEC_COUNT_exec_count_offset 4
+#define SDMA_PKT_COND_EXE_EXEC_COUNT_exec_count_mask 0x00003FFF
+#define SDMA_PKT_COND_EXE_EXEC_COUNT_exec_count_shift 0
+#define SDMA_PKT_COND_EXE_EXEC_COUNT_EXEC_COUNT(x) (((x) & SDMA_PKT_COND_EXE_EXEC_COUNT_exec_count_mask) << SDMA_PKT_COND_EXE_EXEC_COUNT_exec_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_CONSTANT_FILL packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_CONSTANT_FILL_HEADER_op_offset 0
+#define SDMA_PKT_CONSTANT_FILL_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_CONSTANT_FILL_HEADER_op_shift 0
+#define SDMA_PKT_CONSTANT_FILL_HEADER_OP(x) (((x) & SDMA_PKT_CONSTANT_FILL_HEADER_op_mask) << SDMA_PKT_CONSTANT_FILL_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_CONSTANT_FILL_HEADER_sub_op_offset 0
+#define SDMA_PKT_CONSTANT_FILL_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_CONSTANT_FILL_HEADER_sub_op_shift 8
+#define SDMA_PKT_CONSTANT_FILL_HEADER_SUB_OP(x) (((x) & SDMA_PKT_CONSTANT_FILL_HEADER_sub_op_mask) << SDMA_PKT_CONSTANT_FILL_HEADER_sub_op_shift)
+
+/*define for sw field*/
+#define SDMA_PKT_CONSTANT_FILL_HEADER_sw_offset 0
+#define SDMA_PKT_CONSTANT_FILL_HEADER_sw_mask 0x00000003
+#define SDMA_PKT_CONSTANT_FILL_HEADER_sw_shift 16
+#define SDMA_PKT_CONSTANT_FILL_HEADER_SW(x) (((x) & SDMA_PKT_CONSTANT_FILL_HEADER_sw_mask) << SDMA_PKT_CONSTANT_FILL_HEADER_sw_shift)
+
+/*define for fillsize field*/
+#define SDMA_PKT_CONSTANT_FILL_HEADER_fillsize_offset 0
+#define SDMA_PKT_CONSTANT_FILL_HEADER_fillsize_mask 0x00000003
+#define SDMA_PKT_CONSTANT_FILL_HEADER_fillsize_shift 30
+#define SDMA_PKT_CONSTANT_FILL_HEADER_FILLSIZE(x) (((x) & SDMA_PKT_CONSTANT_FILL_HEADER_fillsize_mask) << SDMA_PKT_CONSTANT_FILL_HEADER_fillsize_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_CONSTANT_FILL_DST_ADDR_LO_dst_addr_31_0_offset 1
+#define SDMA_PKT_CONSTANT_FILL_DST_ADDR_LO_dst_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_CONSTANT_FILL_DST_ADDR_LO_dst_addr_31_0_shift 0
+#define SDMA_PKT_CONSTANT_FILL_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_CONSTANT_FILL_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_CONSTANT_FILL_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_CONSTANT_FILL_DST_ADDR_HI_dst_addr_63_32_offset 2
+#define SDMA_PKT_CONSTANT_FILL_DST_ADDR_HI_dst_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_CONSTANT_FILL_DST_ADDR_HI_dst_addr_63_32_shift 0
+#define SDMA_PKT_CONSTANT_FILL_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_CONSTANT_FILL_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_CONSTANT_FILL_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for DATA word*/
+/*define for src_data_31_0 field*/
+#define SDMA_PKT_CONSTANT_FILL_DATA_src_data_31_0_offset 3
+#define SDMA_PKT_CONSTANT_FILL_DATA_src_data_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_CONSTANT_FILL_DATA_src_data_31_0_shift 0
+#define SDMA_PKT_CONSTANT_FILL_DATA_SRC_DATA_31_0(x) (((x) & SDMA_PKT_CONSTANT_FILL_DATA_src_data_31_0_mask) << SDMA_PKT_CONSTANT_FILL_DATA_src_data_31_0_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_CONSTANT_FILL_COUNT_count_offset 4
+#define SDMA_PKT_CONSTANT_FILL_COUNT_count_mask 0x003FFFFF
+#define SDMA_PKT_CONSTANT_FILL_COUNT_count_shift 0
+#define SDMA_PKT_CONSTANT_FILL_COUNT_COUNT(x) (((x) & SDMA_PKT_CONSTANT_FILL_COUNT_count_mask) << SDMA_PKT_CONSTANT_FILL_COUNT_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_DATA_FILL_MULTI packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_op_offset 0
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_op_shift 0
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_OP(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_HEADER_op_mask) << SDMA_PKT_DATA_FILL_MULTI_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_sub_op_offset 0
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_sub_op_shift 8
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_SUB_OP(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_HEADER_sub_op_mask) << SDMA_PKT_DATA_FILL_MULTI_HEADER_sub_op_shift)
+
+/*define for memlog_clr field*/
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_memlog_clr_offset 0
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_memlog_clr_mask 0x00000001
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_memlog_clr_shift 31
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_MEMLOG_CLR(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_HEADER_memlog_clr_mask) << SDMA_PKT_DATA_FILL_MULTI_HEADER_memlog_clr_shift)
+
+/*define for BYTE_STRIDE word*/
+/*define for byte_stride field*/
+#define SDMA_PKT_DATA_FILL_MULTI_BYTE_STRIDE_byte_stride_offset 1
+#define SDMA_PKT_DATA_FILL_MULTI_BYTE_STRIDE_byte_stride_mask 0xFFFFFFFF
+#define SDMA_PKT_DATA_FILL_MULTI_BYTE_STRIDE_byte_stride_shift 0
+#define SDMA_PKT_DATA_FILL_MULTI_BYTE_STRIDE_BYTE_STRIDE(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_BYTE_STRIDE_byte_stride_mask) << SDMA_PKT_DATA_FILL_MULTI_BYTE_STRIDE_byte_stride_shift)
+
+/*define for DMA_COUNT word*/
+/*define for dma_count field*/
+#define SDMA_PKT_DATA_FILL_MULTI_DMA_COUNT_dma_count_offset 2
+#define SDMA_PKT_DATA_FILL_MULTI_DMA_COUNT_dma_count_mask 0xFFFFFFFF
+#define SDMA_PKT_DATA_FILL_MULTI_DMA_COUNT_dma_count_shift 0
+#define SDMA_PKT_DATA_FILL_MULTI_DMA_COUNT_DMA_COUNT(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_DMA_COUNT_dma_count_mask) << SDMA_PKT_DATA_FILL_MULTI_DMA_COUNT_dma_count_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_LO_dst_addr_31_0_offset 3
+#define SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_LO_dst_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_LO_dst_addr_31_0_shift 0
+#define SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_HI_dst_addr_63_32_offset 4
+#define SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_HI_dst_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_HI_dst_addr_63_32_shift 0
+#define SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for BYTE_COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_DATA_FILL_MULTI_BYTE_COUNT_count_offset 5
+#define SDMA_PKT_DATA_FILL_MULTI_BYTE_COUNT_count_mask 0x03FFFFFF
+#define SDMA_PKT_DATA_FILL_MULTI_BYTE_COUNT_count_shift 0
+#define SDMA_PKT_DATA_FILL_MULTI_BYTE_COUNT_COUNT(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_BYTE_COUNT_count_mask) << SDMA_PKT_DATA_FILL_MULTI_BYTE_COUNT_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_POLL_REGMEM packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_POLL_REGMEM_HEADER_op_offset 0
+#define SDMA_PKT_POLL_REGMEM_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_POLL_REGMEM_HEADER_op_shift 0
+#define SDMA_PKT_POLL_REGMEM_HEADER_OP(x) (((x) & SDMA_PKT_POLL_REGMEM_HEADER_op_mask) << SDMA_PKT_POLL_REGMEM_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_POLL_REGMEM_HEADER_sub_op_offset 0
+#define SDMA_PKT_POLL_REGMEM_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_POLL_REGMEM_HEADER_sub_op_shift 8
+#define SDMA_PKT_POLL_REGMEM_HEADER_SUB_OP(x) (((x) & SDMA_PKT_POLL_REGMEM_HEADER_sub_op_mask) << SDMA_PKT_POLL_REGMEM_HEADER_sub_op_shift)
+
+/*define for hdp_flush field*/
+#define SDMA_PKT_POLL_REGMEM_HEADER_hdp_flush_offset 0
+#define SDMA_PKT_POLL_REGMEM_HEADER_hdp_flush_mask 0x00000001
+#define SDMA_PKT_POLL_REGMEM_HEADER_hdp_flush_shift 26
+#define SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(x) (((x) & SDMA_PKT_POLL_REGMEM_HEADER_hdp_flush_mask) << SDMA_PKT_POLL_REGMEM_HEADER_hdp_flush_shift)
+
+/*define for func field*/
+#define SDMA_PKT_POLL_REGMEM_HEADER_func_offset 0
+#define SDMA_PKT_POLL_REGMEM_HEADER_func_mask 0x00000007
+#define SDMA_PKT_POLL_REGMEM_HEADER_func_shift 28
+#define SDMA_PKT_POLL_REGMEM_HEADER_FUNC(x) (((x) & SDMA_PKT_POLL_REGMEM_HEADER_func_mask) << SDMA_PKT_POLL_REGMEM_HEADER_func_shift)
+
+/*define for mem_poll field*/
+#define SDMA_PKT_POLL_REGMEM_HEADER_mem_poll_offset 0
+#define SDMA_PKT_POLL_REGMEM_HEADER_mem_poll_mask 0x00000001
+#define SDMA_PKT_POLL_REGMEM_HEADER_mem_poll_shift 31
+#define SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(x) (((x) & SDMA_PKT_POLL_REGMEM_HEADER_mem_poll_mask) << SDMA_PKT_POLL_REGMEM_HEADER_mem_poll_shift)
+
+/*define for ADDR_LO word*/
+/*define for addr_31_0 field*/
+#define SDMA_PKT_POLL_REGMEM_ADDR_LO_addr_31_0_offset 1
+#define SDMA_PKT_POLL_REGMEM_ADDR_LO_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_REGMEM_ADDR_LO_addr_31_0_shift 0
+#define SDMA_PKT_POLL_REGMEM_ADDR_LO_ADDR_31_0(x) (((x) & SDMA_PKT_POLL_REGMEM_ADDR_LO_addr_31_0_mask) << SDMA_PKT_POLL_REGMEM_ADDR_LO_addr_31_0_shift)
+
+/*define for ADDR_HI word*/
+/*define for addr_63_32 field*/
+#define SDMA_PKT_POLL_REGMEM_ADDR_HI_addr_63_32_offset 2
+#define SDMA_PKT_POLL_REGMEM_ADDR_HI_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_REGMEM_ADDR_HI_addr_63_32_shift 0
+#define SDMA_PKT_POLL_REGMEM_ADDR_HI_ADDR_63_32(x) (((x) & SDMA_PKT_POLL_REGMEM_ADDR_HI_addr_63_32_mask) << SDMA_PKT_POLL_REGMEM_ADDR_HI_addr_63_32_shift)
+
+/*define for VALUE word*/
+/*define for value field*/
+#define SDMA_PKT_POLL_REGMEM_VALUE_value_offset 3
+#define SDMA_PKT_POLL_REGMEM_VALUE_value_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_REGMEM_VALUE_value_shift 0
+#define SDMA_PKT_POLL_REGMEM_VALUE_VALUE(x) (((x) & SDMA_PKT_POLL_REGMEM_VALUE_value_mask) << SDMA_PKT_POLL_REGMEM_VALUE_value_shift)
+
+/*define for MASK word*/
+/*define for mask field*/
+#define SDMA_PKT_POLL_REGMEM_MASK_mask_offset 4
+#define SDMA_PKT_POLL_REGMEM_MASK_mask_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_REGMEM_MASK_mask_shift 0
+#define SDMA_PKT_POLL_REGMEM_MASK_MASK(x) (((x) & SDMA_PKT_POLL_REGMEM_MASK_mask_mask) << SDMA_PKT_POLL_REGMEM_MASK_mask_shift)
+
+/*define for DW5 word*/
+/*define for interval field*/
+#define SDMA_PKT_POLL_REGMEM_DW5_interval_offset 5
+#define SDMA_PKT_POLL_REGMEM_DW5_interval_mask 0x0000FFFF
+#define SDMA_PKT_POLL_REGMEM_DW5_interval_shift 0
+#define SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(x) (((x) & SDMA_PKT_POLL_REGMEM_DW5_interval_mask) << SDMA_PKT_POLL_REGMEM_DW5_interval_shift)
+
+/*define for retry_count field*/
+#define SDMA_PKT_POLL_REGMEM_DW5_retry_count_offset 5
+#define SDMA_PKT_POLL_REGMEM_DW5_retry_count_mask 0x00000FFF
+#define SDMA_PKT_POLL_REGMEM_DW5_retry_count_shift 16
+#define SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(x) (((x) & SDMA_PKT_POLL_REGMEM_DW5_retry_count_mask) << SDMA_PKT_POLL_REGMEM_DW5_retry_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_POLL_REG_WRITE_MEM packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_op_offset 0
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_op_shift 0
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_OP(x) (((x) & SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_op_mask) << SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_sub_op_offset 0
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_sub_op_shift 8
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_SUB_OP(x) (((x) & SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_sub_op_mask) << SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_sub_op_shift)
+
+/*define for SRC_ADDR word*/
+/*define for addr_31_2 field*/
+#define SDMA_PKT_POLL_REG_WRITE_MEM_SRC_ADDR_addr_31_2_offset 1
+#define SDMA_PKT_POLL_REG_WRITE_MEM_SRC_ADDR_addr_31_2_mask 0x3FFFFFFF
+#define SDMA_PKT_POLL_REG_WRITE_MEM_SRC_ADDR_addr_31_2_shift 2
+#define SDMA_PKT_POLL_REG_WRITE_MEM_SRC_ADDR_ADDR_31_2(x) (((x) & SDMA_PKT_POLL_REG_WRITE_MEM_SRC_ADDR_addr_31_2_mask) << SDMA_PKT_POLL_REG_WRITE_MEM_SRC_ADDR_addr_31_2_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for addr_31_0 field*/
+#define SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_LO_addr_31_0_offset 2
+#define SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_LO_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_LO_addr_31_0_shift 0
+#define SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_LO_ADDR_31_0(x) (((x) & SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_LO_addr_31_0_mask) << SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_LO_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for addr_63_32 field*/
+#define SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_HI_addr_63_32_offset 3
+#define SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_HI_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_HI_addr_63_32_shift 0
+#define SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_HI_ADDR_63_32(x) (((x) & SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_HI_addr_63_32_mask) << SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_HI_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_POLL_DBIT_WRITE_MEM packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_op_offset 0
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_op_shift 0
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_OP(x) (((x) & SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_op_mask) << SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_sub_op_offset 0
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_sub_op_shift 8
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_SUB_OP(x) (((x) & SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_sub_op_mask) << SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_sub_op_shift)
+
+/*define for ea field*/
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_ea_offset 0
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_ea_mask 0x00000003
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_ea_shift 16
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_EA(x) (((x) & SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_ea_mask) << SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_ea_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for addr_31_0 field*/
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_LO_addr_31_0_offset 1
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_LO_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_LO_addr_31_0_shift 0
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_LO_ADDR_31_0(x) (((x) & SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_LO_addr_31_0_mask) << SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_LO_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for addr_63_32 field*/
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_HI_addr_63_32_offset 2
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_HI_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_HI_addr_63_32_shift 0
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_HI_ADDR_63_32(x) (((x) & SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_HI_addr_63_32_mask) << SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_HI_addr_63_32_shift)
+
+/*define for START_PAGE word*/
+/*define for addr_31_4 field*/
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_START_PAGE_addr_31_4_offset 3
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_START_PAGE_addr_31_4_mask 0x0FFFFFFF
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_START_PAGE_addr_31_4_shift 4
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_START_PAGE_ADDR_31_4(x) (((x) & SDMA_PKT_POLL_DBIT_WRITE_MEM_START_PAGE_addr_31_4_mask) << SDMA_PKT_POLL_DBIT_WRITE_MEM_START_PAGE_addr_31_4_shift)
+
+/*define for PAGE_NUM word*/
+/*define for page_num_31_0 field*/
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_PAGE_NUM_page_num_31_0_offset 4
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_PAGE_NUM_page_num_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_PAGE_NUM_page_num_31_0_shift 0
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_PAGE_NUM_PAGE_NUM_31_0(x) (((x) & SDMA_PKT_POLL_DBIT_WRITE_MEM_PAGE_NUM_page_num_31_0_mask) << SDMA_PKT_POLL_DBIT_WRITE_MEM_PAGE_NUM_page_num_31_0_shift)
+
+
+/*
+** Definitions for SDMA_PKT_POLL_MEM_VERIFY packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_op_offset 0
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_op_shift 0
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_OP(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_HEADER_op_mask) << SDMA_PKT_POLL_MEM_VERIFY_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_sub_op_offset 0
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_sub_op_shift 8
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_SUB_OP(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_HEADER_sub_op_mask) << SDMA_PKT_POLL_MEM_VERIFY_HEADER_sub_op_shift)
+
+/*define for mode field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_mode_offset 0
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_mode_mask 0x00000001
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_mode_shift 31
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_MODE(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_HEADER_mode_mask) << SDMA_PKT_POLL_MEM_VERIFY_HEADER_mode_shift)
+
+/*define for PATTERN word*/
+/*define for pattern field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_PATTERN_pattern_offset 1
+#define SDMA_PKT_POLL_MEM_VERIFY_PATTERN_pattern_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_PATTERN_pattern_shift 0
+#define SDMA_PKT_POLL_MEM_VERIFY_PATTERN_PATTERN(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_PATTERN_pattern_mask) << SDMA_PKT_POLL_MEM_VERIFY_PATTERN_pattern_shift)
+
+/*define for CMP0_ADDR_START_LO word*/
+/*define for cmp0_start_31_0 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_LO_cmp0_start_31_0_offset 2
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_LO_cmp0_start_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_LO_cmp0_start_31_0_shift 0
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_LO_CMP0_START_31_0(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_LO_cmp0_start_31_0_mask) << SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_LO_cmp0_start_31_0_shift)
+
+/*define for CMP0_ADDR_START_HI word*/
+/*define for cmp0_start_63_32 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_HI_cmp0_start_63_32_offset 3
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_HI_cmp0_start_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_HI_cmp0_start_63_32_shift 0
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_HI_CMP0_START_63_32(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_HI_cmp0_start_63_32_mask) << SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_HI_cmp0_start_63_32_shift)
+
+/*define for CMP0_ADDR_END_LO word*/
+/*define for cmp1_end_31_0 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_LO_cmp1_end_31_0_offset 4
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_LO_cmp1_end_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_LO_cmp1_end_31_0_shift 0
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_LO_CMP1_END_31_0(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_LO_cmp1_end_31_0_mask) << SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_LO_cmp1_end_31_0_shift)
+
+/*define for CMP0_ADDR_END_HI word*/
+/*define for cmp1_end_63_32 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_HI_cmp1_end_63_32_offset 5
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_HI_cmp1_end_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_HI_cmp1_end_63_32_shift 0
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_HI_CMP1_END_63_32(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_HI_cmp1_end_63_32_mask) << SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_HI_cmp1_end_63_32_shift)
+
+/*define for CMP1_ADDR_START_LO word*/
+/*define for cmp1_start_31_0 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_LO_cmp1_start_31_0_offset 6
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_LO_cmp1_start_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_LO_cmp1_start_31_0_shift 0
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_LO_CMP1_START_31_0(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_LO_cmp1_start_31_0_mask) << SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_LO_cmp1_start_31_0_shift)
+
+/*define for CMP1_ADDR_START_HI word*/
+/*define for cmp1_start_63_32 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_HI_cmp1_start_63_32_offset 7
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_HI_cmp1_start_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_HI_cmp1_start_63_32_shift 0
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_HI_CMP1_START_63_32(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_HI_cmp1_start_63_32_mask) << SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_HI_cmp1_start_63_32_shift)
+
+/*define for CMP1_ADDR_END_LO word*/
+/*define for cmp1_end_31_0 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_LO_cmp1_end_31_0_offset 8
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_LO_cmp1_end_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_LO_cmp1_end_31_0_shift 0
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_LO_CMP1_END_31_0(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_LO_cmp1_end_31_0_mask) << SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_LO_cmp1_end_31_0_shift)
+
+/*define for CMP1_ADDR_END_HI word*/
+/*define for cmp1_end_63_32 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_HI_cmp1_end_63_32_offset 9
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_HI_cmp1_end_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_HI_cmp1_end_63_32_shift 0
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_HI_CMP1_END_63_32(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_HI_cmp1_end_63_32_mask) << SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_HI_cmp1_end_63_32_shift)
+
+/*define for REC_ADDR_LO word*/
+/*define for rec_31_0 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_LO_rec_31_0_offset 10
+#define SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_LO_rec_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_LO_rec_31_0_shift 0
+#define SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_LO_REC_31_0(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_LO_rec_31_0_mask) << SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_LO_rec_31_0_shift)
+
+/*define for REC_ADDR_HI word*/
+/*define for rec_63_32 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_HI_rec_63_32_offset 11
+#define SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_HI_rec_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_HI_rec_63_32_shift 0
+#define SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_HI_REC_63_32(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_HI_rec_63_32_mask) << SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_HI_rec_63_32_shift)
+
+/*define for RESERVED word*/
+/*define for reserved field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_RESERVED_reserved_offset 12
+#define SDMA_PKT_POLL_MEM_VERIFY_RESERVED_reserved_mask 0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_RESERVED_reserved_shift 0
+#define SDMA_PKT_POLL_MEM_VERIFY_RESERVED_RESERVED(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_RESERVED_reserved_mask) << SDMA_PKT_POLL_MEM_VERIFY_RESERVED_reserved_shift)
+
+
+/*
+** Definitions for SDMA_PKT_ATOMIC packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_ATOMIC_HEADER_op_offset 0
+#define SDMA_PKT_ATOMIC_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_ATOMIC_HEADER_op_shift 0
+#define SDMA_PKT_ATOMIC_HEADER_OP(x) (((x) & SDMA_PKT_ATOMIC_HEADER_op_mask) << SDMA_PKT_ATOMIC_HEADER_op_shift)
+
+/*define for loop field*/
+#define SDMA_PKT_ATOMIC_HEADER_loop_offset 0
+#define SDMA_PKT_ATOMIC_HEADER_loop_mask 0x00000001
+#define SDMA_PKT_ATOMIC_HEADER_loop_shift 16
+#define SDMA_PKT_ATOMIC_HEADER_LOOP(x) (((x) & SDMA_PKT_ATOMIC_HEADER_loop_mask) << SDMA_PKT_ATOMIC_HEADER_loop_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_ATOMIC_HEADER_tmz_offset 0
+#define SDMA_PKT_ATOMIC_HEADER_tmz_mask 0x00000001
+#define SDMA_PKT_ATOMIC_HEADER_tmz_shift 18
+#define SDMA_PKT_ATOMIC_HEADER_TMZ(x) (((x) & SDMA_PKT_ATOMIC_HEADER_tmz_mask) << SDMA_PKT_ATOMIC_HEADER_tmz_shift)
+
+/*define for atomic_op field*/
+#define SDMA_PKT_ATOMIC_HEADER_atomic_op_offset 0
+#define SDMA_PKT_ATOMIC_HEADER_atomic_op_mask 0x0000007F
+#define SDMA_PKT_ATOMIC_HEADER_atomic_op_shift 25
+#define SDMA_PKT_ATOMIC_HEADER_ATOMIC_OP(x) (((x) & SDMA_PKT_ATOMIC_HEADER_atomic_op_mask) << SDMA_PKT_ATOMIC_HEADER_atomic_op_shift)
+
+/*define for ADDR_LO word*/
+/*define for addr_31_0 field*/
+#define SDMA_PKT_ATOMIC_ADDR_LO_addr_31_0_offset 1
+#define SDMA_PKT_ATOMIC_ADDR_LO_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_ATOMIC_ADDR_LO_addr_31_0_shift 0
+#define SDMA_PKT_ATOMIC_ADDR_LO_ADDR_31_0(x) (((x) & SDMA_PKT_ATOMIC_ADDR_LO_addr_31_0_mask) << SDMA_PKT_ATOMIC_ADDR_LO_addr_31_0_shift)
+
+/*define for ADDR_HI word*/
+/*define for addr_63_32 field*/
+#define SDMA_PKT_ATOMIC_ADDR_HI_addr_63_32_offset 2
+#define SDMA_PKT_ATOMIC_ADDR_HI_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_ATOMIC_ADDR_HI_addr_63_32_shift 0
+#define SDMA_PKT_ATOMIC_ADDR_HI_ADDR_63_32(x) (((x) & SDMA_PKT_ATOMIC_ADDR_HI_addr_63_32_mask) << SDMA_PKT_ATOMIC_ADDR_HI_addr_63_32_shift)
+
+/*define for SRC_DATA_LO word*/
+/*define for src_data_31_0 field*/
+#define SDMA_PKT_ATOMIC_SRC_DATA_LO_src_data_31_0_offset 3
+#define SDMA_PKT_ATOMIC_SRC_DATA_LO_src_data_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_ATOMIC_SRC_DATA_LO_src_data_31_0_shift 0
+#define SDMA_PKT_ATOMIC_SRC_DATA_LO_SRC_DATA_31_0(x) (((x) & SDMA_PKT_ATOMIC_SRC_DATA_LO_src_data_31_0_mask) << SDMA_PKT_ATOMIC_SRC_DATA_LO_src_data_31_0_shift)
+
+/*define for SRC_DATA_HI word*/
+/*define for src_data_63_32 field*/
+#define SDMA_PKT_ATOMIC_SRC_DATA_HI_src_data_63_32_offset 4
+#define SDMA_PKT_ATOMIC_SRC_DATA_HI_src_data_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_ATOMIC_SRC_DATA_HI_src_data_63_32_shift 0
+#define SDMA_PKT_ATOMIC_SRC_DATA_HI_SRC_DATA_63_32(x) (((x) & SDMA_PKT_ATOMIC_SRC_DATA_HI_src_data_63_32_mask) << SDMA_PKT_ATOMIC_SRC_DATA_HI_src_data_63_32_shift)
+
+/*define for CMP_DATA_LO word*/
+/*define for cmp_data_31_0 field*/
+#define SDMA_PKT_ATOMIC_CMP_DATA_LO_cmp_data_31_0_offset 5
+#define SDMA_PKT_ATOMIC_CMP_DATA_LO_cmp_data_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_ATOMIC_CMP_DATA_LO_cmp_data_31_0_shift 0
+#define SDMA_PKT_ATOMIC_CMP_DATA_LO_CMP_DATA_31_0(x) (((x) & SDMA_PKT_ATOMIC_CMP_DATA_LO_cmp_data_31_0_mask) << SDMA_PKT_ATOMIC_CMP_DATA_LO_cmp_data_31_0_shift)
+
+/*define for CMP_DATA_HI word*/
+/*define for cmp_data_63_32 field*/
+#define SDMA_PKT_ATOMIC_CMP_DATA_HI_cmp_data_63_32_offset 6
+#define SDMA_PKT_ATOMIC_CMP_DATA_HI_cmp_data_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_ATOMIC_CMP_DATA_HI_cmp_data_63_32_shift 0
+#define SDMA_PKT_ATOMIC_CMP_DATA_HI_CMP_DATA_63_32(x) (((x) & SDMA_PKT_ATOMIC_CMP_DATA_HI_cmp_data_63_32_mask) << SDMA_PKT_ATOMIC_CMP_DATA_HI_cmp_data_63_32_shift)
+
+/*define for LOOP_INTERVAL word*/
+/*define for loop_interval field*/
+#define SDMA_PKT_ATOMIC_LOOP_INTERVAL_loop_interval_offset 7
+#define SDMA_PKT_ATOMIC_LOOP_INTERVAL_loop_interval_mask 0x00001FFF
+#define SDMA_PKT_ATOMIC_LOOP_INTERVAL_loop_interval_shift 0
+#define SDMA_PKT_ATOMIC_LOOP_INTERVAL_LOOP_INTERVAL(x) (((x) & SDMA_PKT_ATOMIC_LOOP_INTERVAL_loop_interval_mask) << SDMA_PKT_ATOMIC_LOOP_INTERVAL_loop_interval_shift)
+
+
+/*
+** Definitions for SDMA_PKT_TIMESTAMP_SET packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_TIMESTAMP_SET_HEADER_op_offset 0
+#define SDMA_PKT_TIMESTAMP_SET_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_TIMESTAMP_SET_HEADER_op_shift 0
+#define SDMA_PKT_TIMESTAMP_SET_HEADER_OP(x) (((x) & SDMA_PKT_TIMESTAMP_SET_HEADER_op_mask) << SDMA_PKT_TIMESTAMP_SET_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_TIMESTAMP_SET_HEADER_sub_op_offset 0
+#define SDMA_PKT_TIMESTAMP_SET_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_TIMESTAMP_SET_HEADER_sub_op_shift 8
+#define SDMA_PKT_TIMESTAMP_SET_HEADER_SUB_OP(x) (((x) & SDMA_PKT_TIMESTAMP_SET_HEADER_sub_op_mask) << SDMA_PKT_TIMESTAMP_SET_HEADER_sub_op_shift)
+
+/*define for INIT_DATA_LO word*/
+/*define for init_data_31_0 field*/
+#define SDMA_PKT_TIMESTAMP_SET_INIT_DATA_LO_init_data_31_0_offset 1
+#define SDMA_PKT_TIMESTAMP_SET_INIT_DATA_LO_init_data_31_0_mask 0xFFFFFFFF
+#define SDMA_PKT_TIMESTAMP_SET_INIT_DATA_LO_init_data_31_0_shift 0
+#define SDMA_PKT_TIMESTAMP_SET_INIT_DATA_LO_INIT_DATA_31_0(x) (((x) & SDMA_PKT_TIMESTAMP_SET_INIT_DATA_LO_init_data_31_0_mask) << SDMA_PKT_TIMESTAMP_SET_INIT_DATA_LO_init_data_31_0_shift)
+
+/*define for INIT_DATA_HI word*/
+/*define for init_data_63_32 field*/
+#define SDMA_PKT_TIMESTAMP_SET_INIT_DATA_HI_init_data_63_32_offset 2
+#define SDMA_PKT_TIMESTAMP_SET_INIT_DATA_HI_init_data_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_TIMESTAMP_SET_INIT_DATA_HI_init_data_63_32_shift 0
+#define SDMA_PKT_TIMESTAMP_SET_INIT_DATA_HI_INIT_DATA_63_32(x) (((x) & SDMA_PKT_TIMESTAMP_SET_INIT_DATA_HI_init_data_63_32_mask) << SDMA_PKT_TIMESTAMP_SET_INIT_DATA_HI_init_data_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_TIMESTAMP_GET packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_op_offset 0
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_op_shift 0
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_OP(x) (((x) & SDMA_PKT_TIMESTAMP_GET_HEADER_op_mask) << SDMA_PKT_TIMESTAMP_GET_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_sub_op_offset 0
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_sub_op_shift 8
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_SUB_OP(x) (((x) & SDMA_PKT_TIMESTAMP_GET_HEADER_sub_op_mask) << SDMA_PKT_TIMESTAMP_GET_HEADER_sub_op_shift)
+
+/*define for WRITE_ADDR_LO word*/
+/*define for write_addr_31_3 field*/
+#define SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_LO_write_addr_31_3_offset 1
+#define SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_LO_write_addr_31_3_mask 0x1FFFFFFF
+#define SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_LO_write_addr_31_3_shift 3
+#define SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_LO_WRITE_ADDR_31_3(x) (((x) & SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_LO_write_addr_31_3_mask) << SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_LO_write_addr_31_3_shift)
+
+/*define for WRITE_ADDR_HI word*/
+/*define for write_addr_63_32 field*/
+#define SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_HI_write_addr_63_32_offset 2
+#define SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_HI_write_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_HI_write_addr_63_32_shift 0
+#define SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_HI_WRITE_ADDR_63_32(x) (((x) & SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_HI_write_addr_63_32_mask) << SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_HI_write_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_TIMESTAMP_GET_GLOBAL packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_op_offset 0
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_op_shift 0
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_OP(x) (((x) & SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_op_mask) << SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_sub_op_offset 0
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_sub_op_shift 8
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_SUB_OP(x) (((x) & SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_sub_op_mask) << SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_sub_op_shift)
+
+/*define for WRITE_ADDR_LO word*/
+/*define for write_addr_31_3 field*/
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_LO_write_addr_31_3_offset 1
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_LO_write_addr_31_3_mask 0x1FFFFFFF
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_LO_write_addr_31_3_shift 3
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_LO_WRITE_ADDR_31_3(x) (((x) & SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_LO_write_addr_31_3_mask) << SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_LO_write_addr_31_3_shift)
+
+/*define for WRITE_ADDR_HI word*/
+/*define for write_addr_63_32 field*/
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_HI_write_addr_63_32_offset 2
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_HI_write_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_HI_write_addr_63_32_shift 0
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_HI_WRITE_ADDR_63_32(x) (((x) & SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_HI_write_addr_63_32_mask) << SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_HI_write_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_TRAP packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_TRAP_HEADER_op_offset 0
+#define SDMA_PKT_TRAP_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_TRAP_HEADER_op_shift 0
+#define SDMA_PKT_TRAP_HEADER_OP(x) (((x) & SDMA_PKT_TRAP_HEADER_op_mask) << SDMA_PKT_TRAP_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_TRAP_HEADER_sub_op_offset 0
+#define SDMA_PKT_TRAP_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_TRAP_HEADER_sub_op_shift 8
+#define SDMA_PKT_TRAP_HEADER_SUB_OP(x) (((x) & SDMA_PKT_TRAP_HEADER_sub_op_mask) << SDMA_PKT_TRAP_HEADER_sub_op_shift)
+
+/*define for INT_CONTEXT word*/
+/*define for int_context field*/
+#define SDMA_PKT_TRAP_INT_CONTEXT_int_context_offset 1
+#define SDMA_PKT_TRAP_INT_CONTEXT_int_context_mask 0x0FFFFFFF
+#define SDMA_PKT_TRAP_INT_CONTEXT_int_context_shift 0
+#define SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(x) (((x) & SDMA_PKT_TRAP_INT_CONTEXT_int_context_mask) << SDMA_PKT_TRAP_INT_CONTEXT_int_context_shift)
+
+
+/*
+** Definitions for SDMA_PKT_DUMMY_TRAP packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_DUMMY_TRAP_HEADER_op_offset 0
+#define SDMA_PKT_DUMMY_TRAP_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_DUMMY_TRAP_HEADER_op_shift 0
+#define SDMA_PKT_DUMMY_TRAP_HEADER_OP(x) (((x) & SDMA_PKT_DUMMY_TRAP_HEADER_op_mask) << SDMA_PKT_DUMMY_TRAP_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_DUMMY_TRAP_HEADER_sub_op_offset 0
+#define SDMA_PKT_DUMMY_TRAP_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_DUMMY_TRAP_HEADER_sub_op_shift 8
+#define SDMA_PKT_DUMMY_TRAP_HEADER_SUB_OP(x) (((x) & SDMA_PKT_DUMMY_TRAP_HEADER_sub_op_mask) << SDMA_PKT_DUMMY_TRAP_HEADER_sub_op_shift)
+
+/*define for INT_CONTEXT word*/
+/*define for int_context field*/
+#define SDMA_PKT_DUMMY_TRAP_INT_CONTEXT_int_context_offset 1
+#define SDMA_PKT_DUMMY_TRAP_INT_CONTEXT_int_context_mask 0x0FFFFFFF
+#define SDMA_PKT_DUMMY_TRAP_INT_CONTEXT_int_context_shift 0
+#define SDMA_PKT_DUMMY_TRAP_INT_CONTEXT_INT_CONTEXT(x) (((x) & SDMA_PKT_DUMMY_TRAP_INT_CONTEXT_int_context_mask) << SDMA_PKT_DUMMY_TRAP_INT_CONTEXT_int_context_shift)
+
+
+/*
+** Definitions for SDMA_PKT_GPUVM_INV packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_GPUVM_INV_HEADER_op_offset 0
+#define SDMA_PKT_GPUVM_INV_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_GPUVM_INV_HEADER_op_shift 0
+#define SDMA_PKT_GPUVM_INV_HEADER_OP(x) (((x) & SDMA_PKT_GPUVM_INV_HEADER_op_mask) << SDMA_PKT_GPUVM_INV_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_GPUVM_INV_HEADER_sub_op_offset 0
+#define SDMA_PKT_GPUVM_INV_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_GPUVM_INV_HEADER_sub_op_shift 8
+#define SDMA_PKT_GPUVM_INV_HEADER_SUB_OP(x) (((x) & SDMA_PKT_GPUVM_INV_HEADER_sub_op_mask) << SDMA_PKT_GPUVM_INV_HEADER_sub_op_shift)
+
+/*define for PAYLOAD1 word*/
+/*define for per_vmid_inv_req field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_per_vmid_inv_req_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_per_vmid_inv_req_mask 0x0000FFFF
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_per_vmid_inv_req_shift 0
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_PER_VMID_INV_REQ(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_per_vmid_inv_req_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_per_vmid_inv_req_shift)
+
+/*define for flush_type field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_flush_type_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_flush_type_mask 0x00000007
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_flush_type_shift 16
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_FLUSH_TYPE(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_flush_type_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_flush_type_shift)
+
+/*define for l2_ptes field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_ptes_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_ptes_mask 0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_ptes_shift 19
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_L2_PTES(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_ptes_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_ptes_shift)
+
+/*define for l2_pde0 field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde0_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde0_mask 0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde0_shift 20
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_L2_PDE0(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde0_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde0_shift)
+
+/*define for l2_pde1 field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde1_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde1_mask 0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde1_shift 21
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_L2_PDE1(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde1_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde1_shift)
+
+/*define for l2_pde2 field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde2_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde2_mask 0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde2_shift 22
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_L2_PDE2(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde2_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde2_shift)
+
+/*define for l1_ptes field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l1_ptes_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l1_ptes_mask 0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l1_ptes_shift 23
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_L1_PTES(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_l1_ptes_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_l1_ptes_shift)
+
+/*define for clr_protection_fault_status_addr field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_clr_protection_fault_status_addr_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_clr_protection_fault_status_addr_mask 0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_clr_protection_fault_status_addr_shift 24
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_CLR_PROTECTION_FAULT_STATUS_ADDR(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_clr_protection_fault_status_addr_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_clr_protection_fault_status_addr_shift)
+
+/*define for log_request field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_log_request_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_log_request_mask 0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_log_request_shift 25
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_LOG_REQUEST(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_log_request_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_log_request_shift)
+
+/*define for four_kilobytes field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_four_kilobytes_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_four_kilobytes_mask 0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_four_kilobytes_shift 26
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_FOUR_KILOBYTES(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_four_kilobytes_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_four_kilobytes_shift)
+
+/*define for PAYLOAD2 word*/
+/*define for s field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD2_s_offset 2
+#define SDMA_PKT_GPUVM_INV_PAYLOAD2_s_mask 0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD2_s_shift 0
+#define SDMA_PKT_GPUVM_INV_PAYLOAD2_S(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD2_s_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD2_s_shift)
+
+/*define for page_va_42_12 field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD2_page_va_42_12_offset 2
+#define SDMA_PKT_GPUVM_INV_PAYLOAD2_page_va_42_12_mask 0x7FFFFFFF
+#define SDMA_PKT_GPUVM_INV_PAYLOAD2_page_va_42_12_shift 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD2_PAGE_VA_42_12(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD2_page_va_42_12_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD2_page_va_42_12_shift)
+
+/*define for PAYLOAD3 word*/
+/*define for page_va_47_43 field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD3_page_va_47_43_offset 3
+#define SDMA_PKT_GPUVM_INV_PAYLOAD3_page_va_47_43_mask 0x0000003F
+#define SDMA_PKT_GPUVM_INV_PAYLOAD3_page_va_47_43_shift 0
+#define SDMA_PKT_GPUVM_INV_PAYLOAD3_PAGE_VA_47_43(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD3_page_va_47_43_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD3_page_va_47_43_shift)
+
+
+/*
+** Definitions for SDMA_PKT_GCR_REQ packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_GCR_REQ_HEADER_op_offset 0
+#define SDMA_PKT_GCR_REQ_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_GCR_REQ_HEADER_op_shift 0
+#define SDMA_PKT_GCR_REQ_HEADER_OP(x) (((x) & SDMA_PKT_GCR_REQ_HEADER_op_mask) << SDMA_PKT_GCR_REQ_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_GCR_REQ_HEADER_sub_op_offset 0
+#define SDMA_PKT_GCR_REQ_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_GCR_REQ_HEADER_sub_op_shift 8
+#define SDMA_PKT_GCR_REQ_HEADER_SUB_OP(x) (((x) & SDMA_PKT_GCR_REQ_HEADER_sub_op_mask) << SDMA_PKT_GCR_REQ_HEADER_sub_op_shift)
+
+/*define for PAYLOAD1 word*/
+/*define for base_va_31_7 field*/
+#define SDMA_PKT_GCR_REQ_PAYLOAD1_base_va_31_7_offset 1
+#define SDMA_PKT_GCR_REQ_PAYLOAD1_base_va_31_7_mask 0x01FFFFFF
+#define SDMA_PKT_GCR_REQ_PAYLOAD1_base_va_31_7_shift 7
+#define SDMA_PKT_GCR_REQ_PAYLOAD1_BASE_VA_31_7(x) (((x) & SDMA_PKT_GCR_REQ_PAYLOAD1_base_va_31_7_mask) << SDMA_PKT_GCR_REQ_PAYLOAD1_base_va_31_7_shift)
+
+/*define for PAYLOAD2 word*/
+/*define for base_va_47_32 field*/
+#define SDMA_PKT_GCR_REQ_PAYLOAD2_base_va_47_32_offset 2
+#define SDMA_PKT_GCR_REQ_PAYLOAD2_base_va_47_32_mask 0x0000FFFF
+#define SDMA_PKT_GCR_REQ_PAYLOAD2_base_va_47_32_shift 0
+#define SDMA_PKT_GCR_REQ_PAYLOAD2_BASE_VA_47_32(x) (((x) & SDMA_PKT_GCR_REQ_PAYLOAD2_base_va_47_32_mask) << SDMA_PKT_GCR_REQ_PAYLOAD2_base_va_47_32_shift)
+
+/*define for gcr_control_15_0 field*/
+#define SDMA_PKT_GCR_REQ_PAYLOAD2_gcr_control_15_0_offset 2
+#define SDMA_PKT_GCR_REQ_PAYLOAD2_gcr_control_15_0_mask 0x0000FFFF
+#define SDMA_PKT_GCR_REQ_PAYLOAD2_gcr_control_15_0_shift 16
+#define SDMA_PKT_GCR_REQ_PAYLOAD2_GCR_CONTROL_15_0(x) (((x) & SDMA_PKT_GCR_REQ_PAYLOAD2_gcr_control_15_0_mask) << SDMA_PKT_GCR_REQ_PAYLOAD2_gcr_control_15_0_shift)
+
+/*define for PAYLOAD3 word*/
+/*define for gcr_control_18_16 field*/
+#define SDMA_PKT_GCR_REQ_PAYLOAD3_gcr_control_18_16_offset 3
+#define SDMA_PKT_GCR_REQ_PAYLOAD3_gcr_control_18_16_mask 0x00000007
+#define SDMA_PKT_GCR_REQ_PAYLOAD3_gcr_control_18_16_shift 0
+#define SDMA_PKT_GCR_REQ_PAYLOAD3_GCR_CONTROL_18_16(x) (((x) & SDMA_PKT_GCR_REQ_PAYLOAD3_gcr_control_18_16_mask) << SDMA_PKT_GCR_REQ_PAYLOAD3_gcr_control_18_16_shift)
+
+/*define for limit_va_31_7 field*/
+#define SDMA_PKT_GCR_REQ_PAYLOAD3_limit_va_31_7_offset 3
+#define SDMA_PKT_GCR_REQ_PAYLOAD3_limit_va_31_7_mask 0x01FFFFFF
+#define SDMA_PKT_GCR_REQ_PAYLOAD3_limit_va_31_7_shift 7
+#define SDMA_PKT_GCR_REQ_PAYLOAD3_LIMIT_VA_31_7(x) (((x) & SDMA_PKT_GCR_REQ_PAYLOAD3_limit_va_31_7_mask) << SDMA_PKT_GCR_REQ_PAYLOAD3_limit_va_31_7_shift)
+
+/*define for PAYLOAD4 word*/
+/*define for limit_va_47_32 field*/
+#define SDMA_PKT_GCR_REQ_PAYLOAD4_limit_va_47_32_offset 4
+#define SDMA_PKT_GCR_REQ_PAYLOAD4_limit_va_47_32_mask 0x0000FFFF
+#define SDMA_PKT_GCR_REQ_PAYLOAD4_limit_va_47_32_shift 0
+#define SDMA_PKT_GCR_REQ_PAYLOAD4_LIMIT_VA_47_32(x) (((x) & SDMA_PKT_GCR_REQ_PAYLOAD4_limit_va_47_32_mask) << SDMA_PKT_GCR_REQ_PAYLOAD4_limit_va_47_32_shift)
+
+/*define for vmid field*/
+#define SDMA_PKT_GCR_REQ_PAYLOAD4_vmid_offset 4
+#define SDMA_PKT_GCR_REQ_PAYLOAD4_vmid_mask 0x0000000F
+#define SDMA_PKT_GCR_REQ_PAYLOAD4_vmid_shift 24
+#define SDMA_PKT_GCR_REQ_PAYLOAD4_VMID(x) (((x) & SDMA_PKT_GCR_REQ_PAYLOAD4_vmid_mask) << SDMA_PKT_GCR_REQ_PAYLOAD4_vmid_shift)
+
+
+/*
+** Definitions for SDMA_PKT_NOP packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_NOP_HEADER_op_offset 0
+#define SDMA_PKT_NOP_HEADER_op_mask 0x000000FF
+#define SDMA_PKT_NOP_HEADER_op_shift 0
+#define SDMA_PKT_NOP_HEADER_OP(x) (((x) & SDMA_PKT_NOP_HEADER_op_mask) << SDMA_PKT_NOP_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_NOP_HEADER_sub_op_offset 0
+#define SDMA_PKT_NOP_HEADER_sub_op_mask 0x000000FF
+#define SDMA_PKT_NOP_HEADER_sub_op_shift 8
+#define SDMA_PKT_NOP_HEADER_SUB_OP(x) (((x) & SDMA_PKT_NOP_HEADER_sub_op_mask) << SDMA_PKT_NOP_HEADER_sub_op_shift)
+
+/*define for count field*/
+#define SDMA_PKT_NOP_HEADER_count_offset 0
+#define SDMA_PKT_NOP_HEADER_count_mask 0x00003FFF
+#define SDMA_PKT_NOP_HEADER_count_shift 16
+#define SDMA_PKT_NOP_HEADER_COUNT(x) (((x) & SDMA_PKT_NOP_HEADER_count_mask) << SDMA_PKT_NOP_HEADER_count_shift)
+
+/*define for DATA0 word*/
+/*define for data0 field*/
+#define SDMA_PKT_NOP_DATA0_data0_offset 1
+#define SDMA_PKT_NOP_DATA0_data0_mask 0xFFFFFFFF
+#define SDMA_PKT_NOP_DATA0_data0_shift 0
+#define SDMA_PKT_NOP_DATA0_DATA0(x) (((x) & SDMA_PKT_NOP_DATA0_data0_mask) << SDMA_PKT_NOP_DATA0_data0_shift)
+
+
+/*
+** Definitions for SDMA_AQL_PKT_HEADER packet
+*/
+
+/*define for HEADER word*/
+/*define for format field*/
+#define SDMA_AQL_PKT_HEADER_HEADER_format_offset 0
+#define SDMA_AQL_PKT_HEADER_HEADER_format_mask 0x000000FF
+#define SDMA_AQL_PKT_HEADER_HEADER_format_shift 0
+#define SDMA_AQL_PKT_HEADER_HEADER_FORMAT(x) (((x) & SDMA_AQL_PKT_HEADER_HEADER_format_mask) << SDMA_AQL_PKT_HEADER_HEADER_format_shift)
+
+/*define for barrier field*/
+#define SDMA_AQL_PKT_HEADER_HEADER_barrier_offset 0
+#define SDMA_AQL_PKT_HEADER_HEADER_barrier_mask 0x00000001
+#define SDMA_AQL_PKT_HEADER_HEADER_barrier_shift 8
+#define SDMA_AQL_PKT_HEADER_HEADER_BARRIER(x) (((x) & SDMA_AQL_PKT_HEADER_HEADER_barrier_mask) << SDMA_AQL_PKT_HEADER_HEADER_barrier_shift)
+
+/*define for acquire_fence_scope field*/
+#define SDMA_AQL_PKT_HEADER_HEADER_acquire_fence_scope_offset 0
+#define SDMA_AQL_PKT_HEADER_HEADER_acquire_fence_scope_mask 0x00000003
+#define SDMA_AQL_PKT_HEADER_HEADER_acquire_fence_scope_shift 9
+#define SDMA_AQL_PKT_HEADER_HEADER_ACQUIRE_FENCE_SCOPE(x) (((x) & SDMA_AQL_PKT_HEADER_HEADER_acquire_fence_scope_mask) << SDMA_AQL_PKT_HEADER_HEADER_acquire_fence_scope_shift)
+
+/*define for release_fence_scope field*/
+#define SDMA_AQL_PKT_HEADER_HEADER_release_fence_scope_offset 0
+#define SDMA_AQL_PKT_HEADER_HEADER_release_fence_scope_mask 0x00000003
+#define SDMA_AQL_PKT_HEADER_HEADER_release_fence_scope_shift 11
+#define SDMA_AQL_PKT_HEADER_HEADER_RELEASE_FENCE_SCOPE(x) (((x) & SDMA_AQL_PKT_HEADER_HEADER_release_fence_scope_mask) << SDMA_AQL_PKT_HEADER_HEADER_release_fence_scope_shift)
+
+/*define for reserved field*/
+#define SDMA_AQL_PKT_HEADER_HEADER_reserved_offset 0
+#define SDMA_AQL_PKT_HEADER_HEADER_reserved_mask 0x00000007
+#define SDMA_AQL_PKT_HEADER_HEADER_reserved_shift 13
+#define SDMA_AQL_PKT_HEADER_HEADER_RESERVED(x) (((x) & SDMA_AQL_PKT_HEADER_HEADER_reserved_mask) << SDMA_AQL_PKT_HEADER_HEADER_reserved_shift)
+
+/*define for op field*/
+#define SDMA_AQL_PKT_HEADER_HEADER_op_offset 0
+#define SDMA_AQL_PKT_HEADER_HEADER_op_mask 0x0000000F
+#define SDMA_AQL_PKT_HEADER_HEADER_op_shift 16
+#define SDMA_AQL_PKT_HEADER_HEADER_OP(x) (((x) & SDMA_AQL_PKT_HEADER_HEADER_op_mask) << SDMA_AQL_PKT_HEADER_HEADER_op_shift)
+
+/*define for subop field*/
+#define SDMA_AQL_PKT_HEADER_HEADER_subop_offset 0
+#define SDMA_AQL_PKT_HEADER_HEADER_subop_mask 0x00000007
+#define SDMA_AQL_PKT_HEADER_HEADER_subop_shift 20
+#define SDMA_AQL_PKT_HEADER_HEADER_SUBOP(x) (((x) & SDMA_AQL_PKT_HEADER_HEADER_subop_mask) << SDMA_AQL_PKT_HEADER_HEADER_subop_shift)
+
+
+/*
+** Definitions for SDMA_AQL_PKT_COPY_LINEAR packet
+*/
+
+/*define for HEADER word*/
+/*define for format field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_format_offset 0
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_format_mask 0x000000FF
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_format_shift 0
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_FORMAT(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_HEADER_format_mask) << SDMA_AQL_PKT_COPY_LINEAR_HEADER_format_shift)
+
+/*define for barrier field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_barrier_offset 0
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_barrier_mask 0x00000001
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_barrier_shift 8
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_BARRIER(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_HEADER_barrier_mask) << SDMA_AQL_PKT_COPY_LINEAR_HEADER_barrier_shift)
+
+/*define for acquire_fence_scope field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_acquire_fence_scope_offset 0
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_acquire_fence_scope_mask 0x00000003
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_acquire_fence_scope_shift 9
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_ACQUIRE_FENCE_SCOPE(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_HEADER_acquire_fence_scope_mask) << SDMA_AQL_PKT_COPY_LINEAR_HEADER_acquire_fence_scope_shift)
+
+/*define for release_fence_scope field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_release_fence_scope_offset 0
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_release_fence_scope_mask 0x00000003
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_release_fence_scope_shift 11
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_RELEASE_FENCE_SCOPE(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_HEADER_release_fence_scope_mask) << SDMA_AQL_PKT_COPY_LINEAR_HEADER_release_fence_scope_shift)
+
+/*define for reserved field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_reserved_offset 0
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_reserved_mask 0x00000007
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_reserved_shift 13
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_RESERVED(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_HEADER_reserved_mask) << SDMA_AQL_PKT_COPY_LINEAR_HEADER_reserved_shift)
+
+/*define for op field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_op_offset 0
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_op_mask 0x0000000F
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_op_shift 16
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_OP(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_HEADER_op_mask) << SDMA_AQL_PKT_COPY_LINEAR_HEADER_op_shift)
+
+/*define for subop field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_subop_offset 0
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_subop_mask 0x00000007
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_subop_shift 20
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_SUBOP(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_HEADER_subop_mask) << SDMA_AQL_PKT_COPY_LINEAR_HEADER_subop_shift)
+
+/*define for RESERVED_DW1 word*/
+/*define for reserved_dw1 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW1_reserved_dw1_offset 1
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW1_reserved_dw1_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW1_reserved_dw1_shift 0
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW1_RESERVED_DW1(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW1_reserved_dw1_mask) << SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW1_reserved_dw1_shift)
+
+/*define for RETURN_ADDR_LO word*/
+/*define for return_addr_31_0 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_LO_return_addr_31_0_offset 2
+#define SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_LO_return_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_LO_return_addr_31_0_shift 0
+#define SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_LO_RETURN_ADDR_31_0(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_LO_return_addr_31_0_mask) << SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_LO_return_addr_31_0_shift)
+
+/*define for RETURN_ADDR_HI word*/
+/*define for return_addr_63_32 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_HI_return_addr_63_32_offset 3
+#define SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_HI_return_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_HI_return_addr_63_32_shift 0
+#define SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_HI_RETURN_ADDR_63_32(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_HI_return_addr_63_32_mask) << SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_HI_return_addr_63_32_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_COUNT_count_offset 4
+#define SDMA_AQL_PKT_COPY_LINEAR_COUNT_count_mask 0x003FFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_COUNT_count_shift 0
+#define SDMA_AQL_PKT_COPY_LINEAR_COUNT_COUNT(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_COUNT_count_mask) << SDMA_AQL_PKT_COPY_LINEAR_COUNT_count_shift)
+
+/*define for PARAMETER word*/
+/*define for dst_sw field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_dst_sw_offset 5
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_dst_sw_mask 0x00000003
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_dst_sw_shift 16
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_DST_SW(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_dst_sw_mask) << SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_dst_sw_shift)
+
+/*define for src_sw field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_src_sw_offset 5
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_src_sw_mask 0x00000003
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_src_sw_shift 24
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_SRC_SW(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_src_sw_mask) << SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_src_sw_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_offset 6
+#define SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_shift 0
+#define SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_offset 7
+#define SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_shift 0
+#define SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_offset 8
+#define SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_shift 0
+#define SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_offset 9
+#define SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_shift 0
+#define SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for RESERVED_DW10 word*/
+/*define for reserved_dw10 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW10_reserved_dw10_offset 10
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW10_reserved_dw10_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW10_reserved_dw10_shift 0
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW10_RESERVED_DW10(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW10_reserved_dw10_mask) << SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW10_reserved_dw10_shift)
+
+/*define for RESERVED_DW11 word*/
+/*define for reserved_dw11 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW11_reserved_dw11_offset 11
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW11_reserved_dw11_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW11_reserved_dw11_shift 0
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW11_RESERVED_DW11(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW11_reserved_dw11_mask) << SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW11_reserved_dw11_shift)
+
+/*define for RESERVED_DW12 word*/
+/*define for reserved_dw12 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW12_reserved_dw12_offset 12
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW12_reserved_dw12_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW12_reserved_dw12_shift 0
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW12_RESERVED_DW12(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW12_reserved_dw12_mask) << SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW12_reserved_dw12_shift)
+
+/*define for RESERVED_DW13 word*/
+/*define for reserved_dw13 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW13_reserved_dw13_offset 13
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW13_reserved_dw13_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW13_reserved_dw13_shift 0
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW13_RESERVED_DW13(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW13_reserved_dw13_mask) << SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW13_reserved_dw13_shift)
+
+/*define for COMPLETION_SIGNAL_LO word*/
+/*define for completion_signal_31_0 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_LO_completion_signal_31_0_offset 14
+#define SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_LO_completion_signal_31_0_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_LO_completion_signal_31_0_shift 0
+#define SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_LO_COMPLETION_SIGNAL_31_0(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_LO_completion_signal_31_0_mask) << SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_LO_completion_signal_31_0_shift)
+
+/*define for COMPLETION_SIGNAL_HI word*/
+/*define for completion_signal_63_32 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_HI_completion_signal_63_32_offset 15
+#define SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_HI_completion_signal_63_32_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_HI_completion_signal_63_32_shift 0
+#define SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_HI_COMPLETION_SIGNAL_63_32(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_HI_completion_signal_63_32_mask) << SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_HI_completion_signal_63_32_shift)
+
+
+/*
+** Definitions for SDMA_AQL_PKT_BARRIER_OR packet
+*/
+
+/*define for HEADER word*/
+/*define for format field*/
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_format_offset 0
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_format_mask 0x000000FF
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_format_shift 0
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_FORMAT(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_HEADER_format_mask) << SDMA_AQL_PKT_BARRIER_OR_HEADER_format_shift)
+
+/*define for barrier field*/
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_barrier_offset 0
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_barrier_mask 0x00000001
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_barrier_shift 8
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_BARRIER(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_HEADER_barrier_mask) << SDMA_AQL_PKT_BARRIER_OR_HEADER_barrier_shift)
+
+/*define for acquire_fence_scope field*/
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_acquire_fence_scope_offset 0
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_acquire_fence_scope_mask 0x00000003
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_acquire_fence_scope_shift 9
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_ACQUIRE_FENCE_SCOPE(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_HEADER_acquire_fence_scope_mask) << SDMA_AQL_PKT_BARRIER_OR_HEADER_acquire_fence_scope_shift)
+
+/*define for release_fence_scope field*/
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_release_fence_scope_offset 0
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_release_fence_scope_mask 0x00000003
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_release_fence_scope_shift 11
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_RELEASE_FENCE_SCOPE(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_HEADER_release_fence_scope_mask) << SDMA_AQL_PKT_BARRIER_OR_HEADER_release_fence_scope_shift)
+
+/*define for reserved field*/
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_reserved_offset 0
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_reserved_mask 0x00000007
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_reserved_shift 13
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_RESERVED(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_HEADER_reserved_mask) << SDMA_AQL_PKT_BARRIER_OR_HEADER_reserved_shift)
+
+/*define for op field*/
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_op_offset 0
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_op_mask 0x0000000F
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_op_shift 16
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_OP(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_HEADER_op_mask) << SDMA_AQL_PKT_BARRIER_OR_HEADER_op_shift)
+
+/*define for subop field*/
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_subop_offset 0
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_subop_mask 0x00000007
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_subop_shift 20
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_SUBOP(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_HEADER_subop_mask) << SDMA_AQL_PKT_BARRIER_OR_HEADER_subop_shift)
+
+/*define for RESERVED_DW1 word*/
+/*define for reserved_dw1 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW1_reserved_dw1_offset 1
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW1_reserved_dw1_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW1_reserved_dw1_shift 0
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW1_RESERVED_DW1(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW1_reserved_dw1_mask) << SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW1_reserved_dw1_shift)
+
+/*define for DEPENDENT_ADDR_0_LO word*/
+/*define for dependent_addr_0_31_0 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_LO_dependent_addr_0_31_0_offset 2
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_LO_dependent_addr_0_31_0_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_LO_dependent_addr_0_31_0_shift 0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_LO_DEPENDENT_ADDR_0_31_0(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_LO_dependent_addr_0_31_0_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_LO_dependent_addr_0_31_0_shift)
+
+/*define for DEPENDENT_ADDR_0_HI word*/
+/*define for dependent_addr_0_63_32 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_HI_dependent_addr_0_63_32_offset 3
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_HI_dependent_addr_0_63_32_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_HI_dependent_addr_0_63_32_shift 0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_HI_DEPENDENT_ADDR_0_63_32(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_HI_dependent_addr_0_63_32_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_HI_dependent_addr_0_63_32_shift)
+
+/*define for DEPENDENT_ADDR_1_LO word*/
+/*define for dependent_addr_1_31_0 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_LO_dependent_addr_1_31_0_offset 4
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_LO_dependent_addr_1_31_0_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_LO_dependent_addr_1_31_0_shift 0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_LO_DEPENDENT_ADDR_1_31_0(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_LO_dependent_addr_1_31_0_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_LO_dependent_addr_1_31_0_shift)
+
+/*define for DEPENDENT_ADDR_1_HI word*/
+/*define for dependent_addr_1_63_32 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_HI_dependent_addr_1_63_32_offset 5
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_HI_dependent_addr_1_63_32_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_HI_dependent_addr_1_63_32_shift 0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_HI_DEPENDENT_ADDR_1_63_32(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_HI_dependent_addr_1_63_32_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_HI_dependent_addr_1_63_32_shift)
+
+/*define for DEPENDENT_ADDR_2_LO word*/
+/*define for dependent_addr_2_31_0 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_LO_dependent_addr_2_31_0_offset 6
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_LO_dependent_addr_2_31_0_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_LO_dependent_addr_2_31_0_shift 0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_LO_DEPENDENT_ADDR_2_31_0(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_LO_dependent_addr_2_31_0_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_LO_dependent_addr_2_31_0_shift)
+
+/*define for DEPENDENT_ADDR_2_HI word*/
+/*define for dependent_addr_2_63_32 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_HI_dependent_addr_2_63_32_offset 7
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_HI_dependent_addr_2_63_32_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_HI_dependent_addr_2_63_32_shift 0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_HI_DEPENDENT_ADDR_2_63_32(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_HI_dependent_addr_2_63_32_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_HI_dependent_addr_2_63_32_shift)
+
+/*define for DEPENDENT_ADDR_3_LO word*/
+/*define for dependent_addr_3_31_0 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_LO_dependent_addr_3_31_0_offset 8
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_LO_dependent_addr_3_31_0_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_LO_dependent_addr_3_31_0_shift 0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_LO_DEPENDENT_ADDR_3_31_0(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_LO_dependent_addr_3_31_0_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_LO_dependent_addr_3_31_0_shift)
+
+/*define for DEPENDENT_ADDR_3_HI word*/
+/*define for dependent_addr_3_63_32 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_HI_dependent_addr_3_63_32_offset 9
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_HI_dependent_addr_3_63_32_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_HI_dependent_addr_3_63_32_shift 0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_HI_DEPENDENT_ADDR_3_63_32(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_HI_dependent_addr_3_63_32_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_HI_dependent_addr_3_63_32_shift)
+
+/*define for DEPENDENT_ADDR_4_LO word*/
+/*define for dependent_addr_4_31_0 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_LO_dependent_addr_4_31_0_offset 10
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_LO_dependent_addr_4_31_0_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_LO_dependent_addr_4_31_0_shift 0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_LO_DEPENDENT_ADDR_4_31_0(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_LO_dependent_addr_4_31_0_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_LO_dependent_addr_4_31_0_shift)
+
+/*define for DEPENDENT_ADDR_4_HI word*/
+/*define for dependent_addr_4_63_32 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_HI_dependent_addr_4_63_32_offset 11
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_HI_dependent_addr_4_63_32_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_HI_dependent_addr_4_63_32_shift 0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_HI_DEPENDENT_ADDR_4_63_32(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_HI_dependent_addr_4_63_32_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_HI_dependent_addr_4_63_32_shift)
+
+/*define for RESERVED_DW12 word*/
+/*define for reserved_dw12 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW12_reserved_dw12_offset 12
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW12_reserved_dw12_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW12_reserved_dw12_shift 0
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW12_RESERVED_DW12(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW12_reserved_dw12_mask) << SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW12_reserved_dw12_shift)
+
+/*define for RESERVED_DW13 word*/
+/*define for reserved_dw13 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW13_reserved_dw13_offset 13
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW13_reserved_dw13_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW13_reserved_dw13_shift 0
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW13_RESERVED_DW13(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW13_reserved_dw13_mask) << SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW13_reserved_dw13_shift)
+
+/*define for COMPLETION_SIGNAL_LO word*/
+/*define for completion_signal_31_0 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_LO_completion_signal_31_0_offset 14
+#define SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_LO_completion_signal_31_0_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_LO_completion_signal_31_0_shift 0
+#define SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_LO_COMPLETION_SIGNAL_31_0(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_LO_completion_signal_31_0_mask) << SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_LO_completion_signal_31_0_shift)
+
+/*define for COMPLETION_SIGNAL_HI word*/
+/*define for completion_signal_63_32 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_HI_completion_signal_63_32_offset 15
+#define SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_HI_completion_signal_63_32_mask 0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_HI_completion_signal_63_32_shift 0
+#define SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_HI_COMPLETION_SIGNAL_63_32(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_HI_completion_signal_63_32_mask) << SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_HI_completion_signal_63_32_shift)
+
+
+#endif /* __NAVI10_SDMA_PKT_OPEN_H_ */
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v2_3.c b/drivers/gpu/drm/amd/amdgpu/nbio_v2_3.c
new file mode 100644
index 000000000000..835d7b1a841f
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v2_3.c
@@ -0,0 +1,334 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "amdgpu_atombios.h"
+#include "nbio_v2_3.h"
+
+#include "nbio/nbio_2_3_default.h"
+#include "nbio/nbio_2_3_offset.h"
+#include "nbio/nbio_2_3_sh_mask.h"
+
+#define smnPCIE_CONFIG_CNTL 0x11180044
+#define smnCPM_CONTROL 0x11180460
+#define smnPCIE_CNTL2 0x11180070
+
+static u32 nbio_v2_3_get_rev_id(struct amdgpu_device *adev)
+{
+ u32 tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0);
+
+ tmp &= RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK;
+ tmp >>= RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT;
+
+ return tmp;
+}
+
+static void nbio_v2_3_mc_access_enable(struct amdgpu_device *adev, bool enable)
+{
+ if (enable)
+ WREG32_SOC15(NBIO, 0, mmBIF_FB_EN,
+ BIF_FB_EN__FB_READ_EN_MASK |
+ BIF_FB_EN__FB_WRITE_EN_MASK);
+ else
+ WREG32_SOC15(NBIO, 0, mmBIF_FB_EN, 0);
+}
+
+static void nbio_v2_3_hdp_flush(struct amdgpu_device *adev,
+ struct amdgpu_ring *ring)
+{
+ if (!ring || !ring->funcs->emit_wreg)
+ WREG32_SOC15_NO_KIQ(NBIO, 0, mmBIF_BX_PF_HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
+ else
+ amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET(
+ NBIO, 0, mmBIF_BX_PF_HDP_MEM_COHERENCY_FLUSH_CNTL), 0);
+}
+
+static u32 nbio_v2_3_get_memsize(struct amdgpu_device *adev)
+{
+ return RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE);
+}
+
+static void nbio_v2_3_sdma_doorbell_range(struct amdgpu_device *adev, int instance,
+ bool use_doorbell, int doorbell_index,
+ int doorbell_size)
+{
+ u32 reg = instance == 0 ? SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA0_DOORBELL_RANGE) :
+ SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA1_DOORBELL_RANGE);
+
+ u32 doorbell_range = RREG32(reg);
+
+ if (use_doorbell) {
+ doorbell_range = REG_SET_FIELD(doorbell_range,
+ BIF_SDMA0_DOORBELL_RANGE, OFFSET,
+ doorbell_index);
+ doorbell_range = REG_SET_FIELD(doorbell_range,
+ BIF_SDMA0_DOORBELL_RANGE, SIZE,
+ doorbell_size);
+ } else
+ doorbell_range = REG_SET_FIELD(doorbell_range,
+ BIF_SDMA0_DOORBELL_RANGE, SIZE,
+ 0);
+
+ WREG32(reg, doorbell_range);
+}
+
+static void nbio_v2_3_vcn_doorbell_range(struct amdgpu_device *adev, bool use_doorbell,
+ int doorbell_index)
+{
+ u32 reg = SOC15_REG_OFFSET(NBIO, 0, mmBIF_MMSCH0_DOORBELL_RANGE);
+
+ u32 doorbell_range = RREG32(reg);
+
+ if (use_doorbell) {
+ doorbell_range = REG_SET_FIELD(doorbell_range,
+ BIF_MMSCH0_DOORBELL_RANGE, OFFSET,
+ doorbell_index);
+ doorbell_range = REG_SET_FIELD(doorbell_range,
+ BIF_MMSCH0_DOORBELL_RANGE, SIZE, 8);
+ } else
+ doorbell_range = REG_SET_FIELD(doorbell_range,
+ BIF_MMSCH0_DOORBELL_RANGE, SIZE, 0);
+
+ WREG32(reg, doorbell_range);
+}
+
+static void nbio_v2_3_enable_doorbell_aperture(struct amdgpu_device *adev,
+ bool enable)
+{
+ WREG32_FIELD15(NBIO, 0, RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN, BIF_DOORBELL_APER_EN,
+ enable ? 1 : 0);
+}
+
+static void nbio_v2_3_enable_doorbell_selfring_aperture(struct amdgpu_device *adev,
+ bool enable)
+{
+ u32 tmp = 0;
+
+ if (enable) {
+ tmp = REG_SET_FIELD(tmp, BIF_BX_PF_DOORBELL_SELFRING_GPA_APER_CNTL,
+ DOORBELL_SELFRING_GPA_APER_EN, 1) |
+ REG_SET_FIELD(tmp, BIF_BX_PF_DOORBELL_SELFRING_GPA_APER_CNTL,
+ DOORBELL_SELFRING_GPA_APER_MODE, 1) |
+ REG_SET_FIELD(tmp, BIF_BX_PF_DOORBELL_SELFRING_GPA_APER_CNTL,
+ DOORBELL_SELFRING_GPA_APER_SIZE, 0);
+
+ WREG32_SOC15(NBIO, 0, mmBIF_BX_PF_DOORBELL_SELFRING_GPA_APER_BASE_LOW,
+ lower_32_bits(adev->doorbell.base));
+ WREG32_SOC15(NBIO, 0, mmBIF_BX_PF_DOORBELL_SELFRING_GPA_APER_BASE_HIGH,
+ upper_32_bits(adev->doorbell.base));
+ }
+
+ WREG32_SOC15(NBIO, 0, mmBIF_BX_PF_DOORBELL_SELFRING_GPA_APER_CNTL,
+ tmp);
+}
+
+
+static void nbio_v2_3_ih_doorbell_range(struct amdgpu_device *adev,
+ bool use_doorbell, int doorbell_index)
+{
+ u32 ih_doorbell_range = RREG32_SOC15(NBIO, 0, mmBIF_IH_DOORBELL_RANGE);
+
+ if (use_doorbell) {
+ ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+ BIF_IH_DOORBELL_RANGE, OFFSET,
+ doorbell_index);
+ ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+ BIF_IH_DOORBELL_RANGE, SIZE,
+ 2);
+ } else
+ ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+ BIF_IH_DOORBELL_RANGE, SIZE,
+ 0);
+
+ WREG32_SOC15(NBIO, 0, mmBIF_IH_DOORBELL_RANGE, ih_doorbell_range);
+}
+
+static void nbio_v2_3_ih_control(struct amdgpu_device *adev)
+{
+ u32 interrupt_cntl;
+
+ /* setup interrupt control */
+ WREG32_SOC15(NBIO, 0, mmINTERRUPT_CNTL2, adev->dummy_page_addr >> 8);
+
+ interrupt_cntl = RREG32_SOC15(NBIO, 0, mmINTERRUPT_CNTL);
+ /*
+ * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi
+ * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN
+ */
+ interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL,
+ IH_DUMMY_RD_OVERRIDE, 0);
+
+ /* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */
+ interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL,
+ IH_REQ_NONSNOOP_EN, 0);
+
+ WREG32_SOC15(NBIO, 0, mmINTERRUPT_CNTL, interrupt_cntl);
+}
+
+static void nbio_v2_3_update_medium_grain_clock_gating(struct amdgpu_device *adev,
+ bool enable)
+{
+ uint32_t def, data;
+
+ def = data = RREG32_PCIE(smnCPM_CONTROL);
+ if (enable && (adev->cg_flags & AMD_CG_SUPPORT_BIF_MGCG)) {
+ data |= (CPM_CONTROL__LCLK_DYN_GATE_ENABLE_MASK |
+ CPM_CONTROL__TXCLK_DYN_GATE_ENABLE_MASK |
+ CPM_CONTROL__TXCLK_LCNT_GATE_ENABLE_MASK |
+ CPM_CONTROL__TXCLK_REGS_GATE_ENABLE_MASK |
+ CPM_CONTROL__TXCLK_PRBS_GATE_ENABLE_MASK |
+ CPM_CONTROL__REFCLK_REGS_GATE_ENABLE_MASK);
+ } else {
+ data &= ~(CPM_CONTROL__LCLK_DYN_GATE_ENABLE_MASK |
+ CPM_CONTROL__TXCLK_DYN_GATE_ENABLE_MASK |
+ CPM_CONTROL__TXCLK_LCNT_GATE_ENABLE_MASK |
+ CPM_CONTROL__TXCLK_REGS_GATE_ENABLE_MASK |
+ CPM_CONTROL__TXCLK_PRBS_GATE_ENABLE_MASK |
+ CPM_CONTROL__REFCLK_REGS_GATE_ENABLE_MASK);
+ }
+
+ if (def != data)
+ WREG32_PCIE(smnCPM_CONTROL, data);
+}
+
+static void nbio_v2_3_update_medium_grain_light_sleep(struct amdgpu_device *adev,
+ bool enable)
+{
+ uint32_t def, data;
+
+ def = data = RREG32_PCIE(smnPCIE_CNTL2);
+ if (enable && (adev->cg_flags & AMD_CG_SUPPORT_BIF_LS)) {
+ data |= (PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
+ PCIE_CNTL2__MST_MEM_LS_EN_MASK |
+ PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK);
+ } else {
+ data &= ~(PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
+ PCIE_CNTL2__MST_MEM_LS_EN_MASK |
+ PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK);
+ }
+
+ if (def != data)
+ WREG32_PCIE(smnPCIE_CNTL2, data);
+}
+
+static void nbio_v2_3_get_clockgating_state(struct amdgpu_device *adev,
+ u32 *flags)
+{
+ int data;
+
+ /* AMD_CG_SUPPORT_BIF_MGCG */
+ data = RREG32_PCIE(smnCPM_CONTROL);
+ if (data & CPM_CONTROL__LCLK_DYN_GATE_ENABLE_MASK)
+ *flags |= AMD_CG_SUPPORT_BIF_MGCG;
+
+ /* AMD_CG_SUPPORT_BIF_LS */
+ data = RREG32_PCIE(smnPCIE_CNTL2);
+ if (data & PCIE_CNTL2__SLV_MEM_LS_EN_MASK)
+ *flags |= AMD_CG_SUPPORT_BIF_LS;
+}
+
+static u32 nbio_v2_3_get_hdp_flush_req_offset(struct amdgpu_device *adev)
+{
+ return SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF_GPU_HDP_FLUSH_REQ);
+}
+
+static u32 nbio_v2_3_get_hdp_flush_done_offset(struct amdgpu_device *adev)
+{
+ return SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF_GPU_HDP_FLUSH_DONE);
+}
+
+static u32 nbio_v2_3_get_pcie_index_offset(struct amdgpu_device *adev)
+{
+ return SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX2);
+}
+
+static u32 nbio_v2_3_get_pcie_data_offset(struct amdgpu_device *adev)
+{
+ return SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA2);
+}
+
+const struct nbio_hdp_flush_reg nbio_v2_3_hdp_flush_reg = {
+ .ref_and_mask_cp0 = BIF_BX_PF_GPU_HDP_FLUSH_DONE__CP0_MASK,
+ .ref_and_mask_cp1 = BIF_BX_PF_GPU_HDP_FLUSH_DONE__CP1_MASK,
+ .ref_and_mask_cp2 = BIF_BX_PF_GPU_HDP_FLUSH_DONE__CP2_MASK,
+ .ref_and_mask_cp3 = BIF_BX_PF_GPU_HDP_FLUSH_DONE__CP3_MASK,
+ .ref_and_mask_cp4 = BIF_BX_PF_GPU_HDP_FLUSH_DONE__CP4_MASK,
+ .ref_and_mask_cp5 = BIF_BX_PF_GPU_HDP_FLUSH_DONE__CP5_MASK,
+ .ref_and_mask_cp6 = BIF_BX_PF_GPU_HDP_FLUSH_DONE__CP6_MASK,
+ .ref_and_mask_cp7 = BIF_BX_PF_GPU_HDP_FLUSH_DONE__CP7_MASK,
+ .ref_and_mask_cp8 = BIF_BX_PF_GPU_HDP_FLUSH_DONE__CP8_MASK,
+ .ref_and_mask_cp9 = BIF_BX_PF_GPU_HDP_FLUSH_DONE__CP9_MASK,
+ .ref_and_mask_sdma0 = BIF_BX_PF_GPU_HDP_FLUSH_DONE__SDMA0_MASK,
+ .ref_and_mask_sdma1 = BIF_BX_PF_GPU_HDP_FLUSH_DONE__SDMA1_MASK,
+};
+
+static void nbio_v2_3_detect_hw_virt(struct amdgpu_device *adev)
+{
+ uint32_t reg;
+
+ reg = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_RCC_IOV_FUNC_IDENTIFIER);
+ if (reg & 1)
+ adev->virt.caps |= AMDGPU_SRIOV_CAPS_IS_VF;
+
+ if (reg & 0x80000000)
+ adev->virt.caps |= AMDGPU_SRIOV_CAPS_ENABLE_IOV;
+
+ if (!reg) {
+ if (is_virtual_machine()) /* passthrough mode exclus sriov mod */
+ adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
+ }
+}
+
+static void nbio_v2_3_init_registers(struct amdgpu_device *adev)
+{
+ uint32_t def, data;
+
+ def = data = RREG32_PCIE(smnPCIE_CONFIG_CNTL);
+ data = REG_SET_FIELD(data, PCIE_CONFIG_CNTL, CI_SWUS_MAX_READ_REQUEST_SIZE_MODE, 1);
+ data = REG_SET_FIELD(data, PCIE_CONFIG_CNTL, CI_SWUS_MAX_READ_REQUEST_SIZE_PRIV, 1);
+
+ if (def != data)
+ WREG32_PCIE(smnPCIE_CONFIG_CNTL, data);
+}
+
+const struct amdgpu_nbio_funcs nbio_v2_3_funcs = {
+ .hdp_flush_reg = &nbio_v2_3_hdp_flush_reg,
+ .get_hdp_flush_req_offset = nbio_v2_3_get_hdp_flush_req_offset,
+ .get_hdp_flush_done_offset = nbio_v2_3_get_hdp_flush_done_offset,
+ .get_pcie_index_offset = nbio_v2_3_get_pcie_index_offset,
+ .get_pcie_data_offset = nbio_v2_3_get_pcie_data_offset,
+ .get_rev_id = nbio_v2_3_get_rev_id,
+ .mc_access_enable = nbio_v2_3_mc_access_enable,
+ .hdp_flush = nbio_v2_3_hdp_flush,
+ .get_memsize = nbio_v2_3_get_memsize,
+ .sdma_doorbell_range = nbio_v2_3_sdma_doorbell_range,
+ .vcn_doorbell_range = nbio_v2_3_vcn_doorbell_range,
+ .enable_doorbell_aperture = nbio_v2_3_enable_doorbell_aperture,
+ .enable_doorbell_selfring_aperture = nbio_v2_3_enable_doorbell_selfring_aperture,
+ .ih_doorbell_range = nbio_v2_3_ih_doorbell_range,
+ .update_medium_grain_clock_gating = nbio_v2_3_update_medium_grain_clock_gating,
+ .update_medium_grain_light_sleep = nbio_v2_3_update_medium_grain_light_sleep,
+ .get_clockgating_state = nbio_v2_3_get_clockgating_state,
+ .ih_control = nbio_v2_3_ih_control,
+ .init_registers = nbio_v2_3_init_registers,
+ .detect_hw_virt = nbio_v2_3_detect_hw_virt,
+};
diff --git a/drivers/gpu/drm/i915/i915_gemfs.h b/drivers/gpu/drm/amd/amdgpu/nbio_v2_3.h
index cca8bdc5b93e..5ae52085f6b7 100644
--- a/drivers/gpu/drm/i915/i915_gemfs.h
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v2_3.h
@@ -1,5 +1,5 @@
/*
- * Copyright © 2017 Intel Corporation
+ * Copyright 2019 Advanced Micro Devices, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -8,27 +8,24 @@
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
*
*/
-#ifndef __I915_GEMFS_H__
-#define __I915_GEMFS_H__
+#ifndef __NBIO_V2_3_H__
+#define __NBIO_V2_3_H__
-struct drm_i915_private;
+#include "soc15_common.h"
-int i915_gemfs_init(struct drm_i915_private *i915);
-
-void i915_gemfs_fini(struct drm_i915_private *i915);
+extern const struct amdgpu_nbio_funcs nbio_v2_3_funcs;
#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
index 1cdb98ad2db3..73419fa38159 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
@@ -29,9 +29,18 @@
#include "nbio/nbio_7_0_sh_mask.h"
#include "nbio/nbio_7_0_smn.h"
#include "vega10_enum.h"
+#include <uapi/linux/kfd_ioctl.h>
#define smnNBIF_MGCG_CTRL_LCLK 0x1013a05c
+static void nbio_v7_0_remap_hdp_registers(struct amdgpu_device *adev)
+{
+ WREG32_SOC15(NBIO, 0, mmREMAP_HDP_MEM_FLUSH_CNTL,
+ adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL);
+ WREG32_SOC15(NBIO, 0, mmREMAP_HDP_REG_FLUSH_CNTL,
+ adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL);
+}
+
static u32 nbio_v7_0_get_rev_id(struct amdgpu_device *adev)
{
u32 tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0);
@@ -55,10 +64,9 @@ static void nbio_v7_0_hdp_flush(struct amdgpu_device *adev,
struct amdgpu_ring *ring)
{
if (!ring || !ring->funcs->emit_wreg)
- WREG32_SOC15_NO_KIQ(NBIO, 0, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0);
+ WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0);
else
- amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET(
- NBIO, 0, mmHDP_MEM_COHERENCY_FLUSH_CNTL), 0);
+ amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0);
}
static u32 nbio_v7_0_get_memsize(struct amdgpu_device *adev)
@@ -283,4 +291,5 @@ const struct amdgpu_nbio_funcs nbio_v7_0_funcs = {
.ih_control = nbio_v7_0_ih_control,
.init_registers = nbio_v7_0_init_registers,
.detect_hw_virt = nbio_v7_0_detect_hw_virt,
+ .remap_hdp_registers = nbio_v7_0_remap_hdp_registers,
};
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c
index c69d51598cfe..bfaaa327ae3c 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c
@@ -27,9 +27,18 @@
#include "nbio/nbio_7_4_offset.h"
#include "nbio/nbio_7_4_sh_mask.h"
#include "nbio/nbio_7_4_0_smn.h"
+#include <uapi/linux/kfd_ioctl.h>
#define smnNBIF_MGCG_CTRL_LCLK 0x1013a21c
+static void nbio_v7_4_remap_hdp_registers(struct amdgpu_device *adev)
+{
+ WREG32_SOC15(NBIO, 0, mmREMAP_HDP_MEM_FLUSH_CNTL,
+ adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL);
+ WREG32_SOC15(NBIO, 0, mmREMAP_HDP_REG_FLUSH_CNTL,
+ adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL);
+}
+
static u32 nbio_v7_4_get_rev_id(struct amdgpu_device *adev)
{
u32 tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0);
@@ -53,10 +62,9 @@ static void nbio_v7_4_hdp_flush(struct amdgpu_device *adev,
struct amdgpu_ring *ring)
{
if (!ring || !ring->funcs->emit_wreg)
- WREG32_SOC15_NO_KIQ(NBIO, 0, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0);
+ WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0);
else
- amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET(
- NBIO, 0, mmHDP_MEM_COHERENCY_FLUSH_CNTL), 0);
+ amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0);
}
static u32 nbio_v7_4_get_memsize(struct amdgpu_device *adev)
@@ -262,4 +270,5 @@ const struct amdgpu_nbio_funcs nbio_v7_4_funcs = {
.ih_control = nbio_v7_4_ih_control,
.init_registers = nbio_v7_4_init_registers,
.detect_hw_virt = nbio_v7_4_detect_hw_virt,
+ .remap_hdp_registers = nbio_v7_4_remap_hdp_registers,
};
diff --git a/drivers/gpu/drm/amd/amdgpu/nv.c b/drivers/gpu/drm/amd/amdgpu/nv.c
new file mode 100644
index 000000000000..9922bce3fd89
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/nv.c
@@ -0,0 +1,820 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/firmware.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <drm/drmP.h>
+#include "amdgpu.h"
+#include "amdgpu_atombios.h"
+#include "amdgpu_ih.h"
+#include "amdgpu_uvd.h"
+#include "amdgpu_vce.h"
+#include "amdgpu_ucode.h"
+#include "amdgpu_psp.h"
+#include "amdgpu_smu.h"
+#include "atom.h"
+#include "amd_pcie.h"
+
+#include "gc/gc_10_1_0_offset.h"
+#include "gc/gc_10_1_0_sh_mask.h"
+#include "hdp/hdp_5_0_0_offset.h"
+#include "hdp/hdp_5_0_0_sh_mask.h"
+
+#include "soc15.h"
+#include "soc15_common.h"
+#include "gmc_v10_0.h"
+#include "gfxhub_v2_0.h"
+#include "mmhub_v2_0.h"
+#include "nv.h"
+#include "navi10_ih.h"
+#include "gfx_v10_0.h"
+#include "sdma_v5_0.h"
+#include "vcn_v2_0.h"
+#include "dce_virtual.h"
+#include "mes_v10_1.h"
+
+static const struct amd_ip_funcs nv_common_ip_funcs;
+
+/*
+ * Indirect registers accessor
+ */
+static u32 nv_pcie_rreg(struct amdgpu_device *adev, u32 reg)
+{
+ unsigned long flags, address, data;
+ u32 r;
+ address = adev->nbio_funcs->get_pcie_index_offset(adev);
+ data = adev->nbio_funcs->get_pcie_data_offset(adev);
+
+ spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+ WREG32(address, reg);
+ (void)RREG32(address);
+ r = RREG32(data);
+ spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+ return r;
+}
+
+static void nv_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
+{
+ unsigned long flags, address, data;
+
+ address = adev->nbio_funcs->get_pcie_index_offset(adev);
+ data = adev->nbio_funcs->get_pcie_data_offset(adev);
+
+ spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+ WREG32(address, reg);
+ (void)RREG32(address);
+ WREG32(data, v);
+ (void)RREG32(data);
+ spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+}
+
+static u32 nv_didt_rreg(struct amdgpu_device *adev, u32 reg)
+{
+ unsigned long flags, address, data;
+ u32 r;
+
+ address = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_INDEX);
+ data = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_DATA);
+
+ spin_lock_irqsave(&adev->didt_idx_lock, flags);
+ WREG32(address, (reg));
+ r = RREG32(data);
+ spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
+ return r;
+}
+
+static void nv_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
+{
+ unsigned long flags, address, data;
+
+ address = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_INDEX);
+ data = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_DATA);
+
+ spin_lock_irqsave(&adev->didt_idx_lock, flags);
+ WREG32(address, (reg));
+ WREG32(data, (v));
+ spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
+}
+
+static u32 nv_get_config_memsize(struct amdgpu_device *adev)
+{
+ return adev->nbio_funcs->get_memsize(adev);
+}
+
+static u32 nv_get_xclk(struct amdgpu_device *adev)
+{
+ return adev->clock.spll.reference_freq;
+}
+
+
+void nv_grbm_select(struct amdgpu_device *adev,
+ u32 me, u32 pipe, u32 queue, u32 vmid)
+{
+ u32 grbm_gfx_cntl = 0;
+ grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, PIPEID, pipe);
+ grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, MEID, me);
+ grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, VMID, vmid);
+ grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, QUEUEID, queue);
+
+ WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL), grbm_gfx_cntl);
+}
+
+static void nv_vga_set_state(struct amdgpu_device *adev, bool state)
+{
+ /* todo */
+}
+
+static bool nv_read_disabled_bios(struct amdgpu_device *adev)
+{
+ /* todo */
+ return false;
+}
+
+static bool nv_read_bios_from_rom(struct amdgpu_device *adev,
+ u8 *bios, u32 length_bytes)
+{
+ /* TODO: will implement it when SMU header is available */
+ return false;
+}
+
+static struct soc15_allowed_register_entry nv_allowed_read_registers[] = {
+ { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS)},
+ { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS2)},
+ { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE0)},
+ { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE1)},
+ { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE2)},
+ { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE3)},
+#if 0 /* TODO: will set it when SDMA header is available */
+ { SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_STATUS_REG)},
+ { SOC15_REG_ENTRY(SDMA1, 0, mmSDMA1_STATUS_REG)},
+#endif
+ { SOC15_REG_ENTRY(GC, 0, mmCP_STAT)},
+ { SOC15_REG_ENTRY(GC, 0, mmCP_STALLED_STAT1)},
+ { SOC15_REG_ENTRY(GC, 0, mmCP_STALLED_STAT2)},
+ { SOC15_REG_ENTRY(GC, 0, mmCP_STALLED_STAT3)},
+ { SOC15_REG_ENTRY(GC, 0, mmCP_CPF_BUSY_STAT)},
+ { SOC15_REG_ENTRY(GC, 0, mmCP_CPF_STALLED_STAT1)},
+ { SOC15_REG_ENTRY(GC, 0, mmCP_CPF_STATUS)},
+ { SOC15_REG_ENTRY(GC, 0, mmCP_CPC_STALLED_STAT1)},
+ { SOC15_REG_ENTRY(GC, 0, mmCP_CPC_STATUS)},
+ { SOC15_REG_ENTRY(GC, 0, mmGB_ADDR_CONFIG)},
+};
+
+static uint32_t nv_read_indexed_register(struct amdgpu_device *adev, u32 se_num,
+ u32 sh_num, u32 reg_offset)
+{
+ uint32_t val;
+
+ mutex_lock(&adev->grbm_idx_mutex);
+ if (se_num != 0xffffffff || sh_num != 0xffffffff)
+ amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
+
+ val = RREG32(reg_offset);
+
+ if (se_num != 0xffffffff || sh_num != 0xffffffff)
+ amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+ mutex_unlock(&adev->grbm_idx_mutex);
+ return val;
+}
+
+static uint32_t nv_get_register_value(struct amdgpu_device *adev,
+ bool indexed, u32 se_num,
+ u32 sh_num, u32 reg_offset)
+{
+ if (indexed) {
+ return nv_read_indexed_register(adev, se_num, sh_num, reg_offset);
+ } else {
+ if (reg_offset == SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG))
+ return adev->gfx.config.gb_addr_config;
+ return RREG32(reg_offset);
+ }
+}
+
+static int nv_read_register(struct amdgpu_device *adev, u32 se_num,
+ u32 sh_num, u32 reg_offset, u32 *value)
+{
+ uint32_t i;
+ struct soc15_allowed_register_entry *en;
+
+ *value = 0;
+ for (i = 0; i < ARRAY_SIZE(nv_allowed_read_registers); i++) {
+ en = &nv_allowed_read_registers[i];
+ if (reg_offset !=
+ (adev->reg_offset[en->hwip][en->inst][en->seg] + en->reg_offset))
+ continue;
+
+ *value = nv_get_register_value(adev,
+ nv_allowed_read_registers[i].grbm_indexed,
+ se_num, sh_num, reg_offset);
+ return 0;
+ }
+ return -EINVAL;
+}
+
+#if 0
+static void nv_gpu_pci_config_reset(struct amdgpu_device *adev)
+{
+ u32 i;
+
+ dev_info(adev->dev, "GPU pci config reset\n");
+
+ /* disable BM */
+ pci_clear_master(adev->pdev);
+ /* reset */
+ amdgpu_pci_config_reset(adev);
+
+ udelay(100);
+
+ /* wait for asic to come out of reset */
+ for (i = 0; i < adev->usec_timeout; i++) {
+ u32 memsize = nbio_v2_3_get_memsize(adev);
+ if (memsize != 0xffffffff)
+ break;
+ udelay(1);
+ }
+
+}
+#endif
+
+static int nv_asic_mode1_reset(struct amdgpu_device *adev)
+{
+ u32 i;
+ int ret = 0;
+
+ amdgpu_atombios_scratch_regs_engine_hung(adev, true);
+
+ dev_info(adev->dev, "GPU mode1 reset\n");
+
+ /* disable BM */
+ pci_clear_master(adev->pdev);
+
+ pci_save_state(adev->pdev);
+
+ ret = psp_gpu_reset(adev);
+ if (ret)
+ dev_err(adev->dev, "GPU mode1 reset failed\n");
+
+ pci_restore_state(adev->pdev);
+
+ /* wait for asic to come out of reset */
+ for (i = 0; i < adev->usec_timeout; i++) {
+ u32 memsize = adev->nbio_funcs->get_memsize(adev);
+
+ if (memsize != 0xffffffff)
+ break;
+ udelay(1);
+ }
+
+ amdgpu_atombios_scratch_regs_engine_hung(adev, false);
+
+ return ret;
+}
+static int nv_asic_reset(struct amdgpu_device *adev)
+{
+
+ /* FIXME: it doesn't work since vega10 */
+#if 0
+ amdgpu_atombios_scratch_regs_engine_hung(adev, true);
+
+ nv_gpu_pci_config_reset(adev);
+
+ amdgpu_atombios_scratch_regs_engine_hung(adev, false);
+#endif
+ int ret = 0;
+ struct smu_context *smu = &adev->smu;
+
+ if (smu_baco_is_support(smu))
+ ret = smu_baco_reset(smu);
+ else
+ ret = nv_asic_mode1_reset(adev);
+
+ return ret;
+}
+
+static int nv_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
+{
+ /* todo */
+ return 0;
+}
+
+static int nv_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
+{
+ /* todo */
+ return 0;
+}
+
+static void nv_pcie_gen3_enable(struct amdgpu_device *adev)
+{
+ if (pci_is_root_bus(adev->pdev->bus))
+ return;
+
+ if (amdgpu_pcie_gen2 == 0)
+ return;
+
+ if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
+ CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
+ return;
+
+ /* todo */
+}
+
+static void nv_program_aspm(struct amdgpu_device *adev)
+{
+
+ if (amdgpu_aspm == 0)
+ return;
+
+ /* todo */
+}
+
+static void nv_enable_doorbell_aperture(struct amdgpu_device *adev,
+ bool enable)
+{
+ adev->nbio_funcs->enable_doorbell_aperture(adev, enable);
+ adev->nbio_funcs->enable_doorbell_selfring_aperture(adev, enable);
+}
+
+static const struct amdgpu_ip_block_version nv_common_ip_block =
+{
+ .type = AMD_IP_BLOCK_TYPE_COMMON,
+ .major = 1,
+ .minor = 0,
+ .rev = 0,
+ .funcs = &nv_common_ip_funcs,
+};
+
+int nv_set_ip_blocks(struct amdgpu_device *adev)
+{
+ /* Set IP register base before any HW register access */
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ navi10_reg_base_init(adev);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ adev->nbio_funcs = &nbio_v2_3_funcs;
+
+ adev->nbio_funcs->detect_hw_virt(adev);
+
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ amdgpu_device_ip_block_add(adev, &nv_common_ip_block);
+ amdgpu_device_ip_block_add(adev, &gmc_v10_0_ip_block);
+ amdgpu_device_ip_block_add(adev, &navi10_ih_ip_block);
+ amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block);
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP &&
+ is_support_sw_smu(adev))
+ amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block);
+ if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
+ amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
+#if defined(CONFIG_DRM_AMD_DC)
+ else if (amdgpu_device_has_dc_support(adev))
+ amdgpu_device_ip_block_add(adev, &dm_ip_block);
+#endif
+ amdgpu_device_ip_block_add(adev, &gfx_v10_0_ip_block);
+ amdgpu_device_ip_block_add(adev, &sdma_v5_0_ip_block);
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT &&
+ is_support_sw_smu(adev))
+ amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block);
+ amdgpu_device_ip_block_add(adev, &vcn_v2_0_ip_block);
+ if (adev->enable_mes)
+ amdgpu_device_ip_block_add(adev, &mes_v10_1_ip_block);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static uint32_t nv_get_rev_id(struct amdgpu_device *adev)
+{
+ return adev->nbio_funcs->get_rev_id(adev);
+}
+
+static void nv_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
+{
+ adev->nbio_funcs->hdp_flush(adev, ring);
+}
+
+static void nv_invalidate_hdp(struct amdgpu_device *adev,
+ struct amdgpu_ring *ring)
+{
+ if (!ring || !ring->funcs->emit_wreg) {
+ WREG32_SOC15_NO_KIQ(NBIO, 0, mmHDP_READ_CACHE_INVALIDATE, 1);
+ } else {
+ amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET(
+ HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1);
+ }
+}
+
+static bool nv_need_full_reset(struct amdgpu_device *adev)
+{
+ return true;
+}
+
+static void nv_get_pcie_usage(struct amdgpu_device *adev,
+ uint64_t *count0,
+ uint64_t *count1)
+{
+ /*TODO*/
+}
+
+static bool nv_need_reset_on_init(struct amdgpu_device *adev)
+{
+#if 0
+ u32 sol_reg;
+
+ if (adev->flags & AMD_IS_APU)
+ return false;
+
+ /* Check sOS sign of life register to confirm sys driver and sOS
+ * are already been loaded.
+ */
+ sol_reg = RREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_81);
+ if (sol_reg)
+ return true;
+#endif
+ /* TODO: re-enable it when mode1 reset is functional */
+ return false;
+}
+
+static void nv_init_doorbell_index(struct amdgpu_device *adev)
+{
+ adev->doorbell_index.kiq = AMDGPU_NAVI10_DOORBELL_KIQ;
+ adev->doorbell_index.mec_ring0 = AMDGPU_NAVI10_DOORBELL_MEC_RING0;
+ adev->doorbell_index.mec_ring1 = AMDGPU_NAVI10_DOORBELL_MEC_RING1;
+ adev->doorbell_index.mec_ring2 = AMDGPU_NAVI10_DOORBELL_MEC_RING2;
+ adev->doorbell_index.mec_ring3 = AMDGPU_NAVI10_DOORBELL_MEC_RING3;
+ adev->doorbell_index.mec_ring4 = AMDGPU_NAVI10_DOORBELL_MEC_RING4;
+ adev->doorbell_index.mec_ring5 = AMDGPU_NAVI10_DOORBELL_MEC_RING5;
+ adev->doorbell_index.mec_ring6 = AMDGPU_NAVI10_DOORBELL_MEC_RING6;
+ adev->doorbell_index.mec_ring7 = AMDGPU_NAVI10_DOORBELL_MEC_RING7;
+ adev->doorbell_index.userqueue_start = AMDGPU_NAVI10_DOORBELL_USERQUEUE_START;
+ adev->doorbell_index.userqueue_end = AMDGPU_NAVI10_DOORBELL_USERQUEUE_END;
+ adev->doorbell_index.gfx_ring0 = AMDGPU_NAVI10_DOORBELL_GFX_RING0;
+ adev->doorbell_index.gfx_ring1 = AMDGPU_NAVI10_DOORBELL_GFX_RING1;
+ adev->doorbell_index.sdma_engine[0] = AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE0;
+ adev->doorbell_index.sdma_engine[1] = AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE1;
+ adev->doorbell_index.ih = AMDGPU_NAVI10_DOORBELL_IH;
+ adev->doorbell_index.vcn.vcn_ring0_1 = AMDGPU_NAVI10_DOORBELL64_VCN0_1;
+ adev->doorbell_index.vcn.vcn_ring2_3 = AMDGPU_NAVI10_DOORBELL64_VCN2_3;
+ adev->doorbell_index.vcn.vcn_ring4_5 = AMDGPU_NAVI10_DOORBELL64_VCN4_5;
+ adev->doorbell_index.vcn.vcn_ring6_7 = AMDGPU_NAVI10_DOORBELL64_VCN6_7;
+ adev->doorbell_index.first_non_cp = AMDGPU_NAVI10_DOORBELL64_FIRST_NON_CP;
+ adev->doorbell_index.last_non_cp = AMDGPU_NAVI10_DOORBELL64_LAST_NON_CP;
+
+ adev->doorbell_index.max_assignment = AMDGPU_NAVI10_DOORBELL_MAX_ASSIGNMENT << 1;
+ adev->doorbell_index.sdma_doorbell_range = 20;
+}
+
+static const struct amdgpu_asic_funcs nv_asic_funcs =
+{
+ .read_disabled_bios = &nv_read_disabled_bios,
+ .read_bios_from_rom = &nv_read_bios_from_rom,
+ .read_register = &nv_read_register,
+ .reset = &nv_asic_reset,
+ .set_vga_state = &nv_vga_set_state,
+ .get_xclk = &nv_get_xclk,
+ .set_uvd_clocks = &nv_set_uvd_clocks,
+ .set_vce_clocks = &nv_set_vce_clocks,
+ .get_config_memsize = &nv_get_config_memsize,
+ .flush_hdp = &nv_flush_hdp,
+ .invalidate_hdp = &nv_invalidate_hdp,
+ .init_doorbell_index = &nv_init_doorbell_index,
+ .need_full_reset = &nv_need_full_reset,
+ .get_pcie_usage = &nv_get_pcie_usage,
+ .need_reset_on_init = &nv_need_reset_on_init,
+};
+
+static int nv_common_early_init(void *handle)
+{
+ bool psp_enabled = false;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ adev->smc_rreg = NULL;
+ adev->smc_wreg = NULL;
+ adev->pcie_rreg = &nv_pcie_rreg;
+ adev->pcie_wreg = &nv_pcie_wreg;
+
+ /* TODO: will add them during VCN v2 implementation */
+ adev->uvd_ctx_rreg = NULL;
+ adev->uvd_ctx_wreg = NULL;
+
+ adev->didt_rreg = &nv_didt_rreg;
+ adev->didt_wreg = &nv_didt_wreg;
+
+ adev->asic_funcs = &nv_asic_funcs;
+
+ if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP) &&
+ (amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_PSP)))
+ psp_enabled = true;
+
+ adev->rev_id = nv_get_rev_id(adev);
+ adev->external_rev_id = 0xff;
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
+ AMD_CG_SUPPORT_GFX_CGCG |
+ AMD_CG_SUPPORT_IH_CG |
+ AMD_CG_SUPPORT_HDP_MGCG |
+ AMD_CG_SUPPORT_HDP_LS |
+ AMD_CG_SUPPORT_SDMA_MGCG |
+ AMD_CG_SUPPORT_SDMA_LS |
+ AMD_CG_SUPPORT_MC_MGCG |
+ AMD_CG_SUPPORT_MC_LS |
+ AMD_CG_SUPPORT_ATHUB_MGCG |
+ AMD_CG_SUPPORT_ATHUB_LS |
+ AMD_CG_SUPPORT_VCN_MGCG |
+ AMD_CG_SUPPORT_BIF_MGCG |
+ AMD_CG_SUPPORT_BIF_LS;
+ adev->pg_flags = AMD_PG_SUPPORT_VCN |
+ AMD_PG_SUPPORT_VCN_DPG |
+ AMD_PG_SUPPORT_ATHUB;
+ adev->external_rev_id = adev->rev_id + 0x1;
+ break;
+ default:
+ /* FIXME: not supported yet */
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int nv_common_late_init(void *handle)
+{
+ return 0;
+}
+
+static int nv_common_sw_init(void *handle)
+{
+ return 0;
+}
+
+static int nv_common_sw_fini(void *handle)
+{
+ return 0;
+}
+
+static int nv_common_hw_init(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ /* enable pcie gen2/3 link */
+ nv_pcie_gen3_enable(adev);
+ /* enable aspm */
+ nv_program_aspm(adev);
+ /* setup nbio registers */
+ adev->nbio_funcs->init_registers(adev);
+ /* enable the doorbell aperture */
+ nv_enable_doorbell_aperture(adev, true);
+
+ return 0;
+}
+
+static int nv_common_hw_fini(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ /* disable the doorbell aperture */
+ nv_enable_doorbell_aperture(adev, false);
+
+ return 0;
+}
+
+static int nv_common_suspend(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ return nv_common_hw_fini(adev);
+}
+
+static int nv_common_resume(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ return nv_common_hw_init(adev);
+}
+
+static bool nv_common_is_idle(void *handle)
+{
+ return true;
+}
+
+static int nv_common_wait_for_idle(void *handle)
+{
+ return 0;
+}
+
+static int nv_common_soft_reset(void *handle)
+{
+ return 0;
+}
+
+static void nv_update_hdp_mem_power_gating(struct amdgpu_device *adev,
+ bool enable)
+{
+ uint32_t hdp_clk_cntl, hdp_clk_cntl1;
+ uint32_t hdp_mem_pwr_cntl;
+
+ if (!(adev->cg_flags & (AMD_CG_SUPPORT_HDP_LS |
+ AMD_CG_SUPPORT_HDP_DS |
+ AMD_CG_SUPPORT_HDP_SD)))
+ return;
+
+ hdp_clk_cntl = hdp_clk_cntl1 = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL);
+ hdp_mem_pwr_cntl = RREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL);
+
+ /* Before doing clock/power mode switch,
+ * forced on IPH & RC clock */
+ hdp_clk_cntl = REG_SET_FIELD(hdp_clk_cntl, HDP_CLK_CNTL,
+ IPH_MEM_CLK_SOFT_OVERRIDE, 1);
+ hdp_clk_cntl = REG_SET_FIELD(hdp_clk_cntl, HDP_CLK_CNTL,
+ RC_MEM_CLK_SOFT_OVERRIDE, 1);
+ WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl);
+
+ /* HDP 5.0 doesn't support dynamic power mode switch,
+ * disable clock and power gating before any changing */
+ hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL,
+ IPH_MEM_POWER_CTRL_EN, 0);
+ hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL,
+ IPH_MEM_POWER_LS_EN, 0);
+ hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL,
+ IPH_MEM_POWER_DS_EN, 0);
+ hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL,
+ IPH_MEM_POWER_SD_EN, 0);
+ hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL,
+ RC_MEM_POWER_CTRL_EN, 0);
+ hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL,
+ RC_MEM_POWER_LS_EN, 0);
+ hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL,
+ RC_MEM_POWER_DS_EN, 0);
+ hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL,
+ RC_MEM_POWER_SD_EN, 0);
+ WREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL, hdp_mem_pwr_cntl);
+
+ /* only one clock gating mode (LS/DS/SD) can be enabled */
+ if (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS) {
+ hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl,
+ HDP_MEM_POWER_CTRL,
+ IPH_MEM_POWER_LS_EN, enable);
+ hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl,
+ HDP_MEM_POWER_CTRL,
+ RC_MEM_POWER_LS_EN, enable);
+ } else if (adev->cg_flags & AMD_CG_SUPPORT_HDP_DS) {
+ hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl,
+ HDP_MEM_POWER_CTRL,
+ IPH_MEM_POWER_DS_EN, enable);
+ hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl,
+ HDP_MEM_POWER_CTRL,
+ RC_MEM_POWER_DS_EN, enable);
+ } else if (adev->cg_flags & AMD_CG_SUPPORT_HDP_SD) {
+ hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl,
+ HDP_MEM_POWER_CTRL,
+ IPH_MEM_POWER_SD_EN, enable);
+ /* RC should not use shut down mode, fallback to ds */
+ hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl,
+ HDP_MEM_POWER_CTRL,
+ RC_MEM_POWER_DS_EN, enable);
+ }
+
+ WREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL, hdp_mem_pwr_cntl);
+
+ /* restore IPH & RC clock override after clock/power mode changing */
+ WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl1);
+}
+
+static void nv_update_hdp_clock_gating(struct amdgpu_device *adev,
+ bool enable)
+{
+ uint32_t hdp_clk_cntl;
+
+ if (!(adev->cg_flags & AMD_CG_SUPPORT_HDP_MGCG))
+ return;
+
+ hdp_clk_cntl = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL);
+
+ if (enable) {
+ hdp_clk_cntl &=
+ ~(uint32_t)
+ (HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK |
+ HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK |
+ HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK |
+ HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK |
+ HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK |
+ HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK);
+ } else {
+ hdp_clk_cntl |= HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK |
+ HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK |
+ HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK |
+ HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK |
+ HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK |
+ HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK;
+ }
+
+ WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl);
+}
+
+static int nv_common_set_clockgating_state(void *handle,
+ enum amd_clockgating_state state)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ if (amdgpu_sriov_vf(adev))
+ return 0;
+
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ adev->nbio_funcs->update_medium_grain_clock_gating(adev,
+ state == AMD_CG_STATE_GATE ? true : false);
+ adev->nbio_funcs->update_medium_grain_light_sleep(adev,
+ state == AMD_CG_STATE_GATE ? true : false);
+ nv_update_hdp_mem_power_gating(adev,
+ state == AMD_CG_STATE_GATE ? true : false);
+ nv_update_hdp_clock_gating(adev,
+ state == AMD_CG_STATE_GATE ? true : false);
+ break;
+ default:
+ break;
+ }
+ return 0;
+}
+
+static int nv_common_set_powergating_state(void *handle,
+ enum amd_powergating_state state)
+{
+ /* TODO */
+ return 0;
+}
+
+static void nv_common_get_clockgating_state(void *handle, u32 *flags)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ uint32_t tmp;
+
+ if (amdgpu_sriov_vf(adev))
+ *flags = 0;
+
+ adev->nbio_funcs->get_clockgating_state(adev, flags);
+
+ /* AMD_CG_SUPPORT_HDP_MGCG */
+ tmp = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL);
+ if (!(tmp & (HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK |
+ HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK |
+ HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK |
+ HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK |
+ HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK |
+ HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK)))
+ *flags |= AMD_CG_SUPPORT_HDP_MGCG;
+
+ /* AMD_CG_SUPPORT_HDP_LS/DS/SD */
+ tmp = RREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL);
+ if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK)
+ *flags |= AMD_CG_SUPPORT_HDP_LS;
+ else if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_DS_EN_MASK)
+ *flags |= AMD_CG_SUPPORT_HDP_DS;
+ else if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_SD_EN_MASK)
+ *flags |= AMD_CG_SUPPORT_HDP_SD;
+
+ return;
+}
+
+static const struct amd_ip_funcs nv_common_ip_funcs = {
+ .name = "nv_common",
+ .early_init = nv_common_early_init,
+ .late_init = nv_common_late_init,
+ .sw_init = nv_common_sw_init,
+ .sw_fini = nv_common_sw_fini,
+ .hw_init = nv_common_hw_init,
+ .hw_fini = nv_common_hw_fini,
+ .suspend = nv_common_suspend,
+ .resume = nv_common_resume,
+ .is_idle = nv_common_is_idle,
+ .wait_for_idle = nv_common_wait_for_idle,
+ .soft_reset = nv_common_soft_reset,
+ .set_clockgating_state = nv_common_set_clockgating_state,
+ .set_powergating_state = nv_common_set_powergating_state,
+ .get_clockgating_state = nv_common_get_clockgating_state,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/nv.h b/drivers/gpu/drm/amd/amdgpu/nv.h
new file mode 100644
index 000000000000..639c54933cc5
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/nv.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __NV_H__
+#define __NV_H__
+
+#include "nbio_v2_3.h"
+
+void nv_grbm_select(struct amdgpu_device *adev,
+ u32 me, u32 pipe, u32 queue, u32 vmid);
+int nv_set_ip_blocks(struct amdgpu_device *adev);
+int navi10_reg_base_init(struct amdgpu_device *adev);
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/nvd.h b/drivers/gpu/drm/amd/amdgpu/nvd.h
new file mode 100644
index 000000000000..1de984647dbb
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/nvd.h
@@ -0,0 +1,418 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef NVD_H
+#define NVD_H
+
+/**
+ * Navi's PM4 definitions
+ */
+#define PACKET_TYPE0 0
+#define PACKET_TYPE1 1
+#define PACKET_TYPE2 2
+#define PACKET_TYPE3 3
+
+#define CP_PACKET_GET_TYPE(h) (((h) >> 30) & 3)
+#define CP_PACKET_GET_COUNT(h) (((h) >> 16) & 0x3FFF)
+#define CP_PACKET0_GET_REG(h) ((h) & 0xFFFF)
+#define CP_PACKET3_GET_OPCODE(h) (((h) >> 8) & 0xFF)
+#define PACKET0(reg, n) ((PACKET_TYPE0 << 30) | \
+ ((reg) & 0xFFFF) | \
+ ((n) & 0x3FFF) << 16)
+#define CP_PACKET2 0x80000000
+#define PACKET2_PAD_SHIFT 0
+#define PACKET2_PAD_MASK (0x3fffffff << 0)
+
+#define PACKET2(v) (CP_PACKET2 | REG_SET(PACKET2_PAD, (v)))
+
+#define PACKET3(op, n) ((PACKET_TYPE3 << 30) | \
+ (((op) & 0xFF) << 8) | \
+ ((n) & 0x3FFF) << 16)
+
+#define PACKET3_COMPUTE(op, n) (PACKET3(op, n) | 1 << 1)
+
+/* Packet 3 types */
+#define PACKET3_NOP 0x10
+#define PACKET3_SET_BASE 0x11
+#define PACKET3_BASE_INDEX(x) ((x) << 0)
+#define CE_PARTITION_BASE 3
+#define PACKET3_CLEAR_STATE 0x12
+#define PACKET3_INDEX_BUFFER_SIZE 0x13
+#define PACKET3_DISPATCH_DIRECT 0x15
+#define PACKET3_DISPATCH_INDIRECT 0x16
+#define PACKET3_INDIRECT_BUFFER_END 0x17
+#define PACKET3_INDIRECT_BUFFER_CNST_END 0x19
+#define PACKET3_ATOMIC_GDS 0x1D
+#define PACKET3_ATOMIC_MEM 0x1E
+#define PACKET3_OCCLUSION_QUERY 0x1F
+#define PACKET3_SET_PREDICATION 0x20
+#define PACKET3_REG_RMW 0x21
+#define PACKET3_COND_EXEC 0x22
+#define PACKET3_PRED_EXEC 0x23
+#define PACKET3_DRAW_INDIRECT 0x24
+#define PACKET3_DRAW_INDEX_INDIRECT 0x25
+#define PACKET3_INDEX_BASE 0x26
+#define PACKET3_DRAW_INDEX_2 0x27
+#define PACKET3_CONTEXT_CONTROL 0x28
+#define PACKET3_INDEX_TYPE 0x2A
+#define PACKET3_DRAW_INDIRECT_MULTI 0x2C
+#define PACKET3_DRAW_INDEX_AUTO 0x2D
+#define PACKET3_NUM_INSTANCES 0x2F
+#define PACKET3_DRAW_INDEX_MULTI_AUTO 0x30
+#define PACKET3_INDIRECT_BUFFER_PRIV 0x32
+#define PACKET3_INDIRECT_BUFFER_CNST 0x33
+#define PACKET3_COND_INDIRECT_BUFFER_CNST 0x33
+#define PACKET3_STRMOUT_BUFFER_UPDATE 0x34
+#define PACKET3_DRAW_INDEX_OFFSET_2 0x35
+#define PACKET3_DRAW_PREAMBLE 0x36
+#define PACKET3_WRITE_DATA 0x37
+#define WRITE_DATA_DST_SEL(x) ((x) << 8)
+ /* 0 - register
+ * 1 - memory (sync - via GRBM)
+ * 2 - gl2
+ * 3 - gds
+ * 4 - reserved
+ * 5 - memory (async - direct)
+ */
+#define WR_ONE_ADDR (1 << 16)
+#define WR_CONFIRM (1 << 20)
+#define WRITE_DATA_CACHE_POLICY(x) ((x) << 25)
+ /* 0 - LRU
+ * 1 - Stream
+ */
+#define WRITE_DATA_ENGINE_SEL(x) ((x) << 30)
+ /* 0 - me
+ * 1 - pfp
+ * 2 - ce
+ */
+#define PACKET3_DRAW_INDEX_INDIRECT_MULTI 0x38
+#define PACKET3_MEM_SEMAPHORE 0x39
+# define PACKET3_SEM_USE_MAILBOX (0x1 << 16)
+# define PACKET3_SEM_SEL_SIGNAL_TYPE (0x1 << 20) /* 0 = increment, 1 = write 1 */
+# define PACKET3_SEM_SEL_SIGNAL (0x6 << 29)
+# define PACKET3_SEM_SEL_WAIT (0x7 << 29)
+#define PACKET3_DRAW_INDEX_MULTI_INST 0x3A
+#define PACKET3_COPY_DW 0x3B
+#define PACKET3_WAIT_REG_MEM 0x3C
+#define WAIT_REG_MEM_FUNCTION(x) ((x) << 0)
+ /* 0 - always
+ * 1 - <
+ * 2 - <=
+ * 3 - ==
+ * 4 - !=
+ * 5 - >=
+ * 6 - >
+ */
+#define WAIT_REG_MEM_MEM_SPACE(x) ((x) << 4)
+ /* 0 - reg
+ * 1 - mem
+ */
+#define WAIT_REG_MEM_OPERATION(x) ((x) << 6)
+ /* 0 - wait_reg_mem
+ * 1 - wr_wait_wr_reg
+ */
+#define WAIT_REG_MEM_ENGINE(x) ((x) << 8)
+ /* 0 - me
+ * 1 - pfp
+ */
+#define PACKET3_INDIRECT_BUFFER 0x3F
+#define INDIRECT_BUFFER_VALID (1 << 23)
+#define INDIRECT_BUFFER_CACHE_POLICY(x) ((x) << 28)
+ /* 0 - LRU
+ * 1 - Stream
+ * 2 - Bypass
+ */
+#define INDIRECT_BUFFER_PRE_ENB(x) ((x) << 21)
+#define INDIRECT_BUFFER_PRE_RESUME(x) ((x) << 30)
+#define PACKET3_COND_INDIRECT_BUFFER 0x3F
+#define PACKET3_COPY_DATA 0x40
+#define PACKET3_CP_DMA 0x41
+#define PACKET3_PFP_SYNC_ME 0x42
+#define PACKET3_SURFACE_SYNC 0x43
+#define PACKET3_ME_INITIALIZE 0x44
+#define PACKET3_COND_WRITE 0x45
+#define PACKET3_EVENT_WRITE 0x46
+#define EVENT_TYPE(x) ((x) << 0)
+#define EVENT_INDEX(x) ((x) << 8)
+ /* 0 - any non-TS event
+ * 1 - ZPASS_DONE, PIXEL_PIPE_STAT_*
+ * 2 - SAMPLE_PIPELINESTAT
+ * 3 - SAMPLE_STREAMOUTSTAT*
+ * 4 - *S_PARTIAL_FLUSH
+ */
+#define PACKET3_EVENT_WRITE_EOP 0x47
+#define PACKET3_EVENT_WRITE_EOS 0x48
+#define PACKET3_RELEASE_MEM 0x49
+#define PACKET3_RELEASE_MEM_EVENT_TYPE(x) ((x) << 0)
+#define PACKET3_RELEASE_MEM_EVENT_INDEX(x) ((x) << 8)
+#define PACKET3_RELEASE_MEM_GCR_GLM_WB (1 << 12)
+#define PACKET3_RELEASE_MEM_GCR_GLM_INV (1 << 13)
+#define PACKET3_RELEASE_MEM_GCR_GLV_INV (1 << 14)
+#define PACKET3_RELEASE_MEM_GCR_GL1_INV (1 << 15)
+#define PACKET3_RELEASE_MEM_GCR_GL2_US (1 << 16)
+#define PACKET3_RELEASE_MEM_GCR_GL2_RANGE (1 << 17)
+#define PACKET3_RELEASE_MEM_GCR_GL2_DISCARD (1 << 19)
+#define PACKET3_RELEASE_MEM_GCR_GL2_INV (1 << 20)
+#define PACKET3_RELEASE_MEM_GCR_GL2_WB (1 << 21)
+#define PACKET3_RELEASE_MEM_GCR_SEQ (1 << 22)
+#define PACKET3_RELEASE_MEM_CACHE_POLICY(x) ((x) << 25)
+ /* 0 - cache_policy__me_release_mem__lru
+ * 1 - cache_policy__me_release_mem__stream
+ * 2 - cache_policy__me_release_mem__noa
+ * 3 - cache_policy__me_release_mem__bypass
+ */
+#define PACKET3_RELEASE_MEM_EXECUTE (1 << 28)
+
+#define PACKET3_RELEASE_MEM_DATA_SEL(x) ((x) << 29)
+ /* 0 - discard
+ * 1 - send low 32bit data
+ * 2 - send 64bit data
+ * 3 - send 64bit GPU counter value
+ * 4 - send 64bit sys counter value
+ */
+#define PACKET3_RELEASE_MEM_INT_SEL(x) ((x) << 24)
+ /* 0 - none
+ * 1 - interrupt only (DATA_SEL = 0)
+ * 2 - interrupt when data write is confirmed
+ */
+#define PACKET3_RELEASE_MEM_DST_SEL(x) ((x) << 16)
+ /* 0 - MC
+ * 1 - TC/L2
+ */
+
+
+
+#define PACKET3_PREAMBLE_CNTL 0x4A
+# define PACKET3_PREAMBLE_BEGIN_CLEAR_STATE (2 << 28)
+# define PACKET3_PREAMBLE_END_CLEAR_STATE (3 << 28)
+#define PACKET3_DMA_DATA 0x50
+/* 1. header
+ * 2. CONTROL
+ * 3. SRC_ADDR_LO or DATA [31:0]
+ * 4. SRC_ADDR_HI [31:0]
+ * 5. DST_ADDR_LO [31:0]
+ * 6. DST_ADDR_HI [7:0]
+ * 7. COMMAND [31:26] | BYTE_COUNT [25:0]
+ */
+/* CONTROL */
+# define PACKET3_DMA_DATA_ENGINE(x) ((x) << 0)
+ /* 0 - ME
+ * 1 - PFP
+ */
+# define PACKET3_DMA_DATA_SRC_CACHE_POLICY(x) ((x) << 13)
+ /* 0 - LRU
+ * 1 - Stream
+ */
+# define PACKET3_DMA_DATA_DST_SEL(x) ((x) << 20)
+ /* 0 - DST_ADDR using DAS
+ * 1 - GDS
+ * 3 - DST_ADDR using L2
+ */
+# define PACKET3_DMA_DATA_DST_CACHE_POLICY(x) ((x) << 25)
+ /* 0 - LRU
+ * 1 - Stream
+ */
+# define PACKET3_DMA_DATA_SRC_SEL(x) ((x) << 29)
+ /* 0 - SRC_ADDR using SAS
+ * 1 - GDS
+ * 2 - DATA
+ * 3 - SRC_ADDR using L2
+ */
+# define PACKET3_DMA_DATA_CP_SYNC (1 << 31)
+/* COMMAND */
+# define PACKET3_DMA_DATA_CMD_SAS (1 << 26)
+ /* 0 - memory
+ * 1 - register
+ */
+# define PACKET3_DMA_DATA_CMD_DAS (1 << 27)
+ /* 0 - memory
+ * 1 - register
+ */
+# define PACKET3_DMA_DATA_CMD_SAIC (1 << 28)
+# define PACKET3_DMA_DATA_CMD_DAIC (1 << 29)
+# define PACKET3_DMA_DATA_CMD_RAW_WAIT (1 << 30)
+#define PACKET3_CONTEXT_REG_RMW 0x51
+#define PACKET3_GFX_CNTX_UPDATE 0x52
+#define PACKET3_BLK_CNTX_UPDATE 0x53
+#define PACKET3_INCR_UPDT_STATE 0x55
+#define PACKET3_ACQUIRE_MEM 0x58
+#define PACKET3_REWIND 0x59
+#define PACKET3_INTERRUPT 0x5A
+#define PACKET3_GEN_PDEPTE 0x5B
+#define PACKET3_INDIRECT_BUFFER_PASID 0x5C
+#define PACKET3_PRIME_UTCL2 0x5D
+#define PACKET3_LOAD_UCONFIG_REG 0x5E
+#define PACKET3_LOAD_SH_REG 0x5F
+#define PACKET3_LOAD_CONFIG_REG 0x60
+#define PACKET3_LOAD_CONTEXT_REG 0x61
+#define PACKET3_LOAD_COMPUTE_STATE 0x62
+#define PACKET3_LOAD_SH_REG_INDEX 0x63
+#define PACKET3_SET_CONFIG_REG 0x68
+#define PACKET3_SET_CONFIG_REG_START 0x00002000
+#define PACKET3_SET_CONFIG_REG_END 0x00002c00
+#define PACKET3_SET_CONTEXT_REG 0x69
+#define PACKET3_SET_CONTEXT_REG_START 0x0000a000
+#define PACKET3_SET_CONTEXT_REG_END 0x0000a400
+#define PACKET3_SET_CONTEXT_REG_INDEX 0x6A
+#define PACKET3_SET_VGPR_REG_DI_MULTI 0x71
+#define PACKET3_SET_SH_REG_DI 0x72
+#define PACKET3_SET_CONTEXT_REG_INDIRECT 0x73
+#define PACKET3_SET_SH_REG_DI_MULTI 0x74
+#define PACKET3_GFX_PIPE_LOCK 0x75
+#define PACKET3_SET_SH_REG 0x76
+#define PACKET3_SET_SH_REG_START 0x00002c00
+#define PACKET3_SET_SH_REG_END 0x00003000
+#define PACKET3_SET_SH_REG_OFFSET 0x77
+#define PACKET3_SET_QUEUE_REG 0x78
+#define PACKET3_SET_UCONFIG_REG 0x79
+#define PACKET3_SET_UCONFIG_REG_START 0x0000c000
+#define PACKET3_SET_UCONFIG_REG_END 0x0000c400
+#define PACKET3_SET_UCONFIG_REG_INDEX 0x7A
+#define PACKET3_FORWARD_HEADER 0x7C
+#define PACKET3_SCRATCH_RAM_WRITE 0x7D
+#define PACKET3_SCRATCH_RAM_READ 0x7E
+#define PACKET3_LOAD_CONST_RAM 0x80
+#define PACKET3_WRITE_CONST_RAM 0x81
+#define PACKET3_DUMP_CONST_RAM 0x83
+#define PACKET3_INCREMENT_CE_COUNTER 0x84
+#define PACKET3_INCREMENT_DE_COUNTER 0x85
+#define PACKET3_WAIT_ON_CE_COUNTER 0x86
+#define PACKET3_WAIT_ON_DE_COUNTER_DIFF 0x88
+#define PACKET3_SWITCH_BUFFER 0x8B
+#define PACKET3_DISPATCH_DRAW_PREAMBLE 0x8C
+#define PACKET3_DISPATCH_DRAW_PREAMBLE_ACE 0x8C
+#define PACKET3_DISPATCH_DRAW 0x8D
+#define PACKET3_DISPATCH_DRAW_ACE 0x8D
+#define PACKET3_GET_LOD_STATS 0x8E
+#define PACKET3_DRAW_MULTI_PREAMBLE 0x8F
+#define PACKET3_FRAME_CONTROL 0x90
+# define FRAME_CMD(x) ((x) << 28)
+ /*
+ * x=0: tmz_begin
+ * x=1: tmz_end
+ */
+#define PACKET3_INDEX_ATTRIBUTES_INDIRECT 0x91
+#define PACKET3_WAIT_REG_MEM64 0x93
+#define PACKET3_COND_PREEMPT 0x94
+#define PACKET3_HDP_FLUSH 0x95
+#define PACKET3_COPY_DATA_RB 0x96
+#define PACKET3_INVALIDATE_TLBS 0x98
+# define PACKET3_INVALIDATE_TLBS_DST_SEL(x) ((x) << 0)
+# define PACKET3_INVALIDATE_TLBS_ALL_HUB(x) ((x) << 4)
+# define PACKET3_INVALIDATE_TLBS_PASID(x) ((x) << 5)
+#define PACKET3_AQL_PACKET 0x99
+#define PACKET3_DMA_DATA_FILL_MULTI 0x9A
+#define PACKET3_SET_SH_REG_INDEX 0x9B
+#define PACKET3_DRAW_INDIRECT_COUNT_MULTI 0x9C
+#define PACKET3_DRAW_INDEX_INDIRECT_COUNT_MULTI 0x9D
+#define PACKET3_DUMP_CONST_RAM_OFFSET 0x9E
+#define PACKET3_LOAD_CONTEXT_REG_INDEX 0x9F
+#define PACKET3_SET_RESOURCES 0xA0
+/* 1. header
+ * 2. CONTROL
+ * 3. QUEUE_MASK_LO [31:0]
+ * 4. QUEUE_MASK_HI [31:0]
+ * 5. GWS_MASK_LO [31:0]
+ * 6. GWS_MASK_HI [31:0]
+ * 7. OAC_MASK [15:0]
+ * 8. GDS_HEAP_SIZE [16:11] | GDS_HEAP_BASE [5:0]
+ */
+# define PACKET3_SET_RESOURCES_VMID_MASK(x) ((x) << 0)
+# define PACKET3_SET_RESOURCES_UNMAP_LATENTY(x) ((x) << 16)
+# define PACKET3_SET_RESOURCES_QUEUE_TYPE(x) ((x) << 29)
+#define PACKET3_MAP_PROCESS 0xA1
+#define PACKET3_MAP_QUEUES 0xA2
+/* 1. header
+ * 2. CONTROL
+ * 3. CONTROL2
+ * 4. MQD_ADDR_LO [31:0]
+ * 5. MQD_ADDR_HI [31:0]
+ * 6. WPTR_ADDR_LO [31:0]
+ * 7. WPTR_ADDR_HI [31:0]
+ */
+/* CONTROL */
+# define PACKET3_MAP_QUEUES_QUEUE_SEL(x) ((x) << 4)
+# define PACKET3_MAP_QUEUES_VMID(x) ((x) << 8)
+# define PACKET3_MAP_QUEUES_QUEUE(x) ((x) << 13)
+# define PACKET3_MAP_QUEUES_PIPE(x) ((x) << 16)
+# define PACKET3_MAP_QUEUES_ME(x) ((x) << 18)
+# define PACKET3_MAP_QUEUES_QUEUE_TYPE(x) ((x) << 21)
+# define PACKET3_MAP_QUEUES_ALLOC_FORMAT(x) ((x) << 24)
+# define PACKET3_MAP_QUEUES_ENGINE_SEL(x) ((x) << 26)
+# define PACKET3_MAP_QUEUES_NUM_QUEUES(x) ((x) << 29)
+/* CONTROL2 */
+# define PACKET3_MAP_QUEUES_CHECK_DISABLE(x) ((x) << 1)
+# define PACKET3_MAP_QUEUES_DOORBELL_OFFSET(x) ((x) << 2)
+#define PACKET3_UNMAP_QUEUES 0xA3
+/* 1. header
+ * 2. CONTROL
+ * 3. CONTROL2
+ * 4. CONTROL3
+ * 5. CONTROL4
+ * 6. CONTROL5
+ */
+/* CONTROL */
+# define PACKET3_UNMAP_QUEUES_ACTION(x) ((x) << 0)
+ /* 0 - PREEMPT_QUEUES
+ * 1 - RESET_QUEUES
+ * 2 - DISABLE_PROCESS_QUEUES
+ * 3 - PREEMPT_QUEUES_NO_UNMAP
+ */
+# define PACKET3_UNMAP_QUEUES_QUEUE_SEL(x) ((x) << 4)
+# define PACKET3_UNMAP_QUEUES_ENGINE_SEL(x) ((x) << 26)
+# define PACKET3_UNMAP_QUEUES_NUM_QUEUES(x) ((x) << 29)
+/* CONTROL2a */
+# define PACKET3_UNMAP_QUEUES_PASID(x) ((x) << 0)
+/* CONTROL2b */
+# define PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(x) ((x) << 2)
+/* CONTROL3a */
+# define PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET1(x) ((x) << 2)
+/* CONTROL3b */
+# define PACKET3_UNMAP_QUEUES_RB_WPTR(x) ((x) << 0)
+/* CONTROL4 */
+# define PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET2(x) ((x) << 2)
+/* CONTROL5 */
+# define PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET3(x) ((x) << 2)
+#define PACKET3_QUERY_STATUS 0xA4
+/* 1. header
+ * 2. CONTROL
+ * 3. CONTROL2
+ * 4. ADDR_LO [31:0]
+ * 5. ADDR_HI [31:0]
+ * 6. DATA_LO [31:0]
+ * 7. DATA_HI [31:0]
+ */
+/* CONTROL */
+# define PACKET3_QUERY_STATUS_CONTEXT_ID(x) ((x) << 0)
+# define PACKET3_QUERY_STATUS_INTERRUPT_SEL(x) ((x) << 28)
+# define PACKET3_QUERY_STATUS_COMMAND(x) ((x) << 30)
+/* CONTROL2a */
+# define PACKET3_QUERY_STATUS_PASID(x) ((x) << 0)
+/* CONTROL2b */
+# define PACKET3_QUERY_STATUS_DOORBELL_OFFSET(x) ((x) << 2)
+# define PACKET3_QUERY_STATUS_ENG_SEL(x) ((x) << 25)
+#define PACKET3_RUN_LIST 0xA5
+#define PACKET3_MAP_PROCESS_VM 0xA6
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
index 2f79765b4bdb..5080a73a95a5 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
+++ b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
@@ -80,6 +80,18 @@ struct psp_gfx_ctrl
*/
#define GFX_FLAG_RESPONSE 0x80000000
+/* Gbr IH registers ID */
+enum ih_reg_id {
+ IH_RB = 0, // IH_RB_CNTL
+ IH_RB_RNG1 = 1, // IH_RB_CNTL_RING1
+ IH_RB_RNG2 = 2, // IH_RB_CNTL_RING2
+};
+
+/* Command to setup Gibraltar IH register */
+struct psp_gfx_cmd_gbr_ih_reg {
+ uint32_t reg_value; /* Value to be set to the IH_RB_CNTL... register*/
+ enum ih_reg_id reg_id; /* ID of the register */
+};
/* TEE Gfx Command IDs for the ring buffer interface. */
enum psp_gfx_cmd_id
@@ -94,9 +106,12 @@ enum psp_gfx_cmd_id
GFX_CMD_ID_SAVE_RESTORE = 0x00000008, /* save/restore HW IP FW */
GFX_CMD_ID_SETUP_VMR = 0x00000009, /* setup VMR region */
GFX_CMD_ID_DESTROY_VMR = 0x0000000A, /* destroy VMR region */
+ GFX_CMD_ID_PROG_REG = 0x0000000B, /* program regs */
+ /* IDs upto 0x1F are reserved for older programs (Raven, Vega 10/12/20) */
+ GFX_CMD_ID_LOAD_TOC = 0x00000020, /* Load TOC and obtain TMR size */
+ GFX_CMD_ID_AUTOLOAD_RLC = 0x00000021, /* Indicates all graphics fw loaded, start RLC autoload */
};
-
/* Command to load Trusted Application binary into PSP OS. */
struct psp_gfx_cmd_load_ta
{
@@ -168,33 +183,59 @@ struct psp_gfx_cmd_setup_tmr
/* FW types for GFX_CMD_ID_LOAD_IP_FW command. Limit 31. */
-enum psp_gfx_fw_type
-{
- GFX_FW_TYPE_NONE = 0,
- GFX_FW_TYPE_CP_ME = 1,
- GFX_FW_TYPE_CP_PFP = 2,
- GFX_FW_TYPE_CP_CE = 3,
- GFX_FW_TYPE_CP_MEC = 4,
- GFX_FW_TYPE_CP_MEC_ME1 = 5,
- GFX_FW_TYPE_CP_MEC_ME2 = 6,
- GFX_FW_TYPE_RLC_V = 7,
- GFX_FW_TYPE_RLC_G = 8,
- GFX_FW_TYPE_SDMA0 = 9,
- GFX_FW_TYPE_SDMA1 = 10,
- GFX_FW_TYPE_DMCU_ERAM = 11,
- GFX_FW_TYPE_DMCU_ISR = 12,
- GFX_FW_TYPE_VCN = 13,
- GFX_FW_TYPE_UVD = 14,
- GFX_FW_TYPE_VCE = 15,
- GFX_FW_TYPE_ISP = 16,
- GFX_FW_TYPE_ACP = 17,
- GFX_FW_TYPE_SMU = 18,
- GFX_FW_TYPE_MMSCH = 19,
- GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM = 20,
- GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM = 21,
- GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL = 22,
- GFX_FW_TYPE_UVD1 = 23,
- GFX_FW_TYPE_MAX = 24
+enum psp_gfx_fw_type {
+ GFX_FW_TYPE_NONE = 0, /* */
+ GFX_FW_TYPE_CP_ME = 1, /* CP-ME VG + RV */
+ GFX_FW_TYPE_CP_PFP = 2, /* CP-PFP VG + RV */
+ GFX_FW_TYPE_CP_CE = 3, /* CP-CE VG + RV */
+ GFX_FW_TYPE_CP_MEC = 4, /* CP-MEC FW VG + RV */
+ GFX_FW_TYPE_CP_MEC_ME1 = 5, /* CP-MEC Jump Table 1 VG + RV */
+ GFX_FW_TYPE_CP_MEC_ME2 = 6, /* CP-MEC Jump Table 2 VG */
+ GFX_FW_TYPE_RLC_V = 7, /* RLC-V VG */
+ GFX_FW_TYPE_RLC_G = 8, /* RLC-G VG + RV */
+ GFX_FW_TYPE_SDMA0 = 9, /* SDMA0 VG + RV */
+ GFX_FW_TYPE_SDMA1 = 10, /* SDMA1 VG */
+ GFX_FW_TYPE_DMCU_ERAM = 11, /* DMCU-ERAM VG + RV */
+ GFX_FW_TYPE_DMCU_ISR = 12, /* DMCU-ISR VG + RV */
+ GFX_FW_TYPE_VCN = 13, /* VCN RV */
+ GFX_FW_TYPE_UVD = 14, /* UVD VG */
+ GFX_FW_TYPE_VCE = 15, /* VCE VG */
+ GFX_FW_TYPE_ISP = 16, /* ISP RV */
+ GFX_FW_TYPE_ACP = 17, /* ACP RV */
+ GFX_FW_TYPE_SMU = 18, /* SMU VG */
+ GFX_FW_TYPE_MMSCH = 19, /* MMSCH VG */
+ GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM = 20, /* RLC GPM VG + RV */
+ GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM = 21, /* RLC SRM VG + RV */
+ GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL = 22, /* RLC CNTL VG + RV */
+ GFX_FW_TYPE_UVD1 = 23, /* UVD1 VG-20 */
+ GFX_FW_TYPE_TOC = 24, /* TOC NV-10 */
+ GFX_FW_TYPE_RLC_P = 25, /* RLC P NV */
+ GFX_FW_TYPE_RLX6 = 26, /* RLX6 NV */
+ GFX_FW_TYPE_GLOBAL_TAP_DELAYS = 27, /* GLOBAL TAP DELAYS NV */
+ GFX_FW_TYPE_SE0_TAP_DELAYS = 28, /* SE0 TAP DELAYS NV */
+ GFX_FW_TYPE_SE1_TAP_DELAYS = 29, /* SE1 TAP DELAYS NV */
+ GFX_FW_TYPE_GLOBAL_SE0_SE1_SKEW_DELAYS = 30, /* GLOBAL SE0/1 SKEW DELAYS NV */
+ GFX_FW_TYPE_SDMA0_JT = 31, /* SDMA0 JT NV */
+ GFX_FW_TYPE_SDMA1_JT = 32, /* SDNA1 JT NV */
+ GFX_FW_TYPE_CP_MES = 33, /* CP MES NV */
+ GFX_FW_TYPE_MES_STACK = 34, /* MES STACK NV */
+ GFX_FW_TYPE_RLC_SRM_DRAM_SR = 35, /* RLC SRM DRAM NV */
+ GFX_FW_TYPE_RLCG_SCRATCH_SR = 36, /* RLCG SCRATCH NV */
+ GFX_FW_TYPE_RLCP_SCRATCH_SR = 37, /* RLCP SCRATCH NV */
+ GFX_FW_TYPE_RLCV_SCRATCH_SR = 38, /* RLCV SCRATCH NV */
+ GFX_FW_TYPE_RLX6_DRAM_SR = 39, /* RLX6 DRAM NV */
+ GFX_FW_TYPE_SDMA0_PG_CONTEXT = 40, /* SDMA0 PG CONTEXT NV */
+ GFX_FW_TYPE_SDMA1_PG_CONTEXT = 41, /* SDMA1 PG CONTEXT NV */
+ GFX_FW_TYPE_GLOBAL_MUX_SELECT_RAM = 42, /* GLOBAL MUX SEL RAM NV */
+ GFX_FW_TYPE_SE0_MUX_SELECT_RAM = 43, /* SE0 MUX SEL RAM NV */
+ GFX_FW_TYPE_SE1_MUX_SELECT_RAM = 44, /* SE1 MUX SEL RAM NV */
+ GFX_FW_TYPE_ACCUM_CTRL_RAM = 45, /* ACCUM CTRL RAM NV */
+ GFX_FW_TYPE_RLCP_CAM = 46, /* RLCP CAM NV */
+ GFX_FW_TYPE_RLC_SPP_CAM_EXT = 47, /* RLC SPP CAM EXT NV */
+ GFX_FW_TYPE_RLX6_DRAM_BOOT = 48, /* RLX6 DRAM BOOT NV */
+ GFX_FW_TYPE_VCN0_RAM = 49, /* VCN_RAM NV */
+ GFX_FW_TYPE_VCN1_RAM = 50, /* VCN_RAM NV */
+ GFX_FW_TYPE_MAX
};
/* Command to load HW IP FW. */
@@ -217,6 +258,20 @@ struct psp_gfx_cmd_save_restore_ip_fw
enum psp_gfx_fw_type fw_type; /* FW type */
};
+/* Command to setup register program */
+struct psp_gfx_cmd_reg_prog {
+ uint32_t reg_value;
+ uint32_t reg_id;
+};
+
+/* Command to load TOC */
+struct psp_gfx_cmd_load_toc
+{
+ uint32_t toc_phy_addr_lo; /* bits [31:0] of GPU Virtual address of FW location (must be 4 KB aligned) */
+ uint32_t toc_phy_addr_hi; /* bits [63:32] of GPU Virtual address of FW location */
+ uint32_t toc_size; /* FW buffer size in bytes */
+};
+
/* All GFX ring buffer commands. */
union psp_gfx_commands
{
@@ -226,21 +281,24 @@ union psp_gfx_commands
struct psp_gfx_cmd_setup_tmr cmd_setup_tmr;
struct psp_gfx_cmd_load_ip_fw cmd_load_ip_fw;
struct psp_gfx_cmd_save_restore_ip_fw cmd_save_restore_ip_fw;
+ struct psp_gfx_cmd_reg_prog cmd_setup_reg_prog;
+ struct psp_gfx_cmd_setup_tmr cmd_setup_vmr;
+ struct psp_gfx_cmd_load_toc cmd_load_toc;
};
-
/* Structure of GFX Response buffer.
* For GPCOM I/F it is part of GFX_CMD_RESP buffer, for RBI
* it is separate buffer.
*/
struct psp_gfx_resp
{
- uint32_t status; /* +0 status of command execution */
- uint32_t session_id; /* +4 session ID in response to LoadTa command */
- uint32_t fw_addr_lo; /* +8 bits [31:0] of FW address within TMR (in response to cmd_load_ip_fw command) */
- uint32_t fw_addr_hi; /* +12 bits [63:32] of FW address within TMR (in response to cmd_load_ip_fw command) */
+ uint32_t status; /* +0 status of command execution */
+ uint32_t session_id; /* +4 session ID in response to LoadTa command */
+ uint32_t fw_addr_lo; /* +8 bits [31:0] of FW address within TMR (in response to cmd_load_ip_fw command) */
+ uint32_t fw_addr_hi; /* +12 bits [63:32] of FW address within TMR (in response to cmd_load_ip_fw command) */
+ uint32_t tmr_size; /* +16 size of the TMR to be reserved including MM fw and Gfx fw in response to cmd_load_toc command */
- uint32_t reserved[4];
+ uint32_t reserved[3];
/* total 32 bytes */
};
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
index 77c2bc344dfc..ce1ea31feee0 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
@@ -24,6 +24,9 @@
*/
#include <linux/firmware.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
#include "amdgpu.h"
#include "amdgpu_psp.h"
#include "amdgpu_ucode.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c
index b91df7bd1d98..41b72588adcf 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c
@@ -21,6 +21,8 @@
*/
#include <linux/firmware.h>
+#include <linux/module.h>
+
#include "amdgpu.h"
#include "amdgpu_psp.h"
#include "amdgpu_ucode.h"
@@ -39,9 +41,16 @@
MODULE_FIRMWARE("amdgpu/vega20_sos.bin");
MODULE_FIRMWARE("amdgpu/vega20_asd.bin");
MODULE_FIRMWARE("amdgpu/vega20_ta.bin");
+MODULE_FIRMWARE("amdgpu/navi10_sos.bin");
+MODULE_FIRMWARE("amdgpu/navi10_asd.bin");
/* address block */
#define smnMP1_FIRMWARE_FLAGS 0x3010024
+/* navi10 reg offset define */
+#define mmRLC_GPM_UCODE_ADDR_NV10 0x5b61
+#define mmRLC_GPM_UCODE_DATA_NV10 0x5b62
+#define mmSDMA0_UCODE_ADDR_NV10 0x5880
+#define mmSDMA0_UCODE_DATA_NV10 0x5881
static int psp_v11_0_init_microcode(struct psp_context *psp)
{
@@ -50,6 +59,7 @@ static int psp_v11_0_init_microcode(struct psp_context *psp)
char fw_name[30];
int err = 0;
const struct psp_firmware_header_v1_0 *sos_hdr;
+ const struct psp_firmware_header_v1_1 *sos_hdr_v1_1;
const struct psp_firmware_header_v1_0 *asd_hdr;
const struct ta_firmware_header_v1_0 *ta_hdr;
@@ -59,6 +69,9 @@ static int psp_v11_0_init_microcode(struct psp_context *psp)
case CHIP_VEGA20:
chip_name = "vega20";
break;
+ case CHIP_NAVI10:
+ chip_name = "navi10";
+ break;
default:
BUG();
}
@@ -73,15 +86,34 @@ static int psp_v11_0_init_microcode(struct psp_context *psp)
goto out;
sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
- adev->psp.sos_fw_version = le32_to_cpu(sos_hdr->header.ucode_version);
- adev->psp.sos_feature_version = le32_to_cpu(sos_hdr->ucode_feature_version);
- adev->psp.sos_bin_size = le32_to_cpu(sos_hdr->sos_size_bytes);
- adev->psp.sys_bin_size = le32_to_cpu(sos_hdr->header.ucode_size_bytes) -
- le32_to_cpu(sos_hdr->sos_size_bytes);
- adev->psp.sys_start_addr = (uint8_t *)sos_hdr +
+ amdgpu_ucode_print_psp_hdr(&sos_hdr->header);
+
+ switch (sos_hdr->header.header_version_major) {
+ case 1:
+ adev->psp.sos_fw_version = le32_to_cpu(sos_hdr->header.ucode_version);
+ adev->psp.sos_feature_version = le32_to_cpu(sos_hdr->ucode_feature_version);
+ adev->psp.sos_bin_size = le32_to_cpu(sos_hdr->sos_size_bytes);
+ adev->psp.sys_bin_size = le32_to_cpu(sos_hdr->sos_offset_bytes);
+ adev->psp.sys_start_addr = (uint8_t *)sos_hdr +
le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
- adev->psp.sos_start_addr = (uint8_t *)adev->psp.sys_start_addr +
+ adev->psp.sos_start_addr = (uint8_t *)adev->psp.sys_start_addr +
le32_to_cpu(sos_hdr->sos_offset_bytes);
+ if (sos_hdr->header.header_version_minor == 1) {
+ sos_hdr_v1_1 = (const struct psp_firmware_header_v1_1 *)adev->psp.sos_fw->data;
+ adev->psp.toc_bin_size = le32_to_cpu(sos_hdr_v1_1->toc_size_bytes);
+ adev->psp.toc_start_addr = (uint8_t *)adev->psp.sys_start_addr +
+ le32_to_cpu(sos_hdr_v1_1->toc_offset_bytes);
+ adev->psp.kdb_bin_size = le32_to_cpu(sos_hdr_v1_1->kdb_size_bytes);
+ adev->psp.kdb_start_addr = (uint8_t *)adev->psp.sys_start_addr +
+ le32_to_cpu(sos_hdr_v1_1->kdb_offset_bytes);
+ }
+ break;
+ default:
+ dev_err(adev->dev,
+ "Unsupported psp sos firmware\n");
+ err = -EINVAL;
+ goto out;
+ }
snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_asd.bin", chip_name);
err = request_firmware(&adev->psp.asd_fw, fw_name, adev->dev);
@@ -99,30 +131,36 @@ static int psp_v11_0_init_microcode(struct psp_context *psp)
adev->psp.asd_start_addr = (uint8_t *)asd_hdr +
le32_to_cpu(asd_hdr->header.ucode_array_offset_bytes);
- snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ta.bin", chip_name);
- err = request_firmware(&adev->psp.ta_fw, fw_name, adev->dev);
- if (err) {
- release_firmware(adev->psp.ta_fw);
- adev->psp.ta_fw = NULL;
- dev_info(adev->dev,
- "psp v11.0: Failed to load firmware \"%s\"\n", fw_name);
- } else {
- err = amdgpu_ucode_validate(adev->psp.ta_fw);
- if (err)
- goto out2;
-
- ta_hdr = (const struct ta_firmware_header_v1_0 *)adev->psp.ta_fw->data;
- adev->psp.ta_xgmi_ucode_version = le32_to_cpu(ta_hdr->ta_xgmi_ucode_version);
- adev->psp.ta_xgmi_ucode_size = le32_to_cpu(ta_hdr->ta_xgmi_size_bytes);
- adev->psp.ta_xgmi_start_addr = (uint8_t *)ta_hdr +
- le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes);
-
- adev->psp.ta_fw_version = le32_to_cpu(ta_hdr->header.ucode_version);
-
- adev->psp.ta_ras_ucode_version = le32_to_cpu(ta_hdr->ta_ras_ucode_version);
- adev->psp.ta_ras_ucode_size = le32_to_cpu(ta_hdr->ta_ras_size_bytes);
- adev->psp.ta_ras_start_addr = (uint8_t *)adev->psp.ta_xgmi_start_addr +
- le32_to_cpu(ta_hdr->ta_ras_offset_bytes);
+ switch (adev->asic_type) {
+ case CHIP_VEGA20:
+ snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ta.bin", chip_name);
+ err = request_firmware(&adev->psp.ta_fw, fw_name, adev->dev);
+ if (err) {
+ release_firmware(adev->psp.ta_fw);
+ adev->psp.ta_fw = NULL;
+ dev_info(adev->dev,
+ "psp v11.0: Failed to load firmware \"%s\"\n", fw_name);
+ } else {
+ err = amdgpu_ucode_validate(adev->psp.ta_fw);
+ if (err)
+ goto out2;
+
+ ta_hdr = (const struct ta_firmware_header_v1_0 *)adev->psp.ta_fw->data;
+ adev->psp.ta_xgmi_ucode_version = le32_to_cpu(ta_hdr->ta_xgmi_ucode_version);
+ adev->psp.ta_xgmi_ucode_size = le32_to_cpu(ta_hdr->ta_xgmi_size_bytes);
+ adev->psp.ta_xgmi_start_addr = (uint8_t *)ta_hdr +
+ le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes);
+ adev->psp.ta_fw_version = le32_to_cpu(ta_hdr->header.ucode_version);
+ adev->psp.ta_ras_ucode_version = le32_to_cpu(ta_hdr->ta_ras_ucode_version);
+ adev->psp.ta_ras_ucode_size = le32_to_cpu(ta_hdr->ta_ras_size_bytes);
+ adev->psp.ta_ras_start_addr = (uint8_t *)adev->psp.ta_xgmi_start_addr +
+ le32_to_cpu(ta_hdr->ta_ras_offset_bytes);
+ }
+ break;
+ case CHIP_NAVI10:
+ break;
+ default:
+ BUG();
}
return 0;
@@ -142,6 +180,48 @@ out:
return err;
}
+static int psp_v11_0_bootloader_load_kdb(struct psp_context *psp)
+{
+ int ret;
+ uint32_t psp_gfxdrv_command_reg = 0;
+ struct amdgpu_device *adev = psp->adev;
+ uint32_t sol_reg;
+
+ /* Check tOS sign of life register to confirm sys driver and sOS
+ * are already been loaded.
+ */
+ sol_reg = RREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_81);
+ if (sol_reg) {
+ psp->sos_fw_version = RREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_58);
+ dev_info(adev->dev, "sos fw version = 0x%x.\n", psp->sos_fw_version);
+ return 0;
+ }
+
+ /* Wait for bootloader to signify that is ready having bit 31 of C2PMSG_35 set to 1 */
+ ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_35),
+ 0x80000000, 0x80000000, false);
+ if (ret)
+ return ret;
+
+ memset(psp->fw_pri_buf, 0, PSP_1_MEG);
+
+ /* Copy PSP KDB binary to memory */
+ memcpy(psp->fw_pri_buf, psp->kdb_start_addr, psp->kdb_bin_size);
+
+ /* Provide the sys driver to bootloader */
+ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36,
+ (uint32_t)(psp->fw_pri_mc_addr >> 20));
+ psp_gfxdrv_command_reg = PSP_BL__LOAD_KEY_DATABASE;
+ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_35,
+ psp_gfxdrv_command_reg);
+
+ /* Wait for bootloader to signify that is ready having bit 31 of C2PMSG_35 set to 1*/
+ ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_35),
+ 0x80000000, 0x80000000, false);
+
+ return ret;
+}
+
static int psp_v11_0_bootloader_load_sysdrv(struct psp_context *psp)
{
int ret;
@@ -155,7 +235,7 @@ static int psp_v11_0_bootloader_load_sysdrv(struct psp_context *psp)
sol_reg = RREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_81);
if (sol_reg) {
psp->sos_fw_version = RREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_58);
- printk("sos fw version = 0x%x.\n", psp->sos_fw_version);
+ dev_info(adev->dev, "sos fw version = 0x%x.\n", psp->sos_fw_version);
return 0;
}
@@ -173,7 +253,7 @@ static int psp_v11_0_bootloader_load_sysdrv(struct psp_context *psp)
/* Provide the sys driver to bootloader */
WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36,
(uint32_t)(psp->fw_pri_mc_addr >> 20));
- psp_gfxdrv_command_reg = 1 << 16;
+ psp_gfxdrv_command_reg = PSP_BL__LOAD_SYSDRV;
WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_35,
psp_gfxdrv_command_reg);
@@ -214,7 +294,7 @@ static int psp_v11_0_bootloader_load_sos(struct psp_context *psp)
/* Provide the PSP secure OS to bootloader */
WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36,
(uint32_t)(psp->fw_pri_mc_addr >> 20));
- psp_gfxdrv_command_reg = 2 << 16;
+ psp_gfxdrv_command_reg = PSP_BL__LOAD_SOSDRV;
WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_35,
psp_gfxdrv_command_reg);
@@ -499,14 +579,24 @@ psp_v11_0_sram_map(struct amdgpu_device *adev,
case AMDGPU_UCODE_ID_RLC_G:
*sram_offset = 0x2000;
- *sram_addr_reg_offset = SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_UCODE_ADDR);
- *sram_data_reg_offset = SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_UCODE_DATA);
+ if (adev->asic_type < CHIP_NAVI10) {
+ *sram_addr_reg_offset = SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_UCODE_ADDR);
+ *sram_data_reg_offset = SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_UCODE_DATA);
+ } else {
+ *sram_addr_reg_offset = adev->reg_offset[GC_HWIP][0][1] + mmRLC_GPM_UCODE_ADDR_NV10;
+ *sram_data_reg_offset = adev->reg_offset[GC_HWIP][0][1] + mmRLC_GPM_UCODE_DATA_NV10;
+ }
break;
case AMDGPU_UCODE_ID_SDMA0:
*sram_offset = 0x0;
- *sram_addr_reg_offset = SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_UCODE_ADDR);
- *sram_data_reg_offset = SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_UCODE_DATA);
+ if (adev->asic_type < CHIP_NAVI10) {
+ *sram_addr_reg_offset = SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_UCODE_ADDR);
+ *sram_data_reg_offset = SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_UCODE_DATA);
+ } else {
+ *sram_addr_reg_offset = adev->reg_offset[GC_HWIP][0][1] + mmSDMA0_UCODE_ADDR_NV10;
+ *sram_data_reg_offset = adev->reg_offset[GC_HWIP][0][1] + mmSDMA0_UCODE_DATA_NV10;
+ }
break;
/* TODO: needs to confirm */
@@ -770,8 +860,14 @@ static int psp_v11_0_ras_cure_posion(struct psp_context *psp, uint64_t *mode_ptr
#endif
}
+static int psp_v11_0_rlc_autoload_start(struct psp_context *psp)
+{
+ return psp_rlc_autoload_start(psp);
+}
+
static const struct psp_funcs psp_v11_0_funcs = {
.init_microcode = psp_v11_0_init_microcode,
+ .bootloader_load_kdb = psp_v11_0_bootloader_load_kdb,
.bootloader_load_sysdrv = psp_v11_0_bootloader_load_sysdrv,
.bootloader_load_sos = psp_v11_0_bootloader_load_sos,
.ring_init = psp_v11_0_ring_init,
@@ -788,6 +884,7 @@ static const struct psp_funcs psp_v11_0_funcs = {
.support_vmr_ring = psp_v11_0_support_vmr_ring,
.ras_trigger_error = psp_v11_0_ras_trigger_error,
.ras_cure_posion = psp_v11_0_ras_cure_posion,
+ .rlc_autoload_start = psp_v11_0_rlc_autoload_start,
};
void psp_v11_0_set_psp_funcs(struct psp_context *psp)
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
index 143f0fae69d5..019c47feee42 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
@@ -24,7 +24,9 @@
*/
#include <linux/firmware.h>
-#include <drm/drmP.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
#include "amdgpu.h"
#include "amdgpu_psp.h"
#include "amdgpu_ucode.h"
@@ -50,6 +52,10 @@ MODULE_FIRMWARE("amdgpu/vega12_asd.bin");
static uint32_t sos_old_versions[] = {1517616, 1510592, 1448594, 1446554};
+static bool psp_v3_1_support_vmr_ring(struct psp_context *psp);
+static int psp_v3_1_ring_stop(struct psp_context *psp,
+ enum psp_ring_type ring_type);
+
static int psp_v3_1_init_microcode(struct psp_context *psp)
{
struct amdgpu_device *adev = psp->adev;
@@ -149,7 +155,7 @@ static int psp_v3_1_bootloader_load_sysdrv(struct psp_context *psp)
/* Provide the sys driver to bootloader */
WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36,
(uint32_t)(psp->fw_pri_mc_addr >> 20));
- psp_gfxdrv_command_reg = 1 << 16;
+ psp_gfxdrv_command_reg = PSP_BL__LOAD_SYSDRV;
WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_35,
psp_gfxdrv_command_reg);
@@ -212,7 +218,7 @@ static int psp_v3_1_bootloader_load_sos(struct psp_context *psp)
/* Provide the PSP secure OS to bootloader */
WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36,
(uint32_t)(psp->fw_pri_mc_addr >> 20));
- psp_gfxdrv_command_reg = 2 << 16;
+ psp_gfxdrv_command_reg = PSP_BL__LOAD_SOSDRV;
WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_35,
psp_gfxdrv_command_reg);
@@ -296,27 +302,57 @@ static int psp_v3_1_ring_create(struct psp_context *psp,
psp_v3_1_reroute_ih(psp);
- /* Write low address of the ring to C2PMSG_69 */
- psp_ring_reg = lower_32_bits(ring->ring_mem_mc_addr);
- WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_69, psp_ring_reg);
- /* Write high address of the ring to C2PMSG_70 */
- psp_ring_reg = upper_32_bits(ring->ring_mem_mc_addr);
- WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_70, psp_ring_reg);
- /* Write size of ring to C2PMSG_71 */
- psp_ring_reg = ring->ring_size;
- WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_71, psp_ring_reg);
- /* Write the ring initialization command to C2PMSG_64 */
- psp_ring_reg = ring_type;
- psp_ring_reg = psp_ring_reg << 16;
- WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_64, psp_ring_reg);
-
- /* there might be handshake issue with hardware which needs delay */
- mdelay(20);
-
- /* Wait for response flag (bit 31) in C2PMSG_64 */
- ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64),
- 0x80000000, 0x8000FFFF, false);
+ if (psp_v3_1_support_vmr_ring(psp)) {
+ ret = psp_v3_1_ring_stop(psp, ring_type);
+ if (ret) {
+ DRM_ERROR("psp_v3_1_ring_stop_sriov failed!\n");
+ return ret;
+ }
+
+ /* Write low address of the ring to C2PMSG_102 */
+ psp_ring_reg = lower_32_bits(ring->ring_mem_mc_addr);
+ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_102, psp_ring_reg);
+ /* Write high address of the ring to C2PMSG_103 */
+ psp_ring_reg = upper_32_bits(ring->ring_mem_mc_addr);
+ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_103, psp_ring_reg);
+ /* No size initialization for sriov */
+ /* Write the ring initialization command to C2PMSG_101 */
+ psp_ring_reg = ring_type;
+ psp_ring_reg = psp_ring_reg << 16;
+ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_101, psp_ring_reg);
+
+ /* there might be hardware handshake issue which needs delay */
+ mdelay(20);
+
+ /* Wait for response flag (bit 31) in C2PMSG_101 */
+ ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0,
+ mmMP0_SMN_C2PMSG_101), 0x80000000,
+ 0x8000FFFF, false);
+ } else {
+
+ /* Write low address of the ring to C2PMSG_69 */
+ psp_ring_reg = lower_32_bits(ring->ring_mem_mc_addr);
+ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_69, psp_ring_reg);
+ /* Write high address of the ring to C2PMSG_70 */
+ psp_ring_reg = upper_32_bits(ring->ring_mem_mc_addr);
+ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_70, psp_ring_reg);
+ /* Write size of ring to C2PMSG_71 */
+ psp_ring_reg = ring->ring_size;
+ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_71, psp_ring_reg);
+ /* Write the ring initialization command to C2PMSG_64 */
+ psp_ring_reg = ring_type;
+ psp_ring_reg = psp_ring_reg << 16;
+ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_64, psp_ring_reg);
+
+ /* there might be hardware handshake issue which needs delay */
+ mdelay(20);
+
+ /* Wait for response flag (bit 31) in C2PMSG_64 */
+ ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0,
+ mmMP0_SMN_C2PMSG_64), 0x80000000,
+ 0x8000FFFF, false);
+ }
return ret;
}
@@ -327,16 +363,31 @@ static int psp_v3_1_ring_stop(struct psp_context *psp,
unsigned int psp_ring_reg = 0;
struct amdgpu_device *adev = psp->adev;
- /* Write the ring destroy command to C2PMSG_64 */
- psp_ring_reg = 3 << 16;
- WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_64, psp_ring_reg);
-
- /* there might be handshake issue with hardware which needs delay */
- mdelay(20);
-
- /* Wait for response flag (bit 31) in C2PMSG_64 */
- ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64),
- 0x80000000, 0x80000000, false);
+ if (psp_v3_1_support_vmr_ring(psp)) {
+ /* Write the Destroy GPCOM ring command to C2PMSG_101 */
+ psp_ring_reg = GFX_CTRL_CMD_ID_DESTROY_GPCOM_RING;
+ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_101, psp_ring_reg);
+
+ /* there might be handshake issue which needs delay */
+ mdelay(20);
+
+ /* Wait for response flag (bit 31) in C2PMSG_101 */
+ ret = psp_wait_for(psp,
+ SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_101),
+ 0x80000000, 0x80000000, false);
+ } else {
+ /* Write the ring destroy command to C2PMSG_64 */
+ psp_ring_reg = 3 << 16;
+ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_64, psp_ring_reg);
+
+ /* there might be handshake issue which needs delay */
+ mdelay(20);
+
+ /* Wait for response flag (bit 31) in C2PMSG_64 */
+ ret = psp_wait_for(psp,
+ SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64),
+ 0x80000000, 0x80000000, false);
+ }
return ret;
}
@@ -375,7 +426,10 @@ static int psp_v3_1_cmd_submit(struct psp_context *psp,
uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4;
/* KM (GPCOM) prepare write pointer */
- psp_write_ptr_reg = RREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_67);
+ if (psp_v3_1_support_vmr_ring(psp))
+ psp_write_ptr_reg = RREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_102);
+ else
+ psp_write_ptr_reg = RREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_67);
/* Update KM RB frame pointer to new frame */
/* write_frame ptr increments by size of rb_frame in bytes */
@@ -404,7 +458,13 @@ static int psp_v3_1_cmd_submit(struct psp_context *psp,
/* Update the write Pointer in DWORDs */
psp_write_ptr_reg = (psp_write_ptr_reg + rb_frame_size_dw) % ring_size_dw;
- WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_67, psp_write_ptr_reg);
+ if (psp_v3_1_support_vmr_ring(psp)) {
+ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_102, psp_write_ptr_reg);
+ /* send interrupt to PSP for SRIOV ring write pointer update */
+ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_101,
+ GFX_CTRL_CMD_ID_CONSUME_CMD);
+ } else
+ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_67, psp_write_ptr_reg);
return 0;
}
@@ -574,6 +634,14 @@ static int psp_v3_1_mode1_reset(struct psp_context *psp)
return 0;
}
+static bool psp_v3_1_support_vmr_ring(struct psp_context *psp)
+{
+ if (amdgpu_sriov_vf(psp->adev) && psp->sos_fw_version >= 0x80455)
+ return true;
+
+ return false;
+}
+
static const struct psp_funcs psp_v3_1_funcs = {
.init_microcode = psp_v3_1_init_microcode,
.bootloader_load_sysdrv = psp_v3_1_bootloader_load_sysdrv,
@@ -586,6 +654,7 @@ static const struct psp_funcs psp_v3_1_funcs = {
.compare_sram_data = psp_v3_1_compare_sram_data,
.smu_reload_quirk = psp_v3_1_smu_reload_quirk,
.mode1_reset = psp_v3_1_mode1_reset,
+ .support_vmr_ring = psp_v3_1_support_vmr_ring,
};
void psp_v3_1_set_psp_funcs(struct psp_context *psp)
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index 36196372e8db..a10175838013 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -21,8 +21,11 @@
*
* Authors: Alex Deucher
*/
+
+#include <linux/delay.h>
#include <linux/firmware.h>
-#include <drm/drmP.h>
+#include <linux/module.h>
+
#include "amdgpu.h"
#include "amdgpu_ucode.h"
#include "amdgpu_trace.h"
@@ -574,7 +577,7 @@ static int sdma_v2_4_ring_test_ring(struct amdgpu_ring *ring)
tmp = le32_to_cpu(adev->wb.wb[index]);
if (tmp == 0xDEADBEEF)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 6d39544e7829..5f4e2c616241 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -21,8 +21,11 @@
*
* Authors: Alex Deucher
*/
+
+#include <linux/delay.h>
#include <linux/firmware.h>
-#include <drm/drmP.h>
+#include <linux/module.h>
+
#include "amdgpu.h"
#include "amdgpu_ucode.h"
#include "amdgpu_trace.h"
@@ -846,7 +849,7 @@ static int sdma_v3_0_ring_test_ring(struct amdgpu_ring *ring)
tmp = le32_to_cpu(adev->wb.wb[index]);
if (tmp == 0xDEADBEEF)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index 9c88ce513d78..4428018672d3 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -21,8 +21,11 @@
*
*/
+#include <linux/delay.h>
#include <linux/firmware.h>
-#include <drm/drmP.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
#include "amdgpu.h"
#include "amdgpu_ucode.h"
#include "amdgpu_trace.h"
@@ -210,12 +213,14 @@ static void sdma_v4_0_init_golden_registers(struct amdgpu_device *adev)
{
switch (adev->asic_type) {
case CHIP_VEGA10:
- soc15_program_register_sequence(adev,
- golden_settings_sdma_4,
- ARRAY_SIZE(golden_settings_sdma_4));
- soc15_program_register_sequence(adev,
- golden_settings_sdma_vg10,
- ARRAY_SIZE(golden_settings_sdma_vg10));
+ if (!amdgpu_virt_support_skip_setting(adev)) {
+ soc15_program_register_sequence(adev,
+ golden_settings_sdma_4,
+ ARRAY_SIZE(golden_settings_sdma_4));
+ soc15_program_register_sequence(adev,
+ golden_settings_sdma_vg10,
+ ARRAY_SIZE(golden_settings_sdma_vg10));
+ }
break;
case CHIP_VEGA12:
soc15_program_register_sequence(adev,
@@ -1090,7 +1095,7 @@ static int sdma_v4_0_load_microcode(struct amdgpu_device *adev)
static int sdma_v4_0_start(struct amdgpu_device *adev)
{
struct amdgpu_ring *ring;
- int i, r;
+ int i, r = 0;
if (amdgpu_sriov_vf(adev)) {
sdma_v4_0_ctx_switch_enable(adev, false);
@@ -1207,7 +1212,7 @@ static int sdma_v4_0_ring_test_ring(struct amdgpu_ring *ring)
tmp = le32_to_cpu(adev->wb.wb[index]);
if (tmp == 0xDEADBEEF)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
@@ -1521,8 +1526,25 @@ static int sdma_v4_0_late_init(void *handle)
}
/* handle resume path. */
- if (*ras_if)
+ if (*ras_if) {
+ /* resend ras TA enable cmd during resume.
+ * prepare to handle failure.
+ */
+ ih_info.head = **ras_if;
+ r = amdgpu_ras_feature_enable_on_boot(adev, *ras_if, 1);
+ if (r) {
+ if (r == -EAGAIN) {
+ /* request a gpu reset. will run again. */
+ amdgpu_ras_request_reset_on_boot(adev,
+ AMDGPU_RAS_BLOCK__SDMA);
+ return 0;
+ }
+ /* fail to enable ras, cleanup all. */
+ goto irq;
+ }
+ /* enable successfully. continue. */
goto resume;
+ }
*ras_if = kmalloc(sizeof(**ras_if), GFP_KERNEL);
if (!*ras_if)
@@ -1531,8 +1553,14 @@ static int sdma_v4_0_late_init(void *handle)
**ras_if = ras_block;
r = amdgpu_ras_feature_enable_on_boot(adev, *ras_if, 1);
- if (r)
+ if (r) {
+ if (r == -EAGAIN) {
+ amdgpu_ras_request_reset_on_boot(adev,
+ AMDGPU_RAS_BLOCK__SDMA);
+ r = 0;
+ }
goto feature;
+ }
ih_info.head = **ras_if;
fs_info.head = **ras_if;
@@ -1541,9 +1569,7 @@ static int sdma_v4_0_late_init(void *handle)
if (r)
goto interrupt;
- r = amdgpu_ras_debugfs_create(adev, &fs_info);
- if (r)
- goto debugfs;
+ amdgpu_ras_debugfs_create(adev, &fs_info);
r = amdgpu_ras_sysfs_create(adev, &fs_info);
if (r)
@@ -1564,14 +1590,13 @@ irq:
amdgpu_ras_sysfs_remove(adev, *ras_if);
sysfs:
amdgpu_ras_debugfs_remove(adev, *ras_if);
-debugfs:
amdgpu_ras_interrupt_remove_handler(adev, &ih_info);
interrupt:
amdgpu_ras_feature_enable(adev, *ras_if, 0);
feature:
kfree(*ras_if);
*ras_if = NULL;
- return -EINVAL;
+ return r;
}
static int sdma_v4_0_sw_init(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
new file mode 100644
index 000000000000..3747c3f1f0cc
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
@@ -0,0 +1,1687 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/firmware.h>
+#include <drm/drmP.h>
+#include "amdgpu.h"
+#include "amdgpu_ucode.h"
+#include "amdgpu_trace.h"
+
+#include "gc/gc_10_1_0_offset.h"
+#include "gc/gc_10_1_0_sh_mask.h"
+#include "hdp/hdp_5_0_0_offset.h"
+#include "ivsrcid/sdma0/irqsrcs_sdma0_5_0.h"
+#include "ivsrcid/sdma1/irqsrcs_sdma1_5_0.h"
+
+#include "soc15_common.h"
+#include "soc15.h"
+#include "navi10_sdma_pkt_open.h"
+#include "nbio_v2_3.h"
+#include "sdma_v5_0.h"
+
+MODULE_FIRMWARE("amdgpu/navi10_sdma.bin");
+MODULE_FIRMWARE("amdgpu/navi10_sdma1.bin");
+
+#define SDMA1_REG_OFFSET 0x600
+#define SDMA0_HYP_DEC_REG_START 0x5880
+#define SDMA0_HYP_DEC_REG_END 0x5893
+#define SDMA1_HYP_DEC_REG_OFFSET 0x20
+
+static void sdma_v5_0_set_ring_funcs(struct amdgpu_device *adev);
+static void sdma_v5_0_set_buffer_funcs(struct amdgpu_device *adev);
+static void sdma_v5_0_set_vm_pte_funcs(struct amdgpu_device *adev);
+static void sdma_v5_0_set_irq_funcs(struct amdgpu_device *adev);
+
+static const struct soc15_reg_golden golden_settings_sdma_5[] = {
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_CHICKEN_BITS, 0xffbf1f0f, 0x03ab0107),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC2_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC3_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC4_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC5_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC6_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC7_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_UTCL1_PAGE, 0x00ffffff, 0x000c5c00),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_CHICKEN_BITS, 0xffbf1f0f, 0x03ab0107),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_PAGE_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC2_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC3_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC4_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC5_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC6_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC7_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+ SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_UTCL1_PAGE, 0x00ffffff, 0x000c5c00)
+};
+
+static const struct soc15_reg_golden golden_settings_sdma_nv10[] = {
+};
+
+static u32 sdma_v5_0_get_reg_offset(struct amdgpu_device *adev, u32 instance, u32 internal_offset)
+{
+ u32 base;
+
+ if (internal_offset >= SDMA0_HYP_DEC_REG_START &&
+ internal_offset <= SDMA0_HYP_DEC_REG_END) {
+ base = adev->reg_offset[GC_HWIP][0][1];
+ if (instance == 1)
+ internal_offset += SDMA1_HYP_DEC_REG_OFFSET;
+ } else {
+ base = adev->reg_offset[GC_HWIP][0][0];
+ if (instance == 1)
+ internal_offset += SDMA1_REG_OFFSET;
+ }
+
+ return base + internal_offset;
+}
+
+static void sdma_v5_0_init_golden_registers(struct amdgpu_device *adev)
+{
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ soc15_program_register_sequence(adev,
+ golden_settings_sdma_5,
+ (const u32)ARRAY_SIZE(golden_settings_sdma_5));
+ soc15_program_register_sequence(adev,
+ golden_settings_sdma_nv10,
+ (const u32)ARRAY_SIZE(golden_settings_sdma_nv10));
+ break;
+ default:
+ break;
+ }
+}
+
+/**
+ * sdma_v5_0_init_microcode - load ucode images from disk
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Use the firmware interface to load the ucode images into
+ * the driver (not loaded into hw).
+ * Returns 0 on success, error on failure.
+ */
+
+// emulation only, won't work on real chip
+// navi10 real chip need to use PSP to load firmware
+static int sdma_v5_0_init_microcode(struct amdgpu_device *adev)
+{
+ const char *chip_name;
+ char fw_name[30];
+ int err = 0, i;
+ struct amdgpu_firmware_info *info = NULL;
+ const struct common_firmware_header *header = NULL;
+ const struct sdma_firmware_header_v1_0 *hdr;
+
+ DRM_DEBUG("\n");
+
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ chip_name = "navi10";
+ break;
+ default:
+ BUG();
+ }
+
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ if (i == 0)
+ snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma.bin", chip_name);
+ else
+ snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma1.bin", chip_name);
+ err = request_firmware(&adev->sdma.instance[i].fw, fw_name, adev->dev);
+ if (err)
+ goto out;
+ err = amdgpu_ucode_validate(adev->sdma.instance[i].fw);
+ if (err)
+ goto out;
+ hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
+ adev->sdma.instance[i].fw_version = le32_to_cpu(hdr->header.ucode_version);
+ adev->sdma.instance[i].feature_version = le32_to_cpu(hdr->ucode_feature_version);
+ if (adev->sdma.instance[i].feature_version >= 20)
+ adev->sdma.instance[i].burst_nop = true;
+ DRM_DEBUG("psp_load == '%s'\n",
+ adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false");
+
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+ info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];
+ info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i;
+ info->fw = adev->sdma.instance[i].fw;
+ header = (const struct common_firmware_header *)info->fw->data;
+ adev->firmware.fw_size +=
+ ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
+ }
+ }
+out:
+ if (err) {
+ DRM_ERROR("sdma_v5_0: Failed to load firmware \"%s\"\n", fw_name);
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ release_firmware(adev->sdma.instance[i].fw);
+ adev->sdma.instance[i].fw = NULL;
+ }
+ }
+ return err;
+}
+
+static unsigned sdma_v5_0_ring_init_cond_exec(struct amdgpu_ring *ring)
+{
+ unsigned ret;
+
+ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_COND_EXE));
+ amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
+ amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
+ amdgpu_ring_write(ring, 1);
+ ret = ring->wptr & ring->buf_mask;/* this is the offset we need patch later */
+ amdgpu_ring_write(ring, 0x55aa55aa);/* insert dummy here and patch it later */
+
+ return ret;
+}
+
+static void sdma_v5_0_ring_patch_cond_exec(struct amdgpu_ring *ring,
+ unsigned offset)
+{
+ unsigned cur;
+
+ BUG_ON(offset > ring->buf_mask);
+ BUG_ON(ring->ring[offset] != 0x55aa55aa);
+
+ cur = (ring->wptr - 1) & ring->buf_mask;
+ if (cur > offset)
+ ring->ring[offset] = cur - offset;
+ else
+ ring->ring[offset] = (ring->buf_mask + 1) - offset + cur;
+}
+
+/**
+ * sdma_v5_0_ring_get_rptr - get the current read pointer
+ *
+ * @ring: amdgpu ring pointer
+ *
+ * Get the current rptr from the hardware (NAVI10+).
+ */
+static uint64_t sdma_v5_0_ring_get_rptr(struct amdgpu_ring *ring)
+{
+ u64 *rptr;
+
+ /* XXX check if swapping is necessary on BE */
+ rptr = ((u64 *)&ring->adev->wb.wb[ring->rptr_offs]);
+
+ DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr);
+ return ((*rptr) >> 2);
+}
+
+/**
+ * sdma_v5_0_ring_get_wptr - get the current write pointer
+ *
+ * @ring: amdgpu ring pointer
+ *
+ * Get the current wptr from the hardware (NAVI10+).
+ */
+static uint64_t sdma_v5_0_ring_get_wptr(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+ u64 *wptr = NULL;
+ uint64_t local_wptr = 0;
+
+ if (ring->use_doorbell) {
+ /* XXX check if swapping is necessary on BE */
+ wptr = ((u64 *)&adev->wb.wb[ring->wptr_offs]);
+ DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", *wptr);
+ *wptr = (*wptr) >> 2;
+ DRM_DEBUG("wptr/doorbell after shift == 0x%016llx\n", *wptr);
+ } else {
+ u32 lowbit, highbit;
+
+ wptr = &local_wptr;
+ lowbit = RREG32(sdma_v5_0_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR)) >> 2;
+ highbit = RREG32(sdma_v5_0_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI)) >> 2;
+
+ DRM_DEBUG("wptr [%i]high== 0x%08x low==0x%08x\n",
+ ring->me, highbit, lowbit);
+ *wptr = highbit;
+ *wptr = (*wptr) << 32;
+ *wptr |= lowbit;
+ }
+
+ return *wptr;
+}
+
+/**
+ * sdma_v5_0_ring_set_wptr - commit the write pointer
+ *
+ * @ring: amdgpu ring pointer
+ *
+ * Write the wptr back to the hardware (NAVI10+).
+ */
+static void sdma_v5_0_ring_set_wptr(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ DRM_DEBUG("Setting write pointer\n");
+ if (ring->use_doorbell) {
+ DRM_DEBUG("Using doorbell -- "
+ "wptr_offs == 0x%08x "
+ "lower_32_bits(ring->wptr) << 2 == 0x%08x "
+ "upper_32_bits(ring->wptr) << 2 == 0x%08x\n",
+ ring->wptr_offs,
+ lower_32_bits(ring->wptr << 2),
+ upper_32_bits(ring->wptr << 2));
+ /* XXX check if swapping is necessary on BE */
+ adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr << 2);
+ adev->wb.wb[ring->wptr_offs + 1] = upper_32_bits(ring->wptr << 2);
+ DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
+ ring->doorbell_index, ring->wptr << 2);
+ WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
+ } else {
+ DRM_DEBUG("Not using doorbell -- "
+ "mmSDMA%i_GFX_RB_WPTR == 0x%08x "
+ "mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
+ ring->me,
+ lower_32_bits(ring->wptr << 2),
+ ring->me,
+ upper_32_bits(ring->wptr << 2));
+ WREG32(sdma_v5_0_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR),
+ lower_32_bits(ring->wptr << 2));
+ WREG32(sdma_v5_0_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI),
+ upper_32_bits(ring->wptr << 2));
+ }
+}
+
+static void sdma_v5_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
+{
+ struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
+ int i;
+
+ for (i = 0; i < count; i++)
+ if (sdma && sdma->burst_nop && (i == 0))
+ amdgpu_ring_write(ring, ring->funcs->nop |
+ SDMA_PKT_NOP_HEADER_COUNT(count - 1));
+ else
+ amdgpu_ring_write(ring, ring->funcs->nop);
+}
+
+/**
+ * sdma_v5_0_ring_emit_ib - Schedule an IB on the DMA engine
+ *
+ * @ring: amdgpu ring pointer
+ * @ib: IB object to schedule
+ *
+ * Schedule an IB in the DMA ring (NAVI10).
+ */
+static void sdma_v5_0_ring_emit_ib(struct amdgpu_ring *ring,
+ struct amdgpu_job *job,
+ struct amdgpu_ib *ib,
+ uint32_t flags)
+{
+ unsigned vmid = AMDGPU_JOB_GET_VMID(job);
+ uint64_t csa_mc_addr = amdgpu_sdma_get_csa_mc_addr(ring, vmid);
+
+ /* IB packet must end on a 8 DW boundary */
+ sdma_v5_0_ring_insert_nop(ring, (10 - (lower_32_bits(ring->wptr) & 7)) % 8);
+
+ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |
+ SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
+ /* base must be 32 byte aligned */
+ amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
+ amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
+ amdgpu_ring_write(ring, ib->length_dw);
+ amdgpu_ring_write(ring, lower_32_bits(csa_mc_addr));
+ amdgpu_ring_write(ring, upper_32_bits(csa_mc_addr));
+}
+
+/**
+ * sdma_v5_0_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
+ *
+ * @ring: amdgpu ring pointer
+ *
+ * Emit an hdp flush packet on the requested DMA ring.
+ */
+static void sdma_v5_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+ u32 ref_and_mask = 0;
+ const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio_funcs->hdp_flush_reg;
+
+ if (ring->me == 0)
+ ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0;
+ else
+ ref_and_mask = nbio_hf_reg->ref_and_mask_sdma1;
+
+ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
+ SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(1) |
+ SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
+ amdgpu_ring_write(ring, (adev->nbio_funcs->get_hdp_flush_done_offset(adev)) << 2);
+ amdgpu_ring_write(ring, (adev->nbio_funcs->get_hdp_flush_req_offset(adev)) << 2);
+ amdgpu_ring_write(ring, ref_and_mask); /* reference */
+ amdgpu_ring_write(ring, ref_and_mask); /* mask */
+ amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
+ SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */
+}
+
+/**
+ * sdma_v5_0_ring_emit_fence - emit a fence on the DMA ring
+ *
+ * @ring: amdgpu ring pointer
+ * @fence: amdgpu fence object
+ *
+ * Add a DMA fence packet to the ring to write
+ * the fence seq number and DMA trap packet to generate
+ * an interrupt if needed (NAVI10).
+ */
+static void sdma_v5_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
+ unsigned flags)
+{
+ struct amdgpu_device *adev = ring->adev;
+ bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
+ /* write the fence */
+ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) |
+ SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */
+ /* zero in first two bits */
+ BUG_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, lower_32_bits(seq));
+
+ /* optionally write high bits as well */
+ if (write64bit) {
+ addr += 4;
+ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) |
+ SDMA_PKT_FENCE_HEADER_MTYPE(0x3));
+ /* zero in first two bits */
+ BUG_ON(addr & 0x3);
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, upper_32_bits(seq));
+ }
+
+ /* Interrupt not work fine on GFX10.1 model yet. Use fallback instead */
+ if ((flags & AMDGPU_FENCE_FLAG_INT) && adev->pdev->device != 0x50) {
+ /* generate an interrupt */
+ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_TRAP));
+ amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0));
+ }
+}
+
+
+/**
+ * sdma_v5_0_gfx_stop - stop the gfx async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Stop the gfx async dma ring buffers (NAVI10).
+ */
+static void sdma_v5_0_gfx_stop(struct amdgpu_device *adev)
+{
+ struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring;
+ struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring;
+ u32 rb_cntl, ib_cntl;
+ int i;
+
+ if ((adev->mman.buffer_funcs_ring == sdma0) ||
+ (adev->mman.buffer_funcs_ring == sdma1))
+ amdgpu_ttm_set_buffer_funcs_status(adev, false);
+
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ rb_cntl = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL));
+ rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 0);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
+ ib_cntl = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL));
+ ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 0);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL), ib_cntl);
+ }
+
+ sdma0->sched.ready = false;
+ sdma1->sched.ready = false;
+}
+
+/**
+ * sdma_v5_0_rlc_stop - stop the compute async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Stop the compute async dma queues (NAVI10).
+ */
+static void sdma_v5_0_rlc_stop(struct amdgpu_device *adev)
+{
+ /* XXX todo */
+}
+
+/**
+ * sdma_v_0_ctx_switch_enable - stop the async dma engines context switch
+ *
+ * @adev: amdgpu_device pointer
+ * @enable: enable/disable the DMA MEs context switch.
+ *
+ * Halt or unhalt the async dma engines context switch (NAVI10).
+ */
+static void sdma_v5_0_ctx_switch_enable(struct amdgpu_device *adev, bool enable)
+{
+ u32 f32_cntl, phase_quantum = 0;
+ int i;
+
+ if (amdgpu_sdma_phase_quantum) {
+ unsigned value = amdgpu_sdma_phase_quantum;
+ unsigned unit = 0;
+
+ while (value > (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
+ SDMA0_PHASE0_QUANTUM__VALUE__SHIFT)) {
+ value = (value + 1) >> 1;
+ unit++;
+ }
+ if (unit > (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
+ SDMA0_PHASE0_QUANTUM__UNIT__SHIFT)) {
+ value = (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
+ SDMA0_PHASE0_QUANTUM__VALUE__SHIFT);
+ unit = (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
+ SDMA0_PHASE0_QUANTUM__UNIT__SHIFT);
+ WARN_ONCE(1,
+ "clamping sdma_phase_quantum to %uK clock cycles\n",
+ value << unit);
+ }
+ phase_quantum =
+ value << SDMA0_PHASE0_QUANTUM__VALUE__SHIFT |
+ unit << SDMA0_PHASE0_QUANTUM__UNIT__SHIFT;
+ }
+
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ f32_cntl = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_CNTL));
+ f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
+ AUTO_CTXSW_ENABLE, enable ? 1 : 0);
+ if (enable && amdgpu_sdma_phase_quantum) {
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_PHASE0_QUANTUM),
+ phase_quantum);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_PHASE1_QUANTUM),
+ phase_quantum);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_PHASE2_QUANTUM),
+ phase_quantum);
+ }
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_CNTL), f32_cntl);
+ }
+
+}
+
+/**
+ * sdma_v5_0_enable - stop the async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ * @enable: enable/disable the DMA MEs.
+ *
+ * Halt or unhalt the async dma engines (NAVI10).
+ */
+static void sdma_v5_0_enable(struct amdgpu_device *adev, bool enable)
+{
+ u32 f32_cntl;
+ int i;
+
+ if (enable == false) {
+ sdma_v5_0_gfx_stop(adev);
+ sdma_v5_0_rlc_stop(adev);
+ }
+
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ f32_cntl = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_F32_CNTL));
+ f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, enable ? 0 : 1);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_F32_CNTL), f32_cntl);
+ }
+}
+
+/**
+ * sdma_v5_0_gfx_resume - setup and start the async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Set up the gfx DMA ring buffers and enable them (NAVI10).
+ * Returns 0 for success, error for failure.
+ */
+static int sdma_v5_0_gfx_resume(struct amdgpu_device *adev)
+{
+ struct amdgpu_ring *ring;
+ u32 rb_cntl, ib_cntl;
+ u32 rb_bufsz;
+ u32 wb_offset;
+ u32 doorbell;
+ u32 doorbell_offset;
+ u32 temp;
+ u32 wptr_poll_cntl;
+ u64 wptr_gpu_addr;
+ int i, r;
+
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ ring = &adev->sdma.instance[i].ring;
+ wb_offset = (ring->rptr_offs * 4);
+
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL), 0);
+
+ /* Set ring buffer size in dwords */
+ rb_bufsz = order_base_2(ring->ring_size / 4);
+ rb_cntl = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL));
+ rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SIZE, rb_bufsz);
+#ifdef __BIG_ENDIAN
+ rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SWAP_ENABLE, 1);
+ rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL,
+ RPTR_WRITEBACK_SWAP_ENABLE, 1);
+#endif
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
+
+ /* Initialize the ring buffer's read and write pointers */
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR), 0);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_HI), 0);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR), 0);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_HI), 0);
+
+ /* setup the wptr shadow polling */
+ wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO),
+ lower_32_bits(wptr_gpu_addr));
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI),
+ upper_32_bits(wptr_gpu_addr));
+ wptr_poll_cntl = RREG32(sdma_v5_0_get_reg_offset(adev, i,
+ mmSDMA0_GFX_RB_WPTR_POLL_CNTL));
+ wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
+ SDMA0_GFX_RB_WPTR_POLL_CNTL,
+ F32_POLL_ENABLE, 1);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL),
+ wptr_poll_cntl);
+
+ /* set the wb address whether it's enabled or not */
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_ADDR_HI),
+ upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_ADDR_LO),
+ lower_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC);
+
+ rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1);
+
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_BASE), ring->gpu_addr >> 8);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_BASE_HI), ring->gpu_addr >> 40);
+
+ ring->wptr = 0;
+
+ /* before programing wptr to a less value, need set minor_ptr_update first */
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_MINOR_PTR_UPDATE), 1);
+
+ if (!amdgpu_sriov_vf(adev)) { /* only bare-metal use register write for wptr */
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR), lower_32_bits(ring->wptr) << 2);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_HI), upper_32_bits(ring->wptr) << 2);
+ }
+
+ doorbell = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL));
+ doorbell_offset = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL_OFFSET));
+
+ if (ring->use_doorbell) {
+ doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 1);
+ doorbell_offset = REG_SET_FIELD(doorbell_offset, SDMA0_GFX_DOORBELL_OFFSET,
+ OFFSET, ring->doorbell_index);
+ } else {
+ doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 0);
+ }
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL), doorbell);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL_OFFSET), doorbell_offset);
+
+ adev->nbio_funcs->sdma_doorbell_range(adev, i, ring->use_doorbell,
+ ring->doorbell_index, 20);
+
+ if (amdgpu_sriov_vf(adev))
+ sdma_v5_0_ring_set_wptr(ring);
+
+ /* set minor_ptr_update to 0 after wptr programed */
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_MINOR_PTR_UPDATE), 0);
+
+ /* set utc l1 enable flag always to 1 */
+ temp = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_CNTL));
+ temp = REG_SET_FIELD(temp, SDMA0_CNTL, UTC_L1_ENABLE, 1);
+
+ /* enable MCBP */
+ temp = REG_SET_FIELD(temp, SDMA0_CNTL, MIDCMD_PREEMPT_ENABLE, 1);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_CNTL), temp);
+
+ /* Set up RESP_MODE to non-copy addresses */
+ temp = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_UTCL1_CNTL));
+ temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, RESP_MODE, 3);
+ temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, REDO_DELAY, 9);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_UTCL1_CNTL), temp);
+
+ /* program default cache read and write policy */
+ temp = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_UTCL1_PAGE));
+ /* clean read policy and write policy bits */
+ temp &= 0xFF0FFF;
+ temp |= ((CACHE_READ_POLICY_L2__DEFAULT << 12) | (CACHE_WRITE_POLICY_L2__DEFAULT << 14));
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_UTCL1_PAGE), temp);
+
+ if (!amdgpu_sriov_vf(adev)) {
+ /* unhalt engine */
+ temp = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_F32_CNTL));
+ temp = REG_SET_FIELD(temp, SDMA0_F32_CNTL, HALT, 0);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_F32_CNTL), temp);
+ }
+
+ /* enable DMA RB */
+ rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 1);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
+
+ ib_cntl = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL));
+ ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 1);
+#ifdef __BIG_ENDIAN
+ ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_SWAP_ENABLE, 1);
+#endif
+ /* enable DMA IBs */
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL), ib_cntl);
+
+ ring->sched.ready = true;
+
+ if (amdgpu_sriov_vf(adev)) { /* bare-metal sequence doesn't need below to lines */
+ sdma_v5_0_ctx_switch_enable(adev, true);
+ sdma_v5_0_enable(adev, true);
+ }
+
+ r = amdgpu_ring_test_ring(ring);
+ if (r) {
+ ring->sched.ready = false;
+ return r;
+ }
+
+ if (adev->mman.buffer_funcs_ring == ring)
+ amdgpu_ttm_set_buffer_funcs_status(adev, true);
+ }
+
+ return 0;
+}
+
+/**
+ * sdma_v5_0_rlc_resume - setup and start the async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Set up the compute DMA queues and enable them (NAVI10).
+ * Returns 0 for success, error for failure.
+ */
+static int sdma_v5_0_rlc_resume(struct amdgpu_device *adev)
+{
+ return 0;
+}
+
+/**
+ * sdma_v5_0_load_microcode - load the sDMA ME ucode
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Loads the sDMA0/1 ucode.
+ * Returns 0 for success, -EINVAL if the ucode is not available.
+ */
+static int sdma_v5_0_load_microcode(struct amdgpu_device *adev)
+{
+ const struct sdma_firmware_header_v1_0 *hdr;
+ const __le32 *fw_data;
+ u32 fw_size;
+ int i, j;
+
+ /* halt the MEs */
+ sdma_v5_0_enable(adev, false);
+
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ if (!adev->sdma.instance[i].fw)
+ return -EINVAL;
+
+ hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
+ amdgpu_ucode_print_sdma_hdr(&hdr->header);
+ fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
+
+ fw_data = (const __le32 *)
+ (adev->sdma.instance[i].fw->data +
+ le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_UCODE_ADDR), 0);
+
+ for (j = 0; j < fw_size; j++) {
+ if (amdgpu_emu_mode == 1 && j % 500 == 0)
+ msleep(1);
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_UCODE_DATA), le32_to_cpup(fw_data++));
+ }
+
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_UCODE_ADDR), adev->sdma.instance[i].fw_version);
+ }
+
+ return 0;
+}
+
+/**
+ * sdma_v5_0_start - setup and start the async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Set up the DMA engines and enable them (NAVI10).
+ * Returns 0 for success, error for failure.
+ */
+static int sdma_v5_0_start(struct amdgpu_device *adev)
+{
+ int r = 0;
+
+ if (amdgpu_sriov_vf(adev)) {
+ sdma_v5_0_ctx_switch_enable(adev, false);
+ sdma_v5_0_enable(adev, false);
+
+ /* set RB registers */
+ r = sdma_v5_0_gfx_resume(adev);
+ return r;
+ }
+
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+ r = sdma_v5_0_load_microcode(adev);
+ if (r)
+ return r;
+
+ /* The value of mmSDMA_F32_CNTL is invalid the moment after loading fw */
+ if (amdgpu_emu_mode == 1 && adev->pdev->device == 0x4d)
+ msleep(1000);
+ }
+
+ /* unhalt the MEs */
+ sdma_v5_0_enable(adev, true);
+ /* enable sdma ring preemption */
+ sdma_v5_0_ctx_switch_enable(adev, true);
+
+ /* start the gfx rings and rlc compute queues */
+ r = sdma_v5_0_gfx_resume(adev);
+ if (r)
+ return r;
+ r = sdma_v5_0_rlc_resume(adev);
+
+ return r;
+}
+
+/**
+ * sdma_v5_0_ring_test_ring - simple async dma engine test
+ *
+ * @ring: amdgpu_ring structure holding ring information
+ *
+ * Test the DMA engine by writing using it to write an
+ * value to memory. (NAVI10).
+ * Returns 0 for success, error for failure.
+ */
+static int sdma_v5_0_ring_test_ring(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+ unsigned i;
+ unsigned index;
+ int r;
+ u32 tmp;
+ u64 gpu_addr;
+
+ r = amdgpu_device_wb_get(adev, &index);
+ if (r) {
+ dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
+ return r;
+ }
+
+ gpu_addr = adev->wb.gpu_addr + (index * 4);
+ tmp = 0xCAFEDEAD;
+ adev->wb.wb[index] = cpu_to_le32(tmp);
+
+ r = amdgpu_ring_alloc(ring, 5);
+ if (r) {
+ DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
+ amdgpu_device_wb_free(adev, index);
+ return r;
+ }
+
+ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
+ SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
+ amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
+ amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
+ amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));
+ amdgpu_ring_write(ring, 0xDEADBEEF);
+ amdgpu_ring_commit(ring);
+
+ for (i = 0; i < adev->usec_timeout; i++) {
+ tmp = le32_to_cpu(adev->wb.wb[index]);
+ if (tmp == 0xDEADBEEF)
+ break;
+ if (amdgpu_emu_mode == 1)
+ msleep(1);
+ else
+ DRM_UDELAY(1);
+ }
+
+ if (i < adev->usec_timeout) {
+ if (amdgpu_emu_mode == 1)
+ DRM_INFO("ring test on %d succeeded in %d msecs\n", ring->idx, i);
+ else
+ DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
+ } else {
+ DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
+ ring->idx, tmp);
+ r = -EINVAL;
+ }
+ amdgpu_device_wb_free(adev, index);
+
+ return r;
+}
+
+/**
+ * sdma_v5_0_ring_test_ib - test an IB on the DMA engine
+ *
+ * @ring: amdgpu_ring structure holding ring information
+ *
+ * Test a simple IB in the DMA ring (NAVI10).
+ * Returns 0 on success, error on failure.
+ */
+static int sdma_v5_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
+{
+ struct amdgpu_device *adev = ring->adev;
+ struct amdgpu_ib ib;
+ struct dma_fence *f = NULL;
+ unsigned index;
+ long r;
+ u32 tmp = 0;
+ u64 gpu_addr;
+
+ r = amdgpu_device_wb_get(adev, &index);
+ if (r) {
+ dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
+ return r;
+ }
+
+ gpu_addr = adev->wb.gpu_addr + (index * 4);
+ tmp = 0xCAFEDEAD;
+ adev->wb.wb[index] = cpu_to_le32(tmp);
+ memset(&ib, 0, sizeof(ib));
+ r = amdgpu_ib_get(adev, NULL, 256, &ib);
+ if (r) {
+ DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
+ goto err0;
+ }
+
+ ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
+ SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
+ ib.ptr[1] = lower_32_bits(gpu_addr);
+ ib.ptr[2] = upper_32_bits(gpu_addr);
+ ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
+ ib.ptr[4] = 0xDEADBEEF;
+ ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+ ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+ ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+ ib.length_dw = 8;
+
+ r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
+ if (r)
+ goto err1;
+
+ r = dma_fence_wait_timeout(f, false, timeout);
+ if (r == 0) {
+ DRM_ERROR("amdgpu: IB test timed out\n");
+ r = -ETIMEDOUT;
+ goto err1;
+ } else if (r < 0) {
+ DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
+ goto err1;
+ }
+ tmp = le32_to_cpu(adev->wb.wb[index]);
+ if (tmp == 0xDEADBEEF) {
+ DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+ r = 0;
+ } else {
+ DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
+ r = -EINVAL;
+ }
+
+err1:
+ amdgpu_ib_free(adev, &ib, NULL);
+ dma_fence_put(f);
+err0:
+ amdgpu_device_wb_free(adev, index);
+ return r;
+}
+
+
+/**
+ * sdma_v5_0_vm_copy_pte - update PTEs by copying them from the GART
+ *
+ * @ib: indirect buffer to fill with commands
+ * @pe: addr of the page entry
+ * @src: src addr to copy from
+ * @count: number of page entries to update
+ *
+ * Update PTEs by copying them from the GART using sDMA (NAVI10).
+ */
+static void sdma_v5_0_vm_copy_pte(struct amdgpu_ib *ib,
+ uint64_t pe, uint64_t src,
+ unsigned count)
+{
+ unsigned bytes = count * 8;
+
+ ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
+ SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
+ ib->ptr[ib->length_dw++] = bytes - 1;
+ ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
+ ib->ptr[ib->length_dw++] = lower_32_bits(src);
+ ib->ptr[ib->length_dw++] = upper_32_bits(src);
+ ib->ptr[ib->length_dw++] = lower_32_bits(pe);
+ ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+
+}
+
+/**
+ * sdma_v5_0_vm_write_pte - update PTEs by writing them manually
+ *
+ * @ib: indirect buffer to fill with commands
+ * @pe: addr of the page entry
+ * @addr: dst addr to write into pe
+ * @count: number of page entries to update
+ * @incr: increase next addr by incr bytes
+ * @flags: access flags
+ *
+ * Update PTEs by writing them manually using sDMA (NAVI10).
+ */
+static void sdma_v5_0_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
+ uint64_t value, unsigned count,
+ uint32_t incr)
+{
+ unsigned ndw = count * 2;
+
+ ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
+ SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
+ ib->ptr[ib->length_dw++] = lower_32_bits(pe);
+ ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+ ib->ptr[ib->length_dw++] = ndw - 1;
+ for (; ndw > 0; ndw -= 2) {
+ ib->ptr[ib->length_dw++] = lower_32_bits(value);
+ ib->ptr[ib->length_dw++] = upper_32_bits(value);
+ value += incr;
+ }
+}
+
+/**
+ * sdma_v5_0_vm_set_pte_pde - update the page tables using sDMA
+ *
+ * @ib: indirect buffer to fill with commands
+ * @pe: addr of the page entry
+ * @addr: dst addr to write into pe
+ * @count: number of page entries to update
+ * @incr: increase next addr by incr bytes
+ * @flags: access flags
+ *
+ * Update the page tables using sDMA (NAVI10).
+ */
+static void sdma_v5_0_vm_set_pte_pde(struct amdgpu_ib *ib,
+ uint64_t pe,
+ uint64_t addr, unsigned count,
+ uint32_t incr, uint64_t flags)
+{
+ /* for physically contiguous pages (vram) */
+ ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_PTEPDE);
+ ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
+ ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+ ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
+ ib->ptr[ib->length_dw++] = upper_32_bits(flags);
+ ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
+ ib->ptr[ib->length_dw++] = upper_32_bits(addr);
+ ib->ptr[ib->length_dw++] = incr; /* increment size */
+ ib->ptr[ib->length_dw++] = 0;
+ ib->ptr[ib->length_dw++] = count - 1; /* number of entries */
+}
+
+/**
+ * sdma_v5_0_ring_pad_ib - pad the IB to the required number of dw
+ *
+ * @ib: indirect buffer to fill with padding
+ *
+ */
+static void sdma_v5_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
+{
+ struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
+ u32 pad_count;
+ int i;
+
+ pad_count = (8 - (ib->length_dw & 0x7)) % 8;
+ for (i = 0; i < pad_count; i++)
+ if (sdma && sdma->burst_nop && (i == 0))
+ ib->ptr[ib->length_dw++] =
+ SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
+ SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
+ else
+ ib->ptr[ib->length_dw++] =
+ SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
+}
+
+
+/**
+ * sdma_v5_0_ring_emit_pipeline_sync - sync the pipeline
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Make sure all previous operations are completed (CIK).
+ */
+static void sdma_v5_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
+{
+ uint32_t seq = ring->fence_drv.sync_seq;
+ uint64_t addr = ring->fence_drv.gpu_addr;
+
+ /* wait for idle */
+ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
+ SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
+ SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */
+ SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1));
+ amdgpu_ring_write(ring, addr & 0xfffffffc);
+ amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
+ amdgpu_ring_write(ring, seq); /* reference */
+ amdgpu_ring_write(ring, 0xfffffff); /* mask */
+ amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
+ SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */
+}
+
+
+/**
+ * sdma_v5_0_ring_emit_vm_flush - vm flush using sDMA
+ *
+ * @ring: amdgpu_ring pointer
+ * @vm: amdgpu_vm pointer
+ *
+ * Update the page table base and flush the VM TLB
+ * using sDMA (NAVI10).
+ */
+static void sdma_v5_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
+ unsigned vmid, uint64_t pd_addr)
+{
+ amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
+}
+
+static void sdma_v5_0_ring_emit_wreg(struct amdgpu_ring *ring,
+ uint32_t reg, uint32_t val)
+{
+ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
+ SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
+ amdgpu_ring_write(ring, reg);
+ amdgpu_ring_write(ring, val);
+}
+
+static void sdma_v5_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
+ uint32_t val, uint32_t mask)
+{
+ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
+ SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
+ SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* equal */
+ amdgpu_ring_write(ring, reg << 2);
+ amdgpu_ring_write(ring, 0);
+ amdgpu_ring_write(ring, val); /* reference */
+ amdgpu_ring_write(ring, mask); /* mask */
+ amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
+ SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10));
+}
+
+static int sdma_v5_0_early_init(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ adev->sdma.num_instances = 2;
+
+ sdma_v5_0_set_ring_funcs(adev);
+ sdma_v5_0_set_buffer_funcs(adev);
+ sdma_v5_0_set_vm_pte_funcs(adev);
+ sdma_v5_0_set_irq_funcs(adev);
+
+ return 0;
+}
+
+
+static int sdma_v5_0_sw_init(void *handle)
+{
+ struct amdgpu_ring *ring;
+ int r, i;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ /* SDMA trap event */
+ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_SDMA0,
+ SDMA0_5_0__SRCID__SDMA_TRAP,
+ &adev->sdma.trap_irq);
+ if (r)
+ return r;
+
+ /* SDMA trap event */
+ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_SDMA1,
+ SDMA1_5_0__SRCID__SDMA_TRAP,
+ &adev->sdma.trap_irq);
+ if (r)
+ return r;
+
+ r = sdma_v5_0_init_microcode(adev);
+ if (r) {
+ DRM_ERROR("Failed to load sdma firmware!\n");
+ return r;
+ }
+
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ ring = &adev->sdma.instance[i].ring;
+ ring->ring_obj = NULL;
+ ring->use_doorbell = true;
+
+ DRM_INFO("use_doorbell being set to: [%s]\n",
+ ring->use_doorbell?"true":"false");
+
+ ring->doorbell_index = (i == 0) ?
+ (adev->doorbell_index.sdma_engine[0] << 1) //get DWORD offset
+ : (adev->doorbell_index.sdma_engine[1] << 1); // get DWORD offset
+
+ sprintf(ring->name, "sdma%d", i);
+ r = amdgpu_ring_init(adev, ring, 1024,
+ &adev->sdma.trap_irq,
+ (i == 0) ?
+ AMDGPU_SDMA_IRQ_INSTANCE0 :
+ AMDGPU_SDMA_IRQ_INSTANCE1);
+ if (r)
+ return r;
+ }
+
+ return r;
+}
+
+static int sdma_v5_0_sw_fini(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ int i;
+
+ for (i = 0; i < adev->sdma.num_instances; i++)
+ amdgpu_ring_fini(&adev->sdma.instance[i].ring);
+
+ return 0;
+}
+
+static int sdma_v5_0_hw_init(void *handle)
+{
+ int r;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ sdma_v5_0_init_golden_registers(adev);
+
+ r = sdma_v5_0_start(adev);
+
+ return r;
+}
+
+static int sdma_v5_0_hw_fini(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ if (amdgpu_sriov_vf(adev))
+ return 0;
+
+ sdma_v5_0_ctx_switch_enable(adev, false);
+ sdma_v5_0_enable(adev, false);
+
+ return 0;
+}
+
+static int sdma_v5_0_suspend(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ return sdma_v5_0_hw_fini(adev);
+}
+
+static int sdma_v5_0_resume(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ return sdma_v5_0_hw_init(adev);
+}
+
+static bool sdma_v5_0_is_idle(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ u32 i;
+
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ u32 tmp = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_STATUS_REG));
+
+ if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
+ return false;
+ }
+
+ return true;
+}
+
+static int sdma_v5_0_wait_for_idle(void *handle)
+{
+ unsigned i;
+ u32 sdma0, sdma1;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ for (i = 0; i < adev->usec_timeout; i++) {
+ sdma0 = RREG32(sdma_v5_0_get_reg_offset(adev, 0, mmSDMA0_STATUS_REG));
+ sdma1 = RREG32(sdma_v5_0_get_reg_offset(adev, 1, mmSDMA0_STATUS_REG));
+
+ if (sdma0 & sdma1 & SDMA0_STATUS_REG__IDLE_MASK)
+ return 0;
+ udelay(1);
+ }
+ return -ETIMEDOUT;
+}
+
+static int sdma_v5_0_soft_reset(void *handle)
+{
+ /* todo */
+
+ return 0;
+}
+
+static int sdma_v5_0_ring_preempt_ib(struct amdgpu_ring *ring)
+{
+ int i, r = 0;
+ struct amdgpu_device *adev = ring->adev;
+ u32 index = 0;
+ u64 sdma_gfx_preempt;
+
+ amdgpu_sdma_get_index_from_ring(ring, &index);
+ if (index == 0)
+ sdma_gfx_preempt = mmSDMA0_GFX_PREEMPT;
+ else
+ sdma_gfx_preempt = mmSDMA1_GFX_PREEMPT;
+
+ /* assert preemption condition */
+ amdgpu_ring_set_preempt_cond_exec(ring, false);
+
+ /* emit the trailing fence */
+ ring->trail_seq += 1;
+ amdgpu_ring_alloc(ring, 10);
+ sdma_v5_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
+ ring->trail_seq, 0);
+ amdgpu_ring_commit(ring);
+
+ /* assert IB preemption */
+ WREG32(sdma_gfx_preempt, 1);
+
+ /* poll the trailing fence */
+ for (i = 0; i < adev->usec_timeout; i++) {
+ if (ring->trail_seq ==
+ le32_to_cpu(*(ring->trail_fence_cpu_addr)))
+ break;
+ DRM_UDELAY(1);
+ }
+
+ if (i >= adev->usec_timeout) {
+ r = -EINVAL;
+ DRM_ERROR("ring %d failed to be preempted\n", ring->idx);
+ }
+
+ /* deassert IB preemption */
+ WREG32(sdma_gfx_preempt, 0);
+
+ /* deassert the preemption condition */
+ amdgpu_ring_set_preempt_cond_exec(ring, true);
+ return r;
+}
+
+static int sdma_v5_0_set_trap_irq_state(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *source,
+ unsigned type,
+ enum amdgpu_interrupt_state state)
+{
+ u32 sdma_cntl;
+
+ u32 reg_offset = (type == AMDGPU_SDMA_IRQ_INSTANCE0) ?
+ sdma_v5_0_get_reg_offset(adev, 0, mmSDMA0_CNTL) :
+ sdma_v5_0_get_reg_offset(adev, 1, mmSDMA0_CNTL);
+
+ sdma_cntl = RREG32(reg_offset);
+ sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE,
+ state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
+ WREG32(reg_offset, sdma_cntl);
+
+ return 0;
+}
+
+static int sdma_v5_0_process_trap_irq(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *source,
+ struct amdgpu_iv_entry *entry)
+{
+ DRM_DEBUG("IH: SDMA trap\n");
+ switch (entry->client_id) {
+ case SOC15_IH_CLIENTID_SDMA0:
+ switch (entry->ring_id) {
+ case 0:
+ amdgpu_fence_process(&adev->sdma.instance[0].ring);
+ break;
+ case 1:
+ /* XXX compute */
+ break;
+ case 2:
+ /* XXX compute */
+ break;
+ case 3:
+ /* XXX page queue*/
+ break;
+ }
+ break;
+ case SOC15_IH_CLIENTID_SDMA1:
+ switch (entry->ring_id) {
+ case 0:
+ amdgpu_fence_process(&adev->sdma.instance[1].ring);
+ break;
+ case 1:
+ /* XXX compute */
+ break;
+ case 2:
+ /* XXX compute */
+ break;
+ case 3:
+ /* XXX page queue*/
+ break;
+ }
+ break;
+ }
+ return 0;
+}
+
+static int sdma_v5_0_process_illegal_inst_irq(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *source,
+ struct amdgpu_iv_entry *entry)
+{
+ return 0;
+}
+
+static void sdma_v5_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
+ bool enable)
+{
+ uint32_t data, def;
+ int i;
+
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) {
+ /* Enable sdma clock gating */
+ def = data = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL));
+ data &= ~(SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK |
+ SDMA0_CLK_CTRL__SOFT_OVERRIDE6_MASK |
+ SDMA0_CLK_CTRL__SOFT_OVERRIDE5_MASK |
+ SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK |
+ SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK |
+ SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK |
+ SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK |
+ SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK);
+ if (def != data)
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL), data);
+ } else {
+ /* Disable sdma clock gating */
+ def = data = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL));
+ data |= (SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK |
+ SDMA0_CLK_CTRL__SOFT_OVERRIDE6_MASK |
+ SDMA0_CLK_CTRL__SOFT_OVERRIDE5_MASK |
+ SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK |
+ SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK |
+ SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK |
+ SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK |
+ SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK);
+ if (def != data)
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL), data);
+ }
+ }
+}
+
+static void sdma_v5_0_update_medium_grain_light_sleep(struct amdgpu_device *adev,
+ bool enable)
+{
+ uint32_t data, def;
+ int i;
+
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS)) {
+ /* Enable sdma mem light sleep */
+ def = data = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL));
+ data |= SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
+ if (def != data)
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL), data);
+
+ } else {
+ /* Disable sdma mem light sleep */
+ def = data = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL));
+ data &= ~SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
+ if (def != data)
+ WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL), data);
+
+ }
+ }
+}
+
+static int sdma_v5_0_set_clockgating_state(void *handle,
+ enum amd_clockgating_state state)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ if (amdgpu_sriov_vf(adev))
+ return 0;
+
+ switch (adev->asic_type) {
+ case CHIP_NAVI10:
+ sdma_v5_0_update_medium_grain_clock_gating(adev,
+ state == AMD_CG_STATE_GATE ? true : false);
+ sdma_v5_0_update_medium_grain_light_sleep(adev,
+ state == AMD_CG_STATE_GATE ? true : false);
+ break;
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+static int sdma_v5_0_set_powergating_state(void *handle,
+ enum amd_powergating_state state)
+{
+ return 0;
+}
+
+static void sdma_v5_0_get_clockgating_state(void *handle, u32 *flags)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ int data;
+
+ if (amdgpu_sriov_vf(adev))
+ *flags = 0;
+
+ /* AMD_CG_SUPPORT_SDMA_MGCG */
+ data = RREG32(sdma_v5_0_get_reg_offset(adev, 0, mmSDMA0_CLK_CTRL));
+ if (!(data & SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK))
+ *flags |= AMD_CG_SUPPORT_SDMA_MGCG;
+
+ /* AMD_CG_SUPPORT_SDMA_LS */
+ data = RREG32(sdma_v5_0_get_reg_offset(adev, 0, mmSDMA0_POWER_CNTL));
+ if (data & SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK)
+ *flags |= AMD_CG_SUPPORT_SDMA_LS;
+}
+
+const struct amd_ip_funcs sdma_v5_0_ip_funcs = {
+ .name = "sdma_v5_0",
+ .early_init = sdma_v5_0_early_init,
+ .late_init = NULL,
+ .sw_init = sdma_v5_0_sw_init,
+ .sw_fini = sdma_v5_0_sw_fini,
+ .hw_init = sdma_v5_0_hw_init,
+ .hw_fini = sdma_v5_0_hw_fini,
+ .suspend = sdma_v5_0_suspend,
+ .resume = sdma_v5_0_resume,
+ .is_idle = sdma_v5_0_is_idle,
+ .wait_for_idle = sdma_v5_0_wait_for_idle,
+ .soft_reset = sdma_v5_0_soft_reset,
+ .set_clockgating_state = sdma_v5_0_set_clockgating_state,
+ .set_powergating_state = sdma_v5_0_set_powergating_state,
+ .get_clockgating_state = sdma_v5_0_get_clockgating_state,
+};
+
+static const struct amdgpu_ring_funcs sdma_v5_0_ring_funcs = {
+ .type = AMDGPU_RING_TYPE_SDMA,
+ .align_mask = 0xf,
+ .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
+ .support_64bit_ptrs = true,
+ .vmhub = AMDGPU_GFXHUB,
+ .get_rptr = sdma_v5_0_ring_get_rptr,
+ .get_wptr = sdma_v5_0_ring_get_wptr,
+ .set_wptr = sdma_v5_0_ring_set_wptr,
+ .emit_frame_size =
+ 5 + /* sdma_v5_0_ring_init_cond_exec */
+ 6 + /* sdma_v5_0_ring_emit_hdp_flush */
+ 3 + /* hdp_invalidate */
+ 6 + /* sdma_v5_0_ring_emit_pipeline_sync */
+ /* sdma_v5_0_ring_emit_vm_flush */
+ SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
+ SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
+ 10 + 10 + 10, /* sdma_v5_0_ring_emit_fence x3 for user fence, vm fence */
+ .emit_ib_size = 7 + 6, /* sdma_v5_0_ring_emit_ib */
+ .emit_ib = sdma_v5_0_ring_emit_ib,
+ .emit_fence = sdma_v5_0_ring_emit_fence,
+ .emit_pipeline_sync = sdma_v5_0_ring_emit_pipeline_sync,
+ .emit_vm_flush = sdma_v5_0_ring_emit_vm_flush,
+ .emit_hdp_flush = sdma_v5_0_ring_emit_hdp_flush,
+ .test_ring = sdma_v5_0_ring_test_ring,
+ .test_ib = sdma_v5_0_ring_test_ib,
+ .insert_nop = sdma_v5_0_ring_insert_nop,
+ .pad_ib = sdma_v5_0_ring_pad_ib,
+ .emit_wreg = sdma_v5_0_ring_emit_wreg,
+ .emit_reg_wait = sdma_v5_0_ring_emit_reg_wait,
+ .init_cond_exec = sdma_v5_0_ring_init_cond_exec,
+ .patch_cond_exec = sdma_v5_0_ring_patch_cond_exec,
+ .preempt_ib = sdma_v5_0_ring_preempt_ib,
+};
+
+static void sdma_v5_0_set_ring_funcs(struct amdgpu_device *adev)
+{
+ int i;
+
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ adev->sdma.instance[i].ring.funcs = &sdma_v5_0_ring_funcs;
+ adev->sdma.instance[i].ring.me = i;
+ }
+}
+
+static const struct amdgpu_irq_src_funcs sdma_v5_0_trap_irq_funcs = {
+ .set = sdma_v5_0_set_trap_irq_state,
+ .process = sdma_v5_0_process_trap_irq,
+};
+
+static const struct amdgpu_irq_src_funcs sdma_v5_0_illegal_inst_irq_funcs = {
+ .process = sdma_v5_0_process_illegal_inst_irq,
+};
+
+static void sdma_v5_0_set_irq_funcs(struct amdgpu_device *adev)
+{
+ adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
+ adev->sdma.trap_irq.funcs = &sdma_v5_0_trap_irq_funcs;
+ adev->sdma.illegal_inst_irq.funcs = &sdma_v5_0_illegal_inst_irq_funcs;
+}
+
+/**
+ * sdma_v5_0_emit_copy_buffer - copy buffer using the sDMA engine
+ *
+ * @ring: amdgpu_ring structure holding ring information
+ * @src_offset: src GPU address
+ * @dst_offset: dst GPU address
+ * @byte_count: number of bytes to xfer
+ *
+ * Copy GPU buffers using the DMA engine (NAVI10).
+ * Used by the amdgpu ttm implementation to move pages if
+ * registered as the asic copy callback.
+ */
+static void sdma_v5_0_emit_copy_buffer(struct amdgpu_ib *ib,
+ uint64_t src_offset,
+ uint64_t dst_offset,
+ uint32_t byte_count)
+{
+ ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
+ SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
+ ib->ptr[ib->length_dw++] = byte_count - 1;
+ ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
+ ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
+ ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
+ ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
+ ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
+}
+
+/**
+ * sdma_v5_0_emit_fill_buffer - fill buffer using the sDMA engine
+ *
+ * @ring: amdgpu_ring structure holding ring information
+ * @src_data: value to write to buffer
+ * @dst_offset: dst GPU address
+ * @byte_count: number of bytes to xfer
+ *
+ * Fill GPU buffers using the DMA engine (NAVI10).
+ */
+static void sdma_v5_0_emit_fill_buffer(struct amdgpu_ib *ib,
+ uint32_t src_data,
+ uint64_t dst_offset,
+ uint32_t byte_count)
+{
+ ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL);
+ ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
+ ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
+ ib->ptr[ib->length_dw++] = src_data;
+ ib->ptr[ib->length_dw++] = byte_count - 1;
+}
+
+static const struct amdgpu_buffer_funcs sdma_v5_0_buffer_funcs = {
+ .copy_max_bytes = 0x400000,
+ .copy_num_dw = 7,
+ .emit_copy_buffer = sdma_v5_0_emit_copy_buffer,
+
+ .fill_max_bytes = 0x400000,
+ .fill_num_dw = 5,
+ .emit_fill_buffer = sdma_v5_0_emit_fill_buffer,
+};
+
+static void sdma_v5_0_set_buffer_funcs(struct amdgpu_device *adev)
+{
+ if (adev->mman.buffer_funcs == NULL) {
+ adev->mman.buffer_funcs = &sdma_v5_0_buffer_funcs;
+ adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+ }
+}
+
+static const struct amdgpu_vm_pte_funcs sdma_v5_0_vm_pte_funcs = {
+ .copy_pte_num_dw = 7,
+ .copy_pte = sdma_v5_0_vm_copy_pte,
+ .write_pte = sdma_v5_0_vm_write_pte,
+ .set_pte_pde = sdma_v5_0_vm_set_pte_pde,
+};
+
+static void sdma_v5_0_set_vm_pte_funcs(struct amdgpu_device *adev)
+{
+ struct drm_gpu_scheduler *sched;
+ unsigned i;
+
+ if (adev->vm_manager.vm_pte_funcs == NULL) {
+ adev->vm_manager.vm_pte_funcs = &sdma_v5_0_vm_pte_funcs;
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ sched = &adev->sdma.instance[i].ring.sched;
+ adev->vm_manager.vm_pte_rqs[i] =
+ &sched->sched_rq[DRM_SCHED_PRIORITY_KERNEL];
+ }
+ adev->vm_manager.vm_pte_num_rqs = adev->sdma.num_instances;
+ }
+}
+
+const struct amdgpu_ip_block_version sdma_v5_0_ip_block = {
+ .type = AMD_IP_BLOCK_TYPE_SDMA,
+ .major = 5,
+ .minor = 0,
+ .rev = 0,
+ .funcs = &sdma_v5_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.h b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.h
new file mode 100644
index 000000000000..d5a94e3d181c
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __SDMA_V5_0_H__
+#define __SDMA_V5_0_H__
+
+enum sdma_v5_0_utcl2_cache_read_policy {
+ CACHE_READ_POLICY_L2__LRU = 0x00000000,
+ CACHE_READ_POLICY_L2__STREAM = 0x00000001,
+ CACHE_READ_POLICY_L2__NOA = 0x00000002,
+ CACHE_READ_POLICY_L2__DEFAULT = CACHE_READ_POLICY_L2__NOA,
+};
+
+enum sdma_v5_0_utcl2_cache_write_policy {
+ CACHE_WRITE_POLICY_L2__LRU = 0x00000000,
+ CACHE_WRITE_POLICY_L2__STREAM = 0x00000001,
+ CACHE_WRITE_POLICY_L2__NOA = 0x00000002,
+ CACHE_WRITE_POLICY_L2__BYPASS = 0x00000003,
+ CACHE_WRITE_POLICY_L2__DEFAULT = CACHE_WRITE_POLICY_L2__BYPASS,
+};
+
+extern const struct amd_ip_funcs sdma_v5_0_ip_funcs;
+extern const struct amdgpu_ip_block_version sdma_v5_0_ip_block;
+
+#endif /* __SDMA_V5_0_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c
index 9d8df68893b9..4d74453f3cfb 100644
--- a/drivers/gpu/drm/amd/amdgpu/si.c
+++ b/drivers/gpu/drm/amd/amdgpu/si.c
@@ -24,7 +24,8 @@
#include <linux/firmware.h>
#include <linux/slab.h>
#include <linux/module.h>
-#include <drm/drmP.h>
+#include <linux/pci.h>
+
#include "amdgpu.h"
#include "amdgpu_atombios.h"
#include "amdgpu_ih.h"
@@ -1339,8 +1340,8 @@ static void si_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
/* This reports 0 on APUs, so return to avoid writing/reading registers
* that may or may not be different from their GPU counterparts
*/
- if (adev->flags & AMD_IS_APU)
- return;
+ if (adev->flags & AMD_IS_APU)
+ return;
/* Set the 2 events that we wish to watch, defined above */
/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
@@ -1375,6 +1376,18 @@ static void si_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
}
+static uint64_t si_get_pcie_replay_count(struct amdgpu_device *adev)
+{
+ uint64_t nak_r, nak_g;
+
+ /* Get the number of NAKs received and generated */
+ nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
+ nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
+
+ /* Add the total number of NAKs, i.e the number of replays */
+ return (nak_r + nak_g);
+}
+
static const struct amdgpu_asic_funcs si_asic_funcs =
{
.read_disabled_bios = &si_read_disabled_bios,
@@ -1393,6 +1406,7 @@ static const struct amdgpu_asic_funcs si_asic_funcs =
.need_full_reset = &si_need_full_reset,
.get_pcie_usage = &si_get_pcie_usage,
.need_reset_on_init = &si_need_reset_on_init,
+ .get_pcie_replay_count = &si_get_pcie_replay_count,
};
static uint32_t si_get_rev_id(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c
index 3eeefd40dae0..bdda8b4e03f0 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dma.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c
@@ -21,7 +21,7 @@
*
* Authors: Alex Deucher
*/
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_trace.h"
#include "si.h"
@@ -230,7 +230,7 @@ static int si_dma_ring_test_ring(struct amdgpu_ring *ring)
tmp = le32_to_cpu(adev->wb.wb[index]);
if (tmp == 0xDEADBEEF)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
index d57e75e5c71f..4cb4c891120b 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
@@ -21,7 +21,9 @@
*
*/
-#include <drm/drmP.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
#include "amdgpu.h"
#include "amdgpu_pm.h"
#include "amdgpu_dpm.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/si_ih.c b/drivers/gpu/drm/amd/amdgpu/si_ih.c
index 8c50c9cab455..57bb5f9e08b2 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_ih.c
@@ -20,7 +20,9 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
-#include <drm/drmP.h>
+
+#include <linux/pci.h>
+
#include "amdgpu.h"
#include "amdgpu_ih.h"
#include "sid.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/si_smc.c b/drivers/gpu/drm/amd/amdgpu/si_smc.c
index 4a2fd8b61940..8f994ffa9cd1 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_smc.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_smc.c
@@ -23,7 +23,7 @@
*/
#include <linux/firmware.h>
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "sid.h"
#include "ppsmc.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index b7e594c2bfb4..04fbf05d7176 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -23,7 +23,8 @@
#include <linux/firmware.h>
#include <linux/slab.h>
#include <linux/module.h>
-#include <drm/drmP.h>
+#include <linux/pci.h>
+
#include "amdgpu.h"
#include "amdgpu_atombios.h"
#include "amdgpu_ih.h"
@@ -44,6 +45,7 @@
#include "smuio/smuio_9_0_offset.h"
#include "smuio/smuio_9_0_sh_mask.h"
#include "nbio/nbio_7_0_default.h"
+#include "nbio/nbio_7_0_offset.h"
#include "nbio/nbio_7_0_sh_mask.h"
#include "nbio/nbio_7_0_smn.h"
#include "mp/mp_9_0_offset.h"
@@ -64,6 +66,9 @@
#include "dce_virtual.h"
#include "mxgpu_ai.h"
#include "amdgpu_smu.h"
+#include "amdgpu_ras.h"
+#include "amdgpu_xgmi.h"
+#include <uapi/linux/kfd_ioctl.h>
#define mmMP0_MISC_CGTT_CTRL0 0x01b9
#define mmMP0_MISC_CGTT_CTRL0_BASE_IDX 0
@@ -230,7 +235,7 @@ void soc15_grbm_select(struct amdgpu_device *adev,
grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, VMID, vmid);
grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, QUEUEID, queue);
- WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL), grbm_gfx_cntl);
+ WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_CNTL, grbm_gfx_cntl);
}
static void soc15_vga_set_state(struct amdgpu_device *adev, bool state)
@@ -270,15 +275,6 @@ static bool soc15_read_bios_from_rom(struct amdgpu_device *adev,
return true;
}
-struct soc15_allowed_register_entry {
- uint32_t hwip;
- uint32_t inst;
- uint32_t seg;
- uint32_t reg_offset;
- bool grbm_indexed;
-};
-
-
static struct soc15_allowed_register_entry soc15_allowed_read_registers[] = {
{ SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS)},
{ SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS2)},
@@ -383,9 +379,17 @@ void soc15_program_register_sequence(struct amdgpu_device *adev,
} else {
tmp = RREG32(reg);
tmp &= ~(entry->and_mask);
- tmp |= entry->or_mask;
+ tmp |= (entry->or_mask & entry->and_mask);
}
- WREG32(reg, tmp);
+
+ if (reg == SOC15_REG_OFFSET(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3) ||
+ reg == SOC15_REG_OFFSET(GC, 0, mmPA_SC_ENHANCE) ||
+ reg == SOC15_REG_OFFSET(GC, 0, mmPA_SC_ENHANCE_1) ||
+ reg == SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG))
+ WREG32_RLC(reg, tmp);
+ else
+ WREG32(reg, tmp);
+
}
}
@@ -475,6 +479,13 @@ static int soc15_asic_reset(struct amdgpu_device *adev)
soc15_asic_get_baco_capability(adev, &baco_reset);
else
baco_reset = false;
+ if (baco_reset) {
+ struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev, 0);
+ struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
+
+ if (hive || (ras && ras->supported))
+ baco_reset = false;
+ }
break;
default:
baco_reset = false;
@@ -606,12 +617,24 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev)
case CHIP_VEGA20:
amdgpu_device_ip_block_add(adev, &vega10_common_ip_block);
amdgpu_device_ip_block_add(adev, &gmc_v9_0_ip_block);
- amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block);
- if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
- if (adev->asic_type == CHIP_VEGA20)
- amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block);
- else
- amdgpu_device_ip_block_add(adev, &psp_v3_1_ip_block);
+
+ /* For Vega10 SR-IOV, PSP need to be initialized before IH */
+ if (amdgpu_sriov_vf(adev)) {
+ if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
+ if (adev->asic_type == CHIP_VEGA20)
+ amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block);
+ else
+ amdgpu_device_ip_block_add(adev, &psp_v3_1_ip_block);
+ }
+ amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block);
+ } else {
+ amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block);
+ if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
+ if (adev->asic_type == CHIP_VEGA20)
+ amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block);
+ else
+ amdgpu_device_ip_block_add(adev, &psp_v3_1_ip_block);
+ }
}
amdgpu_device_ip_block_add(adev, &gfx_v9_0_ip_block);
amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block);
@@ -626,8 +649,6 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev)
#if defined(CONFIG_DRM_AMD_DC)
else if (amdgpu_device_has_dc_support(adev))
amdgpu_device_ip_block_add(adev, &dm_ip_block);
-#else
-# warning "Enable CONFIG_DRM_AMD_DC for display support on SOC15."
#endif
if (!(adev->asic_type == CHIP_VEGA20 && amdgpu_sriov_vf(adev))) {
amdgpu_device_ip_block_add(adev, &uvd_v7_0_ip_block);
@@ -648,8 +669,6 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev)
#if defined(CONFIG_DRM_AMD_DC)
else if (amdgpu_device_has_dc_support(adev))
amdgpu_device_ip_block_add(adev, &dm_ip_block);
-#else
-# warning "Enable CONFIG_DRM_AMD_DC for display support on SOC15."
#endif
amdgpu_device_ip_block_add(adev, &vcn_v1_0_ip_block);
break;
@@ -690,13 +709,19 @@ static void soc15_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
/* This reports 0 on APUs, so return to avoid writing/reading registers
* that may or may not be different from their GPU counterparts
*/
- if (adev->flags & AMD_IS_APU)
- return;
+ if (adev->flags & AMD_IS_APU)
+ return;
/* Set the 2 events that we wish to watch, defined above */
- /* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
+ /* Reg 40 is # received msgs */
perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
- perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
+ /* Pre-VG20, Reg 104 is # of posted requests sent. On VG20 it's 108 */
+ if (adev->asic_type == CHIP_VEGA20)
+ perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK,
+ EVENT1_SEL, 108);
+ else
+ perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK,
+ EVENT1_SEL, 104);
/* Write to enable desired perf counters */
WREG32_PCIE(smnPCIE_PERF_CNTL_TXCLK, perfctr);
@@ -733,7 +758,8 @@ static bool soc15_need_reset_on_init(struct amdgpu_device *adev)
/* Just return false for soc15 GPUs. Reset does not seem to
* be necessary.
*/
- return false;
+ if (!amdgpu_passthrough(adev))
+ return false;
if (adev->flags & AMD_IS_APU)
return false;
@@ -748,6 +774,18 @@ static bool soc15_need_reset_on_init(struct amdgpu_device *adev)
return false;
}
+static uint64_t soc15_get_pcie_replay_count(struct amdgpu_device *adev)
+{
+ uint64_t nak_r, nak_g;
+
+ /* Get the number of NAKs received and generated */
+ nak_r = RREG32_PCIE(smnPCIE_RX_NUM_NAK);
+ nak_g = RREG32_PCIE(smnPCIE_RX_NUM_NAK_GENERATED);
+
+ /* Add the total number of NAKs, i.e the number of replays */
+ return (nak_r + nak_g);
+}
+
static const struct amdgpu_asic_funcs soc15_asic_funcs =
{
.read_disabled_bios = &soc15_read_disabled_bios,
@@ -765,6 +803,7 @@ static const struct amdgpu_asic_funcs soc15_asic_funcs =
.init_doorbell_index = &vega10_doorbell_index_init,
.get_pcie_usage = &soc15_get_pcie_usage,
.need_reset_on_init = &soc15_need_reset_on_init,
+ .get_pcie_replay_count = &soc15_get_pcie_replay_count,
};
static const struct amdgpu_asic_funcs vega20_asic_funcs =
@@ -784,12 +823,16 @@ static const struct amdgpu_asic_funcs vega20_asic_funcs =
.init_doorbell_index = &vega20_doorbell_index_init,
.get_pcie_usage = &soc15_get_pcie_usage,
.need_reset_on_init = &soc15_need_reset_on_init,
+ .get_pcie_replay_count = &soc15_get_pcie_replay_count,
};
static int soc15_common_early_init(void *handle)
{
+#define MMIO_REG_HOLE_OFFSET (0x80000 - PAGE_SIZE)
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ adev->rmmio_remap.reg_offset = MMIO_REG_HOLE_OFFSET;
+ adev->rmmio_remap.bus_addr = adev->rmmio_base + MMIO_REG_HOLE_OFFSET;
adev->smc_rreg = NULL;
adev->smc_wreg = NULL;
adev->pcie_rreg = &soc15_pcie_rreg;
@@ -949,11 +992,6 @@ static int soc15_common_early_init(void *handle)
adev->pg_flags = AMD_PG_SUPPORT_SDMA | AMD_PG_SUPPORT_VCN;
}
-
- if (adev->pm.pp_feature & PP_GFXOFF_MASK)
- adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
- AMD_PG_SUPPORT_CP |
- AMD_PG_SUPPORT_RLC_SMU_HS;
break;
default:
/* FIXME: not supported yet */
@@ -985,6 +1023,8 @@ static int soc15_common_sw_init(void *handle)
if (amdgpu_sriov_vf(adev))
xgpu_ai_mailbox_add_irq_id(adev);
+ adev->df_funcs->sw_init(adev);
+
return 0;
}
@@ -998,11 +1038,17 @@ static void soc15_doorbell_range_init(struct amdgpu_device *adev)
int i;
struct amdgpu_ring *ring;
- for (i = 0; i < adev->sdma.num_instances; i++) {
- ring = &adev->sdma.instance[i].ring;
- adev->nbio_funcs->sdma_doorbell_range(adev, i,
- ring->use_doorbell, ring->doorbell_index,
- adev->doorbell_index.sdma_doorbell_range);
+ /* Two reasons to skip
+ * 1, Host driver already programmed them
+ * 2, To avoid registers program violations in SR-IOV
+ */
+ if (!amdgpu_virt_support_skip_setting(adev)) {
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ ring = &adev->sdma.instance[i].ring;
+ adev->nbio_funcs->sdma_doorbell_range(adev, i,
+ ring->use_doorbell, ring->doorbell_index,
+ adev->doorbell_index.sdma_doorbell_range);
+ }
}
adev->nbio_funcs->ih_doorbell_range(adev, adev->irq.ih.use_doorbell,
@@ -1019,6 +1065,13 @@ static int soc15_common_hw_init(void *handle)
soc15_program_aspm(adev);
/* setup nbio registers */
adev->nbio_funcs->init_registers(adev);
+ /* remap HDP registers to a hole in mmio space,
+ * for the purpose of expose those registers
+ * to process space
+ */
+ if (adev->nbio_funcs->remap_hdp_registers)
+ adev->nbio_funcs->remap_hdp_registers(adev);
+
/* enable the doorbell aperture */
soc15_enable_doorbell_aperture(adev, true);
/* HW doorbell routing policy: doorbell writing not
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.h b/drivers/gpu/drm/amd/amdgpu/soc15.h
index a66c8bfbbaa6..7a6b2cc6d9f5 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.h
@@ -42,8 +42,28 @@ struct soc15_reg_golden {
u32 or_mask;
};
+struct soc15_reg_entry {
+ uint32_t hwip;
+ uint32_t inst;
+ uint32_t seg;
+ uint32_t reg_offset;
+ uint32_t reg_value;
+ uint32_t se_num;
+ uint32_t instance;
+};
+
+struct soc15_allowed_register_entry {
+ uint32_t hwip;
+ uint32_t inst;
+ uint32_t seg;
+ uint32_t reg_offset;
+ bool grbm_indexed;
+};
+
#define SOC15_REG_ENTRY(ip, inst, reg) ip##_HWIP, inst, reg##_BASE_IDX, reg
+#define SOC15_REG_ENTRY_OFFSET(entry) (adev->reg_offset[entry.hwip][entry.inst][entry.seg] + entry.reg_offset)
+
#define SOC15_REG_GOLDEN_VALUE(ip, inst, reg, and_mask, or_mask) \
{ ip##_HWIP, inst, reg##_BASE_IDX, reg, and_mask, or_mask }
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15_common.h b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
index 49c262540940..47f74dab365d 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15_common.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
@@ -69,26 +69,60 @@
} \
} while (0)
-#define RREG32_SOC15_DPG_MODE(ip, inst, reg, mask, sram_sel) \
- ({ WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_MASK, mask); \
- WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_CTL, \
- UVD_DPG_LMA_CTL__MASK_EN_MASK | \
- ((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) \
- << UVD_DPG_LMA_CTL__READ_WRITE_ADDR__SHIFT) | \
- (sram_sel << UVD_DPG_LMA_CTL__SRAM_SEL__SHIFT)); \
- RREG32_SOC15(ip, inst, mmUVD_DPG_LMA_DATA); })
+#define WREG32_RLC(reg, value) \
+ do { \
+ if (amdgpu_virt_support_rlc_prg_reg(adev)) { \
+ uint32_t i = 0; \
+ uint32_t retries = 50000; \
+ uint32_t r0 = adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0; \
+ uint32_t r1 = adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1; \
+ uint32_t spare_int = adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT; \
+ WREG32(r0, value); \
+ WREG32(r1, (reg | 0x80000000)); \
+ WREG32(spare_int, 0x1); \
+ for (i = 0; i < retries; i++) { \
+ u32 tmp = RREG32(r1); \
+ if (!(tmp & 0x80000000)) \
+ break; \
+ udelay(10); \
+ } \
+ if (i >= retries) \
+ pr_err("timeout: rlcg program reg:0x%05x failed !\n", reg); \
+ } else { \
+ WREG32(reg, value); \
+ } \
+ } while (0)
-#define WREG32_SOC15_DPG_MODE(ip, inst, reg, value, mask, sram_sel) \
+#define WREG32_SOC15_RLC_SHADOW(ip, inst, reg, value) \
do { \
- WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_DATA, value); \
- WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_MASK, mask); \
- WREG32_SOC15(ip, inst, mmUVD_DPG_LMA_CTL, \
- UVD_DPG_LMA_CTL__READ_WRITE_MASK | \
- ((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) \
- << UVD_DPG_LMA_CTL__READ_WRITE_ADDR__SHIFT) | \
- (sram_sel << UVD_DPG_LMA_CTL__SRAM_SEL__SHIFT)); \
+ uint32_t target_reg = adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg;\
+ if (amdgpu_virt_support_rlc_prg_reg(adev)) { \
+ uint32_t r2 = adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2; \
+ uint32_t r3 = adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3; \
+ uint32_t grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL; \
+ uint32_t grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX; \
+ if (target_reg == grbm_cntl) \
+ WREG32(r2, value); \
+ else if (target_reg == grbm_idx) \
+ WREG32(r3, value); \
+ WREG32(target_reg, value); \
+ } else { \
+ WREG32(target_reg, value); \
+ } \
} while (0)
-#endif
+#define WREG32_SOC15_RLC(ip, inst, reg, value) \
+ do { \
+ uint32_t target_reg = adev->reg_offset[GC_HWIP][0][reg##_BASE_IDX] + reg;\
+ WREG32_RLC(target_reg, value); \
+ } while (0)
+
+#define WREG32_FIELD15_RLC(ip, idx, reg, field, val) \
+ WREG32_RLC((adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg), \
+ (RREG32(adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg) \
+ & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
+#define WREG32_SOC15_OFFSET_RLC(ip, inst, reg, offset, value) \
+ WREG32_RLC(((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) + offset), value)
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h b/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h
index 0b4e7b55595a..ca7d05993ca2 100644
--- a/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h
+++ b/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h
@@ -1,29 +1,26 @@
-/****************************************************************************\
-*
-* File Name ta_ras_if.h
-* Project AMD PSP SW IP Module
-*
-* Description Interface to the RAS Trusted Application
-*
-* Copyright 2019 Advanced Micro Devices, Inc.
-*
-* Permission is hereby granted, free of charge, to any person obtaining a copy of this software
-* and associated documentation files (the "Software"), to deal in the Software without restriction,
-* including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
-* and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
-* subject to the following conditions:
-*
-* The above copyright notice and this permission notice shall be included in all copies or substantial
-* portions of the Software.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
-* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-* OTHER DEALINGS IN THE SOFTWARE.
-*/
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
#ifndef _TA_RAS_IF_H
#define _TA_RAS_IF_H
@@ -31,8 +28,8 @@
#define RSP_ID_MASK (1U << 31)
#define RSP_ID(cmdId) (((uint32_t)(cmdId)) | RSP_ID_MASK)
-#define TA_NUM_BLOCK_MAX 14
-
+/* RAS related enumerations */
+/**********************************************************/
enum ras_command {
TA_RAS_COMMAND__ENABLE_FEATURES = 0,
TA_RAS_COMMAND__DISABLE_FEATURES,
@@ -45,7 +42,12 @@ enum ta_ras_status {
TA_RAS_STATUS__ERROR_INVALID_PARAMETER = 0x02,
TA_RAS_STATUS__ERROR_RAS_NOT_AVAILABLE = 0x03,
TA_RAS_STATUS__ERROR_RAS_DUPLICATE_CMD = 0x04,
- TA_RAS_STATUS__ERROR_INJECTION_FAILED = 0x05
+ TA_RAS_STATUS__ERROR_INJECTION_FAILED = 0x05,
+ TA_RAS_STATUS__ERROR_ASD_READ_WRITE = 0x06,
+ TA_RAS_STATUS__ERROR_TOGGLE_DF_CSTATE = 0x07,
+ TA_RAS_STATUS__ERROR_TIMEOUT = 0x08,
+ TA_RAS_STATUS__ERROR_BLOCK_DISABLED = 0x09,
+ TA_RAS_STATUS__ERROR_GENERIC = 0x10,
};
enum ta_ras_block {
@@ -62,47 +64,55 @@ enum ta_ras_block {
TA_RAS_BLOCK__SEM,
TA_RAS_BLOCK__MP0,
TA_RAS_BLOCK__MP1,
- TA_RAS_BLOCK__FUSE = (TA_NUM_BLOCK_MAX - 1),
+ TA_RAS_BLOCK__FUSE,
+ TA_NUM_BLOCK_MAX
};
enum ta_ras_error_type {
- TA_RAS_ERROR__NONE = 0,
- TA_RAS_ERROR__PARITY = 1,
- TA_RAS_ERROR__SINGLE_CORRECTABLE = 2,
- TA_RAS_ERROR__MULTI_UNCORRECTABLE = 4,
- TA_RAS_ERROR__POISON = 8
+ TA_RAS_ERROR__NONE = 0,
+ TA_RAS_ERROR__PARITY = 1,
+ TA_RAS_ERROR__SINGLE_CORRECTABLE = 2,
+ TA_RAS_ERROR__MULTI_UNCORRECTABLE = 4,
+ TA_RAS_ERROR__POISON = 8,
};
+/* Input/output structures for RAS commands */
+/**********************************************************/
+
struct ta_ras_enable_features_input {
- enum ta_ras_block block_id;
- enum ta_ras_error_type error_type;
+ enum ta_ras_block block_id;
+ enum ta_ras_error_type error_type;
};
struct ta_ras_disable_features_input {
- enum ta_ras_block block_id;
- enum ta_ras_error_type error_type;
+ enum ta_ras_block block_id;
+ enum ta_ras_error_type error_type;
};
struct ta_ras_trigger_error_input {
- enum ta_ras_block block_id;
- enum ta_ras_error_type inject_error_type;
- uint32_t sub_block_index;
- uint64_t address;
- uint64_t value;
+ enum ta_ras_block block_id; // ras-block. i.e. umc, gfx
+ enum ta_ras_error_type inject_error_type; // type of error. i.e. single_correctable
+ uint32_t sub_block_index; // mem block. i.e. hbm, sram etc.
+ uint64_t address; // explicit address of error
+ uint64_t value; // method if error injection. i.e persistent, coherent etc.
};
+/* Common input structure for RAS callbacks */
+/**********************************************************/
union ta_ras_cmd_input {
struct ta_ras_enable_features_input enable_features;
struct ta_ras_disable_features_input disable_features;
struct ta_ras_trigger_error_input trigger_error;
};
+/* Shared Memory structures */
+/**********************************************************/
struct ta_ras_shared_memory {
- uint32_t cmd_id;
- uint32_t resp_id;
- enum ta_ras_status ras_status;
- uint32_t reserved;
- union ta_ras_cmd_input ras_in_message;
+ uint32_t cmd_id;
+ uint32_t resp_id;
+ enum ta_ras_status ras_status;
+ uint32_t reserved;
+ union ta_ras_cmd_input ras_in_message;
};
#endif // TL_RAS_IF_H_
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
index a20b711a6756..e40140bf6699 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
@@ -20,7 +20,9 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
-#include <drm/drmP.h>
+
+#include <linux/pci.h>
+
#include "amdgpu.h"
#include "amdgpu_ih.h"
#include "vid.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
index c4fb58667fd4..82abd8e728ab 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
@@ -23,7 +23,7 @@
*/
#include <linux/firmware.h>
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_uvd.h"
#include "cikd.h"
@@ -491,7 +491,7 @@ static int uvd_v4_2_ring_test_ring(struct amdgpu_ring *ring)
tmp = RREG32(mmUVD_CONTEXT_ID);
if (tmp == 0xDEADBEEF)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
@@ -741,6 +741,7 @@ static const struct amdgpu_ring_funcs uvd_v4_2_ring_funcs = {
.type = AMDGPU_RING_TYPE_UVD,
.align_mask = 0xf,
.support_64bit_ptrs = false,
+ .no_user_fence = true,
.get_rptr = uvd_v4_2_ring_get_rptr,
.get_wptr = uvd_v4_2_ring_get_wptr,
.set_wptr = uvd_v4_2_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
index 52bd8a654734..01e62fb8e6e0 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
@@ -22,8 +22,9 @@
* Authors: Christian König <christian.koenig@amd.com>
*/
+#include <linux/delay.h>
#include <linux/firmware.h>
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_uvd.h"
#include "vid.h"
@@ -506,7 +507,7 @@ static int uvd_v5_0_ring_test_ring(struct amdgpu_ring *ring)
tmp = RREG32(mmUVD_CONTEXT_ID);
if (tmp == 0xDEADBEEF)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
@@ -849,6 +850,7 @@ static const struct amdgpu_ring_funcs uvd_v5_0_ring_funcs = {
.type = AMDGPU_RING_TYPE_UVD,
.align_mask = 0xf,
.support_64bit_ptrs = false,
+ .no_user_fence = true,
.get_rptr = uvd_v5_0_ring_get_rptr,
.get_wptr = uvd_v5_0_ring_get_wptr,
.set_wptr = uvd_v5_0_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index be70e6e5f9df..670784a78512 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -23,7 +23,7 @@
*/
#include <linux/firmware.h>
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_uvd.h"
#include "vid.h"
@@ -186,7 +186,7 @@ static int uvd_v6_0_enc_ring_test_ring(struct amdgpu_ring *ring)
for (i = 0; i < adev->usec_timeout; i++) {
if (amdgpu_ring_get_rptr(ring) != rptr)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
@@ -960,7 +960,7 @@ static int uvd_v6_0_ring_test_ring(struct amdgpu_ring *ring)
tmp = RREG32(mmUVD_CONTEXT_ID);
if (tmp == 0xDEADBEEF)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
@@ -1505,6 +1505,7 @@ static const struct amdgpu_ring_funcs uvd_v6_0_ring_phys_funcs = {
.type = AMDGPU_RING_TYPE_UVD,
.align_mask = 0xf,
.support_64bit_ptrs = false,
+ .no_user_fence = true,
.get_rptr = uvd_v6_0_ring_get_rptr,
.get_wptr = uvd_v6_0_ring_get_wptr,
.set_wptr = uvd_v6_0_ring_set_wptr,
@@ -1530,6 +1531,7 @@ static const struct amdgpu_ring_funcs uvd_v6_0_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_UVD,
.align_mask = 0xf,
.support_64bit_ptrs = false,
+ .no_user_fence = true,
.get_rptr = uvd_v6_0_ring_get_rptr,
.get_wptr = uvd_v6_0_ring_get_wptr,
.set_wptr = uvd_v6_0_ring_set_wptr,
@@ -1558,6 +1560,7 @@ static const struct amdgpu_ring_funcs uvd_v6_0_enc_ring_vm_funcs = {
.align_mask = 0x3f,
.nop = HEVC_ENC_CMD_NO_OP,
.support_64bit_ptrs = false,
+ .no_user_fence = true,
.get_rptr = uvd_v6_0_enc_ring_get_rptr,
.get_wptr = uvd_v6_0_enc_ring_get_wptr,
.set_wptr = uvd_v6_0_enc_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
index fc4f0bb9a2e7..a6bfe7651d07 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
@@ -22,7 +22,7 @@
*/
#include <linux/firmware.h>
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_uvd.h"
#include "soc15.h"
@@ -194,7 +194,7 @@ static int uvd_v7_0_enc_ring_test_ring(struct amdgpu_ring *ring)
for (i = 0; i < adev->usec_timeout; i++) {
if (amdgpu_ring_get_rptr(ring) != rptr)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
@@ -1230,7 +1230,7 @@ static int uvd_v7_0_ring_test_ring(struct amdgpu_ring *ring)
tmp = RREG32_SOC15(UVD, ring->me, mmUVD_CONTEXT_ID);
if (tmp == 0xDEADBEEF)
break;
- DRM_UDELAY(1);
+ udelay(1);
}
if (i >= adev->usec_timeout)
@@ -1762,6 +1762,7 @@ static const struct amdgpu_ring_funcs uvd_v7_0_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_UVD,
.align_mask = 0xf,
.support_64bit_ptrs = false,
+ .no_user_fence = true,
.vmhub = AMDGPU_MMHUB,
.get_rptr = uvd_v7_0_ring_get_rptr,
.get_wptr = uvd_v7_0_ring_get_wptr,
@@ -1794,6 +1795,7 @@ static const struct amdgpu_ring_funcs uvd_v7_0_enc_ring_vm_funcs = {
.align_mask = 0x3f,
.nop = HEVC_ENC_CMD_NO_OP,
.support_64bit_ptrs = false,
+ .no_user_fence = true,
.vmhub = AMDGPU_MMHUB,
.get_rptr = uvd_v7_0_enc_ring_get_rptr,
.get_wptr = uvd_v7_0_enc_ring_get_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
index 40363ca6c5f1..b6837fcfdba7 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
@@ -26,7 +26,7 @@
*/
#include <linux/firmware.h>
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_vce.h"
#include "cikd.h"
@@ -605,6 +605,7 @@ static const struct amdgpu_ring_funcs vce_v2_0_ring_funcs = {
.align_mask = 0xf,
.nop = VCE_CMD_NO_OP,
.support_64bit_ptrs = false,
+ .no_user_fence = true,
.get_rptr = vce_v2_0_ring_get_rptr,
.get_wptr = vce_v2_0_ring_get_wptr,
.set_wptr = vce_v2_0_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
index 6ec65cf11112..475ae68f38f5 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
@@ -26,7 +26,7 @@
*/
#include <linux/firmware.h>
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_vce.h"
#include "vid.h"
@@ -894,6 +894,7 @@ static const struct amdgpu_ring_funcs vce_v3_0_ring_phys_funcs = {
.align_mask = 0xf,
.nop = VCE_CMD_NO_OP,
.support_64bit_ptrs = false,
+ .no_user_fence = true,
.get_rptr = vce_v3_0_ring_get_rptr,
.get_wptr = vce_v3_0_ring_get_wptr,
.set_wptr = vce_v3_0_ring_set_wptr,
@@ -917,6 +918,7 @@ static const struct amdgpu_ring_funcs vce_v3_0_ring_vm_funcs = {
.align_mask = 0xf,
.nop = VCE_CMD_NO_OP,
.support_64bit_ptrs = false,
+ .no_user_fence = true,
.get_rptr = vce_v3_0_ring_get_rptr,
.get_wptr = vce_v3_0_ring_get_wptr,
.set_wptr = vce_v3_0_ring_set_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
index c0ec27991c22..eafbe8d8248d 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
@@ -25,7 +25,7 @@
*/
#include <linux/firmware.h>
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_vce.h"
#include "soc15.h"
@@ -1069,6 +1069,7 @@ static const struct amdgpu_ring_funcs vce_v4_0_ring_vm_funcs = {
.align_mask = 0x3f,
.nop = VCE_CMD_NO_OP,
.support_64bit_ptrs = false,
+ .no_user_fence = true,
.vmhub = AMDGPU_MMHUB,
.get_rptr = vce_v4_0_ring_get_rptr,
.get_wptr = vce_v4_0_ring_get_wptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
index 3dbc51f9d3b9..dde22b7d140d 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
@@ -22,7 +22,7 @@
*/
#include <linux/firmware.h>
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_vcn.h"
#include "soc15.h"
@@ -49,6 +49,8 @@ static void vcn_v1_0_set_jpeg_ring_funcs(struct amdgpu_device *adev);
static void vcn_v1_0_set_irq_funcs(struct amdgpu_device *adev);
static void vcn_v1_0_jpeg_ring_set_patch_ring(struct amdgpu_ring *ring, uint32_t ptr);
static int vcn_v1_0_set_powergating_state(void *handle, enum amd_powergating_state state);
+static int vcn_v1_0_pause_dpg_mode(struct amdgpu_device *adev,
+ struct dpg_pause_state *new_state);
/**
* vcn_v1_0_early_init - set function pointers
@@ -126,6 +128,17 @@ static int vcn_v1_0_sw_init(void *handle)
if (r)
return r;
+ adev->vcn.internal.scratch9 = adev->vcn.external.scratch9 =
+ SOC15_REG_OFFSET(UVD, 0, mmUVD_SCRATCH9);
+ adev->vcn.internal.data0 = adev->vcn.external.data0 =
+ SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0);
+ adev->vcn.internal.data1 = adev->vcn.external.data1 =
+ SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1);
+ adev->vcn.internal.cmd = adev->vcn.external.cmd =
+ SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD);
+ adev->vcn.internal.nop = adev->vcn.external.nop =
+ SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP);
+
for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
ring = &adev->vcn.ring_enc[i];
sprintf(ring->name, "vcn_enc%d", i);
@@ -140,7 +153,11 @@ static int vcn_v1_0_sw_init(void *handle)
if (r)
return r;
- return r;
+ adev->vcn.pause_dpg_mode = vcn_v1_0_pause_dpg_mode;
+ adev->vcn.internal.jpeg_pitch = adev->vcn.external.jpeg_pitch =
+ SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_PITCH);
+
+ return 0;
}
/**
@@ -1204,6 +1221,132 @@ static int vcn_v1_0_stop(struct amdgpu_device *adev)
return r;
}
+static int vcn_v1_0_pause_dpg_mode(struct amdgpu_device *adev,
+ struct dpg_pause_state *new_state)
+{
+ int ret_code;
+ uint32_t reg_data = 0;
+ uint32_t reg_data2 = 0;
+ struct amdgpu_ring *ring;
+
+ /* pause/unpause if state is changed */
+ if (adev->vcn.pause_state.fw_based != new_state->fw_based) {
+ DRM_DEBUG("dpg pause state changed %d:%d -> %d:%d",
+ adev->vcn.pause_state.fw_based, adev->vcn.pause_state.jpeg,
+ new_state->fw_based, new_state->jpeg);
+
+ reg_data = RREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE) &
+ (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
+
+ if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
+ ret_code = 0;
+
+ if (!(reg_data & UVD_DPG_PAUSE__JPEG_PAUSE_DPG_ACK_MASK))
+ SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
+ UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF,
+ UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
+
+ if (!ret_code) {
+ /* pause DPG non-jpeg */
+ reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
+ WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
+ SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_DPG_PAUSE,
+ UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
+ UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK, ret_code);
+
+ /* Restore */
+ ring = &adev->vcn.ring_enc[0];
+ WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
+ WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
+ WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
+ WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
+ WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
+
+ ring = &adev->vcn.ring_enc[1];
+ WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
+ WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
+ WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
+ WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
+ WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
+
+ ring = &adev->vcn.ring_dec;
+ WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
+ RREG32_SOC15(UVD, 0, mmUVD_SCRATCH2) & 0x7FFFFFFF);
+ SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
+ UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON,
+ UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
+ }
+ } else {
+ /* unpause dpg non-jpeg, no need to wait */
+ reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
+ WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
+ }
+ adev->vcn.pause_state.fw_based = new_state->fw_based;
+ }
+
+ /* pause/unpause if state is changed */
+ if (adev->vcn.pause_state.jpeg != new_state->jpeg) {
+ DRM_DEBUG("dpg pause state changed %d:%d -> %d:%d",
+ adev->vcn.pause_state.fw_based, adev->vcn.pause_state.jpeg,
+ new_state->fw_based, new_state->jpeg);
+
+ reg_data = RREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE) &
+ (~UVD_DPG_PAUSE__JPEG_PAUSE_DPG_ACK_MASK);
+
+ if (new_state->jpeg == VCN_DPG_STATE__PAUSE) {
+ ret_code = 0;
+
+ if (!(reg_data & UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK))
+ SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
+ UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF,
+ UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
+
+ if (!ret_code) {
+ /* Make sure JPRG Snoop is disabled before sending the pause */
+ reg_data2 = RREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS);
+ reg_data2 |= UVD_POWER_STATUS__JRBC_SNOOP_DIS_MASK;
+ WREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS, reg_data2);
+
+ /* pause DPG jpeg */
+ reg_data |= UVD_DPG_PAUSE__JPEG_PAUSE_DPG_REQ_MASK;
+ WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
+ SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_DPG_PAUSE,
+ UVD_DPG_PAUSE__JPEG_PAUSE_DPG_ACK_MASK,
+ UVD_DPG_PAUSE__JPEG_PAUSE_DPG_ACK_MASK, ret_code);
+
+ /* Restore */
+ ring = &adev->vcn.ring_jpeg;
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_VMID, 0);
+ WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_CNTL,
+ UVD_JRBC_RB_CNTL__RB_NO_FETCH_MASK |
+ UVD_JRBC_RB_CNTL__RB_RPTR_WR_EN_MASK);
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_LOW,
+ lower_32_bits(ring->gpu_addr));
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
+ upper_32_bits(ring->gpu_addr));
+ WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_RPTR, ring->wptr);
+ WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR, ring->wptr);
+ WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_CNTL,
+ UVD_JRBC_RB_CNTL__RB_RPTR_WR_EN_MASK);
+
+ ring = &adev->vcn.ring_dec;
+ WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
+ RREG32_SOC15(UVD, 0, mmUVD_SCRATCH2) & 0x7FFFFFFF);
+ SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
+ UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON,
+ UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
+ }
+ } else {
+ /* unpause dpg jpeg, no need to wait */
+ reg_data &= ~UVD_DPG_PAUSE__JPEG_PAUSE_DPG_REQ_MASK;
+ WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
+ }
+ adev->vcn.pause_state.jpeg = new_state->jpeg;
+ }
+
+ return 0;
+}
+
static bool vcn_v1_0_is_idle(void *handle)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -2054,6 +2197,7 @@ static const struct amdgpu_ring_funcs vcn_v1_0_dec_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCN_DEC,
.align_mask = 0xf,
.support_64bit_ptrs = false,
+ .no_user_fence = true,
.vmhub = AMDGPU_MMHUB,
.get_rptr = vcn_v1_0_dec_ring_get_rptr,
.get_wptr = vcn_v1_0_dec_ring_get_wptr,
@@ -2087,6 +2231,7 @@ static const struct amdgpu_ring_funcs vcn_v1_0_enc_ring_vm_funcs = {
.align_mask = 0x3f,
.nop = VCN_ENC_CMD_NO_OP,
.support_64bit_ptrs = false,
+ .no_user_fence = true,
.vmhub = AMDGPU_MMHUB,
.get_rptr = vcn_v1_0_enc_ring_get_rptr,
.get_wptr = vcn_v1_0_enc_ring_get_wptr,
@@ -2118,6 +2263,7 @@ static const struct amdgpu_ring_funcs vcn_v1_0_jpeg_ring_vm_funcs = {
.align_mask = 0xf,
.nop = PACKET0(0x81ff, 0),
.support_64bit_ptrs = false,
+ .no_user_fence = true,
.vmhub = AMDGPU_MMHUB,
.extra_dw = 64,
.get_rptr = vcn_v1_0_jpeg_ring_get_rptr,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
new file mode 100644
index 000000000000..dfde886cc6bd
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
@@ -0,0 +1,2288 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/firmware.h>
+#include <drm/drmP.h>
+#include "amdgpu.h"
+#include "amdgpu_vcn.h"
+#include "soc15.h"
+#include "soc15d.h"
+#include "amdgpu_pm.h"
+#include "amdgpu_psp.h"
+
+#include "vcn/vcn_2_0_0_offset.h"
+#include "vcn/vcn_2_0_0_sh_mask.h"
+#include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"
+
+#define mmUVD_CONTEXT_ID_INTERNAL_OFFSET 0x1fd
+#define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET 0x503
+#define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET 0x504
+#define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET 0x505
+#define mmUVD_NO_OP_INTERNAL_OFFSET 0x53f
+#define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET 0x54a
+#define mmUVD_SCRATCH9_INTERNAL_OFFSET 0xc01d
+
+#define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET 0x1e1
+#define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET 0x5a6
+#define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET 0x5a7
+#define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET 0x1e2
+
+#define mmUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET 0x1bfff
+#define mmUVD_JPEG_GPCOM_CMD_INTERNAL_OFFSET 0x4029
+#define mmUVD_JPEG_GPCOM_DATA0_INTERNAL_OFFSET 0x402a
+#define mmUVD_JPEG_GPCOM_DATA1_INTERNAL_OFFSET 0x402b
+#define mmUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_LOW_INTERNAL_OFFSET 0x40ea
+#define mmUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_HIGH_INTERNAL_OFFSET 0x40eb
+#define mmUVD_LMI_JRBC_IB_VMID_INTERNAL_OFFSET 0x40cf
+#define mmUVD_LMI_JPEG_VMID_INTERNAL_OFFSET 0x40d1
+#define mmUVD_LMI_JRBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET 0x40e8
+#define mmUVD_LMI_JRBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET 0x40e9
+#define mmUVD_JRBC_IB_SIZE_INTERNAL_OFFSET 0x4082
+#define mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_LOW_INTERNAL_OFFSET 0x40ec
+#define mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_HIGH_INTERNAL_OFFSET 0x40ed
+#define mmUVD_JRBC_RB_COND_RD_TIMER_INTERNAL_OFFSET 0x4085
+#define mmUVD_JRBC_RB_REF_DATA_INTERNAL_OFFSET 0x4084
+#define mmUVD_JRBC_STATUS_INTERNAL_OFFSET 0x4089
+#define mmUVD_JPEG_PITCH_INTERNAL_OFFSET 0x401f
+
+#define JRBC_DEC_EXTERNAL_REG_WRITE_ADDR 0x18000
+
+#define mmUVD_RBC_XX_IB_REG_CHECK 0x026b
+#define mmUVD_RBC_XX_IB_REG_CHECK_BASE_IDX 1
+#define mmUVD_REG_XX_MASK 0x026c
+#define mmUVD_REG_XX_MASK_BASE_IDX 1
+
+static void vcn_v2_0_set_dec_ring_funcs(struct amdgpu_device *adev);
+static void vcn_v2_0_set_enc_ring_funcs(struct amdgpu_device *adev);
+static void vcn_v2_0_set_jpeg_ring_funcs(struct amdgpu_device *adev);
+static void vcn_v2_0_set_irq_funcs(struct amdgpu_device *adev);
+static int vcn_v2_0_set_powergating_state(void *handle,
+ enum amd_powergating_state state);
+static int vcn_v2_0_pause_dpg_mode(struct amdgpu_device *adev,
+ struct dpg_pause_state *new_state);
+
+/**
+ * vcn_v2_0_early_init - set function pointers
+ *
+ * @handle: amdgpu_device pointer
+ *
+ * Set ring and irq function pointers
+ */
+static int vcn_v2_0_early_init(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ adev->vcn.num_enc_rings = 2;
+
+ vcn_v2_0_set_dec_ring_funcs(adev);
+ vcn_v2_0_set_enc_ring_funcs(adev);
+ vcn_v2_0_set_jpeg_ring_funcs(adev);
+ vcn_v2_0_set_irq_funcs(adev);
+
+ return 0;
+}
+
+/**
+ * vcn_v2_0_sw_init - sw init for VCN block
+ *
+ * @handle: amdgpu_device pointer
+ *
+ * Load firmware and sw initialization
+ */
+static int vcn_v2_0_sw_init(void *handle)
+{
+ struct amdgpu_ring *ring;
+ int i, r;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ /* VCN DEC TRAP */
+ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
+ VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT,
+ &adev->vcn.irq);
+ if (r)
+ return r;
+
+ /* VCN ENC TRAP */
+ for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
+ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
+ i + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE,
+ &adev->vcn.irq);
+ if (r)
+ return r;
+ }
+
+ /* VCN JPEG TRAP */
+ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
+ VCN_2_0__SRCID__JPEG_DECODE,
+ &adev->vcn.irq);
+ if (r)
+ return r;
+
+ r = amdgpu_vcn_sw_init(adev);
+ if (r)
+ return r;
+
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+ const struct common_firmware_header *hdr;
+ hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
+ adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].ucode_id = AMDGPU_UCODE_ID_VCN;
+ adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].fw = adev->vcn.fw;
+ adev->firmware.fw_size +=
+ ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);
+ DRM_INFO("PSP loading VCN firmware\n");
+ }
+
+ r = amdgpu_vcn_resume(adev);
+ if (r)
+ return r;
+
+ ring = &adev->vcn.ring_dec;
+
+ ring->use_doorbell = true;
+ ring->doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1 << 1;
+
+ sprintf(ring->name, "vcn_dec");
+ r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.irq, 0);
+ if (r)
+ return r;
+
+ adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;
+ adev->vcn.external.scratch9 = SOC15_REG_OFFSET(UVD, 0, mmUVD_SCRATCH9);
+ adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;
+ adev->vcn.external.data0 = SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0);
+ adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;
+ adev->vcn.external.data1 = SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1);
+ adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;
+ adev->vcn.external.cmd = SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD);
+ adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;
+ adev->vcn.external.nop = SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP);
+
+ for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
+ ring = &adev->vcn.ring_enc[i];
+ ring->use_doorbell = true;
+ ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + i;
+ sprintf(ring->name, "vcn_enc%d", i);
+ r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.irq, 0);
+ if (r)
+ return r;
+ }
+
+ ring = &adev->vcn.ring_jpeg;
+ ring->use_doorbell = true;
+ ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1;
+ sprintf(ring->name, "vcn_jpeg");
+ r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.irq, 0);
+ if (r)
+ return r;
+
+ adev->vcn.pause_dpg_mode = vcn_v2_0_pause_dpg_mode;
+
+ adev->vcn.internal.jpeg_pitch = mmUVD_JPEG_PITCH_INTERNAL_OFFSET;
+ adev->vcn.external.jpeg_pitch = SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_PITCH);
+
+ return 0;
+}
+
+/**
+ * vcn_v2_0_sw_fini - sw fini for VCN block
+ *
+ * @handle: amdgpu_device pointer
+ *
+ * VCN suspend and free up sw allocation
+ */
+static int vcn_v2_0_sw_fini(void *handle)
+{
+ int r;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ r = amdgpu_vcn_suspend(adev);
+ if (r)
+ return r;
+
+ r = amdgpu_vcn_sw_fini(adev);
+
+ return r;
+}
+
+/**
+ * vcn_v2_0_hw_init - start and test VCN block
+ *
+ * @handle: amdgpu_device pointer
+ *
+ * Initialize the hardware, boot up the VCPU and do some testing
+ */
+static int vcn_v2_0_hw_init(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ struct amdgpu_ring *ring = &adev->vcn.ring_dec;
+ int i, r;
+
+ adev->nbio_funcs->vcn_doorbell_range(adev, ring->use_doorbell,
+ ring->doorbell_index);
+
+ ring->sched.ready = true;
+ r = amdgpu_ring_test_ring(ring);
+ if (r) {
+ ring->sched.ready = false;
+ goto done;
+ }
+
+ for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
+ ring = &adev->vcn.ring_enc[i];
+ ring->sched.ready = true;
+ r = amdgpu_ring_test_ring(ring);
+ if (r) {
+ ring->sched.ready = false;
+ goto done;
+ }
+ }
+
+ ring = &adev->vcn.ring_jpeg;
+ ring->sched.ready = true;
+ r = amdgpu_ring_test_ring(ring);
+ if (r) {
+ ring->sched.ready = false;
+ goto done;
+ }
+
+done:
+ if (!r)
+ DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
+ (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
+
+ return r;
+}
+
+/**
+ * vcn_v2_0_hw_fini - stop the hardware block
+ *
+ * @handle: amdgpu_device pointer
+ *
+ * Stop the VCN block, mark ring as not ready any more
+ */
+static int vcn_v2_0_hw_fini(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ struct amdgpu_ring *ring = &adev->vcn.ring_dec;
+ int i;
+
+ if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
+ (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
+ RREG32_SOC15(VCN, 0, mmUVD_STATUS)))
+ vcn_v2_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
+
+ ring->sched.ready = false;
+
+ for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
+ ring = &adev->vcn.ring_enc[i];
+ ring->sched.ready = false;
+ }
+
+ ring = &adev->vcn.ring_jpeg;
+ ring->sched.ready = false;
+
+ return 0;
+}
+
+/**
+ * vcn_v2_0_suspend - suspend VCN block
+ *
+ * @handle: amdgpu_device pointer
+ *
+ * HW fini and suspend VCN block
+ */
+static int vcn_v2_0_suspend(void *handle)
+{
+ int r;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ r = vcn_v2_0_hw_fini(adev);
+ if (r)
+ return r;
+
+ r = amdgpu_vcn_suspend(adev);
+
+ return r;
+}
+
+/**
+ * vcn_v2_0_resume - resume VCN block
+ *
+ * @handle: amdgpu_device pointer
+ *
+ * Resume firmware and hw init VCN block
+ */
+static int vcn_v2_0_resume(void *handle)
+{
+ int r;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ r = amdgpu_vcn_resume(adev);
+ if (r)
+ return r;
+
+ r = vcn_v2_0_hw_init(adev);
+
+ return r;
+}
+
+/**
+ * vcn_v2_0_mc_resume - memory controller programming
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Let the VCN memory controller know it's offsets
+ */
+static void vcn_v2_0_mc_resume(struct amdgpu_device *adev)
+{
+ uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
+ uint32_t offset;
+
+ /* cache window 0: fw */
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
+ (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo));
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
+ (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi));
+ WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0, 0);
+ offset = 0;
+ } else {
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
+ lower_32_bits(adev->vcn.gpu_addr));
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
+ upper_32_bits(adev->vcn.gpu_addr));
+ offset = size;
+ WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0,
+ AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
+ }
+
+ WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE0, size);
+
+ /* cache window 1: stack */
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
+ lower_32_bits(adev->vcn.gpu_addr + offset));
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
+ upper_32_bits(adev->vcn.gpu_addr + offset));
+ WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1, 0);
+ WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
+
+ /* cache window 2: context */
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
+ lower_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
+ upper_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
+ WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2, 0);
+ WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
+
+ WREG32_SOC15(UVD, 0, mmUVD_GFX10_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
+ WREG32_SOC15(UVD, 0, mmJPEG_DEC_GFX10_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
+}
+
+static void vcn_v2_0_mc_resume_dpg_mode(struct amdgpu_device *adev, bool indirect)
+{
+ uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
+ uint32_t offset;
+
+ /* cache window 0: fw */
+ if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+ if (!indirect) {
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
+ (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo), 0, indirect);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
+ (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi), 0, indirect);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
+ } else {
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
+ }
+ offset = 0;
+ } else {
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
+ lower_32_bits(adev->vcn.gpu_addr), 0, indirect);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
+ upper_32_bits(adev->vcn.gpu_addr), 0, indirect);
+ offset = size;
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_VCPU_CACHE_OFFSET0),
+ AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
+ }
+
+ if (!indirect)
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
+ else
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
+
+ /* cache window 1: stack */
+ if (!indirect) {
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
+ lower_32_bits(adev->vcn.gpu_addr + offset), 0, indirect);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
+ upper_32_bits(adev->vcn.gpu_addr + offset), 0, indirect);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
+ } else {
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
+ }
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
+
+ /* cache window 2: context */
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
+ lower_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
+ upper_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
+
+ /* non-cache window */
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), 0, 0, indirect);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), 0, 0, indirect);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_VCPU_NONCACHE_SIZE0), 0, 0, indirect);
+
+ /* VCN global tiling registers */
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
+}
+
+/**
+ * vcn_v2_0_disable_clock_gating - disable VCN clock gating
+ *
+ * @adev: amdgpu_device pointer
+ * @sw: enable SW clock gating
+ *
+ * Disable clock gating for VCN block
+ */
+static void vcn_v2_0_disable_clock_gating(struct amdgpu_device *adev)
+{
+ uint32_t data;
+
+ /* UVD disable CGC */
+ data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
+ if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
+ data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
+ else
+ data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
+ data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
+ data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
+ WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
+
+ data = RREG32_SOC15(VCN, 0, mmUVD_CGC_GATE);
+ data &= ~(UVD_CGC_GATE__SYS_MASK
+ | UVD_CGC_GATE__UDEC_MASK
+ | UVD_CGC_GATE__MPEG2_MASK
+ | UVD_CGC_GATE__REGS_MASK
+ | UVD_CGC_GATE__RBC_MASK
+ | UVD_CGC_GATE__LMI_MC_MASK
+ | UVD_CGC_GATE__LMI_UMC_MASK
+ | UVD_CGC_GATE__IDCT_MASK
+ | UVD_CGC_GATE__MPRD_MASK
+ | UVD_CGC_GATE__MPC_MASK
+ | UVD_CGC_GATE__LBSI_MASK
+ | UVD_CGC_GATE__LRBBM_MASK
+ | UVD_CGC_GATE__UDEC_RE_MASK
+ | UVD_CGC_GATE__UDEC_CM_MASK
+ | UVD_CGC_GATE__UDEC_IT_MASK
+ | UVD_CGC_GATE__UDEC_DB_MASK
+ | UVD_CGC_GATE__UDEC_MP_MASK
+ | UVD_CGC_GATE__WCB_MASK
+ | UVD_CGC_GATE__VCPU_MASK
+ | UVD_CGC_GATE__SCPU_MASK);
+ WREG32_SOC15(VCN, 0, mmUVD_CGC_GATE, data);
+
+ data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
+ data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
+ | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
+ | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
+ | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
+ | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
+ | UVD_CGC_CTRL__SYS_MODE_MASK
+ | UVD_CGC_CTRL__UDEC_MODE_MASK
+ | UVD_CGC_CTRL__MPEG2_MODE_MASK
+ | UVD_CGC_CTRL__REGS_MODE_MASK
+ | UVD_CGC_CTRL__RBC_MODE_MASK
+ | UVD_CGC_CTRL__LMI_MC_MODE_MASK
+ | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
+ | UVD_CGC_CTRL__IDCT_MODE_MASK
+ | UVD_CGC_CTRL__MPRD_MODE_MASK
+ | UVD_CGC_CTRL__MPC_MODE_MASK
+ | UVD_CGC_CTRL__LBSI_MODE_MASK
+ | UVD_CGC_CTRL__LRBBM_MODE_MASK
+ | UVD_CGC_CTRL__WCB_MODE_MASK
+ | UVD_CGC_CTRL__VCPU_MODE_MASK
+ | UVD_CGC_CTRL__SCPU_MODE_MASK);
+ WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
+
+ /* turn on */
+ data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE);
+ data |= (UVD_SUVD_CGC_GATE__SRE_MASK
+ | UVD_SUVD_CGC_GATE__SIT_MASK
+ | UVD_SUVD_CGC_GATE__SMP_MASK
+ | UVD_SUVD_CGC_GATE__SCM_MASK
+ | UVD_SUVD_CGC_GATE__SDB_MASK
+ | UVD_SUVD_CGC_GATE__SRE_H264_MASK
+ | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
+ | UVD_SUVD_CGC_GATE__SIT_H264_MASK
+ | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
+ | UVD_SUVD_CGC_GATE__SCM_H264_MASK
+ | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
+ | UVD_SUVD_CGC_GATE__SDB_H264_MASK
+ | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
+ | UVD_SUVD_CGC_GATE__SCLR_MASK
+ | UVD_SUVD_CGC_GATE__UVD_SC_MASK
+ | UVD_SUVD_CGC_GATE__ENT_MASK
+ | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
+ | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
+ | UVD_SUVD_CGC_GATE__SITE_MASK
+ | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
+ | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
+ | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
+ | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
+ | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
+ WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE, data);
+
+ data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
+ data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
+ WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
+}
+
+static void vcn_v2_0_clock_gating_dpg_mode(struct amdgpu_device *adev,
+ uint8_t sram_sel, uint8_t indirect)
+{
+ uint32_t reg_data = 0;
+
+ /* enable sw clock gating control */
+ if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
+ reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
+ else
+ reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
+ reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
+ reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
+ reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
+ UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
+ UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
+ UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
+ UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
+ UVD_CGC_CTRL__SYS_MODE_MASK |
+ UVD_CGC_CTRL__UDEC_MODE_MASK |
+ UVD_CGC_CTRL__MPEG2_MODE_MASK |
+ UVD_CGC_CTRL__REGS_MODE_MASK |
+ UVD_CGC_CTRL__RBC_MODE_MASK |
+ UVD_CGC_CTRL__LMI_MC_MODE_MASK |
+ UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
+ UVD_CGC_CTRL__IDCT_MODE_MASK |
+ UVD_CGC_CTRL__MPRD_MODE_MASK |
+ UVD_CGC_CTRL__MPC_MODE_MASK |
+ UVD_CGC_CTRL__LBSI_MODE_MASK |
+ UVD_CGC_CTRL__LRBBM_MODE_MASK |
+ UVD_CGC_CTRL__WCB_MODE_MASK |
+ UVD_CGC_CTRL__VCPU_MODE_MASK |
+ UVD_CGC_CTRL__SCPU_MODE_MASK);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);
+
+ /* turn off clock gating */
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_CGC_GATE), 0, sram_sel, indirect);
+
+ /* turn on SUVD clock gating */
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
+
+ /* turn on sw mode in UVD_SUVD_CGC_CTRL */
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
+}
+
+/**
+ * jpeg_v2_0_start - start JPEG block
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Setup and start the JPEG block
+ */
+static int jpeg_v2_0_start(struct amdgpu_device *adev)
+{
+ struct amdgpu_ring *ring = &adev->vcn.ring_jpeg;
+ uint32_t tmp;
+ int r = 0;
+
+ /* disable power gating */
+ tmp = 1 << UVD_PGFSM_CONFIG__UVDJ_PWR_CONFIG__SHIFT;
+ WREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_PGFSM_CONFIG), tmp);
+
+ SOC15_WAIT_ON_RREG(VCN, 0,
+ mmUVD_PGFSM_STATUS, UVD_PGFSM_STATUS_UVDJ_PWR_ON,
+ UVD_PGFSM_STATUS__UVDJ_PWR_STATUS_MASK, r);
+
+ if (r) {
+ DRM_ERROR("amdgpu: JPEG disable power gating failed\n");
+ return r;
+ }
+
+ /* Removing the anti hang mechanism to indicate the UVDJ tile is ON */
+ tmp = RREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_POWER_STATUS)) & ~0x1;
+ WREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_POWER_STATUS), tmp);
+
+ /* JPEG disable CGC */
+ tmp = RREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL);
+ tmp |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
+ tmp |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
+ tmp |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
+ WREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL, tmp);
+
+ tmp = RREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE);
+ tmp &= ~(JPEG_CGC_GATE__JPEG_DEC_MASK
+ | JPEG_CGC_GATE__JPEG2_DEC_MASK
+ | JPEG_CGC_GATE__JPEG_ENC_MASK
+ | JPEG_CGC_GATE__JMCIF_MASK
+ | JPEG_CGC_GATE__JRBBM_MASK);
+ WREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE, tmp);
+
+ /* enable JMI channel */
+ WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_JMI_CNTL), 0,
+ ~UVD_JMI_CNTL__SOFT_RESET_MASK);
+
+ /* enable System Interrupt for JRBC */
+ WREG32_P(SOC15_REG_OFFSET(VCN, 0, mmJPEG_SYS_INT_EN),
+ JPEG_SYS_INT_EN__DJRBC_MASK,
+ ~JPEG_SYS_INT_EN__DJRBC_MASK);
+
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_VMID, 0);
+ WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_CNTL, (0x00000001L | 0x00000002L));
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_LOW,
+ lower_32_bits(ring->gpu_addr));
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
+ upper_32_bits(ring->gpu_addr));
+ WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_RPTR, 0);
+ WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR, 0);
+ WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_CNTL, 0x00000002L);
+ WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_SIZE, ring->ring_size / 4);
+ ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR);
+
+ return 0;
+}
+
+/**
+ * jpeg_v2_0_stop - stop JPEG block
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * stop the JPEG block
+ */
+static int jpeg_v2_0_stop(struct amdgpu_device *adev)
+{
+ uint32_t tmp;
+ int r = 0;
+
+ /* reset JMI */
+ WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_JMI_CNTL),
+ UVD_JMI_CNTL__SOFT_RESET_MASK,
+ ~UVD_JMI_CNTL__SOFT_RESET_MASK);
+
+ /* enable JPEG CGC */
+ tmp = RREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL);
+ tmp |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
+ tmp |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
+ tmp |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
+ WREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL, tmp);
+
+
+ tmp = RREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE);
+ tmp |= (JPEG_CGC_GATE__JPEG_DEC_MASK
+ |JPEG_CGC_GATE__JPEG2_DEC_MASK
+ |JPEG_CGC_GATE__JPEG_ENC_MASK
+ |JPEG_CGC_GATE__JMCIF_MASK
+ |JPEG_CGC_GATE__JRBBM_MASK);
+ WREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE, tmp);
+
+ /* enable power gating */
+ tmp = RREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_POWER_STATUS));
+ tmp &= ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK;
+ tmp |= 0x1; //UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_TILES_OFF;
+ WREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_POWER_STATUS), tmp);
+
+ tmp = 2 << UVD_PGFSM_CONFIG__UVDJ_PWR_CONFIG__SHIFT;
+ WREG32(SOC15_REG_OFFSET(UVD, 0, mmUVD_PGFSM_CONFIG), tmp);
+
+ SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS,
+ (2 << UVD_PGFSM_STATUS__UVDJ_PWR_STATUS__SHIFT),
+ UVD_PGFSM_STATUS__UVDJ_PWR_STATUS_MASK, r);
+
+ if (r) {
+ DRM_ERROR("amdgpu: JPEG enable power gating failed\n");
+ return r;
+ }
+
+ return r;
+}
+
+/**
+ * vcn_v2_0_enable_clock_gating - enable VCN clock gating
+ *
+ * @adev: amdgpu_device pointer
+ * @sw: enable SW clock gating
+ *
+ * Enable clock gating for VCN block
+ */
+static void vcn_v2_0_enable_clock_gating(struct amdgpu_device *adev)
+{
+ uint32_t data = 0;
+
+ /* enable UVD CGC */
+ data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
+ if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
+ data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
+ else
+ data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
+ data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
+ data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
+ WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
+
+ data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
+ data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
+ | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
+ | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
+ | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
+ | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
+ | UVD_CGC_CTRL__SYS_MODE_MASK
+ | UVD_CGC_CTRL__UDEC_MODE_MASK
+ | UVD_CGC_CTRL__MPEG2_MODE_MASK
+ | UVD_CGC_CTRL__REGS_MODE_MASK
+ | UVD_CGC_CTRL__RBC_MODE_MASK
+ | UVD_CGC_CTRL__LMI_MC_MODE_MASK
+ | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
+ | UVD_CGC_CTRL__IDCT_MODE_MASK
+ | UVD_CGC_CTRL__MPRD_MODE_MASK
+ | UVD_CGC_CTRL__MPC_MODE_MASK
+ | UVD_CGC_CTRL__LBSI_MODE_MASK
+ | UVD_CGC_CTRL__LRBBM_MODE_MASK
+ | UVD_CGC_CTRL__WCB_MODE_MASK
+ | UVD_CGC_CTRL__VCPU_MODE_MASK
+ | UVD_CGC_CTRL__SCPU_MODE_MASK);
+ WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
+
+ data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
+ data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
+ WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
+}
+
+static void vcn_v2_0_disable_static_power_gating(struct amdgpu_device *adev)
+{
+ uint32_t data = 0;
+ int ret;
+
+ if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
+ data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
+ | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
+ | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
+ | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
+ | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
+ | 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
+ | 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
+ | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
+ | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
+ | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT);
+
+ WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
+ SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS,
+ UVD_PGFSM_STATUS__UVDM_UVDU_PWR_ON_2_0, 0xFFFFF, ret);
+ } else {
+ data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
+ | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
+ | 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
+ | 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
+ | 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
+ | 1 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
+ | 1 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
+ | 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
+ | 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
+ | 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT);
+ WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
+ SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, 0, 0xFFFFF, ret);
+ }
+
+ /* polling UVD_PGFSM_STATUS to confirm UVDM_PWR_STATUS,
+ * UVDU_PWR_STATUS are 0 (power on) */
+
+ data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS);
+ data &= ~0x103;
+ if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
+ data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
+ UVD_POWER_STATUS__UVD_PG_EN_MASK;
+
+ WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data);
+}
+
+static void vcn_v2_0_enable_static_power_gating(struct amdgpu_device *adev)
+{
+ uint32_t data = 0;
+ int ret;
+
+ if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
+ /* Before power off, this indicator has to be turned on */
+ data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS);
+ data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
+ data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
+ WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data);
+
+
+ data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
+ | 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
+ | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
+ | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
+ | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
+ | 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
+ | 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
+ | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
+ | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
+ | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT);
+
+ WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
+
+ data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
+ | 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT
+ | 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
+ | 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT
+ | 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
+ | 2 << UVD_PGFSM_STATUS__UVDIL_PWR_STATUS__SHIFT
+ | 2 << UVD_PGFSM_STATUS__UVDIR_PWR_STATUS__SHIFT
+ | 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
+ | 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
+ | 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT);
+ SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, data, 0xFFFFF, ret);
+ }
+}
+
+static int vcn_v2_0_start_dpg_mode(struct amdgpu_device *adev, bool indirect)
+{
+ struct amdgpu_ring *ring = &adev->vcn.ring_dec;
+ uint32_t rb_bufsz, tmp;
+
+ vcn_v2_0_enable_static_power_gating(adev);
+
+ /* enable dynamic power gating mode */
+ tmp = RREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS);
+ tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
+ tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
+ WREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS, tmp);
+
+ if (indirect)
+ adev->vcn.dpg_sram_curr_addr = (uint32_t*)adev->vcn.dpg_sram_cpu_addr;
+
+ /* enable clock gating */
+ vcn_v2_0_clock_gating_dpg_mode(adev, 0, indirect);
+
+ /* enable VCPU clock */
+ tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
+ tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
+ tmp |= UVD_VCPU_CNTL__MIF_WR_LOW_THRESHOLD_BP_MASK;
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect);
+
+ /* disable master interupt */
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_MASTINT_EN), 0, 0, indirect);
+
+ /* setup mmUVD_LMI_CTRL */
+ tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
+ UVD_LMI_CTRL__REQ_MODE_MASK |
+ UVD_LMI_CTRL__CRC_RESET_MASK |
+ UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
+ UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
+ UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
+ (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
+ 0x00100000L);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_LMI_CTRL), tmp, 0, indirect);
+
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_MPC_CNTL),
+ 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
+
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_MPC_SET_MUXA0),
+ ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
+ (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
+ (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
+ (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
+
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_MPC_SET_MUXB0),
+ ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
+ (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
+ (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
+ (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
+
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_MPC_SET_MUX),
+ ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
+ (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
+ (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
+
+ vcn_v2_0_mc_resume_dpg_mode(adev, indirect);
+
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_REG_XX_MASK), 0x10, 0, indirect);
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);
+
+ /* release VCPU reset to boot */
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_SOFT_RESET), 0, 0, indirect);
+
+ /* enable LMI MC and UMC channels */
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_LMI_CTRL2),
+ 0x1F << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT, 0, indirect);
+
+ /* enable master interrupt */
+ WREG32_SOC15_DPG_MODE_2_0(SOC15_DPG_MODE_OFFSET_2_0(
+ UVD, 0, mmUVD_MASTINT_EN),
+ UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
+
+ if (indirect)
+ psp_update_vcn_sram(adev, 0, adev->vcn.dpg_sram_gpu_addr,
+ (uint32_t)((uintptr_t)adev->vcn.dpg_sram_curr_addr -
+ (uintptr_t)adev->vcn.dpg_sram_cpu_addr));
+
+ /* force RBC into idle state */
+ rb_bufsz = order_base_2(ring->ring_size);
+ tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
+ tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
+ tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
+ tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
+ tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
+ WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
+
+ /* set the write pointer delay */
+ WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR_CNTL, 0);
+
+ /* set the wb address */
+ WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR_ADDR,
+ (upper_32_bits(ring->gpu_addr) >> 2));
+
+ /* programm the RB_BASE for ring buffer */
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
+ lower_32_bits(ring->gpu_addr));
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
+ upper_32_bits(ring->gpu_addr));
+
+ /* Initialize the ring buffer's read and write pointers */
+ WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
+
+ WREG32_SOC15(UVD, 0, mmUVD_SCRATCH2, 0);
+
+ ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
+ WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
+ lower_32_bits(ring->wptr));
+
+ return 0;
+}
+
+static int vcn_v2_0_start(struct amdgpu_device *adev)
+{
+ struct amdgpu_ring *ring = &adev->vcn.ring_dec;
+ uint32_t rb_bufsz, tmp;
+ uint32_t lmi_swap_cntl;
+ int i, j, r;
+
+ if (adev->pm.dpm_enabled)
+ amdgpu_dpm_enable_uvd(adev, true);
+
+ if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
+ r = vcn_v2_0_start_dpg_mode(adev, adev->vcn.indirect_sram);
+ if (r)
+ return r;
+ goto jpeg;
+ }
+
+ vcn_v2_0_disable_static_power_gating(adev);
+
+ /* set uvd status busy */
+ tmp = RREG32_SOC15(UVD, 0, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
+ WREG32_SOC15(UVD, 0, mmUVD_STATUS, tmp);
+
+ /*SW clock gating */
+ vcn_v2_0_disable_clock_gating(adev);
+
+ /* enable VCPU clock */
+ WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CNTL),
+ UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
+
+ /* disable master interrupt */
+ WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN), 0,
+ ~UVD_MASTINT_EN__VCPU_EN_MASK);
+
+ /* setup mmUVD_LMI_CTRL */
+ tmp = RREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL);
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL, tmp |
+ UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
+ UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
+ UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
+ UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
+
+ /* setup mmUVD_MPC_CNTL */
+ tmp = RREG32_SOC15(UVD, 0, mmUVD_MPC_CNTL);
+ tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
+ tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
+ WREG32_SOC15(VCN, 0, mmUVD_MPC_CNTL, tmp);
+
+ /* setup UVD_MPC_SET_MUXA0 */
+ WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXA0,
+ ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
+ (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
+ (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
+ (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
+
+ /* setup UVD_MPC_SET_MUXB0 */
+ WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXB0,
+ ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
+ (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
+ (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
+ (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
+
+ /* setup mmUVD_MPC_SET_MUX */
+ WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUX,
+ ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
+ (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
+ (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
+
+ vcn_v2_0_mc_resume(adev);
+
+ /* release VCPU reset to boot */
+ WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0,
+ ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
+
+ /* enable LMI MC and UMC channels */
+ WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0,
+ ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
+
+ tmp = RREG32_SOC15(VCN, 0, mmUVD_SOFT_RESET);
+ tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
+ tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
+ WREG32_SOC15(VCN, 0, mmUVD_SOFT_RESET, tmp);
+
+ /* disable byte swapping */
+ lmi_swap_cntl = 0;
+#ifdef __BIG_ENDIAN
+ /* swap (8 in 32) RB and IB */
+ lmi_swap_cntl = 0xa;
+#endif
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl);
+
+ for (i = 0; i < 10; ++i) {
+ uint32_t status;
+
+ for (j = 0; j < 100; ++j) {
+ status = RREG32_SOC15(UVD, 0, mmUVD_STATUS);
+ if (status & 2)
+ break;
+ mdelay(10);
+ }
+ r = 0;
+ if (status & 2)
+ break;
+
+ DRM_ERROR("VCN decode not responding, trying to reset the VCPU!!!\n");
+ WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
+ UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
+ ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
+ mdelay(10);
+ WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0,
+ ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
+ mdelay(10);
+ r = -1;
+ }
+
+ if (r) {
+ DRM_ERROR("VCN decode not responding, giving up!!!\n");
+ return r;
+ }
+
+ /* enable master interrupt */
+ WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
+ UVD_MASTINT_EN__VCPU_EN_MASK,
+ ~UVD_MASTINT_EN__VCPU_EN_MASK);
+
+ /* clear the busy bit of VCN_STATUS */
+ WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0,
+ ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
+
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_VMID, 0);
+
+ /* force RBC into idle state */
+ rb_bufsz = order_base_2(ring->ring_size);
+ tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
+ tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
+ tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
+ tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
+ tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
+ WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
+
+ /* programm the RB_BASE for ring buffer */
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
+ lower_32_bits(ring->gpu_addr));
+ WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
+ upper_32_bits(ring->gpu_addr));
+
+ /* Initialize the ring buffer's read and write pointers */
+ WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
+
+ ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
+ WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
+ lower_32_bits(ring->wptr));
+
+ ring = &adev->vcn.ring_enc[0];
+ WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
+ WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
+ WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
+ WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
+ WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
+
+ ring = &adev->vcn.ring_enc[1];
+ WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
+ WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
+ WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
+ WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
+ WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
+
+jpeg:
+ r = jpeg_v2_0_start(adev);
+
+ return r;
+}
+
+static int vcn_v2_0_stop_dpg_mode(struct amdgpu_device *adev)
+{
+ int ret_code = 0;
+ uint32_t tmp;
+
+ /* Wait for power status to be 1 */
+ SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS, 1,
+ UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
+
+ /* wait for read ptr to be equal to write ptr */
+ tmp = RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR);
+ SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF, ret_code);
+
+ tmp = RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2);
+ SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF, ret_code);
+
+ tmp = RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR);
+ SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_JRBC_RB_RPTR, tmp, 0xFFFFFFFF, ret_code);
+
+ tmp = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
+ SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF, ret_code);
+
+ SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS, 1,
+ UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
+
+ /* disable dynamic power gating mode */
+ WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS), 0,
+ ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
+
+ return 0;
+}
+
+static int vcn_v2_0_stop(struct amdgpu_device *adev)
+{
+ uint32_t tmp;
+ int r;
+
+ r = jpeg_v2_0_stop(adev);
+ if (r)
+ return r;
+
+ if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
+ r = vcn_v2_0_stop_dpg_mode(adev);
+ if (r)
+ return r;
+ goto power_off;
+ }
+
+ /* wait for uvd idle */
+ SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7, r);
+ if (r)
+ return r;
+
+ tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
+ UVD_LMI_STATUS__READ_CLEAN_MASK |
+ UVD_LMI_STATUS__WRITE_CLEAN_MASK |
+ UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
+ SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_LMI_STATUS, tmp, tmp, r);
+ if (r)
+ return r;
+
+ /* stall UMC channel */
+ tmp = RREG32_SOC15(VCN, 0, mmUVD_LMI_CTRL2);
+ tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
+ WREG32_SOC15(VCN, 0, mmUVD_LMI_CTRL2, tmp);
+
+ tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|
+ UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
+ SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_LMI_STATUS, tmp, tmp, r);
+ if (r)
+ return r;
+
+ /* disable VCPU clock */
+ WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CNTL), 0,
+ ~(UVD_VCPU_CNTL__CLK_EN_MASK));
+
+ /* reset LMI UMC */
+ WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
+ UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK,
+ ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK);
+
+ /* reset LMI */
+ WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
+ UVD_SOFT_RESET__LMI_SOFT_RESET_MASK,
+ ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK);
+
+ /* reset VCPU */
+ WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
+ UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
+ ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
+
+ /* clear status */
+ WREG32_SOC15(VCN, 0, mmUVD_STATUS, 0);
+
+ vcn_v2_0_enable_clock_gating(adev);
+ vcn_v2_0_enable_static_power_gating(adev);
+
+power_off:
+ if (adev->pm.dpm_enabled)
+ amdgpu_dpm_enable_uvd(adev, false);
+
+ return 0;
+}
+
+static int vcn_v2_0_pause_dpg_mode(struct amdgpu_device *adev,
+ struct dpg_pause_state *new_state)
+{
+ struct amdgpu_ring *ring;
+ uint32_t reg_data = 0;
+ int ret_code;
+
+ /* pause/unpause if state is changed */
+ if (adev->vcn.pause_state.fw_based != new_state->fw_based) {
+ DRM_DEBUG("dpg pause state changed %d -> %d",
+ adev->vcn.pause_state.fw_based, new_state->fw_based);
+ reg_data = RREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE) &
+ (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
+
+ if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
+ ret_code = 0;
+ SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS, 0x1,
+ UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
+
+ if (!ret_code) {
+ /* pause DPG */
+ reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
+ WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
+
+ /* wait for ACK */
+ SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_DPG_PAUSE,
+ UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
+ UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK, ret_code);
+
+ /* Restore */
+ ring = &adev->vcn.ring_enc[0];
+ WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
+ WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
+ WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
+ WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
+ WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
+
+ ring = &adev->vcn.ring_enc[1];
+ WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
+ WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
+ WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
+ WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
+ WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
+
+ WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
+ RREG32_SOC15(UVD, 0, mmUVD_SCRATCH2) & 0x7FFFFFFF);
+
+ SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
+ UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON,
+ UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
+ }
+ } else {
+ /* unpause dpg, no need to wait */
+ reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
+ WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
+ }
+ adev->vcn.pause_state.fw_based = new_state->fw_based;
+ }
+
+ return 0;
+}
+
+static bool vcn_v2_0_is_idle(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ return (RREG32_SOC15(VCN, 0, mmUVD_STATUS) == UVD_STATUS__IDLE);
+}
+
+static int vcn_v2_0_wait_for_idle(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ int ret = 0;
+
+ SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_STATUS, UVD_STATUS__IDLE,
+ UVD_STATUS__IDLE, ret);
+
+ return ret;
+}
+
+static int vcn_v2_0_set_clockgating_state(void *handle,
+ enum amd_clockgating_state state)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
+
+ if (enable) {
+ /* wait for STATUS to clear */
+ if (vcn_v2_0_is_idle(handle))
+ return -EBUSY;
+ vcn_v2_0_enable_clock_gating(adev);
+ } else {
+ /* disable HW gating and enable Sw gating */
+ vcn_v2_0_disable_clock_gating(adev);
+ }
+ return 0;
+}
+
+/**
+ * vcn_v2_0_dec_ring_get_rptr - get read pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Returns the current hardware read pointer
+ */
+static uint64_t vcn_v2_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
+}
+
+/**
+ * vcn_v2_0_dec_ring_get_wptr - get write pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Returns the current hardware write pointer
+ */
+static uint64_t vcn_v2_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ if (ring->use_doorbell)
+ return adev->wb.wb[ring->wptr_offs];
+ else
+ return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR);
+}
+
+/**
+ * vcn_v2_0_dec_ring_set_wptr - set write pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Commits the write pointer to the hardware
+ */
+static void vcn_v2_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
+ WREG32_SOC15(UVD, 0, mmUVD_SCRATCH2,
+ lower_32_bits(ring->wptr) | 0x80000000);
+
+ if (ring->use_doorbell) {
+ adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
+ WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
+ } else {
+ WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
+ }
+}
+
+/**
+ * vcn_v2_0_dec_ring_insert_start - insert a start command
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Write a start command to the ring.
+ */
+static void vcn_v2_0_dec_ring_insert_start(struct amdgpu_ring *ring)
+{
+ amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, 0);
+ amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_PACKET_START << 1));
+}
+
+/**
+ * vcn_v2_0_dec_ring_insert_end - insert a end command
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Write a end command to the ring.
+ */
+static void vcn_v2_0_dec_ring_insert_end(struct amdgpu_ring *ring)
+{
+ amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_PACKET_END << 1));
+}
+
+/**
+ * vcn_v2_0_dec_ring_insert_nop - insert a nop command
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Write a nop command to the ring.
+ */
+static void vcn_v2_0_dec_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
+{
+ int i;
+
+ WARN_ON(ring->wptr % 2 || count % 2);
+
+ for (i = 0; i < count / 2; i++) {
+ amdgpu_ring_write(ring, PACKET0(mmUVD_NO_OP_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, 0);
+ }
+}
+
+/**
+ * vcn_v2_0_dec_ring_emit_fence - emit an fence & trap command
+ *
+ * @ring: amdgpu_ring pointer
+ * @fence: fence to emit
+ *
+ * Write a fence and a trap command to the ring.
+ */
+static void vcn_v2_0_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
+ unsigned flags)
+{
+ WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
+
+ amdgpu_ring_write(ring, PACKET0(mmUVD_CONTEXT_ID_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, seq);
+
+ amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, addr & 0xffffffff);
+
+ amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, upper_32_bits(addr) & 0xff);
+
+ amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_FENCE << 1));
+
+ amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, 0);
+
+ amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, 0);
+
+ amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET, 0));
+
+ amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_TRAP << 1));
+}
+
+/**
+ * vcn_v2_0_dec_ring_emit_ib - execute indirect buffer
+ *
+ * @ring: amdgpu_ring pointer
+ * @ib: indirect buffer to execute
+ *
+ * Write ring commands to execute the indirect buffer
+ */
+static void vcn_v2_0_dec_ring_emit_ib(struct amdgpu_ring *ring,
+ struct amdgpu_job *job,
+ struct amdgpu_ib *ib,
+ uint32_t flags)
+{
+ unsigned vmid = AMDGPU_JOB_GET_VMID(job);
+
+ amdgpu_ring_write(ring, PACKET0(mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, vmid);
+
+ amdgpu_ring_write(ring, PACKET0(mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
+ amdgpu_ring_write(ring, PACKET0(mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
+ amdgpu_ring_write(ring, PACKET0(mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, ib->length_dw);
+}
+
+static void vcn_v2_0_dec_ring_emit_reg_wait(struct amdgpu_ring *ring,
+ uint32_t reg, uint32_t val,
+ uint32_t mask)
+{
+ amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, reg << 2);
+
+ amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, val);
+
+ amdgpu_ring_write(ring, PACKET0(mmUVD_GP_SCRATCH8_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, mask);
+
+ amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET, 0));
+
+ amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_REG_READ_COND_WAIT << 1));
+}
+
+static void vcn_v2_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
+ unsigned vmid, uint64_t pd_addr)
+{
+ struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
+ uint32_t data0, data1, mask;
+
+ pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
+
+ /* wait for register write */
+ data0 = hub->ctx0_ptb_addr_lo32 + vmid * 2;
+ data1 = lower_32_bits(pd_addr);
+ mask = 0xffffffff;
+ vcn_v2_0_dec_ring_emit_reg_wait(ring, data0, data1, mask);
+}
+
+static void vcn_v2_0_dec_ring_emit_wreg(struct amdgpu_ring *ring,
+ uint32_t reg, uint32_t val)
+{
+ amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, reg << 2);
+
+ amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, val);
+
+ amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET, 0));
+
+ amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_WRITE_REG << 1));
+}
+
+/**
+ * vcn_v2_0_enc_ring_get_rptr - get enc read pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Returns the current hardware enc read pointer
+ */
+static uint64_t vcn_v2_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ if (ring == &adev->vcn.ring_enc[0])
+ return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR);
+ else
+ return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2);
+}
+
+ /**
+ * vcn_v2_0_enc_ring_get_wptr - get enc write pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Returns the current hardware enc write pointer
+ */
+static uint64_t vcn_v2_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ if (ring == &adev->vcn.ring_enc[0]) {
+ if (ring->use_doorbell)
+ return adev->wb.wb[ring->wptr_offs];
+ else
+ return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR);
+ } else {
+ if (ring->use_doorbell)
+ return adev->wb.wb[ring->wptr_offs];
+ else
+ return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2);
+ }
+}
+
+ /**
+ * vcn_v2_0_enc_ring_set_wptr - set enc write pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Commits the enc write pointer to the hardware
+ */
+static void vcn_v2_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ if (ring == &adev->vcn.ring_enc[0]) {
+ if (ring->use_doorbell) {
+ adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
+ WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
+ } else {
+ WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
+ }
+ } else {
+ if (ring->use_doorbell) {
+ adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
+ WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
+ } else {
+ WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
+ }
+ }
+}
+
+/**
+ * vcn_v2_0_enc_ring_emit_fence - emit an enc fence & trap command
+ *
+ * @ring: amdgpu_ring pointer
+ * @fence: fence to emit
+ *
+ * Write enc a fence and a trap command to the ring.
+ */
+static void vcn_v2_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
+ u64 seq, unsigned flags)
+{
+ WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
+
+ amdgpu_ring_write(ring, VCN_ENC_CMD_FENCE);
+ amdgpu_ring_write(ring, addr);
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+ amdgpu_ring_write(ring, seq);
+ amdgpu_ring_write(ring, VCN_ENC_CMD_TRAP);
+}
+
+static void vcn_v2_0_enc_ring_insert_end(struct amdgpu_ring *ring)
+{
+ amdgpu_ring_write(ring, VCN_ENC_CMD_END);
+}
+
+/**
+ * vcn_v2_0_enc_ring_emit_ib - enc execute indirect buffer
+ *
+ * @ring: amdgpu_ring pointer
+ * @ib: indirect buffer to execute
+ *
+ * Write enc ring commands to execute the indirect buffer
+ */
+static void vcn_v2_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
+ struct amdgpu_job *job,
+ struct amdgpu_ib *ib,
+ uint32_t flags)
+{
+ unsigned vmid = AMDGPU_JOB_GET_VMID(job);
+
+ amdgpu_ring_write(ring, VCN_ENC_CMD_IB);
+ amdgpu_ring_write(ring, vmid);
+ amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
+ amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
+ amdgpu_ring_write(ring, ib->length_dw);
+}
+
+static void vcn_v2_0_enc_ring_emit_reg_wait(struct amdgpu_ring *ring,
+ uint32_t reg, uint32_t val,
+ uint32_t mask)
+{
+ amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT);
+ amdgpu_ring_write(ring, reg << 2);
+ amdgpu_ring_write(ring, mask);
+ amdgpu_ring_write(ring, val);
+}
+
+static void vcn_v2_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
+ unsigned int vmid, uint64_t pd_addr)
+{
+ struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
+
+ pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
+
+ /* wait for reg writes */
+ vcn_v2_0_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + vmid * 2,
+ lower_32_bits(pd_addr), 0xffffffff);
+}
+
+static void vcn_v2_0_enc_ring_emit_wreg(struct amdgpu_ring *ring,
+ uint32_t reg, uint32_t val)
+{
+ amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE);
+ amdgpu_ring_write(ring, reg << 2);
+ amdgpu_ring_write(ring, val);
+}
+
+/**
+ * vcn_v2_0_jpeg_ring_get_rptr - get read pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Returns the current hardware read pointer
+ */
+static uint64_t vcn_v2_0_jpeg_ring_get_rptr(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ return RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_RPTR);
+}
+
+/**
+ * vcn_v2_0_jpeg_ring_get_wptr - get write pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Returns the current hardware write pointer
+ */
+static uint64_t vcn_v2_0_jpeg_ring_get_wptr(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ if (ring->use_doorbell)
+ return adev->wb.wb[ring->wptr_offs];
+ else
+ return RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR);
+}
+
+/**
+ * vcn_v2_0_jpeg_ring_set_wptr - set write pointer
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Commits the write pointer to the hardware
+ */
+static void vcn_v2_0_jpeg_ring_set_wptr(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+
+ if (ring->use_doorbell) {
+ adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
+ WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
+ } else {
+ WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR, lower_32_bits(ring->wptr));
+ }
+}
+
+/**
+ * vcn_v2_0_jpeg_ring_insert_start - insert a start command
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Write a start command to the ring.
+ */
+static void vcn_v2_0_jpeg_ring_insert_start(struct amdgpu_ring *ring)
+{
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, 0x68e04);
+
+ amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, 0x80010000);
+}
+
+/**
+ * vcn_v2_0_jpeg_ring_insert_end - insert a end command
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Write a end command to the ring.
+ */
+static void vcn_v2_0_jpeg_ring_insert_end(struct amdgpu_ring *ring)
+{
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, 0x68e04);
+
+ amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, 0x00010000);
+}
+
+/**
+ * vcn_v2_0_jpeg_ring_emit_fence - emit an fence & trap command
+ *
+ * @ring: amdgpu_ring pointer
+ * @fence: fence to emit
+ *
+ * Write a fence and a trap command to the ring.
+ */
+static void vcn_v2_0_jpeg_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
+ unsigned flags)
+{
+ WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_JPEG_GPCOM_DATA0_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, seq);
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_JPEG_GPCOM_DATA1_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, seq);
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_LOW_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, lower_32_bits(addr));
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_HIGH_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, upper_32_bits(addr));
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_JPEG_GPCOM_CMD_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, 0x8);
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_JPEG_GPCOM_CMD_INTERNAL_OFFSET,
+ 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE4));
+ amdgpu_ring_write(ring, 0);
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, 0x3fbc);
+
+ amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, 0x1);
+
+ amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE7));
+ amdgpu_ring_write(ring, 0);
+}
+
+/**
+ * vcn_v2_0_jpeg_ring_emit_ib - execute indirect buffer
+ *
+ * @ring: amdgpu_ring pointer
+ * @ib: indirect buffer to execute
+ *
+ * Write ring commands to execute the indirect buffer.
+ */
+static void vcn_v2_0_jpeg_ring_emit_ib(struct amdgpu_ring *ring,
+ struct amdgpu_job *job,
+ struct amdgpu_ib *ib,
+ uint32_t flags)
+{
+ unsigned vmid = AMDGPU_JOB_GET_VMID(job);
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JRBC_IB_VMID_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, (vmid | (vmid << 4)));
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JPEG_VMID_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, (vmid | (vmid << 4)));
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JRBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JRBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_IB_SIZE_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, ib->length_dw);
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_LOW_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, lower_32_bits(ring->gpu_addr));
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_HIGH_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, upper_32_bits(ring->gpu_addr));
+
+ amdgpu_ring_write(ring, PACKETJ(0, 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE2));
+ amdgpu_ring_write(ring, 0);
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_RB_COND_RD_TIMER_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, 0x01400200);
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_RB_REF_DATA_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, 0x2);
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_STATUS_INTERNAL_OFFSET,
+ 0, PACKETJ_CONDITION_CHECK3, PACKETJ_TYPE3));
+ amdgpu_ring_write(ring, 0x2);
+}
+
+static void vcn_v2_0_jpeg_ring_emit_reg_wait(struct amdgpu_ring *ring,
+ uint32_t reg, uint32_t val,
+ uint32_t mask)
+{
+ uint32_t reg_offset = (reg << 2);
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_RB_COND_RD_TIMER_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, 0x01400200);
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_RB_REF_DATA_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ amdgpu_ring_write(ring, val);
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ if (reg_offset >= 0x10000 && reg_offset <= 0x105ff) {
+ amdgpu_ring_write(ring, 0);
+ amdgpu_ring_write(ring,
+ PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE3));
+ } else {
+ amdgpu_ring_write(ring, reg_offset);
+ amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
+ 0, 0, PACKETJ_TYPE3));
+ }
+ amdgpu_ring_write(ring, mask);
+}
+
+static void vcn_v2_0_jpeg_ring_emit_vm_flush(struct amdgpu_ring *ring,
+ unsigned vmid, uint64_t pd_addr)
+{
+ struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
+ uint32_t data0, data1, mask;
+
+ pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
+
+ /* wait for register write */
+ data0 = hub->ctx0_ptb_addr_lo32 + vmid * 2;
+ data1 = lower_32_bits(pd_addr);
+ mask = 0xffffffff;
+ vcn_v2_0_jpeg_ring_emit_reg_wait(ring, data0, data1, mask);
+}
+
+static void vcn_v2_0_jpeg_ring_emit_wreg(struct amdgpu_ring *ring,
+ uint32_t reg, uint32_t val)
+{
+ uint32_t reg_offset = (reg << 2);
+
+ amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
+ 0, 0, PACKETJ_TYPE0));
+ if (reg_offset >= 0x10000 && reg_offset <= 0x105ff) {
+ amdgpu_ring_write(ring, 0);
+ amdgpu_ring_write(ring,
+ PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE0));
+ } else {
+ amdgpu_ring_write(ring, reg_offset);
+ amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
+ 0, 0, PACKETJ_TYPE0));
+ }
+ amdgpu_ring_write(ring, val);
+}
+
+static void vcn_v2_0_jpeg_ring_nop(struct amdgpu_ring *ring, uint32_t count)
+{
+ int i;
+
+ WARN_ON(ring->wptr % 2 || count % 2);
+
+ for (i = 0; i < count / 2; i++) {
+ amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE6));
+ amdgpu_ring_write(ring, 0);
+ }
+}
+
+static int vcn_v2_0_set_interrupt_state(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *source,
+ unsigned type,
+ enum amdgpu_interrupt_state state)
+{
+ return 0;
+}
+
+static int vcn_v2_0_process_interrupt(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *source,
+ struct amdgpu_iv_entry *entry)
+{
+ DRM_DEBUG("IH: VCN TRAP\n");
+
+ switch (entry->src_id) {
+ case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:
+ amdgpu_fence_process(&adev->vcn.ring_dec);
+ break;
+ case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
+ amdgpu_fence_process(&adev->vcn.ring_enc[0]);
+ break;
+ case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:
+ amdgpu_fence_process(&adev->vcn.ring_enc[1]);
+ break;
+ case VCN_2_0__SRCID__JPEG_DECODE:
+ amdgpu_fence_process(&adev->vcn.ring_jpeg);
+ break;
+ default:
+ DRM_ERROR("Unhandled interrupt: %d %d\n",
+ entry->src_id, entry->src_data[0]);
+ break;
+ }
+
+ return 0;
+}
+
+static int vcn_v2_0_dec_ring_test_ring(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+ uint32_t tmp = 0;
+ unsigned i;
+ int r;
+
+ WREG32(adev->vcn.external.scratch9, 0xCAFEDEAD);
+ r = amdgpu_ring_alloc(ring, 4);
+ if (r)
+ return r;
+ amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET, 0));
+ amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_PACKET_START << 1));
+ amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.scratch9, 0));
+ amdgpu_ring_write(ring, 0xDEADBEEF);
+ amdgpu_ring_commit(ring);
+ for (i = 0; i < adev->usec_timeout; i++) {
+ tmp = RREG32(adev->vcn.external.scratch9);
+ if (tmp == 0xDEADBEEF)
+ break;
+ DRM_UDELAY(1);
+ }
+
+ if (i >= adev->usec_timeout)
+ r = -ETIMEDOUT;
+
+ return r;
+}
+
+
+static int vcn_v2_0_set_powergating_state(void *handle,
+ enum amd_powergating_state state)
+{
+ /* This doesn't actually powergate the VCN block.
+ * That's done in the dpm code via the SMC. This
+ * just re-inits the block as necessary. The actual
+ * gating still happens in the dpm code. We should
+ * revisit this when there is a cleaner line between
+ * the smc and the hw blocks
+ */
+ int ret;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ if (state == adev->vcn.cur_state)
+ return 0;
+
+ if (state == AMD_PG_STATE_GATE)
+ ret = vcn_v2_0_stop(adev);
+ else
+ ret = vcn_v2_0_start(adev);
+
+ if (!ret)
+ adev->vcn.cur_state = state;
+ return ret;
+}
+
+static const struct amd_ip_funcs vcn_v2_0_ip_funcs = {
+ .name = "vcn_v2_0",
+ .early_init = vcn_v2_0_early_init,
+ .late_init = NULL,
+ .sw_init = vcn_v2_0_sw_init,
+ .sw_fini = vcn_v2_0_sw_fini,
+ .hw_init = vcn_v2_0_hw_init,
+ .hw_fini = vcn_v2_0_hw_fini,
+ .suspend = vcn_v2_0_suspend,
+ .resume = vcn_v2_0_resume,
+ .is_idle = vcn_v2_0_is_idle,
+ .wait_for_idle = vcn_v2_0_wait_for_idle,
+ .check_soft_reset = NULL,
+ .pre_soft_reset = NULL,
+ .soft_reset = NULL,
+ .post_soft_reset = NULL,
+ .set_clockgating_state = vcn_v2_0_set_clockgating_state,
+ .set_powergating_state = vcn_v2_0_set_powergating_state,
+};
+
+static const struct amdgpu_ring_funcs vcn_v2_0_dec_ring_vm_funcs = {
+ .type = AMDGPU_RING_TYPE_VCN_DEC,
+ .align_mask = 0xf,
+ .vmhub = AMDGPU_MMHUB,
+ .get_rptr = vcn_v2_0_dec_ring_get_rptr,
+ .get_wptr = vcn_v2_0_dec_ring_get_wptr,
+ .set_wptr = vcn_v2_0_dec_ring_set_wptr,
+ .emit_frame_size =
+ SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
+ SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
+ 8 + /* vcn_v2_0_dec_ring_emit_vm_flush */
+ 14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */
+ 6,
+ .emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */
+ .emit_ib = vcn_v2_0_dec_ring_emit_ib,
+ .emit_fence = vcn_v2_0_dec_ring_emit_fence,
+ .emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
+ .test_ring = vcn_v2_0_dec_ring_test_ring,
+ .test_ib = amdgpu_vcn_dec_ring_test_ib,
+ .insert_nop = vcn_v2_0_dec_ring_insert_nop,
+ .insert_start = vcn_v2_0_dec_ring_insert_start,
+ .insert_end = vcn_v2_0_dec_ring_insert_end,
+ .pad_ib = amdgpu_ring_generic_pad_ib,
+ .begin_use = amdgpu_vcn_ring_begin_use,
+ .end_use = amdgpu_vcn_ring_end_use,
+ .emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
+ .emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
+ .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
+};
+
+static const struct amdgpu_ring_funcs vcn_v2_0_enc_ring_vm_funcs = {
+ .type = AMDGPU_RING_TYPE_VCN_ENC,
+ .align_mask = 0x3f,
+ .nop = VCN_ENC_CMD_NO_OP,
+ .vmhub = AMDGPU_MMHUB,
+ .get_rptr = vcn_v2_0_enc_ring_get_rptr,
+ .get_wptr = vcn_v2_0_enc_ring_get_wptr,
+ .set_wptr = vcn_v2_0_enc_ring_set_wptr,
+ .emit_frame_size =
+ SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
+ SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
+ 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
+ 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
+ 1, /* vcn_v2_0_enc_ring_insert_end */
+ .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
+ .emit_ib = vcn_v2_0_enc_ring_emit_ib,
+ .emit_fence = vcn_v2_0_enc_ring_emit_fence,
+ .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
+ .test_ring = amdgpu_vcn_enc_ring_test_ring,
+ .test_ib = amdgpu_vcn_enc_ring_test_ib,
+ .insert_nop = amdgpu_ring_insert_nop,
+ .insert_end = vcn_v2_0_enc_ring_insert_end,
+ .pad_ib = amdgpu_ring_generic_pad_ib,
+ .begin_use = amdgpu_vcn_ring_begin_use,
+ .end_use = amdgpu_vcn_ring_end_use,
+ .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
+ .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
+ .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
+};
+
+static const struct amdgpu_ring_funcs vcn_v2_0_jpeg_ring_vm_funcs = {
+ .type = AMDGPU_RING_TYPE_VCN_JPEG,
+ .align_mask = 0xf,
+ .vmhub = AMDGPU_MMHUB,
+ .get_rptr = vcn_v2_0_jpeg_ring_get_rptr,
+ .get_wptr = vcn_v2_0_jpeg_ring_get_wptr,
+ .set_wptr = vcn_v2_0_jpeg_ring_set_wptr,
+ .emit_frame_size =
+ SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
+ SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
+ 8 + /* vcn_v2_0_jpeg_ring_emit_vm_flush */
+ 18 + 18 + /* vcn_v2_0_jpeg_ring_emit_fence x2 vm fence */
+ 8 + 16,
+ .emit_ib_size = 22, /* vcn_v2_0_jpeg_ring_emit_ib */
+ .emit_ib = vcn_v2_0_jpeg_ring_emit_ib,
+ .emit_fence = vcn_v2_0_jpeg_ring_emit_fence,
+ .emit_vm_flush = vcn_v2_0_jpeg_ring_emit_vm_flush,
+ .test_ring = amdgpu_vcn_jpeg_ring_test_ring,
+ .test_ib = amdgpu_vcn_jpeg_ring_test_ib,
+ .insert_nop = vcn_v2_0_jpeg_ring_nop,
+ .insert_start = vcn_v2_0_jpeg_ring_insert_start,
+ .insert_end = vcn_v2_0_jpeg_ring_insert_end,
+ .pad_ib = amdgpu_ring_generic_pad_ib,
+ .begin_use = amdgpu_vcn_ring_begin_use,
+ .end_use = amdgpu_vcn_ring_end_use,
+ .emit_wreg = vcn_v2_0_jpeg_ring_emit_wreg,
+ .emit_reg_wait = vcn_v2_0_jpeg_ring_emit_reg_wait,
+ .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
+};
+
+static void vcn_v2_0_set_dec_ring_funcs(struct amdgpu_device *adev)
+{
+ adev->vcn.ring_dec.funcs = &vcn_v2_0_dec_ring_vm_funcs;
+ DRM_INFO("VCN decode is enabled in VM mode\n");
+}
+
+static void vcn_v2_0_set_enc_ring_funcs(struct amdgpu_device *adev)
+{
+ int i;
+
+ for (i = 0; i < adev->vcn.num_enc_rings; ++i)
+ adev->vcn.ring_enc[i].funcs = &vcn_v2_0_enc_ring_vm_funcs;
+
+ DRM_INFO("VCN encode is enabled in VM mode\n");
+}
+
+static void vcn_v2_0_set_jpeg_ring_funcs(struct amdgpu_device *adev)
+{
+ adev->vcn.ring_jpeg.funcs = &vcn_v2_0_jpeg_ring_vm_funcs;
+ DRM_INFO("VCN jpeg decode is enabled in VM mode\n");
+}
+
+static const struct amdgpu_irq_src_funcs vcn_v2_0_irq_funcs = {
+ .set = vcn_v2_0_set_interrupt_state,
+ .process = vcn_v2_0_process_interrupt,
+};
+
+static void vcn_v2_0_set_irq_funcs(struct amdgpu_device *adev)
+{
+ adev->vcn.irq.num_types = adev->vcn.num_enc_rings + 2;
+ adev->vcn.irq.funcs = &vcn_v2_0_irq_funcs;
+}
+
+const struct amdgpu_ip_block_version vcn_v2_0_ip_block =
+{
+ .type = AMD_IP_BLOCK_TYPE_VCN,
+ .major = 2,
+ .minor = 0,
+ .rev = 0,
+ .funcs = &vcn_v2_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.h b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.h
new file mode 100644
index 000000000000..a74227f4663b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __VCN_V2_0_H__
+#define __VCN_V2_0_H__
+
+extern const struct amdgpu_ip_block_version vcn_v2_0_ip_block;
+
+#endif /* __VCN_V2_0_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
index 8d89ab7f0ae8..22260e6963b8 100644
--- a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
@@ -20,7 +20,9 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
-#include <drm/drmP.h>
+
+#include <linux/pci.h>
+
#include "amdgpu.h"
#include "amdgpu_ih.h"
#include "soc15.h"
@@ -48,14 +50,29 @@ static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
- WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
+ if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
+ if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) {
+ DRM_ERROR("PSP program IH_RB_CNTL failed!\n");
+ return;
+ }
+ } else {
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
+ }
adev->irq.ih.enabled = true;
if (adev->irq.ih1.ring_size) {
ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1,
RB_ENABLE, 1);
- WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
+ if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
+ if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1,
+ ih_rb_cntl)) {
+ DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n");
+ return;
+ }
+ } else {
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
+ }
adev->irq.ih1.enabled = true;
}
@@ -63,7 +80,15 @@ static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2,
RB_ENABLE, 1);
- WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
+ if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
+ if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2,
+ ih_rb_cntl)) {
+ DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n");
+ return;
+ }
+ } else {
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
+ }
adev->irq.ih2.enabled = true;
}
}
@@ -81,7 +106,15 @@ static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
- WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
+ if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
+ if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) {
+ DRM_ERROR("PSP program IH_RB_CNTL failed!\n");
+ return;
+ }
+ } else {
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
+ }
+
/* set rptr, wptr to 0 */
WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
@@ -92,7 +125,15 @@ static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1,
RB_ENABLE, 0);
- WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
+ if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
+ if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1,
+ ih_rb_cntl)) {
+ DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n");
+ return;
+ }
+ } else {
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
+ }
/* set rptr, wptr to 0 */
WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0);
WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0);
@@ -104,7 +145,16 @@ static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2,
RB_ENABLE, 0);
- WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
+ if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
+ if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2,
+ ih_rb_cntl)) {
+ DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n");
+ return;
+ }
+ } else {
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
+ }
+
/* set rptr, wptr to 0 */
WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0);
WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0);
@@ -187,7 +237,15 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev)
ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM,
!!adev->irq.msi_enabled);
- WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
+
+ if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
+ if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) {
+ DRM_ERROR("PSP program IH_RB_CNTL failed!\n");
+ return -ETIMEDOUT;
+ }
+ } else {
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
+ }
/* set the writeback address whether it's enabled or not */
WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO,
@@ -214,7 +272,15 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev)
WPTR_OVERFLOW_ENABLE, 0);
ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
RB_FULL_DRAIN_ENABLE, 1);
- WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
+ if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
+ if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1,
+ ih_rb_cntl)) {
+ DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n");
+ return -ETIMEDOUT;
+ }
+ } else {
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
+ }
/* set rptr, wptr to 0 */
WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0);
@@ -232,7 +298,16 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev)
ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2);
ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl);
- WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
+
+ if (amdgpu_virt_support_psp_prg_ih_reg(adev)) {
+ if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2,
+ ih_rb_cntl)) {
+ DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n");
+ return -ETIMEDOUT;
+ }
+ } else {
+ WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
+ }
/* set rptr, wptr to 0 */
WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0);
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 5e5b42a0744a..6575ddcfcf00 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -20,8 +20,10 @@
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
+
+#include <linux/pci.h>
#include <linux/slab.h>
-#include <drm/drmP.h>
+
#include "amdgpu.h"
#include "amdgpu_atombios.h"
#include "amdgpu_ih.h"
@@ -57,7 +59,6 @@
#include "vid.h"
#include "vi.h"
-#include "vi_dpm.h"
#include "gmc_v8_0.h"
#include "gmc_v7_0.h"
#include "gfx_v8_0.h"
@@ -987,6 +988,18 @@ static void vi_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
}
+static uint64_t vi_get_pcie_replay_count(struct amdgpu_device *adev)
+{
+ uint64_t nak_r, nak_g;
+
+ /* Get the number of NAKs received and generated */
+ nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
+ nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
+
+ /* Add the total number of NAKs, i.e the number of replays */
+ return (nak_r + nak_g);
+}
+
static bool vi_need_reset_on_init(struct amdgpu_device *adev)
{
u32 clock_cntl, pc;
@@ -1021,6 +1034,7 @@ static const struct amdgpu_asic_funcs vi_asic_funcs =
.init_doorbell_index = &legacy_doorbell_index_init,
.get_pcie_usage = &vi_get_pcie_usage,
.need_reset_on_init = &vi_need_reset_on_init,
+ .get_pcie_replay_count = &vi_get_pcie_replay_count,
};
#define CZ_REV_BRISTOL(rev) \
diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile
index 69ec96998bb9..48155060a57c 100644
--- a/drivers/gpu/drm/amd/amdkfd/Makefile
+++ b/drivers/gpu/drm/amd/amdkfd/Makefile
@@ -36,16 +36,19 @@ AMDKFD_FILES := $(AMDKFD_PATH)/kfd_module.o \
$(AMDKFD_PATH)/kfd_mqd_manager_cik.o \
$(AMDKFD_PATH)/kfd_mqd_manager_vi.o \
$(AMDKFD_PATH)/kfd_mqd_manager_v9.o \
+ $(AMDKFD_PATH)/kfd_mqd_manager_v10.o \
$(AMDKFD_PATH)/kfd_kernel_queue.o \
$(AMDKFD_PATH)/kfd_kernel_queue_cik.o \
$(AMDKFD_PATH)/kfd_kernel_queue_vi.o \
$(AMDKFD_PATH)/kfd_kernel_queue_v9.o \
+ $(AMDKFD_PATH)/kfd_kernel_queue_v10.o \
$(AMDKFD_PATH)/kfd_packet_manager.o \
$(AMDKFD_PATH)/kfd_process_queue_manager.o \
$(AMDKFD_PATH)/kfd_device_queue_manager.o \
$(AMDKFD_PATH)/kfd_device_queue_manager_cik.o \
$(AMDKFD_PATH)/kfd_device_queue_manager_vi.o \
$(AMDKFD_PATH)/kfd_device_queue_manager_v9.o \
+ $(AMDKFD_PATH)/kfd_device_queue_manager_v10.o \
$(AMDKFD_PATH)/kfd_interrupt.o \
$(AMDKFD_PATH)/kfd_events.o \
$(AMDKFD_PATH)/cik_event_interrupt.o \
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
index 3621efbd5759..826913c70766 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
@@ -21,7 +21,7 @@
*/
static const uint32_t cwsr_trap_gfx8_hex[] = {
- 0xbf820001, 0xbf82012b,
+ 0xbf820001, 0xbf820121,
0xb8f4f802, 0x89748674,
0xb8f5f803, 0x8675ff75,
0x00000400, 0xbf850017,
@@ -36,12 +36,7 @@ static const uint32_t cwsr_trap_gfx8_hex[] = {
0x8671ff71, 0x0000ffff,
0x8f728374, 0xb972e0c2,
0xbf800002, 0xb9740002,
- 0xbe801f70, 0xb8f5f803,
- 0x8675ff75, 0x00000100,
- 0xbf840006, 0xbefa0080,
- 0xb97a0203, 0x8671ff71,
- 0x0000ffff, 0x80f08870,
- 0x82f18071, 0xbefa0080,
+ 0xbe801f70, 0xbefa0080,
0xb97a0283, 0xbef60068,
0xbef70069, 0xb8fa1c07,
0x8e7a9c7a, 0x87717a71,
@@ -279,15 +274,17 @@ static const uint32_t cwsr_trap_gfx8_hex[] = {
static const uint32_t cwsr_trap_gfx9_hex[] = {
- 0xbf820001, 0xbf82015d,
+ 0xbf820001, 0xbf82015e,
0xb8f8f802, 0x89788678,
- 0xb8f1f803, 0x866eff71,
- 0x00000400, 0xbf850037,
- 0x866eff71, 0x00000800,
- 0xbf850003, 0x866eff71,
- 0x00000100, 0xbf840008,
+ 0xb8fbf803, 0x866eff7b,
+ 0x00000400, 0xbf85003b,
+ 0x866eff7b, 0x00000800,
+ 0xbf850003, 0x866eff7b,
+ 0x00000100, 0xbf84000c,
0x866eff78, 0x00002000,
- 0xbf840001, 0xbf810000,
+ 0xbf840005, 0xbf8e0010,
+ 0xb8eef803, 0x866eff6e,
+ 0x00000400, 0xbf84fffb,
0x8778ff78, 0x00002000,
0x80ec886c, 0x82ed806d,
0xb8eef807, 0x866fff6e,
@@ -295,13 +292,13 @@ static const uint32_t cwsr_trap_gfx9_hex[] = {
0x8977ff77, 0xfc000000,
0x87776f77, 0x896eff6e,
0x001f8000, 0xb96ef807,
- 0xb8f0f812, 0xb8f1f813,
- 0x8ef08870, 0xc0071bb8,
+ 0xb8faf812, 0xb8fbf813,
+ 0x8efa887a, 0xc0071bbd,
0x00000000, 0xbf8cc07f,
- 0xc0071c38, 0x00000008,
+ 0xc0071ebd, 0x00000008,
0xbf8cc07f, 0x86ee6e6e,
0xbf840001, 0xbe801d6e,
- 0xb8f1f803, 0x8671ff71,
+ 0xb8fbf803, 0x867bff7b,
0x000001ff, 0xbf850002,
0x806c846c, 0x826d806d,
0x866dff6d, 0x0000ffff,
@@ -311,258 +308,555 @@ static const uint32_t cwsr_trap_gfx9_hex[] = {
0x8f6e8378, 0xb96ee0c2,
0xbf800002, 0xb9780002,
0xbe801f6c, 0x866dff6d,
- 0x0000ffff, 0xbef00080,
- 0xb9700283, 0xb8f02407,
- 0x8e709c70, 0x876d706d,
- 0xb8f003c7, 0x8e709b70,
- 0x876d706d, 0xb8f0f807,
- 0x8670ff70, 0x00007fff,
- 0xb970f807, 0xbeee007e,
+ 0x0000ffff, 0xbefa0080,
+ 0xb97a0283, 0xb8fa2407,
+ 0x8e7a9b7a, 0x876d7a6d,
+ 0xb8fa03c7, 0x8e7a9a7a,
+ 0x876d7a6d, 0xb8faf807,
+ 0x867aff7a, 0x00007fff,
+ 0xb97af807, 0xbeee007e,
0xbeef007f, 0xbefe0180,
- 0xbf900004, 0x87708478,
- 0xb970f802, 0xbf8e0002,
- 0xbf88fffe, 0xb8f02a05,
+ 0xbf900004, 0x877a8478,
+ 0xb97af802, 0xbf8e0002,
+ 0xbf88fffe, 0xb8fa2a05,
+ 0x807a817a, 0x8e7a8a7a,
+ 0xb8fb1605, 0x807b817b,
+ 0x8e7b867b, 0x807a7b7a,
+ 0x807a7e7a, 0x827b807f,
+ 0x867bff7b, 0x0000ffff,
+ 0xc04b1c3d, 0x00000050,
+ 0xbf8cc07f, 0xc04b1d3d,
+ 0x00000060, 0xbf8cc07f,
+ 0xc0431e7d, 0x00000074,
+ 0xbf8cc07f, 0xbef4007e,
+ 0x8675ff7f, 0x0000ffff,
+ 0x8775ff75, 0x00040000,
+ 0xbef60080, 0xbef700ff,
+ 0x00807fac, 0x867aff7f,
+ 0x08000000, 0x8f7a837a,
+ 0x87777a77, 0x867aff7f,
+ 0x70000000, 0x8f7a817a,
+ 0x87777a77, 0xbef1007c,
+ 0xbef00080, 0xb8f02a05,
0x80708170, 0x8e708a70,
- 0xb8f11605, 0x80718171,
- 0x8e718671, 0x80707170,
- 0x80707e70, 0x8271807f,
- 0x8671ff71, 0x0000ffff,
- 0xc0471cb8, 0x00000040,
- 0xbf8cc07f, 0xc04b1d38,
- 0x00000048, 0xbf8cc07f,
- 0xc0431e78, 0x00000058,
- 0xbf8cc07f, 0xc0471eb8,
- 0x0000005c, 0xbf8cc07f,
- 0xbef4007e, 0x8675ff7f,
- 0x0000ffff, 0x8775ff75,
- 0x00040000, 0xbef60080,
- 0xbef700ff, 0x00807fac,
- 0x8670ff7f, 0x08000000,
- 0x8f708370, 0x87777077,
- 0x8670ff7f, 0x70000000,
- 0x8f708170, 0x87777077,
- 0xbefb007c, 0xbefa0080,
- 0xb8fa2a05, 0x807a817a,
- 0x8e7a8a7a, 0xb8f01605,
- 0x80708170, 0x8e708670,
- 0x807a707a, 0xbef60084,
- 0xbef600ff, 0x01000000,
- 0xbefe007c, 0xbefc007a,
- 0xc0611efa, 0x0000007c,
- 0xbf8cc07f, 0x807a847a,
- 0xbefc007e, 0xbefe007c,
- 0xbefc007a, 0xc0611b3a,
+ 0xb8fa1605, 0x807a817a,
+ 0x8e7a867a, 0x80707a70,
+ 0xbef60084, 0xbef600ff,
+ 0x01000000, 0xbefe007c,
+ 0xbefc0070, 0xc0611c7a,
0x0000007c, 0xbf8cc07f,
- 0x807a847a, 0xbefc007e,
- 0xbefe007c, 0xbefc007a,
- 0xc0611b7a, 0x0000007c,
- 0xbf8cc07f, 0x807a847a,
+ 0x80708470, 0xbefc007e,
+ 0xbefe007c, 0xbefc0070,
+ 0xc0611b3a, 0x0000007c,
+ 0xbf8cc07f, 0x80708470,
0xbefc007e, 0xbefe007c,
- 0xbefc007a, 0xc0611bba,
+ 0xbefc0070, 0xc0611b7a,
0x0000007c, 0xbf8cc07f,
- 0x807a847a, 0xbefc007e,
- 0xbefe007c, 0xbefc007a,
- 0xc0611bfa, 0x0000007c,
- 0xbf8cc07f, 0x807a847a,
+ 0x80708470, 0xbefc007e,
+ 0xbefe007c, 0xbefc0070,
+ 0xc0611bba, 0x0000007c,
+ 0xbf8cc07f, 0x80708470,
0xbefc007e, 0xbefe007c,
- 0xbefc007a, 0xc0611e3a,
+ 0xbefc0070, 0xc0611bfa,
0x0000007c, 0xbf8cc07f,
- 0x807a847a, 0xbefc007e,
- 0xb8f1f803, 0xbefe007c,
- 0xbefc007a, 0xc0611c7a,
- 0x0000007c, 0xbf8cc07f,
- 0x807a847a, 0xbefc007e,
- 0xbefe007c, 0xbefc007a,
- 0xc0611a3a, 0x0000007c,
- 0xbf8cc07f, 0x807a847a,
+ 0x80708470, 0xbefc007e,
+ 0xbefe007c, 0xbefc0070,
+ 0xc0611e3a, 0x0000007c,
+ 0xbf8cc07f, 0x80708470,
+ 0xbefc007e, 0xb8fbf803,
+ 0xbefe007c, 0xbefc0070,
+ 0xc0611efa, 0x0000007c,
+ 0xbf8cc07f, 0x80708470,
0xbefc007e, 0xbefe007c,
- 0xbefc007a, 0xc0611a7a,
+ 0xbefc0070, 0xc0611a3a,
0x0000007c, 0xbf8cc07f,
- 0x807a847a, 0xbefc007e,
- 0xb8fbf801, 0xbefe007c,
- 0xbefc007a, 0xc0611efa,
- 0x0000007c, 0xbf8cc07f,
- 0x807a847a, 0xbefc007e,
- 0x8670ff7f, 0x04000000,
- 0xbeef0080, 0x876f6f70,
- 0xb8fa2a05, 0x807a817a,
- 0x8e7a8a7a, 0xb8f11605,
- 0x80718171, 0x8e718471,
- 0x8e768271, 0xbef600ff,
- 0x01000000, 0xbef20174,
- 0x80747a74, 0x82758075,
- 0xbefc0080, 0xbf800000,
- 0xbe802b00, 0xbe822b02,
- 0xbe842b04, 0xbe862b06,
- 0xbe882b08, 0xbe8a2b0a,
- 0xbe8c2b0c, 0xbe8e2b0e,
- 0xc06b003a, 0x00000000,
- 0xbf8cc07f, 0xc06b013a,
- 0x00000010, 0xbf8cc07f,
- 0xc06b023a, 0x00000020,
- 0xbf8cc07f, 0xc06b033a,
- 0x00000030, 0xbf8cc07f,
- 0x8074c074, 0x82758075,
- 0x807c907c, 0xbf0a717c,
- 0xbf85ffe7, 0xbef40172,
- 0xbefa0080, 0xbefe00c1,
- 0xbeff00c1, 0xbee80080,
- 0xbee90080, 0xbef600ff,
- 0x01000000, 0xe0724000,
- 0x7a1d0000, 0xe0724100,
- 0x7a1d0100, 0xe0724200,
- 0x7a1d0200, 0xe0724300,
- 0x7a1d0300, 0xbefe00c1,
- 0xbeff00c1, 0xb8f14306,
- 0x8671c171, 0xbf84002c,
- 0xbf8a0000, 0x8670ff6f,
- 0x04000000, 0xbf840028,
- 0x8e718671, 0x8e718271,
- 0xbef60071, 0xb8fa2a05,
- 0x807a817a, 0x8e7a8a7a,
- 0xb8f01605, 0x80708170,
- 0x8e708670, 0x807a707a,
- 0x807aff7a, 0x00000080,
+ 0x80708470, 0xbefc007e,
+ 0xbefe007c, 0xbefc0070,
+ 0xc0611a7a, 0x0000007c,
+ 0xbf8cc07f, 0x80708470,
+ 0xbefc007e, 0xb8f1f801,
+ 0xbefe007c, 0xbefc0070,
+ 0xc0611c7a, 0x0000007c,
+ 0xbf8cc07f, 0x80708470,
+ 0xbefc007e, 0x867aff7f,
+ 0x04000000, 0xbeef0080,
+ 0x876f6f7a, 0xb8f02a05,
+ 0x80708170, 0x8e708a70,
+ 0xb8fb1605, 0x807b817b,
+ 0x8e7b847b, 0x8e76827b,
0xbef600ff, 0x01000000,
- 0xbefc0080, 0xd28c0002,
- 0x000100c1, 0xd28d0003,
- 0x000204c1, 0xd1060002,
- 0x00011103, 0x7e0602ff,
- 0x00000200, 0xbefc00ff,
- 0x00010000, 0xbe800077,
- 0x8677ff77, 0xff7fffff,
- 0x8777ff77, 0x00058000,
- 0xd8ec0000, 0x00000002,
- 0xbf8cc07f, 0xe0765000,
- 0x7a1d0002, 0x68040702,
- 0xd0c9006a, 0x0000e302,
- 0xbf87fff7, 0xbef70000,
- 0xbefa00ff, 0x00000400,
+ 0xbef20174, 0x80747074,
+ 0x82758075, 0xbefc0080,
+ 0xbf800000, 0xbe802b00,
+ 0xbe822b02, 0xbe842b04,
+ 0xbe862b06, 0xbe882b08,
+ 0xbe8a2b0a, 0xbe8c2b0c,
+ 0xbe8e2b0e, 0xc06b003a,
+ 0x00000000, 0xbf8cc07f,
+ 0xc06b013a, 0x00000010,
+ 0xbf8cc07f, 0xc06b023a,
+ 0x00000020, 0xbf8cc07f,
+ 0xc06b033a, 0x00000030,
+ 0xbf8cc07f, 0x8074c074,
+ 0x82758075, 0x807c907c,
+ 0xbf0a7b7c, 0xbf85ffe7,
+ 0xbef40172, 0xbef00080,
0xbefe00c1, 0xbeff00c1,
- 0xb8f12a05, 0x80718171,
- 0x8e718271, 0x8e768871,
+ 0xbee80080, 0xbee90080,
0xbef600ff, 0x01000000,
- 0xbefc0084, 0xbf0a717c,
- 0xbf840015, 0xbf11017c,
- 0x8071ff71, 0x00001000,
- 0x7e000300, 0x7e020301,
- 0x7e040302, 0x7e060303,
- 0xe0724000, 0x7a1d0000,
- 0xe0724100, 0x7a1d0100,
- 0xe0724200, 0x7a1d0200,
- 0xe0724300, 0x7a1d0300,
- 0x807c847c, 0x807aff7a,
- 0x00000400, 0xbf0a717c,
- 0xbf85ffef, 0xbf9c0000,
- 0xbf8200dc, 0xbef4007e,
- 0x8675ff7f, 0x0000ffff,
- 0x8775ff75, 0x00040000,
- 0xbef60080, 0xbef700ff,
- 0x00807fac, 0x866eff7f,
- 0x08000000, 0x8f6e836e,
- 0x87776e77, 0x866eff7f,
- 0x70000000, 0x8f6e816e,
- 0x87776e77, 0x866eff7f,
- 0x04000000, 0xbf84001e,
+ 0xe0724000, 0x701d0000,
+ 0xe0724100, 0x701d0100,
+ 0xe0724200, 0x701d0200,
+ 0xe0724300, 0x701d0300,
0xbefe00c1, 0xbeff00c1,
- 0xb8ef4306, 0x866fc16f,
- 0xbf840019, 0x8e6f866f,
- 0x8e6f826f, 0xbef6006f,
- 0xb8f82a05, 0x80788178,
- 0x8e788a78, 0xb8ee1605,
- 0x806e816e, 0x8e6e866e,
- 0x80786e78, 0x8078ff78,
+ 0xb8fb4306, 0x867bc17b,
+ 0xbf84002c, 0xbf8a0000,
+ 0x867aff6f, 0x04000000,
+ 0xbf840028, 0x8e7b867b,
+ 0x8e7b827b, 0xbef6007b,
+ 0xb8f02a05, 0x80708170,
+ 0x8e708a70, 0xb8fa1605,
+ 0x807a817a, 0x8e7a867a,
+ 0x80707a70, 0x8070ff70,
0x00000080, 0xbef600ff,
0x01000000, 0xbefc0080,
- 0xe0510000, 0x781d0000,
- 0xe0510100, 0x781d0000,
- 0x807cff7c, 0x00000200,
- 0x8078ff78, 0x00000200,
- 0xbf0a6f7c, 0xbf85fff6,
- 0xbef80080, 0xbefe00c1,
- 0xbeff00c1, 0xb8ef2a05,
- 0x806f816f, 0x8e6f826f,
- 0x8e76886f, 0xbef600ff,
- 0x01000000, 0xbeee0078,
- 0x8078ff78, 0x00000400,
- 0xbefc0084, 0xbf11087c,
- 0x806fff6f, 0x00008000,
- 0xe0524000, 0x781d0000,
- 0xe0524100, 0x781d0100,
- 0xe0524200, 0x781d0200,
- 0xe0524300, 0x781d0300,
- 0xbf8c0f70, 0x7e000300,
+ 0xd28c0002, 0x000100c1,
+ 0xd28d0003, 0x000204c1,
+ 0xd1060002, 0x00011103,
+ 0x7e0602ff, 0x00000200,
+ 0xbefc00ff, 0x00010000,
+ 0xbe800077, 0x8677ff77,
+ 0xff7fffff, 0x8777ff77,
+ 0x00058000, 0xd8ec0000,
+ 0x00000002, 0xbf8cc07f,
+ 0xe0765000, 0x701d0002,
+ 0x68040702, 0xd0c9006a,
+ 0x0000f702, 0xbf87fff7,
+ 0xbef70000, 0xbef000ff,
+ 0x00000400, 0xbefe00c1,
+ 0xbeff00c1, 0xb8fb2a05,
+ 0x807b817b, 0x8e7b827b,
+ 0x8e76887b, 0xbef600ff,
+ 0x01000000, 0xbefc0084,
+ 0xbf0a7b7c, 0xbf840015,
+ 0xbf11017c, 0x807bff7b,
+ 0x00001000, 0x7e000300,
0x7e020301, 0x7e040302,
- 0x7e060303, 0x807c847c,
- 0x8078ff78, 0x00000400,
- 0xbf0a6f7c, 0xbf85ffee,
- 0xbf9c0000, 0xe0524000,
- 0x6e1d0000, 0xe0524100,
- 0x6e1d0100, 0xe0524200,
- 0x6e1d0200, 0xe0524300,
- 0x6e1d0300, 0xb8f82a05,
+ 0x7e060303, 0xe0724000,
+ 0x701d0000, 0xe0724100,
+ 0x701d0100, 0xe0724200,
+ 0x701d0200, 0xe0724300,
+ 0x701d0300, 0x807c847c,
+ 0x8070ff70, 0x00000400,
+ 0xbf0a7b7c, 0xbf85ffef,
+ 0xbf9c0000, 0xbf8200da,
+ 0xbef4007e, 0x8675ff7f,
+ 0x0000ffff, 0x8775ff75,
+ 0x00040000, 0xbef60080,
+ 0xbef700ff, 0x00807fac,
+ 0x866eff7f, 0x08000000,
+ 0x8f6e836e, 0x87776e77,
+ 0x866eff7f, 0x70000000,
+ 0x8f6e816e, 0x87776e77,
+ 0x866eff7f, 0x04000000,
+ 0xbf84001e, 0xbefe00c1,
+ 0xbeff00c1, 0xb8ef4306,
+ 0x866fc16f, 0xbf840019,
+ 0x8e6f866f, 0x8e6f826f,
+ 0xbef6006f, 0xb8f82a05,
0x80788178, 0x8e788a78,
0xb8ee1605, 0x806e816e,
0x8e6e866e, 0x80786e78,
- 0x80f8c078, 0xb8ef1605,
- 0x806f816f, 0x8e6f846f,
- 0x8e76826f, 0xbef600ff,
- 0x01000000, 0xbefc006f,
- 0xc031003a, 0x00000078,
- 0x80f8c078, 0xbf8cc07f,
- 0x80fc907c, 0xbf800000,
- 0xbe802d00, 0xbe822d02,
- 0xbe842d04, 0xbe862d06,
- 0xbe882d08, 0xbe8a2d0a,
- 0xbe8c2d0c, 0xbe8e2d0e,
- 0xbf06807c, 0xbf84fff0,
+ 0x8078ff78, 0x00000080,
+ 0xbef600ff, 0x01000000,
+ 0xbefc0080, 0xe0510000,
+ 0x781d0000, 0xe0510100,
+ 0x781d0000, 0x807cff7c,
+ 0x00000200, 0x8078ff78,
+ 0x00000200, 0xbf0a6f7c,
+ 0xbf85fff6, 0xbef80080,
+ 0xbefe00c1, 0xbeff00c1,
+ 0xb8ef2a05, 0x806f816f,
+ 0x8e6f826f, 0x8e76886f,
+ 0xbef600ff, 0x01000000,
+ 0xbeee0078, 0x8078ff78,
+ 0x00000400, 0xbefc0084,
+ 0xbf11087c, 0x806fff6f,
+ 0x00008000, 0xe0524000,
+ 0x781d0000, 0xe0524100,
+ 0x781d0100, 0xe0524200,
+ 0x781d0200, 0xe0524300,
+ 0x781d0300, 0xbf8c0f70,
+ 0x7e000300, 0x7e020301,
+ 0x7e040302, 0x7e060303,
+ 0x807c847c, 0x8078ff78,
+ 0x00000400, 0xbf0a6f7c,
+ 0xbf85ffee, 0xbf9c0000,
+ 0xe0524000, 0x6e1d0000,
+ 0xe0524100, 0x6e1d0100,
+ 0xe0524200, 0x6e1d0200,
+ 0xe0524300, 0x6e1d0300,
0xb8f82a05, 0x80788178,
0x8e788a78, 0xb8ee1605,
0x806e816e, 0x8e6e866e,
- 0x80786e78, 0xbef60084,
+ 0x80786e78, 0x80f8c078,
+ 0xb8ef1605, 0x806f816f,
+ 0x8e6f846f, 0x8e76826f,
0xbef600ff, 0x01000000,
- 0xc0211bfa, 0x00000078,
- 0x80788478, 0xc0211b3a,
+ 0xbefc006f, 0xc031003a,
+ 0x00000078, 0x80f8c078,
+ 0xbf8cc07f, 0x80fc907c,
+ 0xbf800000, 0xbe802d00,
+ 0xbe822d02, 0xbe842d04,
+ 0xbe862d06, 0xbe882d08,
+ 0xbe8a2d0a, 0xbe8c2d0c,
+ 0xbe8e2d0e, 0xbf06807c,
+ 0xbf84fff0, 0xb8f82a05,
+ 0x80788178, 0x8e788a78,
+ 0xb8ee1605, 0x806e816e,
+ 0x8e6e866e, 0x80786e78,
+ 0xbef60084, 0xbef600ff,
+ 0x01000000, 0xc0211bfa,
0x00000078, 0x80788478,
- 0xc0211b7a, 0x00000078,
- 0x80788478, 0xc0211eba,
+ 0xc0211b3a, 0x00000078,
+ 0x80788478, 0xc0211b7a,
0x00000078, 0x80788478,
- 0xc0211efa, 0x00000078,
- 0x80788478, 0xc0211c3a,
+ 0xc0211c3a, 0x00000078,
+ 0x80788478, 0xc0211c7a,
0x00000078, 0x80788478,
- 0xc0211c7a, 0x00000078,
- 0x80788478, 0xc0211a3a,
+ 0xc0211eba, 0x00000078,
+ 0x80788478, 0xc0211efa,
0x00000078, 0x80788478,
- 0xc0211a7a, 0x00000078,
- 0x80788478, 0xc0211cfa,
+ 0xc0211a3a, 0x00000078,
+ 0x80788478, 0xc0211a7a,
0x00000078, 0x80788478,
- 0xbf8cc07f, 0xbefc006f,
- 0xbefe007a, 0xbeff007b,
- 0x866f71ff, 0x000003ff,
- 0xb96f4803, 0x866f71ff,
- 0xfffff800, 0x8f6f8b6f,
- 0xb96fa2c3, 0xb973f801,
- 0xb8ee2a05, 0x806e816e,
- 0x8e6e8a6e, 0xb8ef1605,
- 0x806f816f, 0x8e6f866f,
- 0x806e6f6e, 0x806e746e,
- 0x826f8075, 0x866fff6f,
- 0x0000ffff, 0xc0071cb7,
- 0x00000040, 0xc00b1d37,
- 0x00000048, 0xc0031e77,
- 0x00000058, 0xc0071eb7,
- 0x0000005c, 0xbf8cc07f,
- 0x866fff6d, 0xf0000000,
- 0x8f6f9c6f, 0x8e6f906f,
- 0xbeee0080, 0x876e6f6e,
- 0x866fff6d, 0x08000000,
- 0x8f6f9b6f, 0x8e6f8f6f,
- 0x876e6f6e, 0x866fff70,
- 0x00800000, 0x8f6f976f,
- 0xb96ef807, 0x866dff6d,
- 0x0000ffff, 0x86fe7e7e,
- 0x86ea6a6a, 0x8f6e8370,
- 0xb96ee0c2, 0xbf800002,
- 0xb9700002, 0xbf8a0000,
- 0x95806f6c, 0xbf810000,
+ 0xc0211cfa, 0x00000078,
+ 0x80788478, 0xbf8cc07f,
+ 0xbefc006f, 0xbefe0070,
+ 0xbeff0071, 0x866f7bff,
+ 0x000003ff, 0xb96f4803,
+ 0x866f7bff, 0xfffff800,
+ 0x8f6f8b6f, 0xb96fa2c3,
+ 0xb973f801, 0xb8ee2a05,
+ 0x806e816e, 0x8e6e8a6e,
+ 0xb8ef1605, 0x806f816f,
+ 0x8e6f866f, 0x806e6f6e,
+ 0x806e746e, 0x826f8075,
+ 0x866fff6f, 0x0000ffff,
+ 0xc00b1c37, 0x00000050,
+ 0xc00b1d37, 0x00000060,
+ 0xc0031e77, 0x00000074,
+ 0xbf8cc07f, 0x866fff6d,
+ 0xf8000000, 0x8f6f9b6f,
+ 0x8e6f906f, 0xbeee0080,
+ 0x876e6f6e, 0x866fff6d,
+ 0x04000000, 0x8f6f9a6f,
+ 0x8e6f8f6f, 0x876e6f6e,
+ 0x866fff7a, 0x00800000,
+ 0x8f6f976f, 0xb96ef807,
+ 0x866dff6d, 0x0000ffff,
+ 0x86fe7e7e, 0x86ea6a6a,
+ 0x8f6e837a, 0xb96ee0c2,
+ 0xbf800002, 0xb97a0002,
+ 0xbf8a0000, 0x95806f6c,
+ 0xbf810000, 0x00000000,
+};
+
+static const uint32_t cwsr_trap_gfx10_hex[] = {
+ 0xbf820001, 0xbf82012e,
+ 0xb0804004, 0xb970f802,
+ 0x8a708670, 0xb971f803,
+ 0x8771ff71, 0x00000400,
+ 0xbf850008, 0xb971f803,
+ 0x8771ff71, 0x000001ff,
+ 0xbf850001, 0x806c846c,
+ 0x876dff6d, 0x0000ffff,
+ 0xbe80226c, 0xb971f803,
+ 0x8771ff71, 0x00000100,
+ 0xbf840006, 0xbef60380,
+ 0xb9f60203, 0x876dff6d,
+ 0x0000ffff, 0x80ec886c,
+ 0x82ed806d, 0xbef60380,
+ 0xb9f60283, 0xb973f816,
+ 0xb9762c07, 0x8f769c76,
+ 0x886d766d, 0xb97603c7,
+ 0x8f769b76, 0x886d766d,
+ 0xb976f807, 0x8776ff76,
+ 0x00007fff, 0xb9f6f807,
+ 0xbeee037e, 0xbeef037f,
+ 0xbefe0480, 0xbf900004,
+ 0xbf8e0002, 0xbf88fffe,
+ 0xbef4037e, 0x8775ff7f,
+ 0x0000ffff, 0x8875ff75,
+ 0x00040000, 0xbef60380,
+ 0xbef703ff, 0x00807fac,
+ 0x8776ff7f, 0x08000000,
+ 0x90768376, 0x88777677,
+ 0x8776ff7f, 0x70000000,
+ 0x90768176, 0x88777677,
+ 0xbefb037c, 0xbefa0380,
+ 0xb97202dc, 0x8872727f,
+ 0xbefe03c1, 0x877c8172,
+ 0xbf06817c, 0xbf850002,
+ 0xbeff0380, 0xbf820001,
+ 0xbeff03c1, 0xb9712a05,
+ 0x80718171, 0x8f718271,
+ 0x877c8172, 0xbf06817c,
+ 0xbf85000d, 0x8f768771,
+ 0xbef603ff, 0x01000000,
+ 0xbefc0380, 0x7e008700,
+ 0xe0704000, 0x7a5d0000,
+ 0x807c817c, 0x807aff7a,
+ 0x00000080, 0xbf0a717c,
+ 0xbf85fff8, 0xbf82001b,
+ 0x8f768871, 0xbef603ff,
+ 0x01000000, 0xbefc0380,
+ 0x7e008700, 0xe0704000,
+ 0x7a5d0000, 0x807c817c,
+ 0x807aff7a, 0x00000100,
+ 0xbf0a717c, 0xbf85fff8,
+ 0xb9711e06, 0x8771c171,
+ 0xbf84000c, 0x8f718371,
+ 0x80717c71, 0xbefe03c1,
+ 0xbeff0380, 0x7e008700,
+ 0xe0704000, 0x7a5d0000,
+ 0x807c817c, 0x807aff7a,
+ 0x00000080, 0xbf0a717c,
+ 0xbf85fff8, 0xbf8a0000,
+ 0x8776ff72, 0x04000000,
+ 0xbf84002b, 0xbefe03c1,
+ 0x877c8172, 0xbf06817c,
+ 0xbf850002, 0xbeff0380,
+ 0xbf820001, 0xbeff03c1,
+ 0xb9714306, 0x8771c171,
+ 0xbf840021, 0x8f718671,
+ 0x8f718271, 0xbef60371,
+ 0xbef603ff, 0x01000000,
+ 0xd7650000, 0x000100c1,
+ 0xd7660000, 0x000200c1,
+ 0x16000084, 0x877c8172,
+ 0xbf06817c, 0xbefc0380,
+ 0xbf85000a, 0x807cff7c,
+ 0x00000080, 0x807aff7a,
+ 0x00000080, 0xd5250000,
+ 0x0001ff00, 0x00000080,
+ 0xbf0a717c, 0xbf85fff7,
+ 0xbf820009, 0x807cff7c,
+ 0x00000100, 0x807aff7a,
+ 0x00000100, 0xd5250000,
+ 0x0001ff00, 0x00000100,
+ 0xbf0a717c, 0xbf85fff7,
+ 0x877c8172, 0xbf06817c,
+ 0xbf850003, 0x8f7687ff,
+ 0x0000006a, 0xbf820002,
+ 0x8f7688ff, 0x0000006a,
+ 0xbef603ff, 0x01000000,
+ 0x877c8172, 0xbf06817c,
+ 0xbefc0380, 0xbf800000,
+ 0xbf85000b, 0xbe802e00,
+ 0x7e000200, 0xe0704000,
+ 0x7a5d0000, 0x807aff7a,
+ 0x00000080, 0x807c817c,
+ 0xbf0aff7c, 0x0000006a,
+ 0xbf85fff6, 0xbf82000a,
+ 0xbe802e00, 0x7e000200,
+ 0xe0704000, 0x7a5d0000,
+ 0x807aff7a, 0x00000100,
+ 0x807c817c, 0xbf0aff7c,
+ 0x0000006a, 0xbf85fff6,
+ 0xbef60384, 0xbef603ff,
+ 0x01000000, 0x877c8172,
+ 0xbf06817c, 0xbf850030,
+ 0x7e00027b, 0xe0704000,
+ 0x7a5d0000, 0x807aff7a,
+ 0x00000080, 0x7e00026c,
+ 0xe0704000, 0x7a5d0000,
+ 0x807aff7a, 0x00000080,
+ 0x7e00026d, 0xe0704000,
+ 0x7a5d0000, 0x807aff7a,
+ 0x00000080, 0x7e00026e,
+ 0xe0704000, 0x7a5d0000,
+ 0x807aff7a, 0x00000080,
+ 0x7e00026f, 0xe0704000,
+ 0x7a5d0000, 0x807aff7a,
+ 0x00000080, 0x7e000270,
+ 0xe0704000, 0x7a5d0000,
+ 0x807aff7a, 0x00000080,
+ 0xb971f803, 0x7e000271,
+ 0xe0704000, 0x7a5d0000,
+ 0x807aff7a, 0x00000080,
+ 0x7e000273, 0xe0704000,
+ 0x7a5d0000, 0x807aff7a,
+ 0x00000080, 0xb97bf801,
+ 0x7e00027b, 0xe0704000,
+ 0x7a5d0000, 0x807aff7a,
+ 0x00000080, 0xbf82002f,
+ 0x7e00027b, 0xe0704000,
+ 0x7a5d0000, 0x807aff7a,
+ 0x00000100, 0x7e00026c,
+ 0xe0704000, 0x7a5d0000,
+ 0x807aff7a, 0x00000100,
+ 0x7e00026d, 0xe0704000,
+ 0x7a5d0000, 0x807aff7a,
+ 0x00000100, 0x7e00026e,
+ 0xe0704000, 0x7a5d0000,
+ 0x807aff7a, 0x00000100,
+ 0x7e00026f, 0xe0704000,
+ 0x7a5d0000, 0x807aff7a,
+ 0x00000100, 0x7e000270,
+ 0xe0704000, 0x7a5d0000,
+ 0x807aff7a, 0x00000100,
+ 0xb971f803, 0x7e000271,
+ 0xe0704000, 0x7a5d0000,
+ 0x807aff7a, 0x00000100,
+ 0x7e000273, 0xe0704000,
+ 0x7a5d0000, 0x807aff7a,
+ 0x00000100, 0xb97bf801,
+ 0x7e00027b, 0xe0704000,
+ 0x7a5d0000, 0x807aff7a,
+ 0x00000100, 0xbf820119,
+ 0xbef4037e, 0x8775ff7f,
+ 0x0000ffff, 0x8875ff75,
+ 0x00040000, 0xbef60380,
+ 0xbef703ff, 0x00807fac,
+ 0x8772ff7f, 0x08000000,
+ 0x90728372, 0x88777277,
+ 0x8772ff7f, 0x70000000,
+ 0x90728172, 0x88777277,
+ 0xb97902dc, 0x8879797f,
+ 0xbef80380, 0xbefe03c1,
+ 0x877c8179, 0xbf06817c,
+ 0xbf850002, 0xbeff0380,
+ 0xbf820001, 0xbeff03c1,
+ 0xb96f2a05, 0x806f816f,
+ 0x8f6f826f, 0x877c8179,
+ 0xbf06817c, 0xbf850013,
+ 0x8f76876f, 0xbef603ff,
+ 0x01000000, 0xbef20378,
+ 0x8078ff78, 0x00000080,
+ 0xbefc0381, 0xe0304000,
+ 0x785d0000, 0xbf8c3f70,
+ 0x7e008500, 0x807c817c,
+ 0x8078ff78, 0x00000080,
+ 0xbf0a6f7c, 0xbf85fff7,
+ 0xe0304000, 0x725d0000,
+ 0xbf820023, 0x8f76886f,
+ 0xbef603ff, 0x01000000,
+ 0xbef20378, 0x8078ff78,
+ 0x00000100, 0xbefc0381,
+ 0xe0304000, 0x785d0000,
+ 0xbf8c3f70, 0x7e008500,
+ 0x807c817c, 0x8078ff78,
+ 0x00000100, 0xbf0a6f7c,
+ 0xbf85fff7, 0xb96f1e06,
+ 0x876fc16f, 0xbf84000e,
+ 0x8f6f836f, 0x806f7c6f,
+ 0xbefe03c1, 0xbeff0380,
+ 0xe0304000, 0x785d0000,
+ 0xbf8c3f70, 0x7e008500,
+ 0x807c817c, 0x8078ff78,
+ 0x00000080, 0xbf0a6f7c,
+ 0xbf85fff7, 0xbeff03c1,
+ 0xe0304000, 0x725d0000,
+ 0x8772ff79, 0x04000000,
+ 0xbf840020, 0xbefe03c1,
+ 0x877c8179, 0xbf06817c,
+ 0xbf850002, 0xbeff0380,
+ 0xbf820001, 0xbeff03c1,
+ 0xb96f4306, 0x876fc16f,
+ 0xbf840016, 0x8f6f866f,
+ 0x8f6f826f, 0xbef6036f,
+ 0xbef603ff, 0x01000000,
+ 0x877c8172, 0xbf06817c,
+ 0xbefc0380, 0xbf850007,
+ 0x807cff7c, 0x00000080,
+ 0x8078ff78, 0x00000080,
+ 0xbf0a6f7c, 0xbf85fffa,
+ 0xbf820006, 0x807cff7c,
+ 0x00000100, 0x8078ff78,
+ 0x00000100, 0xbf0a6f7c,
+ 0xbf85fffa, 0x877c8179,
+ 0xbf06817c, 0xbf850003,
+ 0x8f7687ff, 0x0000006a,
+ 0xbf820002, 0x8f7688ff,
+ 0x0000006a, 0xbef603ff,
+ 0x01000000, 0x877c8179,
+ 0xbf06817c, 0xbf850012,
+ 0xf4211cba, 0xf0000000,
+ 0x8078ff78, 0x00000080,
+ 0xbefc0381, 0xf421003a,
+ 0xf0000000, 0x8078ff78,
+ 0x00000080, 0xbf8cc07f,
+ 0xbe803000, 0xbf800000,
+ 0x807c817c, 0xbf0aff7c,
+ 0x0000006a, 0xbf85fff5,
+ 0xbe800372, 0xbf820011,
+ 0xf4211cba, 0xf0000000,
+ 0x8078ff78, 0x00000100,
+ 0xbefc0381, 0xf421003a,
+ 0xf0000000, 0x8078ff78,
+ 0x00000100, 0xbf8cc07f,
+ 0xbe803000, 0xbf800000,
+ 0x807c817c, 0xbf0aff7c,
+ 0x0000006a, 0xbf85fff5,
+ 0xbe800372, 0xbef60384,
+ 0xbef603ff, 0x01000000,
+ 0x877c8179, 0xbf06817c,
+ 0xbf850025, 0xf4211bfa,
+ 0xf0000000, 0x8078ff78,
+ 0x00000080, 0xf4211b3a,
+ 0xf0000000, 0x8078ff78,
+ 0x00000080, 0xf4211b7a,
+ 0xf0000000, 0x8078ff78,
+ 0x00000080, 0xf4211eba,
+ 0xf0000000, 0x8078ff78,
+ 0x00000080, 0xf4211efa,
+ 0xf0000000, 0x8078ff78,
+ 0x00000080, 0xf4211c3a,
+ 0xf0000000, 0x8078ff78,
+ 0x00000080, 0xf4211c7a,
+ 0xf0000000, 0x8078ff78,
+ 0x00000080, 0xf4211cfa,
+ 0xf0000000, 0x8078ff78,
+ 0x00000080, 0xf4211e7a,
+ 0xf0000000, 0x8078ff78,
+ 0x00000080, 0xbf820024,
+ 0xf4211bfa, 0xf0000000,
+ 0x8078ff78, 0x00000100,
+ 0xf4211b3a, 0xf0000000,
+ 0x8078ff78, 0x00000100,
+ 0xf4211b7a, 0xf0000000,
+ 0x8078ff78, 0x00000100,
+ 0xf4211eba, 0xf0000000,
+ 0x8078ff78, 0x00000100,
+ 0xf4211efa, 0xf0000000,
+ 0x8078ff78, 0x00000100,
+ 0xf4211c3a, 0xf0000000,
+ 0x8078ff78, 0x00000100,
+ 0xf4211c7a, 0xf0000000,
+ 0x8078ff78, 0x00000100,
+ 0xf4211cfa, 0xf0000000,
+ 0x8078ff78, 0x00000100,
+ 0xf4211e7a, 0xf0000000,
+ 0x8078ff78, 0x00000100,
+ 0xbf8cc07f, 0x876dff6d,
+ 0x0000ffff, 0xbefc036f,
+ 0xbefe037a, 0xbeff037b,
+ 0x876f71ff, 0x000003ff,
+ 0xb9ef4803, 0xb9f3f816,
+ 0x876f71ff, 0xfffff800,
+ 0x906f8b6f, 0xb9efa2c3,
+ 0xb9f9f801, 0x876fff6d,
+ 0xf0000000, 0x906f9c6f,
+ 0x8f6f906f, 0xbef20380,
+ 0x88726f72, 0x876fff6d,
+ 0x08000000, 0x906f9b6f,
+ 0x8f6f8f6f, 0x88726f72,
+ 0x876fff70, 0x00800000,
+ 0x906f976f, 0xb9f2f807,
+ 0xb9f0f802, 0xbf8a0000,
+ 0xbe80226c, 0xbf810000,
+ 0xbf9f0000, 0xbf9f0000,
+ 0xbf9f0000, 0xbf9f0000,
+ 0xbf9f0000, 0x00000000,
};
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm
new file mode 100644
index 000000000000..f20e463e748b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm
@@ -0,0 +1,1124 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+
+shader main
+
+asic(DEFAULT)
+
+type(CS)
+
+wave_size(32)
+/*************************************************************************/
+/* control on how to run the shader */
+/*************************************************************************/
+//any hack that needs to be made to run this code in EMU (either becasue various EMU code are not ready or no compute save & restore in EMU run)
+var EMU_RUN_HACK = 0
+var EMU_RUN_HACK_RESTORE_NORMAL = 0
+var EMU_RUN_HACK_SAVE_NORMAL_EXIT = 0
+var EMU_RUN_HACK_SAVE_SINGLE_WAVE = 0
+var EMU_RUN_HACK_SAVE_FIRST_TIME = 0 //for interrupted restore in which the first save is through EMU_RUN_HACK
+var SAVE_LDS = 0
+var WG_BASE_ADDR_LO = 0x9000a000
+var WG_BASE_ADDR_HI = 0x0
+var WAVE_SPACE = 0x9000 //memory size that each wave occupies in workgroup state mem, increase from 5000 to 9000 for more SGPR need to be saved
+var CTX_SAVE_CONTROL = 0x0
+var CTX_RESTORE_CONTROL = CTX_SAVE_CONTROL
+var SIM_RUN_HACK = 0 //any hack that needs to be made to run this code in SIM (either becasue various RTL code are not ready or no compute save & restore in RTL run)
+var SGPR_SAVE_USE_SQC = 0 //use SQC D$ to do the write
+var USE_MTBUF_INSTEAD_OF_MUBUF = 0 //need to change BUF_DATA_FORMAT in S_SAVE_BUF_RSRC_WORD3_MISC from 0 to BUF_DATA_FORMAT_32 if set to 1 (i.e. 0x00827FAC)
+var SWIZZLE_EN = 0 //whether we use swizzled buffer addressing
+var SAVE_RESTORE_HWID_DDID = 0
+var RESTORE_DDID_IN_SGPR18 = 0
+/**************************************************************************/
+/* variables */
+/**************************************************************************/
+var SQ_WAVE_STATUS_INST_ATC_SHIFT = 23
+var SQ_WAVE_STATUS_INST_ATC_MASK = 0x00800000
+var SQ_WAVE_STATUS_SPI_PRIO_MASK = 0x00000006
+
+var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT = 12
+var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE = 9
+var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT = 8
+var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE = 6
+var SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT = 24
+var SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE = 4 //FIXME sq.blk still has 4 bits at this time while SQ programming guide has 3 bits
+var SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT = 24
+var SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE = 4
+var SQ_WAVE_IB_STS2_WAVE64_SHIFT = 11
+var SQ_WAVE_IB_STS2_WAVE64_SIZE = 1
+
+var SQ_WAVE_TRAPSTS_SAVECTX_MASK = 0x400
+var SQ_WAVE_TRAPSTS_EXCE_MASK = 0x1FF // Exception mask
+var SQ_WAVE_TRAPSTS_SAVECTX_SHIFT = 10
+var SQ_WAVE_TRAPSTS_MEM_VIOL_MASK = 0x100
+var SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT = 8
+var SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK = 0x3FF
+var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT = 0x0
+var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE = 10
+var SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK = 0xFFFFF800
+var SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT = 11
+var SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE = 21
+
+var SQ_WAVE_IB_STS_RCNT_SHIFT = 16 //FIXME
+var SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT = 15 //FIXME
+var SQ_WAVE_IB_STS_FIRST_REPLAY_SIZE = 1 //FIXME
+var SQ_WAVE_IB_STS_RCNT_SIZE = 6 //FIXME
+var SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK_NEG = 0x00007FFF //FIXME
+
+var SQ_BUF_RSRC_WORD1_ATC_SHIFT = 24
+var SQ_BUF_RSRC_WORD3_MTYPE_SHIFT = 27
+
+
+/* Save */
+var S_SAVE_BUF_RSRC_WORD1_STRIDE = 0x00040000 //stride is 4 bytes
+var S_SAVE_BUF_RSRC_WORD3_MISC = 0x00807FAC //SQ_SEL_X/Y/Z/W, BUF_NUM_FORMAT_FLOAT, (0 for MUBUF stride[17:14] when ADD_TID_ENABLE and BUF_DATA_FORMAT_32 for MTBUF), ADD_TID_ENABLE
+
+var S_SAVE_SPI_INIT_ATC_MASK = 0x08000000 //bit[27]: ATC bit
+var S_SAVE_SPI_INIT_ATC_SHIFT = 27
+var S_SAVE_SPI_INIT_MTYPE_MASK = 0x70000000 //bit[30:28]: Mtype
+var S_SAVE_SPI_INIT_MTYPE_SHIFT = 28
+var S_SAVE_SPI_INIT_FIRST_WAVE_MASK = 0x04000000 //bit[26]: FirstWaveInTG
+var S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT = 26
+
+var S_SAVE_PC_HI_RCNT_SHIFT = 28 //FIXME check with Brian to ensure all fields other than PC[47:0] can be used
+var S_SAVE_PC_HI_RCNT_MASK = 0xF0000000 //FIXME
+var S_SAVE_PC_HI_FIRST_REPLAY_SHIFT = 27 //FIXME
+var S_SAVE_PC_HI_FIRST_REPLAY_MASK = 0x08000000 //FIXME
+
+var s_save_spi_init_lo = exec_lo
+var s_save_spi_init_hi = exec_hi
+
+var s_save_pc_lo = ttmp0 //{TTMP1, TTMP0} = {3¡¯h0,pc_rewind[3:0], HT[0],trapID[7:0], PC[47:0]}
+var s_save_pc_hi = ttmp1
+var s_save_exec_lo = ttmp2
+var s_save_exec_hi = ttmp3
+var s_save_status = ttmp4
+var s_save_trapsts = ttmp5 //not really used until the end of the SAVE routine
+var s_wave_size = ttmp6 //ttmp6 is not needed now, since it's only 32bit xnack mask, now use it to determine wave32 or wave64 in EMU_HACK
+var s_save_xnack_mask = ttmp7
+var s_save_buf_rsrc0 = ttmp8
+var s_save_buf_rsrc1 = ttmp9
+var s_save_buf_rsrc2 = ttmp10
+var s_save_buf_rsrc3 = ttmp11
+
+var s_save_mem_offset = ttmp14
+var s_sgpr_save_num = 106 //in gfx10, all sgpr must be saved
+var s_save_alloc_size = s_save_trapsts //conflict
+var s_save_tmp = s_save_buf_rsrc2 //shared with s_save_buf_rsrc2 (conflict: should not use mem access with s_save_tmp at the same time)
+var s_save_m0 = ttmp15
+
+/* Restore */
+var S_RESTORE_BUF_RSRC_WORD1_STRIDE = S_SAVE_BUF_RSRC_WORD1_STRIDE
+var S_RESTORE_BUF_RSRC_WORD3_MISC = S_SAVE_BUF_RSRC_WORD3_MISC
+
+var S_RESTORE_SPI_INIT_ATC_MASK = 0x08000000 //bit[27]: ATC bit
+var S_RESTORE_SPI_INIT_ATC_SHIFT = 27
+var S_RESTORE_SPI_INIT_MTYPE_MASK = 0x70000000 //bit[30:28]: Mtype
+var S_RESTORE_SPI_INIT_MTYPE_SHIFT = 28
+var S_RESTORE_SPI_INIT_FIRST_WAVE_MASK = 0x04000000 //bit[26]: FirstWaveInTG
+var S_RESTORE_SPI_INIT_FIRST_WAVE_SHIFT = 26
+
+var S_RESTORE_PC_HI_RCNT_SHIFT = S_SAVE_PC_HI_RCNT_SHIFT
+var S_RESTORE_PC_HI_RCNT_MASK = S_SAVE_PC_HI_RCNT_MASK
+var S_RESTORE_PC_HI_FIRST_REPLAY_SHIFT = S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+var S_RESTORE_PC_HI_FIRST_REPLAY_MASK = S_SAVE_PC_HI_FIRST_REPLAY_MASK
+
+var s_restore_spi_init_lo = exec_lo
+var s_restore_spi_init_hi = exec_hi
+
+var s_restore_mem_offset = ttmp12
+var s_restore_alloc_size = ttmp3
+var s_restore_tmp = ttmp6
+var s_restore_mem_offset_save = s_restore_tmp //no conflict
+
+var s_restore_m0 = s_restore_alloc_size //no conflict
+
+var s_restore_mode = ttmp13
+var s_restore_hwid1 = ttmp2
+var s_restore_ddid = s_restore_hwid1
+var s_restore_pc_lo = ttmp0
+var s_restore_pc_hi = ttmp1
+var s_restore_exec_lo = ttmp14
+var s_restore_exec_hi = ttmp15
+var s_restore_status = ttmp4
+var s_restore_trapsts = ttmp5
+//var s_restore_xnack_mask_lo = xnack_mask_lo
+//var s_restore_xnack_mask_hi = xnack_mask_hi
+var s_restore_xnack_mask = ttmp7
+var s_restore_buf_rsrc0 = ttmp8
+var s_restore_buf_rsrc1 = ttmp9
+var s_restore_buf_rsrc2 = ttmp10
+var s_restore_buf_rsrc3 = ttmp11
+var s_restore_size = ttmp13 //ttmp13 has no conflict
+
+/**************************************************************************/
+/* trap handler entry points */
+/**************************************************************************/
+ if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL)) //hack to use trap_id for determining save/restore
+ //FIXME VCCZ un-init assertion s_getreg_b32 s_save_status, hwreg(HW_REG_STATUS) //save STATUS since we will change SCC
+ s_and_b32 s_save_tmp, s_save_pc_hi, 0xffff0000 //change SCC
+ s_cmp_eq_u32 s_save_tmp, 0x007e0000 //Save: trap_id = 0x7e. Restore: trap_id = 0x7f.
+ s_cbranch_scc0 L_JUMP_TO_RESTORE //do not need to recover STATUS here since we are going to RESTORE
+ //FIXME s_setreg_b32 hwreg(HW_REG_STATUS), s_save_status //need to recover STATUS since we are going to SAVE
+ s_branch L_SKIP_RESTORE //NOT restore, SAVE actually
+ else
+ s_branch L_SKIP_RESTORE //NOT restore. might be a regular trap or save
+ end
+
+L_JUMP_TO_RESTORE:
+ s_branch L_RESTORE //restore
+
+L_SKIP_RESTORE:
+
+ s_getreg_b32 s_save_status, hwreg(HW_REG_STATUS) //save STATUS since we will change SCC
+ s_andn2_b32 s_save_status, s_save_status, SQ_WAVE_STATUS_SPI_PRIO_MASK //check whether this is for save
+ s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+ s_and_b32 s_save_trapsts, s_save_trapsts, SQ_WAVE_TRAPSTS_SAVECTX_MASK //check whether this is for save
+ s_cbranch_scc1 L_SAVE //this is the operation for save
+
+ // ********* Handle non-CWSR traps *******************
+ if (!EMU_RUN_HACK)
+ s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+ s_and_b32 s_save_trapsts, s_save_trapsts, SQ_WAVE_TRAPSTS_EXCE_MASK // Check whether it is an exception
+ s_cbranch_scc1 L_EXCP_CASE // Exception, jump back to the shader program directly.
+ s_add_u32 ttmp0, ttmp0, 4 // S_TRAP case, add 4 to ttmp0
+
+ L_EXCP_CASE:
+ s_and_b32 ttmp1, ttmp1, 0xFFFF
+ s_rfe_b64 [ttmp0, ttmp1]
+ end
+ // ********* End handling of non-CWSR traps *******************
+
+/**************************************************************************/
+/* save routine */
+/**************************************************************************/
+
+L_SAVE:
+
+ //check whether there is mem_viol
+ s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+ s_and_b32 s_save_trapsts, s_save_trapsts, SQ_WAVE_TRAPSTS_MEM_VIOL_MASK
+ s_cbranch_scc0 L_NO_PC_REWIND
+
+ //if so, need rewind PC assuming GDS operation gets NACKed
+ s_mov_b32 s_save_tmp, 0 //clear mem_viol bit
+ s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT, 1), s_save_tmp //clear mem_viol bit
+ s_and_b32 s_save_pc_hi, s_save_pc_hi, 0x0000ffff //pc[47:32]
+ s_sub_u32 s_save_pc_lo, s_save_pc_lo, 8 //pc[31:0]-8
+ s_subb_u32 s_save_pc_hi, s_save_pc_hi, 0x0 // -scc
+
+L_NO_PC_REWIND:
+ s_mov_b32 s_save_tmp, 0 //clear saveCtx bit
+ s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_SAVECTX_SHIFT, 1), s_save_tmp //clear saveCtx bit
+
+ //s_mov_b32 s_save_xnack_mask_lo, xnack_mask_lo //save XNACK_MASK
+ //s_mov_b32 s_save_xnack_mask_hi, xnack_mask_hi
+ s_getreg_b32 s_save_xnack_mask, hwreg(HW_REG_SHADER_XNACK_MASK)
+ s_getreg_b32 s_save_tmp, hwreg(HW_REG_IB_STS, SQ_WAVE_IB_STS_RCNT_SHIFT, SQ_WAVE_IB_STS_RCNT_SIZE) //save RCNT
+ s_lshl_b32 s_save_tmp, s_save_tmp, S_SAVE_PC_HI_RCNT_SHIFT
+ s_or_b32 s_save_pc_hi, s_save_pc_hi, s_save_tmp
+ s_getreg_b32 s_save_tmp, hwreg(HW_REG_IB_STS, SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT, SQ_WAVE_IB_STS_FIRST_REPLAY_SIZE) //save FIRST_REPLAY
+ s_lshl_b32 s_save_tmp, s_save_tmp, S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+ s_or_b32 s_save_pc_hi, s_save_pc_hi, s_save_tmp
+ s_getreg_b32 s_save_tmp, hwreg(HW_REG_IB_STS) //clear RCNT and FIRST_REPLAY in IB_STS
+ s_and_b32 s_save_tmp, s_save_tmp, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK_NEG
+
+ s_setreg_b32 hwreg(HW_REG_IB_STS), s_save_tmp
+
+ /* inform SPI the readiness and wait for SPI's go signal */
+ s_mov_b32 s_save_exec_lo, exec_lo //save EXEC and use EXEC for the go signal from SPI
+ s_mov_b32 s_save_exec_hi, exec_hi
+ s_mov_b64 exec, 0x0 //clear EXEC to get ready to receive
+ if (EMU_RUN_HACK)
+
+ else
+ s_sendmsg sendmsg(MSG_SAVEWAVE) //send SPI a message and wait for SPI's write to EXEC
+ end
+
+ L_SLEEP:
+ s_sleep 0x2
+
+ if (EMU_RUN_HACK)
+
+ else
+ s_cbranch_execz L_SLEEP
+ end
+
+
+ /* setup Resource Contants */
+ if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_SAVE_SINGLE_WAVE))
+ //calculate wd_addr using absolute thread id
+ v_readlane_b32 s_save_tmp, v9, 0
+ //determine it is wave32 or wave64
+ s_getreg_b32 s_wave_size, hwreg(HW_REG_IB_STS2,SQ_WAVE_IB_STS2_WAVE64_SHIFT,SQ_WAVE_IB_STS2_WAVE64_SIZE)
+ s_cmp_eq_u32 s_wave_size, 0
+ s_cbranch_scc1 L_SAVE_WAVE32
+ s_lshr_b32 s_save_tmp, s_save_tmp, 6 //SAVE WAVE64
+ s_branch L_SAVE_CON
+ L_SAVE_WAVE32:
+ s_lshr_b32 s_save_tmp, s_save_tmp, 5 //SAVE WAVE32
+ L_SAVE_CON:
+ s_mul_i32 s_save_tmp, s_save_tmp, WAVE_SPACE
+ s_add_i32 s_save_spi_init_lo, s_save_tmp, WG_BASE_ADDR_LO
+ s_mov_b32 s_save_spi_init_hi, WG_BASE_ADDR_HI
+ s_and_b32 s_save_spi_init_hi, s_save_spi_init_hi, CTX_SAVE_CONTROL
+ else
+ end
+ if ((EMU_RUN_HACK) && (EMU_RUN_HACK_SAVE_SINGLE_WAVE))
+ s_add_i32 s_save_spi_init_lo, s_save_tmp, WG_BASE_ADDR_LO
+ s_mov_b32 s_save_spi_init_hi, WG_BASE_ADDR_HI
+ s_and_b32 s_save_spi_init_hi, s_save_spi_init_hi, CTX_SAVE_CONTROL
+ else
+ end
+
+
+ s_mov_b32 s_save_buf_rsrc0, s_save_spi_init_lo //base_addr_lo
+ s_and_b32 s_save_buf_rsrc1, s_save_spi_init_hi, 0x0000FFFF //base_addr_hi
+ s_or_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE
+ s_mov_b32 s_save_buf_rsrc2, 0 //NUM_RECORDS initial value = 0 (in bytes) although not neccessarily inited
+ s_mov_b32 s_save_buf_rsrc3, S_SAVE_BUF_RSRC_WORD3_MISC
+ s_and_b32 s_save_tmp, s_save_spi_init_hi, S_SAVE_SPI_INIT_ATC_MASK
+ s_lshr_b32 s_save_tmp, s_save_tmp, (S_SAVE_SPI_INIT_ATC_SHIFT-SQ_BUF_RSRC_WORD1_ATC_SHIFT) //get ATC bit into position
+ s_or_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, s_save_tmp //or ATC
+ s_and_b32 s_save_tmp, s_save_spi_init_hi, S_SAVE_SPI_INIT_MTYPE_MASK
+ s_lshr_b32 s_save_tmp, s_save_tmp, (S_SAVE_SPI_INIT_MTYPE_SHIFT-SQ_BUF_RSRC_WORD3_MTYPE_SHIFT) //get MTYPE bits into position
+ s_or_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, s_save_tmp //or MTYPE
+
+ s_mov_b32 s_save_m0, m0 //save M0
+
+ /* global mem offset */
+ s_mov_b32 s_save_mem_offset, 0x0 //mem offset initial value = 0
+ s_getreg_b32 s_wave_size, hwreg(HW_REG_IB_STS2,SQ_WAVE_IB_STS2_WAVE64_SHIFT,SQ_WAVE_IB_STS2_WAVE64_SIZE) //get wave_save_size
+ s_or_b32 s_wave_size, s_save_spi_init_hi, s_wave_size //share s_wave_size with exec_hi
+
+ /* save VGPRs */
+ //////////////////////////////
+ L_SAVE_VGPR:
+
+ s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
+ s_and_b32 m0, s_wave_size, 1
+ s_cmp_eq_u32 m0, 1
+ s_cbranch_scc1 L_ENABLE_SAVE_VGPR_EXEC_HI
+ s_mov_b32 exec_hi, 0x00000000
+ s_branch L_SAVE_VGPR_NORMAL
+ L_ENABLE_SAVE_VGPR_EXEC_HI:
+ s_mov_b32 exec_hi, 0xFFFFFFFF
+ L_SAVE_VGPR_NORMAL:
+ s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE) //vpgr_size
+ //for wave32 and wave64, the num of vgpr function is the same?
+ s_add_u32 s_save_alloc_size, s_save_alloc_size, 1
+ s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 2 //Number of VGPRs = (vgpr_size + 1) * 4 (non-zero value) //FIXME for GFX, zero is possible
+ //determine it is wave32 or wave64
+ s_and_b32 m0, s_wave_size, 1
+ s_cmp_eq_u32 m0, 1
+ s_cbranch_scc1 L_SAVE_VGPR_WAVE64
+
+ //zhenxu added it for save vgpr for wave32
+ s_lshl_b32 s_save_buf_rsrc2, s_save_alloc_size, 7 //NUM_RECORDS in bytes (32 threads*4)
+ if (SWIZZLE_EN)
+ s_add_u32 s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+ else
+ s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+ end
+
+ s_mov_b32 m0, 0x0 //VGPR initial index value =0
+ //s_set_gpr_idx_on m0, 0x1 //M0[7:0] = M0[7:0] and M0[15:12] = 0x1
+ //s_add_u32 s_save_alloc_size, s_save_alloc_size, 0x1000 //add 0x1000 since we compare m0 against it later, doesn't need this in gfx10
+
+ L_SAVE_VGPR_WAVE32_LOOP:
+ v_movrels_b32 v0, v0 //v0 = v[0+m0]
+
+ if(USE_MTBUF_INSTEAD_OF_MUBUF)
+ tbuffer_store_format_x v0, v0, s_save_buf_rsrc0, s_save_mem_offset format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+ else
+ buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+ end
+
+ s_add_u32 m0, m0, 1 //next vgpr index
+ s_add_u32 s_save_mem_offset, s_save_mem_offset, 128 //every buffer_store_dword does 128 bytes
+ s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0
+ s_cbranch_scc1 L_SAVE_VGPR_WAVE32_LOOP //VGPR save is complete?
+ s_branch L_SAVE_LDS
+ //save vgpr for wave32 ends
+
+ L_SAVE_VGPR_WAVE64:
+ s_lshl_b32 s_save_buf_rsrc2, s_save_alloc_size, 8 //NUM_RECORDS in bytes (64 threads*4)
+ if (SWIZZLE_EN)
+ s_add_u32 s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+ else
+ s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+ end
+
+ s_mov_b32 m0, 0x0 //VGPR initial index value =0
+ //s_set_gpr_idx_on m0, 0x1 //M0[7:0] = M0[7:0] and M0[15:12] = 0x1
+ //s_add_u32 s_save_alloc_size, s_save_alloc_size, 0x1000 //add 0x1000 since we compare m0 against it later, doesn't need this in gfx10
+
+ L_SAVE_VGPR_WAVE64_LOOP:
+ v_movrels_b32 v0, v0 //v0 = v[0+m0]
+
+ if(USE_MTBUF_INSTEAD_OF_MUBUF)
+ tbuffer_store_format_x v0, v0, s_save_buf_rsrc0, s_save_mem_offset format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+ else
+ buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+ end
+
+ s_add_u32 m0, m0, 1 //next vgpr index
+ s_add_u32 s_save_mem_offset, s_save_mem_offset, 256 //every buffer_store_dword does 256 bytes
+ s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0
+ s_cbranch_scc1 L_SAVE_VGPR_WAVE64_LOOP //VGPR save is complete?
+ //s_set_gpr_idx_off
+ //
+ //Below part will be the save shared vgpr part (new for gfx10)
+ s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE) //shared_vgpr_size
+ s_and_b32 s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF //shared_vgpr_size is zero?
+ s_cbranch_scc0 L_SAVE_LDS //no shared_vgpr used? jump to L_SAVE_LDS
+ s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 3 //Number of SHARED_VGPRs = shared_vgpr_size * 8 (non-zero value)
+ //m0 now has the value of normal vgpr count, just add the m0 with shared_vgpr count to get the total count.
+ //save shared_vgpr will start from the index of m0
+ s_add_u32 s_save_alloc_size, s_save_alloc_size, m0
+ s_mov_b32 exec_lo, 0xFFFFFFFF
+ s_mov_b32 exec_hi, 0x00000000
+ L_SAVE_SHARED_VGPR_WAVE64_LOOP:
+ v_movrels_b32 v0, v0 //v0 = v[0+m0]
+ buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+ s_add_u32 m0, m0, 1 //next vgpr index
+ s_add_u32 s_save_mem_offset, s_save_mem_offset, 128 //every buffer_store_dword does 256 bytes
+ s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0
+ s_cbranch_scc1 L_SAVE_SHARED_VGPR_WAVE64_LOOP //SHARED_VGPR save is complete?
+
+ /* save LDS */
+ //////////////////////////////
+ L_SAVE_LDS:
+
+ //Only check the first wave need LDS
+ /* the first wave in the threadgroup */
+ s_barrier //FIXME not performance-optimal "LDS is used? wait for other waves in the same TG"
+ s_and_b32 s_save_tmp, s_wave_size, S_SAVE_SPI_INIT_FIRST_WAVE_MASK //exec is still used here
+ s_cbranch_scc0 L_SAVE_SGPR
+
+ s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
+ s_and_b32 m0, s_wave_size, 1
+ s_cmp_eq_u32 m0, 1
+ s_cbranch_scc1 L_ENABLE_SAVE_LDS_EXEC_HI
+ s_mov_b32 exec_hi, 0x00000000
+ s_branch L_SAVE_LDS_NORMAL
+ L_ENABLE_SAVE_LDS_EXEC_HI:
+ s_mov_b32 exec_hi, 0xFFFFFFFF
+ L_SAVE_LDS_NORMAL:
+ s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE) //lds_size
+ s_and_b32 s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF //lds_size is zero?
+ s_cbranch_scc0 L_SAVE_SGPR //no lds used? jump to L_SAVE_VGPR
+ s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 6 //LDS size in dwords = lds_size * 64dw
+ s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 2 //LDS size in bytes
+ s_mov_b32 s_save_buf_rsrc2, s_save_alloc_size //NUM_RECORDS in bytes
+ if (SWIZZLE_EN)
+ s_add_u32 s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+ else
+ s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+ end
+
+ //load 0~63*4(byte address) to vgpr v15
+ v_mbcnt_lo_u32_b32 v0, -1, 0
+ v_mbcnt_hi_u32_b32 v0, -1, v0
+ v_mul_u32_u24 v0, 4, v0
+
+ s_and_b32 m0, s_wave_size, 1
+ s_cmp_eq_u32 m0, 1
+ s_mov_b32 m0, 0x0
+ s_cbranch_scc1 L_SAVE_LDS_LOOP_W64
+
+ L_SAVE_LDS_LOOP_W32:
+ if (SAVE_LDS)
+ ds_read_b32 v1, v0
+ s_waitcnt 0 //ensure data ready
+ buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+ //buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 //save lds to memory doesn't exist in 10
+ end
+ s_add_u32 m0, m0, 128 //every buffer_store_lds does 128 bytes
+ s_add_u32 s_save_mem_offset, s_save_mem_offset, 128 //mem offset increased by 128 bytes
+ v_add_nc_u32 v0, v0, 128
+ s_cmp_lt_u32 m0, s_save_alloc_size //scc=(m0 < s_save_alloc_size) ? 1 : 0
+ s_cbranch_scc1 L_SAVE_LDS_LOOP_W32 //LDS save is complete?
+ s_branch L_SAVE_SGPR
+
+ L_SAVE_LDS_LOOP_W64:
+ if (SAVE_LDS)
+ ds_read_b32 v1, v0
+ s_waitcnt 0 //ensure data ready
+ buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+ //buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 //save lds to memory doesn't exist in 10
+ end
+ s_add_u32 m0, m0, 256 //every buffer_store_lds does 256 bytes
+ s_add_u32 s_save_mem_offset, s_save_mem_offset, 256 //mem offset increased by 256 bytes
+ v_add_nc_u32 v0, v0, 256
+ s_cmp_lt_u32 m0, s_save_alloc_size //scc=(m0 < s_save_alloc_size) ? 1 : 0
+ s_cbranch_scc1 L_SAVE_LDS_LOOP_W64 //LDS save is complete?
+
+
+ /* save SGPRs */
+ //////////////////////////////
+ //s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE) //spgr_size
+ //s_add_u32 s_save_alloc_size, s_save_alloc_size, 1
+ //s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 4 //Number of SGPRs = (sgpr_size + 1) * 16 (non-zero value)
+ //s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 3 //In gfx10, Number of SGPRs = (sgpr_size + 1) * 8 (non-zero value)
+ L_SAVE_SGPR:
+ //need to look at it is wave32 or wave64
+ s_and_b32 m0, s_wave_size, 1
+ s_cmp_eq_u32 m0, 1
+ s_cbranch_scc1 L_SAVE_SGPR_VMEM_WAVE64
+ if (SGPR_SAVE_USE_SQC)
+ s_lshl_b32 s_save_buf_rsrc2, s_sgpr_save_num, 2 //NUM_RECORDS in bytes
+ else
+ s_lshl_b32 s_save_buf_rsrc2, s_sgpr_save_num, 7 //NUM_RECORDS in bytes (32 threads)
+ end
+ s_branch L_SAVE_SGPR_CONT
+ L_SAVE_SGPR_VMEM_WAVE64:
+ if (SGPR_SAVE_USE_SQC)
+ s_lshl_b32 s_save_buf_rsrc2, s_sgpr_save_num, 2 //NUM_RECORDS in bytes
+ else
+ s_lshl_b32 s_save_buf_rsrc2, s_sgpr_save_num, 8 //NUM_RECORDS in bytes (64 threads)
+ end
+ L_SAVE_SGPR_CONT:
+ if (SWIZZLE_EN)
+ s_add_u32 s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+ else
+ s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+ end
+
+ //s_mov_b32 m0, 0x0 //SGPR initial index value =0
+ //s_nop 0x0 //Manually inserted wait states
+
+ s_and_b32 m0, s_wave_size, 1
+ s_cmp_eq_u32 m0, 1
+
+ s_mov_b32 m0, 0x0 //SGPR initial index value =0
+ s_nop 0x0 //Manually inserted wait states
+
+ s_cbranch_scc1 L_SAVE_SGPR_LOOP_WAVE64
+
+ L_SAVE_SGPR_LOOP_WAVE32:
+ s_movrels_b32 s0, s0 //s0 = s[0+m0]
+ //zhenxu, adding one more argument to save sgpr function, this is only for vmem, using sqc is not change
+ write_sgpr_to_mem_wave32(s0, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF) //PV: the best performance should be using s_buffer_store_dwordx4
+ s_add_u32 m0, m0, 1 //next sgpr index
+ s_cmp_lt_u32 m0, s_sgpr_save_num //scc = (m0 < s_sgpr_save_num) ? 1 : 0
+ s_cbranch_scc1 L_SAVE_SGPR_LOOP_WAVE32 //SGPR save is complete?
+ s_branch L_SAVE_HWREG
+
+ L_SAVE_SGPR_LOOP_WAVE64:
+ s_movrels_b32 s0, s0 //s0 = s[0+m0]
+ //zhenxu, adding one more argument to save sgpr function, this is only for vmem, using sqc is not change
+ write_sgpr_to_mem_wave64(s0, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF) //PV: the best performance should be using s_buffer_store_dwordx4
+ s_add_u32 m0, m0, 1 //next sgpr index
+ s_cmp_lt_u32 m0, s_sgpr_save_num //scc = (m0 < s_sgpr_save_num) ? 1 : 0
+ s_cbranch_scc1 L_SAVE_SGPR_LOOP_WAVE64 //SGPR save is complete?
+
+
+ /* save HW registers */
+ //////////////////////////////
+ L_SAVE_HWREG:
+ s_mov_b32 s_save_buf_rsrc2, 0x4 //NUM_RECORDS in bytes
+ if (SWIZZLE_EN)
+ s_add_u32 s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+ else
+ s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+ end
+
+ s_and_b32 m0, s_wave_size, 1
+ s_cmp_eq_u32 m0, 1
+ s_cbranch_scc1 L_SAVE_HWREG_WAVE64
+
+ write_sgpr_to_mem_wave32(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF) //M0
+
+ if ((EMU_RUN_HACK) && (EMU_RUN_HACK_SAVE_FIRST_TIME))
+ s_add_u32 s_save_pc_lo, s_save_pc_lo, 4 //pc[31:0]+4
+ s_addc_u32 s_save_pc_hi, s_save_pc_hi, 0x0 //carry bit over
+ end
+
+ write_sgpr_to_mem_wave32(s_save_pc_lo, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF) //PC
+ write_sgpr_to_mem_wave32(s_save_pc_hi, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF)
+ write_sgpr_to_mem_wave32(s_save_exec_lo, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF) //EXEC
+ write_sgpr_to_mem_wave32(s_save_exec_hi, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF)
+ write_sgpr_to_mem_wave32(s_save_status, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF) //STATUS
+
+ //s_save_trapsts conflicts with s_save_alloc_size
+ s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+ write_sgpr_to_mem_wave32(s_save_trapsts, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF) //TRAPSTS
+
+ //write_sgpr_to_mem_wave32(s_save_xnack_mask_lo, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF) //XNACK_MASK_LO
+ write_sgpr_to_mem_wave32(s_save_xnack_mask, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF) //XNACK_MASK_HI
+
+ //use s_save_tmp would introduce conflict here between s_save_tmp and s_save_buf_rsrc2
+ s_getreg_b32 s_save_m0, hwreg(HW_REG_MODE) //MODE
+ write_sgpr_to_mem_wave32(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF)
+ if(SAVE_RESTORE_HWID_DDID)
+ s_getreg_b32 s_save_m0, hwreg(HW_REG_HW_ID1) //HW_ID1, handler records the SE/SA/WGP/SIMD/wave of the original wave
+ write_sgpr_to_mem_wave32(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF)
+ end
+ s_branch L_S_PGM_END_SAVED
+
+ L_SAVE_HWREG_WAVE64:
+ write_sgpr_to_mem_wave64(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF) //M0
+
+ if ((EMU_RUN_HACK) && (EMU_RUN_HACK_SAVE_FIRST_TIME))
+ s_add_u32 s_save_pc_lo, s_save_pc_lo, 4 //pc[31:0]+4
+ s_addc_u32 s_save_pc_hi, s_save_pc_hi, 0x0 //carry bit over
+ end
+
+ write_sgpr_to_mem_wave64(s_save_pc_lo, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF) //PC
+ write_sgpr_to_mem_wave64(s_save_pc_hi, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF)
+ write_sgpr_to_mem_wave64(s_save_exec_lo, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF) //EXEC
+ write_sgpr_to_mem_wave64(s_save_exec_hi, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF)
+ write_sgpr_to_mem_wave64(s_save_status, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF) //STATUS
+
+ //s_save_trapsts conflicts with s_save_alloc_size
+ s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+ write_sgpr_to_mem_wave64(s_save_trapsts, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF) //TRAPSTS
+
+ //write_sgpr_to_mem_wave64(s_save_xnack_mask_lo, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF) //XNACK_MASK_LO
+ write_sgpr_to_mem_wave64(s_save_xnack_mask, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF) //XNACK_MASK_HI
+
+ //use s_save_tmp would introduce conflict here between s_save_tmp and s_save_buf_rsrc2
+ s_getreg_b32 s_save_m0, hwreg(HW_REG_MODE) //MODE
+ write_sgpr_to_mem_wave64(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF)
+
+
+ if(SAVE_RESTORE_HWID_DDID)
+ s_getreg_b32 s_save_m0, hwreg(HW_REG_HW_ID1) //HW_ID1, handler records the SE/SA/WGP/SIMD/wave of the original wave
+ write_sgpr_to_mem_wave64(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF)
+
+ /* save DDID */
+ //////////////////////////////
+ L_SAVE_DDID:
+ //EXEC has been saved, no vector inst following
+ s_mov_b32 exec_lo, 0x80000000 //Set MSB to 1. Cleared when draw index is returned
+ s_sendmsg sendmsg(MSG_GET_DDID)
+
+ L_WAIT_DDID_LOOP:
+ s_nop 7 // sleep a bit
+ s_bitcmp0_b32 exec_lo, 31 // test to see if MSB is cleared, meaning done
+ s_cbranch_scc0 L_WAIT_DDID_LOOP
+
+ s_mov_b32 s_save_m0, exec_lo
+
+
+ s_mov_b32 s_save_buf_rsrc2, 0x4 //NUM_RECORDS in bytes
+ if (SWIZZLE_EN)
+ s_add_u32 s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+ else
+ s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+ end
+ s_and_b32 m0, s_wave_size, 1
+ s_cmp_eq_u32 m0, 1
+ s_cbranch_scc1 L_SAVE_DDID_WAVE64
+
+ write_sgpr_to_mem_wave32(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF)
+
+ L_SAVE_DDID_WAVE64:
+ write_sgpr_to_mem_wave64(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset, SGPR_SAVE_USE_SQC, USE_MTBUF_INSTEAD_OF_MUBUF)
+
+ end
+
+ L_S_PGM_END_SAVED:
+ /* S_PGM_END_SAVED */ //FIXME graphics ONLY
+ if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_SAVE_NORMAL_EXIT))
+ s_and_b32 s_save_pc_hi, s_save_pc_hi, 0x0000ffff //pc[47:32]
+ s_add_u32 s_save_pc_lo, s_save_pc_lo, 4 //pc[31:0]+4
+ s_addc_u32 s_save_pc_hi, s_save_pc_hi, 0x0 //carry bit over
+ s_rfe_b64 s_save_pc_lo //Return to the main shader program
+ else
+ end
+
+
+ s_branch L_END_PGM
+
+
+
+/**************************************************************************/
+/* restore routine */
+/**************************************************************************/
+
+L_RESTORE:
+ /* Setup Resource Contants */
+ if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL))
+ //calculate wd_addr using absolute thread id
+ v_readlane_b32 s_restore_tmp, v9, 0
+ //determine it is wave32 or wave64
+ s_getreg_b32 s_restore_size, hwreg(HW_REG_IB_STS2,SQ_WAVE_IB_STS2_WAVE64_SHIFT,SQ_WAVE_IB_STS2_WAVE64_SIZE) //change to ttmp13
+ s_cmp_eq_u32 s_restore_size, 0
+ s_cbranch_scc1 L_RESTORE_WAVE32
+ s_lshr_b32 s_restore_tmp, s_restore_tmp, 6 //SAVE WAVE64
+ s_branch L_RESTORE_CON
+ L_RESTORE_WAVE32:
+ s_lshr_b32 s_restore_tmp, s_restore_tmp, 5 //SAVE WAVE32
+ L_RESTORE_CON:
+ s_mul_i32 s_restore_tmp, s_restore_tmp, WAVE_SPACE
+ s_add_i32 s_restore_spi_init_lo, s_restore_tmp, WG_BASE_ADDR_LO
+ s_mov_b32 s_restore_spi_init_hi, WG_BASE_ADDR_HI
+ s_and_b32 s_restore_spi_init_hi, s_restore_spi_init_hi, CTX_RESTORE_CONTROL
+ else
+ end
+
+ s_mov_b32 s_restore_buf_rsrc0, s_restore_spi_init_lo //base_addr_lo
+ s_and_b32 s_restore_buf_rsrc1, s_restore_spi_init_hi, 0x0000FFFF //base_addr_hi
+ s_or_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, S_RESTORE_BUF_RSRC_WORD1_STRIDE
+ s_mov_b32 s_restore_buf_rsrc2, 0 //NUM_RECORDS initial value = 0 (in bytes)
+ s_mov_b32 s_restore_buf_rsrc3, S_RESTORE_BUF_RSRC_WORD3_MISC
+ s_and_b32 s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_ATC_MASK
+ s_lshr_b32 s_restore_tmp, s_restore_tmp, (S_RESTORE_SPI_INIT_ATC_SHIFT-SQ_BUF_RSRC_WORD1_ATC_SHIFT) //get ATC bit into position
+ s_or_b32 s_restore_buf_rsrc3, s_restore_buf_rsrc3, s_restore_tmp //or ATC
+ s_and_b32 s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_MTYPE_MASK
+ s_lshr_b32 s_restore_tmp, s_restore_tmp, (S_RESTORE_SPI_INIT_MTYPE_SHIFT-SQ_BUF_RSRC_WORD3_MTYPE_SHIFT) //get MTYPE bits into position
+ s_or_b32 s_restore_buf_rsrc3, s_restore_buf_rsrc3, s_restore_tmp //or MTYPE
+ //determine it is wave32 or wave64
+ s_getreg_b32 s_restore_size, hwreg(HW_REG_IB_STS2,SQ_WAVE_IB_STS2_WAVE64_SHIFT,SQ_WAVE_IB_STS2_WAVE64_SIZE)
+ s_or_b32 s_restore_size, s_restore_spi_init_hi, s_restore_size //share s_wave_size with exec_hi
+
+ /* global mem offset */
+ s_mov_b32 s_restore_mem_offset, 0x0 //mem offset initial value = 0
+
+ /* restore VGPRs */
+ //////////////////////////////
+ L_RESTORE_VGPR:
+
+ s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on //be consistent with SAVE although can be moved ahead
+ s_and_b32 m0, s_restore_size, 1
+ s_cmp_eq_u32 m0, 1
+ s_cbranch_scc1 L_ENABLE_RESTORE_VGPR_EXEC_HI
+ s_mov_b32 exec_hi, 0x00000000
+ s_branch L_RESTORE_VGPR_NORMAL
+ L_ENABLE_RESTORE_VGPR_EXEC_HI:
+ s_mov_b32 exec_hi, 0xFFFFFFFF
+ L_RESTORE_VGPR_NORMAL:
+ s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE) //vpgr_size
+ s_add_u32 s_restore_alloc_size, s_restore_alloc_size, 1
+ s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 2 //Number of VGPRs = (vgpr_size + 1) * 4 (non-zero value)
+ //determine it is wave32 or wave64
+ s_and_b32 m0, s_restore_size, 1
+ s_cmp_eq_u32 m0, 1
+ s_cbranch_scc1 L_RESTORE_VGPR_WAVE64
+
+ s_lshl_b32 s_restore_buf_rsrc2, s_restore_alloc_size, 7 //NUM_RECORDS in bytes (32 threads*4)
+ if (SWIZZLE_EN)
+ s_add_u32 s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+ else
+ s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+ end
+
+ s_mov_b32 s_restore_mem_offset_save, s_restore_mem_offset // restore start with v1, v0 will be the last
+ s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128
+ s_mov_b32 m0, 1 //VGPR initial index value = 1
+ //s_set_gpr_idx_on m0, 0x8 //M0[7:0] = M0[7:0] and M0[15:12] = 0x8
+ //s_add_u32 s_restore_alloc_size, s_restore_alloc_size, 0x8000 //add 0x8000 since we compare m0 against it later, might not need this in gfx10
+
+ L_RESTORE_VGPR_WAVE32_LOOP:
+ if(USE_MTBUF_INSTEAD_OF_MUBUF)
+ tbuffer_load_format_x v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+ else
+ buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
+ end
+ s_waitcnt vmcnt(0) //ensure data ready
+ v_movreld_b32 v0, v0 //v[0+m0] = v0
+ s_add_u32 m0, m0, 1 //next vgpr index
+ s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128 //every buffer_load_dword does 128 bytes
+ s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0
+ s_cbranch_scc1 L_RESTORE_VGPR_WAVE32_LOOP //VGPR restore (except v0) is complete?
+ //s_set_gpr_idx_off
+ /* VGPR restore on v0 */
+ if(USE_MTBUF_INSTEAD_OF_MUBUF)
+ tbuffer_load_format_x v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+ else
+ buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1
+ end
+
+ s_branch L_RESTORE_LDS
+
+ L_RESTORE_VGPR_WAVE64:
+ s_lshl_b32 s_restore_buf_rsrc2, s_restore_alloc_size, 8 //NUM_RECORDS in bytes (64 threads*4)
+ if (SWIZZLE_EN)
+ s_add_u32 s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+ else
+ s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+ end
+
+ s_mov_b32 s_restore_mem_offset_save, s_restore_mem_offset // restore start with v1, v0 will be the last
+ s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256
+ s_mov_b32 m0, 1 //VGPR initial index value = 1
+ L_RESTORE_VGPR_WAVE64_LOOP:
+ if(USE_MTBUF_INSTEAD_OF_MUBUF)
+ tbuffer_load_format_x v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+ else
+ buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
+ end
+ s_waitcnt vmcnt(0) //ensure data ready
+ v_movreld_b32 v0, v0 //v[0+m0] = v0
+ s_add_u32 m0, m0, 1 //next vgpr index
+ s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256 //every buffer_load_dword does 256 bytes
+ s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0
+ s_cbranch_scc1 L_RESTORE_VGPR_WAVE64_LOOP //VGPR restore (except v0) is complete?
+ //s_set_gpr_idx_off
+ //
+ //Below part will be the restore shared vgpr part (new for gfx10)
+ s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE) //shared_vgpr_size
+ s_and_b32 s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF //shared_vgpr_size is zero?
+ s_cbranch_scc0 L_RESTORE_V0 //no shared_vgpr used? jump to L_SAVE_LDS
+ s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 3 //Number of SHARED_VGPRs = shared_vgpr_size * 8 (non-zero value)
+ //m0 now has the value of normal vgpr count, just add the m0 with shared_vgpr count to get the total count.
+ //restore shared_vgpr will start from the index of m0
+ s_add_u32 s_restore_alloc_size, s_restore_alloc_size, m0
+ s_mov_b32 exec_lo, 0xFFFFFFFF
+ s_mov_b32 exec_hi, 0x00000000
+ L_RESTORE_SHARED_VGPR_WAVE64_LOOP:
+ buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
+ s_waitcnt vmcnt(0) //ensure data ready
+ v_movreld_b32 v0, v0 //v[0+m0] = v0
+ s_add_u32 m0, m0, 1 //next vgpr index
+ s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128 //every buffer_load_dword does 256 bytes
+ s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0
+ s_cbranch_scc1 L_RESTORE_SHARED_VGPR_WAVE64_LOOP //VGPR restore (except v0) is complete?
+
+ s_mov_b32 exec_hi, 0xFFFFFFFF //restore back exec_hi before restoring V0!!
+
+ /* VGPR restore on v0 */
+ L_RESTORE_V0:
+ if(USE_MTBUF_INSTEAD_OF_MUBUF)
+ tbuffer_load_format_x v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+ else
+ buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1
+ end
+
+
+ /* restore LDS */
+ //////////////////////////////
+ L_RESTORE_LDS:
+
+ //Only need to check the first wave
+ /* the first wave in the threadgroup */
+ s_and_b32 s_restore_tmp, s_restore_size, S_RESTORE_SPI_INIT_FIRST_WAVE_MASK
+ s_cbranch_scc0 L_RESTORE_SGPR
+
+ s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on //be consistent with SAVE although can be moved ahead
+ s_and_b32 m0, s_restore_size, 1
+ s_cmp_eq_u32 m0, 1
+ s_cbranch_scc1 L_ENABLE_RESTORE_LDS_EXEC_HI
+ s_mov_b32 exec_hi, 0x00000000
+ s_branch L_RESTORE_LDS_NORMAL
+ L_ENABLE_RESTORE_LDS_EXEC_HI:
+ s_mov_b32 exec_hi, 0xFFFFFFFF
+ L_RESTORE_LDS_NORMAL:
+ s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE) //lds_size
+ s_and_b32 s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF //lds_size is zero?
+ s_cbranch_scc0 L_RESTORE_SGPR //no lds used? jump to L_RESTORE_VGPR
+ s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 6 //LDS size in dwords = lds_size * 64dw
+ s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 2 //LDS size in bytes
+ s_mov_b32 s_restore_buf_rsrc2, s_restore_alloc_size //NUM_RECORDS in bytes
+ if (SWIZZLE_EN)
+ s_add_u32 s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+ else
+ s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+ end
+
+ s_and_b32 m0, s_wave_size, 1
+ s_cmp_eq_u32 m0, 1
+ s_mov_b32 m0, 0x0
+ s_cbranch_scc1 L_RESTORE_LDS_LOOP_W64
+
+ L_RESTORE_LDS_LOOP_W32:
+ if (SAVE_LDS)
+ buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1
+ s_waitcnt 0
+ end
+ s_add_u32 m0, m0, 128 //every buffer_load_dword does 256 bytes
+ s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128 //mem offset increased by 256 bytes
+ s_cmp_lt_u32 m0, s_restore_alloc_size //scc=(m0 < s_restore_alloc_size) ? 1 : 0
+ s_cbranch_scc1 L_RESTORE_LDS_LOOP_W32 //LDS restore is complete?
+ s_branch L_RESTORE_SGPR
+
+ L_RESTORE_LDS_LOOP_W64:
+ if (SAVE_LDS)
+ buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1
+ s_waitcnt 0
+ end
+ s_add_u32 m0, m0, 256 //every buffer_load_dword does 256 bytes
+ s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256 //mem offset increased by 256 bytes
+ s_cmp_lt_u32 m0, s_restore_alloc_size //scc=(m0 < s_restore_alloc_size) ? 1 : 0
+ s_cbranch_scc1 L_RESTORE_LDS_LOOP_W64 //LDS restore is complete?
+
+
+ /* restore SGPRs */
+ //////////////////////////////
+ //s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE) //spgr_size
+ //s_add_u32 s_restore_alloc_size, s_restore_alloc_size, 1
+ //s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 4 //Number of SGPRs = (sgpr_size + 1) * 16 (non-zero value)
+ //s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 3 //Number of SGPRs = (sgpr_size + 1) * 8 (non-zero value)
+ L_RESTORE_SGPR:
+ //need to look at it is wave32 or wave64
+ s_and_b32 m0, s_restore_size, 1
+ s_cmp_eq_u32 m0, 1
+ s_cbranch_scc1 L_RESTORE_SGPR_VMEM_WAVE64
+ if (SGPR_SAVE_USE_SQC)
+ s_lshl_b32 s_restore_buf_rsrc2, s_sgpr_save_num, 2 //NUM_RECORDS in bytes
+ else
+ s_lshl_b32 s_restore_buf_rsrc2, s_sgpr_save_num, 7 //NUM_RECORDS in bytes (32 threads)
+ end
+ s_branch L_RESTORE_SGPR_CONT
+ L_RESTORE_SGPR_VMEM_WAVE64:
+ if (SGPR_SAVE_USE_SQC)
+ s_lshl_b32 s_restore_buf_rsrc2, s_sgpr_save_num, 2 //NUM_RECORDS in bytes
+ else
+ s_lshl_b32 s_restore_buf_rsrc2, s_sgpr_save_num, 8 //NUM_RECORDS in bytes (64 threads)
+ end
+
+ L_RESTORE_SGPR_CONT:
+ if (SWIZZLE_EN)
+ s_add_u32 s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+ else
+ s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+ end
+
+ s_and_b32 m0, s_restore_size, 1
+ s_cmp_eq_u32 m0, 1
+ s_cbranch_scc1 L_RESTORE_SGPR_WAVE64
+
+ read_sgpr_from_mem_wave32(s_restore_tmp, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //save s0 to s_restore_tmp
+ s_mov_b32 m0, 0x1
+
+ L_RESTORE_SGPR_LOOP_WAVE32:
+ read_sgpr_from_mem_wave32(s0, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //PV: further performance improvement can be made
+ s_waitcnt lgkmcnt(0) //ensure data ready
+ s_movreld_b32 s0, s0 //s[0+m0] = s0
+ s_nop 0 // hazard SALU M0=> S_MOVREL
+ s_add_u32 m0, m0, 1 //next sgpr index
+ s_cmp_lt_u32 m0, s_sgpr_save_num //scc = (m0 < s_restore_alloc_size) ? 1 : 0
+ s_cbranch_scc1 L_RESTORE_SGPR_LOOP_WAVE32 //SGPR restore (except s0) is complete?
+ s_mov_b32 s0, s_restore_tmp /* SGPR restore on s0 */
+ s_branch L_RESTORE_HWREG
+
+ L_RESTORE_SGPR_WAVE64:
+ read_sgpr_from_mem_wave64(s_restore_tmp, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //save s0 to s_restore_tmp
+ s_mov_b32 m0, 0x1 //SGPR initial index value =1 //go on with with s1
+
+ L_RESTORE_SGPR_LOOP_WAVE64:
+ read_sgpr_from_mem_wave64(s0, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //PV: further performance improvement can be made
+ s_waitcnt lgkmcnt(0) //ensure data ready
+ s_movreld_b32 s0, s0 //s[0+m0] = s0
+ s_nop 0 // hazard SALU M0=> S_MOVREL
+ s_add_u32 m0, m0, 1 //next sgpr index
+ s_cmp_lt_u32 m0, s_sgpr_save_num //scc = (m0 < s_restore_alloc_size) ? 1 : 0
+ s_cbranch_scc1 L_RESTORE_SGPR_LOOP_WAVE64 //SGPR restore (except s0) is complete?
+ s_mov_b32 s0, s_restore_tmp /* SGPR restore on s0 */
+
+
+ /* restore HW registers */
+ //////////////////////////////
+ L_RESTORE_HWREG:
+ s_mov_b32 s_restore_buf_rsrc2, 0x4 //NUM_RECORDS in bytes
+ if (SWIZZLE_EN)
+ s_add_u32 s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+ else
+ s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+ end
+
+ s_and_b32 m0, s_restore_size, 1
+ s_cmp_eq_u32 m0, 1
+ s_cbranch_scc1 L_RESTORE_HWREG_WAVE64
+
+ read_sgpr_from_mem_wave32(s_restore_m0, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //M0
+ read_sgpr_from_mem_wave32(s_restore_pc_lo, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //PC
+ read_sgpr_from_mem_wave32(s_restore_pc_hi, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC)
+ read_sgpr_from_mem_wave32(s_restore_exec_lo, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //EXEC
+ read_sgpr_from_mem_wave32(s_restore_exec_hi, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC)
+ read_sgpr_from_mem_wave32(s_restore_status, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //STATUS
+ read_sgpr_from_mem_wave32(s_restore_trapsts, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //TRAPSTS
+ //read_sgpr_from_mem_wave32(xnack_mask_lo, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //XNACK_MASK_LO
+ //read_sgpr_from_mem_wave32(xnack_mask_hi, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //XNACK_MASK_HI
+ read_sgpr_from_mem_wave32(s_restore_xnack_mask, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //XNACK_MASK
+ read_sgpr_from_mem_wave32(s_restore_mode, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //MODE
+ if(SAVE_RESTORE_HWID_DDID)
+ read_sgpr_from_mem_wave32(s_restore_hwid1, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //HW_ID1
+ end
+ s_branch L_RESTORE_HWREG_FINISH
+
+ L_RESTORE_HWREG_WAVE64:
+ read_sgpr_from_mem_wave64(s_restore_m0, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //M0
+ read_sgpr_from_mem_wave64(s_restore_pc_lo, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //PC
+ read_sgpr_from_mem_wave64(s_restore_pc_hi, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC)
+ read_sgpr_from_mem_wave64(s_restore_exec_lo, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //EXEC
+ read_sgpr_from_mem_wave64(s_restore_exec_hi, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC)
+ read_sgpr_from_mem_wave64(s_restore_status, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //STATUS
+ read_sgpr_from_mem_wave64(s_restore_trapsts, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //TRAPSTS
+ //read_sgpr_from_mem_wave64(xnack_mask_lo, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //XNACK_MASK_LO
+ //read_sgpr_from_mem_wave64(xnack_mask_hi, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //XNACK_MASK_HI
+ read_sgpr_from_mem_wave64(s_restore_xnack_mask, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //XNACK_MASK
+ read_sgpr_from_mem_wave64(s_restore_mode, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //MODE
+ if(SAVE_RESTORE_HWID_DDID)
+ read_sgpr_from_mem_wave64(s_restore_hwid1, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC) //HW_ID1
+ end
+ L_RESTORE_HWREG_FINISH:
+ s_waitcnt lgkmcnt(0) //from now on, it is safe to restore STATUS and IB_STS
+
+
+
+ if(SAVE_RESTORE_HWID_DDID)
+ L_RESTORE_DDID:
+ s_mov_b32 m0, s_restore_hwid1 //virture ttrace support: The save-context handler records the SE/SA/WGP/SIMD/wave of the original wave
+ s_ttracedata //and then can output it as SHADER_DATA to ttrace on restore to provide a correlation across the save-restore
+
+ s_mov_b32 s_restore_buf_rsrc2, 0x4 //NUM_RECORDS in bytes
+ if (SWIZZLE_EN)
+ s_add_u32 s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0 //FIXME need to use swizzle to enable bounds checking?
+ else
+ s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
+ end
+
+ s_and_b32 m0, s_restore_size, 1
+ s_cmp_eq_u32 m0, 1
+ s_cbranch_scc1 L_RESTORE_DDID_WAVE64
+
+ read_sgpr_from_mem_wave32(s_restore_ddid, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC)
+ s_branch L_RESTORE_DDID_FINISH
+ L_RESTORE_DDID_WAVE64:
+ read_sgpr_from_mem_wave64(s_restore_ddid, s_restore_buf_rsrc0, s_restore_mem_offset, SGPR_SAVE_USE_SQC)
+
+ L_RESTORE_DDID_FINISH:
+ s_waitcnt lgkmcnt(0)
+ //s_mov_b32 m0, s_restore_ddid
+ //s_ttracedata
+ if (RESTORE_DDID_IN_SGPR18)
+ s_mov_b32 s18, s_restore_ddid
+ end
+
+ end
+
+ s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff //pc[47:32] //Do it here in order not to affect STATUS
+
+ //for normal save & restore, the saved PC points to the next inst to execute, no adjustment needs to be made, otherwise:
+ if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL))
+ s_add_u32 s_restore_pc_lo, s_restore_pc_lo, 8 //pc[31:0]+8 //two back-to-back s_trap are used (first for save and second for restore)
+ s_addc_u32 s_restore_pc_hi, s_restore_pc_hi, 0x0 //carry bit over
+ end
+ if ((EMU_RUN_HACK) && (EMU_RUN_HACK_RESTORE_NORMAL))
+ s_add_u32 s_restore_pc_lo, s_restore_pc_lo, 4 //pc[31:0]+4 // save is hack through s_trap but restore is normal
+ s_addc_u32 s_restore_pc_hi, s_restore_pc_hi, 0x0 //carry bit over
+ end
+
+ s_mov_b32 m0, s_restore_m0
+ s_mov_b32 exec_lo, s_restore_exec_lo
+ s_mov_b32 exec_hi, s_restore_exec_hi
+
+ s_and_b32 s_restore_m0, SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK, s_restore_trapsts
+ s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE), s_restore_m0
+ s_setreg_b32 hwreg(HW_REG_SHADER_XNACK_MASK), s_restore_xnack_mask //restore xnack_mask
+ s_and_b32 s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK, s_restore_trapsts
+ s_lshr_b32 s_restore_m0, s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT
+ s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE), s_restore_m0
+ //s_setreg_b32 hwreg(HW_REG_TRAPSTS), s_restore_trapsts //don't overwrite SAVECTX bit as it may be set through external SAVECTX during restore
+ s_setreg_b32 hwreg(HW_REG_MODE), s_restore_mode
+ //reuse s_restore_m0 as a temp register
+ s_and_b32 s_restore_m0, s_restore_pc_hi, S_SAVE_PC_HI_RCNT_MASK
+ s_lshr_b32 s_restore_m0, s_restore_m0, S_SAVE_PC_HI_RCNT_SHIFT
+ s_lshl_b32 s_restore_m0, s_restore_m0, SQ_WAVE_IB_STS_RCNT_SHIFT
+ s_mov_b32 s_restore_tmp, 0x0 //IB_STS is zero
+ s_or_b32 s_restore_tmp, s_restore_tmp, s_restore_m0
+ s_and_b32 s_restore_m0, s_restore_pc_hi, S_SAVE_PC_HI_FIRST_REPLAY_MASK
+ s_lshr_b32 s_restore_m0, s_restore_m0, S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+ s_lshl_b32 s_restore_m0, s_restore_m0, SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT
+ s_or_b32 s_restore_tmp, s_restore_tmp, s_restore_m0
+ s_and_b32 s_restore_m0, s_restore_status, SQ_WAVE_STATUS_INST_ATC_MASK
+ s_lshr_b32 s_restore_m0, s_restore_m0, SQ_WAVE_STATUS_INST_ATC_SHIFT
+ s_setreg_b32 hwreg(HW_REG_IB_STS), s_restore_tmp
+ s_setreg_b32 hwreg(HW_REG_STATUS), s_restore_status
+
+ s_barrier //barrier to ensure the readiness of LDS before access attemps from any other wave in the same TG //FIXME not performance-optimal at this time
+
+
+// s_rfe_b64 s_restore_pc_lo //Return to the main shader program and resume execution
+ s_rfe_b64 s_restore_pc_lo // s_restore_m0[0] is used to set STATUS.inst_atc
+
+
+/**************************************************************************/
+/* the END */
+/**************************************************************************/
+L_END_PGM:
+ s_endpgm
+
+end
+
+
+/**************************************************************************/
+/* the helper functions */
+/**************************************************************************/
+function write_sgpr_to_mem_wave32(s, s_rsrc, s_mem_offset, use_sqc, use_mtbuf)
+ if (use_sqc)
+ s_mov_b32 exec_lo, m0 //assuming exec_lo is not needed anymore from this point on
+ s_mov_b32 m0, s_mem_offset
+ s_buffer_store_dword s, s_rsrc, m0 glc:1
+ s_add_u32 s_mem_offset, s_mem_offset, 4
+ s_mov_b32 m0, exec_lo
+ elsif (use_mtbuf)
+ v_mov_b32 v0, s
+ tbuffer_store_format_x v0, v0, s_rsrc, s_mem_offset format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+ s_add_u32 s_mem_offset, s_mem_offset, 128
+ else
+ v_mov_b32 v0, s
+ buffer_store_dword v0, v0, s_rsrc, s_mem_offset slc:1 glc:1
+ s_add_u32 s_mem_offset, s_mem_offset, 128
+ end
+end
+
+function write_sgpr_to_mem_wave64(s, s_rsrc, s_mem_offset, use_sqc, use_mtbuf)
+ if (use_sqc)
+ s_mov_b32 exec_lo, m0 //assuming exec_lo is not needed anymore from this point on
+ s_mov_b32 m0, s_mem_offset
+ s_buffer_store_dword s, s_rsrc, m0 glc:1
+ s_add_u32 s_mem_offset, s_mem_offset, 4
+ s_mov_b32 m0, exec_lo
+ elsif (use_mtbuf)
+ v_mov_b32 v0, s
+ tbuffer_store_format_x v0, v0, s_rsrc, s_mem_offset format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+ s_add_u32 s_mem_offset, s_mem_offset, 256
+ else
+ v_mov_b32 v0, s
+ buffer_store_dword v0, v0, s_rsrc, s_mem_offset slc:1 glc:1
+ s_add_u32 s_mem_offset, s_mem_offset, 256
+ end
+end
+
+function read_sgpr_from_mem_wave32(s, s_rsrc, s_mem_offset, use_sqc)
+ s_buffer_load_dword s, s_rsrc, s_mem_offset glc:1
+ if (use_sqc)
+ s_add_u32 s_mem_offset, s_mem_offset, 4
+ else
+ s_add_u32 s_mem_offset, s_mem_offset, 128
+ end
+end
+
+function read_sgpr_from_mem_wave64(s, s_rsrc, s_mem_offset, use_sqc)
+ s_buffer_load_dword s, s_rsrc, s_mem_offset glc:1
+ if (use_sqc)
+ s_add_u32 s_mem_offset, s_mem_offset, 4
+ else
+ s_add_u32 s_mem_offset, s_mem_offset, 256
+ end
+end
+
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm
index abe1a5da29fb..a47f5b933120 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm
@@ -282,19 +282,6 @@ if G8SR_DEBUG_TIMESTAMP
s_waitcnt lgkmcnt(0) //FIXME, will cause xnack??
end
- //check whether there is mem_viol
- s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
- s_and_b32 s_save_trapsts, s_save_trapsts, SQ_WAVE_TRAPSTS_MEM_VIOL_MASK
- s_cbranch_scc0 L_NO_PC_REWIND
-
- //if so, need rewind PC assuming GDS operation gets NACKed
- s_mov_b32 s_save_tmp, 0 //clear mem_viol bit
- s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT, 1), s_save_tmp //clear mem_viol bit
- s_and_b32 s_save_pc_hi, s_save_pc_hi, 0x0000ffff //pc[47:32]
- s_sub_u32 s_save_pc_lo, s_save_pc_lo, 8 //pc[31:0]-8
- s_subb_u32 s_save_pc_hi, s_save_pc_hi, 0x0 // -scc
-
-L_NO_PC_REWIND:
s_mov_b32 s_save_tmp, 0 //clear saveCtx bit
s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_SAVECTX_SHIFT, 1), s_save_tmp //clear saveCtx bit
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm
index 0bb9c577b3a2..6bae2e022c6e 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm
@@ -150,10 +150,10 @@ var S_SAVE_SPI_INIT_MTYPE_SHIFT = 28
var S_SAVE_SPI_INIT_FIRST_WAVE_MASK = 0x04000000 //bit[26]: FirstWaveInTG
var S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT = 26
-var S_SAVE_PC_HI_RCNT_SHIFT = 28 //FIXME check with Brian to ensure all fields other than PC[47:0] can be used
-var S_SAVE_PC_HI_RCNT_MASK = 0xF0000000 //FIXME
-var S_SAVE_PC_HI_FIRST_REPLAY_SHIFT = 27 //FIXME
-var S_SAVE_PC_HI_FIRST_REPLAY_MASK = 0x08000000 //FIXME
+var S_SAVE_PC_HI_RCNT_SHIFT = 27 //FIXME check with Brian to ensure all fields other than PC[47:0] can be used
+var S_SAVE_PC_HI_RCNT_MASK = 0xF8000000 //FIXME
+var S_SAVE_PC_HI_FIRST_REPLAY_SHIFT = 26 //FIXME
+var S_SAVE_PC_HI_FIRST_REPLAY_MASK = 0x04000000 //FIXME
var s_save_spi_init_lo = exec_lo
var s_save_spi_init_hi = exec_hi
@@ -162,8 +162,8 @@ var s_save_pc_lo = ttmp0 //{TTMP1, TTMP0} = {3'h0,pc_rewind[3:0], HT[0],tra
var s_save_pc_hi = ttmp1
var s_save_exec_lo = ttmp2
var s_save_exec_hi = ttmp3
-var s_save_tmp = ttmp4
-var s_save_trapsts = ttmp5 //not really used until the end of the SAVE routine
+var s_save_tmp = ttmp14
+var s_save_trapsts = ttmp15 //not really used until the end of the SAVE routine
var s_save_xnack_mask_lo = ttmp6
var s_save_xnack_mask_hi = ttmp7
var s_save_buf_rsrc0 = ttmp8
@@ -171,9 +171,9 @@ var s_save_buf_rsrc1 = ttmp9
var s_save_buf_rsrc2 = ttmp10
var s_save_buf_rsrc3 = ttmp11
var s_save_status = ttmp12
-var s_save_mem_offset = ttmp14
+var s_save_mem_offset = ttmp4
var s_save_alloc_size = s_save_trapsts //conflict
-var s_save_m0 = ttmp15
+var s_save_m0 = ttmp5
var s_save_ttmps_lo = s_save_tmp //no conflict
var s_save_ttmps_hi = s_save_trapsts //no conflict
@@ -207,10 +207,10 @@ var s_restore_mode = ttmp7
var s_restore_pc_lo = ttmp0
var s_restore_pc_hi = ttmp1
-var s_restore_exec_lo = ttmp14
-var s_restore_exec_hi = ttmp15
-var s_restore_status = ttmp4
-var s_restore_trapsts = ttmp5
+var s_restore_exec_lo = ttmp4
+var s_restore_exec_hi = ttmp5
+var s_restore_status = ttmp14
+var s_restore_trapsts = ttmp15
var s_restore_xnack_mask_lo = xnack_mask_lo
var s_restore_xnack_mask_hi = xnack_mask_hi
var s_restore_buf_rsrc0 = ttmp8
@@ -266,10 +266,16 @@ if (!EMU_RUN_HACK)
L_HALT_WAVE:
// If STATUS.HALT is set then this fault must come from SQC instruction fetch.
- // We cannot prevent further faults so just terminate the wavefront.
+ // We cannot prevent further faults. Spin wait until context saved.
s_and_b32 ttmp2, s_save_status, SQ_WAVE_STATUS_HALT_MASK
s_cbranch_scc0 L_NOT_ALREADY_HALTED
- s_endpgm
+
+L_WAIT_CTX_SAVE:
+ s_sleep 0x10
+ s_getreg_b32 ttmp2, hwreg(HW_REG_TRAPSTS)
+ s_and_b32 ttmp2, ttmp2, SQ_WAVE_TRAPSTS_SAVECTX_MASK
+ s_cbranch_scc0 L_WAIT_CTX_SAVE
+
L_NOT_ALREADY_HALTED:
s_or_b32 s_save_status, s_save_status, SQ_WAVE_STATUS_HALT_MASK
@@ -293,12 +299,12 @@ L_FETCH_2ND_TRAP:
// Read second-level TBA/TMA from first-level TMA and jump if available.
// ttmp[2:5] and ttmp12 can be used (others hold SPI-initialized debug data)
// ttmp12 holds SQ_WAVE_STATUS
- s_getreg_b32 ttmp4, hwreg(HW_REG_SQ_SHADER_TMA_LO)
- s_getreg_b32 ttmp5, hwreg(HW_REG_SQ_SHADER_TMA_HI)
- s_lshl_b64 [ttmp4, ttmp5], [ttmp4, ttmp5], 0x8
- s_load_dwordx2 [ttmp2, ttmp3], [ttmp4, ttmp5], 0x0 glc:1 // second-level TBA
+ s_getreg_b32 ttmp14, hwreg(HW_REG_SQ_SHADER_TMA_LO)
+ s_getreg_b32 ttmp15, hwreg(HW_REG_SQ_SHADER_TMA_HI)
+ s_lshl_b64 [ttmp14, ttmp15], [ttmp14, ttmp15], 0x8
+ s_load_dwordx2 [ttmp2, ttmp3], [ttmp14, ttmp15], 0x0 glc:1 // second-level TBA
s_waitcnt lgkmcnt(0)
- s_load_dwordx2 [ttmp4, ttmp5], [ttmp4, ttmp5], 0x8 glc:1 // second-level TMA
+ s_load_dwordx2 [ttmp14, ttmp15], [ttmp14, ttmp15], 0x8 glc:1 // second-level TMA
s_waitcnt lgkmcnt(0)
s_and_b64 [ttmp2, ttmp3], [ttmp2, ttmp3], [ttmp2, ttmp3]
s_cbranch_scc0 L_NO_NEXT_TRAP // second-level trap handler not been set
@@ -405,7 +411,7 @@ end
else
end
- // Save trap temporaries 6-11, 13-15 initialized by SPI debug dispatch logic
+ // Save trap temporaries 4-11, 13 initialized by SPI debug dispatch logic
// ttmp SR memory offset : size(VGPR)+size(SGPR)+0x40
get_vgpr_size_bytes(s_save_ttmps_lo)
get_sgpr_size_bytes(s_save_ttmps_hi)
@@ -413,13 +419,11 @@ end
s_add_u32 s_save_ttmps_lo, s_save_ttmps_lo, s_save_spi_init_lo
s_addc_u32 s_save_ttmps_hi, s_save_spi_init_hi, 0x0
s_and_b32 s_save_ttmps_hi, s_save_ttmps_hi, 0xFFFF
- s_store_dwordx2 [ttmp6, ttmp7], [s_save_ttmps_lo, s_save_ttmps_hi], 0x40 glc:1
- ack_sqc_store_workaround()
- s_store_dwordx4 [ttmp8, ttmp9, ttmp10, ttmp11], [s_save_ttmps_lo, s_save_ttmps_hi], 0x48 glc:1
+ s_store_dwordx4 [ttmp4, ttmp5, ttmp6, ttmp7], [s_save_ttmps_lo, s_save_ttmps_hi], 0x50 glc:1
ack_sqc_store_workaround()
- s_store_dword ttmp13, [s_save_ttmps_lo, s_save_ttmps_hi], 0x58 glc:1
+ s_store_dwordx4 [ttmp8, ttmp9, ttmp10, ttmp11], [s_save_ttmps_lo, s_save_ttmps_hi], 0x60 glc:1
ack_sqc_store_workaround()
- s_store_dwordx2 [ttmp14, ttmp15], [s_save_ttmps_lo, s_save_ttmps_hi], 0x5C glc:1
+ s_store_dword ttmp13, [s_save_ttmps_lo, s_save_ttmps_hi], 0x74 glc:1
ack_sqc_store_workaround()
/* setup Resource Contants */
@@ -1093,7 +1097,7 @@ end
//s_setreg_b32 hwreg(HW_REG_TRAPSTS), s_restore_trapsts //don't overwrite SAVECTX bit as it may be set through external SAVECTX during restore
s_setreg_b32 hwreg(HW_REG_MODE), s_restore_mode
- // Restore trap temporaries 6-11, 13-15 initialized by SPI debug dispatch logic
+ // Restore trap temporaries 4-11, 13 initialized by SPI debug dispatch logic
// ttmp SR memory offset : size(VGPR)+size(SGPR)+0x40
get_vgpr_size_bytes(s_restore_ttmps_lo)
get_sgpr_size_bytes(s_restore_ttmps_hi)
@@ -1101,10 +1105,9 @@ end
s_add_u32 s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_buf_rsrc0
s_addc_u32 s_restore_ttmps_hi, s_restore_buf_rsrc1, 0x0
s_and_b32 s_restore_ttmps_hi, s_restore_ttmps_hi, 0xFFFF
- s_load_dwordx2 [ttmp6, ttmp7], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x40 glc:1
- s_load_dwordx4 [ttmp8, ttmp9, ttmp10, ttmp11], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x48 glc:1
- s_load_dword ttmp13, [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x58 glc:1
- s_load_dwordx2 [ttmp14, ttmp15], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x5C glc:1
+ s_load_dwordx4 [ttmp4, ttmp5, ttmp6, ttmp7], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x50 glc:1
+ s_load_dwordx4 [ttmp8, ttmp9, ttmp10, ttmp11], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x60 glc:1
+ s_load_dword ttmp13, [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x74 glc:1
s_waitcnt lgkmcnt(0)
//reuse s_restore_m0 as a temp register
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 083bd8114db1..1d3cd5c50d5f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -213,6 +213,8 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties,
q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA)
q_properties->type = KFD_QUEUE_TYPE_SDMA;
+ else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI)
+ q_properties->type = KFD_QUEUE_TYPE_SDMA_XGMI;
else
return -ENOTSUPP;
@@ -522,7 +524,7 @@ static int kfd_ioctl_set_trap_handler(struct file *filep,
struct kfd_process_device *pdd;
dev = kfd_device_by_id(args->gpu_id);
- if (dev == NULL)
+ if (!dev)
return -EINVAL;
mutex_lock(&p->mutex);
@@ -837,7 +839,7 @@ static int kfd_ioctl_get_clock_counters(struct file *filep,
/* No access to rdtsc. Using raw monotonic time */
args->cpu_clock_counter = ktime_get_raw_ns();
- args->system_clock_counter = ktime_get_boot_ns();
+ args->system_clock_counter = ktime_get_boottime_ns();
/* Since the counter is in nano-seconds we use 1GHz frequency */
args->system_clock_freq = 1000000000;
@@ -1272,6 +1274,12 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
if (args->size != kfd_doorbell_process_slice(dev))
return -EINVAL;
offset = kfd_get_process_doorbells(dev, p);
+ } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
+ if (args->size != PAGE_SIZE)
+ return -EINVAL;
+ offset = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd);
+ if (!offset)
+ return -ENOMEM;
}
mutex_lock(&p->mutex);
@@ -1301,6 +1309,14 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
args->mmap_offset = offset;
+ /* MMIO is mapped through kfd device
+ * Generate a kfd mmap offset
+ */
+ if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
+ args->mmap_offset = KFD_MMAP_TYPE_MMIO | KFD_MMAP_GPU_ID(args->gpu_id);
+ args->mmap_offset <<= PAGE_SHIFT;
+ }
+
return 0;
err_free:
@@ -1845,6 +1861,39 @@ err_i1:
return retcode;
}
+static int kfd_mmio_mmap(struct kfd_dev *dev, struct kfd_process *process,
+ struct vm_area_struct *vma)
+{
+ phys_addr_t address;
+ int ret;
+
+ if (vma->vm_end - vma->vm_start != PAGE_SIZE)
+ return -EINVAL;
+
+ address = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd);
+
+ vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
+ VM_DONTDUMP | VM_PFNMAP;
+
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+
+ pr_debug("Process %d mapping mmio page\n"
+ " target user address == 0x%08llX\n"
+ " physical address == 0x%08llX\n"
+ " vm_flags == 0x%04lX\n"
+ " size == 0x%04lX\n",
+ process->pasid, (unsigned long long) vma->vm_start,
+ address, vma->vm_flags, PAGE_SIZE);
+
+ ret = io_remap_pfn_range(vma,
+ vma->vm_start,
+ address >> PAGE_SHIFT,
+ PAGE_SIZE,
+ vma->vm_page_prot);
+ return ret;
+}
+
+
static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
{
struct kfd_process *process;
@@ -1875,6 +1924,10 @@ static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
if (!dev)
return -ENODEV;
return kfd_reserved_mem_mmap(dev, process, vma);
+ case KFD_MMAP_TYPE_MMIO:
+ if (!dev)
+ return -ENODEV;
+ return kfd_mmio_mmap(dev, process, vma);
}
return -EFAULT;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
index 2e7c44955f43..4e3fc284f6ac 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
@@ -134,9 +134,12 @@ static struct kfd_gpu_cache_info carrizo_cache_info[] = {
#define polaris10_cache_info carrizo_cache_info
#define polaris11_cache_info carrizo_cache_info
#define polaris12_cache_info carrizo_cache_info
+#define vegam_cache_info carrizo_cache_info
/* TODO - check & update Vega10 cache details */
#define vega10_cache_info carrizo_cache_info
#define raven_cache_info carrizo_cache_info
+/* TODO - check & update Navi10 cache details */
+#define navi10_cache_info carrizo_cache_info
static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
struct crat_subtype_computeunit *cu)
@@ -372,7 +375,7 @@ static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
if (props->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS)
props->weight = 20;
else if (props->iolink_type == CRAT_IOLINK_TYPE_XGMI)
- props->weight = 15;
+ props->weight = 15 * iolink->num_hops_xgmi;
else
props->weight = node_distance(id_from, id_to);
@@ -652,6 +655,10 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev,
pcache_info = polaris12_cache_info;
num_of_cache_types = ARRAY_SIZE(polaris12_cache_info);
break;
+ case CHIP_VEGAM:
+ pcache_info = vegam_cache_info;
+ num_of_cache_types = ARRAY_SIZE(vegam_cache_info);
+ break;
case CHIP_VEGA10:
case CHIP_VEGA12:
case CHIP_VEGA20:
@@ -662,6 +669,10 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev,
pcache_info = raven_cache_info;
num_of_cache_types = ARRAY_SIZE(raven_cache_info);
break;
+ case CHIP_NAVI10:
+ pcache_info = navi10_cache_info;
+ num_of_cache_types = ARRAY_SIZE(navi10_cache_info);
+ break;
default:
return -EINVAL;
}
@@ -1092,6 +1103,7 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,
struct kfd_dev *kdev,
+ struct kfd_dev *peer_kdev,
struct crat_subtype_iolink *sub_type_hdr,
uint32_t proximity_domain_from,
uint32_t proximity_domain_to)
@@ -1110,6 +1122,8 @@ static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,
sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
sub_type_hdr->proximity_domain_from = proximity_domain_from;
sub_type_hdr->proximity_domain_to = proximity_domain_to;
+ sub_type_hdr->num_hops_xgmi =
+ amdgpu_amdkfd_get_xgmi_hops_count(kdev->kgd, peer_kdev->kgd);
return 0;
}
@@ -1287,7 +1301,7 @@ static int kfd_create_vcrat_image_gpu(void *pcrat_image,
(char *)sub_type_hdr +
sizeof(struct crat_subtype_iolink));
ret = kfd_fill_gpu_xgmi_link_to_gpu(
- &avail_size, kdev,
+ &avail_size, kdev, peer_dev->gpu,
(struct crat_subtype_iolink *)sub_type_hdr,
proximity_domain, nid);
if (ret < 0)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
index 7c3f192fe25f..d54ceebd346b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
@@ -274,7 +274,8 @@ struct crat_subtype_iolink {
uint32_t minimum_bandwidth_mbs;
uint32_t maximum_bandwidth_mbs;
uint32_t recommended_transfer_size;
- uint8_t reserved2[CRAT_IOLINK_RESERVED_LENGTH];
+ uint8_t reserved2[CRAT_IOLINK_RESERVED_LENGTH - 1];
+ uint8_t num_hops_xgmi;
};
/*
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c b/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c
index ab37d36d9cd6..15c523027285 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c
@@ -85,36 +85,16 @@ static const struct file_operations kfd_debugfs_hang_hws_fops = {
void kfd_debugfs_init(void)
{
- struct dentry *ent;
-
debugfs_root = debugfs_create_dir("kfd", NULL);
- if (!debugfs_root || debugfs_root == ERR_PTR(-ENODEV)) {
- pr_warn("Failed to create kfd debugfs dir\n");
- return;
- }
-
- ent = debugfs_create_file("mqds", S_IFREG | 0444, debugfs_root,
- kfd_debugfs_mqds_by_process,
- &kfd_debugfs_fops);
- if (!ent)
- pr_warn("Failed to create mqds in kfd debugfs\n");
-
- ent = debugfs_create_file("hqds", S_IFREG | 0444, debugfs_root,
- kfd_debugfs_hqds_by_device,
- &kfd_debugfs_fops);
- if (!ent)
- pr_warn("Failed to create hqds in kfd debugfs\n");
-
- ent = debugfs_create_file("rls", S_IFREG | 0444, debugfs_root,
- kfd_debugfs_rls_by_device,
- &kfd_debugfs_fops);
-
- ent = debugfs_create_file("hang_hws", S_IFREG | 0644, debugfs_root,
- NULL,
- &kfd_debugfs_hang_hws_fops);
- if (!ent)
- pr_warn("Failed to create rls in kfd debugfs\n");
+ debugfs_create_file("mqds", S_IFREG | 0444, debugfs_root,
+ kfd_debugfs_mqds_by_process, &kfd_debugfs_fops);
+ debugfs_create_file("hqds", S_IFREG | 0444, debugfs_root,
+ kfd_debugfs_hqds_by_device, &kfd_debugfs_fops);
+ debugfs_create_file("rls", S_IFREG | 0444, debugfs_root,
+ kfd_debugfs_rls_by_device, &kfd_debugfs_fops);
+ debugfs_create_file("hang_hws", S_IFREG | 0644, debugfs_root,
+ NULL, &kfd_debugfs_hang_hws_fops);
}
void kfd_debugfs_fini(void)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 765b58a17dc7..3322a443dfb2 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -54,6 +54,7 @@ static const struct kfd_device_info kaveri_device_info = {
.needs_iommu_device = true,
.needs_pci_atomics = false,
.num_sdma_engines = 2,
+ .num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 2,
};
@@ -71,6 +72,7 @@ static const struct kfd_device_info carrizo_device_info = {
.needs_iommu_device = true,
.needs_pci_atomics = false,
.num_sdma_engines = 2,
+ .num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 2,
};
@@ -87,6 +89,7 @@ static const struct kfd_device_info raven_device_info = {
.needs_iommu_device = true,
.needs_pci_atomics = true,
.num_sdma_engines = 1,
+ .num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 2,
};
#endif
@@ -105,6 +108,7 @@ static const struct kfd_device_info hawaii_device_info = {
.needs_iommu_device = false,
.needs_pci_atomics = false,
.num_sdma_engines = 2,
+ .num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 2,
};
@@ -121,6 +125,7 @@ static const struct kfd_device_info tonga_device_info = {
.needs_iommu_device = false,
.needs_pci_atomics = true,
.num_sdma_engines = 2,
+ .num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 2,
};
@@ -137,6 +142,7 @@ static const struct kfd_device_info fiji_device_info = {
.needs_iommu_device = false,
.needs_pci_atomics = true,
.num_sdma_engines = 2,
+ .num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 2,
};
@@ -153,6 +159,7 @@ static const struct kfd_device_info fiji_vf_device_info = {
.needs_iommu_device = false,
.needs_pci_atomics = false,
.num_sdma_engines = 2,
+ .num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 2,
};
@@ -170,6 +177,7 @@ static const struct kfd_device_info polaris10_device_info = {
.needs_iommu_device = false,
.needs_pci_atomics = true,
.num_sdma_engines = 2,
+ .num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 2,
};
@@ -186,6 +194,7 @@ static const struct kfd_device_info polaris10_vf_device_info = {
.needs_iommu_device = false,
.needs_pci_atomics = false,
.num_sdma_engines = 2,
+ .num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 2,
};
@@ -202,6 +211,7 @@ static const struct kfd_device_info polaris11_device_info = {
.needs_iommu_device = false,
.needs_pci_atomics = true,
.num_sdma_engines = 2,
+ .num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 2,
};
@@ -218,6 +228,24 @@ static const struct kfd_device_info polaris12_device_info = {
.needs_iommu_device = false,
.needs_pci_atomics = true,
.num_sdma_engines = 2,
+ .num_xgmi_sdma_engines = 0,
+ .num_sdma_queues_per_engine = 2,
+};
+
+static const struct kfd_device_info vegam_device_info = {
+ .asic_family = CHIP_VEGAM,
+ .max_pasid_bits = 16,
+ .max_no_of_hqd = 24,
+ .doorbell_size = 4,
+ .ih_ring_entry_size = 4 * sizeof(uint32_t),
+ .event_interrupt_class = &event_interrupt_class_cik,
+ .num_of_watch_points = 4,
+ .mqd_size_aligned = MQD_SIZE_ALIGNED,
+ .supports_cwsr = true,
+ .needs_iommu_device = false,
+ .needs_pci_atomics = true,
+ .num_sdma_engines = 2,
+ .num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 2,
};
@@ -234,6 +262,7 @@ static const struct kfd_device_info vega10_device_info = {
.needs_iommu_device = false,
.needs_pci_atomics = false,
.num_sdma_engines = 2,
+ .num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 2,
};
@@ -250,6 +279,7 @@ static const struct kfd_device_info vega10_vf_device_info = {
.needs_iommu_device = false,
.needs_pci_atomics = false,
.num_sdma_engines = 2,
+ .num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 2,
};
@@ -266,6 +296,7 @@ static const struct kfd_device_info vega12_device_info = {
.needs_iommu_device = false,
.needs_pci_atomics = false,
.num_sdma_engines = 2,
+ .num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 2,
};
@@ -282,6 +313,24 @@ static const struct kfd_device_info vega20_device_info = {
.needs_iommu_device = false,
.needs_pci_atomics = false,
.num_sdma_engines = 2,
+ .num_xgmi_sdma_engines = 0,
+ .num_sdma_queues_per_engine = 8,
+};
+
+static const struct kfd_device_info navi10_device_info = {
+ .asic_family = CHIP_NAVI10,
+ .max_pasid_bits = 16,
+ .max_no_of_hqd = 24,
+ .doorbell_size = 8,
+ .ih_ring_entry_size = 8 * sizeof(uint32_t),
+ .event_interrupt_class = &event_interrupt_class_v9,
+ .num_of_watch_points = 4,
+ .mqd_size_aligned = MQD_SIZE_ALIGNED,
+ .needs_iommu_device = false,
+ .supports_cwsr = true,
+ .needs_pci_atomics = false,
+ .num_sdma_engines = 2,
+ .num_xgmi_sdma_engines = 0,
.num_sdma_queues_per_engine = 8,
};
@@ -373,6 +422,9 @@ static const struct kfd_deviceid supported_devices[] = {
{ 0x6995, &polaris12_device_info }, /* Polaris12 */
{ 0x6997, &polaris12_device_info }, /* Polaris12 */
{ 0x699F, &polaris12_device_info }, /* Polaris12 */
+ { 0x694C, &vegam_device_info }, /* VegaM */
+ { 0x694E, &vegam_device_info }, /* VegaM */
+ { 0x694F, &vegam_device_info }, /* VegaM */
{ 0x6860, &vega10_device_info }, /* Vega10 */
{ 0x6861, &vega10_device_info }, /* Vega10 */
{ 0x6862, &vega10_device_info }, /* Vega10 */
@@ -399,7 +451,13 @@ static const struct kfd_deviceid supported_devices[] = {
{ 0x66a3, &vega20_device_info }, /* Vega20 */
{ 0x66a4, &vega20_device_info }, /* Vega20 */
{ 0x66a7, &vega20_device_info }, /* Vega20 */
- { 0x66af, &vega20_device_info } /* Vega20 */
+ { 0x66af, &vega20_device_info }, /* Vega20 */
+ /* Navi10 */
+ { 0x7310, &navi10_device_info }, /* Navi10 */
+ { 0x7312, &navi10_device_info }, /* Navi10 */
+ { 0x7318, &navi10_device_info }, /* Navi10 */
+ { 0x731a, &navi10_device_info }, /* Navi10 */
+ { 0x731f, &navi10_device_info }, /* Navi10 */
};
static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
@@ -429,7 +487,6 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
struct pci_dev *pdev, const struct kfd2kgd_calls *f2g)
{
struct kfd_dev *kfd;
- int ret;
const struct kfd_device_info *device_info =
lookup_device_info(pdev->device);
@@ -446,17 +503,15 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
* 32 and 64-bit requests are possible and must be
* supported.
*/
- ret = pci_enable_atomic_ops_to_root(pdev,
- PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
- PCI_EXP_DEVCAP2_ATOMIC_COMP64);
- if (device_info->needs_pci_atomics && ret < 0) {
+ kfd->pci_atomic_requested = amdgpu_amdkfd_have_atomics_support(kgd);
+ if (device_info->needs_pci_atomics &&
+ !kfd->pci_atomic_requested) {
dev_info(kfd_device,
"skipped device %x:%x, PCI rejects atomics\n",
pdev->vendor, pdev->device);
kfree(kfd);
return NULL;
- } else if (!ret)
- kfd->pci_atomic_requested = true;
+ }
kfd->kgd = kgd;
kfd->device_info = device_info;
@@ -481,10 +536,14 @@ static void kfd_cwsr_init(struct kfd_dev *kfd)
BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) > PAGE_SIZE);
kfd->cwsr_isa = cwsr_trap_gfx8_hex;
kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx8_hex);
- } else {
+ } else if (kfd->device_info->asic_family < CHIP_NAVI10) {
BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) > PAGE_SIZE);
kfd->cwsr_isa = cwsr_trap_gfx9_hex;
kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_hex);
+ } else {
+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex) > PAGE_SIZE);
+ kfd->cwsr_isa = cwsr_trap_gfx10_hex;
+ kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx10_hex);
}
kfd->cwsr_enabled = true;
@@ -518,6 +577,13 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
} else
kfd->max_proc_per_quantum = hws_max_conc_proc;
+ /* Allocate global GWS that is shared by all KFD processes */
+ if (hws_gws_support && amdgpu_amdkfd_alloc_gws(kfd->kgd,
+ amdgpu_amdkfd_get_num_gws(kfd->kgd), &kfd->gws)) {
+ dev_err(kfd_device, "Could not allocate %d gws\n",
+ amdgpu_amdkfd_get_num_gws(kfd->kgd));
+ goto out;
+ }
/* calculate max size of mqds needed for queues */
size = max_num_of_queues_per_device *
kfd->device_info->mqd_size_aligned;
@@ -541,7 +607,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
&kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr,
false)) {
dev_err(kfd_device, "Could not allocate %d bytes\n", size);
- goto out;
+ goto alloc_gtt_mem_failure;
}
dev_info(kfd_device, "Allocated %d bytes on gart\n", size);
@@ -561,11 +627,6 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
if (kfd->kfd2kgd->get_hive_id)
kfd->hive_id = kfd->kfd2kgd->get_hive_id(kfd->kgd);
- if (kfd_topology_add_device(kfd)) {
- dev_err(kfd_device, "Error adding device to topology\n");
- goto kfd_topology_add_device_error;
- }
-
if (kfd_interrupt_init(kfd)) {
dev_err(kfd_device, "Error initializing interrupts\n");
goto kfd_interrupt_error;
@@ -589,6 +650,11 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
kfd->dbgmgr = NULL;
+ if (kfd_topology_add_device(kfd)) {
+ dev_err(kfd_device, "Error adding device to topology\n");
+ goto kfd_topology_add_device_error;
+ }
+
kfd->init_complete = true;
dev_info(kfd_device, "added device %x:%x\n", kfd->pdev->vendor,
kfd->pdev->device);
@@ -598,19 +664,21 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
goto out;
+kfd_topology_add_device_error:
kfd_resume_error:
device_iommu_error:
device_queue_manager_uninit(kfd->dqm);
device_queue_manager_error:
kfd_interrupt_exit(kfd);
kfd_interrupt_error:
- kfd_topology_remove_device(kfd);
-kfd_topology_add_device_error:
kfd_doorbell_fini(kfd);
kfd_doorbell_error:
kfd_gtt_sa_fini(kfd);
kfd_gtt_sa_init_error:
amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem);
+alloc_gtt_mem_failure:
+ if (hws_gws_support)
+ amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws);
dev_err(kfd_device,
"device %x:%x NOT added due to errors\n",
kfd->pdev->vendor, kfd->pdev->device);
@@ -628,6 +696,8 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd)
kfd_doorbell_fini(kfd);
kfd_gtt_sa_fini(kfd);
amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem);
+ if (hws_gws_support)
+ amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws);
}
kfree(kfd);
@@ -665,7 +735,6 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd)
if (ret)
return ret;
count = atomic_dec_return(&kfd_locked);
- WARN_ONCE(count != 0, "KFD reset ref. error");
atomic_set(&kfd->sram_ecc_flag, 0);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index ae381450601c..e6a4288bfaa6 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -42,10 +42,6 @@
static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
unsigned int pasid, unsigned int vmid);
-static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
- struct queue *q,
- struct qcm_process_device *qpd);
-
static int execute_queues_cpsch(struct device_queue_manager *dqm,
enum kfd_unmap_queues_filter filter,
uint32_t filter_param);
@@ -55,19 +51,20 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,
static int map_queues_cpsch(struct device_queue_manager *dqm);
-static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
- struct queue *q,
- struct qcm_process_device *qpd);
-
static void deallocate_sdma_queue(struct device_queue_manager *dqm,
- unsigned int sdma_queue_id);
+ struct queue *q);
+static inline void deallocate_hqd(struct device_queue_manager *dqm,
+ struct queue *q);
+static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q);
+static int allocate_sdma_queue(struct device_queue_manager *dqm,
+ struct queue *q);
static void kfd_process_hw_exception(struct work_struct *work);
static inline
enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
{
- if (type == KFD_QUEUE_TYPE_SDMA)
+ if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI)
return KFD_MQD_TYPE_SDMA;
return KFD_MQD_TYPE_CP;
}
@@ -107,12 +104,23 @@ static unsigned int get_num_sdma_engines(struct device_queue_manager *dqm)
return dqm->dev->device_info->num_sdma_engines;
}
+static unsigned int get_num_xgmi_sdma_engines(struct device_queue_manager *dqm)
+{
+ return dqm->dev->device_info->num_xgmi_sdma_engines;
+}
+
unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
{
return dqm->dev->device_info->num_sdma_engines
* dqm->dev->device_info->num_sdma_queues_per_engine;
}
+unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm)
+{
+ return dqm->dev->device_info->num_xgmi_sdma_engines
+ * dqm->dev->device_info->num_sdma_queues_per_engine;
+}
+
void program_sh_mem_settings(struct device_queue_manager *dqm,
struct qcm_process_device *qpd)
{
@@ -133,7 +141,8 @@ static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q)
* preserve the user mode ABI.
*/
q->doorbell_id = q->properties.queue_id;
- } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
+ } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
+ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
/* For SDMA queues on SOC15 with 8-byte doorbell, use static
* doorbell assignments based on the engine and queue id.
* The doobell index distance between RLC (2*i) and (2*i+1)
@@ -174,7 +183,8 @@ static void deallocate_doorbell(struct qcm_process_device *qpd,
struct kfd_dev *dev = qpd->dqm->dev;
if (!KFD_IS_SOC15(dev->device_info->asic_family) ||
- q->properties.type == KFD_QUEUE_TYPE_SDMA)
+ q->properties.type == KFD_QUEUE_TYPE_SDMA ||
+ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
return;
old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap);
@@ -210,6 +220,9 @@ static int allocate_vmid(struct device_queue_manager *dqm,
/* invalidate the VM context after pasid and vmid mapping is set up */
kfd_flush_tlb(qpd_to_pdd(qpd));
+ dqm->dev->kfd2kgd->set_scratch_backing_va(
+ dqm->dev->kgd, qpd->sh_hidden_private_base, qpd->vmid);
+
return 0;
}
@@ -256,6 +269,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
struct queue *q,
struct qcm_process_device *qpd)
{
+ struct mqd_manager *mqd_mgr;
int retval;
print_queue(q);
@@ -276,28 +290,56 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
}
q->properties.vmid = qpd->vmid;
/*
- * Eviction state logic: we only mark active queues as evicted
- * to avoid the overhead of restoring inactive queues later
+ * Eviction state logic: mark all queues as evicted, even ones
+ * not currently active. Restoring inactive queues later only
+ * updates the is_evicted flag but is a no-op otherwise.
*/
- if (qpd->evicted)
- q->properties.is_evicted = (q->properties.queue_size > 0 &&
- q->properties.queue_percent > 0 &&
- q->properties.queue_address != 0);
+ q->properties.is_evicted = !!qpd->evicted;
q->properties.tba_addr = qpd->tba_addr;
q->properties.tma_addr = qpd->tma_addr;
- if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
- retval = create_compute_queue_nocpsch(dqm, q, qpd);
- else if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
- retval = create_sdma_queue_nocpsch(dqm, q, qpd);
- else
- retval = -EINVAL;
+ mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
+ q->properties.type)];
+ if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
+ retval = allocate_hqd(dqm, q);
+ if (retval)
+ goto deallocate_vmid;
+ pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
+ q->pipe, q->queue);
+ } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
+ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
+ retval = allocate_sdma_queue(dqm, q);
+ if (retval)
+ goto deallocate_vmid;
+ dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
+ }
- if (retval) {
- if (list_empty(&qpd->queues_list))
- deallocate_vmid(dqm, qpd, q);
- goto out_unlock;
+ retval = allocate_doorbell(qpd, q);
+ if (retval)
+ goto out_deallocate_hqd;
+
+ /* Temporarily release dqm lock to avoid a circular lock dependency */
+ dqm_unlock(dqm);
+ q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
+ dqm_lock(dqm);
+
+ if (!q->mqd_mem_obj) {
+ retval = -ENOMEM;
+ goto out_deallocate_doorbell;
+ }
+ mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
+ &q->gart_mqd_addr, &q->properties);
+ if (q->properties.is_active) {
+
+ if (WARN(q->process->mm != current->mm,
+ "should only run in user thread"))
+ retval = -EFAULT;
+ else
+ retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
+ q->queue, &q->properties, current->mm);
+ if (retval)
+ goto out_free_mqd;
}
list_add(&q->list, &qpd->queues_list);
@@ -307,6 +349,8 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
dqm->sdma_queue_count++;
+ else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
+ dqm->xgmi_sdma_queue_count++;
/*
* Unconditionally increment this counter, regardless of the queue's
@@ -315,7 +359,21 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
dqm->total_queue_count++;
pr_debug("Total of %d queues are accountable so far\n",
dqm->total_queue_count);
+ goto out_unlock;
+out_free_mqd:
+ mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
+out_deallocate_doorbell:
+ deallocate_doorbell(qpd, q);
+out_deallocate_hqd:
+ if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
+ deallocate_hqd(dqm, q);
+ else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
+ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
+ deallocate_sdma_queue(dqm, q);
+deallocate_vmid:
+ if (list_empty(&qpd->queues_list))
+ deallocate_vmid(dqm, qpd, q);
out_unlock:
dqm_unlock(dqm);
return retval;
@@ -361,60 +419,6 @@ static inline void deallocate_hqd(struct device_queue_manager *dqm,
dqm->allocated_queues[q->pipe] |= (1 << q->queue);
}
-static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
- struct queue *q,
- struct qcm_process_device *qpd)
-{
- struct mqd_manager *mqd_mgr;
- int retval;
-
- mqd_mgr = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
- if (!mqd_mgr)
- return -ENOMEM;
-
- retval = allocate_hqd(dqm, q);
- if (retval)
- return retval;
-
- retval = allocate_doorbell(qpd, q);
- if (retval)
- goto out_deallocate_hqd;
-
- retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
- &q->gart_mqd_addr, &q->properties);
- if (retval)
- goto out_deallocate_doorbell;
-
- pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
- q->pipe, q->queue);
-
- dqm->dev->kfd2kgd->set_scratch_backing_va(
- dqm->dev->kgd, qpd->sh_hidden_private_base, qpd->vmid);
-
- if (!q->properties.is_active)
- return 0;
-
- if (WARN(q->process->mm != current->mm,
- "should only run in user thread"))
- retval = -EFAULT;
- else
- retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, q->queue,
- &q->properties, current->mm);
- if (retval)
- goto out_uninit_mqd;
-
- return 0;
-
-out_uninit_mqd:
- mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
-out_deallocate_doorbell:
- deallocate_doorbell(qpd, q);
-out_deallocate_hqd:
- deallocate_hqd(dqm, q);
-
- return retval;
-}
-
/* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked
* to avoid asynchronized access
*/
@@ -425,16 +429,17 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
int retval;
struct mqd_manager *mqd_mgr;
- mqd_mgr = dqm->ops.get_mqd_manager(dqm,
- get_mqd_type_from_queue_type(q->properties.type));
- if (!mqd_mgr)
- return -ENOMEM;
+ mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
+ q->properties.type)];
if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
deallocate_hqd(dqm, q);
} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
dqm->sdma_queue_count--;
- deallocate_sdma_queue(dqm, q->sdma_id);
+ deallocate_sdma_queue(dqm, q);
+ } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
+ dqm->xgmi_sdma_queue_count--;
+ deallocate_sdma_queue(dqm, q);
} else {
pr_debug("q->properties.type %d is invalid\n",
q->properties.type);
@@ -451,7 +456,7 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
if (retval == -ETIME)
qpd->reset_wavefronts = true;
- mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
+ mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
list_del(&q->list);
if (list_empty(&qpd->queues_list)) {
@@ -490,7 +495,7 @@ static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
static int update_queue(struct device_queue_manager *dqm, struct queue *q)
{
- int retval;
+ int retval = 0;
struct mqd_manager *mqd_mgr;
struct kfd_process_device *pdd;
bool prev_active = false;
@@ -501,20 +506,8 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
retval = -ENODEV;
goto out_unlock;
}
- mqd_mgr = dqm->ops.get_mqd_manager(dqm,
- get_mqd_type_from_queue_type(q->properties.type));
- if (!mqd_mgr) {
- retval = -ENOMEM;
- goto out_unlock;
- }
- /*
- * Eviction state logic: we only mark active queues as evicted
- * to avoid the overhead of restoring inactive queues later
- */
- if (pdd->qpd.evicted)
- q->properties.is_evicted = (q->properties.queue_size > 0 &&
- q->properties.queue_percent > 0 &&
- q->properties.queue_address != 0);
+ mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
+ q->properties.type)];
/* Save previous activity state for counters */
prev_active = q->properties.is_active;
@@ -529,7 +522,8 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
}
} else if (prev_active &&
(q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
- q->properties.type == KFD_QUEUE_TYPE_SDMA)) {
+ q->properties.type == KFD_QUEUE_TYPE_SDMA ||
+ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
@@ -539,7 +533,7 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
}
}
- retval = mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties);
+ mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties);
/*
* check active state vs. the previous state and modify
@@ -556,7 +550,8 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
retval = map_queues_cpsch(dqm);
else if (q->properties.is_active &&
(q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
- q->properties.type == KFD_QUEUE_TYPE_SDMA)) {
+ q->properties.type == KFD_QUEUE_TYPE_SDMA ||
+ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
if (WARN(q->process->mm != current->mm,
"should only run in user thread"))
retval = -EFAULT;
@@ -571,34 +566,13 @@ out_unlock:
return retval;
}
-static struct mqd_manager *get_mqd_manager(
- struct device_queue_manager *dqm, enum KFD_MQD_TYPE type)
-{
- struct mqd_manager *mqd_mgr;
-
- if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
- return NULL;
-
- pr_debug("mqd type %d\n", type);
-
- mqd_mgr = dqm->mqd_mgrs[type];
- if (!mqd_mgr) {
- mqd_mgr = mqd_manager_init(type, dqm->dev);
- if (!mqd_mgr)
- pr_err("mqd manager is NULL");
- dqm->mqd_mgrs[type] = mqd_mgr;
- }
-
- return mqd_mgr;
-}
-
static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
struct qcm_process_device *qpd)
{
struct queue *q;
struct mqd_manager *mqd_mgr;
struct kfd_process_device *pdd;
- int retval = 0;
+ int retval, ret = 0;
dqm_lock(dqm);
if (qpd->evicted++ > 0) /* already evicted, do nothing */
@@ -608,30 +582,31 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
pr_info_ratelimited("Evicting PASID %u queues\n",
pdd->process->pasid);
- /* unactivate all active queues on the qpd */
+ /* Mark all queues as evicted. Deactivate all active queues on
+ * the qpd.
+ */
list_for_each_entry(q, &qpd->queues_list, list) {
+ q->properties.is_evicted = true;
if (!q->properties.is_active)
continue;
- mqd_mgr = dqm->ops.get_mqd_manager(dqm,
- get_mqd_type_from_queue_type(q->properties.type));
- if (!mqd_mgr) { /* should not be here */
- pr_err("Cannot evict queue, mqd mgr is NULL\n");
- retval = -ENOMEM;
- goto out;
- }
- q->properties.is_evicted = true;
+
+ mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
+ q->properties.type)];
q->properties.is_active = false;
retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
- if (retval)
- goto out;
+ if (retval && !ret)
+ /* Return the first error, but keep going to
+ * maintain a consistent eviction state
+ */
+ ret = retval;
dqm->queue_count--;
}
out:
dqm_unlock(dqm);
- return retval;
+ return ret;
}
static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
@@ -649,11 +624,14 @@ static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
pr_info_ratelimited("Evicting PASID %u queues\n",
pdd->process->pasid);
- /* unactivate all active queues on the qpd */
+ /* Mark all queues as evicted. Deactivate all active queues on
+ * the qpd.
+ */
list_for_each_entry(q, &qpd->queues_list, list) {
+ q->properties.is_evicted = true;
if (!q->properties.is_active)
continue;
- q->properties.is_evicted = true;
+
q->properties.is_active = false;
dqm->queue_count--;
}
@@ -675,7 +653,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
struct mqd_manager *mqd_mgr;
struct kfd_process_device *pdd;
uint64_t pd_base;
- int retval = 0;
+ int retval, ret = 0;
pdd = qpd_to_pdd(qpd);
/* Retrieve PD base */
@@ -709,27 +687,28 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
*/
mm = get_task_mm(pdd->process->lead_thread);
if (!mm) {
- retval = -EFAULT;
+ ret = -EFAULT;
goto out;
}
- /* activate all active queues on the qpd */
+ /* Remove the eviction flags. Activate queues that are not
+ * inactive for other reasons.
+ */
list_for_each_entry(q, &qpd->queues_list, list) {
- if (!q->properties.is_evicted)
- continue;
- mqd_mgr = dqm->ops.get_mqd_manager(dqm,
- get_mqd_type_from_queue_type(q->properties.type));
- if (!mqd_mgr) { /* should not be here */
- pr_err("Cannot restore queue, mqd mgr is NULL\n");
- retval = -ENOMEM;
- goto out;
- }
q->properties.is_evicted = false;
+ if (!QUEUE_IS_ACTIVE(q->properties))
+ continue;
+
+ mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
+ q->properties.type)];
q->properties.is_active = true;
retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
q->queue, &q->properties, mm);
- if (retval)
- goto out;
+ if (retval && !ret)
+ /* Return the first error, but keep going to
+ * maintain a consistent eviction state
+ */
+ ret = retval;
dqm->queue_count++;
}
qpd->evicted = 0;
@@ -737,7 +716,7 @@ out:
if (mm)
mmput(mm);
dqm_unlock(dqm);
- return retval;
+ return ret;
}
static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
@@ -769,16 +748,16 @@ static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
/* activate all active queues on the qpd */
list_for_each_entry(q, &qpd->queues_list, list) {
- if (!q->properties.is_evicted)
- continue;
q->properties.is_evicted = false;
+ if (!QUEUE_IS_ACTIVE(q->properties))
+ continue;
+
q->properties.is_active = true;
dqm->queue_count++;
}
retval = execute_queues_cpsch(dqm,
KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
- if (!retval)
- qpd->evicted = 0;
+ qpd->evicted = 0;
out:
dqm_unlock(dqm);
return retval;
@@ -812,10 +791,14 @@ static int register_process(struct device_queue_manager *dqm,
retval = dqm->asic_ops.update_qpd(dqm, qpd);
dqm->processes_count++;
- kfd_inc_compute_active(dqm->dev);
dqm_unlock(dqm);
+ /* Outside the DQM lock because under the DQM lock we can't do
+ * reclaim or take other locks that others hold while reclaiming.
+ */
+ kfd_inc_compute_active(dqm->dev);
+
return retval;
}
@@ -836,7 +819,6 @@ static int unregister_process(struct device_queue_manager *dqm,
list_del(&cur->list);
kfree(cur);
dqm->processes_count--;
- kfd_dec_compute_active(dqm->dev);
goto out;
}
}
@@ -844,6 +826,13 @@ static int unregister_process(struct device_queue_manager *dqm,
retval = 1;
out:
dqm_unlock(dqm);
+
+ /* Outside the DQM lock because under the DQM lock we can't do
+ * reclaim or take other locks that others hold while reclaiming.
+ */
+ if (!retval)
+ kfd_dec_compute_active(dqm->dev);
+
return retval;
}
@@ -879,6 +868,7 @@ static int initialize_nocpsch(struct device_queue_manager *dqm)
INIT_LIST_HEAD(&dqm->queues);
dqm->queue_count = dqm->next_pipe_to_allocate = 0;
dqm->sdma_queue_count = 0;
+ dqm->xgmi_sdma_queue_count = 0;
for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
int pipe_offset = pipe * get_queues_per_pipe(dqm);
@@ -890,7 +880,8 @@ static int initialize_nocpsch(struct device_queue_manager *dqm)
}
dqm->vmid_bitmap = (1 << dqm->dev->vm_info.vmid_num_kfd) - 1;
- dqm->sdma_bitmap = (1 << get_num_sdma_queues(dqm)) - 1;
+ dqm->sdma_bitmap = (1ULL << get_num_sdma_queues(dqm)) - 1;
+ dqm->xgmi_sdma_bitmap = (1ULL << get_num_xgmi_sdma_queues(dqm)) - 1;
return 0;
}
@@ -921,75 +912,56 @@ static int stop_nocpsch(struct device_queue_manager *dqm)
}
static int allocate_sdma_queue(struct device_queue_manager *dqm,
- unsigned int *sdma_queue_id)
+ struct queue *q)
{
int bit;
- if (dqm->sdma_bitmap == 0)
- return -ENOMEM;
+ if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
+ if (dqm->sdma_bitmap == 0)
+ return -ENOMEM;
+ bit = __ffs64(dqm->sdma_bitmap);
+ dqm->sdma_bitmap &= ~(1ULL << bit);
+ q->sdma_id = bit;
+ q->properties.sdma_engine_id = q->sdma_id %
+ get_num_sdma_engines(dqm);
+ q->properties.sdma_queue_id = q->sdma_id /
+ get_num_sdma_engines(dqm);
+ } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
+ if (dqm->xgmi_sdma_bitmap == 0)
+ return -ENOMEM;
+ bit = __ffs64(dqm->xgmi_sdma_bitmap);
+ dqm->xgmi_sdma_bitmap &= ~(1ULL << bit);
+ q->sdma_id = bit;
+ /* sdma_engine_id is sdma id including
+ * both PCIe-optimized SDMAs and XGMI-
+ * optimized SDMAs. The calculation below
+ * assumes the first N engines are always
+ * PCIe-optimized ones
+ */
+ q->properties.sdma_engine_id = get_num_sdma_engines(dqm) +
+ q->sdma_id % get_num_xgmi_sdma_engines(dqm);
+ q->properties.sdma_queue_id = q->sdma_id /
+ get_num_xgmi_sdma_engines(dqm);
+ }
- bit = ffs(dqm->sdma_bitmap) - 1;
- dqm->sdma_bitmap &= ~(1 << bit);
- *sdma_queue_id = bit;
+ pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
+ pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
return 0;
}
static void deallocate_sdma_queue(struct device_queue_manager *dqm,
- unsigned int sdma_queue_id)
-{
- if (sdma_queue_id >= get_num_sdma_queues(dqm))
- return;
- dqm->sdma_bitmap |= (1 << sdma_queue_id);
-}
-
-static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
- struct queue *q,
- struct qcm_process_device *qpd)
+ struct queue *q)
{
- struct mqd_manager *mqd_mgr;
- int retval;
-
- mqd_mgr = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
- if (!mqd_mgr)
- return -ENOMEM;
-
- retval = allocate_sdma_queue(dqm, &q->sdma_id);
- if (retval)
- return retval;
-
- q->properties.sdma_queue_id = q->sdma_id / get_num_sdma_engines(dqm);
- q->properties.sdma_engine_id = q->sdma_id % get_num_sdma_engines(dqm);
-
- retval = allocate_doorbell(qpd, q);
- if (retval)
- goto out_deallocate_sdma_queue;
-
- pr_debug("SDMA id is: %d\n", q->sdma_id);
- pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
- pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
-
- dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
- retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
- &q->gart_mqd_addr, &q->properties);
- if (retval)
- goto out_deallocate_doorbell;
-
- retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, 0, 0, &q->properties,
- NULL);
- if (retval)
- goto out_uninit_mqd;
-
- return 0;
-
-out_uninit_mqd:
- mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
-out_deallocate_doorbell:
- deallocate_doorbell(qpd, q);
-out_deallocate_sdma_queue:
- deallocate_sdma_queue(dqm, q->sdma_id);
-
- return retval;
+ if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
+ if (q->sdma_id >= get_num_sdma_queues(dqm))
+ return;
+ dqm->sdma_bitmap |= (1ULL << q->sdma_id);
+ } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
+ if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm))
+ return;
+ dqm->xgmi_sdma_bitmap |= (1ULL << q->sdma_id);
+ }
}
/*
@@ -1026,8 +998,8 @@ static int set_sched_resources(struct device_queue_manager *dqm)
res.queue_mask |= (1ull << i);
}
- res.gws_mask = res.oac_mask = res.gds_heap_base =
- res.gds_heap_size = 0;
+ res.gws_mask = ~0ull;
+ res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0;
pr_debug("Scheduling resources:\n"
"vmid mask: 0x%8X\n"
@@ -1045,8 +1017,10 @@ static int initialize_cpsch(struct device_queue_manager *dqm)
INIT_LIST_HEAD(&dqm->queues);
dqm->queue_count = dqm->processes_count = 0;
dqm->sdma_queue_count = 0;
+ dqm->xgmi_sdma_queue_count = 0;
dqm->active_runlist = false;
- dqm->sdma_bitmap = (1 << get_num_sdma_queues(dqm)) - 1;
+ dqm->sdma_bitmap = (1ULL << get_num_sdma_queues(dqm)) - 1;
+ dqm->xgmi_sdma_bitmap = (1ULL << get_num_xgmi_sdma_queues(dqm)) - 1;
INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception);
@@ -1161,55 +1135,49 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
int retval;
struct mqd_manager *mqd_mgr;
- retval = 0;
-
- dqm_lock(dqm);
-
if (dqm->total_queue_count >= max_num_of_queues_per_device) {
pr_warn("Can't create new usermode queue because %d queues were already created\n",
dqm->total_queue_count);
retval = -EPERM;
- goto out_unlock;
+ goto out;
}
- if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
- retval = allocate_sdma_queue(dqm, &q->sdma_id);
+ if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
+ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
+ dqm_lock(dqm);
+ retval = allocate_sdma_queue(dqm, q);
+ dqm_unlock(dqm);
if (retval)
- goto out_unlock;
- q->properties.sdma_queue_id =
- q->sdma_id / get_num_sdma_engines(dqm);
- q->properties.sdma_engine_id =
- q->sdma_id % get_num_sdma_engines(dqm);
+ goto out;
}
retval = allocate_doorbell(qpd, q);
if (retval)
goto out_deallocate_sdma_queue;
- mqd_mgr = dqm->ops.get_mqd_manager(dqm,
- get_mqd_type_from_queue_type(q->properties.type));
+ mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
+ q->properties.type)];
- if (!mqd_mgr) {
+ if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
+ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
+ dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
+ q->properties.tba_addr = qpd->tba_addr;
+ q->properties.tma_addr = qpd->tma_addr;
+ q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
+ if (!q->mqd_mem_obj) {
retval = -ENOMEM;
goto out_deallocate_doorbell;
}
+
+ dqm_lock(dqm);
/*
- * Eviction state logic: we only mark active queues as evicted
- * to avoid the overhead of restoring inactive queues later
+ * Eviction state logic: mark all queues as evicted, even ones
+ * not currently active. Restoring inactive queues later only
+ * updates the is_evicted flag but is a no-op otherwise.
*/
- if (qpd->evicted)
- q->properties.is_evicted = (q->properties.queue_size > 0 &&
- q->properties.queue_percent > 0 &&
- q->properties.queue_address != 0);
-
- dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
-
- q->properties.tba_addr = qpd->tba_addr;
- q->properties.tma_addr = qpd->tma_addr;
- retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
+ q->properties.is_evicted = !!qpd->evicted;
+ mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
&q->gart_mqd_addr, &q->properties);
- if (retval)
- goto out_deallocate_doorbell;
list_add(&q->list, &qpd->queues_list);
qpd->queue_count++;
@@ -1221,6 +1189,8 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
dqm->sdma_queue_count++;
+ else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
+ dqm->xgmi_sdma_queue_count++;
/*
* Unconditionally increment this counter, regardless of the queue's
* type or whether the queue is active.
@@ -1236,11 +1206,13 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
out_deallocate_doorbell:
deallocate_doorbell(qpd, q);
out_deallocate_sdma_queue:
- if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
- deallocate_sdma_queue(dqm, q->sdma_id);
-out_unlock:
- dqm_unlock(dqm);
-
+ if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
+ q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
+ dqm_lock(dqm);
+ deallocate_sdma_queue(dqm, q);
+ dqm_unlock(dqm);
+ }
+out:
return retval;
}
@@ -1268,12 +1240,18 @@ int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
return 0;
}
-static int unmap_sdma_queues(struct device_queue_manager *dqm,
- unsigned int sdma_engine)
+static int unmap_sdma_queues(struct device_queue_manager *dqm)
{
- return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA,
- KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, false,
- sdma_engine);
+ int i, retval = 0;
+
+ for (i = 0; i < dqm->dev->device_info->num_sdma_engines +
+ dqm->dev->device_info->num_xgmi_sdma_engines; i++) {
+ retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA,
+ KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, false, i);
+ if (retval)
+ return retval;
+ }
+ return retval;
}
/* dqm->lock mutex has to be locked before calling this function */
@@ -1288,6 +1266,7 @@ static int map_queues_cpsch(struct device_queue_manager *dqm)
return 0;
retval = pm_send_runlist(&dqm->packets, &dqm->queues);
+ pr_debug("%s sent runlist\n", __func__);
if (retval) {
pr_err("failed to execute runlist\n");
return retval;
@@ -1309,13 +1288,11 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,
if (!dqm->active_runlist)
return retval;
- pr_debug("Before destroying queues, sdma queue count is : %u\n",
- dqm->sdma_queue_count);
+ pr_debug("Before destroying queues, sdma queue count is : %u, xgmi sdma queue count is : %u\n",
+ dqm->sdma_queue_count, dqm->xgmi_sdma_queue_count);
- if (dqm->sdma_queue_count > 0) {
- unmap_sdma_queues(dqm, 0);
- unmap_sdma_queues(dqm, 1);
- }
+ if (dqm->sdma_queue_count > 0 || dqm->xgmi_sdma_queue_count)
+ unmap_sdma_queues(dqm);
retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
filter, filter_param, false, 0);
@@ -1327,7 +1304,7 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,
KFD_FENCE_COMPLETED);
/* should be timed out */
retval = amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
- QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
+ queue_preemption_timeout_ms);
if (retval)
return retval;
@@ -1379,18 +1356,17 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
}
- mqd_mgr = dqm->ops.get_mqd_manager(dqm,
- get_mqd_type_from_queue_type(q->properties.type));
- if (!mqd_mgr) {
- retval = -ENOMEM;
- goto failed;
- }
+ mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
+ q->properties.type)];
deallocate_doorbell(qpd, q);
if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
dqm->sdma_queue_count--;
- deallocate_sdma_queue(dqm, q->sdma_id);
+ deallocate_sdma_queue(dqm, q);
+ } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
+ dqm->xgmi_sdma_queue_count--;
+ deallocate_sdma_queue(dqm, q);
}
list_del(&q->list);
@@ -1403,8 +1379,6 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
qpd->reset_wavefronts = true;
}
- mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
-
/*
* Unconditionally decrement this counter, regardless of the queue's
* type
@@ -1415,9 +1389,11 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
dqm_unlock(dqm);
+ /* Do free_mqd after dqm_unlock(dqm) to avoid circular locking */
+ mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
+
return retval;
-failed:
failed_try_destroy_debugged_queue:
dqm_unlock(dqm);
@@ -1520,6 +1496,7 @@ static int process_termination_nocpsch(struct device_queue_manager *dqm,
struct queue *q, *next;
struct device_process_node *cur, *next_dpn;
int retval = 0;
+ bool found = false;
dqm_lock(dqm);
@@ -1538,12 +1515,19 @@ static int process_termination_nocpsch(struct device_queue_manager *dqm,
list_del(&cur->list);
kfree(cur);
dqm->processes_count--;
- kfd_dec_compute_active(dqm->dev);
+ found = true;
break;
}
}
dqm_unlock(dqm);
+
+ /* Outside the DQM lock because under the DQM lock we can't do
+ * reclaim or take other locks that others hold while reclaiming.
+ */
+ if (found)
+ kfd_dec_compute_active(dqm->dev);
+
return retval;
}
@@ -1564,11 +1548,7 @@ static int get_wave_state(struct device_queue_manager *dqm,
goto dqm_unlock;
}
- mqd_mgr = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
- if (!mqd_mgr) {
- r = -ENOMEM;
- goto dqm_unlock;
- }
+ mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_COMPUTE];
if (!mqd_mgr->get_wave_state) {
r = -EINVAL;
@@ -1593,6 +1573,7 @@ static int process_termination_cpsch(struct device_queue_manager *dqm,
struct device_process_node *cur, *next_dpn;
enum kfd_unmap_queues_filter filter =
KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
+ bool found = false;
retval = 0;
@@ -1611,7 +1592,10 @@ static int process_termination_cpsch(struct device_queue_manager *dqm,
list_for_each_entry(q, &qpd->queues_list, list) {
if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
dqm->sdma_queue_count--;
- deallocate_sdma_queue(dqm, q->sdma_id);
+ deallocate_sdma_queue(dqm, q);
+ } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
+ dqm->xgmi_sdma_queue_count--;
+ deallocate_sdma_queue(dqm, q);
}
if (q->properties.is_active)
@@ -1626,7 +1610,7 @@ static int process_termination_cpsch(struct device_queue_manager *dqm,
list_del(&cur->list);
kfree(cur);
dqm->processes_count--;
- kfd_dec_compute_active(dqm->dev);
+ found = true;
break;
}
}
@@ -1638,21 +1622,68 @@ static int process_termination_cpsch(struct device_queue_manager *dqm,
qpd->reset_wavefronts = false;
}
- /* lastly, free mqd resources */
+ dqm_unlock(dqm);
+
+ /* Outside the DQM lock because under the DQM lock we can't do
+ * reclaim or take other locks that others hold while reclaiming.
+ */
+ if (found)
+ kfd_dec_compute_active(dqm->dev);
+
+ /* Lastly, free mqd resources.
+ * Do free_mqd() after dqm_unlock to avoid circular locking.
+ */
list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
- mqd_mgr = dqm->ops.get_mqd_manager(dqm,
- get_mqd_type_from_queue_type(q->properties.type));
- if (!mqd_mgr) {
- retval = -ENOMEM;
- goto out;
- }
+ mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
+ q->properties.type)];
list_del(&q->list);
qpd->queue_count--;
- mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
+ mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
}
-out:
- dqm_unlock(dqm);
+ return retval;
+}
+
+static int init_mqd_managers(struct device_queue_manager *dqm)
+{
+ int i, j;
+ struct mqd_manager *mqd_mgr;
+
+ for (i = 0; i < KFD_MQD_TYPE_MAX; i++) {
+ mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev);
+ if (!mqd_mgr) {
+ pr_err("mqd manager [%d] initialization failed\n", i);
+ goto out_free;
+ }
+ dqm->mqd_mgrs[i] = mqd_mgr;
+ }
+
+ return 0;
+
+out_free:
+ for (j = 0; j < i; j++) {
+ kfree(dqm->mqd_mgrs[j]);
+ dqm->mqd_mgrs[j] = NULL;
+ }
+
+ return -ENOMEM;
+}
+
+/* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/
+static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm)
+{
+ int retval;
+ struct kfd_dev *dev = dqm->dev;
+ struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd;
+ uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size *
+ dev->device_info->num_sdma_engines *
+ dev->device_info->num_sdma_queues_per_engine +
+ dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size;
+
+ retval = amdgpu_amdkfd_alloc_gtt_mem(dev->kgd, size,
+ &(mem_obj->gtt_mem), &(mem_obj->gpu_addr),
+ (void *)&(mem_obj->cpu_ptr), true);
+
return retval;
}
@@ -1693,7 +1724,6 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
dqm->ops.stop = stop_cpsch;
dqm->ops.destroy_queue = destroy_queue_cpsch;
dqm->ops.update_queue = update_queue;
- dqm->ops.get_mqd_manager = get_mqd_manager;
dqm->ops.register_process = register_process;
dqm->ops.unregister_process = unregister_process;
dqm->ops.uninitialize = uninitialize;
@@ -1713,7 +1743,6 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
dqm->ops.create_queue = create_queue_nocpsch;
dqm->ops.destroy_queue = destroy_queue_nocpsch;
dqm->ops.update_queue = update_queue;
- dqm->ops.get_mqd_manager = get_mqd_manager;
dqm->ops.register_process = register_process;
dqm->ops.unregister_process = unregister_process;
dqm->ops.initialize = initialize_nocpsch;
@@ -1749,6 +1778,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
case CHIP_POLARIS10:
case CHIP_POLARIS11:
case CHIP_POLARIS12:
+ case CHIP_VEGAM:
device_queue_manager_init_vi_tonga(&dqm->asic_ops);
break;
@@ -1758,12 +1788,23 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
case CHIP_RAVEN:
device_queue_manager_init_v9(&dqm->asic_ops);
break;
+ case CHIP_NAVI10:
+ device_queue_manager_init_v10_navi10(&dqm->asic_ops);
+ break;
default:
WARN(1, "Unexpected ASIC family %u",
dev->device_info->asic_family);
goto out_free;
}
+ if (init_mqd_managers(dqm))
+ goto out_free;
+
+ if (allocate_hiq_sdma_mqd(dqm)) {
+ pr_err("Failed to allocate hiq sdma mqd trunk buffer\n");
+ goto out_free;
+ }
+
if (!dqm->ops.initialize(dqm))
return dqm;
@@ -1772,9 +1813,17 @@ out_free:
return NULL;
}
+void deallocate_hiq_sdma_mqd(struct kfd_dev *dev, struct kfd_mem_obj *mqd)
+{
+ WARN(!mqd, "No hiq sdma mqd trunk to free");
+
+ amdgpu_amdkfd_free_gtt_mem(dev->kgd, mqd->gtt_mem);
+}
+
void device_queue_manager_uninit(struct device_queue_manager *dqm)
{
dqm->ops.uninitialize(dqm);
+ deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd);
kfree(dqm);
}
@@ -1833,12 +1882,13 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data)
int r = 0;
r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->kgd,
- KFD_CIK_HIQ_PIPE, KFD_CIK_HIQ_QUEUE, &dump, &n_regs);
+ KFD_CIK_HIQ_PIPE, KFD_CIK_HIQ_QUEUE,
+ &dump, &n_regs);
if (!r) {
seq_printf(m, " HIQ on MEC %d Pipe %d Queue %d\n",
- KFD_CIK_HIQ_PIPE/get_pipes_per_mec(dqm)+1,
- KFD_CIK_HIQ_PIPE%get_pipes_per_mec(dqm),
- KFD_CIK_HIQ_QUEUE);
+ KFD_CIK_HIQ_PIPE/get_pipes_per_mec(dqm)+1,
+ KFD_CIK_HIQ_PIPE%get_pipes_per_mec(dqm),
+ KFD_CIK_HIQ_QUEUE);
seq_reg_dump(m, dump, n_regs);
kfree(dump);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index 70e38a2e23b9..90db2c9275f6 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -31,8 +31,6 @@
#include "kfd_priv.h"
#include "kfd_mqd_manager.h"
-#define KFD_UNMAP_LATENCY_MS (4000)
-#define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (2 * KFD_UNMAP_LATENCY_MS + 1000)
struct device_process_node {
struct qcm_process_device *qpd;
@@ -48,8 +46,6 @@ struct device_process_node {
*
* @update_queue: Queue update routine.
*
- * @get_mqd_manager: Returns the mqd manager according to the mqd type.
- *
* @exeute_queues: Dispatches the queues list to the H/W.
*
* @register_process: This routine associates a specific process with device.
@@ -97,10 +93,6 @@ struct device_queue_manager_ops {
int (*update_queue)(struct device_queue_manager *dqm,
struct queue *q);
- struct mqd_manager * (*get_mqd_manager)
- (struct device_queue_manager *dqm,
- enum KFD_MQD_TYPE type);
-
int (*register_process)(struct device_queue_manager *dqm,
struct qcm_process_device *qpd);
@@ -158,6 +150,8 @@ struct device_queue_manager_asic_ops {
void (*init_sdma_vm)(struct device_queue_manager *dqm,
struct queue *q,
struct qcm_process_device *qpd);
+ struct mqd_manager * (*mqd_manager_init)(enum KFD_MQD_TYPE type,
+ struct kfd_dev *dev);
};
/**
@@ -185,10 +179,12 @@ struct device_queue_manager {
unsigned int processes_count;
unsigned int queue_count;
unsigned int sdma_queue_count;
+ unsigned int xgmi_sdma_queue_count;
unsigned int total_queue_count;
unsigned int next_pipe_to_allocate;
unsigned int *allocated_queues;
- unsigned int sdma_bitmap;
+ uint64_t sdma_bitmap;
+ uint64_t xgmi_sdma_bitmap;
unsigned int vmid_bitmap;
uint64_t pipelines_addr;
struct kfd_mem_obj *pipeline_mem;
@@ -201,6 +197,7 @@ struct device_queue_manager {
/* hw exception */
bool is_hws_hang;
struct work_struct hw_exception_work;
+ struct kfd_mem_obj hiq_sdma_mqd;
};
void device_queue_manager_init_cik(
@@ -213,12 +210,15 @@ void device_queue_manager_init_vi_tonga(
struct device_queue_manager_asic_ops *asic_ops);
void device_queue_manager_init_v9(
struct device_queue_manager_asic_ops *asic_ops);
+void device_queue_manager_init_v10_navi10(
+ struct device_queue_manager_asic_ops *asic_ops);
void program_sh_mem_settings(struct device_queue_manager *dqm,
struct qcm_process_device *qpd);
unsigned int get_queues_num(struct device_queue_manager *dqm);
unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);
unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);
unsigned int get_num_sdma_queues(struct device_queue_manager *dqm);
+unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm);
static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
{
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
index aed4c21417bf..0d26506798cf 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
@@ -48,6 +48,7 @@ void device_queue_manager_init_cik(
asic_ops->set_cache_memory_policy = set_cache_memory_policy_cik;
asic_ops->update_qpd = update_qpd_cik;
asic_ops->init_sdma_vm = init_sdma_vm;
+ asic_ops->mqd_manager_init = mqd_manager_init_cik;
}
void device_queue_manager_init_cik_hawaii(
@@ -56,6 +57,7 @@ void device_queue_manager_init_cik_hawaii(
asic_ops->set_cache_memory_policy = set_cache_memory_policy_cik;
asic_ops->update_qpd = update_qpd_cik_hawaii;
asic_ops->init_sdma_vm = init_sdma_vm_hawaii;
+ asic_ops->mqd_manager_init = mqd_manager_init_cik_hawaii;
}
static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v10.c
new file mode 100644
index 000000000000..72e4d61ac752
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v10.c
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "kfd_device_queue_manager.h"
+#include "navi10_enum.h"
+#include "gc/gc_10_1_0_offset.h"
+#include "gc/gc_10_1_0_sh_mask.h"
+
+static int update_qpd_v10(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd);
+static void init_sdma_vm_v10(struct device_queue_manager *dqm, struct queue *q,
+ struct qcm_process_device *qpd);
+
+void device_queue_manager_init_v10_navi10(
+ struct device_queue_manager_asic_ops *asic_ops)
+{
+ asic_ops->update_qpd = update_qpd_v10;
+ asic_ops->init_sdma_vm = init_sdma_vm_v10;
+ asic_ops->mqd_manager_init = mqd_manager_init_v10;
+}
+
+static uint32_t compute_sh_mem_bases_64bit(struct kfd_process_device *pdd)
+{
+ uint32_t shared_base = pdd->lds_base >> 48;
+ uint32_t private_base = pdd->scratch_base >> 48;
+
+ return (shared_base << SH_MEM_BASES__SHARED_BASE__SHIFT) |
+ private_base;
+}
+
+static int update_qpd_v10(struct device_queue_manager *dqm,
+ struct qcm_process_device *qpd)
+{
+ struct kfd_process_device *pdd;
+
+ pdd = qpd_to_pdd(qpd);
+
+ /* check if sh_mem_config register already configured */
+ if (qpd->sh_mem_config == 0) {
+ qpd->sh_mem_config =
+ SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
+ SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
+#if 0
+ /* TODO:
+ * This shouldn't be an issue with Navi10. Verify.
+ */
+ if (vega10_noretry)
+ qpd->sh_mem_config |=
+ 1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
+#endif
+
+ qpd->sh_mem_ape1_limit = 0;
+ qpd->sh_mem_ape1_base = 0;
+ }
+
+ qpd->sh_mem_bases = compute_sh_mem_bases_64bit(pdd);
+
+ pr_debug("sh_mem_bases 0x%X\n", qpd->sh_mem_bases);
+
+ return 0;
+}
+
+static void init_sdma_vm_v10(struct device_queue_manager *dqm, struct queue *q,
+ struct qcm_process_device *qpd)
+{
+ /* Not needed on SDMAv4 onwards any more */
+ q->properties.sdma_vm_addr = 0;
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
index 417515332c35..95a82ac455f2 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c
@@ -37,6 +37,7 @@ void device_queue_manager_init_v9(
{
asic_ops->update_qpd = update_qpd_v9;
asic_ops->init_sdma_vm = init_sdma_vm_v9;
+ asic_ops->mqd_manager_init = mqd_manager_init_v9;
}
static uint32_t compute_sh_mem_bases_64bit(struct kfd_process_device *pdd)
@@ -60,7 +61,7 @@ static int update_qpd_v9(struct device_queue_manager *dqm,
qpd->sh_mem_config =
SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
- if (noretry &&
+ if (amdgpu_noretry &&
!dqm->dev->device_info->needs_iommu_device)
qpd->sh_mem_config |=
1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
index c3a5dcfe877a..3a7cb2f88366 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
@@ -54,6 +54,7 @@ void device_queue_manager_init_vi(
asic_ops->set_cache_memory_policy = set_cache_memory_policy_vi;
asic_ops->update_qpd = update_qpd_vi;
asic_ops->init_sdma_vm = init_sdma_vm;
+ asic_ops->mqd_manager_init = mqd_manager_init_vi;
}
void device_queue_manager_init_vi_tonga(
@@ -62,6 +63,7 @@ void device_queue_manager_init_vi_tonga(
asic_ops->set_cache_memory_policy = set_cache_memory_policy_vi_tonga;
asic_ops->update_qpd = update_qpd_vi_tonga;
asic_ops->init_sdma_vm = init_sdma_vm_tonga;
+ asic_ops->mqd_manager_init = mqd_manager_init_vi_tonga;
}
static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
index 6e1d41c5bf86..d674d4b3340f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
@@ -983,7 +983,7 @@ void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
return; /* Presumably process exited. */
memset(&memory_exception_data, 0, sizeof(memory_exception_data));
memory_exception_data.gpu_id = dev->id;
- memory_exception_data.failure.imprecise = 1;
+ memory_exception_data.failure.imprecise = true;
/* Set failure reason */
if (info) {
memory_exception_data.va = (info->page_addr) << PAGE_SHIFT;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
index 213ea5454d11..60521366dd31 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
@@ -398,12 +398,14 @@ int kfd_init_apertures(struct kfd_process *process)
case CHIP_POLARIS10:
case CHIP_POLARIS11:
case CHIP_POLARIS12:
+ case CHIP_VEGAM:
kfd_init_apertures_vi(pdd, id);
break;
case CHIP_VEGA10:
case CHIP_VEGA12:
case CHIP_VEGA20:
case CHIP_RAVEN:
+ case CHIP_NAVI10:
kfd_init_apertures_v9(pdd, id);
break;
default:
@@ -435,5 +437,3 @@ int kfd_init_apertures(struct kfd_process *process)
return 0;
}
-
-
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c
index 01494752c36a..5f35df23fb18 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c
@@ -66,16 +66,8 @@ int kfd_iommu_device_init(struct kfd_dev *kfd)
top_dev = kfd_topology_device_by_id(kfd->id);
- /*
- * Overwrite ATS capability according to needs_iommu_device to fix
- * potential missing corresponding bit in CRAT of BIOS.
- */
- if (!kfd->device_info->needs_iommu_device) {
- top_dev->node_props.capability &= ~HSA_CAP_ATS_PRESENT;
+ if (!kfd->device_info->needs_iommu_device)
return 0;
- }
-
- top_dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
iommu_info.flags = 0;
err = amd_iommu_device_info(kfd->pdev, &iommu_info);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
index f1596881f20a..29c0bd2d7a5c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
@@ -58,9 +58,10 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
kq->nop_packet = nop.u32all;
switch (type) {
case KFD_QUEUE_TYPE_DIQ:
+ kq->mqd_mgr = dev->dqm->mqd_mgrs[KFD_MQD_TYPE_DIQ];
+ break;
case KFD_QUEUE_TYPE_HIQ:
- kq->mqd_mgr = dev->dqm->ops.get_mqd_manager(dev->dqm,
- KFD_MQD_TYPE_HIQ);
+ kq->mqd_mgr = dev->dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ];
break;
default:
pr_err("Invalid queue type %d\n", type);
@@ -131,13 +132,14 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
kq->queue->device = dev;
kq->queue->process = kfd_get_process(current);
- retval = kq->mqd_mgr->init_mqd(kq->mqd_mgr, &kq->queue->mqd,
- &kq->queue->mqd_mem_obj,
+ kq->queue->mqd_mem_obj = kq->mqd_mgr->allocate_mqd(kq->mqd_mgr->dev,
+ &kq->queue->properties);
+ if (!kq->queue->mqd_mem_obj)
+ goto err_allocate_mqd;
+ kq->mqd_mgr->init_mqd(kq->mqd_mgr, &kq->queue->mqd,
+ kq->queue->mqd_mem_obj,
&kq->queue->gart_mqd_addr,
&kq->queue->properties);
- if (retval != 0)
- goto err_init_mqd;
-
/* assign HIQ to HQD */
if (type == KFD_QUEUE_TYPE_HIQ) {
pr_debug("Assigning hiq to hqd\n");
@@ -163,7 +165,8 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
return true;
err_alloc_fence:
-err_init_mqd:
+ kq->mqd_mgr->free_mqd(kq->mqd_mgr, kq->queue->mqd, kq->queue->mqd_mem_obj);
+err_allocate_mqd:
uninit_queue(kq->queue);
err_init_queue:
kfd_gtt_sa_free(dev, kq->wptr_mem);
@@ -192,7 +195,7 @@ static void uninitialize(struct kernel_queue *kq)
else if (kq->queue->properties.type == KFD_QUEUE_TYPE_DIQ)
kfd_gtt_sa_free(kq->dev, kq->fence_mem_obj);
- kq->mqd_mgr->uninit_mqd(kq->mqd_mgr, kq->queue->mqd,
+ kq->mqd_mgr->free_mqd(kq->mqd_mgr, kq->queue->mqd,
kq->queue->mqd_mem_obj);
kfd_gtt_sa_free(kq->dev, kq->rptr_mem);
@@ -314,6 +317,7 @@ struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
case CHIP_POLARIS10:
case CHIP_POLARIS11:
case CHIP_POLARIS12:
+ case CHIP_VEGAM:
kernel_queue_init_vi(&kq->ops_asic_specific);
break;
@@ -328,6 +332,9 @@ struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
case CHIP_RAVEN:
kernel_queue_init_v9(&kq->ops_asic_specific);
break;
+ case CHIP_NAVI10:
+ kernel_queue_init_v10(&kq->ops_asic_specific);
+ break;
default:
WARN(1, "Unexpected ASIC family %u",
dev->device_info->asic_family);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h
index a7116a939029..365fc674fea4 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h
@@ -102,5 +102,6 @@ struct kernel_queue {
void kernel_queue_init_cik(struct kernel_queue_ops *ops);
void kernel_queue_init_vi(struct kernel_queue_ops *ops);
void kernel_queue_init_v9(struct kernel_queue_ops *ops);
+void kernel_queue_init_v10(struct kernel_queue_ops *ops);
#endif /* KFD_KERNEL_QUEUE_H_ */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v10.c
new file mode 100644
index 000000000000..aed32ab7102e
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v10.c
@@ -0,0 +1,348 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "kfd_kernel_queue.h"
+#include "kfd_device_queue_manager.h"
+#include "kfd_pm4_headers_ai.h"
+#include "kfd_pm4_opcodes.h"
+#include "gc/gc_10_1_0_sh_mask.h"
+
+static bool initialize_v10(struct kernel_queue *kq, struct kfd_dev *dev,
+ enum kfd_queue_type type, unsigned int queue_size);
+static void uninitialize_v10(struct kernel_queue *kq);
+static void submit_packet_v10(struct kernel_queue *kq);
+
+void kernel_queue_init_v10(struct kernel_queue_ops *ops)
+{
+ ops->initialize = initialize_v10;
+ ops->uninitialize = uninitialize_v10;
+ ops->submit_packet = submit_packet_v10;
+}
+
+static bool initialize_v10(struct kernel_queue *kq, struct kfd_dev *dev,
+ enum kfd_queue_type type, unsigned int queue_size)
+{
+ int retval;
+
+ retval = kfd_gtt_sa_allocate(dev, PAGE_SIZE, &kq->eop_mem);
+ if (retval != 0)
+ return false;
+
+ kq->eop_gpu_addr = kq->eop_mem->gpu_addr;
+ kq->eop_kernel_addr = kq->eop_mem->cpu_ptr;
+
+ memset(kq->eop_kernel_addr, 0, PAGE_SIZE);
+
+ return true;
+}
+
+static void uninitialize_v10(struct kernel_queue *kq)
+{
+ kfd_gtt_sa_free(kq->dev, kq->eop_mem);
+}
+
+static void submit_packet_v10(struct kernel_queue *kq)
+{
+ *kq->wptr64_kernel = kq->pending_wptr64;
+ write_kernel_doorbell64(kq->queue->properties.doorbell_ptr,
+ kq->pending_wptr64);
+}
+
+static int pm_map_process_v10(struct packet_manager *pm,
+ uint32_t *buffer, struct qcm_process_device *qpd)
+{
+ struct pm4_mes_map_process *packet;
+ uint64_t vm_page_table_base_addr = qpd->page_table_base;
+
+ packet = (struct pm4_mes_map_process *)buffer;
+ memset(buffer, 0, sizeof(struct pm4_mes_map_process));
+
+ packet->header.u32All = pm_build_pm4_header(IT_MAP_PROCESS,
+ sizeof(struct pm4_mes_map_process));
+ packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
+ packet->bitfields2.process_quantum = 1;
+ packet->bitfields2.pasid = qpd->pqm->process->pasid;
+ packet->bitfields14.gds_size = qpd->gds_size;
+ packet->bitfields14.num_gws = qpd->num_gws;
+ packet->bitfields14.num_oac = qpd->num_oac;
+ packet->bitfields14.sdma_enable = 1;
+
+ packet->bitfields14.num_queues = (qpd->is_debug) ? 0 : qpd->queue_count;
+
+ packet->sh_mem_config = qpd->sh_mem_config;
+ packet->sh_mem_bases = qpd->sh_mem_bases;
+ if (qpd->tba_addr) {
+ packet->sq_shader_tba_lo = lower_32_bits(qpd->tba_addr >> 8);
+ packet->sq_shader_tba_hi = (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT) |
+ upper_32_bits(qpd->tba_addr >> 8);
+ packet->sq_shader_tma_lo = lower_32_bits(qpd->tma_addr >> 8);
+ packet->sq_shader_tma_hi = upper_32_bits(qpd->tma_addr >> 8);
+ }
+
+ packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);
+ packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);
+
+ packet->vm_context_page_table_base_addr_lo32 =
+ lower_32_bits(vm_page_table_base_addr);
+ packet->vm_context_page_table_base_addr_hi32 =
+ upper_32_bits(vm_page_table_base_addr);
+
+ return 0;
+}
+
+static int pm_runlist_v10(struct packet_manager *pm, uint32_t *buffer,
+ uint64_t ib, size_t ib_size_in_dwords, bool chain)
+{
+ struct pm4_mes_runlist *packet;
+
+ int concurrent_proc_cnt = 0;
+ struct kfd_dev *kfd = pm->dqm->dev;
+
+ /* Determine the number of processes to map together to HW:
+ * it can not exceed the number of VMIDs available to the
+ * scheduler, and it is determined by the smaller of the number
+ * of processes in the runlist and kfd module parameter
+ * hws_max_conc_proc.
+ * Note: the arbitration between the number of VMIDs and
+ * hws_max_conc_proc has been done in
+ * kgd2kfd_device_init().
+ */
+ concurrent_proc_cnt = min(pm->dqm->processes_count,
+ kfd->max_proc_per_quantum);
+
+
+ packet = (struct pm4_mes_runlist *)buffer;
+
+ memset(buffer, 0, sizeof(struct pm4_mes_runlist));
+ packet->header.u32All = pm_build_pm4_header(IT_RUN_LIST,
+ sizeof(struct pm4_mes_runlist));
+
+ packet->bitfields4.ib_size = ib_size_in_dwords;
+ packet->bitfields4.chain = chain ? 1 : 0;
+ packet->bitfields4.offload_polling = 0;
+ packet->bitfields4.valid = 1;
+ packet->bitfields4.process_cnt = concurrent_proc_cnt;
+ packet->ordinal2 = lower_32_bits(ib);
+ packet->ib_base_hi = upper_32_bits(ib);
+
+ return 0;
+}
+
+static int pm_map_queues_v10(struct packet_manager *pm, uint32_t *buffer,
+ struct queue *q, bool is_static)
+{
+ struct pm4_mes_map_queues *packet;
+ bool use_static = is_static;
+
+ packet = (struct pm4_mes_map_queues *)buffer;
+ memset(buffer, 0, sizeof(struct pm4_mes_map_queues));
+
+ packet->header.u32All = pm_build_pm4_header(IT_MAP_QUEUES,
+ sizeof(struct pm4_mes_map_queues));
+ packet->bitfields2.num_queues = 1;
+ packet->bitfields2.queue_sel =
+ queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi;
+
+ packet->bitfields2.engine_sel =
+ engine_sel__mes_map_queues__compute_vi;
+ packet->bitfields2.queue_type =
+ queue_type__mes_map_queues__normal_compute_vi;
+
+ switch (q->properties.type) {
+ case KFD_QUEUE_TYPE_COMPUTE:
+ if (use_static)
+ packet->bitfields2.queue_type =
+ queue_type__mes_map_queues__normal_latency_static_queue_vi;
+ break;
+ case KFD_QUEUE_TYPE_DIQ:
+ packet->bitfields2.queue_type =
+ queue_type__mes_map_queues__debug_interface_queue_vi;
+ break;
+ case KFD_QUEUE_TYPE_SDMA:
+ case KFD_QUEUE_TYPE_SDMA_XGMI:
+ packet->bitfields2.engine_sel = q->properties.sdma_engine_id +
+ engine_sel__mes_map_queues__sdma0_vi;
+ use_static = false; /* no static queues under SDMA */
+ break;
+ default:
+ WARN(1, "queue type %d\n", q->properties.type);
+ return -EINVAL;
+ }
+ packet->bitfields3.doorbell_offset =
+ q->properties.doorbell_off;
+
+ packet->mqd_addr_lo =
+ lower_32_bits(q->gart_mqd_addr);
+
+ packet->mqd_addr_hi =
+ upper_32_bits(q->gart_mqd_addr);
+
+ packet->wptr_addr_lo =
+ lower_32_bits((uint64_t)q->properties.write_ptr);
+
+ packet->wptr_addr_hi =
+ upper_32_bits((uint64_t)q->properties.write_ptr);
+
+ return 0;
+}
+
+static int pm_unmap_queues_v10(struct packet_manager *pm, uint32_t *buffer,
+ enum kfd_queue_type type,
+ enum kfd_unmap_queues_filter filter,
+ uint32_t filter_param, bool reset,
+ unsigned int sdma_engine)
+{
+ struct pm4_mes_unmap_queues *packet;
+
+ packet = (struct pm4_mes_unmap_queues *)buffer;
+ memset(buffer, 0, sizeof(struct pm4_mes_unmap_queues));
+
+ packet->header.u32All = pm_build_pm4_header(IT_UNMAP_QUEUES,
+ sizeof(struct pm4_mes_unmap_queues));
+ switch (type) {
+ case KFD_QUEUE_TYPE_COMPUTE:
+ case KFD_QUEUE_TYPE_DIQ:
+ packet->bitfields2.engine_sel =
+ engine_sel__mes_unmap_queues__compute;
+ break;
+ case KFD_QUEUE_TYPE_SDMA:
+ case KFD_QUEUE_TYPE_SDMA_XGMI:
+ packet->bitfields2.engine_sel =
+ engine_sel__mes_unmap_queues__sdma0 + sdma_engine;
+ break;
+ default:
+ WARN(1, "queue type %d\n", type);
+ break;
+ }
+
+ if (reset)
+ packet->bitfields2.action =
+ action__mes_unmap_queues__reset_queues;
+ else
+ packet->bitfields2.action =
+ action__mes_unmap_queues__preempt_queues;
+
+ switch (filter) {
+ case KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE:
+ packet->bitfields2.queue_sel =
+ queue_sel__mes_unmap_queues__perform_request_on_specified_queues;
+ packet->bitfields2.num_queues = 1;
+ packet->bitfields3b.doorbell_offset0 = filter_param;
+ break;
+ case KFD_UNMAP_QUEUES_FILTER_BY_PASID:
+ packet->bitfields2.queue_sel =
+ queue_sel__mes_unmap_queues__perform_request_on_pasid_queues;
+ packet->bitfields3a.pasid = filter_param;
+ break;
+ case KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES:
+ packet->bitfields2.queue_sel =
+ queue_sel__mes_unmap_queues__unmap_all_queues;
+ break;
+ case KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES:
+ /* in this case, we do not preempt static queues */
+ packet->bitfields2.queue_sel =
+ queue_sel__mes_unmap_queues__unmap_all_non_static_queues;
+ break;
+ default:
+ WARN(1, "filter %d\n", filter);
+ break;
+ }
+
+ return 0;
+
+}
+
+static int pm_query_status_v10(struct packet_manager *pm, uint32_t *buffer,
+ uint64_t fence_address, uint32_t fence_value)
+{
+ struct pm4_mes_query_status *packet;
+
+ packet = (struct pm4_mes_query_status *)buffer;
+ memset(buffer, 0, sizeof(struct pm4_mes_query_status));
+
+
+ packet->header.u32All = pm_build_pm4_header(IT_QUERY_STATUS,
+ sizeof(struct pm4_mes_query_status));
+
+ packet->bitfields2.context_id = 0;
+ packet->bitfields2.interrupt_sel =
+ interrupt_sel__mes_query_status__completion_status;
+ packet->bitfields2.command =
+ command__mes_query_status__fence_only_after_write_ack;
+
+ packet->addr_hi = upper_32_bits((uint64_t)fence_address);
+ packet->addr_lo = lower_32_bits((uint64_t)fence_address);
+ packet->data_hi = upper_32_bits((uint64_t)fence_value);
+ packet->data_lo = lower_32_bits((uint64_t)fence_value);
+
+ return 0;
+}
+
+
+static int pm_release_mem_v10(uint64_t gpu_addr, uint32_t *buffer)
+{
+ struct pm4_mec_release_mem *packet;
+
+ WARN_ON(!buffer);
+
+ packet = (struct pm4_mec_release_mem *)buffer;
+ memset(buffer, 0, sizeof(struct pm4_mec_release_mem));
+
+ packet->header.u32All = pm_build_pm4_header(IT_RELEASE_MEM,
+ sizeof(struct pm4_mec_release_mem));
+
+ packet->bitfields2.event_type = CACHE_FLUSH_AND_INV_TS_EVENT;
+ packet->bitfields2.event_index = event_index__mec_release_mem__end_of_pipe;
+ packet->bitfields2.tcl1_action_ena = 1;
+ packet->bitfields2.tc_action_ena = 1;
+ packet->bitfields2.cache_policy = cache_policy__mec_release_mem__lru;
+
+ packet->bitfields3.data_sel = data_sel__mec_release_mem__send_32_bit_low;
+ packet->bitfields3.int_sel =
+ int_sel__mec_release_mem__send_interrupt_after_write_confirm;
+
+ packet->bitfields4.address_lo_32b = (gpu_addr & 0xffffffff) >> 2;
+ packet->address_hi = upper_32_bits(gpu_addr);
+
+ packet->data_lo = 0;
+
+ return sizeof(struct pm4_mec_release_mem) / sizeof(unsigned int);
+}
+
+const struct packet_manager_funcs kfd_v10_pm_funcs = {
+ .map_process = pm_map_process_v10,
+ .runlist = pm_runlist_v10,
+ .set_resources = pm_set_resources_vi,
+ .map_queues = pm_map_queues_v10,
+ .unmap_queues = pm_unmap_queues_v10,
+ .query_status = pm_query_status_v10,
+ .release_mem = pm_release_mem_v10,
+ .map_process_size = sizeof(struct pm4_mes_map_process),
+ .runlist_size = sizeof(struct pm4_mes_runlist),
+ .set_resources_size = sizeof(struct pm4_mes_set_resources),
+ .map_queues_size = sizeof(struct pm4_mes_map_queues),
+ .unmap_queues_size = sizeof(struct pm4_mes_unmap_queues),
+ .query_status_size = sizeof(struct pm4_mes_query_status),
+ .release_mem_size = sizeof(struct pm4_mec_release_mem)
+};
+
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c
index 33830b1a5a54..2d5ddf199bd0 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c
@@ -134,6 +134,7 @@ static int pm_runlist_v9(struct packet_manager *pm, uint32_t *buffer,
packet->bitfields4.ib_size = ib_size_in_dwords;
packet->bitfields4.chain = chain ? 1 : 0;
packet->bitfields4.offload_polling = 0;
+ packet->bitfields4.chained_runlist_idle_disable = chain ? 1 : 0;
packet->bitfields4.valid = 1;
packet->bitfields4.process_cnt = concurrent_proc_cnt;
packet->ordinal2 = lower_32_bits(ib);
@@ -153,14 +154,13 @@ static int pm_map_queues_v9(struct packet_manager *pm, uint32_t *buffer,
packet->header.u32All = pm_build_pm4_header(IT_MAP_QUEUES,
sizeof(struct pm4_mes_map_queues));
- packet->bitfields2.alloc_format =
- alloc_format__mes_map_queues__one_per_pipe_vi;
packet->bitfields2.num_queues = 1;
packet->bitfields2.queue_sel =
queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi;
packet->bitfields2.engine_sel =
engine_sel__mes_map_queues__compute_vi;
+ packet->bitfields2.gws_control_queue = q->gws ? 1 : 0;
packet->bitfields2.queue_type =
queue_type__mes_map_queues__normal_compute_vi;
@@ -175,6 +175,7 @@ static int pm_map_queues_v9(struct packet_manager *pm, uint32_t *buffer,
queue_type__mes_map_queues__debug_interface_queue_vi;
break;
case KFD_QUEUE_TYPE_SDMA:
+ case KFD_QUEUE_TYPE_SDMA_XGMI:
packet->bitfields2.engine_sel = q->properties.sdma_engine_id +
engine_sel__mes_map_queues__sdma0_vi;
use_static = false; /* no static queues under SDMA */
@@ -221,6 +222,7 @@ static int pm_unmap_queues_v9(struct packet_manager *pm, uint32_t *buffer,
engine_sel__mes_unmap_queues__compute;
break;
case KFD_QUEUE_TYPE_SDMA:
+ case KFD_QUEUE_TYPE_SDMA_XGMI:
packet->bitfields2.engine_sel =
engine_sel__mes_unmap_queues__sdma0 + sdma_engine;
break;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c
index bf20c6d32ef3..2adaf40027eb 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c
@@ -190,8 +190,6 @@ static int pm_map_queues_vi(struct packet_manager *pm, uint32_t *buffer,
packet->header.u32All = pm_build_pm4_header(IT_MAP_QUEUES,
sizeof(struct pm4_mes_map_queues));
- packet->bitfields2.alloc_format =
- alloc_format__mes_map_queues__one_per_pipe_vi;
packet->bitfields2.num_queues = 1;
packet->bitfields2.queue_sel =
queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi;
@@ -212,6 +210,7 @@ static int pm_map_queues_vi(struct packet_manager *pm, uint32_t *buffer,
queue_type__mes_map_queues__debug_interface_queue_vi;
break;
case KFD_QUEUE_TYPE_SDMA:
+ case KFD_QUEUE_TYPE_SDMA_XGMI:
packet->bitfields2.engine_sel = q->properties.sdma_engine_id +
engine_sel__mes_map_queues__sdma0_vi;
use_static = false; /* no static queues under SDMA */
@@ -258,6 +257,7 @@ static int pm_unmap_queues_vi(struct packet_manager *pm, uint32_t *buffer,
engine_sel__mes_unmap_queues__compute;
break;
case KFD_QUEUE_TYPE_SDMA:
+ case KFD_QUEUE_TYPE_SDMA_XGMI:
packet->bitfields2.engine_sel =
engine_sel__mes_unmap_queues__sdma0 + sdma_engine;
break;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_module.c b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
index 932007eb9168..986ff52d5750 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_module.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
@@ -56,6 +56,11 @@ static int kfd_init(void)
if (err < 0)
goto err_create_wq;
+ /* Ignore the return value, so that we can continue
+ * to init the KFD, even if procfs isn't craated
+ */
+ kfd_procfs_init();
+
kfd_debugfs_init();
return 0;
@@ -72,6 +77,7 @@ static void kfd_exit(void)
{
kfd_debugfs_fini();
kfd_process_destroy_wq();
+ kfd_procfs_shutdown();
kfd_topology_shutdown();
kfd_chardev_exit();
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
index aed9b9b82213..d6cf391da591 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
@@ -23,34 +23,74 @@
#include "kfd_mqd_manager.h"
#include "amdgpu_amdkfd.h"
+#include "kfd_device_queue_manager.h"
-struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev)
+/* Mapping queue priority to pipe priority, indexed by queue priority */
+int pipe_priority_map[] = {
+ KFD_PIPE_PRIORITY_CS_LOW,
+ KFD_PIPE_PRIORITY_CS_LOW,
+ KFD_PIPE_PRIORITY_CS_LOW,
+ KFD_PIPE_PRIORITY_CS_LOW,
+ KFD_PIPE_PRIORITY_CS_LOW,
+ KFD_PIPE_PRIORITY_CS_LOW,
+ KFD_PIPE_PRIORITY_CS_LOW,
+ KFD_PIPE_PRIORITY_CS_MEDIUM,
+ KFD_PIPE_PRIORITY_CS_MEDIUM,
+ KFD_PIPE_PRIORITY_CS_MEDIUM,
+ KFD_PIPE_PRIORITY_CS_MEDIUM,
+ KFD_PIPE_PRIORITY_CS_HIGH,
+ KFD_PIPE_PRIORITY_CS_HIGH,
+ KFD_PIPE_PRIORITY_CS_HIGH,
+ KFD_PIPE_PRIORITY_CS_HIGH,
+ KFD_PIPE_PRIORITY_CS_HIGH
+};
+
+struct kfd_mem_obj *allocate_hiq_mqd(struct kfd_dev *dev, struct queue_properties *q)
{
- switch (dev->device_info->asic_family) {
- case CHIP_KAVERI:
- return mqd_manager_init_cik(type, dev);
- case CHIP_HAWAII:
- return mqd_manager_init_cik_hawaii(type, dev);
- case CHIP_CARRIZO:
- return mqd_manager_init_vi(type, dev);
- case CHIP_TONGA:
- case CHIP_FIJI:
- case CHIP_POLARIS10:
- case CHIP_POLARIS11:
- case CHIP_POLARIS12:
- return mqd_manager_init_vi_tonga(type, dev);
- case CHIP_VEGA10:
- case CHIP_VEGA12:
- case CHIP_VEGA20:
- case CHIP_RAVEN:
- return mqd_manager_init_v9(type, dev);
- default:
- WARN(1, "Unexpected ASIC family %u",
- dev->device_info->asic_family);
- }
+ struct kfd_mem_obj *mqd_mem_obj = NULL;
+
+ mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
+ if (!mqd_mem_obj)
+ return NULL;
+
+ mqd_mem_obj->gtt_mem = dev->dqm->hiq_sdma_mqd.gtt_mem;
+ mqd_mem_obj->gpu_addr = dev->dqm->hiq_sdma_mqd.gpu_addr;
+ mqd_mem_obj->cpu_ptr = dev->dqm->hiq_sdma_mqd.cpu_ptr;
+
+ return mqd_mem_obj;
+}
+
+struct kfd_mem_obj *allocate_sdma_mqd(struct kfd_dev *dev,
+ struct queue_properties *q)
+{
+ struct kfd_mem_obj *mqd_mem_obj = NULL;
+ uint64_t offset;
- return NULL;
+ mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
+ if (!mqd_mem_obj)
+ return NULL;
+
+ offset = (q->sdma_engine_id *
+ dev->device_info->num_sdma_queues_per_engine +
+ q->sdma_queue_id) *
+ dev->dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size;
+
+ offset += dev->dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size;
+
+ mqd_mem_obj->gtt_mem = (void *)((uint64_t)dev->dqm->hiq_sdma_mqd.gtt_mem
+ + offset);
+ mqd_mem_obj->gpu_addr = dev->dqm->hiq_sdma_mqd.gpu_addr + offset;
+ mqd_mem_obj->cpu_ptr = (uint32_t *)((uint64_t)
+ dev->dqm->hiq_sdma_mqd.cpu_ptr + offset);
+
+ return mqd_mem_obj;
+}
+
+void free_mqd_hiq_sdma(struct mqd_manager *mm, void *mqd,
+ struct kfd_mem_obj *mqd_mem_obj)
+{
+ WARN_ON(!mqd_mem_obj->gtt_mem);
+ kfree(mqd_mem_obj);
}
void mqd_symmetrically_map_cu_mask(struct mqd_manager *mm,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
index f8261313ae7b..550b61e81015 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
@@ -39,7 +39,7 @@
* @destroy_mqd: Destroys the HQD slot and by that preempt the relevant queue.
* Used only for no cp scheduling.
*
- * @uninit_mqd: Releases the mqd buffer from local gpu memory.
+ * @free_mqd: Releases the mqd buffer from local gpu memory.
*
* @is_occupied: Checks if the relevant HQD slot is occupied.
*
@@ -62,10 +62,13 @@
* per KFD_MQD_TYPE for each device.
*
*/
-
+extern int pipe_priority_map[];
struct mqd_manager {
- int (*init_mqd)(struct mqd_manager *mm, void **mqd,
- struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+ struct kfd_mem_obj* (*allocate_mqd)(struct kfd_dev *kfd,
+ struct queue_properties *q);
+
+ void (*init_mqd)(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *q);
int (*load_mqd)(struct mqd_manager *mm, void *mqd,
@@ -73,7 +76,7 @@ struct mqd_manager {
struct queue_properties *p,
struct mm_struct *mms);
- int (*update_mqd)(struct mqd_manager *mm, void *mqd,
+ void (*update_mqd)(struct mqd_manager *mm, void *mqd,
struct queue_properties *q);
int (*destroy_mqd)(struct mqd_manager *mm, void *mqd,
@@ -81,7 +84,7 @@ struct mqd_manager {
unsigned int timeout, uint32_t pipe_id,
uint32_t queue_id);
- void (*uninit_mqd)(struct mqd_manager *mm, void *mqd,
+ void (*free_mqd)(struct mqd_manager *mm, void *mqd,
struct kfd_mem_obj *mqd_mem_obj);
bool (*is_occupied)(struct mqd_manager *mm, void *mqd,
@@ -99,8 +102,17 @@ struct mqd_manager {
struct mutex mqd_mutex;
struct kfd_dev *dev;
+ uint32_t mqd_size;
};
+struct kfd_mem_obj *allocate_hiq_mqd(struct kfd_dev *dev,
+ struct queue_properties *q);
+
+struct kfd_mem_obj *allocate_sdma_mqd(struct kfd_dev *dev,
+ struct queue_properties *q);
+void free_mqd_hiq_sdma(struct mqd_manager *mm, void *mqd,
+ struct kfd_mem_obj *mqd_mem_obj);
+
void mqd_symmetrically_map_cu_mask(struct mqd_manager *mm,
const uint32_t *cu_mask, uint32_t cu_mask_count,
uint32_t *se_mask);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
index ae90a99909ef..28876aceb14b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
@@ -66,22 +66,33 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
m->compute_static_thread_mgmt_se3);
}
-static int init_mqd(struct mqd_manager *mm, void **mqd,
- struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+static void set_priority(struct cik_mqd *m, struct queue_properties *q)
+{
+ m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
+ m->cp_hqd_queue_priority = q->priority;
+}
+
+static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
+ struct queue_properties *q)
+{
+ struct kfd_mem_obj *mqd_mem_obj;
+
+ if (kfd_gtt_sa_allocate(kfd, sizeof(struct cik_mqd),
+ &mqd_mem_obj))
+ return NULL;
+
+ return mqd_mem_obj;
+}
+
+static void init_mqd(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *q)
{
uint64_t addr;
struct cik_mqd *m;
- int retval;
-
- retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd),
- mqd_mem_obj);
- if (retval != 0)
- return -ENOMEM;
-
- m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr;
- addr = (*mqd_mem_obj)->gpu_addr;
+ m = (struct cik_mqd *) mqd_mem_obj->cpu_ptr;
+ addr = mqd_mem_obj->gpu_addr;
memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256));
@@ -116,8 +127,7 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
* 1 = CS_MEDIUM (typically between HP3D and GFX
* 2 = CS_HIGH (typically above HP3D)
*/
- m->cp_hqd_pipe_priority = 1;
- m->cp_hqd_queue_priority = 15;
+ set_priority(m, q);
if (q->format == KFD_QUEUE_FORMAT_AQL)
m->cp_hqd_iq_rptr = AQL_ENABLE;
@@ -125,49 +135,32 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
*mqd = m;
if (gart_addr)
*gart_addr = addr;
- retval = mm->update_mqd(mm, m, q);
-
- return retval;
+ mm->update_mqd(mm, m, q);
}
-static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
- struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *q)
{
- int retval;
struct cik_sdma_rlc_registers *m;
- retval = kfd_gtt_sa_allocate(mm->dev,
- sizeof(struct cik_sdma_rlc_registers),
- mqd_mem_obj);
-
- if (retval != 0)
- return -ENOMEM;
-
- m = (struct cik_sdma_rlc_registers *) (*mqd_mem_obj)->cpu_ptr;
+ m = (struct cik_sdma_rlc_registers *) mqd_mem_obj->cpu_ptr;
memset(m, 0, sizeof(struct cik_sdma_rlc_registers));
*mqd = m;
if (gart_addr)
- *gart_addr = (*mqd_mem_obj)->gpu_addr;
+ *gart_addr = mqd_mem_obj->gpu_addr;
- retval = mm->update_mqd(mm, m, q);
-
- return retval;
+ mm->update_mqd(mm, m, q);
}
-static void uninit_mqd(struct mqd_manager *mm, void *mqd,
+static void free_mqd(struct mqd_manager *mm, void *mqd,
struct kfd_mem_obj *mqd_mem_obj)
{
kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
}
-static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd,
- struct kfd_mem_obj *mqd_mem_obj)
-{
- kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
-}
static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id,
uint32_t queue_id, struct queue_properties *p,
@@ -191,7 +184,7 @@ static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
mms);
}
-static int __update_mqd(struct mqd_manager *mm, void *mqd,
+static void __update_mqd(struct mqd_manager *mm, void *mqd,
struct queue_properties *q, unsigned int atc_bit)
{
struct cik_mqd *m;
@@ -222,28 +215,24 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
m->cp_hqd_pq_control |= NO_UPDATE_RPTR;
update_cu_mask(mm, mqd, q);
+ set_priority(m, q);
- q->is_active = (q->queue_size > 0 &&
- q->queue_address != 0 &&
- q->queue_percent > 0 &&
- !q->is_evicted);
-
- return 0;
+ q->is_active = QUEUE_IS_ACTIVE(*q);
}
-static int update_mqd(struct mqd_manager *mm, void *mqd,
+static void update_mqd(struct mqd_manager *mm, void *mqd,
struct queue_properties *q)
{
- return __update_mqd(mm, mqd, q, 1);
+ __update_mqd(mm, mqd, q, 1);
}
-static int update_mqd_hawaii(struct mqd_manager *mm, void *mqd,
+static void update_mqd_hawaii(struct mqd_manager *mm, void *mqd,
struct queue_properties *q)
{
- return __update_mqd(mm, mqd, q, 0);
+ __update_mqd(mm, mqd, q, 0);
}
-static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
+static void update_mqd_sdma(struct mqd_manager *mm, void *mqd,
struct queue_properties *q)
{
struct cik_sdma_rlc_registers *m;
@@ -267,12 +256,7 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
m->sdma_engine_id = q->sdma_engine_id;
m->sdma_queue_id = q->sdma_queue_id;
- q->is_active = (q->queue_size > 0 &&
- q->queue_address != 0 &&
- q->queue_percent > 0 &&
- !q->is_evicted);
-
- return 0;
+ q->is_active = QUEUE_IS_ACTIVE(*q);
}
static int destroy_mqd(struct mqd_manager *mm, void *mqd,
@@ -319,14 +303,14 @@ static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
* queues but with different initial values.
*/
-static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
- struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *q)
{
- return init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
+ init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
}
-static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
+static void update_mqd_hiq(struct mqd_manager *mm, void *mqd,
struct queue_properties *q)
{
struct cik_mqd *m;
@@ -350,12 +334,9 @@ static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
m->cp_hqd_vmid = q->vmid;
- q->is_active = (q->queue_size > 0 &&
- q->queue_address != 0 &&
- q->queue_percent > 0 &&
- !q->is_evicted);
+ q->is_active = QUEUE_IS_ACTIVE(*q);
- return 0;
+ set_priority(m, q);
}
#if defined(CONFIG_DEBUG_FS)
@@ -394,34 +375,53 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
switch (type) {
case KFD_MQD_TYPE_CP:
case KFD_MQD_TYPE_COMPUTE:
+ mqd->allocate_mqd = allocate_mqd;
mqd->init_mqd = init_mqd;
- mqd->uninit_mqd = uninit_mqd;
+ mqd->free_mqd = free_mqd;
mqd->load_mqd = load_mqd;
mqd->update_mqd = update_mqd;
mqd->destroy_mqd = destroy_mqd;
mqd->is_occupied = is_occupied;
+ mqd->mqd_size = sizeof(struct cik_mqd);
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd;
#endif
break;
case KFD_MQD_TYPE_HIQ:
+ mqd->allocate_mqd = allocate_hiq_mqd;
+ mqd->init_mqd = init_mqd_hiq;
+ mqd->free_mqd = free_mqd_hiq_sdma;
+ mqd->load_mqd = load_mqd;
+ mqd->update_mqd = update_mqd_hiq;
+ mqd->destroy_mqd = destroy_mqd;
+ mqd->is_occupied = is_occupied;
+ mqd->mqd_size = sizeof(struct cik_mqd);
+#if defined(CONFIG_DEBUG_FS)
+ mqd->debugfs_show_mqd = debugfs_show_mqd;
+#endif
+ break;
+ case KFD_MQD_TYPE_DIQ:
+ mqd->allocate_mqd = allocate_hiq_mqd;
mqd->init_mqd = init_mqd_hiq;
- mqd->uninit_mqd = uninit_mqd;
+ mqd->free_mqd = free_mqd;
mqd->load_mqd = load_mqd;
mqd->update_mqd = update_mqd_hiq;
mqd->destroy_mqd = destroy_mqd;
mqd->is_occupied = is_occupied;
+ mqd->mqd_size = sizeof(struct cik_mqd);
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd;
#endif
break;
case KFD_MQD_TYPE_SDMA:
+ mqd->allocate_mqd = allocate_sdma_mqd;
mqd->init_mqd = init_mqd_sdma;
- mqd->uninit_mqd = uninit_mqd_sdma;
+ mqd->free_mqd = free_mqd_hiq_sdma;
mqd->load_mqd = load_mqd_sdma;
mqd->update_mqd = update_mqd_sdma;
mqd->destroy_mqd = destroy_mqd_sdma;
mqd->is_occupied = is_occupied_sdma;
+ mqd->mqd_size = sizeof(struct cik_sdma_rlc_registers);
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
new file mode 100644
index 000000000000..9cd3eb2d90bd
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
@@ -0,0 +1,497 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/printk.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include "kfd_priv.h"
+#include "kfd_mqd_manager.h"
+#include "v10_structs.h"
+#include "gc/gc_10_1_0_offset.h"
+#include "gc/gc_10_1_0_sh_mask.h"
+#include "amdgpu_amdkfd.h"
+
+static inline struct v10_compute_mqd *get_mqd(void *mqd)
+{
+ return (struct v10_compute_mqd *)mqd;
+}
+
+static inline struct v10_sdma_mqd *get_sdma_mqd(void *mqd)
+{
+ return (struct v10_sdma_mqd *)mqd;
+}
+
+static void update_cu_mask(struct mqd_manager *mm, void *mqd,
+ struct queue_properties *q)
+{
+ struct v10_compute_mqd *m;
+ uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */
+
+ if (q->cu_mask_count == 0)
+ return;
+
+ mqd_symmetrically_map_cu_mask(mm,
+ q->cu_mask, q->cu_mask_count, se_mask);
+
+ m = get_mqd(mqd);
+ m->compute_static_thread_mgmt_se0 = se_mask[0];
+ m->compute_static_thread_mgmt_se1 = se_mask[1];
+ m->compute_static_thread_mgmt_se2 = se_mask[2];
+ m->compute_static_thread_mgmt_se3 = se_mask[3];
+
+ pr_debug("update cu mask to %#x %#x %#x %#x\n",
+ m->compute_static_thread_mgmt_se0,
+ m->compute_static_thread_mgmt_se1,
+ m->compute_static_thread_mgmt_se2,
+ m->compute_static_thread_mgmt_se3);
+}
+
+static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
+ struct queue_properties *q)
+{
+ int retval;
+ struct kfd_mem_obj *mqd_mem_obj = NULL;
+
+ /* From V9, for CWSR, the control stack is located on the next page
+ * boundary after the mqd, we will use the gtt allocation function
+ * instead of sub-allocation function.
+ */
+ if (kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {
+ mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_NOIO);
+ if (!mqd_mem_obj)
+ return NULL;
+ retval = amdgpu_amdkfd_alloc_gtt_mem(kfd->kgd,
+ ALIGN(q->ctl_stack_size, PAGE_SIZE) +
+ ALIGN(sizeof(struct v10_compute_mqd), PAGE_SIZE),
+ &(mqd_mem_obj->gtt_mem),
+ &(mqd_mem_obj->gpu_addr),
+ (void *)&(mqd_mem_obj->cpu_ptr), true);
+ } else {
+ retval = kfd_gtt_sa_allocate(kfd, sizeof(struct v10_compute_mqd),
+ &mqd_mem_obj);
+ }
+
+ if (retval) {
+ kfree(mqd_mem_obj);
+ return NULL;
+ }
+
+ return mqd_mem_obj;
+
+}
+
+static void init_mqd(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+ struct queue_properties *q)
+{
+ uint64_t addr;
+ struct v10_compute_mqd *m;
+
+ m = (struct v10_compute_mqd *) mqd_mem_obj->cpu_ptr;
+ addr = mqd_mem_obj->gpu_addr;
+
+ memset(m, 0, sizeof(struct v10_compute_mqd));
+
+ m->header = 0xC0310800;
+ m->compute_pipelinestat_enable = 1;
+ m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
+ m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
+ m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
+ m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
+
+ m->cp_hqd_persistent_state = CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK |
+ 0x53 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT;
+
+ m->cp_mqd_control = 1 << CP_MQD_CONTROL__PRIV_STATE__SHIFT;
+
+ m->cp_mqd_base_addr_lo = lower_32_bits(addr);
+ m->cp_mqd_base_addr_hi = upper_32_bits(addr);
+
+ m->cp_hqd_quantum = 1 << CP_HQD_QUANTUM__QUANTUM_EN__SHIFT |
+ 1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT |
+ 10 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT;
+
+ m->cp_hqd_pipe_priority = 1;
+ m->cp_hqd_queue_priority = 15;
+
+ if (q->format == KFD_QUEUE_FORMAT_AQL) {
+ m->cp_hqd_aql_control =
+ 1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
+ }
+
+ if (mm->dev->cwsr_enabled) {
+ m->cp_hqd_persistent_state |=
+ (1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT);
+ m->cp_hqd_ctx_save_base_addr_lo =
+ lower_32_bits(q->ctx_save_restore_area_address);
+ m->cp_hqd_ctx_save_base_addr_hi =
+ upper_32_bits(q->ctx_save_restore_area_address);
+ m->cp_hqd_ctx_save_size = q->ctx_save_restore_area_size;
+ m->cp_hqd_cntl_stack_size = q->ctl_stack_size;
+ m->cp_hqd_cntl_stack_offset = q->ctl_stack_size;
+ m->cp_hqd_wg_state_offset = q->ctl_stack_size;
+ }
+
+ *mqd = m;
+ if (gart_addr)
+ *gart_addr = addr;
+ mm->update_mqd(mm, m, q);
+}
+
+static int load_mqd(struct mqd_manager *mm, void *mqd,
+ uint32_t pipe_id, uint32_t queue_id,
+ struct queue_properties *p, struct mm_struct *mms)
+{
+ int r = 0;
+ /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
+ uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
+
+ r = mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id,
+ (uint32_t __user *)p->write_ptr,
+ wptr_shift, 0, mms);
+ return r;
+}
+
+static void update_mqd(struct mqd_manager *mm, void *mqd,
+ struct queue_properties *q)
+{
+ struct v10_compute_mqd *m;
+
+ m = get_mqd(mqd);
+
+ m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT;
+ m->cp_hqd_pq_control |=
+ ffs(q->queue_size / sizeof(unsigned int)) - 1 - 1;
+ pr_debug("cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control);
+
+ m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
+ m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
+
+ m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
+ m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+ m->cp_hqd_pq_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr);
+ m->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr);
+
+ m->cp_hqd_pq_doorbell_control =
+ q->doorbell_off <<
+ CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
+ pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
+ m->cp_hqd_pq_doorbell_control);
+
+ m->cp_hqd_ib_control = 3 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT;
+
+ /*
+ * HW does not clamp this field correctly. Maximum EOP queue size
+ * is constrained by per-SE EOP done signal count, which is 8-bit.
+ * Limit is 0xFF EOP entries (= 0x7F8 dwords). CP will not submit
+ * more than (EOP entry count - 1) so a queue size of 0x800 dwords
+ * is safe, giving a maximum field value of 0xA.
+ */
+ m->cp_hqd_eop_control = min(0xA,
+ ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1);
+ m->cp_hqd_eop_base_addr_lo =
+ lower_32_bits(q->eop_ring_buffer_address >> 8);
+ m->cp_hqd_eop_base_addr_hi =
+ upper_32_bits(q->eop_ring_buffer_address >> 8);
+
+ m->cp_hqd_iq_timer = 0;
+
+ m->cp_hqd_vmid = q->vmid;
+
+ if (q->format == KFD_QUEUE_FORMAT_AQL) {
+ /* GC 10 removed WPP_CLAMP from PQ Control */
+ m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK |
+ 2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT |
+ 1 << CP_HQD_PQ_CONTROL__QUEUE_FULL_EN__SHIFT ;
+ m->cp_hqd_pq_doorbell_control |=
+ 1 << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT;
+ }
+ if (mm->dev->cwsr_enabled)
+ m->cp_hqd_ctx_save_control = 0;
+
+ update_cu_mask(mm, mqd, q);
+
+ q->is_active = (q->queue_size > 0 &&
+ q->queue_address != 0 &&
+ q->queue_percent > 0 &&
+ !q->is_evicted);
+}
+
+static int destroy_mqd(struct mqd_manager *mm, void *mqd,
+ enum kfd_preempt_type type,
+ unsigned int timeout, uint32_t pipe_id,
+ uint32_t queue_id)
+{
+ return mm->dev->kfd2kgd->hqd_destroy
+ (mm->dev->kgd, mqd, type, timeout,
+ pipe_id, queue_id);
+}
+
+static void free_mqd(struct mqd_manager *mm, void *mqd,
+ struct kfd_mem_obj *mqd_mem_obj)
+{
+ struct kfd_dev *kfd = mm->dev;
+
+ if (mqd_mem_obj->gtt_mem) {
+ amdgpu_amdkfd_free_gtt_mem(kfd->kgd, mqd_mem_obj->gtt_mem);
+ kfree(mqd_mem_obj);
+ } else {
+ kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
+ }
+}
+
+static bool is_occupied(struct mqd_manager *mm, void *mqd,
+ uint64_t queue_address, uint32_t pipe_id,
+ uint32_t queue_id)
+{
+ return mm->dev->kfd2kgd->hqd_is_occupied(
+ mm->dev->kgd, queue_address,
+ pipe_id, queue_id);
+}
+
+static int get_wave_state(struct mqd_manager *mm, void *mqd,
+ void __user *ctl_stack,
+ u32 *ctl_stack_used_size,
+ u32 *save_area_used_size)
+{
+ struct v10_compute_mqd *m;
+
+ /* Control stack is located one page after MQD. */
+ void *mqd_ctl_stack = (void *)((uintptr_t)mqd + PAGE_SIZE);
+
+ m = get_mqd(mqd);
+
+ *ctl_stack_used_size = m->cp_hqd_cntl_stack_size -
+ m->cp_hqd_cntl_stack_offset;
+ *save_area_used_size = m->cp_hqd_wg_state_offset -
+ m->cp_hqd_cntl_stack_size;
+
+ if (copy_to_user(ctl_stack, mqd_ctl_stack, m->cp_hqd_cntl_stack_size))
+ return -EFAULT;
+
+ return 0;
+}
+
+static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+ struct queue_properties *q)
+{
+ struct v10_compute_mqd *m;
+
+ init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
+
+ m = get_mqd(*mqd);
+
+ m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT |
+ 1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;
+}
+
+static void update_mqd_hiq(struct mqd_manager *mm, void *mqd,
+ struct queue_properties *q)
+{
+ struct v10_compute_mqd *m;
+
+ update_mqd(mm, mqd, q);
+
+ /* TODO: what's the point? update_mqd already does this. */
+ m = get_mqd(mqd);
+ m->cp_hqd_vmid = q->vmid;
+}
+
+static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+ struct queue_properties *q)
+{
+ struct v10_sdma_mqd *m;
+
+ m = (struct v10_sdma_mqd *) mqd_mem_obj->cpu_ptr;
+
+ memset(m, 0, sizeof(struct v10_sdma_mqd));
+
+ *mqd = m;
+ if (gart_addr)
+ *gart_addr = mqd_mem_obj->gpu_addr;
+
+ mm->update_mqd(mm, m, q);
+}
+
+static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
+ uint32_t pipe_id, uint32_t queue_id,
+ struct queue_properties *p, struct mm_struct *mms)
+{
+ return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd,
+ (uint32_t __user *)p->write_ptr,
+ mms);
+}
+
+#define SDMA_RLC_DUMMY_DEFAULT 0xf
+
+static void update_mqd_sdma(struct mqd_manager *mm, void *mqd,
+ struct queue_properties *q)
+{
+ struct v10_sdma_mqd *m;
+
+ m = get_sdma_mqd(mqd);
+ m->sdmax_rlcx_rb_cntl = (ffs(q->queue_size / sizeof(unsigned int)) - 1)
+ << SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
+ q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT |
+ 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
+ 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT;
+
+ m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8);
+ m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8);
+ m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
+ m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+ m->sdmax_rlcx_doorbell_offset =
+ q->doorbell_off << SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT;
+
+ m->sdma_engine_id = q->sdma_engine_id;
+ m->sdma_queue_id = q->sdma_queue_id;
+ m->sdmax_rlcx_dummy_reg = SDMA_RLC_DUMMY_DEFAULT;
+
+
+ q->is_active = (q->queue_size > 0 &&
+ q->queue_address != 0 &&
+ q->queue_percent > 0 &&
+ !q->is_evicted);
+}
+
+/*
+ * * preempt type here is ignored because there is only one way
+ * * to preempt sdma queue
+ */
+static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
+ enum kfd_preempt_type type,
+ unsigned int timeout, uint32_t pipe_id,
+ uint32_t queue_id)
+{
+ return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout);
+}
+
+static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
+ uint64_t queue_address, uint32_t pipe_id,
+ uint32_t queue_id)
+{
+ return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
+}
+
+#if defined(CONFIG_DEBUG_FS)
+
+static int debugfs_show_mqd(struct seq_file *m, void *data)
+{
+ seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4,
+ data, sizeof(struct v10_compute_mqd), false);
+ return 0;
+}
+
+static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
+{
+ seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4,
+ data, sizeof(struct v10_sdma_mqd), false);
+ return 0;
+}
+
+#endif
+
+struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
+ struct kfd_dev *dev)
+{
+ struct mqd_manager *mqd;
+
+ if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
+ return NULL;
+
+ mqd = kzalloc(sizeof(*mqd), GFP_NOIO);
+ if (!mqd)
+ return NULL;
+
+ mqd->dev = dev;
+
+ switch (type) {
+ case KFD_MQD_TYPE_CP:
+ case KFD_MQD_TYPE_COMPUTE:
+ pr_debug("%s@%i\n", __func__, __LINE__);
+ mqd->allocate_mqd = allocate_mqd;
+ mqd->init_mqd = init_mqd;
+ mqd->free_mqd = free_mqd;
+ mqd->load_mqd = load_mqd;
+ mqd->update_mqd = update_mqd;
+ mqd->destroy_mqd = destroy_mqd;
+ mqd->is_occupied = is_occupied;
+ mqd->mqd_size = sizeof(struct v10_compute_mqd);
+ mqd->get_wave_state = get_wave_state;
+#if defined(CONFIG_DEBUG_FS)
+ mqd->debugfs_show_mqd = debugfs_show_mqd;
+#endif
+ pr_debug("%s@%i\n", __func__, __LINE__);
+ break;
+ case KFD_MQD_TYPE_HIQ:
+ pr_debug("%s@%i\n", __func__, __LINE__);
+ mqd->allocate_mqd = allocate_hiq_mqd;
+ mqd->init_mqd = init_mqd_hiq;
+ mqd->free_mqd = free_mqd_hiq_sdma;
+ mqd->load_mqd = load_mqd;
+ mqd->update_mqd = update_mqd_hiq;
+ mqd->destroy_mqd = destroy_mqd;
+ mqd->is_occupied = is_occupied;
+ mqd->mqd_size = sizeof(struct v10_compute_mqd);
+#if defined(CONFIG_DEBUG_FS)
+ mqd->debugfs_show_mqd = debugfs_show_mqd;
+#endif
+ pr_debug("%s@%i\n", __func__, __LINE__);
+ break;
+ case KFD_MQD_TYPE_DIQ:
+ mqd->allocate_mqd = allocate_hiq_mqd;
+ mqd->init_mqd = init_mqd_hiq;
+ mqd->free_mqd = free_mqd;
+ mqd->load_mqd = load_mqd;
+ mqd->update_mqd = update_mqd_hiq;
+ mqd->destroy_mqd = destroy_mqd;
+ mqd->is_occupied = is_occupied;
+ mqd->mqd_size = sizeof(struct v10_compute_mqd);
+#if defined(CONFIG_DEBUG_FS)
+ mqd->debugfs_show_mqd = debugfs_show_mqd;
+#endif
+ break;
+ case KFD_MQD_TYPE_SDMA:
+ pr_debug("%s@%i\n", __func__, __LINE__);
+ mqd->allocate_mqd = allocate_sdma_mqd;
+ mqd->init_mqd = init_mqd_sdma;
+ mqd->free_mqd = free_mqd_hiq_sdma;
+ mqd->load_mqd = load_mqd_sdma;
+ mqd->update_mqd = update_mqd_sdma;
+ mqd->destroy_mqd = destroy_mqd_sdma;
+ mqd->is_occupied = is_occupied_sdma;
+ mqd->mqd_size = sizeof(struct v10_sdma_mqd);
+#if defined(CONFIG_DEBUG_FS)
+ mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
+#endif
+ pr_debug("%s@%i\n", __func__, __LINE__);
+ break;
+ default:
+ kfree(mqd);
+ return NULL;
+ }
+
+ return mqd;
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index 9dbba609450e..0c58f91b3ff3 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -67,37 +67,55 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
m->compute_static_thread_mgmt_se3);
}
-static int init_mqd(struct mqd_manager *mm, void **mqd,
- struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
- struct queue_properties *q)
+static void set_priority(struct v9_mqd *m, struct queue_properties *q)
+{
+ m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
+ m->cp_hqd_queue_priority = q->priority;
+}
+
+static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
+ struct queue_properties *q)
{
int retval;
- uint64_t addr;
- struct v9_mqd *m;
- struct kfd_dev *kfd = mm->dev;
+ struct kfd_mem_obj *mqd_mem_obj = NULL;
/* From V9, for CWSR, the control stack is located on the next page
* boundary after the mqd, we will use the gtt allocation function
* instead of sub-allocation function.
*/
if (kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {
- *mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
- if (!*mqd_mem_obj)
- return -ENOMEM;
+ mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_NOIO);
+ if (!mqd_mem_obj)
+ return NULL;
retval = amdgpu_amdkfd_alloc_gtt_mem(kfd->kgd,
ALIGN(q->ctl_stack_size, PAGE_SIZE) +
ALIGN(sizeof(struct v9_mqd), PAGE_SIZE),
- &((*mqd_mem_obj)->gtt_mem),
- &((*mqd_mem_obj)->gpu_addr),
- (void *)&((*mqd_mem_obj)->cpu_ptr), true);
- } else
- retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct v9_mqd),
- mqd_mem_obj);
- if (retval != 0)
- return -ENOMEM;
-
- m = (struct v9_mqd *) (*mqd_mem_obj)->cpu_ptr;
- addr = (*mqd_mem_obj)->gpu_addr;
+ &(mqd_mem_obj->gtt_mem),
+ &(mqd_mem_obj->gpu_addr),
+ (void *)&(mqd_mem_obj->cpu_ptr), true);
+ } else {
+ retval = kfd_gtt_sa_allocate(kfd, sizeof(struct v9_mqd),
+ &mqd_mem_obj);
+ }
+
+ if (retval) {
+ kfree(mqd_mem_obj);
+ return NULL;
+ }
+
+ return mqd_mem_obj;
+
+}
+
+static void init_mqd(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+ struct queue_properties *q)
+{
+ uint64_t addr;
+ struct v9_mqd *m;
+
+ m = (struct v9_mqd *) mqd_mem_obj->cpu_ptr;
+ addr = mqd_mem_obj->gpu_addr;
memset(m, 0, sizeof(struct v9_mqd));
@@ -120,9 +138,6 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT |
10 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT;
- m->cp_hqd_pipe_priority = 1;
- m->cp_hqd_queue_priority = 15;
-
if (q->format == KFD_QUEUE_FORMAT_AQL) {
m->cp_hqd_aql_control =
1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
@@ -149,9 +164,7 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
*mqd = m;
if (gart_addr)
*gart_addr = addr;
- retval = mm->update_mqd(mm, m, q);
-
- return retval;
+ mm->update_mqd(mm, m, q);
}
static int load_mqd(struct mqd_manager *mm, void *mqd,
@@ -166,7 +179,7 @@ static int load_mqd(struct mqd_manager *mm, void *mqd,
wptr_shift, 0, mms);
}
-static int update_mqd(struct mqd_manager *mm, void *mqd,
+static void update_mqd(struct mqd_manager *mm, void *mqd,
struct queue_properties *q)
{
struct v9_mqd *m;
@@ -225,13 +238,9 @@ static int update_mqd(struct mqd_manager *mm, void *mqd,
m->cp_hqd_ctx_save_control = 0;
update_cu_mask(mm, mqd, q);
+ set_priority(m, q);
- q->is_active = (q->queue_size > 0 &&
- q->queue_address != 0 &&
- q->queue_percent > 0 &&
- !q->is_evicted);
-
- return 0;
+ q->is_active = QUEUE_IS_ACTIVE(*q);
}
@@ -245,7 +254,7 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
pipe_id, queue_id);
}
-static void uninit_mqd(struct mqd_manager *mm, void *mqd,
+static void free_mqd(struct mqd_manager *mm, void *mqd,
struct kfd_mem_obj *mqd_mem_obj)
{
struct kfd_dev *kfd = mm->dev;
@@ -289,71 +298,47 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
return 0;
}
-static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
- struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *q)
{
struct v9_mqd *m;
- int retval = init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
- if (retval != 0)
- return retval;
+ init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
m = get_mqd(*mqd);
m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT |
1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;
-
- return retval;
}
-static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
+static void update_mqd_hiq(struct mqd_manager *mm, void *mqd,
struct queue_properties *q)
{
struct v9_mqd *m;
- int retval = update_mqd(mm, mqd, q);
- if (retval != 0)
- return retval;
+ update_mqd(mm, mqd, q);
/* TODO: what's the point? update_mqd already does this. */
m = get_mqd(mqd);
m->cp_hqd_vmid = q->vmid;
- return retval;
}
-static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
- struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *q)
{
- int retval;
struct v9_sdma_mqd *m;
-
- retval = kfd_gtt_sa_allocate(mm->dev,
- sizeof(struct v9_sdma_mqd),
- mqd_mem_obj);
-
- if (retval != 0)
- return -ENOMEM;
-
- m = (struct v9_sdma_mqd *) (*mqd_mem_obj)->cpu_ptr;
+ m = (struct v9_sdma_mqd *) mqd_mem_obj->cpu_ptr;
memset(m, 0, sizeof(struct v9_sdma_mqd));
*mqd = m;
if (gart_addr)
- *gart_addr = (*mqd_mem_obj)->gpu_addr;
-
- retval = mm->update_mqd(mm, m, q);
+ *gart_addr = mqd_mem_obj->gpu_addr;
- return retval;
-}
-
-static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd,
- struct kfd_mem_obj *mqd_mem_obj)
-{
- kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
+ mm->update_mqd(mm, m, q);
}
static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
@@ -367,7 +352,7 @@ static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
#define SDMA_RLC_DUMMY_DEFAULT 0xf
-static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
+static void update_mqd_sdma(struct mqd_manager *mm, void *mqd,
struct queue_properties *q)
{
struct v9_sdma_mqd *m;
@@ -390,12 +375,7 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
m->sdma_queue_id = q->sdma_queue_id;
m->sdmax_rlcx_dummy_reg = SDMA_RLC_DUMMY_DEFAULT;
- q->is_active = (q->queue_size > 0 &&
- q->queue_address != 0 &&
- q->queue_percent > 0 &&
- !q->is_evicted);
-
- return 0;
+ q->is_active = QUEUE_IS_ACTIVE(*q);
}
/*
@@ -452,35 +432,54 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
switch (type) {
case KFD_MQD_TYPE_CP:
case KFD_MQD_TYPE_COMPUTE:
+ mqd->allocate_mqd = allocate_mqd;
mqd->init_mqd = init_mqd;
- mqd->uninit_mqd = uninit_mqd;
+ mqd->free_mqd = free_mqd;
mqd->load_mqd = load_mqd;
mqd->update_mqd = update_mqd;
mqd->destroy_mqd = destroy_mqd;
mqd->is_occupied = is_occupied;
mqd->get_wave_state = get_wave_state;
+ mqd->mqd_size = sizeof(struct v9_mqd);
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd;
#endif
break;
case KFD_MQD_TYPE_HIQ:
+ mqd->allocate_mqd = allocate_hiq_mqd;
+ mqd->init_mqd = init_mqd_hiq;
+ mqd->free_mqd = free_mqd_hiq_sdma;
+ mqd->load_mqd = load_mqd;
+ mqd->update_mqd = update_mqd_hiq;
+ mqd->destroy_mqd = destroy_mqd;
+ mqd->is_occupied = is_occupied;
+ mqd->mqd_size = sizeof(struct v9_mqd);
+#if defined(CONFIG_DEBUG_FS)
+ mqd->debugfs_show_mqd = debugfs_show_mqd;
+#endif
+ break;
+ case KFD_MQD_TYPE_DIQ:
+ mqd->allocate_mqd = allocate_hiq_mqd;
mqd->init_mqd = init_mqd_hiq;
- mqd->uninit_mqd = uninit_mqd;
+ mqd->free_mqd = free_mqd;
mqd->load_mqd = load_mqd;
mqd->update_mqd = update_mqd_hiq;
mqd->destroy_mqd = destroy_mqd;
mqd->is_occupied = is_occupied;
+ mqd->mqd_size = sizeof(struct v9_mqd);
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd;
#endif
break;
case KFD_MQD_TYPE_SDMA:
+ mqd->allocate_mqd = allocate_sdma_mqd;
mqd->init_mqd = init_mqd_sdma;
- mqd->uninit_mqd = uninit_mqd_sdma;
+ mqd->free_mqd = free_mqd_hiq_sdma;
mqd->load_mqd = load_mqd_sdma;
mqd->update_mqd = update_mqd_sdma;
mqd->destroy_mqd = destroy_mqd_sdma;
mqd->is_occupied = is_occupied_sdma;
+ mqd->mqd_size = sizeof(struct v9_sdma_mqd);
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index 6469b3456f00..7d144f56f421 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -31,6 +31,7 @@
#include "gca/gfx_8_0_sh_mask.h"
#include "gca/gfx_8_0_enum.h"
#include "oss/oss_3_0_sh_mask.h"
+
#define CP_MQD_CONTROL__PRIV_STATE__SHIFT 0x8
static inline struct vi_mqd *get_mqd(void *mqd)
@@ -68,21 +69,33 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
m->compute_static_thread_mgmt_se3);
}
-static int init_mqd(struct mqd_manager *mm, void **mqd,
- struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+static void set_priority(struct vi_mqd *m, struct queue_properties *q)
+{
+ m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
+ m->cp_hqd_queue_priority = q->priority;
+}
+
+static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
+ struct queue_properties *q)
+{
+ struct kfd_mem_obj *mqd_mem_obj;
+
+ if (kfd_gtt_sa_allocate(kfd, sizeof(struct vi_mqd),
+ &mqd_mem_obj))
+ return NULL;
+
+ return mqd_mem_obj;
+}
+
+static void init_mqd(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *q)
{
- int retval;
uint64_t addr;
struct vi_mqd *m;
- retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct vi_mqd),
- mqd_mem_obj);
- if (retval != 0)
- return -ENOMEM;
-
- m = (struct vi_mqd *) (*mqd_mem_obj)->cpu_ptr;
- addr = (*mqd_mem_obj)->gpu_addr;
+ m = (struct vi_mqd *) mqd_mem_obj->cpu_ptr;
+ addr = mqd_mem_obj->gpu_addr;
memset(m, 0, sizeof(struct vi_mqd));
@@ -106,9 +119,7 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT |
10 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT;
- m->cp_hqd_pipe_priority = 1;
- m->cp_hqd_queue_priority = 15;
-
+ set_priority(m, q);
m->cp_hqd_eop_rptr = 1 << CP_HQD_EOP_RPTR__INIT_FETCHER__SHIFT;
if (q->format == KFD_QUEUE_FORMAT_AQL)
@@ -139,9 +150,7 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
*mqd = m;
if (gart_addr)
*gart_addr = addr;
- retval = mm->update_mqd(mm, m, q);
-
- return retval;
+ mm->update_mqd(mm, m, q);
}
static int load_mqd(struct mqd_manager *mm, void *mqd,
@@ -157,7 +166,7 @@ static int load_mqd(struct mqd_manager *mm, void *mqd,
wptr_shift, wptr_mask, mms);
}
-static int __update_mqd(struct mqd_manager *mm, void *mqd,
+static void __update_mqd(struct mqd_manager *mm, void *mqd,
struct queue_properties *q, unsigned int mtype,
unsigned int atc_bit)
{
@@ -222,26 +231,22 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
mtype << CP_HQD_CTX_SAVE_CONTROL__MTYPE__SHIFT;
update_cu_mask(mm, mqd, q);
+ set_priority(m, q);
- q->is_active = (q->queue_size > 0 &&
- q->queue_address != 0 &&
- q->queue_percent > 0 &&
- !q->is_evicted);
-
- return 0;
+ q->is_active = QUEUE_IS_ACTIVE(*q);
}
-static int update_mqd(struct mqd_manager *mm, void *mqd,
+static void update_mqd(struct mqd_manager *mm, void *mqd,
struct queue_properties *q)
{
- return __update_mqd(mm, mqd, q, MTYPE_CC, 1);
+ __update_mqd(mm, mqd, q, MTYPE_CC, 1);
}
-static int update_mqd_tonga(struct mqd_manager *mm, void *mqd,
+static void update_mqd_tonga(struct mqd_manager *mm, void *mqd,
struct queue_properties *q)
{
- return __update_mqd(mm, mqd, q, MTYPE_UC, 0);
+ __update_mqd(mm, mqd, q, MTYPE_UC, 0);
}
static int destroy_mqd(struct mqd_manager *mm, void *mqd,
@@ -254,7 +259,7 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
pipe_id, queue_id);
}
-static void uninit_mqd(struct mqd_manager *mm, void *mqd,
+static void free_mqd(struct mqd_manager *mm, void *mqd,
struct kfd_mem_obj *mqd_mem_obj)
{
kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
@@ -291,70 +296,44 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
return 0;
}
-static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
- struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *q)
{
struct vi_mqd *m;
- int retval = init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
-
- if (retval != 0)
- return retval;
+ init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
m = get_mqd(*mqd);
m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT |
1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;
-
- return retval;
}
-static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
+static void update_mqd_hiq(struct mqd_manager *mm, void *mqd,
struct queue_properties *q)
{
struct vi_mqd *m;
- int retval = __update_mqd(mm, mqd, q, MTYPE_UC, 0);
-
- if (retval != 0)
- return retval;
+ __update_mqd(mm, mqd, q, MTYPE_UC, 0);
m = get_mqd(mqd);
m->cp_hqd_vmid = q->vmid;
- return retval;
}
-static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
- struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *q)
{
- int retval;
struct vi_sdma_mqd *m;
-
- retval = kfd_gtt_sa_allocate(mm->dev,
- sizeof(struct vi_sdma_mqd),
- mqd_mem_obj);
-
- if (retval != 0)
- return -ENOMEM;
-
- m = (struct vi_sdma_mqd *) (*mqd_mem_obj)->cpu_ptr;
+ m = (struct vi_sdma_mqd *) mqd_mem_obj->cpu_ptr;
memset(m, 0, sizeof(struct vi_sdma_mqd));
*mqd = m;
- if (gart_addr != NULL)
- *gart_addr = (*mqd_mem_obj)->gpu_addr;
-
- retval = mm->update_mqd(mm, m, q);
-
- return retval;
-}
+ if (gart_addr)
+ *gart_addr = mqd_mem_obj->gpu_addr;
-static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd,
- struct kfd_mem_obj *mqd_mem_obj)
-{
- kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
+ mm->update_mqd(mm, m, q);
}
static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
@@ -366,7 +345,7 @@ static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
mms);
}
-static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
+static void update_mqd_sdma(struct mqd_manager *mm, void *mqd,
struct queue_properties *q)
{
struct vi_sdma_mqd *m;
@@ -390,12 +369,7 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
m->sdma_engine_id = q->sdma_engine_id;
m->sdma_queue_id = q->sdma_queue_id;
- q->is_active = (q->queue_size > 0 &&
- q->queue_address != 0 &&
- q->queue_percent > 0 &&
- !q->is_evicted);
-
- return 0;
+ q->is_active = QUEUE_IS_ACTIVE(*q);
}
/*
@@ -452,35 +426,54 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
switch (type) {
case KFD_MQD_TYPE_CP:
case KFD_MQD_TYPE_COMPUTE:
+ mqd->allocate_mqd = allocate_mqd;
mqd->init_mqd = init_mqd;
- mqd->uninit_mqd = uninit_mqd;
+ mqd->free_mqd = free_mqd;
mqd->load_mqd = load_mqd;
mqd->update_mqd = update_mqd;
mqd->destroy_mqd = destroy_mqd;
mqd->is_occupied = is_occupied;
mqd->get_wave_state = get_wave_state;
+ mqd->mqd_size = sizeof(struct vi_mqd);
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd;
#endif
break;
case KFD_MQD_TYPE_HIQ:
+ mqd->allocate_mqd = allocate_hiq_mqd;
+ mqd->init_mqd = init_mqd_hiq;
+ mqd->free_mqd = free_mqd_hiq_sdma;
+ mqd->load_mqd = load_mqd;
+ mqd->update_mqd = update_mqd_hiq;
+ mqd->destroy_mqd = destroy_mqd;
+ mqd->is_occupied = is_occupied;
+ mqd->mqd_size = sizeof(struct vi_mqd);
+#if defined(CONFIG_DEBUG_FS)
+ mqd->debugfs_show_mqd = debugfs_show_mqd;
+#endif
+ break;
+ case KFD_MQD_TYPE_DIQ:
+ mqd->allocate_mqd = allocate_hiq_mqd;
mqd->init_mqd = init_mqd_hiq;
- mqd->uninit_mqd = uninit_mqd;
+ mqd->free_mqd = free_mqd;
mqd->load_mqd = load_mqd;
mqd->update_mqd = update_mqd_hiq;
mqd->destroy_mqd = destroy_mqd;
mqd->is_occupied = is_occupied;
+ mqd->mqd_size = sizeof(struct vi_mqd);
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd;
#endif
break;
case KFD_MQD_TYPE_SDMA:
+ mqd->allocate_mqd = allocate_sdma_mqd;
mqd->init_mqd = init_mqd_sdma;
- mqd->uninit_mqd = uninit_mqd_sdma;
+ mqd->free_mqd = free_mqd_hiq_sdma;
mqd->load_mqd = load_mqd_sdma;
mqd->update_mqd = update_mqd_sdma;
mqd->destroy_mqd = destroy_mqd_sdma;
mqd->is_occupied = is_occupied_sdma;
+ mqd->mqd_size = sizeof(struct vi_sdma_mqd);
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
index 045a229436a0..ccf6b2310316 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
@@ -48,7 +48,8 @@ static void pm_calc_rlib_size(struct packet_manager *pm,
process_count = pm->dqm->processes_count;
queue_count = pm->dqm->queue_count;
- compute_queue_count = queue_count - pm->dqm->sdma_queue_count;
+ compute_queue_count = queue_count - pm->dqm->sdma_queue_count -
+ pm->dqm->xgmi_sdma_queue_count;
/* check if there is over subscription
* Note: the arbitration between the number of VMIDs and
@@ -202,11 +203,15 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
pr_debug("Finished map process and queues to runlist\n");
- if (is_over_subscription)
+ if (is_over_subscription) {
+ if (!pm->is_over_subscription)
+ pr_warn("Runlist is getting oversubscribed. Expect reduced ROCm performance.\n");
retval = pm->pmf->runlist(pm, &rl_buffer[rl_wptr],
*rl_gpu_addr,
alloc_size_bytes / sizeof(uint32_t),
true);
+ }
+ pm->is_over_subscription = is_over_subscription;
for (i = 0; i < alloc_size_bytes / sizeof(uint32_t); i++)
pr_debug("0x%2X ", rl_buffer[i]);
@@ -227,6 +232,7 @@ int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm)
case CHIP_POLARIS10:
case CHIP_POLARIS11:
case CHIP_POLARIS12:
+ case CHIP_VEGAM:
pm->pmf = &kfd_vi_pm_funcs;
break;
case CHIP_VEGA10:
@@ -235,6 +241,9 @@ int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm)
case CHIP_RAVEN:
pm->pmf = &kfd_v9_pm_funcs;
break;
+ case CHIP_NAVI10:
+ pm->pmf = &kfd_v10_pm_funcs;
+ break;
default:
WARN(1, "Unexpected ASIC family %u",
dqm->dev->device_info->asic_family);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_ai.h b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_ai.h
index f2bcf5c092ea..e3e21404cfa0 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_ai.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_ai.h
@@ -120,7 +120,7 @@ struct pm4_mes_runlist {
uint32_t ib_size:20;
uint32_t chain:1;
uint32_t offload_polling:1;
- uint32_t reserved2:1;
+ uint32_t chained_runlist_idle_disable:1;
uint32_t valid:1;
uint32_t process_cnt:4;
uint32_t reserved3:4;
@@ -176,8 +176,7 @@ struct pm4_mes_map_process {
union {
struct {
- uint32_t num_gws:6;
- uint32_t reserved7:1;
+ uint32_t num_gws:7;
uint32_t sdma_enable:1;
uint32_t num_oac:4;
uint32_t reserved8:4;
@@ -255,11 +254,6 @@ enum mes_map_queues_queue_type_enum {
queue_type__mes_map_queues__low_latency_static_queue_vi = 3
};
-enum mes_map_queues_alloc_format_enum {
- alloc_format__mes_map_queues__one_per_pipe_vi = 0,
-alloc_format__mes_map_queues__all_on_one_pipe_vi = 1
-};
-
enum mes_map_queues_engine_sel_enum {
engine_sel__mes_map_queues__compute_vi = 0,
engine_sel__mes_map_queues__sdma0_vi = 2,
@@ -277,9 +271,11 @@ struct pm4_mes_map_queues {
struct {
uint32_t reserved1:4;
enum mes_map_queues_queue_sel_enum queue_sel:2;
- uint32_t reserved2:15;
+ uint32_t reserved5:6;
+ uint32_t gws_control_queue:1;
+ uint32_t reserved2:8;
enum mes_map_queues_queue_type_enum queue_type:3;
- enum mes_map_queues_alloc_format_enum alloc_format:2;
+ uint32_t reserved3:2;
enum mes_map_queues_engine_sel_enum engine_sel:3;
uint32_t num_queues:3;
} bitfields2;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h
index 7c8d9b357749..5466cfe1c3cc 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h
@@ -216,11 +216,6 @@ enum mes_map_queues_queue_type_vi_enum {
queue_type__mes_map_queues__low_latency_static_queue_vi = 3
};
-enum mes_map_queues_alloc_format_vi_enum {
- alloc_format__mes_map_queues__one_per_pipe_vi = 0,
-alloc_format__mes_map_queues__all_on_one_pipe_vi = 1
-};
-
enum mes_map_queues_engine_sel_vi_enum {
engine_sel__mes_map_queues__compute_vi = 0,
engine_sel__mes_map_queues__sdma0_vi = 2,
@@ -240,7 +235,7 @@ struct pm4_mes_map_queues {
enum mes_map_queues_queue_sel_vi_enum queue_sel:2;
uint32_t reserved2:15;
enum mes_map_queues_queue_type_vi_enum queue_type:3;
- enum mes_map_queues_alloc_format_vi_enum alloc_format:2;
+ uint32_t reserved3:2;
enum mes_map_queues_engine_sel_vi_enum engine_sel:3;
uint32_t num_queues:3;
} bitfields2;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 487d5da337c1..3933fb6a371e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -35,6 +35,7 @@
#include <linux/kfifo.h>
#include <linux/seq_file.h>
#include <linux/kref.h>
+#include <linux/sysfs.h>
#include <kgd_kfd_interface.h>
#include "amd_shared.h"
@@ -59,6 +60,7 @@
#define KFD_MMAP_TYPE_DOORBELL (0x3ULL << KFD_MMAP_TYPE_SHIFT)
#define KFD_MMAP_TYPE_EVENTS (0x2ULL << KFD_MMAP_TYPE_SHIFT)
#define KFD_MMAP_TYPE_RESERVED_MEM (0x1ULL << KFD_MMAP_TYPE_SHIFT)
+#define KFD_MMAP_TYPE_MMIO (0x0ULL << KFD_MMAP_TYPE_SHIFT)
#define KFD_MMAP_GPU_ID_SHIFT (46 - PAGE_SHIFT)
#define KFD_MMAP_GPU_ID_MASK (((1ULL << KFD_GPU_ID_HASH_WIDTH) - 1) \
@@ -103,6 +105,8 @@
#define KFD_KERNEL_QUEUE_SIZE 2048
+#define KFD_UNMAP_LATENCY_MS (4000)
+
/*
* 512 = 0x200
* The doorbell index distance between SDMA RLC (2*i) and (2*i+1) in the
@@ -153,18 +157,32 @@ extern int ignore_crat;
/*
* Set sh_mem_config.retry_disable on Vega10
*/
-extern int noretry;
+extern int amdgpu_noretry;
/*
* Halt if HWS hang is detected
*/
extern int halt_if_hws_hang;
+/*
+ * Whether MEC FW support GWS barriers
+ */
+extern bool hws_gws_support;
+
+/*
+ * Queue preemption timeout in ms
+ */
+extern int queue_preemption_timeout_ms;
+
enum cache_policy {
cache_policy_coherent,
cache_policy_noncoherent
};
+#define KFD_IS_VI(chip) ((chip) >= CHIP_CARRIZO && (chip) <= CHIP_POLARIS11)
+#define KFD_IS_DGPU(chip) (((chip) >= CHIP_TONGA && \
+ (chip) <= CHIP_NAVI10) || \
+ (chip) == CHIP_HAWAII)
#define KFD_IS_SOC15(chip) ((chip) >= CHIP_VEGA10)
struct kfd_event_interrupt_class {
@@ -188,6 +206,7 @@ struct kfd_device_info {
bool needs_iommu_device;
bool needs_pci_atomics;
unsigned int num_sdma_engines;
+ unsigned int num_xgmi_sdma_engines;
unsigned int num_sdma_queues_per_engine;
};
@@ -258,7 +277,7 @@ struct kfd_dev {
bool interrupts_active;
/* Debug manager */
- struct kfd_dbgmgr *dbgmgr;
+ struct kfd_dbgmgr *dbgmgr;
/* Firmware versions */
uint16_t mec_fw_version;
@@ -282,6 +301,9 @@ struct kfd_dev {
/* Compute Profile ref. count */
atomic_t compute_profile;
+
+ /* Global GWS resource shared b/t processes*/
+ void *gws;
};
enum kfd_mempool {
@@ -329,7 +351,8 @@ enum kfd_queue_type {
KFD_QUEUE_TYPE_COMPUTE,
KFD_QUEUE_TYPE_SDMA,
KFD_QUEUE_TYPE_HIQ,
- KFD_QUEUE_TYPE_DIQ
+ KFD_QUEUE_TYPE_DIQ,
+ KFD_QUEUE_TYPE_SDMA_XGMI
};
enum kfd_queue_format {
@@ -337,6 +360,11 @@ enum kfd_queue_format {
KFD_QUEUE_FORMAT_AQL
};
+enum KFD_QUEUE_PRIORITY {
+ KFD_QUEUE_PRIORITY_MINIMUM = 0,
+ KFD_QUEUE_PRIORITY_MAXIMUM = 15
+};
+
/**
* struct queue_properties
*
@@ -419,6 +447,11 @@ struct queue_properties {
uint32_t *cu_mask;
};
+#define QUEUE_IS_ACTIVE(q) ((q).queue_size > 0 && \
+ (q).queue_address != 0 && \
+ (q).queue_percent > 0 && \
+ !(q).is_evicted)
+
/**
* struct queue
*
@@ -444,6 +477,9 @@ struct queue_properties {
*
* @device: The kfd device that created this queue.
*
+ * @gws: Pointing to gws kgd_mem if this is a gws control queue; NULL
+ * otherwise.
+ *
* This structure represents user mode compute queues.
* It contains all the necessary data to handle such queues.
*
@@ -465,6 +501,7 @@ struct queue {
struct kfd_process *process;
struct kfd_dev *device;
+ void *gws;
};
/*
@@ -475,9 +512,16 @@ enum KFD_MQD_TYPE {
KFD_MQD_TYPE_HIQ, /* for hiq */
KFD_MQD_TYPE_CP, /* for cp queues and diq */
KFD_MQD_TYPE_SDMA, /* for sdma queues */
+ KFD_MQD_TYPE_DIQ, /* for diq */
KFD_MQD_TYPE_MAX
};
+enum KFD_PIPE_PRIORITY {
+ KFD_PIPE_PRIORITY_CS_LOW = 0,
+ KFD_PIPE_PRIORITY_CS_MEDIUM,
+ KFD_PIPE_PRIORITY_CS_HIGH
+};
+
struct scheduling_resources {
unsigned int vmid_mask;
enum kfd_queue_type type;
@@ -686,6 +730,10 @@ struct kfd_process {
* restored after an eviction
*/
unsigned long last_restore_timestamp;
+
+ /* Kobj for our procfs */
+ struct kobject *kobj;
+ struct attribute attr_pasid;
};
#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
@@ -788,6 +836,10 @@ int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj);
extern struct device *kfd_device;
+/* KFD's procfs */
+void kfd_procfs_init(void);
+void kfd_procfs_shutdown(void);
+
/* Topology */
int kfd_topology_init(void);
void kfd_topology_shutdown(void);
@@ -819,8 +871,6 @@ void uninit_queue(struct queue *q);
void print_queue_properties(struct queue_properties *q);
void print_queue(struct queue *q);
-struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev);
struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
struct kfd_dev *dev);
struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
@@ -831,6 +881,8 @@ struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
struct kfd_dev *dev);
struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
struct kfd_dev *dev);
+struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
+ struct kfd_dev *dev);
struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
void device_queue_manager_uninit(struct device_queue_manager *dqm);
struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
@@ -859,6 +911,8 @@ int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
struct queue_properties *p);
int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
struct queue_properties *p);
+int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,
+ void *gws);
struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
unsigned int qid);
int pqm_get_wave_state(struct process_queue_manager *pqm,
@@ -868,8 +922,8 @@ int pqm_get_wave_state(struct process_queue_manager *pqm,
u32 *save_area_used_size);
int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
- unsigned int fence_value,
- unsigned int timeout_ms);
+ unsigned int fence_value,
+ unsigned int timeout_ms);
/* Packet Manager */
@@ -883,6 +937,7 @@ struct packet_manager {
bool allocated;
struct kfd_mem_obj *ib_buffer_obj;
unsigned int ib_size_bytes;
+ bool is_over_subscription;
const struct packet_manager_funcs *pmf;
};
@@ -918,6 +973,7 @@ struct packet_manager_funcs {
extern const struct packet_manager_funcs kfd_vi_pm_funcs;
extern const struct packet_manager_funcs kfd_v9_pm_funcs;
+extern const struct packet_manager_funcs kfd_v10_pm_funcs;
int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
void pm_uninit(struct packet_manager *pm);
@@ -937,7 +993,8 @@ void pm_release_ib(struct packet_manager *pm);
/* Following PM funcs can be shared among VI and AI */
unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size);
int pm_set_resources_vi(struct packet_manager *pm, uint32_t *buffer,
- struct scheduling_resources *res);
+ struct scheduling_resources *res);
+
uint64_t kfd_get_number_elems(struct kfd_dev *kfd);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 4bdae78bab8e..8f1076c0c88a 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -68,6 +68,68 @@ static struct kfd_process *create_process(const struct task_struct *thread,
static void evict_process_worker(struct work_struct *work);
static void restore_process_worker(struct work_struct *work);
+struct kfd_procfs_tree {
+ struct kobject *kobj;
+};
+
+static struct kfd_procfs_tree procfs;
+
+static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
+ char *buffer)
+{
+ int val = 0;
+
+ if (strcmp(attr->name, "pasid") == 0) {
+ struct kfd_process *p = container_of(attr, struct kfd_process,
+ attr_pasid);
+ val = p->pasid;
+ } else {
+ pr_err("Invalid attribute");
+ return -EINVAL;
+ }
+
+ return snprintf(buffer, PAGE_SIZE, "%d\n", val);
+}
+
+static void kfd_procfs_kobj_release(struct kobject *kobj)
+{
+ kfree(kobj);
+}
+
+static const struct sysfs_ops kfd_procfs_ops = {
+ .show = kfd_procfs_show,
+};
+
+static struct kobj_type procfs_type = {
+ .release = kfd_procfs_kobj_release,
+ .sysfs_ops = &kfd_procfs_ops,
+};
+
+void kfd_procfs_init(void)
+{
+ int ret = 0;
+
+ procfs.kobj = kfd_alloc_struct(procfs.kobj);
+ if (!procfs.kobj)
+ return;
+
+ ret = kobject_init_and_add(procfs.kobj, &procfs_type,
+ &kfd_device->kobj, "proc");
+ if (ret) {
+ pr_warn("Could not create procfs proc folder");
+ /* If we fail to create the procfs, clean up */
+ kfd_procfs_shutdown();
+ }
+}
+
+void kfd_procfs_shutdown(void)
+{
+ if (procfs.kobj) {
+ kobject_del(procfs.kobj);
+ kobject_put(procfs.kobj);
+ procfs.kobj = NULL;
+ }
+}
int kfd_process_create_wq(void)
{
@@ -206,6 +268,7 @@ struct kfd_process *kfd_create_process(struct file *filep)
{
struct kfd_process *process;
struct task_struct *thread = current;
+ int ret;
if (!thread->mm)
return ERR_PTR(-EINVAL);
@@ -223,11 +286,36 @@ struct kfd_process *kfd_create_process(struct file *filep)
/* A prior open of /dev/kfd could have already created the process. */
process = find_process(thread);
- if (process)
+ if (process) {
pr_debug("Process already found\n");
- else
+ } else {
process = create_process(thread, filep);
+ if (!procfs.kobj)
+ goto out;
+
+ process->kobj = kfd_alloc_struct(process->kobj);
+ if (!process->kobj) {
+ pr_warn("Creating procfs kobject failed");
+ goto out;
+ }
+ ret = kobject_init_and_add(process->kobj, &procfs_type,
+ procfs.kobj, "%d",
+ (int)process->lead_thread->pid);
+ if (ret) {
+ pr_warn("Creating procfs pid directory failed");
+ goto out;
+ }
+
+ process->attr_pasid.name = "pasid";
+ process->attr_pasid.mode = KFD_SYSFS_FILE_MODE;
+ sysfs_attr_init(&process->attr_pasid);
+ ret = sysfs_create_file(process->kobj, &process->attr_pasid);
+ if (ret)
+ pr_warn("Creating pasid for pid %d failed",
+ (int)process->lead_thread->pid);
+ }
+out:
mutex_unlock(&kfd_processes_mutex);
return process;
@@ -355,6 +443,14 @@ static void kfd_process_wq_release(struct work_struct *work)
struct kfd_process *p = container_of(work, struct kfd_process,
release_work);
+ /* Remove the procfs files */
+ if (p->kobj) {
+ sysfs_remove_file(p->kobj, &p->attr_pasid);
+ kobject_del(p->kobj);
+ kobject_put(p->kobj);
+ p->kobj = NULL;
+ }
+
kfd_iommu_unbind_process(p);
kfd_process_free_outstanding_kfd_bos(p);
@@ -1107,3 +1203,4 @@ int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
}
#endif
+
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index fcaaf93681ac..7e6c3ee82f5b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -26,6 +26,7 @@
#include "kfd_device_queue_manager.h"
#include "kfd_priv.h"
#include "kfd_kernel_queue.h"
+#include "amdgpu_amdkfd.h"
static inline struct process_queue_node *get_queue_by_qid(
struct process_queue_manager *pqm, unsigned int qid)
@@ -74,6 +75,55 @@ void kfd_process_dequeue_from_device(struct kfd_process_device *pdd)
pdd->already_dequeued = true;
}
+int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,
+ void *gws)
+{
+ struct kfd_dev *dev = NULL;
+ struct process_queue_node *pqn;
+ struct kfd_process_device *pdd;
+ struct kgd_mem *mem = NULL;
+ int ret;
+
+ pqn = get_queue_by_qid(pqm, qid);
+ if (!pqn) {
+ pr_err("Queue id does not match any known queue\n");
+ return -EINVAL;
+ }
+
+ if (pqn->q)
+ dev = pqn->q->device;
+ if (WARN_ON(!dev))
+ return -ENODEV;
+
+ pdd = kfd_get_process_device_data(dev, pqm->process);
+ if (!pdd) {
+ pr_err("Process device data doesn't exist\n");
+ return -EINVAL;
+ }
+
+ /* Only allow one queue per process can have GWS assigned */
+ if (gws && pdd->qpd.num_gws)
+ return -EBUSY;
+
+ if (!gws && pdd->qpd.num_gws == 0)
+ return -EINVAL;
+
+ if (gws)
+ ret = amdgpu_amdkfd_add_gws_to_process(pdd->process->kgd_process_info,
+ gws, &mem);
+ else
+ ret = amdgpu_amdkfd_remove_gws_from_process(pdd->process->kgd_process_info,
+ pqn->q->gws);
+ if (unlikely(ret))
+ return ret;
+
+ pqn->q->gws = mem;
+ pdd->qpd.num_gws = gws ? amdgpu_amdkfd_get_num_gws(dev->kgd) : 0;
+
+ return pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
+ pqn->q);
+}
+
void kfd_process_dequeue_from_all_devices(struct kfd_process *p)
{
struct kfd_process_device *pdd;
@@ -100,6 +150,9 @@ void pqm_uninit(struct process_queue_manager *pqm)
struct process_queue_node *pqn, *next;
list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) {
+ if (pqn->q && pqn->q->gws)
+ amdgpu_amdkfd_remove_gws_from_process(pqm->process->kgd_process_info,
+ pqn->q->gws);
uninit_queue(pqn->q);
list_del(&pqn->process_queue_list);
kfree(pqn);
@@ -186,8 +239,13 @@ int pqm_create_queue(struct process_queue_manager *pqm,
switch (type) {
case KFD_QUEUE_TYPE_SDMA:
- if (dev->dqm->queue_count >= get_num_sdma_queues(dev->dqm)) {
- pr_err("Over-subscription is not allowed for SDMA.\n");
+ case KFD_QUEUE_TYPE_SDMA_XGMI:
+ if ((type == KFD_QUEUE_TYPE_SDMA && dev->dqm->sdma_queue_count
+ >= get_num_sdma_queues(dev->dqm)) ||
+ (type == KFD_QUEUE_TYPE_SDMA_XGMI &&
+ dev->dqm->xgmi_sdma_queue_count
+ >= get_num_xgmi_sdma_queues(dev->dqm))) {
+ pr_debug("Over-subscription is not allowed for SDMA.\n");
retval = -EPERM;
goto err_create_queue;
}
@@ -325,6 +383,13 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
if (retval != -ETIME)
goto err_destroy_queue;
}
+
+ if (pqn->q->gws) {
+ amdgpu_amdkfd_remove_gws_from_process(pqm->process->kgd_process_info,
+ pqn->q->gws);
+ pdd->qpd.num_gws = 0;
+ }
+
kfree(pqn->q->properties.cu_mask);
pqn->q->properties.cu_mask = NULL;
uninit_queue(pqn->q);
@@ -446,6 +511,7 @@ int pqm_debugfs_mqds(struct seq_file *m, void *data)
q = pqn->q;
switch (q->properties.type) {
case KFD_QUEUE_TYPE_SDMA:
+ case KFD_QUEUE_TYPE_SDMA_XGMI:
seq_printf(m, " SDMA queue on device %x\n",
q->device->id);
mqd_type = KFD_MQD_TYPE_SDMA;
@@ -461,8 +527,7 @@ int pqm_debugfs_mqds(struct seq_file *m, void *data)
q->properties.type, q->device->id);
continue;
}
- mqd_mgr = q->device->dqm->ops.get_mqd_manager(
- q->device->dqm, mqd_type);
+ mqd_mgr = q->device->dqm->mqd_mgrs[mqd_type];
} else if (pqn->kq) {
q = pqn->kq->queue;
mqd_mgr = pqn->kq->mqd_mgr;
@@ -470,7 +535,6 @@ int pqm_debugfs_mqds(struct seq_file *m, void *data)
case KFD_QUEUE_TYPE_DIQ:
seq_printf(m, " DIQ on device %x\n",
pqn->kq->dev->id);
- mqd_type = KFD_MQD_TYPE_HIQ;
break;
default:
seq_printf(m,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 769dbc7be8cb..c2e6e47abaf2 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -454,6 +454,8 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
dev->node_props.lds_size_in_kb);
sysfs_show_32bit_prop(buffer, "gds_size_in_kb",
dev->node_props.gds_size_in_kb);
+ sysfs_show_32bit_prop(buffer, "num_gws",
+ dev->node_props.num_gws);
sysfs_show_32bit_prop(buffer, "wave_front_size",
dev->node_props.wave_front_size);
sysfs_show_32bit_prop(buffer, "array_count",
@@ -476,6 +478,10 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
dev->node_props.drm_render_minor);
sysfs_show_64bit_prop(buffer, "hive_id",
dev->node_props.hive_id);
+ sysfs_show_32bit_prop(buffer, "num_sdma_engines",
+ dev->node_props.num_sdma_engines);
+ sysfs_show_32bit_prop(buffer, "num_sdma_xgmi_engines",
+ dev->node_props.num_sdma_xgmi_engines);
if (dev->gpu) {
log_max_watch_addr =
@@ -1078,8 +1084,9 @@ static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
local_mem_info.local_mem_size_public;
buf[0] = gpu->pdev->devfn;
- buf[1] = gpu->pdev->subsystem_vendor;
- buf[2] = gpu->pdev->subsystem_device;
+ buf[1] = gpu->pdev->subsystem_vendor |
+ (gpu->pdev->subsystem_device << 16);
+ buf[2] = pci_domain_nr(gpu->pdev->bus);
buf[3] = gpu->pdev->device;
buf[4] = gpu->pdev->bus->number;
buf[5] = lower_32_bits(local_mem_size);
@@ -1281,6 +1288,12 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
gpu->shared_resources.drm_render_minor;
dev->node_props.hive_id = gpu->hive_id;
+ dev->node_props.num_sdma_engines = gpu->device_info->num_sdma_engines;
+ dev->node_props.num_sdma_xgmi_engines =
+ gpu->device_info->num_xgmi_sdma_engines;
+ dev->node_props.num_gws = (hws_gws_support &&
+ dev->gpu->dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) ?
+ amdgpu_amdkfd_get_num_gws(dev->gpu->kgd) : 0;
kfd_fill_mem_clk_max_info(dev);
kfd_fill_iolink_non_crat_info(dev);
@@ -1298,6 +1311,7 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
case CHIP_POLARIS10:
case CHIP_POLARIS11:
case CHIP_POLARIS12:
+ case CHIP_VEGAM:
pr_debug("Adding doorbell packet type capability\n");
dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_1_0 <<
HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
@@ -1307,6 +1321,7 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
case CHIP_VEGA12:
case CHIP_VEGA20:
case CHIP_RAVEN:
+ case CHIP_NAVI10:
dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_2_0 <<
HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
@@ -1316,17 +1331,24 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
dev->gpu->device_info->asic_family);
}
+ /*
+ * Overwrite ATS capability according to needs_iommu_device to fix
+ * potential missing corresponding bit in CRAT of BIOS.
+ */
+ if (dev->gpu->device_info->needs_iommu_device)
+ dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
+ else
+ dev->node_props.capability &= ~HSA_CAP_ATS_PRESENT;
+
/* Fix errors in CZ CRAT.
* simd_count: Carrizo CRAT reports wrong simd_count, probably
* because it doesn't consider masked out CUs
* max_waves_per_simd: Carrizo reports wrong max_waves_per_simd
- * capability flag: Carrizo CRAT doesn't report IOMMU flags
*/
if (dev->gpu->device_info->asic_family == CHIP_CARRIZO) {
dev->node_props.simd_count =
cu_info.simd_per_cu * cu_info.cu_active_number;
dev->node_props.max_waves_per_simd = 10;
- dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
}
ctx = amdgpu_ras_get_context((struct amdgpu_device *)(dev->gpu->kgd));
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
index 84710cfd23c2..276354aa0fcc 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
@@ -65,6 +65,7 @@ struct kfd_node_properties {
uint32_t max_waves_per_simd;
uint32_t lds_size_in_kb;
uint32_t gds_size_in_kb;
+ uint32_t num_gws;
uint32_t wave_front_size;
uint32_t array_count;
uint32_t simd_arrays_per_engine;
@@ -78,6 +79,8 @@ struct kfd_node_properties {
uint32_t max_engine_clk_fcompute;
uint32_t max_engine_clk_ccompute;
int32_t drm_render_minor;
+ uint32_t num_sdma_engines;
+ uint32_t num_sdma_xgmi_engines;
uint16_t marketing_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE];
};
diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig
index 0c25baded852..f954bf61af28 100644
--- a/drivers/gpu/drm/amd/display/Kconfig
+++ b/drivers/gpu/drm/amd/display/Kconfig
@@ -5,8 +5,8 @@ menu "Display Engine Configuration"
config DRM_AMD_DC
bool "AMD DC - Enable new display engine"
default y
+ select SND_HDA_COMPONENT if SND_HDA_CORE
select DRM_AMD_DC_DCN1_0 if X86 && !(KCOV_INSTRUMENT_ALL && KCOV_ENABLE_COMPARISONS)
- select DRM_AMD_DC_DCN1_01 if X86 && !(KCOV_INSTRUMENT_ALL && KCOV_ENABLE_COMPARISONS)
help
Choose this option if you want to use the new display engine
support for AMDGPU. This adds required support for Vega and
@@ -17,10 +17,24 @@ config DRM_AMD_DC_DCN1_0
help
RV family support for display engine
-config DRM_AMD_DC_DCN1_01
- def_bool n
+config DRM_AMD_DC_DCN2_0
+ bool "DCN 2.0 family"
+ default y
+ depends on DRM_AMD_DC && X86
+ depends on DRM_AMD_DC_DCN1_0
+ help
+ Choose this option if you want to have
+ Navi support for display engine
+
+config DRM_AMD_DC_DSC_SUPPORT
+ bool "DSC support"
+ default y
+ depends on DRM_AMD_DC && X86
+ depends on DRM_AMD_DC_DCN1_0
+ depends on DRM_AMD_DC_DCN2_0
help
- RV2 family for display engine
+ Choose this option if you want to have
+ Dynamic Stream Compression support
config DEBUG_KERNEL_DC
bool "Enable kgdb break in DC"
diff --git a/drivers/gpu/drm/amd/display/Makefile b/drivers/gpu/drm/amd/display/Makefile
index cfde1568c79a..496cee000f10 100644
--- a/drivers/gpu/drm/amd/display/Makefile
+++ b/drivers/gpu/drm/amd/display/Makefile
@@ -28,6 +28,7 @@ AMDDALPATH = $(RELATIVE_AMD_DISPLAY_PATH)
subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/inc/
subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/inc/hw
+subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/clk_mgr
subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/modules/inc
subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/modules/freesync
subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/modules/color
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
index ab7c5c3004ee..45be7a2132bb 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
@@ -54,18 +54,22 @@
#include <linux/version.h>
#include <linux/types.h>
#include <linux/pm_runtime.h>
+#include <linux/pci.h>
#include <linux/firmware.h>
+#include <linux/component.h>
-#include <drm/drmP.h>
#include <drm/drm_atomic.h>
#include <drm/drm_atomic_uapi.h>
#include <drm/drm_atomic_helper.h>
#include <drm/drm_dp_mst_helper.h>
#include <drm/drm_fb_helper.h>
+#include <drm/drm_fourcc.h>
#include <drm/drm_edid.h>
+#include <drm/drm_vblank.h>
+#include <drm/drm_audio_component.h>
#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
-#include "ivsrcid/irqsrcs_dcn_1_0.h"
+#include "ivsrcid/dcn/irqsrcs_dcn_1_0.h"
#include "dcn/dcn_1_0_offset.h"
#include "dcn/dcn_1_0_sh_mask.h"
@@ -506,6 +510,139 @@ static void amdgpu_dm_fbc_init(struct drm_connector *connector)
}
+static int amdgpu_dm_audio_component_get_eld(struct device *kdev, int port,
+ int pipe, bool *enabled,
+ unsigned char *buf, int max_bytes)
+{
+ struct drm_device *dev = dev_get_drvdata(kdev);
+ struct amdgpu_device *adev = dev->dev_private;
+ struct drm_connector *connector;
+ struct drm_connector_list_iter conn_iter;
+ struct amdgpu_dm_connector *aconnector;
+ int ret = 0;
+
+ *enabled = false;
+
+ mutex_lock(&adev->dm.audio_lock);
+
+ drm_connector_list_iter_begin(dev, &conn_iter);
+ drm_for_each_connector_iter(connector, &conn_iter) {
+ aconnector = to_amdgpu_dm_connector(connector);
+ if (aconnector->audio_inst != port)
+ continue;
+
+ *enabled = true;
+ ret = drm_eld_size(connector->eld);
+ memcpy(buf, connector->eld, min(max_bytes, ret));
+
+ break;
+ }
+ drm_connector_list_iter_end(&conn_iter);
+
+ mutex_unlock(&adev->dm.audio_lock);
+
+ DRM_DEBUG_KMS("Get ELD : idx=%d ret=%d en=%d\n", port, ret, *enabled);
+
+ return ret;
+}
+
+static const struct drm_audio_component_ops amdgpu_dm_audio_component_ops = {
+ .get_eld = amdgpu_dm_audio_component_get_eld,
+};
+
+static int amdgpu_dm_audio_component_bind(struct device *kdev,
+ struct device *hda_kdev, void *data)
+{
+ struct drm_device *dev = dev_get_drvdata(kdev);
+ struct amdgpu_device *adev = dev->dev_private;
+ struct drm_audio_component *acomp = data;
+
+ acomp->ops = &amdgpu_dm_audio_component_ops;
+ acomp->dev = kdev;
+ adev->dm.audio_component = acomp;
+
+ return 0;
+}
+
+static void amdgpu_dm_audio_component_unbind(struct device *kdev,
+ struct device *hda_kdev, void *data)
+{
+ struct drm_device *dev = dev_get_drvdata(kdev);
+ struct amdgpu_device *adev = dev->dev_private;
+ struct drm_audio_component *acomp = data;
+
+ acomp->ops = NULL;
+ acomp->dev = NULL;
+ adev->dm.audio_component = NULL;
+}
+
+static const struct component_ops amdgpu_dm_audio_component_bind_ops = {
+ .bind = amdgpu_dm_audio_component_bind,
+ .unbind = amdgpu_dm_audio_component_unbind,
+};
+
+static int amdgpu_dm_audio_init(struct amdgpu_device *adev)
+{
+ int i, ret;
+
+ if (!amdgpu_audio)
+ return 0;
+
+ adev->mode_info.audio.enabled = true;
+
+ adev->mode_info.audio.num_pins = adev->dm.dc->res_pool->audio_count;
+
+ for (i = 0; i < adev->mode_info.audio.num_pins; i++) {
+ adev->mode_info.audio.pin[i].channels = -1;
+ adev->mode_info.audio.pin[i].rate = -1;
+ adev->mode_info.audio.pin[i].bits_per_sample = -1;
+ adev->mode_info.audio.pin[i].status_bits = 0;
+ adev->mode_info.audio.pin[i].category_code = 0;
+ adev->mode_info.audio.pin[i].connected = false;
+ adev->mode_info.audio.pin[i].id =
+ adev->dm.dc->res_pool->audios[i]->inst;
+ adev->mode_info.audio.pin[i].offset = 0;
+ }
+
+ ret = component_add(adev->dev, &amdgpu_dm_audio_component_bind_ops);
+ if (ret < 0)
+ return ret;
+
+ adev->dm.audio_registered = true;
+
+ return 0;
+}
+
+static void amdgpu_dm_audio_fini(struct amdgpu_device *adev)
+{
+ if (!amdgpu_audio)
+ return;
+
+ if (!adev->mode_info.audio.enabled)
+ return;
+
+ if (adev->dm.audio_registered) {
+ component_del(adev->dev, &amdgpu_dm_audio_component_bind_ops);
+ adev->dm.audio_registered = false;
+ }
+
+ /* TODO: Disable audio? */
+
+ adev->mode_info.audio.enabled = false;
+}
+
+void amdgpu_dm_audio_eld_notify(struct amdgpu_device *adev, int pin)
+{
+ struct drm_audio_component *acomp = adev->dm.audio_component;
+
+ if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify) {
+ DRM_DEBUG_KMS("Notify ELD: %d\n", pin);
+
+ acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr,
+ pin, -1);
+ }
+}
+
static int amdgpu_dm_init(struct amdgpu_device *adev)
{
struct dc_init_data init_data;
@@ -516,6 +653,7 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
memset(&init_data, 0, sizeof(init_data));
mutex_init(&adev->dm.dc_lock);
+ mutex_init(&adev->dm.audio_lock);
if(amdgpu_dm_irq_init(adev)) {
DRM_ERROR("amdgpu: failed to initialize DM IRQ support.\n");
@@ -558,6 +696,10 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
init_data.flags.power_down_display_on_boot = true;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ init_data.soc_bounding_box = adev->dm.soc_bounding_box;
+#endif
+
/* Display Core create. */
adev->dm.dc = dc_create(&init_data);
@@ -615,7 +757,13 @@ error:
static void amdgpu_dm_fini(struct amdgpu_device *adev)
{
+ amdgpu_dm_audio_fini(adev);
+
amdgpu_dm_destroy_drm_device(&adev->dm);
+
+ /* DC Destroy TODO: Replace destroy DAL */
+ if (adev->dm.dc)
+ dc_destroy(&adev->dm.dc);
/*
* TODO: pageflip, vlank interrupt
*
@@ -630,10 +778,8 @@ static void amdgpu_dm_fini(struct amdgpu_device *adev)
mod_freesync_destroy(adev->dm.freesync_module);
adev->dm.freesync_module = NULL;
}
- /* DC Destroy TODO: Replace destroy DAL */
- if (adev->dm.dc)
- dc_destroy(&adev->dm.dc);
+ mutex_destroy(&adev->dm.audio_lock);
mutex_destroy(&adev->dm.dc_lock);
return;
@@ -662,15 +808,14 @@ static int load_dmcu_fw(struct amdgpu_device *adev)
case CHIP_VEGA10:
case CHIP_VEGA12:
case CHIP_VEGA20:
+ case CHIP_NAVI10:
return 0;
case CHIP_RAVEN:
-#if defined(CONFIG_DRM_AMD_DC_DCN1_01)
if (ASICREV_IS_PICASSO(adev->external_rev_id))
fw_name_dmcu = FIRMWARE_RAVEN_DMCU;
else if (ASICREV_IS_RAVEN2(adev->external_rev_id))
fw_name_dmcu = FIRMWARE_RAVEN_DMCU;
else
-#endif
return 0;
break;
default:
@@ -778,7 +923,7 @@ static int dm_late_init(void *handle)
unsigned int linear_lut[16];
int i;
struct dmcu *dmcu = adev->dm.dc->res_pool->dmcu;
- bool ret;
+ bool ret = false;
for (i = 0; i < 16; i++)
linear_lut[i] = 0xFFFF * i / 15;
@@ -789,10 +934,13 @@ static int dm_late_init(void *handle)
params.backlight_lut_array_size = 16;
params.backlight_lut_array = linear_lut;
- ret = dmcu_load_iram(dmcu, params);
+ /* todo will enable for navi10 */
+ if (adev->asic_type <= CHIP_RAVEN) {
+ ret = dmcu_load_iram(dmcu, params);
- if (!ret)
- return -EINVAL;
+ if (!ret)
+ return -EINVAL;
+ }
return detect_mst_link_for_all_connectors(adev->ddev);
}
@@ -1526,10 +1674,7 @@ static int dce110_register_irq_handlers(struct amdgpu_device *adev)
int i;
unsigned client_id = AMDGPU_IRQ_CLIENTID_LEGACY;
- if (adev->asic_type == CHIP_VEGA10 ||
- adev->asic_type == CHIP_VEGA12 ||
- adev->asic_type == CHIP_VEGA20 ||
- adev->asic_type == CHIP_RAVEN)
+ if (adev->asic_type >= CHIP_VEGA10)
client_id = SOC15_IH_CLIENTID_DCE;
int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
@@ -1882,6 +2027,10 @@ static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev)
if (r)
return r;
+ r = amdgpu_dm_audio_init(adev);
+ if (r)
+ return r;
+
return 0;
}
@@ -2208,6 +2357,9 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
break;
#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
case CHIP_RAVEN:
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ case CHIP_NAVI10:
+#endif
if (dcn10_register_irq_handlers(dm->adev)) {
DRM_ERROR("DM: Failed to initialize IRQ\n");
goto fail;
@@ -2361,6 +2513,13 @@ static int dm_early_init(void *handle)
adev->mode_info.num_dig = 4;
break;
#endif
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ case CHIP_NAVI10:
+ adev->mode_info.num_crtc = 6;
+ adev->mode_info.num_hpd = 6;
+ adev->mode_info.num_dig = 6;
+ break;
+#endif
default:
DRM_ERROR("Unsupported ASIC type: 0x%X\n", adev->asic_type);
return -EINVAL;
@@ -2592,7 +2751,7 @@ fill_plane_buffer_attributes(struct amdgpu_device *adev,
address->type = PLN_ADDR_TYPE_GRAPHICS;
address->grph.addr.low_part = lower_32_bits(afb->address);
address->grph.addr.high_part = upper_32_bits(afb->address);
- } else {
+ } else if (format < SURFACE_PIXEL_FORMAT_INVALID) {
uint64_t chroma_addr = afb->address + fb->offsets[1];
plane_size->video.luma_size.x = 0;
@@ -2653,6 +2812,9 @@ fill_plane_buffer_attributes(struct amdgpu_device *adev,
if (adev->asic_type == CHIP_VEGA10 ||
adev->asic_type == CHIP_VEGA12 ||
adev->asic_type == CHIP_VEGA20 ||
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ adev->asic_type == CHIP_NAVI10 ||
+#endif
adev->asic_type == CHIP_RAVEN) {
/* Fill GFX9 params */
tiling_info->gfx9.num_pipes =
@@ -2858,6 +3020,7 @@ static int fill_dc_plane_attributes(struct amdgpu_device *adev,
struct drm_plane_state *plane_state,
struct drm_crtc_state *crtc_state)
{
+ struct dm_crtc_state *dm_crtc_state = to_dm_crtc_state(crtc_state);
const struct amdgpu_framebuffer *amdgpu_fb =
to_amdgpu_framebuffer(plane_state->fb);
struct dc_scaling_info scaling_info;
@@ -2902,13 +3065,11 @@ static int fill_dc_plane_attributes(struct amdgpu_device *adev,
* Always set input transfer function, since plane state is refreshed
* every time.
*/
- ret = amdgpu_dm_set_degamma_lut(crtc_state, dc_plane_state);
- if (ret) {
- dc_transfer_func_release(dc_plane_state->in_transfer_func);
- dc_plane_state->in_transfer_func = NULL;
- }
+ ret = amdgpu_dm_update_plane_color_mgmt(dm_crtc_state, dc_plane_state);
+ if (ret)
+ return ret;
- return ret;
+ return 0;
}
static void update_stream_scaling_settings(const struct drm_display_mode *mode,
@@ -2967,16 +3128,31 @@ static void update_stream_scaling_settings(const struct drm_display_mode *mode,
}
static enum dc_color_depth
-convert_color_depth_from_display_info(const struct drm_connector *connector)
+convert_color_depth_from_display_info(const struct drm_connector *connector,
+ const struct drm_connector_state *state)
{
- struct dm_connector_state *dm_conn_state =
- to_dm_connector_state(connector->state);
- uint32_t bpc = connector->display_info.bpc;
+ uint8_t bpc = (uint8_t)connector->display_info.bpc;
+
+ /* Assume 8 bpc by default if no bpc is specified. */
+ bpc = bpc ? bpc : 8;
+
+ if (!state)
+ state = connector->state;
- /* TODO: Remove this when there's support for max_bpc in drm */
- if (dm_conn_state && bpc > dm_conn_state->max_bpc)
- /* Round down to nearest even number. */
- bpc = dm_conn_state->max_bpc - (dm_conn_state->max_bpc & 1);
+ if (state) {
+ /*
+ * Cap display bpc based on the user requested value.
+ *
+ * The value for state->max_bpc may not correctly updated
+ * depending on when the connector gets added to the state
+ * or if this was called outside of atomic check, so it
+ * can't be used directly.
+ */
+ bpc = min(bpc, state->max_requested_bpc);
+
+ /* Round down to the nearest even number. */
+ bpc = bpc - (bpc & 1);
+ }
switch (bpc) {
case 0:
@@ -3094,11 +3270,12 @@ static void adjust_colour_depth_from_display_info(struct dc_crtc_timing *timing_
}
-static void
-fill_stream_properties_from_drm_display_mode(struct dc_stream_state *stream,
- const struct drm_display_mode *mode_in,
- const struct drm_connector *connector,
- const struct dc_stream_state *old_stream)
+static void fill_stream_properties_from_drm_display_mode(
+ struct dc_stream_state *stream,
+ const struct drm_display_mode *mode_in,
+ const struct drm_connector *connector,
+ const struct drm_connector_state *connector_state,
+ const struct dc_stream_state *old_stream)
{
struct dc_crtc_timing *timing_out = &stream->timing;
const struct drm_display_info *info = &connector->display_info;
@@ -3121,7 +3298,7 @@ fill_stream_properties_from_drm_display_mode(struct dc_stream_state *stream,
timing_out->timing_3d_format = TIMING_3D_FORMAT_NONE;
timing_out->display_color_depth = convert_color_depth_from_display_info(
- connector);
+ connector, connector_state);
timing_out->scan_type = SCANNING_TYPE_NODATA;
timing_out->hdmi_vic = 0;
@@ -3318,6 +3495,8 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector,
{
struct drm_display_mode *preferred_mode = NULL;
struct drm_connector *drm_connector;
+ const struct drm_connector_state *con_state =
+ dm_state ? &dm_state->base : NULL;
struct dc_stream_state *stream = NULL;
struct drm_display_mode mode = *drm_mode;
bool native_mode_found = false;
@@ -3390,10 +3569,24 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector,
*/
if (!scale || mode_refresh != preferred_refresh)
fill_stream_properties_from_drm_display_mode(stream,
- &mode, &aconnector->base, NULL);
+ &mode, &aconnector->base, con_state, NULL);
else
fill_stream_properties_from_drm_display_mode(stream,
- &mode, &aconnector->base, old_stream);
+ &mode, &aconnector->base, con_state, old_stream);
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ /* stream->timing.flags.DSC = 0; */
+ /* */
+ /* if (aconnector->dc_link && */
+ /* aconnector->dc_link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT #<{(|&& */
+ /* aconnector->dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.is_dsc_supported|)}>#) */
+ /* if (dc_dsc_compute_config(aconnector->dc_link->ctx->dc, */
+ /* &aconnector->dc_link->dpcd_caps.dsc_caps, */
+ /* dc_link_bandwidth_kbps(aconnector->dc_link, dc_link_get_link_cap(aconnector->dc_link)), */
+ /* &stream->timing, */
+ /* &stream->timing.dsc_cfg)) */
+ /* stream->timing.flags.DSC = 1; */
+#endif
update_stream_scaling_settings(&mode, dm_state, stream);
@@ -3477,6 +3670,8 @@ dm_crtc_duplicate_state(struct drm_crtc *crtc)
state->vrr_supported = cur->vrr_supported;
state->freesync_config = cur->freesync_config;
state->crc_enabled = cur->crc_enabled;
+ state->cm_has_degamma = cur->cm_has_degamma;
+ state->cm_is_degamma_srgb = cur->cm_is_degamma_srgb;
/* TODO Duplicate dc_stream after objects are stream object is flattened */
@@ -3618,9 +3813,6 @@ int amdgpu_dm_connector_atomic_set_property(struct drm_connector *connector,
} else if (property == adev->mode_info.underscan_property) {
dm_new_state->underscan_enable = val;
ret = 0;
- } else if (property == adev->mode_info.max_bpc_property) {
- dm_new_state->max_bpc = val;
- ret = 0;
} else if (property == adev->mode_info.abm_level_property) {
dm_new_state->abm_level = val;
ret = 0;
@@ -3666,9 +3858,6 @@ int amdgpu_dm_connector_atomic_get_property(struct drm_connector *connector,
} else if (property == adev->mode_info.underscan_property) {
*val = dm_state->underscan_enable;
ret = 0;
- } else if (property == adev->mode_info.max_bpc_property) {
- *val = dm_state->max_bpc;
- ret = 0;
} else if (property == adev->mode_info.abm_level_property) {
*val = dm_state->abm_level;
ret = 0;
@@ -3677,6 +3866,13 @@ int amdgpu_dm_connector_atomic_get_property(struct drm_connector *connector,
return ret;
}
+static void amdgpu_dm_connector_unregister(struct drm_connector *connector)
+{
+ struct amdgpu_dm_connector *amdgpu_dm_connector = to_amdgpu_dm_connector(connector);
+
+ drm_dp_aux_unregister(&amdgpu_dm_connector->dm_dp_aux.aux);
+}
+
static void amdgpu_dm_connector_destroy(struct drm_connector *connector)
{
struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
@@ -3705,6 +3901,11 @@ static void amdgpu_dm_connector_destroy(struct drm_connector *connector)
drm_dp_cec_unregister_connector(&aconnector->dm_dp_aux.aux);
drm_connector_unregister(connector);
drm_connector_cleanup(connector);
+ if (aconnector->i2c) {
+ i2c_del_adapter(&aconnector->i2c->base);
+ kfree(aconnector->i2c);
+ }
+
kfree(connector);
}
@@ -3725,7 +3926,10 @@ void amdgpu_dm_connector_funcs_reset(struct drm_connector *connector)
state->underscan_enable = false;
state->underscan_hborder = 0;
state->underscan_vborder = 0;
- state->max_bpc = 8;
+ state->base.max_requested_bpc = 8;
+
+ if (connector->connector_type == DRM_MODE_CONNECTOR_eDP)
+ state->abm_level = amdgpu_dm_abm_level;
__drm_atomic_helper_connector_reset(connector, &state->base);
}
@@ -3751,7 +3955,6 @@ amdgpu_dm_connector_atomic_duplicate_state(struct drm_connector *connector)
new_state->underscan_enable = state->underscan_enable;
new_state->underscan_hborder = state->underscan_hborder;
new_state->underscan_vborder = state->underscan_vborder;
- new_state->max_bpc = state->max_bpc;
return &new_state->base;
}
@@ -3764,7 +3967,8 @@ static const struct drm_connector_funcs amdgpu_dm_connector_funcs = {
.atomic_duplicate_state = amdgpu_dm_connector_atomic_duplicate_state,
.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
.atomic_set_property = amdgpu_dm_connector_atomic_set_property,
- .atomic_get_property = amdgpu_dm_connector_atomic_get_property
+ .atomic_get_property = amdgpu_dm_connector_atomic_get_property,
+ .early_unregister = amdgpu_dm_connector_unregister
};
static int get_modes(struct drm_connector *connector)
@@ -3879,6 +4083,129 @@ fail:
return result;
}
+static int fill_hdr_info_packet(const struct drm_connector_state *state,
+ struct dc_info_packet *out)
+{
+ struct hdmi_drm_infoframe frame;
+ unsigned char buf[30]; /* 26 + 4 */
+ ssize_t len;
+ int ret, i;
+
+ memset(out, 0, sizeof(*out));
+
+ if (!state->hdr_output_metadata)
+ return 0;
+
+ ret = drm_hdmi_infoframe_set_hdr_metadata(&frame, state);
+ if (ret)
+ return ret;
+
+ len = hdmi_drm_infoframe_pack_only(&frame, buf, sizeof(buf));
+ if (len < 0)
+ return (int)len;
+
+ /* Static metadata is a fixed 26 bytes + 4 byte header. */
+ if (len != 30)
+ return -EINVAL;
+
+ /* Prepare the infopacket for DC. */
+ switch (state->connector->connector_type) {
+ case DRM_MODE_CONNECTOR_HDMIA:
+ out->hb0 = 0x87; /* type */
+ out->hb1 = 0x01; /* version */
+ out->hb2 = 0x1A; /* length */
+ out->sb[0] = buf[3]; /* checksum */
+ i = 1;
+ break;
+
+ case DRM_MODE_CONNECTOR_DisplayPort:
+ case DRM_MODE_CONNECTOR_eDP:
+ out->hb0 = 0x00; /* sdp id, zero */
+ out->hb1 = 0x87; /* type */
+ out->hb2 = 0x1D; /* payload len - 1 */
+ out->hb3 = (0x13 << 2); /* sdp version */
+ out->sb[0] = 0x01; /* version */
+ out->sb[1] = 0x1A; /* length */
+ i = 2;
+ break;
+
+ default:
+ return -EINVAL;
+ }
+
+ memcpy(&out->sb[i], &buf[4], 26);
+ out->valid = true;
+
+ print_hex_dump(KERN_DEBUG, "HDR SB:", DUMP_PREFIX_NONE, 16, 1, out->sb,
+ sizeof(out->sb), false);
+
+ return 0;
+}
+
+static bool
+is_hdr_metadata_different(const struct drm_connector_state *old_state,
+ const struct drm_connector_state *new_state)
+{
+ struct drm_property_blob *old_blob = old_state->hdr_output_metadata;
+ struct drm_property_blob *new_blob = new_state->hdr_output_metadata;
+
+ if (old_blob != new_blob) {
+ if (old_blob && new_blob &&
+ old_blob->length == new_blob->length)
+ return memcmp(old_blob->data, new_blob->data,
+ old_blob->length);
+
+ return true;
+ }
+
+ return false;
+}
+
+static int
+amdgpu_dm_connector_atomic_check(struct drm_connector *conn,
+ struct drm_atomic_state *state)
+{
+ struct drm_connector_state *new_con_state =
+ drm_atomic_get_new_connector_state(state, conn);
+ struct drm_connector_state *old_con_state =
+ drm_atomic_get_old_connector_state(state, conn);
+ struct drm_crtc *crtc = new_con_state->crtc;
+ struct drm_crtc_state *new_crtc_state;
+ int ret;
+
+ if (!crtc)
+ return 0;
+
+ if (is_hdr_metadata_different(old_con_state, new_con_state)) {
+ struct dc_info_packet hdr_infopacket;
+
+ ret = fill_hdr_info_packet(new_con_state, &hdr_infopacket);
+ if (ret)
+ return ret;
+
+ new_crtc_state = drm_atomic_get_crtc_state(state, crtc);
+ if (IS_ERR(new_crtc_state))
+ return PTR_ERR(new_crtc_state);
+
+ /*
+ * DC considers the stream backends changed if the
+ * static metadata changes. Forcing the modeset also
+ * gives a simple way for userspace to switch from
+ * 8bpc to 10bpc when setting the metadata to enter
+ * or exit HDR.
+ *
+ * Changing the static metadata after it's been
+ * set is permissible, however. So only force a
+ * modeset if we're entering or exiting HDR.
+ */
+ new_crtc_state->mode_changed =
+ !old_con_state->hdr_output_metadata ||
+ !new_con_state->hdr_output_metadata;
+ }
+
+ return 0;
+}
+
static const struct drm_connector_helper_funcs
amdgpu_dm_connector_helper_funcs = {
/*
@@ -3889,6 +4216,7 @@ amdgpu_dm_connector_helper_funcs = {
*/
.get_modes = get_modes,
.mode_valid = amdgpu_dm_connector_mode_valid,
+ .atomic_check = amdgpu_dm_connector_atomic_check,
};
static void dm_crtc_helper_disable(struct drm_crtc *crtc)
@@ -4098,6 +4426,9 @@ static int dm_plane_helper_prepare_fb(struct drm_plane *plane,
struct amdgpu_device *adev;
struct amdgpu_bo *rbo;
struct dm_plane_state *dm_plane_state_new, *dm_plane_state_old;
+ struct list_head list;
+ struct ttm_validate_buffer tv;
+ struct ww_acquire_ctx ticket;
uint64_t tiling_flags;
uint32_t domain;
int r;
@@ -4114,9 +4445,17 @@ static int dm_plane_helper_prepare_fb(struct drm_plane *plane,
obj = new_state->fb->obj[0];
rbo = gem_to_amdgpu_bo(obj);
adev = amdgpu_ttm_adev(rbo->tbo.bdev);
- r = amdgpu_bo_reserve(rbo, false);
- if (unlikely(r != 0))
+ INIT_LIST_HEAD(&list);
+
+ tv.bo = &rbo->tbo;
+ tv.num_shared = 1;
+ list_add(&tv.head, &list);
+
+ r = ttm_eu_reserve_buffers(&ticket, &list, false, NULL, true);
+ if (r) {
+ dev_err(adev->dev, "fail to reserve bo (%d)\n", r);
return r;
+ }
if (plane->type != DRM_PLANE_TYPE_CURSOR)
domain = amdgpu_display_supported_domains(adev);
@@ -4127,21 +4466,21 @@ static int dm_plane_helper_prepare_fb(struct drm_plane *plane,
if (unlikely(r != 0)) {
if (r != -ERESTARTSYS)
DRM_ERROR("Failed to pin framebuffer with error %d\n", r);
- amdgpu_bo_unreserve(rbo);
+ ttm_eu_backoff_reservation(&ticket, &list);
return r;
}
r = amdgpu_ttm_alloc_gart(&rbo->tbo);
if (unlikely(r != 0)) {
amdgpu_bo_unpin(rbo);
- amdgpu_bo_unreserve(rbo);
+ ttm_eu_backoff_reservation(&ticket, &list);
DRM_ERROR("%p bind failed\n", rbo);
return r;
}
amdgpu_bo_get_tiling_flags(rbo, &tiling_flags);
- amdgpu_bo_unreserve(rbo);
+ ttm_eu_backoff_reservation(&ticket, &list);
afb->address = amdgpu_bo_gpu_offset(rbo);
@@ -4592,6 +4931,15 @@ static void amdgpu_dm_connector_ddc_get_modes(struct drm_connector *connector,
amdgpu_dm_connector->num_modes =
drm_add_edid_modes(connector, edid);
+ /* sorting the probed modes before calling function
+ * amdgpu_dm_get_native_mode() since EDID can have
+ * more than one preferred mode. The modes that are
+ * later in the probed mode list could be of higher
+ * and preferred resolution. For example, 3840x2160
+ * resolution in base EDID preferred timing and 4096x2160
+ * preferred resolution in DID extension block later.
+ */
+ drm_mode_sort(&connector->probed_modes);
amdgpu_dm_get_native_mode(connector);
} else {
amdgpu_dm_connector->num_modes = 0;
@@ -4627,6 +4975,13 @@ void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm,
{
struct amdgpu_device *adev = dm->ddev->dev_private;
+ /*
+ * Some of the properties below require access to state, like bpc.
+ * Allocate some default initial connector state with our reset helper.
+ */
+ if (aconnector->base.funcs->reset)
+ aconnector->base.funcs->reset(&aconnector->base);
+
aconnector->connector_id = link_index;
aconnector->dc_link = link;
aconnector->base.interlace_allowed = false;
@@ -4634,6 +4989,7 @@ void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm,
aconnector->base.stereo_allowed = false;
aconnector->base.dpms = DRM_MODE_DPMS_OFF;
aconnector->hpd.hpd = AMDGPU_HPD_NONE; /* not used */
+ aconnector->audio_inst = -1;
mutex_init(&aconnector->hpd_lock);
/*
@@ -4671,9 +5027,12 @@ void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm,
drm_object_attach_property(&aconnector->base.base,
adev->mode_info.underscan_vborder_property,
0);
- drm_object_attach_property(&aconnector->base.base,
- adev->mode_info.max_bpc_property,
- 0);
+
+ drm_connector_attach_max_bpc_property(&aconnector->base, 8, 16);
+
+ /* This defaults to the max in the range, but we want 8bpc. */
+ aconnector->base.state->max_bpc = 8;
+ aconnector->base.state->max_requested_bpc = 8;
if (connector_type == DRM_MODE_CONNECTOR_eDP &&
dc_is_dmcu_initialized(adev->dm.dc)) {
@@ -4684,6 +5043,10 @@ void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm,
if (connector_type == DRM_MODE_CONNECTOR_HDMIA ||
connector_type == DRM_MODE_CONNECTOR_DisplayPort ||
connector_type == DRM_MODE_CONNECTOR_eDP) {
+ drm_object_attach_property(
+ &aconnector->base.base,
+ dm->ddev->mode_config.hdr_output_metadata_property, 0);
+
drm_connector_attach_vrr_capable_property(
&aconnector->base);
}
@@ -4809,9 +5172,6 @@ static int amdgpu_dm_connector_init(struct amdgpu_display_manager *dm,
&aconnector->base,
&amdgpu_dm_connector_helper_funcs);
- if (aconnector->base.funcs->reset)
- aconnector->base.funcs->reset(&aconnector->base);
-
amdgpu_dm_connector_init_helper(
dm,
aconnector,
@@ -4824,11 +5184,7 @@ static int amdgpu_dm_connector_init(struct amdgpu_display_manager *dm,
drm_connector_register(&aconnector->base);
#if defined(CONFIG_DEBUG_FS)
- res = connector_debugfs_init(aconnector);
- if (res) {
- DRM_ERROR("Failed to create debugfs for connector");
- goto out_free;
- }
+ connector_debugfs_init(aconnector);
aconnector->debugfs_dpcd_address = 0;
aconnector->debugfs_dpcd_size = 0;
#endif
@@ -4952,12 +5308,12 @@ static int get_cursor_position(struct drm_plane *plane, struct drm_crtc *crtc,
int x, y;
int xorigin = 0, yorigin = 0;
- if (!crtc || !plane->state->fb) {
- position->enable = false;
- position->x = 0;
- position->y = 0;
+ position->enable = false;
+ position->x = 0;
+ position->y = 0;
+
+ if (!crtc || !plane->state->fb)
return 0;
- }
if ((plane->state->crtc_w > amdgpu_crtc->max_cursor_width) ||
(plane->state->crtc_h > amdgpu_crtc->max_cursor_height)) {
@@ -4971,6 +5327,10 @@ static int get_cursor_position(struct drm_plane *plane, struct drm_crtc *crtc,
x = plane->state->crtc_x;
y = plane->state->crtc_y;
+ if (x <= -amdgpu_crtc->max_cursor_width ||
+ y <= -amdgpu_crtc->max_cursor_height)
+ return 0;
+
if (crtc->primary->state) {
/* avivo cursor are offset into the total surface */
x += crtc->primary->state->src_x >> 16;
@@ -5114,6 +5474,11 @@ static void update_freesync_state_on_stream(
amdgpu_dm_vrr_active(new_crtc_state)) {
mod_freesync_handle_v_update(dm->freesync_module,
new_stream, &vrr_params);
+
+ /* Need to call this before the frame ends. */
+ dc_stream_adjust_vmin_vmax(dm->dc,
+ new_crtc_state->stream,
+ &vrr_params.adjust);
}
}
@@ -5452,11 +5817,6 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
}
if (acrtc_state->stream) {
-
- if (acrtc_state->freesync_timing_changed)
- bundle->stream_update.adjust =
- &acrtc_state->stream->adjust;
-
if (acrtc_state->freesync_vrr_info_changed)
bundle->stream_update.vrr_infopacket =
&acrtc_state->stream->vrr_infopacket;
@@ -5464,19 +5824,44 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
}
/* Update the planes if changed or disable if we don't have any. */
- if (planes_count || acrtc_state->active_planes == 0) {
+ if ((planes_count || acrtc_state->active_planes == 0) &&
+ acrtc_state->stream) {
if (new_pcrtc_state->mode_changed) {
bundle->stream_update.src = acrtc_state->stream->src;
bundle->stream_update.dst = acrtc_state->stream->dst;
}
- if (new_pcrtc_state->color_mgmt_changed)
- bundle->stream_update.out_transfer_func = acrtc_state->stream->out_transfer_func;
+ if (new_pcrtc_state->color_mgmt_changed) {
+ /*
+ * TODO: This isn't fully correct since we've actually
+ * already modified the stream in place.
+ */
+ bundle->stream_update.gamut_remap =
+ &acrtc_state->stream->gamut_remap_matrix;
+ bundle->stream_update.output_csc_transform =
+ &acrtc_state->stream->csc_color_matrix;
+ bundle->stream_update.out_transfer_func =
+ acrtc_state->stream->out_transfer_func;
+ }
acrtc_state->stream->abm_level = acrtc_state->abm_level;
if (acrtc_state->abm_level != dm_old_crtc_state->abm_level)
bundle->stream_update.abm_level = &acrtc_state->abm_level;
+ /*
+ * If FreeSync state on the stream has changed then we need to
+ * re-adjust the min/max bounds now that DC doesn't handle this
+ * as part of commit.
+ */
+ if (amdgpu_dm_vrr_active(dm_old_crtc_state) !=
+ amdgpu_dm_vrr_active(acrtc_state)) {
+ spin_lock_irqsave(&pcrtc->dev->event_lock, flags);
+ dc_stream_adjust_vmin_vmax(
+ dm->dc, acrtc_state->stream,
+ &acrtc_state->vrr_params.adjust);
+ spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags);
+ }
+
mutex_lock(&dm->dc_lock);
dc_commit_updates_for_stream(dm->dc,
bundle->surface_updates,
@@ -5499,6 +5884,81 @@ cleanup:
kfree(bundle);
}
+static void amdgpu_dm_commit_audio(struct drm_device *dev,
+ struct drm_atomic_state *state)
+{
+ struct amdgpu_device *adev = dev->dev_private;
+ struct amdgpu_dm_connector *aconnector;
+ struct drm_connector *connector;
+ struct drm_connector_state *old_con_state, *new_con_state;
+ struct drm_crtc_state *new_crtc_state;
+ struct dm_crtc_state *new_dm_crtc_state;
+ const struct dc_stream_status *status;
+ int i, inst;
+
+ /* Notify device removals. */
+ for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) {
+ if (old_con_state->crtc != new_con_state->crtc) {
+ /* CRTC changes require notification. */
+ goto notify;
+ }
+
+ if (!new_con_state->crtc)
+ continue;
+
+ new_crtc_state = drm_atomic_get_new_crtc_state(
+ state, new_con_state->crtc);
+
+ if (!new_crtc_state)
+ continue;
+
+ if (!drm_atomic_crtc_needs_modeset(new_crtc_state))
+ continue;
+
+ notify:
+ aconnector = to_amdgpu_dm_connector(connector);
+
+ mutex_lock(&adev->dm.audio_lock);
+ inst = aconnector->audio_inst;
+ aconnector->audio_inst = -1;
+ mutex_unlock(&adev->dm.audio_lock);
+
+ amdgpu_dm_audio_eld_notify(adev, inst);
+ }
+
+ /* Notify audio device additions. */
+ for_each_new_connector_in_state(state, connector, new_con_state, i) {
+ if (!new_con_state->crtc)
+ continue;
+
+ new_crtc_state = drm_atomic_get_new_crtc_state(
+ state, new_con_state->crtc);
+
+ if (!new_crtc_state)
+ continue;
+
+ if (!drm_atomic_crtc_needs_modeset(new_crtc_state))
+ continue;
+
+ new_dm_crtc_state = to_dm_crtc_state(new_crtc_state);
+ if (!new_dm_crtc_state->stream)
+ continue;
+
+ status = dc_stream_get_status(new_dm_crtc_state->stream);
+ if (!status)
+ continue;
+
+ aconnector = to_amdgpu_dm_connector(connector);
+
+ mutex_lock(&adev->dm.audio_lock);
+ inst = status->audio_inst;
+ aconnector->audio_inst = inst;
+ mutex_unlock(&adev->dm.audio_lock);
+
+ amdgpu_dm_audio_eld_notify(adev, inst);
+ }
+}
+
/*
* Enable interrupts on CRTCs that are newly active, undergone
* a modeset, or have active planes again.
@@ -5768,7 +6228,9 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
struct amdgpu_crtc *acrtc = to_amdgpu_crtc(dm_new_con_state->base.crtc);
struct dc_surface_update dummy_updates[MAX_SURFACES];
struct dc_stream_update stream_update;
+ struct dc_info_packet hdr_packet;
struct dc_stream_status *status = NULL;
+ bool abm_changed, hdr_changed, scaling_changed;
memset(&dummy_updates, 0, sizeof(dummy_updates));
memset(&stream_update, 0, sizeof(stream_update));
@@ -5785,11 +6247,19 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
dm_old_crtc_state = to_dm_crtc_state(old_crtc_state);
- if (!is_scaling_state_different(dm_new_con_state, dm_old_con_state) &&
- (dm_new_crtc_state->abm_level == dm_old_crtc_state->abm_level))
+ scaling_changed = is_scaling_state_different(dm_new_con_state,
+ dm_old_con_state);
+
+ abm_changed = dm_new_crtc_state->abm_level !=
+ dm_old_crtc_state->abm_level;
+
+ hdr_changed =
+ is_hdr_metadata_different(old_con_state, new_con_state);
+
+ if (!scaling_changed && !abm_changed && !hdr_changed)
continue;
- if (is_scaling_state_different(dm_new_con_state, dm_old_con_state)) {
+ if (scaling_changed) {
update_stream_scaling_settings(&dm_new_con_state->base.crtc->mode,
dm_new_con_state, (struct dc_stream_state *)dm_new_crtc_state->stream);
@@ -5797,12 +6267,17 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
stream_update.dst = dm_new_crtc_state->stream->dst;
}
- if (dm_new_crtc_state->abm_level != dm_old_crtc_state->abm_level) {
+ if (abm_changed) {
dm_new_crtc_state->stream->abm_level = dm_new_crtc_state->abm_level;
stream_update.abm_level = &dm_new_crtc_state->abm_level;
}
+ if (hdr_changed) {
+ fill_hdr_info_packet(new_con_state, &hdr_packet);
+ stream_update.hdr_static_metadata = &hdr_packet;
+ }
+
status = dc_stream_get_status(dm_new_crtc_state->stream);
WARN_ON(!status);
WARN_ON(!status->plane_count);
@@ -5862,6 +6337,9 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
/* Enable interrupts for CRTCs going from 0 to n active planes. */
amdgpu_dm_enable_crtc_interrupts(dev, state, false);
+ /* Update audio instances for each connector. */
+ amdgpu_dm_commit_audio(dev, state);
+
/*
* send vblank event on all events not handled in flip and
* mark consumed event for drm_atomic_helper_commit_hw_done
@@ -6148,7 +6626,22 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm,
dm_new_crtc_state->abm_level = dm_new_conn_state->abm_level;
- if (dc_is_stream_unchanged(new_stream, dm_old_crtc_state->stream) &&
+ ret = fill_hdr_info_packet(drm_new_conn_state,
+ &new_stream->hdr_static_metadata);
+ if (ret)
+ goto fail;
+
+ /*
+ * If we already removed the old stream from the context
+ * (and set the new stream to NULL) then we can't reuse
+ * the old stream even if the stream and scaling are unchanged.
+ * We'll hit the BUG_ON and black screen.
+ *
+ * TODO: Refactor this function to allow this check to work
+ * in all conditions.
+ */
+ if (dm_new_crtc_state->stream &&
+ dc_is_stream_unchanged(new_stream, dm_old_crtc_state->stream) &&
dc_is_stream_scaling_unchanged(new_stream, dm_old_crtc_state->stream)) {
new_crtc_state->mode_changed = false;
DRM_DEBUG_DRIVER("Mode change not required, setting mode_changed to %d",
@@ -6277,10 +6770,9 @@ skip_modeset:
*/
if (dm_new_crtc_state->base.color_mgmt_changed ||
drm_atomic_crtc_needs_modeset(new_crtc_state)) {
- ret = amdgpu_dm_set_regamma_lut(dm_new_crtc_state);
+ ret = amdgpu_dm_update_crtc_color_mgmt(dm_new_crtc_state);
if (ret)
goto fail;
- amdgpu_dm_set_ctm(dm_new_crtc_state);
}
/* Update Freesync settings. */
@@ -6327,6 +6819,10 @@ static bool should_reset_plane(struct drm_atomic_state *state,
if (!new_crtc_state)
return true;
+ /* CRTC Degamma changes currently require us to recreate planes. */
+ if (new_crtc_state->color_mgmt_changed)
+ return true;
+
if (drm_atomic_crtc_needs_modeset(new_crtc_state))
return true;
@@ -6579,6 +7075,8 @@ dm_determine_update_type_for_commit(struct amdgpu_display_manager *dm,
new_dm_plane_state->dc_state->in_transfer_func;
stream_update.gamut_remap =
&new_dm_crtc_state->stream->gamut_remap_matrix;
+ stream_update.output_csc_transform =
+ &new_dm_crtc_state->stream->csc_color_matrix;
stream_update.out_transfer_func =
new_dm_crtc_state->stream->out_transfer_func;
}
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
index 978ff14a7d45..b89cbbfcc0e9 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
@@ -26,8 +26,11 @@
#ifndef __AMDGPU_DM_H__
#define __AMDGPU_DM_H__
-#include <drm/drmP.h>
#include <drm/drm_atomic.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_dp_mst_helper.h>
+#include <drm/drm_plane.h>
/*
* This file contains the definition for amdgpu_display_manager
@@ -141,6 +144,28 @@ struct amdgpu_display_manager {
struct mutex dc_lock;
/**
+ * @audio_lock:
+ *
+ * Guards access to audio instance changes.
+ */
+ struct mutex audio_lock;
+
+ /**
+ * @audio_component:
+ *
+ * Used to notify ELD changes to sound driver.
+ */
+ struct drm_audio_component *audio_component;
+
+ /**
+ * @audio_registered:
+ *
+ * True if the audio component has been registered
+ * successfully, false otherwise.
+ */
+ bool audio_registered;
+
+ /**
* @irq_handler_list_low_tab:
*
* Low priority IRQ handler table.
@@ -206,6 +231,13 @@ struct amdgpu_display_manager {
const struct firmware *fw_dmcu;
uint32_t dmcu_fw_version;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ /**
+ * gpu_info FW provided soc bounding box struct or 0 if not
+ * available in FW
+ */
+ const struct gpu_info_soc_bounding_box_v1_0 *soc_bounding_box;
+#endif
};
struct amdgpu_dm_connector {
@@ -244,6 +276,9 @@ struct amdgpu_dm_connector {
int max_vfreq ;
int pixel_clock_mhz;
+ /* Audio instance - protected by audio_lock. */
+ int audio_inst;
+
struct mutex hpd_lock;
bool fake_enable;
@@ -271,6 +306,9 @@ struct dm_crtc_state {
struct drm_crtc_state base;
struct dc_stream_state *stream;
+ bool cm_has_degamma;
+ bool cm_is_degamma_srgb;
+
int active_planes;
bool interrupts_enabled;
@@ -304,7 +342,6 @@ struct dm_connector_state {
enum amdgpu_rmx_type scaling;
uint8_t underscan_vborder;
uint8_t underscan_hborder;
- uint8_t max_bpc;
bool underscan_enable;
bool freesync_capable;
uint8_t abm_level;
@@ -361,10 +398,9 @@ void amdgpu_dm_crtc_handle_crc_irq(struct drm_crtc *crtc);
#define MAX_COLOR_LEGACY_LUT_ENTRIES 256
void amdgpu_dm_init_color_mod(void);
-int amdgpu_dm_set_degamma_lut(struct drm_crtc_state *crtc_state,
- struct dc_plane_state *dc_plane_state);
-void amdgpu_dm_set_ctm(struct dm_crtc_state *crtc);
-int amdgpu_dm_set_regamma_lut(struct dm_crtc_state *crtc);
+int amdgpu_dm_update_crtc_color_mgmt(struct dm_crtc_state *crtc);
+int amdgpu_dm_update_plane_color_mgmt(struct dm_crtc_state *crtc,
+ struct dc_plane_state *dc_plane_state);
extern const struct drm_encoder_helper_funcs amdgpu_dm_encoder_helper_funcs;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
index 7258c992a2bf..b43bb7f90e4e 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
@@ -27,6 +27,47 @@
#include "amdgpu_dm.h"
#include "dc.h"
#include "modules/color/color_gamma.h"
+#include "basics/conversion.h"
+
+/*
+ * The DC interface to HW gives us the following color management blocks
+ * per pipe (surface):
+ *
+ * - Input gamma LUT (de-normalized)
+ * - Input CSC (normalized)
+ * - Surface degamma LUT (normalized)
+ * - Surface CSC (normalized)
+ * - Surface regamma LUT (normalized)
+ * - Output CSC (normalized)
+ *
+ * But these aren't a direct mapping to DRM color properties. The current DRM
+ * interface exposes CRTC degamma, CRTC CTM and CRTC regamma while our hardware
+ * is essentially giving:
+ *
+ * Plane CTM -> Plane degamma -> Plane CTM -> Plane regamma -> Plane CTM
+ *
+ * The input gamma LUT block isn't really applicable here since it operates
+ * on the actual input data itself rather than the HW fp representation. The
+ * input and output CSC blocks are technically available to use as part of
+ * the DC interface but are typically used internally by DC for conversions
+ * between color spaces. These could be blended together with user
+ * adjustments in the future but for now these should remain untouched.
+ *
+ * The pipe blending also happens after these blocks so we don't actually
+ * support any CRTC props with correct blending with multiple planes - but we
+ * can still support CRTC color management properties in DM in most single
+ * plane cases correctly with clever management of the DC interface in DM.
+ *
+ * As per DRM documentation, blocks should be in hardware bypass when their
+ * respective property is set to NULL. A linear DGM/RGM LUT should also
+ * considered as putting the respective block into bypass mode.
+ *
+ * This means that the following
+ * configuration is assumed to be the default:
+ *
+ * Plane DGM Bypass -> Plane CTM Bypass -> Plane RGM Bypass -> ...
+ * CRTC DGM Bypass -> CRTC CTM Bypass -> CRTC RGM Bypass
+ */
#define MAX_DRM_LUT_VALUE 0xFFFF
@@ -41,6 +82,13 @@ void amdgpu_dm_init_color_mod(void)
setup_x_points_distribution();
}
+/* Extracts the DRM lut and lut size from a blob. */
+static const struct drm_color_lut *
+__extract_blob_lut(const struct drm_property_blob *blob, uint32_t *size)
+{
+ *size = blob ? drm_color_lut_size(blob) : 0;
+ return blob ? (struct drm_color_lut *)blob->data : NULL;
+}
/*
* Return true if the given lut is a linear mapping of values, i.e. it acts
@@ -50,7 +98,7 @@ void amdgpu_dm_init_color_mod(void)
* f(a) = (0xFF00/MAX_COLOR_LUT_ENTRIES-1)a; for integer a in
* [0, MAX_COLOR_LUT_ENTRIES)
*/
-static bool __is_lut_linear(struct drm_color_lut *lut, uint32_t size)
+static bool __is_lut_linear(const struct drm_color_lut *lut, uint32_t size)
{
int i;
uint32_t expected;
@@ -75,9 +123,8 @@ static bool __is_lut_linear(struct drm_color_lut *lut, uint32_t size)
* Convert the drm_color_lut to dc_gamma. The conversion depends on the size
* of the lut - whether or not it's legacy.
*/
-static void __drm_lut_to_dc_gamma(struct drm_color_lut *lut,
- struct dc_gamma *gamma,
- bool is_legacy)
+static void __drm_lut_to_dc_gamma(const struct drm_color_lut *lut,
+ struct dc_gamma *gamma, bool is_legacy)
{
uint32_t r, g, b;
int i;
@@ -107,103 +154,16 @@ static void __drm_lut_to_dc_gamma(struct drm_color_lut *lut,
}
}
-/**
- * amdgpu_dm_set_regamma_lut: Set regamma lut for the given CRTC.
- * @crtc: amdgpu_dm crtc state
- *
- * Update the underlying dc_stream_state's output transfer function (OTF) in
- * preparation for hardware commit. If no lut is specified by user, we default
- * to SRGB.
- *
- * RETURNS:
- * 0 on success, -ENOMEM if memory cannot be allocated to calculate the OTF.
- */
-int amdgpu_dm_set_regamma_lut(struct dm_crtc_state *crtc)
-{
- struct drm_property_blob *blob = crtc->base.gamma_lut;
- struct dc_stream_state *stream = crtc->stream;
- struct amdgpu_device *adev = (struct amdgpu_device *)
- crtc->base.state->dev->dev_private;
- struct drm_color_lut *lut;
- uint32_t lut_size;
- struct dc_gamma *gamma = NULL;
- enum dc_transfer_func_type old_type = stream->out_transfer_func->type;
-
- bool ret;
-
- if (!blob && adev->asic_type <= CHIP_RAVEN) {
- /* By default, use the SRGB predefined curve.*/
- stream->out_transfer_func->type = TF_TYPE_PREDEFINED;
- stream->out_transfer_func->tf = TRANSFER_FUNCTION_SRGB;
- return 0;
- }
-
- if (blob) {
- lut = (struct drm_color_lut *)blob->data;
- lut_size = blob->length / sizeof(struct drm_color_lut);
-
- gamma = dc_create_gamma();
- if (!gamma)
- return -ENOMEM;
-
- gamma->num_entries = lut_size;
- if (gamma->num_entries == MAX_COLOR_LEGACY_LUT_ENTRIES)
- gamma->type = GAMMA_RGB_256;
- else if (gamma->num_entries == MAX_COLOR_LUT_ENTRIES)
- gamma->type = GAMMA_CS_TFM_1D;
- else {
- /* Invalid lut size */
- dc_gamma_release(&gamma);
- return -EINVAL;
- }
-
- /* Convert drm_lut into dc_gamma */
- __drm_lut_to_dc_gamma(lut, gamma, gamma->type == GAMMA_RGB_256);
- }
-
- /* predefined gamma ROM only exist for RAVEN and pre-RAVEN ASIC,
- * set canRomBeUsed accordingly
- */
- stream->out_transfer_func->type = TF_TYPE_DISTRIBUTED_POINTS;
- ret = mod_color_calculate_regamma_params(stream->out_transfer_func,
- gamma, true, adev->asic_type <= CHIP_RAVEN, NULL);
-
- if (gamma)
- dc_gamma_release(&gamma);
-
- if (!ret) {
- stream->out_transfer_func->type = old_type;
- DRM_ERROR("Out of memory when calculating regamma params\n");
- return -ENOMEM;
- }
-
- return 0;
-}
-
-/**
- * amdgpu_dm_set_ctm: Set the color transform matrix for the given CRTC.
- * @crtc: amdgpu_dm crtc state
- *
- * Update the underlying dc_stream_state's gamut remap matrix in preparation
- * for hardware commit. If no matrix is specified by user, gamut remap will be
- * disabled.
+/*
+ * Converts a DRM CTM to a DC CSC float matrix.
+ * The matrix needs to be a 3x4 (12 entry) matrix.
*/
-void amdgpu_dm_set_ctm(struct dm_crtc_state *crtc)
+static void __drm_ctm_to_dc_matrix(const struct drm_color_ctm *ctm,
+ struct fixed31_32 *matrix)
{
-
- struct drm_property_blob *blob = crtc->base.ctm;
- struct dc_stream_state *stream = crtc->stream;
- struct drm_color_ctm *ctm;
int64_t val;
int i;
- if (!blob) {
- stream->gamut_remap_matrix.enable_remap = false;
- return;
- }
-
- stream->gamut_remap_matrix.enable_remap = true;
- ctm = (struct drm_color_ctm *)blob->data;
/*
* DRM gives a 3x3 matrix, but DC wants 3x4. Assuming we're operating
* with homogeneous coordinates, augment the matrix with 0's.
@@ -215,83 +175,306 @@ void amdgpu_dm_set_ctm(struct dm_crtc_state *crtc)
for (i = 0; i < 12; i++) {
/* Skip 4th element */
if (i % 4 == 3) {
- stream->gamut_remap_matrix.matrix[i] = dc_fixpt_zero;
+ matrix[i] = dc_fixpt_zero;
continue;
}
/* gamut_remap_matrix[i] = ctm[i - floor(i/4)] */
- val = ctm->matrix[i - (i/4)];
+ val = ctm->matrix[i - (i / 4)];
/* If negative, convert to 2's complement. */
if (val & (1ULL << 63))
val = -(val & ~(1ULL << 63));
- stream->gamut_remap_matrix.matrix[i].value = val;
+ matrix[i].value = val;
}
}
+/* Calculates the legacy transfer function - only for sRGB input space. */
+static int __set_legacy_tf(struct dc_transfer_func *func,
+ const struct drm_color_lut *lut, uint32_t lut_size,
+ bool has_rom)
+{
+ struct dc_gamma *gamma = NULL;
+ bool res;
-/**
- * amdgpu_dm_set_degamma_lut: Set degamma lut for the given CRTC.
- * @crtc: amdgpu_dm crtc state
- *
- * Update the underlying dc_stream_state's input transfer function (ITF) in
- * preparation for hardware commit. If no lut is specified by user, we default
- * to SRGB degamma.
- *
- * We support degamma bypass, predefined SRGB, and custom degamma
- *
- * RETURNS:
- * 0 on success
- * -EINVAL if crtc_state has a degamma_lut of invalid size
- * -ENOMEM if gamma allocation fails
- */
-int amdgpu_dm_set_degamma_lut(struct drm_crtc_state *crtc_state,
- struct dc_plane_state *dc_plane_state)
+ ASSERT(lut && lut_size == MAX_COLOR_LEGACY_LUT_ENTRIES);
+
+ gamma = dc_create_gamma();
+ if (!gamma)
+ return -ENOMEM;
+
+ gamma->type = GAMMA_RGB_256;
+ gamma->num_entries = lut_size;
+ __drm_lut_to_dc_gamma(lut, gamma, true);
+
+ res = mod_color_calculate_regamma_params(func, gamma, true, has_rom,
+ NULL);
+
+ return res ? 0 : -ENOMEM;
+}
+
+/* Calculates the output transfer function based on expected input space. */
+static int __set_output_tf(struct dc_transfer_func *func,
+ const struct drm_color_lut *lut, uint32_t lut_size,
+ bool has_rom)
{
- struct drm_property_blob *blob = crtc_state->degamma_lut;
- struct drm_color_lut *lut;
- uint32_t lut_size;
- struct dc_gamma *gamma;
- bool ret;
-
- if (!blob) {
- /* Default to SRGB */
- dc_plane_state->in_transfer_func->type = TF_TYPE_PREDEFINED;
- dc_plane_state->in_transfer_func->tf = TRANSFER_FUNCTION_SRGB;
- return 0;
- }
+ struct dc_gamma *gamma = NULL;
+ bool res;
- lut = (struct drm_color_lut *)blob->data;
- if (__is_lut_linear(lut, MAX_COLOR_LUT_ENTRIES)) {
- dc_plane_state->in_transfer_func->type = TF_TYPE_BYPASS;
- dc_plane_state->in_transfer_func->tf = TRANSFER_FUNCTION_LINEAR;
- return 0;
- }
+ ASSERT(lut && lut_size == MAX_COLOR_LUT_ENTRIES);
gamma = dc_create_gamma();
if (!gamma)
return -ENOMEM;
- lut_size = blob->length / sizeof(struct drm_color_lut);
gamma->num_entries = lut_size;
- if (gamma->num_entries == MAX_COLOR_LUT_ENTRIES)
+ __drm_lut_to_dc_gamma(lut, gamma, false);
+
+ if (func->tf == TRANSFER_FUNCTION_LINEAR) {
+ /*
+ * Color module doesn't like calculating regamma params
+ * on top of a linear input. But degamma params can be used
+ * instead to simulate this.
+ */
gamma->type = GAMMA_CUSTOM;
- else {
- dc_gamma_release(&gamma);
- return -EINVAL;
+ res = mod_color_calculate_degamma_params(func, gamma, true);
+ } else {
+ /*
+ * Assume sRGB. The actual mapping will depend on whether the
+ * input was legacy or not.
+ */
+ gamma->type = GAMMA_CS_TFM_1D;
+ res = mod_color_calculate_regamma_params(func, gamma, false,
+ has_rom, NULL);
}
+ dc_gamma_release(&gamma);
+
+ return res ? 0 : -ENOMEM;
+}
+
+/* Caculates the input transfer function based on expected input space. */
+static int __set_input_tf(struct dc_transfer_func *func,
+ const struct drm_color_lut *lut, uint32_t lut_size)
+{
+ struct dc_gamma *gamma = NULL;
+ bool res;
+
+ gamma = dc_create_gamma();
+ if (!gamma)
+ return -ENOMEM;
+
+ gamma->type = GAMMA_CUSTOM;
+ gamma->num_entries = lut_size;
+
__drm_lut_to_dc_gamma(lut, gamma, false);
- dc_plane_state->in_transfer_func->type = TF_TYPE_DISTRIBUTED_POINTS;
- ret = mod_color_calculate_degamma_params(dc_plane_state->in_transfer_func, gamma, true);
+ res = mod_color_calculate_degamma_params(func, gamma, true);
dc_gamma_release(&gamma);
- if (!ret) {
- dc_plane_state->in_transfer_func->type = TF_TYPE_BYPASS;
- DRM_ERROR("Out of memory when calculating degamma params\n");
- return -ENOMEM;
+
+ return res ? 0 : -ENOMEM;
+}
+
+/**
+ * amdgpu_dm_update_crtc_color_mgmt: Maps DRM color management to DC stream.
+ * @crtc: amdgpu_dm crtc state
+ *
+ * With no plane level color management properties we're free to use any
+ * of the HW blocks as long as the CRTC CTM always comes before the
+ * CRTC RGM and after the CRTC DGM.
+ *
+ * The CRTC RGM block will be placed in the RGM LUT block if it is non-linear.
+ * The CRTC DGM block will be placed in the DGM LUT block if it is non-linear.
+ * The CRTC CTM will be placed in the gamut remap block if it is non-linear.
+ *
+ * The RGM block is typically more fully featured and accurate across
+ * all ASICs - DCE can't support a custom non-linear CRTC DGM.
+ *
+ * For supporting both plane level color management and CRTC level color
+ * management at once we have to either restrict the usage of CRTC properties
+ * or blend adjustments together.
+ *
+ * Returns 0 on success.
+ */
+int amdgpu_dm_update_crtc_color_mgmt(struct dm_crtc_state *crtc)
+{
+ struct dc_stream_state *stream = crtc->stream;
+ struct amdgpu_device *adev =
+ (struct amdgpu_device *)crtc->base.state->dev->dev_private;
+ bool has_rom = adev->asic_type <= CHIP_RAVEN;
+ struct drm_color_ctm *ctm = NULL;
+ const struct drm_color_lut *degamma_lut, *regamma_lut;
+ uint32_t degamma_size, regamma_size;
+ bool has_regamma, has_degamma;
+ bool is_legacy;
+ int r;
+
+ degamma_lut = __extract_blob_lut(crtc->base.degamma_lut, &degamma_size);
+ if (degamma_lut && degamma_size != MAX_COLOR_LUT_ENTRIES)
+ return -EINVAL;
+
+ regamma_lut = __extract_blob_lut(crtc->base.gamma_lut, &regamma_size);
+ if (regamma_lut && regamma_size != MAX_COLOR_LUT_ENTRIES &&
+ regamma_size != MAX_COLOR_LEGACY_LUT_ENTRIES)
+ return -EINVAL;
+
+ has_degamma =
+ degamma_lut && !__is_lut_linear(degamma_lut, degamma_size);
+
+ has_regamma =
+ regamma_lut && !__is_lut_linear(regamma_lut, regamma_size);
+
+ is_legacy = regamma_size == MAX_COLOR_LEGACY_LUT_ENTRIES;
+
+ /* Reset all adjustments. */
+ crtc->cm_has_degamma = false;
+ crtc->cm_is_degamma_srgb = false;
+
+ /* Setup regamma and degamma. */
+ if (is_legacy) {
+ /*
+ * Legacy regamma forces us to use the sRGB RGM as a base.
+ * This also means we can't use linear DGM since DGM needs
+ * to use sRGB as a base as well, resulting in incorrect CRTC
+ * DGM and CRTC CTM.
+ *
+ * TODO: Just map this to the standard regamma interface
+ * instead since this isn't really right. One of the cases
+ * where this setup currently fails is trying to do an
+ * inverse color ramp in legacy userspace.
+ */
+ crtc->cm_is_degamma_srgb = true;
+ stream->out_transfer_func->type = TF_TYPE_DISTRIBUTED_POINTS;
+ stream->out_transfer_func->tf = TRANSFER_FUNCTION_SRGB;
+
+ r = __set_legacy_tf(stream->out_transfer_func, regamma_lut,
+ regamma_size, has_rom);
+ if (r)
+ return r;
+ } else if (has_regamma) {
+ /* CRTC RGM goes into RGM LUT. */
+ stream->out_transfer_func->type = TF_TYPE_DISTRIBUTED_POINTS;
+ stream->out_transfer_func->tf = TRANSFER_FUNCTION_LINEAR;
+
+ r = __set_output_tf(stream->out_transfer_func, regamma_lut,
+ regamma_size, has_rom);
+ if (r)
+ return r;
+ } else {
+ /*
+ * No CRTC RGM means we can just put the block into bypass
+ * since we don't have any plane level adjustments using it.
+ */
+ stream->out_transfer_func->type = TF_TYPE_BYPASS;
+ stream->out_transfer_func->tf = TRANSFER_FUNCTION_LINEAR;
+ }
+
+ /*
+ * CRTC DGM goes into DGM LUT. It would be nice to place it
+ * into the RGM since it's a more featured block but we'd
+ * have to place the CTM in the OCSC in that case.
+ */
+ crtc->cm_has_degamma = has_degamma;
+
+ /* Setup CRTC CTM. */
+ if (crtc->base.ctm) {
+ ctm = (struct drm_color_ctm *)crtc->base.ctm->data;
+
+ /*
+ * Gamut remapping must be used for gamma correction
+ * since it comes before the regamma correction.
+ *
+ * OCSC could be used for gamma correction, but we'd need to
+ * blend the adjustments together with the required output
+ * conversion matrix - so just use the gamut remap block
+ * for now.
+ */
+ __drm_ctm_to_dc_matrix(ctm, stream->gamut_remap_matrix.matrix);
+
+ stream->gamut_remap_matrix.enable_remap = true;
+ stream->csc_color_matrix.enable_adjustment = false;
+ } else {
+ /* Bypass CTM. */
+ stream->gamut_remap_matrix.enable_remap = false;
+ stream->csc_color_matrix.enable_adjustment = false;
}
return 0;
}
+/**
+ * amdgpu_dm_update_plane_color_mgmt: Maps DRM color management to DC plane.
+ * @crtc: amdgpu_dm crtc state
+ * @ dc_plane_state: target DC surface
+ *
+ * Update the underlying dc_stream_state's input transfer function (ITF) in
+ * preparation for hardware commit. The transfer function used depends on
+ * the prepartion done on the stream for color management.
+ *
+ * Returns 0 on success.
+ */
+int amdgpu_dm_update_plane_color_mgmt(struct dm_crtc_state *crtc,
+ struct dc_plane_state *dc_plane_state)
+{
+ const struct drm_color_lut *degamma_lut;
+ uint32_t degamma_size;
+ int r;
+
+ if (crtc->cm_has_degamma) {
+ degamma_lut = __extract_blob_lut(crtc->base.degamma_lut,
+ &degamma_size);
+ ASSERT(degamma_size == MAX_COLOR_LUT_ENTRIES);
+
+ dc_plane_state->in_transfer_func->type =
+ TF_TYPE_DISTRIBUTED_POINTS;
+
+ /*
+ * This case isn't fully correct, but also fairly
+ * uncommon. This is userspace trying to use a
+ * legacy gamma LUT + atomic degamma LUT
+ * at the same time.
+ *
+ * Legacy gamma requires the input to be in linear
+ * space, so that means we need to apply an sRGB
+ * degamma. But color module also doesn't support
+ * a user ramp in this case so the degamma will
+ * be lost.
+ *
+ * Even if we did support it, it's still not right:
+ *
+ * Input -> CRTC DGM -> sRGB DGM -> CRTC CTM ->
+ * sRGB RGM -> CRTC RGM -> Output
+ *
+ * The CSC will be done in the wrong space since
+ * we're applying an sRGB DGM on top of the CRTC
+ * DGM.
+ *
+ * TODO: Don't use the legacy gamma interface and just
+ * map these to the atomic one instead.
+ */
+ if (crtc->cm_is_degamma_srgb)
+ dc_plane_state->in_transfer_func->tf =
+ TRANSFER_FUNCTION_SRGB;
+ else
+ dc_plane_state->in_transfer_func->tf =
+ TRANSFER_FUNCTION_LINEAR;
+
+ r = __set_input_tf(dc_plane_state->in_transfer_func,
+ degamma_lut, degamma_size);
+ if (r)
+ return r;
+ } else if (crtc->cm_is_degamma_srgb) {
+ /*
+ * For legacy gamma support we need the regamma input
+ * in linear space. Assume that the input is sRGB.
+ */
+ dc_plane_state->in_transfer_func->type = TF_TYPE_PREDEFINED;
+ dc_plane_state->in_transfer_func->tf = TRANSFER_FUNCTION_SRGB;
+ } else {
+ /* ...Otherwise we can just bypass the DGM block. */
+ dc_plane_state->in_transfer_func->type = TF_TYPE_BYPASS;
+ dc_plane_state->in_transfer_func->tf = TRANSFER_FUNCTION_LINEAR;
+ }
+
+ return 0;
+}
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c
index a10e3a50d9ef..bc67e6502733 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c
@@ -24,6 +24,7 @@
*/
#include <drm/drm_crtc.h>
+#include <drm/drm_vblank.h>
#include "amdgpu.h"
#include "amdgpu_dm.h"
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
index 1d5fc5ad3bee..36a1d794b4af 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
@@ -23,7 +23,9 @@
*
*/
-#include <linux/debugfs.h>
+#include <linux/uaccess.h>
+
+#include <drm/drm_debugfs.h>
#include "dc.h"
#include "amdgpu.h"
@@ -673,6 +675,71 @@ static ssize_t dp_phy_test_pattern_debugfs_write(struct file *f, const char __us
}
/*
+ * Returns the current and maximum output bpc for the connector.
+ * Example usage: cat /sys/kernel/debug/dri/0/DP-1/output_bpc
+ */
+static int output_bpc_show(struct seq_file *m, void *data)
+{
+ struct drm_connector *connector = m->private;
+ struct drm_device *dev = connector->dev;
+ struct drm_crtc *crtc = NULL;
+ struct dm_crtc_state *dm_crtc_state = NULL;
+ int res = -ENODEV;
+ unsigned int bpc;
+
+ mutex_lock(&dev->mode_config.mutex);
+ drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
+
+ if (connector->state == NULL)
+ goto unlock;
+
+ crtc = connector->state->crtc;
+ if (crtc == NULL)
+ goto unlock;
+
+ drm_modeset_lock(&crtc->mutex, NULL);
+ if (crtc->state == NULL)
+ goto unlock;
+
+ dm_crtc_state = to_dm_crtc_state(crtc->state);
+ if (dm_crtc_state->stream == NULL)
+ goto unlock;
+
+ switch (dm_crtc_state->stream->timing.display_color_depth) {
+ case COLOR_DEPTH_666:
+ bpc = 6;
+ break;
+ case COLOR_DEPTH_888:
+ bpc = 8;
+ break;
+ case COLOR_DEPTH_101010:
+ bpc = 10;
+ break;
+ case COLOR_DEPTH_121212:
+ bpc = 12;
+ break;
+ case COLOR_DEPTH_161616:
+ bpc = 16;
+ break;
+ default:
+ goto unlock;
+ }
+
+ seq_printf(m, "Current: %u\n", bpc);
+ seq_printf(m, "Maximum: %u\n", connector->display_info.bpc);
+ res = 0;
+
+unlock:
+ if (crtc)
+ drm_modeset_unlock(&crtc->mutex);
+
+ drm_modeset_unlock(&dev->mode_config.connection_mutex);
+ mutex_unlock(&dev->mode_config.mutex);
+
+ return res;
+}
+
+/*
* Returns the min and max vrr vfreq through the connector's debugfs file.
* Example usage: cat /sys/kernel/debug/dri/0/DP-1/vrr_range
*/
@@ -730,8 +797,6 @@ static ssize_t dp_sdp_message_debugfs_write(struct file *f, const char __user *b
return write_size;
}
-DEFINE_SHOW_ATTRIBUTE(vrr_range);
-
static ssize_t dp_dpcd_address_write(struct file *f, const char __user *buf,
size_t size, loff_t *pos)
{
@@ -814,6 +879,9 @@ static ssize_t dp_dpcd_data_read(struct file *f, char __user *buf,
return read_size - r;
}
+DEFINE_SHOW_ATTRIBUTE(output_bpc);
+DEFINE_SHOW_ATTRIBUTE(vrr_range);
+
static const struct file_operations dp_link_settings_debugfs_fops = {
.owner = THIS_MODULE,
.read = dp_link_settings_read,
@@ -866,6 +934,7 @@ static const struct {
{"link_settings", &dp_link_settings_debugfs_fops},
{"phy_settings", &dp_phy_settings_debugfs_fop},
{"test_pattern", &dp_phy_test_pattern_fops},
+ {"output_bpc", &output_bpc_fops},
{"vrr_range", &vrr_range_fops},
{"sdp_message", &sdp_message_fops},
{"aux_dpcd_address", &dp_dpcd_address_debugfs_fops},
@@ -873,25 +942,19 @@ static const struct {
{"aux_dpcd_data", &dp_dpcd_data_debugfs_fops}
};
-int connector_debugfs_init(struct amdgpu_dm_connector *connector)
+void connector_debugfs_init(struct amdgpu_dm_connector *connector)
{
int i;
- struct dentry *ent, *dir = connector->base.debugfs_entry;
+ struct dentry *dir = connector->base.debugfs_entry;
if (connector->base.connector_type == DRM_MODE_CONNECTOR_DisplayPort ||
connector->base.connector_type == DRM_MODE_CONNECTOR_eDP) {
for (i = 0; i < ARRAY_SIZE(dp_debugfs_entries); i++) {
- ent = debugfs_create_file(dp_debugfs_entries[i].name,
- 0644,
- dir,
- connector,
- dp_debugfs_entries[i].fops);
- if (IS_ERR(ent))
- return PTR_ERR(ent);
+ debugfs_create_file(dp_debugfs_entries[i].name,
+ 0644, dir, connector,
+ dp_debugfs_entries[i].fops);
}
}
-
- return 0;
}
/*
@@ -1034,7 +1097,7 @@ int dtn_debugfs_init(struct amdgpu_device *adev)
};
struct drm_minor *minor = adev->ddev->primary;
- struct dentry *ent, *root = minor->debugfs_root;
+ struct dentry *root = minor->debugfs_root;
int ret;
ret = amdgpu_debugfs_add_files(adev, amdgpu_dm_debugfs_list,
@@ -1042,20 +1105,11 @@ int dtn_debugfs_init(struct amdgpu_device *adev)
if (ret)
return ret;
- ent = debugfs_create_file(
- "amdgpu_dm_dtn_log",
- 0644,
- root,
- adev,
- &dtn_log_fops);
-
- if (IS_ERR(ent))
- return PTR_ERR(ent);
+ debugfs_create_file("amdgpu_dm_dtn_log", 0644, root, adev,
+ &dtn_log_fops);
- ent = debugfs_create_file_unsafe("amdgpu_dm_visual_confirm", 0644, root,
- adev, &visual_confirm_fops);
- if (IS_ERR(ent))
- return PTR_ERR(ent);
+ debugfs_create_file_unsafe("amdgpu_dm_visual_confirm", 0644, root, adev,
+ &visual_confirm_fops);
return 0;
}
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.h
index bdef1587b0a0..5e5b2b2afa31 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.h
@@ -29,7 +29,7 @@
#include "amdgpu.h"
#include "amdgpu_dm.h"
-int connector_debugfs_init(struct amdgpu_dm_connector *connector);
+void connector_debugfs_init(struct amdgpu_dm_connector *connector);
int dtn_debugfs_init(struct amdgpu_device *adev);
#endif
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
index e6cd67342df8..a0ed0154a9f0 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
@@ -28,7 +28,6 @@
#include <linux/version.h>
#include <linux/i2c.h>
-#include <drm/drmP.h>
#include <drm/drm_probe_helper.h>
#include <drm/amdgpu_drm.h>
#include <drm/drm_edid.h>
@@ -542,6 +541,16 @@ bool dm_helpers_submit_i2c(
return result;
}
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+bool dm_helpers_dp_write_dsc_enable(
+ struct dc_context *ctx,
+ const struct dc_stream_state *stream,
+ bool enable
+)
+{
+ return false;
+}
+#endif
bool dm_helpers_is_dp_sink_present(struct dc_link *link)
{
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c
index fd22b4474dbf..fa5d503d379c 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c
@@ -23,8 +23,6 @@
*
*/
-#include <drm/drmP.h>
-
#include "dm_services_types.h"
#include "dc.h"
@@ -279,8 +277,6 @@ void *amdgpu_dm_irq_register_interrupt(struct amdgpu_device *adev,
return DAL_INVALID_IRQ_HANDLER_IDX;
}
- memset(handler_data, 0, sizeof(*handler_data));
-
init_handler_common_data(handler_data, ih, handler_args, &adev->dm);
irq_source = int_params->irq_source;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.c
index 350e7a620d45..592fa499c9f8 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.c
@@ -24,7 +24,6 @@
#include <linux/string.h>
#include <linux/acpi.h>
-#include <drm/drmP.h>
#include <drm/drm_probe_helper.h>
#include <drm/amdgpu_drm.h>
#include "dm_services.h"
@@ -149,6 +148,23 @@ static void get_default_clock_levels(
}
}
+static enum smu_clk_type dc_to_smu_clock_type(
+ enum dm_pp_clock_type dm_pp_clk_type)
+{
+#define DCCLK_MAP_SMUCLK(dcclk, smuclk) \
+ [dcclk] = smuclk
+
+ static int dc_clk_type_map[] = {
+ DCCLK_MAP_SMUCLK(DM_PP_CLOCK_TYPE_DISPLAY_CLK, SMU_DISPCLK),
+ DCCLK_MAP_SMUCLK(DM_PP_CLOCK_TYPE_ENGINE_CLK, SMU_GFXCLK),
+ DCCLK_MAP_SMUCLK(DM_PP_CLOCK_TYPE_MEMORY_CLK, SMU_MCLK),
+ DCCLK_MAP_SMUCLK(DM_PP_CLOCK_TYPE_DCEFCLK, SMU_DCEFCLK),
+ DCCLK_MAP_SMUCLK(DM_PP_CLOCK_TYPE_SOCCLK, SMU_SOCCLK),
+ };
+
+ return dc_clk_type_map[dm_pp_clk_type];
+}
+
static enum amd_pp_clock_type dc_to_pp_clock_type(
enum dm_pp_clock_type dm_pp_clk_type)
{
@@ -292,7 +308,8 @@ static void pp_to_dc_clock_levels_with_voltage(
DC_DECODE_PP_CLOCK_TYPE(dc_clk_type));
for (i = 0; i < clk_level_info->num_levels; i++) {
- DRM_INFO("DM_PPLIB:\t %d in kHz\n", pp_clks->data[i].clocks_in_khz);
+ DRM_INFO("DM_PPLIB:\t %d in kHz, %d in mV\n", pp_clks->data[i].clocks_in_khz,
+ pp_clks->data[i].voltage_in_mv);
clk_level_info->data[i].clocks_in_khz = pp_clks->data[i].clocks_in_khz;
clk_level_info->data[i].voltage_in_mv = pp_clks->data[i].voltage_in_mv;
}
@@ -317,7 +334,7 @@ bool dm_pp_get_clock_levels_by_type(
}
} else if (adev->smu.funcs && adev->smu.funcs->get_clock_by_type) {
if (smu_get_clock_by_type(&adev->smu,
- dc_to_pp_clock_type(clk_type),
+ dc_to_smu_clock_type(clk_type),
&pp_clks)) {
get_default_clock_levels(clk_type, dc_clks);
return true;
@@ -630,16 +647,279 @@ void pp_rv_set_hard_min_fclk_by_freq(struct pp_smu *pp, int mhz)
pp_funcs->set_hard_min_fclk_by_freq(pp_handle, mhz);
}
+enum pp_smu_status pp_nv_set_wm_ranges(struct pp_smu *pp,
+ struct pp_smu_wm_range_sets *ranges)
+{
+ const struct dc_context *ctx = pp->dm;
+ struct amdgpu_device *adev = ctx->driver_context;
+ struct smu_context *smu = &adev->smu;
+ struct dm_pp_wm_sets_with_clock_ranges_soc15 wm_with_clock_ranges;
+ struct dm_pp_clock_range_for_dmif_wm_set_soc15 *wm_dce_clocks =
+ wm_with_clock_ranges.wm_dmif_clocks_ranges;
+ struct dm_pp_clock_range_for_mcif_wm_set_soc15 *wm_soc_clocks =
+ wm_with_clock_ranges.wm_mcif_clocks_ranges;
+ int32_t i;
+
+ wm_with_clock_ranges.num_wm_dmif_sets = ranges->num_reader_wm_sets;
+ wm_with_clock_ranges.num_wm_mcif_sets = ranges->num_writer_wm_sets;
+
+ for (i = 0; i < wm_with_clock_ranges.num_wm_dmif_sets; i++) {
+ if (ranges->reader_wm_sets[i].wm_inst > 3)
+ wm_dce_clocks[i].wm_set_id = WM_SET_A;
+ else
+ wm_dce_clocks[i].wm_set_id =
+ ranges->reader_wm_sets[i].wm_inst;
+ wm_dce_clocks[i].wm_max_dcfclk_clk_in_khz =
+ ranges->reader_wm_sets[i].max_drain_clk_mhz * 1000;
+ wm_dce_clocks[i].wm_min_dcfclk_clk_in_khz =
+ ranges->reader_wm_sets[i].min_drain_clk_mhz * 1000;
+ wm_dce_clocks[i].wm_max_mem_clk_in_khz =
+ ranges->reader_wm_sets[i].max_fill_clk_mhz * 1000;
+ wm_dce_clocks[i].wm_min_mem_clk_in_khz =
+ ranges->reader_wm_sets[i].min_fill_clk_mhz * 1000;
+ }
+
+ for (i = 0; i < wm_with_clock_ranges.num_wm_mcif_sets; i++) {
+ if (ranges->writer_wm_sets[i].wm_inst > 3)
+ wm_soc_clocks[i].wm_set_id = WM_SET_A;
+ else
+ wm_soc_clocks[i].wm_set_id =
+ ranges->writer_wm_sets[i].wm_inst;
+ wm_soc_clocks[i].wm_max_socclk_clk_in_khz =
+ ranges->writer_wm_sets[i].max_fill_clk_mhz * 1000;
+ wm_soc_clocks[i].wm_min_socclk_clk_in_khz =
+ ranges->writer_wm_sets[i].min_fill_clk_mhz * 1000;
+ wm_soc_clocks[i].wm_max_mem_clk_in_khz =
+ ranges->writer_wm_sets[i].max_drain_clk_mhz * 1000;
+ wm_soc_clocks[i].wm_min_mem_clk_in_khz =
+ ranges->writer_wm_sets[i].min_drain_clk_mhz * 1000;
+ }
+
+ if (!smu->funcs)
+ return PP_SMU_RESULT_UNSUPPORTED;
+
+ /* 0: successful or smu.funcs->set_watermarks_for_clock_ranges = NULL;
+ * 1: fail
+ */
+ if (smu_set_watermarks_for_clock_ranges(&adev->smu,
+ &wm_with_clock_ranges))
+ return PP_SMU_RESULT_UNSUPPORTED;
+
+ return PP_SMU_RESULT_OK;
+}
+
+enum pp_smu_status pp_nv_set_pme_wa_enable(struct pp_smu *pp)
+{
+ const struct dc_context *ctx = pp->dm;
+ struct amdgpu_device *adev = ctx->driver_context;
+ struct smu_context *smu = &adev->smu;
+
+ if (!smu->funcs)
+ return PP_SMU_RESULT_UNSUPPORTED;
+
+ /* 0: successful or smu.funcs->set_azalia_d3_pme = NULL; 1: fail */
+ if (smu_set_azalia_d3_pme(smu))
+ return PP_SMU_RESULT_FAIL;
+
+ return PP_SMU_RESULT_OK;
+}
+
+enum pp_smu_status pp_nv_set_display_count(struct pp_smu *pp, int count)
+{
+ const struct dc_context *ctx = pp->dm;
+ struct amdgpu_device *adev = ctx->driver_context;
+ struct smu_context *smu = &adev->smu;
+
+ if (!smu->funcs)
+ return PP_SMU_RESULT_UNSUPPORTED;
+
+ /* 0: successful or smu.funcs->set_display_count = NULL; 1: fail */
+ if (smu_set_display_count(smu, count))
+ return PP_SMU_RESULT_FAIL;
+
+ return PP_SMU_RESULT_OK;
+}
+
+enum pp_smu_status pp_nv_set_min_deep_sleep_dcfclk(struct pp_smu *pp, int mhz)
+{
+ const struct dc_context *ctx = pp->dm;
+ struct amdgpu_device *adev = ctx->driver_context;
+ struct smu_context *smu = &adev->smu;
+
+ if (!smu->funcs)
+ return PP_SMU_RESULT_UNSUPPORTED;
+
+ /* 0: successful or smu.funcs->set_deep_sleep_dcefclk = NULL;1: fail */
+ if (smu_set_deep_sleep_dcefclk(smu, mhz))
+ return PP_SMU_RESULT_FAIL;
+
+ return PP_SMU_RESULT_OK;
+}
+
+enum pp_smu_status pp_nv_set_hard_min_dcefclk_by_freq(
+ struct pp_smu *pp, int mhz)
+{
+ const struct dc_context *ctx = pp->dm;
+ struct amdgpu_device *adev = ctx->driver_context;
+ struct smu_context *smu = &adev->smu;
+ struct pp_display_clock_request clock_req;
+
+ if (!smu->funcs)
+ return PP_SMU_RESULT_UNSUPPORTED;
+
+ clock_req.clock_type = amd_pp_dcef_clock;
+ clock_req.clock_freq_in_khz = mhz * 1000;
+
+ /* 0: successful or smu.funcs->display_clock_voltage_request = NULL
+ * 1: fail
+ */
+ if (smu_display_clock_voltage_request(smu, &clock_req))
+ return PP_SMU_RESULT_FAIL;
+
+ return PP_SMU_RESULT_OK;
+}
+
+enum pp_smu_status pp_nv_set_hard_min_uclk_by_freq(struct pp_smu *pp, int mhz)
+{
+ const struct dc_context *ctx = pp->dm;
+ struct amdgpu_device *adev = ctx->driver_context;
+ struct smu_context *smu = &adev->smu;
+ struct pp_display_clock_request clock_req;
+
+ if (!smu->funcs)
+ return PP_SMU_RESULT_UNSUPPORTED;
+
+ clock_req.clock_type = amd_pp_mem_clock;
+ clock_req.clock_freq_in_khz = mhz * 1000;
+
+ /* 0: successful or smu.funcs->display_clock_voltage_request = NULL
+ * 1: fail
+ */
+ if (smu_display_clock_voltage_request(smu, &clock_req))
+ return PP_SMU_RESULT_FAIL;
+
+ return PP_SMU_RESULT_OK;
+}
+
+enum pp_smu_status pp_nv_set_voltage_by_freq(struct pp_smu *pp,
+ enum pp_smu_nv_clock_id clock_id, int mhz)
+{
+ const struct dc_context *ctx = pp->dm;
+ struct amdgpu_device *adev = ctx->driver_context;
+ struct smu_context *smu = &adev->smu;
+ struct pp_display_clock_request clock_req;
+
+ if (!smu->funcs)
+ return PP_SMU_RESULT_UNSUPPORTED;
+
+ switch (clock_id) {
+ case PP_SMU_NV_DISPCLK:
+ clock_req.clock_type = amd_pp_disp_clock;
+ break;
+ case PP_SMU_NV_PHYCLK:
+ clock_req.clock_type = amd_pp_phy_clock;
+ break;
+ case PP_SMU_NV_PIXELCLK:
+ clock_req.clock_type = amd_pp_pixel_clock;
+ break;
+ default:
+ break;
+ }
+ clock_req.clock_freq_in_khz = mhz * 1000;
+
+ /* 0: successful or smu.funcs->display_clock_voltage_request = NULL
+ * 1: fail
+ */
+ if (smu_display_clock_voltage_request(smu, &clock_req))
+ return PP_SMU_RESULT_FAIL;
+
+ return PP_SMU_RESULT_OK;
+}
+
+enum pp_smu_status pp_nv_get_maximum_sustainable_clocks(
+ struct pp_smu *pp, struct pp_smu_nv_clock_table *max_clocks)
+{
+ const struct dc_context *ctx = pp->dm;
+ struct amdgpu_device *adev = ctx->driver_context;
+ struct smu_context *smu = &adev->smu;
+
+ if (!smu->funcs)
+ return PP_SMU_RESULT_UNSUPPORTED;
+
+ if (!smu->funcs->get_max_sustainable_clocks_by_dc)
+ return PP_SMU_RESULT_UNSUPPORTED;
+
+ if (!smu->funcs->get_max_sustainable_clocks_by_dc(smu, max_clocks))
+ return PP_SMU_RESULT_OK;
+
+ return PP_SMU_RESULT_FAIL;
+}
+
+enum pp_smu_status pp_nv_get_uclk_dpm_states(struct pp_smu *pp,
+ unsigned int *clock_values_in_khz, unsigned int *num_states)
+{
+ const struct dc_context *ctx = pp->dm;
+ struct amdgpu_device *adev = ctx->driver_context;
+ struct smu_context *smu = &adev->smu;
+
+ if (!smu->ppt_funcs)
+ return PP_SMU_RESULT_UNSUPPORTED;
+
+ if (!smu->ppt_funcs->get_uclk_dpm_states)
+ return PP_SMU_RESULT_UNSUPPORTED;
+
+ if (!smu->ppt_funcs->get_uclk_dpm_states(smu,
+ clock_values_in_khz, num_states))
+ return PP_SMU_RESULT_OK;
+
+ return PP_SMU_RESULT_FAIL;
+}
+
void dm_pp_get_funcs(
struct dc_context *ctx,
struct pp_smu_funcs *funcs)
{
- funcs->rv_funcs.pp_smu.dm = ctx;
- funcs->rv_funcs.set_wm_ranges = pp_rv_set_wm_ranges;
- funcs->rv_funcs.set_pme_wa_enable = pp_rv_set_pme_wa_enable;
- funcs->rv_funcs.set_display_count = pp_rv_set_active_display_count;
- funcs->rv_funcs.set_min_deep_sleep_dcfclk = pp_rv_set_min_deep_sleep_dcfclk;
- funcs->rv_funcs.set_hard_min_dcfclk_by_freq = pp_rv_set_hard_min_dcefclk_by_freq;
- funcs->rv_funcs.set_hard_min_fclk_by_freq = pp_rv_set_hard_min_fclk_by_freq;
+ switch (ctx->dce_version) {
+ case DCN_VERSION_1_0:
+ case DCN_VERSION_1_01:
+ funcs->ctx.ver = PP_SMU_VER_RV;
+ funcs->rv_funcs.pp_smu.dm = ctx;
+ funcs->rv_funcs.set_wm_ranges = pp_rv_set_wm_ranges;
+ funcs->rv_funcs.set_pme_wa_enable = pp_rv_set_pme_wa_enable;
+ funcs->rv_funcs.set_display_count =
+ pp_rv_set_active_display_count;
+ funcs->rv_funcs.set_min_deep_sleep_dcfclk =
+ pp_rv_set_min_deep_sleep_dcfclk;
+ funcs->rv_funcs.set_hard_min_dcfclk_by_freq =
+ pp_rv_set_hard_min_dcefclk_by_freq;
+ funcs->rv_funcs.set_hard_min_fclk_by_freq =
+ pp_rv_set_hard_min_fclk_by_freq;
+ break;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ case DCN_VERSION_2_0:
+ funcs->ctx.ver = PP_SMU_VER_NV;
+ funcs->nv_funcs.pp_smu.dm = ctx;
+ funcs->nv_funcs.set_display_count = pp_nv_set_display_count;
+ funcs->nv_funcs.set_hard_min_dcfclk_by_freq =
+ pp_nv_set_hard_min_dcefclk_by_freq;
+ funcs->nv_funcs.set_min_deep_sleep_dcfclk =
+ pp_nv_set_min_deep_sleep_dcfclk;
+ funcs->nv_funcs.set_voltage_by_freq =
+ pp_nv_set_voltage_by_freq;
+ funcs->nv_funcs.set_wm_ranges = pp_nv_set_wm_ranges;
+
+ /* todo set_pme_wa_enable cause 4k@6ohz display not light up */
+ funcs->nv_funcs.set_pme_wa_enable = NULL;
+ /* todo debug waring message */
+ funcs->nv_funcs.set_hard_min_uclk_by_freq = pp_nv_set_hard_min_uclk_by_freq;
+ /* todo compare data with window driver*/
+ funcs->nv_funcs.get_maximum_sustainable_clocks = pp_nv_get_maximum_sustainable_clocks;
+ /*todo compare data with window driver */
+ funcs->nv_funcs.get_uclk_dpm_states = pp_nv_get_uclk_dpm_states;
+ break;
+#endif
+ default:
+ DRM_ERROR("smu version is not supported !\n");
+ break;
+ }
}
-
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
index d915e8c8769b..022da5d45d4d 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
@@ -26,7 +26,6 @@
#include <linux/string.h>
#include <linux/acpi.h>
-#include <drm/drmP.h>
#include <drm/drm_probe_helper.h>
#include <drm/amdgpu_drm.h>
#include "dm_services.h"
diff --git a/drivers/gpu/drm/amd/display/dc/Makefile b/drivers/gpu/drm/amd/display/dc/Makefile
index b8ddb4acccdb..55ce5b657390 100644
--- a/drivers/gpu/drm/amd/display/dc/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/Makefile
@@ -23,7 +23,16 @@
# Makefile for Display Core (dc) component.
#
-DC_LIBS = basics bios calcs dce gpio irq virtual
+DC_LIBS = basics bios calcs clk_mgr dce gpio irq virtual
+
+ifdef CONFIG_DRM_AMD_DC_DCN2_0
+DC_LIBS += dcn20
+endif
+
+
+ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+DC_LIBS += dsc
+endif
ifdef CONFIG_DRM_AMD_DC_DCN1_0
DC_LIBS += dcn10 dml
@@ -41,8 +50,11 @@ AMD_DC = $(addsuffix /Makefile, $(addprefix $(FULL_AMD_DISPLAY_PATH)/dc/,$(DC_LI
include $(AMD_DC)
DISPLAY_CORE = dc.o dc_link.o dc_resource.o dc_hw_sequencer.o dc_sink.o \
-dc_surface.o dc_link_hwss.o dc_link_dp.o dc_link_ddc.o dc_debug.o dc_stream.o \
-dc_vm_helper.o
+dc_surface.o dc_link_hwss.o dc_link_dp.o dc_link_ddc.o dc_debug.o dc_stream.o
+
+ifdef CONFIG_DRM_AMD_DC_DCN2_0
+DISPLAY_CORE += dc_vm_helper.o
+endif
AMD_DISPLAY_CORE = $(addprefix $(AMDDALPATH)/dc/core/,$(DISPLAY_CORE))
diff --git a/drivers/gpu/drm/amd/display/dc/basics/vector.c b/drivers/gpu/drm/amd/display/dc/basics/vector.c
index d28e9cf0e961..8f93d25f91ee 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/vector.c
+++ b/drivers/gpu/drm/amd/display/dc/basics/vector.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "include/vector.h"
diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
index a4c97d32e751..461eef1de124 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "atom.h"
diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
index fd5266a58297..6aa2e56dfb67 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "ObjectID.h"
@@ -1313,6 +1315,8 @@ static enum bp_result bios_parser_get_encoder_cap_info(
ATOM_ENCODER_CAP_RECORD_HBR3_EN) ? 1 : 0;
info->HDMI_6GB_EN = (record->encodercaps &
ATOM_ENCODER_CAP_RECORD_HDMI6Gbps_EN) ? 1 : 0;
+ info->DP_IS_USB_C = (record->encodercaps &
+ ATOM_ENCODER_CAP_RECORD_USB_C_TYPE) ? 1 : 0;
return BP_RESULT_OK;
}
@@ -1398,6 +1402,10 @@ static enum bp_result get_integrated_info_v11(
info->ma_channel_number = info_v11->umachannelnumber;
info->lvds_ss_percentage =
le16_to_cpu(info_v11->lvds_ss_percentage);
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ info->dp_ss_control =
+ le16_to_cpu(info_v11->reserved1);
+#endif
info->lvds_sspread_rate_in_10hz =
le16_to_cpu(info_v11->lvds_ss_rate_10hz);
info->hdmi_ss_percentage =
diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c
index 8196f3bb10c7..f9439dfc7b75 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c
@@ -57,12 +57,13 @@ bool dal_bios_parser_init_cmd_tbl_helper2(
return true;
#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
case DCN_VERSION_1_0:
+ case DCN_VERSION_1_01:
*h = dal_cmd_tbl_helper_dce112_get_table2();
return true;
#endif
-#if defined(CONFIG_DRM_AMD_DC_DCN1_01)
- case DCN_VERSION_1_01:
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ case DCN_VERSION_2_0:
*h = dal_cmd_tbl_helper_dce112_get_table2();
return true;
#endif
diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c b/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c
index f3aa7b53d2aa..7108d51a9c5b 100644
--- a/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "dce_calcs.h"
#include "dc.h"
diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_auto.h b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_auto.h
index 03f06f682ead..ce35de79a6c7 100644
--- a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_auto.h
+++ b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_auto.h
@@ -26,6 +26,7 @@
#ifndef _DCN_CALC_AUTO_H_
#define _DCN_CALC_AUTO_H_
+#include "dc.h"
#include "dcn_calcs.h"
void scaler_settings_calculation(struct dcn_bw_internal_vars *v);
diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_math.c b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_math.c
index 7600a4a4abc7..07d18e78de49 100644
--- a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_math.c
+++ b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_math.c
@@ -73,6 +73,17 @@ float dcn_bw_floor2(const float arg, const float significance)
return 0;
return ((int) (arg / significance)) * significance;
}
+float dcn_bw_floor(const float arg)
+{
+ return ((int) (arg));
+}
+
+float dcn_bw_ceil(const float arg)
+{
+ float flr = dcn_bw_floor2(arg, 1);
+
+ return flr + 0.00001 >= arg ? arg : flr + 1;
+}
float dcn_bw_ceil2(const float arg, const float significance)
{
@@ -109,6 +120,15 @@ float dcn_bw_pow(float a, float exp)
}
}
+double dcn_bw_fabs(double a)
+{
+ if (a > 0)
+ return (a);
+ else
+ return (-a);
+}
+
+
float dcn_bw_log(float a, float b)
{
int * const exp_ptr = (int *)(&a);
diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_math.h b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_math.h
index f46ab0e24ca1..45a07eeffbb6 100644
--- a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_math.h
+++ b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_math.h
@@ -31,10 +31,13 @@ float dcn_bw_min2(const float arg1, const float arg2);
unsigned int dcn_bw_max(const unsigned int arg1, const unsigned int arg2);
float dcn_bw_max2(const float arg1, const float arg2);
float dcn_bw_floor2(const float arg, const float significance);
+float dcn_bw_floor(const float arg);
float dcn_bw_ceil2(const float arg, const float significance);
+float dcn_bw_ceil(const float arg);
float dcn_bw_max3(float v1, float v2, float v3);
float dcn_bw_max5(float v1, float v2, float v3, float v4, float v5);
float dcn_bw_pow(float a, float exp);
float dcn_bw_log(float a, float b);
+double dcn_bw_fabs(double a);
#endif /* _DCN_CALC_MATH_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c
index 1b4b51657f5e..38365dd911a3 100644
--- a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c
@@ -24,14 +24,14 @@
*/
#include "dm_services.h"
+#include "dc.h"
#include "dcn_calcs.h"
#include "dcn_calc_auto.h"
-#include "dc.h"
#include "dal_asic_id.h"
-
#include "resource.h"
#include "dcn10/dcn10_resource.h"
#include "dcn10/dcn10_hubbub.h"
+#include "dml/dml1_display_rq_dlg_calc.h"
#include "dcn_calc_math.h"
@@ -53,7 +53,13 @@
* remain as-is as it provides us with a guarantee from HW that it is correct.
*/
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+/* Defaults from spreadsheet rev#247.
+ * RV2 delta: dram_clock_change_latency, max_num_dpp
+ */
+#else
/* Defaults from spreadsheet rev#247 */
+#endif
const struct dcn_soc_bounding_box dcn10_soc_defaults = {
/* latencies */
.sr_exit_time = 17, /*us*/
@@ -712,7 +718,7 @@ bool dcn_validate_bandwidth(
const struct resource_pool *pool = dc->res_pool;
struct dcn_bw_internal_vars *v = &context->dcn_bw_vars;
- int i, input_idx;
+ int i, input_idx, k;
int vesa_sync_start, asic_blank_end, asic_blank_start;
bool bw_limit_pass;
float bw_limit;
@@ -873,8 +879,19 @@ bool dcn_validate_bandwidth(
v->lb_bit_per_pixel[input_idx] = 30;
v->viewport_width[input_idx] = pipe->stream->timing.h_addressable;
v->viewport_height[input_idx] = pipe->stream->timing.v_addressable;
- v->scaler_rec_out_width[input_idx] = pipe->stream->timing.h_addressable;
- v->scaler_recout_height[input_idx] = pipe->stream->timing.v_addressable;
+ /*
+ * for cases where we have no plane, we want to validate up to 1080p
+ * source size because here we are only interested in if the output
+ * timing is supported or not. if we cannot support native resolution
+ * of the high res display, we still want to support lower res up scale
+ * to native
+ */
+ if (v->viewport_width[input_idx] > 1920)
+ v->viewport_width[input_idx] = 1920;
+ if (v->viewport_height[input_idx] > 1080)
+ v->viewport_height[input_idx] = 1080;
+ v->scaler_rec_out_width[input_idx] = v->viewport_width[input_idx];
+ v->scaler_recout_height[input_idx] = v->viewport_height[input_idx];
v->override_hta_ps[input_idx] = 1;
v->override_vta_ps[input_idx] = 1;
v->override_hta_pschroma[input_idx] = 1;
@@ -1023,6 +1040,43 @@ bool dcn_validate_bandwidth(
mode_support_and_system_configuration(v);
}
+ display_pipe_configuration(v);
+
+ for (k = 0; k <= v->number_of_active_planes - 1; k++) {
+ if (v->source_scan[k] == dcn_bw_hor)
+ v->swath_width_y[k] = v->viewport_width[k] / v->dpp_per_plane[k];
+ else
+ v->swath_width_y[k] = v->viewport_height[k] / v->dpp_per_plane[k];
+ }
+ for (k = 0; k <= v->number_of_active_planes - 1; k++) {
+ if (v->source_pixel_format[k] == dcn_bw_rgb_sub_64) {
+ v->byte_per_pixel_dety[k] = 8.0;
+ v->byte_per_pixel_detc[k] = 0.0;
+ } else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_32) {
+ v->byte_per_pixel_dety[k] = 4.0;
+ v->byte_per_pixel_detc[k] = 0.0;
+ } else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_16) {
+ v->byte_per_pixel_dety[k] = 2.0;
+ v->byte_per_pixel_detc[k] = 0.0;
+ } else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8) {
+ v->byte_per_pixel_dety[k] = 1.0;
+ v->byte_per_pixel_detc[k] = 2.0;
+ } else {
+ v->byte_per_pixel_dety[k] = 4.0f / 3.0f;
+ v->byte_per_pixel_detc[k] = 8.0f / 3.0f;
+ }
+ }
+
+ v->total_data_read_bandwidth = 0.0;
+ for (k = 0; k <= v->number_of_active_planes - 1; k++) {
+ v->read_bandwidth_plane_luma[k] = v->swath_width_y[k] * v->dpp_per_plane[k] *
+ dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0) / (v->htotal[k] / v->pixel_clock[k]) * v->v_ratio[k];
+ v->read_bandwidth_plane_chroma[k] = v->swath_width_y[k] / 2.0 * v->dpp_per_plane[k] *
+ dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0) / (v->htotal[k] / v->pixel_clock[k]) * v->v_ratio[k] / 2.0;
+ v->total_data_read_bandwidth = v->total_data_read_bandwidth +
+ v->read_bandwidth_plane_luma[k] + v->read_bandwidth_plane_chroma[k];
+ }
+
BW_VAL_TRACE_END_VOLTAGE_LEVEL();
if (v->voltage_level != number_of_states_plus_one && !fast_validate) {
@@ -1062,9 +1116,8 @@ bool dcn_validate_bandwidth(
context->bw_ctx.bw.dcn.clk.fclk_khz = (int)(bw_consumed * 1000000 /
(ddr4_dram_factor_single_Channel * v->number_of_channels));
- if (bw_consumed == v->fabric_and_dram_bandwidth_vmin0p65) {
+ if (bw_consumed == v->fabric_and_dram_bandwidth_vmin0p65)
context->bw_ctx.bw.dcn.clk.fclk_khz = (int)(bw_consumed * 1000000 / 32);
- }
context->bw_ctx.bw.dcn.clk.dcfclk_deep_sleep_khz = (int)(v->dcf_clk_deep_sleep * 1000);
context->bw_ctx.bw.dcn.clk.dcfclk_khz = (int)(v->dcfclk * 1000);
@@ -1079,7 +1132,8 @@ bool dcn_validate_bandwidth(
dc->debug.min_disp_clk_khz;
}
- context->bw_ctx.bw.dcn.clk.dppclk_khz = context->bw_ctx.bw.dcn.clk.dispclk_khz / v->dispclk_dppclk_ratio;
+ context->bw_ctx.bw.dcn.clk.dppclk_khz = context->bw_ctx.bw.dcn.clk.dispclk_khz /
+ v->dispclk_dppclk_ratio;
context->bw_ctx.bw.dcn.clk.phyclk_khz = v->phyclk_per_state[v->voltage_level];
switch (v->voltage_level) {
case 0:
@@ -1166,9 +1220,7 @@ bool dcn_validate_bandwidth(
/* pipe not split previously needs split */
hsplit_pipe = find_idle_secondary_pipe(&context->res_ctx, pool, pipe);
ASSERT(hsplit_pipe);
- split_stream_across_pipes(
- &context->res_ctx, pool,
- pipe, hsplit_pipe);
+ split_stream_across_pipes(&context->res_ctx, pool, pipe, hsplit_pipe);
}
dcn_bw_calc_rq_dlg_ttu(dc, v, hsplit_pipe, input_idx);
@@ -1199,7 +1251,6 @@ bool dcn_validate_bandwidth(
}
if (v->voltage_level == 0) {
-
context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us =
dc->dcn_soc->sr_enter_plus_exit_time;
context->bw_ctx.dml.soc.sr_exit_time_us = dc->dcn_soc->sr_exit_time;
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile b/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile
new file mode 100644
index 000000000000..003c27767e9c
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile
@@ -0,0 +1,87 @@
+#
+# Copyright 2017 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+#
+# Makefile for the 'clk_mgr' sub-component of DAL.
+# It provides the control and status of HW CLK_MGR pins.
+
+CLK_MGR = clk_mgr.o
+
+AMD_DAL_CLK_MGR = $(addprefix $(AMDDALPATH)/dc/clk_mgr/,$(CLK_MGR))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR)
+
+
+###############################################################################
+# DCE 100 and DCE8x
+###############################################################################
+CLK_MGR_DCE100 = dce_clk_mgr.o
+
+AMD_DAL_CLK_MGR_DCE100 = $(addprefix $(AMDDALPATH)/dc/clk_mgr/dce100/,$(CLK_MGR_DCE100))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR_DCE100)
+
+###############################################################################
+# DCE 100 and DCE8x
+###############################################################################
+CLK_MGR_DCE110 = dce110_clk_mgr.o
+
+AMD_DAL_CLK_MGR_DCE110 = $(addprefix $(AMDDALPATH)/dc/clk_mgr/dce110/,$(CLK_MGR_DCE110))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR_DCE110)
+###############################################################################
+# DCE 112
+###############################################################################
+CLK_MGR_DCE112 = dce112_clk_mgr.o
+
+AMD_DAL_CLK_MGR_DCE112 = $(addprefix $(AMDDALPATH)/dc/clk_mgr/dce112/,$(CLK_MGR_DCE112))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR_DCE112)
+###############################################################################
+# DCE 120
+###############################################################################
+CLK_MGR_DCE120 = dce120_clk_mgr.o
+
+AMD_DAL_CLK_MGR_DCE120 = $(addprefix $(AMDDALPATH)/dc/clk_mgr/dce120/,$(CLK_MGR_DCE120))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR_DCE120)
+ifdef CONFIG_DRM_AMD_DC_DCN1_0
+###############################################################################
+# DCN10
+###############################################################################
+CLK_MGR_DCN10 = rv1_clk_mgr.o rv1_clk_mgr_vbios_smu.o rv2_clk_mgr.o
+
+AMD_DAL_CLK_MGR_DCN10 = $(addprefix $(AMDDALPATH)/dc/clk_mgr/dcn10/,$(CLK_MGR_DCN10))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR_DCN10)
+endif
+
+ifdef CONFIG_DRM_AMD_DC_DCN2_0
+###############################################################################
+# DCN20
+###############################################################################
+CLK_MGR_DCN20 = dcn20_clk_mgr.o
+
+AMD_DAL_CLK_MGR_DCN20 = $(addprefix $(AMDDALPATH)/dc/clk_mgr/dcn20/,$(CLK_MGR_DCN20))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR_DCN20)
+endif
+
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c
new file mode 100644
index 000000000000..6b8fc5cbabb8
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2012-16 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include <linux/slab.h>
+
+#include "dal_asic_id.h"
+#include "dc_types.h"
+#include "dccg.h"
+#include "clk_mgr_internal.h"
+
+#include "dce100/dce_clk_mgr.h"
+#include "dce110/dce110_clk_mgr.h"
+#include "dce112/dce112_clk_mgr.h"
+#include "dce120/dce120_clk_mgr.h"
+#include "dcn10/rv1_clk_mgr.h"
+#include "dcn10/rv2_clk_mgr.h"
+#include "dcn20/dcn20_clk_mgr.h"
+
+
+int clk_mgr_helper_get_active_display_cnt(
+ struct dc *dc,
+ struct dc_state *context)
+{
+ int i, display_count;
+
+ display_count = 0;
+ for (i = 0; i < context->stream_count; i++) {
+ const struct dc_stream_state *stream = context->streams[i];
+
+ /*
+ * Only notify active stream or virtual stream.
+ * Need to notify virtual stream to work around
+ * headless case. HPD does not fire when system is in
+ * S0i2.
+ */
+ if (!stream->dpms_off || stream->signal == SIGNAL_TYPE_VIRTUAL)
+ display_count++;
+ }
+
+ return display_count;
+}
+
+
+struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *pp_smu, struct dccg *dccg)
+{
+ struct hw_asic_id asic_id = ctx->asic_id;
+
+ struct clk_mgr_internal *clk_mgr = kzalloc(sizeof(*clk_mgr), GFP_KERNEL);
+
+ if (clk_mgr == NULL) {
+ BREAK_TO_DEBUGGER();
+ return NULL;
+ }
+
+ switch (asic_id.chip_family) {
+ case FAMILY_CI:
+ case FAMILY_KV:
+ dce_clk_mgr_construct(ctx, clk_mgr);
+ break;
+ case FAMILY_CZ:
+ dce110_clk_mgr_construct(ctx, clk_mgr);
+ break;
+ case FAMILY_VI:
+ if (ASIC_REV_IS_TONGA_P(asic_id.hw_internal_rev) ||
+ ASIC_REV_IS_FIJI_P(asic_id.hw_internal_rev)) {
+ dce_clk_mgr_construct(ctx, clk_mgr);
+ break;
+ }
+ if (ASIC_REV_IS_POLARIS10_P(asic_id.hw_internal_rev) ||
+ ASIC_REV_IS_POLARIS11_M(asic_id.hw_internal_rev) ||
+ ASIC_REV_IS_POLARIS12_V(asic_id.hw_internal_rev)) {
+ dce112_clk_mgr_construct(ctx, clk_mgr);
+ break;
+ }
+ if (ASIC_REV_IS_VEGAM(asic_id.hw_internal_rev)) {
+ dce112_clk_mgr_construct(ctx, clk_mgr);
+ break;
+ }
+ break;
+ case FAMILY_AI:
+ if (ASICREV_IS_VEGA20_P(asic_id.hw_internal_rev))
+ dce121_clk_mgr_construct(ctx, clk_mgr);
+ else
+ dce120_clk_mgr_construct(ctx, clk_mgr);
+ break;
+
+#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
+ case FAMILY_RV:
+ if (ASICREV_IS_RAVEN2(asic_id.hw_internal_rev)) {
+ rv2_clk_mgr_construct(ctx, clk_mgr, pp_smu);
+ break;
+ }
+ if (ASICREV_IS_RAVEN(asic_id.hw_internal_rev) ||
+ ASICREV_IS_PICASSO(asic_id.hw_internal_rev)) {
+ rv1_clk_mgr_construct(ctx, clk_mgr, pp_smu);
+ break;
+ }
+ break;
+#endif /* Family RV */
+
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ case FAMILY_NV:
+ dcn20_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg);
+ break;
+#endif /* Family NV */
+
+ default:
+ ASSERT(0); /* Unknown Asic */
+ break;
+ }
+
+ return &clk_mgr->base;
+}
+
+void dc_destroy_clk_mgr(struct clk_mgr *clk_mgr_base)
+{
+ struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+
+ kfree(clk_mgr);
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c
new file mode 100644
index 000000000000..814450fefffa
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c
@@ -0,0 +1,471 @@
+/*
+ * Copyright 2012-16 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+
+#include "dccg.h"
+#include "clk_mgr_internal.h"
+#include "dce_clk_mgr.h"
+#include "dce110/dce110_clk_mgr.h"
+#include "dce112/dce112_clk_mgr.h"
+#include "reg_helper.h"
+#include "dmcu.h"
+#include "core_types.h"
+#include "dal_asic_id.h"
+
+/*
+ * Currently the register shifts and masks in this file are used for dce100 and dce80
+ * which has identical definitions.
+ * TODO: remove this when DPREFCLK_CNTL and dpref DENTIST_DISPCLK_CNTL
+ * is moved to dccg, where it belongs
+ */
+#include "dce/dce_8_0_d.h"
+#include "dce/dce_8_0_sh_mask.h"
+
+#define REG(reg) \
+ (clk_mgr->regs->reg)
+
+#undef FN
+#define FN(reg_name, field_name) \
+ clk_mgr->clk_mgr_shift->field_name, clk_mgr->clk_mgr_mask->field_name
+
+static const struct clk_mgr_registers disp_clk_regs = {
+ CLK_COMMON_REG_LIST_DCE_BASE()
+};
+
+static const struct clk_mgr_shift disp_clk_shift = {
+ CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
+};
+
+static const struct clk_mgr_mask disp_clk_mask = {
+ CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
+};
+
+
+/* Max clock values for each state indexed by "enum clocks_state": */
+static const struct state_dependent_clocks dce80_max_clks_by_state[] = {
+/* ClocksStateInvalid - should not be used */
+{ .display_clk_khz = 0, .pixel_clk_khz = 0 },
+/* ClocksStateUltraLow - not expected to be used for DCE 8.0 */
+{ .display_clk_khz = 0, .pixel_clk_khz = 0 },
+/* ClocksStateLow */
+{ .display_clk_khz = 352000, .pixel_clk_khz = 330000},
+/* ClocksStateNominal */
+{ .display_clk_khz = 600000, .pixel_clk_khz = 400000 },
+/* ClocksStatePerformance */
+{ .display_clk_khz = 600000, .pixel_clk_khz = 400000 } };
+
+int dentist_get_divider_from_did(int did)
+{
+ if (did < DENTIST_BASE_DID_1)
+ did = DENTIST_BASE_DID_1;
+ if (did > DENTIST_MAX_DID)
+ did = DENTIST_MAX_DID;
+
+ if (did < DENTIST_BASE_DID_2) {
+ return DENTIST_DIVIDER_RANGE_1_START + DENTIST_DIVIDER_RANGE_1_STEP
+ * (did - DENTIST_BASE_DID_1);
+ } else if (did < DENTIST_BASE_DID_3) {
+ return DENTIST_DIVIDER_RANGE_2_START + DENTIST_DIVIDER_RANGE_2_STEP
+ * (did - DENTIST_BASE_DID_2);
+ } else if (did < DENTIST_BASE_DID_4) {
+ return DENTIST_DIVIDER_RANGE_3_START + DENTIST_DIVIDER_RANGE_3_STEP
+ * (did - DENTIST_BASE_DID_3);
+ } else {
+ return DENTIST_DIVIDER_RANGE_4_START + DENTIST_DIVIDER_RANGE_4_STEP
+ * (did - DENTIST_BASE_DID_4);
+ }
+}
+
+/* SW will adjust DP REF Clock average value for all purposes
+ * (DP DTO / DP Audio DTO and DP GTC)
+ if clock is spread for all cases:
+ -if SS enabled on DP Ref clock and HW de-spreading enabled with SW
+ calculations for DS_INCR/DS_MODULO (this is planned to be default case)
+ -if SS enabled on DP Ref clock and HW de-spreading enabled with HW
+ calculations (not planned to be used, but average clock should still
+ be valid)
+ -if SS enabled on DP Ref clock and HW de-spreading disabled
+ (should not be case with CIK) then SW should program all rates
+ generated according to average value (case as with previous ASICs)
+ */
+
+int dce_adjust_dp_ref_freq_for_ss(struct clk_mgr_internal *clk_mgr_dce, int dp_ref_clk_khz)
+{
+ if (clk_mgr_dce->ss_on_dprefclk && clk_mgr_dce->dprefclk_ss_divider != 0) {
+ struct fixed31_32 ss_percentage = dc_fixpt_div_int(
+ dc_fixpt_from_fraction(clk_mgr_dce->dprefclk_ss_percentage,
+ clk_mgr_dce->dprefclk_ss_divider), 200);
+ struct fixed31_32 adj_dp_ref_clk_khz;
+
+ ss_percentage = dc_fixpt_sub(dc_fixpt_one, ss_percentage);
+ adj_dp_ref_clk_khz = dc_fixpt_mul_int(ss_percentage, dp_ref_clk_khz);
+ dp_ref_clk_khz = dc_fixpt_floor(adj_dp_ref_clk_khz);
+ }
+ return dp_ref_clk_khz;
+}
+
+int dce_get_dp_ref_freq_khz(struct clk_mgr *clk_mgr_base)
+{
+ struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+ int dprefclk_wdivider;
+ int dprefclk_src_sel;
+ int dp_ref_clk_khz = 600000;
+ int target_div;
+
+ /* ASSERT DP Reference Clock source is from DFS*/
+ REG_GET(DPREFCLK_CNTL, DPREFCLK_SRC_SEL, &dprefclk_src_sel);
+ ASSERT(dprefclk_src_sel == 0);
+
+ /* Read the mmDENTIST_DISPCLK_CNTL to get the currently
+ * programmed DID DENTIST_DPREFCLK_WDIVIDER*/
+ REG_GET(DENTIST_DISPCLK_CNTL, DENTIST_DPREFCLK_WDIVIDER, &dprefclk_wdivider);
+
+ /* Convert DENTIST_DPREFCLK_WDIVIDERto actual divider*/
+ target_div = dentist_get_divider_from_did(dprefclk_wdivider);
+
+ /* Calculate the current DFS clock, in kHz.*/
+ dp_ref_clk_khz = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR
+ * clk_mgr->dentist_vco_freq_khz) / target_div;
+
+ return dce_adjust_dp_ref_freq_for_ss(clk_mgr, dp_ref_clk_khz);
+}
+
+int dce12_get_dp_ref_freq_khz(struct clk_mgr *clk_mgr_base)
+{
+ struct clk_mgr_internal *clk_mgr_dce = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+
+ return dce_adjust_dp_ref_freq_for_ss(clk_mgr_dce, clk_mgr_base->dprefclk_khz);
+}
+
+/* unit: in_khz before mode set, get pixel clock from context. ASIC register
+ * may not be programmed yet
+ */
+uint32_t dce_get_max_pixel_clock_for_all_paths(struct dc_state *context)
+{
+ uint32_t max_pix_clk = 0;
+ int i;
+
+ for (i = 0; i < MAX_PIPES; i++) {
+ struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
+
+ if (pipe_ctx->stream == NULL)
+ continue;
+
+ /* do not check under lay */
+ if (pipe_ctx->top_pipe)
+ continue;
+
+ if (pipe_ctx->stream_res.pix_clk_params.requested_pix_clk_100hz / 10 > max_pix_clk)
+ max_pix_clk = pipe_ctx->stream_res.pix_clk_params.requested_pix_clk_100hz / 10;
+
+ /* raise clock state for HBR3/2 if required. Confirmed with HW DCE/DPCS
+ * logic for HBR3 still needs Nominal (0.8V) on VDDC rail
+ */
+ if (dc_is_dp_signal(pipe_ctx->stream->signal) &&
+ pipe_ctx->stream_res.pix_clk_params.requested_sym_clk > max_pix_clk)
+ max_pix_clk = pipe_ctx->stream_res.pix_clk_params.requested_sym_clk;
+ }
+
+ return max_pix_clk;
+}
+
+enum dm_pp_clocks_state dce_get_required_clocks_state(
+ struct clk_mgr *clk_mgr_base,
+ struct dc_state *context)
+{
+ struct clk_mgr_internal *clk_mgr_dce = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+ int i;
+ enum dm_pp_clocks_state low_req_clk;
+ int max_pix_clk = dce_get_max_pixel_clock_for_all_paths(context);
+
+ /* Iterate from highest supported to lowest valid state, and update
+ * lowest RequiredState with the lowest state that satisfies
+ * all required clocks
+ */
+ for (i = clk_mgr_dce->max_clks_state; i >= DM_PP_CLOCKS_STATE_ULTRA_LOW; i--)
+ if (context->bw_ctx.bw.dce.dispclk_khz >
+ clk_mgr_dce->max_clks_by_state[i].display_clk_khz
+ || max_pix_clk >
+ clk_mgr_dce->max_clks_by_state[i].pixel_clk_khz)
+ break;
+
+ low_req_clk = i + 1;
+ if (low_req_clk > clk_mgr_dce->max_clks_state) {
+ /* set max clock state for high phyclock, invalid on exceeding display clock */
+ if (clk_mgr_dce->max_clks_by_state[clk_mgr_dce->max_clks_state].display_clk_khz
+ < context->bw_ctx.bw.dce.dispclk_khz)
+ low_req_clk = DM_PP_CLOCKS_STATE_INVALID;
+ else
+ low_req_clk = clk_mgr_dce->max_clks_state;
+ }
+
+ return low_req_clk;
+}
+
+
+/* TODO: remove use the two broken down functions */
+int dce_set_clock(
+ struct clk_mgr *clk_mgr_base,
+ int requested_clk_khz)
+{
+ struct clk_mgr_internal *clk_mgr_dce = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+ struct bp_pixel_clock_parameters pxl_clk_params = { 0 };
+ struct dc_bios *bp = clk_mgr_base->ctx->dc_bios;
+ int actual_clock = requested_clk_khz;
+ struct dmcu *dmcu = clk_mgr_dce->base.ctx->dc->res_pool->dmcu;
+
+ /* Make sure requested clock isn't lower than minimum threshold*/
+ if (requested_clk_khz > 0)
+ requested_clk_khz = max(requested_clk_khz,
+ clk_mgr_dce->dentist_vco_freq_khz / 64);
+
+ /* Prepare to program display clock*/
+ pxl_clk_params.target_pixel_clock_100hz = requested_clk_khz * 10;
+ pxl_clk_params.pll_id = CLOCK_SOURCE_ID_DFS;
+
+ if (clk_mgr_dce->dfs_bypass_active)
+ pxl_clk_params.flags.SET_DISPCLK_DFS_BYPASS = true;
+
+ bp->funcs->program_display_engine_pll(bp, &pxl_clk_params);
+
+ if (clk_mgr_dce->dfs_bypass_active) {
+ /* Cache the fixed display clock*/
+ clk_mgr_dce->dfs_bypass_disp_clk =
+ pxl_clk_params.dfs_bypass_display_clock;
+ actual_clock = pxl_clk_params.dfs_bypass_display_clock;
+ }
+
+ /* from power down, we need mark the clock state as ClocksStateNominal
+ * from HWReset, so when resume we will call pplib voltage regulator.*/
+ if (requested_clk_khz == 0)
+ clk_mgr_dce->cur_min_clks_state = DM_PP_CLOCKS_STATE_NOMINAL;
+
+ if (dmcu && dmcu->funcs->is_dmcu_initialized(dmcu))
+ dmcu->funcs->set_psr_wait_loop(dmcu, actual_clock / 1000 / 7);
+
+ return actual_clock;
+}
+
+
+static void dce_clock_read_integrated_info(struct clk_mgr_internal *clk_mgr_dce)
+{
+ struct dc_debug_options *debug = &clk_mgr_dce->base.ctx->dc->debug;
+ struct dc_bios *bp = clk_mgr_dce->base.ctx->dc_bios;
+ struct integrated_info info = { { { 0 } } };
+ struct dc_firmware_info fw_info = { { 0 } };
+ int i;
+
+ if (bp->integrated_info)
+ info = *bp->integrated_info;
+
+ clk_mgr_dce->dentist_vco_freq_khz = info.dentist_vco_freq;
+ if (clk_mgr_dce->dentist_vco_freq_khz == 0) {
+ bp->funcs->get_firmware_info(bp, &fw_info);
+ clk_mgr_dce->dentist_vco_freq_khz =
+ fw_info.smu_gpu_pll_output_freq;
+ if (clk_mgr_dce->dentist_vco_freq_khz == 0)
+ clk_mgr_dce->dentist_vco_freq_khz = 3600000;
+ }
+
+ /*update the maximum display clock for each power state*/
+ for (i = 0; i < NUMBER_OF_DISP_CLK_VOLTAGE; ++i) {
+ enum dm_pp_clocks_state clk_state = DM_PP_CLOCKS_STATE_INVALID;
+
+ switch (i) {
+ case 0:
+ clk_state = DM_PP_CLOCKS_STATE_ULTRA_LOW;
+ break;
+
+ case 1:
+ clk_state = DM_PP_CLOCKS_STATE_LOW;
+ break;
+
+ case 2:
+ clk_state = DM_PP_CLOCKS_STATE_NOMINAL;
+ break;
+
+ case 3:
+ clk_state = DM_PP_CLOCKS_STATE_PERFORMANCE;
+ break;
+
+ default:
+ clk_state = DM_PP_CLOCKS_STATE_INVALID;
+ break;
+ }
+
+ /*Do not allow bad VBIOS/SBIOS to override with invalid values,
+ * check for > 100MHz*/
+ if (info.disp_clk_voltage[i].max_supported_clk >= 100000)
+ clk_mgr_dce->max_clks_by_state[clk_state].display_clk_khz =
+ info.disp_clk_voltage[i].max_supported_clk;
+ }
+
+ if (!debug->disable_dfs_bypass && bp->integrated_info)
+ if (bp->integrated_info->gpu_cap_info & DFS_BYPASS_ENABLE)
+ clk_mgr_dce->dfs_bypass_enabled = true;
+}
+
+void dce_clock_read_ss_info(struct clk_mgr_internal *clk_mgr_dce)
+{
+ struct dc_bios *bp = clk_mgr_dce->base.ctx->dc_bios;
+ int ss_info_num = bp->funcs->get_ss_entry_number(
+ bp, AS_SIGNAL_TYPE_GPU_PLL);
+
+ if (ss_info_num) {
+ struct spread_spectrum_info info = { { 0 } };
+ enum bp_result result = bp->funcs->get_spread_spectrum_info(
+ bp, AS_SIGNAL_TYPE_GPU_PLL, 0, &info);
+
+ /* Based on VBIOS, VBIOS will keep entry for GPU PLL SS
+ * even if SS not enabled and in that case
+ * SSInfo.spreadSpectrumPercentage !=0 would be sign
+ * that SS is enabled
+ */
+ if (result == BP_RESULT_OK &&
+ info.spread_spectrum_percentage != 0) {
+ clk_mgr_dce->ss_on_dprefclk = true;
+ clk_mgr_dce->dprefclk_ss_divider = info.spread_percentage_divider;
+
+ if (info.type.CENTER_MODE == 0) {
+ /* TODO: Currently for DP Reference clock we
+ * need only SS percentage for
+ * downspread */
+ clk_mgr_dce->dprefclk_ss_percentage =
+ info.spread_spectrum_percentage;
+ }
+
+ return;
+ }
+
+ result = bp->funcs->get_spread_spectrum_info(
+ bp, AS_SIGNAL_TYPE_DISPLAY_PORT, 0, &info);
+
+ /* Based on VBIOS, VBIOS will keep entry for DPREFCLK SS
+ * even if SS not enabled and in that case
+ * SSInfo.spreadSpectrumPercentage !=0 would be sign
+ * that SS is enabled
+ */
+ if (result == BP_RESULT_OK &&
+ info.spread_spectrum_percentage != 0) {
+ clk_mgr_dce->ss_on_dprefclk = true;
+ clk_mgr_dce->dprefclk_ss_divider = info.spread_percentage_divider;
+
+ if (info.type.CENTER_MODE == 0) {
+ /* Currently for DP Reference clock we
+ * need only SS percentage for
+ * downspread */
+ clk_mgr_dce->dprefclk_ss_percentage =
+ info.spread_spectrum_percentage;
+ }
+ }
+ }
+}
+
+static void dce_pplib_apply_display_requirements(
+ struct dc *dc,
+ struct dc_state *context)
+{
+ struct dm_pp_display_configuration *pp_display_cfg = &context->pp_display_cfg;
+
+ pp_display_cfg->avail_mclk_switch_time_us = dce110_get_min_vblank_time_us(context);
+
+ dce110_fill_display_configs(context, pp_display_cfg);
+
+ if (memcmp(&dc->current_state->pp_display_cfg, pp_display_cfg, sizeof(*pp_display_cfg)) != 0)
+ dm_pp_apply_display_requirements(dc->ctx, pp_display_cfg);
+}
+
+static void dce_update_clocks(struct clk_mgr *clk_mgr_base,
+ struct dc_state *context,
+ bool safe_to_lower)
+{
+ struct clk_mgr_internal *clk_mgr_dce = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+ struct dm_pp_power_level_change_request level_change_req;
+ int patched_disp_clk = context->bw_ctx.bw.dce.dispclk_khz;
+
+ /*TODO: W/A for dal3 linux, investigate why this works */
+ if (!clk_mgr_dce->dfs_bypass_active)
+ patched_disp_clk = patched_disp_clk * 115 / 100;
+
+ level_change_req.power_level = dce_get_required_clocks_state(clk_mgr_base, context);
+ /* get max clock state from PPLIB */
+ if ((level_change_req.power_level < clk_mgr_dce->cur_min_clks_state && safe_to_lower)
+ || level_change_req.power_level > clk_mgr_dce->cur_min_clks_state) {
+ if (dm_pp_apply_power_level_change_request(clk_mgr_base->ctx, &level_change_req))
+ clk_mgr_dce->cur_min_clks_state = level_change_req.power_level;
+ }
+
+ if (should_set_clock(safe_to_lower, patched_disp_clk, clk_mgr_base->clks.dispclk_khz)) {
+ patched_disp_clk = dce_set_clock(clk_mgr_base, patched_disp_clk);
+ clk_mgr_base->clks.dispclk_khz = patched_disp_clk;
+ }
+ dce_pplib_apply_display_requirements(clk_mgr_base->ctx->dc, context);
+}
+
+
+
+
+
+
+
+
+static struct clk_mgr_funcs dce_funcs = {
+ .get_dp_ref_clk_frequency = dce_get_dp_ref_freq_khz,
+ .update_clocks = dce_update_clocks
+};
+
+void dce_clk_mgr_construct(
+ struct dc_context *ctx,
+ struct clk_mgr_internal *clk_mgr)
+{
+ struct clk_mgr *base = &clk_mgr->base;
+ struct dm_pp_static_clock_info static_clk_info = {0};
+
+ memcpy(clk_mgr->max_clks_by_state,
+ dce80_max_clks_by_state,
+ sizeof(dce80_max_clks_by_state));
+
+ base->ctx = ctx;
+ base->funcs = &dce_funcs;
+
+ clk_mgr->regs = &disp_clk_regs;
+ clk_mgr->clk_mgr_shift = &disp_clk_shift;
+ clk_mgr->clk_mgr_mask = &disp_clk_mask;
+ clk_mgr->dfs_bypass_disp_clk = 0;
+
+ clk_mgr->dprefclk_ss_percentage = 0;
+ clk_mgr->dprefclk_ss_divider = 1000;
+ clk_mgr->ss_on_dprefclk = false;
+
+ if (dm_pp_get_static_clocks(ctx, &static_clk_info))
+ clk_mgr->max_clks_state = static_clk_info.max_clocks_state;
+ else
+ clk_mgr->max_clks_state = DM_PP_CLOCKS_STATE_NOMINAL;
+ clk_mgr->cur_min_clks_state = DM_PP_CLOCKS_STATE_INVALID;
+
+ dce_clock_read_integrated_info(clk_mgr);
+ dce_clock_read_ss_info(clk_mgr);
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.h
new file mode 100644
index 000000000000..f6622f58f62e
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.h
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2012-16 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+
+#ifndef _DCE_CLK_MGR_H_
+#define _DCE_CLK_MGR_H_
+
+#include "dc.h"
+
+/* functions shared by other dce clk mgrs */
+int dce_adjust_dp_ref_freq_for_ss(struct clk_mgr_internal *clk_mgr_dce, int dp_ref_clk_khz);
+int dce_get_dp_ref_freq_khz(struct clk_mgr *clk_mgr_base);
+enum dm_pp_clocks_state dce_get_required_clocks_state(
+ struct clk_mgr *clk_mgr_base,
+ struct dc_state *context);
+
+uint32_t dce_get_max_pixel_clock_for_all_paths(struct dc_state *context);
+
+
+void dce_clk_mgr_construct(
+ struct dc_context *ctx,
+ struct clk_mgr_internal *clk_mgr_dce);
+
+void dce_clock_read_ss_info(struct clk_mgr_internal *dccg_dce);
+
+int dce12_get_dp_ref_freq_khz(struct clk_mgr *dccg);
+
+int dce_set_clock(
+ struct clk_mgr *clk_mgr_base,
+ int requested_clk_khz);
+
+
+void dce_clk_mgr_destroy(struct clk_mgr **clk_mgr);
+
+int dentist_get_divider_from_did(int did);
+
+#endif /* _DCE_CLK_MGR_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce110/dce110_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce110/dce110_clk_mgr.c
new file mode 100644
index 000000000000..5cc3acccda2a
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce110/dce110_clk_mgr.c
@@ -0,0 +1,276 @@
+/*
+ * Copyright 2012-16 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "core_types.h"
+#include "clk_mgr_internal.h"
+
+#include "dce/dce_11_0_d.h"
+#include "dce/dce_11_0_sh_mask.h"
+#include "dce110_clk_mgr.h"
+#include "../clk_mgr/dce100/dce_clk_mgr.h"
+
+/* set register offset */
+#define SR(reg_name)\
+ .reg_name = mm ## reg_name
+
+/* set register offset with instance */
+#define SRI(reg_name, block, id)\
+ .reg_name = mm ## block ## id ## _ ## reg_name
+
+static const struct clk_mgr_registers disp_clk_regs = {
+ CLK_COMMON_REG_LIST_DCE_BASE()
+};
+
+static const struct clk_mgr_shift disp_clk_shift = {
+ CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
+};
+
+static const struct clk_mgr_mask disp_clk_mask = {
+ CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
+};
+
+static const struct state_dependent_clocks dce110_max_clks_by_state[] = {
+/*ClocksStateInvalid - should not be used*/
+{ .display_clk_khz = 0, .pixel_clk_khz = 0 },
+/*ClocksStateUltraLow - currently by HW design team not supposed to be used*/
+{ .display_clk_khz = 352000, .pixel_clk_khz = 330000 },
+/*ClocksStateLow*/
+{ .display_clk_khz = 352000, .pixel_clk_khz = 330000 },
+/*ClocksStateNominal*/
+{ .display_clk_khz = 467000, .pixel_clk_khz = 400000 },
+/*ClocksStatePerformance*/
+{ .display_clk_khz = 643000, .pixel_clk_khz = 400000 } };
+
+static int determine_sclk_from_bounding_box(
+ const struct dc *dc,
+ int required_sclk)
+{
+ int i;
+
+ /*
+ * Some asics do not give us sclk levels, so we just report the actual
+ * required sclk
+ */
+ if (dc->sclk_lvls.num_levels == 0)
+ return required_sclk;
+
+ for (i = 0; i < dc->sclk_lvls.num_levels; i++) {
+ if (dc->sclk_lvls.clocks_in_khz[i] >= required_sclk)
+ return dc->sclk_lvls.clocks_in_khz[i];
+ }
+ /*
+ * even maximum level could not satisfy requirement, this
+ * is unexpected at this stage, should have been caught at
+ * validation time
+ */
+ ASSERT(0);
+ return dc->sclk_lvls.clocks_in_khz[dc->sclk_lvls.num_levels - 1];
+}
+
+uint32_t dce110_get_min_vblank_time_us(const struct dc_state *context)
+{
+ uint8_t j;
+ uint32_t min_vertical_blank_time = -1;
+
+ for (j = 0; j < context->stream_count; j++) {
+ struct dc_stream_state *stream = context->streams[j];
+ uint32_t vertical_blank_in_pixels = 0;
+ uint32_t vertical_blank_time = 0;
+
+ vertical_blank_in_pixels = stream->timing.h_total *
+ (stream->timing.v_total
+ - stream->timing.v_addressable);
+
+ vertical_blank_time = vertical_blank_in_pixels
+ * 10000 / stream->timing.pix_clk_100hz;
+
+ if (min_vertical_blank_time > vertical_blank_time)
+ min_vertical_blank_time = vertical_blank_time;
+ }
+
+ return min_vertical_blank_time;
+}
+
+void dce110_fill_display_configs(
+ const struct dc_state *context,
+ struct dm_pp_display_configuration *pp_display_cfg)
+{
+ int j;
+ int num_cfgs = 0;
+
+ for (j = 0; j < context->stream_count; j++) {
+ int k;
+
+ const struct dc_stream_state *stream = context->streams[j];
+ struct dm_pp_single_disp_config *cfg =
+ &pp_display_cfg->disp_configs[num_cfgs];
+ const struct pipe_ctx *pipe_ctx = NULL;
+
+ for (k = 0; k < MAX_PIPES; k++)
+ if (stream == context->res_ctx.pipe_ctx[k].stream) {
+ pipe_ctx = &context->res_ctx.pipe_ctx[k];
+ break;
+ }
+
+ ASSERT(pipe_ctx != NULL);
+
+ /* only notify active stream */
+ if (stream->dpms_off)
+ continue;
+
+ num_cfgs++;
+ cfg->signal = pipe_ctx->stream->signal;
+ cfg->pipe_idx = pipe_ctx->stream_res.tg->inst;
+ cfg->src_height = stream->src.height;
+ cfg->src_width = stream->src.width;
+ cfg->ddi_channel_mapping =
+ stream->link->ddi_channel_mapping.raw;
+ cfg->transmitter =
+ stream->link->link_enc->transmitter;
+ cfg->link_settings.lane_count =
+ stream->link->cur_link_settings.lane_count;
+ cfg->link_settings.link_rate =
+ stream->link->cur_link_settings.link_rate;
+ cfg->link_settings.link_spread =
+ stream->link->cur_link_settings.link_spread;
+ cfg->sym_clock = stream->phy_pix_clk;
+ /* Round v_refresh*/
+ cfg->v_refresh = stream->timing.pix_clk_100hz * 100;
+ cfg->v_refresh /= stream->timing.h_total;
+ cfg->v_refresh = (cfg->v_refresh + stream->timing.v_total / 2)
+ / stream->timing.v_total;
+ }
+
+ pp_display_cfg->display_count = num_cfgs;
+}
+
+void dce11_pplib_apply_display_requirements(
+ struct dc *dc,
+ struct dc_state *context)
+{
+ struct dm_pp_display_configuration *pp_display_cfg = &context->pp_display_cfg;
+
+ pp_display_cfg->all_displays_in_sync =
+ context->bw_ctx.bw.dce.all_displays_in_sync;
+ pp_display_cfg->nb_pstate_switch_disable =
+ context->bw_ctx.bw.dce.nbp_state_change_enable == false;
+ pp_display_cfg->cpu_cc6_disable =
+ context->bw_ctx.bw.dce.cpuc_state_change_enable == false;
+ pp_display_cfg->cpu_pstate_disable =
+ context->bw_ctx.bw.dce.cpup_state_change_enable == false;
+ pp_display_cfg->cpu_pstate_separation_time =
+ context->bw_ctx.bw.dce.blackout_recovery_time_us;
+
+ pp_display_cfg->min_memory_clock_khz = context->bw_ctx.bw.dce.yclk_khz
+ / MEMORY_TYPE_MULTIPLIER_CZ;
+
+ pp_display_cfg->min_engine_clock_khz = determine_sclk_from_bounding_box(
+ dc,
+ context->bw_ctx.bw.dce.sclk_khz);
+
+ /*
+ * As workaround for >4x4K lightup set dcfclock to min_engine_clock value.
+ * This is not required for less than 5 displays,
+ * thus don't request decfclk in dc to avoid impact
+ * on power saving.
+ *
+ */
+ pp_display_cfg->min_dcfclock_khz = (context->stream_count > 4) ?
+ pp_display_cfg->min_engine_clock_khz : 0;
+
+ pp_display_cfg->min_engine_clock_deep_sleep_khz
+ = context->bw_ctx.bw.dce.sclk_deep_sleep_khz;
+
+ pp_display_cfg->avail_mclk_switch_time_us =
+ dce110_get_min_vblank_time_us(context);
+ /* TODO: dce11.2*/
+ pp_display_cfg->avail_mclk_switch_time_in_disp_active_us = 0;
+
+ pp_display_cfg->disp_clk_khz = dc->clk_mgr->clks.dispclk_khz;
+
+ dce110_fill_display_configs(context, pp_display_cfg);
+
+ /* TODO: is this still applicable?*/
+ if (pp_display_cfg->display_count == 1) {
+ const struct dc_crtc_timing *timing =
+ &context->streams[0]->timing;
+
+ pp_display_cfg->crtc_index =
+ pp_display_cfg->disp_configs[0].pipe_idx;
+ pp_display_cfg->line_time_in_us = timing->h_total * 10000 / timing->pix_clk_100hz;
+ }
+
+ if (memcmp(&dc->current_state->pp_display_cfg, pp_display_cfg, sizeof(*pp_display_cfg)) != 0)
+ dm_pp_apply_display_requirements(dc->ctx, pp_display_cfg);
+}
+
+static void dce11_update_clocks(struct clk_mgr *clk_mgr_base,
+ struct dc_state *context,
+ bool safe_to_lower)
+{
+ struct clk_mgr_internal *clk_mgr_dce = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+ struct dm_pp_power_level_change_request level_change_req;
+ int patched_disp_clk = context->bw_ctx.bw.dce.dispclk_khz;
+
+ /*TODO: W/A for dal3 linux, investigate why this works */
+ if (!clk_mgr_dce->dfs_bypass_active)
+ patched_disp_clk = patched_disp_clk * 115 / 100;
+
+ level_change_req.power_level = dce_get_required_clocks_state(clk_mgr_base, context);
+ /* get max clock state from PPLIB */
+ if ((level_change_req.power_level < clk_mgr_dce->cur_min_clks_state && safe_to_lower)
+ || level_change_req.power_level > clk_mgr_dce->cur_min_clks_state) {
+ if (dm_pp_apply_power_level_change_request(clk_mgr_base->ctx, &level_change_req))
+ clk_mgr_dce->cur_min_clks_state = level_change_req.power_level;
+ }
+
+ if (should_set_clock(safe_to_lower, patched_disp_clk, clk_mgr_base->clks.dispclk_khz)) {
+ context->bw_ctx.bw.dce.dispclk_khz = dce_set_clock(clk_mgr_base, patched_disp_clk);
+ clk_mgr_base->clks.dispclk_khz = patched_disp_clk;
+ }
+ dce11_pplib_apply_display_requirements(clk_mgr_base->ctx->dc, context);
+}
+
+static struct clk_mgr_funcs dce110_funcs = {
+ .get_dp_ref_clk_frequency = dce_get_dp_ref_freq_khz,
+ .update_clocks = dce11_update_clocks
+};
+
+void dce110_clk_mgr_construct(
+ struct dc_context *ctx,
+ struct clk_mgr_internal *clk_mgr)
+{
+ dce_clk_mgr_construct(ctx, clk_mgr);
+
+ memcpy(clk_mgr->max_clks_by_state,
+ dce110_max_clks_by_state,
+ sizeof(dce110_max_clks_by_state));
+
+ clk_mgr->regs = &disp_clk_regs;
+ clk_mgr->clk_mgr_shift = &disp_clk_shift;
+ clk_mgr->clk_mgr_mask = &disp_clk_mask;
+ clk_mgr->base.funcs = &dce110_funcs;
+
+}
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce110/dce110_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce110/dce110_clk_mgr.h
new file mode 100644
index 000000000000..c0eb2ea6fb3a
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce110/dce110_clk_mgr.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2012-16 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef DAL_DC_DCE_DCE110_CLK_MGR_H_
+#define DAL_DC_DCE_DCE110_CLK_MGR_H_
+
+void dce110_clk_mgr_construct(
+ struct dc_context *ctx,
+ struct clk_mgr_internal *clk_mgr);
+
+void dce110_fill_display_configs(
+ const struct dc_state *context,
+ struct dm_pp_display_configuration *pp_display_cfg);
+
+/* functions shared with other clk mgr*/
+void dce11_pplib_apply_display_requirements(
+ struct dc *dc,
+ struct dc_state *context);
+
+uint32_t dce110_get_min_vblank_time_us(const struct dc_state *context);
+
+#endif /* DAL_DC_DCE_DCE110_CLK_MGR_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce112/dce112_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce112/dce112_clk_mgr.c
new file mode 100644
index 000000000000..7c746ef1e32e
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce112/dce112_clk_mgr.c
@@ -0,0 +1,239 @@
+/*
+ * Copyright 2012-16 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "core_types.h"
+#include "clk_mgr_internal.h"
+
+#include "dce/dce_11_2_d.h"
+#include "dce/dce_11_2_sh_mask.h"
+#include "dce100/dce_clk_mgr.h"
+#include "dce110/dce110_clk_mgr.h"
+#include "dce112_clk_mgr.h"
+#include "dal_asic_id.h"
+
+/* set register offset */
+#define SR(reg_name)\
+ .reg_name = mm ## reg_name
+
+/* set register offset with instance */
+#define SRI(reg_name, block, id)\
+ .reg_name = mm ## block ## id ## _ ## reg_name
+
+static const struct clk_mgr_registers disp_clk_regs = {
+ CLK_COMMON_REG_LIST_DCE_BASE()
+};
+
+static const struct clk_mgr_shift disp_clk_shift = {
+ CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
+};
+
+static const struct clk_mgr_mask disp_clk_mask = {
+ CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
+};
+
+static const struct state_dependent_clocks dce112_max_clks_by_state[] = {
+/*ClocksStateInvalid - should not be used*/
+{ .display_clk_khz = 0, .pixel_clk_khz = 0 },
+/*ClocksStateUltraLow - currently by HW design team not supposed to be used*/
+{ .display_clk_khz = 389189, .pixel_clk_khz = 346672 },
+/*ClocksStateLow*/
+{ .display_clk_khz = 459000, .pixel_clk_khz = 400000 },
+/*ClocksStateNominal*/
+{ .display_clk_khz = 667000, .pixel_clk_khz = 600000 },
+/*ClocksStatePerformance*/
+{ .display_clk_khz = 1132000, .pixel_clk_khz = 600000 } };
+
+
+//TODO: remove use the two broken down functions
+int dce112_set_clock(struct clk_mgr *clk_mgr_base, int requested_clk_khz)
+{
+ struct clk_mgr_internal *clk_mgr_dce = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+ struct bp_set_dce_clock_parameters dce_clk_params;
+ struct dc_bios *bp = clk_mgr_base->ctx->dc_bios;
+ struct dc *core_dc = clk_mgr_base->ctx->dc;
+ struct dmcu *dmcu = core_dc->res_pool->dmcu;
+ int actual_clock = requested_clk_khz;
+ /* Prepare to program display clock*/
+ memset(&dce_clk_params, 0, sizeof(dce_clk_params));
+
+ /* Make sure requested clock isn't lower than minimum threshold*/
+ if (requested_clk_khz > 0)
+ requested_clk_khz = max(requested_clk_khz,
+ clk_mgr_dce->dentist_vco_freq_khz / 62);
+
+ dce_clk_params.target_clock_frequency = requested_clk_khz;
+ dce_clk_params.pll_id = CLOCK_SOURCE_ID_DFS;
+ dce_clk_params.clock_type = DCECLOCK_TYPE_DISPLAY_CLOCK;
+
+ bp->funcs->set_dce_clock(bp, &dce_clk_params);
+ actual_clock = dce_clk_params.target_clock_frequency;
+
+ /*
+ * from power down, we need mark the clock state as ClocksStateNominal
+ * from HWReset, so when resume we will call pplib voltage regulator.
+ */
+ if (requested_clk_khz == 0)
+ clk_mgr_dce->cur_min_clks_state = DM_PP_CLOCKS_STATE_NOMINAL;
+
+ /*Program DP ref Clock*/
+ /*VBIOS will determine DPREFCLK frequency, so we don't set it*/
+ dce_clk_params.target_clock_frequency = 0;
+ dce_clk_params.clock_type = DCECLOCK_TYPE_DPREFCLK;
+ if (!ASICREV_IS_VEGA20_P(clk_mgr_base->ctx->asic_id.hw_internal_rev))
+ dce_clk_params.flags.USE_GENLOCK_AS_SOURCE_FOR_DPREFCLK =
+ (dce_clk_params.pll_id ==
+ CLOCK_SOURCE_COMBO_DISPLAY_PLL0);
+ else
+ dce_clk_params.flags.USE_GENLOCK_AS_SOURCE_FOR_DPREFCLK = false;
+
+ bp->funcs->set_dce_clock(bp, &dce_clk_params);
+
+ if (!IS_FPGA_MAXIMUS_DC(core_dc->ctx->dce_environment)) {
+ if (dmcu && dmcu->funcs->is_dmcu_initialized(dmcu)) {
+ if (clk_mgr_dce->dfs_bypass_disp_clk != actual_clock)
+ dmcu->funcs->set_psr_wait_loop(dmcu,
+ actual_clock / 1000 / 7);
+ }
+ }
+
+ clk_mgr_dce->dfs_bypass_disp_clk = actual_clock;
+ return actual_clock;
+}
+
+int dce112_set_dispclk(struct clk_mgr_internal *clk_mgr, int requested_clk_khz)
+{
+ struct bp_set_dce_clock_parameters dce_clk_params;
+ struct dc_bios *bp = clk_mgr->base.ctx->dc_bios;
+ struct dc *core_dc = clk_mgr->base.ctx->dc;
+ struct dmcu *dmcu = core_dc->res_pool->dmcu;
+ int actual_clock = requested_clk_khz;
+ /* Prepare to program display clock*/
+ memset(&dce_clk_params, 0, sizeof(dce_clk_params));
+
+ /* Make sure requested clock isn't lower than minimum threshold*/
+ if (requested_clk_khz > 0)
+ requested_clk_khz = max(requested_clk_khz,
+ clk_mgr->dentist_vco_freq_khz / 62);
+
+ dce_clk_params.target_clock_frequency = requested_clk_khz;
+ dce_clk_params.pll_id = CLOCK_SOURCE_ID_DFS;
+ dce_clk_params.clock_type = DCECLOCK_TYPE_DISPLAY_CLOCK;
+
+ bp->funcs->set_dce_clock(bp, &dce_clk_params);
+ actual_clock = dce_clk_params.target_clock_frequency;
+
+ /*
+ * from power down, we need mark the clock state as ClocksStateNominal
+ * from HWReset, so when resume we will call pplib voltage regulator.
+ */
+ if (requested_clk_khz == 0)
+ clk_mgr->cur_min_clks_state = DM_PP_CLOCKS_STATE_NOMINAL;
+
+
+ if (!IS_FPGA_MAXIMUS_DC(core_dc->ctx->dce_environment)) {
+ if (dmcu && dmcu->funcs->is_dmcu_initialized(dmcu)) {
+ if (clk_mgr->dfs_bypass_disp_clk != actual_clock)
+ dmcu->funcs->set_psr_wait_loop(dmcu,
+ actual_clock / 1000 / 7);
+ }
+ }
+
+ clk_mgr->dfs_bypass_disp_clk = actual_clock;
+ return actual_clock;
+
+}
+
+int dce112_set_dprefclk(struct clk_mgr_internal *clk_mgr)
+{
+ struct bp_set_dce_clock_parameters dce_clk_params;
+ struct dc_bios *bp = clk_mgr->base.ctx->dc_bios;
+
+ memset(&dce_clk_params, 0, sizeof(dce_clk_params));
+
+ /*Program DP ref Clock*/
+ /*VBIOS will determine DPREFCLK frequency, so we don't set it*/
+ dce_clk_params.target_clock_frequency = 0;
+ dce_clk_params.pll_id = CLOCK_SOURCE_ID_DFS;
+ dce_clk_params.clock_type = DCECLOCK_TYPE_DPREFCLK;
+ if (!ASICREV_IS_VEGA20_P(clk_mgr->base.ctx->asic_id.hw_internal_rev))
+ dce_clk_params.flags.USE_GENLOCK_AS_SOURCE_FOR_DPREFCLK =
+ (dce_clk_params.pll_id ==
+ CLOCK_SOURCE_COMBO_DISPLAY_PLL0);
+ else
+ dce_clk_params.flags.USE_GENLOCK_AS_SOURCE_FOR_DPREFCLK = false;
+
+ bp->funcs->set_dce_clock(bp, &dce_clk_params);
+
+ /* Returns the dp_refclk that was set */
+ return dce_clk_params.target_clock_frequency;
+}
+
+static void dce112_update_clocks(struct clk_mgr *clk_mgr_base,
+ struct dc_state *context,
+ bool safe_to_lower)
+{
+ struct clk_mgr_internal *clk_mgr_dce = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+ struct dm_pp_power_level_change_request level_change_req;
+ int patched_disp_clk = context->bw_ctx.bw.dce.dispclk_khz;
+
+ /*TODO: W/A for dal3 linux, investigate why this works */
+ if (!clk_mgr_dce->dfs_bypass_active)
+ patched_disp_clk = patched_disp_clk * 115 / 100;
+
+ level_change_req.power_level = dce_get_required_clocks_state(clk_mgr_base, context);
+ /* get max clock state from PPLIB */
+ if ((level_change_req.power_level < clk_mgr_dce->cur_min_clks_state && safe_to_lower)
+ || level_change_req.power_level > clk_mgr_dce->cur_min_clks_state) {
+ if (dm_pp_apply_power_level_change_request(clk_mgr_base->ctx, &level_change_req))
+ clk_mgr_dce->cur_min_clks_state = level_change_req.power_level;
+ }
+
+ if (should_set_clock(safe_to_lower, patched_disp_clk, clk_mgr_base->clks.dispclk_khz)) {
+ patched_disp_clk = dce112_set_clock(clk_mgr_base, patched_disp_clk);
+ clk_mgr_base->clks.dispclk_khz = patched_disp_clk;
+ }
+ dce11_pplib_apply_display_requirements(clk_mgr_base->ctx->dc, context);
+}
+
+static struct clk_mgr_funcs dce112_funcs = {
+ .get_dp_ref_clk_frequency = dce_get_dp_ref_freq_khz,
+ .update_clocks = dce112_update_clocks
+};
+
+void dce112_clk_mgr_construct(
+ struct dc_context *ctx,
+ struct clk_mgr_internal *clk_mgr)
+{
+ dce_clk_mgr_construct(ctx, clk_mgr);
+
+ memcpy(clk_mgr->max_clks_by_state,
+ dce112_max_clks_by_state,
+ sizeof(dce112_max_clks_by_state));
+
+ clk_mgr->regs = &disp_clk_regs;
+ clk_mgr->clk_mgr_shift = &disp_clk_shift;
+ clk_mgr->clk_mgr_mask = &disp_clk_mask;
+ clk_mgr->base.funcs = &dce112_funcs;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce112/dce112_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce112/dce112_clk_mgr.h
new file mode 100644
index 000000000000..dfb06db118e1
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce112/dce112_clk_mgr.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2012-16 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef DAL_DC_DCE_DCE112_CLK_MGR_H_
+#define DAL_DC_DCE_DCE112_CLK_MGR_H_
+
+
+void dce112_clk_mgr_construct(
+ struct dc_context *ctx,
+ struct clk_mgr_internal *clk_mgr);
+
+/* functions shared with other clk mgr */
+int dce112_set_clock(struct clk_mgr *clk_mgr_base, int requested_clk_khz);
+int dce112_set_dispclk(struct clk_mgr_internal *clk_mgr, int requested_clk_khz);
+int dce112_set_dprefclk(struct clk_mgr_internal *clk_mgr);
+
+#endif /* DAL_DC_DCE_DCE112_CLK_MGR_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce120/dce120_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce120/dce120_clk_mgr.c
new file mode 100644
index 000000000000..5399b8cf6b75
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce120/dce120_clk_mgr.c
@@ -0,0 +1,153 @@
+/*
+ * Copyright 2012-16 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "core_types.h"
+#include "clk_mgr_internal.h"
+
+#include "dce112/dce112_clk_mgr.h"
+#include "dce110/dce110_clk_mgr.h"
+#include "dce120_clk_mgr.h"
+#include "dce100/dce_clk_mgr.h"
+#include "dce120/dce120_hw_sequencer.h"
+
+static const struct state_dependent_clocks dce120_max_clks_by_state[] = {
+/*ClocksStateInvalid - should not be used*/
+{ .display_clk_khz = 0, .pixel_clk_khz = 0 },
+/*ClocksStateUltraLow - currently by HW design team not supposed to be used*/
+{ .display_clk_khz = 0, .pixel_clk_khz = 0 },
+/*ClocksStateLow*/
+{ .display_clk_khz = 460000, .pixel_clk_khz = 400000 },
+/*ClocksStateNominal*/
+{ .display_clk_khz = 670000, .pixel_clk_khz = 600000 },
+/*ClocksStatePerformance*/
+{ .display_clk_khz = 1133000, .pixel_clk_khz = 600000 } };
+
+/**
+ * dce121_clock_patch_xgmi_ss_info() - Save XGMI spread spectrum info
+ * @clk_mgr_dce: clock manager internal structure
+ *
+ * Reads from VBIOS the XGMI spread spectrum info and saves it within
+ * the dce clock manager. This operation will overwrite the existing dprefclk
+ * SS values if the vBIOS query succeeds. Otherwise, it does nothing. It also
+ * sets the ->xgmi_enabled flag.
+ */
+static void dce121_clock_patch_xgmi_ss_info(struct clk_mgr_internal *clk_mgr_dce)
+{
+ enum bp_result result;
+ struct spread_spectrum_info info = { { 0 } };
+ struct dc_bios *bp = clk_mgr_dce->base.ctx->dc_bios;
+
+ clk_mgr_dce->xgmi_enabled = false;
+
+ result = bp->funcs->get_spread_spectrum_info(bp, AS_SIGNAL_TYPE_XGMI,
+ 0, &info);
+ if (result == BP_RESULT_OK && info.spread_spectrum_percentage != 0) {
+ clk_mgr_dce->xgmi_enabled = true;
+ clk_mgr_dce->ss_on_dprefclk = true;
+ clk_mgr_dce->dprefclk_ss_divider =
+ info.spread_percentage_divider;
+
+ if (info.type.CENTER_MODE == 0) {
+ /*
+ * Currently for DP Reference clock we
+ * need only SS percentage for
+ * downspread
+ */
+ clk_mgr_dce->dprefclk_ss_percentage =
+ info.spread_spectrum_percentage;
+ }
+ }
+}
+
+static void dce12_update_clocks(struct clk_mgr *clk_mgr_base,
+ struct dc_state *context,
+ bool safe_to_lower)
+{
+ struct clk_mgr_internal *clk_mgr_dce = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+ struct dm_pp_clock_for_voltage_req clock_voltage_req = {0};
+ int max_pix_clk = dce_get_max_pixel_clock_for_all_paths(context);
+ int patched_disp_clk = context->bw_ctx.bw.dce.dispclk_khz;
+
+ /*TODO: W/A for dal3 linux, investigate why this works */
+ if (!clk_mgr_dce->dfs_bypass_active)
+ patched_disp_clk = patched_disp_clk * 115 / 100;
+
+ if (should_set_clock(safe_to_lower, patched_disp_clk, clk_mgr_base->clks.dispclk_khz)) {
+ clock_voltage_req.clk_type = DM_PP_CLOCK_TYPE_DISPLAY_CLK;
+ /*
+ * When xGMI is enabled, the display clk needs to be adjusted
+ * with the WAFL link's SS percentage.
+ */
+ if (clk_mgr_dce->xgmi_enabled)
+ patched_disp_clk = dce_adjust_dp_ref_freq_for_ss(
+ clk_mgr_dce, patched_disp_clk);
+ clock_voltage_req.clocks_in_khz = patched_disp_clk;
+ clk_mgr_base->clks.dispclk_khz = dce112_set_clock(clk_mgr_base, patched_disp_clk);
+
+ dm_pp_apply_clock_for_voltage_request(clk_mgr_base->ctx, &clock_voltage_req);
+ }
+
+ if (should_set_clock(safe_to_lower, max_pix_clk, clk_mgr_base->clks.phyclk_khz)) {
+ clock_voltage_req.clk_type = DM_PP_CLOCK_TYPE_DISPLAYPHYCLK;
+ clock_voltage_req.clocks_in_khz = max_pix_clk;
+ clk_mgr_base->clks.phyclk_khz = max_pix_clk;
+
+ dm_pp_apply_clock_for_voltage_request(clk_mgr_base->ctx, &clock_voltage_req);
+ }
+ dce11_pplib_apply_display_requirements(clk_mgr_base->ctx->dc, context);
+}
+
+
+static struct clk_mgr_funcs dce120_funcs = {
+ .get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+ .update_clocks = dce12_update_clocks
+};
+
+void dce120_clk_mgr_construct(struct dc_context *ctx, struct clk_mgr_internal *clk_mgr)
+{
+ dce_clk_mgr_construct(ctx, clk_mgr);
+
+ memcpy(clk_mgr->max_clks_by_state,
+ dce120_max_clks_by_state,
+ sizeof(dce120_max_clks_by_state));
+
+ clk_mgr->base.dprefclk_khz = 600000;
+ clk_mgr->base.funcs = &dce120_funcs;
+}
+
+void dce121_clk_mgr_construct(struct dc_context *ctx, struct clk_mgr_internal *clk_mgr)
+{
+ dce120_clk_mgr_construct(ctx, clk_mgr);
+ clk_mgr->base.dprefclk_khz = 625000;
+
+ /*
+ * The xGMI enabled info is used to determine if audio and display
+ * clocks need to be adjusted with the WAFL link's SS info.
+ */
+ if (dce121_xgmi_enabled(ctx->dc->hwseq))
+ dce121_clock_patch_xgmi_ss_info(clk_mgr);
+
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce120/dce120_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce120/dce120_clk_mgr.h
new file mode 100644
index 000000000000..d12d6fcb167d
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce120/dce120_clk_mgr.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2012-16 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef DAL_DC_DCE_DCE120_CLK_MGR_H_
+#define DAL_DC_DCE_DCE120_CLK_MGR_H_
+
+void dce120_clk_mgr_construct(struct dc_context *ctx, struct clk_mgr_internal *clk_mgr);
+void dce121_clk_mgr_construct(struct dc_context *ctx, struct clk_mgr_internal *clk_mgr);
+
+
+
+#endif /* DAL_DC_DCE_DCE120_CLK_MGR_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c
index 2b2de1d913c9..caf8a4a4e442 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c
@@ -23,32 +23,23 @@
*
*/
-#include "dcn10_clk_mgr.h"
+#include <linux/slab.h>
#include "reg_helper.h"
#include "core_types.h"
-
-#define TO_DCE_CLK_MGR(clocks)\
- container_of(clocks, struct dce_clk_mgr, base)
-
-#define REG(reg) \
- (clk_mgr_dce->regs->reg)
-
-#undef FN
-#define FN(reg_name, field_name) \
- clk_mgr_dce->clk_mgr_shift->field_name, clk_mgr_dce->clk_mgr_mask->field_name
-
-#define CTX \
- clk_mgr_dce->base.ctx
-#define DC_LOGGER \
- clk_mgr->ctx->logger
-
-static int dcn1_determine_dppclk_threshold(struct clk_mgr *clk_mgr, struct dc_clocks *new_clocks)
+#include "clk_mgr_internal.h"
+#include "rv1_clk_mgr.h"
+#include "dce100/dce_clk_mgr.h"
+#include "dce112/dce112_clk_mgr.h"
+#include "rv1_clk_mgr_vbios_smu.h"
+#include "rv1_clk_mgr_clk.h"
+
+static int rv1_determine_dppclk_threshold(struct clk_mgr_internal *clk_mgr, struct dc_clocks *new_clocks)
{
bool request_dpp_div = new_clocks->dispclk_khz > new_clocks->dppclk_khz;
- bool dispclk_increase = new_clocks->dispclk_khz > clk_mgr->clks.dispclk_khz;
+ bool dispclk_increase = new_clocks->dispclk_khz > clk_mgr->base.clks.dispclk_khz;
int disp_clk_threshold = new_clocks->max_supported_dppclk_khz;
- bool cur_dpp_div = clk_mgr->clks.dispclk_khz > clk_mgr->clks.dppclk_khz;
+ bool cur_dpp_div = clk_mgr->base.clks.dispclk_khz > clk_mgr->base.clks.dppclk_khz;
/* increase clock, looking for div is 0 for current, request div is 1*/
if (dispclk_increase) {
@@ -78,7 +69,7 @@ static int dcn1_determine_dppclk_threshold(struct clk_mgr *clk_mgr, struct dc_cl
/* current disp clk is lower than current maximum dpp clk,
* no need to ramp
*/
- if (clk_mgr->clks.dispclk_khz <= disp_clk_threshold)
+ if (clk_mgr->base.clks.dispclk_khz <= disp_clk_threshold)
return new_clocks->dispclk_khz;
/* request dpp clk need to be divided by 2 */
@@ -89,15 +80,17 @@ static int dcn1_determine_dppclk_threshold(struct clk_mgr *clk_mgr, struct dc_cl
return disp_clk_threshold;
}
-static void dcn1_ramp_up_dispclk_with_dpp(struct clk_mgr *clk_mgr, struct dc_clocks *new_clocks)
+static void ramp_up_dispclk_with_dpp(struct clk_mgr_internal *clk_mgr, struct dc *dc, struct dc_clocks *new_clocks)
{
- struct dc *dc = clk_mgr->ctx->dc;
- int dispclk_to_dpp_threshold = dcn1_determine_dppclk_threshold(clk_mgr, new_clocks);
- bool request_dpp_div = new_clocks->dispclk_khz > new_clocks->dppclk_khz;
int i;
+ int dispclk_to_dpp_threshold = rv1_determine_dppclk_threshold(clk_mgr, new_clocks);
+ bool request_dpp_div = new_clocks->dispclk_khz > new_clocks->dppclk_khz;
/* set disp clk to dpp clk threshold */
- dce112_set_clock(clk_mgr, dispclk_to_dpp_threshold);
+
+ clk_mgr->funcs->set_dispclk(clk_mgr, dispclk_to_dpp_threshold);
+ clk_mgr->funcs->set_dprefclk(clk_mgr);
+
/* update request dpp clk division option */
for (i = 0; i < dc->res_pool->pipe_count; i++) {
@@ -113,42 +106,23 @@ static void dcn1_ramp_up_dispclk_with_dpp(struct clk_mgr *clk_mgr, struct dc_clo
}
/* If target clk not same as dppclk threshold, set to target clock */
- if (dispclk_to_dpp_threshold != new_clocks->dispclk_khz)
- dce112_set_clock(clk_mgr, new_clocks->dispclk_khz);
-
- clk_mgr->clks.dispclk_khz = new_clocks->dispclk_khz;
- clk_mgr->clks.dppclk_khz = new_clocks->dppclk_khz;
- clk_mgr->clks.max_supported_dppclk_khz = new_clocks->max_supported_dppclk_khz;
-}
-
-static int get_active_display_cnt(
- struct dc *dc,
- struct dc_state *context)
-{
- int i, display_count;
-
- display_count = 0;
- for (i = 0; i < context->stream_count; i++) {
- const struct dc_stream_state *stream = context->streams[i];
-
- /*
- * Only notify active stream or virtual stream.
- * Need to notify virtual stream to work around
- * headless case. HPD does not fire when system is in
- * S0i2.
- */
- if (!stream->dpms_off || stream->signal == SIGNAL_TYPE_VIRTUAL)
- display_count++;
+ if (dispclk_to_dpp_threshold != new_clocks->dispclk_khz) {
+ clk_mgr->funcs->set_dispclk(clk_mgr, new_clocks->dispclk_khz);
+ clk_mgr->funcs->set_dprefclk(clk_mgr);
}
- return display_count;
+
+ clk_mgr->base.clks.dispclk_khz = new_clocks->dispclk_khz;
+ clk_mgr->base.clks.dppclk_khz = new_clocks->dppclk_khz;
+ clk_mgr->base.clks.max_supported_dppclk_khz = new_clocks->max_supported_dppclk_khz;
}
-static void dcn1_update_clocks(struct clk_mgr *clk_mgr,
+static void rv1_update_clocks(struct clk_mgr *clk_mgr_base,
struct dc_state *context,
bool safe_to_lower)
{
- struct dc *dc = clk_mgr->ctx->dc;
+ struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+ struct dc *dc = clk_mgr_base->ctx->dc;
struct dc_debug_options *debug = &dc->debug;
struct dc_clocks *new_clocks = &context->bw_ctx.bw.dcn.clk;
struct pp_smu_funcs_rv *pp_smu = NULL;
@@ -158,9 +132,12 @@ static void dcn1_update_clocks(struct clk_mgr *clk_mgr,
bool enter_display_off = false;
- display_count = get_active_display_cnt(dc, context);
- if (dc->res_pool->pp_smu)
- pp_smu = &dc->res_pool->pp_smu->rv_funcs;
+ ASSERT(clk_mgr->pp_smu);
+
+ pp_smu = &clk_mgr->pp_smu->rv_funcs;
+
+ display_count = clk_mgr_helper_get_active_display_cnt(dc, context);
+
if (display_count == 0)
enter_display_off = true;
@@ -170,18 +147,18 @@ static void dcn1_update_clocks(struct clk_mgr *clk_mgr,
* if function pointer not set up, this message is
* sent as part of pplib_apply_display_requirements.
*/
- if (pp_smu && pp_smu->set_display_count)
+ if (pp_smu->set_display_count)
pp_smu->set_display_count(&pp_smu->pp_smu, display_count);
}
- if (new_clocks->dispclk_khz > clk_mgr->clks.dispclk_khz
- || new_clocks->phyclk_khz > clk_mgr->clks.phyclk_khz
- || new_clocks->fclk_khz > clk_mgr->clks.fclk_khz
- || new_clocks->dcfclk_khz > clk_mgr->clks.dcfclk_khz)
+ if (new_clocks->dispclk_khz > clk_mgr_base->clks.dispclk_khz
+ || new_clocks->phyclk_khz > clk_mgr_base->clks.phyclk_khz
+ || new_clocks->fclk_khz > clk_mgr_base->clks.fclk_khz
+ || new_clocks->dcfclk_khz > clk_mgr_base->clks.dcfclk_khz)
send_request_to_increase = true;
- if (should_set_clock(safe_to_lower, new_clocks->phyclk_khz, clk_mgr->clks.phyclk_khz)) {
- clk_mgr->clks.phyclk_khz = new_clocks->phyclk_khz;
+ if (should_set_clock(safe_to_lower, new_clocks->phyclk_khz, clk_mgr_base->clks.phyclk_khz)) {
+ clk_mgr_base->clks.phyclk_khz = new_clocks->phyclk_khz;
send_request_to_lower = true;
}
@@ -189,20 +166,20 @@ static void dcn1_update_clocks(struct clk_mgr *clk_mgr,
if (debug->force_fclk_khz != 0)
new_clocks->fclk_khz = debug->force_fclk_khz;
- if (should_set_clock(safe_to_lower, new_clocks->fclk_khz, clk_mgr->clks.fclk_khz)) {
- clk_mgr->clks.fclk_khz = new_clocks->fclk_khz;
+ if (should_set_clock(safe_to_lower, new_clocks->fclk_khz, clk_mgr_base->clks.fclk_khz)) {
+ clk_mgr_base->clks.fclk_khz = new_clocks->fclk_khz;
send_request_to_lower = true;
}
//DCF Clock
- if (should_set_clock(safe_to_lower, new_clocks->dcfclk_khz, clk_mgr->clks.dcfclk_khz)) {
- clk_mgr->clks.dcfclk_khz = new_clocks->dcfclk_khz;
+ if (should_set_clock(safe_to_lower, new_clocks->dcfclk_khz, clk_mgr_base->clks.dcfclk_khz)) {
+ clk_mgr_base->clks.dcfclk_khz = new_clocks->dcfclk_khz;
send_request_to_lower = true;
}
if (should_set_clock(safe_to_lower,
- new_clocks->dcfclk_deep_sleep_khz, clk_mgr->clks.dcfclk_deep_sleep_khz)) {
- clk_mgr->clks.dcfclk_deep_sleep_khz = new_clocks->dcfclk_deep_sleep_khz;
+ new_clocks->dcfclk_deep_sleep_khz, clk_mgr_base->clks.dcfclk_deep_sleep_khz)) {
+ clk_mgr_base->clks.dcfclk_deep_sleep_khz = new_clocks->dcfclk_deep_sleep_khz;
send_request_to_lower = true;
}
@@ -211,10 +188,9 @@ static void dcn1_update_clocks(struct clk_mgr *clk_mgr,
*/
if (send_request_to_increase) {
/*use dcfclk to request voltage*/
- if (pp_smu && pp_smu->set_hard_min_fclk_by_freq &&
+ if (pp_smu->set_hard_min_fclk_by_freq &&
pp_smu->set_hard_min_dcfclk_by_freq &&
pp_smu->set_min_deep_sleep_dcfclk) {
-
pp_smu->set_hard_min_fclk_by_freq(&pp_smu->pp_smu, new_clocks->fclk_khz / 1000);
pp_smu->set_hard_min_dcfclk_by_freq(&pp_smu->pp_smu, new_clocks->dcfclk_khz / 1000);
pp_smu->set_min_deep_sleep_dcfclk(&pp_smu->pp_smu, (new_clocks->dcfclk_deep_sleep_khz + 999) / 1000);
@@ -223,67 +199,81 @@ static void dcn1_update_clocks(struct clk_mgr *clk_mgr,
/* dcn1 dppclk is tied to dispclk */
/* program dispclk on = as a w/a for sleep resume clock ramping issues */
- if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr->clks.dispclk_khz)
- || new_clocks->dispclk_khz == clk_mgr->clks.dispclk_khz) {
- dcn1_ramp_up_dispclk_with_dpp(clk_mgr, new_clocks);
- clk_mgr->clks.dispclk_khz = new_clocks->dispclk_khz;
+ if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz)
+ || new_clocks->dispclk_khz == clk_mgr_base->clks.dispclk_khz) {
+ ramp_up_dispclk_with_dpp(clk_mgr, dc, new_clocks);
+ clk_mgr_base->clks.dispclk_khz = new_clocks->dispclk_khz;
send_request_to_lower = true;
}
if (!send_request_to_increase && send_request_to_lower) {
/*use dcfclk to request voltage*/
- if (pp_smu && pp_smu->set_hard_min_fclk_by_freq &&
+ if (pp_smu->set_hard_min_fclk_by_freq &&
pp_smu->set_hard_min_dcfclk_by_freq &&
pp_smu->set_min_deep_sleep_dcfclk) {
-
pp_smu->set_hard_min_fclk_by_freq(&pp_smu->pp_smu, new_clocks->fclk_khz / 1000);
pp_smu->set_hard_min_dcfclk_by_freq(&pp_smu->pp_smu, new_clocks->dcfclk_khz / 1000);
pp_smu->set_min_deep_sleep_dcfclk(&pp_smu->pp_smu, (new_clocks->dcfclk_deep_sleep_khz + 999) / 1000);
}
}
}
-static const struct clk_mgr_funcs dcn1_funcs = {
+
+static void rv1_enable_pme_wa(struct clk_mgr *clk_mgr_base)
+{
+ struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+ struct pp_smu_funcs_rv *pp_smu = NULL;
+
+ if (clk_mgr->pp_smu) {
+ pp_smu = &clk_mgr->pp_smu->rv_funcs;
+
+ if (pp_smu->set_pme_wa_enable)
+ pp_smu->set_pme_wa_enable(&pp_smu->pp_smu);
+ }
+}
+
+static struct clk_mgr_funcs rv1_clk_funcs = {
.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
- .update_clocks = dcn1_update_clocks
+ .update_clocks = rv1_update_clocks,
+ .enable_pme_wa = rv1_enable_pme_wa,
+};
+
+static struct clk_mgr_internal_funcs rv1_clk_internal_funcs = {
+ .set_dispclk = rv1_vbios_smu_set_dispclk,
+ .set_dprefclk = dce112_set_dprefclk
};
-struct clk_mgr *dcn1_clk_mgr_create(struct dc_context *ctx)
+
+void rv1_clk_mgr_construct(struct dc_context *ctx, struct clk_mgr_internal *clk_mgr, struct pp_smu_funcs *pp_smu)
{
struct dc_debug_options *debug = &ctx->dc->debug;
struct dc_bios *bp = ctx->dc_bios;
struct dc_firmware_info fw_info = { { 0 } };
- struct dce_clk_mgr *clk_mgr_dce = kzalloc(sizeof(*clk_mgr_dce), GFP_KERNEL);
-
- if (clk_mgr_dce == NULL) {
- BREAK_TO_DEBUGGER();
- return NULL;
- }
- clk_mgr_dce->base.ctx = ctx;
- clk_mgr_dce->base.funcs = &dcn1_funcs;
+ clk_mgr->base.ctx = ctx;
+ clk_mgr->pp_smu = pp_smu;
+ clk_mgr->base.funcs = &rv1_clk_funcs;
+ clk_mgr->funcs = &rv1_clk_internal_funcs;
- clk_mgr_dce->dfs_bypass_disp_clk = 0;
+ clk_mgr->dfs_bypass_disp_clk = 0;
- clk_mgr_dce->dprefclk_ss_percentage = 0;
- clk_mgr_dce->dprefclk_ss_divider = 1000;
- clk_mgr_dce->ss_on_dprefclk = false;
+ clk_mgr->dprefclk_ss_percentage = 0;
+ clk_mgr->dprefclk_ss_divider = 1000;
+ clk_mgr->ss_on_dprefclk = false;
+ clk_mgr->base.dprefclk_khz = 600000;
- clk_mgr_dce->dprefclk_khz = 600000;
if (bp->integrated_info)
- clk_mgr_dce->dentist_vco_freq_khz = bp->integrated_info->dentist_vco_freq;
- if (clk_mgr_dce->dentist_vco_freq_khz == 0) {
+ clk_mgr->dentist_vco_freq_khz = bp->integrated_info->dentist_vco_freq;
+ if (clk_mgr->dentist_vco_freq_khz == 0) {
bp->funcs->get_firmware_info(bp, &fw_info);
- clk_mgr_dce->dentist_vco_freq_khz = fw_info.smu_gpu_pll_output_freq;
- if (clk_mgr_dce->dentist_vco_freq_khz == 0)
- clk_mgr_dce->dentist_vco_freq_khz = 3600000;
+ clk_mgr->dentist_vco_freq_khz = fw_info.smu_gpu_pll_output_freq;
+ if (clk_mgr->dentist_vco_freq_khz == 0)
+ clk_mgr->dentist_vco_freq_khz = 3600000;
}
if (!debug->disable_dfs_bypass && bp->integrated_info)
if (bp->integrated_info->gpu_cap_info & DFS_BYPASS_ENABLE)
- clk_mgr_dce->dfs_bypass_enabled = true;
-
- dce_clock_read_ss_info(clk_mgr_dce);
+ clk_mgr->dfs_bypass_enabled = true;
- return &clk_mgr_dce->base;
+ dce_clock_read_ss_info(clk_mgr);
}
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.h
new file mode 100644
index 000000000000..0807478c8212
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __RV1_CLK_MGR_H__
+#define __RV1_CLK_MGR_H__
+
+void rv1_clk_mgr_construct(struct dc_context *ctx, struct clk_mgr_internal *clk_mgr, struct pp_smu_funcs *pp_smu);
+
+#endif //__DCN10_CLK_MGR_H__
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_clk.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_clk.c
new file mode 100644
index 000000000000..61dd12198a3c
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_clk.c
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2012-16 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "reg_helper.h"
+#include "clk_mgr_internal.h"
+#include "rv1_clk_mgr_clk.h"
+
+#include "ip/Discovery/hwid.h"
+#include "ip/Discovery/v1/ip_offset_1.h"
+#include "ip/CLK/clk_10_0_default.h"
+#include "ip/CLK/clk_10_0_offset.h"
+#include "ip/CLK/clk_10_0_reg.h"
+#include "ip/CLK/clk_10_0_sh_mask.h"
+
+#include "dce100/dce_clk_mgr.h"
+
+#define CLK_BASE_INNER(inst) \
+ CLK_BASE__INST ## inst ## _SEG0
+
+
+#define CLK_REG(reg_name, block, inst)\
+ CLK_BASE(mm ## block ## _ ## inst ## _ ## reg_name ## _BASE_IDX) + \
+ mm ## block ## _ ## inst ## _ ## reg_name
+
+#define REG(reg_name) \
+ CLK_REG(reg_name, CLK0, 0)
+
+
+/* Only used by testing framework*/
+void rv1_dump_clk_registers(struct clk_state_registers *regs, struct clk_bypass *bypass, struct clk_mgr *clk_mgr_base)
+{
+ struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+
+ regs->CLK0_CLK8_CURRENT_CNT = REG_READ(CLK0_CLK8_CURRENT_CNT) / 10; //dcf clk
+
+ bypass->dcfclk_bypass = REG_READ(CLK0_CLK8_BYPASS_CNTL) & 0x0007;
+ if (bypass->dcfclk_bypass < 0 || bypass->dcfclk_bypass > 4)
+ bypass->dcfclk_bypass = 0;
+
+
+ regs->CLK0_CLK8_DS_CNTL = REG_READ(CLK0_CLK8_DS_CNTL) / 10; //dcf deep sleep divider
+
+ regs->CLK0_CLK8_ALLOW_DS = REG_READ(CLK0_CLK8_ALLOW_DS); //dcf deep sleep allow
+
+ regs->CLK0_CLK10_CURRENT_CNT = REG_READ(CLK0_CLK10_CURRENT_CNT) / 10; //dpref clk
+
+ bypass->dispclk_pypass = REG_READ(CLK0_CLK10_BYPASS_CNTL) & 0x0007;
+ if (bypass->dispclk_pypass < 0 || bypass->dispclk_pypass > 4)
+ bypass->dispclk_pypass = 0;
+
+ regs->CLK0_CLK11_CURRENT_CNT = REG_READ(CLK0_CLK11_CURRENT_CNT) / 10; //disp clk
+
+ bypass->dprefclk_bypass = REG_READ(CLK0_CLK11_BYPASS_CNTL) & 0x0007;
+ if (bypass->dprefclk_bypass < 0 || bypass->dprefclk_bypass > 4)
+ bypass->dprefclk_bypass = 0;
+
+}
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_clk.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_clk.h
new file mode 100644
index 000000000000..b68e3452efb9
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_clk.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef DAL_DC_DCN10_RV1_CLK_MGR_CLK_H_
+#define DAL_DC_DCN10_RV1_CLK_MGR_CLK_H_
+
+#endif /* DAL_DC_DCN10_RV1_CLK_MGR_CLK_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_vbios_smu.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_vbios_smu.c
new file mode 100644
index 000000000000..1897e91c8ccb
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_vbios_smu.c
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2012-16 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "core_types.h"
+#include "clk_mgr_internal.h"
+#include "reg_helper.h"
+
+#define MAX_INSTANCE 5
+#define MAX_SEGMENT 5
+
+struct IP_BASE_INSTANCE {
+ unsigned int segment[MAX_SEGMENT];
+};
+
+struct IP_BASE {
+ struct IP_BASE_INSTANCE instance[MAX_INSTANCE];
+};
+
+
+static const struct IP_BASE MP1_BASE = { { { { 0x00016000, 0, 0, 0, 0 } },
+ { { 0, 0, 0, 0, 0 } },
+ { { 0, 0, 0, 0, 0 } },
+ { { 0, 0, 0, 0, 0 } },
+ { { 0, 0, 0, 0, 0 } } } };
+
+#define mmMP1_SMN_C2PMSG_91 0x29B
+#define mmMP1_SMN_C2PMSG_83 0x293
+#define mmMP1_SMN_C2PMSG_67 0x283
+#define mmMP1_SMN_C2PMSG_91_BASE_IDX 0
+#define mmMP1_SMN_C2PMSG_83_BASE_IDX 0
+#define mmMP1_SMN_C2PMSG_67_BASE_IDX 0
+
+#define MP1_SMN_C2PMSG_91__CONTENT_MASK 0xffffffffL
+#define MP1_SMN_C2PMSG_83__CONTENT_MASK 0xffffffffL
+#define MP1_SMN_C2PMSG_67__CONTENT_MASK 0xffffffffL
+#define MP1_SMN_C2PMSG_91__CONTENT__SHIFT 0x00000000
+#define MP1_SMN_C2PMSG_83__CONTENT__SHIFT 0x00000000
+#define MP1_SMN_C2PMSG_67__CONTENT__SHIFT 0x00000000
+
+#define REG(reg_name) \
+ (MP1_BASE.instance[0].segment[mm ## reg_name ## _BASE_IDX] + mm ## reg_name)
+
+#define FN(reg_name, field) \
+ FD(reg_name##__##field)
+
+#define VBIOSSMC_MSG_SetDispclkFreq 0x4
+#define VBIOSSMC_MSG_SetDprefclkFreq 0x5
+
+int rv1_vbios_smu_send_msg_with_param(struct clk_mgr_internal *clk_mgr, unsigned int msg_id, unsigned int param)
+{
+ /* First clear response register */
+ REG_WRITE(MP1_SMN_C2PMSG_91, 0);
+
+ /* Set the parameter register for the SMU message, unit is Mhz */
+ REG_WRITE(MP1_SMN_C2PMSG_83, param);
+
+ /* Trigger the message transaction by writing the message ID */
+ REG_WRITE(MP1_SMN_C2PMSG_67, msg_id);
+
+ REG_WAIT(MP1_SMN_C2PMSG_91, CONTENT, 1, 10, 200000);
+
+ /* Actual dispclk set is returned in the parameter register */
+ return REG_READ(MP1_SMN_C2PMSG_83);
+}
+
+int rv1_vbios_smu_set_dispclk(struct clk_mgr_internal *clk_mgr, int requested_dispclk_khz)
+{
+ int actual_dispclk_set_mhz = -1;
+ struct dc *core_dc = clk_mgr->base.ctx->dc;
+ struct dmcu *dmcu = core_dc->res_pool->dmcu;
+
+ /* Unit of SMU msg parameter is Mhz */
+ actual_dispclk_set_mhz = rv1_vbios_smu_send_msg_with_param(
+ clk_mgr,
+ VBIOSSMC_MSG_SetDispclkFreq,
+ requested_dispclk_khz / 1000);
+
+ /* Actual dispclk set is returned in the parameter register */
+ actual_dispclk_set_mhz = REG_READ(MP1_SMN_C2PMSG_83) * 1000;
+
+ if (!IS_FPGA_MAXIMUS_DC(core_dc->ctx->dce_environment)) {
+ if (dmcu && dmcu->funcs->is_dmcu_initialized(dmcu)) {
+ if (clk_mgr->dfs_bypass_disp_clk != actual_dispclk_set_mhz)
+ dmcu->funcs->set_psr_wait_loop(dmcu,
+ actual_dispclk_set_mhz / 7);
+ }
+ }
+
+ return actual_dispclk_set_mhz * 1000;
+}
+
+int rv1_vbios_smu_set_dprefclk(struct clk_mgr_internal *clk_mgr)
+{
+ int actual_dprefclk_set_mhz = -1;
+
+ actual_dprefclk_set_mhz = rv1_vbios_smu_send_msg_with_param(
+ clk_mgr,
+ VBIOSSMC_MSG_SetDprefclkFreq,
+ clk_mgr->base.dprefclk_khz / 1000);
+
+ /* TODO: add code for programing DP DTO, currently this is down by command table */
+
+ return actual_dprefclk_set_mhz * 1000;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_vbios_smu.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_vbios_smu.h
new file mode 100644
index 000000000000..083cb3158859
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_vbios_smu.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef DAL_DC_DCN10_RV1_CLK_MGR_VBIOS_SMU_H_
+#define DAL_DC_DCN10_RV1_CLK_MGR_VBIOS_SMU_H_
+
+int rv1_vbios_smu_set_dispclk(struct clk_mgr_internal *clk_mgr, int requested_dispclk_khz);
+int rv1_vbios_smu_set_dprefclk(struct clk_mgr_internal *clk_mgr);
+
+#endif /* DAL_DC_DCN10_RV1_CLK_MGR_VBIOS_SMU_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv2_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv2_clk_mgr.c
new file mode 100644
index 000000000000..b9ba6dbc2b46
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv2_clk_mgr.c
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "core_types.h"
+#include "clk_mgr_internal.h"
+#include "rv1_clk_mgr.h"
+#include "rv2_clk_mgr.h"
+#include "dce112/dce112_clk_mgr.h"
+
+static struct clk_mgr_internal_funcs rv2_clk_internal_funcs = {
+ .set_dispclk = dce112_set_dispclk,
+ .set_dprefclk = dce112_set_dprefclk
+};
+
+void rv2_clk_mgr_construct(struct dc_context *ctx, struct clk_mgr_internal *clk_mgr, struct pp_smu_funcs *pp_smu)
+
+{
+ rv1_clk_mgr_construct(ctx, clk_mgr, pp_smu);
+
+ clk_mgr->funcs = &rv2_clk_internal_funcs;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv2_clk_mgr.h
index 97007cf33665..0c1f26ca563b 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_clk_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv2_clk_mgr.h
@@ -23,17 +23,10 @@
*
*/
-#ifndef __DCN10_CLK_MGR_H__
-#define __DCN10_CLK_MGR_H__
+#ifndef __RV2_CLK_MGR_H__
+#define __RV2_CLK_MGR_H__
-#include "../dce/dce_clk_mgr.h"
+void rv2_clk_mgr_construct(struct dc_context *ctx, struct clk_mgr_internal *clk_mgr, struct pp_smu_funcs *pp_smu);
-struct clk_bypass {
- uint32_t dcfclk_bypass;
- uint32_t dispclk_pypass;
- uint32_t dprefclk_bypass;
-};
-
-struct clk_mgr *dcn1_clk_mgr_create(struct dc_context *ctx);
#endif //__DCN10_CLK_MGR_H__
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c
new file mode 100644
index 000000000000..50bfb5921de0
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c
@@ -0,0 +1,394 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dccg.h"
+#include "clk_mgr_internal.h"
+
+
+#include "dcn20/dcn20_clk_mgr.h"
+#include "dce100/dce_clk_mgr.h"
+#include "reg_helper.h"
+#include "core_types.h"
+#include "dm_helpers.h"
+
+#include "navi10_ip_offset.h"
+#include "dcn/dcn_2_0_0_offset.h"
+#include "dcn/dcn_2_0_0_sh_mask.h"
+#include "clk/clk_11_0_0_offset.h"
+#include "clk/clk_11_0_0_sh_mask.h"
+
+#undef FN
+#define FN(reg_name, field_name) \
+ clk_mgr->clk_mgr_shift->field_name, clk_mgr->clk_mgr_mask->field_name
+
+#define REG(reg) \
+ (clk_mgr->regs->reg)
+
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)\
+ .reg_name = BASE(mm ## reg_name ## _BASE_IDX) + \
+ mm ## reg_name
+
+#define CLK_BASE_INNER(seg) \
+ CLK_BASE__INST0_SEG ## seg
+
+
+static const struct clk_mgr_registers clk_mgr_regs = {
+ CLK_REG_LIST_NV10()
+};
+
+static const struct clk_mgr_shift clk_mgr_shift = {
+ CLK_MASK_SH_LIST_NV10(__SHIFT)
+};
+
+static const struct clk_mgr_mask clk_mgr_mask = {
+ CLK_MASK_SH_LIST_NV10(_MASK)
+};
+
+uint32_t dentist_get_did_from_divider(int divider)
+{
+ uint32_t divider_id;
+
+ /* we want to floor here to get higher clock than required rather than lower */
+ if (divider < DENTIST_DIVIDER_RANGE_2_START) {
+ if (divider < DENTIST_DIVIDER_RANGE_1_START)
+ divider_id = DENTIST_BASE_DID_1;
+ else
+ divider_id = DENTIST_BASE_DID_1
+ + (divider - DENTIST_DIVIDER_RANGE_1_START)
+ / DENTIST_DIVIDER_RANGE_1_STEP;
+ } else if (divider < DENTIST_DIVIDER_RANGE_3_START) {
+ divider_id = DENTIST_BASE_DID_2
+ + (divider - DENTIST_DIVIDER_RANGE_2_START)
+ / DENTIST_DIVIDER_RANGE_2_STEP;
+ } else if (divider < DENTIST_DIVIDER_RANGE_4_START) {
+ divider_id = DENTIST_BASE_DID_3
+ + (divider - DENTIST_DIVIDER_RANGE_3_START)
+ / DENTIST_DIVIDER_RANGE_3_STEP;
+ } else {
+ divider_id = DENTIST_BASE_DID_4
+ + (divider - DENTIST_DIVIDER_RANGE_4_START)
+ / DENTIST_DIVIDER_RANGE_4_STEP;
+ if (divider_id > DENTIST_MAX_DID)
+ divider_id = DENTIST_MAX_DID;
+ }
+
+ return divider_id;
+}
+
+void dcn20_update_clocks_update_dpp_dto(struct clk_mgr_internal *clk_mgr,
+ struct dc_state *context)
+{
+ int i;
+
+ clk_mgr->dccg->ref_dppclk = clk_mgr->base.clks.dppclk_khz;
+ for (i = 0; i < clk_mgr->base.ctx->dc->res_pool->pipe_count; i++) {
+ int dpp_inst, dppclk_khz;
+
+ if (!context->res_ctx.pipe_ctx[i].plane_state)
+ continue;
+
+ dpp_inst = context->res_ctx.pipe_ctx[i].plane_res.dpp->inst;
+ dppclk_khz = context->res_ctx.pipe_ctx[i].plane_res.bw.dppclk_khz;
+ clk_mgr->dccg->funcs->update_dpp_dto(
+ clk_mgr->dccg, dpp_inst, dppclk_khz);
+ }
+}
+
+void dcn20_update_clocks_update_dentist(struct clk_mgr_internal *clk_mgr)
+{
+ int dpp_divider = DENTIST_DIVIDER_RANGE_SCALE_FACTOR
+ * clk_mgr->dentist_vco_freq_khz / clk_mgr->base.clks.dppclk_khz;
+ int disp_divider = DENTIST_DIVIDER_RANGE_SCALE_FACTOR
+ * clk_mgr->dentist_vco_freq_khz / clk_mgr->base.clks.dispclk_khz;
+
+ uint32_t dppclk_wdivider = dentist_get_did_from_divider(dpp_divider);
+ uint32_t dispclk_wdivider = dentist_get_did_from_divider(disp_divider);
+
+ REG_UPDATE(DENTIST_DISPCLK_CNTL,
+ DENTIST_DISPCLK_WDIVIDER, dispclk_wdivider);
+// REG_WAIT(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, 1, 5, 100);
+ REG_UPDATE(DENTIST_DISPCLK_CNTL,
+ DENTIST_DPPCLK_WDIVIDER, dppclk_wdivider);
+ REG_WAIT(DENTIST_DISPCLK_CNTL, DENTIST_DPPCLK_CHG_DONE, 1, 5, 100);
+}
+
+
+void dcn2_update_clocks(struct clk_mgr *clk_mgr_base,
+ struct dc_state *context,
+ bool safe_to_lower)
+{
+ struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+ struct dc_clocks *new_clocks = &context->bw_ctx.bw.dcn.clk;
+ struct dc *dc = clk_mgr_base->ctx->dc;
+ struct pp_smu_funcs_nv *pp_smu = NULL;
+ int display_count;
+ bool update_dppclk = false;
+ bool update_dispclk = false;
+ bool enter_display_off = false;
+ bool dpp_clock_lowered = false;
+ struct dmcu *dmcu = clk_mgr_base->ctx->dc->res_pool->dmcu;
+
+ display_count = clk_mgr_helper_get_active_display_cnt(dc, context);
+ if (dc->res_pool->pp_smu)
+ pp_smu = &dc->res_pool->pp_smu->nv_funcs;
+
+ if (display_count == 0)
+ enter_display_off = true;
+
+ if (enter_display_off == safe_to_lower) {
+ if (pp_smu && pp_smu->set_display_count)
+ pp_smu->set_display_count(&pp_smu->pp_smu, display_count);
+ }
+
+ if (should_set_clock(safe_to_lower, new_clocks->phyclk_khz, clk_mgr_base->clks.phyclk_khz)) {
+ clk_mgr_base->clks.phyclk_khz = new_clocks->phyclk_khz;
+ if (pp_smu && pp_smu->set_voltage_by_freq)
+ pp_smu->set_voltage_by_freq(&pp_smu->pp_smu, PP_SMU_NV_PHYCLK, clk_mgr_base->clks.phyclk_khz / 1000);
+ }
+
+ if (dc->debug.force_min_dcfclk_mhz > 0)
+ new_clocks->dcfclk_khz = (new_clocks->dcfclk_khz > (dc->debug.force_min_dcfclk_mhz * 1000)) ?
+ new_clocks->dcfclk_khz : (dc->debug.force_min_dcfclk_mhz * 1000);
+
+ if (should_set_clock(safe_to_lower, new_clocks->dcfclk_khz, clk_mgr_base->clks.dcfclk_khz)) {
+ clk_mgr_base->clks.dcfclk_khz = new_clocks->dcfclk_khz;
+ if (pp_smu && pp_smu->set_hard_min_dcfclk_by_freq)
+ pp_smu->set_hard_min_dcfclk_by_freq(&pp_smu->pp_smu, clk_mgr_base->clks.dcfclk_khz / 1000);
+ }
+
+ if (should_set_clock(safe_to_lower,
+ new_clocks->dcfclk_deep_sleep_khz, clk_mgr_base->clks.dcfclk_deep_sleep_khz)) {
+ clk_mgr_base->clks.dcfclk_deep_sleep_khz = new_clocks->dcfclk_deep_sleep_khz;
+ if (pp_smu && pp_smu->set_min_deep_sleep_dcfclk)
+ pp_smu->set_min_deep_sleep_dcfclk(&pp_smu->pp_smu, clk_mgr_base->clks.dcfclk_deep_sleep_khz / 1000);
+ }
+
+ if (should_set_clock(safe_to_lower, new_clocks->socclk_khz, clk_mgr_base->clks.socclk_khz)) {
+ clk_mgr_base->clks.socclk_khz = new_clocks->socclk_khz;
+ if (pp_smu && pp_smu->set_hard_min_socclk_by_freq)
+ pp_smu->set_hard_min_socclk_by_freq(&pp_smu->pp_smu, clk_mgr_base->clks.socclk_khz / 1000);
+ }
+
+ if (should_update_pstate_support(safe_to_lower, new_clocks->p_state_change_support, clk_mgr_base->clks.p_state_change_support)) {
+ clk_mgr_base->clks.p_state_change_support = new_clocks->p_state_change_support;
+ if (pp_smu && pp_smu->set_pstate_handshake_support)
+ pp_smu->set_pstate_handshake_support(&pp_smu->pp_smu, clk_mgr_base->clks.p_state_change_support);
+ }
+
+ if (should_set_clock(safe_to_lower, new_clocks->dramclk_khz, clk_mgr_base->clks.dramclk_khz)) {
+ clk_mgr_base->clks.dramclk_khz = new_clocks->dramclk_khz;
+ if (pp_smu && pp_smu->set_hard_min_uclk_by_freq)
+ pp_smu->set_hard_min_uclk_by_freq(&pp_smu->pp_smu, clk_mgr_base->clks.dramclk_khz / 1000);
+ }
+
+ if (should_set_clock(safe_to_lower, new_clocks->dppclk_khz, clk_mgr->base.clks.dppclk_khz)) {
+ if (clk_mgr->base.clks.dppclk_khz > new_clocks->dppclk_khz)
+ dpp_clock_lowered = true;
+ clk_mgr->base.clks.dppclk_khz = new_clocks->dppclk_khz;
+
+ if (pp_smu && pp_smu->set_voltage_by_freq)
+ pp_smu->set_voltage_by_freq(&pp_smu->pp_smu, PP_SMU_NV_PIXELCLK, clk_mgr_base->clks.dppclk_khz / 1000);
+
+ update_dppclk = true;
+ }
+
+ if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz)) {
+ clk_mgr_base->clks.dispclk_khz = new_clocks->dispclk_khz;
+ if (pp_smu && pp_smu->set_voltage_by_freq)
+ pp_smu->set_voltage_by_freq(&pp_smu->pp_smu, PP_SMU_NV_DISPCLK, clk_mgr_base->clks.dispclk_khz / 1000);
+
+ update_dispclk = true;
+ }
+ if (dc->config.forced_clocks == false) {
+ if (dpp_clock_lowered) {
+ // if clock is being lowered, increase DTO before lowering refclk
+ dcn20_update_clocks_update_dpp_dto(clk_mgr, context);
+ dcn20_update_clocks_update_dentist(clk_mgr);
+ } else {
+ // if clock is being raised, increase refclk before lowering DTO
+ if (update_dppclk || update_dispclk)
+ dcn20_update_clocks_update_dentist(clk_mgr);
+ if (update_dppclk)
+ dcn20_update_clocks_update_dpp_dto(clk_mgr, context);
+ }
+ }
+ if (update_dispclk &&
+ dmcu && dmcu->funcs->is_dmcu_initialized(dmcu)) {
+ /*update dmcu for wait_loop count*/
+ dmcu->funcs->set_psr_wait_loop(dmcu,
+ clk_mgr_base->clks.dispclk_khz / 1000 / 7);
+ }
+}
+
+void dcn2_update_clocks_fpga(struct clk_mgr *clk_mgr,
+ struct dc_state *context,
+ bool safe_to_lower)
+{
+ struct dc_clocks *new_clocks = &context->bw_ctx.bw.dcn.clk;
+ /* Min fclk = 1.2GHz since all the extra scemi logic seems to run off of it */
+ int fclk_adj = new_clocks->fclk_khz > 1200000 ? new_clocks->fclk_khz : 1200000;
+
+ if (should_set_clock(safe_to_lower, new_clocks->phyclk_khz, clk_mgr->clks.phyclk_khz)) {
+ clk_mgr->clks.phyclk_khz = new_clocks->phyclk_khz;
+ }
+
+ if (should_set_clock(safe_to_lower, new_clocks->dcfclk_khz, clk_mgr->clks.dcfclk_khz)) {
+ clk_mgr->clks.dcfclk_khz = new_clocks->dcfclk_khz;
+ }
+
+ if (should_set_clock(safe_to_lower,
+ new_clocks->dcfclk_deep_sleep_khz, clk_mgr->clks.dcfclk_deep_sleep_khz)) {
+ clk_mgr->clks.dcfclk_deep_sleep_khz = new_clocks->dcfclk_deep_sleep_khz;
+ }
+
+ if (should_set_clock(safe_to_lower, new_clocks->socclk_khz, clk_mgr->clks.socclk_khz)) {
+ clk_mgr->clks.socclk_khz = new_clocks->socclk_khz;
+ }
+
+ if (should_set_clock(safe_to_lower, new_clocks->dramclk_khz, clk_mgr->clks.dramclk_khz)) {
+ clk_mgr->clks.dramclk_khz = new_clocks->dramclk_khz;
+ }
+
+ if (should_set_clock(safe_to_lower, new_clocks->dppclk_khz, clk_mgr->clks.dppclk_khz)) {
+ clk_mgr->clks.dppclk_khz = new_clocks->dppclk_khz;
+ }
+
+ if (should_set_clock(safe_to_lower, fclk_adj, clk_mgr->clks.fclk_khz)) {
+ clk_mgr->clks.fclk_khz = fclk_adj;
+ }
+
+ if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr->clks.dispclk_khz)) {
+ clk_mgr->clks.dispclk_khz = new_clocks->dispclk_khz;
+ }
+
+ /* Both fclk and dppclk ref are run on the same scemi clock so we
+ * need to keep the same value for both
+ */
+ if (clk_mgr->clks.fclk_khz > clk_mgr->clks.dppclk_khz)
+ clk_mgr->clks.dppclk_khz = clk_mgr->clks.fclk_khz;
+ if (clk_mgr->clks.dppclk_khz > clk_mgr->clks.fclk_khz)
+ clk_mgr->clks.fclk_khz = clk_mgr->clks.dppclk_khz;
+
+ dm_set_dcn_clocks(clk_mgr->ctx, &clk_mgr->clks);
+}
+
+void dcn2_init_clocks(struct clk_mgr *clk_mgr)
+{
+ memset(&(clk_mgr->clks), 0, sizeof(struct dc_clocks));
+ // Assumption is that boot state always supports pstate
+ clk_mgr->clks.p_state_change_support = true;
+}
+
+void dcn2_enable_pme_wa(struct clk_mgr *clk_mgr_base)
+{
+ struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+ struct pp_smu_funcs_nv *pp_smu = NULL;
+
+ if (clk_mgr->pp_smu) {
+ pp_smu = &clk_mgr->pp_smu->nv_funcs;
+
+ if (pp_smu->set_pme_wa_enable)
+ pp_smu->set_pme_wa_enable(&pp_smu->pp_smu);
+ }
+}
+
+static struct clk_mgr_funcs dcn2_funcs = {
+ .get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+ .update_clocks = dcn2_update_clocks,
+ .init_clocks = dcn2_init_clocks,
+ .enable_pme_wa = dcn2_enable_pme_wa
+};
+
+
+void dcn20_clk_mgr_construct(
+ struct dc_context *ctx,
+ struct clk_mgr_internal *clk_mgr,
+ struct pp_smu_funcs *pp_smu,
+ struct dccg *dccg)
+{
+ clk_mgr->base.ctx = ctx;
+ clk_mgr->pp_smu = pp_smu;
+ clk_mgr->base.funcs = &dcn2_funcs;
+ clk_mgr->regs = &clk_mgr_regs;
+ clk_mgr->clk_mgr_shift = &clk_mgr_shift;
+ clk_mgr->clk_mgr_mask = &clk_mgr_mask;
+
+ clk_mgr->dccg = dccg;
+ clk_mgr->dfs_bypass_disp_clk = 0;
+
+ clk_mgr->dprefclk_ss_percentage = 0;
+ clk_mgr->dprefclk_ss_divider = 1000;
+ clk_mgr->ss_on_dprefclk = false;
+
+ clk_mgr->base.dprefclk_khz = 700000; // 700 MHz planned if VCO is 3.85 GHz, will be retrieved
+
+ if (IS_FPGA_MAXIMUS_DC(ctx->dce_environment)) {
+ dcn2_funcs.update_clocks = dcn2_update_clocks_fpga;
+ clk_mgr->dentist_vco_freq_khz = 3850000;
+
+ } else {
+ /* DFS Slice 2 should be used for DPREFCLK */
+ int dprefclk_did = REG_READ(CLK3_CLK2_DFS_CNTL);
+ /* Convert DPREFCLK DFS Slice DID to actual divider*/
+ int target_div = dentist_get_divider_from_did(dprefclk_did);
+
+ /* get FbMult value */
+ uint32_t pll_req_reg = REG_READ(CLK3_CLK_PLL_REQ);
+ struct fixed31_32 pll_req;
+
+ /* set up a fixed-point number
+ * this works because the int part is on the right edge of the register
+ * and the frac part is on the left edge
+ */
+
+ pll_req = dc_fixpt_from_int(pll_req_reg & clk_mgr->clk_mgr_mask->FbMult_int);
+ pll_req.value |= pll_req_reg & clk_mgr->clk_mgr_mask->FbMult_frac;
+
+ /* multiply by REFCLK period */
+ pll_req = dc_fixpt_mul_int(pll_req, 100000);
+
+ /* integer part is now VCO frequency in kHz */
+ clk_mgr->dentist_vco_freq_khz = dc_fixpt_floor(pll_req);
+
+ /* in case we don't get a value from the register, use default */
+ if (clk_mgr->dentist_vco_freq_khz == 0)
+ clk_mgr->dentist_vco_freq_khz = 3850000;
+
+ /* Calculate the DPREFCLK in kHz.*/
+ clk_mgr->base.dprefclk_khz = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR
+ * clk_mgr->dentist_vco_freq_khz) / target_div;
+ }
+ //Integrated_info table does not exist on dGPU projects so should not be referenced
+ //anywhere in code for dGPUs.
+ //Also there is no plan for now that DFS BYPASS will be used on NV10/12/14.
+ clk_mgr->dfs_bypass_enabled = false;
+
+ dce_clock_read_ss_info(clk_mgr);
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.h
new file mode 100644
index 000000000000..5661a5a89847
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DCN20_CLK_MGR_H__
+#define __DCN20_CLK_MGR_H__
+
+void dcn2_update_clocks(struct clk_mgr *dccg,
+ struct dc_state *context,
+ bool safe_to_lower);
+
+void dcn2_update_clocks_fpga(struct clk_mgr *clk_mgr,
+ struct dc_state *context,
+ bool safe_to_lower);
+void dcn20_update_clocks_update_dpp_dto(struct clk_mgr_internal *clk_mgr,
+ struct dc_state *context);
+
+void dcn2_init_clocks(struct clk_mgr *clk_mgr);
+
+void dcn20_clk_mgr_construct(struct dc_context *ctx,
+ struct clk_mgr_internal *clk_mgr,
+ struct pp_smu_funcs *pp_smu,
+ struct dccg *dccg);
+
+uint32_t dentist_get_did_from_divider(int divider);
+
+#endif //__DCN20_CLK_MGR_H__
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c
index 18c775a950cc..cbc480a33376 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc.c
@@ -22,6 +22,9 @@
* Authors: AMD
*/
+#include <linux/slab.h>
+#include <linux/mm.h>
+
#include "dm_services.h"
#include "dc.h"
@@ -33,6 +36,7 @@
#include "resource.h"
+#include "clk_mgr.h"
#include "clock_source.h"
#include "dc_bios_types.h"
@@ -55,6 +59,14 @@
#include "dc_link_dp.h"
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+#include "dsc.h"
+#endif
+
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+#include "vm_helper.h"
+#endif
+
#include "dce/dce_i2c.h"
#define DC_LOGGER \
@@ -169,9 +181,14 @@ static bool create_links(
link = link_create(&link_init_params);
if (link) {
- dc->links[dc->link_count] = link;
- link->dc = dc;
- ++dc->link_count;
+ if (dc->config.edp_not_connected &&
+ link->connector_signal == SIGNAL_TYPE_EDP) {
+ link_destroy(&link);
+ } else {
+ dc->links[dc->link_count] = link;
+ link->dc = dc;
+ ++dc->link_count;
+ }
}
}
@@ -257,7 +274,7 @@ bool dc_stream_adjust_vmin_vmax(struct dc *dc,
for (i = 0; i < MAX_PIPES; i++) {
struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
- if (pipe->stream == stream && pipe->stream_res.stream_enc) {
+ if (pipe->stream == stream && pipe->stream_res.tg) {
pipe->stream->adjust = *adjust;
dc->hwss.set_drr(&pipe,
1,
@@ -451,7 +468,7 @@ bool dc_stream_program_csc_matrix(struct dc *dc, struct dc_stream_state *stream)
pipes,
stream->output_color_space,
stream->csc_color_matrix.matrix,
- pipes->plane_res.hubp ? pipes->plane_res.hubp->opp_id : 0);
+ pipes->stream_res.opp->inst);
ret = true;
}
}
@@ -484,135 +501,20 @@ void dc_stream_set_static_screen_events(struct dc *dc,
dc->hwss.set_static_screen_control(pipes_affected, num_pipes_affected, events);
}
-void dc_link_set_drive_settings(struct dc *dc,
- struct link_training_settings *lt_settings,
- const struct dc_link *link)
+static void destruct(struct dc *dc)
{
-
- int i;
-
- for (i = 0; i < dc->link_count; i++) {
- if (dc->links[i] == link)
- break;
+ if (dc->current_state) {
+ dc_release_state(dc->current_state);
+ dc->current_state = NULL;
}
- if (i >= dc->link_count)
- ASSERT_CRITICAL(false);
-
- dc_link_dp_set_drive_settings(dc->links[i], lt_settings);
-}
-
-void dc_link_perform_link_training(struct dc *dc,
- struct dc_link_settings *link_setting,
- bool skip_video_pattern)
-{
- int i;
-
- for (i = 0; i < dc->link_count; i++)
- dc_link_dp_perform_link_training(
- dc->links[i],
- link_setting,
- skip_video_pattern);
-}
-
-void dc_link_set_preferred_link_settings(struct dc *dc,
- struct dc_link_settings *link_setting,
- struct dc_link *link)
-{
- int i;
- struct pipe_ctx *pipe;
- struct dc_stream_state *link_stream;
- struct dc_link_settings store_settings = *link_setting;
-
- link->preferred_link_setting = store_settings;
-
- /* Retrain with preferred link settings only relevant for
- * DP signal type
- */
- if (!dc_is_dp_signal(link->connector_signal))
- return;
+ destroy_links(dc);
- for (i = 0; i < MAX_PIPES; i++) {
- pipe = &dc->current_state->res_ctx.pipe_ctx[i];
- if (pipe->stream && pipe->stream->link) {
- if (pipe->stream->link == link)
- break;
- }
+ if (dc->clk_mgr) {
+ dc_destroy_clk_mgr(dc->clk_mgr);
+ dc->clk_mgr = NULL;
}
- /* Stream not found */
- if (i == MAX_PIPES)
- return;
-
- link_stream = link->dc->current_state->res_ctx.pipe_ctx[i].stream;
-
- /* Cannot retrain link if backend is off */
- if (link_stream->dpms_off)
- return;
-
- if (link_stream)
- decide_link_settings(link_stream, &store_settings);
-
- if ((store_settings.lane_count != LANE_COUNT_UNKNOWN) &&
- (store_settings.link_rate != LINK_RATE_UNKNOWN))
- dp_retrain_link_dp_test(link, &store_settings, false);
-}
-
-void dc_link_enable_hpd(const struct dc_link *link)
-{
- dc_link_dp_enable_hpd(link);
-}
-
-void dc_link_disable_hpd(const struct dc_link *link)
-{
- dc_link_dp_disable_hpd(link);
-}
-
-
-void dc_link_set_test_pattern(struct dc_link *link,
- enum dp_test_pattern test_pattern,
- const struct link_training_settings *p_link_settings,
- const unsigned char *p_custom_pattern,
- unsigned int cust_pattern_size)
-{
- if (link != NULL)
- dc_link_dp_set_test_pattern(
- link,
- test_pattern,
- p_link_settings,
- p_custom_pattern,
- cust_pattern_size);
-}
-
-uint32_t dc_link_bandwidth_kbps(
- const struct dc_link *link,
- const struct dc_link_settings *link_setting)
-{
- uint32_t link_bw_kbps = link_setting->link_rate * LINK_RATE_REF_FREQ_IN_KHZ; /* bytes per sec */
-
- link_bw_kbps *= 8; /* 8 bits per byte*/
- link_bw_kbps *= link_setting->lane_count;
-
- return link_bw_kbps;
-
-}
-
-const struct dc_link_settings *dc_link_get_link_cap(
- const struct dc_link *link)
-{
- if (link->preferred_link_setting.lane_count != LANE_COUNT_UNKNOWN &&
- link->preferred_link_setting.link_rate != LINK_RATE_UNKNOWN)
- return &link->preferred_link_setting;
- return &link->verified_link_cap;
-}
-
-static void destruct(struct dc *dc)
-{
- dc_release_state(dc->current_state);
- dc->current_state = NULL;
-
- destroy_links(dc);
-
dc_destroy_resource_pool(dc);
if (dc->ctx->gpio_service)
@@ -640,6 +542,11 @@ static void destruct(struct dc *dc)
dc->dcn_ip = NULL;
#endif
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ kfree(dc->vm_helper);
+ dc->vm_helper = NULL;
+
+#endif
}
static bool construct(struct dc *dc,
@@ -656,6 +563,11 @@ static bool construct(struct dc *dc,
enum dce_version dc_version = DCE_VERSION_UNKNOWN;
dc->config = init_params->flags;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ // Allocate memory for the vm_helper
+ dc->vm_helper = kzalloc(sizeof(struct vm_helper), GFP_KERNEL);
+
+#endif
memcpy(&dc->bb_overrides, &init_params->bb_overrides, sizeof(dc->bb_overrides));
dc_dceip = kzalloc(sizeof(*dc_dceip), GFP_KERNEL);
@@ -689,6 +601,9 @@ static bool construct(struct dc *dc,
}
dc->dcn_ip = dcn_ip;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ dc->soc_bounding_box = init_params->soc_bounding_box;
+#endif
#endif
dc_ctx = kzalloc(sizeof(*dc_ctx), GFP_KERNEL);
@@ -756,6 +671,10 @@ static bool construct(struct dc *dc,
if (!dc->res_pool)
goto fail;
+ dc->clk_mgr = dc_clk_mgr_create(dc->ctx, dc->res_pool->pp_smu, dc->res_pool->dccg);
+ if (!dc->clk_mgr)
+ goto fail;
+
/* Creation of current_state must occur after dc->dml
* is initialized in dc_create_resource_pool because
* on creation it copies the contents of dc->dml
@@ -781,6 +700,21 @@ fail:
return false;
}
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+static bool disable_all_writeback_pipes_for_stream(
+ const struct dc *dc,
+ struct dc_stream_state *stream,
+ struct dc_state *context)
+{
+ int i;
+
+ for (i = 0; i < stream->num_wb_info; i++)
+ stream->writeback_info[i].wb_enabled = false;
+
+ return true;
+}
+#endif
+
static void disable_dangling_plane(struct dc *dc, struct dc_state *context)
{
int i, j;
@@ -805,6 +739,9 @@ static void disable_dangling_plane(struct dc *dc, struct dc_state *context)
}
if (should_disable && old_stream) {
dc_rem_all_planes_for_stream(dc, old_stream, dangling_context);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ disable_all_writeback_pipes_for_stream(dc, old_stream, dangling_context);
+#endif
dc->hwss.apply_ctx_for_surface(dc, old_stream, 0, dangling_context);
}
}
@@ -1136,10 +1073,6 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c
/* Program all planes within new context*/
for (i = 0; i < context->stream_count; i++) {
const struct dc_link *link = context->streams[i]->link;
- struct dc_stream_status *status;
-
- if (context->streams[i]->apply_seamless_boot_optimization)
- context->streams[i]->apply_seamless_boot_optimization = false;
if (!context->streams[i]->mode_changed)
continue;
@@ -1164,9 +1097,6 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c
}
}
- status = dc_stream_get_status_from_state(context, context->streams[i]);
- context->streams[i]->out.otg_offset = status->primary_otg_inst;
-
CONN_MSG_MODE(link, "{%dx%d, %dx%d@%dKhz}",
context->streams[i]->timing.h_addressable,
context->streams[i]->timing.v_addressable,
@@ -1242,8 +1172,8 @@ bool dc_post_update_surfaces_to_stream(struct dc *dc)
struct dc_state *dc_create_state(struct dc *dc)
{
- struct dc_state *context = kzalloc(sizeof(struct dc_state),
- GFP_KERNEL);
+ struct dc_state *context = kvzalloc(sizeof(struct dc_state),
+ GFP_KERNEL);
if (!context)
return NULL;
@@ -1263,12 +1193,10 @@ struct dc_state *dc_create_state(struct dc *dc)
struct dc_state *dc_copy_state(struct dc_state *src_ctx)
{
int i, j;
- struct dc_state *new_ctx = kzalloc(sizeof(struct dc_state),
- GFP_KERNEL);
+ struct dc_state *new_ctx = kvmalloc(sizeof(struct dc_state), GFP_KERNEL);
if (!new_ctx)
return NULL;
-
memcpy(new_ctx, src_ctx, sizeof(struct dc_state));
for (i = 0; i < MAX_PIPES; i++) {
@@ -1303,7 +1231,7 @@ static void dc_state_free(struct kref *kref)
{
struct dc_state *context = container_of(kref, struct dc_state, refcount);
dc_resource_state_destruct(context);
- kfree(context);
+ kvfree(context);
}
void dc_release_state(struct dc_state *context)
@@ -1331,71 +1259,94 @@ static bool is_surface_in_context(
static enum surface_update_type get_plane_info_update_type(const struct dc_surface_update *u)
{
union surface_update_flags *update_flags = &u->surface->update_flags;
+ enum surface_update_type update_type = UPDATE_TYPE_FAST;
if (!u->plane_info)
return UPDATE_TYPE_FAST;
- if (u->plane_info->color_space != u->surface->color_space)
+ if (u->plane_info->color_space != u->surface->color_space) {
update_flags->bits.color_space_change = 1;
+ elevate_update_type(&update_type, UPDATE_TYPE_MED);
+ }
- if (u->plane_info->horizontal_mirror != u->surface->horizontal_mirror)
+ if (u->plane_info->horizontal_mirror != u->surface->horizontal_mirror) {
update_flags->bits.horizontal_mirror_change = 1;
+ elevate_update_type(&update_type, UPDATE_TYPE_MED);
+ }
- if (u->plane_info->rotation != u->surface->rotation)
+ if (u->plane_info->rotation != u->surface->rotation) {
update_flags->bits.rotation_change = 1;
+ elevate_update_type(&update_type, UPDATE_TYPE_FULL);
+ }
- if (u->plane_info->format != u->surface->format)
+ if (u->plane_info->format != u->surface->format) {
update_flags->bits.pixel_format_change = 1;
+ elevate_update_type(&update_type, UPDATE_TYPE_FULL);
+ }
- if (u->plane_info->stereo_format != u->surface->stereo_format)
+ if (u->plane_info->stereo_format != u->surface->stereo_format) {
update_flags->bits.stereo_format_change = 1;
+ elevate_update_type(&update_type, UPDATE_TYPE_FULL);
+ }
- if (u->plane_info->per_pixel_alpha != u->surface->per_pixel_alpha)
+ if (u->plane_info->per_pixel_alpha != u->surface->per_pixel_alpha) {
update_flags->bits.per_pixel_alpha_change = 1;
+ elevate_update_type(&update_type, UPDATE_TYPE_MED);
+ }
- if (u->plane_info->global_alpha_value != u->surface->global_alpha_value)
+ if (u->plane_info->global_alpha_value != u->surface->global_alpha_value) {
update_flags->bits.global_alpha_change = 1;
+ elevate_update_type(&update_type, UPDATE_TYPE_MED);
+ }
+
+ if (u->plane_info->sdr_white_level != u->surface->sdr_white_level) {
+ update_flags->bits.sdr_white_level = 1;
+ elevate_update_type(&update_type, UPDATE_TYPE_MED);
+ }
if (u->plane_info->dcc.enable != u->surface->dcc.enable
|| u->plane_info->dcc.grph.independent_64b_blks != u->surface->dcc.grph.independent_64b_blks
- || u->plane_info->dcc.grph.meta_pitch != u->surface->dcc.grph.meta_pitch)
+ || u->plane_info->dcc.grph.meta_pitch != u->surface->dcc.grph.meta_pitch) {
update_flags->bits.dcc_change = 1;
+ elevate_update_type(&update_type, UPDATE_TYPE_MED);
+ }
if (resource_pixel_format_to_bpp(u->plane_info->format) !=
- resource_pixel_format_to_bpp(u->surface->format))
+ resource_pixel_format_to_bpp(u->surface->format)) {
/* different bytes per element will require full bandwidth
* and DML calculation
*/
update_flags->bits.bpp_change = 1;
+ elevate_update_type(&update_type, UPDATE_TYPE_FULL);
+ }
if (u->plane_info->plane_size.grph.surface_pitch != u->surface->plane_size.grph.surface_pitch
|| u->plane_info->plane_size.video.luma_pitch != u->surface->plane_size.video.luma_pitch
- || u->plane_info->plane_size.video.chroma_pitch != u->surface->plane_size.video.chroma_pitch)
+ || u->plane_info->plane_size.video.chroma_pitch != u->surface->plane_size.video.chroma_pitch) {
update_flags->bits.plane_size_change = 1;
+ elevate_update_type(&update_type, UPDATE_TYPE_MED);
+ }
if (memcmp(&u->plane_info->tiling_info, &u->surface->tiling_info,
sizeof(union dc_tiling_info)) != 0) {
update_flags->bits.swizzle_change = 1;
+ elevate_update_type(&update_type, UPDATE_TYPE_MED);
+
/* todo: below are HW dependent, we should add a hook to
* DCE/N resource and validated there.
*/
- if (u->plane_info->tiling_info.gfx9.swizzle != DC_SW_LINEAR)
+ if (u->plane_info->tiling_info.gfx9.swizzle != DC_SW_LINEAR) {
/* swizzled mode requires RQ to be setup properly,
* thus need to run DML to calculate RQ settings
*/
update_flags->bits.bandwidth_change = 1;
+ elevate_update_type(&update_type, UPDATE_TYPE_FULL);
+ }
}
- if (update_flags->bits.rotation_change
- || update_flags->bits.stereo_format_change
- || update_flags->bits.pixel_format_change
- || update_flags->bits.bpp_change
- || update_flags->bits.bandwidth_change
- || update_flags->bits.output_tf_change)
- return UPDATE_TYPE_FULL;
-
- return update_flags->raw ? UPDATE_TYPE_MED : UPDATE_TYPE_FAST;
+ /* This should be UPDATE_TYPE_FAST if nothing has changed. */
+ return update_type;
}
static enum surface_update_type get_scaling_info_update_type(
@@ -1459,6 +1410,9 @@ static enum surface_update_type det_surface_update(const struct dc *dc,
update_flags->raw = 0; // Reset all flags
+ if (u->flip_addr)
+ update_flags->bits.addr_update = 1;
+
if (!is_surface_in_context(context, u->surface)) {
update_flags->bits.new_plane = 1;
return UPDATE_TYPE_FULL;
@@ -1475,6 +1429,9 @@ static enum surface_update_type det_surface_update(const struct dc *dc,
type = get_scaling_info_update_type(u);
elevate_update_type(&overall_type, type);
+ if (u->flip_addr)
+ update_flags->bits.addr_update = 1;
+
if (u->in_transfer_func)
update_flags->bits.in_transfer_func_change = 1;
@@ -1542,6 +1499,11 @@ static enum surface_update_type check_update_surfaces_for_stream(
if (stream_update->dpms_off)
return UPDATE_TYPE_FULL;
+
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ if (stream_update->wb_update)
+ return UPDATE_TYPE_FULL;
+#endif
}
for (i = 0 ; i < surface_count; i++) {
@@ -1686,6 +1648,26 @@ static void copy_surface_update_to_plane(
sizeof(struct dc_transfer_func_distributed_points));
}
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ if (srf_update->func_shaper &&
+ (surface->in_shaper_func !=
+ srf_update->func_shaper))
+ memcpy(surface->in_shaper_func, srf_update->func_shaper,
+ sizeof(*surface->in_shaper_func));
+
+ if (srf_update->lut3d_func &&
+ (surface->lut3d_func !=
+ srf_update->lut3d_func))
+ memcpy(surface->lut3d_func, srf_update->lut3d_func,
+ sizeof(*surface->lut3d_func));
+
+ if (srf_update->blend_tf &&
+ (surface->blend_tf !=
+ srf_update->blend_tf))
+ memcpy(surface->blend_tf, srf_update->blend_tf,
+ sizeof(*surface->blend_tf));
+
+#endif
if (srf_update->input_csc_color_matrix)
surface->input_csc_color_matrix =
*srf_update->input_csc_color_matrix;
@@ -1695,6 +1677,101 @@ static void copy_surface_update_to_plane(
*srf_update->coeff_reduction_factor;
}
+static void copy_stream_update_to_stream(struct dc *dc,
+ struct dc_state *context,
+ struct dc_stream_state *stream,
+ const struct dc_stream_update *update)
+{
+ if (update == NULL || stream == NULL)
+ return;
+
+ if (update->src.height && update->src.width)
+ stream->src = update->src;
+
+ if (update->dst.height && update->dst.width)
+ stream->dst = update->dst;
+
+ if (update->out_transfer_func &&
+ stream->out_transfer_func != update->out_transfer_func) {
+ stream->out_transfer_func->sdr_ref_white_level =
+ update->out_transfer_func->sdr_ref_white_level;
+ stream->out_transfer_func->tf = update->out_transfer_func->tf;
+ stream->out_transfer_func->type =
+ update->out_transfer_func->type;
+ memcpy(&stream->out_transfer_func->tf_pts,
+ &update->out_transfer_func->tf_pts,
+ sizeof(struct dc_transfer_func_distributed_points));
+ }
+
+ if (update->hdr_static_metadata)
+ stream->hdr_static_metadata = *update->hdr_static_metadata;
+
+ if (update->abm_level)
+ stream->abm_level = *update->abm_level;
+
+ if (update->periodic_interrupt0)
+ stream->periodic_interrupt0 = *update->periodic_interrupt0;
+
+ if (update->periodic_interrupt1)
+ stream->periodic_interrupt1 = *update->periodic_interrupt1;
+
+ if (update->gamut_remap)
+ stream->gamut_remap_matrix = *update->gamut_remap;
+
+ /* Note: this being updated after mode set is currently not a use case
+ * however if it arises OCSC would need to be reprogrammed at the
+ * minimum
+ */
+ if (update->output_color_space)
+ stream->output_color_space = *update->output_color_space;
+
+ if (update->output_csc_transform)
+ stream->csc_color_matrix = *update->output_csc_transform;
+
+ if (update->vrr_infopacket)
+ stream->vrr_infopacket = *update->vrr_infopacket;
+
+ if (update->dpms_off)
+ stream->dpms_off = *update->dpms_off;
+
+ if (update->vsc_infopacket)
+ stream->vsc_infopacket = *update->vsc_infopacket;
+
+ if (update->vsp_infopacket)
+ stream->vsp_infopacket = *update->vsp_infopacket;
+
+ if (update->dither_option)
+ stream->dither_option = *update->dither_option;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ /* update current stream with writeback info */
+ if (update->wb_update) {
+ int i;
+
+ stream->num_wb_info = update->wb_update->num_wb_info;
+ ASSERT(stream->num_wb_info <= MAX_DWB_PIPES);
+ for (i = 0; i < stream->num_wb_info; i++)
+ stream->writeback_info[i] =
+ update->wb_update->writeback_info[i];
+ }
+#endif
+#if defined(CONFIG_DRM_AMD_DC_DSC_SUPPORT)
+ if (update->dsc_config) {
+ struct dc_dsc_config old_dsc_cfg = stream->timing.dsc_cfg;
+ uint32_t old_dsc_enabled = stream->timing.flags.DSC;
+ uint32_t enable_dsc = (update->dsc_config->num_slices_h != 0 &&
+ update->dsc_config->num_slices_v != 0);
+
+ stream->timing.dsc_cfg = *update->dsc_config;
+ stream->timing.flags.DSC = enable_dsc;
+ if (!dc->res_pool->funcs->validate_bandwidth(dc, context,
+ true)) {
+ stream->timing.dsc_cfg = old_dsc_cfg;
+ stream->timing.flags.DSC = old_dsc_enabled;
+ }
+ }
+#endif
+}
+
static void commit_planes_do_stream_update(struct dc *dc,
struct dc_stream_state *stream,
struct dc_stream_update *stream_update,
@@ -1711,13 +1788,6 @@ static void commit_planes_do_stream_update(struct dc *dc,
pipe_ctx->stream &&
pipe_ctx->stream == stream) {
- /* Fast update*/
- // VRR program can be done as part of FAST UPDATE
- if (stream_update->adjust)
- dc->hwss.set_drr(&pipe_ctx, 1,
- stream_update->adjust->v_total_min,
- stream_update->adjust->v_total_max);
-
if (stream_update->periodic_interrupt0 &&
dc->hwss.setup_periodic_interrupt)
dc->hwss.setup_periodic_interrupt(pipe_ctx, VLINE0);
@@ -1741,13 +1811,29 @@ static void commit_planes_do_stream_update(struct dc *dc,
dc_stream_program_csc_matrix(dc, stream);
if (stream_update->dither_option) {
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ struct pipe_ctx *odm_pipe = dc_res_get_odm_bottom_pipe(pipe_ctx);
+#endif
resource_build_bit_depth_reduction_params(pipe_ctx->stream,
&pipe_ctx->stream->bit_depth_params);
pipe_ctx->stream_res.opp->funcs->opp_program_fmt(pipe_ctx->stream_res.opp,
&stream->bit_depth_params,
&stream->clamping);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ if (odm_pipe)
+ odm_pipe->stream_res.opp->funcs->opp_program_fmt(odm_pipe->stream_res.opp,
+ &stream->bit_depth_params,
+ &stream->clamping);
+#endif
}
+#if defined(CONFIG_DRM_AMD_DC_DSC_SUPPORT)
+ if (stream_update->dsc_config && dc->hwss.pipe_control_lock_global) {
+ dc->hwss.pipe_control_lock_global(dc, pipe_ctx, true);
+ dp_update_dsc_config(pipe_ctx);
+ dc->hwss.pipe_control_lock_global(dc, pipe_ctx, false);
+ }
+#endif
/* Full fe update*/
if (update_type == UPDATE_TYPE_FAST)
continue;
@@ -1792,10 +1878,15 @@ static void commit_planes_for_stream(struct dc *dc,
if (dc->optimize_seamless_boot && surface_count > 0) {
/* Optimize seamless boot flag keeps clocks and watermarks high until
* first flip. After first flip, optimization is required to lower
- * bandwidth.
+ * bandwidth. Important to note that it is expected UEFI will
+ * only light up a single display on POST, therefore we only expect
+ * one stream with seamless boot flag set.
*/
- dc->optimize_seamless_boot = false;
- dc->optimized_required = true;
+ if (stream->apply_seamless_boot_optimization) {
+ stream->apply_seamless_boot_optimization = false;
+ dc->optimize_seamless_boot = false;
+ dc->optimized_required = true;
+ }
}
if (update_type == UPDATE_TYPE_FULL && !dc->optimize_seamless_boot) {
@@ -1816,6 +1907,30 @@ static void commit_planes_for_stream(struct dc *dc,
return;
}
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ if (!IS_DIAG_DC(dc->ctx->dce_environment)) {
+ for (i = 0; i < surface_count; i++) {
+ struct dc_plane_state *plane_state = srf_updates[i].surface;
+ /*set logical flag for lock/unlock use*/
+ for (j = 0; j < dc->res_pool->pipe_count; j++) {
+ struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+ if (!pipe_ctx->plane_state)
+ continue;
+ if (pipe_ctx->plane_state != plane_state)
+ continue;
+ plane_state->triplebuffer_flips = false;
+ if (update_type == UPDATE_TYPE_FAST &&
+ dc->hwss.program_triplebuffer != NULL &&
+ !plane_state->flip_immediate &&
+ !dc->debug.disable_tri_buf) {
+ /*triple buffer for VUpdate only*/
+ plane_state->triplebuffer_flips = true;
+ }
+ }
+ }
+ }
+#endif
+
// Update Type FULL, Surface updates
for (j = 0; j < dc->res_pool->pipe_count; j++) {
struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
@@ -1834,6 +1949,16 @@ static void commit_planes_for_stream(struct dc *dc,
if (update_type == UPDATE_TYPE_FAST)
continue;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ ASSERT(!pipe_ctx->plane_state->triplebuffer_flips);
+
+ if (dc->hwss.program_triplebuffer != NULL &&
+ !dc->debug.disable_tri_buf) {
+ /*turn off triple buffer for full update*/
+ dc->hwss.program_triplebuffer(
+ dc, pipe_ctx, pipe_ctx->plane_state->triplebuffer_flips);
+ }
+#endif
stream_status =
stream_get_status(context, pipe_ctx->stream);
@@ -1850,6 +1975,26 @@ static void commit_planes_for_stream(struct dc *dc,
*/
dc->hwss.pipe_control_lock(dc, top_pipe_to_program, true);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ if (dc->hwss.set_flip_control_gsl)
+ for (i = 0; i < surface_count; i++) {
+ struct dc_plane_state *plane_state = srf_updates[i].surface;
+
+ for (j = 0; j < dc->res_pool->pipe_count; j++) {
+ struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+ if (pipe_ctx->stream != stream)
+ continue;
+
+ if (pipe_ctx->plane_state != plane_state)
+ continue;
+
+ // GSL has to be used for flip immediate
+ dc->hwss.set_flip_control_gsl(pipe_ctx,
+ plane_state->flip_immediate);
+ }
+ }
+#endif
/* Perform requested Updates */
for (i = 0; i < surface_count; i++) {
struct dc_plane_state *plane_state = srf_updates[i].surface;
@@ -1862,7 +2007,15 @@ static void commit_planes_for_stream(struct dc *dc,
if (pipe_ctx->plane_state != plane_state)
continue;
-
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ /*program triple buffer after lock based on flip type*/
+ if (dc->hwss.program_triplebuffer != NULL &&
+ !dc->debug.disable_tri_buf) {
+ /*only enable triplebuffer for fast_update*/
+ dc->hwss.program_triplebuffer(
+ dc, pipe_ctx, plane_state->triplebuffer_flips);
+ }
+#endif
if (srf_updates[i].flip_addr)
dc->hwss.update_plane_addr(dc, pipe_ctx);
}
@@ -1870,6 +2023,20 @@ static void commit_planes_for_stream(struct dc *dc,
dc->hwss.pipe_control_lock(dc, top_pipe_to_program, false);
}
+
+ // Fire manual trigger only when bottom plane is flipped
+ for (j = 0; j < dc->res_pool->pipe_count; j++) {
+ struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+ if (pipe_ctx->bottom_pipe ||
+ !pipe_ctx->stream ||
+ pipe_ctx->stream != stream ||
+ !pipe_ctx->plane_state->update_flags.bits.addr_update)
+ continue;
+
+ if (pipe_ctx->stream_res.tg->funcs->program_manual_trigger)
+ pipe_ctx->stream_res.tg->funcs->program_manual_trigger(pipe_ctx->stream_res.tg);
+ }
}
void dc_commit_updates_for_stream(struct dc *dc,
@@ -1933,6 +2100,8 @@ void dc_commit_updates_for_stream(struct dc *dc,
}
}
+ copy_stream_update_to_stream(dc, context, stream, stream_update);
+
commit_planes_for_stream(
dc,
srf_updates,
@@ -2006,6 +2175,12 @@ void dc_set_power_state(
enum dc_acpi_cm_power_state power_state)
{
struct kref refcount;
+ struct display_mode_lib *dml = kzalloc(sizeof(struct display_mode_lib),
+ GFP_KERNEL);
+
+ ASSERT(dml);
+ if (!dml)
+ return;
switch (power_state) {
case DC_ACPI_CM_POWER_STATE_D0:
@@ -2022,15 +2197,20 @@ void dc_set_power_state(
/* Preserve refcount */
refcount = dc->current_state->refcount;
+ /* Preserve display mode lib */
+ memcpy(dml, &dc->current_state->bw_ctx.dml, sizeof(struct display_mode_lib));
+
dc_resource_state_destruct(dc->current_state);
memset(dc->current_state, 0,
sizeof(*dc->current_state));
dc->current_state->refcount = refcount;
+ dc->current_state->bw_ctx.dml = *dml;
break;
}
+ kfree(dml);
}
void dc_resume(struct dc *dc)
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
index 83d121510ef5..c026b393f3c5 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/delay.h>
+
#include "dm_services.h"
#include "core_types.h"
#include "timing_generator.h"
@@ -45,8 +47,10 @@ enum dc_color_space_type {
COLOR_SPACE_RGB_LIMITED_TYPE,
COLOR_SPACE_YCBCR601_TYPE,
COLOR_SPACE_YCBCR709_TYPE,
+ COLOR_SPACE_YCBCR2020_TYPE,
COLOR_SPACE_YCBCR601_LIMITED_TYPE,
- COLOR_SPACE_YCBCR709_LIMITED_TYPE
+ COLOR_SPACE_YCBCR709_LIMITED_TYPE,
+ COLOR_SPACE_YCBCR709_BLACK_TYPE,
};
static const struct tg_color black_color_format[] = {
@@ -80,7 +84,6 @@ static const struct out_csc_color_matrix_type output_csc_matrix[] = {
{ COLOR_SPACE_YCBCR709_TYPE,
{ 0xE04, 0xF345, 0xFEB7, 0x1004, 0x5D3, 0x1399, 0x1FA,
0x201, 0xFCCA, 0xF533, 0xE04, 0x1004} },
-
/* TODO: correct values below */
{ COLOR_SPACE_YCBCR601_LIMITED_TYPE,
{ 0xE00, 0xF447, 0xFDB9, 0x1000, 0x991,
@@ -88,6 +91,12 @@ static const struct out_csc_color_matrix_type output_csc_matrix[] = {
{ COLOR_SPACE_YCBCR709_LIMITED_TYPE,
{ 0xE00, 0xF349, 0xFEB7, 0x1000, 0x6CE, 0x16E3,
0x24F, 0x200, 0xFCCB, 0xF535, 0xE00, 0x1000} },
+ { COLOR_SPACE_YCBCR2020_TYPE,
+ { 0x1000, 0xF149, 0xFEB7, 0x0000, 0x0868, 0x15B2,
+ 0x01E6, 0x0000, 0xFB88, 0xF478, 0x1000, 0x0000} },
+ { COLOR_SPACE_YCBCR709_BLACK_TYPE,
+ { 0x0000, 0x0000, 0x0000, 0x1000, 0x0000, 0x0000,
+ 0x0000, 0x0200, 0x0000, 0x0000, 0x0000, 0x1000} },
};
static bool is_rgb_type(
@@ -149,6 +158,16 @@ static bool is_ycbcr709_type(
return ret;
}
+static bool is_ycbcr2020_type(
+ enum dc_color_space color_space)
+{
+ bool ret = false;
+
+ if (color_space == COLOR_SPACE_2020_YCBCR)
+ ret = true;
+ return ret;
+}
+
static bool is_ycbcr709_limited_type(
enum dc_color_space color_space)
{
@@ -174,7 +193,12 @@ enum dc_color_space_type get_color_space_type(enum dc_color_space color_space)
type = COLOR_SPACE_YCBCR601_LIMITED_TYPE;
else if (is_ycbcr709_limited_type(color_space))
type = COLOR_SPACE_YCBCR709_LIMITED_TYPE;
-
+ else if (is_ycbcr2020_type(color_space))
+ type = COLOR_SPACE_YCBCR2020_TYPE;
+ else if (color_space == COLOR_SPACE_YCBCR709)
+ type = COLOR_SPACE_YCBCR709_BLACK_TYPE;
+ else if (color_space == COLOR_SPACE_YCBCR709_BLACK)
+ type = COLOR_SPACE_YCBCR709_BLACK_TYPE;
return type;
}
@@ -206,6 +230,7 @@ void color_space_to_black_color(
switch (colorspace) {
case COLOR_SPACE_YCBCR601:
case COLOR_SPACE_YCBCR709:
+ case COLOR_SPACE_YCBCR709_BLACK:
case COLOR_SPACE_YCBCR601_LIMITED:
case COLOR_SPACE_YCBCR709_LIMITED:
case COLOR_SPACE_2020_YCBCR:
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c
index b37ecc3ede61..355b4ba12796 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "atom.h"
#include "dm_helpers.h"
@@ -42,6 +44,11 @@
#include "fixed31_32.h"
#include "dpcd_defs.h"
#include "dmcu.h"
+#include "hw/clk_mgr.h"
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#include "resource.h"
+#endif
+#include "hw/clk_mgr.h"
#define DC_LOGGER_INIT(logger)
@@ -216,8 +223,11 @@ bool dc_link_detect_sink(struct dc_link *link, enum dc_connection_type *type)
return true;
}
- if (link->connector_signal == SIGNAL_TYPE_EDP)
+ if (link->connector_signal == SIGNAL_TYPE_EDP) {
+ /*in case it is not on*/
+ link->dc->hwss.edp_power_control(link, true);
link->dc->hwss.edp_wait_for_hpd_ready(link, true);
+ }
/* todo: may need to lock gpio access */
hpd_pin = get_hpd_gpio(link->ctx->dc_bios, link->link_id, link->ctx->gpio_service);
@@ -519,11 +529,30 @@ static void read_edp_current_link_settings_on_detect(struct dc_link *link)
union lane_count_set lane_count_set = { {0} };
uint8_t link_bw_set;
uint8_t link_rate_set;
+ uint32_t read_dpcd_retry_cnt = 10;
+ enum dc_status status = DC_ERROR_UNEXPECTED;
+ int i;
+ union max_down_spread max_down_spread = { {0} };
// Read DPCD 00101h to find out the number of lanes currently set
- core_link_read_dpcd(link, DP_LANE_COUNT_SET,
- &lane_count_set.raw, sizeof(lane_count_set));
- link->cur_link_settings.lane_count = lane_count_set.bits.LANE_COUNT_SET;
+ for (i = 0; i < read_dpcd_retry_cnt; i++) {
+ status = core_link_read_dpcd(
+ link,
+ DP_LANE_COUNT_SET,
+ &lane_count_set.raw,
+ sizeof(lane_count_set));
+ /* First DPCD read after VDD ON can fail if the particular board
+ * does not have HPD pin wired correctly. So if DPCD read fails,
+ * which it should never happen, retry a few times. Target worst
+ * case scenario of 80 ms.
+ */
+ if (status == DC_OK) {
+ link->cur_link_settings.lane_count = lane_count_set.bits.LANE_COUNT_SET;
+ break;
+ }
+
+ msleep(8);
+ }
// Read DPCD 00100h to find if standard link rates are set
core_link_read_dpcd(link, DP_LINK_BW_SET,
@@ -546,6 +575,12 @@ static void read_edp_current_link_settings_on_detect(struct dc_link *link)
link->cur_link_settings.link_rate = link_bw_set;
link->cur_link_settings.use_link_rate_set = false;
}
+ // Read DPCD 00003h to find the max down spread.
+ core_link_read_dpcd(link, DP_MAX_DOWNSPREAD,
+ &max_down_spread.raw, sizeof(max_down_spread));
+ link->cur_link_settings.link_spread =
+ max_down_spread.bits.MAX_DOWN_SPREAD ?
+ LINK_SPREAD_05_DOWNSPREAD_30KHZ : LINK_SPREAD_DISABLED;
}
static bool detect_dp(
@@ -677,24 +712,16 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
if (dc_is_virtual_signal(link->connector_signal))
return false;
+ if ((link->connector_signal == SIGNAL_TYPE_LVDS ||
+ link->connector_signal == SIGNAL_TYPE_EDP) &&
+ link->local_sink)
+ return true;
+
if (false == dc_link_detect_sink(link, &new_connection_type)) {
BREAK_TO_DEBUGGER();
return false;
}
- if (link->connector_signal == SIGNAL_TYPE_EDP) {
- /* On detect, we want to make sure current link settings are
- * up to date, especially if link was powered on by GOP.
- */
- read_edp_current_link_settings_on_detect(link);
- if (link->local_sink)
- return true;
- }
-
- if (link->connector_signal == SIGNAL_TYPE_LVDS &&
- link->local_sink)
- return true;
-
prev_sink = link->local_sink;
if (prev_sink != NULL) {
dc_sink_retain(prev_sink);
@@ -704,6 +731,7 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
if (new_connection_type != dc_connection_none) {
link->type = new_connection_type;
+ link->link_state_valid = false;
/* From Disconnected-to-Connected. */
switch (link->connector_signal) {
@@ -735,6 +763,7 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
}
case SIGNAL_TYPE_EDP: {
+ read_edp_current_link_settings_on_detect(link);
detect_edp_sink_caps(link);
sink_caps.transaction_type =
DDC_TRANSACTION_TYPE_I2C_OVER_AUX;
@@ -906,10 +935,10 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
sink->sink_signal = SIGNAL_TYPE_DVI_SINGLE_LINK;
/* Connectivity log: detection */
- for (i = 0; i < sink->dc_edid.length / EDID_BLOCK_SIZE; i++) {
+ for (i = 0; i < sink->dc_edid.length / DC_EDID_BLOCK_SIZE; i++) {
CONN_DATA_DETECT(link,
- &sink->dc_edid.raw_edid[i * EDID_BLOCK_SIZE],
- EDID_BLOCK_SIZE,
+ &sink->dc_edid.raw_edid[i * DC_EDID_BLOCK_SIZE],
+ DC_EDID_BLOCK_SIZE,
"%s: [Block %d] ", sink->edid_caps.display_name, i);
}
@@ -960,6 +989,12 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
link->type = dc_connection_none;
sink_caps.signal = SIGNAL_TYPE_NONE;
+ /* When we unplug a passive DP-HDMI dongle connection, dongle_max_pix_clk
+ * is not cleared. If we emulate a DP signal on this connection, it thinks
+ * the dongle is still there and limits the number of modes we can emulate.
+ * Clear dongle_max_pix_clk on disconnect to fix this
+ */
+ link->dongle_max_pix_clk = 0;
}
LINK_INFO("link=%d, dc_sink_in=%p is now %s prev_sink=%p dpcd same=%d edid same=%d\n",
@@ -1156,7 +1191,7 @@ static bool construct(
link->link_id = bios->funcs->get_connector_id(bios, init_params->connector_index);
if (link->link_id.type != OBJECT_TYPE_CONNECTOR) {
- dm_error("%s: Invalid Connector ObjectID from Adapter Service for connector index:%d! type %d expected %d\n",
+ dm_output_to_console("%s: Invalid Connector ObjectID from Adapter Service for connector index:%d! type %d expected %d\n",
__func__, init_params->connector_index,
link->link_id.type, OBJECT_TYPE_CONNECTOR);
goto create_fail;
@@ -1474,6 +1509,10 @@ static enum dc_status enable_link_dp(
if (link_settings.link_rate == LINK_RATE_LOW)
skip_video_pattern = false;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ dp_set_fec_ready(link, true);
+#endif
+
if (perform_link_training_with_retries(
link,
&link_settings,
@@ -1485,6 +1524,9 @@ static enum dc_status enable_link_dp(
else
status = DC_FAIL_DP_LINK_TRAINING;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ dp_set_fec_enable(link, true);
+#endif
return status;
}
@@ -2107,6 +2149,14 @@ static void disable_link(struct dc_link *link, enum signal_type signal)
dp_disable_link_phy(link, signal);
else
dp_disable_link_phy_mst(link, signal);
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+
+ if (dc_is_dp_sst_signal(signal) ||
+ link->mst_stream_alloc_table.stream_count == 0) {
+ dp_set_fec_enable(link, false);
+ dp_set_fec_ready(link, false);
+ }
+#endif
} else
link->link_enc->funcs->disable_output(link->link_enc, signal);
@@ -2278,7 +2328,7 @@ bool dc_link_set_backlight_level(const struct dc_link *link,
if (core_dc->current_state->res_ctx.pipe_ctx[i].stream) {
if (core_dc->current_state->res_ctx.
pipe_ctx[i].stream->link
- == link)
+ == link) {
/* DMCU -1 for all controller id values,
* therefore +1 here
*/
@@ -2286,6 +2336,13 @@ bool dc_link_set_backlight_level(const struct dc_link *link,
core_dc->current_state->
res_ctx.pipe_ctx[i].stream_res.tg->inst +
1;
+
+ /* Disable brightness ramping when the display is blanked
+ * as it can hang the DMCU
+ */
+ if (core_dc->current_state->res_ctx.pipe_ctx[i].plane_state == NULL)
+ frame_ramp = 0;
+ }
}
}
abm->funcs->set_backlight_level_pwm(
@@ -2337,7 +2394,8 @@ void core_link_resume(struct dc_link *link)
static struct fixed31_32 get_pbn_per_slot(struct dc_stream_state *stream)
{
struct fixed31_32 mbytes_per_sec;
- uint32_t link_rate_in_mbytes_per_sec = dc_link_bandwidth_kbps(stream->link, &stream->link->cur_link_settings);
+ uint32_t link_rate_in_mbytes_per_sec = dc_link_bandwidth_kbps(stream->link,
+ &stream->link->cur_link_settings);
link_rate_in_mbytes_per_sec /= 8000; /* Kbits to MBytes */
mbytes_per_sec = dc_fixpt_from_int(link_rate_in_mbytes_per_sec);
@@ -2631,6 +2689,8 @@ void core_link_enable_stream(
stream->phy_pix_clk,
pipe_ctx->stream_res.audio != NULL);
+ pipe_ctx->stream->link->link_state_valid = true;
+
if (dc_is_dvi_signal(pipe_ctx->stream->signal))
pipe_ctx->stream_res.stream_enc->funcs->dvi_set_stream_attribute(
pipe_ctx->stream_res.stream_enc,
@@ -2700,33 +2760,76 @@ void core_link_enable_stream(
if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST)
allocate_mst_payload(pipe_ctx);
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ if (pipe_ctx->stream->timing.flags.DSC &&
+ (dc_is_dp_signal(pipe_ctx->stream->signal) ||
+ dc_is_virtual_signal(pipe_ctx->stream->signal))) {
+ dp_set_dsc_enable(pipe_ctx, true);
+ pipe_ctx->stream_res.tg->funcs->wait_for_state(
+ pipe_ctx->stream_res.tg,
+ CRTC_STATE_VBLANK);
+ }
+#endif
core_dc->hwss.unblank_stream(pipe_ctx,
&pipe_ctx->stream->link->cur_link_settings);
if (dc_is_dp_signal(pipe_ctx->stream->signal))
enable_stream_features(pipe_ctx);
}
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ else { // if (IS_FPGA_MAXIMUS_DC(core_dc->ctx->dce_environment))
+ if (dc_is_dp_signal(pipe_ctx->stream->signal) ||
+ dc_is_virtual_signal(pipe_ctx->stream->signal))
+ dp_set_dsc_enable(pipe_ctx, true);
+ }
+#endif
}
void core_link_disable_stream(struct pipe_ctx *pipe_ctx, int option)
{
struct dc *core_dc = pipe_ctx->stream->ctx->dc;
struct dc_stream_state *stream = pipe_ctx->stream;
+ struct dc_link *link = stream->sink->link;
core_dc->hwss.blank_stream(pipe_ctx);
if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST)
deallocate_mst_payload(pipe_ctx);
- if (dc_is_hdmi_signal(pipe_ctx->stream->signal))
- dal_ddc_service_write_scdc_data(
- stream->link->ddc, 0,
- stream->timing.flags.LTE_340MCSC_SCRAMBLE);
+ if (dc_is_hdmi_signal(pipe_ctx->stream->signal)) {
+ struct ext_hdmi_settings settings = {0};
+ enum engine_id eng_id = pipe_ctx->stream_res.stream_enc->id;
+ unsigned short masked_chip_caps = link->chip_caps &
+ EXT_DISPLAY_PATH_CAPS__EXT_CHIP_MASK;
+ //Need to inform that sink is going to use legacy HDMI mode.
+ dal_ddc_service_write_scdc_data(
+ link->ddc,
+ 165000,//vbios only handles 165Mhz.
+ false);
+ if (masked_chip_caps == EXT_DISPLAY_PATH_CAPS__HDMI20_TISN65DP159RSBT) {
+ /* DP159, Retimer settings */
+ if (get_ext_hdmi_settings(pipe_ctx, eng_id, &settings))
+ write_i2c_retimer_setting(pipe_ctx,
+ false, false, &settings);
+ else
+ write_i2c_default_retimer_setting(pipe_ctx,
+ false, false);
+ } else if (masked_chip_caps == EXT_DISPLAY_PATH_CAPS__HDMI20_PI3EQX1204) {
+ /* PI3EQX1204, Redriver settings */
+ write_i2c_redriver_setting(pipe_ctx, false);
+ }
+ }
core_dc->hwss.disable_stream(pipe_ctx, option);
disable_link(pipe_ctx->stream->link, pipe_ctx->stream->signal);
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ if (pipe_ctx->stream->timing.flags.DSC &&
+ dc_is_dp_signal(pipe_ctx->stream->signal)) {
+ dp_set_dsc_enable(pipe_ctx, false);
+ }
+#endif
}
void core_link_set_avmute(struct pipe_ctx *pipe_ctx, bool enable)
@@ -2794,6 +2897,14 @@ uint32_t dc_bandwidth_in_kbps_from_timing(
uint32_t bits_per_channel = 0;
uint32_t kbps;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ if (timing->flags.DSC) {
+ kbps = (timing->pix_clk_100hz * timing->dsc_cfg.bits_per_pixel);
+ kbps = kbps / 160 + ((kbps % 160) ? 1 : 0);
+ return kbps;
+ }
+#endif
+
switch (timing->display_color_depth) {
case COLOR_DEPTH_666:
bits_per_channel = 6;
@@ -2834,3 +2945,155 @@ uint32_t dc_bandwidth_in_kbps_from_timing(
return kbps;
}
+
+void dc_link_set_drive_settings(struct dc *dc,
+ struct link_training_settings *lt_settings,
+ const struct dc_link *link)
+{
+
+ int i;
+
+ for (i = 0; i < dc->link_count; i++) {
+ if (dc->links[i] == link)
+ break;
+ }
+
+ if (i >= dc->link_count)
+ ASSERT_CRITICAL(false);
+
+ dc_link_dp_set_drive_settings(dc->links[i], lt_settings);
+}
+
+void dc_link_perform_link_training(struct dc *dc,
+ struct dc_link_settings *link_setting,
+ bool skip_video_pattern)
+{
+ int i;
+
+ for (i = 0; i < dc->link_count; i++)
+ dc_link_dp_perform_link_training(
+ dc->links[i],
+ link_setting,
+ skip_video_pattern);
+}
+
+void dc_link_set_preferred_link_settings(struct dc *dc,
+ struct dc_link_settings *link_setting,
+ struct dc_link *link)
+{
+ int i;
+ struct pipe_ctx *pipe;
+ struct dc_stream_state *link_stream;
+ struct dc_link_settings store_settings = *link_setting;
+
+ link->preferred_link_setting = store_settings;
+
+ /* Retrain with preferred link settings only relevant for
+ * DP signal type
+ * Check for non-DP signal or if passive dongle present
+ */
+ if (!dc_is_dp_signal(link->connector_signal) ||
+ link->dongle_max_pix_clk > 0)
+ return;
+
+ for (i = 0; i < MAX_PIPES; i++) {
+ pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+ if (pipe->stream && pipe->stream->link) {
+ if (pipe->stream->link == link)
+ break;
+ }
+ }
+
+ /* Stream not found */
+ if (i == MAX_PIPES)
+ return;
+
+ link_stream = link->dc->current_state->res_ctx.pipe_ctx[i].stream;
+
+ /* Cannot retrain link if backend is off */
+ if (link_stream->dpms_off)
+ return;
+
+ if (link_stream)
+ decide_link_settings(link_stream, &store_settings);
+
+ if ((store_settings.lane_count != LANE_COUNT_UNKNOWN) &&
+ (store_settings.link_rate != LINK_RATE_UNKNOWN))
+ dp_retrain_link_dp_test(link, &store_settings, false);
+}
+
+void dc_link_enable_hpd(const struct dc_link *link)
+{
+ dc_link_dp_enable_hpd(link);
+}
+
+void dc_link_disable_hpd(const struct dc_link *link)
+{
+ dc_link_dp_disable_hpd(link);
+}
+
+
+void dc_link_set_test_pattern(struct dc_link *link,
+ enum dp_test_pattern test_pattern,
+ const struct link_training_settings *p_link_settings,
+ const unsigned char *p_custom_pattern,
+ unsigned int cust_pattern_size)
+{
+ if (link != NULL)
+ dc_link_dp_set_test_pattern(
+ link,
+ test_pattern,
+ p_link_settings,
+ p_custom_pattern,
+ cust_pattern_size);
+}
+
+uint32_t dc_link_bandwidth_kbps(
+ const struct dc_link *link,
+ const struct dc_link_settings *link_setting)
+{
+ uint32_t link_bw_kbps =
+ link_setting->link_rate * LINK_RATE_REF_FREQ_IN_KHZ; /* bytes per sec */
+
+ link_bw_kbps *= 8; /* 8 bits per byte*/
+ link_bw_kbps *= link_setting->lane_count;
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ if (link->dpcd_caps.fec_cap.bits.FEC_CAPABLE) {
+ /* Account for FEC overhead.
+ * We have to do it based on caps,
+ * and not based on FEC being set ready,
+ * because FEC is set ready too late in
+ * the process to correctly be picked up
+ * by mode enumeration.
+ *
+ * There's enough zeros at the end of 'kbps'
+ * that make the below operation 100% precise
+ * for our purposes.
+ * 'long long' makes it work even for HDMI 2.1
+ * max bandwidth (and much, much bigger bandwidths
+ * than that, actually).
+ *
+ * NOTE: Reducing link BW by 3% may not be precise
+ * because it may be a stream BT that increases by 3%, and so
+ * 1/1.03 = 0.970873 factor should have been used instead,
+ * but the difference is minimal and is in a safe direction,
+ * which all works well around potential ambiguity of DP 1.4a spec.
+ */
+ link_bw_kbps = mul_u64_u32_shr(BIT_ULL(32) * 970LL / 1000,
+ link_bw_kbps, 32);
+ }
+#endif
+
+ return link_bw_kbps;
+
+}
+
+const struct dc_link_settings *dc_link_get_link_cap(
+ const struct dc_link *link)
+{
+ if (link->preferred_link_setting.lane_count != LANE_COUNT_UNKNOWN &&
+ link->preferred_link_setting.link_rate != LINK_RATE_UNKNOWN)
+ return &link->preferred_link_setting;
+ return &link->verified_link_cap;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c
index f02092a0dc76..e6da8506128b 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "dm_helpers.h"
#include "gpio_service_interface.h"
@@ -91,6 +93,8 @@ union hdmi_scdc_status_flags_data {
uint8_t CH2_LOCKED:1;
uint8_t RESERVED:4;
uint8_t RESERVED2:8;
+ uint8_t RESERVED3:8;
+
} fields;
};
@@ -107,14 +111,10 @@ union hdmi_scdc_ced_data {
uint8_t CH2_7HIGH:7;
uint8_t CH2_VALID:1;
uint8_t CHECKSUM:8;
- } fields;
-};
-
-union hdmi_scdc_test_config_Data {
- uint8_t byte;
- struct {
- uint8_t TEST_READ_REQUEST_DELAY:7;
- uint8_t TEST_READ_REQUEST: 1;
+ uint8_t RESERVED:8;
+ uint8_t RESERVED2:8;
+ uint8_t RESERVED3:8;
+ uint8_t RESERVED4:4;
} fields;
};
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
index 1ee544a32ebb..2c7aaed907b9 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
@@ -4,6 +4,12 @@
#include "dc_link_dp.h"
#include "dm_helpers.h"
#include "opp.h"
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+#include "dsc.h"
+#endif
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#include "resource.h"
+#endif
#include "inc/core_types.h"
#include "link_hwss.h"
@@ -89,6 +95,29 @@ static void dpcd_set_training_pattern(
dpcd_pattern.v1_4.TRAINING_PATTERN_SET);
}
+static enum hw_dp_training_pattern get_supported_tp(struct dc_link *link)
+{
+ enum hw_dp_training_pattern highest_tp = HW_DP_TRAINING_PATTERN_2;
+ struct encoder_feature_support *features = &link->link_enc->features;
+ struct dpcd_caps *dpcd_caps = &link->dpcd_caps;
+
+ if (features->flags.bits.IS_TPS3_CAPABLE)
+ highest_tp = HW_DP_TRAINING_PATTERN_3;
+
+ if (features->flags.bits.IS_TPS4_CAPABLE)
+ highest_tp = HW_DP_TRAINING_PATTERN_4;
+
+ if (dpcd_caps->max_down_spread.bits.TPS4_SUPPORTED &&
+ highest_tp >= HW_DP_TRAINING_PATTERN_4)
+ return HW_DP_TRAINING_PATTERN_4;
+
+ if (dpcd_caps->max_ln_count.bits.TPS3_SUPPORTED &&
+ highest_tp >= HW_DP_TRAINING_PATTERN_3)
+ return HW_DP_TRAINING_PATTERN_3;
+
+ return HW_DP_TRAINING_PATTERN_2;
+}
+
static void dpcd_set_link_settings(
struct dc_link *link,
const struct link_training_settings *lt_settings)
@@ -97,6 +126,7 @@ static void dpcd_set_link_settings(
union down_spread_ctrl downspread = { {0} };
union lane_count_set lane_count_set = { {0} };
+ enum hw_dp_training_pattern hw_tr_pattern;
downspread.raw = (uint8_t)
(lt_settings->link_settings.link_spread);
@@ -106,8 +136,13 @@ static void dpcd_set_link_settings(
lane_count_set.bits.ENHANCED_FRAMING = 1;
- lane_count_set.bits.POST_LT_ADJ_REQ_GRANTED =
- link->dpcd_caps.max_ln_count.bits.POST_LT_ADJ_REQ_SUPPORTED;
+ lane_count_set.bits.POST_LT_ADJ_REQ_GRANTED = 0;
+
+ hw_tr_pattern = get_supported_tp(link);
+ if (hw_tr_pattern != HW_DP_TRAINING_PATTERN_4) {
+ lane_count_set.bits.POST_LT_ADJ_REQ_GRANTED =
+ link->dpcd_caps.max_ln_count.bits.POST_LT_ADJ_REQ_SUPPORTED;
+ }
core_link_write_dpcd(link, DP_DOWNSPREAD_CTRL,
&downspread.raw, sizeof(downspread));
@@ -698,29 +733,6 @@ static bool perform_post_lt_adj_req_sequence(
}
-static enum hw_dp_training_pattern get_supported_tp(struct dc_link *link)
-{
- enum hw_dp_training_pattern highest_tp = HW_DP_TRAINING_PATTERN_2;
- struct encoder_feature_support *features = &link->link_enc->features;
- struct dpcd_caps *dpcd_caps = &link->dpcd_caps;
-
- if (features->flags.bits.IS_TPS3_CAPABLE)
- highest_tp = HW_DP_TRAINING_PATTERN_3;
-
- if (features->flags.bits.IS_TPS4_CAPABLE)
- highest_tp = HW_DP_TRAINING_PATTERN_4;
-
- if (dpcd_caps->max_down_spread.bits.TPS4_SUPPORTED &&
- highest_tp >= HW_DP_TRAINING_PATTERN_4)
- return HW_DP_TRAINING_PATTERN_4;
-
- if (dpcd_caps->max_ln_count.bits.TPS3_SUPPORTED &&
- highest_tp >= HW_DP_TRAINING_PATTERN_3)
- return HW_DP_TRAINING_PATTERN_3;
-
- return HW_DP_TRAINING_PATTERN_2;
-}
-
static enum link_training_result get_cr_failure(enum dc_lane_count ln_count,
union lane_status *dpcd_lane_status)
{
@@ -1624,8 +1636,7 @@ static bool decide_edp_link_settings(struct dc_link *link, struct dc_link_settin
uint32_t link_bw;
if (link->dpcd_caps.dpcd_rev.raw < DPCD_REV_14 ||
- link->dpcd_caps.edp_supported_link_rates_count == 0 ||
- link->dc->config.optimize_edp_link_rate == false) {
+ link->dpcd_caps.edp_supported_link_rates_count == 0) {
*link_setting = link->verified_link_cap;
return true;
}
@@ -2219,18 +2230,25 @@ static void get_active_converter_info(
link->dpcd_caps.dongle_type = DISPLAY_DONGLE_NONE;
ddc_service_set_dongle_type(link->ddc,
link->dpcd_caps.dongle_type);
+ link->dpcd_caps.is_branch_dev = false;
return;
}
/* DPCD 0x5 bit 0 = 1, it indicate it's branch device */
- link->dpcd_caps.is_branch_dev = ds_port.fields.PORT_PRESENT;
+ if (ds_port.fields.PORT_TYPE == DOWNSTREAM_DP) {
+ link->dpcd_caps.is_branch_dev = false;
+ }
+
+ else {
+ link->dpcd_caps.is_branch_dev = ds_port.fields.PORT_PRESENT;
+ }
switch (ds_port.fields.PORT_TYPE) {
case DOWNSTREAM_VGA:
link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_VGA_CONVERTER;
break;
- case DOWNSTREAM_DVI_HDMI:
- /* At this point we don't know is it DVI or HDMI,
+ case DOWNSTREAM_DVI_HDMI_DP_PLUS_PLUS:
+ /* At this point we don't know is it DVI or HDMI or DP++,
* assume DVI.*/
link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_DVI_CONVERTER;
break;
@@ -2247,6 +2265,10 @@ static void get_active_converter_info(
det_caps, sizeof(det_caps));
switch (port_caps->bits.DWN_STRM_PORTX_TYPE) {
+ /*Handle DP case as DONGLE_NONE*/
+ case DOWN_STREAM_DETAILED_DP:
+ link->dpcd_caps.dongle_type = DISPLAY_DONGLE_NONE;
+ break;
case DOWN_STREAM_DETAILED_VGA:
link->dpcd_caps.dongle_type =
DISPLAY_DONGLE_DP_VGA_CONVERTER;
@@ -2256,6 +2278,8 @@ static void get_active_converter_info(
DISPLAY_DONGLE_DP_DVI_CONVERTER;
break;
case DOWN_STREAM_DETAILED_HDMI:
+ case DOWN_STREAM_DETAILED_DP_PLUS_PLUS:
+ /*Handle DP++ active converter case, process DP++ case as HDMI case according DP1.4 spec*/
link->dpcd_caps.dongle_type =
DISPLAY_DONGLE_DP_HDMI_CONVERTER;
@@ -2271,14 +2295,18 @@ static void get_active_converter_info(
link->dpcd_caps.dongle_caps.is_dp_hdmi_s3d_converter =
hdmi_caps.bits.FRAME_SEQ_TO_FRAME_PACK;
- link->dpcd_caps.dongle_caps.is_dp_hdmi_ycbcr422_pass_through =
- hdmi_caps.bits.YCrCr422_PASS_THROUGH;
- link->dpcd_caps.dongle_caps.is_dp_hdmi_ycbcr420_pass_through =
- hdmi_caps.bits.YCrCr420_PASS_THROUGH;
- link->dpcd_caps.dongle_caps.is_dp_hdmi_ycbcr422_converter =
- hdmi_caps.bits.YCrCr422_CONVERSION;
- link->dpcd_caps.dongle_caps.is_dp_hdmi_ycbcr420_converter =
- hdmi_caps.bits.YCrCr420_CONVERSION;
+ /*YCBCR capability only for HDMI case*/
+ if (port_caps->bits.DWN_STRM_PORTX_TYPE
+ == DOWN_STREAM_DETAILED_HDMI) {
+ link->dpcd_caps.dongle_caps.is_dp_hdmi_ycbcr422_pass_through =
+ hdmi_caps.bits.YCrCr422_PASS_THROUGH;
+ link->dpcd_caps.dongle_caps.is_dp_hdmi_ycbcr420_pass_through =
+ hdmi_caps.bits.YCrCr420_PASS_THROUGH;
+ link->dpcd_caps.dongle_caps.is_dp_hdmi_ycbcr422_converter =
+ hdmi_caps.bits.YCrCr422_CONVERSION;
+ link->dpcd_caps.dongle_caps.is_dp_hdmi_ycbcr420_converter =
+ hdmi_caps.bits.YCrCr420_CONVERSION;
+ }
link->dpcd_caps.dongle_caps.dp_hdmi_max_bpc =
translate_dpcd_max_bpc(
@@ -2361,6 +2389,7 @@ static bool retrieve_link_cap(struct dc_link *link)
/*Only need to read 1 byte starting from DP_DPRX_FEATURE_ENUMERATION_LIST.
*/
uint8_t dpcd_dprx_data = '\0';
+ uint8_t dpcd_power_state = '\0';
struct dp_device_vendor_id sink_id;
union down_stream_port_count down_strm_port_count;
@@ -2377,6 +2406,17 @@ static bool retrieve_link_cap(struct dc_link *link)
memset(&edp_config_cap, '\0',
sizeof(union edp_configuration_cap));
+ status = core_link_read_dpcd(link, DP_SET_POWER,
+ &dpcd_power_state, sizeof(dpcd_power_state));
+
+ /* Delay 1 ms if AUX CH is in power down state. Based on spec
+ * section 2.3.1.2, if AUX CH may be powered down due to
+ * write to DPCD 600h = 2. Sink AUX CH is monitoring differential
+ * signal and may need up to 1 ms before being able to reply.
+ */
+ if (status != DC_OK || dpcd_power_state == DP_SET_POWER_D3)
+ udelay(1000);
+
for (i = 0; i < read_dpcd_retry_cnt; i++) {
status = core_link_read_dpcd(
link,
@@ -2530,6 +2570,30 @@ static bool retrieve_link_cap(struct dc_link *link)
dp_hw_fw_revision.ieee_fw_rev,
sizeof(dp_hw_fw_revision.ieee_fw_rev));
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ memset(&link->dpcd_caps.dsc_caps, '\0',
+ sizeof(link->dpcd_caps.dsc_caps));
+ memset(&link->dpcd_caps.fec_cap, '\0', sizeof(link->dpcd_caps.fec_cap));
+ /* Read DSC and FEC sink capabilities if DP revision is 1.4 and up */
+ if (link->dpcd_caps.dpcd_rev.raw >= DPCD_REV_14) {
+ status = core_link_read_dpcd(
+ link,
+ DP_FEC_CAPABILITY,
+ &link->dpcd_caps.fec_cap.raw,
+ sizeof(link->dpcd_caps.fec_cap.raw));
+ status = core_link_read_dpcd(
+ link,
+ DP_DSC_SUPPORT,
+ link->dpcd_caps.dsc_caps.dsc_basic_caps.raw,
+ sizeof(link->dpcd_caps.dsc_caps.dsc_basic_caps.raw));
+ status = core_link_read_dpcd(
+ link,
+ DP_DSC_BRANCH_OVERALL_THROUGHPUT_0,
+ link->dpcd_caps.dsc_caps.dsc_ext_caps.raw,
+ sizeof(link->dpcd_caps.dsc_caps.dsc_ext_caps.raw));
+ }
+#endif
+
/* Connectivity log: detection */
CONN_DATA_DETECT(link, dpcd_data, sizeof(dpcd_data), "Rx Caps: ");
@@ -2597,7 +2661,8 @@ void detect_edp_sink_caps(struct dc_link *link)
memset(supported_link_rates, 0, sizeof(supported_link_rates));
if (link->dpcd_caps.dpcd_rev.raw >= DPCD_REV_14 &&
- link->dc->config.optimize_edp_link_rate) {
+ (link->dc->config.optimize_edp_link_rate ||
+ link->reported_link_cap.link_rate == LINK_RATE_UNKNOWN)) {
// Read DPCD 00010h - 0001Fh 16 bytes at one shot
core_link_read_dpcd(link, DP_SUPPORTED_LINK_RATES,
supported_link_rates, sizeof(supported_link_rates));
@@ -2612,6 +2677,9 @@ void detect_edp_sink_caps(struct dc_link *link)
link_rate = linkRateInKHzToLinkRateMultiplier(link_rate_in_khz);
link->dpcd_caps.edp_supported_link_rates[link->dpcd_caps.edp_supported_link_rates_count] = link_rate;
link->dpcd_caps.edp_supported_link_rates_count++;
+
+ if (link->reported_link_cap.link_rate < link_rate)
+ link->reported_link_cap.link_rate = link_rate;
}
}
}
@@ -2653,6 +2721,14 @@ static void set_crtc_test_pattern(struct dc_link *link,
stream->timing.display_color_depth;
struct bit_depth_reduction_params params;
struct output_pixel_processor *opp = pipe_ctx->stream_res.opp;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ int width = pipe_ctx->stream->timing.h_addressable +
+ pipe_ctx->stream->timing.h_border_left +
+ pipe_ctx->stream->timing.h_border_right;
+ int height = pipe_ctx->stream->timing.v_addressable +
+ pipe_ctx->stream->timing.v_border_bottom +
+ pipe_ctx->stream->timing.v_border_top;
+#endif
memset(&params, 0, sizeof(params));
@@ -2696,6 +2772,30 @@ static void set_crtc_test_pattern(struct dc_link *link,
if (pipe_ctx->stream_res.tg->funcs->set_test_pattern)
pipe_ctx->stream_res.tg->funcs->set_test_pattern(pipe_ctx->stream_res.tg,
controller_test_pattern, color_depth);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ else if (opp->funcs->opp_set_disp_pattern_generator) {
+ struct pipe_ctx *bot_odm_pipe = dc_res_get_odm_bottom_pipe(pipe_ctx);
+
+ if (bot_odm_pipe) {
+ struct output_pixel_processor *bot_opp = bot_odm_pipe->stream_res.opp;
+
+ bot_opp->funcs->opp_program_bit_depth_reduction(bot_opp, &params);
+ width /= 2;
+ bot_opp->funcs->opp_set_disp_pattern_generator(bot_opp,
+ controller_test_pattern,
+ color_depth,
+ NULL,
+ width,
+ height);
+ }
+ opp->funcs->opp_set_disp_pattern_generator(opp,
+ controller_test_pattern,
+ color_depth,
+ NULL,
+ width,
+ height);
+ }
+#endif
}
break;
case DP_TEST_PATTERN_VIDEO_MODE:
@@ -2708,6 +2808,30 @@ static void set_crtc_test_pattern(struct dc_link *link,
pipe_ctx->stream_res.tg->funcs->set_test_pattern(pipe_ctx->stream_res.tg,
CONTROLLER_DP_TEST_PATTERN_VIDEOMODE,
color_depth);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ else if (opp->funcs->opp_set_disp_pattern_generator) {
+ struct pipe_ctx *bot_odm_pipe = dc_res_get_odm_bottom_pipe(pipe_ctx);
+
+ if (bot_odm_pipe) {
+ struct output_pixel_processor *bot_opp = bot_odm_pipe->stream_res.opp;
+
+ bot_opp->funcs->opp_program_bit_depth_reduction(bot_opp, &params);
+ width /= 2;
+ bot_opp->funcs->opp_set_disp_pattern_generator(bot_opp,
+ CONTROLLER_DP_TEST_PATTERN_VIDEOMODE,
+ color_depth,
+ NULL,
+ width,
+ height);
+ }
+ opp->funcs->opp_set_disp_pattern_generator(opp,
+ CONTROLLER_DP_TEST_PATTERN_VIDEOMODE,
+ color_depth,
+ NULL,
+ width,
+ height);
+ }
+#endif
}
break;
@@ -2882,3 +3006,67 @@ void dp_enable_mst_on_sink(struct dc_link *link, bool enable)
core_link_write_dpcd(link, DP_MSTM_CTRL, &mstmCntl, 1);
}
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+void dp_set_fec_ready(struct dc_link *link, bool ready)
+{
+ /* FEC has to be "set ready" before the link training.
+ * The policy is to always train with FEC
+ * if the sink supports it and leave it enabled on link.
+ * If FEC is not supported, disable it.
+ */
+ struct link_encoder *link_enc = link->link_enc;
+ uint8_t fec_config = 0;
+
+ if (link->dc->debug.disable_fec ||
+ IS_FPGA_MAXIMUS_DC(link->ctx->dce_environment))
+ return;
+
+ if (link_enc->funcs->fec_set_ready &&
+ link->dpcd_caps.fec_cap.bits.FEC_CAPABLE) {
+ if (link->fec_state == dc_link_fec_not_ready && ready) {
+ fec_config = 1;
+ if (core_link_write_dpcd(link,
+ DP_FEC_CONFIGURATION,
+ &fec_config,
+ sizeof(fec_config)) == DC_OK) {
+ link_enc->funcs->fec_set_ready(link_enc, true);
+ link->fec_state = dc_link_fec_ready;
+ } else {
+ dm_error("dpcd write failed to set fec_ready");
+ }
+ } else if (link->fec_state == dc_link_fec_ready && !ready) {
+ fec_config = 0;
+ core_link_write_dpcd(link,
+ DP_FEC_CONFIGURATION,
+ &fec_config,
+ sizeof(fec_config));
+ link->link_enc->funcs->fec_set_ready(
+ link->link_enc, false);
+ link->fec_state = dc_link_fec_not_ready;
+ }
+ }
+}
+
+void dp_set_fec_enable(struct dc_link *link, bool enable)
+{
+ struct link_encoder *link_enc = link->link_enc;
+
+ if (link->dc->debug.disable_fec ||
+ IS_FPGA_MAXIMUS_DC(link->ctx->dce_environment))
+ return;
+
+ if (link_enc->funcs->fec_set_enable &&
+ link->dpcd_caps.fec_cap.bits.FEC_CAPABLE) {
+ if (link->fec_state == dc_link_fec_ready && enable) {
+ msleep(1);
+ link_enc->funcs->fec_set_enable(link_enc, true);
+ link->fec_state = dc_link_fec_enabled;
+ } else if (link->fec_state == dc_link_fec_enabled && !enable) {
+ link_enc->funcs->fec_set_enable(link_enc, false);
+ link->fec_state = dc_link_fec_ready;
+ }
+ }
+}
+#endif
+
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c
index b0dea759cd86..2d019e1f6135 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c
@@ -12,6 +12,12 @@
#include "dc_link_ddc.h"
#include "dm_helpers.h"
#include "dpcd_defs.h"
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+#include "dsc.h"
+#endif
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#include "resource.h"
+#endif
enum dc_status core_link_read_dpcd(
struct dc_link *link,
@@ -360,3 +366,141 @@ void dp_retrain_link_dp_test(struct dc_link *link,
}
}
}
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+#define DC_LOGGER \
+ dsc->ctx->logger
+static void dsc_optc_config_log(struct display_stream_compressor *dsc,
+ struct dsc_optc_config *config)
+{
+ DC_LOG_DSC("Setting optc DSC config at DSC inst %d", dsc->inst);
+ DC_LOG_DSC("\n\tbytes_per_pixel %d\n\tis_pixel_format_444 %d\n\tslice_width %d",
+ config->bytes_per_pixel,
+ config->is_pixel_format_444, config->slice_width);
+}
+
+static bool dp_set_dsc_on_rx(struct pipe_ctx *pipe_ctx, bool enable)
+{
+ struct dc *core_dc = pipe_ctx->stream->ctx->dc;
+ struct dc_stream_state *stream = pipe_ctx->stream;
+ bool result = false;
+
+ if (IS_FPGA_MAXIMUS_DC(core_dc->ctx->dce_environment))
+ result = true;
+ else
+ result = dm_helpers_dp_write_dsc_enable(core_dc->ctx, stream, enable);
+ return result;
+}
+
+/* This has to be done after DSC was enabled on RX first, i.e. after dp_enable_dsc_on_rx() had been called
+ */
+static void dp_set_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
+{
+ struct display_stream_compressor *dsc = pipe_ctx->stream_res.dsc;
+ struct dc *core_dc = pipe_ctx->stream->ctx->dc;
+ struct dc_stream_state *stream = pipe_ctx->stream;
+ struct pipe_ctx *odm_pipe = dc_res_get_odm_bottom_pipe(pipe_ctx);
+
+ if (enable) {
+ /* TODO proper function */
+ struct dsc_config dsc_cfg;
+ struct dsc_optc_config dsc_optc_cfg;
+ enum optc_dsc_mode optc_dsc_mode;
+ uint8_t dsc_packed_pps[128];
+
+ /* Enable DSC hw block */
+ dsc_cfg.pic_width = stream->timing.h_addressable + stream->timing.h_border_left + stream->timing.h_border_right;
+ dsc_cfg.pic_height = stream->timing.v_addressable + stream->timing.v_border_top + stream->timing.v_border_bottom;
+ dsc_cfg.pixel_encoding = stream->timing.pixel_encoding;
+ dsc_cfg.color_depth = stream->timing.display_color_depth;
+ dsc_cfg.dc_dsc_cfg = stream->timing.dsc_cfg;
+
+ dsc->funcs->dsc_set_config(dsc, &dsc_cfg, &dsc_optc_cfg, &dsc_packed_pps[0]);
+ if (odm_pipe) {
+ struct display_stream_compressor *bot_dsc = odm_pipe->stream_res.dsc;
+ uint8_t dsc_packed_pps_odm[128];
+
+ dsc_cfg.pic_width /= 2;
+ ASSERT(dsc_cfg.dc_dsc_cfg.num_slices_h % 2 == 0);
+ dsc_cfg.dc_dsc_cfg.num_slices_h /= 2;
+ dsc->funcs->dsc_set_config(dsc, &dsc_cfg, &dsc_optc_cfg, &dsc_packed_pps_odm[0]);
+ bot_dsc->funcs->dsc_set_config(bot_dsc, &dsc_cfg, &dsc_optc_cfg, &dsc_packed_pps_odm[0]);
+ bot_dsc->funcs->dsc_enable(bot_dsc, odm_pipe->stream_res.opp->inst);
+ }
+ dsc->funcs->dsc_enable(dsc, pipe_ctx->stream_res.opp->inst);
+
+ optc_dsc_mode = dsc_optc_cfg.is_pixel_format_444 ? OPTC_DSC_ENABLED_444 : OPTC_DSC_ENABLED_NATIVE_SUBSAMPLED;
+
+ dsc_optc_config_log(dsc, &dsc_optc_cfg);
+ /* Enable DSC in encoder */
+ if (!IS_FPGA_MAXIMUS_DC(core_dc->ctx->dce_environment) && pipe_ctx->stream_res.stream_enc->funcs->dp_set_dsc_config)
+ pipe_ctx->stream_res.stream_enc->funcs->dp_set_dsc_config(pipe_ctx->stream_res.stream_enc,
+ optc_dsc_mode,
+ dsc_optc_cfg.bytes_per_pixel,
+ dsc_optc_cfg.slice_width,
+ &dsc_packed_pps[0]);
+
+ /* Enable DSC in OPTC */
+ pipe_ctx->stream_res.tg->funcs->set_dsc_config(pipe_ctx->stream_res.tg,
+ optc_dsc_mode,
+ dsc_optc_cfg.bytes_per_pixel,
+ dsc_optc_cfg.slice_width);
+ } else {
+ /* disable DSC in OPTC */
+ pipe_ctx->stream_res.tg->funcs->set_dsc_config(
+ pipe_ctx->stream_res.tg,
+ OPTC_DSC_DISABLED, 0, 0);
+
+ /* disable DSC in stream encoder */
+ if (!IS_FPGA_MAXIMUS_DC(core_dc->ctx->dce_environment)) {
+ pipe_ctx->stream_res.stream_enc->funcs->dp_set_dsc_config(
+ pipe_ctx->stream_res.stream_enc,
+ OPTC_DSC_DISABLED, 0, 0, NULL);
+ }
+
+ /* disable DSC block */
+ pipe_ctx->stream_res.dsc->funcs->dsc_disable(pipe_ctx->stream_res.dsc);
+ if (odm_pipe)
+ odm_pipe->stream_res.dsc->funcs->dsc_disable(odm_pipe->stream_res.dsc);
+ }
+}
+
+bool dp_set_dsc_enable(struct pipe_ctx *pipe_ctx, bool enable)
+{
+ struct display_stream_compressor *dsc = pipe_ctx->stream_res.dsc;
+ bool result = false;
+
+ if (!pipe_ctx->stream->timing.flags.DSC)
+ goto out;
+ if (!dsc)
+ goto out;
+
+ if (enable) {
+ if (dp_set_dsc_on_rx(pipe_ctx, true)) {
+ dp_set_dsc_on_stream(pipe_ctx, true);
+ result = true;
+ }
+ } else {
+ dp_set_dsc_on_rx(pipe_ctx, false);
+ dp_set_dsc_on_stream(pipe_ctx, false);
+ result = true;
+ }
+out:
+ return result;
+}
+
+bool dp_update_dsc_config(struct pipe_ctx *pipe_ctx)
+{
+ struct display_stream_compressor *dsc = pipe_ctx->stream_res.dsc;
+
+ if (!pipe_ctx->stream->timing.flags.DSC)
+ return false;
+ if (!dsc)
+ return false;
+
+ dp_set_dsc_on_stream(pipe_ctx, true);
+ return true;
+}
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
index eac7186e4f08..2ceaab4fb5de 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
@@ -22,6 +22,9 @@
* Authors: AMD
*
*/
+
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "resource.h"
@@ -46,6 +49,9 @@
#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
#include "dcn10/dcn10_resource.h"
#endif
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#include "dcn20/dcn20_resource.h"
+#endif
#include "dce120/dce120_resource.h"
#define DC_LOGGER_INIT(logger)
@@ -93,10 +99,14 @@ enum dce_version resource_parse_asic_id(struct hw_asic_id asic_id)
#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
case FAMILY_RV:
dc_version = DCN_VERSION_1_0;
-#if defined(CONFIG_DRM_AMD_DC_DCN1_01)
if (ASICREV_IS_RAVEN2(asic_id.hw_internal_rev))
dc_version = DCN_VERSION_1_01;
+ break;
#endif
+
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ case FAMILY_NV:
+ dc_version = DCN_VERSION_2_0;
break;
#endif
default:
@@ -147,46 +157,40 @@ struct resource_pool *dc_create_resource_pool(struct dc *dc,
#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
case DCN_VERSION_1_0:
-#if defined(CONFIG_DRM_AMD_DC_DCN1_01)
case DCN_VERSION_1_01:
-#endif
res_pool = dcn10_create_resource_pool(init_data, dc);
break;
#endif
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ case DCN_VERSION_2_0:
+ res_pool = dcn20_create_resource_pool(init_data, dc);
+ break;
+#endif
+
default:
break;
}
if (res_pool != NULL) {
struct dc_firmware_info fw_info = { { 0 } };
- if (dc->ctx->dc_bios->funcs->get_firmware_info(
- dc->ctx->dc_bios, &fw_info) == BP_RESULT_OK) {
- res_pool->ref_clocks.xtalin_clock_inKhz = fw_info.pll_info.crystal_frequency;
-
- if (IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) {
- // On FPGA these dividers are currently not configured by GDB
- res_pool->ref_clocks.dccg_ref_clock_inKhz = res_pool->ref_clocks.xtalin_clock_inKhz;
- res_pool->ref_clocks.dchub_ref_clock_inKhz = res_pool->ref_clocks.xtalin_clock_inKhz;
- } else if (res_pool->dccg && res_pool->hubbub) {
- // If DCCG reference frequency cannot be determined (usually means not set to xtalin) then this is a critical error
- // as this value must be known for DCHUB programming
- (res_pool->dccg->funcs->get_dccg_ref_freq)(res_pool->dccg,
- fw_info.pll_info.crystal_frequency,
- &res_pool->ref_clocks.dccg_ref_clock_inKhz);
-
- // Similarly, if DCHUB reference frequency cannot be determined, then it is also a critical error
- (res_pool->hubbub->funcs->get_dchub_ref_freq)(res_pool->hubbub,
- res_pool->ref_clocks.dccg_ref_clock_inKhz,
- &res_pool->ref_clocks.dchub_ref_clock_inKhz);
- } else {
- // Not all ASICs have DCCG sw component
- res_pool->ref_clocks.dccg_ref_clock_inKhz = res_pool->ref_clocks.xtalin_clock_inKhz;
- res_pool->ref_clocks.dchub_ref_clock_inKhz = res_pool->ref_clocks.xtalin_clock_inKhz;
- }
- } else
- ASSERT_CRITICAL(false);
+ if (dc->ctx->dc_bios->funcs->get_firmware_info(dc->ctx->dc_bios,
+ &fw_info) == BP_RESULT_OK) {
+ res_pool->ref_clocks.xtalin_clock_inKhz =
+ fw_info.pll_info.crystal_frequency;
+ /* initialize with firmware data first, no all
+ * ASIC have DCCG SW component. FPGA or
+ * simulation need initialization of
+ * dccg_ref_clock_inKhz, dchub_ref_clock_inKhz
+ * with xtalin_clock_inKhz
+ */
+ res_pool->ref_clocks.dccg_ref_clock_inKhz =
+ res_pool->ref_clocks.xtalin_clock_inKhz;
+ res_pool->ref_clocks.dchub_ref_clock_inKhz =
+ res_pool->ref_clocks.xtalin_clock_inKhz;
+ } else
+ ASSERT_CRITICAL(false);
}
return res_pool;
@@ -254,7 +258,7 @@ bool resource_construct(
* PORT_CONNECTIVITY == 1 (as instructed by HW team).
*/
update_num_audio(&straps, &num_audio, &pool->audio_support);
- for (i = 0; i < pool->pipe_count && i < num_audio; i++) {
+ for (i = 0; i < caps->num_audio; i++) {
struct audio *aud = create_funcs->create_audio(ctx, i);
if (aud == NULL) {
@@ -1184,24 +1188,27 @@ static int acquire_first_split_pipe(
int i;
for (i = 0; i < pool->pipe_count; i++) {
- struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
-
- if (pipe_ctx->top_pipe &&
- pipe_ctx->top_pipe->plane_state == pipe_ctx->plane_state) {
- pipe_ctx->top_pipe->bottom_pipe = pipe_ctx->bottom_pipe;
- if (pipe_ctx->bottom_pipe)
- pipe_ctx->bottom_pipe->top_pipe = pipe_ctx->top_pipe;
-
- memset(pipe_ctx, 0, sizeof(*pipe_ctx));
- pipe_ctx->stream_res.tg = pool->timing_generators[i];
- pipe_ctx->plane_res.hubp = pool->hubps[i];
- pipe_ctx->plane_res.ipp = pool->ipps[i];
- pipe_ctx->plane_res.dpp = pool->dpps[i];
- pipe_ctx->stream_res.opp = pool->opps[i];
- pipe_ctx->plane_res.mpcc_inst = pool->dpps[i]->inst;
- pipe_ctx->pipe_idx = i;
-
- pipe_ctx->stream = stream;
+ struct pipe_ctx *split_pipe = &res_ctx->pipe_ctx[i];
+
+ if (split_pipe->top_pipe && !dc_res_is_odm_head_pipe(split_pipe) &&
+ split_pipe->top_pipe->plane_state == split_pipe->plane_state) {
+ split_pipe->top_pipe->bottom_pipe = split_pipe->bottom_pipe;
+ if (split_pipe->bottom_pipe)
+ split_pipe->bottom_pipe->top_pipe = split_pipe->top_pipe;
+
+ if (split_pipe->top_pipe->plane_state)
+ resource_build_scaling_params(split_pipe->top_pipe);
+
+ memset(split_pipe, 0, sizeof(*split_pipe));
+ split_pipe->stream_res.tg = pool->timing_generators[i];
+ split_pipe->plane_res.hubp = pool->hubps[i];
+ split_pipe->plane_res.ipp = pool->ipps[i];
+ split_pipe->plane_res.dpp = pool->dpps[i];
+ split_pipe->stream_res.opp = pool->opps[i];
+ split_pipe->plane_res.mpcc_inst = pool->dpps[i]->inst;
+ split_pipe->pipe_idx = i;
+
+ split_pipe->stream = stream;
return i;
}
}
@@ -1647,46 +1654,6 @@ static int acquire_first_free_pipe(
return -1;
}
-static struct stream_encoder *find_first_free_match_stream_enc_for_link(
- struct resource_context *res_ctx,
- const struct resource_pool *pool,
- struct dc_stream_state *stream)
-{
- int i;
- int j = -1;
- struct dc_link *link = stream->link;
-
- for (i = 0; i < pool->stream_enc_count; i++) {
- if (!res_ctx->is_stream_enc_acquired[i] &&
- pool->stream_enc[i]) {
- /* Store first available for MST second display
- * in daisy chain use case */
- j = i;
- if (pool->stream_enc[i]->id ==
- link->link_enc->preferred_engine)
- return pool->stream_enc[i];
- }
- }
-
- /*
- * below can happen in cases when stream encoder is acquired:
- * 1) for second MST display in chain, so preferred engine already
- * acquired;
- * 2) for another link, which preferred engine already acquired by any
- * MST configuration.
- *
- * If signal is of DP type and preferred engine not found, return last available
- *
- * TODO - This is just a patch up and a generic solution is
- * required for non DP connectors.
- */
-
- if (j >= 0 && link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT)
- return pool->stream_enc[j];
-
- return NULL;
-}
-
static struct audio *find_first_free_audio(
struct resource_context *res_ctx,
const struct resource_pool *pool,
@@ -1702,6 +1669,12 @@ static struct audio *find_first_free_audio(
return pool->audios[i];
}
}
+
+ /* use engine id to find free audio */
+ if ((id < pool->audio_count) && (res_ctx->is_audio_acquired[id] == false)) {
+ return pool->audios[id];
+ }
+
/*not found the matching one, first come first serve*/
for (i = 0; i < pool->audio_count; i++) {
if (res_ctx->is_audio_acquired[i] == false) {
@@ -1866,6 +1839,7 @@ static int get_norm_pix_clk(const struct dc_crtc_timing *timing)
pix_clk /= 2;
if (timing->pixel_encoding != PIXEL_ENCODING_YCBCR422) {
switch (timing->display_color_depth) {
+ case COLOR_DEPTH_666:
case COLOR_DEPTH_888:
normalized_pix_clk = pix_clk;
break;
@@ -1998,7 +1972,7 @@ enum dc_status resource_map_pool_resources(
pipe_ctx = &context->res_ctx.pipe_ctx[pipe_idx];
pipe_ctx->stream_res.stream_enc =
- find_first_free_match_stream_enc_for_link(
+ dc->res_pool->funcs->find_first_free_match_stream_enc_for_link(
&context->res_ctx, pool, stream);
if (!pipe_ctx->stream_res.stream_enc)
@@ -2012,7 +1986,7 @@ enum dc_status resource_map_pool_resources(
/* TODO: Add check if ASIC support and EDID audio */
if (!stream->converter_disable_audio &&
dc_is_audio_capable_signal(pipe_ctx->stream->signal) &&
- stream->audio_info.mode_count) {
+ stream->audio_info.mode_count && stream->audio_info.flags.all) {
pipe_ctx->stream_res.audio = find_first_free_audio(
&context->res_ctx, pool, pipe_ctx->stream_res.stream_enc->id);
@@ -2034,6 +2008,9 @@ enum dc_status resource_map_pool_resources(
if (context->streams[i] == stream) {
context->stream_status[i].primary_otg_inst = pipe_ctx->stream_res.tg->inst;
context->stream_status[i].stream_enc_inst = pipe_ctx->stream_res.stream_enc->id;
+ context->stream_status[i].audio_inst =
+ pipe_ctx->stream_res.audio ? pipe_ctx->stream_res.audio->inst : -1;
+
return DC_OK;
}
@@ -2059,7 +2036,7 @@ void dc_resource_state_construct(
const struct dc *dc,
struct dc_state *dst_ctx)
{
- dst_ctx->clk_mgr = dc->res_pool->clk_mgr;
+ dst_ctx->clk_mgr = dc->clk_mgr;
}
/**
@@ -2354,7 +2331,18 @@ static void set_avi_info_frame(
break;
}
}
+ /* If VIC >= 128, the Source shall use AVI InfoFrame Version 3*/
hdmi_info.bits.VIC0_VIC7 = vic;
+ if (vic >= 128)
+ hdmi_info.bits.header.version = 3;
+ /* If (C1, C0)=(1, 1) and (EC2, EC1, EC0)=(1, 1, 1),
+ * the Source shall use 20 AVI InfoFrame Version 4
+ */
+ if (hdmi_info.bits.C0_C1 == COLORIMETRY_EXTENDED &&
+ hdmi_info.bits.EC0_EC2 == COLORIMETRYEX_RESERVED) {
+ hdmi_info.bits.header.version = 4;
+ hdmi_info.bits.header.length = 14;
+ }
/* pixel repetition
* PR0 - PR3 start from 0 whereas pHwPathMode->mode.timing.flags.pixel
@@ -2373,12 +2361,19 @@ static void set_avi_info_frame(
hdmi_info.bits.bar_right = (stream->timing.h_total
- stream->timing.h_border_right + 1);
+ /* Additional Colorimetry Extension
+ * Used in conduction with C0-C1 and EC0-EC2
+ * 0 = DCI-P3 RGB (D65)
+ * 1 = DCI-P3 RGB (theater)
+ */
+ hdmi_info.bits.ACE0_ACE3 = 0;
+
/* check_sum - Calculate AFMT_AVI_INFO0 ~ AFMT_AVI_INFO3 */
check_sum = &hdmi_info.packet_raw_data.sb[0];
- *check_sum = HDMI_INFOFRAME_TYPE_AVI + HDMI_AVI_INFOFRAME_SIZE + 2;
+ *check_sum = HDMI_INFOFRAME_TYPE_AVI + hdmi_info.bits.header.length + hdmi_info.bits.header.version;
- for (byte_index = 1; byte_index <= HDMI_AVI_INFOFRAME_SIZE; byte_index++)
+ for (byte_index = 1; byte_index <= hdmi_info.bits.header.length; byte_index++)
*check_sum += hdmi_info.packet_raw_data.sb[byte_index];
/* one byte complement */
@@ -2425,21 +2420,6 @@ static void set_spd_info_packet(
*info_packet = stream->vrr_infopacket;
}
-static void set_dp_sdp_info_packet(
- struct dc_info_packet *info_packet,
- struct dc_stream_state *stream)
-{
- /* SPD info packet for custom sdp message */
-
- /* Return if false. If true,
- * set the corresponding bit in the info packet
- */
- if (!stream->dpsdp_infopacket.valid)
- return;
-
- *info_packet = stream->dpsdp_infopacket;
-}
-
static void set_hdr_static_info_packet(
struct dc_info_packet *info_packet,
struct dc_stream_state *stream)
@@ -2495,7 +2475,6 @@ void dc_resource_state_copy_construct(
if (cur_pipe->bottom_pipe)
cur_pipe->bottom_pipe = &dst_ctx->res_ctx.pipe_ctx[cur_pipe->bottom_pipe->pipe_idx];
-
}
for (i = 0; i < dst_ctx->stream_count; i++) {
@@ -2536,7 +2515,6 @@ void resource_build_info_frame(struct pipe_ctx *pipe_ctx)
info->spd.valid = false;
info->hdrsmd.valid = false;
info->vsc.valid = false;
- info->dpsdp.valid = false;
signal = pipe_ctx->stream->signal;
@@ -2556,8 +2534,6 @@ void resource_build_info_frame(struct pipe_ctx *pipe_ctx)
set_spd_info_packet(&info->spd, pipe_ctx->stream);
set_hdr_static_info_packet(&info->hdrsmd, pipe_ctx->stream);
-
- set_dp_sdp_info_packet(&info->dpsdp, pipe_ctx->stream);
}
patch_gamut_packet_checksum(&info->gamut);
@@ -2644,6 +2620,10 @@ bool pipe_need_reprogram(
if (is_vsc_info_packet_changed(pipe_ctx_old->stream, pipe_ctx->stream))
return true;
+ if (false == pipe_ctx_old->stream->link->link_state_valid &&
+ false == pipe_ctx_old->stream->dpms_off)
+ return true;
+
return false;
}
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_sink.c b/drivers/gpu/drm/amd/display/dc/core/dc_sink.c
index 9971b515c3eb..5cbfdf1c4b11 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_sink.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_sink.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "dm_helpers.h"
#include "core_types.h"
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
index 96e97d25d639..352862370390 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
@@ -23,6 +23,9 @@
*
*/
+#include <linux/delay.h>
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "dc.h"
#include "core_types.h"
@@ -47,8 +50,8 @@ void update_stream_signal(struct dc_stream_state *stream, struct dc_sink *sink)
if (dc_is_dvi_signal(stream->signal)) {
if (stream->ctx->dc->caps.dual_link_dvi &&
- (stream->timing.pix_clk_100hz / 10) > TMDS_MAX_PIXEL_CLOCK &&
- sink->sink_signal != SIGNAL_TYPE_DVI_SINGLE_LINK)
+ (stream->timing.pix_clk_100hz / 10) > TMDS_MAX_PIXEL_CLOCK &&
+ sink->sink_signal != SIGNAL_TYPE_DVI_SINGLE_LINK)
stream->signal = SIGNAL_TYPE_DVI_DUAL_LINK;
else
stream->signal = SIGNAL_TYPE_DVI_SINGLE_LINK;
@@ -105,6 +108,17 @@ static void construct(struct dc_stream_state *stream,
/* EDID CAP translation for HDMI 2.0 */
stream->timing.flags.LTE_340MCSC_SCRAMBLE = dc_sink_data->edid_caps.lte_340mcsc_scramble;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ memset(&stream->timing.dsc_cfg, 0, sizeof(stream->timing.dsc_cfg));
+ stream->timing.dsc_cfg.num_slices_h = 0;
+ stream->timing.dsc_cfg.num_slices_v = 0;
+ stream->timing.dsc_cfg.bits_per_pixel = 128;
+ stream->timing.dsc_cfg.block_pred_enable = 1;
+ stream->timing.dsc_cfg.linebuf_depth = 9;
+ stream->timing.dsc_cfg.version_minor = 2;
+ stream->timing.dsc_cfg.ycbcr422_simple = 0;
+#endif
+
update_stream_signal(stream, dc_sink_data);
stream->out_transfer_func = dc_create_transfer_func();
@@ -167,18 +181,19 @@ struct dc_stream_state *dc_copy_stream(const struct dc_stream_state *stream)
{
struct dc_stream_state *new_stream;
- new_stream = kzalloc(sizeof(struct dc_stream_state), GFP_KERNEL);
+ new_stream = kmemdup(stream, sizeof(struct dc_stream_state), GFP_KERNEL);
if (!new_stream)
return NULL;
- memcpy(new_stream, stream, sizeof(struct dc_stream_state));
-
if (new_stream->sink)
dc_sink_retain(new_stream->sink);
if (new_stream->out_transfer_func)
dc_transfer_func_retain(new_stream->out_transfer_func);
+ new_stream->stream_id = new_stream->ctx->dc_stream_id_count;
+ new_stream->ctx->dc_stream_id_count++;
+
kref_init(&new_stream->refcount);
return new_stream;
@@ -229,7 +244,7 @@ static void delay_cursor_until_vupdate(struct pipe_ctx *pipe_ctx, struct dc *dc)
unsigned int us_per_line;
if (stream->ctx->asic_id.chip_family == FAMILY_RV &&
- ASIC_REV_IS_RAVEN(stream->ctx->asic_id.hw_internal_rev)) {
+ ASICREV_IS_RAVEN(stream->ctx->asic_id.hw_internal_rev)) {
vupdate_line = get_vupdate_offset_from_vsync(pipe_ctx);
if (!dc_stream_get_crtc_position(dc, &stream, 1, &vpos, &nvpos))
@@ -352,53 +367,138 @@ bool dc_stream_set_cursor_position(
return true;
}
-uint32_t dc_stream_get_vblank_counter(const struct dc_stream_state *stream)
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+bool dc_stream_add_writeback(struct dc *dc,
+ struct dc_stream_state *stream,
+ struct dc_writeback_info *wb_info)
{
- uint8_t i;
- struct dc *core_dc = stream->ctx->dc;
- struct resource_context *res_ctx =
- &core_dc->current_state->res_ctx;
+ bool isDrc = false;
+ int i = 0;
+ struct dwbc *dwb;
- for (i = 0; i < MAX_PIPES; i++) {
- struct timing_generator *tg = res_ctx->pipe_ctx[i].stream_res.tg;
+ if (stream == NULL) {
+ dm_error("DC: dc_stream is NULL!\n");
+ return false;
+ }
- if (res_ctx->pipe_ctx[i].stream != stream)
- continue;
+ if (wb_info == NULL) {
+ dm_error("DC: dc_writeback_info is NULL!\n");
+ return false;
+ }
- return tg->funcs->get_frame_count(tg);
+ if (wb_info->dwb_pipe_inst >= MAX_DWB_PIPES) {
+ dm_error("DC: writeback pipe is invalid!\n");
+ return false;
}
- return 0;
+ wb_info->dwb_params.out_transfer_func = stream->out_transfer_func;
+
+ dwb = dc->res_pool->dwbc[wb_info->dwb_pipe_inst];
+ dwb->dwb_is_drc = false;
+
+ /* recalculate and apply DML parameters */
+
+ for (i = 0; i < stream->num_wb_info; i++) {
+ /*dynamic update*/
+ if (stream->writeback_info[i].wb_enabled &&
+ stream->writeback_info[i].dwb_pipe_inst == wb_info->dwb_pipe_inst) {
+ stream->writeback_info[i] = *wb_info;
+ isDrc = true;
+ }
+ }
+
+ if (!isDrc) {
+ stream->writeback_info[stream->num_wb_info++] = *wb_info;
+ }
+
+ if (!dc->hwss.update_bandwidth(dc, dc->current_state)) {
+ dm_error("DC: update_bandwidth failed!\n");
+ return false;
+ }
+
+ /* enable writeback */
+ if (dc->hwss.enable_writeback) {
+ struct dc_stream_status *stream_status = dc_stream_get_status(stream);
+ struct dwbc *dwb = dc->res_pool->dwbc[wb_info->dwb_pipe_inst];
+
+ if (dwb->funcs->is_enabled(dwb)) {
+ /* writeback pipe already enabled, only need to update */
+ dc->hwss.update_writeback(dc, stream_status, wb_info);
+ } else {
+ /* Enable writeback pipe from scratch*/
+ dc->hwss.enable_writeback(dc, stream_status, wb_info);
+ }
+ }
+
+ return true;
}
-static void build_dp_sdp_info_frame(struct pipe_ctx *pipe_ctx,
- const uint8_t *custom_sdp_message,
- unsigned int sdp_message_size)
+bool dc_stream_remove_writeback(struct dc *dc,
+ struct dc_stream_state *stream,
+ uint32_t dwb_pipe_inst)
{
- uint8_t i;
- struct encoder_info_frame *info = &pipe_ctx->stream_res.encoder_info_frame;
+ int i = 0, j = 0;
+ if (stream == NULL) {
+ dm_error("DC: dc_stream is NULL!\n");
+ return false;
+ }
- /* set valid info */
- info->dpsdp.valid = true;
+ if (dwb_pipe_inst >= MAX_DWB_PIPES) {
+ dm_error("DC: writeback pipe is invalid!\n");
+ return false;
+ }
- /* set sdp message header */
- info->dpsdp.hb0 = custom_sdp_message[0]; /* package id */
- info->dpsdp.hb1 = custom_sdp_message[1]; /* package type */
- info->dpsdp.hb2 = custom_sdp_message[2]; /* package specific byte 0 any data */
- info->dpsdp.hb3 = custom_sdp_message[3]; /* package specific byte 0 any data */
+// stream->writeback_info[dwb_pipe_inst].wb_enabled = false;
+ for (i = 0; i < stream->num_wb_info; i++) {
+ /*dynamic update*/
+ if (stream->writeback_info[i].wb_enabled &&
+ stream->writeback_info[i].dwb_pipe_inst == dwb_pipe_inst) {
+ stream->writeback_info[i].wb_enabled = false;
+ }
+ }
- /* set sdp message data */
- for (i = 0; i < 32; i++)
- info->dpsdp.sb[i] = (custom_sdp_message[i+4]);
+ /* remove writeback info for disabled writeback pipes from stream */
+ for (i = 0, j = 0; i < stream->num_wb_info; i++) {
+ if (stream->writeback_info[i].wb_enabled) {
+ if (i != j)
+ /* trim the array */
+ stream->writeback_info[j] = stream->writeback_info[i];
+ j++;
+ }
+ }
+ stream->num_wb_info = j;
+ /* recalculate and apply DML parameters */
+ if (!dc->hwss.update_bandwidth(dc, dc->current_state)) {
+ dm_error("DC: update_bandwidth failed!\n");
+ return false;
+ }
+
+ /* disable writeback */
+ if (dc->hwss.disable_writeback)
+ dc->hwss.disable_writeback(dc, dwb_pipe_inst);
+
+ return true;
}
+#endif
-static void invalid_dp_sdp_info_frame(struct pipe_ctx *pipe_ctx)
+uint32_t dc_stream_get_vblank_counter(const struct dc_stream_state *stream)
{
- struct encoder_info_frame *info = &pipe_ctx->stream_res.encoder_info_frame;
+ uint8_t i;
+ struct dc *core_dc = stream->ctx->dc;
+ struct resource_context *res_ctx =
+ &core_dc->current_state->res_ctx;
+
+ for (i = 0; i < MAX_PIPES; i++) {
+ struct timing_generator *tg = res_ctx->pipe_ctx[i].stream_res.tg;
+
+ if (res_ctx->pipe_ctx[i].stream != stream)
+ continue;
- /* in-valid info */
- info->dpsdp.valid = false;
+ return tg->funcs->get_frame_count(tg);
+ }
+
+ return 0;
}
bool dc_stream_send_dp_sdp(const struct dc_stream_state *stream,
@@ -406,7 +506,7 @@ bool dc_stream_send_dp_sdp(const struct dc_stream_state *stream,
unsigned int sdp_message_size)
{
int i;
- struct dc *core_dc;
+ struct dc *dc;
struct resource_context *res_ctx;
if (stream == NULL) {
@@ -414,8 +514,8 @@ bool dc_stream_send_dp_sdp(const struct dc_stream_state *stream,
return false;
}
- core_dc = stream->ctx->dc;
- res_ctx = &core_dc->current_state->res_ctx;
+ dc = stream->ctx->dc;
+ res_ctx = &dc->current_state->res_ctx;
for (i = 0; i < MAX_PIPES; i++) {
struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
@@ -423,11 +523,14 @@ bool dc_stream_send_dp_sdp(const struct dc_stream_state *stream,
if (pipe_ctx->stream != stream)
continue;
- build_dp_sdp_info_frame(pipe_ctx, custom_sdp_message, sdp_message_size);
-
- core_dc->hwss.update_info_frame(pipe_ctx);
+ if (dc->hwss.send_immediate_sdp_message != NULL)
+ dc->hwss.send_immediate_sdp_message(pipe_ctx,
+ custom_sdp_message,
+ sdp_message_size);
+ else
+ DC_LOG_WARNING("%s:send_immediate_sdp_message not implemented on this ASIC\n",
+ __func__);
- invalid_dp_sdp_info_frame(pipe_ctx);
}
return true;
@@ -463,6 +566,78 @@ bool dc_stream_get_scanoutpos(const struct dc_stream_state *stream,
return ret;
}
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+bool dc_stream_dmdata_status_done(struct dc *dc, struct dc_stream_state *stream)
+{
+ bool status = true;
+ struct pipe_ctx *pipe = NULL;
+ int i;
+
+ if (!dc->hwss.dmdata_status_done)
+ return false;
+
+ for (i = 0; i < MAX_PIPES; i++) {
+ pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+ if (pipe->stream == stream)
+ break;
+ }
+ /* Stream not found, by default we'll assume HUBP fetched dm data */
+ if (i == MAX_PIPES)
+ return true;
+
+ status = dc->hwss.dmdata_status_done(pipe);
+ return status;
+}
+
+bool dc_stream_set_dynamic_metadata(struct dc *dc,
+ struct dc_stream_state *stream,
+ struct dc_dmdata_attributes *attr)
+{
+ struct pipe_ctx *pipe_ctx = NULL;
+ struct hubp *hubp;
+ int i;
+
+ for (i = 0; i < MAX_PIPES; i++) {
+ pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
+ if (pipe_ctx->stream == stream)
+ break;
+ }
+
+ if (i == MAX_PIPES)
+ return false;
+
+ hubp = pipe_ctx->plane_res.hubp;
+ if (hubp == NULL)
+ return false;
+
+ pipe_ctx->stream->dmdata_address = attr->address;
+
+ if (pipe_ctx->stream_res.stream_enc &&
+ pipe_ctx->stream_res.stream_enc->funcs->set_dynamic_metadata != NULL) {
+ if (pipe_ctx->stream->dmdata_address.quad_part != 0) {
+ /* if using dynamic meta, don't set up generic infopackets */
+ pipe_ctx->stream_res.encoder_info_frame.hdrsmd.valid = false;
+ pipe_ctx->stream_res.stream_enc->funcs->set_dynamic_metadata(
+ pipe_ctx->stream_res.stream_enc,
+ true, pipe_ctx->plane_res.hubp->inst,
+ dc_is_dp_signal(pipe_ctx->stream->signal) ?
+ dmdata_dp : dmdata_hdmi);
+ } else
+ pipe_ctx->stream_res.stream_enc->funcs->set_dynamic_metadata(
+ pipe_ctx->stream_res.stream_enc,
+ false, pipe_ctx->plane_res.hubp->inst,
+ dc_is_dp_signal(pipe_ctx->stream->signal) ?
+ dmdata_dp : dmdata_hdmi);
+ }
+
+ if (hubp->funcs->dmdata_set_attributes != NULL &&
+ pipe_ctx->stream->dmdata_address.quad_part != 0) {
+ hubp->funcs->dmdata_set_attributes(hubp, attr);
+ }
+
+ return true;
+}
+#endif
void dc_stream_log(const struct dc *dc, const struct dc_stream_state *stream)
{
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
index a5e86f9b148f..f40e4fd52fa2 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/mm.h>
+
/* DC interface (public) */
#include "dm_services.h"
#include "dc.h"
@@ -48,6 +50,25 @@ static void construct(struct dc_context *ctx, struct dc_plane_state *plane_state
plane_state->in_transfer_func->type = TF_TYPE_BYPASS;
plane_state->in_transfer_func->ctx = ctx;
}
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ plane_state->in_shaper_func = dc_create_transfer_func();
+ if (plane_state->in_shaper_func != NULL) {
+ plane_state->in_shaper_func->type = TF_TYPE_BYPASS;
+ plane_state->in_shaper_func->ctx = ctx;
+ }
+
+ plane_state->lut3d_func = dc_create_3dlut_func();
+ if (plane_state->lut3d_func != NULL) {
+ plane_state->lut3d_func->ctx = ctx;
+ plane_state->lut3d_func->initialized = false;
+ }
+ plane_state->blend_tf = dc_create_transfer_func();
+ if (plane_state->blend_tf != NULL) {
+ plane_state->blend_tf->type = TF_TYPE_BYPASS;
+ plane_state->blend_tf->ctx = ctx;
+ }
+
+#endif
}
static void destruct(struct dc_plane_state *plane_state)
@@ -60,6 +81,24 @@ static void destruct(struct dc_plane_state *plane_state)
plane_state->in_transfer_func);
plane_state->in_transfer_func = NULL;
}
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ if (plane_state->in_shaper_func != NULL) {
+ dc_transfer_func_release(
+ plane_state->in_shaper_func);
+ plane_state->in_shaper_func = NULL;
+ }
+ if (plane_state->lut3d_func != NULL) {
+ dc_3dlut_func_release(
+ plane_state->lut3d_func);
+ plane_state->lut3d_func = NULL;
+ }
+ if (plane_state->blend_tf != NULL) {
+ dc_transfer_func_release(
+ plane_state->blend_tf);
+ plane_state->blend_tf = NULL;
+ }
+
+#endif
}
/*******************************************************************************
@@ -224,4 +263,40 @@ alloc_fail:
return NULL;
}
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+static void dc_3dlut_func_free(struct kref *kref)
+{
+ struct dc_3dlut *lut = container_of(kref, struct dc_3dlut, refcount);
+
+ kvfree(lut);
+}
+
+struct dc_3dlut *dc_create_3dlut_func(void)
+{
+ struct dc_3dlut *lut = kvzalloc(sizeof(*lut), GFP_KERNEL);
+
+ if (lut == NULL)
+ goto alloc_fail;
+
+ kref_init(&lut->refcount);
+ lut->initialized = false;
+
+ return lut;
+
+alloc_fail:
+ return NULL;
+
+}
+
+void dc_3dlut_func_release(struct dc_3dlut *lut)
+{
+ kref_put(&lut->refcount, dc_3dlut_func_free);
+}
+
+void dc_3dlut_func_retain(struct dc_3dlut *lut)
+{
+ kref_get(&lut->refcount);
+}
+#endif
+
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c b/drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c
index 6ce87b682a32..a96d8de9380e 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c
@@ -24,8 +24,9 @@
*/
#include "vm_helper.h"
+#include "dc.h"
-static void mark_vmid_used(struct vm_helper *vm_helper, unsigned int pos, uint8_t hubp_idx)
+void vm_helper_mark_vmid_used(struct vm_helper *vm_helper, unsigned int pos, uint8_t hubp_idx)
{
struct vmid_usage vmids = vm_helper->hubp_vmid_usage[hubp_idx];
@@ -33,91 +34,43 @@ static void mark_vmid_used(struct vm_helper *vm_helper, unsigned int pos, uint8_
vmids.vmid_usage[1] = 1 << pos;
}
-static void add_ptb_to_table(struct vm_helper *vm_helper, unsigned int vmid, uint64_t ptb)
+int dc_setup_system_context(struct dc *dc, struct dc_phy_addr_space_config *pa_config)
{
- vm_helper->ptb_assigned_to_vmid[vmid] = ptb;
- vm_helper->num_vmids_available--;
-}
-
-static void clear_entry_from_vmid_table(struct vm_helper *vm_helper, unsigned int vmid)
-{
- vm_helper->ptb_assigned_to_vmid[vmid] = 0;
- vm_helper->num_vmids_available++;
-}
-
-static void evict_vmids(struct vm_helper *vm_helper)
-{
- int i;
- uint16_t ord = 0;
+ int num_vmids = 0;
- for (i = 0; i < vm_helper->num_vmid; i++)
- ord |= vm_helper->hubp_vmid_usage[i].vmid_usage[0] | vm_helper->hubp_vmid_usage[i].vmid_usage[1];
+ /* Call HWSS to setup HUBBUB for address config */
+ if (dc->hwss.init_sys_ctx) {
+ num_vmids = dc->hwss.init_sys_ctx(dc->hwseq, dc, pa_config);
- // At this point any positions with value 0 are unused vmids, evict them
- for (i = 1; i < vm_helper->num_vmid; i++) {
- if (ord & (1u << i))
- clear_entry_from_vmid_table(vm_helper, i);
+ /* Pre-init system aperture start/end for all HUBP instances (if not gating?)
+ * or cache system aperture if using power gating
+ */
+ memcpy(&dc->vm_pa_config, pa_config, sizeof(struct dc_phy_addr_space_config));
+ dc->vm_pa_config.valid = true;
}
-}
-
-// Return value of -1 indicates vmid table unitialized or ptb dne in the table
-static int get_existing_vmid_for_ptb(struct vm_helper *vm_helper, uint64_t ptb)
-{
- int i;
- for (i = 0; i < vm_helper->num_vmid; i++) {
- if (vm_helper->ptb_assigned_to_vmid[i] == ptb)
- return i;
- }
-
- return -1;
+ return num_vmids;
}
-// Expected to be called only when there's an available vmid
-static int get_next_available_vmid(struct vm_helper *vm_helper)
+void dc_setup_vm_context(struct dc *dc, struct dc_virtual_addr_space_config *va_config, int vmid)
{
- int i;
-
- for (i = 1; i < vm_helper->num_vmid; i++) {
- if (vm_helper->ptb_assigned_to_vmid[i] == 0)
- return i;
- }
-
- return -1;
+ dc->hwss.init_vm_ctx(dc->hwseq, dc, va_config, vmid);
}
-uint8_t get_vmid_for_ptb(struct vm_helper *vm_helper, int64_t ptb, uint8_t hubp_idx)
+int dc_get_vmid_use_vector(struct dc *dc)
{
- unsigned int vmid = 0;
- int vmid_exists = -1;
-
- // Physical address gets vmid 0
- if (ptb == 0)
- return 0;
-
- vmid_exists = get_existing_vmid_for_ptb(vm_helper, ptb);
-
- if (vmid_exists != -1) {
- mark_vmid_used(vm_helper, vmid_exists, hubp_idx);
- vmid = vmid_exists;
- } else {
- if (vm_helper->num_vmids_available == 0)
- evict_vmids(vm_helper);
-
- vmid = get_next_available_vmid(vm_helper);
- mark_vmid_used(vm_helper, vmid, hubp_idx);
- add_ptb_to_table(vm_helper, vmid, ptb);
- }
+ int i;
+ int in_use = 0;
- return vmid;
+ for (i = 0; i < dc->vm_helper->num_vmid; i++)
+ in_use |= dc->vm_helper->hubp_vmid_usage[i].vmid_usage[0]
+ | dc->vm_helper->hubp_vmid_usage[i].vmid_usage[1];
+ return in_use;
}
-void init_vm_helper(struct vm_helper *vm_helper, unsigned int num_vmid, unsigned int num_hubp)
+void vm_helper_init(struct vm_helper *vm_helper, unsigned int num_vmid)
{
vm_helper->num_vmid = num_vmid;
- vm_helper->num_hubp = num_hubp;
- vm_helper->num_vmids_available = num_vmid - 1;
memset(vm_helper->hubp_vmid_usage, 0, sizeof(vm_helper->hubp_vmid_usage[0]) * MAX_HUBP);
- memset(vm_helper->ptb_assigned_to_vmid, 0, sizeof(vm_helper->ptb_assigned_to_vmid[0]) * MAX_VMID);
}
diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h
index 44e4b0465587..e513028faefa 100644
--- a/drivers/gpu/drm/amd/display/dc/dc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc.h
@@ -39,7 +39,7 @@
#include "inc/hw/dmcu.h"
#include "dml/display_mode_lib.h"
-#define DC_VER "3.2.27"
+#define DC_VER "3.2.35"
#define MAX_SURFACES 3
#define MAX_PLANES 6
@@ -70,6 +70,8 @@ struct dc_plane_cap {
uint32_t argb8888 : 1;
uint32_t nv12 : 1;
uint32_t fp16 : 1;
+ uint32_t p010 : 1;
+ uint32_t ayuv : 1;
} pixel_format_support;
// max upscaling factor x1000
// upscaling factors are always >= 1
@@ -109,9 +111,19 @@ struct dc_caps {
bool force_dp_tps4_for_cp2520;
bool disable_dp_clk_share;
bool psp_setup_panel_mode;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ bool hw_3d_lut;
+#endif
struct dc_plane_cap planes[MAX_PLANES];
};
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+struct dc_bug_wa {
+ bool no_connect_phy_config;
+ bool dedcn20_305_wa;
+};
+#endif
+
struct dc_dcc_surface_param {
struct dc_size surface_size;
enum surface_pixel_format format;
@@ -205,6 +217,9 @@ struct dc_config {
bool disable_fractional_pwm;
bool allow_seamless_boot_optimization;
bool power_down_display_on_boot;
+ bool edp_not_connected;
+ bool forced_clocks;
+
};
enum visual_confirm {
@@ -320,6 +335,9 @@ struct dc_debug_options {
bool disable_dfs_bypass;
bool disable_dpp_power_gate;
bool disable_hubp_power_gate;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ bool disable_dsc_power_gate;
+#endif
bool disable_pplib_wm_range;
enum wm_report_mode pplib_wm_report_mode;
unsigned int min_disp_clk_khz;
@@ -328,6 +346,7 @@ struct dc_debug_options {
int sr_exit_time_ns;
int sr_enter_plus_exit_time_ns;
int urgent_latency_ns;
+ uint32_t underflow_assert_delay_us;
int percent_of_ideal_drambw;
int dram_clock_change_latency_ns;
bool optimized_watermark;
@@ -352,6 +371,13 @@ struct dc_debug_options {
unsigned int force_fclk_khz;
bool disable_tri_buf;
struct dc_bw_validation_profile bw_val_profile;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ bool disable_fec;
+#endif
+ /* This forces a hard min on the DCFCLK requested to SMU/PP
+ * watermarks are not affected.
+ */
+ unsigned int force_min_dcfclk_mhz;
};
struct dc_debug_data {
@@ -360,17 +386,54 @@ struct dc_debug_data {
uint32_t auxErrorCount;
};
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+struct dc_phy_addr_space_config {
+ struct {
+ uint64_t start_addr;
+ uint64_t end_addr;
+ uint64_t fb_top;
+ uint64_t fb_offset;
+ uint64_t fb_base;
+ uint64_t agp_top;
+ uint64_t agp_bot;
+ uint64_t agp_base;
+ } system_aperture;
+
+ struct {
+ uint64_t page_table_start_addr;
+ uint64_t page_table_end_addr;
+ uint64_t page_table_base_addr;
+ } gart_config;
+
+ bool valid;
+};
+
+struct dc_virtual_addr_space_config {
+ uint64_t page_table_base_addr;
+ uint64_t page_table_start_addr;
+ uint64_t page_table_end_addr;
+ uint32_t page_table_block_size_in_bytes;
+ uint8_t page_table_depth; // 1 = 1 level, 2 = 2 level, etc. 0 = invalid
+};
+#endif
+
struct dc_bounding_box_overrides {
int sr_exit_time_ns;
int sr_enter_plus_exit_time_ns;
int urgent_latency_ns;
int percent_of_ideal_drambw;
int dram_clock_change_latency_ns;
+ /* This forces a hard min on the DCFCLK we use
+ * for DML. Unlike the debug option for forcing
+ * DCFCLK, this override affects watermark calculations
+ */
+ int min_dcfclk_mhz;
};
struct dc_state;
struct resource_pool;
struct dce_hwseq;
+struct gpu_info_soc_bounding_box_v1_0;
struct dc {
struct dc_versions versions;
struct dc_caps caps;
@@ -378,7 +441,13 @@ struct dc {
struct dc_config config;
struct dc_debug_options debug;
struct dc_bounding_box_overrides bb_overrides;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ struct dc_bug_wa work_arounds;
+#endif
struct dc_context *ctx;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ struct dc_phy_addr_space_config vm_pa_config;
+#endif
uint8_t link_count;
struct dc_link *links[MAX_PIPES * 2];
@@ -386,6 +455,8 @@ struct dc {
struct dc_state *current_state;
struct resource_pool *res_pool;
+ struct clk_mgr *clk_mgr;
+
/* Display Engine Clock levels */
struct dm_pp_clock_levels sclk_lvls;
@@ -414,6 +485,10 @@ struct dc {
struct dc_debug_data debug_data;
const char *build_id;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ struct vm_helper *vm_helper;
+ const struct gpu_info_soc_bounding_box_v1_0 *soc_bounding_box;
+#endif
};
enum frame_buffer_mode {
@@ -447,6 +522,13 @@ struct dc_init_data {
struct dc_config flags;
uint32_t log_mask;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ /**
+ * gpu_info FW provided soc bounding box struct or 0 if not
+ * available in FW
+ */
+ const struct gpu_info_soc_bounding_box_v1_0 *soc_bounding_box;
+#endif
};
struct dc_callback_init {
@@ -454,6 +536,12 @@ struct dc_callback_init {
};
struct dc *dc_create(const struct dc_init_data *init_params);
+int dc_get_vmid_use_vector(struct dc *dc);
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+void dc_setup_vm_context(struct dc *dc, struct dc_virtual_addr_space_config *va_config, int vmid);
+/* Returns the number of vmids supported */
+int dc_setup_system_context(struct dc *dc, struct dc_phy_addr_space_config *pa_config);
+#endif
void dc_init_callbacks(struct dc *dc,
const struct dc_callback_init *init_params);
void dc_destroy(struct dc **dc);
@@ -525,6 +613,17 @@ struct dc_transfer_func {
};
};
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+
+
+struct dc_3dlut {
+ struct kref refcount;
+ struct tetrahedral_params lut_3d;
+ uint32_t hdr_multiplier;
+ bool initialized;
+ struct dc_context *ctx;
+};
+#endif
/*
* This structure is filled in by dc_surface_get_status and contains
* the last requested address and the currently active address so the called
@@ -540,12 +639,14 @@ struct dc_plane_status {
union surface_update_flags {
struct {
+ uint32_t addr_update:1;
/* Medium updates */
uint32_t dcc_change:1;
uint32_t color_space_change:1;
uint32_t horizontal_mirror_change:1;
uint32_t per_pixel_alpha_change:1;
uint32_t global_alpha_change:1;
+ uint32_t sdr_white_level:1;
uint32_t rotation_change:1;
uint32_t swizzle_change:1;
uint32_t scaling_change:1;
@@ -573,6 +674,9 @@ union surface_update_flags {
struct dc_plane_state {
struct dc_plane_address address;
struct dc_plane_flip_time time;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ bool triplebuffer_flips;
+#endif
struct scaling_taps scaling_quality;
struct rect src_rect;
struct rect dst_rect;
@@ -595,6 +699,12 @@ struct dc_plane_state {
enum dc_color_space color_space;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ struct dc_3dlut *lut3d_func;
+ struct dc_transfer_func *in_shaper_func;
+ struct dc_transfer_func *blend_tf;
+#endif
+
enum surface_pixel_format format;
enum dc_rotation_angle rotation;
enum plane_stereo_format stereo_format;
@@ -660,6 +770,11 @@ struct dc_surface_update {
const struct dc_csc_transform *input_csc_color_matrix;
const struct fixed31_32 *coeff_reduction_factor;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ const struct dc_transfer_func *func_shaper;
+ const struct dc_3dlut *lut3d_func;
+ const struct dc_transfer_func *blend_tf;
+#endif
};
/*
@@ -680,6 +795,11 @@ void dc_transfer_func_retain(struct dc_transfer_func *dc_tf);
void dc_transfer_func_release(struct dc_transfer_func *dc_tf);
struct dc_transfer_func *dc_create_transfer_func(void);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+struct dc_3dlut *dc_create_3dlut_func(void);
+void dc_3dlut_func_release(struct dc_3dlut *lut);
+void dc_3dlut_func_retain(struct dc_3dlut *lut);
+#endif
/*
* This structure holds a surface address. There could be multiple addresses
* in cases such as Stereo 3D, Planar YUV, etc. Other per-flip attributes such
@@ -793,6 +913,10 @@ struct dpcd_caps {
bool panel_mode_edp;
bool dpcd_display_control_capable;
bool ext_receiver_cap_field_present;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ union dpcd_fec_capability fec_cap;
+ struct dpcd_dsc_capabilities dsc_caps;
+#endif
};
#include "dc_link.h"
@@ -813,6 +937,14 @@ struct dc_container_id {
};
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+struct dc_sink_dsc_caps {
+ // 'true' if these are virtual DPCD's DSC caps (immediately upstream of sink in MST topology),
+ // 'false' if they are sink's DSC caps
+ bool is_virtual_dpcd_dsc;
+ struct dsc_dec_dpcd_caps dsc_dec_caps;
+};
+#endif
/*
* The sink structure contains EDID and other display device properties
@@ -827,6 +959,10 @@ struct dc_sink {
struct stereo_3d_features features_3d[TIMING_3D_FORMAT_MAX];
bool converter_disable_audio;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ struct dc_sink_dsc_caps sink_dsc_caps;
+#endif
+
/* private to DC core */
struct dc_link *link;
struct dc_context *ctx;
@@ -884,4 +1020,10 @@ unsigned int dc_get_target_backlight_pwm(struct dc *dc);
bool dc_is_dmcu_initialized(struct dc *dc);
+#if defined(CONFIG_DRM_AMD_DC_DSC_SUPPORT)
+/*******************************************************************************
+ * DSC Interfaces
+ ******************************************************************************/
+#include "dc_dsc.h"
+#endif
#endif /* DC_INTERFACE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
index 11c68a399267..dfcec4d3e9c0 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
@@ -512,4 +512,131 @@ union test_misc {
unsigned char raw;
};
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+/* FEC capability DPCD register field bits-*/
+union dpcd_fec_capability {
+ struct {
+ uint8_t FEC_CAPABLE:1;
+ uint8_t UNCORRECTED_BLOCK_ERROR_COUNT_CAPABLE:1;
+ uint8_t CORRECTED_BLOCK_ERROR_COUNT_CAPABLE:1;
+ uint8_t BIT_ERROR_COUNT_CAPABLE:1;
+ uint8_t RESERVED:4;
+ } bits;
+ uint8_t raw;
+};
+
+/* DSC capability DPCD register field bits-*/
+struct dpcd_dsc_support {
+ uint8_t DSC_SUPPORT :1;
+ uint8_t DSC_PASSTHROUGH_SUPPORT :1;
+ uint8_t RESERVED :6;
+};
+
+struct dpcd_dsc_algorithm_revision {
+ uint8_t DSC_VERSION_MAJOR :4;
+ uint8_t DSC_VERSION_MINOR :4;
+};
+
+struct dpcd_dsc_rc_buffer_block_size {
+ uint8_t RC_BLOCK_BUFFER_SIZE :2;
+ uint8_t RESERVED :6;
+};
+
+struct dpcd_dsc_slice_capability1 {
+ uint8_t ONE_SLICE_PER_DP_DSC_SINK_DEVICE :1;
+ uint8_t TWO_SLICES_PER_DP_DSC_SINK_DEVICE :1;
+ uint8_t RESERVED :1;
+ uint8_t FOUR_SLICES_PER_DP_DSC_SINK_DEVICE :1;
+ uint8_t SIX_SLICES_PER_DP_DSC_SINK_DEVICE :1;
+ uint8_t EIGHT_SLICES_PER_DP_DSC_SINK_DEVICE :1;
+ uint8_t TEN_SLICES_PER_DP_DSC_SINK_DEVICE :1;
+ uint8_t TWELVE_SLICES_PER_DP_DSC_SINK_DEVICE :1;
+};
+
+struct dpcd_dsc_line_buffer_bit_depth {
+ uint8_t LINE_BUFFER_BIT_DEPTH :4;
+ uint8_t RESERVED :4;
+};
+
+struct dpcd_dsc_block_prediction_support {
+ uint8_t BLOCK_PREDICTION_SUPPORT:1;
+ uint8_t RESERVED :7;
+};
+
+struct dpcd_maximum_bits_per_pixel_supported_by_the_decompressor {
+ uint8_t MAXIMUM_BITS_PER_PIXEL_SUPPORTED_BY_THE_DECOMPRESSOR_LOW :7;
+ uint8_t MAXIMUM_BITS_PER_PIXEL_SUPPORTED_BY_THE_DECOMPRESSOR_HIGH :7;
+ uint8_t RESERVED :2;
+};
+
+struct dpcd_dsc_decoder_color_format_capabilities {
+ uint8_t RGB_SUPPORT :1;
+ uint8_t Y_CB_CR_444_SUPPORT :1;
+ uint8_t Y_CB_CR_SIMPLE_422_SUPPORT :1;
+ uint8_t Y_CB_CR_NATIVE_422_SUPPORT :1;
+ uint8_t Y_CB_CR_NATIVE_420_SUPPORT :1;
+ uint8_t RESERVED :3;
+};
+
+struct dpcd_dsc_decoder_color_depth_capabilities {
+ uint8_t RESERVED0 :1;
+ uint8_t EIGHT_BITS_PER_COLOR_SUPPORT :1;
+ uint8_t TEN_BITS_PER_COLOR_SUPPORT :1;
+ uint8_t TWELVE_BITS_PER_COLOR_SUPPORT :1;
+ uint8_t RESERVED1 :4;
+};
+
+struct dpcd_peak_dsc_throughput_dsc_sink {
+ uint8_t THROUGHPUT_MODE_0:4;
+ uint8_t THROUGHPUT_MODE_1:4;
+};
+
+struct dpcd_dsc_slice_capabilities_2 {
+ uint8_t SIXTEEN_SLICES_PER_DSC_SINK_DEVICE :1;
+ uint8_t TWENTY_SLICES_PER_DSC_SINK_DEVICE :1;
+ uint8_t TWENTYFOUR_SLICES_PER_DSC_SINK_DEVICE :1;
+ uint8_t RESERVED :5;
+};
+
+struct dpcd_bits_per_pixel_increment{
+ uint8_t INCREMENT_OF_BITS_PER_PIXEL_SUPPORTED :3;
+ uint8_t RESERVED :5;
+};
+union dpcd_dsc_basic_capabilities {
+ struct {
+ struct dpcd_dsc_support dsc_support;
+ struct dpcd_dsc_algorithm_revision dsc_algorithm_revision;
+ struct dpcd_dsc_rc_buffer_block_size dsc_rc_buffer_block_size;
+ uint8_t dsc_rc_buffer_size;
+ struct dpcd_dsc_slice_capability1 dsc_slice_capabilities_1;
+ struct dpcd_dsc_line_buffer_bit_depth dsc_line_buffer_bit_depth;
+ struct dpcd_dsc_block_prediction_support dsc_block_prediction_support;
+ struct dpcd_maximum_bits_per_pixel_supported_by_the_decompressor maximum_bits_per_pixel_supported_by_the_decompressor;
+ struct dpcd_dsc_decoder_color_format_capabilities dsc_decoder_color_format_capabilities;
+ struct dpcd_dsc_decoder_color_depth_capabilities dsc_decoder_color_depth_capabilities;
+ struct dpcd_peak_dsc_throughput_dsc_sink peak_dsc_throughput_dsc_sink;
+ uint8_t dsc_maximum_slice_width;
+ struct dpcd_dsc_slice_capabilities_2 dsc_slice_capabilities_2;
+ uint8_t reserved;
+ struct dpcd_bits_per_pixel_increment bits_per_pixel_increment;
+ } fields;
+ uint8_t raw[16];
+};
+
+union dpcd_dsc_ext_capabilities {
+ struct {
+ uint8_t BRANCH_OVERALL_THROUGHPUT_0;
+ uint8_t BRANCH_OVERALL_THROUGHPUT_1;
+ uint8_t BRANCH_MAX_LINE_WIDTH;
+ } fields;
+ uint8_t raw[3];
+};
+
+struct dpcd_dsc_capabilities {
+ union dpcd_dsc_basic_capabilities dsc_basic_caps;
+ union dpcd_dsc_ext_capabilities dsc_ext_caps;
+};
+
+#endif /* CONFIG_DRM_AMD_DC_DSC_SUPPORT */
+
#endif /* DC_DP_TYPES_H */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dsc.h b/drivers/gpu/drm/amd/display/dc/dc_dsc.h
new file mode 100644
index 000000000000..6e42209f0e20
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dc_dsc.h
@@ -0,0 +1,62 @@
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+#ifndef DC_DSC_H_
+#define DC_DSC_H_
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Author: AMD
+ */
+
+/* put it here temporarily until linux has the new addresses official defined */
+/* DP Extended DSC Capabilities */
+#define DP_DSC_BRANCH_OVERALL_THROUGHPUT_0 0x0a0 /* DP 1.4a SCR */
+#define DP_DSC_BRANCH_OVERALL_THROUGHPUT_1 0x0a1
+#define DP_DSC_BRANCH_MAX_LINE_WIDTH 0x0a2
+
+struct dc_dsc_bw_range {
+ uint32_t min_kbps; /* Bandwidth if min_target_bpp_x16 is used */
+ uint32_t min_target_bpp_x16;
+ uint32_t max_kbps; /* Bandwidth if max_target_bpp_x16 is used */
+ uint32_t max_target_bpp_x16;
+ uint32_t stream_kbps; /* Uncompressed stream bandwidth */
+};
+
+
+bool dc_dsc_parse_dsc_dpcd(const uint8_t *dpcd_dsc_basic_data,
+ const uint8_t *dpcd_dsc_ext_data,
+ struct dsc_dec_dpcd_caps *dsc_sink_caps);
+
+bool dc_dsc_compute_bandwidth_range(
+ const struct dc *dc,
+ const uint32_t min_kbps,
+ const uint32_t max_kbps,
+ const struct dsc_dec_dpcd_caps *dsc_sink_caps,
+ const struct dc_crtc_timing *timing,
+ struct dc_dsc_bw_range *range);
+
+bool dc_dsc_compute_config(
+ const struct dc *dc,
+ const struct dsc_dec_dpcd_caps *dsc_sink_caps,
+ uint32_t target_bandwidth_kbps,
+ const struct dc_crtc_timing *timing,
+ struct dc_dsc_config *dsc_cfg);
+#endif
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dc_helper.c b/drivers/gpu/drm/amd/display/dc/dc_helper.c
index 5e6c5eff49cf..30b2f9edd42f 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_helper.c
@@ -26,6 +26,9 @@
* Created on: Aug 30, 2016
* Author: agrodzov
*/
+
+#include <linux/delay.h>
+
#include "dm_services.h"
#include <stdarg.h>
@@ -297,7 +300,7 @@ void generic_reg_wait(const struct dc_context *ctx,
int i;
/* something is terribly wrong if time out is > 200ms. (5Hz) */
- ASSERT(delay_between_poll_us * time_out_num_tries <= 200000);
+ ASSERT(delay_between_poll_us * time_out_num_tries <= 3000000);
for (i = 0; i <= time_out_num_tries; i++) {
if (i) {
diff --git a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
index da55d623647a..22db5682aa6c 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
@@ -99,6 +99,8 @@ struct dc_plane_address {
};
union large_integer page_table_base;
+
+ uint8_t vmid;
};
struct dc_size {
@@ -194,6 +196,12 @@ enum surface_pixel_format {
/*swaped & float*/
SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F,
/*grow graphics here if necessary */
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ SURFACE_PIXEL_FORMAT_GRPH_RGB111110_FIX,
+ SURFACE_PIXEL_FORMAT_GRPH_BGR101111_FIX,
+ SURFACE_PIXEL_FORMAT_GRPH_RGB111110_FLOAT,
+ SURFACE_PIXEL_FORMAT_GRPH_BGR101111_FLOAT,
+#endif
SURFACE_PIXEL_FORMAT_VIDEO_BEGIN,
SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr =
SURFACE_PIXEL_FORMAT_VIDEO_BEGIN,
@@ -201,6 +209,10 @@ enum surface_pixel_format {
SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCbCr,
SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb,
SURFACE_PIXEL_FORMAT_SUBSAMPLE_END,
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ SURFACE_PIXEL_FORMAT_VIDEO_ACrYCb2101010,
+ SURFACE_PIXEL_FORMAT_VIDEO_CrYCbA1010102,
+#endif
SURFACE_PIXEL_FORMAT_VIDEO_AYCrCb8888,
SURFACE_PIXEL_FORMAT_INVALID
@@ -239,6 +251,13 @@ enum tile_split_values {
DC_ROTATED_MICRO_TILING = 0x3,
};
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+enum tripleBuffer_enable {
+ DC_TRIPLEBUFFER_DISABLE = 0x0,
+ DC_TRIPLEBUFFER_ENABLE = 0x1,
+};
+#endif
+
/* TODO: These values come from hardware spec. We need to readdress this
* if they ever change.
*/
@@ -437,6 +456,14 @@ struct dc_csc_transform {
bool enable_adjustment;
};
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+struct dc_rgb_fixed {
+ struct fixed31_32 red;
+ struct fixed31_32 green;
+ struct fixed31_32 blue;
+};
+#endif
+
struct dc_gamma {
struct kref refcount;
enum dc_gamma_type type;
@@ -451,7 +478,11 @@ struct dc_gamma {
/* private to DC core */
struct dc_context *ctx;
+ /* is_identity is used for RGB256 gamma identity which can also be programmed in INPUT_LUT.
+ * is_logical_identity indicates the given gamma ramp regardless of type is identity.
+ */
bool is_identity;
+ bool is_logical_identity;
};
/* Used by both ipp amd opp functions*/
@@ -466,7 +497,11 @@ enum dc_cursor_color_format {
CURSOR_MODE_MONO,
CURSOR_MODE_COLOR_1BIT_AND,
CURSOR_MODE_COLOR_PRE_MULTIPLIED_ALPHA,
- CURSOR_MODE_COLOR_UN_PRE_MULTIPLIED_ALPHA
+ CURSOR_MODE_COLOR_UN_PRE_MULTIPLIED_ALPHA,
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ CURSOR_MODE_COLOR_64BIT_FP_PRE_MULTIPLIED,
+ CURSOR_MODE_COLOR_64BIT_FP_UN_PRE_MULTIPLIED
+#endif
};
/*
@@ -534,6 +569,7 @@ enum dc_color_space {
COLOR_SPACE_DOLBYVISION,
COLOR_SPACE_APPCTRL,
COLOR_SPACE_CUSTOMPOINTS,
+ COLOR_SPACE_YCBCR709_BLACK,
};
enum dc_dither_option {
@@ -611,6 +647,10 @@ enum dc_color_depth {
COLOR_DEPTH_121212,
COLOR_DEPTH_141414,
COLOR_DEPTH_161616,
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ COLOR_DEPTH_999,
+ COLOR_DEPTH_111111,
+#endif
COLOR_DEPTH_COUNT
};
@@ -671,6 +711,9 @@ struct dc_crtc_timing_flags {
* rates less than or equal to 340Mcsc */
uint32_t LTE_340MCSC_SCRAMBLE:1;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ uint32_t DSC : 1; /* Use DSC with this timing */
+#endif
};
enum dc_timing_3d_format {
@@ -717,6 +760,18 @@ struct dc_crtc_timing_adjust {
uint32_t v_total_max;
};
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+struct dc_dsc_config {
+ uint32_t num_slices_h; /* Number of DSC slices - horizontal */
+ uint32_t num_slices_v; /* Number of DSC slices - vertical */
+ uint32_t bits_per_pixel; /* DSC target bitrate in 1/16 of bpp (e.g. 128 -> 8bpp) */
+ bool block_pred_enable; /* DSC block prediction enable */
+ uint32_t linebuf_depth; /* DSC line buffer depth */
+ uint32_t version_minor; /* DSC minor version. Full version is formed as 1.version_minor. */
+ bool ycbcr422_simple; /* Tell DSC engine to convert YCbCr 4:2:2 to 'YCbCr 4:2:2 simple'. */
+ int32_t rc_buffer_size; /* DSC RC buffer block size in bytes */
+};
+#endif
struct dc_crtc_timing {
uint32_t h_total;
uint32_t h_border_left;
@@ -743,8 +798,73 @@ struct dc_crtc_timing {
enum scanning_type scan_type;
struct dc_crtc_timing_flags flags;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ struct dc_dsc_config dsc_cfg;
+#endif
};
+/* Passed on init */
+enum vram_type {
+ VIDEO_MEMORY_TYPE_GDDR5 = 2,
+ VIDEO_MEMORY_TYPE_DDR3 = 3,
+ VIDEO_MEMORY_TYPE_DDR4 = 4,
+ VIDEO_MEMORY_TYPE_HBM = 5,
+ VIDEO_MEMORY_TYPE_GDDR6 = 6,
+};
+
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+enum dwb_cnv_out_bpc {
+ DWB_CNV_OUT_BPC_8BPC = 0,
+ DWB_CNV_OUT_BPC_10BPC = 1,
+};
+
+enum dwb_output_depth {
+ DWB_OUTPUT_PIXEL_DEPTH_8BPC = 0,
+ DWB_OUTPUT_PIXEL_DEPTH_10BPC = 1,
+};
+
+enum dwb_capture_rate {
+ dwb_capture_rate_0 = 0, /* Every frame is captured. */
+ dwb_capture_rate_1 = 1, /* Every other frame is captured. */
+ dwb_capture_rate_2 = 2, /* Every 3rd frame is captured. */
+ dwb_capture_rate_3 = 3, /* Every 4th frame is captured. */
+};
+
+enum dwb_scaler_mode {
+ dwb_scaler_mode_bypass444 = 0,
+ dwb_scaler_mode_rgb444 = 1,
+ dwb_scaler_mode_yuv444 = 2,
+ dwb_scaler_mode_yuv420 = 3
+};
+
+enum dwb_subsample_position {
+ DWB_INTERSTITIAL_SUBSAMPLING = 0,
+ DWB_COSITED_SUBSAMPLING = 1
+};
+
+enum dwb_stereo_eye_select {
+ DWB_STEREO_EYE_LEFT = 1, /* Capture left eye only */
+ DWB_STEREO_EYE_RIGHT = 2, /* Capture right eye only */
+};
+
+enum dwb_stereo_type {
+ DWB_STEREO_TYPE_FRAME_PACKING = 0, /* Frame packing */
+ DWB_STEREO_TYPE_FRAME_SEQUENTIAL = 3, /* Frame sequential */
+};
+
+#define MCIF_BUF_COUNT 4
+
+struct mcif_buf_params {
+ unsigned long long luma_address[MCIF_BUF_COUNT];
+ unsigned long long chroma_address[MCIF_BUF_COUNT];
+ unsigned int luma_pitch;
+ unsigned int chroma_pitch;
+ unsigned int warmup_pitch;
+ unsigned int swlock;
+};
+
+#endif
+
#define MAX_TG_COLOR_VALUE 0x3FF
struct tg_color {
/* Maximum 10 bits color value */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_link.h b/drivers/gpu/drm/amd/display/dc/dc_link.h
index 7b9429e30d82..6f0b80111e58 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_link.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_link.h
@@ -29,6 +29,13 @@
#include "dc_types.h"
#include "grph_object_defs.h"
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+enum dc_link_fec_state {
+ dc_link_fec_not_ready,
+ dc_link_fec_ready,
+ dc_link_fec_enabled
+};
+#endif
struct dc_link_status {
bool link_active;
struct dpcd_caps *dpcd_caps;
@@ -75,6 +82,7 @@ struct dc_link {
enum dc_irq_source irq_source_hpd_rx;/* aka DP Short Pulse */
bool is_hpd_filter_disabled;
bool dp_ss_off;
+ bool link_state_valid;
/* caps is the same as reported_link_cap. link_traing use
* reported_link_cap. Will clean up. TODO
@@ -128,6 +136,9 @@ struct dc_link {
struct link_trace link_trace;
struct gpio *hpd_gpio;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ enum dc_link_fec_state fec_state;
+#endif
};
const struct dc_link_status *dc_link_get_status(const struct dc_link *dc_link);
diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h
index 189bdab929a5..0fa1c26bc20d 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_stream.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h
@@ -42,6 +42,7 @@ struct dc_stream_status {
int primary_otg_inst;
int stream_enc_inst;
int plane_count;
+ int audio_inst;
struct timing_sync_info timing_sync_info;
struct dc_plane_state *plane_states[MAX_SURFACE_NUM];
};
@@ -51,6 +52,52 @@ struct freesync_context {
bool dummy;
};
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+enum hubp_dmdata_mode {
+ DMDATA_SW_MODE,
+ DMDATA_HW_MODE
+};
+
+struct dc_dmdata_attributes {
+ /* Specifies whether dynamic meta data will be updated by software
+ * or has to be fetched by hardware (DMA mode)
+ */
+ enum hubp_dmdata_mode dmdata_mode;
+ /* Specifies if current dynamic meta data is to be used only for the current frame */
+ bool dmdata_repeat;
+ /* Specifies the size of Dynamic Metadata surface in byte. Size of 0 means no Dynamic metadata is fetched */
+ uint32_t dmdata_size;
+ /* Specifies if a new dynamic meta data should be fetched for an upcoming frame */
+ bool dmdata_updated;
+ /* If hardware mode is used, the base address where DMDATA surface is located */
+ PHYSICAL_ADDRESS_LOC address;
+ /* Specifies whether QOS level will be provided by TTU or it will come from DMDATA_QOS_LEVEL */
+ bool dmdata_qos_mode;
+ /* If qos_mode = 1, this is the QOS value to be used: */
+ uint32_t dmdata_qos_level;
+ /* Specifies the value in unit of REFCLK cycles to be added to the
+ * current time to produce the Amortized deadline for Dynamic Metadata chunk request
+ */
+ uint32_t dmdata_dl_delta;
+ /* An unbounded array of uint32s, represents software dmdata to be loaded */
+ uint32_t *dmdata_sw_data;
+};
+#endif
+
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+struct dc_writeback_info {
+ bool wb_enabled;
+ int dwb_pipe_inst;
+ struct dc_dwb_params dwb_params;
+ struct mcif_buf_params mcif_buf_params;
+};
+
+struct dc_writeback_update {
+ unsigned int num_wb_info;
+ struct dc_writeback_info writeback_info[MAX_DWB_PIPES];
+};
+#endif
+
enum vertical_interrupt_ref_point {
START_V_UPDATE = 0,
START_V_SYNC,
@@ -80,7 +127,6 @@ struct dc_stream_state {
struct dc_info_packet vrr_infopacket;
struct dc_info_packet vsc_infopacket;
struct dc_info_packet vsp_infopacket;
- struct dc_info_packet dpsdp_infopacket;
struct rect src; /* composition area */
struct rect dst; /* stream addressable area */
@@ -142,6 +188,11 @@ struct dc_stream_state {
struct crtc_trigger_info triggered_crtc_reset;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ /* writeback */
+ unsigned int num_wb_info;
+ struct dc_writeback_info writeback_info[MAX_DWB_PIPES];
+#endif
/* Computed state bits */
bool mode_changed : 1;
@@ -160,6 +211,9 @@ struct dc_stream_state {
bool apply_seamless_boot_optimization;
uint32_t stream_id;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ bool is_dsc_enabled;
+#endif
};
struct dc_stream_update {
@@ -172,7 +226,6 @@ struct dc_stream_update {
struct periodic_interrupt_config *periodic_interrupt0;
struct periodic_interrupt_config *periodic_interrupt1;
- struct dc_crtc_timing_adjust *adjust;
struct dc_info_packet *vrr_infopacket;
struct dc_info_packet *vsc_infopacket;
struct dc_info_packet *vsp_infopacket;
@@ -185,6 +238,12 @@ struct dc_stream_update {
struct dc_csc_transform *output_csc_transform;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ struct dc_writeback_update *wb_update;
+#endif
+#if defined(CONFIG_DRM_AMD_DC_DSC_SUPPORT)
+ struct dc_dsc_config *dsc_config;
+#endif
};
bool dc_is_stream_unchanged(
@@ -274,6 +333,19 @@ bool dc_add_all_planes_for_stream(
int plane_count,
struct dc_state *context);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+bool dc_stream_add_writeback(struct dc *dc,
+ struct dc_stream_state *stream,
+ struct dc_writeback_info *wb_info);
+bool dc_stream_remove_writeback(struct dc *dc,
+ struct dc_stream_state *stream,
+ uint32_t dwb_pipe_inst);
+bool dc_stream_dmdata_status_done(struct dc *dc, struct dc_stream_state *stream);
+bool dc_stream_set_dynamic_metadata(struct dc *dc,
+ struct dc_stream_state *stream,
+ struct dc_dmdata_attributes *dmdata_attr);
+#endif
+
enum dc_status dc_validate_stream(struct dc *dc, struct dc_stream_state *stream);
/*
diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h
index 6c2a3d9a4c2e..6eabb6491a3d 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_types.h
@@ -104,7 +104,7 @@ struct dc_context {
#define DC_MAX_EDID_BUFFER_SIZE 1024
-#define EDID_BLOCK_SIZE 128
+#define DC_EDID_BLOCK_SIZE 128
#define MAX_SURFACE_NUM 4
#define NUM_PIXEL_FORMATS 10
@@ -421,6 +421,39 @@ enum display_content_type {
DISPLAY_CONTENT_TYPE_GAME = 8
};
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+/* writeback */
+struct dwb_stereo_params {
+ bool stereo_enabled; /* false: normal mode, true: 3D stereo */
+ enum dwb_stereo_type stereo_type; /* indicates stereo format */
+ bool stereo_polarity; /* indicates left eye or right eye comes first in stereo mode */
+ enum dwb_stereo_eye_select stereo_eye_select; /* indicate which eye should be captured */
+};
+
+struct dc_dwb_cnv_params {
+ unsigned int src_width; /* input active width */
+ unsigned int src_height; /* input active height (half-active height in interlaced mode) */
+ unsigned int crop_width; /* cropped window width at cnv output */
+ bool crop_en; /* window cropping enable in cnv */
+ unsigned int crop_height; /* cropped window height at cnv output */
+ unsigned int crop_x; /* cropped window start x value at cnv output */
+ unsigned int crop_y; /* cropped window start y value at cnv output */
+ enum dwb_cnv_out_bpc cnv_out_bpc; /* cnv output pixel depth - 8bpc or 10bpc */
+};
+
+struct dc_dwb_params {
+ struct dc_dwb_cnv_params cnv_params; /* CNV source size and cropping window parameters */
+ unsigned int dest_width; /* Destination width */
+ unsigned int dest_height; /* Destination height */
+ enum dwb_scaler_mode out_format; /* default = YUV420 - TODO: limit this to 0 and 1 on dcn3 */
+ enum dwb_output_depth output_depth; /* output pixel depth - 8bpc or 10bpc */
+ enum dwb_capture_rate capture_rate; /* controls the frame capture rate */
+ struct scaling_taps scaler_taps; /* Scaling taps */
+ enum dwb_subsample_position subsample_position;
+ struct dc_transfer_func *out_transfer_func;
+};
+#endif
+
/* audio*/
union audio_sample_rates {
@@ -527,6 +560,9 @@ enum dc_infoframe_type {
DC_HDMI_INFOFRAME_TYPE_AVI = 0x82,
DC_HDMI_INFOFRAME_TYPE_SPD = 0x83,
DC_HDMI_INFOFRAME_TYPE_AUDIO = 0x84,
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ DC_DP_INFOFRAME_TYPE_PPS = 0x10,
+#endif
};
struct dc_info_packet {
@@ -538,6 +574,15 @@ struct dc_info_packet {
uint8_t sb[32];
};
+struct dc_info_packet_128 {
+ bool valid;
+ uint8_t hb0;
+ uint8_t hb1;
+ uint8_t hb2;
+ uint8_t hb3;
+ uint8_t sb[128];
+};
+
#define DC_PLANE_UPDATE_TIMES_MAX 10
struct dc_plane_flip_time {
@@ -680,4 +725,75 @@ struct AsicStateEx {
unsigned int phyClock;
};
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+/* DSC DPCD capabilities */
+union dsc_slice_caps1 {
+ struct {
+ uint8_t NUM_SLICES_1 : 1;
+ uint8_t NUM_SLICES_2 : 1;
+ uint8_t RESERVED : 1;
+ uint8_t NUM_SLICES_4 : 1;
+ uint8_t NUM_SLICES_6 : 1;
+ uint8_t NUM_SLICES_8 : 1;
+ uint8_t NUM_SLICES_10 : 1;
+ uint8_t NUM_SLICES_12 : 1;
+ } bits;
+ uint8_t raw;
+};
+
+union dsc_slice_caps2 {
+ struct {
+ uint8_t NUM_SLICES_16 : 1;
+ uint8_t NUM_SLICES_20 : 1;
+ uint8_t NUM_SLICES_24 : 1;
+ uint8_t RESERVED : 5;
+ } bits;
+ uint8_t raw;
+};
+
+union dsc_color_formats {
+ struct {
+ uint8_t RGB : 1;
+ uint8_t YCBCR_444 : 1;
+ uint8_t YCBCR_SIMPLE_422 : 1;
+ uint8_t YCBCR_NATIVE_422 : 1;
+ uint8_t YCBCR_NATIVE_420 : 1;
+ uint8_t RESERVED : 3;
+ } bits;
+ uint8_t raw;
+};
+
+union dsc_color_depth {
+ struct {
+ uint8_t RESERVED1 : 1;
+ uint8_t COLOR_DEPTH_8_BPC : 1;
+ uint8_t COLOR_DEPTH_10_BPC : 1;
+ uint8_t COLOR_DEPTH_12_BPC : 1;
+ uint8_t RESERVED2 : 3;
+ } bits;
+ uint8_t raw;
+};
+
+struct dsc_dec_dpcd_caps {
+ bool is_dsc_supported;
+ uint8_t dsc_version;
+ int32_t rc_buffer_size; /* DSC RC buffer block size in bytes */
+ union dsc_slice_caps1 slice_caps1;
+ union dsc_slice_caps2 slice_caps2;
+ int32_t lb_bit_depth;
+ bool is_block_pred_supported;
+ int32_t edp_max_bits_per_pixel; /* Valid only in eDP */
+ union dsc_color_formats color_formats;
+ union dsc_color_depth color_depth;
+ int32_t throughput_mode_0_mps; /* In MPs */
+ int32_t throughput_mode_1_mps; /* In MPs */
+ int32_t max_slice_width;
+ uint32_t bpp_increment_div; /* bpp increment divisor, e.g. if 16, it's 1/16th of a bit */
+
+ /* Extended DSC caps */
+ uint32_t branch_overall_throughput_0_mps; /* In MPs */
+ uint32_t branch_overall_throughput_1_mps; /* In MPs */
+ uint32_t branch_max_line_width;
+};
+#endif
#endif /* DC_TYPES_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dce/Makefile b/drivers/gpu/drm/amd/display/dc/dce/Makefile
index 6d7b64a743ca..fdf3d8f87eee 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dce/Makefile
@@ -28,7 +28,7 @@
DCE = dce_audio.o dce_stream_encoder.o dce_link_encoder.o dce_hwseq.o \
dce_mem_input.o dce_clock_source.o dce_scl_filters.o dce_transform.o \
-dce_clk_mgr.o dce_opp.o dce_dmcu.o dce_abm.o dce_ipp.o dce_aux.o \
+dce_opp.o dce_dmcu.o dce_abm.o dce_ipp.o dce_aux.o \
dce_i2c.o dce_i2c_hw.o dce_i2c_sw.o
AMD_DAL_DCE = $(addprefix $(AMDDALPATH)/dc/dce/,$(DCE))
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c
index da96229db53a..58bd131d5b48 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dce_abm.h"
#include "dm_services.h"
#include "reg_helper.h"
@@ -58,6 +60,9 @@ static bool dce_abm_set_pipe(struct abm *abm, uint32_t controller_id)
struct dce_abm *abm_dce = TO_DCE_ABM(abm);
uint32_t rampingBoundary = 0xFFFF;
+ if (abm->dmcu_is_running == false)
+ return true;
+
REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0,
1, 80000);
@@ -234,6 +239,10 @@ static void dmcu_set_backlight_level(
s2 |= (backlight_8_bit << ATOM_S2_CURRENT_BL_LEVEL_SHIFT);
REG_WRITE(BIOS_SCRATCH_2, s2);
+
+ /* waitDMCUReadyForCmd */
+ REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT,
+ 0, 1, 80000);
}
static void dce_abm_init(struct abm *abm)
@@ -302,6 +311,9 @@ static bool dce_abm_set_level(struct abm *abm, uint32_t level)
{
struct dce_abm *abm_dce = TO_DCE_ABM(abm);
+ if (abm->dmcu_is_running == false)
+ return true;
+
REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0,
1, 80000);
@@ -320,6 +332,9 @@ static bool dce_abm_immediate_disable(struct abm *abm)
{
struct dce_abm *abm_dce = TO_DCE_ABM(abm);
+ if (abm->dmcu_is_running == false)
+ return true;
+
dce_abm_set_pipe(abm, MCP_DISABLE_ABM_IMMEDIATELY);
abm->stored_backlight_registers.BL_PWM_CNTL =
@@ -443,6 +458,7 @@ static void dce_abm_construct(
base->stored_backlight_registers.BL_PWM_CNTL2 = 0;
base->stored_backlight_registers.BL_PWM_PERIOD_CNTL = 0;
base->stored_backlight_registers.LVTMA_PWRSEQ_REF_DIV_BL_PWM_REF_DIV = 0;
+ base->dmcu_is_running = false;
abm_dce->regs = regs;
abm_dce->abm_shift = abm_shift;
@@ -473,6 +489,9 @@ void dce_abm_destroy(struct abm **abm)
{
struct dce_abm *abm_dce = TO_DCE_ABM(*abm);
+ if (abm_dce->base.dmcu_is_running == true)
+ abm_dce->base.funcs->set_abm_immediate_disable(*abm);
+
kfree(abm_dce);
*abm = NULL;
}
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.h b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.h
index ff9436966041..7ba7e6f722f6 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.h
@@ -67,6 +67,22 @@
SRI(DC_ABM1_HGLS_REG_READ_PROGRESS, ABM, id), \
NBIO_SR(BIOS_SCRATCH_2)
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#define ABM_DCN20_REG_LIST() \
+ ABM_COMMON_REG_LIST_DCE_BASE(), \
+ SR(DC_ABM1_HG_SAMPLE_RATE), \
+ SR(DC_ABM1_LS_SAMPLE_RATE), \
+ SR(BL1_PWM_BL_UPDATE_SAMPLE_RATE), \
+ SR(DC_ABM1_HG_MISC_CTRL), \
+ SR(DC_ABM1_IPCSC_COEFF_SEL), \
+ SR(BL1_PWM_CURRENT_ABM_LEVEL), \
+ SR(BL1_PWM_TARGET_ABM_LEVEL), \
+ SR(BL1_PWM_USER_LEVEL), \
+ SR(DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES), \
+ SR(DC_ABM1_HGLS_REG_READ_PROGRESS), \
+ NBIO_SR(BIOS_SCRATCH_2)
+#endif
+
#define ABM_SF(reg_name, field_name, post_fix)\
.field_name = reg_name ## __ ## field_name ## post_fix
@@ -147,6 +163,10 @@
ABM_SF(ABM0_DC_ABM1_HGLS_REG_READ_PROGRESS, \
ABM1_BL_REG_READ_MISSED_FRAME_CLEAR, mask_sh)
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#define ABM_MASK_SH_LIST_DCN20(mask_sh) ABM_MASK_SH_LIST_DCE110(mask_sh)
+#endif
+
#define ABM_REG_FIELD_LIST(type) \
type ABM1_HG_NUM_OF_BINS_SEL; \
type ABM1_HG_VMAX_SEL; \
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c
index 7f6d724686f1..4a10a5d22c90 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "reg_helper.h"
#include "dce_audio.h"
#include "dce/dce_11_0_d.h"
@@ -841,8 +843,6 @@ void dce_aud_wall_dto_setup(
REG_UPDATE(DCCG_AUDIO_DTO_SOURCE,
DCCG_AUDIO_DTO_SEL, 1);
- REG_UPDATE(DCCG_AUDIO_DTO_SOURCE,
- DCCG_AUDIO_DTO_SEL, 1);
/* DCCG_AUDIO_DTO2_USE_512FBR_DTO, 1)
* Select 512fs for DP TODO: web register definition
* does not match register header file
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.h b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.h
index 0dc5ff137c7a..a0d5724aab31 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.h
@@ -49,6 +49,8 @@
SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO0_SOURCE_SEL, mask_sh),\
SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO_SEL, mask_sh),\
SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO2_USE_512FBR_DTO, mask_sh),\
+ SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO0_USE_512FBR_DTO, mask_sh),\
+ SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO1_USE_512FBR_DTO, mask_sh),\
SF(DCCG_AUDIO_DTO0_MODULE, DCCG_AUDIO_DTO0_MODULE, mask_sh),\
SF(DCCG_AUDIO_DTO0_PHASE, DCCG_AUDIO_DTO0_PHASE, mask_sh),\
SF(DCCG_AUDIO_DTO1_MODULE, DCCG_AUDIO_DTO1_MODULE, mask_sh),\
@@ -95,6 +97,8 @@ struct dce_audio_shift {
uint8_t DCCG_AUDIO_DTO1_MODULE;
uint8_t DCCG_AUDIO_DTO1_PHASE;
uint8_t DCCG_AUDIO_DTO2_USE_512FBR_DTO;
+ uint32_t DCCG_AUDIO_DTO0_USE_512FBR_DTO;
+ uint32_t DCCG_AUDIO_DTO1_USE_512FBR_DTO;
};
struct dce_aduio_mask {
@@ -112,6 +116,9 @@ struct dce_aduio_mask {
uint32_t DCCG_AUDIO_DTO1_MODULE;
uint32_t DCCG_AUDIO_DTO1_PHASE;
uint32_t DCCG_AUDIO_DTO2_USE_512FBR_DTO;
+ uint32_t DCCG_AUDIO_DTO0_USE_512FBR_DTO;
+ uint32_t DCCG_AUDIO_DTO1_USE_512FBR_DTO;
+
};
struct dce_audio {
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
index bd33c47183fc..f2295e780031 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
@@ -23,6 +23,9 @@
*
*/
+#include <linux/delay.h>
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "core_types.h"
#include "dce_aux.h"
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h
index ce6a26d189b0..ed7fec8fe253 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h
@@ -29,6 +29,16 @@
#include "i2caux_interface.h"
#include "inc/hw/aux_engine.h"
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+#define AUX_COMMON_REG_LIST0(id)\
+ SRI(AUX_CONTROL, DP_AUX, id), \
+ SRI(AUX_ARB_CONTROL, DP_AUX, id), \
+ SRI(AUX_SW_DATA, DP_AUX, id), \
+ SRI(AUX_SW_CONTROL, DP_AUX, id), \
+ SRI(AUX_INTERRUPT_CONTROL, DP_AUX, id), \
+ SRI(AUX_SW_STATUS, DP_AUX, id)
+#endif
+
#define AUX_COMMON_REG_LIST(id)\
SRI(AUX_CONTROL, DP_AUX, id), \
SRI(AUX_ARB_CONTROL, DP_AUX, id), \
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c
index 963686380738..29d69dfc9848 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dce_clk_mgr.h"
#include "reg_helper.h"
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
index f70437aae8e0..5fae77e201d5 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
@@ -33,6 +35,7 @@
#include "include/logger_interface.h"
#include "dce_clock_source.h"
+#include "clk_mgr.h"
#include "reg_helper.h"
@@ -52,6 +55,8 @@
#define CALC_PLL_CLK_SRC_ERR_TOLERANCE 1
#define MAX_PLL_CALC_ERROR 0xFFFFFFFF
+#define NUM_ELEMENTS(a) (sizeof(a) / sizeof((a)[0]))
+
static const struct spread_spectrum_data *get_ss_data_entry(
struct dce110_clk_src *clk_src,
enum signal_type signal,
@@ -183,8 +188,8 @@ static bool calculate_fb_and_fractional_fb_divider(
*RETURNS:
* It fills the PLLSettings structure with PLL Dividers values
* if calculated values are within required tolerance
-* It returns - true if eror is within tolerance
-* - false if eror is not within tolerance
+* It returns - true if error is within tolerance
+* - false if error is not within tolerance
*/
static bool calc_fb_divider_checking_tolerance(
struct calc_pll_clock_source *calc_pll_cs,
@@ -999,6 +1004,67 @@ static bool get_pixel_clk_frequency_100hz(
return false;
}
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+
+/* this table is use to find *1.001 and /1.001 pixel rates from non-precise pixel rate */
+struct pixel_rate_range_table_entry {
+ unsigned int range_min_khz;
+ unsigned int range_max_khz;
+ unsigned int target_pixel_rate_khz;
+ unsigned short mult_factor;
+ unsigned short div_factor;
+};
+
+static const struct pixel_rate_range_table_entry video_optimized_pixel_rates[] = {
+ // /1.001 rates
+ {25170, 25180, 25200, 1000, 1001}, //25.2MHz -> 25.17
+ {59340, 59350, 59400, 1000, 1001}, //59.4Mhz -> 59.340
+ {74170, 74180, 74250, 1000, 1001}, //74.25Mhz -> 74.1758
+ {125870, 125880, 126000, 1000, 1001}, //126Mhz -> 125.87
+ {148350, 148360, 148500, 1000, 1001}, //148.5Mhz -> 148.3516
+ {167830, 167840, 168000, 1000, 1001}, //168Mhz -> 167.83
+ {222520, 222530, 222750, 1000, 1001}, //222.75Mhz -> 222.527
+ {257140, 257150, 257400, 1000, 1001}, //257.4Mhz -> 257.1429
+ {296700, 296710, 297000, 1000, 1001}, //297Mhz -> 296.7033
+ {342850, 342860, 343200, 1000, 1001}, //343.2Mhz -> 342.857
+ {395600, 395610, 396000, 1000, 1001}, //396Mhz -> 395.6
+ {409090, 409100, 409500, 1000, 1001}, //409.5Mhz -> 409.091
+ {445050, 445060, 445500, 1000, 1001}, //445.5Mhz -> 445.055
+ {467530, 467540, 468000, 1000, 1001}, //468Mhz -> 467.5325
+ {519230, 519240, 519750, 1000, 1001}, //519.75Mhz -> 519.231
+ {525970, 525980, 526500, 1000, 1001}, //526.5Mhz -> 525.974
+ {545450, 545460, 546000, 1000, 1001}, //546Mhz -> 545.455
+ {593400, 593410, 594000, 1000, 1001}, //594Mhz -> 593.4066
+ {623370, 623380, 624000, 1000, 1001}, //624Mhz -> 623.377
+ {692300, 692310, 693000, 1000, 1001}, //693Mhz -> 692.308
+ {701290, 701300, 702000, 1000, 1001}, //702Mhz -> 701.2987
+ {791200, 791210, 792000, 1000, 1001}, //792Mhz -> 791.209
+ {890100, 890110, 891000, 1000, 1001}, //891Mhz -> 890.1099
+ {1186810, 1186820, 1188000, 1000, 1001},//1188Mhz -> 1186.8131
+
+ // *1.001 rates
+ {27020, 27030, 27000, 1001, 1000}, //27Mhz
+ {54050, 54060, 54000, 1001, 1000}, //54Mhz
+ {108100, 108110, 108000, 1001, 1000},//108Mhz
+};
+
+static bool dcn20_program_pix_clk(
+ struct clock_source *clock_source,
+ struct pixel_clk_params *pix_clk_params,
+ struct pll_settings *pll_settings)
+{
+ dce112_program_pix_clk(clock_source, pix_clk_params, pll_settings);
+
+ return true;
+}
+
+static const struct clock_source_funcs dcn20_clk_src_funcs = {
+ .cs_power_down = dce110_clock_source_power_down,
+ .program_pix_clk = dcn20_program_pix_clk,
+ .get_pix_clk_dividers = dce112_get_pix_clk_dividers
+};
+#endif
+
/*****************************************/
/* Constructor */
/*****************************************/
@@ -1375,3 +1441,20 @@ bool dce112_clk_src_construct(
return true;
}
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+bool dcn20_clk_src_construct(
+ struct dce110_clk_src *clk_src,
+ struct dc_context *ctx,
+ struct dc_bios *bios,
+ enum clock_source_id id,
+ const struct dce110_clk_src_regs *regs,
+ const struct dce110_clk_src_shift *cs_shift,
+ const struct dce110_clk_src_mask *cs_mask)
+{
+ bool ret = dce112_clk_src_construct(clk_src, ctx, bios, id, regs, cs_shift, cs_mask);
+
+ clk_src->base.funcs = &dcn20_clk_src_funcs;
+
+ return ret;
+}
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.h b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.h
index 1ed7695a76d3..adae03b1f3a7 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.h
@@ -55,6 +55,37 @@
CS_SF(PHYPLLA_PIXCLK_RESYNC_CNTL, PHYPLLA_DCCG_DEEP_COLOR_CNTL, mask_sh),\
CS_SF(PHYPLLA_PIXCLK_RESYNC_CNTL, PHYPLLA_PIXCLK_DOUBLE_RATE_ENABLE, mask_sh)
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#define CS_COMMON_REG_LIST_DCN2_0(index, pllid) \
+ SRI(PIXCLK_RESYNC_CNTL, PHYPLL, pllid),\
+ SRII(PHASE, DP_DTO, 0),\
+ SRII(PHASE, DP_DTO, 1),\
+ SRII(PHASE, DP_DTO, 2),\
+ SRII(PHASE, DP_DTO, 3),\
+ SRII(PHASE, DP_DTO, 4),\
+ SRII(PHASE, DP_DTO, 5),\
+ SRII(MODULO, DP_DTO, 0),\
+ SRII(MODULO, DP_DTO, 1),\
+ SRII(MODULO, DP_DTO, 2),\
+ SRII(MODULO, DP_DTO, 3),\
+ SRII(MODULO, DP_DTO, 4),\
+ SRII(MODULO, DP_DTO, 5),\
+ SRII(PIXEL_RATE_CNTL, OTG, 0),\
+ SRII(PIXEL_RATE_CNTL, OTG, 1),\
+ SRII(PIXEL_RATE_CNTL, OTG, 2),\
+ SRII(PIXEL_RATE_CNTL, OTG, 3),\
+ SRII(PIXEL_RATE_CNTL, OTG, 4),\
+ SRII(PIXEL_RATE_CNTL, OTG, 5)
+#endif
+
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#define CS_COMMON_MASK_SH_LIST_DCN2_0(mask_sh)\
+ CS_SF(DP_DTO0_PHASE, DP_DTO0_PHASE, mask_sh),\
+ CS_SF(DP_DTO0_MODULO, DP_DTO0_MODULO, mask_sh),\
+ CS_SF(PHYPLLA_PIXCLK_RESYNC_CNTL, PHYPLLA_DCCG_DEEP_COLOR_CNTL, mask_sh),\
+ CS_SF(OTG0_PIXEL_RATE_CNTL, DP_DTO0_ENABLE, mask_sh)
+#endif
+
#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
#define CS_COMMON_REG_LIST_DCN1_0(index, pllid) \
@@ -153,4 +184,15 @@ bool dce112_clk_src_construct(
const struct dce110_clk_src_shift *cs_shift,
const struct dce110_clk_src_mask *cs_mask);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+bool dcn20_clk_src_construct(
+ struct dce110_clk_src *clk_src,
+ struct dc_context *ctx,
+ struct dc_bios *bios,
+ enum clock_source_id id,
+ const struct dce110_clk_src_regs *regs,
+ const struct dce110_clk_src_shift *cs_shift,
+ const struct dce110_clk_src_mask *cs_mask);
+#endif
+
#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
index 818536eea00a..0b86cee4876f 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
@@ -23,6 +23,9 @@
*
*/
+#include <linux/delay.h>
+#include <linux/slab.h>
+
#include "core_types.h"
#include "link_encoder.h"
#include "dce_dmcu.h"
@@ -388,6 +391,9 @@ static bool dcn10_dmcu_init(struct dmcu *dmcu)
/* Set initialized ramping boundary value */
REG_WRITE(MASTER_COMM_DATA_REG1, 0xFFFF);
+ /* Set backlight ramping stepsize */
+ REG_WRITE(MASTER_COMM_DATA_REG2, abm_gain_stepsize);
+
/* Set command to initialize microcontroller */
REG_UPDATE(MASTER_COMM_CMD_REG, MASTER_COMM_CMD_REG_BYTE0,
MCP_INIT_DMCU);
@@ -723,6 +729,56 @@ static bool dcn10_is_dmcu_initialized(struct dmcu *dmcu)
#endif //(CONFIG_DRM_AMD_DC_DCN1_0)
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+
+static bool dcn20_lock_phy(struct dmcu *dmcu)
+{
+ struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu);
+
+ /* If microcontroller is not running, do nothing */
+ if (dmcu->dmcu_state != DMCU_RUNNING)
+ return false;
+
+ /* waitDMCUReadyForCmd */
+ REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 1, 10000);
+
+ /* setDMCUParam_Cmd */
+ REG_UPDATE(MASTER_COMM_CMD_REG, MASTER_COMM_CMD_REG_BYTE0, MCP_SYNC_PHY_LOCK);
+
+ /* notifyDMCUMsg */
+ REG_UPDATE(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 1);
+
+ /* waitDMCUReadyForCmd */
+ REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 1, 10000);
+
+ return true;
+}
+
+static bool dcn20_unlock_phy(struct dmcu *dmcu)
+{
+ struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu);
+
+ /* If microcontroller is not running, do nothing */
+ if (dmcu->dmcu_state != DMCU_RUNNING)
+ return false;
+
+ /* waitDMCUReadyForCmd */
+ REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 1, 10000);
+
+ /* setDMCUParam_Cmd */
+ REG_UPDATE(MASTER_COMM_CMD_REG, MASTER_COMM_CMD_REG_BYTE0, MCP_SYNC_PHY_UNLOCK);
+
+ /* notifyDMCUMsg */
+ REG_UPDATE(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 1);
+
+ /* waitDMCUReadyForCmd */
+ REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 1, 10000);
+
+ return true;
+}
+
+#endif //(CONFIG_DRM_AMD_DC_DCN2_0)
+
static const struct dmcu_funcs dce_funcs = {
.dmcu_init = dce_dmcu_init,
.load_iram = dce_dmcu_load_iram,
@@ -747,6 +803,21 @@ static const struct dmcu_funcs dcn10_funcs = {
};
#endif
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+static const struct dmcu_funcs dcn20_funcs = {
+ .dmcu_init = dcn10_dmcu_init,
+ .load_iram = dcn10_dmcu_load_iram,
+ .set_psr_enable = dcn10_dmcu_set_psr_enable,
+ .setup_psr = dcn10_dmcu_setup_psr,
+ .get_psr_state = dcn10_get_dmcu_psr_state,
+ .set_psr_wait_loop = dcn10_psr_wait_loop,
+ .get_psr_wait_loop = dcn10_get_psr_wait_loop,
+ .is_dmcu_initialized = dcn10_is_dmcu_initialized,
+ .lock_phy = dcn20_lock_phy,
+ .unlock_phy = dcn20_unlock_phy
+};
+#endif
+
static void dce_dmcu_construct(
struct dce_dmcu *dmcu_dce,
struct dc_context *ctx,
@@ -809,10 +880,36 @@ struct dmcu *dcn10_dmcu_create(
}
#endif
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+struct dmcu *dcn20_dmcu_create(
+ struct dc_context *ctx,
+ const struct dce_dmcu_registers *regs,
+ const struct dce_dmcu_shift *dmcu_shift,
+ const struct dce_dmcu_mask *dmcu_mask)
+{
+ struct dce_dmcu *dmcu_dce = kzalloc(sizeof(*dmcu_dce), GFP_KERNEL);
+
+ if (dmcu_dce == NULL) {
+ BREAK_TO_DEBUGGER();
+ return NULL;
+ }
+
+ dce_dmcu_construct(
+ dmcu_dce, ctx, regs, dmcu_shift, dmcu_mask);
+
+ dmcu_dce->base.funcs = &dcn20_funcs;
+
+ return &dmcu_dce->base;
+}
+#endif
+
void dce_dmcu_destroy(struct dmcu **dmcu)
{
struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(*dmcu);
+ if (dmcu_dce->base.dmcu_state == DMCU_RUNNING)
+ dmcu_dce->base.funcs->set_psr_enable(*dmcu, false, true);
+
kfree(dmcu_dce);
*dmcu = NULL;
}
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h
index 60ce56f60ae3..cc8587683b4b 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h
@@ -261,6 +261,16 @@ struct dmcu *dcn10_dmcu_create(
const struct dce_dmcu_shift *dmcu_shift,
const struct dce_dmcu_mask *dmcu_mask);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+struct dmcu *dcn20_dmcu_create(
+ struct dc_context *ctx,
+ const struct dce_dmcu_registers *regs,
+ const struct dce_dmcu_shift *dmcu_shift,
+ const struct dce_dmcu_mask *dmcu_mask);
+#endif
+
void dce_dmcu_destroy(struct dmcu **dmcu);
+static const uint32_t abm_gain_stepsize = 0x0060;
+
#endif /* _DCE_ABM_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h b/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h
index 956bdf14503f..cb0a037b1c4a 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h
@@ -199,6 +199,70 @@
SR(DC_IP_REQUEST_CNTL), \
BL_REG_LIST()
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#define HWSEQ_DCN2_REG_LIST()\
+ HWSEQ_DCN_REG_LIST(), \
+ HWSEQ_PIXEL_RATE_REG_LIST(OTG), \
+ HWSEQ_PHYPLL_REG_LIST(OTG), \
+ SR(MICROSECOND_TIME_BASE_DIV), \
+ SR(MILLISECOND_TIME_BASE_DIV), \
+ SR(DISPCLK_FREQ_CHANGE_CNTL), \
+ SR(RBBMIF_TIMEOUT_DIS), \
+ SR(RBBMIF_TIMEOUT_DIS_2), \
+ SR(DCHUBBUB_CRC_CTRL), \
+ SR(DPP_TOP0_DPP_CRC_CTRL), \
+ SR(DPP_TOP0_DPP_CRC_VAL_B_A), \
+ SR(DPP_TOP0_DPP_CRC_VAL_R_G), \
+ SR(MPC_CRC_CTRL), \
+ SR(MPC_CRC_RESULT_GB), \
+ SR(MPC_CRC_RESULT_C), \
+ SR(MPC_CRC_RESULT_AR), \
+ SR(DOMAIN0_PG_CONFIG), \
+ SR(DOMAIN1_PG_CONFIG), \
+ SR(DOMAIN2_PG_CONFIG), \
+ SR(DOMAIN3_PG_CONFIG), \
+ SR(DOMAIN4_PG_CONFIG), \
+ SR(DOMAIN5_PG_CONFIG), \
+ SR(DOMAIN6_PG_CONFIG), \
+ SR(DOMAIN7_PG_CONFIG), \
+ SR(DOMAIN8_PG_CONFIG), \
+ SR(DOMAIN9_PG_CONFIG), \
+ SR(DOMAIN10_PG_CONFIG), \
+ SR(DOMAIN11_PG_CONFIG), \
+ SR(DOMAIN16_PG_CONFIG), \
+ SR(DOMAIN17_PG_CONFIG), \
+ SR(DOMAIN18_PG_CONFIG), \
+ SR(DOMAIN19_PG_CONFIG), \
+ SR(DOMAIN20_PG_CONFIG), \
+ SR(DOMAIN21_PG_CONFIG), \
+ SR(DOMAIN0_PG_STATUS), \
+ SR(DOMAIN1_PG_STATUS), \
+ SR(DOMAIN2_PG_STATUS), \
+ SR(DOMAIN3_PG_STATUS), \
+ SR(DOMAIN4_PG_STATUS), \
+ SR(DOMAIN5_PG_STATUS), \
+ SR(DOMAIN6_PG_STATUS), \
+ SR(DOMAIN7_PG_STATUS), \
+ SR(DOMAIN8_PG_STATUS), \
+ SR(DOMAIN9_PG_STATUS), \
+ SR(DOMAIN10_PG_STATUS), \
+ SR(DOMAIN11_PG_STATUS), \
+ SR(DOMAIN16_PG_STATUS), \
+ SR(DOMAIN17_PG_STATUS), \
+ SR(DOMAIN18_PG_STATUS), \
+ SR(DOMAIN19_PG_STATUS), \
+ SR(DOMAIN20_PG_STATUS), \
+ SR(DOMAIN21_PG_STATUS), \
+ SR(D1VGA_CONTROL), \
+ SR(D2VGA_CONTROL), \
+ SR(D3VGA_CONTROL), \
+ SR(D4VGA_CONTROL), \
+ SR(D5VGA_CONTROL), \
+ SR(D6VGA_CONTROL), \
+ SR(DC_IP_REQUEST_CNTL), \
+ BL_REG_LIST()
+#endif
+
struct dce_hwseq_registers {
/* Backlight registers */
@@ -453,6 +517,69 @@ struct dce_hwseq_registers {
HWS_SF(, LVTMA_PWRSEQ_CNTL, LVTMA_DIGON_OVRD, mask_sh), \
HWS_SF(, LVTMA_PWRSEQ_STATE, LVTMA_PWRSEQ_TARGET_STATE_R, mask_sh)
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#define HWSEQ_DCN2_MASK_SH_LIST(mask_sh)\
+ HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
+ HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
+ HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN0_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN0_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN1_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN1_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN2_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN2_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN3_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN3_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN4_PG_CONFIG, DOMAIN4_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN4_PG_CONFIG, DOMAIN4_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN5_PG_CONFIG, DOMAIN5_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN5_PG_CONFIG, DOMAIN5_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN6_PG_CONFIG, DOMAIN6_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN6_PG_CONFIG, DOMAIN6_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN7_PG_CONFIG, DOMAIN7_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN7_PG_CONFIG, DOMAIN7_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN8_PG_CONFIG, DOMAIN8_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN8_PG_CONFIG, DOMAIN8_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN9_PG_CONFIG, DOMAIN9_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN9_PG_CONFIG, DOMAIN9_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN10_PG_CONFIG, DOMAIN10_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN10_PG_CONFIG, DOMAIN10_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN11_PG_CONFIG, DOMAIN11_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN11_PG_CONFIG, DOMAIN11_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN16_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN16_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN17_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN17_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN18_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN18_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN19_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN19_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN20_PG_CONFIG, DOMAIN20_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN20_PG_CONFIG, DOMAIN20_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN21_PG_CONFIG, DOMAIN21_POWER_FORCEON, mask_sh), \
+ HWS_SF(, DOMAIN21_PG_CONFIG, DOMAIN21_POWER_GATE, mask_sh), \
+ HWS_SF(, DOMAIN0_PG_STATUS, DOMAIN0_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DOMAIN1_PG_STATUS, DOMAIN1_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DOMAIN2_PG_STATUS, DOMAIN2_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DOMAIN3_PG_STATUS, DOMAIN3_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DOMAIN4_PG_STATUS, DOMAIN4_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DOMAIN5_PG_STATUS, DOMAIN5_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DOMAIN6_PG_STATUS, DOMAIN6_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DOMAIN7_PG_STATUS, DOMAIN7_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DOMAIN8_PG_STATUS, DOMAIN8_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DOMAIN9_PG_STATUS, DOMAIN9_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DOMAIN10_PG_STATUS, DOMAIN10_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DOMAIN11_PG_STATUS, DOMAIN11_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DOMAIN16_PG_STATUS, DOMAIN16_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DOMAIN17_PG_STATUS, DOMAIN17_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DOMAIN18_PG_STATUS, DOMAIN18_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DOMAIN19_PG_STATUS, DOMAIN19_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DOMAIN20_PG_STATUS, DOMAIN20_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DOMAIN21_PG_STATUS, DOMAIN21_PGFSM_PWR_STATUS, mask_sh), \
+ HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
+ HWS_SF(, LVTMA_PWRSEQ_CNTL, LVTMA_BLON, mask_sh), \
+ HWS_SF(, LVTMA_PWRSEQ_STATE, LVTMA_PWRSEQ_TARGET_STATE_R, mask_sh)
+#endif
+
#define HWSEQ_REG_FIELD_LIST(type) \
type DCFE_CLOCK_ENABLE; \
type DCFEV_CLOCK_ENABLE; \
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c
index cd26161bcc4d..a9061aaf1562 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c
@@ -22,6 +22,9 @@
* Authors: AMD
*
*/
+
+#include <linux/delay.h>
+
#include "dce_i2c.h"
#include "dce_i2c_hw.h"
#include "reg_helper.h"
@@ -149,6 +152,36 @@ static void process_channel_reply(
}
}
+static bool is_engine_available(struct dce_i2c_hw *dce_i2c_hw)
+{
+ unsigned int arbitrate;
+ unsigned int i2c_hw_status;
+
+ REG_GET(HW_STATUS, DC_I2C_DDC1_HW_STATUS, &i2c_hw_status);
+ if (i2c_hw_status == DC_I2C_STATUS__DC_I2C_STATUS_USED_BY_HW)
+ return false;
+
+ REG_GET(DC_I2C_ARBITRATION, DC_I2C_REG_RW_CNTL_STATUS, &arbitrate);
+ if (arbitrate == DC_I2C_REG_RW_CNTL_STATUS_DMCU_ONLY)
+ return false;
+
+ return true;
+}
+
+static bool is_hw_busy(struct dce_i2c_hw *dce_i2c_hw)
+{
+ uint32_t i2c_sw_status = 0;
+
+ REG_GET(DC_I2C_SW_STATUS, DC_I2C_SW_STATUS, &i2c_sw_status);
+ if (i2c_sw_status == DC_I2C_STATUS__DC_I2C_STATUS_IDLE)
+ return false;
+
+ if (is_engine_available(dce_i2c_hw))
+ return false;
+
+ return true;
+}
+
static bool process_transaction(
struct dce_i2c_hw *dce_i2c_hw,
struct i2c_request_transaction_data *request)
@@ -159,6 +192,11 @@ static bool process_transaction(
bool last_transaction = false;
uint32_t value = 0;
+ if (is_hw_busy(dce_i2c_hw)) {
+ request->status = I2C_CHANNEL_OPERATION_ENGINE_BUSY;
+ return false;
+ }
+
last_transaction = ((dce_i2c_hw->transaction_count == 3) ||
(request->action == DCE_I2C_TRANSACTION_ACTION_I2C_WRITE) ||
(request->action & DCE_I2C_TRANSACTION_ACTION_I2C_READ));
@@ -268,6 +306,14 @@ static bool setup_engine(
struct dce_i2c_hw *dce_i2c_hw)
{
uint32_t i2c_setup_limit = I2C_SETUP_TIME_LIMIT_DCE;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ uint32_t reset_length = 0;
+#endif
+ /* we have checked I2c not used by DMCU, set SW use I2C REQ to 1 to indicate SW using it*/
+ REG_UPDATE(DC_I2C_ARBITRATION, DC_I2C_SW_USE_I2C_REG_REQ, 1);
+
+ /* we have checked I2c not used by DMCU, set SW use I2C REQ to 1 to indicate SW using it*/
+ REG_UPDATE(DC_I2C_ARBITRATION, DC_I2C_SW_USE_I2C_REG_REQ, 1);
if (dce_i2c_hw->setup_limit != 0)
i2c_setup_limit = dce_i2c_hw->setup_limit;
@@ -286,33 +332,26 @@ static bool setup_engine(
REG_UPDATE_N(SETUP, 2,
FN(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_TIME_LIMIT), i2c_setup_limit,
FN(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_ENABLE), 1);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ } else {
+ reset_length = dce_i2c_hw->send_reset_length;
+ REG_UPDATE_N(SETUP, 3,
+ FN(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_TIME_LIMIT), i2c_setup_limit,
+ FN(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_SEND_RESET_LENGTH), reset_length,
+ FN(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_ENABLE), 1);
+#endif
}
/* Program HW priority
* set to High - interrupt software I2C at any time
* Enable restart of SW I2C that was interrupted by HW
* disable queuing of software while I2C is in use by HW
*/
- REG_UPDATE_2(DC_I2C_ARBITRATION,
- DC_I2C_NO_QUEUED_SW_GO, 0,
- DC_I2C_SW_PRIORITY, DC_I2C_ARBITRATION__DC_I2C_SW_PRIORITY_NORMAL);
+ REG_UPDATE(DC_I2C_ARBITRATION,
+ DC_I2C_NO_QUEUED_SW_GO, 0);
return true;
}
-static bool is_hw_busy(struct dce_i2c_hw *dce_i2c_hw)
-{
- uint32_t i2c_sw_status = 0;
-
- REG_GET(DC_I2C_SW_STATUS, DC_I2C_SW_STATUS, &i2c_sw_status);
- if (i2c_sw_status == DC_I2C_STATUS__DC_I2C_STATUS_IDLE)
- return false;
-
- reset_hw_engine(dce_i2c_hw);
-
- REG_GET(DC_I2C_SW_STATUS, DC_I2C_SW_STATUS, &i2c_sw_status);
- return i2c_sw_status != DC_I2C_STATUS__DC_I2C_STATUS_IDLE;
-}
-
static void release_engine(
struct dce_i2c_hw *dce_i2c_hw)
{
@@ -322,8 +361,6 @@ static void release_engine(
set_speed(dce_i2c_hw, dce_i2c_hw->original_speed);
- /* Release I2C */
- REG_UPDATE(DC_I2C_ARBITRATION, DC_I2C_SW_DONE_USING_I2C_REG, 1);
/* Reset HW engine */
{
@@ -343,19 +380,12 @@ static void release_engine(
/* HW I2c engine - clock gating feature */
if (!dce_i2c_hw->engine_keep_power_up_count)
REG_UPDATE_N(SETUP, 1, FN(SETUP, DC_I2C_DDC1_ENABLE), 0);
+ /* Release I2C after reset, so HW or DMCU could use it */
+ REG_UPDATE_2(DC_I2C_ARBITRATION, DC_I2C_SW_DONE_USING_I2C_REG, 1,
+ DC_I2C_SW_USE_I2C_REG_REQ, 0);
}
-static bool is_engine_available(struct dce_i2c_hw *dce_i2c_hw)
-{
- unsigned int arbitrate;
-
- REG_GET(DC_I2C_ARBITRATION, DC_I2C_REG_RW_CNTL_STATUS, &arbitrate);
- if (arbitrate == DC_I2C_REG_RW_CNTL_STATUS_DMCU_ONLY)
- return false;
- return true;
-}
-
struct dce_i2c_hw *acquire_i2c_hw_engine(
struct resource_pool *pool,
struct ddc *ddc)
@@ -453,6 +483,7 @@ static void submit_channel_request_hw(
request->status = I2C_CHANNEL_OPERATION_ENGINE_BUSY;
return;
}
+ reset_hw_engine(dce_i2c_hw);
execute_transaction(dce_i2c_hw);
@@ -684,3 +715,23 @@ void dcn1_i2c_hw_construct(
dce_i2c_hw->setup_limit = I2C_SETUP_TIME_LIMIT_DCN;
}
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+void dcn2_i2c_hw_construct(
+ struct dce_i2c_hw *dce_i2c_hw,
+ struct dc_context *ctx,
+ uint32_t engine_id,
+ const struct dce_i2c_registers *regs,
+ const struct dce_i2c_shift *shifts,
+ const struct dce_i2c_mask *masks)
+{
+ dcn1_i2c_hw_construct(dce_i2c_hw,
+ ctx,
+ engine_id,
+ regs,
+ shifts,
+ masks);
+ dce_i2c_hw->send_reset_length = I2C_SEND_RESET_LENGTH_9;
+ if (ctx->dc->debug.scl_reset_length10)
+ dce_i2c_hw->send_reset_length = I2C_SEND_RESET_LENGTH_10;
+}
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.h b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.h
index 575500755b2e..cb0234e5d597 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.h
@@ -84,6 +84,7 @@ enum {
#define I2C_HW_ENGINE_COMMON_REG_LIST(id)\
SRI(SETUP, DC_I2C_DDC, id),\
SRI(SPEED, DC_I2C_DDC, id),\
+ SRI(HW_STATUS, DC_I2C_DDC, id),\
SR(DC_I2C_ARBITRATION),\
SR(DC_I2C_CONTROL),\
SR(DC_I2C_SW_STATUS),\
@@ -105,6 +106,8 @@ enum {
I2C_SF(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_DATA_DRIVE_SEL, mask_sh),\
I2C_SF(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_INTRA_TRANSACTION_DELAY, mask_sh),\
I2C_SF(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_INTRA_BYTE_DELAY, mask_sh),\
+ I2C_SF(DC_I2C_DDC1_HW_STATUS, DC_I2C_DDC1_HW_STATUS, mask_sh),\
+ I2C_SF(DC_I2C_ARBITRATION, DC_I2C_SW_USE_I2C_REG_REQ, mask_sh),\
I2C_SF(DC_I2C_ARBITRATION, DC_I2C_SW_DONE_USING_I2C_REG, mask_sh),\
I2C_SF(DC_I2C_ARBITRATION, DC_I2C_NO_QUEUED_SW_GO, mask_sh),\
I2C_SF(DC_I2C_ARBITRATION, DC_I2C_SW_PRIORITY, mask_sh),\
@@ -145,7 +148,9 @@ struct dce_i2c_shift {
uint8_t DC_I2C_DDC1_DATA_DRIVE_SEL;
uint8_t DC_I2C_DDC1_INTRA_TRANSACTION_DELAY;
uint8_t DC_I2C_DDC1_INTRA_BYTE_DELAY;
+ uint8_t DC_I2C_DDC1_HW_STATUS;
uint8_t DC_I2C_SW_DONE_USING_I2C_REG;
+ uint8_t DC_I2C_SW_USE_I2C_REG_REQ;
uint8_t DC_I2C_NO_QUEUED_SW_GO;
uint8_t DC_I2C_SW_PRIORITY;
uint8_t DC_I2C_SOFT_RESET;
@@ -172,6 +177,9 @@ struct dce_i2c_shift {
uint8_t DC_I2C_INDEX;
uint8_t DC_I2C_INDEX_WRITE;
uint8_t XTAL_REF_DIV;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ uint8_t DC_I2C_DDC1_SEND_RESET_LENGTH;
+#endif
uint8_t DC_I2C_REG_RW_CNTL_STATUS;
};
@@ -183,7 +191,9 @@ struct dce_i2c_mask {
uint32_t DC_I2C_DDC1_DATA_DRIVE_SEL;
uint32_t DC_I2C_DDC1_INTRA_TRANSACTION_DELAY;
uint32_t DC_I2C_DDC1_INTRA_BYTE_DELAY;
+ uint32_t DC_I2C_DDC1_HW_STATUS;
uint32_t DC_I2C_SW_DONE_USING_I2C_REG;
+ uint32_t DC_I2C_SW_USE_I2C_REG_REQ;
uint32_t DC_I2C_NO_QUEUED_SW_GO;
uint32_t DC_I2C_SW_PRIORITY;
uint32_t DC_I2C_SOFT_RESET;
@@ -210,12 +220,22 @@ struct dce_i2c_mask {
uint32_t DC_I2C_INDEX;
uint32_t DC_I2C_INDEX_WRITE;
uint32_t XTAL_REF_DIV;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ uint32_t DC_I2C_DDC1_SEND_RESET_LENGTH;
+#endif
uint32_t DC_I2C_REG_RW_CNTL_STATUS;
};
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#define I2C_COMMON_MASK_SH_LIST_DCN2(mask_sh)\
+ I2C_COMMON_MASK_SH_LIST_DCE110(mask_sh),\
+ I2C_SF(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_SEND_RESET_LENGTH, mask_sh)
+#endif
+
struct dce_i2c_registers {
uint32_t SETUP;
uint32_t SPEED;
+ uint32_t HW_STATUS;
uint32_t DC_I2C_ARBITRATION;
uint32_t DC_I2C_CONTROL;
uint32_t DC_I2C_SW_STATUS;
@@ -292,6 +312,16 @@ void dcn1_i2c_hw_construct(
const struct dce_i2c_shift *shifts,
const struct dce_i2c_mask *masks);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+void dcn2_i2c_hw_construct(
+ struct dce_i2c_hw *dce_i2c_hw,
+ struct dc_context *ctx,
+ uint32_t engine_id,
+ const struct dce_i2c_registers *regs,
+ const struct dce_i2c_shift *shifts,
+ const struct dce_i2c_mask *masks);
+#endif
+
bool dce_i2c_submit_command_hw(
struct resource_pool *pool,
struct ddc *ddc,
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c
index f0266694cb56..a5a11c251e25 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c
@@ -22,6 +22,9 @@
* Authors: AMD
*
*/
+
+#include <linux/delay.h>
+
#include "dce_i2c.h"
#include "dce_i2c_sw.h"
#include "include/gpio_service_interface.h"
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c b/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c
index 5d9506b3d46b..ce30dbf579d4 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dce_ipp.h"
#include "reg_helper.h"
#include "dm_services.h"
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
index 314c04a915d2..8527cce81c6f 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
@@ -23,6 +23,9 @@
*
*/
+#include <linux/delay.h>
+#include <linux/slab.h>
+
#include "reg_helper.h"
#include "core_types.h"
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c
index 87093894ea9e..51081d9ae3fb 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "basics/conversion.h"
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c
index 14309fe6f2e6..5e2b4d47c548 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/delay.h>
+
#include "dc_bios_types.h"
#include "dce_stream_encoder.h"
#include "reg_helper.h"
@@ -418,6 +420,7 @@ static void dce110_stream_encoder_dp_set_stream_attribute(
break;
case COLOR_SPACE_YCBCR709:
case COLOR_SPACE_YCBCR709_LIMITED:
+ case COLOR_SPACE_YCBCR709_BLACK:
misc0 = misc0 | 0x18; /* bit3=1, bit4=1 */
misc1 = misc1 & ~0x80; /* bit7 = 0*/
dynamic_range_ycbcr = 1; /*bt709*/
@@ -1123,19 +1126,6 @@ union audio_cea_channels {
} channels;
};
-struct audio_clock_info {
- /* pixel clock frequency*/
- uint32_t pixel_clock_in_10khz;
- /* N - 32KHz audio */
- uint32_t n_32khz;
- /* CTS - 32KHz audio*/
- uint32_t cts_32khz;
- uint32_t n_44khz;
- uint32_t cts_44khz;
- uint32_t n_48khz;
- uint32_t cts_48khz;
-};
-
/* 25.2MHz/1.001*/
/* 25.2MHz/1.001*/
/* 25.2MHz*/
diff --git a/drivers/gpu/drm/amd/display/dc/dce100/dce100_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce100/dce100_hw_sequencer.c
index 87771676acac..799d36299c9b 100644
--- a/drivers/gpu/drm/amd/display/dc/dce100/dce100_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dce100/dce100_hw_sequencer.c
@@ -25,6 +25,7 @@
#include "dm_services.h"
#include "dc.h"
#include "core_types.h"
+#include "clk_mgr.h"
#include "hw_sequencer.h"
#include "dce100_hw_sequencer.h"
#include "resource.h"
@@ -111,8 +112,8 @@ void dce100_prepare_bandwidth(
{
dce110_set_safe_displaymarks(&context->res_ctx, dc->res_pool);
- dc->res_pool->clk_mgr->funcs->update_clocks(
- dc->res_pool->clk_mgr,
+ dc->clk_mgr->funcs->update_clocks(
+ dc->clk_mgr,
context,
false);
}
@@ -123,8 +124,8 @@ void dce100_optimize_bandwidth(
{
dce110_set_safe_displaymarks(&context->res_ctx, dc->res_pool);
- dc->res_pool->clk_mgr->funcs->update_clocks(
- dc->res_pool->clk_mgr,
+ dc->clk_mgr->funcs->update_clocks(
+ dc->clk_mgr,
context,
true);
}
diff --git a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c
index e938bf9986d3..6248c8455314 100644
--- a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c
@@ -22,6 +22,9 @@
* Authors: AMD
*
*/
+
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "link_encoder.h"
@@ -35,8 +38,6 @@
#include "irq/dce110/irq_service_dce110.h"
#include "dce/dce_link_encoder.h"
#include "dce/dce_stream_encoder.h"
-
-#include "dce/dce_clk_mgr.h"
#include "dce/dce_mem_input.h"
#include "dce/dce_ipp.h"
#include "dce/dce_transform.h"
@@ -137,19 +138,6 @@ static const struct dce110_timing_generator_offsets dce100_tg_offsets[] = {
#define SRI(reg_name, block, id)\
.reg_name = mm ## block ## id ## _ ## reg_name
-
-static const struct clk_mgr_registers disp_clk_regs = {
- CLK_COMMON_REG_LIST_DCE_BASE()
-};
-
-static const struct clk_mgr_shift disp_clk_shift = {
- CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
-};
-
-static const struct clk_mgr_mask disp_clk_mask = {
- CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
-};
-
#define ipp_regs(id)\
[id] = {\
IPP_DCE100_REG_LIST_DCE_BASE(id)\
@@ -746,9 +734,6 @@ static void destruct(struct dce110_resource_pool *pool)
dce_aud_destroy(&pool->base.audios[i]);
}
- if (pool->base.clk_mgr != NULL)
- dce_clk_mgr_destroy(&pool->base.clk_mgr);
-
if (pool->base.abm != NULL)
dce_abm_destroy(&pool->base.abm);
@@ -867,13 +852,55 @@ enum dc_status dce100_validate_plane(const struct dc_plane_state *plane_state, s
return DC_FAIL_SURFACE_VALIDATE;
}
+struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link(
+ struct resource_context *res_ctx,
+ const struct resource_pool *pool,
+ struct dc_stream_state *stream)
+{
+ int i;
+ int j = -1;
+ struct dc_link *link = stream->link;
+
+ for (i = 0; i < pool->stream_enc_count; i++) {
+ if (!res_ctx->is_stream_enc_acquired[i] &&
+ pool->stream_enc[i]) {
+ /* Store first available for MST second display
+ * in daisy chain use case
+ */
+ j = i;
+ if (pool->stream_enc[i]->id ==
+ link->link_enc->preferred_engine)
+ return pool->stream_enc[i];
+ }
+ }
+
+ /*
+ * below can happen in cases when stream encoder is acquired:
+ * 1) for second MST display in chain, so preferred engine already
+ * acquired;
+ * 2) for another link, which preferred engine already acquired by any
+ * MST configuration.
+ *
+ * If signal is of DP type and preferred engine not found, return last available
+ *
+ * TODO - This is just a patch up and a generic solution is
+ * required for non DP connectors.
+ */
+
+ if (j >= 0 && link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT)
+ return pool->stream_enc[j];
+
+ return NULL;
+}
+
static const struct resource_funcs dce100_res_pool_funcs = {
.destroy = dce100_destroy_resource_pool,
.link_enc_create = dce100_link_encoder_create,
.validate_bandwidth = dce100_validate_bandwidth,
.validate_plane = dce100_validate_plane,
.add_stream_to_ctx = dce100_add_stream_to_ctx,
- .validate_global = dce100_validate_global
+ .validate_global = dce100_validate_global,
+ .find_first_free_match_stream_enc_for_link = dce100_find_first_free_match_stream_enc_for_link
};
static bool construct(
@@ -932,16 +959,6 @@ static bool construct(
}
}
- pool->base.clk_mgr = dce_clk_mgr_create(ctx,
- &disp_clk_regs,
- &disp_clk_shift,
- &disp_clk_mask);
- if (pool->base.clk_mgr == NULL) {
- dm_error("DC: failed to create display clock!\n");
- BREAK_TO_DEBUGGER();
- goto res_create_fail;
- }
-
pool->base.dmcu = dce_dmcu_create(ctx,
&dmcu_regs,
&dmcu_shift,
diff --git a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.h b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.h
index 2f366d66635d..fecab7c560f5 100644
--- a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.h
@@ -46,4 +46,9 @@ enum dc_status dce100_add_stream_to_ctx(
struct dc_state *new_ctx,
struct dc_stream_state *dc_stream);
+struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link(
+ struct resource_context *res_ctx,
+ const struct resource_pool *pool,
+ struct dc_stream_state *stream);
+
#endif /* DCE100_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c
index 7b23239d33fe..72b580a4eb85 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c
@@ -23,6 +23,9 @@
*
*/
+#include <linux/delay.h>
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "dce/dce_11_0_d.h"
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
index 7ac50ab1b762..fafb4b470140 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
@@ -22,6 +22,9 @@
* Authors: AMD
*
*/
+
+#include <linux/delay.h>
+
#include "dm_services.h"
#include "dc.h"
#include "dc_bios_types.h"
@@ -46,6 +49,7 @@
#include "link_encoder.h"
#include "link_hwss.h"
#include "clock_source.h"
+#include "clk_mgr.h"
#include "abm.h"
#include "audio.h"
#include "reg_helper.h"
@@ -242,6 +246,9 @@ static void build_prescale_params(struct ipp_prescale_params *prescale_params,
prescale_params->mode = IPP_PRESCALE_MODE_FIXED_UNSIGNED;
switch (plane_state->format) {
+ case SURFACE_PIXEL_FORMAT_GRPH_RGB565:
+ prescale_params->scale = 0x2082;
+ break;
case SURFACE_PIXEL_FORMAT_GRPH_ARGB8888:
case SURFACE_PIXEL_FORMAT_GRPH_ABGR8888:
prescale_params->scale = 0x2020;
@@ -662,6 +669,26 @@ void dce110_enable_stream(struct pipe_ctx *pipe_ctx)
/* update AVI info frame (HDMI, DP)*/
/* TODO: FPGA may change to hwss.update_info_frame */
+
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ if (pipe_ctx->stream_res.stream_enc->funcs->set_dynamic_metadata != NULL &&
+ pipe_ctx->plane_res.hubp != NULL) {
+ if (pipe_ctx->stream->dmdata_address.quad_part != 0) {
+ /* if using dynamic meta, don't set up generic infopackets */
+ pipe_ctx->stream_res.encoder_info_frame.hdrsmd.valid = false;
+ pipe_ctx->stream_res.stream_enc->funcs->set_dynamic_metadata(
+ pipe_ctx->stream_res.stream_enc,
+ true, pipe_ctx->plane_res.hubp->inst,
+ dc_is_dp_signal(pipe_ctx->stream->signal) ?
+ dmdata_dp : dmdata_hdmi);
+ } else
+ pipe_ctx->stream_res.stream_enc->funcs->set_dynamic_metadata(
+ pipe_ctx->stream_res.stream_enc,
+ false, pipe_ctx->plane_res.hubp->inst,
+ dc_is_dp_signal(pipe_ctx->stream->signal) ?
+ dmdata_dp : dmdata_hdmi);
+ }
+#endif
dce110_update_info_frame(pipe_ctx);
/* enable early control to avoid corruption on DP monitor*/
@@ -935,28 +962,23 @@ void hwss_edp_backlight_control(
edp_receiver_ready_T9(link);
}
-// Static helper function which calls the correct function
-// based on pp_smu version
-static void set_pme_wa_enable_by_version(struct dc *dc)
-{
- struct pp_smu_funcs *pp_smu = NULL;
-
- if (dc->res_pool->pp_smu)
- pp_smu = dc->res_pool->pp_smu;
-
- if (pp_smu) {
- if (pp_smu->ctx.ver == PP_SMU_VER_RV && pp_smu->rv_funcs.set_pme_wa_enable)
- pp_smu->rv_funcs.set_pme_wa_enable(&(pp_smu->ctx));
- }
-}
-
void dce110_enable_audio_stream(struct pipe_ctx *pipe_ctx)
{
/* notify audio driver for audio modes of monitor */
- struct dc *core_dc = pipe_ctx->stream->ctx->dc;
+ struct dc *core_dc;
struct pp_smu_funcs *pp_smu = NULL;
+ struct clk_mgr *clk_mgr;
unsigned int i, num_audio = 1;
+ if (!pipe_ctx->stream)
+ return;
+
+ core_dc = pipe_ctx->stream->ctx->dc;
+ clk_mgr = core_dc->clk_mgr;
+
+ if (pipe_ctx->stream_res.audio && pipe_ctx->stream_res.audio->enabled == true)
+ return;
+
if (core_dc->res_pool->pp_smu)
pp_smu = core_dc->res_pool->pp_smu;
@@ -969,24 +991,38 @@ void dce110_enable_audio_stream(struct pipe_ctx *pipe_ctx)
pipe_ctx->stream_res.audio->funcs->az_enable(pipe_ctx->stream_res.audio);
- if (num_audio >= 1 && pp_smu != NULL)
+ if (num_audio >= 1 && clk_mgr->funcs->enable_pme_wa)
/*this is the first audio. apply the PME w/a in order to wake AZ from D3*/
- set_pme_wa_enable_by_version(core_dc);
+ clk_mgr->funcs->enable_pme_wa(clk_mgr);
/* un-mute audio */
/* TODO: audio should be per stream rather than per link */
pipe_ctx->stream_res.stream_enc->funcs->audio_mute_control(
pipe_ctx->stream_res.stream_enc, false);
+ if (pipe_ctx->stream_res.audio)
+ pipe_ctx->stream_res.audio->enabled = true;
}
}
void dce110_disable_audio_stream(struct pipe_ctx *pipe_ctx, int option)
{
- struct dc *dc = pipe_ctx->stream->ctx->dc;
+ struct dc *dc;
struct pp_smu_funcs *pp_smu = NULL;
+ struct clk_mgr *clk_mgr;
+
+ if (!pipe_ctx || !pipe_ctx->stream)
+ return;
+
+ dc = pipe_ctx->stream->ctx->dc;
+ clk_mgr = dc->clk_mgr;
+
+ if (pipe_ctx->stream_res.audio && pipe_ctx->stream_res.audio->enabled == false)
+ return;
pipe_ctx->stream_res.stream_enc->funcs->audio_mute_control(
pipe_ctx->stream_res.stream_enc, true);
if (pipe_ctx->stream_res.audio) {
+ pipe_ctx->stream_res.audio->enabled = false;
+
if (dc->res_pool->pp_smu)
pp_smu = dc->res_pool->pp_smu;
@@ -1008,9 +1044,9 @@ void dce110_disable_audio_stream(struct pipe_ctx *pipe_ctx, int option)
update_audio_usage(&dc->current_state->res_ctx, dc->res_pool, pipe_ctx->stream_res.audio, false);
pipe_ctx->stream_res.audio = NULL;
}
- if (pp_smu != NULL)
+ if (clk_mgr->funcs->enable_pme_wa)
/*this is the first audio. apply the PME w/a in order to wake AZ from D3*/
- set_pme_wa_enable_by_version(dc);
+ clk_mgr->funcs->enable_pme_wa(clk_mgr);
/* TODO: notify audio driver for if audio modes list changed
* add audio mode list change flag */
@@ -1296,6 +1332,11 @@ static enum dc_status dce110_enable_stream_timing(
pipe_ctx->stream_res.tg->funcs->program_timing(
pipe_ctx->stream_res.tg,
&stream->timing,
+ 0,
+ 0,
+ 0,
+ 0,
+ pipe_ctx->stream->signal,
true);
}
@@ -1318,6 +1359,9 @@ static enum dc_status apply_single_controller_ctx_to_hw(
struct dc_stream_state *stream = pipe_ctx->stream;
struct drr_params params = {0};
unsigned int event_triggers = 0;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ struct pipe_ctx *odm_pipe = dc_res_get_odm_bottom_pipe(pipe_ctx);
+#endif
if (dc->hwss.disable_stream_gating) {
dc->hwss.disable_stream_gating(dc, pipe_ctx);
@@ -1383,6 +1427,20 @@ static enum dc_status apply_single_controller_ctx_to_hw(
pipe_ctx->stream_res.opp,
&stream->bit_depth_params,
&stream->clamping);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ if (odm_pipe) {
+ odm_pipe->stream_res.opp->funcs->opp_set_dyn_expansion(
+ odm_pipe->stream_res.opp,
+ COLOR_SPACE_YCBCR601,
+ stream->timing.display_color_depth,
+ stream->signal);
+
+ odm_pipe->stream_res.opp->funcs->opp_program_fmt(
+ odm_pipe->stream_res.opp,
+ &stream->bit_depth_params,
+ &stream->clamping);
+ }
+#endif
if (!stream->dpms_off)
core_link_enable_stream(context, pipe_ctx);
@@ -1488,10 +1546,23 @@ static void disable_vga_and_power_gate_all_controllers(
}
}
-static struct dc_link *get_link_for_edp(struct dc *dc)
+
+static struct dc_stream_state *get_edp_stream(struct dc_state *context)
{
int i;
+ for (i = 0; i < context->stream_count; i++) {
+ if (context->streams[i]->signal == SIGNAL_TYPE_EDP)
+ return context->streams[i];
+ }
+ return NULL;
+}
+
+static struct dc_link *get_edp_link(struct dc *dc)
+{
+ int i;
+
+ // report any eDP links, even unconnected DDI's
for (i = 0; i < dc->link_count; i++) {
if (dc->links[i]->connector_signal == SIGNAL_TYPE_EDP)
return dc->links[i];
@@ -1499,23 +1570,13 @@ static struct dc_link *get_link_for_edp(struct dc *dc)
return NULL;
}
-static struct dc_link *get_link_for_edp_to_turn_off(
+static struct dc_link *get_edp_link_with_sink(
struct dc *dc,
struct dc_state *context)
{
int i;
struct dc_link *link = NULL;
- /* check if eDP panel is suppose to be set mode, if yes, no need to disable */
- for (i = 0; i < context->stream_count; i++) {
- if (context->streams[i]->signal == SIGNAL_TYPE_EDP) {
- if (context->streams[i]->dpms_off == true)
- return context->streams[i]->sink->link;
- else
- return NULL;
- }
- }
-
/* check if there is an eDP panel not in use */
for (i = 0; i < dc->link_count; i++) {
if (dc->links[i]->local_sink &&
@@ -1538,59 +1599,58 @@ static struct dc_link *get_link_for_edp_to_turn_off(
void dce110_enable_accelerated_mode(struct dc *dc, struct dc_state *context)
{
int i;
- struct dc_link *edp_link_to_turnoff = NULL;
- struct dc_link *edp_link = get_link_for_edp(dc);
- bool can_edp_fast_boot_optimize = false;
- bool apply_edp_fast_boot_optimization = false;
+ struct dc_link *edp_link_with_sink = get_edp_link_with_sink(dc, context);
+ struct dc_link *edp_link = get_edp_link(dc);
+ struct dc_stream_state *edp_stream = NULL;
+ bool can_apply_edp_fast_boot = false;
bool can_apply_seamless_boot = false;
-
- for (i = 0; i < context->stream_count; i++) {
- if (context->streams[i]->apply_seamless_boot_optimization) {
- can_apply_seamless_boot = true;
- break;
- }
- }
+ bool keep_edp_vdd_on = false;
if (dc->hwss.init_pipes)
dc->hwss.init_pipes(dc, context);
- if (edp_link) {
- /* this seems to cause blank screens on DCE8 */
- if ((dc->ctx->dce_version == DCE_VERSION_8_0) ||
- (dc->ctx->dce_version == DCE_VERSION_8_1) ||
- (dc->ctx->dce_version == DCE_VERSION_8_3))
- can_edp_fast_boot_optimize = false;
- else
- can_edp_fast_boot_optimize =
- edp_link->link_enc->funcs->is_dig_enabled(edp_link->link_enc);
- }
+ edp_stream = get_edp_stream(context);
- if (can_edp_fast_boot_optimize)
- edp_link_to_turnoff = get_link_for_edp_to_turn_off(dc, context);
+ // Check fastboot support, disable on DCE8 because of blank screens
+ if (edp_link && dc->ctx->dce_version != DCE_VERSION_8_0 &&
+ dc->ctx->dce_version != DCE_VERSION_8_1 &&
+ dc->ctx->dce_version != DCE_VERSION_8_3) {
- /* if OS doesn't light up eDP and eDP link is available, we want to disable
- * If resume from S4/S5, should optimization.
- */
- if (can_edp_fast_boot_optimize && !edp_link_to_turnoff) {
- /* Find eDP stream and set optimization flag */
- for (i = 0; i < context->stream_count; i++) {
- if (context->streams[i]->signal == SIGNAL_TYPE_EDP) {
- context->streams[i]->apply_edp_fast_boot_optimization = true;
- apply_edp_fast_boot_optimization = true;
+ // enable fastboot if backend is enabled on eDP
+ if (edp_link->link_enc->funcs->is_dig_enabled(edp_link->link_enc)) {
+ /* Set optimization flag on eDP stream*/
+ if (edp_stream) {
+ edp_stream->apply_edp_fast_boot_optimization = true;
+ can_apply_edp_fast_boot = true;
}
}
+
+ // We are trying to enable eDP, don't power down VDD
+ if (edp_stream)
+ keep_edp_vdd_on = true;
+ }
+
+ // Check seamless boot support
+ for (i = 0; i < context->stream_count; i++) {
+ if (context->streams[i]->apply_seamless_boot_optimization) {
+ can_apply_seamless_boot = true;
+ break;
+ }
}
- if (!apply_edp_fast_boot_optimization && !can_apply_seamless_boot) {
- if (edp_link_to_turnoff) {
+ /* eDP should not have stream in resume from S4 and so even with VBios post
+ * it should get turned off
+ */
+ if (!can_apply_edp_fast_boot && !can_apply_seamless_boot) {
+ if (edp_link_with_sink && !keep_edp_vdd_on) {
/*turn off backlight before DP_blank and encoder powered down*/
- dc->hwss.edp_backlight_control(edp_link_to_turnoff, false);
+ dc->hwss.edp_backlight_control(edp_link_with_sink, false);
}
/*resume from S3, no vbios posting, no need to power down again*/
power_down_all_hw_blocks(dc);
disable_vga_and_power_gate_all_controllers(dc);
- if (edp_link_to_turnoff)
- dc->hwss.edp_power_control(edp_link_to_turnoff, false);
+ if (edp_link_with_sink && !keep_edp_vdd_on)
+ dc->hwss.edp_power_control(edp_link_with_sink, false);
}
bios_set_scratch_acc_mode_change(dc->ctx->dc_bios);
}
@@ -2030,8 +2090,10 @@ enum dc_status dce110_apply_ctx_to_hw(
if (pipe_ctx->stream == NULL)
continue;
- if (pipe_ctx->stream == pipe_ctx_old->stream)
+ if (pipe_ctx->stream == pipe_ctx_old->stream &&
+ pipe_ctx->stream->link->link_state_valid) {
continue;
+ }
if (pipe_ctx_old->stream && !pipe_need_reprogram(pipe_ctx_old, pipe_ctx))
continue;
@@ -2318,6 +2380,7 @@ static void init_hw(struct dc *dc)
struct dc_bios *bp;
struct transform *xfm;
struct abm *abm;
+ struct dmcu *dmcu;
bp = dc->ctx->dc_bios;
for (i = 0; i < dc->res_pool->pipe_count; i++) {
@@ -2345,9 +2408,6 @@ static void init_hw(struct dc *dc)
* default signal on connector). */
struct dc_link *link = dc->links[i];
- if (link->link_enc->connector.id == CONNECTOR_ID_EDP)
- dc->hwss.edp_power_control(link, true);
-
link->link_enc->funcs->hw_init(link->link_enc);
}
@@ -2373,6 +2433,10 @@ static void init_hw(struct dc *dc)
abm->funcs->abm_init(abm);
}
+ dmcu = dc->res_pool->dmcu;
+ if (dmcu != NULL && abm != NULL)
+ abm->dmcu_is_running = dmcu->funcs->is_dmcu_initialized(dmcu);
+
if (dc->fbc_compressor)
dc->fbc_compressor->funcs->power_up_fbc(dc->fbc_compressor);
@@ -2383,7 +2447,7 @@ void dce110_prepare_bandwidth(
struct dc *dc,
struct dc_state *context)
{
- struct clk_mgr *dccg = dc->res_pool->clk_mgr;
+ struct clk_mgr *dccg = dc->clk_mgr;
dce110_set_safe_displaymarks(&context->res_ctx, dc->res_pool);
@@ -2397,7 +2461,7 @@ void dce110_optimize_bandwidth(
struct dc *dc,
struct dc_state *context)
{
- struct clk_mgr *dccg = dc->res_pool->clk_mgr;
+ struct clk_mgr *dccg = dc->clk_mgr;
dce110_set_displaymarks(dc, context);
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_opp_regamma_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_opp_regamma_v.c
index 9b65b77e8823..34c5e3c7c6d2 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_opp_regamma_v.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_opp_regamma_v.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/delay.h>
+
#include "dm_services.h"
/* include DCE11 register header files */
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
index dcd04e9ea76b..764329264c3b 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "link_encoder.h"
@@ -30,8 +32,6 @@
#include "resource.h"
#include "dce110/dce110_resource.h"
-
-#include "dce/dce_clk_mgr.h"
#include "include/irq_service_interface.h"
#include "dce/dce_audio.h"
#include "dce110/dce110_timing_generator.h"
@@ -149,18 +149,6 @@ static const struct dce110_timing_generator_offsets dce110_tg_offsets[] = {
#define SRI(reg_name, block, id)\
.reg_name = mm ## block ## id ## _ ## reg_name
-static const struct clk_mgr_registers disp_clk_regs = {
- CLK_COMMON_REG_LIST_DCE_BASE()
-};
-
-static const struct clk_mgr_shift disp_clk_shift = {
- CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
-};
-
-static const struct clk_mgr_mask disp_clk_mask = {
- CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
-};
-
static const struct dce_dmcu_registers dmcu_regs = {
DMCU_DCE110_COMMON_REG_LIST()
};
@@ -811,9 +799,6 @@ static void destruct(struct dce110_resource_pool *pool)
if (pool->base.dmcu != NULL)
dce_dmcu_destroy(&pool->base.dmcu);
- if (pool->base.clk_mgr != NULL)
- dce_clk_mgr_destroy(&pool->base.clk_mgr);
-
if (pool->base.irqs != NULL) {
dal_irq_service_destroy(&pool->base.irqs);
}
@@ -1097,6 +1082,11 @@ static struct pipe_ctx *dce110_acquire_underlay(
pipe_ctx->stream_res.tg->funcs->program_timing(pipe_ctx->stream_res.tg,
&stream->timing,
+ 0,
+ 0,
+ 0,
+ 0,
+ pipe_ctx->stream->signal,
false);
pipe_ctx->stream_res.tg->funcs->enable_advanced_request(
@@ -1129,6 +1119,38 @@ static void dce110_destroy_resource_pool(struct resource_pool **pool)
*pool = NULL;
}
+struct stream_encoder *dce110_find_first_free_match_stream_enc_for_link(
+ struct resource_context *res_ctx,
+ const struct resource_pool *pool,
+ struct dc_stream_state *stream)
+{
+ int i;
+ int j = -1;
+ struct dc_link *link = stream->link;
+
+ for (i = 0; i < pool->stream_enc_count; i++) {
+ if (!res_ctx->is_stream_enc_acquired[i] &&
+ pool->stream_enc[i]) {
+ /* Store first available for MST second display
+ * in daisy chain use case
+ */
+ j = i;
+ if (pool->stream_enc[i]->id ==
+ link->link_enc->preferred_engine)
+ return pool->stream_enc[i];
+ }
+ }
+
+ /*
+ * For CZ and later, we can allow DIG FE and BE to differ for all display types
+ */
+
+ if (j >= 0)
+ return pool->stream_enc[j];
+
+ return NULL;
+}
+
static const struct resource_funcs dce110_res_pool_funcs = {
.destroy = dce110_destroy_resource_pool,
@@ -1137,7 +1159,8 @@ static const struct resource_funcs dce110_res_pool_funcs = {
.validate_plane = dce110_validate_plane,
.acquire_idle_pipe_for_layer = dce110_acquire_underlay,
.add_stream_to_ctx = dce110_add_stream_to_ctx,
- .validate_global = dce110_validate_global
+ .validate_global = dce110_validate_global,
+ .find_first_free_match_stream_enc_for_link = dce110_find_first_free_match_stream_enc_for_link
};
static bool underlay_create(struct dc_context *ctx, struct resource_pool *pool)
@@ -1308,16 +1331,6 @@ static bool construct(
}
}
- pool->base.clk_mgr = dce110_clk_mgr_create(ctx,
- &disp_clk_regs,
- &disp_clk_shift,
- &disp_clk_mask);
- if (pool->base.clk_mgr == NULL) {
- dm_error("DC: failed to create display clock!\n");
- BREAK_TO_DEBUGGER();
- goto res_create_fail;
- }
-
pool->base.dmcu = dce_dmcu_create(ctx,
&dmcu_regs,
&dmcu_shift,
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.h b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.h
index e5f168c1f8c8..aa4531e0800e 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.h
@@ -45,5 +45,10 @@ struct resource_pool *dce110_create_resource_pool(
struct dc *dc,
struct hw_asic_id asic_id);
+struct stream_encoder *dce110_find_first_free_match_stream_enc_for_link(
+ struct resource_context *res_ctx,
+ const struct resource_pool *pool,
+ struct dc_stream_state *stream);
+
#endif /* __DC_RESOURCE_DCE110_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c
index 1b2fe0df347f..5f7c2c5641c4 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c
@@ -1952,6 +1952,11 @@ void dce110_tg_set_overscan_color(struct timing_generator *tg,
void dce110_tg_program_timing(struct timing_generator *tg,
const struct dc_crtc_timing *timing,
+ int vready_offset,
+ int vstartup_start,
+ int vupdate_offset,
+ int vupdate_width,
+ const enum signal_type signal,
bool use_vbios)
{
if (use_vbios)
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.h b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.h
index 734d4965dab1..768ccf27ada9 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.h
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.h
@@ -256,6 +256,11 @@ void dce110_tg_set_overscan_color(struct timing_generator *tg,
void dce110_tg_program_timing(struct timing_generator *tg,
const struct dc_crtc_timing *timing,
+ int vready_offset,
+ int vstartup_start,
+ int vupdate_offset,
+ int vupdate_width,
+ const enum signal_type signal,
bool use_vbios);
bool dce110_tg_is_blanked(struct timing_generator *tg);
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator_v.c
index a3cef60380ed..a13a2f58944e 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator_v.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator_v.c
@@ -435,6 +435,11 @@ static void dce110_timing_generator_v_set_blank(struct timing_generator *tg,
static void dce110_timing_generator_v_program_timing(struct timing_generator *tg,
const struct dc_crtc_timing *timing,
+ int vready_offset,
+ int vstartup_start,
+ int vupdate_offset,
+ int vupdate_width,
+ const enum signal_type signal,
bool use_vbios)
{
if (use_vbios)
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c
index aa8d6b10d2c3..b1aaab5590cc 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/delay.h>
+
#include "dce110_transform_v.h"
#include "dm_services.h"
#include "dc.h"
diff --git a/drivers/gpu/drm/amd/display/dc/dce112/dce112_compressor.c b/drivers/gpu/drm/amd/display/dc/dce112/dce112_compressor.c
index faae12cf7968..51cb45d8b9ab 100644
--- a/drivers/gpu/drm/amd/display/dc/dce112/dce112_compressor.c
+++ b/drivers/gpu/drm/amd/display/dc/dce112/dce112_compressor.c
@@ -23,6 +23,9 @@
*
*/
+#include <linux/delay.h>
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "dce/dce_11_2_d.h"
diff --git a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c
index a480b15f6885..c6136e0ed1a4 100644
--- a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "link_encoder.h"
@@ -34,8 +36,6 @@
#include "dce110/dce110_timing_generator.h"
#include "irq/dce110/irq_service_dce110.h"
-
-#include "dce/dce_clk_mgr.h"
#include "dce/dce_mem_input.h"
#include "dce/dce_transform.h"
#include "dce/dce_link_encoder.h"
@@ -148,19 +148,6 @@ static const struct dce110_timing_generator_offsets dce112_tg_offsets[] = {
#define SRI(reg_name, block, id)\
.reg_name = mm ## block ## id ## _ ## reg_name
-
-static const struct clk_mgr_registers disp_clk_regs = {
- CLK_COMMON_REG_LIST_DCE_BASE()
-};
-
-static const struct clk_mgr_shift disp_clk_shift = {
- CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
-};
-
-static const struct clk_mgr_mask disp_clk_mask = {
- CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
-};
-
static const struct dce_dmcu_registers dmcu_regs = {
DMCU_DCE110_COMMON_REG_LIST()
};
@@ -774,9 +761,6 @@ static void destruct(struct dce110_resource_pool *pool)
if (pool->base.dmcu != NULL)
dce_dmcu_destroy(&pool->base.dmcu);
- if (pool->base.clk_mgr != NULL)
- dce_clk_mgr_destroy(&pool->base.clk_mgr);
-
if (pool->base.irqs != NULL) {
dal_irq_service_destroy(&pool->base.irqs);
}
@@ -993,7 +977,8 @@ static const struct resource_funcs dce112_res_pool_funcs = {
.validate_bandwidth = dce112_validate_bandwidth,
.validate_plane = dce100_validate_plane,
.add_stream_to_ctx = dce112_add_stream_to_ctx,
- .validate_global = dce112_validate_global
+ .validate_global = dce112_validate_global,
+ .find_first_free_match_stream_enc_for_link = dce110_find_first_free_match_stream_enc_for_link
};
static void bw_calcs_data_update_from_pplib(struct dc *dc)
@@ -1224,16 +1209,6 @@ static bool construct(
}
}
- pool->base.clk_mgr = dce112_clk_mgr_create(ctx,
- &disp_clk_regs,
- &disp_clk_shift,
- &disp_clk_mask);
- if (pool->base.clk_mgr == NULL) {
- dm_error("DC: failed to create display clock!\n");
- BREAK_TO_DEBUGGER();
- goto res_create_fail;
- }
-
pool->base.dmcu = dce_dmcu_create(ctx,
&dmcu_regs,
&dmcu_shift,
diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c
index 6d49c7143c67..4a6ba3173a5a 100644
--- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c
@@ -24,6 +24,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
@@ -46,8 +48,7 @@
#include "dce110/dce110_hw_sequencer.h"
#include "dce120/dce120_hw_sequencer.h"
#include "dce/dce_transform.h"
-
-#include "dce/dce_clk_mgr.h"
+#include "clk_mgr.h"
#include "dce/dce_audio.h"
#include "dce/dce_link_encoder.h"
#include "dce/dce_stream_encoder.h"
@@ -480,7 +481,7 @@ static const struct dc_debug_options debug_defaults = {
.disable_clock_gate = true,
};
-struct clock_source *dce120_clock_source_create(
+static struct clock_source *dce120_clock_source_create(
struct dc_context *ctx,
struct dc_bios *bios,
enum clock_source_id id,
@@ -503,14 +504,14 @@ struct clock_source *dce120_clock_source_create(
return NULL;
}
-void dce120_clock_source_destroy(struct clock_source **clk_src)
+static void dce120_clock_source_destroy(struct clock_source **clk_src)
{
kfree(TO_DCE110_CLK_SRC(*clk_src));
*clk_src = NULL;
}
-bool dce120_hw_sequencer_create(struct dc *dc)
+static bool dce120_hw_sequencer_create(struct dc *dc)
{
/* All registers used by dce11.2 match those in dce11 in offset and
* structure
@@ -609,9 +610,6 @@ static void destruct(struct dce110_resource_pool *pool)
if (pool->base.dmcu != NULL)
dce_dmcu_destroy(&pool->base.dmcu);
-
- if (pool->base.clk_mgr != NULL)
- dce_clk_mgr_destroy(&pool->base.clk_mgr);
}
static void read_dce_straps(
@@ -837,7 +835,8 @@ static const struct resource_funcs dce120_res_pool_funcs = {
.link_enc_create = dce120_link_encoder_create,
.validate_bandwidth = dce112_validate_bandwidth,
.validate_plane = dce100_validate_plane,
- .add_stream_to_ctx = dce112_add_stream_to_ctx
+ .add_stream_to_ctx = dce112_add_stream_to_ctx,
+ .find_first_free_match_stream_enc_for_link = dce110_find_first_free_match_stream_enc_for_link
};
static void bw_calcs_data_update_from_pplib(struct dc *dc)
@@ -1047,17 +1046,6 @@ static bool construct(
}
}
- if (is_vg20)
- pool->base.clk_mgr = dce121_clk_mgr_create(ctx);
- else
- pool->base.clk_mgr = dce120_clk_mgr_create(ctx);
-
- if (pool->base.clk_mgr == NULL) {
- dm_error("DC: failed to create display clock!\n");
- BREAK_TO_DEBUGGER();
- goto dccg_create_fail;
- }
-
pool->base.dmcu = dce_dmcu_create(ctx,
&dmcu_regs,
&dmcu_shift,
@@ -1177,16 +1165,6 @@ static bool construct(
if (!resource_construct(num_virtual_links, dc, &pool->base, res_funcs))
goto res_create_fail;
- /*
- * This is a bit of a hack. The xGMI enabled info is used to determine
- * if audio and display clocks need to be adjusted with the WAFL link's
- * SS info. This is a responsiblity of the clk_mgr. But since MMHUB is
- * under hwseq, and the relevant register is in MMHUB, we have to do it
- * here.
- */
- if (is_vg20 && dce121_xgmi_enabled(dc->hwseq))
- dce121_clock_patch_xgmi_ss_info(pool->base.clk_mgr);
-
/* Create hardware sequencer */
if (!dce120_hw_sequencer_create(dc))
goto controller_create_fail;
@@ -1204,7 +1182,6 @@ static bool construct(
irqs_create_fail:
controller_create_fail:
-dccg_create_fail:
clk_src_create_fail:
res_create_fail:
diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c
index 04b866f0fa1f..098e56962f2a 100644
--- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c
+++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c
@@ -734,8 +734,13 @@ void dce120_tg_set_overscan_color(struct timing_generator *tg,
CRTC_OVERSCAN_COLOR_RED, overscan_color->color_r_cr);
}
-void dce120_tg_program_timing(struct timing_generator *tg,
+static void dce120_tg_program_timing(struct timing_generator *tg,
const struct dc_crtc_timing *timing,
+ int vready_offset,
+ int vstartup_start,
+ int vupdate_offset,
+ int vupdate_width,
+ const enum signal_type signal,
bool use_vbios)
{
if (use_vbios)
@@ -1109,6 +1114,92 @@ static bool dce120_arm_vert_intr(
return true;
}
+
+static bool dce120_is_tg_enabled(struct timing_generator *tg)
+{
+ struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg);
+ uint32_t value, field;
+
+ value = dm_read_reg_soc15(tg->ctx, mmCRTC0_CRTC_CONTROL,
+ tg110->offsets.crtc);
+ field = get_reg_field_value(value, CRTC0_CRTC_CONTROL,
+ CRTC_CURRENT_MASTER_EN_STATE);
+
+ return field == 1;
+}
+
+static bool dce120_configure_crc(struct timing_generator *tg,
+ const struct crc_params *params)
+{
+ struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg);
+
+ /* Cannot configure crc on a CRTC that is disabled */
+ if (!dce120_is_tg_enabled(tg))
+ return false;
+
+ /* First, disable CRC before we configure it. */
+ dm_write_reg_soc15(tg->ctx, mmCRTC0_CRTC_CRC_CNTL,
+ tg110->offsets.crtc, 0);
+
+ if (!params->enable)
+ return true;
+
+ /* Program frame boundaries */
+ /* Window A x axis start and end. */
+ CRTC_REG_UPDATE_2(CRTC0_CRTC_CRC0_WINDOWA_X_CONTROL,
+ CRTC_CRC0_WINDOWA_X_START, params->windowa_x_start,
+ CRTC_CRC0_WINDOWA_X_END, params->windowa_x_end);
+
+ /* Window A y axis start and end. */
+ CRTC_REG_UPDATE_2(CRTC0_CRTC_CRC0_WINDOWA_Y_CONTROL,
+ CRTC_CRC0_WINDOWA_Y_START, params->windowa_y_start,
+ CRTC_CRC0_WINDOWA_Y_END, params->windowa_y_end);
+
+ /* Window B x axis start and end. */
+ CRTC_REG_UPDATE_2(CRTC0_CRTC_CRC0_WINDOWB_X_CONTROL,
+ CRTC_CRC0_WINDOWB_X_START, params->windowb_x_start,
+ CRTC_CRC0_WINDOWB_X_END, params->windowb_x_end);
+
+ /* Window B y axis start and end. */
+ CRTC_REG_UPDATE_2(CRTC0_CRTC_CRC0_WINDOWB_Y_CONTROL,
+ CRTC_CRC0_WINDOWB_Y_START, params->windowb_y_start,
+ CRTC_CRC0_WINDOWB_Y_END, params->windowb_y_end);
+
+ /* Set crc mode and selection, and enable. Only using CRC0*/
+ CRTC_REG_UPDATE_3(CRTC0_CRTC_CRC_CNTL,
+ CRTC_CRC_EN, params->continuous_mode ? 1 : 0,
+ CRTC_CRC0_SELECT, params->selection,
+ CRTC_CRC_EN, 1);
+
+ return true;
+}
+
+static bool dce120_get_crc(struct timing_generator *tg, uint32_t *r_cr,
+ uint32_t *g_y, uint32_t *b_cb)
+{
+ struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg);
+ uint32_t value, field;
+
+ value = dm_read_reg_soc15(tg->ctx, mmCRTC0_CRTC_CRC_CNTL,
+ tg110->offsets.crtc);
+ field = get_reg_field_value(value, CRTC0_CRTC_CRC_CNTL, CRTC_CRC_EN);
+
+ /* Early return if CRC is not enabled for this CRTC */
+ if (!field)
+ return false;
+
+ value = dm_read_reg_soc15(tg->ctx, mmCRTC0_CRTC_CRC0_DATA_RG,
+ tg110->offsets.crtc);
+ *r_cr = get_reg_field_value(value, CRTC0_CRTC_CRC0_DATA_RG, CRC0_R_CR);
+ *g_y = get_reg_field_value(value, CRTC0_CRTC_CRC0_DATA_RG, CRC0_G_Y);
+
+ value = dm_read_reg_soc15(tg->ctx, mmCRTC0_CRTC_CRC0_DATA_B,
+ tg110->offsets.crtc);
+ *b_cb = get_reg_field_value(value, CRTC0_CRTC_CRC0_DATA_B, CRC0_B_CB);
+
+ return true;
+}
+
static const struct timing_generator_funcs dce120_tg_funcs = {
.validate_timing = dce120_tg_validate_timing,
.program_timing = dce120_tg_program_timing,
@@ -1140,6 +1231,9 @@ static const struct timing_generator_funcs dce120_tg_funcs = {
.set_static_screen_control = dce120_timing_generator_set_static_screen_control,
.set_test_pattern = dce120_timing_generator_set_test_pattern,
.arm_vert_intr = dce120_arm_vert_intr,
+ .is_tg_enabled = dce120_is_tg_enabled,
+ .configure_crc = dce120_configure_crc,
+ .get_crc = dce120_get_crc,
};
diff --git a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c
index 27d0cc394963..860a524ebcfa 100644
--- a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dce/dce_8_0_d.h"
#include "dce/dce_8_0_sh_mask.h"
@@ -37,7 +39,6 @@
#include "dce110/dce110_timing_generator.h"
#include "dce110/dce110_resource.h"
#include "dce80/dce80_timing_generator.h"
-#include "dce/dce_clk_mgr.h"
#include "dce/dce_mem_input.h"
#include "dce/dce_link_encoder.h"
#include "dce/dce_stream_encoder.h"
@@ -154,19 +155,6 @@ static const struct dce110_timing_generator_offsets dce80_tg_offsets[] = {
#define SRI(reg_name, block, id)\
.reg_name = mm ## block ## id ## _ ## reg_name
-
-static const struct clk_mgr_registers disp_clk_regs = {
- CLK_COMMON_REG_LIST_DCE_BASE()
-};
-
-static const struct clk_mgr_shift disp_clk_shift = {
- CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
-};
-
-static const struct clk_mgr_mask disp_clk_mask = {
- CLK_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
-};
-
#define ipp_regs(id)\
[id] = {\
IPP_COMMON_REG_LIST_DCE_BASE(id)\
@@ -802,9 +790,6 @@ static void destruct(struct dce110_resource_pool *pool)
}
}
- if (pool->base.clk_mgr != NULL)
- dce_clk_mgr_destroy(&pool->base.clk_mgr);
-
if (pool->base.irqs != NULL) {
dal_irq_service_destroy(&pool->base.irqs);
}
@@ -880,7 +865,8 @@ static const struct resource_funcs dce80_res_pool_funcs = {
.validate_bandwidth = dce80_validate_bandwidth,
.validate_plane = dce100_validate_plane,
.add_stream_to_ctx = dce100_add_stream_to_ctx,
- .validate_global = dce80_validate_global
+ .validate_global = dce80_validate_global,
+ .find_first_free_match_stream_enc_for_link = dce100_find_first_free_match_stream_enc_for_link
};
static bool dce80_construct(
@@ -954,16 +940,6 @@ static bool dce80_construct(
}
}
- pool->base.clk_mgr = dce_clk_mgr_create(ctx,
- &disp_clk_regs,
- &disp_clk_shift,
- &disp_clk_mask);
- if (pool->base.clk_mgr == NULL) {
- dm_error("DC: failed to create display clock!\n");
- BREAK_TO_DEBUGGER();
- goto res_create_fail;
- }
-
pool->base.dmcu = dce_dmcu_create(ctx,
&dmcu_regs,
&dmcu_shift,
@@ -1163,16 +1139,6 @@ static bool dce81_construct(
}
}
- pool->base.clk_mgr = dce_clk_mgr_create(ctx,
- &disp_clk_regs,
- &disp_clk_shift,
- &disp_clk_mask);
- if (pool->base.clk_mgr == NULL) {
- dm_error("DC: failed to create display clock!\n");
- BREAK_TO_DEBUGGER();
- goto res_create_fail;
- }
-
pool->base.dmcu = dce_dmcu_create(ctx,
&dmcu_regs,
&dmcu_shift,
@@ -1368,16 +1334,6 @@ static bool dce83_construct(
}
}
- pool->base.clk_mgr = dce_clk_mgr_create(ctx,
- &disp_clk_regs,
- &disp_clk_shift,
- &disp_clk_mask);
- if (pool->base.clk_mgr == NULL) {
- dm_error("DC: failed to create display clock!\n");
- BREAK_TO_DEBUGGER();
- goto res_create_fail;
- }
-
pool->base.dmcu = dce_dmcu_create(ctx,
&dmcu_regs,
&dmcu_shift,
diff --git a/drivers/gpu/drm/amd/display/dc/dce80/dce80_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dce80/dce80_timing_generator.c
index 8b5ce557ee71..397e7f94e1e8 100644
--- a/drivers/gpu/drm/amd/display/dc/dce80/dce80_timing_generator.c
+++ b/drivers/gpu/drm/amd/display/dc/dce80/dce80_timing_generator.c
@@ -107,12 +107,17 @@ static void program_pix_dur(struct timing_generator *tg, uint32_t pix_clk_100hz)
static void program_timing(struct timing_generator *tg,
const struct dc_crtc_timing *timing,
+ int vready_offset,
+ int vstartup_start,
+ int vupdate_offset,
+ int vupdate_width,
+ const enum signal_type signal,
bool use_vbios)
{
if (!use_vbios)
program_pix_dur(tg, timing->pix_clk_100hz);
- dce110_tg_program_timing(tg, timing, use_vbios);
+ dce110_tg_program_timing(tg, timing, 0, 0, 0, 0, 0, use_vbios);
}
static void dce80_timing_generator_enable_advanced_request(
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/Makefile b/drivers/gpu/drm/amd/display/dc/dcn10/Makefile
index 55f293c8a3c0..032f872be89c 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/Makefile
@@ -24,7 +24,7 @@
DCN10 = dcn10_resource.o dcn10_ipp.o dcn10_hw_sequencer.o dcn10_hw_sequencer_debug.o \
dcn10_dpp.o dcn10_opp.o dcn10_optc.o \
- dcn10_hubp.o dcn10_mpc.o dcn10_clk_mgr.o \
+ dcn10_hubp.o dcn10_mpc.o \
dcn10_dpp_dscl.o dcn10_dpp_cm.o dcn10_cm_common.o \
dcn10_hubbub.o dcn10_stream_encoder.o dcn10_link_encoder.o
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.h
index 5ae4d69391a5..3b8cd7410498 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.h
@@ -38,6 +38,22 @@
type exp_resion_start_segment;\
type field_region_linear_slope
+#define TF_HELPER_REG_LIST \
+ uint32_t start_cntl_b; \
+ uint32_t start_cntl_g; \
+ uint32_t start_cntl_r; \
+ uint32_t start_slope_cntl_b; \
+ uint32_t start_slope_cntl_g; \
+ uint32_t start_slope_cntl_r; \
+ uint32_t start_end_cntl1_b; \
+ uint32_t start_end_cntl2_b; \
+ uint32_t start_end_cntl1_g; \
+ uint32_t start_end_cntl2_g; \
+ uint32_t start_end_cntl1_r; \
+ uint32_t start_end_cntl2_r; \
+ uint32_t region_start; \
+ uint32_t region_end
+
#define TF_CM_REG_FIELD_LIST(type) \
type csc_c11; \
type csc_c12
@@ -54,20 +70,7 @@ struct xfer_func_reg {
struct xfer_func_shift shifts;
struct xfer_func_mask masks;
- uint32_t start_cntl_b;
- uint32_t start_cntl_g;
- uint32_t start_cntl_r;
- uint32_t start_slope_cntl_b;
- uint32_t start_slope_cntl_g;
- uint32_t start_slope_cntl_r;
- uint32_t start_end_cntl1_b;
- uint32_t start_end_cntl2_b;
- uint32_t start_end_cntl1_g;
- uint32_t start_end_cntl2_g;
- uint32_t start_end_cntl1_r;
- uint32_t start_end_cntl2_r;
- uint32_t region_start;
- uint32_t region_end;
+ TF_HELPER_REG_LIST;
};
struct cm_color_matrix_shift {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
index 6f4b24756323..b95ec73fcae3 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
@@ -290,7 +290,12 @@ void dpp1_cnv_setup (
enum surface_pixel_format format,
enum expansion_mode mode,
struct dc_csc_transform input_csc_color_matrix,
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ enum dc_color_space input_color_space,
+ struct cnv_alpha_2bit_lut *alpha_2bit_lut)
+#else
enum dc_color_space input_color_space)
+#endif
{
uint32_t pixel_format;
uint32_t alpha_en;
@@ -523,6 +528,11 @@ static const struct dpp_funcs dcn10_dpp_funcs = {
.set_optional_cursor_attributes = dpp1_cnv_set_optional_cursor_attributes,
.dpp_dppclk_control = dpp1_dppclk_control,
.dpp_set_hdr_multiplier = dpp1_set_hdr_multiplier,
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ .dpp_program_blnd_lut = NULL,
+ .dpp_program_shaper_lut = NULL,
+ .dpp_program_3dlut = NULL
+#endif
};
static struct dpp_caps dcn10_dpp_cap = {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h
index 282e22f9b175..8a5517eebb7c 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h
@@ -1486,7 +1486,12 @@ void dpp1_cnv_setup (
enum surface_pixel_format format,
enum expansion_mode mode,
struct dc_csc_transform input_csc_color_matrix,
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ enum dc_color_space input_color_space,
+ struct cnv_alpha_2bit_lut *alpha_2bit_lut);
+#else
enum dc_color_space input_color_space);
+#endif
void dpp1_full_bypass(struct dpp *dpp_base);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c
index 882bcc5a40f6..aa0c7a7d13a0 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c
@@ -731,6 +731,10 @@ void dpp1_full_bypass(struct dpp *dpp_base)
/* COLOR_KEYER_CONTROL.COLOR_KEYER_EN = 0 this should be default */
if (dpp->tf_mask->CM_BYPASS_EN)
REG_SET(CM_CONTROL, 0, CM_BYPASS_EN, 1);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ else
+ REG_SET(CM_CONTROL, 0, CM_BYPASS, 1);
+#endif
/* Setting degamma bypass for now */
REG_SET(CM_DGAM_CONTROL, 0, CM_DGAM_LUT_MODE, 0);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
index ce21a290bf3e..d67e0abeee93 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
@@ -218,6 +218,14 @@ static void dpp1_dscl_set_lb(
INTERLEAVE_EN, lb_params->interleave_en, /* Interleave source enable */
LB_DATA_FORMAT__ALPHA_EN, lb_params->alpha_en); /* Alpha enable */
}
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ else {
+ /* DSCL caps: pixel data processed in float format */
+ REG_SET_2(LB_DATA_FORMAT, 0,
+ INTERLEAVE_EN, lb_params->interleave_en, /* Interleave source enable */
+ LB_DATA_FORMAT__ALPHA_EN, lb_params->alpha_en); /* Alpha enable */
+ }
+#endif
REG_SET_2(LB_MEMORY_CTRL, 0,
MEMORY_CONFIG, mem_size_config,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dwb.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dwb.c
new file mode 100644
index 000000000000..374cc9acda3b
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dwb.c
@@ -0,0 +1,136 @@
+/*
+ * Copyright 2012-17 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
+
+#include "reg_helper.h"
+#include "resource.h"
+#include "dwb.h"
+#include "dcn10_dwb.h"
+
+
+#define REG(reg)\
+ dwbc10->dwbc_regs->reg
+
+#define CTX \
+ dwbc10->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name) \
+ dwbc10->dwbc_shift->field_name, dwbc10->dwbc_mask->field_name
+
+#define TO_DCN10_DWBC(dwbc_base) \
+ container_of(dwbc_base, struct dcn10_dwbc, base)
+
+static bool dwb1_get_caps(struct dwbc *dwbc, struct dwb_caps *caps)
+{
+ if (caps) {
+ caps->adapter_id = 0; /* we only support 1 adapter currently */
+ caps->hw_version = DCN_VERSION_1_0;
+ caps->num_pipes = 2;
+ memset(&caps->reserved, 0, sizeof(caps->reserved));
+ memset(&caps->reserved2, 0, sizeof(caps->reserved2));
+ caps->sw_version = dwb_ver_1_0;
+ caps->caps.support_dwb = true;
+ caps->caps.support_ogam = false;
+ caps->caps.support_wbscl = true;
+ caps->caps.support_ocsc = false;
+ return true;
+ } else {
+ return false;
+ }
+}
+
+static bool dwb1_enable(struct dwbc *dwbc, struct dc_dwb_params *params)
+{
+ struct dcn10_dwbc *dwbc10 = TO_DCN10_DWBC(dwbc);
+
+ /* disable first. */
+ dwbc->funcs->disable(dwbc);
+
+ /* disable power gating */
+ REG_UPDATE_5(WB_EC_CONFIG, DISPCLK_R_WB_GATE_DIS, 1,
+ DISPCLK_G_WB_GATE_DIS, 1, DISPCLK_G_WBSCL_GATE_DIS, 1,
+ WB_LB_LS_DIS, 1, WB_LUT_LS_DIS, 1);
+
+ REG_UPDATE(WB_ENABLE, WB_ENABLE, 1);
+
+ return true;
+}
+
+static bool dwb1_disable(struct dwbc *dwbc)
+{
+ struct dcn10_dwbc *dwbc10 = TO_DCN10_DWBC(dwbc);
+
+ /* disable CNV */
+ REG_UPDATE(CNV_MODE, CNV_FRAME_CAPTURE_EN, 0);
+
+ /* disable WB */
+ REG_UPDATE(WB_ENABLE, WB_ENABLE, 0);
+
+ /* soft reset */
+ REG_UPDATE(WB_SOFT_RESET, WB_SOFT_RESET, 1);
+ REG_UPDATE(WB_SOFT_RESET, WB_SOFT_RESET, 0);
+
+ /* enable power gating */
+ REG_UPDATE_5(WB_EC_CONFIG, DISPCLK_R_WB_GATE_DIS, 0,
+ DISPCLK_G_WB_GATE_DIS, 0, DISPCLK_G_WBSCL_GATE_DIS, 0,
+ WB_LB_LS_DIS, 0, WB_LUT_LS_DIS, 0);
+
+ return true;
+}
+
+const struct dwbc_funcs dcn10_dwbc_funcs = {
+ .get_caps = dwb1_get_caps,
+ .enable = dwb1_enable,
+ .disable = dwb1_disable,
+ .update = NULL,
+ .set_stereo = NULL,
+ .set_new_content = NULL,
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ .set_warmup = NULL,
+#endif
+ .dwb_set_scaler = NULL,
+};
+
+void dcn10_dwbc_construct(struct dcn10_dwbc *dwbc10,
+ struct dc_context *ctx,
+ const struct dcn10_dwbc_registers *dwbc_regs,
+ const struct dcn10_dwbc_shift *dwbc_shift,
+ const struct dcn10_dwbc_mask *dwbc_mask,
+ int inst)
+{
+ dwbc10->base.ctx = ctx;
+
+ dwbc10->base.inst = inst;
+ dwbc10->base.funcs = &dcn10_dwbc_funcs;
+
+ dwbc10->dwbc_regs = dwbc_regs;
+ dwbc10->dwbc_shift = dwbc_shift;
+ dwbc10->dwbc_mask = dwbc_mask;
+}
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dwb.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dwb.h
new file mode 100644
index 000000000000..c175edd0bae7
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dwb.h
@@ -0,0 +1,271 @@
+/* Copyright 2012-17 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+#ifndef __DC_DWBC_DCN10_H__
+#define __DC_DWBC_DCN10_H__
+
+#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
+
+/* DCN */
+#define BASE_INNER(seg) \
+ DCE_BASE__INST0_SEG ## seg
+
+#define BASE(seg) \
+ BASE_INNER(seg)
+
+#define SR(reg_name)\
+ .reg_name = BASE(mm ## reg_name ## _BASE_IDX) + \
+ mm ## reg_name
+
+#define SRI(reg_name, block, id)\
+ .reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+ mm ## block ## id ## _ ## reg_name
+
+
+#define SRII(reg_name, block, id)\
+ .reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+ mm ## block ## id ## _ ## reg_name
+
+#define SF(reg_name, field_name, post_fix)\
+ .field_name = reg_name ## __ ## field_name ## post_fix
+
+
+#define DWBC_COMMON_REG_LIST_DCN1_0(inst) \
+ SRI(WB_ENABLE, CNV, inst),\
+ SRI(WB_EC_CONFIG, CNV, inst),\
+ SRI(CNV_MODE, CNV, inst),\
+ SRI(WB_SOFT_RESET, CNV, inst),\
+ SRI(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_PITCH, MCIF_WB, inst),\
+ SRI(MCIF_WB_ARBITRATION_CONTROL, MCIF_WB, inst),\
+ SRI(MCIF_WB_SCLK_CHANGE, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_1_ADDR_Y, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_1_ADDR_Y_OFFSET, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_1_ADDR_C, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_1_ADDR_C_OFFSET, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_2_ADDR_Y, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_2_ADDR_Y_OFFSET, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_2_ADDR_C, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_2_ADDR_C_OFFSET, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_3_ADDR_Y, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_3_ADDR_Y_OFFSET, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_3_ADDR_C, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_3_ADDR_C_OFFSET, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_4_ADDR_Y, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_4_ADDR_Y_OFFSET, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_4_ADDR_C, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_4_ADDR_C_OFFSET, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB, inst),\
+ SRI(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, MCIF_WB, inst),\
+ SRI(MCIF_WB_NB_PSTATE_CONTROL, MCIF_WB, inst),\
+ SRI(MCIF_WB_WATERMARK, MCIF_WB, inst),\
+ SRI(MCIF_WB_WARM_UP_CNTL, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_LUMA_SIZE, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_CHROMA_SIZE, MCIF_WB, inst)
+
+#define DWBC_COMMON_MASK_SH_LIST_DCN1_0(mask_sh) \
+ SF(CNV0_WB_ENABLE, WB_ENABLE, mask_sh),\
+ SF(CNV0_WB_EC_CONFIG, DISPCLK_R_WB_GATE_DIS, mask_sh),\
+ SF(CNV0_WB_EC_CONFIG, DISPCLK_G_WB_GATE_DIS, mask_sh),\
+ SF(CNV0_WB_EC_CONFIG, DISPCLK_G_WBSCL_GATE_DIS, mask_sh),\
+ SF(CNV0_WB_EC_CONFIG, WB_LB_LS_DIS, mask_sh),\
+ SF(CNV0_WB_EC_CONFIG, WB_LUT_LS_DIS, mask_sh),\
+ SF(CNV0_CNV_MODE, CNV_WINDOW_CROP_EN, mask_sh),\
+ SF(CNV0_CNV_MODE, CNV_STEREO_TYPE, mask_sh),\
+ SF(CNV0_CNV_MODE, CNV_INTERLACED_MODE, mask_sh),\
+ SF(CNV0_CNV_MODE, CNV_EYE_SELECTION, mask_sh),\
+ SF(CNV0_CNV_MODE, CNV_STEREO_POLARITY, mask_sh),\
+ SF(CNV0_CNV_MODE, CNV_INTERLACED_FIELD_ORDER, mask_sh),\
+ SF(CNV0_CNV_MODE, CNV_STEREO_SPLIT, mask_sh),\
+ SF(CNV0_CNV_MODE, CNV_NEW_CONTENT, mask_sh),\
+ SF(CNV0_CNV_MODE, CNV_FRAME_CAPTURE_EN, mask_sh),\
+ SF(CNV0_WB_SOFT_RESET, WB_SOFT_RESET, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_ENABLE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUF_DUALSIZE_REQ, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_INT_EN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_INT_ACK, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_SLICE_INT_EN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_OVERRUN_INT_EN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_LOCK, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_P_VMID, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUF_ADDR_FENCE_EN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_PITCH, MCIF_WB_BUF_LUMA_PITCH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_PITCH, MCIF_WB_BUF_CHROMA_PITCH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_ARBITRATION_CONTROL, MCIF_WB_CLIENT_ARBITRATION_SLICE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_ARBITRATION_CONTROL, MCIF_WB_TIME_PER_PIXEL, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_SCLK_CHANGE, WM_CHANGE_ACK_FORCE_ON, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_SCLK_CHANGE, MCIF_WB_CLI_WATERMARK_MASK, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_ADDR_Y, MCIF_WB_BUF_1_ADDR_Y, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_ADDR_Y_OFFSET, MCIF_WB_BUF_1_ADDR_Y_OFFSET, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_ADDR_C, MCIF_WB_BUF_1_ADDR_C, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_ADDR_C_OFFSET, MCIF_WB_BUF_1_ADDR_C_OFFSET, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_ADDR_Y, MCIF_WB_BUF_2_ADDR_Y, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_ADDR_Y_OFFSET, MCIF_WB_BUF_2_ADDR_Y_OFFSET, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_ADDR_C, MCIF_WB_BUF_2_ADDR_C, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_ADDR_C_OFFSET, MCIF_WB_BUF_2_ADDR_C_OFFSET, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_ADDR_Y, MCIF_WB_BUF_3_ADDR_Y, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_ADDR_Y_OFFSET, MCIF_WB_BUF_3_ADDR_Y_OFFSET, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_ADDR_C, MCIF_WB_BUF_3_ADDR_C, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_ADDR_C_OFFSET, MCIF_WB_BUF_3_ADDR_C_OFFSET, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_ADDR_Y, MCIF_WB_BUF_4_ADDR_Y, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_ADDR_Y_OFFSET, MCIF_WB_BUF_4_ADDR_Y_OFFSET, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_ADDR_C, MCIF_WB_BUF_4_ADDR_C, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_ADDR_C_OFFSET, MCIF_WB_BUF_4_ADDR_C_OFFSET, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_LOCK_IGNORE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_EN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_ACK, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_SLICE_INT_EN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_LOCK, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_SLICE_SIZE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, NB_PSTATE_CHANGE_REFRESH_WATERMARK, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_NB_PSTATE_CONTROL, NB_PSTATE_CHANGE_URGENT_DURING_REQUEST, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_NB_PSTATE_CONTROL, NB_PSTATE_CHANGE_FORCE_ON, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_NB_PSTATE_CONTROL, NB_PSTATE_ALLOW_FOR_URGENT, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_NB_PSTATE_CONTROL, NB_PSTATE_CHANGE_WATERMARK_MASK, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_WATERMARK, MCIF_WB_CLI_WATERMARK, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_WARM_UP_CNTL, MCIF_WB_PITCH_SIZE_WARMUP, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_LUMA_SIZE, MCIF_WB_BUF_LUMA_SIZE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_CHROMA_SIZE, MCIF_WB_BUF_CHROMA_SIZE, mask_sh)
+
+#define DWBC_REG_FIELD_LIST(type) \
+ type WB_ENABLE;\
+ type DISPCLK_R_WB_GATE_DIS;\
+ type DISPCLK_G_WB_GATE_DIS;\
+ type DISPCLK_G_WBSCL_GATE_DIS;\
+ type WB_LB_LS_DIS;\
+ type WB_LB_SD_DIS;\
+ type WB_LUT_LS_DIS;\
+ type CNV_WINDOW_CROP_EN;\
+ type CNV_STEREO_TYPE;\
+ type CNV_INTERLACED_MODE;\
+ type CNV_EYE_SELECTION;\
+ type CNV_STEREO_POLARITY;\
+ type CNV_INTERLACED_FIELD_ORDER;\
+ type CNV_STEREO_SPLIT;\
+ type CNV_NEW_CONTENT;\
+ type CNV_FRAME_CAPTURE_EN;\
+ type WB_SOFT_RESET;\
+ type MCIF_WB_BUFMGR_ENABLE;\
+ type MCIF_WB_BUF_DUALSIZE_REQ;\
+ type MCIF_WB_BUFMGR_SW_INT_EN;\
+ type MCIF_WB_BUFMGR_SW_INT_ACK;\
+ type MCIF_WB_BUFMGR_SW_SLICE_INT_EN;\
+ type MCIF_WB_BUFMGR_SW_OVERRUN_INT_EN;\
+ type MCIF_WB_BUFMGR_SW_LOCK;\
+ type MCIF_WB_P_VMID;\
+ type MCIF_WB_BUF_ADDR_FENCE_EN;\
+ type MCIF_WB_BUF_LUMA_PITCH;\
+ type MCIF_WB_BUF_CHROMA_PITCH;\
+ type MCIF_WB_CLIENT_ARBITRATION_SLICE;\
+ type MCIF_WB_TIME_PER_PIXEL;\
+ type WM_CHANGE_ACK_FORCE_ON;\
+ type MCIF_WB_CLI_WATERMARK_MASK;\
+ type MCIF_WB_BUF_1_ADDR_Y;\
+ type MCIF_WB_BUF_1_ADDR_Y_OFFSET;\
+ type MCIF_WB_BUF_1_ADDR_C;\
+ type MCIF_WB_BUF_1_ADDR_C_OFFSET;\
+ type MCIF_WB_BUF_2_ADDR_Y;\
+ type MCIF_WB_BUF_2_ADDR_Y_OFFSET;\
+ type MCIF_WB_BUF_2_ADDR_C;\
+ type MCIF_WB_BUF_2_ADDR_C_OFFSET;\
+ type MCIF_WB_BUF_3_ADDR_Y;\
+ type MCIF_WB_BUF_3_ADDR_Y_OFFSET;\
+ type MCIF_WB_BUF_3_ADDR_C;\
+ type MCIF_WB_BUF_3_ADDR_C_OFFSET;\
+ type MCIF_WB_BUF_4_ADDR_Y;\
+ type MCIF_WB_BUF_4_ADDR_Y_OFFSET;\
+ type MCIF_WB_BUF_4_ADDR_C;\
+ type MCIF_WB_BUF_4_ADDR_C_OFFSET;\
+ type MCIF_WB_BUFMGR_VCE_LOCK_IGNORE;\
+ type MCIF_WB_BUFMGR_VCE_INT_EN;\
+ type MCIF_WB_BUFMGR_VCE_INT_ACK;\
+ type MCIF_WB_BUFMGR_VCE_SLICE_INT_EN;\
+ type MCIF_WB_BUFMGR_VCE_LOCK;\
+ type MCIF_WB_BUFMGR_SLICE_SIZE;\
+ type NB_PSTATE_CHANGE_REFRESH_WATERMARK;\
+ type NB_PSTATE_CHANGE_URGENT_DURING_REQUEST;\
+ type NB_PSTATE_CHANGE_FORCE_ON;\
+ type NB_PSTATE_ALLOW_FOR_URGENT;\
+ type NB_PSTATE_CHANGE_WATERMARK_MASK;\
+ type MCIF_WB_CLI_WATERMARK;\
+ type MCIF_WB_CLI_CLOCK_GATER_OVERRIDE;\
+ type MCIF_WB_PITCH_SIZE_WARMUP;\
+ type MCIF_WB_BUF_LUMA_SIZE;\
+ type MCIF_WB_BUF_CHROMA_SIZE;\
+
+struct dcn10_dwbc_registers {
+ uint32_t WB_ENABLE;
+ uint32_t WB_EC_CONFIG;
+ uint32_t CNV_MODE;
+ uint32_t WB_SOFT_RESET;
+ uint32_t MCIF_WB_BUFMGR_SW_CONTROL;
+ uint32_t MCIF_WB_BUF_PITCH;
+ uint32_t MCIF_WB_ARBITRATION_CONTROL;
+ uint32_t MCIF_WB_SCLK_CHANGE;
+ uint32_t MCIF_WB_BUF_1_ADDR_Y;
+ uint32_t MCIF_WB_BUF_1_ADDR_Y_OFFSET;
+ uint32_t MCIF_WB_BUF_1_ADDR_C;
+ uint32_t MCIF_WB_BUF_1_ADDR_C_OFFSET;
+ uint32_t MCIF_WB_BUF_2_ADDR_Y;
+ uint32_t MCIF_WB_BUF_2_ADDR_Y_OFFSET;
+ uint32_t MCIF_WB_BUF_2_ADDR_C;
+ uint32_t MCIF_WB_BUF_2_ADDR_C_OFFSET;
+ uint32_t MCIF_WB_BUF_3_ADDR_Y;
+ uint32_t MCIF_WB_BUF_3_ADDR_Y_OFFSET;
+ uint32_t MCIF_WB_BUF_3_ADDR_C;
+ uint32_t MCIF_WB_BUF_3_ADDR_C_OFFSET;
+ uint32_t MCIF_WB_BUF_4_ADDR_Y;
+ uint32_t MCIF_WB_BUF_4_ADDR_Y_OFFSET;
+ uint32_t MCIF_WB_BUF_4_ADDR_C;
+ uint32_t MCIF_WB_BUF_4_ADDR_C_OFFSET;
+ uint32_t MCIF_WB_BUFMGR_VCE_CONTROL;
+ uint32_t MCIF_WB_NB_PSTATE_LATENCY_WATERMARK;
+ uint32_t MCIF_WB_NB_PSTATE_CONTROL;
+ uint32_t MCIF_WB_WATERMARK;
+ uint32_t MCIF_WB_WARM_UP_CNTL;
+ uint32_t MCIF_WB_BUF_LUMA_SIZE;
+ uint32_t MCIF_WB_BUF_CHROMA_SIZE;
+};
+struct dcn10_dwbc_mask {
+ DWBC_REG_FIELD_LIST(uint32_t)
+};
+struct dcn10_dwbc_shift {
+ DWBC_REG_FIELD_LIST(uint8_t)
+};
+struct dcn10_dwbc {
+ struct dwbc base;
+ const struct dcn10_dwbc_registers *dwbc_regs;
+ const struct dcn10_dwbc_shift *dwbc_shift;
+ const struct dcn10_dwbc_mask *dwbc_mask;
+};
+
+void dcn10_dwbc_construct(struct dcn10_dwbc *dwbc10,
+ struct dc_context *ctx,
+ const struct dcn10_dwbc_registers *dwbc_regs,
+ const struct dcn10_dwbc_shift *dwbc_shift,
+ const struct dcn10_dwbc_mask *dwbc_mask,
+ int inst);
+
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
index 0db2a6e96fc0..a780057e2dbc 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/delay.h>
+
#include "dm_services.h"
#include "dcn10_hubp.h"
#include "dcn10_hubbub.h"
@@ -180,8 +182,43 @@ bool hubbub1_verify_allow_pstate_change_high(
* 29: WB1 Allow Pstate Change
* 30: Arbiter's allow_pstate_change
* 31: SOC pstate change request"
- *
- * RV1:
+ */
+ /*DCN2.x:
+ HUBBUB:DCHUBBUB_TEST_ARB_DEBUG10 DCHUBBUBDEBUGIND:0xB
+ 0: Pipe0 Plane0 Allow P-state Change
+ 1: Pipe0 Plane1 Allow P-state Change
+ 2: Pipe0 Cursor0 Allow P-state Change
+ 3: Pipe0 Cursor1 Allow P-state Change
+ 4: Pipe1 Plane0 Allow P-state Change
+ 5: Pipe1 Plane1 Allow P-state Change
+ 6: Pipe1 Cursor0 Allow P-state Change
+ 7: Pipe1 Cursor1 Allow P-state Change
+ 8: Pipe2 Plane0 Allow P-state Change
+ 9: Pipe2 Plane1 Allow P-state Change
+ 10: Pipe2 Cursor0 Allow P-state Change
+ 11: Pipe2 Cursor1 Allow P-state Change
+ 12: Pipe3 Plane0 Allow P-state Change
+ 13: Pipe3 Plane1 Allow P-state Change
+ 14: Pipe3 Cursor0 Allow P-state Change
+ 15: Pipe3 Cursor1 Allow P-state Change
+ 16: Pipe4 Plane0 Allow P-state Change
+ 17: Pipe4 Plane1 Allow P-state Change
+ 18: Pipe4 Cursor0 Allow P-state Change
+ 19: Pipe4 Cursor1 Allow P-state Change
+ 20: Pipe5 Plane0 Allow P-state Change
+ 21: Pipe5 Plane1 Allow P-state Change
+ 22: Pipe5 Cursor0 Allow P-state Change
+ 23: Pipe5 Cursor1 Allow P-state Change
+ 24: Pipe6 Plane0 Allow P-state Change
+ 25: Pipe6 Plane1 Allow P-state Change
+ 26: Pipe6 Cursor0 Allow P-state Change
+ 27: Pipe6 Cursor1 Allow P-state Change
+ 28: WB0 Allow P-state Change
+ 29: WB1 Allow P-state Change
+ 30: Arbiter`s Allow P-state Change
+ 31: SOC P-state Change request
+ */
+ /* RV1:
* dchubbubdebugind, at: 0x7
* description "3-0: Pipe0 cursor0 QOS
* 7-4: Pipe1 cursor0 QOS
@@ -263,20 +300,15 @@ void hubbub1_wm_change_req_wa(struct hubbub *hubbub)
DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, 1);
}
-void hubbub1_program_watermarks(
+void hubbub1_program_urgent_watermarks(
struct hubbub *hubbub,
struct dcn_watermark_set *watermarks,
unsigned int refclk_mhz,
bool safe_to_lower)
{
struct dcn10_hubbub *hubbub1 = TO_DCN10_HUBBUB(hubbub);
- /*
- * Need to clamp to max of the register values (i.e. no wrap)
- * for dcn1, all wm registers are 21-bit wide
- */
uint32_t prog_wm_value;
-
/* Repeat for water mark set A, B, C and D. */
/* clock state A */
if (safe_to_lower || watermarks->a.urgent_ns > hubbub1->watermarks.a.urgent_ns) {
@@ -291,60 +323,14 @@ void hubbub1_program_watermarks(
watermarks->a.urgent_ns, prog_wm_value);
}
- if (REG(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A)) {
- if (safe_to_lower || watermarks->a.pte_meta_urgent_ns > hubbub1->watermarks.a.pte_meta_urgent_ns) {
- hubbub1->watermarks.a.pte_meta_urgent_ns = watermarks->a.pte_meta_urgent_ns;
- prog_wm_value = convert_and_clamp(watermarks->a.pte_meta_urgent_ns,
- refclk_mhz, 0x1fffff);
- REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A, prog_wm_value);
- DC_LOG_BANDWIDTH_CALCS("PTE_META_URGENCY_WATERMARK_A calculated =%d\n"
- "HW register value = 0x%x\n",
- watermarks->a.pte_meta_urgent_ns, prog_wm_value);
- }
- }
-
- if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A)) {
- if (safe_to_lower || watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns
- > hubbub1->watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns) {
- hubbub1->watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns =
- watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns;
- prog_wm_value = convert_and_clamp(
- watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns,
- refclk_mhz, 0x1fffff);
- REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, 0,
- DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, prog_wm_value);
- DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_A calculated =%d\n"
- "HW register value = 0x%x\n",
- watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
- }
-
- if (safe_to_lower || watermarks->a.cstate_pstate.cstate_exit_ns
- > hubbub1->watermarks.a.cstate_pstate.cstate_exit_ns) {
- hubbub1->watermarks.a.cstate_pstate.cstate_exit_ns =
- watermarks->a.cstate_pstate.cstate_exit_ns;
- prog_wm_value = convert_and_clamp(
- watermarks->a.cstate_pstate.cstate_exit_ns,
- refclk_mhz, 0x1fffff);
- REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, 0,
- DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, prog_wm_value);
- DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_A calculated =%d\n"
- "HW register value = 0x%x\n",
- watermarks->a.cstate_pstate.cstate_exit_ns, prog_wm_value);
- }
- }
-
- if (safe_to_lower || watermarks->a.cstate_pstate.pstate_change_ns
- > hubbub1->watermarks.a.cstate_pstate.pstate_change_ns) {
- hubbub1->watermarks.a.cstate_pstate.pstate_change_ns =
- watermarks->a.cstate_pstate.pstate_change_ns;
- prog_wm_value = convert_and_clamp(
- watermarks->a.cstate_pstate.pstate_change_ns,
+ if (safe_to_lower || watermarks->a.pte_meta_urgent_ns > hubbub1->watermarks.a.pte_meta_urgent_ns) {
+ hubbub1->watermarks.a.pte_meta_urgent_ns = watermarks->a.pte_meta_urgent_ns;
+ prog_wm_value = convert_and_clamp(watermarks->a.pte_meta_urgent_ns,
refclk_mhz, 0x1fffff);
- REG_SET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, 0,
- DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, prog_wm_value);
- DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_A calculated =%d\n"
- "HW register value = 0x%x\n\n",
- watermarks->a.cstate_pstate.pstate_change_ns, prog_wm_value);
+ REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("PTE_META_URGENCY_WATERMARK_A calculated =%d\n"
+ "HW register value = 0x%x\n",
+ watermarks->a.pte_meta_urgent_ns, prog_wm_value);
}
/* clock state B */
@@ -360,60 +346,14 @@ void hubbub1_program_watermarks(
watermarks->b.urgent_ns, prog_wm_value);
}
- if (REG(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B)) {
- if (safe_to_lower || watermarks->b.pte_meta_urgent_ns > hubbub1->watermarks.b.pte_meta_urgent_ns) {
- hubbub1->watermarks.b.pte_meta_urgent_ns = watermarks->b.pte_meta_urgent_ns;
- prog_wm_value = convert_and_clamp(watermarks->b.pte_meta_urgent_ns,
- refclk_mhz, 0x1fffff);
- REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B, prog_wm_value);
- DC_LOG_BANDWIDTH_CALCS("PTE_META_URGENCY_WATERMARK_B calculated =%d\n"
- "HW register value = 0x%x\n",
- watermarks->b.pte_meta_urgent_ns, prog_wm_value);
- }
- }
-
- if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B)) {
- if (safe_to_lower || watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns
- > hubbub1->watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns) {
- hubbub1->watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns =
- watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns;
- prog_wm_value = convert_and_clamp(
- watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns,
- refclk_mhz, 0x1fffff);
- REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, 0,
- DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, prog_wm_value);
- DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_B calculated =%d\n"
- "HW register value = 0x%x\n",
- watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
- }
-
- if (safe_to_lower || watermarks->b.cstate_pstate.cstate_exit_ns
- > hubbub1->watermarks.b.cstate_pstate.cstate_exit_ns) {
- hubbub1->watermarks.b.cstate_pstate.cstate_exit_ns =
- watermarks->b.cstate_pstate.cstate_exit_ns;
- prog_wm_value = convert_and_clamp(
- watermarks->b.cstate_pstate.cstate_exit_ns,
- refclk_mhz, 0x1fffff);
- REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, 0,
- DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, prog_wm_value);
- DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_B calculated =%d\n"
- "HW register value = 0x%x\n",
- watermarks->b.cstate_pstate.cstate_exit_ns, prog_wm_value);
- }
- }
-
- if (safe_to_lower || watermarks->b.cstate_pstate.pstate_change_ns
- > hubbub1->watermarks.b.cstate_pstate.pstate_change_ns) {
- hubbub1->watermarks.b.cstate_pstate.pstate_change_ns =
- watermarks->b.cstate_pstate.pstate_change_ns;
- prog_wm_value = convert_and_clamp(
- watermarks->b.cstate_pstate.pstate_change_ns,
+ if (safe_to_lower || watermarks->b.pte_meta_urgent_ns > hubbub1->watermarks.b.pte_meta_urgent_ns) {
+ hubbub1->watermarks.b.pte_meta_urgent_ns = watermarks->b.pte_meta_urgent_ns;
+ prog_wm_value = convert_and_clamp(watermarks->b.pte_meta_urgent_ns,
refclk_mhz, 0x1fffff);
- REG_SET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, 0,
- DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, prog_wm_value);
- DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_B calculated =%d\n"
- "HW register value = 0x%x\n\n",
- watermarks->b.cstate_pstate.pstate_change_ns, prog_wm_value);
+ REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("PTE_META_URGENCY_WATERMARK_B calculated =%d\n"
+ "HW register value = 0x%x\n",
+ watermarks->b.pte_meta_urgent_ns, prog_wm_value);
}
/* clock state C */
@@ -429,60 +369,14 @@ void hubbub1_program_watermarks(
watermarks->c.urgent_ns, prog_wm_value);
}
- if (REG(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C)) {
- if (safe_to_lower || watermarks->c.pte_meta_urgent_ns > hubbub1->watermarks.c.pte_meta_urgent_ns) {
- hubbub1->watermarks.c.pte_meta_urgent_ns = watermarks->c.pte_meta_urgent_ns;
- prog_wm_value = convert_and_clamp(watermarks->c.pte_meta_urgent_ns,
- refclk_mhz, 0x1fffff);
- REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C, prog_wm_value);
- DC_LOG_BANDWIDTH_CALCS("PTE_META_URGENCY_WATERMARK_C calculated =%d\n"
- "HW register value = 0x%x\n",
- watermarks->c.pte_meta_urgent_ns, prog_wm_value);
- }
- }
-
- if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C)) {
- if (safe_to_lower || watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns
- > hubbub1->watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns) {
- hubbub1->watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns =
- watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns;
- prog_wm_value = convert_and_clamp(
- watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns,
- refclk_mhz, 0x1fffff);
- REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, 0,
- DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, prog_wm_value);
- DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_C calculated =%d\n"
- "HW register value = 0x%x\n",
- watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
- }
-
- if (safe_to_lower || watermarks->c.cstate_pstate.cstate_exit_ns
- > hubbub1->watermarks.c.cstate_pstate.cstate_exit_ns) {
- hubbub1->watermarks.c.cstate_pstate.cstate_exit_ns =
- watermarks->c.cstate_pstate.cstate_exit_ns;
- prog_wm_value = convert_and_clamp(
- watermarks->c.cstate_pstate.cstate_exit_ns,
- refclk_mhz, 0x1fffff);
- REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, 0,
- DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, prog_wm_value);
- DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_C calculated =%d\n"
- "HW register value = 0x%x\n",
- watermarks->c.cstate_pstate.cstate_exit_ns, prog_wm_value);
- }
- }
-
- if (safe_to_lower || watermarks->c.cstate_pstate.pstate_change_ns
- > hubbub1->watermarks.c.cstate_pstate.pstate_change_ns) {
- hubbub1->watermarks.c.cstate_pstate.pstate_change_ns =
- watermarks->c.cstate_pstate.pstate_change_ns;
- prog_wm_value = convert_and_clamp(
- watermarks->c.cstate_pstate.pstate_change_ns,
+ if (safe_to_lower || watermarks->c.pte_meta_urgent_ns > hubbub1->watermarks.c.pte_meta_urgent_ns) {
+ hubbub1->watermarks.c.pte_meta_urgent_ns = watermarks->c.pte_meta_urgent_ns;
+ prog_wm_value = convert_and_clamp(watermarks->c.pte_meta_urgent_ns,
refclk_mhz, 0x1fffff);
- REG_SET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, 0,
- DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, prog_wm_value);
- DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_C calculated =%d\n"
- "HW register value = 0x%x\n\n",
- watermarks->c.cstate_pstate.pstate_change_ns, prog_wm_value);
+ REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("PTE_META_URGENCY_WATERMARK_C calculated =%d\n"
+ "HW register value = 0x%x\n",
+ watermarks->c.pte_meta_urgent_ns, prog_wm_value);
}
/* clock state D */
@@ -498,48 +392,199 @@ void hubbub1_program_watermarks(
watermarks->d.urgent_ns, prog_wm_value);
}
- if (REG(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D)) {
- if (safe_to_lower || watermarks->d.pte_meta_urgent_ns > hubbub1->watermarks.d.pte_meta_urgent_ns) {
- hubbub1->watermarks.d.pte_meta_urgent_ns = watermarks->d.pte_meta_urgent_ns;
- prog_wm_value = convert_and_clamp(watermarks->d.pte_meta_urgent_ns,
- refclk_mhz, 0x1fffff);
- REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D, prog_wm_value);
- DC_LOG_BANDWIDTH_CALCS("PTE_META_URGENCY_WATERMARK_D calculated =%d\n"
- "HW register value = 0x%x\n",
- watermarks->d.pte_meta_urgent_ns, prog_wm_value);
- }
+ if (safe_to_lower || watermarks->d.pte_meta_urgent_ns > hubbub1->watermarks.d.pte_meta_urgent_ns) {
+ hubbub1->watermarks.d.pte_meta_urgent_ns = watermarks->d.pte_meta_urgent_ns;
+ prog_wm_value = convert_and_clamp(watermarks->d.pte_meta_urgent_ns,
+ refclk_mhz, 0x1fffff);
+ REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("PTE_META_URGENCY_WATERMARK_D calculated =%d\n"
+ "HW register value = 0x%x\n",
+ watermarks->d.pte_meta_urgent_ns, prog_wm_value);
}
+}
- if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D)) {
- if (safe_to_lower || watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns
- > hubbub1->watermarks.d.cstate_pstate.cstate_enter_plus_exit_ns) {
- hubbub1->watermarks.d.cstate_pstate.cstate_enter_plus_exit_ns =
- watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns;
- prog_wm_value = convert_and_clamp(
- watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns,
- refclk_mhz, 0x1fffff);
- REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, 0,
- DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, prog_wm_value);
- DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_D calculated =%d\n"
- "HW register value = 0x%x\n",
- watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
- }
+void hubbub1_program_stutter_watermarks(
+ struct hubbub *hubbub,
+ struct dcn_watermark_set *watermarks,
+ unsigned int refclk_mhz,
+ bool safe_to_lower)
+{
+ struct dcn10_hubbub *hubbub1 = TO_DCN10_HUBBUB(hubbub);
+ uint32_t prog_wm_value;
- if (safe_to_lower || watermarks->d.cstate_pstate.cstate_exit_ns
- > hubbub1->watermarks.d.cstate_pstate.cstate_exit_ns) {
- hubbub1->watermarks.d.cstate_pstate.cstate_exit_ns =
- watermarks->d.cstate_pstate.cstate_exit_ns;
- prog_wm_value = convert_and_clamp(
- watermarks->d.cstate_pstate.cstate_exit_ns,
- refclk_mhz, 0x1fffff);
- REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, 0,
- DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, prog_wm_value);
- DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_D calculated =%d\n"
- "HW register value = 0x%x\n",
- watermarks->d.cstate_pstate.cstate_exit_ns, prog_wm_value);
- }
+ /* clock state A */
+ if (safe_to_lower || watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns
+ > hubbub1->watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns) {
+ hubbub1->watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns =
+ watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns,
+ refclk_mhz, 0x1fffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, 0,
+ DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_A calculated =%d\n"
+ "HW register value = 0x%x\n",
+ watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
+ }
+
+ if (safe_to_lower || watermarks->a.cstate_pstate.cstate_exit_ns
+ > hubbub1->watermarks.a.cstate_pstate.cstate_exit_ns) {
+ hubbub1->watermarks.a.cstate_pstate.cstate_exit_ns =
+ watermarks->a.cstate_pstate.cstate_exit_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->a.cstate_pstate.cstate_exit_ns,
+ refclk_mhz, 0x1fffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, 0,
+ DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_A calculated =%d\n"
+ "HW register value = 0x%x\n",
+ watermarks->a.cstate_pstate.cstate_exit_ns, prog_wm_value);
+ }
+
+ /* clock state B */
+ if (safe_to_lower || watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns
+ > hubbub1->watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns) {
+ hubbub1->watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns =
+ watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns,
+ refclk_mhz, 0x1fffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, 0,
+ DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_B calculated =%d\n"
+ "HW register value = 0x%x\n",
+ watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
+ }
+
+ if (safe_to_lower || watermarks->b.cstate_pstate.cstate_exit_ns
+ > hubbub1->watermarks.b.cstate_pstate.cstate_exit_ns) {
+ hubbub1->watermarks.b.cstate_pstate.cstate_exit_ns =
+ watermarks->b.cstate_pstate.cstate_exit_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->b.cstate_pstate.cstate_exit_ns,
+ refclk_mhz, 0x1fffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, 0,
+ DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_B calculated =%d\n"
+ "HW register value = 0x%x\n",
+ watermarks->b.cstate_pstate.cstate_exit_ns, prog_wm_value);
+ }
+
+ /* clock state C */
+ if (safe_to_lower || watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns
+ > hubbub1->watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns) {
+ hubbub1->watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns =
+ watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns,
+ refclk_mhz, 0x1fffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, 0,
+ DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_C calculated =%d\n"
+ "HW register value = 0x%x\n",
+ watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
+ }
+
+ if (safe_to_lower || watermarks->c.cstate_pstate.cstate_exit_ns
+ > hubbub1->watermarks.c.cstate_pstate.cstate_exit_ns) {
+ hubbub1->watermarks.c.cstate_pstate.cstate_exit_ns =
+ watermarks->c.cstate_pstate.cstate_exit_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->c.cstate_pstate.cstate_exit_ns,
+ refclk_mhz, 0x1fffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, 0,
+ DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_C calculated =%d\n"
+ "HW register value = 0x%x\n",
+ watermarks->c.cstate_pstate.cstate_exit_ns, prog_wm_value);
+ }
+
+ /* clock state D */
+ if (safe_to_lower || watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns
+ > hubbub1->watermarks.d.cstate_pstate.cstate_enter_plus_exit_ns) {
+ hubbub1->watermarks.d.cstate_pstate.cstate_enter_plus_exit_ns =
+ watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns,
+ refclk_mhz, 0x1fffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, 0,
+ DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_D calculated =%d\n"
+ "HW register value = 0x%x\n",
+ watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
+ }
+
+ if (safe_to_lower || watermarks->d.cstate_pstate.cstate_exit_ns
+ > hubbub1->watermarks.d.cstate_pstate.cstate_exit_ns) {
+ hubbub1->watermarks.d.cstate_pstate.cstate_exit_ns =
+ watermarks->d.cstate_pstate.cstate_exit_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->d.cstate_pstate.cstate_exit_ns,
+ refclk_mhz, 0x1fffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, 0,
+ DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_D calculated =%d\n"
+ "HW register value = 0x%x\n",
+ watermarks->d.cstate_pstate.cstate_exit_ns, prog_wm_value);
}
+}
+
+void hubbub1_program_pstate_watermarks(
+ struct hubbub *hubbub,
+ struct dcn_watermark_set *watermarks,
+ unsigned int refclk_mhz,
+ bool safe_to_lower)
+{
+ struct dcn10_hubbub *hubbub1 = TO_DCN10_HUBBUB(hubbub);
+ uint32_t prog_wm_value;
+
+ /* clock state A */
+ if (safe_to_lower || watermarks->a.cstate_pstate.pstate_change_ns
+ > hubbub1->watermarks.a.cstate_pstate.pstate_change_ns) {
+ hubbub1->watermarks.a.cstate_pstate.pstate_change_ns =
+ watermarks->a.cstate_pstate.pstate_change_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->a.cstate_pstate.pstate_change_ns,
+ refclk_mhz, 0x1fffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, 0,
+ DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_A calculated =%d\n"
+ "HW register value = 0x%x\n\n",
+ watermarks->a.cstate_pstate.pstate_change_ns, prog_wm_value);
+ }
+
+ /* clock state B */
+ if (safe_to_lower || watermarks->b.cstate_pstate.pstate_change_ns
+ > hubbub1->watermarks.b.cstate_pstate.pstate_change_ns) {
+ hubbub1->watermarks.b.cstate_pstate.pstate_change_ns =
+ watermarks->b.cstate_pstate.pstate_change_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->b.cstate_pstate.pstate_change_ns,
+ refclk_mhz, 0x1fffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, 0,
+ DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_B calculated =%d\n"
+ "HW register value = 0x%x\n\n",
+ watermarks->b.cstate_pstate.pstate_change_ns, prog_wm_value);
+ }
+
+ /* clock state C */
+ if (safe_to_lower || watermarks->c.cstate_pstate.pstate_change_ns
+ > hubbub1->watermarks.c.cstate_pstate.pstate_change_ns) {
+ hubbub1->watermarks.c.cstate_pstate.pstate_change_ns =
+ watermarks->c.cstate_pstate.pstate_change_ns;
+ prog_wm_value = convert_and_clamp(
+ watermarks->c.cstate_pstate.pstate_change_ns,
+ refclk_mhz, 0x1fffff);
+ REG_SET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, 0,
+ DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, prog_wm_value);
+ DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_C calculated =%d\n"
+ "HW register value = 0x%x\n\n",
+ watermarks->c.cstate_pstate.pstate_change_ns, prog_wm_value);
+ }
+
+ /* clock state D */
if (safe_to_lower || watermarks->d.cstate_pstate.pstate_change_ns
> hubbub1->watermarks.d.cstate_pstate.pstate_change_ns) {
hubbub1->watermarks.d.cstate_pstate.pstate_change_ns =
@@ -553,6 +598,22 @@ void hubbub1_program_watermarks(
"HW register value = 0x%x\n\n",
watermarks->d.cstate_pstate.pstate_change_ns, prog_wm_value);
}
+}
+
+void hubbub1_program_watermarks(
+ struct hubbub *hubbub,
+ struct dcn_watermark_set *watermarks,
+ unsigned int refclk_mhz,
+ bool safe_to_lower)
+{
+ struct dcn10_hubbub *hubbub1 = TO_DCN10_HUBBUB(hubbub);
+ /*
+ * Need to clamp to max of the register values (i.e. no wrap)
+ * for dcn1, all wm registers are 21-bit wide
+ */
+ hubbub1_program_urgent_watermarks(hubbub, watermarks, refclk_mhz, safe_to_lower);
+ hubbub1_program_stutter_watermarks(hubbub, watermarks, refclk_mhz, safe_to_lower);
+ hubbub1_program_pstate_watermarks(hubbub, watermarks, refclk_mhz, safe_to_lower);
REG_UPDATE(DCHUBBUB_ARB_SAT_LEVEL,
DCHUBBUB_ARB_SAT_LEVEL, 60 * refclk_mhz);
@@ -903,9 +964,7 @@ void hubbub1_construct(struct hubbub *hubbub,
hubbub1->masks = hubbub_mask;
hubbub1->debug_test_index_pstate = 0x7;
-#if defined(CONFIG_DRM_AMD_DC_DCN1_01)
if (ctx->dce_version == DCN_VERSION_1_01)
hubbub1->debug_test_index_pstate = 0xB;
-#endif
}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
index 85811b24a497..7c2559c9ae23 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
@@ -262,4 +262,20 @@ void hubbub1_construct(struct hubbub *hubbub,
const struct dcn_hubbub_shift *hubbub_shift,
const struct dcn_hubbub_mask *hubbub_mask);
+void hubbub1_program_urgent_watermarks(
+ struct hubbub *hubbub,
+ struct dcn_watermark_set *watermarks,
+ unsigned int refclk_mhz,
+ bool safe_to_lower);
+void hubbub1_program_stutter_watermarks(
+ struct hubbub *hubbub,
+ struct dcn_watermark_set *watermarks,
+ unsigned int refclk_mhz,
+ bool safe_to_lower);
+void hubbub1_program_pstate_watermarks(
+ struct hubbub *hubbub,
+ struct dcn_watermark_set *watermarks,
+ unsigned int refclk_mhz,
+ bool safe_to_lower);
+
#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
index 54b219a710d8..934bacc0c6ad 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
@@ -63,7 +63,7 @@ void hubp1_set_blank(struct hubp *hubp, bool blank)
}
hubp->mpcc_id = 0xf;
- hubp->opp_id = 0xf;
+ hubp->opp_id = OPP_ID_INVALID;
}
}
@@ -306,6 +306,28 @@ void hubp1_program_pixel_format(
REG_UPDATE(DCSURF_SURFACE_CONFIG,
SURFACE_PIXEL_FORMAT, 12);
break;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ case SURFACE_PIXEL_FORMAT_GRPH_RGB111110_FIX:
+ REG_UPDATE(DCSURF_SURFACE_CONFIG,
+ SURFACE_PIXEL_FORMAT, 112);
+ break;
+ case SURFACE_PIXEL_FORMAT_GRPH_BGR101111_FIX:
+ REG_UPDATE(DCSURF_SURFACE_CONFIG,
+ SURFACE_PIXEL_FORMAT, 113);
+ break;
+ case SURFACE_PIXEL_FORMAT_VIDEO_ACrYCb2101010:
+ REG_UPDATE(DCSURF_SURFACE_CONFIG,
+ SURFACE_PIXEL_FORMAT, 114);
+ break;
+ case SURFACE_PIXEL_FORMAT_GRPH_RGB111110_FLOAT:
+ REG_UPDATE(DCSURF_SURFACE_CONFIG,
+ SURFACE_PIXEL_FORMAT, 118);
+ break;
+ case SURFACE_PIXEL_FORMAT_GRPH_BGR101111_FLOAT:
+ REG_UPDATE(DCSURF_SURFACE_CONFIG,
+ SURFACE_PIXEL_FORMAT, 119);
+ break;
+#endif
default:
BREAK_TO_DEBUGGER();
break;
@@ -317,8 +339,7 @@ void hubp1_program_pixel_format(
bool hubp1_program_surface_flip_and_addr(
struct hubp *hubp,
const struct dc_plane_address *address,
- bool flip_immediate,
- uint8_t vmid)
+ bool flip_immediate)
{
struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
@@ -1206,6 +1227,11 @@ static const struct hubp_funcs dcn10_hubp_funcs = {
.hubp_disable_control = hubp1_disable_control,
.hubp_get_underflow_status = hubp1_get_underflow_status,
.hubp_init = hubp1_init,
+
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ .dmdata_set_attributes = NULL,
+ .dmdata_load = NULL,
+#endif
};
/*****************************************/
@@ -1226,7 +1252,7 @@ void dcn10_hubp_construct(
hubp1->hubp_shift = hubp_shift;
hubp1->hubp_mask = hubp_mask;
hubp1->base.inst = inst;
- hubp1->base.opp_id = 0xf;
+ hubp1->base.opp_id = OPP_ID_INVALID;
hubp1->base.mpcc_id = 0xf;
}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
index 99d2b7e2a578..31c8fdd3206c 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
@@ -459,6 +459,7 @@
type ROTATION_ANGLE;\
type H_MIRROR_EN;\
type SURFACE_PIXEL_FORMAT;\
+ type ALPHA_PLANE_EN;\
type SURFACE_FLIP_TYPE;\
type SURFACE_FLIP_MODE_FOR_STEREOSYNC;\
type SURFACE_FLIP_IN_STEREOSYNC;\
@@ -715,6 +716,13 @@ void hubp1_dcc_control(struct hubp *hubp,
bool enable,
bool independent_64b_blks);
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+bool hubp1_program_surface_flip_and_addr(
+ struct hubp *hubp,
+ const struct dc_plane_address *address,
+ bool flip_immediate);
+
+#endif
bool hubp1_is_flip_pending(struct hubp *hubp);
void hubp1_cursor_set_attributes(
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
index 33d311cea28c..2118ea21d7e9 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
@@ -23,6 +23,7 @@
*
*/
+#include <linux/delay.h>
#include "dm_services.h"
#include "core_types.h"
#include "resource.h"
@@ -45,6 +46,12 @@
#include "dcn10_cm_common.h"
#include "dc_link_dp.h"
#include "dccg.h"
+#include "clk_mgr.h"
+
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+#include "dsc.h"
+#endif
#define DC_LOGGER_INIT(logger)
@@ -343,6 +350,62 @@ void dcn10_log_hw_state(struct dc *dc,
}
DTN_INFO("\n");
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ DTN_INFO("DSC: CLOCK_EN SLICE_WIDTH Bytes_pp\n");
+ for (i = 0; i < pool->res_cap->num_dsc; i++) {
+ struct display_stream_compressor *dsc = pool->dscs[i];
+ struct dcn_dsc_state s = {0};
+
+ dsc->funcs->dsc_read_state(dsc, &s);
+ DTN_INFO("[%d]: %-9d %-12d %-10d\n",
+ dsc->inst,
+ s.dsc_clock_en,
+ s.dsc_slice_width,
+ s.dsc_bytes_per_pixel);
+ DTN_INFO("\n");
+ }
+ DTN_INFO("\n");
+
+ DTN_INFO("S_ENC: DSC_MODE SEC_GSP7_LINE_NUM"
+ " VBID6_LINE_REFERENCE VBID6_LINE_NUM SEC_GSP7_ENABLE SEC_STREAM_ENABLE\n");
+ for (i = 0; i < pool->stream_enc_count; i++) {
+ struct stream_encoder *enc = pool->stream_enc[i];
+ struct enc_state s = {0};
+
+ if (enc->funcs->enc_read_state) {
+ enc->funcs->enc_read_state(enc, &s);
+ DTN_INFO("[%-3d]: %-9d %-18d %-21d %-15d %-16d %-17d\n",
+ enc->id,
+ s.dsc_mode,
+ s.sec_gsp_pps_line_num,
+ s.vbid6_line_reference,
+ s.vbid6_line_num,
+ s.sec_gsp_pps_enable,
+ s.sec_stream_enable);
+ DTN_INFO("\n");
+ }
+ }
+ DTN_INFO("\n");
+
+ DTN_INFO("L_ENC: DPHY_FEC_EN DPHY_FEC_READY_SHADOW DPHY_FEC_ACTIVE_STATUS\n");
+ for (i = 0; i < dc->link_count; i++) {
+ struct link_encoder *lenc = dc->links[i]->link_enc;
+
+ struct link_enc_state s = {0};
+
+ if (lenc->funcs->read_state) {
+ lenc->funcs->read_state(lenc, &s);
+ DTN_INFO("[%-3d]: %-12d %-22d %-22d\n",
+ i,
+ s.dphy_fec_en,
+ s.dphy_fec_ready_shadow,
+ s.dphy_fec_active_status);
+ DTN_INFO("\n");
+ }
+ }
+ DTN_INFO("\n");
+#endif
+
DTN_INFO("\nCALCULATED Clocks: dcfclk_khz:%d dcfclk_deep_sleep_khz:%d dispclk_khz:%d\n"
"dppclk_khz:%d max_supported_dppclk_khz:%d fclk_khz:%d socclk_khz:%d\n\n",
dc->current_state->bw_ctx.bw.dcn.clk.dcfclk_khz,
@@ -358,6 +421,23 @@ void dcn10_log_hw_state(struct dc *dc,
DTN_INFO_END();
}
+bool dcn10_did_underflow_occur(struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+ struct hubp *hubp = pipe_ctx->plane_res.hubp;
+ struct timing_generator *tg = pipe_ctx->stream_res.tg;
+
+ if (tg->funcs->is_optc_underflow_occurred(tg)) {
+ tg->funcs->clear_optc_underflow(tg);
+ return true;
+ }
+
+ if (hubp->funcs->hubp_get_underflow_status(hubp)) {
+ hubp->funcs->hubp_clear_underflow(hubp);
+ return true;
+ }
+ return false;
+}
+
static void enable_power_gating_plane(
struct dce_hwseq *hws,
bool enable)
@@ -658,16 +738,15 @@ static enum dc_status dcn10_enable_stream_timing(
BREAK_TO_DEBUGGER();
return DC_ERROR_UNEXPECTED;
}
- pipe_ctx->stream_res.tg->dlg_otg_param.vready_offset = pipe_ctx->pipe_dlg_param.vready_offset;
- pipe_ctx->stream_res.tg->dlg_otg_param.vstartup_start = pipe_ctx->pipe_dlg_param.vstartup_start;
- pipe_ctx->stream_res.tg->dlg_otg_param.vupdate_offset = pipe_ctx->pipe_dlg_param.vupdate_offset;
- pipe_ctx->stream_res.tg->dlg_otg_param.vupdate_width = pipe_ctx->pipe_dlg_param.vupdate_width;
-
- pipe_ctx->stream_res.tg->dlg_otg_param.signal = pipe_ctx->stream->signal;
pipe_ctx->stream_res.tg->funcs->program_timing(
pipe_ctx->stream_res.tg,
&stream->timing,
+ pipe_ctx->pipe_dlg_param.vready_offset,
+ pipe_ctx->pipe_dlg_param.vstartup_start,
+ pipe_ctx->pipe_dlg_param.vupdate_offset,
+ pipe_ctx->pipe_dlg_param.vupdate_width,
+ pipe_ctx->stream->signal,
true);
#if 0 /* move to after enable_crtc */
@@ -1024,7 +1103,7 @@ static void dcn10_init_pipes(struct dc *dc, struct dc_state *context)
pipe_ctx->plane_res.dpp = dpp;
pipe_ctx->plane_res.mpcc_inst = dpp->inst;
hubp->mpcc_id = dpp->inst;
- hubp->opp_id = 0xf;
+ hubp->opp_id = OPP_ID_INVALID;
hubp->power_gated = false;
dc->res_pool->opps[i]->mpc_tree_params.opp_id = dc->res_pool->opps[i]->inst;
@@ -1101,9 +1180,6 @@ static void dcn10_init_hw(struct dc *dc)
*/
struct dc_link *link = dc->links[i];
- if (link->link_enc->connector.id == CONNECTOR_ID_EDP)
- dc->hwss.edp_power_control(link, true);
-
link->link_enc->funcs->hw_init(link->link_enc);
/* Check for enabled DIG to identify enabled display */
@@ -1119,16 +1195,7 @@ static void dcn10_init_hw(struct dc *dc)
* everything down.
*/
if (dcb->funcs->is_accelerated_mode(dcb) || dc->config.power_down_display_on_boot) {
- for (i = 0; i < dc->res_pool->pipe_count; i++) {
- struct hubp *hubp = dc->res_pool->hubps[i];
- struct dpp *dpp = dc->res_pool->dpps[i];
-
- hubp->funcs->hubp_init(hubp);
- dc->res_pool->opps[i]->mpc_tree_params.opp_id = dc->res_pool->opps[i]->inst;
- plane_atomic_power_down(dc, dpp, hubp);
- }
-
- apply_DEGVIDCN10_253_wa(dc);
+ dc->hwss.init_pipes(dc, dc->current_state);
}
for (i = 0; i < dc->res_pool->audio_count; i++) {
@@ -1145,6 +1212,9 @@ static void dcn10_init_hw(struct dc *dc)
if (dmcu != NULL)
dmcu->funcs->dmcu_init(dmcu);
+ if (abm != NULL && dmcu != NULL)
+ abm->dmcu_is_running = dmcu->funcs->is_dmcu_initialized(dmcu);
+
/* power AFMT HDMI memory TODO: may move to dis/en output save power*/
REG_WRITE(DIO_MEM_PWR_CTRL, 0);
@@ -1159,7 +1229,7 @@ static void dcn10_init_hw(struct dc *dc)
enable_power_gating_plane(dc->hwseq, true);
- memset(&dc->res_pool->clk_mgr->clks, 0, sizeof(dc->res_pool->clk_mgr->clks));
+ memset(&dc->clk_mgr->clks, 0, sizeof(dc->clk_mgr->clks));
}
static void dcn10_reset_hw_ctx_wrap(
@@ -1235,8 +1305,7 @@ static void dcn10_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_c
pipe_ctx->plane_res.hubp->funcs->hubp_program_surface_flip_and_addr(
pipe_ctx->plane_res.hubp,
&plane_state->address,
- plane_state->flip_immediate,
- 0);
+ plane_state->flip_immediate);
plane_state->status.requested_address = plane_state->address;
@@ -1297,10 +1366,6 @@ static bool dcn10_set_input_transfer_func(struct pipe_ctx *pipe_ctx,
return result;
}
-
-
-
-
static bool
dcn10_set_output_transfer_func(struct pipe_ctx *pipe_ctx,
const struct dc_stream_state *stream)
@@ -1756,7 +1821,7 @@ static void dcn10_program_output_csc(struct dc *dc,
bool is_lower_pipe_tree_visible(struct pipe_ctx *pipe_ctx)
{
- if (pipe_ctx->plane_state->visible)
+ if (pipe_ctx->plane_state && pipe_ctx->plane_state->visible)
return true;
if (pipe_ctx->bottom_pipe && is_lower_pipe_tree_visible(pipe_ctx->bottom_pipe))
return true;
@@ -1765,7 +1830,7 @@ bool is_lower_pipe_tree_visible(struct pipe_ctx *pipe_ctx)
bool is_upper_pipe_tree_visible(struct pipe_ctx *pipe_ctx)
{
- if (pipe_ctx->plane_state->visible)
+ if (pipe_ctx->plane_state && pipe_ctx->plane_state->visible)
return true;
if (pipe_ctx->top_pipe && is_upper_pipe_tree_visible(pipe_ctx->top_pipe))
return true;
@@ -1774,7 +1839,7 @@ bool is_upper_pipe_tree_visible(struct pipe_ctx *pipe_ctx)
bool is_pipe_tree_visible(struct pipe_ctx *pipe_ctx)
{
- if (pipe_ctx->plane_state->visible)
+ if (pipe_ctx->plane_state && pipe_ctx->plane_state->visible)
return true;
if (pipe_ctx->top_pipe && is_upper_pipe_tree_visible(pipe_ctx->top_pipe))
return true;
@@ -1920,7 +1985,7 @@ static uint16_t fixed_point_to_int_frac(
return result;
}
-void build_prescale_params(struct dc_bias_and_scale *bias_and_scale,
+void dcn10_build_prescale_params(struct dc_bias_and_scale *bias_and_scale,
const struct dc_plane_state *plane_state)
{
if (plane_state->format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN
@@ -1950,10 +2015,15 @@ static void update_dpp(struct dpp *dpp, struct dc_plane_state *plane_state)
plane_state->format,
EXPANSION_MODE_ZERO,
plane_state->input_csc_color_matrix,
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ plane_state->color_space,
+ NULL);
+#else
plane_state->color_space);
+#endif
//set scale and bias registers
- build_prescale_params(&bns_params, plane_state);
+ dcn10_build_prescale_params(&bns_params, plane_state);
if (dpp->funcs->dpp_program_bias_and_scale)
dpp->funcs->dpp_program_bias_and_scale(dpp, &bns_params);
}
@@ -2071,7 +2141,7 @@ void update_dchubp_dpp(
*/
if (plane_state->update_flags.bits.full_update) {
bool should_divided_by_2 = context->bw_ctx.bw.dcn.clk.dppclk_khz <=
- dc->res_pool->clk_mgr->clks.dispclk_khz / 2;
+ dc->clk_mgr->clks.dispclk_khz / 2;
dpp->funcs->dpp_dppclk_control(
dpp,
@@ -2084,9 +2154,9 @@ void update_dchubp_dpp(
dpp->inst,
pipe_ctx->plane_res.bw.dppclk_khz);
else
- dc->res_pool->clk_mgr->clks.dppclk_khz = should_divided_by_2 ?
- dc->res_pool->clk_mgr->clks.dispclk_khz / 2 :
- dc->res_pool->clk_mgr->clks.dispclk_khz;
+ dc->clk_mgr->clks.dppclk_khz = should_divided_by_2 ?
+ dc->clk_mgr->clks.dispclk_khz / 2 :
+ dc->clk_mgr->clks.dispclk_khz;
}
/* TODO: Need input parameter to tell current DCHUB pipe tie to which OTG
@@ -2152,7 +2222,7 @@ void update_dchubp_dpp(
pipe_ctx,
pipe_ctx->stream->output_color_space,
pipe_ctx->stream->csc_color_matrix.matrix,
- hubp->opp_id);
+ pipe_ctx->stream_res.opp->inst);
}
if (plane_state->update_flags.bits.full_update ||
@@ -2279,14 +2349,15 @@ static void program_all_pipe_in_tree(
if (pipe_ctx->top_pipe == NULL) {
bool blank = !is_pipe_tree_visible(pipe_ctx);
- pipe_ctx->stream_res.tg->dlg_otg_param.vready_offset = pipe_ctx->pipe_dlg_param.vready_offset;
- pipe_ctx->stream_res.tg->dlg_otg_param.vstartup_start = pipe_ctx->pipe_dlg_param.vstartup_start;
- pipe_ctx->stream_res.tg->dlg_otg_param.vupdate_offset = pipe_ctx->pipe_dlg_param.vupdate_offset;
- pipe_ctx->stream_res.tg->dlg_otg_param.vupdate_width = pipe_ctx->pipe_dlg_param.vupdate_width;
- pipe_ctx->stream_res.tg->dlg_otg_param.signal = pipe_ctx->stream->signal;
-
pipe_ctx->stream_res.tg->funcs->program_global_sync(
- pipe_ctx->stream_res.tg);
+ pipe_ctx->stream_res.tg,
+ pipe_ctx->pipe_dlg_param.vready_offset,
+ pipe_ctx->pipe_dlg_param.vstartup_start,
+ pipe_ctx->pipe_dlg_param.vupdate_offset,
+ pipe_ctx->pipe_dlg_param.vupdate_width);
+
+ pipe_ctx->stream_res.tg->funcs->set_vtg_params(
+ pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing);
dc->hwss.blank_pixel_data(dc, pipe_ctx, blank);
@@ -2331,6 +2402,7 @@ static void dcn10_apply_ctx_for_surface(
{
int i;
struct timing_generator *tg;
+ uint32_t underflow_check_delay_us;
bool removed_pipe[4] = { false };
bool interdependent_update = false;
struct pipe_ctx *top_pipe_to_program =
@@ -2345,11 +2417,22 @@ static void dcn10_apply_ctx_for_surface(
interdependent_update = top_pipe_to_program->plane_state &&
top_pipe_to_program->plane_state->update_flags.bits.full_update;
+ underflow_check_delay_us = dc->debug.underflow_assert_delay_us;
+
+ if (underflow_check_delay_us != 0xFFFFFFFF && dc->hwss.did_underflow_occur)
+ ASSERT(dc->hwss.did_underflow_occur(dc, top_pipe_to_program));
+
if (interdependent_update)
lock_all_pipes(dc, context, true);
else
dcn10_pipe_control_lock(dc, top_pipe_to_program, true);
+ if (underflow_check_delay_us != 0xFFFFFFFF)
+ udelay(underflow_check_delay_us);
+
+ if (underflow_check_delay_us != 0xFFFFFFFF && dc->hwss.did_underflow_occur)
+ ASSERT(dc->hwss.did_underflow_occur(dc, top_pipe_to_program));
+
if (num_planes == 0) {
/* OTG blank before remove all front end */
dc->hwss.blank_pixel_data(dc, top_pipe_to_program, true);
@@ -2369,7 +2452,7 @@ static void dcn10_apply_ctx_for_surface(
if (pipe_ctx->plane_state && !old_pipe_ctx->plane_state) {
if (old_pipe_ctx->stream_res.tg == tg &&
old_pipe_ctx->plane_res.hubp &&
- old_pipe_ctx->plane_res.hubp->opp_id != 0xf)
+ old_pipe_ctx->plane_res.hubp->opp_id != OPP_ID_INVALID)
dcn10_disable_plane(dc, old_pipe_ctx);
}
@@ -2389,6 +2472,11 @@ static void dcn10_apply_ctx_for_surface(
if (num_planes > 0)
program_all_pipe_in_tree(dc, top_pipe_to_program, context);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ /* Program secondary blending tree and writeback pipes */
+ if ((stream->num_wb_info > 0) && (dc->hwss.program_all_writeback_pipes_in_tree))
+ dc->hwss.program_all_writeback_pipes_in_tree(dc, stream, context);
+#endif
if (interdependent_update)
for (i = 0; i < dc->res_pool->pipe_count; i++) {
struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
@@ -2415,6 +2503,12 @@ static void dcn10_apply_ctx_for_surface(
if (removed_pipe[i])
dcn10_disable_plane(dc, &dc->current_state->res_ctx.pipe_ctx[i]);
+ for (i = 0; i < dc->res_pool->pipe_count; i++)
+ if (removed_pipe[i]) {
+ dc->hwss.optimize_bandwidth(dc, context);
+ break;
+ }
+
if (dc->hwseq->wa.DEGVIDCN10_254)
hubbub1_wm_change_req_wa(dc->res_pool->hubbub);
}
@@ -2448,8 +2542,8 @@ static void dcn10_prepare_bandwidth(
if (context->stream_count == 0)
context->bw_ctx.bw.dcn.clk.phyclk_khz = 0;
- dc->res_pool->clk_mgr->funcs->update_clocks(
- dc->res_pool->clk_mgr,
+ dc->clk_mgr->funcs->update_clocks(
+ dc->clk_mgr,
context,
false);
}
@@ -2480,8 +2574,8 @@ static void dcn10_optimize_bandwidth(
if (context->stream_count == 0)
context->bw_ctx.bw.dcn.clk.phyclk_khz = 0;
- dc->res_pool->clk_mgr->funcs->update_clocks(
- dc->res_pool->clk_mgr,
+ dc->clk_mgr->funcs->update_clocks(
+ dc->clk_mgr,
context,
true);
}
@@ -2504,8 +2598,8 @@ static void set_drr(struct pipe_ctx **pipe_ctx,
{
int i = 0;
struct drr_params params = {0};
- // DRR should set trigger event to monitor surface update event
- unsigned int event_triggers = 0x80;
+ // DRR set trigger event mapped to OTG_TRIG_A (bit 11) for manual control flow
+ unsigned int event_triggers = 0x800;
params.vertical_total_max = vmax;
params.vertical_total_min = vmin;
@@ -2644,9 +2738,6 @@ static void dcn10_wait_for_mpcc_disconnect(
res_pool->mpc->funcs->wait_for_idle(res_pool->mpc, mpcc_inst);
pipe_ctx->stream_res.opp->mpcc_disconnect_pending[mpcc_inst] = false;
hubp->funcs->set_blank(hubp, true);
- /*DC_LOG_ERROR(dc->ctx->logger,
- "[debug_mpo: wait_for_mpcc finished waiting on mpcc %d]\n",
- i);*/
}
}
@@ -2790,7 +2881,6 @@ static void apply_front_porch_workaround(
int get_vupdate_offset_from_vsync(struct pipe_ctx *pipe_ctx)
{
- struct timing_generator *optc = pipe_ctx->stream_res.tg;
const struct dc_crtc_timing *dc_crtc_timing = &pipe_ctx->stream->timing;
struct dc_crtc_timing patched_crtc_timing;
int vesa_sync_start;
@@ -2813,7 +2903,7 @@ int get_vupdate_offset_from_vsync(struct pipe_ctx *pipe_ctx)
* interlace_factor;
vertical_line_start = asic_blank_end -
- optc->dlg_otg_param.vstartup_start + 1;
+ pipe_ctx->pipe_dlg_param.vstartup_start + 1;
return vertical_line_start;
}
@@ -2961,6 +3051,18 @@ static void dcn10_unblank_stream(struct pipe_ctx *pipe_ctx,
}
}
+static void dcn10_send_immediate_sdp_message(struct pipe_ctx *pipe_ctx,
+ const uint8_t *custom_sdp_message,
+ unsigned int sdp_message_size)
+{
+ if (dc_is_dp_signal(pipe_ctx->stream->signal)) {
+ pipe_ctx->stream_res.stream_enc->funcs->send_immediate_sdp_message(
+ pipe_ctx->stream_res.stream_enc,
+ custom_sdp_message,
+ sdp_message_size);
+ }
+}
+
static const struct hw_sequencer_funcs dcn10_funcs = {
.program_gamut_remap = program_gamut_remap,
.init_hw = dcn10_init_hw,
@@ -2980,6 +3082,7 @@ static const struct hw_sequencer_funcs dcn10_funcs = {
.enable_timing_synchronization = dcn10_enable_timing_synchronization,
.enable_per_frame_crtc_position_reset = dcn10_enable_per_frame_crtc_position_reset,
.update_info_frame = dce110_update_info_frame,
+ .send_immediate_sdp_message = dcn10_send_immediate_sdp_message,
.enable_stream = dce110_enable_stream,
.disable_stream = dce110_disable_stream,
.unblank_stream = dcn10_unblank_stream,
@@ -3012,7 +3115,8 @@ static const struct hw_sequencer_funcs dcn10_funcs = {
.disable_stream_gating = NULL,
.enable_stream_gating = NULL,
.setup_periodic_interrupt = dcn10_setup_periodic_interrupt,
- .setup_vupdate_interrupt = dcn10_setup_vupdate_interrupt
+ .setup_vupdate_interrupt = dcn10_setup_vupdate_interrupt,
+ .did_underflow_occur = dcn10_did_underflow_occur
};
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
index 4b3b27a5d23b..d3616b1948cc 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
@@ -71,6 +71,8 @@ void dcn10_get_hdr_visual_confirm_color(
struct pipe_ctx *pipe_ctx,
struct tg_color *color);
+bool dcn10_did_underflow_occur(struct dc *dc, struct pipe_ctx *pipe_ctx);
+
void update_dchubp_dpp(
struct dc *dc,
struct pipe_ctx *pipe_ctx,
@@ -83,6 +85,8 @@ struct pipe_ctx *find_top_pipe_for_stream(
int get_vupdate_offset_from_vsync(struct pipe_ctx *pipe_ctx);
+void dcn10_build_prescale_params(struct dc_bias_and_scale *bias_and_scale,
+ const struct dc_plane_state *plane_state);
void lock_all_pipes(struct dc *dc,
struct dc_state *context,
bool lock);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c
index 991622da9ed5..6e47444109d7 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c
@@ -43,7 +43,7 @@
#include "dcn10_hubp.h"
#include "dcn10_hubbub.h"
#include "dcn10_cm_common.h"
-#include "dcn10_clk_mgr.h"
+#include "clk_mgr.h"
static unsigned int snprintf_count(char *pBuf, unsigned int bufSize, char *fmt, ...)
{
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_ipp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_ipp.c
index 08db1e6b5166..0fb9e440cb9d 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_ipp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_ipp.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "dcn10_ipp.h"
#include "reg_helper.h"
@@ -51,6 +53,12 @@ static const struct ipp_funcs dcn10_ipp_funcs = {
.ipp_destroy = dcn10_ipp_destroy
};
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+static const struct ipp_funcs dcn20_ipp_funcs = {
+ .ipp_destroy = dcn10_ipp_destroy
+};
+#endif
+
void dcn10_ipp_construct(
struct dcn10_ipp *ippn10,
struct dc_context *ctx,
@@ -68,3 +76,21 @@ void dcn10_ipp_construct(
ippn10->ipp_mask = ipp_mask;
}
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+void dcn20_ipp_construct(
+ struct dcn10_ipp *ippn10,
+ struct dc_context *ctx,
+ int inst,
+ const struct dcn10_ipp_registers *regs,
+ const struct dcn10_ipp_shift *ipp_shift,
+ const struct dcn10_ipp_mask *ipp_mask)
+{
+ ippn10->base.ctx = ctx;
+ ippn10->base.inst = inst;
+ ippn10->base.funcs = &dcn20_ipp_funcs;
+
+ ippn10->regs = regs;
+ ippn10->ipp_shift = ipp_shift;
+ ippn10->ipp_mask = ipp_mask;
+}
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_ipp.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_ipp.h
index 819b749c6e31..cfa24459242b 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_ipp.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_ipp.h
@@ -49,6 +49,19 @@
SRI(CURSOR_HOT_SPOT, CURSOR, id), \
SRI(CURSOR_DST_OFFSET, CURSOR, id)
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#define IPP_REG_LIST_DCN20(id) \
+ IPP_REG_LIST_DCN(id), \
+ SRI(CURSOR_SETTINGS, HUBPREQ, id), \
+ SRI(CURSOR_SURFACE_ADDRESS_HIGH, CURSOR0_, id), \
+ SRI(CURSOR_SURFACE_ADDRESS, CURSOR0_, id), \
+ SRI(CURSOR_SIZE, CURSOR0_, id), \
+ SRI(CURSOR_CONTROL, CURSOR0_, id), \
+ SRI(CURSOR_POSITION, CURSOR0_, id), \
+ SRI(CURSOR_HOT_SPOT, CURSOR0_, id), \
+ SRI(CURSOR_DST_OFFSET, CURSOR0_, id)
+#endif
+
#define CURSOR0_CURSOR_CONTROL__CURSOR_2X_MAGNIFY__SHIFT 0x4
#define CURSOR0_CURSOR_CONTROL__CURSOR_2X_MAGNIFY_MASK 0x00000010L
#define CURSOR1_CURSOR_CONTROL__CURSOR_2X_MAGNIFY__SHIFT 0x4
@@ -92,6 +105,27 @@
IPP_SF(CURSOR0_CURSOR_DST_OFFSET, CURSOR_DST_X_OFFSET, mask_sh), \
IPP_SF(CNVC_CFG0_FORMAT_CONTROL, OUTPUT_FP, mask_sh)
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#define IPP_MASK_SH_LIST_DCN20(mask_sh) \
+ IPP_MASK_SH_LIST_DCN(mask_sh), \
+ IPP_SF(HUBPREQ0_CURSOR_SETTINGS, CURSOR0_DST_Y_OFFSET, mask_sh), \
+ IPP_SF(HUBPREQ0_CURSOR_SETTINGS, CURSOR0_CHUNK_HDL_ADJUST, mask_sh), \
+ IPP_SF(CURSOR0_0_CURSOR_SURFACE_ADDRESS_HIGH, CURSOR_SURFACE_ADDRESS_HIGH, mask_sh), \
+ IPP_SF(CURSOR0_0_CURSOR_SURFACE_ADDRESS, CURSOR_SURFACE_ADDRESS, mask_sh), \
+ IPP_SF(CURSOR0_0_CURSOR_SIZE, CURSOR_WIDTH, mask_sh), \
+ IPP_SF(CURSOR0_0_CURSOR_SIZE, CURSOR_HEIGHT, mask_sh), \
+ IPP_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_MODE, mask_sh), \
+ IPP_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_2X_MAGNIFY, mask_sh), \
+ IPP_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_PITCH, mask_sh), \
+ IPP_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_LINES_PER_CHUNK, mask_sh), \
+ IPP_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_ENABLE, mask_sh), \
+ IPP_SF(CURSOR0_0_CURSOR_POSITION, CURSOR_X_POSITION, mask_sh), \
+ IPP_SF(CURSOR0_0_CURSOR_POSITION, CURSOR_Y_POSITION, mask_sh), \
+ IPP_SF(CURSOR0_0_CURSOR_HOT_SPOT, CURSOR_HOT_SPOT_X, mask_sh), \
+ IPP_SF(CURSOR0_0_CURSOR_HOT_SPOT, CURSOR_HOT_SPOT_Y, mask_sh), \
+ IPP_SF(CURSOR0_0_CURSOR_DST_OFFSET, CURSOR_DST_X_OFFSET, mask_sh)
+#endif
+
#define IPP_DCN10_REG_FIELD_LIST(type) \
type CNVC_SURFACE_PIXEL_FORMAT; \
type CNVC_BYPASS; \
@@ -162,4 +196,13 @@ void dcn10_ipp_construct(struct dcn10_ipp *ippn10,
const struct dcn10_ipp_shift *ipp_shift,
const struct dcn10_ipp_mask *ipp_mask);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+void dcn20_ipp_construct(struct dcn10_ipp *ippn10,
+ struct dc_context *ctx,
+ int inst,
+ const struct dcn10_ipp_registers *regs,
+ const struct dcn10_ipp_shift *ipp_shift,
+ const struct dcn10_ipp_mask *ipp_mask);
+#endif
+
#endif /* _DCN10_IPP_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
index 0126a44ba012..549d423a01f6 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
@@ -23,6 +23,9 @@
*
*/
+#include <linux/delay.h>
+#include <linux/slab.h>
+
#include "reg_helper.h"
#include "core_types.h"
@@ -229,7 +232,9 @@ static void setup_panel_mode(
{
uint32_t value;
- ASSERT(REG(DP_DPHY_INTERNAL_CTRL));
+ if (!REG(DP_DPHY_INTERNAL_CTRL))
+ return;
+
value = REG_READ(DP_DPHY_INTERNAL_CTRL);
switch (panel_mode) {
@@ -726,6 +731,8 @@ void dcn10_link_encoder_construct(
enc10->base.features.flags.bits.IS_HBR3_CAPABLE =
bp_cap_info.DP_HBR3_EN;
enc10->base.features.flags.bits.HDMI_6GB_EN = bp_cap_info.HDMI_6GB_EN;
+ enc10->base.features.flags.bits.DP_IS_USB_C =
+ bp_cap_info.DP_IS_USB_C;
} else {
DC_LOG_WARNING("%s: Failed to get encoder_cap_info from VBIOS with error code %d!\n",
__func__,
@@ -1357,5 +1364,5 @@ void dcn10_aux_initialize(struct dcn10_link_encoder *enc10)
/* 1/4 window (the maximum allowed) */
AUX_REG_UPDATE(AUX_DPHY_RX_CONTROL0,
- AUX_RX_RECEIVE_WINDOW, 1);
+ AUX_RX_RECEIVE_WINDOW, 0);
}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h
index b74b80a247ec..33b2af1a181c 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h
@@ -72,6 +72,9 @@
struct dcn10_link_enc_aux_registers {
uint32_t AUX_CONTROL;
uint32_t AUX_DPHY_RX_CONTROL0;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ uint32_t AUX_DPHY_TX_CONTROL;
+#endif
};
struct dcn10_link_enc_hpd_registers {
@@ -103,6 +106,23 @@ struct dcn10_link_enc_registers {
uint32_t DP_DPHY_HBR2_PATTERN_CONTROL;
uint32_t DP_SEC_CNTL1;
uint32_t TMDS_CTL_BITS;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ /* DCCG */
+ uint32_t CLOCK_ENABLE;
+ /* DIG */
+ uint32_t DIG_LANE_ENABLE;
+ /* UNIPHY */
+ uint32_t CHANNEL_XBAR_CNTL;
+ /* indirect registers */
+ uint32_t RAWLANE0_DIG_PCS_XF_RX_OVRD_IN_2;
+ uint32_t RAWLANE0_DIG_PCS_XF_RX_OVRD_IN_3;
+ uint32_t RAWLANE1_DIG_PCS_XF_RX_OVRD_IN_2;
+ uint32_t RAWLANE1_DIG_PCS_XF_RX_OVRD_IN_3;
+ uint32_t RAWLANE2_DIG_PCS_XF_RX_OVRD_IN_2;
+ uint32_t RAWLANE2_DIG_PCS_XF_RX_OVRD_IN_3;
+ uint32_t RAWLANE3_DIG_PCS_XF_RX_OVRD_IN_2;
+ uint32_t RAWLANE3_DIG_PCS_XF_RX_OVRD_IN_3;
+#endif
};
#define LE_SF(reg_name, field_name, post_fix)\
@@ -208,12 +228,166 @@ struct dcn10_link_enc_registers {
type AUX_LS_READ_EN;\
type AUX_RX_RECEIVE_WINDOW
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+
+#define DCN20_LINK_ENCODER_DPCS_REG_FIELD_LIST(type) \
+ type RDPCS_PHY_DP_TX0_DATA_EN;\
+ type RDPCS_PHY_DP_TX1_DATA_EN;\
+ type RDPCS_PHY_DP_TX2_DATA_EN;\
+ type RDPCS_PHY_DP_TX3_DATA_EN;\
+ type RDPCS_PHY_DP_TX0_PSTATE;\
+ type RDPCS_PHY_DP_TX1_PSTATE;\
+ type RDPCS_PHY_DP_TX2_PSTATE;\
+ type RDPCS_PHY_DP_TX3_PSTATE;\
+ type RDPCS_PHY_DP_TX0_MPLL_EN;\
+ type RDPCS_PHY_DP_TX1_MPLL_EN;\
+ type RDPCS_PHY_DP_TX2_MPLL_EN;\
+ type RDPCS_PHY_DP_TX3_MPLL_EN;\
+ type RDPCS_TX_FIFO_LANE0_EN;\
+ type RDPCS_TX_FIFO_LANE1_EN;\
+ type RDPCS_TX_FIFO_LANE2_EN;\
+ type RDPCS_TX_FIFO_LANE3_EN;\
+ type RDPCS_EXT_REFCLK_EN;\
+ type RDPCS_TX_FIFO_EN;\
+ type UNIPHY_LINK_ENABLE;\
+ type UNIPHY_CHANNEL0_INVERT;\
+ type UNIPHY_CHANNEL1_INVERT;\
+ type UNIPHY_CHANNEL2_INVERT;\
+ type UNIPHY_CHANNEL3_INVERT;\
+ type UNIPHY_LINK_ENABLE_HPD_MASK;\
+ type UNIPHY_LANE_STAGGER_DELAY;\
+ type RDPCS_SRAMCLK_BYPASS;\
+ type RDPCS_SRAMCLK_EN;\
+ type RDPCS_SRAMCLK_CLOCK_ON;\
+ type DPCS_TX_FIFO_EN;\
+ type RDPCS_PHY_DP_TX0_DISABLE;\
+ type RDPCS_PHY_DP_TX1_DISABLE;\
+ type RDPCS_PHY_DP_TX2_DISABLE;\
+ type RDPCS_PHY_DP_TX3_DISABLE;\
+ type RDPCS_PHY_DP_TX0_CLK_RDY;\
+ type RDPCS_PHY_DP_TX1_CLK_RDY;\
+ type RDPCS_PHY_DP_TX2_CLK_RDY;\
+ type RDPCS_PHY_DP_TX3_CLK_RDY;\
+ type RDPCS_PHY_DP_TX0_REQ;\
+ type RDPCS_PHY_DP_TX1_REQ;\
+ type RDPCS_PHY_DP_TX2_REQ;\
+ type RDPCS_PHY_DP_TX3_REQ;\
+ type RDPCS_PHY_DP_TX0_ACK;\
+ type RDPCS_PHY_DP_TX1_ACK;\
+ type RDPCS_PHY_DP_TX2_ACK;\
+ type RDPCS_PHY_DP_TX3_ACK;\
+ type RDPCS_PHY_DP_TX0_RESET;\
+ type RDPCS_PHY_DP_TX1_RESET;\
+ type RDPCS_PHY_DP_TX2_RESET;\
+ type RDPCS_PHY_DP_TX3_RESET;\
+ type RDPCS_PHY_RESET;\
+ type RDPCS_PHY_CR_MUX_SEL;\
+ type RDPCS_PHY_REF_RANGE;\
+ type RDPCS_PHY_DP4_POR;\
+ type RDPCS_SRAM_BYPASS;\
+ type RDPCS_SRAM_EXT_LD_DONE;\
+ type RDPCS_PHY_DP_TX0_TERM_CTRL;\
+ type RDPCS_PHY_DP_TX1_TERM_CTRL;\
+ type RDPCS_PHY_DP_TX2_TERM_CTRL;\
+ type RDPCS_PHY_DP_TX3_TERM_CTRL;\
+ type RDPCS_PHY_DP_REF_CLK_MPLLB_DIV;\
+ type RDPCS_PHY_DP_MPLLB_MULTIPLIER;\
+ type RDPCS_PHY_DP_MPLLB_SSC_EN;\
+ type RDPCS_PHY_DP_MPLLB_DIV5_CLK_EN;\
+ type RDPCS_PHY_DP_MPLLB_TX_CLK_DIV;\
+ type RDPCS_PHY_DP_MPLLB_WORD_DIV2_EN;\
+ type RDPCS_PHY_DP_MPLLB_FRACN_EN;\
+ type RDPCS_PHY_DP_MPLLB_PMIX_EN;\
+ type RDPCS_PHY_DP_MPLLB_FRACN_QUOT;\
+ type RDPCS_PHY_DP_MPLLB_FRACN_DEN;\
+ type RDPCS_PHY_DP_MPLLB_FRACN_REM;\
+ type RDPCS_PHY_DP_MPLLB_SSC_UP_SPREAD;\
+ type RDPCS_PHY_DP_MPLLB_SSC_STEPSIZE;\
+ type RDPCS_PHY_DP_MPLLB_SSC_PEAK;\
+ type RDPCS_PHY_DP_MPLLB_DIV_CLK_EN;\
+ type RDPCS_PHY_DP_MPLLB_DIV_MULTIPLIER;\
+ type RDPCS_PHY_TX_VBOOST_LVL;\
+ type RDPCS_PHY_HDMIMODE_ENABLE;\
+ type RDPCS_PHY_DP_REF_CLK_EN;\
+ type RDPCS_PLL_UPDATE_DATA;\
+ type RDPCS_SRAM_INIT_DONE;\
+ type RDPCS_TX_CR_ADDR;\
+ type RDPCS_TX_CR_DATA;\
+ type RDPCS_PHY_HDMI_MPLLB_HDMI_DIV;\
+ type RDPCS_PHY_DP_MPLLB_STATE;\
+ type RDPCS_PHY_DP_TX0_WIDTH;\
+ type RDPCS_PHY_DP_TX0_RATE;\
+ type RDPCS_PHY_DP_TX1_WIDTH;\
+ type RDPCS_PHY_DP_TX1_RATE;\
+ type RDPCS_PHY_DP_TX2_WIDTH;\
+ type RDPCS_PHY_DP_TX2_RATE;\
+ type RDPCS_PHY_DP_TX3_WIDTH;\
+ type RDPCS_PHY_DP_TX3_RATE;\
+ type DPCS_SYMCLK_CLOCK_ON;\
+ type DPCS_SYMCLK_GATE_DIS;\
+ type DPCS_SYMCLK_EN;\
+ type RDPCS_SYMCLK_DIV2_CLOCK_ON;\
+ type RDPCS_SYMCLK_DIV2_GATE_DIS;\
+ type RDPCS_SYMCLK_DIV2_EN;\
+ type DPCS_TX_DATA_SWAP;\
+ type DPCS_TX_DATA_ORDER_INVERT;\
+ type DPCS_TX_FIFO_RD_START_DELAY;\
+ type RDPCS_TX_FIFO_RD_START_DELAY;\
+ type RDPCS_REG_FIFO_ERROR_MASK;\
+ type RDPCS_TX_FIFO_ERROR_MASK;\
+ type RDPCS_DPALT_DISABLE_TOGGLE_MASK;\
+ type RDPCS_DPALT_4LANE_TOGGLE_MASK;\
+ type RDPCS_PHY_DPALT_DISABLE_ACK;\
+ type RDPCS_PHY_DP_MPLLB_V2I;\
+ type RDPCS_PHY_DP_MPLLB_FREQ_VCO;\
+ type RDPCS_PHY_DP_MPLLB_CP_INT;\
+ type RDPCS_PHY_DP_MPLLB_CP_PROP;\
+ type RDPCS_PHY_RX_REF_LD_VAL;\
+ type RDPCS_PHY_RX_VCO_LD_VAL;\
+ type DPCSTX_DEBUG_CONFIG; \
+ type RDPCSTX_DEBUG_CONFIG
+
+#define DCN20_LINK_ENCODER_REG_FIELD_LIST(type) \
+ type DIG_LANE0EN;\
+ type DIG_LANE1EN;\
+ type DIG_LANE2EN;\
+ type DIG_LANE3EN;\
+ type DIG_CLK_EN;\
+ type SYMCLKA_CLOCK_ENABLE;\
+ type DPHY_FEC_EN;\
+ type DPHY_FEC_READY_SHADOW;\
+ type DPHY_FEC_ACTIVE_STATUS;\
+ DCN20_LINK_ENCODER_DPCS_REG_FIELD_LIST(type);\
+ type VCO_LD_VAL_OVRD;\
+ type VCO_LD_VAL_OVRD_EN;\
+ type REF_LD_VAL_OVRD;\
+ type REF_LD_VAL_OVRD_EN;\
+ type AUX_RX_START_WINDOW; \
+ type AUX_RX_HALF_SYM_DETECT_LEN; \
+ type AUX_RX_TRANSITION_FILTER_EN; \
+ type AUX_RX_ALLOW_BELOW_THRESHOLD_PHASE_DETECT; \
+ type AUX_RX_ALLOW_BELOW_THRESHOLD_START; \
+ type AUX_RX_ALLOW_BELOW_THRESHOLD_STOP; \
+ type AUX_RX_PHASE_DETECT_LEN; \
+ type AUX_RX_DETECTION_THRESHOLD; \
+ type AUX_TX_PRECHARGE_LEN; \
+ type AUX_TX_PRECHARGE_SYMBOLS; \
+ type AUX_MODE_DET_CHECK_DELAY;\
+ type DPCS_DBG_CBUS_DIS
+#endif
+
struct dcn10_link_enc_shift {
DCN_LINK_ENCODER_REG_FIELD_LIST(uint8_t);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ DCN20_LINK_ENCODER_REG_FIELD_LIST(uint8_t);
+#endif
};
struct dcn10_link_enc_mask {
DCN_LINK_ENCODER_REG_FIELD_LIST(uint32_t);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ DCN20_LINK_ENCODER_REG_FIELD_LIST(uint32_t);
+#endif
};
struct dcn10_link_encoder {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c
index 958994edf2c4..0bca011ed7c9 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c
@@ -438,6 +438,12 @@ static const struct mpc_funcs dcn10_mpc_funcs = {
.assert_mpcc_idle_before_connect = mpc1_assert_mpcc_idle_before_connect,
.init_mpcc_list_from_hw = mpc1_init_mpcc_list_from_hw,
.update_blending = mpc1_update_blending,
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ .set_denorm = NULL,
+ .set_denorm_clamp = NULL,
+ .set_output_csc = NULL,
+ .set_output_gamma = NULL,
+#endif
};
void dcn10_mpc_construct(struct dcn10_mpc *mpc10,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c
index ab958cff3b76..e9ebbbe256b4 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "dcn10_opp.h"
#include "reg_helper.h"
@@ -365,6 +367,11 @@ void opp1_program_oppbuf(
*/
REG_UPDATE(OPPBUF_CONTROL, OPPBUF_PIXEL_REPETITION, oppbuf->pixel_repetition);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ /* Controls the number of padded pixels at the end of a segment */
+ if (REG(OPPBUF_CONTROL1))
+ REG_UPDATE(OPPBUF_CONTROL1, OPPBUF_NUM_SEGMENT_PADDED_PIXELS, oppbuf->num_segment_padded_pixels);
+#endif
}
void opp1_pipe_clock_control(struct output_pixel_processor *opp, bool enable)
@@ -391,6 +398,9 @@ static const struct opp_funcs dcn10_opp_funcs = {
.opp_program_bit_depth_reduction = opp1_program_bit_depth_reduction,
.opp_program_stereo = opp1_program_stereo,
.opp_pipe_clock_control = opp1_pipe_clock_control,
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ .opp_set_disp_pattern_generator = NULL,
+#endif
.opp_destroy = opp1_destroy
};
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c
index 0345d51e9d6f..a546c2bc9129 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c
@@ -46,9 +46,7 @@
* This is a workaround for a bug that has existed since R5xx and has not been
* fixed keep Front porch at minimum 2 for Interlaced mode or 1 for progressive.
*/
-static void optc1_apply_front_porch_workaround(
- struct timing_generator *optc,
- struct dc_crtc_timing *timing)
+static void apply_front_porch_workaround(struct dc_crtc_timing *timing)
{
if (timing->flags.INTERLACE == 1) {
if (timing->v_front_porch < 2)
@@ -60,24 +58,33 @@ static void optc1_apply_front_porch_workaround(
}
void optc1_program_global_sync(
- struct timing_generator *optc)
+ struct timing_generator *optc,
+ int vready_offset,
+ int vstartup_start,
+ int vupdate_offset,
+ int vupdate_width)
{
struct optc *optc1 = DCN10TG_FROM_TG(optc);
- if (optc->dlg_otg_param.vstartup_start == 0) {
+ optc1->vready_offset = vready_offset;
+ optc1->vstartup_start = vstartup_start;
+ optc1->vupdate_offset = vupdate_offset;
+ optc1->vupdate_width = vupdate_width;
+
+ if (optc1->vstartup_start == 0) {
BREAK_TO_DEBUGGER();
return;
}
REG_SET(OTG_VSTARTUP_PARAM, 0,
- VSTARTUP_START, optc->dlg_otg_param.vstartup_start);
+ VSTARTUP_START, optc1->vstartup_start);
REG_SET_2(OTG_VUPDATE_PARAM, 0,
- VUPDATE_OFFSET, optc->dlg_otg_param.vupdate_offset,
- VUPDATE_WIDTH, optc->dlg_otg_param.vupdate_width);
+ VUPDATE_OFFSET, optc1->vupdate_offset,
+ VUPDATE_WIDTH, optc1->vupdate_width);
REG_SET(OTG_VREADY_PARAM, 0,
- VREADY_OFFSET, optc->dlg_otg_param.vready_offset);
+ VREADY_OFFSET, optc1->vready_offset);
}
static void optc1_disable_stereo(struct timing_generator *optc)
@@ -132,25 +139,32 @@ void optc1_setup_vertical_interrupt2(
void optc1_program_timing(
struct timing_generator *optc,
const struct dc_crtc_timing *dc_crtc_timing,
+ int vready_offset,
+ int vstartup_start,
+ int vupdate_offset,
+ int vupdate_width,
+ const enum signal_type signal,
bool use_vbios)
{
struct dc_crtc_timing patched_crtc_timing;
- uint32_t vesa_sync_start;
uint32_t asic_blank_end;
uint32_t asic_blank_start;
uint32_t v_total;
uint32_t v_sync_end;
- uint32_t v_init, v_fp2;
uint32_t h_sync_polarity, v_sync_polarity;
uint32_t start_point = 0;
uint32_t field_num = 0;
uint32_t h_div_2;
- int32_t vertical_line_start;
struct optc *optc1 = DCN10TG_FROM_TG(optc);
+ optc1->signal = signal;
+ optc1->vready_offset = vready_offset;
+ optc1->vstartup_start = vstartup_start;
+ optc1->vupdate_offset = vupdate_offset;
+ optc1->vupdate_width = vupdate_width;
patched_crtc_timing = *dc_crtc_timing;
- optc1_apply_front_porch_workaround(optc, &patched_crtc_timing);
+ apply_front_porch_workaround(&patched_crtc_timing);
/* Load horizontal timing */
@@ -163,24 +177,16 @@ void optc1_program_timing(
OTG_H_SYNC_A_START, 0,
OTG_H_SYNC_A_END, patched_crtc_timing.h_sync_width);
- /* asic_h_blank_end = HsyncWidth + HbackPorch =
- * vesa. usHorizontalTotal - vesa. usHorizontalSyncStart -
- * vesa.h_left_border
- */
- vesa_sync_start = patched_crtc_timing.h_addressable +
- patched_crtc_timing.h_border_right +
+ /* blank_start = line end - front porch */
+ asic_blank_start = patched_crtc_timing.h_total -
patched_crtc_timing.h_front_porch;
- asic_blank_end = patched_crtc_timing.h_total -
- vesa_sync_start -
+ /* blank_end = blank_start - active */
+ asic_blank_end = asic_blank_start -
+ patched_crtc_timing.h_border_right -
+ patched_crtc_timing.h_addressable -
patched_crtc_timing.h_border_left;
- /* h_blank_start = v_blank_end + v_active */
- asic_blank_start = asic_blank_end +
- patched_crtc_timing.h_border_left +
- patched_crtc_timing.h_addressable +
- patched_crtc_timing.h_border_right;
-
REG_UPDATE_2(OTG_H_BLANK_START_END,
OTG_H_BLANK_START, asic_blank_start,
OTG_H_BLANK_END, asic_blank_end);
@@ -212,24 +218,15 @@ void optc1_program_timing(
OTG_V_SYNC_A_START, 0,
OTG_V_SYNC_A_END, v_sync_end);
- vesa_sync_start = patched_crtc_timing.v_addressable +
- patched_crtc_timing.v_border_bottom +
+ /* blank_start = frame end - front porch */
+ asic_blank_start = patched_crtc_timing.v_total -
patched_crtc_timing.v_front_porch;
- asic_blank_end = (patched_crtc_timing.v_total -
- vesa_sync_start -
- patched_crtc_timing.v_border_top);
-
- /* v_blank_start = v_blank_end + v_active */
- asic_blank_start = asic_blank_end +
- (patched_crtc_timing.v_border_top +
- patched_crtc_timing.v_addressable +
- patched_crtc_timing.v_border_bottom);
-
- vertical_line_start = asic_blank_end - optc->dlg_otg_param.vstartup_start + 1;
- v_fp2 = 0;
- if (vertical_line_start < 0)
- v_fp2 = -vertical_line_start;
+ /* blank_end = blank_start - active */
+ asic_blank_end = asic_blank_start -
+ patched_crtc_timing.v_border_bottom -
+ patched_crtc_timing.v_addressable -
+ patched_crtc_timing.v_border_top;
REG_UPDATE_2(OTG_V_BLANK_START_END,
OTG_V_BLANK_START, asic_blank_start,
@@ -242,10 +239,9 @@ void optc1_program_timing(
REG_UPDATE(OTG_V_SYNC_A_CNTL,
OTG_V_SYNC_A_POL, v_sync_polarity);
- v_init = asic_blank_start;
- if (optc->dlg_otg_param.signal == SIGNAL_TYPE_DISPLAY_PORT ||
- optc->dlg_otg_param.signal == SIGNAL_TYPE_DISPLAY_PORT_MST ||
- optc->dlg_otg_param.signal == SIGNAL_TYPE_EDP) {
+ if (optc1->signal == SIGNAL_TYPE_DISPLAY_PORT ||
+ optc1->signal == SIGNAL_TYPE_DISPLAY_PORT_MST ||
+ optc1->signal == SIGNAL_TYPE_EDP) {
start_point = 1;
if (patched_crtc_timing.flags.INTERLACE == 1)
field_num = 1;
@@ -253,13 +249,10 @@ void optc1_program_timing(
/* Interlace */
if (REG(OTG_INTERLACE_CONTROL)) {
- if (patched_crtc_timing.flags.INTERLACE == 1) {
+ if (patched_crtc_timing.flags.INTERLACE == 1)
REG_UPDATE(OTG_INTERLACE_CONTROL,
OTG_INTERLACE_ENABLE, 1);
- v_init = v_init / 2;
- if ((optc->dlg_otg_param.vstartup_start/2)*2 > asic_blank_end)
- v_fp2 = v_fp2 / 2;
- } else
+ else
REG_UPDATE(OTG_INTERLACE_CONTROL,
OTG_INTERLACE_ENABLE, 0);
}
@@ -268,16 +261,18 @@ void optc1_program_timing(
REG_UPDATE(CONTROL,
VTG0_ENABLE, 0);
- REG_UPDATE_2(CONTROL,
- VTG0_FP2, v_fp2,
- VTG0_VCOUNT_INIT, v_init);
-
/* original code is using VTG offset to address OTG reg, seems wrong */
REG_UPDATE_2(OTG_CONTROL,
OTG_START_POINT_CNTL, start_point,
OTG_FIELD_NUMBER_CNTL, field_num);
- optc1_program_global_sync(optc);
+ optc->funcs->program_global_sync(optc,
+ vready_offset,
+ vstartup_start,
+ vupdate_offset,
+ vupdate_width);
+
+ optc->funcs->set_vtg_params(optc, dc_crtc_timing);
/* TODO
* patched_crtc_timing.flags.HORZ_COUNT_BY_TWO == 1
@@ -296,6 +291,48 @@ void optc1_program_timing(
}
+void optc1_set_vtg_params(struct timing_generator *optc,
+ const struct dc_crtc_timing *dc_crtc_timing)
+{
+ struct dc_crtc_timing patched_crtc_timing;
+ uint32_t asic_blank_end;
+ uint32_t v_init;
+ uint32_t v_fp2 = 0;
+ int32_t vertical_line_start;
+
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ patched_crtc_timing = *dc_crtc_timing;
+ apply_front_porch_workaround(&patched_crtc_timing);
+
+ /* VCOUNT_INIT is the start of blank */
+ v_init = patched_crtc_timing.v_total - patched_crtc_timing.v_front_porch;
+
+ /* end of blank = v_init - active */
+ asic_blank_end = v_init -
+ patched_crtc_timing.v_border_bottom -
+ patched_crtc_timing.v_addressable -
+ patched_crtc_timing.v_border_top;
+
+ /* if VSTARTUP is before VSYNC, FP2 is the offset, otherwise 0 */
+ vertical_line_start = asic_blank_end - optc1->vstartup_start + 1;
+ if (vertical_line_start < 0)
+ v_fp2 = -vertical_line_start;
+
+ /* Interlace */
+ if (REG(OTG_INTERLACE_CONTROL)) {
+ if (patched_crtc_timing.flags.INTERLACE == 1) {
+ v_init = v_init / 2;
+ if ((optc1->vstartup_start/2)*2 > asic_blank_end)
+ v_fp2 = v_fp2 / 2;
+ }
+ }
+
+ REG_UPDATE_2(CONTROL,
+ VTG0_FP2, v_fp2,
+ VTG0_VCOUNT_INIT, v_init);
+}
+
void optc1_set_blank_data_double_buffer(struct timing_generator *optc, bool enable)
{
struct optc *optc1 = DCN10TG_FROM_TG(optc);
@@ -547,6 +584,13 @@ uint32_t optc1_get_vblank_counter(struct timing_generator *optc)
void optc1_lock(struct timing_generator *optc)
{
struct optc *optc1 = DCN10TG_FROM_TG(optc);
+ uint32_t regval = 0;
+
+ regval = REG_READ(OTG_CONTROL);
+
+ /* otg is not running, do not need to be locked */
+ if ((regval & 0x1) == 0x0)
+ return;
REG_SET(OTG_GLOBAL_CONTROL0, 0,
OTG_MASTER_UPDATE_LOCK_SEL, optc->inst);
@@ -554,10 +598,12 @@ void optc1_lock(struct timing_generator *optc)
OTG_MASTER_UPDATE_LOCK, 1);
/* Should be fast, status does not update on maximus */
- if (optc->ctx->dce_environment != DCE_ENV_FPGA_MAXIMUS)
+ if (optc->ctx->dce_environment != DCE_ENV_FPGA_MAXIMUS) {
+
REG_WAIT(OTG_MASTER_UPDATE_LOCK,
UPDATE_LOCK_STATUS, 1,
1, 10);
+ }
}
void optc1_unlock(struct timing_generator *optc)
@@ -754,6 +800,32 @@ void optc1_set_static_screen_control(
OTG_STATIC_SCREEN_FRAME_COUNT, 2);
}
+void optc1_setup_manual_trigger(struct timing_generator *optc)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ REG_SET(OTG_GLOBAL_CONTROL2, 0,
+ MANUAL_FLOW_CONTROL_SEL, optc->inst);
+
+ REG_SET_8(OTG_TRIGA_CNTL, 0,
+ OTG_TRIGA_SOURCE_SELECT, 22,
+ OTG_TRIGA_SOURCE_PIPE_SELECT, optc->inst,
+ OTG_TRIGA_RISING_EDGE_DETECT_CNTL, 1,
+ OTG_TRIGA_FALLING_EDGE_DETECT_CNTL, 0,
+ OTG_TRIGA_POLARITY_SELECT, 0,
+ OTG_TRIGA_FREQUENCY_SELECT, 0,
+ OTG_TRIGA_DELAY, 0,
+ OTG_TRIGA_CLEAR, 1);
+}
+
+void optc1_program_manual_trigger(struct timing_generator *optc)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ REG_SET(OTG_MANUAL_FLOW_CONTROL, 0,
+ MANUAL_FLOW_CONTROL, 1);
+}
+
/**
*****************************************************************************
@@ -786,6 +858,10 @@ void optc1_set_drr(
OTG_FORCE_LOCK_ON_EVENT, 0,
OTG_SET_V_TOTAL_MIN_MASK_EN, 0,
OTG_SET_V_TOTAL_MIN_MASK, 0);
+
+ // Setup manual flow control for EOF via TRIG_A
+ optc->funcs->setup_manual_trigger(optc);
+
} else {
REG_UPDATE_4(OTG_V_TOTAL_CONTROL,
OTG_SET_V_TOTAL_MIN_MASK, 0,
@@ -1420,6 +1496,9 @@ static const struct timing_generator_funcs dcn10_tg_funcs = {
.clear_optc_underflow = optc1_clear_optc_underflow,
.get_crc = optc1_get_crc,
.configure_crc = optc1_configure_crc,
+ .set_vtg_params = optc1_set_vtg_params,
+ .program_manual_trigger = optc1_program_manual_trigger,
+ .setup_manual_trigger = optc1_setup_manual_trigger
};
void dcn10_timing_generator_init(struct optc *optc1)
@@ -1437,10 +1516,28 @@ void dcn10_timing_generator_init(struct optc *optc1)
optc1->comb_opp_id = 0xf;
}
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+/* "Containter" vs. "pixel" is a concept within HW blocks, mostly those closer to the back-end. It works like this:
+ *
+ * - In most of the formats (RGB or YCbCr 4:4:4, 4:2:2 uncompressed and DSC 4:2:2 Simple) pixel rate is the same as
+ * containter rate.
+ *
+ * - In 4:2:0 (DSC or uncompressed) there are two pixels per container, hence the target container rate has to be
+ * halved to maintain the correct pixel rate.
+ *
+ * - Unlike 4:2:2 uncompressed, DSC 4:2:2 Native also has two pixels per container (this happens when DSC is applied
+ * to it) and has to be treated the same as 4:2:0, i.e. target containter rate has to be halved in this case as well.
+ *
+ */
+#endif
bool optc1_is_two_pixels_per_containter(const struct dc_crtc_timing *timing)
{
bool two_pix = timing->pixel_encoding == PIXEL_ENCODING_YCBCR420;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ two_pix = two_pix || (timing->flags.DSC && timing->pixel_encoding == PIXEL_ENCODING_YCBCR422
+ && !timing->dsc_cfg.ycbcr422_simple);
+#endif
return two_pix;
}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
index 4eb9a898c237..02599eb92ca6 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
@@ -84,13 +84,17 @@
SRI(OTG_CRC0_WINDOWA_X_CONTROL, OTG, inst),\
SRI(OTG_CRC0_WINDOWA_Y_CONTROL, OTG, inst),\
SRI(OTG_CRC0_WINDOWB_X_CONTROL, OTG, inst),\
- SRI(OTG_CRC0_WINDOWB_Y_CONTROL, OTG, inst)
+ SRI(OTG_CRC0_WINDOWB_Y_CONTROL, OTG, inst),\
+ SR(GSL_SOURCE_SELECT),\
+ SRI(OTG_GLOBAL_CONTROL2, OTG, inst),\
+ SRI(OTG_TRIGA_MANUAL_TRIG, OTG, inst)
#define TG_COMMON_REG_LIST_DCN1_0(inst) \
TG_COMMON_REG_LIST_DCN(inst),\
SRI(OTG_TEST_PATTERN_PARAMETERS, OTG, inst),\
SRI(OTG_TEST_PATTERN_CONTROL, OTG, inst),\
- SRI(OTG_TEST_PATTERN_COLOR, OTG, inst)
+ SRI(OTG_TEST_PATTERN_COLOR, OTG, inst),\
+ SRI(OTG_MANUAL_FLOW_CONTROL, OTG, inst)
struct dcn_optc_registers {
@@ -124,6 +128,8 @@ struct dcn_optc_registers {
uint32_t OTG_V_TOTAL_MIN;
uint32_t OTG_V_TOTAL_CONTROL;
uint32_t OTG_TRIGA_CNTL;
+ uint32_t OTG_TRIGA_MANUAL_TRIG;
+ uint32_t OTG_MANUAL_FLOW_CONTROL;
uint32_t OTG_FORCE_COUNT_NOW_CNTL;
uint32_t OTG_STATIC_SCREEN_CONTROL;
uint32_t OTG_STATUS_FRAME_COUNT;
@@ -156,6 +162,14 @@ struct dcn_optc_registers {
uint32_t OTG_CRC0_WINDOWA_Y_CONTROL;
uint32_t OTG_CRC0_WINDOWB_X_CONTROL;
uint32_t OTG_CRC0_WINDOWB_Y_CONTROL;
+ uint32_t GSL_SOURCE_SELECT;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ uint32_t DWB_SOURCE_SELECT;
+ uint32_t OTG_DSC_START_POSITION;
+ uint32_t OPTC_DATA_FORMAT_CONTROL;
+ uint32_t OPTC_BYTES_PER_PIXEL;
+ uint32_t OPTC_WIDTH_CONTROL;
+#endif
};
#define TG_COMMON_MASK_SH_LIST_DCN(mask_sh)\
@@ -213,6 +227,11 @@ struct dcn_optc_registers {
SF(OTG0_OTG_TRIGA_CNTL, OTG_TRIGA_SOURCE_PIPE_SELECT, mask_sh),\
SF(OTG0_OTG_TRIGA_CNTL, OTG_TRIGA_RISING_EDGE_DETECT_CNTL, mask_sh),\
SF(OTG0_OTG_TRIGA_CNTL, OTG_TRIGA_FALLING_EDGE_DETECT_CNTL, mask_sh),\
+ SF(OTG0_OTG_TRIGA_CNTL, OTG_TRIGA_POLARITY_SELECT, mask_sh),\
+ SF(OTG0_OTG_TRIGA_CNTL, OTG_TRIGA_FREQUENCY_SELECT, mask_sh),\
+ SF(OTG0_OTG_TRIGA_CNTL, OTG_TRIGA_DELAY, mask_sh),\
+ SF(OTG0_OTG_TRIGA_CNTL, OTG_TRIGA_CLEAR, mask_sh),\
+ SF(OTG0_OTG_TRIGA_MANUAL_TRIG, OTG_TRIGA_MANUAL_TRIG, mask_sh),\
SF(OTG0_OTG_STATIC_SCREEN_CONTROL, OTG_STATIC_SCREEN_EVENT_MASK, mask_sh),\
SF(OTG0_OTG_STATIC_SCREEN_CONTROL, OTG_STATIC_SCREEN_FRAME_COUNT, mask_sh),\
SF(OTG0_OTG_STATUS_FRAME_COUNT, OTG_FRAME_COUNT, mask_sh),\
@@ -266,8 +285,11 @@ struct dcn_optc_registers {
SF(OTG0_OTG_CRC0_WINDOWB_X_CONTROL, OTG_CRC0_WINDOWB_X_START, mask_sh),\
SF(OTG0_OTG_CRC0_WINDOWB_X_CONTROL, OTG_CRC0_WINDOWB_X_END, mask_sh),\
SF(OTG0_OTG_CRC0_WINDOWB_Y_CONTROL, OTG_CRC0_WINDOWB_Y_START, mask_sh),\
- SF(OTG0_OTG_CRC0_WINDOWB_Y_CONTROL, OTG_CRC0_WINDOWB_Y_END, mask_sh)
-
+ SF(OTG0_OTG_CRC0_WINDOWB_Y_CONTROL, OTG_CRC0_WINDOWB_Y_END, mask_sh),\
+ SF(GSL_SOURCE_SELECT, GSL0_READY_SOURCE_SEL, mask_sh),\
+ SF(GSL_SOURCE_SELECT, GSL1_READY_SOURCE_SEL, mask_sh),\
+ SF(GSL_SOURCE_SELECT, GSL2_READY_SOURCE_SEL, mask_sh),\
+ SF(OTG0_OTG_GLOBAL_CONTROL2, MANUAL_FLOW_CONTROL_SEL, mask_sh)
#define TG_COMMON_MASK_SH_LIST_DCN1_0(mask_sh)\
TG_COMMON_MASK_SH_LIST_DCN(mask_sh),\
@@ -282,7 +304,8 @@ struct dcn_optc_registers {
SF(OTG0_OTG_TEST_PATTERN_CONTROL, OTG_TEST_PATTERN_COLOR_FORMAT, mask_sh),\
SF(OTG0_OTG_TEST_PATTERN_COLOR, OTG_TEST_PATTERN_MASK, mask_sh),\
SF(OTG0_OTG_TEST_PATTERN_COLOR, OTG_TEST_PATTERN_DATA, mask_sh),\
- SF(ODM0_OPTC_DATA_SOURCE_SELECT, OPTC_SRC_SEL, mask_sh)
+ SF(ODM0_OPTC_DATA_SOURCE_SELECT, OPTC_SRC_SEL, mask_sh),\
+ SF(OTG0_OTG_MANUAL_FLOW_CONTROL, MANUAL_FLOW_CONTROL, mask_sh),\
#define TG_REG_FIELD_LIST_DCN1_0(type) \
type VSTARTUP_START;\
@@ -338,6 +361,11 @@ struct dcn_optc_registers {
type OTG_TRIGA_SOURCE_PIPE_SELECT;\
type OTG_TRIGA_RISING_EDGE_DETECT_CNTL;\
type OTG_TRIGA_FALLING_EDGE_DETECT_CNTL;\
+ type OTG_TRIGA_POLARITY_SELECT;\
+ type OTG_TRIGA_FREQUENCY_SELECT;\
+ type OTG_TRIGA_DELAY;\
+ type OTG_TRIGA_CLEAR;\
+ type OTG_TRIGA_MANUAL_TRIG;\
type OTG_STATIC_SCREEN_EVENT_MASK;\
type OTG_STATIC_SCREEN_FRAME_COUNT;\
type OTG_FRAME_COUNT;\
@@ -413,12 +441,43 @@ struct dcn_optc_registers {
type OTG_CRC0_WINDOWB_X_START;\
type OTG_CRC0_WINDOWB_X_END;\
type OTG_CRC0_WINDOWB_Y_START;\
- type OTG_CRC0_WINDOWB_Y_END;
+ type OTG_CRC0_WINDOWB_Y_END;\
+ type GSL0_READY_SOURCE_SEL;\
+ type GSL1_READY_SOURCE_SEL;\
+ type GSL2_READY_SOURCE_SEL;\
+ type MANUAL_FLOW_CONTROL;\
+ type MANUAL_FLOW_CONTROL_SEL;
+
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+#define TG_REG_FIELD_LIST(type) \
+ TG_REG_FIELD_LIST_DCN1_0(type)\
+ type MASTER_UPDATE_LOCK_DB_X;\
+ type MASTER_UPDATE_LOCK_DB_Y;\
+ type MASTER_UPDATE_LOCK_DB_EN;\
+ type GLOBAL_UPDATE_LOCK_EN;\
+ type DIG_UPDATE_LOCATION;\
+ type OTG_DSC_START_POSITION_X;\
+ type OTG_DSC_START_POSITION_LINE_NUM;\
+ type OPTC_NUM_OF_INPUT_SEGMENT;\
+ type OPTC_SEG0_SRC_SEL;\
+ type OPTC_SEG1_SRC_SEL;\
+ type OPTC_MEM_SEL;\
+ type OPTC_DATA_FORMAT;\
+ type OPTC_DSC_MODE;\
+ type OPTC_DSC_BYTES_PER_PIXEL;\
+ type OPTC_DSC_SLICE_WIDTH;\
+ type OPTC_SEGMENT_WIDTH;\
+ type OPTC_DWB0_SOURCE_SELECT;\
+ type OPTC_DWB1_SOURCE_SELECT;
+
+#else
#define TG_REG_FIELD_LIST(type) \
TG_REG_FIELD_LIST_DCN1_0(type)
+#endif
+
struct dcn_optc_shift {
TG_REG_FIELD_LIST(uint8_t)
@@ -446,6 +505,12 @@ struct optc {
uint32_t min_v_sync_width;
uint32_t min_v_blank;
uint32_t min_v_blank_interlace;
+
+ int vstartup_start;
+ int vupdate_offset;
+ int vupdate_width;
+ int vready_offset;
+ enum signal_type signal;
};
void dcn10_timing_generator_init(struct optc *optc);
@@ -481,6 +546,11 @@ bool optc1_validate_timing(
void optc1_program_timing(
struct timing_generator *optc,
const struct dc_crtc_timing *dc_crtc_timing,
+ int vready_offset,
+ int vstartup_start,
+ int vupdate_offset,
+ int vupdate_width,
+ const enum signal_type signal,
bool use_vbios);
void optc1_setup_vertical_interrupt0(
@@ -495,7 +565,11 @@ void optc1_setup_vertical_interrupt2(
uint32_t start_line);
void optc1_program_global_sync(
- struct timing_generator *optc);
+ struct timing_generator *optc,
+ int vready_offset,
+ int vstartup_start,
+ int vupdate_offset,
+ int vupdate_width);
bool optc1_disable_crtc(struct timing_generator *optc);
@@ -582,4 +656,7 @@ bool optc1_get_crc(struct timing_generator *optc,
bool optc1_is_two_pixels_per_containter(const struct dc_crtc_timing *timing);
+void optc1_set_vtg_params(struct timing_generator *optc,
+ const struct dc_crtc_timing *dc_crtc_timing);
+
#endif /* __DC_TIMING_GENERATOR_DCN10_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
index 7eccb54c421d..a12530a3ab9c 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
@@ -23,13 +23,14 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "dc.h"
#include "resource.h"
#include "include/irq_service_interface.h"
#include "dcn10_resource.h"
-
#include "dcn10_ipp.h"
#include "dcn10_mpc.h"
#include "irq/dcn10/irq_service_dcn10.h"
@@ -40,7 +41,6 @@
#include "dcn10_opp.h"
#include "dcn10_link_encoder.h"
#include "dcn10_stream_encoder.h"
-#include "dcn10_clk_mgr.h"
#include "dce/dce_clock_source.h"
#include "dce/dce_audio.h"
#include "dce/dce_hwseq.h"
@@ -153,9 +153,7 @@ enum dcn10_clk_src_array_id {
DCN10_CLK_SRC_PLL2,
DCN10_CLK_SRC_PLL3,
DCN10_CLK_SRC_TOTAL,
-#if defined(CONFIG_DRM_AMD_DC_DCN1_01)
DCN101_CLK_SRC_TOTAL = DCN10_CLK_SRC_PLL3
-#endif
};
/* begin *********************
@@ -202,6 +200,7 @@ enum dcn10_clk_src_array_id {
#define MMHUB_SR(reg_name)\
.reg_name = MMHUB_BASE(mm ## reg_name ## _BASE_IDX) + \
mm ## reg_name
+
/* macros to expend register list macro defined in HW object header file
* end *********************/
@@ -445,7 +444,6 @@ static const struct bios_registers bios_regs = {
HUBP_REG_LIST_DCN10(id)\
}
-
static const struct dcn_mi_registers hubp_regs[] = {
hubp_regs(0),
hubp_regs(1),
@@ -461,7 +459,6 @@ static const struct dcn_mi_mask hubp_mask = {
HUBP_MASK_SH_LIST_DCN10(_MASK)
};
-
static const struct dcn_hubbub_registers hubbub_reg = {
HUBBUB_REG_LIST_DCN10(0)
};
@@ -504,7 +501,6 @@ static const struct resource_caps res_cap = {
.num_ddc = 4,
};
-#if defined(CONFIG_DRM_AMD_DC_DCN1_01)
static const struct resource_caps rv2_res_cap = {
.num_timing_generator = 3,
.num_opp = 3,
@@ -512,9 +508,8 @@ static const struct resource_caps rv2_res_cap = {
.num_audio = 3,
.num_stream_encoder = 3,
.num_pll = 3,
- .num_ddc = 3,
+ .num_ddc = 4,
};
-#endif
static const struct dc_plane_cap plane_cap = {
.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
@@ -567,6 +562,7 @@ static const struct dc_debug_options debug_defaults_drv = {
.az_endpoint_mute_only = true,
.recovery_enabled = false, /*enable this by default after testing.*/
.max_downscale_src_width = 3840,
+ .underflow_assert_delay_us = 0xFFFFFFFF,
};
static const struct dc_debug_options debug_defaults_diags = {
@@ -576,7 +572,8 @@ static const struct dc_debug_options debug_defaults_diags = {
.clock_trace = true,
.disable_stutter = true,
.disable_pplib_clock_request = true,
- .disable_pplib_wm_range = true
+ .disable_pplib_wm_range = true,
+ .underflow_assert_delay_us = 0xFFFFFFFF,
};
static void dcn10_dpp_destroy(struct dpp **dpp)
@@ -966,9 +963,6 @@ static void destruct(struct dcn10_resource_pool *pool)
if (pool->base.dmcu != NULL)
dce_dmcu_destroy(&pool->base.dmcu);
- if (pool->base.clk_mgr != NULL)
- dce_clk_mgr_destroy(&pool->base.clk_mgr);
-
kfree(pool->base.pp_smu);
}
@@ -1217,6 +1211,38 @@ static enum dc_status dcn10_get_default_swizzle_mode(struct dc_plane_state *plan
return result;
}
+struct stream_encoder *dcn10_find_first_free_match_stream_enc_for_link(
+ struct resource_context *res_ctx,
+ const struct resource_pool *pool,
+ struct dc_stream_state *stream)
+{
+ int i;
+ int j = -1;
+ struct dc_link *link = stream->link;
+
+ for (i = 0; i < pool->stream_enc_count; i++) {
+ if (!res_ctx->is_stream_enc_acquired[i] &&
+ pool->stream_enc[i]) {
+ /* Store first available for MST second display
+ * in daisy chain use case
+ */
+ j = i;
+ if (pool->stream_enc[i]->id ==
+ link->link_enc->preferred_engine)
+ return pool->stream_enc[i];
+ }
+ }
+
+ /*
+ * For CZ and later, we can allow DIG FE and BE to differ for all display types
+ */
+
+ if (j >= 0)
+ return pool->stream_enc[j];
+
+ return NULL;
+}
+
static const struct dc_cap_funcs cap_funcs = {
.get_dcc_compression_cap = dcn10_get_dcc_compression_cap
};
@@ -1229,7 +1255,8 @@ static const struct resource_funcs dcn10_res_pool_funcs = {
.validate_plane = dcn10_validate_plane,
.validate_global = dcn10_validate_global,
.add_stream_to_ctx = dcn10_add_stream_to_ctx,
- .get_default_swizzle_mode = dcn10_get_default_swizzle_mode
+ .get_default_swizzle_mode = dcn10_get_default_swizzle_mode,
+ .find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link
};
static uint32_t read_pipe_fuses(struct dc_context *ctx)
@@ -1252,11 +1279,9 @@ static bool construct(
ctx->dc_bios->regs = &bios_regs;
-#if defined(CONFIG_DRM_AMD_DC_DCN1_01)
if (ctx->dce_version == DCN_VERSION_1_01)
pool->base.res_cap = &rv2_res_cap;
else
-#endif
pool->base.res_cap = &res_cap;
pool->base.funcs = &dcn10_res_pool_funcs;
@@ -1273,10 +1298,8 @@ static bool construct(
/* max pipe num for ASIC before check pipe fuses */
pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
-#if defined(CONFIG_DRM_AMD_DC_DCN1_01)
if (dc->ctx->dce_version == DCN_VERSION_1_01)
pool->base.pipe_count = 3;
-#endif
dc->caps.max_video_width = 3840;
dc->caps.max_downscale_ratio = 200;
dc->caps.i2c_speed_in_khz = 100;
@@ -1309,26 +1332,17 @@ static bool construct(
CLOCK_SOURCE_COMBO_PHY_PLL2,
&clk_src_regs[2], false);
-#ifdef CONFIG_DRM_AMD_DC_DCN1_01
if (dc->ctx->dce_version == DCN_VERSION_1_0) {
pool->base.clock_sources[DCN10_CLK_SRC_PLL3] =
dcn10_clock_source_create(ctx, ctx->dc_bios,
CLOCK_SOURCE_COMBO_PHY_PLL3,
&clk_src_regs[3], false);
}
-#else
- pool->base.clock_sources[DCN10_CLK_SRC_PLL3] =
- dcn10_clock_source_create(ctx, ctx->dc_bios,
- CLOCK_SOURCE_COMBO_PHY_PLL3,
- &clk_src_regs[3], false);
-#endif
pool->base.clk_src_count = DCN10_CLK_SRC_TOTAL;
-#if defined(CONFIG_DRM_AMD_DC_DCN1_01)
if (dc->ctx->dce_version == DCN_VERSION_1_01)
pool->base.clk_src_count = DCN101_CLK_SRC_TOTAL;
-#endif
pool->base.dp_clock_source =
dcn10_clock_source_create(ctx, ctx->dc_bios,
@@ -1343,12 +1357,6 @@ static bool construct(
goto fail;
}
}
- pool->base.clk_mgr = dcn1_clk_mgr_create(ctx);
- if (pool->base.clk_mgr == NULL) {
- dm_error("DC: failed to create display clock!\n");
- BREAK_TO_DEBUGGER();
- goto fail;
- }
pool->base.dmcu = dcn10_dmcu_create(ctx,
&dmcu_regs,
@@ -1374,7 +1382,6 @@ static bool construct(
memcpy(dc->dcn_ip, &dcn10_ip_defaults, sizeof(dcn10_ip_defaults));
memcpy(dc->dcn_soc, &dcn10_soc_defaults, sizeof(dcn10_soc_defaults));
-#if defined(CONFIG_DRM_AMD_DC_DCN1_01)
if (dc->ctx->dce_version == DCN_VERSION_1_01) {
struct dcn_soc_bounding_box *dcn_soc = dc->dcn_soc;
struct dcn_ip_params *dcn_ip = dc->dcn_ip;
@@ -1385,7 +1392,6 @@ static bool construct(
dcn_soc->dram_clock_change_latency = 23;
dcn_ip->max_num_dpp = 3;
}
-#endif
if (ASICREV_IS_RV1_F0(dc->ctx->asic_id.hw_internal_rev)) {
dc->dcn_soc->urgent_latency = 3;
dc->debug.disable_dmcu = true;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.h
index 999c684a0b36..633025ccb870 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.h
@@ -42,6 +42,11 @@ struct resource_pool *dcn10_create_resource_pool(
const struct dc_init_data *init_data,
struct dc *dc);
+struct stream_encoder *dcn10_find_first_free_match_stream_enc_for_link(
+ struct resource_context *res_ctx,
+ const struct resource_pool *pool,
+ struct dc_stream_state *stream);
+
#endif /* __DC_RESOURCE_DCN10_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c
index 8ee9f6dc1d62..b9ffbf6b58ff 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c
@@ -23,6 +23,7 @@
*
*/
+#include <linux/delay.h>
#include "dc_bios_types.h"
#include "dcn10_stream_encoder.h"
@@ -415,6 +416,7 @@ void enc1_stream_encoder_dp_set_stream_attribute(
case COLOR_SPACE_APPCTRL:
case COLOR_SPACE_CUSTOMPOINTS:
case COLOR_SPACE_UNKNOWN:
+ case COLOR_SPACE_YCBCR709_BLACK:
/* do nothing */
break;
}
@@ -471,7 +473,7 @@ void enc1_stream_encoder_dp_set_stream_attribute(
hw_crtc_timing.v_addressable + hw_crtc_timing.v_border_bottom);
}
-static void enc1_stream_encoder_set_stream_attribute_helper(
+void enc1_stream_encoder_set_stream_attribute_helper(
struct dcn10_stream_encoder *enc1,
struct dc_crtc_timing *crtc_timing)
{
@@ -726,11 +728,9 @@ void enc1_stream_encoder_update_dp_info_packets(
3, /* packetIndex */
&info_frame->hdrsmd);
- if (info_frame->dpsdp.valid)
- enc1_update_generic_info_packet(
- enc1,
- 4,/* packetIndex */
- &info_frame->dpsdp);
+ /* packetIndex 4 is used for send immediate sdp message, and please
+ * use other packetIndex (such as 5,6) for other info packet
+ */
/* enable/disable transmission of packet(s).
* If enabled, packet transmission begins on the next frame
@@ -738,7 +738,101 @@ void enc1_stream_encoder_update_dp_info_packets(
REG_UPDATE(DP_SEC_CNTL, DP_SEC_GSP0_ENABLE, info_frame->vsc.valid);
REG_UPDATE(DP_SEC_CNTL, DP_SEC_GSP2_ENABLE, info_frame->spd.valid);
REG_UPDATE(DP_SEC_CNTL, DP_SEC_GSP3_ENABLE, info_frame->hdrsmd.valid);
- REG_UPDATE(DP_SEC_CNTL, DP_SEC_GSP4_ENABLE, info_frame->dpsdp.valid);
+
+
+ /* This bit is the master enable bit.
+ * When enabling secondary stream engine,
+ * this master bit must also be set.
+ * This register shared with audio info frame.
+ * Therefore we need to enable master bit
+ * if at least on of the fields is not 0
+ */
+ value = REG_READ(DP_SEC_CNTL);
+ if (value)
+ REG_UPDATE(DP_SEC_CNTL, DP_SEC_STREAM_ENABLE, 1);
+}
+
+void enc1_stream_encoder_send_immediate_sdp_message(
+ struct stream_encoder *enc,
+ const uint8_t *custom_sdp_message,
+ unsigned int sdp_message_size)
+{
+ struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+ uint32_t value = 0;
+
+ /* TODOFPGA Figure out a proper number for max_retries polling for lock
+ * use 50 for now.
+ */
+ uint32_t max_retries = 50;
+
+ /* check if GSP4 is transmitted */
+ REG_WAIT(DP_SEC_CNTL2, DP_SEC_GSP4_SEND_PENDING,
+ 0, 10, max_retries);
+
+ /* disable GSP4 transmitting */
+ REG_UPDATE(DP_SEC_CNTL2, DP_SEC_GSP4_SEND, 0);
+
+ /* transmit GSP4 at the earliest time in a frame */
+ REG_UPDATE(DP_SEC_CNTL2, DP_SEC_GSP4_SEND_ANY_LINE, 1);
+
+ /*we need turn on clock before programming AFMT block*/
+ REG_UPDATE(AFMT_CNTL, AFMT_AUDIO_CLOCK_EN, 1);
+
+ /* check if HW reading GSP memory */
+ REG_WAIT(AFMT_VBI_PACKET_CONTROL, AFMT_GENERIC_CONFLICT,
+ 0, 10, max_retries);
+
+ /* HW does is not reading GSP memory not reading too long ->
+ * something wrong. clear GPS memory access and notify?
+ * hw SW is writing to GSP memory
+ */
+ REG_UPDATE(AFMT_VBI_PACKET_CONTROL, AFMT_GENERIC_CONFLICT_CLR, 1);
+
+ /* use generic packet 4 for immediate sdp message */
+ REG_UPDATE(AFMT_VBI_PACKET_CONTROL,
+ AFMT_GENERIC_INDEX, 4);
+
+ /* write generic packet header
+ * (4th byte is for GENERIC0 only)
+ */
+ REG_SET_4(AFMT_GENERIC_HDR, 0,
+ AFMT_GENERIC_HB0, custom_sdp_message[0],
+ AFMT_GENERIC_HB1, custom_sdp_message[1],
+ AFMT_GENERIC_HB2, custom_sdp_message[2],
+ AFMT_GENERIC_HB3, custom_sdp_message[3]);
+
+ /* write generic packet contents
+ * (we never use last 4 bytes)
+ * there are 8 (0-7) mmDIG0_AFMT_GENERIC0_x registers
+ */
+ {
+ const uint32_t *content =
+ (const uint32_t *) &custom_sdp_message[4];
+
+ REG_WRITE(AFMT_GENERIC_0, *content++);
+ REG_WRITE(AFMT_GENERIC_1, *content++);
+ REG_WRITE(AFMT_GENERIC_2, *content++);
+ REG_WRITE(AFMT_GENERIC_3, *content++);
+ REG_WRITE(AFMT_GENERIC_4, *content++);
+ REG_WRITE(AFMT_GENERIC_5, *content++);
+ REG_WRITE(AFMT_GENERIC_6, *content++);
+ REG_WRITE(AFMT_GENERIC_7, *content);
+ }
+
+ /* check whether GENERIC4 registers double buffer update in immediate mode
+ * is pending
+ */
+ REG_WAIT(AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC4_IMMEDIATE_UPDATE_PENDING,
+ 0, 10, max_retries);
+
+ /* atomically update double-buffered GENERIC4 registers in immediate mode
+ * (update immediately)
+ */
+ REG_UPDATE(AFMT_VBI_PACKET_CONTROL1,
+ AFMT_GENERIC4_IMMEDIATE_UPDATE, 1);
+
+ /* enable GSP4 transmitting */
+ REG_UPDATE(DP_SEC_CNTL2, DP_SEC_GSP4_SEND, 1);
/* This bit is the master enable bit.
* When enabling secondary stream engine,
@@ -998,19 +1092,6 @@ union audio_cea_channels {
} channels;
};
-struct audio_clock_info {
- /* pixel clock frequency*/
- uint32_t pixel_clock_in_10khz;
- /* N - 32KHz audio */
- uint32_t n_32khz;
- /* CTS - 32KHz audio*/
- uint32_t cts_32khz;
- uint32_t n_44khz;
- uint32_t cts_44khz;
- uint32_t n_48khz;
- uint32_t cts_48khz;
-};
-
/* 25.2MHz/1.001*/
/* 25.2MHz/1.001*/
/* 25.2MHz*/
@@ -1113,7 +1194,7 @@ static union audio_cea_channels speakers_to_channels(
return cea_channels;
}
-static void get_audio_clock_info(
+void get_audio_clock_info(
enum dc_color_depth color_depth,
uint32_t crtc_pixel_clock_in_khz,
uint32_t actual_pixel_clock_in_khz,
@@ -1317,7 +1398,7 @@ static void enc1_se_setup_dp_audio(
REG_UPDATE(AFMT_60958_0, AFMT_60958_CS_CLOCK_ACCURACY, 0);
}
-static void enc1_se_enable_audio_clock(
+void enc1_se_enable_audio_clock(
struct stream_encoder *enc,
bool enable)
{
@@ -1339,7 +1420,7 @@ static void enc1_se_enable_audio_clock(
*/
}
-static void enc1_se_enable_dp_audio(
+void enc1_se_enable_dp_audio(
struct stream_encoder *enc)
{
struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
@@ -1462,6 +1543,8 @@ static const struct stream_encoder_funcs dcn10_str_enc_funcs = {
enc1_stream_encoder_stop_hdmi_info_packets,
.update_dp_info_packets =
enc1_stream_encoder_update_dp_info_packets,
+ .send_immediate_sdp_message =
+ enc1_stream_encoder_send_immediate_sdp_message,
.stop_dp_info_packets =
enc1_stream_encoder_stop_dp_info_packets,
.dp_blank =
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h
index e654c2f55971..bc2b4af9543b 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h
@@ -81,6 +81,8 @@
SRI(DP_MSE_RATE_UPDATE, DP, id), \
SRI(DP_PIXEL_FORMAT, DP, id), \
SRI(DP_SEC_CNTL, DP, id), \
+ SRI(DP_SEC_CNTL2, DP, id), \
+ SRI(DP_SEC_CNTL6, DP, id), \
SRI(DP_STEER_FIFO, DP, id), \
SRI(DP_VID_M, DP, id), \
SRI(DP_VID_N, DP, id), \
@@ -118,10 +120,13 @@ struct dcn10_stream_enc_registers {
uint32_t AFMT_60958_1;
uint32_t AFMT_60958_2;
uint32_t DIG_FE_CNTL;
+ uint32_t DIG_FE_CNTL2;
uint32_t DP_MSE_RATE_CNTL;
uint32_t DP_MSE_RATE_UPDATE;
uint32_t DP_PIXEL_FORMAT;
uint32_t DP_SEC_CNTL;
+ uint32_t DP_SEC_CNTL2;
+ uint32_t DP_SEC_CNTL6;
uint32_t DP_STEER_FIFO;
uint32_t DP_VID_M;
uint32_t DP_VID_N;
@@ -150,12 +155,21 @@ struct dcn10_stream_enc_registers {
uint32_t HDMI_ACR_48_1;
uint32_t DP_DB_CNTL;
uint32_t DP_MSA_MISC;
+ uint32_t DP_MSA_VBID_MISC;
uint32_t DP_MSA_COLORIMETRY;
uint32_t DP_MSA_TIMING_PARAM1;
uint32_t DP_MSA_TIMING_PARAM2;
uint32_t DP_MSA_TIMING_PARAM3;
uint32_t DP_MSA_TIMING_PARAM4;
uint32_t HDMI_DB_CONTROL;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ uint32_t DP_DSC_CNTL;
+ uint32_t DP_DSC_BYTES_PER_PIXEL;
+ uint32_t DME_CONTROL;
+ uint32_t DP_SEC_METADATA_TRANSMISSION;
+ uint32_t HDMI_METADATA_PACKET_CONTROL;
+ uint32_t DP_SEC_FRAMING4;
+#endif
};
@@ -191,6 +205,10 @@ struct dcn10_stream_enc_registers {
SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP2_ENABLE, mask_sh),\
SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP3_ENABLE, mask_sh),\
SE_SF(DP0_DP_SEC_CNTL, DP_SEC_MPG_ENABLE, mask_sh),\
+ SE_SF(DP0_DP_SEC_CNTL2, DP_SEC_GSP4_SEND, mask_sh),\
+ SE_SF(DP0_DP_SEC_CNTL2, DP_SEC_GSP4_SEND_PENDING, mask_sh),\
+ SE_SF(DP0_DP_SEC_CNTL4, DP_SEC_GSP4_LINE_NUM, mask_sh),\
+ SE_SF(DP0_DP_SEC_CNTL2, DP_SEC_GSP4_SEND_ANY_LINE, mask_sh),\
SE_SF(DP0_DP_VID_STREAM_CNTL, DP_VID_STREAM_DIS_DEFER, mask_sh),\
SE_SF(DP0_DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, mask_sh),\
SE_SF(DP0_DP_VID_STREAM_CNTL, DP_VID_STREAM_STATUS, mask_sh),\
@@ -245,6 +263,7 @@ struct dcn10_stream_enc_registers {
SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC2_FRAME_UPDATE_PENDING, mask_sh),\
SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC3_FRAME_UPDATE_PENDING, mask_sh),\
SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC4_FRAME_UPDATE_PENDING, mask_sh),\
+ SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC4_IMMEDIATE_UPDATE_PENDING, mask_sh),\
SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC5_FRAME_UPDATE_PENDING, mask_sh),\
SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC6_FRAME_UPDATE_PENDING, mask_sh),\
SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC7_FRAME_UPDATE_PENDING, mask_sh),\
@@ -253,6 +272,7 @@ struct dcn10_stream_enc_registers {
SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC2_FRAME_UPDATE, mask_sh),\
SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC3_FRAME_UPDATE, mask_sh),\
SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC4_FRAME_UPDATE, mask_sh),\
+ SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC4_IMMEDIATE_UPDATE, mask_sh),\
SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC5_FRAME_UPDATE, mask_sh),\
SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC6_FRAME_UPDATE, mask_sh),\
SE_SF(DIG0_AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC7_FRAME_UPDATE, mask_sh),\
@@ -260,7 +280,9 @@ struct dcn10_stream_enc_registers {
SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP5_ENABLE, mask_sh),\
SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP6_ENABLE, mask_sh),\
SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP7_ENABLE, mask_sh),\
+ SE_SF(DP0_DP_SEC_CNTL2, DP_SEC_GSP7_PPS, mask_sh),\
SE_SF(DP0_DP_SEC_CNTL2, DP_SEC_GSP7_SEND, mask_sh),\
+ SE_SF(DP0_DP_SEC_CNTL6, DP_SEC_GSP7_LINE_NUM, mask_sh),\
SE_SF(DP0_DP_DB_CNTL, DP_DB_DISABLE, mask_sh),\
SE_SF(DP0_DP_MSA_COLORIMETRY, DP_MSA_MISC0, mask_sh),\
SE_SF(DP0_DP_MSA_TIMING_PARAM1, DP_MSA_HTOTAL, mask_sh),\
@@ -304,6 +326,7 @@ struct dcn10_stream_enc_registers {
type AFMT_GENERIC2_FRAME_UPDATE_PENDING;\
type AFMT_GENERIC3_FRAME_UPDATE_PENDING;\
type AFMT_GENERIC4_FRAME_UPDATE_PENDING;\
+ type AFMT_GENERIC4_IMMEDIATE_UPDATE_PENDING;\
type AFMT_GENERIC5_FRAME_UPDATE_PENDING;\
type AFMT_GENERIC6_FRAME_UPDATE_PENDING;\
type AFMT_GENERIC7_FRAME_UPDATE_PENDING;\
@@ -312,6 +335,7 @@ struct dcn10_stream_enc_registers {
type AFMT_GENERIC2_FRAME_UPDATE;\
type AFMT_GENERIC3_FRAME_UPDATE;\
type AFMT_GENERIC4_FRAME_UPDATE;\
+ type AFMT_GENERIC4_IMMEDIATE_UPDATE;\
type AFMT_GENERIC5_FRAME_UPDATE;\
type AFMT_GENERIC6_FRAME_UPDATE;\
type AFMT_GENERIC7_FRAME_UPDATE;\
@@ -366,7 +390,12 @@ struct dcn10_stream_enc_registers {
type DP_SEC_GSP5_ENABLE;\
type DP_SEC_GSP6_ENABLE;\
type DP_SEC_GSP7_ENABLE;\
+ type DP_SEC_GSP7_PPS;\
type DP_SEC_GSP7_SEND;\
+ type DP_SEC_GSP4_SEND;\
+ type DP_SEC_GSP4_SEND_PENDING;\
+ type DP_SEC_GSP4_LINE_NUM;\
+ type DP_SEC_GSP4_SEND_ANY_LINE;\
type DP_SEC_MPG_ENABLE;\
type DP_VID_STREAM_DIS_DEFER;\
type DP_VID_STREAM_ENABLE;\
@@ -407,6 +436,7 @@ struct dcn10_stream_enc_registers {
type DP_SEC_ATP_ENABLE;\
type DP_SEC_AIP_ENABLE;\
type DP_SEC_ACM_ENABLE;\
+ type DP_SEC_GSP7_LINE_NUM;\
type AFMT_AUDIO_SAMPLE_SEND;\
type AFMT_AUDIO_CLOCK_EN;\
type TMDS_PIXEL_ENCODING;\
@@ -430,12 +460,39 @@ struct dcn10_stream_enc_registers {
type DP_VID_M_DOUBLE_VALUE_EN;\
type DIG_SOURCE_SELECT
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#define SE_REG_FIELD_LIST_DCN2_0(type) \
+ type DP_DSC_MODE;\
+ type DP_DSC_SLICE_WIDTH;\
+ type DP_DSC_BYTES_PER_PIXEL;\
+ type DP_VBID6_LINE_REFERENCE;\
+ type DP_VBID6_LINE_NUM;\
+ type METADATA_ENGINE_EN;\
+ type METADATA_HUBP_REQUESTOR_ID;\
+ type METADATA_STREAM_TYPE;\
+ type DP_SEC_METADATA_PACKET_ENABLE;\
+ type DP_SEC_METADATA_PACKET_LINE_REFERENCE;\
+ type DP_SEC_METADATA_PACKET_LINE;\
+ type HDMI_METADATA_PACKET_ENABLE;\
+ type HDMI_METADATA_PACKET_LINE_REFERENCE;\
+ type HDMI_METADATA_PACKET_LINE;\
+ type DOLBY_VISION_EN;\
+ type DP_PIXEL_COMBINE;\
+ type DP_SST_SDP_SPLITTING
+#endif
+
struct dcn10_stream_encoder_shift {
SE_REG_FIELD_LIST_DCN1_0(uint8_t);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ SE_REG_FIELD_LIST_DCN2_0(uint8_t);
+#endif
};
struct dcn10_stream_encoder_mask {
SE_REG_FIELD_LIST_DCN1_0(uint32_t);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ SE_REG_FIELD_LIST_DCN2_0(uint32_t);
+#endif
};
struct dcn10_stream_encoder {
@@ -484,6 +541,11 @@ void enc1_stream_encoder_update_dp_info_packets(
struct stream_encoder *enc,
const struct encoder_info_frame *info_frame);
+void enc1_stream_encoder_send_immediate_sdp_message(
+ struct stream_encoder *enc,
+ const uint8_t *custom_sdp_message,
+ unsigned int sdp_message_size);
+
void enc1_stream_encoder_stop_dp_info_packets(
struct stream_encoder *enc);
@@ -530,4 +592,21 @@ void enc1_dig_connect_to_otg(
struct stream_encoder *enc,
int tg_inst);
+void enc1_stream_encoder_set_stream_attribute_helper(
+ struct dcn10_stream_encoder *enc1,
+ struct dc_crtc_timing *crtc_timing);
+
+void enc1_se_enable_audio_clock(
+ struct stream_encoder *enc,
+ bool enable);
+
+void enc1_se_enable_dp_audio(
+ struct stream_encoder *enc);
+
+void get_audio_clock_info(
+ enum dc_color_depth color_depth,
+ uint32_t crtc_pixel_clock_in_khz,
+ uint32_t actual_pixel_clock_in_khz,
+ struct audio_clock_info *audio_clock_info);
+
#endif /* __DC_STREAM_ENCODER_DCN10_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/Makefile b/drivers/gpu/drm/amd/display/dc/dcn20/Makefile
new file mode 100644
index 000000000000..e9721a906592
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/Makefile
@@ -0,0 +1,23 @@
+#
+# Makefile for DCN.
+
+DCN20 = dcn20_resource.o dcn20_hwseq.o dcn20_dpp.o dcn20_dpp_cm.o dcn20_hubp.o \
+ dcn20_mpc.o dcn20_opp.o dcn20_hubbub.o dcn20_optc.o dcn20_mmhubbub.o \
+ dcn20_stream_encoder.o dcn20_link_encoder.o dcn20_dccg.o \
+ dcn20_vmid.o dcn20_dwb.o dcn20_dwb_scl.o
+
+ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+DCN20 += dcn20_dsc.o
+endif
+
+ifneq ($(call cc-option, -mpreferred-stack-boundary=4),)
+ cc_stack_align := -mpreferred-stack-boundary=4
+else ifneq ($(call cc-option, -mstack-alignment=16),)
+ cc_stack_align := -mstack-alignment=16
+endif
+
+CFLAGS_dcn20_resource.o := -mhard-float -msse $(cc_stack_align)
+
+AMD_DAL_DCN20 = $(addprefix $(AMDDALPATH)/dc/dcn20/,$(DCN20))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_DCN20)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.c
new file mode 100644
index 000000000000..31aa6ee5cd5b
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.c
@@ -0,0 +1,164 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include <linux/slab.h>
+
+#include "reg_helper.h"
+#include "core_types.h"
+#include "dcn20_dccg.h"
+
+#define TO_DCN_DCCG(dccg)\
+ container_of(dccg, struct dcn_dccg, base)
+
+#define REG(reg) \
+ (dccg_dcn->regs->reg)
+
+#undef FN
+#define FN(reg_name, field_name) \
+ dccg_dcn->dccg_shift->field_name, dccg_dcn->dccg_mask->field_name
+
+#define CTX \
+ dccg_dcn->base.ctx
+#define DC_LOGGER \
+ dccg->ctx->logger
+
+void dccg2_update_dpp_dto(struct dccg *dccg, int dpp_inst, int req_dppclk)
+{
+ struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+
+ if (dccg->ref_dppclk && req_dppclk) {
+ int ref_dppclk = dccg->ref_dppclk;
+
+ ASSERT(req_dppclk <= ref_dppclk);
+ /* need to clamp to 8 bits */
+ if (ref_dppclk > 0xff) {
+ int divider = (ref_dppclk + 0xfe) / 0xff;
+
+ ref_dppclk /= divider;
+ req_dppclk = (req_dppclk + divider - 1) / divider;
+ if (req_dppclk > ref_dppclk)
+ req_dppclk = ref_dppclk;
+ }
+ REG_SET_2(DPPCLK_DTO_PARAM[dpp_inst], 0,
+ DPPCLK0_DTO_PHASE, req_dppclk,
+ DPPCLK0_DTO_MODULO, ref_dppclk);
+ REG_UPDATE(DPPCLK_DTO_CTRL,
+ DPPCLK_DTO_ENABLE[dpp_inst], 1);
+ } else {
+ REG_UPDATE(DPPCLK_DTO_CTRL,
+ DPPCLK_DTO_ENABLE[dpp_inst], 0);
+ }
+}
+
+void dccg2_get_dccg_ref_freq(struct dccg *dccg,
+ unsigned int xtalin_freq_inKhz,
+ unsigned int *dccg_ref_freq_inKhz)
+{
+ struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+ uint32_t clk_en = 0;
+ uint32_t clk_sel = 0;
+
+ REG_GET_2(REFCLK_CNTL, REFCLK_CLOCK_EN, &clk_en, REFCLK_SRC_SEL, &clk_sel);
+
+ if (clk_en != 0) {
+ // DCN20 has never been validated for non-xtalin as reference
+ // frequency. There's actually no way for DC to determine what
+ // frequency a non-xtalin source is.
+ ASSERT_CRITICAL(false);
+ }
+
+ *dccg_ref_freq_inKhz = xtalin_freq_inKhz;
+
+ return;
+}
+
+void dccg2_init(struct dccg *dccg)
+{
+ struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+
+ // Fallthrough intentional to program all available dpp_dto's
+ switch (dccg_dcn->base.ctx->dc->res_pool->pipe_count) {
+ case 6:
+ REG_UPDATE(DPPCLK_DTO_CTRL, DPPCLK_DTO_DB_EN[5], 1);
+ /* Fall through */
+ case 5:
+ REG_UPDATE(DPPCLK_DTO_CTRL, DPPCLK_DTO_DB_EN[4], 1);
+ /* Fall through */
+ case 4:
+ REG_UPDATE(DPPCLK_DTO_CTRL, DPPCLK_DTO_DB_EN[3], 1);
+ /* Fall through */
+ case 3:
+ REG_UPDATE(DPPCLK_DTO_CTRL, DPPCLK_DTO_DB_EN[2], 1);
+ /* Fall through */
+ case 2:
+ REG_UPDATE(DPPCLK_DTO_CTRL, DPPCLK_DTO_DB_EN[1], 1);
+ /* Fall through */
+ case 1:
+ REG_UPDATE(DPPCLK_DTO_CTRL, DPPCLK_DTO_DB_EN[0], 1);
+ break;
+ default:
+ ASSERT(false);
+ break;
+ }
+}
+
+static const struct dccg_funcs dccg2_funcs = {
+ .update_dpp_dto = dccg2_update_dpp_dto,
+ .get_dccg_ref_freq = dccg2_get_dccg_ref_freq,
+ .dccg_init = dccg2_init
+};
+
+struct dccg *dccg2_create(
+ struct dc_context *ctx,
+ const struct dccg_registers *regs,
+ const struct dccg_shift *dccg_shift,
+ const struct dccg_mask *dccg_mask)
+{
+ struct dcn_dccg *dccg_dcn = kzalloc(sizeof(*dccg_dcn), GFP_KERNEL);
+ struct dccg *base;
+
+ if (dccg_dcn == NULL) {
+ BREAK_TO_DEBUGGER();
+ return NULL;
+ }
+
+ base = &dccg_dcn->base;
+ base->ctx = ctx;
+ base->funcs = &dccg2_funcs;
+
+ dccg_dcn->regs = regs;
+ dccg_dcn->dccg_shift = dccg_shift;
+ dccg_dcn->dccg_mask = dccg_mask;
+
+ return &dccg_dcn->base;
+}
+
+void dcn_dccg_destroy(struct dccg **dccg)
+{
+ struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(*dccg);
+
+ kfree(dccg_dcn);
+ *dccg = NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h
new file mode 100644
index 000000000000..2205cb0204e7
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h
@@ -0,0 +1,116 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DCN20_DCCG_H__
+#define __DCN20_DCCG_H__
+
+#include "dccg.h"
+
+#define DCCG_COMMON_REG_LIST_DCN_BASE() \
+ SR(DPPCLK_DTO_CTRL),\
+ DCCG_SRII(DTO_PARAM, DPPCLK, 0),\
+ DCCG_SRII(DTO_PARAM, DPPCLK, 1),\
+ DCCG_SRII(DTO_PARAM, DPPCLK, 2),\
+ DCCG_SRII(DTO_PARAM, DPPCLK, 3),\
+ SR(REFCLK_CNTL)
+
+#define DCCG_REG_LIST_DCN2() \
+ DCCG_COMMON_REG_LIST_DCN_BASE(),\
+ DCCG_SRII(DTO_PARAM, DPPCLK, 4),\
+ DCCG_SRII(DTO_PARAM, DPPCLK, 5)
+
+#define DCCG_SF(reg_name, field_name, post_fix)\
+ .field_name = reg_name ## __ ## field_name ## post_fix
+
+#define DCCG_SFI(reg_name, field_name, field_prefix, inst, post_fix)\
+ .field_prefix ## _ ## field_name[inst] = reg_name ## __ ## field_prefix ## inst ## _ ## field_name ## post_fix
+
+#define DCCG_COMMON_MASK_SH_LIST_DCN_COMMON_BASE(mask_sh) \
+ DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 0, mask_sh),\
+ DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 0, mask_sh),\
+ DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 1, mask_sh),\
+ DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 1, mask_sh),\
+ DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 2, mask_sh),\
+ DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 2, mask_sh),\
+ DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 3, mask_sh),\
+ DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 3, mask_sh),\
+ DCCG_SF(DPPCLK0_DTO_PARAM, DPPCLK0_DTO_PHASE, mask_sh),\
+ DCCG_SF(DPPCLK0_DTO_PARAM, DPPCLK0_DTO_MODULO, mask_sh),\
+ DCCG_SF(REFCLK_CNTL, REFCLK_CLOCK_EN, mask_sh),\
+ DCCG_SF(REFCLK_CNTL, REFCLK_SRC_SEL, mask_sh)
+
+#define DCCG_MASK_SH_LIST_DCN2(mask_sh) \
+ DCCG_COMMON_MASK_SH_LIST_DCN_COMMON_BASE(mask_sh),\
+ DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 4, mask_sh),\
+ DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 4, mask_sh),\
+ DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 5, mask_sh),\
+ DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 5, mask_sh)
+
+#define DCCG_REG_FIELD_LIST(type) \
+ type DPPCLK0_DTO_PHASE;\
+ type DPPCLK0_DTO_MODULO;\
+ type DPPCLK_DTO_ENABLE[6];\
+ type DPPCLK_DTO_DB_EN[6];\
+ type REFCLK_CLOCK_EN;\
+ type REFCLK_SRC_SEL;
+
+struct dccg_shift {
+ DCCG_REG_FIELD_LIST(uint8_t)
+};
+
+struct dccg_mask {
+ DCCG_REG_FIELD_LIST(uint32_t)
+};
+
+struct dccg_registers {
+ uint32_t DPPCLK_DTO_CTRL;
+ uint32_t DPPCLK_DTO_PARAM[6];
+ uint32_t REFCLK_CNTL;
+};
+
+struct dcn_dccg {
+ struct dccg base;
+ const struct dccg_registers *regs;
+ const struct dccg_shift *dccg_shift;
+ const struct dccg_mask *dccg_mask;
+};
+
+void dccg2_update_dpp_dto(struct dccg *dccg, int dpp_inst, int req_dppclk);
+
+void dccg2_get_dccg_ref_freq(struct dccg *dccg,
+ unsigned int xtalin_freq_inKhz,
+ unsigned int *dccg_ref_freq_inKhz);
+
+void dccg2_init(struct dccg *dccg);
+
+struct dccg *dccg2_create(
+ struct dc_context *ctx,
+ const struct dccg_registers *regs,
+ const struct dccg_shift *dccg_shift,
+ const struct dccg_mask *dccg_mask);
+
+void dcn_dccg_destroy(struct dccg **dccg);
+
+#endif //__DCN20_DCCG_H__
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp.c
new file mode 100644
index 000000000000..9bc5dd23d297
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp.c
@@ -0,0 +1,502 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services.h"
+
+#include "core_types.h"
+
+#include "reg_helper.h"
+#include "dcn20_dpp.h"
+#include "basics/conversion.h"
+
+#define NUM_PHASES 64
+#define HORZ_MAX_TAPS 8
+#define VERT_MAX_TAPS 8
+
+#define BLACK_OFFSET_RGB_Y 0x0
+#define BLACK_OFFSET_CBCR 0x8000
+
+#define REG(reg)\
+ dpp->tf_regs->reg
+
+#define CTX \
+ dpp->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name) \
+ dpp->tf_shift->field_name, dpp->tf_mask->field_name
+
+void dpp20_read_state(struct dpp *dpp_base,
+ struct dcn_dpp_state *s)
+{
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ REG_GET(DPP_CONTROL,
+ DPP_CLOCK_ENABLE, &s->is_enabled);
+ REG_GET(CM_DGAM_CONTROL,
+ CM_DGAM_LUT_MODE, &s->dgam_lut_mode);
+ // BGAM has no ROM, and definition is different, can't reuse same dump
+ //REG_GET(CM_BLNDGAM_CONTROL,
+ // CM_BLNDGAM_LUT_MODE, &s->rgam_lut_mode);
+ REG_GET(CM_GAMUT_REMAP_CONTROL,
+ CM_GAMUT_REMAP_MODE, &s->gamut_remap_mode);
+ if (s->gamut_remap_mode) {
+ s->gamut_remap_c11_c12 = REG_READ(CM_GAMUT_REMAP_C11_C12);
+ s->gamut_remap_c13_c14 = REG_READ(CM_GAMUT_REMAP_C13_C14);
+ s->gamut_remap_c21_c22 = REG_READ(CM_GAMUT_REMAP_C21_C22);
+ s->gamut_remap_c23_c24 = REG_READ(CM_GAMUT_REMAP_C23_C24);
+ s->gamut_remap_c31_c32 = REG_READ(CM_GAMUT_REMAP_C31_C32);
+ s->gamut_remap_c33_c34 = REG_READ(CM_GAMUT_REMAP_C33_C34);
+ }
+}
+
+void dpp2_dummy_program_input_lut(
+ struct dpp *dpp_base,
+ const struct dc_gamma *gamma)
+{}
+
+static void dpp2_cnv_setup (
+ struct dpp *dpp_base,
+ enum surface_pixel_format format,
+ enum expansion_mode mode,
+ struct dc_csc_transform input_csc_color_matrix,
+ enum dc_color_space input_color_space,
+ struct cnv_alpha_2bit_lut *alpha_2bit_lut)
+{
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+ uint32_t pixel_format = 0;
+ uint32_t alpha_en = 1;
+ enum dc_color_space color_space = COLOR_SPACE_SRGB;
+ enum dcn10_input_csc_select select = INPUT_CSC_SELECT_BYPASS;
+ bool force_disable_cursor = false;
+ struct out_csc_color_matrix tbl_entry;
+ uint32_t is_2bit = 0;
+ int i = 0;
+
+ REG_SET_2(FORMAT_CONTROL, 0,
+ CNVC_BYPASS, 0,
+ FORMAT_EXPANSION_MODE, mode);
+
+ //hardcode default
+ //FORMAT_CONTROL. FORMAT_CNV16 default 0: U0.16/S.1.15; 1: U1.15/ S.1.14
+ //FORMAT_CONTROL. CNVC_BYPASS_MSB_ALIGN default 0: disabled 1: enabled
+ //FORMAT_CONTROL. CLAMP_POSITIVE default 0: disabled 1: enabled
+ //FORMAT_CONTROL. CLAMP_POSITIVE_C default 0: disabled 1: enabled
+ REG_UPDATE(FORMAT_CONTROL, FORMAT_CNV16, 0);
+ REG_UPDATE(FORMAT_CONTROL, CNVC_BYPASS_MSB_ALIGN, 0);
+ REG_UPDATE(FORMAT_CONTROL, CLAMP_POSITIVE, 0);
+ REG_UPDATE(FORMAT_CONTROL, CLAMP_POSITIVE_C, 0);
+
+ switch (format) {
+ case SURFACE_PIXEL_FORMAT_GRPH_ARGB1555:
+ pixel_format = 1;
+ break;
+ case SURFACE_PIXEL_FORMAT_GRPH_RGB565:
+ pixel_format = 3;
+ alpha_en = 0;
+ break;
+ case SURFACE_PIXEL_FORMAT_GRPH_ARGB8888:
+ case SURFACE_PIXEL_FORMAT_GRPH_ABGR8888:
+ pixel_format = 8;
+ break;
+ case SURFACE_PIXEL_FORMAT_GRPH_ARGB2101010:
+ case SURFACE_PIXEL_FORMAT_GRPH_ABGR2101010:
+ pixel_format = 10;
+ is_2bit = 1;
+ break;
+ case SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr:
+ force_disable_cursor = false;
+ pixel_format = 65;
+ color_space = COLOR_SPACE_YCBCR709;
+ select = INPUT_CSC_SELECT_ICSC;
+ break;
+ case SURFACE_PIXEL_FORMAT_VIDEO_420_YCrCb:
+ force_disable_cursor = true;
+ pixel_format = 64;
+ color_space = COLOR_SPACE_YCBCR709;
+ select = INPUT_CSC_SELECT_ICSC;
+ break;
+ case SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCbCr:
+ force_disable_cursor = true;
+ pixel_format = 67;
+ color_space = COLOR_SPACE_YCBCR709;
+ select = INPUT_CSC_SELECT_ICSC;
+ break;
+ case SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb:
+ force_disable_cursor = true;
+ pixel_format = 66;
+ color_space = COLOR_SPACE_YCBCR709;
+ select = INPUT_CSC_SELECT_ICSC;
+ break;
+ case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616:
+ pixel_format = 22;
+ break;
+ case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616F:
+ pixel_format = 24;
+ break;
+ case SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F:
+ pixel_format = 25;
+ break;
+ case SURFACE_PIXEL_FORMAT_VIDEO_AYCrCb8888:
+ pixel_format = 12;
+ color_space = COLOR_SPACE_YCBCR709;
+ select = INPUT_CSC_SELECT_ICSC;
+ break;
+ case SURFACE_PIXEL_FORMAT_GRPH_RGB111110_FIX:
+ pixel_format = 112;
+ break;
+ case SURFACE_PIXEL_FORMAT_GRPH_BGR101111_FIX:
+ pixel_format = 113;
+ break;
+ case SURFACE_PIXEL_FORMAT_VIDEO_ACrYCb2101010:
+ pixel_format = 114;
+ color_space = COLOR_SPACE_YCBCR709;
+ select = INPUT_CSC_SELECT_ICSC;
+ is_2bit = 1;
+ break;
+ case SURFACE_PIXEL_FORMAT_VIDEO_CrYCbA1010102:
+ pixel_format = 115;
+ color_space = COLOR_SPACE_YCBCR709;
+ select = INPUT_CSC_SELECT_ICSC;
+ is_2bit = 1;
+ break;
+ case SURFACE_PIXEL_FORMAT_GRPH_RGB111110_FLOAT:
+ pixel_format = 118;
+ break;
+ case SURFACE_PIXEL_FORMAT_GRPH_BGR101111_FLOAT:
+ pixel_format = 119;
+ break;
+ default:
+ break;
+ }
+
+ if (is_2bit == 1 && alpha_2bit_lut != NULL) {
+ REG_UPDATE(ALPHA_2BIT_LUT, ALPHA_2BIT_LUT0, alpha_2bit_lut->lut0);
+ REG_UPDATE(ALPHA_2BIT_LUT, ALPHA_2BIT_LUT1, alpha_2bit_lut->lut1);
+ REG_UPDATE(ALPHA_2BIT_LUT, ALPHA_2BIT_LUT2, alpha_2bit_lut->lut2);
+ REG_UPDATE(ALPHA_2BIT_LUT, ALPHA_2BIT_LUT3, alpha_2bit_lut->lut3);
+ }
+
+ REG_SET(CNVC_SURFACE_PIXEL_FORMAT, 0,
+ CNVC_SURFACE_PIXEL_FORMAT, pixel_format);
+ REG_UPDATE(FORMAT_CONTROL, FORMAT_CONTROL__ALPHA_EN, alpha_en);
+
+ // if input adjustments exist, program icsc with those values
+ if (input_csc_color_matrix.enable_adjustment
+ == true) {
+ for (i = 0; i < 12; i++)
+ tbl_entry.regval[i] = input_csc_color_matrix.matrix[i];
+
+ tbl_entry.color_space = input_color_space;
+
+ if (color_space >= COLOR_SPACE_YCBCR601)
+ select = INPUT_CSC_SELECT_ICSC;
+ else
+ select = INPUT_CSC_SELECT_BYPASS;
+
+ dpp1_program_input_csc(dpp_base, color_space, select, &tbl_entry);
+ } else
+ dpp1_program_input_csc(dpp_base, color_space, select, NULL);
+
+ if (force_disable_cursor) {
+ REG_UPDATE(CURSOR_CONTROL,
+ CURSOR_ENABLE, 0);
+ REG_UPDATE(CURSOR0_CONTROL,
+ CUR0_ENABLE, 0);
+
+ }
+
+}
+
+void dpp2_cnv_set_bias_scale(
+ struct dpp *dpp_base,
+ struct dc_bias_and_scale *bias_and_scale)
+{
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ REG_UPDATE(FCNV_FP_BIAS_R, FCNV_FP_BIAS_R, bias_and_scale->bias_red);
+ REG_UPDATE(FCNV_FP_BIAS_G, FCNV_FP_BIAS_G, bias_and_scale->bias_green);
+ REG_UPDATE(FCNV_FP_BIAS_B, FCNV_FP_BIAS_B, bias_and_scale->bias_blue);
+ REG_UPDATE(FCNV_FP_SCALE_R, FCNV_FP_SCALE_R, bias_and_scale->scale_red);
+ REG_UPDATE(FCNV_FP_SCALE_G, FCNV_FP_SCALE_G, bias_and_scale->scale_green);
+ REG_UPDATE(FCNV_FP_SCALE_B, FCNV_FP_SCALE_B, bias_and_scale->scale_blue);
+}
+
+/*compute the maximum number of lines that we can fit in the line buffer*/
+void dscl2_calc_lb_num_partitions(
+ const struct scaler_data *scl_data,
+ enum lb_memory_config lb_config,
+ int *num_part_y,
+ int *num_part_c)
+{
+ int memory_line_size_y, memory_line_size_c, memory_line_size_a,
+ lb_memory_size, lb_memory_size_c, lb_memory_size_a, num_partitions_a;
+
+ int line_size = scl_data->viewport.width < scl_data->recout.width ?
+ scl_data->viewport.width : scl_data->recout.width;
+ int line_size_c = scl_data->viewport_c.width < scl_data->recout.width ?
+ scl_data->viewport_c.width : scl_data->recout.width;
+
+ if (line_size == 0)
+ line_size = 1;
+
+ if (line_size_c == 0)
+ line_size_c = 1;
+
+ memory_line_size_y = (line_size + 5) / 6; /* +5 to ceil */
+ memory_line_size_c = (line_size_c + 5) / 6; /* +5 to ceil */
+ memory_line_size_a = (line_size + 5) / 6; /* +5 to ceil */
+
+ if (lb_config == LB_MEMORY_CONFIG_1) {
+ lb_memory_size = 970;
+ lb_memory_size_c = 970;
+ lb_memory_size_a = 970;
+ } else if (lb_config == LB_MEMORY_CONFIG_2) {
+ lb_memory_size = 1290;
+ lb_memory_size_c = 1290;
+ lb_memory_size_a = 1290;
+ } else if (lb_config == LB_MEMORY_CONFIG_3) {
+ /* 420 mode: using 3rd mem from Y, Cr and Cb */
+ lb_memory_size = 970 + 1290 + 484 + 484 + 484;
+ lb_memory_size_c = 970 + 1290;
+ lb_memory_size_a = 970 + 1290 + 484;
+ } else {
+ lb_memory_size = 970 + 1290 + 484;
+ lb_memory_size_c = 970 + 1290 + 484;
+ lb_memory_size_a = 970 + 1290 + 484;
+ }
+ *num_part_y = lb_memory_size / memory_line_size_y;
+ *num_part_c = lb_memory_size_c / memory_line_size_c;
+ num_partitions_a = lb_memory_size_a / memory_line_size_a;
+
+ if (scl_data->lb_params.alpha_en
+ && (num_partitions_a < *num_part_y))
+ *num_part_y = num_partitions_a;
+
+ if (*num_part_y > 64)
+ *num_part_y = 64;
+ if (*num_part_c > 64)
+ *num_part_c = 64;
+}
+
+void dpp2_cnv_set_alpha_keyer(
+ struct dpp *dpp_base,
+ struct cnv_color_keyer_params *color_keyer)
+{
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ REG_UPDATE(COLOR_KEYER_CONTROL, COLOR_KEYER_EN, color_keyer->color_keyer_en);
+
+ REG_UPDATE(COLOR_KEYER_CONTROL, COLOR_KEYER_MODE, color_keyer->color_keyer_mode);
+
+ REG_UPDATE(COLOR_KEYER_ALPHA, COLOR_KEYER_ALPHA_LOW, color_keyer->color_keyer_alpha_low);
+ REG_UPDATE(COLOR_KEYER_ALPHA, COLOR_KEYER_ALPHA_HIGH, color_keyer->color_keyer_alpha_high);
+
+ REG_UPDATE(COLOR_KEYER_RED, COLOR_KEYER_RED_LOW, color_keyer->color_keyer_red_low);
+ REG_UPDATE(COLOR_KEYER_RED, COLOR_KEYER_RED_HIGH, color_keyer->color_keyer_red_high);
+
+ REG_UPDATE(COLOR_KEYER_GREEN, COLOR_KEYER_GREEN_LOW, color_keyer->color_keyer_green_low);
+ REG_UPDATE(COLOR_KEYER_GREEN, COLOR_KEYER_GREEN_HIGH, color_keyer->color_keyer_green_high);
+
+ REG_UPDATE(COLOR_KEYER_BLUE, COLOR_KEYER_BLUE_LOW, color_keyer->color_keyer_blue_low);
+ REG_UPDATE(COLOR_KEYER_BLUE, COLOR_KEYER_BLUE_HIGH, color_keyer->color_keyer_blue_high);
+}
+
+void dpp2_set_cursor_attributes(
+ struct dpp *dpp_base,
+ enum dc_cursor_color_format color_format)
+{
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+ int cur_rom_en = 0;
+
+ if (color_format == CURSOR_MODE_COLOR_PRE_MULTIPLIED_ALPHA ||
+ color_format == CURSOR_MODE_COLOR_UN_PRE_MULTIPLIED_ALPHA)
+ cur_rom_en = 1;
+
+ REG_UPDATE_3(CURSOR0_CONTROL,
+ CUR0_MODE, color_format,
+ CUR0_EXPANSION_MODE, 0,
+ CUR0_ROM_EN, cur_rom_en);
+
+ if (color_format == CURSOR_MODE_MONO) {
+ /* todo: clarify what to program these to */
+ REG_UPDATE(CURSOR0_COLOR0,
+ CUR0_COLOR0, 0x00000000);
+ REG_UPDATE(CURSOR0_COLOR1,
+ CUR0_COLOR1, 0xFFFFFFFF);
+ }
+}
+
+#define IDENTITY_RATIO(ratio) (dc_fixpt_u3d19(ratio) == (1 << 19))
+
+bool dpp2_get_optimal_number_of_taps(
+ struct dpp *dpp,
+ struct scaler_data *scl_data,
+ const struct scaling_taps *in_taps)
+{
+ uint32_t pixel_width;
+
+ if (scl_data->viewport.width > scl_data->recout.width)
+ pixel_width = scl_data->recout.width;
+ else
+ pixel_width = scl_data->viewport.width;
+
+ /* Some ASICs does not support FP16 scaling, so we reject modes require this*/
+ if (scl_data->viewport.width != scl_data->h_active &&
+ scl_data->viewport.height != scl_data->v_active &&
+ dpp->caps->dscl_data_proc_format == DSCL_DATA_PRCESSING_FIXED_FORMAT &&
+ scl_data->format == PIXEL_FORMAT_FP16)
+ return false;
+
+ if (scl_data->viewport.width > scl_data->h_active &&
+ dpp->ctx->dc->debug.max_downscale_src_width != 0 &&
+ scl_data->viewport.width > dpp->ctx->dc->debug.max_downscale_src_width)
+ return false;
+
+ /* TODO: add lb check */
+
+ /* No support for programming ratio of 8, drop to 7.99999.. */
+ if (scl_data->ratios.horz.value == (8ll << 32))
+ scl_data->ratios.horz.value--;
+ if (scl_data->ratios.vert.value == (8ll << 32))
+ scl_data->ratios.vert.value--;
+ if (scl_data->ratios.horz_c.value == (8ll << 32))
+ scl_data->ratios.horz_c.value--;
+ if (scl_data->ratios.vert_c.value == (8ll << 32))
+ scl_data->ratios.vert_c.value--;
+
+ /* Set default taps if none are provided */
+ if (in_taps->h_taps == 0) {
+ if (dc_fixpt_ceil(scl_data->ratios.horz) > 4)
+ scl_data->taps.h_taps = 8;
+ else
+ scl_data->taps.h_taps = 4;
+ } else
+ scl_data->taps.h_taps = in_taps->h_taps;
+ if (in_taps->v_taps == 0) {
+ if (dc_fixpt_ceil(scl_data->ratios.vert) > 4)
+ scl_data->taps.v_taps = 8;
+ else
+ scl_data->taps.v_taps = 4;
+ } else
+ scl_data->taps.v_taps = in_taps->v_taps;
+ if (in_taps->v_taps_c == 0) {
+ if (dc_fixpt_ceil(scl_data->ratios.vert_c) > 4)
+ scl_data->taps.v_taps_c = 4;
+ else
+ scl_data->taps.v_taps_c = 2;
+ } else
+ scl_data->taps.v_taps_c = in_taps->v_taps_c;
+ if (in_taps->h_taps_c == 0) {
+ if (dc_fixpt_ceil(scl_data->ratios.horz_c) > 4)
+ scl_data->taps.h_taps_c = 4;
+ else
+ scl_data->taps.h_taps_c = 2;
+ } else if ((in_taps->h_taps_c % 2) != 0 && in_taps->h_taps_c != 1)
+ /* Only 1 and even h_taps_c are supported by hw */
+ scl_data->taps.h_taps_c = in_taps->h_taps_c - 1;
+ else
+ scl_data->taps.h_taps_c = in_taps->h_taps_c;
+
+ if (!dpp->ctx->dc->debug.always_scale) {
+ if (IDENTITY_RATIO(scl_data->ratios.horz))
+ scl_data->taps.h_taps = 1;
+ if (IDENTITY_RATIO(scl_data->ratios.vert))
+ scl_data->taps.v_taps = 1;
+ if (IDENTITY_RATIO(scl_data->ratios.horz_c))
+ scl_data->taps.h_taps_c = 1;
+ if (IDENTITY_RATIO(scl_data->ratios.vert_c))
+ scl_data->taps.v_taps_c = 1;
+ }
+
+ return true;
+}
+
+void oppn20_dummy_program_regamma_pwl(
+ struct dpp *dpp,
+ const struct pwl_params *params,
+ enum opp_regamma mode)
+{}
+
+static struct dpp_funcs dcn20_dpp_funcs = {
+ .dpp_read_state = dpp20_read_state,
+ .dpp_reset = dpp_reset,
+ .dpp_set_scaler = dpp1_dscl_set_scaler_manual_scale,
+ .dpp_get_optimal_number_of_taps = dpp2_get_optimal_number_of_taps,
+ .dpp_set_gamut_remap = dpp1_cm_set_gamut_remap,
+ .dpp_set_csc_adjustment = NULL,
+ .dpp_set_csc_default = NULL,
+ .dpp_program_regamma_pwl = oppn20_dummy_program_regamma_pwl,
+ .dpp_set_degamma = dpp2_set_degamma,
+ .dpp_program_input_lut = dpp2_dummy_program_input_lut,
+ .dpp_full_bypass = dpp1_full_bypass,
+ .dpp_setup = dpp2_cnv_setup,
+ .dpp_program_degamma_pwl = dpp2_set_degamma_pwl,
+ .dpp_program_blnd_lut = dpp20_program_blnd_lut,
+ .dpp_program_shaper_lut = dpp20_program_shaper,
+ .dpp_program_3dlut = dpp20_program_3dlut,
+ .dpp_program_bias_and_scale = NULL,
+ .dpp_cnv_set_alpha_keyer = dpp2_cnv_set_alpha_keyer,
+ .set_cursor_attributes = dpp2_set_cursor_attributes,
+ .set_cursor_position = dpp1_set_cursor_position,
+ .set_optional_cursor_attributes = dpp1_cnv_set_optional_cursor_attributes,
+ .dpp_dppclk_control = dpp1_dppclk_control,
+ .dpp_set_hdr_multiplier = dpp2_set_hdr_multiplier,
+};
+
+static struct dpp_caps dcn20_dpp_cap = {
+ .dscl_data_proc_format = DSCL_DATA_PRCESSING_FLOAT_FORMAT,
+ .dscl_calc_lb_num_partitions = dscl2_calc_lb_num_partitions,
+};
+
+bool dpp2_construct(
+ struct dcn20_dpp *dpp,
+ struct dc_context *ctx,
+ uint32_t inst,
+ const struct dcn2_dpp_registers *tf_regs,
+ const struct dcn2_dpp_shift *tf_shift,
+ const struct dcn2_dpp_mask *tf_mask)
+{
+ dpp->base.ctx = ctx;
+
+ dpp->base.inst = inst;
+ dpp->base.funcs = &dcn20_dpp_funcs;
+ dpp->base.caps = &dcn20_dpp_cap;
+
+ dpp->tf_regs = tf_regs;
+ dpp->tf_shift = tf_shift;
+ dpp->tf_mask = tf_mask;
+
+ dpp->lb_pixel_depth_supported =
+ LB_PIXEL_DEPTH_18BPP |
+ LB_PIXEL_DEPTH_24BPP |
+ LB_PIXEL_DEPTH_30BPP;
+
+ dpp->lb_bits_per_entry = LB_BITS_PER_ENTRY;
+ dpp->lb_memory_size = LB_TOTAL_NUMBER_OF_ENTRIES; /*0x1404*/
+
+ return true;
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp.h
new file mode 100644
index 000000000000..59b67ed57c19
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp.h
@@ -0,0 +1,698 @@
+/* Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DCN20_DPP_H__
+#define __DCN20_DPP_H__
+
+#include "dcn10/dcn10_dpp.h"
+
+#define TO_DCN20_DPP(dpp)\
+ container_of(dpp, struct dcn20_dpp, base)
+
+#define TF_REG_LIST_DCN20(id) \
+ TF_REG_LIST_DCN(id), \
+ SRI(CM_BLNDGAM_LUT_WRITE_EN_MASK, CM, id), \
+ SRI(CM_BLNDGAM_CONTROL, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_START_CNTL_B, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_START_CNTL_G, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_START_CNTL_R, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_SLOPE_CNTL_B, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_SLOPE_CNTL_G, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_SLOPE_CNTL_R, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_END_CNTL1_B, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_END_CNTL2_B, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_END_CNTL1_G, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_END_CNTL2_G, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_END_CNTL1_R, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_END_CNTL2_R, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_REGION_0_1, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_REGION_2_3, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_REGION_4_5, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_REGION_6_7, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_REGION_8_9, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_REGION_10_11, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_REGION_12_13, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_REGION_14_15, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_REGION_16_17, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_REGION_18_19, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_REGION_20_21, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_REGION_22_23, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_REGION_24_25, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_REGION_26_27, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_REGION_28_29, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_REGION_30_31, CM, id), \
+ SRI(CM_BLNDGAM_RAMB_REGION_32_33, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_START_CNTL_B, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_START_CNTL_G, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_START_CNTL_R, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_SLOPE_CNTL_B, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_SLOPE_CNTL_G, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_SLOPE_CNTL_R, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_END_CNTL1_B, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_END_CNTL2_B, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_END_CNTL1_G, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_END_CNTL2_G, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_END_CNTL1_R, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_END_CNTL2_R, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_REGION_0_1, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_REGION_2_3, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_REGION_4_5, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_REGION_6_7, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_REGION_8_9, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_REGION_10_11, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_REGION_12_13, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_REGION_14_15, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_REGION_16_17, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_REGION_18_19, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_REGION_20_21, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_REGION_22_23, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_REGION_24_25, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_REGION_26_27, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_REGION_28_29, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_REGION_30_31, CM, id), \
+ SRI(CM_BLNDGAM_RAMA_REGION_32_33, CM, id), \
+ SRI(CM_BLNDGAM_LUT_INDEX, CM, id), \
+ SRI(CM_BLNDGAM_LUT_DATA, CM, id), \
+ SRI(CM_3DLUT_MODE, CM, id), \
+ SRI(CM_3DLUT_INDEX, CM, id), \
+ SRI(CM_3DLUT_DATA, CM, id), \
+ SRI(CM_3DLUT_DATA_30BIT, CM, id), \
+ SRI(CM_3DLUT_READ_WRITE_CONTROL, CM, id), \
+ SRI(CM_SHAPER_LUT_WRITE_EN_MASK, CM, id), \
+ SRI(CM_SHAPER_CONTROL, CM, id), \
+ SRI(CM_SHAPER_RAMB_START_CNTL_B, CM, id), \
+ SRI(CM_SHAPER_RAMB_START_CNTL_G, CM, id), \
+ SRI(CM_SHAPER_RAMB_START_CNTL_R, CM, id), \
+ SRI(CM_SHAPER_RAMB_END_CNTL_B, CM, id), \
+ SRI(CM_SHAPER_RAMB_END_CNTL_G, CM, id), \
+ SRI(CM_SHAPER_RAMB_END_CNTL_R, CM, id), \
+ SRI(CM_SHAPER_RAMB_REGION_0_1, CM, id), \
+ SRI(CM_SHAPER_RAMB_REGION_2_3, CM, id), \
+ SRI(CM_SHAPER_RAMB_REGION_4_5, CM, id), \
+ SRI(CM_SHAPER_RAMB_REGION_6_7, CM, id), \
+ SRI(CM_SHAPER_RAMB_REGION_8_9, CM, id), \
+ SRI(CM_SHAPER_RAMB_REGION_10_11, CM, id), \
+ SRI(CM_SHAPER_RAMB_REGION_12_13, CM, id), \
+ SRI(CM_SHAPER_RAMB_REGION_14_15, CM, id), \
+ SRI(CM_SHAPER_RAMB_REGION_16_17, CM, id), \
+ SRI(CM_SHAPER_RAMB_REGION_18_19, CM, id), \
+ SRI(CM_SHAPER_RAMB_REGION_20_21, CM, id), \
+ SRI(CM_SHAPER_RAMB_REGION_22_23, CM, id), \
+ SRI(CM_SHAPER_RAMB_REGION_24_25, CM, id), \
+ SRI(CM_SHAPER_RAMB_REGION_26_27, CM, id), \
+ SRI(CM_SHAPER_RAMB_REGION_28_29, CM, id), \
+ SRI(CM_SHAPER_RAMB_REGION_30_31, CM, id), \
+ SRI(CM_SHAPER_RAMB_REGION_32_33, CM, id), \
+ SRI(CM_SHAPER_RAMA_START_CNTL_B, CM, id), \
+ SRI(CM_SHAPER_RAMA_START_CNTL_G, CM, id), \
+ SRI(CM_SHAPER_RAMA_START_CNTL_R, CM, id), \
+ SRI(CM_SHAPER_RAMA_END_CNTL_B, CM, id), \
+ SRI(CM_SHAPER_RAMA_END_CNTL_G, CM, id), \
+ SRI(CM_SHAPER_RAMA_END_CNTL_R, CM, id), \
+ SRI(CM_SHAPER_RAMA_REGION_0_1, CM, id), \
+ SRI(CM_SHAPER_RAMA_REGION_2_3, CM, id), \
+ SRI(CM_SHAPER_RAMA_REGION_4_5, CM, id), \
+ SRI(CM_SHAPER_RAMA_REGION_6_7, CM, id), \
+ SRI(CM_SHAPER_RAMA_REGION_8_9, CM, id), \
+ SRI(CM_SHAPER_RAMA_REGION_10_11, CM, id), \
+ SRI(CM_SHAPER_RAMA_REGION_12_13, CM, id), \
+ SRI(CM_SHAPER_RAMA_REGION_14_15, CM, id), \
+ SRI(CM_SHAPER_RAMA_REGION_16_17, CM, id), \
+ SRI(CM_SHAPER_RAMA_REGION_18_19, CM, id), \
+ SRI(CM_SHAPER_RAMA_REGION_20_21, CM, id), \
+ SRI(CM_SHAPER_RAMA_REGION_22_23, CM, id), \
+ SRI(CM_SHAPER_RAMA_REGION_24_25, CM, id), \
+ SRI(CM_SHAPER_RAMA_REGION_26_27, CM, id), \
+ SRI(CM_SHAPER_RAMA_REGION_28_29, CM, id), \
+ SRI(CM_SHAPER_RAMA_REGION_30_31, CM, id), \
+ SRI(CM_SHAPER_RAMA_REGION_32_33, CM, id), \
+ SRI(CM_SHAPER_LUT_INDEX, CM, id), \
+ SRI(CURSOR_CONTROL, CURSOR0_, id), \
+ SRI(ALPHA_2BIT_LUT, CNVC_CFG, id), \
+ SRI(FCNV_FP_BIAS_R, CNVC_CFG, id), \
+ SRI(FCNV_FP_BIAS_G, CNVC_CFG, id), \
+ SRI(FCNV_FP_BIAS_B, CNVC_CFG, id), \
+ SRI(FCNV_FP_SCALE_R, CNVC_CFG, id), \
+ SRI(FCNV_FP_SCALE_G, CNVC_CFG, id), \
+ SRI(FCNV_FP_SCALE_B, CNVC_CFG, id), \
+ SRI(COLOR_KEYER_CONTROL, CNVC_CFG, id), \
+ SRI(COLOR_KEYER_ALPHA, CNVC_CFG, id), \
+ SRI(COLOR_KEYER_RED, CNVC_CFG, id), \
+ SRI(COLOR_KEYER_GREEN, CNVC_CFG, id), \
+ SRI(COLOR_KEYER_BLUE, CNVC_CFG, id), \
+ SRI(CM_SHAPER_LUT_DATA, CM, id), \
+ SRI(CURSOR_CONTROL, CURSOR0_, id)
+
+#define TF_REG_LIST_SH_MASK_DCN20(mask_sh)\
+ TF_REG_LIST_SH_MASK_DCN(mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_CONTROL, CM_BLNDGAM_LUT_MODE, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_START_CNTL_B, CM_BLNDGAM_RAMB_EXP_REGION_START_B, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_START_CNTL_B, CM_BLNDGAM_RAMB_EXP_REGION_START_SEGMENT_B, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_START_CNTL_G, CM_BLNDGAM_RAMB_EXP_REGION_START_G, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_START_CNTL_G, CM_BLNDGAM_RAMB_EXP_REGION_START_SEGMENT_G, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_START_CNTL_R, CM_BLNDGAM_RAMB_EXP_REGION_START_R, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_START_CNTL_R, CM_BLNDGAM_RAMB_EXP_REGION_START_SEGMENT_R, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_SLOPE_CNTL_B, CM_BLNDGAM_RAMB_EXP_REGION_LINEAR_SLOPE_B, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_SLOPE_CNTL_G, CM_BLNDGAM_RAMB_EXP_REGION_LINEAR_SLOPE_G, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_SLOPE_CNTL_R, CM_BLNDGAM_RAMB_EXP_REGION_LINEAR_SLOPE_R, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_END_CNTL1_B, CM_BLNDGAM_RAMB_EXP_REGION_END_B, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_END_CNTL2_B, CM_BLNDGAM_RAMB_EXP_REGION_END_SLOPE_B, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_END_CNTL2_B, CM_BLNDGAM_RAMB_EXP_REGION_END_BASE_B, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_END_CNTL1_G, CM_BLNDGAM_RAMB_EXP_REGION_END_G, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_END_CNTL2_G, CM_BLNDGAM_RAMB_EXP_REGION_END_SLOPE_G, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_END_CNTL2_G, CM_BLNDGAM_RAMB_EXP_REGION_END_BASE_G, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_END_CNTL1_R, CM_BLNDGAM_RAMB_EXP_REGION_END_R, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_END_CNTL2_R, CM_BLNDGAM_RAMB_EXP_REGION_END_SLOPE_R, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_END_CNTL2_R, CM_BLNDGAM_RAMB_EXP_REGION_END_BASE_R, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_0_1, CM_BLNDGAM_RAMB_EXP_REGION0_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_0_1, CM_BLNDGAM_RAMB_EXP_REGION0_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_0_1, CM_BLNDGAM_RAMB_EXP_REGION1_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_0_1, CM_BLNDGAM_RAMB_EXP_REGION1_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_2_3, CM_BLNDGAM_RAMB_EXP_REGION2_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_2_3, CM_BLNDGAM_RAMB_EXP_REGION2_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_2_3, CM_BLNDGAM_RAMB_EXP_REGION3_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_2_3, CM_BLNDGAM_RAMB_EXP_REGION3_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_4_5, CM_BLNDGAM_RAMB_EXP_REGION4_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_4_5, CM_BLNDGAM_RAMB_EXP_REGION4_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_4_5, CM_BLNDGAM_RAMB_EXP_REGION5_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_4_5, CM_BLNDGAM_RAMB_EXP_REGION5_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_6_7, CM_BLNDGAM_RAMB_EXP_REGION6_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_6_7, CM_BLNDGAM_RAMB_EXP_REGION6_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_6_7, CM_BLNDGAM_RAMB_EXP_REGION7_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_6_7, CM_BLNDGAM_RAMB_EXP_REGION7_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_8_9, CM_BLNDGAM_RAMB_EXP_REGION8_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_8_9, CM_BLNDGAM_RAMB_EXP_REGION8_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_8_9, CM_BLNDGAM_RAMB_EXP_REGION9_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_8_9, CM_BLNDGAM_RAMB_EXP_REGION9_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_10_11, CM_BLNDGAM_RAMB_EXP_REGION10_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_10_11, CM_BLNDGAM_RAMB_EXP_REGION10_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_10_11, CM_BLNDGAM_RAMB_EXP_REGION11_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_10_11, CM_BLNDGAM_RAMB_EXP_REGION11_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_12_13, CM_BLNDGAM_RAMB_EXP_REGION12_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_12_13, CM_BLNDGAM_RAMB_EXP_REGION12_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_12_13, CM_BLNDGAM_RAMB_EXP_REGION13_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_12_13, CM_BLNDGAM_RAMB_EXP_REGION13_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_14_15, CM_BLNDGAM_RAMB_EXP_REGION14_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_14_15, CM_BLNDGAM_RAMB_EXP_REGION14_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_14_15, CM_BLNDGAM_RAMB_EXP_REGION15_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_14_15, CM_BLNDGAM_RAMB_EXP_REGION15_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_16_17, CM_BLNDGAM_RAMB_EXP_REGION16_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_16_17, CM_BLNDGAM_RAMB_EXP_REGION16_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_16_17, CM_BLNDGAM_RAMB_EXP_REGION17_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_16_17, CM_BLNDGAM_RAMB_EXP_REGION17_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_18_19, CM_BLNDGAM_RAMB_EXP_REGION18_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_18_19, CM_BLNDGAM_RAMB_EXP_REGION18_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_18_19, CM_BLNDGAM_RAMB_EXP_REGION19_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_18_19, CM_BLNDGAM_RAMB_EXP_REGION19_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_20_21, CM_BLNDGAM_RAMB_EXP_REGION20_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_20_21, CM_BLNDGAM_RAMB_EXP_REGION20_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_20_21, CM_BLNDGAM_RAMB_EXP_REGION21_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_20_21, CM_BLNDGAM_RAMB_EXP_REGION21_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_22_23, CM_BLNDGAM_RAMB_EXP_REGION22_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_22_23, CM_BLNDGAM_RAMB_EXP_REGION22_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_22_23, CM_BLNDGAM_RAMB_EXP_REGION23_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_22_23, CM_BLNDGAM_RAMB_EXP_REGION23_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_24_25, CM_BLNDGAM_RAMB_EXP_REGION24_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_24_25, CM_BLNDGAM_RAMB_EXP_REGION24_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_24_25, CM_BLNDGAM_RAMB_EXP_REGION25_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_24_25, CM_BLNDGAM_RAMB_EXP_REGION25_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_26_27, CM_BLNDGAM_RAMB_EXP_REGION26_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_26_27, CM_BLNDGAM_RAMB_EXP_REGION26_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_26_27, CM_BLNDGAM_RAMB_EXP_REGION27_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_26_27, CM_BLNDGAM_RAMB_EXP_REGION27_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_28_29, CM_BLNDGAM_RAMB_EXP_REGION28_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_28_29, CM_BLNDGAM_RAMB_EXP_REGION28_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_28_29, CM_BLNDGAM_RAMB_EXP_REGION29_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_28_29, CM_BLNDGAM_RAMB_EXP_REGION29_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_30_31, CM_BLNDGAM_RAMB_EXP_REGION30_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_30_31, CM_BLNDGAM_RAMB_EXP_REGION30_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_30_31, CM_BLNDGAM_RAMB_EXP_REGION31_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_30_31, CM_BLNDGAM_RAMB_EXP_REGION31_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_32_33, CM_BLNDGAM_RAMB_EXP_REGION32_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_32_33, CM_BLNDGAM_RAMB_EXP_REGION32_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_32_33, CM_BLNDGAM_RAMB_EXP_REGION33_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMB_REGION_32_33, CM_BLNDGAM_RAMB_EXP_REGION33_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_START_CNTL_B, CM_BLNDGAM_RAMA_EXP_REGION_START_B, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_START_CNTL_B, CM_BLNDGAM_RAMA_EXP_REGION_START_SEGMENT_B, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_START_CNTL_G, CM_BLNDGAM_RAMA_EXP_REGION_START_G, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_START_CNTL_G, CM_BLNDGAM_RAMA_EXP_REGION_START_SEGMENT_G, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_START_CNTL_R, CM_BLNDGAM_RAMA_EXP_REGION_START_R, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_START_CNTL_R, CM_BLNDGAM_RAMA_EXP_REGION_START_SEGMENT_R, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_SLOPE_CNTL_B, CM_BLNDGAM_RAMA_EXP_REGION_LINEAR_SLOPE_B, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_SLOPE_CNTL_G, CM_BLNDGAM_RAMA_EXP_REGION_LINEAR_SLOPE_G, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_SLOPE_CNTL_R, CM_BLNDGAM_RAMA_EXP_REGION_LINEAR_SLOPE_R, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_END_CNTL1_B, CM_BLNDGAM_RAMA_EXP_REGION_END_B, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_END_CNTL2_B, CM_BLNDGAM_RAMA_EXP_REGION_END_SLOPE_B, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_END_CNTL2_B, CM_BLNDGAM_RAMA_EXP_REGION_END_BASE_B, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_END_CNTL1_G, CM_BLNDGAM_RAMA_EXP_REGION_END_G, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_END_CNTL2_G, CM_BLNDGAM_RAMA_EXP_REGION_END_SLOPE_G, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_END_CNTL2_G, CM_BLNDGAM_RAMA_EXP_REGION_END_BASE_G, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_END_CNTL1_R, CM_BLNDGAM_RAMA_EXP_REGION_END_R, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_END_CNTL2_R, CM_BLNDGAM_RAMA_EXP_REGION_END_SLOPE_R, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_END_CNTL2_R, CM_BLNDGAM_RAMA_EXP_REGION_END_BASE_R, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_0_1, CM_BLNDGAM_RAMA_EXP_REGION0_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_0_1, CM_BLNDGAM_RAMA_EXP_REGION0_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_0_1, CM_BLNDGAM_RAMA_EXP_REGION1_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_0_1, CM_BLNDGAM_RAMA_EXP_REGION1_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_2_3, CM_BLNDGAM_RAMA_EXP_REGION2_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_2_3, CM_BLNDGAM_RAMA_EXP_REGION2_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_2_3, CM_BLNDGAM_RAMA_EXP_REGION3_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_2_3, CM_BLNDGAM_RAMA_EXP_REGION3_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_4_5, CM_BLNDGAM_RAMA_EXP_REGION4_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_4_5, CM_BLNDGAM_RAMA_EXP_REGION4_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_4_5, CM_BLNDGAM_RAMA_EXP_REGION5_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_4_5, CM_BLNDGAM_RAMA_EXP_REGION5_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_6_7, CM_BLNDGAM_RAMA_EXP_REGION6_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_6_7, CM_BLNDGAM_RAMA_EXP_REGION6_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_6_7, CM_BLNDGAM_RAMA_EXP_REGION7_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_6_7, CM_BLNDGAM_RAMA_EXP_REGION7_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_8_9, CM_BLNDGAM_RAMA_EXP_REGION8_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_8_9, CM_BLNDGAM_RAMA_EXP_REGION8_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_8_9, CM_BLNDGAM_RAMA_EXP_REGION9_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_8_9, CM_BLNDGAM_RAMA_EXP_REGION9_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_10_11, CM_BLNDGAM_RAMA_EXP_REGION10_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_10_11, CM_BLNDGAM_RAMA_EXP_REGION10_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_10_11, CM_BLNDGAM_RAMA_EXP_REGION11_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_10_11, CM_BLNDGAM_RAMA_EXP_REGION11_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_12_13, CM_BLNDGAM_RAMA_EXP_REGION12_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_12_13, CM_BLNDGAM_RAMA_EXP_REGION12_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_12_13, CM_BLNDGAM_RAMA_EXP_REGION13_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_12_13, CM_BLNDGAM_RAMA_EXP_REGION13_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_14_15, CM_BLNDGAM_RAMA_EXP_REGION14_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_14_15, CM_BLNDGAM_RAMA_EXP_REGION14_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_14_15, CM_BLNDGAM_RAMA_EXP_REGION15_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_14_15, CM_BLNDGAM_RAMA_EXP_REGION15_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_16_17, CM_BLNDGAM_RAMA_EXP_REGION16_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_16_17, CM_BLNDGAM_RAMA_EXP_REGION16_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_16_17, CM_BLNDGAM_RAMA_EXP_REGION17_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_16_17, CM_BLNDGAM_RAMA_EXP_REGION17_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_18_19, CM_BLNDGAM_RAMA_EXP_REGION18_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_18_19, CM_BLNDGAM_RAMA_EXP_REGION18_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_18_19, CM_BLNDGAM_RAMA_EXP_REGION19_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_18_19, CM_BLNDGAM_RAMA_EXP_REGION19_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_20_21, CM_BLNDGAM_RAMA_EXP_REGION20_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_20_21, CM_BLNDGAM_RAMA_EXP_REGION20_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_20_21, CM_BLNDGAM_RAMA_EXP_REGION21_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_20_21, CM_BLNDGAM_RAMA_EXP_REGION21_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_22_23, CM_BLNDGAM_RAMA_EXP_REGION22_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_22_23, CM_BLNDGAM_RAMA_EXP_REGION22_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_22_23, CM_BLNDGAM_RAMA_EXP_REGION23_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_22_23, CM_BLNDGAM_RAMA_EXP_REGION23_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_24_25, CM_BLNDGAM_RAMA_EXP_REGION24_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_24_25, CM_BLNDGAM_RAMA_EXP_REGION24_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_24_25, CM_BLNDGAM_RAMA_EXP_REGION25_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_24_25, CM_BLNDGAM_RAMA_EXP_REGION25_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_26_27, CM_BLNDGAM_RAMA_EXP_REGION26_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_26_27, CM_BLNDGAM_RAMA_EXP_REGION26_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_26_27, CM_BLNDGAM_RAMA_EXP_REGION27_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_26_27, CM_BLNDGAM_RAMA_EXP_REGION27_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_28_29, CM_BLNDGAM_RAMA_EXP_REGION28_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_28_29, CM_BLNDGAM_RAMA_EXP_REGION28_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_28_29, CM_BLNDGAM_RAMA_EXP_REGION29_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_28_29, CM_BLNDGAM_RAMA_EXP_REGION29_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_30_31, CM_BLNDGAM_RAMA_EXP_REGION30_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_30_31, CM_BLNDGAM_RAMA_EXP_REGION30_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_30_31, CM_BLNDGAM_RAMA_EXP_REGION31_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_30_31, CM_BLNDGAM_RAMA_EXP_REGION31_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_32_33, CM_BLNDGAM_RAMA_EXP_REGION32_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_32_33, CM_BLNDGAM_RAMA_EXP_REGION32_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_32_33, CM_BLNDGAM_RAMA_EXP_REGION33_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_RAMA_REGION_32_33, CM_BLNDGAM_RAMA_EXP_REGION33_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_LUT_WRITE_EN_MASK, CM_BLNDGAM_LUT_WRITE_EN_MASK, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_LUT_WRITE_EN_MASK, CM_BLNDGAM_LUT_WRITE_SEL, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_LUT_WRITE_EN_MASK, CM_BLNDGAM_CONFIG_STATUS, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_LUT_INDEX, CM_BLNDGAM_LUT_INDEX, mask_sh), \
+ TF_SF(CM0_CM_BLNDGAM_LUT_DATA, CM_BLNDGAM_LUT_DATA, mask_sh), \
+ TF_SF(CM0_CM_MEM_PWR_CTRL, BLNDGAM_MEM_PWR_FORCE, mask_sh), \
+ TF_SF(CM0_CM_3DLUT_MODE, CM_3DLUT_MODE, mask_sh), \
+ TF_SF(CM0_CM_3DLUT_MODE, CM_3DLUT_SIZE, mask_sh), \
+ TF_SF(CM0_CM_3DLUT_INDEX, CM_3DLUT_INDEX, mask_sh), \
+ TF_SF(CM0_CM_3DLUT_DATA, CM_3DLUT_DATA0, mask_sh), \
+ TF_SF(CM0_CM_3DLUT_DATA, CM_3DLUT_DATA1, mask_sh), \
+ TF_SF(CM0_CM_3DLUT_DATA_30BIT, CM_3DLUT_DATA_30BIT, mask_sh), \
+ TF_SF(CM0_CM_3DLUT_READ_WRITE_CONTROL, CM_3DLUT_WRITE_EN_MASK, mask_sh), \
+ TF_SF(CM0_CM_3DLUT_READ_WRITE_CONTROL, CM_3DLUT_RAM_SEL, mask_sh), \
+ TF_SF(CM0_CM_3DLUT_READ_WRITE_CONTROL, CM_3DLUT_30BIT_EN, mask_sh), \
+ TF_SF(CM0_CM_3DLUT_READ_WRITE_CONTROL, CM_3DLUT_CONFIG_STATUS, mask_sh), \
+ TF_SF(CM0_CM_3DLUT_READ_WRITE_CONTROL, CM_3DLUT_READ_SEL, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_CONTROL, CM_SHAPER_LUT_MODE, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_START_CNTL_B, CM_SHAPER_RAMB_EXP_REGION_START_B, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_START_CNTL_B, CM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_START_CNTL_G, CM_SHAPER_RAMB_EXP_REGION_START_G, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_START_CNTL_G, CM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_START_CNTL_R, CM_SHAPER_RAMB_EXP_REGION_START_R, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_START_CNTL_R, CM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_END_CNTL_B, CM_SHAPER_RAMB_EXP_REGION_END_B, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_END_CNTL_B, CM_SHAPER_RAMB_EXP_REGION_END_BASE_B, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_END_CNTL_G, CM_SHAPER_RAMB_EXP_REGION_END_G, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_END_CNTL_G, CM_SHAPER_RAMB_EXP_REGION_END_BASE_G, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_END_CNTL_R, CM_SHAPER_RAMB_EXP_REGION_END_R, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_END_CNTL_R, CM_SHAPER_RAMB_EXP_REGION_END_BASE_R, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_0_1, CM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_0_1, CM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_0_1, CM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_0_1, CM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_2_3, CM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_2_3, CM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_2_3, CM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_2_3, CM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_4_5, CM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_4_5, CM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_4_5, CM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_4_5, CM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_6_7, CM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_6_7, CM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_6_7, CM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_6_7, CM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_8_9, CM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_8_9, CM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_8_9, CM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_8_9, CM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_10_11, CM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_10_11, CM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_10_11, CM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_10_11, CM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_12_13, CM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_12_13, CM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_12_13, CM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_12_13, CM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_14_15, CM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_14_15, CM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_14_15, CM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_14_15, CM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_16_17, CM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_16_17, CM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_16_17, CM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_16_17, CM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_18_19, CM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_18_19, CM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_18_19, CM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_18_19, CM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_20_21, CM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_20_21, CM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_20_21, CM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_20_21, CM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_22_23, CM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_22_23, CM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_22_23, CM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_22_23, CM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_24_25, CM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_24_25, CM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_24_25, CM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_24_25, CM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_26_27, CM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_26_27, CM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_26_27, CM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_26_27, CM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_28_29, CM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_28_29, CM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_28_29, CM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_28_29, CM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_30_31, CM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_30_31, CM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_30_31, CM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_30_31, CM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_32_33, CM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_32_33, CM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_32_33, CM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMB_REGION_32_33, CM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_START_CNTL_B, CM_SHAPER_RAMA_EXP_REGION_START_B, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_START_CNTL_B, CM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_START_CNTL_G, CM_SHAPER_RAMA_EXP_REGION_START_G, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_START_CNTL_G, CM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_START_CNTL_R, CM_SHAPER_RAMA_EXP_REGION_START_R, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_START_CNTL_R, CM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_END_CNTL_B, CM_SHAPER_RAMA_EXP_REGION_END_B, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_END_CNTL_B, CM_SHAPER_RAMA_EXP_REGION_END_BASE_B, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_END_CNTL_G, CM_SHAPER_RAMA_EXP_REGION_END_G, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_END_CNTL_G, CM_SHAPER_RAMA_EXP_REGION_END_BASE_G, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_END_CNTL_R, CM_SHAPER_RAMA_EXP_REGION_END_R, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_END_CNTL_R, CM_SHAPER_RAMA_EXP_REGION_END_BASE_R, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_0_1, CM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_0_1, CM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_0_1, CM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_0_1, CM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_2_3, CM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_2_3, CM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_2_3, CM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_2_3, CM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_4_5, CM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_4_5, CM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_4_5, CM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_4_5, CM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_6_7, CM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_6_7, CM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_6_7, CM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_6_7, CM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_8_9, CM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_8_9, CM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_8_9, CM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_8_9, CM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_10_11, CM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_10_11, CM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_10_11, CM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_10_11, CM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_12_13, CM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_12_13, CM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_12_13, CM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_12_13, CM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_14_15, CM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_14_15, CM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_14_15, CM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_14_15, CM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_16_17, CM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_16_17, CM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_16_17, CM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_16_17, CM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_18_19, CM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_18_19, CM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_18_19, CM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_18_19, CM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_20_21, CM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_20_21, CM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_20_21, CM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_20_21, CM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_22_23, CM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_22_23, CM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_22_23, CM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_22_23, CM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_24_25, CM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_24_25, CM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_24_25, CM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_24_25, CM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_26_27, CM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_26_27, CM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_26_27, CM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_26_27, CM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_28_29, CM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_28_29, CM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_28_29, CM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_28_29, CM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_30_31, CM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_30_31, CM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_30_31, CM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_30_31, CM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_32_33, CM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_32_33, CM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_32_33, CM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_RAMA_REGION_32_33, CM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_LUT_WRITE_EN_MASK, CM_SHAPER_LUT_WRITE_EN_MASK, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_LUT_WRITE_EN_MASK, CM_SHAPER_LUT_WRITE_SEL, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_LUT_WRITE_EN_MASK, CM_SHAPER_CONFIG_STATUS, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_LUT_INDEX, CM_SHAPER_LUT_INDEX, mask_sh), \
+ TF_SF(CM0_CM_SHAPER_LUT_DATA, CM_SHAPER_LUT_DATA, mask_sh), \
+ TF_SF(CM0_CM_DGAM_LUT_WRITE_EN_MASK, CM_DGAM_CONFIG_STATUS, mask_sh), \
+ TF_SF(CM0_CM_CONTROL, CM_BYPASS, mask_sh), \
+ TF_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_MODE, mask_sh), \
+ TF_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_PITCH, mask_sh), \
+ TF_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_LINES_PER_CHUNK, mask_sh), \
+ TF_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_ENABLE, mask_sh), \
+ TF_SF(CNVC_CFG0_FORMAT_CONTROL, FORMAT_CNV16, mask_sh), \
+ TF_SF(CNVC_CFG0_FORMAT_CONTROL, CNVC_BYPASS_MSB_ALIGN, mask_sh), \
+ TF_SF(CNVC_CFG0_FORMAT_CONTROL, CLAMP_POSITIVE, mask_sh), \
+ TF_SF(CNVC_CFG0_FORMAT_CONTROL, CLAMP_POSITIVE_C, mask_sh), \
+ TF_SF(CNVC_CFG0_ALPHA_2BIT_LUT, ALPHA_2BIT_LUT0, mask_sh), \
+ TF_SF(CNVC_CFG0_ALPHA_2BIT_LUT, ALPHA_2BIT_LUT1, mask_sh), \
+ TF_SF(CNVC_CFG0_ALPHA_2BIT_LUT, ALPHA_2BIT_LUT2, mask_sh), \
+ TF_SF(CNVC_CFG0_ALPHA_2BIT_LUT, ALPHA_2BIT_LUT3, mask_sh), \
+ TF_SF(CNVC_CFG0_FCNV_FP_BIAS_R, FCNV_FP_BIAS_R, mask_sh), \
+ TF_SF(CNVC_CFG0_FCNV_FP_BIAS_G, FCNV_FP_BIAS_G, mask_sh), \
+ TF_SF(CNVC_CFG0_FCNV_FP_BIAS_B, FCNV_FP_BIAS_B, mask_sh), \
+ TF_SF(CNVC_CFG0_FCNV_FP_SCALE_R, FCNV_FP_SCALE_R, mask_sh), \
+ TF_SF(CNVC_CFG0_FCNV_FP_SCALE_G, FCNV_FP_SCALE_G, mask_sh), \
+ TF_SF(CNVC_CFG0_FCNV_FP_SCALE_B, FCNV_FP_SCALE_B, mask_sh), \
+ TF_SF(CNVC_CFG0_COLOR_KEYER_CONTROL, COLOR_KEYER_EN, mask_sh), \
+ TF_SF(CNVC_CFG0_COLOR_KEYER_CONTROL, COLOR_KEYER_MODE, mask_sh), \
+ TF_SF(CNVC_CFG0_COLOR_KEYER_ALPHA, COLOR_KEYER_ALPHA_LOW, mask_sh), \
+ TF_SF(CNVC_CFG0_COLOR_KEYER_ALPHA, COLOR_KEYER_ALPHA_HIGH, mask_sh), \
+ TF_SF(CNVC_CFG0_COLOR_KEYER_RED, COLOR_KEYER_RED_LOW, mask_sh), \
+ TF_SF(CNVC_CFG0_COLOR_KEYER_RED, COLOR_KEYER_RED_HIGH, mask_sh), \
+ TF_SF(CNVC_CFG0_COLOR_KEYER_GREEN, COLOR_KEYER_GREEN_LOW, mask_sh), \
+ TF_SF(CNVC_CFG0_COLOR_KEYER_GREEN, COLOR_KEYER_GREEN_HIGH, mask_sh), \
+ TF_SF(CNVC_CFG0_COLOR_KEYER_BLUE, COLOR_KEYER_BLUE_LOW, mask_sh), \
+ TF_SF(CNVC_CFG0_COLOR_KEYER_BLUE, COLOR_KEYER_BLUE_HIGH, mask_sh), \
+ TF_SF(CNVC_CUR0_CURSOR0_CONTROL, CUR0_PIX_INV_MODE, mask_sh), \
+ TF_SF(CNVC_CUR0_CURSOR0_CONTROL, CUR0_PIXEL_ALPHA_MOD_EN, mask_sh), \
+ TF_SF(CNVC_CUR0_CURSOR0_CONTROL, CUR0_ROM_EN, mask_sh)
+
+#define TF_REG_FIELD_LIST_DCN2_0(type) \
+ TF_REG_FIELD_LIST(type) \
+ type CM_BLNDGAM_LUT_DATA; \
+ type FORMAT_CNV16; \
+ type CNVC_BYPASS_MSB_ALIGN; \
+ type CLAMP_POSITIVE; \
+ type CLAMP_POSITIVE_C; \
+ type ALPHA_2BIT_LUT0; \
+ type ALPHA_2BIT_LUT1; \
+ type ALPHA_2BIT_LUT2; \
+ type ALPHA_2BIT_LUT3; \
+ type FCNV_FP_BIAS_R; \
+ type FCNV_FP_BIAS_G; \
+ type FCNV_FP_BIAS_B; \
+ type FCNV_FP_SCALE_R; \
+ type FCNV_FP_SCALE_G; \
+ type FCNV_FP_SCALE_B; \
+ type COLOR_KEYER_EN; \
+ type COLOR_KEYER_MODE; \
+ type COLOR_KEYER_ALPHA_LOW; \
+ type COLOR_KEYER_ALPHA_HIGH; \
+ type COLOR_KEYER_RED_LOW; \
+ type COLOR_KEYER_RED_HIGH; \
+ type COLOR_KEYER_GREEN_LOW; \
+ type COLOR_KEYER_GREEN_HIGH; \
+ type COLOR_KEYER_BLUE_LOW; \
+ type COLOR_KEYER_BLUE_HIGH; \
+ type CUR0_PIX_INV_MODE; \
+ type CUR0_PIXEL_ALPHA_MOD_EN; \
+ type CUR0_ROM_EN
+
+struct dcn2_dpp_shift {
+ TF_REG_FIELD_LIST_DCN2_0(uint8_t);
+};
+
+struct dcn2_dpp_mask {
+ TF_REG_FIELD_LIST_DCN2_0(uint32_t);
+};
+
+#define DPP_DCN2_REG_VARIABLE_LIST \
+ DPP_COMMON_REG_VARIABLE_LIST \
+ uint32_t CM_BLNDGAM_LUT_DATA; \
+ uint32_t ALPHA_2BIT_LUT; \
+ uint32_t FCNV_FP_BIAS_R; \
+ uint32_t FCNV_FP_BIAS_G; \
+ uint32_t FCNV_FP_BIAS_B; \
+ uint32_t FCNV_FP_SCALE_R; \
+ uint32_t FCNV_FP_SCALE_G; \
+ uint32_t FCNV_FP_SCALE_B; \
+ uint32_t COLOR_KEYER_CONTROL; \
+ uint32_t COLOR_KEYER_ALPHA; \
+ uint32_t COLOR_KEYER_RED; \
+ uint32_t COLOR_KEYER_GREEN; \
+ uint32_t COLOR_KEYER_BLUE
+
+struct dcn2_dpp_registers {
+ DPP_DCN2_REG_VARIABLE_LIST;
+};
+
+struct dcn20_dpp {
+ struct dpp base;
+
+ const struct dcn2_dpp_registers *tf_regs;
+ const struct dcn2_dpp_shift *tf_shift;
+ const struct dcn2_dpp_mask *tf_mask;
+
+ const uint16_t *filter_v;
+ const uint16_t *filter_h;
+ const uint16_t *filter_v_c;
+ const uint16_t *filter_h_c;
+ int lb_pixel_depth_supported;
+ int lb_memory_size;
+ int lb_bits_per_entry;
+ bool is_write_to_ram_a_safe;
+ struct scaler_data scl_data;
+ struct pwl_params pwl_data;
+};
+
+void dpp20_read_state(struct dpp *dpp_base,
+ struct dcn_dpp_state *s);
+
+void dpp2_set_degamma_pwl(
+ struct dpp *dpp_base,
+ const struct pwl_params *params);
+
+void dpp2_set_degamma(
+ struct dpp *dpp_base,
+ enum ipp_degamma_mode mode);
+
+bool dpp20_program_blnd_lut(
+ struct dpp *dpp_base, const struct pwl_params *params);
+
+bool dpp20_program_shaper(
+ struct dpp *dpp_base,
+ const struct pwl_params *params);
+
+bool dpp20_program_3dlut(
+ struct dpp *dpp_base,
+ struct tetrahedral_params *params);
+
+void dpp2_cnv_set_alpha_keyer(
+ struct dpp *dpp_base,
+ struct cnv_color_keyer_params *color_keyer);
+
+void dscl2_calc_lb_num_partitions(
+ const struct scaler_data *scl_data,
+ enum lb_memory_config lb_config,
+ int *num_part_y,
+ int *num_part_c);
+
+void dpp2_set_cursor_attributes(
+ struct dpp *dpp_base,
+ enum dc_cursor_color_format color_format);
+
+void dpp2_dummy_program_input_lut(
+ struct dpp *dpp_base,
+ const struct dc_gamma *gamma);
+
+void oppn20_dummy_program_regamma_pwl(
+ struct dpp *dpp,
+ const struct pwl_params *params,
+ enum opp_regamma mode);
+
+void dpp2_set_hdr_multiplier(
+ struct dpp *dpp_base,
+ uint32_t multiplier);
+
+bool dpp2_get_optimal_number_of_taps(
+ struct dpp *dpp,
+ struct scaler_data *scl_data,
+ const struct scaling_taps *in_taps);
+
+bool dpp2_construct(struct dcn20_dpp *dpp2,
+ struct dc_context *ctx,
+ uint32_t inst,
+ const struct dcn2_dpp_registers *tf_regs,
+ const struct dcn2_dpp_shift *tf_shift,
+ const struct dcn2_dpp_mask *tf_mask);
+
+#endif /* __DC_HWSS_DCN20_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp_cm.c
new file mode 100644
index 000000000000..e28b8e7bedf5
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp_cm.c
@@ -0,0 +1,990 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services.h"
+
+#include "core_types.h"
+
+#include "reg_helper.h"
+#include "dcn20_dpp.h"
+#include "basics/conversion.h"
+
+#include "dcn10/dcn10_cm_common.h"
+
+#define REG(reg)\
+ dpp->tf_regs->reg
+
+#define CTX \
+ dpp->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name) \
+ dpp->tf_shift->field_name, dpp->tf_mask->field_name
+
+
+
+
+
+static void dpp2_enable_cm_block(
+ struct dpp *dpp_base)
+{
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ REG_UPDATE(CM_CONTROL, CM_BYPASS, 0);
+}
+
+
+static bool dpp2_degamma_ram_inuse(
+ struct dpp *dpp_base,
+ bool *ram_a_inuse)
+{
+ bool ret = false;
+ uint32_t status_reg = 0;
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ REG_GET(CM_DGAM_LUT_WRITE_EN_MASK, CM_DGAM_CONFIG_STATUS,
+ &status_reg);
+
+ if (status_reg == 3) {
+ *ram_a_inuse = true;
+ ret = true;
+ } else if (status_reg == 4) {
+ *ram_a_inuse = false;
+ ret = true;
+ }
+ return ret;
+}
+
+static void dpp2_program_degamma_lut(
+ struct dpp *dpp_base,
+ const struct pwl_result_data *rgb,
+ uint32_t num,
+ bool is_ram_a)
+{
+ uint32_t i;
+
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+ REG_UPDATE(CM_DGAM_LUT_WRITE_EN_MASK,
+ CM_DGAM_LUT_WRITE_EN_MASK, 7);
+ REG_UPDATE(CM_DGAM_LUT_WRITE_EN_MASK, CM_DGAM_LUT_WRITE_SEL,
+ is_ram_a == true ? 0:1);
+
+ REG_SET(CM_DGAM_LUT_INDEX, 0, CM_DGAM_LUT_INDEX, 0);
+ for (i = 0 ; i < num; i++) {
+ REG_SET(CM_DGAM_LUT_DATA, 0, CM_DGAM_LUT_DATA, rgb[i].red_reg);
+ REG_SET(CM_DGAM_LUT_DATA, 0, CM_DGAM_LUT_DATA, rgb[i].green_reg);
+ REG_SET(CM_DGAM_LUT_DATA, 0, CM_DGAM_LUT_DATA, rgb[i].blue_reg);
+
+ REG_SET(CM_DGAM_LUT_DATA, 0,
+ CM_DGAM_LUT_DATA, rgb[i].delta_red_reg);
+ REG_SET(CM_DGAM_LUT_DATA, 0,
+ CM_DGAM_LUT_DATA, rgb[i].delta_green_reg);
+ REG_SET(CM_DGAM_LUT_DATA, 0,
+ CM_DGAM_LUT_DATA, rgb[i].delta_blue_reg);
+
+ }
+
+}
+
+void dpp2_set_degamma_pwl(
+ struct dpp *dpp_base,
+ const struct pwl_params *params)
+{
+ bool is_ram_a = true;
+
+ dpp1_power_on_degamma_lut(dpp_base, true);
+ dpp2_enable_cm_block(dpp_base);
+ dpp2_degamma_ram_inuse(dpp_base, &is_ram_a);
+ if (is_ram_a == true)
+ dpp1_program_degamma_lutb_settings(dpp_base, params);
+ else
+ dpp1_program_degamma_luta_settings(dpp_base, params);
+
+ dpp2_program_degamma_lut(dpp_base, params->rgb_resulted, params->hw_points_num, !is_ram_a);
+ dpp1_degamma_ram_select(dpp_base, !is_ram_a);
+}
+
+void dpp2_set_degamma(
+ struct dpp *dpp_base,
+ enum ipp_degamma_mode mode)
+{
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+ dpp2_enable_cm_block(dpp_base);
+
+ switch (mode) {
+ case IPP_DEGAMMA_MODE_BYPASS:
+ /* Setting de gamma bypass for now */
+ REG_UPDATE(CM_DGAM_CONTROL, CM_DGAM_LUT_MODE, 0);
+ break;
+ case IPP_DEGAMMA_MODE_HW_sRGB:
+ REG_UPDATE(CM_DGAM_CONTROL, CM_DGAM_LUT_MODE, 1);
+ break;
+ case IPP_DEGAMMA_MODE_HW_xvYCC:
+ REG_UPDATE(CM_DGAM_CONTROL, CM_DGAM_LUT_MODE, 2);
+ break;
+ default:
+ BREAK_TO_DEBUGGER();
+ break;
+ }
+}
+
+static void dpp20_power_on_blnd_lut(
+ struct dpp *dpp_base,
+ bool power_on)
+{
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ REG_SET(CM_MEM_PWR_CTRL, 0,
+ BLNDGAM_MEM_PWR_FORCE, power_on == true ? 0:1);
+
+}
+
+static void dpp20_configure_blnd_lut(
+ struct dpp *dpp_base,
+ bool is_ram_a)
+{
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ REG_UPDATE(CM_BLNDGAM_LUT_WRITE_EN_MASK,
+ CM_BLNDGAM_LUT_WRITE_EN_MASK, 7);
+ REG_UPDATE(CM_BLNDGAM_LUT_WRITE_EN_MASK,
+ CM_BLNDGAM_LUT_WRITE_SEL, is_ram_a == true ? 0:1);
+ REG_SET(CM_BLNDGAM_LUT_INDEX, 0, CM_BLNDGAM_LUT_INDEX, 0);
+}
+
+static void dpp20_program_blnd_pwl(
+ struct dpp *dpp_base,
+ const struct pwl_result_data *rgb,
+ uint32_t num)
+{
+ uint32_t i;
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ for (i = 0 ; i < num; i++) {
+ REG_SET(CM_BLNDGAM_LUT_DATA, 0, CM_BLNDGAM_LUT_DATA, rgb[i].red_reg);
+ REG_SET(CM_BLNDGAM_LUT_DATA, 0, CM_BLNDGAM_LUT_DATA, rgb[i].green_reg);
+ REG_SET(CM_BLNDGAM_LUT_DATA, 0, CM_BLNDGAM_LUT_DATA, rgb[i].blue_reg);
+
+ REG_SET(CM_BLNDGAM_LUT_DATA, 0,
+ CM_BLNDGAM_LUT_DATA, rgb[i].delta_red_reg);
+ REG_SET(CM_BLNDGAM_LUT_DATA, 0,
+ CM_BLNDGAM_LUT_DATA, rgb[i].delta_green_reg);
+ REG_SET(CM_BLNDGAM_LUT_DATA, 0,
+ CM_BLNDGAM_LUT_DATA, rgb[i].delta_blue_reg);
+
+ }
+
+}
+
+static void dcn20_dpp_cm_get_reg_field(
+ struct dcn20_dpp *dpp,
+ struct xfer_func_reg *reg)
+{
+ reg->shifts.exp_region0_lut_offset = dpp->tf_shift->CM_BLNDGAM_RAMA_EXP_REGION0_LUT_OFFSET;
+ reg->masks.exp_region0_lut_offset = dpp->tf_mask->CM_BLNDGAM_RAMA_EXP_REGION0_LUT_OFFSET;
+ reg->shifts.exp_region0_num_segments = dpp->tf_shift->CM_BLNDGAM_RAMA_EXP_REGION0_NUM_SEGMENTS;
+ reg->masks.exp_region0_num_segments = dpp->tf_mask->CM_BLNDGAM_RAMA_EXP_REGION0_NUM_SEGMENTS;
+ reg->shifts.exp_region1_lut_offset = dpp->tf_shift->CM_BLNDGAM_RAMA_EXP_REGION1_LUT_OFFSET;
+ reg->masks.exp_region1_lut_offset = dpp->tf_mask->CM_BLNDGAM_RAMA_EXP_REGION1_LUT_OFFSET;
+ reg->shifts.exp_region1_num_segments = dpp->tf_shift->CM_BLNDGAM_RAMA_EXP_REGION1_NUM_SEGMENTS;
+ reg->masks.exp_region1_num_segments = dpp->tf_mask->CM_BLNDGAM_RAMA_EXP_REGION1_NUM_SEGMENTS;
+
+ reg->shifts.field_region_end = dpp->tf_shift->CM_BLNDGAM_RAMA_EXP_REGION_END_B;
+ reg->masks.field_region_end = dpp->tf_mask->CM_BLNDGAM_RAMA_EXP_REGION_END_B;
+ reg->shifts.field_region_end_slope = dpp->tf_shift->CM_BLNDGAM_RAMA_EXP_REGION_END_SLOPE_B;
+ reg->masks.field_region_end_slope = dpp->tf_mask->CM_BLNDGAM_RAMA_EXP_REGION_END_SLOPE_B;
+ reg->shifts.field_region_end_base = dpp->tf_shift->CM_BLNDGAM_RAMA_EXP_REGION_END_BASE_B;
+ reg->masks.field_region_end_base = dpp->tf_mask->CM_BLNDGAM_RAMA_EXP_REGION_END_BASE_B;
+ reg->shifts.field_region_linear_slope = dpp->tf_shift->CM_BLNDGAM_RAMA_EXP_REGION_LINEAR_SLOPE_B;
+ reg->masks.field_region_linear_slope = dpp->tf_mask->CM_BLNDGAM_RAMA_EXP_REGION_LINEAR_SLOPE_B;
+ reg->shifts.exp_region_start = dpp->tf_shift->CM_BLNDGAM_RAMA_EXP_REGION_START_B;
+ reg->masks.exp_region_start = dpp->tf_mask->CM_BLNDGAM_RAMA_EXP_REGION_START_B;
+ reg->shifts.exp_resion_start_segment = dpp->tf_shift->CM_BLNDGAM_RAMA_EXP_REGION_START_SEGMENT_B;
+ reg->masks.exp_resion_start_segment = dpp->tf_mask->CM_BLNDGAM_RAMA_EXP_REGION_START_SEGMENT_B;
+}
+
+/*program blnd lut RAM A*/
+static void dpp20_program_blnd_luta_settings(
+ struct dpp *dpp_base,
+ const struct pwl_params *params)
+{
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+ struct xfer_func_reg gam_regs;
+
+ dcn20_dpp_cm_get_reg_field(dpp, &gam_regs);
+
+ gam_regs.start_cntl_b = REG(CM_BLNDGAM_RAMA_START_CNTL_B);
+ gam_regs.start_cntl_g = REG(CM_BLNDGAM_RAMA_START_CNTL_G);
+ gam_regs.start_cntl_r = REG(CM_BLNDGAM_RAMA_START_CNTL_R);
+ gam_regs.start_slope_cntl_b = REG(CM_BLNDGAM_RAMA_SLOPE_CNTL_B);
+ gam_regs.start_slope_cntl_g = REG(CM_BLNDGAM_RAMA_SLOPE_CNTL_G);
+ gam_regs.start_slope_cntl_r = REG(CM_BLNDGAM_RAMA_SLOPE_CNTL_R);
+ gam_regs.start_end_cntl1_b = REG(CM_BLNDGAM_RAMA_END_CNTL1_B);
+ gam_regs.start_end_cntl2_b = REG(CM_BLNDGAM_RAMA_END_CNTL2_B);
+ gam_regs.start_end_cntl1_g = REG(CM_BLNDGAM_RAMA_END_CNTL1_G);
+ gam_regs.start_end_cntl2_g = REG(CM_BLNDGAM_RAMA_END_CNTL2_G);
+ gam_regs.start_end_cntl1_r = REG(CM_BLNDGAM_RAMA_END_CNTL1_R);
+ gam_regs.start_end_cntl2_r = REG(CM_BLNDGAM_RAMA_END_CNTL2_R);
+ gam_regs.region_start = REG(CM_BLNDGAM_RAMA_REGION_0_1);
+ gam_regs.region_end = REG(CM_BLNDGAM_RAMA_REGION_32_33);
+
+ cm_helper_program_xfer_func(dpp->base.ctx, params, &gam_regs);
+}
+
+/*program blnd lut RAM B*/
+static void dpp20_program_blnd_lutb_settings(
+ struct dpp *dpp_base,
+ const struct pwl_params *params)
+{
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+ struct xfer_func_reg gam_regs;
+
+ dcn20_dpp_cm_get_reg_field(dpp, &gam_regs);
+
+ gam_regs.start_cntl_b = REG(CM_BLNDGAM_RAMB_START_CNTL_B);
+ gam_regs.start_cntl_g = REG(CM_BLNDGAM_RAMB_START_CNTL_G);
+ gam_regs.start_cntl_r = REG(CM_BLNDGAM_RAMB_START_CNTL_R);
+ gam_regs.start_slope_cntl_b = REG(CM_BLNDGAM_RAMB_SLOPE_CNTL_B);
+ gam_regs.start_slope_cntl_g = REG(CM_BLNDGAM_RAMB_SLOPE_CNTL_G);
+ gam_regs.start_slope_cntl_r = REG(CM_BLNDGAM_RAMB_SLOPE_CNTL_R);
+ gam_regs.start_end_cntl1_b = REG(CM_BLNDGAM_RAMB_END_CNTL1_B);
+ gam_regs.start_end_cntl2_b = REG(CM_BLNDGAM_RAMB_END_CNTL2_B);
+ gam_regs.start_end_cntl1_g = REG(CM_BLNDGAM_RAMB_END_CNTL1_G);
+ gam_regs.start_end_cntl2_g = REG(CM_BLNDGAM_RAMB_END_CNTL2_G);
+ gam_regs.start_end_cntl1_r = REG(CM_BLNDGAM_RAMB_END_CNTL1_R);
+ gam_regs.start_end_cntl2_r = REG(CM_BLNDGAM_RAMB_END_CNTL2_R);
+ gam_regs.region_start = REG(CM_BLNDGAM_RAMB_REGION_0_1);
+ gam_regs.region_end = REG(CM_BLNDGAM_RAMB_REGION_32_33);
+
+ cm_helper_program_xfer_func(dpp->base.ctx, params, &gam_regs);
+}
+
+static enum dc_lut_mode dpp20_get_blndgam_current(struct dpp *dpp_base)
+{
+ enum dc_lut_mode mode;
+ uint32_t state_mode;
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ REG_GET(CM_BLNDGAM_LUT_WRITE_EN_MASK,
+ CM_BLNDGAM_CONFIG_STATUS, &state_mode);
+
+ switch (state_mode) {
+ case 0:
+ mode = LUT_BYPASS;
+ break;
+ case 1:
+ mode = LUT_RAM_A;
+ break;
+ case 2:
+ mode = LUT_RAM_B;
+ break;
+ default:
+ mode = LUT_BYPASS;
+ break;
+ }
+ return mode;
+}
+
+bool dpp20_program_blnd_lut(
+ struct dpp *dpp_base, const struct pwl_params *params)
+{
+ enum dc_lut_mode current_mode;
+ enum dc_lut_mode next_mode;
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ if (params == NULL) {
+ REG_SET(CM_BLNDGAM_CONTROL, 0, CM_BLNDGAM_LUT_MODE, 0);
+ return false;
+ }
+ current_mode = dpp20_get_blndgam_current(dpp_base);
+ if (current_mode == LUT_BYPASS || current_mode == LUT_RAM_A)
+ next_mode = LUT_RAM_B;
+ else
+ next_mode = LUT_RAM_A;
+
+ dpp20_power_on_blnd_lut(dpp_base, true);
+ dpp20_configure_blnd_lut(dpp_base, next_mode == LUT_RAM_A ? true:false);
+
+ if (next_mode == LUT_RAM_A)
+ dpp20_program_blnd_luta_settings(dpp_base, params);
+ else
+ dpp20_program_blnd_lutb_settings(dpp_base, params);
+
+ dpp20_program_blnd_pwl(
+ dpp_base, params->rgb_resulted, params->hw_points_num);
+
+ REG_SET(CM_BLNDGAM_CONTROL, 0, CM_BLNDGAM_LUT_MODE,
+ next_mode == LUT_RAM_A ? 1:2);
+
+ return true;
+}
+
+
+static void dpp20_program_shaper_lut(
+ struct dpp *dpp_base,
+ const struct pwl_result_data *rgb,
+ uint32_t num)
+{
+ uint32_t i, red, green, blue;
+ uint32_t red_delta, green_delta, blue_delta;
+ uint32_t red_value, green_value, blue_value;
+
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ for (i = 0 ; i < num; i++) {
+
+ red = rgb[i].red_reg;
+ green = rgb[i].green_reg;
+ blue = rgb[i].blue_reg;
+
+ red_delta = rgb[i].delta_red_reg;
+ green_delta = rgb[i].delta_green_reg;
+ blue_delta = rgb[i].delta_blue_reg;
+
+ red_value = ((red_delta & 0x3ff) << 14) | (red & 0x3fff);
+ green_value = ((green_delta & 0x3ff) << 14) | (green & 0x3fff);
+ blue_value = ((blue_delta & 0x3ff) << 14) | (blue & 0x3fff);
+
+ REG_SET(CM_SHAPER_LUT_DATA, 0, CM_SHAPER_LUT_DATA, red_value);
+ REG_SET(CM_SHAPER_LUT_DATA, 0, CM_SHAPER_LUT_DATA, green_value);
+ REG_SET(CM_SHAPER_LUT_DATA, 0, CM_SHAPER_LUT_DATA, blue_value);
+ }
+
+}
+
+static enum dc_lut_mode dpp20_get_shaper_current(struct dpp *dpp_base)
+{
+ enum dc_lut_mode mode;
+ uint32_t state_mode;
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ REG_GET(CM_SHAPER_LUT_WRITE_EN_MASK,
+ CM_SHAPER_CONFIG_STATUS, &state_mode);
+
+ switch (state_mode) {
+ case 0:
+ mode = LUT_BYPASS;
+ break;
+ case 1:
+ mode = LUT_RAM_A;
+ break;
+ case 2:
+ mode = LUT_RAM_B;
+ break;
+ default:
+ mode = LUT_BYPASS;
+ break;
+ }
+ return mode;
+}
+
+static void dpp20_configure_shaper_lut(
+ struct dpp *dpp_base,
+ bool is_ram_a)
+{
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ REG_UPDATE(CM_SHAPER_LUT_WRITE_EN_MASK,
+ CM_SHAPER_LUT_WRITE_EN_MASK, 7);
+ REG_UPDATE(CM_SHAPER_LUT_WRITE_EN_MASK,
+ CM_SHAPER_LUT_WRITE_SEL, is_ram_a == true ? 0:1);
+ REG_SET(CM_SHAPER_LUT_INDEX, 0, CM_SHAPER_LUT_INDEX, 0);
+}
+
+/*program shaper RAM A*/
+
+static void dpp20_program_shaper_luta_settings(
+ struct dpp *dpp_base,
+ const struct pwl_params *params)
+{
+ const struct gamma_curve *curve;
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ REG_SET_2(CM_SHAPER_RAMA_START_CNTL_B, 0,
+ CM_SHAPER_RAMA_EXP_REGION_START_B, params->corner_points[0].blue.custom_float_x,
+ CM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, 0);
+ REG_SET_2(CM_SHAPER_RAMA_START_CNTL_G, 0,
+ CM_SHAPER_RAMA_EXP_REGION_START_G, params->corner_points[0].green.custom_float_x,
+ CM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G, 0);
+ REG_SET_2(CM_SHAPER_RAMA_START_CNTL_R, 0,
+ CM_SHAPER_RAMA_EXP_REGION_START_R, params->corner_points[0].red.custom_float_x,
+ CM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R, 0);
+
+ REG_SET_2(CM_SHAPER_RAMA_END_CNTL_B, 0,
+ CM_SHAPER_RAMA_EXP_REGION_END_B, params->corner_points[1].blue.custom_float_x,
+ CM_SHAPER_RAMA_EXP_REGION_END_BASE_B, params->corner_points[1].blue.custom_float_y);
+
+ REG_SET_2(CM_SHAPER_RAMA_END_CNTL_G, 0,
+ CM_SHAPER_RAMA_EXP_REGION_END_G, params->corner_points[1].green.custom_float_x,
+ CM_SHAPER_RAMA_EXP_REGION_END_BASE_G, params->corner_points[1].green.custom_float_y);
+
+ REG_SET_2(CM_SHAPER_RAMA_END_CNTL_R, 0,
+ CM_SHAPER_RAMA_EXP_REGION_END_R, params->corner_points[1].red.custom_float_x,
+ CM_SHAPER_RAMA_EXP_REGION_END_BASE_R, params->corner_points[1].red.custom_float_y);
+
+ curve = params->arr_curve_points;
+ REG_SET_4(CM_SHAPER_RAMA_REGION_0_1, 0,
+ CM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMA_REGION_2_3, 0,
+ CM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMA_REGION_4_5, 0,
+ CM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMA_REGION_6_7, 0,
+ CM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMA_REGION_8_9, 0,
+ CM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMA_REGION_10_11, 0,
+ CM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMA_REGION_12_13, 0,
+ CM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMA_REGION_14_15, 0,
+ CM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMA_REGION_16_17, 0,
+ CM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMA_REGION_18_19, 0,
+ CM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMA_REGION_20_21, 0,
+ CM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMA_REGION_22_23, 0,
+ CM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMA_REGION_24_25, 0,
+ CM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMA_REGION_26_27, 0,
+ CM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMA_REGION_28_29, 0,
+ CM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMA_REGION_30_31, 0,
+ CM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMA_REGION_32_33, 0,
+ CM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS, curve[1].segments_num);
+}
+
+/*program shaper RAM B*/
+static void dpp20_program_shaper_lutb_settings(
+ struct dpp *dpp_base,
+ const struct pwl_params *params)
+{
+ const struct gamma_curve *curve;
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ REG_SET_2(CM_SHAPER_RAMB_START_CNTL_B, 0,
+ CM_SHAPER_RAMB_EXP_REGION_START_B, params->corner_points[0].blue.custom_float_x,
+ CM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B, 0);
+ REG_SET_2(CM_SHAPER_RAMB_START_CNTL_G, 0,
+ CM_SHAPER_RAMB_EXP_REGION_START_G, params->corner_points[0].green.custom_float_x,
+ CM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G, 0);
+ REG_SET_2(CM_SHAPER_RAMB_START_CNTL_R, 0,
+ CM_SHAPER_RAMB_EXP_REGION_START_R, params->corner_points[0].red.custom_float_x,
+ CM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R, 0);
+
+ REG_SET_2(CM_SHAPER_RAMB_END_CNTL_B, 0,
+ CM_SHAPER_RAMB_EXP_REGION_END_B, params->corner_points[1].blue.custom_float_x,
+ CM_SHAPER_RAMB_EXP_REGION_END_BASE_B, params->corner_points[1].blue.custom_float_y);
+
+ REG_SET_2(CM_SHAPER_RAMB_END_CNTL_G, 0,
+ CM_SHAPER_RAMB_EXP_REGION_END_G, params->corner_points[1].green.custom_float_x,
+ CM_SHAPER_RAMB_EXP_REGION_END_BASE_G, params->corner_points[1].green.custom_float_y);
+
+ REG_SET_2(CM_SHAPER_RAMB_END_CNTL_R, 0,
+ CM_SHAPER_RAMB_EXP_REGION_END_R, params->corner_points[1].red.custom_float_x,
+ CM_SHAPER_RAMB_EXP_REGION_END_BASE_R, params->corner_points[1].red.custom_float_y);
+
+ curve = params->arr_curve_points;
+ REG_SET_4(CM_SHAPER_RAMB_REGION_0_1, 0,
+ CM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMB_REGION_2_3, 0,
+ CM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMB_REGION_4_5, 0,
+ CM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMB_REGION_6_7, 0,
+ CM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMB_REGION_8_9, 0,
+ CM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMB_REGION_10_11, 0,
+ CM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMB_REGION_12_13, 0,
+ CM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMB_REGION_14_15, 0,
+ CM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMB_REGION_16_17, 0,
+ CM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMB_REGION_18_19, 0,
+ CM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMB_REGION_20_21, 0,
+ CM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMB_REGION_22_23, 0,
+ CM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMB_REGION_24_25, 0,
+ CM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMB_REGION_26_27, 0,
+ CM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMB_REGION_28_29, 0,
+ CM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMB_REGION_30_31, 0,
+ CM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS, curve[1].segments_num);
+
+ curve += 2;
+ REG_SET_4(CM_SHAPER_RAMB_REGION_32_33, 0,
+ CM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET, curve[0].offset,
+ CM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS, curve[0].segments_num,
+ CM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET, curve[1].offset,
+ CM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS, curve[1].segments_num);
+
+}
+
+
+bool dpp20_program_shaper(
+ struct dpp *dpp_base,
+ const struct pwl_params *params)
+{
+ enum dc_lut_mode current_mode;
+ enum dc_lut_mode next_mode;
+
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ if (params == NULL) {
+ REG_SET(CM_SHAPER_CONTROL, 0, CM_SHAPER_LUT_MODE, 0);
+ return false;
+ }
+ current_mode = dpp20_get_shaper_current(dpp_base);
+
+ if (current_mode == LUT_BYPASS || current_mode == LUT_RAM_A)
+ next_mode = LUT_RAM_B;
+ else
+ next_mode = LUT_RAM_A;
+
+ dpp20_configure_shaper_lut(dpp_base, next_mode == LUT_RAM_A ? true:false);
+
+ if (next_mode == LUT_RAM_A)
+ dpp20_program_shaper_luta_settings(dpp_base, params);
+ else
+ dpp20_program_shaper_lutb_settings(dpp_base, params);
+
+ dpp20_program_shaper_lut(
+ dpp_base, params->rgb_resulted, params->hw_points_num);
+
+ REG_SET(CM_SHAPER_CONTROL, 0, CM_SHAPER_LUT_MODE, next_mode == LUT_RAM_A ? 1:2);
+
+ return true;
+
+}
+
+static enum dc_lut_mode get3dlut_config(
+ struct dpp *dpp_base,
+ bool *is_17x17x17,
+ bool *is_12bits_color_channel)
+{
+ uint32_t i_mode, i_enable_10bits, lut_size;
+ enum dc_lut_mode mode;
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ REG_GET_2(CM_3DLUT_READ_WRITE_CONTROL,
+ CM_3DLUT_CONFIG_STATUS, &i_mode,
+ CM_3DLUT_30BIT_EN, &i_enable_10bits);
+
+ switch (i_mode) {
+ case 0:
+ mode = LUT_BYPASS;
+ break;
+ case 1:
+ mode = LUT_RAM_A;
+ break;
+ case 2:
+ mode = LUT_RAM_B;
+ break;
+ default:
+ mode = LUT_BYPASS;
+ break;
+ }
+ if (i_enable_10bits > 0)
+ *is_12bits_color_channel = false;
+ else
+ *is_12bits_color_channel = true;
+
+ REG_GET(CM_3DLUT_MODE, CM_3DLUT_SIZE, &lut_size);
+
+ if (lut_size == 0)
+ *is_17x17x17 = true;
+ else
+ *is_17x17x17 = false;
+
+ return mode;
+}
+/*
+ * select ramA or ramB, or bypass
+ * select color channel size 10 or 12 bits
+ * select 3dlut size 17x17x17 or 9x9x9
+ */
+static void dpp20_set_3dlut_mode(
+ struct dpp *dpp_base,
+ enum dc_lut_mode mode,
+ bool is_color_channel_12bits,
+ bool is_lut_size17x17x17)
+{
+ uint32_t lut_mode;
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ if (mode == LUT_BYPASS)
+ lut_mode = 0;
+ else if (mode == LUT_RAM_A)
+ lut_mode = 1;
+ else
+ lut_mode = 2;
+
+ REG_UPDATE_2(CM_3DLUT_MODE,
+ CM_3DLUT_MODE, lut_mode,
+ CM_3DLUT_SIZE, is_lut_size17x17x17 == true ? 0 : 1);
+}
+
+static void dpp20_select_3dlut_ram(
+ struct dpp *dpp_base,
+ enum dc_lut_mode mode,
+ bool is_color_channel_12bits)
+{
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ REG_UPDATE_2(CM_3DLUT_READ_WRITE_CONTROL,
+ CM_3DLUT_RAM_SEL, mode == LUT_RAM_A ? 0 : 1,
+ CM_3DLUT_30BIT_EN,
+ is_color_channel_12bits == true ? 0:1);
+}
+
+
+
+static void dpp20_set3dlut_ram12(
+ struct dpp *dpp_base,
+ const struct dc_rgb *lut,
+ uint32_t entries)
+{
+ uint32_t i, red, green, blue, red1, green1, blue1;
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ for (i = 0 ; i < entries; i += 2) {
+ red = lut[i].red<<4;
+ green = lut[i].green<<4;
+ blue = lut[i].blue<<4;
+ red1 = lut[i+1].red<<4;
+ green1 = lut[i+1].green<<4;
+ blue1 = lut[i+1].blue<<4;
+
+ REG_SET_2(CM_3DLUT_DATA, 0,
+ CM_3DLUT_DATA0, red,
+ CM_3DLUT_DATA1, red1);
+
+ REG_SET_2(CM_3DLUT_DATA, 0,
+ CM_3DLUT_DATA0, green,
+ CM_3DLUT_DATA1, green1);
+
+ REG_SET_2(CM_3DLUT_DATA, 0,
+ CM_3DLUT_DATA0, blue,
+ CM_3DLUT_DATA1, blue1);
+
+ }
+}
+
+/*
+ * load selected lut with 10 bits color channels
+ */
+static void dpp20_set3dlut_ram10(
+ struct dpp *dpp_base,
+ const struct dc_rgb *lut,
+ uint32_t entries)
+{
+ uint32_t i, red, green, blue, value;
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ for (i = 0; i < entries; i++) {
+ red = lut[i].red;
+ green = lut[i].green;
+ blue = lut[i].blue;
+
+ value = (red<<20) | (green<<10) | blue;
+
+ REG_SET(CM_3DLUT_DATA_30BIT, 0, CM_3DLUT_DATA_30BIT, value);
+ }
+
+}
+
+
+static void dpp20_select_3dlut_ram_mask(
+ struct dpp *dpp_base,
+ uint32_t ram_selection_mask)
+{
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ REG_UPDATE(CM_3DLUT_READ_WRITE_CONTROL, CM_3DLUT_WRITE_EN_MASK,
+ ram_selection_mask);
+ REG_SET(CM_3DLUT_INDEX, 0, CM_3DLUT_INDEX, 0);
+}
+
+bool dpp20_program_3dlut(
+ struct dpp *dpp_base,
+ struct tetrahedral_params *params)
+{
+ enum dc_lut_mode mode;
+ bool is_17x17x17;
+ bool is_12bits_color_channel;
+ struct dc_rgb *lut0;
+ struct dc_rgb *lut1;
+ struct dc_rgb *lut2;
+ struct dc_rgb *lut3;
+ int lut_size0;
+ int lut_size;
+
+ if (params == NULL) {
+ dpp20_set_3dlut_mode(dpp_base, LUT_BYPASS, false, false);
+ return false;
+ }
+ mode = get3dlut_config(dpp_base, &is_17x17x17, &is_12bits_color_channel);
+
+ if (mode == LUT_BYPASS || mode == LUT_RAM_B)
+ mode = LUT_RAM_A;
+ else
+ mode = LUT_RAM_B;
+
+ is_17x17x17 = !params->use_tetrahedral_9;
+ is_12bits_color_channel = params->use_12bits;
+ if (is_17x17x17) {
+ lut0 = params->tetrahedral_17.lut0;
+ lut1 = params->tetrahedral_17.lut1;
+ lut2 = params->tetrahedral_17.lut2;
+ lut3 = params->tetrahedral_17.lut3;
+ lut_size0 = sizeof(params->tetrahedral_17.lut0)/
+ sizeof(params->tetrahedral_17.lut0[0]);
+ lut_size = sizeof(params->tetrahedral_17.lut1)/
+ sizeof(params->tetrahedral_17.lut1[0]);
+ } else {
+ lut0 = params->tetrahedral_9.lut0;
+ lut1 = params->tetrahedral_9.lut1;
+ lut2 = params->tetrahedral_9.lut2;
+ lut3 = params->tetrahedral_9.lut3;
+ lut_size0 = sizeof(params->tetrahedral_9.lut0)/
+ sizeof(params->tetrahedral_9.lut0[0]);
+ lut_size = sizeof(params->tetrahedral_9.lut1)/
+ sizeof(params->tetrahedral_9.lut1[0]);
+ }
+
+ dpp20_select_3dlut_ram(dpp_base, mode,
+ is_12bits_color_channel);
+ dpp20_select_3dlut_ram_mask(dpp_base, 0x1);
+ if (is_12bits_color_channel)
+ dpp20_set3dlut_ram12(dpp_base, lut0, lut_size0);
+ else
+ dpp20_set3dlut_ram10(dpp_base, lut0, lut_size0);
+
+ dpp20_select_3dlut_ram_mask(dpp_base, 0x2);
+ if (is_12bits_color_channel)
+ dpp20_set3dlut_ram12(dpp_base, lut1, lut_size);
+ else
+ dpp20_set3dlut_ram10(dpp_base, lut1, lut_size);
+
+ dpp20_select_3dlut_ram_mask(dpp_base, 0x4);
+ if (is_12bits_color_channel)
+ dpp20_set3dlut_ram12(dpp_base, lut2, lut_size);
+ else
+ dpp20_set3dlut_ram10(dpp_base, lut2, lut_size);
+
+ dpp20_select_3dlut_ram_mask(dpp_base, 0x8);
+ if (is_12bits_color_channel)
+ dpp20_set3dlut_ram12(dpp_base, lut3, lut_size);
+ else
+ dpp20_set3dlut_ram10(dpp_base, lut3, lut_size);
+
+
+ dpp20_set_3dlut_mode(dpp_base, mode, is_12bits_color_channel,
+ is_17x17x17);
+
+ return true;
+}
+
+void dpp2_set_hdr_multiplier(
+ struct dpp *dpp_base,
+ uint32_t multiplier)
+{
+ struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+ REG_UPDATE(CM_HDR_MULT_COEF, CM_HDR_MULT_COEF, multiplier);
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.c
new file mode 100644
index 000000000000..ffd0014ec3b5
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.c
@@ -0,0 +1,694 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+#include "reg_helper.h"
+#include "dcn20_dsc.h"
+#include "dsc/dscc_types.h"
+
+static void dsc_log_pps(struct display_stream_compressor *dsc, struct drm_dsc_config *pps);
+static bool dsc_prepare_config(struct display_stream_compressor *dsc, const struct dsc_config *dsc_cfg,
+ struct dsc_optc_config *dsc_optc_cfg);
+static void dsc_init_reg_values(struct dsc_reg_values *reg_vals);
+static void dsc_update_from_dsc_parameters(struct dsc_reg_values *reg_vals, const struct dsc_parameters *dsc_params);
+static void dsc_write_to_registers(struct display_stream_compressor *dsc, const struct dsc_reg_values *reg_vals);
+static enum dsc_pixel_format dsc_dc_pixel_encoding_to_dsc_pixel_format(enum dc_pixel_encoding dc_pix_enc, bool is_ycbcr422_simple);
+static enum dsc_bits_per_comp dsc_dc_color_depth_to_dsc_bits_per_comp(enum dc_color_depth);
+
+/* Object I/F functions */
+static void dsc2_get_enc_caps(struct dsc_enc_caps *dsc_enc_caps, int pixel_clock_100Hz);
+static void dsc2_read_state(struct display_stream_compressor *dsc, struct dcn_dsc_state *s);
+static bool dsc2_validate_stream(struct display_stream_compressor *dsc, const struct dsc_config *dsc_cfg);
+static void dsc2_set_config(struct display_stream_compressor *dsc, const struct dsc_config *dsc_cfg,
+ struct dsc_optc_config *dsc_optc_cfg, uint8_t *dsc_packed_pps);
+static void dsc2_enable(struct display_stream_compressor *dsc, int opp_pipe);
+static void dsc2_disable(struct display_stream_compressor *dsc);
+
+const struct dsc_funcs dcn20_dsc_funcs = {
+ .dsc_get_enc_caps = dsc2_get_enc_caps,
+ .dsc_read_state = dsc2_read_state,
+ .dsc_validate_stream = dsc2_validate_stream,
+ .dsc_set_config = dsc2_set_config,
+ .dsc_enable = dsc2_enable,
+ .dsc_disable = dsc2_disable,
+};
+
+/* Macro definitios for REG_SET macros*/
+#define CTX \
+ dsc20->base.ctx
+
+#define REG(reg)\
+ dsc20->dsc_regs->reg
+
+#undef FN
+#define FN(reg_name, field_name) \
+ dsc20->dsc_shift->field_name, dsc20->dsc_mask->field_name
+#define DC_LOGGER \
+ dsc->ctx->logger
+
+enum dsc_bits_per_comp {
+ DSC_BPC_8 = 8,
+ DSC_BPC_10 = 10,
+ DSC_BPC_12 = 12,
+ DSC_BPC_UNKNOWN
+};
+
+/* API functions (external or via structure->function_pointer) */
+
+void dsc2_construct(struct dcn20_dsc *dsc,
+ struct dc_context *ctx,
+ int inst,
+ const struct dcn20_dsc_registers *dsc_regs,
+ const struct dcn20_dsc_shift *dsc_shift,
+ const struct dcn20_dsc_mask *dsc_mask)
+{
+ dsc->base.ctx = ctx;
+ dsc->base.inst = inst;
+ dsc->base.funcs = &dcn20_dsc_funcs;
+
+ dsc->dsc_regs = dsc_regs;
+ dsc->dsc_shift = dsc_shift;
+ dsc->dsc_mask = dsc_mask;
+
+ dsc->max_image_width = 5184;
+}
+
+
+#define DCN20_MAX_PIXEL_CLOCK_Mhz 1188
+#define DCN20_MAX_DISPLAY_CLOCK_Mhz 1200
+
+/* This returns the capabilities for a single DSC encoder engine. Number of slices and total throughput
+ * can be doubled, tripled etc. by using additional DSC engines.
+ */
+static void dsc2_get_enc_caps(struct dsc_enc_caps *dsc_enc_caps, int pixel_clock_100Hz)
+{
+ dsc_enc_caps->dsc_version = 0x21; /* v1.2 - DP spec defined it in reverse order and we kept it */
+
+ dsc_enc_caps->slice_caps.bits.NUM_SLICES_1 = 1;
+ dsc_enc_caps->slice_caps.bits.NUM_SLICES_2 = 1;
+ dsc_enc_caps->slice_caps.bits.NUM_SLICES_3 = 1;
+ dsc_enc_caps->slice_caps.bits.NUM_SLICES_4 = 1;
+
+ dsc_enc_caps->lb_bit_depth = 13;
+ dsc_enc_caps->is_block_pred_supported = true;
+
+ dsc_enc_caps->color_formats.bits.RGB = 1;
+ dsc_enc_caps->color_formats.bits.YCBCR_444 = 1;
+ dsc_enc_caps->color_formats.bits.YCBCR_SIMPLE_422 = 1;
+ dsc_enc_caps->color_formats.bits.YCBCR_NATIVE_422 = 1;
+ dsc_enc_caps->color_formats.bits.YCBCR_NATIVE_420 = 1;
+
+ dsc_enc_caps->color_depth.bits.COLOR_DEPTH_8_BPC = 1;
+ dsc_enc_caps->color_depth.bits.COLOR_DEPTH_10_BPC = 1;
+ dsc_enc_caps->color_depth.bits.COLOR_DEPTH_12_BPC = 1;
+
+ /* Maximum total throughput with all the slices combined. This is different from how DP spec specifies it.
+ * Our decoder's total throughput in Pix/s is equal to DISPCLK. This is then shared between slices.
+ * The value below is the absolute maximum value. The actual throughput may be lower, but it'll always
+ * be sufficient to process the input pixel rate fed into a single DSC engine.
+ */
+ dsc_enc_caps->max_total_throughput_mps = DCN20_MAX_DISPLAY_CLOCK_Mhz;
+
+ /* For pixel clock bigger than a single-pipe limit we'll need two engines, which then doubles our
+ * throughput and number of slices, but also introduces a lower limit of 2 slices
+ */
+ if (pixel_clock_100Hz >= DCN20_MAX_PIXEL_CLOCK_Mhz*10000) {
+ dsc_enc_caps->slice_caps.bits.NUM_SLICES_1 = 0;
+ dsc_enc_caps->slice_caps.bits.NUM_SLICES_8 = 1;
+ dsc_enc_caps->max_total_throughput_mps = DCN20_MAX_DISPLAY_CLOCK_Mhz * 2;
+ }
+
+ // TODO DSC: This is actually image width limitation, not a slice width. This should be added to the criteria to use ODM.
+ dsc_enc_caps->max_slice_width = 5184; /* (including 64 overlap pixels for eDP MSO mode) */
+ dsc_enc_caps->bpp_increment_div = 16; /* 1/16th of a bit */
+}
+
+
+/* this function read dsc related register fields to be logged later in dcn10_log_hw_state
+ * into a dcn_dsc_state struct.
+ */
+static void dsc2_read_state(struct display_stream_compressor *dsc, struct dcn_dsc_state *s)
+{
+ struct dcn20_dsc *dsc20 = TO_DCN20_DSC(dsc);
+
+ REG_GET(DSC_TOP_CONTROL, DSC_CLOCK_EN, &s->dsc_clock_en);
+ REG_GET(DSCC_PPS_CONFIG3, SLICE_WIDTH, &s->dsc_slice_width);
+ REG_GET(DSCC_PPS_CONFIG1, BITS_PER_PIXEL, &s->dsc_bytes_per_pixel);
+}
+
+
+static bool dsc2_validate_stream(struct display_stream_compressor *dsc, const struct dsc_config *dsc_cfg)
+{
+ struct dsc_optc_config dsc_optc_cfg;
+
+ if (dsc_cfg->pic_width > TO_DCN20_DSC(dsc)->max_image_width)
+ return false;
+
+ return dsc_prepare_config(dsc, dsc_cfg, &dsc_optc_cfg);
+}
+
+
+static void dsc_config_log(struct display_stream_compressor *dsc,
+ const struct dsc_config *config)
+{
+ DC_LOG_DSC("Setting DSC Config at DSC inst %d", dsc->inst);
+ DC_LOG_DSC("\n\tnum_slices_h %d\n\tnum_slices_v %d\n\tbits_per_pixel %d\n\tcolor_depth %d",
+ config->dc_dsc_cfg.num_slices_h,
+ config->dc_dsc_cfg.num_slices_v,
+ config->dc_dsc_cfg.bits_per_pixel,
+ config->color_depth);
+}
+
+static void dsc2_set_config(struct display_stream_compressor *dsc, const struct dsc_config *dsc_cfg,
+ struct dsc_optc_config *dsc_optc_cfg, uint8_t *dsc_packed_pps)
+{
+ bool is_config_ok;
+ struct dcn20_dsc *dsc20 = TO_DCN20_DSC(dsc);
+
+ dsc_config_log(dsc, dsc_cfg);
+ is_config_ok = dsc_prepare_config(dsc, dsc_cfg, dsc_optc_cfg);
+ ASSERT(is_config_ok);
+ drm_dsc_pps_payload_pack((struct drm_dsc_picture_parameter_set *)dsc_packed_pps, &dsc20->reg_vals.pps);
+ dsc_log_pps(dsc, &dsc20->reg_vals.pps);
+ dsc_write_to_registers(dsc, &dsc20->reg_vals);
+}
+
+
+static void dsc2_enable(struct display_stream_compressor *dsc, int opp_pipe)
+{
+ struct dcn20_dsc *dsc20 = TO_DCN20_DSC(dsc);
+
+ /* TODO Check if DSC alreay in use? */
+ DC_LOG_DSC("enable DSC at opp pipe %d", opp_pipe);
+
+ REG_UPDATE(DSC_TOP_CONTROL,
+ DSC_CLOCK_EN, 1);
+
+ REG_UPDATE_2(DSCRM_DSC_FORWARD_CONFIG,
+ DSCRM_DSC_FORWARD_EN, 1,
+ DSCRM_DSC_OPP_PIPE_SOURCE, opp_pipe);
+}
+
+
+static void dsc2_disable(struct display_stream_compressor *dsc)
+{
+ struct dcn20_dsc *dsc20 = TO_DCN20_DSC(dsc);
+
+ DC_LOG_DSC("disable DSC");
+
+ REG_UPDATE(DSCRM_DSC_FORWARD_CONFIG,
+ DSCRM_DSC_FORWARD_EN, 0);
+
+ REG_UPDATE(DSC_TOP_CONTROL,
+ DSC_CLOCK_EN, 0);
+}
+
+
+/* This module's internal functions */
+static void dsc_log_pps(struct display_stream_compressor *dsc, struct drm_dsc_config *pps)
+{
+ int i;
+ int bits_per_pixel = pps->bits_per_pixel;
+
+ DC_LOG_DSC("programming DSC Picture Parameter Set (PPS):");
+ DC_LOG_DSC("\tdsc_version_major %d", pps->dsc_version_major);
+ DC_LOG_DSC("\tdsc_version_minor %d", pps->dsc_version_minor);
+ DC_LOG_DSC("\tbits_per_component %d", pps->bits_per_component);
+ DC_LOG_DSC("\tline_buf_depth %d", pps->line_buf_depth);
+ DC_LOG_DSC("\tblock_pred_enable %d", pps->block_pred_enable);
+ DC_LOG_DSC("\tconvert_rgb %d", pps->convert_rgb);
+ DC_LOG_DSC("\tsimple_422 %d", pps->simple_422);
+ DC_LOG_DSC("\tvbr_enable %d", pps->vbr_enable);
+ DC_LOG_DSC("\tbits_per_pixel %d (%d.%04d)", bits_per_pixel, bits_per_pixel / 16, ((bits_per_pixel % 16) * 10000) / 16);
+ DC_LOG_DSC("\tpic_height %d", pps->pic_height);
+ DC_LOG_DSC("\tpic_width %d", pps->pic_width);
+ DC_LOG_DSC("\tslice_height %d", pps->slice_height);
+ DC_LOG_DSC("\tslice_width %d", pps->slice_width);
+ DC_LOG_DSC("\tslice_chunk_size %d", pps->slice_chunk_size);
+ DC_LOG_DSC("\tinitial_xmit_delay %d", pps->initial_xmit_delay);
+ DC_LOG_DSC("\tinitial_dec_delay %d", pps->initial_dec_delay);
+ DC_LOG_DSC("\tinitial_scale_value %d", pps->initial_scale_value);
+ DC_LOG_DSC("\tscale_increment_interval %d", pps->scale_increment_interval);
+ DC_LOG_DSC("\tscale_decrement_interval %d", pps->scale_decrement_interval);
+ DC_LOG_DSC("\tfirst_line_bpg_offset %d", pps->first_line_bpg_offset);
+ DC_LOG_DSC("\tnfl_bpg_offset %d", pps->nfl_bpg_offset);
+ DC_LOG_DSC("\tslice_bpg_offset %d", pps->slice_bpg_offset);
+ DC_LOG_DSC("\tinitial_offset %d", pps->initial_offset);
+ DC_LOG_DSC("\tfinal_offset %d", pps->final_offset);
+ DC_LOG_DSC("\tflatness_min_qp %d", pps->flatness_min_qp);
+ DC_LOG_DSC("\tflatness_max_qp %d", pps->flatness_max_qp);
+ /* DC_LOG_DSC("\trc_parameter_set %d", pps->rc_parameter_set); */
+ DC_LOG_DSC("\tnative_420 %d", pps->native_420);
+ DC_LOG_DSC("\tnative_422 %d", pps->native_422);
+ DC_LOG_DSC("\tsecond_line_bpg_offset %d", pps->second_line_bpg_offset);
+ DC_LOG_DSC("\tnsl_bpg_offset %d", pps->nsl_bpg_offset);
+ DC_LOG_DSC("\tsecond_line_offset_adj %d", pps->second_line_offset_adj);
+ DC_LOG_DSC("\trc_model_size %d", pps->rc_model_size);
+ DC_LOG_DSC("\trc_edge_factor %d", pps->rc_edge_factor);
+ DC_LOG_DSC("\trc_quant_incr_limit0 %d", pps->rc_quant_incr_limit0);
+ DC_LOG_DSC("\trc_quant_incr_limit1 %d", pps->rc_quant_incr_limit1);
+ DC_LOG_DSC("\trc_tgt_offset_high %d", pps->rc_tgt_offset_high);
+ DC_LOG_DSC("\trc_tgt_offset_low %d", pps->rc_tgt_offset_low);
+
+ for (i = 0; i < NUM_BUF_RANGES - 1; i++)
+ DC_LOG_DSC("\trc_buf_thresh[%d] %d", i, pps->rc_buf_thresh[i]);
+
+ for (i = 0; i < NUM_BUF_RANGES; i++) {
+ DC_LOG_DSC("\trc_range_parameters[%d].range_min_qp %d", i, pps->rc_range_params[i].range_min_qp);
+ DC_LOG_DSC("\trc_range_parameters[%d].range_max_qp %d", i, pps->rc_range_params[i].range_max_qp);
+ DC_LOG_DSC("\trc_range_parameters[%d].range_bpg_offset %d", i, pps->rc_range_params[i].range_bpg_offset);
+ }
+}
+
+static bool dsc_prepare_config(struct display_stream_compressor *dsc, const struct dsc_config *dsc_cfg,
+ struct dsc_optc_config *dsc_optc_cfg)
+{
+ struct dsc_parameters dsc_params;
+
+ struct dcn20_dsc *dsc20 = TO_DCN20_DSC(dsc);
+
+ /* Validate input parameters */
+ ASSERT(dsc_cfg->dc_dsc_cfg.num_slices_h);
+ ASSERT(dsc_cfg->dc_dsc_cfg.num_slices_v);
+ ASSERT(dsc_cfg->dc_dsc_cfg.version_minor == 1 || dsc_cfg->dc_dsc_cfg.version_minor == 2);
+ ASSERT(dsc_cfg->pic_width);
+ ASSERT(dsc_cfg->pic_height);
+ ASSERT((dsc_cfg->dc_dsc_cfg.version_minor == 1 &&
+ (8 <= dsc_cfg->dc_dsc_cfg.linebuf_depth && dsc_cfg->dc_dsc_cfg.linebuf_depth <= 13)) ||
+ (dsc_cfg->dc_dsc_cfg.version_minor == 2 &&
+ ((8 <= dsc_cfg->dc_dsc_cfg.linebuf_depth && dsc_cfg->dc_dsc_cfg.linebuf_depth <= 15) ||
+ dsc_cfg->dc_dsc_cfg.linebuf_depth == 0)));
+ ASSERT(96 <= dsc_cfg->dc_dsc_cfg.bits_per_pixel && dsc_cfg->dc_dsc_cfg.bits_per_pixel <= 0x3ff); // 6.0 <= bits_per_pixel <= 63.9375
+
+ if (!dsc_cfg->dc_dsc_cfg.num_slices_v || !dsc_cfg->dc_dsc_cfg.num_slices_v ||
+ !(dsc_cfg->dc_dsc_cfg.version_minor == 1 || dsc_cfg->dc_dsc_cfg.version_minor == 2) ||
+ !dsc_cfg->pic_width || !dsc_cfg->pic_height ||
+ !((dsc_cfg->dc_dsc_cfg.version_minor == 1 && // v1.1 line buffer depth range:
+ 8 <= dsc_cfg->dc_dsc_cfg.linebuf_depth && dsc_cfg->dc_dsc_cfg.linebuf_depth <= 13) ||
+ (dsc_cfg->dc_dsc_cfg.version_minor == 2 && // v1.2 line buffer depth range:
+ ((8 <= dsc_cfg->dc_dsc_cfg.linebuf_depth && dsc_cfg->dc_dsc_cfg.linebuf_depth <= 15) ||
+ dsc_cfg->dc_dsc_cfg.linebuf_depth == 0))) ||
+ !(96 <= dsc_cfg->dc_dsc_cfg.bits_per_pixel && dsc_cfg->dc_dsc_cfg.bits_per_pixel <= 0x3ff)) {
+ dm_output_to_console("%s: Invalid parameters\n", __func__);
+ return false;
+ }
+
+ dsc_init_reg_values(&dsc20->reg_vals);
+
+ /* Copy input config */
+ dsc20->reg_vals.pixel_format = dsc_dc_pixel_encoding_to_dsc_pixel_format(dsc_cfg->pixel_encoding, dsc_cfg->dc_dsc_cfg.ycbcr422_simple);
+ dsc20->reg_vals.num_slices_h = dsc_cfg->dc_dsc_cfg.num_slices_h;
+ dsc20->reg_vals.num_slices_v = dsc_cfg->dc_dsc_cfg.num_slices_v;
+ dsc20->reg_vals.pps.dsc_version_minor = dsc_cfg->dc_dsc_cfg.version_minor;
+ dsc20->reg_vals.pps.pic_width = dsc_cfg->pic_width;
+ dsc20->reg_vals.pps.pic_height = dsc_cfg->pic_height;
+ dsc20->reg_vals.pps.bits_per_component = dsc_dc_color_depth_to_dsc_bits_per_comp(dsc_cfg->color_depth);
+ dsc20->reg_vals.pps.block_pred_enable = dsc_cfg->dc_dsc_cfg.block_pred_enable;
+ dsc20->reg_vals.pps.line_buf_depth = dsc_cfg->dc_dsc_cfg.linebuf_depth;
+ dsc20->reg_vals.alternate_ich_encoding_en = dsc20->reg_vals.pps.dsc_version_minor == 1 ? 0 : 1;
+
+ // TODO: in addition to validating slice height (pic height must be divisible by slice height),
+ // see what happens when the same condition doesn't apply for slice_width/pic_width.
+ dsc20->reg_vals.pps.slice_width = dsc_cfg->pic_width / dsc_cfg->dc_dsc_cfg.num_slices_h;
+ dsc20->reg_vals.pps.slice_height = dsc_cfg->pic_height / dsc_cfg->dc_dsc_cfg.num_slices_v;
+
+ ASSERT(dsc20->reg_vals.pps.slice_height * dsc_cfg->dc_dsc_cfg.num_slices_v == dsc_cfg->pic_height);
+ if (!(dsc20->reg_vals.pps.slice_height * dsc_cfg->dc_dsc_cfg.num_slices_v == dsc_cfg->pic_height)) {
+ dm_output_to_console("%s: pix height %d not divisible by num_slices_v %d\n\n", __func__, dsc_cfg->pic_height, dsc_cfg->dc_dsc_cfg.num_slices_v);
+ return false;
+ }
+
+ dsc20->reg_vals.bpp_x32 = dsc_cfg->dc_dsc_cfg.bits_per_pixel << 1;
+ if (dsc20->reg_vals.pixel_format == DSC_PIXFMT_NATIVE_YCBCR420 || dsc20->reg_vals.pixel_format == DSC_PIXFMT_NATIVE_YCBCR422)
+ dsc20->reg_vals.pps.bits_per_pixel = dsc20->reg_vals.bpp_x32;
+ else
+ dsc20->reg_vals.pps.bits_per_pixel = dsc20->reg_vals.bpp_x32 >> 1;
+
+ dsc20->reg_vals.pps.convert_rgb = dsc20->reg_vals.pixel_format == DSC_PIXFMT_RGB ? 1 : 0;
+ dsc20->reg_vals.pps.native_422 = (dsc20->reg_vals.pixel_format == DSC_PIXFMT_NATIVE_YCBCR422);
+ dsc20->reg_vals.pps.native_420 = (dsc20->reg_vals.pixel_format == DSC_PIXFMT_NATIVE_YCBCR420);
+ dsc20->reg_vals.pps.simple_422 = (dsc20->reg_vals.pixel_format == DSC_PIXFMT_SIMPLE_YCBCR422);
+
+ if (dscc_compute_dsc_parameters(&dsc20->reg_vals.pps, &dsc_params)) {
+ dm_output_to_console("%s: DSC config failed\n", __func__);
+ return false;
+ }
+
+ dsc_update_from_dsc_parameters(&dsc20->reg_vals, &dsc_params);
+
+ dsc_optc_cfg->bytes_per_pixel = dsc_params.bytes_per_pixel;
+ dsc_optc_cfg->slice_width = dsc20->reg_vals.pps.slice_width;
+ dsc_optc_cfg->is_pixel_format_444 = dsc20->reg_vals.pixel_format == DSC_PIXFMT_RGB ||
+ dsc20->reg_vals.pixel_format == DSC_PIXFMT_YCBCR444 ||
+ dsc20->reg_vals.pixel_format == DSC_PIXFMT_SIMPLE_YCBCR422;
+
+ return true;
+}
+
+
+static enum dsc_pixel_format dsc_dc_pixel_encoding_to_dsc_pixel_format(enum dc_pixel_encoding dc_pix_enc, bool is_ycbcr422_simple)
+{
+ enum dsc_pixel_format dsc_pix_fmt = DSC_PIXFMT_UNKNOWN;
+
+ /* NOTE: We don't support DSC_PIXFMT_SIMPLE_YCBCR422 */
+
+ switch (dc_pix_enc) {
+ case PIXEL_ENCODING_RGB:
+ dsc_pix_fmt = DSC_PIXFMT_RGB;
+ break;
+ case PIXEL_ENCODING_YCBCR422:
+ if (is_ycbcr422_simple)
+ dsc_pix_fmt = DSC_PIXFMT_SIMPLE_YCBCR422;
+ else
+ dsc_pix_fmt = DSC_PIXFMT_NATIVE_YCBCR422;
+ break;
+ case PIXEL_ENCODING_YCBCR444:
+ dsc_pix_fmt = DSC_PIXFMT_YCBCR444;
+ break;
+ case PIXEL_ENCODING_YCBCR420:
+ dsc_pix_fmt = DSC_PIXFMT_NATIVE_YCBCR420;
+ break;
+ default:
+ dsc_pix_fmt = DSC_PIXFMT_UNKNOWN;
+ break;
+ }
+
+ ASSERT(dsc_pix_fmt != DSC_PIXFMT_UNKNOWN);
+ return dsc_pix_fmt;
+}
+
+
+static enum dsc_bits_per_comp dsc_dc_color_depth_to_dsc_bits_per_comp(enum dc_color_depth dc_color_depth)
+{
+ enum dsc_bits_per_comp bpc = DSC_BPC_UNKNOWN;
+
+ switch (dc_color_depth) {
+ case COLOR_DEPTH_888:
+ bpc = DSC_BPC_8;
+ break;
+ case COLOR_DEPTH_101010:
+ bpc = DSC_BPC_10;
+ break;
+ case COLOR_DEPTH_121212:
+ bpc = DSC_BPC_12;
+ break;
+ default:
+ bpc = DSC_BPC_UNKNOWN;
+ break;
+ }
+
+ return bpc;
+}
+
+
+static void dsc_init_reg_values(struct dsc_reg_values *reg_vals)
+{
+ int i;
+
+ /* Non-PPS values */
+ reg_vals->dsc_clock_enable = 1;
+ reg_vals->dsc_clock_gating_disable = 0;
+ reg_vals->underflow_recovery_en = 0;
+ reg_vals->underflow_occurred_int_en = 0;
+ reg_vals->underflow_occurred_status = 0;
+ reg_vals->ich_reset_at_eol = 0;
+ reg_vals->alternate_ich_encoding_en = 0;
+ reg_vals->rc_buffer_model_size = 0;
+ reg_vals->disable_ich = 0;
+ reg_vals->dsc_dbg_en = 0;
+
+ for (i = 0; i < 4; i++)
+ reg_vals->rc_buffer_model_overflow_int_en[i] = 0;
+
+ /* PPS values */
+ reg_vals->pps.dsc_version_minor = 2;
+ reg_vals->pps.dsc_version_major = 1;
+ reg_vals->pps.line_buf_depth = 9;
+ reg_vals->pps.bits_per_component = 8;
+ reg_vals->pps.block_pred_enable = 1;
+ reg_vals->pps.slice_chunk_size = 0;
+ reg_vals->pps.pic_width = 0;
+ reg_vals->pps.pic_height = 0;
+ reg_vals->pps.slice_width = 0;
+ reg_vals->pps.slice_height = 0;
+ reg_vals->pps.initial_xmit_delay = 170;
+ reg_vals->pps.initial_dec_delay = 0;
+ reg_vals->pps.initial_scale_value = 0;
+ reg_vals->pps.scale_increment_interval = 0;
+ reg_vals->pps.scale_decrement_interval = 0;
+ reg_vals->pps.nfl_bpg_offset = 0;
+ reg_vals->pps.slice_bpg_offset = 0;
+ reg_vals->pps.nsl_bpg_offset = 0;
+ reg_vals->pps.initial_offset = 6144;
+ reg_vals->pps.final_offset = 0;
+ reg_vals->pps.flatness_min_qp = 3;
+ reg_vals->pps.flatness_max_qp = 12;
+ reg_vals->pps.rc_model_size = 8192;
+ reg_vals->pps.rc_edge_factor = 6;
+ reg_vals->pps.rc_quant_incr_limit0 = 11;
+ reg_vals->pps.rc_quant_incr_limit1 = 11;
+ reg_vals->pps.rc_tgt_offset_low = 3;
+ reg_vals->pps.rc_tgt_offset_high = 3;
+}
+
+/* Updates dsc_reg_values::reg_vals::xxx fields based on the values from computed params.
+ * This is required because dscc_compute_dsc_parameters returns a modified PPS, which in turn
+ * affects non-PPS register values.
+ */
+static void dsc_update_from_dsc_parameters(struct dsc_reg_values *reg_vals, const struct dsc_parameters *dsc_params)
+{
+ int i;
+
+ reg_vals->pps = dsc_params->pps;
+
+ // pps_computed will have the "expanded" values; need to shift them to make them fit for regs.
+ for (i = 0; i < NUM_BUF_RANGES - 1; i++)
+ reg_vals->pps.rc_buf_thresh[i] = reg_vals->pps.rc_buf_thresh[i] >> 6;
+
+ reg_vals->rc_buffer_model_size = dsc_params->rc_buffer_model_size;
+ reg_vals->ich_reset_at_eol = reg_vals->num_slices_h == 1 ? 0 : 0xf;
+}
+
+static void dsc_write_to_registers(struct display_stream_compressor *dsc, const struct dsc_reg_values *reg_vals)
+{
+ uint32_t temp_int;
+ struct dcn20_dsc *dsc20 = TO_DCN20_DSC(dsc);
+
+ REG_SET(DSC_DEBUG_CONTROL, 0,
+ DSC_DBG_EN, reg_vals->dsc_dbg_en);
+
+ // dsccif registers
+ REG_SET_5(DSCCIF_CONFIG0, 0,
+ INPUT_INTERFACE_UNDERFLOW_RECOVERY_EN, reg_vals->underflow_recovery_en,
+ INPUT_INTERFACE_UNDERFLOW_OCCURRED_INT_EN, reg_vals->underflow_occurred_int_en,
+ INPUT_INTERFACE_UNDERFLOW_OCCURRED_STATUS, reg_vals->underflow_occurred_status,
+ INPUT_PIXEL_FORMAT, reg_vals->pixel_format,
+ DSCCIF_CONFIG0__BITS_PER_COMPONENT, reg_vals->pps.bits_per_component);
+
+ REG_SET_2(DSCCIF_CONFIG1, 0,
+ PIC_WIDTH, reg_vals->pps.pic_width,
+ PIC_HEIGHT, reg_vals->pps.pic_height);
+
+ // dscc registers
+ REG_SET_4(DSCC_CONFIG0, 0,
+ ICH_RESET_AT_END_OF_LINE, reg_vals->ich_reset_at_eol,
+ NUMBER_OF_SLICES_PER_LINE, reg_vals->num_slices_h - 1,
+ ALTERNATE_ICH_ENCODING_EN, reg_vals->alternate_ich_encoding_en,
+ NUMBER_OF_SLICES_IN_VERTICAL_DIRECTION, reg_vals->num_slices_v - 1);
+
+ REG_SET_2(DSCC_CONFIG1, 0,
+ DSCC_RATE_CONTROL_BUFFER_MODEL_SIZE, reg_vals->rc_buffer_model_size,
+ DSCC_DISABLE_ICH, reg_vals->disable_ich);
+
+ REG_SET_4(DSCC_INTERRUPT_CONTROL_STATUS, 0,
+ DSCC_RATE_CONTROL_BUFFER_MODEL0_OVERFLOW_OCCURRED_INT_EN, reg_vals->rc_buffer_model_overflow_int_en[0],
+ DSCC_RATE_CONTROL_BUFFER_MODEL1_OVERFLOW_OCCURRED_INT_EN, reg_vals->rc_buffer_model_overflow_int_en[1],
+ DSCC_RATE_CONTROL_BUFFER_MODEL2_OVERFLOW_OCCURRED_INT_EN, reg_vals->rc_buffer_model_overflow_int_en[2],
+ DSCC_RATE_CONTROL_BUFFER_MODEL3_OVERFLOW_OCCURRED_INT_EN, reg_vals->rc_buffer_model_overflow_int_en[3]);
+
+ REG_SET_3(DSCC_PPS_CONFIG0, 0,
+ DSC_VERSION_MINOR, reg_vals->pps.dsc_version_minor,
+ LINEBUF_DEPTH, reg_vals->pps.line_buf_depth,
+ DSCC_PPS_CONFIG0__BITS_PER_COMPONENT, reg_vals->pps.bits_per_component);
+
+ if (reg_vals->pixel_format == DSC_PIXFMT_NATIVE_YCBCR420 || reg_vals->pixel_format == DSC_PIXFMT_NATIVE_YCBCR422)
+ temp_int = reg_vals->bpp_x32;
+ else
+ temp_int = reg_vals->bpp_x32 >> 1;
+
+ REG_SET_7(DSCC_PPS_CONFIG1, 0,
+ BITS_PER_PIXEL, temp_int,
+ SIMPLE_422, reg_vals->pixel_format == DSC_PIXFMT_SIMPLE_YCBCR422,
+ CONVERT_RGB, reg_vals->pixel_format == DSC_PIXFMT_RGB,
+ BLOCK_PRED_ENABLE, reg_vals->pps.block_pred_enable,
+ NATIVE_422, reg_vals->pixel_format == DSC_PIXFMT_NATIVE_YCBCR422,
+ NATIVE_420, reg_vals->pixel_format == DSC_PIXFMT_NATIVE_YCBCR420,
+ CHUNK_SIZE, reg_vals->pps.slice_chunk_size);
+
+ REG_SET_2(DSCC_PPS_CONFIG2, 0,
+ PIC_WIDTH, reg_vals->pps.pic_width,
+ PIC_HEIGHT, reg_vals->pps.pic_height);
+
+ REG_SET_2(DSCC_PPS_CONFIG3, 0,
+ SLICE_WIDTH, reg_vals->pps.slice_width,
+ SLICE_HEIGHT, reg_vals->pps.slice_height);
+
+ REG_SET(DSCC_PPS_CONFIG4, 0,
+ INITIAL_XMIT_DELAY, reg_vals->pps.initial_xmit_delay);
+
+ REG_SET_2(DSCC_PPS_CONFIG5, 0,
+ INITIAL_SCALE_VALUE, reg_vals->pps.initial_scale_value,
+ SCALE_INCREMENT_INTERVAL, reg_vals->pps.scale_increment_interval);
+
+ REG_SET_3(DSCC_PPS_CONFIG6, 0,
+ SCALE_DECREMENT_INTERVAL, reg_vals->pps.scale_decrement_interval,
+ FIRST_LINE_BPG_OFFSET, reg_vals->pps.first_line_bpg_offset,
+ SECOND_LINE_BPG_OFFSET, reg_vals->pps.second_line_bpg_offset);
+
+ REG_SET_2(DSCC_PPS_CONFIG7, 0,
+ NFL_BPG_OFFSET, reg_vals->pps.nfl_bpg_offset,
+ SLICE_BPG_OFFSET, reg_vals->pps.slice_bpg_offset);
+
+ REG_SET_2(DSCC_PPS_CONFIG8, 0,
+ NSL_BPG_OFFSET, reg_vals->pps.nsl_bpg_offset,
+ SECOND_LINE_OFFSET_ADJ, reg_vals->pps.second_line_offset_adj);
+
+ REG_SET_2(DSCC_PPS_CONFIG9, 0,
+ INITIAL_OFFSET, reg_vals->pps.initial_offset,
+ FINAL_OFFSET, reg_vals->pps.final_offset);
+
+ REG_SET_3(DSCC_PPS_CONFIG10, 0,
+ FLATNESS_MIN_QP, reg_vals->pps.flatness_min_qp,
+ FLATNESS_MAX_QP, reg_vals->pps.flatness_max_qp,
+ RC_MODEL_SIZE, reg_vals->pps.rc_model_size);
+
+ REG_SET_5(DSCC_PPS_CONFIG11, 0,
+ RC_EDGE_FACTOR, reg_vals->pps.rc_edge_factor,
+ RC_QUANT_INCR_LIMIT0, reg_vals->pps.rc_quant_incr_limit0,
+ RC_QUANT_INCR_LIMIT1, reg_vals->pps.rc_quant_incr_limit1,
+ RC_TGT_OFFSET_LO, reg_vals->pps.rc_tgt_offset_low,
+ RC_TGT_OFFSET_HI, reg_vals->pps.rc_tgt_offset_high);
+
+ REG_SET_4(DSCC_PPS_CONFIG12, 0,
+ RC_BUF_THRESH0, reg_vals->pps.rc_buf_thresh[0],
+ RC_BUF_THRESH1, reg_vals->pps.rc_buf_thresh[1],
+ RC_BUF_THRESH2, reg_vals->pps.rc_buf_thresh[2],
+ RC_BUF_THRESH3, reg_vals->pps.rc_buf_thresh[3]);
+
+ REG_SET_4(DSCC_PPS_CONFIG13, 0,
+ RC_BUF_THRESH4, reg_vals->pps.rc_buf_thresh[4],
+ RC_BUF_THRESH5, reg_vals->pps.rc_buf_thresh[5],
+ RC_BUF_THRESH6, reg_vals->pps.rc_buf_thresh[6],
+ RC_BUF_THRESH7, reg_vals->pps.rc_buf_thresh[7]);
+
+ REG_SET_4(DSCC_PPS_CONFIG14, 0,
+ RC_BUF_THRESH8, reg_vals->pps.rc_buf_thresh[8],
+ RC_BUF_THRESH9, reg_vals->pps.rc_buf_thresh[9],
+ RC_BUF_THRESH10, reg_vals->pps.rc_buf_thresh[10],
+ RC_BUF_THRESH11, reg_vals->pps.rc_buf_thresh[11]);
+
+ REG_SET_5(DSCC_PPS_CONFIG15, 0,
+ RC_BUF_THRESH12, reg_vals->pps.rc_buf_thresh[12],
+ RC_BUF_THRESH13, reg_vals->pps.rc_buf_thresh[13],
+ RANGE_MIN_QP0, reg_vals->pps.rc_range_params[0].range_min_qp,
+ RANGE_MAX_QP0, reg_vals->pps.rc_range_params[0].range_max_qp,
+ RANGE_BPG_OFFSET0, reg_vals->pps.rc_range_params[0].range_bpg_offset);
+
+ REG_SET_6(DSCC_PPS_CONFIG16, 0,
+ RANGE_MIN_QP1, reg_vals->pps.rc_range_params[1].range_min_qp,
+ RANGE_MAX_QP1, reg_vals->pps.rc_range_params[1].range_max_qp,
+ RANGE_BPG_OFFSET1, reg_vals->pps.rc_range_params[1].range_bpg_offset,
+ RANGE_MIN_QP2, reg_vals->pps.rc_range_params[2].range_min_qp,
+ RANGE_MAX_QP2, reg_vals->pps.rc_range_params[2].range_max_qp,
+ RANGE_BPG_OFFSET2, reg_vals->pps.rc_range_params[2].range_bpg_offset);
+
+ REG_SET_6(DSCC_PPS_CONFIG17, 0,
+ RANGE_MIN_QP3, reg_vals->pps.rc_range_params[3].range_min_qp,
+ RANGE_MAX_QP3, reg_vals->pps.rc_range_params[3].range_max_qp,
+ RANGE_BPG_OFFSET3, reg_vals->pps.rc_range_params[3].range_bpg_offset,
+ RANGE_MIN_QP4, reg_vals->pps.rc_range_params[4].range_min_qp,
+ RANGE_MAX_QP4, reg_vals->pps.rc_range_params[4].range_max_qp,
+ RANGE_BPG_OFFSET4, reg_vals->pps.rc_range_params[4].range_bpg_offset);
+
+ REG_SET_6(DSCC_PPS_CONFIG18, 0,
+ RANGE_MIN_QP5, reg_vals->pps.rc_range_params[5].range_min_qp,
+ RANGE_MAX_QP5, reg_vals->pps.rc_range_params[5].range_max_qp,
+ RANGE_BPG_OFFSET5, reg_vals->pps.rc_range_params[5].range_bpg_offset,
+ RANGE_MIN_QP6, reg_vals->pps.rc_range_params[6].range_min_qp,
+ RANGE_MAX_QP6, reg_vals->pps.rc_range_params[6].range_max_qp,
+ RANGE_BPG_OFFSET6, reg_vals->pps.rc_range_params[6].range_bpg_offset);
+
+ REG_SET_6(DSCC_PPS_CONFIG19, 0,
+ RANGE_MIN_QP7, reg_vals->pps.rc_range_params[7].range_min_qp,
+ RANGE_MAX_QP7, reg_vals->pps.rc_range_params[7].range_max_qp,
+ RANGE_BPG_OFFSET7, reg_vals->pps.rc_range_params[7].range_bpg_offset,
+ RANGE_MIN_QP8, reg_vals->pps.rc_range_params[8].range_min_qp,
+ RANGE_MAX_QP8, reg_vals->pps.rc_range_params[8].range_max_qp,
+ RANGE_BPG_OFFSET8, reg_vals->pps.rc_range_params[8].range_bpg_offset);
+
+ REG_SET_6(DSCC_PPS_CONFIG20, 0,
+ RANGE_MIN_QP9, reg_vals->pps.rc_range_params[9].range_min_qp,
+ RANGE_MAX_QP9, reg_vals->pps.rc_range_params[9].range_max_qp,
+ RANGE_BPG_OFFSET9, reg_vals->pps.rc_range_params[9].range_bpg_offset,
+ RANGE_MIN_QP10, reg_vals->pps.rc_range_params[10].range_min_qp,
+ RANGE_MAX_QP10, reg_vals->pps.rc_range_params[10].range_max_qp,
+ RANGE_BPG_OFFSET10, reg_vals->pps.rc_range_params[10].range_bpg_offset);
+
+ REG_SET_6(DSCC_PPS_CONFIG21, 0,
+ RANGE_MIN_QP11, reg_vals->pps.rc_range_params[11].range_min_qp,
+ RANGE_MAX_QP11, reg_vals->pps.rc_range_params[11].range_max_qp,
+ RANGE_BPG_OFFSET11, reg_vals->pps.rc_range_params[11].range_bpg_offset,
+ RANGE_MIN_QP12, reg_vals->pps.rc_range_params[12].range_min_qp,
+ RANGE_MAX_QP12, reg_vals->pps.rc_range_params[12].range_max_qp,
+ RANGE_BPG_OFFSET12, reg_vals->pps.rc_range_params[12].range_bpg_offset);
+
+ REG_SET_6(DSCC_PPS_CONFIG22, 0,
+ RANGE_MIN_QP13, reg_vals->pps.rc_range_params[13].range_min_qp,
+ RANGE_MAX_QP13, reg_vals->pps.rc_range_params[13].range_max_qp,
+ RANGE_BPG_OFFSET13, reg_vals->pps.rc_range_params[13].range_bpg_offset,
+ RANGE_MIN_QP14, reg_vals->pps.rc_range_params[14].range_min_qp,
+ RANGE_MAX_QP14, reg_vals->pps.rc_range_params[14].range_max_qp,
+ RANGE_BPG_OFFSET14, reg_vals->pps.rc_range_params[14].range_bpg_offset);
+
+ if (IS_FPGA_MAXIMUS_DC(dsc20->base.ctx->dce_environment)) {
+ /* It's safe to do this as long as debug bus is not being used in DAL Diag environment.
+ *
+ * This is because DSCC_PPS_CONFIG4.INITIAL_DEC_DELAY is a read-only register field (because it's a decoder
+ * value not required by DSC encoder). However, since decoding fails when this value is missing from PPS, it's
+ * required to communicate this value to the PPS header. When testing on FPGA, the values for PPS header are
+ * being read from Diag register dump. The register below is used in place of a scratch register to make
+ * 'initial_dec_delay' available.
+ */
+
+ temp_int = reg_vals->pps.initial_dec_delay;
+ REG_SET_4(DSCC_TEST_DEBUG_BUS_ROTATE, 0,
+ DSCC_TEST_DEBUG_BUS0_ROTATE, temp_int & 0x1f,
+ DSCC_TEST_DEBUG_BUS1_ROTATE, temp_int >> 5 & 0x1f,
+ DSCC_TEST_DEBUG_BUS2_ROTATE, temp_int >> 10 & 0x1f,
+ DSCC_TEST_DEBUG_BUS3_ROTATE, temp_int >> 15 & 0x1);
+ }
+}
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.h
new file mode 100644
index 000000000000..168865a16288
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.h
@@ -0,0 +1,575 @@
+/* Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+#ifndef __DCN20_DSC_H__
+#define __DCN20_DSC_H__
+
+#include "dsc.h"
+#include "dsc/dscc_types.h"
+#include <drm/drm_dsc.h>
+
+#define TO_DCN20_DSC(dsc)\
+ container_of(dsc, struct dcn20_dsc, base)
+
+#define DSC_REG_LIST_DCN20(id) \
+ SRI(DSC_TOP_CONTROL, DSC_TOP, id),\
+ SRI(DSC_DEBUG_CONTROL, DSC_TOP, id),\
+ SRI(DSCC_CONFIG0, DSCC, id),\
+ SRI(DSCC_CONFIG1, DSCC, id),\
+ SRI(DSCC_STATUS, DSCC, id),\
+ SRI(DSCC_INTERRUPT_CONTROL_STATUS, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG0, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG1, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG2, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG3, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG4, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG5, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG6, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG7, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG8, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG9, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG10, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG11, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG12, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG13, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG14, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG15, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG16, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG17, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG18, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG19, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG20, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG21, DSCC, id),\
+ SRI(DSCC_PPS_CONFIG22, DSCC, id),\
+ SRI(DSCC_MEM_POWER_CONTROL, DSCC, id),\
+ SRI(DSCC_R_Y_SQUARED_ERROR_LOWER, DSCC, id),\
+ SRI(DSCC_R_Y_SQUARED_ERROR_UPPER, DSCC, id),\
+ SRI(DSCC_G_CB_SQUARED_ERROR_LOWER, DSCC, id),\
+ SRI(DSCC_G_CB_SQUARED_ERROR_UPPER, DSCC, id),\
+ SRI(DSCC_B_CR_SQUARED_ERROR_LOWER, DSCC, id),\
+ SRI(DSCC_B_CR_SQUARED_ERROR_UPPER, DSCC, id),\
+ SRI(DSCC_MAX_ABS_ERROR0, DSCC, id),\
+ SRI(DSCC_MAX_ABS_ERROR1, DSCC, id),\
+ SRI(DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL, DSCC, id),\
+ SRI(DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL, DSCC, id),\
+ SRI(DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL, DSCC, id),\
+ SRI(DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL, DSCC, id),\
+ SRI(DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL, DSCC, id),\
+ SRI(DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL, DSCC, id),\
+ SRI(DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL, DSCC, id),\
+ SRI(DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL, DSCC, id),\
+ SRI(DSCC_TEST_DEBUG_BUS_ROTATE, DSCC, id),\
+ SRI(DSCCIF_CONFIG0, DSCCIF, id),\
+ SRI(DSCCIF_CONFIG1, DSCCIF, id),\
+ SRI(DSCRM_DSC_FORWARD_CONFIG, DSCRM, id)
+
+
+#define DSC_SF(reg_name, field_name, post_fix)\
+ .field_name = reg_name ## __ ## field_name ## post_fix
+
+//Used in resolving the corner case with duplicate field name
+#define DSC2_SF(reg_name, field_name, post_fix)\
+ .field_name = reg_name ## _ ## field_name ## post_fix
+
+#define DSC_REG_LIST_SH_MASK_DCN20(mask_sh)\
+ DSC_SF(DSC_TOP0_DSC_TOP_CONTROL, DSC_CLOCK_EN, mask_sh), \
+ DSC_SF(DSC_TOP0_DSC_TOP_CONTROL, DSC_DISPCLK_R_GATE_DIS, mask_sh), \
+ DSC_SF(DSC_TOP0_DSC_TOP_CONTROL, DSC_DSCCLK_R_GATE_DIS, mask_sh), \
+ DSC_SF(DSC_TOP0_DSC_DEBUG_CONTROL, DSC_DBG_EN, mask_sh), \
+ DSC_SF(DSC_TOP0_DSC_DEBUG_CONTROL, DSC_TEST_CLOCK_MUX_SEL, mask_sh), \
+ DSC_SF(DSCC0_DSCC_CONFIG0, ICH_RESET_AT_END_OF_LINE, mask_sh), \
+ DSC_SF(DSCC0_DSCC_CONFIG0, NUMBER_OF_SLICES_PER_LINE, mask_sh), \
+ DSC_SF(DSCC0_DSCC_CONFIG0, ALTERNATE_ICH_ENCODING_EN, mask_sh), \
+ DSC_SF(DSCC0_DSCC_CONFIG0, NUMBER_OF_SLICES_IN_VERTICAL_DIRECTION, mask_sh), \
+ DSC_SF(DSCC0_DSCC_CONFIG1, DSCC_RATE_CONTROL_BUFFER_MODEL_SIZE, mask_sh), \
+ DSC_SF(DSCC0_DSCC_CONFIG1, DSCC_DISABLE_ICH, mask_sh), \
+ DSC_SF(DSCC0_DSCC_STATUS, DSCC_DOUBLE_BUFFER_REG_UPDATE_PENDING, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER0_OVERFLOW_OCCURRED, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER1_OVERFLOW_OCCURRED, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER2_OVERFLOW_OCCURRED, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER3_OVERFLOW_OCCURRED, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER0_UNDERFLOW_OCCURRED, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER1_UNDERFLOW_OCCURRED, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER2_UNDERFLOW_OCCURRED, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER3_UNDERFLOW_OCCURRED, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_CONTROL_BUFFER_MODEL0_OVERFLOW_OCCURRED, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_CONTROL_BUFFER_MODEL1_OVERFLOW_OCCURRED, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_CONTROL_BUFFER_MODEL2_OVERFLOW_OCCURRED, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_CONTROL_BUFFER_MODEL3_OVERFLOW_OCCURRED, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER0_OVERFLOW_OCCURRED_INT_EN, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER1_OVERFLOW_OCCURRED_INT_EN, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER2_OVERFLOW_OCCURRED_INT_EN, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER3_OVERFLOW_OCCURRED_INT_EN, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER0_UNDERFLOW_OCCURRED_INT_EN, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER1_UNDERFLOW_OCCURRED_INT_EN, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER2_UNDERFLOW_OCCURRED_INT_EN, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER3_UNDERFLOW_OCCURRED_INT_EN, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_CONTROL_BUFFER_MODEL0_OVERFLOW_OCCURRED_INT_EN, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_CONTROL_BUFFER_MODEL1_OVERFLOW_OCCURRED_INT_EN, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_CONTROL_BUFFER_MODEL2_OVERFLOW_OCCURRED_INT_EN, mask_sh), \
+ DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_CONTROL_BUFFER_MODEL3_OVERFLOW_OCCURRED_INT_EN, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG0, DSC_VERSION_MINOR, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG0, DSC_VERSION_MAJOR, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG0, PPS_IDENTIFIER, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG0, LINEBUF_DEPTH, mask_sh), \
+ DSC2_SF(DSCC0, DSCC_PPS_CONFIG0__BITS_PER_COMPONENT, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG1, BITS_PER_PIXEL, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG1, VBR_ENABLE, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG1, SIMPLE_422, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG1, CONVERT_RGB, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG1, BLOCK_PRED_ENABLE, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG1, NATIVE_422, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG1, NATIVE_420, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG1, CHUNK_SIZE, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG2, PIC_WIDTH, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG2, PIC_HEIGHT, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG3, SLICE_WIDTH, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG3, SLICE_HEIGHT, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG4, INITIAL_XMIT_DELAY, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG4, INITIAL_DEC_DELAY, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG5, INITIAL_SCALE_VALUE, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG5, SCALE_INCREMENT_INTERVAL, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG6, SCALE_DECREMENT_INTERVAL, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG6, FIRST_LINE_BPG_OFFSET, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG6, SECOND_LINE_BPG_OFFSET, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG7, NFL_BPG_OFFSET, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG7, SLICE_BPG_OFFSET, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG8, NSL_BPG_OFFSET, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG8, SECOND_LINE_OFFSET_ADJ, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG9, INITIAL_OFFSET, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG9, FINAL_OFFSET, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG10, FLATNESS_MIN_QP, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG10, FLATNESS_MAX_QP, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG10, RC_MODEL_SIZE, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG11, RC_EDGE_FACTOR, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG11, RC_QUANT_INCR_LIMIT0, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG11, RC_QUANT_INCR_LIMIT1, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG11, RC_TGT_OFFSET_LO, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG11, RC_TGT_OFFSET_HI, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG12, RC_BUF_THRESH0, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG12, RC_BUF_THRESH1, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG12, RC_BUF_THRESH2, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG12, RC_BUF_THRESH3, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG13, RC_BUF_THRESH4, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG13, RC_BUF_THRESH5, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG13, RC_BUF_THRESH6, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG13, RC_BUF_THRESH7, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG14, RC_BUF_THRESH8, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG14, RC_BUF_THRESH9, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG14, RC_BUF_THRESH10, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG14, RC_BUF_THRESH11, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG15, RC_BUF_THRESH12, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG15, RC_BUF_THRESH13, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG15, RANGE_MIN_QP0, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG15, RANGE_MAX_QP0, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG15, RANGE_BPG_OFFSET0, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG16, RANGE_MIN_QP1, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG16, RANGE_MAX_QP1, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG16, RANGE_BPG_OFFSET1, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG16, RANGE_MIN_QP2, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG16, RANGE_MAX_QP2, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG16, RANGE_BPG_OFFSET2, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG17, RANGE_MIN_QP3, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG17, RANGE_MAX_QP3, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG17, RANGE_BPG_OFFSET3, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG17, RANGE_MIN_QP4, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG17, RANGE_MAX_QP4, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG17, RANGE_BPG_OFFSET4, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG18, RANGE_MIN_QP5, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG18, RANGE_MAX_QP5, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG18, RANGE_BPG_OFFSET5, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG18, RANGE_MIN_QP6, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG18, RANGE_MAX_QP6, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG18, RANGE_BPG_OFFSET6, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG19, RANGE_MIN_QP7, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG19, RANGE_MAX_QP7, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG19, RANGE_BPG_OFFSET7, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG19, RANGE_MIN_QP8, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG19, RANGE_MAX_QP8, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG19, RANGE_BPG_OFFSET8, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG20, RANGE_MIN_QP9, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG20, RANGE_MAX_QP9, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG20, RANGE_BPG_OFFSET9, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG20, RANGE_MIN_QP10, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG20, RANGE_MAX_QP10, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG20, RANGE_BPG_OFFSET10, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG21, RANGE_MIN_QP11, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG21, RANGE_MAX_QP11, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG21, RANGE_BPG_OFFSET11, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG21, RANGE_MIN_QP12, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG21, RANGE_MAX_QP12, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG21, RANGE_BPG_OFFSET12, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG22, RANGE_MIN_QP13, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG22, RANGE_MAX_QP13, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG22, RANGE_BPG_OFFSET13, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG22, RANGE_MIN_QP14, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG22, RANGE_MAX_QP14, mask_sh), \
+ DSC_SF(DSCC0_DSCC_PPS_CONFIG22, RANGE_BPG_OFFSET14, mask_sh), \
+ DSC_SF(DSCC0_DSCC_MEM_POWER_CONTROL, DSCC_DEFAULT_MEM_LOW_POWER_STATE, mask_sh), \
+ DSC_SF(DSCC0_DSCC_MEM_POWER_CONTROL, DSCC_MEM_PWR_FORCE, mask_sh), \
+ DSC_SF(DSCC0_DSCC_MEM_POWER_CONTROL, DSCC_MEM_PWR_DIS, mask_sh), \
+ DSC_SF(DSCC0_DSCC_MEM_POWER_CONTROL, DSCC_MEM_PWR_STATE, mask_sh), \
+ DSC_SF(DSCC0_DSCC_MEM_POWER_CONTROL, DSCC_NATIVE_422_MEM_PWR_FORCE, mask_sh), \
+ DSC_SF(DSCC0_DSCC_MEM_POWER_CONTROL, DSCC_NATIVE_422_MEM_PWR_DIS, mask_sh), \
+ DSC_SF(DSCC0_DSCC_MEM_POWER_CONTROL, DSCC_NATIVE_422_MEM_PWR_STATE, mask_sh), \
+ DSC_SF(DSCC0_DSCC_R_Y_SQUARED_ERROR_LOWER, DSCC_R_Y_SQUARED_ERROR_LOWER, mask_sh), \
+ DSC_SF(DSCC0_DSCC_R_Y_SQUARED_ERROR_UPPER, DSCC_R_Y_SQUARED_ERROR_UPPER, mask_sh), \
+ DSC_SF(DSCC0_DSCC_G_CB_SQUARED_ERROR_LOWER, DSCC_G_CB_SQUARED_ERROR_LOWER, mask_sh), \
+ DSC_SF(DSCC0_DSCC_G_CB_SQUARED_ERROR_UPPER, DSCC_G_CB_SQUARED_ERROR_UPPER, mask_sh), \
+ DSC_SF(DSCC0_DSCC_B_CR_SQUARED_ERROR_LOWER, DSCC_B_CR_SQUARED_ERROR_LOWER, mask_sh), \
+ DSC_SF(DSCC0_DSCC_B_CR_SQUARED_ERROR_UPPER, DSCC_B_CR_SQUARED_ERROR_UPPER, mask_sh), \
+ DSC_SF(DSCC0_DSCC_MAX_ABS_ERROR0, DSCC_R_Y_MAX_ABS_ERROR, mask_sh), \
+ DSC_SF(DSCC0_DSCC_MAX_ABS_ERROR0, DSCC_G_CB_MAX_ABS_ERROR, mask_sh), \
+ DSC_SF(DSCC0_DSCC_MAX_ABS_ERROR1, DSCC_B_CR_MAX_ABS_ERROR, mask_sh), \
+ DSC_SF(DSCC0_DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL, DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL, mask_sh), \
+ DSC_SF(DSCC0_DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL, DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL, mask_sh), \
+ DSC_SF(DSCC0_DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL, DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL, mask_sh), \
+ DSC_SF(DSCC0_DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL, DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL, mask_sh), \
+ DSC_SF(DSCC0_DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL, DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL, mask_sh), \
+ DSC_SF(DSCC0_DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL, DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL, mask_sh), \
+ DSC_SF(DSCC0_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL, DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL, mask_sh), \
+ DSC_SF(DSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL, DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL, mask_sh), \
+ DSC_SF(DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE, DSCC_TEST_DEBUG_BUS0_ROTATE, mask_sh), \
+ DSC_SF(DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE, DSCC_TEST_DEBUG_BUS1_ROTATE, mask_sh), \
+ DSC_SF(DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE, DSCC_TEST_DEBUG_BUS2_ROTATE, mask_sh), \
+ DSC_SF(DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE, DSCC_TEST_DEBUG_BUS3_ROTATE, mask_sh), \
+ DSC_SF(DSCCIF0_DSCCIF_CONFIG0, INPUT_INTERFACE_UNDERFLOW_RECOVERY_EN, mask_sh), \
+ DSC_SF(DSCCIF0_DSCCIF_CONFIG0, INPUT_INTERFACE_UNDERFLOW_OCCURRED_INT_EN, mask_sh), \
+ DSC_SF(DSCCIF0_DSCCIF_CONFIG0, INPUT_INTERFACE_UNDERFLOW_OCCURRED_STATUS, mask_sh), \
+ DSC_SF(DSCCIF0_DSCCIF_CONFIG0, INPUT_PIXEL_FORMAT, mask_sh), \
+ DSC2_SF(DSCCIF0, DSCCIF_CONFIG0__BITS_PER_COMPONENT, mask_sh), \
+ DSC_SF(DSCCIF0_DSCCIF_CONFIG0, DOUBLE_BUFFER_REG_UPDATE_PENDING, mask_sh), \
+ DSC_SF(DSCCIF0_DSCCIF_CONFIG1, PIC_WIDTH, mask_sh), \
+ DSC_SF(DSCCIF0_DSCCIF_CONFIG1, PIC_HEIGHT, mask_sh), \
+ DSC_SF(DSCRM0_DSCRM_DSC_FORWARD_CONFIG, DSCRM_DSC_FORWARD_EN, mask_sh), \
+ DSC_SF(DSCRM0_DSCRM_DSC_FORWARD_CONFIG, DSCRM_DSC_OPP_PIPE_SOURCE, mask_sh)
+
+
+
+#define DSC_FIELD_LIST_DCN20(type)\
+ type DSC_CLOCK_EN; \
+ type DSC_DISPCLK_R_GATE_DIS; \
+ type DSC_DSCCLK_R_GATE_DIS; \
+ type DSC_DBG_EN; \
+ type DSC_TEST_CLOCK_MUX_SEL; \
+ type ICH_RESET_AT_END_OF_LINE; \
+ type NUMBER_OF_SLICES_PER_LINE; \
+ type ALTERNATE_ICH_ENCODING_EN; \
+ type NUMBER_OF_SLICES_IN_VERTICAL_DIRECTION; \
+ type DSCC_RATE_CONTROL_BUFFER_MODEL_SIZE; \
+ type DSCC_DISABLE_ICH; \
+ type DSCC_DOUBLE_BUFFER_REG_UPDATE_PENDING; \
+ type DSCC_RATE_BUFFER0_OVERFLOW_OCCURRED; \
+ type DSCC_RATE_BUFFER1_OVERFLOW_OCCURRED; \
+ type DSCC_RATE_BUFFER2_OVERFLOW_OCCURRED; \
+ type DSCC_RATE_BUFFER3_OVERFLOW_OCCURRED; \
+ type DSCC_RATE_BUFFER0_UNDERFLOW_OCCURRED; \
+ type DSCC_RATE_BUFFER1_UNDERFLOW_OCCURRED; \
+ type DSCC_RATE_BUFFER2_UNDERFLOW_OCCURRED; \
+ type DSCC_RATE_BUFFER3_UNDERFLOW_OCCURRED; \
+ type DSCC_RATE_CONTROL_BUFFER_MODEL0_OVERFLOW_OCCURRED; \
+ type DSCC_RATE_CONTROL_BUFFER_MODEL1_OVERFLOW_OCCURRED; \
+ type DSCC_RATE_CONTROL_BUFFER_MODEL2_OVERFLOW_OCCURRED; \
+ type DSCC_RATE_CONTROL_BUFFER_MODEL3_OVERFLOW_OCCURRED; \
+ type DSCC_RATE_BUFFER0_OVERFLOW_OCCURRED_INT_EN; \
+ type DSCC_RATE_BUFFER1_OVERFLOW_OCCURRED_INT_EN; \
+ type DSCC_RATE_BUFFER2_OVERFLOW_OCCURRED_INT_EN; \
+ type DSCC_RATE_BUFFER3_OVERFLOW_OCCURRED_INT_EN; \
+ type DSCC_RATE_BUFFER0_UNDERFLOW_OCCURRED_INT_EN; \
+ type DSCC_RATE_BUFFER1_UNDERFLOW_OCCURRED_INT_EN; \
+ type DSCC_RATE_BUFFER2_UNDERFLOW_OCCURRED_INT_EN; \
+ type DSCC_RATE_BUFFER3_UNDERFLOW_OCCURRED_INT_EN; \
+ type DSCC_RATE_CONTROL_BUFFER_MODEL0_OVERFLOW_OCCURRED_INT_EN; \
+ type DSCC_RATE_CONTROL_BUFFER_MODEL1_OVERFLOW_OCCURRED_INT_EN; \
+ type DSCC_RATE_CONTROL_BUFFER_MODEL2_OVERFLOW_OCCURRED_INT_EN; \
+ type DSCC_RATE_CONTROL_BUFFER_MODEL3_OVERFLOW_OCCURRED_INT_EN; \
+ type DSC_VERSION_MINOR; \
+ type DSC_VERSION_MAJOR; \
+ type PPS_IDENTIFIER; \
+ type LINEBUF_DEPTH; \
+ type DSCC_PPS_CONFIG0__BITS_PER_COMPONENT; \
+ type BITS_PER_PIXEL; \
+ type VBR_ENABLE; \
+ type SIMPLE_422; \
+ type CONVERT_RGB; \
+ type BLOCK_PRED_ENABLE; \
+ type NATIVE_422; \
+ type NATIVE_420; \
+ type CHUNK_SIZE; \
+ type PIC_WIDTH; \
+ type PIC_HEIGHT; \
+ type SLICE_WIDTH; \
+ type SLICE_HEIGHT; \
+ type INITIAL_XMIT_DELAY; \
+ type INITIAL_DEC_DELAY; \
+ type INITIAL_SCALE_VALUE; \
+ type SCALE_INCREMENT_INTERVAL; \
+ type SCALE_DECREMENT_INTERVAL; \
+ type FIRST_LINE_BPG_OFFSET; \
+ type SECOND_LINE_BPG_OFFSET; \
+ type NFL_BPG_OFFSET; \
+ type SLICE_BPG_OFFSET; \
+ type NSL_BPG_OFFSET; \
+ type SECOND_LINE_OFFSET_ADJ; \
+ type INITIAL_OFFSET; \
+ type FINAL_OFFSET; \
+ type FLATNESS_MIN_QP; \
+ type FLATNESS_MAX_QP; \
+ type RC_MODEL_SIZE; \
+ type RC_EDGE_FACTOR; \
+ type RC_QUANT_INCR_LIMIT0; \
+ type RC_QUANT_INCR_LIMIT1; \
+ type RC_TGT_OFFSET_LO; \
+ type RC_TGT_OFFSET_HI; \
+ type RC_BUF_THRESH0; \
+ type RC_BUF_THRESH1; \
+ type RC_BUF_THRESH2; \
+ type RC_BUF_THRESH3; \
+ type RC_BUF_THRESH4; \
+ type RC_BUF_THRESH5; \
+ type RC_BUF_THRESH6; \
+ type RC_BUF_THRESH7; \
+ type RC_BUF_THRESH8; \
+ type RC_BUF_THRESH9; \
+ type RC_BUF_THRESH10; \
+ type RC_BUF_THRESH11; \
+ type RC_BUF_THRESH12; \
+ type RC_BUF_THRESH13; \
+ type RANGE_MIN_QP0; \
+ type RANGE_MAX_QP0; \
+ type RANGE_BPG_OFFSET0; \
+ type RANGE_MIN_QP1; \
+ type RANGE_MAX_QP1; \
+ type RANGE_BPG_OFFSET1; \
+ type RANGE_MIN_QP2; \
+ type RANGE_MAX_QP2; \
+ type RANGE_BPG_OFFSET2; \
+ type RANGE_MIN_QP3; \
+ type RANGE_MAX_QP3; \
+ type RANGE_BPG_OFFSET3; \
+ type RANGE_MIN_QP4; \
+ type RANGE_MAX_QP4; \
+ type RANGE_BPG_OFFSET4; \
+ type RANGE_MIN_QP5; \
+ type RANGE_MAX_QP5; \
+ type RANGE_BPG_OFFSET5; \
+ type RANGE_MIN_QP6; \
+ type RANGE_MAX_QP6; \
+ type RANGE_BPG_OFFSET6; \
+ type RANGE_MIN_QP7; \
+ type RANGE_MAX_QP7; \
+ type RANGE_BPG_OFFSET7; \
+ type RANGE_MIN_QP8; \
+ type RANGE_MAX_QP8; \
+ type RANGE_BPG_OFFSET8; \
+ type RANGE_MIN_QP9; \
+ type RANGE_MAX_QP9; \
+ type RANGE_BPG_OFFSET9; \
+ type RANGE_MIN_QP10; \
+ type RANGE_MAX_QP10; \
+ type RANGE_BPG_OFFSET10; \
+ type RANGE_MIN_QP11; \
+ type RANGE_MAX_QP11; \
+ type RANGE_BPG_OFFSET11; \
+ type RANGE_MIN_QP12; \
+ type RANGE_MAX_QP12; \
+ type RANGE_BPG_OFFSET12; \
+ type RANGE_MIN_QP13; \
+ type RANGE_MAX_QP13; \
+ type RANGE_BPG_OFFSET13; \
+ type RANGE_MIN_QP14; \
+ type RANGE_MAX_QP14; \
+ type RANGE_BPG_OFFSET14; \
+ type DSCC_DEFAULT_MEM_LOW_POWER_STATE; \
+ type DSCC_MEM_PWR_FORCE; \
+ type DSCC_MEM_PWR_DIS; \
+ type DSCC_MEM_PWR_STATE; \
+ type DSCC_NATIVE_422_MEM_PWR_FORCE; \
+ type DSCC_NATIVE_422_MEM_PWR_DIS; \
+ type DSCC_NATIVE_422_MEM_PWR_STATE; \
+ type DSCC_R_Y_SQUARED_ERROR_LOWER; \
+ type DSCC_R_Y_SQUARED_ERROR_UPPER; \
+ type DSCC_G_CB_SQUARED_ERROR_LOWER; \
+ type DSCC_G_CB_SQUARED_ERROR_UPPER; \
+ type DSCC_B_CR_SQUARED_ERROR_LOWER; \
+ type DSCC_B_CR_SQUARED_ERROR_UPPER; \
+ type DSCC_R_Y_MAX_ABS_ERROR; \
+ type DSCC_G_CB_MAX_ABS_ERROR; \
+ type DSCC_B_CR_MAX_ABS_ERROR; \
+ type DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL; \
+ type DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL; \
+ type DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL; \
+ type DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL; \
+ type DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL; \
+ type DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL; \
+ type DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL; \
+ type DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL; \
+ type DSCC_UPDATE_PENDING_STATUS; \
+ type DSCC_UPDATE_TAKEN_STATUS; \
+ type DSCC_UPDATE_TAKEN_ACK; \
+ type DSCC_TEST_DEBUG_BUS0_ROTATE; \
+ type DSCC_TEST_DEBUG_BUS1_ROTATE; \
+ type DSCC_TEST_DEBUG_BUS2_ROTATE; \
+ type DSCC_TEST_DEBUG_BUS3_ROTATE; \
+ type DSCC_RATE_BUFFER0_FULLNESS_LEVEL; \
+ type DSCC_RATE_BUFFER1_FULLNESS_LEVEL; \
+ type DSCC_RATE_BUFFER2_FULLNESS_LEVEL; \
+ type DSCC_RATE_BUFFER3_FULLNESS_LEVEL; \
+ type DSCC_RATE_CONTROL_BUFFER0_FULLNESS_LEVEL; \
+ type DSCC_RATE_CONTROL_BUFFER1_FULLNESS_LEVEL; \
+ type DSCC_RATE_CONTROL_BUFFER2_FULLNESS_LEVEL; \
+ type DSCC_RATE_CONTROL_BUFFER3_FULLNESS_LEVEL; \
+ type DSCC_RATE_BUFFER0_INITIAL_XMIT_DELAY_REACHED; \
+ type DSCC_RATE_BUFFER1_INITIAL_XMIT_DELAY_REACHED; \
+ type DSCC_RATE_BUFFER2_INITIAL_XMIT_DELAY_REACHED; \
+ type DSCC_RATE_BUFFER3_INITIAL_XMIT_DELAY_REACHED; \
+ type INPUT_INTERFACE_UNDERFLOW_RECOVERY_EN; \
+ type INPUT_INTERFACE_UNDERFLOW_OCCURRED_INT_EN; \
+ type INPUT_INTERFACE_UNDERFLOW_OCCURRED_STATUS; \
+ type INPUT_PIXEL_FORMAT; \
+ type DSCCIF_CONFIG0__BITS_PER_COMPONENT; \
+ type DOUBLE_BUFFER_REG_UPDATE_PENDING; \
+ type DSCCIF_UPDATE_PENDING_STATUS; \
+ type DSCCIF_UPDATE_TAKEN_STATUS; \
+ type DSCCIF_UPDATE_TAKEN_ACK; \
+ type DSCRM_DSC_FORWARD_EN; \
+ type DSCRM_DSC_OPP_PIPE_SOURCE
+
+
+struct dcn20_dsc_registers {
+ uint32_t DSC_TOP_CONTROL;
+ uint32_t DSC_DEBUG_CONTROL;
+ uint32_t DSCC_CONFIG0;
+ uint32_t DSCC_CONFIG1;
+ uint32_t DSCC_STATUS;
+ uint32_t DSCC_INTERRUPT_CONTROL_STATUS;
+ uint32_t DSCC_PPS_CONFIG0;
+ uint32_t DSCC_PPS_CONFIG1;
+ uint32_t DSCC_PPS_CONFIG2;
+ uint32_t DSCC_PPS_CONFIG3;
+ uint32_t DSCC_PPS_CONFIG4;
+ uint32_t DSCC_PPS_CONFIG5;
+ uint32_t DSCC_PPS_CONFIG6;
+ uint32_t DSCC_PPS_CONFIG7;
+ uint32_t DSCC_PPS_CONFIG8;
+ uint32_t DSCC_PPS_CONFIG9;
+ uint32_t DSCC_PPS_CONFIG10;
+ uint32_t DSCC_PPS_CONFIG11;
+ uint32_t DSCC_PPS_CONFIG12;
+ uint32_t DSCC_PPS_CONFIG13;
+ uint32_t DSCC_PPS_CONFIG14;
+ uint32_t DSCC_PPS_CONFIG15;
+ uint32_t DSCC_PPS_CONFIG16;
+ uint32_t DSCC_PPS_CONFIG17;
+ uint32_t DSCC_PPS_CONFIG18;
+ uint32_t DSCC_PPS_CONFIG19;
+ uint32_t DSCC_PPS_CONFIG20;
+ uint32_t DSCC_PPS_CONFIG21;
+ uint32_t DSCC_PPS_CONFIG22;
+ uint32_t DSCC_MEM_POWER_CONTROL;
+ uint32_t DSCC_R_Y_SQUARED_ERROR_LOWER;
+ uint32_t DSCC_R_Y_SQUARED_ERROR_UPPER;
+ uint32_t DSCC_G_CB_SQUARED_ERROR_LOWER;
+ uint32_t DSCC_G_CB_SQUARED_ERROR_UPPER;
+ uint32_t DSCC_B_CR_SQUARED_ERROR_LOWER;
+ uint32_t DSCC_B_CR_SQUARED_ERROR_UPPER;
+ uint32_t DSCC_MAX_ABS_ERROR0;
+ uint32_t DSCC_MAX_ABS_ERROR1;
+ uint32_t DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL;
+ uint32_t DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL;
+ uint32_t DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL;
+ uint32_t DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL;
+ uint32_t DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL;
+ uint32_t DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL;
+ uint32_t DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL;
+ uint32_t DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL;
+ uint32_t DSCC_TEST_DEBUG_BUS_ROTATE;
+ uint32_t DSCCIF_CONFIG0;
+ uint32_t DSCCIF_CONFIG1;
+ uint32_t DSCRM_DSC_FORWARD_CONFIG;
+};
+
+
+struct dcn20_dsc_shift {
+ DSC_FIELD_LIST_DCN20(uint8_t);
+};
+
+struct dcn20_dsc_mask {
+ DSC_FIELD_LIST_DCN20(uint32_t);
+};
+
+/* DSCCIF_CONFIG.INPUT_PIXEL_FORMAT values */
+enum dsc_pixel_format {
+ DSC_PIXFMT_RGB,
+ DSC_PIXFMT_YCBCR444,
+ DSC_PIXFMT_SIMPLE_YCBCR422,
+ DSC_PIXFMT_NATIVE_YCBCR422,
+ DSC_PIXFMT_NATIVE_YCBCR420,
+ DSC_PIXFMT_UNKNOWN
+};
+
+struct dsc_reg_values {
+ /* PPS registers */
+ struct drm_dsc_config pps;
+
+ /* Additional registers */
+ uint32_t dsc_clock_enable;
+ uint32_t dsc_clock_gating_disable;
+ uint32_t underflow_recovery_en;
+ uint32_t underflow_occurred_int_en;
+ uint32_t underflow_occurred_status;
+ enum dsc_pixel_format pixel_format;
+ uint32_t ich_reset_at_eol;
+ uint32_t alternate_ich_encoding_en;
+ uint32_t num_slices_h;
+ uint32_t num_slices_v;
+ uint32_t rc_buffer_model_size;
+ uint32_t disable_ich;
+ uint32_t bpp_x32;
+ uint32_t dsc_dbg_en;
+ uint32_t rc_buffer_model_overflow_int_en[4];
+};
+
+struct dcn20_dsc {
+ struct display_stream_compressor base;
+ const struct dcn20_dsc_registers *dsc_regs;
+ const struct dcn20_dsc_shift *dsc_shift;
+ const struct dcn20_dsc_mask *dsc_mask;
+
+ struct dsc_reg_values reg_vals;
+
+ int max_image_width;
+};
+
+
+void dsc2_construct(struct dcn20_dsc *dsc,
+ struct dc_context *ctx,
+ int inst,
+ const struct dcn20_dsc_registers *dsc_regs,
+ const struct dcn20_dsc_shift *dsc_shift,
+ const struct dcn20_dsc_mask *dsc_mask);
+
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb.c
new file mode 100644
index 000000000000..8d3884b306dd
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb.c
@@ -0,0 +1,332 @@
+/*
+ * Copyright 2012-17 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+
+#include "reg_helper.h"
+#include "resource.h"
+#include "dwb.h"
+#include "dcn20_dwb.h"
+
+
+#define REG(reg)\
+ dwbc20->dwbc_regs->reg
+
+#define CTX \
+ dwbc20->base.ctx
+
+#define DC_LOGGER \
+ dwbc20->base.ctx->logger
+#undef FN
+#define FN(reg_name, field_name) \
+ dwbc20->dwbc_shift->field_name, dwbc20->dwbc_mask->field_name
+
+enum dwb_outside_pix_strategy {
+ DWB_OUTSIDE_PIX_STRATEGY_BLACK = 0,
+ DWB_OUTSIDE_PIX_STRATEGY_EDGE = 1
+};
+
+static bool dwb2_get_caps(struct dwbc *dwbc, struct dwb_caps *caps)
+{
+ struct dcn20_dwbc *dwbc20 = TO_DCN20_DWBC(dwbc);
+ if (caps) {
+ caps->adapter_id = 0; /* we only support 1 adapter currently */
+ caps->hw_version = DCN_VERSION_2_0;
+ caps->num_pipes = 1;
+ memset(&caps->reserved, 0, sizeof(caps->reserved));
+ memset(&caps->reserved2, 0, sizeof(caps->reserved2));
+ caps->sw_version = dwb_ver_1_0;
+ caps->caps.support_dwb = true;
+ caps->caps.support_ogam = false;
+ caps->caps.support_wbscl = false;
+ caps->caps.support_ocsc = false;
+ DC_LOG_DWB("%s SUPPORTED! inst = %d", __func__, dwbc20->base.inst);
+ return true;
+ } else {
+ DC_LOG_DWB("%s NOT SUPPORTED! inst = %d", __func__, dwbc20->base.inst);
+ return false;
+ }
+}
+
+void dwb2_config_dwb_cnv(struct dwbc *dwbc, struct dc_dwb_params *params)
+{
+ struct dcn20_dwbc *dwbc20 = TO_DCN20_DWBC(dwbc);
+ DC_LOG_DWB("%s inst = %d", __func__, dwbc20->base.inst);
+
+ /* Set DWB source size */
+ REG_UPDATE_2(CNV_SOURCE_SIZE, CNV_SOURCE_WIDTH, params->cnv_params.src_width,
+ CNV_SOURCE_HEIGHT, params->cnv_params.src_height);
+
+ /* source size is not equal the source size, then enable cropping. */
+ if (params->cnv_params.crop_en) {
+ REG_UPDATE(CNV_MODE, CNV_WINDOW_CROP_EN, 1);
+ REG_UPDATE(CNV_WINDOW_START, CNV_WINDOW_START_X, params->cnv_params.crop_x);
+ REG_UPDATE(CNV_WINDOW_START, CNV_WINDOW_START_Y, params->cnv_params.crop_y);
+ REG_UPDATE(CNV_WINDOW_SIZE, CNV_WINDOW_WIDTH, params->cnv_params.crop_width);
+ REG_UPDATE(CNV_WINDOW_SIZE, CNV_WINDOW_HEIGHT, params->cnv_params.crop_height);
+ } else {
+ REG_UPDATE(CNV_MODE, CNV_WINDOW_CROP_EN, 0);
+ }
+
+ /* Set CAPTURE_RATE */
+ REG_UPDATE(CNV_MODE, CNV_FRAME_CAPTURE_RATE, params->capture_rate);
+
+ /* Set CNV output pixel depth */
+ REG_UPDATE(CNV_MODE, CNV_OUT_BPC, params->cnv_params.cnv_out_bpc);
+}
+
+static bool dwb2_enable(struct dwbc *dwbc, struct dc_dwb_params *params)
+{
+ struct dcn20_dwbc *dwbc20 = TO_DCN20_DWBC(dwbc);
+
+ /* Only chroma scaling (sub-sampling) is supported in DCN2 */
+if ((params->cnv_params.src_width != params->dest_width) ||
+ (params->cnv_params.src_height != params->dest_height)) {
+
+ DC_LOG_DWB("%s inst = %d, FAILED!LUMA SCALING NOT SUPPORTED", __func__, dwbc20->base.inst);
+ return false;
+ }
+ DC_LOG_DWB("%s inst = %d, ENABLED", __func__, dwbc20->base.inst);
+
+ /* disable power gating */
+ //REG_UPDATE_5(WB_EC_CONFIG, DISPCLK_R_WB_GATE_DIS, 1,
+ // DISPCLK_G_WB_GATE_DIS, 1, DISPCLK_G_WBSCL_GATE_DIS, 1,
+ // WB_LB_LS_DIS, 1, WB_LUT_LS_DIS, 1);
+
+ /* Set WB_ENABLE (not double buffered; capture not enabled) */
+ REG_UPDATE(WB_ENABLE, WB_ENABLE, 1);
+
+ /* Set CNV parameters */
+ dwb2_config_dwb_cnv(dwbc, params);
+
+ /* Set scaling parameters */
+ dwb2_set_scaler(dwbc, params);
+
+ /* Enable DWB capture enable (double buffered) */
+ REG_UPDATE(CNV_MODE, CNV_FRAME_CAPTURE_EN, DWB_FRAME_CAPTURE_ENABLE);
+
+ // disable warmup
+ REG_UPDATE(WB_WARM_UP_MODE_CTL1, GMC_WARM_UP_ENABLE, 0);
+
+ return true;
+}
+
+bool dwb2_disable(struct dwbc *dwbc)
+{
+ struct dcn20_dwbc *dwbc20 = TO_DCN20_DWBC(dwbc);
+ DC_LOG_DWB("%s inst = %d, Disabled", __func__, dwbc20->base.inst);
+
+ /* disable CNV */
+ REG_UPDATE(CNV_MODE, CNV_FRAME_CAPTURE_EN, DWB_FRAME_CAPTURE_DISABLE);
+
+ /* disable WB */
+ REG_UPDATE(WB_ENABLE, WB_ENABLE, 0);
+
+ /* soft reset */
+ REG_UPDATE(WB_SOFT_RESET, WB_SOFT_RESET, 1);
+ REG_UPDATE(WB_SOFT_RESET, WB_SOFT_RESET, 0);
+
+ /* enable power gating */
+ //REG_UPDATE_5(WB_EC_CONFIG, DISPCLK_R_WB_GATE_DIS, 0,
+ // DISPCLK_G_WB_GATE_DIS, 0, DISPCLK_G_WBSCL_GATE_DIS, 0,
+ // WB_LB_LS_DIS, 0, WB_LUT_LS_DIS, 0);
+
+ return true;
+}
+
+static bool dwb2_update(struct dwbc *dwbc, struct dc_dwb_params *params)
+{
+ struct dcn20_dwbc *dwbc20 = TO_DCN20_DWBC(dwbc);
+ unsigned int pre_locked;
+
+ /* Only chroma scaling (sub-sampling) is supported in DCN2 */
+ if ((params->cnv_params.src_width != params->dest_width) ||
+ (params->cnv_params.src_height != params->dest_height)) {
+ DC_LOG_DWB("%s inst = %d, FAILED!LUMA SCALING NOT SUPPORTED", __func__, dwbc20->base.inst);
+ return false;
+ }
+ DC_LOG_DWB("%s inst = %d, scaling", __func__, dwbc20->base.inst);
+
+ /*
+ * Check if the caller has already locked CNV registers.
+ * If so: assume the caller will unlock, so don't touch the lock.
+ * If not: lock them for this update, then unlock after the
+ * update is complete.
+ */
+ REG_GET(CNV_UPDATE, CNV_UPDATE_LOCK, &pre_locked);
+
+ if (pre_locked == 0) {
+ /* Lock DWB registers */
+ REG_UPDATE(CNV_UPDATE, CNV_UPDATE_LOCK, 1);
+ }
+
+ /* Set CNV parameters */
+ dwb2_config_dwb_cnv(dwbc, params);
+
+ /* Set scaling parameters */
+ dwb2_set_scaler(dwbc, params);
+
+ if (pre_locked == 0) {
+ /* Unlock DWB registers */
+ REG_UPDATE(CNV_UPDATE, CNV_UPDATE_LOCK, 0);
+ }
+
+ return true;
+}
+
+bool dwb2_is_enabled(struct dwbc *dwbc)
+{
+ struct dcn20_dwbc *dwbc20 = TO_DCN20_DWBC(dwbc);
+ unsigned int wb_enabled = 0;
+ unsigned int cnv_frame_capture_en = 0;
+
+ REG_GET(WB_ENABLE, WB_ENABLE, &wb_enabled);
+ REG_GET(CNV_MODE, CNV_FRAME_CAPTURE_EN, &cnv_frame_capture_en);
+
+ return ((wb_enabled != 0) && (cnv_frame_capture_en != 0));
+}
+
+void dwb2_set_stereo(struct dwbc *dwbc,
+ struct dwb_stereo_params *stereo_params)
+{
+ struct dcn20_dwbc *dwbc20 = TO_DCN20_DWBC(dwbc);
+ DC_LOG_DWB("%s inst = %d, enabled =%d", __func__,\
+ dwbc20->base.inst, stereo_params->stereo_enabled);
+
+ if (stereo_params->stereo_enabled) {
+ REG_UPDATE(CNV_MODE, CNV_STEREO_TYPE, stereo_params->stereo_type);
+ REG_UPDATE(CNV_MODE, CNV_EYE_SELECTION, stereo_params->stereo_eye_select);
+ REG_UPDATE(CNV_MODE, CNV_STEREO_POLARITY, stereo_params->stereo_polarity);
+ } else {
+ REG_UPDATE(CNV_MODE, CNV_EYE_SELECTION, 0);
+ }
+}
+
+void dwb2_set_new_content(struct dwbc *dwbc,
+ bool is_new_content)
+{
+ struct dcn20_dwbc *dwbc20 = TO_DCN20_DWBC(dwbc);
+ DC_LOG_DWB("%s inst = %d", __func__, dwbc20->base.inst);
+
+ REG_UPDATE(CNV_MODE, CNV_NEW_CONTENT, is_new_content);
+}
+
+static void dwb2_set_warmup(struct dwbc *dwbc,
+ struct dwb_warmup_params *warmup_params)
+{
+ struct dcn20_dwbc *dwbc20 = TO_DCN20_DWBC(dwbc);
+ DC_LOG_DWB("%s inst = %d", __func__, dwbc20->base.inst);
+
+ REG_UPDATE(WB_WARM_UP_MODE_CTL1, GMC_WARM_UP_ENABLE, warmup_params->warmup_en);
+ REG_UPDATE(WB_WARM_UP_MODE_CTL1, WIDTH_WARMUP, warmup_params->warmup_width);
+ REG_UPDATE(WB_WARM_UP_MODE_CTL1, HEIGHT_WARMUP, warmup_params->warmup_height);
+
+ REG_UPDATE(WB_WARM_UP_MODE_CTL2, DATA_VALUE_WARMUP, warmup_params->warmup_data);
+ REG_UPDATE(WB_WARM_UP_MODE_CTL2, MODE_WARMUP, warmup_params->warmup_mode);
+ REG_UPDATE(WB_WARM_UP_MODE_CTL2, DATA_DEPTH_WARMUP, warmup_params->warmup_depth);
+}
+
+void dwb2_set_scaler(struct dwbc *dwbc, struct dc_dwb_params *params)
+{
+ struct dcn20_dwbc *dwbc20 = TO_DCN20_DWBC(dwbc);
+ DC_LOG_DWB("%s inst = %d", __func__, dwbc20->base.inst);
+
+ /* Program scaling mode */
+ REG_UPDATE_2(WBSCL_MODE, WBSCL_MODE, params->out_format,
+ WBSCL_OUT_BIT_DEPTH, params->output_depth);
+
+ if (params->out_format != dwb_scaler_mode_bypass444) {
+ /* Program output size */
+ REG_UPDATE(WBSCL_DEST_SIZE, WBSCL_DEST_WIDTH, params->dest_width);
+ REG_UPDATE(WBSCL_DEST_SIZE, WBSCL_DEST_HEIGHT, params->dest_height);
+
+ /* Program round offsets */
+ REG_UPDATE(WBSCL_ROUND_OFFSET, WBSCL_ROUND_OFFSET_Y_RGB, 0x40);
+ REG_UPDATE(WBSCL_ROUND_OFFSET, WBSCL_ROUND_OFFSET_CBCR, 0x200);
+
+ /* Program clamp values */
+ REG_UPDATE(WBSCL_CLAMP_Y_RGB, WBSCL_CLAMP_UPPER_Y_RGB, 0x3fe);
+ REG_UPDATE(WBSCL_CLAMP_Y_RGB, WBSCL_CLAMP_LOWER_Y_RGB, 0x1);
+ REG_UPDATE(WBSCL_CLAMP_CBCR, WBSCL_CLAMP_UPPER_CBCR, 0x3fe);
+ REG_UPDATE(WBSCL_CLAMP_CBCR, WBSCL_CLAMP_LOWER_CBCR, 0x1);
+
+ /* Program outside pixel strategy to use edge pixels */
+ REG_UPDATE(WBSCL_OUTSIDE_PIX_STRATEGY, WBSCL_OUTSIDE_PIX_STRATEGY, DWB_OUTSIDE_PIX_STRATEGY_EDGE);
+
+ if (params->cnv_params.crop_en) {
+ /* horizontal scale */
+ dwb_program_horz_scalar(dwbc20, params->cnv_params.crop_width,
+ params->dest_width,
+ params->scaler_taps);
+
+ /* vertical scale */
+ dwb_program_vert_scalar(dwbc20, params->cnv_params.crop_height,
+ params->dest_height,
+ params->scaler_taps,
+ params->subsample_position);
+ } else {
+ /* horizontal scale */
+ dwb_program_horz_scalar(dwbc20, params->cnv_params.src_width,
+ params->dest_width,
+ params->scaler_taps);
+
+ /* vertical scale */
+ dwb_program_vert_scalar(dwbc20, params->cnv_params.src_height,
+ params->dest_height,
+ params->scaler_taps,
+ params->subsample_position);
+ }
+ }
+
+}
+
+const struct dwbc_funcs dcn20_dwbc_funcs = {
+ .get_caps = dwb2_get_caps,
+ .enable = dwb2_enable,
+ .disable = dwb2_disable,
+ .update = dwb2_update,
+ .is_enabled = dwb2_is_enabled,
+ .set_stereo = dwb2_set_stereo,
+ .set_new_content = dwb2_set_new_content,
+ .set_warmup = dwb2_set_warmup,
+ .dwb_set_scaler = dwb2_set_scaler,
+};
+
+void dcn20_dwbc_construct(struct dcn20_dwbc *dwbc20,
+ struct dc_context *ctx,
+ const struct dcn20_dwbc_registers *dwbc_regs,
+ const struct dcn20_dwbc_shift *dwbc_shift,
+ const struct dcn20_dwbc_mask *dwbc_mask,
+ int inst)
+{
+ dwbc20->base.ctx = ctx;
+
+ dwbc20->base.inst = inst;
+ dwbc20->base.funcs = &dcn20_dwbc_funcs;
+
+ dwbc20->dwbc_regs = dwbc_regs;
+ dwbc20->dwbc_shift = dwbc_shift;
+ dwbc20->dwbc_mask = dwbc_mask;
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb.h
new file mode 100644
index 000000000000..a85ed228dfc2
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb.h
@@ -0,0 +1,458 @@
+/* Copyright 2012-17 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+#ifndef __DC_DWBC_DCN20_H__
+#define __DC_DWBC_DCN20_H__
+
+#define TO_DCN20_DWBC(dwbc_base) \
+ container_of(dwbc_base, struct dcn20_dwbc, base)
+
+/* DCN */
+#define BASE_INNER(seg) \
+ DCE_BASE__INST0_SEG ## seg
+
+#define BASE(seg) \
+ BASE_INNER(seg)
+
+#define SR(reg_name)\
+ .reg_name = BASE(mm ## reg_name ## _BASE_IDX) + \
+ mm ## reg_name
+
+#define SRI(reg_name, block, id)\
+ .reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+ mm ## block ## id ## _ ## reg_name
+
+#define SRI2(reg_name, block, id)\
+ .reg_name = BASE(mm ## reg_name ## _BASE_IDX) + \
+ mm ## reg_name
+
+#define SRII(reg_name, block, id)\
+ .reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+ mm ## block ## id ## _ ## reg_name
+
+#define SF(reg_name, field_name, post_fix)\
+ .field_name = reg_name ## __ ## field_name ## post_fix
+
+
+#define DWBC_COMMON_REG_LIST_DCN2_0(inst) \
+ SRI2(WB_ENABLE, CNV, inst),\
+ SRI2(WB_EC_CONFIG, CNV, inst),\
+ SRI2(CNV_MODE, CNV, inst),\
+ SRI2(CNV_WINDOW_START, CNV, inst),\
+ SRI2(CNV_WINDOW_SIZE, CNV, inst),\
+ SRI2(CNV_UPDATE, CNV, inst),\
+ SRI2(CNV_SOURCE_SIZE, CNV, inst),\
+ SRI2(CNV_TEST_CNTL, CNV, inst),\
+ SRI2(CNV_TEST_CRC_RED, CNV, inst),\
+ SRI2(CNV_TEST_CRC_GREEN, CNV, inst),\
+ SRI2(CNV_TEST_CRC_BLUE, CNV, inst),\
+ SRI2(WBSCL_COEF_RAM_SELECT, WBSCL, inst),\
+ SRI2(WBSCL_COEF_RAM_TAP_DATA, WBSCL, inst),\
+ SRI2(WBSCL_MODE, WBSCL, inst),\
+ SRI2(WBSCL_TAP_CONTROL, WBSCL, inst),\
+ SRI2(WBSCL_DEST_SIZE, WBSCL, inst),\
+ SRI2(WBSCL_HORZ_FILTER_SCALE_RATIO, WBSCL, inst),\
+ SRI2(WBSCL_HORZ_FILTER_INIT_Y_RGB, WBSCL, inst),\
+ SRI2(WBSCL_HORZ_FILTER_INIT_CBCR, WBSCL, inst),\
+ SRI2(WBSCL_VERT_FILTER_SCALE_RATIO, WBSCL, inst),\
+ SRI2(WBSCL_VERT_FILTER_INIT_Y_RGB, WBSCL, inst),\
+ SRI2(WBSCL_VERT_FILTER_INIT_CBCR, WBSCL, inst),\
+ SRI2(WBSCL_ROUND_OFFSET, WBSCL, inst),\
+ SRI2(WBSCL_OVERFLOW_STATUS, WBSCL, inst),\
+ SRI2(WBSCL_COEF_RAM_CONFLICT_STATUS, WBSCL, inst),\
+ SRI2(WBSCL_TEST_CNTL, WBSCL, inst),\
+ SRI2(WBSCL_TEST_CRC_RED, WBSCL, inst),\
+ SRI2(WBSCL_TEST_CRC_GREEN, WBSCL, inst),\
+ SRI2(WBSCL_TEST_CRC_BLUE, WBSCL, inst),\
+ SRI2(WBSCL_BACKPRESSURE_CNT_EN, WBSCL, inst),\
+ SRI2(WB_MCIF_BACKPRESSURE_CNT, WBSCL, inst),\
+ SRI2(WBSCL_CLAMP_Y_RGB, WBSCL, inst),\
+ SRI2(WBSCL_CLAMP_CBCR, WBSCL, inst),\
+ SRI2(WBSCL_OUTSIDE_PIX_STRATEGY, WBSCL, inst),\
+ SRI2(WBSCL_OUTSIDE_PIX_STRATEGY_CBCR, WBSCL, inst),\
+ SRI2(WBSCL_DEBUG, WBSCL, inst),\
+ SRI2(WBSCL_TEST_DEBUG_INDEX, WBSCL, inst),\
+ SRI2(WBSCL_TEST_DEBUG_DATA, WBSCL, inst),\
+ SRI2(WB_DEBUG_CTRL, CNV, inst),\
+ SRI2(WB_DBG_MODE, CNV, inst),\
+ SRI2(WB_HW_DEBUG, CNV, inst),\
+ SRI2(CNV_TEST_DEBUG_INDEX, CNV, inst),\
+ SRI2(CNV_TEST_DEBUG_DATA, CNV, inst),\
+ SRI2(WB_SOFT_RESET, CNV, inst),\
+ SRI2(WB_WARM_UP_MODE_CTL1, CNV, inst),\
+ SRI2(WB_WARM_UP_MODE_CTL2, CNV, inst)
+
+#define DWBC_COMMON_MASK_SH_LIST_DCN2_0(mask_sh) \
+ SF(WB_ENABLE, WB_ENABLE, mask_sh),\
+ SF(WB_EC_CONFIG, DISPCLK_R_WB_GATE_DIS, mask_sh),\
+ SF(WB_EC_CONFIG, DISPCLK_G_WB_GATE_DIS, mask_sh),\
+ SF(WB_EC_CONFIG, DISPCLK_G_WBSCL_GATE_DIS, mask_sh),\
+ SF(WB_EC_CONFIG, WB_TEST_CLK_SEL, mask_sh),\
+ SF(WB_EC_CONFIG, WB_LB_LS_DIS, mask_sh),\
+ SF(WB_EC_CONFIG, WB_LB_SD_DIS, mask_sh),\
+ SF(WB_EC_CONFIG, WB_LUT_LS_DIS, mask_sh),\
+ SF(WB_EC_CONFIG, WBSCL_LB_MEM_PWR_MODE_SEL, mask_sh),\
+ SF(WB_EC_CONFIG, WBSCL_LB_MEM_PWR_DIS, mask_sh),\
+ SF(WB_EC_CONFIG, WBSCL_LB_MEM_PWR_FORCE, mask_sh),\
+ SF(WB_EC_CONFIG, WBSCL_LB_MEM_PWR_STATE, mask_sh),\
+ SF(WB_EC_CONFIG, WB_RAM_PW_SAVE_MODE, mask_sh),\
+ SF(WB_EC_CONFIG, WBSCL_LUT_MEM_PWR_STATE, mask_sh),\
+ SF(CNV_MODE, CNV_OUT_BPC, mask_sh),\
+ SF(CNV_MODE, CNV_FRAME_CAPTURE_RATE, mask_sh),\
+ SF(CNV_MODE, CNV_WINDOW_CROP_EN, mask_sh),\
+ SF(CNV_MODE, CNV_STEREO_TYPE, mask_sh),\
+ SF(CNV_MODE, CNV_INTERLACED_MODE, mask_sh),\
+ SF(CNV_MODE, CNV_EYE_SELECTION, mask_sh),\
+ SF(CNV_MODE, CNV_STEREO_POLARITY, mask_sh),\
+ SF(CNV_MODE, CNV_INTERLACED_FIELD_ORDER, mask_sh),\
+ SF(CNV_MODE, CNV_STEREO_SPLIT, mask_sh),\
+ SF(CNV_MODE, CNV_NEW_CONTENT, mask_sh),\
+ SF(CNV_MODE, CNV_FRAME_CAPTURE_EN_CURRENT, mask_sh),\
+ SF(CNV_MODE, CNV_FRAME_CAPTURE_EN, mask_sh),\
+ SF(CNV_WINDOW_START, CNV_WINDOW_START_X, mask_sh),\
+ SF(CNV_WINDOW_START, CNV_WINDOW_START_Y, mask_sh),\
+ SF(CNV_WINDOW_SIZE, CNV_WINDOW_WIDTH, mask_sh),\
+ SF(CNV_WINDOW_SIZE, CNV_WINDOW_HEIGHT, mask_sh),\
+ SF(CNV_UPDATE, CNV_UPDATE_PENDING, mask_sh),\
+ SF(CNV_UPDATE, CNV_UPDATE_TAKEN, mask_sh),\
+ SF(CNV_UPDATE, CNV_UPDATE_LOCK, mask_sh),\
+ SF(CNV_SOURCE_SIZE, CNV_SOURCE_WIDTH, mask_sh),\
+ SF(CNV_SOURCE_SIZE, CNV_SOURCE_HEIGHT, mask_sh),\
+ SF(CNV_TEST_CNTL, CNV_TEST_CRC_EN, mask_sh),\
+ SF(CNV_TEST_CNTL, CNV_TEST_CRC_CONT_EN, mask_sh),\
+ SF(CNV_TEST_CRC_RED, CNV_TEST_CRC_RED_MASK, mask_sh),\
+ SF(CNV_TEST_CRC_RED, CNV_TEST_CRC_SIG_RED, mask_sh),\
+ SF(CNV_TEST_CRC_GREEN, CNV_TEST_CRC_GREEN_MASK, mask_sh),\
+ SF(CNV_TEST_CRC_GREEN, CNV_TEST_CRC_SIG_GREEN, mask_sh),\
+ SF(CNV_TEST_CRC_BLUE, CNV_TEST_CRC_BLUE_MASK, mask_sh),\
+ SF(CNV_TEST_CRC_BLUE, CNV_TEST_CRC_SIG_BLUE, mask_sh),\
+ SF(WB_DEBUG_CTRL, WB_DEBUG_EN, mask_sh),\
+ SF(WB_DEBUG_CTRL, WB_DEBUG_SEL, mask_sh),\
+ SF(WB_DBG_MODE, WB_DBG_MODE_EN, mask_sh),\
+ SF(WB_DBG_MODE, WB_DBG_DIN_FMT, mask_sh),\
+ SF(WB_DBG_MODE, WB_DBG_36MODE, mask_sh),\
+ SF(WB_DBG_MODE, WB_DBG_CMAP, mask_sh),\
+ SF(WB_DBG_MODE, WB_DBG_PXLRATE_ERROR, mask_sh),\
+ SF(WB_DBG_MODE, WB_DBG_SOURCE_WIDTH, mask_sh),\
+ SF(WB_HW_DEBUG, WB_HW_DEBUG, mask_sh),\
+ SF(WB_SOFT_RESET, WB_SOFT_RESET, mask_sh),\
+ SF(CNV_TEST_DEBUG_INDEX, CNV_TEST_DEBUG_INDEX, mask_sh),\
+ SF(CNV_TEST_DEBUG_INDEX, CNV_TEST_DEBUG_WRITE_EN, mask_sh),\
+ SF(CNV_TEST_DEBUG_DATA, CNV_TEST_DEBUG_DATA, mask_sh),\
+ SF(WBSCL_COEF_RAM_SELECT, WBSCL_COEF_RAM_TAP_PAIR_IDX, mask_sh),\
+ SF(WBSCL_COEF_RAM_SELECT, WBSCL_COEF_RAM_PHASE, mask_sh),\
+ SF(WBSCL_COEF_RAM_SELECT, WBSCL_COEF_RAM_FILTER_TYPE, mask_sh),\
+ SF(WBSCL_COEF_RAM_TAP_DATA, WBSCL_COEF_RAM_EVEN_TAP_COEF, mask_sh),\
+ SF(WBSCL_COEF_RAM_TAP_DATA, WBSCL_COEF_RAM_EVEN_TAP_COEF_EN, mask_sh),\
+ SF(WBSCL_COEF_RAM_TAP_DATA, WBSCL_COEF_RAM_ODD_TAP_COEF, mask_sh),\
+ SF(WBSCL_COEF_RAM_TAP_DATA, WBSCL_COEF_RAM_ODD_TAP_COEF_EN, mask_sh),\
+ SF(WBSCL_MODE, WBSCL_MODE, mask_sh),\
+ SF(WBSCL_MODE, WBSCL_OUT_BIT_DEPTH, mask_sh),\
+ SF(WBSCL_TAP_CONTROL, WBSCL_V_NUM_OF_TAPS_Y_RGB, mask_sh),\
+ SF(WBSCL_TAP_CONTROL, WBSCL_V_NUM_OF_TAPS_CBCR, mask_sh),\
+ SF(WBSCL_TAP_CONTROL, WBSCL_H_NUM_OF_TAPS_Y_RGB, mask_sh),\
+ SF(WBSCL_TAP_CONTROL, WBSCL_H_NUM_OF_TAPS_CBCR, mask_sh),\
+ SF(WBSCL_DEST_SIZE, WBSCL_DEST_HEIGHT, mask_sh),\
+ SF(WBSCL_DEST_SIZE, WBSCL_DEST_WIDTH, mask_sh),\
+ SF(WBSCL_HORZ_FILTER_SCALE_RATIO, WBSCL_H_SCALE_RATIO, mask_sh),\
+ SF(WBSCL_HORZ_FILTER_INIT_Y_RGB, WBSCL_H_INIT_FRAC_Y_RGB, mask_sh),\
+ SF(WBSCL_HORZ_FILTER_INIT_Y_RGB, WBSCL_H_INIT_INT_Y_RGB, mask_sh),\
+ SF(WBSCL_HORZ_FILTER_INIT_CBCR, WBSCL_H_INIT_FRAC_CBCR, mask_sh),\
+ SF(WBSCL_HORZ_FILTER_INIT_CBCR, WBSCL_H_INIT_INT_CBCR, mask_sh),\
+ SF(WBSCL_VERT_FILTER_SCALE_RATIO, WBSCL_V_SCALE_RATIO, mask_sh),\
+ SF(WBSCL_VERT_FILTER_INIT_Y_RGB, WBSCL_V_INIT_FRAC_Y_RGB, mask_sh),\
+ SF(WBSCL_VERT_FILTER_INIT_Y_RGB, WBSCL_V_INIT_INT_Y_RGB, mask_sh),\
+ SF(WBSCL_VERT_FILTER_INIT_CBCR, WBSCL_V_INIT_FRAC_CBCR, mask_sh),\
+ SF(WBSCL_VERT_FILTER_INIT_CBCR, WBSCL_V_INIT_INT_CBCR, mask_sh),\
+ SF(WBSCL_ROUND_OFFSET, WBSCL_ROUND_OFFSET_Y_RGB, mask_sh),\
+ SF(WBSCL_ROUND_OFFSET, WBSCL_ROUND_OFFSET_CBCR, mask_sh),\
+ SF(WBSCL_OVERFLOW_STATUS, WBSCL_DATA_OVERFLOW_FLAG, mask_sh),\
+ SF(WBSCL_OVERFLOW_STATUS, WBSCL_DATA_OVERFLOW_ACK, mask_sh),\
+ SF(WBSCL_OVERFLOW_STATUS, WBSCL_DATA_OVERFLOW_MASK, mask_sh),\
+ SF(WBSCL_OVERFLOW_STATUS, WBSCL_DATA_OVERFLOW_INT_STATUS, mask_sh),\
+ SF(WBSCL_OVERFLOW_STATUS, WBSCL_DATA_OVERFLOW_INT_TYPE, mask_sh),\
+ SF(WBSCL_COEF_RAM_CONFLICT_STATUS, WBSCL_HOST_CONFLICT_FLAG, mask_sh),\
+ SF(WBSCL_COEF_RAM_CONFLICT_STATUS, WBSCL_HOST_CONFLICT_ACK, mask_sh),\
+ SF(WBSCL_COEF_RAM_CONFLICT_STATUS, WBSCL_HOST_CONFLICT_MASK, mask_sh),\
+ SF(WBSCL_COEF_RAM_CONFLICT_STATUS, WBSCL_HOST_CONFLICT_INT_STATUS, mask_sh),\
+ SF(WBSCL_COEF_RAM_CONFLICT_STATUS, WBSCL_HOST_CONFLICT_INT_TYPE, mask_sh),\
+ SF(WBSCL_TEST_CNTL, WBSCL_TEST_CRC_EN, mask_sh),\
+ SF(WBSCL_TEST_CNTL, WBSCL_TEST_CRC_CONT_EN, mask_sh),\
+ SF(WBSCL_TEST_CRC_RED, WBSCL_TEST_CRC_RED_MASK, mask_sh),\
+ SF(WBSCL_TEST_CRC_RED, WBSCL_TEST_CRC_SIG_RED, mask_sh),\
+ SF(WBSCL_TEST_CRC_GREEN, WBSCL_TEST_CRC_GREEN_MASK, mask_sh),\
+ SF(WBSCL_TEST_CRC_GREEN, WBSCL_TEST_CRC_SIG_GREEN, mask_sh),\
+ SF(WBSCL_TEST_CRC_BLUE, WBSCL_TEST_CRC_BLUE_MASK, mask_sh),\
+ SF(WBSCL_TEST_CRC_BLUE, WBSCL_TEST_CRC_SIG_BLUE, mask_sh),\
+ SF(WBSCL_BACKPRESSURE_CNT_EN, WBSCL_BACKPRESSURE_CNT_EN, mask_sh),\
+ SF(WB_MCIF_BACKPRESSURE_CNT, WB_MCIF_Y_MAX_BACKPRESSURE, mask_sh),\
+ SF(WB_MCIF_BACKPRESSURE_CNT, WB_MCIF_C_MAX_BACKPRESSURE, mask_sh),\
+ SF(WBSCL_CLAMP_Y_RGB, WBSCL_CLAMP_UPPER_Y_RGB, mask_sh),\
+ SF(WBSCL_CLAMP_Y_RGB, WBSCL_CLAMP_LOWER_Y_RGB, mask_sh),\
+ SF(WBSCL_CLAMP_CBCR, WBSCL_CLAMP_UPPER_CBCR, mask_sh),\
+ SF(WBSCL_CLAMP_CBCR, WBSCL_CLAMP_LOWER_CBCR, mask_sh),\
+ SF(WBSCL_OUTSIDE_PIX_STRATEGY, WBSCL_OUTSIDE_PIX_STRATEGY, mask_sh),\
+ SF(WBSCL_OUTSIDE_PIX_STRATEGY, WBSCL_BLACK_COLOR_G_Y, mask_sh),\
+ SF(WBSCL_OUTSIDE_PIX_STRATEGY_CBCR, WBSCL_BLACK_COLOR_B_CB, mask_sh),\
+ SF(WBSCL_OUTSIDE_PIX_STRATEGY_CBCR, WBSCL_BLACK_COLOR_R_CR, mask_sh),\
+ SF(WBSCL_DEBUG, WBSCL_DEBUG, mask_sh),\
+ SF(WBSCL_TEST_DEBUG_INDEX, WBSCL_TEST_DEBUG_INDEX, mask_sh),\
+ SF(WBSCL_TEST_DEBUG_INDEX, WBSCL_TEST_DEBUG_WRITE_EN, mask_sh),\
+ SF(WBSCL_TEST_DEBUG_DATA, WBSCL_TEST_DEBUG_DATA, mask_sh),\
+ SF(WB_WARM_UP_MODE_CTL1, WIDTH_WARMUP, mask_sh),\
+ SF(WB_WARM_UP_MODE_CTL1, HEIGHT_WARMUP, mask_sh),\
+ SF(WB_WARM_UP_MODE_CTL1, GMC_WARM_UP_ENABLE, mask_sh),\
+ SF(WB_WARM_UP_MODE_CTL2, DATA_VALUE_WARMUP, mask_sh),\
+ SF(WB_WARM_UP_MODE_CTL2, MODE_WARMUP, mask_sh),\
+ SF(WB_WARM_UP_MODE_CTL2, DATA_DEPTH_WARMUP, mask_sh)
+
+#define DWBC_REG_FIELD_LIST_DCN2_0(type) \
+ type WB_ENABLE;\
+ type DISPCLK_R_WB_GATE_DIS;\
+ type DISPCLK_G_WB_GATE_DIS;\
+ type DISPCLK_G_WBSCL_GATE_DIS;\
+ type WB_TEST_CLK_SEL;\
+ type WB_LB_LS_DIS;\
+ type WB_LB_SD_DIS;\
+ type WB_LUT_LS_DIS;\
+ type WBSCL_LB_MEM_PWR_MODE_SEL;\
+ type WBSCL_LB_MEM_PWR_DIS;\
+ type WBSCL_LB_MEM_PWR_FORCE;\
+ type WBSCL_LB_MEM_PWR_STATE;\
+ type WB_RAM_PW_SAVE_MODE;\
+ type WBSCL_LUT_MEM_PWR_STATE;\
+ type CNV_OUT_BPC;\
+ type CNV_FRAME_CAPTURE_RATE;\
+ type CNV_WINDOW_CROP_EN;\
+ type CNV_STEREO_TYPE;\
+ type CNV_INTERLACED_MODE;\
+ type CNV_EYE_SELECTION;\
+ type CNV_STEREO_POLARITY;\
+ type CNV_INTERLACED_FIELD_ORDER;\
+ type CNV_STEREO_SPLIT;\
+ type CNV_NEW_CONTENT;\
+ type CNV_FRAME_CAPTURE_EN_CURRENT;\
+ type CNV_FRAME_CAPTURE_EN;\
+ type CNV_WINDOW_START_X;\
+ type CNV_WINDOW_START_Y;\
+ type CNV_WINDOW_WIDTH;\
+ type CNV_WINDOW_HEIGHT;\
+ type CNV_UPDATE_PENDING;\
+ type CNV_UPDATE_TAKEN;\
+ type CNV_UPDATE_LOCK;\
+ type CNV_SOURCE_WIDTH;\
+ type CNV_SOURCE_HEIGHT;\
+ type CNV_TEST_CRC_EN;\
+ type CNV_TEST_CRC_CONT_EN;\
+ type CNV_TEST_CRC_RED_MASK;\
+ type CNV_TEST_CRC_SIG_RED;\
+ type CNV_TEST_CRC_GREEN_MASK;\
+ type CNV_TEST_CRC_SIG_GREEN;\
+ type CNV_TEST_CRC_BLUE_MASK;\
+ type CNV_TEST_CRC_SIG_BLUE;\
+ type WB_DEBUG_EN;\
+ type WB_DEBUG_SEL;\
+ type WB_DBG_MODE_EN;\
+ type WB_DBG_DIN_FMT;\
+ type WB_DBG_36MODE;\
+ type WB_DBG_CMAP;\
+ type WB_DBG_PXLRATE_ERROR;\
+ type WB_DBG_SOURCE_WIDTH;\
+ type WB_HW_DEBUG;\
+ type CNV_TEST_DEBUG_INDEX;\
+ type CNV_TEST_DEBUG_WRITE_EN;\
+ type CNV_TEST_DEBUG_DATA;\
+ type WB_SOFT_RESET;\
+ type WBSCL_COEF_RAM_TAP_PAIR_IDX;\
+ type WBSCL_COEF_RAM_PHASE;\
+ type WBSCL_COEF_RAM_FILTER_TYPE;\
+ type WBSCL_COEF_RAM_SEL;\
+ type WBSCL_COEF_RAM_SEL_CURRENT;\
+ type WBSCL_COEF_RAM_RD_SEL;\
+ type WBSCL_COEF_RAM_EVEN_TAP_COEF;\
+ type WBSCL_COEF_RAM_EVEN_TAP_COEF_EN;\
+ type WBSCL_COEF_RAM_ODD_TAP_COEF;\
+ type WBSCL_COEF_RAM_ODD_TAP_COEF_EN;\
+ type WBSCL_MODE;\
+ type WBSCL_OUT_BIT_DEPTH;\
+ type WBSCL_V_NUM_OF_TAPS_Y_RGB;\
+ type WBSCL_V_NUM_OF_TAPS_CBCR;\
+ type WBSCL_H_NUM_OF_TAPS_Y_RGB;\
+ type WBSCL_H_NUM_OF_TAPS_CBCR;\
+ type WBSCL_DEST_HEIGHT;\
+ type WBSCL_DEST_WIDTH;\
+ type WBSCL_H_SCALE_RATIO;\
+ type WBSCL_H_INIT_FRAC_Y_RGB;\
+ type WBSCL_H_INIT_INT_Y_RGB;\
+ type WBSCL_H_INIT_FRAC_CBCR;\
+ type WBSCL_H_INIT_INT_CBCR;\
+ type WBSCL_V_SCALE_RATIO;\
+ type WBSCL_V_INIT_FRAC_Y_RGB;\
+ type WBSCL_V_INIT_INT_Y_RGB;\
+ type WBSCL_V_INIT_FRAC_CBCR;\
+ type WBSCL_V_INIT_INT_CBCR;\
+ type WBSCL_ROUND_OFFSET_Y_RGB;\
+ type WBSCL_ROUND_OFFSET_CBCR;\
+ type WBSCL_DATA_OVERFLOW_FLAG;\
+ type WBSCL_DATA_OVERFLOW_ACK;\
+ type WBSCL_DATA_OVERFLOW_MASK;\
+ type WBSCL_DATA_OVERFLOW_INT_STATUS;\
+ type WBSCL_DATA_OVERFLOW_INT_TYPE;\
+ type WBSCL_HOST_CONFLICT_FLAG;\
+ type WBSCL_HOST_CONFLICT_ACK;\
+ type WBSCL_HOST_CONFLICT_MASK;\
+ type WBSCL_HOST_CONFLICT_INT_STATUS;\
+ type WBSCL_HOST_CONFLICT_INT_TYPE;\
+ type WBSCL_TEST_CRC_EN;\
+ type WBSCL_TEST_CRC_CONT_EN;\
+ type WBSCL_TEST_CRC_RED_MASK;\
+ type WBSCL_TEST_CRC_SIG_RED;\
+ type WBSCL_TEST_CRC_GREEN_MASK;\
+ type WBSCL_TEST_CRC_SIG_GREEN;\
+ type WBSCL_TEST_CRC_BLUE_MASK;\
+ type WBSCL_TEST_CRC_SIG_BLUE;\
+ type WBSCL_BACKPRESSURE_CNT_EN;\
+ type WB_MCIF_Y_MAX_BACKPRESSURE;\
+ type WB_MCIF_C_MAX_BACKPRESSURE;\
+ type WBSCL_CLAMP_UPPER_Y_RGB;\
+ type WBSCL_CLAMP_LOWER_Y_RGB;\
+ type WBSCL_CLAMP_UPPER_CBCR;\
+ type WBSCL_CLAMP_LOWER_CBCR;\
+ type WBSCL_OUTSIDE_PIX_STRATEGY;\
+ type WBSCL_BLACK_COLOR_G_Y;\
+ type WBSCL_BLACK_COLOR_B_CB;\
+ type WBSCL_BLACK_COLOR_R_CR;\
+ type WBSCL_DEBUG;\
+ type WBSCL_TEST_DEBUG_INDEX;\
+ type WBSCL_TEST_DEBUG_WRITE_EN;\
+ type WBSCL_TEST_DEBUG_DATA;\
+ type WIDTH_WARMUP;\
+ type HEIGHT_WARMUP;\
+ type GMC_WARM_UP_ENABLE;\
+ type DATA_VALUE_WARMUP;\
+ type MODE_WARMUP;\
+ type DATA_DEPTH_WARMUP; \
+
+struct dcn20_dwbc_registers {
+ /* DCN2.0 */
+ uint32_t WB_ENABLE;
+ uint32_t WB_EC_CONFIG;
+ uint32_t CNV_MODE;
+ uint32_t CNV_WINDOW_START;
+ uint32_t CNV_WINDOW_SIZE;
+ uint32_t CNV_UPDATE;
+ uint32_t CNV_SOURCE_SIZE;
+ uint32_t CNV_TEST_CNTL;
+ uint32_t CNV_TEST_CRC_RED;
+ uint32_t CNV_TEST_CRC_GREEN;
+ uint32_t CNV_TEST_CRC_BLUE;
+ uint32_t WB_DEBUG_CTRL;
+ uint32_t WB_DBG_MODE;
+ uint32_t WB_HW_DEBUG;
+ uint32_t CNV_TEST_DEBUG_INDEX;
+ uint32_t CNV_TEST_DEBUG_DATA;
+ uint32_t WB_SOFT_RESET;
+ uint32_t WBSCL_COEF_RAM_SELECT;
+ uint32_t WBSCL_COEF_RAM_TAP_DATA;
+ uint32_t WBSCL_MODE;
+ uint32_t WBSCL_TAP_CONTROL;
+ uint32_t WBSCL_DEST_SIZE;
+ uint32_t WBSCL_HORZ_FILTER_SCALE_RATIO;
+ uint32_t WBSCL_HORZ_FILTER_INIT_Y_RGB;
+ uint32_t WBSCL_HORZ_FILTER_INIT_CBCR;
+ uint32_t WBSCL_VERT_FILTER_SCALE_RATIO;
+ uint32_t WBSCL_VERT_FILTER_INIT_Y_RGB;
+ uint32_t WBSCL_VERT_FILTER_INIT_CBCR;
+ uint32_t WBSCL_ROUND_OFFSET;
+ uint32_t WBSCL_OVERFLOW_STATUS;
+ uint32_t WBSCL_COEF_RAM_CONFLICT_STATUS;
+ uint32_t WBSCL_TEST_CNTL;
+ uint32_t WBSCL_TEST_CRC_RED;
+ uint32_t WBSCL_TEST_CRC_GREEN;
+ uint32_t WBSCL_TEST_CRC_BLUE;
+ uint32_t WBSCL_BACKPRESSURE_CNT_EN;
+ uint32_t WB_MCIF_BACKPRESSURE_CNT;
+ uint32_t WBSCL_CLAMP_Y_RGB;
+ uint32_t WBSCL_CLAMP_CBCR;
+ uint32_t WBSCL_OUTSIDE_PIX_STRATEGY;
+ uint32_t WBSCL_OUTSIDE_PIX_STRATEGY_CBCR;
+ uint32_t WBSCL_DEBUG;
+ uint32_t WBSCL_TEST_DEBUG_INDEX;
+ uint32_t WBSCL_TEST_DEBUG_DATA;
+ uint32_t WB_WARM_UP_MODE_CTL1;
+ uint32_t WB_WARM_UP_MODE_CTL2;
+};
+
+
+struct dcn20_dwbc_mask {
+ DWBC_REG_FIELD_LIST_DCN2_0(uint32_t)
+};
+
+struct dcn20_dwbc_shift {
+ DWBC_REG_FIELD_LIST_DCN2_0(uint8_t)
+};
+
+struct dcn20_dwbc {
+ struct dwbc base;
+ const struct dcn20_dwbc_registers *dwbc_regs;
+ const struct dcn20_dwbc_shift *dwbc_shift;
+ const struct dcn20_dwbc_mask *dwbc_mask;
+};
+
+void dcn20_dwbc_construct(struct dcn20_dwbc *dwbc20,
+ struct dc_context *ctx,
+ const struct dcn20_dwbc_registers *dwbc_regs,
+ const struct dcn20_dwbc_shift *dwbc_shift,
+ const struct dcn20_dwbc_mask *dwbc_mask,
+ int inst);
+
+bool dwb2_disable(struct dwbc *dwbc);
+
+bool dwb2_is_enabled(struct dwbc *dwbc);
+
+void dwb2_set_stereo(struct dwbc *dwbc,
+ struct dwb_stereo_params *stereo_params);
+
+void dwb2_set_new_content(struct dwbc *dwbc,
+ bool is_new_content);
+
+void dwb2_config_dwb_cnv(struct dwbc *dwbc,
+ struct dc_dwb_params *params);
+
+void dwb2_set_scaler(struct dwbc *dwbc, struct dc_dwb_params *params);
+
+bool dwb_program_vert_scalar(struct dcn20_dwbc *dwbc20,
+ uint32_t src_height,
+ uint32_t dest_height,
+ struct scaling_taps num_taps,
+ enum dwb_subsample_position subsample_position);
+
+bool dwb_program_horz_scalar(struct dcn20_dwbc *dwbc20,
+ uint32_t src_width,
+ uint32_t dest_width,
+ struct scaling_taps num_taps);
+
+
+#endif
+
+
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb_scl.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb_scl.c
new file mode 100644
index 000000000000..cd8bc92ce3ba
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb_scl.c
@@ -0,0 +1,877 @@
+/*
+ * Copyright 2012-17 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "reg_helper.h"
+#include "fixed31_32.h"
+#include "resource.h"
+#include "dwb.h"
+#include "dcn20_dwb.h"
+
+#define NUM_PHASES 16
+#define HORZ_MAX_TAPS 12
+#define VERT_MAX_TAPS 12
+
+#define REG(reg)\
+ dwbc20->dwbc_regs->reg
+
+#define CTX \
+ dwbc20->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name) \
+ dwbc20->dwbc_shift->field_name, dwbc20->dwbc_mask->field_name
+
+#define TO_DCN20_DWBC(dwbc_base) \
+ container_of(dwbc_base, struct dcn20_dwbc, base)
+
+
+static const uint16_t filter_3tap_16p_upscale[27] = {
+ 2048, 2048, 0,
+ 1708, 2424, 16348,
+ 1372, 2796, 16308,
+ 1056, 3148, 16272,
+ 768, 3464, 16244,
+ 512, 3728, 16236,
+ 296, 3928, 16252,
+ 124, 4052, 16296,
+ 0, 4096, 0
+};
+
+static const uint16_t filter_3tap_16p_117[27] = {
+ 2048, 2048, 0,
+ 1824, 2276, 16376,
+ 1600, 2496, 16380,
+ 1376, 2700, 16,
+ 1156, 2880, 52,
+ 948, 3032, 108,
+ 756, 3144, 192,
+ 580, 3212, 296,
+ 428, 3236, 428
+};
+
+static const uint16_t filter_3tap_16p_150[27] = {
+ 2048, 2048, 0,
+ 1872, 2184, 36,
+ 1692, 2308, 88,
+ 1516, 2420, 156,
+ 1340, 2516, 236,
+ 1168, 2592, 328,
+ 1004, 2648, 440,
+ 844, 2684, 560,
+ 696, 2696, 696
+};
+
+static const uint16_t filter_3tap_16p_183[27] = {
+ 2048, 2048, 0,
+ 1892, 2104, 92,
+ 1744, 2152, 196,
+ 1592, 2196, 300,
+ 1448, 2232, 412,
+ 1304, 2256, 528,
+ 1168, 2276, 648,
+ 1032, 2288, 772,
+ 900, 2292, 900
+};
+
+static const uint16_t filter_4tap_16p_upscale[36] = {
+ 0, 4096, 0, 0,
+ 16240, 4056, 180, 16380,
+ 16136, 3952, 404, 16364,
+ 16072, 3780, 664, 16344,
+ 16040, 3556, 952, 16312,
+ 16036, 3284, 1268, 16272,
+ 16052, 2980, 1604, 16224,
+ 16084, 2648, 1952, 16176,
+ 16128, 2304, 2304, 16128
+};
+
+static const uint16_t filter_4tap_16p_117[36] = {
+ 428, 3236, 428, 0,
+ 276, 3232, 604, 16364,
+ 148, 3184, 800, 16340,
+ 44, 3104, 1016, 16312,
+ 16344, 2984, 1244, 16284,
+ 16284, 2832, 1488, 16256,
+ 16244, 2648, 1732, 16236,
+ 16220, 2440, 1976, 16220,
+ 16212, 2216, 2216, 16212
+};
+
+static const uint16_t filter_4tap_16p_150[36] = {
+ 696, 2700, 696, 0,
+ 560, 2700, 848, 16364,
+ 436, 2676, 1008, 16348,
+ 328, 2628, 1180, 16336,
+ 232, 2556, 1356, 16328,
+ 152, 2460, 1536, 16328,
+ 84, 2344, 1716, 16332,
+ 28, 2208, 1888, 16348,
+ 16376, 2052, 2052, 16376
+};
+
+static const uint16_t filter_4tap_16p_183[36] = {
+ 940, 2208, 940, 0,
+ 832, 2200, 1052, 4,
+ 728, 2180, 1164, 16,
+ 628, 2148, 1280, 36,
+ 536, 2100, 1392, 60,
+ 448, 2044, 1504, 92,
+ 368, 1976, 1612, 132,
+ 296, 1900, 1716, 176,
+ 232, 1812, 1812, 232
+};
+
+static const uint16_t filter_5tap_16p_upscale[45] = {
+ 15936, 2496, 2496, 15936, 0,
+ 15992, 2128, 2832, 15896, 12,
+ 16056, 1760, 3140, 15876, 24,
+ 16120, 1404, 3420, 15876, 36,
+ 16188, 1060, 3652, 15908, 44,
+ 16248, 744, 3844, 15972, 44,
+ 16304, 460, 3980, 16072, 40,
+ 16348, 212, 4064, 16208, 24,
+ 0, 0, 4096, 0, 0,
+};
+
+static const uint16_t filter_5tap_16p_117[45] = {
+ 16056, 2372, 2372, 16056, 0,
+ 16052, 2124, 2600, 16076, 0,
+ 16060, 1868, 2808, 16120, 0,
+ 16080, 1612, 2992, 16180, 16376,
+ 16112, 1356, 3144, 16268, 16364,
+ 16144, 1108, 3268, 16376, 16344,
+ 16184, 872, 3356, 124, 16320,
+ 16220, 656, 3412, 276, 16292,
+ 16256, 456, 3428, 456, 16256,
+};
+
+static const uint16_t filter_5tap_16p_150[45] = {
+ 16368, 2064, 2064, 16368, 0,
+ 16316, 1924, 2204, 44, 16372,
+ 16280, 1772, 2328, 116, 16356,
+ 16256, 1616, 2440, 204, 16340,
+ 16240, 1456, 2536, 304, 16320,
+ 16232, 1296, 2612, 416, 16300,
+ 16232, 1132, 2664, 544, 16284,
+ 16240, 976, 2700, 680, 16264,
+ 16248, 824, 2708, 824, 16248,
+};
+
+static const uint16_t filter_5tap_16p_183[45] = {
+ 228, 1816, 1816, 228, 0,
+ 168, 1728, 1904, 300, 16372,
+ 116, 1632, 1988, 376, 16360,
+ 72, 1528, 2060, 460, 16348,
+ 36, 1424, 2120, 552, 16340,
+ 4, 1312, 2168, 652, 16336,
+ 16368, 1200, 2204, 752, 16332,
+ 16352, 1084, 2224, 860, 16332,
+ 16340, 972, 2232, 972, 16340,
+};
+
+static const uint16_t filter_6tap_16p_upscale[54] = {
+ 0, 0, 4092, 0, 0, 0,
+ 44, 16188, 4064, 228, 16324, 0,
+ 80, 16036, 3980, 492, 16256, 4,
+ 108, 15916, 3844, 788, 16184, 16,
+ 120, 15836, 3656, 1108, 16104, 28,
+ 128, 15792, 3420, 1448, 16024, 44,
+ 124, 15776, 3144, 1800, 15948, 64,
+ 112, 15792, 2836, 2152, 15880, 80,
+ 100, 15828, 2504, 2504, 15828, 100,
+};
+
+static const uint16_t filter_6tap_16p_117[54] = {
+ 16168, 476, 3568, 476, 16168, 0,
+ 16216, 280, 3540, 692, 16116, 8,
+ 16264, 104, 3472, 924, 16068, 16,
+ 16304, 16340, 3372, 1168, 16024, 28,
+ 16344, 16212, 3236, 1424, 15988, 36,
+ 16372, 16112, 3072, 1680, 15956, 44,
+ 12, 16036, 2880, 1936, 15940, 48,
+ 28, 15984, 2668, 2192, 15936, 48,
+ 40, 15952, 2436, 2436, 15952, 40,
+};
+
+static const uint16_t filter_6tap_16p_150[54] = {
+ 16148, 920, 2724, 920, 16148, 0,
+ 16156, 768, 2712, 1072, 16144, 0,
+ 16172, 628, 2684, 1232, 16148, 16380,
+ 16192, 492, 2632, 1388, 16160, 16372,
+ 16212, 368, 2564, 1548, 16180, 16364,
+ 16232, 256, 2480, 1704, 16212, 16352,
+ 16256, 156, 2380, 1856, 16256, 16336,
+ 16276, 64, 2268, 2004, 16308, 16320,
+ 16300, 16372, 2140, 2140, 16372, 16300,
+};
+
+static const uint16_t filter_6tap_16p_183[54] = {
+ 16296, 1032, 2196, 1032, 16296, 0,
+ 16284, 924, 2196, 1144, 16320, 16376,
+ 16272, 820, 2180, 1256, 16348, 16364,
+ 16268, 716, 2156, 1364, 16380, 16352,
+ 16264, 620, 2116, 1472, 36, 16340,
+ 16268, 524, 2068, 1576, 88, 16328,
+ 16272, 436, 2008, 1680, 144, 16316,
+ 16280, 352, 1940, 1772, 204, 16304,
+ 16292, 276, 1860, 1860, 276, 16292,
+};
+
+static const uint16_t filter_7tap_16p_upscale[63] = {
+ 176, 15760, 2488, 2488, 15760, 176, 0,
+ 160, 15812, 2152, 2816, 15728, 192, 16376,
+ 136, 15884, 1812, 3124, 15720, 196, 16368,
+ 108, 15964, 1468, 3400, 15740, 196, 16364,
+ 84, 16048, 1132, 3640, 15792, 180, 16360,
+ 56, 16140, 812, 3832, 15884, 152, 16360,
+ 32, 16228, 512, 3976, 16012, 116, 16364,
+ 12, 16308, 240, 4064, 16180, 60, 16372,
+ 0, 0, 0, 4096, 0, 0, 0,
+};
+
+static const uint16_t filter_7tap_16p_117[63] = {
+ 92, 15868, 2464, 2464, 15868, 92, 0,
+ 108, 15852, 2216, 2700, 15904, 72, 0,
+ 112, 15856, 1960, 2916, 15964, 44, 0,
+ 116, 15876, 1696, 3108, 16048, 8, 8,
+ 112, 15908, 1428, 3268, 16156, 16348, 12,
+ 104, 15952, 1168, 3400, 16288, 16300, 24,
+ 92, 16004, 916, 3496, 64, 16244, 36,
+ 80, 16064, 676, 3556, 248, 16184, 48,
+ 64, 16124, 452, 3576, 452, 16124, 64,
+};
+
+static const uint16_t filter_7tap_16p_150[63] = {
+ 16224, 16380, 2208, 2208, 16380, 16224, 0,
+ 16252, 16304, 2072, 2324, 84, 16196, 4,
+ 16276, 16240, 1924, 2432, 184, 16172, 8,
+ 16300, 16184, 1772, 2524, 296, 16144, 12,
+ 16324, 16144, 1616, 2600, 416, 16124, 12,
+ 16344, 16112, 1456, 2660, 548, 16104, 12,
+ 16360, 16092, 1296, 2704, 688, 16088, 12,
+ 16372, 16080, 1140, 2732, 832, 16080, 8,
+ 0, 16076, 984, 2740, 984, 16076, 0,
+};
+
+static const uint16_t filter_7tap_16p_183[63] = {
+ 16216, 324, 1884, 1884, 324, 16216, 0,
+ 16228, 248, 1804, 1960, 408, 16212, 16380,
+ 16240, 176, 1716, 2028, 496, 16208, 16376,
+ 16252, 112, 1624, 2084, 588, 16208, 16372,
+ 16264, 56, 1524, 2132, 684, 16212, 16364,
+ 16280, 4, 1424, 2168, 788, 16220, 16356,
+ 16292, 16344, 1320, 2196, 892, 16232, 16344,
+ 16308, 16308, 1212, 2212, 996, 16252, 16332,
+ 16320, 16276, 1104, 2216, 1104, 16276, 16320,
+};
+
+static const uint16_t filter_8tap_16p_upscale[72] = {
+ 0, 0, 0, 4096, 0, 0, 0, 0,
+ 16360, 76, 16172, 4064, 244, 16296, 24, 16380,
+ 16340, 136, 15996, 3980, 524, 16204, 56, 16380,
+ 16328, 188, 15860, 3844, 828, 16104, 92, 16372,
+ 16320, 224, 15760, 3656, 1156, 16008, 128, 16368,
+ 16320, 248, 15696, 3428, 1496, 15912, 160, 16360,
+ 16320, 256, 15668, 3156, 1844, 15828, 192, 16348,
+ 16324, 256, 15672, 2856, 2192, 15756, 220, 16340,
+ 16332, 244, 15704, 2532, 2532, 15704, 244, 16332,
+};
+
+static const uint16_t filter_8tap_16p_117[72] = {
+ 116, 16100, 428, 3564, 428, 16100, 116, 0,
+ 96, 16168, 220, 3548, 656, 16032, 136, 16376,
+ 76, 16236, 32, 3496, 904, 15968, 152, 16372,
+ 56, 16300, 16252, 3408, 1164, 15908, 164, 16368,
+ 36, 16360, 16116, 3284, 1428, 15856, 172, 16364,
+ 20, 28, 16000, 3124, 1700, 15820, 176, 16364,
+ 4, 76, 15912, 2940, 1972, 15800, 172, 16364,
+ 16380, 112, 15848, 2724, 2236, 15792, 160, 16364,
+ 16372, 140, 15812, 2488, 2488, 15812, 140, 16372,
+};
+
+static const uint16_t filter_8tap_16p_150[72] = {
+ 16380, 16020, 1032, 2756, 1032, 16020, 16380, 0,
+ 12, 16020, 876, 2744, 1184, 16032, 16364, 4,
+ 24, 16028, 728, 2716, 1344, 16052, 16340, 8,
+ 36, 16040, 584, 2668, 1500, 16080, 16316, 16,
+ 40, 16060, 448, 2608, 1652, 16120, 16288, 20,
+ 44, 16080, 320, 2528, 1804, 16168, 16260, 28,
+ 48, 16108, 204, 2436, 1948, 16232, 16228, 32,
+ 44, 16136, 100, 2328, 2084, 16304, 16200, 40,
+ 44, 16168, 4, 2212, 2212, 4, 16168, 44,
+};
+
+static const uint16_t filter_8tap_16p_183[72] = {
+ 16264, 16264, 1164, 2244, 1164, 16264, 16264, 0,
+ 16280, 16232, 1056, 2236, 1268, 16300, 16248, 0,
+ 16296, 16204, 948, 2220, 1372, 16348, 16232, 0,
+ 16312, 16184, 844, 2192, 1472, 12, 16216, 4,
+ 16328, 16172, 740, 2156, 1572, 72, 16200, 0,
+ 16340, 16160, 640, 2108, 1668, 136, 16188, 0,
+ 16352, 16156, 544, 2052, 1756, 204, 16176, 16380,
+ 16360, 16156, 452, 1988, 1840, 280, 16164, 16376,
+ 16368, 16160, 364, 1920, 1920, 364, 16160, 16368,
+};
+
+static const uint16_t filter_9tap_16p_upscale[81] = {
+ 16284, 296, 15660, 2572, 2572, 15660, 296, 16284, 0,
+ 16296, 272, 15712, 2228, 2896, 15632, 304, 16276, 4,
+ 16308, 240, 15788, 1876, 3192, 15632, 304, 16276, 4,
+ 16320, 204, 15876, 1520, 3452, 15664, 288, 16280, 8,
+ 16336, 164, 15976, 1176, 3676, 15732, 260, 16288, 12,
+ 16348, 120, 16080, 844, 3856, 15840, 216, 16300, 12,
+ 16364, 76, 16188, 532, 3988, 15984, 156, 16324, 8,
+ 16376, 36, 16288, 252, 4068, 16164, 84, 16352, 4,
+ 0, 0, 0, 0, 4096, 0, 0, 0, 0,
+};
+
+static const uint16_t filter_9tap_16p_117[81] = {
+ 16356, 172, 15776, 2504, 2504, 15776, 172, 16356, 0,
+ 16344, 200, 15756, 2252, 2740, 15816, 136, 16372, 16380,
+ 16336, 216, 15756, 1988, 2956, 15884, 92, 8, 16380,
+ 16332, 224, 15780, 1720, 3144, 15976, 40, 28, 16376,
+ 16328, 224, 15816, 1448, 3304, 16096, 16364, 52, 16372,
+ 16328, 216, 15868, 1180, 3432, 16240, 16296, 80, 16364,
+ 16332, 200, 15928, 916, 3524, 24, 16224, 108, 16356,
+ 16336, 184, 15996, 668, 3580, 220, 16148, 132, 16352,
+ 16344, 160, 16072, 436, 3600, 436, 16072, 160, 16344,
+};
+
+static const uint16_t filter_9tap_16p_150[81] = {
+ 84, 16128, 0, 2216, 2216, 0, 16128, 84, 0,
+ 80, 16160, 16296, 2088, 2332, 100, 16092, 84, 0,
+ 76, 16196, 16220, 1956, 2432, 208, 16064, 80, 0,
+ 72, 16232, 16152, 1812, 2524, 328, 16036, 76, 4,
+ 64, 16264, 16096, 1664, 2600, 460, 16012, 64, 8,
+ 56, 16300, 16052, 1508, 2656, 596, 15996, 52, 12,
+ 48, 16328, 16020, 1356, 2700, 740, 15984, 36, 20,
+ 40, 16356, 15996, 1196, 2728, 888, 15980, 20, 24,
+ 32, 0, 15984, 1044, 2736, 1044, 15984, 0, 32,
+};
+
+static const uint16_t filter_9tap_16p_183[81] = {
+ 16356, 16112, 388, 1952, 1952, 388, 16112, 16356, 0,
+ 16368, 16116, 304, 1876, 2020, 480, 16112, 16344, 4,
+ 16376, 16124, 224, 1792, 2080, 576, 16116, 16328, 8,
+ 0, 16136, 148, 1700, 2132, 672, 16124, 16312, 8,
+ 8, 16148, 80, 1604, 2176, 772, 16140, 16296, 12,
+ 12, 16164, 16, 1504, 2208, 876, 16156, 16276, 16,
+ 16, 16180, 16344, 1404, 2232, 980, 16184, 16256, 20,
+ 20, 16200, 16296, 1300, 2244, 1088, 16212, 16240, 20,
+ 20, 16220, 16252, 1196, 2252, 1196, 16252, 16220, 20,
+};
+
+static const uint16_t filter_10tap_16p_upscale[90] = {
+ 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0,
+ 12, 16344, 88, 16160, 4068, 252, 16280, 44, 16368, 0,
+ 24, 16308, 168, 15976, 3988, 540, 16176, 92, 16348, 0,
+ 32, 16280, 236, 15828, 3852, 852, 16064, 140, 16328, 4,
+ 36, 16260, 284, 15720, 3672, 1184, 15956, 188, 16308, 8,
+ 36, 16244, 320, 15648, 3448, 1528, 15852, 236, 16288, 12,
+ 36, 16240, 336, 15612, 3184, 1880, 15764, 276, 16272, 20,
+ 32, 16240, 340, 15608, 2888, 2228, 15688, 308, 16256, 24,
+ 28, 16244, 332, 15636, 2568, 2568, 15636, 332, 16244, 28,
+};
+
+static const uint16_t filter_10tap_16p_117[90] = {
+ 16308, 196, 16048, 440, 3636, 440, 16048, 196, 16308, 0,
+ 16316, 164, 16132, 220, 3612, 676, 15972, 220, 16300, 0,
+ 16324, 132, 16212, 20, 3552, 932, 15900, 240, 16296, 4,
+ 16336, 100, 16292, 16232, 3456, 1192, 15836, 256, 16296, 4,
+ 16348, 68, 16364, 16084, 3324, 1464, 15784, 264, 16296, 8,
+ 16356, 36, 48, 15960, 3164, 1736, 15748, 260, 16304, 4,
+ 16364, 8, 108, 15864, 2972, 2008, 15728, 252, 16312, 4,
+ 16372, 16368, 160, 15792, 2756, 2268, 15724, 228, 16328, 0,
+ 16380, 16344, 200, 15748, 2520, 2520, 15748, 200, 16344, 16380,
+};
+
+static const uint16_t filter_10tap_16p_150[90] = {
+ 64, 0, 15956, 1048, 2716, 1048, 15956, 0, 64, 0,
+ 52, 24, 15952, 896, 2708, 1204, 15972, 16356, 72, 16380,
+ 44, 48, 15952, 748, 2684, 1360, 16000, 16320, 84, 16380,
+ 32, 68, 15964, 604, 2644, 1516, 16032, 16288, 92, 16376,
+ 24, 88, 15980, 464, 2588, 1668, 16080, 16248, 100, 16376,
+ 16, 100, 16004, 332, 2516, 1816, 16140, 16212, 108, 16376,
+ 8, 108, 16032, 212, 2428, 1956, 16208, 16172, 112, 16376,
+ 4, 116, 16060, 100, 2328, 2092, 16288, 16132, 116, 16380,
+ 0, 116, 16096, 16380, 2216, 2216, 16380, 16096, 116, 0,
+};
+
+static const uint16_t filter_10tap_16p_183[90] = {
+ 40, 16180, 16240, 1216, 2256, 1216, 16240, 16180, 40, 0,
+ 44, 16204, 16200, 1112, 2252, 1320, 16288, 16160, 36, 0,
+ 44, 16224, 16168, 1004, 2236, 1424, 16344, 16144, 28, 4,
+ 44, 16248, 16136, 900, 2208, 1524, 16, 16124, 24, 8,
+ 44, 16268, 16116, 796, 2176, 1620, 84, 16108, 12, 12,
+ 40, 16288, 16100, 692, 2132, 1712, 156, 16096, 4, 16,
+ 36, 16308, 16088, 592, 2080, 1796, 232, 16088, 16376, 20,
+ 32, 16328, 16080, 496, 2020, 1876, 316, 16080, 16360, 24,
+ 28, 16344, 16080, 404, 1952, 1952, 404, 16080, 16344, 28,
+};
+
+static const uint16_t filter_11tap_16p_upscale[99] = {
+ 60, 16216, 356, 15620, 2556, 2556, 15620, 356, 16216, 60, 0,
+ 52, 16224, 336, 15672, 2224, 2876, 15592, 368, 16208, 64, 16380,
+ 44, 16244, 304, 15744, 1876, 3176, 15596, 364, 16212, 64, 16376,
+ 36, 16264, 260, 15836, 1532, 3440, 15636, 340, 16220, 60, 16376,
+ 28, 16288, 212, 15940, 1188, 3668, 15708, 304, 16236, 56, 16376,
+ 20, 16312, 160, 16052, 856, 3848, 15820, 248, 16264, 48, 16376,
+ 12, 16336, 104, 16164, 544, 3984, 15968, 180, 16296, 36, 16376,
+ 4, 16360, 48, 16276, 256, 4068, 16160, 96, 16336, 16, 16380,
+ 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0,
+};
+
+static const uint16_t filter_11tap_16p_117[99] = {
+ 16380, 16332, 220, 15728, 2536, 2536, 15728, 220, 16332, 16380, 0,
+ 4, 16308, 256, 15704, 2280, 2768, 15772, 176, 16360, 16368, 0,
+ 12, 16292, 280, 15704, 2016, 2984, 15848, 120, 8, 16356, 0,
+ 20, 16276, 292, 15724, 1744, 3172, 15948, 56, 40, 16340, 4,
+ 24, 16268, 292, 15760, 1468, 3328, 16072, 16368, 80, 16324, 8,
+ 24, 16264, 288, 15816, 1196, 3456, 16224, 16288, 116, 16312, 12,
+ 24, 16264, 272, 15880, 932, 3548, 16, 16208, 152, 16296, 16,
+ 24, 16268, 248, 15956, 676, 3604, 216, 16120, 188, 16284, 20,
+ 24, 16276, 220, 16036, 436, 3624, 436, 16036, 220, 16276, 24,
+};
+
+static const uint16_t filter_11tap_16p_150[99] = {
+ 0, 144, 16072, 0, 2212, 2212, 0, 16072, 144, 0, 0,
+ 16376, 144, 16112, 16288, 2092, 2324, 104, 16036, 140, 8, 16380,
+ 16368, 144, 16152, 16204, 1960, 2424, 216, 16004, 132, 16, 16376,
+ 16364, 140, 16192, 16132, 1820, 2512, 340, 15976, 116, 28, 16376,
+ 16364, 132, 16232, 16072, 1676, 2584, 476, 15952, 100, 40, 16372,
+ 16360, 124, 16272, 16020, 1528, 2644, 612, 15936, 80, 52, 16368,
+ 16360, 116, 16312, 15980, 1372, 2684, 760, 15928, 56, 64, 16364,
+ 16360, 104, 16348, 15952, 1216, 2712, 908, 15928, 28, 76, 16364,
+ 16360, 92, 0, 15936, 1064, 2720, 1064, 15936, 0, 92, 16360,
+};
+
+static const uint16_t filter_11tap_16p_183[99] = {
+ 60, 16336, 16052, 412, 1948, 1948, 412, 16052, 16336, 60, 0,
+ 56, 16356, 16052, 324, 1876, 2016, 504, 16056, 16316, 64, 0,
+ 48, 16372, 16060, 240, 1796, 2072, 604, 16064, 16292, 64, 0,
+ 44, 4, 16068, 160, 1712, 2124, 700, 16080, 16272, 68, 0,
+ 40, 20, 16080, 84, 1620, 2164, 804, 16096, 16248, 68, 4,
+ 32, 32, 16096, 16, 1524, 2200, 908, 16124, 16224, 68, 4,
+ 28, 40, 16112, 16340, 1428, 2220, 1012, 16152, 16200, 64, 8,
+ 24, 52, 16132, 16284, 1328, 2236, 1120, 16192, 16176, 64, 12,
+ 16, 56, 16156, 16236, 1224, 2240, 1224, 16236, 16156, 56, 16,
+};
+
+static const uint16_t filter_12tap_16p_upscale[108] = {
+ 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0,
+ 16376, 24, 16332, 100, 16156, 4068, 260, 16272, 56, 16356, 8, 0,
+ 16368, 44, 16284, 188, 15964, 3988, 548, 16156, 112, 16328, 20, 16380,
+ 16360, 64, 16248, 260, 15812, 3856, 864, 16040, 172, 16296, 32, 16380,
+ 16360, 76, 16216, 320, 15696, 3672, 1196, 15928, 228, 16268, 44, 16376,
+ 16356, 84, 16196, 360, 15620, 3448, 1540, 15820, 280, 16240, 56, 16372,
+ 16356, 88, 16184, 384, 15580, 3188, 1888, 15728, 324, 16216, 68, 16368,
+ 16360, 88, 16180, 392, 15576, 2892, 2236, 15652, 360, 16200, 80, 16364,
+ 16360, 84, 16188, 384, 15600, 2576, 2576, 15600, 384, 16188, 84, 16360,
+};
+
+static const uint16_t filter_12tap_16p_117[108] = {
+ 48, 16248, 240, 16028, 436, 3612, 436, 16028, 240, 16248, 48, 0,
+ 44, 16260, 208, 16116, 212, 3596, 676, 15944, 272, 16240, 48, 16380,
+ 40, 16276, 168, 16204, 12, 3540, 932, 15868, 296, 16240, 48, 16380,
+ 36, 16292, 128, 16288, 16220, 3452, 1196, 15800, 312, 16240, 44, 16380,
+ 28, 16308, 84, 16372, 16064, 3324, 1472, 15748, 316, 16244, 40, 16380,
+ 24, 16328, 44, 64, 15936, 3168, 1744, 15708, 312, 16256, 32, 16380,
+ 16, 16344, 8, 132, 15836, 2980, 2016, 15688, 300, 16272, 20, 0,
+ 12, 16364, 16356, 188, 15760, 2768, 2280, 15688, 272, 16296, 8, 4,
+ 8, 16380, 16324, 236, 15712, 2532, 2532, 15712, 236, 16324, 16380, 8,
+};
+
+static const uint16_t filter_12tap_16p_150[108] = {
+ 16340, 116, 0, 15916, 1076, 2724, 1076, 15916, 0, 116, 16340, 0,
+ 16340, 100, 32, 15908, 920, 2716, 1232, 15936, 16344, 128, 16340, 0,
+ 16344, 84, 64, 15908, 772, 2692, 1388, 15968, 16304, 140, 16344, 16380,
+ 16344, 68, 92, 15912, 624, 2652, 1540, 16008, 16264, 152, 16344, 16380,
+ 16348, 52, 112, 15928, 484, 2592, 1688, 16060, 16220, 160, 16348, 16380,
+ 16352, 40, 132, 15952, 348, 2520, 1836, 16124, 16176, 168, 16356, 16376,
+ 16356, 24, 148, 15980, 224, 2436, 1976, 16200, 16132, 172, 16364, 16372,
+ 16360, 12, 160, 16012, 108, 2336, 2104, 16288, 16088, 172, 16372, 16368,
+ 16364, 0, 168, 16048, 0, 2228, 2228, 0, 16048, 168, 0, 16364,
+};
+
+static const uint16_t filter_12tap_16p_183[108] = {
+ 36, 72, 16132, 16228, 1224, 2224, 1224, 16228, 16132, 72, 36, 0,
+ 28, 80, 16156, 16184, 1120, 2224, 1328, 16280, 16112, 64, 40, 16380,
+ 24, 84, 16180, 16144, 1016, 2208, 1428, 16340, 16092, 52, 48, 16380,
+ 16, 88, 16208, 16112, 912, 2188, 1524, 16, 16072, 36, 56, 16380,
+ 12, 92, 16232, 16084, 812, 2156, 1620, 88, 16056, 24, 64, 16380,
+ 8, 92, 16256, 16064, 708, 2116, 1708, 164, 16044, 4, 68, 16380,
+ 4, 88, 16280, 16048, 608, 2068, 1792, 244, 16036, 16372, 76, 16380,
+ 0, 88, 16308, 16036, 512, 2008, 1872, 328, 16032, 16352, 80, 16380,
+ 0, 84, 16328, 16032, 416, 1944, 1944, 416, 16032, 16328, 84, 0,
+};
+
+const uint16_t *wbscl_get_filter_3tap_16p(struct fixed31_32 ratio)
+{
+ if (ratio.value < dc_fixpt_one.value)
+ return filter_3tap_16p_upscale;
+ else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
+ return filter_3tap_16p_117;
+ else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
+ return filter_3tap_16p_150;
+ else
+ return filter_3tap_16p_183;
+}
+
+const uint16_t *wbscl_get_filter_4tap_16p(struct fixed31_32 ratio)
+{
+ if (ratio.value < dc_fixpt_one.value)
+ return filter_4tap_16p_upscale;
+ else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
+ return filter_4tap_16p_117;
+ else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
+ return filter_4tap_16p_150;
+ else
+ return filter_4tap_16p_183;
+}
+
+static const uint16_t *wbscl_get_filter_5tap_16p(struct fixed31_32 ratio)
+{
+ if (ratio.value < dc_fixpt_one.value)
+ return filter_5tap_16p_upscale;
+ else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
+ return filter_5tap_16p_117;
+ else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
+ return filter_5tap_16p_150;
+ else
+ return filter_5tap_16p_183;
+}
+
+static const uint16_t *wbscl_get_filter_6tap_16p(struct fixed31_32 ratio)
+{
+ if (ratio.value < dc_fixpt_one.value)
+ return filter_6tap_16p_upscale;
+ else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
+ return filter_6tap_16p_117;
+ else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
+ return filter_6tap_16p_150;
+ else
+ return filter_6tap_16p_183;
+}
+
+static const uint16_t *wbscl_get_filter_7tap_16p(struct fixed31_32 ratio)
+{
+ if (ratio.value < dc_fixpt_one.value)
+ return filter_7tap_16p_upscale;
+ else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
+ return filter_7tap_16p_117;
+ else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
+ return filter_7tap_16p_150;
+ else
+ return filter_7tap_16p_183;
+}
+
+static const uint16_t *wbscl_get_filter_8tap_16p(struct fixed31_32 ratio)
+{
+ if (ratio.value < dc_fixpt_one.value)
+ return filter_8tap_16p_upscale;
+ else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
+ return filter_8tap_16p_117;
+ else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
+ return filter_8tap_16p_150;
+ else
+ return filter_8tap_16p_183;
+}
+
+static const uint16_t *wbscl_get_filter_9tap_16p(struct fixed31_32 ratio)
+{
+ if (ratio.value < dc_fixpt_one.value)
+ return filter_9tap_16p_upscale;
+ else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
+ return filter_9tap_16p_117;
+ else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
+ return filter_9tap_16p_150;
+ else
+ return filter_9tap_16p_183;
+}
+static const uint16_t *wbscl_get_filter_10tap_16p(struct fixed31_32 ratio)
+{
+ if (ratio.value < dc_fixpt_one.value)
+ return filter_10tap_16p_upscale;
+ else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
+ return filter_10tap_16p_117;
+ else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
+ return filter_10tap_16p_150;
+ else
+ return filter_10tap_16p_183;
+}
+
+static const uint16_t *wbscl_get_filter_11tap_16p(struct fixed31_32 ratio)
+{
+ if (ratio.value < dc_fixpt_one.value)
+ return filter_11tap_16p_upscale;
+ else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
+ return filter_11tap_16p_117;
+ else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
+ return filter_11tap_16p_150;
+ else
+ return filter_11tap_16p_183;
+}
+
+static const uint16_t *wbscl_get_filter_12tap_16p(struct fixed31_32 ratio)
+{
+ if (ratio.value < dc_fixpt_one.value)
+ return filter_12tap_16p_upscale;
+ else if (ratio.value < dc_fixpt_from_fraction(4, 3).value)
+ return filter_12tap_16p_117;
+ else if (ratio.value < dc_fixpt_from_fraction(5, 3).value)
+ return filter_12tap_16p_150;
+ else
+ return filter_12tap_16p_183;
+}
+
+static const uint16_t *wbscl_get_filter_coeffs_16p(int taps, struct fixed31_32 ratio)
+{
+ if (taps == 12)
+ return wbscl_get_filter_12tap_16p(ratio);
+ else if (taps == 11)
+ return wbscl_get_filter_11tap_16p(ratio);
+ else if (taps == 10)
+ return wbscl_get_filter_10tap_16p(ratio);
+ else if (taps == 9)
+ return wbscl_get_filter_9tap_16p(ratio);
+ else if (taps == 8)
+ return wbscl_get_filter_8tap_16p(ratio);
+ else if (taps == 7)
+ return wbscl_get_filter_7tap_16p(ratio);
+ else if (taps == 6)
+ return wbscl_get_filter_6tap_16p(ratio);
+ else if (taps == 5)
+ return wbscl_get_filter_5tap_16p(ratio);
+ else if (taps == 4)
+ return wbscl_get_filter_4tap_16p(ratio);
+ else if (taps == 3)
+ return wbscl_get_filter_3tap_16p(ratio);
+ else if (taps == 2)
+ return get_filter_2tap_16p();
+ else if (taps == 1)
+ return NULL;
+ else {
+ /* should never happen, bug */
+ BREAK_TO_DEBUGGER();
+ return NULL;
+ }
+}
+
+static void wbscl_set_scaler_filter(
+ struct dcn20_dwbc *dwbc20,
+ uint32_t taps,
+ enum wbscl_coef_filter_type_sel filter_type,
+ const uint16_t *filter)
+{
+ const int tap_pairs = (taps + 1) / 2;
+ int phase;
+ int pair;
+ uint16_t odd_coef, even_coef;
+
+ for (phase = 0; phase < (NUM_PHASES / 2 + 1); phase++) {
+ for (pair = 0; pair < tap_pairs; pair++) {
+ even_coef = filter[phase * taps + 2 * pair];
+ if ((pair * 2 + 1) < taps)
+ odd_coef = filter[phase * taps + 2 * pair + 1];
+ else
+ odd_coef = 0;
+
+ REG_SET_3(WBSCL_COEF_RAM_SELECT, 0,
+ WBSCL_COEF_RAM_TAP_PAIR_IDX, pair,
+ WBSCL_COEF_RAM_PHASE, phase,
+ WBSCL_COEF_RAM_FILTER_TYPE, filter_type);
+
+ REG_SET_4(WBSCL_COEF_RAM_TAP_DATA, 0,
+ /* Even tap coefficient (bits 1:0 fixed to 0) */
+ WBSCL_COEF_RAM_EVEN_TAP_COEF, even_coef,
+ /* Write/read control for even coefficient */
+ WBSCL_COEF_RAM_EVEN_TAP_COEF_EN, 1,
+ /* Odd tap coefficient (bits 1:0 fixed to 0) */
+ WBSCL_COEF_RAM_ODD_TAP_COEF, odd_coef,
+ /* Write/read control for odd coefficient */
+ WBSCL_COEF_RAM_ODD_TAP_COEF_EN, 1);
+ }
+ }
+}
+
+bool dwb_program_horz_scalar(struct dcn20_dwbc *dwbc20,
+ uint32_t src_width,
+ uint32_t dest_width,
+ struct scaling_taps num_taps)
+{
+ uint32_t h_ratio_luma = 1;
+ uint32_t h_ratio_chroma = 1;
+ uint32_t h_taps_luma = num_taps.h_taps;
+ uint32_t h_taps_chroma = num_taps.h_taps_c;
+ int32_t h_init_phase_luma = 0;
+ int32_t h_init_phase_chroma = 0;
+ uint32_t h_init_phase_luma_int = 0;
+ uint32_t h_init_phase_luma_frac = 0;
+ uint32_t h_init_phase_chroma_int = 0;
+ uint32_t h_init_phase_chroma_frac = 0;
+ const uint16_t *filter_h = NULL;
+ const uint16_t *filter_h_c = NULL;
+
+
+ struct fixed31_32 tmp_h_init_phase_luma = dc_fixpt_from_int(0);
+ struct fixed31_32 tmp_h_init_phase_chroma = dc_fixpt_from_int(0);
+
+
+ /*Calculate ratio*/
+ struct fixed31_32 tmp_h_ratio_luma = dc_fixpt_from_fraction(
+ src_width, dest_width);
+
+ if (dc_fixpt_floor(tmp_h_ratio_luma) == 8)
+ h_ratio_luma = -1;
+ else
+ h_ratio_luma = dc_fixpt_u3d19(tmp_h_ratio_luma) << 5;
+ h_ratio_chroma = h_ratio_luma * 2;
+
+ /*Program ratio*/
+ REG_UPDATE(WBSCL_HORZ_FILTER_SCALE_RATIO, WBSCL_H_SCALE_RATIO, h_ratio_luma);
+
+ /* Program taps*/
+ REG_UPDATE(WBSCL_TAP_CONTROL, WBSCL_H_NUM_OF_TAPS_Y_RGB, h_taps_luma - 1);
+ REG_UPDATE(WBSCL_TAP_CONTROL, WBSCL_H_NUM_OF_TAPS_CBCR, h_taps_chroma - 1);
+
+ /* Calculate phase*/
+ tmp_h_init_phase_luma = dc_fixpt_add_int(tmp_h_ratio_luma, h_taps_luma + 1);
+ tmp_h_init_phase_luma = dc_fixpt_div_int(tmp_h_init_phase_luma, 2);
+ tmp_h_init_phase_luma = dc_fixpt_sub_int(tmp_h_init_phase_luma, h_taps_luma);
+
+ h_init_phase_luma = dc_fixpt_s4d19(tmp_h_init_phase_luma);
+ h_init_phase_luma_int = (h_init_phase_luma >> 19) & 0x1f;
+ h_init_phase_luma_frac = (h_init_phase_luma & 0x7ffff) << 5;
+
+ tmp_h_init_phase_chroma = dc_fixpt_mul_int(tmp_h_ratio_luma, 2);
+ tmp_h_init_phase_chroma = dc_fixpt_add_int(tmp_h_init_phase_chroma, h_taps_chroma + 1);
+ tmp_h_init_phase_chroma = dc_fixpt_div_int(tmp_h_init_phase_chroma, 2);
+ tmp_h_init_phase_chroma = dc_fixpt_sub_int(tmp_h_init_phase_chroma, h_taps_chroma);
+ tmp_h_init_phase_chroma = dc_fixpt_add(tmp_h_init_phase_chroma, dc_fixpt_from_fraction(1, 4));
+
+ h_init_phase_chroma = dc_fixpt_s4d19(tmp_h_init_phase_chroma);
+ h_init_phase_chroma_int = (h_init_phase_chroma >> 19) & 0x1f;
+ h_init_phase_chroma_frac = (h_init_phase_chroma & 0x7ffff) << 5;
+
+ /* Program phase*/
+ REG_UPDATE(WBSCL_HORZ_FILTER_INIT_Y_RGB, WBSCL_H_INIT_INT_Y_RGB, h_init_phase_luma_int);
+ REG_UPDATE(WBSCL_HORZ_FILTER_INIT_Y_RGB, WBSCL_H_INIT_FRAC_Y_RGB, h_init_phase_luma_frac);
+ REG_UPDATE(WBSCL_HORZ_FILTER_INIT_CBCR, WBSCL_H_INIT_INT_CBCR, h_init_phase_chroma_int);
+ REG_UPDATE(WBSCL_HORZ_FILTER_INIT_CBCR, WBSCL_H_INIT_FRAC_CBCR, h_init_phase_chroma_frac);
+
+ /* Program LUT coefficients*/
+ filter_h = wbscl_get_filter_coeffs_16p(
+ h_taps_luma, tmp_h_ratio_luma);
+ filter_h_c = wbscl_get_filter_coeffs_16p(
+ h_taps_chroma, dc_fixpt_from_int(h_ratio_luma * 2));
+
+ wbscl_set_scaler_filter(dwbc20, h_taps_luma,
+ WBSCL_COEF_LUMA_HORZ_FILTER, filter_h);
+
+ wbscl_set_scaler_filter(dwbc20, h_taps_chroma,
+ WBSCL_COEF_CHROMA_HORZ_FILTER, filter_h_c);
+
+ return true;
+}
+
+bool dwb_program_vert_scalar(struct dcn20_dwbc *dwbc20,
+ uint32_t src_height,
+ uint32_t dest_height,
+ struct scaling_taps num_taps,
+ enum dwb_subsample_position subsample_position)
+{
+ uint32_t v_ratio_luma = 1;
+ uint32_t v_ratio_chroma = 1;
+ uint32_t v_taps_luma = num_taps.v_taps;
+ uint32_t v_taps_chroma = num_taps.v_taps_c;
+ int32_t v_init_phase_luma = 0;
+ int32_t v_init_phase_chroma = 0;
+ uint32_t v_init_phase_luma_int = 0;
+ uint32_t v_init_phase_luma_frac = 0;
+ uint32_t v_init_phase_chroma_int = 0;
+ uint32_t v_init_phase_chroma_frac = 0;
+
+ const uint16_t *filter_v = NULL;
+ const uint16_t *filter_v_c = NULL;
+
+ struct fixed31_32 tmp_v_init_phase_luma = dc_fixpt_from_int(0);
+ struct fixed31_32 tmp_v_init_phase_chroma = dc_fixpt_from_int(0);
+
+ /*Calculate ratio*/
+ struct fixed31_32 tmp_v_ratio_luma = dc_fixpt_from_fraction(
+ src_height, dest_height);
+
+ if (dc_fixpt_floor(tmp_v_ratio_luma) == 8)
+ v_ratio_luma = -1;
+ else
+ v_ratio_luma = dc_fixpt_u3d19(tmp_v_ratio_luma) << 5;
+ v_ratio_chroma = v_ratio_luma * 2;
+
+ /*Program ratio*/
+ REG_UPDATE(WBSCL_VERT_FILTER_SCALE_RATIO, WBSCL_V_SCALE_RATIO, v_ratio_luma);
+
+ /* Program taps*/
+ REG_UPDATE(WBSCL_TAP_CONTROL, WBSCL_V_NUM_OF_TAPS_Y_RGB, v_taps_luma - 1);
+ REG_UPDATE(WBSCL_TAP_CONTROL, WBSCL_V_NUM_OF_TAPS_CBCR, v_taps_chroma - 1);
+
+ /* Calculate phase*/
+ tmp_v_init_phase_luma = dc_fixpt_add_int(tmp_v_ratio_luma, v_taps_luma + 1);
+ tmp_v_init_phase_luma = dc_fixpt_div_int(tmp_v_init_phase_luma, 2);
+ tmp_v_init_phase_luma = dc_fixpt_sub_int(tmp_v_init_phase_luma, v_taps_luma);
+
+ v_init_phase_luma = dc_fixpt_s4d19(tmp_v_init_phase_luma);
+ v_init_phase_luma_int = (v_init_phase_luma >> 19) & 0x1f;
+ v_init_phase_luma_frac = (v_init_phase_luma & 0x7ffff) << 5;
+
+ tmp_v_init_phase_chroma = dc_fixpt_mul_int(tmp_v_ratio_luma, 2);
+ tmp_v_init_phase_chroma = dc_fixpt_add_int(tmp_v_init_phase_chroma, v_taps_chroma + 1);
+ tmp_v_init_phase_chroma = dc_fixpt_div_int(tmp_v_init_phase_chroma, 2);
+ tmp_v_init_phase_chroma = dc_fixpt_sub_int(tmp_v_init_phase_chroma, v_taps_chroma);
+ if (subsample_position == DWB_COSITED_SUBSAMPLING)
+ tmp_v_init_phase_chroma = dc_fixpt_add(tmp_v_init_phase_chroma, dc_fixpt_from_fraction(1, 4));
+
+ v_init_phase_chroma = dc_fixpt_s4d19(tmp_v_init_phase_chroma);
+ v_init_phase_chroma_int = (v_init_phase_chroma >> 19) & 0x1f;
+ v_init_phase_chroma_frac = (v_init_phase_chroma & 0x7ffff) << 5;
+
+ /* Program phase*/
+ REG_UPDATE(WBSCL_VERT_FILTER_INIT_Y_RGB, WBSCL_V_INIT_INT_Y_RGB, v_init_phase_luma_int);
+ REG_UPDATE(WBSCL_VERT_FILTER_INIT_Y_RGB, WBSCL_V_INIT_FRAC_Y_RGB, v_init_phase_luma_frac);
+ REG_UPDATE(WBSCL_VERT_FILTER_INIT_CBCR, WBSCL_V_INIT_INT_CBCR, v_init_phase_chroma_int);
+ REG_UPDATE(WBSCL_VERT_FILTER_INIT_CBCR, WBSCL_V_INIT_FRAC_CBCR, v_init_phase_chroma_frac);
+
+
+ /* Program LUT coefficients*/
+ filter_v = wbscl_get_filter_coeffs_16p(
+ v_taps_luma, tmp_v_ratio_luma);
+ filter_v_c = wbscl_get_filter_coeffs_16p(
+ v_taps_chroma, dc_fixpt_from_int(v_ratio_luma * 2));
+ wbscl_set_scaler_filter(dwbc20, v_taps_luma,
+ WBSCL_COEF_LUMA_VERT_FILTER, filter_v);
+
+ wbscl_set_scaler_filter(dwbc20, v_taps_chroma,
+ WBSCL_COEF_CHROMA_VERT_FILTER, filter_v_c);
+ return true;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.c
new file mode 100644
index 000000000000..6e2dbd03f9bf
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.c
@@ -0,0 +1,592 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+
+#include "dcn20_hubbub.h"
+#include "reg_helper.h"
+
+#define REG(reg)\
+ hubbub1->regs->reg
+
+#define CTX \
+ hubbub1->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name) \
+ hubbub1->shifts->field_name, hubbub1->masks->field_name
+
+#define REG(reg)\
+ hubbub1->regs->reg
+
+#define CTX \
+ hubbub1->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name) \
+ hubbub1->shifts->field_name, hubbub1->masks->field_name
+
+#ifdef NUM_VMID
+#undef NUM_VMID
+#endif
+#define NUM_VMID 16
+
+bool hubbub2_dcc_support_swizzle(
+ enum swizzle_mode_values swizzle,
+ unsigned int bytes_per_element,
+ enum segment_order *segment_order_horz,
+ enum segment_order *segment_order_vert)
+{
+ bool standard_swizzle = false;
+ bool display_swizzle = false;
+ bool render_swizzle = false;
+
+ switch (swizzle) {
+ case DC_SW_4KB_S:
+ case DC_SW_64KB_S:
+ case DC_SW_VAR_S:
+ case DC_SW_4KB_S_X:
+ case DC_SW_64KB_S_X:
+ case DC_SW_VAR_S_X:
+ standard_swizzle = true;
+ break;
+ case DC_SW_64KB_R_X:
+ render_swizzle = true;
+ break;
+ case DC_SW_4KB_D:
+ case DC_SW_64KB_D:
+ case DC_SW_VAR_D:
+ case DC_SW_4KB_D_X:
+ case DC_SW_64KB_D_X:
+ case DC_SW_VAR_D_X:
+ display_swizzle = true;
+ break;
+ default:
+ break;
+ }
+
+ if (standard_swizzle) {
+ if (bytes_per_element == 1) {
+ *segment_order_horz = segment_order__contiguous;
+ *segment_order_vert = segment_order__na;
+ return true;
+ }
+ if (bytes_per_element == 2) {
+ *segment_order_horz = segment_order__non_contiguous;
+ *segment_order_vert = segment_order__contiguous;
+ return true;
+ }
+ if (bytes_per_element == 4) {
+ *segment_order_horz = segment_order__non_contiguous;
+ *segment_order_vert = segment_order__contiguous;
+ return true;
+ }
+ if (bytes_per_element == 8) {
+ *segment_order_horz = segment_order__na;
+ *segment_order_vert = segment_order__contiguous;
+ return true;
+ }
+ }
+ if (render_swizzle) {
+ if (bytes_per_element == 2) {
+ *segment_order_horz = segment_order__contiguous;
+ *segment_order_vert = segment_order__contiguous;
+ return true;
+ }
+ if (bytes_per_element == 4) {
+ *segment_order_horz = segment_order__non_contiguous;
+ *segment_order_vert = segment_order__contiguous;
+ return true;
+ }
+ if (bytes_per_element == 8) {
+ *segment_order_horz = segment_order__contiguous;
+ *segment_order_vert = segment_order__non_contiguous;
+ return true;
+ }
+ }
+ if (display_swizzle && bytes_per_element == 8) {
+ *segment_order_horz = segment_order__contiguous;
+ *segment_order_vert = segment_order__non_contiguous;
+ return true;
+ }
+
+ return false;
+}
+
+bool hubbub2_dcc_support_pixel_format(
+ enum surface_pixel_format format,
+ unsigned int *bytes_per_element)
+{
+ /* DML: get_bytes_per_element */
+ switch (format) {
+ case SURFACE_PIXEL_FORMAT_GRPH_ARGB1555:
+ case SURFACE_PIXEL_FORMAT_GRPH_RGB565:
+ *bytes_per_element = 2;
+ return true;
+ case SURFACE_PIXEL_FORMAT_GRPH_ARGB8888:
+ case SURFACE_PIXEL_FORMAT_GRPH_ABGR8888:
+ case SURFACE_PIXEL_FORMAT_GRPH_ARGB2101010:
+ case SURFACE_PIXEL_FORMAT_GRPH_ABGR2101010:
+ case SURFACE_PIXEL_FORMAT_GRPH_RGB111110_FIX:
+ case SURFACE_PIXEL_FORMAT_GRPH_BGR101111_FIX:
+ case SURFACE_PIXEL_FORMAT_GRPH_RGB111110_FLOAT:
+ case SURFACE_PIXEL_FORMAT_GRPH_BGR101111_FLOAT:
+ *bytes_per_element = 4;
+ return true;
+ case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616:
+ case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616F:
+ case SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F:
+ *bytes_per_element = 8;
+ return true;
+ default:
+ return false;
+ }
+}
+
+static void hubbub2_get_blk256_size(unsigned int *blk256_width, unsigned int *blk256_height,
+ unsigned int bytes_per_element)
+{
+ /* copied from DML. might want to refactor DML to leverage from DML */
+ /* DML : get_blk256_size */
+ if (bytes_per_element == 1) {
+ *blk256_width = 16;
+ *blk256_height = 16;
+ } else if (bytes_per_element == 2) {
+ *blk256_width = 16;
+ *blk256_height = 8;
+ } else if (bytes_per_element == 4) {
+ *blk256_width = 8;
+ *blk256_height = 8;
+ } else if (bytes_per_element == 8) {
+ *blk256_width = 8;
+ *blk256_height = 4;
+ }
+}
+
+static void hubbub2_det_request_size(
+ unsigned int height,
+ unsigned int width,
+ unsigned int bpe,
+ bool *req128_horz_wc,
+ bool *req128_vert_wc)
+{
+ unsigned int detile_buf_size = 164 * 1024; /* 164KB for DCN1.0 */
+
+ unsigned int blk256_height = 0;
+ unsigned int blk256_width = 0;
+ unsigned int swath_bytes_horz_wc, swath_bytes_vert_wc;
+
+ hubbub2_get_blk256_size(&blk256_width, &blk256_height, bpe);
+
+ swath_bytes_horz_wc = width * blk256_height * bpe;
+ swath_bytes_vert_wc = height * blk256_width * bpe;
+
+ *req128_horz_wc = (2 * swath_bytes_horz_wc <= detile_buf_size) ?
+ false : /* full 256B request */
+ true; /* half 128b request */
+
+ *req128_vert_wc = (2 * swath_bytes_vert_wc <= detile_buf_size) ?
+ false : /* full 256B request */
+ true; /* half 128b request */
+}
+
+bool hubbub2_get_dcc_compression_cap(struct hubbub *hubbub,
+ const struct dc_dcc_surface_param *input,
+ struct dc_surface_dcc_cap *output)
+{
+ struct dc *dc = hubbub->ctx->dc;
+ /* implement section 1.6.2.1 of DCN1_Programming_Guide.docx */
+ enum dcc_control dcc_control;
+ unsigned int bpe;
+ enum segment_order segment_order_horz, segment_order_vert;
+ bool req128_horz_wc, req128_vert_wc;
+
+ memset(output, 0, sizeof(*output));
+
+ if (dc->debug.disable_dcc == DCC_DISABLE)
+ return false;
+
+ if (!hubbub->funcs->dcc_support_pixel_format(input->format,
+ &bpe))
+ return false;
+
+ if (!hubbub->funcs->dcc_support_swizzle(input->swizzle_mode, bpe,
+ &segment_order_horz, &segment_order_vert))
+ return false;
+
+ hubbub2_det_request_size(input->surface_size.height, input->surface_size.width,
+ bpe, &req128_horz_wc, &req128_vert_wc);
+
+ if (!req128_horz_wc && !req128_vert_wc) {
+ dcc_control = dcc_control__256_256_xxx;
+ } else if (input->scan == SCAN_DIRECTION_HORIZONTAL) {
+ if (!req128_horz_wc)
+ dcc_control = dcc_control__256_256_xxx;
+ else if (segment_order_horz == segment_order__contiguous)
+ dcc_control = dcc_control__128_128_xxx;
+ else
+ dcc_control = dcc_control__256_64_64;
+ } else if (input->scan == SCAN_DIRECTION_VERTICAL) {
+ if (!req128_vert_wc)
+ dcc_control = dcc_control__256_256_xxx;
+ else if (segment_order_vert == segment_order__contiguous)
+ dcc_control = dcc_control__128_128_xxx;
+ else
+ dcc_control = dcc_control__256_64_64;
+ } else {
+ if ((req128_horz_wc &&
+ segment_order_horz == segment_order__non_contiguous) ||
+ (req128_vert_wc &&
+ segment_order_vert == segment_order__non_contiguous))
+ /* access_dir not known, must use most constraining */
+ dcc_control = dcc_control__256_64_64;
+ else
+ /* reg128 is true for either horz and vert
+ * but segment_order is contiguous
+ */
+ dcc_control = dcc_control__128_128_xxx;
+ }
+
+ /* Exception for 64KB_R_X */
+ if ((bpe == 2) && (input->swizzle_mode == DC_SW_64KB_R_X))
+ dcc_control = dcc_control__128_128_xxx;
+
+ if (dc->debug.disable_dcc == DCC_HALF_REQ_DISALBE &&
+ dcc_control != dcc_control__256_256_xxx)
+ return false;
+
+ switch (dcc_control) {
+ case dcc_control__256_256_xxx:
+ output->grph.rgb.max_uncompressed_blk_size = 256;
+ output->grph.rgb.max_compressed_blk_size = 256;
+ output->grph.rgb.independent_64b_blks = false;
+ break;
+ case dcc_control__128_128_xxx:
+ output->grph.rgb.max_uncompressed_blk_size = 128;
+ output->grph.rgb.max_compressed_blk_size = 128;
+ output->grph.rgb.independent_64b_blks = false;
+ break;
+ case dcc_control__256_64_64:
+ output->grph.rgb.max_uncompressed_blk_size = 256;
+ output->grph.rgb.max_compressed_blk_size = 64;
+ output->grph.rgb.independent_64b_blks = true;
+ break;
+ }
+ output->capable = true;
+ output->const_color_support = true;
+
+ return true;
+}
+
+static enum dcn_hubbub_page_table_depth page_table_depth_to_hw(unsigned int page_table_depth)
+{
+ enum dcn_hubbub_page_table_depth depth = 0;
+
+ switch (page_table_depth) {
+ case 1:
+ depth = DCN_PAGE_TABLE_DEPTH_1_LEVEL;
+ break;
+ case 2:
+ depth = DCN_PAGE_TABLE_DEPTH_2_LEVEL;
+ break;
+ case 3:
+ depth = DCN_PAGE_TABLE_DEPTH_3_LEVEL;
+ break;
+ case 4:
+ depth = DCN_PAGE_TABLE_DEPTH_4_LEVEL;
+ break;
+ default:
+ ASSERT(false);
+ break;
+ }
+
+ return depth;
+}
+
+static enum dcn_hubbub_page_table_block_size page_table_block_size_to_hw(unsigned int page_table_block_size)
+{
+ enum dcn_hubbub_page_table_block_size block_size = 0;
+
+ switch (page_table_block_size) {
+ case 4096:
+ block_size = DCN_PAGE_TABLE_BLOCK_SIZE_4KB;
+ break;
+ case 65536:
+ block_size = DCN_PAGE_TABLE_BLOCK_SIZE_64KB;
+ break;
+ default:
+ ASSERT(false);
+ block_size = page_table_block_size;
+ break;
+ }
+
+ return block_size;
+}
+
+void hubbub2_init_vm_ctx(struct hubbub *hubbub,
+ struct dcn_hubbub_virt_addr_config *va_config,
+ int vmid)
+{
+ struct dcn20_hubbub *hubbub1 = TO_DCN20_HUBBUB(hubbub);
+ struct dcn_vmid_page_table_config virt_config;
+
+ virt_config.page_table_start_addr = va_config->page_table_start_addr >> 12;
+ virt_config.page_table_end_addr = va_config->page_table_end_addr >> 12;
+ virt_config.depth = page_table_depth_to_hw(va_config->page_table_depth);
+ virt_config.block_size = page_table_block_size_to_hw(va_config->page_table_block_size);
+ virt_config.page_table_base_addr = va_config->page_table_base_addr;
+
+ dcn20_vmid_setup(&hubbub1->vmid[vmid], &virt_config);
+}
+
+int hubbub2_init_dchub_sys_ctx(struct hubbub *hubbub,
+ struct dcn_hubbub_phys_addr_config *pa_config)
+{
+ struct dcn20_hubbub *hubbub1 = TO_DCN20_HUBBUB(hubbub);
+ struct dcn_vmid_page_table_config phys_config;
+
+ REG_SET(DCN_VM_FB_LOCATION_BASE, 0,
+ FB_BASE, pa_config->system_aperture.fb_base >> 24);
+ REG_SET(DCN_VM_FB_LOCATION_TOP, 0,
+ FB_TOP, pa_config->system_aperture.fb_top >> 24);
+ REG_SET(DCN_VM_FB_OFFSET, 0,
+ FB_OFFSET, pa_config->system_aperture.fb_offset >> 24);
+ REG_SET(DCN_VM_AGP_BOT, 0,
+ AGP_BOT, pa_config->system_aperture.agp_bot >> 24);
+ REG_SET(DCN_VM_AGP_TOP, 0,
+ AGP_TOP, pa_config->system_aperture.agp_top >> 24);
+ REG_SET(DCN_VM_AGP_BASE, 0,
+ AGP_BASE, pa_config->system_aperture.agp_base >> 24);
+
+ if (pa_config->gart_config.page_table_start_addr != pa_config->gart_config.page_table_end_addr) {
+ phys_config.page_table_start_addr = pa_config->gart_config.page_table_start_addr >> 12;
+ phys_config.page_table_end_addr = pa_config->gart_config.page_table_end_addr >> 12;
+ phys_config.page_table_base_addr = pa_config->gart_config.page_table_base_addr;
+ phys_config.depth = 0;
+ phys_config.block_size = 0;
+ // Init VMID 0 based on PA config
+ dcn20_vmid_setup(&hubbub1->vmid[0], &phys_config);
+ }
+
+ return NUM_VMID;
+}
+
+void hubbub2_update_dchub(struct hubbub *hubbub,
+ struct dchub_init_data *dh_data)
+{
+ struct dcn20_hubbub *hubbub1 = TO_DCN20_HUBBUB(hubbub);
+
+ if (REG(DCHUBBUB_SDPIF_FB_TOP) == 0) {
+ ASSERT(false);
+ /*should not come here*/
+ return;
+ }
+ /* TODO: port code from dal2 */
+ switch (dh_data->fb_mode) {
+ case FRAME_BUFFER_MODE_ZFB_ONLY:
+ /*For ZFB case need to put DCHUB FB BASE and TOP upside down to indicate ZFB mode*/
+ REG_UPDATE(DCHUBBUB_SDPIF_FB_TOP,
+ SDPIF_FB_TOP, 0);
+
+ REG_UPDATE(DCHUBBUB_SDPIF_FB_BASE,
+ SDPIF_FB_BASE, 0x0FFFF);
+
+ REG_UPDATE(DCHUBBUB_SDPIF_AGP_BASE,
+ SDPIF_AGP_BASE, dh_data->zfb_phys_addr_base >> 22);
+
+ REG_UPDATE(DCHUBBUB_SDPIF_AGP_BOT,
+ SDPIF_AGP_BOT, dh_data->zfb_mc_base_addr >> 22);
+
+ REG_UPDATE(DCHUBBUB_SDPIF_AGP_TOP,
+ SDPIF_AGP_TOP, (dh_data->zfb_mc_base_addr +
+ dh_data->zfb_size_in_byte - 1) >> 22);
+ break;
+ case FRAME_BUFFER_MODE_MIXED_ZFB_AND_LOCAL:
+ /*Should not touch FB LOCATION (done by VBIOS on AsicInit table)*/
+
+ REG_UPDATE(DCHUBBUB_SDPIF_AGP_BASE,
+ SDPIF_AGP_BASE, dh_data->zfb_phys_addr_base >> 22);
+
+ REG_UPDATE(DCHUBBUB_SDPIF_AGP_BOT,
+ SDPIF_AGP_BOT, dh_data->zfb_mc_base_addr >> 22);
+
+ REG_UPDATE(DCHUBBUB_SDPIF_AGP_TOP,
+ SDPIF_AGP_TOP, (dh_data->zfb_mc_base_addr +
+ dh_data->zfb_size_in_byte - 1) >> 22);
+ break;
+ case FRAME_BUFFER_MODE_LOCAL_ONLY:
+ /*Should not touch FB LOCATION (done by VBIOS on AsicInit table)*/
+ REG_UPDATE(DCHUBBUB_SDPIF_AGP_BASE,
+ SDPIF_AGP_BASE, 0);
+
+ REG_UPDATE(DCHUBBUB_SDPIF_AGP_BOT,
+ SDPIF_AGP_BOT, 0X03FFFF);
+
+ REG_UPDATE(DCHUBBUB_SDPIF_AGP_TOP,
+ SDPIF_AGP_TOP, 0);
+ break;
+ default:
+ break;
+ }
+
+ dh_data->dchub_initialzied = true;
+ dh_data->dchub_info_valid = false;
+}
+
+void hubbub2_wm_read_state(struct hubbub *hubbub,
+ struct dcn_hubbub_wm *wm)
+{
+ struct dcn20_hubbub *hubbub1 = TO_DCN20_HUBBUB(hubbub);
+
+ struct dcn_hubbub_wm_set *s;
+
+ memset(wm, 0, sizeof(struct dcn_hubbub_wm));
+
+ s = &wm->sets[0];
+ s->wm_set = 0;
+ s->data_urgent = REG_READ(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A);
+ if (REG(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A))
+ s->pte_meta_urgent = REG_READ(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A);
+ if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A)) {
+ s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A);
+ s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A);
+ }
+ s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A);
+
+ s = &wm->sets[1];
+ s->wm_set = 1;
+ s->data_urgent = REG_READ(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B);
+ if (REG(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B))
+ s->pte_meta_urgent = REG_READ(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B);
+ if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B)) {
+ s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B);
+ s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B);
+ }
+ s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B);
+
+ s = &wm->sets[2];
+ s->wm_set = 2;
+ s->data_urgent = REG_READ(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C);
+ if (REG(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C))
+ s->pte_meta_urgent = REG_READ(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C);
+ if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C)) {
+ s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C);
+ s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C);
+ }
+ s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C);
+
+ s = &wm->sets[3];
+ s->wm_set = 3;
+ s->data_urgent = REG_READ(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D);
+ if (REG(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D))
+ s->pte_meta_urgent = REG_READ(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D);
+ if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D)) {
+ s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D);
+ s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D);
+ }
+ s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D);
+}
+
+void hubbub2_get_dchub_ref_freq(struct hubbub *hubbub,
+ unsigned int dccg_ref_freq_inKhz,
+ unsigned int *dchub_ref_freq_inKhz)
+{
+ struct dcn20_hubbub *hubbub1 = TO_DCN20_HUBBUB(hubbub);
+ uint32_t ref_div = 0;
+ uint32_t ref_en = 0;
+
+ REG_GET_2(DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, &ref_div,
+ DCHUBBUB_GLOBAL_TIMER_ENABLE, &ref_en);
+
+ if (ref_en) {
+ if (ref_div == 2)
+ *dchub_ref_freq_inKhz = dccg_ref_freq_inKhz / 2;
+ else
+ *dchub_ref_freq_inKhz = dccg_ref_freq_inKhz;
+
+ // DC hub reference frequency must be around 50Mhz, otherwise there may be
+ // overflow/underflow issues when doing HUBBUB programming
+ if (*dchub_ref_freq_inKhz < 40000 || *dchub_ref_freq_inKhz > 60000)
+ ASSERT_CRITICAL(false);
+
+ return;
+ } else {
+ *dchub_ref_freq_inKhz = dccg_ref_freq_inKhz;
+
+ // HUBBUB global timer must be enabled.
+ ASSERT_CRITICAL(false);
+ return;
+ }
+}
+
+static void hubbub2_program_watermarks(
+ struct hubbub *hubbub,
+ struct dcn_watermark_set *watermarks,
+ unsigned int refclk_mhz,
+ bool safe_to_lower)
+{
+ struct dcn20_hubbub *hubbub1 = TO_DCN20_HUBBUB(hubbub);
+ /*
+ * Need to clamp to max of the register values (i.e. no wrap)
+ * for dcn1, all wm registers are 21-bit wide
+ */
+ hubbub1_program_urgent_watermarks(hubbub, watermarks, refclk_mhz, safe_to_lower);
+ hubbub1_program_stutter_watermarks(hubbub, watermarks, refclk_mhz, safe_to_lower);
+ hubbub1_program_pstate_watermarks(hubbub, watermarks, refclk_mhz, safe_to_lower);
+
+ REG_SET(DCHUBBUB_ARB_SAT_LEVEL, 0,
+ DCHUBBUB_ARB_SAT_LEVEL, 60 * refclk_mhz);
+ REG_UPDATE(DCHUBBUB_ARB_DF_REQ_OUTSTAND, DCHUBBUB_ARB_MIN_REQ_OUTSTAND, 180);
+
+ hubbub1_allow_self_refresh_control(hubbub, !hubbub->ctx->dc->debug.disable_stutter);
+}
+
+static const struct hubbub_funcs hubbub2_funcs = {
+ .update_dchub = hubbub2_update_dchub,
+ .init_dchub_sys_ctx = hubbub2_init_dchub_sys_ctx,
+ .init_vm_ctx = hubbub2_init_vm_ctx,
+ .dcc_support_swizzle = hubbub2_dcc_support_swizzle,
+ .dcc_support_pixel_format = hubbub2_dcc_support_pixel_format,
+ .get_dcc_compression_cap = hubbub2_get_dcc_compression_cap,
+ .wm_read_state = hubbub2_wm_read_state,
+ .get_dchub_ref_freq = hubbub2_get_dchub_ref_freq,
+ .program_watermarks = hubbub2_program_watermarks,
+};
+
+void hubbub2_construct(struct dcn20_hubbub *hubbub,
+ struct dc_context *ctx,
+ const struct dcn_hubbub_registers *hubbub_regs,
+ const struct dcn_hubbub_shift *hubbub_shift,
+ const struct dcn_hubbub_mask *hubbub_mask)
+{
+ hubbub->base.ctx = ctx;
+
+ hubbub->base.funcs = &hubbub2_funcs;
+
+ hubbub->regs = hubbub_regs;
+ hubbub->shifts = hubbub_shift;
+ hubbub->masks = hubbub_mask;
+
+ hubbub->debug_test_index_pstate = 0xB;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.h
new file mode 100644
index 000000000000..a7b6ca26a9ad
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.h
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_HUBBUB_DCN20_H__
+#define __DC_HUBBUB_DCN20_H__
+
+#include "dcn10/dcn10_hubbub.h"
+#include "dcn20_vmid.h"
+
+#define TO_DCN20_HUBBUB(hubbub)\
+ container_of(hubbub, struct dcn20_hubbub, base)
+
+#define HUBBUB_REG_LIST_DCN20(id)\
+ HUBBUB_REG_LIST_DCN_COMMON(), \
+ HUBBUB_VM_REG_LIST(), \
+ HUBBUB_SR_WATERMARK_REG_LIST(), \
+ SR(DCHUBBUB_CRC_CTRL), \
+ SR(DCN_VM_FB_LOCATION_BASE),\
+ SR(DCN_VM_FB_LOCATION_TOP),\
+ SR(DCN_VM_FB_OFFSET),\
+ SR(DCN_VM_AGP_BOT),\
+ SR(DCN_VM_AGP_TOP),\
+ SR(DCN_VM_AGP_BASE)
+
+#define HUBBUB_MASK_SH_LIST_DCN20(mask_sh)\
+ HUBBUB_MASK_SH_LIST_DCN_COMMON(mask_sh), \
+ HUBBUB_MASK_SH_LIST_STUTTER(mask_sh), \
+ HUBBUB_SF(DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
+ HUBBUB_SF(DCN_VM_FB_LOCATION_BASE, FB_BASE, mask_sh), \
+ HUBBUB_SF(DCN_VM_FB_LOCATION_TOP, FB_TOP, mask_sh), \
+ HUBBUB_SF(DCN_VM_FB_OFFSET, FB_OFFSET, mask_sh), \
+ HUBBUB_SF(DCN_VM_AGP_BOT, AGP_BOT, mask_sh), \
+ HUBBUB_SF(DCN_VM_AGP_TOP, AGP_TOP, mask_sh), \
+ HUBBUB_SF(DCN_VM_AGP_BASE, AGP_BASE, mask_sh)
+
+struct dcn20_hubbub {
+ struct hubbub base;
+ const struct dcn_hubbub_registers *regs;
+ const struct dcn_hubbub_shift *shifts;
+ const struct dcn_hubbub_mask *masks;
+ unsigned int debug_test_index_pstate;
+ struct dcn_watermark_set watermarks;
+ struct dcn20_vmid vmid[16];
+};
+
+void hubbub2_construct(struct dcn20_hubbub *hubbub,
+ struct dc_context *ctx,
+ const struct dcn_hubbub_registers *hubbub_regs,
+ const struct dcn_hubbub_shift *hubbub_shift,
+ const struct dcn_hubbub_mask *hubbub_mask);
+
+bool hubbub2_dcc_support_swizzle(
+ enum swizzle_mode_values swizzle,
+ unsigned int bytes_per_element,
+ enum segment_order *segment_order_horz,
+ enum segment_order *segment_order_vert);
+
+bool hubbub2_dcc_support_pixel_format(
+ enum surface_pixel_format format,
+ unsigned int *bytes_per_element);
+
+bool hubbub2_get_dcc_compression_cap(struct hubbub *hubbub,
+ const struct dc_dcc_surface_param *input,
+ struct dc_surface_dcc_cap *output);
+
+bool hubbub2_initialize_vmids(struct hubbub *hubbub,
+ const struct dc_dcc_surface_param *input,
+ struct dc_surface_dcc_cap *output);
+
+int hubbub2_init_dchub_sys_ctx(struct hubbub *hubbub,
+ struct dcn_hubbub_phys_addr_config *pa_config);
+void hubbub2_init_vm_ctx(struct hubbub *hubbub,
+ struct dcn_hubbub_virt_addr_config *va_config,
+ int vmid);
+void hubbub2_update_dchub(struct hubbub *hubbub,
+ struct dchub_init_data *dh_data);
+
+void hubbub2_get_dchub_ref_freq(struct hubbub *hubbub,
+ unsigned int dccg_ref_freq_inKhz,
+ unsigned int *dchub_ref_freq_inKhz);
+
+void hubbub2_wm_read_state(struct hubbub *hubbub,
+ struct dcn_hubbub_wm *wm);
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c
new file mode 100644
index 000000000000..d3f7dd374d50
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c
@@ -0,0 +1,700 @@
+/*
+ * Copyright 2012-17 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dcn20_hubp.h"
+
+#include "dm_services.h"
+#include "dce_calcs.h"
+#include "reg_helper.h"
+#include "basics/conversion.h"
+
+#define REG(reg)\
+ hubp2->hubp_regs->reg
+
+#define CTX \
+ hubp2->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name) \
+ hubp2->hubp_shift->field_name, hubp2->hubp_mask->field_name
+
+void hubp2_update_dchub(
+ struct hubp *hubp,
+ struct dchub_init_data *dh_data)
+{
+ struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+ if (REG(DCN_VM_FB_LOCATION_TOP) == 0)
+ return;
+
+ switch (dh_data->fb_mode) {
+ case FRAME_BUFFER_MODE_ZFB_ONLY:
+ /*For ZFB case need to put DCHUB FB BASE and TOP upside down to indicate ZFB mode*/
+ REG_UPDATE(DCN_VM_FB_LOCATION_TOP,
+ FB_TOP, 0);
+
+ REG_UPDATE(DCN_VM_FB_LOCATION_BASE,
+ FB_BASE, 0xFFFFFF);
+
+ /*This field defines the 24 MSBs, bits [47:24] of the 48 bit AGP Base*/
+ REG_UPDATE(DCN_VM_AGP_BASE,
+ AGP_BASE, dh_data->zfb_phys_addr_base >> 24);
+
+ /*This field defines the bottom range of the AGP aperture and represents the 24*/
+ /*MSBs, bits [47:24] of the 48 address bits*/
+ REG_UPDATE(DCN_VM_AGP_BOT,
+ AGP_BOT, dh_data->zfb_mc_base_addr >> 24);
+
+ /*This field defines the top range of the AGP aperture and represents the 24*/
+ /*MSBs, bits [47:24] of the 48 address bits*/
+ REG_UPDATE(DCN_VM_AGP_TOP,
+ AGP_TOP, (dh_data->zfb_mc_base_addr +
+ dh_data->zfb_size_in_byte - 1) >> 24);
+ break;
+ case FRAME_BUFFER_MODE_MIXED_ZFB_AND_LOCAL:
+ /*Should not touch FB LOCATION (done by VBIOS on AsicInit table)*/
+
+ /*This field defines the 24 MSBs, bits [47:24] of the 48 bit AGP Base*/
+ REG_UPDATE(DCN_VM_AGP_BASE,
+ AGP_BASE, dh_data->zfb_phys_addr_base >> 24);
+
+ /*This field defines the bottom range of the AGP aperture and represents the 24*/
+ /*MSBs, bits [47:24] of the 48 address bits*/
+ REG_UPDATE(DCN_VM_AGP_BOT,
+ AGP_BOT, dh_data->zfb_mc_base_addr >> 24);
+
+ /*This field defines the top range of the AGP aperture and represents the 24*/
+ /*MSBs, bits [47:24] of the 48 address bits*/
+ REG_UPDATE(DCN_VM_AGP_TOP,
+ AGP_TOP, (dh_data->zfb_mc_base_addr +
+ dh_data->zfb_size_in_byte - 1) >> 24);
+ break;
+ case FRAME_BUFFER_MODE_LOCAL_ONLY:
+ /*Should not touch FB LOCATION (should be done by VBIOS)*/
+
+ /*This field defines the 24 MSBs, bits [47:24] of the 48 bit AGP Base*/
+ REG_UPDATE(DCN_VM_AGP_BASE,
+ AGP_BASE, 0);
+
+ /*This field defines the bottom range of the AGP aperture and represents the 24*/
+ /*MSBs, bits [47:24] of the 48 address bits*/
+ REG_UPDATE(DCN_VM_AGP_BOT,
+ AGP_BOT, 0xFFFFFF);
+
+ /*This field defines the top range of the AGP aperture and represents the 24*/
+ /*MSBs, bits [47:24] of the 48 address bits*/
+ REG_UPDATE(DCN_VM_AGP_TOP,
+ AGP_TOP, 0);
+ break;
+ default:
+ break;
+ }
+
+ dh_data->dchub_initialzied = true;
+ dh_data->dchub_info_valid = false;
+}
+
+void hubp2_set_vm_system_aperture_settings(struct hubp *hubp,
+ struct vm_system_aperture_param *apt)
+{
+ struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+ PHYSICAL_ADDRESS_LOC mc_vm_apt_default;
+ PHYSICAL_ADDRESS_LOC mc_vm_apt_low;
+ PHYSICAL_ADDRESS_LOC mc_vm_apt_high;
+
+ // The format of default addr is 48:12 of the 48 bit addr
+ mc_vm_apt_default.quad_part = apt->sys_default.quad_part >> 12;
+
+ // The format of high/low are 48:18 of the 48 bit addr
+ mc_vm_apt_low.quad_part = apt->sys_low.quad_part >> 18;
+ mc_vm_apt_high.quad_part = apt->sys_high.quad_part >> 18;
+
+ REG_UPDATE_2(DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
+ DCN_VM_SYSTEM_APERTURE_DEFAULT_SYSTEM, 1, /* 1 = system physical memory */
+ DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, mc_vm_apt_default.high_part);
+
+ REG_SET(DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, 0,
+ DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, mc_vm_apt_default.low_part);
+
+ REG_SET(DCN_VM_SYSTEM_APERTURE_LOW_ADDR, 0,
+ MC_VM_SYSTEM_APERTURE_LOW_ADDR, mc_vm_apt_low.quad_part);
+
+ REG_SET(DCN_VM_SYSTEM_APERTURE_HIGH_ADDR, 0,
+ MC_VM_SYSTEM_APERTURE_HIGH_ADDR, mc_vm_apt_high.quad_part);
+
+ REG_SET_2(DCN_VM_MX_L1_TLB_CNTL, 0,
+ ENABLE_L1_TLB, 1,
+ SYSTEM_ACCESS_MODE, 0x3);
+}
+
+void hubp2_program_deadline(
+ struct hubp *hubp,
+ struct _vcs_dpi_display_dlg_regs_st *dlg_attr,
+ struct _vcs_dpi_display_ttu_regs_st *ttu_attr)
+{
+ struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+ hubp1_program_deadline(hubp, dlg_attr, ttu_attr);
+
+ REG_SET(FLIP_PARAMETERS_1, 0,
+ REFCYC_PER_PTE_GROUP_FLIP_L, dlg_attr->refcyc_per_pte_group_flip_l);
+}
+
+void hubp2_vready_at_or_After_vsync(struct hubp *hubp,
+ struct _vcs_dpi_display_pipe_dest_params_st *pipe_dest)
+{
+ uint32_t value = 0;
+ struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+ /* disable_dlg_test_mode Set 9th bit to 1 to disable "dv" mode */
+ REG_WRITE(HUBPREQ_DEBUG_DB, 1 << 8);
+ /*
+ if (VSTARTUP_START - (VREADY_OFFSET+VUPDATE_WIDTH+VUPDATE_OFFSET)/htotal)
+ <= OTG_V_BLANK_END
+ Set HUBP_VREADY_AT_OR_AFTER_VSYNC = 1
+ else
+ Set HUBP_VREADY_AT_OR_AFTER_VSYNC = 0
+ */
+ if ((pipe_dest->vstartup_start - (pipe_dest->vready_offset+pipe_dest->vupdate_width
+ + pipe_dest->vupdate_offset) / pipe_dest->htotal) <= pipe_dest->vblank_end) {
+ value = 1;
+ } else
+ value = 0;
+ REG_UPDATE(DCHUBP_CNTL, HUBP_VREADY_AT_OR_AFTER_VSYNC, value);
+}
+
+static void hubp2_setup(
+ struct hubp *hubp,
+ struct _vcs_dpi_display_dlg_regs_st *dlg_attr,
+ struct _vcs_dpi_display_ttu_regs_st *ttu_attr,
+ struct _vcs_dpi_display_rq_regs_st *rq_regs,
+ struct _vcs_dpi_display_pipe_dest_params_st *pipe_dest)
+{
+ /* otg is locked when this func is called. Register are double buffered.
+ * disable the requestors is not needed
+ */
+
+ hubp2_vready_at_or_After_vsync(hubp, pipe_dest);
+ hubp1_program_requestor(hubp, rq_regs);
+ hubp2_program_deadline(hubp, dlg_attr, ttu_attr);
+
+}
+
+void hubp2_setup_interdependent(
+ struct hubp *hubp,
+ struct _vcs_dpi_display_dlg_regs_st *dlg_attr,
+ struct _vcs_dpi_display_ttu_regs_st *ttu_attr)
+{
+ struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+ REG_SET_2(PREFETCH_SETTINGS, 0,
+ DST_Y_PREFETCH, dlg_attr->dst_y_prefetch,
+ VRATIO_PREFETCH, dlg_attr->vratio_prefetch);
+
+ REG_SET(PREFETCH_SETTINGS_C, 0,
+ VRATIO_PREFETCH_C, dlg_attr->vratio_prefetch_c);
+
+ REG_SET_2(VBLANK_PARAMETERS_0, 0,
+ DST_Y_PER_VM_VBLANK, dlg_attr->dst_y_per_vm_vblank,
+ DST_Y_PER_ROW_VBLANK, dlg_attr->dst_y_per_row_vblank);
+
+ REG_SET_2(FLIP_PARAMETERS_0, 0,
+ DST_Y_PER_VM_FLIP, dlg_attr->dst_y_per_vm_flip,
+ DST_Y_PER_ROW_FLIP, dlg_attr->dst_y_per_row_flip);
+
+ REG_SET(VBLANK_PARAMETERS_3, 0,
+ REFCYC_PER_META_CHUNK_VBLANK_L, dlg_attr->refcyc_per_meta_chunk_vblank_l);
+
+ REG_SET(VBLANK_PARAMETERS_4, 0,
+ REFCYC_PER_META_CHUNK_VBLANK_C, dlg_attr->refcyc_per_meta_chunk_vblank_c);
+
+ REG_SET(FLIP_PARAMETERS_2, 0,
+ REFCYC_PER_META_CHUNK_FLIP_L, dlg_attr->refcyc_per_meta_chunk_flip_l);
+
+ REG_SET_2(PER_LINE_DELIVERY_PRE, 0,
+ REFCYC_PER_LINE_DELIVERY_PRE_L, dlg_attr->refcyc_per_line_delivery_pre_l,
+ REFCYC_PER_LINE_DELIVERY_PRE_C, dlg_attr->refcyc_per_line_delivery_pre_c);
+
+ REG_SET(DCN_SURF0_TTU_CNTL1, 0,
+ REFCYC_PER_REQ_DELIVERY_PRE,
+ ttu_attr->refcyc_per_req_delivery_pre_l);
+ REG_SET(DCN_SURF1_TTU_CNTL1, 0,
+ REFCYC_PER_REQ_DELIVERY_PRE,
+ ttu_attr->refcyc_per_req_delivery_pre_c);
+ REG_SET(DCN_CUR0_TTU_CNTL1, 0,
+ REFCYC_PER_REQ_DELIVERY_PRE, ttu_attr->refcyc_per_req_delivery_pre_cur0);
+ REG_SET(DCN_CUR1_TTU_CNTL1, 0,
+ REFCYC_PER_REQ_DELIVERY_PRE, ttu_attr->refcyc_per_req_delivery_pre_cur1);
+
+ REG_SET_2(DCN_GLOBAL_TTU_CNTL, 0,
+ MIN_TTU_VBLANK, ttu_attr->min_ttu_vblank,
+ QoS_LEVEL_FLIP, ttu_attr->qos_level_flip);
+}
+
+/* DCN2 (GFX10), the following GFX fields are deprecated. They can be set but they will not be used:
+ * NUM_BANKS
+ * NUM_SE
+ * NUM_RB_PER_SE
+ * RB_ALIGNED
+ * Other things can be defaulted, since they never change:
+ * PIPE_ALIGNED = 0
+ * META_LINEAR = 0
+ * In GFX10, only these apply:
+ * PIPE_INTERLEAVE
+ * NUM_PIPES
+ * MAX_COMPRESSED_FRAGS
+ * SW_MODE
+ */
+static void hubp2_program_tiling(
+ struct dcn20_hubp *hubp2,
+ const union dc_tiling_info *info,
+ const enum surface_pixel_format pixel_format)
+{
+ REG_UPDATE_3(DCSURF_ADDR_CONFIG,
+ NUM_PIPES, log_2(info->gfx9.num_pipes),
+ PIPE_INTERLEAVE, info->gfx9.pipe_interleave,
+ MAX_COMPRESSED_FRAGS, log_2(info->gfx9.max_compressed_frags));
+
+ REG_UPDATE_4(DCSURF_TILING_CONFIG,
+ SW_MODE, info->gfx9.swizzle,
+ META_LINEAR, 0,
+ RB_ALIGNED, 0,
+ PIPE_ALIGNED, 0);
+}
+
+void hubp2_program_surface_config(
+ struct hubp *hubp,
+ enum surface_pixel_format format,
+ union dc_tiling_info *tiling_info,
+ union plane_size *plane_size,
+ enum dc_rotation_angle rotation,
+ struct dc_plane_dcc_param *dcc,
+ bool horizontal_mirror,
+ unsigned int compat_level)
+{
+ struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+ hubp1_dcc_control(hubp, dcc->enable, dcc->grph.independent_64b_blks);
+ hubp2_program_tiling(hubp2, tiling_info, format);
+ hubp1_program_size(hubp, format, plane_size, dcc);
+ hubp1_program_rotation(hubp, rotation, horizontal_mirror);
+ hubp1_program_pixel_format(hubp, format);
+}
+
+enum cursor_lines_per_chunk hubp2_get_lines_per_chunk(
+ unsigned int cursor_width,
+ enum dc_cursor_color_format cursor_mode)
+{
+ enum cursor_lines_per_chunk line_per_chunk = CURSOR_LINE_PER_CHUNK_16;
+
+ if (cursor_mode == CURSOR_MODE_MONO)
+ line_per_chunk = CURSOR_LINE_PER_CHUNK_16;
+ else if (cursor_mode == CURSOR_MODE_COLOR_1BIT_AND ||
+ cursor_mode == CURSOR_MODE_COLOR_PRE_MULTIPLIED_ALPHA ||
+ cursor_mode == CURSOR_MODE_COLOR_UN_PRE_MULTIPLIED_ALPHA) {
+ if (cursor_width >= 1 && cursor_width <= 32)
+ line_per_chunk = CURSOR_LINE_PER_CHUNK_16;
+ else if (cursor_width >= 33 && cursor_width <= 64)
+ line_per_chunk = CURSOR_LINE_PER_CHUNK_8;
+ else if (cursor_width >= 65 && cursor_width <= 128)
+ line_per_chunk = CURSOR_LINE_PER_CHUNK_4;
+ else if (cursor_width >= 129 && cursor_width <= 256)
+ line_per_chunk = CURSOR_LINE_PER_CHUNK_2;
+ } else if (cursor_mode == CURSOR_MODE_COLOR_64BIT_FP_PRE_MULTIPLIED ||
+ cursor_mode == CURSOR_MODE_COLOR_64BIT_FP_UN_PRE_MULTIPLIED) {
+ if (cursor_width >= 1 && cursor_width <= 16)
+ line_per_chunk = CURSOR_LINE_PER_CHUNK_16;
+ else if (cursor_width >= 17 && cursor_width <= 32)
+ line_per_chunk = CURSOR_LINE_PER_CHUNK_8;
+ else if (cursor_width >= 33 && cursor_width <= 64)
+ line_per_chunk = CURSOR_LINE_PER_CHUNK_4;
+ else if (cursor_width >= 65 && cursor_width <= 128)
+ line_per_chunk = CURSOR_LINE_PER_CHUNK_2;
+ else if (cursor_width >= 129 && cursor_width <= 256)
+ line_per_chunk = CURSOR_LINE_PER_CHUNK_1;
+ }
+
+ return line_per_chunk;
+}
+
+void hubp2_cursor_set_attributes(
+ struct hubp *hubp,
+ const struct dc_cursor_attributes *attr)
+{
+ struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+ enum cursor_pitch hw_pitch = hubp1_get_cursor_pitch(attr->pitch);
+ enum cursor_lines_per_chunk lpc = hubp2_get_lines_per_chunk(
+ attr->width, attr->color_format);
+
+ hubp->curs_attr = *attr;
+
+ REG_UPDATE(CURSOR_SURFACE_ADDRESS_HIGH,
+ CURSOR_SURFACE_ADDRESS_HIGH, attr->address.high_part);
+ REG_UPDATE(CURSOR_SURFACE_ADDRESS,
+ CURSOR_SURFACE_ADDRESS, attr->address.low_part);
+
+ REG_UPDATE_2(CURSOR_SIZE,
+ CURSOR_WIDTH, attr->width,
+ CURSOR_HEIGHT, attr->height);
+
+ REG_UPDATE_4(CURSOR_CONTROL,
+ CURSOR_MODE, attr->color_format,
+ CURSOR_2X_MAGNIFY, attr->attribute_flags.bits.ENABLE_MAGNIFICATION,
+ CURSOR_PITCH, hw_pitch,
+ CURSOR_LINES_PER_CHUNK, lpc);
+
+ REG_SET_2(CURSOR_SETTINGS, 0,
+ /* no shift of the cursor HDL schedule */
+ CURSOR0_DST_Y_OFFSET, 0,
+ /* used to shift the cursor chunk request deadline */
+ CURSOR0_CHUNK_HDL_ADJUST, 3);
+}
+
+void hubp2_dmdata_set_attributes(
+ struct hubp *hubp,
+ const struct dc_dmdata_attributes *attr)
+{
+ struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+ if (attr->dmdata_mode == DMDATA_HW_MODE) {
+ /* set to HW mode */
+ REG_UPDATE(DMDATA_CNTL,
+ DMDATA_MODE, 1);
+
+ /* for DMDATA flip, need to use SURFACE_UPDATE_LOCK */
+ REG_UPDATE(DCSURF_FLIP_CONTROL, SURFACE_UPDATE_LOCK, 1);
+
+ /* toggle DMDATA_UPDATED and set repeat and size */
+ REG_UPDATE(DMDATA_CNTL,
+ DMDATA_UPDATED, 0);
+ REG_UPDATE_3(DMDATA_CNTL,
+ DMDATA_UPDATED, 1,
+ DMDATA_REPEAT, attr->dmdata_repeat,
+ DMDATA_SIZE, attr->dmdata_size);
+
+ /* set DMDATA address */
+ REG_WRITE(DMDATA_ADDRESS_LOW, attr->address.low_part);
+ REG_UPDATE(DMDATA_ADDRESS_HIGH,
+ DMDATA_ADDRESS_HIGH, attr->address.high_part);
+
+ REG_UPDATE(DCSURF_FLIP_CONTROL, SURFACE_UPDATE_LOCK, 0);
+
+ } else {
+ /* set to SW mode before loading data */
+ REG_SET(DMDATA_CNTL, 0,
+ DMDATA_MODE, 0);
+ /* toggle DMDATA_SW_UPDATED to start loading sequence */
+ REG_UPDATE(DMDATA_SW_CNTL,
+ DMDATA_SW_UPDATED, 0);
+ REG_UPDATE_3(DMDATA_SW_CNTL,
+ DMDATA_SW_UPDATED, 1,
+ DMDATA_SW_REPEAT, attr->dmdata_repeat,
+ DMDATA_SW_SIZE, attr->dmdata_size);
+ /* load data into hubp dmdata buffer */
+ hubp2_dmdata_load(hubp, attr->dmdata_size, attr->dmdata_sw_data);
+ }
+
+ /* Note that DL_DELTA must be programmed if we want to use TTU mode */
+ REG_SET_3(DMDATA_QOS_CNTL, 0,
+ DMDATA_QOS_MODE, attr->dmdata_qos_mode,
+ DMDATA_QOS_LEVEL, attr->dmdata_qos_level,
+ DMDATA_DL_DELTA, attr->dmdata_dl_delta);
+}
+
+void hubp2_dmdata_load(
+ struct hubp *hubp,
+ uint32_t dmdata_sw_size,
+ const uint32_t *dmdata_sw_data)
+{
+ int i;
+ struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+ /* load dmdata into HUBP buffer in SW mode */
+ for (i = 0; i < dmdata_sw_size / 4; i++)
+ REG_WRITE(DMDATA_SW_DATA, dmdata_sw_data[i]);
+}
+
+bool hubp2_dmdata_status_done(struct hubp *hubp)
+{
+ uint32_t status;
+ struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+ REG_GET(DMDATA_STATUS, DMDATA_DONE, &status);
+ return (status == 1);
+}
+
+bool hubp2_program_surface_flip_and_addr(
+ struct hubp *hubp,
+ const struct dc_plane_address *address,
+ bool flip_immediate)
+{
+ struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+ //program flip type
+ REG_UPDATE(DCSURF_FLIP_CONTROL,
+ SURFACE_FLIP_TYPE, flip_immediate);
+
+ // Program VMID reg
+ REG_UPDATE(VMID_SETTINGS_0,
+ VMID, address->vmid);
+
+ if (address->type == PLN_ADDR_TYPE_GRPH_STEREO) {
+ REG_UPDATE(DCSURF_FLIP_CONTROL, SURFACE_FLIP_MODE_FOR_STEREOSYNC, 0x1);
+ REG_UPDATE(DCSURF_FLIP_CONTROL, SURFACE_FLIP_IN_STEREOSYNC, 0x1);
+
+ } else {
+ // turn off stereo if not in stereo
+ REG_UPDATE(DCSURF_FLIP_CONTROL, SURFACE_FLIP_MODE_FOR_STEREOSYNC, 0x0);
+ REG_UPDATE(DCSURF_FLIP_CONTROL, SURFACE_FLIP_IN_STEREOSYNC, 0x0);
+ }
+
+
+
+ /* HW automatically latch rest of address register on write to
+ * DCSURF_PRIMARY_SURFACE_ADDRESS if SURFACE_UPDATE_LOCK is not used
+ *
+ * program high first and then the low addr, order matters!
+ */
+ switch (address->type) {
+ case PLN_ADDR_TYPE_GRAPHICS:
+ /* DCN1.0 does not support const color
+ * TODO: program DCHUBBUB_RET_PATH_DCC_CFGx_0/1
+ * base on address->grph.dcc_const_color
+ * x = 0, 2, 4, 6 for pipe 0, 1, 2, 3 for rgb and luma
+ * x = 1, 3, 5, 7 for pipe 0, 1, 2, 3 for chroma
+ */
+
+ if (address->grph.addr.quad_part == 0)
+ break;
+
+ REG_UPDATE_2(DCSURF_SURFACE_CONTROL,
+ PRIMARY_SURFACE_TMZ, address->tmz_surface,
+ PRIMARY_META_SURFACE_TMZ, address->tmz_surface);
+
+ if (address->grph.meta_addr.quad_part != 0) {
+ REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH, 0,
+ PRIMARY_META_SURFACE_ADDRESS_HIGH,
+ address->grph.meta_addr.high_part);
+
+ REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS, 0,
+ PRIMARY_META_SURFACE_ADDRESS,
+ address->grph.meta_addr.low_part);
+ }
+
+ REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH, 0,
+ PRIMARY_SURFACE_ADDRESS_HIGH,
+ address->grph.addr.high_part);
+
+ REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS, 0,
+ PRIMARY_SURFACE_ADDRESS,
+ address->grph.addr.low_part);
+ break;
+ case PLN_ADDR_TYPE_VIDEO_PROGRESSIVE:
+ if (address->video_progressive.luma_addr.quad_part == 0
+ || address->video_progressive.chroma_addr.quad_part == 0)
+ break;
+
+ REG_UPDATE_4(DCSURF_SURFACE_CONTROL,
+ PRIMARY_SURFACE_TMZ, address->tmz_surface,
+ PRIMARY_SURFACE_TMZ_C, address->tmz_surface,
+ PRIMARY_META_SURFACE_TMZ, address->tmz_surface,
+ PRIMARY_META_SURFACE_TMZ_C, address->tmz_surface);
+
+ if (address->video_progressive.luma_meta_addr.quad_part != 0) {
+ REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C, 0,
+ PRIMARY_META_SURFACE_ADDRESS_HIGH_C,
+ address->video_progressive.chroma_meta_addr.high_part);
+
+ REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS_C, 0,
+ PRIMARY_META_SURFACE_ADDRESS_C,
+ address->video_progressive.chroma_meta_addr.low_part);
+
+ REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH, 0,
+ PRIMARY_META_SURFACE_ADDRESS_HIGH,
+ address->video_progressive.luma_meta_addr.high_part);
+
+ REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS, 0,
+ PRIMARY_META_SURFACE_ADDRESS,
+ address->video_progressive.luma_meta_addr.low_part);
+ }
+
+ REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C, 0,
+ PRIMARY_SURFACE_ADDRESS_HIGH_C,
+ address->video_progressive.chroma_addr.high_part);
+
+ REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_C, 0,
+ PRIMARY_SURFACE_ADDRESS_C,
+ address->video_progressive.chroma_addr.low_part);
+
+ REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH, 0,
+ PRIMARY_SURFACE_ADDRESS_HIGH,
+ address->video_progressive.luma_addr.high_part);
+
+ REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS, 0,
+ PRIMARY_SURFACE_ADDRESS,
+ address->video_progressive.luma_addr.low_part);
+ break;
+ case PLN_ADDR_TYPE_GRPH_STEREO:
+ if (address->grph_stereo.left_addr.quad_part == 0)
+ break;
+ if (address->grph_stereo.right_addr.quad_part == 0)
+ break;
+
+ REG_UPDATE_8(DCSURF_SURFACE_CONTROL,
+ PRIMARY_SURFACE_TMZ, address->tmz_surface,
+ PRIMARY_SURFACE_TMZ_C, address->tmz_surface,
+ PRIMARY_META_SURFACE_TMZ, address->tmz_surface,
+ PRIMARY_META_SURFACE_TMZ_C, address->tmz_surface,
+ SECONDARY_SURFACE_TMZ, address->tmz_surface,
+ SECONDARY_SURFACE_TMZ_C, address->tmz_surface,
+ SECONDARY_META_SURFACE_TMZ, address->tmz_surface,
+ SECONDARY_META_SURFACE_TMZ_C, address->tmz_surface);
+
+ if (address->grph_stereo.right_meta_addr.quad_part != 0) {
+
+ REG_SET(DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH, 0,
+ SECONDARY_META_SURFACE_ADDRESS_HIGH,
+ address->grph_stereo.right_meta_addr.high_part);
+
+ REG_SET(DCSURF_SECONDARY_META_SURFACE_ADDRESS, 0,
+ SECONDARY_META_SURFACE_ADDRESS,
+ address->grph_stereo.right_meta_addr.low_part);
+ }
+ if (address->grph_stereo.left_meta_addr.quad_part != 0) {
+
+ REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH, 0,
+ PRIMARY_META_SURFACE_ADDRESS_HIGH,
+ address->grph_stereo.left_meta_addr.high_part);
+
+ REG_SET(DCSURF_PRIMARY_META_SURFACE_ADDRESS, 0,
+ PRIMARY_META_SURFACE_ADDRESS,
+ address->grph_stereo.left_meta_addr.low_part);
+ }
+
+ REG_SET(DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH, 0,
+ SECONDARY_SURFACE_ADDRESS_HIGH,
+ address->grph_stereo.right_addr.high_part);
+
+ REG_SET(DCSURF_SECONDARY_SURFACE_ADDRESS, 0,
+ SECONDARY_SURFACE_ADDRESS,
+ address->grph_stereo.right_addr.low_part);
+
+ REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH, 0,
+ PRIMARY_SURFACE_ADDRESS_HIGH,
+ address->grph_stereo.left_addr.high_part);
+
+ REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS, 0,
+ PRIMARY_SURFACE_ADDRESS,
+ address->grph_stereo.left_addr.low_part);
+ break;
+ default:
+ BREAK_TO_DEBUGGER();
+ break;
+ }
+
+ hubp->request_address = *address;
+
+ return true;
+}
+
+void hubp2_enable_triplebuffer(
+ struct hubp *hubp,
+ bool enable)
+{
+ struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+ uint32_t triple_buffer_en = 0;
+ bool tri_buffer_en;
+
+ REG_GET(DCSURF_FLIP_CONTROL2, SURFACE_TRIPLE_BUFFER_ENABLE, &triple_buffer_en);
+ tri_buffer_en = (triple_buffer_en == 1);
+ if (tri_buffer_en != enable) {
+ REG_UPDATE(DCSURF_FLIP_CONTROL2,
+ SURFACE_TRIPLE_BUFFER_ENABLE, enable ? DC_TRIPLEBUFFER_ENABLE : DC_TRIPLEBUFFER_DISABLE);
+ }
+}
+
+bool hubp2_is_triplebuffer_enabled(
+ struct hubp *hubp)
+{
+ struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+ uint32_t triple_buffer_en = 0;
+
+ REG_GET(DCSURF_FLIP_CONTROL2, SURFACE_TRIPLE_BUFFER_ENABLE, &triple_buffer_en);
+
+ return (bool)triple_buffer_en;
+}
+
+void hubp2_set_flip_control_surface_gsl(struct hubp *hubp, bool enable)
+{
+ struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+ REG_UPDATE(DCSURF_FLIP_CONTROL2, SURFACE_GSL_ENABLE, enable ? 1 : 0);
+}
+
+static struct hubp_funcs dcn20_hubp_funcs = {
+ .hubp_enable_tripleBuffer = hubp2_enable_triplebuffer,
+ .hubp_is_triplebuffer_enabled = hubp2_is_triplebuffer_enabled,
+ .hubp_program_surface_flip_and_addr = hubp2_program_surface_flip_and_addr,
+ .hubp_program_surface_config = hubp2_program_surface_config,
+ .hubp_is_flip_pending = hubp1_is_flip_pending,
+ .hubp_setup = hubp2_setup,
+ .hubp_setup_interdependent = hubp2_setup_interdependent,
+ .hubp_set_vm_system_aperture_settings = hubp2_set_vm_system_aperture_settings,
+ .set_blank = hubp1_set_blank,
+ .dcc_control = hubp1_dcc_control,
+ .hubp_update_dchub = hubp2_update_dchub,
+ .mem_program_viewport = min_set_viewport,
+ .set_cursor_attributes = hubp2_cursor_set_attributes,
+ .set_cursor_position = hubp1_cursor_set_position,
+ .hubp_clk_cntl = hubp1_clk_cntl,
+ .hubp_vtg_sel = hubp1_vtg_sel,
+ .dmdata_set_attributes = hubp2_dmdata_set_attributes,
+ .dmdata_load = hubp2_dmdata_load,
+ .dmdata_status_done = hubp2_dmdata_status_done,
+ .hubp_read_state = hubp1_read_state,
+ .hubp_clear_underflow = hubp1_clear_underflow,
+ .hubp_set_flip_control_surface_gsl = hubp2_set_flip_control_surface_gsl,
+ .hubp_init = hubp1_init,
+};
+
+
+bool hubp2_construct(
+ struct dcn20_hubp *hubp2,
+ struct dc_context *ctx,
+ uint32_t inst,
+ const struct dcn_hubp2_registers *hubp_regs,
+ const struct dcn_hubp2_shift *hubp_shift,
+ const struct dcn_hubp2_mask *hubp_mask)
+{
+ hubp2->base.funcs = &dcn20_hubp_funcs;
+ hubp2->base.ctx = ctx;
+ hubp2->hubp_regs = hubp_regs;
+ hubp2->hubp_shift = hubp_shift;
+ hubp2->hubp_mask = hubp_mask;
+ hubp2->base.inst = inst;
+ hubp2->base.opp_id = OPP_ID_INVALID;
+ hubp2->base.mpcc_id = 0xf;
+
+ return true;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.h
new file mode 100644
index 000000000000..d5acc348be22
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.h
@@ -0,0 +1,277 @@
+/*
+ * Copyright 2012-17 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_MEM_INPUT_DCN20_H__
+#define __DC_MEM_INPUT_DCN20_H__
+
+#include "../dcn10/dcn10_hubp.h"
+
+#define TO_DCN20_HUBP(hubp)\
+ container_of(hubp, struct dcn20_hubp, base)
+
+#define HUBP_REG_LIST_DCN2_COMMON(id)\
+ HUBP_REG_LIST_DCN(id),\
+ HUBP_REG_LIST_DCN_VM(id),\
+ SRI(PREFETCH_SETTINGS, HUBPREQ, id),\
+ SRI(PREFETCH_SETTINGS_C, HUBPREQ, id),\
+ SRI(DCN_VM_SYSTEM_APERTURE_LOW_ADDR, HUBPREQ, id),\
+ SRI(DCN_VM_SYSTEM_APERTURE_HIGH_ADDR, HUBPREQ, id),\
+ SR(DCN_VM_FB_LOCATION_TOP),\
+ SR(DCN_VM_FB_LOCATION_BASE),\
+ SR(DCN_VM_FB_OFFSET),\
+ SR(DCN_VM_AGP_BASE),\
+ SR(DCN_VM_AGP_BOT),\
+ SR(DCN_VM_AGP_TOP),\
+ SRI(CURSOR_SETTINGS, HUBPREQ, id), \
+ SRI(CURSOR_SURFACE_ADDRESS_HIGH, CURSOR0_, id), \
+ SRI(CURSOR_SURFACE_ADDRESS, CURSOR0_, id), \
+ SRI(CURSOR_SIZE, CURSOR0_, id), \
+ SRI(CURSOR_CONTROL, CURSOR0_, id), \
+ SRI(CURSOR_POSITION, CURSOR0_, id), \
+ SRI(CURSOR_HOT_SPOT, CURSOR0_, id), \
+ SRI(CURSOR_DST_OFFSET, CURSOR0_, id), \
+ SRI(DMDATA_ADDRESS_HIGH, CURSOR0_, id), \
+ SRI(DMDATA_ADDRESS_LOW, CURSOR0_, id), \
+ SRI(DMDATA_CNTL, CURSOR0_, id), \
+ SRI(DMDATA_SW_CNTL, CURSOR0_, id), \
+ SRI(DMDATA_QOS_CNTL, CURSOR0_, id), \
+ SRI(DMDATA_SW_DATA, CURSOR0_, id), \
+ SRI(DMDATA_STATUS, CURSOR0_, id),\
+ SRI(FLIP_PARAMETERS_0, HUBPREQ, id),\
+ SRI(FLIP_PARAMETERS_1, HUBPREQ, id),\
+ SRI(FLIP_PARAMETERS_2, HUBPREQ, id),\
+ SRI(DCN_CUR1_TTU_CNTL0, HUBPREQ, id),\
+ SRI(DCN_CUR1_TTU_CNTL1, HUBPREQ, id),\
+ SRI(DCSURF_FLIP_CONTROL2, HUBPREQ, id), \
+ SRI(VMID_SETTINGS_0, HUBPREQ, id)
+
+#define HUBP_REG_LIST_DCN20(id)\
+ HUBP_REG_LIST_DCN2_COMMON(id),\
+ SR(DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB),\
+ SR(DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB)
+
+#define HUBP_MASK_SH_LIST_DCN2_COMMON(mask_sh)\
+ HUBP_MASK_SH_LIST_DCN(mask_sh),\
+ HUBP_MASK_SH_LIST_DCN_VM(mask_sh),\
+ HUBP_SF(HUBP0_DCSURF_SURFACE_CONFIG, ROTATION_ANGLE, mask_sh),\
+ HUBP_SF(HUBP0_DCSURF_SURFACE_CONFIG, H_MIRROR_EN, mask_sh),\
+ HUBP_SF(HUBPREQ0_PREFETCH_SETTINGS, DST_Y_PREFETCH, mask_sh),\
+ HUBP_SF(HUBPREQ0_PREFETCH_SETTINGS, VRATIO_PREFETCH, mask_sh),\
+ HUBP_SF(HUBPREQ0_PREFETCH_SETTINGS_C, VRATIO_PREFETCH_C, mask_sh),\
+ HUBP_SF(HUBPREQ0_DCN_VM_SYSTEM_APERTURE_LOW_ADDR, MC_VM_SYSTEM_APERTURE_LOW_ADDR, mask_sh),\
+ HUBP_SF(HUBPREQ0_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR, MC_VM_SYSTEM_APERTURE_HIGH_ADDR, mask_sh),\
+ HUBP_SF(DCN_VM_FB_LOCATION_TOP, FB_TOP, mask_sh),\
+ HUBP_SF(DCN_VM_FB_LOCATION_BASE, FB_BASE, mask_sh),\
+ HUBP_SF(DCN_VM_FB_OFFSET, FB_OFFSET, mask_sh),\
+ HUBP_SF(DCN_VM_AGP_BASE, AGP_BASE, mask_sh),\
+ HUBP_SF(DCN_VM_AGP_BOT, AGP_BOT, mask_sh),\
+ HUBP_SF(DCN_VM_AGP_TOP, AGP_TOP, mask_sh),\
+ HUBP_SF(HUBPREQ0_CURSOR_SETTINGS, CURSOR0_DST_Y_OFFSET, mask_sh), \
+ HUBP_SF(HUBPREQ0_CURSOR_SETTINGS, CURSOR0_CHUNK_HDL_ADJUST, mask_sh), \
+ HUBP_SF(CURSOR0_0_CURSOR_SURFACE_ADDRESS_HIGH, CURSOR_SURFACE_ADDRESS_HIGH, mask_sh), \
+ HUBP_SF(CURSOR0_0_CURSOR_SURFACE_ADDRESS, CURSOR_SURFACE_ADDRESS, mask_sh), \
+ HUBP_SF(CURSOR0_0_CURSOR_SIZE, CURSOR_WIDTH, mask_sh), \
+ HUBP_SF(CURSOR0_0_CURSOR_SIZE, CURSOR_HEIGHT, mask_sh), \
+ HUBP_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_MODE, mask_sh), \
+ HUBP_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_2X_MAGNIFY, mask_sh), \
+ HUBP_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_PITCH, mask_sh), \
+ HUBP_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_LINES_PER_CHUNK, mask_sh), \
+ HUBP_SF(CURSOR0_0_CURSOR_CONTROL, CURSOR_ENABLE, mask_sh), \
+ HUBP_SF(CURSOR0_0_CURSOR_POSITION, CURSOR_X_POSITION, mask_sh), \
+ HUBP_SF(CURSOR0_0_CURSOR_POSITION, CURSOR_Y_POSITION, mask_sh), \
+ HUBP_SF(CURSOR0_0_CURSOR_HOT_SPOT, CURSOR_HOT_SPOT_X, mask_sh), \
+ HUBP_SF(CURSOR0_0_CURSOR_HOT_SPOT, CURSOR_HOT_SPOT_Y, mask_sh), \
+ HUBP_SF(CURSOR0_0_CURSOR_DST_OFFSET, CURSOR_DST_X_OFFSET, mask_sh), \
+ HUBP_SF(CURSOR0_0_DMDATA_ADDRESS_HIGH, DMDATA_ADDRESS_HIGH, mask_sh), \
+ HUBP_SF(CURSOR0_0_DMDATA_CNTL, DMDATA_MODE, mask_sh), \
+ HUBP_SF(CURSOR0_0_DMDATA_CNTL, DMDATA_UPDATED, mask_sh), \
+ HUBP_SF(CURSOR0_0_DMDATA_CNTL, DMDATA_REPEAT, mask_sh), \
+ HUBP_SF(CURSOR0_0_DMDATA_CNTL, DMDATA_SIZE, mask_sh), \
+ HUBP_SF(CURSOR0_0_DMDATA_SW_CNTL, DMDATA_SW_UPDATED, mask_sh), \
+ HUBP_SF(CURSOR0_0_DMDATA_SW_CNTL, DMDATA_SW_REPEAT, mask_sh), \
+ HUBP_SF(CURSOR0_0_DMDATA_SW_CNTL, DMDATA_SW_SIZE, mask_sh), \
+ HUBP_SF(CURSOR0_0_DMDATA_QOS_CNTL, DMDATA_QOS_MODE, mask_sh), \
+ HUBP_SF(CURSOR0_0_DMDATA_QOS_CNTL, DMDATA_QOS_LEVEL, mask_sh), \
+ HUBP_SF(CURSOR0_0_DMDATA_QOS_CNTL, DMDATA_DL_DELTA, mask_sh), \
+ HUBP_SF(CURSOR0_0_DMDATA_STATUS, DMDATA_DONE, mask_sh),\
+ HUBP_SF(HUBPREQ0_FLIP_PARAMETERS_0, DST_Y_PER_VM_FLIP, mask_sh),\
+ HUBP_SF(HUBPREQ0_FLIP_PARAMETERS_0, DST_Y_PER_ROW_FLIP, mask_sh),\
+ HUBP_SF(HUBPREQ0_FLIP_PARAMETERS_1, REFCYC_PER_PTE_GROUP_FLIP_L, mask_sh),\
+ HUBP_SF(HUBPREQ0_FLIP_PARAMETERS_2, REFCYC_PER_META_CHUNK_FLIP_L, mask_sh),\
+ HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_VREADY_AT_OR_AFTER_VSYNC, mask_sh),\
+ HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_DISABLE_STOP_DATA_DURING_VM, mask_sh),\
+ HUBP_SF(HUBPREQ0_DCSURF_FLIP_CONTROL, HUBPREQ_MASTER_UPDATE_LOCK_STATUS, mask_sh),\
+ HUBP_SF(HUBPREQ0_DCSURF_FLIP_CONTROL2, SURFACE_GSL_ENABLE, mask_sh),\
+ HUBP_SF(HUBPREQ0_DCSURF_FLIP_CONTROL2, SURFACE_TRIPLE_BUFFER_ENABLE, mask_sh),\
+ HUBP_SF(HUBPREQ0_VMID_SETTINGS_0, VMID, mask_sh)
+
+#define HUBP_MASK_SH_LIST_DCN20(mask_sh)\
+ HUBP_MASK_SH_LIST_DCN2_COMMON(mask_sh),\
+ HUBP_SF(DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, DCN_VM_SYSTEM_APERTURE_DEFAULT_SYSTEM, mask_sh),\
+ HUBP_SF(DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, mask_sh),\
+ HUBP_SF(DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, mask_sh)
+
+
+#define DCN2_HUBP_REG_COMMON_VARIABLE_LIST \
+ HUBP_COMMON_REG_VARIABLE_LIST; \
+ uint32_t DMDATA_ADDRESS_HIGH; \
+ uint32_t DMDATA_ADDRESS_LOW; \
+ uint32_t DMDATA_CNTL; \
+ uint32_t DMDATA_SW_CNTL; \
+ uint32_t DMDATA_QOS_CNTL; \
+ uint32_t DMDATA_SW_DATA; \
+ uint32_t DMDATA_STATUS;\
+ uint32_t DCSURF_FLIP_CONTROL2;\
+ uint32_t FLIP_PARAMETERS_0;\
+ uint32_t FLIP_PARAMETERS_1;\
+ uint32_t FLIP_PARAMETERS_2;\
+ uint32_t DCN_CUR1_TTU_CNTL0;\
+ uint32_t DCN_CUR1_TTU_CNTL1;\
+ uint32_t VMID_SETTINGS_0;\
+ uint32_t FLIP_PARAMETERS_3;\
+ uint32_t FLIP_PARAMETERS_4;\
+ uint32_t VBLANK_PARAMETERS_5;\
+ uint32_t VBLANK_PARAMETERS_6
+
+#define DCN2_HUBP_REG_FIELD_VARIABLE_LIST(type) \
+ DCN_HUBP_REG_FIELD_LIST(type); \
+ type DMDATA_ADDRESS_HIGH;\
+ type DMDATA_MODE;\
+ type DMDATA_UPDATED;\
+ type DMDATA_REPEAT;\
+ type DMDATA_SIZE;\
+ type DMDATA_SW_UPDATED;\
+ type DMDATA_SW_REPEAT;\
+ type DMDATA_SW_SIZE;\
+ type DMDATA_QOS_MODE;\
+ type DMDATA_QOS_LEVEL;\
+ type DMDATA_DL_DELTA;\
+ type DMDATA_DONE;\
+ type DST_Y_PER_VM_FLIP;\
+ type DST_Y_PER_ROW_FLIP;\
+ type REFCYC_PER_PTE_GROUP_FLIP_L;\
+ type REFCYC_PER_META_CHUNK_FLIP_L;\
+ type HUBP_VREADY_AT_OR_AFTER_VSYNC;\
+ type HUBP_DISABLE_STOP_DATA_DURING_VM;\
+ type HUBPREQ_MASTER_UPDATE_LOCK_STATUS;\
+ type SURFACE_GSL_ENABLE;\
+ type SURFACE_TRIPLE_BUFFER_ENABLE;\
+ type VMID
+
+
+struct dcn_hubp2_registers {
+ DCN2_HUBP_REG_COMMON_VARIABLE_LIST;
+};
+
+struct dcn_hubp2_shift {
+ DCN2_HUBP_REG_FIELD_VARIABLE_LIST(uint8_t);
+};
+
+struct dcn_hubp2_mask {
+ DCN2_HUBP_REG_FIELD_VARIABLE_LIST(uint32_t);
+};
+
+struct dcn20_hubp {
+ struct hubp base;
+ struct dcn_hubp_state state;
+ const struct dcn_hubp2_registers *hubp_regs;
+ const struct dcn_hubp2_shift *hubp_shift;
+ const struct dcn_hubp2_mask *hubp_mask;
+};
+
+bool hubp2_construct(
+ struct dcn20_hubp *hubp2,
+ struct dc_context *ctx,
+ uint32_t inst,
+ const struct dcn_hubp2_registers *hubp_regs,
+ const struct dcn_hubp2_shift *hubp_shift,
+ const struct dcn_hubp2_mask *hubp_mask);
+
+void hubp2_setup_interdependent(
+ struct hubp *hubp,
+ struct _vcs_dpi_display_dlg_regs_st *dlg_attr,
+ struct _vcs_dpi_display_ttu_regs_st *ttu_attr);
+
+void hubp2_vready_at_or_After_vsync(struct hubp *hubp,
+ struct _vcs_dpi_display_pipe_dest_params_st *pipe_dest);
+
+void hubp2_update_dchub(
+ struct hubp *hubp,
+ struct dchub_init_data *dh_data);
+
+void hubp2_cursor_set_attributes(
+ struct hubp *hubp,
+ const struct dc_cursor_attributes *attr);
+
+void hubp2_set_vm_system_aperture_settings(struct hubp *hubp,
+ struct vm_system_aperture_param *apt);
+
+enum cursor_lines_per_chunk hubp2_get_lines_per_chunk(
+ unsigned int cursor_width,
+ enum dc_cursor_color_format cursor_mode);
+
+void hubp2_dmdata_set_attributes(
+ struct hubp *hubp,
+ const struct dc_dmdata_attributes *attr);
+
+void hubp2_dmdata_load(
+ struct hubp *hubp,
+ uint32_t dmdata_sw_size,
+ const uint32_t *dmdata_sw_data);
+
+bool hubp2_dmdata_status_done(struct hubp *hubp);
+
+void hubp2_enable_triplebuffer(
+ struct hubp *hubp,
+ bool enable);
+
+bool hubp2_is_triplebuffer_enabled(
+ struct hubp *hubp);
+
+void hubp2_set_flip_control_surface_gsl(struct hubp *hubp, bool enable);
+
+void hubp2_program_deadline(
+ struct hubp *hubp,
+ struct _vcs_dpi_display_dlg_regs_st *dlg_attr,
+ struct _vcs_dpi_display_ttu_regs_st *ttu_attr);
+
+bool hubp2_program_surface_flip_and_addr(
+ struct hubp *hubp,
+ const struct dc_plane_address *address,
+ bool flip_immediate);
+
+void hubp2_program_surface_config(
+ struct hubp *hubp,
+ enum surface_pixel_format format,
+ union dc_tiling_info *tiling_info,
+ union plane_size *plane_size,
+ enum dc_rotation_angle rotation,
+ struct dc_plane_dcc_param *dcc,
+ bool horizontal_mirror,
+ unsigned int compat_level);
+
+#endif /* __DC_MEM_INPUT_DCN20_H__ */
+
+
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c
new file mode 100644
index 000000000000..d810c8940129
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c
@@ -0,0 +1,2049 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+#include <linux/delay.h>
+
+#include "dm_services.h"
+#include "dm_helpers.h"
+#include "core_types.h"
+#include "resource.h"
+#include "dcn20/dcn20_resource.h"
+#include "dce110/dce110_hw_sequencer.h"
+#include "dcn10/dcn10_hw_sequencer.h"
+#include "dcn20_hwseq.h"
+#include "dce/dce_hwseq.h"
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+#include "dcn20/dcn20_dsc.h"
+#endif
+#include "abm.h"
+#include "clk_mgr.h"
+#include "dmcu.h"
+#include "hubp.h"
+#include "timing_generator.h"
+#include "opp.h"
+#include "ipp.h"
+#include "mpc.h"
+#include "mcif_wb.h"
+#include "reg_helper.h"
+#include "dcn10/dcn10_cm_common.h"
+#include "dcn10/dcn10_hubbub.h"
+#include "dcn10/dcn10_optc.h"
+#include "dc_link_dp.h"
+#include "vm_helper.h"
+#include "dccg.h"
+
+#define DC_LOGGER_INIT(logger)
+
+#define CTX \
+ hws->ctx
+#define REG(reg)\
+ hws->regs->reg
+
+#undef FN
+#define FN(reg_name, field_name) \
+ hws->shifts->field_name, hws->masks->field_name
+
+static void bios_golden_init(struct dc *dc)
+{
+ struct dc_bios *bp = dc->ctx->dc_bios;
+ int i;
+
+ /* initialize dcn global */
+ bp->funcs->enable_disp_power_gating(bp,
+ CONTROLLER_ID_D0, ASIC_PIPE_INIT);
+
+ for (i = 0; i < dc->res_pool->pipe_count; i++) {
+ /* initialize dcn per pipe */
+ bp->funcs->enable_disp_power_gating(bp,
+ CONTROLLER_ID_D0 + i, ASIC_PIPE_DISABLE);
+ }
+}
+
+static void enable_power_gating_plane(
+ struct dce_hwseq *hws,
+ bool enable)
+{
+ bool force_on = 1; /* disable power gating */
+
+ if (enable)
+ force_on = 0;
+
+ /* DCHUBP0/1/2/3/4/5 */
+ REG_UPDATE(DOMAIN0_PG_CONFIG, DOMAIN0_POWER_FORCEON, force_on);
+ REG_UPDATE(DOMAIN2_PG_CONFIG, DOMAIN2_POWER_FORCEON, force_on);
+ REG_UPDATE(DOMAIN4_PG_CONFIG, DOMAIN4_POWER_FORCEON, force_on);
+ REG_UPDATE(DOMAIN6_PG_CONFIG, DOMAIN6_POWER_FORCEON, force_on);
+ REG_UPDATE(DOMAIN8_PG_CONFIG, DOMAIN8_POWER_FORCEON, force_on);
+ /*Do not power gate DCHUB5, should be left at HW default, power on permanently*/
+ /*REG_UPDATE(DOMAIN10_PG_CONFIG, DOMAIN10_POWER_FORCEON, force_on);*/
+
+ /* DPP0/1/2/3/4/5 */
+ REG_UPDATE(DOMAIN1_PG_CONFIG, DOMAIN1_POWER_FORCEON, force_on);
+ REG_UPDATE(DOMAIN3_PG_CONFIG, DOMAIN3_POWER_FORCEON, force_on);
+ REG_UPDATE(DOMAIN5_PG_CONFIG, DOMAIN5_POWER_FORCEON, force_on);
+ REG_UPDATE(DOMAIN7_PG_CONFIG, DOMAIN7_POWER_FORCEON, force_on);
+ REG_UPDATE(DOMAIN9_PG_CONFIG, DOMAIN9_POWER_FORCEON, force_on);
+ /*Do not power gate DPP5, should be left at HW default, power on permanently*/
+ /*REG_UPDATE(DOMAIN11_PG_CONFIG, DOMAIN11_POWER_FORCEON, force_on);*/
+
+ REG_UPDATE(DOMAIN16_PG_CONFIG, DOMAIN16_POWER_FORCEON, force_on);
+ REG_UPDATE(DOMAIN17_PG_CONFIG, DOMAIN17_POWER_FORCEON, force_on);
+ REG_UPDATE(DOMAIN18_PG_CONFIG, DOMAIN18_POWER_FORCEON, force_on);
+ REG_UPDATE(DOMAIN19_PG_CONFIG, DOMAIN19_POWER_FORCEON, force_on);
+ REG_UPDATE(DOMAIN20_PG_CONFIG, DOMAIN20_POWER_FORCEON, force_on);
+ REG_UPDATE(DOMAIN21_PG_CONFIG, DOMAIN21_POWER_FORCEON, force_on);
+}
+
+static void dcn20_dccg_init(struct dce_hwseq *hws)
+{
+ /*
+ * set MICROSECOND_TIME_BASE_DIV
+ * 100Mhz refclk -> 0x120264
+ * 27Mhz refclk -> 0x12021b
+ * 48Mhz refclk -> 0x120230
+ *
+ */
+ REG_WRITE(MICROSECOND_TIME_BASE_DIV, 0x120264);
+
+ /*
+ * set MILLISECOND_TIME_BASE_DIV
+ * 100Mhz refclk -> 0x1186a0
+ * 27Mhz refclk -> 0x106978
+ * 48Mhz refclk -> 0x10bb80
+ *
+ */
+ REG_WRITE(MILLISECOND_TIME_BASE_DIV, 0x1186a0);
+
+ /* This value is dependent on the hardware pipeline delay so set once per SOC */
+ REG_WRITE(DISPCLK_FREQ_CHANGE_CNTL, 0x801003c);
+}
+
+static void disable_vga(
+ struct dce_hwseq *hws)
+{
+ REG_WRITE(D1VGA_CONTROL, 0);
+ REG_WRITE(D2VGA_CONTROL, 0);
+ REG_WRITE(D3VGA_CONTROL, 0);
+ REG_WRITE(D4VGA_CONTROL, 0);
+ REG_WRITE(D5VGA_CONTROL, 0);
+ REG_WRITE(D6VGA_CONTROL, 0);
+}
+
+void dcn20_program_tripleBuffer(
+ const struct dc *dc,
+ struct pipe_ctx *pipe_ctx,
+ bool enableTripleBuffer)
+{
+ if (pipe_ctx->plane_res.hubp && pipe_ctx->plane_res.hubp->funcs) {
+ pipe_ctx->plane_res.hubp->funcs->hubp_enable_tripleBuffer(
+ pipe_ctx->plane_res.hubp,
+ enableTripleBuffer);
+ }
+}
+
+/* Blank pixel data during initialization */
+static void dcn20_init_blank(
+ struct dc *dc,
+ struct timing_generator *tg)
+{
+ enum dc_color_space color_space;
+ struct tg_color black_color = {0};
+ struct output_pixel_processor *opp = NULL;
+ struct output_pixel_processor *bottom_opp = NULL;
+ uint32_t num_opps, opp_id_src0, opp_id_src1;
+ uint32_t otg_active_width, otg_active_height;
+
+ /* program opp dpg blank color */
+ color_space = COLOR_SPACE_SRGB;
+ color_space_to_black_color(dc, color_space, &black_color);
+
+ /* get the OTG active size */
+ tg->funcs->get_otg_active_size(tg,
+ &otg_active_width,
+ &otg_active_height);
+
+ /* get the OPTC source */
+ tg->funcs->get_optc_source(tg, &num_opps, &opp_id_src0, &opp_id_src1);
+ ASSERT(opp_id_src0 < dc->res_pool->res_cap->num_opp);
+ opp = dc->res_pool->opps[opp_id_src0];
+
+ if (num_opps == 2) {
+ otg_active_width = otg_active_width / 2;
+ ASSERT(opp_id_src1 < dc->res_pool->res_cap->num_opp);
+ bottom_opp = dc->res_pool->opps[opp_id_src1];
+ }
+
+ opp->funcs->opp_set_disp_pattern_generator(
+ opp,
+ CONTROLLER_DP_TEST_PATTERN_SOLID_COLOR,
+ COLOR_DEPTH_UNDEFINED,
+ &black_color,
+ otg_active_width,
+ otg_active_height);
+
+ if (num_opps == 2) {
+ bottom_opp->funcs->opp_set_disp_pattern_generator(
+ bottom_opp,
+ CONTROLLER_DP_TEST_PATTERN_SOLID_COLOR,
+ COLOR_DEPTH_UNDEFINED,
+ &black_color,
+ otg_active_width,
+ otg_active_height);
+ }
+
+ dcn20_hwss_wait_for_blank_complete(opp);
+}
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+static void dcn20_dsc_pg_control(
+ struct dce_hwseq *hws,
+ unsigned int dsc_inst,
+ bool power_on)
+{
+ uint32_t power_gate = power_on ? 0 : 1;
+ uint32_t pwr_status = power_on ? 0 : 2;
+ uint32_t org_ip_request_cntl = 0;
+
+ if (hws->ctx->dc->debug.disable_dsc_power_gate)
+ return;
+
+ if (REG(DOMAIN16_PG_CONFIG) == 0)
+ return;
+
+ REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
+ if (org_ip_request_cntl == 0)
+ REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 1);
+
+ switch (dsc_inst) {
+ case 0: /* DSC0 */
+ REG_UPDATE(DOMAIN16_PG_CONFIG,
+ DOMAIN16_POWER_GATE, power_gate);
+
+ REG_WAIT(DOMAIN16_PG_STATUS,
+ DOMAIN16_PGFSM_PWR_STATUS, pwr_status,
+ 1, 1000);
+ break;
+ case 1: /* DSC1 */
+ REG_UPDATE(DOMAIN17_PG_CONFIG,
+ DOMAIN17_POWER_GATE, power_gate);
+
+ REG_WAIT(DOMAIN17_PG_STATUS,
+ DOMAIN17_PGFSM_PWR_STATUS, pwr_status,
+ 1, 1000);
+ break;
+ case 2: /* DSC2 */
+ REG_UPDATE(DOMAIN18_PG_CONFIG,
+ DOMAIN18_POWER_GATE, power_gate);
+
+ REG_WAIT(DOMAIN18_PG_STATUS,
+ DOMAIN18_PGFSM_PWR_STATUS, pwr_status,
+ 1, 1000);
+ break;
+ case 3: /* DSC3 */
+ REG_UPDATE(DOMAIN19_PG_CONFIG,
+ DOMAIN19_POWER_GATE, power_gate);
+
+ REG_WAIT(DOMAIN19_PG_STATUS,
+ DOMAIN19_PGFSM_PWR_STATUS, pwr_status,
+ 1, 1000);
+ break;
+ case 4: /* DSC4 */
+ REG_UPDATE(DOMAIN20_PG_CONFIG,
+ DOMAIN20_POWER_GATE, power_gate);
+
+ REG_WAIT(DOMAIN20_PG_STATUS,
+ DOMAIN20_PGFSM_PWR_STATUS, pwr_status,
+ 1, 1000);
+ break;
+ case 5: /* DSC5 */
+ REG_UPDATE(DOMAIN21_PG_CONFIG,
+ DOMAIN21_POWER_GATE, power_gate);
+
+ REG_WAIT(DOMAIN21_PG_STATUS,
+ DOMAIN21_PGFSM_PWR_STATUS, pwr_status,
+ 1, 1000);
+ break;
+ default:
+ BREAK_TO_DEBUGGER();
+ break;
+ }
+
+ if (org_ip_request_cntl == 0)
+ REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 0);
+}
+#endif
+
+static void dcn20_dpp_pg_control(
+ struct dce_hwseq *hws,
+ unsigned int dpp_inst,
+ bool power_on)
+{
+ uint32_t power_gate = power_on ? 0 : 1;
+ uint32_t pwr_status = power_on ? 0 : 2;
+
+ if (hws->ctx->dc->debug.disable_dpp_power_gate)
+ return;
+ if (REG(DOMAIN1_PG_CONFIG) == 0)
+ return;
+
+ switch (dpp_inst) {
+ case 0: /* DPP0 */
+ REG_UPDATE(DOMAIN1_PG_CONFIG,
+ DOMAIN1_POWER_GATE, power_gate);
+
+ REG_WAIT(DOMAIN1_PG_STATUS,
+ DOMAIN1_PGFSM_PWR_STATUS, pwr_status,
+ 1, 1000);
+ break;
+ case 1: /* DPP1 */
+ REG_UPDATE(DOMAIN3_PG_CONFIG,
+ DOMAIN3_POWER_GATE, power_gate);
+
+ REG_WAIT(DOMAIN3_PG_STATUS,
+ DOMAIN3_PGFSM_PWR_STATUS, pwr_status,
+ 1, 1000);
+ break;
+ case 2: /* DPP2 */
+ REG_UPDATE(DOMAIN5_PG_CONFIG,
+ DOMAIN5_POWER_GATE, power_gate);
+
+ REG_WAIT(DOMAIN5_PG_STATUS,
+ DOMAIN5_PGFSM_PWR_STATUS, pwr_status,
+ 1, 1000);
+ break;
+ case 3: /* DPP3 */
+ REG_UPDATE(DOMAIN7_PG_CONFIG,
+ DOMAIN7_POWER_GATE, power_gate);
+
+ REG_WAIT(DOMAIN7_PG_STATUS,
+ DOMAIN7_PGFSM_PWR_STATUS, pwr_status,
+ 1, 1000);
+ break;
+ case 4: /* DPP4 */
+ REG_UPDATE(DOMAIN9_PG_CONFIG,
+ DOMAIN9_POWER_GATE, power_gate);
+
+ REG_WAIT(DOMAIN9_PG_STATUS,
+ DOMAIN9_PGFSM_PWR_STATUS, pwr_status,
+ 1, 1000);
+ break;
+ case 5: /* DPP5 */
+ /*
+ * Do not power gate DPP5, should be left at HW default, power on permanently.
+ * PG on Pipe5 is De-featured, attempting to put it to PG state may result in hard
+ * reset.
+ * REG_UPDATE(DOMAIN11_PG_CONFIG,
+ * DOMAIN11_POWER_GATE, power_gate);
+ *
+ * REG_WAIT(DOMAIN11_PG_STATUS,
+ * DOMAIN11_PGFSM_PWR_STATUS, pwr_status,
+ * 1, 1000);
+ */
+ break;
+ default:
+ BREAK_TO_DEBUGGER();
+ break;
+ }
+}
+
+
+static void dcn20_hubp_pg_control(
+ struct dce_hwseq *hws,
+ unsigned int hubp_inst,
+ bool power_on)
+{
+ uint32_t power_gate = power_on ? 0 : 1;
+ uint32_t pwr_status = power_on ? 0 : 2;
+
+ if (hws->ctx->dc->debug.disable_hubp_power_gate)
+ return;
+ if (REG(DOMAIN0_PG_CONFIG) == 0)
+ return;
+
+ switch (hubp_inst) {
+ case 0: /* DCHUBP0 */
+ REG_UPDATE(DOMAIN0_PG_CONFIG,
+ DOMAIN0_POWER_GATE, power_gate);
+
+ REG_WAIT(DOMAIN0_PG_STATUS,
+ DOMAIN0_PGFSM_PWR_STATUS, pwr_status,
+ 1, 1000);
+ break;
+ case 1: /* DCHUBP1 */
+ REG_UPDATE(DOMAIN2_PG_CONFIG,
+ DOMAIN2_POWER_GATE, power_gate);
+
+ REG_WAIT(DOMAIN2_PG_STATUS,
+ DOMAIN2_PGFSM_PWR_STATUS, pwr_status,
+ 1, 1000);
+ break;
+ case 2: /* DCHUBP2 */
+ REG_UPDATE(DOMAIN4_PG_CONFIG,
+ DOMAIN4_POWER_GATE, power_gate);
+
+ REG_WAIT(DOMAIN4_PG_STATUS,
+ DOMAIN4_PGFSM_PWR_STATUS, pwr_status,
+ 1, 1000);
+ break;
+ case 3: /* DCHUBP3 */
+ REG_UPDATE(DOMAIN6_PG_CONFIG,
+ DOMAIN6_POWER_GATE, power_gate);
+
+ REG_WAIT(DOMAIN6_PG_STATUS,
+ DOMAIN6_PGFSM_PWR_STATUS, pwr_status,
+ 1, 1000);
+ break;
+ case 4: /* DCHUBP4 */
+ REG_UPDATE(DOMAIN8_PG_CONFIG,
+ DOMAIN8_POWER_GATE, power_gate);
+
+ REG_WAIT(DOMAIN8_PG_STATUS,
+ DOMAIN8_PGFSM_PWR_STATUS, pwr_status,
+ 1, 1000);
+ break;
+ case 5: /* DCHUBP5 */
+ /*
+ * Do not power gate DCHUB5, should be left at HW default, power on permanently.
+ * PG on Pipe5 is De-featured, attempting to put it to PG state may result in hard
+ * reset.
+ * REG_UPDATE(DOMAIN10_PG_CONFIG,
+ * DOMAIN10_POWER_GATE, power_gate);
+ *
+ * REG_WAIT(DOMAIN10_PG_STATUS,
+ * DOMAIN10_PGFSM_PWR_STATUS, pwr_status,
+ * 1, 1000);
+ */
+ break;
+ default:
+ BREAK_TO_DEBUGGER();
+ break;
+ }
+}
+
+
+
+static void dcn20_plane_atomic_power_down(struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+ struct dce_hwseq *hws = dc->hwseq;
+ struct dpp *dpp = pipe_ctx->plane_res.dpp;
+
+ DC_LOGGER_INIT(dc->ctx->logger);
+
+ if (REG(DC_IP_REQUEST_CNTL)) {
+ REG_SET(DC_IP_REQUEST_CNTL, 0,
+ IP_REQUEST_EN, 1);
+ dcn20_dpp_pg_control(hws, dpp->inst, false);
+ dcn20_hubp_pg_control(hws, pipe_ctx->plane_res.hubp->inst, false);
+ dpp->funcs->dpp_reset(dpp);
+ REG_SET(DC_IP_REQUEST_CNTL, 0,
+ IP_REQUEST_EN, 0);
+ DC_LOG_DEBUG(
+ "Power gated front end %d\n", pipe_ctx->pipe_idx);
+ }
+}
+
+
+
+/* disable HW used by plane.
+ * note: cannot disable until disconnect is complete
+ */
+static void dcn20_plane_atomic_disable(struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+ struct hubp *hubp = pipe_ctx->plane_res.hubp;
+ struct dpp *dpp = pipe_ctx->plane_res.dpp;
+
+ dc->hwss.wait_for_mpcc_disconnect(dc, dc->res_pool, pipe_ctx);
+
+ /* In flip immediate with pipe splitting case GSL is used for
+ * synchronization so we must disable it when the plane is disabled.
+ */
+ if (pipe_ctx->stream_res.gsl_group != 0)
+ dcn20_setup_gsl_group_as_lock(dc, pipe_ctx, false);
+
+ dc->hwss.set_flip_control_gsl(pipe_ctx, false);
+
+ hubp->funcs->hubp_clk_cntl(hubp, false);
+
+ dpp->funcs->dpp_dppclk_control(dpp, false, false);
+
+ hubp->power_gated = true;
+ dc->optimized_required = false; /* We're powering off, no need to optimize */
+
+ dcn20_plane_atomic_power_down(dc, pipe_ctx);
+
+ pipe_ctx->stream = NULL;
+ memset(&pipe_ctx->stream_res, 0, sizeof(pipe_ctx->stream_res));
+ memset(&pipe_ctx->plane_res, 0, sizeof(pipe_ctx->plane_res));
+ pipe_ctx->top_pipe = NULL;
+ pipe_ctx->bottom_pipe = NULL;
+ pipe_ctx->plane_state = NULL;
+}
+
+
+void dcn20_disable_plane(struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+ DC_LOGGER_INIT(dc->ctx->logger);
+
+ if (!pipe_ctx->plane_res.hubp || pipe_ctx->plane_res.hubp->power_gated)
+ return;
+
+ dcn20_plane_atomic_disable(dc, pipe_ctx);
+
+ DC_LOG_DC("Power down front end %d\n",
+ pipe_ctx->pipe_idx);
+}
+
+static void dcn20_init_hw(struct dc *dc)
+{
+ int i, j;
+ struct abm *abm = dc->res_pool->abm;
+ struct dmcu *dmcu = dc->res_pool->dmcu;
+ struct dce_hwseq *hws = dc->hwseq;
+ struct dc_bios *dcb = dc->ctx->dc_bios;
+ struct resource_pool *res_pool = dc->res_pool;
+ struct dc_state *context = dc->current_state;
+ struct dc_firmware_info fw_info = { { 0 } };
+
+ if (dc->clk_mgr && dc->clk_mgr->funcs->init_clocks)
+ dc->clk_mgr->funcs->init_clocks(dc->clk_mgr);
+
+ // Initialize the dccg
+ if (res_pool->dccg->funcs->dccg_init)
+ res_pool->dccg->funcs->dccg_init(res_pool->dccg);
+
+ //Enable ability to power gate / don't force power on permanently
+ enable_power_gating_plane(dc->hwseq, true);
+
+ if (IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) {
+ REG_WRITE(RBBMIF_TIMEOUT_DIS, 0xFFFFFFFF);
+ REG_WRITE(RBBMIF_TIMEOUT_DIS_2, 0xFFFFFFFF);
+
+ dcn20_dccg_init(hws);
+
+ REG_UPDATE(DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, 2);
+ REG_UPDATE(DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_ENABLE, 1);
+ REG_WRITE(REFCLK_CNTL, 0);
+ } else {
+ if (!dcb->funcs->is_accelerated_mode(dcb)) {
+ bios_golden_init(dc);
+ if (dc->ctx->dc_bios->funcs->get_firmware_info(
+ dc->ctx->dc_bios, &fw_info) == BP_RESULT_OK) {
+ res_pool->ref_clocks.xtalin_clock_inKhz = fw_info.pll_info.crystal_frequency;
+
+ if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) {
+ if (res_pool->dccg && res_pool->hubbub) {
+
+ (res_pool->dccg->funcs->get_dccg_ref_freq)(res_pool->dccg,
+ fw_info.pll_info.crystal_frequency,
+ &res_pool->ref_clocks.dccg_ref_clock_inKhz);
+
+ (res_pool->hubbub->funcs->get_dchub_ref_freq)(res_pool->hubbub,
+ res_pool->ref_clocks.dccg_ref_clock_inKhz,
+ &res_pool->ref_clocks.dchub_ref_clock_inKhz);
+ } else {
+ // Not all ASICs have DCCG sw component
+ res_pool->ref_clocks.dccg_ref_clock_inKhz =
+ res_pool->ref_clocks.xtalin_clock_inKhz;
+ res_pool->ref_clocks.dchub_ref_clock_inKhz =
+ res_pool->ref_clocks.xtalin_clock_inKhz;
+ }
+ }
+ } else
+ ASSERT_CRITICAL(false);
+ disable_vga(dc->hwseq);
+ }
+
+ for (i = 0; i < dc->link_count; i++) {
+ /* Power up AND update implementation according to the
+ * required signal (which may be different from the
+ * default signal on connector).
+ */
+ struct dc_link *link = dc->links[i];
+
+ link->link_enc->funcs->hw_init(link->link_enc);
+ }
+ }
+
+ /* Blank pixel data with OPP DPG */
+ for (i = 0; i < dc->res_pool->timing_generator_count; i++) {
+ struct timing_generator *tg = dc->res_pool->timing_generators[i];
+
+ if (tg->funcs->is_tg_enabled(tg)) {
+ dcn20_init_blank(dc, tg);
+ }
+ }
+
+ for (i = 0; i < res_pool->timing_generator_count; i++) {
+ struct timing_generator *tg = dc->res_pool->timing_generators[i];
+
+ if (tg->funcs->is_tg_enabled(tg))
+ tg->funcs->lock(tg);
+ }
+
+ for (i = 0; i < dc->res_pool->pipe_count; i++) {
+ struct dpp *dpp = res_pool->dpps[i];
+
+ dpp->funcs->dpp_reset(dpp);
+ }
+
+ /* Reset all MPCC muxes */
+ res_pool->mpc->funcs->mpc_init(res_pool->mpc);
+
+ /* initialize OPP mpc_tree parameter */
+ for (i = 0; i < dc->res_pool->res_cap->num_opp; i++) {
+ res_pool->opps[i]->mpc_tree_params.opp_id = res_pool->opps[i]->inst;
+ res_pool->opps[i]->mpc_tree_params.opp_list = NULL;
+ for (j = 0; j < MAX_PIPES; j++)
+ res_pool->opps[i]->mpcc_disconnect_pending[j] = false;
+ }
+
+ for (i = 0; i < dc->res_pool->pipe_count; i++) {
+ struct timing_generator *tg = dc->res_pool->timing_generators[i];
+ struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
+ struct hubp *hubp = dc->res_pool->hubps[i];
+ struct dpp *dpp = dc->res_pool->dpps[i];
+
+ pipe_ctx->stream_res.tg = tg;
+ pipe_ctx->pipe_idx = i;
+
+ pipe_ctx->plane_res.hubp = hubp;
+ pipe_ctx->plane_res.dpp = dpp;
+ pipe_ctx->plane_res.mpcc_inst = dpp->inst;
+ hubp->mpcc_id = dpp->inst;
+ hubp->opp_id = OPP_ID_INVALID;
+ hubp->power_gated = false;
+ pipe_ctx->stream_res.opp = NULL;
+
+ hubp->funcs->hubp_init(hubp);
+
+ //dc->res_pool->opps[i]->mpc_tree_params.opp_id = dc->res_pool->opps[i]->inst;
+ //dc->res_pool->opps[i]->mpc_tree_params.opp_list = NULL;
+ dc->res_pool->opps[i]->mpcc_disconnect_pending[pipe_ctx->plane_res.mpcc_inst] = true;
+ pipe_ctx->stream_res.opp = dc->res_pool->opps[i];
+ /*to do*/
+ hwss1_plane_atomic_disconnect(dc, pipe_ctx);
+ }
+
+ /* initialize DWB pointer to MCIF_WB */
+ for (i = 0; i < res_pool->res_cap->num_dwb; i++)
+ res_pool->dwbc[i]->mcif = res_pool->mcif_wb[i];
+
+ for (i = 0; i < dc->res_pool->timing_generator_count; i++) {
+ struct timing_generator *tg = dc->res_pool->timing_generators[i];
+
+ if (tg->funcs->is_tg_enabled(tg))
+ tg->funcs->unlock(tg);
+ }
+
+ for (i = 0; i < dc->res_pool->pipe_count; i++) {
+ struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
+
+ dc->hwss.disable_plane(dc, pipe_ctx);
+
+ pipe_ctx->stream_res.tg = NULL;
+ pipe_ctx->plane_res.hubp = NULL;
+ }
+
+ for (i = 0; i < dc->res_pool->timing_generator_count; i++) {
+ struct timing_generator *tg = dc->res_pool->timing_generators[i];
+
+ tg->funcs->tg_init(tg);
+ }
+
+ /* end of FPGA. Below if real ASIC */
+ if (IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment))
+ return;
+
+
+ for (i = 0; i < res_pool->audio_count; i++) {
+ struct audio *audio = res_pool->audios[i];
+
+ audio->funcs->hw_init(audio);
+ }
+
+ if (abm != NULL) {
+ abm->funcs->init_backlight(abm);
+ abm->funcs->abm_init(abm);
+ }
+
+ if (dmcu != NULL)
+ dmcu->funcs->dmcu_init(dmcu);
+
+ if (abm != NULL && dmcu != NULL)
+ abm->dmcu_is_running = dmcu->funcs->is_dmcu_initialized(dmcu);
+
+ /* power AFMT HDMI memory TODO: may move to dis/en output save power*/
+ REG_WRITE(DIO_MEM_PWR_CTRL, 0);
+
+ if (!dc->debug.disable_clock_gate) {
+ /* enable all DCN clock gating */
+ REG_WRITE(DCCG_GATE_DISABLE_CNTL, 0);
+
+ REG_WRITE(DCCG_GATE_DISABLE_CNTL2, 0);
+
+ REG_UPDATE(DCFCLK_CNTL, DCFCLK_GATE_DIS, 0);
+ }
+
+}
+
+enum dc_status dcn20_enable_stream_timing(
+ struct pipe_ctx *pipe_ctx,
+ struct dc_state *context,
+ struct dc *dc)
+{
+ struct dc_stream_state *stream = pipe_ctx->stream;
+ struct drr_params params = {0};
+ unsigned int event_triggers = 0;
+
+
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ struct pipe_ctx *odm_pipe = dc_res_get_odm_bottom_pipe(pipe_ctx);
+#endif
+
+ /* by upper caller loop, pipe0 is parent pipe and be called first.
+ * back end is set up by for pipe0. Other children pipe share back end
+ * with pipe 0. No program is needed.
+ */
+ if (pipe_ctx->top_pipe != NULL)
+ return DC_OK;
+
+ /* TODO check if timing_changed, disable stream if timing changed */
+
+ if (odm_pipe)
+ pipe_ctx->stream_res.tg->funcs->set_odm_combine(
+ pipe_ctx->stream_res.tg,
+ odm_pipe->stream_res.opp->inst,
+ pipe_ctx->stream->timing.h_addressable/2,
+ pipe_ctx->stream->timing.pixel_encoding);
+ /* HW program guide assume display already disable
+ * by unplug sequence. OTG assume stop.
+ */
+ pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, true);
+
+ if (false == pipe_ctx->clock_source->funcs->program_pix_clk(
+ pipe_ctx->clock_source,
+ &pipe_ctx->stream_res.pix_clk_params,
+ &pipe_ctx->pll_settings)) {
+ BREAK_TO_DEBUGGER();
+ return DC_ERROR_UNEXPECTED;
+ }
+
+ pipe_ctx->stream_res.tg->funcs->program_timing(
+ pipe_ctx->stream_res.tg,
+ &stream->timing,
+ pipe_ctx->pipe_dlg_param.vready_offset,
+ pipe_ctx->pipe_dlg_param.vstartup_start,
+ pipe_ctx->pipe_dlg_param.vupdate_offset,
+ pipe_ctx->pipe_dlg_param.vupdate_width,
+ pipe_ctx->stream->signal,
+ true);
+
+ if (pipe_ctx->stream_res.tg->funcs->setup_global_lock)
+ pipe_ctx->stream_res.tg->funcs->setup_global_lock(
+ pipe_ctx->stream_res.tg);
+
+ if (odm_pipe)
+ odm_pipe->stream_res.opp->funcs->opp_pipe_clock_control(
+ odm_pipe->stream_res.opp,
+ true);
+
+ pipe_ctx->stream_res.opp->funcs->opp_pipe_clock_control(
+ pipe_ctx->stream_res.opp,
+ true);
+
+ dc->hwss.blank_pixel_data(dc, pipe_ctx, true);
+
+ /* VTG is within DCHUB command block. DCFCLK is always on */
+ if (false == pipe_ctx->stream_res.tg->funcs->enable_crtc(pipe_ctx->stream_res.tg)) {
+ BREAK_TO_DEBUGGER();
+ return DC_ERROR_UNEXPECTED;
+ }
+
+ dcn20_hwss_wait_for_blank_complete(pipe_ctx->stream_res.opp);
+
+ params.vertical_total_min = stream->adjust.v_total_min;
+ params.vertical_total_max = stream->adjust.v_total_max;
+ if (pipe_ctx->stream_res.tg->funcs->set_drr)
+ pipe_ctx->stream_res.tg->funcs->set_drr(
+ pipe_ctx->stream_res.tg, &params);
+
+ // DRR should set trigger event to monitor surface update event
+ if (stream->adjust.v_total_min != 0 && stream->adjust.v_total_max != 0)
+ event_triggers = 0x80;
+ if (pipe_ctx->stream_res.tg->funcs->set_static_screen_control)
+ pipe_ctx->stream_res.tg->funcs->set_static_screen_control(
+ pipe_ctx->stream_res.tg, event_triggers);
+
+ /* TODO program crtc source select for non-virtual signal*/
+ /* TODO program FMT */
+ /* TODO setup link_enc */
+ /* TODO set stream attributes */
+ /* TODO program audio */
+ /* TODO enable stream if timing changed */
+ /* TODO unblank stream if DP */
+
+ return DC_OK;
+}
+
+void dcn20_program_output_csc(struct dc *dc,
+ struct pipe_ctx *pipe_ctx,
+ enum dc_color_space colorspace,
+ uint16_t *matrix,
+ int opp_id)
+{
+ struct mpc *mpc = dc->res_pool->mpc;
+ enum mpc_output_csc_mode ocsc_mode = MPC_OUTPUT_CSC_COEF_A;
+
+ if (pipe_ctx->stream->csc_color_matrix.enable_adjustment == true) {
+ if (mpc->funcs->set_output_csc != NULL)
+ mpc->funcs->set_output_csc(mpc,
+ opp_id,
+ matrix,
+ ocsc_mode);
+ } else {
+ if (mpc->funcs->set_ocsc_default != NULL)
+ mpc->funcs->set_ocsc_default(mpc,
+ opp_id,
+ colorspace,
+ ocsc_mode);
+ }
+}
+
+bool dcn20_set_output_transfer_func(struct pipe_ctx *pipe_ctx,
+ const struct dc_stream_state *stream)
+{
+ int mpcc_id = pipe_ctx->plane_res.hubp->inst;
+ struct mpc *mpc = pipe_ctx->stream_res.opp->ctx->dc->res_pool->mpc;
+ struct pwl_params *params = NULL;
+ /*
+ * program OGAM only for the top pipe
+ * if there is a pipe split then fix diagnostic is required:
+ * how to pass OGAM parameter for stream.
+ * if programming for all pipes is required then remove condition
+ * pipe_ctx->top_pipe == NULL ,but then fix the diagnostic.
+ */
+ if ((pipe_ctx->top_pipe == NULL || dc_res_is_odm_head_pipe(pipe_ctx))
+ && mpc->funcs->set_output_gamma && stream->out_transfer_func) {
+ if (stream->out_transfer_func->type == TF_TYPE_HWPWL)
+ params = &stream->out_transfer_func->pwl;
+ else if (pipe_ctx->stream->out_transfer_func->type ==
+ TF_TYPE_DISTRIBUTED_POINTS &&
+ cm_helper_translate_curve_to_hw_format(
+ stream->out_transfer_func,
+ &mpc->blender_params, false))
+ params = &mpc->blender_params;
+ /*
+ * there is no ROM
+ */
+ if (stream->out_transfer_func->type == TF_TYPE_PREDEFINED)
+ BREAK_TO_DEBUGGER();
+ }
+ /*
+ * if above if is not executed then 'params' equal to 0 and set in bypass
+ */
+ mpc->funcs->set_output_gamma(mpc, mpcc_id, params);
+
+ return true;
+}
+
+static bool dcn20_set_blend_lut(
+ struct pipe_ctx *pipe_ctx, const struct dc_plane_state *plane_state)
+{
+ struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
+ bool result = true;
+ struct pwl_params *blend_lut = NULL;
+
+ if (plane_state->blend_tf) {
+ if (plane_state->blend_tf->type == TF_TYPE_HWPWL)
+ blend_lut = &plane_state->blend_tf->pwl;
+ else if (plane_state->blend_tf->type == TF_TYPE_DISTRIBUTED_POINTS) {
+ cm_helper_translate_curve_to_hw_format(
+ plane_state->blend_tf,
+ &dpp_base->regamma_params, false);
+ blend_lut = &dpp_base->regamma_params;
+ }
+ }
+ result = dpp_base->funcs->dpp_program_blnd_lut(dpp_base, blend_lut);
+
+ return result;
+}
+
+static bool dcn20_set_shaper_3dlut(
+ struct pipe_ctx *pipe_ctx, const struct dc_plane_state *plane_state)
+{
+ struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
+ bool result = true;
+ struct pwl_params *shaper_lut = NULL;
+
+ if (plane_state->in_shaper_func) {
+ if (plane_state->in_shaper_func->type == TF_TYPE_HWPWL)
+ shaper_lut = &plane_state->in_shaper_func->pwl;
+ else if (plane_state->in_shaper_func->type == TF_TYPE_DISTRIBUTED_POINTS) {
+ cm_helper_translate_curve_to_hw_format(
+ plane_state->in_shaper_func,
+ &dpp_base->shaper_params, true);
+ shaper_lut = &dpp_base->shaper_params;
+ }
+ }
+
+ result = dpp_base->funcs->dpp_program_shaper_lut(dpp_base, shaper_lut);
+ if (plane_state->lut3d_func &&
+ plane_state->lut3d_func->initialized == true)
+ result = dpp_base->funcs->dpp_program_3dlut(dpp_base,
+ &plane_state->lut3d_func->lut_3d);
+ else
+ result = dpp_base->funcs->dpp_program_3dlut(dpp_base, NULL);
+
+ if (plane_state->lut3d_func &&
+ plane_state->lut3d_func->initialized == true &&
+ plane_state->lut3d_func->hdr_multiplier != 0)
+ dpp_base->funcs->dpp_set_hdr_multiplier(dpp_base,
+ plane_state->lut3d_func->hdr_multiplier);
+ else
+ dpp_base->funcs->dpp_set_hdr_multiplier(dpp_base, 0x1f000);
+
+ return result;
+}
+
+bool dcn20_set_input_transfer_func(struct pipe_ctx *pipe_ctx,
+ const struct dc_plane_state *plane_state)
+{
+ struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
+ const struct dc_transfer_func *tf = NULL;
+ bool result = true;
+ bool use_degamma_ram = false;
+
+ if (dpp_base == NULL || plane_state == NULL)
+ return false;
+
+ dcn20_set_shaper_3dlut(pipe_ctx, plane_state);
+ dcn20_set_blend_lut(pipe_ctx, plane_state);
+
+ if (plane_state->in_transfer_func)
+ tf = plane_state->in_transfer_func;
+
+
+ if (tf == NULL) {
+ dpp_base->funcs->dpp_set_degamma(dpp_base,
+ IPP_DEGAMMA_MODE_BYPASS);
+ return true;
+ }
+
+ if (tf->type == TF_TYPE_HWPWL || tf->type == TF_TYPE_DISTRIBUTED_POINTS)
+ use_degamma_ram = true;
+
+ if (use_degamma_ram == true) {
+ if (tf->type == TF_TYPE_HWPWL)
+ dpp_base->funcs->dpp_program_degamma_pwl(dpp_base,
+ &tf->pwl);
+ else if (tf->type == TF_TYPE_DISTRIBUTED_POINTS) {
+ cm_helper_translate_curve_to_degamma_hw_format(tf,
+ &dpp_base->degamma_params);
+ dpp_base->funcs->dpp_program_degamma_pwl(dpp_base,
+ &dpp_base->degamma_params);
+ }
+ return true;
+ }
+ /* handle here the optimized cases when de-gamma ROM could be used.
+ *
+ */
+ if (tf->type == TF_TYPE_PREDEFINED) {
+ switch (tf->tf) {
+ case TRANSFER_FUNCTION_SRGB:
+ dpp_base->funcs->dpp_set_degamma(dpp_base,
+ IPP_DEGAMMA_MODE_HW_sRGB);
+ break;
+ case TRANSFER_FUNCTION_BT709:
+ dpp_base->funcs->dpp_set_degamma(dpp_base,
+ IPP_DEGAMMA_MODE_HW_xvYCC);
+ break;
+ case TRANSFER_FUNCTION_LINEAR:
+ dpp_base->funcs->dpp_set_degamma(dpp_base,
+ IPP_DEGAMMA_MODE_BYPASS);
+ break;
+ case TRANSFER_FUNCTION_PQ:
+ default:
+ result = false;
+ break;
+ }
+ } else if (tf->type == TF_TYPE_BYPASS)
+ dpp_base->funcs->dpp_set_degamma(dpp_base,
+ IPP_DEGAMMA_MODE_BYPASS);
+ else {
+ /*
+ * if we are here, we did not handle correctly.
+ * fix is required for this use case
+ */
+ BREAK_TO_DEBUGGER();
+ dpp_base->funcs->dpp_set_degamma(dpp_base,
+ IPP_DEGAMMA_MODE_BYPASS);
+ }
+
+ return result;
+}
+
+static void dcn20_update_odm(struct dc *dc, struct dc_state *context, struct pipe_ctx *pipe_ctx)
+{
+ struct pipe_ctx *combine_pipe = dc_res_get_odm_bottom_pipe(pipe_ctx);
+
+ if (combine_pipe)
+ pipe_ctx->stream_res.tg->funcs->set_odm_combine(
+ pipe_ctx->stream_res.tg,
+ combine_pipe->stream_res.opp->inst,
+ pipe_ctx->plane_res.scl_data.h_active,
+ pipe_ctx->stream->timing.pixel_encoding);
+ else
+ pipe_ctx->stream_res.tg->funcs->set_odm_bypass(
+ pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing);
+}
+
+void dcn20_blank_pixel_data(
+ struct dc *dc,
+ struct pipe_ctx *pipe_ctx,
+ bool blank)
+{
+ struct tg_color black_color = {0};
+ struct stream_resource *stream_res = &pipe_ctx->stream_res;
+ struct dc_stream_state *stream = pipe_ctx->stream;
+ enum dc_color_space color_space = stream->output_color_space;
+ enum controller_dp_test_pattern test_pattern = CONTROLLER_DP_TEST_PATTERN_SOLID_COLOR;
+ struct pipe_ctx *bot_odm_pipe = dc_res_get_odm_bottom_pipe(pipe_ctx);
+
+ int width = stream->timing.h_addressable + stream->timing.h_border_left + stream->timing.h_border_right;
+ int height = stream->timing.v_addressable + stream->timing.v_border_bottom + stream->timing.v_border_top;
+
+ /* get opp dpg blank color */
+ color_space_to_black_color(dc, color_space, &black_color);
+
+ if (bot_odm_pipe)
+ width = width / 2;
+
+ if (blank) {
+ if (stream_res->abm)
+ stream_res->abm->funcs->set_abm_immediate_disable(stream_res->abm);
+
+ if (dc->debug.visual_confirm != VISUAL_CONFIRM_DISABLE)
+ test_pattern = CONTROLLER_DP_TEST_PATTERN_COLORSQUARES;
+ } else {
+ test_pattern = CONTROLLER_DP_TEST_PATTERN_VIDEOMODE;
+ }
+
+ stream_res->opp->funcs->opp_set_disp_pattern_generator(
+ stream_res->opp,
+ test_pattern,
+ stream->timing.display_color_depth,
+ &black_color,
+ width,
+ height);
+
+ if (bot_odm_pipe) {
+ bot_odm_pipe->stream_res.opp->funcs->opp_set_disp_pattern_generator(
+ bot_odm_pipe->stream_res.opp,
+ dc->debug.visual_confirm != VISUAL_CONFIRM_DISABLE ?
+ CONTROLLER_DP_TEST_PATTERN_COLORRAMP : test_pattern,
+ stream->timing.display_color_depth,
+ &black_color,
+ width,
+ height);
+ }
+
+ if (!blank)
+ if (stream_res->abm) {
+ stream_res->abm->funcs->set_pipe(stream_res->abm, stream_res->tg->inst + 1);
+ stream_res->abm->funcs->set_abm_level(stream_res->abm, stream->abm_level);
+ }
+}
+
+
+static void dcn20_power_on_plane(
+ struct dce_hwseq *hws,
+ struct pipe_ctx *pipe_ctx)
+{
+ DC_LOGGER_INIT(hws->ctx->logger);
+ if (REG(DC_IP_REQUEST_CNTL)) {
+ REG_SET(DC_IP_REQUEST_CNTL, 0,
+ IP_REQUEST_EN, 1);
+ dcn20_dpp_pg_control(hws, pipe_ctx->plane_res.dpp->inst, true);
+ dcn20_hubp_pg_control(hws, pipe_ctx->plane_res.hubp->inst, true);
+ REG_SET(DC_IP_REQUEST_CNTL, 0,
+ IP_REQUEST_EN, 0);
+ DC_LOG_DEBUG(
+ "Un-gated front end for pipe %d\n", pipe_ctx->plane_res.hubp->inst);
+ }
+}
+
+void dcn20_enable_plane(
+ struct dc *dc,
+ struct pipe_ctx *pipe_ctx,
+ struct dc_state *context)
+{
+ //if (dc->debug.sanity_checks) {
+ // dcn10_verify_allow_pstate_change_high(dc);
+ //}
+ dcn20_power_on_plane(dc->hwseq, pipe_ctx);
+
+ /* enable DCFCLK current DCHUB */
+ pipe_ctx->plane_res.hubp->funcs->hubp_clk_cntl(pipe_ctx->plane_res.hubp, true);
+
+ /* make sure OPP_PIPE_CLOCK_EN = 1 */
+ pipe_ctx->stream_res.opp->funcs->opp_pipe_clock_control(
+ pipe_ctx->stream_res.opp,
+ true);
+
+/* TODO: enable/disable in dm as per update type.
+ if (plane_state) {
+ DC_LOG_DC(dc->ctx->logger,
+ "Pipe:%d 0x%x: addr hi:0x%x, "
+ "addr low:0x%x, "
+ "src: %d, %d, %d,"
+ " %d; dst: %d, %d, %d, %d;\n",
+ pipe_ctx->pipe_idx,
+ plane_state,
+ plane_state->address.grph.addr.high_part,
+ plane_state->address.grph.addr.low_part,
+ plane_state->src_rect.x,
+ plane_state->src_rect.y,
+ plane_state->src_rect.width,
+ plane_state->src_rect.height,
+ plane_state->dst_rect.x,
+ plane_state->dst_rect.y,
+ plane_state->dst_rect.width,
+ plane_state->dst_rect.height);
+
+ DC_LOG_DC(dc->ctx->logger,
+ "Pipe %d: width, height, x, y format:%d\n"
+ "viewport:%d, %d, %d, %d\n"
+ "recout: %d, %d, %d, %d\n",
+ pipe_ctx->pipe_idx,
+ plane_state->format,
+ pipe_ctx->plane_res.scl_data.viewport.width,
+ pipe_ctx->plane_res.scl_data.viewport.height,
+ pipe_ctx->plane_res.scl_data.viewport.x,
+ pipe_ctx->plane_res.scl_data.viewport.y,
+ pipe_ctx->plane_res.scl_data.recout.width,
+ pipe_ctx->plane_res.scl_data.recout.height,
+ pipe_ctx->plane_res.scl_data.recout.x,
+ pipe_ctx->plane_res.scl_data.recout.y);
+ print_rq_dlg_ttu(dc, pipe_ctx);
+ }
+*/
+ if (dc->vm_pa_config.valid) {
+ struct vm_system_aperture_param apt;
+
+ apt.sys_default.quad_part = 0;
+
+ apt.sys_low.quad_part = dc->vm_pa_config.system_aperture.start_addr;
+ apt.sys_high.quad_part = dc->vm_pa_config.system_aperture.end_addr;
+
+ // Program system aperture settings
+ pipe_ctx->plane_res.hubp->funcs->hubp_set_vm_system_aperture_settings(pipe_ctx->plane_res.hubp, &apt);
+ }
+
+// if (dc->debug.sanity_checks) {
+// dcn10_verify_allow_pstate_change_high(dc);
+// }
+}
+
+
+static void dcn20_program_pipe(
+ struct dc *dc,
+ struct pipe_ctx *pipe_ctx,
+ struct dc_state *context)
+{
+ pipe_ctx->plane_state->update_flags.bits.full_update =
+ context->commit_hints.full_update_needed ? 1 : pipe_ctx->plane_state->update_flags.bits.full_update;
+
+ if (pipe_ctx->plane_state->update_flags.bits.full_update)
+ dcn20_enable_plane(dc, pipe_ctx, context);
+
+ update_dchubp_dpp(dc, pipe_ctx, context);
+
+ set_hdr_multiplier(pipe_ctx);
+
+ if (pipe_ctx->plane_state->update_flags.bits.full_update ||
+ pipe_ctx->plane_state->update_flags.bits.in_transfer_func_change ||
+ pipe_ctx->plane_state->update_flags.bits.gamma_change)
+ dc->hwss.set_input_transfer_func(pipe_ctx, pipe_ctx->plane_state);
+
+ /* dcn10_translate_regamma_to_hw_format takes 750us to finish
+ * only do gamma programming for full update.
+ * TODO: This can be further optimized/cleaned up
+ * Always call this for now since it does memcmp inside before
+ * doing heavy calculation and programming
+ */
+ if (pipe_ctx->plane_state->update_flags.bits.full_update)
+ dc->hwss.set_output_transfer_func(pipe_ctx, pipe_ctx->stream);
+}
+
+static void dcn20_program_all_pipe_in_tree(
+ struct dc *dc,
+ struct pipe_ctx *pipe_ctx,
+ struct dc_state *context)
+{
+ if (pipe_ctx->top_pipe == NULL) {
+ bool blank = !is_pipe_tree_visible(pipe_ctx);
+
+ pipe_ctx->stream_res.tg->funcs->program_global_sync(
+ pipe_ctx->stream_res.tg,
+ pipe_ctx->pipe_dlg_param.vready_offset,
+ pipe_ctx->pipe_dlg_param.vstartup_start,
+ pipe_ctx->pipe_dlg_param.vupdate_offset,
+ pipe_ctx->pipe_dlg_param.vupdate_width);
+
+ pipe_ctx->stream_res.tg->funcs->set_vtg_params(
+ pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing);
+
+ dc->hwss.blank_pixel_data(dc, pipe_ctx, blank);
+
+ if (dc->hwss.update_odm)
+ dc->hwss.update_odm(dc, context, pipe_ctx);
+ }
+
+ if (pipe_ctx->plane_state != NULL)
+ dcn20_program_pipe(dc, pipe_ctx, context);
+
+ if (pipe_ctx->bottom_pipe != NULL && pipe_ctx->bottom_pipe != pipe_ctx)
+ dcn20_program_all_pipe_in_tree(dc, pipe_ctx->bottom_pipe, context);
+}
+
+void dcn20_pipe_control_lock_global(
+ struct dc *dc,
+ struct pipe_ctx *pipe,
+ bool lock)
+{
+ if (lock) {
+ pipe->stream_res.tg->funcs->lock_doublebuffer_enable(
+ pipe->stream_res.tg);
+ pipe->stream_res.tg->funcs->lock(pipe->stream_res.tg);
+ } else {
+ pipe->stream_res.tg->funcs->unlock(pipe->stream_res.tg);
+ pipe->stream_res.tg->funcs->wait_for_state(pipe->stream_res.tg,
+ CRTC_STATE_VACTIVE);
+ pipe->stream_res.tg->funcs->wait_for_state(pipe->stream_res.tg,
+ CRTC_STATE_VBLANK);
+ pipe->stream_res.tg->funcs->wait_for_state(pipe->stream_res.tg,
+ CRTC_STATE_VACTIVE);
+ pipe->stream_res.tg->funcs->lock_doublebuffer_disable(
+ pipe->stream_res.tg);
+ }
+}
+
+void dcn20_pipe_control_lock(
+ struct dc *dc,
+ struct pipe_ctx *pipe,
+ bool lock)
+{
+ bool flip_immediate = false;
+
+ /* use TG master update lock to lock everything on the TG
+ * therefore only top pipe need to lock
+ */
+ if (pipe->top_pipe)
+ return;
+
+ if (pipe->plane_state != NULL)
+ flip_immediate = pipe->plane_state->flip_immediate;
+
+ if (flip_immediate && lock) {
+ while (pipe->plane_res.hubp->funcs->hubp_is_flip_pending(pipe->plane_res.hubp)) {
+ udelay(1);
+ }
+
+ if (pipe->bottom_pipe != NULL)
+ while (pipe->bottom_pipe->plane_res.hubp->funcs->hubp_is_flip_pending(pipe->bottom_pipe->plane_res.hubp)) {
+ udelay(1);
+ }
+ }
+
+ /* In flip immediate and pipe splitting case, we need to use GSL
+ * for synchronization. Only do setup on locking and on flip type change.
+ */
+ if (lock && pipe->bottom_pipe != NULL)
+ if ((flip_immediate && pipe->stream_res.gsl_group == 0) ||
+ (!flip_immediate && pipe->stream_res.gsl_group > 0))
+ dcn20_setup_gsl_group_as_lock(dc, pipe, flip_immediate);
+
+ if (pipe->plane_state != NULL && pipe->plane_state->triplebuffer_flips) {
+ if (lock)
+ pipe->stream_res.tg->funcs->triplebuffer_lock(pipe->stream_res.tg);
+ else
+ pipe->stream_res.tg->funcs->triplebuffer_unlock(pipe->stream_res.tg);
+ } else {
+ if (lock)
+ pipe->stream_res.tg->funcs->lock(pipe->stream_res.tg);
+ else
+ pipe->stream_res.tg->funcs->unlock(pipe->stream_res.tg);
+ }
+}
+
+static void dcn20_apply_ctx_for_surface(
+ struct dc *dc,
+ const struct dc_stream_state *stream,
+ int num_planes,
+ struct dc_state *context)
+{
+
+ int i;
+ struct timing_generator *tg;
+ bool removed_pipe[6] = { false };
+ bool interdependent_update = false;
+ struct pipe_ctx *top_pipe_to_program =
+ find_top_pipe_for_stream(dc, context, stream);
+ DC_LOGGER_INIT(dc->ctx->logger);
+
+ if (!top_pipe_to_program)
+ return;
+
+ tg = top_pipe_to_program->stream_res.tg;
+
+ interdependent_update = top_pipe_to_program->plane_state &&
+ top_pipe_to_program->plane_state->update_flags.bits.full_update;
+
+ if (interdependent_update)
+ lock_all_pipes(dc, context, true);
+ else
+ dcn20_pipe_control_lock(dc, top_pipe_to_program, true);
+
+ if (num_planes == 0) {
+ /* OTG blank before remove all front end */
+ dc->hwss.blank_pixel_data(dc, top_pipe_to_program, true);
+ }
+
+ /* Disconnect unused mpcc */
+ for (i = 0; i < dc->res_pool->pipe_count; i++) {
+ struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
+ struct pipe_ctx *old_pipe_ctx =
+ &dc->current_state->res_ctx.pipe_ctx[i];
+ /*
+ * Powergate reused pipes that are not powergated
+ * fairly hacky right now, using opp_id as indicator
+ * TODO: After move dc_post to dc_update, this will
+ * be removed.
+ */
+ if (pipe_ctx->plane_state && !old_pipe_ctx->plane_state) {
+ if (old_pipe_ctx->stream_res.tg == tg &&
+ old_pipe_ctx->plane_res.hubp &&
+ old_pipe_ctx->plane_res.hubp->opp_id != OPP_ID_INVALID)
+ dcn20_disable_plane(dc, old_pipe_ctx);
+ }
+
+ if ((!pipe_ctx->plane_state ||
+ pipe_ctx->stream_res.tg != old_pipe_ctx->stream_res.tg) &&
+ old_pipe_ctx->plane_state &&
+ old_pipe_ctx->stream_res.tg == tg) {
+
+ dc->hwss.plane_atomic_disconnect(dc, old_pipe_ctx);
+ removed_pipe[i] = true;
+
+ DC_LOG_DC("Reset mpcc for pipe %d\n",
+ old_pipe_ctx->pipe_idx);
+ }
+ }
+
+ if (num_planes > 0)
+ dcn20_program_all_pipe_in_tree(dc, top_pipe_to_program, context);
+
+ /* Program secondary blending tree and writeback pipes */
+ if ((stream->num_wb_info > 0) && (dc->hwss.program_all_writeback_pipes_in_tree))
+ dc->hwss.program_all_writeback_pipes_in_tree(dc, stream, context);
+
+ if (interdependent_update)
+ for (i = 0; i < dc->res_pool->pipe_count; i++) {
+ struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
+
+ /* Skip inactive pipes and ones already updated */
+ if (!pipe_ctx->stream || pipe_ctx->stream == stream ||
+ !pipe_ctx->plane_state || !tg->funcs->is_tg_enabled(tg))
+ continue;
+
+ pipe_ctx->plane_res.hubp->funcs->hubp_setup_interdependent(
+ pipe_ctx->plane_res.hubp,
+ &pipe_ctx->dlg_regs,
+ &pipe_ctx->ttu_regs);
+ }
+
+ if (interdependent_update)
+ lock_all_pipes(dc, context, false);
+ else
+ dcn20_pipe_control_lock(dc, top_pipe_to_program, false);
+
+ for (i = 0; i < dc->res_pool->pipe_count; i++)
+ if (removed_pipe[i])
+ dcn20_disable_plane(dc, &dc->current_state->res_ctx.pipe_ctx[i]);
+}
+
+
+void dcn20_prepare_bandwidth(
+ struct dc *dc,
+ struct dc_state *context)
+{
+ struct hubbub *hubbub = dc->res_pool->hubbub;
+
+ /* program dchubbub watermarks */
+ hubbub->funcs->program_watermarks(hubbub,
+ &context->bw_ctx.bw.dcn.watermarks,
+ dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000,
+ false);
+
+ dc->clk_mgr->funcs->update_clocks(
+ dc->clk_mgr,
+ context,
+ false);
+}
+
+void dcn20_optimize_bandwidth(
+ struct dc *dc,
+ struct dc_state *context)
+{
+ struct hubbub *hubbub = dc->res_pool->hubbub;
+
+ /* program dchubbub watermarks */
+ hubbub->funcs->program_watermarks(hubbub,
+ &context->bw_ctx.bw.dcn.watermarks,
+ dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000,
+ true);
+
+ dc->clk_mgr->funcs->update_clocks(
+ dc->clk_mgr,
+ context,
+ true);
+}
+
+bool dcn20_update_bandwidth(
+ struct dc *dc,
+ struct dc_state *context)
+{
+ int i;
+
+ /* recalculate DML parameters */
+ if (!dc->res_pool->funcs->validate_bandwidth(dc, context, false))
+ return false;
+
+ /* apply updated bandwidth parameters */
+ dc->hwss.prepare_bandwidth(dc, context);
+
+ /* update hubp configs for all pipes */
+ for (i = 0; i < dc->res_pool->pipe_count; i++) {
+ struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
+
+ if (pipe_ctx->plane_state == NULL)
+ continue;
+
+ if (pipe_ctx->top_pipe == NULL) {
+ bool blank = !is_pipe_tree_visible(pipe_ctx);
+
+ pipe_ctx->stream_res.tg->funcs->program_global_sync(
+ pipe_ctx->stream_res.tg,
+ pipe_ctx->pipe_dlg_param.vready_offset,
+ pipe_ctx->pipe_dlg_param.vstartup_start,
+ pipe_ctx->pipe_dlg_param.vupdate_offset,
+ pipe_ctx->pipe_dlg_param.vupdate_width);
+
+ pipe_ctx->stream_res.tg->funcs->set_vtg_params(
+ pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing);
+
+ dc->hwss.blank_pixel_data(dc, pipe_ctx, blank);
+ }
+
+ pipe_ctx->plane_res.hubp->funcs->hubp_setup(
+ pipe_ctx->plane_res.hubp,
+ &pipe_ctx->dlg_regs,
+ &pipe_ctx->ttu_regs,
+ &pipe_ctx->rq_regs,
+ &pipe_ctx->pipe_dlg_param);
+ }
+
+ return true;
+}
+
+static void dcn20_enable_writeback(
+ struct dc *dc,
+ const struct dc_stream_status *stream_status,
+ struct dc_writeback_info *wb_info)
+{
+ struct dwbc *dwb;
+ struct mcif_wb *mcif_wb;
+ struct timing_generator *optc;
+
+ ASSERT(wb_info->dwb_pipe_inst < MAX_DWB_PIPES);
+ ASSERT(wb_info->wb_enabled);
+ dwb = dc->res_pool->dwbc[wb_info->dwb_pipe_inst];
+ mcif_wb = dc->res_pool->mcif_wb[wb_info->dwb_pipe_inst];
+
+ /* set the OPTC source mux */
+ ASSERT(stream_status->primary_otg_inst < MAX_PIPES);
+ optc = dc->res_pool->timing_generators[stream_status->primary_otg_inst];
+ optc->funcs->set_dwb_source(optc, wb_info->dwb_pipe_inst);
+ /* set MCIF_WB buffer and arbitration configuration */
+ mcif_wb->funcs->config_mcif_buf(mcif_wb, &wb_info->mcif_buf_params, wb_info->dwb_params.dest_height);
+ mcif_wb->funcs->config_mcif_arb(mcif_wb, &dc->current_state->bw_ctx.bw.dcn.bw_writeback.mcif_wb_arb[wb_info->dwb_pipe_inst]);
+ /* Enable MCIF_WB */
+ mcif_wb->funcs->enable_mcif(mcif_wb);
+ /* Enable DWB */
+ dwb->funcs->enable(dwb, &wb_info->dwb_params);
+ /* TODO: add sequence to enable/disable warmup */
+}
+
+void dcn20_disable_writeback(
+ struct dc *dc,
+ unsigned int dwb_pipe_inst)
+{
+ struct dwbc *dwb;
+ struct mcif_wb *mcif_wb;
+
+ ASSERT(dwb_pipe_inst < MAX_DWB_PIPES);
+ dwb = dc->res_pool->dwbc[dwb_pipe_inst];
+ mcif_wb = dc->res_pool->mcif_wb[dwb_pipe_inst];
+
+ dwb->funcs->disable(dwb);
+ mcif_wb->funcs->disable_mcif(mcif_wb);
+}
+
+bool dcn20_hwss_wait_for_blank_complete(
+ struct output_pixel_processor *opp)
+{
+ int counter;
+
+ for (counter = 0; counter < 1000; counter++) {
+ if (opp->funcs->dpg_is_blanked(opp))
+ break;
+
+ udelay(100);
+ }
+
+ if (counter == 1000) {
+ dm_error("DC: failed to blank crtc!\n");
+ return false;
+ }
+
+ return true;
+}
+
+bool dcn20_dmdata_status_done(struct pipe_ctx *pipe_ctx)
+{
+ struct hubp *hubp = pipe_ctx->plane_res.hubp;
+
+ if (!hubp)
+ return false;
+ return hubp->funcs->dmdata_status_done(hubp);
+}
+
+static void dcn20_disable_stream_gating(struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ struct dce_hwseq *hws = dc->hwseq;
+ struct pipe_ctx *bot_odm_pipe = dc_res_get_odm_bottom_pipe(pipe_ctx);
+
+ if (pipe_ctx->stream_res.dsc) {
+ dcn20_dsc_pg_control(hws, pipe_ctx->stream_res.dsc->inst, true);
+ if (bot_odm_pipe)
+ dcn20_dsc_pg_control(hws, bot_odm_pipe->stream_res.dsc->inst, true);
+ }
+#endif
+}
+
+static void dcn20_enable_stream_gating(struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ struct dce_hwseq *hws = dc->hwseq;
+ struct pipe_ctx *bot_odm_pipe = dc_res_get_odm_bottom_pipe(pipe_ctx);
+
+ if (pipe_ctx->stream_res.dsc) {
+ dcn20_dsc_pg_control(hws, pipe_ctx->stream_res.dsc->inst, false);
+ if (bot_odm_pipe)
+ dcn20_dsc_pg_control(hws, bot_odm_pipe->stream_res.dsc->inst, false);
+ }
+#endif
+}
+
+void dcn20_set_dmdata_attributes(struct pipe_ctx *pipe_ctx)
+{
+ struct dc_dmdata_attributes attr = { 0 };
+ struct hubp *hubp = pipe_ctx->plane_res.hubp;
+
+ attr.dmdata_mode = DMDATA_HW_MODE;
+ attr.dmdata_size =
+ dc_is_hdmi_signal(pipe_ctx->stream->signal) ? 32 : 36;
+ attr.address.quad_part =
+ pipe_ctx->stream->dmdata_address.quad_part;
+ attr.dmdata_dl_delta = 0;
+ attr.dmdata_qos_mode = 0;
+ attr.dmdata_qos_level = 0;
+ attr.dmdata_repeat = 1; /* always repeat */
+ attr.dmdata_updated = 1;
+ attr.dmdata_sw_data = NULL;
+
+ hubp->funcs->dmdata_set_attributes(hubp, &attr);
+}
+
+void dcn20_disable_stream(struct pipe_ctx *pipe_ctx, int option)
+{
+ dce110_disable_stream(pipe_ctx, option);
+}
+
+static void dcn20_init_vm_ctx(
+ struct dce_hwseq *hws,
+ struct dc *dc,
+ struct dc_virtual_addr_space_config *va_config,
+ int vmid)
+{
+ struct dcn_hubbub_virt_addr_config config;
+
+ if (vmid == 0) {
+ ASSERT(0); /* VMID cannot be 0 for vm context */
+ return;
+ }
+
+ config.page_table_start_addr = va_config->page_table_start_addr;
+ config.page_table_end_addr = va_config->page_table_end_addr;
+ config.page_table_block_size = va_config->page_table_block_size_in_bytes;
+ config.page_table_depth = va_config->page_table_depth;
+ config.page_table_base_addr = va_config->page_table_base_addr;
+
+ dc->res_pool->hubbub->funcs->init_vm_ctx(dc->res_pool->hubbub, &config, vmid);
+}
+
+static int dcn20_init_sys_ctx(struct dce_hwseq *hws, struct dc *dc, struct dc_phy_addr_space_config *pa_config)
+{
+ struct dcn_hubbub_phys_addr_config config;
+
+ config.system_aperture.fb_top = pa_config->system_aperture.fb_top;
+ config.system_aperture.fb_offset = pa_config->system_aperture.fb_offset;
+ config.system_aperture.fb_base = pa_config->system_aperture.fb_base;
+ config.system_aperture.agp_top = pa_config->system_aperture.agp_top;
+ config.system_aperture.agp_bot = pa_config->system_aperture.agp_bot;
+ config.system_aperture.agp_base = pa_config->system_aperture.agp_base;
+ config.gart_config.page_table_start_addr = pa_config->gart_config.page_table_start_addr;
+ config.gart_config.page_table_end_addr = pa_config->gart_config.page_table_end_addr;
+ config.gart_config.page_table_base_addr = pa_config->gart_config.page_table_base_addr;
+
+ return dc->res_pool->hubbub->funcs->init_dchub_sys_ctx(dc->res_pool->hubbub, &config);
+}
+
+static bool patch_address_for_sbs_tb_stereo(
+ struct pipe_ctx *pipe_ctx, PHYSICAL_ADDRESS_LOC *addr)
+{
+ struct dc_plane_state *plane_state = pipe_ctx->plane_state;
+ bool sec_split = pipe_ctx->top_pipe &&
+ pipe_ctx->top_pipe->plane_state == pipe_ctx->plane_state;
+ if (sec_split && plane_state->address.type == PLN_ADDR_TYPE_GRPH_STEREO &&
+ (pipe_ctx->stream->timing.timing_3d_format ==
+ TIMING_3D_FORMAT_SIDE_BY_SIDE ||
+ pipe_ctx->stream->timing.timing_3d_format ==
+ TIMING_3D_FORMAT_TOP_AND_BOTTOM)) {
+ *addr = plane_state->address.grph_stereo.left_addr;
+ plane_state->address.grph_stereo.left_addr =
+ plane_state->address.grph_stereo.right_addr;
+ return true;
+ }
+
+ if (pipe_ctx->stream->view_format != VIEW_3D_FORMAT_NONE &&
+ plane_state->address.type != PLN_ADDR_TYPE_GRPH_STEREO) {
+ plane_state->address.type = PLN_ADDR_TYPE_GRPH_STEREO;
+ plane_state->address.grph_stereo.right_addr =
+ plane_state->address.grph_stereo.left_addr;
+ }
+ return false;
+}
+
+
+static void dcn20_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+ bool addr_patched = false;
+ PHYSICAL_ADDRESS_LOC addr;
+ struct dc_plane_state *plane_state = pipe_ctx->plane_state;
+
+ if (plane_state == NULL)
+ return;
+
+ addr_patched = patch_address_for_sbs_tb_stereo(pipe_ctx, &addr);
+
+ // Call Helper to track VMID use
+ vm_helper_mark_vmid_used(dc->vm_helper, plane_state->address.vmid, pipe_ctx->plane_res.hubp->inst);
+
+ pipe_ctx->plane_res.hubp->funcs->hubp_program_surface_flip_and_addr(
+ pipe_ctx->plane_res.hubp,
+ &plane_state->address,
+ plane_state->flip_immediate);
+
+ plane_state->status.requested_address = plane_state->address;
+
+ if (plane_state->flip_immediate)
+ plane_state->status.current_address = plane_state->address;
+
+ if (addr_patched)
+ pipe_ctx->plane_state->address.grph_stereo.left_addr = addr;
+}
+
+void dcn20_unblank_stream(struct pipe_ctx *pipe_ctx,
+ struct dc_link_settings *link_settings)
+{
+ struct encoder_unblank_param params = { { 0 } };
+ struct dc_stream_state *stream = pipe_ctx->stream;
+ struct dc_link *link = stream->link;
+ params.odm = dc_res_get_odm_bottom_pipe(pipe_ctx);
+
+ /* only 3 items below are used by unblank */
+ params.timing = pipe_ctx->stream->timing;
+
+ params.link_settings.link_rate = link_settings->link_rate;
+
+ if (dc_is_dp_signal(pipe_ctx->stream->signal)) {
+ if (optc1_is_two_pixels_per_containter(&stream->timing) || params.odm)
+ params.timing.pix_clk_100hz /= 2;
+ pipe_ctx->stream_res.stream_enc->funcs->dp_set_odm_combine(
+ pipe_ctx->stream_res.stream_enc, params.odm);
+ pipe_ctx->stream_res.stream_enc->funcs->dp_unblank(pipe_ctx->stream_res.stream_enc, &params);
+ }
+
+ if (link->local_sink && link->local_sink->sink_signal == SIGNAL_TYPE_EDP) {
+ link->dc->hwss.edp_backlight_control(link, true);
+ }
+}
+
+void dcn20_setup_vupdate_interrupt(struct pipe_ctx *pipe_ctx)
+{
+ struct timing_generator *tg = pipe_ctx->stream_res.tg;
+ int start_line = get_vupdate_offset_from_vsync(pipe_ctx);
+
+ if (start_line < 0)
+ start_line = 0;
+
+ if (tg->funcs->setup_vertical_interrupt2)
+ tg->funcs->setup_vertical_interrupt2(tg, start_line);
+}
+
+static void dcn20_reset_back_end_for_pipe(
+ struct dc *dc,
+ struct pipe_ctx *pipe_ctx,
+ struct dc_state *context)
+{
+ int i;
+ DC_LOGGER_INIT(dc->ctx->logger);
+ if (pipe_ctx->stream_res.stream_enc == NULL) {
+ pipe_ctx->stream = NULL;
+ return;
+ }
+
+ if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) {
+ /* DPMS may already disable */
+ if (!pipe_ctx->stream->dpms_off)
+ core_link_disable_stream(pipe_ctx, FREE_ACQUIRED_RESOURCE);
+ else if (pipe_ctx->stream_res.audio) {
+ dc->hwss.disable_audio_stream(pipe_ctx, FREE_ACQUIRED_RESOURCE);
+ }
+ }
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ else if (pipe_ctx->stream_res.dsc)
+ dp_set_dsc_enable(pipe_ctx, false);
+#endif
+
+ /* by upper caller loop, parent pipe: pipe0, will be reset last.
+ * back end share by all pipes and will be disable only when disable
+ * parent pipe.
+ */
+ if (pipe_ctx->top_pipe == NULL) {
+ pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg);
+
+ pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false);
+ if (pipe_ctx->stream_res.tg->funcs->set_odm_bypass)
+ pipe_ctx->stream_res.tg->funcs->set_odm_bypass(
+ pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing);
+ }
+
+ for (i = 0; i < dc->res_pool->pipe_count; i++)
+ if (&dc->current_state->res_ctx.pipe_ctx[i] == pipe_ctx)
+ break;
+
+ if (i == dc->res_pool->pipe_count)
+ return;
+
+ pipe_ctx->stream = NULL;
+ DC_LOG_DEBUG("Reset back end for pipe %d, tg:%d\n",
+ pipe_ctx->pipe_idx, pipe_ctx->stream_res.tg->inst);
+}
+
+static void dcn20_reset_hw_ctx_wrap(
+ struct dc *dc,
+ struct dc_state *context)
+{
+ int i;
+
+ /* Reset Back End*/
+ for (i = dc->res_pool->pipe_count - 1; i >= 0 ; i--) {
+ struct pipe_ctx *pipe_ctx_old =
+ &dc->current_state->res_ctx.pipe_ctx[i];
+ struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
+
+ if (!pipe_ctx_old->stream)
+ continue;
+
+ if (pipe_ctx_old->top_pipe)
+ continue;
+
+ if (!pipe_ctx->stream ||
+ pipe_need_reprogram(pipe_ctx_old, pipe_ctx)) {
+ struct clock_source *old_clk = pipe_ctx_old->clock_source;
+
+ dcn20_reset_back_end_for_pipe(dc, pipe_ctx_old, dc->current_state);
+ if (dc->hwss.enable_stream_gating)
+ dc->hwss.enable_stream_gating(dc, pipe_ctx);
+ if (old_clk)
+ old_clk->funcs->cs_power_down(old_clk);
+ }
+ }
+}
+
+static void dcn20_update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+ struct hubp *hubp = pipe_ctx->plane_res.hubp;
+ struct mpcc_blnd_cfg blnd_cfg = { {0} };
+ bool per_pixel_alpha = pipe_ctx->plane_state->per_pixel_alpha && pipe_ctx->bottom_pipe;
+ int mpcc_id;
+ struct mpcc *new_mpcc;
+ struct mpc *mpc = dc->res_pool->mpc;
+ struct mpc_tree *mpc_tree_params = &(pipe_ctx->stream_res.opp->mpc_tree_params);
+
+ // input to MPCC is always RGB, by default leave black_color at 0
+ if (dc->debug.visual_confirm == VISUAL_CONFIRM_HDR) {
+ dcn10_get_hdr_visual_confirm_color(
+ pipe_ctx, &blnd_cfg.black_color);
+ } else if (dc->debug.visual_confirm == VISUAL_CONFIRM_SURFACE) {
+ dcn10_get_surface_visual_confirm_color(
+ pipe_ctx, &blnd_cfg.black_color);
+ }
+
+ if (per_pixel_alpha)
+ blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA;
+ else
+ blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_GLOBAL_ALPHA;
+
+ blnd_cfg.overlap_only = false;
+ blnd_cfg.global_gain = 0xff;
+
+ if (pipe_ctx->plane_state->global_alpha)
+ blnd_cfg.global_alpha = pipe_ctx->plane_state->global_alpha_value;
+ else
+ blnd_cfg.global_alpha = 0xff;
+
+ blnd_cfg.background_color_bpc = 4;
+ blnd_cfg.bottom_gain_mode = 0;
+ blnd_cfg.top_gain = 0x1f000;
+ blnd_cfg.bottom_inside_gain = 0x1f000;
+ blnd_cfg.bottom_outside_gain = 0x1f000;
+ blnd_cfg.pre_multiplied_alpha = per_pixel_alpha;
+
+ /*
+ * TODO: remove hack
+ * Note: currently there is a bug in init_hw such that
+ * on resume from hibernate, BIOS sets up MPCC0, and
+ * we do mpcc_remove but the mpcc cannot go to idle
+ * after remove. This cause us to pick mpcc1 here,
+ * which causes a pstate hang for yet unknown reason.
+ */
+ mpcc_id = hubp->inst;
+
+ /* If there is no full update, don't need to touch MPC tree*/
+ if (!pipe_ctx->plane_state->update_flags.bits.full_update) {
+ mpc->funcs->update_blending(mpc, &blnd_cfg, mpcc_id);
+ return;
+ }
+
+ /* check if this MPCC is already being used */
+ new_mpcc = mpc->funcs->get_mpcc_for_dpp(mpc_tree_params, mpcc_id);
+ /* remove MPCC if being used */
+ if (new_mpcc != NULL)
+ mpc->funcs->remove_mpcc(mpc, mpc_tree_params, new_mpcc);
+ else
+ if (dc->debug.sanity_checks)
+ mpc->funcs->assert_mpcc_idle_before_connect(
+ dc->res_pool->mpc, mpcc_id);
+
+ /* Call MPC to insert new plane */
+ new_mpcc = mpc->funcs->insert_plane(dc->res_pool->mpc,
+ mpc_tree_params,
+ &blnd_cfg,
+ NULL,
+ NULL,
+ hubp->inst,
+ mpcc_id);
+
+ ASSERT(new_mpcc != NULL);
+ hubp->opp_id = pipe_ctx->stream_res.opp->inst;
+ hubp->mpcc_id = mpcc_id;
+}
+
+static int find_free_gsl_group(const struct dc *dc)
+{
+ if (dc->res_pool->gsl_groups.gsl_0 == 0)
+ return 1;
+ if (dc->res_pool->gsl_groups.gsl_1 == 0)
+ return 2;
+ if (dc->res_pool->gsl_groups.gsl_2 == 0)
+ return 3;
+
+ return 0;
+}
+
+/* NOTE: This is not a generic setup_gsl function (hence the suffix as_lock)
+ * This is only used to lock pipes in pipe splitting case with immediate flip
+ * Ordinary MPC/OTG locks suppress VUPDATE which doesn't help with immediate,
+ * so we get tearing with freesync since we cannot flip multiple pipes
+ * atomically.
+ * We use GSL for this:
+ * - immediate flip: find first available GSL group if not already assigned
+ * program gsl with that group, set current OTG as master
+ * and always us 0x4 = AND of flip_ready from all pipes
+ * - vsync flip: disable GSL if used
+ *
+ * Groups in stream_res are stored as +1 from HW registers, i.e.
+ * gsl_0 <=> pipe_ctx->stream_res.gsl_group == 1
+ * Using a magic value like -1 would require tracking all inits/resets
+ */
+void dcn20_setup_gsl_group_as_lock(
+ const struct dc *dc,
+ struct pipe_ctx *pipe_ctx,
+ bool enable)
+{
+ struct gsl_params gsl;
+ int group_idx;
+
+ memset(&gsl, 0, sizeof(struct gsl_params));
+
+ if (enable) {
+ /* return if group already assigned since GSL was set up
+ * for vsync flip, we would unassign so it can't be "left over"
+ */
+ if (pipe_ctx->stream_res.gsl_group > 0)
+ return;
+
+ group_idx = find_free_gsl_group(dc);
+ ASSERT(group_idx != 0);
+ pipe_ctx->stream_res.gsl_group = group_idx;
+
+ /* set gsl group reg field and mark resource used */
+ switch (group_idx) {
+ case 1:
+ gsl.gsl0_en = 1;
+ dc->res_pool->gsl_groups.gsl_0 = 1;
+ break;
+ case 2:
+ gsl.gsl1_en = 1;
+ dc->res_pool->gsl_groups.gsl_1 = 1;
+ break;
+ case 3:
+ gsl.gsl2_en = 1;
+ dc->res_pool->gsl_groups.gsl_2 = 1;
+ break;
+ default:
+ BREAK_TO_DEBUGGER();
+ return; // invalid case
+ }
+ gsl.gsl_master_en = 1;
+ } else {
+ group_idx = pipe_ctx->stream_res.gsl_group;
+ if (group_idx == 0)
+ return; // if not in use, just return
+
+ pipe_ctx->stream_res.gsl_group = 0;
+
+ /* unset gsl group reg field and mark resource free */
+ switch (group_idx) {
+ case 1:
+ gsl.gsl0_en = 0;
+ dc->res_pool->gsl_groups.gsl_0 = 0;
+ break;
+ case 2:
+ gsl.gsl1_en = 0;
+ dc->res_pool->gsl_groups.gsl_1 = 0;
+ break;
+ case 3:
+ gsl.gsl2_en = 0;
+ dc->res_pool->gsl_groups.gsl_2 = 0;
+ break;
+ default:
+ BREAK_TO_DEBUGGER();
+ return;
+ }
+ gsl.gsl_master_en = 0;
+ }
+
+ /* at this point we want to program whether it's to enable or disable */
+ if (pipe_ctx->stream_res.tg->funcs->set_gsl != NULL &&
+ pipe_ctx->stream_res.tg->funcs->set_gsl_source_select != NULL) {
+ pipe_ctx->stream_res.tg->funcs->set_gsl(
+ pipe_ctx->stream_res.tg,
+ &gsl);
+
+ pipe_ctx->stream_res.tg->funcs->set_gsl_source_select(
+ pipe_ctx->stream_res.tg, group_idx, enable ? 4 : 0);
+ } else
+ BREAK_TO_DEBUGGER();
+}
+
+static void dcn20_set_flip_control_gsl(
+ struct pipe_ctx *pipe_ctx,
+ bool flip_immediate)
+{
+ if (pipe_ctx && pipe_ctx->plane_res.hubp->funcs->hubp_set_flip_control_surface_gsl)
+ pipe_ctx->plane_res.hubp->funcs->hubp_set_flip_control_surface_gsl(
+ pipe_ctx->plane_res.hubp, flip_immediate);
+
+}
+
+void dcn20_hw_sequencer_construct(struct dc *dc)
+{
+ dcn10_hw_sequencer_construct(dc);
+ dc->hwss.init_hw = dcn20_init_hw;
+ dc->hwss.init_pipes = NULL;
+ dc->hwss.unblank_stream = dcn20_unblank_stream;
+ dc->hwss.update_plane_addr = dcn20_update_plane_addr;
+ dc->hwss.disable_plane = dcn20_disable_plane,
+ dc->hwss.enable_stream_timing = dcn20_enable_stream_timing;
+ dc->hwss.program_triplebuffer = dcn20_program_tripleBuffer;
+ dc->hwss.set_input_transfer_func = dcn20_set_input_transfer_func;
+ dc->hwss.set_output_transfer_func = dcn20_set_output_transfer_func;
+ dc->hwss.apply_ctx_for_surface = dcn20_apply_ctx_for_surface;
+ dc->hwss.pipe_control_lock = dcn20_pipe_control_lock;
+ dc->hwss.pipe_control_lock_global = dcn20_pipe_control_lock_global;
+ dc->hwss.optimize_bandwidth = dcn20_optimize_bandwidth;
+ dc->hwss.prepare_bandwidth = dcn20_prepare_bandwidth;
+ dc->hwss.update_bandwidth = dcn20_update_bandwidth;
+ dc->hwss.enable_writeback = dcn20_enable_writeback;
+ dc->hwss.disable_writeback = dcn20_disable_writeback;
+ dc->hwss.program_output_csc = dcn20_program_output_csc;
+ dc->hwss.update_odm = dcn20_update_odm;
+ dc->hwss.blank_pixel_data = dcn20_blank_pixel_data;
+ dc->hwss.dmdata_status_done = dcn20_dmdata_status_done;
+ dc->hwss.disable_stream = dcn20_disable_stream;
+ dc->hwss.init_sys_ctx = dcn20_init_sys_ctx;
+ dc->hwss.init_vm_ctx = dcn20_init_vm_ctx;
+ dc->hwss.disable_stream_gating = dcn20_disable_stream_gating;
+ dc->hwss.enable_stream_gating = dcn20_enable_stream_gating;
+ dc->hwss.setup_vupdate_interrupt = dcn20_setup_vupdate_interrupt;
+ dc->hwss.reset_hw_ctx_wrap = dcn20_reset_hw_ctx_wrap;
+ dc->hwss.update_mpcc = dcn20_update_mpcc;
+ dc->hwss.set_flip_control_gsl = dcn20_set_flip_control_gsl;
+ dc->hwss.did_underflow_occur = dcn10_did_underflow_occur;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.h
new file mode 100644
index 000000000000..2b0409454073
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.h
@@ -0,0 +1,103 @@
+/*
+* Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_HWSS_DCN20_H__
+#define __DC_HWSS_DCN20_H__
+
+struct dc;
+
+void dcn20_hw_sequencer_construct(struct dc *dc);
+
+enum dc_status dcn20_enable_stream_timing(
+ struct pipe_ctx *pipe_ctx,
+ struct dc_state *context,
+ struct dc *dc);
+
+void dcn20_blank_pixel_data(
+ struct dc *dc,
+ struct pipe_ctx *pipe_ctx,
+ bool blank);
+
+void dcn20_program_output_csc(struct dc *dc,
+ struct pipe_ctx *pipe_ctx,
+ enum dc_color_space colorspace,
+ uint16_t *matrix,
+ int opp_id);
+
+void dcn20_prepare_bandwidth(
+ struct dc *dc,
+ struct dc_state *context);
+
+void dcn20_optimize_bandwidth(
+ struct dc *dc,
+ struct dc_state *context);
+
+bool dcn20_update_bandwidth(
+ struct dc *dc,
+ struct dc_state *context);
+
+void dcn20_disable_writeback(
+ struct dc *dc,
+ unsigned int dwb_pipe_inst);
+
+bool dcn20_hwss_wait_for_blank_complete(
+ struct output_pixel_processor *opp);
+
+bool dcn20_set_output_transfer_func(struct pipe_ctx *pipe_ctx,
+ const struct dc_stream_state *stream);
+
+bool dcn20_set_input_transfer_func(struct pipe_ctx *pipe_ctx,
+ const struct dc_plane_state *plane_state);
+
+bool dcn20_dmdata_status_done(struct pipe_ctx *pipe_ctx);
+
+void dcn20_set_dmdata_attributes(struct pipe_ctx *pipe_ctx);
+
+void dcn20_disable_stream(struct pipe_ctx *pipe_ctx, int option);
+
+void dcn20_program_tripleBuffer(
+ const struct dc *dc,
+ struct pipe_ctx *pipe_ctx,
+ bool enableTripleBuffer);
+
+void dcn20_setup_vupdate_interrupt(struct pipe_ctx *pipe_ctx);
+
+void dcn20_pipe_control_lock_global(
+ struct dc *dc,
+ struct pipe_ctx *pipe,
+ bool lock);
+void dcn20_setup_gsl_group_as_lock(const struct dc *dc,
+ struct pipe_ctx *pipe_ctx,
+ bool enable);
+void dcn20_pipe_control_lock(
+ struct dc *dc,
+ struct pipe_ctx *pipe,
+ bool lock);
+void dcn20_disable_plane(struct dc *dc, struct pipe_ctx *pipe_ctx);
+void dcn20_enable_plane(
+ struct dc *dc,
+ struct pipe_ctx *pipe_ctx,
+ struct dc_state *context);
+#endif /* __DC_HWSS_DCN20_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.c
new file mode 100644
index 000000000000..f495582e9e87
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.c
@@ -0,0 +1,460 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "reg_helper.h"
+
+#include "core_types.h"
+#include "link_encoder.h"
+#include "dcn20_link_encoder.h"
+#include "stream_encoder.h"
+#include "i2caux_interface.h"
+#include "dc_bios_types.h"
+
+#include "gpio_service_interface.h"
+
+#define CTX \
+ enc10->base.ctx
+#define DC_LOGGER \
+ enc10->base.ctx->logger
+
+#define REG(reg)\
+ (enc10->link_regs->reg)
+
+#undef FN
+#define FN(reg_name, field_name) \
+ enc10->link_shift->field_name, enc10->link_mask->field_name
+
+#define IND_REG(index) \
+ (enc10->link_regs->index)
+
+
+static struct mpll_cfg dcn2_mpll_cfg[] = {
+ // RBR
+ {
+ .hdmimode_enable = 1,
+ .ref_range = 3,
+ .ref_clk_mpllb_div = 2,
+ .mpllb_ssc_en = 1,
+ .mpllb_div5_clk_en = 1,
+ .mpllb_multiplier = 226,
+ .mpllb_fracn_en = 1,
+ .mpllb_fracn_quot = 39321,
+ .mpllb_fracn_rem = 3,
+ .mpllb_fracn_den = 5,
+ .mpllb_ssc_up_spread = 0,
+ .mpllb_ssc_peak = 38221,
+ .mpllb_ssc_stepsize = 49314,
+ .mpllb_div_clk_en = 0,
+ .mpllb_div_multiplier = 0,
+ .mpllb_hdmi_div = 0,
+ .mpllb_tx_clk_div = 2,
+ .tx_vboost_lvl = 4,
+ .mpllb_pmix_en = 1,
+ .mpllb_word_div2_en = 0,
+ .mpllb_ana_v2i = 2,
+ .mpllb_ana_freq_vco = 2,
+ .mpllb_ana_cp_int = 7,
+ .mpllb_ana_cp_prop = 18,
+ .hdmi_pixel_clk_div = 0,
+ },
+ // HBR
+ {
+ .hdmimode_enable = 1,
+ .ref_range = 3,
+ .ref_clk_mpllb_div = 2,
+ .mpllb_ssc_en = 1,
+ .mpllb_div5_clk_en = 1,
+ .mpllb_multiplier = 184,
+ .mpllb_fracn_en = 0,
+ .mpllb_fracn_quot = 0,
+ .mpllb_fracn_rem = 0,
+ .mpllb_fracn_den = 1,
+ .mpllb_ssc_up_spread = 0,
+ .mpllb_ssc_peak = 31850,
+ .mpllb_ssc_stepsize = 41095,
+ .mpllb_div_clk_en = 0,
+ .mpllb_div_multiplier = 0,
+ .mpllb_hdmi_div = 0,
+ .mpllb_tx_clk_div = 1,
+ .tx_vboost_lvl = 4,
+ .mpllb_pmix_en = 1,
+ .mpllb_word_div2_en = 0,
+ .mpllb_ana_v2i = 2,
+ .mpllb_ana_freq_vco = 3,
+ .mpllb_ana_cp_int = 7,
+ .mpllb_ana_cp_prop = 18,
+ .hdmi_pixel_clk_div = 0,
+ },
+ //HBR2
+ {
+ .hdmimode_enable = 1,
+ .ref_range = 3,
+ .ref_clk_mpllb_div = 2,
+ .mpllb_ssc_en = 1,
+ .mpllb_div5_clk_en = 1,
+ .mpllb_multiplier = 184,
+ .mpllb_fracn_en = 0,
+ .mpllb_fracn_quot = 0,
+ .mpllb_fracn_rem = 0,
+ .mpllb_fracn_den = 1,
+ .mpllb_ssc_up_spread = 0,
+ .mpllb_ssc_peak = 31850,
+ .mpllb_ssc_stepsize = 41095,
+ .mpllb_div_clk_en = 0,
+ .mpllb_div_multiplier = 0,
+ .mpllb_hdmi_div = 0,
+ .mpllb_tx_clk_div = 0,
+ .tx_vboost_lvl = 4,
+ .mpllb_pmix_en = 1,
+ .mpllb_word_div2_en = 0,
+ .mpllb_ana_v2i = 2,
+ .mpllb_ana_freq_vco = 3,
+ .mpllb_ana_cp_int = 7,
+ .mpllb_ana_cp_prop = 18,
+ .hdmi_pixel_clk_div = 0,
+ },
+ //HBR3
+ {
+ .hdmimode_enable = 1,
+ .ref_range = 3,
+ .ref_clk_mpllb_div = 2,
+ .mpllb_ssc_en = 1,
+ .mpllb_div5_clk_en = 1,
+ .mpllb_multiplier = 292,
+ .mpllb_fracn_en = 0,
+ .mpllb_fracn_quot = 0,
+ .mpllb_fracn_rem = 0,
+ .mpllb_fracn_den = 1,
+ .mpllb_ssc_up_spread = 0,
+ .mpllb_ssc_peak = 47776,
+ .mpllb_ssc_stepsize = 61642,
+ .mpllb_div_clk_en = 0,
+ .mpllb_div_multiplier = 0,
+ .mpllb_hdmi_div = 0,
+ .mpllb_tx_clk_div = 0,
+ .tx_vboost_lvl = 4,
+ .mpllb_pmix_en = 1,
+ .mpllb_word_div2_en = 0,
+ .mpllb_ana_v2i = 2,
+ .mpllb_ana_freq_vco = 0,
+ .mpllb_ana_cp_int = 7,
+ .mpllb_ana_cp_prop = 18,
+ .hdmi_pixel_clk_div = 0,
+ },
+};
+
+void enc2_fec_set_enable(struct link_encoder *enc, bool enable)
+{
+ struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ DC_LOG_DSC("%s FEC at link encoder inst %d",
+ enable ? "Enabling" : "Disabling", enc->id.enum_id);
+#endif
+ REG_UPDATE(DP_DPHY_CNTL, DPHY_FEC_EN, enable);
+}
+
+void enc2_fec_set_ready(struct link_encoder *enc, bool ready)
+{
+ struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+
+ REG_UPDATE(DP_DPHY_CNTL, DPHY_FEC_READY_SHADOW, ready);
+}
+
+bool enc2_fec_is_active(struct link_encoder *enc)
+{
+ uint32_t active = 0;
+ struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+
+ REG_GET(DP_DPHY_CNTL, DPHY_FEC_ACTIVE_STATUS, &active);
+
+ return (active != 0);
+}
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+/* this function reads dsc related register fields to be logged later in dcn10_log_hw_state
+ * into a dcn_dsc_state struct.
+ */
+void link_enc2_read_state(struct link_encoder *enc, struct link_enc_state *s)
+{
+ struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+
+ REG_GET(DP_DPHY_CNTL, DPHY_FEC_EN, &s->dphy_fec_en);
+ REG_GET(DP_DPHY_CNTL, DPHY_FEC_READY_SHADOW, &s->dphy_fec_ready_shadow);
+ REG_GET(DP_DPHY_CNTL, DPHY_FEC_ACTIVE_STATUS, &s->dphy_fec_active_status);
+}
+#endif
+
+static bool update_cfg_data(
+ struct dcn10_link_encoder *enc10,
+ const struct dc_link_settings *link_settings,
+ struct dpcssys_phy_seq_cfg *cfg)
+{
+ int i;
+
+ cfg->load_sram_fw = false;
+
+ for (i = 0; i < link_settings->lane_count; i++)
+ cfg->lane_en[i] = true;
+
+ switch (link_settings->link_rate) {
+ case LINK_RATE_LOW:
+ cfg->mpll_cfg = dcn2_mpll_cfg[0];
+ break;
+ case LINK_RATE_HIGH:
+ cfg->mpll_cfg = dcn2_mpll_cfg[1];
+ break;
+ case LINK_RATE_HIGH2:
+ cfg->mpll_cfg = dcn2_mpll_cfg[2];
+ break;
+ case LINK_RATE_HIGH3:
+ cfg->mpll_cfg = dcn2_mpll_cfg[3];
+ break;
+ default:
+ DC_LOG_ERROR("%s: No supported link rate found %X!\n",
+ __func__, link_settings->link_rate);
+ return false;
+ }
+
+ return true;
+}
+
+void dcn20_link_encoder_enable_dp_output(
+ struct link_encoder *enc,
+ const struct dc_link_settings *link_settings,
+ enum clock_source_id clock_source)
+{
+ struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+ struct dcn20_link_encoder *enc20 = (struct dcn20_link_encoder *) enc10;
+ struct dpcssys_phy_seq_cfg *cfg = &enc20->phy_seq_cfg;
+
+ if (!enc->ctx->dc->debug.avoid_vbios_exec_table) {
+ dcn10_link_encoder_enable_dp_output(enc, link_settings, clock_source);
+ return;
+ }
+
+ if (!update_cfg_data(enc10, link_settings, cfg))
+ return;
+
+ enc1_configure_encoder(enc10, link_settings);
+
+ dcn10_link_encoder_setup(enc, SIGNAL_TYPE_DISPLAY_PORT);
+
+}
+
+#define AUX_REG(reg)\
+ (enc10->aux_regs->reg)
+
+#define AUX_REG_READ(reg_name) \
+ dm_read_reg(CTX, AUX_REG(reg_name))
+
+#define AUX_REG_WRITE(reg_name, val) \
+ dm_write_reg(CTX, AUX_REG(reg_name), val)
+void enc2_hw_init(struct link_encoder *enc)
+{
+ struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
+
+/*
+ 00 - DP_AUX_DPHY_RX_DETECTION_THRESHOLD__1to2 : 1/2
+ 01 - DP_AUX_DPHY_RX_DETECTION_THRESHOLD__3to4 : 3/4
+ 02 - DP_AUX_DPHY_RX_DETECTION_THRESHOLD__7to8 : 7/8
+ 03 - DP_AUX_DPHY_RX_DETECTION_THRESHOLD__15to16 : 15/16
+ 04 - DP_AUX_DPHY_RX_DETECTION_THRESHOLD__31to32 : 31/32
+ 05 - DP_AUX_DPHY_RX_DETECTION_THRESHOLD__63to64 : 63/64
+ 06 - DP_AUX_DPHY_RX_DETECTION_THRESHOLD__127to128 : 127/128
+ 07 - DP_AUX_DPHY_RX_DETECTION_THRESHOLD__255to256 : 255/256
+*/
+
+/*
+ AUX_REG_UPDATE_5(AUX_DPHY_RX_CONTROL0,
+ AUX_RX_START_WINDOW = 1 [6:4]
+ AUX_RX_RECEIVE_WINDOW = 1 default is 2 [10:8]
+ AUX_RX_HALF_SYM_DETECT_LEN = 1 [13:12] default is 1
+ AUX_RX_TRANSITION_FILTER_EN = 1 [16] default is 1
+ AUX_RX_ALLOW_BELOW_THRESHOLD_PHASE_DETECT [17] is 0 default is 0
+ AUX_RX_ALLOW_BELOW_THRESHOLD_START [18] is 1 default is 1
+ AUX_RX_ALLOW_BELOW_THRESHOLD_STOP [19] is 1 default is 1
+ AUX_RX_PHASE_DETECT_LEN, [21,20] = 0x3 default is 3
+ AUX_RX_DETECTION_THRESHOLD [30:28] = 1
+*/
+ AUX_REG_WRITE(AUX_DPHY_RX_CONTROL0, 0x103d1110);
+
+ AUX_REG_WRITE(AUX_DPHY_TX_CONTROL, 0x21c7a);
+
+ //AUX_DPHY_TX_REF_CONTROL'AUX_TX_REF_DIV HW default is 0x32;
+ // Set AUX_TX_REF_DIV Divider to generate 2 MHz reference from refclk
+ // 27MHz -> 0xd
+ // 100MHz -> 0x32
+ // 48MHz -> 0x18
+
+ // Set TMDS_CTL0 to 1. This is a legacy setting.
+ REG_UPDATE(TMDS_CTL_BITS, TMDS_CTL0, 1);
+
+ dcn10_aux_initialize(enc10);
+}
+
+static const struct link_encoder_funcs dcn20_link_enc_funcs = {
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ .read_state = link_enc2_read_state,
+#endif
+ .validate_output_with_stream =
+ dcn10_link_encoder_validate_output_with_stream,
+ .hw_init = enc2_hw_init,
+ .setup = dcn10_link_encoder_setup,
+ .enable_tmds_output = dcn10_link_encoder_enable_tmds_output,
+ .enable_dp_output = dcn20_link_encoder_enable_dp_output,
+ .enable_dp_mst_output = dcn10_link_encoder_enable_dp_mst_output,
+ .disable_output = dcn10_link_encoder_disable_output,
+ .dp_set_lane_settings = dcn10_link_encoder_dp_set_lane_settings,
+ .dp_set_phy_pattern = dcn10_link_encoder_dp_set_phy_pattern,
+ .update_mst_stream_allocation_table =
+ dcn10_link_encoder_update_mst_stream_allocation_table,
+ .psr_program_dp_dphy_fast_training =
+ dcn10_psr_program_dp_dphy_fast_training,
+ .psr_program_secondary_packet = dcn10_psr_program_secondary_packet,
+ .connect_dig_be_to_fe = dcn10_link_encoder_connect_dig_be_to_fe,
+ .enable_hpd = dcn10_link_encoder_enable_hpd,
+ .disable_hpd = dcn10_link_encoder_disable_hpd,
+ .is_dig_enabled = dcn10_is_dig_enabled,
+ .destroy = dcn10_link_encoder_destroy,
+ .fec_set_enable = enc2_fec_set_enable,
+ .fec_set_ready = enc2_fec_set_ready,
+ .fec_is_active = enc2_fec_is_active,
+ .get_dig_frontend = dcn10_get_dig_frontend,
+};
+
+void dcn20_link_encoder_construct(
+ struct dcn20_link_encoder *enc20,
+ const struct encoder_init_data *init_data,
+ const struct encoder_feature_support *enc_features,
+ const struct dcn10_link_enc_registers *link_regs,
+ const struct dcn10_link_enc_aux_registers *aux_regs,
+ const struct dcn10_link_enc_hpd_registers *hpd_regs,
+ const struct dcn10_link_enc_shift *link_shift,
+ const struct dcn10_link_enc_mask *link_mask)
+{
+ struct bp_encoder_cap_info bp_cap_info = {0};
+ const struct dc_vbios_funcs *bp_funcs = init_data->ctx->dc_bios->funcs;
+ enum bp_result result = BP_RESULT_OK;
+ struct dcn10_link_encoder *enc10 = &enc20->enc10;
+
+ enc10->base.funcs = &dcn20_link_enc_funcs;
+ enc10->base.ctx = init_data->ctx;
+ enc10->base.id = init_data->encoder;
+
+ enc10->base.hpd_source = init_data->hpd_source;
+ enc10->base.connector = init_data->connector;
+
+ enc10->base.preferred_engine = ENGINE_ID_UNKNOWN;
+
+ enc10->base.features = *enc_features;
+
+ enc10->base.transmitter = init_data->transmitter;
+
+ /* set the flag to indicate whether driver poll the I2C data pin
+ * while doing the DP sink detect
+ */
+
+/* if (dal_adapter_service_is_feature_supported(as,
+ FEATURE_DP_SINK_DETECT_POLL_DATA_PIN))
+ enc10->base.features.flags.bits.
+ DP_SINK_DETECT_POLL_DATA_PIN = true;*/
+
+ enc10->base.output_signals =
+ SIGNAL_TYPE_DVI_SINGLE_LINK |
+ SIGNAL_TYPE_DVI_DUAL_LINK |
+ SIGNAL_TYPE_LVDS |
+ SIGNAL_TYPE_DISPLAY_PORT |
+ SIGNAL_TYPE_DISPLAY_PORT_MST |
+ SIGNAL_TYPE_EDP |
+ SIGNAL_TYPE_HDMI_TYPE_A;
+
+ /* For DCE 8.0 and 8.1, by design, UNIPHY is hardwired to DIG_BE.
+ * SW always assign DIG_FE 1:1 mapped to DIG_FE for non-MST UNIPHY.
+ * SW assign DIG_FE to non-MST UNIPHY first and MST last. So prefer
+ * DIG is per UNIPHY and used by SST DP, eDP, HDMI, DVI and LVDS.
+ * Prefer DIG assignment is decided by board design.
+ * For DCE 8.0, there are only max 6 UNIPHYs, we assume board design
+ * and VBIOS will filter out 7 UNIPHY for DCE 8.0.
+ * By this, adding DIGG should not hurt DCE 8.0.
+ * This will let DCE 8.1 share DCE 8.0 as much as possible
+ */
+
+ enc10->link_regs = link_regs;
+ enc10->aux_regs = aux_regs;
+ enc10->hpd_regs = hpd_regs;
+ enc10->link_shift = link_shift;
+ enc10->link_mask = link_mask;
+
+ switch (enc10->base.transmitter) {
+ case TRANSMITTER_UNIPHY_A:
+ enc10->base.preferred_engine = ENGINE_ID_DIGA;
+ break;
+ case TRANSMITTER_UNIPHY_B:
+ enc10->base.preferred_engine = ENGINE_ID_DIGB;
+ break;
+ case TRANSMITTER_UNIPHY_C:
+ enc10->base.preferred_engine = ENGINE_ID_DIGC;
+ break;
+ case TRANSMITTER_UNIPHY_D:
+ enc10->base.preferred_engine = ENGINE_ID_DIGD;
+ break;
+ case TRANSMITTER_UNIPHY_E:
+ enc10->base.preferred_engine = ENGINE_ID_DIGE;
+ break;
+ case TRANSMITTER_UNIPHY_F:
+ enc10->base.preferred_engine = ENGINE_ID_DIGF;
+ break;
+ case TRANSMITTER_UNIPHY_G:
+ enc10->base.preferred_engine = ENGINE_ID_DIGG;
+ break;
+ default:
+ ASSERT_CRITICAL(false);
+ enc10->base.preferred_engine = ENGINE_ID_UNKNOWN;
+ }
+
+ /* default to one to mirror Windows behavior */
+ enc10->base.features.flags.bits.HDMI_6GB_EN = 1;
+
+ result = bp_funcs->get_encoder_cap_info(enc10->base.ctx->dc_bios,
+ enc10->base.id, &bp_cap_info);
+
+ /* Override features with DCE-specific values */
+ if (result == BP_RESULT_OK) {
+ enc10->base.features.flags.bits.IS_HBR2_CAPABLE =
+ bp_cap_info.DP_HBR2_EN;
+ enc10->base.features.flags.bits.IS_HBR3_CAPABLE =
+ bp_cap_info.DP_HBR3_EN;
+ enc10->base.features.flags.bits.HDMI_6GB_EN = bp_cap_info.HDMI_6GB_EN;
+ enc10->base.features.flags.bits.DP_IS_USB_C =
+ bp_cap_info.DP_IS_USB_C;
+ } else {
+ DC_LOG_WARNING("%s: Failed to get encoder_cap_info from VBIOS with error code %d!\n",
+ __func__,
+ result);
+ }
+ if (enc10->base.ctx->dc->debug.hdmi20_disable) {
+ enc10->base.features.flags.bits.HDMI_6GB_EN = 0;
+ }
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.h
new file mode 100644
index 000000000000..3736b5548a25
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.h
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_LINK_ENCODER__DCN20_H__
+#define __DC_LINK_ENCODER__DCN20_H__
+
+#include "dcn10/dcn10_link_encoder.h"
+
+#define DCN2_AUX_REG_LIST(id)\
+ AUX_REG_LIST(id), \
+ SRI(AUX_DPHY_TX_CONTROL, DP_AUX, id)
+
+#define UNIPHY_MASK_SH_LIST(mask_sh)\
+ LE_SF(UNIPHYA_CHANNEL_XBAR_CNTL, UNIPHY_LINK_ENABLE, mask_sh)
+
+#define LINK_ENCODER_MASK_SH_LIST_DCN20(mask_sh)\
+ LINK_ENCODER_MASK_SH_LIST_DCN10(mask_sh),\
+ LE_SF(DP0_DP_DPHY_CNTL, DPHY_FEC_EN, mask_sh),\
+ LE_SF(DP0_DP_DPHY_CNTL, DPHY_FEC_READY_SHADOW, mask_sh),\
+ LE_SF(DP0_DP_DPHY_CNTL, DPHY_FEC_ACTIVE_STATUS, mask_sh),\
+ LE_SF(DIG0_DIG_LANE_ENABLE, DIG_LANE0EN, mask_sh),\
+ LE_SF(DIG0_DIG_LANE_ENABLE, DIG_LANE1EN, mask_sh),\
+ LE_SF(DIG0_DIG_LANE_ENABLE, DIG_LANE2EN, mask_sh),\
+ LE_SF(DIG0_DIG_LANE_ENABLE, DIG_LANE3EN, mask_sh),\
+ LE_SF(DIG0_DIG_LANE_ENABLE, DIG_CLK_EN, mask_sh),\
+ LE_SF(DIG0_TMDS_CTL_BITS, TMDS_CTL0, mask_sh), \
+ UNIPHY_MASK_SH_LIST(mask_sh),\
+ LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_START_WINDOW, mask_sh),\
+ LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_HALF_SYM_DETECT_LEN, mask_sh),\
+ LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_TRANSITION_FILTER_EN, mask_sh),\
+ LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_ALLOW_BELOW_THRESHOLD_PHASE_DETECT, mask_sh),\
+ LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_ALLOW_BELOW_THRESHOLD_START, mask_sh),\
+ LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_ALLOW_BELOW_THRESHOLD_STOP, mask_sh),\
+ LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_PHASE_DETECT_LEN, mask_sh),\
+ LE_SF(DP_AUX0_AUX_DPHY_RX_CONTROL0, AUX_RX_DETECTION_THRESHOLD, mask_sh), \
+ LE_SF(DP_AUX0_AUX_DPHY_TX_CONTROL, AUX_TX_PRECHARGE_LEN, mask_sh),\
+ LE_SF(DP_AUX0_AUX_DPHY_TX_CONTROL, AUX_TX_PRECHARGE_SYMBOLS, mask_sh),\
+ LE_SF(DP_AUX0_AUX_DPHY_TX_CONTROL, AUX_MODE_DET_CHECK_DELAY, mask_sh)
+
+#define UNIPHY_DCN2_REG_LIST(id) \
+ SRI(CLOCK_ENABLE, SYMCLK, id), \
+ SRI(CHANNEL_XBAR_CNTL, UNIPHY, id)
+
+struct mpll_cfg {
+ uint32_t mpllb_ana_v2i;
+ uint32_t mpllb_ana_freq_vco;
+ uint32_t mpllb_ana_cp_int;
+ uint32_t mpllb_ana_cp_prop;
+ uint32_t mpllb_multiplier;
+ uint32_t ref_clk_mpllb_div;
+ bool mpllb_word_div2_en;
+ bool mpllb_ssc_en;
+ bool mpllb_div5_clk_en;
+ bool mpllb_div_clk_en;
+ bool mpllb_fracn_en;
+ bool mpllb_pmix_en;
+ uint32_t mpllb_div_multiplier;
+ uint32_t mpllb_tx_clk_div;
+ uint32_t mpllb_fracn_quot;
+ uint32_t mpllb_fracn_den;
+ uint32_t mpllb_ssc_peak;
+ uint32_t mpllb_ssc_stepsize;
+ uint32_t mpllb_ssc_up_spread;
+ uint32_t mpllb_fracn_rem;
+ uint32_t mpllb_hdmi_div;
+ // TODO: May not mpll params, need to figure out.
+ uint32_t tx_vboost_lvl;
+ uint32_t hdmi_pixel_clk_div;
+ uint32_t ref_range;
+ uint32_t ref_clk;
+ bool hdmimode_enable;
+};
+
+struct dpcssys_phy_seq_cfg {
+ bool program_fuse;
+ bool bypass_sram;
+ bool lane_en[4];
+ bool use_calibration_setting;
+ struct mpll_cfg mpll_cfg;
+ bool load_sram_fw;
+#if 0
+
+ bool hdmimode_enable;
+ bool silver2;
+ bool ext_refclk_en;
+ uint32_t dp_tx0_term_ctrl;
+ uint32_t dp_tx1_term_ctrl;
+ uint32_t dp_tx2_term_ctrl;
+ uint32_t dp_tx3_term_ctrl;
+ uint32_t fw_data[0x1000];
+ uint32_t dp_tx0_width;
+ uint32_t dp_tx1_width;
+ uint32_t dp_tx2_width;
+ uint32_t dp_tx3_width;
+ uint32_t dp_tx0_rate;
+ uint32_t dp_tx1_rate;
+ uint32_t dp_tx2_rate;
+ uint32_t dp_tx3_rate;
+ uint32_t dp_tx0_eq_main;
+ uint32_t dp_tx0_eq_pre;
+ uint32_t dp_tx0_eq_post;
+ uint32_t dp_tx1_eq_main;
+ uint32_t dp_tx1_eq_pre;
+ uint32_t dp_tx1_eq_post;
+ uint32_t dp_tx2_eq_main;
+ uint32_t dp_tx2_eq_pre;
+ uint32_t dp_tx2_eq_post;
+ uint32_t dp_tx3_eq_main;
+ uint32_t dp_tx3_eq_pre;
+ uint32_t dp_tx3_eq_post;
+ bool data_swap_en;
+ bool data_order_invert_en;
+ uint32_t ldpcs_fifo_start_delay;
+ uint32_t rdpcs_fifo_start_delay;
+ bool rdpcs_reg_fifo_error_mask;
+ bool rdpcs_tx_fifo_error_mask;
+ bool rdpcs_dpalt_disable_mask;
+ bool rdpcs_dpalt_4lane_mask;
+#endif
+};
+
+struct dcn20_link_encoder {
+ struct dcn10_link_encoder enc10;
+ struct dpcssys_phy_seq_cfg phy_seq_cfg;
+};
+
+void enc2_fec_set_enable(struct link_encoder *enc, bool enable);
+void enc2_fec_set_ready(struct link_encoder *enc, bool ready);
+bool enc2_fec_is_active(struct link_encoder *enc);
+void enc2_hw_init(struct link_encoder *enc);
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+void link_enc2_read_state(struct link_encoder *enc, struct link_enc_state *s);
+#endif
+
+void dcn20_link_encoder_enable_dp_output(
+ struct link_encoder *enc,
+ const struct dc_link_settings *link_settings,
+ enum clock_source_id clock_source);
+
+void dcn20_link_encoder_construct(
+ struct dcn20_link_encoder *enc20,
+ const struct encoder_init_data *init_data,
+ const struct encoder_feature_support *enc_features,
+ const struct dcn10_link_enc_registers *link_regs,
+ const struct dcn10_link_enc_aux_registers *aux_regs,
+ const struct dcn10_link_enc_hpd_registers *hpd_regs,
+ const struct dcn10_link_enc_shift *link_shift,
+ const struct dcn10_link_enc_mask *link_mask);
+
+#endif /* __DC_LINK_ENCODER__DCN20_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.c
new file mode 100644
index 000000000000..694260c10a01
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.c
@@ -0,0 +1,323 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+
+#include "reg_helper.h"
+#include "resource.h"
+#include "mcif_wb.h"
+#include "dcn20_mmhubbub.h"
+
+
+#define REG(reg)\
+ mcif_wb20->mcif_wb_regs->reg
+
+#define CTX \
+ mcif_wb20->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name) \
+ mcif_wb20->mcif_wb_shift->field_name, mcif_wb20->mcif_wb_mask->field_name
+
+#define MCIF_ADDR(addr) (((unsigned long long)addr & 0xffffffffff) + 0xFE) >> 8
+#define MCIF_ADDR_HIGH(addr) (unsigned long long)addr >> 40
+
+/* wbif programming guide:
+ * 1. set up wbif parameter:
+ * unsigned long long luma_address[4]; //4 frame buffer
+ * unsigned long long chroma_address[4];
+ * unsigned int luma_pitch;
+ * unsigned int chroma_pitch;
+ * unsigned int warmup_pitch=0x10; //256B align, the page size is 4KB when it is 0x10
+ * unsigned int slice_lines; //slice size
+ * unsigned int time_per_pixel; // time per pixel, in ns
+ * unsigned int arbitration_slice; // 0: 512 bytes 1: 1024 bytes 2: 2048 Bytes
+ * unsigned int max_scaled_time; // used for QOS generation
+ * unsigned int swlock=0x0;
+ * unsigned int cli_watermark[4]; //4 group urgent watermark
+ * unsigned int pstate_watermark[4]; //4 group pstate watermark
+ * unsigned int sw_int_en; // Software interrupt enable, frame end and overflow
+ * unsigned int sw_slice_int_en; // slice end interrupt enable
+ * unsigned int sw_overrun_int_en; // overrun error interrupt enable
+ * unsigned int vce_int_en; // VCE interrupt enable, frame end and overflow
+ * unsigned int vce_slice_int_en; // VCE slice end interrupt enable, frame end and overflow
+ *
+ * 2. configure wbif register
+ * a. call mmhubbub_config_wbif()
+ *
+ * 3. Enable wbif
+ * call set_wbif_bufmgr_enable();
+ *
+ * 4. wbif_dump_status(), option, for debug purpose
+ * the bufmgr status can show the progress of write back, can be used for debug purpose
+ */
+
+static void mmhubbub2_config_mcif_buf(struct mcif_wb *mcif_wb,
+ struct mcif_buf_params *params,
+ unsigned int dest_height)
+{
+ struct dcn20_mmhubbub *mcif_wb20 = TO_DCN20_MMHUBBUB(mcif_wb);
+
+ /* sw lock buffer0~buffer3, default is 0 */
+ REG_UPDATE(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_LOCK, params->swlock);
+
+ /* buffer address for packing mode or Luma in planar mode */
+ REG_UPDATE(MCIF_WB_BUF_1_ADDR_Y, MCIF_WB_BUF_1_ADDR_Y, MCIF_ADDR(params->luma_address[0]));
+ REG_UPDATE(MCIF_WB_BUF_1_ADDR_Y_HIGH, MCIF_WB_BUF_1_ADDR_Y_HIGH, MCIF_ADDR_HIGH(params->luma_address[0]));
+ /* right eye sub-buffer address offset for packing mode or Luma in planar mode */
+ REG_UPDATE(MCIF_WB_BUF_1_ADDR_Y_OFFSET, MCIF_WB_BUF_1_ADDR_Y_OFFSET, 0);
+
+ /* buffer address for Chroma in planar mode (unused in packing mode) */
+ REG_UPDATE(MCIF_WB_BUF_1_ADDR_C, MCIF_WB_BUF_1_ADDR_C, MCIF_ADDR(params->chroma_address[0]));
+ REG_UPDATE(MCIF_WB_BUF_1_ADDR_C_HIGH, MCIF_WB_BUF_1_ADDR_C_HIGH, MCIF_ADDR_HIGH(params->chroma_address[0]));
+ /* right eye offset for packing mode or Luma in planar mode */
+ REG_UPDATE(MCIF_WB_BUF_1_ADDR_C_OFFSET, MCIF_WB_BUF_1_ADDR_C_OFFSET, 0);
+
+ /* buffer address for packing mode or Luma in planar mode */
+ REG_UPDATE(MCIF_WB_BUF_2_ADDR_Y, MCIF_WB_BUF_2_ADDR_Y, MCIF_ADDR(params->luma_address[1]));
+ REG_UPDATE(MCIF_WB_BUF_2_ADDR_Y_HIGH, MCIF_WB_BUF_2_ADDR_Y_HIGH, MCIF_ADDR_HIGH(params->luma_address[1]));
+ /* right eye sub-buffer address offset for packing mode or Luma in planar mode */
+ REG_UPDATE(MCIF_WB_BUF_2_ADDR_Y_OFFSET, MCIF_WB_BUF_2_ADDR_Y_OFFSET, 0);
+
+ /* buffer address for Chroma in planar mode (unused in packing mode) */
+ REG_UPDATE(MCIF_WB_BUF_2_ADDR_C, MCIF_WB_BUF_2_ADDR_C, MCIF_ADDR(params->chroma_address[1]));
+ REG_UPDATE(MCIF_WB_BUF_2_ADDR_C_HIGH, MCIF_WB_BUF_2_ADDR_C_HIGH, MCIF_ADDR_HIGH(params->chroma_address[1]));
+ /* right eye offset for packing mode or Luma in planar mode */
+ REG_UPDATE(MCIF_WB_BUF_2_ADDR_C_OFFSET, MCIF_WB_BUF_2_ADDR_C_OFFSET, 0);
+
+ /* buffer address for packing mode or Luma in planar mode */
+ REG_UPDATE(MCIF_WB_BUF_3_ADDR_Y, MCIF_WB_BUF_3_ADDR_Y, MCIF_ADDR(params->luma_address[2]));
+ REG_UPDATE(MCIF_WB_BUF_3_ADDR_Y_HIGH, MCIF_WB_BUF_3_ADDR_Y_HIGH, MCIF_ADDR_HIGH(params->luma_address[2]));
+ /* right eye sub-buffer address offset for packing mode or Luma in planar mode */
+ REG_UPDATE(MCIF_WB_BUF_3_ADDR_Y_OFFSET, MCIF_WB_BUF_3_ADDR_Y_OFFSET, 0);
+
+ /* buffer address for Chroma in planar mode (unused in packing mode) */
+ REG_UPDATE(MCIF_WB_BUF_3_ADDR_C, MCIF_WB_BUF_3_ADDR_C, MCIF_ADDR(params->chroma_address[2]));
+ REG_UPDATE(MCIF_WB_BUF_3_ADDR_C_HIGH, MCIF_WB_BUF_3_ADDR_C_HIGH, MCIF_ADDR_HIGH(params->chroma_address[2]));
+ /* right eye offset for packing mode or Luma in planar mode */
+ REG_UPDATE(MCIF_WB_BUF_3_ADDR_C_OFFSET, MCIF_WB_BUF_3_ADDR_C_OFFSET, 0);
+
+ /* buffer address for packing mode or Luma in planar mode */
+ REG_UPDATE(MCIF_WB_BUF_4_ADDR_Y, MCIF_WB_BUF_4_ADDR_Y, MCIF_ADDR(params->luma_address[3]));
+ REG_UPDATE(MCIF_WB_BUF_4_ADDR_Y_HIGH, MCIF_WB_BUF_4_ADDR_Y_HIGH, MCIF_ADDR_HIGH(params->luma_address[3]));
+ /* right eye sub-buffer address offset for packing mode or Luma in planar mode */
+ REG_UPDATE(MCIF_WB_BUF_4_ADDR_Y_OFFSET, MCIF_WB_BUF_4_ADDR_Y_OFFSET, 0);
+
+ /* buffer address for Chroma in planar mode (unused in packing mode) */
+ REG_UPDATE(MCIF_WB_BUF_4_ADDR_C, MCIF_WB_BUF_4_ADDR_C, MCIF_ADDR(params->chroma_address[3]));
+ REG_UPDATE(MCIF_WB_BUF_4_ADDR_C_HIGH, MCIF_WB_BUF_4_ADDR_C_HIGH, MCIF_ADDR_HIGH(params->chroma_address[3]));
+ /* right eye offset for packing mode or Luma in planar mode */
+ REG_UPDATE(MCIF_WB_BUF_4_ADDR_C_OFFSET, MCIF_WB_BUF_4_ADDR_C_OFFSET, 0);
+
+ /* setup luma & chroma size
+ * should be enough to contain a whole frame Luma data,
+ * the programmed value is frame buffer size [27:8], 256-byte aligned
+ */
+ REG_UPDATE(MCIF_WB_BUF_LUMA_SIZE, MCIF_WB_BUF_LUMA_SIZE, (params->luma_pitch>>8) * dest_height);
+ REG_UPDATE(MCIF_WB_BUF_CHROMA_SIZE, MCIF_WB_BUF_CHROMA_SIZE, (params->chroma_pitch>>8) * dest_height);
+
+ /* enable address fence */
+ REG_UPDATE(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUF_ADDR_FENCE_EN, 1);
+
+ /* setup pitch, the programmed value is [15:8], 256B align */
+ REG_UPDATE_2(MCIF_WB_BUF_PITCH, MCIF_WB_BUF_LUMA_PITCH, params->luma_pitch >> 8,
+ MCIF_WB_BUF_CHROMA_PITCH, params->chroma_pitch >> 8);
+
+ /* Set pitch for MC cache warm up mode */
+ /* Pitch is 256 bytes aligned. The default pitch is 4K */
+ /* default is 0x10 */
+ REG_UPDATE(MCIF_WB_WARM_UP_CNTL, MCIF_WB_PITCH_SIZE_WARMUP, params->warmup_pitch);
+}
+
+static void mmhubbub2_config_mcif_arb(struct mcif_wb *mcif_wb,
+ struct mcif_arb_params *params)
+{
+ struct dcn20_mmhubbub *mcif_wb20 = TO_DCN20_MMHUBBUB(mcif_wb);
+
+ /* Programmed by the video driver based on the CRTC timing (for DWB) */
+ REG_UPDATE(MCIF_WB_ARBITRATION_CONTROL, MCIF_WB_TIME_PER_PIXEL, params->time_per_pixel);
+
+ /* Programming dwb watermark */
+ /* Watermark to generate urgent in MCIF_WB_CLI, value is determined by MCIF_WB_CLI_WATERMARK_MASK. */
+ /* Program in ns. A formula will be provided in the pseudo code to calculate the value. */
+ REG_UPDATE(MCIF_WB_SCLK_CHANGE, MCIF_WB_CLI_WATERMARK_MASK, 0x0);
+ /* urgent_watermarkA */
+ REG_UPDATE(MCIF_WB_WATERMARK, MCIF_WB_CLI_WATERMARK, params->cli_watermark[0]);
+ REG_UPDATE(MCIF_WB_SCLK_CHANGE, MCIF_WB_CLI_WATERMARK_MASK, 0x1);
+ /* urgent_watermarkB */
+ REG_UPDATE(MCIF_WB_WATERMARK, MCIF_WB_CLI_WATERMARK, params->cli_watermark[1]);
+ REG_UPDATE(MCIF_WB_SCLK_CHANGE, MCIF_WB_CLI_WATERMARK_MASK, 0x2);
+ /* urgent_watermarkC */
+ REG_UPDATE(MCIF_WB_WATERMARK, MCIF_WB_CLI_WATERMARK, params->cli_watermark[2]);
+ REG_UPDATE(MCIF_WB_SCLK_CHANGE, MCIF_WB_CLI_WATERMARK_MASK, 0x3);
+ /* urgent_watermarkD */
+ REG_UPDATE(MCIF_WB_WATERMARK, MCIF_WB_CLI_WATERMARK, params->cli_watermark[3]);
+
+ /* Programming nb pstate watermark */
+ /* nbp_state_change_watermarkA */
+ REG_UPDATE(MCIF_WB_NB_PSTATE_CONTROL, NB_PSTATE_CHANGE_WATERMARK_MASK, 0x0);
+ REG_UPDATE(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK,
+ NB_PSTATE_CHANGE_REFRESH_WATERMARK, params->pstate_watermark[0]);
+ /* nbp_state_change_watermarkB */
+ REG_UPDATE(MCIF_WB_NB_PSTATE_CONTROL, NB_PSTATE_CHANGE_WATERMARK_MASK, 0x1);
+ REG_UPDATE(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK,
+ NB_PSTATE_CHANGE_REFRESH_WATERMARK, params->pstate_watermark[1]);
+ /* nbp_state_change_watermarkC */
+ REG_UPDATE(MCIF_WB_NB_PSTATE_CONTROL, NB_PSTATE_CHANGE_WATERMARK_MASK, 0x2);
+ REG_UPDATE(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK,
+ NB_PSTATE_CHANGE_REFRESH_WATERMARK, params->pstate_watermark[2]);
+ /* nbp_state_change_watermarkD */
+ REG_UPDATE(MCIF_WB_NB_PSTATE_CONTROL, NB_PSTATE_CHANGE_WATERMARK_MASK, 0x3);
+ REG_UPDATE(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK,
+ NB_PSTATE_CHANGE_REFRESH_WATERMARK, params->pstate_watermark[3]);
+
+ /* max_scaled_time */
+ REG_UPDATE(MULTI_LEVEL_QOS_CTRL, MAX_SCALED_TIME_TO_URGENT, params->max_scaled_time);
+
+ /* slice_lines */
+ REG_UPDATE(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_SLICE_SIZE, params->slice_lines-1);
+
+ /* Set arbitration unit for Luma/Chroma */
+ /* arb_unit=2 should be chosen for more efficiency */
+ /* Arbitration size, 0: 512 bytes 1: 1024 bytes 2: 2048 Bytes */
+ REG_UPDATE(MCIF_WB_ARBITRATION_CONTROL, MCIF_WB_CLIENT_ARBITRATION_SLICE, params->arbitration_slice);
+}
+
+void mmhubbub2_config_mcif_irq(struct mcif_wb *mcif_wb,
+ struct mcif_irq_params *params)
+{
+ struct dcn20_mmhubbub *mcif_wb20 = TO_DCN20_MMHUBBUB(mcif_wb);
+
+ /* Set interrupt mask */
+ REG_UPDATE(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_INT_EN, params->sw_int_en);
+ REG_UPDATE(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_SLICE_INT_EN, params->sw_slice_int_en);
+ REG_UPDATE(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_OVERRUN_INT_EN, params->sw_overrun_int_en);
+
+ REG_UPDATE(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_EN, params->vce_int_en);
+ REG_UPDATE(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_SLICE_INT_EN, params->vce_slice_int_en);
+}
+
+void mmhubbub2_enable_mcif(struct mcif_wb *mcif_wb)
+{
+ struct dcn20_mmhubbub *mcif_wb20 = TO_DCN20_MMHUBBUB(mcif_wb);
+
+ /* Enable Mcifwb */
+ REG_UPDATE(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_ENABLE, 1);
+}
+
+void mmhubbub2_disable_mcif(struct mcif_wb *mcif_wb)
+{
+ struct dcn20_mmhubbub *mcif_wb20 = TO_DCN20_MMHUBBUB(mcif_wb);
+
+ /* disable buffer manager */
+ REG_UPDATE(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_ENABLE, 0);
+}
+
+/* set which group of pstate watermark to use and set wbif watermark change request */
+/*
+static void mmhubbub2_wbif_watermark_change_req(struct mcif_wb *mcif_wb, unsigned int wm_set)
+{
+ struct dcn20_mmhubbub *mcif_wb20 = TO_DCN20_MMHUBBUB(mcif_wb);
+ uint32_t change_req;
+
+ REG_GET(SMU_WM_CONTROL, MCIF_WB0_WM_CHG_REQ, &change_req);
+ change_req = (change_req == 0) ? 1 : 0;
+ REG_UPDATE(SMU_WM_CONTROL, MCIF_WB0_WM_CHG_SEL, wm_set);
+ REG_UPDATE(SMU_WM_CONTROL, MCIF_WB0_WM_CHG_REQ, change_req);
+}
+*/
+/* Set watermark change interrupt disable bit */
+/*
+static void mmhubbub2_set_wbif_watermark_change_int_disable(struct mcif_wb *mcif_wb, unsigned int ack_int_dis)
+{
+ struct dcn20_mmhubbub *mcif_wb20 = TO_DCN20_MMHUBBUB(mcif_wb);
+
+ REG_UPDATE(SMU_WM_CONTROL, MCIF_WB0_WM_CHG_ACK_INT_DIS, ack_int_dis);
+}
+*/
+/* Read watermark change interrupt status */
+/*
+unsigned int mmhubbub2_get_wbif_watermark_change_int_status(struct mcif_wb *mcif_wb)
+{
+ struct dcn20_mmhubbub *mcif_wb20 = TO_DCN20_MMHUBBUB(mcif_wb);
+ uint32_t irq_status;
+
+ REG_GET(SMU_WM_CONTROL, MCIF_WB0_WM_CHG_ACK_INT_STATUS, &irq_status);
+ return irq_status;
+}
+*/
+
+void mcifwb2_dump_frame(struct mcif_wb *mcif_wb,
+ struct mcif_buf_params *mcif_params,
+ enum dwb_scaler_mode out_format,
+ unsigned int dest_width,
+ unsigned int dest_height,
+ struct mcif_wb_frame_dump_info *dump_info,
+ unsigned char *luma_buffer,
+ unsigned char *chroma_buffer,
+ unsigned char *dest_luma_buffer,
+ unsigned char *dest_chroma_buffer)
+{
+ struct dcn20_mmhubbub *mcif_wb20 = TO_DCN20_MMHUBBUB(mcif_wb);
+
+ REG_UPDATE(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_LOCK, 0xf);
+
+ memcpy(dest_luma_buffer, luma_buffer, mcif_params->luma_pitch * dest_height);
+ memcpy(dest_chroma_buffer, chroma_buffer, mcif_params->chroma_pitch * dest_height / 2);
+
+ REG_UPDATE(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_LOCK, 0x0);
+
+ dump_info->format = out_format;
+ dump_info->width = dest_width;
+ dump_info->height = dest_height;
+ dump_info->luma_pitch = mcif_params->luma_pitch;
+ dump_info->chroma_pitch = mcif_params->chroma_pitch;
+ dump_info->size = dest_height * (mcif_params->luma_pitch + mcif_params->chroma_pitch);
+}
+
+const struct mcif_wb_funcs dcn20_mmhubbub_funcs = {
+ .enable_mcif = mmhubbub2_enable_mcif,
+ .disable_mcif = mmhubbub2_disable_mcif,
+ .config_mcif_buf = mmhubbub2_config_mcif_buf,
+ .config_mcif_arb = mmhubbub2_config_mcif_arb,
+ .config_mcif_irq = mmhubbub2_config_mcif_irq,
+ .dump_frame = mcifwb2_dump_frame,
+};
+
+void dcn20_mmhubbub_construct(struct dcn20_mmhubbub *mcif_wb20,
+ struct dc_context *ctx,
+ const struct dcn20_mmhubbub_registers *mcif_wb_regs,
+ const struct dcn20_mmhubbub_shift *mcif_wb_shift,
+ const struct dcn20_mmhubbub_mask *mcif_wb_mask,
+ int inst)
+{
+ mcif_wb20->base.ctx = ctx;
+
+ mcif_wb20->base.inst = inst;
+ mcif_wb20->base.funcs = &dcn20_mmhubbub_funcs;
+
+ mcif_wb20->mcif_wb_regs = mcif_wb_regs;
+ mcif_wb20->mcif_wb_shift = mcif_wb_shift;
+ mcif_wb20->mcif_wb_mask = mcif_wb_mask;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.h
new file mode 100644
index 000000000000..3fccd5eeecbb
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.h
@@ -0,0 +1,544 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_MCIF_WB_DCN20_H__
+#define __DC_MCIF_WB_DCN20_H__
+
+#define TO_DCN20_MMHUBBUB(mcif_wb_base) \
+ container_of(mcif_wb_base, struct dcn20_mmhubbub, base)
+
+/* DCN */
+#define BASE_INNER(seg) \
+ DCE_BASE__INST0_SEG ## seg
+
+#define BASE(seg) \
+ BASE_INNER(seg)
+
+#define SR(reg_name)\
+ .reg_name = BASE(mm ## reg_name ## _BASE_IDX) + \
+ mm ## reg_name
+
+#define SRI(reg_name, block, id)\
+ .reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+ mm ## block ## id ## _ ## reg_name
+
+#define SRI2(reg_name, block, id)\
+ .reg_name = BASE(mm ## reg_name ## _BASE_IDX) + \
+ mm ## reg_name
+
+#define SRII(reg_name, block, id)\
+ .reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+ mm ## block ## id ## _ ## reg_name
+
+#define SF(reg_name, field_name, post_fix)\
+ .field_name = reg_name ## __ ## field_name ## post_fix
+
+
+#define MCIF_WB_COMMON_REG_LIST_DCN2_0(inst) \
+ SRI(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUFMGR_CUR_LINE_R, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUFMGR_STATUS, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_PITCH, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_1_STATUS, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_1_STATUS2, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_2_STATUS, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_2_STATUS2, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_3_STATUS, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_3_STATUS2, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_4_STATUS, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_4_STATUS2, MCIF_WB, inst),\
+ SRI(MCIF_WB_ARBITRATION_CONTROL, MCIF_WB, inst),\
+ SRI(MCIF_WB_SCLK_CHANGE, MCIF_WB, inst),\
+ SRI(MCIF_WB_TEST_DEBUG_INDEX, MCIF_WB, inst),\
+ SRI(MCIF_WB_TEST_DEBUG_DATA, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_1_ADDR_Y, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_1_ADDR_Y_OFFSET, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_1_ADDR_C, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_1_ADDR_C_OFFSET, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_2_ADDR_Y, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_2_ADDR_Y_OFFSET, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_2_ADDR_C, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_2_ADDR_C_OFFSET, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_3_ADDR_Y, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_3_ADDR_Y_OFFSET, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_3_ADDR_C, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_3_ADDR_C_OFFSET, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_4_ADDR_Y, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_4_ADDR_Y_OFFSET, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_4_ADDR_C, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_4_ADDR_C_OFFSET, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB, inst),\
+ SRI(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, MCIF_WB, inst),\
+ SRI(MCIF_WB_NB_PSTATE_CONTROL, MCIF_WB, inst),\
+ SRI(MCIF_WB_WATERMARK, MCIF_WB, inst),\
+ SRI(MCIF_WB_CLOCK_GATER_CONTROL, MCIF_WB, inst),\
+ SRI(MCIF_WB_WARM_UP_CNTL, MCIF_WB, inst),\
+ SRI(MCIF_WB_SELF_REFRESH_CONTROL, MCIF_WB, inst),\
+ SRI(MULTI_LEVEL_QOS_CTRL, MCIF_WB, inst),\
+ SRI(MCIF_WB_SECURITY_LEVEL, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_LUMA_SIZE, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_CHROMA_SIZE, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_1_ADDR_Y_HIGH, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_1_ADDR_C_HIGH, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_2_ADDR_Y_HIGH, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_2_ADDR_C_HIGH, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_3_ADDR_Y_HIGH, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_3_ADDR_C_HIGH, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_4_ADDR_Y_HIGH, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_4_ADDR_C_HIGH, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_1_RESOLUTION, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_2_RESOLUTION, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_3_RESOLUTION, MCIF_WB, inst),\
+ SRI(MCIF_WB_BUF_4_RESOLUTION, MCIF_WB, inst),\
+ SRI(SMU_WM_CONTROL, WBIF, inst)
+
+#define MCIF_WB_COMMON_MASK_SH_LIST_DCN2_0(mask_sh) \
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_ENABLE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_INT_EN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_INT_ACK, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_SLICE_INT_EN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_OVERRUN_INT_EN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_LOCK, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_P_VMID, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUF_ADDR_FENCE_EN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_CUR_LINE_R, MCIF_WB_BUFMGR_CUR_LINE_R, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_STATUS, MCIF_WB_BUFMGR_VCE_INT_STATUS, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_STATUS, MCIF_WB_BUFMGR_SW_INT_STATUS, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_STATUS, MCIF_WB_BUFMGR_SW_OVERRUN_INT_STATUS, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_STATUS, MCIF_WB_BUFMGR_CUR_BUF, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_STATUS, MCIF_WB_BUFMGR_BUFTAG, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_STATUS, MCIF_WB_BUFMGR_CUR_LINE_L, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_STATUS, MCIF_WB_BUFMGR_NEXT_BUF, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_PITCH, MCIF_WB_BUF_LUMA_PITCH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_PITCH, MCIF_WB_BUF_CHROMA_PITCH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_ACTIVE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_SW_LOCKED, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_VCE_LOCKED, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_OVERFLOW, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_DISABLE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_MODE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_BUFTAG, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_NXT_BUF, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_FIELD, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_CUR_LINE_L, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_LONG_LINE_ERROR, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_SHORT_LINE_ERROR, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_FRAME_LENGTH_ERROR, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS2, MCIF_WB_BUF_1_CUR_LINE_R, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS2, MCIF_WB_BUF_1_NEW_CONTENT, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS2, MCIF_WB_BUF_1_COLOR_DEPTH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS2, MCIF_WB_BUF_1_TMZ_BLACK_PIXEL, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS2, MCIF_WB_BUF_1_TMZ, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS2, MCIF_WB_BUF_1_Y_OVERRUN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_STATUS2, MCIF_WB_BUF_1_C_OVERRUN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_ACTIVE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_SW_LOCKED, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_VCE_LOCKED, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_OVERFLOW, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_DISABLE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_MODE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_BUFTAG, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_NXT_BUF, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_FIELD, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_CUR_LINE_L, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_LONG_LINE_ERROR, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_SHORT_LINE_ERROR, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_FRAME_LENGTH_ERROR, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS2, MCIF_WB_BUF_2_CUR_LINE_R, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS2, MCIF_WB_BUF_2_NEW_CONTENT, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS2, MCIF_WB_BUF_2_COLOR_DEPTH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS2, MCIF_WB_BUF_2_TMZ_BLACK_PIXEL, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS2, MCIF_WB_BUF_2_TMZ, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS2, MCIF_WB_BUF_2_Y_OVERRUN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_STATUS2, MCIF_WB_BUF_2_C_OVERRUN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_ACTIVE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_SW_LOCKED, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_VCE_LOCKED, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_OVERFLOW, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_DISABLE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_MODE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_BUFTAG, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_NXT_BUF, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_FIELD, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_CUR_LINE_L, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_LONG_LINE_ERROR, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_SHORT_LINE_ERROR, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_FRAME_LENGTH_ERROR, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS2, MCIF_WB_BUF_3_CUR_LINE_R, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS2, MCIF_WB_BUF_3_NEW_CONTENT, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS2, MCIF_WB_BUF_3_COLOR_DEPTH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS2, MCIF_WB_BUF_3_TMZ_BLACK_PIXEL, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS2, MCIF_WB_BUF_3_TMZ, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS2, MCIF_WB_BUF_3_Y_OVERRUN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_STATUS2, MCIF_WB_BUF_3_C_OVERRUN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_ACTIVE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_SW_LOCKED, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_VCE_LOCKED, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_OVERFLOW, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_DISABLE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_MODE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_BUFTAG, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_NXT_BUF, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_FIELD, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_CUR_LINE_L, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_LONG_LINE_ERROR, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_SHORT_LINE_ERROR, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_FRAME_LENGTH_ERROR, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS2, MCIF_WB_BUF_4_CUR_LINE_R, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS2, MCIF_WB_BUF_4_NEW_CONTENT, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS2, MCIF_WB_BUF_4_COLOR_DEPTH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS2, MCIF_WB_BUF_4_TMZ_BLACK_PIXEL, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS2, MCIF_WB_BUF_4_TMZ, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS2, MCIF_WB_BUF_4_Y_OVERRUN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_STATUS2, MCIF_WB_BUF_4_C_OVERRUN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_ARBITRATION_CONTROL, MCIF_WB_CLIENT_ARBITRATION_SLICE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_ARBITRATION_CONTROL, MCIF_WB_TIME_PER_PIXEL, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_SCLK_CHANGE, WM_CHANGE_ACK_FORCE_ON, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_SCLK_CHANGE, MCIF_WB_CLI_WATERMARK_MASK, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_TEST_DEBUG_INDEX, MCIF_WB_TEST_DEBUG_INDEX, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_TEST_DEBUG_DATA, MCIF_WB_TEST_DEBUG_DATA, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_ADDR_Y, MCIF_WB_BUF_1_ADDR_Y, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_ADDR_Y_OFFSET, MCIF_WB_BUF_1_ADDR_Y_OFFSET, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_ADDR_C, MCIF_WB_BUF_1_ADDR_C, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_ADDR_C_OFFSET, MCIF_WB_BUF_1_ADDR_C_OFFSET, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_ADDR_Y, MCIF_WB_BUF_2_ADDR_Y, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_ADDR_Y_OFFSET, MCIF_WB_BUF_2_ADDR_Y_OFFSET, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_ADDR_C, MCIF_WB_BUF_2_ADDR_C, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_ADDR_C_OFFSET, MCIF_WB_BUF_2_ADDR_C_OFFSET, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_ADDR_Y, MCIF_WB_BUF_3_ADDR_Y, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_ADDR_Y_OFFSET, MCIF_WB_BUF_3_ADDR_Y_OFFSET, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_ADDR_C, MCIF_WB_BUF_3_ADDR_C, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_ADDR_C_OFFSET, MCIF_WB_BUF_3_ADDR_C_OFFSET, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_ADDR_Y, MCIF_WB_BUF_4_ADDR_Y, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_ADDR_Y_OFFSET, MCIF_WB_BUF_4_ADDR_Y_OFFSET, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_ADDR_C, MCIF_WB_BUF_4_ADDR_C, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_ADDR_C_OFFSET, MCIF_WB_BUF_4_ADDR_C_OFFSET, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_LOCK_IGNORE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_EN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_ACK, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_SLICE_INT_EN, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_LOCK, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_SLICE_SIZE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, NB_PSTATE_CHANGE_REFRESH_WATERMARK, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_NB_PSTATE_CONTROL, NB_PSTATE_CHANGE_URGENT_DURING_REQUEST, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_NB_PSTATE_CONTROL, NB_PSTATE_CHANGE_FORCE_ON, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_NB_PSTATE_CONTROL, NB_PSTATE_ALLOW_FOR_URGENT, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_NB_PSTATE_CONTROL, NB_PSTATE_CHANGE_WATERMARK_MASK, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_WATERMARK, MCIF_WB_CLI_WATERMARK, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_CLOCK_GATER_CONTROL, MCIF_WB_CLI_CLOCK_GATER_OVERRIDE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_WARM_UP_CNTL, MCIF_WB_PITCH_SIZE_WARMUP, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_SELF_REFRESH_CONTROL, DIS_REFRESH_UNDER_NBPREQ, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_SELF_REFRESH_CONTROL, PERFRAME_SELF_REFRESH, mask_sh),\
+ SF(MCIF_WB0_MULTI_LEVEL_QOS_CTRL, MAX_SCALED_TIME_TO_URGENT, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_SECURITY_LEVEL, MCIF_WB_SECURITY_LEVEL, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_LUMA_SIZE, MCIF_WB_BUF_LUMA_SIZE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_CHROMA_SIZE, MCIF_WB_BUF_CHROMA_SIZE, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_ADDR_Y_HIGH, MCIF_WB_BUF_1_ADDR_Y_HIGH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_ADDR_C_HIGH, MCIF_WB_BUF_1_ADDR_C_HIGH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_ADDR_Y_HIGH, MCIF_WB_BUF_2_ADDR_Y_HIGH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_ADDR_C_HIGH, MCIF_WB_BUF_2_ADDR_C_HIGH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_ADDR_Y_HIGH, MCIF_WB_BUF_3_ADDR_Y_HIGH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_ADDR_C_HIGH, MCIF_WB_BUF_3_ADDR_C_HIGH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_ADDR_Y_HIGH, MCIF_WB_BUF_4_ADDR_Y_HIGH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_ADDR_C_HIGH, MCIF_WB_BUF_4_ADDR_C_HIGH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_RESOLUTION, MCIF_WB_BUF_1_RESOLUTION_WIDTH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_1_RESOLUTION, MCIF_WB_BUF_1_RESOLUTION_HEIGHT, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_RESOLUTION, MCIF_WB_BUF_2_RESOLUTION_WIDTH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_2_RESOLUTION, MCIF_WB_BUF_2_RESOLUTION_HEIGHT, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_RESOLUTION, MCIF_WB_BUF_3_RESOLUTION_WIDTH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_3_RESOLUTION, MCIF_WB_BUF_3_RESOLUTION_HEIGHT, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_RESOLUTION, MCIF_WB_BUF_4_RESOLUTION_WIDTH, mask_sh),\
+ SF(MCIF_WB0_MCIF_WB_BUF_4_RESOLUTION, MCIF_WB_BUF_4_RESOLUTION_HEIGHT, mask_sh),\
+ SF(WBIF0_SMU_WM_CONTROL, MCIF_WB0_WM_CHG_SEL, mask_sh),\
+ SF(WBIF0_SMU_WM_CONTROL, MCIF_WB0_WM_CHG_REQ, mask_sh),\
+ SF(WBIF0_SMU_WM_CONTROL, MCIF_WB0_WM_CHG_ACK_INT_DIS, mask_sh),\
+ SF(WBIF0_SMU_WM_CONTROL, MCIF_WB0_WM_CHG_ACK_INT_STATUS, mask_sh)
+
+#define MCIF_WB_REG_FIELD_LIST_DCN2_0(type) \
+ type MCIF_WB_BUFMGR_ENABLE;\
+ type MCIF_WB_BUFMGR_SW_INT_EN;\
+ type MCIF_WB_BUFMGR_SW_INT_ACK;\
+ type MCIF_WB_BUFMGR_SW_SLICE_INT_EN;\
+ type MCIF_WB_BUFMGR_SW_OVERRUN_INT_EN;\
+ type MCIF_WB_BUFMGR_SW_LOCK;\
+ type MCIF_WB_P_VMID;\
+ type MCIF_WB_BUF_ADDR_FENCE_EN;\
+ type MCIF_WB_BUFMGR_CUR_LINE_R;\
+ type MCIF_WB_BUFMGR_VCE_INT_STATUS;\
+ type MCIF_WB_BUFMGR_SW_INT_STATUS;\
+ type MCIF_WB_BUFMGR_SW_OVERRUN_INT_STATUS;\
+ type MCIF_WB_BUFMGR_CUR_BUF;\
+ type MCIF_WB_BUFMGR_BUFTAG;\
+ type MCIF_WB_BUFMGR_CUR_LINE_L;\
+ type MCIF_WB_BUFMGR_NEXT_BUF;\
+ type MCIF_WB_BUF_LUMA_PITCH;\
+ type MCIF_WB_BUF_CHROMA_PITCH;\
+ type MCIF_WB_BUF_1_ACTIVE;\
+ type MCIF_WB_BUF_1_SW_LOCKED;\
+ type MCIF_WB_BUF_1_VCE_LOCKED;\
+ type MCIF_WB_BUF_1_OVERFLOW;\
+ type MCIF_WB_BUF_1_DISABLE;\
+ type MCIF_WB_BUF_1_MODE;\
+ type MCIF_WB_BUF_1_BUFTAG;\
+ type MCIF_WB_BUF_1_NXT_BUF;\
+ type MCIF_WB_BUF_1_FIELD;\
+ type MCIF_WB_BUF_1_CUR_LINE_L;\
+ type MCIF_WB_BUF_1_LONG_LINE_ERROR;\
+ type MCIF_WB_BUF_1_SHORT_LINE_ERROR;\
+ type MCIF_WB_BUF_1_FRAME_LENGTH_ERROR;\
+ type MCIF_WB_BUF_1_CUR_LINE_R;\
+ type MCIF_WB_BUF_1_NEW_CONTENT;\
+ type MCIF_WB_BUF_1_COLOR_DEPTH;\
+ type MCIF_WB_BUF_1_TMZ_BLACK_PIXEL;\
+ type MCIF_WB_BUF_1_TMZ;\
+ type MCIF_WB_BUF_1_Y_OVERRUN;\
+ type MCIF_WB_BUF_1_C_OVERRUN;\
+ type MCIF_WB_BUF_2_ACTIVE;\
+ type MCIF_WB_BUF_2_SW_LOCKED;\
+ type MCIF_WB_BUF_2_VCE_LOCKED;\
+ type MCIF_WB_BUF_2_OVERFLOW;\
+ type MCIF_WB_BUF_2_DISABLE;\
+ type MCIF_WB_BUF_2_MODE;\
+ type MCIF_WB_BUF_2_BUFTAG;\
+ type MCIF_WB_BUF_2_NXT_BUF;\
+ type MCIF_WB_BUF_2_FIELD;\
+ type MCIF_WB_BUF_2_CUR_LINE_L;\
+ type MCIF_WB_BUF_2_LONG_LINE_ERROR;\
+ type MCIF_WB_BUF_2_SHORT_LINE_ERROR;\
+ type MCIF_WB_BUF_2_FRAME_LENGTH_ERROR;\
+ type MCIF_WB_BUF_2_CUR_LINE_R;\
+ type MCIF_WB_BUF_2_NEW_CONTENT;\
+ type MCIF_WB_BUF_2_COLOR_DEPTH;\
+ type MCIF_WB_BUF_2_TMZ_BLACK_PIXEL;\
+ type MCIF_WB_BUF_2_TMZ;\
+ type MCIF_WB_BUF_2_Y_OVERRUN;\
+ type MCIF_WB_BUF_2_C_OVERRUN;\
+ type MCIF_WB_BUF_3_ACTIVE;\
+ type MCIF_WB_BUF_3_SW_LOCKED;\
+ type MCIF_WB_BUF_3_VCE_LOCKED;\
+ type MCIF_WB_BUF_3_OVERFLOW;\
+ type MCIF_WB_BUF_3_DISABLE;\
+ type MCIF_WB_BUF_3_MODE;\
+ type MCIF_WB_BUF_3_BUFTAG;\
+ type MCIF_WB_BUF_3_NXT_BUF;\
+ type MCIF_WB_BUF_3_FIELD;\
+ type MCIF_WB_BUF_3_CUR_LINE_L;\
+ type MCIF_WB_BUF_3_LONG_LINE_ERROR;\
+ type MCIF_WB_BUF_3_SHORT_LINE_ERROR;\
+ type MCIF_WB_BUF_3_FRAME_LENGTH_ERROR;\
+ type MCIF_WB_BUF_3_CUR_LINE_R;\
+ type MCIF_WB_BUF_3_NEW_CONTENT;\
+ type MCIF_WB_BUF_3_COLOR_DEPTH;\
+ type MCIF_WB_BUF_3_TMZ_BLACK_PIXEL;\
+ type MCIF_WB_BUF_3_TMZ;\
+ type MCIF_WB_BUF_3_Y_OVERRUN;\
+ type MCIF_WB_BUF_3_C_OVERRUN;\
+ type MCIF_WB_BUF_4_ACTIVE;\
+ type MCIF_WB_BUF_4_SW_LOCKED;\
+ type MCIF_WB_BUF_4_VCE_LOCKED;\
+ type MCIF_WB_BUF_4_OVERFLOW;\
+ type MCIF_WB_BUF_4_DISABLE;\
+ type MCIF_WB_BUF_4_MODE;\
+ type MCIF_WB_BUF_4_BUFTAG;\
+ type MCIF_WB_BUF_4_NXT_BUF;\
+ type MCIF_WB_BUF_4_FIELD;\
+ type MCIF_WB_BUF_4_CUR_LINE_L;\
+ type MCIF_WB_BUF_4_LONG_LINE_ERROR;\
+ type MCIF_WB_BUF_4_SHORT_LINE_ERROR;\
+ type MCIF_WB_BUF_4_FRAME_LENGTH_ERROR;\
+ type MCIF_WB_BUF_4_CUR_LINE_R;\
+ type MCIF_WB_BUF_4_NEW_CONTENT;\
+ type MCIF_WB_BUF_4_COLOR_DEPTH;\
+ type MCIF_WB_BUF_4_TMZ_BLACK_PIXEL;\
+ type MCIF_WB_BUF_4_TMZ;\
+ type MCIF_WB_BUF_4_Y_OVERRUN;\
+ type MCIF_WB_BUF_4_C_OVERRUN;\
+ type MCIF_WB_CLIENT_ARBITRATION_SLICE;\
+ type MCIF_WB_TIME_PER_PIXEL;\
+ type WM_CHANGE_ACK_FORCE_ON;\
+ type MCIF_WB_CLI_WATERMARK_MASK;\
+ type MCIF_WB_TEST_DEBUG_INDEX;\
+ type MCIF_WB_TEST_DEBUG_DATA;\
+ type MCIF_WB_BUF_1_ADDR_Y;\
+ type MCIF_WB_BUF_1_ADDR_Y_OFFSET;\
+ type MCIF_WB_BUF_1_ADDR_C;\
+ type MCIF_WB_BUF_1_ADDR_C_OFFSET;\
+ type MCIF_WB_BUF_2_ADDR_Y;\
+ type MCIF_WB_BUF_2_ADDR_Y_OFFSET;\
+ type MCIF_WB_BUF_2_ADDR_C;\
+ type MCIF_WB_BUF_2_ADDR_C_OFFSET;\
+ type MCIF_WB_BUF_3_ADDR_Y;\
+ type MCIF_WB_BUF_3_ADDR_Y_OFFSET;\
+ type MCIF_WB_BUF_3_ADDR_C;\
+ type MCIF_WB_BUF_3_ADDR_C_OFFSET;\
+ type MCIF_WB_BUF_4_ADDR_Y;\
+ type MCIF_WB_BUF_4_ADDR_Y_OFFSET;\
+ type MCIF_WB_BUF_4_ADDR_C;\
+ type MCIF_WB_BUF_4_ADDR_C_OFFSET;\
+ type MCIF_WB_BUFMGR_VCE_LOCK_IGNORE;\
+ type MCIF_WB_BUFMGR_VCE_INT_EN;\
+ type MCIF_WB_BUFMGR_VCE_INT_ACK;\
+ type MCIF_WB_BUFMGR_VCE_SLICE_INT_EN;\
+ type MCIF_WB_BUFMGR_VCE_LOCK;\
+ type MCIF_WB_BUFMGR_SLICE_SIZE;\
+ type NB_PSTATE_CHANGE_REFRESH_WATERMARK;\
+ type NB_PSTATE_CHANGE_URGENT_DURING_REQUEST;\
+ type NB_PSTATE_CHANGE_FORCE_ON;\
+ type NB_PSTATE_ALLOW_FOR_URGENT;\
+ type NB_PSTATE_CHANGE_WATERMARK_MASK;\
+ type MCIF_WB_CLI_WATERMARK;\
+ type MCIF_WB_CLI_CLOCK_GATER_OVERRIDE;\
+ type MCIF_WB_PITCH_SIZE_WARMUP;\
+ type DIS_REFRESH_UNDER_NBPREQ;\
+ type PERFRAME_SELF_REFRESH;\
+ type MAX_SCALED_TIME_TO_URGENT;\
+ type MCIF_WB_SECURITY_LEVEL;\
+ type MCIF_WB_BUF_LUMA_SIZE;\
+ type MCIF_WB_BUF_CHROMA_SIZE;\
+ type MCIF_WB_BUF_1_ADDR_Y_HIGH;\
+ type MCIF_WB_BUF_1_ADDR_C_HIGH;\
+ type MCIF_WB_BUF_2_ADDR_Y_HIGH;\
+ type MCIF_WB_BUF_2_ADDR_C_HIGH;\
+ type MCIF_WB_BUF_3_ADDR_Y_HIGH;\
+ type MCIF_WB_BUF_3_ADDR_C_HIGH;\
+ type MCIF_WB_BUF_4_ADDR_Y_HIGH;\
+ type MCIF_WB_BUF_4_ADDR_C_HIGH;\
+ type MCIF_WB_BUF_1_RESOLUTION_WIDTH;\
+ type MCIF_WB_BUF_1_RESOLUTION_HEIGHT;\
+ type MCIF_WB_BUF_2_RESOLUTION_WIDTH;\
+ type MCIF_WB_BUF_2_RESOLUTION_HEIGHT;\
+ type MCIF_WB_BUF_3_RESOLUTION_WIDTH;\
+ type MCIF_WB_BUF_3_RESOLUTION_HEIGHT;\
+ type MCIF_WB_BUF_4_RESOLUTION_WIDTH;\
+ type MCIF_WB_BUF_4_RESOLUTION_HEIGHT;\
+ type MCIF_WB0_WM_CHG_SEL;\
+ type MCIF_WB0_WM_CHG_REQ;\
+ type MCIF_WB0_WM_CHG_ACK_INT_DIS;\
+ type MCIF_WB0_WM_CHG_ACK_INT_STATUS
+
+#define MCIF_WB_REG_VARIABLE_LIST_DCN2_0 \
+ uint32_t MCIF_WB_BUFMGR_SW_CONTROL;\
+ uint32_t MCIF_WB_BUFMGR_CUR_LINE_R;\
+ uint32_t MCIF_WB_BUFMGR_STATUS;\
+ uint32_t MCIF_WB_BUF_PITCH;\
+ uint32_t MCIF_WB_BUF_1_STATUS;\
+ uint32_t MCIF_WB_BUF_1_STATUS2;\
+ uint32_t MCIF_WB_BUF_2_STATUS;\
+ uint32_t MCIF_WB_BUF_2_STATUS2;\
+ uint32_t MCIF_WB_BUF_3_STATUS;\
+ uint32_t MCIF_WB_BUF_3_STATUS2;\
+ uint32_t MCIF_WB_BUF_4_STATUS;\
+ uint32_t MCIF_WB_BUF_4_STATUS2;\
+ uint32_t MCIF_WB_ARBITRATION_CONTROL;\
+ uint32_t MCIF_WB_SCLK_CHANGE;\
+ uint32_t MCIF_WB_TEST_DEBUG_INDEX;\
+ uint32_t MCIF_WB_TEST_DEBUG_DATA;\
+ uint32_t MCIF_WB_BUF_1_ADDR_Y;\
+ uint32_t MCIF_WB_BUF_1_ADDR_Y_OFFSET;\
+ uint32_t MCIF_WB_BUF_1_ADDR_C;\
+ uint32_t MCIF_WB_BUF_1_ADDR_C_OFFSET;\
+ uint32_t MCIF_WB_BUF_2_ADDR_Y;\
+ uint32_t MCIF_WB_BUF_2_ADDR_Y_OFFSET;\
+ uint32_t MCIF_WB_BUF_2_ADDR_C;\
+ uint32_t MCIF_WB_BUF_2_ADDR_C_OFFSET;\
+ uint32_t MCIF_WB_BUF_3_ADDR_Y;\
+ uint32_t MCIF_WB_BUF_3_ADDR_Y_OFFSET;\
+ uint32_t MCIF_WB_BUF_3_ADDR_C;\
+ uint32_t MCIF_WB_BUF_3_ADDR_C_OFFSET;\
+ uint32_t MCIF_WB_BUF_4_ADDR_Y;\
+ uint32_t MCIF_WB_BUF_4_ADDR_Y_OFFSET;\
+ uint32_t MCIF_WB_BUF_4_ADDR_C;\
+ uint32_t MCIF_WB_BUF_4_ADDR_C_OFFSET;\
+ uint32_t MCIF_WB_BUFMGR_VCE_CONTROL;\
+ uint32_t MCIF_WB_NB_PSTATE_LATENCY_WATERMARK;\
+ uint32_t MCIF_WB_NB_PSTATE_CONTROL;\
+ uint32_t MCIF_WB_WATERMARK;\
+ uint32_t MCIF_WB_CLOCK_GATER_CONTROL;\
+ uint32_t MCIF_WB_WARM_UP_CNTL;\
+ uint32_t MCIF_WB_SELF_REFRESH_CONTROL;\
+ uint32_t MULTI_LEVEL_QOS_CTRL;\
+ uint32_t MCIF_WB_SECURITY_LEVEL;\
+ uint32_t MCIF_WB_BUF_LUMA_SIZE;\
+ uint32_t MCIF_WB_BUF_CHROMA_SIZE;\
+ uint32_t MCIF_WB_BUF_1_ADDR_Y_HIGH;\
+ uint32_t MCIF_WB_BUF_1_ADDR_C_HIGH;\
+ uint32_t MCIF_WB_BUF_2_ADDR_Y_HIGH;\
+ uint32_t MCIF_WB_BUF_2_ADDR_C_HIGH;\
+ uint32_t MCIF_WB_BUF_3_ADDR_Y_HIGH;\
+ uint32_t MCIF_WB_BUF_3_ADDR_C_HIGH;\
+ uint32_t MCIF_WB_BUF_4_ADDR_Y_HIGH;\
+ uint32_t MCIF_WB_BUF_4_ADDR_C_HIGH;\
+ uint32_t MCIF_WB_BUF_1_RESOLUTION;\
+ uint32_t MCIF_WB_BUF_2_RESOLUTION;\
+ uint32_t MCIF_WB_BUF_3_RESOLUTION;\
+ uint32_t MCIF_WB_BUF_4_RESOLUTION;\
+ uint32_t SMU_WM_CONTROL
+
+struct dcn20_mmhubbub_registers {
+ MCIF_WB_REG_VARIABLE_LIST_DCN2_0;
+};
+
+
+struct dcn20_mmhubbub_mask {
+ MCIF_WB_REG_FIELD_LIST_DCN2_0(uint32_t);
+};
+
+struct dcn20_mmhubbub_shift {
+ MCIF_WB_REG_FIELD_LIST_DCN2_0(uint8_t);
+};
+
+struct dcn20_mmhubbub {
+ struct mcif_wb base;
+ const struct dcn20_mmhubbub_registers *mcif_wb_regs;
+ const struct dcn20_mmhubbub_shift *mcif_wb_shift;
+ const struct dcn20_mmhubbub_mask *mcif_wb_mask;
+};
+
+void mmhubbub2_config_mcif_irq(struct mcif_wb *mcif_wb,
+ struct mcif_irq_params *params);
+
+void mmhubbub2_enable_mcif(struct mcif_wb *mcif_wb);
+
+void mmhubbub2_disable_mcif(struct mcif_wb *mcif_wb);
+
+void mcifwb2_dump_frame(struct mcif_wb *mcif_wb,
+ struct mcif_buf_params *mcif_params,
+ enum dwb_scaler_mode out_format,
+ unsigned int dest_width,
+ unsigned int dest_height,
+ struct mcif_wb_frame_dump_info *dump_info,
+ unsigned char *luma_buffer,
+ unsigned char *chroma_buffer,
+ unsigned char *dest_luma_buffer,
+ unsigned char *dest_chroma_buffer);
+
+void dcn20_mmhubbub_construct(struct dcn20_mmhubbub *mcif_wb20,
+ struct dc_context *ctx,
+ const struct dcn20_mmhubbub_registers *mcif_wb_regs,
+ const struct dcn20_mmhubbub_shift *mcif_wb_shift,
+ const struct dcn20_mmhubbub_mask *mcif_wb_mask,
+ int inst);
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mpc.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mpc.c
new file mode 100644
index 000000000000..240749e4cf83
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mpc.c
@@ -0,0 +1,526 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dcn20_mpc.h"
+
+#include "reg_helper.h"
+#include "dc.h"
+#include "mem_input.h"
+#include "dcn10/dcn10_cm_common.h"
+
+#define REG(reg)\
+ mpc20->mpc_regs->reg
+
+#define CTX \
+ mpc20->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name) \
+ mpc20->mpc_shift->field_name, mpc20->mpc_mask->field_name
+
+#define NUM_ELEMENTS(a) (sizeof(a) / sizeof((a)[0]))
+
+void mpc2_update_blending(
+ struct mpc *mpc,
+ struct mpcc_blnd_cfg *blnd_cfg,
+ int mpcc_id)
+{
+ struct dcn20_mpc *mpc20 = TO_DCN20_MPC(mpc);
+
+ struct mpcc *mpcc = mpc1_get_mpcc(mpc, mpcc_id);
+
+ REG_UPDATE_7(MPCC_CONTROL[mpcc_id],
+ MPCC_ALPHA_BLND_MODE, blnd_cfg->alpha_mode,
+ MPCC_ALPHA_MULTIPLIED_MODE, blnd_cfg->pre_multiplied_alpha,
+ MPCC_BLND_ACTIVE_OVERLAP_ONLY, blnd_cfg->overlap_only,
+ MPCC_GLOBAL_ALPHA, blnd_cfg->global_alpha,
+ MPCC_GLOBAL_GAIN, blnd_cfg->global_gain,
+ MPCC_BG_BPC, blnd_cfg->background_color_bpc,
+ MPCC_BOT_GAIN_MODE, blnd_cfg->bottom_gain_mode);
+
+ REG_SET(MPCC_TOP_GAIN[mpcc_id], 0, MPCC_TOP_GAIN, blnd_cfg->top_gain);
+ REG_SET(MPCC_BOT_GAIN_INSIDE[mpcc_id], 0, MPCC_BOT_GAIN_INSIDE, blnd_cfg->bottom_inside_gain);
+ REG_SET(MPCC_BOT_GAIN_OUTSIDE[mpcc_id], 0, MPCC_BOT_GAIN_OUTSIDE, blnd_cfg->bottom_outside_gain);
+
+ mpc1_set_bg_color(mpc, &blnd_cfg->black_color, mpcc_id);
+ mpcc->blnd_cfg = *blnd_cfg;
+}
+
+void mpc2_set_denorm(
+ struct mpc *mpc,
+ int opp_id,
+ enum dc_color_depth output_depth)
+{
+ struct dcn20_mpc *mpc20 = TO_DCN20_MPC(mpc);
+ int denorm_mode = 0;
+
+ switch (output_depth) {
+ case COLOR_DEPTH_666:
+ denorm_mode = 1;
+ break;
+ case COLOR_DEPTH_888:
+ denorm_mode = 2;
+ break;
+ case COLOR_DEPTH_999:
+ denorm_mode = 3;
+ break;
+ case COLOR_DEPTH_101010:
+ denorm_mode = 4;
+ break;
+ case COLOR_DEPTH_111111:
+ denorm_mode = 5;
+ break;
+ case COLOR_DEPTH_121212:
+ denorm_mode = 6;
+ break;
+ case COLOR_DEPTH_141414:
+ case COLOR_DEPTH_161616:
+ default:
+ /* not valid used case! */
+ break;
+ }
+
+ REG_UPDATE(DENORM_CONTROL[opp_id],
+ MPC_OUT_DENORM_MODE, denorm_mode);
+}
+
+void mpc2_set_denorm_clamp(
+ struct mpc *mpc,
+ int opp_id,
+ struct mpc_denorm_clamp denorm_clamp)
+{
+ struct dcn20_mpc *mpc20 = TO_DCN20_MPC(mpc);
+
+ REG_UPDATE_2(DENORM_CONTROL[opp_id],
+ MPC_OUT_DENORM_CLAMP_MAX_R_CR, denorm_clamp.clamp_max_r_cr,
+ MPC_OUT_DENORM_CLAMP_MIN_R_CR, denorm_clamp.clamp_min_r_cr);
+ REG_UPDATE_2(DENORM_CLAMP_G_Y[opp_id],
+ MPC_OUT_DENORM_CLAMP_MAX_G_Y, denorm_clamp.clamp_max_g_y,
+ MPC_OUT_DENORM_CLAMP_MIN_G_Y, denorm_clamp.clamp_min_g_y);
+ REG_UPDATE_2(DENORM_CLAMP_B_CB[opp_id],
+ MPC_OUT_DENORM_CLAMP_MAX_B_CB, denorm_clamp.clamp_max_b_cb,
+ MPC_OUT_DENORM_CLAMP_MIN_B_CB, denorm_clamp.clamp_min_b_cb);
+}
+
+
+
+void mpc2_set_output_csc(
+ struct mpc *mpc,
+ int opp_id,
+ const uint16_t *regval,
+ enum mpc_output_csc_mode ocsc_mode)
+{
+ struct dcn20_mpc *mpc20 = TO_DCN20_MPC(mpc);
+ struct color_matrices_reg ocsc_regs;
+
+ REG_SET(CSC_MODE[opp_id], 0, MPC_OCSC_MODE, ocsc_mode);
+
+ if (ocsc_mode == MPC_OUTPUT_CSC_DISABLE)
+ return;
+
+ if (regval == NULL) {
+ BREAK_TO_DEBUGGER();
+ return;
+ }
+
+ ocsc_regs.shifts.csc_c11 = mpc20->mpc_shift->MPC_OCSC_C11_A;
+ ocsc_regs.masks.csc_c11 = mpc20->mpc_mask->MPC_OCSC_C11_A;
+ ocsc_regs.shifts.csc_c12 = mpc20->mpc_shift->MPC_OCSC_C12_A;
+ ocsc_regs.masks.csc_c12 = mpc20->mpc_mask->MPC_OCSC_C12_A;
+
+ if (ocsc_mode == MPC_OUTPUT_CSC_COEF_A) {
+ ocsc_regs.csc_c11_c12 = REG(CSC_C11_C12_A[opp_id]);
+ ocsc_regs.csc_c33_c34 = REG(CSC_C33_C34_A[opp_id]);
+ } else {
+ ocsc_regs.csc_c11_c12 = REG(CSC_C11_C12_B[opp_id]);
+ ocsc_regs.csc_c33_c34 = REG(CSC_C33_C34_B[opp_id]);
+ }
+ cm_helper_program_color_matrices(
+ mpc20->base.ctx,
+ regval,
+ &ocsc_regs);
+}
+
+void mpc2_set_ocsc_default(
+ struct mpc *mpc,
+ int opp_id,
+ enum dc_color_space color_space,
+ enum mpc_output_csc_mode ocsc_mode)
+{
+ struct dcn20_mpc *mpc20 = TO_DCN20_MPC(mpc);
+ uint32_t arr_size;
+ struct color_matrices_reg ocsc_regs;
+ const uint16_t *regval = NULL;
+
+ REG_SET(CSC_MODE[opp_id], 0, MPC_OCSC_MODE, ocsc_mode);
+ if (ocsc_mode == MPC_OUTPUT_CSC_DISABLE)
+ return;
+
+ regval = find_color_matrix(color_space, &arr_size);
+
+ if (regval == NULL) {
+ BREAK_TO_DEBUGGER();
+ return;
+ }
+
+ ocsc_regs.shifts.csc_c11 = mpc20->mpc_shift->MPC_OCSC_C11_A;
+ ocsc_regs.masks.csc_c11 = mpc20->mpc_mask->MPC_OCSC_C11_A;
+ ocsc_regs.shifts.csc_c12 = mpc20->mpc_shift->MPC_OCSC_C12_A;
+ ocsc_regs.masks.csc_c12 = mpc20->mpc_mask->MPC_OCSC_C12_A;
+
+
+ if (ocsc_mode == MPC_OUTPUT_CSC_COEF_A) {
+ ocsc_regs.csc_c11_c12 = REG(CSC_C11_C12_A[opp_id]);
+ ocsc_regs.csc_c33_c34 = REG(CSC_C33_C34_A[opp_id]);
+ } else {
+ ocsc_regs.csc_c11_c12 = REG(CSC_C11_C12_B[opp_id]);
+ ocsc_regs.csc_c33_c34 = REG(CSC_C33_C34_B[opp_id]);
+ }
+
+ cm_helper_program_color_matrices(
+ mpc20->base.ctx,
+ regval,
+ &ocsc_regs);
+}
+
+static void mpc2_ogam_get_reg_field(
+ struct mpc *mpc,
+ struct xfer_func_reg *reg)
+{
+ struct dcn20_mpc *mpc20 = TO_DCN20_MPC(mpc);
+
+ reg->shifts.exp_region0_lut_offset = mpc20->mpc_shift->MPCC_OGAM_RAMA_EXP_REGION0_LUT_OFFSET;
+ reg->masks.exp_region0_lut_offset = mpc20->mpc_mask->MPCC_OGAM_RAMA_EXP_REGION0_LUT_OFFSET;
+ reg->shifts.exp_region0_num_segments = mpc20->mpc_shift->MPCC_OGAM_RAMA_EXP_REGION0_NUM_SEGMENTS;
+ reg->masks.exp_region0_num_segments = mpc20->mpc_mask->MPCC_OGAM_RAMA_EXP_REGION0_NUM_SEGMENTS;
+ reg->shifts.exp_region1_lut_offset = mpc20->mpc_shift->MPCC_OGAM_RAMA_EXP_REGION1_LUT_OFFSET;
+ reg->masks.exp_region1_lut_offset = mpc20->mpc_mask->MPCC_OGAM_RAMA_EXP_REGION1_LUT_OFFSET;
+ reg->shifts.exp_region1_num_segments = mpc20->mpc_shift->MPCC_OGAM_RAMA_EXP_REGION1_NUM_SEGMENTS;
+ reg->masks.exp_region1_num_segments = mpc20->mpc_mask->MPCC_OGAM_RAMA_EXP_REGION1_NUM_SEGMENTS;
+ reg->shifts.field_region_end = mpc20->mpc_shift->MPCC_OGAM_RAMA_EXP_REGION_END_B;
+ reg->masks.field_region_end = mpc20->mpc_mask->MPCC_OGAM_RAMA_EXP_REGION_END_B;
+ reg->shifts.field_region_end_slope = mpc20->mpc_shift->MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_B;
+ reg->masks.field_region_end_slope = mpc20->mpc_mask->MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_B;
+ reg->shifts.field_region_end_base = mpc20->mpc_shift->MPCC_OGAM_RAMA_EXP_REGION_END_BASE_B;
+ reg->masks.field_region_end_base = mpc20->mpc_mask->MPCC_OGAM_RAMA_EXP_REGION_END_BASE_B;
+ reg->shifts.field_region_linear_slope = mpc20->mpc_shift->MPCC_OGAM_RAMA_EXP_REGION_LINEAR_SLOPE_B;
+ reg->masks.field_region_linear_slope = mpc20->mpc_mask->MPCC_OGAM_RAMA_EXP_REGION_LINEAR_SLOPE_B;
+ reg->shifts.exp_region_start = mpc20->mpc_shift->MPCC_OGAM_RAMA_EXP_REGION_START_B;
+ reg->masks.exp_region_start = mpc20->mpc_mask->MPCC_OGAM_RAMA_EXP_REGION_START_B;
+ reg->shifts.exp_resion_start_segment = mpc20->mpc_shift->MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_B;
+ reg->masks.exp_resion_start_segment = mpc20->mpc_mask->MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_B;
+}
+
+static void mpc20_power_on_ogam_lut(
+ struct mpc *mpc, int mpcc_id,
+ bool power_on)
+{
+ struct dcn20_mpc *mpc20 = TO_DCN20_MPC(mpc);
+
+ REG_SET(MPCC_MEM_PWR_CTRL[mpcc_id], 0,
+ MPCC_OGAM_MEM_PWR_FORCE, power_on == true ? 0:1);
+
+}
+
+static void mpc20_configure_ogam_lut(
+ struct mpc *mpc, int mpcc_id,
+ bool is_ram_a)
+{
+ struct dcn20_mpc *mpc20 = TO_DCN20_MPC(mpc);
+
+ REG_UPDATE_2(MPCC_OGAM_LUT_RAM_CONTROL[mpcc_id],
+ MPCC_OGAM_LUT_WRITE_EN_MASK, 7,
+ MPCC_OGAM_LUT_RAM_SEL, is_ram_a == true ? 0:1);
+
+ REG_SET(MPCC_OGAM_LUT_INDEX[mpcc_id], 0, MPCC_OGAM_LUT_INDEX, 0);
+}
+
+static enum dc_lut_mode mpc20_get_ogam_current(struct mpc *mpc, int mpcc_id)
+{
+ enum dc_lut_mode mode;
+ uint32_t state_mode;
+ struct dcn20_mpc *mpc20 = TO_DCN20_MPC(mpc);
+
+ REG_GET(MPCC_OGAM_LUT_RAM_CONTROL[mpcc_id],
+ MPCC_OGAM_CONFIG_STATUS, &state_mode);
+
+ switch (state_mode) {
+ case 0:
+ mode = LUT_BYPASS;
+ break;
+ case 1:
+ mode = LUT_RAM_A;
+ break;
+ case 2:
+ mode = LUT_RAM_B;
+ break;
+ default:
+ mode = LUT_BYPASS;
+ break;
+ }
+ return mode;
+}
+
+static void mpc2_program_lutb(struct mpc *mpc, int mpcc_id,
+ const struct pwl_params *params)
+{
+ struct dcn20_mpc *mpc20 = TO_DCN20_MPC(mpc);
+ struct xfer_func_reg gam_regs;
+
+ mpc2_ogam_get_reg_field(mpc, &gam_regs);
+
+ gam_regs.start_cntl_b = REG(MPCC_OGAM_RAMB_START_CNTL_B[mpcc_id]);
+ gam_regs.start_cntl_g = REG(MPCC_OGAM_RAMB_START_CNTL_G[mpcc_id]);
+ gam_regs.start_cntl_r = REG(MPCC_OGAM_RAMB_START_CNTL_R[mpcc_id]);
+ gam_regs.start_slope_cntl_b = REG(MPCC_OGAM_RAMB_SLOPE_CNTL_B[mpcc_id]);
+ gam_regs.start_slope_cntl_g = REG(MPCC_OGAM_RAMB_SLOPE_CNTL_G[mpcc_id]);
+ gam_regs.start_slope_cntl_r = REG(MPCC_OGAM_RAMB_SLOPE_CNTL_R[mpcc_id]);
+ gam_regs.start_end_cntl1_b = REG(MPCC_OGAM_RAMB_END_CNTL1_B[mpcc_id]);
+ gam_regs.start_end_cntl2_b = REG(MPCC_OGAM_RAMB_END_CNTL2_B[mpcc_id]);
+ gam_regs.start_end_cntl1_g = REG(MPCC_OGAM_RAMB_END_CNTL1_G[mpcc_id]);
+ gam_regs.start_end_cntl2_g = REG(MPCC_OGAM_RAMB_END_CNTL2_G[mpcc_id]);
+ gam_regs.start_end_cntl1_r = REG(MPCC_OGAM_RAMB_END_CNTL1_R[mpcc_id]);
+ gam_regs.start_end_cntl2_r = REG(MPCC_OGAM_RAMB_END_CNTL2_R[mpcc_id]);
+ gam_regs.region_start = REG(MPCC_OGAM_RAMB_REGION_0_1[mpcc_id]);
+ gam_regs.region_end = REG(MPCC_OGAM_RAMB_REGION_32_33[mpcc_id]);
+
+ cm_helper_program_xfer_func(mpc20->base.ctx, params, &gam_regs);
+
+}
+
+static void mpc2_program_luta(struct mpc *mpc, int mpcc_id,
+ const struct pwl_params *params)
+{
+ struct dcn20_mpc *mpc20 = TO_DCN20_MPC(mpc);
+ struct xfer_func_reg gam_regs;
+
+ mpc2_ogam_get_reg_field(mpc, &gam_regs);
+
+ gam_regs.start_cntl_b = REG(MPCC_OGAM_RAMA_START_CNTL_B[mpcc_id]);
+ gam_regs.start_cntl_g = REG(MPCC_OGAM_RAMA_START_CNTL_G[mpcc_id]);
+ gam_regs.start_cntl_r = REG(MPCC_OGAM_RAMA_START_CNTL_R[mpcc_id]);
+ gam_regs.start_slope_cntl_b = REG(MPCC_OGAM_RAMA_SLOPE_CNTL_B[mpcc_id]);
+ gam_regs.start_slope_cntl_g = REG(MPCC_OGAM_RAMA_SLOPE_CNTL_G[mpcc_id]);
+ gam_regs.start_slope_cntl_r = REG(MPCC_OGAM_RAMA_SLOPE_CNTL_R[mpcc_id]);
+ gam_regs.start_end_cntl1_b = REG(MPCC_OGAM_RAMA_END_CNTL1_B[mpcc_id]);
+ gam_regs.start_end_cntl2_b = REG(MPCC_OGAM_RAMA_END_CNTL2_B[mpcc_id]);
+ gam_regs.start_end_cntl1_g = REG(MPCC_OGAM_RAMA_END_CNTL1_G[mpcc_id]);
+ gam_regs.start_end_cntl2_g = REG(MPCC_OGAM_RAMA_END_CNTL2_G[mpcc_id]);
+ gam_regs.start_end_cntl1_r = REG(MPCC_OGAM_RAMA_END_CNTL1_R[mpcc_id]);
+ gam_regs.start_end_cntl2_r = REG(MPCC_OGAM_RAMA_END_CNTL2_R[mpcc_id]);
+ gam_regs.region_start = REG(MPCC_OGAM_RAMA_REGION_0_1[mpcc_id]);
+ gam_regs.region_end = REG(MPCC_OGAM_RAMA_REGION_32_33[mpcc_id]);
+
+ cm_helper_program_xfer_func(mpc20->base.ctx, params, &gam_regs);
+
+}
+
+static void mpc20_program_ogam_pwl(
+ struct mpc *mpc, int mpcc_id,
+ const struct pwl_result_data *rgb,
+ uint32_t num)
+{
+ uint32_t i;
+ struct dcn20_mpc *mpc20 = TO_DCN20_MPC(mpc);
+
+ for (i = 0 ; i < num; i++) {
+ REG_SET(MPCC_OGAM_LUT_DATA[mpcc_id], 0, MPCC_OGAM_LUT_DATA, rgb[i].red_reg);
+ REG_SET(MPCC_OGAM_LUT_DATA[mpcc_id], 0, MPCC_OGAM_LUT_DATA, rgb[i].green_reg);
+ REG_SET(MPCC_OGAM_LUT_DATA[mpcc_id], 0, MPCC_OGAM_LUT_DATA, rgb[i].blue_reg);
+
+ REG_SET(MPCC_OGAM_LUT_DATA[mpcc_id], 0,
+ MPCC_OGAM_LUT_DATA, rgb[i].delta_red_reg);
+ REG_SET(MPCC_OGAM_LUT_DATA[mpcc_id], 0,
+ MPCC_OGAM_LUT_DATA, rgb[i].delta_green_reg);
+ REG_SET(MPCC_OGAM_LUT_DATA[mpcc_id], 0,
+ MPCC_OGAM_LUT_DATA, rgb[i].delta_blue_reg);
+
+ }
+
+}
+
+void apply_DEDCN20_305_wa(
+ struct mpc *mpc,
+ int mpcc_id, enum dc_lut_mode current_mode,
+ enum dc_lut_mode next_mode)
+{
+ struct dcn20_mpc *mpc20 = TO_DCN20_MPC(mpc);
+
+ if (mpc->ctx->dc->work_arounds.dedcn20_305_wa == false) {
+ /*hw fixed in new review*/
+ return;
+ }
+ if (current_mode == LUT_BYPASS)
+ /*this will only work if OTG is locked.
+ *if we were to support OTG unlock case,
+ *the workaround will be more complex
+ */
+ REG_SET(MPCC_OGAM_MODE[mpcc_id], 0, MPCC_OGAM_MODE,
+ next_mode == LUT_RAM_A ? 1:2);
+}
+
+void mpc2_set_output_gamma(
+ struct mpc *mpc,
+ int mpcc_id,
+ const struct pwl_params *params)
+{
+ enum dc_lut_mode current_mode;
+ enum dc_lut_mode next_mode;
+ struct dcn20_mpc *mpc20 = TO_DCN20_MPC(mpc);
+
+ if (params == NULL) {
+ REG_SET(MPCC_OGAM_MODE[mpcc_id], 0, MPCC_OGAM_MODE, 0);
+ return;
+ }
+ current_mode = mpc20_get_ogam_current(mpc, mpcc_id);
+ if (current_mode == LUT_BYPASS || current_mode == LUT_RAM_A)
+ next_mode = LUT_RAM_B;
+ else
+ next_mode = LUT_RAM_A;
+
+ mpc20_power_on_ogam_lut(mpc, mpcc_id, true);
+ mpc20_configure_ogam_lut(mpc, mpcc_id, next_mode == LUT_RAM_A ? true:false);
+
+ if (next_mode == LUT_RAM_A)
+ mpc2_program_luta(mpc, mpcc_id, params);
+ else
+ mpc2_program_lutb(mpc, mpcc_id, params);
+
+ apply_DEDCN20_305_wa(mpc, mpcc_id, current_mode, next_mode);
+
+ mpc20_program_ogam_pwl(
+ mpc, mpcc_id, params->rgb_resulted, params->hw_points_num);
+
+ REG_SET(MPCC_OGAM_MODE[mpcc_id], 0, MPCC_OGAM_MODE,
+ next_mode == LUT_RAM_A ? 1:2);
+}
+void mpc2_assert_idle_mpcc(struct mpc *mpc, int id)
+{
+ struct dcn20_mpc *mpc20 = TO_DCN20_MPC(mpc);
+ unsigned int mpc_disabled;
+
+ ASSERT(!(mpc20->mpcc_in_use_mask & 1 << id));
+ REG_GET(MPCC_STATUS[id], MPCC_DISABLED, &mpc_disabled);
+ if (mpc_disabled)
+ return;
+
+ REG_WAIT(MPCC_STATUS[id],
+ MPCC_IDLE, 1,
+ 1, 100000);
+}
+
+void mpc2_assert_mpcc_idle_before_connect(struct mpc *mpc, int mpcc_id)
+{
+ struct dcn20_mpc *mpc20 = TO_DCN20_MPC(mpc);
+ unsigned int top_sel, mpc_busy, mpc_idle, mpc_disabled;
+ REG_GET(MPCC_STATUS[mpcc_id], MPCC_DISABLED, &mpc_disabled);
+
+ if (mpc_disabled) {
+ ASSERT(0);
+ return;
+ }
+
+ REG_GET(MPCC_TOP_SEL[mpcc_id],
+ MPCC_TOP_SEL, &top_sel);
+
+ if (top_sel == 0xf) {
+ REG_GET_2(MPCC_STATUS[mpcc_id],
+ MPCC_BUSY, &mpc_busy,
+ MPCC_IDLE, &mpc_idle);
+
+ ASSERT(mpc_busy == 0);
+ ASSERT(mpc_idle == 1);
+ }
+}
+
+static void mpc2_init_mpcc(struct mpcc *mpcc, int mpcc_inst)
+{
+ mpcc->mpcc_id = mpcc_inst;
+ mpcc->dpp_id = 0xf;
+ mpcc->mpcc_bot = NULL;
+ mpcc->blnd_cfg.overlap_only = false;
+ mpcc->blnd_cfg.global_alpha = 0xff;
+ mpcc->blnd_cfg.global_gain = 0xff;
+ mpcc->blnd_cfg.background_color_bpc = 4;
+ mpcc->blnd_cfg.bottom_gain_mode = 0;
+ mpcc->blnd_cfg.top_gain = 0x1f000;
+ mpcc->blnd_cfg.bottom_inside_gain = 0x1f000;
+ mpcc->blnd_cfg.bottom_outside_gain = 0x1f000;
+ mpcc->sm_cfg.enable = false;
+}
+
+struct mpcc *mpc2_get_mpcc_for_dpp(struct mpc_tree *tree, int dpp_id)
+{
+ struct mpcc *tmp_mpcc = tree->opp_list;
+
+ while (tmp_mpcc != NULL) {
+ if (tmp_mpcc->dpp_id == 0xf || tmp_mpcc->dpp_id == dpp_id)
+ return tmp_mpcc;
+ tmp_mpcc = tmp_mpcc->mpcc_bot;
+ }
+ return NULL;
+}
+
+const struct mpc_funcs dcn20_mpc_funcs = {
+ .read_mpcc_state = mpc1_read_mpcc_state,
+ .insert_plane = mpc1_insert_plane,
+ .remove_mpcc = mpc1_remove_mpcc,
+ .mpc_init = mpc1_mpc_init,
+ .update_blending = mpc2_update_blending,
+ .get_mpcc_for_dpp = mpc2_get_mpcc_for_dpp,
+ .wait_for_idle = mpc2_assert_idle_mpcc,
+ .assert_mpcc_idle_before_connect = mpc2_assert_mpcc_idle_before_connect,
+ .init_mpcc_list_from_hw = mpc1_init_mpcc_list_from_hw,
+ .set_denorm = mpc2_set_denorm,
+ .set_denorm_clamp = mpc2_set_denorm_clamp,
+ .set_output_csc = mpc2_set_output_csc,
+ .set_ocsc_default = mpc2_set_ocsc_default,
+ .set_output_gamma = mpc2_set_output_gamma,
+};
+
+void dcn20_mpc_construct(struct dcn20_mpc *mpc20,
+ struct dc_context *ctx,
+ const struct dcn20_mpc_registers *mpc_regs,
+ const struct dcn20_mpc_shift *mpc_shift,
+ const struct dcn20_mpc_mask *mpc_mask,
+ int num_mpcc)
+{
+ int i;
+
+ mpc20->base.ctx = ctx;
+
+ mpc20->base.funcs = &dcn20_mpc_funcs;
+
+ mpc20->mpc_regs = mpc_regs;
+ mpc20->mpc_shift = mpc_shift;
+ mpc20->mpc_mask = mpc_mask;
+
+ mpc20->mpcc_in_use_mask = 0;
+ mpc20->num_mpcc = num_mpcc;
+
+ for (i = 0; i < MAX_MPCC; i++)
+ mpc2_init_mpcc(&mpc20->base.mpcc_array[i], i);
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mpc.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mpc.h
new file mode 100644
index 000000000000..9750095d2d73
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mpc.h
@@ -0,0 +1,285 @@
+/* Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_MPCC_DCN20_H__
+#define __DC_MPCC_DCN20_H__
+
+#include "dcn10/dcn10_mpc.h"
+
+#define TO_DCN20_MPC(mpc_base) \
+ container_of(mpc_base, struct dcn20_mpc, base)
+
+#define MPC_REG_LIST_DCN2_0(inst)\
+ MPC_COMMON_REG_LIST_DCN1_0(inst),\
+ SRII(MPCC_TOP_GAIN, MPCC, inst),\
+ SRII(MPCC_BOT_GAIN_INSIDE, MPCC, inst),\
+ SRII(MPCC_BOT_GAIN_OUTSIDE, MPCC, inst),\
+ SRII(MPCC_OGAM_RAMA_START_CNTL_B, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMA_START_CNTL_G, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMA_START_CNTL_R, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMA_SLOPE_CNTL_B, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMA_SLOPE_CNTL_G, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMA_SLOPE_CNTL_R, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMA_END_CNTL1_B, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMA_END_CNTL2_B, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMA_END_CNTL1_G, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMA_END_CNTL2_G, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMA_END_CNTL1_R, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMA_END_CNTL2_R, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMA_REGION_0_1, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMA_REGION_32_33, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMB_START_CNTL_B, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMB_START_CNTL_G, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMB_START_CNTL_R, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMB_SLOPE_CNTL_B, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMB_SLOPE_CNTL_G, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMB_SLOPE_CNTL_R, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMB_END_CNTL1_B, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMB_END_CNTL2_B, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMB_END_CNTL1_G, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMB_END_CNTL2_G, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMB_END_CNTL1_R, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMB_END_CNTL2_R, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMB_REGION_0_1, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_RAMB_REGION_32_33, MPCC_OGAM, inst),\
+ SRII(MPCC_MEM_PWR_CTRL, MPCC, inst),\
+ SRII(MPCC_OGAM_LUT_INDEX, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_LUT_RAM_CONTROL, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_LUT_DATA, MPCC_OGAM, inst),\
+ SRII(MPCC_OGAM_MODE, MPCC_OGAM, inst)
+
+#define MPC_OUT_MUX_REG_LIST_DCN2_0(inst) \
+ MPC_OUT_MUX_COMMON_REG_LIST_DCN1_0(inst),\
+ SRII(CSC_MODE, MPC_OUT, inst),\
+ SRII(CSC_C11_C12_A, MPC_OUT, inst),\
+ SRII(CSC_C33_C34_A, MPC_OUT, inst),\
+ SRII(CSC_C11_C12_B, MPC_OUT, inst),\
+ SRII(CSC_C33_C34_B, MPC_OUT, inst),\
+ SRII(DENORM_CONTROL, MPC_OUT, inst),\
+ SRII(DENORM_CLAMP_G_Y, MPC_OUT, inst),\
+ SRII(DENORM_CLAMP_B_CB, MPC_OUT, inst)
+
+#define MPC_REG_VARIABLE_LIST_DCN2_0 \
+ MPC_COMMON_REG_VARIABLE_LIST \
+ uint32_t MPCC_TOP_GAIN[MAX_MPCC]; \
+ uint32_t MPCC_BOT_GAIN_INSIDE[MAX_MPCC]; \
+ uint32_t MPCC_BOT_GAIN_OUTSIDE[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMA_START_CNTL_B[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMA_START_CNTL_G[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMA_START_CNTL_R[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMA_SLOPE_CNTL_B[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMA_SLOPE_CNTL_G[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMA_SLOPE_CNTL_R[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMA_END_CNTL1_B[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMA_END_CNTL2_B[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMA_END_CNTL1_G[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMA_END_CNTL2_G[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMA_END_CNTL1_R[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMA_END_CNTL2_R[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMA_REGION_0_1[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMA_REGION_32_33[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMB_START_CNTL_B[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMB_START_CNTL_G[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMB_START_CNTL_R[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMB_SLOPE_CNTL_B[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMB_SLOPE_CNTL_G[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMB_SLOPE_CNTL_R[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMB_END_CNTL1_B[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMB_END_CNTL2_B[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMB_END_CNTL1_G[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMB_END_CNTL2_G[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMB_END_CNTL1_R[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMB_END_CNTL2_R[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMB_REGION_0_1[MAX_MPCC]; \
+ uint32_t MPCC_OGAM_RAMB_REGION_32_33[MAX_MPCC];\
+ uint32_t MPCC_MEM_PWR_CTRL[MAX_MPCC];\
+ uint32_t MPCC_OGAM_LUT_INDEX[MAX_MPCC];\
+ uint32_t MPCC_OGAM_LUT_RAM_CONTROL[MAX_MPCC];\
+ uint32_t MPCC_OGAM_LUT_DATA[MAX_MPCC];\
+ uint32_t MPCC_OGAM_MODE[MAX_MPCC];\
+ uint32_t CSC_MODE[MAX_OPP]; \
+ uint32_t CSC_C11_C12_A[MAX_OPP]; \
+ uint32_t CSC_C33_C34_A[MAX_OPP]; \
+ uint32_t CSC_C11_C12_B[MAX_OPP]; \
+ uint32_t CSC_C33_C34_B[MAX_OPP]; \
+ uint32_t DENORM_CONTROL[MAX_OPP]; \
+ uint32_t DENORM_CLAMP_G_Y[MAX_OPP]; \
+ uint32_t DENORM_CLAMP_B_CB[MAX_OPP];
+
+#define MPC_COMMON_MASK_SH_LIST_DCN2_0(mask_sh) \
+ MPC_COMMON_MASK_SH_LIST_DCN1_0(mask_sh),\
+ SF(MPCC0_MPCC_CONTROL, MPCC_BG_BPC, mask_sh),\
+ SF(MPCC0_MPCC_CONTROL, MPCC_BOT_GAIN_MODE, mask_sh),\
+ SF(MPCC0_MPCC_TOP_GAIN, MPCC_TOP_GAIN, mask_sh),\
+ SF(MPCC0_MPCC_BOT_GAIN_INSIDE, MPCC_BOT_GAIN_INSIDE, mask_sh),\
+ SF(MPCC0_MPCC_BOT_GAIN_OUTSIDE, MPCC_BOT_GAIN_OUTSIDE, mask_sh),\
+ SF(MPC_OUT0_CSC_MODE, MPC_OCSC_MODE, mask_sh),\
+ SF(MPC_OUT0_CSC_C11_C12_A, MPC_OCSC_C11_A, mask_sh),\
+ SF(MPC_OUT0_CSC_C11_C12_A, MPC_OCSC_C12_A, mask_sh),\
+ SF(MPCC0_MPCC_STATUS, MPCC_DISABLED, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1, MPCC_OGAM_RAMA_EXP_REGION0_LUT_OFFSET, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1, MPCC_OGAM_RAMA_EXP_REGION0_NUM_SEGMENTS, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1, MPCC_OGAM_RAMA_EXP_REGION1_LUT_OFFSET, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1, MPCC_OGAM_RAMA_EXP_REGION1_NUM_SEGMENTS, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_B, MPCC_OGAM_RAMA_EXP_REGION_END_B, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_B, MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_B, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_B, MPCC_OGAM_RAMA_EXP_REGION_END_BASE_B, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMA_SLOPE_CNTL_B, MPCC_OGAM_RAMA_EXP_REGION_LINEAR_SLOPE_B, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_B, MPCC_OGAM_RAMA_EXP_REGION_START_B, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_B, MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_B, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_0_1, MPCC_OGAM_RAMB_EXP_REGION0_LUT_OFFSET, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_0_1, MPCC_OGAM_RAMB_EXP_REGION0_NUM_SEGMENTS, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_0_1, MPCC_OGAM_RAMB_EXP_REGION1_LUT_OFFSET, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMB_REGION_0_1, MPCC_OGAM_RAMB_EXP_REGION1_NUM_SEGMENTS, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_B, MPCC_OGAM_RAMB_EXP_REGION_END_B, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_B, MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_B, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_B, MPCC_OGAM_RAMB_EXP_REGION_END_BASE_B, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMB_SLOPE_CNTL_B, MPCC_OGAM_RAMB_EXP_REGION_LINEAR_SLOPE_B, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_B, MPCC_OGAM_RAMB_EXP_REGION_START_B, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_B, MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_B, mask_sh),\
+ SF(MPCC0_MPCC_MEM_PWR_CTRL, MPCC_OGAM_MEM_PWR_FORCE, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_LUT_INDEX, MPCC_OGAM_LUT_INDEX, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_LUT_RAM_CONTROL, MPCC_OGAM_LUT_WRITE_EN_MASK, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_LUT_RAM_CONTROL, MPCC_OGAM_LUT_RAM_SEL, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_LUT_RAM_CONTROL, MPCC_OGAM_CONFIG_STATUS, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_LUT_DATA, MPCC_OGAM_LUT_DATA, mask_sh),\
+ SF(MPCC_OGAM0_MPCC_OGAM_MODE, MPCC_OGAM_MODE, mask_sh),\
+ SF(MPC_OUT0_DENORM_CONTROL, MPC_OUT_DENORM_MODE, mask_sh),\
+ SF(MPC_OUT0_DENORM_CONTROL, MPC_OUT_DENORM_CLAMP_MAX_R_CR, mask_sh),\
+ SF(MPC_OUT0_DENORM_CONTROL, MPC_OUT_DENORM_CLAMP_MIN_R_CR, mask_sh),\
+ SF(MPC_OUT0_DENORM_CLAMP_G_Y, MPC_OUT_DENORM_CLAMP_MAX_G_Y, mask_sh),\
+ SF(MPC_OUT0_DENORM_CLAMP_G_Y, MPC_OUT_DENORM_CLAMP_MIN_G_Y, mask_sh),\
+ SF(MPC_OUT0_DENORM_CLAMP_B_CB, MPC_OUT_DENORM_CLAMP_MAX_B_CB, mask_sh),\
+ SF(MPC_OUT0_DENORM_CLAMP_B_CB, MPC_OUT_DENORM_CLAMP_MIN_B_CB, mask_sh)
+
+#define MPC_REG_FIELD_LIST_DCN2_0(type) \
+ MPC_REG_FIELD_LIST(type)\
+ type MPCC_BG_BPC;\
+ type MPCC_BOT_GAIN_MODE;\
+ type MPCC_TOP_GAIN;\
+ type MPCC_BOT_GAIN_INSIDE;\
+ type MPCC_BOT_GAIN_OUTSIDE;\
+ type MPC_OCSC_MODE;\
+ type MPC_OCSC_C11_A;\
+ type MPC_OCSC_C12_A;\
+ type MPCC_OGAM_RAMA_EXP_REGION0_LUT_OFFSET;\
+ type MPCC_OGAM_RAMA_EXP_REGION0_NUM_SEGMENTS;\
+ type MPCC_OGAM_RAMA_EXP_REGION1_LUT_OFFSET;\
+ type MPCC_OGAM_RAMA_EXP_REGION1_NUM_SEGMENTS;\
+ type MPCC_OGAM_RAMA_EXP_REGION_END_B;\
+ type MPCC_OGAM_RAMA_EXP_REGION_END_SLOPE_B;\
+ type MPCC_OGAM_RAMA_EXP_REGION_END_BASE_B;\
+ type MPCC_OGAM_RAMA_EXP_REGION_LINEAR_SLOPE_B;\
+ type MPCC_OGAM_RAMA_EXP_REGION_START_B;\
+ type MPCC_OGAM_RAMA_EXP_REGION_START_SEGMENT_B;\
+ type MPCC_OGAM_RAMB_EXP_REGION0_LUT_OFFSET;\
+ type MPCC_OGAM_RAMB_EXP_REGION0_NUM_SEGMENTS;\
+ type MPCC_OGAM_RAMB_EXP_REGION1_LUT_OFFSET;\
+ type MPCC_OGAM_RAMB_EXP_REGION1_NUM_SEGMENTS;\
+ type MPCC_OGAM_RAMB_EXP_REGION_END_B;\
+ type MPCC_OGAM_RAMB_EXP_REGION_END_SLOPE_B;\
+ type MPCC_OGAM_RAMB_EXP_REGION_END_BASE_B;\
+ type MPCC_OGAM_RAMB_EXP_REGION_LINEAR_SLOPE_B;\
+ type MPCC_OGAM_RAMB_EXP_REGION_START_B;\
+ type MPCC_OGAM_RAMB_EXP_REGION_START_SEGMENT_B;\
+ type MPCC_OGAM_MEM_PWR_FORCE;\
+ type MPCC_OGAM_LUT_INDEX;\
+ type MPCC_OGAM_LUT_WRITE_EN_MASK;\
+ type MPCC_OGAM_LUT_RAM_SEL;\
+ type MPCC_OGAM_CONFIG_STATUS;\
+ type MPCC_OGAM_LUT_DATA;\
+ type MPCC_OGAM_MODE;\
+ type MPC_OUT_DENORM_MODE;\
+ type MPC_OUT_DENORM_CLAMP_MAX_R_CR;\
+ type MPC_OUT_DENORM_CLAMP_MIN_R_CR;\
+ type MPC_OUT_DENORM_CLAMP_MAX_G_Y;\
+ type MPC_OUT_DENORM_CLAMP_MIN_G_Y;\
+ type MPC_OUT_DENORM_CLAMP_MAX_B_CB;\
+ type MPC_OUT_DENORM_CLAMP_MIN_B_CB;\
+ type MPCC_DISABLED;
+
+struct dcn20_mpc_registers {
+ MPC_REG_VARIABLE_LIST_DCN2_0
+};
+
+struct dcn20_mpc_shift {
+ MPC_REG_FIELD_LIST_DCN2_0(uint8_t)
+};
+
+struct dcn20_mpc_mask {
+ MPC_REG_FIELD_LIST_DCN2_0(uint32_t)
+};
+
+struct dcn20_mpc {
+ struct mpc base;
+
+ int mpcc_in_use_mask;
+ int num_mpcc;
+ const struct dcn20_mpc_registers *mpc_regs;
+ const struct dcn20_mpc_shift *mpc_shift;
+ const struct dcn20_mpc_mask *mpc_mask;
+};
+
+void dcn20_mpc_construct(struct dcn20_mpc *mpcc20,
+ struct dc_context *ctx,
+ const struct dcn20_mpc_registers *mpc_regs,
+ const struct dcn20_mpc_shift *mpc_shift,
+ const struct dcn20_mpc_mask *mpc_mask,
+ int num_mpcc);
+
+void mpc2_update_blending(
+ struct mpc *mpc,
+ struct mpcc_blnd_cfg *blnd_cfg,
+ int mpcc_id);
+
+void mpc2_set_denorm(
+ struct mpc *mpc,
+ int opp_id,
+ enum dc_color_depth output_depth);
+
+void mpc2_set_denorm_clamp(
+ struct mpc *mpc,
+ int opp_id,
+ struct mpc_denorm_clamp denorm_clamp);
+
+void mpc2_set_output_csc(
+ struct mpc *mpc,
+ int opp_id,
+ const uint16_t *regval,
+ enum mpc_output_csc_mode ocsc_mode);
+
+void mpc2_set_ocsc_default(
+ struct mpc *mpc,
+ int opp_id,
+ enum dc_color_space color_space,
+ enum mpc_output_csc_mode ocsc_mode);
+
+void mpc2_set_output_gamma(
+ struct mpc *mpc,
+ int mpcc_id,
+ const struct pwl_params *params);
+
+void mpc2_assert_idle_mpcc(struct mpc *mpc, int id);
+void mpc2_assert_mpcc_idle_before_connect(struct mpc *mpc, int mpcc_id);
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_opp.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_opp.c
new file mode 100644
index 000000000000..d9e7c711a71c
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_opp.c
@@ -0,0 +1,355 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services.h"
+#include "dcn20_opp.h"
+#include "reg_helper.h"
+
+#define REG(reg) \
+ (oppn20->regs->reg)
+
+#undef FN
+#define FN(reg_name, field_name) \
+ oppn20->opp_shift->field_name, oppn20->opp_mask->field_name
+
+#define CTX \
+ oppn20->base.ctx
+
+
+void opp2_set_disp_pattern_generator(
+ struct output_pixel_processor *opp,
+ enum controller_dp_test_pattern test_pattern,
+ enum dc_color_depth color_depth,
+ const struct tg_color *solid_color,
+ int width,
+ int height)
+{
+ struct dcn20_opp *oppn20 = TO_DCN20_OPP(opp);
+ enum test_pattern_color_format bit_depth;
+ enum test_pattern_dyn_range dyn_range;
+ enum test_pattern_mode mode;
+
+ /* color ramp generator mixes 16-bits color */
+ uint32_t src_bpc = 16;
+ /* requested bpc */
+ uint32_t dst_bpc;
+ uint32_t index;
+ /* RGB values of the color bars.
+ * Produce two RGB colors: RGB0 - white (all Fs)
+ * and RGB1 - black (all 0s)
+ * (three RGB components for two colors)
+ */
+ uint16_t src_color[6] = {0xFFFF, 0xFFFF, 0xFFFF, 0x0000,
+ 0x0000, 0x0000};
+ /* dest color (converted to the specified color format) */
+ uint16_t dst_color[6];
+ uint32_t inc_base;
+
+ /* translate to bit depth */
+ switch (color_depth) {
+ case COLOR_DEPTH_666:
+ bit_depth = TEST_PATTERN_COLOR_FORMAT_BPC_6;
+ break;
+ case COLOR_DEPTH_888:
+ bit_depth = TEST_PATTERN_COLOR_FORMAT_BPC_8;
+ break;
+ case COLOR_DEPTH_101010:
+ bit_depth = TEST_PATTERN_COLOR_FORMAT_BPC_10;
+ break;
+ case COLOR_DEPTH_121212:
+ bit_depth = TEST_PATTERN_COLOR_FORMAT_BPC_12;
+ break;
+ default:
+ bit_depth = TEST_PATTERN_COLOR_FORMAT_BPC_8;
+ break;
+ }
+
+ /* set DPG dimentions */
+ REG_SET_2(DPG_DIMENSIONS, 0,
+ DPG_ACTIVE_WIDTH, width,
+ DPG_ACTIVE_HEIGHT, height);
+
+ switch (test_pattern) {
+ case CONTROLLER_DP_TEST_PATTERN_COLORSQUARES:
+ case CONTROLLER_DP_TEST_PATTERN_COLORSQUARES_CEA:
+ {
+ dyn_range = (test_pattern ==
+ CONTROLLER_DP_TEST_PATTERN_COLORSQUARES_CEA ?
+ TEST_PATTERN_DYN_RANGE_CEA :
+ TEST_PATTERN_DYN_RANGE_VESA);
+
+ REG_UPDATE_6(DPG_CONTROL,
+ DPG_EN, 1,
+ DPG_MODE, TEST_PATTERN_MODE_COLORSQUARES_RGB,
+ DPG_DYNAMIC_RANGE, dyn_range,
+ DPG_BIT_DEPTH, bit_depth,
+ DPG_VRES, 6,
+ DPG_HRES, 6);
+ }
+ break;
+
+ case CONTROLLER_DP_TEST_PATTERN_VERTICALBARS:
+ case CONTROLLER_DP_TEST_PATTERN_HORIZONTALBARS:
+ {
+ mode = (test_pattern ==
+ CONTROLLER_DP_TEST_PATTERN_VERTICALBARS ?
+ TEST_PATTERN_MODE_VERTICALBARS :
+ TEST_PATTERN_MODE_HORIZONTALBARS);
+
+ switch (bit_depth) {
+ case TEST_PATTERN_COLOR_FORMAT_BPC_6:
+ dst_bpc = 6;
+ break;
+ case TEST_PATTERN_COLOR_FORMAT_BPC_8:
+ dst_bpc = 8;
+ break;
+ case TEST_PATTERN_COLOR_FORMAT_BPC_10:
+ dst_bpc = 10;
+ break;
+ default:
+ dst_bpc = 8;
+ break;
+ }
+
+ /* adjust color to the required colorFormat */
+ for (index = 0; index < 6; index++) {
+ /* dst = 2^dstBpc * src / 2^srcBpc = src >>
+ * (srcBpc - dstBpc);
+ */
+ dst_color[index] =
+ src_color[index] >> (src_bpc - dst_bpc);
+ /* DPG_COLOUR registers are 16-bit MSB aligned value with bits 3:0 hardwired to ZERO.
+ * XXXXXXXXXX000000 for 10 bit,
+ * XXXXXXXX00000000 for 8 bit,
+ * XXXXXX0000000000 for 6 bits
+ */
+ dst_color[index] <<= (16 - dst_bpc);
+ }
+
+ REG_SET_2(DPG_COLOUR_R_CR, 0,
+ DPG_COLOUR1_R_CR, dst_color[0],
+ DPG_COLOUR0_R_CR, dst_color[3]);
+ REG_SET_2(DPG_COLOUR_G_Y, 0,
+ DPG_COLOUR1_G_Y, dst_color[1],
+ DPG_COLOUR0_G_Y, dst_color[4]);
+ REG_SET_2(DPG_COLOUR_B_CB, 0,
+ DPG_COLOUR1_B_CB, dst_color[2],
+ DPG_COLOUR0_B_CB, dst_color[5]);
+
+ /* enable test pattern */
+ REG_UPDATE_6(DPG_CONTROL,
+ DPG_EN, 1,
+ DPG_MODE, mode,
+ DPG_DYNAMIC_RANGE, 0,
+ DPG_BIT_DEPTH, bit_depth,
+ DPG_VRES, 0,
+ DPG_HRES, 0);
+ }
+ break;
+
+ case CONTROLLER_DP_TEST_PATTERN_COLORRAMP:
+ {
+ mode = (bit_depth ==
+ TEST_PATTERN_COLOR_FORMAT_BPC_10 ?
+ TEST_PATTERN_MODE_DUALRAMP_RGB :
+ TEST_PATTERN_MODE_SINGLERAMP_RGB);
+
+ switch (bit_depth) {
+ case TEST_PATTERN_COLOR_FORMAT_BPC_6:
+ dst_bpc = 6;
+ break;
+ case TEST_PATTERN_COLOR_FORMAT_BPC_8:
+ dst_bpc = 8;
+ break;
+ case TEST_PATTERN_COLOR_FORMAT_BPC_10:
+ dst_bpc = 10;
+ break;
+ default:
+ dst_bpc = 8;
+ break;
+ }
+
+ /* increment for the first ramp for one color gradation
+ * 1 gradation for 6-bit color is 2^10
+ * gradations in 16-bit color
+ */
+ inc_base = (src_bpc - dst_bpc);
+
+ switch (bit_depth) {
+ case TEST_PATTERN_COLOR_FORMAT_BPC_6:
+ {
+ REG_SET_3(DPG_RAMP_CONTROL, 0,
+ DPG_RAMP0_OFFSET, 0,
+ DPG_INC0, inc_base,
+ DPG_INC1, 0);
+ REG_UPDATE_2(DPG_CONTROL,
+ DPG_VRES, 6,
+ DPG_HRES, 6);
+ }
+ break;
+ case TEST_PATTERN_COLOR_FORMAT_BPC_8:
+ {
+ REG_SET_3(DPG_RAMP_CONTROL, 0,
+ DPG_RAMP0_OFFSET, 0,
+ DPG_INC0, inc_base,
+ DPG_INC1, 0);
+ REG_UPDATE_2(DPG_CONTROL,
+ DPG_VRES, 6,
+ DPG_HRES, 8);
+ }
+ break;
+ case TEST_PATTERN_COLOR_FORMAT_BPC_10:
+ {
+ REG_SET_3(DPG_RAMP_CONTROL, 0,
+ DPG_RAMP0_OFFSET, 384 << 6,
+ DPG_INC0, inc_base,
+ DPG_INC1, inc_base + 2);
+ REG_UPDATE_2(DPG_CONTROL,
+ DPG_VRES, 5,
+ DPG_HRES, 8);
+ }
+ break;
+ default:
+ break;
+ }
+
+ /* enable test pattern */
+ REG_UPDATE_4(DPG_CONTROL,
+ DPG_EN, 1,
+ DPG_MODE, mode,
+ DPG_DYNAMIC_RANGE, 0,
+ DPG_BIT_DEPTH, bit_depth);
+ }
+ break;
+ case CONTROLLER_DP_TEST_PATTERN_VIDEOMODE:
+ {
+ REG_WRITE(DPG_CONTROL, 0);
+ REG_WRITE(DPG_COLOUR_R_CR, 0);
+ REG_WRITE(DPG_COLOUR_G_Y, 0);
+ REG_WRITE(DPG_COLOUR_B_CB, 0);
+ REG_WRITE(DPG_RAMP_CONTROL, 0);
+ }
+ break;
+ case CONTROLLER_DP_TEST_PATTERN_SOLID_COLOR:
+ {
+ opp2_dpg_set_blank_color(opp, solid_color);
+ REG_UPDATE_2(DPG_CONTROL,
+ DPG_EN, 1,
+ DPG_MODE, TEST_PATTERN_MODE_HORIZONTALBARS);
+
+ REG_SET_2(DPG_DIMENSIONS, 0,
+ DPG_ACTIVE_WIDTH, width,
+ DPG_ACTIVE_HEIGHT, height);
+ }
+ break;
+ default:
+ break;
+
+ }
+}
+
+void opp2_dpg_set_blank_color(
+ struct output_pixel_processor *opp,
+ const struct tg_color *color)
+{
+ struct dcn20_opp *oppn20 = TO_DCN20_OPP(opp);
+
+ /* 16-bit MSB aligned value. Bits 3:0 of this field are hardwired to ZERO */
+ ASSERT(color);
+ REG_SET_2(DPG_COLOUR_B_CB, 0,
+ DPG_COLOUR1_B_CB, color->color_b_cb << 6,
+ DPG_COLOUR0_B_CB, color->color_b_cb << 6);
+ REG_SET_2(DPG_COLOUR_G_Y, 0,
+ DPG_COLOUR1_G_Y, color->color_g_y << 6,
+ DPG_COLOUR0_G_Y, color->color_g_y << 6);
+ REG_SET_2(DPG_COLOUR_R_CR, 0,
+ DPG_COLOUR1_R_CR, color->color_r_cr << 6,
+ DPG_COLOUR0_R_CR, color->color_r_cr << 6);
+}
+
+bool opp2_dpg_is_blanked(struct output_pixel_processor *opp)
+{
+ struct dcn20_opp *oppn20 = TO_DCN20_OPP(opp);
+ uint32_t dpg_en, dpg_mode;
+ uint32_t double_buffer_pending;
+
+ REG_GET_2(DPG_CONTROL,
+ DPG_EN, &dpg_en,
+ DPG_MODE, &dpg_mode);
+
+ REG_GET(DPG_STATUS,
+ DPG_DOUBLE_BUFFER_PENDING, &double_buffer_pending);
+
+ return (dpg_en == 1) &&
+ (double_buffer_pending == 0);
+}
+
+void opp2_program_left_edge_extra_pixel (
+ struct output_pixel_processor *opp,
+ bool count)
+{
+ struct dcn20_opp *oppn20 = TO_DCN20_OPP(opp);
+
+ /* Specifies the number of extra left edge pixels that are supplied to
+ * the 422 horizontal chroma sub-sample filter.
+ * Note that when left edge pixel is not "0", fmt pixel encoding can be in either 420 or 422 mode
+ * */
+ REG_UPDATE(FMT_422_CONTROL, FMT_LEFT_EDGE_EXTRA_PIXEL_COUNT, count);
+}
+
+/*****************************************/
+/* Constructor, Destructor */
+/*****************************************/
+
+static struct opp_funcs dcn20_opp_funcs = {
+ .opp_set_dyn_expansion = opp1_set_dyn_expansion,
+ .opp_program_fmt = opp1_program_fmt,
+ .opp_program_bit_depth_reduction = opp1_program_bit_depth_reduction,
+ .opp_program_stereo = opp1_program_stereo,
+ .opp_pipe_clock_control = opp1_pipe_clock_control,
+ .opp_set_disp_pattern_generator = opp2_set_disp_pattern_generator,
+ .dpg_is_blanked = opp2_dpg_is_blanked,
+ .opp_dpg_set_blank_color = opp2_dpg_set_blank_color,
+ .opp_convert_pti = NULL,
+ .opp_destroy = opp1_destroy,
+ .opp_program_left_edge_extra_pixel = opp2_program_left_edge_extra_pixel,
+};
+
+void dcn20_opp_construct(struct dcn20_opp *oppn20,
+ struct dc_context *ctx,
+ uint32_t inst,
+ const struct dcn20_opp_registers *regs,
+ const struct dcn20_opp_shift *opp_shift,
+ const struct dcn20_opp_mask *opp_mask)
+{
+ oppn20->base.ctx = ctx;
+ oppn20->base.inst = inst;
+ oppn20->base.funcs = &dcn20_opp_funcs;
+
+ oppn20->regs = regs;
+ oppn20->opp_shift = opp_shift;
+ oppn20->opp_mask = opp_mask;
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_opp.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_opp.h
new file mode 100644
index 000000000000..abd8de9a78f8
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_opp.h
@@ -0,0 +1,158 @@
+/* Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_OPP_DCN20_H__
+#define __DC_OPP_DCN20_H__
+
+#include "dcn10/dcn10_opp.h"
+
+#define TO_DCN20_OPP(opp)\
+ container_of(opp, struct dcn20_opp, base)
+
+#define OPP_SF(reg_name, field_name, post_fix)\
+ .field_name = reg_name ## __ ## field_name ## post_fix
+
+#define OPP_DPG_REG_LIST(id) \
+ SRI(DPG_CONTROL, DPG, id), \
+ SRI(DPG_DIMENSIONS, DPG, id), \
+ SRI(DPG_COLOUR_B_CB, DPG, id), \
+ SRI(DPG_COLOUR_G_Y, DPG, id), \
+ SRI(DPG_COLOUR_R_CR, DPG, id), \
+ SRI(DPG_RAMP_CONTROL, DPG, id), \
+ SRI(DPG_STATUS, DPG, id)
+
+#define OPP_REG_LIST_DCN20(id) \
+ OPP_REG_LIST_DCN10(id), \
+ OPP_DPG_REG_LIST(id), \
+ SRI(FMT_422_CONTROL, FMT, id), \
+ SRI(OPPBUF_CONTROL1, OPPBUF, id)
+
+#define OPP_REG_VARIABLE_LIST_DCN2_0 \
+ OPP_COMMON_REG_VARIABLE_LIST; \
+ uint32_t FMT_422_CONTROL; \
+ uint32_t DPG_CONTROL; \
+ uint32_t DPG_DIMENSIONS; \
+ uint32_t DPG_COLOUR_B_CB; \
+ uint32_t DPG_COLOUR_G_Y; \
+ uint32_t DPG_COLOUR_R_CR; \
+ uint32_t DPG_RAMP_CONTROL; \
+ uint32_t DPG_STATUS
+
+#define OPP_DPG_MASK_SH_LIST(mask_sh) \
+ OPP_SF(DPG0_DPG_CONTROL, DPG_EN, mask_sh), \
+ OPP_SF(DPG0_DPG_CONTROL, DPG_MODE, mask_sh), \
+ OPP_SF(DPG0_DPG_CONTROL, DPG_DYNAMIC_RANGE, mask_sh), \
+ OPP_SF(DPG0_DPG_CONTROL, DPG_BIT_DEPTH, mask_sh), \
+ OPP_SF(DPG0_DPG_CONTROL, DPG_VRES, mask_sh), \
+ OPP_SF(DPG0_DPG_CONTROL, DPG_HRES, mask_sh), \
+ OPP_SF(DPG0_DPG_DIMENSIONS, DPG_ACTIVE_WIDTH, mask_sh), \
+ OPP_SF(DPG0_DPG_DIMENSIONS, DPG_ACTIVE_HEIGHT, mask_sh), \
+ OPP_SF(DPG0_DPG_COLOUR_R_CR, DPG_COLOUR0_R_CR, mask_sh), \
+ OPP_SF(DPG0_DPG_COLOUR_R_CR, DPG_COLOUR1_R_CR, mask_sh), \
+ OPP_SF(DPG0_DPG_COLOUR_B_CB, DPG_COLOUR0_B_CB, mask_sh), \
+ OPP_SF(DPG0_DPG_COLOUR_B_CB, DPG_COLOUR1_B_CB, mask_sh), \
+ OPP_SF(DPG0_DPG_COLOUR_G_Y, DPG_COLOUR0_G_Y, mask_sh), \
+ OPP_SF(DPG0_DPG_COLOUR_G_Y, DPG_COLOUR1_G_Y, mask_sh), \
+ OPP_SF(DPG0_DPG_RAMP_CONTROL, DPG_RAMP0_OFFSET, mask_sh), \
+ OPP_SF(DPG0_DPG_RAMP_CONTROL, DPG_INC0, mask_sh), \
+ OPP_SF(DPG0_DPG_RAMP_CONTROL, DPG_INC1, mask_sh), \
+ OPP_SF(DPG0_DPG_STATUS, DPG_DOUBLE_BUFFER_PENDING, mask_sh)
+
+#define OPP_MASK_SH_LIST_DCN20(mask_sh) \
+ OPP_MASK_SH_LIST_DCN(mask_sh), \
+ OPP_DPG_MASK_SH_LIST(mask_sh), \
+ OPP_SF(OPPBUF0_OPPBUF_CONTROL, OPPBUF_DISPLAY_SEGMENTATION, mask_sh),\
+ OPP_SF(OPPBUF0_OPPBUF_CONTROL, OPPBUF_OVERLAP_PIXEL_NUM, mask_sh), \
+ OPP_SF(FMT0_FMT_422_CONTROL, FMT_LEFT_EDGE_EXTRA_PIXEL_COUNT, mask_sh)
+
+#define OPP_DCN20_REG_FIELD_LIST(type) \
+ OPP_DCN10_REG_FIELD_LIST(type); \
+ type FMT_LEFT_EDGE_EXTRA_PIXEL_COUNT; \
+ type DPG_EN; \
+ type DPG_MODE; \
+ type DPG_DYNAMIC_RANGE; \
+ type DPG_BIT_DEPTH; \
+ type DPG_VRES; \
+ type DPG_HRES; \
+ type DPG_ACTIVE_WIDTH; \
+ type DPG_ACTIVE_HEIGHT; \
+ type DPG_COLOUR0_R_CR; \
+ type DPG_COLOUR1_R_CR; \
+ type DPG_COLOUR0_B_CB; \
+ type DPG_COLOUR1_B_CB; \
+ type DPG_COLOUR0_G_Y; \
+ type DPG_COLOUR1_G_Y; \
+ type DPG_RAMP0_OFFSET; \
+ type DPG_INC0; \
+ type DPG_INC1; \
+ type DPG_DOUBLE_BUFFER_PENDING
+
+struct dcn20_opp_registers {
+ OPP_REG_VARIABLE_LIST_DCN2_0;
+};
+
+struct dcn20_opp_shift {
+ OPP_DCN20_REG_FIELD_LIST(uint8_t);
+};
+
+struct dcn20_opp_mask {
+ OPP_DCN20_REG_FIELD_LIST(uint32_t);
+};
+
+struct dcn20_opp {
+ struct output_pixel_processor base;
+
+ const struct dcn20_opp_registers *regs;
+ const struct dcn20_opp_shift *opp_shift;
+ const struct dcn20_opp_mask *opp_mask;
+
+ bool is_write_to_ram_a_safe;
+};
+
+void dcn20_opp_construct(struct dcn20_opp *oppn20,
+ struct dc_context *ctx,
+ uint32_t inst,
+ const struct dcn20_opp_registers *regs,
+ const struct dcn20_opp_shift *opp_shift,
+ const struct dcn20_opp_mask *opp_mask);
+
+void opp2_set_disp_pattern_generator(
+ struct output_pixel_processor *opp,
+ enum controller_dp_test_pattern test_pattern,
+ enum dc_color_depth color_depth,
+ const struct tg_color *solid_color,
+ int width,
+ int height);
+
+bool opp2_dpg_is_blanked(struct output_pixel_processor *opp);
+
+void opp2_dpg_set_blank_color(
+ struct output_pixel_processor *opp,
+ const struct tg_color *color);
+
+void opp2_program_left_edge_extra_pixel (
+ struct output_pixel_processor *opp,
+ bool count);
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_optc.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_optc.c
new file mode 100644
index 000000000000..1ae973962d53
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_optc.c
@@ -0,0 +1,542 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "reg_helper.h"
+#include "dcn20_optc.h"
+#include "dc.h"
+
+#define REG(reg)\
+ optc1->tg_regs->reg
+
+#define CTX \
+ optc1->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name) \
+ optc1->tg_shift->field_name, optc1->tg_mask->field_name
+
+/**
+ * Enable CRTC
+ * Enable CRTC - call ASIC Control Object to enable Timing generator.
+ */
+bool optc2_enable_crtc(struct timing_generator *optc)
+{
+ /* TODO FPGA wait for answer
+ * OTG_MASTER_UPDATE_MODE != CRTC_MASTER_UPDATE_MODE
+ * OTG_MASTER_UPDATE_LOCK != CRTC_MASTER_UPDATE_LOCK
+ */
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ /* opp instance for OTG. For DCN1.0, ODM is remoed.
+ * OPP and OPTC should 1:1 mapping
+ */
+ REG_UPDATE(OPTC_DATA_SOURCE_SELECT,
+ OPTC_SEG0_SRC_SEL, optc->inst);
+
+ /* VTG enable first is for HW workaround */
+ REG_UPDATE(CONTROL,
+ VTG0_ENABLE, 1);
+
+ /* Enable CRTC */
+ REG_UPDATE_2(OTG_CONTROL,
+ OTG_DISABLE_POINT_CNTL, 3,
+ OTG_MASTER_EN, 1);
+
+ return true;
+}
+
+/**
+ * DRR double buffering control to select buffer point
+ * for V_TOTAL, H_TOTAL, VTOTAL_MIN, VTOTAL_MAX, VTOTAL_MIN_SEL and VTOTAL_MAX_SEL registers
+ * Options: anytime, start of frame, dp start of frame (range timing)
+ */
+void optc2_set_timing_db_mode(struct timing_generator *optc, bool enable)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ uint32_t blank_data_double_buffer_enable = enable ? 1 : 0;
+
+ REG_UPDATE(OTG_DOUBLE_BUFFER_CONTROL,
+ OTG_RANGE_TIMING_DBUF_UPDATE_MODE, blank_data_double_buffer_enable);
+}
+
+/**
+ *For the below, I'm not sure how your GSL parameters are stored in your env,
+ * so I will assume a gsl_params struct for now
+ */
+void optc2_set_gsl(struct timing_generator *optc,
+ const struct gsl_params *params)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+/**
+ * There are (MAX_OPTC+1)/2 gsl groups available for use.
+ * In each group (assign an OTG to a group by setting OTG_GSLX_EN = 1,
+ * set one of the OTGs to be the master (OTG_GSL_MASTER_EN = 1) and the rest are slaves.
+ */
+ REG_UPDATE_5(OTG_GSL_CONTROL,
+ OTG_GSL0_EN, params->gsl0_en,
+ OTG_GSL1_EN, params->gsl1_en,
+ OTG_GSL2_EN, params->gsl2_en,
+ OTG_GSL_MASTER_EN, params->gsl_master_en,
+ OTG_GSL_MASTER_MODE, params->gsl_master_mode);
+}
+
+
+/* Use the gsl allow flip as the master update lock */
+void optc2_use_gsl_as_master_update_lock(struct timing_generator *optc,
+ const struct gsl_params *params)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ REG_UPDATE(OTG_GSL_CONTROL,
+ OTG_MASTER_UPDATE_LOCK_GSL_EN, params->master_update_lock_gsl_en);
+}
+
+/* You can control the GSL timing by limiting GSL to a window (X,Y) */
+void optc2_set_gsl_window(struct timing_generator *optc,
+ const struct gsl_params *params)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ REG_SET_2(OTG_GSL_WINDOW_X, 0,
+ OTG_GSL_WINDOW_START_X, params->gsl_window_start_x,
+ OTG_GSL_WINDOW_END_X, params->gsl_window_end_x);
+ REG_SET_2(OTG_GSL_WINDOW_Y, 0,
+ OTG_GSL_WINDOW_START_Y, params->gsl_window_start_y,
+ OTG_GSL_WINDOW_END_Y, params->gsl_window_end_y);
+}
+
+/**
+ * Vupdate keepout can be set to a window to block the update lock for that pipe from changing.
+ * Start offset begins with vstartup and goes for x number of clocks,
+ * end offset starts from end of vupdate to x number of clocks.
+ */
+void optc2_set_vupdate_keepout(struct timing_generator *optc,
+ const struct vupdate_keepout_params *params)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ REG_SET_3(OTG_VUPDATE_KEEPOUT, 0,
+ MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_START_OFFSET, params->start_offset,
+ MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_END_OFFSET, params->end_offset,
+ OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN, params->enable);
+}
+
+void optc2_set_gsl_source_select(
+ struct timing_generator *optc,
+ int group_idx,
+ uint32_t gsl_ready_signal)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ switch (group_idx) {
+ case 1:
+ REG_UPDATE(GSL_SOURCE_SELECT, GSL0_READY_SOURCE_SEL, gsl_ready_signal);
+ break;
+ case 2:
+ REG_UPDATE(GSL_SOURCE_SELECT, GSL1_READY_SOURCE_SEL, gsl_ready_signal);
+ break;
+ case 3:
+ REG_UPDATE(GSL_SOURCE_SELECT, GSL2_READY_SOURCE_SEL, gsl_ready_signal);
+ break;
+ default:
+ break;
+ }
+}
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+/* DSC encoder frame start controls: x = h position, line_num = # of lines from vstartup */
+void optc2_set_dsc_encoder_frame_start(struct timing_generator *optc,
+ int x_position,
+ int line_num)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ REG_SET_2(OTG_DSC_START_POSITION, 0,
+ OTG_DSC_START_POSITION_X, x_position,
+ OTG_DSC_START_POSITION_LINE_NUM, line_num);
+}
+
+/* Set DSC-related configuration.
+ * dsc_mode: 0 disables DSC, other values enable DSC in specified format
+ * sc_bytes_per_pixel: Bytes per pixel in u3.28 format
+ * dsc_slice_width: Slice width in pixels
+ */
+void optc2_set_dsc_config(struct timing_generator *optc,
+ enum optc_dsc_mode dsc_mode,
+ uint32_t dsc_bytes_per_pixel,
+ uint32_t dsc_slice_width)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+ uint32_t data_format = 0;
+ /* skip if dsc mode is not changed */
+ data_format = dm_read_reg(CTX, REG(OPTC_DATA_FORMAT_CONTROL));
+
+ data_format = data_format & 0x30; /* bit5:4 */
+ data_format = data_format >> 4;
+
+ if (data_format == dsc_mode)
+ return;
+
+ REG_UPDATE(OPTC_DATA_FORMAT_CONTROL,
+ OPTC_DSC_MODE, dsc_mode);
+
+ REG_SET(OPTC_BYTES_PER_PIXEL, 0,
+ OPTC_DSC_BYTES_PER_PIXEL, dsc_bytes_per_pixel);
+
+ REG_UPDATE(OPTC_WIDTH_CONTROL,
+ OPTC_DSC_SLICE_WIDTH, dsc_slice_width);
+}
+#endif
+
+/**
+ * PTI i think is already done somewhere else for 2ka
+ * (opp?, please double check.
+ * OPTC side only has 1 register to set for PTI_ENABLE)
+ */
+
+void optc2_set_odm_bypass(struct timing_generator *optc,
+ const struct dc_crtc_timing *dc_crtc_timing)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+ uint32_t h_div_2 = 0;
+
+ optc1->comb_opp_id = 0xf;
+ REG_SET_3(OPTC_DATA_SOURCE_SELECT, 0,
+ OPTC_NUM_OF_INPUT_SEGMENT, 0,
+ OPTC_SEG0_SRC_SEL, optc->inst,
+ OPTC_SEG1_SRC_SEL, 0xf);
+ REG_WRITE(OTG_H_TIMING_CNTL, 0);
+
+ h_div_2 = optc1_is_two_pixels_per_containter(dc_crtc_timing);
+ REG_UPDATE(OTG_H_TIMING_CNTL,
+ OTG_H_TIMING_DIV_BY2, h_div_2);
+ REG_SET(OPTC_MEMORY_CONFIG, 0,
+ OPTC_MEM_SEL, 0);
+}
+
+void optc2_set_odm_combine(struct timing_generator *optc, int combine_opp_id,
+ int mpcc_hactive, enum dc_pixel_encoding pixel_encoding)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+ /* 2 pieces of memory required for up to 5120 displays, 4 for up to 8192 */
+ int memory_mask = mpcc_hactive <= 2560 ? 0x3 : 0xf;
+ uint32_t data_fmt = 0;
+
+ /* TODO: In pseudocode but does not affect maximus, delete comment if we dont need on asic
+ * REG_SET(OTG_GLOBAL_CONTROL2, 0, GLOBAL_UPDATE_LOCK_EN, 1);
+ * Program OTG register MASTER_UPDATE_LOCK_DB_X/Y to the position before DP frame start
+ * REG_SET_2(OTG_GLOBAL_CONTROL1, 0,
+ * MASTER_UPDATE_LOCK_DB_X, 160,
+ * MASTER_UPDATE_LOCK_DB_Y, 240);
+ */
+ if (REG(OPTC_MEMORY_CONFIG))
+ REG_SET(OPTC_MEMORY_CONFIG, 0,
+ OPTC_MEM_SEL, memory_mask << (optc->inst * 4));
+
+ if (pixel_encoding == PIXEL_ENCODING_YCBCR422)
+ data_fmt = 1;
+ else if (pixel_encoding == PIXEL_ENCODING_YCBCR420)
+ data_fmt = 2;
+
+ REG_UPDATE(OPTC_DATA_FORMAT_CONTROL, OPTC_DATA_FORMAT, data_fmt);
+
+ REG_SET_3(OPTC_DATA_SOURCE_SELECT, 0,
+ OPTC_NUM_OF_INPUT_SEGMENT, 1,
+ OPTC_SEG0_SRC_SEL, optc->inst,
+ OPTC_SEG1_SRC_SEL, combine_opp_id);
+
+ REG_UPDATE(OPTC_WIDTH_CONTROL,
+ OPTC_SEGMENT_WIDTH, mpcc_hactive);
+
+ REG_SET(OTG_H_TIMING_CNTL, 0, OTG_H_TIMING_DIV_BY2, 1);
+ optc1->comb_opp_id = combine_opp_id;
+}
+
+void optc2_get_optc_source(struct timing_generator *optc,
+ uint32_t *num_of_src_opp,
+ uint32_t *src_opp_id_0,
+ uint32_t *src_opp_id_1)
+{
+ uint32_t num_of_input_segments;
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ REG_GET_3(OPTC_DATA_SOURCE_SELECT,
+ OPTC_NUM_OF_INPUT_SEGMENT, &num_of_input_segments,
+ OPTC_SEG0_SRC_SEL, src_opp_id_0,
+ OPTC_SEG1_SRC_SEL, src_opp_id_1);
+
+ if (num_of_input_segments == 1)
+ *num_of_src_opp = 2;
+ else
+ *num_of_src_opp = 1;
+}
+
+void optc2_set_dwb_source(struct timing_generator *optc,
+ uint32_t dwb_pipe_inst)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ if (dwb_pipe_inst == 0)
+ REG_UPDATE(DWB_SOURCE_SELECT,
+ OPTC_DWB0_SOURCE_SELECT, optc->inst);
+ else if (dwb_pipe_inst == 1)
+ REG_UPDATE(DWB_SOURCE_SELECT,
+ OPTC_DWB1_SOURCE_SELECT, optc->inst);
+}
+
+void optc2_triplebuffer_lock(struct timing_generator *optc)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ REG_SET(OTG_GLOBAL_CONTROL0, 0,
+ OTG_MASTER_UPDATE_LOCK_SEL, optc->inst);
+
+ REG_SET(OTG_VUPDATE_KEEPOUT, 0,
+ OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN, 1);
+
+ REG_SET(OTG_MASTER_UPDATE_LOCK, 0,
+ OTG_MASTER_UPDATE_LOCK, 1);
+
+ if (optc->ctx->dce_environment != DCE_ENV_FPGA_MAXIMUS)
+ REG_WAIT(OTG_MASTER_UPDATE_LOCK,
+ UPDATE_LOCK_STATUS, 1,
+ 1, 10);
+}
+
+void optc2_triplebuffer_unlock(struct timing_generator *optc)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ REG_SET(OTG_MASTER_UPDATE_LOCK, 0,
+ OTG_MASTER_UPDATE_LOCK, 0);
+
+ REG_SET(OTG_VUPDATE_KEEPOUT, 0,
+ OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN, 0);
+
+}
+
+
+void optc2_setup_global_lock(struct timing_generator *optc)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+ uint32_t v_blank_start = 0;
+ uint32_t h_blank_start = 0, h_total = 0;
+
+ REG_SET(OTG_GLOBAL_CONTROL1, 0, MASTER_UPDATE_LOCK_DB_EN, 1);
+
+ REG_SET(OTG_GLOBAL_CONTROL2, 0, DIG_UPDATE_LOCATION, 20);
+
+ REG_GET(OTG_V_BLANK_START_END, OTG_V_BLANK_START, &v_blank_start);
+
+ REG_GET(OTG_H_BLANK_START_END, OTG_H_BLANK_START, &h_blank_start);
+
+ REG_GET(OTG_H_TOTAL, OTG_H_TOTAL, &h_total);
+ REG_UPDATE_2(OTG_GLOBAL_CONTROL1,
+ MASTER_UPDATE_LOCK_DB_X,
+ h_blank_start - 200 - 1,
+ MASTER_UPDATE_LOCK_DB_Y,
+ v_blank_start - 1);
+}
+
+void optc2_lock_global(struct timing_generator *optc)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ REG_UPDATE(OTG_GLOBAL_CONTROL2, GLOBAL_UPDATE_LOCK_EN, 1);
+
+ REG_SET(OTG_GLOBAL_CONTROL0, 0,
+ OTG_MASTER_UPDATE_LOCK_SEL, optc->inst);
+ REG_SET(OTG_MASTER_UPDATE_LOCK, 0,
+ OTG_MASTER_UPDATE_LOCK, 1);
+
+ /* Should be fast, status does not update on maximus */
+ if (optc->ctx->dce_environment != DCE_ENV_FPGA_MAXIMUS)
+ REG_WAIT(OTG_MASTER_UPDATE_LOCK,
+ UPDATE_LOCK_STATUS, 1,
+ 1, 10);
+}
+
+void optc2_lock(struct timing_generator *optc)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ REG_UPDATE(OTG_GLOBAL_CONTROL2, GLOBAL_UPDATE_LOCK_EN, 0);
+
+ REG_SET(OTG_GLOBAL_CONTROL0, 0,
+ OTG_MASTER_UPDATE_LOCK_SEL, optc->inst);
+ REG_SET(OTG_MASTER_UPDATE_LOCK, 0,
+ OTG_MASTER_UPDATE_LOCK, 1);
+
+ /* Should be fast, status does not update on maximus */
+ if (optc->ctx->dce_environment != DCE_ENV_FPGA_MAXIMUS)
+ REG_WAIT(OTG_MASTER_UPDATE_LOCK,
+ UPDATE_LOCK_STATUS, 1,
+ 1, 10);
+}
+
+void optc2_lock_doublebuffer_enable(struct timing_generator *optc)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+ uint32_t v_blank_start = 0;
+ uint32_t h_blank_start = 0;
+
+ REG_UPDATE(OTG_GLOBAL_CONTROL1, MASTER_UPDATE_LOCK_DB_EN, 1);
+
+ REG_UPDATE_2(OTG_GLOBAL_CONTROL2, GLOBAL_UPDATE_LOCK_EN, 1,
+ DIG_UPDATE_LOCATION, 20);
+
+ REG_GET(OTG_V_BLANK_START_END, OTG_V_BLANK_START, &v_blank_start);
+
+ REG_GET(OTG_H_BLANK_START_END, OTG_H_BLANK_START, &h_blank_start);
+
+ REG_UPDATE_2(OTG_GLOBAL_CONTROL1,
+ MASTER_UPDATE_LOCK_DB_X,
+ h_blank_start - 200 - 1,
+ MASTER_UPDATE_LOCK_DB_Y,
+ v_blank_start - 1);
+}
+
+void optc2_lock_doublebuffer_disable(struct timing_generator *optc)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ REG_UPDATE_2(OTG_GLOBAL_CONTROL1,
+ MASTER_UPDATE_LOCK_DB_X,
+ 0,
+ MASTER_UPDATE_LOCK_DB_Y,
+ 0);
+
+ REG_UPDATE_2(OTG_GLOBAL_CONTROL2, GLOBAL_UPDATE_LOCK_EN, 0,
+ DIG_UPDATE_LOCATION, 0);
+
+ REG_UPDATE(OTG_GLOBAL_CONTROL1, MASTER_UPDATE_LOCK_DB_EN, 0);
+}
+
+void optc2_setup_manual_trigger(struct timing_generator *optc)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ REG_SET(OTG_MANUAL_FLOW_CONTROL, 0,
+ MANUAL_FLOW_CONTROL, 1);
+
+ REG_SET(OTG_GLOBAL_CONTROL2, 0,
+ MANUAL_FLOW_CONTROL_SEL, optc->inst);
+
+ REG_SET_8(OTG_TRIGA_CNTL, 0,
+ OTG_TRIGA_SOURCE_SELECT, 22,
+ OTG_TRIGA_SOURCE_PIPE_SELECT, optc->inst,
+ OTG_TRIGA_RISING_EDGE_DETECT_CNTL, 1,
+ OTG_TRIGA_FALLING_EDGE_DETECT_CNTL, 0,
+ OTG_TRIGA_POLARITY_SELECT, 0,
+ OTG_TRIGA_FREQUENCY_SELECT, 0,
+ OTG_TRIGA_DELAY, 0,
+ OTG_TRIGA_CLEAR, 1);
+}
+
+void optc2_program_manual_trigger(struct timing_generator *optc)
+{
+ struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+ REG_SET(OTG_TRIGA_MANUAL_TRIG, 0,
+ OTG_TRIGA_MANUAL_TRIG, 1);
+}
+
+static struct timing_generator_funcs dcn20_tg_funcs = {
+ .validate_timing = optc1_validate_timing,
+ .program_timing = optc1_program_timing,
+ .setup_vertical_interrupt0 = optc1_setup_vertical_interrupt0,
+ .setup_vertical_interrupt1 = optc1_setup_vertical_interrupt1,
+ .setup_vertical_interrupt2 = optc1_setup_vertical_interrupt2,
+ .program_global_sync = optc1_program_global_sync,
+ .enable_crtc = optc2_enable_crtc,
+ .disable_crtc = optc1_disable_crtc,
+ /* used by enable_timing_synchronization. Not need for FPGA */
+ .is_counter_moving = optc1_is_counter_moving,
+ .get_position = optc1_get_position,
+ .get_frame_count = optc1_get_vblank_counter,
+ .get_scanoutpos = optc1_get_crtc_scanoutpos,
+ .get_otg_active_size = optc1_get_otg_active_size,
+ .set_early_control = optc1_set_early_control,
+ /* used by enable_timing_synchronization. Not need for FPGA */
+ .wait_for_state = optc1_wait_for_state,
+ .set_blank = optc1_set_blank,
+ .is_blanked = optc1_is_blanked,
+ .set_blank_color = optc1_program_blank_color,
+ .enable_reset_trigger = optc1_enable_reset_trigger,
+ .enable_crtc_reset = optc1_enable_crtc_reset,
+ .did_triggered_reset_occur = optc1_did_triggered_reset_occur,
+ .triplebuffer_lock = optc2_triplebuffer_lock,
+ .triplebuffer_unlock = optc2_triplebuffer_unlock,
+ .disable_reset_trigger = optc1_disable_reset_trigger,
+ .lock = optc2_lock,
+ .unlock = optc1_unlock,
+ .lock_global = optc2_lock_global,
+ .setup_global_lock = optc2_setup_global_lock,
+ .lock_doublebuffer_enable = optc2_lock_doublebuffer_enable,
+ .lock_doublebuffer_disable = optc2_lock_doublebuffer_disable,
+ .enable_optc_clock = optc1_enable_optc_clock,
+ .set_drr = optc1_set_drr,
+ .set_static_screen_control = optc1_set_static_screen_control,
+ .program_stereo = optc1_program_stereo,
+ .is_stereo_left_eye = optc1_is_stereo_left_eye,
+ .set_blank_data_double_buffer = optc1_set_blank_data_double_buffer,
+ .tg_init = optc1_tg_init,
+ .is_tg_enabled = optc1_is_tg_enabled,
+ .is_optc_underflow_occurred = optc1_is_optc_underflow_occurred,
+ .clear_optc_underflow = optc1_clear_optc_underflow,
+ .setup_global_swap_lock = NULL,
+ .get_crc = optc1_get_crc,
+ .configure_crc = optc1_configure_crc,
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ .set_dsc_config = optc2_set_dsc_config,
+#endif
+ .set_dwb_source = optc2_set_dwb_source,
+ .set_odm_bypass = optc2_set_odm_bypass,
+ .set_odm_combine = optc2_set_odm_combine,
+ .get_optc_source = optc2_get_optc_source,
+ .set_gsl = optc2_set_gsl,
+ .set_gsl_source_select = optc2_set_gsl_source_select,
+ .set_vtg_params = optc1_set_vtg_params,
+ .program_manual_trigger = optc2_program_manual_trigger,
+ .setup_manual_trigger = optc2_setup_manual_trigger
+};
+
+void dcn20_timing_generator_init(struct optc *optc1)
+{
+ optc1->base.funcs = &dcn20_tg_funcs;
+
+ optc1->max_h_total = optc1->tg_mask->OTG_H_TOTAL + 1;
+ optc1->max_v_total = optc1->tg_mask->OTG_V_TOTAL + 1;
+
+ optc1->min_h_blank = 32;
+ optc1->min_v_blank = 3;
+ optc1->min_v_blank_interlace = 5;
+ optc1->min_h_sync_width = 4;// Minimum HSYNC = 8 pixels asked By HW in the first place for no actual reason. Oculus Rift S will not light up with 8 as it's hsyncWidth is 6. Changing it to 4 to fix that issue.
+ optc1->min_v_sync_width = 1;
+ optc1->comb_opp_id = 0xf;
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_optc.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_optc.h
new file mode 100644
index 000000000000..ebf07c582da2
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_optc.h
@@ -0,0 +1,116 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_OPTC_DCN20_H__
+#define __DC_OPTC_DCN20_H__
+
+#include "../dcn10/dcn10_optc.h"
+
+#define TG_COMMON_REG_LIST_DCN2_0(inst) \
+ TG_COMMON_REG_LIST_DCN(inst),\
+ SRI(OTG_GLOBAL_CONTROL1, OTG, inst),\
+ SRI(OTG_GLOBAL_CONTROL2, OTG, inst),\
+ SRI(OTG_GSL_WINDOW_X, OTG, inst),\
+ SRI(OTG_GSL_WINDOW_Y, OTG, inst),\
+ SRI(OTG_VUPDATE_KEEPOUT, OTG, inst),\
+ SRI(OTG_DSC_START_POSITION, OTG, inst),\
+ SRI(OPTC_DATA_FORMAT_CONTROL, ODM, inst),\
+ SRI(OPTC_BYTES_PER_PIXEL, ODM, inst),\
+ SRI(OPTC_WIDTH_CONTROL, ODM, inst),\
+ SRI(OPTC_MEMORY_CONFIG, ODM, inst),\
+ SR(DWB_SOURCE_SELECT),\
+ SRI(OTG_MANUAL_FLOW_CONTROL, OTG, inst)
+
+#define TG_COMMON_MASK_SH_LIST_DCN2_0(mask_sh)\
+ TG_COMMON_MASK_SH_LIST_DCN(mask_sh),\
+ SF(OTG0_OTG_GLOBAL_CONTROL1, MASTER_UPDATE_LOCK_DB_X, mask_sh),\
+ SF(OTG0_OTG_GLOBAL_CONTROL1, MASTER_UPDATE_LOCK_DB_Y, mask_sh),\
+ SF(OTG0_OTG_GLOBAL_CONTROL1, MASTER_UPDATE_LOCK_DB_EN, mask_sh),\
+ SF(OTG0_OTG_GLOBAL_CONTROL2, GLOBAL_UPDATE_LOCK_EN, mask_sh),\
+ SF(OTG0_OTG_GLOBAL_CONTROL2, DIG_UPDATE_LOCATION, mask_sh),\
+ SF(OTG0_OTG_DOUBLE_BUFFER_CONTROL, OTG_RANGE_TIMING_DBUF_UPDATE_MODE, mask_sh),\
+ SF(OTG0_OTG_GSL_WINDOW_X, OTG_GSL_WINDOW_START_X, mask_sh),\
+ SF(OTG0_OTG_GSL_WINDOW_X, OTG_GSL_WINDOW_END_X, mask_sh), \
+ SF(OTG0_OTG_GSL_WINDOW_Y, OTG_GSL_WINDOW_START_Y, mask_sh),\
+ SF(OTG0_OTG_GSL_WINDOW_Y, OTG_GSL_WINDOW_END_Y, mask_sh),\
+ SF(OTG0_OTG_VUPDATE_KEEPOUT, OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN, mask_sh), \
+ SF(OTG0_OTG_VUPDATE_KEEPOUT, MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_START_OFFSET, mask_sh), \
+ SF(OTG0_OTG_VUPDATE_KEEPOUT, MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_END_OFFSET, mask_sh), \
+ SF(OTG0_OTG_GSL_CONTROL, OTG_GSL_MASTER_MODE, mask_sh), \
+ SF(OTG0_OTG_GSL_CONTROL, OTG_MASTER_UPDATE_LOCK_GSL_EN, mask_sh), \
+ SF(OTG0_OTG_DSC_START_POSITION, OTG_DSC_START_POSITION_X, mask_sh), \
+ SF(OTG0_OTG_DSC_START_POSITION, OTG_DSC_START_POSITION_LINE_NUM, mask_sh),\
+ SF(ODM0_OPTC_DATA_SOURCE_SELECT, OPTC_SEG0_SRC_SEL, mask_sh),\
+ SF(ODM0_OPTC_DATA_SOURCE_SELECT, OPTC_SEG1_SRC_SEL, mask_sh),\
+ SF(ODM0_OPTC_DATA_SOURCE_SELECT, OPTC_NUM_OF_INPUT_SEGMENT, mask_sh),\
+ SF(ODM0_OPTC_MEMORY_CONFIG, OPTC_MEM_SEL, mask_sh),\
+ SF(ODM0_OPTC_DATA_FORMAT_CONTROL, OPTC_DATA_FORMAT, mask_sh),\
+ SF(ODM0_OPTC_DATA_FORMAT_CONTROL, OPTC_DSC_MODE, mask_sh),\
+ SF(ODM0_OPTC_BYTES_PER_PIXEL, OPTC_DSC_BYTES_PER_PIXEL, mask_sh),\
+ SF(ODM0_OPTC_WIDTH_CONTROL, OPTC_DSC_SLICE_WIDTH, mask_sh),\
+ SF(ODM0_OPTC_WIDTH_CONTROL, OPTC_SEGMENT_WIDTH, mask_sh),\
+ SF(DWB_SOURCE_SELECT, OPTC_DWB0_SOURCE_SELECT, mask_sh),\
+ SF(DWB_SOURCE_SELECT, OPTC_DWB1_SOURCE_SELECT, mask_sh),\
+ SF(OTG0_OTG_MANUAL_FLOW_CONTROL, MANUAL_FLOW_CONTROL, mask_sh)
+
+void dcn20_timing_generator_init(struct optc *optc);
+
+bool optc2_enable_crtc(struct timing_generator *optc);
+
+void optc2_set_gsl(struct timing_generator *optc,
+ const struct gsl_params *params);
+
+void optc2_set_gsl_source_select(struct timing_generator *optc,
+ int group_idx,
+ uint32_t gsl_ready_signal);
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+void optc2_set_dsc_config(struct timing_generator *optc,
+ enum optc_dsc_mode dsc_mode,
+ uint32_t dsc_bytes_per_pixel,
+ uint32_t dsc_slice_width);
+#endif
+
+void optc2_set_odm_bypass(struct timing_generator *optc,
+ const struct dc_crtc_timing *dc_crtc_timing);
+
+void optc2_set_odm_combine(struct timing_generator *optc, int combine_opp_id,
+ int mpcc_hactive, enum dc_pixel_encoding pixel_encoding);
+
+void optc2_get_optc_source(struct timing_generator *optc,
+ uint32_t *num_of_src_opp,
+ uint32_t *src_opp_id_0,
+ uint32_t *src_opp_id_1);
+
+void optc2_triplebuffer_lock(struct timing_generator *optc);
+void optc2_triplebuffer_unlock(struct timing_generator *optc);
+void optc2_lock(struct timing_generator *optc);
+void optc2_lock_global(struct timing_generator *optc);
+void optc2_setup_global_lock(struct timing_generator *optc);
+void optc2_lock_doublebuffer_disable(struct timing_generator *optc);
+void optc2_lock_doublebuffer_enable(struct timing_generator *optc);
+void optc2_program_manual_trigger(struct timing_generator *optc);
+
+#endif /* __DC_OPTC_DCN20_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c
new file mode 100644
index 000000000000..b949e202d6cb
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c
@@ -0,0 +1,3191 @@
+/*
+* Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include <linux/slab.h>
+
+#include "dm_services.h"
+#include "dc.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "dcn20/dcn20_resource.h"
+
+#include "dcn10/dcn10_hubp.h"
+#include "dcn10/dcn10_ipp.h"
+#include "dcn20_hubbub.h"
+#include "dcn20_mpc.h"
+#include "dcn20_hubp.h"
+#include "irq/dcn20/irq_service_dcn20.h"
+#include "dcn20_dpp.h"
+#include "dcn20_optc.h"
+#include "dcn20_hwseq.h"
+#include "dce110/dce110_hw_sequencer.h"
+#include "dcn10/dcn10_resource.h"
+#include "dcn20_opp.h"
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+#include "dcn20_dsc.h"
+#endif
+
+#include "dcn20_link_encoder.h"
+#include "dcn20_stream_encoder.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "virtual/virtual_stream_encoder.h"
+#include "dce110/dce110_resource.h"
+#include "dml/display_mode_vba.h"
+#include "dcn20_dccg.h"
+#include "dcn20_vmid.h"
+
+#include "navi10_ip_offset.h"
+
+#include "dcn/dcn_2_0_0_offset.h"
+#include "dcn/dcn_2_0_0_sh_mask.h"
+
+#include "nbio/nbio_2_3_offset.h"
+
+#include "dcn20/dcn20_dwb.h"
+#include "dcn20/dcn20_mmhubbub.h"
+
+#include "mmhub/mmhub_2_0_0_offset.h"
+#include "mmhub/mmhub_2_0_0_sh_mask.h"
+
+#include "reg_helper.h"
+#include "dce/dce_abm.h"
+#include "dce/dce_dmcu.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+#include "vm_helper.h"
+
+#include "amdgpu_socbb.h"
+
+#define SOC_BOUNDING_BOX_VALID false
+#define DC_LOGGER_INIT(logger)
+
+struct _vcs_dpi_ip_params_st dcn2_0_ip = {
+ .odm_capable = 1,
+ .gpuvm_enable = 0,
+ .hostvm_enable = 0,
+ .gpuvm_max_page_table_levels = 4,
+ .hostvm_max_page_table_levels = 4,
+ .hostvm_cached_page_table_levels = 0,
+ .pte_group_size_bytes = 2048,
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ .num_dsc = 6,
+#else
+ .num_dsc = 0,
+#endif
+ .rob_buffer_size_kbytes = 168,
+ .det_buffer_size_kbytes = 164,
+ .dpte_buffer_size_in_pte_reqs_luma = 84,
+ .pde_proc_buffer_size_64k_reqs = 48,
+ .dpp_output_buffer_pixels = 2560,
+ .opp_output_buffer_lines = 1,
+ .pixel_chunk_size_kbytes = 8,
+ .pte_chunk_size_kbytes = 2,
+ .meta_chunk_size_kbytes = 2,
+ .writeback_chunk_size_kbytes = 2,
+ .line_buffer_size_bits = 789504,
+ .is_line_buffer_bpp_fixed = 0,
+ .line_buffer_fixed_bpp = 0,
+ .dcc_supported = true,
+ .max_line_buffer_lines = 12,
+ .writeback_luma_buffer_size_kbytes = 12,
+ .writeback_chroma_buffer_size_kbytes = 8,
+ .writeback_chroma_line_buffer_width_pixels = 4,
+ .writeback_max_hscl_ratio = 1,
+ .writeback_max_vscl_ratio = 1,
+ .writeback_min_hscl_ratio = 1,
+ .writeback_min_vscl_ratio = 1,
+ .writeback_max_hscl_taps = 12,
+ .writeback_max_vscl_taps = 12,
+ .writeback_line_buffer_luma_buffer_size = 0,
+ .writeback_line_buffer_chroma_buffer_size = 14643,
+ .cursor_buffer_size = 8,
+ .cursor_chunk_size = 2,
+ .max_num_otg = 6,
+ .max_num_dpp = 6,
+ .max_num_wb = 1,
+ .max_dchub_pscl_bw_pix_per_clk = 4,
+ .max_pscl_lb_bw_pix_per_clk = 2,
+ .max_lb_vscl_bw_pix_per_clk = 4,
+ .max_vscl_hscl_bw_pix_per_clk = 4,
+ .max_hscl_ratio = 8,
+ .max_vscl_ratio = 8,
+ .hscl_mults = 4,
+ .vscl_mults = 4,
+ .max_hscl_taps = 8,
+ .max_vscl_taps = 8,
+ .dispclk_ramp_margin_percent = 1,
+ .underscan_factor = 1.10,
+ .min_vblank_lines = 32, //
+ .dppclk_delay_subtotal = 77, //
+ .dppclk_delay_scl_lb_only = 16,
+ .dppclk_delay_scl = 50,
+ .dppclk_delay_cnvc_formatter = 8,
+ .dppclk_delay_cnvc_cursor = 6,
+ .dispclk_delay_subtotal = 87, //
+ .dcfclk_cstate_latency = 10, // SRExitTime
+ .max_inter_dcn_tile_repeaters = 8,
+
+ .xfc_supported = true,
+ .xfc_fill_bw_overhead_percent = 10.0,
+ .xfc_fill_constant_bytes = 0,
+};
+
+struct _vcs_dpi_soc_bounding_box_st dcn2_0_soc = { 0 };
+
+
+#ifndef mmDP0_DP_DPHY_INTERNAL_CTRL
+ #define mmDP0_DP_DPHY_INTERNAL_CTRL 0x210f
+ #define mmDP0_DP_DPHY_INTERNAL_CTRL_BASE_IDX 2
+ #define mmDP1_DP_DPHY_INTERNAL_CTRL 0x220f
+ #define mmDP1_DP_DPHY_INTERNAL_CTRL_BASE_IDX 2
+ #define mmDP2_DP_DPHY_INTERNAL_CTRL 0x230f
+ #define mmDP2_DP_DPHY_INTERNAL_CTRL_BASE_IDX 2
+ #define mmDP3_DP_DPHY_INTERNAL_CTRL 0x240f
+ #define mmDP3_DP_DPHY_INTERNAL_CTRL_BASE_IDX 2
+ #define mmDP4_DP_DPHY_INTERNAL_CTRL 0x250f
+ #define mmDP4_DP_DPHY_INTERNAL_CTRL_BASE_IDX 2
+ #define mmDP5_DP_DPHY_INTERNAL_CTRL 0x260f
+ #define mmDP5_DP_DPHY_INTERNAL_CTRL_BASE_IDX 2
+ #define mmDP6_DP_DPHY_INTERNAL_CTRL 0x270f
+ #define mmDP6_DP_DPHY_INTERNAL_CTRL_BASE_IDX 2
+#endif
+
+
+enum dcn20_clk_src_array_id {
+ DCN20_CLK_SRC_PLL0,
+ DCN20_CLK_SRC_PLL1,
+ DCN20_CLK_SRC_PLL2,
+ DCN20_CLK_SRC_PLL3,
+ DCN20_CLK_SRC_PLL4,
+ DCN20_CLK_SRC_PLL5,
+ DCN20_CLK_SRC_TOTAL
+};
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file */
+
+/* DCN */
+/* TODO awful hack. fixup dcn20_dwb.h */
+#undef BASE_INNER
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)\
+ .reg_name = BASE(mm ## reg_name ## _BASE_IDX) + \
+ mm ## reg_name
+
+#define SRI(reg_name, block, id)\
+ .reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+ mm ## block ## id ## _ ## reg_name
+
+#define SRIR(var_name, reg_name, block, id)\
+ .var_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+ mm ## block ## id ## _ ## reg_name
+
+#define SRII(reg_name, block, id)\
+ .reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+ mm ## block ## id ## _ ## reg_name
+
+#define DCCG_SRII(reg_name, block, id)\
+ .block ## _ ## reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+ mm ## block ## id ## _ ## reg_name
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) \
+ NBIO_BASE__INST0_SEG ## seg
+
+#define NBIO_BASE(seg) \
+ NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)\
+ .reg_name = NBIO_BASE(mm ## reg_name ## _BASE_IDX) + \
+ mm ## reg_name
+
+/* MMHUB */
+#define MMHUB_BASE_INNER(seg) \
+ MMHUB_BASE__INST0_SEG ## seg
+
+#define MMHUB_BASE(seg) \
+ MMHUB_BASE_INNER(seg)
+
+#define MMHUB_SR(reg_name)\
+ .reg_name = MMHUB_BASE(mmMM ## reg_name ## _BASE_IDX) + \
+ mmMM ## reg_name
+
+static const struct bios_registers bios_regs = {
+ NBIO_SR(BIOS_SCRATCH_3),
+ NBIO_SR(BIOS_SCRATCH_6)
+};
+
+#define clk_src_regs(index, pllid)\
+[index] = {\
+ CS_COMMON_REG_LIST_DCN2_0(index, pllid),\
+}
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+ clk_src_regs(0, A),
+ clk_src_regs(1, B),
+ clk_src_regs(2, C),
+ clk_src_regs(3, D),
+ clk_src_regs(4, E),
+ clk_src_regs(5, F)
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+ CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+ CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+static const struct dce_dmcu_registers dmcu_regs = {
+ DMCU_DCN10_REG_LIST()
+};
+
+static const struct dce_dmcu_shift dmcu_shift = {
+ DMCU_MASK_SH_LIST_DCN10(__SHIFT)
+};
+
+static const struct dce_dmcu_mask dmcu_mask = {
+ DMCU_MASK_SH_LIST_DCN10(_MASK)
+};
+
+static const struct dce_abm_registers abm_regs = {
+ ABM_DCN20_REG_LIST()
+};
+
+static const struct dce_abm_shift abm_shift = {
+ ABM_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+ ABM_MASK_SH_LIST_DCN20(_MASK)
+};
+
+#define audio_regs(id)\
+[id] = {\
+ AUD_COMMON_REG_LIST(id)\
+}
+
+static const struct dce_audio_registers audio_regs[] = {
+ audio_regs(0),
+ audio_regs(1),
+ audio_regs(2),
+ audio_regs(3),
+ audio_regs(4),
+ audio_regs(5),
+ audio_regs(6),
+};
+
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
+ SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
+ SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+ AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+ DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_aduio_mask audio_mask = {
+ DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define stream_enc_regs(id)\
+[id] = {\
+ SE_DCN2_REG_LIST(id)\
+}
+
+static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
+ stream_enc_regs(0),
+ stream_enc_regs(1),
+ stream_enc_regs(2),
+ stream_enc_regs(3),
+ stream_enc_regs(4),
+ stream_enc_regs(5),
+};
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+ SE_COMMON_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+ SE_COMMON_MASK_SH_LIST_DCN20(_MASK)
+};
+
+
+#define aux_regs(id)\
+[id] = {\
+ DCN2_AUX_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
+ aux_regs(0),
+ aux_regs(1),
+ aux_regs(2),
+ aux_regs(3),
+ aux_regs(4),
+ aux_regs(5)
+};
+
+#define hpd_regs(id)\
+[id] = {\
+ HPD_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
+ hpd_regs(0),
+ hpd_regs(1),
+ hpd_regs(2),
+ hpd_regs(3),
+ hpd_regs(4),
+ hpd_regs(5)
+};
+
+#define link_regs(id, phyid)\
+[id] = {\
+ LE_DCN10_REG_LIST(id), \
+ UNIPHY_DCN2_REG_LIST(phyid), \
+ SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
+}
+
+static const struct dcn10_link_enc_registers link_enc_regs[] = {
+ link_regs(0, A),
+ link_regs(1, B),
+ link_regs(2, C),
+ link_regs(3, D),
+ link_regs(4, E),
+ link_regs(5, F)
+};
+
+static const struct dcn10_link_enc_shift le_shift = {
+ LINK_ENCODER_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn10_link_enc_mask le_mask = {
+ LINK_ENCODER_MASK_SH_LIST_DCN20(_MASK)
+};
+
+#define ipp_regs(id)\
+[id] = {\
+ IPP_REG_LIST_DCN20(id),\
+}
+
+static const struct dcn10_ipp_registers ipp_regs[] = {
+ ipp_regs(0),
+ ipp_regs(1),
+ ipp_regs(2),
+ ipp_regs(3),
+ ipp_regs(4),
+ ipp_regs(5),
+};
+
+static const struct dcn10_ipp_shift ipp_shift = {
+ IPP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn10_ipp_mask ipp_mask = {
+ IPP_MASK_SH_LIST_DCN20(_MASK),
+};
+
+#define opp_regs(id)\
+[id] = {\
+ OPP_REG_LIST_DCN20(id),\
+}
+
+static const struct dcn20_opp_registers opp_regs[] = {
+ opp_regs(0),
+ opp_regs(1),
+ opp_regs(2),
+ opp_regs(3),
+ opp_regs(4),
+ opp_regs(5),
+};
+
+static const struct dcn20_opp_shift opp_shift = {
+ OPP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn20_opp_mask opp_mask = {
+ OPP_MASK_SH_LIST_DCN20(_MASK)
+};
+
+#define aux_engine_regs(id)\
+[id] = {\
+ AUX_COMMON_REG_LIST0(id), \
+ .AUXN_IMPCAL = 0, \
+ .AUXP_IMPCAL = 0, \
+ .AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+ aux_engine_regs(0),
+ aux_engine_regs(1),
+ aux_engine_regs(2),
+ aux_engine_regs(3),
+ aux_engine_regs(4),
+ aux_engine_regs(5)
+};
+
+#define tf_regs(id)\
+[id] = {\
+ TF_REG_LIST_DCN20(id),\
+}
+
+static const struct dcn2_dpp_registers tf_regs[] = {
+ tf_regs(0),
+ tf_regs(1),
+ tf_regs(2),
+ tf_regs(3),
+ tf_regs(4),
+ tf_regs(5),
+};
+
+static const struct dcn2_dpp_shift tf_shift = {
+ TF_REG_LIST_SH_MASK_DCN20(__SHIFT)
+};
+
+static const struct dcn2_dpp_mask tf_mask = {
+ TF_REG_LIST_SH_MASK_DCN20(_MASK)
+};
+
+#define dwbc_regs_dcn2(id)\
+[id] = {\
+ DWBC_COMMON_REG_LIST_DCN2_0(id),\
+ }
+
+static const struct dcn20_dwbc_registers dwbc20_regs[] = {
+ dwbc_regs_dcn2(0),
+};
+
+static const struct dcn20_dwbc_shift dwbc20_shift = {
+ DWBC_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dcn20_dwbc_mask dwbc20_mask = {
+ DWBC_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+#define mcif_wb_regs_dcn2(id)\
+[id] = {\
+ MCIF_WB_COMMON_REG_LIST_DCN2_0(id),\
+ }
+
+static const struct dcn20_mmhubbub_registers mcif_wb20_regs[] = {
+ mcif_wb_regs_dcn2(0),
+};
+
+static const struct dcn20_mmhubbub_shift mcif_wb20_shift = {
+ MCIF_WB_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dcn20_mmhubbub_mask mcif_wb20_mask = {
+ MCIF_WB_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+static const struct dcn20_mpc_registers mpc_regs = {
+ MPC_REG_LIST_DCN2_0(0),
+ MPC_REG_LIST_DCN2_0(1),
+ MPC_REG_LIST_DCN2_0(2),
+ MPC_REG_LIST_DCN2_0(3),
+ MPC_REG_LIST_DCN2_0(4),
+ MPC_REG_LIST_DCN2_0(5),
+ MPC_OUT_MUX_REG_LIST_DCN2_0(0),
+ MPC_OUT_MUX_REG_LIST_DCN2_0(1),
+ MPC_OUT_MUX_REG_LIST_DCN2_0(2),
+ MPC_OUT_MUX_REG_LIST_DCN2_0(3),
+ MPC_OUT_MUX_REG_LIST_DCN2_0(4),
+ MPC_OUT_MUX_REG_LIST_DCN2_0(5),
+};
+
+static const struct dcn20_mpc_shift mpc_shift = {
+ MPC_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dcn20_mpc_mask mpc_mask = {
+ MPC_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+#define tg_regs(id)\
+[id] = {TG_COMMON_REG_LIST_DCN2_0(id)}
+
+
+static const struct dcn_optc_registers tg_regs[] = {
+ tg_regs(0),
+ tg_regs(1),
+ tg_regs(2),
+ tg_regs(3),
+ tg_regs(4),
+ tg_regs(5)
+};
+
+static const struct dcn_optc_shift tg_shift = {
+ TG_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dcn_optc_mask tg_mask = {
+ TG_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+#define hubp_regs(id)\
+[id] = {\
+ HUBP_REG_LIST_DCN20(id)\
+}
+
+static const struct dcn_hubp2_registers hubp_regs[] = {
+ hubp_regs(0),
+ hubp_regs(1),
+ hubp_regs(2),
+ hubp_regs(3),
+ hubp_regs(4),
+ hubp_regs(5)
+};
+
+static const struct dcn_hubp2_shift hubp_shift = {
+ HUBP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn_hubp2_mask hubp_mask = {
+ HUBP_MASK_SH_LIST_DCN20(_MASK)
+};
+
+static const struct dcn_hubbub_registers hubbub_reg = {
+ HUBBUB_REG_LIST_DCN20(0)
+};
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+ HUBBUB_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+ HUBBUB_MASK_SH_LIST_DCN20(_MASK)
+};
+
+#define vmid_regs(id)\
+[id] = {\
+ DCN20_VMID_REG_LIST(id)\
+}
+
+static const struct dcn_vmid_registers vmid_regs[] = {
+ vmid_regs(0),
+ vmid_regs(1),
+ vmid_regs(2),
+ vmid_regs(3),
+ vmid_regs(4),
+ vmid_regs(5),
+ vmid_regs(6),
+ vmid_regs(7),
+ vmid_regs(8),
+ vmid_regs(9),
+ vmid_regs(10),
+ vmid_regs(11),
+ vmid_regs(12),
+ vmid_regs(13),
+ vmid_regs(14),
+ vmid_regs(15)
+};
+
+static const struct dcn20_vmid_shift vmid_shifts = {
+ DCN20_VMID_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn20_vmid_mask vmid_masks = {
+ DCN20_VMID_MASK_SH_LIST(_MASK)
+};
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+#define dsc_regsDCN20(id)\
+[id] = {\
+ DSC_REG_LIST_DCN20(id)\
+}
+
+static const struct dcn20_dsc_registers dsc_regs[] = {
+ dsc_regsDCN20(0),
+ dsc_regsDCN20(1),
+ dsc_regsDCN20(2),
+ dsc_regsDCN20(3),
+ dsc_regsDCN20(4),
+ dsc_regsDCN20(5)
+};
+
+static const struct dcn20_dsc_shift dsc_shift = {
+ DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
+};
+
+static const struct dcn20_dsc_mask dsc_mask = {
+ DSC_REG_LIST_SH_MASK_DCN20(_MASK)
+};
+#endif
+
+static const struct dccg_registers dccg_regs = {
+ DCCG_REG_LIST_DCN2()
+};
+
+static const struct dccg_shift dccg_shift = {
+ DCCG_MASK_SH_LIST_DCN2(__SHIFT)
+};
+
+static const struct dccg_mask dccg_mask = {
+ DCCG_MASK_SH_LIST_DCN2(_MASK)
+};
+
+static const struct resource_caps res_cap_nv10 = {
+ .num_timing_generator = 6,
+ .num_opp = 6,
+ .num_video_plane = 6,
+ .num_audio = 7,
+ .num_stream_encoder = 6,
+ .num_pll = 6,
+ .num_dwb = 1,
+ .num_ddc = 6,
+ .num_vmid = 16,
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ .num_dsc = 6,
+#endif
+};
+
+static const struct dc_plane_cap plane_cap = {
+ .type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+ .blends_with_above = true,
+ .blends_with_below = true,
+ .per_pixel_alpha = true,
+
+ .pixel_format_support = {
+ .argb8888 = true,
+ .nv12 = true,
+ .fp16 = true
+ },
+
+ .max_upscale_factor = {
+ .argb8888 = 16000,
+ .nv12 = 16000,
+ .fp16 = 1
+ },
+
+ .max_downscale_factor = {
+ .argb8888 = 250,
+ .nv12 = 250,
+ .fp16 = 1
+ }
+};
+
+static const struct dc_debug_options debug_defaults_drv = {
+ .disable_dmcu = true,
+ .force_abm_enable = false,
+ .timing_trace = false,
+ .clock_trace = true,
+ .disable_pplib_clock_request = true,
+ .pipe_split_policy = MPC_SPLIT_DYNAMIC,
+ .force_single_disp_pipe_split = true,
+ .disable_dcc = DCC_ENABLE,
+ .vsr_support = true,
+ .performance_trace = false,
+ .max_downscale_src_width = 5120,/*upto 5K*/
+ .disable_pplib_wm_range = false,
+ .scl_reset_length10 = true,
+ .sanity_checks = false,
+ .disable_tri_buf = true,
+ .underflow_assert_delay_us = 0xFFFFFFFF,
+};
+
+static const struct dc_debug_options debug_defaults_diags = {
+ .disable_dmcu = true,
+ .force_abm_enable = false,
+ .timing_trace = true,
+ .clock_trace = true,
+ .disable_dpp_power_gate = true,
+ .disable_hubp_power_gate = true,
+ .disable_clock_gate = true,
+ .disable_pplib_clock_request = true,
+ .disable_pplib_wm_range = true,
+ .disable_stutter = true,
+ .scl_reset_length10 = true,
+ .underflow_assert_delay_us = 0xFFFFFFFF,
+};
+
+void dcn20_dpp_destroy(struct dpp **dpp)
+{
+ kfree(TO_DCN20_DPP(*dpp));
+ *dpp = NULL;
+}
+
+struct dpp *dcn20_dpp_create(
+ struct dc_context *ctx,
+ uint32_t inst)
+{
+ struct dcn20_dpp *dpp =
+ kzalloc(sizeof(struct dcn20_dpp), GFP_KERNEL);
+
+ if (!dpp)
+ return NULL;
+
+ if (dpp2_construct(dpp, ctx, inst,
+ &tf_regs[inst], &tf_shift, &tf_mask))
+ return &dpp->base;
+
+ BREAK_TO_DEBUGGER();
+ kfree(dpp);
+ return NULL;
+}
+
+struct input_pixel_processor *dcn20_ipp_create(
+ struct dc_context *ctx, uint32_t inst)
+{
+ struct dcn10_ipp *ipp =
+ kzalloc(sizeof(struct dcn10_ipp), GFP_KERNEL);
+
+ if (!ipp) {
+ BREAK_TO_DEBUGGER();
+ return NULL;
+ }
+
+ dcn20_ipp_construct(ipp, ctx, inst,
+ &ipp_regs[inst], &ipp_shift, &ipp_mask);
+ return &ipp->base;
+}
+
+
+struct output_pixel_processor *dcn20_opp_create(
+ struct dc_context *ctx, uint32_t inst)
+{
+ struct dcn20_opp *opp =
+ kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
+
+ if (!opp) {
+ BREAK_TO_DEBUGGER();
+ return NULL;
+ }
+
+ dcn20_opp_construct(opp, ctx, inst,
+ &opp_regs[inst], &opp_shift, &opp_mask);
+ return &opp->base;
+}
+
+struct dce_aux *dcn20_aux_engine_create(
+ struct dc_context *ctx,
+ uint32_t inst)
+{
+ struct aux_engine_dce110 *aux_engine =
+ kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+ if (!aux_engine)
+ return NULL;
+
+ dce110_aux_engine_construct(aux_engine, ctx, inst,
+ SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+ &aux_engine_regs[inst]);
+
+ return &aux_engine->base;
+}
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+ i2c_inst_regs(1),
+ i2c_inst_regs(2),
+ i2c_inst_regs(3),
+ i2c_inst_regs(4),
+ i2c_inst_regs(5),
+ i2c_inst_regs(6),
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+ I2C_COMMON_MASK_SH_LIST_DCN2(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+ I2C_COMMON_MASK_SH_LIST_DCN2(_MASK)
+};
+
+struct dce_i2c_hw *dcn20_i2c_hw_create(
+ struct dc_context *ctx,
+ uint32_t inst)
+{
+ struct dce_i2c_hw *dce_i2c_hw =
+ kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+ if (!dce_i2c_hw)
+ return NULL;
+
+ dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+ &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+ return dce_i2c_hw;
+}
+struct mpc *dcn20_mpc_create(struct dc_context *ctx)
+{
+ struct dcn20_mpc *mpc20 = kzalloc(sizeof(struct dcn20_mpc),
+ GFP_KERNEL);
+
+ if (!mpc20)
+ return NULL;
+
+ dcn20_mpc_construct(mpc20, ctx,
+ &mpc_regs,
+ &mpc_shift,
+ &mpc_mask,
+ 6);
+
+ return &mpc20->base;
+}
+
+struct hubbub *dcn20_hubbub_create(struct dc_context *ctx)
+{
+ int i;
+ struct dcn20_hubbub *hubbub = kzalloc(sizeof(struct dcn20_hubbub),
+ GFP_KERNEL);
+
+ if (!hubbub)
+ return NULL;
+
+ hubbub2_construct(hubbub, ctx,
+ &hubbub_reg,
+ &hubbub_shift,
+ &hubbub_mask);
+
+ for (i = 0; i < res_cap_nv10.num_vmid; i++) {
+ struct dcn20_vmid *vmid = &hubbub->vmid[i];
+
+ vmid->ctx = ctx;
+
+ vmid->regs = &vmid_regs[i];
+ vmid->shifts = &vmid_shifts;
+ vmid->masks = &vmid_masks;
+ }
+
+ return &hubbub->base;
+}
+
+struct timing_generator *dcn20_timing_generator_create(
+ struct dc_context *ctx,
+ uint32_t instance)
+{
+ struct optc *tgn10 =
+ kzalloc(sizeof(struct optc), GFP_KERNEL);
+
+ if (!tgn10)
+ return NULL;
+
+ tgn10->base.inst = instance;
+ tgn10->base.ctx = ctx;
+
+ tgn10->tg_regs = &tg_regs[instance];
+ tgn10->tg_shift = &tg_shift;
+ tgn10->tg_mask = &tg_mask;
+
+ dcn20_timing_generator_init(tgn10);
+
+ return &tgn10->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+ .max_hdmi_deep_color = COLOR_DEPTH_121212,
+ .max_hdmi_pixel_clock = 600000,
+ .hdmi_ycbcr420_supported = true,
+ .dp_ycbcr420_supported = true,
+ .flags.bits.IS_HBR2_CAPABLE = true,
+ .flags.bits.IS_HBR3_CAPABLE = true,
+ .flags.bits.IS_TPS3_CAPABLE = true,
+ .flags.bits.IS_TPS4_CAPABLE = true
+};
+
+struct link_encoder *dcn20_link_encoder_create(
+ const struct encoder_init_data *enc_init_data)
+{
+ struct dcn20_link_encoder *enc20 =
+ kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+
+ if (!enc20)
+ return NULL;
+
+ dcn20_link_encoder_construct(enc20,
+ enc_init_data,
+ &link_enc_feature,
+ &link_enc_regs[enc_init_data->transmitter],
+ &link_enc_aux_regs[enc_init_data->channel - 1],
+ &link_enc_hpd_regs[enc_init_data->hpd_source],
+ &le_shift,
+ &le_mask);
+
+ return &enc20->enc10.base;
+}
+
+struct clock_source *dcn20_clock_source_create(
+ struct dc_context *ctx,
+ struct dc_bios *bios,
+ enum clock_source_id id,
+ const struct dce110_clk_src_regs *regs,
+ bool dp_clk_src)
+{
+ struct dce110_clk_src *clk_src =
+ kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+ if (!clk_src)
+ return NULL;
+
+ if (dcn20_clk_src_construct(clk_src, ctx, bios, id,
+ regs, &cs_shift, &cs_mask)) {
+ clk_src->base.dp_clk_src = dp_clk_src;
+ return &clk_src->base;
+ }
+
+ BREAK_TO_DEBUGGER();
+ return NULL;
+}
+
+static void read_dce_straps(
+ struct dc_context *ctx,
+ struct resource_straps *straps)
+{
+ generic_reg_get(ctx, mmDC_PINSTRAPS + BASE(mmDC_PINSTRAPS_BASE_IDX),
+ FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+}
+
+static struct audio *dcn20_create_audio(
+ struct dc_context *ctx, unsigned int inst)
+{
+ return dce_audio_create(ctx, inst,
+ &audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+struct stream_encoder *dcn20_stream_encoder_create(
+ enum engine_id eng_id,
+ struct dc_context *ctx)
+{
+ struct dcn10_stream_encoder *enc1 =
+ kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
+
+ if (!enc1)
+ return NULL;
+
+ dcn20_stream_encoder_construct(enc1, ctx, ctx->dc_bios, eng_id,
+ &stream_enc_regs[eng_id],
+ &se_shift, &se_mask);
+
+ return &enc1->base;
+}
+
+static const struct dce_hwseq_registers hwseq_reg = {
+ HWSEQ_DCN2_REG_LIST()
+};
+
+static const struct dce_hwseq_shift hwseq_shift = {
+ HWSEQ_DCN2_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+ HWSEQ_DCN2_MASK_SH_LIST(_MASK)
+};
+
+struct dce_hwseq *dcn20_hwseq_create(
+ struct dc_context *ctx)
+{
+ struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+ if (hws) {
+ hws->ctx = ctx;
+ hws->regs = &hwseq_reg;
+ hws->shifts = &hwseq_shift;
+ hws->masks = &hwseq_mask;
+ }
+ return hws;
+}
+
+static const struct resource_create_funcs res_create_funcs = {
+ .read_dce_straps = read_dce_straps,
+ .create_audio = dcn20_create_audio,
+ .create_stream_encoder = dcn20_stream_encoder_create,
+ .create_hwseq = dcn20_hwseq_create,
+};
+
+static const struct resource_create_funcs res_create_maximus_funcs = {
+ .read_dce_straps = NULL,
+ .create_audio = NULL,
+ .create_stream_encoder = NULL,
+ .create_hwseq = dcn20_hwseq_create,
+};
+
+void dcn20_clock_source_destroy(struct clock_source **clk_src)
+{
+ kfree(TO_DCE110_CLK_SRC(*clk_src));
+ *clk_src = NULL;
+}
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+
+struct display_stream_compressor *dcn20_dsc_create(
+ struct dc_context *ctx, uint32_t inst)
+{
+ struct dcn20_dsc *dsc =
+ kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
+
+ if (!dsc) {
+ BREAK_TO_DEBUGGER();
+ return NULL;
+ }
+
+ dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
+ return &dsc->base;
+}
+
+void dcn20_dsc_destroy(struct display_stream_compressor **dsc)
+{
+ kfree(container_of(*dsc, struct dcn20_dsc, base));
+ *dsc = NULL;
+}
+
+#endif
+
+static void destruct(struct dcn20_resource_pool *pool)
+{
+ unsigned int i;
+
+ for (i = 0; i < pool->base.stream_enc_count; i++) {
+ if (pool->base.stream_enc[i] != NULL) {
+ kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+ pool->base.stream_enc[i] = NULL;
+ }
+ }
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+ if (pool->base.dscs[i] != NULL)
+ dcn20_dsc_destroy(&pool->base.dscs[i]);
+ }
+#endif
+
+ if (pool->base.mpc != NULL) {
+ kfree(TO_DCN20_MPC(pool->base.mpc));
+ pool->base.mpc = NULL;
+ }
+ if (pool->base.hubbub != NULL) {
+ kfree(pool->base.hubbub);
+ pool->base.hubbub = NULL;
+ }
+ for (i = 0; i < pool->base.pipe_count; i++) {
+ if (pool->base.dpps[i] != NULL)
+ dcn20_dpp_destroy(&pool->base.dpps[i]);
+
+ if (pool->base.ipps[i] != NULL)
+ pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
+
+ if (pool->base.hubps[i] != NULL) {
+ kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
+ pool->base.hubps[i] = NULL;
+ }
+
+ if (pool->base.irqs != NULL) {
+ dal_irq_service_destroy(&pool->base.irqs);
+ }
+ }
+
+ for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+ if (pool->base.engines[i] != NULL)
+ dce110_engine_destroy(&pool->base.engines[i]);
+ if (pool->base.hw_i2cs[i] != NULL) {
+ kfree(pool->base.hw_i2cs[i]);
+ pool->base.hw_i2cs[i] = NULL;
+ }
+ if (pool->base.sw_i2cs[i] != NULL) {
+ kfree(pool->base.sw_i2cs[i]);
+ pool->base.sw_i2cs[i] = NULL;
+ }
+ }
+
+ for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+ if (pool->base.opps[i] != NULL)
+ pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
+ }
+
+ for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+ if (pool->base.timing_generators[i] != NULL) {
+ kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
+ pool->base.timing_generators[i] = NULL;
+ }
+ }
+
+ for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+ if (pool->base.dwbc[i] != NULL) {
+ kfree(TO_DCN20_DWBC(pool->base.dwbc[i]));
+ pool->base.dwbc[i] = NULL;
+ }
+ if (pool->base.mcif_wb[i] != NULL) {
+ kfree(TO_DCN20_MMHUBBUB(pool->base.mcif_wb[i]));
+ pool->base.mcif_wb[i] = NULL;
+ }
+ }
+
+ for (i = 0; i < pool->base.audio_count; i++) {
+ if (pool->base.audios[i])
+ dce_aud_destroy(&pool->base.audios[i]);
+ }
+
+ for (i = 0; i < pool->base.clk_src_count; i++) {
+ if (pool->base.clock_sources[i] != NULL) {
+ dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
+ pool->base.clock_sources[i] = NULL;
+ }
+ }
+
+ if (pool->base.dp_clock_source != NULL) {
+ dcn20_clock_source_destroy(&pool->base.dp_clock_source);
+ pool->base.dp_clock_source = NULL;
+ }
+
+
+ if (pool->base.abm != NULL)
+ dce_abm_destroy(&pool->base.abm);
+
+ if (pool->base.dmcu != NULL)
+ dce_dmcu_destroy(&pool->base.dmcu);
+
+ if (pool->base.dccg != NULL)
+ dcn_dccg_destroy(&pool->base.dccg);
+
+ if (pool->base.pp_smu != NULL)
+ dcn20_pp_smu_destroy(&pool->base.pp_smu);
+
+}
+
+struct hubp *dcn20_hubp_create(
+ struct dc_context *ctx,
+ uint32_t inst)
+{
+ struct dcn20_hubp *hubp2 =
+ kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
+
+ if (!hubp2)
+ return NULL;
+
+ if (hubp2_construct(hubp2, ctx, inst,
+ &hubp_regs[inst], &hubp_shift, &hubp_mask))
+ return &hubp2->base;
+
+ BREAK_TO_DEBUGGER();
+ kfree(hubp2);
+ return NULL;
+}
+
+static void get_pixel_clock_parameters(
+ struct pipe_ctx *pipe_ctx,
+ struct pixel_clk_params *pixel_clk_params)
+{
+ const struct dc_stream_state *stream = pipe_ctx->stream;
+ bool odm_combine = dc_res_get_odm_bottom_pipe(pipe_ctx) != NULL;
+
+ pixel_clk_params->requested_pix_clk_100hz = stream->timing.pix_clk_100hz;
+ pixel_clk_params->encoder_object_id = stream->link->link_enc->id;
+ pixel_clk_params->signal_type = pipe_ctx->stream->signal;
+ pixel_clk_params->controller_id = pipe_ctx->stream_res.tg->inst + 1;
+ /* TODO: un-hardcode*/
+ pixel_clk_params->requested_sym_clk = LINK_RATE_LOW *
+ LINK_RATE_REF_FREQ_IN_KHZ;
+ pixel_clk_params->flags.ENABLE_SS = 0;
+ pixel_clk_params->color_depth =
+ stream->timing.display_color_depth;
+ pixel_clk_params->flags.DISPLAY_BLANKED = 1;
+ pixel_clk_params->pixel_encoding = stream->timing.pixel_encoding;
+
+ if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR422)
+ pixel_clk_params->color_depth = COLOR_DEPTH_888;
+
+ if (optc1_is_two_pixels_per_containter(&stream->timing) || odm_combine)
+ pixel_clk_params->requested_pix_clk_100hz /= 2;
+
+ if (stream->timing.timing_3d_format == TIMING_3D_FORMAT_HW_FRAME_PACKING)
+ pixel_clk_params->requested_pix_clk_100hz *= 2;
+
+}
+
+static void build_clamping_params(struct dc_stream_state *stream)
+{
+ stream->clamping.clamping_level = CLAMPING_FULL_RANGE;
+ stream->clamping.c_depth = stream->timing.display_color_depth;
+ stream->clamping.pixel_encoding = stream->timing.pixel_encoding;
+}
+
+static enum dc_status build_pipe_hw_param(struct pipe_ctx *pipe_ctx)
+{
+
+ get_pixel_clock_parameters(pipe_ctx, &pipe_ctx->stream_res.pix_clk_params);
+
+ pipe_ctx->clock_source->funcs->get_pix_clk_dividers(
+ pipe_ctx->clock_source,
+ &pipe_ctx->stream_res.pix_clk_params,
+ &pipe_ctx->pll_settings);
+
+ pipe_ctx->stream->clamping.pixel_encoding = pipe_ctx->stream->timing.pixel_encoding;
+
+ resource_build_bit_depth_reduction_params(pipe_ctx->stream,
+ &pipe_ctx->stream->bit_depth_params);
+ build_clamping_params(pipe_ctx->stream);
+
+ return DC_OK;
+}
+
+enum dc_status dcn20_build_mapped_resource(const struct dc *dc, struct dc_state *context, struct dc_stream_state *stream)
+{
+ enum dc_status status = DC_OK;
+ struct pipe_ctx *pipe_ctx = resource_get_head_pipe_for_stream(&context->res_ctx, stream);
+
+ /*TODO Seems unneeded anymore */
+ /* if (old_context && resource_is_stream_unchanged(old_context, stream)) {
+ if (stream != NULL && old_context->streams[i] != NULL) {
+ todo: shouldn't have to copy missing parameter here
+ resource_build_bit_depth_reduction_params(stream,
+ &stream->bit_depth_params);
+ stream->clamping.pixel_encoding =
+ stream->timing.pixel_encoding;
+
+ resource_build_bit_depth_reduction_params(stream,
+ &stream->bit_depth_params);
+ build_clamping_params(stream);
+
+ continue;
+ }
+ }
+ */
+
+ if (!pipe_ctx)
+ return DC_ERROR_UNEXPECTED;
+
+
+ status = build_pipe_hw_param(pipe_ctx);
+
+ return status;
+}
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+
+static void acquire_dsc(struct resource_context *res_ctx,
+ const struct resource_pool *pool,
+ struct display_stream_compressor **dsc)
+{
+ int i;
+
+ ASSERT(*dsc == NULL);
+ *dsc = NULL;
+
+ /* Find first free DSC */
+ for (i = 0; i < pool->res_cap->num_dsc; i++)
+ if (!res_ctx->is_dsc_acquired[i]) {
+ *dsc = pool->dscs[i];
+ res_ctx->is_dsc_acquired[i] = true;
+ break;
+ }
+}
+
+static void release_dsc(struct resource_context *res_ctx,
+ const struct resource_pool *pool,
+ struct display_stream_compressor **dsc)
+{
+ int i;
+
+ for (i = 0; i < pool->res_cap->num_dsc; i++)
+ if (pool->dscs[i] == *dsc) {
+ res_ctx->is_dsc_acquired[i] = false;
+ *dsc = NULL;
+ break;
+ }
+}
+
+#endif
+
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+static enum dc_status add_dsc_to_stream_resource(struct dc *dc,
+ struct dc_state *dc_ctx,
+ struct dc_stream_state *dc_stream)
+{
+ enum dc_status result = DC_OK;
+ int i;
+ const struct resource_pool *pool = dc->res_pool;
+
+ /* Get a DSC if required and available */
+ for (i = 0; i < dc->res_pool->pipe_count; i++) {
+ struct pipe_ctx *pipe_ctx = &dc_ctx->res_ctx.pipe_ctx[i];
+
+ if (pipe_ctx->stream != dc_stream)
+ continue;
+
+ acquire_dsc(&dc_ctx->res_ctx, pool, &pipe_ctx->stream_res.dsc);
+
+ /* The number of DSCs can be less than the number of pipes */
+ if (!pipe_ctx->stream_res.dsc) {
+ dm_output_to_console("No DSCs available\n");
+ result = DC_NO_DSC_RESOURCE;
+ }
+
+ break;
+ }
+
+ return result;
+}
+
+
+static enum dc_status remove_dsc_from_stream_resource(struct dc *dc,
+ struct dc_state *new_ctx,
+ struct dc_stream_state *dc_stream)
+{
+ struct pipe_ctx *pipe_ctx = NULL;
+ int i;
+
+ for (i = 0; i < MAX_PIPES; i++) {
+ if (new_ctx->res_ctx.pipe_ctx[i].stream == dc_stream && !new_ctx->res_ctx.pipe_ctx[i].top_pipe) {
+ pipe_ctx = &new_ctx->res_ctx.pipe_ctx[i];
+ break;
+ }
+ }
+
+ if (!pipe_ctx)
+ return DC_ERROR_UNEXPECTED;
+
+ if (pipe_ctx->stream_res.dsc) {
+ struct pipe_ctx *odm_pipe = dc_res_get_odm_bottom_pipe(pipe_ctx);
+
+ release_dsc(&new_ctx->res_ctx, dc->res_pool, &pipe_ctx->stream_res.dsc);
+ if (odm_pipe)
+ release_dsc(&new_ctx->res_ctx, dc->res_pool, &odm_pipe->stream_res.dsc);
+ }
+
+ return DC_OK;
+}
+#endif
+
+
+enum dc_status dcn20_add_stream_to_ctx(struct dc *dc, struct dc_state *new_ctx, struct dc_stream_state *dc_stream)
+{
+ enum dc_status result = DC_ERROR_UNEXPECTED;
+
+ result = resource_map_pool_resources(dc, new_ctx, dc_stream);
+
+ if (result == DC_OK)
+ result = resource_map_phy_clock_resources(dc, new_ctx, dc_stream);
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ /* Get a DSC if required and available */
+ if (result == DC_OK && dc_stream->timing.flags.DSC)
+ result = add_dsc_to_stream_resource(dc, new_ctx, dc_stream);
+#endif
+
+ if (result == DC_OK)
+ result = dcn20_build_mapped_resource(dc, new_ctx, dc_stream);
+
+ return result;
+}
+
+
+enum dc_status dcn20_remove_stream_from_ctx(struct dc *dc, struct dc_state *new_ctx, struct dc_stream_state *dc_stream)
+{
+ enum dc_status result = DC_OK;
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ result = remove_dsc_from_stream_resource(dc, new_ctx, dc_stream);
+#endif
+
+ return result;
+}
+
+
+static void swizzle_to_dml_params(
+ enum swizzle_mode_values swizzle,
+ unsigned int *sw_mode)
+{
+ switch (swizzle) {
+ case DC_SW_LINEAR:
+ *sw_mode = dm_sw_linear;
+ break;
+ case DC_SW_4KB_S:
+ *sw_mode = dm_sw_4kb_s;
+ break;
+ case DC_SW_4KB_S_X:
+ *sw_mode = dm_sw_4kb_s_x;
+ break;
+ case DC_SW_4KB_D:
+ *sw_mode = dm_sw_4kb_d;
+ break;
+ case DC_SW_4KB_D_X:
+ *sw_mode = dm_sw_4kb_d_x;
+ break;
+ case DC_SW_64KB_S:
+ *sw_mode = dm_sw_64kb_s;
+ break;
+ case DC_SW_64KB_S_X:
+ *sw_mode = dm_sw_64kb_s_x;
+ break;
+ case DC_SW_64KB_S_T:
+ *sw_mode = dm_sw_64kb_s_t;
+ break;
+ case DC_SW_64KB_D:
+ *sw_mode = dm_sw_64kb_d;
+ break;
+ case DC_SW_64KB_D_X:
+ *sw_mode = dm_sw_64kb_d_x;
+ break;
+ case DC_SW_64KB_D_T:
+ *sw_mode = dm_sw_64kb_d_t;
+ break;
+ case DC_SW_64KB_R_X:
+ *sw_mode = dm_sw_64kb_r_x;
+ break;
+ case DC_SW_VAR_S:
+ *sw_mode = dm_sw_var_s;
+ break;
+ case DC_SW_VAR_S_X:
+ *sw_mode = dm_sw_var_s_x;
+ break;
+ case DC_SW_VAR_D:
+ *sw_mode = dm_sw_var_d;
+ break;
+ case DC_SW_VAR_D_X:
+ *sw_mode = dm_sw_var_d_x;
+ break;
+
+ default:
+ ASSERT(0); /* Not supported */
+ break;
+ }
+}
+
+static bool dcn20_split_stream_for_combine(
+ struct resource_context *res_ctx,
+ const struct resource_pool *pool,
+ struct pipe_ctx *primary_pipe,
+ struct pipe_ctx *secondary_pipe,
+ bool is_odm_combine)
+{
+ int pipe_idx = secondary_pipe->pipe_idx;
+ struct scaler_data *sd = &primary_pipe->plane_res.scl_data;
+ struct pipe_ctx *sec_bot_pipe = secondary_pipe->bottom_pipe;
+ int new_width;
+
+ *secondary_pipe = *primary_pipe;
+ secondary_pipe->bottom_pipe = sec_bot_pipe;
+
+ secondary_pipe->pipe_idx = pipe_idx;
+ secondary_pipe->plane_res.mi = pool->mis[secondary_pipe->pipe_idx];
+ secondary_pipe->plane_res.hubp = pool->hubps[secondary_pipe->pipe_idx];
+ secondary_pipe->plane_res.ipp = pool->ipps[secondary_pipe->pipe_idx];
+ secondary_pipe->plane_res.xfm = pool->transforms[secondary_pipe->pipe_idx];
+ secondary_pipe->plane_res.dpp = pool->dpps[secondary_pipe->pipe_idx];
+ secondary_pipe->plane_res.mpcc_inst = pool->dpps[secondary_pipe->pipe_idx]->inst;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ secondary_pipe->stream_res.dsc = NULL;
+#endif
+ if (primary_pipe->bottom_pipe && primary_pipe->bottom_pipe != secondary_pipe) {
+ ASSERT(!secondary_pipe->bottom_pipe);
+ secondary_pipe->bottom_pipe = primary_pipe->bottom_pipe;
+ secondary_pipe->bottom_pipe->top_pipe = secondary_pipe;
+ }
+ primary_pipe->bottom_pipe = secondary_pipe;
+ secondary_pipe->top_pipe = primary_pipe;
+
+ if (is_odm_combine) {
+ if (primary_pipe->plane_state) {
+ /* HACTIVE halved for odm combine */
+ sd->h_active /= 2;
+ /* Copy scl_data to secondary pipe */
+ secondary_pipe->plane_res.scl_data = *sd;
+
+ /* Calculate new vp and recout for left pipe */
+ /* Need at least 16 pixels width per side */
+ if (sd->recout.x + 16 >= sd->h_active)
+ return false;
+ new_width = sd->h_active - sd->recout.x;
+ sd->viewport.width -= dc_fixpt_floor(dc_fixpt_mul_int(
+ sd->ratios.horz, sd->recout.width - new_width));
+ sd->viewport_c.width -= dc_fixpt_floor(dc_fixpt_mul_int(
+ sd->ratios.horz_c, sd->recout.width - new_width));
+ sd->recout.width = new_width;
+
+ /* Calculate new vp and recout for right pipe */
+ sd = &secondary_pipe->plane_res.scl_data;
+ new_width = sd->recout.width + sd->recout.x - sd->h_active;
+ /* Need at least 16 pixels width per side */
+ if (new_width <= 16)
+ return false;
+ sd->viewport.width -= dc_fixpt_floor(dc_fixpt_mul_int(
+ sd->ratios.horz, sd->recout.width - new_width));
+ sd->viewport_c.width -= dc_fixpt_floor(dc_fixpt_mul_int(
+ sd->ratios.horz_c, sd->recout.width - new_width));
+ sd->recout.width = new_width;
+ sd->viewport.x += dc_fixpt_floor(dc_fixpt_mul_int(
+ sd->ratios.horz, sd->h_active - sd->recout.x));
+ sd->viewport_c.x += dc_fixpt_floor(dc_fixpt_mul_int(
+ sd->ratios.horz_c, sd->h_active - sd->recout.x));
+ sd->recout.x = 0;
+ }
+ secondary_pipe->stream_res.opp = pool->opps[secondary_pipe->pipe_idx];
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ if (secondary_pipe->stream->timing.flags.DSC == 1) {
+ acquire_dsc(res_ctx, pool, &secondary_pipe->stream_res.dsc);
+ ASSERT(secondary_pipe->stream_res.dsc);
+ if (secondary_pipe->stream_res.dsc == NULL)
+ return false;
+ }
+#endif
+ } else {
+ ASSERT(primary_pipe->plane_state);
+ resource_build_scaling_params(primary_pipe);
+ resource_build_scaling_params(secondary_pipe);
+ }
+
+ return true;
+}
+
+void dcn20_populate_dml_writeback_from_context(
+ struct dc *dc, struct resource_context *res_ctx, display_e2e_pipe_params_st *pipes)
+{
+ int pipe_cnt, i;
+
+ for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
+ struct dc_writeback_info *wb_info = &res_ctx->pipe_ctx[i].stream->writeback_info[0];
+
+ if (!res_ctx->pipe_ctx[i].stream)
+ continue;
+
+ /* Set writeback information */
+ pipes[pipe_cnt].dout.wb_enable = (wb_info->wb_enabled == true) ? 1 : 0;
+ pipes[pipe_cnt].dout.num_active_wb++;
+ pipes[pipe_cnt].dout.wb.wb_src_height = wb_info->dwb_params.cnv_params.crop_height;
+ pipes[pipe_cnt].dout.wb.wb_src_width = wb_info->dwb_params.cnv_params.crop_width;
+ pipes[pipe_cnt].dout.wb.wb_dst_width = wb_info->dwb_params.dest_width;
+ pipes[pipe_cnt].dout.wb.wb_dst_height = wb_info->dwb_params.dest_height;
+ pipes[pipe_cnt].dout.wb.wb_htaps_luma = 1;
+ pipes[pipe_cnt].dout.wb.wb_vtaps_luma = 1;
+ pipes[pipe_cnt].dout.wb.wb_htaps_chroma = wb_info->dwb_params.scaler_taps.h_taps_c;
+ pipes[pipe_cnt].dout.wb.wb_vtaps_chroma = wb_info->dwb_params.scaler_taps.v_taps_c;
+ pipes[pipe_cnt].dout.wb.wb_hratio = 1.0;
+ pipes[pipe_cnt].dout.wb.wb_vratio = 1.0;
+ if (wb_info->dwb_params.out_format == dwb_scaler_mode_yuv420) {
+ if (wb_info->dwb_params.output_depth == DWB_OUTPUT_PIXEL_DEPTH_8BPC)
+ pipes[pipe_cnt].dout.wb.wb_pixel_format = dm_420_8;
+ else
+ pipes[pipe_cnt].dout.wb.wb_pixel_format = dm_420_10;
+ } else
+ pipes[pipe_cnt].dout.wb.wb_pixel_format = dm_444_32;
+
+ pipe_cnt++;
+ }
+
+}
+
+int dcn20_populate_dml_pipes_from_context(
+ struct dc *dc, struct resource_context *res_ctx, display_e2e_pipe_params_st *pipes)
+{
+ int pipe_cnt, i;
+ bool synchronized_vblank = true;
+
+ for (i = 0, pipe_cnt = -1; i < dc->res_pool->pipe_count; i++) {
+ if (!res_ctx->pipe_ctx[i].stream)
+ continue;
+
+ if (pipe_cnt < 0) {
+ pipe_cnt = i;
+ continue;
+ }
+ if (!resource_are_streams_timing_synchronizable(
+ res_ctx->pipe_ctx[pipe_cnt].stream,
+ res_ctx->pipe_ctx[i].stream)) {
+ synchronized_vblank = false;
+ break;
+ }
+ }
+
+ for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
+ struct dc_crtc_timing *timing = &res_ctx->pipe_ctx[i].stream->timing;
+ int output_bpc;
+
+ if (!res_ctx->pipe_ctx[i].stream)
+ continue;
+ /* todo:
+ pipes[pipe_cnt].pipe.src.dynamic_metadata_enable = 0;
+ pipes[pipe_cnt].pipe.src.dcc = 0;
+ pipes[pipe_cnt].pipe.src.vm = 0;*/
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ pipes[pipe_cnt].dout.dsc_enable = res_ctx->pipe_ctx[i].stream->timing.flags.DSC;
+ /* todo: rotation?*/
+ pipes[pipe_cnt].dout.dsc_slices = res_ctx->pipe_ctx[i].stream->timing.dsc_cfg.num_slices_h;
+#endif
+ if (res_ctx->pipe_ctx[i].stream->use_dynamic_meta) {
+ pipes[pipe_cnt].pipe.src.dynamic_metadata_enable = true;
+ /* 1/2 vblank */
+ pipes[pipe_cnt].pipe.src.dynamic_metadata_lines_before_active =
+ (timing->v_total - timing->v_addressable
+ - timing->v_border_top - timing->v_border_bottom) / 2;
+ /* 36 bytes dp, 32 hdmi */
+ pipes[pipe_cnt].pipe.src.dynamic_metadata_xmit_bytes =
+ dc_is_dp_signal(res_ctx->pipe_ctx[i].stream->signal) ? 36 : 32;
+ }
+ pipes[pipe_cnt].pipe.src.dcc = false;
+ pipes[pipe_cnt].pipe.src.dcc_rate = 1;
+ pipes[pipe_cnt].pipe.dest.synchronized_vblank_all_planes = synchronized_vblank;
+ pipes[pipe_cnt].pipe.dest.hblank_start = timing->h_total - timing->h_front_porch;
+ pipes[pipe_cnt].pipe.dest.hblank_end = pipes[pipe_cnt].pipe.dest.hblank_start
+ - timing->h_addressable
+ - timing->h_border_left
+ - timing->h_border_right;
+ pipes[pipe_cnt].pipe.dest.vblank_start = timing->v_total - timing->v_front_porch;
+ pipes[pipe_cnt].pipe.dest.vblank_end = pipes[pipe_cnt].pipe.dest.vblank_start
+ - timing->v_addressable
+ - timing->v_border_top
+ - timing->v_border_bottom;
+ pipes[pipe_cnt].pipe.dest.htotal = timing->h_total;
+ pipes[pipe_cnt].pipe.dest.vtotal = timing->v_total;
+ pipes[pipe_cnt].pipe.dest.hactive = timing->h_addressable;
+ pipes[pipe_cnt].pipe.dest.vactive = timing->v_addressable;
+ pipes[pipe_cnt].pipe.dest.interlaced = timing->flags.INTERLACE;
+ pipes[pipe_cnt].pipe.dest.pixel_rate_mhz = timing->pix_clk_100hz/10000.0;
+ if (timing->timing_3d_format == TIMING_3D_FORMAT_HW_FRAME_PACKING)
+ pipes[pipe_cnt].pipe.dest.pixel_rate_mhz *= 2;
+ pipes[pipe_cnt].pipe.dest.otg_inst = res_ctx->pipe_ctx[i].stream_res.tg->inst;
+ pipes[pipe_cnt].dout.dp_lanes = 4;
+ pipes[pipe_cnt].pipe.dest.vtotal_min = res_ctx->pipe_ctx[i].stream->adjust.v_total_min;
+ pipes[pipe_cnt].pipe.dest.vtotal_max = res_ctx->pipe_ctx[i].stream->adjust.v_total_max;
+
+ switch (res_ctx->pipe_ctx[i].stream->signal) {
+ case SIGNAL_TYPE_DISPLAY_PORT_MST:
+ case SIGNAL_TYPE_DISPLAY_PORT:
+ pipes[pipe_cnt].dout.output_type = dm_dp;
+ break;
+ case SIGNAL_TYPE_EDP:
+ pipes[pipe_cnt].dout.output_type = dm_edp;
+ break;
+ case SIGNAL_TYPE_HDMI_TYPE_A:
+ case SIGNAL_TYPE_DVI_SINGLE_LINK:
+ case SIGNAL_TYPE_DVI_DUAL_LINK:
+ pipes[pipe_cnt].dout.output_type = dm_hdmi;
+ break;
+ default:
+ /* In case there is no signal, set dp with 4 lanes to allow max config */
+ pipes[pipe_cnt].dout.output_type = dm_dp;
+ pipes[pipe_cnt].dout.dp_lanes = 4;
+ }
+
+ switch (res_ctx->pipe_ctx[i].stream->timing.display_color_depth) {
+ case COLOR_DEPTH_666:
+ output_bpc = 6;
+ break;
+ case COLOR_DEPTH_888:
+ output_bpc = 8;
+ break;
+ case COLOR_DEPTH_101010:
+ output_bpc = 10;
+ break;
+ case COLOR_DEPTH_121212:
+ output_bpc = 12;
+ break;
+ case COLOR_DEPTH_141414:
+ output_bpc = 14;
+ break;
+ case COLOR_DEPTH_161616:
+ output_bpc = 16;
+ break;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ case COLOR_DEPTH_999:
+ output_bpc = 9;
+ break;
+ case COLOR_DEPTH_111111:
+ output_bpc = 11;
+ break;
+#endif
+ default:
+ output_bpc = 8;
+ break;
+ }
+
+
+ switch (res_ctx->pipe_ctx[i].stream->timing.pixel_encoding) {
+ case PIXEL_ENCODING_RGB:
+ case PIXEL_ENCODING_YCBCR444:
+ pipes[pipe_cnt].dout.output_format = dm_444;
+ pipes[pipe_cnt].dout.output_bpp = output_bpc * 3;
+ break;
+ case PIXEL_ENCODING_YCBCR420:
+ pipes[pipe_cnt].dout.output_format = dm_420;
+ pipes[pipe_cnt].dout.output_bpp = (output_bpc * 3) / 2;
+ break;
+ case PIXEL_ENCODING_YCBCR422:
+ if (true) /* todo */
+ pipes[pipe_cnt].dout.output_format = dm_s422;
+ else
+ pipes[pipe_cnt].dout.output_format = dm_n422;
+ pipes[pipe_cnt].dout.output_bpp = output_bpc * 2;
+ break;
+ default:
+ pipes[pipe_cnt].dout.output_format = dm_444;
+ pipes[pipe_cnt].dout.output_bpp = output_bpc * 3;
+ }
+ pipes[pipe_cnt].pipe.src.hsplit_grp = res_ctx->pipe_ctx[i].pipe_idx;
+ if (res_ctx->pipe_ctx[i].top_pipe && res_ctx->pipe_ctx[i].top_pipe->plane_state
+ == res_ctx->pipe_ctx[i].plane_state)
+ pipes[pipe_cnt].pipe.src.hsplit_grp = res_ctx->pipe_ctx[i].top_pipe->pipe_idx;
+
+ /* todo: default max for now, until there is logic reflecting this in dc*/
+ pipes[pipe_cnt].dout.output_bpc = 12;
+ /*
+ * Use max cursor settings for calculations to minimize
+ * bw calculations due to cursor on/off
+ */
+ pipes[pipe_cnt].pipe.src.num_cursors = 2;
+ pipes[pipe_cnt].pipe.src.cur0_src_width = 256;
+ pipes[pipe_cnt].pipe.src.cur0_bpp = dm_cur_32bit;
+ pipes[pipe_cnt].pipe.src.cur1_src_width = 256;
+ pipes[pipe_cnt].pipe.src.cur1_bpp = dm_cur_32bit;
+
+ if (!res_ctx->pipe_ctx[i].plane_state) {
+ pipes[pipe_cnt].pipe.src.source_scan = dm_horz;
+ pipes[pipe_cnt].pipe.src.sw_mode = dm_sw_linear;
+ pipes[pipe_cnt].pipe.src.macro_tile_size = dm_64k_tile;
+ pipes[pipe_cnt].pipe.src.viewport_width = timing->h_addressable;
+ if (pipes[pipe_cnt].pipe.src.viewport_width > 1920)
+ pipes[pipe_cnt].pipe.src.viewport_width = 1920;
+ pipes[pipe_cnt].pipe.src.viewport_height = timing->v_addressable;
+ if (pipes[pipe_cnt].pipe.src.viewport_height > 1080)
+ pipes[pipe_cnt].pipe.src.viewport_height = 1080;
+ pipes[pipe_cnt].pipe.src.data_pitch = ((pipes[pipe_cnt].pipe.src.viewport_width + 63) / 64) * 64; /* linear sw only */
+ pipes[pipe_cnt].pipe.src.source_format = dm_444_32;
+ pipes[pipe_cnt].pipe.dest.recout_width = pipes[pipe_cnt].pipe.src.viewport_width; /*vp_width/hratio*/
+ pipes[pipe_cnt].pipe.dest.recout_height = pipes[pipe_cnt].pipe.src.viewport_height; /*vp_height/vratio*/
+ pipes[pipe_cnt].pipe.dest.full_recout_width = pipes[pipe_cnt].pipe.dest.recout_width; /*when is_hsplit != 1*/
+ pipes[pipe_cnt].pipe.dest.full_recout_height = pipes[pipe_cnt].pipe.dest.recout_height; /*when is_hsplit != 1*/
+ pipes[pipe_cnt].pipe.scale_ratio_depth.lb_depth = dm_lb_16;
+ pipes[pipe_cnt].pipe.scale_ratio_depth.hscl_ratio = 1.0;
+ pipes[pipe_cnt].pipe.scale_ratio_depth.vscl_ratio = 1.0;
+ pipes[pipe_cnt].pipe.scale_ratio_depth.scl_enable = 0; /*Lb only or Full scl*/
+ pipes[pipe_cnt].pipe.scale_taps.htaps = 1;
+ pipes[pipe_cnt].pipe.scale_taps.vtaps = 1;
+ pipes[pipe_cnt].pipe.src.is_hsplit = 0;
+ pipes[pipe_cnt].pipe.dest.odm_combine = 0;
+ pipes[pipe_cnt].pipe.dest.vtotal_min = timing->v_total;
+ pipes[pipe_cnt].pipe.dest.vtotal_max = timing->v_total;
+ } else {
+ struct dc_plane_state *pln = res_ctx->pipe_ctx[i].plane_state;
+ struct scaler_data *scl = &res_ctx->pipe_ctx[i].plane_res.scl_data;
+
+ pipes[pipe_cnt].pipe.src.immediate_flip = pln->flip_immediate;
+ pipes[pipe_cnt].pipe.src.is_hsplit = (res_ctx->pipe_ctx[i].bottom_pipe
+ && res_ctx->pipe_ctx[i].bottom_pipe->plane_state == pln)
+ || (res_ctx->pipe_ctx[i].top_pipe
+ && res_ctx->pipe_ctx[i].top_pipe->plane_state == pln);
+ pipes[pipe_cnt].pipe.dest.odm_combine = (res_ctx->pipe_ctx[i].bottom_pipe
+ && res_ctx->pipe_ctx[i].bottom_pipe->plane_state == pln
+ && res_ctx->pipe_ctx[i].bottom_pipe->stream_res.opp
+ != res_ctx->pipe_ctx[i].stream_res.opp)
+ || (res_ctx->pipe_ctx[i].top_pipe
+ && res_ctx->pipe_ctx[i].top_pipe->plane_state == pln
+ && res_ctx->pipe_ctx[i].top_pipe->stream_res.opp
+ != res_ctx->pipe_ctx[i].stream_res.opp);
+ pipes[pipe_cnt].pipe.src.source_scan = pln->rotation == ROTATION_ANGLE_90
+ || pln->rotation == ROTATION_ANGLE_270 ? dm_vert : dm_horz;
+ pipes[pipe_cnt].pipe.src.viewport_y_y = scl->viewport.y;
+ pipes[pipe_cnt].pipe.src.viewport_y_c = scl->viewport_c.y;
+ pipes[pipe_cnt].pipe.src.viewport_width = scl->viewport.width;
+ pipes[pipe_cnt].pipe.src.viewport_width_c = scl->viewport_c.width;
+ pipes[pipe_cnt].pipe.src.viewport_height = scl->viewport.height;
+ pipes[pipe_cnt].pipe.src.viewport_height_c = scl->viewport_c.height;
+ if (pln->format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) {
+ pipes[pipe_cnt].pipe.src.data_pitch = pln->plane_size.video.luma_pitch;
+ pipes[pipe_cnt].pipe.src.data_pitch_c = pln->plane_size.video.chroma_pitch;
+ pipes[pipe_cnt].pipe.src.meta_pitch = pln->dcc.video.meta_pitch_l;
+ pipes[pipe_cnt].pipe.src.meta_pitch_c = pln->dcc.video.meta_pitch_c;
+ } else {
+ pipes[pipe_cnt].pipe.src.data_pitch = pln->plane_size.grph.surface_pitch;
+ pipes[pipe_cnt].pipe.src.meta_pitch = pln->dcc.grph.meta_pitch;
+ }
+ pipes[pipe_cnt].pipe.src.dcc = pln->dcc.enable;
+ pipes[pipe_cnt].pipe.dest.recout_width = scl->recout.width;
+ pipes[pipe_cnt].pipe.dest.recout_height = scl->recout.height;
+ pipes[pipe_cnt].pipe.dest.full_recout_width = scl->recout.width;
+ pipes[pipe_cnt].pipe.dest.full_recout_height = scl->recout.height;
+ if (res_ctx->pipe_ctx[i].bottom_pipe && res_ctx->pipe_ctx[i].bottom_pipe->plane_state == pln) {
+ pipes[pipe_cnt].pipe.dest.full_recout_width +=
+ res_ctx->pipe_ctx[i].bottom_pipe->plane_res.scl_data.recout.width;
+ pipes[pipe_cnt].pipe.dest.full_recout_height +=
+ res_ctx->pipe_ctx[i].bottom_pipe->plane_res.scl_data.recout.height;
+ } else if (res_ctx->pipe_ctx[i].top_pipe && res_ctx->pipe_ctx[i].top_pipe->plane_state == pln) {
+ pipes[pipe_cnt].pipe.dest.full_recout_width +=
+ res_ctx->pipe_ctx[i].top_pipe->plane_res.scl_data.recout.width;
+ pipes[pipe_cnt].pipe.dest.full_recout_height +=
+ res_ctx->pipe_ctx[i].top_pipe->plane_res.scl_data.recout.height;
+ }
+
+ pipes[pipe_cnt].pipe.scale_ratio_depth.lb_depth = dm_lb_16;
+ pipes[pipe_cnt].pipe.scale_ratio_depth.hscl_ratio = (double) scl->ratios.horz.value / (1ULL<<32);
+ pipes[pipe_cnt].pipe.scale_ratio_depth.hscl_ratio_c = (double) scl->ratios.horz_c.value / (1ULL<<32);
+ pipes[pipe_cnt].pipe.scale_ratio_depth.vscl_ratio = (double) scl->ratios.vert.value / (1ULL<<32);
+ pipes[pipe_cnt].pipe.scale_ratio_depth.vscl_ratio_c = (double) scl->ratios.vert_c.value / (1ULL<<32);
+ pipes[pipe_cnt].pipe.scale_ratio_depth.scl_enable =
+ scl->ratios.vert.value != dc_fixpt_one.value
+ || scl->ratios.horz.value != dc_fixpt_one.value
+ || scl->ratios.vert_c.value != dc_fixpt_one.value
+ || scl->ratios.horz_c.value != dc_fixpt_one.value /*Lb only or Full scl*/
+ || dc->debug.always_scale; /*support always scale*/
+ pipes[pipe_cnt].pipe.scale_taps.htaps = scl->taps.h_taps;
+ pipes[pipe_cnt].pipe.scale_taps.htaps_c = scl->taps.h_taps_c;
+ pipes[pipe_cnt].pipe.scale_taps.vtaps = scl->taps.v_taps;
+ pipes[pipe_cnt].pipe.scale_taps.vtaps_c = scl->taps.v_taps_c;
+
+ pipes[pipe_cnt].pipe.src.macro_tile_size =
+ swizzle_mode_to_macro_tile_size(pln->tiling_info.gfx9.swizzle);
+ swizzle_to_dml_params(pln->tiling_info.gfx9.swizzle,
+ &pipes[pipe_cnt].pipe.src.sw_mode);
+
+ switch (pln->format) {
+ case SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr:
+ case SURFACE_PIXEL_FORMAT_VIDEO_420_YCrCb:
+ pipes[pipe_cnt].pipe.src.source_format = dm_420_8;
+ break;
+ case SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCbCr:
+ case SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb:
+ pipes[pipe_cnt].pipe.src.source_format = dm_420_10;
+ break;
+ case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616:
+ case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616F:
+ case SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F:
+ pipes[pipe_cnt].pipe.src.source_format = dm_444_64;
+ break;
+ case SURFACE_PIXEL_FORMAT_GRPH_ARGB1555:
+ case SURFACE_PIXEL_FORMAT_GRPH_RGB565:
+ pipes[pipe_cnt].pipe.src.source_format = dm_444_16;
+ break;
+ case SURFACE_PIXEL_FORMAT_GRPH_PALETA_256_COLORS:
+ pipes[pipe_cnt].pipe.src.source_format = dm_444_8;
+ break;
+ default:
+ pipes[pipe_cnt].pipe.src.source_format = dm_444_32;
+ break;
+ }
+ }
+
+ pipe_cnt++;
+ }
+
+ /* populate writeback information */
+ dc->res_pool->funcs->populate_dml_writeback_from_context(dc, res_ctx, pipes);
+
+ return pipe_cnt;
+}
+
+unsigned int dcn20_calc_max_scaled_time(
+ unsigned int time_per_pixel,
+ enum mmhubbub_wbif_mode mode,
+ unsigned int urgent_watermark)
+{
+ unsigned int time_per_byte = 0;
+ unsigned int total_y_free_entry = 0x200; /* two memory piece for luma */
+ unsigned int total_c_free_entry = 0x140; /* two memory piece for chroma */
+ unsigned int small_free_entry, max_free_entry;
+ unsigned int buf_lh_capability;
+ unsigned int max_scaled_time;
+
+ if (mode == PACKED_444) /* packed mode */
+ time_per_byte = time_per_pixel/4;
+ else if (mode == PLANAR_420_8BPC)
+ time_per_byte = time_per_pixel;
+ else if (mode == PLANAR_420_10BPC) /* p010 */
+ time_per_byte = time_per_pixel * 819/1024;
+
+ if (time_per_byte == 0)
+ time_per_byte = 1;
+
+ small_free_entry = (total_y_free_entry > total_c_free_entry) ? total_c_free_entry : total_y_free_entry;
+ max_free_entry = (mode == PACKED_444) ? total_y_free_entry + total_c_free_entry : small_free_entry;
+ buf_lh_capability = max_free_entry*time_per_byte*32/16; /* there is 4bit fraction */
+ max_scaled_time = buf_lh_capability - urgent_watermark;
+ return max_scaled_time;
+}
+
+void dcn20_set_mcif_arb_params(
+ struct dc *dc,
+ struct dc_state *context,
+ display_e2e_pipe_params_st *pipes,
+ int pipe_cnt)
+{
+ enum mmhubbub_wbif_mode wbif_mode;
+ struct mcif_arb_params *wb_arb_params;
+ int i, j, k, dwb_pipe;
+
+ /* Writeback MCIF_WB arbitration parameters */
+ dwb_pipe = 0;
+ for (i = 0; i < dc->res_pool->pipe_count; i++) {
+
+ if (!context->res_ctx.pipe_ctx[i].stream)
+ continue;
+
+ for (j = 0; j < MAX_DWB_PIPES; j++) {
+ if (context->res_ctx.pipe_ctx[i].stream->writeback_info[j].wb_enabled == false)
+ continue;
+
+ //wb_arb_params = &context->res_ctx.pipe_ctx[i].stream->writeback_info[j].mcif_arb_params;
+ wb_arb_params = &context->bw_ctx.bw.dcn.bw_writeback.mcif_wb_arb[dwb_pipe];
+
+ if (context->res_ctx.pipe_ctx[i].stream->writeback_info[j].dwb_params.out_format == dwb_scaler_mode_yuv420) {
+ if (context->res_ctx.pipe_ctx[i].stream->writeback_info[j].dwb_params.output_depth == DWB_OUTPUT_PIXEL_DEPTH_8BPC)
+ wbif_mode = PLANAR_420_8BPC;
+ else
+ wbif_mode = PLANAR_420_10BPC;
+ } else
+ wbif_mode = PACKED_444;
+
+ for (k = 0; k < sizeof(wb_arb_params->cli_watermark)/sizeof(wb_arb_params->cli_watermark[0]); k++) {
+ wb_arb_params->cli_watermark[k] = get_wm_writeback_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ wb_arb_params->pstate_watermark[k] = get_wm_writeback_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ }
+ wb_arb_params->time_per_pixel = 16.0 / context->res_ctx.pipe_ctx[i].stream->phy_pix_clk; /* 4 bit fraction, ms */
+ wb_arb_params->slice_lines = 32;
+ wb_arb_params->arbitration_slice = 2;
+ wb_arb_params->max_scaled_time = dcn20_calc_max_scaled_time(wb_arb_params->time_per_pixel,
+ wbif_mode,
+ wb_arb_params->cli_watermark[0]); /* assume 4 watermark sets have the same value */
+
+ dwb_pipe++;
+
+ if (dwb_pipe >= MAX_DWB_PIPES)
+ return;
+ }
+ if (dwb_pipe >= MAX_DWB_PIPES)
+ return;
+ }
+}
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+static bool dcn20_validate_dsc(struct dc *dc, struct dc_state *new_ctx)
+{
+ int i;
+
+ /* Validate DSC config, dsc count validation is already done */
+ for (i = 0; i < dc->res_pool->pipe_count; i++) {
+ struct pipe_ctx *pipe_ctx = &new_ctx->res_ctx.pipe_ctx[i];
+ struct dc_stream_state *stream = pipe_ctx->stream;
+ struct dsc_config dsc_cfg;
+
+ /* Only need to validate top pipe */
+ if (pipe_ctx->top_pipe || !stream || !stream->timing.flags.DSC)
+ continue;
+
+ dsc_cfg.pic_width = stream->timing.h_addressable + stream->timing.h_border_left
+ + stream->timing.h_border_right;
+ dsc_cfg.pic_height = stream->timing.v_addressable + stream->timing.v_border_top
+ + stream->timing.v_border_bottom;
+ if (dc_res_get_odm_bottom_pipe(pipe_ctx))
+ dsc_cfg.pic_width /= 2;
+ dsc_cfg.pixel_encoding = stream->timing.pixel_encoding;
+ dsc_cfg.color_depth = stream->timing.display_color_depth;
+ dsc_cfg.dc_dsc_cfg = stream->timing.dsc_cfg;
+
+ if (!pipe_ctx->stream_res.dsc->funcs->dsc_validate_stream(pipe_ctx->stream_res.dsc, &dsc_cfg))
+ return false;
+ }
+ return true;
+}
+#endif
+
+bool dcn20_validate_bandwidth(struct dc *dc, struct dc_state *context,
+ bool fast_validate)
+{
+ bool out = false;
+
+ BW_VAL_TRACE_SETUP();
+
+ int pipe_cnt, i, pipe_idx, vlevel, vlevel_unsplit;
+ int pipe_split_from[MAX_PIPES];
+ bool odm_capable = context->bw_ctx.dml.ip.odm_capable;
+ bool force_split = false;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ bool failed_non_odm_dsc = false;
+#endif
+ int split_threshold = dc->res_pool->pipe_count / 2;
+ bool avoid_split = dc->debug.pipe_split_policy != MPC_SPLIT_DYNAMIC;
+ display_e2e_pipe_params_st *pipes = kzalloc(dc->res_pool->pipe_count * sizeof(display_e2e_pipe_params_st), GFP_KERNEL);
+ DC_LOGGER_INIT(dc->ctx->logger);
+
+ BW_VAL_TRACE_COUNT();
+
+ ASSERT(pipes);
+ if (!pipes)
+ return false;
+
+ for (i = 0; i < dc->res_pool->pipe_count; i++) {
+ struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+ struct pipe_ctx *hsplit_pipe = pipe->bottom_pipe;
+
+ if (!hsplit_pipe || hsplit_pipe->plane_state != pipe->plane_state)
+ continue;
+
+ /* merge previously split pipe since mode support needs to make the decision */
+ pipe->bottom_pipe = hsplit_pipe->bottom_pipe;
+ if (hsplit_pipe->bottom_pipe)
+ hsplit_pipe->bottom_pipe->top_pipe = pipe;
+ hsplit_pipe->plane_state = NULL;
+ hsplit_pipe->stream = NULL;
+ hsplit_pipe->top_pipe = NULL;
+ hsplit_pipe->bottom_pipe = NULL;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ if (hsplit_pipe->stream_res.dsc && hsplit_pipe->stream_res.dsc != pipe->stream_res.dsc)
+ release_dsc(&context->res_ctx, dc->res_pool, &hsplit_pipe->stream_res.dsc);
+#endif
+ /* Clear plane_res and stream_res */
+ memset(&hsplit_pipe->plane_res, 0, sizeof(hsplit_pipe->plane_res));
+ memset(&hsplit_pipe->stream_res, 0, sizeof(hsplit_pipe->stream_res));
+ if (pipe->plane_state)
+ resource_build_scaling_params(pipe);
+ }
+
+ if (dc->res_pool->funcs->populate_dml_pipes)
+ pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc,
+ &context->res_ctx, pipes);
+ else
+ pipe_cnt = dcn20_populate_dml_pipes_from_context(dc,
+ &context->res_ctx, pipes);
+
+ if (!pipe_cnt) {
+ BW_VAL_TRACE_SKIP(pass);
+ out = true;
+ goto validate_out;
+ }
+
+ context->bw_ctx.dml.ip.odm_capable = 0;
+
+ vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
+
+ context->bw_ctx.dml.ip.odm_capable = odm_capable;
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ /* 1 dsc per stream dsc validation */
+ if (vlevel <= context->bw_ctx.dml.soc.num_states)
+ if (!dcn20_validate_dsc(dc, context)) {
+ failed_non_odm_dsc = true;
+ vlevel = context->bw_ctx.dml.soc.num_states + 1;
+ }
+#endif
+
+ if (vlevel > context->bw_ctx.dml.soc.num_states && odm_capable)
+ vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
+
+ if (vlevel > context->bw_ctx.dml.soc.num_states)
+ goto validate_fail;
+
+ if ((context->stream_count > split_threshold && dc->current_state->stream_count <= split_threshold)
+ || (context->stream_count <= split_threshold && dc->current_state->stream_count > split_threshold))
+ context->commit_hints.full_update_needed = true;
+
+ /*initialize pipe_just_split_from to invalid idx*/
+ for (i = 0; i < MAX_PIPES; i++)
+ pipe_split_from[i] = -1;
+
+ /* Single display only conditionals get set here */
+ for (i = 0; i < dc->res_pool->pipe_count; i++) {
+ struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+ bool exit_loop = false;
+
+ if (!pipe->stream || pipe->top_pipe)
+ continue;
+
+ if (dc->debug.force_single_disp_pipe_split) {
+ if (!force_split)
+ force_split = true;
+ else {
+ force_split = false;
+ exit_loop = true;
+ }
+ }
+ if (dc->debug.pipe_split_policy == MPC_SPLIT_AVOID_MULT_DISP) {
+ if (avoid_split)
+ avoid_split = false;
+ else {
+ avoid_split = true;
+ exit_loop = true;
+ }
+ }
+ if (exit_loop)
+ break;
+ }
+
+ if (context->stream_count > split_threshold)
+ avoid_split = true;
+
+ vlevel_unsplit = vlevel;
+ for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
+ if (!context->res_ctx.pipe_ctx[i].stream)
+ continue;
+ for (; vlevel_unsplit <= context->bw_ctx.dml.soc.num_states; vlevel_unsplit++)
+ if (context->bw_ctx.dml.vba.NoOfDPP[vlevel_unsplit][0][pipe_idx] == 1)
+ break;
+ pipe_idx++;
+ }
+
+ for (i = 0, pipe_idx = -1; i < dc->res_pool->pipe_count; i++) {
+ struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+ struct pipe_ctx *hsplit_pipe = pipe->bottom_pipe;
+ bool need_split = true;
+ bool need_split3d;
+
+ if (!pipe->stream || pipe_split_from[i] >= 0)
+ continue;
+
+ pipe_idx++;
+
+ if (dc->debug.force_odm_combine & (1 << pipe->stream_res.tg->inst)) {
+ force_split = true;
+ context->bw_ctx.dml.vba.ODMCombineEnabled[pipe_idx] = true;
+ context->bw_ctx.dml.vba.ODMCombineEnablePerState[vlevel][pipe_idx] = true;
+ }
+ if (force_split && context->bw_ctx.dml.vba.NoOfDPP[vlevel][context->bw_ctx.dml.vba.maxMpcComb][pipe_idx] == 1)
+ context->bw_ctx.dml.vba.RequiredDPPCLK[vlevel][context->bw_ctx.dml.vba.maxMpcComb][pipe_idx] /= 2;
+ if (dc->config.forced_clocks == true) {
+ context->bw_ctx.dml.vba.RequiredDPPCLK[vlevel][context->bw_ctx.dml.vba.maxMpcComb][pipe_idx] =
+ context->bw_ctx.dml.soc.clock_limits[0].dppclk_mhz;
+ }
+ if (!pipe->top_pipe && !pipe->plane_state && context->bw_ctx.dml.vba.ODMCombineEnabled[pipe_idx]) {
+ hsplit_pipe = find_idle_secondary_pipe(&context->res_ctx, dc->res_pool, pipe);
+ ASSERT(hsplit_pipe);
+ if (!dcn20_split_stream_for_combine(
+ &context->res_ctx, dc->res_pool,
+ pipe, hsplit_pipe,
+ true))
+ goto validate_fail;
+ pipe_split_from[hsplit_pipe->pipe_idx] = pipe_idx;
+ dcn20_build_mapped_resource(dc, context, pipe->stream);
+ }
+
+ if (!pipe->plane_state)
+ continue;
+ /* Skip 2nd half of already split pipe */
+ if (pipe->top_pipe && pipe->plane_state == pipe->top_pipe->plane_state)
+ continue;
+
+ need_split3d = ((pipe->stream->view_format ==
+ VIEW_3D_FORMAT_SIDE_BY_SIDE ||
+ pipe->stream->view_format ==
+ VIEW_3D_FORMAT_TOP_AND_BOTTOM) &&
+ (pipe->stream->timing.timing_3d_format ==
+ TIMING_3D_FORMAT_TOP_AND_BOTTOM ||
+ pipe->stream->timing.timing_3d_format ==
+ TIMING_3D_FORMAT_SIDE_BY_SIDE));
+
+ if (avoid_split && vlevel_unsplit <= context->bw_ctx.dml.soc.num_states && !force_split && !need_split3d) {
+ need_split = false;
+ vlevel = vlevel_unsplit;
+ context->bw_ctx.dml.vba.maxMpcComb = 0;
+ } else
+ need_split = context->bw_ctx.dml.vba.NoOfDPP[vlevel][context->bw_ctx.dml.vba.maxMpcComb][pipe_idx] == 2;
+
+ /* We do not support mpo + odm at the moment */
+ if (hsplit_pipe && hsplit_pipe->plane_state != pipe->plane_state
+ && context->bw_ctx.dml.vba.ODMCombineEnabled[pipe_idx])
+ goto validate_fail;
+
+ if (need_split3d || need_split || force_split) {
+ if (!hsplit_pipe || hsplit_pipe->plane_state != pipe->plane_state) {
+ /* pipe not split previously needs split */
+ hsplit_pipe = find_idle_secondary_pipe(&context->res_ctx, dc->res_pool, pipe);
+ ASSERT(hsplit_pipe || force_split);
+ if (!hsplit_pipe)
+ continue;
+
+ if (!dcn20_split_stream_for_combine(
+ &context->res_ctx, dc->res_pool,
+ pipe, hsplit_pipe,
+ context->bw_ctx.dml.vba.ODMCombineEnabled[pipe_idx]))
+ goto validate_fail;
+ pipe_split_from[hsplit_pipe->pipe_idx] = pipe_idx;
+ }
+ } else if (hsplit_pipe && hsplit_pipe->plane_state == pipe->plane_state) {
+ /* merge should already have been done */
+ ASSERT(0);
+ }
+ }
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ /* Actual dsc count per stream dsc validation*/
+ if (failed_non_odm_dsc && !dcn20_validate_dsc(dc, context)) {
+ context->bw_ctx.dml.vba.ValidationStatus[context->bw_ctx.dml.vba.soc.num_states] =
+ DML_FAIL_DSC_VALIDATION_FAILURE;
+ goto validate_fail;
+ }
+#endif
+
+ BW_VAL_TRACE_END_VOLTAGE_LEVEL();
+
+ if (fast_validate) {
+ BW_VAL_TRACE_SKIP(fast);
+ out = true;
+ goto validate_out;
+ }
+
+ for (i = 0, pipe_idx = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
+ if (!context->res_ctx.pipe_ctx[i].stream)
+ continue;
+
+ pipes[pipe_cnt].clks_cfg.refclk_mhz = dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000.0;
+ pipes[pipe_cnt].clks_cfg.dispclk_mhz = context->bw_ctx.dml.vba.RequiredDISPCLK[vlevel][context->bw_ctx.dml.vba.maxMpcComb];
+
+ if (pipe_split_from[i] < 0) {
+ pipes[pipe_cnt].clks_cfg.dppclk_mhz =
+ context->bw_ctx.dml.vba.RequiredDPPCLK[vlevel][context->bw_ctx.dml.vba.maxMpcComb][pipe_idx];
+ if (context->bw_ctx.dml.vba.BlendingAndTiming[pipe_idx] == pipe_idx)
+ pipes[pipe_cnt].pipe.dest.odm_combine =
+ context->bw_ctx.dml.vba.ODMCombineEnablePerState[vlevel][pipe_idx];
+ else
+ pipes[pipe_cnt].pipe.dest.odm_combine = 0;
+ pipe_idx++;
+ } else {
+ pipes[pipe_cnt].clks_cfg.dppclk_mhz =
+ context->bw_ctx.dml.vba.RequiredDPPCLK[vlevel][context->bw_ctx.dml.vba.maxMpcComb][pipe_split_from[i]];
+ if (context->bw_ctx.dml.vba.BlendingAndTiming[pipe_split_from[i]] == pipe_split_from[i])
+ pipes[pipe_cnt].pipe.dest.odm_combine =
+ context->bw_ctx.dml.vba.ODMCombineEnablePerState[vlevel][pipe_split_from[i]];
+ else
+ pipes[pipe_cnt].pipe.dest.odm_combine = 0;
+ }
+ if (dc->config.forced_clocks) {
+ pipes[pipe_cnt].clks_cfg.dispclk_mhz = context->bw_ctx.dml.soc.clock_limits[0].dispclk_mhz;
+ pipes[pipe_cnt].clks_cfg.dppclk_mhz = context->bw_ctx.dml.soc.clock_limits[0].dppclk_mhz;
+ }
+ pipe_cnt++;
+ }
+
+ if (pipe_cnt != pipe_idx) {
+ if (dc->res_pool->funcs->populate_dml_pipes)
+ pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc,
+ &context->res_ctx, pipes);
+ else
+ pipe_cnt = dcn20_populate_dml_pipes_from_context(dc,
+ &context->res_ctx, pipes);
+ }
+
+ pipes[0].clks_cfg.voltage = vlevel;
+ pipes[0].clks_cfg.dcfclk_mhz = context->bw_ctx.dml.soc.clock_limits[vlevel].dcfclk_mhz;
+ pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[vlevel].socclk_mhz;
+
+ /* only pipe 0 is read for voltage and dcf/soc clocks */
+ if (vlevel < 1) {
+ pipes[0].clks_cfg.voltage = 1;
+ pipes[0].clks_cfg.dcfclk_mhz = context->bw_ctx.dml.soc.clock_limits[1].dcfclk_mhz;
+ pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[1].socclk_mhz;
+ }
+ context->bw_ctx.bw.dcn.watermarks.b.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ context->bw_ctx.bw.dcn.watermarks.b.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+
+ if (vlevel < 2) {
+ pipes[0].clks_cfg.voltage = 2;
+ pipes[0].clks_cfg.dcfclk_mhz = context->bw_ctx.dml.soc.clock_limits[2].dcfclk_mhz;
+ pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[2].socclk_mhz;
+ }
+ context->bw_ctx.bw.dcn.watermarks.c.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ context->bw_ctx.bw.dcn.watermarks.c.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+
+ if (vlevel < 3) {
+ pipes[0].clks_cfg.voltage = 3;
+ pipes[0].clks_cfg.dcfclk_mhz = context->bw_ctx.dml.soc.clock_limits[2].dcfclk_mhz;
+ pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[2].socclk_mhz;
+ }
+ context->bw_ctx.bw.dcn.watermarks.d.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ context->bw_ctx.bw.dcn.watermarks.d.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+
+ pipes[0].clks_cfg.voltage = vlevel;
+ pipes[0].clks_cfg.dcfclk_mhz = context->bw_ctx.dml.soc.clock_limits[vlevel].dcfclk_mhz;
+ pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[vlevel].socclk_mhz;
+ context->bw_ctx.bw.dcn.watermarks.a.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ context->bw_ctx.bw.dcn.watermarks.a.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+ /* Writeback MCIF_WB arbitration parameters */
+ dc->res_pool->funcs->set_mcif_arb_params(dc, context, pipes, pipe_cnt);
+
+ context->bw_ctx.bw.dcn.clk.dispclk_khz = context->bw_ctx.dml.vba.DISPCLK * 1000;
+ context->bw_ctx.bw.dcn.clk.dcfclk_khz = context->bw_ctx.dml.vba.DCFCLK * 1000;
+ context->bw_ctx.bw.dcn.clk.socclk_khz = context->bw_ctx.dml.vba.SOCCLK * 1000;
+ context->bw_ctx.bw.dcn.clk.dramclk_khz = context->bw_ctx.dml.vba.DRAMSpeed * 1000 / 16;
+ context->bw_ctx.bw.dcn.clk.dcfclk_deep_sleep_khz = context->bw_ctx.dml.vba.DCFCLKDeepSleep * 1000;
+ context->bw_ctx.bw.dcn.clk.fclk_khz = context->bw_ctx.dml.vba.FabricClock * 1000;
+ context->bw_ctx.bw.dcn.clk.p_state_change_support =
+ context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][context->bw_ctx.dml.vba.maxMpcComb]
+ != dm_dram_clock_change_unsupported;
+ context->bw_ctx.bw.dcn.clk.dppclk_khz = 0;
+
+ BW_VAL_TRACE_END_WATERMARKS();
+
+ for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
+ if (!context->res_ctx.pipe_ctx[i].stream)
+ continue;
+ pipes[pipe_idx].pipe.dest.vstartup_start = context->bw_ctx.dml.vba.VStartup[pipe_idx];
+ pipes[pipe_idx].pipe.dest.vupdate_offset = context->bw_ctx.dml.vba.VUpdateOffsetPix[pipe_idx];
+ pipes[pipe_idx].pipe.dest.vupdate_width = context->bw_ctx.dml.vba.VUpdateWidthPix[pipe_idx];
+ pipes[pipe_idx].pipe.dest.vready_offset = context->bw_ctx.dml.vba.VReadyOffsetPix[pipe_idx];
+ if (context->bw_ctx.bw.dcn.clk.dppclk_khz < pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000)
+ context->bw_ctx.bw.dcn.clk.dppclk_khz = pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000;
+ context->res_ctx.pipe_ctx[i].plane_res.bw.dppclk_khz =
+ pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ context->res_ctx.pipe_ctx[i].stream_res.dscclk_khz =
+ context->bw_ctx.dml.vba.DSCCLK_calculated[pipe_idx] * 1000;
+#endif
+ context->res_ctx.pipe_ctx[i].pipe_dlg_param = pipes[pipe_idx].pipe.dest;
+ pipe_idx++;
+ }
+
+ for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
+ bool cstate_en = context->bw_ctx.dml.vba.PrefetchMode[vlevel][context->bw_ctx.dml.vba.maxMpcComb] != 2;
+
+ if (!context->res_ctx.pipe_ctx[i].stream)
+ continue;
+
+ context->bw_ctx.dml.funcs.rq_dlg_get_dlg_reg(&context->bw_ctx.dml,
+ &context->res_ctx.pipe_ctx[i].dlg_regs,
+ &context->res_ctx.pipe_ctx[i].ttu_regs,
+ pipes,
+ pipe_cnt,
+ pipe_idx,
+ cstate_en,
+ context->bw_ctx.bw.dcn.clk.p_state_change_support,
+ false, false, false);
+
+ context->bw_ctx.dml.funcs.rq_dlg_get_rq_reg(&context->bw_ctx.dml,
+ &context->res_ctx.pipe_ctx[i].rq_regs,
+ pipes[pipe_idx].pipe);
+ pipe_idx++;
+ }
+
+ out = true;
+ goto validate_out;
+
+validate_fail:
+ DC_LOG_WARNING("Mode Validation Warning: %s failed validation.\n",
+ dml_get_status_message(context->bw_ctx.dml.vba.ValidationStatus[context->bw_ctx.dml.vba.soc.num_states]));
+
+ BW_VAL_TRACE_SKIP(fail);
+ out = false;
+
+validate_out:
+ kfree(pipes);
+
+ BW_VAL_TRACE_FINISH();
+
+ return out;
+}
+
+struct pipe_ctx *dcn20_acquire_idle_pipe_for_layer(
+ struct dc_state *state,
+ const struct resource_pool *pool,
+ struct dc_stream_state *stream)
+{
+ struct resource_context *res_ctx = &state->res_ctx;
+ struct pipe_ctx *head_pipe = resource_get_head_pipe_for_stream(res_ctx, stream);
+ struct pipe_ctx *idle_pipe = find_idle_secondary_pipe(res_ctx, pool, head_pipe);
+
+ if (!head_pipe)
+ ASSERT(0);
+
+ if (!idle_pipe)
+ return NULL;
+
+ idle_pipe->stream = head_pipe->stream;
+ idle_pipe->stream_res.tg = head_pipe->stream_res.tg;
+ idle_pipe->stream_res.opp = head_pipe->stream_res.opp;
+
+ idle_pipe->plane_res.hubp = pool->hubps[idle_pipe->pipe_idx];
+ idle_pipe->plane_res.ipp = pool->ipps[idle_pipe->pipe_idx];
+ idle_pipe->plane_res.dpp = pool->dpps[idle_pipe->pipe_idx];
+ idle_pipe->plane_res.mpcc_inst = pool->dpps[idle_pipe->pipe_idx]->inst;
+
+ return idle_pipe;
+}
+
+bool dcn20_get_dcc_compression_cap(const struct dc *dc,
+ const struct dc_dcc_surface_param *input,
+ struct dc_surface_dcc_cap *output)
+{
+ return dc->res_pool->hubbub->funcs->get_dcc_compression_cap(
+ dc->res_pool->hubbub,
+ input,
+ output);
+}
+
+static void dcn20_destroy_resource_pool(struct resource_pool **pool)
+{
+ struct dcn20_resource_pool *dcn20_pool = TO_DCN20_RES_POOL(*pool);
+
+ destruct(dcn20_pool);
+ kfree(dcn20_pool);
+ *pool = NULL;
+}
+
+
+static struct dc_cap_funcs cap_funcs = {
+ .get_dcc_compression_cap = dcn20_get_dcc_compression_cap
+};
+
+
+enum dc_status dcn20_get_default_swizzle_mode(struct dc_plane_state *plane_state)
+{
+ enum dc_status result = DC_OK;
+
+ enum surface_pixel_format surf_pix_format = plane_state->format;
+ unsigned int bpp = resource_pixel_format_to_bpp(surf_pix_format);
+
+ enum swizzle_mode_values swizzle = DC_SW_LINEAR;
+
+ if (bpp == 64)
+ swizzle = DC_SW_64KB_D;
+ else
+ swizzle = DC_SW_64KB_S;
+
+ plane_state->tiling_info.gfx9.swizzle = swizzle;
+ return result;
+}
+
+static struct resource_funcs dcn20_res_pool_funcs = {
+ .destroy = dcn20_destroy_resource_pool,
+ .link_enc_create = dcn20_link_encoder_create,
+ .validate_bandwidth = dcn20_validate_bandwidth,
+ .acquire_idle_pipe_for_layer = dcn20_acquire_idle_pipe_for_layer,
+ .add_stream_to_ctx = dcn20_add_stream_to_ctx,
+ .remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
+ .populate_dml_writeback_from_context = dcn20_populate_dml_writeback_from_context,
+ .get_default_swizzle_mode = dcn20_get_default_swizzle_mode,
+ .set_mcif_arb_params = dcn20_set_mcif_arb_params,
+ .find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link
+};
+
+bool dcn20_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+ int i;
+ uint32_t pipe_count = pool->res_cap->num_dwb;
+
+ ASSERT(pipe_count > 0);
+
+ for (i = 0; i < pipe_count; i++) {
+ struct dcn20_dwbc *dwbc20 = kzalloc(sizeof(struct dcn20_dwbc),
+ GFP_KERNEL);
+
+ if (!dwbc20) {
+ dm_error("DC: failed to create dwbc20!\n");
+ return false;
+ }
+ dcn20_dwbc_construct(dwbc20, ctx,
+ &dwbc20_regs[i],
+ &dwbc20_shift,
+ &dwbc20_mask,
+ i);
+ pool->dwbc[i] = &dwbc20->base;
+ }
+ return true;
+}
+
+bool dcn20_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+ int i;
+ uint32_t pipe_count = pool->res_cap->num_dwb;
+
+ ASSERT(pipe_count > 0);
+
+ for (i = 0; i < pipe_count; i++) {
+ struct dcn20_mmhubbub *mcif_wb20 = kzalloc(sizeof(struct dcn20_mmhubbub),
+ GFP_KERNEL);
+
+ if (!mcif_wb20) {
+ dm_error("DC: failed to create mcif_wb20!\n");
+ return false;
+ }
+
+ dcn20_mmhubbub_construct(mcif_wb20, ctx,
+ &mcif_wb20_regs[i],
+ &mcif_wb20_shift,
+ &mcif_wb20_mask,
+ i);
+
+ pool->mcif_wb[i] = &mcif_wb20->base;
+ }
+ return true;
+}
+
+struct pp_smu_funcs *dcn20_pp_smu_create(struct dc_context *ctx)
+{
+ struct pp_smu_funcs *pp_smu = kzalloc(sizeof(*pp_smu), GFP_KERNEL);
+
+ if (!pp_smu)
+ return pp_smu;
+
+ dm_pp_get_funcs(ctx, pp_smu);
+
+ if (pp_smu->ctx.ver != PP_SMU_VER_NV)
+ pp_smu = memset(pp_smu, 0, sizeof(struct pp_smu_funcs));
+
+ return pp_smu;
+}
+
+void dcn20_pp_smu_destroy(struct pp_smu_funcs **pp_smu)
+{
+ if (pp_smu && *pp_smu) {
+ kfree(*pp_smu);
+ *pp_smu = NULL;
+ }
+}
+
+static void cap_soc_clocks(
+ struct _vcs_dpi_soc_bounding_box_st *bb,
+ struct pp_smu_nv_clock_table max_clocks)
+{
+ int i;
+
+ // First pass - cap all clocks higher than the reported max
+ for (i = 0; i < bb->num_states; i++) {
+ if ((bb->clock_limits[i].dcfclk_mhz > (max_clocks.dcfClockInKhz / 1000))
+ && max_clocks.dcfClockInKhz != 0)
+ bb->clock_limits[i].dcfclk_mhz = (max_clocks.dcfClockInKhz / 1000);
+
+ if ((bb->clock_limits[i].dram_speed_mts > (max_clocks.uClockInKhz / 1000) * 16)
+ && max_clocks.uClockInKhz != 0)
+ bb->clock_limits[i].dram_speed_mts = (max_clocks.uClockInKhz / 1000) * 16;
+
+ // HACK: Force every uclk to max for now to "disable" uclk switching.
+ bb->clock_limits[i].dram_speed_mts = (max_clocks.uClockInKhz / 1000) * 16;
+
+ if ((bb->clock_limits[i].fabricclk_mhz > (max_clocks.fabricClockInKhz / 1000))
+ && max_clocks.fabricClockInKhz != 0)
+ bb->clock_limits[i].fabricclk_mhz = (max_clocks.fabricClockInKhz / 1000);
+
+ if ((bb->clock_limits[i].dispclk_mhz > (max_clocks.displayClockInKhz / 1000))
+ && max_clocks.displayClockInKhz != 0)
+ bb->clock_limits[i].dispclk_mhz = (max_clocks.displayClockInKhz / 1000);
+
+ if ((bb->clock_limits[i].dppclk_mhz > (max_clocks.dppClockInKhz / 1000))
+ && max_clocks.dppClockInKhz != 0)
+ bb->clock_limits[i].dppclk_mhz = (max_clocks.dppClockInKhz / 1000);
+
+ if ((bb->clock_limits[i].phyclk_mhz > (max_clocks.phyClockInKhz / 1000))
+ && max_clocks.phyClockInKhz != 0)
+ bb->clock_limits[i].phyclk_mhz = (max_clocks.phyClockInKhz / 1000);
+
+ if ((bb->clock_limits[i].socclk_mhz > (max_clocks.socClockInKhz / 1000))
+ && max_clocks.socClockInKhz != 0)
+ bb->clock_limits[i].socclk_mhz = (max_clocks.socClockInKhz / 1000);
+
+ if ((bb->clock_limits[i].dscclk_mhz > (max_clocks.dscClockInKhz / 1000))
+ && max_clocks.dscClockInKhz != 0)
+ bb->clock_limits[i].dscclk_mhz = (max_clocks.dscClockInKhz / 1000);
+ }
+
+ // Second pass - remove all duplicate clock states
+ for (i = bb->num_states - 1; i > 1; i--) {
+ bool duplicate = true;
+
+ if (bb->clock_limits[i-1].dcfclk_mhz != bb->clock_limits[i].dcfclk_mhz)
+ duplicate = false;
+ if (bb->clock_limits[i-1].dispclk_mhz != bb->clock_limits[i].dispclk_mhz)
+ duplicate = false;
+ if (bb->clock_limits[i-1].dppclk_mhz != bb->clock_limits[i].dppclk_mhz)
+ duplicate = false;
+ if (bb->clock_limits[i-1].dram_speed_mts != bb->clock_limits[i].dram_speed_mts)
+ duplicate = false;
+ if (bb->clock_limits[i-1].dscclk_mhz != bb->clock_limits[i].dscclk_mhz)
+ duplicate = false;
+ if (bb->clock_limits[i-1].fabricclk_mhz != bb->clock_limits[i].fabricclk_mhz)
+ duplicate = false;
+ if (bb->clock_limits[i-1].phyclk_mhz != bb->clock_limits[i].phyclk_mhz)
+ duplicate = false;
+ if (bb->clock_limits[i-1].socclk_mhz != bb->clock_limits[i].socclk_mhz)
+ duplicate = false;
+
+ if (duplicate)
+ bb->num_states--;
+ }
+}
+
+static void update_bounding_box(struct dc *dc, struct _vcs_dpi_soc_bounding_box_st *bb,
+ struct pp_smu_nv_clock_table *max_clocks, unsigned int *uclk_states, unsigned int num_states)
+{
+ struct _vcs_dpi_voltage_scaling_st calculated_states[MAX_CLOCK_LIMIT_STATES] = {0};
+ int i;
+ int num_calculated_states = 0;
+ int min_dcfclk = 0;
+
+ if (num_states == 0)
+ return;
+
+ if (dc->bb_overrides.min_dcfclk_mhz > 0)
+ min_dcfclk = dc->bb_overrides.min_dcfclk_mhz;
+ else
+ // Accounting for SOC/DCF relationship, we can go as high as
+ // 506Mhz in Vmin. We need to code 507 since SMU will round down to 506.
+ min_dcfclk = 507;
+
+ for (i = 0; i < num_states; i++) {
+ int min_fclk_required_by_uclk;
+ calculated_states[i].state = i;
+ calculated_states[i].dram_speed_mts = uclk_states[i] * 16 / 1000;
+
+ // FCLK:UCLK ratio is 1.08
+ min_fclk_required_by_uclk = mul_u64_u32_shr(BIT_ULL(32) * 1080 / 1000000, uclk_states[i], 32);
+
+ calculated_states[i].fabricclk_mhz = (min_fclk_required_by_uclk < min_dcfclk) ?
+ min_dcfclk : min_fclk_required_by_uclk;
+
+ calculated_states[i].socclk_mhz = (calculated_states[i].fabricclk_mhz > max_clocks->socClockInKhz / 1000) ?
+ max_clocks->socClockInKhz / 1000 : calculated_states[i].fabricclk_mhz;
+
+ calculated_states[i].dcfclk_mhz = (calculated_states[i].fabricclk_mhz > max_clocks->dcfClockInKhz / 1000) ?
+ max_clocks->dcfClockInKhz / 1000 : calculated_states[i].fabricclk_mhz;
+
+ calculated_states[i].dispclk_mhz = max_clocks->displayClockInKhz / 1000;
+ calculated_states[i].dppclk_mhz = max_clocks->displayClockInKhz / 1000;
+ calculated_states[i].dscclk_mhz = max_clocks->displayClockInKhz / (1000 * 3);
+
+ calculated_states[i].phyclk_mhz = max_clocks->phyClockInKhz / 1000;
+
+ num_calculated_states++;
+ }
+
+ memcpy(bb->clock_limits, calculated_states, sizeof(bb->clock_limits));
+ bb->num_states = num_calculated_states;
+
+ // Duplicate the last state, DML always an extra state identical to max state to work
+ memcpy(&bb->clock_limits[num_calculated_states], &bb->clock_limits[num_calculated_states - 1], sizeof(struct _vcs_dpi_voltage_scaling_st));
+ bb->clock_limits[num_calculated_states].state = bb->num_states;
+}
+
+static void patch_bounding_box(struct dc *dc, struct _vcs_dpi_soc_bounding_box_st *bb)
+{
+ kernel_fpu_begin();
+ if ((int)(bb->sr_exit_time_us * 1000) != dc->bb_overrides.sr_exit_time_ns
+ && dc->bb_overrides.sr_exit_time_ns) {
+ bb->sr_exit_time_us = dc->bb_overrides.sr_exit_time_ns / 1000.0;
+ }
+
+ if ((int)(bb->sr_enter_plus_exit_time_us * 1000)
+ != dc->bb_overrides.sr_enter_plus_exit_time_ns
+ && dc->bb_overrides.sr_enter_plus_exit_time_ns) {
+ bb->sr_enter_plus_exit_time_us =
+ dc->bb_overrides.sr_enter_plus_exit_time_ns / 1000.0;
+ }
+
+ if ((int)(bb->urgent_latency_us * 1000) != dc->bb_overrides.urgent_latency_ns
+ && dc->bb_overrides.urgent_latency_ns) {
+ bb->urgent_latency_us = dc->bb_overrides.urgent_latency_ns / 1000.0;
+ }
+
+ if ((int)(bb->dram_clock_change_latency_us * 1000)
+ != dc->bb_overrides.dram_clock_change_latency_ns
+ && dc->bb_overrides.dram_clock_change_latency_ns) {
+ bb->dram_clock_change_latency_us =
+ dc->bb_overrides.dram_clock_change_latency_ns / 1000.0;
+ }
+ kernel_fpu_end();
+}
+
+#define fixed16_to_double(x) (((double) x) / ((double) (1 << 16)))
+#define fixed16_to_double_to_cpu(x) fixed16_to_double(le32_to_cpu(x))
+
+static bool init_soc_bounding_box(struct dc *dc,
+ struct dcn20_resource_pool *pool)
+{
+ const struct gpu_info_soc_bounding_box_v1_0 *bb = dc->soc_bounding_box;
+ DC_LOGGER_INIT(dc->ctx->logger);
+
+ if (!bb && !SOC_BOUNDING_BOX_VALID) {
+ DC_LOG_ERROR("%s: not valid soc bounding box/n", __func__);
+ return false;
+ }
+
+ if (bb && !SOC_BOUNDING_BOX_VALID) {
+ int i;
+
+ dcn2_0_soc.sr_exit_time_us =
+ fixed16_to_double_to_cpu(bb->sr_exit_time_us);
+ dcn2_0_soc.sr_enter_plus_exit_time_us =
+ fixed16_to_double_to_cpu(bb->sr_enter_plus_exit_time_us);
+ dcn2_0_soc.urgent_latency_us =
+ fixed16_to_double_to_cpu(bb->urgent_latency_us);
+ dcn2_0_soc.urgent_latency_pixel_data_only_us =
+ fixed16_to_double_to_cpu(bb->urgent_latency_pixel_data_only_us);
+ dcn2_0_soc.urgent_latency_pixel_mixed_with_vm_data_us =
+ fixed16_to_double_to_cpu(bb->urgent_latency_pixel_mixed_with_vm_data_us);
+ dcn2_0_soc.urgent_latency_vm_data_only_us =
+ fixed16_to_double_to_cpu(bb->urgent_latency_vm_data_only_us);
+ dcn2_0_soc.urgent_out_of_order_return_per_channel_pixel_only_bytes =
+ le32_to_cpu(bb->urgent_out_of_order_return_per_channel_pixel_only_bytes);
+ dcn2_0_soc.urgent_out_of_order_return_per_channel_pixel_and_vm_bytes =
+ le32_to_cpu(bb->urgent_out_of_order_return_per_channel_pixel_and_vm_bytes);
+ dcn2_0_soc.urgent_out_of_order_return_per_channel_vm_only_bytes =
+ le32_to_cpu(bb->urgent_out_of_order_return_per_channel_vm_only_bytes);
+ dcn2_0_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only =
+ fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_pixel_only);
+ dcn2_0_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm =
+ fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm);
+ dcn2_0_soc.pct_ideal_dram_sdp_bw_after_urgent_vm_only =
+ fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_vm_only);
+ dcn2_0_soc.max_avg_sdp_bw_use_normal_percent =
+ fixed16_to_double_to_cpu(bb->max_avg_sdp_bw_use_normal_percent);
+ dcn2_0_soc.max_avg_dram_bw_use_normal_percent =
+ fixed16_to_double_to_cpu(bb->max_avg_dram_bw_use_normal_percent);
+ dcn2_0_soc.writeback_latency_us =
+ fixed16_to_double_to_cpu(bb->writeback_latency_us);
+ dcn2_0_soc.ideal_dram_bw_after_urgent_percent =
+ fixed16_to_double_to_cpu(bb->ideal_dram_bw_after_urgent_percent);
+ dcn2_0_soc.max_request_size_bytes =
+ le32_to_cpu(bb->max_request_size_bytes);
+ dcn2_0_soc.dram_channel_width_bytes =
+ le32_to_cpu(bb->dram_channel_width_bytes);
+ dcn2_0_soc.fabric_datapath_to_dcn_data_return_bytes =
+ le32_to_cpu(bb->fabric_datapath_to_dcn_data_return_bytes);
+ dcn2_0_soc.dcn_downspread_percent =
+ fixed16_to_double_to_cpu(bb->dcn_downspread_percent);
+ dcn2_0_soc.downspread_percent =
+ fixed16_to_double_to_cpu(bb->downspread_percent);
+ dcn2_0_soc.dram_page_open_time_ns =
+ fixed16_to_double_to_cpu(bb->dram_page_open_time_ns);
+ dcn2_0_soc.dram_rw_turnaround_time_ns =
+ fixed16_to_double_to_cpu(bb->dram_rw_turnaround_time_ns);
+ dcn2_0_soc.dram_return_buffer_per_channel_bytes =
+ le32_to_cpu(bb->dram_return_buffer_per_channel_bytes);
+ dcn2_0_soc.round_trip_ping_latency_dcfclk_cycles =
+ le32_to_cpu(bb->round_trip_ping_latency_dcfclk_cycles);
+ dcn2_0_soc.urgent_out_of_order_return_per_channel_bytes =
+ le32_to_cpu(bb->urgent_out_of_order_return_per_channel_bytes);
+ dcn2_0_soc.channel_interleave_bytes =
+ le32_to_cpu(bb->channel_interleave_bytes);
+ dcn2_0_soc.num_banks =
+ le32_to_cpu(bb->num_banks);
+ dcn2_0_soc.num_chans =
+ le32_to_cpu(bb->num_chans);
+ dcn2_0_soc.vmm_page_size_bytes =
+ le32_to_cpu(bb->vmm_page_size_bytes);
+ dcn2_0_soc.dram_clock_change_latency_us =
+ fixed16_to_double_to_cpu(bb->dram_clock_change_latency_us);
+ // HACK!! Lower uclock latency switch time so we don't switch
+ dcn2_0_soc.dram_clock_change_latency_us = 10;
+ dcn2_0_soc.writeback_dram_clock_change_latency_us =
+ fixed16_to_double_to_cpu(bb->writeback_dram_clock_change_latency_us);
+ dcn2_0_soc.return_bus_width_bytes =
+ le32_to_cpu(bb->return_bus_width_bytes);
+ dcn2_0_soc.dispclk_dppclk_vco_speed_mhz =
+ le32_to_cpu(bb->dispclk_dppclk_vco_speed_mhz);
+ dcn2_0_soc.xfc_bus_transport_time_us =
+ le32_to_cpu(bb->xfc_bus_transport_time_us);
+ dcn2_0_soc.xfc_xbuf_latency_tolerance_us =
+ le32_to_cpu(bb->xfc_xbuf_latency_tolerance_us);
+ dcn2_0_soc.use_urgent_burst_bw =
+ le32_to_cpu(bb->use_urgent_burst_bw);
+ dcn2_0_soc.num_states =
+ le32_to_cpu(bb->num_states);
+
+ for (i = 0; i < dcn2_0_soc.num_states; i++) {
+ dcn2_0_soc.clock_limits[i].state =
+ le32_to_cpu(bb->clock_limits[i].state);
+ dcn2_0_soc.clock_limits[i].dcfclk_mhz =
+ fixed16_to_double_to_cpu(bb->clock_limits[i].dcfclk_mhz);
+ dcn2_0_soc.clock_limits[i].fabricclk_mhz =
+ fixed16_to_double_to_cpu(bb->clock_limits[i].fabricclk_mhz);
+ dcn2_0_soc.clock_limits[i].dispclk_mhz =
+ fixed16_to_double_to_cpu(bb->clock_limits[i].dispclk_mhz);
+ dcn2_0_soc.clock_limits[i].dppclk_mhz =
+ fixed16_to_double_to_cpu(bb->clock_limits[i].dppclk_mhz);
+ dcn2_0_soc.clock_limits[i].phyclk_mhz =
+ fixed16_to_double_to_cpu(bb->clock_limits[i].phyclk_mhz);
+ dcn2_0_soc.clock_limits[i].socclk_mhz =
+ fixed16_to_double_to_cpu(bb->clock_limits[i].socclk_mhz);
+ dcn2_0_soc.clock_limits[i].dscclk_mhz =
+ fixed16_to_double_to_cpu(bb->clock_limits[i].dscclk_mhz);
+ dcn2_0_soc.clock_limits[i].dram_speed_mts =
+ fixed16_to_double_to_cpu(bb->clock_limits[i].dram_speed_mts);
+ }
+ }
+
+ if (pool->base.pp_smu) {
+ struct pp_smu_nv_clock_table max_clocks = {0};
+ unsigned int uclk_states[8] = {0};
+ unsigned int num_states = 0;
+ int i;
+ enum pp_smu_status status;
+ bool clock_limits_available = false;
+ bool uclk_states_available = false;
+
+ if (pool->base.pp_smu->nv_funcs.get_uclk_dpm_states) {
+ status = (pool->base.pp_smu->nv_funcs.get_uclk_dpm_states)
+ (&pool->base.pp_smu->nv_funcs.pp_smu, uclk_states, &num_states);
+
+ uclk_states_available = (status == PP_SMU_RESULT_OK);
+ }
+
+ if (pool->base.pp_smu->nv_funcs.get_maximum_sustainable_clocks) {
+ status = (*pool->base.pp_smu->nv_funcs.get_maximum_sustainable_clocks)
+ (&pool->base.pp_smu->nv_funcs.pp_smu, &max_clocks);
+ /* SMU cannot set DCF clock to anything equal to or higher than SOC clock
+ */
+ if (max_clocks.dcfClockInKhz >= max_clocks.socClockInKhz)
+ max_clocks.dcfClockInKhz = max_clocks.socClockInKhz - 1000;
+ clock_limits_available = (status == PP_SMU_RESULT_OK);
+ }
+
+ // HACK: Use the max uclk_states value for all elements.
+ for (i = 0; i < num_states; i++)
+ uclk_states[i] = uclk_states[num_states - 1];
+
+ if (clock_limits_available && uclk_states_available && num_states)
+ update_bounding_box(dc, &dcn2_0_soc, &max_clocks, uclk_states, num_states);
+ else if (clock_limits_available)
+ cap_soc_clocks(&dcn2_0_soc, max_clocks);
+ }
+
+ dcn2_0_ip.max_num_otg = pool->base.res_cap->num_timing_generator;
+ dcn2_0_ip.max_num_dpp = pool->base.pipe_count;
+ patch_bounding_box(dc, &dcn2_0_soc);
+
+ return true;
+}
+
+static bool construct(
+ uint8_t num_virtual_links,
+ struct dc *dc,
+ struct dcn20_resource_pool *pool)
+{
+ int i;
+ struct dc_context *ctx = dc->ctx;
+ struct irq_service_init_data init_data;
+
+ ctx->dc_bios->regs = &bios_regs;
+
+ pool->base.res_cap = &res_cap_nv10;
+ pool->base.funcs = &dcn20_res_pool_funcs;
+
+ /*************************************************
+ * Resource + asic cap harcoding *
+ *************************************************/
+ pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+
+ pool->base.pipe_count = 6;
+ pool->base.mpcc_count = 6;
+ dc->caps.max_downscale_ratio = 200;
+ dc->caps.i2c_speed_in_khz = 100;
+ dc->caps.max_cursor_size = 256;
+ dc->caps.dmdata_alloc_size = 2048;
+
+ dc->caps.max_slave_planes = 1;
+ dc->caps.post_blend_color_processing = true;
+ dc->caps.force_dp_tps4_for_cp2520 = true;
+ dc->caps.hw_3d_lut = true;
+
+ if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV) {
+ dc->debug = debug_defaults_drv;
+ } else if (dc->ctx->dce_environment == DCE_ENV_FPGA_MAXIMUS) {
+ pool->base.pipe_count = 4;
+ pool->base.mpcc_count = pool->base.pipe_count;
+ dc->debug = debug_defaults_diags;
+ } else {
+ dc->debug = debug_defaults_diags;
+ }
+ //dcn2.0x
+ dc->work_arounds.dedcn20_305_wa = true;
+
+ // Init the vm_helper
+ if (dc->vm_helper)
+ vm_helper_init(dc->vm_helper, 16);
+
+ /*************************************************
+ * Create resources *
+ *************************************************/
+
+ pool->base.clock_sources[DCN20_CLK_SRC_PLL0] =
+ dcn20_clock_source_create(ctx, ctx->dc_bios,
+ CLOCK_SOURCE_COMBO_PHY_PLL0,
+ &clk_src_regs[0], false);
+ pool->base.clock_sources[DCN20_CLK_SRC_PLL1] =
+ dcn20_clock_source_create(ctx, ctx->dc_bios,
+ CLOCK_SOURCE_COMBO_PHY_PLL1,
+ &clk_src_regs[1], false);
+ pool->base.clock_sources[DCN20_CLK_SRC_PLL2] =
+ dcn20_clock_source_create(ctx, ctx->dc_bios,
+ CLOCK_SOURCE_COMBO_PHY_PLL2,
+ &clk_src_regs[2], false);
+ pool->base.clock_sources[DCN20_CLK_SRC_PLL3] =
+ dcn20_clock_source_create(ctx, ctx->dc_bios,
+ CLOCK_SOURCE_COMBO_PHY_PLL3,
+ &clk_src_regs[3], false);
+ pool->base.clock_sources[DCN20_CLK_SRC_PLL4] =
+ dcn20_clock_source_create(ctx, ctx->dc_bios,
+ CLOCK_SOURCE_COMBO_PHY_PLL4,
+ &clk_src_regs[4], false);
+ pool->base.clock_sources[DCN20_CLK_SRC_PLL5] =
+ dcn20_clock_source_create(ctx, ctx->dc_bios,
+ CLOCK_SOURCE_COMBO_PHY_PLL5,
+ &clk_src_regs[5], false);
+ pool->base.clk_src_count = DCN20_CLK_SRC_TOTAL;
+ /* todo: not reuse phy_pll registers */
+ pool->base.dp_clock_source =
+ dcn20_clock_source_create(ctx, ctx->dc_bios,
+ CLOCK_SOURCE_ID_DP_DTO,
+ &clk_src_regs[0], true);
+
+ for (i = 0; i < pool->base.clk_src_count; i++) {
+ if (pool->base.clock_sources[i] == NULL) {
+ dm_error("DC: failed to create clock sources!\n");
+ BREAK_TO_DEBUGGER();
+ goto create_fail;
+ }
+ }
+
+ pool->base.dccg = dccg2_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
+ if (pool->base.dccg == NULL) {
+ dm_error("DC: failed to create dccg!\n");
+ BREAK_TO_DEBUGGER();
+ goto create_fail;
+ }
+
+ pool->base.dmcu = dcn20_dmcu_create(ctx,
+ &dmcu_regs,
+ &dmcu_shift,
+ &dmcu_mask);
+ if (pool->base.dmcu == NULL) {
+ dm_error("DC: failed to create dmcu!\n");
+ BREAK_TO_DEBUGGER();
+ goto create_fail;
+ }
+
+ pool->base.abm = dce_abm_create(ctx,
+ &abm_regs,
+ &abm_shift,
+ &abm_mask);
+ if (pool->base.abm == NULL) {
+ dm_error("DC: failed to create abm!\n");
+ BREAK_TO_DEBUGGER();
+ goto create_fail;
+ }
+
+ pool->base.pp_smu = dcn20_pp_smu_create(ctx);
+
+
+ if (!init_soc_bounding_box(dc, pool)) {
+ dm_error("DC: failed to initialize soc bounding box!\n");
+ BREAK_TO_DEBUGGER();
+ goto create_fail;
+ }
+
+ dml_init_instance(&dc->dml, &dcn2_0_soc, &dcn2_0_ip, DML_PROJECT_NAVI10);
+
+ if (!dc->debug.disable_pplib_wm_range) {
+ struct pp_smu_wm_range_sets ranges = {0};
+ int i = 0;
+
+ ranges.num_reader_wm_sets = 0;
+
+ if (dcn2_0_soc.num_states == 1) {
+ ranges.reader_wm_sets[0].wm_inst = i;
+ ranges.reader_wm_sets[0].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+ ranges.reader_wm_sets[0].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+ ranges.reader_wm_sets[0].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+ ranges.reader_wm_sets[0].max_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+
+ ranges.num_reader_wm_sets = 1;
+ } else if (dcn2_0_soc.num_states > 1) {
+ for (i = 0; i < 4 && i < dcn2_0_soc.num_states; i++) {
+ ranges.reader_wm_sets[i].wm_inst = i;
+ ranges.reader_wm_sets[i].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+ ranges.reader_wm_sets[i].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+ ranges.reader_wm_sets[i].min_fill_clk_mhz = (i > 0) ? (dcn2_0_soc.clock_limits[i - 1].dram_speed_mts / 16) + 1 : 0;
+ ranges.reader_wm_sets[i].max_fill_clk_mhz = dcn2_0_soc.clock_limits[i].dram_speed_mts / 16;
+
+ ranges.num_reader_wm_sets = i + 1;
+ }
+
+ ranges.reader_wm_sets[0].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+ ranges.reader_wm_sets[ranges.num_reader_wm_sets - 1].max_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+ }
+
+ ranges.num_writer_wm_sets = 1;
+
+ ranges.writer_wm_sets[0].wm_inst = 0;
+ ranges.writer_wm_sets[0].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+ ranges.writer_wm_sets[0].max_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+ ranges.writer_wm_sets[0].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+ ranges.writer_wm_sets[0].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+
+ /* Notify PP Lib/SMU which Watermarks to use for which clock ranges */
+ if (pool->base.pp_smu->nv_funcs.set_wm_ranges)
+ pool->base.pp_smu->nv_funcs.set_wm_ranges(&pool->base.pp_smu->nv_funcs.pp_smu, &ranges);
+ }
+
+ init_data.ctx = dc->ctx;
+ pool->base.irqs = dal_irq_service_dcn20_create(&init_data);
+ if (!pool->base.irqs)
+ goto create_fail;
+
+ /* mem input -> ipp -> dpp -> opp -> TG */
+ for (i = 0; i < pool->base.pipe_count; i++) {
+ pool->base.hubps[i] = dcn20_hubp_create(ctx, i);
+ if (pool->base.hubps[i] == NULL) {
+ BREAK_TO_DEBUGGER();
+ dm_error(
+ "DC: failed to create memory input!\n");
+ goto create_fail;
+ }
+
+ pool->base.ipps[i] = dcn20_ipp_create(ctx, i);
+ if (pool->base.ipps[i] == NULL) {
+ BREAK_TO_DEBUGGER();
+ dm_error(
+ "DC: failed to create input pixel processor!\n");
+ goto create_fail;
+ }
+
+ pool->base.dpps[i] = dcn20_dpp_create(ctx, i);
+ if (pool->base.dpps[i] == NULL) {
+ BREAK_TO_DEBUGGER();
+ dm_error(
+ "DC: failed to create dpps!\n");
+ goto create_fail;
+ }
+ }
+ for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+ pool->base.engines[i] = dcn20_aux_engine_create(ctx, i);
+ if (pool->base.engines[i] == NULL) {
+ BREAK_TO_DEBUGGER();
+ dm_error(
+ "DC:failed to create aux engine!!\n");
+ goto create_fail;
+ }
+ pool->base.hw_i2cs[i] = dcn20_i2c_hw_create(ctx, i);
+ if (pool->base.hw_i2cs[i] == NULL) {
+ BREAK_TO_DEBUGGER();
+ dm_error(
+ "DC:failed to create hw i2c!!\n");
+ goto create_fail;
+ }
+ pool->base.sw_i2cs[i] = NULL;
+ }
+
+ for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+ pool->base.opps[i] = dcn20_opp_create(ctx, i);
+ if (pool->base.opps[i] == NULL) {
+ BREAK_TO_DEBUGGER();
+ dm_error(
+ "DC: failed to create output pixel processor!\n");
+ goto create_fail;
+ }
+ }
+
+ for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+ pool->base.timing_generators[i] = dcn20_timing_generator_create(
+ ctx, i);
+ if (pool->base.timing_generators[i] == NULL) {
+ BREAK_TO_DEBUGGER();
+ dm_error("DC: failed to create tg!\n");
+ goto create_fail;
+ }
+ }
+
+ pool->base.timing_generator_count = i;
+
+ pool->base.mpc = dcn20_mpc_create(ctx);
+ if (pool->base.mpc == NULL) {
+ BREAK_TO_DEBUGGER();
+ dm_error("DC: failed to create mpc!\n");
+ goto create_fail;
+ }
+
+ pool->base.hubbub = dcn20_hubbub_create(ctx);
+ if (pool->base.hubbub == NULL) {
+ BREAK_TO_DEBUGGER();
+ dm_error("DC: failed to create hubbub!\n");
+ goto create_fail;
+ }
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+ pool->base.dscs[i] = dcn20_dsc_create(ctx, i);
+ if (pool->base.dscs[i] == NULL) {
+ BREAK_TO_DEBUGGER();
+ dm_error("DC: failed to create display stream compressor %d!\n", i);
+ goto create_fail;
+ }
+ }
+#endif
+
+ if (!dcn20_dwbc_create(ctx, &pool->base)) {
+ BREAK_TO_DEBUGGER();
+ dm_error("DC: failed to create dwbc!\n");
+ goto create_fail;
+ }
+ if (!dcn20_mmhubbub_create(ctx, &pool->base)) {
+ BREAK_TO_DEBUGGER();
+ dm_error("DC: failed to create mcif_wb!\n");
+ goto create_fail;
+ }
+
+ if (!resource_construct(num_virtual_links, dc, &pool->base,
+ (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment) ?
+ &res_create_funcs : &res_create_maximus_funcs)))
+ goto create_fail;
+
+ dcn20_hw_sequencer_construct(dc);
+
+ dc->caps.max_planes = pool->base.pipe_count;
+
+ for (i = 0; i < dc->caps.max_planes; ++i)
+ dc->caps.planes[i] = plane_cap;
+
+ dc->cap_funcs = cap_funcs;
+
+ return true;
+
+create_fail:
+
+ destruct(pool);
+
+ return false;
+}
+
+struct resource_pool *dcn20_create_resource_pool(
+ const struct dc_init_data *init_data,
+ struct dc *dc)
+{
+ struct dcn20_resource_pool *pool =
+ kzalloc(sizeof(struct dcn20_resource_pool), GFP_KERNEL);
+
+ if (!pool)
+ return NULL;
+
+ if (construct(init_data->num_virtual_links, dc, pool))
+ return &pool->base;
+
+ BREAK_TO_DEBUGGER();
+ kfree(pool);
+ return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.h
new file mode 100644
index 000000000000..b5a75289f444
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.h
@@ -0,0 +1,133 @@
+/*
+* Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_RESOURCE_DCN20_H__
+#define __DC_RESOURCE_DCN20_H__
+
+#include "core_types.h"
+
+#define TO_DCN20_RES_POOL(pool)\
+ container_of(pool, struct dcn20_resource_pool, base)
+
+struct dc;
+struct resource_pool;
+struct _vcs_dpi_display_pipe_params_st;
+
+struct dcn20_resource_pool {
+ struct resource_pool base;
+};
+struct resource_pool *dcn20_create_resource_pool(
+ const struct dc_init_data *init_data,
+ struct dc *dc);
+
+struct link_encoder *dcn20_link_encoder_create(
+ const struct encoder_init_data *enc_init_data);
+
+unsigned int dcn20_calc_max_scaled_time(
+ unsigned int time_per_pixel,
+ enum mmhubbub_wbif_mode mode,
+ unsigned int urgent_watermark);
+int dcn20_populate_dml_pipes_from_context(
+ struct dc *dc, struct resource_context *res_ctx, display_e2e_pipe_params_st *pipes);
+struct pipe_ctx *dcn20_acquire_idle_pipe_for_layer(
+ struct dc_state *state,
+ const struct resource_pool *pool,
+ struct dc_stream_state *stream);
+void dcn20_populate_dml_writeback_from_context(
+ struct dc *dc, struct resource_context *res_ctx, display_e2e_pipe_params_st *pipes);
+
+struct stream_encoder *dcn20_stream_encoder_create(
+ enum engine_id eng_id,
+ struct dc_context *ctx);
+
+struct dce_hwseq *dcn20_hwseq_create(
+ struct dc_context *ctx);
+
+bool dcn20_get_dcc_compression_cap(const struct dc *dc,
+ const struct dc_dcc_surface_param *input,
+ struct dc_surface_dcc_cap *output);
+
+void dcn20_dpp_destroy(struct dpp **dpp);
+
+struct dpp *dcn20_dpp_create(
+ struct dc_context *ctx,
+ uint32_t inst);
+
+struct input_pixel_processor *dcn20_ipp_create(
+ struct dc_context *ctx, uint32_t inst);
+
+
+struct output_pixel_processor *dcn20_opp_create(
+ struct dc_context *ctx, uint32_t inst);
+
+struct dce_aux *dcn20_aux_engine_create(
+ struct dc_context *ctx, uint32_t inst);
+
+struct dce_i2c_hw *dcn20_i2c_hw_create(
+ struct dc_context *ctx,
+ uint32_t inst);
+
+void dcn20_clock_source_destroy(struct clock_source **clk_src);
+
+struct display_stream_compressor *dcn20_dsc_create(
+ struct dc_context *ctx, uint32_t inst);
+void dcn20_dsc_destroy(struct display_stream_compressor **dsc);
+
+struct pp_smu_funcs *dcn20_pp_smu_create(struct dc_context *ctx);
+void dcn20_pp_smu_destroy(struct pp_smu_funcs **pp_smu);
+
+struct hubp *dcn20_hubp_create(
+ struct dc_context *ctx,
+ uint32_t inst);
+struct timing_generator *dcn20_timing_generator_create(
+ struct dc_context *ctx,
+ uint32_t instance);
+struct mpc *dcn20_mpc_create(struct dc_context *ctx);
+struct hubbub *dcn20_hubbub_create(struct dc_context *ctx);
+
+bool dcn20_dwbc_create(struct dc_context *ctx, struct resource_pool *pool);
+bool dcn20_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool);
+
+void dcn20_set_mcif_arb_params(
+ struct dc *dc,
+ struct dc_state *context,
+ display_e2e_pipe_params_st *pipes,
+ int pipe_cnt);
+bool dcn20_validate_bandwidth(struct dc *dc, struct dc_state *context, bool fast_validate);
+
+enum dc_status dcn20_build_mapped_resource(const struct dc *dc, struct dc_state *context, struct dc_stream_state *stream);
+enum dc_status dcn20_add_stream_to_ctx(struct dc *dc, struct dc_state *new_ctx, struct dc_stream_state *dc_stream);
+enum dc_status dcn20_remove_stream_from_ctx(struct dc *dc, struct dc_state *new_ctx, struct dc_stream_state *dc_stream);
+enum dc_status dcn20_get_default_swizzle_mode(struct dc_plane_state *plane_state);
+
+void dcn20_patch_bounding_box(
+ struct dc *dc,
+ struct _vcs_dpi_soc_bounding_box_st *bb);
+void dcn20_cap_soc_clocks(
+ struct _vcs_dpi_soc_bounding_box_st *bb,
+ struct pp_smu_nv_clock_table max_clocks);
+
+#endif /* __DC_RESOURCE_DCN20_H__ */
+
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_stream_encoder.c
new file mode 100644
index 000000000000..f5bcffc426b8
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_stream_encoder.c
@@ -0,0 +1,610 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include <linux/delay.h>
+
+#include "dc_bios_types.h"
+#include "dcn20_stream_encoder.h"
+#include "reg_helper.h"
+#include "hw_shared.h"
+
+#define DC_LOGGER \
+ enc1->base.ctx->logger
+
+
+#define REG(reg)\
+ (enc1->regs->reg)
+
+#undef FN
+#define FN(reg_name, field_name) \
+ enc1->se_shift->field_name, enc1->se_mask->field_name
+
+
+#define CTX \
+ enc1->base.ctx
+
+
+static void enc2_update_hdmi_info_packet(
+ struct dcn10_stream_encoder *enc1,
+ uint32_t packet_index,
+ const struct dc_info_packet *info_packet)
+{
+ uint32_t cont, send, line;
+
+ if (info_packet->valid) {
+ enc1_update_generic_info_packet(
+ enc1,
+ packet_index,
+ info_packet);
+
+ /* enable transmission of packet(s) -
+ * packet transmission begins on the next frame */
+ cont = 1;
+ /* send packet(s) every frame */
+ send = 1;
+ /* select line number to send packets on */
+ line = 2;
+ } else {
+ cont = 0;
+ send = 0;
+ line = 0;
+ }
+
+ /* DP_SEC_GSP[x]_LINE_REFERENCE - keep default value REFER_TO_DP_SOF */
+
+ /* choose which generic packet control to use */
+ switch (packet_index) {
+ case 0:
+ REG_UPDATE_2(HDMI_GENERIC_PACKET_CONTROL0,
+ HDMI_GENERIC0_CONT, cont,
+ HDMI_GENERIC0_SEND, send);
+ REG_UPDATE(HDMI_GENERIC_PACKET_CONTROL1,
+ HDMI_GENERIC0_LINE, line);
+ break;
+ case 1:
+ REG_UPDATE_2(HDMI_GENERIC_PACKET_CONTROL0,
+ HDMI_GENERIC1_CONT, cont,
+ HDMI_GENERIC1_SEND, send);
+ REG_UPDATE(HDMI_GENERIC_PACKET_CONTROL1,
+ HDMI_GENERIC1_LINE, line);
+ break;
+ case 2:
+ REG_UPDATE_2(HDMI_GENERIC_PACKET_CONTROL0,
+ HDMI_GENERIC2_CONT, cont,
+ HDMI_GENERIC2_SEND, send);
+ REG_UPDATE(HDMI_GENERIC_PACKET_CONTROL2,
+ HDMI_GENERIC2_LINE, line);
+ break;
+ case 3:
+ REG_UPDATE_2(HDMI_GENERIC_PACKET_CONTROL0,
+ HDMI_GENERIC3_CONT, cont,
+ HDMI_GENERIC3_SEND, send);
+ REG_UPDATE(HDMI_GENERIC_PACKET_CONTROL2,
+ HDMI_GENERIC3_LINE, line);
+ break;
+ case 4:
+ REG_UPDATE_2(HDMI_GENERIC_PACKET_CONTROL0,
+ HDMI_GENERIC4_CONT, cont,
+ HDMI_GENERIC4_SEND, send);
+ REG_UPDATE(HDMI_GENERIC_PACKET_CONTROL3,
+ HDMI_GENERIC4_LINE, line);
+ break;
+ case 5:
+ REG_UPDATE_2(HDMI_GENERIC_PACKET_CONTROL0,
+ HDMI_GENERIC5_CONT, cont,
+ HDMI_GENERIC5_SEND, send);
+ REG_UPDATE(HDMI_GENERIC_PACKET_CONTROL3,
+ HDMI_GENERIC5_LINE, line);
+ break;
+ case 6:
+ REG_UPDATE_2(HDMI_GENERIC_PACKET_CONTROL0,
+ HDMI_GENERIC6_CONT, cont,
+ HDMI_GENERIC6_SEND, send);
+ REG_UPDATE(HDMI_GENERIC_PACKET_CONTROL4,
+ HDMI_GENERIC6_LINE, line);
+ break;
+ case 7:
+ REG_UPDATE_2(HDMI_GENERIC_PACKET_CONTROL0,
+ HDMI_GENERIC7_CONT, cont,
+ HDMI_GENERIC7_SEND, send);
+ REG_UPDATE(HDMI_GENERIC_PACKET_CONTROL4,
+ HDMI_GENERIC7_LINE, line);
+ break;
+ default:
+ /* invalid HW packet index */
+ DC_LOG_WARNING(
+ "Invalid HW packet index: %s()\n",
+ __func__);
+ return;
+ }
+}
+
+static void enc2_stream_encoder_update_hdmi_info_packets(
+ struct stream_encoder *enc,
+ const struct encoder_info_frame *info_frame)
+{
+ struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+ /* for bring up, disable dp double TODO */
+ REG_UPDATE(HDMI_DB_CONTROL, HDMI_DB_DISABLE, 1);
+
+ /*Always add mandatory packets first followed by optional ones*/
+ enc2_update_hdmi_info_packet(enc1, 0, &info_frame->avi);
+ enc2_update_hdmi_info_packet(enc1, 5, &info_frame->hfvsif);
+ enc2_update_hdmi_info_packet(enc1, 2, &info_frame->gamut);
+ enc2_update_hdmi_info_packet(enc1, 1, &info_frame->vendor);
+ enc2_update_hdmi_info_packet(enc1, 3, &info_frame->spd);
+ enc2_update_hdmi_info_packet(enc1, 4, &info_frame->hdrsmd);
+}
+
+static void enc2_stream_encoder_stop_hdmi_info_packets(
+ struct stream_encoder *enc)
+{
+ struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+ /* stop generic packets 0,1 on HDMI */
+ REG_SET_4(HDMI_GENERIC_PACKET_CONTROL0, 0,
+ HDMI_GENERIC0_CONT, 0,
+ HDMI_GENERIC0_SEND, 0,
+ HDMI_GENERIC1_CONT, 0,
+ HDMI_GENERIC1_SEND, 0);
+ REG_SET_2(HDMI_GENERIC_PACKET_CONTROL1, 0,
+ HDMI_GENERIC0_LINE, 0,
+ HDMI_GENERIC1_LINE, 0);
+
+ /* stop generic packets 2,3 on HDMI */
+ REG_SET_4(HDMI_GENERIC_PACKET_CONTROL0, 0,
+ HDMI_GENERIC2_CONT, 0,
+ HDMI_GENERIC2_SEND, 0,
+ HDMI_GENERIC3_CONT, 0,
+ HDMI_GENERIC3_SEND, 0);
+ REG_SET_2(HDMI_GENERIC_PACKET_CONTROL2, 0,
+ HDMI_GENERIC2_LINE, 0,
+ HDMI_GENERIC3_LINE, 0);
+
+ /* stop generic packets 4,5 on HDMI */
+ REG_SET_4(HDMI_GENERIC_PACKET_CONTROL0, 0,
+ HDMI_GENERIC4_CONT, 0,
+ HDMI_GENERIC4_SEND, 0,
+ HDMI_GENERIC5_CONT, 0,
+ HDMI_GENERIC5_SEND, 0);
+ REG_SET_2(HDMI_GENERIC_PACKET_CONTROL3, 0,
+ HDMI_GENERIC4_LINE, 0,
+ HDMI_GENERIC5_LINE, 0);
+
+ /* stop generic packets 6,7 on HDMI */
+ REG_SET_4(HDMI_GENERIC_PACKET_CONTROL0, 0,
+ HDMI_GENERIC6_CONT, 0,
+ HDMI_GENERIC6_SEND, 0,
+ HDMI_GENERIC7_CONT, 0,
+ HDMI_GENERIC7_SEND, 0);
+ REG_SET_2(HDMI_GENERIC_PACKET_CONTROL4, 0,
+ HDMI_GENERIC6_LINE, 0,
+ HDMI_GENERIC7_LINE, 0);
+}
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+
+
+/* Update GSP7 SDP 128 byte long */
+static void enc2_send_gsp7_128_info_packet(
+ struct dcn10_stream_encoder *enc1,
+ const struct dc_info_packet_128 *info_packet)
+{
+ uint32_t i;
+
+ /* TODOFPGA Figure out a proper number for max_retries polling for lock
+ * use 50 for now.
+ */
+ uint32_t max_retries = 50;
+ const uint32_t *content = (const uint32_t *) &info_packet->sb[0];
+
+ ASSERT(info_packet->hb1 == DC_DP_INFOFRAME_TYPE_PPS);
+
+ /* Configure for PPS packet size (128 bytes) */
+ REG_UPDATE(DP_SEC_CNTL2, DP_SEC_GSP7_PPS, 1);
+
+ /* We need turn on clock before programming AFMT block*/
+ REG_UPDATE(AFMT_CNTL, AFMT_AUDIO_CLOCK_EN, 1);
+
+ /* Poll dig_update_lock is not locked -> asic internal signal
+ * assumes otg master lock will unlock it
+ */
+ /*REG_WAIT(AFMT_VBI_PACKET_CONTROL, AFMT_GENERIC_LOCK_STATUS, 0, 10, max_retries);*/
+
+ /* Wait for HW/SW GSP memory access conflict to go away */
+ REG_WAIT(AFMT_VBI_PACKET_CONTROL, AFMT_GENERIC_CONFLICT,
+ 0, 10, max_retries);
+
+ /* Clear HW/SW memory access conflict flag */
+ REG_UPDATE(AFMT_VBI_PACKET_CONTROL, AFMT_GENERIC_CONFLICT_CLR, 1);
+
+ /* write generic packet header */
+ REG_UPDATE(AFMT_VBI_PACKET_CONTROL, AFMT_GENERIC_INDEX, 7);
+ REG_SET_4(AFMT_GENERIC_HDR, 0,
+ AFMT_GENERIC_HB0, info_packet->hb0,
+ AFMT_GENERIC_HB1, info_packet->hb1,
+ AFMT_GENERIC_HB2, info_packet->hb2,
+ AFMT_GENERIC_HB3, info_packet->hb3);
+
+ /* Write generic packet content 128 bytes long. Four sets are used (indexes 7
+ * through 10) to fit 128 bytes.
+ */
+ for (i = 0; i < 4; i++) {
+ uint32_t packet_index = 7 + i;
+ REG_UPDATE(AFMT_VBI_PACKET_CONTROL, AFMT_GENERIC_INDEX, packet_index);
+
+ REG_WRITE(AFMT_GENERIC_0, *content++);
+ REG_WRITE(AFMT_GENERIC_1, *content++);
+ REG_WRITE(AFMT_GENERIC_2, *content++);
+ REG_WRITE(AFMT_GENERIC_3, *content++);
+ REG_WRITE(AFMT_GENERIC_4, *content++);
+ REG_WRITE(AFMT_GENERIC_5, *content++);
+ REG_WRITE(AFMT_GENERIC_6, *content++);
+ REG_WRITE(AFMT_GENERIC_7, *content++);
+ }
+
+ REG_UPDATE(AFMT_VBI_PACKET_CONTROL1, AFMT_GENERIC7_FRAME_UPDATE, 1);
+}
+
+/* Set DSC-related configuration.
+ * dsc_mode: 0 disables DSC, other values enable DSC in specified format
+ * sc_bytes_per_pixel: Bytes per pixel in u3.28 format
+ * dsc_slice_width: Slice width in pixels
+ */
+static void enc2_dp_set_dsc_config(struct stream_encoder *enc,
+ enum optc_dsc_mode dsc_mode,
+ uint32_t dsc_bytes_per_pixel,
+ uint32_t dsc_slice_width,
+ uint8_t *dsc_packed_pps)
+{
+ struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+ uint32_t dsc_value = 0;
+
+ dsc_value = REG_READ(DP_DSC_CNTL);
+
+ /* dsc disable skip */
+ if ((dsc_value & 0x3) == 0x0)
+ return;
+
+
+ REG_UPDATE_2(DP_DSC_CNTL,
+ DP_DSC_MODE, dsc_mode,
+ DP_DSC_SLICE_WIDTH, dsc_slice_width);
+
+ REG_SET(DP_DSC_BYTES_PER_PIXEL, 0,
+ DP_DSC_BYTES_PER_PIXEL, dsc_bytes_per_pixel);
+
+ if (dsc_mode != OPTC_DSC_DISABLED) {
+ struct dc_info_packet_128 pps_sdp;
+
+ ASSERT(dsc_packed_pps);
+
+ /* Load PPS into infoframe (SDP) registers */
+ pps_sdp.valid = true;
+ pps_sdp.hb0 = 0;
+ pps_sdp.hb1 = DC_DP_INFOFRAME_TYPE_PPS;
+ pps_sdp.hb2 = 127;
+ pps_sdp.hb3 = 0;
+ memcpy(&pps_sdp.sb[0], dsc_packed_pps, sizeof(pps_sdp.sb));
+ enc2_send_gsp7_128_info_packet(enc1, &pps_sdp);
+
+ /* Enable Generic Stream Packet 7 (GSP) transmission */
+ //REG_UPDATE(DP_SEC_CNTL,
+ // DP_SEC_GSP7_ENABLE, 1);
+
+ /* SW should make sure VBID[6] update line number is bigger
+ * than PPS transmit line number
+ */
+ REG_UPDATE(DP_SEC_CNTL6,
+ DP_SEC_GSP7_LINE_NUM, 2);
+ REG_UPDATE_2(DP_MSA_VBID_MISC,
+ DP_VBID6_LINE_REFERENCE, 0,
+ DP_VBID6_LINE_NUM, 3);
+
+ /* Send PPS data at the line number specified above.
+ * DP spec requires PPS to be sent only when it changes, however since
+ * decoder has to be able to handle its change on every frame, we're
+ * sending it always (i.e. on every frame) to reduce the chance it'd be
+ * missed by decoder. If it turns out required to send PPS only when it
+ * changes, we can use DP_SEC_GSP7_SEND register.
+ */
+ REG_UPDATE_2(DP_SEC_CNTL,
+ DP_SEC_GSP7_ENABLE, 1,
+ DP_SEC_STREAM_ENABLE, 1);
+ } else {
+ /* Disable Generic Stream Packet 7 (GSP) transmission */
+ REG_UPDATE(DP_SEC_CNTL, DP_SEC_GSP7_ENABLE, 0);
+ REG_UPDATE(DP_SEC_CNTL2, DP_SEC_GSP7_PPS, 0);
+ }
+}
+#endif
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+/* this function read dsc related register fields to be logged later in dcn10_log_hw_state
+ * into a dcn_dsc_state struct.
+ */
+static void enc2_read_state(struct stream_encoder *enc, struct enc_state *s)
+{
+ struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+ //if dsc is enabled, continue to read
+ REG_GET(DP_DSC_CNTL, DP_DSC_MODE, &s->dsc_mode);
+ if (s->dsc_mode) {
+ REG_GET(DP_DSC_CNTL, DP_DSC_SLICE_WIDTH, &s->dsc_slice_width);
+ REG_GET(DP_SEC_CNTL6, DP_SEC_GSP7_LINE_NUM, &s->sec_gsp_pps_line_num);
+
+ REG_GET(DP_MSA_VBID_MISC, DP_VBID6_LINE_REFERENCE, &s->vbid6_line_reference);
+ REG_GET(DP_MSA_VBID_MISC, DP_VBID6_LINE_NUM, &s->vbid6_line_num);
+
+ REG_GET(DP_SEC_CNTL, DP_SEC_GSP7_ENABLE, &s->sec_gsp_pps_enable);
+ REG_GET(DP_SEC_CNTL, DP_SEC_STREAM_ENABLE, &s->sec_stream_enable);
+ }
+}
+#endif
+
+/* Set Dynamic Metadata-configuration.
+ * enable_dme: TRUE: enables Dynamic Metadata Enfine, FALSE: disables DME
+ * hubp_requestor_id: HUBP physical instance that is the source of dynamic metadata
+ * only needs to be set when enable_dme is TRUE
+ * dmdata_mode: dynamic metadata packet type: DP, HDMI, or Dolby Vision
+ *
+ * Ensure the OTG master update lock is set when changing DME configuration.
+ */
+static void enc2_set_dynamic_metadata(struct stream_encoder *enc,
+ bool enable_dme,
+ uint32_t hubp_requestor_id,
+ enum dynamic_metadata_mode dmdata_mode)
+{
+ struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+ if (enable_dme) {
+ REG_UPDATE_2(DME_CONTROL,
+ METADATA_HUBP_REQUESTOR_ID, hubp_requestor_id,
+ METADATA_STREAM_TYPE, (dmdata_mode == dmdata_dolby_vision) ? 1 : 0);
+
+ /* Use default line reference DP_SOF for bringup.
+ * Should use OTG_SOF for DRR cases
+ */
+ if (dmdata_mode == dmdata_dp)
+ REG_UPDATE_3(DP_SEC_METADATA_TRANSMISSION,
+ DP_SEC_METADATA_PACKET_ENABLE, 1,
+ DP_SEC_METADATA_PACKET_LINE_REFERENCE, 0,
+ DP_SEC_METADATA_PACKET_LINE, 20);
+ else {
+ REG_UPDATE_3(HDMI_METADATA_PACKET_CONTROL,
+ HDMI_METADATA_PACKET_ENABLE, 1,
+ HDMI_METADATA_PACKET_LINE_REFERENCE, 0,
+ HDMI_METADATA_PACKET_LINE, 2);
+
+ if (dmdata_mode == dmdata_dolby_vision)
+ REG_UPDATE(DIG_FE_CNTL,
+ DOLBY_VISION_EN, 1);
+ }
+
+ REG_UPDATE(DME_CONTROL,
+ METADATA_ENGINE_EN, 1);
+ } else {
+ REG_UPDATE(DME_CONTROL,
+ METADATA_ENGINE_EN, 0);
+
+ if (dmdata_mode == dmdata_dp)
+ REG_UPDATE(DP_SEC_METADATA_TRANSMISSION,
+ DP_SEC_METADATA_PACKET_ENABLE, 0);
+ else {
+ REG_UPDATE(HDMI_METADATA_PACKET_CONTROL,
+ HDMI_METADATA_PACKET_ENABLE, 0);
+ REG_UPDATE(DIG_FE_CNTL,
+ DOLBY_VISION_EN, 0);
+ }
+ }
+}
+
+static void enc2_stream_encoder_update_dp_info_packets(
+ struct stream_encoder *enc,
+ const struct encoder_info_frame *info_frame)
+{
+ struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+ uint32_t dmdata_packet_enabled = 0;
+
+ enc1_stream_encoder_update_dp_info_packets(enc, info_frame);
+
+ /* check if dynamic metadata packet transmission is enabled */
+ REG_GET(DP_SEC_METADATA_TRANSMISSION,
+ DP_SEC_METADATA_PACKET_ENABLE, &dmdata_packet_enabled);
+
+ if (dmdata_packet_enabled)
+ REG_UPDATE(DP_SEC_CNTL, DP_SEC_STREAM_ENABLE, 1);
+}
+
+static bool is_two_pixels_per_containter(const struct dc_crtc_timing *timing)
+{
+ bool two_pix = timing->pixel_encoding == PIXEL_ENCODING_YCBCR420;
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ two_pix = two_pix || (timing->flags.DSC && timing->pixel_encoding == PIXEL_ENCODING_YCBCR422
+ && !timing->dsc_cfg.ycbcr422_simple);
+#endif
+ return two_pix;
+}
+
+void enc2_stream_encoder_dp_unblank(
+ struct stream_encoder *enc,
+ const struct encoder_unblank_param *param)
+{
+ struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+ if (param->link_settings.link_rate != LINK_RATE_UNKNOWN) {
+ uint32_t n_vid = 0x8000;
+ uint32_t m_vid;
+ uint32_t n_multiply = 0;
+ uint64_t m_vid_l = n_vid;
+
+ /* YCbCr 4:2:0 : Computed VID_M will be 2X the input rate */
+ if (is_two_pixels_per_containter(&param->timing) || param->odm) {
+ /*this logic should be the same in get_pixel_clock_parameters() */
+ n_multiply = 1;
+ }
+ /* M / N = Fstream / Flink
+ * m_vid / n_vid = pixel rate / link rate
+ */
+
+ m_vid_l *= param->timing.pix_clk_100hz / 10;
+ m_vid_l = div_u64(m_vid_l,
+ param->link_settings.link_rate
+ * LINK_RATE_REF_FREQ_IN_KHZ);
+
+ m_vid = (uint32_t) m_vid_l;
+
+ /* enable auto measurement */
+
+ REG_UPDATE(DP_VID_TIMING, DP_VID_M_N_GEN_EN, 0);
+
+ /* auto measurement need 1 full 0x8000 symbol cycle to kick in,
+ * therefore program initial value for Mvid and Nvid
+ */
+
+ REG_UPDATE(DP_VID_N, DP_VID_N, n_vid);
+
+ REG_UPDATE(DP_VID_M, DP_VID_M, m_vid);
+
+ REG_UPDATE_2(DP_VID_TIMING,
+ DP_VID_M_N_GEN_EN, 1,
+ DP_VID_N_MUL, n_multiply);
+ }
+
+ /* set DIG_START to 0x1 to reset FIFO */
+
+ REG_UPDATE(DIG_FE_CNTL, DIG_START, 1);
+
+ /* write 0 to take the FIFO out of reset */
+
+ REG_UPDATE(DIG_FE_CNTL, DIG_START, 0);
+
+ /* switch DP encoder to CRTC data */
+
+ REG_UPDATE(DP_STEER_FIFO, DP_STEER_FIFO_RESET, 0);
+
+ /* wait 100us for DIG/DP logic to prime
+ * (i.e. a few video lines)
+ */
+ udelay(100);
+
+ /* the hardware would start sending video at the start of the next DP
+ * frame (i.e. rising edge of the vblank).
+ * NOTE: We used to program DP_VID_STREAM_DIS_DEFER = 2 here, but this
+ * register has no effect on enable transition! HW always guarantees
+ * VID_STREAM enable at start of next frame, and this is not
+ * programmable
+ */
+
+ REG_UPDATE(DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, true);
+}
+
+static void enc2_dp_set_odm_combine(
+ struct stream_encoder *enc,
+ bool odm_combine)
+{
+ struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+ REG_UPDATE(DP_PIXEL_FORMAT, DP_PIXEL_COMBINE, odm_combine);
+}
+
+void enc2_stream_encoder_dp_set_stream_attribute(
+ struct stream_encoder *enc,
+ struct dc_crtc_timing *crtc_timing,
+ enum dc_color_space output_color_space,
+ uint32_t enable_sdp_splitting)
+{
+ struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
+
+ enc1_stream_encoder_dp_set_stream_attribute(enc, crtc_timing, output_color_space, enable_sdp_splitting);
+
+ REG_UPDATE(DP_SEC_FRAMING4,
+ DP_SST_SDP_SPLITTING, enable_sdp_splitting);
+}
+
+static const struct stream_encoder_funcs dcn20_str_enc_funcs = {
+ .dp_set_odm_combine =
+ enc2_dp_set_odm_combine,
+ .dp_set_stream_attribute =
+ enc2_stream_encoder_dp_set_stream_attribute,
+ .hdmi_set_stream_attribute =
+ enc1_stream_encoder_hdmi_set_stream_attribute,
+ .dvi_set_stream_attribute =
+ enc1_stream_encoder_dvi_set_stream_attribute,
+ .set_mst_bandwidth =
+ enc1_stream_encoder_set_mst_bandwidth,
+ .update_hdmi_info_packets =
+ enc2_stream_encoder_update_hdmi_info_packets,
+ .stop_hdmi_info_packets =
+ enc2_stream_encoder_stop_hdmi_info_packets,
+ .update_dp_info_packets =
+ enc2_stream_encoder_update_dp_info_packets,
+ .stop_dp_info_packets =
+ enc1_stream_encoder_stop_dp_info_packets,
+ .dp_blank =
+ enc1_stream_encoder_dp_blank,
+ .dp_unblank =
+ enc2_stream_encoder_dp_unblank,
+ .audio_mute_control = enc1_se_audio_mute_control,
+
+ .dp_audio_setup = enc1_se_dp_audio_setup,
+ .dp_audio_enable = enc1_se_dp_audio_enable,
+ .dp_audio_disable = enc1_se_dp_audio_disable,
+
+ .hdmi_audio_setup = enc1_se_hdmi_audio_setup,
+ .hdmi_audio_disable = enc1_se_hdmi_audio_disable,
+ .setup_stereo_sync = enc1_setup_stereo_sync,
+ .set_avmute = enc1_stream_encoder_set_avmute,
+ .dig_connect_to_otg = enc1_dig_connect_to_otg,
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ .enc_read_state = enc2_read_state,
+#endif
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ .dp_set_dsc_config = enc2_dp_set_dsc_config,
+#endif
+ .set_dynamic_metadata = enc2_set_dynamic_metadata,
+};
+
+void dcn20_stream_encoder_construct(
+ struct dcn10_stream_encoder *enc1,
+ struct dc_context *ctx,
+ struct dc_bios *bp,
+ enum engine_id eng_id,
+ const struct dcn10_stream_enc_registers *regs,
+ const struct dcn10_stream_encoder_shift *se_shift,
+ const struct dcn10_stream_encoder_mask *se_mask)
+{
+ enc1->base.funcs = &dcn20_str_enc_funcs;
+ enc1->base.ctx = ctx;
+ enc1->base.id = eng_id;
+ enc1->base.bp = bp;
+ enc1->regs = regs;
+ enc1->se_shift = se_shift;
+ enc1->se_mask = se_mask;
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_stream_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_stream_encoder.h
new file mode 100644
index 000000000000..6d40e8c9b78f
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_stream_encoder.h
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_STREAM_ENCODER_DCN20_H__
+#define __DC_STREAM_ENCODER_DCN20_H__
+
+#include "stream_encoder.h"
+#include "dcn10/dcn10_stream_encoder.h"
+
+
+#define SE_DCN2_REG_LIST(id)\
+ SE_COMMON_DCN_REG_LIST(id),\
+ SRI(HDMI_GENERIC_PACKET_CONTROL4, DIG, id), \
+ SRI(HDMI_GENERIC_PACKET_CONTROL5, DIG, id), \
+ SRI(DP_DSC_CNTL, DP, id), \
+ SRI(DP_DSC_BYTES_PER_PIXEL, DP, id), \
+ SRI(DME_CONTROL, DIG, id),\
+ SRI(DP_SEC_METADATA_TRANSMISSION, DP, id), \
+ SRI(HDMI_METADATA_PACKET_CONTROL, DIG, id), \
+ SRI(DP_SEC_FRAMING4, DP, id)
+
+#define SE_COMMON_MASK_SH_LIST_DCN20(mask_sh)\
+ SE_COMMON_MASK_SH_LIST_SOC(mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC0_CONT, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC0_SEND, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC1_CONT, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC1_SEND, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC2_CONT, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC2_SEND, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC3_CONT, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC3_SEND, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC4_CONT, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC4_SEND, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC5_CONT, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC5_SEND, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC6_CONT, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC6_SEND, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC7_CONT, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL0, HDMI_GENERIC7_SEND, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL1, HDMI_GENERIC0_LINE, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL1, HDMI_GENERIC1_LINE, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL2, HDMI_GENERIC2_LINE, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL2, HDMI_GENERIC3_LINE, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL3, HDMI_GENERIC4_LINE, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL3, HDMI_GENERIC5_LINE, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL4, HDMI_GENERIC6_LINE, mask_sh),\
+ SE_SF(DIG0_HDMI_GENERIC_PACKET_CONTROL4, HDMI_GENERIC7_LINE, mask_sh),\
+ SE_SF(DP0_DP_DSC_CNTL, DP_DSC_MODE, mask_sh),\
+ SE_SF(DP0_DP_DSC_CNTL, DP_DSC_SLICE_WIDTH, mask_sh),\
+ SE_SF(DP0_DP_DSC_BYTES_PER_PIXEL, DP_DSC_BYTES_PER_PIXEL, mask_sh),\
+ SE_SF(DP0_DP_MSA_VBID_MISC, DP_VBID6_LINE_REFERENCE, mask_sh),\
+ SE_SF(DP0_DP_MSA_VBID_MISC, DP_VBID6_LINE_NUM, mask_sh),\
+ SE_SF(DIG0_DME_CONTROL, METADATA_ENGINE_EN, mask_sh),\
+ SE_SF(DIG0_DME_CONTROL, METADATA_HUBP_REQUESTOR_ID, mask_sh),\
+ SE_SF(DIG0_DME_CONTROL, METADATA_STREAM_TYPE, mask_sh),\
+ SE_SF(DP0_DP_SEC_METADATA_TRANSMISSION, DP_SEC_METADATA_PACKET_ENABLE, mask_sh),\
+ SE_SF(DP0_DP_SEC_METADATA_TRANSMISSION, DP_SEC_METADATA_PACKET_LINE_REFERENCE, mask_sh),\
+ SE_SF(DP0_DP_SEC_METADATA_TRANSMISSION, DP_SEC_METADATA_PACKET_LINE, mask_sh),\
+ SE_SF(DIG0_HDMI_METADATA_PACKET_CONTROL, HDMI_METADATA_PACKET_ENABLE, mask_sh),\
+ SE_SF(DIG0_HDMI_METADATA_PACKET_CONTROL, HDMI_METADATA_PACKET_LINE_REFERENCE, mask_sh),\
+ SE_SF(DIG0_HDMI_METADATA_PACKET_CONTROL, HDMI_METADATA_PACKET_LINE, mask_sh),\
+ SE_SF(DIG0_DIG_FE_CNTL, DOLBY_VISION_EN, mask_sh),\
+ SE_SF(DP0_DP_PIXEL_FORMAT, DP_PIXEL_COMBINE, mask_sh),\
+ SE_SF(DP0_DP_SEC_FRAMING4, DP_SST_SDP_SPLITTING, mask_sh)
+
+void dcn20_stream_encoder_construct(
+ struct dcn10_stream_encoder *enc1,
+ struct dc_context *ctx,
+ struct dc_bios *bp,
+ enum engine_id eng_id,
+ const struct dcn10_stream_enc_registers *regs,
+ const struct dcn10_stream_encoder_shift *se_shift,
+ const struct dcn10_stream_encoder_mask *se_mask);
+
+void enc2_stream_encoder_dp_set_stream_attribute(
+ struct stream_encoder *enc,
+ struct dc_crtc_timing *crtc_timing,
+ enum dc_color_space output_color_space,
+ uint32_t enable_sdp_splitting);
+
+void enc2_stream_encoder_dp_unblank(
+ struct stream_encoder *enc,
+ const struct encoder_unblank_param *param);
+
+#endif /* __DC_STREAM_ENCODER_DCN20_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_vmid.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_vmid.c
new file mode 100644
index 000000000000..96c263223315
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_vmid.c
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include <linux/delay.h>
+
+#include "dcn20_vmid.h"
+#include "reg_helper.h"
+
+#define REG(reg)\
+ vmid->regs->reg
+
+#define CTX \
+ vmid->ctx
+
+#undef FN
+#define FN(reg_name, field_name) \
+ vmid->shifts->field_name, vmid->masks->field_name
+
+static void dcn20_wait_for_vmid_ready(struct dcn20_vmid *vmid)
+{
+ /* According the hardware spec, we need to poll for the lowest
+ * bit of PAGE_TABLE_BASE_ADDR_LO32 = 1 any time a GPUVM
+ * context is updated. We can't use REG_WAIT here since we
+ * don't have a seperate field to wait on.
+ *
+ * TODO: Confirm timeout / poll interval with hardware team
+ */
+
+ int max_times = 10000;
+ int delay_us = 5;
+ int i;
+
+ for (i = 0; i < max_times; ++i) {
+ uint32_t entry_lo32;
+
+ REG_GET(PAGE_TABLE_BASE_ADDR_LO32,
+ VM_CONTEXT0_PAGE_DIRECTORY_ENTRY_LO32,
+ &entry_lo32);
+
+ if (entry_lo32 & 0x1)
+ return;
+
+ udelay(delay_us);
+ }
+
+ /* VM setup timed out */
+ DC_LOG_WARNING("Timeout while waiting for GPUVM context update\n");
+ ASSERT(0);
+}
+
+void dcn20_vmid_setup(struct dcn20_vmid *vmid, const struct dcn_vmid_page_table_config *config)
+{
+ REG_SET(PAGE_TABLE_START_ADDR_HI32, 0,
+ VM_CONTEXT0_START_LOGICAL_PAGE_NUMBER_HI4, (config->page_table_start_addr >> 32) & 0xF);
+ REG_SET(PAGE_TABLE_START_ADDR_LO32, 0,
+ VM_CONTEXT0_START_LOGICAL_PAGE_NUMBER_LO32, config->page_table_start_addr & 0xFFFFFFFF);
+
+ REG_SET(PAGE_TABLE_END_ADDR_HI32, 0,
+ VM_CONTEXT0_END_LOGICAL_PAGE_NUMBER_HI4, (config->page_table_end_addr >> 32) & 0xF);
+ REG_SET(PAGE_TABLE_END_ADDR_LO32, 0,
+ VM_CONTEXT0_END_LOGICAL_PAGE_NUMBER_LO32, config->page_table_end_addr & 0xFFFFFFFF);
+
+ REG_SET_2(CNTL, 0,
+ VM_CONTEXT0_PAGE_TABLE_DEPTH, config->depth,
+ VM_CONTEXT0_PAGE_TABLE_BLOCK_SIZE, config->block_size);
+
+ REG_SET(PAGE_TABLE_BASE_ADDR_HI32, 0,
+ VM_CONTEXT0_PAGE_DIRECTORY_ENTRY_HI32, (config->page_table_base_addr >> 32) & 0xFFFFFFFF);
+ /* Note: per hardware spec PAGE_TABLE_BASE_ADDR_LO32 must be programmed last in sequence */
+ REG_SET(PAGE_TABLE_BASE_ADDR_LO32, 0,
+ VM_CONTEXT0_PAGE_DIRECTORY_ENTRY_LO32, config->page_table_base_addr & 0xFFFFFFFF);
+
+ dcn20_wait_for_vmid_ready(vmid);
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_vmid.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_vmid.h
new file mode 100644
index 000000000000..02fafb013fc6
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_vmid.h
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef DAL_DC_DCN20_DCN20_VMID_H_
+#define DAL_DC_DCN20_DCN20_VMID_H_
+
+#include "vmid.h"
+
+#define BASE_INNER(seg) \
+ DCE_BASE__INST0_SEG ## seg
+
+#define BASE(seg) \
+ BASE_INNER(seg)
+
+#define SRI(reg_name, block, id)\
+ .reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+ mm ## block ## id ## _ ## reg_name
+
+#define SF(reg_name, field_name, post_fix)\
+ .field_name = reg_name ## __ ## field_name ## post_fix
+
+#define DCN20_VMID_REG_LIST(id)\
+ SRI(CNTL, DCN_VM_CONTEXT, id),\
+ SRI(PAGE_TABLE_BASE_ADDR_HI32, DCN_VM_CONTEXT, id),\
+ SRI(PAGE_TABLE_BASE_ADDR_LO32, DCN_VM_CONTEXT, id),\
+ SRI(PAGE_TABLE_START_ADDR_HI32, DCN_VM_CONTEXT, id),\
+ SRI(PAGE_TABLE_START_ADDR_LO32, DCN_VM_CONTEXT, id),\
+ SRI(PAGE_TABLE_END_ADDR_HI32, DCN_VM_CONTEXT, id),\
+ SRI(PAGE_TABLE_END_ADDR_LO32, DCN_VM_CONTEXT, id)
+
+#define DCN20_VMID_MASK_SH_LIST(mask_sh)\
+ SF(DCN_VM_CONTEXT0_CNTL, VM_CONTEXT0_PAGE_TABLE_DEPTH, mask_sh),\
+ SF(DCN_VM_CONTEXT0_CNTL, VM_CONTEXT0_PAGE_TABLE_BLOCK_SIZE, mask_sh),\
+ SF(DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, VM_CONTEXT0_PAGE_DIRECTORY_ENTRY_HI32, mask_sh),\
+ SF(DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, VM_CONTEXT0_PAGE_DIRECTORY_ENTRY_LO32, mask_sh),\
+ SF(DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, VM_CONTEXT0_START_LOGICAL_PAGE_NUMBER_HI4, mask_sh),\
+ SF(DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, VM_CONTEXT0_START_LOGICAL_PAGE_NUMBER_LO32, mask_sh),\
+ SF(DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, VM_CONTEXT0_END_LOGICAL_PAGE_NUMBER_HI4, mask_sh),\
+ SF(DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, VM_CONTEXT0_END_LOGICAL_PAGE_NUMBER_LO32, mask_sh)
+
+#define DCN20_VMID_REG_FIELD_LIST(type)\
+ type VM_CONTEXT0_PAGE_TABLE_DEPTH;\
+ type VM_CONTEXT0_PAGE_TABLE_BLOCK_SIZE;\
+ type VM_CONTEXT0_PAGE_DIRECTORY_ENTRY_HI32;\
+ type VM_CONTEXT0_PAGE_DIRECTORY_ENTRY_LO32;\
+ type VM_CONTEXT0_START_LOGICAL_PAGE_NUMBER_HI4;\
+ type VM_CONTEXT0_START_LOGICAL_PAGE_NUMBER_LO32;\
+ type VM_CONTEXT0_END_LOGICAL_PAGE_NUMBER_HI4;\
+ type VM_CONTEXT0_END_LOGICAL_PAGE_NUMBER_LO32
+
+struct dcn20_vmid_shift {
+ DCN20_VMID_REG_FIELD_LIST(uint8_t);
+};
+
+struct dcn20_vmid_mask {
+ DCN20_VMID_REG_FIELD_LIST(uint32_t);
+};
+
+struct dcn20_vmid {
+ struct dc_context *ctx;
+ const struct dcn_vmid_registers *regs;
+ const struct dcn20_vmid_shift *shifts;
+ const struct dcn20_vmid_mask *masks;
+};
+
+void dcn20_vmid_setup(struct dcn20_vmid *vmid, const struct dcn_vmid_page_table_config *config);
+
+#endif /* DAL_DC_DCN20_DCN20_VMID_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dm_helpers.h b/drivers/gpu/drm/amd/display/dc/dm_helpers.h
index ccbfe9680d27..b6b4333737f2 100644
--- a/drivers/gpu/drm/amd/display/dc/dm_helpers.h
+++ b/drivers/gpu/drm/amd/display/dc/dm_helpers.h
@@ -118,6 +118,13 @@ bool dm_helpers_submit_i2c(
const struct dc_link *link,
struct i2c_command *cmd);
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+bool dm_helpers_dp_write_dsc_enable(
+ struct dc_context *ctx,
+ const struct dc_stream_state *stream,
+ bool enable
+);
+#endif
bool dm_helpers_is_dp_sink_present(
struct dc_link *link);
diff --git a/drivers/gpu/drm/amd/display/dc/dm_pp_smu.h b/drivers/gpu/drm/amd/display/dc/dm_pp_smu.h
index 4fc4208d1472..680689cab5dd 100644
--- a/drivers/gpu/drm/amd/display/dc/dm_pp_smu.h
+++ b/drivers/gpu/drm/amd/display/dc/dm_pp_smu.h
@@ -41,6 +41,10 @@ enum pp_smu_ver {
*/
PP_SMU_UNSUPPORTED,
PP_SMU_VER_RV,
+#ifndef CONFIG_TRIM_DRM_AMD_DC_DCN2_0
+ PP_SMU_VER_NV,
+#endif
+
PP_SMU_VER_MAX
};
@@ -56,12 +60,30 @@ struct pp_smu {
const void *dm;
};
+enum pp_smu_status {
+ PP_SMU_RESULT_UNDEFINED = 0,
+ PP_SMU_RESULT_OK = 1,
+ PP_SMU_RESULT_FAIL,
+ PP_SMU_RESULT_UNSUPPORTED
+};
+
+#define PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN 0x0
+#define PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX 0xFFFF
+
+enum wm_type {
+ WM_TYPE_PSTATE_CHG = 0,
+ WM_TYPE_RETRAINING = 1,
+};
+
+/* This structure is a copy of WatermarkRowGeneric_t defined by smuxx_driver_if.h*/
struct pp_smu_wm_set_range {
- unsigned int wm_inst;
- uint32_t min_fill_clk_mhz;
- uint32_t max_fill_clk_mhz;
- uint32_t min_drain_clk_mhz;
- uint32_t max_drain_clk_mhz;
+ uint16_t min_fill_clk_mhz;
+ uint16_t max_fill_clk_mhz;
+ uint16_t min_drain_clk_mhz;
+ uint16_t max_drain_clk_mhz;
+
+ uint8_t wm_inst;
+ uint8_t wm_type;
};
#define MAX_WATERMARK_SETS 4
@@ -80,6 +102,7 @@ struct pp_smu_funcs_rv {
/* PPSMC_MSG_SetDisplayCount
* 0 triggers S0i2 optimization
*/
+
void (*set_display_count)(struct pp_smu *pp, int count);
/* reader and writer WM's are sent together as part of one table*/
@@ -115,13 +138,122 @@ struct pp_smu_funcs_rv {
/* PME w/a */
void (*set_pme_wa_enable)(struct pp_smu *pp);
+};
+
+#ifndef CONFIG_TRIM_DRM_AMD_DC_DCN2_0
+/* Used by pp_smu_funcs_nv.set_voltage_by_freq
+ *
+ */
+enum pp_smu_nv_clock_id {
+ PP_SMU_NV_DISPCLK,
+ PP_SMU_NV_PHYCLK,
+ PP_SMU_NV_PIXELCLK
+};
+
+/*
+ * Used by pp_smu_funcs_nv.get_maximum_sustainable_clocks
+ */
+struct pp_smu_nv_clock_table {
+ // voltage managed SMU, freq set by driver
+ unsigned int displayClockInKhz;
+ unsigned int dppClockInKhz;
+ unsigned int phyClockInKhz;
+ unsigned int pixelClockInKhz;
+ unsigned int dscClockInKhz;
+ // freq/voltage managed by SMU
+ unsigned int fabricClockInKhz;
+ unsigned int socClockInKhz;
+ unsigned int dcfClockInKhz;
+ unsigned int uClockInKhz;
};
+struct pp_smu_funcs_nv {
+ struct pp_smu pp_smu;
+
+ /* PPSMC_MSG_SetDisplayCount
+ * 0 triggers S0i2 optimization
+ */
+ enum pp_smu_status (*set_display_count)(struct pp_smu *pp, int count);
+
+ /* PPSMC_MSG_SetHardMinDcfclkByFreq
+ * fixed clock at requested freq, either from FCH bypass or DFS
+ */
+ enum pp_smu_status (*set_hard_min_dcfclk_by_freq)(struct pp_smu *pp, int Mhz);
+
+ /* PPSMC_MSG_SetMinDeepSleepDcfclk
+ * when DF is in cstate, dcf clock is further divided down
+ * to just above given frequency
+ */
+ enum pp_smu_status (*set_min_deep_sleep_dcfclk)(struct pp_smu *pp, int Mhz);
+
+ /* PPSMC_MSG_SetHardMinUclkByFreq
+ * UCLK will vary with DPM, but never below requested hard min
+ */
+ enum pp_smu_status (*set_hard_min_uclk_by_freq)(struct pp_smu *pp, int Mhz);
+
+ /* PPSMC_MSG_SetHardMinSocclkByFreq
+ * Needed for DWB support
+ */
+ enum pp_smu_status (*set_hard_min_socclk_by_freq)(struct pp_smu *pp, int Mhz);
+
+ /* PME w/a */
+ enum pp_smu_status (*set_pme_wa_enable)(struct pp_smu *pp);
+
+ /* PPSMC_MSG_SetHardMinByFreq
+ * Needed to set ASIC voltages for clocks programmed by DAL
+ */
+ enum pp_smu_status (*set_voltage_by_freq)(struct pp_smu *pp,
+ enum pp_smu_nv_clock_id clock_id, int Mhz);
+
+ /* reader and writer WM's are sent together as part of one table*/
+ /*
+ * PPSMC_MSG_SetDriverDramAddrHigh
+ * PPSMC_MSG_SetDriverDramAddrLow
+ * PPSMC_MSG_TransferTableDram2Smu
+ *
+ * on DCN20:
+ * reader fill clk = uclk
+ * reader drain clk = dcfclk
+ * writer fill clk = socclk
+ * writer drain clk = uclk
+ * */
+ enum pp_smu_status (*set_wm_ranges)(struct pp_smu *pp,
+ struct pp_smu_wm_range_sets *ranges);
+
+ /* Not a single SMU message. This call should return maximum sustainable limit for all
+ * clocks that DC depends on. These will be used as basis for mode enumeration.
+ */
+ enum pp_smu_status (*get_maximum_sustainable_clocks)(struct pp_smu *pp,
+ struct pp_smu_nv_clock_table *max_clocks);
+
+ /* This call should return the discrete uclk DPM states available
+ */
+ enum pp_smu_status (*get_uclk_dpm_states)(struct pp_smu *pp,
+ unsigned int *clock_values_in_khz, unsigned int *num_states);
+
+ /* Not a single SMU message. This call informs PPLIB that display will not be able
+ * to perform pstate handshaking in its current state. Typically this handshake
+ * is used to perform uCLK switching, so disabling pstate disables uCLK switching.
+ *
+ * Note that when setting handshake to unsupported, the call is pre-emptive. That means
+ * DC will make the call BEFORE setting up the display state which would cause pstate
+ * request to go un-acked. Only when the call completes should such a state be applied to
+ * DC hardware
+ */
+ enum pp_smu_status (*set_pstate_handshake_support)(struct pp_smu *pp,
+ BOOLEAN pstate_handshake_supported);
+};
+#endif
+
struct pp_smu_funcs {
struct pp_smu ctx;
union {
struct pp_smu_funcs_rv rv_funcs;
+#ifndef CONFIG_TRIM_DRM_AMD_DC_DCN2_0
+ struct pp_smu_funcs_nv nv_funcs;
+#endif
+
};
};
diff --git a/drivers/gpu/drm/amd/display/dc/dml/Makefile b/drivers/gpu/drm/amd/display/dc/dml/Makefile
index d97ca6528f9d..0bb7a20675c4 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dml/Makefile
@@ -33,7 +33,15 @@ endif
dml_ccflags := -mhard-float -msse $(cc_stack_align)
CFLAGS_display_mode_lib.o := $(dml_ccflags)
-CFLAGS_display_pipe_clocks.o := $(dml_ccflags)
+
+ifdef CONFIG_DRM_AMD_DC_DCN2_0
+CFLAGS_display_mode_vba.o := $(dml_ccflags)
+CFLAGS_display_mode_vba_20.o := $(dml_ccflags)
+CFLAGS_display_rq_dlg_calc_20.o := $(dml_ccflags)
+endif
+ifdef CONFIG_DRM_AMD_DCN3AG
+CFLAGS_display_mode_vba_3ag.o := $(dml_ccflags)
+endif
CFLAGS_dml1_display_rq_dlg_calc.o := $(dml_ccflags)
CFLAGS_display_rq_dlg_helpers.o := $(dml_ccflags)
CFLAGS_dml_common_defs.o := $(dml_ccflags)
@@ -41,6 +49,10 @@ CFLAGS_dml_common_defs.o := $(dml_ccflags)
DML = display_mode_lib.o display_rq_dlg_helpers.o dml1_display_rq_dlg_calc.o \
dml_common_defs.o
+ifdef CONFIG_DRM_AMD_DC_DCN2_0
+DML += display_mode_vba.o dcn20/display_rq_dlg_calc_20.o dcn20/display_mode_vba_20.o
+endif
+
AMD_DAL_DML = $(addprefix $(AMDDALPATH)/dc/dml/,$(DML))
AMD_DISPLAY_FILES += $(AMD_DAL_DML)
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c
new file mode 100644
index 000000000000..649883777f62
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c
@@ -0,0 +1,5104 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "../display_mode_lib.h"
+#include "display_mode_vba_20.h"
+#include "../dml_inline_defs.h"
+
+/*
+ * NOTE:
+ * This file is gcc-parseable HW gospel, coming straight from HW engineers.
+ *
+ * It doesn't adhere to Linux kernel style and sometimes will do things in odd
+ * ways. Unless there is something clearly wrong with it the code should
+ * remain as-is as it provides us with a guarantee from HW that it is correct.
+ */
+
+#define BPP_INVALID 0
+#define BPP_BLENDED_PIPE 0xffffffff
+
+static double adjust_ReturnBW(
+ struct display_mode_lib *mode_lib,
+ double ReturnBW,
+ bool DCCEnabledAnyPlane,
+ double ReturnBandwidthToDCN);
+static unsigned int dscceComputeDelay(
+ unsigned int bpc,
+ double bpp,
+ unsigned int sliceWidth,
+ unsigned int numSlices,
+ enum output_format_class pixelFormat);
+static unsigned int dscComputeDelay(enum output_format_class pixelFormat);
+// Super monster function with some 45 argument
+static bool CalculatePrefetchSchedule(
+ struct display_mode_lib *mode_lib,
+ double DPPCLK,
+ double DISPCLK,
+ double PixelClock,
+ double DCFCLKDeepSleep,
+ unsigned int DSCDelay,
+ unsigned int DPPPerPlane,
+ bool ScalerEnabled,
+ unsigned int NumberOfCursors,
+ double DPPCLKDelaySubtotal,
+ double DPPCLKDelaySCL,
+ double DPPCLKDelaySCLLBOnly,
+ double DPPCLKDelayCNVCFormater,
+ double DPPCLKDelayCNVCCursor,
+ double DISPCLKDelaySubtotal,
+ unsigned int ScalerRecoutWidth,
+ enum output_format_class OutputFormat,
+ unsigned int VBlank,
+ unsigned int HTotal,
+ unsigned int MaxInterDCNTileRepeaters,
+ unsigned int VStartup,
+ unsigned int PageTableLevels,
+ bool GPUVMEnable,
+ bool DynamicMetadataEnable,
+ unsigned int DynamicMetadataLinesBeforeActiveRequired,
+ unsigned int DynamicMetadataTransmittedBytes,
+ bool DCCEnable,
+ double UrgentLatencyPixelDataOnly,
+ double UrgentExtraLatency,
+ double TCalc,
+ unsigned int PDEAndMetaPTEBytesFrame,
+ unsigned int MetaRowByte,
+ unsigned int PixelPTEBytesPerRow,
+ double PrefetchSourceLinesY,
+ unsigned int SwathWidthY,
+ double BytePerPixelDETY,
+ double VInitPreFillY,
+ unsigned int MaxNumSwathY,
+ double PrefetchSourceLinesC,
+ double BytePerPixelDETC,
+ double VInitPreFillC,
+ unsigned int MaxNumSwathC,
+ unsigned int SwathHeightY,
+ unsigned int SwathHeightC,
+ double TWait,
+ bool XFCEnabled,
+ double XFCRemoteSurfaceFlipDelay,
+ bool InterlaceEnable,
+ bool ProgressiveToInterlaceUnitInOPP,
+ double *DSTXAfterScaler,
+ double *DSTYAfterScaler,
+ double *DestinationLinesForPrefetch,
+ double *PrefetchBandwidth,
+ double *DestinationLinesToRequestVMInVBlank,
+ double *DestinationLinesToRequestRowInVBlank,
+ double *VRatioPrefetchY,
+ double *VRatioPrefetchC,
+ double *RequiredPrefetchPixDataBW,
+ unsigned int *VStartupRequiredWhenNotEnoughTimeForDynamicMetadata,
+ double *Tno_bw,
+ unsigned int *VUpdateOffsetPix,
+ double *VUpdateWidthPix,
+ double *VReadyOffsetPix);
+static double RoundToDFSGranularityUp(double Clock, double VCOSpeed);
+static double RoundToDFSGranularityDown(double Clock, double VCOSpeed);
+static double CalculatePrefetchSourceLines(
+ struct display_mode_lib *mode_lib,
+ double VRatio,
+ double vtaps,
+ bool Interlace,
+ bool ProgressiveToInterlaceUnitInOPP,
+ unsigned int SwathHeight,
+ unsigned int ViewportYStart,
+ double *VInitPreFill,
+ unsigned int *MaxNumSwath);
+static unsigned int CalculateVMAndRowBytes(
+ struct display_mode_lib *mode_lib,
+ bool DCCEnable,
+ unsigned int BlockHeight256Bytes,
+ unsigned int BlockWidth256Bytes,
+ enum source_format_class SourcePixelFormat,
+ unsigned int SurfaceTiling,
+ unsigned int BytePerPixel,
+ enum scan_direction_class ScanDirection,
+ unsigned int ViewportWidth,
+ unsigned int ViewportHeight,
+ unsigned int SwathWidthY,
+ bool GPUVMEnable,
+ unsigned int VMMPageSize,
+ unsigned int PTEBufferSizeInRequestsLuma,
+ unsigned int PDEProcessingBufIn64KBReqs,
+ unsigned int Pitch,
+ unsigned int DCCMetaPitch,
+ unsigned int *MacroTileWidth,
+ unsigned int *MetaRowByte,
+ unsigned int *PixelPTEBytesPerRow,
+ bool *PTEBufferSizeNotExceeded,
+ unsigned int *dpte_row_height,
+ unsigned int *meta_row_height);
+static double CalculateTWait(
+ unsigned int PrefetchMode,
+ double DRAMClockChangeLatency,
+ double UrgentLatencyPixelDataOnly,
+ double SREnterPlusExitTime);
+static double CalculateRemoteSurfaceFlipDelay(
+ struct display_mode_lib *mode_lib,
+ double VRatio,
+ double SwathWidth,
+ double Bpp,
+ double LineTime,
+ double XFCTSlvVupdateOffset,
+ double XFCTSlvVupdateWidth,
+ double XFCTSlvVreadyOffset,
+ double XFCXBUFLatencyTolerance,
+ double XFCFillBWOverhead,
+ double XFCSlvChunkSize,
+ double XFCBusTransportTime,
+ double TCalc,
+ double TWait,
+ double *SrcActiveDrainRate,
+ double *TInitXFill,
+ double *TslvChk);
+static void CalculateActiveRowBandwidth(
+ bool GPUVMEnable,
+ enum source_format_class SourcePixelFormat,
+ double VRatio,
+ bool DCCEnable,
+ double LineTime,
+ unsigned int MetaRowByteLuma,
+ unsigned int MetaRowByteChroma,
+ unsigned int meta_row_height_luma,
+ unsigned int meta_row_height_chroma,
+ unsigned int PixelPTEBytesPerRowLuma,
+ unsigned int PixelPTEBytesPerRowChroma,
+ unsigned int dpte_row_height_luma,
+ unsigned int dpte_row_height_chroma,
+ double *meta_row_bw,
+ double *dpte_row_bw,
+ double *qual_row_bw);
+static void CalculateFlipSchedule(
+ struct display_mode_lib *mode_lib,
+ double UrgentExtraLatency,
+ double UrgentLatencyPixelDataOnly,
+ unsigned int GPUVMMaxPageTableLevels,
+ bool GPUVMEnable,
+ double BandwidthAvailableForImmediateFlip,
+ unsigned int TotImmediateFlipBytes,
+ enum source_format_class SourcePixelFormat,
+ unsigned int ImmediateFlipBytes,
+ double LineTime,
+ double VRatio,
+ double Tno_bw,
+ double PDEAndMetaPTEBytesFrame,
+ unsigned int MetaRowByte,
+ unsigned int PixelPTEBytesPerRow,
+ bool DCCEnable,
+ unsigned int dpte_row_height,
+ unsigned int meta_row_height,
+ double qual_row_bw,
+ double *DestinationLinesToRequestVMInImmediateFlip,
+ double *DestinationLinesToRequestRowInImmediateFlip,
+ double *final_flip_bw,
+ bool *ImmediateFlipSupportedForPipe);
+static double CalculateWriteBackDelay(
+ enum source_format_class WritebackPixelFormat,
+ double WritebackHRatio,
+ double WritebackVRatio,
+ unsigned int WritebackLumaHTaps,
+ unsigned int WritebackLumaVTaps,
+ unsigned int WritebackChromaHTaps,
+ unsigned int WritebackChromaVTaps,
+ unsigned int WritebackDestinationWidth);
+
+static void dml20_DisplayPipeConfiguration(struct display_mode_lib *mode_lib);
+static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation(
+ struct display_mode_lib *mode_lib);
+
+void dml20_recalculate(struct display_mode_lib *mode_lib)
+{
+ ModeSupportAndSystemConfiguration(mode_lib);
+ mode_lib->vba.FabricAndDRAMBandwidth = dml_min(
+ mode_lib->vba.DRAMSpeed * mode_lib->vba.NumberOfChannels * mode_lib->vba.DRAMChannelWidth,
+ mode_lib->vba.FabricClock * mode_lib->vba.FabricDatapathToDCNDataReturn) / 1000.0;
+ PixelClockAdjustmentForProgressiveToInterlaceUnit(mode_lib);
+ dml20_DisplayPipeConfiguration(mode_lib);
+ dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation(mode_lib);
+}
+
+static double adjust_ReturnBW(
+ struct display_mode_lib *mode_lib,
+ double ReturnBW,
+ bool DCCEnabledAnyPlane,
+ double ReturnBandwidthToDCN)
+{
+ double CriticalCompression;
+
+ if (DCCEnabledAnyPlane
+ && ReturnBandwidthToDCN
+ > mode_lib->vba.DCFCLK * mode_lib->vba.ReturnBusWidth / 4.0)
+ ReturnBW =
+ dml_min(
+ ReturnBW,
+ ReturnBandwidthToDCN * 4
+ * (1.0
+ - mode_lib->vba.UrgentLatencyPixelDataOnly
+ / ((mode_lib->vba.ROBBufferSizeInKByte
+ - mode_lib->vba.PixelChunkSizeInKByte)
+ * 1024
+ / ReturnBandwidthToDCN
+ - mode_lib->vba.DCFCLK
+ * mode_lib->vba.ReturnBusWidth
+ / 4)
+ + mode_lib->vba.UrgentLatencyPixelDataOnly));
+
+ CriticalCompression = 2.0 * mode_lib->vba.ReturnBusWidth * mode_lib->vba.DCFCLK
+ * mode_lib->vba.UrgentLatencyPixelDataOnly
+ / (ReturnBandwidthToDCN * mode_lib->vba.UrgentLatencyPixelDataOnly
+ + (mode_lib->vba.ROBBufferSizeInKByte
+ - mode_lib->vba.PixelChunkSizeInKByte)
+ * 1024);
+
+ if (DCCEnabledAnyPlane && CriticalCompression > 1.0 && CriticalCompression < 4.0)
+ ReturnBW =
+ dml_min(
+ ReturnBW,
+ 4.0 * ReturnBandwidthToDCN
+ * (mode_lib->vba.ROBBufferSizeInKByte
+ - mode_lib->vba.PixelChunkSizeInKByte)
+ * 1024
+ * mode_lib->vba.ReturnBusWidth
+ * mode_lib->vba.DCFCLK
+ * mode_lib->vba.UrgentLatencyPixelDataOnly
+ / dml_pow(
+ (ReturnBandwidthToDCN
+ * mode_lib->vba.UrgentLatencyPixelDataOnly
+ + (mode_lib->vba.ROBBufferSizeInKByte
+ - mode_lib->vba.PixelChunkSizeInKByte)
+ * 1024),
+ 2));
+
+ return ReturnBW;
+}
+
+static unsigned int dscceComputeDelay(
+ unsigned int bpc,
+ double bpp,
+ unsigned int sliceWidth,
+ unsigned int numSlices,
+ enum output_format_class pixelFormat)
+{
+ // valid bpc = source bits per component in the set of {8, 10, 12}
+ // valid bpp = increments of 1/16 of a bit
+ // min = 6/7/8 in N420/N422/444, respectively
+ // max = such that compression is 1:1
+ //valid sliceWidth = number of pixels per slice line, must be less than or equal to 5184/numSlices (or 4096/numSlices in 420 mode)
+ //valid numSlices = number of slices in the horiziontal direction per DSC engine in the set of {1, 2, 3, 4}
+ //valid pixelFormat = pixel/color format in the set of {:N444_RGB, :S422, :N422, :N420}
+
+ // fixed value
+ unsigned int rcModelSize = 8192;
+
+ // N422/N420 operate at 2 pixels per clock
+ unsigned int pixelsPerClock, lstall, D, initalXmitDelay, w, s, ix, wx, p, l0, a, ax, l,
+ Delay, pixels;
+
+ if (pixelFormat == dm_n422 || pixelFormat == dm_420)
+ pixelsPerClock = 2;
+ // #all other modes operate at 1 pixel per clock
+ else
+ pixelsPerClock = 1;
+
+ //initial transmit delay as per PPS
+ initalXmitDelay = dml_round(rcModelSize / 2.0 / bpp / pixelsPerClock);
+
+ //compute ssm delay
+ if (bpc == 8)
+ D = 81;
+ else if (bpc == 10)
+ D = 89;
+ else
+ D = 113;
+
+ //divide by pixel per cycle to compute slice width as seen by DSC
+ w = sliceWidth / pixelsPerClock;
+
+ //422 mode has an additional cycle of delay
+ if (pixelFormat == dm_s422)
+ s = 1;
+ else
+ s = 0;
+
+ //main calculation for the dscce
+ ix = initalXmitDelay + 45;
+ wx = (w + 2) / 3;
+ p = 3 * wx - w;
+ l0 = ix / w;
+ a = ix + p * l0;
+ ax = (a + 2) / 3 + D + 6 + 1;
+ l = (ax + wx - 1) / wx;
+ if ((ix % w) == 0 && p != 0)
+ lstall = 1;
+ else
+ lstall = 0;
+ Delay = l * wx * (numSlices - 1) + ax + s + lstall + 22;
+
+ //dsc processes 3 pixel containers per cycle and a container can contain 1 or 2 pixels
+ pixels = Delay * 3 * pixelsPerClock;
+ return pixels;
+}
+
+static unsigned int dscComputeDelay(enum output_format_class pixelFormat)
+{
+ unsigned int Delay = 0;
+
+ if (pixelFormat == dm_420) {
+ // sfr
+ Delay = Delay + 2;
+ // dsccif
+ Delay = Delay + 0;
+ // dscc - input deserializer
+ Delay = Delay + 3;
+ // dscc gets pixels every other cycle
+ Delay = Delay + 2;
+ // dscc - input cdc fifo
+ Delay = Delay + 12;
+ // dscc gets pixels every other cycle
+ Delay = Delay + 13;
+ // dscc - cdc uncertainty
+ Delay = Delay + 2;
+ // dscc - output cdc fifo
+ Delay = Delay + 7;
+ // dscc gets pixels every other cycle
+ Delay = Delay + 3;
+ // dscc - cdc uncertainty
+ Delay = Delay + 2;
+ // dscc - output serializer
+ Delay = Delay + 1;
+ // sft
+ Delay = Delay + 1;
+ } else if (pixelFormat == dm_n422) {
+ // sfr
+ Delay = Delay + 2;
+ // dsccif
+ Delay = Delay + 1;
+ // dscc - input deserializer
+ Delay = Delay + 5;
+ // dscc - input cdc fifo
+ Delay = Delay + 25;
+ // dscc - cdc uncertainty
+ Delay = Delay + 2;
+ // dscc - output cdc fifo
+ Delay = Delay + 10;
+ // dscc - cdc uncertainty
+ Delay = Delay + 2;
+ // dscc - output serializer
+ Delay = Delay + 1;
+ // sft
+ Delay = Delay + 1;
+ } else {
+ // sfr
+ Delay = Delay + 2;
+ // dsccif
+ Delay = Delay + 0;
+ // dscc - input deserializer
+ Delay = Delay + 3;
+ // dscc - input cdc fifo
+ Delay = Delay + 12;
+ // dscc - cdc uncertainty
+ Delay = Delay + 2;
+ // dscc - output cdc fifo
+ Delay = Delay + 7;
+ // dscc - output serializer
+ Delay = Delay + 1;
+ // dscc - cdc uncertainty
+ Delay = Delay + 2;
+ // sft
+ Delay = Delay + 1;
+ }
+
+ return Delay;
+}
+
+static bool CalculatePrefetchSchedule(
+ struct display_mode_lib *mode_lib,
+ double DPPCLK,
+ double DISPCLK,
+ double PixelClock,
+ double DCFCLKDeepSleep,
+ unsigned int DSCDelay,
+ unsigned int DPPPerPlane,
+ bool ScalerEnabled,
+ unsigned int NumberOfCursors,
+ double DPPCLKDelaySubtotal,
+ double DPPCLKDelaySCL,
+ double DPPCLKDelaySCLLBOnly,
+ double DPPCLKDelayCNVCFormater,
+ double DPPCLKDelayCNVCCursor,
+ double DISPCLKDelaySubtotal,
+ unsigned int ScalerRecoutWidth,
+ enum output_format_class OutputFormat,
+ unsigned int VBlank,
+ unsigned int HTotal,
+ unsigned int MaxInterDCNTileRepeaters,
+ unsigned int VStartup,
+ unsigned int PageTableLevels,
+ bool GPUVMEnable,
+ bool DynamicMetadataEnable,
+ unsigned int DynamicMetadataLinesBeforeActiveRequired,
+ unsigned int DynamicMetadataTransmittedBytes,
+ bool DCCEnable,
+ double UrgentLatencyPixelDataOnly,
+ double UrgentExtraLatency,
+ double TCalc,
+ unsigned int PDEAndMetaPTEBytesFrame,
+ unsigned int MetaRowByte,
+ unsigned int PixelPTEBytesPerRow,
+ double PrefetchSourceLinesY,
+ unsigned int SwathWidthY,
+ double BytePerPixelDETY,
+ double VInitPreFillY,
+ unsigned int MaxNumSwathY,
+ double PrefetchSourceLinesC,
+ double BytePerPixelDETC,
+ double VInitPreFillC,
+ unsigned int MaxNumSwathC,
+ unsigned int SwathHeightY,
+ unsigned int SwathHeightC,
+ double TWait,
+ bool XFCEnabled,
+ double XFCRemoteSurfaceFlipDelay,
+ bool InterlaceEnable,
+ bool ProgressiveToInterlaceUnitInOPP,
+ double *DSTXAfterScaler,
+ double *DSTYAfterScaler,
+ double *DestinationLinesForPrefetch,
+ double *PrefetchBandwidth,
+ double *DestinationLinesToRequestVMInVBlank,
+ double *DestinationLinesToRequestRowInVBlank,
+ double *VRatioPrefetchY,
+ double *VRatioPrefetchC,
+ double *RequiredPrefetchPixDataBW,
+ unsigned int *VStartupRequiredWhenNotEnoughTimeForDynamicMetadata,
+ double *Tno_bw,
+ unsigned int *VUpdateOffsetPix,
+ double *VUpdateWidthPix,
+ double *VReadyOffsetPix)
+{
+ bool MyError = false;
+ unsigned int DPPCycles, DISPCLKCycles;
+ double DSTTotalPixelsAfterScaler, TotalRepeaterDelayTime;
+ double Tdm, LineTime, Tsetup;
+ double dst_y_prefetch_equ;
+ double Tsw_oto;
+ double prefetch_bw_oto;
+ double Tvm_oto;
+ double Tr0_oto;
+ double Tpre_oto;
+ double dst_y_prefetch_oto;
+ double TimeForFetchingMetaPTE = 0;
+ double TimeForFetchingRowInVBlank = 0;
+ double LinesToRequestPrefetchPixelData = 0;
+
+ if (ScalerEnabled)
+ DPPCycles = DPPCLKDelaySubtotal + DPPCLKDelaySCL;
+ else
+ DPPCycles = DPPCLKDelaySubtotal + DPPCLKDelaySCLLBOnly;
+
+ DPPCycles = DPPCycles + DPPCLKDelayCNVCFormater + NumberOfCursors * DPPCLKDelayCNVCCursor;
+
+ DISPCLKCycles = DISPCLKDelaySubtotal;
+
+ if (DPPCLK == 0.0 || DISPCLK == 0.0)
+ return true;
+
+ *DSTXAfterScaler = DPPCycles * PixelClock / DPPCLK + DISPCLKCycles * PixelClock / DISPCLK
+ + DSCDelay;
+
+ if (DPPPerPlane > 1)
+ *DSTXAfterScaler = *DSTXAfterScaler + ScalerRecoutWidth;
+
+ if (OutputFormat == dm_420 || (InterlaceEnable && ProgressiveToInterlaceUnitInOPP))
+ *DSTYAfterScaler = 1;
+ else
+ *DSTYAfterScaler = 0;
+
+ DSTTotalPixelsAfterScaler = ((double) (*DSTYAfterScaler * HTotal)) + *DSTXAfterScaler;
+ *DSTYAfterScaler = dml_floor(DSTTotalPixelsAfterScaler / HTotal, 1);
+ *DSTXAfterScaler = DSTTotalPixelsAfterScaler - ((double) (*DSTYAfterScaler * HTotal));
+
+ *VUpdateOffsetPix = dml_ceil(HTotal / 4.0, 1);
+ TotalRepeaterDelayTime = MaxInterDCNTileRepeaters * (2.0 / DPPCLK + 3.0 / DISPCLK);
+ *VUpdateWidthPix = (14.0 / DCFCLKDeepSleep + 12.0 / DPPCLK + TotalRepeaterDelayTime)
+ * PixelClock;
+
+ *VReadyOffsetPix = dml_max(
+ 150.0 / DPPCLK,
+ TotalRepeaterDelayTime + 20.0 / DCFCLKDeepSleep + 10.0 / DPPCLK)
+ * PixelClock;
+
+ Tsetup = (double) (*VUpdateOffsetPix + *VUpdateWidthPix + *VReadyOffsetPix) / PixelClock;
+
+ LineTime = (double) HTotal / PixelClock;
+
+ if (DynamicMetadataEnable) {
+ double Tdmbf, Tdmec, Tdmsks;
+
+ Tdm = dml_max(0.0, UrgentExtraLatency - TCalc);
+ Tdmbf = DynamicMetadataTransmittedBytes / 4.0 / DISPCLK;
+ Tdmec = LineTime;
+ if (DynamicMetadataLinesBeforeActiveRequired == 0)
+ Tdmsks = VBlank * LineTime / 2.0;
+ else
+ Tdmsks = DynamicMetadataLinesBeforeActiveRequired * LineTime;
+ if (InterlaceEnable && !ProgressiveToInterlaceUnitInOPP)
+ Tdmsks = Tdmsks / 2;
+ if (VStartup * LineTime
+ < Tsetup + TWait + UrgentExtraLatency + Tdmbf + Tdmec + Tdmsks) {
+ MyError = true;
+ *VStartupRequiredWhenNotEnoughTimeForDynamicMetadata = (Tsetup + TWait
+ + UrgentExtraLatency + Tdmbf + Tdmec + Tdmsks) / LineTime;
+ } else
+ *VStartupRequiredWhenNotEnoughTimeForDynamicMetadata = 0.0;
+ } else
+ Tdm = 0;
+
+ if (GPUVMEnable) {
+ if (PageTableLevels == 4)
+ *Tno_bw = UrgentExtraLatency + UrgentLatencyPixelDataOnly;
+ else if (PageTableLevels == 3)
+ *Tno_bw = UrgentExtraLatency;
+ else
+ *Tno_bw = 0;
+ } else if (DCCEnable)
+ *Tno_bw = LineTime;
+ else
+ *Tno_bw = LineTime / 4;
+
+ dst_y_prefetch_equ = VStartup - dml_max(TCalc + TWait, XFCRemoteSurfaceFlipDelay) / LineTime
+ - (Tsetup + Tdm) / LineTime
+ - (*DSTYAfterScaler + *DSTXAfterScaler / HTotal);
+
+ Tsw_oto = dml_max(PrefetchSourceLinesY, PrefetchSourceLinesC) * LineTime;
+
+ prefetch_bw_oto = (MetaRowByte + PixelPTEBytesPerRow
+ + PrefetchSourceLinesY * SwathWidthY * dml_ceil(BytePerPixelDETY, 1)
+ + PrefetchSourceLinesC * SwathWidthY / 2 * dml_ceil(BytePerPixelDETC, 2))
+ / Tsw_oto;
+
+ if (GPUVMEnable == true) {
+ Tvm_oto =
+ dml_max(
+ *Tno_bw + PDEAndMetaPTEBytesFrame / prefetch_bw_oto,
+ dml_max(
+ UrgentExtraLatency
+ + UrgentLatencyPixelDataOnly
+ * (PageTableLevels
+ - 1),
+ LineTime / 4.0));
+ } else
+ Tvm_oto = LineTime / 4.0;
+
+ if ((GPUVMEnable == true || DCCEnable == true)) {
+ Tr0_oto = dml_max(
+ (MetaRowByte + PixelPTEBytesPerRow) / prefetch_bw_oto,
+ dml_max(UrgentLatencyPixelDataOnly, dml_max(LineTime - Tvm_oto, LineTime / 4)));
+ } else
+ Tr0_oto = LineTime - Tvm_oto;
+
+ Tpre_oto = Tvm_oto + Tr0_oto + Tsw_oto;
+
+ dst_y_prefetch_oto = Tpre_oto / LineTime;
+
+ if (dst_y_prefetch_oto < dst_y_prefetch_equ)
+ *DestinationLinesForPrefetch = dst_y_prefetch_oto;
+ else
+ *DestinationLinesForPrefetch = dst_y_prefetch_equ;
+
+ *DestinationLinesForPrefetch = dml_floor(4.0 * (*DestinationLinesForPrefetch + 0.125), 1)
+ / 4;
+
+ dml_print("DML: VStartup: %d\n", VStartup);
+ dml_print("DML: TCalc: %f\n", TCalc);
+ dml_print("DML: TWait: %f\n", TWait);
+ dml_print("DML: XFCRemoteSurfaceFlipDelay: %f\n", XFCRemoteSurfaceFlipDelay);
+ dml_print("DML: LineTime: %f\n", LineTime);
+ dml_print("DML: Tsetup: %f\n", Tsetup);
+ dml_print("DML: Tdm: %f\n", Tdm);
+ dml_print("DML: DSTYAfterScaler: %f\n", *DSTYAfterScaler);
+ dml_print("DML: DSTXAfterScaler: %f\n", *DSTXAfterScaler);
+ dml_print("DML: HTotal: %d\n", HTotal);
+
+ *PrefetchBandwidth = 0;
+ *DestinationLinesToRequestVMInVBlank = 0;
+ *DestinationLinesToRequestRowInVBlank = 0;
+ *VRatioPrefetchY = 0;
+ *VRatioPrefetchC = 0;
+ *RequiredPrefetchPixDataBW = 0;
+ if (*DestinationLinesForPrefetch > 1) {
+ *PrefetchBandwidth = (PDEAndMetaPTEBytesFrame + 2 * MetaRowByte
+ + 2 * PixelPTEBytesPerRow
+ + PrefetchSourceLinesY * SwathWidthY * dml_ceil(BytePerPixelDETY, 1)
+ + PrefetchSourceLinesC * SwathWidthY / 2
+ * dml_ceil(BytePerPixelDETC, 2))
+ / (*DestinationLinesForPrefetch * LineTime - *Tno_bw);
+ if (GPUVMEnable) {
+ TimeForFetchingMetaPTE =
+ dml_max(
+ *Tno_bw
+ + (double) PDEAndMetaPTEBytesFrame
+ / *PrefetchBandwidth,
+ dml_max(
+ UrgentExtraLatency
+ + UrgentLatencyPixelDataOnly
+ * (PageTableLevels
+ - 1),
+ LineTime / 4));
+ } else {
+ if (NumberOfCursors > 0 || XFCEnabled)
+ TimeForFetchingMetaPTE = LineTime / 4;
+ else
+ TimeForFetchingMetaPTE = 0.0;
+ }
+
+ if ((GPUVMEnable == true || DCCEnable == true)) {
+ TimeForFetchingRowInVBlank =
+ dml_max(
+ (MetaRowByte + PixelPTEBytesPerRow)
+ / *PrefetchBandwidth,
+ dml_max(
+ UrgentLatencyPixelDataOnly,
+ dml_max(
+ LineTime
+ - TimeForFetchingMetaPTE,
+ LineTime
+ / 4.0)));
+ } else {
+ if (NumberOfCursors > 0 || XFCEnabled)
+ TimeForFetchingRowInVBlank = LineTime - TimeForFetchingMetaPTE;
+ else
+ TimeForFetchingRowInVBlank = 0.0;
+ }
+
+ *DestinationLinesToRequestVMInVBlank = dml_floor(
+ 4.0 * (TimeForFetchingMetaPTE / LineTime + 0.125),
+ 1) / 4.0;
+
+ *DestinationLinesToRequestRowInVBlank = dml_floor(
+ 4.0 * (TimeForFetchingRowInVBlank / LineTime + 0.125),
+ 1) / 4.0;
+
+ LinesToRequestPrefetchPixelData =
+ *DestinationLinesForPrefetch
+ - ((NumberOfCursors > 0 || GPUVMEnable
+ || DCCEnable) ?
+ (*DestinationLinesToRequestVMInVBlank
+ + *DestinationLinesToRequestRowInVBlank) :
+ 0.0);
+
+ if (LinesToRequestPrefetchPixelData > 0) {
+
+ *VRatioPrefetchY = (double) PrefetchSourceLinesY
+ / LinesToRequestPrefetchPixelData;
+ *VRatioPrefetchY = dml_max(*VRatioPrefetchY, 1.0);
+ if ((SwathHeightY > 4) && (VInitPreFillY > 3)) {
+ if (LinesToRequestPrefetchPixelData > (VInitPreFillY - 3.0) / 2.0) {
+ *VRatioPrefetchY =
+ dml_max(
+ (double) PrefetchSourceLinesY
+ / LinesToRequestPrefetchPixelData,
+ (double) MaxNumSwathY
+ * SwathHeightY
+ / (LinesToRequestPrefetchPixelData
+ - (VInitPreFillY
+ - 3.0)
+ / 2.0));
+ *VRatioPrefetchY = dml_max(*VRatioPrefetchY, 1.0);
+ } else {
+ MyError = true;
+ *VRatioPrefetchY = 0;
+ }
+ }
+
+ *VRatioPrefetchC = (double) PrefetchSourceLinesC
+ / LinesToRequestPrefetchPixelData;
+ *VRatioPrefetchC = dml_max(*VRatioPrefetchC, 1.0);
+
+ if ((SwathHeightC > 4)) {
+ if (LinesToRequestPrefetchPixelData > (VInitPreFillC - 3.0) / 2.0) {
+ *VRatioPrefetchC =
+ dml_max(
+ *VRatioPrefetchC,
+ (double) MaxNumSwathC
+ * SwathHeightC
+ / (LinesToRequestPrefetchPixelData
+ - (VInitPreFillC
+ - 3.0)
+ / 2.0));
+ *VRatioPrefetchC = dml_max(*VRatioPrefetchC, 1.0);
+ } else {
+ MyError = true;
+ *VRatioPrefetchC = 0;
+ }
+ }
+
+ *RequiredPrefetchPixDataBW =
+ DPPPerPlane
+ * ((double) PrefetchSourceLinesY
+ / LinesToRequestPrefetchPixelData
+ * dml_ceil(
+ BytePerPixelDETY,
+ 1)
+ + (double) PrefetchSourceLinesC
+ / LinesToRequestPrefetchPixelData
+ * dml_ceil(
+ BytePerPixelDETC,
+ 2)
+ / 2)
+ * SwathWidthY / LineTime;
+ } else {
+ MyError = true;
+ *VRatioPrefetchY = 0;
+ *VRatioPrefetchC = 0;
+ *RequiredPrefetchPixDataBW = 0;
+ }
+
+ } else {
+ MyError = true;
+ }
+
+ if (MyError) {
+ *PrefetchBandwidth = 0;
+ TimeForFetchingMetaPTE = 0;
+ TimeForFetchingRowInVBlank = 0;
+ *DestinationLinesToRequestVMInVBlank = 0;
+ *DestinationLinesToRequestRowInVBlank = 0;
+ *DestinationLinesForPrefetch = 0;
+ LinesToRequestPrefetchPixelData = 0;
+ *VRatioPrefetchY = 0;
+ *VRatioPrefetchC = 0;
+ *RequiredPrefetchPixDataBW = 0;
+ }
+
+ return MyError;
+}
+
+static double RoundToDFSGranularityUp(double Clock, double VCOSpeed)
+{
+ return VCOSpeed * 4 / dml_floor(VCOSpeed * 4 / Clock, 1);
+}
+
+static double RoundToDFSGranularityDown(double Clock, double VCOSpeed)
+{
+ return VCOSpeed * 4 / dml_ceil(VCOSpeed * 4 / Clock, 1);
+}
+
+static double CalculatePrefetchSourceLines(
+ struct display_mode_lib *mode_lib,
+ double VRatio,
+ double vtaps,
+ bool Interlace,
+ bool ProgressiveToInterlaceUnitInOPP,
+ unsigned int SwathHeight,
+ unsigned int ViewportYStart,
+ double *VInitPreFill,
+ unsigned int *MaxNumSwath)
+{
+ unsigned int MaxPartialSwath;
+
+ if (ProgressiveToInterlaceUnitInOPP)
+ *VInitPreFill = dml_floor((VRatio + vtaps + 1) / 2.0, 1);
+ else
+ *VInitPreFill = dml_floor((VRatio + vtaps + 1 + Interlace * 0.5 * VRatio) / 2.0, 1);
+
+ if (!mode_lib->vba.IgnoreViewportPositioning) {
+
+ *MaxNumSwath = dml_ceil((*VInitPreFill - 1.0) / SwathHeight, 1) + 1.0;
+
+ if (*VInitPreFill > 1.0)
+ MaxPartialSwath = (unsigned int) (*VInitPreFill - 2) % SwathHeight;
+ else
+ MaxPartialSwath = (unsigned int) (*VInitPreFill + SwathHeight - 2)
+ % SwathHeight;
+ MaxPartialSwath = dml_max(1U, MaxPartialSwath);
+
+ } else {
+
+ if (ViewportYStart != 0)
+ dml_print(
+ "WARNING DML: using viewport y position of 0 even though actual viewport y position is non-zero in prefetch source lines calculation\n");
+
+ *MaxNumSwath = dml_ceil(*VInitPreFill / SwathHeight, 1);
+
+ if (*VInitPreFill > 1.0)
+ MaxPartialSwath = (unsigned int) (*VInitPreFill - 1) % SwathHeight;
+ else
+ MaxPartialSwath = (unsigned int) (*VInitPreFill + SwathHeight - 1)
+ % SwathHeight;
+ }
+
+ return *MaxNumSwath * SwathHeight + MaxPartialSwath;
+}
+
+static unsigned int CalculateVMAndRowBytes(
+ struct display_mode_lib *mode_lib,
+ bool DCCEnable,
+ unsigned int BlockHeight256Bytes,
+ unsigned int BlockWidth256Bytes,
+ enum source_format_class SourcePixelFormat,
+ unsigned int SurfaceTiling,
+ unsigned int BytePerPixel,
+ enum scan_direction_class ScanDirection,
+ unsigned int ViewportWidth,
+ unsigned int ViewportHeight,
+ unsigned int SwathWidth,
+ bool GPUVMEnable,
+ unsigned int VMMPageSize,
+ unsigned int PTEBufferSizeInRequestsLuma,
+ unsigned int PDEProcessingBufIn64KBReqs,
+ unsigned int Pitch,
+ unsigned int DCCMetaPitch,
+ unsigned int *MacroTileWidth,
+ unsigned int *MetaRowByte,
+ unsigned int *PixelPTEBytesPerRow,
+ bool *PTEBufferSizeNotExceeded,
+ unsigned int *dpte_row_height,
+ unsigned int *meta_row_height)
+{
+ unsigned int MetaRequestHeight;
+ unsigned int MetaRequestWidth;
+ unsigned int MetaSurfWidth;
+ unsigned int MetaSurfHeight;
+ unsigned int MPDEBytesFrame;
+ unsigned int MetaPTEBytesFrame;
+ unsigned int DCCMetaSurfaceBytes;
+
+ unsigned int MacroTileSizeBytes;
+ unsigned int MacroTileHeight;
+ unsigned int DPDE0BytesFrame;
+ unsigned int ExtraDPDEBytesFrame;
+ unsigned int PDEAndMetaPTEBytesFrame;
+
+ if (DCCEnable == true) {
+ MetaRequestHeight = 8 * BlockHeight256Bytes;
+ MetaRequestWidth = 8 * BlockWidth256Bytes;
+ if (ScanDirection == dm_horz) {
+ *meta_row_height = MetaRequestHeight;
+ MetaSurfWidth = dml_ceil((double) SwathWidth - 1, MetaRequestWidth)
+ + MetaRequestWidth;
+ *MetaRowByte = MetaSurfWidth * MetaRequestHeight * BytePerPixel / 256.0;
+ } else {
+ *meta_row_height = MetaRequestWidth;
+ MetaSurfHeight = dml_ceil((double) SwathWidth - 1, MetaRequestHeight)
+ + MetaRequestHeight;
+ *MetaRowByte = MetaSurfHeight * MetaRequestWidth * BytePerPixel / 256.0;
+ }
+ if (ScanDirection == dm_horz) {
+ DCCMetaSurfaceBytes = DCCMetaPitch
+ * (dml_ceil(ViewportHeight - 1, 64 * BlockHeight256Bytes)
+ + 64 * BlockHeight256Bytes) * BytePerPixel
+ / 256;
+ } else {
+ DCCMetaSurfaceBytes = DCCMetaPitch
+ * (dml_ceil(
+ (double) ViewportHeight - 1,
+ 64 * BlockHeight256Bytes)
+ + 64 * BlockHeight256Bytes) * BytePerPixel
+ / 256;
+ }
+ if (GPUVMEnable == true) {
+ MetaPTEBytesFrame = (dml_ceil(
+ (double) (DCCMetaSurfaceBytes - VMMPageSize)
+ / (8 * VMMPageSize),
+ 1) + 1) * 64;
+ MPDEBytesFrame = 128 * (mode_lib->vba.GPUVMMaxPageTableLevels - 1);
+ } else {
+ MetaPTEBytesFrame = 0;
+ MPDEBytesFrame = 0;
+ }
+ } else {
+ MetaPTEBytesFrame = 0;
+ MPDEBytesFrame = 0;
+ *MetaRowByte = 0;
+ }
+
+ if (SurfaceTiling == dm_sw_linear || SurfaceTiling == dm_sw_gfx7_2d_thin_gl || SurfaceTiling == dm_sw_gfx7_2d_thin_lvp) {
+ MacroTileSizeBytes = 256;
+ MacroTileHeight = BlockHeight256Bytes;
+ } else if (SurfaceTiling == dm_sw_4kb_s || SurfaceTiling == dm_sw_4kb_s_x
+ || SurfaceTiling == dm_sw_4kb_d || SurfaceTiling == dm_sw_4kb_d_x) {
+ MacroTileSizeBytes = 4096;
+ MacroTileHeight = 4 * BlockHeight256Bytes;
+ } else if (SurfaceTiling == dm_sw_64kb_s || SurfaceTiling == dm_sw_64kb_s_t
+ || SurfaceTiling == dm_sw_64kb_s_x || SurfaceTiling == dm_sw_64kb_d
+ || SurfaceTiling == dm_sw_64kb_d_t || SurfaceTiling == dm_sw_64kb_d_x
+ || SurfaceTiling == dm_sw_64kb_r_x) {
+ MacroTileSizeBytes = 65536;
+ MacroTileHeight = 16 * BlockHeight256Bytes;
+ } else {
+ MacroTileSizeBytes = 262144;
+ MacroTileHeight = 32 * BlockHeight256Bytes;
+ }
+ *MacroTileWidth = MacroTileSizeBytes / BytePerPixel / MacroTileHeight;
+
+ if (GPUVMEnable == true && mode_lib->vba.GPUVMMaxPageTableLevels > 1) {
+ if (ScanDirection == dm_horz) {
+ DPDE0BytesFrame =
+ 64
+ * (dml_ceil(
+ ((Pitch
+ * (dml_ceil(
+ ViewportHeight
+ - 1,
+ MacroTileHeight)
+ + MacroTileHeight)
+ * BytePerPixel)
+ - MacroTileSizeBytes)
+ / (8
+ * 2097152),
+ 1) + 1);
+ } else {
+ DPDE0BytesFrame =
+ 64
+ * (dml_ceil(
+ ((Pitch
+ * (dml_ceil(
+ (double) SwathWidth
+ - 1,
+ MacroTileHeight)
+ + MacroTileHeight)
+ * BytePerPixel)
+ - MacroTileSizeBytes)
+ / (8
+ * 2097152),
+ 1) + 1);
+ }
+ ExtraDPDEBytesFrame = 128 * (mode_lib->vba.GPUVMMaxPageTableLevels - 2);
+ } else {
+ DPDE0BytesFrame = 0;
+ ExtraDPDEBytesFrame = 0;
+ }
+
+ PDEAndMetaPTEBytesFrame = MetaPTEBytesFrame + MPDEBytesFrame + DPDE0BytesFrame
+ + ExtraDPDEBytesFrame;
+
+ if (GPUVMEnable == true) {
+ unsigned int PTERequestSize;
+ unsigned int PixelPTEReqHeight;
+ unsigned int PixelPTEReqWidth;
+ double FractionOfPTEReturnDrop;
+ unsigned int EffectivePDEProcessingBufIn64KBReqs;
+
+ if (SurfaceTiling == dm_sw_linear) {
+ PixelPTEReqHeight = 1;
+ PixelPTEReqWidth = 8.0 * VMMPageSize / BytePerPixel;
+ PTERequestSize = 64;
+ FractionOfPTEReturnDrop = 0;
+ } else if (MacroTileSizeBytes == 4096) {
+ PixelPTEReqHeight = MacroTileHeight;
+ PixelPTEReqWidth = 8 * *MacroTileWidth;
+ PTERequestSize = 64;
+ if (ScanDirection == dm_horz)
+ FractionOfPTEReturnDrop = 0;
+ else
+ FractionOfPTEReturnDrop = 7 / 8;
+ } else if (VMMPageSize == 4096 && MacroTileSizeBytes > 4096) {
+ PixelPTEReqHeight = 16 * BlockHeight256Bytes;
+ PixelPTEReqWidth = 16 * BlockWidth256Bytes;
+ PTERequestSize = 128;
+ FractionOfPTEReturnDrop = 0;
+ } else {
+ PixelPTEReqHeight = MacroTileHeight;
+ PixelPTEReqWidth = 8 * *MacroTileWidth;
+ PTERequestSize = 64;
+ FractionOfPTEReturnDrop = 0;
+ }
+
+ if (SourcePixelFormat == dm_420_8 || SourcePixelFormat == dm_420_10)
+ EffectivePDEProcessingBufIn64KBReqs = PDEProcessingBufIn64KBReqs / 2;
+ else
+ EffectivePDEProcessingBufIn64KBReqs = PDEProcessingBufIn64KBReqs;
+
+ if (SurfaceTiling == dm_sw_linear) {
+ *dpte_row_height =
+ dml_min(
+ 128,
+ 1
+ << (unsigned int) dml_floor(
+ dml_log2(
+ dml_min(
+ (double) PTEBufferSizeInRequestsLuma
+ * PixelPTEReqWidth,
+ EffectivePDEProcessingBufIn64KBReqs
+ * 65536.0
+ / BytePerPixel)
+ / Pitch),
+ 1));
+ *PixelPTEBytesPerRow = PTERequestSize
+ * (dml_ceil(
+ (double) (Pitch * *dpte_row_height - 1)
+ / PixelPTEReqWidth,
+ 1) + 1);
+ } else if (ScanDirection == dm_horz) {
+ *dpte_row_height = PixelPTEReqHeight;
+ *PixelPTEBytesPerRow = PTERequestSize
+ * (dml_ceil(((double) SwathWidth - 1) / PixelPTEReqWidth, 1)
+ + 1);
+ } else {
+ *dpte_row_height = dml_min(PixelPTEReqWidth, *MacroTileWidth);
+ *PixelPTEBytesPerRow = PTERequestSize
+ * (dml_ceil(
+ ((double) SwathWidth - 1)
+ / PixelPTEReqHeight,
+ 1) + 1);
+ }
+ if (*PixelPTEBytesPerRow * (1 - FractionOfPTEReturnDrop)
+ <= 64 * PTEBufferSizeInRequestsLuma) {
+ *PTEBufferSizeNotExceeded = true;
+ } else {
+ *PTEBufferSizeNotExceeded = false;
+ }
+ } else {
+ *PixelPTEBytesPerRow = 0;
+ *PTEBufferSizeNotExceeded = true;
+ }
+
+ return PDEAndMetaPTEBytesFrame;
+}
+
+static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation(
+ struct display_mode_lib *mode_lib)
+{
+ unsigned int j, k;
+
+ mode_lib->vba.WritebackDISPCLK = 0.0;
+ mode_lib->vba.DISPCLKWithRamping = 0;
+ mode_lib->vba.DISPCLKWithoutRamping = 0;
+ mode_lib->vba.GlobalDPPCLK = 0.0;
+
+ // dml_ml->vba.DISPCLK and dml_ml->vba.DPPCLK Calculation
+ //
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.WritebackEnable[k]) {
+ mode_lib->vba.WritebackDISPCLK =
+ dml_max(
+ mode_lib->vba.WritebackDISPCLK,
+ CalculateWriteBackDISPCLK(
+ mode_lib->vba.WritebackPixelFormat[k],
+ mode_lib->vba.PixelClock[k],
+ mode_lib->vba.WritebackHRatio[k],
+ mode_lib->vba.WritebackVRatio[k],
+ mode_lib->vba.WritebackLumaHTaps[k],
+ mode_lib->vba.WritebackLumaVTaps[k],
+ mode_lib->vba.WritebackChromaHTaps[k],
+ mode_lib->vba.WritebackChromaVTaps[k],
+ mode_lib->vba.WritebackDestinationWidth[k],
+ mode_lib->vba.HTotal[k],
+ mode_lib->vba.WritebackChromaLineBufferWidth));
+ }
+ }
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.HRatio[k] > 1) {
+ mode_lib->vba.PSCL_THROUGHPUT_LUMA[k] = dml_min(
+ mode_lib->vba.MaxDCHUBToPSCLThroughput,
+ mode_lib->vba.MaxPSCLToLBThroughput
+ * mode_lib->vba.HRatio[k]
+ / dml_ceil(
+ mode_lib->vba.htaps[k]
+ / 6.0,
+ 1));
+ } else {
+ mode_lib->vba.PSCL_THROUGHPUT_LUMA[k] = dml_min(
+ mode_lib->vba.MaxDCHUBToPSCLThroughput,
+ mode_lib->vba.MaxPSCLToLBThroughput);
+ }
+
+ mode_lib->vba.DPPCLKUsingSingleDPPLuma =
+ mode_lib->vba.PixelClock[k]
+ * dml_max(
+ mode_lib->vba.vtaps[k] / 6.0
+ * dml_min(
+ 1.0,
+ mode_lib->vba.HRatio[k]),
+ dml_max(
+ mode_lib->vba.HRatio[k]
+ * mode_lib->vba.VRatio[k]
+ / mode_lib->vba.PSCL_THROUGHPUT_LUMA[k],
+ 1.0));
+
+ if ((mode_lib->vba.htaps[k] > 6 || mode_lib->vba.vtaps[k] > 6)
+ && mode_lib->vba.DPPCLKUsingSingleDPPLuma
+ < 2 * mode_lib->vba.PixelClock[k]) {
+ mode_lib->vba.DPPCLKUsingSingleDPPLuma = 2 * mode_lib->vba.PixelClock[k];
+ }
+
+ if ((mode_lib->vba.SourcePixelFormat[k] != dm_420_8
+ && mode_lib->vba.SourcePixelFormat[k] != dm_420_10)) {
+ mode_lib->vba.PSCL_THROUGHPUT_CHROMA[k] = 0.0;
+ mode_lib->vba.DPPCLKUsingSingleDPP[k] =
+ mode_lib->vba.DPPCLKUsingSingleDPPLuma;
+ } else {
+ if (mode_lib->vba.HRatio[k] > 1) {
+ mode_lib->vba.PSCL_THROUGHPUT_CHROMA[k] =
+ dml_min(
+ mode_lib->vba.MaxDCHUBToPSCLThroughput,
+ mode_lib->vba.MaxPSCLToLBThroughput
+ * mode_lib->vba.HRatio[k]
+ / 2
+ / dml_ceil(
+ mode_lib->vba.HTAPsChroma[k]
+ / 6.0,
+ 1.0));
+ } else {
+ mode_lib->vba.PSCL_THROUGHPUT_CHROMA[k] = dml_min(
+ mode_lib->vba.MaxDCHUBToPSCLThroughput,
+ mode_lib->vba.MaxPSCLToLBThroughput);
+ }
+ mode_lib->vba.DPPCLKUsingSingleDPPChroma =
+ mode_lib->vba.PixelClock[k]
+ * dml_max(
+ mode_lib->vba.VTAPsChroma[k]
+ / 6.0
+ * dml_min(
+ 1.0,
+ mode_lib->vba.HRatio[k]
+ / 2),
+ dml_max(
+ mode_lib->vba.HRatio[k]
+ * mode_lib->vba.VRatio[k]
+ / 4
+ / mode_lib->vba.PSCL_THROUGHPUT_CHROMA[k],
+ 1.0));
+
+ if ((mode_lib->vba.HTAPsChroma[k] > 6 || mode_lib->vba.VTAPsChroma[k] > 6)
+ && mode_lib->vba.DPPCLKUsingSingleDPPChroma
+ < 2 * mode_lib->vba.PixelClock[k]) {
+ mode_lib->vba.DPPCLKUsingSingleDPPChroma = 2
+ * mode_lib->vba.PixelClock[k];
+ }
+
+ mode_lib->vba.DPPCLKUsingSingleDPP[k] = dml_max(
+ mode_lib->vba.DPPCLKUsingSingleDPPLuma,
+ mode_lib->vba.DPPCLKUsingSingleDPPChroma);
+ }
+ }
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.BlendingAndTiming[k] != k)
+ continue;
+ if (mode_lib->vba.ODMCombineEnabled[k]) {
+ mode_lib->vba.DISPCLKWithRamping =
+ dml_max(
+ mode_lib->vba.DISPCLKWithRamping,
+ mode_lib->vba.PixelClock[k] / 2
+ * (1
+ + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading
+ / 100)
+ * (1
+ + mode_lib->vba.DISPCLKRampingMargin
+ / 100));
+ mode_lib->vba.DISPCLKWithoutRamping =
+ dml_max(
+ mode_lib->vba.DISPCLKWithoutRamping,
+ mode_lib->vba.PixelClock[k] / 2
+ * (1
+ + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading
+ / 100));
+ } else if (!mode_lib->vba.ODMCombineEnabled[k]) {
+ mode_lib->vba.DISPCLKWithRamping =
+ dml_max(
+ mode_lib->vba.DISPCLKWithRamping,
+ mode_lib->vba.PixelClock[k]
+ * (1
+ + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading
+ / 100)
+ * (1
+ + mode_lib->vba.DISPCLKRampingMargin
+ / 100));
+ mode_lib->vba.DISPCLKWithoutRamping =
+ dml_max(
+ mode_lib->vba.DISPCLKWithoutRamping,
+ mode_lib->vba.PixelClock[k]
+ * (1
+ + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading
+ / 100));
+ }
+ }
+
+ mode_lib->vba.DISPCLKWithRamping = dml_max(
+ mode_lib->vba.DISPCLKWithRamping,
+ mode_lib->vba.WritebackDISPCLK);
+ mode_lib->vba.DISPCLKWithoutRamping = dml_max(
+ mode_lib->vba.DISPCLKWithoutRamping,
+ mode_lib->vba.WritebackDISPCLK);
+
+ ASSERT(mode_lib->vba.DISPCLKDPPCLKVCOSpeed != 0);
+ mode_lib->vba.DISPCLKWithRampingRoundedToDFSGranularity = RoundToDFSGranularityUp(
+ mode_lib->vba.DISPCLKWithRamping,
+ mode_lib->vba.DISPCLKDPPCLKVCOSpeed);
+ mode_lib->vba.DISPCLKWithoutRampingRoundedToDFSGranularity = RoundToDFSGranularityUp(
+ mode_lib->vba.DISPCLKWithoutRamping,
+ mode_lib->vba.DISPCLKDPPCLKVCOSpeed);
+ mode_lib->vba.MaxDispclkRoundedToDFSGranularity = RoundToDFSGranularityDown(
+ mode_lib->vba.soc.clock_limits[mode_lib->vba.soc.num_states].dispclk_mhz,
+ mode_lib->vba.DISPCLKDPPCLKVCOSpeed);
+ if (mode_lib->vba.DISPCLKWithoutRampingRoundedToDFSGranularity
+ > mode_lib->vba.MaxDispclkRoundedToDFSGranularity) {
+ mode_lib->vba.DISPCLK_calculated =
+ mode_lib->vba.DISPCLKWithoutRampingRoundedToDFSGranularity;
+ } else if (mode_lib->vba.DISPCLKWithRampingRoundedToDFSGranularity
+ > mode_lib->vba.MaxDispclkRoundedToDFSGranularity) {
+ mode_lib->vba.DISPCLK_calculated = mode_lib->vba.MaxDispclkRoundedToDFSGranularity;
+ } else {
+ mode_lib->vba.DISPCLK_calculated =
+ mode_lib->vba.DISPCLKWithRampingRoundedToDFSGranularity;
+ }
+ DTRACE(" dispclk_mhz (calculated) = %f", mode_lib->vba.DISPCLK_calculated);
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.DPPPerPlane[k] == 0) {
+ mode_lib->vba.DPPCLK_calculated[k] = 0;
+ } else {
+ mode_lib->vba.DPPCLK_calculated[k] = mode_lib->vba.DPPCLKUsingSingleDPP[k]
+ / mode_lib->vba.DPPPerPlane[k]
+ * (1 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100);
+ }
+ mode_lib->vba.GlobalDPPCLK = dml_max(
+ mode_lib->vba.GlobalDPPCLK,
+ mode_lib->vba.DPPCLK_calculated[k]);
+ }
+ mode_lib->vba.GlobalDPPCLK = RoundToDFSGranularityUp(
+ mode_lib->vba.GlobalDPPCLK,
+ mode_lib->vba.DISPCLKDPPCLKVCOSpeed);
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ mode_lib->vba.DPPCLK_calculated[k] = mode_lib->vba.GlobalDPPCLK / 255
+ * dml_ceil(
+ mode_lib->vba.DPPCLK_calculated[k] * 255
+ / mode_lib->vba.GlobalDPPCLK,
+ 1);
+ DTRACE(" dppclk_mhz[%i] (calculated) = %f", k, mode_lib->vba.DPPCLK_calculated[k]);
+ }
+
+ // Urgent Watermark
+ mode_lib->vba.DCCEnabledAnyPlane = false;
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k)
+ if (mode_lib->vba.DCCEnable[k])
+ mode_lib->vba.DCCEnabledAnyPlane = true;
+
+ mode_lib->vba.ReturnBandwidthToDCN = dml_min(
+ mode_lib->vba.ReturnBusWidth * mode_lib->vba.DCFCLK,
+ mode_lib->vba.FabricAndDRAMBandwidth * 1000)
+ * mode_lib->vba.PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelDataOnly / 100;
+
+ mode_lib->vba.ReturnBW = mode_lib->vba.ReturnBandwidthToDCN;
+ mode_lib->vba.ReturnBW = adjust_ReturnBW(
+ mode_lib,
+ mode_lib->vba.ReturnBW,
+ mode_lib->vba.DCCEnabledAnyPlane,
+ mode_lib->vba.ReturnBandwidthToDCN);
+
+ // Let's do this calculation again??
+ mode_lib->vba.ReturnBandwidthToDCN = dml_min(
+ mode_lib->vba.ReturnBusWidth * mode_lib->vba.DCFCLK,
+ mode_lib->vba.FabricAndDRAMBandwidth * 1000);
+ mode_lib->vba.ReturnBW = adjust_ReturnBW(
+ mode_lib,
+ mode_lib->vba.ReturnBW,
+ mode_lib->vba.DCCEnabledAnyPlane,
+ mode_lib->vba.ReturnBandwidthToDCN);
+
+ DTRACE(" dcfclk_mhz = %f", mode_lib->vba.DCFCLK);
+ DTRACE(" return_bw_to_dcn = %f", mode_lib->vba.ReturnBandwidthToDCN);
+ DTRACE(" return_bus_bw = %f", mode_lib->vba.ReturnBW);
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ bool MainPlaneDoesODMCombine = false;
+
+ if (mode_lib->vba.SourceScan[k] == dm_horz)
+ mode_lib->vba.SwathWidthSingleDPPY[k] = mode_lib->vba.ViewportWidth[k];
+ else
+ mode_lib->vba.SwathWidthSingleDPPY[k] = mode_lib->vba.ViewportHeight[k];
+
+ if (mode_lib->vba.ODMCombineEnabled[k] == true)
+ MainPlaneDoesODMCombine = true;
+ for (j = 0; j < mode_lib->vba.NumberOfActivePlanes; ++j)
+ if (mode_lib->vba.BlendingAndTiming[k] == j
+ && mode_lib->vba.ODMCombineEnabled[j] == true)
+ MainPlaneDoesODMCombine = true;
+
+ if (MainPlaneDoesODMCombine == true)
+ mode_lib->vba.SwathWidthY[k] = dml_min(
+ (double) mode_lib->vba.SwathWidthSingleDPPY[k],
+ dml_round(
+ mode_lib->vba.HActive[k] / 2.0
+ * mode_lib->vba.HRatio[k]));
+ else {
+ if (mode_lib->vba.DPPPerPlane[k] == 0) {
+ mode_lib->vba.SwathWidthY[k] = 0;
+ } else {
+ mode_lib->vba.SwathWidthY[k] = mode_lib->vba.SwathWidthSingleDPPY[k]
+ / mode_lib->vba.DPPPerPlane[k];
+ }
+ }
+ }
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.SourcePixelFormat[k] == dm_444_64) {
+ mode_lib->vba.BytePerPixelDETY[k] = 8;
+ mode_lib->vba.BytePerPixelDETC[k] = 0;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_444_32) {
+ mode_lib->vba.BytePerPixelDETY[k] = 4;
+ mode_lib->vba.BytePerPixelDETC[k] = 0;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_444_16) {
+ mode_lib->vba.BytePerPixelDETY[k] = 2;
+ mode_lib->vba.BytePerPixelDETC[k] = 0;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_444_8) {
+ mode_lib->vba.BytePerPixelDETY[k] = 1;
+ mode_lib->vba.BytePerPixelDETC[k] = 0;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_420_8) {
+ mode_lib->vba.BytePerPixelDETY[k] = 1;
+ mode_lib->vba.BytePerPixelDETC[k] = 2;
+ } else { // dm_420_10
+ mode_lib->vba.BytePerPixelDETY[k] = 4.0 / 3.0;
+ mode_lib->vba.BytePerPixelDETC[k] = 8.0 / 3.0;
+ }
+ }
+
+ mode_lib->vba.TotalDataReadBandwidth = 0.0;
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ mode_lib->vba.ReadBandwidthPlaneLuma[k] = mode_lib->vba.SwathWidthSingleDPPY[k]
+ * dml_ceil(mode_lib->vba.BytePerPixelDETY[k], 1)
+ / (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k])
+ * mode_lib->vba.VRatio[k];
+ mode_lib->vba.ReadBandwidthPlaneChroma[k] = mode_lib->vba.SwathWidthSingleDPPY[k]
+ / 2 * dml_ceil(mode_lib->vba.BytePerPixelDETC[k], 2)
+ / (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k])
+ * mode_lib->vba.VRatio[k] / 2;
+ DTRACE(
+ " read_bw[%i] = %fBps",
+ k,
+ mode_lib->vba.ReadBandwidthPlaneLuma[k]
+ + mode_lib->vba.ReadBandwidthPlaneChroma[k]);
+ mode_lib->vba.TotalDataReadBandwidth += mode_lib->vba.ReadBandwidthPlaneLuma[k]
+ + mode_lib->vba.ReadBandwidthPlaneChroma[k];
+ }
+
+ mode_lib->vba.TotalDCCActiveDPP = 0;
+ mode_lib->vba.TotalActiveDPP = 0;
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ mode_lib->vba.TotalActiveDPP = mode_lib->vba.TotalActiveDPP
+ + mode_lib->vba.DPPPerPlane[k];
+ if (mode_lib->vba.DCCEnable[k])
+ mode_lib->vba.TotalDCCActiveDPP = mode_lib->vba.TotalDCCActiveDPP
+ + mode_lib->vba.DPPPerPlane[k];
+ }
+
+ mode_lib->vba.UrgentRoundTripAndOutOfOrderLatency =
+ (mode_lib->vba.RoundTripPingLatencyCycles + 32) / mode_lib->vba.DCFCLK
+ + mode_lib->vba.UrgentOutOfOrderReturnPerChannelPixelDataOnly
+ * mode_lib->vba.NumberOfChannels
+ / mode_lib->vba.ReturnBW;
+
+ mode_lib->vba.LastPixelOfLineExtraWatermark = 0;
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ double DataFabricLineDeliveryTimeLuma, DataFabricLineDeliveryTimeChroma;
+
+ if (mode_lib->vba.VRatio[k] <= 1.0)
+ mode_lib->vba.DisplayPipeLineDeliveryTimeLuma[k] =
+ (double) mode_lib->vba.SwathWidthY[k]
+ * mode_lib->vba.DPPPerPlane[k]
+ / mode_lib->vba.HRatio[k]
+ / mode_lib->vba.PixelClock[k];
+ else
+ mode_lib->vba.DisplayPipeLineDeliveryTimeLuma[k] =
+ (double) mode_lib->vba.SwathWidthY[k]
+ / mode_lib->vba.PSCL_THROUGHPUT_LUMA[k]
+ / mode_lib->vba.DPPCLK[k];
+
+ DataFabricLineDeliveryTimeLuma = mode_lib->vba.SwathWidthSingleDPPY[k]
+ * mode_lib->vba.SwathHeightY[k]
+ * dml_ceil(mode_lib->vba.BytePerPixelDETY[k], 1)
+ / (mode_lib->vba.ReturnBW * mode_lib->vba.ReadBandwidthPlaneLuma[k]
+ / mode_lib->vba.TotalDataReadBandwidth);
+ mode_lib->vba.LastPixelOfLineExtraWatermark = dml_max(
+ mode_lib->vba.LastPixelOfLineExtraWatermark,
+ DataFabricLineDeliveryTimeLuma
+ - mode_lib->vba.DisplayPipeLineDeliveryTimeLuma[k]);
+
+ if (mode_lib->vba.BytePerPixelDETC[k] == 0)
+ mode_lib->vba.DisplayPipeLineDeliveryTimeChroma[k] = 0.0;
+ else if (mode_lib->vba.VRatio[k] / 2.0 <= 1.0)
+ mode_lib->vba.DisplayPipeLineDeliveryTimeChroma[k] =
+ mode_lib->vba.SwathWidthY[k] / 2.0
+ * mode_lib->vba.DPPPerPlane[k]
+ / (mode_lib->vba.HRatio[k] / 2.0)
+ / mode_lib->vba.PixelClock[k];
+ else
+ mode_lib->vba.DisplayPipeLineDeliveryTimeChroma[k] =
+ mode_lib->vba.SwathWidthY[k] / 2.0
+ / mode_lib->vba.PSCL_THROUGHPUT_CHROMA[k]
+ / mode_lib->vba.DPPCLK[k];
+
+ DataFabricLineDeliveryTimeChroma = mode_lib->vba.SwathWidthSingleDPPY[k] / 2.0
+ * mode_lib->vba.SwathHeightC[k]
+ * dml_ceil(mode_lib->vba.BytePerPixelDETC[k], 2)
+ / (mode_lib->vba.ReturnBW
+ * mode_lib->vba.ReadBandwidthPlaneChroma[k]
+ / mode_lib->vba.TotalDataReadBandwidth);
+ mode_lib->vba.LastPixelOfLineExtraWatermark =
+ dml_max(
+ mode_lib->vba.LastPixelOfLineExtraWatermark,
+ DataFabricLineDeliveryTimeChroma
+ - mode_lib->vba.DisplayPipeLineDeliveryTimeChroma[k]);
+ }
+
+ mode_lib->vba.UrgentExtraLatency = mode_lib->vba.UrgentRoundTripAndOutOfOrderLatency
+ + (mode_lib->vba.TotalActiveDPP * mode_lib->vba.PixelChunkSizeInKByte
+ + mode_lib->vba.TotalDCCActiveDPP
+ * mode_lib->vba.MetaChunkSize) * 1024.0
+ / mode_lib->vba.ReturnBW;
+
+ if (mode_lib->vba.GPUVMEnable)
+ mode_lib->vba.UrgentExtraLatency += mode_lib->vba.TotalActiveDPP
+ * mode_lib->vba.PTEGroupSize / mode_lib->vba.ReturnBW;
+
+ mode_lib->vba.UrgentWatermark = mode_lib->vba.UrgentLatencyPixelDataOnly
+ + mode_lib->vba.LastPixelOfLineExtraWatermark
+ + mode_lib->vba.UrgentExtraLatency;
+
+ DTRACE(" urgent_extra_latency = %fus", mode_lib->vba.UrgentExtraLatency);
+ DTRACE(" wm_urgent = %fus", mode_lib->vba.UrgentWatermark);
+
+ mode_lib->vba.UrgentLatency = mode_lib->vba.UrgentLatencyPixelDataOnly;
+
+ mode_lib->vba.TotalActiveWriteback = 0;
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.WritebackEnable[k])
+ mode_lib->vba.TotalActiveWriteback = mode_lib->vba.TotalActiveWriteback + mode_lib->vba.ActiveWritebacksPerPlane[k];
+ }
+
+ if (mode_lib->vba.TotalActiveWriteback <= 1)
+ mode_lib->vba.WritebackUrgentWatermark = mode_lib->vba.WritebackLatency;
+ else
+ mode_lib->vba.WritebackUrgentWatermark = mode_lib->vba.WritebackLatency
+ + mode_lib->vba.WritebackChunkSize * 1024.0 / 32
+ / mode_lib->vba.SOCCLK;
+
+ DTRACE(" wm_wb_urgent = %fus", mode_lib->vba.WritebackUrgentWatermark);
+
+ // NB P-State/DRAM Clock Change Watermark
+ mode_lib->vba.DRAMClockChangeWatermark = mode_lib->vba.DRAMClockChangeLatency
+ + mode_lib->vba.UrgentWatermark;
+
+ DTRACE(" wm_pstate_change = %fus", mode_lib->vba.DRAMClockChangeWatermark);
+
+ DTRACE(" calculating wb pstate watermark");
+ DTRACE(" total wb outputs %d", mode_lib->vba.TotalActiveWriteback);
+ DTRACE(" socclk frequency %f Mhz", mode_lib->vba.SOCCLK);
+
+ if (mode_lib->vba.TotalActiveWriteback <= 1)
+ mode_lib->vba.WritebackDRAMClockChangeWatermark =
+ mode_lib->vba.DRAMClockChangeLatency
+ + mode_lib->vba.WritebackLatency;
+ else
+ mode_lib->vba.WritebackDRAMClockChangeWatermark =
+ mode_lib->vba.DRAMClockChangeLatency
+ + mode_lib->vba.WritebackLatency
+ + mode_lib->vba.WritebackChunkSize * 1024.0 / 32
+ / mode_lib->vba.SOCCLK;
+
+ DTRACE(" wm_wb_pstate %fus", mode_lib->vba.WritebackDRAMClockChangeWatermark);
+
+ // Stutter Efficiency
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ mode_lib->vba.LinesInDETY[k] = mode_lib->vba.DETBufferSizeY[k]
+ / mode_lib->vba.BytePerPixelDETY[k] / mode_lib->vba.SwathWidthY[k];
+ mode_lib->vba.LinesInDETYRoundedDownToSwath[k] = dml_floor(
+ mode_lib->vba.LinesInDETY[k],
+ mode_lib->vba.SwathHeightY[k]);
+ mode_lib->vba.FullDETBufferingTimeY[k] =
+ mode_lib->vba.LinesInDETYRoundedDownToSwath[k]
+ * (mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k])
+ / mode_lib->vba.VRatio[k];
+ if (mode_lib->vba.BytePerPixelDETC[k] > 0) {
+ mode_lib->vba.LinesInDETC[k] = mode_lib->vba.DETBufferSizeC[k]
+ / mode_lib->vba.BytePerPixelDETC[k]
+ / (mode_lib->vba.SwathWidthY[k] / 2);
+ mode_lib->vba.LinesInDETCRoundedDownToSwath[k] = dml_floor(
+ mode_lib->vba.LinesInDETC[k],
+ mode_lib->vba.SwathHeightC[k]);
+ mode_lib->vba.FullDETBufferingTimeC[k] =
+ mode_lib->vba.LinesInDETCRoundedDownToSwath[k]
+ * (mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k])
+ / (mode_lib->vba.VRatio[k] / 2);
+ } else {
+ mode_lib->vba.LinesInDETC[k] = 0;
+ mode_lib->vba.LinesInDETCRoundedDownToSwath[k] = 0;
+ mode_lib->vba.FullDETBufferingTimeC[k] = 999999;
+ }
+ }
+
+ mode_lib->vba.MinFullDETBufferingTime = 999999.0;
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.FullDETBufferingTimeY[k]
+ < mode_lib->vba.MinFullDETBufferingTime) {
+ mode_lib->vba.MinFullDETBufferingTime =
+ mode_lib->vba.FullDETBufferingTimeY[k];
+ mode_lib->vba.FrameTimeForMinFullDETBufferingTime =
+ (double) mode_lib->vba.VTotal[k] * mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k];
+ }
+ if (mode_lib->vba.FullDETBufferingTimeC[k]
+ < mode_lib->vba.MinFullDETBufferingTime) {
+ mode_lib->vba.MinFullDETBufferingTime =
+ mode_lib->vba.FullDETBufferingTimeC[k];
+ mode_lib->vba.FrameTimeForMinFullDETBufferingTime =
+ (double) mode_lib->vba.VTotal[k] * mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k];
+ }
+ }
+
+ mode_lib->vba.AverageReadBandwidthGBytePerSecond = 0.0;
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.DCCEnable[k]) {
+ mode_lib->vba.AverageReadBandwidthGBytePerSecond =
+ mode_lib->vba.AverageReadBandwidthGBytePerSecond
+ + mode_lib->vba.ReadBandwidthPlaneLuma[k]
+ / mode_lib->vba.DCCRate[k]
+ / 1000
+ + mode_lib->vba.ReadBandwidthPlaneChroma[k]
+ / mode_lib->vba.DCCRate[k]
+ / 1000;
+ } else {
+ mode_lib->vba.AverageReadBandwidthGBytePerSecond =
+ mode_lib->vba.AverageReadBandwidthGBytePerSecond
+ + mode_lib->vba.ReadBandwidthPlaneLuma[k]
+ / 1000
+ + mode_lib->vba.ReadBandwidthPlaneChroma[k]
+ / 1000;
+ }
+ if (mode_lib->vba.DCCEnable[k]) {
+ mode_lib->vba.AverageReadBandwidthGBytePerSecond =
+ mode_lib->vba.AverageReadBandwidthGBytePerSecond
+ + mode_lib->vba.ReadBandwidthPlaneLuma[k]
+ / 1000 / 256
+ + mode_lib->vba.ReadBandwidthPlaneChroma[k]
+ / 1000 / 256;
+ }
+ if (mode_lib->vba.GPUVMEnable) {
+ mode_lib->vba.AverageReadBandwidthGBytePerSecond =
+ mode_lib->vba.AverageReadBandwidthGBytePerSecond
+ + mode_lib->vba.ReadBandwidthPlaneLuma[k]
+ / 1000 / 512
+ + mode_lib->vba.ReadBandwidthPlaneChroma[k]
+ / 1000 / 512;
+ }
+ }
+
+ mode_lib->vba.PartOfBurstThatFitsInROB =
+ dml_min(
+ mode_lib->vba.MinFullDETBufferingTime
+ * mode_lib->vba.TotalDataReadBandwidth,
+ mode_lib->vba.ROBBufferSizeInKByte * 1024
+ * mode_lib->vba.TotalDataReadBandwidth
+ / (mode_lib->vba.AverageReadBandwidthGBytePerSecond
+ * 1000));
+ mode_lib->vba.StutterBurstTime = mode_lib->vba.PartOfBurstThatFitsInROB
+ * (mode_lib->vba.AverageReadBandwidthGBytePerSecond * 1000)
+ / mode_lib->vba.TotalDataReadBandwidth / mode_lib->vba.ReturnBW
+ + (mode_lib->vba.MinFullDETBufferingTime
+ * mode_lib->vba.TotalDataReadBandwidth
+ - mode_lib->vba.PartOfBurstThatFitsInROB)
+ / (mode_lib->vba.DCFCLK * 64);
+ if (mode_lib->vba.TotalActiveWriteback == 0) {
+ mode_lib->vba.StutterEfficiencyNotIncludingVBlank = (1
+ - (mode_lib->vba.SRExitTime + mode_lib->vba.StutterBurstTime)
+ / mode_lib->vba.MinFullDETBufferingTime) * 100;
+ } else {
+ mode_lib->vba.StutterEfficiencyNotIncludingVBlank = 0;
+ }
+
+ mode_lib->vba.SmallestVBlank = 999999;
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.SynchronizedVBlank || mode_lib->vba.NumberOfActivePlanes == 1) {
+ mode_lib->vba.VBlankTime = (double) (mode_lib->vba.VTotal[k]
+ - mode_lib->vba.VActive[k]) * mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k];
+ } else {
+ mode_lib->vba.VBlankTime = 0;
+ }
+ mode_lib->vba.SmallestVBlank = dml_min(
+ mode_lib->vba.SmallestVBlank,
+ mode_lib->vba.VBlankTime);
+ }
+
+ mode_lib->vba.StutterEfficiency = (mode_lib->vba.StutterEfficiencyNotIncludingVBlank / 100
+ * (mode_lib->vba.FrameTimeForMinFullDETBufferingTime
+ - mode_lib->vba.SmallestVBlank)
+ + mode_lib->vba.SmallestVBlank)
+ / mode_lib->vba.FrameTimeForMinFullDETBufferingTime * 100;
+
+ // dml_ml->vba.DCFCLK Deep Sleep
+ mode_lib->vba.DCFCLKDeepSleep = 8.0;
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; k++) {
+ if (mode_lib->vba.BytePerPixelDETC[k] > 0) {
+ mode_lib->vba.DCFCLKDeepSleepPerPlane[k] =
+ dml_max(
+ 1.1 * mode_lib->vba.SwathWidthY[k]
+ * dml_ceil(
+ mode_lib->vba.BytePerPixelDETY[k],
+ 1) / 32
+ / mode_lib->vba.DisplayPipeLineDeliveryTimeLuma[k],
+ 1.1 * mode_lib->vba.SwathWidthY[k] / 2.0
+ * dml_ceil(
+ mode_lib->vba.BytePerPixelDETC[k],
+ 2) / 32
+ / mode_lib->vba.DisplayPipeLineDeliveryTimeChroma[k]);
+ } else
+ mode_lib->vba.DCFCLKDeepSleepPerPlane[k] = 1.1 * mode_lib->vba.SwathWidthY[k]
+ * dml_ceil(mode_lib->vba.BytePerPixelDETY[k], 1) / 64.0
+ / mode_lib->vba.DisplayPipeLineDeliveryTimeLuma[k];
+ mode_lib->vba.DCFCLKDeepSleepPerPlane[k] = dml_max(
+ mode_lib->vba.DCFCLKDeepSleepPerPlane[k],
+ mode_lib->vba.PixelClock[k] / 16.0);
+ mode_lib->vba.DCFCLKDeepSleep = dml_max(
+ mode_lib->vba.DCFCLKDeepSleep,
+ mode_lib->vba.DCFCLKDeepSleepPerPlane[k]);
+
+ DTRACE(
+ " dcfclk_deepsleep_per_plane[%i] = %fMHz",
+ k,
+ mode_lib->vba.DCFCLKDeepSleepPerPlane[k]);
+ }
+
+ DTRACE(" dcfclk_deepsleep_mhz = %fMHz", mode_lib->vba.DCFCLKDeepSleep);
+
+ // Stutter Watermark
+ mode_lib->vba.StutterExitWatermark = mode_lib->vba.SRExitTime
+ + mode_lib->vba.LastPixelOfLineExtraWatermark
+ + mode_lib->vba.UrgentExtraLatency + 10 / mode_lib->vba.DCFCLKDeepSleep;
+ mode_lib->vba.StutterEnterPlusExitWatermark = mode_lib->vba.SREnterPlusExitTime
+ + mode_lib->vba.LastPixelOfLineExtraWatermark
+ + mode_lib->vba.UrgentExtraLatency;
+
+ DTRACE(" wm_cstate_exit = %fus", mode_lib->vba.StutterExitWatermark);
+ DTRACE(" wm_cstate_enter_exit = %fus", mode_lib->vba.StutterEnterPlusExitWatermark);
+
+ // Urgent Latency Supported
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ mode_lib->vba.EffectiveDETPlusLBLinesLuma =
+ dml_floor(
+ mode_lib->vba.LinesInDETY[k]
+ + dml_min(
+ mode_lib->vba.LinesInDETY[k]
+ * mode_lib->vba.DPPCLK[k]
+ * mode_lib->vba.BytePerPixelDETY[k]
+ * mode_lib->vba.PSCL_THROUGHPUT_LUMA[k]
+ / (mode_lib->vba.ReturnBW
+ / mode_lib->vba.DPPPerPlane[k]),
+ (double) mode_lib->vba.EffectiveLBLatencyHidingSourceLinesLuma),
+ mode_lib->vba.SwathHeightY[k]);
+
+ mode_lib->vba.UrgentLatencySupportUsLuma = mode_lib->vba.EffectiveDETPlusLBLinesLuma
+ * (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k])
+ / mode_lib->vba.VRatio[k]
+ - mode_lib->vba.EffectiveDETPlusLBLinesLuma
+ * mode_lib->vba.SwathWidthY[k]
+ * mode_lib->vba.BytePerPixelDETY[k]
+ / (mode_lib->vba.ReturnBW
+ / mode_lib->vba.DPPPerPlane[k]);
+
+ if (mode_lib->vba.BytePerPixelDETC[k] > 0) {
+ mode_lib->vba.EffectiveDETPlusLBLinesChroma =
+ dml_floor(
+ mode_lib->vba.LinesInDETC[k]
+ + dml_min(
+ mode_lib->vba.LinesInDETC[k]
+ * mode_lib->vba.DPPCLK[k]
+ * mode_lib->vba.BytePerPixelDETC[k]
+ * mode_lib->vba.PSCL_THROUGHPUT_CHROMA[k]
+ / (mode_lib->vba.ReturnBW
+ / mode_lib->vba.DPPPerPlane[k]),
+ (double) mode_lib->vba.EffectiveLBLatencyHidingSourceLinesChroma),
+ mode_lib->vba.SwathHeightC[k]);
+ mode_lib->vba.UrgentLatencySupportUsChroma =
+ mode_lib->vba.EffectiveDETPlusLBLinesChroma
+ * (mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k])
+ / (mode_lib->vba.VRatio[k] / 2)
+ - mode_lib->vba.EffectiveDETPlusLBLinesChroma
+ * (mode_lib->vba.SwathWidthY[k]
+ / 2)
+ * mode_lib->vba.BytePerPixelDETC[k]
+ / (mode_lib->vba.ReturnBW
+ / mode_lib->vba.DPPPerPlane[k]);
+ mode_lib->vba.UrgentLatencySupportUs[k] = dml_min(
+ mode_lib->vba.UrgentLatencySupportUsLuma,
+ mode_lib->vba.UrgentLatencySupportUsChroma);
+ } else {
+ mode_lib->vba.UrgentLatencySupportUs[k] =
+ mode_lib->vba.UrgentLatencySupportUsLuma;
+ }
+ }
+
+ mode_lib->vba.MinUrgentLatencySupportUs = 999999;
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ mode_lib->vba.MinUrgentLatencySupportUs = dml_min(
+ mode_lib->vba.MinUrgentLatencySupportUs,
+ mode_lib->vba.UrgentLatencySupportUs[k]);
+ }
+
+ // Non-Urgent Latency Tolerance
+ mode_lib->vba.NonUrgentLatencyTolerance = mode_lib->vba.MinUrgentLatencySupportUs
+ - mode_lib->vba.UrgentWatermark;
+
+ // DSCCLK
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if ((mode_lib->vba.BlendingAndTiming[k] != k) || !mode_lib->vba.DSCEnabled[k]) {
+ mode_lib->vba.DSCCLK_calculated[k] = 0.0;
+ } else {
+ if (mode_lib->vba.OutputFormat[k] == dm_420
+ || mode_lib->vba.OutputFormat[k] == dm_n422)
+ mode_lib->vba.DSCFormatFactor = 2;
+ else
+ mode_lib->vba.DSCFormatFactor = 1;
+ if (mode_lib->vba.ODMCombineEnabled[k])
+ mode_lib->vba.DSCCLK_calculated[k] =
+ mode_lib->vba.PixelClockBackEnd[k] / 6
+ / mode_lib->vba.DSCFormatFactor
+ / (1
+ - mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading
+ / 100);
+ else
+ mode_lib->vba.DSCCLK_calculated[k] =
+ mode_lib->vba.PixelClockBackEnd[k] / 3
+ / mode_lib->vba.DSCFormatFactor
+ / (1
+ - mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading
+ / 100);
+ }
+ }
+
+ // DSC Delay
+ // TODO
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ double bpp = mode_lib->vba.OutputBpp[k];
+ unsigned int slices = mode_lib->vba.NumberOfDSCSlices[k];
+
+ if (mode_lib->vba.DSCEnabled[k] && bpp != 0) {
+ if (!mode_lib->vba.ODMCombineEnabled[k]) {
+ mode_lib->vba.DSCDelay[k] =
+ dscceComputeDelay(
+ mode_lib->vba.DSCInputBitPerComponent[k],
+ bpp,
+ dml_ceil(
+ (double) mode_lib->vba.HActive[k]
+ / mode_lib->vba.NumberOfDSCSlices[k],
+ 1),
+ slices,
+ mode_lib->vba.OutputFormat[k])
+ + dscComputeDelay(
+ mode_lib->vba.OutputFormat[k]);
+ } else {
+ mode_lib->vba.DSCDelay[k] =
+ 2
+ * (dscceComputeDelay(
+ mode_lib->vba.DSCInputBitPerComponent[k],
+ bpp,
+ dml_ceil(
+ (double) mode_lib->vba.HActive[k]
+ / mode_lib->vba.NumberOfDSCSlices[k],
+ 1),
+ slices / 2.0,
+ mode_lib->vba.OutputFormat[k])
+ + dscComputeDelay(
+ mode_lib->vba.OutputFormat[k]));
+ }
+ mode_lib->vba.DSCDelay[k] = mode_lib->vba.DSCDelay[k]
+ * mode_lib->vba.PixelClock[k]
+ / mode_lib->vba.PixelClockBackEnd[k];
+ } else {
+ mode_lib->vba.DSCDelay[k] = 0;
+ }
+ }
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k)
+ for (j = 0; j < mode_lib->vba.NumberOfActivePlanes; ++j) // NumberOfPlanes
+ if (j != k && mode_lib->vba.BlendingAndTiming[k] == j
+ && mode_lib->vba.DSCEnabled[j])
+ mode_lib->vba.DSCDelay[k] = mode_lib->vba.DSCDelay[j];
+
+ // Prefetch
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ unsigned int PDEAndMetaPTEBytesFrameY;
+ unsigned int PixelPTEBytesPerRowY;
+ unsigned int MetaRowByteY;
+ unsigned int MetaRowByteC;
+ unsigned int PDEAndMetaPTEBytesFrameC;
+ unsigned int PixelPTEBytesPerRowC;
+
+ Calculate256BBlockSizes(
+ mode_lib->vba.SourcePixelFormat[k],
+ mode_lib->vba.SurfaceTiling[k],
+ dml_ceil(mode_lib->vba.BytePerPixelDETY[k], 1),
+ dml_ceil(mode_lib->vba.BytePerPixelDETC[k], 2),
+ &mode_lib->vba.BlockHeight256BytesY[k],
+ &mode_lib->vba.BlockHeight256BytesC[k],
+ &mode_lib->vba.BlockWidth256BytesY[k],
+ &mode_lib->vba.BlockWidth256BytesC[k]);
+ PDEAndMetaPTEBytesFrameY = CalculateVMAndRowBytes(
+ mode_lib,
+ mode_lib->vba.DCCEnable[k],
+ mode_lib->vba.BlockHeight256BytesY[k],
+ mode_lib->vba.BlockWidth256BytesY[k],
+ mode_lib->vba.SourcePixelFormat[k],
+ mode_lib->vba.SurfaceTiling[k],
+ dml_ceil(mode_lib->vba.BytePerPixelDETY[k], 1),
+ mode_lib->vba.SourceScan[k],
+ mode_lib->vba.ViewportWidth[k],
+ mode_lib->vba.ViewportHeight[k],
+ mode_lib->vba.SwathWidthY[k],
+ mode_lib->vba.GPUVMEnable,
+ mode_lib->vba.VMMPageSize,
+ mode_lib->vba.PTEBufferSizeInRequestsLuma,
+ mode_lib->vba.PDEProcessingBufIn64KBReqs,
+ mode_lib->vba.PitchY[k],
+ mode_lib->vba.DCCMetaPitchY[k],
+ &mode_lib->vba.MacroTileWidthY[k],
+ &MetaRowByteY,
+ &PixelPTEBytesPerRowY,
+ &mode_lib->vba.PTEBufferSizeNotExceeded[mode_lib->vba.VoltageLevel][0],
+ &mode_lib->vba.dpte_row_height[k],
+ &mode_lib->vba.meta_row_height[k]);
+ mode_lib->vba.PrefetchSourceLinesY[k] = CalculatePrefetchSourceLines(
+ mode_lib,
+ mode_lib->vba.VRatio[k],
+ mode_lib->vba.vtaps[k],
+ mode_lib->vba.Interlace[k],
+ mode_lib->vba.ProgressiveToInterlaceUnitInOPP,
+ mode_lib->vba.SwathHeightY[k],
+ mode_lib->vba.ViewportYStartY[k],
+ &mode_lib->vba.VInitPreFillY[k],
+ &mode_lib->vba.MaxNumSwathY[k]);
+
+ if ((mode_lib->vba.SourcePixelFormat[k] != dm_444_64
+ && mode_lib->vba.SourcePixelFormat[k] != dm_444_32
+ && mode_lib->vba.SourcePixelFormat[k] != dm_444_16
+ && mode_lib->vba.SourcePixelFormat[k] != dm_444_8)) {
+ PDEAndMetaPTEBytesFrameC =
+ CalculateVMAndRowBytes(
+ mode_lib,
+ mode_lib->vba.DCCEnable[k],
+ mode_lib->vba.BlockHeight256BytesC[k],
+ mode_lib->vba.BlockWidth256BytesC[k],
+ mode_lib->vba.SourcePixelFormat[k],
+ mode_lib->vba.SurfaceTiling[k],
+ dml_ceil(
+ mode_lib->vba.BytePerPixelDETC[k],
+ 2),
+ mode_lib->vba.SourceScan[k],
+ mode_lib->vba.ViewportWidth[k] / 2,
+ mode_lib->vba.ViewportHeight[k] / 2,
+ mode_lib->vba.SwathWidthY[k] / 2,
+ mode_lib->vba.GPUVMEnable,
+ mode_lib->vba.VMMPageSize,
+ mode_lib->vba.PTEBufferSizeInRequestsLuma,
+ mode_lib->vba.PDEProcessingBufIn64KBReqs,
+ mode_lib->vba.PitchC[k],
+ 0,
+ &mode_lib->vba.MacroTileWidthC[k],
+ &MetaRowByteC,
+ &PixelPTEBytesPerRowC,
+ &mode_lib->vba.PTEBufferSizeNotExceeded[mode_lib->vba.VoltageLevel][0],
+ &mode_lib->vba.dpte_row_height_chroma[k],
+ &mode_lib->vba.meta_row_height_chroma[k]);
+ mode_lib->vba.PrefetchSourceLinesC[k] = CalculatePrefetchSourceLines(
+ mode_lib,
+ mode_lib->vba.VRatio[k] / 2,
+ mode_lib->vba.VTAPsChroma[k],
+ mode_lib->vba.Interlace[k],
+ mode_lib->vba.ProgressiveToInterlaceUnitInOPP,
+ mode_lib->vba.SwathHeightC[k],
+ mode_lib->vba.ViewportYStartC[k],
+ &mode_lib->vba.VInitPreFillC[k],
+ &mode_lib->vba.MaxNumSwathC[k]);
+ } else {
+ PixelPTEBytesPerRowC = 0;
+ PDEAndMetaPTEBytesFrameC = 0;
+ MetaRowByteC = 0;
+ mode_lib->vba.MaxNumSwathC[k] = 0;
+ mode_lib->vba.PrefetchSourceLinesC[k] = 0;
+ }
+
+ mode_lib->vba.PixelPTEBytesPerRow[k] = PixelPTEBytesPerRowY + PixelPTEBytesPerRowC;
+ mode_lib->vba.PDEAndMetaPTEBytesFrame[k] = PDEAndMetaPTEBytesFrameY
+ + PDEAndMetaPTEBytesFrameC;
+ mode_lib->vba.MetaRowByte[k] = MetaRowByteY + MetaRowByteC;
+
+ CalculateActiveRowBandwidth(
+ mode_lib->vba.GPUVMEnable,
+ mode_lib->vba.SourcePixelFormat[k],
+ mode_lib->vba.VRatio[k],
+ mode_lib->vba.DCCEnable[k],
+ mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k],
+ MetaRowByteY,
+ MetaRowByteC,
+ mode_lib->vba.meta_row_height[k],
+ mode_lib->vba.meta_row_height_chroma[k],
+ PixelPTEBytesPerRowY,
+ PixelPTEBytesPerRowC,
+ mode_lib->vba.dpte_row_height[k],
+ mode_lib->vba.dpte_row_height_chroma[k],
+ &mode_lib->vba.meta_row_bw[k],
+ &mode_lib->vba.dpte_row_bw[k],
+ &mode_lib->vba.qual_row_bw[k]);
+ }
+
+ mode_lib->vba.TCalc = 24.0 / mode_lib->vba.DCFCLKDeepSleep;
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.BlendingAndTiming[k] == k) {
+ if (mode_lib->vba.WritebackEnable[k] == true) {
+ mode_lib->vba.WritebackDelay[mode_lib->vba.VoltageLevel][k] =
+ mode_lib->vba.WritebackLatency
+ + CalculateWriteBackDelay(
+ mode_lib->vba.WritebackPixelFormat[k],
+ mode_lib->vba.WritebackHRatio[k],
+ mode_lib->vba.WritebackVRatio[k],
+ mode_lib->vba.WritebackLumaHTaps[k],
+ mode_lib->vba.WritebackLumaVTaps[k],
+ mode_lib->vba.WritebackChromaHTaps[k],
+ mode_lib->vba.WritebackChromaVTaps[k],
+ mode_lib->vba.WritebackDestinationWidth[k])
+ / mode_lib->vba.DISPCLK;
+ } else
+ mode_lib->vba.WritebackDelay[mode_lib->vba.VoltageLevel][k] = 0;
+ for (j = 0; j < mode_lib->vba.NumberOfActivePlanes; ++j) {
+ if (mode_lib->vba.BlendingAndTiming[j] == k
+ && mode_lib->vba.WritebackEnable[j] == true) {
+ mode_lib->vba.WritebackDelay[mode_lib->vba.VoltageLevel][k] =
+ dml_max(
+ mode_lib->vba.WritebackDelay[mode_lib->vba.VoltageLevel][k],
+ mode_lib->vba.WritebackLatency
+ + CalculateWriteBackDelay(
+ mode_lib->vba.WritebackPixelFormat[j],
+ mode_lib->vba.WritebackHRatio[j],
+ mode_lib->vba.WritebackVRatio[j],
+ mode_lib->vba.WritebackLumaHTaps[j],
+ mode_lib->vba.WritebackLumaVTaps[j],
+ mode_lib->vba.WritebackChromaHTaps[j],
+ mode_lib->vba.WritebackChromaVTaps[j],
+ mode_lib->vba.WritebackDestinationWidth[j])
+ / mode_lib->vba.DISPCLK);
+ }
+ }
+ }
+ }
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k)
+ for (j = 0; j < mode_lib->vba.NumberOfActivePlanes; ++j)
+ if (mode_lib->vba.BlendingAndTiming[k] == j)
+ mode_lib->vba.WritebackDelay[mode_lib->vba.VoltageLevel][k] =
+ mode_lib->vba.WritebackDelay[mode_lib->vba.VoltageLevel][j];
+
+ mode_lib->vba.VStartupLines = 13;
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ mode_lib->vba.MaxVStartupLines[k] =
+ mode_lib->vba.VTotal[k] - mode_lib->vba.VActive[k]
+ - dml_max(
+ 1.0,
+ dml_ceil(
+ mode_lib->vba.WritebackDelay[mode_lib->vba.VoltageLevel][k]
+ / (mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k]),
+ 1));
+ }
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k)
+ mode_lib->vba.MaximumMaxVStartupLines = dml_max(
+ mode_lib->vba.MaximumMaxVStartupLines,
+ mode_lib->vba.MaxVStartupLines[k]);
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ mode_lib->vba.cursor_bw[k] = 0.0;
+ for (j = 0; j < mode_lib->vba.NumberOfCursors[k]; ++j)
+ mode_lib->vba.cursor_bw[k] += mode_lib->vba.CursorWidth[k][j]
+ * mode_lib->vba.CursorBPP[k][j] / 8.0
+ / (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k])
+ * mode_lib->vba.VRatio[k];
+ }
+
+ do {
+ double MaxTotalRDBandwidth = 0;
+ bool DestinationLineTimesForPrefetchLessThan2 = false;
+ bool VRatioPrefetchMoreThan4 = false;
+ bool prefetch_vm_bw_valid = true;
+ bool prefetch_row_bw_valid = true;
+ double TWait = CalculateTWait(
+ mode_lib->vba.PrefetchMode[mode_lib->vba.VoltageLevel][mode_lib->vba.maxMpcComb],
+ mode_lib->vba.DRAMClockChangeLatency,
+ mode_lib->vba.UrgentLatencyPixelDataOnly,
+ mode_lib->vba.SREnterPlusExitTime);
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.XFCEnabled[k] == true) {
+ mode_lib->vba.XFCRemoteSurfaceFlipDelay =
+ CalculateRemoteSurfaceFlipDelay(
+ mode_lib,
+ mode_lib->vba.VRatio[k],
+ mode_lib->vba.SwathWidthY[k],
+ dml_ceil(
+ mode_lib->vba.BytePerPixelDETY[k],
+ 1),
+ mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k],
+ mode_lib->vba.XFCTSlvVupdateOffset,
+ mode_lib->vba.XFCTSlvVupdateWidth,
+ mode_lib->vba.XFCTSlvVreadyOffset,
+ mode_lib->vba.XFCXBUFLatencyTolerance,
+ mode_lib->vba.XFCFillBWOverhead,
+ mode_lib->vba.XFCSlvChunkSize,
+ mode_lib->vba.XFCBusTransportTime,
+ mode_lib->vba.TCalc,
+ TWait,
+ &mode_lib->vba.SrcActiveDrainRate,
+ &mode_lib->vba.TInitXFill,
+ &mode_lib->vba.TslvChk);
+ } else {
+ mode_lib->vba.XFCRemoteSurfaceFlipDelay = 0;
+ }
+ mode_lib->vba.ErrorResult[k] =
+ CalculatePrefetchSchedule(
+ mode_lib,
+ mode_lib->vba.DPPCLK[k],
+ mode_lib->vba.DISPCLK,
+ mode_lib->vba.PixelClock[k],
+ mode_lib->vba.DCFCLKDeepSleep,
+ mode_lib->vba.DSCDelay[k],
+ mode_lib->vba.DPPPerPlane[k],
+ mode_lib->vba.ScalerEnabled[k],
+ mode_lib->vba.NumberOfCursors[k],
+ mode_lib->vba.DPPCLKDelaySubtotal,
+ mode_lib->vba.DPPCLKDelaySCL,
+ mode_lib->vba.DPPCLKDelaySCLLBOnly,
+ mode_lib->vba.DPPCLKDelayCNVCFormater,
+ mode_lib->vba.DPPCLKDelayCNVCCursor,
+ mode_lib->vba.DISPCLKDelaySubtotal,
+ (unsigned int) (mode_lib->vba.SwathWidthY[k]
+ / mode_lib->vba.HRatio[k]),
+ mode_lib->vba.OutputFormat[k],
+ mode_lib->vba.VTotal[k]
+ - mode_lib->vba.VActive[k],
+ mode_lib->vba.HTotal[k],
+ mode_lib->vba.MaxInterDCNTileRepeaters,
+ dml_min(
+ mode_lib->vba.VStartupLines,
+ mode_lib->vba.MaxVStartupLines[k]),
+ mode_lib->vba.GPUVMMaxPageTableLevels,
+ mode_lib->vba.GPUVMEnable,
+ mode_lib->vba.DynamicMetadataEnable[k],
+ mode_lib->vba.DynamicMetadataLinesBeforeActiveRequired[k],
+ mode_lib->vba.DynamicMetadataTransmittedBytes[k],
+ mode_lib->vba.DCCEnable[k],
+ mode_lib->vba.UrgentLatencyPixelDataOnly,
+ mode_lib->vba.UrgentExtraLatency,
+ mode_lib->vba.TCalc,
+ mode_lib->vba.PDEAndMetaPTEBytesFrame[k],
+ mode_lib->vba.MetaRowByte[k],
+ mode_lib->vba.PixelPTEBytesPerRow[k],
+ mode_lib->vba.PrefetchSourceLinesY[k],
+ mode_lib->vba.SwathWidthY[k],
+ mode_lib->vba.BytePerPixelDETY[k],
+ mode_lib->vba.VInitPreFillY[k],
+ mode_lib->vba.MaxNumSwathY[k],
+ mode_lib->vba.PrefetchSourceLinesC[k],
+ mode_lib->vba.BytePerPixelDETC[k],
+ mode_lib->vba.VInitPreFillC[k],
+ mode_lib->vba.MaxNumSwathC[k],
+ mode_lib->vba.SwathHeightY[k],
+ mode_lib->vba.SwathHeightC[k],
+ TWait,
+ mode_lib->vba.XFCEnabled[k],
+ mode_lib->vba.XFCRemoteSurfaceFlipDelay,
+ mode_lib->vba.Interlace[k],
+ mode_lib->vba.ProgressiveToInterlaceUnitInOPP,
+ &mode_lib->vba.DSTXAfterScaler[k],
+ &mode_lib->vba.DSTYAfterScaler[k],
+ &mode_lib->vba.DestinationLinesForPrefetch[k],
+ &mode_lib->vba.PrefetchBandwidth[k],
+ &mode_lib->vba.DestinationLinesToRequestVMInVBlank[k],
+ &mode_lib->vba.DestinationLinesToRequestRowInVBlank[k],
+ &mode_lib->vba.VRatioPrefetchY[k],
+ &mode_lib->vba.VRatioPrefetchC[k],
+ &mode_lib->vba.RequiredPrefetchPixDataBWLuma[k],
+ &mode_lib->vba.VStartupRequiredWhenNotEnoughTimeForDynamicMetadata,
+ &mode_lib->vba.Tno_bw[k],
+ &mode_lib->vba.VUpdateOffsetPix[k],
+ &mode_lib->vba.VUpdateWidthPix[k],
+ &mode_lib->vba.VReadyOffsetPix[k]);
+ if (mode_lib->vba.BlendingAndTiming[k] == k) {
+ mode_lib->vba.VStartup[k] = dml_min(
+ mode_lib->vba.VStartupLines,
+ mode_lib->vba.MaxVStartupLines[k]);
+ if (mode_lib->vba.VStartupRequiredWhenNotEnoughTimeForDynamicMetadata
+ != 0) {
+ mode_lib->vba.VStartup[k] =
+ mode_lib->vba.VStartupRequiredWhenNotEnoughTimeForDynamicMetadata;
+ }
+ } else {
+ mode_lib->vba.VStartup[k] =
+ dml_min(
+ mode_lib->vba.VStartupLines,
+ mode_lib->vba.MaxVStartupLines[mode_lib->vba.BlendingAndTiming[k]]);
+ }
+ }
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+
+ if (mode_lib->vba.PDEAndMetaPTEBytesFrame[k] == 0)
+ mode_lib->vba.prefetch_vm_bw[k] = 0;
+ else if (mode_lib->vba.DestinationLinesToRequestVMInVBlank[k] > 0) {
+ mode_lib->vba.prefetch_vm_bw[k] =
+ (double) mode_lib->vba.PDEAndMetaPTEBytesFrame[k]
+ / (mode_lib->vba.DestinationLinesToRequestVMInVBlank[k]
+ * mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k]);
+ } else {
+ mode_lib->vba.prefetch_vm_bw[k] = 0;
+ prefetch_vm_bw_valid = false;
+ }
+ if (mode_lib->vba.MetaRowByte[k] + mode_lib->vba.PixelPTEBytesPerRow[k]
+ == 0)
+ mode_lib->vba.prefetch_row_bw[k] = 0;
+ else if (mode_lib->vba.DestinationLinesToRequestRowInVBlank[k] > 0) {
+ mode_lib->vba.prefetch_row_bw[k] =
+ (double) (mode_lib->vba.MetaRowByte[k]
+ + mode_lib->vba.PixelPTEBytesPerRow[k])
+ / (mode_lib->vba.DestinationLinesToRequestRowInVBlank[k]
+ * mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k]);
+ } else {
+ mode_lib->vba.prefetch_row_bw[k] = 0;
+ prefetch_row_bw_valid = false;
+ }
+
+ MaxTotalRDBandwidth =
+ MaxTotalRDBandwidth + mode_lib->vba.cursor_bw[k]
+ + dml_max(
+ mode_lib->vba.prefetch_vm_bw[k],
+ dml_max(
+ mode_lib->vba.prefetch_row_bw[k],
+ dml_max(
+ mode_lib->vba.ReadBandwidthPlaneLuma[k]
+ + mode_lib->vba.ReadBandwidthPlaneChroma[k],
+ mode_lib->vba.RequiredPrefetchPixDataBWLuma[k])
+ + mode_lib->vba.meta_row_bw[k]
+ + mode_lib->vba.dpte_row_bw[k]));
+
+ if (mode_lib->vba.DestinationLinesForPrefetch[k] < 2)
+ DestinationLineTimesForPrefetchLessThan2 = true;
+ if (mode_lib->vba.VRatioPrefetchY[k] > 4
+ || mode_lib->vba.VRatioPrefetchC[k] > 4)
+ VRatioPrefetchMoreThan4 = true;
+ }
+
+ if (MaxTotalRDBandwidth <= mode_lib->vba.ReturnBW && prefetch_vm_bw_valid
+ && prefetch_row_bw_valid && !VRatioPrefetchMoreThan4
+ && !DestinationLineTimesForPrefetchLessThan2)
+ mode_lib->vba.PrefetchModeSupported = true;
+ else {
+ mode_lib->vba.PrefetchModeSupported = false;
+ dml_print(
+ "DML: CalculatePrefetchSchedule ***failed***. Bandwidth violation. Results are NOT valid\n");
+ }
+
+ if (mode_lib->vba.PrefetchModeSupported == true) {
+ double final_flip_bw[DC__NUM_DPP__MAX];
+ unsigned int ImmediateFlipBytes[DC__NUM_DPP__MAX];
+ double total_dcn_read_bw_with_flip = 0;
+
+ mode_lib->vba.BandwidthAvailableForImmediateFlip = mode_lib->vba.ReturnBW;
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ mode_lib->vba.BandwidthAvailableForImmediateFlip =
+ mode_lib->vba.BandwidthAvailableForImmediateFlip
+ - mode_lib->vba.cursor_bw[k]
+ - dml_max(
+ mode_lib->vba.ReadBandwidthPlaneLuma[k]
+ + mode_lib->vba.ReadBandwidthPlaneChroma[k]
+ + mode_lib->vba.qual_row_bw[k],
+ mode_lib->vba.PrefetchBandwidth[k]);
+ }
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ ImmediateFlipBytes[k] = 0;
+ if ((mode_lib->vba.SourcePixelFormat[k] != dm_420_8
+ && mode_lib->vba.SourcePixelFormat[k] != dm_420_10)) {
+ ImmediateFlipBytes[k] =
+ mode_lib->vba.PDEAndMetaPTEBytesFrame[k]
+ + mode_lib->vba.MetaRowByte[k]
+ + mode_lib->vba.PixelPTEBytesPerRow[k];
+ }
+ }
+ mode_lib->vba.TotImmediateFlipBytes = 0;
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if ((mode_lib->vba.SourcePixelFormat[k] != dm_420_8
+ && mode_lib->vba.SourcePixelFormat[k] != dm_420_10)) {
+ mode_lib->vba.TotImmediateFlipBytes =
+ mode_lib->vba.TotImmediateFlipBytes
+ + ImmediateFlipBytes[k];
+ }
+ }
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ CalculateFlipSchedule(
+ mode_lib,
+ mode_lib->vba.UrgentExtraLatency,
+ mode_lib->vba.UrgentLatencyPixelDataOnly,
+ mode_lib->vba.GPUVMMaxPageTableLevels,
+ mode_lib->vba.GPUVMEnable,
+ mode_lib->vba.BandwidthAvailableForImmediateFlip,
+ mode_lib->vba.TotImmediateFlipBytes,
+ mode_lib->vba.SourcePixelFormat[k],
+ ImmediateFlipBytes[k],
+ mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k],
+ mode_lib->vba.VRatio[k],
+ mode_lib->vba.Tno_bw[k],
+ mode_lib->vba.PDEAndMetaPTEBytesFrame[k],
+ mode_lib->vba.MetaRowByte[k],
+ mode_lib->vba.PixelPTEBytesPerRow[k],
+ mode_lib->vba.DCCEnable[k],
+ mode_lib->vba.dpte_row_height[k],
+ mode_lib->vba.meta_row_height[k],
+ mode_lib->vba.qual_row_bw[k],
+ &mode_lib->vba.DestinationLinesToRequestVMInImmediateFlip[k],
+ &mode_lib->vba.DestinationLinesToRequestRowInImmediateFlip[k],
+ &final_flip_bw[k],
+ &mode_lib->vba.ImmediateFlipSupportedForPipe[k]);
+ }
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ total_dcn_read_bw_with_flip =
+ total_dcn_read_bw_with_flip
+ + mode_lib->vba.cursor_bw[k]
+ + dml_max(
+ mode_lib->vba.prefetch_vm_bw[k],
+ dml_max(
+ mode_lib->vba.prefetch_row_bw[k],
+ final_flip_bw[k]
+ + dml_max(
+ mode_lib->vba.ReadBandwidthPlaneLuma[k]
+ + mode_lib->vba.ReadBandwidthPlaneChroma[k],
+ mode_lib->vba.RequiredPrefetchPixDataBWLuma[k])));
+ }
+ mode_lib->vba.ImmediateFlipSupported = true;
+ if (total_dcn_read_bw_with_flip > mode_lib->vba.ReturnBW) {
+ mode_lib->vba.ImmediateFlipSupported = false;
+ }
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.ImmediateFlipSupportedForPipe[k] == false) {
+ mode_lib->vba.ImmediateFlipSupported = false;
+ }
+ }
+ } else {
+ mode_lib->vba.ImmediateFlipSupported = false;
+ }
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.ErrorResult[k]) {
+ mode_lib->vba.PrefetchModeSupported = false;
+ dml_print(
+ "DML: CalculatePrefetchSchedule ***failed***. Prefetch schedule violation. Results are NOT valid\n");
+ }
+ }
+
+ mode_lib->vba.VStartupLines = mode_lib->vba.VStartupLines + 1;
+ } while (!((mode_lib->vba.PrefetchModeSupported
+ && (!mode_lib->vba.ImmediateFlipSupport
+ || mode_lib->vba.ImmediateFlipSupported))
+ || mode_lib->vba.MaximumMaxVStartupLines < mode_lib->vba.VStartupLines));
+
+ //Display Pipeline Delivery Time in Prefetch
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.VRatioPrefetchY[k] <= 1) {
+ mode_lib->vba.DisplayPipeLineDeliveryTimeLumaPrefetch[k] =
+ mode_lib->vba.SwathWidthY[k] * mode_lib->vba.DPPPerPlane[k]
+ / mode_lib->vba.HRatio[k]
+ / mode_lib->vba.PixelClock[k];
+ } else {
+ mode_lib->vba.DisplayPipeLineDeliveryTimeLumaPrefetch[k] =
+ mode_lib->vba.SwathWidthY[k]
+ / mode_lib->vba.PSCL_THROUGHPUT_LUMA[k]
+ / mode_lib->vba.DPPCLK[k];
+ }
+ if (mode_lib->vba.BytePerPixelDETC[k] == 0) {
+ mode_lib->vba.DisplayPipeLineDeliveryTimeChromaPrefetch[k] = 0;
+ } else {
+ if (mode_lib->vba.VRatioPrefetchC[k] <= 1) {
+ mode_lib->vba.DisplayPipeLineDeliveryTimeChromaPrefetch[k] =
+ mode_lib->vba.SwathWidthY[k]
+ * mode_lib->vba.DPPPerPlane[k]
+ / mode_lib->vba.HRatio[k]
+ / mode_lib->vba.PixelClock[k];
+ } else {
+ mode_lib->vba.DisplayPipeLineDeliveryTimeChromaPrefetch[k] =
+ mode_lib->vba.SwathWidthY[k]
+ / mode_lib->vba.PSCL_THROUGHPUT_LUMA[k]
+ / mode_lib->vba.DPPCLK[k];
+ }
+ }
+ }
+
+ // Min TTUVBlank
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.PrefetchMode[mode_lib->vba.VoltageLevel][mode_lib->vba.maxMpcComb] == 0) {
+ mode_lib->vba.AllowDRAMClockChangeDuringVBlank[k] = true;
+ mode_lib->vba.AllowDRAMSelfRefreshDuringVBlank[k] = true;
+ mode_lib->vba.MinTTUVBlank[k] = dml_max(
+ mode_lib->vba.DRAMClockChangeWatermark,
+ dml_max(
+ mode_lib->vba.StutterEnterPlusExitWatermark,
+ mode_lib->vba.UrgentWatermark));
+ } else if (mode_lib->vba.PrefetchMode[mode_lib->vba.VoltageLevel][mode_lib->vba.maxMpcComb] == 1) {
+ mode_lib->vba.AllowDRAMClockChangeDuringVBlank[k] = false;
+ mode_lib->vba.AllowDRAMSelfRefreshDuringVBlank[k] = true;
+ mode_lib->vba.MinTTUVBlank[k] = dml_max(
+ mode_lib->vba.StutterEnterPlusExitWatermark,
+ mode_lib->vba.UrgentWatermark);
+ } else {
+ mode_lib->vba.AllowDRAMClockChangeDuringVBlank[k] = false;
+ mode_lib->vba.AllowDRAMSelfRefreshDuringVBlank[k] = false;
+ mode_lib->vba.MinTTUVBlank[k] = mode_lib->vba.UrgentWatermark;
+ }
+ if (!mode_lib->vba.DynamicMetadataEnable[k])
+ mode_lib->vba.MinTTUVBlank[k] = mode_lib->vba.TCalc
+ + mode_lib->vba.MinTTUVBlank[k];
+ }
+
+ // DCC Configuration
+ mode_lib->vba.ActiveDPPs = 0;
+ // NB P-State/DRAM Clock Change Support
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ mode_lib->vba.ActiveDPPs = mode_lib->vba.ActiveDPPs + mode_lib->vba.DPPPerPlane[k];
+ }
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ double EffectiveLBLatencyHidingY;
+ double EffectiveLBLatencyHidingC;
+ double DPPOutputBufferLinesY;
+ double DPPOutputBufferLinesC;
+ double DPPOPPBufferingY;
+ double MaxDETBufferingTimeY;
+ double ActiveDRAMClockChangeLatencyMarginY;
+
+ mode_lib->vba.LBLatencyHidingSourceLinesY =
+ dml_min(
+ mode_lib->vba.MaxLineBufferLines,
+ (unsigned int) dml_floor(
+ (double) mode_lib->vba.LineBufferSize
+ / mode_lib->vba.LBBitPerPixel[k]
+ / (mode_lib->vba.SwathWidthY[k]
+ / dml_max(
+ mode_lib->vba.HRatio[k],
+ 1.0)),
+ 1)) - (mode_lib->vba.vtaps[k] - 1);
+
+ mode_lib->vba.LBLatencyHidingSourceLinesC =
+ dml_min(
+ mode_lib->vba.MaxLineBufferLines,
+ (unsigned int) dml_floor(
+ (double) mode_lib->vba.LineBufferSize
+ / mode_lib->vba.LBBitPerPixel[k]
+ / (mode_lib->vba.SwathWidthY[k]
+ / 2.0
+ / dml_max(
+ mode_lib->vba.HRatio[k]
+ / 2,
+ 1.0)),
+ 1))
+ - (mode_lib->vba.VTAPsChroma[k] - 1);
+
+ EffectiveLBLatencyHidingY = mode_lib->vba.LBLatencyHidingSourceLinesY
+ / mode_lib->vba.VRatio[k]
+ * (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k]);
+
+ EffectiveLBLatencyHidingC = mode_lib->vba.LBLatencyHidingSourceLinesC
+ / (mode_lib->vba.VRatio[k] / 2)
+ * (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k]);
+
+ if (mode_lib->vba.SwathWidthY[k] > 2 * mode_lib->vba.DPPOutputBufferPixels) {
+ DPPOutputBufferLinesY = mode_lib->vba.DPPOutputBufferPixels
+ / mode_lib->vba.SwathWidthY[k];
+ } else if (mode_lib->vba.SwathWidthY[k] > mode_lib->vba.DPPOutputBufferPixels) {
+ DPPOutputBufferLinesY = 0.5;
+ } else {
+ DPPOutputBufferLinesY = 1;
+ }
+
+ if (mode_lib->vba.SwathWidthY[k] / 2 > 2 * mode_lib->vba.DPPOutputBufferPixels) {
+ DPPOutputBufferLinesC = mode_lib->vba.DPPOutputBufferPixels
+ / (mode_lib->vba.SwathWidthY[k] / 2);
+ } else if (mode_lib->vba.SwathWidthY[k] / 2 > mode_lib->vba.DPPOutputBufferPixels) {
+ DPPOutputBufferLinesC = 0.5;
+ } else {
+ DPPOutputBufferLinesC = 1;
+ }
+
+ DPPOPPBufferingY = (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k])
+ * (DPPOutputBufferLinesY + mode_lib->vba.OPPOutputBufferLines);
+ MaxDETBufferingTimeY = mode_lib->vba.FullDETBufferingTimeY[k]
+ + (mode_lib->vba.LinesInDETY[k]
+ - mode_lib->vba.LinesInDETYRoundedDownToSwath[k])
+ / mode_lib->vba.SwathHeightY[k]
+ * (mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k]);
+
+ ActiveDRAMClockChangeLatencyMarginY = DPPOPPBufferingY + EffectiveLBLatencyHidingY
+ + MaxDETBufferingTimeY - mode_lib->vba.DRAMClockChangeWatermark;
+
+ if (mode_lib->vba.ActiveDPPs > 1) {
+ ActiveDRAMClockChangeLatencyMarginY =
+ ActiveDRAMClockChangeLatencyMarginY
+ - (1 - 1 / (mode_lib->vba.ActiveDPPs - 1))
+ * mode_lib->vba.SwathHeightY[k]
+ * (mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k]);
+ }
+
+ if (mode_lib->vba.BytePerPixelDETC[k] > 0) {
+ double DPPOPPBufferingC = (mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k])
+ * (DPPOutputBufferLinesC
+ + mode_lib->vba.OPPOutputBufferLines);
+ double MaxDETBufferingTimeC =
+ mode_lib->vba.FullDETBufferingTimeC[k]
+ + (mode_lib->vba.LinesInDETC[k]
+ - mode_lib->vba.LinesInDETCRoundedDownToSwath[k])
+ / mode_lib->vba.SwathHeightC[k]
+ * (mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k]);
+ double ActiveDRAMClockChangeLatencyMarginC = DPPOPPBufferingC
+ + EffectiveLBLatencyHidingC + MaxDETBufferingTimeC
+ - mode_lib->vba.DRAMClockChangeWatermark;
+
+ if (mode_lib->vba.ActiveDPPs > 1) {
+ ActiveDRAMClockChangeLatencyMarginC =
+ ActiveDRAMClockChangeLatencyMarginC
+ - (1
+ - 1
+ / (mode_lib->vba.ActiveDPPs
+ - 1))
+ * mode_lib->vba.SwathHeightC[k]
+ * (mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k]);
+ }
+ mode_lib->vba.ActiveDRAMClockChangeLatencyMargin[k] = dml_min(
+ ActiveDRAMClockChangeLatencyMarginY,
+ ActiveDRAMClockChangeLatencyMarginC);
+ } else {
+ mode_lib->vba.ActiveDRAMClockChangeLatencyMargin[k] =
+ ActiveDRAMClockChangeLatencyMarginY;
+ }
+
+ if (mode_lib->vba.WritebackEnable[k]) {
+ double WritebackDRAMClockChangeLatencyMargin;
+
+ if (mode_lib->vba.WritebackPixelFormat[k] == dm_444_32) {
+ WritebackDRAMClockChangeLatencyMargin =
+ (double) (mode_lib->vba.WritebackInterfaceLumaBufferSize
+ + mode_lib->vba.WritebackInterfaceChromaBufferSize)
+ / (mode_lib->vba.WritebackDestinationWidth[k]
+ * mode_lib->vba.WritebackDestinationHeight[k]
+ / (mode_lib->vba.WritebackSourceHeight[k]
+ * mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k])
+ * 4)
+ - mode_lib->vba.WritebackDRAMClockChangeWatermark;
+ } else if (mode_lib->vba.WritebackPixelFormat[k] == dm_420_10) {
+ WritebackDRAMClockChangeLatencyMargin =
+ dml_min(
+ (double) mode_lib->vba.WritebackInterfaceLumaBufferSize
+ * 8.0 / 10,
+ 2.0
+ * mode_lib->vba.WritebackInterfaceChromaBufferSize
+ * 8 / 10)
+ / (mode_lib->vba.WritebackDestinationWidth[k]
+ * mode_lib->vba.WritebackDestinationHeight[k]
+ / (mode_lib->vba.WritebackSourceHeight[k]
+ * mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k]))
+ - mode_lib->vba.WritebackDRAMClockChangeWatermark;
+ } else {
+ WritebackDRAMClockChangeLatencyMargin =
+ dml_min(
+ (double) mode_lib->vba.WritebackInterfaceLumaBufferSize,
+ 2.0
+ * mode_lib->vba.WritebackInterfaceChromaBufferSize)
+ / (mode_lib->vba.WritebackDestinationWidth[k]
+ * mode_lib->vba.WritebackDestinationHeight[k]
+ / (mode_lib->vba.WritebackSourceHeight[k]
+ * mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k]))
+ - mode_lib->vba.WritebackDRAMClockChangeWatermark;
+ }
+ mode_lib->vba.ActiveDRAMClockChangeLatencyMargin[k] = dml_min(
+ mode_lib->vba.ActiveDRAMClockChangeLatencyMargin[k],
+ WritebackDRAMClockChangeLatencyMargin);
+ }
+ }
+
+ mode_lib->vba.MinActiveDRAMClockChangeMargin = 999999;
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.ActiveDRAMClockChangeLatencyMargin[k]
+ < mode_lib->vba.MinActiveDRAMClockChangeMargin) {
+ mode_lib->vba.MinActiveDRAMClockChangeMargin =
+ mode_lib->vba.ActiveDRAMClockChangeLatencyMargin[k];
+ }
+ }
+
+ mode_lib->vba.MinActiveDRAMClockChangeLatencySupported =
+ mode_lib->vba.MinActiveDRAMClockChangeMargin
+ + mode_lib->vba.DRAMClockChangeLatency;
+
+ if (mode_lib->vba.MinActiveDRAMClockChangeMargin > 0) {
+ mode_lib->vba.DRAMClockChangeSupport[0][0] = dm_dram_clock_change_vactive;
+ } else {
+ if (mode_lib->vba.SynchronizedVBlank || mode_lib->vba.NumberOfActivePlanes == 1) {
+ mode_lib->vba.DRAMClockChangeSupport[0][0] = dm_dram_clock_change_vblank;
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (!mode_lib->vba.AllowDRAMClockChangeDuringVBlank[k]) {
+ mode_lib->vba.DRAMClockChangeSupport[0][0] =
+ dm_dram_clock_change_unsupported;
+ }
+ }
+ } else {
+ mode_lib->vba.DRAMClockChangeSupport[0][0] = dm_dram_clock_change_unsupported;
+ }
+ }
+ for (k = 0; k <= mode_lib->vba.soc.num_states; k++)
+ for (j = 0; j < 2; j++)
+ mode_lib->vba.DRAMClockChangeSupport[k][j] = mode_lib->vba.DRAMClockChangeSupport[0][0];
+
+ //XFC Parameters:
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.XFCEnabled[k] == true) {
+ double TWait;
+
+ mode_lib->vba.XFCSlaveVUpdateOffset[k] = mode_lib->vba.XFCTSlvVupdateOffset;
+ mode_lib->vba.XFCSlaveVupdateWidth[k] = mode_lib->vba.XFCTSlvVupdateWidth;
+ mode_lib->vba.XFCSlaveVReadyOffset[k] = mode_lib->vba.XFCTSlvVreadyOffset;
+ TWait = CalculateTWait(
+ mode_lib->vba.PrefetchMode[mode_lib->vba.VoltageLevel][mode_lib->vba.maxMpcComb],
+ mode_lib->vba.DRAMClockChangeLatency,
+ mode_lib->vba.UrgentLatencyPixelDataOnly,
+ mode_lib->vba.SREnterPlusExitTime);
+ mode_lib->vba.XFCRemoteSurfaceFlipDelay = CalculateRemoteSurfaceFlipDelay(
+ mode_lib,
+ mode_lib->vba.VRatio[k],
+ mode_lib->vba.SwathWidthY[k],
+ dml_ceil(mode_lib->vba.BytePerPixelDETY[k], 1),
+ mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k],
+ mode_lib->vba.XFCTSlvVupdateOffset,
+ mode_lib->vba.XFCTSlvVupdateWidth,
+ mode_lib->vba.XFCTSlvVreadyOffset,
+ mode_lib->vba.XFCXBUFLatencyTolerance,
+ mode_lib->vba.XFCFillBWOverhead,
+ mode_lib->vba.XFCSlvChunkSize,
+ mode_lib->vba.XFCBusTransportTime,
+ mode_lib->vba.TCalc,
+ TWait,
+ &mode_lib->vba.SrcActiveDrainRate,
+ &mode_lib->vba.TInitXFill,
+ &mode_lib->vba.TslvChk);
+ mode_lib->vba.XFCRemoteSurfaceFlipLatency[k] =
+ dml_floor(
+ mode_lib->vba.XFCRemoteSurfaceFlipDelay
+ / (mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k]),
+ 1);
+ mode_lib->vba.XFCTransferDelay[k] =
+ dml_ceil(
+ mode_lib->vba.XFCBusTransportTime
+ / (mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k]),
+ 1);
+ mode_lib->vba.XFCPrechargeDelay[k] =
+ dml_ceil(
+ (mode_lib->vba.XFCBusTransportTime
+ + mode_lib->vba.TInitXFill
+ + mode_lib->vba.TslvChk)
+ / (mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k]),
+ 1);
+ mode_lib->vba.InitFillLevel = mode_lib->vba.XFCXBUFLatencyTolerance
+ * mode_lib->vba.SrcActiveDrainRate;
+ mode_lib->vba.FinalFillMargin =
+ (mode_lib->vba.DestinationLinesToRequestVMInVBlank[k]
+ + mode_lib->vba.DestinationLinesToRequestRowInVBlank[k])
+ * mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k]
+ * mode_lib->vba.SrcActiveDrainRate
+ + mode_lib->vba.XFCFillConstant;
+ mode_lib->vba.FinalFillLevel = mode_lib->vba.XFCRemoteSurfaceFlipDelay
+ * mode_lib->vba.SrcActiveDrainRate
+ + mode_lib->vba.FinalFillMargin;
+ mode_lib->vba.RemainingFillLevel = dml_max(
+ 0.0,
+ mode_lib->vba.FinalFillLevel - mode_lib->vba.InitFillLevel);
+ mode_lib->vba.TFinalxFill = mode_lib->vba.RemainingFillLevel
+ / (mode_lib->vba.SrcActiveDrainRate
+ * mode_lib->vba.XFCFillBWOverhead / 100);
+ mode_lib->vba.XFCPrefetchMargin[k] =
+ mode_lib->vba.XFCRemoteSurfaceFlipDelay
+ + mode_lib->vba.TFinalxFill
+ + (mode_lib->vba.DestinationLinesToRequestVMInVBlank[k]
+ + mode_lib->vba.DestinationLinesToRequestRowInVBlank[k])
+ * mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k];
+ } else {
+ mode_lib->vba.XFCSlaveVUpdateOffset[k] = 0;
+ mode_lib->vba.XFCSlaveVupdateWidth[k] = 0;
+ mode_lib->vba.XFCSlaveVReadyOffset[k] = 0;
+ mode_lib->vba.XFCRemoteSurfaceFlipLatency[k] = 0;
+ mode_lib->vba.XFCPrechargeDelay[k] = 0;
+ mode_lib->vba.XFCTransferDelay[k] = 0;
+ mode_lib->vba.XFCPrefetchMargin[k] = 0;
+ }
+ }
+ {
+ unsigned int VStartupMargin = 0;
+ bool FirstMainPlane = true;
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.BlendingAndTiming[k] == k) {
+ unsigned int Margin = (mode_lib->vba.MaxVStartupLines[k] - mode_lib->vba.VStartup[k])
+ * mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k];
+
+ if (FirstMainPlane) {
+ VStartupMargin = Margin;
+ FirstMainPlane = false;
+ } else
+ VStartupMargin = dml_min(VStartupMargin, Margin);
+ }
+
+ if (mode_lib->vba.UseMaximumVStartup) {
+ if (mode_lib->vba.VTotal_Max[k] == mode_lib->vba.VTotal[k]) {
+ //only use max vstart if it is not drr or lateflip.
+ mode_lib->vba.VStartup[k] = mode_lib->vba.MaxVStartupLines[mode_lib->vba.BlendingAndTiming[k]];
+ }
+ }
+ }
+}
+}
+
+static void dml20_DisplayPipeConfiguration(struct display_mode_lib *mode_lib)
+{
+ double BytePerPixDETY;
+ double BytePerPixDETC;
+ double Read256BytesBlockHeightY;
+ double Read256BytesBlockHeightC;
+ double Read256BytesBlockWidthY;
+ double Read256BytesBlockWidthC;
+ double MaximumSwathHeightY;
+ double MaximumSwathHeightC;
+ double MinimumSwathHeightY;
+ double MinimumSwathHeightC;
+ double SwathWidth;
+ double SwathWidthGranularityY;
+ double SwathWidthGranularityC;
+ double RoundedUpMaxSwathSizeBytesY;
+ double RoundedUpMaxSwathSizeBytesC;
+ unsigned int j, k;
+
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ bool MainPlaneDoesODMCombine = false;
+
+ if (mode_lib->vba.SourcePixelFormat[k] == dm_444_64) {
+ BytePerPixDETY = 8;
+ BytePerPixDETC = 0;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_444_32) {
+ BytePerPixDETY = 4;
+ BytePerPixDETC = 0;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_444_16) {
+ BytePerPixDETY = 2;
+ BytePerPixDETC = 0;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_444_8) {
+ BytePerPixDETY = 1;
+ BytePerPixDETC = 0;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_420_8) {
+ BytePerPixDETY = 1;
+ BytePerPixDETC = 2;
+ } else {
+ BytePerPixDETY = 4.0 / 3.0;
+ BytePerPixDETC = 8.0 / 3.0;
+ }
+
+ if ((mode_lib->vba.SourcePixelFormat[k] == dm_444_64
+ || mode_lib->vba.SourcePixelFormat[k] == dm_444_32
+ || mode_lib->vba.SourcePixelFormat[k] == dm_444_16
+ || mode_lib->vba.SourcePixelFormat[k] == dm_444_8)) {
+ if (mode_lib->vba.SurfaceTiling[k] == dm_sw_linear) {
+ Read256BytesBlockHeightY = 1;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_444_64) {
+ Read256BytesBlockHeightY = 4;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_444_32
+ || mode_lib->vba.SourcePixelFormat[k] == dm_444_16) {
+ Read256BytesBlockHeightY = 8;
+ } else {
+ Read256BytesBlockHeightY = 16;
+ }
+ Read256BytesBlockWidthY = 256 / dml_ceil(BytePerPixDETY, 1)
+ / Read256BytesBlockHeightY;
+ Read256BytesBlockHeightC = 0;
+ Read256BytesBlockWidthC = 0;
+ } else {
+ if (mode_lib->vba.SurfaceTiling[k] == dm_sw_linear) {
+ Read256BytesBlockHeightY = 1;
+ Read256BytesBlockHeightC = 1;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_420_8) {
+ Read256BytesBlockHeightY = 16;
+ Read256BytesBlockHeightC = 8;
+ } else {
+ Read256BytesBlockHeightY = 8;
+ Read256BytesBlockHeightC = 8;
+ }
+ Read256BytesBlockWidthY = 256 / dml_ceil(BytePerPixDETY, 1)
+ / Read256BytesBlockHeightY;
+ Read256BytesBlockWidthC = 256 / dml_ceil(BytePerPixDETC, 2)
+ / Read256BytesBlockHeightC;
+ }
+
+ if (mode_lib->vba.SourceScan[k] == dm_horz) {
+ MaximumSwathHeightY = Read256BytesBlockHeightY;
+ MaximumSwathHeightC = Read256BytesBlockHeightC;
+ } else {
+ MaximumSwathHeightY = Read256BytesBlockWidthY;
+ MaximumSwathHeightC = Read256BytesBlockWidthC;
+ }
+
+ if ((mode_lib->vba.SourcePixelFormat[k] == dm_444_64
+ || mode_lib->vba.SourcePixelFormat[k] == dm_444_32
+ || mode_lib->vba.SourcePixelFormat[k] == dm_444_16
+ || mode_lib->vba.SourcePixelFormat[k] == dm_444_8)) {
+ if (mode_lib->vba.SurfaceTiling[k] == dm_sw_linear
+ || (mode_lib->vba.SourcePixelFormat[k] == dm_444_64
+ && (mode_lib->vba.SurfaceTiling[k]
+ == dm_sw_4kb_s
+ || mode_lib->vba.SurfaceTiling[k]
+ == dm_sw_4kb_s_x
+ || mode_lib->vba.SurfaceTiling[k]
+ == dm_sw_64kb_s
+ || mode_lib->vba.SurfaceTiling[k]
+ == dm_sw_64kb_s_t
+ || mode_lib->vba.SurfaceTiling[k]
+ == dm_sw_64kb_s_x
+ || mode_lib->vba.SurfaceTiling[k]
+ == dm_sw_var_s
+ || mode_lib->vba.SurfaceTiling[k]
+ == dm_sw_var_s_x)
+ && mode_lib->vba.SourceScan[k] == dm_horz)) {
+ MinimumSwathHeightY = MaximumSwathHeightY;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_444_8
+ && mode_lib->vba.SourceScan[k] != dm_horz) {
+ MinimumSwathHeightY = MaximumSwathHeightY;
+ } else {
+ MinimumSwathHeightY = MaximumSwathHeightY / 2.0;
+ }
+ MinimumSwathHeightC = MaximumSwathHeightC;
+ } else {
+ if (mode_lib->vba.SurfaceTiling[k] == dm_sw_linear) {
+ MinimumSwathHeightY = MaximumSwathHeightY;
+ MinimumSwathHeightC = MaximumSwathHeightC;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_420_8
+ && mode_lib->vba.SourceScan[k] == dm_horz) {
+ MinimumSwathHeightY = MaximumSwathHeightY / 2.0;
+ MinimumSwathHeightC = MaximumSwathHeightC;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_420_10
+ && mode_lib->vba.SourceScan[k] == dm_horz) {
+ MinimumSwathHeightC = MaximumSwathHeightC / 2.0;
+ MinimumSwathHeightY = MaximumSwathHeightY;
+ } else {
+ MinimumSwathHeightY = MaximumSwathHeightY;
+ MinimumSwathHeightC = MaximumSwathHeightC;
+ }
+ }
+
+ if (mode_lib->vba.SourceScan[k] == dm_horz) {
+ SwathWidth = mode_lib->vba.ViewportWidth[k];
+ } else {
+ SwathWidth = mode_lib->vba.ViewportHeight[k];
+ }
+
+ if (mode_lib->vba.ODMCombineEnabled[k] == true) {
+ MainPlaneDoesODMCombine = true;
+ }
+ for (j = 0; j < mode_lib->vba.NumberOfActivePlanes; ++j) {
+ if (mode_lib->vba.BlendingAndTiming[k] == j
+ && mode_lib->vba.ODMCombineEnabled[j] == true) {
+ MainPlaneDoesODMCombine = true;
+ }
+ }
+
+ if (MainPlaneDoesODMCombine == true) {
+ SwathWidth = dml_min(
+ SwathWidth,
+ mode_lib->vba.HActive[k] / 2.0 * mode_lib->vba.HRatio[k]);
+ } else {
+ if (mode_lib->vba.DPPPerPlane[k] == 0)
+ SwathWidth = 0;
+ else
+ SwathWidth = SwathWidth / mode_lib->vba.DPPPerPlane[k];
+ }
+
+ SwathWidthGranularityY = 256 / dml_ceil(BytePerPixDETY, 1) / MaximumSwathHeightY;
+ RoundedUpMaxSwathSizeBytesY = (dml_ceil(
+ (double) (SwathWidth - 1),
+ SwathWidthGranularityY) + SwathWidthGranularityY) * BytePerPixDETY
+ * MaximumSwathHeightY;
+ if (mode_lib->vba.SourcePixelFormat[k] == dm_420_10) {
+ RoundedUpMaxSwathSizeBytesY = dml_ceil(RoundedUpMaxSwathSizeBytesY, 256)
+ + 256;
+ }
+ if (MaximumSwathHeightC > 0) {
+ SwathWidthGranularityC = 256.0 / dml_ceil(BytePerPixDETC, 2)
+ / MaximumSwathHeightC;
+ RoundedUpMaxSwathSizeBytesC = (dml_ceil(
+ (double) (SwathWidth / 2.0 - 1),
+ SwathWidthGranularityC) + SwathWidthGranularityC)
+ * BytePerPixDETC * MaximumSwathHeightC;
+ if (mode_lib->vba.SourcePixelFormat[k] == dm_420_10) {
+ RoundedUpMaxSwathSizeBytesC = dml_ceil(
+ RoundedUpMaxSwathSizeBytesC,
+ 256) + 256;
+ }
+ } else
+ RoundedUpMaxSwathSizeBytesC = 0.0;
+
+ if (RoundedUpMaxSwathSizeBytesY + RoundedUpMaxSwathSizeBytesC
+ <= mode_lib->vba.DETBufferSizeInKByte * 1024.0 / 2.0) {
+ mode_lib->vba.SwathHeightY[k] = MaximumSwathHeightY;
+ mode_lib->vba.SwathHeightC[k] = MaximumSwathHeightC;
+ } else {
+ mode_lib->vba.SwathHeightY[k] = MinimumSwathHeightY;
+ mode_lib->vba.SwathHeightC[k] = MinimumSwathHeightC;
+ }
+
+ if (mode_lib->vba.SwathHeightC[k] == 0) {
+ mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte * 1024;
+ mode_lib->vba.DETBufferSizeC[k] = 0;
+ } else if (mode_lib->vba.SwathHeightY[k] <= mode_lib->vba.SwathHeightC[k]) {
+ mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte
+ * 1024.0 / 2;
+ mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte
+ * 1024.0 / 2;
+ } else {
+ mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte
+ * 1024.0 * 2 / 3;
+ mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte
+ * 1024.0 / 3;
+ }
+ }
+}
+
+static double CalculateTWait(
+ unsigned int PrefetchMode,
+ double DRAMClockChangeLatency,
+ double UrgentLatencyPixelDataOnly,
+ double SREnterPlusExitTime)
+{
+ if (PrefetchMode == 0) {
+ return dml_max(
+ DRAMClockChangeLatency + UrgentLatencyPixelDataOnly,
+ dml_max(SREnterPlusExitTime, UrgentLatencyPixelDataOnly));
+ } else if (PrefetchMode == 1) {
+ return dml_max(SREnterPlusExitTime, UrgentLatencyPixelDataOnly);
+ } else {
+ return UrgentLatencyPixelDataOnly;
+ }
+}
+
+static double CalculateRemoteSurfaceFlipDelay(
+ struct display_mode_lib *mode_lib,
+ double VRatio,
+ double SwathWidth,
+ double Bpp,
+ double LineTime,
+ double XFCTSlvVupdateOffset,
+ double XFCTSlvVupdateWidth,
+ double XFCTSlvVreadyOffset,
+ double XFCXBUFLatencyTolerance,
+ double XFCFillBWOverhead,
+ double XFCSlvChunkSize,
+ double XFCBusTransportTime,
+ double TCalc,
+ double TWait,
+ double *SrcActiveDrainRate,
+ double *TInitXFill,
+ double *TslvChk)
+{
+ double TSlvSetup, AvgfillRate, result;
+
+ *SrcActiveDrainRate = VRatio * SwathWidth * Bpp / LineTime;
+ TSlvSetup = XFCTSlvVupdateOffset + XFCTSlvVupdateWidth + XFCTSlvVreadyOffset;
+ *TInitXFill = XFCXBUFLatencyTolerance / (1 + XFCFillBWOverhead / 100);
+ AvgfillRate = *SrcActiveDrainRate * (1 + XFCFillBWOverhead / 100);
+ *TslvChk = XFCSlvChunkSize / AvgfillRate;
+ dml_print(
+ "DML::CalculateRemoteSurfaceFlipDelay: SrcActiveDrainRate: %f\n",
+ *SrcActiveDrainRate);
+ dml_print("DML::CalculateRemoteSurfaceFlipDelay: TSlvSetup: %f\n", TSlvSetup);
+ dml_print("DML::CalculateRemoteSurfaceFlipDelay: TInitXFill: %f\n", *TInitXFill);
+ dml_print("DML::CalculateRemoteSurfaceFlipDelay: AvgfillRate: %f\n", AvgfillRate);
+ dml_print("DML::CalculateRemoteSurfaceFlipDelay: TslvChk: %f\n", *TslvChk);
+ result = 2 * XFCBusTransportTime + TSlvSetup + TCalc + TWait + *TslvChk + *TInitXFill; // TODO: This doesn't seem to match programming guide
+ dml_print("DML::CalculateRemoteSurfaceFlipDelay: RemoteSurfaceFlipDelay: %f\n", result);
+ return result;
+}
+
+static double CalculateWriteBackDelay(
+ enum source_format_class WritebackPixelFormat,
+ double WritebackHRatio,
+ double WritebackVRatio,
+ unsigned int WritebackLumaHTaps,
+ unsigned int WritebackLumaVTaps,
+ unsigned int WritebackChromaHTaps,
+ unsigned int WritebackChromaVTaps,
+ unsigned int WritebackDestinationWidth)
+{
+ double CalculateWriteBackDelay =
+ dml_max(
+ dml_ceil(WritebackLumaHTaps / 4.0, 1) / WritebackHRatio,
+ WritebackLumaVTaps * dml_ceil(1.0 / WritebackVRatio, 1)
+ * dml_ceil(
+ WritebackDestinationWidth
+ / 4.0,
+ 1)
+ + dml_ceil(1.0 / WritebackVRatio, 1)
+ * (dml_ceil(
+ WritebackLumaVTaps
+ / 4.0,
+ 1) + 4));
+
+ if (WritebackPixelFormat != dm_444_32) {
+ CalculateWriteBackDelay =
+ dml_max(
+ CalculateWriteBackDelay,
+ dml_max(
+ dml_ceil(
+ WritebackChromaHTaps
+ / 2.0,
+ 1)
+ / (2
+ * WritebackHRatio),
+ WritebackChromaVTaps
+ * dml_ceil(
+ 1
+ / (2
+ * WritebackVRatio),
+ 1)
+ * dml_ceil(
+ WritebackDestinationWidth
+ / 2.0
+ / 2.0,
+ 1)
+ + dml_ceil(
+ 1
+ / (2
+ * WritebackVRatio),
+ 1)
+ * (dml_ceil(
+ WritebackChromaVTaps
+ / 4.0,
+ 1)
+ + 4)));
+ }
+ return CalculateWriteBackDelay;
+}
+
+static void CalculateActiveRowBandwidth(
+ bool GPUVMEnable,
+ enum source_format_class SourcePixelFormat,
+ double VRatio,
+ bool DCCEnable,
+ double LineTime,
+ unsigned int MetaRowByteLuma,
+ unsigned int MetaRowByteChroma,
+ unsigned int meta_row_height_luma,
+ unsigned int meta_row_height_chroma,
+ unsigned int PixelPTEBytesPerRowLuma,
+ unsigned int PixelPTEBytesPerRowChroma,
+ unsigned int dpte_row_height_luma,
+ unsigned int dpte_row_height_chroma,
+ double *meta_row_bw,
+ double *dpte_row_bw,
+ double *qual_row_bw)
+{
+ if (DCCEnable != true) {
+ *meta_row_bw = 0;
+ } else if (SourcePixelFormat == dm_420_8 || SourcePixelFormat == dm_420_10) {
+ *meta_row_bw = VRatio * MetaRowByteLuma / (meta_row_height_luma * LineTime)
+ + VRatio / 2 * MetaRowByteChroma
+ / (meta_row_height_chroma * LineTime);
+ } else {
+ *meta_row_bw = VRatio * MetaRowByteLuma / (meta_row_height_luma * LineTime);
+ }
+
+ if (GPUVMEnable != true) {
+ *dpte_row_bw = 0;
+ } else if (SourcePixelFormat == dm_420_8 || SourcePixelFormat == dm_420_10) {
+ *dpte_row_bw = VRatio * PixelPTEBytesPerRowLuma / (dpte_row_height_luma * LineTime)
+ + VRatio / 2 * PixelPTEBytesPerRowChroma
+ / (dpte_row_height_chroma * LineTime);
+ } else {
+ *dpte_row_bw = VRatio * PixelPTEBytesPerRowLuma / (dpte_row_height_luma * LineTime);
+ }
+
+ if ((SourcePixelFormat == dm_420_8 || SourcePixelFormat == dm_420_10)) {
+ *qual_row_bw = *meta_row_bw + *dpte_row_bw;
+ } else {
+ *qual_row_bw = 0;
+ }
+}
+
+static void CalculateFlipSchedule(
+ struct display_mode_lib *mode_lib,
+ double UrgentExtraLatency,
+ double UrgentLatencyPixelDataOnly,
+ unsigned int GPUVMMaxPageTableLevels,
+ bool GPUVMEnable,
+ double BandwidthAvailableForImmediateFlip,
+ unsigned int TotImmediateFlipBytes,
+ enum source_format_class SourcePixelFormat,
+ unsigned int ImmediateFlipBytes,
+ double LineTime,
+ double VRatio,
+ double Tno_bw,
+ double PDEAndMetaPTEBytesFrame,
+ unsigned int MetaRowByte,
+ unsigned int PixelPTEBytesPerRow,
+ bool DCCEnable,
+ unsigned int dpte_row_height,
+ unsigned int meta_row_height,
+ double qual_row_bw,
+ double *DestinationLinesToRequestVMInImmediateFlip,
+ double *DestinationLinesToRequestRowInImmediateFlip,
+ double *final_flip_bw,
+ bool *ImmediateFlipSupportedForPipe)
+{
+ double min_row_time = 0.0;
+
+ if (SourcePixelFormat == dm_420_8 || SourcePixelFormat == dm_420_10) {
+ *DestinationLinesToRequestVMInImmediateFlip = 0.0;
+ *DestinationLinesToRequestRowInImmediateFlip = 0.0;
+ *final_flip_bw = qual_row_bw;
+ *ImmediateFlipSupportedForPipe = true;
+ } else {
+ double TimeForFetchingMetaPTEImmediateFlip;
+ double TimeForFetchingRowInVBlankImmediateFlip;
+
+ if (GPUVMEnable == true) {
+ mode_lib->vba.ImmediateFlipBW[0] = BandwidthAvailableForImmediateFlip
+ * ImmediateFlipBytes / TotImmediateFlipBytes;
+ TimeForFetchingMetaPTEImmediateFlip =
+ dml_max(
+ Tno_bw
+ + PDEAndMetaPTEBytesFrame
+ / mode_lib->vba.ImmediateFlipBW[0],
+ dml_max(
+ UrgentExtraLatency
+ + UrgentLatencyPixelDataOnly
+ * (GPUVMMaxPageTableLevels
+ - 1),
+ LineTime / 4.0));
+ } else {
+ TimeForFetchingMetaPTEImmediateFlip = 0;
+ }
+
+ *DestinationLinesToRequestVMInImmediateFlip = dml_floor(
+ 4.0 * (TimeForFetchingMetaPTEImmediateFlip / LineTime + 0.125),
+ 1) / 4.0;
+
+ if ((GPUVMEnable == true || DCCEnable == true)) {
+ mode_lib->vba.ImmediateFlipBW[0] = BandwidthAvailableForImmediateFlip
+ * ImmediateFlipBytes / TotImmediateFlipBytes;
+ TimeForFetchingRowInVBlankImmediateFlip = dml_max(
+ (MetaRowByte + PixelPTEBytesPerRow)
+ / mode_lib->vba.ImmediateFlipBW[0],
+ dml_max(UrgentLatencyPixelDataOnly, LineTime / 4.0));
+ } else {
+ TimeForFetchingRowInVBlankImmediateFlip = 0;
+ }
+
+ *DestinationLinesToRequestRowInImmediateFlip = dml_floor(
+ 4.0 * (TimeForFetchingRowInVBlankImmediateFlip / LineTime + 0.125),
+ 1) / 4.0;
+
+ if (GPUVMEnable == true) {
+ *final_flip_bw =
+ dml_max(
+ PDEAndMetaPTEBytesFrame
+ / (*DestinationLinesToRequestVMInImmediateFlip
+ * LineTime),
+ (MetaRowByte + PixelPTEBytesPerRow)
+ / (TimeForFetchingRowInVBlankImmediateFlip
+ * LineTime));
+ } else if (MetaRowByte + PixelPTEBytesPerRow > 0) {
+ *final_flip_bw = (MetaRowByte + PixelPTEBytesPerRow)
+ / (TimeForFetchingRowInVBlankImmediateFlip * LineTime);
+ } else {
+ *final_flip_bw = 0;
+ }
+
+ if (GPUVMEnable && !DCCEnable)
+ min_row_time = dpte_row_height * LineTime / VRatio;
+ else if (!GPUVMEnable && DCCEnable)
+ min_row_time = meta_row_height * LineTime / VRatio;
+ else
+ min_row_time = dml_min(dpte_row_height, meta_row_height) * LineTime
+ / VRatio;
+
+ if (*DestinationLinesToRequestVMInImmediateFlip >= 8
+ || *DestinationLinesToRequestRowInImmediateFlip >= 16
+ || TimeForFetchingMetaPTEImmediateFlip
+ + 2 * TimeForFetchingRowInVBlankImmediateFlip
+ > min_row_time)
+ *ImmediateFlipSupportedForPipe = false;
+ else
+ *ImmediateFlipSupportedForPipe = true;
+ }
+}
+
+static unsigned int TruncToValidBPP(
+ double DecimalBPP,
+ bool DSCEnabled,
+ enum output_encoder_class Output,
+ enum output_format_class Format,
+ unsigned int DSCInputBitPerComponent)
+{
+ if (Output == dm_hdmi) {
+ if (Format == dm_420) {
+ if (DecimalBPP >= 18)
+ return 18;
+ else if (DecimalBPP >= 15)
+ return 15;
+ else if (DecimalBPP >= 12)
+ return 12;
+ else
+ return BPP_INVALID;
+ } else if (Format == dm_444) {
+ if (DecimalBPP >= 36)
+ return 36;
+ else if (DecimalBPP >= 30)
+ return 30;
+ else if (DecimalBPP >= 24)
+ return 24;
+ else if (DecimalBPP >= 18)
+ return 18;
+ else
+ return BPP_INVALID;
+ } else {
+ if (DecimalBPP / 1.5 >= 24)
+ return 24;
+ else if (DecimalBPP / 1.5 >= 20)
+ return 20;
+ else if (DecimalBPP / 1.5 >= 16)
+ return 16;
+ else
+ return BPP_INVALID;
+ }
+ } else {
+ if (DSCEnabled) {
+ if (Format == dm_420) {
+ if (DecimalBPP < 6)
+ return BPP_INVALID;
+ else if (DecimalBPP >= 1.5 * DSCInputBitPerComponent - 1 / 16)
+ return 1.5 * DSCInputBitPerComponent - 1 / 16;
+ else
+ return dml_floor(16 * DecimalBPP, 1) / 16;
+ } else if (Format == dm_n422) {
+ if (DecimalBPP < 7)
+ return BPP_INVALID;
+ else if (DecimalBPP >= 2 * DSCInputBitPerComponent - 1 / 16)
+ return 2 * DSCInputBitPerComponent - 1 / 16;
+ else
+ return dml_floor(16 * DecimalBPP, 1) / 16;
+ } else {
+ if (DecimalBPP < 8)
+ return BPP_INVALID;
+ else if (DecimalBPP >= 3 * DSCInputBitPerComponent - 1 / 16)
+ return 3 * DSCInputBitPerComponent - 1 / 16;
+ else
+ return dml_floor(16 * DecimalBPP, 1) / 16;
+ }
+ } else if (Format == dm_420) {
+ if (DecimalBPP >= 18)
+ return 18;
+ else if (DecimalBPP >= 15)
+ return 15;
+ else if (DecimalBPP >= 12)
+ return 12;
+ else
+ return BPP_INVALID;
+ } else if (Format == dm_s422 || Format == dm_n422) {
+ if (DecimalBPP >= 24)
+ return 24;
+ else if (DecimalBPP >= 20)
+ return 20;
+ else if (DecimalBPP >= 16)
+ return 16;
+ else
+ return BPP_INVALID;
+ } else {
+ if (DecimalBPP >= 36)
+ return 36;
+ else if (DecimalBPP >= 30)
+ return 30;
+ else if (DecimalBPP >= 24)
+ return 24;
+ else if (DecimalBPP >= 18)
+ return 18;
+ else
+ return BPP_INVALID;
+ }
+ }
+}
+
+void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_lib)
+{
+ struct vba_vars_st *locals = &mode_lib->vba;
+
+ int i;
+ unsigned int j, k, m;
+
+ /*MODE SUPPORT, VOLTAGE STATE AND SOC CONFIGURATION*/
+
+ /*Scale Ratio, taps Support Check*/
+
+ mode_lib->vba.ScaleRatioAndTapsSupport = true;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (mode_lib->vba.ScalerEnabled[k] == false
+ && ((mode_lib->vba.SourcePixelFormat[k] != dm_444_64
+ && mode_lib->vba.SourcePixelFormat[k] != dm_444_32
+ && mode_lib->vba.SourcePixelFormat[k] != dm_444_16
+ && mode_lib->vba.SourcePixelFormat[k] != dm_mono_16
+ && mode_lib->vba.SourcePixelFormat[k] != dm_mono_8)
+ || mode_lib->vba.HRatio[k] != 1.0
+ || mode_lib->vba.htaps[k] != 1.0
+ || mode_lib->vba.VRatio[k] != 1.0
+ || mode_lib->vba.vtaps[k] != 1.0)) {
+ mode_lib->vba.ScaleRatioAndTapsSupport = false;
+ } else if (mode_lib->vba.vtaps[k] < 1.0 || mode_lib->vba.vtaps[k] > 8.0
+ || mode_lib->vba.htaps[k] < 1.0 || mode_lib->vba.htaps[k] > 8.0
+ || (mode_lib->vba.htaps[k] > 1.0
+ && (mode_lib->vba.htaps[k] % 2) == 1)
+ || mode_lib->vba.HRatio[k] > mode_lib->vba.MaxHSCLRatio
+ || mode_lib->vba.VRatio[k] > mode_lib->vba.MaxVSCLRatio
+ || mode_lib->vba.HRatio[k] > mode_lib->vba.htaps[k]
+ || mode_lib->vba.VRatio[k] > mode_lib->vba.vtaps[k]
+ || (mode_lib->vba.SourcePixelFormat[k] != dm_444_64
+ && mode_lib->vba.SourcePixelFormat[k] != dm_444_32
+ && mode_lib->vba.SourcePixelFormat[k] != dm_444_16
+ && mode_lib->vba.SourcePixelFormat[k] != dm_mono_16
+ && mode_lib->vba.SourcePixelFormat[k] != dm_mono_8
+ && (mode_lib->vba.HRatio[k] / 2.0
+ > mode_lib->vba.HTAPsChroma[k]
+ || mode_lib->vba.VRatio[k] / 2.0
+ > mode_lib->vba.VTAPsChroma[k]))) {
+ mode_lib->vba.ScaleRatioAndTapsSupport = false;
+ }
+ }
+ /*Source Format, Pixel Format and Scan Support Check*/
+
+ mode_lib->vba.SourceFormatPixelAndScanSupport = true;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if ((mode_lib->vba.SurfaceTiling[k] == dm_sw_linear
+ && mode_lib->vba.SourceScan[k] != dm_horz)
+ || ((mode_lib->vba.SurfaceTiling[k] == dm_sw_4kb_d
+ || mode_lib->vba.SurfaceTiling[k] == dm_sw_4kb_d_x
+ || mode_lib->vba.SurfaceTiling[k] == dm_sw_64kb_d
+ || mode_lib->vba.SurfaceTiling[k] == dm_sw_64kb_d_t
+ || mode_lib->vba.SurfaceTiling[k] == dm_sw_64kb_d_x
+ || mode_lib->vba.SurfaceTiling[k] == dm_sw_var_d
+ || mode_lib->vba.SurfaceTiling[k] == dm_sw_var_d_x)
+ && mode_lib->vba.SourcePixelFormat[k] != dm_444_64)
+ || (mode_lib->vba.SurfaceTiling[k] == dm_sw_64kb_r_x
+ && (mode_lib->vba.SourcePixelFormat[k] == dm_mono_8
+ || mode_lib->vba.SourcePixelFormat[k]
+ == dm_420_8
+ || mode_lib->vba.SourcePixelFormat[k]
+ == dm_420_10))
+ || (((mode_lib->vba.SurfaceTiling[k] == dm_sw_gfx7_2d_thin_gl
+ || mode_lib->vba.SurfaceTiling[k]
+ == dm_sw_gfx7_2d_thin_lvp)
+ && !((mode_lib->vba.SourcePixelFormat[k]
+ == dm_444_64
+ || mode_lib->vba.SourcePixelFormat[k]
+ == dm_444_32)
+ && mode_lib->vba.SourceScan[k]
+ == dm_horz
+ && mode_lib->vba.SupportGFX7CompatibleTilingIn32bppAnd64bpp
+ == true
+ && mode_lib->vba.DCCEnable[k]
+ == false))
+ || (mode_lib->vba.DCCEnable[k] == true
+ && (mode_lib->vba.SurfaceTiling[k]
+ == dm_sw_linear
+ || mode_lib->vba.SourcePixelFormat[k]
+ == dm_420_8
+ || mode_lib->vba.SourcePixelFormat[k]
+ == dm_420_10)))) {
+ mode_lib->vba.SourceFormatPixelAndScanSupport = false;
+ }
+ }
+ /*Bandwidth Support Check*/
+
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (mode_lib->vba.SourcePixelFormat[k] == dm_444_64) {
+ locals->BytePerPixelInDETY[k] = 8.0;
+ locals->BytePerPixelInDETC[k] = 0.0;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_444_32) {
+ locals->BytePerPixelInDETY[k] = 4.0;
+ locals->BytePerPixelInDETC[k] = 0.0;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_444_16
+ || mode_lib->vba.SourcePixelFormat[k] == dm_mono_16) {
+ locals->BytePerPixelInDETY[k] = 2.0;
+ locals->BytePerPixelInDETC[k] = 0.0;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_mono_8) {
+ locals->BytePerPixelInDETY[k] = 1.0;
+ locals->BytePerPixelInDETC[k] = 0.0;
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_420_8) {
+ locals->BytePerPixelInDETY[k] = 1.0;
+ locals->BytePerPixelInDETC[k] = 2.0;
+ } else {
+ locals->BytePerPixelInDETY[k] = 4.0 / 3;
+ locals->BytePerPixelInDETC[k] = 8.0 / 3;
+ }
+ if (mode_lib->vba.SourceScan[k] == dm_horz) {
+ locals->SwathWidthYSingleDPP[k] = mode_lib->vba.ViewportWidth[k];
+ } else {
+ locals->SwathWidthYSingleDPP[k] = mode_lib->vba.ViewportHeight[k];
+ }
+ }
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ locals->ReadBandwidthLuma[k] = locals->SwathWidthYSingleDPP[k] * dml_ceil(locals->BytePerPixelInDETY[k], 1.0)
+ / (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k]) * mode_lib->vba.VRatio[k];
+ locals->ReadBandwidthChroma[k] = locals->SwathWidthYSingleDPP[k] / 2 * dml_ceil(locals->BytePerPixelInDETC[k], 2.0)
+ / (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k]) * mode_lib->vba.VRatio[k] / 2.0;
+ locals->ReadBandwidth[k] = locals->ReadBandwidthLuma[k] + locals->ReadBandwidthChroma[k];
+ }
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (mode_lib->vba.WritebackEnable[k] == true
+ && mode_lib->vba.WritebackPixelFormat[k] == dm_444_32) {
+ locals->WriteBandwidth[k] = mode_lib->vba.WritebackDestinationWidth[k]
+ * mode_lib->vba.WritebackDestinationHeight[k]
+ / (mode_lib->vba.WritebackSourceHeight[k]
+ * mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k]) * 4.0;
+ } else if (mode_lib->vba.WritebackEnable[k] == true
+ && mode_lib->vba.WritebackPixelFormat[k] == dm_420_10) {
+ locals->WriteBandwidth[k] = mode_lib->vba.WritebackDestinationWidth[k]
+ * mode_lib->vba.WritebackDestinationHeight[k]
+ / (mode_lib->vba.WritebackSourceHeight[k]
+ * mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k]) * 3.0;
+ } else if (mode_lib->vba.WritebackEnable[k] == true) {
+ locals->WriteBandwidth[k] = mode_lib->vba.WritebackDestinationWidth[k]
+ * mode_lib->vba.WritebackDestinationHeight[k]
+ / (mode_lib->vba.WritebackSourceHeight[k]
+ * mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k]) * 1.5;
+ } else {
+ locals->WriteBandwidth[k] = 0.0;
+ }
+ }
+ mode_lib->vba.DCCEnabledInAnyPlane = false;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (mode_lib->vba.DCCEnable[k] == true) {
+ mode_lib->vba.DCCEnabledInAnyPlane = true;
+ }
+ }
+ for (i = 0; i <= mode_lib->vba.soc.num_states; i++) {
+ locals->FabricAndDRAMBandwidthPerState[i] = dml_min(
+ mode_lib->vba.DRAMSpeedPerState[i] * mode_lib->vba.NumberOfChannels
+ * mode_lib->vba.DRAMChannelWidth,
+ mode_lib->vba.FabricClockPerState[i]
+ * mode_lib->vba.FabricDatapathToDCNDataReturn) / 1000;
+ locals->ReturnBWToDCNPerState = dml_min(locals->ReturnBusWidth * locals->DCFCLKPerState[i],
+ locals->FabricAndDRAMBandwidthPerState[i] * 1000)
+ * locals->PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelDataOnly / 100;
+
+ locals->ReturnBWPerState[i] = locals->ReturnBWToDCNPerState;
+
+ if (locals->DCCEnabledInAnyPlane == true && locals->ReturnBWToDCNPerState > locals->DCFCLKPerState[i] * locals->ReturnBusWidth / 4) {
+ locals->ReturnBWPerState[i] = dml_min(locals->ReturnBWPerState[i],
+ locals->ReturnBWToDCNPerState * 4 * (1 - locals->UrgentLatency /
+ ((locals->ROBBufferSizeInKByte - locals->PixelChunkSizeInKByte) * 1024
+ / (locals->ReturnBWToDCNPerState - locals->DCFCLKPerState[i]
+ * locals->ReturnBusWidth / 4) + locals->UrgentLatency)));
+ }
+ locals->CriticalPoint = 2 * locals->ReturnBusWidth * locals->DCFCLKPerState[i] *
+ locals->UrgentLatency / (locals->ReturnBWToDCNPerState * locals->UrgentLatency
+ + (locals->ROBBufferSizeInKByte - locals->PixelChunkSizeInKByte) * 1024);
+
+ if (locals->DCCEnabledInAnyPlane && locals->CriticalPoint > 1 && locals->CriticalPoint < 4) {
+ locals->ReturnBWPerState[i] = dml_min(locals->ReturnBWPerState[i],
+ 4 * locals->ReturnBWToDCNPerState *
+ (locals->ROBBufferSizeInKByte - locals->PixelChunkSizeInKByte) * 1024
+ * locals->ReturnBusWidth * locals->DCFCLKPerState[i] * locals->UrgentLatency /
+ dml_pow((locals->ReturnBWToDCNPerState * locals->UrgentLatency
+ + (locals->ROBBufferSizeInKByte - locals->PixelChunkSizeInKByte) * 1024), 2));
+ }
+
+ locals->ReturnBWToDCNPerState = dml_min(locals->ReturnBusWidth *
+ locals->DCFCLKPerState[i], locals->FabricAndDRAMBandwidthPerState[i] * 1000);
+
+ if (locals->DCCEnabledInAnyPlane == true && locals->ReturnBWToDCNPerState > locals->DCFCLKPerState[i] * locals->ReturnBusWidth / 4) {
+ locals->ReturnBWPerState[i] = dml_min(locals->ReturnBWPerState[i],
+ locals->ReturnBWToDCNPerState * 4 * (1 - locals->UrgentLatency /
+ ((locals->ROBBufferSizeInKByte - locals->PixelChunkSizeInKByte) * 1024
+ / (locals->ReturnBWToDCNPerState - locals->DCFCLKPerState[i]
+ * locals->ReturnBusWidth / 4) + locals->UrgentLatency)));
+ }
+ locals->CriticalPoint = 2 * locals->ReturnBusWidth * locals->DCFCLKPerState[i] *
+ locals->UrgentLatency / (locals->ReturnBWToDCNPerState * locals->UrgentLatency
+ + (locals->ROBBufferSizeInKByte - locals->PixelChunkSizeInKByte) * 1024);
+
+ if (locals->DCCEnabledInAnyPlane && locals->CriticalPoint > 1 && locals->CriticalPoint < 4) {
+ locals->ReturnBWPerState[i] = dml_min(locals->ReturnBWPerState[i],
+ 4 * locals->ReturnBWToDCNPerState *
+ (locals->ROBBufferSizeInKByte - locals->PixelChunkSizeInKByte) * 1024
+ * locals->ReturnBusWidth * locals->DCFCLKPerState[i] * locals->UrgentLatency /
+ dml_pow((locals->ReturnBWToDCNPerState * locals->UrgentLatency
+ + (locals->ROBBufferSizeInKByte - locals->PixelChunkSizeInKByte) * 1024), 2));
+ }
+ }
+ /*Writeback Latency support check*/
+
+ mode_lib->vba.WritebackLatencySupport = true;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (mode_lib->vba.WritebackEnable[k] == true) {
+ if (mode_lib->vba.WritebackPixelFormat[k] == dm_444_32) {
+ if (locals->WriteBandwidth[k]
+ > (mode_lib->vba.WritebackInterfaceLumaBufferSize
+ + mode_lib->vba.WritebackInterfaceChromaBufferSize)
+ / mode_lib->vba.WritebackLatency) {
+ mode_lib->vba.WritebackLatencySupport = false;
+ }
+ } else {
+ if (locals->WriteBandwidth[k]
+ > 1.5
+ * dml_min(
+ mode_lib->vba.WritebackInterfaceLumaBufferSize,
+ 2.0
+ * mode_lib->vba.WritebackInterfaceChromaBufferSize)
+ / mode_lib->vba.WritebackLatency) {
+ mode_lib->vba.WritebackLatencySupport = false;
+ }
+ }
+ }
+ }
+ /*Re-ordering Buffer Support Check*/
+
+ for (i = 0; i <= mode_lib->vba.soc.num_states; i++) {
+ locals->UrgentRoundTripAndOutOfOrderLatencyPerState[i] =
+ (mode_lib->vba.RoundTripPingLatencyCycles + 32.0) / mode_lib->vba.DCFCLKPerState[i]
+ + locals->UrgentOutOfOrderReturnPerChannel * mode_lib->vba.NumberOfChannels / locals->ReturnBWPerState[i];
+ if ((mode_lib->vba.ROBBufferSizeInKByte - mode_lib->vba.PixelChunkSizeInKByte) * 1024.0 / locals->ReturnBWPerState[i]
+ > locals->UrgentRoundTripAndOutOfOrderLatencyPerState[i]) {
+ locals->ROBSupport[i] = true;
+ } else {
+ locals->ROBSupport[i] = false;
+ }
+ }
+ /*Writeback Mode Support Check*/
+
+ mode_lib->vba.TotalNumberOfActiveWriteback = 0;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (mode_lib->vba.WritebackEnable[k] == true) {
+ if (mode_lib->vba.ActiveWritebacksPerPlane[k] == 0)
+ mode_lib->vba.ActiveWritebacksPerPlane[k] = 1;
+ mode_lib->vba.TotalNumberOfActiveWriteback =
+ mode_lib->vba.TotalNumberOfActiveWriteback
+ + mode_lib->vba.ActiveWritebacksPerPlane[k];
+ }
+ }
+ mode_lib->vba.WritebackModeSupport = true;
+ if (mode_lib->vba.TotalNumberOfActiveWriteback > mode_lib->vba.MaxNumWriteback) {
+ mode_lib->vba.WritebackModeSupport = false;
+ }
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (mode_lib->vba.WritebackEnable[k] == true
+ && mode_lib->vba.Writeback10bpc420Supported != true
+ && mode_lib->vba.WritebackPixelFormat[k] == dm_420_10) {
+ mode_lib->vba.WritebackModeSupport = false;
+ }
+ }
+ /*Writeback Scale Ratio and Taps Support Check*/
+
+ mode_lib->vba.WritebackScaleRatioAndTapsSupport = true;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (mode_lib->vba.WritebackEnable[k] == true) {
+ if (mode_lib->vba.WritebackLumaAndChromaScalingSupported == false
+ && (mode_lib->vba.WritebackHRatio[k] != 1.0
+ || mode_lib->vba.WritebackVRatio[k] != 1.0)) {
+ mode_lib->vba.WritebackScaleRatioAndTapsSupport = false;
+ }
+ if (mode_lib->vba.WritebackHRatio[k] > mode_lib->vba.WritebackMaxHSCLRatio
+ || mode_lib->vba.WritebackVRatio[k]
+ > mode_lib->vba.WritebackMaxVSCLRatio
+ || mode_lib->vba.WritebackHRatio[k]
+ < mode_lib->vba.WritebackMinHSCLRatio
+ || mode_lib->vba.WritebackVRatio[k]
+ < mode_lib->vba.WritebackMinVSCLRatio
+ || mode_lib->vba.WritebackLumaHTaps[k]
+ > mode_lib->vba.WritebackMaxHSCLTaps
+ || mode_lib->vba.WritebackLumaVTaps[k]
+ > mode_lib->vba.WritebackMaxVSCLTaps
+ || mode_lib->vba.WritebackHRatio[k]
+ > mode_lib->vba.WritebackLumaHTaps[k]
+ || mode_lib->vba.WritebackVRatio[k]
+ > mode_lib->vba.WritebackLumaVTaps[k]
+ || (mode_lib->vba.WritebackLumaHTaps[k] > 2.0
+ && ((mode_lib->vba.WritebackLumaHTaps[k] % 2)
+ == 1))
+ || (mode_lib->vba.WritebackPixelFormat[k] != dm_444_32
+ && (mode_lib->vba.WritebackChromaHTaps[k]
+ > mode_lib->vba.WritebackMaxHSCLTaps
+ || mode_lib->vba.WritebackChromaVTaps[k]
+ > mode_lib->vba.WritebackMaxVSCLTaps
+ || 2.0
+ * mode_lib->vba.WritebackHRatio[k]
+ > mode_lib->vba.WritebackChromaHTaps[k]
+ || 2.0
+ * mode_lib->vba.WritebackVRatio[k]
+ > mode_lib->vba.WritebackChromaVTaps[k]
+ || (mode_lib->vba.WritebackChromaHTaps[k] > 2.0
+ && ((mode_lib->vba.WritebackChromaHTaps[k] % 2) == 1))))) {
+ mode_lib->vba.WritebackScaleRatioAndTapsSupport = false;
+ }
+ if (mode_lib->vba.WritebackVRatio[k] < 1.0) {
+ mode_lib->vba.WritebackLumaVExtra =
+ dml_max(1.0 - 2.0 / dml_ceil(1.0 / mode_lib->vba.WritebackVRatio[k], 1.0), 0.0);
+ } else {
+ mode_lib->vba.WritebackLumaVExtra = -1;
+ }
+ if ((mode_lib->vba.WritebackPixelFormat[k] == dm_444_32
+ && mode_lib->vba.WritebackLumaVTaps[k]
+ > (mode_lib->vba.WritebackLineBufferLumaBufferSize
+ + mode_lib->vba.WritebackLineBufferChromaBufferSize)
+ / 3.0
+ / mode_lib->vba.WritebackDestinationWidth[k]
+ - mode_lib->vba.WritebackLumaVExtra)
+ || (mode_lib->vba.WritebackPixelFormat[k] == dm_420_8
+ && mode_lib->vba.WritebackLumaVTaps[k]
+ > mode_lib->vba.WritebackLineBufferLumaBufferSize
+ * 8.0 / 10.0 / mode_lib->vba.WritebackDestinationWidth[k]
+ - mode_lib->vba.WritebackLumaVExtra)
+ || (mode_lib->vba.WritebackPixelFormat[k] == dm_420_10
+ && mode_lib->vba.WritebackLumaVTaps[k]
+ > mode_lib->vba.WritebackLineBufferLumaBufferSize
+ * 8.0 / 10.0
+ / mode_lib->vba.WritebackDestinationWidth[k]
+ - mode_lib->vba.WritebackLumaVExtra)) {
+ mode_lib->vba.WritebackScaleRatioAndTapsSupport = false;
+ }
+ if (2.0 * mode_lib->vba.WritebackVRatio[k] < 1) {
+ mode_lib->vba.WritebackChromaVExtra = 0.0;
+ } else {
+ mode_lib->vba.WritebackChromaVExtra = -1;
+ }
+ if ((mode_lib->vba.WritebackPixelFormat[k] == dm_420_8
+ && mode_lib->vba.WritebackChromaVTaps[k]
+ > mode_lib->vba.WritebackLineBufferChromaBufferSize
+ * 8.0 / 10.0 / mode_lib->vba.WritebackDestinationWidth[k]
+ - mode_lib->vba.WritebackChromaVExtra)
+ || (mode_lib->vba.WritebackPixelFormat[k] == dm_420_10
+ && mode_lib->vba.WritebackChromaVTaps[k]
+ > mode_lib->vba.WritebackLineBufferChromaBufferSize
+ * 8.0 / 10.0
+ / mode_lib->vba.WritebackDestinationWidth[k]
+ - mode_lib->vba.WritebackChromaVExtra)) {
+ mode_lib->vba.WritebackScaleRatioAndTapsSupport = false;
+ }
+ }
+ }
+ /*Maximum DISPCLK/DPPCLK Support check*/
+
+ mode_lib->vba.WritebackRequiredDISPCLK = 0.0;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (mode_lib->vba.WritebackEnable[k] == true) {
+ mode_lib->vba.WritebackRequiredDISPCLK =
+ dml_max(
+ mode_lib->vba.WritebackRequiredDISPCLK,
+ CalculateWriteBackDISPCLK(
+ mode_lib->vba.WritebackPixelFormat[k],
+ mode_lib->vba.PixelClock[k],
+ mode_lib->vba.WritebackHRatio[k],
+ mode_lib->vba.WritebackVRatio[k],
+ mode_lib->vba.WritebackLumaHTaps[k],
+ mode_lib->vba.WritebackLumaVTaps[k],
+ mode_lib->vba.WritebackChromaHTaps[k],
+ mode_lib->vba.WritebackChromaVTaps[k],
+ mode_lib->vba.WritebackDestinationWidth[k],
+ mode_lib->vba.HTotal[k],
+ mode_lib->vba.WritebackChromaLineBufferWidth));
+ }
+ }
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (mode_lib->vba.HRatio[k] > 1.0) {
+ locals->PSCL_FACTOR[k] = dml_min(
+ mode_lib->vba.MaxDCHUBToPSCLThroughput,
+ mode_lib->vba.MaxPSCLToLBThroughput
+ * mode_lib->vba.HRatio[k]
+ / dml_ceil(
+ mode_lib->vba.htaps[k]
+ / 6.0,
+ 1.0));
+ } else {
+ locals->PSCL_FACTOR[k] = dml_min(
+ mode_lib->vba.MaxDCHUBToPSCLThroughput,
+ mode_lib->vba.MaxPSCLToLBThroughput);
+ }
+ if (locals->BytePerPixelInDETC[k] == 0.0) {
+ locals->PSCL_FACTOR_CHROMA[k] = 0.0;
+ locals->MinDPPCLKUsingSingleDPP[k] =
+ mode_lib->vba.PixelClock[k]
+ * dml_max3(
+ mode_lib->vba.vtaps[k] / 6.0
+ * dml_min(
+ 1.0,
+ mode_lib->vba.HRatio[k]),
+ mode_lib->vba.HRatio[k]
+ * mode_lib->vba.VRatio[k]
+ / locals->PSCL_FACTOR[k],
+ 1.0);
+ if ((mode_lib->vba.htaps[k] > 6.0 || mode_lib->vba.vtaps[k] > 6.0)
+ && locals->MinDPPCLKUsingSingleDPP[k]
+ < 2.0 * mode_lib->vba.PixelClock[k]) {
+ locals->MinDPPCLKUsingSingleDPP[k] = 2.0
+ * mode_lib->vba.PixelClock[k];
+ }
+ } else {
+ if (mode_lib->vba.HRatio[k] / 2.0 > 1.0) {
+ locals->PSCL_FACTOR_CHROMA[k] =
+ dml_min(
+ mode_lib->vba.MaxDCHUBToPSCLThroughput,
+ mode_lib->vba.MaxPSCLToLBThroughput
+ * mode_lib->vba.HRatio[k]
+ / 2.0
+ / dml_ceil(
+ mode_lib->vba.HTAPsChroma[k]
+ / 6.0,
+ 1.0));
+ } else {
+ locals->PSCL_FACTOR_CHROMA[k] = dml_min(
+ mode_lib->vba.MaxDCHUBToPSCLThroughput,
+ mode_lib->vba.MaxPSCLToLBThroughput);
+ }
+ locals->MinDPPCLKUsingSingleDPP[k] =
+ mode_lib->vba.PixelClock[k]
+ * dml_max5(
+ mode_lib->vba.vtaps[k] / 6.0
+ * dml_min(
+ 1.0,
+ mode_lib->vba.HRatio[k]),
+ mode_lib->vba.HRatio[k]
+ * mode_lib->vba.VRatio[k]
+ / locals->PSCL_FACTOR[k],
+ mode_lib->vba.VTAPsChroma[k]
+ / 6.0
+ * dml_min(
+ 1.0,
+ mode_lib->vba.HRatio[k]
+ / 2.0),
+ mode_lib->vba.HRatio[k]
+ * mode_lib->vba.VRatio[k]
+ / 4.0
+ / locals->PSCL_FACTOR_CHROMA[k],
+ 1.0);
+ if ((mode_lib->vba.htaps[k] > 6.0 || mode_lib->vba.vtaps[k] > 6.0
+ || mode_lib->vba.HTAPsChroma[k] > 6.0
+ || mode_lib->vba.VTAPsChroma[k] > 6.0)
+ && locals->MinDPPCLKUsingSingleDPP[k]
+ < 2.0 * mode_lib->vba.PixelClock[k]) {
+ locals->MinDPPCLKUsingSingleDPP[k] = 2.0
+ * mode_lib->vba.PixelClock[k];
+ }
+ }
+ }
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ Calculate256BBlockSizes(
+ mode_lib->vba.SourcePixelFormat[k],
+ mode_lib->vba.SurfaceTiling[k],
+ dml_ceil(locals->BytePerPixelInDETY[k], 1.0),
+ dml_ceil(locals->BytePerPixelInDETC[k], 2.0),
+ &locals->Read256BlockHeightY[k],
+ &locals->Read256BlockHeightC[k],
+ &locals->Read256BlockWidthY[k],
+ &locals->Read256BlockWidthC[k]);
+ if (mode_lib->vba.SourceScan[k] == dm_horz) {
+ locals->MaxSwathHeightY[k] = locals->Read256BlockHeightY[k];
+ locals->MaxSwathHeightC[k] = locals->Read256BlockHeightC[k];
+ } else {
+ locals->MaxSwathHeightY[k] = locals->Read256BlockWidthY[k];
+ locals->MaxSwathHeightC[k] = locals->Read256BlockWidthC[k];
+ }
+ if ((mode_lib->vba.SourcePixelFormat[k] == dm_444_64
+ || mode_lib->vba.SourcePixelFormat[k] == dm_444_32
+ || mode_lib->vba.SourcePixelFormat[k] == dm_444_16
+ || mode_lib->vba.SourcePixelFormat[k] == dm_mono_16
+ || mode_lib->vba.SourcePixelFormat[k] == dm_mono_8)) {
+ if (mode_lib->vba.SurfaceTiling[k] == dm_sw_linear
+ || (mode_lib->vba.SourcePixelFormat[k] == dm_444_64
+ && (mode_lib->vba.SurfaceTiling[k]
+ == dm_sw_4kb_s
+ || mode_lib->vba.SurfaceTiling[k]
+ == dm_sw_4kb_s_x
+ || mode_lib->vba.SurfaceTiling[k]
+ == dm_sw_64kb_s
+ || mode_lib->vba.SurfaceTiling[k]
+ == dm_sw_64kb_s_t
+ || mode_lib->vba.SurfaceTiling[k]
+ == dm_sw_64kb_s_x
+ || mode_lib->vba.SurfaceTiling[k]
+ == dm_sw_var_s
+ || mode_lib->vba.SurfaceTiling[k]
+ == dm_sw_var_s_x)
+ && mode_lib->vba.SourceScan[k] == dm_horz)) {
+ locals->MinSwathHeightY[k] = locals->MaxSwathHeightY[k];
+ } else {
+ locals->MinSwathHeightY[k] = locals->MaxSwathHeightY[k]
+ / 2.0;
+ }
+ locals->MinSwathHeightC[k] = locals->MaxSwathHeightC[k];
+ } else {
+ if (mode_lib->vba.SurfaceTiling[k] == dm_sw_linear) {
+ locals->MinSwathHeightY[k] = locals->MaxSwathHeightY[k];
+ locals->MinSwathHeightC[k] = locals->MaxSwathHeightC[k];
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_420_8
+ && mode_lib->vba.SourceScan[k] == dm_horz) {
+ locals->MinSwathHeightY[k] = locals->MaxSwathHeightY[k]
+ / 2.0;
+ locals->MinSwathHeightC[k] = locals->MaxSwathHeightC[k];
+ } else if (mode_lib->vba.SourcePixelFormat[k] == dm_420_10
+ && mode_lib->vba.SourceScan[k] == dm_horz) {
+ locals->MinSwathHeightC[k] = locals->MaxSwathHeightC[k]
+ / 2.0;
+ locals->MinSwathHeightY[k] = locals->MaxSwathHeightY[k];
+ } else {
+ locals->MinSwathHeightY[k] = locals->MaxSwathHeightY[k];
+ locals->MinSwathHeightC[k] = locals->MaxSwathHeightC[k];
+ }
+ }
+ if (mode_lib->vba.SurfaceTiling[k] == dm_sw_linear) {
+ mode_lib->vba.MaximumSwathWidthSupport = 8192.0;
+ } else {
+ mode_lib->vba.MaximumSwathWidthSupport = 5120.0;
+ }
+ mode_lib->vba.MaximumSwathWidthInDETBuffer =
+ dml_min(
+ mode_lib->vba.MaximumSwathWidthSupport,
+ mode_lib->vba.DETBufferSizeInKByte * 1024.0 / 2.0
+ / (locals->BytePerPixelInDETY[k]
+ * locals->MinSwathHeightY[k]
+ + locals->BytePerPixelInDETC[k]
+ / 2.0
+ * locals->MinSwathHeightC[k]));
+ if (locals->BytePerPixelInDETC[k] == 0.0) {
+ mode_lib->vba.MaximumSwathWidthInLineBuffer =
+ mode_lib->vba.LineBufferSize
+ * dml_max(mode_lib->vba.HRatio[k], 1.0)
+ / mode_lib->vba.LBBitPerPixel[k]
+ / (mode_lib->vba.vtaps[k]
+ + dml_max(
+ dml_ceil(
+ mode_lib->vba.VRatio[k],
+ 1.0)
+ - 2,
+ 0.0));
+ } else {
+ mode_lib->vba.MaximumSwathWidthInLineBuffer =
+ dml_min(
+ mode_lib->vba.LineBufferSize
+ * dml_max(
+ mode_lib->vba.HRatio[k],
+ 1.0)
+ / mode_lib->vba.LBBitPerPixel[k]
+ / (mode_lib->vba.vtaps[k]
+ + dml_max(
+ dml_ceil(
+ mode_lib->vba.VRatio[k],
+ 1.0)
+ - 2,
+ 0.0)),
+ 2.0 * mode_lib->vba.LineBufferSize
+ * dml_max(
+ mode_lib->vba.HRatio[k]
+ / 2.0,
+ 1.0)
+ / mode_lib->vba.LBBitPerPixel[k]
+ / (mode_lib->vba.VTAPsChroma[k]
+ + dml_max(
+ dml_ceil(
+ mode_lib->vba.VRatio[k]
+ / 2.0,
+ 1.0)
+ - 2,
+ 0.0)));
+ }
+ locals->MaximumSwathWidth[k] = dml_min(
+ mode_lib->vba.MaximumSwathWidthInDETBuffer,
+ mode_lib->vba.MaximumSwathWidthInLineBuffer);
+ }
+ for (i = 0; i <= mode_lib->vba.soc.num_states; i++) {
+ for (j = 0; j < 2; j++) {
+ mode_lib->vba.MaxDispclkRoundedDownToDFSGranularity = RoundToDFSGranularityDown(
+ mode_lib->vba.MaxDispclk[i],
+ mode_lib->vba.DISPCLKDPPCLKVCOSpeed);
+ mode_lib->vba.MaxDppclkRoundedDownToDFSGranularity = RoundToDFSGranularityDown(
+ mode_lib->vba.MaxDppclk[i],
+ mode_lib->vba.DISPCLKDPPCLKVCOSpeed);
+ locals->RequiredDISPCLK[i][j] = 0.0;
+ locals->DISPCLK_DPPCLK_Support[i][j] = true;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ mode_lib->vba.PlaneRequiredDISPCLKWithoutODMCombine =
+ mode_lib->vba.PixelClock[k] * (1.0 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0)
+ * (1.0 + mode_lib->vba.DISPCLKRampingMargin / 100.0);
+ if (mode_lib->vba.PlaneRequiredDISPCLKWithoutODMCombine >= mode_lib->vba.MaxDispclk[i]
+ && i == mode_lib->vba.soc.num_states)
+ mode_lib->vba.PlaneRequiredDISPCLKWithoutODMCombine = mode_lib->vba.PixelClock[k]
+ * (1 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0);
+
+ mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine = mode_lib->vba.PixelClock[k] / 2
+ * (1 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0) * (1 + mode_lib->vba.DISPCLKRampingMargin / 100.0);
+ if (mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine >= mode_lib->vba.MaxDispclk[i]
+ && i == mode_lib->vba.soc.num_states)
+ mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine = mode_lib->vba.PixelClock[k] / 2
+ * (1 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0);
+ if (mode_lib->vba.ODMCapability == false || mode_lib->vba.PlaneRequiredDISPCLKWithoutODMCombine <= mode_lib->vba.MaxDispclkRoundedDownToDFSGranularity) {
+ locals->ODMCombineEnablePerState[i][k] = false;
+ mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithoutODMCombine;
+ } else {
+ locals->ODMCombineEnablePerState[i][k] = true;
+ mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PlaneRequiredDISPCLKWithODMCombine;
+ }
+ if (locals->MinDPPCLKUsingSingleDPP[k] * (1.0 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0) <= mode_lib->vba.MaxDppclkRoundedDownToDFSGranularity
+ && locals->SwathWidthYSingleDPP[k] <= locals->MaximumSwathWidth[k]
+ && locals->ODMCombineEnablePerState[i][k] == false) {
+ locals->NoOfDPP[i][j][k] = 1;
+ locals->RequiredDPPCLK[i][j][k] =
+ locals->MinDPPCLKUsingSingleDPP[k] * (1.0 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0);
+ } else {
+ locals->NoOfDPP[i][j][k] = 2;
+ locals->RequiredDPPCLK[i][j][k] =
+ locals->MinDPPCLKUsingSingleDPP[k] * (1.0 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0) / 2.0;
+ }
+ locals->RequiredDISPCLK[i][j] = dml_max(
+ locals->RequiredDISPCLK[i][j],
+ mode_lib->vba.PlaneRequiredDISPCLK);
+ if ((locals->MinDPPCLKUsingSingleDPP[k] / locals->NoOfDPP[i][j][k] * (1.0 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0)
+ > mode_lib->vba.MaxDppclkRoundedDownToDFSGranularity)
+ || (mode_lib->vba.PlaneRequiredDISPCLK > mode_lib->vba.MaxDispclkRoundedDownToDFSGranularity)) {
+ locals->DISPCLK_DPPCLK_Support[i][j] = false;
+ }
+ }
+ locals->TotalNumberOfActiveDPP[i][j] = 0.0;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++)
+ locals->TotalNumberOfActiveDPP[i][j] = locals->TotalNumberOfActiveDPP[i][j] + locals->NoOfDPP[i][j][k];
+ if (j == 1) {
+ while (locals->TotalNumberOfActiveDPP[i][j] < mode_lib->vba.MaxNumDPP
+ && locals->TotalNumberOfActiveDPP[i][j] < 2 * mode_lib->vba.NumberOfActivePlanes) {
+ double BWOfNonSplitPlaneOfMaximumBandwidth;
+ unsigned int NumberOfNonSplitPlaneOfMaximumBandwidth;
+
+ BWOfNonSplitPlaneOfMaximumBandwidth = 0;
+ NumberOfNonSplitPlaneOfMaximumBandwidth = 0;
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (locals->ReadBandwidth[k] > BWOfNonSplitPlaneOfMaximumBandwidth && locals->NoOfDPP[i][j][k] == 1) {
+ BWOfNonSplitPlaneOfMaximumBandwidth = locals->ReadBandwidth[k];
+ NumberOfNonSplitPlaneOfMaximumBandwidth = k;
+ }
+ }
+ locals->NoOfDPP[i][j][NumberOfNonSplitPlaneOfMaximumBandwidth] = 2;
+ locals->RequiredDPPCLK[i][j][NumberOfNonSplitPlaneOfMaximumBandwidth] =
+ locals->MinDPPCLKUsingSingleDPP[NumberOfNonSplitPlaneOfMaximumBandwidth]
+ * (1 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100) / 2;
+ locals->TotalNumberOfActiveDPP[i][j] = locals->TotalNumberOfActiveDPP[i][j] + 1;
+ }
+ }
+ if (locals->TotalNumberOfActiveDPP[i][j] > mode_lib->vba.MaxNumDPP) {
+ locals->RequiredDISPCLK[i][j] = 0.0;
+ locals->DISPCLK_DPPCLK_Support[i][j] = true;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ locals->ODMCombineEnablePerState[i][k] = false;
+ if (locals->SwathWidthYSingleDPP[k] <= locals->MaximumSwathWidth[k]) {
+ locals->NoOfDPP[i][j][k] = 1;
+ locals->RequiredDPPCLK[i][j][k] = locals->MinDPPCLKUsingSingleDPP[k]
+ * (1.0 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0);
+ } else {
+ locals->NoOfDPP[i][j][k] = 2;
+ locals->RequiredDPPCLK[i][j][k] = locals->MinDPPCLKUsingSingleDPP[k]
+ * (1.0 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0) / 2.0;
+ }
+ if (i != mode_lib->vba.soc.num_states) {
+ mode_lib->vba.PlaneRequiredDISPCLK =
+ mode_lib->vba.PixelClock[k]
+ * (1.0 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0)
+ * (1.0 + mode_lib->vba.DISPCLKRampingMargin / 100.0);
+ } else {
+ mode_lib->vba.PlaneRequiredDISPCLK = mode_lib->vba.PixelClock[k]
+ * (1.0 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0);
+ }
+ locals->RequiredDISPCLK[i][j] = dml_max(
+ locals->RequiredDISPCLK[i][j],
+ mode_lib->vba.PlaneRequiredDISPCLK);
+ if (locals->MinDPPCLKUsingSingleDPP[k] / locals->NoOfDPP[i][j][k] * (1.0 + mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0)
+ > mode_lib->vba.MaxDppclkRoundedDownToDFSGranularity
+ || mode_lib->vba.PlaneRequiredDISPCLK > mode_lib->vba.MaxDispclkRoundedDownToDFSGranularity)
+ locals->DISPCLK_DPPCLK_Support[i][j] = false;
+ }
+ locals->TotalNumberOfActiveDPP[i][j] = 0.0;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++)
+ locals->TotalNumberOfActiveDPP[i][j] = locals->TotalNumberOfActiveDPP[i][j] + locals->NoOfDPP[i][j][k];
+ }
+ locals->RequiredDISPCLK[i][j] = dml_max(
+ locals->RequiredDISPCLK[i][j],
+ mode_lib->vba.WritebackRequiredDISPCLK);
+ if (mode_lib->vba.MaxDispclkRoundedDownToDFSGranularity
+ < mode_lib->vba.WritebackRequiredDISPCLK) {
+ locals->DISPCLK_DPPCLK_Support[i][j] = false;
+ }
+ }
+ }
+ /*Viewport Size Check*/
+
+ for (i = 0; i <= mode_lib->vba.soc.num_states; i++) {
+ locals->ViewportSizeSupport[i] = true;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (locals->ODMCombineEnablePerState[i][k] == true) {
+ if (dml_min(locals->SwathWidthYSingleDPP[k], dml_round(mode_lib->vba.HActive[k] / 2.0 * mode_lib->vba.HRatio[k]))
+ > locals->MaximumSwathWidth[k]) {
+ locals->ViewportSizeSupport[i] = false;
+ }
+ } else {
+ if (locals->SwathWidthYSingleDPP[k] / 2.0 > locals->MaximumSwathWidth[k]) {
+ locals->ViewportSizeSupport[i] = false;
+ }
+ }
+ }
+ }
+ /*Total Available Pipes Support Check*/
+
+ for (i = 0; i <= mode_lib->vba.soc.num_states; i++) {
+ for (j = 0; j < 2; j++) {
+ if (locals->TotalNumberOfActiveDPP[i][j] <= mode_lib->vba.MaxNumDPP)
+ locals->TotalAvailablePipesSupport[i][j] = true;
+ else
+ locals->TotalAvailablePipesSupport[i][j] = false;
+ }
+ }
+ /*Total Available OTG Support Check*/
+
+ mode_lib->vba.TotalNumberOfActiveOTG = 0.0;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (mode_lib->vba.BlendingAndTiming[k] == k) {
+ mode_lib->vba.TotalNumberOfActiveOTG = mode_lib->vba.TotalNumberOfActiveOTG
+ + 1.0;
+ }
+ }
+ if (mode_lib->vba.TotalNumberOfActiveOTG <= mode_lib->vba.MaxNumOTG) {
+ mode_lib->vba.NumberOfOTGSupport = true;
+ } else {
+ mode_lib->vba.NumberOfOTGSupport = false;
+ }
+ /*Display IO and DSC Support Check*/
+
+ mode_lib->vba.NonsupportedDSCInputBPC = false;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (!(mode_lib->vba.DSCInputBitPerComponent[k] == 12.0
+ || mode_lib->vba.DSCInputBitPerComponent[k] == 10.0
+ || mode_lib->vba.DSCInputBitPerComponent[k] == 8.0)) {
+ mode_lib->vba.NonsupportedDSCInputBPC = true;
+ }
+ }
+ for (i = 0; i <= mode_lib->vba.soc.num_states; i++) {
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ locals->RequiresDSC[i][k] = 0;
+ locals->RequiresFEC[i][k] = 0;
+ if (mode_lib->vba.BlendingAndTiming[k] == k) {
+ if (mode_lib->vba.Output[k] == dm_hdmi) {
+ locals->RequiresDSC[i][k] = 0;
+ locals->RequiresFEC[i][k] = 0;
+ locals->OutputBppPerState[i][k] = TruncToValidBPP(
+ dml_min(600.0, mode_lib->vba.PHYCLKPerState[i]) / mode_lib->vba.PixelClockBackEnd[k] * 24,
+ false,
+ mode_lib->vba.Output[k],
+ mode_lib->vba.OutputFormat[k],
+ mode_lib->vba.DSCInputBitPerComponent[k]);
+ } else if (mode_lib->vba.Output[k] == dm_dp
+ || mode_lib->vba.Output[k] == dm_edp) {
+ if (mode_lib->vba.Output[k] == dm_edp) {
+ mode_lib->vba.EffectiveFECOverhead = 0.0;
+ } else {
+ mode_lib->vba.EffectiveFECOverhead =
+ mode_lib->vba.FECOverhead;
+ }
+ if (mode_lib->vba.PHYCLKPerState[i] >= 270.0) {
+ mode_lib->vba.Outbpp = TruncToValidBPP(
+ (1.0 - mode_lib->vba.Downspreading / 100.0) * 270.0
+ * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0,
+ false,
+ mode_lib->vba.Output[k],
+ mode_lib->vba.OutputFormat[k],
+ mode_lib->vba.DSCInputBitPerComponent[k]);
+ mode_lib->vba.OutbppDSC = TruncToValidBPP(
+ (1.0 - mode_lib->vba.Downspreading / 100.0) * (1.0 - mode_lib->vba.EffectiveFECOverhead / 100.0) * 270.0
+ * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0,
+ true,
+ mode_lib->vba.Output[k],
+ mode_lib->vba.OutputFormat[k],
+ mode_lib->vba.DSCInputBitPerComponent[k]);
+ if (mode_lib->vba.DSCEnabled[k] == true) {
+ locals->RequiresDSC[i][k] = true;
+ if (mode_lib->vba.Output[k] == dm_dp) {
+ locals->RequiresFEC[i][k] = true;
+ } else {
+ locals->RequiresFEC[i][k] = false;
+ }
+ mode_lib->vba.Outbpp = mode_lib->vba.OutbppDSC;
+ } else {
+ locals->RequiresDSC[i][k] = false;
+ locals->RequiresFEC[i][k] = false;
+ }
+ locals->OutputBppPerState[i][k] = mode_lib->vba.Outbpp;
+ }
+ if (mode_lib->vba.Outbpp == BPP_INVALID && mode_lib->vba.PHYCLKPerState[i] >= 540.0) {
+ mode_lib->vba.Outbpp = TruncToValidBPP(
+ (1.0 - mode_lib->vba.Downspreading / 100.0) * 540.0
+ * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0,
+ false,
+ mode_lib->vba.Output[k],
+ mode_lib->vba.OutputFormat[k],
+ mode_lib->vba.DSCInputBitPerComponent[k]);
+ mode_lib->vba.OutbppDSC = TruncToValidBPP(
+ (1.0 - mode_lib->vba.Downspreading / 100.0) * (1.0 - mode_lib->vba.EffectiveFECOverhead / 100.0) * 540.0
+ * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0,
+ true,
+ mode_lib->vba.Output[k],
+ mode_lib->vba.OutputFormat[k],
+ mode_lib->vba.DSCInputBitPerComponent[k]);
+ if (mode_lib->vba.DSCEnabled[k] == true) {
+ locals->RequiresDSC[i][k] = true;
+ if (mode_lib->vba.Output[k] == dm_dp) {
+ locals->RequiresFEC[i][k] = true;
+ } else {
+ locals->RequiresFEC[i][k] = false;
+ }
+ mode_lib->vba.Outbpp = mode_lib->vba.OutbppDSC;
+ } else {
+ locals->RequiresDSC[i][k] = false;
+ locals->RequiresFEC[i][k] = false;
+ }
+ locals->OutputBppPerState[i][k] = mode_lib->vba.Outbpp;
+ }
+ if (mode_lib->vba.Outbpp == BPP_INVALID
+ && mode_lib->vba.PHYCLKPerState[i]
+ >= 810.0) {
+ mode_lib->vba.Outbpp = TruncToValidBPP(
+ (1.0 - mode_lib->vba.Downspreading / 100.0) * 810.0
+ * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0,
+ false,
+ mode_lib->vba.Output[k],
+ mode_lib->vba.OutputFormat[k],
+ mode_lib->vba.DSCInputBitPerComponent[k]);
+ mode_lib->vba.OutbppDSC = TruncToValidBPP(
+ (1.0 - mode_lib->vba.Downspreading / 100.0) * (1.0 - mode_lib->vba.EffectiveFECOverhead / 100.0) * 810.0
+ * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0,
+ true,
+ mode_lib->vba.Output[k],
+ mode_lib->vba.OutputFormat[k],
+ mode_lib->vba.DSCInputBitPerComponent[k]);
+ if (mode_lib->vba.DSCEnabled[k] == true || mode_lib->vba.Outbpp == BPP_INVALID) {
+ locals->RequiresDSC[i][k] = true;
+ if (mode_lib->vba.Output[k] == dm_dp) {
+ locals->RequiresFEC[i][k] = true;
+ } else {
+ locals->RequiresFEC[i][k] = false;
+ }
+ mode_lib->vba.Outbpp = mode_lib->vba.OutbppDSC;
+ } else {
+ locals->RequiresDSC[i][k] = false;
+ locals->RequiresFEC[i][k] = false;
+ }
+ locals->OutputBppPerState[i][k] =
+ mode_lib->vba.Outbpp;
+ }
+ }
+ } else {
+ locals->OutputBppPerState[i][k] = BPP_BLENDED_PIPE;
+ }
+ }
+ }
+ for (i = 0; i <= mode_lib->vba.soc.num_states; i++) {
+ locals->DIOSupport[i] = true;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (locals->OutputBppPerState[i][k] == BPP_INVALID
+ || (mode_lib->vba.OutputFormat[k] == dm_420
+ && mode_lib->vba.Interlace[k] == true
+ && mode_lib->vba.ProgressiveToInterlaceUnitInOPP == true)) {
+ locals->DIOSupport[i] = false;
+ }
+ }
+ }
+ for (i = 0; i <= mode_lib->vba.soc.num_states; i++) {
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ locals->DSCCLKRequiredMoreThanSupported[i] = false;
+ if (mode_lib->vba.BlendingAndTiming[k] == k) {
+ if ((mode_lib->vba.Output[k] == dm_dp
+ || mode_lib->vba.Output[k] == dm_edp)) {
+ if (mode_lib->vba.OutputFormat[k] == dm_420
+ || mode_lib->vba.OutputFormat[k]
+ == dm_n422) {
+ mode_lib->vba.DSCFormatFactor = 2;
+ } else {
+ mode_lib->vba.DSCFormatFactor = 1;
+ }
+ if (locals->RequiresDSC[i][k] == true) {
+ if (locals->ODMCombineEnablePerState[i][k]
+ == true) {
+ if (mode_lib->vba.PixelClockBackEnd[k] / 6.0 / mode_lib->vba.DSCFormatFactor
+ > (1.0 - mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0) * mode_lib->vba.MaxDSCCLK[i]) {
+ locals->DSCCLKRequiredMoreThanSupported[i] =
+ true;
+ }
+ } else {
+ if (mode_lib->vba.PixelClockBackEnd[k] / 3.0 / mode_lib->vba.DSCFormatFactor
+ > (1.0 - mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading / 100.0) * mode_lib->vba.MaxDSCCLK[i]) {
+ locals->DSCCLKRequiredMoreThanSupported[i] =
+ true;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ for (i = 0; i <= mode_lib->vba.soc.num_states; i++) {
+ locals->NotEnoughDSCUnits[i] = false;
+ mode_lib->vba.TotalDSCUnitsRequired = 0.0;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (locals->RequiresDSC[i][k] == true) {
+ if (locals->ODMCombineEnablePerState[i][k] == true) {
+ mode_lib->vba.TotalDSCUnitsRequired =
+ mode_lib->vba.TotalDSCUnitsRequired + 2.0;
+ } else {
+ mode_lib->vba.TotalDSCUnitsRequired =
+ mode_lib->vba.TotalDSCUnitsRequired + 1.0;
+ }
+ }
+ }
+ if (mode_lib->vba.TotalDSCUnitsRequired > mode_lib->vba.NumberOfDSC) {
+ locals->NotEnoughDSCUnits[i] = true;
+ }
+ }
+ /*DSC Delay per state*/
+
+ for (i = 0; i <= mode_lib->vba.soc.num_states; i++) {
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (mode_lib->vba.BlendingAndTiming[k] != k) {
+ mode_lib->vba.slices = 0;
+ } else if (locals->RequiresDSC[i][k] == 0
+ || locals->RequiresDSC[i][k] == false) {
+ mode_lib->vba.slices = 0;
+ } else if (mode_lib->vba.PixelClockBackEnd[k] > 3200.0) {
+ mode_lib->vba.slices = dml_ceil(
+ mode_lib->vba.PixelClockBackEnd[k] / 400.0,
+ 4.0);
+ } else if (mode_lib->vba.PixelClockBackEnd[k] > 1360.0) {
+ mode_lib->vba.slices = 8.0;
+ } else if (mode_lib->vba.PixelClockBackEnd[k] > 680.0) {
+ mode_lib->vba.slices = 4.0;
+ } else if (mode_lib->vba.PixelClockBackEnd[k] > 340.0) {
+ mode_lib->vba.slices = 2.0;
+ } else {
+ mode_lib->vba.slices = 1.0;
+ }
+ if (locals->OutputBppPerState[i][k] == BPP_BLENDED_PIPE
+ || locals->OutputBppPerState[i][k] == BPP_INVALID) {
+ mode_lib->vba.bpp = 0.0;
+ } else {
+ mode_lib->vba.bpp = locals->OutputBppPerState[i][k];
+ }
+ if (locals->RequiresDSC[i][k] == true && mode_lib->vba.bpp != 0.0) {
+ if (locals->ODMCombineEnablePerState[i][k] == false) {
+ locals->DSCDelayPerState[i][k] =
+ dscceComputeDelay(
+ mode_lib->vba.DSCInputBitPerComponent[k],
+ mode_lib->vba.bpp,
+ dml_ceil(
+ mode_lib->vba.HActive[k]
+ / mode_lib->vba.slices,
+ 1.0),
+ mode_lib->vba.slices,
+ mode_lib->vba.OutputFormat[k])
+ + dscComputeDelay(
+ mode_lib->vba.OutputFormat[k]);
+ } else {
+ locals->DSCDelayPerState[i][k] =
+ 2.0 * (dscceComputeDelay(
+ mode_lib->vba.DSCInputBitPerComponent[k],
+ mode_lib->vba.bpp,
+ dml_ceil(mode_lib->vba.HActive[k] / mode_lib->vba.slices, 1.0),
+ mode_lib->vba.slices / 2,
+ mode_lib->vba.OutputFormat[k])
+ + dscComputeDelay(mode_lib->vba.OutputFormat[k]));
+ }
+ locals->DSCDelayPerState[i][k] =
+ locals->DSCDelayPerState[i][k] * mode_lib->vba.PixelClock[k] / mode_lib->vba.PixelClockBackEnd[k];
+ } else {
+ locals->DSCDelayPerState[i][k] = 0.0;
+ }
+ }
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ for (m = 0; m <= mode_lib->vba.NumberOfActivePlanes - 1; m++) {
+ for (j = 0; j <= mode_lib->vba.NumberOfActivePlanes - 1; j++) {
+ if (mode_lib->vba.BlendingAndTiming[k] == m && locals->RequiresDSC[i][m] == true)
+ locals->DSCDelayPerState[i][k] = locals->DSCDelayPerState[i][m];
+ }
+ }
+ }
+ }
+
+ //Prefetch Check
+ for (i = 0; i <= mode_lib->vba.soc.num_states; i++) {
+ for (j = 0; j < 2; j++) {
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (locals->ODMCombineEnablePerState[i][k] == true)
+ locals->SwathWidthYPerState[i][j][k] = dml_min(locals->SwathWidthYSingleDPP[k], dml_round(locals->HActive[k] / 2 * locals->HRatio[k]));
+ else
+ locals->SwathWidthYPerState[i][j][k] = locals->SwathWidthYSingleDPP[k] / locals->NoOfDPP[i][j][k];
+ locals->SwathWidthGranularityY = 256 / dml_ceil(locals->BytePerPixelInDETY[k], 1) / locals->MaxSwathHeightY[k];
+ locals->RoundedUpMaxSwathSizeBytesY = (dml_ceil(locals->SwathWidthYPerState[i][j][k] - 1, locals->SwathWidthGranularityY)
+ + locals->SwathWidthGranularityY) * locals->BytePerPixelInDETY[k] * locals->MaxSwathHeightY[k];
+ if (locals->SourcePixelFormat[k] == dm_420_10) {
+ locals->RoundedUpMaxSwathSizeBytesY = dml_ceil(locals->RoundedUpMaxSwathSizeBytesY, 256) + 256;
+ }
+ if (locals->MaxSwathHeightC[k] > 0) {
+ locals->SwathWidthGranularityC = 256 / dml_ceil(locals->BytePerPixelInDETC[k], 2) / locals->MaxSwathHeightC[k];
+
+ locals->RoundedUpMaxSwathSizeBytesC = (dml_ceil(locals->SwathWidthYPerState[i][j][k] / 2 - 1, locals->SwathWidthGranularityC)
+ + locals->SwathWidthGranularityC) * locals->BytePerPixelInDETC[k] * locals->MaxSwathHeightC[k];
+ }
+ if (locals->SourcePixelFormat[k] == dm_420_10) {
+ locals->RoundedUpMaxSwathSizeBytesC = dml_ceil(locals->RoundedUpMaxSwathSizeBytesC, 256) + 256;
+ } else {
+ locals->RoundedUpMaxSwathSizeBytesC = 0;
+ }
+
+ if (locals->RoundedUpMaxSwathSizeBytesY + locals->RoundedUpMaxSwathSizeBytesC <= locals->DETBufferSizeInKByte * 1024 / 2) {
+ locals->SwathHeightYPerState[i][j][k] = locals->MaxSwathHeightY[k];
+ locals->SwathHeightCPerState[i][j][k] = locals->MaxSwathHeightC[k];
+ } else {
+ locals->SwathHeightYPerState[i][j][k] = locals->MinSwathHeightY[k];
+ locals->SwathHeightCPerState[i][j][k] = locals->MinSwathHeightC[k];
+ }
+
+ if (locals->BytePerPixelInDETC[k] == 0) {
+ locals->LinesInDETLuma = locals->DETBufferSizeInKByte * 1024 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];
+ locals->LinesInDETChroma = 0;
+ } else if (locals->SwathHeightYPerState[i][j][k] <= locals->SwathHeightCPerState[i][j][k]) {
+ locals->LinesInDETLuma = locals->DETBufferSizeInKByte * 1024 / 2 / locals->BytePerPixelInDETY[k] /
+ locals->SwathWidthYPerState[i][j][k];
+ locals->LinesInDETChroma = locals->DETBufferSizeInKByte * 1024 / 2 / locals->BytePerPixelInDETC[k] / (locals->SwathWidthYPerState[i][j][k] / 2);
+ } else {
+ locals->LinesInDETLuma = locals->DETBufferSizeInKByte * 1024 * 2 / 3 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];
+ locals->LinesInDETChroma = locals->DETBufferSizeInKByte * 1024 / 3 / locals->BytePerPixelInDETY[k] / (locals->SwathWidthYPerState[i][j][k] / 2);
+ }
+
+ locals->EffectiveLBLatencyHidingSourceLinesLuma = dml_min(locals->MaxLineBufferLines,
+ dml_floor(locals->LineBufferSize / locals->LBBitPerPixel[k] / (locals->SwathWidthYPerState[i][j][k]
+ / dml_max(locals->HRatio[k], 1)), 1)) - (locals->vtaps[k] - 1);
+
+ locals->EffectiveLBLatencyHidingSourceLinesChroma = dml_min(locals->MaxLineBufferLines,
+ dml_floor(locals->LineBufferSize / locals->LBBitPerPixel[k]
+ / (locals->SwathWidthYPerState[i][j][k] / 2
+ / dml_max(locals->HRatio[k] / 2, 1)), 1)) - (locals->VTAPsChroma[k] - 1);
+
+ locals->EffectiveDETLBLinesLuma = dml_floor(locals->LinesInDETLuma + dml_min(
+ locals->LinesInDETLuma * locals->RequiredDISPCLK[i][j] * locals->BytePerPixelInDETY[k] *
+ locals->PSCL_FACTOR[k] / locals->ReturnBWPerState[i],
+ locals->EffectiveLBLatencyHidingSourceLinesLuma),
+ locals->SwathHeightYPerState[i][j][k]);
+
+ locals->EffectiveDETLBLinesChroma = dml_floor(locals->LinesInDETChroma + dml_min(
+ locals->LinesInDETChroma * locals->RequiredDISPCLK[i][j] * locals->BytePerPixelInDETC[k] *
+ locals->PSCL_FACTOR_CHROMA[k] / locals->ReturnBWPerState[i],
+ locals->EffectiveLBLatencyHidingSourceLinesChroma),
+ locals->SwathHeightCPerState[i][j][k]);
+
+ if (locals->BytePerPixelInDETC[k] == 0) {
+ locals->UrgentLatencySupportUsPerState[i][j][k] = locals->EffectiveDETLBLinesLuma * (locals->HTotal[k] / locals->PixelClock[k])
+ / locals->VRatio[k] - locals->EffectiveDETLBLinesLuma * locals->SwathWidthYPerState[i][j][k] *
+ dml_ceil(locals->BytePerPixelInDETY[k], 1) / (locals->ReturnBWPerState[i] / locals->NoOfDPP[i][j][k]);
+ } else {
+ locals->UrgentLatencySupportUsPerState[i][j][k] = dml_min(
+ locals->EffectiveDETLBLinesLuma * (locals->HTotal[k] / locals->PixelClock[k])
+ / locals->VRatio[k] - locals->EffectiveDETLBLinesLuma * locals->SwathWidthYPerState[i][j][k] *
+ dml_ceil(locals->BytePerPixelInDETY[k], 1) / (locals->ReturnBWPerState[i] / locals->NoOfDPP[i][j][k]),
+ locals->EffectiveDETLBLinesChroma * (locals->HTotal[k] / locals->PixelClock[k]) / (locals->VRatio[k] / 2) -
+ locals->EffectiveDETLBLinesChroma * locals->SwathWidthYPerState[i][j][k] / 2 *
+ dml_ceil(locals->BytePerPixelInDETC[k], 2) / (locals->ReturnBWPerState[i] / locals->NoOfDPP[i][j][k]));
+ }
+ }
+ }
+ }
+
+ for (i = 0; i <= locals->soc.num_states; i++) {
+ for (j = 0; j < 2; j++) {
+ locals->UrgentLatencySupport[i][j] = true;
+ for (k = 0; k < locals->NumberOfActivePlanes; k++) {
+ if (locals->UrgentLatencySupportUsPerState[i][j][k] < locals->UrgentLatency)
+ locals->UrgentLatencySupport[i][j] = false;
+ }
+ }
+ }
+
+
+ /*Prefetch Check*/
+ for (i = 0; i <= locals->soc.num_states; i++) {
+ for (j = 0; j < 2; j++) {
+ locals->TotalNumberOfDCCActiveDPP[i][j] = 0;
+ for (k = 0; k < locals->NumberOfActivePlanes; k++) {
+ if (locals->DCCEnable[k] == true) {
+ locals->TotalNumberOfDCCActiveDPP[i][j] =
+ locals->TotalNumberOfDCCActiveDPP[i][j] + locals->NoOfDPP[i][j][k];
+ }
+ }
+ }
+ }
+
+ CalculateMinAndMaxPrefetchMode(locals->AllowDRAMSelfRefreshOrDRAMClockChangeInVblank, &locals->MinPrefetchMode, &locals->MaxPrefetchMode);
+
+ for (i = 0; i <= locals->soc.num_states; i++) {
+ for (j = 0; j < 2; j++) {
+ for (k = 0; k < locals->NumberOfActivePlanes; k++) {
+ locals->NoOfDPPThisState[k] = locals->NoOfDPP[i][j][k];
+ locals->RequiredDPPCLKThisState[k] = locals->RequiredDPPCLK[i][j][k];
+ locals->SwathHeightYThisState[k] = locals->SwathHeightYPerState[i][j][k];
+ locals->SwathHeightCThisState[k] = locals->SwathHeightCPerState[i][j][k];
+ locals->SwathWidthYThisState[k] = locals->SwathWidthYPerState[i][j][k];
+ mode_lib->vba.ProjectedDCFCLKDeepSleep = dml_max(
+ mode_lib->vba.ProjectedDCFCLKDeepSleep,
+ mode_lib->vba.PixelClock[k] / 16.0);
+ if (mode_lib->vba.BytePerPixelInDETC[k] == 0.0) {
+ if (mode_lib->vba.VRatio[k] <= 1.0) {
+ mode_lib->vba.ProjectedDCFCLKDeepSleep =
+ dml_max(
+ mode_lib->vba.ProjectedDCFCLKDeepSleep,
+ 1.1
+ * dml_ceil(
+ mode_lib->vba.BytePerPixelInDETY[k],
+ 1.0)
+ / 64.0
+ * mode_lib->vba.HRatio[k]
+ * mode_lib->vba.PixelClock[k]
+ / mode_lib->vba.NoOfDPP[i][j][k]);
+ } else {
+ mode_lib->vba.ProjectedDCFCLKDeepSleep =
+ dml_max(
+ mode_lib->vba.ProjectedDCFCLKDeepSleep,
+ 1.1
+ * dml_ceil(
+ mode_lib->vba.BytePerPixelInDETY[k],
+ 1.0)
+ / 64.0
+ * mode_lib->vba.PSCL_FACTOR[k]
+ * mode_lib->vba.RequiredDPPCLK[i][j][k]);
+ }
+ } else {
+ if (mode_lib->vba.VRatio[k] <= 1.0) {
+ mode_lib->vba.ProjectedDCFCLKDeepSleep =
+ dml_max(
+ mode_lib->vba.ProjectedDCFCLKDeepSleep,
+ 1.1
+ * dml_ceil(
+ mode_lib->vba.BytePerPixelInDETY[k],
+ 1.0)
+ / 32.0
+ * mode_lib->vba.HRatio[k]
+ * mode_lib->vba.PixelClock[k]
+ / mode_lib->vba.NoOfDPP[i][j][k]);
+ } else {
+ mode_lib->vba.ProjectedDCFCLKDeepSleep =
+ dml_max(
+ mode_lib->vba.ProjectedDCFCLKDeepSleep,
+ 1.1
+ * dml_ceil(
+ mode_lib->vba.BytePerPixelInDETY[k],
+ 1.0)
+ / 32.0
+ * mode_lib->vba.PSCL_FACTOR[k]
+ * mode_lib->vba.RequiredDPPCLK[i][j][k]);
+ }
+ if (mode_lib->vba.VRatio[k] / 2.0 <= 1.0) {
+ mode_lib->vba.ProjectedDCFCLKDeepSleep =
+ dml_max(
+ mode_lib->vba.ProjectedDCFCLKDeepSleep,
+ 1.1
+ * dml_ceil(
+ mode_lib->vba.BytePerPixelInDETC[k],
+ 2.0)
+ / 32.0
+ * mode_lib->vba.HRatio[k]
+ / 2.0
+ * mode_lib->vba.PixelClock[k]
+ / mode_lib->vba.NoOfDPP[i][j][k]);
+ } else {
+ mode_lib->vba.ProjectedDCFCLKDeepSleep =
+ dml_max(
+ mode_lib->vba.ProjectedDCFCLKDeepSleep,
+ 1.1
+ * dml_ceil(
+ mode_lib->vba.BytePerPixelInDETC[k],
+ 2.0)
+ / 32.0
+ * mode_lib->vba.PSCL_FACTOR_CHROMA[k]
+ * mode_lib->vba.RequiredDPPCLK[i][j][k]);
+ }
+ }
+ }
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ mode_lib->vba.PDEAndMetaPTEBytesPerFrameY = CalculateVMAndRowBytes(
+ mode_lib,
+ mode_lib->vba.DCCEnable[k],
+ mode_lib->vba.Read256BlockHeightY[k],
+ mode_lib->vba.Read256BlockWidthY[k],
+ mode_lib->vba.SourcePixelFormat[k],
+ mode_lib->vba.SurfaceTiling[k],
+ dml_ceil(mode_lib->vba.BytePerPixelInDETY[k], 1.0),
+ mode_lib->vba.SourceScan[k],
+ mode_lib->vba.ViewportWidth[k],
+ mode_lib->vba.ViewportHeight[k],
+ mode_lib->vba.SwathWidthYPerState[i][j][k],
+ mode_lib->vba.GPUVMEnable,
+ mode_lib->vba.VMMPageSize,
+ mode_lib->vba.PTEBufferSizeInRequestsLuma,
+ mode_lib->vba.PDEProcessingBufIn64KBReqs,
+ mode_lib->vba.PitchY[k],
+ mode_lib->vba.DCCMetaPitchY[k],
+ &mode_lib->vba.MacroTileWidthY[k],
+ &mode_lib->vba.MetaRowBytesY,
+ &mode_lib->vba.DPTEBytesPerRowY,
+ &mode_lib->vba.PTEBufferSizeNotExceededY[i][j][k],
+ &mode_lib->vba.dpte_row_height[k],
+ &mode_lib->vba.meta_row_height[k]);
+ mode_lib->vba.PrefetchLinesY[k] = CalculatePrefetchSourceLines(
+ mode_lib,
+ mode_lib->vba.VRatio[k],
+ mode_lib->vba.vtaps[k],
+ mode_lib->vba.Interlace[k],
+ mode_lib->vba.ProgressiveToInterlaceUnitInOPP,
+ mode_lib->vba.SwathHeightYPerState[i][j][k],
+ mode_lib->vba.ViewportYStartY[k],
+ &mode_lib->vba.PrefillY[k],
+ &mode_lib->vba.MaxNumSwY[k]);
+ if ((mode_lib->vba.SourcePixelFormat[k] != dm_444_64
+ && mode_lib->vba.SourcePixelFormat[k] != dm_444_32
+ && mode_lib->vba.SourcePixelFormat[k] != dm_444_16
+ && mode_lib->vba.SourcePixelFormat[k] != dm_mono_16
+ && mode_lib->vba.SourcePixelFormat[k] != dm_mono_8)) {
+ mode_lib->vba.PDEAndMetaPTEBytesPerFrameC = CalculateVMAndRowBytes(
+ mode_lib,
+ mode_lib->vba.DCCEnable[k],
+ mode_lib->vba.Read256BlockHeightY[k],
+ mode_lib->vba.Read256BlockWidthY[k],
+ mode_lib->vba.SourcePixelFormat[k],
+ mode_lib->vba.SurfaceTiling[k],
+ dml_ceil(mode_lib->vba.BytePerPixelInDETC[k], 2.0),
+ mode_lib->vba.SourceScan[k],
+ mode_lib->vba.ViewportWidth[k] / 2.0,
+ mode_lib->vba.ViewportHeight[k] / 2.0,
+ mode_lib->vba.SwathWidthYPerState[i][j][k] / 2.0,
+ mode_lib->vba.GPUVMEnable,
+ mode_lib->vba.VMMPageSize,
+ mode_lib->vba.PTEBufferSizeInRequestsLuma,
+ mode_lib->vba.PDEProcessingBufIn64KBReqs,
+ mode_lib->vba.PitchC[k],
+ 0.0,
+ &mode_lib->vba.MacroTileWidthC[k],
+ &mode_lib->vba.MetaRowBytesC,
+ &mode_lib->vba.DPTEBytesPerRowC,
+ &mode_lib->vba.PTEBufferSizeNotExceededC[i][j][k],
+ &mode_lib->vba.dpte_row_height_chroma[k],
+ &mode_lib->vba.meta_row_height_chroma[k]);
+ mode_lib->vba.PrefetchLinesC[k] = CalculatePrefetchSourceLines(
+ mode_lib,
+ mode_lib->vba.VRatio[k] / 2.0,
+ mode_lib->vba.VTAPsChroma[k],
+ mode_lib->vba.Interlace[k],
+ mode_lib->vba.ProgressiveToInterlaceUnitInOPP,
+ mode_lib->vba.SwathHeightCPerState[i][j][k],
+ mode_lib->vba.ViewportYStartC[k],
+ &mode_lib->vba.PrefillC[k],
+ &mode_lib->vba.MaxNumSwC[k]);
+ } else {
+ mode_lib->vba.PDEAndMetaPTEBytesPerFrameC = 0.0;
+ mode_lib->vba.MetaRowBytesC = 0.0;
+ mode_lib->vba.DPTEBytesPerRowC = 0.0;
+ locals->PrefetchLinesC[k] = 0.0;
+ locals->PTEBufferSizeNotExceededC[i][j][k] = true;
+ locals->PTEBufferSizeInRequestsForLuma = mode_lib->vba.PTEBufferSizeInRequestsLuma + mode_lib->vba.PTEBufferSizeInRequestsChroma;
+ }
+ locals->PDEAndMetaPTEBytesPerFrame[k] =
+ mode_lib->vba.PDEAndMetaPTEBytesPerFrameY + mode_lib->vba.PDEAndMetaPTEBytesPerFrameC;
+ locals->MetaRowBytes[k] = mode_lib->vba.MetaRowBytesY + mode_lib->vba.MetaRowBytesC;
+ locals->DPTEBytesPerRow[k] = mode_lib->vba.DPTEBytesPerRowY + mode_lib->vba.DPTEBytesPerRowC;
+
+ CalculateActiveRowBandwidth(
+ mode_lib->vba.GPUVMEnable,
+ mode_lib->vba.SourcePixelFormat[k],
+ mode_lib->vba.VRatio[k],
+ mode_lib->vba.DCCEnable[k],
+ mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k],
+ mode_lib->vba.MetaRowBytesY,
+ mode_lib->vba.MetaRowBytesC,
+ mode_lib->vba.meta_row_height[k],
+ mode_lib->vba.meta_row_height_chroma[k],
+ mode_lib->vba.DPTEBytesPerRowY,
+ mode_lib->vba.DPTEBytesPerRowC,
+ mode_lib->vba.dpte_row_height[k],
+ mode_lib->vba.dpte_row_height_chroma[k],
+ &mode_lib->vba.meta_row_bw[k],
+ &mode_lib->vba.dpte_row_bw[k],
+ &mode_lib->vba.qual_row_bw[k]);
+ }
+ mode_lib->vba.ExtraLatency =
+ mode_lib->vba.UrgentRoundTripAndOutOfOrderLatencyPerState[i]
+ + (mode_lib->vba.TotalNumberOfActiveDPP[i][j]
+ * mode_lib->vba.PixelChunkSizeInKByte
+ + mode_lib->vba.TotalNumberOfDCCActiveDPP[i][j]
+ * mode_lib->vba.MetaChunkSize)
+ * 1024.0
+ / mode_lib->vba.ReturnBWPerState[i];
+ if (mode_lib->vba.GPUVMEnable == true) {
+ mode_lib->vba.ExtraLatency = mode_lib->vba.ExtraLatency
+ + mode_lib->vba.TotalNumberOfActiveDPP[i][j]
+ * mode_lib->vba.PTEGroupSize
+ / mode_lib->vba.ReturnBWPerState[i];
+ }
+ mode_lib->vba.TimeCalc = 24.0 / mode_lib->vba.ProjectedDCFCLKDeepSleep;
+
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (mode_lib->vba.BlendingAndTiming[k] == k) {
+ if (mode_lib->vba.WritebackEnable[k] == true) {
+ locals->WritebackDelay[i][k] = mode_lib->vba.WritebackLatency
+ + CalculateWriteBackDelay(
+ mode_lib->vba.WritebackPixelFormat[k],
+ mode_lib->vba.WritebackHRatio[k],
+ mode_lib->vba.WritebackVRatio[k],
+ mode_lib->vba.WritebackLumaHTaps[k],
+ mode_lib->vba.WritebackLumaVTaps[k],
+ mode_lib->vba.WritebackChromaHTaps[k],
+ mode_lib->vba.WritebackChromaVTaps[k],
+ mode_lib->vba.WritebackDestinationWidth[k]) / locals->RequiredDISPCLK[i][j];
+ } else {
+ locals->WritebackDelay[i][k] = 0.0;
+ }
+ for (m = 0; m <= mode_lib->vba.NumberOfActivePlanes - 1; m++) {
+ if (mode_lib->vba.BlendingAndTiming[m] == k
+ && mode_lib->vba.WritebackEnable[m]
+ == true) {
+ locals->WritebackDelay[i][k] = dml_max(locals->WritebackDelay[i][k],
+ mode_lib->vba.WritebackLatency + CalculateWriteBackDelay(
+ mode_lib->vba.WritebackPixelFormat[m],
+ mode_lib->vba.WritebackHRatio[m],
+ mode_lib->vba.WritebackVRatio[m],
+ mode_lib->vba.WritebackLumaHTaps[m],
+ mode_lib->vba.WritebackLumaVTaps[m],
+ mode_lib->vba.WritebackChromaHTaps[m],
+ mode_lib->vba.WritebackChromaVTaps[m],
+ mode_lib->vba.WritebackDestinationWidth[m]) / locals->RequiredDISPCLK[i][j]);
+ }
+ }
+ }
+ }
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ for (m = 0; m <= mode_lib->vba.NumberOfActivePlanes - 1; m++) {
+ if (mode_lib->vba.BlendingAndTiming[k] == m) {
+ locals->WritebackDelay[i][k] = locals->WritebackDelay[i][m];
+ }
+ }
+ }
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ for (m = 0; m < locals->NumberOfCursors[k]; m++)
+ locals->cursor_bw[k] = locals->NumberOfCursors[k] * locals->CursorWidth[k][m] * locals->CursorBPP[k][m]
+ / 8 / (locals->HTotal[k] / locals->PixelClock[k]) * locals->VRatio[k];
+ }
+
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ locals->MaximumVStartup[k] = mode_lib->vba.VTotal[k] - mode_lib->vba.VActive[k]
+ - dml_max(1.0, dml_ceil(locals->WritebackDelay[i][k] / (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k]), 1.0));
+ }
+
+ mode_lib->vba.NextPrefetchMode = mode_lib->vba.MinPrefetchMode;
+ do {
+ mode_lib->vba.PrefetchMode[i][j] = mode_lib->vba.NextPrefetchMode;
+ mode_lib->vba.NextPrefetchMode = mode_lib->vba.NextPrefetchMode + 1;
+
+ mode_lib->vba.TWait = CalculateTWait(
+ mode_lib->vba.PrefetchMode[i][j],
+ mode_lib->vba.DRAMClockChangeLatency,
+ mode_lib->vba.UrgentLatency,
+ mode_lib->vba.SREnterPlusExitTime);
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+
+ if (mode_lib->vba.XFCEnabled[k] == true) {
+ mode_lib->vba.XFCRemoteSurfaceFlipDelay =
+ CalculateRemoteSurfaceFlipDelay(
+ mode_lib,
+ mode_lib->vba.VRatio[k],
+ locals->SwathWidthYPerState[i][j][k],
+ dml_ceil(locals->BytePerPixelInDETY[k], 1.0),
+ mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k],
+ mode_lib->vba.XFCTSlvVupdateOffset,
+ mode_lib->vba.XFCTSlvVupdateWidth,
+ mode_lib->vba.XFCTSlvVreadyOffset,
+ mode_lib->vba.XFCXBUFLatencyTolerance,
+ mode_lib->vba.XFCFillBWOverhead,
+ mode_lib->vba.XFCSlvChunkSize,
+ mode_lib->vba.XFCBusTransportTime,
+ mode_lib->vba.TimeCalc,
+ mode_lib->vba.TWait,
+ &mode_lib->vba.SrcActiveDrainRate,
+ &mode_lib->vba.TInitXFill,
+ &mode_lib->vba.TslvChk);
+ } else {
+ mode_lib->vba.XFCRemoteSurfaceFlipDelay = 0.0;
+ }
+ mode_lib->vba.IsErrorResult[i][j][k] =
+ CalculatePrefetchSchedule(
+ mode_lib,
+ mode_lib->vba.RequiredDPPCLK[i][j][k],
+ mode_lib->vba.RequiredDISPCLK[i][j],
+ mode_lib->vba.PixelClock[k],
+ mode_lib->vba.ProjectedDCFCLKDeepSleep,
+ mode_lib->vba.DSCDelayPerState[i][k],
+ mode_lib->vba.NoOfDPP[i][j][k],
+ mode_lib->vba.ScalerEnabled[k],
+ mode_lib->vba.NumberOfCursors[k],
+ mode_lib->vba.DPPCLKDelaySubtotal,
+ mode_lib->vba.DPPCLKDelaySCL,
+ mode_lib->vba.DPPCLKDelaySCLLBOnly,
+ mode_lib->vba.DPPCLKDelayCNVCFormater,
+ mode_lib->vba.DPPCLKDelayCNVCCursor,
+ mode_lib->vba.DISPCLKDelaySubtotal,
+ mode_lib->vba.SwathWidthYPerState[i][j][k]
+ / mode_lib->vba.HRatio[k],
+ mode_lib->vba.OutputFormat[k],
+ mode_lib->vba.VTotal[k]
+ - mode_lib->vba.VActive[k],
+ mode_lib->vba.HTotal[k],
+ mode_lib->vba.MaxInterDCNTileRepeaters,
+ mode_lib->vba.MaximumVStartup[k],
+ mode_lib->vba.GPUVMMaxPageTableLevels,
+ mode_lib->vba.GPUVMEnable,
+ mode_lib->vba.DynamicMetadataEnable[k],
+ mode_lib->vba.DynamicMetadataLinesBeforeActiveRequired[k],
+ mode_lib->vba.DynamicMetadataTransmittedBytes[k],
+ mode_lib->vba.DCCEnable[k],
+ mode_lib->vba.UrgentLatencyPixelDataOnly,
+ mode_lib->vba.ExtraLatency,
+ mode_lib->vba.TimeCalc,
+ mode_lib->vba.PDEAndMetaPTEBytesPerFrame[k],
+ mode_lib->vba.MetaRowBytes[k],
+ mode_lib->vba.DPTEBytesPerRow[k],
+ mode_lib->vba.PrefetchLinesY[k],
+ mode_lib->vba.SwathWidthYPerState[i][j][k],
+ mode_lib->vba.BytePerPixelInDETY[k],
+ mode_lib->vba.PrefillY[k],
+ mode_lib->vba.MaxNumSwY[k],
+ mode_lib->vba.PrefetchLinesC[k],
+ mode_lib->vba.BytePerPixelInDETC[k],
+ mode_lib->vba.PrefillC[k],
+ mode_lib->vba.MaxNumSwC[k],
+ mode_lib->vba.SwathHeightYPerState[i][j][k],
+ mode_lib->vba.SwathHeightCPerState[i][j][k],
+ mode_lib->vba.TWait,
+ mode_lib->vba.XFCEnabled[k],
+ mode_lib->vba.XFCRemoteSurfaceFlipDelay,
+ mode_lib->vba.Interlace[k],
+ mode_lib->vba.ProgressiveToInterlaceUnitInOPP,
+ mode_lib->vba.DSTXAfterScaler,
+ mode_lib->vba.DSTYAfterScaler,
+ &mode_lib->vba.LineTimesForPrefetch[k],
+ &mode_lib->vba.PrefetchBW[k],
+ &mode_lib->vba.LinesForMetaPTE[k],
+ &mode_lib->vba.LinesForMetaAndDPTERow[k],
+ &mode_lib->vba.VRatioPreY[i][j][k],
+ &mode_lib->vba.VRatioPreC[i][j][k],
+ &mode_lib->vba.RequiredPrefetchPixelDataBWLuma[i][j][k],
+ &mode_lib->vba.VStartupRequiredWhenNotEnoughTimeForDynamicMetadata,
+ &mode_lib->vba.Tno_bw[k],
+ &mode_lib->vba.VUpdateOffsetPix[k],
+ &mode_lib->vba.VUpdateWidthPix[k],
+ &mode_lib->vba.VReadyOffsetPix[k]);
+ }
+ mode_lib->vba.MaximumReadBandwidthWithoutPrefetch = 0.0;
+ mode_lib->vba.MaximumReadBandwidthWithPrefetch = 0.0;
+ locals->prefetch_vm_bw_valid = true;
+ locals->prefetch_row_bw_valid = true;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (locals->PDEAndMetaPTEBytesPerFrame[k] == 0)
+ locals->prefetch_vm_bw[k] = 0;
+ else if (locals->LinesForMetaPTE[k] > 0)
+ locals->prefetch_vm_bw[k] = locals->PDEAndMetaPTEBytesPerFrame[k]
+ / (locals->LinesForMetaPTE[k] * locals->HTotal[k] / locals->PixelClock[k]);
+ else {
+ locals->prefetch_vm_bw[k] = 0;
+ locals->prefetch_vm_bw_valid = false;
+ }
+ if (locals->MetaRowBytes[k] + locals->DPTEBytesPerRow[k] == 0)
+ locals->prefetch_row_bw[k] = 0;
+ else if (locals->LinesForMetaAndDPTERow[k] > 0)
+ locals->prefetch_row_bw[k] = (locals->MetaRowBytes[k] + locals->DPTEBytesPerRow[k])
+ / (locals->LinesForMetaAndDPTERow[k] * locals->HTotal[k] / locals->PixelClock[k]);
+ else {
+ locals->prefetch_row_bw[k] = 0;
+ locals->prefetch_row_bw_valid = false;
+ }
+
+ mode_lib->vba.MaximumReadBandwidthWithoutPrefetch = mode_lib->vba.MaximumReadBandwidthWithPrefetch
+ + mode_lib->vba.cursor_bw[k] + mode_lib->vba.ReadBandwidth[k] + mode_lib->vba.meta_row_bw[k] + mode_lib->vba.dpte_row_bw[k];
+ mode_lib->vba.MaximumReadBandwidthWithPrefetch =
+ mode_lib->vba.MaximumReadBandwidthWithPrefetch
+ + mode_lib->vba.cursor_bw[k]
+ + dml_max3(
+ mode_lib->vba.prefetch_vm_bw[k],
+ mode_lib->vba.prefetch_row_bw[k],
+ dml_max(mode_lib->vba.ReadBandwidth[k],
+ mode_lib->vba.RequiredPrefetchPixelDataBWLuma[i][j][k])
+ + mode_lib->vba.meta_row_bw[k] + mode_lib->vba.dpte_row_bw[k]);
+ }
+ locals->BandwidthWithoutPrefetchSupported[i] = true;
+ if (mode_lib->vba.MaximumReadBandwidthWithoutPrefetch > locals->ReturnBWPerState[i]) {
+ locals->BandwidthWithoutPrefetchSupported[i] = false;
+ }
+
+ locals->PrefetchSupported[i][j] = true;
+ if (mode_lib->vba.MaximumReadBandwidthWithPrefetch > locals->ReturnBWPerState[i]) {
+ locals->PrefetchSupported[i][j] = false;
+ }
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (locals->LineTimesForPrefetch[k] < 2.0
+ || locals->LinesForMetaPTE[k] >= 8.0
+ || locals->LinesForMetaAndDPTERow[k] >= 16.0
+ || mode_lib->vba.IsErrorResult[i][j][k] == true) {
+ locals->PrefetchSupported[i][j] = false;
+ }
+ }
+ locals->VRatioInPrefetchSupported[i][j] = true;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (locals->VRatioPreY[i][j][k] > 4.0
+ || locals->VRatioPreC[i][j][k] > 4.0
+ || mode_lib->vba.IsErrorResult[i][j][k] == true) {
+ locals->VRatioInPrefetchSupported[i][j] = false;
+ }
+ }
+ } while ((locals->PrefetchSupported[i][j] != true || locals->VRatioInPrefetchSupported[i][j] != true)
+ && mode_lib->vba.NextPrefetchMode < mode_lib->vba.MaxPrefetchMode);
+
+ if (mode_lib->vba.PrefetchSupported[i][j] == true
+ && mode_lib->vba.VRatioInPrefetchSupported[i][j] == true) {
+ mode_lib->vba.BandwidthAvailableForImmediateFlip =
+ mode_lib->vba.ReturnBWPerState[i];
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ mode_lib->vba.BandwidthAvailableForImmediateFlip =
+ mode_lib->vba.BandwidthAvailableForImmediateFlip
+ - mode_lib->vba.cursor_bw[k]
+ - dml_max(
+ mode_lib->vba.ReadBandwidth[k] + mode_lib->vba.qual_row_bw[k],
+ mode_lib->vba.PrefetchBW[k]);
+ }
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ mode_lib->vba.ImmediateFlipBytes[k] = 0.0;
+ if ((mode_lib->vba.SourcePixelFormat[k] != dm_420_8
+ && mode_lib->vba.SourcePixelFormat[k] != dm_420_10)) {
+ mode_lib->vba.ImmediateFlipBytes[k] =
+ mode_lib->vba.PDEAndMetaPTEBytesPerFrame[k]
+ + mode_lib->vba.MetaRowBytes[k]
+ + mode_lib->vba.DPTEBytesPerRow[k];
+ }
+ }
+ mode_lib->vba.TotImmediateFlipBytes = 0.0;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if ((mode_lib->vba.SourcePixelFormat[k] != dm_420_8
+ && mode_lib->vba.SourcePixelFormat[k] != dm_420_10)) {
+ mode_lib->vba.TotImmediateFlipBytes =
+ mode_lib->vba.TotImmediateFlipBytes
+ + mode_lib->vba.ImmediateFlipBytes[k];
+ }
+ }
+
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ CalculateFlipSchedule(
+ mode_lib,
+ mode_lib->vba.ExtraLatency,
+ mode_lib->vba.UrgentLatencyPixelDataOnly,
+ mode_lib->vba.GPUVMMaxPageTableLevels,
+ mode_lib->vba.GPUVMEnable,
+ mode_lib->vba.BandwidthAvailableForImmediateFlip,
+ mode_lib->vba.TotImmediateFlipBytes,
+ mode_lib->vba.SourcePixelFormat[k],
+ mode_lib->vba.ImmediateFlipBytes[k],
+ mode_lib->vba.HTotal[k]
+ / mode_lib->vba.PixelClock[k],
+ mode_lib->vba.VRatio[k],
+ mode_lib->vba.Tno_bw[k],
+ mode_lib->vba.PDEAndMetaPTEBytesPerFrame[k],
+ mode_lib->vba.MetaRowBytes[k],
+ mode_lib->vba.DPTEBytesPerRow[k],
+ mode_lib->vba.DCCEnable[k],
+ mode_lib->vba.dpte_row_height[k],
+ mode_lib->vba.meta_row_height[k],
+ mode_lib->vba.qual_row_bw[k],
+ &mode_lib->vba.DestinationLinesToRequestVMInImmediateFlip[k],
+ &mode_lib->vba.DestinationLinesToRequestRowInImmediateFlip[k],
+ &mode_lib->vba.final_flip_bw[k],
+ &mode_lib->vba.ImmediateFlipSupportedForPipe[k]);
+ }
+ mode_lib->vba.total_dcn_read_bw_with_flip = 0.0;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ mode_lib->vba.total_dcn_read_bw_with_flip =
+ mode_lib->vba.total_dcn_read_bw_with_flip
+ + mode_lib->vba.cursor_bw[k]
+ + dml_max3(
+ mode_lib->vba.prefetch_vm_bw[k],
+ mode_lib->vba.prefetch_row_bw[k],
+ mode_lib->vba.final_flip_bw[k]
+ + dml_max(
+ mode_lib->vba.ReadBandwidth[k],
+ mode_lib->vba.RequiredPrefetchPixelDataBWLuma[i][j][k]));
+ }
+ mode_lib->vba.ImmediateFlipSupportedForState[i][j] = true;
+ if (mode_lib->vba.total_dcn_read_bw_with_flip
+ > mode_lib->vba.ReturnBWPerState[i]) {
+ mode_lib->vba.ImmediateFlipSupportedForState[i][j] = false;
+ }
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (mode_lib->vba.ImmediateFlipSupportedForPipe[k] == false) {
+ mode_lib->vba.ImmediateFlipSupportedForState[i][j] = false;
+ }
+ }
+ } else {
+ mode_lib->vba.ImmediateFlipSupportedForState[i][j] = false;
+ }
+ }
+ }
+
+ /*Vertical Active BW support*/
+ mode_lib->vba.MaxTotalVActiveRDBandwidth = 0;
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; k++)
+ mode_lib->vba.MaxTotalVActiveRDBandwidth = mode_lib->vba.MaxTotalVActiveRDBandwidth + mode_lib->vba.ReadBandwidth[k];
+ for (i = 0; i <= mode_lib->vba.soc.num_states; i++) {
+ mode_lib->vba.MaxTotalVerticalActiveAvailableBandwidth[i] = dml_min(mode_lib->vba.ReturnBusWidth *
+ mode_lib->vba.DCFCLKPerState[i], mode_lib->vba.FabricAndDRAMBandwidthPerState[i] * 1000) *
+ mode_lib->vba.MaxAveragePercentOfIdealDRAMBWDisplayCanUseInNormalSystemOperation / 100;
+ if (mode_lib->vba.MaxTotalVActiveRDBandwidth <= mode_lib->vba.MaxTotalVerticalActiveAvailableBandwidth[i])
+ mode_lib->vba.TotalVerticalActiveBandwidthSupport[i] = true;
+ else
+ mode_lib->vba.TotalVerticalActiveBandwidthSupport[i] = false;
+ }
+
+ /*PTE Buffer Size Check*/
+
+ for (i = 0; i <= mode_lib->vba.soc.num_states; i++) {
+ for (j = 0; j < 2; j++) {
+ locals->PTEBufferSizeNotExceeded[i][j] = true;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (locals->PTEBufferSizeNotExceededY[i][j][k] == false
+ || locals->PTEBufferSizeNotExceededC[i][j][k] == false) {
+ locals->PTEBufferSizeNotExceeded[i][j] = false;
+ }
+ }
+ }
+ }
+ /*Cursor Support Check*/
+ mode_lib->vba.CursorSupport = true;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ for (j = 0; j < 2; j++) {
+ if (mode_lib->vba.CursorWidth[k][j] > 0.0) {
+ if (dml_floor(
+ dml_floor(
+ mode_lib->vba.CursorBufferSize
+ - mode_lib->vba.CursorChunkSize,
+ mode_lib->vba.CursorChunkSize) * 1024.0
+ / (mode_lib->vba.CursorWidth[k][j]
+ * mode_lib->vba.CursorBPP[k][j]
+ / 8.0),
+ 1.0)
+ * (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k])
+ / mode_lib->vba.VRatio[k] < mode_lib->vba.UrgentLatencyPixelDataOnly
+ || (mode_lib->vba.CursorBPP[k][j] == 64.0
+ && mode_lib->vba.Cursor64BppSupport == false)) {
+ mode_lib->vba.CursorSupport = false;
+ }
+ }
+ }
+ }
+ /*Valid Pitch Check*/
+
+ mode_lib->vba.PitchSupport = true;
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ locals->AlignedYPitch[k] = dml_ceil(
+ dml_max(mode_lib->vba.PitchY[k], mode_lib->vba.ViewportWidth[k]),
+ locals->MacroTileWidthY[k]);
+ if (locals->AlignedYPitch[k] > mode_lib->vba.PitchY[k]) {
+ mode_lib->vba.PitchSupport = false;
+ }
+ if (mode_lib->vba.DCCEnable[k] == true) {
+ locals->AlignedDCCMetaPitch[k] = dml_ceil(
+ dml_max(
+ mode_lib->vba.DCCMetaPitchY[k],
+ mode_lib->vba.ViewportWidth[k]),
+ 64.0 * locals->Read256BlockWidthY[k]);
+ } else {
+ locals->AlignedDCCMetaPitch[k] = mode_lib->vba.DCCMetaPitchY[k];
+ }
+ if (locals->AlignedDCCMetaPitch[k] > mode_lib->vba.DCCMetaPitchY[k]) {
+ mode_lib->vba.PitchSupport = false;
+ }
+ if (mode_lib->vba.SourcePixelFormat[k] != dm_444_64
+ && mode_lib->vba.SourcePixelFormat[k] != dm_444_32
+ && mode_lib->vba.SourcePixelFormat[k] != dm_444_16
+ && mode_lib->vba.SourcePixelFormat[k] != dm_mono_16
+ && mode_lib->vba.SourcePixelFormat[k] != dm_mono_8) {
+ locals->AlignedCPitch[k] = dml_ceil(
+ dml_max(
+ mode_lib->vba.PitchC[k],
+ mode_lib->vba.ViewportWidth[k] / 2.0),
+ locals->MacroTileWidthC[k]);
+ } else {
+ locals->AlignedCPitch[k] = mode_lib->vba.PitchC[k];
+ }
+ if (locals->AlignedCPitch[k] > mode_lib->vba.PitchC[k]) {
+ mode_lib->vba.PitchSupport = false;
+ }
+ }
+ /*Mode Support, Voltage State and SOC Configuration*/
+
+ for (i = mode_lib->vba.soc.num_states; i >= 0; i--) {
+ for (j = 0; j < 2; j++) {
+ enum dm_validation_status status = DML_VALIDATION_OK;
+
+ if (mode_lib->vba.ScaleRatioAndTapsSupport != true) {
+ status = DML_FAIL_SCALE_RATIO_TAP;
+ } else if (mode_lib->vba.SourceFormatPixelAndScanSupport != true) {
+ status = DML_FAIL_SOURCE_PIXEL_FORMAT;
+ } else if (locals->ViewportSizeSupport[i] != true) {
+ status = DML_FAIL_VIEWPORT_SIZE;
+ } else if (locals->DIOSupport[i] != true) {
+ status = DML_FAIL_DIO_SUPPORT;
+ } else if (locals->NotEnoughDSCUnits[i] != false) {
+ status = DML_FAIL_NOT_ENOUGH_DSC;
+ } else if (locals->DSCCLKRequiredMoreThanSupported[i] != false) {
+ status = DML_FAIL_DSC_CLK_REQUIRED;
+ } else if (locals->UrgentLatencySupport[i][j] != true) {
+ status = DML_FAIL_URGENT_LATENCY;
+ } else if (locals->ROBSupport[i] != true) {
+ status = DML_FAIL_REORDERING_BUFFER;
+ } else if (locals->DISPCLK_DPPCLK_Support[i][j] != true) {
+ status = DML_FAIL_DISPCLK_DPPCLK;
+ } else if (locals->TotalAvailablePipesSupport[i][j] != true) {
+ status = DML_FAIL_TOTAL_AVAILABLE_PIPES;
+ } else if (mode_lib->vba.NumberOfOTGSupport != true) {
+ status = DML_FAIL_NUM_OTG;
+ } else if (mode_lib->vba.WritebackModeSupport != true) {
+ status = DML_FAIL_WRITEBACK_MODE;
+ } else if (mode_lib->vba.WritebackLatencySupport != true) {
+ status = DML_FAIL_WRITEBACK_LATENCY;
+ } else if (mode_lib->vba.WritebackScaleRatioAndTapsSupport != true) {
+ status = DML_FAIL_WRITEBACK_SCALE_RATIO_TAP;
+ } else if (mode_lib->vba.CursorSupport != true) {
+ status = DML_FAIL_CURSOR_SUPPORT;
+ } else if (mode_lib->vba.PitchSupport != true) {
+ status = DML_FAIL_PITCH_SUPPORT;
+ } else if (locals->PrefetchSupported[i][j] != true) {
+ status = DML_FAIL_PREFETCH_SUPPORT;
+ } else if (locals->TotalVerticalActiveBandwidthSupport[i] != true) {
+ status = DML_FAIL_TOTAL_V_ACTIVE_BW;
+ } else if (locals->VRatioInPrefetchSupported[i][j] != true) {
+ status = DML_FAIL_V_RATIO_PREFETCH;
+ } else if (locals->PTEBufferSizeNotExceeded[i][j] != true) {
+ status = DML_FAIL_PTE_BUFFER_SIZE;
+ } else if (mode_lib->vba.NonsupportedDSCInputBPC != false) {
+ status = DML_FAIL_DSC_INPUT_BPC;
+ }
+
+ if (status == DML_VALIDATION_OK) {
+ locals->ModeSupport[i][j] = true;
+ } else {
+ locals->ModeSupport[i][j] = false;
+ }
+ locals->ValidationStatus[i] = status;
+ }
+ }
+ {
+ unsigned int MaximumMPCCombine = 0;
+ mode_lib->vba.VoltageLevel = mode_lib->vba.soc.num_states + 1;
+ for (i = mode_lib->vba.VoltageOverrideLevel; i <= mode_lib->vba.soc.num_states; i++) {
+ if (locals->ModeSupport[i][0] == true || locals->ModeSupport[i][1] == true) {
+ mode_lib->vba.VoltageLevel = i;
+ if (locals->ModeSupport[i][1] == true && (locals->ModeSupport[i][0] == false
+ || mode_lib->vba.WhenToDoMPCCombine == dm_mpc_always_when_possible)) {
+ MaximumMPCCombine = 1;
+ } else {
+ MaximumMPCCombine = 0;
+ }
+ break;
+ }
+ }
+ mode_lib->vba.ImmediateFlipSupport =
+ locals->ImmediateFlipSupportedForState[mode_lib->vba.VoltageLevel][MaximumMPCCombine];
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ mode_lib->vba.DPPPerPlane[k] = locals->NoOfDPP[mode_lib->vba.VoltageLevel][MaximumMPCCombine][k];
+ locals->DPPCLK[k] = locals->RequiredDPPCLK[mode_lib->vba.VoltageLevel][MaximumMPCCombine][k];
+ }
+ mode_lib->vba.DISPCLK = locals->RequiredDISPCLK[mode_lib->vba.VoltageLevel][MaximumMPCCombine];
+ mode_lib->vba.maxMpcComb = MaximumMPCCombine;
+ }
+ mode_lib->vba.DCFCLK = mode_lib->vba.DCFCLKPerState[mode_lib->vba.VoltageLevel];
+ mode_lib->vba.DRAMSpeed = mode_lib->vba.DRAMSpeedPerState[mode_lib->vba.VoltageLevel];
+ mode_lib->vba.FabricClock = mode_lib->vba.FabricClockPerState[mode_lib->vba.VoltageLevel];
+ mode_lib->vba.SOCCLK = mode_lib->vba.SOCCLKPerState[mode_lib->vba.VoltageLevel];
+ mode_lib->vba.ReturnBW = locals->ReturnBWPerState[mode_lib->vba.VoltageLevel];
+ mode_lib->vba.FabricAndDRAMBandwidth = locals->FabricAndDRAMBandwidthPerState[mode_lib->vba.VoltageLevel];
+ for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
+ if (mode_lib->vba.BlendingAndTiming[k] == k) {
+ mode_lib->vba.ODMCombineEnabled[k] =
+ locals->ODMCombineEnablePerState[mode_lib->vba.VoltageLevel][k];
+ } else {
+ mode_lib->vba.ODMCombineEnabled[k] = 0;
+ }
+ mode_lib->vba.DSCEnabled[k] =
+ locals->RequiresDSC[mode_lib->vba.VoltageLevel][k];
+ mode_lib->vba.OutputBpp[k] =
+ locals->OutputBppPerState[mode_lib->vba.VoltageLevel][k];
+ }
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.h b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.h
new file mode 100644
index 000000000000..92b6805f4342
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef _DCN20_DISPLAY_MODE_VBA_H_
+#define _DCN20_DISPLAY_MODE_VBA_H_
+
+void dml20_recalculate(struct display_mode_lib *mode_lib);
+void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_lib);
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c
new file mode 100644
index 000000000000..878bf4782ce6
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c
@@ -0,0 +1,1701 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "../display_mode_lib.h"
+#include "../display_mode_vba.h"
+#include "display_rq_dlg_calc_20.h"
+
+// Function: dml20_rq_dlg_get_rq_params
+// Calculate requestor related parameters that register definition agnostic
+// (i.e. this layer does try to separate real values from register definition)
+// Input:
+// pipe_src_param - pipe source configuration (e.g. vp, pitch, etc.)
+// Output:
+// rq_param - values that can be used to setup RQ (e.g. swath_height, plane1_addr, etc.)
+//
+static void dml20_rq_dlg_get_rq_params(
+ struct display_mode_lib *mode_lib,
+ display_rq_params_st * rq_param,
+ const display_pipe_source_params_st pipe_src_param);
+
+// Function: dml20_rq_dlg_get_dlg_params
+// Calculate deadline related parameters
+//
+static void dml20_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
+ const display_e2e_pipe_params_st *e2e_pipe_param,
+ const unsigned int num_pipes,
+ const unsigned int pipe_idx,
+ display_dlg_regs_st *disp_dlg_regs,
+ display_ttu_regs_st *disp_ttu_regs,
+ const display_rq_dlg_params_st rq_dlg_param,
+ const display_dlg_sys_params_st dlg_sys_param,
+ const bool cstate_en,
+ const bool pstate_en);
+/*
+ * NOTE:
+ * This file is gcc-parseable HW gospel, coming straight from HW engineers.
+ *
+ * It doesn't adhere to Linux kernel style and sometimes will do things in odd
+ * ways. Unless there is something clearly wrong with it the code should
+ * remain as-is as it provides us with a guarantee from HW that it is correct.
+ */
+
+static void calculate_ttu_cursor(struct display_mode_lib *mode_lib,
+ double *refcyc_per_req_delivery_pre_cur,
+ double *refcyc_per_req_delivery_cur,
+ double refclk_freq_in_mhz,
+ double ref_freq_to_pix_freq,
+ double hscale_pixel_rate_l,
+ double hscl_ratio,
+ double vratio_pre_l,
+ double vratio_l,
+ unsigned int cur_width,
+ enum cursor_bpp cur_bpp);
+
+#include "../dml_inline_defs.h"
+
+static unsigned int get_bytes_per_element(enum source_format_class source_format, bool is_chroma)
+{
+ unsigned int ret_val = 0;
+
+ if (source_format == dm_444_16) {
+ if (!is_chroma)
+ ret_val = 2;
+ } else if (source_format == dm_444_32) {
+ if (!is_chroma)
+ ret_val = 4;
+ } else if (source_format == dm_444_64) {
+ if (!is_chroma)
+ ret_val = 8;
+ } else if (source_format == dm_420_8) {
+ if (is_chroma)
+ ret_val = 2;
+ else
+ ret_val = 1;
+ } else if (source_format == dm_420_10) {
+ if (is_chroma)
+ ret_val = 4;
+ else
+ ret_val = 2;
+ } else if (source_format == dm_444_8) {
+ ret_val = 1;
+ }
+ return ret_val;
+}
+
+static bool is_dual_plane(enum source_format_class source_format)
+{
+ bool ret_val = 0;
+
+ if ((source_format == dm_420_8) || (source_format == dm_420_10))
+ ret_val = 1;
+
+ return ret_val;
+}
+
+static double get_refcyc_per_delivery(struct display_mode_lib *mode_lib,
+ double refclk_freq_in_mhz,
+ double pclk_freq_in_mhz,
+ bool odm_combine,
+ unsigned int recout_width,
+ unsigned int hactive,
+ double vratio,
+ double hscale_pixel_rate,
+ unsigned int delivery_width,
+ unsigned int req_per_swath_ub)
+{
+ double refcyc_per_delivery = 0.0;
+
+ if (vratio <= 1.0) {
+ if (odm_combine)
+ refcyc_per_delivery = (double) refclk_freq_in_mhz
+ * dml_min((double) recout_width, (double) hactive / 2.0)
+ / pclk_freq_in_mhz / (double) req_per_swath_ub;
+ else
+ refcyc_per_delivery = (double) refclk_freq_in_mhz * (double) recout_width
+ / pclk_freq_in_mhz / (double) req_per_swath_ub;
+ } else {
+ refcyc_per_delivery = (double) refclk_freq_in_mhz * (double) delivery_width
+ / (double) hscale_pixel_rate / (double) req_per_swath_ub;
+ }
+
+ dml_print("DML_DLG: %s: refclk_freq_in_mhz = %3.2f\n", __func__, refclk_freq_in_mhz);
+ dml_print("DML_DLG: %s: pclk_freq_in_mhz = %3.2f\n", __func__, pclk_freq_in_mhz);
+ dml_print("DML_DLG: %s: recout_width = %d\n", __func__, recout_width);
+ dml_print("DML_DLG: %s: vratio = %3.2f\n", __func__, vratio);
+ dml_print("DML_DLG: %s: req_per_swath_ub = %d\n", __func__, req_per_swath_ub);
+ dml_print("DML_DLG: %s: refcyc_per_delivery= %3.2f\n", __func__, refcyc_per_delivery);
+
+ return refcyc_per_delivery;
+
+}
+
+static unsigned int get_blk_size_bytes(const enum source_macro_tile_size tile_size)
+{
+ if (tile_size == dm_256k_tile)
+ return (256 * 1024);
+ else if (tile_size == dm_64k_tile)
+ return (64 * 1024);
+ else
+ return (4 * 1024);
+}
+
+static void extract_rq_sizing_regs(struct display_mode_lib *mode_lib,
+ display_data_rq_regs_st *rq_regs,
+ const display_data_rq_sizing_params_st rq_sizing)
+{
+ dml_print("DML_DLG: %s: rq_sizing param\n", __func__);
+ print__data_rq_sizing_params_st(mode_lib, rq_sizing);
+
+ rq_regs->chunk_size = dml_log2(rq_sizing.chunk_bytes) - 10;
+
+ if (rq_sizing.min_chunk_bytes == 0)
+ rq_regs->min_chunk_size = 0;
+ else
+ rq_regs->min_chunk_size = dml_log2(rq_sizing.min_chunk_bytes) - 8 + 1;
+
+ rq_regs->meta_chunk_size = dml_log2(rq_sizing.meta_chunk_bytes) - 10;
+ if (rq_sizing.min_meta_chunk_bytes == 0)
+ rq_regs->min_meta_chunk_size = 0;
+ else
+ rq_regs->min_meta_chunk_size = dml_log2(rq_sizing.min_meta_chunk_bytes) - 6 + 1;
+
+ rq_regs->dpte_group_size = dml_log2(rq_sizing.dpte_group_bytes) - 6;
+ rq_regs->mpte_group_size = dml_log2(rq_sizing.mpte_group_bytes) - 6;
+}
+
+static void extract_rq_regs(struct display_mode_lib *mode_lib,
+ display_rq_regs_st *rq_regs,
+ const display_rq_params_st rq_param)
+{
+ unsigned int detile_buf_size_in_bytes = mode_lib->ip.det_buffer_size_kbytes * 1024;
+ unsigned int detile_buf_plane1_addr = 0;
+
+ extract_rq_sizing_regs(mode_lib, &(rq_regs->rq_regs_l), rq_param.sizing.rq_l);
+
+ rq_regs->rq_regs_l.pte_row_height_linear = dml_floor(dml_log2(rq_param.dlg.rq_l.dpte_row_height),
+ 1) - 3;
+
+ if (rq_param.yuv420) {
+ extract_rq_sizing_regs(mode_lib, &(rq_regs->rq_regs_c), rq_param.sizing.rq_c);
+ rq_regs->rq_regs_c.pte_row_height_linear = dml_floor(dml_log2(rq_param.dlg.rq_c.dpte_row_height),
+ 1) - 3;
+ }
+
+ rq_regs->rq_regs_l.swath_height = dml_log2(rq_param.dlg.rq_l.swath_height);
+ rq_regs->rq_regs_c.swath_height = dml_log2(rq_param.dlg.rq_c.swath_height);
+
+ // FIXME: take the max between luma, chroma chunk size?
+ // okay for now, as we are setting chunk_bytes to 8kb anyways
+ if (rq_param.sizing.rq_l.chunk_bytes >= 32 * 1024) { //32kb
+ rq_regs->drq_expansion_mode = 0;
+ } else {
+ rq_regs->drq_expansion_mode = 2;
+ }
+ rq_regs->prq_expansion_mode = 1;
+ rq_regs->mrq_expansion_mode = 1;
+ rq_regs->crq_expansion_mode = 1;
+
+ if (rq_param.yuv420) {
+ if ((double) rq_param.misc.rq_l.stored_swath_bytes
+ / (double) rq_param.misc.rq_c.stored_swath_bytes <= 1.5) {
+ detile_buf_plane1_addr = (detile_buf_size_in_bytes / 2.0 / 64.0); // half to chroma
+ } else {
+ detile_buf_plane1_addr = dml_round_to_multiple((unsigned int) ((2.0 * detile_buf_size_in_bytes) / 3.0),
+ 256,
+ 0) / 64.0; // 2/3 to chroma
+ }
+ }
+ rq_regs->plane1_base_address = detile_buf_plane1_addr;
+}
+
+static void handle_det_buf_split(struct display_mode_lib *mode_lib,
+ display_rq_params_st *rq_param,
+ const display_pipe_source_params_st pipe_src_param)
+{
+ unsigned int total_swath_bytes = 0;
+ unsigned int swath_bytes_l = 0;
+ unsigned int swath_bytes_c = 0;
+ unsigned int full_swath_bytes_packed_l = 0;
+ unsigned int full_swath_bytes_packed_c = 0;
+ bool req128_l = 0;
+ bool req128_c = 0;
+ bool surf_linear = (pipe_src_param.sw_mode == dm_sw_linear);
+ bool surf_vert = (pipe_src_param.source_scan == dm_vert);
+ unsigned int log2_swath_height_l = 0;
+ unsigned int log2_swath_height_c = 0;
+ unsigned int detile_buf_size_in_bytes = mode_lib->ip.det_buffer_size_kbytes * 1024;
+
+ full_swath_bytes_packed_l = rq_param->misc.rq_l.full_swath_bytes;
+ full_swath_bytes_packed_c = rq_param->misc.rq_c.full_swath_bytes;
+
+ if (rq_param->yuv420_10bpc) {
+ full_swath_bytes_packed_l = dml_round_to_multiple(rq_param->misc.rq_l.full_swath_bytes * 2 / 3,
+ 256,
+ 1) + 256;
+ full_swath_bytes_packed_c = dml_round_to_multiple(rq_param->misc.rq_c.full_swath_bytes * 2 / 3,
+ 256,
+ 1) + 256;
+ }
+
+ if (rq_param->yuv420) {
+ total_swath_bytes = 2 * full_swath_bytes_packed_l + 2 * full_swath_bytes_packed_c;
+
+ if (total_swath_bytes <= detile_buf_size_in_bytes) { //full 256b request
+ req128_l = 0;
+ req128_c = 0;
+ swath_bytes_l = full_swath_bytes_packed_l;
+ swath_bytes_c = full_swath_bytes_packed_c;
+ } else { //128b request (for luma only for yuv420 8bpc)
+ req128_l = 1;
+ req128_c = 0;
+ swath_bytes_l = full_swath_bytes_packed_l / 2;
+ swath_bytes_c = full_swath_bytes_packed_c;
+ }
+ // Note: assumption, the config that pass in will fit into
+ // the detiled buffer.
+ } else {
+ total_swath_bytes = 2 * full_swath_bytes_packed_l;
+
+ if (total_swath_bytes <= detile_buf_size_in_bytes)
+ req128_l = 0;
+ else
+ req128_l = 1;
+
+ swath_bytes_l = total_swath_bytes;
+ swath_bytes_c = 0;
+ }
+ rq_param->misc.rq_l.stored_swath_bytes = swath_bytes_l;
+ rq_param->misc.rq_c.stored_swath_bytes = swath_bytes_c;
+
+ if (surf_linear) {
+ log2_swath_height_l = 0;
+ log2_swath_height_c = 0;
+ } else if (!surf_vert) {
+ log2_swath_height_l = dml_log2(rq_param->misc.rq_l.blk256_height) - req128_l;
+ log2_swath_height_c = dml_log2(rq_param->misc.rq_c.blk256_height) - req128_c;
+ } else {
+ log2_swath_height_l = dml_log2(rq_param->misc.rq_l.blk256_width) - req128_l;
+ log2_swath_height_c = dml_log2(rq_param->misc.rq_c.blk256_width) - req128_c;
+ }
+ rq_param->dlg.rq_l.swath_height = 1 << log2_swath_height_l;
+ rq_param->dlg.rq_c.swath_height = 1 << log2_swath_height_c;
+
+ dml_print("DML_DLG: %s: req128_l = %0d\n", __func__, req128_l);
+ dml_print("DML_DLG: %s: req128_c = %0d\n", __func__, req128_c);
+ dml_print("DML_DLG: %s: full_swath_bytes_packed_l = %0d\n",
+ __func__,
+ full_swath_bytes_packed_l);
+ dml_print("DML_DLG: %s: full_swath_bytes_packed_c = %0d\n",
+ __func__,
+ full_swath_bytes_packed_c);
+}
+
+static void get_meta_and_pte_attr(struct display_mode_lib *mode_lib,
+ display_data_rq_dlg_params_st *rq_dlg_param,
+ display_data_rq_misc_params_st *rq_misc_param,
+ display_data_rq_sizing_params_st *rq_sizing_param,
+ unsigned int vp_width,
+ unsigned int vp_height,
+ unsigned int data_pitch,
+ unsigned int meta_pitch,
+ unsigned int source_format,
+ unsigned int tiling,
+ unsigned int macro_tile_size,
+ unsigned int source_scan,
+ unsigned int is_chroma)
+{
+ bool surf_linear = (tiling == dm_sw_linear);
+ bool surf_vert = (source_scan == dm_vert);
+
+ unsigned int bytes_per_element;
+ unsigned int bytes_per_element_y = get_bytes_per_element((enum source_format_class)(source_format),
+ false);
+ unsigned int bytes_per_element_c = get_bytes_per_element((enum source_format_class)(source_format),
+ true);
+
+ unsigned int blk256_width = 0;
+ unsigned int blk256_height = 0;
+
+ unsigned int blk256_width_y = 0;
+ unsigned int blk256_height_y = 0;
+ unsigned int blk256_width_c = 0;
+ unsigned int blk256_height_c = 0;
+ unsigned int log2_bytes_per_element;
+ unsigned int log2_blk256_width;
+ unsigned int log2_blk256_height;
+ unsigned int blk_bytes;
+ unsigned int log2_blk_bytes;
+ unsigned int log2_blk_height;
+ unsigned int log2_blk_width;
+ unsigned int log2_meta_req_bytes;
+ unsigned int log2_meta_req_height;
+ unsigned int log2_meta_req_width;
+ unsigned int meta_req_width;
+ unsigned int meta_req_height;
+ unsigned int log2_meta_row_height;
+ unsigned int meta_row_width_ub;
+ unsigned int log2_meta_chunk_bytes;
+ unsigned int log2_meta_chunk_height;
+
+ //full sized meta chunk width in unit of data elements
+ unsigned int log2_meta_chunk_width;
+ unsigned int log2_min_meta_chunk_bytes;
+ unsigned int min_meta_chunk_width;
+ unsigned int meta_chunk_width;
+ unsigned int meta_chunk_per_row_int;
+ unsigned int meta_row_remainder;
+ unsigned int meta_chunk_threshold;
+ unsigned int meta_blk_bytes;
+ unsigned int meta_blk_height;
+ unsigned int meta_blk_width;
+ unsigned int meta_surface_bytes;
+ unsigned int vmpg_bytes;
+ unsigned int meta_pte_req_per_frame_ub;
+ unsigned int meta_pte_bytes_per_frame_ub;
+ const unsigned int log2_vmpg_bytes = dml_log2(mode_lib->soc.vmm_page_size_bytes);
+ const unsigned int dpte_buf_in_pte_reqs = mode_lib->ip.dpte_buffer_size_in_pte_reqs_luma;
+ const unsigned int pde_proc_buffer_size_64k_reqs =
+ mode_lib->ip.pde_proc_buffer_size_64k_reqs;
+
+ unsigned int log2_vmpg_height = 0;
+ unsigned int log2_vmpg_width = 0;
+ unsigned int log2_dpte_req_height_ptes = 0;
+ unsigned int log2_dpte_req_height = 0;
+ unsigned int log2_dpte_req_width = 0;
+ unsigned int log2_dpte_row_height_linear = 0;
+ unsigned int log2_dpte_row_height = 0;
+ unsigned int log2_dpte_group_width = 0;
+ unsigned int dpte_row_width_ub = 0;
+ unsigned int dpte_req_height = 0;
+ unsigned int dpte_req_width = 0;
+ unsigned int dpte_group_width = 0;
+ unsigned int log2_dpte_group_bytes = 0;
+ unsigned int log2_dpte_group_length = 0;
+ unsigned int pde_buf_entries;
+ bool yuv420 = (source_format == dm_420_8 || source_format == dm_420_10);
+
+ Calculate256BBlockSizes((enum source_format_class)(source_format),
+ (enum dm_swizzle_mode)(tiling),
+ bytes_per_element_y,
+ bytes_per_element_c,
+ &blk256_height_y,
+ &blk256_height_c,
+ &blk256_width_y,
+ &blk256_width_c);
+
+ if (!is_chroma) {
+ blk256_width = blk256_width_y;
+ blk256_height = blk256_height_y;
+ bytes_per_element = bytes_per_element_y;
+ } else {
+ blk256_width = blk256_width_c;
+ blk256_height = blk256_height_c;
+ bytes_per_element = bytes_per_element_c;
+ }
+
+ log2_bytes_per_element = dml_log2(bytes_per_element);
+
+ dml_print("DML_DLG: %s: surf_linear = %d\n", __func__, surf_linear);
+ dml_print("DML_DLG: %s: surf_vert = %d\n", __func__, surf_vert);
+ dml_print("DML_DLG: %s: blk256_width = %d\n", __func__, blk256_width);
+ dml_print("DML_DLG: %s: blk256_height = %d\n", __func__, blk256_height);
+
+ log2_blk256_width = dml_log2((double) blk256_width);
+ log2_blk256_height = dml_log2((double) blk256_height);
+ blk_bytes = surf_linear ?
+ 256 : get_blk_size_bytes((enum source_macro_tile_size) macro_tile_size);
+ log2_blk_bytes = dml_log2((double) blk_bytes);
+ log2_blk_height = 0;
+ log2_blk_width = 0;
+
+ // remember log rule
+ // "+" in log is multiply
+ // "-" in log is divide
+ // "/2" is like square root
+ // blk is vertical biased
+ if (tiling != dm_sw_linear)
+ log2_blk_height = log2_blk256_height
+ + dml_ceil((double) (log2_blk_bytes - 8) / 2.0, 1);
+ else
+ log2_blk_height = 0; // blk height of 1
+
+ log2_blk_width = log2_blk_bytes - log2_bytes_per_element - log2_blk_height;
+
+ if (!surf_vert) {
+ rq_dlg_param->swath_width_ub = dml_round_to_multiple(vp_width - 1, blk256_width, 1)
+ + blk256_width;
+ rq_dlg_param->req_per_swath_ub = rq_dlg_param->swath_width_ub >> log2_blk256_width;
+ } else {
+ rq_dlg_param->swath_width_ub = dml_round_to_multiple(vp_height - 1, blk256_height, 1)
+ + blk256_height;
+ rq_dlg_param->req_per_swath_ub = rq_dlg_param->swath_width_ub >> log2_blk256_height;
+ }
+
+ if (!surf_vert)
+ rq_misc_param->full_swath_bytes = rq_dlg_param->swath_width_ub * blk256_height
+ * bytes_per_element;
+ else
+ rq_misc_param->full_swath_bytes = rq_dlg_param->swath_width_ub * blk256_width
+ * bytes_per_element;
+
+ rq_misc_param->blk256_height = blk256_height;
+ rq_misc_param->blk256_width = blk256_width;
+
+ // -------
+ // meta
+ // -------
+ log2_meta_req_bytes = 6; // meta request is 64b and is 8x8byte meta element
+
+ // each 64b meta request for dcn is 8x8 meta elements and
+ // a meta element covers one 256b block of the the data surface.
+ log2_meta_req_height = log2_blk256_height + 3; // meta req is 8x8 byte, each byte represent 1 blk256
+ log2_meta_req_width = log2_meta_req_bytes + 8 - log2_bytes_per_element
+ - log2_meta_req_height;
+ meta_req_width = 1 << log2_meta_req_width;
+ meta_req_height = 1 << log2_meta_req_height;
+ log2_meta_row_height = 0;
+ meta_row_width_ub = 0;
+
+ // the dimensions of a meta row are meta_row_width x meta_row_height in elements.
+ // calculate upper bound of the meta_row_width
+ if (!surf_vert) {
+ log2_meta_row_height = log2_meta_req_height;
+ meta_row_width_ub = dml_round_to_multiple(vp_width - 1, meta_req_width, 1)
+ + meta_req_width;
+ rq_dlg_param->meta_req_per_row_ub = meta_row_width_ub / meta_req_width;
+ } else {
+ log2_meta_row_height = log2_meta_req_width;
+ meta_row_width_ub = dml_round_to_multiple(vp_height - 1, meta_req_height, 1)
+ + meta_req_height;
+ rq_dlg_param->meta_req_per_row_ub = meta_row_width_ub / meta_req_height;
+ }
+ rq_dlg_param->meta_bytes_per_row_ub = rq_dlg_param->meta_req_per_row_ub * 64;
+
+ rq_dlg_param->meta_row_height = 1 << log2_meta_row_height;
+
+ log2_meta_chunk_bytes = dml_log2(rq_sizing_param->meta_chunk_bytes);
+ log2_meta_chunk_height = log2_meta_row_height;
+
+ //full sized meta chunk width in unit of data elements
+ log2_meta_chunk_width = log2_meta_chunk_bytes + 8 - log2_bytes_per_element
+ - log2_meta_chunk_height;
+ log2_min_meta_chunk_bytes = dml_log2(rq_sizing_param->min_meta_chunk_bytes);
+ min_meta_chunk_width = 1
+ << (log2_min_meta_chunk_bytes + 8 - log2_bytes_per_element
+ - log2_meta_chunk_height);
+ meta_chunk_width = 1 << log2_meta_chunk_width;
+ meta_chunk_per_row_int = (unsigned int) (meta_row_width_ub / meta_chunk_width);
+ meta_row_remainder = meta_row_width_ub % meta_chunk_width;
+ meta_chunk_threshold = 0;
+ meta_blk_bytes = 4096;
+ meta_blk_height = blk256_height * 64;
+ meta_blk_width = meta_blk_bytes * 256 / bytes_per_element / meta_blk_height;
+ meta_surface_bytes = meta_pitch
+ * (dml_round_to_multiple(vp_height - 1, meta_blk_height, 1) + meta_blk_height)
+ * bytes_per_element / 256;
+ vmpg_bytes = mode_lib->soc.vmm_page_size_bytes;
+ meta_pte_req_per_frame_ub = (dml_round_to_multiple(meta_surface_bytes - vmpg_bytes,
+ 8 * vmpg_bytes,
+ 1) + 8 * vmpg_bytes) / (8 * vmpg_bytes);
+ meta_pte_bytes_per_frame_ub = meta_pte_req_per_frame_ub * 64; //64B mpte request
+ rq_dlg_param->meta_pte_bytes_per_frame_ub = meta_pte_bytes_per_frame_ub;
+
+ dml_print("DML_DLG: %s: meta_blk_height = %d\n", __func__, meta_blk_height);
+ dml_print("DML_DLG: %s: meta_blk_width = %d\n", __func__, meta_blk_width);
+ dml_print("DML_DLG: %s: meta_surface_bytes = %d\n", __func__, meta_surface_bytes);
+ dml_print("DML_DLG: %s: meta_pte_req_per_frame_ub = %d\n",
+ __func__,
+ meta_pte_req_per_frame_ub);
+ dml_print("DML_DLG: %s: meta_pte_bytes_per_frame_ub = %d\n",
+ __func__,
+ meta_pte_bytes_per_frame_ub);
+
+ if (!surf_vert)
+ meta_chunk_threshold = 2 * min_meta_chunk_width - meta_req_width;
+ else
+ meta_chunk_threshold = 2 * min_meta_chunk_width - meta_req_height;
+
+ if (meta_row_remainder <= meta_chunk_threshold)
+ rq_dlg_param->meta_chunks_per_row_ub = meta_chunk_per_row_int + 1;
+ else
+ rq_dlg_param->meta_chunks_per_row_ub = meta_chunk_per_row_int + 2;
+
+ // ------
+ // dpte
+ // ------
+ if (surf_linear) {
+ log2_vmpg_height = 0; // one line high
+ } else {
+ log2_vmpg_height = (log2_vmpg_bytes - 8) / 2 + log2_blk256_height;
+ }
+ log2_vmpg_width = log2_vmpg_bytes - log2_bytes_per_element - log2_vmpg_height;
+
+ // only 3 possible shapes for dpte request in dimensions of ptes: 8x1, 4x2, 2x4.
+ if (surf_linear) { //one 64B PTE request returns 8 PTEs
+ log2_dpte_req_height_ptes = 0;
+ log2_dpte_req_width = log2_vmpg_width + 3;
+ log2_dpte_req_height = 0;
+ } else if (log2_blk_bytes == 12) { //4KB tile means 4kB page size
+ //one 64B req gives 8x1 PTEs for 4KB tile
+ log2_dpte_req_height_ptes = 0;
+ log2_dpte_req_width = log2_blk_width + 3;
+ log2_dpte_req_height = log2_blk_height + 0;
+ } else if ((log2_blk_bytes >= 16) && (log2_vmpg_bytes == 12)) { // tile block >= 64KB
+ //two 64B reqs of 2x4 PTEs give 16 PTEs to cover 64KB
+ log2_dpte_req_height_ptes = 4;
+ log2_dpte_req_width = log2_blk256_width + 4; // log2_64KB_width
+ log2_dpte_req_height = log2_blk256_height + 4; // log2_64KB_height
+ } else { //64KB page size and must 64KB tile block
+ //one 64B req gives 8x1 PTEs for 64KB tile
+ log2_dpte_req_height_ptes = 0;
+ log2_dpte_req_width = log2_blk_width + 3;
+ log2_dpte_req_height = log2_blk_height + 0;
+ }
+
+ // The dpte request dimensions in data elements is dpte_req_width x dpte_req_height
+ // log2_vmpg_width is how much 1 pte represent, now calculating how much a 64b pte req represent
+ // That depends on the pte shape (i.e. 8x1, 4x2, 2x4)
+ //log2_dpte_req_height = log2_vmpg_height + log2_dpte_req_height_ptes;
+ //log2_dpte_req_width = log2_vmpg_width + log2_dpte_req_width_ptes;
+ dpte_req_height = 1 << log2_dpte_req_height;
+ dpte_req_width = 1 << log2_dpte_req_width;
+
+ // calculate pitch dpte row buffer can hold
+ // round the result down to a power of two.
+ pde_buf_entries = yuv420 ? (pde_proc_buffer_size_64k_reqs >> 1) : pde_proc_buffer_size_64k_reqs;
+ if (surf_linear) {
+ unsigned int dpte_row_height;
+
+ log2_dpte_row_height_linear = dml_floor(dml_log2(dml_min(64 * 1024 * pde_buf_entries
+ / bytes_per_element,
+ dpte_buf_in_pte_reqs
+ * dpte_req_width)
+ / data_pitch),
+ 1);
+
+ ASSERT(log2_dpte_row_height_linear >= 3);
+
+ if (log2_dpte_row_height_linear > 7)
+ log2_dpte_row_height_linear = 7;
+
+ log2_dpte_row_height = log2_dpte_row_height_linear;
+ // For linear, the dpte row is pitch dependent and the pte requests wrap at the pitch boundary.
+ // the dpte_row_width_ub is the upper bound of data_pitch*dpte_row_height in elements with this unique buffering.
+ dpte_row_height = 1 << log2_dpte_row_height;
+ dpte_row_width_ub = dml_round_to_multiple(data_pitch * dpte_row_height - 1,
+ dpte_req_width,
+ 1) + dpte_req_width;
+ rq_dlg_param->dpte_req_per_row_ub = dpte_row_width_ub / dpte_req_width;
+ } else {
+ // the upper bound of the dpte_row_width without dependency on viewport position follows.
+ // for tiled mode, row height is the same as req height and row store up to vp size upper bound
+ if (!surf_vert) {
+ log2_dpte_row_height = log2_dpte_req_height;
+ dpte_row_width_ub = dml_round_to_multiple(vp_width - 1, dpte_req_width, 1)
+ + dpte_req_width;
+ rq_dlg_param->dpte_req_per_row_ub = dpte_row_width_ub / dpte_req_width;
+ } else {
+ log2_dpte_row_height =
+ (log2_blk_width < log2_dpte_req_width) ?
+ log2_blk_width : log2_dpte_req_width;
+ dpte_row_width_ub = dml_round_to_multiple(vp_height - 1, dpte_req_height, 1)
+ + dpte_req_height;
+ rq_dlg_param->dpte_req_per_row_ub = dpte_row_width_ub / dpte_req_height;
+ }
+ }
+ if (log2_blk_bytes >= 16 && log2_vmpg_bytes == 12) // tile block >= 64KB
+ rq_dlg_param->dpte_bytes_per_row_ub = rq_dlg_param->dpte_req_per_row_ub * 128; //2*64B dpte request
+ else
+ rq_dlg_param->dpte_bytes_per_row_ub = rq_dlg_param->dpte_req_per_row_ub * 64; //64B dpte request
+
+ rq_dlg_param->dpte_row_height = 1 << log2_dpte_row_height;
+
+ // the dpte_group_bytes is reduced for the specific case of vertical
+ // access of a tile surface that has dpte request of 8x1 ptes.
+ if (!surf_linear & (log2_dpte_req_height_ptes == 0) & surf_vert) //reduced, in this case, will have page fault within a group
+ rq_sizing_param->dpte_group_bytes = 512;
+ else
+ //full size
+ rq_sizing_param->dpte_group_bytes = 2048;
+
+ //since pte request size is 64byte, the number of data pte requests per full sized group is as follows.
+ log2_dpte_group_bytes = dml_log2(rq_sizing_param->dpte_group_bytes);
+ log2_dpte_group_length = log2_dpte_group_bytes - 6; //length in 64b requests
+
+ // full sized data pte group width in elements
+ if (!surf_vert)
+ log2_dpte_group_width = log2_dpte_group_length + log2_dpte_req_width;
+ else
+ log2_dpte_group_width = log2_dpte_group_length + log2_dpte_req_height;
+
+ //But if the tile block >=64KB and the page size is 4KB, then each dPTE request is 2*64B
+ if ((log2_blk_bytes >= 16) && (log2_vmpg_bytes == 12)) // tile block >= 64KB
+ log2_dpte_group_width = log2_dpte_group_width - 1;
+
+ dpte_group_width = 1 << log2_dpte_group_width;
+
+ // since dpte groups are only aligned to dpte_req_width and not dpte_group_width,
+ // the upper bound for the dpte groups per row is as follows.
+ rq_dlg_param->dpte_groups_per_row_ub = dml_ceil((double) dpte_row_width_ub / dpte_group_width,
+ 1);
+}
+
+static void get_surf_rq_param(struct display_mode_lib *mode_lib,
+ display_data_rq_sizing_params_st *rq_sizing_param,
+ display_data_rq_dlg_params_st *rq_dlg_param,
+ display_data_rq_misc_params_st *rq_misc_param,
+ const display_pipe_source_params_st pipe_src_param,
+ bool is_chroma)
+{
+ bool mode_422 = 0;
+ unsigned int vp_width = 0;
+ unsigned int vp_height = 0;
+ unsigned int data_pitch = 0;
+ unsigned int meta_pitch = 0;
+ unsigned int ppe = mode_422 ? 2 : 1;
+
+ // FIXME check if ppe apply for both luma and chroma in 422 case
+ if (is_chroma) {
+ vp_width = pipe_src_param.viewport_width_c / ppe;
+ vp_height = pipe_src_param.viewport_height_c;
+ data_pitch = pipe_src_param.data_pitch_c;
+ meta_pitch = pipe_src_param.meta_pitch_c;
+ } else {
+ vp_width = pipe_src_param.viewport_width / ppe;
+ vp_height = pipe_src_param.viewport_height;
+ data_pitch = pipe_src_param.data_pitch;
+ meta_pitch = pipe_src_param.meta_pitch;
+ }
+
+ rq_sizing_param->chunk_bytes = 8192;
+
+ if (rq_sizing_param->chunk_bytes == 64 * 1024)
+ rq_sizing_param->min_chunk_bytes = 0;
+ else
+ rq_sizing_param->min_chunk_bytes = 1024;
+
+ rq_sizing_param->meta_chunk_bytes = 2048;
+ rq_sizing_param->min_meta_chunk_bytes = 256;
+
+ rq_sizing_param->mpte_group_bytes = 2048;
+
+ get_meta_and_pte_attr(mode_lib,
+ rq_dlg_param,
+ rq_misc_param,
+ rq_sizing_param,
+ vp_width,
+ vp_height,
+ data_pitch,
+ meta_pitch,
+ pipe_src_param.source_format,
+ pipe_src_param.sw_mode,
+ pipe_src_param.macro_tile_size,
+ pipe_src_param.source_scan,
+ is_chroma);
+}
+
+static void dml20_rq_dlg_get_rq_params(struct display_mode_lib *mode_lib,
+ display_rq_params_st *rq_param,
+ const display_pipe_source_params_st pipe_src_param)
+{
+ // get param for luma surface
+ rq_param->yuv420 = pipe_src_param.source_format == dm_420_8
+ || pipe_src_param.source_format == dm_420_10;
+ rq_param->yuv420_10bpc = pipe_src_param.source_format == dm_420_10;
+
+ get_surf_rq_param(mode_lib,
+ &(rq_param->sizing.rq_l),
+ &(rq_param->dlg.rq_l),
+ &(rq_param->misc.rq_l),
+ pipe_src_param,
+ 0);
+
+ if (is_dual_plane((enum source_format_class)(pipe_src_param.source_format))) {
+ // get param for chroma surface
+ get_surf_rq_param(mode_lib,
+ &(rq_param->sizing.rq_c),
+ &(rq_param->dlg.rq_c),
+ &(rq_param->misc.rq_c),
+ pipe_src_param,
+ 1);
+ }
+
+ // calculate how to split the det buffer space between luma and chroma
+ handle_det_buf_split(mode_lib, rq_param, pipe_src_param);
+ print__rq_params_st(mode_lib, *rq_param);
+}
+
+void dml20_rq_dlg_get_rq_reg(struct display_mode_lib *mode_lib,
+ display_rq_regs_st *rq_regs,
+ const display_pipe_params_st pipe_param)
+{
+ display_rq_params_st rq_param = {0};
+
+ memset(rq_regs, 0, sizeof(*rq_regs));
+ dml20_rq_dlg_get_rq_params(mode_lib, &rq_param, pipe_param.src);
+ extract_rq_regs(mode_lib, rq_regs, rq_param);
+
+ print__rq_regs_st(mode_lib, *rq_regs);
+}
+
+// Note: currently taken in as is.
+// Nice to decouple code from hw register implement and extract code that are repeated for luma and chroma.
+static void dml20_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
+ const display_e2e_pipe_params_st *e2e_pipe_param,
+ const unsigned int num_pipes,
+ const unsigned int pipe_idx,
+ display_dlg_regs_st *disp_dlg_regs,
+ display_ttu_regs_st *disp_ttu_regs,
+ const display_rq_dlg_params_st rq_dlg_param,
+ const display_dlg_sys_params_st dlg_sys_param,
+ const bool cstate_en,
+ const bool pstate_en)
+{
+ const display_pipe_source_params_st *src = &e2e_pipe_param[pipe_idx].pipe.src;
+ const display_pipe_dest_params_st *dst = &e2e_pipe_param[pipe_idx].pipe.dest;
+ const display_output_params_st *dout = &e2e_pipe_param[pipe_idx].dout;
+ const display_clocks_and_cfg_st *clks = &e2e_pipe_param[pipe_idx].clks_cfg;
+ const scaler_ratio_depth_st *scl = &e2e_pipe_param[pipe_idx].pipe.scale_ratio_depth;
+ const scaler_taps_st *taps = &e2e_pipe_param[pipe_idx].pipe.scale_taps;
+
+ // -------------------------
+ // Section 1.15.2.1: OTG dependent Params
+ // -------------------------
+ // Timing
+ unsigned int htotal = dst->htotal;
+// unsigned int hblank_start = dst.hblank_start; // TODO: Remove
+ unsigned int hblank_end = dst->hblank_end;
+ unsigned int vblank_start = dst->vblank_start;
+ unsigned int vblank_end = dst->vblank_end;
+ unsigned int min_vblank = mode_lib->ip.min_vblank_lines;
+
+ double dppclk_freq_in_mhz = clks->dppclk_mhz;
+ double dispclk_freq_in_mhz = clks->dispclk_mhz;
+ double refclk_freq_in_mhz = clks->refclk_mhz;
+ double pclk_freq_in_mhz = dst->pixel_rate_mhz;
+ bool interlaced = dst->interlaced;
+
+ double ref_freq_to_pix_freq = refclk_freq_in_mhz / pclk_freq_in_mhz;
+
+ double min_dcfclk_mhz;
+ double t_calc_us;
+ double min_ttu_vblank;
+
+ double min_dst_y_ttu_vblank;
+ unsigned int dlg_vblank_start;
+ bool dual_plane;
+ bool mode_422;
+ unsigned int access_dir;
+ unsigned int vp_height_l;
+ unsigned int vp_width_l;
+ unsigned int vp_height_c;
+ unsigned int vp_width_c;
+
+ // Scaling
+ unsigned int htaps_l;
+ unsigned int htaps_c;
+ double hratio_l;
+ double hratio_c;
+ double vratio_l;
+ double vratio_c;
+ bool scl_enable;
+
+ double line_time_in_us;
+ // double vinit_l;
+ // double vinit_c;
+ // double vinit_bot_l;
+ // double vinit_bot_c;
+
+ // unsigned int swath_height_l;
+ unsigned int swath_width_ub_l;
+ // unsigned int dpte_bytes_per_row_ub_l;
+ unsigned int dpte_groups_per_row_ub_l;
+ // unsigned int meta_pte_bytes_per_frame_ub_l;
+ // unsigned int meta_bytes_per_row_ub_l;
+
+ // unsigned int swath_height_c;
+ unsigned int swath_width_ub_c;
+ // unsigned int dpte_bytes_per_row_ub_c;
+ unsigned int dpte_groups_per_row_ub_c;
+
+ unsigned int meta_chunks_per_row_ub_l;
+ unsigned int meta_chunks_per_row_ub_c;
+ unsigned int vupdate_offset;
+ unsigned int vupdate_width;
+ unsigned int vready_offset;
+
+ unsigned int dppclk_delay_subtotal;
+ unsigned int dispclk_delay_subtotal;
+ unsigned int pixel_rate_delay_subtotal;
+
+ unsigned int vstartup_start;
+ unsigned int dst_x_after_scaler;
+ unsigned int dst_y_after_scaler;
+ double line_wait;
+ double dst_y_prefetch;
+ double dst_y_per_vm_vblank;
+ double dst_y_per_row_vblank;
+ double dst_y_per_vm_flip;
+ double dst_y_per_row_flip;
+ double min_dst_y_per_vm_vblank;
+ double min_dst_y_per_row_vblank;
+ double lsw;
+ double vratio_pre_l;
+ double vratio_pre_c;
+ unsigned int req_per_swath_ub_l;
+ unsigned int req_per_swath_ub_c;
+ unsigned int meta_row_height_l;
+ unsigned int meta_row_height_c;
+ unsigned int swath_width_pixels_ub_l;
+ unsigned int swath_width_pixels_ub_c;
+ unsigned int scaler_rec_in_width_l;
+ unsigned int scaler_rec_in_width_c;
+ unsigned int dpte_row_height_l;
+ unsigned int dpte_row_height_c;
+ double hscale_pixel_rate_l;
+ double hscale_pixel_rate_c;
+ double min_hratio_fact_l;
+ double min_hratio_fact_c;
+ double refcyc_per_line_delivery_pre_l;
+ double refcyc_per_line_delivery_pre_c;
+ double refcyc_per_line_delivery_l;
+ double refcyc_per_line_delivery_c;
+
+ double refcyc_per_req_delivery_pre_l;
+ double refcyc_per_req_delivery_pre_c;
+ double refcyc_per_req_delivery_l;
+ double refcyc_per_req_delivery_c;
+
+ unsigned int full_recout_width;
+ double xfc_transfer_delay;
+ double xfc_precharge_delay;
+ double xfc_remote_surface_flip_latency;
+ double xfc_dst_y_delta_drq_limit;
+ double xfc_prefetch_margin;
+ double refcyc_per_req_delivery_pre_cur0;
+ double refcyc_per_req_delivery_cur0;
+ double refcyc_per_req_delivery_pre_cur1;
+ double refcyc_per_req_delivery_cur1;
+
+ memset(disp_dlg_regs, 0, sizeof(*disp_dlg_regs));
+ memset(disp_ttu_regs, 0, sizeof(*disp_ttu_regs));
+
+ dml_print("DML_DLG: %s: cstate_en = %d\n", __func__, cstate_en);
+ dml_print("DML_DLG: %s: pstate_en = %d\n", __func__, pstate_en);
+
+ dml_print("DML_DLG: %s: dppclk_freq_in_mhz = %3.2f\n", __func__, dppclk_freq_in_mhz);
+ dml_print("DML_DLG: %s: dispclk_freq_in_mhz = %3.2f\n", __func__, dispclk_freq_in_mhz);
+ dml_print("DML_DLG: %s: refclk_freq_in_mhz = %3.2f\n", __func__, refclk_freq_in_mhz);
+ dml_print("DML_DLG: %s: pclk_freq_in_mhz = %3.2f\n", __func__, pclk_freq_in_mhz);
+ dml_print("DML_DLG: %s: interlaced = %d\n", __func__, interlaced);
+ ASSERT(ref_freq_to_pix_freq < 4.0);
+
+ disp_dlg_regs->ref_freq_to_pix_freq =
+ (unsigned int) (ref_freq_to_pix_freq * dml_pow(2, 19));
+ disp_dlg_regs->refcyc_per_htotal = (unsigned int) (ref_freq_to_pix_freq * (double) htotal
+ * dml_pow(2, 8));
+ disp_dlg_regs->dlg_vblank_end = interlaced ? (vblank_end / 2) : vblank_end; // 15 bits
+ disp_dlg_regs->refcyc_h_blank_end = (unsigned int) ((double) hblank_end
+ * (double) ref_freq_to_pix_freq);
+ ASSERT(disp_dlg_regs->refcyc_h_blank_end < (unsigned int) dml_pow(2, 13));
+
+ min_dcfclk_mhz = dlg_sys_param.deepsleep_dcfclk_mhz;
+ t_calc_us = get_tcalc(mode_lib, e2e_pipe_param, num_pipes);
+ min_ttu_vblank = get_min_ttu_vblank(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+
+ min_dst_y_ttu_vblank = min_ttu_vblank * pclk_freq_in_mhz / (double) htotal;
+ dlg_vblank_start = interlaced ? (vblank_start / 2) : vblank_start;
+
+ disp_dlg_regs->min_dst_y_next_start = (unsigned int) (((double) dlg_vblank_start
+ + min_dst_y_ttu_vblank) * dml_pow(2, 2));
+ ASSERT(disp_dlg_regs->min_dst_y_next_start < (unsigned int) dml_pow(2, 18));
+
+ dml_print("DML_DLG: %s: min_dcfclk_mhz = %3.2f\n",
+ __func__,
+ min_dcfclk_mhz);
+ dml_print("DML_DLG: %s: min_ttu_vblank = %3.2f\n",
+ __func__,
+ min_ttu_vblank);
+ dml_print("DML_DLG: %s: min_dst_y_ttu_vblank = %3.2f\n",
+ __func__,
+ min_dst_y_ttu_vblank);
+ dml_print("DML_DLG: %s: t_calc_us = %3.2f\n",
+ __func__,
+ t_calc_us);
+ dml_print("DML_DLG: %s: disp_dlg_regs->min_dst_y_next_start = 0x%0x\n",
+ __func__,
+ disp_dlg_regs->min_dst_y_next_start);
+ dml_print("DML_DLG: %s: ref_freq_to_pix_freq = %3.2f\n",
+ __func__,
+ ref_freq_to_pix_freq);
+
+ // -------------------------
+ // Section 1.15.2.2: Prefetch, Active and TTU
+ // -------------------------
+ // Prefetch Calc
+ // Source
+// dcc_en = src.dcc;
+ dual_plane = is_dual_plane((enum source_format_class)(src->source_format));
+ mode_422 = 0; // FIXME
+ access_dir = (src->source_scan == dm_vert); // vp access direction: horizontal or vertical accessed
+// bytes_per_element_l = get_bytes_per_element(source_format_class(src.source_format), 0);
+// bytes_per_element_c = get_bytes_per_element(source_format_class(src.source_format), 1);
+ vp_height_l = src->viewport_height;
+ vp_width_l = src->viewport_width;
+ vp_height_c = src->viewport_height_c;
+ vp_width_c = src->viewport_width_c;
+
+ // Scaling
+ htaps_l = taps->htaps;
+ htaps_c = taps->htaps_c;
+ hratio_l = scl->hscl_ratio;
+ hratio_c = scl->hscl_ratio_c;
+ vratio_l = scl->vscl_ratio;
+ vratio_c = scl->vscl_ratio_c;
+ scl_enable = scl->scl_enable;
+
+ line_time_in_us = (htotal / pclk_freq_in_mhz);
+// vinit_l = scl.vinit;
+// vinit_c = scl.vinit_c;
+// vinit_bot_l = scl.vinit_bot;
+// vinit_bot_c = scl.vinit_bot_c;
+
+// unsigned int swath_height_l = rq_dlg_param.rq_l.swath_height;
+ swath_width_ub_l = rq_dlg_param.rq_l.swath_width_ub;
+// unsigned int dpte_bytes_per_row_ub_l = rq_dlg_param.rq_l.dpte_bytes_per_row_ub;
+ dpte_groups_per_row_ub_l = rq_dlg_param.rq_l.dpte_groups_per_row_ub;
+// unsigned int meta_pte_bytes_per_frame_ub_l = rq_dlg_param.rq_l.meta_pte_bytes_per_frame_ub;
+// unsigned int meta_bytes_per_row_ub_l = rq_dlg_param.rq_l.meta_bytes_per_row_ub;
+
+// unsigned int swath_height_c = rq_dlg_param.rq_c.swath_height;
+ swath_width_ub_c = rq_dlg_param.rq_c.swath_width_ub;
+ // dpte_bytes_per_row_ub_c = rq_dlg_param.rq_c.dpte_bytes_per_row_ub;
+ dpte_groups_per_row_ub_c = rq_dlg_param.rq_c.dpte_groups_per_row_ub;
+
+ meta_chunks_per_row_ub_l = rq_dlg_param.rq_l.meta_chunks_per_row_ub;
+ meta_chunks_per_row_ub_c = rq_dlg_param.rq_c.meta_chunks_per_row_ub;
+ vupdate_offset = dst->vupdate_offset;
+ vupdate_width = dst->vupdate_width;
+ vready_offset = dst->vready_offset;
+
+ dppclk_delay_subtotal = mode_lib->ip.dppclk_delay_subtotal;
+ dispclk_delay_subtotal = mode_lib->ip.dispclk_delay_subtotal;
+
+ if (scl_enable)
+ dppclk_delay_subtotal += mode_lib->ip.dppclk_delay_scl;
+ else
+ dppclk_delay_subtotal += mode_lib->ip.dppclk_delay_scl_lb_only;
+
+ dppclk_delay_subtotal += mode_lib->ip.dppclk_delay_cnvc_formatter
+ + src->num_cursors * mode_lib->ip.dppclk_delay_cnvc_cursor;
+
+ if (dout->dsc_enable) {
+ double dsc_delay = get_dsc_delay(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+
+ dispclk_delay_subtotal += dsc_delay;
+ }
+
+ pixel_rate_delay_subtotal = dppclk_delay_subtotal * pclk_freq_in_mhz / dppclk_freq_in_mhz
+ + dispclk_delay_subtotal * pclk_freq_in_mhz / dispclk_freq_in_mhz;
+
+ vstartup_start = dst->vstartup_start;
+ if (interlaced) {
+ if (vstartup_start / 2.0
+ - (double) (vready_offset + vupdate_width + vupdate_offset) / htotal
+ <= vblank_end / 2.0)
+ disp_dlg_regs->vready_after_vcount0 = 1;
+ else
+ disp_dlg_regs->vready_after_vcount0 = 0;
+ } else {
+ if (vstartup_start
+ - (double) (vready_offset + vupdate_width + vupdate_offset) / htotal
+ <= vblank_end)
+ disp_dlg_regs->vready_after_vcount0 = 1;
+ else
+ disp_dlg_regs->vready_after_vcount0 = 0;
+ }
+
+ // TODO: Where is this coming from?
+ if (interlaced)
+ vstartup_start = vstartup_start / 2;
+
+ // TODO: What if this min_vblank doesn't match the value in the dml_config_settings.cpp?
+ if (vstartup_start >= min_vblank) {
+ dml_print("WARNING: DML_DLG: %s: vblank_start=%d vblank_end=%d\n",
+ __func__,
+ vblank_start,
+ vblank_end);
+ dml_print("WARNING: DML_DLG: %s: vstartup_start=%d should be less than min_vblank=%d\n",
+ __func__,
+ vstartup_start,
+ min_vblank);
+ min_vblank = vstartup_start + 1;
+ dml_print("WARNING: DML_DLG: %s: vstartup_start=%d should be less than min_vblank=%d\n",
+ __func__,
+ vstartup_start,
+ min_vblank);
+ }
+
+ dst_x_after_scaler = get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+ dst_y_after_scaler = get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+
+ dml_print("DML_DLG: %s: htotal = %d\n", __func__, htotal);
+ dml_print("DML_DLG: %s: pixel_rate_delay_subtotal = %d\n",
+ __func__,
+ pixel_rate_delay_subtotal);
+ dml_print("DML_DLG: %s: dst_x_after_scaler = %d\n",
+ __func__,
+ dst_x_after_scaler);
+ dml_print("DML_DLG: %s: dst_y_after_scaler = %d\n",
+ __func__,
+ dst_y_after_scaler);
+
+ // Lwait
+ line_wait = mode_lib->soc.urgent_latency_us;
+ if (cstate_en)
+ line_wait = dml_max(mode_lib->soc.sr_enter_plus_exit_time_us, line_wait);
+ if (pstate_en)
+ line_wait = dml_max(mode_lib->soc.dram_clock_change_latency_us
+ + mode_lib->soc.urgent_latency_us,
+ line_wait);
+ line_wait = line_wait / line_time_in_us;
+
+ dst_y_prefetch = get_dst_y_prefetch(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+ dml_print("DML_DLG: %s: dst_y_prefetch (after rnd) = %3.2f\n", __func__, dst_y_prefetch);
+
+ dst_y_per_vm_vblank = get_dst_y_per_vm_vblank(mode_lib,
+ e2e_pipe_param,
+ num_pipes,
+ pipe_idx);
+ dst_y_per_row_vblank = get_dst_y_per_row_vblank(mode_lib,
+ e2e_pipe_param,
+ num_pipes,
+ pipe_idx);
+ dst_y_per_vm_flip = get_dst_y_per_vm_flip(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+ dst_y_per_row_flip = get_dst_y_per_row_flip(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+
+ min_dst_y_per_vm_vblank = 8.0;
+ min_dst_y_per_row_vblank = 16.0;
+
+ // magic!
+ if (htotal <= 75) {
+ min_vblank = 300;
+ min_dst_y_per_vm_vblank = 100.0;
+ min_dst_y_per_row_vblank = 100.0;
+ }
+
+ dml_print("DML_DLG: %s: dst_y_per_vm_vblank = %3.2f\n", __func__, dst_y_per_vm_vblank);
+ dml_print("DML_DLG: %s: dst_y_per_row_vblank = %3.2f\n", __func__, dst_y_per_row_vblank);
+
+ ASSERT(dst_y_per_vm_vblank < min_dst_y_per_vm_vblank);
+ ASSERT(dst_y_per_row_vblank < min_dst_y_per_row_vblank);
+
+ ASSERT(dst_y_prefetch > (dst_y_per_vm_vblank + dst_y_per_row_vblank));
+ lsw = dst_y_prefetch - (dst_y_per_vm_vblank + dst_y_per_row_vblank);
+
+ dml_print("DML_DLG: %s: lsw = %3.2f\n", __func__, lsw);
+
+ vratio_pre_l = get_vratio_prefetch_l(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+ vratio_pre_c = get_vratio_prefetch_c(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+
+ dml_print("DML_DLG: %s: vratio_pre_l=%3.2f\n", __func__, vratio_pre_l);
+ dml_print("DML_DLG: %s: vratio_pre_c=%3.2f\n", __func__, vratio_pre_c);
+
+ // Active
+ req_per_swath_ub_l = rq_dlg_param.rq_l.req_per_swath_ub;
+ req_per_swath_ub_c = rq_dlg_param.rq_c.req_per_swath_ub;
+ meta_row_height_l = rq_dlg_param.rq_l.meta_row_height;
+ meta_row_height_c = rq_dlg_param.rq_c.meta_row_height;
+ swath_width_pixels_ub_l = 0;
+ swath_width_pixels_ub_c = 0;
+ scaler_rec_in_width_l = 0;
+ scaler_rec_in_width_c = 0;
+ dpte_row_height_l = rq_dlg_param.rq_l.dpte_row_height;
+ dpte_row_height_c = rq_dlg_param.rq_c.dpte_row_height;
+
+ if (mode_422) {
+ swath_width_pixels_ub_l = swath_width_ub_l * 2; // *2 for 2 pixel per element
+ swath_width_pixels_ub_c = swath_width_ub_c * 2;
+ } else {
+ swath_width_pixels_ub_l = swath_width_ub_l * 1;
+ swath_width_pixels_ub_c = swath_width_ub_c * 1;
+ }
+
+ hscale_pixel_rate_l = 0.;
+ hscale_pixel_rate_c = 0.;
+ min_hratio_fact_l = 1.0;
+ min_hratio_fact_c = 1.0;
+
+ if (htaps_l <= 1)
+ min_hratio_fact_l = 2.0;
+ else if (htaps_l <= 6) {
+ if ((hratio_l * 2.0) > 4.0)
+ min_hratio_fact_l = 4.0;
+ else
+ min_hratio_fact_l = hratio_l * 2.0;
+ } else {
+ if (hratio_l > 4.0)
+ min_hratio_fact_l = 4.0;
+ else
+ min_hratio_fact_l = hratio_l;
+ }
+
+ hscale_pixel_rate_l = min_hratio_fact_l * dppclk_freq_in_mhz;
+
+ if (htaps_c <= 1)
+ min_hratio_fact_c = 2.0;
+ else if (htaps_c <= 6) {
+ if ((hratio_c * 2.0) > 4.0)
+ min_hratio_fact_c = 4.0;
+ else
+ min_hratio_fact_c = hratio_c * 2.0;
+ } else {
+ if (hratio_c > 4.0)
+ min_hratio_fact_c = 4.0;
+ else
+ min_hratio_fact_c = hratio_c;
+ }
+
+ hscale_pixel_rate_c = min_hratio_fact_c * dppclk_freq_in_mhz;
+
+ refcyc_per_line_delivery_pre_l = 0.;
+ refcyc_per_line_delivery_pre_c = 0.;
+ refcyc_per_line_delivery_l = 0.;
+ refcyc_per_line_delivery_c = 0.;
+
+ refcyc_per_req_delivery_pre_l = 0.;
+ refcyc_per_req_delivery_pre_c = 0.;
+ refcyc_per_req_delivery_l = 0.;
+ refcyc_per_req_delivery_c = 0.;
+
+ full_recout_width = 0;
+ // In ODM
+ if (src->is_hsplit) {
+ // This "hack" is only allowed (and valid) for MPC combine. In ODM
+ // combine, you MUST specify the full_recout_width...according to Oswin
+ if (dst->full_recout_width == 0 && !dst->odm_combine) {
+ dml_print("DML_DLG: %s: Warning: full_recout_width not set in hsplit mode\n",
+ __func__);
+ full_recout_width = dst->recout_width * 2; // assume half split for dcn1
+ } else
+ full_recout_width = dst->full_recout_width;
+ } else
+ full_recout_width = dst->recout_width;
+
+ // As of DCN2, mpc_combine and odm_combine are mutually exclusive
+ refcyc_per_line_delivery_pre_l = get_refcyc_per_delivery(mode_lib,
+ refclk_freq_in_mhz,
+ pclk_freq_in_mhz,
+ dst->odm_combine,
+ full_recout_width,
+ dst->hactive,
+ vratio_pre_l,
+ hscale_pixel_rate_l,
+ swath_width_pixels_ub_l,
+ 1); // per line
+
+ refcyc_per_line_delivery_l = get_refcyc_per_delivery(mode_lib,
+ refclk_freq_in_mhz,
+ pclk_freq_in_mhz,
+ dst->odm_combine,
+ full_recout_width,
+ dst->hactive,
+ vratio_l,
+ hscale_pixel_rate_l,
+ swath_width_pixels_ub_l,
+ 1); // per line
+
+ dml_print("DML_DLG: %s: full_recout_width = %d\n",
+ __func__,
+ full_recout_width);
+ dml_print("DML_DLG: %s: hscale_pixel_rate_l = %3.2f\n",
+ __func__,
+ hscale_pixel_rate_l);
+ dml_print("DML_DLG: %s: refcyc_per_line_delivery_pre_l = %3.2f\n",
+ __func__,
+ refcyc_per_line_delivery_pre_l);
+ dml_print("DML_DLG: %s: refcyc_per_line_delivery_l = %3.2f\n",
+ __func__,
+ refcyc_per_line_delivery_l);
+
+ if (dual_plane) {
+ refcyc_per_line_delivery_pre_c = get_refcyc_per_delivery(mode_lib,
+ refclk_freq_in_mhz,
+ pclk_freq_in_mhz,
+ dst->odm_combine,
+ full_recout_width,
+ dst->hactive,
+ vratio_pre_c,
+ hscale_pixel_rate_c,
+ swath_width_pixels_ub_c,
+ 1); // per line
+
+ refcyc_per_line_delivery_c = get_refcyc_per_delivery(mode_lib,
+ refclk_freq_in_mhz,
+ pclk_freq_in_mhz,
+ dst->odm_combine,
+ full_recout_width,
+ dst->hactive,
+ vratio_c,
+ hscale_pixel_rate_c,
+ swath_width_pixels_ub_c,
+ 1); // per line
+
+ dml_print("DML_DLG: %s: refcyc_per_line_delivery_pre_c = %3.2f\n",
+ __func__,
+ refcyc_per_line_delivery_pre_c);
+ dml_print("DML_DLG: %s: refcyc_per_line_delivery_c = %3.2f\n",
+ __func__,
+ refcyc_per_line_delivery_c);
+ }
+
+ // TTU - Luma / Chroma
+ if (access_dir) { // vertical access
+ scaler_rec_in_width_l = vp_height_l;
+ scaler_rec_in_width_c = vp_height_c;
+ } else {
+ scaler_rec_in_width_l = vp_width_l;
+ scaler_rec_in_width_c = vp_width_c;
+ }
+
+ refcyc_per_req_delivery_pre_l = get_refcyc_per_delivery(mode_lib,
+ refclk_freq_in_mhz,
+ pclk_freq_in_mhz,
+ dst->odm_combine,
+ full_recout_width,
+ dst->hactive,
+ vratio_pre_l,
+ hscale_pixel_rate_l,
+ scaler_rec_in_width_l,
+ req_per_swath_ub_l); // per req
+ refcyc_per_req_delivery_l = get_refcyc_per_delivery(mode_lib,
+ refclk_freq_in_mhz,
+ pclk_freq_in_mhz,
+ dst->odm_combine,
+ full_recout_width,
+ dst->hactive,
+ vratio_l,
+ hscale_pixel_rate_l,
+ scaler_rec_in_width_l,
+ req_per_swath_ub_l); // per req
+
+ dml_print("DML_DLG: %s: refcyc_per_req_delivery_pre_l = %3.2f\n",
+ __func__,
+ refcyc_per_req_delivery_pre_l);
+ dml_print("DML_DLG: %s: refcyc_per_req_delivery_l = %3.2f\n",
+ __func__,
+ refcyc_per_req_delivery_l);
+
+ ASSERT(refcyc_per_req_delivery_pre_l < dml_pow(2, 13));
+ ASSERT(refcyc_per_req_delivery_l < dml_pow(2, 13));
+
+ if (dual_plane) {
+ refcyc_per_req_delivery_pre_c = get_refcyc_per_delivery(mode_lib,
+ refclk_freq_in_mhz,
+ pclk_freq_in_mhz,
+ dst->odm_combine,
+ full_recout_width,
+ dst->hactive,
+ vratio_pre_c,
+ hscale_pixel_rate_c,
+ scaler_rec_in_width_c,
+ req_per_swath_ub_c); // per req
+ refcyc_per_req_delivery_c = get_refcyc_per_delivery(mode_lib,
+ refclk_freq_in_mhz,
+ pclk_freq_in_mhz,
+ dst->odm_combine,
+ full_recout_width,
+ dst->hactive,
+ vratio_c,
+ hscale_pixel_rate_c,
+ scaler_rec_in_width_c,
+ req_per_swath_ub_c); // per req
+
+ dml_print("DML_DLG: %s: refcyc_per_req_delivery_pre_c = %3.2f\n",
+ __func__,
+ refcyc_per_req_delivery_pre_c);
+ dml_print("DML_DLG: %s: refcyc_per_req_delivery_c = %3.2f\n",
+ __func__,
+ refcyc_per_req_delivery_c);
+
+ ASSERT(refcyc_per_req_delivery_pre_c < dml_pow(2, 13));
+ ASSERT(refcyc_per_req_delivery_c < dml_pow(2, 13));
+ }
+
+ // XFC
+ xfc_transfer_delay = get_xfc_transfer_delay(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+ xfc_precharge_delay = get_xfc_precharge_delay(mode_lib,
+ e2e_pipe_param,
+ num_pipes,
+ pipe_idx);
+ xfc_remote_surface_flip_latency = get_xfc_remote_surface_flip_latency(mode_lib,
+ e2e_pipe_param,
+ num_pipes,
+ pipe_idx);
+ xfc_dst_y_delta_drq_limit = xfc_remote_surface_flip_latency;
+ xfc_prefetch_margin = get_xfc_prefetch_margin(mode_lib,
+ e2e_pipe_param,
+ num_pipes,
+ pipe_idx);
+
+ // TTU - Cursor
+ refcyc_per_req_delivery_pre_cur0 = 0.0;
+ refcyc_per_req_delivery_cur0 = 0.0;
+ if (src->num_cursors > 0) {
+ calculate_ttu_cursor(mode_lib,
+ &refcyc_per_req_delivery_pre_cur0,
+ &refcyc_per_req_delivery_cur0,
+ refclk_freq_in_mhz,
+ ref_freq_to_pix_freq,
+ hscale_pixel_rate_l,
+ scl->hscl_ratio,
+ vratio_pre_l,
+ vratio_l,
+ src->cur0_src_width,
+ (enum cursor_bpp)(src->cur0_bpp));
+ }
+
+ refcyc_per_req_delivery_pre_cur1 = 0.0;
+ refcyc_per_req_delivery_cur1 = 0.0;
+ if (src->num_cursors > 1) {
+ calculate_ttu_cursor(mode_lib,
+ &refcyc_per_req_delivery_pre_cur1,
+ &refcyc_per_req_delivery_cur1,
+ refclk_freq_in_mhz,
+ ref_freq_to_pix_freq,
+ hscale_pixel_rate_l,
+ scl->hscl_ratio,
+ vratio_pre_l,
+ vratio_l,
+ src->cur1_src_width,
+ (enum cursor_bpp)(src->cur1_bpp));
+ }
+
+ // TTU - Misc
+ // all hard-coded
+
+ // Assignment to register structures
+ disp_dlg_regs->dst_y_after_scaler = dst_y_after_scaler; // in terms of line
+ disp_dlg_regs->refcyc_x_after_scaler = dst_x_after_scaler * ref_freq_to_pix_freq; // in terms of refclk
+ ASSERT(disp_dlg_regs->refcyc_x_after_scaler < (unsigned int) dml_pow(2, 13));
+ disp_dlg_regs->dst_y_prefetch = (unsigned int) (dst_y_prefetch * dml_pow(2, 2));
+ disp_dlg_regs->dst_y_per_vm_vblank = (unsigned int) (dst_y_per_vm_vblank * dml_pow(2, 2));
+ disp_dlg_regs->dst_y_per_row_vblank = (unsigned int) (dst_y_per_row_vblank * dml_pow(2, 2));
+ disp_dlg_regs->dst_y_per_vm_flip = (unsigned int) (dst_y_per_vm_flip * dml_pow(2, 2));
+ disp_dlg_regs->dst_y_per_row_flip = (unsigned int) (dst_y_per_row_flip * dml_pow(2, 2));
+
+ disp_dlg_regs->vratio_prefetch = (unsigned int) (vratio_pre_l * dml_pow(2, 19));
+ disp_dlg_regs->vratio_prefetch_c = (unsigned int) (vratio_pre_c * dml_pow(2, 19));
+
+ disp_dlg_regs->refcyc_per_pte_group_vblank_l =
+ (unsigned int) (dst_y_per_row_vblank * (double) htotal
+ * ref_freq_to_pix_freq / (double) dpte_groups_per_row_ub_l);
+ ASSERT(disp_dlg_regs->refcyc_per_pte_group_vblank_l < (unsigned int) dml_pow(2, 13));
+
+ if (dual_plane) {
+ disp_dlg_regs->refcyc_per_pte_group_vblank_c = (unsigned int) (dst_y_per_row_vblank
+ * (double) htotal * ref_freq_to_pix_freq
+ / (double) dpte_groups_per_row_ub_c);
+ ASSERT(disp_dlg_regs->refcyc_per_pte_group_vblank_c
+ < (unsigned int) dml_pow(2, 13));
+ }
+
+ disp_dlg_regs->refcyc_per_meta_chunk_vblank_l =
+ (unsigned int) (dst_y_per_row_vblank * (double) htotal
+ * ref_freq_to_pix_freq / (double) meta_chunks_per_row_ub_l);
+ ASSERT(disp_dlg_regs->refcyc_per_meta_chunk_vblank_l < (unsigned int) dml_pow(2, 13));
+
+ disp_dlg_regs->refcyc_per_meta_chunk_vblank_c =
+ disp_dlg_regs->refcyc_per_meta_chunk_vblank_l; // dcc for 4:2:0 is not supported in dcn1.0. assigned to be the same as _l for now
+
+ disp_dlg_regs->refcyc_per_pte_group_flip_l = (unsigned int) (dst_y_per_row_flip * htotal
+ * ref_freq_to_pix_freq) / dpte_groups_per_row_ub_l;
+ disp_dlg_regs->refcyc_per_meta_chunk_flip_l = (unsigned int) (dst_y_per_row_flip * htotal
+ * ref_freq_to_pix_freq) / meta_chunks_per_row_ub_l;
+
+ if (dual_plane) {
+ disp_dlg_regs->refcyc_per_pte_group_flip_c = (unsigned int) (dst_y_per_row_flip
+ * htotal * ref_freq_to_pix_freq) / dpte_groups_per_row_ub_c;
+ disp_dlg_regs->refcyc_per_meta_chunk_flip_c = (unsigned int) (dst_y_per_row_flip
+ * htotal * ref_freq_to_pix_freq) / meta_chunks_per_row_ub_c;
+ }
+
+ disp_dlg_regs->dst_y_per_pte_row_nom_l = (unsigned int) ((double) dpte_row_height_l
+ / (double) vratio_l * dml_pow(2, 2));
+ ASSERT(disp_dlg_regs->dst_y_per_pte_row_nom_l < (unsigned int) dml_pow(2, 17));
+
+ if (dual_plane) {
+ disp_dlg_regs->dst_y_per_pte_row_nom_c = (unsigned int) ((double) dpte_row_height_c
+ / (double) vratio_c * dml_pow(2, 2));
+ if (disp_dlg_regs->dst_y_per_pte_row_nom_c >= (unsigned int) dml_pow(2, 17)) {
+ dml_print("DML_DLG: %s: Warning dst_y_per_pte_row_nom_c %u larger than supported by register format U15.2 %u\n",
+ __func__,
+ disp_dlg_regs->dst_y_per_pte_row_nom_c,
+ (unsigned int) dml_pow(2, 17) - 1);
+ }
+ }
+
+ disp_dlg_regs->dst_y_per_meta_row_nom_l = (unsigned int) ((double) meta_row_height_l
+ / (double) vratio_l * dml_pow(2, 2));
+ ASSERT(disp_dlg_regs->dst_y_per_meta_row_nom_l < (unsigned int) dml_pow(2, 17));
+
+ disp_dlg_regs->dst_y_per_meta_row_nom_c = disp_dlg_regs->dst_y_per_meta_row_nom_l; // TODO: dcc for 4:2:0 is not supported in dcn1.0. assigned to be the same as _l for now
+
+ disp_dlg_regs->refcyc_per_pte_group_nom_l = (unsigned int) ((double) dpte_row_height_l
+ / (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
+ / (double) dpte_groups_per_row_ub_l);
+ if (disp_dlg_regs->refcyc_per_pte_group_nom_l >= (unsigned int) dml_pow(2, 23))
+ disp_dlg_regs->refcyc_per_pte_group_nom_l = dml_pow(2, 23) - 1;
+ disp_dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int) ((double) meta_row_height_l
+ / (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
+ / (double) meta_chunks_per_row_ub_l);
+ if (disp_dlg_regs->refcyc_per_meta_chunk_nom_l >= (unsigned int) dml_pow(2, 23))
+ disp_dlg_regs->refcyc_per_meta_chunk_nom_l = dml_pow(2, 23) - 1;
+
+ if (dual_plane) {
+ disp_dlg_regs->refcyc_per_pte_group_nom_c =
+ (unsigned int) ((double) dpte_row_height_c / (double) vratio_c
+ * (double) htotal * ref_freq_to_pix_freq
+ / (double) dpte_groups_per_row_ub_c);
+ if (disp_dlg_regs->refcyc_per_pte_group_nom_c >= (unsigned int) dml_pow(2, 23))
+ disp_dlg_regs->refcyc_per_pte_group_nom_c = dml_pow(2, 23) - 1;
+
+ // TODO: Is this the right calculation? Does htotal need to be halved?
+ disp_dlg_regs->refcyc_per_meta_chunk_nom_c =
+ (unsigned int) ((double) meta_row_height_c / (double) vratio_c
+ * (double) htotal * ref_freq_to_pix_freq
+ / (double) meta_chunks_per_row_ub_c);
+ if (disp_dlg_regs->refcyc_per_meta_chunk_nom_c >= (unsigned int) dml_pow(2, 23))
+ disp_dlg_regs->refcyc_per_meta_chunk_nom_c = dml_pow(2, 23) - 1;
+ }
+
+ disp_dlg_regs->refcyc_per_line_delivery_pre_l = (unsigned int) dml_floor(refcyc_per_line_delivery_pre_l,
+ 1);
+ disp_dlg_regs->refcyc_per_line_delivery_l = (unsigned int) dml_floor(refcyc_per_line_delivery_l,
+ 1);
+ ASSERT(disp_dlg_regs->refcyc_per_line_delivery_pre_l < (unsigned int) dml_pow(2, 13));
+ ASSERT(disp_dlg_regs->refcyc_per_line_delivery_l < (unsigned int) dml_pow(2, 13));
+
+ disp_dlg_regs->refcyc_per_line_delivery_pre_c = (unsigned int) dml_floor(refcyc_per_line_delivery_pre_c,
+ 1);
+ disp_dlg_regs->refcyc_per_line_delivery_c = (unsigned int) dml_floor(refcyc_per_line_delivery_c,
+ 1);
+ ASSERT(disp_dlg_regs->refcyc_per_line_delivery_pre_c < (unsigned int) dml_pow(2, 13));
+ ASSERT(disp_dlg_regs->refcyc_per_line_delivery_c < (unsigned int) dml_pow(2, 13));
+
+ disp_dlg_regs->chunk_hdl_adjust_cur0 = 3;
+ disp_dlg_regs->dst_y_offset_cur0 = 0;
+ disp_dlg_regs->chunk_hdl_adjust_cur1 = 3;
+ disp_dlg_regs->dst_y_offset_cur1 = 0;
+
+ disp_dlg_regs->xfc_reg_transfer_delay = xfc_transfer_delay;
+ disp_dlg_regs->xfc_reg_precharge_delay = xfc_precharge_delay;
+ disp_dlg_regs->xfc_reg_remote_surface_flip_latency = xfc_remote_surface_flip_latency;
+ disp_dlg_regs->xfc_reg_prefetch_margin = dml_ceil(xfc_prefetch_margin * refclk_freq_in_mhz,
+ 1);
+
+ // slave has to have this value also set to off
+ if (src->xfc_enable && !src->xfc_slave)
+ disp_dlg_regs->dst_y_delta_drq_limit = dml_ceil(xfc_dst_y_delta_drq_limit, 1);
+ else
+ disp_dlg_regs->dst_y_delta_drq_limit = 0x7fff; // off
+
+ disp_ttu_regs->refcyc_per_req_delivery_pre_l = (unsigned int) (refcyc_per_req_delivery_pre_l
+ * dml_pow(2, 10));
+ disp_ttu_regs->refcyc_per_req_delivery_l = (unsigned int) (refcyc_per_req_delivery_l
+ * dml_pow(2, 10));
+ disp_ttu_regs->refcyc_per_req_delivery_pre_c = (unsigned int) (refcyc_per_req_delivery_pre_c
+ * dml_pow(2, 10));
+ disp_ttu_regs->refcyc_per_req_delivery_c = (unsigned int) (refcyc_per_req_delivery_c
+ * dml_pow(2, 10));
+ disp_ttu_regs->refcyc_per_req_delivery_pre_cur0 =
+ (unsigned int) (refcyc_per_req_delivery_pre_cur0 * dml_pow(2, 10));
+ disp_ttu_regs->refcyc_per_req_delivery_cur0 = (unsigned int) (refcyc_per_req_delivery_cur0
+ * dml_pow(2, 10));
+ disp_ttu_regs->refcyc_per_req_delivery_pre_cur1 =
+ (unsigned int) (refcyc_per_req_delivery_pre_cur1 * dml_pow(2, 10));
+ disp_ttu_regs->refcyc_per_req_delivery_cur1 = (unsigned int) (refcyc_per_req_delivery_cur1
+ * dml_pow(2, 10));
+ disp_ttu_regs->qos_level_low_wm = 0;
+ ASSERT(disp_ttu_regs->qos_level_low_wm < dml_pow(2, 14));
+ disp_ttu_regs->qos_level_high_wm = (unsigned int) (4.0 * (double) htotal
+ * ref_freq_to_pix_freq);
+ /*ASSERT(disp_ttu_regs->qos_level_high_wm < dml_pow(2, 14));*/
+
+ disp_ttu_regs->qos_level_flip = 14;
+ disp_ttu_regs->qos_level_fixed_l = 8;
+ disp_ttu_regs->qos_level_fixed_c = 8;
+ disp_ttu_regs->qos_level_fixed_cur0 = 8;
+ disp_ttu_regs->qos_ramp_disable_l = 0;
+ disp_ttu_regs->qos_ramp_disable_c = 0;
+ disp_ttu_regs->qos_ramp_disable_cur0 = 0;
+
+ disp_ttu_regs->min_ttu_vblank = min_ttu_vblank * refclk_freq_in_mhz;
+ ASSERT(disp_ttu_regs->min_ttu_vblank < dml_pow(2, 24));
+
+ print__ttu_regs_st(mode_lib, *disp_ttu_regs);
+ print__dlg_regs_st(mode_lib, *disp_dlg_regs);
+}
+
+void dml20_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
+ display_dlg_regs_st *dlg_regs,
+ display_ttu_regs_st *ttu_regs,
+ display_e2e_pipe_params_st *e2e_pipe_param,
+ const unsigned int num_pipes,
+ const unsigned int pipe_idx,
+ const bool cstate_en,
+ const bool pstate_en,
+ const bool vm_en,
+ const bool ignore_viewport_pos,
+ const bool immediate_flip_support)
+{
+ display_rq_params_st rq_param = {0};
+ display_dlg_sys_params_st dlg_sys_param = {0};
+
+ // Get watermark and Tex.
+ dlg_sys_param.t_urg_wm_us = get_wm_urgent(mode_lib, e2e_pipe_param, num_pipes);
+ dlg_sys_param.deepsleep_dcfclk_mhz = get_clk_dcf_deepsleep(mode_lib,
+ e2e_pipe_param,
+ num_pipes);
+ dlg_sys_param.t_extra_us = get_urgent_extra_latency(mode_lib, e2e_pipe_param, num_pipes);
+ dlg_sys_param.mem_trip_us = get_wm_memory_trip(mode_lib, e2e_pipe_param, num_pipes);
+ dlg_sys_param.t_mclk_wm_us = get_wm_dram_clock_change(mode_lib, e2e_pipe_param, num_pipes);
+ dlg_sys_param.t_sr_wm_us = get_wm_stutter_enter_exit(mode_lib, e2e_pipe_param, num_pipes);
+ dlg_sys_param.total_flip_bw = get_total_immediate_flip_bw(mode_lib,
+ e2e_pipe_param,
+ num_pipes);
+ dlg_sys_param.total_flip_bytes = get_total_immediate_flip_bytes(mode_lib,
+ e2e_pipe_param,
+ num_pipes);
+ dlg_sys_param.t_srx_delay_us = mode_lib->ip.dcfclk_cstate_latency
+ / dlg_sys_param.deepsleep_dcfclk_mhz; // TODO: Deprecated
+
+ print__dlg_sys_params_st(mode_lib, dlg_sys_param);
+
+ // system parameter calculation done
+
+ dml_print("DML_DLG: Calculation for pipe[%d] start\n\n", pipe_idx);
+ dml20_rq_dlg_get_rq_params(mode_lib, &rq_param, e2e_pipe_param[pipe_idx].pipe.src);
+ dml20_rq_dlg_get_dlg_params(mode_lib,
+ e2e_pipe_param,
+ num_pipes,
+ pipe_idx,
+ dlg_regs,
+ ttu_regs,
+ rq_param.dlg,
+ dlg_sys_param,
+ cstate_en,
+ pstate_en);
+ dml_print("DML_DLG: Calculation for pipe[%d] end\n", pipe_idx);
+}
+
+static void calculate_ttu_cursor(struct display_mode_lib *mode_lib,
+ double *refcyc_per_req_delivery_pre_cur,
+ double *refcyc_per_req_delivery_cur,
+ double refclk_freq_in_mhz,
+ double ref_freq_to_pix_freq,
+ double hscale_pixel_rate_l,
+ double hscl_ratio,
+ double vratio_pre_l,
+ double vratio_l,
+ unsigned int cur_width,
+ enum cursor_bpp cur_bpp)
+{
+ unsigned int cur_src_width = cur_width;
+ unsigned int cur_req_size = 0;
+ unsigned int cur_req_width = 0;
+ double cur_width_ub = 0.0;
+ double cur_req_per_width = 0.0;
+ double hactive_cur = 0.0;
+
+ ASSERT(cur_src_width <= 256);
+
+ *refcyc_per_req_delivery_pre_cur = 0.0;
+ *refcyc_per_req_delivery_cur = 0.0;
+ if (cur_src_width > 0) {
+ unsigned int cur_bit_per_pixel = 0;
+
+ if (cur_bpp == dm_cur_2bit) {
+ cur_req_size = 64; // byte
+ cur_bit_per_pixel = 2;
+ } else { // 32bit
+ cur_bit_per_pixel = 32;
+ if (cur_src_width >= 1 && cur_src_width <= 16)
+ cur_req_size = 64;
+ else if (cur_src_width >= 17 && cur_src_width <= 31)
+ cur_req_size = 128;
+ else
+ cur_req_size = 256;
+ }
+
+ cur_req_width = (double) cur_req_size / ((double) cur_bit_per_pixel / 8.0);
+ cur_width_ub = dml_ceil((double) cur_src_width / (double) cur_req_width, 1)
+ * (double) cur_req_width;
+ cur_req_per_width = cur_width_ub / (double) cur_req_width;
+ hactive_cur = (double) cur_src_width / hscl_ratio; // FIXME: oswin to think about what to do for cursor
+
+ if (vratio_pre_l <= 1.0) {
+ *refcyc_per_req_delivery_pre_cur = hactive_cur * ref_freq_to_pix_freq
+ / (double) cur_req_per_width;
+ } else {
+ *refcyc_per_req_delivery_pre_cur = (double) refclk_freq_in_mhz
+ * (double) cur_src_width / hscale_pixel_rate_l
+ / (double) cur_req_per_width;
+ }
+
+ ASSERT(*refcyc_per_req_delivery_pre_cur < dml_pow(2, 13));
+
+ if (vratio_l <= 1.0) {
+ *refcyc_per_req_delivery_cur = hactive_cur * ref_freq_to_pix_freq
+ / (double) cur_req_per_width;
+ } else {
+ *refcyc_per_req_delivery_cur = (double) refclk_freq_in_mhz
+ * (double) cur_src_width / hscale_pixel_rate_l
+ / (double) cur_req_per_width;
+ }
+
+ dml_print("DML_DLG: %s: cur_req_width = %d\n",
+ __func__,
+ cur_req_width);
+ dml_print("DML_DLG: %s: cur_width_ub = %3.2f\n",
+ __func__,
+ cur_width_ub);
+ dml_print("DML_DLG: %s: cur_req_per_width = %3.2f\n",
+ __func__,
+ cur_req_per_width);
+ dml_print("DML_DLG: %s: hactive_cur = %3.2f\n",
+ __func__,
+ hactive_cur);
+ dml_print("DML_DLG: %s: refcyc_per_req_delivery_pre_cur = %3.2f\n",
+ __func__,
+ *refcyc_per_req_delivery_pre_cur);
+ dml_print("DML_DLG: %s: refcyc_per_req_delivery_cur = %3.2f\n",
+ __func__,
+ *refcyc_per_req_delivery_cur);
+
+ ASSERT(*refcyc_per_req_delivery_cur < dml_pow(2, 13));
+ }
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.h b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.h
new file mode 100644
index 000000000000..8c86b63ddf07
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.h
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DML20_DISPLAY_RQ_DLG_CALC_H__
+#define __DML20_DISPLAY_RQ_DLG_CALC_H__
+
+#include "../dml_common_defs.h"
+#include "../display_rq_dlg_helpers.h"
+
+struct display_mode_lib;
+
+
+// Function: dml_rq_dlg_get_rq_reg
+// Main entry point for test to get the register values out of this DML class.
+// This function calls <get_rq_param> and <extract_rq_regs> fucntions to calculate
+// and then populate the rq_regs struct
+// Input:
+// pipe_src_param - pipe source configuration (e.g. vp, pitch, etc.)
+// Output:
+// rq_regs - struct that holds all the RQ registers field value.
+// See also: <display_rq_regs_st>
+void dml20_rq_dlg_get_rq_reg(
+ struct display_mode_lib *mode_lib,
+ display_rq_regs_st *rq_regs,
+ const display_pipe_params_st pipe_param);
+
+
+// Function: dml_rq_dlg_get_dlg_reg
+// Calculate and return DLG and TTU register struct given the system setting
+// Output:
+// dlg_regs - output DLG register struct
+// ttu_regs - output DLG TTU register struct
+// Input:
+// e2e_pipe_param - "compacted" array of e2e pipe param struct
+// num_pipes - num of active "pipe" or "route"
+// pipe_idx - index that identifies the e2e_pipe_param that corresponding to this dlg
+// cstate - 0: when calculate min_ttu_vblank it is assumed cstate is not required. 1: Normal mode, cstate is considered.
+// Added for legacy or unrealistic timing tests.
+void dml20_rq_dlg_get_dlg_reg(
+ struct display_mode_lib *mode_lib,
+ display_dlg_regs_st *dlg_regs,
+ display_ttu_regs_st *ttu_regs,
+ display_e2e_pipe_params_st *e2e_pipe_param,
+ const unsigned int num_pipes,
+ const unsigned int pipe_idx,
+ const bool cstate_en,
+ const bool pstate_en,
+ const bool vm_en,
+ const bool ignore_viewport_pos,
+ const bool immediate_flip_support);
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_enums.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_enums.h
index c59e582c1f40..0c2fab1e93b6 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_enums.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_enums.h
@@ -86,7 +86,8 @@ enum dm_swizzle_mode {
dm_sw_gfx7_2d_thin_gl
};
enum lb_depth {
- dm_lb_10 = 0, dm_lb_8 = 1, dm_lb_6 = 2, dm_lb_12 = 3, dm_lb_16
+ dm_lb_10 = 0, dm_lb_8 = 1, dm_lb_6 = 2, dm_lb_12 = 3, dm_lb_16 = 4,
+ dm_lb_19 = 5
};
enum voltage_state {
dm_vmin = 0, dm_vmid = 1, dm_vnom = 2, dm_vmax = 3
@@ -130,6 +131,9 @@ enum dm_validation_status {
DML_FAIL_DIO_SUPPORT,
DML_FAIL_NOT_ENOUGH_DSC,
DML_FAIL_DSC_CLK_REQUIRED,
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ DML_FAIL_DSC_VALIDATION_FAILURE,
+#endif
DML_FAIL_URGENT_LATENCY,
DML_FAIL_REORDERING_BUFFER,
DML_FAIL_DISPCLK_DPPCLK,
@@ -147,4 +151,10 @@ enum dm_validation_status {
DML_FAIL_V_RATIO_PREFETCH,
};
+enum writeback_config {
+ dm_normal,
+ dm_whole_buffer_for_single_stream_no_interleave,
+ dm_whole_buffer_for_single_stream_interleave,
+};
+
#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
index 80ffd7d958b2..91810c7d5cf5 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
@@ -25,6 +25,19 @@
#include "display_mode_lib.h"
#include "dc_features.h"
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#include "dcn20/display_mode_vba_20.h"
+#include "dcn20/display_rq_dlg_calc_20.h"
+#endif
+
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+const struct dml_funcs dml20_funcs = {
+ .validate = dml20_ModeSupportAndSystemConfigurationFull,
+ .recalculate = dml20_recalculate,
+ .rq_dlg_get_dlg_reg = dml20_rq_dlg_get_dlg_reg,
+ .rq_dlg_get_rq_reg = dml20_rq_dlg_get_rq_reg
+};
+#endif
void dml_init_instance(struct display_mode_lib *lib,
const struct _vcs_dpi_soc_bounding_box_st *soc_bb,
@@ -34,6 +47,15 @@ void dml_init_instance(struct display_mode_lib *lib,
lib->soc = *soc_bb;
lib->ip = *ip_params;
lib->project = project;
+ switch (project) {
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ case DML_PROJECT_NAVI10:
+ lib->funcs = dml20_funcs;
+ break;
+#endif
+ default:
+ break;
+ }
}
const char *dml_get_status_message(enum dm_validation_status status)
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h
index 1b546dba34bd..5bf13d67f289 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h
@@ -27,18 +27,50 @@
#include "dml_common_defs.h"
-#include "dml1_display_rq_dlg_calc.h"
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+#include "display_mode_vba.h"
+#endif
enum dml_project {
DML_PROJECT_UNDEFINED,
- DML_PROJECT_RAVEN1
+ DML_PROJECT_RAVEN1,
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ DML_PROJECT_NAVI10,
+#endif
+};
+
+struct display_mode_lib;
+
+struct dml_funcs {
+ void (*rq_dlg_get_dlg_reg)(
+ struct display_mode_lib *mode_lib,
+ display_dlg_regs_st *dlg_regs,
+ display_ttu_regs_st *ttu_regs,
+ display_e2e_pipe_params_st *e2e_pipe_param,
+ const unsigned int num_pipes,
+ const unsigned int pipe_idx,
+ const bool cstate_en,
+ const bool pstate_en,
+ const bool vm_en,
+ const bool ignore_viewport_pos,
+ const bool immediate_flip_support);
+ void (*rq_dlg_get_rq_reg)(
+ struct display_mode_lib *mode_lib,
+ display_rq_regs_st *rq_regs,
+ const display_pipe_params_st pipe_param);
+ void (*recalculate)(struct display_mode_lib *mode_lib);
+ void (*validate)(struct display_mode_lib *mode_lib);
};
struct display_mode_lib {
struct _vcs_dpi_ip_params_st ip;
struct _vcs_dpi_soc_bounding_box_st soc;
enum dml_project project;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ struct vba_vars_st vba;
+#endif
struct dal_logger *logger;
+ struct dml_funcs funcs;
};
void dml_init_instance(struct display_mode_lib *lib,
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
index c5b791d158a7..5678472546ab 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
@@ -57,6 +57,7 @@ struct _vcs_dpi_voltage_scaling_st {
double dscclk_mhz;
double dcfclk_mhz;
double socclk_mhz;
+ double phyclk_d18_mhz;
double dram_speed_mts;
double fabricclk_mhz;
double dispclk_mhz;
@@ -97,6 +98,7 @@ struct _vcs_dpi_soc_bounding_box_st {
unsigned int num_banks;
unsigned int num_chans;
unsigned int vmm_page_size_bytes;
+ unsigned int hostvm_min_page_size_bytes;
double dram_clock_change_latency_us;
double writeback_dram_clock_change_latency_us;
unsigned int return_bus_width_bytes;
@@ -135,6 +137,22 @@ struct _vcs_dpi_ip_params_st {
unsigned int writeback_luma_buffer_size_kbytes;
unsigned int writeback_chroma_buffer_size_kbytes;
unsigned int writeback_chroma_line_buffer_width_pixels;
+
+ unsigned int writeback_interface_buffer_size_kbytes;
+ unsigned int writeback_line_buffer_buffer_size;
+
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ unsigned int writeback_10bpc420_supported;
+ double writeback_max_hscl_ratio;
+ double writeback_max_vscl_ratio;
+ double writeback_min_hscl_ratio;
+ double writeback_min_vscl_ratio;
+ unsigned int writeback_max_hscl_taps;
+ unsigned int writeback_max_vscl_taps;
+ unsigned int writeback_line_buffer_luma_buffer_size;
+ unsigned int writeback_line_buffer_chroma_buffer_size;
+#endif
+
unsigned int max_page_table_levels;
unsigned int max_num_dpp;
unsigned int max_num_otg;
@@ -152,6 +170,13 @@ struct _vcs_dpi_ip_params_st {
unsigned int max_hscl_taps;
unsigned int max_vscl_taps;
unsigned int xfc_supported;
+ unsigned int ptoi_supported;
+ unsigned int gfx7_compat_tiling_supported;
+
+ bool odm_combine_4to1_supported;
+ bool dynamic_metadata_vm_enabled;
+ unsigned int max_num_hdmi_frl_outputs;
+
unsigned int xfc_fill_constant_bytes;
double dispclk_ramp_margin_percent;
double xfc_fill_bw_overhead_percent;
@@ -218,10 +243,15 @@ struct _vcs_dpi_display_pipe_source_params_st {
unsigned int hsplit_grp;
unsigned char xfc_enable;
unsigned char xfc_slave;
+ unsigned char immediate_flip;
struct _vcs_dpi_display_xfc_params_st xfc_params;
+ //for vstartuplines calculation freesync
+ unsigned char v_total_min;
+ unsigned char v_total_max;
};
struct writeback_st {
int wb_src_height;
+ int wb_src_width;
int wb_dst_width;
int wb_dst_height;
int wb_pixel_format;
@@ -289,6 +319,8 @@ struct _vcs_dpi_display_pipe_dest_params_st {
unsigned char otg_inst;
unsigned char odm_combine;
unsigned char use_maximum_vstartup;
+ unsigned int vtotal_max;
+ unsigned int vtotal_min;
};
struct _vcs_dpi_display_pipe_params_st {
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c
new file mode 100644
index 000000000000..4d2a1262d9db
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c
@@ -0,0 +1,839 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+
+#include "display_mode_lib.h"
+#include "display_mode_vba.h"
+#include "dml_inline_defs.h"
+
+/*
+ * NOTE:
+ * This file is gcc-parsable HW gospel, coming straight from HW engineers.
+ *
+ * It doesn't adhere to Linux kernel style and sometimes will do things in odd
+ * ways. Unless there is something clearly wrong with it the code should
+ * remain as-is as it provides us with a guarantee from HW that it is correct.
+ */
+
+
+static void fetch_socbb_params(struct display_mode_lib *mode_lib);
+static void fetch_ip_params(struct display_mode_lib *mode_lib);
+static void fetch_pipe_params(struct display_mode_lib *mode_lib);
+static void recalculate_params(
+ struct display_mode_lib *mode_lib,
+ const display_e2e_pipe_params_st *pipes,
+ unsigned int num_pipes);
+
+static unsigned int CursorBppEnumToBits(enum cursor_bpp ebpp);
+
+unsigned int dml_get_voltage_level(
+ struct display_mode_lib *mode_lib,
+ const display_e2e_pipe_params_st *pipes,
+ unsigned int num_pipes)
+{
+ bool need_recalculate = memcmp(&mode_lib->soc, &mode_lib->vba.soc, sizeof(mode_lib->vba.soc)) != 0
+ || memcmp(&mode_lib->ip, &mode_lib->vba.ip, sizeof(mode_lib->vba.ip)) != 0
+ || num_pipes != mode_lib->vba.cache_num_pipes
+ || memcmp(pipes, mode_lib->vba.cache_pipes,
+ sizeof(display_e2e_pipe_params_st) * num_pipes) != 0;
+
+ mode_lib->vba.soc = mode_lib->soc;
+ mode_lib->vba.ip = mode_lib->ip;
+ memcpy(mode_lib->vba.cache_pipes, pipes, sizeof(*pipes) * num_pipes);
+ mode_lib->vba.cache_num_pipes = num_pipes;
+
+ if (need_recalculate && pipes[0].clks_cfg.dppclk_mhz != 0)
+ mode_lib->funcs.recalculate(mode_lib);
+ else {
+ fetch_socbb_params(mode_lib);
+ fetch_ip_params(mode_lib);
+ fetch_pipe_params(mode_lib);
+ PixelClockAdjustmentForProgressiveToInterlaceUnit(mode_lib);
+ }
+ mode_lib->funcs.validate(mode_lib);
+
+ return mode_lib->vba.VoltageLevel;
+}
+
+#define dml_get_attr_func(attr, var) double get_##attr(struct display_mode_lib *mode_lib, const display_e2e_pipe_params_st *pipes, unsigned int num_pipes) \
+{ \
+ recalculate_params(mode_lib, pipes, num_pipes); \
+ return var; \
+}
+
+dml_get_attr_func(clk_dcf_deepsleep, mode_lib->vba.DCFCLKDeepSleep);
+dml_get_attr_func(wm_urgent, mode_lib->vba.UrgentWatermark);
+dml_get_attr_func(wm_memory_trip, mode_lib->vba.UrgentLatency);
+dml_get_attr_func(wm_writeback_urgent, mode_lib->vba.WritebackUrgentWatermark);
+dml_get_attr_func(wm_stutter_exit, mode_lib->vba.StutterExitWatermark);
+dml_get_attr_func(wm_stutter_enter_exit, mode_lib->vba.StutterEnterPlusExitWatermark);
+dml_get_attr_func(wm_dram_clock_change, mode_lib->vba.DRAMClockChangeWatermark);
+dml_get_attr_func(wm_writeback_dram_clock_change, mode_lib->vba.WritebackDRAMClockChangeWatermark);
+dml_get_attr_func(wm_xfc_underflow, mode_lib->vba.UrgentWatermark); // xfc_underflow maps to urgent
+dml_get_attr_func(stutter_efficiency, mode_lib->vba.StutterEfficiency);
+dml_get_attr_func(stutter_efficiency_no_vblank, mode_lib->vba.StutterEfficiencyNotIncludingVBlank);
+dml_get_attr_func(urgent_latency, mode_lib->vba.UrgentLatency);
+dml_get_attr_func(urgent_extra_latency, mode_lib->vba.UrgentExtraLatency);
+dml_get_attr_func(nonurgent_latency, mode_lib->vba.NonUrgentLatencyTolerance);
+dml_get_attr_func(
+ dram_clock_change_latency,
+ mode_lib->vba.MinActiveDRAMClockChangeLatencySupported);
+dml_get_attr_func(dispclk_calculated, mode_lib->vba.DISPCLK_calculated);
+dml_get_attr_func(total_data_read_bw, mode_lib->vba.TotalDataReadBandwidth);
+dml_get_attr_func(return_bw, mode_lib->vba.ReturnBW);
+dml_get_attr_func(tcalc, mode_lib->vba.TCalc);
+dml_get_attr_func(fraction_of_urgent_bandwidth, mode_lib->vba.FractionOfUrgentBandwidth);
+dml_get_attr_func(fraction_of_urgent_bandwidth_imm_flip, mode_lib->vba.FractionOfUrgentBandwidthImmediateFlip);
+
+#define dml_get_pipe_attr_func(attr, var) double get_##attr(struct display_mode_lib *mode_lib, const display_e2e_pipe_params_st *pipes, unsigned int num_pipes, unsigned int which_pipe) \
+{\
+ unsigned int which_plane; \
+ recalculate_params(mode_lib, pipes, num_pipes); \
+ which_plane = mode_lib->vba.pipe_plane[which_pipe]; \
+ return var[which_plane]; \
+}
+
+dml_get_pipe_attr_func(dsc_delay, mode_lib->vba.DSCDelay);
+dml_get_pipe_attr_func(dppclk_calculated, mode_lib->vba.DPPCLK_calculated);
+dml_get_pipe_attr_func(dscclk_calculated, mode_lib->vba.DSCCLK_calculated);
+dml_get_pipe_attr_func(min_ttu_vblank, mode_lib->vba.MinTTUVBlank);
+dml_get_pipe_attr_func(vratio_prefetch_l, mode_lib->vba.VRatioPrefetchY);
+dml_get_pipe_attr_func(vratio_prefetch_c, mode_lib->vba.VRatioPrefetchC);
+dml_get_pipe_attr_func(dst_x_after_scaler, mode_lib->vba.DSTXAfterScaler);
+dml_get_pipe_attr_func(dst_y_after_scaler, mode_lib->vba.DSTYAfterScaler);
+dml_get_pipe_attr_func(dst_y_per_vm_vblank, mode_lib->vba.DestinationLinesToRequestVMInVBlank);
+dml_get_pipe_attr_func(dst_y_per_row_vblank, mode_lib->vba.DestinationLinesToRequestRowInVBlank);
+dml_get_pipe_attr_func(dst_y_prefetch, mode_lib->vba.DestinationLinesForPrefetch);
+dml_get_pipe_attr_func(dst_y_per_vm_flip, mode_lib->vba.DestinationLinesToRequestVMInImmediateFlip);
+dml_get_pipe_attr_func(
+ dst_y_per_row_flip,
+ mode_lib->vba.DestinationLinesToRequestRowInImmediateFlip);
+
+dml_get_pipe_attr_func(xfc_transfer_delay, mode_lib->vba.XFCTransferDelay);
+dml_get_pipe_attr_func(xfc_precharge_delay, mode_lib->vba.XFCPrechargeDelay);
+dml_get_pipe_attr_func(xfc_remote_surface_flip_latency, mode_lib->vba.XFCRemoteSurfaceFlipLatency);
+dml_get_pipe_attr_func(xfc_prefetch_margin, mode_lib->vba.XFCPrefetchMargin);
+dml_get_pipe_attr_func(refcyc_per_vm_group_vblank, mode_lib->vba.TimePerVMGroupVBlank);
+dml_get_pipe_attr_func(refcyc_per_vm_group_flip, mode_lib->vba.TimePerVMGroupFlip);
+dml_get_pipe_attr_func(refcyc_per_vm_req_vblank, mode_lib->vba.TimePerVMRequestVBlank);
+dml_get_pipe_attr_func(refcyc_per_vm_req_flip, mode_lib->vba.TimePerVMRequestFlip);
+
+unsigned int get_vstartup_calculated(
+ struct display_mode_lib *mode_lib,
+ const display_e2e_pipe_params_st *pipes,
+ unsigned int num_pipes,
+ unsigned int which_pipe)
+{
+ unsigned int which_plane;
+
+ recalculate_params(mode_lib, pipes, num_pipes);
+ which_plane = mode_lib->vba.pipe_plane[which_pipe];
+ return mode_lib->vba.VStartup[which_plane];
+}
+
+double get_total_immediate_flip_bytes(
+ struct display_mode_lib *mode_lib,
+ const display_e2e_pipe_params_st *pipes,
+ unsigned int num_pipes)
+{
+ recalculate_params(mode_lib, pipes, num_pipes);
+ return mode_lib->vba.TotImmediateFlipBytes;
+}
+
+double get_total_immediate_flip_bw(
+ struct display_mode_lib *mode_lib,
+ const display_e2e_pipe_params_st *pipes,
+ unsigned int num_pipes)
+{
+ unsigned int k;
+ double immediate_flip_bw = 0.0;
+ recalculate_params(mode_lib, pipes, num_pipes);
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k)
+ immediate_flip_bw += mode_lib->vba.ImmediateFlipBW[k];
+ return immediate_flip_bw;
+}
+
+double get_total_prefetch_bw(
+ struct display_mode_lib *mode_lib,
+ const display_e2e_pipe_params_st *pipes,
+ unsigned int num_pipes)
+{
+ unsigned int k;
+ double total_prefetch_bw = 0.0;
+
+ recalculate_params(mode_lib, pipes, num_pipes);
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k)
+ total_prefetch_bw += mode_lib->vba.PrefetchBandwidth[k];
+ return total_prefetch_bw;
+}
+
+static void fetch_socbb_params(struct display_mode_lib *mode_lib)
+{
+ soc_bounding_box_st *soc = &mode_lib->vba.soc;
+ int i;
+
+ // SOC Bounding Box Parameters
+ mode_lib->vba.ReturnBusWidth = soc->return_bus_width_bytes;
+ mode_lib->vba.NumberOfChannels = soc->num_chans;
+ mode_lib->vba.PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelDataOnly =
+ soc->pct_ideal_dram_sdp_bw_after_urgent_pixel_only; // there's always that one bastard variable that's so long it throws everything out of alignment!
+ mode_lib->vba.PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData =
+ soc->pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm;
+ mode_lib->vba.PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly =
+ soc->pct_ideal_dram_sdp_bw_after_urgent_vm_only;
+ mode_lib->vba.MaxAveragePercentOfIdealSDPPortBWDisplayCanUseInNormalSystemOperation =
+ soc->max_avg_sdp_bw_use_normal_percent;
+ mode_lib->vba.MaxAveragePercentOfIdealDRAMBWDisplayCanUseInNormalSystemOperation =
+ soc->max_avg_dram_bw_use_normal_percent;
+ mode_lib->vba.UrgentLatencyPixelDataOnly = soc->urgent_latency_pixel_data_only_us;
+ mode_lib->vba.UrgentLatencyPixelMixedWithVMData = soc->urgent_latency_pixel_mixed_with_vm_data_us;
+ mode_lib->vba.UrgentLatencyVMDataOnly = soc->urgent_latency_vm_data_only_us;
+ mode_lib->vba.RoundTripPingLatencyCycles = soc->round_trip_ping_latency_dcfclk_cycles;
+ mode_lib->vba.UrgentOutOfOrderReturnPerChannelPixelDataOnly =
+ soc->urgent_out_of_order_return_per_channel_pixel_only_bytes;
+ mode_lib->vba.UrgentOutOfOrderReturnPerChannelPixelMixedWithVMData =
+ soc->urgent_out_of_order_return_per_channel_pixel_and_vm_bytes;
+ mode_lib->vba.UrgentOutOfOrderReturnPerChannelVMDataOnly =
+ soc->urgent_out_of_order_return_per_channel_vm_only_bytes;
+ mode_lib->vba.WritebackLatency = soc->writeback_latency_us;
+ mode_lib->vba.SRExitTime = soc->sr_exit_time_us;
+ mode_lib->vba.SREnterPlusExitTime = soc->sr_enter_plus_exit_time_us;
+ mode_lib->vba.DRAMClockChangeLatency = soc->dram_clock_change_latency_us;
+ mode_lib->vba.Downspreading = soc->downspread_percent;
+ mode_lib->vba.DRAMChannelWidth = soc->dram_channel_width_bytes; // new!
+ mode_lib->vba.FabricDatapathToDCNDataReturn = soc->fabric_datapath_to_dcn_data_return_bytes; // new!
+ mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading = soc->dcn_downspread_percent; // new
+ mode_lib->vba.DISPCLKDPPCLKVCOSpeed = soc->dispclk_dppclk_vco_speed_mhz; // new
+ mode_lib->vba.VMMPageSize = soc->vmm_page_size_bytes;
+ mode_lib->vba.GPUVMMinPageSize = soc->vmm_page_size_bytes / 1024;
+ mode_lib->vba.HostVMMinPageSize = soc->hostvm_min_page_size_bytes / 1024;
+ // Set the voltage scaling clocks as the defaults. Most of these will
+ // be set to different values by the test
+ for (i = 0; i < mode_lib->vba.soc.num_states; i++)
+ if (soc->clock_limits[i].state == mode_lib->vba.VoltageLevel)
+ break;
+
+ mode_lib->vba.DCFCLK = soc->clock_limits[i].dcfclk_mhz;
+ mode_lib->vba.SOCCLK = soc->clock_limits[i].socclk_mhz;
+ mode_lib->vba.DRAMSpeed = soc->clock_limits[i].dram_speed_mts;
+ mode_lib->vba.FabricClock = soc->clock_limits[i].fabricclk_mhz;
+
+ mode_lib->vba.XFCBusTransportTime = soc->xfc_bus_transport_time_us;
+ mode_lib->vba.XFCXBUFLatencyTolerance = soc->xfc_xbuf_latency_tolerance_us;
+ mode_lib->vba.UseUrgentBurstBandwidth = soc->use_urgent_burst_bw;
+
+ mode_lib->vba.SupportGFX7CompatibleTilingIn32bppAnd64bpp = false;
+ mode_lib->vba.WritebackLumaAndChromaScalingSupported = true;
+ mode_lib->vba.MaxHSCLRatio = 4;
+ mode_lib->vba.MaxVSCLRatio = 4;
+ mode_lib->vba.Cursor64BppSupport = true;
+ for (i = 0; i <= mode_lib->vba.soc.num_states; i++) {
+ mode_lib->vba.DCFCLKPerState[i] = soc->clock_limits[i].dcfclk_mhz;
+ mode_lib->vba.FabricClockPerState[i] = soc->clock_limits[i].fabricclk_mhz;
+ mode_lib->vba.SOCCLKPerState[i] = soc->clock_limits[i].socclk_mhz;
+ mode_lib->vba.PHYCLKPerState[i] = soc->clock_limits[i].phyclk_mhz;
+ mode_lib->vba.PHYCLKD18PerState[i] = soc->clock_limits[i].phyclk_d18_mhz;
+ mode_lib->vba.MaxDppclk[i] = soc->clock_limits[i].dppclk_mhz;
+ mode_lib->vba.MaxDSCCLK[i] = soc->clock_limits[i].dscclk_mhz;
+ mode_lib->vba.DRAMSpeedPerState[i] = soc->clock_limits[i].dram_speed_mts;
+ //mode_lib->vba.DRAMSpeedPerState[i] = soc->clock_limits[i].dram_speed_mhz;
+ mode_lib->vba.MaxDispclk[i] = soc->clock_limits[i].dispclk_mhz;
+ }
+}
+
+static void fetch_ip_params(struct display_mode_lib *mode_lib)
+{
+ ip_params_st *ip = &mode_lib->vba.ip;
+
+ // IP Parameters
+ mode_lib->vba.MaxNumDPP = ip->max_num_dpp;
+ mode_lib->vba.MaxNumOTG = ip->max_num_otg;
+ mode_lib->vba.MaxNumHDMIFRLOutputs = ip->max_num_hdmi_frl_outputs;
+ mode_lib->vba.MaxNumWriteback = ip->max_num_wb;
+ mode_lib->vba.CursorChunkSize = ip->cursor_chunk_size;
+ mode_lib->vba.CursorBufferSize = ip->cursor_buffer_size;
+
+ mode_lib->vba.MaxDCHUBToPSCLThroughput = ip->max_dchub_pscl_bw_pix_per_clk;
+ mode_lib->vba.MaxPSCLToLBThroughput = ip->max_pscl_lb_bw_pix_per_clk;
+ mode_lib->vba.ROBBufferSizeInKByte = ip->rob_buffer_size_kbytes;
+ mode_lib->vba.DETBufferSizeInKByte = ip->det_buffer_size_kbytes;
+ mode_lib->vba.PixelChunkSizeInKByte = ip->pixel_chunk_size_kbytes;
+ mode_lib->vba.MetaChunkSize = ip->meta_chunk_size_kbytes;
+ mode_lib->vba.WritebackChunkSize = ip->writeback_chunk_size_kbytes;
+ mode_lib->vba.LineBufferSize = ip->line_buffer_size_bits;
+ mode_lib->vba.MaxLineBufferLines = ip->max_line_buffer_lines;
+ mode_lib->vba.PTEBufferSizeInRequestsLuma = ip->dpte_buffer_size_in_pte_reqs_luma;
+ mode_lib->vba.PTEBufferSizeInRequestsChroma = ip->dpte_buffer_size_in_pte_reqs_chroma;
+ mode_lib->vba.DPPOutputBufferPixels = ip->dpp_output_buffer_pixels;
+ mode_lib->vba.OPPOutputBufferLines = ip->opp_output_buffer_lines;
+ mode_lib->vba.MaxHSCLRatio = ip->max_hscl_ratio;
+ mode_lib->vba.MaxVSCLRatio = ip->max_vscl_ratio;
+ mode_lib->vba.WritebackInterfaceLumaBufferSize = ip->writeback_luma_buffer_size_kbytes * 1024;
+ mode_lib->vba.WritebackInterfaceChromaBufferSize = ip->writeback_chroma_buffer_size_kbytes * 1024;
+
+ mode_lib->vba.WritebackInterfaceBufferSize = ip->writeback_interface_buffer_size_kbytes;
+ mode_lib->vba.WritebackLineBufferSize = ip->writeback_line_buffer_buffer_size;
+ mode_lib->vba.MinVoltageLevel = 0;
+ mode_lib->vba.MaxVoltageLevel = 5;
+
+ mode_lib->vba.WritebackChromaLineBufferWidth =
+ ip->writeback_chroma_line_buffer_width_pixels;
+ mode_lib->vba.WritebackLineBufferLumaBufferSize =
+ ip->writeback_line_buffer_luma_buffer_size;
+ mode_lib->vba.WritebackLineBufferChromaBufferSize =
+ ip->writeback_line_buffer_chroma_buffer_size;
+ mode_lib->vba.Writeback10bpc420Supported = ip->writeback_10bpc420_supported;
+ mode_lib->vba.WritebackMaxHSCLRatio = ip->writeback_max_hscl_ratio;
+ mode_lib->vba.WritebackMaxVSCLRatio = ip->writeback_max_vscl_ratio;
+ mode_lib->vba.WritebackMinHSCLRatio = ip->writeback_min_hscl_ratio;
+ mode_lib->vba.WritebackMinVSCLRatio = ip->writeback_min_vscl_ratio;
+ mode_lib->vba.WritebackMaxHSCLTaps = ip->writeback_max_hscl_taps;
+ mode_lib->vba.WritebackMaxVSCLTaps = ip->writeback_max_vscl_taps;
+ mode_lib->vba.WritebackConfiguration = dm_normal;
+ mode_lib->vba.GPUVMMaxPageTableLevels = ip->gpuvm_max_page_table_levels;
+ mode_lib->vba.HostVMMaxNonCachedPageTableLevels = ip->hostvm_max_page_table_levels;
+ mode_lib->vba.HostVMMaxPageTableLevels = ip->hostvm_max_page_table_levels;
+ mode_lib->vba.HostVMCachedPageTableLevels = ip->hostvm_cached_page_table_levels;
+ mode_lib->vba.MaxInterDCNTileRepeaters = ip->max_inter_dcn_tile_repeaters;
+ mode_lib->vba.NumberOfDSC = ip->num_dsc;
+ mode_lib->vba.ODMCapability = ip->odm_capable;
+ mode_lib->vba.DISPCLKRampingMargin = ip->dispclk_ramp_margin_percent;
+
+ mode_lib->vba.XFCSupported = ip->xfc_supported;
+ mode_lib->vba.XFCFillBWOverhead = ip->xfc_fill_bw_overhead_percent;
+ mode_lib->vba.XFCFillConstant = ip->xfc_fill_constant_bytes;
+ mode_lib->vba.DPPCLKDelaySubtotal = ip->dppclk_delay_subtotal;
+ mode_lib->vba.DPPCLKDelaySCL = ip->dppclk_delay_scl;
+ mode_lib->vba.DPPCLKDelaySCLLBOnly = ip->dppclk_delay_scl_lb_only;
+ mode_lib->vba.DPPCLKDelayCNVCFormater = ip->dppclk_delay_cnvc_formatter;
+ mode_lib->vba.DPPCLKDelayCNVCCursor = ip->dppclk_delay_cnvc_cursor;
+ mode_lib->vba.DISPCLKDelaySubtotal = ip->dispclk_delay_subtotal;
+ mode_lib->vba.DynamicMetadataVMEnabled = ip->dynamic_metadata_vm_enabled;
+ mode_lib->vba.ODMCombine4To1Supported = ip->odm_combine_4to1_supported;
+ mode_lib->vba.ProgressiveToInterlaceUnitInOPP = ip->ptoi_supported;
+ mode_lib->vba.PDEProcessingBufIn64KBReqs = ip->pde_proc_buffer_size_64k_reqs;
+ mode_lib->vba.PTEGroupSize = ip->pte_group_size_bytes;
+ mode_lib->vba.SupportGFX7CompatibleTilingIn32bppAnd64bpp = ip->gfx7_compat_tiling_supported;
+}
+
+static void fetch_pipe_params(struct display_mode_lib *mode_lib)
+{
+ display_e2e_pipe_params_st *pipes = mode_lib->vba.cache_pipes;
+ ip_params_st *ip = &mode_lib->vba.ip;
+
+ unsigned int OTGInstPlane[DC__NUM_DPP__MAX];
+ unsigned int j, k;
+ bool PlaneVisited[DC__NUM_DPP__MAX];
+ bool visited[DC__NUM_DPP__MAX];
+
+ // Convert Pipes to Planes
+ for (k = 0; k < mode_lib->vba.cache_num_pipes; ++k)
+ visited[k] = false;
+
+ mode_lib->vba.NumberOfActivePlanes = 0;
+ for (j = 0; j < mode_lib->vba.cache_num_pipes; ++j) {
+ display_pipe_source_params_st *src = &pipes[j].pipe.src;
+ display_pipe_dest_params_st *dst = &pipes[j].pipe.dest;
+ scaler_ratio_depth_st *scl = &pipes[j].pipe.scale_ratio_depth;
+ scaler_taps_st *taps = &pipes[j].pipe.scale_taps;
+ display_output_params_st *dout = &pipes[j].dout;
+ display_clocks_and_cfg_st *clks = &pipes[j].clks_cfg;
+
+ if (visited[j])
+ continue;
+ visited[j] = true;
+
+ mode_lib->vba.pipe_plane[j] = mode_lib->vba.NumberOfActivePlanes;
+
+ mode_lib->vba.DPPPerPlane[mode_lib->vba.NumberOfActivePlanes] = 1;
+ mode_lib->vba.SourceScan[mode_lib->vba.NumberOfActivePlanes] =
+ (enum scan_direction_class) (src->source_scan);
+ mode_lib->vba.ViewportWidth[mode_lib->vba.NumberOfActivePlanes] =
+ src->viewport_width;
+ mode_lib->vba.ViewportWidthChroma[mode_lib->vba.NumberOfActivePlanes] =
+ src->viewport_width_c;
+ mode_lib->vba.ViewportHeight[mode_lib->vba.NumberOfActivePlanes] =
+ src->viewport_height;
+ mode_lib->vba.ViewportHeightChroma[mode_lib->vba.NumberOfActivePlanes] =
+ src->viewport_height_c;
+ mode_lib->vba.ViewportYStartY[mode_lib->vba.NumberOfActivePlanes] =
+ src->viewport_y_y;
+ mode_lib->vba.ViewportYStartC[mode_lib->vba.NumberOfActivePlanes] =
+ src->viewport_y_c;
+ mode_lib->vba.PitchY[mode_lib->vba.NumberOfActivePlanes] = src->data_pitch;
+ mode_lib->vba.SurfaceHeightY[mode_lib->vba.NumberOfActivePlanes] = src->viewport_height;
+ mode_lib->vba.PitchC[mode_lib->vba.NumberOfActivePlanes] = src->data_pitch_c;
+ mode_lib->vba.SurfaceHeightC[mode_lib->vba.NumberOfActivePlanes] = src->viewport_height_c;
+ mode_lib->vba.DCCMetaPitchY[mode_lib->vba.NumberOfActivePlanes] = src->meta_pitch;
+ mode_lib->vba.DCCMetaPitchC[mode_lib->vba.NumberOfActivePlanes] = src->meta_pitch_c;
+ mode_lib->vba.HRatio[mode_lib->vba.NumberOfActivePlanes] = scl->hscl_ratio;
+ mode_lib->vba.HRatioChroma[mode_lib->vba.NumberOfActivePlanes] = scl->hscl_ratio_c;
+ mode_lib->vba.VRatio[mode_lib->vba.NumberOfActivePlanes] = scl->vscl_ratio;
+ mode_lib->vba.VRatioChroma[mode_lib->vba.NumberOfActivePlanes] = scl->vscl_ratio_c;
+ mode_lib->vba.ScalerEnabled[mode_lib->vba.NumberOfActivePlanes] = scl->scl_enable;
+ mode_lib->vba.Interlace[mode_lib->vba.NumberOfActivePlanes] = dst->interlaced;
+ if (dst->interlaced && !ip->ptoi_supported) {
+ mode_lib->vba.VRatio[mode_lib->vba.NumberOfActivePlanes] *= 2.0;
+ mode_lib->vba.VRatioChroma[mode_lib->vba.NumberOfActivePlanes] *= 2.0;
+ }
+ mode_lib->vba.htaps[mode_lib->vba.NumberOfActivePlanes] = taps->htaps;
+ mode_lib->vba.vtaps[mode_lib->vba.NumberOfActivePlanes] = taps->vtaps;
+ mode_lib->vba.HTAPsChroma[mode_lib->vba.NumberOfActivePlanes] = taps->htaps_c;
+ mode_lib->vba.VTAPsChroma[mode_lib->vba.NumberOfActivePlanes] = taps->vtaps_c;
+ mode_lib->vba.HTotal[mode_lib->vba.NumberOfActivePlanes] = dst->htotal;
+ mode_lib->vba.VTotal[mode_lib->vba.NumberOfActivePlanes] = dst->vtotal;
+ mode_lib->vba.DCCEnable[mode_lib->vba.NumberOfActivePlanes] =
+ src->dcc_use_global ?
+ ip->dcc_supported : src->dcc && ip->dcc_supported;
+ mode_lib->vba.DCCRate[mode_lib->vba.NumberOfActivePlanes] = src->dcc_rate;
+ /* TODO: Needs to be set based on src->dcc_rate_luma/chroma */
+ mode_lib->vba.DCCRateLuma[mode_lib->vba.NumberOfActivePlanes] = 0;
+ mode_lib->vba.DCCRateChroma[mode_lib->vba.NumberOfActivePlanes] = 0;
+
+ mode_lib->vba.SourcePixelFormat[mode_lib->vba.NumberOfActivePlanes] =
+ (enum source_format_class) (src->source_format);
+ mode_lib->vba.HActive[mode_lib->vba.NumberOfActivePlanes] = dst->hactive;
+ mode_lib->vba.VActive[mode_lib->vba.NumberOfActivePlanes] = dst->vactive;
+ mode_lib->vba.SurfaceTiling[mode_lib->vba.NumberOfActivePlanes] =
+ (enum dm_swizzle_mode) (src->sw_mode);
+ mode_lib->vba.ScalerRecoutWidth[mode_lib->vba.NumberOfActivePlanes] =
+ dst->recout_width; // TODO: or should this be full_recout_width???...maybe only when in hsplit mode?
+ mode_lib->vba.ODMCombineEnabled[mode_lib->vba.NumberOfActivePlanes] =
+ dst->odm_combine;
+ mode_lib->vba.OutputFormat[mode_lib->vba.NumberOfActivePlanes] =
+ (enum output_format_class) (dout->output_format);
+ mode_lib->vba.Output[mode_lib->vba.NumberOfActivePlanes] =
+ (enum output_encoder_class) (dout->output_type);
+
+ if (!dout->dsc_enable)
+ mode_lib->vba.ForcedOutputLinkBPP[mode_lib->vba.NumberOfActivePlanes] = dout->output_bpp;
+ else
+ mode_lib->vba.ForcedOutputLinkBPP[mode_lib->vba.NumberOfActivePlanes] = 0.0;
+
+ mode_lib->vba.OutputLinkDPLanes[mode_lib->vba.NumberOfActivePlanes] =
+ dout->dp_lanes;
+ /* TODO: Needs to be set based on dout->audio.audio_sample_rate_khz/sample_layout */
+ mode_lib->vba.AudioSampleRate[mode_lib->vba.NumberOfActivePlanes] =
+ 44.1 * 1000;
+ mode_lib->vba.AudioSampleLayout[mode_lib->vba.NumberOfActivePlanes] =
+ 1;
+ mode_lib->vba.DRAMClockChangeLatencyOverride = 0.0;
+ mode_lib->vba.DSCEnabled[mode_lib->vba.NumberOfActivePlanes] = dout->dsc_enable;
+ mode_lib->vba.NumberOfDSCSlices[mode_lib->vba.NumberOfActivePlanes] =
+ dout->dsc_slices;
+ mode_lib->vba.DSCInputBitPerComponent[mode_lib->vba.NumberOfActivePlanes] =
+ dout->output_bpc == 0 ? 12 : dout->output_bpc;
+ mode_lib->vba.WritebackEnable[mode_lib->vba.NumberOfActivePlanes] = dout->wb_enable;
+ mode_lib->vba.ActiveWritebacksPerPlane[mode_lib->vba.NumberOfActivePlanes] =
+ dout->num_active_wb;
+ mode_lib->vba.WritebackSourceHeight[mode_lib->vba.NumberOfActivePlanes] =
+ dout->wb.wb_src_height;
+ mode_lib->vba.WritebackSourceWidth[mode_lib->vba.NumberOfActivePlanes] =
+ dout->wb.wb_src_width;
+ mode_lib->vba.WritebackDestinationWidth[mode_lib->vba.NumberOfActivePlanes] =
+ dout->wb.wb_dst_width;
+ mode_lib->vba.WritebackDestinationHeight[mode_lib->vba.NumberOfActivePlanes] =
+ dout->wb.wb_dst_height;
+ mode_lib->vba.WritebackPixelFormat[mode_lib->vba.NumberOfActivePlanes] =
+ (enum source_format_class) (dout->wb.wb_pixel_format);
+ mode_lib->vba.WritebackHTaps[mode_lib->vba.NumberOfActivePlanes] =
+ dout->wb.wb_htaps_luma;
+ mode_lib->vba.WritebackVTaps[mode_lib->vba.NumberOfActivePlanes] =
+ dout->wb.wb_vtaps_luma;
+ mode_lib->vba.WritebackLumaHTaps[mode_lib->vba.NumberOfActivePlanes] =
+ dout->wb.wb_htaps_luma;
+ mode_lib->vba.WritebackLumaVTaps[mode_lib->vba.NumberOfActivePlanes] =
+ dout->wb.wb_vtaps_luma;
+ mode_lib->vba.WritebackChromaHTaps[mode_lib->vba.NumberOfActivePlanes] =
+ dout->wb.wb_htaps_chroma;
+ mode_lib->vba.WritebackChromaVTaps[mode_lib->vba.NumberOfActivePlanes] =
+ dout->wb.wb_vtaps_chroma;
+ mode_lib->vba.WritebackHRatio[mode_lib->vba.NumberOfActivePlanes] =
+ dout->wb.wb_hratio;
+ mode_lib->vba.WritebackVRatio[mode_lib->vba.NumberOfActivePlanes] =
+ dout->wb.wb_vratio;
+
+ mode_lib->vba.DynamicMetadataEnable[mode_lib->vba.NumberOfActivePlanes] =
+ src->dynamic_metadata_enable;
+ mode_lib->vba.DynamicMetadataLinesBeforeActiveRequired[mode_lib->vba.NumberOfActivePlanes] =
+ src->dynamic_metadata_lines_before_active;
+ mode_lib->vba.DynamicMetadataTransmittedBytes[mode_lib->vba.NumberOfActivePlanes] =
+ src->dynamic_metadata_xmit_bytes;
+
+ mode_lib->vba.XFCEnabled[mode_lib->vba.NumberOfActivePlanes] = src->xfc_enable
+ && ip->xfc_supported;
+ mode_lib->vba.XFCSlvChunkSize = src->xfc_params.xfc_slv_chunk_size_bytes;
+ mode_lib->vba.XFCTSlvVupdateOffset = src->xfc_params.xfc_tslv_vupdate_offset_us;
+ mode_lib->vba.XFCTSlvVupdateWidth = src->xfc_params.xfc_tslv_vupdate_width_us;
+ mode_lib->vba.XFCTSlvVreadyOffset = src->xfc_params.xfc_tslv_vready_offset_us;
+ mode_lib->vba.PixelClock[mode_lib->vba.NumberOfActivePlanes] = dst->pixel_rate_mhz;
+ mode_lib->vba.PixelClockBackEnd[mode_lib->vba.NumberOfActivePlanes] = dst->pixel_rate_mhz;
+ mode_lib->vba.DPPCLK[mode_lib->vba.NumberOfActivePlanes] = clks->dppclk_mhz;
+ if (ip->is_line_buffer_bpp_fixed)
+ mode_lib->vba.LBBitPerPixel[mode_lib->vba.NumberOfActivePlanes] =
+ ip->line_buffer_fixed_bpp;
+ else {
+ unsigned int lb_depth;
+
+ switch (scl->lb_depth) {
+ case dm_lb_6:
+ lb_depth = 18;
+ break;
+ case dm_lb_8:
+ lb_depth = 24;
+ break;
+ case dm_lb_10:
+ lb_depth = 30;
+ break;
+ case dm_lb_12:
+ lb_depth = 36;
+ break;
+ case dm_lb_16:
+ lb_depth = 48;
+ break;
+ case dm_lb_19:
+ lb_depth = 57;
+ break;
+ default:
+ lb_depth = 36;
+ }
+ mode_lib->vba.LBBitPerPixel[mode_lib->vba.NumberOfActivePlanes] = lb_depth;
+ }
+ mode_lib->vba.NumberOfCursors[mode_lib->vba.NumberOfActivePlanes] = 0;
+ // The DML spreadsheet assumes that the two cursors utilize the same amount of bandwidth. We'll
+ // calculate things a little more accurately
+ for (k = 0; k < DC__NUM_CURSOR__MAX; ++k) {
+ switch (k) {
+ case 0:
+ mode_lib->vba.CursorBPP[mode_lib->vba.NumberOfActivePlanes][0] =
+ CursorBppEnumToBits(
+ (enum cursor_bpp) (src->cur0_bpp));
+ mode_lib->vba.CursorWidth[mode_lib->vba.NumberOfActivePlanes][0] =
+ src->cur0_src_width;
+ if (src->cur0_src_width > 0)
+ mode_lib->vba.NumberOfCursors[mode_lib->vba.NumberOfActivePlanes]++;
+ break;
+ case 1:
+ mode_lib->vba.CursorBPP[mode_lib->vba.NumberOfActivePlanes][1] =
+ CursorBppEnumToBits(
+ (enum cursor_bpp) (src->cur1_bpp));
+ mode_lib->vba.CursorWidth[mode_lib->vba.NumberOfActivePlanes][1] =
+ src->cur1_src_width;
+ if (src->cur1_src_width > 0)
+ mode_lib->vba.NumberOfCursors[mode_lib->vba.NumberOfActivePlanes]++;
+ break;
+ default:
+ dml_print(
+ "ERROR: Number of cursors specified exceeds supported maximum\n")
+ ;
+ }
+ }
+
+ OTGInstPlane[mode_lib->vba.NumberOfActivePlanes] = dst->otg_inst;
+
+ if (j == 0)
+ mode_lib->vba.UseMaximumVStartup = dst->use_maximum_vstartup;
+ else
+ mode_lib->vba.UseMaximumVStartup = mode_lib->vba.UseMaximumVStartup
+ || dst->use_maximum_vstartup;
+
+ if (dst->odm_combine && !src->is_hsplit)
+ dml_print(
+ "ERROR: ODM Combine is specified but is_hsplit has not be specified for pipe %i\n",
+ j);
+
+ if (src->is_hsplit) {
+ for (k = j + 1; k < mode_lib->vba.cache_num_pipes; ++k) {
+ display_pipe_source_params_st *src_k = &pipes[k].pipe.src;
+
+ if (src_k->is_hsplit && !visited[k]
+ && src->hsplit_grp == src_k->hsplit_grp) {
+ mode_lib->vba.pipe_plane[k] =
+ mode_lib->vba.NumberOfActivePlanes;
+ mode_lib->vba.DPPPerPlane[mode_lib->vba.NumberOfActivePlanes]++;
+ if (mode_lib->vba.SourceScan[mode_lib->vba.NumberOfActivePlanes]
+ == dm_horz)
+ mode_lib->vba.ViewportWidth[mode_lib->vba.NumberOfActivePlanes] +=
+ src_k->viewport_width;
+ else
+ mode_lib->vba.ViewportHeight[mode_lib->vba.NumberOfActivePlanes] +=
+ src_k->viewport_height;
+
+ visited[k] = true;
+ }
+ }
+ }
+
+ if (pipes[k].pipe.src.immediate_flip)
+ mode_lib->vba.ImmediateFlipSupport = true;
+
+ mode_lib->vba.NumberOfActivePlanes++;
+ }
+
+ // handle overlays through BlendingAndTiming
+ // BlendingAndTiming tells you which instance to look at to get timing, the so called 'master'
+
+ for (j = 0; j < mode_lib->vba.NumberOfActivePlanes; ++j)
+ PlaneVisited[j] = false;
+
+ for (j = 0; j < mode_lib->vba.NumberOfActivePlanes; ++j) {
+ for (k = j + 1; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (!PlaneVisited[k] && OTGInstPlane[j] == OTGInstPlane[k]) {
+ // doesn't matter, so choose the smaller one
+ mode_lib->vba.BlendingAndTiming[j] = j;
+ PlaneVisited[j] = true;
+ mode_lib->vba.BlendingAndTiming[k] = j;
+ PlaneVisited[k] = true;
+ }
+ }
+
+ if (!PlaneVisited[j]) {
+ mode_lib->vba.BlendingAndTiming[j] = j;
+ PlaneVisited[j] = true;
+ }
+ }
+
+ // TODO: ODMCombineEnabled => 2 * DPPPerPlane...actually maybe not since all pipes are specified
+ // Do we want the dscclk to automatically be halved? Guess not since the value is specified
+
+ mode_lib->vba.SynchronizedVBlank = pipes[0].pipe.dest.synchronized_vblank_all_planes;
+ for (k = 1; k < mode_lib->vba.cache_num_pipes; ++k)
+ ASSERT(mode_lib->vba.SynchronizedVBlank == pipes[k].pipe.dest.synchronized_vblank_all_planes);
+
+ mode_lib->vba.GPUVMEnable = false;
+ mode_lib->vba.HostVMEnable = false;
+ mode_lib->vba.OverrideGPUVMPageTableLevels = 0;
+ mode_lib->vba.OverrideHostVMPageTableLevels = 0;
+
+ for (k = 0; k < mode_lib->vba.cache_num_pipes; ++k) {
+ mode_lib->vba.GPUVMEnable = mode_lib->vba.GPUVMEnable || !!pipes[k].pipe.src.gpuvm || !!pipes[k].pipe.src.vm;
+ mode_lib->vba.OverrideGPUVMPageTableLevels =
+ (pipes[k].pipe.src.gpuvm_levels_force_en
+ && mode_lib->vba.OverrideGPUVMPageTableLevels
+ < pipes[k].pipe.src.gpuvm_levels_force) ?
+ pipes[k].pipe.src.gpuvm_levels_force :
+ mode_lib->vba.OverrideGPUVMPageTableLevels;
+
+ mode_lib->vba.HostVMEnable = mode_lib->vba.HostVMEnable || !!pipes[k].pipe.src.hostvm || !!pipes[k].pipe.src.vm;
+ mode_lib->vba.OverrideHostVMPageTableLevels =
+ (pipes[k].pipe.src.hostvm_levels_force_en
+ && mode_lib->vba.OverrideHostVMPageTableLevels
+ < pipes[k].pipe.src.hostvm_levels_force) ?
+ pipes[k].pipe.src.hostvm_levels_force :
+ mode_lib->vba.OverrideHostVMPageTableLevels;
+ }
+
+ mode_lib->vba.AllowDRAMSelfRefreshOrDRAMClockChangeInVblank = dm_try_to_allow_self_refresh_and_mclk_switch;
+
+ if (mode_lib->vba.OverrideGPUVMPageTableLevels)
+ mode_lib->vba.GPUVMMaxPageTableLevels = mode_lib->vba.OverrideGPUVMPageTableLevels;
+
+ if (mode_lib->vba.OverrideHostVMPageTableLevels)
+ mode_lib->vba.HostVMMaxPageTableLevels = mode_lib->vba.OverrideHostVMPageTableLevels;
+
+ mode_lib->vba.GPUVMEnable = mode_lib->vba.GPUVMEnable && !!ip->gpuvm_enable;
+ mode_lib->vba.HostVMEnable = mode_lib->vba.HostVMEnable && !!ip->hostvm_enable;
+}
+
+// in wm mode we pull the parameters needed from the display_e2e_pipe_params_st structs
+// rather than working them out as in recalculate_ms
+static void recalculate_params(
+ struct display_mode_lib *mode_lib,
+ const display_e2e_pipe_params_st *pipes,
+ unsigned int num_pipes)
+{
+ // This is only safe to use memcmp because there are non-POD types in struct display_mode_lib
+ if (memcmp(&mode_lib->soc, &mode_lib->vba.soc, sizeof(mode_lib->vba.soc)) != 0
+ || memcmp(&mode_lib->ip, &mode_lib->vba.ip, sizeof(mode_lib->vba.ip)) != 0
+ || num_pipes != mode_lib->vba.cache_num_pipes
+ || memcmp(
+ pipes,
+ mode_lib->vba.cache_pipes,
+ sizeof(display_e2e_pipe_params_st) * num_pipes) != 0) {
+ mode_lib->vba.soc = mode_lib->soc;
+ mode_lib->vba.ip = mode_lib->ip;
+ memcpy(mode_lib->vba.cache_pipes, pipes, sizeof(*pipes) * num_pipes);
+ mode_lib->vba.cache_num_pipes = num_pipes;
+ mode_lib->funcs.recalculate(mode_lib);
+ }
+}
+
+bool Calculate256BBlockSizes(
+ enum source_format_class SourcePixelFormat,
+ enum dm_swizzle_mode SurfaceTiling,
+ unsigned int BytePerPixelY,
+ unsigned int BytePerPixelC,
+ unsigned int *BlockHeight256BytesY,
+ unsigned int *BlockHeight256BytesC,
+ unsigned int *BlockWidth256BytesY,
+ unsigned int *BlockWidth256BytesC)
+{
+ if ((SourcePixelFormat == dm_444_64 || SourcePixelFormat == dm_444_32
+ || SourcePixelFormat == dm_444_16 || SourcePixelFormat == dm_444_8)) {
+ if (SurfaceTiling == dm_sw_linear) {
+ *BlockHeight256BytesY = 1;
+ } else if (SourcePixelFormat == dm_444_64) {
+ *BlockHeight256BytesY = 4;
+ } else if (SourcePixelFormat == dm_444_8) {
+ *BlockHeight256BytesY = 16;
+ } else {
+ *BlockHeight256BytesY = 8;
+ }
+ *BlockWidth256BytesY = 256 / BytePerPixelY / *BlockHeight256BytesY;
+ *BlockHeight256BytesC = 0;
+ *BlockWidth256BytesC = 0;
+ } else {
+ if (SurfaceTiling == dm_sw_linear) {
+ *BlockHeight256BytesY = 1;
+ *BlockHeight256BytesC = 1;
+ } else if (SourcePixelFormat == dm_420_8) {
+ *BlockHeight256BytesY = 16;
+ *BlockHeight256BytesC = 8;
+ } else {
+ *BlockHeight256BytesY = 8;
+ *BlockHeight256BytesC = 8;
+ }
+ *BlockWidth256BytesY = 256 / BytePerPixelY / *BlockHeight256BytesY;
+ *BlockWidth256BytesC = 256 / BytePerPixelC / *BlockHeight256BytesC;
+ }
+ return true;
+}
+
+bool CalculateMinAndMaxPrefetchMode(
+ enum self_refresh_affinity AllowDRAMSelfRefreshOrDRAMClockChangeInVblank,
+ unsigned int *MinPrefetchMode,
+ unsigned int *MaxPrefetchMode)
+{
+ if (AllowDRAMSelfRefreshOrDRAMClockChangeInVblank
+ == dm_neither_self_refresh_nor_mclk_switch) {
+ *MinPrefetchMode = 2;
+ *MaxPrefetchMode = 2;
+ return false;
+ } else if (AllowDRAMSelfRefreshOrDRAMClockChangeInVblank == dm_allow_self_refresh) {
+ *MinPrefetchMode = 1;
+ *MaxPrefetchMode = 1;
+ return false;
+ } else if (AllowDRAMSelfRefreshOrDRAMClockChangeInVblank
+ == dm_allow_self_refresh_and_mclk_switch) {
+ *MinPrefetchMode = 0;
+ *MaxPrefetchMode = 0;
+ return false;
+ } else if (AllowDRAMSelfRefreshOrDRAMClockChangeInVblank
+ == dm_try_to_allow_self_refresh_and_mclk_switch) {
+ *MinPrefetchMode = 0;
+ *MaxPrefetchMode = 2;
+ return false;
+ }
+ *MinPrefetchMode = 0;
+ *MaxPrefetchMode = 2;
+ return true;
+}
+
+void PixelClockAdjustmentForProgressiveToInterlaceUnit(struct display_mode_lib *mode_lib)
+{
+ unsigned int k;
+
+ //Progressive To Interlace Unit Effect
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
+ if (mode_lib->vba.Interlace[k] == 1
+ && mode_lib->vba.ProgressiveToInterlaceUnitInOPP == true) {
+ mode_lib->vba.PixelClock[k] = 2 * mode_lib->vba.PixelClockBackEnd[k];
+ }
+ }
+}
+
+static unsigned int CursorBppEnumToBits(enum cursor_bpp ebpp)
+{
+ switch (ebpp) {
+ case dm_cur_2bit:
+ return 2;
+ case dm_cur_32bit:
+ return 32;
+ case dm_cur_64bit:
+ return 64;
+ default:
+ return 0;
+ }
+}
+
+void ModeSupportAndSystemConfiguration(struct display_mode_lib *mode_lib)
+{
+ soc_bounding_box_st *soc = &mode_lib->vba.soc;
+ unsigned int k;
+ unsigned int total_pipes = 0;
+
+ mode_lib->vba.VoltageLevel = mode_lib->vba.cache_pipes[0].clks_cfg.voltage;
+ mode_lib->vba.ReturnBW = mode_lib->vba.ReturnBWPerState[mode_lib->vba.VoltageLevel];
+ mode_lib->vba.FabricAndDRAMBandwidth = mode_lib->vba.FabricAndDRAMBandwidthPerState[mode_lib->vba.VoltageLevel];
+
+ fetch_socbb_params(mode_lib);
+ fetch_ip_params(mode_lib);
+ fetch_pipe_params(mode_lib);
+
+ mode_lib->vba.DCFCLK = mode_lib->vba.cache_pipes[0].clks_cfg.dcfclk_mhz;
+ mode_lib->vba.SOCCLK = mode_lib->vba.cache_pipes[0].clks_cfg.socclk_mhz;
+ if (mode_lib->vba.cache_pipes[0].clks_cfg.dispclk_mhz > 0.0)
+ mode_lib->vba.DISPCLK = mode_lib->vba.cache_pipes[0].clks_cfg.dispclk_mhz;
+ else
+ mode_lib->vba.DISPCLK = soc->clock_limits[mode_lib->vba.VoltageLevel].dispclk_mhz;
+
+ // Total Available Pipes Support Check
+ for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k)
+ total_pipes += mode_lib->vba.DPPPerPlane[k];
+ ASSERT(total_pipes <= DC__NUM_DPP__MAX);
+}
+
+double CalculateWriteBackDISPCLK(
+ enum source_format_class WritebackPixelFormat,
+ double PixelClock,
+ double WritebackHRatio,
+ double WritebackVRatio,
+ unsigned int WritebackLumaHTaps,
+ unsigned int WritebackLumaVTaps,
+ unsigned int WritebackChromaHTaps,
+ unsigned int WritebackChromaVTaps,
+ double WritebackDestinationWidth,
+ unsigned int HTotal,
+ unsigned int WritebackChromaLineBufferWidth)
+{
+ double CalculateWriteBackDISPCLK = 1.01 * PixelClock * dml_max(
+ dml_ceil(WritebackLumaHTaps / 4.0, 1) / WritebackHRatio,
+ dml_max((WritebackLumaVTaps * dml_ceil(1.0 / WritebackVRatio, 1) * dml_ceil(WritebackDestinationWidth / 4.0, 1)
+ + dml_ceil(WritebackDestinationWidth / 4.0, 1)) / (double) HTotal + dml_ceil(1.0 / WritebackVRatio, 1)
+ * (dml_ceil(WritebackLumaVTaps / 4.0, 1) + 4.0) / (double) HTotal,
+ dml_ceil(1.0 / WritebackVRatio, 1) * WritebackDestinationWidth / (double) HTotal));
+ if (WritebackPixelFormat != dm_444_32) {
+ CalculateWriteBackDISPCLK = dml_max(CalculateWriteBackDISPCLK, 1.01 * PixelClock * dml_max(
+ dml_ceil(WritebackChromaHTaps / 2.0, 1) / (2 * WritebackHRatio),
+ dml_max((WritebackChromaVTaps * dml_ceil(1 / (2 * WritebackVRatio), 1) * dml_ceil(WritebackDestinationWidth / 2.0 / 2.0, 1)
+ + dml_ceil(WritebackDestinationWidth / 2.0 / WritebackChromaLineBufferWidth, 1)) / HTotal
+ + dml_ceil(1 / (2 * WritebackVRatio), 1) * (dml_ceil(WritebackChromaVTaps / 4.0, 1) + 4) / HTotal,
+ dml_ceil(1.0 / (2 * WritebackVRatio), 1) * WritebackDestinationWidth / 2.0 / HTotal)));
+ }
+ return CalculateWriteBackDISPCLK;
+}
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h
new file mode 100644
index 000000000000..0347f74cda3a
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h
@@ -0,0 +1,854 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+
+#ifndef __DML2_DISPLAY_MODE_VBA_H__
+#define __DML2_DISPLAY_MODE_VBA_H__
+
+#include "dml_common_defs.h"
+
+struct display_mode_lib;
+
+void ModeSupportAndSystemConfiguration(struct display_mode_lib *mode_lib);
+
+#define dml_get_attr_decl(attr) double get_##attr(struct display_mode_lib *mode_lib, const display_e2e_pipe_params_st *pipes, unsigned int num_pipes)
+
+dml_get_attr_decl(clk_dcf_deepsleep);
+dml_get_attr_decl(wm_urgent);
+dml_get_attr_decl(wm_memory_trip);
+dml_get_attr_decl(wm_writeback_urgent);
+dml_get_attr_decl(wm_stutter_exit);
+dml_get_attr_decl(wm_stutter_enter_exit);
+dml_get_attr_decl(wm_dram_clock_change);
+dml_get_attr_decl(wm_writeback_dram_clock_change);
+dml_get_attr_decl(wm_xfc_underflow);
+dml_get_attr_decl(stutter_efficiency_no_vblank);
+dml_get_attr_decl(stutter_efficiency);
+dml_get_attr_decl(urgent_latency);
+dml_get_attr_decl(urgent_extra_latency);
+dml_get_attr_decl(nonurgent_latency);
+dml_get_attr_decl(dram_clock_change_latency);
+dml_get_attr_decl(dispclk_calculated);
+dml_get_attr_decl(total_data_read_bw);
+dml_get_attr_decl(return_bw);
+dml_get_attr_decl(tcalc);
+dml_get_attr_decl(fraction_of_urgent_bandwidth);
+dml_get_attr_decl(fraction_of_urgent_bandwidth_imm_flip);
+
+#define dml_get_pipe_attr_decl(attr) double get_##attr(struct display_mode_lib *mode_lib, const display_e2e_pipe_params_st *pipes, unsigned int num_pipes, unsigned int which_pipe)
+
+dml_get_pipe_attr_decl(dsc_delay);
+dml_get_pipe_attr_decl(dppclk_calculated);
+dml_get_pipe_attr_decl(dscclk_calculated);
+dml_get_pipe_attr_decl(min_ttu_vblank);
+dml_get_pipe_attr_decl(vratio_prefetch_l);
+dml_get_pipe_attr_decl(vratio_prefetch_c);
+dml_get_pipe_attr_decl(dst_x_after_scaler);
+dml_get_pipe_attr_decl(dst_y_after_scaler);
+dml_get_pipe_attr_decl(dst_y_per_vm_vblank);
+dml_get_pipe_attr_decl(dst_y_per_row_vblank);
+dml_get_pipe_attr_decl(dst_y_prefetch);
+dml_get_pipe_attr_decl(dst_y_per_vm_flip);
+dml_get_pipe_attr_decl(dst_y_per_row_flip);
+dml_get_pipe_attr_decl(xfc_transfer_delay);
+dml_get_pipe_attr_decl(xfc_precharge_delay);
+dml_get_pipe_attr_decl(xfc_remote_surface_flip_latency);
+dml_get_pipe_attr_decl(xfc_prefetch_margin);
+dml_get_pipe_attr_decl(refcyc_per_vm_group_vblank);
+dml_get_pipe_attr_decl(refcyc_per_vm_group_flip);
+dml_get_pipe_attr_decl(refcyc_per_vm_req_vblank);
+dml_get_pipe_attr_decl(refcyc_per_vm_req_flip);
+
+unsigned int get_vstartup_calculated(
+ struct display_mode_lib *mode_lib,
+ const display_e2e_pipe_params_st *pipes,
+ unsigned int num_pipes,
+ unsigned int which_pipe);
+
+double get_total_immediate_flip_bytes(
+ struct display_mode_lib *mode_lib,
+ const display_e2e_pipe_params_st *pipes,
+ unsigned int num_pipes);
+double get_total_immediate_flip_bw(
+ struct display_mode_lib *mode_lib,
+ const display_e2e_pipe_params_st *pipes,
+ unsigned int num_pipes);
+double get_total_prefetch_bw(
+ struct display_mode_lib *mode_lib,
+ const display_e2e_pipe_params_st *pipes,
+ unsigned int num_pipes);
+unsigned int dml_get_voltage_level(
+ struct display_mode_lib *mode_lib,
+ const display_e2e_pipe_params_st *pipes,
+ unsigned int num_pipes);
+
+void PixelClockAdjustmentForProgressiveToInterlaceUnit(struct display_mode_lib *mode_lib);
+
+bool Calculate256BBlockSizes(
+ enum source_format_class SourcePixelFormat,
+ enum dm_swizzle_mode SurfaceTiling,
+ unsigned int BytePerPixelY,
+ unsigned int BytePerPixelC,
+ unsigned int *BlockHeight256BytesY,
+ unsigned int *BlockHeight256BytesC,
+ unsigned int *BlockWidth256BytesY,
+ unsigned int *BlockWidth256BytesC);
+
+struct vba_vars_st {
+ ip_params_st ip;
+ soc_bounding_box_st soc;
+
+ int maxMpcComb;
+ bool UseMaximumVStartup;
+
+ double WritebackDISPCLK;
+ double DPPCLKUsingSingleDPPLuma;
+ double DPPCLKUsingSingleDPPChroma;
+ double DISPCLKWithRamping;
+ double DISPCLKWithoutRamping;
+ double GlobalDPPCLK;
+ double DISPCLKWithRampingRoundedToDFSGranularity;
+ double DISPCLKWithoutRampingRoundedToDFSGranularity;
+ double MaxDispclkRoundedToDFSGranularity;
+ bool DCCEnabledAnyPlane;
+ double ReturnBandwidthToDCN;
+ unsigned int TotalActiveDPP;
+ unsigned int TotalDCCActiveDPP;
+ double UrgentRoundTripAndOutOfOrderLatency;
+ double StutterPeriod;
+ double FrameTimeForMinFullDETBufferingTime;
+ double AverageReadBandwidth;
+ double TotalRowReadBandwidth;
+ double PartOfBurstThatFitsInROB;
+ double StutterBurstTime;
+ unsigned int NextPrefetchMode;
+ double NextMaxVStartup;
+ double VBlankTime;
+ double SmallestVBlank;
+ double DCFCLKDeepSleepPerPlane[DC__NUM_DPP__MAX];
+ double EffectiveDETPlusLBLinesLuma;
+ double EffectiveDETPlusLBLinesChroma;
+ double UrgentLatencySupportUsLuma;
+ double UrgentLatencySupportUsChroma;
+ unsigned int DSCFormatFactor;
+
+ bool PrefetchModeSupported;
+ enum self_refresh_affinity AllowDRAMSelfRefreshOrDRAMClockChangeInVblank; // Mode Support only
+ double XFCRemoteSurfaceFlipDelay;
+ double TInitXFill;
+ double TslvChk;
+ double SrcActiveDrainRate;
+ bool ImmediateFlipSupported;
+ enum mpc_combine_affinity WhenToDoMPCCombine; // Mode Support only
+
+ bool PrefetchERROR;
+
+ unsigned int VStartupLines;
+ unsigned int ActiveDPPs;
+ unsigned int LBLatencyHidingSourceLinesY;
+ unsigned int LBLatencyHidingSourceLinesC;
+ double ActiveDRAMClockChangeLatencyMargin[DC__NUM_DPP__MAX];
+ double MinActiveDRAMClockChangeMargin;
+ double InitFillLevel;
+ double FinalFillMargin;
+ double FinalFillLevel;
+ double RemainingFillLevel;
+ double TFinalxFill;
+
+ //
+ // SOC Bounding Box Parameters
+ //
+ double SRExitTime;
+ double SREnterPlusExitTime;
+ double UrgentLatencyPixelDataOnly;
+ double UrgentLatencyPixelMixedWithVMData;
+ double UrgentLatencyVMDataOnly;
+ double UrgentLatency; // max of the above three
+ double WritebackLatency;
+ double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelDataOnly; // Mode Support
+ double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData; // Mode Support
+ double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly; // Mode Support
+ double MaxAveragePercentOfIdealSDPPortBWDisplayCanUseInNormalSystemOperation; // Mode Support
+ double MaxAveragePercentOfIdealDRAMBWDisplayCanUseInNormalSystemOperation; // Mode Support
+ double NumberOfChannels;
+ double DRAMChannelWidth;
+ double FabricDatapathToDCNDataReturn;
+ double ReturnBusWidth;
+ double Downspreading;
+ double DISPCLKDPPCLKDSCCLKDownSpreading;
+ double DISPCLKDPPCLKVCOSpeed;
+ double RoundTripPingLatencyCycles;
+ double UrgentOutOfOrderReturnPerChannel;
+ double UrgentOutOfOrderReturnPerChannelPixelDataOnly;
+ double UrgentOutOfOrderReturnPerChannelPixelMixedWithVMData;
+ double UrgentOutOfOrderReturnPerChannelVMDataOnly;
+ unsigned int VMMPageSize;
+ double DRAMClockChangeLatency;
+ double XFCBusTransportTime;
+ bool UseUrgentBurstBandwidth;
+ double XFCXBUFLatencyTolerance;
+
+ //
+ // IP Parameters
+ //
+ unsigned int ROBBufferSizeInKByte;
+ double DETBufferSizeInKByte;
+ double DETBufferSizeInTime;
+ unsigned int DPPOutputBufferPixels;
+ unsigned int OPPOutputBufferLines;
+ unsigned int PixelChunkSizeInKByte;
+ double ReturnBW;
+ bool GPUVMEnable;
+ bool HostVMEnable;
+ unsigned int GPUVMMaxPageTableLevels;
+ unsigned int HostVMMaxPageTableLevels;
+ unsigned int HostVMCachedPageTableLevels;
+ unsigned int OverrideGPUVMPageTableLevels;
+ unsigned int OverrideHostVMPageTableLevels;
+ unsigned int MetaChunkSize;
+ double MinPixelChunkSizeBytes;
+ double MinMetaChunkSizeBytes;
+ unsigned int WritebackChunkSize;
+ bool ODMCapability;
+ unsigned int NumberOfDSC;
+ unsigned int LineBufferSize;
+ unsigned int MaxLineBufferLines;
+ unsigned int WritebackInterfaceLumaBufferSize;
+ unsigned int WritebackInterfaceChromaBufferSize;
+ unsigned int WritebackChromaLineBufferWidth;
+ enum writeback_config WritebackConfiguration;
+ double MaxDCHUBToPSCLThroughput;
+ double MaxPSCLToLBThroughput;
+ unsigned int PTEBufferSizeInRequestsLuma;
+ unsigned int PTEBufferSizeInRequestsChroma;
+ double DISPCLKRampingMargin;
+ unsigned int MaxInterDCNTileRepeaters;
+ bool XFCSupported;
+ double XFCSlvChunkSize;
+ double XFCFillBWOverhead;
+ double XFCFillConstant;
+ double XFCTSlvVupdateOffset;
+ double XFCTSlvVupdateWidth;
+ double XFCTSlvVreadyOffset;
+ double DPPCLKDelaySubtotal;
+ double DPPCLKDelaySCL;
+ double DPPCLKDelaySCLLBOnly;
+ double DPPCLKDelayCNVCFormater;
+ double DPPCLKDelayCNVCCursor;
+ double DISPCLKDelaySubtotal;
+ bool ProgressiveToInterlaceUnitInOPP;
+ // Pipe/Plane Parameters
+ int VoltageLevel;
+ double FabricClock;
+ double DRAMSpeed;
+ double DISPCLK;
+ double SOCCLK;
+ double DCFCLK;
+
+ unsigned int NumberOfActivePlanes;
+ unsigned int NumberOfDSCSlices[DC__NUM_DPP__MAX];
+ unsigned int ViewportWidth[DC__NUM_DPP__MAX];
+ unsigned int ViewportHeight[DC__NUM_DPP__MAX];
+ unsigned int ViewportYStartY[DC__NUM_DPP__MAX];
+ unsigned int ViewportYStartC[DC__NUM_DPP__MAX];
+ unsigned int PitchY[DC__NUM_DPP__MAX];
+ unsigned int PitchC[DC__NUM_DPP__MAX];
+ double HRatio[DC__NUM_DPP__MAX];
+ double VRatio[DC__NUM_DPP__MAX];
+ unsigned int htaps[DC__NUM_DPP__MAX];
+ unsigned int vtaps[DC__NUM_DPP__MAX];
+ unsigned int HTAPsChroma[DC__NUM_DPP__MAX];
+ unsigned int VTAPsChroma[DC__NUM_DPP__MAX];
+ unsigned int HTotal[DC__NUM_DPP__MAX];
+ unsigned int VTotal[DC__NUM_DPP__MAX];
+ unsigned int VTotal_Max[DC__NUM_DPP__MAX];
+ unsigned int VTotal_Min[DC__NUM_DPP__MAX];
+ int DPPPerPlane[DC__NUM_DPP__MAX];
+ double PixelClock[DC__NUM_DPP__MAX];
+ double PixelClockBackEnd[DC__NUM_DPP__MAX];
+ bool DCCEnable[DC__NUM_DPP__MAX];
+ unsigned int DCCMetaPitchY[DC__NUM_DPP__MAX];
+ unsigned int DCCMetaPitchC[DC__NUM_DPP__MAX];
+ enum scan_direction_class SourceScan[DC__NUM_DPP__MAX];
+ enum source_format_class SourcePixelFormat[DC__NUM_DPP__MAX];
+ bool WritebackEnable[DC__NUM_DPP__MAX];
+ unsigned int ActiveWritebacksPerPlane[DC__NUM_DPP__MAX];
+ double WritebackDestinationWidth[DC__NUM_DPP__MAX];
+ double WritebackDestinationHeight[DC__NUM_DPP__MAX];
+ double WritebackSourceHeight[DC__NUM_DPP__MAX];
+ enum source_format_class WritebackPixelFormat[DC__NUM_DPP__MAX];
+ unsigned int WritebackLumaHTaps[DC__NUM_DPP__MAX];
+ unsigned int WritebackLumaVTaps[DC__NUM_DPP__MAX];
+ unsigned int WritebackChromaHTaps[DC__NUM_DPP__MAX];
+ unsigned int WritebackChromaVTaps[DC__NUM_DPP__MAX];
+ double WritebackHRatio[DC__NUM_DPP__MAX];
+ double WritebackVRatio[DC__NUM_DPP__MAX];
+ unsigned int HActive[DC__NUM_DPP__MAX];
+ unsigned int VActive[DC__NUM_DPP__MAX];
+ bool Interlace[DC__NUM_DPP__MAX];
+ enum dm_swizzle_mode SurfaceTiling[DC__NUM_DPP__MAX];
+ unsigned int ScalerRecoutWidth[DC__NUM_DPP__MAX];
+ bool DynamicMetadataEnable[DC__NUM_DPP__MAX];
+ int DynamicMetadataLinesBeforeActiveRequired[DC__NUM_DPP__MAX];
+ unsigned int DynamicMetadataTransmittedBytes[DC__NUM_DPP__MAX];
+ double DCCRate[DC__NUM_DPP__MAX];
+ double AverageDCCCompressionRate;
+ bool ODMCombineEnabled[DC__NUM_DPP__MAX];
+ double OutputBpp[DC__NUM_DPP__MAX];
+ bool DSCEnabled[DC__NUM_DPP__MAX];
+ unsigned int DSCInputBitPerComponent[DC__NUM_DPP__MAX];
+ enum output_format_class OutputFormat[DC__NUM_DPP__MAX];
+ enum output_encoder_class Output[DC__NUM_DPP__MAX];
+ unsigned int BlendingAndTiming[DC__NUM_DPP__MAX];
+ bool SynchronizedVBlank;
+ unsigned int NumberOfCursors[DC__NUM_DPP__MAX];
+ unsigned int CursorWidth[DC__NUM_DPP__MAX][DC__NUM_CURSOR__MAX];
+ unsigned int CursorBPP[DC__NUM_DPP__MAX][DC__NUM_CURSOR__MAX];
+ bool XFCEnabled[DC__NUM_DPP__MAX];
+ bool ScalerEnabled[DC__NUM_DPP__MAX];
+
+ // Intermediates/Informational
+ bool ImmediateFlipSupport;
+ double DETBufferSizeY[DC__NUM_DPP__MAX];
+ double DETBufferSizeC[DC__NUM_DPP__MAX];
+ unsigned int SwathHeightY[DC__NUM_DPP__MAX];
+ unsigned int SwathHeightC[DC__NUM_DPP__MAX];
+ unsigned int LBBitPerPixel[DC__NUM_DPP__MAX];
+ double LastPixelOfLineExtraWatermark;
+ double TotalDataReadBandwidth;
+ unsigned int TotalActiveWriteback;
+ unsigned int EffectiveLBLatencyHidingSourceLinesLuma;
+ unsigned int EffectiveLBLatencyHidingSourceLinesChroma;
+ double BandwidthAvailableForImmediateFlip;
+ unsigned int PrefetchMode[DC__VOLTAGE_STATES + 1][2];
+ unsigned int MinPrefetchMode;
+ unsigned int MaxPrefetchMode;
+ bool AnyLinesForVMOrRowTooLarge;
+ double MaxVStartup;
+ bool IgnoreViewportPositioning;
+ bool ErrorResult[DC__NUM_DPP__MAX];
+ //
+ // Calculated dml_ml->vba.Outputs
+ //
+ double DCFCLKDeepSleep;
+ double UrgentWatermark;
+ double UrgentExtraLatency;
+ double WritebackUrgentWatermark;
+ double StutterExitWatermark;
+ double StutterEnterPlusExitWatermark;
+ double DRAMClockChangeWatermark;
+ double WritebackDRAMClockChangeWatermark;
+ double StutterEfficiency;
+ double StutterEfficiencyNotIncludingVBlank;
+ double NonUrgentLatencyTolerance;
+ double MinActiveDRAMClockChangeLatencySupported;
+
+ // These are the clocks calcuated by the library but they are not actually
+ // used explicitly. They are fetched by tests and then possibly used. The
+ // ultimate values to use are the ones specified by the parameters to DML
+ double DISPCLK_calculated;
+ double DPPCLK_calculated[DC__NUM_DPP__MAX];
+
+ unsigned int VUpdateOffsetPix[DC__NUM_DPP__MAX];
+ double VUpdateWidthPix[DC__NUM_DPP__MAX];
+ double VReadyOffsetPix[DC__NUM_DPP__MAX];
+
+ unsigned int TotImmediateFlipBytes;
+ double TCalc;
+
+ display_e2e_pipe_params_st cache_pipes[DC__NUM_DPP__MAX];
+ unsigned int cache_num_pipes;
+ unsigned int pipe_plane[DC__NUM_DPP__MAX];
+
+ /* vba mode support */
+ /*inputs*/
+ bool SupportGFX7CompatibleTilingIn32bppAnd64bpp;
+ double MaxHSCLRatio;
+ double MaxVSCLRatio;
+ unsigned int MaxNumWriteback;
+ bool WritebackLumaAndChromaScalingSupported;
+ bool Cursor64BppSupport;
+ double DCFCLKPerState[DC__VOLTAGE_STATES + 1];
+ double FabricClockPerState[DC__VOLTAGE_STATES + 1];
+ double SOCCLKPerState[DC__VOLTAGE_STATES + 1];
+ double PHYCLKPerState[DC__VOLTAGE_STATES + 1];
+ double MaxDppclk[DC__VOLTAGE_STATES + 1];
+ double MaxDSCCLK[DC__VOLTAGE_STATES + 1];
+ double DRAMSpeedPerState[DC__VOLTAGE_STATES + 1];
+ double MaxDispclk[DC__VOLTAGE_STATES + 1];
+ int VoltageOverrideLevel;
+
+ /*outputs*/
+ bool ScaleRatioAndTapsSupport;
+ bool SourceFormatPixelAndScanSupport;
+ double TotalBandwidthConsumedGBytePerSecond;
+ bool DCCEnabledInAnyPlane;
+ bool WritebackLatencySupport;
+ bool WritebackModeSupport;
+ bool Writeback10bpc420Supported;
+ bool BandwidthSupport[DC__VOLTAGE_STATES + 1];
+ unsigned int TotalNumberOfActiveWriteback;
+ double CriticalPoint;
+ double ReturnBWToDCNPerState;
+ bool IsErrorResult[DC__VOLTAGE_STATES + 1][2][DC__NUM_DPP__MAX];
+ bool prefetch_vm_bw_valid;
+ bool prefetch_row_bw_valid;
+ bool NumberOfOTGSupport;
+ bool NonsupportedDSCInputBPC;
+ bool WritebackScaleRatioAndTapsSupport;
+ bool CursorSupport;
+ bool PitchSupport;
+ enum dm_validation_status ValidationStatus[DC__VOLTAGE_STATES + 1];
+
+ double WritebackLineBufferLumaBufferSize;
+ double WritebackLineBufferChromaBufferSize;
+ double WritebackMinHSCLRatio;
+ double WritebackMinVSCLRatio;
+ double WritebackMaxHSCLRatio;
+ double WritebackMaxVSCLRatio;
+ double WritebackMaxHSCLTaps;
+ double WritebackMaxVSCLTaps;
+ unsigned int MaxNumDPP;
+ unsigned int MaxNumOTG;
+ double CursorBufferSize;
+ double CursorChunkSize;
+ unsigned int Mode;
+ double OutputLinkDPLanes[DC__NUM_DPP__MAX];
+ double ForcedOutputLinkBPP[DC__NUM_DPP__MAX]; // Mode Support only
+ double ImmediateFlipBW[DC__NUM_DPP__MAX];
+ double MaxMaxVStartup;
+
+ double WritebackLumaVExtra;
+ double WritebackChromaVExtra;
+ double WritebackRequiredDISPCLK;
+ double MaximumSwathWidthSupport;
+ double MaximumSwathWidthInDETBuffer;
+ double MaximumSwathWidthInLineBuffer;
+ double MaxDispclkRoundedDownToDFSGranularity;
+ double MaxDppclkRoundedDownToDFSGranularity;
+ double PlaneRequiredDISPCLKWithoutODMCombine;
+ double PlaneRequiredDISPCLKWithODMCombine;
+ double PlaneRequiredDISPCLK;
+ double TotalNumberOfActiveOTG;
+ double FECOverhead;
+ double EffectiveFECOverhead;
+ double Outbpp;
+ unsigned int OutbppDSC;
+ double TotalDSCUnitsRequired;
+ double bpp;
+ unsigned int slices;
+ double SwathWidthGranularityY;
+ double RoundedUpMaxSwathSizeBytesY;
+ double SwathWidthGranularityC;
+ double RoundedUpMaxSwathSizeBytesC;
+ double EffectiveDETLBLinesLuma;
+ double EffectiveDETLBLinesChroma;
+ double ProjectedDCFCLKDeepSleep;
+ double PDEAndMetaPTEBytesPerFrameY;
+ double PDEAndMetaPTEBytesPerFrameC;
+ unsigned int MetaRowBytesY;
+ unsigned int MetaRowBytesC;
+ unsigned int DPTEBytesPerRowC;
+ unsigned int DPTEBytesPerRowY;
+ double ExtraLatency;
+ double TimeCalc;
+ double TWait;
+ double MaximumReadBandwidthWithPrefetch;
+ double MaximumReadBandwidthWithoutPrefetch;
+ double total_dcn_read_bw_with_flip;
+ double total_dcn_read_bw_with_flip_no_urgent_burst;
+ double FractionOfUrgentBandwidth;
+ double FractionOfUrgentBandwidthImmediateFlip; // Mode Support debugging output
+
+ /* ms locals */
+ double IdealSDPPortBandwidthPerState[DC__VOLTAGE_STATES + 1];
+ unsigned int NoOfDPP[DC__VOLTAGE_STATES + 1][2][DC__NUM_DPP__MAX];
+ int NoOfDPPThisState[DC__NUM_DPP__MAX];
+ bool ODMCombineEnablePerState[DC__VOLTAGE_STATES + 1][DC__NUM_DPP__MAX];
+ unsigned int SwathWidthYThisState[DC__NUM_DPP__MAX];
+ unsigned int SwathHeightCPerState[DC__VOLTAGE_STATES + 1][2][DC__NUM_DPP__MAX];
+ unsigned int SwathHeightYThisState[DC__NUM_DPP__MAX];
+ unsigned int SwathHeightCThisState[DC__NUM_DPP__MAX];
+ double VRatioPreY[DC__VOLTAGE_STATES + 1][2][DC__NUM_DPP__MAX];
+ double VRatioPreC[DC__VOLTAGE_STATES + 1][2][DC__NUM_DPP__MAX];
+ double RequiredPrefetchPixelDataBWLuma[DC__VOLTAGE_STATES + 1][2][DC__NUM_DPP__MAX];
+ double RequiredPrefetchPixelDataBWChroma[DC__VOLTAGE_STATES + 1][2][DC__NUM_DPP__MAX];
+ double RequiredDPPCLK[DC__VOLTAGE_STATES + 1][2][DC__NUM_DPP__MAX];
+ double RequiredDPPCLKThisState[DC__NUM_DPP__MAX];
+ bool PTEBufferSizeNotExceededY[DC__VOLTAGE_STATES + 1][2][DC__NUM_DPP__MAX];
+ bool PTEBufferSizeNotExceededC[DC__VOLTAGE_STATES + 1][2][DC__NUM_DPP__MAX];
+ bool BandwidthWithoutPrefetchSupported[DC__VOLTAGE_STATES + 1];
+ bool PrefetchSupported[DC__VOLTAGE_STATES + 1][2];
+ bool VRatioInPrefetchSupported[DC__VOLTAGE_STATES + 1][2];
+ double RequiredDISPCLK[DC__VOLTAGE_STATES + 1][2];
+ bool DISPCLK_DPPCLK_Support[DC__VOLTAGE_STATES + 1][2];
+ bool TotalAvailablePipesSupport[DC__VOLTAGE_STATES + 1][2];
+ unsigned int TotalNumberOfActiveDPP[DC__VOLTAGE_STATES + 1][2];
+ unsigned int TotalNumberOfDCCActiveDPP[DC__VOLTAGE_STATES + 1][2];
+ bool ModeSupport[DC__VOLTAGE_STATES + 1][2];
+ double ReturnBWPerState[DC__VOLTAGE_STATES + 1];
+ bool DIOSupport[DC__VOLTAGE_STATES + 1];
+ bool NotEnoughDSCUnits[DC__VOLTAGE_STATES + 1];
+ bool DSCCLKRequiredMoreThanSupported[DC__VOLTAGE_STATES + 1];
+ double UrgentRoundTripAndOutOfOrderLatencyPerState[DC__VOLTAGE_STATES + 1];
+ bool ROBSupport[DC__VOLTAGE_STATES + 1];
+ bool PTEBufferSizeNotExceeded[DC__VOLTAGE_STATES + 1][2];
+ bool TotalVerticalActiveBandwidthSupport[DC__VOLTAGE_STATES + 1];
+ double MaxTotalVerticalActiveAvailableBandwidth[DC__VOLTAGE_STATES + 1];
+ double PrefetchBW[DC__NUM_DPP__MAX];
+ double PDEAndMetaPTEBytesPerFrame[DC__NUM_DPP__MAX];
+ double MetaRowBytes[DC__NUM_DPP__MAX];
+ double DPTEBytesPerRow[DC__NUM_DPP__MAX];
+ double PrefetchLinesY[DC__NUM_DPP__MAX];
+ double PrefetchLinesC[DC__NUM_DPP__MAX];
+ unsigned int MaxNumSwY[DC__NUM_DPP__MAX];
+ unsigned int MaxNumSwC[DC__NUM_DPP__MAX];
+ double PrefillY[DC__NUM_DPP__MAX];
+ double PrefillC[DC__NUM_DPP__MAX];
+ double LineTimesForPrefetch[DC__NUM_DPP__MAX];
+ double LinesForMetaPTE[DC__NUM_DPP__MAX];
+ double LinesForMetaAndDPTERow[DC__NUM_DPP__MAX];
+ double MinDPPCLKUsingSingleDPP[DC__NUM_DPP__MAX];
+ unsigned int SwathWidthYSingleDPP[DC__NUM_DPP__MAX];
+ double BytePerPixelInDETY[DC__NUM_DPP__MAX];
+ double BytePerPixelInDETC[DC__NUM_DPP__MAX];
+ bool RequiresDSC[DC__VOLTAGE_STATES + 1][DC__NUM_DPP__MAX];
+ unsigned int NumberOfDSCSlice[DC__VOLTAGE_STATES + 1][DC__NUM_DPP__MAX];
+ double RequiresFEC[DC__VOLTAGE_STATES + 1][DC__NUM_DPP__MAX];
+ double OutputBppPerState[DC__VOLTAGE_STATES + 1][DC__NUM_DPP__MAX];
+ double DSCDelayPerState[DC__VOLTAGE_STATES + 1][DC__NUM_DPP__MAX];
+ bool ViewportSizeSupport[DC__VOLTAGE_STATES + 1];
+ unsigned int Read256BlockHeightY[DC__NUM_DPP__MAX];
+ unsigned int Read256BlockWidthY[DC__NUM_DPP__MAX];
+ unsigned int Read256BlockHeightC[DC__NUM_DPP__MAX];
+ unsigned int Read256BlockWidthC[DC__NUM_DPP__MAX];
+ double MaxSwathHeightY[DC__NUM_DPP__MAX];
+ double MaxSwathHeightC[DC__NUM_DPP__MAX];
+ double MinSwathHeightY[DC__NUM_DPP__MAX];
+ double MinSwathHeightC[DC__NUM_DPP__MAX];
+ double ReadBandwidthLuma[DC__NUM_DPP__MAX];
+ double ReadBandwidthChroma[DC__NUM_DPP__MAX];
+ double ReadBandwidth[DC__NUM_DPP__MAX];
+ double WriteBandwidth[DC__NUM_DPP__MAX];
+ double PSCL_FACTOR[DC__NUM_DPP__MAX];
+ double PSCL_FACTOR_CHROMA[DC__NUM_DPP__MAX];
+ double MaximumVStartup[DC__NUM_DPP__MAX];
+ unsigned int MacroTileWidthY[DC__NUM_DPP__MAX];
+ unsigned int MacroTileWidthC[DC__NUM_DPP__MAX];
+ double AlignedDCCMetaPitch[DC__NUM_DPP__MAX];
+ double AlignedYPitch[DC__NUM_DPP__MAX];
+ double AlignedCPitch[DC__NUM_DPP__MAX];
+ double MaximumSwathWidth[DC__NUM_DPP__MAX];
+ double cursor_bw[DC__NUM_DPP__MAX];
+ double cursor_bw_pre[DC__NUM_DPP__MAX];
+ double Tno_bw[DC__NUM_DPP__MAX];
+ double prefetch_vmrow_bw[DC__NUM_DPP__MAX];
+ double DestinationLinesToRequestVMInImmediateFlip[DC__NUM_DPP__MAX];
+ double DestinationLinesToRequestRowInImmediateFlip[DC__NUM_DPP__MAX];
+ double final_flip_bw[DC__NUM_DPP__MAX];
+ bool ImmediateFlipSupportedForState[DC__VOLTAGE_STATES + 1][2];
+ double WritebackDelay[DC__VOLTAGE_STATES + 1][DC__NUM_DPP__MAX];
+ unsigned int vm_group_bytes[DC__NUM_DPP__MAX];
+ long dpte_group_bytes[DC__NUM_DPP__MAX];
+ unsigned int dpte_row_height[DC__NUM_DPP__MAX];
+ unsigned int meta_req_height[DC__NUM_DPP__MAX];
+ unsigned int meta_req_width[DC__NUM_DPP__MAX];
+ unsigned int meta_row_height[DC__NUM_DPP__MAX];
+ unsigned int meta_row_width[DC__NUM_DPP__MAX];
+ unsigned int dpte_row_height_chroma[DC__NUM_DPP__MAX];
+ unsigned int meta_req_height_chroma[DC__NUM_DPP__MAX];
+ unsigned int meta_req_width_chroma[DC__NUM_DPP__MAX];
+ unsigned int meta_row_height_chroma[DC__NUM_DPP__MAX];
+ unsigned int meta_row_width_chroma[DC__NUM_DPP__MAX];
+ bool ImmediateFlipSupportedForPipe[DC__NUM_DPP__MAX];
+ double meta_row_bw[DC__NUM_DPP__MAX];
+ double dpte_row_bw[DC__NUM_DPP__MAX];
+ double DisplayPipeLineDeliveryTimeLuma[DC__NUM_DPP__MAX]; // WM
+ double DisplayPipeLineDeliveryTimeChroma[DC__NUM_DPP__MAX]; // WM
+ double DisplayPipeRequestDeliveryTimeLuma[DC__NUM_DPP__MAX];
+ double DisplayPipeRequestDeliveryTimeChroma[DC__NUM_DPP__MAX];
+ enum clock_change_support DRAMClockChangeSupport[DC__VOLTAGE_STATES + 1][2];
+ double UrgentBurstFactorCursor[DC__NUM_DPP__MAX];
+ double UrgentBurstFactorCursorPre[DC__NUM_DPP__MAX];
+ double UrgentBurstFactorLuma[DC__NUM_DPP__MAX];
+ double UrgentBurstFactorLumaPre[DC__NUM_DPP__MAX];
+ double UrgentBurstFactorChroma[DC__NUM_DPP__MAX];
+ double UrgentBurstFactorChromaPre[DC__NUM_DPP__MAX];
+
+ bool MPCCombine[DC__VOLTAGE_STATES + 1][2][DC__NUM_DPP__MAX];
+ double SwathWidthCSingleDPP[DC__NUM_DPP__MAX];
+ double MaximumSwathWidthInLineBufferLuma;
+ double MaximumSwathWidthInLineBufferChroma;
+ double MaximumSwathWidthLuma[DC__NUM_DPP__MAX];
+ double MaximumSwathWidthChroma[DC__NUM_DPP__MAX];
+ bool odm_combine_dummy[DC__NUM_DPP__MAX];
+ double dummy1[DC__NUM_DPP__MAX];
+ double dummy2[DC__NUM_DPP__MAX];
+ double dummy3[DC__NUM_DPP__MAX];
+ double dummy4[DC__NUM_DPP__MAX];
+ double dummy5;
+ double dummy6;
+ double dummy7[DC__NUM_DPP__MAX];
+ double dummy8[DC__NUM_DPP__MAX];
+ unsigned int dummyinteger1ms[DC__NUM_DPP__MAX];
+ unsigned int dummyinteger2ms[DC__NUM_DPP__MAX];
+ unsigned int dummyinteger3[DC__NUM_DPP__MAX];
+ unsigned int dummyinteger4;
+ unsigned int dummyinteger5;
+ unsigned int dummyinteger6;
+ unsigned int dummyinteger7;
+ unsigned int dummyinteger8;
+ unsigned int dummyinteger9;
+ unsigned int dummyinteger10;
+ unsigned int dummyinteger11;
+ unsigned int dummyinteger12;
+ bool dummysinglestring;
+ bool SingleDPPViewportSizeSupportPerPlane[DC__NUM_DPP__MAX];
+ double PlaneRequiredDISPCLKWithODMCombine2To1;
+ double PlaneRequiredDISPCLKWithODMCombine4To1;
+ unsigned int TotalNumberOfSingleDPPPlanes[DC__VOLTAGE_STATES + 1][2];
+ bool LinkDSCEnable;
+ bool ODMCombine4To1SupportCheckOK[DC__VOLTAGE_STATES + 1];
+ bool ODMCombineEnableThisState[DC__NUM_DPP__MAX];
+ unsigned int SwathWidthCThisState[DC__NUM_DPP__MAX];
+ bool ViewportSizeSupportPerPlane[DC__NUM_DPP__MAX];
+ double AlignedDCCMetaPitchY[DC__NUM_DPP__MAX];
+ double AlignedDCCMetaPitchC[DC__NUM_DPP__MAX];
+
+ unsigned int NotEnoughUrgentLatencyHiding;
+ unsigned int NotEnoughUrgentLatencyHidingPre;
+ long PTEBufferSizeInRequestsForLuma;
+
+ // Missing from VBA
+ long dpte_group_bytes_chroma;
+ unsigned int vm_group_bytes_chroma;
+ double dst_x_after_scaler;
+ double dst_y_after_scaler;
+ unsigned int VStartupRequiredWhenNotEnoughTimeForDynamicMetadata;
+
+ /* perf locals*/
+ double PrefetchBandwidth[DC__NUM_DPP__MAX];
+ double VInitPreFillY[DC__NUM_DPP__MAX];
+ double VInitPreFillC[DC__NUM_DPP__MAX];
+ unsigned int MaxNumSwathY[DC__NUM_DPP__MAX];
+ unsigned int MaxNumSwathC[DC__NUM_DPP__MAX];
+ unsigned int VStartup[DC__NUM_DPP__MAX];
+ double DSTYAfterScaler[DC__NUM_DPP__MAX];
+ double DSTXAfterScaler[DC__NUM_DPP__MAX];
+ bool AllowDRAMClockChangeDuringVBlank[DC__NUM_DPP__MAX];
+ bool AllowDRAMSelfRefreshDuringVBlank[DC__NUM_DPP__MAX];
+ double VRatioPrefetchY[DC__NUM_DPP__MAX];
+ double VRatioPrefetchC[DC__NUM_DPP__MAX];
+ double DestinationLinesForPrefetch[DC__NUM_DPP__MAX];
+ double DestinationLinesToRequestVMInVBlank[DC__NUM_DPP__MAX];
+ double DestinationLinesToRequestRowInVBlank[DC__NUM_DPP__MAX];
+ double MinTTUVBlank[DC__NUM_DPP__MAX];
+ double BytePerPixelDETY[DC__NUM_DPP__MAX];
+ double BytePerPixelDETC[DC__NUM_DPP__MAX];
+ unsigned int SwathWidthY[DC__NUM_DPP__MAX];
+ unsigned int SwathWidthSingleDPPY[DC__NUM_DPP__MAX];
+ double CursorRequestDeliveryTime[DC__NUM_DPP__MAX];
+ double CursorRequestDeliveryTimePrefetch[DC__NUM_DPP__MAX];
+ double ReadBandwidthPlaneLuma[DC__NUM_DPP__MAX];
+ double ReadBandwidthPlaneChroma[DC__NUM_DPP__MAX];
+ double DisplayPipeLineDeliveryTimeLumaPrefetch[DC__NUM_DPP__MAX];
+ double DisplayPipeLineDeliveryTimeChromaPrefetch[DC__NUM_DPP__MAX];
+ double DisplayPipeRequestDeliveryTimeLumaPrefetch[DC__NUM_DPP__MAX];
+ double DisplayPipeRequestDeliveryTimeChromaPrefetch[DC__NUM_DPP__MAX];
+ double PixelPTEBytesPerRow[DC__NUM_DPP__MAX];
+ double PDEAndMetaPTEBytesFrame[DC__NUM_DPP__MAX];
+ double MetaRowByte[DC__NUM_DPP__MAX];
+ double PrefetchSourceLinesY[DC__NUM_DPP__MAX];
+ double RequiredPrefetchPixDataBWLuma[DC__NUM_DPP__MAX];
+ double RequiredPrefetchPixDataBWChroma[DC__NUM_DPP__MAX];
+ double PrefetchSourceLinesC[DC__NUM_DPP__MAX];
+ double PSCL_THROUGHPUT_LUMA[DC__NUM_DPP__MAX];
+ double PSCL_THROUGHPUT_CHROMA[DC__NUM_DPP__MAX];
+ double DSCCLK_calculated[DC__NUM_DPP__MAX];
+ unsigned int DSCDelay[DC__NUM_DPP__MAX];
+ unsigned int MaxVStartupLines[DC__NUM_DPP__MAX];
+ double DPPCLKUsingSingleDPP[DC__NUM_DPP__MAX];
+ double DPPCLK[DC__NUM_DPP__MAX];
+ unsigned int DCCYMaxUncompressedBlock[DC__NUM_DPP__MAX];
+ unsigned int DCCYMaxCompressedBlock[DC__NUM_DPP__MAX];
+ unsigned int DCCYIndependent64ByteBlock[DC__NUM_DPP__MAX];
+ double MaximumDCCCompressionYSurface[DC__NUM_DPP__MAX];
+ unsigned int BlockHeight256BytesY[DC__NUM_DPP__MAX];
+ unsigned int BlockHeight256BytesC[DC__NUM_DPP__MAX];
+ unsigned int BlockWidth256BytesY[DC__NUM_DPP__MAX];
+ unsigned int BlockWidth256BytesC[DC__NUM_DPP__MAX];
+ double XFCSlaveVUpdateOffset[DC__NUM_DPP__MAX];
+ double XFCSlaveVupdateWidth[DC__NUM_DPP__MAX];
+ double XFCSlaveVReadyOffset[DC__NUM_DPP__MAX];
+ double XFCTransferDelay[DC__NUM_DPP__MAX];
+ double XFCPrechargeDelay[DC__NUM_DPP__MAX];
+ double XFCRemoteSurfaceFlipLatency[DC__NUM_DPP__MAX];
+ double XFCPrefetchMargin[DC__NUM_DPP__MAX];
+ unsigned int dpte_row_width_luma_ub[DC__NUM_DPP__MAX];
+ unsigned int dpte_row_width_chroma_ub[DC__NUM_DPP__MAX];
+ double FullDETBufferingTimeY[DC__NUM_DPP__MAX]; // WM
+ double FullDETBufferingTimeC[DC__NUM_DPP__MAX]; // WM
+ double DST_Y_PER_PTE_ROW_NOM_L[DC__NUM_DPP__MAX];
+ double DST_Y_PER_PTE_ROW_NOM_C[DC__NUM_DPP__MAX];
+ double DST_Y_PER_META_ROW_NOM_L[DC__NUM_DPP__MAX];
+ double TimePerMetaChunkNominal[DC__NUM_DPP__MAX];
+ double TimePerMetaChunkVBlank[DC__NUM_DPP__MAX];
+ double TimePerMetaChunkFlip[DC__NUM_DPP__MAX];
+ unsigned int swath_width_luma_ub[DC__NUM_DPP__MAX];
+ unsigned int swath_width_chroma_ub[DC__NUM_DPP__MAX];
+ unsigned int PixelPTEReqWidthY[DC__NUM_DPP__MAX];
+ unsigned int PixelPTEReqHeightY[DC__NUM_DPP__MAX];
+ unsigned int PTERequestSizeY[DC__NUM_DPP__MAX];
+ unsigned int PixelPTEReqWidthC[DC__NUM_DPP__MAX];
+ unsigned int PixelPTEReqHeightC[DC__NUM_DPP__MAX];
+ unsigned int PTERequestSizeC[DC__NUM_DPP__MAX];
+ double time_per_pte_group_nom_luma[DC__NUM_DPP__MAX];
+ double time_per_pte_group_nom_chroma[DC__NUM_DPP__MAX];
+ double time_per_pte_group_vblank_luma[DC__NUM_DPP__MAX];
+ double time_per_pte_group_vblank_chroma[DC__NUM_DPP__MAX];
+ double time_per_pte_group_flip_luma[DC__NUM_DPP__MAX];
+ double time_per_pte_group_flip_chroma[DC__NUM_DPP__MAX];
+ double TimePerVMGroupVBlank[DC__NUM_DPP__MAX];
+ double TimePerVMGroupFlip[DC__NUM_DPP__MAX];
+ double TimePerVMRequestVBlank[DC__NUM_DPP__MAX];
+ double TimePerVMRequestFlip[DC__NUM_DPP__MAX];
+ unsigned int dpde0_bytes_per_frame_ub_l[DC__NUM_DPP__MAX];
+ unsigned int meta_pte_bytes_per_frame_ub_l[DC__NUM_DPP__MAX];
+ unsigned int dpde0_bytes_per_frame_ub_c[DC__NUM_DPP__MAX];
+ unsigned int meta_pte_bytes_per_frame_ub_c[DC__NUM_DPP__MAX];
+ double LinesToFinishSwathTransferStutterCriticalPlane;
+ unsigned int BytePerPixelYCriticalPlane;
+ double SwathWidthYCriticalPlane;
+ double LinesInDETY[DC__NUM_DPP__MAX];
+ double LinesInDETYRoundedDownToSwath[DC__NUM_DPP__MAX];
+
+ unsigned int SwathWidthSingleDPPC[DC__NUM_DPP__MAX];
+ unsigned int SwathWidthC[DC__NUM_DPP__MAX];
+ unsigned int BytePerPixelY[DC__NUM_DPP__MAX];
+ unsigned int BytePerPixelC[DC__NUM_DPP__MAX];
+ long dummyinteger1;
+ long dummyinteger2;
+ double FinalDRAMClockChangeLatency;
+ double Tdmdl_vm[DC__NUM_DPP__MAX];
+ double Tdmdl[DC__NUM_DPP__MAX];
+ unsigned int ThisVStartup;
+ bool WritebackAllowDRAMClockChangeEndPosition[DC__NUM_DPP__MAX];
+ double DST_Y_PER_META_ROW_NOM_C[DC__NUM_DPP__MAX];
+ double TimePerChromaMetaChunkNominal[DC__NUM_DPP__MAX];
+ double TimePerChromaMetaChunkVBlank[DC__NUM_DPP__MAX];
+ double TimePerChromaMetaChunkFlip[DC__NUM_DPP__MAX];
+ unsigned int DCCCMaxUncompressedBlock[DC__NUM_DPP__MAX];
+ unsigned int DCCCMaxCompressedBlock[DC__NUM_DPP__MAX];
+ unsigned int DCCCIndependent64ByteBlock[DC__NUM_DPP__MAX];
+ double VStartupMargin;
+
+ /* Missing from VBA */
+ unsigned int MaximumMaxVStartupLines;
+ double FabricAndDRAMBandwidth;
+ double LinesInDETLuma;
+ double LinesInDETChroma;
+ unsigned int ImmediateFlipBytes[DC__NUM_DPP__MAX];
+ unsigned int LinesInDETC[DC__NUM_DPP__MAX];
+ unsigned int LinesInDETCRoundedDownToSwath[DC__NUM_DPP__MAX];
+ double UrgentLatencySupportUsPerState[DC__VOLTAGE_STATES + 1][2][DC__NUM_DPP__MAX];
+ double UrgentLatencySupportUs[DC__NUM_DPP__MAX];
+ double FabricAndDRAMBandwidthPerState[DC__VOLTAGE_STATES + 1];
+ bool UrgentLatencySupport[DC__VOLTAGE_STATES + 1][2];
+ unsigned int SwathWidthYPerState[DC__VOLTAGE_STATES + 1][2][DC__NUM_DPP__MAX];
+ unsigned int SwathHeightYPerState[DC__VOLTAGE_STATES + 1][2][DC__NUM_DPP__MAX];
+ double qual_row_bw[DC__NUM_DPP__MAX];
+ double prefetch_row_bw[DC__NUM_DPP__MAX];
+ double prefetch_vm_bw[DC__NUM_DPP__MAX];
+
+ double PTEGroupSize;
+ unsigned int PDEProcessingBufIn64KBReqs;
+
+ double MaxTotalVActiveRDBandwidth;
+ double MinUrgentLatencySupportUs;
+ double MinFullDETBufferingTime;
+ double AverageReadBandwidthGBytePerSecond;
+ bool FirstMainPlane;
+
+ unsigned int ViewportWidthChroma[DC__NUM_DPP__MAX];
+ unsigned int ViewportHeightChroma[DC__NUM_DPP__MAX];
+ double HRatioChroma[DC__NUM_DPP__MAX];
+ double VRatioChroma[DC__NUM_DPP__MAX];
+ long WritebackSourceWidth[DC__NUM_DPP__MAX];
+
+ bool ModeIsSupported;
+ bool ODMCombine4To1Supported;
+
+ unsigned int SurfaceHeightY[DC__NUM_DPP__MAX];
+ unsigned int SurfaceHeightC[DC__NUM_DPP__MAX];
+ unsigned int WritebackHTaps[DC__NUM_DPP__MAX];
+ unsigned int WritebackVTaps[DC__NUM_DPP__MAX];
+ bool DSCEnable[DC__NUM_DPP__MAX];
+
+ double DRAMClockChangeLatencyOverride;
+
+ double GPUVMMinPageSize;
+ double HostVMMinPageSize;
+
+ bool MPCCombineEnable[DC__NUM_DPP__MAX];
+ unsigned int HostVMMaxNonCachedPageTableLevels;
+ bool DynamicMetadataVMEnabled;
+ double WritebackInterfaceBufferSize;
+ double WritebackLineBufferSize;
+
+ double DCCRateLuma[DC__NUM_DPP__MAX];
+ double DCCRateChroma[DC__NUM_DPP__MAX];
+
+ double PHYCLKD18PerState[DC__VOLTAGE_STATES + 1];
+ int MinVoltageLevel;
+ int MaxVoltageLevel;
+
+ bool WritebackSupportInterleaveAndUsingWholeBufferForASingleStream;
+ bool NumberOfHDMIFRLSupport;
+ unsigned int MaxNumHDMIFRLOutputs;
+ int AudioSampleRate[DC__NUM_DPP__MAX];
+ int AudioSampleLayout[DC__NUM_DPP__MAX];
+};
+
+bool CalculateMinAndMaxPrefetchMode(
+ enum self_refresh_affinity AllowDRAMSelfRefreshOrDRAMClockChangeInVblank,
+ unsigned int *MinPrefetchMode,
+ unsigned int *MaxPrefetchMode);
+
+double CalculateWriteBackDISPCLK(
+ enum source_format_class WritebackPixelFormat,
+ double PixelClock,
+ double WritebackHRatio,
+ double WritebackVRatio,
+ unsigned int WritebackLumaHTaps,
+ unsigned int WritebackLumaVTaps,
+ unsigned int WritebackChromaHTaps,
+ unsigned int WritebackChromaVTaps,
+ double WritebackDestinationWidth,
+ unsigned int HTotal,
+ unsigned int WritebackChromaLineBufferWidth);
+
+#endif /* _DML2_DISPLAY_MODE_VBA_H_ */
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h b/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h
index e8ce08567cd8..eca140da13d8 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h
@@ -129,4 +129,12 @@ static inline unsigned int dml_round_to_multiple(unsigned int num,
else
return (num - remainder);
}
+static inline double dml_abs(double a)
+{
+ if (a > 0)
+ return a;
+ else
+ return (a*(-1));
+}
+
#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/Makefile b/drivers/gpu/drm/amd/display/dc/dsc/Makefile
new file mode 100644
index 000000000000..e019cd9447e8
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dsc/Makefile
@@ -0,0 +1,21 @@
+#
+# Makefile for the 'dsc' sub-component of DAL.
+
+ifneq ($(call cc-option, -mpreferred-stack-boundary=4),)
+ cc_stack_align := -mpreferred-stack-boundary=4
+else ifneq ($(call cc-option, -mstack-alignment=16),)
+ cc_stack_align := -mstack-alignment=16
+endif
+
+dsc_ccflags := -mhard-float -msse $(cc_stack_align)
+
+CFLAGS_rc_calc.o := $(dsc_ccflags)
+CFLAGS_rc_calc_dpi.o := $(dsc_ccflags)
+CFLAGS_codec_main_amd.o := $(dsc_ccflags)
+CFLAGS_dc_dsc.o := $(dsc_ccflags)
+
+DSC = dc_dsc.o rc_calc.o rc_calc_dpi.o
+
+AMD_DAL_DSC = $(addprefix $(AMDDALPATH)/dc/dsc/,$(DSC))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_DSC)
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c
new file mode 100644
index 000000000000..ef5f84a144c3
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c
@@ -0,0 +1,858 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Author: AMD
+ */
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+#include "dc.h"
+#include "core_types.h"
+#include "dsc.h"
+#include <drm/drm_dp_helper.h>
+
+/* This module's internal functions */
+
+static bool dsc_buff_block_size_from_dpcd(int dpcd_buff_block_size, int *buff_block_size)
+{
+
+ switch (dpcd_buff_block_size) {
+ case DP_DSC_RC_BUF_BLK_SIZE_1:
+ *buff_block_size = 1024;
+ break;
+ case DP_DSC_RC_BUF_BLK_SIZE_4:
+ *buff_block_size = 4 * 1024;
+ break;
+ case DP_DSC_RC_BUF_BLK_SIZE_16:
+ *buff_block_size = 16 * 1024;
+ break;
+ case DP_DSC_RC_BUF_BLK_SIZE_64:
+ *buff_block_size = 64 * 1024;
+ break;
+ default: {
+ dm_error("%s: DPCD DSC buffer size not recognized.\n", __func__);
+ return false;
+ }
+ }
+
+ return true;
+}
+
+
+static bool dsc_line_buff_depth_from_dpcd(int dpcd_line_buff_bit_depth, int *line_buff_bit_depth)
+{
+ if (0 <= dpcd_line_buff_bit_depth && dpcd_line_buff_bit_depth <= 7)
+ *line_buff_bit_depth = dpcd_line_buff_bit_depth + 9;
+ else if (dpcd_line_buff_bit_depth == 8)
+ *line_buff_bit_depth = 8;
+ else {
+ dm_error("%s: DPCD DSC buffer depth not recognized.\n", __func__);
+ return false;
+ }
+
+ return true;
+}
+
+
+static bool dsc_throughput_from_dpcd(int dpcd_throughput, int *throughput)
+{
+ switch (dpcd_throughput) {
+ case DP_DSC_THROUGHPUT_MODE_0_UPSUPPORTED:
+ *throughput = 0;
+ break;
+ case DP_DSC_THROUGHPUT_MODE_0_170:
+ *throughput = 170;
+ break;
+ case DP_DSC_THROUGHPUT_MODE_0_340:
+ *throughput = 340;
+ break;
+ case DP_DSC_THROUGHPUT_MODE_0_400:
+ *throughput = 400;
+ break;
+ case DP_DSC_THROUGHPUT_MODE_0_450:
+ *throughput = 450;
+ break;
+ case DP_DSC_THROUGHPUT_MODE_0_500:
+ *throughput = 500;
+ break;
+ case DP_DSC_THROUGHPUT_MODE_0_550:
+ *throughput = 550;
+ break;
+ case DP_DSC_THROUGHPUT_MODE_0_600:
+ *throughput = 600;
+ break;
+ case DP_DSC_THROUGHPUT_MODE_0_650:
+ *throughput = 650;
+ break;
+ case DP_DSC_THROUGHPUT_MODE_0_700:
+ *throughput = 700;
+ break;
+ case DP_DSC_THROUGHPUT_MODE_0_750:
+ *throughput = 750;
+ break;
+ case DP_DSC_THROUGHPUT_MODE_0_800:
+ *throughput = 800;
+ break;
+ case DP_DSC_THROUGHPUT_MODE_0_850:
+ *throughput = 850;
+ break;
+ case DP_DSC_THROUGHPUT_MODE_0_900:
+ *throughput = 900;
+ break;
+ case DP_DSC_THROUGHPUT_MODE_0_950:
+ *throughput = 950;
+ break;
+ case DP_DSC_THROUGHPUT_MODE_0_1000:
+ *throughput = 1000;
+ break;
+ default: {
+ dm_error("%s: DPCD DSC throughput mode not recognized.\n", __func__);
+ return false;
+ }
+ }
+
+ return true;
+}
+
+
+static bool dsc_bpp_increment_div_from_dpcd(int bpp_increment_dpcd, uint32_t *bpp_increment_div)
+{
+
+ switch (bpp_increment_dpcd) {
+ case 0:
+ *bpp_increment_div = 16;
+ break;
+ case 1:
+ *bpp_increment_div = 8;
+ break;
+ case 2:
+ *bpp_increment_div = 4;
+ break;
+ case 3:
+ *bpp_increment_div = 2;
+ break;
+ case 4:
+ *bpp_increment_div = 1;
+ break;
+ default: {
+ dm_error("%s: DPCD DSC bits-per-pixel increment not recognized.\n", __func__);
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static void get_dsc_enc_caps(
+ const struct dc *dc,
+ struct dsc_enc_caps *dsc_enc_caps,
+ int pixel_clock_100Hz)
+{
+ // This is a static HW query, so we can use any DSC
+ struct display_stream_compressor *dsc = dc->res_pool->dscs[0];
+
+ memset(dsc_enc_caps, 0, sizeof(struct dsc_enc_caps));
+ if (dsc)
+ dsc->funcs->dsc_get_enc_caps(dsc_enc_caps, pixel_clock_100Hz);
+}
+
+/* Returns 'false' if no intersection was found for at least one capablity.
+ * It also implicitly validates some sink caps against invalid value of zero.
+ */
+static bool intersect_dsc_caps(
+ const struct dsc_dec_dpcd_caps *dsc_sink_caps,
+ const struct dsc_enc_caps *dsc_enc_caps,
+ enum dc_pixel_encoding pixel_encoding,
+ struct dsc_enc_caps *dsc_common_caps)
+{
+ int32_t max_slices;
+ int32_t total_sink_throughput;
+
+ memset(dsc_common_caps, 0, sizeof(struct dsc_enc_caps));
+
+ dsc_common_caps->dsc_version = min(dsc_sink_caps->dsc_version, dsc_enc_caps->dsc_version);
+ if (!dsc_common_caps->dsc_version)
+ return false;
+
+ dsc_common_caps->slice_caps.bits.NUM_SLICES_1 = dsc_sink_caps->slice_caps1.bits.NUM_SLICES_1 && dsc_enc_caps->slice_caps.bits.NUM_SLICES_1;
+ dsc_common_caps->slice_caps.bits.NUM_SLICES_2 = dsc_sink_caps->slice_caps1.bits.NUM_SLICES_2 && dsc_enc_caps->slice_caps.bits.NUM_SLICES_2;
+ dsc_common_caps->slice_caps.bits.NUM_SLICES_4 = dsc_sink_caps->slice_caps1.bits.NUM_SLICES_4 && dsc_enc_caps->slice_caps.bits.NUM_SLICES_4;
+ dsc_common_caps->slice_caps.bits.NUM_SLICES_8 = dsc_sink_caps->slice_caps1.bits.NUM_SLICES_8 && dsc_enc_caps->slice_caps.bits.NUM_SLICES_8;
+ if (!dsc_common_caps->slice_caps.raw)
+ return false;
+
+ dsc_common_caps->lb_bit_depth = min(dsc_sink_caps->lb_bit_depth, dsc_enc_caps->lb_bit_depth);
+ if (!dsc_common_caps->lb_bit_depth)
+ return false;
+
+ dsc_common_caps->is_block_pred_supported = dsc_sink_caps->is_block_pred_supported && dsc_enc_caps->is_block_pred_supported;
+
+ dsc_common_caps->color_formats.raw = dsc_sink_caps->color_formats.raw & dsc_enc_caps->color_formats.raw;
+ if (!dsc_common_caps->color_formats.raw)
+ return false;
+
+ dsc_common_caps->color_depth.raw = dsc_sink_caps->color_depth.raw & dsc_enc_caps->color_depth.raw;
+ if (!dsc_common_caps->color_depth.raw)
+ return false;
+
+ max_slices = 0;
+ if (dsc_common_caps->slice_caps.bits.NUM_SLICES_1)
+ max_slices = 1;
+
+ if (dsc_common_caps->slice_caps.bits.NUM_SLICES_2)
+ max_slices = 2;
+
+ if (dsc_common_caps->slice_caps.bits.NUM_SLICES_4)
+ max_slices = 4;
+
+ total_sink_throughput = max_slices * dsc_sink_caps->throughput_mode_0_mps;
+ if (pixel_encoding == PIXEL_ENCODING_YCBCR422 || pixel_encoding == PIXEL_ENCODING_YCBCR420)
+ total_sink_throughput = max_slices * dsc_sink_caps->throughput_mode_1_mps;
+
+ dsc_common_caps->max_total_throughput_mps = min(total_sink_throughput, dsc_enc_caps->max_total_throughput_mps);
+
+ dsc_common_caps->max_slice_width = min(dsc_sink_caps->max_slice_width, dsc_enc_caps->max_slice_width);
+ if (!dsc_common_caps->max_slice_width)
+ return false;
+
+ dsc_common_caps->bpp_increment_div = min(dsc_sink_caps->bpp_increment_div, dsc_enc_caps->bpp_increment_div);
+
+ // TODO DSC: Remove this workaround for N422 and 420 once it's fixed, or move it to get_dsc_encoder_caps()
+ if (pixel_encoding == PIXEL_ENCODING_YCBCR422 || pixel_encoding == PIXEL_ENCODING_YCBCR420)
+ dsc_common_caps->bpp_increment_div = min(dsc_common_caps->bpp_increment_div, (uint32_t)8);
+
+ return true;
+}
+
+struct dc_dsc_policy {
+ bool use_min_slices_h;
+ int max_slices_h; // Maximum available if 0
+ int num_slices_v;
+ int max_target_bpp;
+ int min_target_bpp; // Minimum target bits per pixel
+};
+
+static inline uint32_t dsc_div_by_10_round_up(uint32_t value)
+{
+ return (value + 9) / 10;
+}
+
+static inline uint32_t calc_dsc_bpp_x16(uint32_t stream_bandwidth_kbps, uint32_t pix_clk_100hz, uint32_t bpp_increment_div)
+{
+ uint32_t dsc_target_bpp_x16;
+ float f_dsc_target_bpp;
+ float f_stream_bandwidth_100bps = stream_bandwidth_kbps * 10.0f;
+ uint32_t precision = bpp_increment_div; // bpp_increment_div is actually precision
+
+ f_dsc_target_bpp = f_stream_bandwidth_100bps / pix_clk_100hz;
+
+ // Round down to the nearest precision stop to bring it into DSC spec range
+ dsc_target_bpp_x16 = (uint32_t)(f_dsc_target_bpp * precision);
+ dsc_target_bpp_x16 = (dsc_target_bpp_x16 * 16) / precision;
+
+ return dsc_target_bpp_x16;
+}
+
+const struct dc_dsc_policy dsc_policy = {
+ .use_min_slices_h = true, // DSC Policy: Use minimum number of slices that fits the pixel clock
+ .max_slices_h = 0, // DSC Policy: Use max available slices (in our case 4 for or 8, depending on the mode)
+ /* DSC Policy: Number of vertical slices set to 2 for no particular reason.
+ * Seems small enough to not affect the quality too much, while still providing some error
+ * propagation control (which may also help debugging).
+ */
+ .num_slices_v = 16,
+ .max_target_bpp = 16,
+ .min_target_bpp = 8,
+};
+
+
+/* Get DSC bandwidth range based on [min_bpp, max_bpp] target bitrate range, and timing's pixel clock
+ * and uncompressed bandwidth.
+ */
+static void get_dsc_bandwidth_range(
+ const uint32_t min_bpp,
+ const uint32_t max_bpp,
+ const struct dsc_enc_caps *dsc_caps,
+ const struct dc_crtc_timing *timing,
+ struct dc_dsc_bw_range *range)
+{
+ /* native stream bandwidth */
+ range->stream_kbps = dc_bandwidth_in_kbps_from_timing(timing);
+
+ /* max dsc target bpp */
+ range->max_kbps = dsc_div_by_10_round_up(max_bpp * timing->pix_clk_100hz);
+ range->max_target_bpp_x16 = max_bpp * 16;
+ if (range->max_kbps > range->stream_kbps) {
+ /* max dsc target bpp is capped to native bandwidth */
+ range->max_kbps = range->stream_kbps;
+ range->max_target_bpp_x16 = calc_dsc_bpp_x16(range->stream_kbps, timing->pix_clk_100hz, dsc_caps->bpp_increment_div);
+ }
+
+ /* min dsc target bpp */
+ range->min_kbps = dsc_div_by_10_round_up(min_bpp * timing->pix_clk_100hz);
+ range->min_target_bpp_x16 = min_bpp * 16;
+ if (range->min_kbps > range->max_kbps) {
+ /* min dsc target bpp is capped to max dsc bandwidth*/
+ range->min_kbps = range->max_kbps;
+ range->min_target_bpp_x16 = range->max_target_bpp_x16;
+ }
+}
+
+
+/* Decides if DSC should be used and calculates target bpp if it should, applying DSC policy.
+ *
+ * Returns:
+ * - 'true' if DSC was required by policy and was successfully applied
+ * - 'false' if DSC was not necessary (e.g. if uncompressed stream fits 'target_bandwidth_kbps'),
+ * or if it couldn't be applied based on DSC policy.
+ */
+static bool decide_dsc_target_bpp_x16(
+ const struct dc_dsc_policy *policy,
+ const struct dsc_enc_caps *dsc_common_caps,
+ const int target_bandwidth_kbps,
+ const struct dc_crtc_timing *timing,
+ int *target_bpp_x16)
+{
+ bool should_use_dsc = false;
+ struct dc_dsc_bw_range range;
+
+ memset(&range, 0, sizeof(range));
+
+ get_dsc_bandwidth_range(policy->min_target_bpp, policy->max_target_bpp,
+ dsc_common_caps, timing, &range);
+ if (target_bandwidth_kbps >= range.stream_kbps) {
+ /* enough bandwidth without dsc */
+ *target_bpp_x16 = 0;
+ should_use_dsc = false;
+ } else if (target_bandwidth_kbps >= range.max_kbps) {
+ /* use max target bpp allowed */
+ *target_bpp_x16 = range.max_target_bpp_x16;
+ should_use_dsc = true;
+ } else if (target_bandwidth_kbps >= range.min_kbps) {
+ /* use target bpp that can take entire target bandwidth */
+ *target_bpp_x16 = calc_dsc_bpp_x16(target_bandwidth_kbps, timing->pix_clk_100hz, dsc_common_caps->bpp_increment_div);
+ should_use_dsc = true;
+ } else {
+ /* not enough bandwidth to fulfill minimum requirement */
+ *target_bpp_x16 = 0;
+ should_use_dsc = false;
+ }
+
+ return should_use_dsc;
+}
+
+#define MIN_AVAILABLE_SLICES_SIZE 4
+
+static int get_available_dsc_slices(union dsc_enc_slice_caps slice_caps, int *available_slices)
+{
+ int idx = 0;
+
+ memset(available_slices, -1, MIN_AVAILABLE_SLICES_SIZE);
+
+ if (slice_caps.bits.NUM_SLICES_1)
+ available_slices[idx++] = 1;
+
+ if (slice_caps.bits.NUM_SLICES_2)
+ available_slices[idx++] = 2;
+
+ if (slice_caps.bits.NUM_SLICES_4)
+ available_slices[idx++] = 4;
+
+ if (slice_caps.bits.NUM_SLICES_8)
+ available_slices[idx++] = 8;
+
+ return idx;
+}
+
+
+static int get_max_dsc_slices(union dsc_enc_slice_caps slice_caps)
+{
+ int max_slices = 0;
+ int available_slices[MIN_AVAILABLE_SLICES_SIZE];
+ int end_idx = get_available_dsc_slices(slice_caps, &available_slices[0]);
+
+ if (end_idx > 0)
+ max_slices = available_slices[end_idx - 1];
+
+ return max_slices;
+}
+
+
+// Increment sice number in available sice numbers stops if possible, or just increment if not
+static int inc_num_slices(union dsc_enc_slice_caps slice_caps, int num_slices)
+{
+ // Get next bigger num slices available in common caps
+ int available_slices[MIN_AVAILABLE_SLICES_SIZE];
+ int end_idx;
+ int i;
+ int new_num_slices = num_slices;
+
+ end_idx = get_available_dsc_slices(slice_caps, &available_slices[0]);
+ if (end_idx == 0) {
+ // No available slices found
+ new_num_slices++;
+ return new_num_slices;
+ }
+
+ // Numbers of slices found - get the next bigger number
+ for (i = 0; i < end_idx; i++) {
+ if (new_num_slices < available_slices[i]) {
+ new_num_slices = available_slices[i];
+ break;
+ }
+ }
+
+ if (new_num_slices == num_slices) // No biger number of slices found
+ new_num_slices++;
+
+ return new_num_slices;
+}
+
+
+// Decrement sice number in available sice numbers stops if possible, or just decrement if not. Stop at zero.
+static int dec_num_slices(union dsc_enc_slice_caps slice_caps, int num_slices)
+{
+ // Get next bigger num slices available in common caps
+ int available_slices[MIN_AVAILABLE_SLICES_SIZE];
+ int end_idx;
+ int i;
+ int new_num_slices = num_slices;
+
+ end_idx = get_available_dsc_slices(slice_caps, &available_slices[0]);
+ if (end_idx == 0 && new_num_slices > 0) {
+ // No numbers of slices found
+ new_num_slices++;
+ return new_num_slices;
+ }
+
+ // Numbers of slices found - get the next smaller number
+ for (i = end_idx - 1; i >= 0; i--) {
+ if (new_num_slices > available_slices[i]) {
+ new_num_slices = available_slices[i];
+ break;
+ }
+ }
+
+ if (new_num_slices == num_slices) {
+ // No smaller number of slices found
+ new_num_slices--;
+ if (new_num_slices < 0)
+ new_num_slices = 0;
+ }
+
+ return new_num_slices;
+}
+
+
+// Choose next bigger number of slices if the requested number of slices is not available
+static int fit_num_slices_up(union dsc_enc_slice_caps slice_caps, int num_slices)
+{
+ // Get next bigger num slices available in common caps
+ int available_slices[MIN_AVAILABLE_SLICES_SIZE];
+ int end_idx;
+ int i;
+ int new_num_slices = num_slices;
+
+ end_idx = get_available_dsc_slices(slice_caps, &available_slices[0]);
+ if (end_idx == 0) {
+ // No available slices found
+ new_num_slices++;
+ return new_num_slices;
+ }
+
+ // Numbers of slices found - get the equal or next bigger number
+ for (i = 0; i < end_idx; i++) {
+ if (new_num_slices <= available_slices[i]) {
+ new_num_slices = available_slices[i];
+ break;
+ }
+ }
+
+ return new_num_slices;
+}
+
+
+/* Attempts to set DSC configuration for the stream, applying DSC policy.
+ * Returns 'true' if successful or 'false' if not.
+ *
+ * Parameters:
+ *
+ * dsc_sink_caps - DSC sink decoder capabilities (from DPCD)
+ *
+ * dsc_enc_caps - DSC encoder capabilities
+ *
+ * target_bandwidth_kbps - Target bandwidth to fit the stream into.
+ * If 0, do not calculate target bpp.
+ *
+ * timing - The stream timing to fit into 'target_bandwidth_kbps' or apply
+ * maximum compression to, if 'target_badwidth == 0'
+ *
+ * dsc_cfg - DSC configuration to use if it was possible to come up with
+ * one for the given inputs.
+ * The target bitrate after DSC can be calculated by multiplying
+ * dsc_cfg.bits_per_pixel (in U6.4 format) by pixel rate, e.g.
+ *
+ * dsc_stream_bitrate_kbps = (int)ceil(timing->pix_clk_khz * dsc_cfg.bits_per_pixel / 16.0);
+ */
+static bool setup_dsc_config(
+ const struct dsc_dec_dpcd_caps *dsc_sink_caps,
+ const struct dsc_enc_caps *dsc_enc_caps,
+ int target_bandwidth_kbps,
+ const struct dc_crtc_timing *timing,
+ struct dc_dsc_config *dsc_cfg)
+{
+ struct dsc_enc_caps dsc_common_caps;
+ int max_slices_h;
+ int min_slices_h;
+ int num_slices_h;
+ int pic_width;
+ int slice_width;
+ int target_bpp;
+ int sink_per_slice_throughput_mps;
+ int branch_max_throughput_mps = 0;
+ bool is_dsc_possible = false;
+ int num_slices_v;
+ int pic_height;
+
+ memset(dsc_cfg, 0, sizeof(struct dc_dsc_config));
+
+ pic_width = timing->h_addressable + timing->h_border_left + timing->h_border_right;
+ pic_height = timing->v_addressable + timing->v_border_top + timing->v_border_bottom;
+
+ if (!dsc_sink_caps->is_dsc_supported)
+ goto done;
+
+ if (dsc_sink_caps->branch_max_line_width && dsc_sink_caps->branch_max_line_width < pic_width)
+ goto done;
+
+ // Intersect decoder with encoder DSC caps and validate DSC settings
+ is_dsc_possible = intersect_dsc_caps(dsc_sink_caps, dsc_enc_caps, timing->pixel_encoding, &dsc_common_caps);
+ if (!is_dsc_possible)
+ goto done;
+
+ if (target_bandwidth_kbps > 0) {
+ is_dsc_possible = decide_dsc_target_bpp_x16(&dsc_policy, &dsc_common_caps, target_bandwidth_kbps, timing, &target_bpp);
+ dsc_cfg->bits_per_pixel = target_bpp;
+ }
+ if (!is_dsc_possible)
+ goto done;
+
+ sink_per_slice_throughput_mps = 0;
+
+ // Validate available DSC settings against the mode timing
+
+ // Validate color format (and pick up the throughput values)
+ dsc_cfg->ycbcr422_simple = false;
+ switch (timing->pixel_encoding) {
+ case PIXEL_ENCODING_RGB:
+ is_dsc_possible = (bool)dsc_common_caps.color_formats.bits.RGB;
+ sink_per_slice_throughput_mps = dsc_sink_caps->throughput_mode_0_mps;
+ branch_max_throughput_mps = dsc_sink_caps->branch_overall_throughput_0_mps;
+ break;
+ case PIXEL_ENCODING_YCBCR444:
+ is_dsc_possible = (bool)dsc_common_caps.color_formats.bits.YCBCR_444;
+ sink_per_slice_throughput_mps = dsc_sink_caps->throughput_mode_0_mps;
+ branch_max_throughput_mps = dsc_sink_caps->branch_overall_throughput_0_mps;
+ break;
+ case PIXEL_ENCODING_YCBCR422:
+ is_dsc_possible = (bool)dsc_common_caps.color_formats.bits.YCBCR_NATIVE_422;
+ sink_per_slice_throughput_mps = dsc_sink_caps->throughput_mode_1_mps;
+ branch_max_throughput_mps = dsc_sink_caps->branch_overall_throughput_1_mps;
+ if (!is_dsc_possible) {
+ is_dsc_possible = (bool)dsc_common_caps.color_formats.bits.YCBCR_SIMPLE_422;
+ dsc_cfg->ycbcr422_simple = is_dsc_possible;
+ sink_per_slice_throughput_mps = dsc_sink_caps->throughput_mode_0_mps;
+ }
+ break;
+ case PIXEL_ENCODING_YCBCR420:
+ is_dsc_possible = (bool)dsc_common_caps.color_formats.bits.YCBCR_NATIVE_420;
+ sink_per_slice_throughput_mps = dsc_sink_caps->throughput_mode_1_mps;
+ branch_max_throughput_mps = dsc_sink_caps->branch_overall_throughput_1_mps;
+ break;
+ default:
+ is_dsc_possible = false;
+ }
+
+ // Validate branch's maximum throughput
+ if (branch_max_throughput_mps && dsc_div_by_10_round_up(timing->pix_clk_100hz) > branch_max_throughput_mps * 1000)
+ is_dsc_possible = false;
+
+ if (!is_dsc_possible)
+ goto done;
+
+ // Color depth
+ switch (timing->display_color_depth) {
+ case COLOR_DEPTH_888:
+ is_dsc_possible = (bool)dsc_common_caps.color_depth.bits.COLOR_DEPTH_8_BPC;
+ break;
+ case COLOR_DEPTH_101010:
+ is_dsc_possible = (bool)dsc_common_caps.color_depth.bits.COLOR_DEPTH_10_BPC;
+ break;
+ case COLOR_DEPTH_121212:
+ is_dsc_possible = (bool)dsc_common_caps.color_depth.bits.COLOR_DEPTH_12_BPC;
+ break;
+ default:
+ is_dsc_possible = false;
+ }
+
+ if (!is_dsc_possible)
+ goto done;
+
+ // DSC slicing
+ max_slices_h = get_max_dsc_slices(dsc_common_caps.slice_caps);
+
+ while (max_slices_h > 0) {
+ if (pic_width % max_slices_h == 0)
+ break;
+
+ max_slices_h = dec_num_slices(dsc_common_caps.slice_caps, max_slices_h);
+ }
+
+ is_dsc_possible = (dsc_common_caps.max_slice_width > 0);
+ if (!is_dsc_possible)
+ goto done;
+
+ min_slices_h = pic_width / dsc_common_caps.max_slice_width;
+ if (pic_width % dsc_common_caps.max_slice_width)
+ min_slices_h++;
+
+ min_slices_h = fit_num_slices_up(dsc_common_caps.slice_caps, min_slices_h);
+
+ while (min_slices_h <= max_slices_h) {
+ int pix_clk_per_slice_khz = dsc_div_by_10_round_up(timing->pix_clk_100hz) / min_slices_h;
+ if (pix_clk_per_slice_khz <= sink_per_slice_throughput_mps * 1000)
+ break;
+
+ min_slices_h = inc_num_slices(dsc_common_caps.slice_caps, min_slices_h);
+ }
+
+ if (pic_width % min_slices_h != 0)
+ min_slices_h = 0; // DSC TODO: Maybe try increasing the number of slices first?
+
+ is_dsc_possible = (min_slices_h <= max_slices_h);
+ if (!is_dsc_possible)
+ goto done;
+
+ if (dsc_policy.use_min_slices_h) {
+ if (min_slices_h > 0)
+ num_slices_h = min_slices_h;
+ else if (max_slices_h > 0) { // Fall back to max slices if min slices is not working out
+ if (dsc_policy.max_slices_h)
+ num_slices_h = min(dsc_policy.max_slices_h, max_slices_h);
+ else
+ num_slices_h = max_slices_h;
+ } else
+ is_dsc_possible = false;
+ } else {
+ if (max_slices_h > 0) {
+ if (dsc_policy.max_slices_h)
+ num_slices_h = min(dsc_policy.max_slices_h, max_slices_h);
+ else
+ num_slices_h = max_slices_h;
+ } else if (min_slices_h > 0) // Fall back to min slices if max slices is not possible
+ num_slices_h = min_slices_h;
+ else
+ is_dsc_possible = false;
+ }
+
+ if (!is_dsc_possible)
+ goto done;
+
+ dsc_cfg->num_slices_h = num_slices_h;
+ slice_width = pic_width / num_slices_h;
+
+ // Vertical number of slices: start from policy and pick the first one that height is divisible by.
+ // For 4:2:0 make sure the slice height is divisible by 2 as well.
+ num_slices_v = dsc_policy.num_slices_v;
+ if (num_slices_v < 1)
+ num_slices_v = 1;
+
+ while (num_slices_v >= 1) {
+ if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR420) {
+ int slice_height = pic_height / num_slices_v;
+ if (pic_height % num_slices_v == 0 && slice_height % 2 == 0)
+ break;
+ } else if (pic_height % num_slices_v == 0)
+ break;
+
+ num_slices_v--;
+ }
+
+ dsc_cfg->num_slices_v = num_slices_v;
+
+ is_dsc_possible = slice_width <= dsc_common_caps.max_slice_width;
+ if (!is_dsc_possible)
+ goto done;
+
+ // Final decission: can we do DSC or not?
+ if (is_dsc_possible) {
+ // Fill out the rest of DSC settings
+ dsc_cfg->block_pred_enable = dsc_common_caps.is_block_pred_supported;
+ dsc_cfg->linebuf_depth = dsc_common_caps.lb_bit_depth;
+ dsc_cfg->version_minor = (dsc_common_caps.dsc_version & 0xf0) >> 4;
+ }
+
+done:
+ if (!is_dsc_possible)
+ memset(dsc_cfg, 0, sizeof(struct dc_dsc_config));
+
+ return is_dsc_possible;
+}
+
+bool dc_dsc_parse_dsc_dpcd(const uint8_t *dpcd_dsc_basic_data, const uint8_t *dpcd_dsc_ext_data, struct dsc_dec_dpcd_caps *dsc_sink_caps)
+{
+ if (!dpcd_dsc_basic_data)
+ return false;
+
+ dsc_sink_caps->is_dsc_supported = (dpcd_dsc_basic_data[DP_DSC_SUPPORT - DP_DSC_SUPPORT] & DP_DSC_DECOMPRESSION_IS_SUPPORTED) != 0;
+ if (!dsc_sink_caps->is_dsc_supported)
+ return false;
+
+ dsc_sink_caps->dsc_version = dpcd_dsc_basic_data[DP_DSC_REV - DP_DSC_SUPPORT];
+
+ {
+ int buff_block_size;
+ int buff_size;
+
+ if (!dsc_buff_block_size_from_dpcd(dpcd_dsc_basic_data[DP_DSC_RC_BUF_BLK_SIZE - DP_DSC_SUPPORT], &buff_block_size))
+ return false;
+
+ buff_size = dpcd_dsc_basic_data[DP_DSC_RC_BUF_SIZE - DP_DSC_SUPPORT] + 1;
+ dsc_sink_caps->rc_buffer_size = buff_size * buff_block_size;
+ }
+
+ dsc_sink_caps->slice_caps1.raw = dpcd_dsc_basic_data[DP_DSC_SLICE_CAP_1 - DP_DSC_SUPPORT];
+ if (!dsc_line_buff_depth_from_dpcd(dpcd_dsc_basic_data[DP_DSC_LINE_BUF_BIT_DEPTH - DP_DSC_SUPPORT], &dsc_sink_caps->lb_bit_depth))
+ return false;
+
+ dsc_sink_caps->is_block_pred_supported =
+ (dpcd_dsc_basic_data[DP_DSC_BLK_PREDICTION_SUPPORT - DP_DSC_SUPPORT] & DP_DSC_BLK_PREDICTION_IS_SUPPORTED) != 0;
+
+ dsc_sink_caps->edp_max_bits_per_pixel =
+ dpcd_dsc_basic_data[DP_DSC_MAX_BITS_PER_PIXEL_LOW - DP_DSC_SUPPORT] |
+ dpcd_dsc_basic_data[DP_DSC_MAX_BITS_PER_PIXEL_HI - DP_DSC_SUPPORT] << 8;
+
+ dsc_sink_caps->color_formats.raw = dpcd_dsc_basic_data[DP_DSC_DEC_COLOR_FORMAT_CAP - DP_DSC_SUPPORT];
+ dsc_sink_caps->color_depth.raw = dpcd_dsc_basic_data[DP_DSC_DEC_COLOR_DEPTH_CAP - DP_DSC_SUPPORT];
+
+ {
+ int dpcd_throughput = dpcd_dsc_basic_data[DP_DSC_PEAK_THROUGHPUT - DP_DSC_SUPPORT];
+
+ if (!dsc_throughput_from_dpcd(dpcd_throughput & DP_DSC_THROUGHPUT_MODE_0_MASK, &dsc_sink_caps->throughput_mode_0_mps))
+ return false;
+
+ dpcd_throughput = (dpcd_throughput & DP_DSC_THROUGHPUT_MODE_1_MASK) >> DP_DSC_THROUGHPUT_MODE_1_SHIFT;
+ if (!dsc_throughput_from_dpcd(dpcd_throughput, &dsc_sink_caps->throughput_mode_1_mps))
+ return false;
+ }
+
+ dsc_sink_caps->max_slice_width = dpcd_dsc_basic_data[DP_DSC_MAX_SLICE_WIDTH - DP_DSC_SUPPORT] * 320;
+ dsc_sink_caps->slice_caps2.raw = dpcd_dsc_basic_data[DP_DSC_SLICE_CAP_2 - DP_DSC_SUPPORT];
+
+ if (!dsc_bpp_increment_div_from_dpcd(dpcd_dsc_basic_data[DP_DSC_BITS_PER_PIXEL_INC - DP_DSC_SUPPORT], &dsc_sink_caps->bpp_increment_div))
+ return false;
+
+ /* Extended caps */
+ if (dpcd_dsc_ext_data == NULL) { // Extended DPCD DSC data can be null, e.g. because it doesn't apply to SST
+ dsc_sink_caps->branch_overall_throughput_0_mps = 0;
+ dsc_sink_caps->branch_overall_throughput_1_mps = 0;
+ dsc_sink_caps->branch_max_line_width = 0;
+ return true;
+ }
+
+ dsc_sink_caps->branch_overall_throughput_0_mps = dpcd_dsc_ext_data[DP_DSC_BRANCH_OVERALL_THROUGHPUT_0 - DP_DSC_BRANCH_OVERALL_THROUGHPUT_0];
+ if (dsc_sink_caps->branch_overall_throughput_0_mps == 0)
+ dsc_sink_caps->branch_overall_throughput_0_mps = 0;
+ else if (dsc_sink_caps->branch_overall_throughput_0_mps == 1)
+ dsc_sink_caps->branch_overall_throughput_0_mps = 680;
+ else {
+ dsc_sink_caps->branch_overall_throughput_0_mps *= 50;
+ dsc_sink_caps->branch_overall_throughput_0_mps += 600;
+ }
+
+ dsc_sink_caps->branch_overall_throughput_1_mps = dpcd_dsc_ext_data[DP_DSC_BRANCH_OVERALL_THROUGHPUT_1 - DP_DSC_BRANCH_OVERALL_THROUGHPUT_0];
+ if (dsc_sink_caps->branch_overall_throughput_1_mps == 0)
+ dsc_sink_caps->branch_overall_throughput_1_mps = 0;
+ else if (dsc_sink_caps->branch_overall_throughput_1_mps == 1)
+ dsc_sink_caps->branch_overall_throughput_1_mps = 680;
+ else {
+ dsc_sink_caps->branch_overall_throughput_1_mps *= 50;
+ dsc_sink_caps->branch_overall_throughput_1_mps += 600;
+ }
+
+ dsc_sink_caps->branch_max_line_width = dpcd_dsc_ext_data[DP_DSC_BRANCH_MAX_LINE_WIDTH - DP_DSC_BRANCH_OVERALL_THROUGHPUT_0] * 320;
+ ASSERT(dsc_sink_caps->branch_max_line_width == 0 || dsc_sink_caps->branch_max_line_width >= 5120);
+
+ return true;
+}
+
+
+/* If DSC is possbile, get DSC bandwidth range based on [min_bpp, max_bpp] target bitrate range and
+ * timing's pixel clock and uncompressed bandwidth.
+ * If DSC is not possible, leave '*range' untouched.
+ */
+bool dc_dsc_compute_bandwidth_range(
+ const struct dc *dc,
+ const uint32_t min_bpp,
+ const uint32_t max_bpp,
+ const struct dsc_dec_dpcd_caps *dsc_sink_caps,
+ const struct dc_crtc_timing *timing,
+ struct dc_dsc_bw_range *range)
+{
+ bool is_dsc_possible = false;
+ struct dsc_enc_caps dsc_enc_caps;
+ struct dsc_enc_caps dsc_common_caps;
+ struct dc_dsc_config config;
+
+ get_dsc_enc_caps(dc, &dsc_enc_caps, timing->pix_clk_100hz);
+
+ is_dsc_possible = intersect_dsc_caps(dsc_sink_caps, &dsc_enc_caps,
+ timing->pixel_encoding, &dsc_common_caps);
+
+ if (is_dsc_possible)
+ is_dsc_possible = setup_dsc_config(dsc_sink_caps,
+ &dsc_enc_caps,
+ 0,
+ timing, &config);
+
+ if (is_dsc_possible)
+ get_dsc_bandwidth_range(min_bpp, max_bpp, &dsc_common_caps, timing, range);
+
+ return is_dsc_possible;
+}
+
+bool dc_dsc_compute_config(
+ const struct dc *dc,
+ const struct dsc_dec_dpcd_caps *dsc_sink_caps,
+ uint32_t target_bandwidth_kbps,
+ const struct dc_crtc_timing *timing,
+ struct dc_dsc_config *dsc_cfg)
+{
+ bool is_dsc_possible = false;
+ struct dsc_enc_caps dsc_enc_caps;
+
+ get_dsc_enc_caps(dc, &dsc_enc_caps, timing->pix_clk_100hz);
+ is_dsc_possible = setup_dsc_config(dsc_sink_caps,
+ &dsc_enc_caps,
+ target_bandwidth_kbps,
+ timing, dsc_cfg);
+ return is_dsc_possible;
+}
+#endif /* CONFIG_DRM_AMD_DC_DSC_SUPPORT */
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/drm_dsc_dc.c b/drivers/gpu/drm/amd/display/dc/dsc/drm_dsc_dc.c
new file mode 100644
index 000000000000..340ef4d41ebd
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dsc/drm_dsc_dc.c
@@ -0,0 +1,388 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2018 Intel Corp
+ *
+ * Author:
+ * Manasi Navare <manasi.d.navare@intel.com>
+ */
+
+/* DC versions of linux includes */
+#include <include/drm_dsc_dc.h>
+
+#define EXPORT_SYMBOL(symbol) /* nothing */
+#define BUILD_BUG_ON(cond) /* nothing */
+#define DIV_ROUND_UP(a, b) (((b) + (a) - 1) / (b))
+#define ERANGE -1
+#define DRM_DEBUG_KMS(msg) /* nothing */
+#define cpu_to_be16(__x) little_to_big(__x)
+
+static unsigned short little_to_big(int data)
+{
+ /* Swap lower and upper byte. DMCU uses big endian format. */
+ return (0xff & (data >> 8)) + ((data & 0xff) << 8);
+}
+
+/*
+ * Everything below this comment was copied directly from drm_dsc.c.
+ * Only the functions needed in DC are included.
+ * Please keep this file synced with upstream.
+ */
+
+/**
+ * DOC: dsc helpers
+ *
+ * These functions contain some common logic and helpers to deal with VESA
+ * Display Stream Compression standard required for DSC on Display Port/eDP or
+ * MIPI display interfaces.
+ */
+
+/**
+ * drm_dsc_pps_payload_pack() - Populates the DSC PPS
+ *
+ * @pps_payload:
+ * Bitwise struct for DSC Picture Parameter Set. This is defined
+ * by &struct drm_dsc_picture_parameter_set
+ * @dsc_cfg:
+ * DSC Configuration data filled by driver as defined by
+ * &struct drm_dsc_config
+ *
+ * DSC source device sends a picture parameter set (PPS) containing the
+ * information required by the sink to decode the compressed frame. Driver
+ * populates the DSC PPS struct using the DSC configuration parameters in
+ * the order expected by the DSC Display Sink device. For the DSC, the sink
+ * device expects the PPS payload in big endian format for fields
+ * that span more than 1 byte.
+ */
+void drm_dsc_pps_payload_pack(struct drm_dsc_picture_parameter_set *pps_payload,
+ const struct drm_dsc_config *dsc_cfg)
+{
+ int i;
+
+ /* Protect against someone accidently changing struct size */
+ BUILD_BUG_ON(sizeof(*pps_payload) !=
+ DP_SDP_PPS_HEADER_PAYLOAD_BYTES_MINUS_1 + 1);
+
+ memset(pps_payload, 0, sizeof(*pps_payload));
+
+ /* PPS 0 */
+ pps_payload->dsc_version =
+ dsc_cfg->dsc_version_minor |
+ dsc_cfg->dsc_version_major << DSC_PPS_VERSION_MAJOR_SHIFT;
+
+ /* PPS 1, 2 is 0 */
+
+ /* PPS 3 */
+ pps_payload->pps_3 =
+ dsc_cfg->line_buf_depth |
+ dsc_cfg->bits_per_component << DSC_PPS_BPC_SHIFT;
+
+ /* PPS 4 */
+ pps_payload->pps_4 =
+ ((dsc_cfg->bits_per_pixel & DSC_PPS_BPP_HIGH_MASK) >>
+ DSC_PPS_MSB_SHIFT) |
+ dsc_cfg->vbr_enable << DSC_PPS_VBR_EN_SHIFT |
+ dsc_cfg->simple_422 << DSC_PPS_SIMPLE422_SHIFT |
+ dsc_cfg->convert_rgb << DSC_PPS_CONVERT_RGB_SHIFT |
+ dsc_cfg->block_pred_enable << DSC_PPS_BLOCK_PRED_EN_SHIFT;
+
+ /* PPS 5 */
+ pps_payload->bits_per_pixel_low =
+ (dsc_cfg->bits_per_pixel & DSC_PPS_LSB_MASK);
+
+ /*
+ * The DSC panel expects the PPS packet to have big endian format
+ * for data spanning 2 bytes. Use a macro cpu_to_be16() to convert
+ * to big endian format. If format is little endian, it will swap
+ * bytes to convert to Big endian else keep it unchanged.
+ */
+
+ /* PPS 6, 7 */
+ pps_payload->pic_height = cpu_to_be16(dsc_cfg->pic_height);
+
+ /* PPS 8, 9 */
+ pps_payload->pic_width = cpu_to_be16(dsc_cfg->pic_width);
+
+ /* PPS 10, 11 */
+ pps_payload->slice_height = cpu_to_be16(dsc_cfg->slice_height);
+
+ /* PPS 12, 13 */
+ pps_payload->slice_width = cpu_to_be16(dsc_cfg->slice_width);
+
+ /* PPS 14, 15 */
+ pps_payload->chunk_size = cpu_to_be16(dsc_cfg->slice_chunk_size);
+
+ /* PPS 16 */
+ pps_payload->initial_xmit_delay_high =
+ ((dsc_cfg->initial_xmit_delay &
+ DSC_PPS_INIT_XMIT_DELAY_HIGH_MASK) >>
+ DSC_PPS_MSB_SHIFT);
+
+ /* PPS 17 */
+ pps_payload->initial_xmit_delay_low =
+ (dsc_cfg->initial_xmit_delay & DSC_PPS_LSB_MASK);
+
+ /* PPS 18, 19 */
+ pps_payload->initial_dec_delay =
+ cpu_to_be16(dsc_cfg->initial_dec_delay);
+
+ /* PPS 20 is 0 */
+
+ /* PPS 21 */
+ pps_payload->initial_scale_value =
+ dsc_cfg->initial_scale_value;
+
+ /* PPS 22, 23 */
+ pps_payload->scale_increment_interval =
+ cpu_to_be16(dsc_cfg->scale_increment_interval);
+
+ /* PPS 24 */
+ pps_payload->scale_decrement_interval_high =
+ ((dsc_cfg->scale_decrement_interval &
+ DSC_PPS_SCALE_DEC_INT_HIGH_MASK) >>
+ DSC_PPS_MSB_SHIFT);
+
+ /* PPS 25 */
+ pps_payload->scale_decrement_interval_low =
+ (dsc_cfg->scale_decrement_interval & DSC_PPS_LSB_MASK);
+
+ /* PPS 26[7:0], PPS 27[7:5] RESERVED */
+
+ /* PPS 27 */
+ pps_payload->first_line_bpg_offset =
+ dsc_cfg->first_line_bpg_offset;
+
+ /* PPS 28, 29 */
+ pps_payload->nfl_bpg_offset =
+ cpu_to_be16(dsc_cfg->nfl_bpg_offset);
+
+ /* PPS 30, 31 */
+ pps_payload->slice_bpg_offset =
+ cpu_to_be16(dsc_cfg->slice_bpg_offset);
+
+ /* PPS 32, 33 */
+ pps_payload->initial_offset =
+ cpu_to_be16(dsc_cfg->initial_offset);
+
+ /* PPS 34, 35 */
+ pps_payload->final_offset = cpu_to_be16(dsc_cfg->final_offset);
+
+ /* PPS 36 */
+ pps_payload->flatness_min_qp = dsc_cfg->flatness_min_qp;
+
+ /* PPS 37 */
+ pps_payload->flatness_max_qp = dsc_cfg->flatness_max_qp;
+
+ /* PPS 38, 39 */
+ pps_payload->rc_model_size =
+ cpu_to_be16(DSC_RC_MODEL_SIZE_CONST);
+
+ /* PPS 40 */
+ pps_payload->rc_edge_factor = DSC_RC_EDGE_FACTOR_CONST;
+
+ /* PPS 41 */
+ pps_payload->rc_quant_incr_limit0 =
+ dsc_cfg->rc_quant_incr_limit0;
+
+ /* PPS 42 */
+ pps_payload->rc_quant_incr_limit1 =
+ dsc_cfg->rc_quant_incr_limit1;
+
+ /* PPS 43 */
+ pps_payload->rc_tgt_offset = DSC_RC_TGT_OFFSET_LO_CONST |
+ DSC_RC_TGT_OFFSET_HI_CONST << DSC_PPS_RC_TGT_OFFSET_HI_SHIFT;
+
+ /* PPS 44 - 57 */
+ for (i = 0; i < DSC_NUM_BUF_RANGES - 1; i++)
+ pps_payload->rc_buf_thresh[i] =
+ dsc_cfg->rc_buf_thresh[i];
+
+ /* PPS 58 - 87 */
+ /*
+ * For DSC sink programming the RC Range parameter fields
+ * are as follows: Min_qp[15:11], max_qp[10:6], offset[5:0]
+ */
+ for (i = 0; i < DSC_NUM_BUF_RANGES; i++) {
+ pps_payload->rc_range_parameters[i] =
+ ((dsc_cfg->rc_range_params[i].range_min_qp <<
+ DSC_PPS_RC_RANGE_MINQP_SHIFT) |
+ (dsc_cfg->rc_range_params[i].range_max_qp <<
+ DSC_PPS_RC_RANGE_MAXQP_SHIFT) |
+ (dsc_cfg->rc_range_params[i].range_bpg_offset));
+ pps_payload->rc_range_parameters[i] =
+ cpu_to_be16(pps_payload->rc_range_parameters[i]);
+ }
+
+ /* PPS 88 */
+ pps_payload->native_422_420 = dsc_cfg->native_422 |
+ dsc_cfg->native_420 << DSC_PPS_NATIVE_420_SHIFT;
+
+ /* PPS 89 */
+ pps_payload->second_line_bpg_offset =
+ dsc_cfg->second_line_bpg_offset;
+
+ /* PPS 90, 91 */
+ pps_payload->nsl_bpg_offset =
+ cpu_to_be16(dsc_cfg->nsl_bpg_offset);
+
+ /* PPS 92, 93 */
+ pps_payload->second_line_offset_adj =
+ cpu_to_be16(dsc_cfg->second_line_offset_adj);
+
+ /* PPS 94 - 127 are O */
+}
+EXPORT_SYMBOL(drm_dsc_pps_payload_pack);
+
+/**
+ * drm_dsc_compute_rc_parameters() - Write rate control
+ * parameters to the dsc configuration defined in
+ * &struct drm_dsc_config in accordance with the DSC 1.2
+ * specification. Some configuration fields must be present
+ * beforehand.
+ *
+ * @vdsc_cfg:
+ * DSC Configuration data partially filled by driver
+ */
+int drm_dsc_compute_rc_parameters(struct drm_dsc_config *vdsc_cfg)
+{
+ unsigned long groups_per_line = 0;
+ unsigned long groups_total = 0;
+ unsigned long num_extra_mux_bits = 0;
+ unsigned long slice_bits = 0;
+ unsigned long hrd_delay = 0;
+ unsigned long final_scale = 0;
+ unsigned long rbs_min = 0;
+
+ if (vdsc_cfg->native_420 || vdsc_cfg->native_422) {
+ /* Number of groups used to code each line of a slice */
+ groups_per_line = DIV_ROUND_UP(vdsc_cfg->slice_width / 2,
+ DSC_RC_PIXELS_PER_GROUP);
+
+ /* chunksize in Bytes */
+ vdsc_cfg->slice_chunk_size = DIV_ROUND_UP(vdsc_cfg->slice_width / 2 *
+ vdsc_cfg->bits_per_pixel,
+ (8 * 16));
+ } else {
+ /* Number of groups used to code each line of a slice */
+ groups_per_line = DIV_ROUND_UP(vdsc_cfg->slice_width,
+ DSC_RC_PIXELS_PER_GROUP);
+
+ /* chunksize in Bytes */
+ vdsc_cfg->slice_chunk_size = DIV_ROUND_UP(vdsc_cfg->slice_width *
+ vdsc_cfg->bits_per_pixel,
+ (8 * 16));
+ }
+
+ if (vdsc_cfg->convert_rgb)
+ num_extra_mux_bits = 3 * (vdsc_cfg->mux_word_size +
+ (4 * vdsc_cfg->bits_per_component + 4)
+ - 2);
+ else if (vdsc_cfg->native_422)
+ num_extra_mux_bits = 4 * vdsc_cfg->mux_word_size +
+ (4 * vdsc_cfg->bits_per_component + 4) +
+ 3 * (4 * vdsc_cfg->bits_per_component) - 2;
+ else
+ num_extra_mux_bits = 3 * vdsc_cfg->mux_word_size +
+ (4 * vdsc_cfg->bits_per_component + 4) +
+ 2 * (4 * vdsc_cfg->bits_per_component) - 2;
+ /* Number of bits in one Slice */
+ slice_bits = 8 * vdsc_cfg->slice_chunk_size * vdsc_cfg->slice_height;
+
+ while ((num_extra_mux_bits > 0) &&
+ ((slice_bits - num_extra_mux_bits) % vdsc_cfg->mux_word_size))
+ num_extra_mux_bits--;
+
+ if (groups_per_line < vdsc_cfg->initial_scale_value - 8)
+ vdsc_cfg->initial_scale_value = groups_per_line + 8;
+
+ /* scale_decrement_interval calculation according to DSC spec 1.11 */
+ if (vdsc_cfg->initial_scale_value > 8)
+ vdsc_cfg->scale_decrement_interval = groups_per_line /
+ (vdsc_cfg->initial_scale_value - 8);
+ else
+ vdsc_cfg->scale_decrement_interval = DSC_SCALE_DECREMENT_INTERVAL_MAX;
+
+ vdsc_cfg->final_offset = vdsc_cfg->rc_model_size -
+ (vdsc_cfg->initial_xmit_delay *
+ vdsc_cfg->bits_per_pixel + 8) / 16 + num_extra_mux_bits;
+
+ if (vdsc_cfg->final_offset >= vdsc_cfg->rc_model_size) {
+ DRM_DEBUG_KMS("FinalOfs < RcModelSze for this InitialXmitDelay\n");
+ return -ERANGE;
+ }
+
+ final_scale = (vdsc_cfg->rc_model_size * 8) /
+ (vdsc_cfg->rc_model_size - vdsc_cfg->final_offset);
+ if (vdsc_cfg->slice_height > 1)
+ /*
+ * NflBpgOffset is 16 bit value with 11 fractional bits
+ * hence we multiply by 2^11 for preserving the
+ * fractional part
+ */
+ vdsc_cfg->nfl_bpg_offset = DIV_ROUND_UP((vdsc_cfg->first_line_bpg_offset << 11),
+ (vdsc_cfg->slice_height - 1));
+ else
+ vdsc_cfg->nfl_bpg_offset = 0;
+
+ /* 2^16 - 1 */
+ if (vdsc_cfg->nfl_bpg_offset > 65535) {
+ DRM_DEBUG_KMS("NflBpgOffset is too large for this slice height\n");
+ return -ERANGE;
+ }
+
+ /* Number of groups used to code the entire slice */
+ groups_total = groups_per_line * vdsc_cfg->slice_height;
+
+ /* slice_bpg_offset is 16 bit value with 11 fractional bits */
+ vdsc_cfg->slice_bpg_offset = DIV_ROUND_UP(((vdsc_cfg->rc_model_size -
+ vdsc_cfg->initial_offset +
+ num_extra_mux_bits) << 11),
+ groups_total);
+
+ if (final_scale > 9) {
+ /*
+ * ScaleIncrementInterval =
+ * finaloffset/((NflBpgOffset + SliceBpgOffset)*8(finalscale - 1.125))
+ * as (NflBpgOffset + SliceBpgOffset) has 11 bit fractional value,
+ * we need divide by 2^11 from pstDscCfg values
+ */
+ vdsc_cfg->scale_increment_interval =
+ (vdsc_cfg->final_offset * (1 << 11)) /
+ ((vdsc_cfg->nfl_bpg_offset +
+ vdsc_cfg->slice_bpg_offset) *
+ (final_scale - 9));
+ } else {
+ /*
+ * If finalScaleValue is less than or equal to 9, a value of 0 should
+ * be used to disable the scale increment at the end of the slice
+ */
+ vdsc_cfg->scale_increment_interval = 0;
+ }
+
+ if (vdsc_cfg->scale_increment_interval > 65535) {
+ DRM_DEBUG_KMS("ScaleIncrementInterval is large for slice height\n");
+ return -ERANGE;
+ }
+
+ /*
+ * DSC spec mentions that bits_per_pixel specifies the target
+ * bits/pixel (bpp) rate that is used by the encoder,
+ * in steps of 1/16 of a bit per pixel
+ */
+ rbs_min = vdsc_cfg->rc_model_size - vdsc_cfg->initial_offset +
+ DIV_ROUND_UP(vdsc_cfg->initial_xmit_delay *
+ vdsc_cfg->bits_per_pixel, 16) +
+ groups_per_line * vdsc_cfg->first_line_bpg_offset;
+
+ hrd_delay = DIV_ROUND_UP((rbs_min * 16), vdsc_cfg->bits_per_pixel);
+ vdsc_cfg->rc_bits = (hrd_delay * vdsc_cfg->bits_per_pixel) / 16;
+ vdsc_cfg->initial_dec_delay = hrd_delay - vdsc_cfg->initial_xmit_delay;
+
+ /* As per DSC spec v1.2a recommendation: */
+ if (vdsc_cfg->native_420)
+ vdsc_cfg->second_line_offset_adj = 512;
+ else
+ vdsc_cfg->second_line_offset_adj = 0;
+
+ return 0;
+}
+EXPORT_SYMBOL(drm_dsc_compute_rc_parameters);
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/dscc_types.h b/drivers/gpu/drm/amd/display/dc/dsc/dscc_types.h
new file mode 100644
index 000000000000..020ad8f685ea
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dsc/dscc_types.h
@@ -0,0 +1,54 @@
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+#ifndef __DSCC_TYPES_H__
+#define __DSCC_TYPES_H__
+
+#include <drm/drm_dsc.h>
+
+#ifndef NUM_BUF_RANGES
+#define NUM_BUF_RANGES 15
+#endif
+
+struct dsc_pps_rc_range {
+ int range_min_qp;
+ int range_max_qp;
+ int range_bpg_offset;
+};
+
+struct dsc_parameters {
+ struct drm_dsc_config pps;
+
+ /* Additional parameters for register programming */
+ uint32_t bytes_per_pixel; /* In u3.28 format */
+ uint32_t rc_buffer_model_size;
+};
+
+int dscc_compute_dsc_parameters(const struct drm_dsc_config *pps, struct dsc_parameters *dsc_params);
+
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/qp_tables.h b/drivers/gpu/drm/amd/display/dc/dsc/qp_tables.h
new file mode 100644
index 000000000000..f66d006eac5d
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dsc/qp_tables.h
@@ -0,0 +1,706 @@
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+
+const qp_table qp_table_422_10bpc_min = {
+ { 6, { 0, 4, 5, 6, 6, 6, 6, 7, 7, 8, 9, 9, 9, 12, 16} },
+ { 6.5, { 0, 4, 5, 6, 6, 6, 6, 7, 7, 8, 9, 9, 9, 12, 16} },
+ { 7, { 0, 4, 5, 6, 6, 6, 6, 7, 7, 7, 9, 9, 9, 11, 15} },
+ { 7.5, { 0, 2, 4, 6, 6, 6, 6, 7, 7, 7, 8, 9, 9, 11, 15} },
+ { 8, { 0, 2, 3, 5, 5, 6, 6, 7, 7, 7, 8, 8, 9, 11, 14} },
+ { 8.5, { 0, 2, 3, 4, 5, 5, 5, 6, 6, 7, 8, 8, 9, 11, 14} },
+ { 9, { 0, 2, 3, 4, 5, 5, 5, 6, 6, 7, 8, 8, 9, 11, 13} },
+ { 9.5, { 0, 2, 3, 4, 4, 5, 5, 6, 6, 7, 8, 8, 9, 11, 13} },
+ { 10, { 0, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 8, 9, 11, 12} },
+ {10.5, { 0, 2, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 11, 12} },
+ { 11, { 0, 2, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 10, 11} },
+ {11.5, { 0, 2, 2, 3, 4, 4, 5, 5, 6, 7, 7, 8, 8, 10, 11} },
+ { 12, { 0, 2, 2, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 9, 10} },
+ {12.5, { 0, 1, 2, 2, 4, 4, 4, 5, 5, 6, 6, 7, 8, 9, 10} },
+ { 13, { 0, 1, 2, 2, 4, 4, 4, 5, 5, 6, 6, 6, 8, 8, 9} },
+ {13.5, { 0, 1, 2, 2, 3, 4, 4, 4, 5, 6, 6, 6, 7, 8, 9} },
+ { 14, { 0, 1, 2, 2, 3, 4, 4, 4, 4, 5, 5, 6, 7, 7, 8} },
+ {14.5, { 0, 1, 1, 2, 3, 3, 4, 4, 4, 5, 5, 6, 6, 7, 8} },
+ { 15, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 6, 6, 6, 8} },
+ {15.5, { 0, 0, 1, 1, 2, 3, 3, 3, 4, 4, 5, 5, 5, 6, 7} },
+ { 16, { 0, 0, 1, 1, 2, 3, 3, 3, 4, 4, 5, 5, 5, 5, 7} },
+ {16.5, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 5, 6} },
+ { 17, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 6} },
+ {17.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5} },
+ { 18, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 5} },
+ {18.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 5} },
+ { 19, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 4} },
+ {19.5, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 4} },
+ { 20, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 3} }
+};
+
+
+const qp_table qp_table_444_8bpc_max = {
+ { 6, { 4, 6, 8, 8, 9, 9, 9, 10, 11, 12, 12, 12, 12, 13, 15} },
+ { 6.5, { 4, 6, 7, 8, 8, 8, 9, 10, 11, 11, 12, 12, 12, 13, 15} },
+ { 7, { 4, 5, 7, 7, 8, 8, 8, 9, 10, 11, 11, 12, 12, 13, 14} },
+ { 7.5, { 4, 5, 6, 7, 7, 8, 8, 9, 10, 10, 11, 11, 12, 13, 14} },
+ { 8, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} },
+ { 8.5, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} },
+ { 9, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 11, 13} },
+ { 9.5, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 11, 13} },
+ { 10, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 11, 12} },
+ {10.5, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 10, 11, 12} },
+ { 11, { 2, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11} },
+ {11.5, { 2, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 10, 11} },
+ { 12, { 2, 3, 4, 5, 6, 6, 7, 8, 8, 9, 9, 9, 9, 10, 11} },
+ {12.5, { 2, 3, 4, 5, 6, 6, 6, 7, 7, 8, 8, 9, 9, 10, 11} },
+ { 13, { 1, 3, 4, 5, 5, 6, 6, 7, 7, 8, 8, 8, 8, 9, 10} },
+ {13.5, { 1, 2, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 10} },
+ { 14, { 1, 2, 2, 3, 4, 4, 4, 5, 6, 6, 7, 8, 8, 8, 10} },
+ {14.5, { 0, 1, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 9} },
+ { 15, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9} },
+ {15.5, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9} },
+ { 16, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 8} },
+ {16.5, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 8} },
+ { 17, { 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 8} },
+ {17.5, { 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 8} },
+ { 18, { 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 7} },
+ {18.5, { 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 7} },
+ { 19, { 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 6} },
+ {19.5, { 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 6} },
+ { 20, { 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 4, 6} },
+ {20.5, { 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 4, 6} },
+ { 21, { 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5} },
+ {21.5, { 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5} },
+ { 22, { 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 5} },
+ {22.5, { 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 4} },
+ { 23, { 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 4} },
+ {23.5, { 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 4} },
+ { 24, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 4} }
+};
+
+
+const qp_table qp_table_420_12bpc_max = {
+ { 4, {11, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 21, 22} },
+ { 4.5, {10, 11, 12, 13, 14, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} },
+ { 5, { 9, 11, 12, 13, 14, 15, 15, 16, 17, 17, 18, 18, 19, 20, 21} },
+ { 5.5, { 8, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18, 19, 20} },
+ { 6, { 6, 9, 11, 12, 13, 14, 15, 16, 16, 17, 17, 17, 17, 18, 19} },
+ { 6.5, { 6, 8, 10, 11, 11, 13, 14, 15, 15, 16, 16, 17, 17, 18, 19} },
+ { 7, { 5, 7, 9, 10, 10, 12, 13, 14, 14, 15, 16, 16, 17, 17, 18} },
+ { 7.5, { 5, 7, 8, 9, 9, 11, 12, 13, 14, 14, 15, 15, 16, 16, 17} },
+ { 8, { 4, 6, 7, 8, 8, 10, 11, 12, 13, 13, 14, 15, 15, 16, 17} },
+ { 8.5, { 3, 6, 6, 7, 7, 10, 11, 12, 13, 13, 14, 14, 15, 15, 16} },
+ { 9, { 3, 5, 6, 7, 7, 10, 11, 12, 12, 13, 13, 14, 14, 14, 15} },
+ { 9.5, { 2, 5, 6, 6, 7, 9, 10, 11, 12, 12, 13, 13, 13, 14, 15} },
+ { 10, { 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 13, 13, 13, 15} },
+ {10.5, { 2, 3, 5, 5, 6, 7, 8, 9, 11, 11, 12, 12, 12, 12, 14} },
+ { 11, { 1, 3, 4, 5, 6, 6, 7, 9, 10, 11, 11, 11, 12, 12, 13} },
+ {11.5, { 1, 2, 3, 4, 5, 6, 6, 8, 9, 10, 10, 11, 11, 11, 13} },
+ { 12, { 1, 1, 3, 3, 4, 5, 6, 7, 8, 9, 9, 10, 10, 10, 12} },
+ {12.5, { 1, 1, 2, 3, 4, 5, 6, 7, 8, 8, 9, 9, 9, 10, 11} },
+ { 13, { 1, 1, 1, 2, 4, 4, 6, 6, 7, 8, 8, 9, 9, 9, 11} },
+ {13.5, { 1, 1, 1, 2, 3, 4, 5, 5, 6, 7, 8, 8, 8, 9, 11} },
+ { 14, { 1, 1, 1, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 8, 10} },
+ {14.5, { 0, 1, 1, 1, 2, 3, 4, 4, 5, 5, 6, 7, 7, 7, 9} },
+ { 15, { 0, 1, 1, 1, 1, 2, 3, 3, 5, 5, 5, 6, 6, 7, 9} },
+ {15.5, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 8} },
+ { 16, { 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 7} },
+ {16.5, { 0, 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 4, 4, 5, 7} },
+ { 17, { 0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 4, 6} },
+ {17.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 6} },
+ { 18, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 5} }
+};
+
+
+const qp_table qp_table_444_10bpc_min = {
+ { 6, { 0, 4, 7, 7, 9, 9, 9, 9, 9, 10, 10, 10, 10, 12, 18} },
+ { 6.5, { 0, 4, 6, 7, 8, 8, 9, 9, 9, 9, 10, 10, 10, 12, 18} },
+ { 7, { 0, 4, 6, 6, 8, 8, 8, 8, 8, 9, 9, 10, 10, 12, 17} },
+ { 7.5, { 0, 4, 6, 6, 7, 8, 8, 8, 8, 8, 9, 9, 10, 12, 17} },
+ { 8, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 8, 9, 9, 9, 12, 16} },
+ { 8.5, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 8, 9, 9, 9, 12, 16} },
+ { 9, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 16} },
+ { 9.5, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 16} },
+ { 10, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 15} },
+ {10.5, { 0, 4, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 15} },
+ { 11, { 0, 3, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 11, 14} },
+ {11.5, { 0, 3, 5, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 11, 14} },
+ { 12, { 0, 2, 4, 4, 6, 6, 7, 7, 7, 7, 9, 9, 9, 11, 14} },
+ {12.5, { 0, 2, 4, 4, 6, 6, 7, 7, 7, 7, 8, 9, 9, 11, 14} },
+ { 13, { 0, 2, 4, 4, 5, 6, 7, 7, 7, 7, 8, 9, 9, 11, 13} },
+ {13.5, { 0, 2, 3, 4, 5, 6, 6, 7, 7, 7, 8, 9, 9, 11, 13} },
+ { 14, { 0, 2, 3, 4, 5, 5, 6, 6, 7, 7, 8, 9, 9, 11, 13} },
+ {14.5, { 0, 2, 3, 4, 5, 5, 6, 6, 6, 7, 7, 8, 9, 11, 12} },
+ { 15, { 0, 2, 3, 4, 4, 5, 5, 6, 6, 6, 7, 8, 9, 11, 12} },
+ {15.5, { 0, 2, 3, 4, 4, 5, 5, 6, 6, 6, 7, 8, 9, 11, 12} },
+ { 16, { 0, 2, 3, 4, 4, 5, 5, 6, 6, 6, 7, 8, 8, 10, 11} },
+ {16.5, { 0, 1, 2, 3, 4, 5, 5, 6, 6, 6, 7, 8, 8, 10, 11} },
+ { 17, { 0, 1, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 8, 9, 11} },
+ {17.5, { 0, 1, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 8, 9, 11} },
+ { 18, { 0, 1, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 8, 9, 10} },
+ {18.5, { 0, 1, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 8, 9, 10} },
+ { 19, { 0, 1, 1, 2, 3, 3, 3, 4, 5, 6, 6, 7, 7, 8, 9} },
+ {19.5, { 0, 1, 1, 2, 3, 3, 3, 4, 5, 6, 6, 7, 7, 8, 9} },
+ { 20, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 9} },
+ {20.5, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 9} },
+ { 21, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 6, 6, 7, 9} },
+ {21.5, { 0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, 6, 6, 7, 8} },
+ { 22, { 0, 0, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 8} },
+ {22.5, { 0, 0, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7} },
+ { 23, { 0, 0, 1, 2, 2, 2, 3, 3, 3, 3, 5, 5, 5, 5, 7} },
+ {23.5, { 0, 0, 0, 2, 2, 2, 3, 3, 3, 3, 5, 5, 5, 5, 7} },
+ { 24, { 0, 0, 0, 1, 1, 2, 3, 3, 3, 3, 4, 4, 4, 5, 7} },
+ {24.5, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 7} },
+ { 25, { 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 4, 4, 4, 4, 6} },
+ {25.5, { 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 6} },
+ { 26, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 3, 5} },
+ {26.5, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 5} },
+ { 27, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 5} },
+ {27.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 5} },
+ { 28, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 4} },
+ {28.5, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 4} },
+ { 29, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3} },
+ {29.5, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3} },
+ { 30, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 3} }
+};
+
+
+const qp_table qp_table_420_8bpc_max = {
+ { 4, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 13, 14} },
+ { 4.5, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} },
+ { 5, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 12, 13} },
+ { 5.5, { 3, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 10, 10, 11, 12} },
+ { 6, { 2, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 10, 11} },
+ { 6.5, { 2, 3, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 11} },
+ { 7, { 1, 2, 3, 4, 4, 5, 5, 6, 6, 7, 8, 8, 9, 9, 10} },
+ { 7.5, { 1, 2, 2, 3, 3, 4, 4, 5, 6, 6, 7, 7, 8, 8, 9} },
+ { 8, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9} },
+ { 8.5, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 6, 7, 7, 8} },
+ { 9, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6, 7} },
+ { 9.5, { 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7} },
+ { 10, { 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6} },
+ {10.5, { 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 4, 6} },
+ { 11, { 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5} },
+ {11.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5} },
+ { 12, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 4} }
+};
+
+
+const qp_table qp_table_444_8bpc_min = {
+ { 6, { 0, 1, 3, 3, 5, 5, 5, 5, 5, 6, 6, 6, 6, 9, 14} },
+ { 6.5, { 0, 1, 2, 3, 4, 4, 5, 5, 5, 5, 6, 6, 6, 9, 14} },
+ { 7, { 0, 0, 2, 2, 4, 4, 4, 4, 4, 5, 5, 6, 6, 9, 13} },
+ { 7.5, { 0, 0, 2, 2, 3, 4, 4, 4, 4, 4, 5, 5, 6, 9, 13} },
+ { 8, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 4, 5, 5, 5, 8, 12} },
+ { 8.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 4, 5, 5, 5, 8, 12} },
+ { 9, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 12} },
+ { 9.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 12} },
+ { 10, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 11} },
+ {10.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 11} },
+ { 11, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 10} },
+ {11.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 10} },
+ { 12, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 10} },
+ {12.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 4, 5, 5, 7, 10} },
+ { 13, { 0, 0, 1, 1, 2, 3, 3, 3, 3, 3, 4, 5, 5, 7, 9} },
+ {13.5, { 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 4, 5, 5, 7, 9} },
+ { 14, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 4, 5, 5, 7, 9} },
+ {14.5, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 4, 4, 5, 7, 8} },
+ { 15, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 8} },
+ {15.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 8} },
+ { 16, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 6, 7} },
+ {16.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 6, 7} },
+ { 17, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7} },
+ {17.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7} },
+ { 18, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6} },
+ {18.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6} },
+ { 19, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5} },
+ {19.5, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5} },
+ { 20, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 3, 5} },
+ {20.5, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 3, 5} },
+ { 21, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4} },
+ {21.5, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4} },
+ { 22, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 4} },
+ {22.5, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 3} },
+ { 23, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 3} },
+ {23.5, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 3} },
+ { 24, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3} }
+};
+
+
+const qp_table qp_table_444_12bpc_min = {
+ { 6, { 0, 5, 11, 11, 13, 13, 13, 13, 13, 14, 14, 14, 14, 17, 22} },
+ { 6.5, { 0, 5, 10, 11, 12, 12, 13, 13, 13, 13, 14, 14, 14, 17, 22} },
+ { 7, { 0, 5, 10, 10, 12, 12, 12, 12, 12, 13, 13, 14, 14, 17, 21} },
+ { 7.5, { 0, 5, 9, 10, 11, 12, 12, 12, 12, 12, 13, 13, 14, 17, 21} },
+ { 8, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 12, 13, 13, 13, 16, 20} },
+ { 8.5, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 12, 13, 13, 13, 16, 20} },
+ { 9, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 20} },
+ { 9.5, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 20} },
+ { 10, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 19} },
+ {10.5, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 19} },
+ { 11, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
+ {11.5, { 0, 4, 8, 9, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
+ { 12, { 0, 4, 7, 8, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
+ {12.5, { 0, 4, 7, 8, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
+ { 13, { 0, 4, 7, 8, 9, 11, 11, 11, 11, 11, 13, 13, 13, 15, 17} },
+ {13.5, { 0, 3, 6, 7, 9, 10, 10, 11, 11, 11, 12, 13, 13, 15, 17} },
+ { 14, { 0, 3, 5, 6, 9, 9, 9, 10, 11, 11, 12, 13, 13, 15, 17} },
+ {14.5, { 0, 2, 5, 6, 8, 9, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
+ { 15, { 0, 2, 4, 6, 7, 8, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
+ {15.5, { 0, 2, 4, 6, 7, 8, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
+ { 16, { 0, 2, 4, 6, 7, 8, 9, 10, 11, 11, 11, 12, 12, 14, 15} },
+ {16.5, { 0, 2, 3, 5, 7, 8, 9, 10, 11, 11, 11, 12, 12, 14, 15} },
+ { 17, { 0, 2, 3, 5, 5, 6, 9, 9, 10, 10, 11, 11, 12, 13, 15} },
+ {17.5, { 0, 2, 3, 5, 5, 6, 8, 9, 10, 10, 11, 11, 12, 13, 15} },
+ { 18, { 0, 2, 3, 5, 5, 6, 8, 9, 10, 10, 11, 11, 12, 13, 14} },
+ {18.5, { 0, 2, 3, 5, 5, 6, 8, 9, 10, 10, 11, 11, 12, 13, 14} },
+ { 19, { 0, 1, 2, 4, 5, 5, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
+ {19.5, { 0, 1, 2, 4, 5, 5, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
+ { 20, { 0, 1, 2, 3, 4, 5, 7, 8, 8, 8, 9, 10, 10, 11, 13} },
+ {20.5, { 0, 1, 2, 3, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11, 13} },
+ { 21, { 0, 1, 2, 3, 4, 5, 5, 7, 7, 8, 9, 10, 10, 11, 13} },
+ {21.5, { 0, 1, 2, 3, 3, 4, 5, 7, 7, 8, 9, 10, 10, 11, 12} },
+ { 22, { 0, 0, 1, 3, 3, 4, 5, 6, 7, 8, 9, 9, 9, 10, 12} },
+ {22.5, { 0, 0, 1, 3, 3, 4, 5, 6, 7, 8, 9, 9, 9, 10, 11} },
+ { 23, { 0, 0, 1, 3, 3, 4, 5, 6, 6, 7, 9, 9, 9, 9, 11} },
+ {23.5, { 0, 0, 1, 3, 3, 4, 5, 6, 6, 7, 9, 9, 9, 9, 11} },
+ { 24, { 0, 0, 1, 2, 3, 4, 5, 6, 6, 7, 8, 8, 8, 9, 11} },
+ {24.5, { 0, 0, 1, 2, 3, 4, 4, 6, 6, 7, 8, 8, 8, 9, 11} },
+ { 25, { 0, 0, 1, 2, 3, 4, 4, 5, 6, 7, 8, 8, 8, 8, 10} },
+ {25.5, { 0, 0, 1, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 8, 10} },
+ { 26, { 0, 0, 1, 2, 2, 3, 4, 5, 5, 6, 7, 7, 7, 7, 9} },
+ {26.5, { 0, 0, 1, 2, 2, 3, 4, 5, 5, 5, 7, 7, 7, 7, 9} },
+ { 27, { 0, 0, 1, 2, 2, 3, 4, 4, 5, 5, 6, 7, 7, 7, 9} },
+ {27.5, { 0, 0, 1, 1, 2, 2, 4, 4, 4, 5, 6, 7, 7, 7, 9} },
+ { 28, { 0, 0, 0, 1, 1, 2, 3, 4, 4, 4, 6, 6, 6, 7, 9} },
+ {28.5, { 0, 0, 0, 1, 1, 2, 3, 3, 4, 4, 5, 6, 6, 6, 8} },
+ { 29, { 0, 0, 0, 1, 1, 2, 3, 3, 4, 4, 5, 5, 6, 6, 8} },
+ {29.5, { 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7} },
+ { 30, { 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 5, 5, 5, 5, 7} },
+ {30.5, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 7} },
+ { 31, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 7} },
+ {31.5, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7} },
+ { 32, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 4, 6} },
+ {32.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 4, 6} },
+ { 33, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 5} },
+ {33.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 5} },
+ { 34, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 5} },
+ {34.5, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 2, 3, 5} },
+ { 35, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 4} },
+ {35.5, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 4} },
+ { 36, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 3} }
+};
+
+
+const qp_table qp_table_420_12bpc_min = {
+ { 4, { 0, 4, 9, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 21} },
+ { 4.5, { 0, 4, 8, 9, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 20} },
+ { 5, { 0, 4, 8, 9, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 20} },
+ { 5.5, { 0, 4, 7, 8, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 19} },
+ { 6, { 0, 4, 7, 8, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
+ { 6.5, { 0, 4, 6, 8, 9, 10, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
+ { 7, { 0, 3, 5, 7, 9, 10, 10, 11, 11, 11, 13, 13, 13, 15, 17} },
+ { 7.5, { 0, 3, 5, 7, 8, 9, 10, 10, 11, 11, 12, 13, 13, 15, 16} },
+ { 8, { 0, 2, 4, 6, 7, 9, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
+ { 8.5, { 0, 2, 4, 6, 6, 9, 9, 10, 11, 11, 12, 12, 13, 14, 15} },
+ { 9, { 0, 2, 4, 6, 6, 9, 9, 10, 10, 11, 11, 12, 13, 13, 14} },
+ { 9.5, { 0, 2, 4, 5, 6, 8, 8, 9, 10, 10, 11, 12, 12, 13, 14} },
+ { 10, { 0, 2, 3, 5, 6, 7, 8, 8, 9, 10, 10, 12, 12, 12, 14} },
+ {10.5, { 0, 2, 3, 4, 5, 6, 7, 8, 9, 9, 10, 11, 11, 11, 13} },
+ { 11, { 0, 2, 3, 4, 5, 5, 6, 8, 8, 9, 9, 10, 11, 11, 12} },
+ {11.5, { 0, 1, 2, 3, 4, 5, 5, 7, 8, 8, 9, 10, 10, 10, 12} },
+ { 12, { 0, 0, 2, 2, 3, 4, 5, 6, 7, 8, 8, 9, 9, 9, 11} },
+ {12.5, { 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 8, 8, 8, 9, 10} },
+ { 13, { 0, 0, 0, 1, 3, 3, 5, 5, 6, 7, 7, 8, 8, 8, 10} },
+ {13.5, { 0, 0, 0, 1, 2, 3, 4, 4, 5, 6, 7, 7, 7, 8, 10} },
+ { 14, { 0, 0, 0, 1, 2, 3, 3, 4, 5, 5, 6, 7, 7, 7, 9} },
+ {14.5, { 0, 0, 0, 0, 1, 2, 3, 3, 4, 4, 5, 6, 6, 6, 8} },
+ { 15, { 0, 0, 0, 0, 0, 1, 2, 2, 4, 4, 4, 5, 5, 6, 8} },
+ {15.5, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7} },
+ { 16, { 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 6} },
+ {16.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 6} },
+ { 17, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 5} },
+ {17.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 3, 5} },
+ { 18, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 4} }
+};
+
+
+const qp_table qp_table_422_12bpc_min = {
+ { 6, { 0, 4, 9, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 16, 20} },
+ { 6.5, { 0, 4, 9, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 16, 20} },
+ { 7, { 0, 4, 9, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 19} },
+ { 7.5, { 0, 4, 8, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 19} },
+ { 8, { 0, 4, 7, 8, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 18} },
+ { 8.5, { 0, 3, 6, 8, 9, 10, 10, 11, 11, 11, 12, 13, 13, 15, 18} },
+ { 9, { 0, 3, 5, 8, 9, 10, 10, 10, 11, 11, 12, 13, 13, 15, 17} },
+ { 9.5, { 0, 3, 5, 7, 8, 9, 10, 10, 11, 11, 12, 13, 13, 15, 17} },
+ { 10, { 0, 2, 4, 6, 7, 9, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
+ {10.5, { 0, 2, 4, 6, 7, 8, 9, 10, 11, 11, 12, 13, 13, 15, 16} },
+ { 11, { 0, 2, 4, 6, 7, 8, 9, 10, 11, 11, 12, 12, 13, 14, 15} },
+ {11.5, { 0, 2, 4, 6, 7, 7, 9, 9, 10, 11, 11, 12, 12, 14, 15} },
+ { 12, { 0, 2, 4, 6, 6, 6, 8, 8, 9, 9, 11, 11, 12, 13, 14} },
+ {12.5, { 0, 1, 4, 5, 6, 6, 7, 8, 8, 9, 10, 11, 11, 13, 14} },
+ { 13, { 0, 1, 3, 4, 5, 5, 7, 8, 8, 9, 10, 10, 11, 12, 13} },
+ {13.5, { 0, 1, 3, 3, 4, 5, 7, 7, 8, 8, 10, 10, 10, 12, 13} },
+ { 14, { 0, 0, 2, 3, 4, 5, 6, 6, 7, 7, 9, 10, 10, 11, 12} },
+ {14.5, { 0, 0, 1, 3, 4, 4, 6, 6, 6, 7, 9, 9, 9, 11, 12} },
+ { 15, { 0, 0, 1, 3, 3, 4, 5, 6, 6, 6, 8, 9, 9, 10, 12} },
+ {15.5, { 0, 0, 1, 2, 3, 4, 5, 5, 6, 6, 8, 8, 8, 10, 11} },
+ { 16, { 0, 0, 1, 2, 3, 4, 5, 5, 6, 6, 8, 8, 8, 9, 11} },
+ {16.5, { 0, 0, 0, 2, 2, 3, 4, 5, 5, 5, 6, 7, 7, 9, 10} },
+ { 17, { 0, 0, 0, 1, 2, 2, 4, 4, 4, 5, 6, 6, 6, 8, 10} },
+ {17.5, { 0, 0, 0, 1, 2, 2, 3, 4, 4, 4, 5, 6, 6, 8, 9} },
+ { 18, { 0, 0, 0, 1, 2, 2, 3, 3, 3, 4, 5, 5, 6, 7, 9} },
+ {18.5, { 0, 0, 0, 1, 2, 2, 3, 3, 3, 3, 5, 5, 5, 7, 9} },
+ { 19, { 0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 6, 8} },
+ {19.5, { 0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 4, 4, 4, 6, 8} },
+ { 20, { 0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 4, 4, 4, 5, 7} },
+ {20.5, { 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 7} },
+ { 21, { 0, 0, 0, 0, 0, 1, 2, 2, 3, 3, 3, 4, 4, 4, 6} },
+ {21.5, { 0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 4, 6} },
+ { 22, { 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 6} },
+ {22.5, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 5} },
+ { 23, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 5} },
+ {23.5, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 4} },
+ { 24, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 4} }
+};
+
+
+const qp_table qp_table_422_12bpc_max = {
+ { 6, {12, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} },
+ { 6.5, {12, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} },
+ { 7, {11, 12, 13, 14, 15, 15, 15, 16, 17, 17, 18, 18, 19, 19, 20} },
+ { 7.5, { 9, 10, 12, 14, 15, 15, 15, 16, 16, 17, 17, 18, 18, 19, 20} },
+ { 8, { 6, 9, 10, 12, 14, 15, 15, 16, 16, 17, 17, 17, 17, 18, 19} },
+ { 8.5, { 6, 8, 9, 11, 13, 14, 14, 15, 15, 16, 16, 17, 17, 18, 19} },
+ { 9, { 5, 7, 8, 10, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 18} },
+ { 9.5, { 5, 7, 7, 9, 10, 12, 12, 13, 14, 14, 15, 15, 16, 17, 18} },
+ { 10, { 4, 6, 6, 8, 9, 11, 11, 12, 13, 13, 14, 15, 15, 16, 17} },
+ {10.5, { 4, 6, 6, 8, 9, 10, 11, 12, 13, 13, 14, 15, 15, 16, 17} },
+ { 11, { 4, 5, 6, 8, 9, 10, 11, 12, 13, 13, 14, 14, 15, 15, 16} },
+ {11.5, { 3, 5, 6, 8, 9, 9, 11, 11, 12, 13, 13, 14, 14, 15, 16} },
+ { 12, { 3, 5, 6, 8, 8, 8, 10, 10, 11, 11, 13, 13, 14, 14, 15} },
+ {12.5, { 3, 4, 6, 7, 8, 8, 9, 10, 10, 11, 12, 13, 13, 14, 15} },
+ { 13, { 2, 4, 5, 6, 7, 7, 9, 10, 10, 11, 12, 12, 13, 13, 14} },
+ {13.5, { 2, 4, 5, 5, 6, 7, 9, 9, 10, 10, 12, 12, 12, 13, 14} },
+ { 14, { 2, 3, 4, 5, 6, 7, 8, 8, 9, 9, 11, 12, 12, 12, 13} },
+ {14.5, { 2, 3, 3, 5, 6, 6, 8, 8, 8, 9, 11, 11, 11, 12, 13} },
+ { 15, { 2, 3, 3, 5, 5, 6, 7, 8, 8, 8, 10, 11, 11, 11, 13} },
+ {15.5, { 2, 2, 3, 4, 5, 6, 7, 7, 8, 8, 10, 10, 10, 11, 12} },
+ { 16, { 2, 2, 3, 4, 5, 6, 7, 7, 8, 8, 10, 10, 10, 10, 12} },
+ {16.5, { 1, 2, 2, 4, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 11} },
+ { 17, { 1, 1, 2, 3, 4, 4, 6, 6, 6, 7, 8, 8, 8, 9, 11} },
+ {17.5, { 1, 1, 2, 3, 4, 4, 5, 6, 6, 6, 7, 8, 8, 9, 10} },
+ { 18, { 1, 1, 1, 2, 3, 3, 5, 5, 5, 6, 7, 7, 8, 8, 10} },
+ {18.5, { 1, 1, 1, 2, 3, 3, 5, 5, 5, 5, 7, 7, 7, 8, 10} },
+ { 19, { 1, 1, 1, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 7, 9} },
+ {19.5, { 1, 1, 1, 2, 2, 2, 4, 5, 5, 5, 6, 6, 6, 7, 9} },
+ { 20, { 1, 1, 1, 2, 2, 2, 4, 5, 5, 5, 6, 6, 6, 6, 8} },
+ {20.5, { 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 8} },
+ { 21, { 0, 0, 0, 1, 1, 2, 3, 3, 4, 4, 4, 5, 5, 5, 7} },
+ {21.5, { 0, 0, 0, 0, 1, 1, 2, 3, 3, 3, 4, 4, 4, 5, 7} },
+ { 22, { 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 7} },
+ {22.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 6} },
+ { 23, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6} },
+ {23.5, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 3, 5} },
+ { 24, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 3, 5} }
+};
+
+
+const qp_table qp_table_444_12bpc_max = {
+ { 6, {12, 14, 16, 16, 17, 17, 17, 18, 19, 20, 20, 20, 20, 21, 23} },
+ { 6.5, {12, 14, 15, 16, 16, 16, 17, 18, 19, 19, 20, 20, 20, 21, 23} },
+ { 7, {12, 13, 15, 15, 16, 16, 16, 17, 18, 19, 19, 20, 20, 21, 22} },
+ { 7.5, {12, 13, 14, 15, 15, 16, 16, 17, 18, 18, 19, 19, 20, 21, 22} },
+ { 8, {12, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} },
+ { 8.5, {12, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} },
+ { 9, {11, 12, 13, 14, 15, 15, 15, 16, 17, 17, 18, 18, 19, 19, 21} },
+ { 9.5, {11, 12, 13, 14, 15, 15, 15, 16, 17, 17, 18, 18, 19, 19, 21} },
+ { 10, {11, 12, 13, 14, 15, 15, 15, 16, 17, 17, 18, 18, 19, 19, 20} },
+ {10.5, {10, 12, 13, 14, 15, 15, 15, 16, 17, 17, 18, 18, 18, 19, 20} },
+ { 11, { 9, 11, 13, 14, 15, 15, 15, 16, 16, 17, 17, 17, 18, 18, 19} },
+ {11.5, { 9, 11, 13, 14, 15, 15, 15, 16, 16, 17, 17, 17, 17, 18, 19} },
+ { 12, { 6, 9, 12, 13, 14, 14, 15, 16, 16, 17, 17, 17, 17, 18, 19} },
+ {12.5, { 6, 9, 12, 13, 14, 14, 14, 15, 15, 16, 16, 17, 17, 18, 19} },
+ { 13, { 5, 9, 12, 13, 13, 14, 14, 15, 15, 16, 16, 16, 16, 17, 18} },
+ {13.5, { 5, 8, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 18} },
+ { 14, { 5, 8, 10, 11, 12, 12, 12, 13, 14, 14, 15, 16, 16, 16, 18} },
+ {14.5, { 4, 7, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 17} },
+ { 15, { 4, 7, 9, 10, 10, 11, 11, 12, 13, 13, 14, 15, 15, 16, 17} },
+ {15.5, { 4, 7, 9, 10, 10, 11, 11, 12, 13, 13, 14, 15, 15, 16, 17} },
+ { 16, { 4, 7, 9, 10, 10, 11, 11, 12, 13, 13, 13, 14, 14, 15, 16} },
+ {16.5, { 4, 5, 7, 8, 10, 11, 11, 12, 13, 13, 13, 14, 14, 15, 16} },
+ { 17, { 4, 5, 7, 8, 8, 9, 11, 11, 12, 12, 12, 13, 13, 14, 16} },
+ {17.5, { 3, 5, 7, 8, 8, 9, 10, 11, 12, 12, 12, 13, 13, 14, 16} },
+ { 18, { 3, 5, 7, 8, 8, 9, 10, 11, 12, 12, 12, 13, 13, 14, 15} },
+ {18.5, { 3, 5, 7, 8, 8, 9, 10, 11, 12, 12, 12, 13, 13, 14, 15} },
+ { 19, { 3, 4, 6, 7, 8, 8, 9, 10, 11, 11, 11, 12, 12, 13, 14} },
+ {19.5, { 3, 4, 6, 7, 8, 8, 9, 10, 11, 11, 11, 12, 12, 13, 14} },
+ { 20, { 2, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10, 11, 11, 12, 14} },
+ {20.5, { 2, 3, 5, 5, 7, 8, 8, 8, 9, 10, 10, 11, 11, 12, 14} },
+ { 21, { 2, 3, 5, 5, 7, 7, 7, 8, 8, 9, 10, 11, 11, 12, 14} },
+ {21.5, { 2, 3, 5, 5, 6, 6, 7, 8, 8, 9, 10, 11, 11, 12, 13} },
+ { 22, { 2, 2, 4, 5, 6, 6, 7, 7, 8, 9, 10, 10, 10, 11, 13} },
+ {22.5, { 2, 2, 4, 5, 5, 6, 7, 7, 8, 9, 10, 10, 10, 11, 12} },
+ { 23, { 2, 2, 4, 5, 5, 6, 7, 7, 7, 8, 10, 10, 10, 10, 12} },
+ {23.5, { 2, 2, 3, 5, 5, 6, 7, 7, 7, 8, 10, 10, 10, 10, 12} },
+ { 24, { 2, 2, 3, 4, 4, 5, 7, 7, 7, 8, 9, 9, 9, 10, 12} },
+ {24.5, { 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 9, 9, 10, 12} },
+ { 25, { 1, 2, 3, 3, 4, 5, 6, 6, 7, 8, 9, 9, 9, 9, 11} },
+ {25.5, { 1, 1, 3, 3, 4, 5, 6, 6, 7, 7, 8, 9, 9, 9, 11} },
+ { 26, { 1, 1, 3, 3, 3, 4, 5, 6, 6, 7, 8, 8, 8, 8, 10} },
+ {26.5, { 1, 1, 2, 3, 3, 4, 5, 6, 6, 6, 8, 8, 8, 8, 10} },
+ { 27, { 1, 1, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8, 8, 8, 10} },
+ {27.5, { 1, 1, 2, 2, 3, 3, 5, 5, 5, 6, 7, 8, 8, 8, 10} },
+ { 28, { 0, 1, 1, 2, 2, 3, 4, 5, 5, 5, 7, 7, 7, 8, 10} },
+ {28.5, { 0, 1, 1, 2, 2, 3, 4, 4, 5, 5, 6, 7, 7, 7, 9} },
+ { 29, { 0, 1, 1, 2, 2, 3, 4, 4, 5, 5, 6, 6, 7, 7, 9} },
+ {29.5, { 0, 1, 1, 2, 2, 2, 3, 3, 4, 5, 6, 6, 7, 7, 8} },
+ { 30, { 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 6, 6, 6, 6, 8} },
+ {30.5, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 5, 6, 8} },
+ { 31, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 5, 6, 8} },
+ {31.5, { 0, 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 8} },
+ { 32, { 0, 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 4, 5, 7} },
+ {32.5, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, 7} },
+ { 33, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 6} },
+ {33.5, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 6} },
+ { 34, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 3, 4, 6} },
+ {34.5, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 3, 3, 3, 3, 4, 6} },
+ { 35, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 5} },
+ {35.5, { 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 5} },
+ { 36, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 4} }
+};
+
+
+const qp_table qp_table_420_8bpc_min = {
+ { 4, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 9, 13} },
+ { 4.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 8, 12} },
+ { 5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 8, 12} },
+ { 5.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 11} },
+ { 6, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 10} },
+ { 6.5, { 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 4, 5, 5, 7, 10} },
+ { 7, { 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 4, 5, 5, 7, 9} },
+ { 7.5, { 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 4, 4, 5, 7, 8} },
+ { 8, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 8} },
+ { 8.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 6, 7} },
+ { 9, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6} },
+ { 9.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6} },
+ { 10, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5} },
+ {10.5, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 3, 5} },
+ { 11, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4} },
+ {11.5, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4} },
+ { 12, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 3} }
+};
+
+
+const qp_table qp_table_422_8bpc_min = {
+ { 6, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 8, 12} },
+ { 6.5, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 8, 12} },
+ { 7, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 11} },
+ { 7.5, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 11} },
+ { 8, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 10} },
+ { 8.5, { 0, 0, 1, 2, 2, 2, 2, 3, 3, 3, 4, 5, 5, 7, 10} },
+ { 9, { 0, 0, 0, 1, 2, 2, 2, 2, 2, 3, 4, 5, 5, 7, 9} },
+ { 9.5, { 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 9} },
+ { 10, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 8} },
+ {10.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 7, 8} },
+ { 11, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 6, 7} },
+ {11.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 6, 7} },
+ { 12, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 4, 4, 5, 5, 6} },
+ {12.5, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6} },
+ { 13, { 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5} },
+ {13.5, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 4, 5} },
+ { 14, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4} },
+ {14.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 4} },
+ { 15, { 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 4} },
+ {15.5, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 3} },
+ { 16, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 3} }
+};
+
+
+const qp_table qp_table_422_10bpc_max = {
+ { 6, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} },
+ { 6.5, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} },
+ { 7, { 7, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16} },
+ { 7.5, { 5, 6, 8, 10, 11, 11, 11, 12, 12, 13, 13, 14, 14, 15, 16} },
+ { 8, { 4, 6, 7, 9, 10, 11, 11, 12, 12, 13, 13, 13, 13, 14, 15} },
+ { 8.5, { 4, 5, 6, 8, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 15} },
+ { 9, { 3, 4, 5, 7, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14} },
+ { 9.5, { 3, 4, 4, 6, 6, 8, 8, 9, 10, 10, 11, 11, 12, 13, 14} },
+ { 10, { 2, 3, 3, 5, 5, 7, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
+ {10.5, { 2, 3, 3, 5, 5, 6, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
+ { 11, { 2, 3, 3, 5, 5, 6, 7, 8, 9, 9, 10, 10, 11, 11, 12} },
+ {11.5, { 2, 3, 3, 5, 5, 5, 7, 7, 8, 9, 9, 10, 10, 11, 12} },
+ { 12, { 2, 3, 3, 5, 5, 5, 7, 7, 8, 8, 9, 9, 10, 10, 11} },
+ {12.5, { 2, 2, 3, 4, 5, 5, 6, 7, 7, 8, 8, 9, 9, 10, 11} },
+ { 13, { 1, 2, 3, 4, 5, 5, 6, 7, 7, 8, 8, 8, 9, 9, 10} },
+ {13.5, { 1, 2, 3, 3, 4, 5, 6, 6, 7, 7, 8, 8, 8, 9, 10} },
+ { 14, { 1, 2, 3, 3, 4, 5, 5, 5, 6, 6, 7, 8, 8, 8, 9} },
+ {14.5, { 1, 2, 2, 3, 4, 4, 5, 5, 5, 6, 7, 7, 7, 8, 9} },
+ { 15, { 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7, 7, 7, 9} },
+ {15.5, { 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 6, 6, 6, 7, 8} },
+ { 16, { 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6, 8} },
+ {16.5, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4, 5, 5, 6, 7} },
+ { 17, { 0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 7} },
+ {17.5, { 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 5, 6} },
+ { 18, { 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 6} },
+ {18.5, { 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 6} },
+ { 19, { 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 5} },
+ {19.5, { 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 5} },
+ { 20, { 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 4} }
+};
+
+
+const qp_table qp_table_420_10bpc_max = {
+ { 4, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 17, 18} },
+ { 4.5, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} },
+ { 5, { 7, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 15, 16, 17} },
+ { 5.5, { 6, 7, 8, 9, 10, 10, 11, 12, 12, 13, 13, 14, 14, 15, 16} },
+ { 6, { 4, 6, 8, 9, 10, 10, 11, 12, 12, 13, 13, 13, 13, 14, 15} },
+ { 6.5, { 4, 5, 7, 8, 8, 9, 10, 11, 11, 12, 12, 13, 13, 14, 15} },
+ { 7, { 3, 4, 6, 7, 7, 8, 9, 10, 10, 11, 12, 12, 13, 13, 14} },
+ { 7.5, { 3, 4, 5, 6, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 13} },
+ { 8, { 2, 3, 4, 5, 5, 6, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
+ { 8.5, { 1, 3, 3, 4, 4, 6, 7, 8, 9, 9, 10, 10, 11, 11, 12} },
+ { 9, { 1, 3, 3, 4, 4, 6, 7, 8, 8, 9, 9, 10, 10, 10, 11} },
+ { 9.5, { 1, 3, 3, 3, 4, 5, 6, 7, 8, 8, 9, 9, 9, 10, 11} },
+ { 10, { 1, 2, 3, 3, 4, 4, 5, 6, 7, 8, 8, 9, 9, 9, 11} },
+ {10.5, { 1, 1, 3, 3, 3, 4, 5, 5, 7, 7, 8, 8, 8, 8, 10} },
+ { 11, { 0, 1, 2, 3, 3, 3, 4, 5, 6, 7, 7, 7, 8, 8, 9} },
+ {11.5, { 0, 1, 1, 2, 3, 3, 3, 4, 5, 6, 6, 7, 7, 7, 9} },
+ { 12, { 0, 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 6, 6, 8} },
+ {12.5, { 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 7} },
+ { 13, { 0, 0, 0, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, 7} },
+ {13.5, { 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 4, 6} },
+ { 14, { 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 6} },
+ {14.5, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 5} },
+ { 15, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 2, 3, 5} }
+};
+
+
+const qp_table qp_table_420_10bpc_min = {
+ { 4, { 0, 4, 4, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 13, 17} },
+ { 4.5, { 0, 4, 4, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 16} },
+ { 5, { 0, 4, 4, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 16} },
+ { 5.5, { 0, 3, 3, 4, 6, 7, 7, 7, 7, 7, 9, 9, 9, 11, 15} },
+ { 6, { 0, 2, 3, 4, 6, 7, 7, 7, 7, 7, 9, 9, 9, 11, 14} },
+ { 6.5, { 0, 2, 3, 4, 5, 6, 6, 7, 7, 7, 8, 9, 9, 11, 14} },
+ { 7, { 0, 2, 3, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 11, 13} },
+ { 7.5, { 0, 2, 3, 4, 4, 5, 5, 6, 7, 7, 8, 8, 9, 11, 12} },
+ { 8, { 0, 2, 3, 4, 4, 5, 5, 6, 6, 7, 8, 8, 9, 11, 12} },
+ { 8.5, { 0, 2, 2, 3, 3, 5, 5, 6, 6, 7, 8, 8, 9, 10, 11} },
+ { 9, { 0, 2, 2, 3, 3, 5, 5, 6, 6, 7, 7, 8, 9, 9, 10} },
+ { 9.5, { 0, 2, 2, 2, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 10} },
+ { 10, { 0, 1, 2, 2, 3, 3, 4, 4, 5, 6, 6, 8, 8, 8, 10} },
+ {10.5, { 0, 0, 2, 2, 2, 3, 4, 4, 5, 5, 6, 7, 7, 7, 9} },
+ { 11, { 0, 0, 1, 2, 2, 2, 3, 4, 4, 5, 5, 6, 7, 7, 8} },
+ {11.5, { 0, 0, 0, 1, 2, 2, 2, 3, 4, 4, 5, 6, 6, 6, 8} },
+ { 12, { 0, 0, 0, 0, 1, 1, 2, 2, 3, 4, 4, 5, 5, 5, 7} },
+ {12.5, { 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 6} },
+ { 13, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 6} },
+ {13.5, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 3, 5} },
+ { 14, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 5} },
+ {14.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 4} },
+ { 15, { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 4} }
+};
+
+
+const qp_table qp_table_444_10bpc_max = {
+ { 6, { 8, 10, 12, 12, 13, 13, 13, 14, 15, 16, 16, 16, 16, 17, 19} },
+ { 6.5, { 8, 10, 11, 12, 12, 12, 13, 14, 15, 15, 16, 16, 16, 17, 19} },
+ { 7, { 8, 9, 11, 11, 12, 12, 12, 13, 14, 15, 15, 16, 16, 17, 18} },
+ { 7.5, { 8, 9, 10, 11, 11, 12, 12, 13, 14, 14, 15, 15, 16, 17, 18} },
+ { 8, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} },
+ { 8.5, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} },
+ { 9, { 7, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 15, 15, 17} },
+ { 9.5, { 7, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 15, 15, 17} },
+ { 10, { 7, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16} },
+ {10.5, { 6, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 14, 15, 16} },
+ { 11, { 5, 7, 9, 10, 11, 11, 11, 12, 12, 13, 13, 13, 14, 14, 15} },
+ {11.5, { 5, 7, 9, 10, 11, 11, 11, 12, 12, 13, 13, 13, 13, 14, 15} },
+ { 12, { 4, 6, 8, 9, 10, 10, 11, 12, 12, 13, 13, 13, 13, 14, 15} },
+ {12.5, { 4, 6, 8, 9, 10, 10, 10, 11, 11, 12, 12, 13, 13, 14, 15} },
+ { 13, { 3, 6, 8, 9, 9, 10, 10, 11, 11, 12, 12, 12, 12, 13, 14} },
+ {13.5, { 3, 5, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14} },
+ { 14, { 3, 5, 6, 7, 8, 8, 8, 9, 10, 10, 11, 12, 12, 12, 14} },
+ {14.5, { 2, 4, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 13} },
+ { 15, { 2, 4, 5, 6, 6, 7, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
+ {15.5, { 2, 4, 5, 6, 6, 7, 7, 8, 9, 9, 10, 11, 11, 12, 13} },
+ { 16, { 2, 4, 5, 6, 6, 7, 7, 8, 9, 9, 9, 10, 10, 11, 12} },
+ {16.5, { 2, 3, 4, 5, 6, 7, 7, 8, 9, 9, 9, 10, 10, 11, 12} },
+ { 17, { 2, 3, 4, 5, 5, 6, 7, 7, 8, 8, 8, 9, 9, 10, 12} },
+ {17.5, { 1, 3, 4, 5, 5, 6, 6, 7, 8, 8, 8, 9, 9, 10, 12} },
+ { 18, { 1, 3, 4, 5, 5, 6, 6, 7, 8, 8, 8, 9, 9, 10, 11} },
+ {18.5, { 1, 3, 4, 5, 5, 6, 6, 7, 8, 8, 8, 9, 9, 10, 11} },
+ { 19, { 1, 2, 3, 4, 5, 5, 5, 6, 7, 7, 7, 8, 8, 9, 10} },
+ {19.5, { 1, 2, 3, 4, 5, 5, 5, 6, 7, 7, 7, 8, 8, 9, 10} },
+ { 20, { 1, 2, 3, 3, 4, 5, 5, 6, 6, 6, 6, 7, 7, 8, 10} },
+ {20.5, { 1, 2, 3, 3, 4, 5, 5, 5, 5, 6, 6, 7, 7, 8, 10} },
+ { 21, { 1, 2, 3, 3, 4, 4, 4, 5, 5, 5, 6, 7, 7, 8, 10} },
+ {21.5, { 1, 2, 3, 3, 3, 3, 4, 5, 5, 5, 6, 7, 7, 8, 9} },
+ { 22, { 1, 1, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6, 6, 7, 9} },
+ {22.5, { 1, 1, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6, 6, 7, 8} },
+ { 23, { 1, 1, 2, 3, 3, 3, 4, 4, 4, 4, 6, 6, 6, 6, 8} },
+ {23.5, { 1, 1, 1, 3, 3, 3, 4, 4, 4, 4, 6, 6, 6, 6, 8} },
+ { 24, { 1, 1, 1, 2, 2, 3, 4, 4, 4, 4, 5, 5, 5, 6, 8} },
+ {24.5, { 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 6, 8} },
+ { 25, { 0, 1, 1, 1, 2, 3, 3, 3, 4, 4, 5, 5, 5, 5, 7} },
+ {25.5, { 0, 0, 1, 1, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 7} },
+ { 26, { 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4, 4, 6} },
+ {26.5, { 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 6} },
+ { 27, { 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 6} },
+ {27.5, { 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 6} },
+ { 28, { 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 5} },
+ {28.5, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 5} },
+ { 29, { 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4} },
+ {29.5, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4} },
+ { 30, { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 4} }
+};
+
+
+const qp_table qp_table_422_8bpc_max = {
+ { 6, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} },
+ { 6.5, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} },
+ { 7, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 11, 12} },
+ { 7.5, { 3, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 10, 10, 11, 12} },
+ { 8, { 2, 4, 5, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 10, 11} },
+ { 8.5, { 2, 3, 4, 5, 6, 6, 6, 7, 7, 8, 8, 9, 9, 10, 11} },
+ { 9, { 1, 2, 3, 4, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 10} },
+ { 9.5, { 1, 2, 2, 3, 3, 4, 4, 5, 6, 6, 7, 7, 8, 9, 10} },
+ { 10, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9} },
+ {10.5, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 7, 8, 9} },
+ { 11, { 0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 6, 7, 7, 8} },
+ {11.5, { 0, 1, 1, 2, 2, 2, 3, 3, 4, 5, 5, 6, 6, 7, 8} },
+ { 12, { 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7} },
+ {12.5, { 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7} },
+ { 13, { 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 6} },
+ {13.5, { 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 6} },
+ { 14, { 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5} },
+ {14.5, { 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 5} },
+ { 15, { 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 5} },
+ {15.5, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 4} },
+ { 16, { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 4} }
+};
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c
new file mode 100644
index 000000000000..ca51e83f8764
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c
@@ -0,0 +1,258 @@
+#if defined(CONFIG_DRM_AMD_DC_DSC_SUPPORT)
+
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "os_types.h"
+#include "rc_calc.h"
+#include "qp_tables.h"
+
+#define table_hash(mode, bpc, max_min) ((mode << 16) | (bpc << 8) | max_min)
+
+#define MODE_SELECT(val444, val422, val420) \
+ (cm == CM_444 || cm == CM_RGB) ? (val444) : (cm == CM_422 ? (val422) : (val420))
+
+
+#define TABLE_CASE(mode, bpc, max) case (table_hash(mode, BPC_##bpc, max)): \
+ table = qp_table_##mode##_##bpc##bpc_##max; \
+ table_size = sizeof(qp_table_##mode##_##bpc##bpc_##max)/sizeof(*qp_table_##mode##_##bpc##bpc_##max); \
+ break
+
+
+void get_qp_set(qp_set qps, enum colour_mode cm, enum bits_per_comp bpc, enum max_min max_min, float bpp)
+{
+ int mode = MODE_SELECT(444, 422, 420);
+ int sel = table_hash(mode, bpc, max_min);
+ int table_size = 0;
+ int index;
+ const struct qp_entry *table = 0L;
+
+ // alias enum
+ enum { min = MM_MIN, max = MM_MAX };
+ switch (sel) {
+ TABLE_CASE(444, 8, max);
+ TABLE_CASE(444, 8, min);
+ TABLE_CASE(444, 10, max);
+ TABLE_CASE(444, 10, min);
+ TABLE_CASE(444, 12, max);
+ TABLE_CASE(444, 12, min);
+ TABLE_CASE(422, 8, max);
+ TABLE_CASE(422, 8, min);
+ TABLE_CASE(422, 10, max);
+ TABLE_CASE(422, 10, min);
+ TABLE_CASE(422, 12, max);
+ TABLE_CASE(422, 12, min);
+ TABLE_CASE(420, 8, max);
+ TABLE_CASE(420, 8, min);
+ TABLE_CASE(420, 10, max);
+ TABLE_CASE(420, 10, min);
+ TABLE_CASE(420, 12, max);
+ TABLE_CASE(420, 12, min);
+ }
+
+ if (table == 0)
+ return;
+
+ index = (bpp - table[0].bpp) * 2;
+
+ /* requested size is bigger than the table */
+ if (index >= table_size) {
+ dm_error("ERROR: Requested rc_calc to find a bpp entry that exceeds the table size\n");
+ return;
+ }
+
+ memcpy(qps, table[index].qps, sizeof(qp_set));
+}
+
+double dsc_roundf(double num)
+{
+ if (num < 0.0)
+ num = num - 0.5;
+ else
+ num = num + 0.5;
+
+ return (int)(num);
+}
+
+double dsc_ceil(double num)
+{
+ double retval = (int)num;
+
+ if (retval != num && num > 0)
+ retval = num + 1;
+
+ return (int)retval;
+}
+
+void get_ofs_set(qp_set ofs, enum colour_mode mode, float bpp)
+{
+ int *p = ofs;
+
+ if (mode == CM_444 || mode == CM_RGB) {
+ *p++ = (bpp <= 6) ? (0) : ((((bpp >= 8) && (bpp <= 12))) ? (2) : ((bpp >= 15) ? (10) : ((((bpp > 6) && (bpp < 8))) ? (0 + dsc_roundf((bpp - 6) * (2 / 2.0))) : (2 + dsc_roundf((bpp - 12) * (8 / 3.0))))));
+ *p++ = (bpp <= 6) ? (-2) : ((((bpp >= 8) && (bpp <= 12))) ? (0) : ((bpp >= 15) ? (8) : ((((bpp > 6) && (bpp < 8))) ? (-2 + dsc_roundf((bpp - 6) * (2 / 2.0))) : (0 + dsc_roundf((bpp - 12) * (8 / 3.0))))));
+ *p++ = (bpp <= 6) ? (-2) : ((((bpp >= 8) && (bpp <= 12))) ? (0) : ((bpp >= 15) ? (6) : ((((bpp > 6) && (bpp < 8))) ? (-2 + dsc_roundf((bpp - 6) * (2 / 2.0))) : (0 + dsc_roundf((bpp - 12) * (6 / 3.0))))));
+ *p++ = (bpp <= 6) ? (-4) : ((((bpp >= 8) && (bpp <= 12))) ? (-2) : ((bpp >= 15) ? (4) : ((((bpp > 6) && (bpp < 8))) ? (-4 + dsc_roundf((bpp - 6) * (2 / 2.0))) : (-2 + dsc_roundf((bpp - 12) * (6 / 3.0))))));
+ *p++ = (bpp <= 6) ? (-6) : ((((bpp >= 8) && (bpp <= 12))) ? (-4) : ((bpp >= 15) ? (2) : ((((bpp > 6) && (bpp < 8))) ? (-6 + dsc_roundf((bpp - 6) * (2 / 2.0))) : (-4 + dsc_roundf((bpp - 12) * (6 / 3.0))))));
+ *p++ = (bpp <= 12) ? (-6) : ((bpp >= 15) ? (0) : (-6 + dsc_roundf((bpp - 12) * (6 / 3.0))));
+ *p++ = (bpp <= 12) ? (-8) : ((bpp >= 15) ? (-2) : (-8 + dsc_roundf((bpp - 12) * (6 / 3.0))));
+ *p++ = (bpp <= 12) ? (-8) : ((bpp >= 15) ? (-4) : (-8 + dsc_roundf((bpp - 12) * (4 / 3.0))));
+ *p++ = (bpp <= 12) ? (-8) : ((bpp >= 15) ? (-6) : (-8 + dsc_roundf((bpp - 12) * (2 / 3.0))));
+ *p++ = (bpp <= 12) ? (-10) : ((bpp >= 15) ? (-8) : (-10 + dsc_roundf((bpp - 12) * (2 / 3.0))));
+ *p++ = -10;
+ *p++ = (bpp <= 6) ? (-12) : ((bpp >= 8) ? (-10) : (-12 + dsc_roundf((bpp - 6) * (2 / 2.0))));
+ *p++ = -12;
+ *p++ = -12;
+ *p++ = -12;
+ } else if (mode == CM_422) {
+ *p++ = (bpp <= 8) ? (2) : ((bpp >= 10) ? (10) : (2 + dsc_roundf((bpp - 8) * (8 / 2.0))));
+ *p++ = (bpp <= 8) ? (0) : ((bpp >= 10) ? (8) : (0 + dsc_roundf((bpp - 8) * (8 / 2.0))));
+ *p++ = (bpp <= 8) ? (0) : ((bpp >= 10) ? (6) : (0 + dsc_roundf((bpp - 8) * (6 / 2.0))));
+ *p++ = (bpp <= 8) ? (-2) : ((bpp >= 10) ? (4) : (-2 + dsc_roundf((bpp - 8) * (6 / 2.0))));
+ *p++ = (bpp <= 8) ? (-4) : ((bpp >= 10) ? (2) : (-4 + dsc_roundf((bpp - 8) * (6 / 2.0))));
+ *p++ = (bpp <= 8) ? (-6) : ((bpp >= 10) ? (0) : (-6 + dsc_roundf((bpp - 8) * (6 / 2.0))));
+ *p++ = (bpp <= 8) ? (-8) : ((bpp >= 10) ? (-2) : (-8 + dsc_roundf((bpp - 8) * (6 / 2.0))));
+ *p++ = (bpp <= 8) ? (-8) : ((bpp >= 10) ? (-4) : (-8 + dsc_roundf((bpp - 8) * (4 / 2.0))));
+ *p++ = (bpp <= 8) ? (-8) : ((bpp >= 10) ? (-6) : (-8 + dsc_roundf((bpp - 8) * (2 / 2.0))));
+ *p++ = (bpp <= 8) ? (-10) : ((bpp >= 10) ? (-8) : (-10 + dsc_roundf((bpp - 8) * (2 / 2.0))));
+ *p++ = -10;
+ *p++ = (bpp <= 6) ? (-12) : ((bpp >= 7) ? (-10) : (-12 + dsc_roundf((bpp - 6) * (2.0 / 1))));
+ *p++ = -12;
+ *p++ = -12;
+ *p++ = -12;
+ } else {
+ *p++ = (bpp <= 6) ? (2) : ((bpp >= 8) ? (10) : (2 + dsc_roundf((bpp - 6) * (8 / 2.0))));
+ *p++ = (bpp <= 6) ? (0) : ((bpp >= 8) ? (8) : (0 + dsc_roundf((bpp - 6) * (8 / 2.0))));
+ *p++ = (bpp <= 6) ? (0) : ((bpp >= 8) ? (6) : (0 + dsc_roundf((bpp - 6) * (6 / 2.0))));
+ *p++ = (bpp <= 6) ? (-2) : ((bpp >= 8) ? (4) : (-2 + dsc_roundf((bpp - 6) * (6 / 2.0))));
+ *p++ = (bpp <= 6) ? (-4) : ((bpp >= 8) ? (2) : (-4 + dsc_roundf((bpp - 6) * (6 / 2.0))));
+ *p++ = (bpp <= 6) ? (-6) : ((bpp >= 8) ? (0) : (-6 + dsc_roundf((bpp - 6) * (6 / 2.0))));
+ *p++ = (bpp <= 6) ? (-8) : ((bpp >= 8) ? (-2) : (-8 + dsc_roundf((bpp - 6) * (6 / 2.0))));
+ *p++ = (bpp <= 6) ? (-8) : ((bpp >= 8) ? (-4) : (-8 + dsc_roundf((bpp - 6) * (4 / 2.0))));
+ *p++ = (bpp <= 6) ? (-8) : ((bpp >= 8) ? (-6) : (-8 + dsc_roundf((bpp - 6) * (2 / 2.0))));
+ *p++ = (bpp <= 6) ? (-10) : ((bpp >= 8) ? (-8) : (-10 + dsc_roundf((bpp - 6) * (2 / 2.0))));
+ *p++ = -10;
+ *p++ = (bpp <= 4) ? (-12) : ((bpp >= 5) ? (-10) : (-12 + dsc_roundf((bpp - 4) * (2 / 1.0))));
+ *p++ = -12;
+ *p++ = -12;
+ *p++ = -12;
+ }
+}
+
+int median3(int a, int b, int c)
+{
+ if (a > b)
+ swap(a, b);
+ if (b > c)
+ swap(b, c);
+ if (a > b)
+ swap(b, c);
+
+ return b;
+}
+
+void calc_rc_params(struct rc_params *rc, enum colour_mode cm, enum bits_per_comp bpc, float bpp, int slice_width, int slice_height, int minor_version)
+{
+ float bpp_group;
+ float initial_xmit_delay_factor;
+ int source_bpp;
+ int padding_pixels;
+ int i;
+
+ rc->rc_quant_incr_limit0 = ((bpc == BPC_8) ? 11 : (bpc == BPC_10 ? 15 : 19)) - ((minor_version == 1 && cm == CM_444) ? 1 : 0);
+ rc->rc_quant_incr_limit1 = ((bpc == BPC_8) ? 11 : (bpc == BPC_10 ? 15 : 19)) - ((minor_version == 1 && cm == CM_444) ? 1 : 0);
+
+ bpp_group = MODE_SELECT(bpp, bpp * 2.0, bpp * 2.0);
+
+ switch (cm) {
+ case CM_420:
+ rc->initial_fullness_offset = (bpp >= 6) ? (2048) : ((bpp <= 4) ? (6144) : ((((bpp > 4) && (bpp <= 5))) ? (6144 - dsc_roundf((bpp - 4) * (512))) : (5632 - dsc_roundf((bpp - 5) * (3584)))));
+ rc->first_line_bpg_offset = median3(0, (12 + (int) (0.09 * min(34, slice_height - 8))), (int)((3 * bpc * 3) - (3 * bpp_group)));
+ rc->second_line_bpg_offset = median3(0, 12, (int)((3 * bpc * 3) - (3 * bpp_group)));
+ break;
+ case CM_422:
+ rc->initial_fullness_offset = (bpp >= 8) ? (2048) : ((bpp <= 7) ? (5632) : (5632 - dsc_roundf((bpp - 7) * (3584))));
+ rc->first_line_bpg_offset = median3(0, (12 + (int) (0.09 * min(34, slice_height - 8))), (int)((3 * bpc * 4) - (3 * bpp_group)));
+ rc->second_line_bpg_offset = 0;
+ break;
+ case CM_444:
+ case CM_RGB:
+ rc->initial_fullness_offset = (bpp >= 12) ? (2048) : ((bpp <= 8) ? (6144) : ((((bpp > 8) && (bpp <= 10))) ? (6144 - dsc_roundf((bpp - 8) * (512 / 2))) : (5632 - dsc_roundf((bpp - 10) * (3584 / 2)))));
+ rc->first_line_bpg_offset = median3(0, (12 + (int) (0.09 * min(34, slice_height - 8))), (int)(((3 * bpc + (cm == CM_444 ? 0 : 2)) * 3) - (3 * bpp_group)));
+ rc->second_line_bpg_offset = 0;
+ break;
+ }
+
+ initial_xmit_delay_factor = (cm == CM_444 || cm == CM_RGB) ? 1.0 : 2.0;
+ rc->initial_xmit_delay = dsc_roundf(8192.0/2.0/bpp/initial_xmit_delay_factor);
+
+ if (cm == CM_422 || cm == CM_420)
+ slice_width /= 2;
+
+ padding_pixels = ((slice_width % 3) != 0) ? (3 - (slice_width % 3)) * (rc->initial_xmit_delay / slice_width) : 0;
+ if (3 * bpp_group >= (((rc->initial_xmit_delay + 2) / 3) * (3 + (cm == CM_422)))) {
+ if ((rc->initial_xmit_delay + padding_pixels) % 3 == 1)
+ rc->initial_xmit_delay++;
+ }
+
+ source_bpp = MODE_SELECT(bpc * 3, bpc * 2, bpc * 1.5);
+
+ rc->flatness_min_qp = ((bpc == BPC_8) ? (3) : ((bpc == BPC_10) ? (7) : (11))) - ((minor_version == 1 && cm == CM_444) ? 1 : 0);
+ rc->flatness_max_qp = ((bpc == BPC_8) ? (12) : ((bpc == BPC_10) ? (16) : (20))) - ((minor_version == 1 && cm == CM_444) ? 1 : 0);
+ rc->flatness_det_thresh = 2 << (bpc - 8);
+
+ get_qp_set(rc->qp_min, cm, bpc, MM_MIN, bpp);
+ get_qp_set(rc->qp_max, cm, bpc, MM_MAX, bpp);
+ if (cm == CM_444 && minor_version == 1) {
+ for (i = 0; i < QP_SET_SIZE; ++i) {
+ rc->qp_min[i] = rc->qp_min[i] > 0 ? rc->qp_min[i] - 1 : 0;
+ rc->qp_max[i] = rc->qp_max[i] > 0 ? rc->qp_max[i] - 1 : 0;
+ }
+ }
+ get_ofs_set(rc->ofs, cm, bpp);
+
+ /* fixed parameters */
+ rc->rc_model_size = 8192;
+ rc->rc_edge_factor = 6;
+ rc->rc_tgt_offset_hi = 3;
+ rc->rc_tgt_offset_lo = 3;
+
+ rc->rc_buf_thresh[0] = 896;
+ rc->rc_buf_thresh[1] = 1792;
+ rc->rc_buf_thresh[2] = 2688;
+ rc->rc_buf_thresh[3] = 3584;
+ rc->rc_buf_thresh[4] = 4480;
+ rc->rc_buf_thresh[5] = 5376;
+ rc->rc_buf_thresh[6] = 6272;
+ rc->rc_buf_thresh[7] = 6720;
+ rc->rc_buf_thresh[8] = 7168;
+ rc->rc_buf_thresh[9] = 7616;
+ rc->rc_buf_thresh[10] = 7744;
+ rc->rc_buf_thresh[11] = 7872;
+ rc->rc_buf_thresh[12] = 8000;
+ rc->rc_buf_thresh[13] = 8064;
+}
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h
new file mode 100644
index 000000000000..f1d6e793bc61
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h
@@ -0,0 +1,85 @@
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __RC_CALC_H__
+#define __RC_CALC_H__
+
+
+#define QP_SET_SIZE 15
+
+typedef int qp_set[QP_SET_SIZE];
+
+struct rc_params {
+ int rc_quant_incr_limit0;
+ int rc_quant_incr_limit1;
+ int initial_fullness_offset;
+ int initial_xmit_delay;
+ int first_line_bpg_offset;
+ int second_line_bpg_offset;
+ int flatness_min_qp;
+ int flatness_max_qp;
+ int flatness_det_thresh;
+ qp_set qp_min;
+ qp_set qp_max;
+ qp_set ofs;
+ int rc_model_size;
+ int rc_edge_factor;
+ int rc_tgt_offset_hi;
+ int rc_tgt_offset_lo;
+ int rc_buf_thresh[QP_SET_SIZE - 1];
+};
+
+enum colour_mode {
+ CM_RGB, /* 444 RGB */
+ CM_444, /* 444 YUV or simple 422 */
+ CM_422, /* native 422 */
+ CM_420 /* native 420 */
+};
+
+enum bits_per_comp {
+ BPC_8 = 8,
+ BPC_10 = 10,
+ BPC_12 = 12
+};
+
+enum max_min {
+ MM_MIN = 0,
+ MM_MAX = 1
+};
+
+struct qp_entry {
+ float bpp;
+ const qp_set qps;
+};
+
+typedef struct qp_entry qp_table[];
+
+void calc_rc_params(struct rc_params *rc, enum colour_mode cm, enum bits_per_comp bpc, float bpp, int slice_width, int slice_height, int minor_version);
+
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c
new file mode 100644
index 000000000000..73172fd0b529
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c
@@ -0,0 +1,147 @@
+#if defined(CONFIG_DRM_AMD_DC_DSC_SUPPORT)
+/*
+ * Copyright 2012-17 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+#include "os_types.h"
+#include <drm/drm_dsc.h>
+#include "dscc_types.h"
+#include "rc_calc.h"
+
+double dsc_ceil(double num);
+
+static void copy_pps_fields(struct drm_dsc_config *to, const struct drm_dsc_config *from)
+{
+ to->line_buf_depth = from->line_buf_depth;
+ to->bits_per_component = from->bits_per_component;
+ to->convert_rgb = from->convert_rgb;
+ to->slice_width = from->slice_width;
+ to->slice_height = from->slice_height;
+ to->simple_422 = from->simple_422;
+ to->native_422 = from->native_422;
+ to->native_420 = from->native_420;
+ to->pic_width = from->pic_width;
+ to->pic_height = from->pic_height;
+ to->rc_tgt_offset_high = from->rc_tgt_offset_high;
+ to->rc_tgt_offset_low = from->rc_tgt_offset_low;
+ to->bits_per_pixel = from->bits_per_pixel;
+ to->rc_edge_factor = from->rc_edge_factor;
+ to->rc_quant_incr_limit1 = from->rc_quant_incr_limit1;
+ to->rc_quant_incr_limit0 = from->rc_quant_incr_limit0;
+ to->initial_xmit_delay = from->initial_xmit_delay;
+ to->initial_dec_delay = from->initial_dec_delay;
+ to->block_pred_enable = from->block_pred_enable;
+ to->first_line_bpg_offset = from->first_line_bpg_offset;
+ to->second_line_bpg_offset = from->second_line_bpg_offset;
+ to->initial_offset = from->initial_offset;
+ memcpy(&to->rc_buf_thresh, &from->rc_buf_thresh, sizeof(from->rc_buf_thresh));
+ memcpy(&to->rc_range_params, &from->rc_range_params, sizeof(from->rc_range_params));
+ to->rc_model_size = from->rc_model_size;
+ to->flatness_min_qp = from->flatness_min_qp;
+ to->flatness_max_qp = from->flatness_max_qp;
+ to->initial_scale_value = from->initial_scale_value;
+ to->scale_decrement_interval = from->scale_decrement_interval;
+ to->scale_increment_interval = from->scale_increment_interval;
+ to->nfl_bpg_offset = from->nfl_bpg_offset;
+ to->nsl_bpg_offset = from->nsl_bpg_offset;
+ to->slice_bpg_offset = from->slice_bpg_offset;
+ to->final_offset = from->final_offset;
+ to->vbr_enable = from->vbr_enable;
+ to->slice_chunk_size = from->slice_chunk_size;
+ to->second_line_offset_adj = from->second_line_offset_adj;
+ to->dsc_version_minor = from->dsc_version_minor;
+}
+
+static void copy_rc_to_cfg(struct drm_dsc_config *dsc_cfg, const struct rc_params *rc)
+{
+ int i;
+
+ dsc_cfg->rc_quant_incr_limit0 = rc->rc_quant_incr_limit0;
+ dsc_cfg->rc_quant_incr_limit1 = rc->rc_quant_incr_limit1;
+ dsc_cfg->initial_offset = rc->initial_fullness_offset;
+ dsc_cfg->initial_xmit_delay = rc->initial_xmit_delay;
+ dsc_cfg->first_line_bpg_offset = rc->first_line_bpg_offset;
+ dsc_cfg->second_line_bpg_offset = rc->second_line_bpg_offset;
+ dsc_cfg->flatness_min_qp = rc->flatness_min_qp;
+ dsc_cfg->flatness_max_qp = rc->flatness_max_qp;
+ for (i = 0; i < QP_SET_SIZE; ++i) {
+ dsc_cfg->rc_range_params[i].range_min_qp = rc->qp_min[i];
+ dsc_cfg->rc_range_params[i].range_max_qp = rc->qp_max[i];
+ /* Truncate 8-bit signed value to 6-bit signed value */
+ dsc_cfg->rc_range_params[i].range_bpg_offset = 0x3f & rc->ofs[i];
+ }
+ dsc_cfg->rc_model_size = rc->rc_model_size;
+ dsc_cfg->rc_edge_factor = rc->rc_edge_factor;
+ dsc_cfg->rc_tgt_offset_high = rc->rc_tgt_offset_hi;
+ dsc_cfg->rc_tgt_offset_low = rc->rc_tgt_offset_lo;
+
+ for (i = 0; i < QP_SET_SIZE - 1; ++i)
+ dsc_cfg->rc_buf_thresh[i] = rc->rc_buf_thresh[i];
+}
+
+int dscc_compute_dsc_parameters(const struct drm_dsc_config *pps, struct dsc_parameters *dsc_params)
+{
+ enum colour_mode mode = pps->convert_rgb ? CM_RGB :
+ (pps->simple_422 ? CM_444 :
+ (pps->native_422 ? CM_422 :
+ pps->native_420 ? CM_420 : CM_444));
+ enum bits_per_comp bpc = (pps->bits_per_component == 8) ? BPC_8 :
+ (pps->bits_per_component == 10) ? BPC_10 : BPC_12;
+ float bpp = ((float) pps->bits_per_pixel / 16.0);
+ int slice_width = pps->slice_width;
+ int slice_height = pps->slice_height;
+ int ret;
+ struct rc_params rc;
+ struct drm_dsc_config dsc_cfg;
+
+ double d_bytes_per_pixel = dsc_ceil(bpp * slice_width / 8.0) / slice_width;
+
+ // TODO: Make sure the formula for calculating this is precise (ceiling vs. floor, and at what point they should be applied)
+ if (pps->native_422 || pps->native_420)
+ d_bytes_per_pixel /= 2;
+
+ dsc_params->bytes_per_pixel = (uint32_t)dsc_ceil(d_bytes_per_pixel * 0x10000000);
+
+ /* in native_422 or native_420 modes, the bits_per_pixel is double the target bpp
+ * (the latter is what calc_rc_params expects)
+ */
+ if (pps->native_422 || pps->native_420)
+ bpp /= 2.0;
+
+ calc_rc_params(&rc, mode, bpc, bpp, slice_width, slice_height, pps->dsc_version_minor);
+ dsc_params->pps = *pps;
+ dsc_params->pps.initial_scale_value = 8 * rc.rc_model_size / (rc.rc_model_size - rc.initial_fullness_offset);
+
+ copy_pps_fields(&dsc_cfg, &dsc_params->pps);
+ copy_rc_to_cfg(&dsc_cfg, &rc);
+
+ dsc_cfg.mux_word_size = dsc_params->pps.bits_per_component <= 10 ? 48 : 64;
+
+ ret = drm_dsc_compute_rc_parameters(&dsc_cfg);
+
+ copy_pps_fields(&dsc_params->pps, &dsc_cfg);
+ dsc_params->rc_buffer_model_size = dsc_cfg.rc_bits;
+ return ret;
+}
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/Makefile b/drivers/gpu/drm/amd/display/dc/gpio/Makefile
index 562ee189d780..c3d92878875d 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/gpio/Makefile
@@ -70,6 +70,17 @@ AMD_DISPLAY_FILES += $(AMD_DAL_GPIO_DCN10)
endif
###############################################################################
+# DCN 2
+###############################################################################
+ifdef CONFIG_DRM_AMD_DC_DCN2_0
+GPIO_DCN20 = hw_translate_dcn20.o hw_factory_dcn20.o
+
+AMD_DAL_GPIO_DCN20 = $(addprefix $(AMDDALPATH)/dc/gpio/dcn20/,$(GPIO_DCN20))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_GPIO_DCN20)
+endif
+
+###############################################################################
# Diagnostics on FPGA
###############################################################################
GPIO_DIAG_FPGA = hw_translate_diag.o hw_factory_diag.o
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_factory_dcn20.c b/drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_factory_dcn20.c
new file mode 100644
index 000000000000..abd76d855375
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_factory_dcn20.c
@@ -0,0 +1,212 @@
+/*
+ * Copyright 2013-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#include "dm_services.h"
+#include "include/gpio_types.h"
+#include "../hw_factory.h"
+
+
+#include "../hw_gpio.h"
+#include "../hw_ddc.h"
+#include "../hw_hpd.h"
+
+#include "hw_factory_dcn20.h"
+
+
+#include "dcn/dcn_2_0_0_offset.h"
+#include "dcn/dcn_2_0_0_sh_mask.h"
+#include "navi10_ip_offset.h"
+
+
+#include "reg_helper.h"
+#include "../hpd_regs.h"
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file */
+
+/* DCN */
+#define block HPD
+#define reg_num 0
+
+#undef BASE_INNER
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+
+
+#define REG(reg_name)\
+ BASE(mm ## reg_name ## _BASE_IDX) + mm ## reg_name
+
+#define SF_HPD(reg_name, field_name, post_fix)\
+ .field_name = HPD0_ ## reg_name ## __ ## field_name ## post_fix
+
+#define REGI(reg_name, block, id)\
+ BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+ mm ## block ## id ## _ ## reg_name
+
+#define SF(reg_name, field_name, post_fix)\
+ .field_name = reg_name ## __ ## field_name ## post_fix
+
+/* macros to expend register list macro defined in HW object header file
+ * end *********************/
+
+
+
+#define hpd_regs(id) \
+{\
+ HPD_REG_LIST(id)\
+}
+
+static const struct hpd_registers hpd_regs[] = {
+ hpd_regs(0),
+ hpd_regs(1),
+ hpd_regs(2),
+ hpd_regs(3),
+ hpd_regs(4),
+ hpd_regs(5),
+};
+
+static const struct hpd_sh_mask hpd_shift = {
+ HPD_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct hpd_sh_mask hpd_mask = {
+ HPD_MASK_SH_LIST(_MASK)
+};
+
+#include "../ddc_regs.h"
+
+ /* set field name */
+#define SF_DDC(reg_name, field_name, post_fix)\
+ .field_name = reg_name ## __ ## field_name ## post_fix
+
+static const struct ddc_registers ddc_data_regs_dcn[] = {
+ ddc_data_regs_dcn2(1),
+ ddc_data_regs_dcn2(2),
+ ddc_data_regs_dcn2(3),
+ ddc_data_regs_dcn2(4),
+ ddc_data_regs_dcn2(5),
+ ddc_data_regs_dcn2(6),
+};
+
+static const struct ddc_registers ddc_clk_regs_dcn[] = {
+ ddc_clk_regs_dcn2(1),
+ ddc_clk_regs_dcn2(2),
+ ddc_clk_regs_dcn2(3),
+ ddc_clk_regs_dcn2(4),
+ ddc_clk_regs_dcn2(5),
+ ddc_clk_regs_dcn2(6),
+};
+
+static const struct ddc_sh_mask ddc_shift[] = {
+ DDC_MASK_SH_LIST_DCN2(__SHIFT, 1),
+ DDC_MASK_SH_LIST_DCN2(__SHIFT, 2),
+ DDC_MASK_SH_LIST_DCN2(__SHIFT, 3),
+ DDC_MASK_SH_LIST_DCN2(__SHIFT, 4),
+ DDC_MASK_SH_LIST_DCN2(__SHIFT, 5),
+ DDC_MASK_SH_LIST_DCN2(__SHIFT, 6)
+};
+
+static const struct ddc_sh_mask ddc_mask[] = {
+ DDC_MASK_SH_LIST_DCN2(_MASK, 1),
+ DDC_MASK_SH_LIST_DCN2(_MASK, 2),
+ DDC_MASK_SH_LIST_DCN2(_MASK, 3),
+ DDC_MASK_SH_LIST_DCN2(_MASK, 4),
+ DDC_MASK_SH_LIST_DCN2(_MASK, 5),
+ DDC_MASK_SH_LIST_DCN2(_MASK, 6)
+};
+
+static void define_ddc_registers(
+ struct hw_gpio_pin *pin,
+ uint32_t en)
+{
+ struct hw_ddc *ddc = HW_DDC_FROM_BASE(pin);
+
+ switch (pin->id) {
+ case GPIO_ID_DDC_DATA:
+ ddc->regs = &ddc_data_regs_dcn[en];
+ ddc->base.regs = &ddc_data_regs_dcn[en].gpio;
+ break;
+ case GPIO_ID_DDC_CLOCK:
+ ddc->regs = &ddc_clk_regs_dcn[en];
+ ddc->base.regs = &ddc_clk_regs_dcn[en].gpio;
+ break;
+ default:
+ ASSERT_CRITICAL(false);
+ return;
+ }
+
+ ddc->shifts = &ddc_shift[en];
+ ddc->masks = &ddc_mask[en];
+
+}
+
+static void define_hpd_registers(struct hw_gpio_pin *pin, uint32_t en)
+{
+ struct hw_hpd *hpd = HW_HPD_FROM_BASE(pin);
+
+ hpd->regs = &hpd_regs[en];
+ hpd->shifts = &hpd_shift;
+ hpd->masks = &hpd_mask;
+ hpd->base.regs = &hpd_regs[en].gpio;
+}
+
+
+/* fucntion table */
+static const struct hw_factory_funcs funcs = {
+ .create_ddc_data = dal_hw_ddc_create,
+ .create_ddc_clock = dal_hw_ddc_create,
+ .create_generic = NULL,
+ .create_hpd = dal_hw_hpd_create,
+ .create_sync = NULL,
+ .create_gsl = NULL,
+ .define_hpd_registers = define_hpd_registers,
+ .define_ddc_registers = define_ddc_registers
+};
+/*
+ * dal_hw_factory_dcn10_init
+ *
+ * @brief
+ * Initialize HW factory function pointers and pin info
+ *
+ * @param
+ * struct hw_factory *factory - [out] struct of function pointers
+ */
+void dal_hw_factory_dcn20_init(struct hw_factory *factory)
+{
+ /*TODO check ASIC CAPs*/
+ factory->number_of_pins[GPIO_ID_DDC_DATA] = 8;
+ factory->number_of_pins[GPIO_ID_DDC_CLOCK] = 8;
+ factory->number_of_pins[GPIO_ID_GENERIC] = 4;
+ factory->number_of_pins[GPIO_ID_HPD] = 6;
+ factory->number_of_pins[GPIO_ID_GPIO_PAD] = 28;
+ factory->number_of_pins[GPIO_ID_VIP_PAD] = 0;
+ factory->number_of_pins[GPIO_ID_SYNC] = 0;
+ factory->number_of_pins[GPIO_ID_GSL] = 0;/*add this*/
+
+ factory->funcs = &funcs;
+}
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_factory_dcn20.h b/drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_factory_dcn20.h
new file mode 100644
index 000000000000..43a4ce7aa3bf
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_factory_dcn20.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#ifndef __DAL_HW_FACTORY_DCN20_H__
+#define __DAL_HW_FACTORY_DCN20_H__
+
+/* Initialize HW factory function pointers and pin info */
+void dal_hw_factory_dcn20_init(struct hw_factory *factory);
+
+#endif /* __DAL_HW_FACTORY_DCN20_H__ */
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_translate_dcn20.c b/drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_translate_dcn20.c
new file mode 100644
index 000000000000..b393cc13298a
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_translate_dcn20.c
@@ -0,0 +1,382 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+/*
+ * Pre-requisites: headers required by header of this unit
+ */
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#include "hw_translate_dcn20.h"
+
+#include "dm_services.h"
+#include "include/gpio_types.h"
+#include "../hw_translate.h"
+
+#include "dcn/dcn_1_0_offset.h"
+#include "dcn/dcn_1_0_sh_mask.h"
+#include "soc15_hw_ip.h"
+#include "vega10_ip_offset.h"
+
+
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file */
+
+/* DCN */
+#define block HPD
+#define reg_num 0
+
+#undef BASE_INNER
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+#undef REG
+#define REG(reg_name)\
+ BASE(mm ## reg_name ## _BASE_IDX) + mm ## reg_name
+#define SF_HPD(reg_name, field_name, post_fix)\
+ .field_name = reg_name ## __ ## field_name ## post_fix
+
+
+/* macros to expend register list macro defined in HW object header file
+ * end *********************/
+
+
+static bool offset_to_id(
+ uint32_t offset,
+ uint32_t mask,
+ enum gpio_id *id,
+ uint32_t *en)
+{
+ switch (offset) {
+ /* GENERIC */
+ case REG(DC_GENERICA):
+ *id = GPIO_ID_GENERIC;
+ switch (mask) {
+ case DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK:
+ *en = GPIO_GENERIC_A;
+ return true;
+ case DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK:
+ *en = GPIO_GENERIC_B;
+ return true;
+ case DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK:
+ *en = GPIO_GENERIC_C;
+ return true;
+ case DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK:
+ *en = GPIO_GENERIC_D;
+ return true;
+ case DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK:
+ *en = GPIO_GENERIC_E;
+ return true;
+ case DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK:
+ *en = GPIO_GENERIC_F;
+ return true;
+ case DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK:
+ *en = GPIO_GENERIC_G;
+ return true;
+ default:
+ ASSERT_CRITICAL(false);
+ return false;
+ }
+ break;
+ /* HPD */
+ case REG(DC_GPIO_HPD_A):
+ *id = GPIO_ID_HPD;
+ switch (mask) {
+ case DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK:
+ *en = GPIO_HPD_1;
+ return true;
+ case DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK:
+ *en = GPIO_HPD_2;
+ return true;
+ case DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK:
+ *en = GPIO_HPD_3;
+ return true;
+ case DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK:
+ *en = GPIO_HPD_4;
+ return true;
+ case DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK:
+ *en = GPIO_HPD_5;
+ return true;
+ case DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK:
+ *en = GPIO_HPD_6;
+ return true;
+ default:
+ ASSERT_CRITICAL(false);
+ return false;
+ }
+ break;
+ /* REG(DC_GPIO_GENLK_MASK */
+ case REG(DC_GPIO_GENLK_A):
+ *id = GPIO_ID_GSL;
+ switch (mask) {
+ case DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK:
+ *en = GPIO_GSL_GENLOCK_CLOCK;
+ return true;
+ case DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK:
+ *en = GPIO_GSL_GENLOCK_VSYNC;
+ return true;
+ case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK:
+ *en = GPIO_GSL_SWAPLOCK_A;
+ return true;
+ case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK:
+ *en = GPIO_GSL_SWAPLOCK_B;
+ return true;
+ default:
+ ASSERT_CRITICAL(false);
+ return false;
+ }
+ break;
+ /* DDC */
+ /* we don't care about the GPIO_ID for DDC
+ * in DdcHandle it will use GPIO_ID_DDC_DATA/GPIO_ID_DDC_CLOCK
+ * directly in the create method */
+ case REG(DC_GPIO_DDC1_A):
+ *en = GPIO_DDC_LINE_DDC1;
+ return true;
+ case REG(DC_GPIO_DDC2_A):
+ *en = GPIO_DDC_LINE_DDC2;
+ return true;
+ case REG(DC_GPIO_DDC3_A):
+ *en = GPIO_DDC_LINE_DDC3;
+ return true;
+ case REG(DC_GPIO_DDC4_A):
+ *en = GPIO_DDC_LINE_DDC4;
+ return true;
+ case REG(DC_GPIO_DDC5_A):
+ *en = GPIO_DDC_LINE_DDC5;
+ return true;
+ case REG(DC_GPIO_DDC6_A):
+ *en = GPIO_DDC_LINE_DDC6;
+ return true;
+ case REG(DC_GPIO_DDCVGA_A):
+ *en = GPIO_DDC_LINE_DDC_VGA;
+ return true;
+
+// case REG(DC_GPIO_I2CPAD_A): not exit
+// case REG(DC_GPIO_PWRSEQ_A):
+// case REG(DC_GPIO_PAD_STRENGTH_1):
+// case REG(DC_GPIO_PAD_STRENGTH_2):
+// case REG(DC_GPIO_DEBUG):
+ /* UNEXPECTED */
+ default:
+// case REG(DC_GPIO_SYNCA_A): not exist
+ ASSERT_CRITICAL(false);
+ return false;
+ }
+}
+
+static bool id_to_offset(
+ enum gpio_id id,
+ uint32_t en,
+ struct gpio_pin_info *info)
+{
+ bool result = true;
+
+ switch (id) {
+ case GPIO_ID_DDC_DATA:
+ info->mask = DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK;
+ switch (en) {
+ case GPIO_DDC_LINE_DDC1:
+ info->offset = REG(DC_GPIO_DDC1_A);
+ break;
+ case GPIO_DDC_LINE_DDC2:
+ info->offset = REG(DC_GPIO_DDC2_A);
+ break;
+ case GPIO_DDC_LINE_DDC3:
+ info->offset = REG(DC_GPIO_DDC3_A);
+ break;
+ case GPIO_DDC_LINE_DDC4:
+ info->offset = REG(DC_GPIO_DDC4_A);
+ break;
+ case GPIO_DDC_LINE_DDC5:
+ info->offset = REG(DC_GPIO_DDC5_A);
+ break;
+ case GPIO_DDC_LINE_DDC6:
+ info->offset = REG(DC_GPIO_DDC6_A);
+ break;
+ case GPIO_DDC_LINE_DDC_VGA:
+ info->offset = REG(DC_GPIO_DDCVGA_A);
+ break;
+ case GPIO_DDC_LINE_I2C_PAD:
+ default:
+ ASSERT_CRITICAL(false);
+ result = false;
+ }
+ break;
+ case GPIO_ID_DDC_CLOCK:
+ info->mask = DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK;
+ switch (en) {
+ case GPIO_DDC_LINE_DDC1:
+ info->offset = REG(DC_GPIO_DDC1_A);
+ break;
+ case GPIO_DDC_LINE_DDC2:
+ info->offset = REG(DC_GPIO_DDC2_A);
+ break;
+ case GPIO_DDC_LINE_DDC3:
+ info->offset = REG(DC_GPIO_DDC3_A);
+ break;
+ case GPIO_DDC_LINE_DDC4:
+ info->offset = REG(DC_GPIO_DDC4_A);
+ break;
+ case GPIO_DDC_LINE_DDC5:
+ info->offset = REG(DC_GPIO_DDC5_A);
+ break;
+ case GPIO_DDC_LINE_DDC6:
+ info->offset = REG(DC_GPIO_DDC6_A);
+ break;
+ case GPIO_DDC_LINE_DDC_VGA:
+ info->offset = REG(DC_GPIO_DDCVGA_A);
+ break;
+ case GPIO_DDC_LINE_I2C_PAD:
+ default:
+ ASSERT_CRITICAL(false);
+ result = false;
+ }
+ break;
+ case GPIO_ID_GENERIC:
+ info->offset = REG(DC_GPIO_GENERIC_A);
+ switch (en) {
+ case GPIO_GENERIC_A:
+ info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK;
+ break;
+ case GPIO_GENERIC_B:
+ info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK;
+ break;
+ case GPIO_GENERIC_C:
+ info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK;
+ break;
+ case GPIO_GENERIC_D:
+ info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK;
+ break;
+ case GPIO_GENERIC_E:
+ info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK;
+ break;
+ case GPIO_GENERIC_F:
+ info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK;
+ break;
+ case GPIO_GENERIC_G:
+ info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK;
+ break;
+ default:
+ ASSERT_CRITICAL(false);
+ result = false;
+ }
+ break;
+ case GPIO_ID_HPD:
+ info->offset = REG(DC_GPIO_HPD_A);
+ switch (en) {
+ case GPIO_HPD_1:
+ info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK;
+ break;
+ case GPIO_HPD_2:
+ info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK;
+ break;
+ case GPIO_HPD_3:
+ info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK;
+ break;
+ case GPIO_HPD_4:
+ info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK;
+ break;
+ case GPIO_HPD_5:
+ info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK;
+ break;
+ case GPIO_HPD_6:
+ info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK;
+ break;
+ default:
+ ASSERT_CRITICAL(false);
+ result = false;
+ }
+ break;
+ case GPIO_ID_GSL:
+ switch (en) {
+ case GPIO_GSL_GENLOCK_CLOCK:
+ /*not implmented*/
+ ASSERT_CRITICAL(false);
+ result = false;
+ break;
+ case GPIO_GSL_GENLOCK_VSYNC:
+ /*not implmented*/
+ ASSERT_CRITICAL(false);
+ result = false;
+ break;
+ case GPIO_GSL_SWAPLOCK_A:
+ /*not implmented*/
+ ASSERT_CRITICAL(false);
+ result = false;
+ break;
+ case GPIO_GSL_SWAPLOCK_B:
+ /*not implmented*/
+ ASSERT_CRITICAL(false);
+ result = false;
+
+ break;
+ default:
+ ASSERT_CRITICAL(false);
+ result = false;
+ }
+ break;
+ case GPIO_ID_SYNC:
+ case GPIO_ID_VIP_PAD:
+ default:
+ ASSERT_CRITICAL(false);
+ result = false;
+ }
+
+ if (result) {
+ info->offset_y = info->offset + 2;
+ info->offset_en = info->offset + 1;
+ info->offset_mask = info->offset - 1;
+
+ info->mask_y = info->mask;
+ info->mask_en = info->mask;
+ info->mask_mask = info->mask;
+ }
+
+ return result;
+}
+
+/* function table */
+static const struct hw_translate_funcs funcs = {
+ .offset_to_id = offset_to_id,
+ .id_to_offset = id_to_offset,
+};
+
+/*
+ * dal_hw_translate_dcn10_init
+ *
+ * @brief
+ * Initialize Hw translate function pointers.
+ *
+ * @param
+ * struct hw_translate *tr - [out] struct of function pointers
+ *
+ */
+void dal_hw_translate_dcn20_init(struct hw_translate *tr)
+{
+ tr->funcs = &funcs;
+}
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_translate_dcn20.h b/drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_translate_dcn20.h
new file mode 100644
index 000000000000..01f52c7bed86
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_translate_dcn20.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#ifndef __DAL_HW_TRANSLATE_DCN20_H__
+#define __DAL_HW_TRANSLATE_DCN20_H__
+
+struct hw_translate;
+
+/* Initialize Hw translate function pointers */
+void dal_hw_translate_dcn20_init(struct hw_translate *tr);
+
+#endif /* __DAL_HW_TRANSLATE_DCN20_H__ */
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/ddc_regs.h b/drivers/gpu/drm/amd/display/dc/gpio/ddc_regs.h
index bf40725f982f..f91e85b04956 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/ddc_regs.h
+++ b/drivers/gpu/drm/amd/display/dc/gpio/ddc_regs.h
@@ -48,6 +48,14 @@
DDC_GPIO_REG_LIST(cd,id),\
.ddc_setup = REG(DC_I2C_DDC ## id ## _SETUP)
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ #define DDC_REG_LIST_DCN2(cd, id) \
+ DDC_GPIO_REG_LIST(cd, id),\
+ .ddc_setup = REG(DC_I2C_DDC ## id ## _SETUP),\
+ .phy_aux_cntl = REG(PHY_AUX_CNTL), \
+ .dc_gpio_aux_ctrl_5 = REG(DC_GPIO_AUX_CTRL_5)
+#endif
+
#define DDC_GPIO_VGA_REG_LIST_ENTRY(type,cd)\
.type ## _reg = REG(DC_GPIO_DDCVGA_ ## type),\
.type ## _mask = DC_GPIO_DDCVGA_ ## type ## __DC_GPIO_DDCVGA ## cd ## _ ## type ## _MASK,\
@@ -82,6 +90,13 @@
DDC_GPIO_I2C_REG_LIST(cd),\
.ddc_setup = 0
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#define DDC_I2C_REG_LIST_DCN2(cd) \
+ DDC_GPIO_I2C_REG_LIST(cd),\
+ .ddc_setup = 0,\
+ .phy_aux_cntl = REG(PHY_AUX_CNTL), \
+ .dc_gpio_aux_ctrl_5 = REG(DC_GPIO_AUX_CTRL_5)
+#endif
#define DDC_MASK_SH_LIST_COMMON(mask_sh) \
SF_DDC(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_ENABLE, mask_sh),\
SF_DDC(DC_I2C_DDC1_SETUP, DC_I2C_DDC1_EDID_DETECT_ENABLE, mask_sh),\
@@ -95,10 +110,22 @@
SF_DDC(DC_GPIO_I2CPAD_MASK, DC_GPIO_SDA_PD_DIS, mask_sh),\
SF_DDC(DC_GPIO_I2CPAD_MASK, DC_GPIO_SCL_PD_DIS, mask_sh)
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#define DDC_MASK_SH_LIST_DCN2(mask_sh, cd) \
+ {DDC_MASK_SH_LIST_COMMON(mask_sh),\
+ 0,\
+ 0,\
+ (PHY_AUX_CNTL__AUX## cd ##_PAD_RXSEL## mask_sh),\
+ (DC_GPIO_AUX_CTRL_5__DDC_PAD## cd ##_I2CMODE## mask_sh)}
+#endif
struct ddc_registers {
struct gpio_registers gpio;
uint32_t ddc_setup;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ uint32_t phy_aux_cntl;
+ uint32_t dc_gpio_aux_ctrl_5;
+#endif
};
struct ddc_sh_mask {
@@ -113,6 +140,11 @@ struct ddc_sh_mask {
/* i2cpad_mask */
uint32_t DC_GPIO_SDA_PD_DIS;
uint32_t DC_GPIO_SCL_PD_DIS;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ //phy_aux_cntl
+ uint32_t AUX_PAD_RXSEL;
+ uint32_t DDC_PAD_I2CMODE;
+#endif
};
@@ -148,6 +180,27 @@ struct ddc_sh_mask {
{\
DDC_I2C_REG_LIST(SCL)\
}
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#define ddc_data_regs_dcn2(id) \
+{\
+ DDC_REG_LIST_DCN2(DATA, id)\
+}
+
+#define ddc_clk_regs_dcn2(id) \
+{\
+ DDC_REG_LIST_DCN2(CLK, id)\
+}
+
+#define ddc_i2c_data_regs_dcn2 \
+{\
+ DDC_I2C_REG_LIST_DCN2(SDA)\
+}
+
+#define ddc_i2c_clk_regs_dcn2 \
+{\
+ DDC_REG_LIST_DCN2(SCL)\
+}
+#endif
#endif /* DRIVERS_GPU_DRM_AMD_DC_DEV_DC_GPIO_DDC_REGS_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/gpio_base.c b/drivers/gpu/drm/amd/display/dc/gpio/gpio_base.c
index cf76ea2d9f5a..d03b38e80d9b 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/gpio_base.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/gpio_base.c
@@ -27,6 +27,8 @@
* Pre-requisites: headers required by header of this unit
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "include/gpio_interface.h"
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c b/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c
index 3c63a3c04dbb..a7fab44f66b6 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c
@@ -27,6 +27,8 @@
* Pre-requisites: headers required by header of this unit
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "include/gpio_interface.h"
#include "include/gpio_service_interface.h"
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c
index 310f48965b27..408857d19c84 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c
@@ -23,6 +23,9 @@
*
*/
+#include <linux/delay.h>
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "include/gpio_types.h"
@@ -144,6 +147,15 @@ static enum gpio_result set_config(
AUX_PAD1_MODE, 0);
}
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ if (ddc->regs->dc_gpio_aux_ctrl_5 != 0) {
+ REG_UPDATE(dc_gpio_aux_ctrl_5, DDC_PAD_I2CMODE, 1);
+ }
+ //set DC_IO_aux_rxsel = 2'b01
+ if (ddc->regs->phy_aux_cntl != 0) {
+ REG_UPDATE(phy_aux_cntl, AUX_PAD_RXSEL, 1);
+ }
+#endif
return GPIO_RESULT_OK;
case GPIO_DDC_CONFIG_TYPE_MODE_AUX:
/* set the AUX pad mode */
@@ -151,6 +163,12 @@ static enum gpio_result set_config(
REG_SET(gpio.MASK_reg, regval,
AUX_PAD1_MODE, 1);
}
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ if (ddc->regs->dc_gpio_aux_ctrl_5 != 0) {
+ REG_UPDATE(dc_gpio_aux_ctrl_5,
+ DDC_PAD_I2CMODE, 0);
+ }
+#endif
return GPIO_RESULT_OK;
case GPIO_DDC_CONFIG_TYPE_POLL_FOR_CONNECT:
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c
index c2028c4744a6..78f528f92907 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
/*
@@ -46,6 +48,9 @@
#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
#include "dcn10/hw_factory_dcn10.h"
#endif
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#include "dcn20/hw_factory_dcn20.h"
+#endif
#include "diagnostics/hw_factory_diag.h"
@@ -84,12 +89,14 @@ bool dal_hw_factory_init(
return true;
#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
case DCN_VERSION_1_0:
+ case DCN_VERSION_1_01:
dal_hw_factory_dcn10_init(factory);
return true;
#endif
-#if defined(CONFIG_DRM_AMD_DC_DCN1_01)
- case DCN_VERSION_1_01:
- dal_hw_factory_dcn10_init(factory);
+
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ case DCN_VERSION_2_0:
+ dal_hw_factory_dcn20_init(factory);
return true;
#endif
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_hpd.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_hpd.c
index 784feccc5853..5e11d748e6f3 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_hpd.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_hpd.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "include/gpio_types.h"
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c
index 236ca28784a9..c35fe201d335 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c
@@ -46,6 +46,9 @@
#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
#include "dcn10/hw_translate_dcn10.h"
#endif
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#include "dcn20/hw_translate_dcn20.h"
+#endif
#include "diagnostics/hw_translate_diag.h"
@@ -81,12 +84,14 @@ bool dal_hw_translate_init(
return true;
#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
case DCN_VERSION_1_0:
+ case DCN_VERSION_1_01:
dal_hw_translate_dcn10_init(translate);
return true;
#endif
-#if defined(CONFIG_DRM_AMD_DC_DCN1_01)
- case DCN_VERSION_1_01:
- dal_hw_translate_dcn10_init(translate);
+
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ case DCN_VERSION_2_0:
+ dal_hw_translate_dcn20_init(translate);
return true;
#endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_status.h b/drivers/gpu/drm/amd/display/dc/inc/core_status.h
index 8dca3b7700e5..0a094d7c9380 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/core_status.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/core_status.h
@@ -43,7 +43,12 @@ enum dc_status {
DC_FAIL_BANDWIDTH_VALIDATE = 13, /* BW and Watermark validation */
DC_FAIL_SCALING = 14,
DC_FAIL_DP_LINK_TRAINING = 15,
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ DC_FAIL_DSC_VALIDATE = 16,
+ DC_NO_DSC_RESOURCE = 17,
+#endif
DC_FAIL_UNSUPPORTED_1 = 18,
+
DC_ERROR_UNEXPECTED = -1
};
diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
index 6f5ab05d6467..a148ffde8b12 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
@@ -36,6 +36,10 @@
#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
#include "mpc.h"
#endif
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#include "dwb.h"
+#include "mcif_wb.h"
+#endif
#define MAX_CLOCK_SOURCES 7
@@ -82,7 +86,6 @@ void core_link_disable_stream(struct pipe_ctx *pipe_ctx, int option);
void core_link_set_avmute(struct pipe_ctx *pipe_ctx, bool enable);
/********** DAL Core*********************/
-#include "hw/clk_mgr.h"
#include "transform.h"
#include "dpp.h"
@@ -100,6 +103,11 @@ struct resource_funcs {
struct dc_state *context,
bool fast_validate);
+ int (*populate_dml_pipes)(
+ struct dc *dc,
+ struct resource_context *res_ctx,
+ display_e2e_pipe_params_st *pipes);
+
enum dc_status (*validate_global)(
struct dc *dc,
struct dc_state *context);
@@ -123,6 +131,23 @@ struct resource_funcs {
enum dc_status (*get_default_swizzle_mode)(
struct dc_plane_state *plane_state);
+ struct stream_encoder *(*find_first_free_match_stream_enc_for_link)(
+ struct resource_context *res_ctx,
+ const struct resource_pool *pool,
+ struct dc_stream_state *stream);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ void (*populate_dml_writeback_from_context)(
+ struct dc *dc,
+ struct resource_context *res_ctx,
+ display_e2e_pipe_params_st *pipes);
+
+ void (*set_mcif_arb_params)(
+ struct dc *dc,
+ struct dc_state *context,
+ display_e2e_pipe_params_st *pipes,
+ int pipe_cnt);
+#endif
+
};
struct audio_support{
@@ -150,6 +175,20 @@ struct resource_pool {
struct dce_i2c_sw *sw_i2cs[MAX_PIPES];
bool i2c_hw_buffer_in_use;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ struct dwbc *dwbc[MAX_DWB_PIPES];
+ struct mcif_wb *mcif_wb[MAX_DWB_PIPES];
+ struct {
+ unsigned int gsl_0:1;
+ unsigned int gsl_1:1;
+ unsigned int gsl_2:1;
+ } gsl_groups;
+#endif
+
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ struct display_stream_compressor *dscs[MAX_PIPES];
+#endif
+
unsigned int pipe_count;
unsigned int underlay_pipe_index;
unsigned int stream_enc_count;
@@ -160,7 +199,11 @@ struct resource_pool {
unsigned int dchub_ref_clock_inKhz;
} ref_clocks;
unsigned int timing_generator_count;
+ unsigned int mpcc_count;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ unsigned int writeback_pipe_count;
+#endif
/*
* reserved clock source for DP
*/
@@ -169,11 +212,10 @@ struct resource_pool {
struct clock_source *clock_sources[MAX_CLOCK_SOURCES];
unsigned int clk_src_count;
- struct audio *audios[MAX_PIPES];
+ struct audio *audios[MAX_AUDIOS];
unsigned int audio_count;
struct audio_support audio_support;
- struct clk_mgr *clk_mgr;
struct dccg *dccg;
struct irq_service *irqs;
@@ -186,10 +228,15 @@ struct resource_pool {
struct dcn_fe_bandwidth {
int dppclk_khz;
+
};
struct stream_resource {
struct output_pixel_processor *opp;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ struct display_stream_compressor *dsc;
+ int dscclk_khz;
+#endif
struct timing_generator *tg;
struct stream_encoder *stream_enc;
struct audio *audio;
@@ -198,6 +245,12 @@ struct stream_resource {
struct encoder_info_frame encoder_info_frame;
struct abm *abm;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ /* There are only (num_pipes+1)/2 groups. 0 means unassigned,
+ * otherwise it's using group number 'gsl_group-1'
+ */
+ uint8_t gsl_group;
+#endif
};
struct plane_resource {
@@ -212,6 +265,25 @@ struct plane_resource {
struct dcn_fe_bandwidth bw;
};
+union pipe_update_flags {
+ struct {
+ uint32_t enable : 1;
+ uint32_t disable : 1;
+ uint32_t odm : 1;
+ uint32_t global_sync : 1;
+ uint32_t opp_changed : 1;
+ uint32_t tg_changed : 1;
+ uint32_t mpcc : 1;
+ uint32_t dppclk : 1;
+ uint32_t hubp_interdependent : 1;
+ uint32_t hubp_rq_dlg_ttu : 1;
+ uint32_t gamut_remap : 1;
+ uint32_t scaler : 1;
+ uint32_t viewport : 1;
+ } bits;
+ uint32_t raw;
+};
+
struct pipe_ctx {
struct dc_plane_state *plane_state;
struct dc_stream_state *stream;
@@ -234,6 +306,11 @@ struct pipe_ctx {
struct _vcs_dpi_display_rq_regs_st rq_regs;
struct _vcs_dpi_display_pipe_dest_params_st pipe_dlg_param;
#endif
+ union pipe_update_flags update_flags;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ struct dwbc *dwbc;
+ struct mcif_wb *mcif_wb;
+#endif
};
struct resource_context {
@@ -242,6 +319,9 @@ struct resource_context {
bool is_audio_acquired[MAX_PIPES];
uint8_t clock_source_ref_count[MAX_CLOCK_SOURCES];
uint8_t dp_clock_source_ref_count;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ bool is_dsc_acquired[MAX_PIPES];
+#endif
};
struct dce_bw_output {
@@ -261,9 +341,18 @@ struct dce_bw_output {
int blackout_recovery_time_us;
};
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+struct dcn_bw_writeback {
+ struct mcif_arb_params mcif_wb_arb[MAX_DWB_PIPES];
+};
+#endif
+
struct dcn_bw_output {
struct dc_clocks clk;
struct dcn_watermark_set watermarks;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ struct dcn_bw_writeback bw_writeback;
+#endif
};
union bw_output {
diff --git a/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h b/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h
index a37255c757e0..2d95eff94239 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h
@@ -62,4 +62,11 @@ bool is_dp_active_dongle(const struct dc_link *link);
void dp_enable_mst_on_sink(struct dc_link *link, bool enable);
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+void dp_set_fec_ready(struct dc_link *link, bool ready);
+void dp_set_fec_enable(struct dc_link *link, bool enable);
+bool dp_set_dsc_enable(struct pipe_ctx *pipe_ctx, bool enable);
+bool dp_update_dsc_config(struct pipe_ctx *pipe_ctx);
+#endif
+
#endif /* __DC_LINK_DP_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h b/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h
index 263c09630c06..806f3041db14 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h
@@ -32,7 +32,7 @@
#include "bw_fixed.h"
#include "../dml/display_mode_lib.h"
-#include "hw/clk_mgr.h"
+
struct dc;
struct dc_state;
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h b/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h
index 86dc39a02408..d607b3191954 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h
@@ -37,7 +37,7 @@ struct abm_backlight_registers {
struct abm {
struct dc_context *ctx;
const struct abm_funcs *funcs;
-
+ bool dmcu_is_running;
/* registers setting needs to be saved and restored at InitBacklight */
struct abm_backlight_registers stored_backlight_registers;
};
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/audio.h b/drivers/gpu/drm/amd/display/dc/inc/hw/audio.h
index 925204f49717..6ed1fb8c9300 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/audio.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/audio.h
@@ -57,6 +57,7 @@ struct audio {
const struct audio_funcs *funcs;
struct dc_context *ctx;
unsigned int inst;
+ bool enabled;
};
#endif /* __DAL_AUDIO__ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
index 31bd6d5183ab..36ebd5bc7863 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
@@ -26,17 +26,22 @@
#ifndef __DAL_CLK_MGR_H__
#define __DAL_CLK_MGR_H__
-#include "dm_services_types.h"
#include "dc.h"
-struct clk_mgr {
- struct dc_context *ctx;
- const struct clk_mgr_funcs *funcs;
+/* Public interfaces */
- struct dc_clocks clks;
+struct clk_states {
+ uint32_t dprefclk_khz;
};
struct clk_mgr_funcs {
+ /*
+ * This function should set new clocks based on the input "safe_to_lower".
+ * If safe_to_lower == false, then only clocks which are to be increased
+ * should changed.
+ * If safe_to_lower == true, then only clocks which are to be decreased
+ * should be changed.
+ */
void (*update_clocks)(struct clk_mgr *clk_mgr,
struct dc_state *context,
bool safe_to_lower);
@@ -44,6 +49,22 @@ struct clk_mgr_funcs {
int (*get_dp_ref_clk_frequency)(struct clk_mgr *clk_mgr);
void (*init_clocks)(struct clk_mgr *clk_mgr);
+
+ void (*enable_pme_wa) (struct clk_mgr *clk_mgr);
+};
+
+struct clk_mgr {
+ struct dc_context *ctx;
+ struct clk_mgr_funcs *funcs;
+ struct dc_clocks clks;
+ int dprefclk_khz; // Used by program pixel clock in clock source funcs, need to figureout where this goes
};
+/* forward declarations */
+struct dccg;
+
+struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *pp_smu, struct dccg *dccg);
+
+void dc_destroy_clk_mgr(struct clk_mgr *clk_mgr);
+
#endif /* __DAL_CLK_MGR_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h
index c8f8c442142a..0835ac041acf 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h
@@ -1,5 +1,5 @@
/*
- * Copyright 2012-16 Advanced Micro Devices, Inc.
+ * Copyright 2018 Advanced Micro Devices, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -23,14 +23,64 @@
*
*/
-
-#ifndef _DCE_CLK_MGR_H_
-#define _DCE_CLK_MGR_H_
+#ifndef __DAL_CLK_MGR_INTERNAL_H__
+#define __DAL_CLK_MGR_INTERNAL_H__
#include "clk_mgr.h"
-#include "dccg.h"
+#include "dc.h"
+
+/*
+ * only thing needed from here is MEMORY_TYPE_MULTIPLIER_CZ, which is also
+ * used in resource, perhaps this should be defined somewhere more common.
+ */
+#include "resource.h"
+
-#define MEMORY_TYPE_MULTIPLIER_CZ 4
+/* Starting DID for each range */
+enum dentist_base_divider_id {
+ DENTIST_BASE_DID_1 = 0x08,
+ DENTIST_BASE_DID_2 = 0x40,
+ DENTIST_BASE_DID_3 = 0x60,
+ DENTIST_BASE_DID_4 = 0x7e,
+ DENTIST_MAX_DID = 0x7f
+};
+
+/* Starting point and step size for each divider range.*/
+enum dentist_divider_range {
+ DENTIST_DIVIDER_RANGE_1_START = 8, /* 2.00 */
+ DENTIST_DIVIDER_RANGE_1_STEP = 1, /* 0.25 */
+ DENTIST_DIVIDER_RANGE_2_START = 64, /* 16.00 */
+ DENTIST_DIVIDER_RANGE_2_STEP = 2, /* 0.50 */
+ DENTIST_DIVIDER_RANGE_3_START = 128, /* 32.00 */
+ DENTIST_DIVIDER_RANGE_3_STEP = 4, /* 1.00 */
+ DENTIST_DIVIDER_RANGE_4_START = 248, /* 62.00 */
+ DENTIST_DIVIDER_RANGE_4_STEP = 264, /* 66.00 */
+ DENTIST_DIVIDER_RANGE_SCALE_FACTOR = 4
+};
+
+/*
+ ***************************************************************************************
+ ****************** Clock Manager Private Macros and Defines ***************************
+ ***************************************************************************************
+ */
+
+#define TO_CLK_MGR_INTERNAL(clk_mgr)\
+ container_of(clk_mgr, struct clk_mgr_internal, base)
+
+#define CTX \
+ clk_mgr->base.ctx
+#define DC_LOGGER \
+ clk_mgr->ctx->logger
+
+
+
+
+#define CLK_BASE(inst) \
+ CLK_BASE_INNER(inst)
+
+#define CLK_SRI(reg_name, block, inst)\
+ .reg_name = CLK_BASE(mm ## block ## _ ## inst ## _ ## reg_name ## _BASE_IDX) + \
+ mm ## block ## _ ## inst ## _ ## reg_name
#define CLK_COMMON_REG_LIST_DCE_BASE() \
.DPREFCLK_CNTL = mmDPREFCLK_CNTL, \
@@ -39,6 +89,18 @@
#define CLK_COMMON_REG_LIST_DCN_BASE() \
SR(DENTIST_DISPCLK_CNTL)
+#define VBIOS_SMU_MSG_BOX_REG_LIST_RV() \
+ .MP1_SMN_C2PMSG_91 = mmMP1_SMN_C2PMSG_91, \
+ .MP1_SMN_C2PMSG_83 = mmMP1_SMN_C2PMSG_83, \
+ .MP1_SMN_C2PMSG_67 = mmMP1_SMN_C2PMSG_67
+
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+#define CLK_REG_LIST_NV10() \
+ SR(DENTIST_DISPCLK_CNTL), \
+ CLK_SRI(CLK3_CLK_PLL_REQ, CLK3, 0), \
+ CLK_SRI(CLK3_CLK2_DFS_CNTL, CLK3, 0)
+#endif
+
#define CLK_SF(reg_name, field_name, post_fix)\
.field_name = reg_name ## __ ## field_name ## post_fix
@@ -50,23 +112,74 @@
CLK_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, mask_sh),\
CLK_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, mask_sh)
+#define CLK_MASK_SH_LIST_RV1(mask_sh) \
+ CLK_COMMON_MASK_SH_LIST_DCN_COMMON_BASE(mask_sh),\
+ CLK_SF(MP1_SMN_C2PMSG_67, CONTENT, mask_sh),\
+ CLK_SF(MP1_SMN_C2PMSG_83, CONTENT, mask_sh),\
+ CLK_SF(MP1_SMN_C2PMSG_91, CONTENT, mask_sh),
+
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+#define CLK_COMMON_MASK_SH_LIST_DCN20_BASE(mask_sh) \
+ CLK_COMMON_MASK_SH_LIST_DCN_COMMON_BASE(mask_sh),\
+ CLK_SF(DENTIST_DISPCLK_CNTL, DENTIST_DPPCLK_WDIVIDER, mask_sh),\
+ CLK_SF(DENTIST_DISPCLK_CNTL, DENTIST_DPPCLK_CHG_DONE, mask_sh)
+
+#define CLK_MASK_SH_LIST_NV10(mask_sh) \
+ CLK_COMMON_MASK_SH_LIST_DCN20_BASE(mask_sh),\
+ CLK_SF(CLK3_0_CLK3_CLK_PLL_REQ, FbMult_int, mask_sh),\
+ CLK_SF(CLK3_0_CLK3_CLK_PLL_REQ, FbMult_frac, mask_sh)
+#endif
+
#define CLK_REG_FIELD_LIST(type) \
type DPREFCLK_SRC_SEL; \
type DENTIST_DPREFCLK_WDIVIDER; \
type DENTIST_DISPCLK_WDIVIDER; \
type DENTIST_DISPCLK_CHG_DONE;
+/*
+ ***************************************************************************************
+ ****************** Clock Manager Private Structures ***********************************
+ ***************************************************************************************
+ */
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+#define CLK20_REG_FIELD_LIST(type) \
+ type DENTIST_DPPCLK_WDIVIDER; \
+ type DENTIST_DPPCLK_CHG_DONE; \
+ type FbMult_int; \
+ type FbMult_frac;
+#endif
+
+#define VBIOS_SMU_REG_FIELD_LIST(type) \
+ type CONTENT;
+
struct clk_mgr_shift {
CLK_REG_FIELD_LIST(uint8_t)
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ CLK20_REG_FIELD_LIST(uint8_t)
+#endif
+ VBIOS_SMU_REG_FIELD_LIST(uint32_t)
};
struct clk_mgr_mask {
CLK_REG_FIELD_LIST(uint32_t)
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ CLK20_REG_FIELD_LIST(uint32_t)
+#endif
+ VBIOS_SMU_REG_FIELD_LIST(uint32_t)
};
struct clk_mgr_registers {
uint32_t DPREFCLK_CNTL;
uint32_t DENTIST_DISPCLK_CNTL;
+
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ uint32_t CLK3_CLK2_DFS_CNTL;
+ uint32_t CLK3_CLK_PLL_REQ;
+#endif
+
+ uint32_t MP1_SMN_C2PMSG_67;
+ uint32_t MP1_SMN_C2PMSG_83;
+ uint32_t MP1_SMN_C2PMSG_91;
};
struct state_dependent_clocks {
@@ -74,24 +187,36 @@ struct state_dependent_clocks {
int pixel_clk_khz;
};
-struct dce_clk_mgr {
+struct clk_mgr_internal {
struct clk_mgr base;
+ struct pp_smu_funcs *pp_smu;
+ struct clk_mgr_internal_funcs *funcs;
+
+ struct dccg *dccg;
+
+ /*
+ * For backwards compatbility with previous implementation
+ * TODO: remove these after everything transitions to new pattern
+ * Rationale is that clk registers change a lot across DCE versions
+ * and a shared data structure doesn't really make sense.
+ */
const struct clk_mgr_registers *regs;
const struct clk_mgr_shift *clk_mgr_shift;
const struct clk_mgr_mask *clk_mgr_mask;
- struct dccg *dccg;
-
struct state_dependent_clocks max_clks_by_state[DM_PP_CLOCKS_MAX_STATES];
+ /*TODO: figure out which of the below fields should be here vs in asic specific portion */
int dentist_vco_freq_khz;
/* Cache the status of DFS-bypass feature*/
bool dfs_bypass_enabled;
/* True if the DFS-bypass feature is enabled and active. */
bool dfs_bypass_active;
- /* Cache the display clock returned by VBIOS if DFS-bypass is enabled.
- * This is basically "Crystal Frequency In KHz" (XTALIN) frequency */
+ /*
+ * Cache the display clock returned by VBIOS if DFS-bypass is enabled.
+ * This is basically "Crystal Frequency In KHz" (XTALIN) frequency
+ */
int dfs_bypass_disp_clk;
/**
@@ -126,74 +251,39 @@ struct dce_clk_mgr {
* DPREFCLK SS percentage Divider (100 or 1000).
*/
int dprefclk_ss_divider;
- int dprefclk_khz;
enum dm_pp_clocks_state max_clks_state;
enum dm_pp_clocks_state cur_min_clks_state;
};
-/* Starting DID for each range */
-enum dentist_base_divider_id {
- DENTIST_BASE_DID_1 = 0x08,
- DENTIST_BASE_DID_2 = 0x40,
- DENTIST_BASE_DID_3 = 0x60,
- DENTIST_BASE_DID_4 = 0x7e,
- DENTIST_MAX_DID = 0x7f
+struct clk_mgr_internal_funcs {
+ int (*set_dispclk)(struct clk_mgr_internal *clk_mgr, int requested_dispclk_khz);
+ int (*set_dprefclk)(struct clk_mgr_internal *clk_mgr);
};
-/* Starting point and step size for each divider range.*/
-enum dentist_divider_range {
- DENTIST_DIVIDER_RANGE_1_START = 8, /* 2.00 */
- DENTIST_DIVIDER_RANGE_1_STEP = 1, /* 0.25 */
- DENTIST_DIVIDER_RANGE_2_START = 64, /* 16.00 */
- DENTIST_DIVIDER_RANGE_2_STEP = 2, /* 0.50 */
- DENTIST_DIVIDER_RANGE_3_START = 128, /* 32.00 */
- DENTIST_DIVIDER_RANGE_3_STEP = 4, /* 1.00 */
- DENTIST_DIVIDER_RANGE_4_START = 248, /* 62.00 */
- DENTIST_DIVIDER_RANGE_4_STEP = 264, /* 66.00 */
- DENTIST_DIVIDER_RANGE_SCALE_FACTOR = 4
-};
+
+/*
+ ***************************************************************************************
+ ****************** Clock Manager Level Helper functions *******************************
+ ***************************************************************************************
+ */
+
static inline bool should_set_clock(bool safe_to_lower, int calc_clk, int cur_clk)
{
return ((safe_to_lower && calc_clk < cur_clk) || calc_clk > cur_clk);
}
-void dce_clock_read_ss_info(struct dce_clk_mgr *dccg_dce);
-
-int dce12_get_dp_ref_freq_khz(struct clk_mgr *dccg);
-
-void dce110_fill_display_configs(
- const struct dc_state *context,
- struct dm_pp_display_configuration *pp_display_cfg);
-
-int dce112_set_clock(struct clk_mgr *dccg, int requested_clk_khz);
-
-struct clk_mgr *dce_clk_mgr_create(
- struct dc_context *ctx,
- const struct clk_mgr_registers *regs,
- const struct clk_mgr_shift *clk_shift,
- const struct clk_mgr_mask *clk_mask);
-
-struct clk_mgr *dce110_clk_mgr_create(
- struct dc_context *ctx,
- const struct clk_mgr_registers *regs,
- const struct clk_mgr_shift *clk_shift,
- const struct clk_mgr_mask *clk_mask);
-
-struct clk_mgr *dce112_clk_mgr_create(
- struct dc_context *ctx,
- const struct clk_mgr_registers *regs,
- const struct clk_mgr_shift *clk_shift,
- const struct clk_mgr_mask *clk_mask);
-
-struct clk_mgr *dce120_clk_mgr_create(struct dc_context *ctx);
+static inline bool should_update_pstate_support(bool safe_to_lower, bool calc_support, bool cur_support)
+{
+ // Whenever we are transitioning pstate support, we always want to notify prior to committing state
+ return (calc_support != cur_support) ? !safe_to_lower : false;
+}
-struct clk_mgr *dce121_clk_mgr_create(struct dc_context *ctx);
-void dce121_clock_patch_xgmi_ss_info(struct clk_mgr *clk_mgr);
+int clk_mgr_helper_get_active_display_cnt(
+ struct dc *dc,
+ struct dc_state *context);
-void dce_clk_mgr_destroy(struct clk_mgr **clk_mgr);
-int dentist_get_divider_from_did(int did);
-#endif /* _DCE_CLK_MGR_H_ */
+#endif //__DAL_CLK_MGR_INTERNAL_H__
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
index 93667e8b23b3..9502478c4a1b 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
@@ -52,11 +52,69 @@ struct dcn_hubbub_wm {
struct dcn_hubbub_wm_set sets[4];
};
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+enum dcn_hubbub_page_table_depth {
+ DCN_PAGE_TABLE_DEPTH_1_LEVEL,
+ DCN_PAGE_TABLE_DEPTH_2_LEVEL,
+ DCN_PAGE_TABLE_DEPTH_3_LEVEL,
+ DCN_PAGE_TABLE_DEPTH_4_LEVEL
+};
+
+enum dcn_hubbub_page_table_block_size {
+ DCN_PAGE_TABLE_BLOCK_SIZE_4KB = 0,
+ DCN_PAGE_TABLE_BLOCK_SIZE_64KB = 4,
+};
+
+struct dcn_hubbub_phys_addr_config {
+ struct {
+ uint64_t fb_top;
+ uint64_t fb_offset;
+ uint64_t fb_base;
+ uint64_t agp_top;
+ uint64_t agp_bot;
+ uint64_t agp_base;
+ } system_aperture;
+
+ struct {
+ uint64_t page_table_start_addr;
+ uint64_t page_table_end_addr;
+ uint64_t page_table_base_addr;
+ } gart_config;
+};
+
+struct dcn_hubbub_virt_addr_config {
+ uint64_t page_table_start_addr;
+ uint64_t page_table_end_addr;
+ enum dcn_hubbub_page_table_block_size page_table_block_size;
+ enum dcn_hubbub_page_table_depth page_table_depth;
+ uint64_t page_table_base_addr;
+};
+
+struct hubbub_addr_config {
+ struct dcn_hubbub_phys_addr_config pa_config;
+ struct dcn_hubbub_virt_addr_config va_config;
+ struct {
+ uint64_t aperture_check_fault;
+ uint64_t generic_fault;
+ } default_addrs;
+};
+
+#endif
struct hubbub_funcs {
void (*update_dchub)(
struct hubbub *hubbub,
struct dchub_init_data *dh_data);
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ int (*init_dchub_sys_ctx)(
+ struct hubbub *hubbub,
+ struct dcn_hubbub_phys_addr_config *pa_config);
+ void (*init_vm_ctx)(
+ struct hubbub *hubbub,
+ struct dcn_hubbub_virt_addr_config *va_config,
+ int vmid);
+
+#endif
bool (*get_dcc_compression_cap)(struct hubbub *hubbub,
const struct dc_dcc_surface_param *input,
struct dc_surface_dcc_cap *output);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
index fb7967b39edb..60c671fcf186 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
@@ -36,7 +36,13 @@ struct dpp {
struct dpp_caps *caps;
struct pwl_params regamma_params;
struct pwl_params degamma_params;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ struct dpp_cursor_attributes cur_attr;
+#endif
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ struct pwl_params shaper_params;
+#endif
};
struct dpp_input_csc_matrix {
@@ -49,6 +55,34 @@ struct dpp_grph_csc_adjustment {
enum graphics_gamut_adjust_type gamut_adjust_type;
};
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+struct cnv_color_keyer_params {
+ int color_keyer_en;
+ int color_keyer_mode;
+ int color_keyer_alpha_low;
+ int color_keyer_alpha_high;
+ int color_keyer_red_low;
+ int color_keyer_red_high;
+ int color_keyer_green_low;
+ int color_keyer_green_high;
+ int color_keyer_blue_low;
+ int color_keyer_blue_high;
+};
+
+/* new for dcn2: set the 8bit alpha values based on the 2 bit alpha
+ *ALPHA_2BIT_LUT. ALPHA_2BIT_LUT0 default: 0b00000000
+ *ALPHA_2BIT_LUT. ALPHA_2BIT_LUT1 default: 0b01010101
+ *ALPHA_2BIT_LUT. ALPHA_2BIT_LUT2 default: 0b10101010
+ *ALPHA_2BIT_LUT. ALPHA_2BIT_LUT3 default: 0b11111111
+ */
+struct cnv_alpha_2bit_lut {
+ int lut0;
+ int lut1;
+ int lut2;
+ int lut3;
+};
+#endif
+
struct dcn_dpp_state {
uint32_t is_enabled;
uint32_t igam_lut_mode;
@@ -64,7 +98,22 @@ struct dcn_dpp_state {
uint32_t gamut_remap_c33_c34;
};
+struct CM_bias_params {
+ uint32_t cm_bias_cr_r;
+ uint32_t cm_bias_y_g;
+ uint32_t cm_bias_cb_b;
+ uint32_t cm_bias_format;
+};
+
struct dpp_funcs {
+
+ void (*dpp_program_cm_dealpha)(struct dpp *dpp_base,
+ uint32_t enable, uint32_t additive_blending);
+
+ void (*dpp_program_cm_bias)(
+ struct dpp *dpp_base,
+ struct CM_bias_params *bias_params);
+
void (*dpp_read_state)(struct dpp *dpp, struct dcn_dpp_state *s);
void (*dpp_reset)(struct dpp *dpp);
@@ -140,7 +189,12 @@ struct dpp_funcs {
enum surface_pixel_format format,
enum expansion_mode mode,
struct dc_csc_transform input_csc_color_matrix,
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ enum dc_color_space input_color_space,
+ struct cnv_alpha_2bit_lut *alpha_2bit_lut);
+#else
enum dc_color_space input_color_space);
+#endif
void (*dpp_full_bypass)(struct dpp *dpp_base);
@@ -155,9 +209,11 @@ struct dpp_funcs {
uint32_t width,
uint32_t height
);
+
void (*dpp_set_hdr_multiplier)(
struct dpp *dpp_base,
uint32_t multiplier);
+
void (*set_optional_cursor_attributes)(
struct dpp *dpp_base,
struct dpp_cursor_attributes *attr);
@@ -167,6 +223,20 @@ struct dpp_funcs {
bool dppclk_div,
bool enable);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ bool (*dpp_program_blnd_lut)(
+ struct dpp *dpp,
+ const struct pwl_params *params);
+ bool (*dpp_program_shaper_lut)(
+ struct dpp *dpp,
+ const struct pwl_params *params);
+ bool (*dpp_program_3dlut)(
+ struct dpp *dpp,
+ struct tetrahedral_params *params);
+ void (*dpp_cnv_set_alpha_keyer)(
+ struct dpp *dpp_base,
+ struct cnv_color_keyer_params *color_keyer);
+#endif
};
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dsc.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dsc.h
new file mode 100644
index 000000000000..c905d020b59e
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dsc.h
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+#ifndef __DAL_DSC_H__
+#define __DAL_DSC_H__
+
+#include "dc_dsc.h"
+#include "dc_hw_types.h"
+#include "dc_dp_types.h"
+
+/* Input parameters for configuring DSC from the outside of DSC */
+struct dsc_config {
+ uint32_t pic_width;
+ uint32_t pic_height;
+ enum dc_pixel_encoding pixel_encoding;
+ enum dc_color_depth color_depth; /* Bits per component */
+ struct dc_dsc_config dc_dsc_cfg;
+};
+
+
+/* Output parameters for configuring DSC-related part of OPTC */
+struct dsc_optc_config {
+ uint32_t slice_width; /* Slice width in pixels */
+ uint32_t bytes_per_pixel; /* Bytes per pixel in u3.28 format */
+ bool is_pixel_format_444; /* 'true' if pixel format is 'RGB 444' or 'Simple YCbCr 4:2:2' (4:2:2 upsampled to 4:4:4)' */
+};
+
+
+struct dcn_dsc_state {
+ uint32_t dsc_clock_en;
+ uint32_t dsc_slice_width;
+ uint32_t dsc_bytes_per_pixel;
+};
+
+
+/* DSC encoder capabilities
+ * They differ from the DPCD DSC caps because they are based on AMD DSC encoder caps.
+ */
+union dsc_enc_slice_caps {
+ struct {
+ uint8_t NUM_SLICES_1 : 1;
+ uint8_t NUM_SLICES_2 : 1;
+ uint8_t NUM_SLICES_3 : 1; /* This one is not per DSC spec, but our encoder supports it */
+ uint8_t NUM_SLICES_4 : 1;
+ uint8_t NUM_SLICES_8 : 1;
+ } bits;
+ uint8_t raw;
+};
+
+struct dsc_enc_caps {
+ uint8_t dsc_version;
+ union dsc_enc_slice_caps slice_caps;
+ int32_t lb_bit_depth;
+ bool is_block_pred_supported;
+ union dsc_color_formats color_formats;
+ union dsc_color_depth color_depth;
+ int32_t max_total_throughput_mps; /* Maximum total throughput with all the slices combined */
+ int32_t max_slice_width;
+ uint32_t bpp_increment_div; /* bpp increment divisor, e.g. if 16, it's 1/16th of a bit */
+};
+
+struct display_stream_compressor {
+ const struct dsc_funcs *funcs;
+ struct dc_context *ctx;
+ int inst;
+};
+
+struct dsc_funcs {
+ void (*dsc_get_enc_caps)(struct dsc_enc_caps *dsc_enc_caps, int pixel_clock_100Hz);
+ void (*dsc_read_state)(struct display_stream_compressor *dsc, struct dcn_dsc_state *s);
+ bool (*dsc_validate_stream)(struct display_stream_compressor *dsc, const struct dsc_config *dsc_cfg);
+ void (*dsc_set_config)(struct display_stream_compressor *dsc, const struct dsc_config *dsc_cfg,
+ struct dsc_optc_config *dsc_optc_cfg, uint8_t *dsc_packed_pps);
+ void (*dsc_enable)(struct display_stream_compressor *dsc, int opp_pipe);
+ void (*dsc_disable)(struct display_stream_compressor *dsc);
+};
+
+#endif
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dwb.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dwb.h
new file mode 100644
index 000000000000..a3409294ae0c
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dwb.h
@@ -0,0 +1,180 @@
+/* Copyright 2012-17 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_DWBC_H__
+#define __DC_DWBC_H__
+
+#include "dc_hw_types.h"
+
+
+#define DWB_SW_V2 1
+#define DWB_MCIF_BUF_COUNT 4
+
+/* forward declaration of mcif_wb struct */
+struct mcif_wb;
+
+enum dce_version;
+
+enum dwb_sw_version {
+ dwb_ver_1_0 = 1,
+ dwb_ver_2_0 = 2,
+};
+
+enum dwb_source {
+ dwb_src_scl = 0, /* for DCE7x/9x, DCN won't support. */
+ dwb_src_blnd, /* for DCE7x/9x */
+ dwb_src_fmt, /* for DCE7x/9x */
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ dwb_src_otg0 = 0x100, /* for DCN1.x/DCN2.x, register: mmDWB_SOURCE_SELECT */
+ dwb_src_otg1, /* for DCN1.x/DCN2.x */
+ dwb_src_otg2, /* for DCN1.x/DCN2.x */
+ dwb_src_otg3, /* for DCN1.x/DCN2.x */
+#else
+ dwb_src_otg0 = 0x100, /* for DCN1.x, register: mmDWB_SOURCE_SELECT */
+ dwb_src_otg1, /* for DCN1.x */
+ dwb_src_otg2, /* for DCN1.x */
+ dwb_src_otg3, /* for DCN1.x */
+#endif
+ dwb_src_mpc0 = 0x200, /* for DCN2, register: mmMPC_DWB0_MUX, mmMPC_DWB1_MUX, mmMPC_DWB2_MUX */
+ dwb_src_mpc1, /* for DCN2 */
+ dwb_src_mpc2, /* for DCN2 */
+ dwb_src_mpc3, /* for DCN2 */
+ dwb_src_mpc4, /* for DCN2 */
+};
+
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+/* DCN1.x, DCN2.x support 2 pipes */
+#else
+/* DCN1.x supports 2 pipes */
+#endif
+enum dwb_pipe {
+ dwb_pipe0 = 0,
+#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
+ dwb_pipe1,
+#endif
+ dwb_pipe_max_num,
+};
+
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+enum dwb_frame_capture_enable {
+ DWB_FRAME_CAPTURE_DISABLE = 0,
+ DWB_FRAME_CAPTURE_ENABLE = 1,
+};
+
+enum wbscl_coef_filter_type_sel {
+ WBSCL_COEF_LUMA_VERT_FILTER = 0,
+ WBSCL_COEF_CHROMA_VERT_FILTER = 1,
+ WBSCL_COEF_LUMA_HORZ_FILTER = 2,
+ WBSCL_COEF_CHROMA_HORZ_FILTER = 3
+};
+
+#endif
+
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+struct dwb_warmup_params {
+ bool warmup_en; /* false: normal mode, true: enable pattern generator */
+ bool warmup_mode; /* false: 420, true: 444 */
+ bool warmup_depth; /* false: 8bit, true: 10bit */
+ int warmup_data; /* Data to be sent by pattern generator (same for each pixel component) */
+ int warmup_width; /* Pattern width (pixels) */
+ int warmup_height; /* Pattern height (lines) */
+};
+#endif
+
+struct dwb_caps {
+ enum dce_version hw_version; /* DCN engine version. */
+ enum dwb_sw_version sw_version; /* DWB sw implementation version. */
+ unsigned int reserved[6]; /* Reserved for future use, MUST BE 0. */
+ unsigned int adapter_id;
+ unsigned int num_pipes; /* number of DWB pipes */
+ struct {
+ unsigned int support_dwb :1;
+ unsigned int support_ogam :1;
+ unsigned int support_wbscl :1;
+ unsigned int support_ocsc :1;
+ unsigned int support_stereo :1;
+ } caps;
+ unsigned int reserved2[9]; /* Reserved for future use, MUST BE 0. */
+};
+
+struct dwbc {
+ const struct dwbc_funcs *funcs;
+ struct dc_context *ctx;
+ int inst;
+ struct mcif_wb *mcif;
+ bool status;
+ int inputSrcSelect;
+ bool dwb_output_black;
+ enum dc_transfer_func_predefined tf;
+ enum dc_color_space output_color_space;
+ bool dwb_is_efc_transition;
+ bool dwb_is_drc;
+ int wb_src_plane_inst;/*hubp, mpcc, inst*/
+ bool update_privacymask;
+ uint32_t mask_id;
+
+};
+
+struct dwbc_funcs {
+ bool (*get_caps)(
+ struct dwbc *dwbc,
+ struct dwb_caps *caps);
+
+ bool (*enable)(
+ struct dwbc *dwbc,
+ struct dc_dwb_params *params);
+
+ bool (*disable)(struct dwbc *dwbc);
+
+ bool (*update)(
+ struct dwbc *dwbc,
+ struct dc_dwb_params *params);
+
+ bool (*is_enabled)(
+ struct dwbc *dwbc);
+
+ void (*set_stereo)(
+ struct dwbc *dwbc,
+ struct dwb_stereo_params *stereo_params);
+
+ void (*set_new_content)(
+ struct dwbc *dwbc,
+ bool is_new_content);
+
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+
+ void (*set_warmup)(
+ struct dwbc *dwbc,
+ struct dwb_warmup_params *warmup_params);
+
+#endif
+
+ bool (*get_dwb_status)(
+ struct dwbc *dwbc);
+ void (*dwb_set_scaler)(
+ struct dwbc *dwbc,
+ struct dc_dwb_params *params);
+};
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
index 455df4999797..51bff8717cc9 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
@@ -28,6 +28,8 @@
#include "mem_input.h"
+#define OPP_ID_INVALID 0xf
+
enum cursor_pitch {
CURSOR_PITCH_64_PIXELS = 0,
@@ -36,6 +38,9 @@ enum cursor_pitch {
};
enum cursor_lines_per_chunk {
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ CURSOR_LINE_PER_CHUNK_1 = 0, /* new for DCN2 */
+#endif
CURSOR_LINE_PER_CHUNK_2 = 1,
CURSOR_LINE_PER_CHUNK_4,
CURSOR_LINE_PER_CHUNK_8,
@@ -78,8 +83,7 @@ struct hubp_funcs {
bool (*hubp_program_surface_flip_and_addr)(
struct hubp *hubp,
const struct dc_plane_address *address,
- bool flip_immediate,
- uint8_t vmid);
+ bool flip_immediate);
void (*hubp_program_pte_vm)(
struct hubp *hubp,
@@ -132,6 +136,28 @@ struct hubp_funcs {
unsigned int (*hubp_get_underflow_status)(struct hubp *hubp);
void (*hubp_init)(struct hubp *hubp);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ void (*dmdata_set_attributes)(
+ struct hubp *hubp,
+ const struct dc_dmdata_attributes *attr);
+
+ void (*dmdata_load)(
+ struct hubp *hubp,
+ uint32_t dmdata_sw_size,
+ const uint32_t *dmdata_sw_data);
+ bool (*dmdata_status_done)(struct hubp *hubp);
+ void (*hubp_enable_tripleBuffer)(
+ struct hubp *hubp,
+ bool enable);
+
+ bool (*hubp_is_triplebuffer_enabled)(
+ struct hubp *hubp);
+
+ void (*hubp_set_flip_control_surface_gsl)(
+ struct hubp *hubp,
+ bool enable);
+#endif
+
};
#endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
index 4c8e2c6fb6db..f82365e2d03c 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
@@ -34,7 +34,11 @@
* Data types shared between different Virtual HW blocks
******************************************************************************/
+#define MAX_AUDIOS 7
#define MAX_PIPES 6
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#define MAX_DWB_PIPES 1
+#endif
struct gamma_curve {
uint32_t offset;
@@ -77,6 +81,37 @@ struct pwl_result_data {
uint32_t delta_blue_reg;
};
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+struct dc_rgb {
+ uint32_t red;
+ uint32_t green;
+ uint32_t blue;
+};
+
+struct tetrahedral_17x17x17 {
+ struct dc_rgb lut0[1229];
+ struct dc_rgb lut1[1228];
+ struct dc_rgb lut2[1228];
+ struct dc_rgb lut3[1228];
+};
+struct tetrahedral_9x9x9 {
+ struct dc_rgb lut0[183];
+ struct dc_rgb lut1[182];
+ struct dc_rgb lut2[182];
+ struct dc_rgb lut3[182];
+};
+
+struct tetrahedral_params {
+ union {
+ struct tetrahedral_17x17x17 tetrahedral_17;
+ struct tetrahedral_9x9x9 tetrahedral_9;
+ };
+ bool use_tetrahedral_9;
+ bool use_12bits;
+
+};
+#endif
+
/* arr_curve_points - regamma regions/segments specification
* arr_points - beginning and end point specified separately (only one on DCE)
* corner_points - beginning and end point for all 3 colors (DCN)
@@ -160,6 +195,14 @@ enum opp_regamma {
OPP_REGAMMA_USER
};
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+enum optc_dsc_mode {
+ OPTC_DSC_DISABLED = 0,
+ OPTC_DSC_ENABLED_444 = 1, /* 'RGB 444' or 'Simple YCbCr 4:2:2' (4:2:2 upsampled to 4:4:4) */
+ OPTC_DSC_ENABLED_NATIVE_SUBSAMPLED = 2 /* Native 4:2:2 or 4:2:0 */
+};
+#endif
+
struct dc_bias_and_scale {
uint16_t scale_red;
uint16_t bias_red;
@@ -181,7 +224,12 @@ enum test_pattern_mode {
TEST_PATTERN_MODE_VERTICALBARS,
TEST_PATTERN_MODE_HORIZONTALBARS,
TEST_PATTERN_MODE_SINGLERAMP_RGB,
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ TEST_PATTERN_MODE_DUALRAMP_RGB,
+ TEST_PATTERN_MODE_XR_BIAS_RGB
+#else
TEST_PATTERN_MODE_DUALRAMP_RGB
+#endif
};
enum test_pattern_color_format {
@@ -203,7 +251,8 @@ enum controller_dp_test_pattern {
CONTROLLER_DP_TEST_PATTERN_RESERVED_8,
CONTROLLER_DP_TEST_PATTERN_RESERVED_9,
CONTROLLER_DP_TEST_PATTERN_RESERVED_A,
- CONTROLLER_DP_TEST_PATTERN_COLORSQUARES_CEA
+ CONTROLLER_DP_TEST_PATTERN_COLORSQUARES_CEA,
+ CONTROLLER_DP_TEST_PATTERN_SOLID_COLOR
};
enum dc_lut_mode {
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
index c9d3e37e9531..e5e8640a9ef3 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
@@ -59,6 +59,7 @@ struct encoder_feature_support {
uint32_t IS_TPS3_CAPABLE:1;
uint32_t IS_TPS4_CAPABLE:1;
uint32_t HDMI_6GB_EN:1;
+ uint32_t DP_IS_USB_C:1;
} bits;
uint32_t raw;
} flags;
@@ -112,9 +113,26 @@ struct link_encoder {
struct encoder_feature_support features;
enum transmitter transmitter;
enum hpd_source_id hpd_source;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ bool usbc_combo_phy;
+#endif
};
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+struct link_enc_state {
+
+ uint32_t dphy_fec_en;
+ uint32_t dphy_fec_ready_shadow;
+ uint32_t dphy_fec_active_status;
+
+};
+#endif
+
struct link_encoder_funcs {
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ void (*read_state)(
+ struct link_encoder *enc, struct link_enc_state *s);
+#endif
bool (*validate_output_with_stream)(
struct link_encoder *enc, const struct dc_stream_state *stream);
void (*hw_init)(struct link_encoder *enc);
@@ -155,6 +173,16 @@ struct link_encoder_funcs {
bool (*is_dig_enabled)(struct link_encoder *enc);
unsigned int (*get_dig_frontend)(struct link_encoder *enc);
void (*destroy)(struct link_encoder **enc);
+
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ void (*fec_set_enable)(struct link_encoder *enc,
+ bool enable);
+
+ void (*fec_set_ready)(struct link_encoder *enc,
+ bool ready);
+
+ bool (*fec_is_active)(struct link_encoder *enc);
+#endif
};
#endif /* LINK_ENCODER_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/mcif_wb.h b/drivers/gpu/drm/amd/display/dc/inc/hw/mcif_wb.h
new file mode 100644
index 000000000000..a5c8d92fc5c2
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/mcif_wb.h
@@ -0,0 +1,105 @@
+/* Copyright 2012-17 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_MCIF_WB_H__
+#define __DC_MCIF_WB_H__
+
+#include "dc_hw_types.h"
+
+
+enum mmhubbub_wbif_mode {
+ PACKED_444 = 0,
+ PACKED_444_FP16 = 1,
+ PLANAR_420_8BPC = 2,
+ PLANAR_420_10BPC = 3
+};
+
+struct mcif_arb_params {
+
+ unsigned int time_per_pixel;
+ unsigned int cli_watermark[4];
+ unsigned int pstate_watermark[4];
+ unsigned int arbitration_slice;
+ unsigned int slice_lines;
+ unsigned int max_scaled_time;
+};
+
+struct mcif_irq_params {
+ unsigned int sw_int_en;
+ unsigned int sw_slice_int_en;
+ unsigned int sw_overrun_int_en;
+ unsigned int vce_int_en;
+ unsigned int vce_slice_int_en;
+};
+
+
+/* / - mcif_wb_frame_dump_info is the info of the dumping WB data */
+struct mcif_wb_frame_dump_info {
+ unsigned int size;
+ unsigned int width;
+ unsigned int height;
+ unsigned int luma_pitch;
+ unsigned int chroma_pitch;
+ enum dwb_scaler_mode format;
+};
+
+struct mcif_wb {
+ const struct mcif_wb_funcs *funcs;
+ struct dc_context *ctx;
+ int inst;
+};
+
+struct mcif_wb_funcs {
+
+ void (*enable_mcif)(struct mcif_wb *mcif_wb);
+
+ void (*disable_mcif)(struct mcif_wb *mcif_wb);
+
+ void (*config_mcif_buf)(
+ struct mcif_wb *mcif_wb,
+ struct mcif_buf_params *params,
+ unsigned int dest_height);
+
+ void (*config_mcif_arb)(
+ struct mcif_wb *mcif_wb,
+ struct mcif_arb_params *params);
+
+ void (*config_mcif_irq)(
+ struct mcif_wb *mcif_wb,
+ struct mcif_irq_params *params);
+
+ void (*dump_frame)(
+ struct mcif_wb *mcif_wb,
+ struct mcif_buf_params *mcif_params,
+ enum dwb_scaler_mode out_format,
+ unsigned int dest_width,
+ unsigned int dest_height,
+ struct mcif_wb_frame_dump_info *dump_info,
+ unsigned char *luma_buffer,
+ unsigned char *chroma_buffer,
+ unsigned char *dest_luma_buffer,
+ unsigned char *dest_chroma_buffer);
+};
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h b/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
index caf74e3c836f..45b94e319cd4 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
@@ -31,6 +31,10 @@
#define MAX_MPCC 6
#define MAX_OPP 6
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#define MAX_DWB 1
+#endif
+
enum mpc_output_csc_mode {
MPC_OUTPUT_CSC_DISABLE = 0,
MPC_OUTPUT_CSC_COEF_A,
@@ -62,6 +66,14 @@ struct mpcc_blnd_cfg {
int global_alpha;
bool overlap_only;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ /* MPCC top/bottom gain settings */
+ int bottom_gain_mode;
+ int background_color_bpc;
+ int top_gain;
+ int bottom_inside_gain;
+ int bottom_outside_gain;
+#endif
};
struct mpcc_sm_cfg {
@@ -78,6 +90,17 @@ struct mpcc_sm_cfg {
int force_next_field_polarity;
};
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+struct mpc_denorm_clamp {
+ int clamp_max_r_cr;
+ int clamp_min_r_cr;
+ int clamp_max_g_y;
+ int clamp_min_g_y;
+ int clamp_max_b_cb;
+ int clamp_min_b_cb;
+};
+#endif
+
/*
* MPCC connection and blending configuration for a single MPCC instance.
* This struct is used as a node in an MPC tree.
@@ -103,6 +126,9 @@ struct mpc {
struct dc_context *ctx;
struct mpcc mpcc_array[MAX_MPCC];
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ struct pwl_params blender_params;
+#endif
};
struct mpcc_state {
@@ -200,6 +226,32 @@ struct mpc_funcs {
struct mpc *mpc,
struct mpc_tree *tree);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ void (*set_denorm)(struct mpc *mpc,
+ int opp_id,
+ enum dc_color_depth output_depth);
+
+ void (*set_denorm_clamp)(
+ struct mpc *mpc,
+ int opp_id,
+ struct mpc_denorm_clamp denorm_clamp);
+
+ void (*set_output_csc)(struct mpc *mpc,
+ int opp_id,
+ const uint16_t *regval,
+ enum mpc_output_csc_mode ocsc_mode);
+
+ void (*set_ocsc_default)(struct mpc *mpc,
+ int opp_id,
+ enum dc_color_space color_space,
+ enum mpc_output_csc_mode ocsc_mode);
+
+ void (*set_output_gamma)(
+ struct mpc *mpc,
+ int mpcc_id,
+ const struct pwl_params *params);
+#endif
+
};
#endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/opp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/opp.h
index d974d9e18612..5d8a7bcccc6f 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/opp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/opp.h
@@ -262,6 +262,9 @@ struct oppbuf_params {
enum oppbuf_display_segmentation mso_segmentation;
uint32_t mso_overlap_pixel_num;
uint32_t pixel_repetition;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ uint32_t num_segment_padded_pixels;
+#endif
};
struct opp_funcs {
@@ -301,6 +304,32 @@ struct opp_funcs {
struct output_pixel_processor *opp,
bool enable);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ void (*opp_set_disp_pattern_generator)(
+ struct output_pixel_processor *opp,
+ enum controller_dp_test_pattern test_pattern,
+ enum dc_color_depth color_depth,
+ const struct tg_color *solid_color,
+ int width,
+ int height);
+
+ bool (*dpg_is_blanked)(
+ struct output_pixel_processor *opp);
+
+ void (*opp_convert_pti)(
+ struct output_pixel_processor *opp,
+ bool enable,
+ bool polarity);
+
+ void (*opp_dpg_set_blank_color)(
+ struct output_pixel_processor *opp,
+ const struct tg_color *color);
+
+ void (*opp_program_left_edge_extra_pixel)(
+ struct output_pixel_processor *opp,
+ bool count);
+#endif
+
};
#endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
index 49854eb73d1d..ed7d9588b309 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
@@ -52,24 +52,47 @@ enum dp_component_depth {
DP_COMPONENT_PIXEL_DEPTH_16BPC = 0x00000004
};
+struct audio_clock_info {
+ /* pixel clock frequency*/
+ uint32_t pixel_clock_in_10khz;
+ /* N - 32KHz audio */
+ uint32_t n_32khz;
+ /* CTS - 32KHz audio*/
+ uint32_t cts_32khz;
+ uint32_t n_44khz;
+ uint32_t cts_44khz;
+ uint32_t n_48khz;
+ uint32_t cts_48khz;
+};
+
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+enum dynamic_metadata_mode {
+ dmdata_dp,
+ dmdata_hdmi,
+ dmdata_dolby_vision
+};
+#endif
+
struct encoder_info_frame {
/* auxiliary video information */
struct dc_info_packet avi;
struct dc_info_packet gamut;
struct dc_info_packet vendor;
+ struct dc_info_packet hfvsif;
/* source product description */
struct dc_info_packet spd;
/* video stream configuration */
struct dc_info_packet vsc;
/* HDR Static MetaData */
struct dc_info_packet hdrsmd;
- /* custom sdp message */
- struct dc_info_packet dpsdp;
};
struct encoder_unblank_param {
struct dc_link_settings link_settings;
struct dc_crtc_timing timing;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ bool odm;
+#endif
};
struct encoder_set_dp_phy_pattern_param {
@@ -86,7 +109,22 @@ struct stream_encoder {
enum engine_id id;
};
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+struct enc_state {
+ uint32_t dsc_mode; // DISABLED 0; 1 or 2 indicate enabled state.
+ uint32_t dsc_slice_width;
+ uint32_t sec_gsp_pps_line_num;
+ uint32_t vbid6_line_reference;
+ uint32_t vbid6_line_num;
+ uint32_t sec_gsp_pps_enable;
+ uint32_t sec_stream_enable;
+};
+#endif
+
struct stream_encoder_funcs {
+ #ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ void (*enc_read_state)(struct stream_encoder *enc, struct enc_state *s);
+ #endif
void (*dp_set_stream_attribute)(
struct stream_encoder *enc,
struct dc_crtc_timing *crtc_timing,
@@ -123,6 +161,11 @@ struct stream_encoder_funcs {
struct stream_encoder *enc,
const struct encoder_info_frame *info_frame);
+ void (*send_immediate_sdp_message)(
+ struct stream_encoder *enc,
+ const uint8_t *custom_sdp_message,
+ unsigned int sdp_message_size);
+
void (*stop_dp_info_packets)(
struct stream_encoder *enc);
@@ -168,6 +211,25 @@ struct stream_encoder_funcs {
struct stream_encoder *enc,
int tg_inst);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ void (*dp_set_dsc_config)(
+ struct stream_encoder *enc,
+ enum optc_dsc_mode dsc_mode,
+ uint32_t dsc_bytes_per_pixel,
+ uint32_t dsc_slice_width,
+ uint8_t *dsc_packed_pps);
+#endif
+
+ void (*set_dynamic_metadata)(struct stream_encoder *enc,
+ bool enable,
+ uint32_t hubp_requestor_id,
+ enum dynamic_metadata_mode dmdata_mode);
+
+ void (*dp_set_odm_combine)(
+ struct stream_encoder *enc,
+ bool odm_combine);
+#endif
};
#endif /* STREAM_ENCODER_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
index 067d53caf28a..5e93bc0e8ff9 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
@@ -70,14 +70,6 @@ enum crtc_state {
CRTC_STATE_VACTIVE
};
-struct _dlg_otg_param {
- int vstartup_start;
- int vupdate_offset;
- int vupdate_width;
- int vready_offset;
- enum signal_type signal;
-};
-
struct vupdate_keepout_params {
int start_offset;
int end_offset;
@@ -126,7 +118,6 @@ struct timing_generator {
const struct timing_generator_funcs *funcs;
struct dc_bios *bp;
struct dc_context *ctx;
- struct _dlg_otg_param dlg_otg_param;
int inst;
};
@@ -140,7 +131,13 @@ struct timing_generator_funcs {
const struct dc_crtc_timing *timing);
void (*program_timing)(struct timing_generator *tg,
const struct dc_crtc_timing *timing,
- bool use_vbios);
+ int vready_offset,
+ int vstartup_start,
+ int vupdate_offset,
+ int vupdate_width,
+ const enum signal_type signal,
+ bool use_vbios
+ );
void (*setup_vertical_interrupt0)(
struct timing_generator *optc,
uint32_t start_line,
@@ -191,6 +188,12 @@ struct timing_generator_funcs {
void (*unlock)(struct timing_generator *tg);
void (*lock)(struct timing_generator *tg);
void (*lock_global)(struct timing_generator *tg);
+ void (*lock_doublebuffer_disable)(struct timing_generator *tg);
+ void (*lock_doublebuffer_enable)(struct timing_generator *tg);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ void(*triplebuffer_unlock)(struct timing_generator *tg);
+ void(*triplebuffer_lock)(struct timing_generator *tg);
+#endif
void (*enable_reset_trigger)(struct timing_generator *tg,
int source_tg_inst);
void (*enable_crtc_reset)(struct timing_generator *tg,
@@ -210,7 +213,11 @@ struct timing_generator_funcs {
bool (*arm_vert_intr)(struct timing_generator *tg, uint8_t width);
- void (*program_global_sync)(struct timing_generator *tg);
+ void (*program_global_sync)(struct timing_generator *tg,
+ int vready_offset,
+ int vstartup_start,
+ int vupdate_offset,
+ int vupdate_width);
void (*enable_optc_clock)(struct timing_generator *tg, bool enable);
void (*program_stereo)(struct timing_generator *tg,
const struct dc_crtc_timing *timing, struct crtc_stereo_flags *flags);
@@ -223,6 +230,16 @@ struct timing_generator_funcs {
bool (*is_optc_underflow_occurred)(struct timing_generator *tg);
void (*clear_optc_underflow)(struct timing_generator *tg);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ void (*set_dwb_source)(struct timing_generator *optc,
+ uint32_t dwb_pipe_inst);
+
+ void (*get_optc_source)(struct timing_generator *optc,
+ uint32_t *num_of_input_segments,
+ uint32_t *seg0_src_sel,
+ uint32_t *seg1_src_sel);
+#endif
+
/**
* Configure CRCs for the given timing generator. Return false if TG is
* not on.
@@ -237,6 +254,27 @@ struct timing_generator_funcs {
bool (*get_crc)(struct timing_generator *tg,
uint32_t *r_cr, uint32_t *g_y, uint32_t *b_cb);
+ void (*program_manual_trigger)(struct timing_generator *optc);
+ void (*setup_manual_trigger)(struct timing_generator *optc);
+
+ void (*set_vtg_params)(struct timing_generator *optc,
+ const struct dc_crtc_timing *dc_crtc_timing);
+
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ void (*set_dsc_config)(struct timing_generator *optc,
+ enum optc_dsc_mode dsc_mode,
+ uint32_t dsc_bytes_per_pixel,
+ uint32_t dsc_slice_width);
+#endif
+ void (*set_odm_bypass)(struct timing_generator *tg, const struct dc_crtc_timing *dc_crtc_timing);
+ void (*set_odm_combine)(struct timing_generator *tg, int combine_opp_id,
+ int mpcc_hactive, enum dc_pixel_encoding pixel_encoding);
+ void (*set_gsl)(struct timing_generator *optc, const struct gsl_params *params);
+ void (*set_gsl_source_select)(struct timing_generator *optc,
+ int group_idx,
+ uint32_t gsl_ready_signal);
+#endif
};
#endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/vmid.h b/drivers/gpu/drm/amd/display/dc/inc/hw/vmid.h
index 037beb0a2a27..76de0e4284e0 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/vmid.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/vmid.h
@@ -44,6 +44,7 @@ struct dcn_vmid_page_table_config {
uint64_t page_table_end_addr;
enum dcn_hubbub_page_table_depth depth;
enum dcn_hubbub_page_table_block_size block_size;
+ uint64_t page_table_base_addr;
};
#endif /* DAL_DC_INC_HW_VMID_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
index 33905468e2b9..4d56d48a3179 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
@@ -65,11 +65,19 @@ struct dce_hwseq {
struct pipe_ctx;
struct dc_state;
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+struct dc_stream_status;
+struct dc_writeback_info;
+#endif
struct dchub_init_data;
struct dc_static_screen_events;
struct resource_pool;
struct resource_context;
struct stream_resource;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+struct dc_phy_addr_space_config;
+struct dc_virtual_addr_space_config;
+#endif
struct hw_sequencer_funcs {
@@ -102,6 +110,16 @@ struct hw_sequencer_funcs {
uint16_t *matrix,
int opp_id);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ void (*program_triplebuffer)(
+ const struct dc *dc,
+ struct pipe_ctx *pipe_ctx,
+ bool enableTripleBuffer);
+ void (*set_flip_control_gsl)(
+ struct pipe_ctx *pipe_ctx,
+ bool flip_immediate);
+#endif
+
void (*update_plane_addr)(
const struct dc *dc,
struct pipe_ctx *pipe_ctx);
@@ -114,6 +132,17 @@ struct hw_sequencer_funcs {
struct dce_hwseq *hws,
struct dchub_init_data *dh_data);
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ int (*init_sys_ctx)(
+ struct dce_hwseq *hws,
+ struct dc *dc,
+ struct dc_phy_addr_space_config *pa_config);
+ void (*init_vm_ctx)(
+ struct dce_hwseq *hws,
+ struct dc *dc,
+ struct dc_virtual_addr_space_config *va_config,
+ int vmid);
+#endif
void (*update_mpcc)(
struct dc *dc,
struct pipe_ctx *pipe_ctx);
@@ -158,6 +187,11 @@ struct hw_sequencer_funcs {
void (*update_info_frame)(struct pipe_ctx *pipe_ctx);
+ void (*send_immediate_sdp_message)(
+ struct pipe_ctx *pipe_ctx,
+ const uint8_t *custom_sdp_message,
+ unsigned int sdp_message_size);
+
void (*enable_stream)(struct pipe_ctx *pipe_ctx);
void (*disable_stream)(struct pipe_ctx *pipe_ctx,
@@ -176,6 +210,7 @@ struct hw_sequencer_funcs {
struct dc *dc,
struct pipe_ctx *pipe,
bool lock);
+
void (*pipe_control_lock_global)(
struct dc *dc,
struct pipe_ctx *pipe,
@@ -192,6 +227,13 @@ struct hw_sequencer_funcs {
struct dc *dc,
struct dc_state *context);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ bool (*update_bandwidth)(
+ struct dc *dc,
+ struct dc_state *context);
+ bool (*dmdata_status_done)(struct pipe_ctx *pipe_ctx);
+#endif
+
void (*set_drr)(struct pipe_ctx **pipe_ctx, int num_pipes,
int vmin, int vmax);
@@ -235,7 +277,23 @@ struct hw_sequencer_funcs {
void (*setup_periodic_interrupt)(struct pipe_ctx *pipe_ctx, enum vline_select vline);
void (*setup_vupdate_interrupt)(struct pipe_ctx *pipe_ctx);
+ bool (*did_underflow_occur)(struct dc *dc, struct pipe_ctx *pipe_ctx);
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ void (*update_odm)(struct dc *dc, struct dc_state *context, struct pipe_ctx *pipe_ctx);
+ void (*program_all_writeback_pipes_in_tree)(
+ struct dc *dc,
+ const struct dc_stream_state *stream,
+ struct dc_state *context);
+ void (*update_writeback)(struct dc *dc,
+ const struct dc_stream_status *stream_status,
+ struct dc_writeback_info *wb_info);
+ void (*enable_writeback)(struct dc *dc,
+ const struct dc_stream_status *stream_status,
+ struct dc_writeback_info *wb_info);
+ void (*disable_writeback)(struct dc *dc,
+ unsigned int dwb_pipe_inst);
+#endif
};
void color_space_to_black_color(
diff --git a/drivers/gpu/drm/amd/display/dc/inc/resource.h b/drivers/gpu/drm/amd/display/dc/inc/resource.h
index 3ce0a4fc5822..47f81072d7e9 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/resource.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/resource.h
@@ -30,6 +30,8 @@
#include "dal_asic_id.h"
#include "dm_pp_smu.h"
+#define MEMORY_TYPE_MULTIPLIER_CZ 4
+
enum dce_version resource_parse_asic_id(
struct hw_asic_id asic_id);
@@ -42,6 +44,12 @@ struct resource_caps {
int num_pll;
int num_dwb;
int num_ddc;
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+ int num_vmid;
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ int num_dsc;
+#endif
+#endif
};
struct resource_straps {
diff --git a/drivers/gpu/drm/amd/display/dc/inc/vm_helper.h b/drivers/gpu/drm/amd/display/dc/inc/vm_helper.h
index 193407f76a80..8bfcef0a3675 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/vm_helper.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/vm_helper.h
@@ -28,29 +28,21 @@
#include "dc_types.h"
-#define MAX_VMID 16
#define MAX_HUBP 6
struct vmid_usage {
- uint16_t vmid_usage[2];
+ int vmid_usage[2];
};
struct vm_helper {
unsigned int num_vmid;
- unsigned int num_hubp;
- unsigned int num_vmids_available;
- uint64_t ptb_assigned_to_vmid[MAX_VMID];
struct vmid_usage hubp_vmid_usage[MAX_HUBP];
};
-uint8_t get_vmid_for_ptb(
- struct vm_helper *vm_helper,
- int64_t ptb,
- uint8_t pipe_idx);
+void vm_helper_mark_vmid_used(struct vm_helper *vm_helper, unsigned int pos, uint8_t hubp_idx);
-void init_vm_helper(
+void vm_helper_init(
struct vm_helper *vm_helper,
- unsigned int num_vmid,
- unsigned int num_hubp);
+ unsigned int num_vmid);
#endif /* DC_INC_VM_HELPER_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/irq/Makefile b/drivers/gpu/drm/amd/display/dc/irq/Makefile
index 498515aad4a5..ad87c2f093e2 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/irq/Makefile
@@ -67,3 +67,13 @@ AMD_DAL_IRQ_DCN1 = $(addprefix $(AMDDALPATH)/dc/irq/dcn10/,$(IRQ_DCN1))
AMD_DISPLAY_FILES += $(AMD_DAL_IRQ_DCN1)
endif
+###############################################################################
+# DCN 20
+###############################################################################
+ifdef CONFIG_DRM_AMD_DC_DCN2_0
+IRQ_DCN2 = irq_service_dcn20.o
+
+AMD_DAL_IRQ_DCN2 = $(addprefix $(AMDDALPATH)/dc/irq/dcn20/,$(IRQ_DCN2))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_IRQ_DCN2)
+endif
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c b/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
index 86987f5e8bd5..1a581c464345 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
+++ b/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "include/logger_interface.h"
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dce120/irq_service_dce120.c b/drivers/gpu/drm/amd/display/dc/irq/dce120/irq_service_dce120.c
index 750ba0ab4106..15380336cb51 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/dce120/irq_service_dce120.c
+++ b/drivers/gpu/drm/amd/display/dc/irq/dce120/irq_service_dce120.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "include/logger_interface.h"
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dce80/irq_service_dce80.c b/drivers/gpu/drm/amd/display/dc/irq/dce80/irq_service_dce80.c
index de218fe84a43..281fee8ad1e5 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/dce80/irq_service_dce80.c
+++ b/drivers/gpu/drm/amd/display/dc/irq/dce80/irq_service_dce80.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "include/logger_interface.h"
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c b/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c
index 10ac6deff5ff..cc8e7dedccce 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c
+++ b/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "include/logger_interface.h"
@@ -36,7 +38,7 @@
#include "irq_service_dcn10.h"
-#include "ivsrcid/irqsrcs_dcn_1_0.h"
+#include "ivsrcid/dcn/irqsrcs_dcn_1_0.h"
enum dc_irq_source to_dal_irq_source_dcn10(
struct irq_service *irq_service,
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dcn20/irq_service_dcn20.c b/drivers/gpu/drm/amd/display/dc/irq/dcn20/irq_service_dcn20.c
new file mode 100644
index 000000000000..3cc0f2a1f77c
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/irq/dcn20/irq_service_dcn20.c
@@ -0,0 +1,375 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include <linux/slab.h>
+
+#include "dm_services.h"
+
+#include "include/logger_interface.h"
+
+#include "../dce110/irq_service_dce110.h"
+
+#include "dcn/dcn_2_0_0_offset.h"
+#include "dcn/dcn_2_0_0_sh_mask.h"
+#include "navi10_ip_offset.h"
+
+
+#include "irq_service_dcn20.h"
+
+#include "ivsrcid/dcn/irqsrcs_dcn_1_0.h"
+
+enum dc_irq_source to_dal_irq_source_dcn20(
+ struct irq_service *irq_service,
+ uint32_t src_id,
+ uint32_t ext_id)
+{
+ switch (src_id) {
+ case DCN_1_0__SRCID__DC_D1_OTG_VSTARTUP:
+ return DC_IRQ_SOURCE_VBLANK1;
+ case DCN_1_0__SRCID__DC_D2_OTG_VSTARTUP:
+ return DC_IRQ_SOURCE_VBLANK2;
+ case DCN_1_0__SRCID__DC_D3_OTG_VSTARTUP:
+ return DC_IRQ_SOURCE_VBLANK3;
+ case DCN_1_0__SRCID__DC_D4_OTG_VSTARTUP:
+ return DC_IRQ_SOURCE_VBLANK4;
+ case DCN_1_0__SRCID__DC_D5_OTG_VSTARTUP:
+ return DC_IRQ_SOURCE_VBLANK5;
+ case DCN_1_0__SRCID__DC_D6_OTG_VSTARTUP:
+ return DC_IRQ_SOURCE_VBLANK6;
+ case DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT:
+ return DC_IRQ_SOURCE_PFLIP1;
+ case DCN_1_0__SRCID__HUBP1_FLIP_INTERRUPT:
+ return DC_IRQ_SOURCE_PFLIP2;
+ case DCN_1_0__SRCID__HUBP2_FLIP_INTERRUPT:
+ return DC_IRQ_SOURCE_PFLIP3;
+ case DCN_1_0__SRCID__HUBP3_FLIP_INTERRUPT:
+ return DC_IRQ_SOURCE_PFLIP4;
+ case DCN_1_0__SRCID__HUBP4_FLIP_INTERRUPT:
+ return DC_IRQ_SOURCE_PFLIP5;
+ case DCN_1_0__SRCID__HUBP5_FLIP_INTERRUPT:
+ return DC_IRQ_SOURCE_PFLIP6;
+ case DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+ return DC_IRQ_SOURCE_VUPDATE1;
+ case DCN_1_0__SRCID__OTG1_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+ return DC_IRQ_SOURCE_VUPDATE2;
+ case DCN_1_0__SRCID__OTG2_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+ return DC_IRQ_SOURCE_VUPDATE3;
+ case DCN_1_0__SRCID__OTG3_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+ return DC_IRQ_SOURCE_VUPDATE4;
+ case DCN_1_0__SRCID__OTG4_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+ return DC_IRQ_SOURCE_VUPDATE5;
+ case DCN_1_0__SRCID__OTG5_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+ return DC_IRQ_SOURCE_VUPDATE6;
+
+ case DCN_1_0__SRCID__DC_HPD1_INT:
+ /* generic src_id for all HPD and HPDRX interrupts */
+ switch (ext_id) {
+ case DCN_1_0__CTXID__DC_HPD1_INT:
+ return DC_IRQ_SOURCE_HPD1;
+ case DCN_1_0__CTXID__DC_HPD2_INT:
+ return DC_IRQ_SOURCE_HPD2;
+ case DCN_1_0__CTXID__DC_HPD3_INT:
+ return DC_IRQ_SOURCE_HPD3;
+ case DCN_1_0__CTXID__DC_HPD4_INT:
+ return DC_IRQ_SOURCE_HPD4;
+ case DCN_1_0__CTXID__DC_HPD5_INT:
+ return DC_IRQ_SOURCE_HPD5;
+ case DCN_1_0__CTXID__DC_HPD6_INT:
+ return DC_IRQ_SOURCE_HPD6;
+ case DCN_1_0__CTXID__DC_HPD1_RX_INT:
+ return DC_IRQ_SOURCE_HPD1RX;
+ case DCN_1_0__CTXID__DC_HPD2_RX_INT:
+ return DC_IRQ_SOURCE_HPD2RX;
+ case DCN_1_0__CTXID__DC_HPD3_RX_INT:
+ return DC_IRQ_SOURCE_HPD3RX;
+ case DCN_1_0__CTXID__DC_HPD4_RX_INT:
+ return DC_IRQ_SOURCE_HPD4RX;
+ case DCN_1_0__CTXID__DC_HPD5_RX_INT:
+ return DC_IRQ_SOURCE_HPD5RX;
+ case DCN_1_0__CTXID__DC_HPD6_RX_INT:
+ return DC_IRQ_SOURCE_HPD6RX;
+ default:
+ return DC_IRQ_SOURCE_INVALID;
+ }
+ break;
+
+ default:
+ return DC_IRQ_SOURCE_INVALID;
+ }
+}
+
+static bool hpd_ack(
+ struct irq_service *irq_service,
+ const struct irq_source_info *info)
+{
+ uint32_t addr = info->status_reg;
+ uint32_t value = dm_read_reg(irq_service->ctx, addr);
+ uint32_t current_status =
+ get_reg_field_value(
+ value,
+ HPD0_DC_HPD_INT_STATUS,
+ DC_HPD_SENSE_DELAYED);
+
+ dal_irq_service_ack_generic(irq_service, info);
+
+ value = dm_read_reg(irq_service->ctx, info->enable_reg);
+
+ set_reg_field_value(
+ value,
+ current_status ? 0 : 1,
+ HPD0_DC_HPD_INT_CONTROL,
+ DC_HPD_INT_POLARITY);
+
+ dm_write_reg(irq_service->ctx, info->enable_reg, value);
+
+ return true;
+}
+
+static const struct irq_source_info_funcs hpd_irq_info_funcs = {
+ .set = NULL,
+ .ack = hpd_ack
+};
+
+static const struct irq_source_info_funcs hpd_rx_irq_info_funcs = {
+ .set = NULL,
+ .ack = NULL
+};
+
+static const struct irq_source_info_funcs pflip_irq_info_funcs = {
+ .set = NULL,
+ .ack = NULL
+};
+
+static const struct irq_source_info_funcs vblank_irq_info_funcs = {
+ .set = NULL,
+ .ack = NULL
+};
+
+#undef BASE_INNER
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+/* compile time expand base address. */
+#define BASE(seg) \
+ BASE_INNER(seg)
+
+
+#define SRI(reg_name, block, id)\
+ BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+ mm ## block ## id ## _ ## reg_name
+
+
+#define IRQ_REG_ENTRY(block, reg_num, reg1, mask1, reg2, mask2)\
+ .enable_reg = SRI(reg1, block, reg_num),\
+ .enable_mask = \
+ block ## reg_num ## _ ## reg1 ## __ ## mask1 ## _MASK,\
+ .enable_value = {\
+ block ## reg_num ## _ ## reg1 ## __ ## mask1 ## _MASK,\
+ ~block ## reg_num ## _ ## reg1 ## __ ## mask1 ## _MASK \
+ },\
+ .ack_reg = SRI(reg2, block, reg_num),\
+ .ack_mask = \
+ block ## reg_num ## _ ## reg2 ## __ ## mask2 ## _MASK,\
+ .ack_value = \
+ block ## reg_num ## _ ## reg2 ## __ ## mask2 ## _MASK \
+
+
+
+#define hpd_int_entry(reg_num)\
+ [DC_IRQ_SOURCE_HPD1 + reg_num] = {\
+ IRQ_REG_ENTRY(HPD, reg_num,\
+ DC_HPD_INT_CONTROL, DC_HPD_INT_EN,\
+ DC_HPD_INT_CONTROL, DC_HPD_INT_ACK),\
+ .status_reg = SRI(DC_HPD_INT_STATUS, HPD, reg_num),\
+ .funcs = &hpd_irq_info_funcs\
+ }
+
+#define hpd_rx_int_entry(reg_num)\
+ [DC_IRQ_SOURCE_HPD1RX + reg_num] = {\
+ IRQ_REG_ENTRY(HPD, reg_num,\
+ DC_HPD_INT_CONTROL, DC_HPD_RX_INT_EN,\
+ DC_HPD_INT_CONTROL, DC_HPD_RX_INT_ACK),\
+ .status_reg = SRI(DC_HPD_INT_STATUS, HPD, reg_num),\
+ .funcs = &hpd_rx_irq_info_funcs\
+ }
+#define pflip_int_entry(reg_num)\
+ [DC_IRQ_SOURCE_PFLIP1 + reg_num] = {\
+ IRQ_REG_ENTRY(HUBPREQ, reg_num,\
+ DCSURF_SURFACE_FLIP_INTERRUPT, SURFACE_FLIP_INT_MASK,\
+ DCSURF_SURFACE_FLIP_INTERRUPT, SURFACE_FLIP_CLEAR),\
+ .funcs = &pflip_irq_info_funcs\
+ }
+
+#define vupdate_int_entry(reg_num)\
+ [DC_IRQ_SOURCE_VUPDATE1 + reg_num] = {\
+ IRQ_REG_ENTRY(OTG, reg_num,\
+ OTG_GLOBAL_SYNC_STATUS, VUPDATE_INT_EN,\
+ OTG_GLOBAL_SYNC_STATUS, VUPDATE_EVENT_CLEAR),\
+ .funcs = &vblank_irq_info_funcs\
+ }
+
+#define vblank_int_entry(reg_num)\
+ [DC_IRQ_SOURCE_VBLANK1 + reg_num] = {\
+ IRQ_REG_ENTRY(OTG, reg_num,\
+ OTG_GLOBAL_SYNC_STATUS, VSTARTUP_INT_EN,\
+ OTG_GLOBAL_SYNC_STATUS, VSTARTUP_EVENT_CLEAR),\
+ .funcs = &vblank_irq_info_funcs\
+ }
+
+#define dummy_irq_entry() \
+ {\
+ .funcs = &dummy_irq_info_funcs\
+ }
+
+#define i2c_int_entry(reg_num) \
+ [DC_IRQ_SOURCE_I2C_DDC ## reg_num] = dummy_irq_entry()
+
+#define dp_sink_int_entry(reg_num) \
+ [DC_IRQ_SOURCE_DPSINK ## reg_num] = dummy_irq_entry()
+
+#define gpio_pad_int_entry(reg_num) \
+ [DC_IRQ_SOURCE_GPIOPAD ## reg_num] = dummy_irq_entry()
+
+#define dc_underflow_int_entry(reg_num) \
+ [DC_IRQ_SOURCE_DC ## reg_num ## UNDERFLOW] = dummy_irq_entry()
+
+static const struct irq_source_info_funcs dummy_irq_info_funcs = {
+ .set = dal_irq_service_dummy_set,
+ .ack = dal_irq_service_dummy_ack
+};
+
+static const struct irq_source_info
+irq_source_info_dcn20[DAL_IRQ_SOURCES_NUMBER] = {
+ [DC_IRQ_SOURCE_INVALID] = dummy_irq_entry(),
+ hpd_int_entry(0),
+ hpd_int_entry(1),
+ hpd_int_entry(2),
+ hpd_int_entry(3),
+ hpd_int_entry(4),
+ hpd_int_entry(5),
+ hpd_rx_int_entry(0),
+ hpd_rx_int_entry(1),
+ hpd_rx_int_entry(2),
+ hpd_rx_int_entry(3),
+ hpd_rx_int_entry(4),
+ hpd_rx_int_entry(5),
+ i2c_int_entry(1),
+ i2c_int_entry(2),
+ i2c_int_entry(3),
+ i2c_int_entry(4),
+ i2c_int_entry(5),
+ i2c_int_entry(6),
+ dp_sink_int_entry(1),
+ dp_sink_int_entry(2),
+ dp_sink_int_entry(3),
+ dp_sink_int_entry(4),
+ dp_sink_int_entry(5),
+ dp_sink_int_entry(6),
+ [DC_IRQ_SOURCE_TIMER] = dummy_irq_entry(),
+ pflip_int_entry(0),
+ pflip_int_entry(1),
+ pflip_int_entry(2),
+ pflip_int_entry(3),
+ [DC_IRQ_SOURCE_PFLIP5] = dummy_irq_entry(),
+ [DC_IRQ_SOURCE_PFLIP6] = dummy_irq_entry(),
+ [DC_IRQ_SOURCE_PFLIP_UNDERLAY0] = dummy_irq_entry(),
+ gpio_pad_int_entry(0),
+ gpio_pad_int_entry(1),
+ gpio_pad_int_entry(2),
+ gpio_pad_int_entry(3),
+ gpio_pad_int_entry(4),
+ gpio_pad_int_entry(5),
+ gpio_pad_int_entry(6),
+ gpio_pad_int_entry(7),
+ gpio_pad_int_entry(8),
+ gpio_pad_int_entry(9),
+ gpio_pad_int_entry(10),
+ gpio_pad_int_entry(11),
+ gpio_pad_int_entry(12),
+ gpio_pad_int_entry(13),
+ gpio_pad_int_entry(14),
+ gpio_pad_int_entry(15),
+ gpio_pad_int_entry(16),
+ gpio_pad_int_entry(17),
+ gpio_pad_int_entry(18),
+ gpio_pad_int_entry(19),
+ gpio_pad_int_entry(20),
+ gpio_pad_int_entry(21),
+ gpio_pad_int_entry(22),
+ gpio_pad_int_entry(23),
+ gpio_pad_int_entry(24),
+ gpio_pad_int_entry(25),
+ gpio_pad_int_entry(26),
+ gpio_pad_int_entry(27),
+ gpio_pad_int_entry(28),
+ gpio_pad_int_entry(29),
+ gpio_pad_int_entry(30),
+ dc_underflow_int_entry(1),
+ dc_underflow_int_entry(2),
+ dc_underflow_int_entry(3),
+ dc_underflow_int_entry(4),
+ dc_underflow_int_entry(5),
+ dc_underflow_int_entry(6),
+ [DC_IRQ_SOURCE_DMCU_SCP] = dummy_irq_entry(),
+ [DC_IRQ_SOURCE_VBIOS_SW] = dummy_irq_entry(),
+ vupdate_int_entry(0),
+ vupdate_int_entry(1),
+ vupdate_int_entry(2),
+ vupdate_int_entry(3),
+ vupdate_int_entry(4),
+ vupdate_int_entry(5),
+ vblank_int_entry(0),
+ vblank_int_entry(1),
+ vblank_int_entry(2),
+ vblank_int_entry(3),
+ vblank_int_entry(4),
+ vblank_int_entry(5),
+};
+
+static const struct irq_service_funcs irq_service_funcs_dcn20 = {
+ .to_dal_irq_source = to_dal_irq_source_dcn20
+};
+
+static void construct(
+ struct irq_service *irq_service,
+ struct irq_service_init_data *init_data)
+{
+ dal_irq_service_construct(irq_service, init_data);
+
+ irq_service->info = irq_source_info_dcn20;
+ irq_service->funcs = &irq_service_funcs_dcn20;
+}
+
+struct irq_service *dal_irq_service_dcn20_create(
+ struct irq_service_init_data *init_data)
+{
+ struct irq_service *irq_service = kzalloc(sizeof(*irq_service),
+ GFP_KERNEL);
+
+ if (!irq_service)
+ return NULL;
+
+ construct(irq_service, init_data);
+ return irq_service;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dcn20/irq_service_dcn20.h b/drivers/gpu/drm/amd/display/dc/irq/dcn20/irq_service_dcn20.h
new file mode 100644
index 000000000000..aee4b37999f1
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/irq/dcn20/irq_service_dcn20.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DAL_IRQ_SERVICE_DCN20_H__
+#define __DAL_IRQ_SERVICE_DCN20_H__
+
+#include "../irq_service.h"
+
+struct irq_service *dal_irq_service_dcn20_create(
+ struct irq_service_init_data *init_data);
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/irq/irq_service.c b/drivers/gpu/drm/amd/display/dc/irq/irq_service.c
index 604bea01fc13..0878550a8178 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/irq_service.c
+++ b/drivers/gpu/drm/amd/display/dc/irq/irq_service.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "include/irq_service_interface.h"
diff --git a/drivers/gpu/drm/amd/display/dc/os_types.h b/drivers/gpu/drm/amd/display/dc/os_types.h
index c0d9f332baed..30ec80ac6fc8 100644
--- a/drivers/gpu/drm/amd/display/dc/os_types.h
+++ b/drivers/gpu/drm/amd/display/dc/os_types.h
@@ -26,11 +26,13 @@
#ifndef _OS_TYPES_H_
#define _OS_TYPES_H_
-#include <asm/byteorder.h>
+#include <linux/kgdb.h>
+#include <linux/kref.h>
#include <linux/types.h>
-#include <drm/drmP.h>
-#include <linux/kref.h>
+#include <asm/byteorder.h>
+
+#include <drm/drm_print.h>
#include "cgs_common.h"
diff --git a/drivers/gpu/drm/amd/display/dc/virtual/virtual_link_encoder.c b/drivers/gpu/drm/amd/display/dc/virtual/virtual_link_encoder.c
index 1c079ba37c30..3464b2d5b89a 100644
--- a/drivers/gpu/drm/amd/display/dc/virtual/virtual_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/virtual/virtual_link_encoder.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "dm_services_types.h"
diff --git a/drivers/gpu/drm/amd/display/dc/virtual/virtual_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/virtual/virtual_stream_encoder.c
index fdcf9e66d852..c9a6dd878d9b 100644
--- a/drivers/gpu/drm/amd/display/dc/virtual/virtual_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/virtual/virtual_stream_encoder.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "virtual_stream_encoder.h"
@@ -75,7 +77,22 @@ static void virtual_audio_mute_control(
struct stream_encoder *enc,
bool mute) {}
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+static void virtual_enc_dp_set_odm_combine(
+ struct stream_encoder *enc,
+ bool odm_combine)
+{}
+#endif
+#endif
+
static const struct stream_encoder_funcs virtual_str_enc_funcs = {
+#ifdef CONFIG_DRM_AMD_DC_DCN2_0
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ .dp_set_odm_combine =
+ virtual_enc_dp_set_odm_combine,
+#endif
+#endif
.dp_set_stream_attribute =
virtual_stream_encoder_dp_set_stream_attribute,
.hdmi_set_stream_attribute =
diff --git a/drivers/gpu/drm/amd/display/include/bios_parser_types.h b/drivers/gpu/drm/amd/display/include/bios_parser_types.h
index 01bf01a34a08..c30437ae8395 100644
--- a/drivers/gpu/drm/amd/display/include/bios_parser_types.h
+++ b/drivers/gpu/drm/amd/display/include/bios_parser_types.h
@@ -307,7 +307,8 @@ struct bp_encoder_cap_info {
uint32_t DP_HBR2_EN:1;
uint32_t DP_HBR3_EN:1;
uint32_t HDMI_6GB_EN:1;
- uint32_t RESERVED:30;
+ uint32_t DP_IS_USB_C:1;
+ uint32_t RESERVED:27;
};
#endif /*__DAL_BIOS_PARSER_TYPES_H__ */
diff --git a/drivers/gpu/drm/amd/display/include/dal_asic_id.h b/drivers/gpu/drm/amd/display/include/dal_asic_id.h
index 4c8ce7938f01..887e6a8597c4 100644
--- a/drivers/gpu/drm/amd/display/include/dal_asic_id.h
+++ b/drivers/gpu/drm/amd/display/include/dal_asic_id.h
@@ -132,23 +132,27 @@
#define RAVEN_A0 0x01
#define RAVEN_B0 0x21
#define PICASSO_A0 0x41
-#if defined(CONFIG_DRM_AMD_DC_DCN1_01)
/* DCN1_01 */
#define RAVEN2_A0 0x81
-#endif
-#define RAVEN_UNKNOWN 0xFF
-
-#define ASIC_REV_IS_RAVEN(eChipRev) ((eChipRev >= RAVEN_A0) && eChipRev < RAVEN_UNKNOWN)
#define RAVEN1_F0 0xF0
-#define ASICREV_IS_RV1_F0(eChipRev) ((eChipRev >= RAVEN1_F0) && (eChipRev < RAVEN_UNKNOWN))
+#define RAVEN_UNKNOWN 0xFF
-#if defined(CONFIG_DRM_AMD_DC_DCN1_01)
+#define ASICREV_IS_RAVEN(eChipRev) ((eChipRev >= RAVEN_A0) && eChipRev < RAVEN_UNKNOWN)
#define ASICREV_IS_PICASSO(eChipRev) ((eChipRev >= PICASSO_A0) && (eChipRev < RAVEN2_A0))
#define ASICREV_IS_RAVEN2(eChipRev) ((eChipRev >= RAVEN2_A0) && (eChipRev < 0xF0))
-#endif /* DCN1_01 */
+
+
+#define ASICREV_IS_RV1_F0(eChipRev) ((eChipRev >= RAVEN1_F0) && (eChipRev < RAVEN_UNKNOWN))
+
#define FAMILY_RV 142 /* DCN 1*/
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+
+#define FAMILY_NV 143 /* DCN 2*/
+
+#endif
+
/*
* ASIC chip ID
*/
diff --git a/drivers/gpu/drm/amd/display/include/dal_types.h b/drivers/gpu/drm/amd/display/include/dal_types.h
index f5bd869d4320..1e3ce4d847ae 100644
--- a/drivers/gpu/drm/amd/display/include/dal_types.h
+++ b/drivers/gpu/drm/amd/display/include/dal_types.h
@@ -45,9 +45,10 @@ enum dce_version {
DCE_VERSION_12_1,
DCE_VERSION_MAX,
DCN_VERSION_1_0,
-#if defined(CONFIG_DRM_AMD_DC_DCN1_01)
DCN_VERSION_1_01,
-#endif /* DCN1_01 */
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+ DCN_VERSION_2_0,
+#endif
DCN_VERSION_MAX
};
diff --git a/drivers/gpu/drm/amd/display/include/dpcd_defs.h b/drivers/gpu/drm/amd/display/include/dpcd_defs.h
index 1c66166d0a94..2c90d1b46c8b 100644
--- a/drivers/gpu/drm/amd/display/include/dpcd_defs.h
+++ b/drivers/gpu/drm/amd/display/include/dpcd_defs.h
@@ -43,7 +43,7 @@ enum dpcd_revision {
enum dpcd_downstream_port_type {
DOWNSTREAM_DP = 0,
DOWNSTREAM_VGA,
- DOWNSTREAM_DVI_HDMI,
+ DOWNSTREAM_DVI_HDMI_DP_PLUS_PLUS,/* DVI, HDMI, DP++ */
DOWNSTREAM_NONDDC /* has no EDID (TV,CV) */
};
diff --git a/drivers/gpu/drm/amd/display/include/logger_types.h b/drivers/gpu/drm/amd/display/include/logger_types.h
index d96550d6434d..ea8d445816b8 100644
--- a/drivers/gpu/drm/amd/display/include/logger_types.h
+++ b/drivers/gpu/drm/amd/display/include/logger_types.h
@@ -63,6 +63,12 @@
#define DC_LOG_IF_TRACE(...) pr_debug("[IF_TRACE]:"__VA_ARGS__)
#define DC_LOG_PERF_TRACE(...) DRM_DEBUG_KMS(__VA_ARGS__)
#define DC_LOG_RETIMER_REDRIVER(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+#define DC_LOG_DSC(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#endif
+#if defined(CONFIG_DRM_AMD_DC_DCN3_0) || defined(CONFIG_DRM_AMD_DC_DCN2_0)
+#define DC_LOG_DWB(...) DRM_DEBUG_KMS(__VA_ARGS__)
+#endif
struct dal_logger;
@@ -107,6 +113,10 @@ enum dc_log_type {
LOG_PERF_TRACE,
LOG_DISPLAYSTATS,
LOG_HDMI_RETIMER_REDRIVER,
+#ifdef CONFIG_DRM_AMD_DC_DSC_SUPPORT
+ LOG_DSC,
+#endif
+ LOG_DWB,
LOG_SECTION_TOTAL_COUNT
};
diff --git a/drivers/gpu/drm/amd/display/include/set_mode_types.h b/drivers/gpu/drm/amd/display/include/set_mode_types.h
index 2b836e582c08..845fea8a387f 100644
--- a/drivers/gpu/drm/amd/display/include/set_mode_types.h
+++ b/drivers/gpu/drm/amd/display/include/set_mode_types.h
@@ -84,7 +84,10 @@ union hdmi_info_packet {
uint16_t bar_left;
uint16_t bar_right;
- uint8_t reserved[14];
+ uint8_t F140_F143:4;
+ uint8_t ACE0_ACE3:4;
+
+ uint8_t reserved[13];
} bits;
struct info_packet_raw_data packet_raw_data;
diff --git a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
index a1055413bade..88898935a5e6 100644
--- a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
+++ b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
@@ -23,6 +23,9 @@
*
*/
+#include <linux/mm.h>
+#include <linux/slab.h>
+
#include "dc.h"
#include "opp.h"
#include "color_gamma.h"
@@ -240,16 +243,27 @@ struct dividers {
struct fixed31_32 divider3;
};
-static void build_coefficients(struct gamma_coefficients *coefficients, bool is_2_4)
+enum gamma_type_index {
+ gamma_type_index_2_4,
+ gamma_type_index_2_2,
+ gamma_type_index_2_2_flat
+};
+
+static void build_coefficients(struct gamma_coefficients *coefficients, enum gamma_type_index type)
{
- static const int32_t numerator01[] = { 31308, 180000};
- static const int32_t numerator02[] = { 12920, 4500};
- static const int32_t numerator03[] = { 55, 99};
- static const int32_t numerator04[] = { 55, 99};
- static const int32_t numerator05[] = { 2400, 2200};
+ static const int32_t numerator01[] = { 31308, 180000, 0};
+ static const int32_t numerator02[] = { 12920, 4500, 0};
+ static const int32_t numerator03[] = { 55, 99, 0};
+ static const int32_t numerator04[] = { 55, 99, 0};
+ static const int32_t numerator05[] = { 2400, 2200, 2200};
uint32_t i = 0;
- uint32_t index = is_2_4 == true ? 0:1;
+ uint32_t index = 0;
+
+ if (type == gamma_type_index_2_2)
+ index = 1;
+ else if (type == gamma_type_index_2_2_flat)
+ index = 2;
do {
coefficients->a0[i] = dc_fixpt_from_fraction(
@@ -697,7 +711,7 @@ static void build_de_pq(struct pwl_float_data_ex *de_pq,
static void build_regamma(struct pwl_float_data_ex *rgb_regamma,
uint32_t hw_points_num,
- const struct hw_x_point *coordinate_x, bool is_2_4)
+ const struct hw_x_point *coordinate_x, enum gamma_type_index type)
{
uint32_t i;
@@ -705,7 +719,7 @@ static void build_regamma(struct pwl_float_data_ex *rgb_regamma,
struct pwl_float_data_ex *rgb = rgb_regamma;
const struct hw_x_point *coord_x = coordinate_x;
- build_coefficients(&coeff, is_2_4);
+ build_coefficients(&coeff, type);
i = 0;
@@ -892,13 +906,13 @@ static bool build_freesync_hdr(struct pwl_float_data_ex *rgb_regamma,
static void build_degamma(struct pwl_float_data_ex *curve,
uint32_t hw_points_num,
- const struct hw_x_point *coordinate_x, bool is_2_4)
+ const struct hw_x_point *coordinate_x, enum gamma_type_index type)
{
uint32_t i;
struct gamma_coefficients coeff;
uint32_t begin_index, end_index;
- build_coefficients(&coeff, is_2_4);
+ build_coefficients(&coeff, type);
i = 0;
/* X points is 2^-25 to 2^7
@@ -1558,7 +1572,8 @@ bool mod_color_calculate_regamma_params(struct dc_transfer_func *output_tf,
output_tf->tf == TRANSFER_FUNCTION_SRGB) {
if (ramp == NULL)
return true;
- if (ramp->is_identity || (!mapUserRamp && ramp->type == GAMMA_RGB_256))
+ if ((ramp->is_logical_identity) ||
+ (!mapUserRamp && ramp->type == GAMMA_RGB_256))
return true;
}
@@ -1614,7 +1629,7 @@ bool mod_color_calculate_regamma_params(struct dc_transfer_func *output_tf,
coordinates_x,
output_tf->sdr_ref_white_level);
} else if (tf == TRANSFER_FUNCTION_GAMMA22 &&
- fs_params != NULL) {
+ fs_params != NULL && fs_params->skip_tm == 0) {
build_freesync_hdr(rgb_regamma,
MAX_HW_POINTS,
coordinates_x,
@@ -1627,7 +1642,9 @@ bool mod_color_calculate_regamma_params(struct dc_transfer_func *output_tf,
build_regamma(rgb_regamma,
MAX_HW_POINTS,
- coordinates_x, tf == TRANSFER_FUNCTION_SRGB ? true:false);
+ coordinates_x, tf == TRANSFER_FUNCTION_SRGB ? gamma_type_index_2_4 :
+ tf == TRANSFER_FUNCTION_GAMMA22 ?
+ gamma_type_index_2_2_flat : gamma_type_index_2_2);
}
map_regamma_hw_to_x_user(ramp, coeff, rgb_user,
coordinates_x, axis_x, rgb_regamma,
@@ -1832,7 +1849,9 @@ bool mod_color_calculate_degamma_params(struct dc_transfer_func *input_tf,
build_degamma(curve,
MAX_HW_POINTS,
coordinates_x,
- tf == TRANSFER_FUNCTION_SRGB ? true : false);
+ tf == TRANSFER_FUNCTION_SRGB ?
+ gamma_type_index_2_4 : tf == TRANSFER_FUNCTION_GAMMA22 ?
+ gamma_type_index_2_2_flat : gamma_type_index_2_2);
else if (tf == TRANSFER_FUNCTION_LINEAR) {
// just copy coordinates_x into curve
i = 0;
@@ -1932,7 +1951,10 @@ bool mod_color_calculate_curve(enum dc_transfer_func_predefined trans,
build_regamma(rgb_regamma,
MAX_HW_POINTS,
- coordinates_x, trans == TRANSFER_FUNCTION_SRGB ? true:false);
+ coordinates_x,
+ trans == TRANSFER_FUNCTION_SRGB ?
+ gamma_type_index_2_4 : trans == TRANSFER_FUNCTION_GAMMA22 ?
+ gamma_type_index_2_2_flat : gamma_type_index_2_2);
for (i = 0; i <= MAX_HW_POINTS ; i++) {
points->red[i] = rgb_regamma[i].r;
points->green[i] = rgb_regamma[i].g;
@@ -2002,7 +2024,8 @@ bool mod_color_calculate_degamma_curve(enum dc_transfer_func_predefined trans,
kvfree(rgb_degamma);
} else if (trans == TRANSFER_FUNCTION_SRGB ||
- trans == TRANSFER_FUNCTION_BT709) {
+ trans == TRANSFER_FUNCTION_BT709 ||
+ trans == TRANSFER_FUNCTION_GAMMA22) {
rgb_degamma = kvcalloc(MAX_HW_POINTS + _EXTRA_POINTS,
sizeof(*rgb_degamma),
GFP_KERNEL);
@@ -2011,7 +2034,10 @@ bool mod_color_calculate_degamma_curve(enum dc_transfer_func_predefined trans,
build_degamma(rgb_degamma,
MAX_HW_POINTS,
- coordinates_x, trans == TRANSFER_FUNCTION_SRGB ? true:false);
+ coordinates_x,
+ trans == TRANSFER_FUNCTION_SRGB ?
+ gamma_type_index_2_4 : trans == TRANSFER_FUNCTION_GAMMA22 ?
+ gamma_type_index_2_2_flat : gamma_type_index_2_2);
for (i = 0; i <= MAX_HW_POINTS ; i++) {
points->red[i] = rgb_degamma[i].r;
points->green[i] = rgb_degamma[i].g;
diff --git a/drivers/gpu/drm/amd/display/modules/color/color_gamma.h b/drivers/gpu/drm/amd/display/modules/color/color_gamma.h
index a6e164df090a..369953fafadf 100644
--- a/drivers/gpu/drm/amd/display/modules/color/color_gamma.h
+++ b/drivers/gpu/drm/amd/display/modules/color/color_gamma.h
@@ -79,6 +79,7 @@ struct freesync_hdr_tf_params {
unsigned int max_content; // luminance in nits
unsigned int min_display; // luminance in 1/10000 nits
unsigned int max_display; // luminance in nits
+ unsigned int skip_tm; // skip tm
};
void setup_x_points_distribution(void);
diff --git a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
index 19b1eaebe484..7c20171a3b6d 100644
--- a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
+++ b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
@@ -23,6 +23,8 @@
*
*/
+#include <linux/slab.h>
+
#include "dm_services.h"
#include "dc.h"
#include "mod_freesync.h"
diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_shared.h b/drivers/gpu/drm/amd/display/modules/inc/mod_shared.h
index b711e7e6c204..b45f7d65e76a 100644
--- a/drivers/gpu/drm/amd/display/modules/inc/mod_shared.h
+++ b/drivers/gpu/drm/amd/display/modules/inc/mod_shared.h
@@ -45,5 +45,65 @@ enum vrr_packet_type {
PACKET_TYPE_VTEM
};
+#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
+union lut3d_control_flags {
+ unsigned int raw;
+ struct {
+ unsigned int do_chroma_scale :1;
+ unsigned int spec_version :3;
+ unsigned int use_zero_display_black :1;
+ unsigned int use_zero_source_black :1;
+ unsigned int force_display_black :6;
+ unsigned int apply_display_gamma :1;
+ unsigned int exp_shaper_max :6;
+ unsigned int unity_3dlut :1;
+ unsigned int bypass_3dlut :1;
+ unsigned int use_3dlut :1;
+ unsigned int less_than_dcip3 :1;
+ unsigned int override_lum :1;
+ unsigned int use_gamut_map_lib :1;
+ unsigned int chromatic_adaptation_src :1;
+ unsigned int chromatic_adaptation_dst :1;
+ unsigned int do_blender_lut_degamma :1;
+ unsigned int reseved :4;
+ } bits;
+};
+
+enum tm_show_option_internal {
+ tm_show_option_internal_single_file = 0,/*flags2 not in use*/
+ tm_show_option_internal_duplicate_file, /*use flags2*/
+ tm_show_option_internal_duplicate_sidebyside/*use flags2*/
+};
+
+enum lut3d_control_gamut_map {
+ lut3d_control_gamut_map_none = 0,
+ lut3d_control_gamut_map_tonemap,
+ lut3d_control_gamut_map_chto,
+ lut3d_control_gamut_map_chso,
+ lut3d_control_gamut_map_chci
+};
+
+enum lut3d_control_rotation_mode {
+ lut3d_control_rotation_mode_none = 0,
+ lut3d_control_rotation_mode_hue,
+ lut3d_control_rotation_mode_cc,
+ lut3d_control_rotation_mode_hue_cc
+};
+
+struct lut3d_settings {
+ unsigned char version;
+ union lut3d_control_flags flags;
+ union lut3d_control_flags flags2;
+ enum tm_show_option_internal option;
+ unsigned int min_lum;/*multiplied by 100*/
+ unsigned int max_lum;
+ unsigned int min_lum2;
+ unsigned int max_lum2;
+ enum lut3d_control_gamut_map map;
+ enum lut3d_control_rotation_mode rotation;
+ enum lut3d_control_gamut_map map2;
+ enum lut3d_control_rotation_mode rotation2;
+};
+#endif
#endif /* MOD_SHARED_H_ */
diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_vmid.h b/drivers/gpu/drm/amd/display/modules/inc/mod_vmid.h
new file mode 100644
index 000000000000..a3787fdf0c08
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/modules/inc/mod_vmid.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef MOD_VMID_H_
+#define MOD_VMID_H_
+
+#define MAX_VMID 16
+
+#include "dc.h"
+
+struct mod_vmid {
+ int dummy;
+};
+
+uint8_t mod_vmid_get_for_ptb(struct mod_vmid *mod_vmid, uint64_t ptb);
+void mod_vmid_reset(struct mod_vmid *mod_vmid);
+struct mod_vmid *mod_vmid_create(
+ struct dc *dc,
+ unsigned int num_vmid,
+ struct dc_virtual_addr_space_config *va_config);
+
+void mod_vmid_destroy(struct mod_vmid *mod_vmid);
+
+#endif /* MOD_VMID_H_ */
diff --git a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
index db06fab2ad5c..bc13c552797f 100644
--- a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
+++ b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
@@ -63,7 +63,9 @@ void mod_build_vsc_infopacket(const struct dc_stream_state *stream,
if (stream->psr_version != 0)
vscPacketRevision = 2;
- if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR420)
+ /* Update to revision 5 for extended colorimetry support for DPCD 1.4+ */
+ if (stream->link->dpcd_caps.dpcd_rev.raw >= 0x14 &&
+ stream->link->dpcd_caps.dprx_feature.bits.VSC_SDP_COLORIMETRY_SUPPORTED)
vscPacketRevision = 5;
/* VSC packet not needed based on the features
diff --git a/drivers/gpu/drm/amd/display/modules/power/Makefile b/drivers/gpu/drm/amd/display/modules/power/Makefile
index 87851f892a52..9d1b22d35ece 100644
--- a/drivers/gpu/drm/amd/display/modules/power/Makefile
+++ b/drivers/gpu/drm/amd/display/modules/power/Makefile
@@ -28,4 +28,4 @@ MOD_POWER = power_helpers.o
AMD_DAL_MOD_POWER = $(addprefix $(AMDDALPATH)/modules/power/,$(MOD_POWER))
#$(info ************ DAL POWER MODULE MAKEFILE ************)
-AMD_DISPLAY_FILES += $(AMD_DAL_MOD_POWER) \ No newline at end of file
+AMD_DISPLAY_FILES += $(AMD_DAL_MOD_POWER)
diff --git a/drivers/gpu/drm/amd/display/modules/vmid/vmid.c b/drivers/gpu/drm/amd/display/modules/vmid/vmid.c
new file mode 100644
index 000000000000..f0a153704f6e
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/modules/vmid/vmid.c
@@ -0,0 +1,167 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "mod_vmid.h"
+
+struct core_vmid {
+ struct mod_vmid public;
+ struct dc *dc;
+
+ unsigned int num_vmid;
+ unsigned int num_vmids_available;
+ uint64_t ptb_assigned_to_vmid[MAX_VMID];
+ struct dc_virtual_addr_space_config base_config;
+};
+
+#define MOD_VMID_TO_CORE(mod_vmid)\
+ container_of(mod_vmid, struct core_vmid, public)
+
+static void add_ptb_to_table(struct core_vmid *core_vmid, unsigned int vmid, uint64_t ptb)
+{
+ core_vmid->ptb_assigned_to_vmid[vmid] = ptb;
+ core_vmid->num_vmids_available--;
+}
+
+static void clear_entry_from_vmid_table(struct core_vmid *core_vmid, unsigned int vmid)
+{
+ core_vmid->ptb_assigned_to_vmid[vmid] = 0;
+ core_vmid->num_vmids_available++;
+}
+
+static void evict_vmids(struct core_vmid *core_vmid)
+{
+ int i;
+ uint16_t ord = dc_get_vmid_use_vector(core_vmid->dc);
+
+ // At this point any positions with value 0 are unused vmids, evict them
+ for (i = 1; i < core_vmid->num_vmid; i++) {
+ if (ord & (1u << i))
+ clear_entry_from_vmid_table(core_vmid, i);
+ }
+}
+
+// Return value of -1 indicates vmid table unitialized or ptb dne in the table
+static int get_existing_vmid_for_ptb(struct core_vmid *core_vmid, uint64_t ptb)
+{
+ int i;
+
+ for (i = 0; i < core_vmid->num_vmid; i++) {
+ if (core_vmid->ptb_assigned_to_vmid[i] == ptb)
+ return i;
+ }
+
+ return -1;
+}
+
+// Expected to be called only when there's an available vmid
+static int get_next_available_vmid(struct core_vmid *core_vmid)
+{
+ int i;
+
+ for (i = 1; i < core_vmid->num_vmid; i++) {
+ if (core_vmid->ptb_assigned_to_vmid[i] == 0)
+ return i;
+ }
+
+ return -1;
+}
+
+uint8_t mod_vmid_get_for_ptb(struct mod_vmid *mod_vmid, uint64_t ptb)
+{
+ struct core_vmid *core_vmid = MOD_VMID_TO_CORE(mod_vmid);
+ unsigned int vmid = 0;
+
+ // Physical address gets vmid 0
+ if (ptb == 0)
+ return 0;
+
+ vmid = get_existing_vmid_for_ptb(core_vmid, ptb);
+
+ if (vmid == -1) {
+ struct dc_virtual_addr_space_config va_config = core_vmid->base_config;
+
+ va_config.page_table_base_addr = ptb;
+
+ if (core_vmid->num_vmids_available == 0)
+ evict_vmids(core_vmid);
+
+ vmid = get_next_available_vmid(core_vmid);
+ add_ptb_to_table(core_vmid, vmid, ptb);
+
+ dc_setup_vm_context(core_vmid->dc, &va_config, vmid);
+ }
+
+ return vmid;
+}
+
+void mod_vmid_reset(struct mod_vmid *mod_vmid)
+{
+ struct core_vmid *core_vmid = MOD_VMID_TO_CORE(mod_vmid);
+
+ core_vmid->num_vmids_available = core_vmid->num_vmid - 1;
+ memset(core_vmid->ptb_assigned_to_vmid, 0, sizeof(core_vmid->ptb_assigned_to_vmid[0]) * MAX_VMID);
+}
+
+struct mod_vmid *mod_vmid_create(
+ struct dc *dc,
+ unsigned int num_vmid,
+ struct dc_virtual_addr_space_config *va_config)
+{
+ struct core_vmid *core_vmid;
+
+ if (num_vmid <= 1)
+ goto fail_no_vm_ctx;
+
+ if (dc == NULL)
+ goto fail_dc_null;
+
+ core_vmid = kzalloc(sizeof(struct core_vmid), GFP_KERNEL);
+
+ if (core_vmid == NULL)
+ goto fail_alloc_context;
+
+ core_vmid->dc = dc;
+ core_vmid->num_vmid = num_vmid;
+ core_vmid->num_vmids_available = num_vmid - 1;
+ core_vmid->base_config = *va_config;
+
+ memset(core_vmid->ptb_assigned_to_vmid, 0, sizeof(core_vmid->ptb_assigned_to_vmid[0]) * MAX_VMID);
+
+ return &core_vmid->public;
+
+fail_no_vm_ctx:
+fail_alloc_context:
+fail_dc_null:
+ return NULL;
+}
+
+void mod_vmid_destroy(struct mod_vmid *mod_vmid)
+{
+ if (mod_vmid != NULL) {
+ struct core_vmid *core_vmid = MOD_VMID_TO_CORE(mod_vmid);
+
+ kfree(core_vmid);
+ }
+}
diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
index 574bf6e70763..a0a7211438f2 100644
--- a/drivers/gpu/drm/amd/include/amd_shared.h
+++ b/drivers/gpu/drm/amd/include/amd_shared.h
@@ -26,7 +26,7 @@
#include <drm/amd_asic_type.h>
-#define AMD_MAX_USEC_TIMEOUT 200000 /* 200 ms */
+#define AMD_MAX_USEC_TIMEOUT 1000000 /* 1000 ms */
/*
* Chip flags
@@ -52,7 +52,8 @@ enum amd_ip_block_type {
AMD_IP_BLOCK_TYPE_UVD,
AMD_IP_BLOCK_TYPE_VCE,
AMD_IP_BLOCK_TYPE_ACP,
- AMD_IP_BLOCK_TYPE_VCN
+ AMD_IP_BLOCK_TYPE_VCN,
+ AMD_IP_BLOCK_TYPE_MES
};
enum amd_clockgating_state {
@@ -93,6 +94,11 @@ enum amd_powergating_state {
#define AMD_CG_SUPPORT_DRM_MGCG (1 << 22)
#define AMD_CG_SUPPORT_DF_MGCG (1 << 23)
#define AMD_CG_SUPPORT_VCN_MGCG (1 << 24)
+#define AMD_CG_SUPPORT_HDP_DS (1 << 25)
+#define AMD_CG_SUPPORT_HDP_SD (1 << 26)
+#define AMD_CG_SUPPORT_IH_CG (1 << 27)
+#define AMD_CG_SUPPORT_ATHUB_LS (1 << 28)
+#define AMD_CG_SUPPORT_ATHUB_MGCG (1 << 29)
/* PG flags */
#define AMD_PG_SUPPORT_GFX_PG (1 << 0)
#define AMD_PG_SUPPORT_GFX_SMG (1 << 1)
@@ -109,7 +115,8 @@ enum amd_powergating_state {
#define AMD_PG_SUPPORT_GFX_PIPELINE (1 << 12)
#define AMD_PG_SUPPORT_MMHUB (1 << 13)
#define AMD_PG_SUPPORT_VCN (1 << 14)
-#define AMD_PG_SUPPORT_VCN_DPG (1 << 15)
+#define AMD_PG_SUPPORT_VCN_DPG (1 << 15)
+#define AMD_PG_SUPPORT_ATHUB (1 << 16)
enum PP_FEATURE_MASK {
PP_SCLK_DPM_MASK = 0x1,
diff --git a/drivers/gpu/drm/amd/include/asic_reg/athub/athub_2_0_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/athub/athub_2_0_0_default.h
new file mode 100644
index 000000000000..5a998c75caf0
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/athub/athub_2_0_0_default.h
@@ -0,0 +1,272 @@
+/*
+ * Copyright (C) 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _athub_2_0_0_DEFAULT_HEADER
+#define _athub_2_0_0_DEFAULT_HEADER
+
+
+// addressBlock: athub_atsdec
+#define mmATC_ATS_CNTL_DEFAULT 0x009a0c00
+#define mmATC_ATS_STATUS_DEFAULT 0x00000000
+#define mmATC_ATS_FAULT_CNTL_DEFAULT 0x000001ff
+#define mmATC_ATS_FAULT_STATUS_INFO_DEFAULT 0x00000000
+#define mmATC_ATS_FAULT_STATUS_ADDR_DEFAULT 0x00000000
+#define mmATC_ATS_DEFAULT_PAGE_LOW_DEFAULT 0x00000000
+#define mmATC_TRANS_FAULT_RSPCNTRL_DEFAULT 0xffffffff
+#define mmATC_ATS_FAULT_STATUS_INFO2_DEFAULT 0x00000000
+#define mmATHUB_MISC_CNTL_DEFAULT 0x001c0200
+#define mmATC_VMID_PASID_MAPPING_UPDATE_STATUS_DEFAULT 0x00000000
+#define mmATC_VMID0_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID1_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID2_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID3_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID4_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID5_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID6_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID7_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID8_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID9_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID10_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID11_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID12_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID13_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID14_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID15_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_ATS_VMID_STATUS_DEFAULT 0x00000000
+#define mmATC_ATS_GFX_ATCL2_STATUS_DEFAULT 0x00000000
+#define mmATC_PERFCOUNTER0_CFG_DEFAULT 0x00000000
+#define mmATC_PERFCOUNTER1_CFG_DEFAULT 0x00000000
+#define mmATC_PERFCOUNTER2_CFG_DEFAULT 0x00000000
+#define mmATC_PERFCOUNTER3_CFG_DEFAULT 0x00000000
+#define mmATC_PERFCOUNTER_RSLT_CNTL_DEFAULT 0x04000000
+#define mmATC_PERFCOUNTER_LO_DEFAULT 0x00000000
+#define mmATC_PERFCOUNTER_HI_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_DEFAULT 0x00000000
+#define mmATHUB_PCIE_PASID_CNTL_DEFAULT 0x00000000
+#define mmATHUB_PCIE_PAGE_REQ_CNTL_DEFAULT 0x00000000
+#define mmATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC_DEFAULT 0x00000000
+#define mmATHUB_COMMAND_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_0_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_1_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_2_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_3_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_4_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_5_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_6_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_7_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_8_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_9_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_10_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_11_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_12_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_13_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_14_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_15_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_16_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_17_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_18_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_19_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_20_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_21_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_22_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_23_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_24_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_25_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_26_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_27_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_28_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_29_DEFAULT 0x00000000
+#define mmATHUB_PCIE_ATS_CNTL_VF_30_DEFAULT 0x00000000
+#define mmATHUB_MEM_POWER_LS_DEFAULT 0x00000208
+#define mmATS_IH_CREDIT_DEFAULT 0x00150002
+#define mmATHUB_IH_CREDIT_DEFAULT 0x00020002
+#define mmATC_VMID16_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID17_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID18_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID19_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID20_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID21_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID22_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID23_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID24_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID25_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID26_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID27_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID28_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID29_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID30_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_VMID31_PASID_MAPPING_DEFAULT 0x00000000
+#define mmATC_ATS_MMHUB_ATCL2_STATUS_DEFAULT 0x00000000
+#define mmATHUB_SHARED_VIRT_RESET_REQ_DEFAULT 0x00000000
+#define mmATHUB_SHARED_ACTIVE_FCN_ID_DEFAULT 0x00000000
+#define mmATC_ATS_SDPPORT_CNTL_DEFAULT 0x03ffa210
+#define mmATC_ATS_VMID_SNAPSHOT_GFX_STAT_DEFAULT 0x00000000
+#define mmATC_ATS_VMID_SNAPSHOT_MMHUB_STAT_DEFAULT 0x00000000
+
+
+// addressBlock: athub_xpbdec
+#define mmXPB_RTR_SRC_APRTR0_DEFAULT 0x00000000
+#define mmXPB_RTR_SRC_APRTR1_DEFAULT 0x00000000
+#define mmXPB_RTR_SRC_APRTR2_DEFAULT 0x00000000
+#define mmXPB_RTR_SRC_APRTR3_DEFAULT 0x00000000
+#define mmXPB_RTR_SRC_APRTR4_DEFAULT 0x00000000
+#define mmXPB_RTR_SRC_APRTR5_DEFAULT 0x00000000
+#define mmXPB_RTR_SRC_APRTR6_DEFAULT 0x00000000
+#define mmXPB_RTR_SRC_APRTR7_DEFAULT 0x00000000
+#define mmXPB_RTR_SRC_APRTR8_DEFAULT 0x00000000
+#define mmXPB_RTR_SRC_APRTR9_DEFAULT 0x00000000
+#define mmXPB_XDMA_RTR_SRC_APRTR0_DEFAULT 0x00000000
+#define mmXPB_XDMA_RTR_SRC_APRTR1_DEFAULT 0x00000000
+#define mmXPB_XDMA_RTR_SRC_APRTR2_DEFAULT 0x00000000
+#define mmXPB_XDMA_RTR_SRC_APRTR3_DEFAULT 0x00000000
+#define mmXPB_RTR_DEST_MAP0_DEFAULT 0x00000000
+#define mmXPB_RTR_DEST_MAP1_DEFAULT 0x00000000
+#define mmXPB_RTR_DEST_MAP2_DEFAULT 0x00000000
+#define mmXPB_RTR_DEST_MAP3_DEFAULT 0x00000000
+#define mmXPB_RTR_DEST_MAP4_DEFAULT 0x00000000
+#define mmXPB_RTR_DEST_MAP5_DEFAULT 0x00000000
+#define mmXPB_RTR_DEST_MAP6_DEFAULT 0x00000000
+#define mmXPB_RTR_DEST_MAP7_DEFAULT 0x00000000
+#define mmXPB_RTR_DEST_MAP8_DEFAULT 0x00000000
+#define mmXPB_RTR_DEST_MAP9_DEFAULT 0x00000000
+#define mmXPB_XDMA_RTR_DEST_MAP0_DEFAULT 0x00000000
+#define mmXPB_XDMA_RTR_DEST_MAP1_DEFAULT 0x00000000
+#define mmXPB_XDMA_RTR_DEST_MAP2_DEFAULT 0x00000000
+#define mmXPB_XDMA_RTR_DEST_MAP3_DEFAULT 0x00000000
+#define mmXPB_CLG_CFG0_DEFAULT 0x00000000
+#define mmXPB_CLG_CFG1_DEFAULT 0x00000000
+#define mmXPB_CLG_CFG2_DEFAULT 0x00000000
+#define mmXPB_CLG_CFG3_DEFAULT 0x00000000
+#define mmXPB_CLG_CFG4_DEFAULT 0x00000000
+#define mmXPB_CLG_CFG5_DEFAULT 0x00000000
+#define mmXPB_CLG_CFG6_DEFAULT 0x00000000
+#define mmXPB_CLG_CFG7_DEFAULT 0x00000000
+#define mmXPB_CLG_EXTRA_DEFAULT 0x00000000
+#define mmXPB_CLG_EXTRA_MSK_DEFAULT 0x00000000
+#define mmXPB_LB_ADDR_DEFAULT 0x00000000
+#define mmXPB_WCB_STS_DEFAULT 0x00000000
+#define mmXPB_HST_CFG_DEFAULT 0x00000000
+#define mmXPB_P2P_BAR_CFG_DEFAULT 0x0000000f
+#define mmXPB_P2P_BAR0_DEFAULT 0x00000000
+#define mmXPB_P2P_BAR1_DEFAULT 0x00000000
+#define mmXPB_P2P_BAR2_DEFAULT 0x00000000
+#define mmXPB_P2P_BAR3_DEFAULT 0x00000000
+#define mmXPB_P2P_BAR4_DEFAULT 0x00000000
+#define mmXPB_P2P_BAR5_DEFAULT 0x00000000
+#define mmXPB_P2P_BAR6_DEFAULT 0x00000000
+#define mmXPB_P2P_BAR7_DEFAULT 0x00000000
+#define mmXPB_P2P_BAR_SETUP_DEFAULT 0x00000000
+#define mmXPB_P2P_BAR_DELTA_ABOVE_DEFAULT 0x00000000
+#define mmXPB_P2P_BAR_DELTA_BELOW_DEFAULT 0x00000000
+#define mmXPB_PEER_SYS_BAR0_DEFAULT 0x00000000
+#define mmXPB_PEER_SYS_BAR1_DEFAULT 0x00000000
+#define mmXPB_PEER_SYS_BAR2_DEFAULT 0x00000000
+#define mmXPB_PEER_SYS_BAR3_DEFAULT 0x00000000
+#define mmXPB_PEER_SYS_BAR4_DEFAULT 0x00000000
+#define mmXPB_PEER_SYS_BAR5_DEFAULT 0x00000000
+#define mmXPB_PEER_SYS_BAR6_DEFAULT 0x00000000
+#define mmXPB_PEER_SYS_BAR7_DEFAULT 0x00000000
+#define mmXPB_PEER_SYS_BAR8_DEFAULT 0x00000000
+#define mmXPB_PEER_SYS_BAR9_DEFAULT 0x00000000
+#define mmXPB_XDMA_PEER_SYS_BAR0_DEFAULT 0x00000000
+#define mmXPB_XDMA_PEER_SYS_BAR1_DEFAULT 0x00000000
+#define mmXPB_XDMA_PEER_SYS_BAR2_DEFAULT 0x00000000
+#define mmXPB_XDMA_PEER_SYS_BAR3_DEFAULT 0x00000000
+#define mmXPB_CLK_GAT_DEFAULT 0x00040400
+#define mmXPB_INTF_CFG_DEFAULT 0x000f1040
+#define mmXPB_INTF_STS_DEFAULT 0x00000000
+#define mmXPB_PIPE_STS_DEFAULT 0x00000000
+#define mmXPB_SUB_CTRL_DEFAULT 0x00000000
+#define mmXPB_MAP_INVERT_FLUSH_NUM_LSB_DEFAULT 0x00000000
+#define mmXPB_PERF_KNOBS_DEFAULT 0x00000000
+#define mmXPB_STICKY_DEFAULT 0x00000000
+#define mmXPB_STICKY_W1C_DEFAULT 0x00000000
+#define mmXPB_MISC_CFG_DEFAULT 0x4d585042
+#define mmXPB_INTF_CFG2_DEFAULT 0x00000040
+#define mmXPB_CLG_EXTRA_RD_DEFAULT 0x00000000
+#define mmXPB_CLG_EXTRA_MSK_RD_DEFAULT 0x00000000
+#define mmXPB_CLG_GFX_MATCH_DEFAULT 0x03000000
+#define mmXPB_CLG_GFX_MATCH_MSK_DEFAULT 0x003cf3cf
+#define mmXPB_CLG_MM_MATCH_DEFAULT 0x00003000
+#define mmXPB_CLG_MM_MATCH_MSK_DEFAULT 0x00000000
+#define mmXPB_CLG_GUS_MATCH_DEFAULT 0x00000040
+#define mmXPB_CLG_GUS_MATCH_MSK_DEFAULT 0x00000000
+#define mmXPB_CLG_GFX_UNITID_MAPPING0_DEFAULT 0x00000000
+#define mmXPB_CLG_GFX_UNITID_MAPPING1_DEFAULT 0x00000040
+#define mmXPB_CLG_GFX_UNITID_MAPPING2_DEFAULT 0x00000080
+#define mmXPB_CLG_GFX_UNITID_MAPPING3_DEFAULT 0x000000c0
+#define mmXPB_CLG_GFX_UNITID_MAPPING4_DEFAULT 0x00000100
+#define mmXPB_CLG_GFX_UNITID_MAPPING5_DEFAULT 0x00000140
+#define mmXPB_CLG_GFX_UNITID_MAPPING6_DEFAULT 0x00000000
+#define mmXPB_CLG_GFX_UNITID_MAPPING7_DEFAULT 0x000001c0
+#define mmXPB_CLG_MM_UNITID_MAPPING0_DEFAULT 0x00000000
+#define mmXPB_CLG_MM_UNITID_MAPPING1_DEFAULT 0x00000040
+#define mmXPB_CLG_MM_UNITID_MAPPING2_DEFAULT 0x00000080
+#define mmXPB_CLG_MM_UNITID_MAPPING3_DEFAULT 0x000000c0
+#define mmXPB_CLG_GUS_UNITID_MAPPING0_DEFAULT 0x00000000
+#define mmXPB_CLG_GUS_UNITID_MAPPING1_DEFAULT 0x00000040
+#define mmXPB_CLG_GUS_UNITID_MAPPING2_DEFAULT 0x00000080
+#define mmXPB_CLG_GUS_UNITID_MAPPING3_DEFAULT 0x000000c0
+#define mmXPB_CLG_GUS_UNITID_MAPPING4_DEFAULT 0x00000100
+#define mmXPB_CLG_GUS_UNITID_MAPPING5_DEFAULT 0x00000140
+#define mmXPB_CLG_GUS_UNITID_MAPPING6_DEFAULT 0x00000180
+#define mmXPB_CLG_GUS_UNITID_MAPPING7_DEFAULT 0x000001c0
+
+
+// addressBlock: athub_rpbdec
+#define mmRPB_PASSPW_CONF_DEFAULT 0x00000230
+#define mmRPB_BLOCKLEVEL_CONF_DEFAULT 0x000000f0
+#define mmRPB_TAG_CONF_DEFAULT 0x08040080
+#define mmRPB_EFF_CNTL_DEFAULT 0x00001010
+#define mmRPB_ARB_CNTL_DEFAULT 0x00040404
+#define mmRPB_ARB_CNTL2_DEFAULT 0x00040104
+#define mmRPB_BIF_CNTL_DEFAULT 0x01000404
+#define mmRPB_WR_SWITCH_CNTL_DEFAULT 0x02040810
+#define mmRPB_WR_COMBINE_CNTL_DEFAULT 0x00000013
+#define mmRPB_RD_SWITCH_CNTL_DEFAULT 0x02040810
+#define mmRPB_CID_QUEUE_WR_DEFAULT 0x00000000
+#define mmRPB_CID_QUEUE_RD_DEFAULT 0x00000000
+#define mmRPB_PERF_COUNTER_CNTL_DEFAULT 0x00000010
+#define mmRPB_PERF_COUNTER_STATUS_DEFAULT 0x00000000
+#define mmRPB_CID_QUEUE_EX_DEFAULT 0x00000000
+#define mmRPB_CID_QUEUE_EX_DATA_DEFAULT 0x00000000
+#define mmRPB_SWITCH_CNTL2_DEFAULT 0x02040810
+#define mmRPB_DEINTRLV_COMBINE_CNTL_DEFAULT 0x00000204
+#define mmRPB_VC_SWITCH_RDWR_DEFAULT 0x00204040
+#define mmRPB_PERFCOUNTER_LO_DEFAULT 0x00000000
+#define mmRPB_PERFCOUNTER_HI_DEFAULT 0x00000000
+#define mmRPB_PERFCOUNTER0_CFG_DEFAULT 0x00000000
+#define mmRPB_PERFCOUNTER1_CFG_DEFAULT 0x00000000
+#define mmRPB_PERFCOUNTER2_CFG_DEFAULT 0x00000000
+#define mmRPB_PERFCOUNTER3_CFG_DEFAULT 0x00000000
+#define mmRPB_PERFCOUNTER_RSLT_CNTL_DEFAULT 0x04000000
+#define mmRPB_BIF_CNTL2_DEFAULT 0x00000000
+#define mmRPB_RD_QUEUE_CNTL_DEFAULT 0x00000000
+#define mmRPB_RD_QUEUE_CNTL2_DEFAULT 0x00000000
+#define mmRPB_WR_QUEUE_CNTL_DEFAULT 0x00000000
+#define mmRPB_WR_QUEUE_CNTL2_DEFAULT 0x00000000
+#define mmRPB_EA_QUEUE_WR_DEFAULT 0x00000000
+#define mmRPB_ATS_CNTL_DEFAULT 0x58088422
+#define mmRPB_ATS_CNTL2_DEFAULT 0x00050b13
+#define mmRPB_DF_SDPPORT_CNTL_DEFAULT 0x00003820
+#define mmRPB_SDPPORT_CNTL_DEFAULT 0x0fd14010
+#define mmRPB_NBIF_SDPPORT_CNTL_DEFAULT 0x08084020
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/athub/athub_2_0_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/athub/athub_2_0_0_offset.h
new file mode 100644
index 000000000000..f86def814430
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/athub/athub_2_0_0_offset.h
@@ -0,0 +1,514 @@
+/*
+ * Copyright (C) 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _athub_2_0_0_OFFSET_HEADER
+#define _athub_2_0_0_OFFSET_HEADER
+
+
+// addressBlock: athub_atsdec
+// base address: 0x3000
+#define mmATC_ATS_CNTL 0x0000
+#define mmATC_ATS_CNTL_BASE_IDX 0
+#define mmATC_ATS_STATUS 0x0003
+#define mmATC_ATS_STATUS_BASE_IDX 0
+#define mmATC_ATS_FAULT_CNTL 0x0004
+#define mmATC_ATS_FAULT_CNTL_BASE_IDX 0
+#define mmATC_ATS_FAULT_STATUS_INFO 0x0005
+#define mmATC_ATS_FAULT_STATUS_INFO_BASE_IDX 0
+#define mmATC_ATS_FAULT_STATUS_ADDR 0x0006
+#define mmATC_ATS_FAULT_STATUS_ADDR_BASE_IDX 0
+#define mmATC_ATS_DEFAULT_PAGE_LOW 0x0007
+#define mmATC_ATS_DEFAULT_PAGE_LOW_BASE_IDX 0
+#define mmATC_TRANS_FAULT_RSPCNTRL 0x0008
+#define mmATC_TRANS_FAULT_RSPCNTRL_BASE_IDX 0
+#define mmATC_ATS_FAULT_STATUS_INFO2 0x0009
+#define mmATC_ATS_FAULT_STATUS_INFO2_BASE_IDX 0
+#define mmATHUB_MISC_CNTL 0x000a
+#define mmATHUB_MISC_CNTL_BASE_IDX 0
+#define mmATC_VMID_PASID_MAPPING_UPDATE_STATUS 0x000b
+#define mmATC_VMID_PASID_MAPPING_UPDATE_STATUS_BASE_IDX 0
+#define mmATC_VMID0_PASID_MAPPING 0x000c
+#define mmATC_VMID0_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID1_PASID_MAPPING 0x000d
+#define mmATC_VMID1_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID2_PASID_MAPPING 0x000e
+#define mmATC_VMID2_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID3_PASID_MAPPING 0x000f
+#define mmATC_VMID3_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID4_PASID_MAPPING 0x0010
+#define mmATC_VMID4_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID5_PASID_MAPPING 0x0011
+#define mmATC_VMID5_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID6_PASID_MAPPING 0x0012
+#define mmATC_VMID6_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID7_PASID_MAPPING 0x0013
+#define mmATC_VMID7_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID8_PASID_MAPPING 0x0014
+#define mmATC_VMID8_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID9_PASID_MAPPING 0x0015
+#define mmATC_VMID9_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID10_PASID_MAPPING 0x0016
+#define mmATC_VMID10_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID11_PASID_MAPPING 0x0017
+#define mmATC_VMID11_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID12_PASID_MAPPING 0x0018
+#define mmATC_VMID12_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID13_PASID_MAPPING 0x0019
+#define mmATC_VMID13_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID14_PASID_MAPPING 0x001a
+#define mmATC_VMID14_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID15_PASID_MAPPING 0x001b
+#define mmATC_VMID15_PASID_MAPPING_BASE_IDX 0
+#define mmATC_ATS_VMID_STATUS 0x001c
+#define mmATC_ATS_VMID_STATUS_BASE_IDX 0
+#define mmATC_ATS_GFX_ATCL2_STATUS 0x001d
+#define mmATC_ATS_GFX_ATCL2_STATUS_BASE_IDX 0
+#define mmATC_PERFCOUNTER0_CFG 0x001e
+#define mmATC_PERFCOUNTER0_CFG_BASE_IDX 0
+#define mmATC_PERFCOUNTER1_CFG 0x001f
+#define mmATC_PERFCOUNTER1_CFG_BASE_IDX 0
+#define mmATC_PERFCOUNTER2_CFG 0x0020
+#define mmATC_PERFCOUNTER2_CFG_BASE_IDX 0
+#define mmATC_PERFCOUNTER3_CFG 0x0021
+#define mmATC_PERFCOUNTER3_CFG_BASE_IDX 0
+#define mmATC_PERFCOUNTER_RSLT_CNTL 0x0022
+#define mmATC_PERFCOUNTER_RSLT_CNTL_BASE_IDX 0
+#define mmATC_PERFCOUNTER_LO 0x0023
+#define mmATC_PERFCOUNTER_LO_BASE_IDX 0
+#define mmATC_PERFCOUNTER_HI 0x0024
+#define mmATC_PERFCOUNTER_HI_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL 0x0025
+#define mmATHUB_PCIE_ATS_CNTL_BASE_IDX 0
+#define mmATHUB_PCIE_PASID_CNTL 0x0026
+#define mmATHUB_PCIE_PASID_CNTL_BASE_IDX 0
+#define mmATHUB_PCIE_PAGE_REQ_CNTL 0x0027
+#define mmATHUB_PCIE_PAGE_REQ_CNTL_BASE_IDX 0
+#define mmATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC 0x0028
+#define mmATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC_BASE_IDX 0
+#define mmATHUB_COMMAND 0x0029
+#define mmATHUB_COMMAND_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_0 0x002a
+#define mmATHUB_PCIE_ATS_CNTL_VF_0_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_1 0x002b
+#define mmATHUB_PCIE_ATS_CNTL_VF_1_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_2 0x002c
+#define mmATHUB_PCIE_ATS_CNTL_VF_2_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_3 0x002d
+#define mmATHUB_PCIE_ATS_CNTL_VF_3_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_4 0x002e
+#define mmATHUB_PCIE_ATS_CNTL_VF_4_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_5 0x002f
+#define mmATHUB_PCIE_ATS_CNTL_VF_5_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_6 0x0030
+#define mmATHUB_PCIE_ATS_CNTL_VF_6_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_7 0x0031
+#define mmATHUB_PCIE_ATS_CNTL_VF_7_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_8 0x0032
+#define mmATHUB_PCIE_ATS_CNTL_VF_8_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_9 0x0033
+#define mmATHUB_PCIE_ATS_CNTL_VF_9_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_10 0x0034
+#define mmATHUB_PCIE_ATS_CNTL_VF_10_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_11 0x0035
+#define mmATHUB_PCIE_ATS_CNTL_VF_11_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_12 0x0036
+#define mmATHUB_PCIE_ATS_CNTL_VF_12_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_13 0x0037
+#define mmATHUB_PCIE_ATS_CNTL_VF_13_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_14 0x0038
+#define mmATHUB_PCIE_ATS_CNTL_VF_14_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_15 0x0039
+#define mmATHUB_PCIE_ATS_CNTL_VF_15_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_16 0x003a
+#define mmATHUB_PCIE_ATS_CNTL_VF_16_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_17 0x003b
+#define mmATHUB_PCIE_ATS_CNTL_VF_17_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_18 0x003c
+#define mmATHUB_PCIE_ATS_CNTL_VF_18_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_19 0x003d
+#define mmATHUB_PCIE_ATS_CNTL_VF_19_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_20 0x003e
+#define mmATHUB_PCIE_ATS_CNTL_VF_20_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_21 0x003f
+#define mmATHUB_PCIE_ATS_CNTL_VF_21_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_22 0x0040
+#define mmATHUB_PCIE_ATS_CNTL_VF_22_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_23 0x0041
+#define mmATHUB_PCIE_ATS_CNTL_VF_23_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_24 0x0042
+#define mmATHUB_PCIE_ATS_CNTL_VF_24_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_25 0x0043
+#define mmATHUB_PCIE_ATS_CNTL_VF_25_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_26 0x0044
+#define mmATHUB_PCIE_ATS_CNTL_VF_26_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_27 0x0045
+#define mmATHUB_PCIE_ATS_CNTL_VF_27_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_28 0x0046
+#define mmATHUB_PCIE_ATS_CNTL_VF_28_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_29 0x0047
+#define mmATHUB_PCIE_ATS_CNTL_VF_29_BASE_IDX 0
+#define mmATHUB_PCIE_ATS_CNTL_VF_30 0x0048
+#define mmATHUB_PCIE_ATS_CNTL_VF_30_BASE_IDX 0
+#define mmATHUB_MEM_POWER_LS 0x0049
+#define mmATHUB_MEM_POWER_LS_BASE_IDX 0
+#define mmATS_IH_CREDIT 0x004a
+#define mmATS_IH_CREDIT_BASE_IDX 0
+#define mmATHUB_IH_CREDIT 0x004b
+#define mmATHUB_IH_CREDIT_BASE_IDX 0
+#define mmATC_VMID16_PASID_MAPPING 0x004c
+#define mmATC_VMID16_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID17_PASID_MAPPING 0x004d
+#define mmATC_VMID17_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID18_PASID_MAPPING 0x004e
+#define mmATC_VMID18_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID19_PASID_MAPPING 0x004f
+#define mmATC_VMID19_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID20_PASID_MAPPING 0x0050
+#define mmATC_VMID20_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID21_PASID_MAPPING 0x0051
+#define mmATC_VMID21_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID22_PASID_MAPPING 0x0052
+#define mmATC_VMID22_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID23_PASID_MAPPING 0x0053
+#define mmATC_VMID23_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID24_PASID_MAPPING 0x0054
+#define mmATC_VMID24_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID25_PASID_MAPPING 0x0055
+#define mmATC_VMID25_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID26_PASID_MAPPING 0x0056
+#define mmATC_VMID26_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID27_PASID_MAPPING 0x0057
+#define mmATC_VMID27_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID28_PASID_MAPPING 0x0058
+#define mmATC_VMID28_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID29_PASID_MAPPING 0x0059
+#define mmATC_VMID29_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID30_PASID_MAPPING 0x005a
+#define mmATC_VMID30_PASID_MAPPING_BASE_IDX 0
+#define mmATC_VMID31_PASID_MAPPING 0x005b
+#define mmATC_VMID31_PASID_MAPPING_BASE_IDX 0
+#define mmATC_ATS_MMHUB_ATCL2_STATUS 0x005c
+#define mmATC_ATS_MMHUB_ATCL2_STATUS_BASE_IDX 0
+#define mmATHUB_SHARED_VIRT_RESET_REQ 0x005d
+#define mmATHUB_SHARED_VIRT_RESET_REQ_BASE_IDX 0
+#define mmATHUB_SHARED_ACTIVE_FCN_ID 0x005e
+#define mmATHUB_SHARED_ACTIVE_FCN_ID_BASE_IDX 0
+#define mmATC_ATS_SDPPORT_CNTL 0x005f
+#define mmATC_ATS_SDPPORT_CNTL_BASE_IDX 0
+#define mmATC_ATS_VMID_SNAPSHOT_GFX_STAT 0x0061
+#define mmATC_ATS_VMID_SNAPSHOT_GFX_STAT_BASE_IDX 0
+#define mmATC_ATS_VMID_SNAPSHOT_MMHUB_STAT 0x0062
+#define mmATC_ATS_VMID_SNAPSHOT_MMHUB_STAT_BASE_IDX 0
+
+
+// addressBlock: athub_xpbdec
+// base address: 0x3190
+#define mmXPB_RTR_SRC_APRTR0 0x0064
+#define mmXPB_RTR_SRC_APRTR0_BASE_IDX 0
+#define mmXPB_RTR_SRC_APRTR1 0x0065
+#define mmXPB_RTR_SRC_APRTR1_BASE_IDX 0
+#define mmXPB_RTR_SRC_APRTR2 0x0066
+#define mmXPB_RTR_SRC_APRTR2_BASE_IDX 0
+#define mmXPB_RTR_SRC_APRTR3 0x0067
+#define mmXPB_RTR_SRC_APRTR3_BASE_IDX 0
+#define mmXPB_RTR_SRC_APRTR4 0x0068
+#define mmXPB_RTR_SRC_APRTR4_BASE_IDX 0
+#define mmXPB_RTR_SRC_APRTR5 0x0069
+#define mmXPB_RTR_SRC_APRTR5_BASE_IDX 0
+#define mmXPB_RTR_SRC_APRTR6 0x006a
+#define mmXPB_RTR_SRC_APRTR6_BASE_IDX 0
+#define mmXPB_RTR_SRC_APRTR7 0x006b
+#define mmXPB_RTR_SRC_APRTR7_BASE_IDX 0
+#define mmXPB_RTR_SRC_APRTR8 0x006c
+#define mmXPB_RTR_SRC_APRTR8_BASE_IDX 0
+#define mmXPB_RTR_SRC_APRTR9 0x006d
+#define mmXPB_RTR_SRC_APRTR9_BASE_IDX 0
+#define mmXPB_XDMA_RTR_SRC_APRTR0 0x006e
+#define mmXPB_XDMA_RTR_SRC_APRTR0_BASE_IDX 0
+#define mmXPB_XDMA_RTR_SRC_APRTR1 0x006f
+#define mmXPB_XDMA_RTR_SRC_APRTR1_BASE_IDX 0
+#define mmXPB_XDMA_RTR_SRC_APRTR2 0x0070
+#define mmXPB_XDMA_RTR_SRC_APRTR2_BASE_IDX 0
+#define mmXPB_XDMA_RTR_SRC_APRTR3 0x0071
+#define mmXPB_XDMA_RTR_SRC_APRTR3_BASE_IDX 0
+#define mmXPB_RTR_DEST_MAP0 0x0072
+#define mmXPB_RTR_DEST_MAP0_BASE_IDX 0
+#define mmXPB_RTR_DEST_MAP1 0x0073
+#define mmXPB_RTR_DEST_MAP1_BASE_IDX 0
+#define mmXPB_RTR_DEST_MAP2 0x0074
+#define mmXPB_RTR_DEST_MAP2_BASE_IDX 0
+#define mmXPB_RTR_DEST_MAP3 0x0075
+#define mmXPB_RTR_DEST_MAP3_BASE_IDX 0
+#define mmXPB_RTR_DEST_MAP4 0x0076
+#define mmXPB_RTR_DEST_MAP4_BASE_IDX 0
+#define mmXPB_RTR_DEST_MAP5 0x0077
+#define mmXPB_RTR_DEST_MAP5_BASE_IDX 0
+#define mmXPB_RTR_DEST_MAP6 0x0078
+#define mmXPB_RTR_DEST_MAP6_BASE_IDX 0
+#define mmXPB_RTR_DEST_MAP7 0x0079
+#define mmXPB_RTR_DEST_MAP7_BASE_IDX 0
+#define mmXPB_RTR_DEST_MAP8 0x007a
+#define mmXPB_RTR_DEST_MAP8_BASE_IDX 0
+#define mmXPB_RTR_DEST_MAP9 0x007b
+#define mmXPB_RTR_DEST_MAP9_BASE_IDX 0
+#define mmXPB_XDMA_RTR_DEST_MAP0 0x007c
+#define mmXPB_XDMA_RTR_DEST_MAP0_BASE_IDX 0
+#define mmXPB_XDMA_RTR_DEST_MAP1 0x007d
+#define mmXPB_XDMA_RTR_DEST_MAP1_BASE_IDX 0
+#define mmXPB_XDMA_RTR_DEST_MAP2 0x007e
+#define mmXPB_XDMA_RTR_DEST_MAP2_BASE_IDX 0
+#define mmXPB_XDMA_RTR_DEST_MAP3 0x007f
+#define mmXPB_XDMA_RTR_DEST_MAP3_BASE_IDX 0
+#define mmXPB_CLG_CFG0 0x0080
+#define mmXPB_CLG_CFG0_BASE_IDX 0
+#define mmXPB_CLG_CFG1 0x0081
+#define mmXPB_CLG_CFG1_BASE_IDX 0
+#define mmXPB_CLG_CFG2 0x0082
+#define mmXPB_CLG_CFG2_BASE_IDX 0
+#define mmXPB_CLG_CFG3 0x0083
+#define mmXPB_CLG_CFG3_BASE_IDX 0
+#define mmXPB_CLG_CFG4 0x0084
+#define mmXPB_CLG_CFG4_BASE_IDX 0
+#define mmXPB_CLG_CFG5 0x0085
+#define mmXPB_CLG_CFG5_BASE_IDX 0
+#define mmXPB_CLG_CFG6 0x0086
+#define mmXPB_CLG_CFG6_BASE_IDX 0
+#define mmXPB_CLG_CFG7 0x0087
+#define mmXPB_CLG_CFG7_BASE_IDX 0
+#define mmXPB_CLG_EXTRA 0x0088
+#define mmXPB_CLG_EXTRA_BASE_IDX 0
+#define mmXPB_CLG_EXTRA_MSK 0x0089
+#define mmXPB_CLG_EXTRA_MSK_BASE_IDX 0
+#define mmXPB_LB_ADDR 0x008a
+#define mmXPB_LB_ADDR_BASE_IDX 0
+#define mmXPB_WCB_STS 0x008b
+#define mmXPB_WCB_STS_BASE_IDX 0
+#define mmXPB_HST_CFG 0x008c
+#define mmXPB_HST_CFG_BASE_IDX 0
+#define mmXPB_P2P_BAR_CFG 0x008d
+#define mmXPB_P2P_BAR_CFG_BASE_IDX 0
+#define mmXPB_P2P_BAR0 0x008e
+#define mmXPB_P2P_BAR0_BASE_IDX 0
+#define mmXPB_P2P_BAR1 0x008f
+#define mmXPB_P2P_BAR1_BASE_IDX 0
+#define mmXPB_P2P_BAR2 0x0090
+#define mmXPB_P2P_BAR2_BASE_IDX 0
+#define mmXPB_P2P_BAR3 0x0091
+#define mmXPB_P2P_BAR3_BASE_IDX 0
+#define mmXPB_P2P_BAR4 0x0092
+#define mmXPB_P2P_BAR4_BASE_IDX 0
+#define mmXPB_P2P_BAR5 0x0093
+#define mmXPB_P2P_BAR5_BASE_IDX 0
+#define mmXPB_P2P_BAR6 0x0094
+#define mmXPB_P2P_BAR6_BASE_IDX 0
+#define mmXPB_P2P_BAR7 0x0095
+#define mmXPB_P2P_BAR7_BASE_IDX 0
+#define mmXPB_P2P_BAR_SETUP 0x0096
+#define mmXPB_P2P_BAR_SETUP_BASE_IDX 0
+#define mmXPB_P2P_BAR_DELTA_ABOVE 0x0098
+#define mmXPB_P2P_BAR_DELTA_ABOVE_BASE_IDX 0
+#define mmXPB_P2P_BAR_DELTA_BELOW 0x0099
+#define mmXPB_P2P_BAR_DELTA_BELOW_BASE_IDX 0
+#define mmXPB_PEER_SYS_BAR0 0x009a
+#define mmXPB_PEER_SYS_BAR0_BASE_IDX 0
+#define mmXPB_PEER_SYS_BAR1 0x009b
+#define mmXPB_PEER_SYS_BAR1_BASE_IDX 0
+#define mmXPB_PEER_SYS_BAR2 0x009c
+#define mmXPB_PEER_SYS_BAR2_BASE_IDX 0
+#define mmXPB_PEER_SYS_BAR3 0x009d
+#define mmXPB_PEER_SYS_BAR3_BASE_IDX 0
+#define mmXPB_PEER_SYS_BAR4 0x009e
+#define mmXPB_PEER_SYS_BAR4_BASE_IDX 0
+#define mmXPB_PEER_SYS_BAR5 0x009f
+#define mmXPB_PEER_SYS_BAR5_BASE_IDX 0
+#define mmXPB_PEER_SYS_BAR6 0x00a0
+#define mmXPB_PEER_SYS_BAR6_BASE_IDX 0
+#define mmXPB_PEER_SYS_BAR7 0x00a1
+#define mmXPB_PEER_SYS_BAR7_BASE_IDX 0
+#define mmXPB_PEER_SYS_BAR8 0x00a2
+#define mmXPB_PEER_SYS_BAR8_BASE_IDX 0
+#define mmXPB_PEER_SYS_BAR9 0x00a3
+#define mmXPB_PEER_SYS_BAR9_BASE_IDX 0
+#define mmXPB_XDMA_PEER_SYS_BAR0 0x00a4
+#define mmXPB_XDMA_PEER_SYS_BAR0_BASE_IDX 0
+#define mmXPB_XDMA_PEER_SYS_BAR1 0x00a5
+#define mmXPB_XDMA_PEER_SYS_BAR1_BASE_IDX 0
+#define mmXPB_XDMA_PEER_SYS_BAR2 0x00a6
+#define mmXPB_XDMA_PEER_SYS_BAR2_BASE_IDX 0
+#define mmXPB_XDMA_PEER_SYS_BAR3 0x00a7
+#define mmXPB_XDMA_PEER_SYS_BAR3_BASE_IDX 0
+#define mmXPB_CLK_GAT 0x00a8
+#define mmXPB_CLK_GAT_BASE_IDX 0
+#define mmXPB_INTF_CFG 0x00a9
+#define mmXPB_INTF_CFG_BASE_IDX 0
+#define mmXPB_INTF_STS 0x00aa
+#define mmXPB_INTF_STS_BASE_IDX 0
+#define mmXPB_PIPE_STS 0x00ab
+#define mmXPB_PIPE_STS_BASE_IDX 0
+#define mmXPB_SUB_CTRL 0x00ac
+#define mmXPB_SUB_CTRL_BASE_IDX 0
+#define mmXPB_MAP_INVERT_FLUSH_NUM_LSB 0x00ad
+#define mmXPB_MAP_INVERT_FLUSH_NUM_LSB_BASE_IDX 0
+#define mmXPB_PERF_KNOBS 0x00ae
+#define mmXPB_PERF_KNOBS_BASE_IDX 0
+#define mmXPB_STICKY 0x00af
+#define mmXPB_STICKY_BASE_IDX 0
+#define mmXPB_STICKY_W1C 0x00b0
+#define mmXPB_STICKY_W1C_BASE_IDX 0
+#define mmXPB_MISC_CFG 0x00b1
+#define mmXPB_MISC_CFG_BASE_IDX 0
+#define mmXPB_INTF_CFG2 0x00b2
+#define mmXPB_INTF_CFG2_BASE_IDX 0
+#define mmXPB_CLG_EXTRA_RD 0x00b3
+#define mmXPB_CLG_EXTRA_RD_BASE_IDX 0
+#define mmXPB_CLG_EXTRA_MSK_RD 0x00b4
+#define mmXPB_CLG_EXTRA_MSK_RD_BASE_IDX 0
+#define mmXPB_CLG_GFX_MATCH 0x00b5
+#define mmXPB_CLG_GFX_MATCH_BASE_IDX 0
+#define mmXPB_CLG_GFX_MATCH_MSK 0x00b6
+#define mmXPB_CLG_GFX_MATCH_MSK_BASE_IDX 0
+#define mmXPB_CLG_MM_MATCH 0x00b7
+#define mmXPB_CLG_MM_MATCH_BASE_IDX 0
+#define mmXPB_CLG_MM_MATCH_MSK 0x00b8
+#define mmXPB_CLG_MM_MATCH_MSK_BASE_IDX 0
+#define mmXPB_CLG_GUS_MATCH 0x00b9
+#define mmXPB_CLG_GUS_MATCH_BASE_IDX 0
+#define mmXPB_CLG_GUS_MATCH_MSK 0x00ba
+#define mmXPB_CLG_GUS_MATCH_MSK_BASE_IDX 0
+#define mmXPB_CLG_GFX_UNITID_MAPPING0 0x00bb
+#define mmXPB_CLG_GFX_UNITID_MAPPING0_BASE_IDX 0
+#define mmXPB_CLG_GFX_UNITID_MAPPING1 0x00bc
+#define mmXPB_CLG_GFX_UNITID_MAPPING1_BASE_IDX 0
+#define mmXPB_CLG_GFX_UNITID_MAPPING2 0x00bd
+#define mmXPB_CLG_GFX_UNITID_MAPPING2_BASE_IDX 0
+#define mmXPB_CLG_GFX_UNITID_MAPPING3 0x00be
+#define mmXPB_CLG_GFX_UNITID_MAPPING3_BASE_IDX 0
+#define mmXPB_CLG_GFX_UNITID_MAPPING4 0x00bf
+#define mmXPB_CLG_GFX_UNITID_MAPPING4_BASE_IDX 0
+#define mmXPB_CLG_GFX_UNITID_MAPPING5 0x00c0
+#define mmXPB_CLG_GFX_UNITID_MAPPING5_BASE_IDX 0
+#define mmXPB_CLG_GFX_UNITID_MAPPING6 0x00c1
+#define mmXPB_CLG_GFX_UNITID_MAPPING6_BASE_IDX 0
+#define mmXPB_CLG_GFX_UNITID_MAPPING7 0x00c2
+#define mmXPB_CLG_GFX_UNITID_MAPPING7_BASE_IDX 0
+#define mmXPB_CLG_MM_UNITID_MAPPING0 0x00c3
+#define mmXPB_CLG_MM_UNITID_MAPPING0_BASE_IDX 0
+#define mmXPB_CLG_MM_UNITID_MAPPING1 0x00c4
+#define mmXPB_CLG_MM_UNITID_MAPPING1_BASE_IDX 0
+#define mmXPB_CLG_MM_UNITID_MAPPING2 0x00c5
+#define mmXPB_CLG_MM_UNITID_MAPPING2_BASE_IDX 0
+#define mmXPB_CLG_MM_UNITID_MAPPING3 0x00c6
+#define mmXPB_CLG_MM_UNITID_MAPPING3_BASE_IDX 0
+#define mmXPB_CLG_GUS_UNITID_MAPPING0 0x00c7
+#define mmXPB_CLG_GUS_UNITID_MAPPING0_BASE_IDX 0
+#define mmXPB_CLG_GUS_UNITID_MAPPING1 0x00c8
+#define mmXPB_CLG_GUS_UNITID_MAPPING1_BASE_IDX 0
+#define mmXPB_CLG_GUS_UNITID_MAPPING2 0x00c9
+#define mmXPB_CLG_GUS_UNITID_MAPPING2_BASE_IDX 0
+#define mmXPB_CLG_GUS_UNITID_MAPPING3 0x00ca
+#define mmXPB_CLG_GUS_UNITID_MAPPING3_BASE_IDX 0
+#define mmXPB_CLG_GUS_UNITID_MAPPING4 0x00cb
+#define mmXPB_CLG_GUS_UNITID_MAPPING4_BASE_IDX 0
+#define mmXPB_CLG_GUS_UNITID_MAPPING5 0x00cc
+#define mmXPB_CLG_GUS_UNITID_MAPPING5_BASE_IDX 0
+#define mmXPB_CLG_GUS_UNITID_MAPPING6 0x00cd
+#define mmXPB_CLG_GUS_UNITID_MAPPING6_BASE_IDX 0
+#define mmXPB_CLG_GUS_UNITID_MAPPING7 0x00ce
+#define mmXPB_CLG_GUS_UNITID_MAPPING7_BASE_IDX 0
+
+
+// addressBlock: athub_rpbdec
+// base address: 0x3350
+#define mmRPB_PASSPW_CONF 0x00d4
+#define mmRPB_PASSPW_CONF_BASE_IDX 0
+#define mmRPB_BLOCKLEVEL_CONF 0x00d5
+#define mmRPB_BLOCKLEVEL_CONF_BASE_IDX 0
+#define mmRPB_TAG_CONF 0x00d7
+#define mmRPB_TAG_CONF_BASE_IDX 0
+#define mmRPB_EFF_CNTL 0x00d9
+#define mmRPB_EFF_CNTL_BASE_IDX 0
+#define mmRPB_ARB_CNTL 0x00da
+#define mmRPB_ARB_CNTL_BASE_IDX 0
+#define mmRPB_ARB_CNTL2 0x00db
+#define mmRPB_ARB_CNTL2_BASE_IDX 0
+#define mmRPB_BIF_CNTL 0x00dc
+#define mmRPB_BIF_CNTL_BASE_IDX 0
+#define mmRPB_WR_SWITCH_CNTL 0x00dd
+#define mmRPB_WR_SWITCH_CNTL_BASE_IDX 0
+#define mmRPB_WR_COMBINE_CNTL 0x00de
+#define mmRPB_WR_COMBINE_CNTL_BASE_IDX 0
+#define mmRPB_RD_SWITCH_CNTL 0x00df
+#define mmRPB_RD_SWITCH_CNTL_BASE_IDX 0
+#define mmRPB_CID_QUEUE_WR 0x00e0
+#define mmRPB_CID_QUEUE_WR_BASE_IDX 0
+#define mmRPB_CID_QUEUE_RD 0x00e1
+#define mmRPB_CID_QUEUE_RD_BASE_IDX 0
+#define mmRPB_PERF_COUNTER_CNTL 0x00e2
+#define mmRPB_PERF_COUNTER_CNTL_BASE_IDX 0
+#define mmRPB_PERF_COUNTER_STATUS 0x00e3
+#define mmRPB_PERF_COUNTER_STATUS_BASE_IDX 0
+#define mmRPB_CID_QUEUE_EX 0x00e4
+#define mmRPB_CID_QUEUE_EX_BASE_IDX 0
+#define mmRPB_CID_QUEUE_EX_DATA 0x00e5
+#define mmRPB_CID_QUEUE_EX_DATA_BASE_IDX 0
+#define mmRPB_SWITCH_CNTL2 0x00e6
+#define mmRPB_SWITCH_CNTL2_BASE_IDX 0
+#define mmRPB_DEINTRLV_COMBINE_CNTL 0x00e7
+#define mmRPB_DEINTRLV_COMBINE_CNTL_BASE_IDX 0
+#define mmRPB_VC_SWITCH_RDWR 0x00e8
+#define mmRPB_VC_SWITCH_RDWR_BASE_IDX 0
+#define mmRPB_PERFCOUNTER_LO 0x00e9
+#define mmRPB_PERFCOUNTER_LO_BASE_IDX 0
+#define mmRPB_PERFCOUNTER_HI 0x00ea
+#define mmRPB_PERFCOUNTER_HI_BASE_IDX 0
+#define mmRPB_PERFCOUNTER0_CFG 0x00eb
+#define mmRPB_PERFCOUNTER0_CFG_BASE_IDX 0
+#define mmRPB_PERFCOUNTER1_CFG 0x00ec
+#define mmRPB_PERFCOUNTER1_CFG_BASE_IDX 0
+#define mmRPB_PERFCOUNTER2_CFG 0x00ed
+#define mmRPB_PERFCOUNTER2_CFG_BASE_IDX 0
+#define mmRPB_PERFCOUNTER3_CFG 0x00ee
+#define mmRPB_PERFCOUNTER3_CFG_BASE_IDX 0
+#define mmRPB_PERFCOUNTER_RSLT_CNTL 0x00ef
+#define mmRPB_PERFCOUNTER_RSLT_CNTL_BASE_IDX 0
+#define mmRPB_BIF_CNTL2 0x00f0
+#define mmRPB_BIF_CNTL2_BASE_IDX 0
+#define mmRPB_RD_QUEUE_CNTL 0x00f1
+#define mmRPB_RD_QUEUE_CNTL_BASE_IDX 0
+#define mmRPB_RD_QUEUE_CNTL2 0x00f2
+#define mmRPB_RD_QUEUE_CNTL2_BASE_IDX 0
+#define mmRPB_WR_QUEUE_CNTL 0x00f3
+#define mmRPB_WR_QUEUE_CNTL_BASE_IDX 0
+#define mmRPB_WR_QUEUE_CNTL2 0x00f4
+#define mmRPB_WR_QUEUE_CNTL2_BASE_IDX 0
+#define mmRPB_EA_QUEUE_WR 0x00f5
+#define mmRPB_EA_QUEUE_WR_BASE_IDX 0
+#define mmRPB_ATS_CNTL 0x00f6
+#define mmRPB_ATS_CNTL_BASE_IDX 0
+#define mmRPB_ATS_CNTL2 0x00f7
+#define mmRPB_ATS_CNTL2_BASE_IDX 0
+#define mmRPB_DF_SDPPORT_CNTL 0x00f8
+#define mmRPB_DF_SDPPORT_CNTL_BASE_IDX 0
+#define mmRPB_SDPPORT_CNTL 0x00f9
+#define mmRPB_SDPPORT_CNTL_BASE_IDX 0
+#define mmRPB_NBIF_SDPPORT_CNTL 0x00fa
+#define mmRPB_NBIF_SDPPORT_CNTL_BASE_IDX 0
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/athub/athub_2_0_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/athub/athub_2_0_0_sh_mask.h
new file mode 100644
index 000000000000..88f77e90c5cb
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/athub/athub_2_0_0_sh_mask.h
@@ -0,0 +1,2264 @@
+/*
+ * Copyright (C) 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _athub_2_0_0_SH_MASK_HEADER
+#define _athub_2_0_0_SH_MASK_HEADER
+
+
+// addressBlock: athub_atsdec
+//ATC_ATS_CNTL
+#define ATC_ATS_CNTL__DISABLE_ATC__SHIFT 0x0
+#define ATC_ATS_CNTL__DISABLE_PRI__SHIFT 0x1
+#define ATC_ATS_CNTL__DISABLE_PASID__SHIFT 0x2
+#define ATC_ATS_CNTL__CREDITS_ATS_RPB__SHIFT 0x8
+#define ATC_ATS_CNTL__INVALIDATION_LOG_KEEP_ORDER__SHIFT 0x14
+#define ATC_ATS_CNTL__TRANS_LOG_KEEP_ORDER__SHIFT 0x15
+#define ATC_ATS_CNTL__TRANS_EXE_RETURN__SHIFT 0x16
+#define ATC_ATS_CNTL__DISABLE_ATC_MASK 0x00000001L
+#define ATC_ATS_CNTL__DISABLE_PRI_MASK 0x00000002L
+#define ATC_ATS_CNTL__DISABLE_PASID_MASK 0x00000004L
+#define ATC_ATS_CNTL__CREDITS_ATS_RPB_MASK 0x00003F00L
+#define ATC_ATS_CNTL__INVALIDATION_LOG_KEEP_ORDER_MASK 0x00100000L
+#define ATC_ATS_CNTL__TRANS_LOG_KEEP_ORDER_MASK 0x00200000L
+#define ATC_ATS_CNTL__TRANS_EXE_RETURN_MASK 0x00C00000L
+//ATC_ATS_STATUS
+#define ATC_ATS_STATUS__BUSY__SHIFT 0x0
+#define ATC_ATS_STATUS__CRASHED__SHIFT 0x1
+#define ATC_ATS_STATUS__DEADLOCK_DETECTION__SHIFT 0x2
+#define ATC_ATS_STATUS__FLUSH_INVALIDATION_OUTSTANDING__SHIFT 0x3
+#define ATC_ATS_STATUS__NONFLUSH_INVALIDATION_OUTSTANDING__SHIFT 0x6
+#define ATC_ATS_STATUS__BUSY_MASK 0x00000001L
+#define ATC_ATS_STATUS__CRASHED_MASK 0x00000002L
+#define ATC_ATS_STATUS__DEADLOCK_DETECTION_MASK 0x00000004L
+#define ATC_ATS_STATUS__FLUSH_INVALIDATION_OUTSTANDING_MASK 0x00000038L
+#define ATC_ATS_STATUS__NONFLUSH_INVALIDATION_OUTSTANDING_MASK 0x000001C0L
+//ATC_ATS_FAULT_CNTL
+#define ATC_ATS_FAULT_CNTL__FAULT_REGISTER_LOG__SHIFT 0x0
+#define ATC_ATS_FAULT_CNTL__FAULT_INTERRUPT_TABLE__SHIFT 0xa
+#define ATC_ATS_FAULT_CNTL__FAULT_CRASH_TABLE__SHIFT 0x14
+#define ATC_ATS_FAULT_CNTL__FAULT_REGISTER_LOG_MASK 0x000001FFL
+#define ATC_ATS_FAULT_CNTL__FAULT_INTERRUPT_TABLE_MASK 0x0007FC00L
+#define ATC_ATS_FAULT_CNTL__FAULT_CRASH_TABLE_MASK 0x1FF00000L
+//ATC_ATS_FAULT_STATUS_INFO
+#define ATC_ATS_FAULT_STATUS_INFO__FAULT_TYPE__SHIFT 0x0
+#define ATC_ATS_FAULT_STATUS_INFO__VMID__SHIFT 0xa
+#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO__SHIFT 0xf
+#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO2__SHIFT 0x10
+#define ATC_ATS_FAULT_STATUS_INFO__INVALIDATION__SHIFT 0x11
+#define ATC_ATS_FAULT_STATUS_INFO__PAGE_REQUEST__SHIFT 0x12
+#define ATC_ATS_FAULT_STATUS_INFO__STATUS__SHIFT 0x13
+#define ATC_ATS_FAULT_STATUS_INFO__PAGE_ADDR_HIGH__SHIFT 0x18
+#define ATC_ATS_FAULT_STATUS_INFO__FAULT_TYPE_MASK 0x000001FFL
+#define ATC_ATS_FAULT_STATUS_INFO__VMID_MASK 0x00007C00L
+#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO_MASK 0x00008000L
+#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO2_MASK 0x00010000L
+#define ATC_ATS_FAULT_STATUS_INFO__INVALIDATION_MASK 0x00020000L
+#define ATC_ATS_FAULT_STATUS_INFO__PAGE_REQUEST_MASK 0x00040000L
+#define ATC_ATS_FAULT_STATUS_INFO__STATUS_MASK 0x00F80000L
+#define ATC_ATS_FAULT_STATUS_INFO__PAGE_ADDR_HIGH_MASK 0x0F000000L
+//ATC_ATS_FAULT_STATUS_ADDR
+#define ATC_ATS_FAULT_STATUS_ADDR__PAGE_ADDR__SHIFT 0x0
+#define ATC_ATS_FAULT_STATUS_ADDR__PAGE_ADDR_MASK 0xFFFFFFFFL
+//ATC_ATS_DEFAULT_PAGE_LOW
+#define ATC_ATS_DEFAULT_PAGE_LOW__DEFAULT_PAGE__SHIFT 0x0
+#define ATC_ATS_DEFAULT_PAGE_LOW__DEFAULT_PAGE_MASK 0xFFFFFFFFL
+//ATC_TRANS_FAULT_RSPCNTRL
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID0__SHIFT 0x0
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID1__SHIFT 0x1
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID2__SHIFT 0x2
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID3__SHIFT 0x3
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID4__SHIFT 0x4
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID5__SHIFT 0x5
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID6__SHIFT 0x6
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID7__SHIFT 0x7
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID8__SHIFT 0x8
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID9__SHIFT 0x9
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID10__SHIFT 0xa
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID11__SHIFT 0xb
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID12__SHIFT 0xc
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID13__SHIFT 0xd
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID14__SHIFT 0xe
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID15__SHIFT 0xf
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID16__SHIFT 0x10
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID17__SHIFT 0x11
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID18__SHIFT 0x12
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID19__SHIFT 0x13
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID20__SHIFT 0x14
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID21__SHIFT 0x15
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID22__SHIFT 0x16
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID23__SHIFT 0x17
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID24__SHIFT 0x18
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID25__SHIFT 0x19
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID26__SHIFT 0x1a
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID27__SHIFT 0x1b
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID28__SHIFT 0x1c
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID29__SHIFT 0x1d
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID30__SHIFT 0x1e
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID31__SHIFT 0x1f
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID0_MASK 0x00000001L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID1_MASK 0x00000002L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID2_MASK 0x00000004L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID3_MASK 0x00000008L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID4_MASK 0x00000010L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID5_MASK 0x00000020L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID6_MASK 0x00000040L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID7_MASK 0x00000080L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID8_MASK 0x00000100L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID9_MASK 0x00000200L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID10_MASK 0x00000400L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID11_MASK 0x00000800L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID12_MASK 0x00001000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID13_MASK 0x00002000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID14_MASK 0x00004000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID15_MASK 0x00008000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID16_MASK 0x00010000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID17_MASK 0x00020000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID18_MASK 0x00040000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID19_MASK 0x00080000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID20_MASK 0x00100000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID21_MASK 0x00200000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID22_MASK 0x00400000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID23_MASK 0x00800000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID24_MASK 0x01000000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID25_MASK 0x02000000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID26_MASK 0x04000000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID27_MASK 0x08000000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID28_MASK 0x10000000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID29_MASK 0x20000000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID30_MASK 0x40000000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID31_MASK 0x80000000L
+//ATC_ATS_FAULT_STATUS_INFO2
+#define ATC_ATS_FAULT_STATUS_INFO2__VF__SHIFT 0x0
+#define ATC_ATS_FAULT_STATUS_INFO2__VFID__SHIFT 0x1
+#define ATC_ATS_FAULT_STATUS_INFO2__MMHUB_INV_VMID__SHIFT 0x9
+#define ATC_ATS_FAULT_STATUS_INFO2__VF_MASK 0x00000001L
+#define ATC_ATS_FAULT_STATUS_INFO2__VFID_MASK 0x0000003EL
+#define ATC_ATS_FAULT_STATUS_INFO2__MMHUB_INV_VMID_MASK 0x00003E00L
+//ATHUB_MISC_CNTL
+#define ATHUB_MISC_CNTL__CG_OFFDLY__SHIFT 0x6
+#define ATHUB_MISC_CNTL__CG_ENABLE__SHIFT 0x12
+#define ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE__SHIFT 0x13
+#define ATHUB_MISC_CNTL__PG_ENABLE__SHIFT 0x14
+#define ATHUB_MISC_CNTL__PG_OFFDLY__SHIFT 0x15
+#define ATHUB_MISC_CNTL__CG_STATUS__SHIFT 0x1b
+#define ATHUB_MISC_CNTL__PG_STATUS__SHIFT 0x1c
+#define ATHUB_MISC_CNTL__CG_OFFDLY_MASK 0x00000FC0L
+#define ATHUB_MISC_CNTL__CG_ENABLE_MASK 0x00040000L
+#define ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK 0x00080000L
+#define ATHUB_MISC_CNTL__PG_ENABLE_MASK 0x00100000L
+#define ATHUB_MISC_CNTL__PG_OFFDLY_MASK 0x07E00000L
+#define ATHUB_MISC_CNTL__CG_STATUS_MASK 0x08000000L
+#define ATHUB_MISC_CNTL__PG_STATUS_MASK 0x10000000L
+//ATC_VMID_PASID_MAPPING_UPDATE_STATUS
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID0_REMAPPING_FINISHED__SHIFT 0x0
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID1_REMAPPING_FINISHED__SHIFT 0x1
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID2_REMAPPING_FINISHED__SHIFT 0x2
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID3_REMAPPING_FINISHED__SHIFT 0x3
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID4_REMAPPING_FINISHED__SHIFT 0x4
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID5_REMAPPING_FINISHED__SHIFT 0x5
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID6_REMAPPING_FINISHED__SHIFT 0x6
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID7_REMAPPING_FINISHED__SHIFT 0x7
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID8_REMAPPING_FINISHED__SHIFT 0x8
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID9_REMAPPING_FINISHED__SHIFT 0x9
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID10_REMAPPING_FINISHED__SHIFT 0xa
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID11_REMAPPING_FINISHED__SHIFT 0xb
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID12_REMAPPING_FINISHED__SHIFT 0xc
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID13_REMAPPING_FINISHED__SHIFT 0xd
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID14_REMAPPING_FINISHED__SHIFT 0xe
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID15_REMAPPING_FINISHED__SHIFT 0xf
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID16_REMAPPING_FINISHED__SHIFT 0x10
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID17_REMAPPING_FINISHED__SHIFT 0x11
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID18_REMAPPING_FINISHED__SHIFT 0x12
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID19_REMAPPING_FINISHED__SHIFT 0x13
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID20_REMAPPING_FINISHED__SHIFT 0x14
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID21_REMAPPING_FINISHED__SHIFT 0x15
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID22_REMAPPING_FINISHED__SHIFT 0x16
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID23_REMAPPING_FINISHED__SHIFT 0x17
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID24_REMAPPING_FINISHED__SHIFT 0x18
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID25_REMAPPING_FINISHED__SHIFT 0x19
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID26_REMAPPING_FINISHED__SHIFT 0x1a
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID27_REMAPPING_FINISHED__SHIFT 0x1b
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID28_REMAPPING_FINISHED__SHIFT 0x1c
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID29_REMAPPING_FINISHED__SHIFT 0x1d
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID30_REMAPPING_FINISHED__SHIFT 0x1e
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID31_REMAPPING_FINISHED__SHIFT 0x1f
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID0_REMAPPING_FINISHED_MASK 0x00000001L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID1_REMAPPING_FINISHED_MASK 0x00000002L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID2_REMAPPING_FINISHED_MASK 0x00000004L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID3_REMAPPING_FINISHED_MASK 0x00000008L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID4_REMAPPING_FINISHED_MASK 0x00000010L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID5_REMAPPING_FINISHED_MASK 0x00000020L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID6_REMAPPING_FINISHED_MASK 0x00000040L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID7_REMAPPING_FINISHED_MASK 0x00000080L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID8_REMAPPING_FINISHED_MASK 0x00000100L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID9_REMAPPING_FINISHED_MASK 0x00000200L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID10_REMAPPING_FINISHED_MASK 0x00000400L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID11_REMAPPING_FINISHED_MASK 0x00000800L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID12_REMAPPING_FINISHED_MASK 0x00001000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID13_REMAPPING_FINISHED_MASK 0x00002000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID14_REMAPPING_FINISHED_MASK 0x00004000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID15_REMAPPING_FINISHED_MASK 0x00008000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID16_REMAPPING_FINISHED_MASK 0x00010000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID17_REMAPPING_FINISHED_MASK 0x00020000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID18_REMAPPING_FINISHED_MASK 0x00040000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID19_REMAPPING_FINISHED_MASK 0x00080000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID20_REMAPPING_FINISHED_MASK 0x00100000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID21_REMAPPING_FINISHED_MASK 0x00200000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID22_REMAPPING_FINISHED_MASK 0x00400000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID23_REMAPPING_FINISHED_MASK 0x00800000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID24_REMAPPING_FINISHED_MASK 0x01000000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID25_REMAPPING_FINISHED_MASK 0x02000000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID26_REMAPPING_FINISHED_MASK 0x04000000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID27_REMAPPING_FINISHED_MASK 0x08000000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID28_REMAPPING_FINISHED_MASK 0x10000000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID29_REMAPPING_FINISHED_MASK 0x20000000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID30_REMAPPING_FINISHED_MASK 0x40000000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID31_REMAPPING_FINISHED_MASK 0x80000000L
+//ATC_VMID0_PASID_MAPPING
+#define ATC_VMID0_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID0_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID0_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID0_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID0_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID0_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID1_PASID_MAPPING
+#define ATC_VMID1_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID1_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID1_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID1_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID1_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID1_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID2_PASID_MAPPING
+#define ATC_VMID2_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID2_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID2_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID2_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID2_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID2_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID3_PASID_MAPPING
+#define ATC_VMID3_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID3_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID3_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID3_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID3_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID3_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID4_PASID_MAPPING
+#define ATC_VMID4_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID4_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID4_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID4_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID4_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID4_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID5_PASID_MAPPING
+#define ATC_VMID5_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID5_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID5_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID5_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID5_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID5_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID6_PASID_MAPPING
+#define ATC_VMID6_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID6_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID6_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID6_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID6_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID6_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID7_PASID_MAPPING
+#define ATC_VMID7_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID7_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID7_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID7_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID7_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID7_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID8_PASID_MAPPING
+#define ATC_VMID8_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID8_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID8_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID8_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID8_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID8_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID9_PASID_MAPPING
+#define ATC_VMID9_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID9_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID9_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID9_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID9_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID9_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID10_PASID_MAPPING
+#define ATC_VMID10_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID10_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID10_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID10_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID10_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID10_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID11_PASID_MAPPING
+#define ATC_VMID11_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID11_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID11_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID11_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID11_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID11_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID12_PASID_MAPPING
+#define ATC_VMID12_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID12_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID12_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID12_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID12_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID12_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID13_PASID_MAPPING
+#define ATC_VMID13_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID13_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID13_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID13_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID13_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID13_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID14_PASID_MAPPING
+#define ATC_VMID14_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID14_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID14_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID14_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID14_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID14_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID15_PASID_MAPPING
+#define ATC_VMID15_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID15_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID15_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID15_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID15_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID15_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_ATS_VMID_STATUS
+#define ATC_ATS_VMID_STATUS__VMID0_OUTSTANDING__SHIFT 0x0
+#define ATC_ATS_VMID_STATUS__VMID1_OUTSTANDING__SHIFT 0x1
+#define ATC_ATS_VMID_STATUS__VMID2_OUTSTANDING__SHIFT 0x2
+#define ATC_ATS_VMID_STATUS__VMID3_OUTSTANDING__SHIFT 0x3
+#define ATC_ATS_VMID_STATUS__VMID4_OUTSTANDING__SHIFT 0x4
+#define ATC_ATS_VMID_STATUS__VMID5_OUTSTANDING__SHIFT 0x5
+#define ATC_ATS_VMID_STATUS__VMID6_OUTSTANDING__SHIFT 0x6
+#define ATC_ATS_VMID_STATUS__VMID7_OUTSTANDING__SHIFT 0x7
+#define ATC_ATS_VMID_STATUS__VMID8_OUTSTANDING__SHIFT 0x8
+#define ATC_ATS_VMID_STATUS__VMID9_OUTSTANDING__SHIFT 0x9
+#define ATC_ATS_VMID_STATUS__VMID10_OUTSTANDING__SHIFT 0xa
+#define ATC_ATS_VMID_STATUS__VMID11_OUTSTANDING__SHIFT 0xb
+#define ATC_ATS_VMID_STATUS__VMID12_OUTSTANDING__SHIFT 0xc
+#define ATC_ATS_VMID_STATUS__VMID13_OUTSTANDING__SHIFT 0xd
+#define ATC_ATS_VMID_STATUS__VMID14_OUTSTANDING__SHIFT 0xe
+#define ATC_ATS_VMID_STATUS__VMID15_OUTSTANDING__SHIFT 0xf
+#define ATC_ATS_VMID_STATUS__VMID16_OUTSTANDING__SHIFT 0x10
+#define ATC_ATS_VMID_STATUS__VMID17_OUTSTANDING__SHIFT 0x11
+#define ATC_ATS_VMID_STATUS__VMID18_OUTSTANDING__SHIFT 0x12
+#define ATC_ATS_VMID_STATUS__VMID19_OUTSTANDING__SHIFT 0x13
+#define ATC_ATS_VMID_STATUS__VMID20_OUTSTANDING__SHIFT 0x14
+#define ATC_ATS_VMID_STATUS__VMID21_OUTSTANDING__SHIFT 0x15
+#define ATC_ATS_VMID_STATUS__VMID22_OUTSTANDING__SHIFT 0x16
+#define ATC_ATS_VMID_STATUS__VMID23_OUTSTANDING__SHIFT 0x17
+#define ATC_ATS_VMID_STATUS__VMID24_OUTSTANDING__SHIFT 0x18
+#define ATC_ATS_VMID_STATUS__VMID25_OUTSTANDING__SHIFT 0x19
+#define ATC_ATS_VMID_STATUS__VMID26_OUTSTANDING__SHIFT 0x1a
+#define ATC_ATS_VMID_STATUS__VMID27_OUTSTANDING__SHIFT 0x1b
+#define ATC_ATS_VMID_STATUS__VMID28_OUTSTANDING__SHIFT 0x1c
+#define ATC_ATS_VMID_STATUS__VMID29_OUTSTANDING__SHIFT 0x1d
+#define ATC_ATS_VMID_STATUS__VMID30_OUTSTANDING__SHIFT 0x1e
+#define ATC_ATS_VMID_STATUS__VMID31_OUTSTANDING__SHIFT 0x1f
+#define ATC_ATS_VMID_STATUS__VMID0_OUTSTANDING_MASK 0x00000001L
+#define ATC_ATS_VMID_STATUS__VMID1_OUTSTANDING_MASK 0x00000002L
+#define ATC_ATS_VMID_STATUS__VMID2_OUTSTANDING_MASK 0x00000004L
+#define ATC_ATS_VMID_STATUS__VMID3_OUTSTANDING_MASK 0x00000008L
+#define ATC_ATS_VMID_STATUS__VMID4_OUTSTANDING_MASK 0x00000010L
+#define ATC_ATS_VMID_STATUS__VMID5_OUTSTANDING_MASK 0x00000020L
+#define ATC_ATS_VMID_STATUS__VMID6_OUTSTANDING_MASK 0x00000040L
+#define ATC_ATS_VMID_STATUS__VMID7_OUTSTANDING_MASK 0x00000080L
+#define ATC_ATS_VMID_STATUS__VMID8_OUTSTANDING_MASK 0x00000100L
+#define ATC_ATS_VMID_STATUS__VMID9_OUTSTANDING_MASK 0x00000200L
+#define ATC_ATS_VMID_STATUS__VMID10_OUTSTANDING_MASK 0x00000400L
+#define ATC_ATS_VMID_STATUS__VMID11_OUTSTANDING_MASK 0x00000800L
+#define ATC_ATS_VMID_STATUS__VMID12_OUTSTANDING_MASK 0x00001000L
+#define ATC_ATS_VMID_STATUS__VMID13_OUTSTANDING_MASK 0x00002000L
+#define ATC_ATS_VMID_STATUS__VMID14_OUTSTANDING_MASK 0x00004000L
+#define ATC_ATS_VMID_STATUS__VMID15_OUTSTANDING_MASK 0x00008000L
+#define ATC_ATS_VMID_STATUS__VMID16_OUTSTANDING_MASK 0x00010000L
+#define ATC_ATS_VMID_STATUS__VMID17_OUTSTANDING_MASK 0x00020000L
+#define ATC_ATS_VMID_STATUS__VMID18_OUTSTANDING_MASK 0x00040000L
+#define ATC_ATS_VMID_STATUS__VMID19_OUTSTANDING_MASK 0x00080000L
+#define ATC_ATS_VMID_STATUS__VMID20_OUTSTANDING_MASK 0x00100000L
+#define ATC_ATS_VMID_STATUS__VMID21_OUTSTANDING_MASK 0x00200000L
+#define ATC_ATS_VMID_STATUS__VMID22_OUTSTANDING_MASK 0x00400000L
+#define ATC_ATS_VMID_STATUS__VMID23_OUTSTANDING_MASK 0x00800000L
+#define ATC_ATS_VMID_STATUS__VMID24_OUTSTANDING_MASK 0x01000000L
+#define ATC_ATS_VMID_STATUS__VMID25_OUTSTANDING_MASK 0x02000000L
+#define ATC_ATS_VMID_STATUS__VMID26_OUTSTANDING_MASK 0x04000000L
+#define ATC_ATS_VMID_STATUS__VMID27_OUTSTANDING_MASK 0x08000000L
+#define ATC_ATS_VMID_STATUS__VMID28_OUTSTANDING_MASK 0x10000000L
+#define ATC_ATS_VMID_STATUS__VMID29_OUTSTANDING_MASK 0x20000000L
+#define ATC_ATS_VMID_STATUS__VMID30_OUTSTANDING_MASK 0x40000000L
+#define ATC_ATS_VMID_STATUS__VMID31_OUTSTANDING_MASK 0x80000000L
+//ATC_ATS_GFX_ATCL2_STATUS
+#define ATC_ATS_GFX_ATCL2_STATUS__POWERED_DOWN__SHIFT 0x0
+#define ATC_ATS_GFX_ATCL2_STATUS__POWERED_DOWN_MASK 0x00000001L
+//ATC_PERFCOUNTER0_CFG
+#define ATC_PERFCOUNTER0_CFG__PERF_SEL__SHIFT 0x0
+#define ATC_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT 0x8
+#define ATC_PERFCOUNTER0_CFG__PERF_MODE__SHIFT 0x18
+#define ATC_PERFCOUNTER0_CFG__ENABLE__SHIFT 0x1c
+#define ATC_PERFCOUNTER0_CFG__CLEAR__SHIFT 0x1d
+#define ATC_PERFCOUNTER0_CFG__PERF_SEL_MASK 0x000000FFL
+#define ATC_PERFCOUNTER0_CFG__PERF_SEL_END_MASK 0x0000FF00L
+#define ATC_PERFCOUNTER0_CFG__PERF_MODE_MASK 0x0F000000L
+#define ATC_PERFCOUNTER0_CFG__ENABLE_MASK 0x10000000L
+#define ATC_PERFCOUNTER0_CFG__CLEAR_MASK 0x20000000L
+//ATC_PERFCOUNTER1_CFG
+#define ATC_PERFCOUNTER1_CFG__PERF_SEL__SHIFT 0x0
+#define ATC_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT 0x8
+#define ATC_PERFCOUNTER1_CFG__PERF_MODE__SHIFT 0x18
+#define ATC_PERFCOUNTER1_CFG__ENABLE__SHIFT 0x1c
+#define ATC_PERFCOUNTER1_CFG__CLEAR__SHIFT 0x1d
+#define ATC_PERFCOUNTER1_CFG__PERF_SEL_MASK 0x000000FFL
+#define ATC_PERFCOUNTER1_CFG__PERF_SEL_END_MASK 0x0000FF00L
+#define ATC_PERFCOUNTER1_CFG__PERF_MODE_MASK 0x0F000000L
+#define ATC_PERFCOUNTER1_CFG__ENABLE_MASK 0x10000000L
+#define ATC_PERFCOUNTER1_CFG__CLEAR_MASK 0x20000000L
+//ATC_PERFCOUNTER2_CFG
+#define ATC_PERFCOUNTER2_CFG__PERF_SEL__SHIFT 0x0
+#define ATC_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT 0x8
+#define ATC_PERFCOUNTER2_CFG__PERF_MODE__SHIFT 0x18
+#define ATC_PERFCOUNTER2_CFG__ENABLE__SHIFT 0x1c
+#define ATC_PERFCOUNTER2_CFG__CLEAR__SHIFT 0x1d
+#define ATC_PERFCOUNTER2_CFG__PERF_SEL_MASK 0x000000FFL
+#define ATC_PERFCOUNTER2_CFG__PERF_SEL_END_MASK 0x0000FF00L
+#define ATC_PERFCOUNTER2_CFG__PERF_MODE_MASK 0x0F000000L
+#define ATC_PERFCOUNTER2_CFG__ENABLE_MASK 0x10000000L
+#define ATC_PERFCOUNTER2_CFG__CLEAR_MASK 0x20000000L
+//ATC_PERFCOUNTER3_CFG
+#define ATC_PERFCOUNTER3_CFG__PERF_SEL__SHIFT 0x0
+#define ATC_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT 0x8
+#define ATC_PERFCOUNTER3_CFG__PERF_MODE__SHIFT 0x18
+#define ATC_PERFCOUNTER3_CFG__ENABLE__SHIFT 0x1c
+#define ATC_PERFCOUNTER3_CFG__CLEAR__SHIFT 0x1d
+#define ATC_PERFCOUNTER3_CFG__PERF_SEL_MASK 0x000000FFL
+#define ATC_PERFCOUNTER3_CFG__PERF_SEL_END_MASK 0x0000FF00L
+#define ATC_PERFCOUNTER3_CFG__PERF_MODE_MASK 0x0F000000L
+#define ATC_PERFCOUNTER3_CFG__ENABLE_MASK 0x10000000L
+#define ATC_PERFCOUNTER3_CFG__CLEAR_MASK 0x20000000L
+//ATC_PERFCOUNTER_RSLT_CNTL
+#define ATC_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT 0x0
+#define ATC_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT 0x8
+#define ATC_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT 0x10
+#define ATC_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT 0x18
+#define ATC_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT 0x19
+#define ATC_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT 0x1a
+#define ATC_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK 0x0000000FL
+#define ATC_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK 0x0000FF00L
+#define ATC_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK 0x00FF0000L
+#define ATC_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK 0x01000000L
+#define ATC_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK 0x02000000L
+#define ATC_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK 0x04000000L
+//ATC_PERFCOUNTER_LO
+#define ATC_PERFCOUNTER_LO__COUNTER_LO__SHIFT 0x0
+#define ATC_PERFCOUNTER_LO__COUNTER_LO_MASK 0xFFFFFFFFL
+//ATC_PERFCOUNTER_HI
+#define ATC_PERFCOUNTER_HI__COUNTER_HI__SHIFT 0x0
+#define ATC_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT 0x10
+#define ATC_PERFCOUNTER_HI__COUNTER_HI_MASK 0x0000FFFFL
+#define ATC_PERFCOUNTER_HI__COMPARE_VALUE_MASK 0xFFFF0000L
+//ATHUB_PCIE_ATS_CNTL
+#define ATHUB_PCIE_ATS_CNTL__STU__SHIFT 0x10
+#define ATHUB_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL__STU_MASK 0x001F0000L
+#define ATHUB_PCIE_ATS_CNTL__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_PASID_CNTL
+#define ATHUB_PCIE_PASID_CNTL__PASID_EN__SHIFT 0x10
+#define ATHUB_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT 0x11
+#define ATHUB_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT 0x12
+#define ATHUB_PCIE_PASID_CNTL__PASID_EN_MASK 0x00010000L
+#define ATHUB_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK 0x00020000L
+#define ATHUB_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK 0x00040000L
+//ATHUB_PCIE_PAGE_REQ_CNTL
+#define ATHUB_PCIE_PAGE_REQ_CNTL__PRI_ENABLE__SHIFT 0x0
+#define ATHUB_PCIE_PAGE_REQ_CNTL__PRI_RESET__SHIFT 0x1
+#define ATHUB_PCIE_PAGE_REQ_CNTL__PRI_ENABLE_MASK 0x00000001L
+#define ATHUB_PCIE_PAGE_REQ_CNTL__PRI_RESET_MASK 0x00000002L
+//ATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC
+#define ATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC__SHIFT 0x0
+#define ATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC_MASK 0xFFFFFFFFL
+//ATHUB_COMMAND
+#define ATHUB_COMMAND__BUS_MASTER_EN__SHIFT 0x2
+#define ATHUB_COMMAND__BUS_MASTER_EN_MASK 0x00000004L
+//ATHUB_PCIE_ATS_CNTL_VF_0
+#define ATHUB_PCIE_ATS_CNTL_VF_0__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_0__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_1
+#define ATHUB_PCIE_ATS_CNTL_VF_1__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_1__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_2
+#define ATHUB_PCIE_ATS_CNTL_VF_2__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_2__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_3
+#define ATHUB_PCIE_ATS_CNTL_VF_3__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_3__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_4
+#define ATHUB_PCIE_ATS_CNTL_VF_4__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_4__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_5
+#define ATHUB_PCIE_ATS_CNTL_VF_5__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_5__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_6
+#define ATHUB_PCIE_ATS_CNTL_VF_6__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_6__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_7
+#define ATHUB_PCIE_ATS_CNTL_VF_7__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_7__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_8
+#define ATHUB_PCIE_ATS_CNTL_VF_8__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_8__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_9
+#define ATHUB_PCIE_ATS_CNTL_VF_9__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_9__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_10
+#define ATHUB_PCIE_ATS_CNTL_VF_10__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_10__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_11
+#define ATHUB_PCIE_ATS_CNTL_VF_11__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_11__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_12
+#define ATHUB_PCIE_ATS_CNTL_VF_12__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_12__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_13
+#define ATHUB_PCIE_ATS_CNTL_VF_13__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_13__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_14
+#define ATHUB_PCIE_ATS_CNTL_VF_14__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_14__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_15
+#define ATHUB_PCIE_ATS_CNTL_VF_15__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_15__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_16
+#define ATHUB_PCIE_ATS_CNTL_VF_16__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_16__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_17
+#define ATHUB_PCIE_ATS_CNTL_VF_17__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_17__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_18
+#define ATHUB_PCIE_ATS_CNTL_VF_18__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_18__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_19
+#define ATHUB_PCIE_ATS_CNTL_VF_19__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_19__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_20
+#define ATHUB_PCIE_ATS_CNTL_VF_20__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_20__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_21
+#define ATHUB_PCIE_ATS_CNTL_VF_21__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_21__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_22
+#define ATHUB_PCIE_ATS_CNTL_VF_22__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_22__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_23
+#define ATHUB_PCIE_ATS_CNTL_VF_23__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_23__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_24
+#define ATHUB_PCIE_ATS_CNTL_VF_24__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_24__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_25
+#define ATHUB_PCIE_ATS_CNTL_VF_25__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_25__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_26
+#define ATHUB_PCIE_ATS_CNTL_VF_26__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_26__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_27
+#define ATHUB_PCIE_ATS_CNTL_VF_27__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_27__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_28
+#define ATHUB_PCIE_ATS_CNTL_VF_28__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_28__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_29
+#define ATHUB_PCIE_ATS_CNTL_VF_29__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_29__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_30
+#define ATHUB_PCIE_ATS_CNTL_VF_30__ATC_ENABLE__SHIFT 0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_30__ATC_ENABLE_MASK 0x80000000L
+//ATHUB_MEM_POWER_LS
+#define ATHUB_MEM_POWER_LS__LS_SETUP__SHIFT 0x0
+#define ATHUB_MEM_POWER_LS__LS_HOLD__SHIFT 0x6
+#define ATHUB_MEM_POWER_LS__LS_SETUP_MASK 0x0000003FL
+#define ATHUB_MEM_POWER_LS__LS_HOLD_MASK 0x00000FC0L
+//ATS_IH_CREDIT
+#define ATS_IH_CREDIT__CREDIT_VALUE__SHIFT 0x0
+#define ATS_IH_CREDIT__IH_CLIENT_ID__SHIFT 0x10
+#define ATS_IH_CREDIT__CREDIT_VALUE_MASK 0x00000003L
+#define ATS_IH_CREDIT__IH_CLIENT_ID_MASK 0x00FF0000L
+//ATHUB_IH_CREDIT
+#define ATHUB_IH_CREDIT__CREDIT_VALUE__SHIFT 0x0
+#define ATHUB_IH_CREDIT__IH_CLIENT_ID__SHIFT 0x10
+#define ATHUB_IH_CREDIT__CREDIT_VALUE_MASK 0x00000003L
+#define ATHUB_IH_CREDIT__IH_CLIENT_ID_MASK 0x00FF0000L
+//ATC_VMID16_PASID_MAPPING
+#define ATC_VMID16_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID16_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID16_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID16_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID16_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID16_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID17_PASID_MAPPING
+#define ATC_VMID17_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID17_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID17_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID17_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID17_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID17_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID18_PASID_MAPPING
+#define ATC_VMID18_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID18_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID18_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID18_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID18_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID18_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID19_PASID_MAPPING
+#define ATC_VMID19_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID19_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID19_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID19_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID19_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID19_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID20_PASID_MAPPING
+#define ATC_VMID20_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID20_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID20_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID20_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID20_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID20_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID21_PASID_MAPPING
+#define ATC_VMID21_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID21_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID21_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID21_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID21_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID21_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID22_PASID_MAPPING
+#define ATC_VMID22_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID22_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID22_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID22_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID22_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID22_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID23_PASID_MAPPING
+#define ATC_VMID23_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID23_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID23_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID23_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID23_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID23_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID24_PASID_MAPPING
+#define ATC_VMID24_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID24_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID24_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID24_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID24_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID24_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID25_PASID_MAPPING
+#define ATC_VMID25_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID25_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID25_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID25_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID25_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID25_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID26_PASID_MAPPING
+#define ATC_VMID26_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID26_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID26_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID26_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID26_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID26_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID27_PASID_MAPPING
+#define ATC_VMID27_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID27_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID27_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID27_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID27_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID27_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID28_PASID_MAPPING
+#define ATC_VMID28_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID28_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID28_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID28_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID28_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID28_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID29_PASID_MAPPING
+#define ATC_VMID29_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID29_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID29_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID29_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID29_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID29_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID30_PASID_MAPPING
+#define ATC_VMID30_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID30_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID30_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID30_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID30_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID30_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_VMID31_PASID_MAPPING
+#define ATC_VMID31_PASID_MAPPING__PASID__SHIFT 0x0
+#define ATC_VMID31_PASID_MAPPING__NO_INVALIDATION__SHIFT 0x1e
+#define ATC_VMID31_PASID_MAPPING__VALID__SHIFT 0x1f
+#define ATC_VMID31_PASID_MAPPING__PASID_MASK 0x0000FFFFL
+#define ATC_VMID31_PASID_MAPPING__NO_INVALIDATION_MASK 0x40000000L
+#define ATC_VMID31_PASID_MAPPING__VALID_MASK 0x80000000L
+//ATC_ATS_MMHUB_ATCL2_STATUS
+#define ATC_ATS_MMHUB_ATCL2_STATUS__POWERED_DOWN__SHIFT 0x0
+#define ATC_ATS_MMHUB_ATCL2_STATUS__POWERED_DOWN_MASK 0x00000001L
+//ATHUB_SHARED_VIRT_RESET_REQ
+#define ATHUB_SHARED_VIRT_RESET_REQ__VF__SHIFT 0x0
+#define ATHUB_SHARED_VIRT_RESET_REQ__PF__SHIFT 0x1f
+#define ATHUB_SHARED_VIRT_RESET_REQ__VF_MASK 0x7FFFFFFFL
+#define ATHUB_SHARED_VIRT_RESET_REQ__PF_MASK 0x80000000L
+//ATHUB_SHARED_ACTIVE_FCN_ID
+#define ATHUB_SHARED_ACTIVE_FCN_ID__VFID__SHIFT 0x0
+#define ATHUB_SHARED_ACTIVE_FCN_ID__VF__SHIFT 0x1f
+#define ATHUB_SHARED_ACTIVE_FCN_ID__VFID_MASK 0x0000001FL
+#define ATHUB_SHARED_ACTIVE_FCN_ID__VF_MASK 0x80000000L
+//ATC_ATS_SDPPORT_CNTL
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_SELF_ACTIVATE__SHIFT 0x0
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_CFG_MODE__SHIFT 0x1
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_HALT_THRESHOLD__SHIFT 0x3
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_SELF_ACTIVATE__SHIFT 0x7
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_QUICK_COMACK__SHIFT 0x8
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_HALT_THRESHOLD__SHIFT 0x9
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_PASSIVE_MODE__SHIFT 0xd
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_RDY_MODE__SHIFT 0xe
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_MMHUB_RDY_MODE__SHIFT 0xf
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPCKEN__SHIFT 0x10
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPCKENRCV__SHIFT 0x11
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPDATACKEN__SHIFT 0x12
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPDATACKENRCV__SHIFT 0x13
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_WRRSPCKEN__SHIFT 0x14
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_WRRSPCKENRCV__SHIFT 0x15
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_REQCKEN__SHIFT 0x16
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_REQCKENRCV__SHIFT 0x17
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_ORIGDATACKEN__SHIFT 0x18
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_ORIGDATACKENRCV__SHIFT 0x19
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_SELF_ACTIVATE_MASK 0x00000001L
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_CFG_MODE_MASK 0x00000006L
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_HALT_THRESHOLD_MASK 0x00000078L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_SELF_ACTIVATE_MASK 0x00000080L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_QUICK_COMACK_MASK 0x00000100L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_HALT_THRESHOLD_MASK 0x00001E00L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_PASSIVE_MODE_MASK 0x00002000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_RDY_MODE_MASK 0x00004000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_MMHUB_RDY_MODE_MASK 0x00008000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPCKEN_MASK 0x00010000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPCKENRCV_MASK 0x00020000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPDATACKEN_MASK 0x00040000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPDATACKENRCV_MASK 0x00080000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_WRRSPCKEN_MASK 0x00100000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_WRRSPCKENRCV_MASK 0x00200000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_REQCKEN_MASK 0x00400000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_REQCKENRCV_MASK 0x00800000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_ORIGDATACKEN_MASK 0x01000000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_ORIGDATACKENRCV_MASK 0x02000000L
+//ATC_ATS_VMID_SNAPSHOT_GFX_STAT
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID0__SHIFT 0x0
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID1__SHIFT 0x1
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID2__SHIFT 0x2
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID3__SHIFT 0x3
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID4__SHIFT 0x4
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID5__SHIFT 0x5
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID6__SHIFT 0x6
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID7__SHIFT 0x7
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID8__SHIFT 0x8
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID9__SHIFT 0x9
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID10__SHIFT 0xa
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID11__SHIFT 0xb
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID12__SHIFT 0xc
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID13__SHIFT 0xd
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID14__SHIFT 0xe
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID15__SHIFT 0xf
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID0_MASK 0x00000001L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID1_MASK 0x00000002L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID2_MASK 0x00000004L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID3_MASK 0x00000008L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID4_MASK 0x00000010L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID5_MASK 0x00000020L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID6_MASK 0x00000040L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID7_MASK 0x00000080L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID8_MASK 0x00000100L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID9_MASK 0x00000200L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID10_MASK 0x00000400L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID11_MASK 0x00000800L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID12_MASK 0x00001000L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID13_MASK 0x00002000L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID14_MASK 0x00004000L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID15_MASK 0x00008000L
+//ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID0__SHIFT 0x0
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID1__SHIFT 0x1
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID2__SHIFT 0x2
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID3__SHIFT 0x3
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID4__SHIFT 0x4
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID5__SHIFT 0x5
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID6__SHIFT 0x6
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID7__SHIFT 0x7
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID8__SHIFT 0x8
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID9__SHIFT 0x9
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID10__SHIFT 0xa
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID11__SHIFT 0xb
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID12__SHIFT 0xc
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID13__SHIFT 0xd
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID14__SHIFT 0xe
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID15__SHIFT 0xf
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID0_MASK 0x00000001L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID1_MASK 0x00000002L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID2_MASK 0x00000004L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID3_MASK 0x00000008L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID4_MASK 0x00000010L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID5_MASK 0x00000020L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID6_MASK 0x00000040L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID7_MASK 0x00000080L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID8_MASK 0x00000100L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID9_MASK 0x00000200L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID10_MASK 0x00000400L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID11_MASK 0x00000800L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID12_MASK 0x00001000L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID13_MASK 0x00002000L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID14_MASK 0x00004000L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID15_MASK 0x00008000L
+
+
+// addressBlock: athub_xpbdec
+//XPB_RTR_SRC_APRTR0
+#define XPB_RTR_SRC_APRTR0__BASE_ADDR__SHIFT 0x0
+#define XPB_RTR_SRC_APRTR0__BASE_ADDR_MASK 0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR1
+#define XPB_RTR_SRC_APRTR1__BASE_ADDR__SHIFT 0x0
+#define XPB_RTR_SRC_APRTR1__BASE_ADDR_MASK 0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR2
+#define XPB_RTR_SRC_APRTR2__BASE_ADDR__SHIFT 0x0
+#define XPB_RTR_SRC_APRTR2__BASE_ADDR_MASK 0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR3
+#define XPB_RTR_SRC_APRTR3__BASE_ADDR__SHIFT 0x0
+#define XPB_RTR_SRC_APRTR3__BASE_ADDR_MASK 0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR4
+#define XPB_RTR_SRC_APRTR4__BASE_ADDR__SHIFT 0x0
+#define XPB_RTR_SRC_APRTR4__BASE_ADDR_MASK 0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR5
+#define XPB_RTR_SRC_APRTR5__BASE_ADDR__SHIFT 0x0
+#define XPB_RTR_SRC_APRTR5__BASE_ADDR_MASK 0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR6
+#define XPB_RTR_SRC_APRTR6__BASE_ADDR__SHIFT 0x0
+#define XPB_RTR_SRC_APRTR6__BASE_ADDR_MASK 0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR7
+#define XPB_RTR_SRC_APRTR7__BASE_ADDR__SHIFT 0x0
+#define XPB_RTR_SRC_APRTR7__BASE_ADDR_MASK 0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR8
+#define XPB_RTR_SRC_APRTR8__BASE_ADDR__SHIFT 0x0
+#define XPB_RTR_SRC_APRTR8__BASE_ADDR_MASK 0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR9
+#define XPB_RTR_SRC_APRTR9__BASE_ADDR__SHIFT 0x0
+#define XPB_RTR_SRC_APRTR9__BASE_ADDR_MASK 0x7FFFFFFFL
+//XPB_XDMA_RTR_SRC_APRTR0
+#define XPB_XDMA_RTR_SRC_APRTR0__BASE_ADDR__SHIFT 0x0
+#define XPB_XDMA_RTR_SRC_APRTR0__BASE_ADDR_MASK 0x7FFFFFFFL
+//XPB_XDMA_RTR_SRC_APRTR1
+#define XPB_XDMA_RTR_SRC_APRTR1__BASE_ADDR__SHIFT 0x0
+#define XPB_XDMA_RTR_SRC_APRTR1__BASE_ADDR_MASK 0x7FFFFFFFL
+//XPB_XDMA_RTR_SRC_APRTR2
+#define XPB_XDMA_RTR_SRC_APRTR2__BASE_ADDR__SHIFT 0x0
+#define XPB_XDMA_RTR_SRC_APRTR2__BASE_ADDR_MASK 0x7FFFFFFFL
+//XPB_XDMA_RTR_SRC_APRTR3
+#define XPB_XDMA_RTR_SRC_APRTR3__BASE_ADDR__SHIFT 0x0
+#define XPB_XDMA_RTR_SRC_APRTR3__BASE_ADDR_MASK 0x7FFFFFFFL
+//XPB_RTR_DEST_MAP0
+#define XPB_RTR_DEST_MAP0__NMR__SHIFT 0x0
+#define XPB_RTR_DEST_MAP0__DEST_OFFSET__SHIFT 0x1
+#define XPB_RTR_DEST_MAP0__DEST_SEL__SHIFT 0x14
+#define XPB_RTR_DEST_MAP0__DEST_SEL_RPB__SHIFT 0x18
+#define XPB_RTR_DEST_MAP0__SIDE_OK__SHIFT 0x19
+#define XPB_RTR_DEST_MAP0__APRTR_SIZE__SHIFT 0x1a
+#define XPB_RTR_DEST_MAP0__NMR_MASK 0x00000001L
+#define XPB_RTR_DEST_MAP0__DEST_OFFSET_MASK 0x000FFFFEL
+#define XPB_RTR_DEST_MAP0__DEST_SEL_MASK 0x00F00000L
+#define XPB_RTR_DEST_MAP0__DEST_SEL_RPB_MASK 0x01000000L
+#define XPB_RTR_DEST_MAP0__SIDE_OK_MASK 0x02000000L
+#define XPB_RTR_DEST_MAP0__APRTR_SIZE_MASK 0x7C000000L
+//XPB_RTR_DEST_MAP1
+#define XPB_RTR_DEST_MAP1__NMR__SHIFT 0x0
+#define XPB_RTR_DEST_MAP1__DEST_OFFSET__SHIFT 0x1
+#define XPB_RTR_DEST_MAP1__DEST_SEL__SHIFT 0x14
+#define XPB_RTR_DEST_MAP1__DEST_SEL_RPB__SHIFT 0x18
+#define XPB_RTR_DEST_MAP1__SIDE_OK__SHIFT 0x19
+#define XPB_RTR_DEST_MAP1__APRTR_SIZE__SHIFT 0x1a
+#define XPB_RTR_DEST_MAP1__NMR_MASK 0x00000001L
+#define XPB_RTR_DEST_MAP1__DEST_OFFSET_MASK 0x000FFFFEL
+#define XPB_RTR_DEST_MAP1__DEST_SEL_MASK 0x00F00000L
+#define XPB_RTR_DEST_MAP1__DEST_SEL_RPB_MASK 0x01000000L
+#define XPB_RTR_DEST_MAP1__SIDE_OK_MASK 0x02000000L
+#define XPB_RTR_DEST_MAP1__APRTR_SIZE_MASK 0x7C000000L
+//XPB_RTR_DEST_MAP2
+#define XPB_RTR_DEST_MAP2__NMR__SHIFT 0x0
+#define XPB_RTR_DEST_MAP2__DEST_OFFSET__SHIFT 0x1
+#define XPB_RTR_DEST_MAP2__DEST_SEL__SHIFT 0x14
+#define XPB_RTR_DEST_MAP2__DEST_SEL_RPB__SHIFT 0x18
+#define XPB_RTR_DEST_MAP2__SIDE_OK__SHIFT 0x19
+#define XPB_RTR_DEST_MAP2__APRTR_SIZE__SHIFT 0x1a
+#define XPB_RTR_DEST_MAP2__NMR_MASK 0x00000001L
+#define XPB_RTR_DEST_MAP2__DEST_OFFSET_MASK 0x000FFFFEL
+#define XPB_RTR_DEST_MAP2__DEST_SEL_MASK 0x00F00000L
+#define XPB_RTR_DEST_MAP2__DEST_SEL_RPB_MASK 0x01000000L
+#define XPB_RTR_DEST_MAP2__SIDE_OK_MASK 0x02000000L
+#define XPB_RTR_DEST_MAP2__APRTR_SIZE_MASK 0x7C000000L
+//XPB_RTR_DEST_MAP3
+#define XPB_RTR_DEST_MAP3__NMR__SHIFT 0x0
+#define XPB_RTR_DEST_MAP3__DEST_OFFSET__SHIFT 0x1
+#define XPB_RTR_DEST_MAP3__DEST_SEL__SHIFT 0x14
+#define XPB_RTR_DEST_MAP3__DEST_SEL_RPB__SHIFT 0x18
+#define XPB_RTR_DEST_MAP3__SIDE_OK__SHIFT 0x19
+#define XPB_RTR_DEST_MAP3__APRTR_SIZE__SHIFT 0x1a
+#define XPB_RTR_DEST_MAP3__NMR_MASK 0x00000001L
+#define XPB_RTR_DEST_MAP3__DEST_OFFSET_MASK 0x000FFFFEL
+#define XPB_RTR_DEST_MAP3__DEST_SEL_MASK 0x00F00000L
+#define XPB_RTR_DEST_MAP3__DEST_SEL_RPB_MASK 0x01000000L
+#define XPB_RTR_DEST_MAP3__SIDE_OK_MASK 0x02000000L
+#define XPB_RTR_DEST_MAP3__APRTR_SIZE_MASK 0x7C000000L
+//XPB_RTR_DEST_MAP4
+#define XPB_RTR_DEST_MAP4__NMR__SHIFT 0x0
+#define XPB_RTR_DEST_MAP4__DEST_OFFSET__SHIFT 0x1
+#define XPB_RTR_DEST_MAP4__DEST_SEL__SHIFT 0x14
+#define XPB_RTR_DEST_MAP4__DEST_SEL_RPB__SHIFT 0x18
+#define XPB_RTR_DEST_MAP4__SIDE_OK__SHIFT 0x19
+#define XPB_RTR_DEST_MAP4__APRTR_SIZE__SHIFT 0x1a
+#define XPB_RTR_DEST_MAP4__NMR_MASK 0x00000001L
+#define XPB_RTR_DEST_MAP4__DEST_OFFSET_MASK 0x000FFFFEL
+#define XPB_RTR_DEST_MAP4__DEST_SEL_MASK 0x00F00000L
+#define XPB_RTR_DEST_MAP4__DEST_SEL_RPB_MASK 0x01000000L
+#define XPB_RTR_DEST_MAP4__SIDE_OK_MASK 0x02000000L
+#define XPB_RTR_DEST_MAP4__APRTR_SIZE_MASK 0x7C000000L
+//XPB_RTR_DEST_MAP5
+#define XPB_RTR_DEST_MAP5__NMR__SHIFT 0x0
+#define XPB_RTR_DEST_MAP5__DEST_OFFSET__SHIFT 0x1
+#define XPB_RTR_DEST_MAP5__DEST_SEL__SHIFT 0x14
+#define XPB_RTR_DEST_MAP5__DEST_SEL_RPB__SHIFT 0x18
+#define XPB_RTR_DEST_MAP5__SIDE_OK__SHIFT 0x19
+#define XPB_RTR_DEST_MAP5__APRTR_SIZE__SHIFT 0x1a
+#define XPB_RTR_DEST_MAP5__NMR_MASK 0x00000001L
+#define XPB_RTR_DEST_MAP5__DEST_OFFSET_MASK 0x000FFFFEL
+#define XPB_RTR_DEST_MAP5__DEST_SEL_MASK 0x00F00000L
+#define XPB_RTR_DEST_MAP5__DEST_SEL_RPB_MASK 0x01000000L
+#define XPB_RTR_DEST_MAP5__SIDE_OK_MASK 0x02000000L
+#define XPB_RTR_DEST_MAP5__APRTR_SIZE_MASK 0x7C000000L
+//XPB_RTR_DEST_MAP6
+#define XPB_RTR_DEST_MAP6__NMR__SHIFT 0x0
+#define XPB_RTR_DEST_MAP6__DEST_OFFSET__SHIFT 0x1
+#define XPB_RTR_DEST_MAP6__DEST_SEL__SHIFT 0x14
+#define XPB_RTR_DEST_MAP6__DEST_SEL_RPB__SHIFT 0x18
+#define XPB_RTR_DEST_MAP6__SIDE_OK__SHIFT 0x19
+#define XPB_RTR_DEST_MAP6__APRTR_SIZE__SHIFT 0x1a
+#define XPB_RTR_DEST_MAP6__NMR_MASK 0x00000001L
+#define XPB_RTR_DEST_MAP6__DEST_OFFSET_MASK 0x000FFFFEL
+#define XPB_RTR_DEST_MAP6__DEST_SEL_MASK 0x00F00000L
+#define XPB_RTR_DEST_MAP6__DEST_SEL_RPB_MASK 0x01000000L
+#define XPB_RTR_DEST_MAP6__SIDE_OK_MASK 0x02000000L
+#define XPB_RTR_DEST_MAP6__APRTR_SIZE_MASK 0x7C000000L
+//XPB_RTR_DEST_MAP7
+#define XPB_RTR_DEST_MAP7__NMR__SHIFT 0x0
+#define XPB_RTR_DEST_MAP7__DEST_OFFSET__SHIFT 0x1
+#define XPB_RTR_DEST_MAP7__DEST_SEL__SHIFT 0x14
+#define XPB_RTR_DEST_MAP7__DEST_SEL_RPB__SHIFT 0x18
+#define XPB_RTR_DEST_MAP7__SIDE_OK__SHIFT 0x19
+#define XPB_RTR_DEST_MAP7__APRTR_SIZE__SHIFT 0x1a
+#define XPB_RTR_DEST_MAP7__NMR_MASK 0x00000001L
+#define XPB_RTR_DEST_MAP7__DEST_OFFSET_MASK 0x000FFFFEL
+#define XPB_RTR_DEST_MAP7__DEST_SEL_MASK 0x00F00000L
+#define XPB_RTR_DEST_MAP7__DEST_SEL_RPB_MASK 0x01000000L
+#define XPB_RTR_DEST_MAP7__SIDE_OK_MASK 0x02000000L
+#define XPB_RTR_DEST_MAP7__APRTR_SIZE_MASK 0x7C000000L
+//XPB_RTR_DEST_MAP8
+#define XPB_RTR_DEST_MAP8__NMR__SHIFT 0x0
+#define XPB_RTR_DEST_MAP8__DEST_OFFSET__SHIFT 0x1
+#define XPB_RTR_DEST_MAP8__DEST_SEL__SHIFT 0x14
+#define XPB_RTR_DEST_MAP8__DEST_SEL_RPB__SHIFT 0x18
+#define XPB_RTR_DEST_MAP8__SIDE_OK__SHIFT 0x19
+#define XPB_RTR_DEST_MAP8__APRTR_SIZE__SHIFT 0x1a
+#define XPB_RTR_DEST_MAP8__NMR_MASK 0x00000001L
+#define XPB_RTR_DEST_MAP8__DEST_OFFSET_MASK 0x000FFFFEL
+#define XPB_RTR_DEST_MAP8__DEST_SEL_MASK 0x00F00000L
+#define XPB_RTR_DEST_MAP8__DEST_SEL_RPB_MASK 0x01000000L
+#define XPB_RTR_DEST_MAP8__SIDE_OK_MASK 0x02000000L
+#define XPB_RTR_DEST_MAP8__APRTR_SIZE_MASK 0x7C000000L
+//XPB_RTR_DEST_MAP9
+#define XPB_RTR_DEST_MAP9__NMR__SHIFT 0x0
+#define XPB_RTR_DEST_MAP9__DEST_OFFSET__SHIFT 0x1
+#define XPB_RTR_DEST_MAP9__DEST_SEL__SHIFT 0x14
+#define XPB_RTR_DEST_MAP9__DEST_SEL_RPB__SHIFT 0x18
+#define XPB_RTR_DEST_MAP9__SIDE_OK__SHIFT 0x19
+#define XPB_RTR_DEST_MAP9__APRTR_SIZE__SHIFT 0x1a
+#define XPB_RTR_DEST_MAP9__NMR_MASK 0x00000001L
+#define XPB_RTR_DEST_MAP9__DEST_OFFSET_MASK 0x000FFFFEL
+#define XPB_RTR_DEST_MAP9__DEST_SEL_MASK 0x00F00000L
+#define XPB_RTR_DEST_MAP9__DEST_SEL_RPB_MASK 0x01000000L
+#define XPB_RTR_DEST_MAP9__SIDE_OK_MASK 0x02000000L
+#define XPB_RTR_DEST_MAP9__APRTR_SIZE_MASK 0x7C000000L
+//XPB_XDMA_RTR_DEST_MAP0
+#define XPB_XDMA_RTR_DEST_MAP0__NMR__SHIFT 0x0
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_OFFSET__SHIFT 0x1
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_SEL__SHIFT 0x14
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_SEL_RPB__SHIFT 0x18
+#define XPB_XDMA_RTR_DEST_MAP0__SIDE_OK__SHIFT 0x19
+#define XPB_XDMA_RTR_DEST_MAP0__APRTR_SIZE__SHIFT 0x1a
+#define XPB_XDMA_RTR_DEST_MAP0__NMR_MASK 0x00000001L
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_OFFSET_MASK 0x000FFFFEL
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_SEL_MASK 0x00F00000L
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_SEL_RPB_MASK 0x01000000L
+#define XPB_XDMA_RTR_DEST_MAP0__SIDE_OK_MASK 0x02000000L
+#define XPB_XDMA_RTR_DEST_MAP0__APRTR_SIZE_MASK 0x7C000000L
+//XPB_XDMA_RTR_DEST_MAP1
+#define XPB_XDMA_RTR_DEST_MAP1__NMR__SHIFT 0x0
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_OFFSET__SHIFT 0x1
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_SEL__SHIFT 0x14
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_SEL_RPB__SHIFT 0x18
+#define XPB_XDMA_RTR_DEST_MAP1__SIDE_OK__SHIFT 0x19
+#define XPB_XDMA_RTR_DEST_MAP1__APRTR_SIZE__SHIFT 0x1a
+#define XPB_XDMA_RTR_DEST_MAP1__NMR_MASK 0x00000001L
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_OFFSET_MASK 0x000FFFFEL
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_SEL_MASK 0x00F00000L
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_SEL_RPB_MASK 0x01000000L
+#define XPB_XDMA_RTR_DEST_MAP1__SIDE_OK_MASK 0x02000000L
+#define XPB_XDMA_RTR_DEST_MAP1__APRTR_SIZE_MASK 0x7C000000L
+//XPB_XDMA_RTR_DEST_MAP2
+#define XPB_XDMA_RTR_DEST_MAP2__NMR__SHIFT 0x0
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_OFFSET__SHIFT 0x1
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_SEL__SHIFT 0x14
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_SEL_RPB__SHIFT 0x18
+#define XPB_XDMA_RTR_DEST_MAP2__SIDE_OK__SHIFT 0x19
+#define XPB_XDMA_RTR_DEST_MAP2__APRTR_SIZE__SHIFT 0x1a
+#define XPB_XDMA_RTR_DEST_MAP2__NMR_MASK 0x00000001L
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_OFFSET_MASK 0x000FFFFEL
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_SEL_MASK 0x00F00000L
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_SEL_RPB_MASK 0x01000000L
+#define XPB_XDMA_RTR_DEST_MAP2__SIDE_OK_MASK 0x02000000L
+#define XPB_XDMA_RTR_DEST_MAP2__APRTR_SIZE_MASK 0x7C000000L
+//XPB_XDMA_RTR_DEST_MAP3
+#define XPB_XDMA_RTR_DEST_MAP3__NMR__SHIFT 0x0
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_OFFSET__SHIFT 0x1
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_SEL__SHIFT 0x14
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_SEL_RPB__SHIFT 0x18
+#define XPB_XDMA_RTR_DEST_MAP3__SIDE_OK__SHIFT 0x19
+#define XPB_XDMA_RTR_DEST_MAP3__APRTR_SIZE__SHIFT 0x1a
+#define XPB_XDMA_RTR_DEST_MAP3__NMR_MASK 0x00000001L
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_OFFSET_MASK 0x000FFFFEL
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_SEL_MASK 0x00F00000L
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_SEL_RPB_MASK 0x01000000L
+#define XPB_XDMA_RTR_DEST_MAP3__SIDE_OK_MASK 0x02000000L
+#define XPB_XDMA_RTR_DEST_MAP3__APRTR_SIZE_MASK 0x7C000000L
+//XPB_CLG_CFG0
+#define XPB_CLG_CFG0__WCB_NUM__SHIFT 0x0
+#define XPB_CLG_CFG0__LB_TYPE__SHIFT 0x4
+#define XPB_CLG_CFG0__P2P_BAR__SHIFT 0x7
+#define XPB_CLG_CFG0__HOST_FLUSH__SHIFT 0xa
+#define XPB_CLG_CFG0__SIDE_FLUSH__SHIFT 0xe
+#define XPB_CLG_CFG0__WCB_NUM_MASK 0x0000000FL
+#define XPB_CLG_CFG0__LB_TYPE_MASK 0x00000070L
+#define XPB_CLG_CFG0__P2P_BAR_MASK 0x00000380L
+#define XPB_CLG_CFG0__HOST_FLUSH_MASK 0x00003C00L
+#define XPB_CLG_CFG0__SIDE_FLUSH_MASK 0x0003C000L
+//XPB_CLG_CFG1
+#define XPB_CLG_CFG1__WCB_NUM__SHIFT 0x0
+#define XPB_CLG_CFG1__LB_TYPE__SHIFT 0x4
+#define XPB_CLG_CFG1__P2P_BAR__SHIFT 0x7
+#define XPB_CLG_CFG1__HOST_FLUSH__SHIFT 0xa
+#define XPB_CLG_CFG1__SIDE_FLUSH__SHIFT 0xe
+#define XPB_CLG_CFG1__WCB_NUM_MASK 0x0000000FL
+#define XPB_CLG_CFG1__LB_TYPE_MASK 0x00000070L
+#define XPB_CLG_CFG1__P2P_BAR_MASK 0x00000380L
+#define XPB_CLG_CFG1__HOST_FLUSH_MASK 0x00003C00L
+#define XPB_CLG_CFG1__SIDE_FLUSH_MASK 0x0003C000L
+//XPB_CLG_CFG2
+#define XPB_CLG_CFG2__WCB_NUM__SHIFT 0x0
+#define XPB_CLG_CFG2__LB_TYPE__SHIFT 0x4
+#define XPB_CLG_CFG2__P2P_BAR__SHIFT 0x7
+#define XPB_CLG_CFG2__HOST_FLUSH__SHIFT 0xa
+#define XPB_CLG_CFG2__SIDE_FLUSH__SHIFT 0xe
+#define XPB_CLG_CFG2__WCB_NUM_MASK 0x0000000FL
+#define XPB_CLG_CFG2__LB_TYPE_MASK 0x00000070L
+#define XPB_CLG_CFG2__P2P_BAR_MASK 0x00000380L
+#define XPB_CLG_CFG2__HOST_FLUSH_MASK 0x00003C00L
+#define XPB_CLG_CFG2__SIDE_FLUSH_MASK 0x0003C000L
+//XPB_CLG_CFG3
+#define XPB_CLG_CFG3__WCB_NUM__SHIFT 0x0
+#define XPB_CLG_CFG3__LB_TYPE__SHIFT 0x4
+#define XPB_CLG_CFG3__P2P_BAR__SHIFT 0x7
+#define XPB_CLG_CFG3__HOST_FLUSH__SHIFT 0xa
+#define XPB_CLG_CFG3__SIDE_FLUSH__SHIFT 0xe
+#define XPB_CLG_CFG3__WCB_NUM_MASK 0x0000000FL
+#define XPB_CLG_CFG3__LB_TYPE_MASK 0x00000070L
+#define XPB_CLG_CFG3__P2P_BAR_MASK 0x00000380L
+#define XPB_CLG_CFG3__HOST_FLUSH_MASK 0x00003C00L
+#define XPB_CLG_CFG3__SIDE_FLUSH_MASK 0x0003C000L
+//XPB_CLG_CFG4
+#define XPB_CLG_CFG4__WCB_NUM__SHIFT 0x0
+#define XPB_CLG_CFG4__LB_TYPE__SHIFT 0x4
+#define XPB_CLG_CFG4__P2P_BAR__SHIFT 0x7
+#define XPB_CLG_CFG4__HOST_FLUSH__SHIFT 0xa
+#define XPB_CLG_CFG4__SIDE_FLUSH__SHIFT 0xe
+#define XPB_CLG_CFG4__WCB_NUM_MASK 0x0000000FL
+#define XPB_CLG_CFG4__LB_TYPE_MASK 0x00000070L
+#define XPB_CLG_CFG4__P2P_BAR_MASK 0x00000380L
+#define XPB_CLG_CFG4__HOST_FLUSH_MASK 0x00003C00L
+#define XPB_CLG_CFG4__SIDE_FLUSH_MASK 0x0003C000L
+//XPB_CLG_CFG5
+#define XPB_CLG_CFG5__WCB_NUM__SHIFT 0x0
+#define XPB_CLG_CFG5__LB_TYPE__SHIFT 0x4
+#define XPB_CLG_CFG5__P2P_BAR__SHIFT 0x7
+#define XPB_CLG_CFG5__HOST_FLUSH__SHIFT 0xa
+#define XPB_CLG_CFG5__SIDE_FLUSH__SHIFT 0xe
+#define XPB_CLG_CFG5__WCB_NUM_MASK 0x0000000FL
+#define XPB_CLG_CFG5__LB_TYPE_MASK 0x00000070L
+#define XPB_CLG_CFG5__P2P_BAR_MASK 0x00000380L
+#define XPB_CLG_CFG5__HOST_FLUSH_MASK 0x00003C00L
+#define XPB_CLG_CFG5__SIDE_FLUSH_MASK 0x0003C000L
+//XPB_CLG_CFG6
+#define XPB_CLG_CFG6__WCB_NUM__SHIFT 0x0
+#define XPB_CLG_CFG6__LB_TYPE__SHIFT 0x4
+#define XPB_CLG_CFG6__P2P_BAR__SHIFT 0x7
+#define XPB_CLG_CFG6__HOST_FLUSH__SHIFT 0xa
+#define XPB_CLG_CFG6__SIDE_FLUSH__SHIFT 0xe
+#define XPB_CLG_CFG6__WCB_NUM_MASK 0x0000000FL
+#define XPB_CLG_CFG6__LB_TYPE_MASK 0x00000070L
+#define XPB_CLG_CFG6__P2P_BAR_MASK 0x00000380L
+#define XPB_CLG_CFG6__HOST_FLUSH_MASK 0x00003C00L
+#define XPB_CLG_CFG6__SIDE_FLUSH_MASK 0x0003C000L
+//XPB_CLG_CFG7
+#define XPB_CLG_CFG7__WCB_NUM__SHIFT 0x0
+#define XPB_CLG_CFG7__LB_TYPE__SHIFT 0x4
+#define XPB_CLG_CFG7__P2P_BAR__SHIFT 0x7
+#define XPB_CLG_CFG7__HOST_FLUSH__SHIFT 0xa
+#define XPB_CLG_CFG7__SIDE_FLUSH__SHIFT 0xe
+#define XPB_CLG_CFG7__WCB_NUM_MASK 0x0000000FL
+#define XPB_CLG_CFG7__LB_TYPE_MASK 0x00000070L
+#define XPB_CLG_CFG7__P2P_BAR_MASK 0x00000380L
+#define XPB_CLG_CFG7__HOST_FLUSH_MASK 0x00003C00L
+#define XPB_CLG_CFG7__SIDE_FLUSH_MASK 0x0003C000L
+//XPB_CLG_EXTRA
+#define XPB_CLG_EXTRA__CMP0_HIGH__SHIFT 0x0
+#define XPB_CLG_EXTRA__CMP0_LOW__SHIFT 0x6
+#define XPB_CLG_EXTRA__VLD0__SHIFT 0xb
+#define XPB_CLG_EXTRA__CLG0_NUM__SHIFT 0xc
+#define XPB_CLG_EXTRA__CMP1_HIGH__SHIFT 0xf
+#define XPB_CLG_EXTRA__CMP1_LOW__SHIFT 0x15
+#define XPB_CLG_EXTRA__VLD1__SHIFT 0x1a
+#define XPB_CLG_EXTRA__CLG1_NUM__SHIFT 0x1b
+#define XPB_CLG_EXTRA__CMP0_HIGH_MASK 0x0000003FL
+#define XPB_CLG_EXTRA__CMP0_LOW_MASK 0x000007C0L
+#define XPB_CLG_EXTRA__VLD0_MASK 0x00000800L
+#define XPB_CLG_EXTRA__CLG0_NUM_MASK 0x00007000L
+#define XPB_CLG_EXTRA__CMP1_HIGH_MASK 0x001F8000L
+#define XPB_CLG_EXTRA__CMP1_LOW_MASK 0x03E00000L
+#define XPB_CLG_EXTRA__VLD1_MASK 0x04000000L
+#define XPB_CLG_EXTRA__CLG1_NUM_MASK 0x38000000L
+//XPB_CLG_EXTRA_MSK
+#define XPB_CLG_EXTRA_MSK__MSK0_HIGH__SHIFT 0x0
+#define XPB_CLG_EXTRA_MSK__MSK0_LOW__SHIFT 0x6
+#define XPB_CLG_EXTRA_MSK__MSK1_HIGH__SHIFT 0xb
+#define XPB_CLG_EXTRA_MSK__MSK1_LOW__SHIFT 0x11
+#define XPB_CLG_EXTRA_MSK__MSK0_HIGH_MASK 0x0000003FL
+#define XPB_CLG_EXTRA_MSK__MSK0_LOW_MASK 0x000007C0L
+#define XPB_CLG_EXTRA_MSK__MSK1_HIGH_MASK 0x0001F800L
+#define XPB_CLG_EXTRA_MSK__MSK1_LOW_MASK 0x003E0000L
+//XPB_LB_ADDR
+#define XPB_LB_ADDR__CMP0__SHIFT 0x0
+#define XPB_LB_ADDR__MASK0__SHIFT 0xa
+#define XPB_LB_ADDR__CMP1__SHIFT 0x14
+#define XPB_LB_ADDR__MASK1__SHIFT 0x1a
+#define XPB_LB_ADDR__CMP0_MASK 0x000003FFL
+#define XPB_LB_ADDR__MASK0_MASK 0x000FFC00L
+#define XPB_LB_ADDR__CMP1_MASK 0x03F00000L
+#define XPB_LB_ADDR__MASK1_MASK 0xFC000000L
+//XPB_WCB_STS
+#define XPB_WCB_STS__PBUF_VLD__SHIFT 0x0
+#define XPB_WCB_STS__WCB_HST_DATA_BUF_CNT__SHIFT 0x10
+#define XPB_WCB_STS__WCB_SID_DATA_BUF_CNT__SHIFT 0x17
+#define XPB_WCB_STS__PBUF_VLD_MASK 0x0000FFFFL
+#define XPB_WCB_STS__WCB_HST_DATA_BUF_CNT_MASK 0x007F0000L
+#define XPB_WCB_STS__WCB_SID_DATA_BUF_CNT_MASK 0x3F800000L
+//XPB_HST_CFG
+#define XPB_HST_CFG__BAR_UP_WR_CMD__SHIFT 0x0
+#define XPB_HST_CFG__BAR_UP_WR_CMD_MASK 0x00000001L
+//XPB_P2P_BAR_CFG
+#define XPB_P2P_BAR_CFG__ADDR_SIZE__SHIFT 0x0
+#define XPB_P2P_BAR_CFG__SEND_BAR__SHIFT 0x4
+#define XPB_P2P_BAR_CFG__SNOOP__SHIFT 0x6
+#define XPB_P2P_BAR_CFG__SEND_DIS__SHIFT 0x7
+#define XPB_P2P_BAR_CFG__COMPRESS_DIS__SHIFT 0x8
+#define XPB_P2P_BAR_CFG__UPDATE_DIS__SHIFT 0x9
+#define XPB_P2P_BAR_CFG__REGBAR_FROM_SYSBAR__SHIFT 0xa
+#define XPB_P2P_BAR_CFG__RD_EN__SHIFT 0xb
+#define XPB_P2P_BAR_CFG__ATC_TRANSLATED__SHIFT 0xc
+#define XPB_P2P_BAR_CFG__ADDR_SIZE_MASK 0x0000000FL
+#define XPB_P2P_BAR_CFG__SEND_BAR_MASK 0x00000030L
+#define XPB_P2P_BAR_CFG__SNOOP_MASK 0x00000040L
+#define XPB_P2P_BAR_CFG__SEND_DIS_MASK 0x00000080L
+#define XPB_P2P_BAR_CFG__COMPRESS_DIS_MASK 0x00000100L
+#define XPB_P2P_BAR_CFG__UPDATE_DIS_MASK 0x00000200L
+#define XPB_P2P_BAR_CFG__REGBAR_FROM_SYSBAR_MASK 0x00000400L
+#define XPB_P2P_BAR_CFG__RD_EN_MASK 0x00000800L
+#define XPB_P2P_BAR_CFG__ATC_TRANSLATED_MASK 0x00001000L
+//XPB_P2P_BAR0
+#define XPB_P2P_BAR0__HOST_FLUSH__SHIFT 0x0
+#define XPB_P2P_BAR0__REG_SYS_BAR__SHIFT 0x4
+#define XPB_P2P_BAR0__MEM_SYS_BAR__SHIFT 0x8
+#define XPB_P2P_BAR0__VALID__SHIFT 0xc
+#define XPB_P2P_BAR0__SEND_DIS__SHIFT 0xd
+#define XPB_P2P_BAR0__COMPRESS_DIS__SHIFT 0xe
+#define XPB_P2P_BAR0__RESERVED__SHIFT 0xf
+#define XPB_P2P_BAR0__ADDRESS__SHIFT 0x10
+#define XPB_P2P_BAR0__HOST_FLUSH_MASK 0x0000000FL
+#define XPB_P2P_BAR0__REG_SYS_BAR_MASK 0x000000F0L
+#define XPB_P2P_BAR0__MEM_SYS_BAR_MASK 0x00000F00L
+#define XPB_P2P_BAR0__VALID_MASK 0x00001000L
+#define XPB_P2P_BAR0__SEND_DIS_MASK 0x00002000L
+#define XPB_P2P_BAR0__COMPRESS_DIS_MASK 0x00004000L
+#define XPB_P2P_BAR0__RESERVED_MASK 0x00008000L
+#define XPB_P2P_BAR0__ADDRESS_MASK 0xFFFF0000L
+//XPB_P2P_BAR1
+#define XPB_P2P_BAR1__HOST_FLUSH__SHIFT 0x0
+#define XPB_P2P_BAR1__REG_SYS_BAR__SHIFT 0x4
+#define XPB_P2P_BAR1__MEM_SYS_BAR__SHIFT 0x8
+#define XPB_P2P_BAR1__VALID__SHIFT 0xc
+#define XPB_P2P_BAR1__SEND_DIS__SHIFT 0xd
+#define XPB_P2P_BAR1__COMPRESS_DIS__SHIFT 0xe
+#define XPB_P2P_BAR1__RESERVED__SHIFT 0xf
+#define XPB_P2P_BAR1__ADDRESS__SHIFT 0x10
+#define XPB_P2P_BAR1__HOST_FLUSH_MASK 0x0000000FL
+#define XPB_P2P_BAR1__REG_SYS_BAR_MASK 0x000000F0L
+#define XPB_P2P_BAR1__MEM_SYS_BAR_MASK 0x00000F00L
+#define XPB_P2P_BAR1__VALID_MASK 0x00001000L
+#define XPB_P2P_BAR1__SEND_DIS_MASK 0x00002000L
+#define XPB_P2P_BAR1__COMPRESS_DIS_MASK 0x00004000L
+#define XPB_P2P_BAR1__RESERVED_MASK 0x00008000L
+#define XPB_P2P_BAR1__ADDRESS_MASK 0xFFFF0000L
+//XPB_P2P_BAR2
+#define XPB_P2P_BAR2__HOST_FLUSH__SHIFT 0x0
+#define XPB_P2P_BAR2__REG_SYS_BAR__SHIFT 0x4
+#define XPB_P2P_BAR2__MEM_SYS_BAR__SHIFT 0x8
+#define XPB_P2P_BAR2__VALID__SHIFT 0xc
+#define XPB_P2P_BAR2__SEND_DIS__SHIFT 0xd
+#define XPB_P2P_BAR2__COMPRESS_DIS__SHIFT 0xe
+#define XPB_P2P_BAR2__RESERVED__SHIFT 0xf
+#define XPB_P2P_BAR2__ADDRESS__SHIFT 0x10
+#define XPB_P2P_BAR2__HOST_FLUSH_MASK 0x0000000FL
+#define XPB_P2P_BAR2__REG_SYS_BAR_MASK 0x000000F0L
+#define XPB_P2P_BAR2__MEM_SYS_BAR_MASK 0x00000F00L
+#define XPB_P2P_BAR2__VALID_MASK 0x00001000L
+#define XPB_P2P_BAR2__SEND_DIS_MASK 0x00002000L
+#define XPB_P2P_BAR2__COMPRESS_DIS_MASK 0x00004000L
+#define XPB_P2P_BAR2__RESERVED_MASK 0x00008000L
+#define XPB_P2P_BAR2__ADDRESS_MASK 0xFFFF0000L
+//XPB_P2P_BAR3
+#define XPB_P2P_BAR3__HOST_FLUSH__SHIFT 0x0
+#define XPB_P2P_BAR3__REG_SYS_BAR__SHIFT 0x4
+#define XPB_P2P_BAR3__MEM_SYS_BAR__SHIFT 0x8
+#define XPB_P2P_BAR3__VALID__SHIFT 0xc
+#define XPB_P2P_BAR3__SEND_DIS__SHIFT 0xd
+#define XPB_P2P_BAR3__COMPRESS_DIS__SHIFT 0xe
+#define XPB_P2P_BAR3__RESERVED__SHIFT 0xf
+#define XPB_P2P_BAR3__ADDRESS__SHIFT 0x10
+#define XPB_P2P_BAR3__HOST_FLUSH_MASK 0x0000000FL
+#define XPB_P2P_BAR3__REG_SYS_BAR_MASK 0x000000F0L
+#define XPB_P2P_BAR3__MEM_SYS_BAR_MASK 0x00000F00L
+#define XPB_P2P_BAR3__VALID_MASK 0x00001000L
+#define XPB_P2P_BAR3__SEND_DIS_MASK 0x00002000L
+#define XPB_P2P_BAR3__COMPRESS_DIS_MASK 0x00004000L
+#define XPB_P2P_BAR3__RESERVED_MASK 0x00008000L
+#define XPB_P2P_BAR3__ADDRESS_MASK 0xFFFF0000L
+//XPB_P2P_BAR4
+#define XPB_P2P_BAR4__HOST_FLUSH__SHIFT 0x0
+#define XPB_P2P_BAR4__REG_SYS_BAR__SHIFT 0x4
+#define XPB_P2P_BAR4__MEM_SYS_BAR__SHIFT 0x8
+#define XPB_P2P_BAR4__VALID__SHIFT 0xc
+#define XPB_P2P_BAR4__SEND_DIS__SHIFT 0xd
+#define XPB_P2P_BAR4__COMPRESS_DIS__SHIFT 0xe
+#define XPB_P2P_BAR4__RESERVED__SHIFT 0xf
+#define XPB_P2P_BAR4__ADDRESS__SHIFT 0x10
+#define XPB_P2P_BAR4__HOST_FLUSH_MASK 0x0000000FL
+#define XPB_P2P_BAR4__REG_SYS_BAR_MASK 0x000000F0L
+#define XPB_P2P_BAR4__MEM_SYS_BAR_MASK 0x00000F00L
+#define XPB_P2P_BAR4__VALID_MASK 0x00001000L
+#define XPB_P2P_BAR4__SEND_DIS_MASK 0x00002000L
+#define XPB_P2P_BAR4__COMPRESS_DIS_MASK 0x00004000L
+#define XPB_P2P_BAR4__RESERVED_MASK 0x00008000L
+#define XPB_P2P_BAR4__ADDRESS_MASK 0xFFFF0000L
+//XPB_P2P_BAR5
+#define XPB_P2P_BAR5__HOST_FLUSH__SHIFT 0x0
+#define XPB_P2P_BAR5__REG_SYS_BAR__SHIFT 0x4
+#define XPB_P2P_BAR5__MEM_SYS_BAR__SHIFT 0x8
+#define XPB_P2P_BAR5__VALID__SHIFT 0xc
+#define XPB_P2P_BAR5__SEND_DIS__SHIFT 0xd
+#define XPB_P2P_BAR5__COMPRESS_DIS__SHIFT 0xe
+#define XPB_P2P_BAR5__RESERVED__SHIFT 0xf
+#define XPB_P2P_BAR5__ADDRESS__SHIFT 0x10
+#define XPB_P2P_BAR5__HOST_FLUSH_MASK 0x0000000FL
+#define XPB_P2P_BAR5__REG_SYS_BAR_MASK 0x000000F0L
+#define XPB_P2P_BAR5__MEM_SYS_BAR_MASK 0x00000F00L
+#define XPB_P2P_BAR5__VALID_MASK 0x00001000L
+#define XPB_P2P_BAR5__SEND_DIS_MASK 0x00002000L
+#define XPB_P2P_BAR5__COMPRESS_DIS_MASK 0x00004000L
+#define XPB_P2P_BAR5__RESERVED_MASK 0x00008000L
+#define XPB_P2P_BAR5__ADDRESS_MASK 0xFFFF0000L
+//XPB_P2P_BAR6
+#define XPB_P2P_BAR6__HOST_FLUSH__SHIFT 0x0
+#define XPB_P2P_BAR6__REG_SYS_BAR__SHIFT 0x4
+#define XPB_P2P_BAR6__MEM_SYS_BAR__SHIFT 0x8
+#define XPB_P2P_BAR6__VALID__SHIFT 0xc
+#define XPB_P2P_BAR6__SEND_DIS__SHIFT 0xd
+#define XPB_P2P_BAR6__COMPRESS_DIS__SHIFT 0xe
+#define XPB_P2P_BAR6__RESERVED__SHIFT 0xf
+#define XPB_P2P_BAR6__ADDRESS__SHIFT 0x10
+#define XPB_P2P_BAR6__HOST_FLUSH_MASK 0x0000000FL
+#define XPB_P2P_BAR6__REG_SYS_BAR_MASK 0x000000F0L
+#define XPB_P2P_BAR6__MEM_SYS_BAR_MASK 0x00000F00L
+#define XPB_P2P_BAR6__VALID_MASK 0x00001000L
+#define XPB_P2P_BAR6__SEND_DIS_MASK 0x00002000L
+#define XPB_P2P_BAR6__COMPRESS_DIS_MASK 0x00004000L
+#define XPB_P2P_BAR6__RESERVED_MASK 0x00008000L
+#define XPB_P2P_BAR6__ADDRESS_MASK 0xFFFF0000L
+//XPB_P2P_BAR7
+#define XPB_P2P_BAR7__HOST_FLUSH__SHIFT 0x0
+#define XPB_P2P_BAR7__REG_SYS_BAR__SHIFT 0x4
+#define XPB_P2P_BAR7__MEM_SYS_BAR__SHIFT 0x8
+#define XPB_P2P_BAR7__VALID__SHIFT 0xc
+#define XPB_P2P_BAR7__SEND_DIS__SHIFT 0xd
+#define XPB_P2P_BAR7__COMPRESS_DIS__SHIFT 0xe
+#define XPB_P2P_BAR7__RESERVED__SHIFT 0xf
+#define XPB_P2P_BAR7__ADDRESS__SHIFT 0x10
+#define XPB_P2P_BAR7__HOST_FLUSH_MASK 0x0000000FL
+#define XPB_P2P_BAR7__REG_SYS_BAR_MASK 0x000000F0L
+#define XPB_P2P_BAR7__MEM_SYS_BAR_MASK 0x00000F00L
+#define XPB_P2P_BAR7__VALID_MASK 0x00001000L
+#define XPB_P2P_BAR7__SEND_DIS_MASK 0x00002000L
+#define XPB_P2P_BAR7__COMPRESS_DIS_MASK 0x00004000L
+#define XPB_P2P_BAR7__RESERVED_MASK 0x00008000L
+#define XPB_P2P_BAR7__ADDRESS_MASK 0xFFFF0000L
+//XPB_P2P_BAR_SETUP
+#define XPB_P2P_BAR_SETUP__SEL__SHIFT 0x0
+#define XPB_P2P_BAR_SETUP__REG_SYS_BAR__SHIFT 0x8
+#define XPB_P2P_BAR_SETUP__VALID__SHIFT 0xc
+#define XPB_P2P_BAR_SETUP__SEND_DIS__SHIFT 0xd
+#define XPB_P2P_BAR_SETUP__COMPRESS_DIS__SHIFT 0xe
+#define XPB_P2P_BAR_SETUP__RESERVED__SHIFT 0xf
+#define XPB_P2P_BAR_SETUP__ADDRESS__SHIFT 0x10
+#define XPB_P2P_BAR_SETUP__SEL_MASK 0x000000FFL
+#define XPB_P2P_BAR_SETUP__REG_SYS_BAR_MASK 0x00000F00L
+#define XPB_P2P_BAR_SETUP__VALID_MASK 0x00001000L
+#define XPB_P2P_BAR_SETUP__SEND_DIS_MASK 0x00002000L
+#define XPB_P2P_BAR_SETUP__COMPRESS_DIS_MASK 0x00004000L
+#define XPB_P2P_BAR_SETUP__RESERVED_MASK 0x00008000L
+#define XPB_P2P_BAR_SETUP__ADDRESS_MASK 0xFFFF0000L
+//XPB_P2P_BAR_DELTA_ABOVE
+#define XPB_P2P_BAR_DELTA_ABOVE__EN__SHIFT 0x0
+#define XPB_P2P_BAR_DELTA_ABOVE__DELTA__SHIFT 0x8
+#define XPB_P2P_BAR_DELTA_ABOVE__EN_MASK 0x000000FFL
+#define XPB_P2P_BAR_DELTA_ABOVE__DELTA_MASK 0x0FFFFF00L
+//XPB_P2P_BAR_DELTA_BELOW
+#define XPB_P2P_BAR_DELTA_BELOW__EN__SHIFT 0x0
+#define XPB_P2P_BAR_DELTA_BELOW__DELTA__SHIFT 0x8
+#define XPB_P2P_BAR_DELTA_BELOW__EN_MASK 0x000000FFL
+#define XPB_P2P_BAR_DELTA_BELOW__DELTA_MASK 0x0FFFFF00L
+//XPB_PEER_SYS_BAR0
+#define XPB_PEER_SYS_BAR0__VALID__SHIFT 0x0
+#define XPB_PEER_SYS_BAR0__ADDR__SHIFT 0x1
+#define XPB_PEER_SYS_BAR0__VALID_MASK 0x00000001L
+#define XPB_PEER_SYS_BAR0__ADDR_MASK 0xFFFFFFFEL
+//XPB_PEER_SYS_BAR1
+#define XPB_PEER_SYS_BAR1__VALID__SHIFT 0x0
+#define XPB_PEER_SYS_BAR1__ADDR__SHIFT 0x1
+#define XPB_PEER_SYS_BAR1__VALID_MASK 0x00000001L
+#define XPB_PEER_SYS_BAR1__ADDR_MASK 0xFFFFFFFEL
+//XPB_PEER_SYS_BAR2
+#define XPB_PEER_SYS_BAR2__VALID__SHIFT 0x0
+#define XPB_PEER_SYS_BAR2__ADDR__SHIFT 0x1
+#define XPB_PEER_SYS_BAR2__VALID_MASK 0x00000001L
+#define XPB_PEER_SYS_BAR2__ADDR_MASK 0xFFFFFFFEL
+//XPB_PEER_SYS_BAR3
+#define XPB_PEER_SYS_BAR3__VALID__SHIFT 0x0
+#define XPB_PEER_SYS_BAR3__ADDR__SHIFT 0x1
+#define XPB_PEER_SYS_BAR3__VALID_MASK 0x00000001L
+#define XPB_PEER_SYS_BAR3__ADDR_MASK 0xFFFFFFFEL
+//XPB_PEER_SYS_BAR4
+#define XPB_PEER_SYS_BAR4__VALID__SHIFT 0x0
+#define XPB_PEER_SYS_BAR4__ADDR__SHIFT 0x1
+#define XPB_PEER_SYS_BAR4__VALID_MASK 0x00000001L
+#define XPB_PEER_SYS_BAR4__ADDR_MASK 0xFFFFFFFEL
+//XPB_PEER_SYS_BAR5
+#define XPB_PEER_SYS_BAR5__VALID__SHIFT 0x0
+#define XPB_PEER_SYS_BAR5__ADDR__SHIFT 0x1
+#define XPB_PEER_SYS_BAR5__VALID_MASK 0x00000001L
+#define XPB_PEER_SYS_BAR5__ADDR_MASK 0xFFFFFFFEL
+//XPB_PEER_SYS_BAR6
+#define XPB_PEER_SYS_BAR6__VALID__SHIFT 0x0
+#define XPB_PEER_SYS_BAR6__ADDR__SHIFT 0x1
+#define XPB_PEER_SYS_BAR6__VALID_MASK 0x00000001L
+#define XPB_PEER_SYS_BAR6__ADDR_MASK 0xFFFFFFFEL
+//XPB_PEER_SYS_BAR7
+#define XPB_PEER_SYS_BAR7__VALID__SHIFT 0x0
+#define XPB_PEER_SYS_BAR7__ADDR__SHIFT 0x1
+#define XPB_PEER_SYS_BAR7__VALID_MASK 0x00000001L
+#define XPB_PEER_SYS_BAR7__ADDR_MASK 0xFFFFFFFEL
+//XPB_PEER_SYS_BAR8
+#define XPB_PEER_SYS_BAR8__VALID__SHIFT 0x0
+#define XPB_PEER_SYS_BAR8__ADDR__SHIFT 0x1
+#define XPB_PEER_SYS_BAR8__VALID_MASK 0x00000001L
+#define XPB_PEER_SYS_BAR8__ADDR_MASK 0xFFFFFFFEL
+//XPB_PEER_SYS_BAR9
+#define XPB_PEER_SYS_BAR9__VALID__SHIFT 0x0
+#define XPB_PEER_SYS_BAR9__ADDR__SHIFT 0x1
+#define XPB_PEER_SYS_BAR9__VALID_MASK 0x00000001L
+#define XPB_PEER_SYS_BAR9__ADDR_MASK 0xFFFFFFFEL
+//XPB_XDMA_PEER_SYS_BAR0
+#define XPB_XDMA_PEER_SYS_BAR0__VALID__SHIFT 0x0
+#define XPB_XDMA_PEER_SYS_BAR0__ADDR__SHIFT 0x1
+#define XPB_XDMA_PEER_SYS_BAR0__VALID_MASK 0x00000001L
+#define XPB_XDMA_PEER_SYS_BAR0__ADDR_MASK 0xFFFFFFFEL
+//XPB_XDMA_PEER_SYS_BAR1
+#define XPB_XDMA_PEER_SYS_BAR1__VALID__SHIFT 0x0
+#define XPB_XDMA_PEER_SYS_BAR1__ADDR__SHIFT 0x1
+#define XPB_XDMA_PEER_SYS_BAR1__VALID_MASK 0x00000001L
+#define XPB_XDMA_PEER_SYS_BAR1__ADDR_MASK 0xFFFFFFFEL
+//XPB_XDMA_PEER_SYS_BAR2
+#define XPB_XDMA_PEER_SYS_BAR2__VALID__SHIFT 0x0
+#define XPB_XDMA_PEER_SYS_BAR2__ADDR__SHIFT 0x1
+#define XPB_XDMA_PEER_SYS_BAR2__VALID_MASK 0x00000001L
+#define XPB_XDMA_PEER_SYS_BAR2__ADDR_MASK 0xFFFFFFFEL
+//XPB_XDMA_PEER_SYS_BAR3
+#define XPB_XDMA_PEER_SYS_BAR3__VALID__SHIFT 0x0
+#define XPB_XDMA_PEER_SYS_BAR3__ADDR__SHIFT 0x1
+#define XPB_XDMA_PEER_SYS_BAR3__VALID_MASK 0x00000001L
+#define XPB_XDMA_PEER_SYS_BAR3__ADDR_MASK 0xFFFFFFFEL
+//XPB_CLK_GAT
+#define XPB_CLK_GAT__ONDLY__SHIFT 0x0
+#define XPB_CLK_GAT__OFFDLY__SHIFT 0x6
+#define XPB_CLK_GAT__RDYDLY__SHIFT 0xc
+#define XPB_CLK_GAT__ENABLE__SHIFT 0x12
+#define XPB_CLK_GAT__MEM_LS_ENABLE__SHIFT 0x13
+#define XPB_CLK_GAT__ONDLY_MASK 0x0000003FL
+#define XPB_CLK_GAT__OFFDLY_MASK 0x00000FC0L
+#define XPB_CLK_GAT__RDYDLY_MASK 0x0003F000L
+#define XPB_CLK_GAT__ENABLE_MASK 0x00040000L
+#define XPB_CLK_GAT__MEM_LS_ENABLE_MASK 0x00080000L
+//XPB_INTF_CFG
+#define XPB_INTF_CFG__RPB_WRREQ_CRD__SHIFT 0x0
+#define XPB_INTF_CFG__MC_WRRET_ASK__SHIFT 0x8
+#define XPB_INTF_CFG__XSP_REQ_CRD__SHIFT 0x10
+#define XPB_INTF_CFG__BIF_REG_SNOOP_SEL__SHIFT 0x17
+#define XPB_INTF_CFG__BIF_REG_SNOOP_VAL__SHIFT 0x18
+#define XPB_INTF_CFG__BIF_MEM_SNOOP_SEL__SHIFT 0x19
+#define XPB_INTF_CFG__BIF_MEM_SNOOP_VAL__SHIFT 0x1a
+#define XPB_INTF_CFG__XSP_SNOOP_SEL__SHIFT 0x1b
+#define XPB_INTF_CFG__XSP_SNOOP_VAL__SHIFT 0x1d
+#define XPB_INTF_CFG__XSP_ORDERING_SEL__SHIFT 0x1e
+#define XPB_INTF_CFG__XSP_ORDERING_VAL__SHIFT 0x1f
+#define XPB_INTF_CFG__RPB_WRREQ_CRD_MASK 0x000000FFL
+#define XPB_INTF_CFG__MC_WRRET_ASK_MASK 0x0000FF00L
+#define XPB_INTF_CFG__XSP_REQ_CRD_MASK 0x007F0000L
+#define XPB_INTF_CFG__BIF_REG_SNOOP_SEL_MASK 0x00800000L
+#define XPB_INTF_CFG__BIF_REG_SNOOP_VAL_MASK 0x01000000L
+#define XPB_INTF_CFG__BIF_MEM_SNOOP_SEL_MASK 0x02000000L
+#define XPB_INTF_CFG__BIF_MEM_SNOOP_VAL_MASK 0x04000000L
+#define XPB_INTF_CFG__XSP_SNOOP_SEL_MASK 0x18000000L
+#define XPB_INTF_CFG__XSP_SNOOP_VAL_MASK 0x20000000L
+#define XPB_INTF_CFG__XSP_ORDERING_SEL_MASK 0x40000000L
+#define XPB_INTF_CFG__XSP_ORDERING_VAL_MASK 0x80000000L
+//XPB_INTF_STS
+#define XPB_INTF_STS__RPB_WRREQ_CRD__SHIFT 0x0
+#define XPB_INTF_STS__XSP_REQ_CRD__SHIFT 0x8
+#define XPB_INTF_STS__HOP_DATA_BUF_FULL__SHIFT 0xf
+#define XPB_INTF_STS__HOP_ATTR_BUF_FULL__SHIFT 0x10
+#define XPB_INTF_STS__CNS_BUF_FULL__SHIFT 0x11
+#define XPB_INTF_STS__CNS_BUF_BUSY__SHIFT 0x12
+#define XPB_INTF_STS__RPB_RDREQ_CRD__SHIFT 0x13
+#define XPB_INTF_STS__RPB_WRREQ_CRD_MASK 0x000000FFL
+#define XPB_INTF_STS__XSP_REQ_CRD_MASK 0x00007F00L
+#define XPB_INTF_STS__HOP_DATA_BUF_FULL_MASK 0x00008000L
+#define XPB_INTF_STS__HOP_ATTR_BUF_FULL_MASK 0x00010000L
+#define XPB_INTF_STS__CNS_BUF_FULL_MASK 0x00020000L
+#define XPB_INTF_STS__CNS_BUF_BUSY_MASK 0x00040000L
+#define XPB_INTF_STS__RPB_RDREQ_CRD_MASK 0x07F80000L
+//XPB_PIPE_STS
+#define XPB_PIPE_STS__WCB_ANY_PBUF__SHIFT 0x0
+#define XPB_PIPE_STS__WCB_HST_DATA_BUF_CNT__SHIFT 0x1
+#define XPB_PIPE_STS__WCB_SID_DATA_BUF_CNT__SHIFT 0x8
+#define XPB_PIPE_STS__WCB_HST_RD_PTR_BUF_FULL__SHIFT 0xf
+#define XPB_PIPE_STS__WCB_SID_RD_PTR_BUF_FULL__SHIFT 0x10
+#define XPB_PIPE_STS__WCB_HST_REQ_FIFO_FULL__SHIFT 0x11
+#define XPB_PIPE_STS__WCB_SID_REQ_FIFO_FULL__SHIFT 0x12
+#define XPB_PIPE_STS__WCB_HST_REQ_OBUF_FULL__SHIFT 0x13
+#define XPB_PIPE_STS__WCB_SID_REQ_OBUF_FULL__SHIFT 0x14
+#define XPB_PIPE_STS__WCB_HST_DATA_OBUF_FULL__SHIFT 0x15
+#define XPB_PIPE_STS__WCB_SID_DATA_OBUF_FULL__SHIFT 0x16
+#define XPB_PIPE_STS__RET_BUF_FULL__SHIFT 0x17
+#define XPB_PIPE_STS__XPB_CLK_BUSY_BITS__SHIFT 0x18
+#define XPB_PIPE_STS__WCB_ANY_PBUF_MASK 0x00000001L
+#define XPB_PIPE_STS__WCB_HST_DATA_BUF_CNT_MASK 0x000000FEL
+#define XPB_PIPE_STS__WCB_SID_DATA_BUF_CNT_MASK 0x00007F00L
+#define XPB_PIPE_STS__WCB_HST_RD_PTR_BUF_FULL_MASK 0x00008000L
+#define XPB_PIPE_STS__WCB_SID_RD_PTR_BUF_FULL_MASK 0x00010000L
+#define XPB_PIPE_STS__WCB_HST_REQ_FIFO_FULL_MASK 0x00020000L
+#define XPB_PIPE_STS__WCB_SID_REQ_FIFO_FULL_MASK 0x00040000L
+#define XPB_PIPE_STS__WCB_HST_REQ_OBUF_FULL_MASK 0x00080000L
+#define XPB_PIPE_STS__WCB_SID_REQ_OBUF_FULL_MASK 0x00100000L
+#define XPB_PIPE_STS__WCB_HST_DATA_OBUF_FULL_MASK 0x00200000L
+#define XPB_PIPE_STS__WCB_SID_DATA_OBUF_FULL_MASK 0x00400000L
+#define XPB_PIPE_STS__RET_BUF_FULL_MASK 0x00800000L
+#define XPB_PIPE_STS__XPB_CLK_BUSY_BITS_MASK 0xFF000000L
+//XPB_SUB_CTRL
+#define XPB_SUB_CTRL__WRREQ_BYPASS_XPB__SHIFT 0x0
+#define XPB_SUB_CTRL__STALL_CNS_RTR_REQ__SHIFT 0x1
+#define XPB_SUB_CTRL__STALL_RTR_RPB_WRREQ__SHIFT 0x2
+#define XPB_SUB_CTRL__STALL_RTR_MAP_REQ__SHIFT 0x3
+#define XPB_SUB_CTRL__STALL_MAP_WCB_REQ__SHIFT 0x4
+#define XPB_SUB_CTRL__STALL_WCB_SID_REQ__SHIFT 0x5
+#define XPB_SUB_CTRL__STALL_MC_XSP_REQ_SEND__SHIFT 0x6
+#define XPB_SUB_CTRL__STALL_WCB_HST_REQ__SHIFT 0x7
+#define XPB_SUB_CTRL__STALL_HST_HOP_REQ__SHIFT 0x8
+#define XPB_SUB_CTRL__STALL_XPB_RPB_REQ_ATTR__SHIFT 0x9
+#define XPB_SUB_CTRL__RESET_CNS__SHIFT 0xa
+#define XPB_SUB_CTRL__RESET_RTR__SHIFT 0xb
+#define XPB_SUB_CTRL__RESET_RET__SHIFT 0xc
+#define XPB_SUB_CTRL__RESET_MAP__SHIFT 0xd
+#define XPB_SUB_CTRL__RESET_WCB__SHIFT 0xe
+#define XPB_SUB_CTRL__RESET_HST__SHIFT 0xf
+#define XPB_SUB_CTRL__RESET_HOP__SHIFT 0x10
+#define XPB_SUB_CTRL__RESET_SID__SHIFT 0x11
+#define XPB_SUB_CTRL__RESET_SRB__SHIFT 0x12
+#define XPB_SUB_CTRL__RESET_CGR__SHIFT 0x13
+#define XPB_SUB_CTRL__WRREQ_BYPASS_XPB_MASK 0x00000001L
+#define XPB_SUB_CTRL__STALL_CNS_RTR_REQ_MASK 0x00000002L
+#define XPB_SUB_CTRL__STALL_RTR_RPB_WRREQ_MASK 0x00000004L
+#define XPB_SUB_CTRL__STALL_RTR_MAP_REQ_MASK 0x00000008L
+#define XPB_SUB_CTRL__STALL_MAP_WCB_REQ_MASK 0x00000010L
+#define XPB_SUB_CTRL__STALL_WCB_SID_REQ_MASK 0x00000020L
+#define XPB_SUB_CTRL__STALL_MC_XSP_REQ_SEND_MASK 0x00000040L
+#define XPB_SUB_CTRL__STALL_WCB_HST_REQ_MASK 0x00000080L
+#define XPB_SUB_CTRL__STALL_HST_HOP_REQ_MASK 0x00000100L
+#define XPB_SUB_CTRL__STALL_XPB_RPB_REQ_ATTR_MASK 0x00000200L
+#define XPB_SUB_CTRL__RESET_CNS_MASK 0x00000400L
+#define XPB_SUB_CTRL__RESET_RTR_MASK 0x00000800L
+#define XPB_SUB_CTRL__RESET_RET_MASK 0x00001000L
+#define XPB_SUB_CTRL__RESET_MAP_MASK 0x00002000L
+#define XPB_SUB_CTRL__RESET_WCB_MASK 0x00004000L
+#define XPB_SUB_CTRL__RESET_HST_MASK 0x00008000L
+#define XPB_SUB_CTRL__RESET_HOP_MASK 0x00010000L
+#define XPB_SUB_CTRL__RESET_SID_MASK 0x00020000L
+#define XPB_SUB_CTRL__RESET_SRB_MASK 0x00040000L
+#define XPB_SUB_CTRL__RESET_CGR_MASK 0x00080000L
+//XPB_MAP_INVERT_FLUSH_NUM_LSB
+#define XPB_MAP_INVERT_FLUSH_NUM_LSB__ALTER_FLUSH_NUM__SHIFT 0x0
+#define XPB_MAP_INVERT_FLUSH_NUM_LSB__ALTER_FLUSH_NUM_MASK 0x0000FFFFL
+//XPB_PERF_KNOBS
+#define XPB_PERF_KNOBS__CNS_FIFO_DEPTH__SHIFT 0x0
+#define XPB_PERF_KNOBS__WCB_HST_FIFO_DEPTH__SHIFT 0x6
+#define XPB_PERF_KNOBS__WCB_SID_FIFO_DEPTH__SHIFT 0xc
+#define XPB_PERF_KNOBS__CNS_FIFO_DEPTH_MASK 0x0000003FL
+#define XPB_PERF_KNOBS__WCB_HST_FIFO_DEPTH_MASK 0x00000FC0L
+#define XPB_PERF_KNOBS__WCB_SID_FIFO_DEPTH_MASK 0x0003F000L
+//XPB_STICKY
+#define XPB_STICKY__BITS__SHIFT 0x0
+#define XPB_STICKY__BITS_MASK 0xFFFFFFFFL
+//XPB_STICKY_W1C
+#define XPB_STICKY_W1C__BITS__SHIFT 0x0
+#define XPB_STICKY_W1C__BITS_MASK 0xFFFFFFFFL
+//XPB_MISC_CFG
+#define XPB_MISC_CFG__FIELDNAME0__SHIFT 0x0
+#define XPB_MISC_CFG__FIELDNAME1__SHIFT 0x8
+#define XPB_MISC_CFG__FIELDNAME2__SHIFT 0x10
+#define XPB_MISC_CFG__FIELDNAME3__SHIFT 0x18
+#define XPB_MISC_CFG__TRIGGERNAME__SHIFT 0x1f
+#define XPB_MISC_CFG__FIELDNAME0_MASK 0x000000FFL
+#define XPB_MISC_CFG__FIELDNAME1_MASK 0x0000FF00L
+#define XPB_MISC_CFG__FIELDNAME2_MASK 0x00FF0000L
+#define XPB_MISC_CFG__FIELDNAME3_MASK 0x7F000000L
+#define XPB_MISC_CFG__TRIGGERNAME_MASK 0x80000000L
+//XPB_INTF_CFG2
+#define XPB_INTF_CFG2__RPB_RDREQ_CRD__SHIFT 0x0
+#define XPB_INTF_CFG2__RPB_RDREQ_CRD_MASK 0x000000FFL
+//XPB_CLG_EXTRA_RD
+#define XPB_CLG_EXTRA_RD__CMP0_HIGH__SHIFT 0x0
+#define XPB_CLG_EXTRA_RD__CMP0_LOW__SHIFT 0x6
+#define XPB_CLG_EXTRA_RD__VLD0__SHIFT 0xb
+#define XPB_CLG_EXTRA_RD__CLG0_NUM__SHIFT 0xc
+#define XPB_CLG_EXTRA_RD__CMP1_HIGH__SHIFT 0xf
+#define XPB_CLG_EXTRA_RD__CMP1_LOW__SHIFT 0x15
+#define XPB_CLG_EXTRA_RD__VLD1__SHIFT 0x1a
+#define XPB_CLG_EXTRA_RD__CLG1_NUM__SHIFT 0x1b
+#define XPB_CLG_EXTRA_RD__CMP0_HIGH_MASK 0x0000003FL
+#define XPB_CLG_EXTRA_RD__CMP0_LOW_MASK 0x000007C0L
+#define XPB_CLG_EXTRA_RD__VLD0_MASK 0x00000800L
+#define XPB_CLG_EXTRA_RD__CLG0_NUM_MASK 0x00007000L
+#define XPB_CLG_EXTRA_RD__CMP1_HIGH_MASK 0x001F8000L
+#define XPB_CLG_EXTRA_RD__CMP1_LOW_MASK 0x03E00000L
+#define XPB_CLG_EXTRA_RD__VLD1_MASK 0x04000000L
+#define XPB_CLG_EXTRA_RD__CLG1_NUM_MASK 0x38000000L
+//XPB_CLG_EXTRA_MSK_RD
+#define XPB_CLG_EXTRA_MSK_RD__MSK0_HIGH__SHIFT 0x0
+#define XPB_CLG_EXTRA_MSK_RD__MSK0_LOW__SHIFT 0x6
+#define XPB_CLG_EXTRA_MSK_RD__MSK1_HIGH__SHIFT 0xb
+#define XPB_CLG_EXTRA_MSK_RD__MSK1_LOW__SHIFT 0x11
+#define XPB_CLG_EXTRA_MSK_RD__MSK0_HIGH_MASK 0x0000003FL
+#define XPB_CLG_EXTRA_MSK_RD__MSK0_LOW_MASK 0x000007C0L
+#define XPB_CLG_EXTRA_MSK_RD__MSK1_HIGH_MASK 0x0001F800L
+#define XPB_CLG_EXTRA_MSK_RD__MSK1_LOW_MASK 0x003E0000L
+//XPB_CLG_GFX_MATCH
+#define XPB_CLG_GFX_MATCH__FARBIRC0_ID__SHIFT 0x0
+#define XPB_CLG_GFX_MATCH__FARBIRC1_ID__SHIFT 0x6
+#define XPB_CLG_GFX_MATCH__FARBIRC2_ID__SHIFT 0xc
+#define XPB_CLG_GFX_MATCH__FARBIRC3_ID__SHIFT 0x12
+#define XPB_CLG_GFX_MATCH__FARBIRC0_VLD__SHIFT 0x18
+#define XPB_CLG_GFX_MATCH__FARBIRC1_VLD__SHIFT 0x19
+#define XPB_CLG_GFX_MATCH__FARBIRC2_VLD__SHIFT 0x1a
+#define XPB_CLG_GFX_MATCH__FARBIRC3_VLD__SHIFT 0x1b
+#define XPB_CLG_GFX_MATCH__FARBIRC0_ID_MASK 0x0000003FL
+#define XPB_CLG_GFX_MATCH__FARBIRC1_ID_MASK 0x00000FC0L
+#define XPB_CLG_GFX_MATCH__FARBIRC2_ID_MASK 0x0003F000L
+#define XPB_CLG_GFX_MATCH__FARBIRC3_ID_MASK 0x00FC0000L
+#define XPB_CLG_GFX_MATCH__FARBIRC0_VLD_MASK 0x01000000L
+#define XPB_CLG_GFX_MATCH__FARBIRC1_VLD_MASK 0x02000000L
+#define XPB_CLG_GFX_MATCH__FARBIRC2_VLD_MASK 0x04000000L
+#define XPB_CLG_GFX_MATCH__FARBIRC3_VLD_MASK 0x08000000L
+//XPB_CLG_GFX_MATCH_MSK
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC0_ID_MSK__SHIFT 0x0
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC1_ID_MSK__SHIFT 0x6
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC2_ID_MSK__SHIFT 0xc
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC3_ID_MSK__SHIFT 0x12
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC0_ID_MSK_MASK 0x0000003FL
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC1_ID_MSK_MASK 0x00000FC0L
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC2_ID_MSK_MASK 0x0003F000L
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC3_ID_MSK_MASK 0x00FC0000L
+//XPB_CLG_MM_MATCH
+#define XPB_CLG_MM_MATCH__FARBIRC0_ID__SHIFT 0x0
+#define XPB_CLG_MM_MATCH__FARBIRC1_ID__SHIFT 0x6
+#define XPB_CLG_MM_MATCH__FARBIRC0_VLD__SHIFT 0xc
+#define XPB_CLG_MM_MATCH__FARBIRC1_VLD__SHIFT 0xd
+#define XPB_CLG_MM_MATCH__FARBIRC0_ID_MASK 0x0000003FL
+#define XPB_CLG_MM_MATCH__FARBIRC1_ID_MASK 0x00000FC0L
+#define XPB_CLG_MM_MATCH__FARBIRC0_VLD_MASK 0x00001000L
+#define XPB_CLG_MM_MATCH__FARBIRC1_VLD_MASK 0x00002000L
+//XPB_CLG_MM_MATCH_MSK
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC0_ID_MSK__SHIFT 0x0
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC1_ID_MSK__SHIFT 0x6
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC0_ID_MSK_MASK 0x0000003FL
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC1_ID_MSK_MASK 0x00000FC0L
+//XPB_CLG_GUS_MATCH
+#define XPB_CLG_GUS_MATCH__FARBIRC0_ID__SHIFT 0x0
+#define XPB_CLG_GUS_MATCH__FARBIRC0_VLD__SHIFT 0x6
+#define XPB_CLG_GUS_MATCH__FARBIRC0_ID_MASK 0x0000003FL
+#define XPB_CLG_GUS_MATCH__FARBIRC0_VLD_MASK 0x00000040L
+//XPB_CLG_GUS_MATCH_MSK
+#define XPB_CLG_GUS_MATCH_MSK__FARBIRC0_ID_MSK__SHIFT 0x0
+#define XPB_CLG_GUS_MATCH_MSK__FARBIRC0_ID_MSK_MASK 0x0000003FL
+//XPB_CLG_GFX_UNITID_MAPPING0
+#define XPB_CLG_GFX_UNITID_MAPPING0__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_GFX_UNITID_MAPPING0__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_GFX_UNITID_MAPPING0__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_GFX_UNITID_MAPPING0__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING0__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING0__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING1
+#define XPB_CLG_GFX_UNITID_MAPPING1__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_GFX_UNITID_MAPPING1__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_GFX_UNITID_MAPPING1__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_GFX_UNITID_MAPPING1__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING1__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING1__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING2
+#define XPB_CLG_GFX_UNITID_MAPPING2__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_GFX_UNITID_MAPPING2__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_GFX_UNITID_MAPPING2__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_GFX_UNITID_MAPPING2__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING2__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING2__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING3
+#define XPB_CLG_GFX_UNITID_MAPPING3__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_GFX_UNITID_MAPPING3__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_GFX_UNITID_MAPPING3__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_GFX_UNITID_MAPPING3__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING3__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING3__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING4
+#define XPB_CLG_GFX_UNITID_MAPPING4__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_GFX_UNITID_MAPPING4__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_GFX_UNITID_MAPPING4__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_GFX_UNITID_MAPPING4__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING4__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING4__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING5
+#define XPB_CLG_GFX_UNITID_MAPPING5__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_GFX_UNITID_MAPPING5__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_GFX_UNITID_MAPPING5__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_GFX_UNITID_MAPPING5__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING5__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING5__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING6
+#define XPB_CLG_GFX_UNITID_MAPPING6__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_GFX_UNITID_MAPPING6__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_GFX_UNITID_MAPPING6__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_GFX_UNITID_MAPPING6__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING6__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING6__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING7
+#define XPB_CLG_GFX_UNITID_MAPPING7__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_GFX_UNITID_MAPPING7__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_GFX_UNITID_MAPPING7__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_GFX_UNITID_MAPPING7__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING7__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING7__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_MM_UNITID_MAPPING0
+#define XPB_CLG_MM_UNITID_MAPPING0__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_MM_UNITID_MAPPING0__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_MM_UNITID_MAPPING0__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_MM_UNITID_MAPPING0__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_MM_UNITID_MAPPING0__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_MM_UNITID_MAPPING0__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_MM_UNITID_MAPPING1
+#define XPB_CLG_MM_UNITID_MAPPING1__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_MM_UNITID_MAPPING1__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_MM_UNITID_MAPPING1__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_MM_UNITID_MAPPING1__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_MM_UNITID_MAPPING1__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_MM_UNITID_MAPPING1__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_MM_UNITID_MAPPING2
+#define XPB_CLG_MM_UNITID_MAPPING2__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_MM_UNITID_MAPPING2__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_MM_UNITID_MAPPING2__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_MM_UNITID_MAPPING2__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_MM_UNITID_MAPPING2__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_MM_UNITID_MAPPING2__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_MM_UNITID_MAPPING3
+#define XPB_CLG_MM_UNITID_MAPPING3__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_MM_UNITID_MAPPING3__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_MM_UNITID_MAPPING3__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_MM_UNITID_MAPPING3__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_MM_UNITID_MAPPING3__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_MM_UNITID_MAPPING3__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_GUS_UNITID_MAPPING0
+#define XPB_CLG_GUS_UNITID_MAPPING0__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_GUS_UNITID_MAPPING0__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_GUS_UNITID_MAPPING0__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_GUS_UNITID_MAPPING0__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_GUS_UNITID_MAPPING0__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_GUS_UNITID_MAPPING0__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_GUS_UNITID_MAPPING1
+#define XPB_CLG_GUS_UNITID_MAPPING1__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_GUS_UNITID_MAPPING1__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_GUS_UNITID_MAPPING1__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_GUS_UNITID_MAPPING1__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_GUS_UNITID_MAPPING1__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_GUS_UNITID_MAPPING1__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_GUS_UNITID_MAPPING2
+#define XPB_CLG_GUS_UNITID_MAPPING2__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_GUS_UNITID_MAPPING2__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_GUS_UNITID_MAPPING2__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_GUS_UNITID_MAPPING2__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_GUS_UNITID_MAPPING2__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_GUS_UNITID_MAPPING2__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_GUS_UNITID_MAPPING3
+#define XPB_CLG_GUS_UNITID_MAPPING3__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_GUS_UNITID_MAPPING3__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_GUS_UNITID_MAPPING3__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_GUS_UNITID_MAPPING3__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_GUS_UNITID_MAPPING3__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_GUS_UNITID_MAPPING3__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_GUS_UNITID_MAPPING4
+#define XPB_CLG_GUS_UNITID_MAPPING4__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_GUS_UNITID_MAPPING4__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_GUS_UNITID_MAPPING4__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_GUS_UNITID_MAPPING4__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_GUS_UNITID_MAPPING4__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_GUS_UNITID_MAPPING4__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_GUS_UNITID_MAPPING5
+#define XPB_CLG_GUS_UNITID_MAPPING5__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_GUS_UNITID_MAPPING5__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_GUS_UNITID_MAPPING5__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_GUS_UNITID_MAPPING5__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_GUS_UNITID_MAPPING5__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_GUS_UNITID_MAPPING5__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_GUS_UNITID_MAPPING6
+#define XPB_CLG_GUS_UNITID_MAPPING6__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_GUS_UNITID_MAPPING6__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_GUS_UNITID_MAPPING6__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_GUS_UNITID_MAPPING6__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_GUS_UNITID_MAPPING6__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_GUS_UNITID_MAPPING6__DEST_CLG_NUM_MASK 0x000001C0L
+//XPB_CLG_GUS_UNITID_MAPPING7
+#define XPB_CLG_GUS_UNITID_MAPPING7__UNITID_LOW__SHIFT 0x0
+#define XPB_CLG_GUS_UNITID_MAPPING7__UNITID_VLD__SHIFT 0x5
+#define XPB_CLG_GUS_UNITID_MAPPING7__DEST_CLG_NUM__SHIFT 0x6
+#define XPB_CLG_GUS_UNITID_MAPPING7__UNITID_LOW_MASK 0x0000001FL
+#define XPB_CLG_GUS_UNITID_MAPPING7__UNITID_VLD_MASK 0x00000020L
+#define XPB_CLG_GUS_UNITID_MAPPING7__DEST_CLG_NUM_MASK 0x000001C0L
+
+
+// addressBlock: athub_rpbdec
+//RPB_PASSPW_CONF
+#define RPB_PASSPW_CONF__XPB_PASSPW_OVERRIDE__SHIFT 0x0
+#define RPB_PASSPW_CONF__XPB_RSPPASSPW_OVERRIDE__SHIFT 0x1
+#define RPB_PASSPW_CONF__ATC_TR_PASSPW_OVERRIDE__SHIFT 0x2
+#define RPB_PASSPW_CONF__ATC_PAGE_PASSPW_OVERRIDE__SHIFT 0x3
+#define RPB_PASSPW_CONF__WR_PASSPW_OVERRIDE__SHIFT 0x4
+#define RPB_PASSPW_CONF__RD_PASSPW_OVERRIDE__SHIFT 0x5
+#define RPB_PASSPW_CONF__WR_RSPPASSPW_OVERRIDE__SHIFT 0x6
+#define RPB_PASSPW_CONF__RD_RSPPASSPW_OVERRIDE__SHIFT 0x7
+#define RPB_PASSPW_CONF__ATC_RSPPASSPW_OVERRIDE__SHIFT 0x8
+#define RPB_PASSPW_CONF__ATOMIC_PASSPW_OVERRIDE__SHIFT 0x9
+#define RPB_PASSPW_CONF__ATOMIC_RSPPASSPW_OVERRIDE__SHIFT 0xa
+#define RPB_PASSPW_CONF__ATC_TR_PASSPW_OVERRIDE_EN__SHIFT 0xb
+#define RPB_PASSPW_CONF__ATC_PAGE_PASSPW_OVERRIDE_EN__SHIFT 0xc
+#define RPB_PASSPW_CONF__ATC_RSPPASSPW_OVERRIDE_EN__SHIFT 0xd
+#define RPB_PASSPW_CONF__WRRSP_PASSPW_OVERRIDE__SHIFT 0xe
+#define RPB_PASSPW_CONF__WRRSP_PASSPW_OVERRIDE_EN__SHIFT 0xf
+#define RPB_PASSPW_CONF__RDRSP_PASSPW_OVERRIDE__SHIFT 0x10
+#define RPB_PASSPW_CONF__RDRSP_PASSPW_OVERRIDE_EN__SHIFT 0x11
+#define RPB_PASSPW_CONF__XPB_PASSPW_OVERRIDE_MASK 0x00000001L
+#define RPB_PASSPW_CONF__XPB_RSPPASSPW_OVERRIDE_MASK 0x00000002L
+#define RPB_PASSPW_CONF__ATC_TR_PASSPW_OVERRIDE_MASK 0x00000004L
+#define RPB_PASSPW_CONF__ATC_PAGE_PASSPW_OVERRIDE_MASK 0x00000008L
+#define RPB_PASSPW_CONF__WR_PASSPW_OVERRIDE_MASK 0x00000010L
+#define RPB_PASSPW_CONF__RD_PASSPW_OVERRIDE_MASK 0x00000020L
+#define RPB_PASSPW_CONF__WR_RSPPASSPW_OVERRIDE_MASK 0x00000040L
+#define RPB_PASSPW_CONF__RD_RSPPASSPW_OVERRIDE_MASK 0x00000080L
+#define RPB_PASSPW_CONF__ATC_RSPPASSPW_OVERRIDE_MASK 0x00000100L
+#define RPB_PASSPW_CONF__ATOMIC_PASSPW_OVERRIDE_MASK 0x00000200L
+#define RPB_PASSPW_CONF__ATOMIC_RSPPASSPW_OVERRIDE_MASK 0x00000400L
+#define RPB_PASSPW_CONF__ATC_TR_PASSPW_OVERRIDE_EN_MASK 0x00000800L
+#define RPB_PASSPW_CONF__ATC_PAGE_PASSPW_OVERRIDE_EN_MASK 0x00001000L
+#define RPB_PASSPW_CONF__ATC_RSPPASSPW_OVERRIDE_EN_MASK 0x00002000L
+#define RPB_PASSPW_CONF__WRRSP_PASSPW_OVERRIDE_MASK 0x00004000L
+#define RPB_PASSPW_CONF__WRRSP_PASSPW_OVERRIDE_EN_MASK 0x00008000L
+#define RPB_PASSPW_CONF__RDRSP_PASSPW_OVERRIDE_MASK 0x00010000L
+#define RPB_PASSPW_CONF__RDRSP_PASSPW_OVERRIDE_EN_MASK 0x00020000L
+//RPB_BLOCKLEVEL_CONF
+#define RPB_BLOCKLEVEL_CONF__XPB_BLOCKLEVEL_OVERRIDE__SHIFT 0x0
+#define RPB_BLOCKLEVEL_CONF__ATC_TR_BLOCKLEVEL__SHIFT 0x2
+#define RPB_BLOCKLEVEL_CONF__ATC_PAGE_BLOCKLEVEL__SHIFT 0x4
+#define RPB_BLOCKLEVEL_CONF__ATC_INV_BLOCKLEVEL__SHIFT 0x6
+#define RPB_BLOCKLEVEL_CONF__IO_WR_BLOCKLEVEL_OVERRIDE__SHIFT 0x8
+#define RPB_BLOCKLEVEL_CONF__IO_RD_BLOCKLEVEL_OVERRIDE__SHIFT 0xa
+#define RPB_BLOCKLEVEL_CONF__ATOMIC_BLOCKLEVEL_OVERRIDE__SHIFT 0xc
+#define RPB_BLOCKLEVEL_CONF__XPB_BLOCKLEVEL_OVERRIDE_EN__SHIFT 0xe
+#define RPB_BLOCKLEVEL_CONF__IO_WR_BLOCKLEVEL_OVERRIDE_EN__SHIFT 0xf
+#define RPB_BLOCKLEVEL_CONF__IO_RD_BLOCKLEVEL_OVERRIDE_EN__SHIFT 0x10
+#define RPB_BLOCKLEVEL_CONF__ATOMIC_BLOCKLEVEL_OVERRIDE_EN__SHIFT 0x11
+#define RPB_BLOCKLEVEL_CONF__XPB_BLOCKLEVEL_OVERRIDE_MASK 0x00000003L
+#define RPB_BLOCKLEVEL_CONF__ATC_TR_BLOCKLEVEL_MASK 0x0000000CL
+#define RPB_BLOCKLEVEL_CONF__ATC_PAGE_BLOCKLEVEL_MASK 0x00000030L
+#define RPB_BLOCKLEVEL_CONF__ATC_INV_BLOCKLEVEL_MASK 0x000000C0L
+#define RPB_BLOCKLEVEL_CONF__IO_WR_BLOCKLEVEL_OVERRIDE_MASK 0x00000300L
+#define RPB_BLOCKLEVEL_CONF__IO_RD_BLOCKLEVEL_OVERRIDE_MASK 0x00000C00L
+#define RPB_BLOCKLEVEL_CONF__ATOMIC_BLOCKLEVEL_OVERRIDE_MASK 0x00003000L
+#define RPB_BLOCKLEVEL_CONF__XPB_BLOCKLEVEL_OVERRIDE_EN_MASK 0x00004000L
+#define RPB_BLOCKLEVEL_CONF__IO_WR_BLOCKLEVEL_OVERRIDE_EN_MASK 0x00008000L
+#define RPB_BLOCKLEVEL_CONF__IO_RD_BLOCKLEVEL_OVERRIDE_EN_MASK 0x00010000L
+#define RPB_BLOCKLEVEL_CONF__ATOMIC_BLOCKLEVEL_OVERRIDE_EN_MASK 0x00020000L
+//RPB_TAG_CONF
+#define RPB_TAG_CONF__RPB_ATS_TR__SHIFT 0x0
+#define RPB_TAG_CONF__RPB_IO_WR__SHIFT 0xa
+#define RPB_TAG_CONF__RPB_ATS_PR__SHIFT 0x14
+#define RPB_TAG_CONF__RPB_ATS_TR_MASK 0x000003FFL
+#define RPB_TAG_CONF__RPB_IO_WR_MASK 0x000FFC00L
+#define RPB_TAG_CONF__RPB_ATS_PR_MASK 0x3FF00000L
+//RPB_EFF_CNTL
+#define RPB_EFF_CNTL__WR_LAZY_TIMER__SHIFT 0x0
+#define RPB_EFF_CNTL__RD_LAZY_TIMER__SHIFT 0x8
+#define RPB_EFF_CNTL__WR_LAZY_TIMER_MASK 0x000000FFL
+#define RPB_EFF_CNTL__RD_LAZY_TIMER_MASK 0x0000FF00L
+//RPB_ARB_CNTL
+#define RPB_ARB_CNTL__RD_SWITCH_NUM__SHIFT 0x0
+#define RPB_ARB_CNTL__WR_SWITCH_NUM__SHIFT 0x8
+#define RPB_ARB_CNTL__ATC_TR_SWITCH_NUM__SHIFT 0x10
+#define RPB_ARB_CNTL__ARB_MODE__SHIFT 0x18
+#define RPB_ARB_CNTL__SWITCH_NUM_MODE__SHIFT 0x19
+#define RPB_ARB_CNTL__RD_SWITCH_NUM_MASK 0x000000FFL
+#define RPB_ARB_CNTL__WR_SWITCH_NUM_MASK 0x0000FF00L
+#define RPB_ARB_CNTL__ATC_TR_SWITCH_NUM_MASK 0x00FF0000L
+#define RPB_ARB_CNTL__ARB_MODE_MASK 0x01000000L
+#define RPB_ARB_CNTL__SWITCH_NUM_MODE_MASK 0x02000000L
+//RPB_ARB_CNTL2
+#define RPB_ARB_CNTL2__P2P_SWITCH_NUM__SHIFT 0x0
+#define RPB_ARB_CNTL2__ATOMIC_SWITCH_NUM__SHIFT 0x8
+#define RPB_ARB_CNTL2__ATC_PAGE_SWITCH_NUM__SHIFT 0x10
+#define RPB_ARB_CNTL2__P2P_SWITCH_NUM_MASK 0x000000FFL
+#define RPB_ARB_CNTL2__ATOMIC_SWITCH_NUM_MASK 0x0000FF00L
+#define RPB_ARB_CNTL2__ATC_PAGE_SWITCH_NUM_MASK 0x00FF0000L
+//RPB_BIF_CNTL
+#define RPB_BIF_CNTL__VC0_SWITCH_NUM__SHIFT 0x0
+#define RPB_BIF_CNTL__VC1_SWITCH_NUM__SHIFT 0x8
+#define RPB_BIF_CNTL__ARB_MODE__SHIFT 0x10
+#define RPB_BIF_CNTL__DRAIN_VC_NUM__SHIFT 0x11
+#define RPB_BIF_CNTL__SWITCH_ENABLE__SHIFT 0x12
+#define RPB_BIF_CNTL__SWITCH_THRESHOLD__SHIFT 0x13
+#define RPB_BIF_CNTL__PAGE_PRI_EN__SHIFT 0x1b
+#define RPB_BIF_CNTL__TR_PRI_EN__SHIFT 0x1c
+#define RPB_BIF_CNTL__VC0_CHAINED_OVERRIDE__SHIFT 0x1d
+#define RPB_BIF_CNTL__PARITY_CHECK_EN__SHIFT 0x1e
+#define RPB_BIF_CNTL__NBIF_DMA_ORIGCLKCTL_EN__SHIFT 0x1f
+#define RPB_BIF_CNTL__VC0_SWITCH_NUM_MASK 0x000000FFL
+#define RPB_BIF_CNTL__VC1_SWITCH_NUM_MASK 0x0000FF00L
+#define RPB_BIF_CNTL__ARB_MODE_MASK 0x00010000L
+#define RPB_BIF_CNTL__DRAIN_VC_NUM_MASK 0x00020000L
+#define RPB_BIF_CNTL__SWITCH_ENABLE_MASK 0x00040000L
+#define RPB_BIF_CNTL__SWITCH_THRESHOLD_MASK 0x07F80000L
+#define RPB_BIF_CNTL__PAGE_PRI_EN_MASK 0x08000000L
+#define RPB_BIF_CNTL__TR_PRI_EN_MASK 0x10000000L
+#define RPB_BIF_CNTL__VC0_CHAINED_OVERRIDE_MASK 0x20000000L
+#define RPB_BIF_CNTL__PARITY_CHECK_EN_MASK 0x40000000L
+#define RPB_BIF_CNTL__NBIF_DMA_ORIGCLKCTL_EN_MASK 0x80000000L
+//RPB_WR_SWITCH_CNTL
+#define RPB_WR_SWITCH_CNTL__QUEUE0_SWITCH_NUM__SHIFT 0x0
+#define RPB_WR_SWITCH_CNTL__QUEUE1_SWITCH_NUM__SHIFT 0x7
+#define RPB_WR_SWITCH_CNTL__QUEUE2_SWITCH_NUM__SHIFT 0xe
+#define RPB_WR_SWITCH_CNTL__QUEUE3_SWITCH_NUM__SHIFT 0x15
+#define RPB_WR_SWITCH_CNTL__SWITCH_NUM_MODE__SHIFT 0x1c
+#define RPB_WR_SWITCH_CNTL__WORKLOAD_ADJUST_EN__SHIFT 0x1d
+#define RPB_WR_SWITCH_CNTL__WEIGHT_ADJUST_STEP__SHIFT 0x1e
+#define RPB_WR_SWITCH_CNTL__QUEUE0_SWITCH_NUM_MASK 0x0000007FL
+#define RPB_WR_SWITCH_CNTL__QUEUE1_SWITCH_NUM_MASK 0x00003F80L
+#define RPB_WR_SWITCH_CNTL__QUEUE2_SWITCH_NUM_MASK 0x001FC000L
+#define RPB_WR_SWITCH_CNTL__QUEUE3_SWITCH_NUM_MASK 0x0FE00000L
+#define RPB_WR_SWITCH_CNTL__SWITCH_NUM_MODE_MASK 0x10000000L
+#define RPB_WR_SWITCH_CNTL__WORKLOAD_ADJUST_EN_MASK 0x20000000L
+#define RPB_WR_SWITCH_CNTL__WEIGHT_ADJUST_STEP_MASK 0xC0000000L
+//RPB_WR_COMBINE_CNTL
+#define RPB_WR_COMBINE_CNTL__WC_MAX_PACKET_SIZE__SHIFT 0x0
+#define RPB_WR_COMBINE_CNTL__WC_FLUSH_TIMER__SHIFT 0x2
+#define RPB_WR_COMBINE_CNTL__WC_ALIGN__SHIFT 0x6
+#define RPB_WR_COMBINE_CNTL__WC_MAX_PACKET_SIZE_MASK 0x00000003L
+#define RPB_WR_COMBINE_CNTL__WC_FLUSH_TIMER_MASK 0x0000003CL
+#define RPB_WR_COMBINE_CNTL__WC_ALIGN_MASK 0x00000040L
+//RPB_RD_SWITCH_CNTL
+#define RPB_RD_SWITCH_CNTL__QUEUE0_SWITCH_NUM__SHIFT 0x0
+#define RPB_RD_SWITCH_CNTL__QUEUE1_SWITCH_NUM__SHIFT 0x7
+#define RPB_RD_SWITCH_CNTL__QUEUE2_SWITCH_NUM__SHIFT 0xe
+#define RPB_RD_SWITCH_CNTL__QUEUE3_SWITCH_NUM__SHIFT 0x15
+#define RPB_RD_SWITCH_CNTL__SWITCH_NUM_MODE__SHIFT 0x1c
+#define RPB_RD_SWITCH_CNTL__WORKLOAD_ADJUST_EN__SHIFT 0x1d
+#define RPB_RD_SWITCH_CNTL__WEIGHT_ADJUST_STEP__SHIFT 0x1e
+#define RPB_RD_SWITCH_CNTL__QUEUE0_SWITCH_NUM_MASK 0x0000007FL
+#define RPB_RD_SWITCH_CNTL__QUEUE1_SWITCH_NUM_MASK 0x00003F80L
+#define RPB_RD_SWITCH_CNTL__QUEUE2_SWITCH_NUM_MASK 0x001FC000L
+#define RPB_RD_SWITCH_CNTL__QUEUE3_SWITCH_NUM_MASK 0x0FE00000L
+#define RPB_RD_SWITCH_CNTL__SWITCH_NUM_MODE_MASK 0x10000000L
+#define RPB_RD_SWITCH_CNTL__WORKLOAD_ADJUST_EN_MASK 0x20000000L
+#define RPB_RD_SWITCH_CNTL__WEIGHT_ADJUST_STEP_MASK 0xC0000000L
+//RPB_CID_QUEUE_WR
+#define RPB_CID_QUEUE_WR__CLIENT_ID_LOW__SHIFT 0x0
+#define RPB_CID_QUEUE_WR__CLIENT_ID_HIGH__SHIFT 0x5
+#define RPB_CID_QUEUE_WR__UPDATE_MODE__SHIFT 0xb
+#define RPB_CID_QUEUE_WR__WRITE_QUEUE__SHIFT 0xc
+#define RPB_CID_QUEUE_WR__READ_QUEUE__SHIFT 0xf
+#define RPB_CID_QUEUE_WR__UPDATE__SHIFT 0x12
+#define RPB_CID_QUEUE_WR__CLIENT_ID_LOW_MASK 0x0000001FL
+#define RPB_CID_QUEUE_WR__CLIENT_ID_HIGH_MASK 0x000007E0L
+#define RPB_CID_QUEUE_WR__UPDATE_MODE_MASK 0x00000800L
+#define RPB_CID_QUEUE_WR__WRITE_QUEUE_MASK 0x00007000L
+#define RPB_CID_QUEUE_WR__READ_QUEUE_MASK 0x00038000L
+#define RPB_CID_QUEUE_WR__UPDATE_MASK 0x00040000L
+//RPB_CID_QUEUE_RD
+#define RPB_CID_QUEUE_RD__CLIENT_ID_LOW__SHIFT 0x0
+#define RPB_CID_QUEUE_RD__CLIENT_ID_HIGH__SHIFT 0x5
+#define RPB_CID_QUEUE_RD__WRITE_QUEUE__SHIFT 0xb
+#define RPB_CID_QUEUE_RD__READ_QUEUE__SHIFT 0xe
+#define RPB_CID_QUEUE_RD__CLIENT_ID_LOW_MASK 0x0000001FL
+#define RPB_CID_QUEUE_RD__CLIENT_ID_HIGH_MASK 0x000007E0L
+#define RPB_CID_QUEUE_RD__WRITE_QUEUE_MASK 0x00003800L
+#define RPB_CID_QUEUE_RD__READ_QUEUE_MASK 0x0001C000L
+//RPB_PERF_COUNTER_CNTL
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_SELECT__SHIFT 0x0
+#define RPB_PERF_COUNTER_CNTL__CLEAR_SELECTED_PERF_COUNTER__SHIFT 0x2
+#define RPB_PERF_COUNTER_CNTL__CLEAR_ALL_PERF_COUNTERS__SHIFT 0x3
+#define RPB_PERF_COUNTER_CNTL__STOP_ON_COUNTER_SATURATION__SHIFT 0x4
+#define RPB_PERF_COUNTER_CNTL__ENABLE_PERF_COUNTERS__SHIFT 0x5
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_0__SHIFT 0x9
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_1__SHIFT 0xe
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_2__SHIFT 0x13
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_3__SHIFT 0x18
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_SELECT_MASK 0x00000003L
+#define RPB_PERF_COUNTER_CNTL__CLEAR_SELECTED_PERF_COUNTER_MASK 0x00000004L
+#define RPB_PERF_COUNTER_CNTL__CLEAR_ALL_PERF_COUNTERS_MASK 0x00000008L
+#define RPB_PERF_COUNTER_CNTL__STOP_ON_COUNTER_SATURATION_MASK 0x00000010L
+#define RPB_PERF_COUNTER_CNTL__ENABLE_PERF_COUNTERS_MASK 0x000001E0L
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_0_MASK 0x00003E00L
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_1_MASK 0x0007C000L
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_2_MASK 0x00F80000L
+#define RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_3_MASK 0x1F000000L
+//RPB_PERF_COUNTER_STATUS
+#define RPB_PERF_COUNTER_STATUS__PERFORMANCE_COUNTER_VALUE__SHIFT 0x0
+#define RPB_PERF_COUNTER_STATUS__PERFORMANCE_COUNTER_VALUE_MASK 0xFFFFFFFFL
+//RPB_CID_QUEUE_EX
+#define RPB_CID_QUEUE_EX__START__SHIFT 0x0
+#define RPB_CID_QUEUE_EX__OFFSET__SHIFT 0x1
+#define RPB_CID_QUEUE_EX__START_MASK 0x00000001L
+#define RPB_CID_QUEUE_EX__OFFSET_MASK 0x000001FEL
+//RPB_CID_QUEUE_EX_DATA
+#define RPB_CID_QUEUE_EX_DATA__WRITE_ENTRIES__SHIFT 0x0
+#define RPB_CID_QUEUE_EX_DATA__READ_ENTRIES__SHIFT 0x10
+#define RPB_CID_QUEUE_EX_DATA__WRITE_ENTRIES_MASK 0x0000FFFFL
+#define RPB_CID_QUEUE_EX_DATA__READ_ENTRIES_MASK 0xFFFF0000L
+//RPB_SWITCH_CNTL2
+#define RPB_SWITCH_CNTL2__RD_QUEUE4_SWITCH_NUM__SHIFT 0x0
+#define RPB_SWITCH_CNTL2__RD_QUEUE5_SWITCH_NUM__SHIFT 0x7
+#define RPB_SWITCH_CNTL2__WR_QUEUE4_SWITCH_NUM__SHIFT 0xe
+#define RPB_SWITCH_CNTL2__WR_QUEUE5_SWITCH_NUM__SHIFT 0x15
+#define RPB_SWITCH_CNTL2__RD_QUEUE4_SWITCH_NUM_MASK 0x0000007FL
+#define RPB_SWITCH_CNTL2__RD_QUEUE5_SWITCH_NUM_MASK 0x00003F80L
+#define RPB_SWITCH_CNTL2__WR_QUEUE4_SWITCH_NUM_MASK 0x001FC000L
+#define RPB_SWITCH_CNTL2__WR_QUEUE5_SWITCH_NUM_MASK 0x0FE00000L
+//RPB_DEINTRLV_COMBINE_CNTL
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_CHAINED_FLUSH_TIMER__SHIFT 0x0
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_CHAINED_BREAK_EN__SHIFT 0x4
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_HANDLE_CHECK_DISABLE__SHIFT 0x5
+#define RPB_DEINTRLV_COMBINE_CNTL__XPB_WRREQ_CRD__SHIFT 0x6
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_CHAINED_FLUSH_TIMER_MASK 0x0000000FL
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_CHAINED_BREAK_EN_MASK 0x00000010L
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_HANDLE_CHECK_DISABLE_MASK 0x00000020L
+#define RPB_DEINTRLV_COMBINE_CNTL__XPB_WRREQ_CRD_MASK 0x00003FC0L
+//RPB_VC_SWITCH_RDWR
+#define RPB_VC_SWITCH_RDWR__MODE__SHIFT 0x0
+#define RPB_VC_SWITCH_RDWR__NUM_RD__SHIFT 0x2
+#define RPB_VC_SWITCH_RDWR__NUM_WR__SHIFT 0xa
+#define RPB_VC_SWITCH_RDWR__XPB_RDREQ_CRD__SHIFT 0x12
+#define RPB_VC_SWITCH_RDWR__MODE_MASK 0x00000003L
+#define RPB_VC_SWITCH_RDWR__NUM_RD_MASK 0x000003FCL
+#define RPB_VC_SWITCH_RDWR__NUM_WR_MASK 0x0003FC00L
+#define RPB_VC_SWITCH_RDWR__XPB_RDREQ_CRD_MASK 0x03FC0000L
+//RPB_PERFCOUNTER_LO
+#define RPB_PERFCOUNTER_LO__COUNTER_LO__SHIFT 0x0
+#define RPB_PERFCOUNTER_LO__COUNTER_LO_MASK 0xFFFFFFFFL
+//RPB_PERFCOUNTER_HI
+#define RPB_PERFCOUNTER_HI__COUNTER_HI__SHIFT 0x0
+#define RPB_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT 0x10
+#define RPB_PERFCOUNTER_HI__COUNTER_HI_MASK 0x0000FFFFL
+#define RPB_PERFCOUNTER_HI__COMPARE_VALUE_MASK 0xFFFF0000L
+//RPB_PERFCOUNTER0_CFG
+#define RPB_PERFCOUNTER0_CFG__PERF_SEL__SHIFT 0x0
+#define RPB_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT 0x8
+#define RPB_PERFCOUNTER0_CFG__PERF_MODE__SHIFT 0x18
+#define RPB_PERFCOUNTER0_CFG__ENABLE__SHIFT 0x1c
+#define RPB_PERFCOUNTER0_CFG__CLEAR__SHIFT 0x1d
+#define RPB_PERFCOUNTER0_CFG__PERF_SEL_MASK 0x000000FFL
+#define RPB_PERFCOUNTER0_CFG__PERF_SEL_END_MASK 0x0000FF00L
+#define RPB_PERFCOUNTER0_CFG__PERF_MODE_MASK 0x0F000000L
+#define RPB_PERFCOUNTER0_CFG__ENABLE_MASK 0x10000000L
+#define RPB_PERFCOUNTER0_CFG__CLEAR_MASK 0x20000000L
+//RPB_PERFCOUNTER1_CFG
+#define RPB_PERFCOUNTER1_CFG__PERF_SEL__SHIFT 0x0
+#define RPB_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT 0x8
+#define RPB_PERFCOUNTER1_CFG__PERF_MODE__SHIFT 0x18
+#define RPB_PERFCOUNTER1_CFG__ENABLE__SHIFT 0x1c
+#define RPB_PERFCOUNTER1_CFG__CLEAR__SHIFT 0x1d
+#define RPB_PERFCOUNTER1_CFG__PERF_SEL_MASK 0x000000FFL
+#define RPB_PERFCOUNTER1_CFG__PERF_SEL_END_MASK 0x0000FF00L
+#define RPB_PERFCOUNTER1_CFG__PERF_MODE_MASK 0x0F000000L
+#define RPB_PERFCOUNTER1_CFG__ENABLE_MASK 0x10000000L
+#define RPB_PERFCOUNTER1_CFG__CLEAR_MASK 0x20000000L
+//RPB_PERFCOUNTER2_CFG
+#define RPB_PERFCOUNTER2_CFG__PERF_SEL__SHIFT 0x0
+#define RPB_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT 0x8
+#define RPB_PERFCOUNTER2_CFG__PERF_MODE__SHIFT 0x18
+#define RPB_PERFCOUNTER2_CFG__ENABLE__SHIFT 0x1c
+#define RPB_PERFCOUNTER2_CFG__CLEAR__SHIFT 0x1d
+#define RPB_PERFCOUNTER2_CFG__PERF_SEL_MASK 0x000000FFL
+#define RPB_PERFCOUNTER2_CFG__PERF_SEL_END_MASK 0x0000FF00L
+#define RPB_PERFCOUNTER2_CFG__PERF_MODE_MASK 0x0F000000L
+#define RPB_PERFCOUNTER2_CFG__ENABLE_MASK 0x10000000L
+#define RPB_PERFCOUNTER2_CFG__CLEAR_MASK 0x20000000L
+//RPB_PERFCOUNTER3_CFG
+#define RPB_PERFCOUNTER3_CFG__PERF_SEL__SHIFT 0x0
+#define RPB_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT 0x8
+#define RPB_PERFCOUNTER3_CFG__PERF_MODE__SHIFT 0x18
+#define RPB_PERFCOUNTER3_CFG__ENABLE__SHIFT 0x1c
+#define RPB_PERFCOUNTER3_CFG__CLEAR__SHIFT 0x1d
+#define RPB_PERFCOUNTER3_CFG__PERF_SEL_MASK 0x000000FFL
+#define RPB_PERFCOUNTER3_CFG__PERF_SEL_END_MASK 0x0000FF00L
+#define RPB_PERFCOUNTER3_CFG__PERF_MODE_MASK 0x0F000000L
+#define RPB_PERFCOUNTER3_CFG__ENABLE_MASK 0x10000000L
+#define RPB_PERFCOUNTER3_CFG__CLEAR_MASK 0x20000000L
+//RPB_PERFCOUNTER_RSLT_CNTL
+#define RPB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT 0x0
+#define RPB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT 0x8
+#define RPB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT 0x10
+#define RPB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT 0x18
+#define RPB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT 0x19
+#define RPB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT 0x1a
+#define RPB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK 0x0000000FL
+#define RPB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK 0x0000FF00L
+#define RPB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK 0x00FF0000L
+#define RPB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK 0x01000000L
+#define RPB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK 0x02000000L
+#define RPB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK 0x04000000L
+//RPB_BIF_CNTL2
+#define RPB_BIF_CNTL2__NBIF_HST_COMPCLKCTL_EN__SHIFT 0x0
+#define RPB_BIF_CNTL2__NBIF_HST_COMPCLKCTL_EN_MASK 0x00000001L
+//RPB_RD_QUEUE_CNTL
+#define RPB_RD_QUEUE_CNTL__ARB_MODE__SHIFT 0x0
+#define RPB_RD_QUEUE_CNTL__Q4_SHARED__SHIFT 0x1
+#define RPB_RD_QUEUE_CNTL__Q5_SHARED__SHIFT 0x2
+#define RPB_RD_QUEUE_CNTL__Q4_UNITID_EA_MODE__SHIFT 0x3
+#define RPB_RD_QUEUE_CNTL__Q5_UNITID_EA_MODE__SHIFT 0x4
+#define RPB_RD_QUEUE_CNTL__Q4_PATTERN_LOW__SHIFT 0x5
+#define RPB_RD_QUEUE_CNTL__Q4_PATTERN_HIGH__SHIFT 0xa
+#define RPB_RD_QUEUE_CNTL__Q5_PATTERN_LOW__SHIFT 0x10
+#define RPB_RD_QUEUE_CNTL__Q5_PATTERN_HIGH__SHIFT 0x15
+#define RPB_RD_QUEUE_CNTL__ARB_MODE_MASK 0x00000001L
+#define RPB_RD_QUEUE_CNTL__Q4_SHARED_MASK 0x00000002L
+#define RPB_RD_QUEUE_CNTL__Q5_SHARED_MASK 0x00000004L
+#define RPB_RD_QUEUE_CNTL__Q4_UNITID_EA_MODE_MASK 0x00000008L
+#define RPB_RD_QUEUE_CNTL__Q5_UNITID_EA_MODE_MASK 0x00000010L
+#define RPB_RD_QUEUE_CNTL__Q4_PATTERN_LOW_MASK 0x000003E0L
+#define RPB_RD_QUEUE_CNTL__Q4_PATTERN_HIGH_MASK 0x0000FC00L
+#define RPB_RD_QUEUE_CNTL__Q5_PATTERN_LOW_MASK 0x001F0000L
+#define RPB_RD_QUEUE_CNTL__Q5_PATTERN_HIGH_MASK 0x07E00000L
+//RPB_RD_QUEUE_CNTL2
+#define RPB_RD_QUEUE_CNTL2__Q4_PATTERN_MASK_LOW__SHIFT 0x0
+#define RPB_RD_QUEUE_CNTL2__Q4_PATTERN_MASK_HIGH__SHIFT 0x5
+#define RPB_RD_QUEUE_CNTL2__Q5_PATTERN_MASK_LOW__SHIFT 0xb
+#define RPB_RD_QUEUE_CNTL2__Q5_PATTERN_MASK_HIGH__SHIFT 0x10
+#define RPB_RD_QUEUE_CNTL2__Q4_PATTERN_MASK_LOW_MASK 0x0000001FL
+#define RPB_RD_QUEUE_CNTL2__Q4_PATTERN_MASK_HIGH_MASK 0x000007E0L
+#define RPB_RD_QUEUE_CNTL2__Q5_PATTERN_MASK_LOW_MASK 0x0000F800L
+#define RPB_RD_QUEUE_CNTL2__Q5_PATTERN_MASK_HIGH_MASK 0x003F0000L
+//RPB_WR_QUEUE_CNTL
+#define RPB_WR_QUEUE_CNTL__ARB_MODE__SHIFT 0x0
+#define RPB_WR_QUEUE_CNTL__Q4_SHARED__SHIFT 0x1
+#define RPB_WR_QUEUE_CNTL__Q5_SHARED__SHIFT 0x2
+#define RPB_WR_QUEUE_CNTL__Q4_UNITID_EA_MODE__SHIFT 0x3
+#define RPB_WR_QUEUE_CNTL__Q5_UNITID_EA_MODE__SHIFT 0x4
+#define RPB_WR_QUEUE_CNTL__Q4_PATTERN_LOW__SHIFT 0x5
+#define RPB_WR_QUEUE_CNTL__Q4_PATTERN_HIGH__SHIFT 0xa
+#define RPB_WR_QUEUE_CNTL__Q5_PATTERN_LOW__SHIFT 0x10
+#define RPB_WR_QUEUE_CNTL__Q5_PATTERN_HIGH__SHIFT 0x15
+#define RPB_WR_QUEUE_CNTL__ARB_MODE_MASK 0x00000001L
+#define RPB_WR_QUEUE_CNTL__Q4_SHARED_MASK 0x00000002L
+#define RPB_WR_QUEUE_CNTL__Q5_SHARED_MASK 0x00000004L
+#define RPB_WR_QUEUE_CNTL__Q4_UNITID_EA_MODE_MASK 0x00000008L
+#define RPB_WR_QUEUE_CNTL__Q5_UNITID_EA_MODE_MASK 0x00000010L
+#define RPB_WR_QUEUE_CNTL__Q4_PATTERN_LOW_MASK 0x000003E0L
+#define RPB_WR_QUEUE_CNTL__Q4_PATTERN_HIGH_MASK 0x0000FC00L
+#define RPB_WR_QUEUE_CNTL__Q5_PATTERN_LOW_MASK 0x001F0000L
+#define RPB_WR_QUEUE_CNTL__Q5_PATTERN_HIGH_MASK 0x07E00000L
+//RPB_WR_QUEUE_CNTL2
+#define RPB_WR_QUEUE_CNTL2__Q4_PATTERN_MASK_LOW__SHIFT 0x0
+#define RPB_WR_QUEUE_CNTL2__Q4_PATTERN_MASK_HIGH__SHIFT 0x5
+#define RPB_WR_QUEUE_CNTL2__Q5_PATTERN_MASK_LOW__SHIFT 0xb
+#define RPB_WR_QUEUE_CNTL2__Q5_PATTERN_MASK_HIGH__SHIFT 0x10
+#define RPB_WR_QUEUE_CNTL2__Q4_PATTERN_MASK_LOW_MASK 0x0000001FL
+#define RPB_WR_QUEUE_CNTL2__Q4_PATTERN_MASK_HIGH_MASK 0x000007E0L
+#define RPB_WR_QUEUE_CNTL2__Q5_PATTERN_MASK_LOW_MASK 0x0000F800L
+#define RPB_WR_QUEUE_CNTL2__Q5_PATTERN_MASK_HIGH_MASK 0x003F0000L
+//RPB_EA_QUEUE_WR
+#define RPB_EA_QUEUE_WR__EA_NUMBER__SHIFT 0x0
+#define RPB_EA_QUEUE_WR__WRITE_QUEUE__SHIFT 0x5
+#define RPB_EA_QUEUE_WR__READ_QUEUE__SHIFT 0x8
+#define RPB_EA_QUEUE_WR__UPDATE__SHIFT 0xb
+#define RPB_EA_QUEUE_WR__EA_NUMBER_MASK 0x0000001FL
+#define RPB_EA_QUEUE_WR__WRITE_QUEUE_MASK 0x000000E0L
+#define RPB_EA_QUEUE_WR__READ_QUEUE_MASK 0x00000700L
+#define RPB_EA_QUEUE_WR__UPDATE_MASK 0x00000800L
+//RPB_ATS_CNTL
+#define RPB_ATS_CNTL__PAGE_MIN_LATENCY_ENABLE__SHIFT 0x0
+#define RPB_ATS_CNTL__TR_MIN_LATENCY_ENABLE__SHIFT 0x1
+#define RPB_ATS_CNTL__SWITCH_THRESHOLD__SHIFT 0x2
+#define RPB_ATS_CNTL__TIME_SLICE__SHIFT 0x7
+#define RPB_ATS_CNTL__ATCTR_SWITCH_NUM__SHIFT 0xf
+#define RPB_ATS_CNTL__ATCPAGE_SWITCH_NUM__SHIFT 0x13
+#define RPB_ATS_CNTL__WR_AT__SHIFT 0x17
+#define RPB_ATS_CNTL__INVAL_COM_CMD__SHIFT 0x19
+#define RPB_ATS_CNTL__PAGE_MIN_LATENCY_ENABLE_MASK 0x00000001L
+#define RPB_ATS_CNTL__TR_MIN_LATENCY_ENABLE_MASK 0x00000002L
+#define RPB_ATS_CNTL__SWITCH_THRESHOLD_MASK 0x0000007CL
+#define RPB_ATS_CNTL__TIME_SLICE_MASK 0x00007F80L
+#define RPB_ATS_CNTL__ATCTR_SWITCH_NUM_MASK 0x00078000L
+#define RPB_ATS_CNTL__ATCPAGE_SWITCH_NUM_MASK 0x00780000L
+#define RPB_ATS_CNTL__WR_AT_MASK 0x01800000L
+#define RPB_ATS_CNTL__INVAL_COM_CMD_MASK 0x7E000000L
+//RPB_ATS_CNTL2
+#define RPB_ATS_CNTL2__TRANS_CMD__SHIFT 0x0
+#define RPB_ATS_CNTL2__PAGE_REQ_CMD__SHIFT 0x6
+#define RPB_ATS_CNTL2__PAGE_ROUTING_CODE__SHIFT 0xc
+#define RPB_ATS_CNTL2__INVAL_COM_ROUTING_CODE__SHIFT 0xf
+#define RPB_ATS_CNTL2__VENDOR_ID__SHIFT 0x12
+#define RPB_ATS_CNTL2__TRANS_CMD_MASK 0x0000003FL
+#define RPB_ATS_CNTL2__PAGE_REQ_CMD_MASK 0x00000FC0L
+#define RPB_ATS_CNTL2__PAGE_ROUTING_CODE_MASK 0x00007000L
+#define RPB_ATS_CNTL2__INVAL_COM_ROUTING_CODE_MASK 0x00038000L
+#define RPB_ATS_CNTL2__VENDOR_ID_MASK 0x000C0000L
+//RPB_DF_SDPPORT_CNTL
+#define RPB_DF_SDPPORT_CNTL__DF_REQ_CRD__SHIFT 0x0
+#define RPB_DF_SDPPORT_CNTL__DF_DATA_CRD__SHIFT 0x6
+#define RPB_DF_SDPPORT_CNTL__DF_HALT_THRESHOLD__SHIFT 0xc
+#define RPB_DF_SDPPORT_CNTL__DF_REQ_CRD_MASK 0x0000003FL
+#define RPB_DF_SDPPORT_CNTL__DF_DATA_CRD_MASK 0x00000FC0L
+#define RPB_DF_SDPPORT_CNTL__DF_HALT_THRESHOLD_MASK 0x0000F000L
+//RPB_SDPPORT_CNTL
+#define RPB_SDPPORT_CNTL__NBIF_DMA_SELF_ACTIVATE__SHIFT 0x0
+#define RPB_SDPPORT_CNTL__NBIF_DMA_CFG_MODE__SHIFT 0x1
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_REISSUE_CREDIT__SHIFT 0x3
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_SATURATE_COUNTER__SHIFT 0x4
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_DISRUPT_FULLDIS__SHIFT 0x5
+#define RPB_SDPPORT_CNTL__NBIF_DMA_HALT_THRESHOLD__SHIFT 0x6
+#define RPB_SDPPORT_CNTL__NBIF_HST_SELF_ACTIVATE__SHIFT 0xa
+#define RPB_SDPPORT_CNTL__NBIF_HST_CFG_MODE__SHIFT 0xb
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_REISSUE_CREDIT__SHIFT 0xd
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_SATURATE_COUNTER__SHIFT 0xe
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_DISRUPT_FULLDIS__SHIFT 0xf
+#define RPB_SDPPORT_CNTL__NBIF_HST_HALT_THRESHOLD__SHIFT 0x10
+#define RPB_SDPPORT_CNTL__NBIF_HST_PASSIVE_MODE__SHIFT 0x14
+#define RPB_SDPPORT_CNTL__NBIF_HST_QUICK_COMACK__SHIFT 0x15
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPCKEN__SHIFT 0x16
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPCKENRCV__SHIFT 0x17
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPDATACKEN__SHIFT 0x18
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPDATACKENRCV__SHIFT 0x19
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_WRRSPCKEN__SHIFT 0x1a
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_WRRSPCKENRCV__SHIFT 0x1b
+#define RPB_SDPPORT_CNTL__NBIF_DMA_SELF_ACTIVATE_MASK 0x00000001L
+#define RPB_SDPPORT_CNTL__NBIF_DMA_CFG_MODE_MASK 0x00000006L
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_REISSUE_CREDIT_MASK 0x00000008L
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_SATURATE_COUNTER_MASK 0x00000010L
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_DISRUPT_FULLDIS_MASK 0x00000020L
+#define RPB_SDPPORT_CNTL__NBIF_DMA_HALT_THRESHOLD_MASK 0x000003C0L
+#define RPB_SDPPORT_CNTL__NBIF_HST_SELF_ACTIVATE_MASK 0x00000400L
+#define RPB_SDPPORT_CNTL__NBIF_HST_CFG_MODE_MASK 0x00001800L
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_REISSUE_CREDIT_MASK 0x00002000L
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_SATURATE_COUNTER_MASK 0x00004000L
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_DISRUPT_FULLDIS_MASK 0x00008000L
+#define RPB_SDPPORT_CNTL__NBIF_HST_HALT_THRESHOLD_MASK 0x000F0000L
+#define RPB_SDPPORT_CNTL__NBIF_HST_PASSIVE_MODE_MASK 0x00100000L
+#define RPB_SDPPORT_CNTL__NBIF_HST_QUICK_COMACK_MASK 0x00200000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPCKEN_MASK 0x00400000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPCKENRCV_MASK 0x00800000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPDATACKEN_MASK 0x01000000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPDATACKENRCV_MASK 0x02000000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_WRRSPCKEN_MASK 0x04000000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_WRRSPCKENRCV_MASK 0x08000000L
+//RPB_NBIF_SDPPORT_CNTL
+#define RPB_NBIF_SDPPORT_CNTL__NBIF_DMA_WRRSP_CRD__SHIFT 0x0
+#define RPB_NBIF_SDPPORT_CNTL__NBIF_DMA_RDRSP_CRD__SHIFT 0x8
+#define RPB_NBIF_SDPPORT_CNTL__NBIF_HST_REQ_CRD__SHIFT 0x10
+#define RPB_NBIF_SDPPORT_CNTL__NBIF_HST_DATA_CRD__SHIFT 0x18
+#define RPB_NBIF_SDPPORT_CNTL__NBIF_DMA_WRRSP_CRD_MASK 0x000000FFL
+#define RPB_NBIF_SDPPORT_CNTL__NBIF_DMA_RDRSP_CRD_MASK 0x0000FF00L
+#define RPB_NBIF_SDPPORT_CNTL__NBIF_HST_REQ_CRD_MASK 0x00FF0000L
+#define RPB_NBIF_SDPPORT_CNTL__NBIF_HST_DATA_CRD_MASK 0xFF000000L
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/clk/clk_11_0_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/clk/clk_11_0_0_offset.h
new file mode 100644
index 000000000000..63759f8c0fa2
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/clk/clk_11_0_0_offset.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _clk_11_0_0_OFFSET_HEADER
+#define _clk_11_0_0_OFFSET_HEADER
+
+
+// addressBlock: clk_clk3_0_SmuClkDec
+// base address: 0x5c800
+#define mmCLK3_0_CLK3_CLK_PLL_REQ 0x000e
+#define mmCLK3_0_CLK3_CLK_PLL_REQ_BASE_IDX 3
+#define mmCLK3_0_CLK3_CLK2_DFS_CNTL 0x0054
+#define mmCLK3_0_CLK3_CLK2_DFS_CNTL_BASE_IDX 3
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/clk/clk_11_0_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/clk/clk_11_0_0_sh_mask.h
new file mode 100644
index 000000000000..e3d954434fcc
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/clk/clk_11_0_0_sh_mask.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _clk_11_0_0_SH_MASK_HEADER
+#define _clk_11_0_0_SH_MASK_HEADER
+
+
+// addressBlock: clk_clk3_0_SmuClkDec
+//CLK3_0_CLK3_CLK_PLL_REQ
+#define CLK3_0_CLK3_CLK_PLL_REQ__FbMult_int__SHIFT 0x0
+#define CLK3_0_CLK3_CLK_PLL_REQ__PllSpineDiv__SHIFT 0xc
+#define CLK3_0_CLK3_CLK_PLL_REQ__FbMult_frac__SHIFT 0x10
+#define CLK3_0_CLK3_CLK_PLL_REQ__FbMult_int_MASK 0x000001FFL
+#define CLK3_0_CLK3_CLK_PLL_REQ__PllSpineDiv_MASK 0x0000F000L
+#define CLK3_0_CLK3_CLK_PLL_REQ__FbMult_frac_MASK 0xFFFF0000L
+//CLK3_0_CLK3_CLK2_DFS_CNTL
+#define CLK3_0_CLK3_CLK2_DFS_CNTL__CLK2_DIVIDER__SHIFT 0x0
+#define CLK3_0_CLK3_CLK2_DFS_CNTL__CLK2_DIVIDER_MASK 0x0000007FL
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_2_0_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_2_0_0_offset.h
new file mode 100644
index 000000000000..cff8f91555d3
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_2_0_0_offset.h
@@ -0,0 +1,17535 @@
+/*
+ * Copyright (C) 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _dcn_2_0_0_OFFSET_HEADER
+#define _dcn_2_0_0_OFFSET_HEADER
+
+
+
+// addressBlock: dce_dc_mmhubbub_vga_dispdec
+// base address: 0x0
+#define mmVGA_MEM_WRITE_PAGE_ADDR 0x0000
+#define mmVGA_MEM_WRITE_PAGE_ADDR_BASE_IDX 0
+#define mmVGA_MEM_READ_PAGE_ADDR 0x0001
+#define mmVGA_MEM_READ_PAGE_ADDR_BASE_IDX 0
+#define mmVGA_RENDER_CONTROL 0x0000
+#define mmVGA_RENDER_CONTROL_BASE_IDX 1
+#define mmVGA_SEQUENCER_RESET_CONTROL 0x0001
+#define mmVGA_SEQUENCER_RESET_CONTROL_BASE_IDX 1
+#define mmVGA_MODE_CONTROL 0x0002
+#define mmVGA_MODE_CONTROL_BASE_IDX 1
+#define mmVGA_SURFACE_PITCH_SELECT 0x0003
+#define mmVGA_SURFACE_PITCH_SELECT_BASE_IDX 1
+#define mmVGA_MEMORY_BASE_ADDRESS 0x0004
+#define mmVGA_MEMORY_BASE_ADDRESS_BASE_IDX 1
+#define mmVGA_DISPBUF1_SURFACE_ADDR 0x0006
+#define mmVGA_DISPBUF1_SURFACE_ADDR_BASE_IDX 1
+#define mmVGA_DISPBUF2_SURFACE_ADDR 0x0008
+#define mmVGA_DISPBUF2_SURFACE_ADDR_BASE_IDX 1
+#define mmVGA_MEMORY_BASE_ADDRESS_HIGH 0x0009
+#define mmVGA_MEMORY_BASE_ADDRESS_HIGH_BASE_IDX 1
+#define mmVGA_HDP_CONTROL 0x000a
+#define mmVGA_HDP_CONTROL_BASE_IDX 1
+#define mmVGA_CACHE_CONTROL 0x000b
+#define mmVGA_CACHE_CONTROL_BASE_IDX 1
+#define mmD1VGA_CONTROL 0x000c
+#define mmD1VGA_CONTROL_BASE_IDX 1
+#define mmD2VGA_CONTROL 0x000e
+#define mmD2VGA_CONTROL_BASE_IDX 1
+#define mmVGA_STATUS 0x0010
+#define mmVGA_STATUS_BASE_IDX 1
+#define mmVGA_INTERRUPT_CONTROL 0x0011
+#define mmVGA_INTERRUPT_CONTROL_BASE_IDX 1
+#define mmVGA_STATUS_CLEAR 0x0012
+#define mmVGA_STATUS_CLEAR_BASE_IDX 1
+#define mmVGA_INTERRUPT_STATUS 0x0013
+#define mmVGA_INTERRUPT_STATUS_BASE_IDX 1
+#define mmVGA_MAIN_CONTROL 0x0014
+#define mmVGA_MAIN_CONTROL_BASE_IDX 1
+#define mmVGA_TEST_CONTROL 0x0015
+#define mmVGA_TEST_CONTROL_BASE_IDX 1
+#define mmVGA_QOS_CTRL 0x0018
+#define mmVGA_QOS_CTRL_BASE_IDX 1
+#define mmCRTC8_IDX 0x002d
+#define mmCRTC8_IDX_BASE_IDX 1
+#define mmCRTC8_DATA 0x002d
+#define mmCRTC8_DATA_BASE_IDX 1
+#define mmGENFC_WT 0x002e
+#define mmGENFC_WT_BASE_IDX 1
+#define mmGENS1 0x002e
+#define mmGENS1_BASE_IDX 1
+#define mmATTRDW 0x0030
+#define mmATTRDW_BASE_IDX 1
+#define mmATTRX 0x0030
+#define mmATTRX_BASE_IDX 1
+#define mmATTRDR 0x0030
+#define mmATTRDR_BASE_IDX 1
+#define mmGENMO_WT 0x0030
+#define mmGENMO_WT_BASE_IDX 1
+#define mmGENS0 0x0030
+#define mmGENS0_BASE_IDX 1
+#define mmGENENB 0x0030
+#define mmGENENB_BASE_IDX 1
+#define mmSEQ8_IDX 0x0031
+#define mmSEQ8_IDX_BASE_IDX 1
+#define mmSEQ8_DATA 0x0031
+#define mmSEQ8_DATA_BASE_IDX 1
+#define mmDAC_MASK 0x0031
+#define mmDAC_MASK_BASE_IDX 1
+#define mmDAC_R_INDEX 0x0031
+#define mmDAC_R_INDEX_BASE_IDX 1
+#define mmDAC_W_INDEX 0x0032
+#define mmDAC_W_INDEX_BASE_IDX 1
+#define mmDAC_DATA 0x0032
+#define mmDAC_DATA_BASE_IDX 1
+#define mmGENFC_RD 0x0032
+#define mmGENFC_RD_BASE_IDX 1
+#define mmGENMO_RD 0x0033
+#define mmGENMO_RD_BASE_IDX 1
+#define mmGRPH8_IDX 0x0033
+#define mmGRPH8_IDX_BASE_IDX 1
+#define mmGRPH8_DATA 0x0033
+#define mmGRPH8_DATA_BASE_IDX 1
+#define mmCRTC8_IDX_1 0x0035
+#define mmCRTC8_IDX_1_BASE_IDX 1
+#define mmCRTC8_DATA_1 0x0035
+#define mmCRTC8_DATA_1_BASE_IDX 1
+#define mmGENFC_WT_1 0x0036
+#define mmGENFC_WT_1_BASE_IDX 1
+#define mmGENS1_1 0x0036
+#define mmGENS1_1_BASE_IDX 1
+#define mmD3VGA_CONTROL 0x0038
+#define mmD3VGA_CONTROL_BASE_IDX 1
+#define mmD4VGA_CONTROL 0x0039
+#define mmD4VGA_CONTROL_BASE_IDX 1
+#define mmD5VGA_CONTROL 0x003a
+#define mmD5VGA_CONTROL_BASE_IDX 1
+#define mmD6VGA_CONTROL 0x003b
+#define mmD6VGA_CONTROL_BASE_IDX 1
+#define mmVGA_SOURCE_SELECT 0x003c
+#define mmVGA_SOURCE_SELECT_BASE_IDX 1
+
+
+// addressBlock: dce_dc_dccg_dccg_dispdec
+// base address: 0x0
+#define mmPHYPLLA_PIXCLK_RESYNC_CNTL 0x0040
+#define mmPHYPLLA_PIXCLK_RESYNC_CNTL_BASE_IDX 1
+#define mmPHYPLLB_PIXCLK_RESYNC_CNTL 0x0041
+#define mmPHYPLLB_PIXCLK_RESYNC_CNTL_BASE_IDX 1
+#define mmPHYPLLC_PIXCLK_RESYNC_CNTL 0x0042
+#define mmPHYPLLC_PIXCLK_RESYNC_CNTL_BASE_IDX 1
+#define mmPHYPLLD_PIXCLK_RESYNC_CNTL 0x0043
+#define mmPHYPLLD_PIXCLK_RESYNC_CNTL_BASE_IDX 1
+#define mmDP_DTO_DBUF_EN 0x0044
+#define mmDP_DTO_DBUF_EN_BASE_IDX 1
+#define mmDSCCLK3_DTO_PARAM 0x0045
+#define mmDSCCLK3_DTO_PARAM_BASE_IDX 1
+#define mmDSCCLK4_DTO_PARAM 0x0046
+#define mmDSCCLK4_DTO_PARAM_BASE_IDX 1
+#define mmDSCCLK5_DTO_PARAM 0x0047
+#define mmDSCCLK5_DTO_PARAM_BASE_IDX 1
+#define mmDPREFCLK_CGTT_BLK_CTRL_REG 0x0048
+#define mmDPREFCLK_CGTT_BLK_CTRL_REG_BASE_IDX 1
+#define mmREFCLK_CNTL 0x0049
+#define mmREFCLK_CNTL_BASE_IDX 1
+#define mmREFCLK_CGTT_BLK_CTRL_REG 0x004b
+#define mmREFCLK_CGTT_BLK_CTRL_REG_BASE_IDX 1
+#define mmPHYPLLE_PIXCLK_RESYNC_CNTL 0x004c
+#define mmPHYPLLE_PIXCLK_RESYNC_CNTL_BASE_IDX 1
+#define mmDCCG_PERFMON_CNTL2 0x004e
+#define mmDCCG_PERFMON_CNTL2_BASE_IDX 1
+#define mmDCCG_DS_DTO_INCR 0x0053
+#define mmDCCG_DS_DTO_INCR_BASE_IDX 1
+#define mmDCCG_DS_DTO_MODULO 0x0054
+#define mmDCCG_DS_DTO_MODULO_BASE_IDX 1
+#define mmDCCG_DS_CNTL 0x0055
+#define mmDCCG_DS_CNTL_BASE_IDX 1
+#define mmDCCG_DS_HW_CAL_INTERVAL 0x0056
+#define mmDCCG_DS_HW_CAL_INTERVAL_BASE_IDX 1
+#define mmDPREFCLK_CNTL 0x0058
+#define mmDPREFCLK_CNTL_BASE_IDX 1
+#define mmDCE_VERSION 0x005e
+#define mmDCE_VERSION_BASE_IDX 1
+#define mmDCCG_GTC_CNTL 0x0060
+#define mmDCCG_GTC_CNTL_BASE_IDX 1
+#define mmDCCG_GTC_DTO_INCR 0x0061
+#define mmDCCG_GTC_DTO_INCR_BASE_IDX 1
+#define mmDCCG_GTC_DTO_MODULO 0x0062
+#define mmDCCG_GTC_DTO_MODULO_BASE_IDX 1
+#define mmDCCG_GTC_CURRENT 0x0063
+#define mmDCCG_GTC_CURRENT_BASE_IDX 1
+#define mmDSCCLK0_DTO_PARAM 0x006c
+#define mmDSCCLK0_DTO_PARAM_BASE_IDX 1
+#define mmDSCCLK1_DTO_PARAM 0x006d
+#define mmDSCCLK1_DTO_PARAM_BASE_IDX 1
+#define mmDSCCLK2_DTO_PARAM 0x006e
+#define mmDSCCLK2_DTO_PARAM_BASE_IDX 1
+#define mmMILLISECOND_TIME_BASE_DIV 0x0070
+#define mmMILLISECOND_TIME_BASE_DIV_BASE_IDX 1
+#define mmDISPCLK_FREQ_CHANGE_CNTL 0x0071
+#define mmDISPCLK_FREQ_CHANGE_CNTL_BASE_IDX 1
+#define mmDC_MEM_GLOBAL_PWR_REQ_CNTL 0x0072
+#define mmDC_MEM_GLOBAL_PWR_REQ_CNTL_BASE_IDX 1
+#define mmDCCG_PERFMON_CNTL 0x0073
+#define mmDCCG_PERFMON_CNTL_BASE_IDX 1
+#define mmDCCG_GATE_DISABLE_CNTL 0x0074
+#define mmDCCG_GATE_DISABLE_CNTL_BASE_IDX 1
+#define mmDISPCLK_CGTT_BLK_CTRL_REG 0x0075
+#define mmDISPCLK_CGTT_BLK_CTRL_REG_BASE_IDX 1
+#define mmSOCCLK_CGTT_BLK_CTRL_REG 0x0076
+#define mmSOCCLK_CGTT_BLK_CTRL_REG_BASE_IDX 1
+#define mmDCCG_CAC_STATUS 0x0077
+#define mmDCCG_CAC_STATUS_BASE_IDX 1
+#define mmMICROSECOND_TIME_BASE_DIV 0x007b
+#define mmMICROSECOND_TIME_BASE_DIV_BASE_IDX 1
+#define mmDCCG_GATE_DISABLE_CNTL2 0x007c
+#define mmDCCG_GATE_DISABLE_CNTL2_BASE_IDX 1
+#define mmSYMCLK_CGTT_BLK_CTRL_REG 0x007d
+#define mmSYMCLK_CGTT_BLK_CTRL_REG_BASE_IDX 1
+#define mmPHYPLLF_PIXCLK_RESYNC_CNTL 0x007e
+#define mmPHYPLLF_PIXCLK_RESYNC_CNTL_BASE_IDX 1
+#define mmDCCG_DISP_CNTL_REG 0x007f
+#define mmDCCG_DISP_CNTL_REG_BASE_IDX 1
+#define mmOTG0_PIXEL_RATE_CNTL 0x0080
+#define mmOTG0_PIXEL_RATE_CNTL_BASE_IDX 1
+#define mmDP_DTO0_PHASE 0x0081
+#define mmDP_DTO0_PHASE_BASE_IDX 1
+#define mmDP_DTO0_MODULO 0x0082
+#define mmDP_DTO0_MODULO_BASE_IDX 1
+#define mmOTG0_PHYPLL_PIXEL_RATE_CNTL 0x0083
+#define mmOTG0_PHYPLL_PIXEL_RATE_CNTL_BASE_IDX 1
+#define mmOTG1_PIXEL_RATE_CNTL 0x0084
+#define mmOTG1_PIXEL_RATE_CNTL_BASE_IDX 1
+#define mmDP_DTO1_PHASE 0x0085
+#define mmDP_DTO1_PHASE_BASE_IDX 1
+#define mmDP_DTO1_MODULO 0x0086
+#define mmDP_DTO1_MODULO_BASE_IDX 1
+#define mmOTG1_PHYPLL_PIXEL_RATE_CNTL 0x0087
+#define mmOTG1_PHYPLL_PIXEL_RATE_CNTL_BASE_IDX 1
+#define mmOTG2_PIXEL_RATE_CNTL 0x0088
+#define mmOTG2_PIXEL_RATE_CNTL_BASE_IDX 1
+#define mmDP_DTO2_PHASE 0x0089
+#define mmDP_DTO2_PHASE_BASE_IDX 1
+#define mmDP_DTO2_MODULO 0x008a
+#define mmDP_DTO2_MODULO_BASE_IDX 1
+#define mmOTG2_PHYPLL_PIXEL_RATE_CNTL 0x008b
+#define mmOTG2_PHYPLL_PIXEL_RATE_CNTL_BASE_IDX 1
+#define mmOTG3_PIXEL_RATE_CNTL 0x008c
+#define mmOTG3_PIXEL_RATE_CNTL_BASE_IDX 1
+#define mmDP_DTO3_PHASE 0x008d
+#define mmDP_DTO3_PHASE_BASE_IDX 1
+#define mmDP_DTO3_MODULO 0x008e
+#define mmDP_DTO3_MODULO_BASE_IDX 1
+#define mmOTG3_PHYPLL_PIXEL_RATE_CNTL 0x008f
+#define mmOTG3_PHYPLL_PIXEL_RATE_CNTL_BASE_IDX 1
+#define mmOTG4_PIXEL_RATE_CNTL 0x0090
+#define mmOTG4_PIXEL_RATE_CNTL_BASE_IDX 1
+#define mmDP_DTO4_PHASE 0x0091
+#define mmDP_DTO4_PHASE_BASE_IDX 1
+#define mmDP_DTO4_MODULO 0x0092
+#define mmDP_DTO4_MODULO_BASE_IDX 1
+#define mmOTG4_PHYPLL_PIXEL_RATE_CNTL 0x0093
+#define mmOTG4_PHYPLL_PIXEL_RATE_CNTL_BASE_IDX 1
+#define mmOTG5_PIXEL_RATE_CNTL 0x0094
+#define mmOTG5_PIXEL_RATE_CNTL_BASE_IDX 1
+#define mmDP_DTO5_PHASE 0x0095
+#define mmDP_DTO5_PHASE_BASE_IDX 1
+#define mmDP_DTO5_MODULO 0x0096
+#define mmDP_DTO5_MODULO_BASE_IDX 1
+#define mmOTG5_PHYPLL_PIXEL_RATE_CNTL 0x0097
+#define mmOTG5_PHYPLL_PIXEL_RATE_CNTL_BASE_IDX 1
+#define mmDPPCLK_CGTT_BLK_CTRL_REG 0x0098
+#define mmDPPCLK_CGTT_BLK_CTRL_REG_BASE_IDX 1
+#define mmDPPCLK0_DTO_PARAM 0x0099
+#define mmDPPCLK0_DTO_PARAM_BASE_IDX 1
+#define mmDPPCLK1_DTO_PARAM 0x009a
+#define mmDPPCLK1_DTO_PARAM_BASE_IDX 1
+#define mmDPPCLK2_DTO_PARAM 0x009b
+#define mmDPPCLK2_DTO_PARAM_BASE_IDX 1
+#define mmDPPCLK3_DTO_PARAM 0x009c
+#define mmDPPCLK3_DTO_PARAM_BASE_IDX 1
+#define mmDPPCLK4_DTO_PARAM 0x009d
+#define mmDPPCLK4_DTO_PARAM_BASE_IDX 1
+#define mmDPPCLK5_DTO_PARAM 0x009e
+#define mmDPPCLK5_DTO_PARAM_BASE_IDX 1
+#define mmDCCG_CAC_STATUS2 0x009f
+#define mmDCCG_CAC_STATUS2_BASE_IDX 1
+#define mmSYMCLKA_CLOCK_ENABLE 0x00a0
+#define mmSYMCLKA_CLOCK_ENABLE_BASE_IDX 1
+#define mmSYMCLKB_CLOCK_ENABLE 0x00a1
+#define mmSYMCLKB_CLOCK_ENABLE_BASE_IDX 1
+#define mmSYMCLKC_CLOCK_ENABLE 0x00a2
+#define mmSYMCLKC_CLOCK_ENABLE_BASE_IDX 1
+#define mmSYMCLKD_CLOCK_ENABLE 0x00a3
+#define mmSYMCLKD_CLOCK_ENABLE_BASE_IDX 1
+#define mmSYMCLKE_CLOCK_ENABLE 0x00a4
+#define mmSYMCLKE_CLOCK_ENABLE_BASE_IDX 1
+#define mmSYMCLKF_CLOCK_ENABLE 0x00a5
+#define mmSYMCLKF_CLOCK_ENABLE_BASE_IDX 1
+#define mmDCCG_SOFT_RESET 0x00a6
+#define mmDCCG_SOFT_RESET_BASE_IDX 1
+#define mmDSCCLK_DTO_CTRL 0x00a7
+#define mmDSCCLK_DTO_CTRL_BASE_IDX 1
+#define mmDCCG_AUDIO_DTO_SOURCE 0x00ab
+#define mmDCCG_AUDIO_DTO_SOURCE_BASE_IDX 1
+#define mmDCCG_AUDIO_DTO0_PHASE 0x00ac
+#define mmDCCG_AUDIO_DTO0_PHASE_BASE_IDX 1
+#define mmDCCG_AUDIO_DTO0_MODULE 0x00ad
+#define mmDCCG_AUDIO_DTO0_MODULE_BASE_IDX 1
+#define mmDCCG_AUDIO_DTO1_PHASE 0x00ae
+#define mmDCCG_AUDIO_DTO1_PHASE_BASE_IDX 1
+#define mmDCCG_AUDIO_DTO1_MODULE 0x00af
+#define mmDCCG_AUDIO_DTO1_MODULE_BASE_IDX 1
+#define mmDCCG_VSYNC_OTG0_LATCH_VALUE 0x00b0
+#define mmDCCG_VSYNC_OTG0_LATCH_VALUE_BASE_IDX 1
+#define mmDCCG_VSYNC_OTG1_LATCH_VALUE 0x00b1
+#define mmDCCG_VSYNC_OTG1_LATCH_VALUE_BASE_IDX 1
+#define mmDCCG_VSYNC_OTG2_LATCH_VALUE 0x00b2
+#define mmDCCG_VSYNC_OTG2_LATCH_VALUE_BASE_IDX 1
+#define mmDCCG_VSYNC_OTG3_LATCH_VALUE 0x00b3
+#define mmDCCG_VSYNC_OTG3_LATCH_VALUE_BASE_IDX 1
+#define mmDCCG_VSYNC_OTG4_LATCH_VALUE 0x00b4
+#define mmDCCG_VSYNC_OTG4_LATCH_VALUE_BASE_IDX 1
+#define mmDCCG_VSYNC_OTG5_LATCH_VALUE 0x00b5
+#define mmDCCG_VSYNC_OTG5_LATCH_VALUE_BASE_IDX 1
+#define mmDPPCLK_DTO_CTRL 0x00b6
+#define mmDPPCLK_DTO_CTRL_BASE_IDX 1
+#define mmDCCG_VSYNC_CNT_CTRL 0x00b8
+#define mmDCCG_VSYNC_CNT_CTRL_BASE_IDX 1
+#define mmDCCG_VSYNC_CNT_INT_CTRL 0x00b9
+#define mmDCCG_VSYNC_CNT_INT_CTRL_BASE_IDX 1
+#define mmFORCE_SYMCLK_DISABLE 0x00ba
+#define mmFORCE_SYMCLK_DISABLE_BASE_IDX 1
+#define mmDCCG_TEST_CLK_SEL 0x00be
+#define mmDCCG_TEST_CLK_SEL_BASE_IDX 1
+
+
+// addressBlock: dce_dc_dccg_dccg_dfs_dispdec
+// base address: 0x0
+#define mmDENTIST_DISPCLK_CNTL 0x0064
+#define mmDENTIST_DISPCLK_CNTL_BASE_IDX 1
+
+
+// addressBlock: dce_dc_dccg_dccg_dcperfmon0_dc_perfmon_dispdec
+// base address: 0x0
+#define mmDC_PERFMON0_PERFCOUNTER_CNTL 0x0000
+#define mmDC_PERFMON0_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON0_PERFCOUNTER_CNTL2 0x0001
+#define mmDC_PERFMON0_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON0_PERFCOUNTER_STATE 0x0002
+#define mmDC_PERFMON0_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON0_PERFMON_CNTL 0x0003
+#define mmDC_PERFMON0_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON0_PERFMON_CNTL2 0x0004
+#define mmDC_PERFMON0_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON0_PERFMON_CVALUE_INT_MISC 0x0005
+#define mmDC_PERFMON0_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON0_PERFMON_CVALUE_LOW 0x0006
+#define mmDC_PERFMON0_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON0_PERFMON_HI 0x0007
+#define mmDC_PERFMON0_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON0_PERFMON_LOW 0x0008
+#define mmDC_PERFMON0_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dccg_dccg_dcperfmon1_dc_perfmon_dispdec
+// base address: 0x30
+#define mmDC_PERFMON1_PERFCOUNTER_CNTL 0x000c
+#define mmDC_PERFMON1_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON1_PERFCOUNTER_CNTL2 0x000d
+#define mmDC_PERFMON1_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON1_PERFCOUNTER_STATE 0x000e
+#define mmDC_PERFMON1_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON1_PERFMON_CNTL 0x000f
+#define mmDC_PERFMON1_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON1_PERFMON_CNTL2 0x0010
+#define mmDC_PERFMON1_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON1_PERFMON_CVALUE_INT_MISC 0x0011
+#define mmDC_PERFMON1_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON1_PERFMON_CVALUE_LOW 0x0012
+#define mmDC_PERFMON1_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON1_PERFMON_HI 0x0013
+#define mmDC_PERFMON1_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON1_PERFMON_LOW 0x0014
+#define mmDC_PERFMON1_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dccg_dccg_pll_dispdec
+// base address: 0x0
+#define mmPLL_MACRO_CNTL_RESERVED0 0x0018
+#define mmPLL_MACRO_CNTL_RESERVED0_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED1 0x0019
+#define mmPLL_MACRO_CNTL_RESERVED1_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED2 0x001a
+#define mmPLL_MACRO_CNTL_RESERVED2_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED3 0x001b
+#define mmPLL_MACRO_CNTL_RESERVED3_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED4 0x001c
+#define mmPLL_MACRO_CNTL_RESERVED4_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED5 0x001d
+#define mmPLL_MACRO_CNTL_RESERVED5_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED6 0x001e
+#define mmPLL_MACRO_CNTL_RESERVED6_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED7 0x001f
+#define mmPLL_MACRO_CNTL_RESERVED7_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED8 0x0020
+#define mmPLL_MACRO_CNTL_RESERVED8_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED9 0x0021
+#define mmPLL_MACRO_CNTL_RESERVED9_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED10 0x0022
+#define mmPLL_MACRO_CNTL_RESERVED10_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED11 0x0023
+#define mmPLL_MACRO_CNTL_RESERVED11_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED12 0x0024
+#define mmPLL_MACRO_CNTL_RESERVED12_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED13 0x0025
+#define mmPLL_MACRO_CNTL_RESERVED13_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED14 0x0026
+#define mmPLL_MACRO_CNTL_RESERVED14_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED15 0x0027
+#define mmPLL_MACRO_CNTL_RESERVED15_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED16 0x0028
+#define mmPLL_MACRO_CNTL_RESERVED16_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED17 0x0029
+#define mmPLL_MACRO_CNTL_RESERVED17_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED18 0x002a
+#define mmPLL_MACRO_CNTL_RESERVED18_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED19 0x002b
+#define mmPLL_MACRO_CNTL_RESERVED19_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED20 0x002c
+#define mmPLL_MACRO_CNTL_RESERVED20_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED21 0x002d
+#define mmPLL_MACRO_CNTL_RESERVED21_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED22 0x002e
+#define mmPLL_MACRO_CNTL_RESERVED22_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED23 0x002f
+#define mmPLL_MACRO_CNTL_RESERVED23_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED24 0x0030
+#define mmPLL_MACRO_CNTL_RESERVED24_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED25 0x0031
+#define mmPLL_MACRO_CNTL_RESERVED25_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED26 0x0032
+#define mmPLL_MACRO_CNTL_RESERVED26_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED27 0x0033
+#define mmPLL_MACRO_CNTL_RESERVED27_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED28 0x0034
+#define mmPLL_MACRO_CNTL_RESERVED28_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED29 0x0035
+#define mmPLL_MACRO_CNTL_RESERVED29_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED30 0x0036
+#define mmPLL_MACRO_CNTL_RESERVED30_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED31 0x0037
+#define mmPLL_MACRO_CNTL_RESERVED31_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED32 0x0038
+#define mmPLL_MACRO_CNTL_RESERVED32_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED33 0x0039
+#define mmPLL_MACRO_CNTL_RESERVED33_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED34 0x003a
+#define mmPLL_MACRO_CNTL_RESERVED34_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED35 0x003b
+#define mmPLL_MACRO_CNTL_RESERVED35_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED36 0x003c
+#define mmPLL_MACRO_CNTL_RESERVED36_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED37 0x003d
+#define mmPLL_MACRO_CNTL_RESERVED37_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED38 0x003e
+#define mmPLL_MACRO_CNTL_RESERVED38_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED39 0x003f
+#define mmPLL_MACRO_CNTL_RESERVED39_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED40 0x0040
+#define mmPLL_MACRO_CNTL_RESERVED40_BASE_IDX 2
+#define mmPLL_MACRO_CNTL_RESERVED41 0x0041
+#define mmPLL_MACRO_CNTL_RESERVED41_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dmu_rbbmif_dispdec
+// base address: 0x0
+#define mmRBBMIF_TIMEOUT 0x005b
+#define mmRBBMIF_TIMEOUT_BASE_IDX 2
+#define mmRBBMIF_STATUS 0x005c
+#define mmRBBMIF_STATUS_BASE_IDX 2
+#define mmRBBMIF_STATUS_2 0x005d
+#define mmRBBMIF_STATUS_2_BASE_IDX 2
+#define mmRBBMIF_INT_STATUS 0x005e
+#define mmRBBMIF_INT_STATUS_BASE_IDX 2
+#define mmRBBMIF_TIMEOUT_DIS 0x005f
+#define mmRBBMIF_TIMEOUT_DIS_BASE_IDX 2
+#define mmRBBMIF_TIMEOUT_DIS_2 0x0060
+#define mmRBBMIF_TIMEOUT_DIS_2_BASE_IDX 2
+#define mmRBBMIF_STATUS_FLAG 0x0061
+#define mmRBBMIF_STATUS_FLAG_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dmu_dc_pg_dispdec
+// base address: 0x0
+#define mmDOMAIN0_PG_CONFIG 0x0080
+#define mmDOMAIN0_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN0_PG_STATUS 0x0081
+#define mmDOMAIN0_PG_STATUS_BASE_IDX 2
+#define mmDOMAIN1_PG_CONFIG 0x0082
+#define mmDOMAIN1_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN1_PG_STATUS 0x0083
+#define mmDOMAIN1_PG_STATUS_BASE_IDX 2
+#define mmDOMAIN2_PG_CONFIG 0x0084
+#define mmDOMAIN2_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN2_PG_STATUS 0x0085
+#define mmDOMAIN2_PG_STATUS_BASE_IDX 2
+#define mmDOMAIN3_PG_CONFIG 0x0086
+#define mmDOMAIN3_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN3_PG_STATUS 0x0087
+#define mmDOMAIN3_PG_STATUS_BASE_IDX 2
+#define mmDOMAIN4_PG_CONFIG 0x0088
+#define mmDOMAIN4_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN4_PG_STATUS 0x0089
+#define mmDOMAIN4_PG_STATUS_BASE_IDX 2
+#define mmDOMAIN5_PG_CONFIG 0x008a
+#define mmDOMAIN5_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN5_PG_STATUS 0x008b
+#define mmDOMAIN5_PG_STATUS_BASE_IDX 2
+#define mmDOMAIN6_PG_CONFIG 0x008c
+#define mmDOMAIN6_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN6_PG_STATUS 0x008d
+#define mmDOMAIN6_PG_STATUS_BASE_IDX 2
+#define mmDOMAIN7_PG_CONFIG 0x008e
+#define mmDOMAIN7_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN7_PG_STATUS 0x008f
+#define mmDOMAIN7_PG_STATUS_BASE_IDX 2
+#define mmDOMAIN8_PG_CONFIG 0x0090
+#define mmDOMAIN8_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN8_PG_STATUS 0x0091
+#define mmDOMAIN8_PG_STATUS_BASE_IDX 2
+#define mmDOMAIN9_PG_CONFIG 0x0092
+#define mmDOMAIN9_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN9_PG_STATUS 0x0093
+#define mmDOMAIN9_PG_STATUS_BASE_IDX 2
+#define mmDOMAIN10_PG_CONFIG 0x0094
+#define mmDOMAIN10_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN10_PG_STATUS 0x0095
+#define mmDOMAIN10_PG_STATUS_BASE_IDX 2
+#define mmDOMAIN11_PG_CONFIG 0x0096
+#define mmDOMAIN11_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN11_PG_STATUS 0x0097
+#define mmDOMAIN11_PG_STATUS_BASE_IDX 2
+#define mmDOMAIN16_PG_CONFIG 0x00a1
+#define mmDOMAIN16_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN16_PG_STATUS 0x00a2
+#define mmDOMAIN16_PG_STATUS_BASE_IDX 2
+#define mmDOMAIN17_PG_CONFIG 0x00a3
+#define mmDOMAIN17_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN17_PG_STATUS 0x00a4
+#define mmDOMAIN17_PG_STATUS_BASE_IDX 2
+#define mmDOMAIN18_PG_CONFIG 0x00a5
+#define mmDOMAIN18_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN18_PG_STATUS 0x00a6
+#define mmDOMAIN18_PG_STATUS_BASE_IDX 2
+#define mmDOMAIN19_PG_CONFIG 0x00a7
+#define mmDOMAIN19_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN19_PG_STATUS 0x00a8
+#define mmDOMAIN19_PG_STATUS_BASE_IDX 2
+#define mmDOMAIN20_PG_CONFIG 0x00a9
+#define mmDOMAIN20_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN20_PG_STATUS 0x00aa
+#define mmDOMAIN20_PG_STATUS_BASE_IDX 2
+#define mmDOMAIN21_PG_CONFIG 0x00ab
+#define mmDOMAIN21_PG_CONFIG_BASE_IDX 2
+#define mmDOMAIN21_PG_STATUS 0x00ac
+#define mmDOMAIN21_PG_STATUS_BASE_IDX 2
+#define mmDCPG_INTERRUPT_STATUS 0x00ad
+#define mmDCPG_INTERRUPT_STATUS_BASE_IDX 2
+#define mmDCPG_INTERRUPT_STATUS_2 0x00ae
+#define mmDCPG_INTERRUPT_STATUS_2_BASE_IDX 2
+#define mmDCPG_INTERRUPT_CONTROL_1 0x00af
+#define mmDCPG_INTERRUPT_CONTROL_1_BASE_IDX 2
+#define mmDCPG_INTERRUPT_CONTROL_2 0x00b0
+#define mmDCPG_INTERRUPT_CONTROL_2_BASE_IDX 2
+#define mmDCPG_INTERRUPT_CONTROL_3 0x00b1
+#define mmDCPG_INTERRUPT_CONTROL_3_BASE_IDX 2
+#define mmDC_IP_REQUEST_CNTL 0x00b2
+#define mmDC_IP_REQUEST_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dmu_dmu_dcperfmon_dc_perfmon_dispdec
+// base address: 0x2f8
+#define mmDC_PERFMON2_PERFCOUNTER_CNTL 0x00be
+#define mmDC_PERFMON2_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON2_PERFCOUNTER_CNTL2 0x00bf
+#define mmDC_PERFMON2_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON2_PERFCOUNTER_STATE 0x00c0
+#define mmDC_PERFMON2_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON2_PERFMON_CNTL 0x00c1
+#define mmDC_PERFMON2_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON2_PERFMON_CNTL2 0x00c2
+#define mmDC_PERFMON2_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON2_PERFMON_CVALUE_INT_MISC 0x00c3
+#define mmDC_PERFMON2_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON2_PERFMON_CVALUE_LOW 0x00c4
+#define mmDC_PERFMON2_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON2_PERFMON_HI 0x00c5
+#define mmDC_PERFMON2_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON2_PERFMON_LOW 0x00c6
+#define mmDC_PERFMON2_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dmu_dmu_misc_dispdec
+// base address: 0x0
+#define mmCC_DC_PIPE_DIS 0x00ca
+#define mmCC_DC_PIPE_DIS_BASE_IDX 2
+#define mmDMU_CLK_CNTL 0x00cb
+#define mmDMU_CLK_CNTL_BASE_IDX 2
+#define mmDMU_MEM_PWR_CNTL 0x00cc
+#define mmDMU_MEM_PWR_CNTL_BASE_IDX 2
+#define mmDMCU_SMU_INTERRUPT_CNTL 0x00cd
+#define mmDMCU_SMU_INTERRUPT_CNTL_BASE_IDX 2
+#define mmSMU_INTERRUPT_CONTROL 0x00ce
+#define mmSMU_INTERRUPT_CONTROL_BASE_IDX 2
+#define mmDMU_MISC_ALLOW_DS_FORCE 0x00d6
+#define mmDMU_MISC_ALLOW_DS_FORCE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dmu_dmcu_dispdec
+// base address: 0x0
+#define mmDMCU_CTRL 0x00da
+#define mmDMCU_CTRL_BASE_IDX 2
+#define mmDMCU_STATUS 0x00db
+#define mmDMCU_STATUS_BASE_IDX 2
+#define mmDMCU_PC_START_ADDR 0x00dc
+#define mmDMCU_PC_START_ADDR_BASE_IDX 2
+#define mmDMCU_FW_START_ADDR 0x00dd
+#define mmDMCU_FW_START_ADDR_BASE_IDX 2
+#define mmDMCU_FW_END_ADDR 0x00de
+#define mmDMCU_FW_END_ADDR_BASE_IDX 2
+#define mmDMCU_FW_ISR_START_ADDR 0x00df
+#define mmDMCU_FW_ISR_START_ADDR_BASE_IDX 2
+#define mmDMCU_FW_CS_HI 0x00e0
+#define mmDMCU_FW_CS_HI_BASE_IDX 2
+#define mmDMCU_FW_CS_LO 0x00e1
+#define mmDMCU_FW_CS_LO_BASE_IDX 2
+#define mmDMCU_RAM_ACCESS_CTRL 0x00e2
+#define mmDMCU_RAM_ACCESS_CTRL_BASE_IDX 2
+#define mmDMCU_ERAM_WR_CTRL 0x00e3
+#define mmDMCU_ERAM_WR_CTRL_BASE_IDX 2
+#define mmDMCU_ERAM_WR_DATA 0x00e4
+#define mmDMCU_ERAM_WR_DATA_BASE_IDX 2
+#define mmDMCU_ERAM_RD_CTRL 0x00e5
+#define mmDMCU_ERAM_RD_CTRL_BASE_IDX 2
+#define mmDMCU_ERAM_RD_DATA 0x00e6
+#define mmDMCU_ERAM_RD_DATA_BASE_IDX 2
+#define mmDMCU_IRAM_WR_CTRL 0x00e7
+#define mmDMCU_IRAM_WR_CTRL_BASE_IDX 2
+#define mmDMCU_IRAM_WR_DATA 0x00e8
+#define mmDMCU_IRAM_WR_DATA_BASE_IDX 2
+#define mmDMCU_IRAM_RD_CTRL 0x00e9
+#define mmDMCU_IRAM_RD_CTRL_BASE_IDX 2
+#define mmDMCU_IRAM_RD_DATA 0x00ea
+#define mmDMCU_IRAM_RD_DATA_BASE_IDX 2
+#define mmDMCU_EVENT_TRIGGER 0x00eb
+#define mmDMCU_EVENT_TRIGGER_BASE_IDX 2
+#define mmDMCU_UC_INTERNAL_INT_STATUS 0x00ec
+#define mmDMCU_UC_INTERNAL_INT_STATUS_BASE_IDX 2
+#define mmDMCU_SS_INTERRUPT_CNTL_STATUS 0x00ed
+#define mmDMCU_SS_INTERRUPT_CNTL_STATUS_BASE_IDX 2
+#define mmDMCU_INTERRUPT_STATUS 0x00ee
+#define mmDMCU_INTERRUPT_STATUS_BASE_IDX 2
+#define mmDMCU_INTERRUPT_STATUS_1 0x00ef
+#define mmDMCU_INTERRUPT_STATUS_1_BASE_IDX 2
+#define mmDMCU_INTERRUPT_TO_HOST_EN_MASK 0x00f0
+#define mmDMCU_INTERRUPT_TO_HOST_EN_MASK_BASE_IDX 2
+#define mmDMCU_INTERRUPT_TO_UC_EN_MASK 0x00f1
+#define mmDMCU_INTERRUPT_TO_UC_EN_MASK_BASE_IDX 2
+#define mmDMCU_INTERRUPT_TO_UC_EN_MASK_1 0x00f2
+#define mmDMCU_INTERRUPT_TO_UC_EN_MASK_1_BASE_IDX 2
+#define mmDMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL 0x00f3
+#define mmDMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL_BASE_IDX 2
+#define mmDMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL_1 0x00f4
+#define mmDMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL_1_BASE_IDX 2
+#define mmDC_DMCU_SCRATCH 0x00f5
+#define mmDC_DMCU_SCRATCH_BASE_IDX 2
+#define mmDMCU_INT_CNT 0x00f6
+#define mmDMCU_INT_CNT_BASE_IDX 2
+#define mmDMCU_FW_CHECKSUM_SMPL_BYTE_POS 0x00f7
+#define mmDMCU_FW_CHECKSUM_SMPL_BYTE_POS_BASE_IDX 2
+#define mmDMCU_UC_CLK_GATING_CNTL 0x00f8
+#define mmDMCU_UC_CLK_GATING_CNTL_BASE_IDX 2
+#define mmMASTER_COMM_DATA_REG1 0x00f9
+#define mmMASTER_COMM_DATA_REG1_BASE_IDX 2
+#define mmMASTER_COMM_DATA_REG2 0x00fa
+#define mmMASTER_COMM_DATA_REG2_BASE_IDX 2
+#define mmMASTER_COMM_DATA_REG3 0x00fb
+#define mmMASTER_COMM_DATA_REG3_BASE_IDX 2
+#define mmMASTER_COMM_CMD_REG 0x00fc
+#define mmMASTER_COMM_CMD_REG_BASE_IDX 2
+#define mmMASTER_COMM_CNTL_REG 0x00fd
+#define mmMASTER_COMM_CNTL_REG_BASE_IDX 2
+#define mmSLAVE_COMM_DATA_REG1 0x00fe
+#define mmSLAVE_COMM_DATA_REG1_BASE_IDX 2
+#define mmSLAVE_COMM_DATA_REG2 0x00ff
+#define mmSLAVE_COMM_DATA_REG2_BASE_IDX 2
+#define mmSLAVE_COMM_DATA_REG3 0x0100
+#define mmSLAVE_COMM_DATA_REG3_BASE_IDX 2
+#define mmSLAVE_COMM_CMD_REG 0x0101
+#define mmSLAVE_COMM_CMD_REG_BASE_IDX 2
+#define mmSLAVE_COMM_CNTL_REG 0x0102
+#define mmSLAVE_COMM_CNTL_REG_BASE_IDX 2
+#define mmDMCU_PERFMON_INTERRUPT_STATUS1 0x0105
+#define mmDMCU_PERFMON_INTERRUPT_STATUS1_BASE_IDX 2
+#define mmDMCU_PERFMON_INTERRUPT_STATUS2 0x0106
+#define mmDMCU_PERFMON_INTERRUPT_STATUS2_BASE_IDX 2
+#define mmDMCU_PERFMON_INTERRUPT_STATUS3 0x0107
+#define mmDMCU_PERFMON_INTERRUPT_STATUS3_BASE_IDX 2
+#define mmDMCU_PERFMON_INTERRUPT_STATUS4 0x0108
+#define mmDMCU_PERFMON_INTERRUPT_STATUS4_BASE_IDX 2
+#define mmDMCU_PERFMON_INTERRUPT_STATUS5 0x0109
+#define mmDMCU_PERFMON_INTERRUPT_STATUS5_BASE_IDX 2
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK1 0x010a
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK1_BASE_IDX 2
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK2 0x010b
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK2_BASE_IDX 2
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK3 0x010c
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK3_BASE_IDX 2
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK4 0x010d
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK4_BASE_IDX 2
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK5 0x010e
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK5_BASE_IDX 2
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL1 0x010f
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL1_BASE_IDX 2
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL2 0x0110
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL2_BASE_IDX 2
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL3 0x0111
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL3_BASE_IDX 2
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL4 0x0112
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL4_BASE_IDX 2
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL5 0x0113
+#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL5_BASE_IDX 2
+#define mmDMCU_DPRX_INTERRUPT_STATUS1 0x0114
+#define mmDMCU_DPRX_INTERRUPT_STATUS1_BASE_IDX 2
+#define mmDMCU_DPRX_INTERRUPT_TO_UC_EN_MASK1 0x0115
+#define mmDMCU_DPRX_INTERRUPT_TO_UC_EN_MASK1_BASE_IDX 2
+#define mmDMCU_DPRX_INTERRUPT_TO_UC_XIRQ_IRQ_SEL1 0x0116
+#define mmDMCU_DPRX_INTERRUPT_TO_UC_XIRQ_IRQ_SEL1_BASE_IDX 2
+#define mmDMCU_INTERRUPT_STATUS_CONTINUE 0x0119
+#define mmDMCU_INTERRUPT_STATUS_CONTINUE_BASE_IDX 2
+#define mmDMCU_INTERRUPT_TO_UC_EN_MASK_CONTINUE 0x011a
+#define mmDMCU_INTERRUPT_TO_UC_EN_MASK_CONTINUE_BASE_IDX 2
+#define mmDMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL_CONTINUE 0x011b
+#define mmDMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL_CONTINUE_BASE_IDX 2
+#define mmDMCU_INT_CNT_CONTINUE 0x011c
+#define mmDMCU_INT_CNT_CONTINUE_BASE_IDX 2
+#define mmDMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL_CONT2 0x011d
+#define mmDMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL_CONT2_BASE_IDX 2
+#define mmDMCU_INTERRUPT_STATUS_2 0x011e
+#define mmDMCU_INTERRUPT_STATUS_2_BASE_IDX 2
+#define mmDMCU_INTERRUPT_TO_UC_EN_MASK_2 0x011f
+#define mmDMCU_INTERRUPT_TO_UC_EN_MASK_2_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dmu_ihc_dispdec
+// base address: 0x0
+#define mmDC_GPU_TIMER_START_POSITION_V_UPDATE 0x0126
+#define mmDC_GPU_TIMER_START_POSITION_V_UPDATE_BASE_IDX 2
+#define mmDC_GPU_TIMER_START_POSITION_VSTARTUP 0x0127
+#define mmDC_GPU_TIMER_START_POSITION_VSTARTUP_BASE_IDX 2
+#define mmDC_GPU_TIMER_READ 0x0128
+#define mmDC_GPU_TIMER_READ_BASE_IDX 2
+#define mmDC_GPU_TIMER_READ_CNTL 0x0129
+#define mmDC_GPU_TIMER_READ_CNTL_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS 0x012a
+#define mmDISP_INTERRUPT_STATUS_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE 0x012b
+#define mmDISP_INTERRUPT_STATUS_CONTINUE_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE2 0x012c
+#define mmDISP_INTERRUPT_STATUS_CONTINUE2_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE3 0x012d
+#define mmDISP_INTERRUPT_STATUS_CONTINUE3_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE4 0x012e
+#define mmDISP_INTERRUPT_STATUS_CONTINUE4_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE5 0x012f
+#define mmDISP_INTERRUPT_STATUS_CONTINUE5_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE6 0x0130
+#define mmDISP_INTERRUPT_STATUS_CONTINUE6_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE7 0x0131
+#define mmDISP_INTERRUPT_STATUS_CONTINUE7_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE8 0x0132
+#define mmDISP_INTERRUPT_STATUS_CONTINUE8_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE9 0x0133
+#define mmDISP_INTERRUPT_STATUS_CONTINUE9_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE10 0x0134
+#define mmDISP_INTERRUPT_STATUS_CONTINUE10_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE11 0x0135
+#define mmDISP_INTERRUPT_STATUS_CONTINUE11_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE12 0x0136
+#define mmDISP_INTERRUPT_STATUS_CONTINUE12_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE13 0x0137
+#define mmDISP_INTERRUPT_STATUS_CONTINUE13_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE14 0x0138
+#define mmDISP_INTERRUPT_STATUS_CONTINUE14_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE15 0x0139
+#define mmDISP_INTERRUPT_STATUS_CONTINUE15_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE16 0x013a
+#define mmDISP_INTERRUPT_STATUS_CONTINUE16_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE17 0x013b
+#define mmDISP_INTERRUPT_STATUS_CONTINUE17_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE18 0x013c
+#define mmDISP_INTERRUPT_STATUS_CONTINUE18_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE19 0x013d
+#define mmDISP_INTERRUPT_STATUS_CONTINUE19_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE20 0x013e
+#define mmDISP_INTERRUPT_STATUS_CONTINUE20_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE21 0x013f
+#define mmDISP_INTERRUPT_STATUS_CONTINUE21_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE22 0x0140
+#define mmDISP_INTERRUPT_STATUS_CONTINUE22_BASE_IDX 2
+#define mmDC_GPU_TIMER_START_POSITION_VREADY 0x0141
+#define mmDC_GPU_TIMER_START_POSITION_VREADY_BASE_IDX 2
+#define mmDC_GPU_TIMER_START_POSITION_FLIP 0x0142
+#define mmDC_GPU_TIMER_START_POSITION_FLIP_BASE_IDX 2
+#define mmDC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK 0x0143
+#define mmDC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK_BASE_IDX 2
+#define mmDC_GPU_TIMER_START_POSITION_FLIP_AWAY 0x0144
+#define mmDC_GPU_TIMER_START_POSITION_FLIP_AWAY_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE23 0x0145
+#define mmDISP_INTERRUPT_STATUS_CONTINUE23_BASE_IDX 2
+#define mmDISP_INTERRUPT_STATUS_CONTINUE24 0x0146
+#define mmDISP_INTERRUPT_STATUS_CONTINUE24_BASE_IDX 2
+#define mmDCCG_INTERRUPT_DEST 0x0147
+#define mmDCCG_INTERRUPT_DEST_BASE_IDX 2
+#define mmDMU_INTERRUPT_DEST 0x0148
+#define mmDMU_INTERRUPT_DEST_BASE_IDX 2
+#define mmDCPG_INTERRUPT_DEST 0x0149
+#define mmDCPG_INTERRUPT_DEST_BASE_IDX 2
+#define mmDCPG_INTERRUPT_DEST2 0x014a
+#define mmDCPG_INTERRUPT_DEST2_BASE_IDX 2
+#define mmMMHUBBUB_INTERRUPT_DEST 0x014b
+#define mmMMHUBBUB_INTERRUPT_DEST_BASE_IDX 2
+#define mmWB_INTERRUPT_DEST 0x014c
+#define mmWB_INTERRUPT_DEST_BASE_IDX 2
+#define mmDCHUB_INTERRUPT_DEST 0x014d
+#define mmDCHUB_INTERRUPT_DEST_BASE_IDX 2
+#define mmDCHUB_PERFCOUNTER_INTERRUPT_DEST 0x014e
+#define mmDCHUB_PERFCOUNTER_INTERRUPT_DEST_BASE_IDX 2
+#define mmDCHUB_INTERRUPT_DEST2 0x014f
+#define mmDCHUB_INTERRUPT_DEST2_BASE_IDX 2
+#define mmDPP_PERFCOUNTER_INTERRUPT_DEST 0x0150
+#define mmDPP_PERFCOUNTER_INTERRUPT_DEST_BASE_IDX 2
+#define mmMPC_INTERRUPT_DEST 0x0151
+#define mmMPC_INTERRUPT_DEST_BASE_IDX 2
+#define mmOPP_INTERRUPT_DEST 0x0152
+#define mmOPP_INTERRUPT_DEST_BASE_IDX 2
+#define mmOPTC_INTERRUPT_DEST 0x0153
+#define mmOPTC_INTERRUPT_DEST_BASE_IDX 2
+#define mmOTG0_INTERRUPT_DEST 0x0154
+#define mmOTG0_INTERRUPT_DEST_BASE_IDX 2
+#define mmOTG1_INTERRUPT_DEST 0x0155
+#define mmOTG1_INTERRUPT_DEST_BASE_IDX 2
+#define mmOTG2_INTERRUPT_DEST 0x0156
+#define mmOTG2_INTERRUPT_DEST_BASE_IDX 2
+#define mmOTG3_INTERRUPT_DEST 0x0157
+#define mmOTG3_INTERRUPT_DEST_BASE_IDX 2
+#define mmOTG4_INTERRUPT_DEST 0x0158
+#define mmOTG4_INTERRUPT_DEST_BASE_IDX 2
+#define mmOTG5_INTERRUPT_DEST 0x0159
+#define mmOTG5_INTERRUPT_DEST_BASE_IDX 2
+#define mmDIG_INTERRUPT_DEST 0x015a
+#define mmDIG_INTERRUPT_DEST_BASE_IDX 2
+#define mmI2C_DDC_HPD_INTERRUPT_DEST 0x015b
+#define mmI2C_DDC_HPD_INTERRUPT_DEST_BASE_IDX 2
+#define mmDIO_INTERRUPT_DEST 0x015d
+#define mmDIO_INTERRUPT_DEST_BASE_IDX 2
+#define mmDCIO_INTERRUPT_DEST 0x015e
+#define mmDCIO_INTERRUPT_DEST_BASE_IDX 2
+#define mmHPD_INTERRUPT_DEST 0x015f
+#define mmHPD_INTERRUPT_DEST_BASE_IDX 2
+#define mmAZ_INTERRUPT_DEST 0x0160
+#define mmAZ_INTERRUPT_DEST_BASE_IDX 2
+#define mmAUX_INTERRUPT_DEST 0x0161
+#define mmAUX_INTERRUPT_DEST_BASE_IDX 2
+#define mmDSC_INTERRUPT_DEST 0x0162
+#define mmDSC_INTERRUPT_DEST_BASE_IDX 2
+
+
+// addressBlock: dce_dc_wb0_dispdec_cnv_dispdec
+// base address: 0x0
+#define mmWB_ENABLE 0x01da
+#define mmWB_ENABLE_BASE_IDX 2
+#define mmWB_EC_CONFIG 0x01db
+#define mmWB_EC_CONFIG_BASE_IDX 2
+#define mmCNV_MODE 0x01dc
+#define mmCNV_MODE_BASE_IDX 2
+#define mmCNV_WINDOW_START 0x01dd
+#define mmCNV_WINDOW_START_BASE_IDX 2
+#define mmCNV_WINDOW_SIZE 0x01de
+#define mmCNV_WINDOW_SIZE_BASE_IDX 2
+#define mmCNV_UPDATE 0x01df
+#define mmCNV_UPDATE_BASE_IDX 2
+#define mmCNV_SOURCE_SIZE 0x01e0
+#define mmCNV_SOURCE_SIZE_BASE_IDX 2
+#define mmCNV_TEST_CNTL 0x01ee
+#define mmCNV_TEST_CNTL_BASE_IDX 2
+#define mmCNV_TEST_CRC_RED 0x01ef
+#define mmCNV_TEST_CRC_RED_BASE_IDX 2
+#define mmCNV_TEST_CRC_GREEN 0x01f0
+#define mmCNV_TEST_CRC_GREEN_BASE_IDX 2
+#define mmCNV_TEST_CRC_BLUE 0x01f1
+#define mmCNV_TEST_CRC_BLUE_BASE_IDX 2
+#define mmWB_DEBUG_CTRL 0x01f2
+#define mmWB_DEBUG_CTRL_BASE_IDX 2
+#define mmWB_DBG_MODE 0x01f3
+#define mmWB_DBG_MODE_BASE_IDX 2
+#define mmWB_HW_DEBUG 0x01f4
+#define mmWB_HW_DEBUG_BASE_IDX 2
+#define mmWB_SOFT_RESET 0x01f5
+#define mmWB_SOFT_RESET_BASE_IDX 2
+#define mmWB_WARM_UP_MODE_CTL1 0x01f6
+#define mmWB_WARM_UP_MODE_CTL1_BASE_IDX 2
+#define mmWB_WARM_UP_MODE_CTL2 0x01f7
+#define mmWB_WARM_UP_MODE_CTL2_BASE_IDX 2
+#define mmCNV_TEST_DEBUG_INDEX 0x01f8
+#define mmCNV_TEST_DEBUG_INDEX_BASE_IDX 2
+#define mmCNV_TEST_DEBUG_DATA 0x01f9
+#define mmCNV_TEST_DEBUG_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_wb0_dispdec_wbscl_dispdec
+// base address: 0x0
+#define mmWBSCL_COEF_RAM_SELECT 0x020a
+#define mmWBSCL_COEF_RAM_SELECT_BASE_IDX 2
+#define mmWBSCL_COEF_RAM_TAP_DATA 0x020b
+#define mmWBSCL_COEF_RAM_TAP_DATA_BASE_IDX 2
+#define mmWBSCL_MODE 0x020c
+#define mmWBSCL_MODE_BASE_IDX 2
+#define mmWBSCL_TAP_CONTROL 0x020d
+#define mmWBSCL_TAP_CONTROL_BASE_IDX 2
+#define mmWBSCL_DEST_SIZE 0x020e
+#define mmWBSCL_DEST_SIZE_BASE_IDX 2
+#define mmWBSCL_HORZ_FILTER_SCALE_RATIO 0x020f
+#define mmWBSCL_HORZ_FILTER_SCALE_RATIO_BASE_IDX 2
+#define mmWBSCL_HORZ_FILTER_INIT_Y_RGB 0x0210
+#define mmWBSCL_HORZ_FILTER_INIT_Y_RGB_BASE_IDX 2
+#define mmWBSCL_HORZ_FILTER_INIT_CBCR 0x0211
+#define mmWBSCL_HORZ_FILTER_INIT_CBCR_BASE_IDX 2
+#define mmWBSCL_VERT_FILTER_SCALE_RATIO 0x0212
+#define mmWBSCL_VERT_FILTER_SCALE_RATIO_BASE_IDX 2
+#define mmWBSCL_VERT_FILTER_INIT_Y_RGB 0x0213
+#define mmWBSCL_VERT_FILTER_INIT_Y_RGB_BASE_IDX 2
+#define mmWBSCL_VERT_FILTER_INIT_CBCR 0x0214
+#define mmWBSCL_VERT_FILTER_INIT_CBCR_BASE_IDX 2
+#define mmWBSCL_ROUND_OFFSET 0x0215
+#define mmWBSCL_ROUND_OFFSET_BASE_IDX 2
+#define mmWBSCL_OVERFLOW_STATUS 0x0216
+#define mmWBSCL_OVERFLOW_STATUS_BASE_IDX 2
+#define mmWBSCL_COEF_RAM_CONFLICT_STATUS 0x0217
+#define mmWBSCL_COEF_RAM_CONFLICT_STATUS_BASE_IDX 2
+#define mmWBSCL_TEST_CNTL 0x0218
+#define mmWBSCL_TEST_CNTL_BASE_IDX 2
+#define mmWBSCL_TEST_CRC_RED 0x0219
+#define mmWBSCL_TEST_CRC_RED_BASE_IDX 2
+#define mmWBSCL_TEST_CRC_GREEN 0x021a
+#define mmWBSCL_TEST_CRC_GREEN_BASE_IDX 2
+#define mmWBSCL_TEST_CRC_BLUE 0x021b
+#define mmWBSCL_TEST_CRC_BLUE_BASE_IDX 2
+#define mmWBSCL_BACKPRESSURE_CNT_EN 0x021c
+#define mmWBSCL_BACKPRESSURE_CNT_EN_BASE_IDX 2
+#define mmWB_MCIF_BACKPRESSURE_CNT 0x021d
+#define mmWB_MCIF_BACKPRESSURE_CNT_BASE_IDX 2
+#define mmWBSCL_CLAMP_Y_RGB 0x021e
+#define mmWBSCL_CLAMP_Y_RGB_BASE_IDX 2
+#define mmWBSCL_CLAMP_CBCR 0x021f
+#define mmWBSCL_CLAMP_CBCR_BASE_IDX 2
+#define mmWBSCL_OUTSIDE_PIX_STRATEGY 0x0220
+#define mmWBSCL_OUTSIDE_PIX_STRATEGY_BASE_IDX 2
+#define mmWBSCL_OUTSIDE_PIX_STRATEGY_CBCR 0x0221
+#define mmWBSCL_OUTSIDE_PIX_STRATEGY_CBCR_BASE_IDX 2
+#define mmWBSCL_DEBUG 0x0222
+#define mmWBSCL_DEBUG_BASE_IDX 2
+#define mmWBSCL_TEST_DEBUG_INDEX 0x0223
+#define mmWBSCL_TEST_DEBUG_INDEX_BASE_IDX 2
+#define mmWBSCL_TEST_DEBUG_DATA 0x0224
+#define mmWBSCL_TEST_DEBUG_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_wb0_dispdec_wb_dcperfmon_dc_perfmon_dispdec
+// base address: 0x8e8
+#define mmDC_PERFMON3_PERFCOUNTER_CNTL 0x023a
+#define mmDC_PERFMON3_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON3_PERFCOUNTER_CNTL2 0x023b
+#define mmDC_PERFMON3_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON3_PERFCOUNTER_STATE 0x023c
+#define mmDC_PERFMON3_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON3_PERFMON_CNTL 0x023d
+#define mmDC_PERFMON3_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON3_PERFMON_CNTL2 0x023e
+#define mmDC_PERFMON3_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON3_PERFMON_CVALUE_INT_MISC 0x023f
+#define mmDC_PERFMON3_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON3_PERFMON_CVALUE_LOW 0x0240
+#define mmDC_PERFMON3_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON3_PERFMON_HI 0x0241
+#define mmDC_PERFMON3_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON3_PERFMON_LOW 0x0242
+#define mmDC_PERFMON3_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mmhubbub_mcif_wb0_dispdec
+// base address: 0x0
+#define mmMCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL 0x02b2
+#define mmMCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUFMGR_CUR_LINE_R 0x02b3
+#define mmMCIF_WB0_MCIF_WB_BUFMGR_CUR_LINE_R_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUFMGR_STATUS 0x02b4
+#define mmMCIF_WB0_MCIF_WB_BUFMGR_STATUS_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_PITCH 0x02b5
+#define mmMCIF_WB0_MCIF_WB_BUF_PITCH_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_1_STATUS 0x02b6
+#define mmMCIF_WB0_MCIF_WB_BUF_1_STATUS_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_1_STATUS2 0x02b7
+#define mmMCIF_WB0_MCIF_WB_BUF_1_STATUS2_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_2_STATUS 0x02b8
+#define mmMCIF_WB0_MCIF_WB_BUF_2_STATUS_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_2_STATUS2 0x02b9
+#define mmMCIF_WB0_MCIF_WB_BUF_2_STATUS2_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_3_STATUS 0x02ba
+#define mmMCIF_WB0_MCIF_WB_BUF_3_STATUS_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_3_STATUS2 0x02bb
+#define mmMCIF_WB0_MCIF_WB_BUF_3_STATUS2_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_4_STATUS 0x02bc
+#define mmMCIF_WB0_MCIF_WB_BUF_4_STATUS_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_4_STATUS2 0x02bd
+#define mmMCIF_WB0_MCIF_WB_BUF_4_STATUS2_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_ARBITRATION_CONTROL 0x02be
+#define mmMCIF_WB0_MCIF_WB_ARBITRATION_CONTROL_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_SCLK_CHANGE 0x02bf
+#define mmMCIF_WB0_MCIF_WB_SCLK_CHANGE_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_TEST_DEBUG_INDEX 0x02c0
+#define mmMCIF_WB0_MCIF_WB_TEST_DEBUG_INDEX_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_TEST_DEBUG_DATA 0x02c1
+#define mmMCIF_WB0_MCIF_WB_TEST_DEBUG_DATA_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_Y 0x02c2
+#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_Y_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_Y_OFFSET 0x02c3
+#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_Y_OFFSET_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_C 0x02c4
+#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_C_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_C_OFFSET 0x02c5
+#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_C_OFFSET_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_Y 0x02c6
+#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_Y_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_Y_OFFSET 0x02c7
+#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_Y_OFFSET_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_C 0x02c8
+#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_C_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_C_OFFSET 0x02c9
+#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_C_OFFSET_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_Y 0x02ca
+#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_Y_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_Y_OFFSET 0x02cb
+#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_Y_OFFSET_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_C 0x02cc
+#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_C_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_C_OFFSET 0x02cd
+#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_C_OFFSET_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_Y 0x02ce
+#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_Y_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_Y_OFFSET 0x02cf
+#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_Y_OFFSET_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_C 0x02d0
+#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_C_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_C_OFFSET 0x02d1
+#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_C_OFFSET_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL 0x02d2
+#define mmMCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_NB_PSTATE_LATENCY_WATERMARK 0x02d3
+#define mmMCIF_WB0_MCIF_WB_NB_PSTATE_LATENCY_WATERMARK_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_NB_PSTATE_CONTROL 0x02d4
+#define mmMCIF_WB0_MCIF_WB_NB_PSTATE_CONTROL_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_WATERMARK 0x02d5
+#define mmMCIF_WB0_MCIF_WB_WATERMARK_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_CLOCK_GATER_CONTROL 0x02d6
+#define mmMCIF_WB0_MCIF_WB_CLOCK_GATER_CONTROL_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_WARM_UP_CNTL 0x02d7
+#define mmMCIF_WB0_MCIF_WB_WARM_UP_CNTL_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_SELF_REFRESH_CONTROL 0x02d8
+#define mmMCIF_WB0_MCIF_WB_SELF_REFRESH_CONTROL_BASE_IDX 2
+#define mmMCIF_WB0_MULTI_LEVEL_QOS_CTRL 0x02d9
+#define mmMCIF_WB0_MULTI_LEVEL_QOS_CTRL_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_SECURITY_LEVEL 0x02da
+#define mmMCIF_WB0_MCIF_WB_SECURITY_LEVEL_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_LUMA_SIZE 0x02db
+#define mmMCIF_WB0_MCIF_WB_BUF_LUMA_SIZE_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_CHROMA_SIZE 0x02dc
+#define mmMCIF_WB0_MCIF_WB_BUF_CHROMA_SIZE_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_Y_HIGH 0x02dd
+#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_Y_HIGH_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_C_HIGH 0x02de
+#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_C_HIGH_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_Y_HIGH 0x02df
+#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_Y_HIGH_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_C_HIGH 0x02e0
+#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_C_HIGH_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_Y_HIGH 0x02e1
+#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_Y_HIGH_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_C_HIGH 0x02e2
+#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_C_HIGH_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_Y_HIGH 0x02e3
+#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_Y_HIGH_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_C_HIGH 0x02e4
+#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_C_HIGH_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_1_RESOLUTION 0x02e5
+#define mmMCIF_WB0_MCIF_WB_BUF_1_RESOLUTION_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_2_RESOLUTION 0x02e6
+#define mmMCIF_WB0_MCIF_WB_BUF_2_RESOLUTION_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_3_RESOLUTION 0x02e7
+#define mmMCIF_WB0_MCIF_WB_BUF_3_RESOLUTION_BASE_IDX 2
+#define mmMCIF_WB0_MCIF_WB_BUF_4_RESOLUTION 0x02e8
+#define mmMCIF_WB0_MCIF_WB_BUF_4_RESOLUTION_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mmhubbub_mcif_wb1_dispdec
+// base address: 0x100
+#define mmMCIF_WB1_MCIF_WB_BUFMGR_SW_CONTROL 0x02f2
+#define mmMCIF_WB1_MCIF_WB_BUFMGR_SW_CONTROL_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUFMGR_CUR_LINE_R 0x02f3
+#define mmMCIF_WB1_MCIF_WB_BUFMGR_CUR_LINE_R_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUFMGR_STATUS 0x02f4
+#define mmMCIF_WB1_MCIF_WB_BUFMGR_STATUS_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_PITCH 0x02f5
+#define mmMCIF_WB1_MCIF_WB_BUF_PITCH_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_1_STATUS 0x02f6
+#define mmMCIF_WB1_MCIF_WB_BUF_1_STATUS_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_1_STATUS2 0x02f7
+#define mmMCIF_WB1_MCIF_WB_BUF_1_STATUS2_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_2_STATUS 0x02f8
+#define mmMCIF_WB1_MCIF_WB_BUF_2_STATUS_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_2_STATUS2 0x02f9
+#define mmMCIF_WB1_MCIF_WB_BUF_2_STATUS2_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_3_STATUS 0x02fa
+#define mmMCIF_WB1_MCIF_WB_BUF_3_STATUS_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_3_STATUS2 0x02fb
+#define mmMCIF_WB1_MCIF_WB_BUF_3_STATUS2_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_4_STATUS 0x02fc
+#define mmMCIF_WB1_MCIF_WB_BUF_4_STATUS_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_4_STATUS2 0x02fd
+#define mmMCIF_WB1_MCIF_WB_BUF_4_STATUS2_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_ARBITRATION_CONTROL 0x02fe
+#define mmMCIF_WB1_MCIF_WB_ARBITRATION_CONTROL_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_SCLK_CHANGE 0x02ff
+#define mmMCIF_WB1_MCIF_WB_SCLK_CHANGE_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_TEST_DEBUG_INDEX 0x0300
+#define mmMCIF_WB1_MCIF_WB_TEST_DEBUG_INDEX_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_TEST_DEBUG_DATA 0x0301
+#define mmMCIF_WB1_MCIF_WB_TEST_DEBUG_DATA_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_Y 0x0302
+#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_Y_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_Y_OFFSET 0x0303
+#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_Y_OFFSET_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_C 0x0304
+#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_C_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_C_OFFSET 0x0305
+#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_C_OFFSET_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_Y 0x0306
+#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_Y_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_Y_OFFSET 0x0307
+#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_Y_OFFSET_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_C 0x0308
+#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_C_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_C_OFFSET 0x0309
+#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_C_OFFSET_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_Y 0x030a
+#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_Y_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_Y_OFFSET 0x030b
+#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_Y_OFFSET_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_C 0x030c
+#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_C_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_C_OFFSET 0x030d
+#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_C_OFFSET_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_Y 0x030e
+#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_Y_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_Y_OFFSET 0x030f
+#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_Y_OFFSET_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_C 0x0310
+#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_C_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_C_OFFSET 0x0311
+#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_C_OFFSET_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUFMGR_VCE_CONTROL 0x0312
+#define mmMCIF_WB1_MCIF_WB_BUFMGR_VCE_CONTROL_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_NB_PSTATE_LATENCY_WATERMARK 0x0313
+#define mmMCIF_WB1_MCIF_WB_NB_PSTATE_LATENCY_WATERMARK_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_NB_PSTATE_CONTROL 0x0314
+#define mmMCIF_WB1_MCIF_WB_NB_PSTATE_CONTROL_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_WATERMARK 0x0315
+#define mmMCIF_WB1_MCIF_WB_WATERMARK_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_CLOCK_GATER_CONTROL 0x0316
+#define mmMCIF_WB1_MCIF_WB_CLOCK_GATER_CONTROL_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_WARM_UP_CNTL 0x0317
+#define mmMCIF_WB1_MCIF_WB_WARM_UP_CNTL_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_SELF_REFRESH_CONTROL 0x0318
+#define mmMCIF_WB1_MCIF_WB_SELF_REFRESH_CONTROL_BASE_IDX 2
+#define mmMCIF_WB1_MULTI_LEVEL_QOS_CTRL 0x0319
+#define mmMCIF_WB1_MULTI_LEVEL_QOS_CTRL_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_LUMA_SIZE 0x031b
+#define mmMCIF_WB1_MCIF_WB_BUF_LUMA_SIZE_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_CHROMA_SIZE 0x031c
+#define mmMCIF_WB1_MCIF_WB_BUF_CHROMA_SIZE_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_Y_HIGH 0x031d
+#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_Y_HIGH_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_C_HIGH 0x031e
+#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_C_HIGH_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_Y_HIGH 0x031f
+#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_Y_HIGH_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_C_HIGH 0x0320
+#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_C_HIGH_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_Y_HIGH 0x0321
+#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_Y_HIGH_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_C_HIGH 0x0322
+#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_C_HIGH_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_Y_HIGH 0x0323
+#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_Y_HIGH_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_C_HIGH 0x0324
+#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_C_HIGH_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_1_RESOLUTION 0x0325
+#define mmMCIF_WB1_MCIF_WB_BUF_1_RESOLUTION_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_2_RESOLUTION 0x0326
+#define mmMCIF_WB1_MCIF_WB_BUF_2_RESOLUTION_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_3_RESOLUTION 0x0327
+#define mmMCIF_WB1_MCIF_WB_BUF_3_RESOLUTION_BASE_IDX 2
+#define mmMCIF_WB1_MCIF_WB_BUF_4_RESOLUTION 0x0328
+#define mmMCIF_WB1_MCIF_WB_BUF_4_RESOLUTION_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mmhubbub_mmhubbub_dispdec
+// base address: 0x0
+#define mmWBIF0_MISC_CTRL 0x0333
+#define mmWBIF0_MISC_CTRL_BASE_IDX 2
+#define mmWBIF0_SMU_WM_CONTROL 0x0334
+#define mmWBIF0_SMU_WM_CONTROL_BASE_IDX 2
+#define mmWBIF0_PHASE0_OUTSTANDING_COUNTER 0x0335
+#define mmWBIF0_PHASE0_OUTSTANDING_COUNTER_BASE_IDX 2
+#define mmWBIF0_PHASE1_OUTSTANDING_COUNTER 0x0336
+#define mmWBIF0_PHASE1_OUTSTANDING_COUNTER_BASE_IDX 2
+#define mmVGA_SRC_SPLIT_CNTL 0x033f
+#define mmVGA_SRC_SPLIT_CNTL_BASE_IDX 2
+#define mmMMHUBBUB_MEM_PWR_STATUS 0x0340
+#define mmMMHUBBUB_MEM_PWR_STATUS_BASE_IDX 2
+#define mmMMHUBBUB_MEM_PWR_CNTL 0x0341
+#define mmMMHUBBUB_MEM_PWR_CNTL_BASE_IDX 2
+#define mmMMHUBBUB_CLOCK_CNTL 0x0342
+#define mmMMHUBBUB_CLOCK_CNTL_BASE_IDX 2
+#define mmMMHUBBUB_SOFT_RESET 0x0343
+#define mmMMHUBBUB_SOFT_RESET_BASE_IDX 2
+#define mmDMU_IF_ERR_STATUS 0x0347
+#define mmDMU_IF_ERR_STATUS_BASE_IDX 2
+#define mmMMHUBBUB_CLIENT_UNIT_ID 0x0348
+#define mmMMHUBBUB_CLIENT_UNIT_ID_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mmhubbub_vgaif_dispdec
+// base address: 0x0
+#define mmMCIF_CONTROL 0x034a
+#define mmMCIF_CONTROL_BASE_IDX 2
+#define mmMCIF_WRITE_COMBINE_CONTROL 0x034b
+#define mmMCIF_WRITE_COMBINE_CONTROL_BASE_IDX 2
+#define mmMCIF_PHASE0_OUTSTANDING_COUNTER 0x034e
+#define mmMCIF_PHASE0_OUTSTANDING_COUNTER_BASE_IDX 2
+#define mmMCIF_PHASE1_OUTSTANDING_COUNTER 0x034f
+#define mmMCIF_PHASE1_OUTSTANDING_COUNTER_BASE_IDX 2
+#define mmMCIF_PHASE2_OUTSTANDING_COUNTER 0x0350
+#define mmMCIF_PHASE2_OUTSTANDING_COUNTER_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mmhubbub_mmhubbub_dcperfmon_dc_perfmon_dispdec
+// base address: 0xd48
+#define mmDC_PERFMON4_PERFCOUNTER_CNTL 0x0352
+#define mmDC_PERFMON4_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON4_PERFCOUNTER_CNTL2 0x0353
+#define mmDC_PERFMON4_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON4_PERFCOUNTER_STATE 0x0354
+#define mmDC_PERFMON4_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON4_PERFMON_CNTL 0x0355
+#define mmDC_PERFMON4_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON4_PERFMON_CNTL2 0x0356
+#define mmDC_PERFMON4_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON4_PERFMON_CVALUE_INT_MISC 0x0357
+#define mmDC_PERFMON4_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON4_PERFMON_CVALUE_LOW 0x0358
+#define mmDC_PERFMON4_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON4_PERFMON_HI 0x0359
+#define mmDC_PERFMON4_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON4_PERFMON_LOW 0x035a
+#define mmDC_PERFMON4_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0stream0_dispdec
+// base address: 0x0
+#define mmAZF0STREAM0_AZALIA_STREAM_INDEX 0x035e
+#define mmAZF0STREAM0_AZALIA_STREAM_INDEX_BASE_IDX 2
+#define mmAZF0STREAM0_AZALIA_STREAM_DATA 0x035f
+#define mmAZF0STREAM0_AZALIA_STREAM_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0stream1_dispdec
+// base address: 0x8
+#define mmAZF0STREAM1_AZALIA_STREAM_INDEX 0x0360
+#define mmAZF0STREAM1_AZALIA_STREAM_INDEX_BASE_IDX 2
+#define mmAZF0STREAM1_AZALIA_STREAM_DATA 0x0361
+#define mmAZF0STREAM1_AZALIA_STREAM_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0stream2_dispdec
+// base address: 0x10
+#define mmAZF0STREAM2_AZALIA_STREAM_INDEX 0x0362
+#define mmAZF0STREAM2_AZALIA_STREAM_INDEX_BASE_IDX 2
+#define mmAZF0STREAM2_AZALIA_STREAM_DATA 0x0363
+#define mmAZF0STREAM2_AZALIA_STREAM_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0stream3_dispdec
+// base address: 0x18
+#define mmAZF0STREAM3_AZALIA_STREAM_INDEX 0x0364
+#define mmAZF0STREAM3_AZALIA_STREAM_INDEX_BASE_IDX 2
+#define mmAZF0STREAM3_AZALIA_STREAM_DATA 0x0365
+#define mmAZF0STREAM3_AZALIA_STREAM_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0stream4_dispdec
+// base address: 0x20
+#define mmAZF0STREAM4_AZALIA_STREAM_INDEX 0x0366
+#define mmAZF0STREAM4_AZALIA_STREAM_INDEX_BASE_IDX 2
+#define mmAZF0STREAM4_AZALIA_STREAM_DATA 0x0367
+#define mmAZF0STREAM4_AZALIA_STREAM_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0stream5_dispdec
+// base address: 0x28
+#define mmAZF0STREAM5_AZALIA_STREAM_INDEX 0x0368
+#define mmAZF0STREAM5_AZALIA_STREAM_INDEX_BASE_IDX 2
+#define mmAZF0STREAM5_AZALIA_STREAM_DATA 0x0369
+#define mmAZF0STREAM5_AZALIA_STREAM_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0stream6_dispdec
+// base address: 0x30
+#define mmAZF0STREAM6_AZALIA_STREAM_INDEX 0x036a
+#define mmAZF0STREAM6_AZALIA_STREAM_INDEX_BASE_IDX 2
+#define mmAZF0STREAM6_AZALIA_STREAM_DATA 0x036b
+#define mmAZF0STREAM6_AZALIA_STREAM_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0stream7_dispdec
+// base address: 0x38
+#define mmAZF0STREAM7_AZALIA_STREAM_INDEX 0x036c
+#define mmAZF0STREAM7_AZALIA_STREAM_INDEX_BASE_IDX 2
+#define mmAZF0STREAM7_AZALIA_STREAM_DATA 0x036d
+#define mmAZF0STREAM7_AZALIA_STREAM_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_az_misc_dispdec
+// base address: 0x0
+#define mmAZ_CLOCK_CNTL 0x0372
+#define mmAZ_CLOCK_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_az_dcperfmon_dc_perfmon_dispdec
+// base address: 0xde8
+#define mmDC_PERFMON5_PERFCOUNTER_CNTL 0x037a
+#define mmDC_PERFMON5_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON5_PERFCOUNTER_CNTL2 0x037b
+#define mmDC_PERFMON5_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON5_PERFCOUNTER_STATE 0x037c
+#define mmDC_PERFMON5_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON5_PERFMON_CNTL 0x037d
+#define mmDC_PERFMON5_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON5_PERFMON_CNTL2 0x037e
+#define mmDC_PERFMON5_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON5_PERFMON_CVALUE_INT_MISC 0x037f
+#define mmDC_PERFMON5_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON5_PERFMON_CVALUE_LOW 0x0380
+#define mmDC_PERFMON5_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON5_PERFMON_HI 0x0381
+#define mmDC_PERFMON5_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON5_PERFMON_LOW 0x0382
+#define mmDC_PERFMON5_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0endpoint0_dispdec
+// base address: 0x0
+#define mmAZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX 0x0386
+#define mmAZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX_BASE_IDX 2
+#define mmAZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA 0x0387
+#define mmAZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0endpoint1_dispdec
+// base address: 0x18
+#define mmAZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_INDEX 0x038c
+#define mmAZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_INDEX_BASE_IDX 2
+#define mmAZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_DATA 0x038d
+#define mmAZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0endpoint2_dispdec
+// base address: 0x30
+#define mmAZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_INDEX 0x0392
+#define mmAZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_INDEX_BASE_IDX 2
+#define mmAZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_DATA 0x0393
+#define mmAZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0endpoint3_dispdec
+// base address: 0x48
+#define mmAZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_INDEX 0x0398
+#define mmAZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_INDEX_BASE_IDX 2
+#define mmAZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_DATA 0x0399
+#define mmAZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0endpoint4_dispdec
+// base address: 0x60
+#define mmAZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_INDEX 0x039e
+#define mmAZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_INDEX_BASE_IDX 2
+#define mmAZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_DATA 0x039f
+#define mmAZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0endpoint5_dispdec
+// base address: 0x78
+#define mmAZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_INDEX 0x03a4
+#define mmAZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_INDEX_BASE_IDX 2
+#define mmAZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_DATA 0x03a5
+#define mmAZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0endpoint6_dispdec
+// base address: 0x90
+#define mmAZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_INDEX 0x03aa
+#define mmAZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_INDEX_BASE_IDX 2
+#define mmAZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_DATA 0x03ab
+#define mmAZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0endpoint7_dispdec
+// base address: 0xa8
+#define mmAZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_INDEX 0x03b0
+#define mmAZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_INDEX_BASE_IDX 2
+#define mmAZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_DATA 0x03b1
+#define mmAZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0controller_dispdec
+// base address: 0x0
+#define mmAZALIA_CONTROLLER_CLOCK_GATING 0x03c2
+#define mmAZALIA_CONTROLLER_CLOCK_GATING_BASE_IDX 2
+#define mmAZALIA_AUDIO_DTO 0x03c3
+#define mmAZALIA_AUDIO_DTO_BASE_IDX 2
+#define mmAZALIA_AUDIO_DTO_CONTROL 0x03c4
+#define mmAZALIA_AUDIO_DTO_CONTROL_BASE_IDX 2
+#define mmAZALIA_SOCCLK_CONTROL 0x03c5
+#define mmAZALIA_SOCCLK_CONTROL_BASE_IDX 2
+#define mmAZALIA_UNDERFLOW_FILLER_SAMPLE 0x03c6
+#define mmAZALIA_UNDERFLOW_FILLER_SAMPLE_BASE_IDX 2
+#define mmAZALIA_DATA_DMA_CONTROL 0x03c7
+#define mmAZALIA_DATA_DMA_CONTROL_BASE_IDX 2
+#define mmAZALIA_BDL_DMA_CONTROL 0x03c8
+#define mmAZALIA_BDL_DMA_CONTROL_BASE_IDX 2
+#define mmAZALIA_RIRB_AND_DP_CONTROL 0x03c9
+#define mmAZALIA_RIRB_AND_DP_CONTROL_BASE_IDX 2
+#define mmAZALIA_CORB_DMA_CONTROL 0x03ca
+#define mmAZALIA_CORB_DMA_CONTROL_BASE_IDX 2
+#define mmAZALIA_APPLICATION_POSITION_IN_CYCLIC_BUFFER 0x03d1
+#define mmAZALIA_APPLICATION_POSITION_IN_CYCLIC_BUFFER_BASE_IDX 2
+#define mmAZALIA_CYCLIC_BUFFER_SYNC 0x03d2
+#define mmAZALIA_CYCLIC_BUFFER_SYNC_BASE_IDX 2
+#define mmAZALIA_GLOBAL_CAPABILITIES 0x03d3
+#define mmAZALIA_GLOBAL_CAPABILITIES_BASE_IDX 2
+#define mmAZALIA_OUTPUT_PAYLOAD_CAPABILITY 0x03d4
+#define mmAZALIA_OUTPUT_PAYLOAD_CAPABILITY_BASE_IDX 2
+#define mmAZALIA_OUTPUT_STREAM_ARBITER_CONTROL 0x03d5
+#define mmAZALIA_OUTPUT_STREAM_ARBITER_CONTROL_BASE_IDX 2
+#define mmAZALIA_INPUT_PAYLOAD_CAPABILITY 0x03d6
+#define mmAZALIA_INPUT_PAYLOAD_CAPABILITY_BASE_IDX 2
+#define mmAZALIA_INPUT_CRC0_CONTROL0 0x03d9
+#define mmAZALIA_INPUT_CRC0_CONTROL0_BASE_IDX 2
+#define mmAZALIA_INPUT_CRC0_CONTROL1 0x03da
+#define mmAZALIA_INPUT_CRC0_CONTROL1_BASE_IDX 2
+#define mmAZALIA_INPUT_CRC0_CONTROL2 0x03db
+#define mmAZALIA_INPUT_CRC0_CONTROL2_BASE_IDX 2
+#define mmAZALIA_INPUT_CRC0_CONTROL3 0x03dc
+#define mmAZALIA_INPUT_CRC0_CONTROL3_BASE_IDX 2
+#define mmAZALIA_INPUT_CRC0_RESULT 0x03dd
+#define mmAZALIA_INPUT_CRC0_RESULT_BASE_IDX 2
+#define mmAZALIA_INPUT_CRC1_CONTROL0 0x03de
+#define mmAZALIA_INPUT_CRC1_CONTROL0_BASE_IDX 2
+#define mmAZALIA_INPUT_CRC1_CONTROL1 0x03df
+#define mmAZALIA_INPUT_CRC1_CONTROL1_BASE_IDX 2
+#define mmAZALIA_INPUT_CRC1_CONTROL2 0x03e0
+#define mmAZALIA_INPUT_CRC1_CONTROL2_BASE_IDX 2
+#define mmAZALIA_INPUT_CRC1_CONTROL3 0x03e1
+#define mmAZALIA_INPUT_CRC1_CONTROL3_BASE_IDX 2
+#define mmAZALIA_INPUT_CRC1_RESULT 0x03e2
+#define mmAZALIA_INPUT_CRC1_RESULT_BASE_IDX 2
+#define mmAZALIA_CRC0_CONTROL0 0x03e3
+#define mmAZALIA_CRC0_CONTROL0_BASE_IDX 2
+#define mmAZALIA_CRC0_CONTROL1 0x03e4
+#define mmAZALIA_CRC0_CONTROL1_BASE_IDX 2
+#define mmAZALIA_CRC0_CONTROL2 0x03e5
+#define mmAZALIA_CRC0_CONTROL2_BASE_IDX 2
+#define mmAZALIA_CRC0_CONTROL3 0x03e6
+#define mmAZALIA_CRC0_CONTROL3_BASE_IDX 2
+#define mmAZALIA_CRC0_RESULT 0x03e7
+#define mmAZALIA_CRC0_RESULT_BASE_IDX 2
+#define mmAZALIA_CRC1_CONTROL0 0x03e8
+#define mmAZALIA_CRC1_CONTROL0_BASE_IDX 2
+#define mmAZALIA_CRC1_CONTROL1 0x03e9
+#define mmAZALIA_CRC1_CONTROL1_BASE_IDX 2
+#define mmAZALIA_CRC1_CONTROL2 0x03ea
+#define mmAZALIA_CRC1_CONTROL2_BASE_IDX 2
+#define mmAZALIA_CRC1_CONTROL3 0x03eb
+#define mmAZALIA_CRC1_CONTROL3_BASE_IDX 2
+#define mmAZALIA_CRC1_RESULT 0x03ec
+#define mmAZALIA_CRC1_RESULT_BASE_IDX 2
+#define mmAZALIA_MEM_PWR_CTRL 0x03ee
+#define mmAZALIA_MEM_PWR_CTRL_BASE_IDX 2
+#define mmAZALIA_MEM_PWR_STATUS 0x03ef
+#define mmAZALIA_MEM_PWR_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0root_dispdec
+// base address: 0x0
+#define mmAZALIA_F0_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID 0x0406
+#define mmAZALIA_F0_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID_BASE_IDX 2
+#define mmAZALIA_F0_CODEC_ROOT_PARAMETER_REVISION_ID 0x0407
+#define mmAZALIA_F0_CODEC_ROOT_PARAMETER_REVISION_ID_BASE_IDX 2
+#define mmAZALIA_F0_CODEC_CHANNEL_COUNT_CONTROL 0x0408
+#define mmAZALIA_F0_CODEC_CHANNEL_COUNT_CONTROL_BASE_IDX 2
+#define mmAZALIA_F0_CODEC_RESYNC_FIFO_CONTROL 0x0409
+#define mmAZALIA_F0_CODEC_RESYNC_FIFO_CONTROL_BASE_IDX 2
+#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_GROUP_TYPE 0x040a
+#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_GROUP_TYPE_BASE_IDX 2
+#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES 0x040b
+#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES_BASE_IDX 2
+#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS 0x040c
+#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS_BASE_IDX 2
+#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES 0x040d
+#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES_BASE_IDX 2
+#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE 0x040e
+#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE_BASE_IDX 2
+#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_RESET 0x040f
+#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_RESET_BASE_IDX 2
+#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID 0x0410
+#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_BASE_IDX 2
+#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION 0x0411
+#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION_BASE_IDX 2
+#define mmCC_RCU_DC_AUDIO_PORT_CONNECTIVITY 0x0412
+#define mmCC_RCU_DC_AUDIO_PORT_CONNECTIVITY_BASE_IDX 2
+#define mmCC_RCU_DC_AUDIO_INPUT_PORT_CONNECTIVITY 0x0413
+#define mmCC_RCU_DC_AUDIO_INPUT_PORT_CONNECTIVITY_BASE_IDX 2
+#define mmAZALIA_F0_GTC_GROUP_OFFSET0 0x0415
+#define mmAZALIA_F0_GTC_GROUP_OFFSET0_BASE_IDX 2
+#define mmAZALIA_F0_GTC_GROUP_OFFSET1 0x0416
+#define mmAZALIA_F0_GTC_GROUP_OFFSET1_BASE_IDX 2
+#define mmAZALIA_F0_GTC_GROUP_OFFSET2 0x0417
+#define mmAZALIA_F0_GTC_GROUP_OFFSET2_BASE_IDX 2
+#define mmAZALIA_F0_GTC_GROUP_OFFSET3 0x0418
+#define mmAZALIA_F0_GTC_GROUP_OFFSET3_BASE_IDX 2
+#define mmAZALIA_F0_GTC_GROUP_OFFSET4 0x0419
+#define mmAZALIA_F0_GTC_GROUP_OFFSET4_BASE_IDX 2
+#define mmAZALIA_F0_GTC_GROUP_OFFSET5 0x041a
+#define mmAZALIA_F0_GTC_GROUP_OFFSET5_BASE_IDX 2
+#define mmAZALIA_F0_GTC_GROUP_OFFSET6 0x041b
+#define mmAZALIA_F0_GTC_GROUP_OFFSET6_BASE_IDX 2
+#define mmREG_DC_AUDIO_PORT_CONNECTIVITY 0x041c
+#define mmREG_DC_AUDIO_PORT_CONNECTIVITY_BASE_IDX 2
+#define mmREG_DC_AUDIO_INPUT_PORT_CONNECTIVITY 0x041d
+#define mmREG_DC_AUDIO_INPUT_PORT_CONNECTIVITY_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0stream8_dispdec
+// base address: 0x320
+#define mmAZF0STREAM8_AZALIA_STREAM_INDEX 0x0426
+#define mmAZF0STREAM8_AZALIA_STREAM_INDEX_BASE_IDX 2
+#define mmAZF0STREAM8_AZALIA_STREAM_DATA 0x0427
+#define mmAZF0STREAM8_AZALIA_STREAM_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0stream9_dispdec
+// base address: 0x328
+#define mmAZF0STREAM9_AZALIA_STREAM_INDEX 0x0428
+#define mmAZF0STREAM9_AZALIA_STREAM_INDEX_BASE_IDX 2
+#define mmAZF0STREAM9_AZALIA_STREAM_DATA 0x0429
+#define mmAZF0STREAM9_AZALIA_STREAM_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0stream10_dispdec
+// base address: 0x330
+#define mmAZF0STREAM10_AZALIA_STREAM_INDEX 0x042a
+#define mmAZF0STREAM10_AZALIA_STREAM_INDEX_BASE_IDX 2
+#define mmAZF0STREAM10_AZALIA_STREAM_DATA 0x042b
+#define mmAZF0STREAM10_AZALIA_STREAM_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0stream11_dispdec
+// base address: 0x338
+#define mmAZF0STREAM11_AZALIA_STREAM_INDEX 0x042c
+#define mmAZF0STREAM11_AZALIA_STREAM_INDEX_BASE_IDX 2
+#define mmAZF0STREAM11_AZALIA_STREAM_DATA 0x042d
+#define mmAZF0STREAM11_AZALIA_STREAM_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0stream12_dispdec
+// base address: 0x340
+#define mmAZF0STREAM12_AZALIA_STREAM_INDEX 0x042e
+#define mmAZF0STREAM12_AZALIA_STREAM_INDEX_BASE_IDX 2
+#define mmAZF0STREAM12_AZALIA_STREAM_DATA 0x042f
+#define mmAZF0STREAM12_AZALIA_STREAM_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0stream13_dispdec
+// base address: 0x348
+#define mmAZF0STREAM13_AZALIA_STREAM_INDEX 0x0430
+#define mmAZF0STREAM13_AZALIA_STREAM_INDEX_BASE_IDX 2
+#define mmAZF0STREAM13_AZALIA_STREAM_DATA 0x0431
+#define mmAZF0STREAM13_AZALIA_STREAM_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0stream14_dispdec
+// base address: 0x350
+#define mmAZF0STREAM14_AZALIA_STREAM_INDEX 0x0432
+#define mmAZF0STREAM14_AZALIA_STREAM_INDEX_BASE_IDX 2
+#define mmAZF0STREAM14_AZALIA_STREAM_DATA 0x0433
+#define mmAZF0STREAM14_AZALIA_STREAM_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0stream15_dispdec
+// base address: 0x358
+#define mmAZF0STREAM15_AZALIA_STREAM_INDEX 0x0434
+#define mmAZF0STREAM15_AZALIA_STREAM_INDEX_BASE_IDX 2
+#define mmAZF0STREAM15_AZALIA_STREAM_DATA 0x0435
+#define mmAZF0STREAM15_AZALIA_STREAM_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint0_dispdec
+// base address: 0x0
+#define mmAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX 0x043a
+#define mmAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_BASE_IDX 2
+#define mmAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA 0x043b
+#define mmAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint1_dispdec
+// base address: 0x10
+#define mmAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX 0x043e
+#define mmAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_BASE_IDX 2
+#define mmAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA 0x043f
+#define mmAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint2_dispdec
+// base address: 0x20
+#define mmAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX 0x0442
+#define mmAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_BASE_IDX 2
+#define mmAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA 0x0443
+#define mmAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint3_dispdec
+// base address: 0x30
+#define mmAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX 0x0446
+#define mmAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_BASE_IDX 2
+#define mmAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA 0x0447
+#define mmAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint4_dispdec
+// base address: 0x40
+#define mmAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX 0x044a
+#define mmAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_BASE_IDX 2
+#define mmAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA 0x044b
+#define mmAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint5_dispdec
+// base address: 0x50
+#define mmAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX 0x044e
+#define mmAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_BASE_IDX 2
+#define mmAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA 0x044f
+#define mmAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint6_dispdec
+// base address: 0x60
+#define mmAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX 0x0452
+#define mmAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_BASE_IDX 2
+#define mmAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA 0x0453
+#define mmAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azf0inputendpoint7_dispdec
+// base address: 0x70
+#define mmAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX 0x0456
+#define mmAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_BASE_IDX 2
+#define mmAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA 0x0457
+#define mmAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dchubbub_hubbub_sdpif_dispdec
+// base address: 0x0
+#define mmDCHUBBUB_SDPIF_CFG0 0x048f
+#define mmDCHUBBUB_SDPIF_CFG0_BASE_IDX 2
+#define mmVM_REQUEST_PHYSICAL 0x0490
+#define mmVM_REQUEST_PHYSICAL_BASE_IDX 2
+#define mmDCHUBBUB_FORCE_IO_STATUS_0 0x0491
+#define mmDCHUBBUB_FORCE_IO_STATUS_0_BASE_IDX 2
+#define mmDCHUBBUB_FORCE_IO_STATUS_1 0x0492
+#define mmDCHUBBUB_FORCE_IO_STATUS_1_BASE_IDX 2
+#define mmDCN_VM_FB_LOCATION_BASE 0x0493
+#define mmDCN_VM_FB_LOCATION_BASE_BASE_IDX 2
+#define mmDCN_VM_FB_LOCATION_TOP 0x0494
+#define mmDCN_VM_FB_LOCATION_TOP_BASE_IDX 2
+#define mmDCN_VM_FB_OFFSET 0x0495
+#define mmDCN_VM_FB_OFFSET_BASE_IDX 2
+#define mmDCN_VM_AGP_BOT 0x0496
+#define mmDCN_VM_AGP_BOT_BASE_IDX 2
+#define mmDCN_VM_AGP_TOP 0x0497
+#define mmDCN_VM_AGP_TOP_BASE_IDX 2
+#define mmDCN_VM_AGP_BASE 0x0498
+#define mmDCN_VM_AGP_BASE_BASE_IDX 2
+#define mmDCN_VM_LOCAL_HBM_ADDRESS_START 0x0499
+#define mmDCN_VM_LOCAL_HBM_ADDRESS_START_BASE_IDX 2
+#define mmDCN_VM_LOCAL_HBM_ADDRESS_END 0x049a
+#define mmDCN_VM_LOCAL_HBM_ADDRESS_END_BASE_IDX 2
+#define mmDCN_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL 0x049b
+#define mmDCN_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL_BASE_IDX 2
+#define mmDCHUBBUB_SDPIF_PIPE_SEC_LVL 0x04b8
+#define mmDCHUBBUB_SDPIF_PIPE_SEC_LVL_BASE_IDX 2
+#define mmDCHUBBUB_SDPIF_PIPE_DMDATA_SEC_LVL 0x04b9
+#define mmDCHUBBUB_SDPIF_PIPE_DMDATA_SEC_LVL_BASE_IDX 2
+#define mmDCHUBBUB_SDPIF_MEM_PWR_CTRL 0x04ba
+#define mmDCHUBBUB_SDPIF_MEM_PWR_CTRL_BASE_IDX 2
+#define mmDCHUBBUB_SDPIF_MEM_PWR_STATUS 0x04bb
+#define mmDCHUBBUB_SDPIF_MEM_PWR_STATUS_BASE_IDX 2
+#define mmDCHUBBUB_SDPIF_CFG1 0x04bf
+#define mmDCHUBBUB_SDPIF_CFG1_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dchubbub_hubbub_ret_path_dispdec
+// base address: 0x0
+#define mmDCHUBBUB_RET_PATH_DCC_CFG 0x04cf
+#define mmDCHUBBUB_RET_PATH_DCC_CFG_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG0_0 0x04d0
+#define mmDCHUBBUB_RET_PATH_DCC_CFG0_0_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG0_1 0x04d1
+#define mmDCHUBBUB_RET_PATH_DCC_CFG0_1_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG1_0 0x04d2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG1_0_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG1_1 0x04d3
+#define mmDCHUBBUB_RET_PATH_DCC_CFG1_1_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG2_0 0x04d4
+#define mmDCHUBBUB_RET_PATH_DCC_CFG2_0_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG2_1 0x04d5
+#define mmDCHUBBUB_RET_PATH_DCC_CFG2_1_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG3_0 0x04d6
+#define mmDCHUBBUB_RET_PATH_DCC_CFG3_0_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG3_1 0x04d7
+#define mmDCHUBBUB_RET_PATH_DCC_CFG3_1_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG4_0 0x04d8
+#define mmDCHUBBUB_RET_PATH_DCC_CFG4_0_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG4_1 0x04d9
+#define mmDCHUBBUB_RET_PATH_DCC_CFG4_1_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG5_0 0x04da
+#define mmDCHUBBUB_RET_PATH_DCC_CFG5_0_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG5_1 0x04db
+#define mmDCHUBBUB_RET_PATH_DCC_CFG5_1_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG6_0 0x04dc
+#define mmDCHUBBUB_RET_PATH_DCC_CFG6_0_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG6_1 0x04dd
+#define mmDCHUBBUB_RET_PATH_DCC_CFG6_1_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG7_0 0x04de
+#define mmDCHUBBUB_RET_PATH_DCC_CFG7_0_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG7_1 0x04df
+#define mmDCHUBBUB_RET_PATH_DCC_CFG7_1_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG8_0 0x04e0
+#define mmDCHUBBUB_RET_PATH_DCC_CFG8_0_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG8_1 0x04e1
+#define mmDCHUBBUB_RET_PATH_DCC_CFG8_1_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG9_0 0x04e2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG9_0_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG9_1 0x04e3
+#define mmDCHUBBUB_RET_PATH_DCC_CFG9_1_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG10_0 0x04e4
+#define mmDCHUBBUB_RET_PATH_DCC_CFG10_0_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG10_1 0x04e5
+#define mmDCHUBBUB_RET_PATH_DCC_CFG10_1_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG11_0 0x04e6
+#define mmDCHUBBUB_RET_PATH_DCC_CFG11_0_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_DCC_CFG11_1 0x04e7
+#define mmDCHUBBUB_RET_PATH_DCC_CFG11_1_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_MEM_PWR_CTRL 0x04ef
+#define mmDCHUBBUB_RET_PATH_MEM_PWR_CTRL_BASE_IDX 2
+#define mmDCHUBBUB_RET_PATH_MEM_PWR_STATUS 0x04f0
+#define mmDCHUBBUB_RET_PATH_MEM_PWR_STATUS_BASE_IDX 2
+#define mmDCHUBBUB_CRC_CTRL 0x04f1
+#define mmDCHUBBUB_CRC_CTRL_BASE_IDX 2
+#define mmDCHUBBUB_CRC0_VAL_R_G 0x04f2
+#define mmDCHUBBUB_CRC0_VAL_R_G_BASE_IDX 2
+#define mmDCHUBBUB_CRC0_VAL_B_A 0x04f3
+#define mmDCHUBBUB_CRC0_VAL_B_A_BASE_IDX 2
+#define mmDCHUBBUB_CRC1_VAL_R_G 0x04f4
+#define mmDCHUBBUB_CRC1_VAL_R_G_BASE_IDX 2
+#define mmDCHUBBUB_CRC1_VAL_B_A 0x04f5
+#define mmDCHUBBUB_CRC1_VAL_B_A_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dchubbub_hubbub_dispdec
+// base address: 0x0
+#define mmDCHUBBUB_ARB_DF_REQ_OUTSTAND 0x0505
+#define mmDCHUBBUB_ARB_DF_REQ_OUTSTAND_BASE_IDX 2
+#define mmDCHUBBUB_ARB_SAT_LEVEL 0x0506
+#define mmDCHUBBUB_ARB_SAT_LEVEL_BASE_IDX 2
+#define mmDCHUBBUB_ARB_QOS_FORCE 0x0507
+#define mmDCHUBBUB_ARB_QOS_FORCE_BASE_IDX 2
+#define mmDCHUBBUB_ARB_DRAM_STATE_CNTL 0x0508
+#define mmDCHUBBUB_ARB_DRAM_STATE_CNTL_BASE_IDX 2
+#define mmDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A 0x0509
+#define mmDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A_BASE_IDX 2
+#define mmDCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A 0x050a
+#define mmDCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A_BASE_IDX 2
+#define mmDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A 0x050b
+#define mmDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A_BASE_IDX 2
+#define mmDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A 0x050c
+#define mmDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A_BASE_IDX 2
+#define mmDCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A 0x050d
+#define mmDCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A_BASE_IDX 2
+#define mmDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B 0x050e
+#define mmDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B_BASE_IDX 2
+#define mmDCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B 0x050f
+#define mmDCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B_BASE_IDX 2
+#define mmDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B 0x0510
+#define mmDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B_BASE_IDX 2
+#define mmDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B 0x0511
+#define mmDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B_BASE_IDX 2
+#define mmDCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B 0x0512
+#define mmDCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B_BASE_IDX 2
+#define mmDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C 0x0513
+#define mmDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C_BASE_IDX 2
+#define mmDCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C 0x0514
+#define mmDCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C_BASE_IDX 2
+#define mmDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C 0x0515
+#define mmDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C_BASE_IDX 2
+#define mmDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C 0x0516
+#define mmDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C_BASE_IDX 2
+#define mmDCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C 0x0517
+#define mmDCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C_BASE_IDX 2
+#define mmDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D 0x0518
+#define mmDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D_BASE_IDX 2
+#define mmDCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D 0x0519
+#define mmDCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D_BASE_IDX 2
+#define mmDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D 0x051a
+#define mmDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D_BASE_IDX 2
+#define mmDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D 0x051b
+#define mmDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D_BASE_IDX 2
+#define mmDCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D 0x051c
+#define mmDCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D_BASE_IDX 2
+#define mmDCHUBBUB_ARB_WATERMARK_CHANGE_CNTL 0x051d
+#define mmDCHUBBUB_ARB_WATERMARK_CHANGE_CNTL_BASE_IDX 2
+#define mmDCHUBBUB_ARB_TIMEOUT_ENABLE 0x051e
+#define mmDCHUBBUB_ARB_TIMEOUT_ENABLE_BASE_IDX 2
+#define mmDCHUBBUB_GLOBAL_TIMER_CNTL 0x051f
+#define mmDCHUBBUB_GLOBAL_TIMER_CNTL_BASE_IDX 2
+#define mmSURFACE_CHECK0_ADDRESS_LSB 0x0520
+#define mmSURFACE_CHECK0_ADDRESS_LSB_BASE_IDX 2
+#define mmSURFACE_CHECK0_ADDRESS_MSB 0x0521
+#define mmSURFACE_CHECK0_ADDRESS_MSB_BASE_IDX 2
+#define mmSURFACE_CHECK1_ADDRESS_LSB 0x0522
+#define mmSURFACE_CHECK1_ADDRESS_LSB_BASE_IDX 2
+#define mmSURFACE_CHECK1_ADDRESS_MSB 0x0523
+#define mmSURFACE_CHECK1_ADDRESS_MSB_BASE_IDX 2
+#define mmSURFACE_CHECK2_ADDRESS_LSB 0x0524
+#define mmSURFACE_CHECK2_ADDRESS_LSB_BASE_IDX 2
+#define mmSURFACE_CHECK2_ADDRESS_MSB 0x0525
+#define mmSURFACE_CHECK2_ADDRESS_MSB_BASE_IDX 2
+#define mmSURFACE_CHECK3_ADDRESS_LSB 0x0526
+#define mmSURFACE_CHECK3_ADDRESS_LSB_BASE_IDX 2
+#define mmSURFACE_CHECK3_ADDRESS_MSB 0x0527
+#define mmSURFACE_CHECK3_ADDRESS_MSB_BASE_IDX 2
+#define mmVTG0_CONTROL 0x0528
+#define mmVTG0_CONTROL_BASE_IDX 2
+#define mmVTG1_CONTROL 0x0529
+#define mmVTG1_CONTROL_BASE_IDX 2
+#define mmVTG2_CONTROL 0x052a
+#define mmVTG2_CONTROL_BASE_IDX 2
+#define mmVTG3_CONTROL 0x052b
+#define mmVTG3_CONTROL_BASE_IDX 2
+#define mmVTG4_CONTROL 0x052c
+#define mmVTG4_CONTROL_BASE_IDX 2
+#define mmVTG5_CONTROL 0x052d
+#define mmVTG5_CONTROL_BASE_IDX 2
+#define mmDCHUBBUB_SOFT_RESET 0x052e
+#define mmDCHUBBUB_SOFT_RESET_BASE_IDX 2
+#define mmDCHUBBUB_CLOCK_CNTL 0x052f
+#define mmDCHUBBUB_CLOCK_CNTL_BASE_IDX 2
+#define mmDCFCLK_CNTL 0x0530
+#define mmDCFCLK_CNTL_BASE_IDX 2
+#define mmDCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL 0x0531
+#define mmDCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL_BASE_IDX 2
+#define mmDCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2 0x0532
+#define mmDCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2_BASE_IDX 2
+#define mmDCHUBBUB_VLINE_SNAPSHOT 0x0533
+#define mmDCHUBBUB_VLINE_SNAPSHOT_BASE_IDX 2
+#define mmDCHUBBUB_CTRL_STATUS 0x0534
+#define mmDCHUBBUB_CTRL_STATUS_BASE_IDX 2
+#define mmDCHUBBUB_TIMEOUT_DETECTION_CTRL1 0x053a
+#define mmDCHUBBUB_TIMEOUT_DETECTION_CTRL1_BASE_IDX 2
+#define mmDCHUBBUB_TIMEOUT_DETECTION_CTRL2 0x053b
+#define mmDCHUBBUB_TIMEOUT_DETECTION_CTRL2_BASE_IDX 2
+#define mmDCHUBBUB_TIMEOUT_INTERRUPT_STATUS 0x053c
+#define mmDCHUBBUB_TIMEOUT_INTERRUPT_STATUS_BASE_IDX 2
+#define mmDCHUBBUB_TEST_DEBUG_INDEX 0x053d
+#define mmDCHUBBUB_TEST_DEBUG_INDEX_BASE_IDX 2
+#define mmDCHUBBUB_TEST_DEBUG_DATA 0x053e
+#define mmDCHUBBUB_TEST_DEBUG_DATA_BASE_IDX 2
+#define mmFMON_CTRL 0x0548
+#define mmFMON_CTRL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dchubbub_dchubbub_dcperfmon_dc_perfmon_dispdec
+// base address: 0x1534
+#define mmDC_PERFMON6_PERFCOUNTER_CNTL 0x054d
+#define mmDC_PERFMON6_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON6_PERFCOUNTER_CNTL2 0x054e
+#define mmDC_PERFMON6_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON6_PERFCOUNTER_STATE 0x054f
+#define mmDC_PERFMON6_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON6_PERFMON_CNTL 0x0550
+#define mmDC_PERFMON6_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON6_PERFMON_CNTL2 0x0551
+#define mmDC_PERFMON6_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON6_PERFMON_CVALUE_INT_MISC 0x0552
+#define mmDC_PERFMON6_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON6_PERFMON_CVALUE_LOW 0x0553
+#define mmDC_PERFMON6_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON6_PERFMON_HI 0x0554
+#define mmDC_PERFMON6_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON6_PERFMON_LOW 0x0555
+#define mmDC_PERFMON6_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dchubbub_hubbub_vmrq_if_dispdec
+// base address: 0x0
+#define mmDCN_VM_CONTEXT0_CNTL 0x0559
+#define mmDCN_VM_CONTEXT0_CNTL_BASE_IDX 2
+#define mmDCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32 0x055a
+#define mmDCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32 0x055b
+#define mmDCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32 0x055c
+#define mmDCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32 0x055d
+#define mmDCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32 0x055e
+#define mmDCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32 0x055f
+#define mmDCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT1_CNTL 0x0560
+#define mmDCN_VM_CONTEXT1_CNTL_BASE_IDX 2
+#define mmDCN_VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32 0x0561
+#define mmDCN_VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 0x0562
+#define mmDCN_VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32 0x0563
+#define mmDCN_VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32 0x0564
+#define mmDCN_VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32 0x0565
+#define mmDCN_VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32 0x0566
+#define mmDCN_VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT2_CNTL 0x0567
+#define mmDCN_VM_CONTEXT2_CNTL_BASE_IDX 2
+#define mmDCN_VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32 0x0568
+#define mmDCN_VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32 0x0569
+#define mmDCN_VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32 0x056a
+#define mmDCN_VM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32 0x056b
+#define mmDCN_VM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32 0x056c
+#define mmDCN_VM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32 0x056d
+#define mmDCN_VM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT3_CNTL 0x056e
+#define mmDCN_VM_CONTEXT3_CNTL_BASE_IDX 2
+#define mmDCN_VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32 0x056f
+#define mmDCN_VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32 0x0570
+#define mmDCN_VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32 0x0571
+#define mmDCN_VM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32 0x0572
+#define mmDCN_VM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32 0x0573
+#define mmDCN_VM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32 0x0574
+#define mmDCN_VM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT4_CNTL 0x0575
+#define mmDCN_VM_CONTEXT4_CNTL_BASE_IDX 2
+#define mmDCN_VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32 0x0576
+#define mmDCN_VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32 0x0577
+#define mmDCN_VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32 0x0578
+#define mmDCN_VM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32 0x0579
+#define mmDCN_VM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32 0x057a
+#define mmDCN_VM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32 0x057b
+#define mmDCN_VM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT5_CNTL 0x057c
+#define mmDCN_VM_CONTEXT5_CNTL_BASE_IDX 2
+#define mmDCN_VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32 0x057d
+#define mmDCN_VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32 0x057e
+#define mmDCN_VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32 0x057f
+#define mmDCN_VM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32 0x0580
+#define mmDCN_VM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32 0x0581
+#define mmDCN_VM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32 0x0582
+#define mmDCN_VM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT6_CNTL 0x0583
+#define mmDCN_VM_CONTEXT6_CNTL_BASE_IDX 2
+#define mmDCN_VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32 0x0584
+#define mmDCN_VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32 0x0585
+#define mmDCN_VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32 0x0586
+#define mmDCN_VM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32 0x0587
+#define mmDCN_VM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32 0x0588
+#define mmDCN_VM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32 0x0589
+#define mmDCN_VM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT7_CNTL 0x058a
+#define mmDCN_VM_CONTEXT7_CNTL_BASE_IDX 2
+#define mmDCN_VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32 0x058b
+#define mmDCN_VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32 0x058c
+#define mmDCN_VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32 0x058d
+#define mmDCN_VM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32 0x058e
+#define mmDCN_VM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32 0x058f
+#define mmDCN_VM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32 0x0590
+#define mmDCN_VM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT8_CNTL 0x0591
+#define mmDCN_VM_CONTEXT8_CNTL_BASE_IDX 2
+#define mmDCN_VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32 0x0592
+#define mmDCN_VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32 0x0593
+#define mmDCN_VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32 0x0594
+#define mmDCN_VM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32 0x0595
+#define mmDCN_VM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32 0x0596
+#define mmDCN_VM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32 0x0597
+#define mmDCN_VM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT9_CNTL 0x0598
+#define mmDCN_VM_CONTEXT9_CNTL_BASE_IDX 2
+#define mmDCN_VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32 0x0599
+#define mmDCN_VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32 0x059a
+#define mmDCN_VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32 0x059b
+#define mmDCN_VM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32 0x059c
+#define mmDCN_VM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32 0x059d
+#define mmDCN_VM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32 0x059e
+#define mmDCN_VM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT10_CNTL 0x059f
+#define mmDCN_VM_CONTEXT10_CNTL_BASE_IDX 2
+#define mmDCN_VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32 0x05a0
+#define mmDCN_VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32 0x05a1
+#define mmDCN_VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32 0x05a2
+#define mmDCN_VM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32 0x05a3
+#define mmDCN_VM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32 0x05a4
+#define mmDCN_VM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32 0x05a5
+#define mmDCN_VM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT11_CNTL 0x05a6
+#define mmDCN_VM_CONTEXT11_CNTL_BASE_IDX 2
+#define mmDCN_VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32 0x05a7
+#define mmDCN_VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32 0x05a8
+#define mmDCN_VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32 0x05a9
+#define mmDCN_VM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32 0x05aa
+#define mmDCN_VM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32 0x05ab
+#define mmDCN_VM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32 0x05ac
+#define mmDCN_VM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT12_CNTL 0x05ad
+#define mmDCN_VM_CONTEXT12_CNTL_BASE_IDX 2
+#define mmDCN_VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32 0x05ae
+#define mmDCN_VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32 0x05af
+#define mmDCN_VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32 0x05b0
+#define mmDCN_VM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32 0x05b1
+#define mmDCN_VM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32 0x05b2
+#define mmDCN_VM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32 0x05b3
+#define mmDCN_VM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT13_CNTL 0x05b4
+#define mmDCN_VM_CONTEXT13_CNTL_BASE_IDX 2
+#define mmDCN_VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32 0x05b5
+#define mmDCN_VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32 0x05b6
+#define mmDCN_VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32 0x05b7
+#define mmDCN_VM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32 0x05b8
+#define mmDCN_VM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32 0x05b9
+#define mmDCN_VM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32 0x05ba
+#define mmDCN_VM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT14_CNTL 0x05bb
+#define mmDCN_VM_CONTEXT14_CNTL_BASE_IDX 2
+#define mmDCN_VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32 0x05bc
+#define mmDCN_VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32 0x05bd
+#define mmDCN_VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32 0x05be
+#define mmDCN_VM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32 0x05bf
+#define mmDCN_VM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32 0x05c0
+#define mmDCN_VM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32 0x05c1
+#define mmDCN_VM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT15_CNTL 0x05c2
+#define mmDCN_VM_CONTEXT15_CNTL_BASE_IDX 2
+#define mmDCN_VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32 0x05c3
+#define mmDCN_VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32 0x05c4
+#define mmDCN_VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32 0x05c5
+#define mmDCN_VM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32 0x05c6
+#define mmDCN_VM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32 0x05c7
+#define mmDCN_VM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32_BASE_IDX 2
+#define mmDCN_VM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32 0x05c8
+#define mmDCN_VM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32_BASE_IDX 2
+#define mmDCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB 0x05c9
+#define mmDCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_BASE_IDX 2
+#define mmDCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB 0x05ca
+#define mmDCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_BASE_IDX 2
+#define mmDCN_VM_PROTECTION_FAULT_DEFAULT_ADDR_MSB 0x05cb
+#define mmDCN_VM_PROTECTION_FAULT_DEFAULT_ADDR_MSB_BASE_IDX 2
+#define mmDCN_VM_PROTECTION_FAULT_DEFAULT_ADDR_LSB 0x05cc
+#define mmDCN_VM_PROTECTION_FAULT_DEFAULT_ADDR_LSB_BASE_IDX 2
+#define mmDCN_VM_FAULT_CNTL 0x05cd
+#define mmDCN_VM_FAULT_CNTL_BASE_IDX 2
+#define mmDCN_VM_FAULT_STATUS 0x05ce
+#define mmDCN_VM_FAULT_STATUS_BASE_IDX 2
+#define mmDCN_VM_FAULT_ADDR_MSB 0x05cf
+#define mmDCN_VM_FAULT_ADDR_MSB_BASE_IDX 2
+#define mmDCN_VM_FAULT_ADDR_LSB 0x05d0
+#define mmDCN_VM_FAULT_ADDR_LSB_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp0_dispdec_hubp_dispdec
+// base address: 0x0
+#define mmHUBP0_DCSURF_SURFACE_CONFIG 0x05e5
+#define mmHUBP0_DCSURF_SURFACE_CONFIG_BASE_IDX 2
+#define mmHUBP0_DCSURF_ADDR_CONFIG 0x05e6
+#define mmHUBP0_DCSURF_ADDR_CONFIG_BASE_IDX 2
+#define mmHUBP0_DCSURF_TILING_CONFIG 0x05e7
+#define mmHUBP0_DCSURF_TILING_CONFIG_BASE_IDX 2
+#define mmHUBP0_DCSURF_PRI_VIEWPORT_START 0x05e9
+#define mmHUBP0_DCSURF_PRI_VIEWPORT_START_BASE_IDX 2
+#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x05ea
+#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2
+#define mmHUBP0_DCSURF_PRI_VIEWPORT_START_C 0x05eb
+#define mmHUBP0_DCSURF_PRI_VIEWPORT_START_C_BASE_IDX 2
+#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_C 0x05ec
+#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_C_BASE_IDX 2
+#define mmHUBP0_DCSURF_SEC_VIEWPORT_START 0x05ed
+#define mmHUBP0_DCSURF_SEC_VIEWPORT_START_BASE_IDX 2
+#define mmHUBP0_DCSURF_SEC_VIEWPORT_DIMENSION 0x05ee
+#define mmHUBP0_DCSURF_SEC_VIEWPORT_DIMENSION_BASE_IDX 2
+#define mmHUBP0_DCSURF_SEC_VIEWPORT_START_C 0x05ef
+#define mmHUBP0_DCSURF_SEC_VIEWPORT_START_C_BASE_IDX 2
+#define mmHUBP0_DCSURF_SEC_VIEWPORT_DIMENSION_C 0x05f0
+#define mmHUBP0_DCSURF_SEC_VIEWPORT_DIMENSION_C_BASE_IDX 2
+#define mmHUBP0_DCHUBP_REQ_SIZE_CONFIG 0x05f1
+#define mmHUBP0_DCHUBP_REQ_SIZE_CONFIG_BASE_IDX 2
+#define mmHUBP0_DCHUBP_REQ_SIZE_CONFIG_C 0x05f2
+#define mmHUBP0_DCHUBP_REQ_SIZE_CONFIG_C_BASE_IDX 2
+#define mmHUBP0_DCHUBP_CNTL 0x05f3
+#define mmHUBP0_DCHUBP_CNTL_BASE_IDX 2
+#define mmHUBP0_HUBP_CLK_CNTL 0x05f4
+#define mmHUBP0_HUBP_CLK_CNTL_BASE_IDX 2
+#define mmHUBP0_DCHUBP_VMPG_CONFIG 0x05f5
+#define mmHUBP0_DCHUBP_VMPG_CONFIG_BASE_IDX 2
+#define mmHUBP0_HUBPREQ_DEBUG_DB 0x05f6
+#define mmHUBP0_HUBPREQ_DEBUG_DB_BASE_IDX 2
+#define mmHUBP0_HUBPREQ_DEBUG 0x05f7
+#define mmHUBP0_HUBPREQ_DEBUG_BASE_IDX 2
+#define mmHUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK 0x05fb
+#define mmHUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK_BASE_IDX 2
+#define mmHUBP0_HUBP_MEASURE_WIN_CTRL_DPPCLK 0x05fc
+#define mmHUBP0_HUBP_MEASURE_WIN_CTRL_DPPCLK_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp0_dispdec_hubpreq_dispdec
+// base address: 0x0
+#define mmHUBPREQ0_DCSURF_SURFACE_PITCH 0x0607
+#define mmHUBPREQ0_DCSURF_SURFACE_PITCH_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SURFACE_PITCH_C 0x0608
+#define mmHUBPREQ0_DCSURF_SURFACE_PITCH_C_BASE_IDX 2
+#define mmHUBPREQ0_VMID_SETTINGS_0 0x0609
+#define mmHUBPREQ0_VMID_SETTINGS_0_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS 0x060a
+#define mmHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH 0x060b
+#define mmHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_C 0x060c
+#define mmHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C 0x060d
+#define mmHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS 0x060e
+#define mmHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH 0x060f
+#define mmHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_C 0x0610
+#define mmHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C 0x0611
+#define mmHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS 0x0612
+#define mmHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH 0x0613
+#define mmHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C 0x0614
+#define mmHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C 0x0615
+#define mmHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS 0x0616
+#define mmHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH 0x0617
+#define mmHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C 0x0618
+#define mmHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C 0x0619
+#define mmHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SURFACE_CONTROL 0x061a
+#define mmHUBPREQ0_DCSURF_SURFACE_CONTROL_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_FLIP_CONTROL 0x061b
+#define mmHUBPREQ0_DCSURF_FLIP_CONTROL_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_FLIP_CONTROL2 0x061c
+#define mmHUBPREQ0_DCSURF_FLIP_CONTROL2_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_QUEUE_CONTROL 0x061d
+#define mmHUBPREQ0_DCSURF_QUEUE_CONTROL_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_FRAME_PACING_TIME 0x061e
+#define mmHUBPREQ0_DCSURF_FRAME_PACING_TIME_BASE_IDX 2
+#define mmHUBPREQ0_SURFACE_CURRENT_PACING_COUNTER 0x061f
+#define mmHUBPREQ0_SURFACE_CURRENT_PACING_COUNTER_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT 0x0620
+#define mmHUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SURFACE_INUSE 0x0621
+#define mmHUBPREQ0_DCSURF_SURFACE_INUSE_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SURFACE_INUSE_HIGH 0x0622
+#define mmHUBPREQ0_DCSURF_SURFACE_INUSE_HIGH_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SURFACE_INUSE_C 0x0623
+#define mmHUBPREQ0_DCSURF_SURFACE_INUSE_C_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SURFACE_INUSE_HIGH_C 0x0624
+#define mmHUBPREQ0_DCSURF_SURFACE_INUSE_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE 0x0625
+#define mmHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH 0x0626
+#define mmHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_C 0x0627
+#define mmHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_C_BASE_IDX 2
+#define mmHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C 0x0628
+#define mmHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ0_DCN_EXPANSION_MODE 0x062c
+#define mmHUBPREQ0_DCN_EXPANSION_MODE_BASE_IDX 2
+#define mmHUBPREQ0_DCN_TTU_QOS_WM 0x062d
+#define mmHUBPREQ0_DCN_TTU_QOS_WM_BASE_IDX 2
+#define mmHUBPREQ0_DCN_GLOBAL_TTU_CNTL 0x062e
+#define mmHUBPREQ0_DCN_GLOBAL_TTU_CNTL_BASE_IDX 2
+#define mmHUBPREQ0_DCN_SURF0_TTU_CNTL0 0x062f
+#define mmHUBPREQ0_DCN_SURF0_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ0_DCN_SURF0_TTU_CNTL1 0x0630
+#define mmHUBPREQ0_DCN_SURF0_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ0_DCN_SURF1_TTU_CNTL0 0x0631
+#define mmHUBPREQ0_DCN_SURF1_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ0_DCN_SURF1_TTU_CNTL1 0x0632
+#define mmHUBPREQ0_DCN_SURF1_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ0_DCN_CUR0_TTU_CNTL0 0x0633
+#define mmHUBPREQ0_DCN_CUR0_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ0_DCN_CUR0_TTU_CNTL1 0x0634
+#define mmHUBPREQ0_DCN_CUR0_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ0_DCN_CUR1_TTU_CNTL0 0x0635
+#define mmHUBPREQ0_DCN_CUR1_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ0_DCN_CUR1_TTU_CNTL1 0x0636
+#define mmHUBPREQ0_DCN_CUR1_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ0_DCN_VM_SYSTEM_APERTURE_LOW_ADDR 0x0637
+#define mmHUBPREQ0_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_BASE_IDX 2
+#define mmHUBPREQ0_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR 0x0638
+#define mmHUBPREQ0_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_BASE_IDX 2
+#define mmHUBPREQ0_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB 0x0639
+#define mmHUBPREQ0_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ0_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB 0x063a
+#define mmHUBPREQ0_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB 0x063b
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB 0x063c
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB 0x063d
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB 0x063e
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB 0x063f
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB 0x0640
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB 0x0641
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB 0x0642
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PROTECTION_FAULT_ADDR_LSB 0x0643
+#define mmHUBPREQ0_DC_VM_CONTEXT0_PROTECTION_FAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ0_DC_VM_CONTEXT0_CNTL 0x0644
+#define mmHUBPREQ0_DC_VM_CONTEXT0_CNTL_BASE_IDX 2
+#define mmHUBPREQ0_DCN_VM_MX_L1_TLB_CNTL 0x0645
+#define mmHUBPREQ0_DCN_VM_MX_L1_TLB_CNTL_BASE_IDX 2
+#define mmHUBPREQ0_BLANK_OFFSET_0 0x0646
+#define mmHUBPREQ0_BLANK_OFFSET_0_BASE_IDX 2
+#define mmHUBPREQ0_BLANK_OFFSET_1 0x0647
+#define mmHUBPREQ0_BLANK_OFFSET_1_BASE_IDX 2
+#define mmHUBPREQ0_DST_DIMENSIONS 0x0648
+#define mmHUBPREQ0_DST_DIMENSIONS_BASE_IDX 2
+#define mmHUBPREQ0_DST_AFTER_SCALER 0x0649
+#define mmHUBPREQ0_DST_AFTER_SCALER_BASE_IDX 2
+#define mmHUBPREQ0_PREFETCH_SETTINGS 0x064a
+#define mmHUBPREQ0_PREFETCH_SETTINGS_BASE_IDX 2
+#define mmHUBPREQ0_PREFETCH_SETTINGS_C 0x064b
+#define mmHUBPREQ0_PREFETCH_SETTINGS_C_BASE_IDX 2
+#define mmHUBPREQ0_VBLANK_PARAMETERS_0 0x064c
+#define mmHUBPREQ0_VBLANK_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ0_VBLANK_PARAMETERS_1 0x064d
+#define mmHUBPREQ0_VBLANK_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ0_VBLANK_PARAMETERS_2 0x064e
+#define mmHUBPREQ0_VBLANK_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ0_VBLANK_PARAMETERS_3 0x064f
+#define mmHUBPREQ0_VBLANK_PARAMETERS_3_BASE_IDX 2
+#define mmHUBPREQ0_VBLANK_PARAMETERS_4 0x0650
+#define mmHUBPREQ0_VBLANK_PARAMETERS_4_BASE_IDX 2
+#define mmHUBPREQ0_FLIP_PARAMETERS_0 0x0651
+#define mmHUBPREQ0_FLIP_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ0_FLIP_PARAMETERS_1 0x0652
+#define mmHUBPREQ0_FLIP_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ0_FLIP_PARAMETERS_2 0x0653
+#define mmHUBPREQ0_FLIP_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ0_NOM_PARAMETERS_0 0x0654
+#define mmHUBPREQ0_NOM_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ0_NOM_PARAMETERS_1 0x0655
+#define mmHUBPREQ0_NOM_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ0_NOM_PARAMETERS_2 0x0656
+#define mmHUBPREQ0_NOM_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ0_NOM_PARAMETERS_3 0x0657
+#define mmHUBPREQ0_NOM_PARAMETERS_3_BASE_IDX 2
+#define mmHUBPREQ0_NOM_PARAMETERS_4 0x0658
+#define mmHUBPREQ0_NOM_PARAMETERS_4_BASE_IDX 2
+#define mmHUBPREQ0_NOM_PARAMETERS_5 0x0659
+#define mmHUBPREQ0_NOM_PARAMETERS_5_BASE_IDX 2
+#define mmHUBPREQ0_NOM_PARAMETERS_6 0x065a
+#define mmHUBPREQ0_NOM_PARAMETERS_6_BASE_IDX 2
+#define mmHUBPREQ0_NOM_PARAMETERS_7 0x065b
+#define mmHUBPREQ0_NOM_PARAMETERS_7_BASE_IDX 2
+#define mmHUBPREQ0_PER_LINE_DELIVERY_PRE 0x065c
+#define mmHUBPREQ0_PER_LINE_DELIVERY_PRE_BASE_IDX 2
+#define mmHUBPREQ0_PER_LINE_DELIVERY 0x065d
+#define mmHUBPREQ0_PER_LINE_DELIVERY_BASE_IDX 2
+#define mmHUBPREQ0_CURSOR_SETTINGS 0x065e
+#define mmHUBPREQ0_CURSOR_SETTINGS_BASE_IDX 2
+#define mmHUBPREQ0_REF_FREQ_TO_PIX_FREQ 0x065f
+#define mmHUBPREQ0_REF_FREQ_TO_PIX_FREQ_BASE_IDX 2
+#define mmHUBPREQ0_DST_Y_DELTA_DRQ_LIMIT 0x0660
+#define mmHUBPREQ0_DST_Y_DELTA_DRQ_LIMIT_BASE_IDX 2
+#define mmHUBPREQ0_HUBPREQ_MEM_PWR_CTRL 0x0661
+#define mmHUBPREQ0_HUBPREQ_MEM_PWR_CTRL_BASE_IDX 2
+#define mmHUBPREQ0_HUBPREQ_MEM_PWR_STATUS 0x0662
+#define mmHUBPREQ0_HUBPREQ_MEM_PWR_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp0_dispdec_hubpret_dispdec
+// base address: 0x0
+#define mmHUBPRET0_HUBPRET_CONTROL 0x066c
+#define mmHUBPRET0_HUBPRET_CONTROL_BASE_IDX 2
+#define mmHUBPRET0_HUBPRET_MEM_PWR_CTRL 0x066d
+#define mmHUBPRET0_HUBPRET_MEM_PWR_CTRL_BASE_IDX 2
+#define mmHUBPRET0_HUBPRET_MEM_PWR_STATUS 0x066e
+#define mmHUBPRET0_HUBPRET_MEM_PWR_STATUS_BASE_IDX 2
+#define mmHUBPRET0_HUBPRET_READ_LINE_CTRL0 0x066f
+#define mmHUBPRET0_HUBPRET_READ_LINE_CTRL0_BASE_IDX 2
+#define mmHUBPRET0_HUBPRET_READ_LINE_CTRL1 0x0670
+#define mmHUBPRET0_HUBPRET_READ_LINE_CTRL1_BASE_IDX 2
+#define mmHUBPRET0_HUBPRET_READ_LINE0 0x0671
+#define mmHUBPRET0_HUBPRET_READ_LINE0_BASE_IDX 2
+#define mmHUBPRET0_HUBPRET_READ_LINE1 0x0672
+#define mmHUBPRET0_HUBPRET_READ_LINE1_BASE_IDX 2
+#define mmHUBPRET0_HUBPRET_INTERRUPT 0x0673
+#define mmHUBPRET0_HUBPRET_INTERRUPT_BASE_IDX 2
+#define mmHUBPRET0_HUBPRET_READ_LINE_VALUE 0x0674
+#define mmHUBPRET0_HUBPRET_READ_LINE_VALUE_BASE_IDX 2
+#define mmHUBPRET0_HUBPRET_READ_LINE_STATUS 0x0675
+#define mmHUBPRET0_HUBPRET_READ_LINE_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp0_dispdec_cursor0_dispdec
+// base address: 0x0
+#define mmCURSOR0_0_CURSOR_CONTROL 0x0678
+#define mmCURSOR0_0_CURSOR_CONTROL_BASE_IDX 2
+#define mmCURSOR0_0_CURSOR_SURFACE_ADDRESS 0x0679
+#define mmCURSOR0_0_CURSOR_SURFACE_ADDRESS_BASE_IDX 2
+#define mmCURSOR0_0_CURSOR_SURFACE_ADDRESS_HIGH 0x067a
+#define mmCURSOR0_0_CURSOR_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmCURSOR0_0_CURSOR_SIZE 0x067b
+#define mmCURSOR0_0_CURSOR_SIZE_BASE_IDX 2
+#define mmCURSOR0_0_CURSOR_POSITION 0x067c
+#define mmCURSOR0_0_CURSOR_POSITION_BASE_IDX 2
+#define mmCURSOR0_0_CURSOR_HOT_SPOT 0x067d
+#define mmCURSOR0_0_CURSOR_HOT_SPOT_BASE_IDX 2
+#define mmCURSOR0_0_CURSOR_STEREO_CONTROL 0x067e
+#define mmCURSOR0_0_CURSOR_STEREO_CONTROL_BASE_IDX 2
+#define mmCURSOR0_0_CURSOR_DST_OFFSET 0x067f
+#define mmCURSOR0_0_CURSOR_DST_OFFSET_BASE_IDX 2
+#define mmCURSOR0_0_CURSOR_MEM_PWR_CTRL 0x0680
+#define mmCURSOR0_0_CURSOR_MEM_PWR_CTRL_BASE_IDX 2
+#define mmCURSOR0_0_CURSOR_MEM_PWR_STATUS 0x0681
+#define mmCURSOR0_0_CURSOR_MEM_PWR_STATUS_BASE_IDX 2
+#define mmCURSOR0_0_DMDATA_ADDRESS_HIGH 0x0682
+#define mmCURSOR0_0_DMDATA_ADDRESS_HIGH_BASE_IDX 2
+#define mmCURSOR0_0_DMDATA_ADDRESS_LOW 0x0683
+#define mmCURSOR0_0_DMDATA_ADDRESS_LOW_BASE_IDX 2
+#define mmCURSOR0_0_DMDATA_CNTL 0x0684
+#define mmCURSOR0_0_DMDATA_CNTL_BASE_IDX 2
+#define mmCURSOR0_0_DMDATA_QOS_CNTL 0x0685
+#define mmCURSOR0_0_DMDATA_QOS_CNTL_BASE_IDX 2
+#define mmCURSOR0_0_DMDATA_STATUS 0x0686
+#define mmCURSOR0_0_DMDATA_STATUS_BASE_IDX 2
+#define mmCURSOR0_0_DMDATA_SW_CNTL 0x0687
+#define mmCURSOR0_0_DMDATA_SW_CNTL_BASE_IDX 2
+#define mmCURSOR0_0_DMDATA_SW_DATA 0x0688
+#define mmCURSOR0_0_DMDATA_SW_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp0_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x1a74
+#define mmDC_PERFMON7_PERFCOUNTER_CNTL 0x069d
+#define mmDC_PERFMON7_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON7_PERFCOUNTER_CNTL2 0x069e
+#define mmDC_PERFMON7_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON7_PERFCOUNTER_STATE 0x069f
+#define mmDC_PERFMON7_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON7_PERFMON_CNTL 0x06a0
+#define mmDC_PERFMON7_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON7_PERFMON_CNTL2 0x06a1
+#define mmDC_PERFMON7_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON7_PERFMON_CVALUE_INT_MISC 0x06a2
+#define mmDC_PERFMON7_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON7_PERFMON_CVALUE_LOW 0x06a3
+#define mmDC_PERFMON7_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON7_PERFMON_HI 0x06a4
+#define mmDC_PERFMON7_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON7_PERFMON_LOW 0x06a5
+#define mmDC_PERFMON7_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp0_dispdec_hubpxfc_dispdec
+// base address: 0x0
+#define mmHUBPXFC0_HUBP_XFC_CNTL 0x06a9
+#define mmHUBPXFC0_HUBP_XFC_CNTL_BASE_IDX 2
+#define mmHUBPXFC0_HUBP_XFC_XBUF_RD_BASE0_ADDR_LSB 0x06aa
+#define mmHUBPXFC0_HUBP_XFC_XBUF_RD_BASE0_ADDR_LSB_BASE_IDX 2
+#define mmHUBPXFC0_HUBP_XFC_XBUF_RD_BASE0_ADDR_MSB 0x06ab
+#define mmHUBPXFC0_HUBP_XFC_XBUF_RD_BASE0_ADDR_MSB_BASE_IDX 2
+#define mmHUBPXFC0_HUBP_XFC_XBUF_RD_BASE1_ADDR_LSB 0x06ac
+#define mmHUBPXFC0_HUBP_XFC_XBUF_RD_BASE1_ADDR_LSB_BASE_IDX 2
+#define mmHUBPXFC0_HUBP_XFC_XBUF_RD_BASE1_ADDR_MSB 0x06ad
+#define mmHUBPXFC0_HUBP_XFC_XBUF_RD_BASE1_ADDR_MSB_BASE_IDX 2
+#define mmHUBPXFC0_HUBP_XFC_XBUF_RD_PITCH 0x06ae
+#define mmHUBPXFC0_HUBP_XFC_XBUF_RD_PITCH_BASE_IDX 2
+#define mmHUBPXFC0_HUBP_XFC_DELAY_CONFIG0 0x06af
+#define mmHUBPXFC0_HUBP_XFC_DELAY_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC0_HUBP_XFC_DELAY_CONFIG1 0x06b0
+#define mmHUBPXFC0_HUBP_XFC_DELAY_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC0_HUBP_XFC_DELAY_CONFIG2 0x06b1
+#define mmHUBPXFC0_HUBP_XFC_DELAY_CONFIG2_BASE_IDX 2
+#define mmHUBPXFC0_HUBP_XFC_UNDERFLOW_STATUS 0x06b2
+#define mmHUBPXFC0_HUBP_XFC_UNDERFLOW_STATUS_BASE_IDX 2
+#define mmHUBPXFC0_HUBP_XFC_SLV_VTG_CONFIG0 0x06b3
+#define mmHUBPXFC0_HUBP_XFC_SLV_VTG_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC0_HUBP_XFC_SLV_VTG_CONFIG1 0x06b4
+#define mmHUBPXFC0_HUBP_XFC_SLV_VTG_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC0_HUBP_XFC_SLV_SCALER_CONFIG0 0x06b5
+#define mmHUBPXFC0_HUBP_XFC_SLV_SCALER_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC0_HUBP_XFC_SLV_SCALER_CONFIG1 0x06b6
+#define mmHUBPXFC0_HUBP_XFC_SLV_SCALER_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC0_HUBP_XFC_MPC_CONFIG 0x06b7
+#define mmHUBPXFC0_HUBP_XFC_MPC_CONFIG_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp1_dispdec_hubp_dispdec
+// base address: 0x370
+#define mmHUBP1_DCSURF_SURFACE_CONFIG 0x06c1
+#define mmHUBP1_DCSURF_SURFACE_CONFIG_BASE_IDX 2
+#define mmHUBP1_DCSURF_ADDR_CONFIG 0x06c2
+#define mmHUBP1_DCSURF_ADDR_CONFIG_BASE_IDX 2
+#define mmHUBP1_DCSURF_TILING_CONFIG 0x06c3
+#define mmHUBP1_DCSURF_TILING_CONFIG_BASE_IDX 2
+#define mmHUBP1_DCSURF_PRI_VIEWPORT_START 0x06c5
+#define mmHUBP1_DCSURF_PRI_VIEWPORT_START_BASE_IDX 2
+#define mmHUBP1_DCSURF_PRI_VIEWPORT_DIMENSION 0x06c6
+#define mmHUBP1_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2
+#define mmHUBP1_DCSURF_PRI_VIEWPORT_START_C 0x06c7
+#define mmHUBP1_DCSURF_PRI_VIEWPORT_START_C_BASE_IDX 2
+#define mmHUBP1_DCSURF_PRI_VIEWPORT_DIMENSION_C 0x06c8
+#define mmHUBP1_DCSURF_PRI_VIEWPORT_DIMENSION_C_BASE_IDX 2
+#define mmHUBP1_DCSURF_SEC_VIEWPORT_START 0x06c9
+#define mmHUBP1_DCSURF_SEC_VIEWPORT_START_BASE_IDX 2
+#define mmHUBP1_DCSURF_SEC_VIEWPORT_DIMENSION 0x06ca
+#define mmHUBP1_DCSURF_SEC_VIEWPORT_DIMENSION_BASE_IDX 2
+#define mmHUBP1_DCSURF_SEC_VIEWPORT_START_C 0x06cb
+#define mmHUBP1_DCSURF_SEC_VIEWPORT_START_C_BASE_IDX 2
+#define mmHUBP1_DCSURF_SEC_VIEWPORT_DIMENSION_C 0x06cc
+#define mmHUBP1_DCSURF_SEC_VIEWPORT_DIMENSION_C_BASE_IDX 2
+#define mmHUBP1_DCHUBP_REQ_SIZE_CONFIG 0x06cd
+#define mmHUBP1_DCHUBP_REQ_SIZE_CONFIG_BASE_IDX 2
+#define mmHUBP1_DCHUBP_REQ_SIZE_CONFIG_C 0x06ce
+#define mmHUBP1_DCHUBP_REQ_SIZE_CONFIG_C_BASE_IDX 2
+#define mmHUBP1_DCHUBP_CNTL 0x06cf
+#define mmHUBP1_DCHUBP_CNTL_BASE_IDX 2
+#define mmHUBP1_HUBP_CLK_CNTL 0x06d0
+#define mmHUBP1_HUBP_CLK_CNTL_BASE_IDX 2
+#define mmHUBP1_DCHUBP_VMPG_CONFIG 0x06d1
+#define mmHUBP1_DCHUBP_VMPG_CONFIG_BASE_IDX 2
+#define mmHUBP1_HUBPREQ_DEBUG_DB 0x06d2
+#define mmHUBP1_HUBPREQ_DEBUG_DB_BASE_IDX 2
+#define mmHUBP1_HUBPREQ_DEBUG 0x06d3
+#define mmHUBP1_HUBPREQ_DEBUG_BASE_IDX 2
+#define mmHUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK 0x06d7
+#define mmHUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK_BASE_IDX 2
+#define mmHUBP1_HUBP_MEASURE_WIN_CTRL_DPPCLK 0x06d8
+#define mmHUBP1_HUBP_MEASURE_WIN_CTRL_DPPCLK_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp1_dispdec_hubpreq_dispdec
+// base address: 0x370
+#define mmHUBPREQ1_DCSURF_SURFACE_PITCH 0x06e3
+#define mmHUBPREQ1_DCSURF_SURFACE_PITCH_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SURFACE_PITCH_C 0x06e4
+#define mmHUBPREQ1_DCSURF_SURFACE_PITCH_C_BASE_IDX 2
+#define mmHUBPREQ1_VMID_SETTINGS_0 0x06e5
+#define mmHUBPREQ1_VMID_SETTINGS_0_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS 0x06e6
+#define mmHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH 0x06e7
+#define mmHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_C 0x06e8
+#define mmHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C 0x06e9
+#define mmHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS 0x06ea
+#define mmHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH 0x06eb
+#define mmHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_C 0x06ec
+#define mmHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C 0x06ed
+#define mmHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS 0x06ee
+#define mmHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH 0x06ef
+#define mmHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C 0x06f0
+#define mmHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C 0x06f1
+#define mmHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS 0x06f2
+#define mmHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH 0x06f3
+#define mmHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C 0x06f4
+#define mmHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C 0x06f5
+#define mmHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SURFACE_CONTROL 0x06f6
+#define mmHUBPREQ1_DCSURF_SURFACE_CONTROL_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_FLIP_CONTROL 0x06f7
+#define mmHUBPREQ1_DCSURF_FLIP_CONTROL_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_FLIP_CONTROL2 0x06f8
+#define mmHUBPREQ1_DCSURF_FLIP_CONTROL2_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_QUEUE_CONTROL 0x06f9
+#define mmHUBPREQ1_DCSURF_QUEUE_CONTROL_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_FRAME_PACING_TIME 0x06fa
+#define mmHUBPREQ1_DCSURF_FRAME_PACING_TIME_BASE_IDX 2
+#define mmHUBPREQ1_SURFACE_CURRENT_PACING_COUNTER 0x06fb
+#define mmHUBPREQ1_SURFACE_CURRENT_PACING_COUNTER_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT 0x06fc
+#define mmHUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SURFACE_INUSE 0x06fd
+#define mmHUBPREQ1_DCSURF_SURFACE_INUSE_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SURFACE_INUSE_HIGH 0x06fe
+#define mmHUBPREQ1_DCSURF_SURFACE_INUSE_HIGH_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SURFACE_INUSE_C 0x06ff
+#define mmHUBPREQ1_DCSURF_SURFACE_INUSE_C_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SURFACE_INUSE_HIGH_C 0x0700
+#define mmHUBPREQ1_DCSURF_SURFACE_INUSE_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE 0x0701
+#define mmHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH 0x0702
+#define mmHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_C 0x0703
+#define mmHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_C_BASE_IDX 2
+#define mmHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C 0x0704
+#define mmHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ1_DCN_EXPANSION_MODE 0x0708
+#define mmHUBPREQ1_DCN_EXPANSION_MODE_BASE_IDX 2
+#define mmHUBPREQ1_DCN_TTU_QOS_WM 0x0709
+#define mmHUBPREQ1_DCN_TTU_QOS_WM_BASE_IDX 2
+#define mmHUBPREQ1_DCN_GLOBAL_TTU_CNTL 0x070a
+#define mmHUBPREQ1_DCN_GLOBAL_TTU_CNTL_BASE_IDX 2
+#define mmHUBPREQ1_DCN_SURF0_TTU_CNTL0 0x070b
+#define mmHUBPREQ1_DCN_SURF0_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ1_DCN_SURF0_TTU_CNTL1 0x070c
+#define mmHUBPREQ1_DCN_SURF0_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ1_DCN_SURF1_TTU_CNTL0 0x070d
+#define mmHUBPREQ1_DCN_SURF1_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ1_DCN_SURF1_TTU_CNTL1 0x070e
+#define mmHUBPREQ1_DCN_SURF1_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ1_DCN_CUR0_TTU_CNTL0 0x070f
+#define mmHUBPREQ1_DCN_CUR0_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ1_DCN_CUR0_TTU_CNTL1 0x0710
+#define mmHUBPREQ1_DCN_CUR0_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ1_DCN_CUR1_TTU_CNTL0 0x0711
+#define mmHUBPREQ1_DCN_CUR1_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ1_DCN_CUR1_TTU_CNTL1 0x0712
+#define mmHUBPREQ1_DCN_CUR1_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ1_DCN_VM_SYSTEM_APERTURE_LOW_ADDR 0x0713
+#define mmHUBPREQ1_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_BASE_IDX 2
+#define mmHUBPREQ1_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR 0x0714
+#define mmHUBPREQ1_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_BASE_IDX 2
+#define mmHUBPREQ1_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB 0x0715
+#define mmHUBPREQ1_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ1_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB 0x0716
+#define mmHUBPREQ1_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB 0x0717
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB 0x0718
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB 0x0719
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB 0x071a
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB 0x071b
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB 0x071c
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB 0x071d
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB 0x071e
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PROTECTION_FAULT_ADDR_LSB 0x071f
+#define mmHUBPREQ1_DC_VM_CONTEXT0_PROTECTION_FAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ1_DC_VM_CONTEXT0_CNTL 0x0720
+#define mmHUBPREQ1_DC_VM_CONTEXT0_CNTL_BASE_IDX 2
+#define mmHUBPREQ1_DCN_VM_MX_L1_TLB_CNTL 0x0721
+#define mmHUBPREQ1_DCN_VM_MX_L1_TLB_CNTL_BASE_IDX 2
+#define mmHUBPREQ1_BLANK_OFFSET_0 0x0722
+#define mmHUBPREQ1_BLANK_OFFSET_0_BASE_IDX 2
+#define mmHUBPREQ1_BLANK_OFFSET_1 0x0723
+#define mmHUBPREQ1_BLANK_OFFSET_1_BASE_IDX 2
+#define mmHUBPREQ1_DST_DIMENSIONS 0x0724
+#define mmHUBPREQ1_DST_DIMENSIONS_BASE_IDX 2
+#define mmHUBPREQ1_DST_AFTER_SCALER 0x0725
+#define mmHUBPREQ1_DST_AFTER_SCALER_BASE_IDX 2
+#define mmHUBPREQ1_PREFETCH_SETTINGS 0x0726
+#define mmHUBPREQ1_PREFETCH_SETTINGS_BASE_IDX 2
+#define mmHUBPREQ1_PREFETCH_SETTINGS_C 0x0727
+#define mmHUBPREQ1_PREFETCH_SETTINGS_C_BASE_IDX 2
+#define mmHUBPREQ1_VBLANK_PARAMETERS_0 0x0728
+#define mmHUBPREQ1_VBLANK_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ1_VBLANK_PARAMETERS_1 0x0729
+#define mmHUBPREQ1_VBLANK_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ1_VBLANK_PARAMETERS_2 0x072a
+#define mmHUBPREQ1_VBLANK_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ1_VBLANK_PARAMETERS_3 0x072b
+#define mmHUBPREQ1_VBLANK_PARAMETERS_3_BASE_IDX 2
+#define mmHUBPREQ1_VBLANK_PARAMETERS_4 0x072c
+#define mmHUBPREQ1_VBLANK_PARAMETERS_4_BASE_IDX 2
+#define mmHUBPREQ1_FLIP_PARAMETERS_0 0x072d
+#define mmHUBPREQ1_FLIP_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ1_FLIP_PARAMETERS_1 0x072e
+#define mmHUBPREQ1_FLIP_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ1_FLIP_PARAMETERS_2 0x072f
+#define mmHUBPREQ1_FLIP_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ1_NOM_PARAMETERS_0 0x0730
+#define mmHUBPREQ1_NOM_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ1_NOM_PARAMETERS_1 0x0731
+#define mmHUBPREQ1_NOM_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ1_NOM_PARAMETERS_2 0x0732
+#define mmHUBPREQ1_NOM_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ1_NOM_PARAMETERS_3 0x0733
+#define mmHUBPREQ1_NOM_PARAMETERS_3_BASE_IDX 2
+#define mmHUBPREQ1_NOM_PARAMETERS_4 0x0734
+#define mmHUBPREQ1_NOM_PARAMETERS_4_BASE_IDX 2
+#define mmHUBPREQ1_NOM_PARAMETERS_5 0x0735
+#define mmHUBPREQ1_NOM_PARAMETERS_5_BASE_IDX 2
+#define mmHUBPREQ1_NOM_PARAMETERS_6 0x0736
+#define mmHUBPREQ1_NOM_PARAMETERS_6_BASE_IDX 2
+#define mmHUBPREQ1_NOM_PARAMETERS_7 0x0737
+#define mmHUBPREQ1_NOM_PARAMETERS_7_BASE_IDX 2
+#define mmHUBPREQ1_PER_LINE_DELIVERY_PRE 0x0738
+#define mmHUBPREQ1_PER_LINE_DELIVERY_PRE_BASE_IDX 2
+#define mmHUBPREQ1_PER_LINE_DELIVERY 0x0739
+#define mmHUBPREQ1_PER_LINE_DELIVERY_BASE_IDX 2
+#define mmHUBPREQ1_CURSOR_SETTINGS 0x073a
+#define mmHUBPREQ1_CURSOR_SETTINGS_BASE_IDX 2
+#define mmHUBPREQ1_REF_FREQ_TO_PIX_FREQ 0x073b
+#define mmHUBPREQ1_REF_FREQ_TO_PIX_FREQ_BASE_IDX 2
+#define mmHUBPREQ1_DST_Y_DELTA_DRQ_LIMIT 0x073c
+#define mmHUBPREQ1_DST_Y_DELTA_DRQ_LIMIT_BASE_IDX 2
+#define mmHUBPREQ1_HUBPREQ_MEM_PWR_CTRL 0x073d
+#define mmHUBPREQ1_HUBPREQ_MEM_PWR_CTRL_BASE_IDX 2
+#define mmHUBPREQ1_HUBPREQ_MEM_PWR_STATUS 0x073e
+#define mmHUBPREQ1_HUBPREQ_MEM_PWR_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp1_dispdec_hubpret_dispdec
+// base address: 0x370
+#define mmHUBPRET1_HUBPRET_CONTROL 0x0748
+#define mmHUBPRET1_HUBPRET_CONTROL_BASE_IDX 2
+#define mmHUBPRET1_HUBPRET_MEM_PWR_CTRL 0x0749
+#define mmHUBPRET1_HUBPRET_MEM_PWR_CTRL_BASE_IDX 2
+#define mmHUBPRET1_HUBPRET_MEM_PWR_STATUS 0x074a
+#define mmHUBPRET1_HUBPRET_MEM_PWR_STATUS_BASE_IDX 2
+#define mmHUBPRET1_HUBPRET_READ_LINE_CTRL0 0x074b
+#define mmHUBPRET1_HUBPRET_READ_LINE_CTRL0_BASE_IDX 2
+#define mmHUBPRET1_HUBPRET_READ_LINE_CTRL1 0x074c
+#define mmHUBPRET1_HUBPRET_READ_LINE_CTRL1_BASE_IDX 2
+#define mmHUBPRET1_HUBPRET_READ_LINE0 0x074d
+#define mmHUBPRET1_HUBPRET_READ_LINE0_BASE_IDX 2
+#define mmHUBPRET1_HUBPRET_READ_LINE1 0x074e
+#define mmHUBPRET1_HUBPRET_READ_LINE1_BASE_IDX 2
+#define mmHUBPRET1_HUBPRET_INTERRUPT 0x074f
+#define mmHUBPRET1_HUBPRET_INTERRUPT_BASE_IDX 2
+#define mmHUBPRET1_HUBPRET_READ_LINE_VALUE 0x0750
+#define mmHUBPRET1_HUBPRET_READ_LINE_VALUE_BASE_IDX 2
+#define mmHUBPRET1_HUBPRET_READ_LINE_STATUS 0x0751
+#define mmHUBPRET1_HUBPRET_READ_LINE_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp1_dispdec_cursor0_dispdec
+// base address: 0x370
+#define mmCURSOR0_1_CURSOR_CONTROL 0x0754
+#define mmCURSOR0_1_CURSOR_CONTROL_BASE_IDX 2
+#define mmCURSOR0_1_CURSOR_SURFACE_ADDRESS 0x0755
+#define mmCURSOR0_1_CURSOR_SURFACE_ADDRESS_BASE_IDX 2
+#define mmCURSOR0_1_CURSOR_SURFACE_ADDRESS_HIGH 0x0756
+#define mmCURSOR0_1_CURSOR_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmCURSOR0_1_CURSOR_SIZE 0x0757
+#define mmCURSOR0_1_CURSOR_SIZE_BASE_IDX 2
+#define mmCURSOR0_1_CURSOR_POSITION 0x0758
+#define mmCURSOR0_1_CURSOR_POSITION_BASE_IDX 2
+#define mmCURSOR0_1_CURSOR_HOT_SPOT 0x0759
+#define mmCURSOR0_1_CURSOR_HOT_SPOT_BASE_IDX 2
+#define mmCURSOR0_1_CURSOR_STEREO_CONTROL 0x075a
+#define mmCURSOR0_1_CURSOR_STEREO_CONTROL_BASE_IDX 2
+#define mmCURSOR0_1_CURSOR_DST_OFFSET 0x075b
+#define mmCURSOR0_1_CURSOR_DST_OFFSET_BASE_IDX 2
+#define mmCURSOR0_1_CURSOR_MEM_PWR_CTRL 0x075c
+#define mmCURSOR0_1_CURSOR_MEM_PWR_CTRL_BASE_IDX 2
+#define mmCURSOR0_1_CURSOR_MEM_PWR_STATUS 0x075d
+#define mmCURSOR0_1_CURSOR_MEM_PWR_STATUS_BASE_IDX 2
+#define mmCURSOR0_1_DMDATA_ADDRESS_HIGH 0x075e
+#define mmCURSOR0_1_DMDATA_ADDRESS_HIGH_BASE_IDX 2
+#define mmCURSOR0_1_DMDATA_ADDRESS_LOW 0x075f
+#define mmCURSOR0_1_DMDATA_ADDRESS_LOW_BASE_IDX 2
+#define mmCURSOR0_1_DMDATA_CNTL 0x0760
+#define mmCURSOR0_1_DMDATA_CNTL_BASE_IDX 2
+#define mmCURSOR0_1_DMDATA_QOS_CNTL 0x0761
+#define mmCURSOR0_1_DMDATA_QOS_CNTL_BASE_IDX 2
+#define mmCURSOR0_1_DMDATA_STATUS 0x0762
+#define mmCURSOR0_1_DMDATA_STATUS_BASE_IDX 2
+#define mmCURSOR0_1_DMDATA_SW_CNTL 0x0763
+#define mmCURSOR0_1_DMDATA_SW_CNTL_BASE_IDX 2
+#define mmCURSOR0_1_DMDATA_SW_DATA 0x0764
+#define mmCURSOR0_1_DMDATA_SW_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp1_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x1de4
+#define mmDC_PERFMON8_PERFCOUNTER_CNTL 0x0779
+#define mmDC_PERFMON8_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON8_PERFCOUNTER_CNTL2 0x077a
+#define mmDC_PERFMON8_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON8_PERFCOUNTER_STATE 0x077b
+#define mmDC_PERFMON8_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON8_PERFMON_CNTL 0x077c
+#define mmDC_PERFMON8_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON8_PERFMON_CNTL2 0x077d
+#define mmDC_PERFMON8_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON8_PERFMON_CVALUE_INT_MISC 0x077e
+#define mmDC_PERFMON8_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON8_PERFMON_CVALUE_LOW 0x077f
+#define mmDC_PERFMON8_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON8_PERFMON_HI 0x0780
+#define mmDC_PERFMON8_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON8_PERFMON_LOW 0x0781
+#define mmDC_PERFMON8_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp1_dispdec_hubpxfc_dispdec
+// base address: 0x370
+#define mmHUBPXFC1_HUBP_XFC_CNTL 0x0785
+#define mmHUBPXFC1_HUBP_XFC_CNTL_BASE_IDX 2
+#define mmHUBPXFC1_HUBP_XFC_XBUF_RD_BASE0_ADDR_LSB 0x0786
+#define mmHUBPXFC1_HUBP_XFC_XBUF_RD_BASE0_ADDR_LSB_BASE_IDX 2
+#define mmHUBPXFC1_HUBP_XFC_XBUF_RD_BASE0_ADDR_MSB 0x0787
+#define mmHUBPXFC1_HUBP_XFC_XBUF_RD_BASE0_ADDR_MSB_BASE_IDX 2
+#define mmHUBPXFC1_HUBP_XFC_XBUF_RD_BASE1_ADDR_LSB 0x0788
+#define mmHUBPXFC1_HUBP_XFC_XBUF_RD_BASE1_ADDR_LSB_BASE_IDX 2
+#define mmHUBPXFC1_HUBP_XFC_XBUF_RD_BASE1_ADDR_MSB 0x0789
+#define mmHUBPXFC1_HUBP_XFC_XBUF_RD_BASE1_ADDR_MSB_BASE_IDX 2
+#define mmHUBPXFC1_HUBP_XFC_XBUF_RD_PITCH 0x078a
+#define mmHUBPXFC1_HUBP_XFC_XBUF_RD_PITCH_BASE_IDX 2
+#define mmHUBPXFC1_HUBP_XFC_DELAY_CONFIG0 0x078b
+#define mmHUBPXFC1_HUBP_XFC_DELAY_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC1_HUBP_XFC_DELAY_CONFIG1 0x078c
+#define mmHUBPXFC1_HUBP_XFC_DELAY_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC1_HUBP_XFC_DELAY_CONFIG2 0x078d
+#define mmHUBPXFC1_HUBP_XFC_DELAY_CONFIG2_BASE_IDX 2
+#define mmHUBPXFC1_HUBP_XFC_UNDERFLOW_STATUS 0x078e
+#define mmHUBPXFC1_HUBP_XFC_UNDERFLOW_STATUS_BASE_IDX 2
+#define mmHUBPXFC1_HUBP_XFC_SLV_VTG_CONFIG0 0x078f
+#define mmHUBPXFC1_HUBP_XFC_SLV_VTG_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC1_HUBP_XFC_SLV_VTG_CONFIG1 0x0790
+#define mmHUBPXFC1_HUBP_XFC_SLV_VTG_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC1_HUBP_XFC_SLV_SCALER_CONFIG0 0x0791
+#define mmHUBPXFC1_HUBP_XFC_SLV_SCALER_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC1_HUBP_XFC_SLV_SCALER_CONFIG1 0x0792
+#define mmHUBPXFC1_HUBP_XFC_SLV_SCALER_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC1_HUBP_XFC_MPC_CONFIG 0x0793
+#define mmHUBPXFC1_HUBP_XFC_MPC_CONFIG_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp2_dispdec_hubp_dispdec
+// base address: 0x6e0
+#define mmHUBP2_DCSURF_SURFACE_CONFIG 0x079d
+#define mmHUBP2_DCSURF_SURFACE_CONFIG_BASE_IDX 2
+#define mmHUBP2_DCSURF_ADDR_CONFIG 0x079e
+#define mmHUBP2_DCSURF_ADDR_CONFIG_BASE_IDX 2
+#define mmHUBP2_DCSURF_TILING_CONFIG 0x079f
+#define mmHUBP2_DCSURF_TILING_CONFIG_BASE_IDX 2
+#define mmHUBP2_DCSURF_PRI_VIEWPORT_START 0x07a1
+#define mmHUBP2_DCSURF_PRI_VIEWPORT_START_BASE_IDX 2
+#define mmHUBP2_DCSURF_PRI_VIEWPORT_DIMENSION 0x07a2
+#define mmHUBP2_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2
+#define mmHUBP2_DCSURF_PRI_VIEWPORT_START_C 0x07a3
+#define mmHUBP2_DCSURF_PRI_VIEWPORT_START_C_BASE_IDX 2
+#define mmHUBP2_DCSURF_PRI_VIEWPORT_DIMENSION_C 0x07a4
+#define mmHUBP2_DCSURF_PRI_VIEWPORT_DIMENSION_C_BASE_IDX 2
+#define mmHUBP2_DCSURF_SEC_VIEWPORT_START 0x07a5
+#define mmHUBP2_DCSURF_SEC_VIEWPORT_START_BASE_IDX 2
+#define mmHUBP2_DCSURF_SEC_VIEWPORT_DIMENSION 0x07a6
+#define mmHUBP2_DCSURF_SEC_VIEWPORT_DIMENSION_BASE_IDX 2
+#define mmHUBP2_DCSURF_SEC_VIEWPORT_START_C 0x07a7
+#define mmHUBP2_DCSURF_SEC_VIEWPORT_START_C_BASE_IDX 2
+#define mmHUBP2_DCSURF_SEC_VIEWPORT_DIMENSION_C 0x07a8
+#define mmHUBP2_DCSURF_SEC_VIEWPORT_DIMENSION_C_BASE_IDX 2
+#define mmHUBP2_DCHUBP_REQ_SIZE_CONFIG 0x07a9
+#define mmHUBP2_DCHUBP_REQ_SIZE_CONFIG_BASE_IDX 2
+#define mmHUBP2_DCHUBP_REQ_SIZE_CONFIG_C 0x07aa
+#define mmHUBP2_DCHUBP_REQ_SIZE_CONFIG_C_BASE_IDX 2
+#define mmHUBP2_DCHUBP_CNTL 0x07ab
+#define mmHUBP2_DCHUBP_CNTL_BASE_IDX 2
+#define mmHUBP2_HUBP_CLK_CNTL 0x07ac
+#define mmHUBP2_HUBP_CLK_CNTL_BASE_IDX 2
+#define mmHUBP2_DCHUBP_VMPG_CONFIG 0x07ad
+#define mmHUBP2_DCHUBP_VMPG_CONFIG_BASE_IDX 2
+#define mmHUBP2_HUBPREQ_DEBUG_DB 0x07ae
+#define mmHUBP2_HUBPREQ_DEBUG_DB_BASE_IDX 2
+#define mmHUBP2_HUBPREQ_DEBUG 0x07af
+#define mmHUBP2_HUBPREQ_DEBUG_BASE_IDX 2
+#define mmHUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK 0x07b3
+#define mmHUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK_BASE_IDX 2
+#define mmHUBP2_HUBP_MEASURE_WIN_CTRL_DPPCLK 0x07b4
+#define mmHUBP2_HUBP_MEASURE_WIN_CTRL_DPPCLK_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp2_dispdec_hubpreq_dispdec
+// base address: 0x6e0
+#define mmHUBPREQ2_DCSURF_SURFACE_PITCH 0x07bf
+#define mmHUBPREQ2_DCSURF_SURFACE_PITCH_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SURFACE_PITCH_C 0x07c0
+#define mmHUBPREQ2_DCSURF_SURFACE_PITCH_C_BASE_IDX 2
+#define mmHUBPREQ2_VMID_SETTINGS_0 0x07c1
+#define mmHUBPREQ2_VMID_SETTINGS_0_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS 0x07c2
+#define mmHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH 0x07c3
+#define mmHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_C 0x07c4
+#define mmHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C 0x07c5
+#define mmHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS 0x07c6
+#define mmHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH 0x07c7
+#define mmHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_C 0x07c8
+#define mmHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C 0x07c9
+#define mmHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS 0x07ca
+#define mmHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH 0x07cb
+#define mmHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C 0x07cc
+#define mmHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C 0x07cd
+#define mmHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS 0x07ce
+#define mmHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH 0x07cf
+#define mmHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C 0x07d0
+#define mmHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C 0x07d1
+#define mmHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SURFACE_CONTROL 0x07d2
+#define mmHUBPREQ2_DCSURF_SURFACE_CONTROL_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_FLIP_CONTROL 0x07d3
+#define mmHUBPREQ2_DCSURF_FLIP_CONTROL_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_FLIP_CONTROL2 0x07d4
+#define mmHUBPREQ2_DCSURF_FLIP_CONTROL2_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_QUEUE_CONTROL 0x07d5
+#define mmHUBPREQ2_DCSURF_QUEUE_CONTROL_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_FRAME_PACING_TIME 0x07d6
+#define mmHUBPREQ2_DCSURF_FRAME_PACING_TIME_BASE_IDX 2
+#define mmHUBPREQ2_SURFACE_CURRENT_PACING_COUNTER 0x07d7
+#define mmHUBPREQ2_SURFACE_CURRENT_PACING_COUNTER_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT 0x07d8
+#define mmHUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SURFACE_INUSE 0x07d9
+#define mmHUBPREQ2_DCSURF_SURFACE_INUSE_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SURFACE_INUSE_HIGH 0x07da
+#define mmHUBPREQ2_DCSURF_SURFACE_INUSE_HIGH_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SURFACE_INUSE_C 0x07db
+#define mmHUBPREQ2_DCSURF_SURFACE_INUSE_C_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SURFACE_INUSE_HIGH_C 0x07dc
+#define mmHUBPREQ2_DCSURF_SURFACE_INUSE_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE 0x07dd
+#define mmHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH 0x07de
+#define mmHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_C 0x07df
+#define mmHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_C_BASE_IDX 2
+#define mmHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C 0x07e0
+#define mmHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ2_DCN_EXPANSION_MODE 0x07e4
+#define mmHUBPREQ2_DCN_EXPANSION_MODE_BASE_IDX 2
+#define mmHUBPREQ2_DCN_TTU_QOS_WM 0x07e5
+#define mmHUBPREQ2_DCN_TTU_QOS_WM_BASE_IDX 2
+#define mmHUBPREQ2_DCN_GLOBAL_TTU_CNTL 0x07e6
+#define mmHUBPREQ2_DCN_GLOBAL_TTU_CNTL_BASE_IDX 2
+#define mmHUBPREQ2_DCN_SURF0_TTU_CNTL0 0x07e7
+#define mmHUBPREQ2_DCN_SURF0_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ2_DCN_SURF0_TTU_CNTL1 0x07e8
+#define mmHUBPREQ2_DCN_SURF0_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ2_DCN_SURF1_TTU_CNTL0 0x07e9
+#define mmHUBPREQ2_DCN_SURF1_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ2_DCN_SURF1_TTU_CNTL1 0x07ea
+#define mmHUBPREQ2_DCN_SURF1_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ2_DCN_CUR0_TTU_CNTL0 0x07eb
+#define mmHUBPREQ2_DCN_CUR0_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ2_DCN_CUR0_TTU_CNTL1 0x07ec
+#define mmHUBPREQ2_DCN_CUR0_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ2_DCN_CUR1_TTU_CNTL0 0x07ed
+#define mmHUBPREQ2_DCN_CUR1_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ2_DCN_CUR1_TTU_CNTL1 0x07ee
+#define mmHUBPREQ2_DCN_CUR1_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ2_DCN_VM_SYSTEM_APERTURE_LOW_ADDR 0x07ef
+#define mmHUBPREQ2_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_BASE_IDX 2
+#define mmHUBPREQ2_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR 0x07f0
+#define mmHUBPREQ2_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_BASE_IDX 2
+#define mmHUBPREQ2_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB 0x07f1
+#define mmHUBPREQ2_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ2_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB 0x07f2
+#define mmHUBPREQ2_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB 0x07f3
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB 0x07f4
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB 0x07f5
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB 0x07f6
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB 0x07f7
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB 0x07f8
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB 0x07f9
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB 0x07fa
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PROTECTION_FAULT_ADDR_LSB 0x07fb
+#define mmHUBPREQ2_DC_VM_CONTEXT0_PROTECTION_FAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ2_DC_VM_CONTEXT0_CNTL 0x07fc
+#define mmHUBPREQ2_DC_VM_CONTEXT0_CNTL_BASE_IDX 2
+#define mmHUBPREQ2_DCN_VM_MX_L1_TLB_CNTL 0x07fd
+#define mmHUBPREQ2_DCN_VM_MX_L1_TLB_CNTL_BASE_IDX 2
+#define mmHUBPREQ2_BLANK_OFFSET_0 0x07fe
+#define mmHUBPREQ2_BLANK_OFFSET_0_BASE_IDX 2
+#define mmHUBPREQ2_BLANK_OFFSET_1 0x07ff
+#define mmHUBPREQ2_BLANK_OFFSET_1_BASE_IDX 2
+#define mmHUBPREQ2_DST_DIMENSIONS 0x0800
+#define mmHUBPREQ2_DST_DIMENSIONS_BASE_IDX 2
+#define mmHUBPREQ2_DST_AFTER_SCALER 0x0801
+#define mmHUBPREQ2_DST_AFTER_SCALER_BASE_IDX 2
+#define mmHUBPREQ2_PREFETCH_SETTINGS 0x0802
+#define mmHUBPREQ2_PREFETCH_SETTINGS_BASE_IDX 2
+#define mmHUBPREQ2_PREFETCH_SETTINGS_C 0x0803
+#define mmHUBPREQ2_PREFETCH_SETTINGS_C_BASE_IDX 2
+#define mmHUBPREQ2_VBLANK_PARAMETERS_0 0x0804
+#define mmHUBPREQ2_VBLANK_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ2_VBLANK_PARAMETERS_1 0x0805
+#define mmHUBPREQ2_VBLANK_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ2_VBLANK_PARAMETERS_2 0x0806
+#define mmHUBPREQ2_VBLANK_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ2_VBLANK_PARAMETERS_3 0x0807
+#define mmHUBPREQ2_VBLANK_PARAMETERS_3_BASE_IDX 2
+#define mmHUBPREQ2_VBLANK_PARAMETERS_4 0x0808
+#define mmHUBPREQ2_VBLANK_PARAMETERS_4_BASE_IDX 2
+#define mmHUBPREQ2_FLIP_PARAMETERS_0 0x0809
+#define mmHUBPREQ2_FLIP_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ2_FLIP_PARAMETERS_1 0x080a
+#define mmHUBPREQ2_FLIP_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ2_FLIP_PARAMETERS_2 0x080b
+#define mmHUBPREQ2_FLIP_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ2_NOM_PARAMETERS_0 0x080c
+#define mmHUBPREQ2_NOM_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ2_NOM_PARAMETERS_1 0x080d
+#define mmHUBPREQ2_NOM_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ2_NOM_PARAMETERS_2 0x080e
+#define mmHUBPREQ2_NOM_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ2_NOM_PARAMETERS_3 0x080f
+#define mmHUBPREQ2_NOM_PARAMETERS_3_BASE_IDX 2
+#define mmHUBPREQ2_NOM_PARAMETERS_4 0x0810
+#define mmHUBPREQ2_NOM_PARAMETERS_4_BASE_IDX 2
+#define mmHUBPREQ2_NOM_PARAMETERS_5 0x0811
+#define mmHUBPREQ2_NOM_PARAMETERS_5_BASE_IDX 2
+#define mmHUBPREQ2_NOM_PARAMETERS_6 0x0812
+#define mmHUBPREQ2_NOM_PARAMETERS_6_BASE_IDX 2
+#define mmHUBPREQ2_NOM_PARAMETERS_7 0x0813
+#define mmHUBPREQ2_NOM_PARAMETERS_7_BASE_IDX 2
+#define mmHUBPREQ2_PER_LINE_DELIVERY_PRE 0x0814
+#define mmHUBPREQ2_PER_LINE_DELIVERY_PRE_BASE_IDX 2
+#define mmHUBPREQ2_PER_LINE_DELIVERY 0x0815
+#define mmHUBPREQ2_PER_LINE_DELIVERY_BASE_IDX 2
+#define mmHUBPREQ2_CURSOR_SETTINGS 0x0816
+#define mmHUBPREQ2_CURSOR_SETTINGS_BASE_IDX 2
+#define mmHUBPREQ2_REF_FREQ_TO_PIX_FREQ 0x0817
+#define mmHUBPREQ2_REF_FREQ_TO_PIX_FREQ_BASE_IDX 2
+#define mmHUBPREQ2_DST_Y_DELTA_DRQ_LIMIT 0x0818
+#define mmHUBPREQ2_DST_Y_DELTA_DRQ_LIMIT_BASE_IDX 2
+#define mmHUBPREQ2_HUBPREQ_MEM_PWR_CTRL 0x0819
+#define mmHUBPREQ2_HUBPREQ_MEM_PWR_CTRL_BASE_IDX 2
+#define mmHUBPREQ2_HUBPREQ_MEM_PWR_STATUS 0x081a
+#define mmHUBPREQ2_HUBPREQ_MEM_PWR_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp2_dispdec_hubpret_dispdec
+// base address: 0x6e0
+#define mmHUBPRET2_HUBPRET_CONTROL 0x0824
+#define mmHUBPRET2_HUBPRET_CONTROL_BASE_IDX 2
+#define mmHUBPRET2_HUBPRET_MEM_PWR_CTRL 0x0825
+#define mmHUBPRET2_HUBPRET_MEM_PWR_CTRL_BASE_IDX 2
+#define mmHUBPRET2_HUBPRET_MEM_PWR_STATUS 0x0826
+#define mmHUBPRET2_HUBPRET_MEM_PWR_STATUS_BASE_IDX 2
+#define mmHUBPRET2_HUBPRET_READ_LINE_CTRL0 0x0827
+#define mmHUBPRET2_HUBPRET_READ_LINE_CTRL0_BASE_IDX 2
+#define mmHUBPRET2_HUBPRET_READ_LINE_CTRL1 0x0828
+#define mmHUBPRET2_HUBPRET_READ_LINE_CTRL1_BASE_IDX 2
+#define mmHUBPRET2_HUBPRET_READ_LINE0 0x0829
+#define mmHUBPRET2_HUBPRET_READ_LINE0_BASE_IDX 2
+#define mmHUBPRET2_HUBPRET_READ_LINE1 0x082a
+#define mmHUBPRET2_HUBPRET_READ_LINE1_BASE_IDX 2
+#define mmHUBPRET2_HUBPRET_INTERRUPT 0x082b
+#define mmHUBPRET2_HUBPRET_INTERRUPT_BASE_IDX 2
+#define mmHUBPRET2_HUBPRET_READ_LINE_VALUE 0x082c
+#define mmHUBPRET2_HUBPRET_READ_LINE_VALUE_BASE_IDX 2
+#define mmHUBPRET2_HUBPRET_READ_LINE_STATUS 0x082d
+#define mmHUBPRET2_HUBPRET_READ_LINE_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp2_dispdec_cursor0_dispdec
+// base address: 0x6e0
+#define mmCURSOR0_2_CURSOR_CONTROL 0x0830
+#define mmCURSOR0_2_CURSOR_CONTROL_BASE_IDX 2
+#define mmCURSOR0_2_CURSOR_SURFACE_ADDRESS 0x0831
+#define mmCURSOR0_2_CURSOR_SURFACE_ADDRESS_BASE_IDX 2
+#define mmCURSOR0_2_CURSOR_SURFACE_ADDRESS_HIGH 0x0832
+#define mmCURSOR0_2_CURSOR_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmCURSOR0_2_CURSOR_SIZE 0x0833
+#define mmCURSOR0_2_CURSOR_SIZE_BASE_IDX 2
+#define mmCURSOR0_2_CURSOR_POSITION 0x0834
+#define mmCURSOR0_2_CURSOR_POSITION_BASE_IDX 2
+#define mmCURSOR0_2_CURSOR_HOT_SPOT 0x0835
+#define mmCURSOR0_2_CURSOR_HOT_SPOT_BASE_IDX 2
+#define mmCURSOR0_2_CURSOR_STEREO_CONTROL 0x0836
+#define mmCURSOR0_2_CURSOR_STEREO_CONTROL_BASE_IDX 2
+#define mmCURSOR0_2_CURSOR_DST_OFFSET 0x0837
+#define mmCURSOR0_2_CURSOR_DST_OFFSET_BASE_IDX 2
+#define mmCURSOR0_2_CURSOR_MEM_PWR_CTRL 0x0838
+#define mmCURSOR0_2_CURSOR_MEM_PWR_CTRL_BASE_IDX 2
+#define mmCURSOR0_2_CURSOR_MEM_PWR_STATUS 0x0839
+#define mmCURSOR0_2_CURSOR_MEM_PWR_STATUS_BASE_IDX 2
+#define mmCURSOR0_2_DMDATA_ADDRESS_HIGH 0x083a
+#define mmCURSOR0_2_DMDATA_ADDRESS_HIGH_BASE_IDX 2
+#define mmCURSOR0_2_DMDATA_ADDRESS_LOW 0x083b
+#define mmCURSOR0_2_DMDATA_ADDRESS_LOW_BASE_IDX 2
+#define mmCURSOR0_2_DMDATA_CNTL 0x083c
+#define mmCURSOR0_2_DMDATA_CNTL_BASE_IDX 2
+#define mmCURSOR0_2_DMDATA_QOS_CNTL 0x083d
+#define mmCURSOR0_2_DMDATA_QOS_CNTL_BASE_IDX 2
+#define mmCURSOR0_2_DMDATA_STATUS 0x083e
+#define mmCURSOR0_2_DMDATA_STATUS_BASE_IDX 2
+#define mmCURSOR0_2_DMDATA_SW_CNTL 0x083f
+#define mmCURSOR0_2_DMDATA_SW_CNTL_BASE_IDX 2
+#define mmCURSOR0_2_DMDATA_SW_DATA 0x0840
+#define mmCURSOR0_2_DMDATA_SW_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp2_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x2154
+#define mmDC_PERFMON9_PERFCOUNTER_CNTL 0x0855
+#define mmDC_PERFMON9_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON9_PERFCOUNTER_CNTL2 0x0856
+#define mmDC_PERFMON9_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON9_PERFCOUNTER_STATE 0x0857
+#define mmDC_PERFMON9_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON9_PERFMON_CNTL 0x0858
+#define mmDC_PERFMON9_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON9_PERFMON_CNTL2 0x0859
+#define mmDC_PERFMON9_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON9_PERFMON_CVALUE_INT_MISC 0x085a
+#define mmDC_PERFMON9_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON9_PERFMON_CVALUE_LOW 0x085b
+#define mmDC_PERFMON9_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON9_PERFMON_HI 0x085c
+#define mmDC_PERFMON9_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON9_PERFMON_LOW 0x085d
+#define mmDC_PERFMON9_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp2_dispdec_hubpxfc_dispdec
+// base address: 0x6e0
+#define mmHUBPXFC2_HUBP_XFC_CNTL 0x0861
+#define mmHUBPXFC2_HUBP_XFC_CNTL_BASE_IDX 2
+#define mmHUBPXFC2_HUBP_XFC_XBUF_RD_BASE0_ADDR_LSB 0x0862
+#define mmHUBPXFC2_HUBP_XFC_XBUF_RD_BASE0_ADDR_LSB_BASE_IDX 2
+#define mmHUBPXFC2_HUBP_XFC_XBUF_RD_BASE0_ADDR_MSB 0x0863
+#define mmHUBPXFC2_HUBP_XFC_XBUF_RD_BASE0_ADDR_MSB_BASE_IDX 2
+#define mmHUBPXFC2_HUBP_XFC_XBUF_RD_BASE1_ADDR_LSB 0x0864
+#define mmHUBPXFC2_HUBP_XFC_XBUF_RD_BASE1_ADDR_LSB_BASE_IDX 2
+#define mmHUBPXFC2_HUBP_XFC_XBUF_RD_BASE1_ADDR_MSB 0x0865
+#define mmHUBPXFC2_HUBP_XFC_XBUF_RD_BASE1_ADDR_MSB_BASE_IDX 2
+#define mmHUBPXFC2_HUBP_XFC_XBUF_RD_PITCH 0x0866
+#define mmHUBPXFC2_HUBP_XFC_XBUF_RD_PITCH_BASE_IDX 2
+#define mmHUBPXFC2_HUBP_XFC_DELAY_CONFIG0 0x0867
+#define mmHUBPXFC2_HUBP_XFC_DELAY_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC2_HUBP_XFC_DELAY_CONFIG1 0x0868
+#define mmHUBPXFC2_HUBP_XFC_DELAY_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC2_HUBP_XFC_DELAY_CONFIG2 0x0869
+#define mmHUBPXFC2_HUBP_XFC_DELAY_CONFIG2_BASE_IDX 2
+#define mmHUBPXFC2_HUBP_XFC_UNDERFLOW_STATUS 0x086a
+#define mmHUBPXFC2_HUBP_XFC_UNDERFLOW_STATUS_BASE_IDX 2
+#define mmHUBPXFC2_HUBP_XFC_SLV_VTG_CONFIG0 0x086b
+#define mmHUBPXFC2_HUBP_XFC_SLV_VTG_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC2_HUBP_XFC_SLV_VTG_CONFIG1 0x086c
+#define mmHUBPXFC2_HUBP_XFC_SLV_VTG_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC2_HUBP_XFC_SLV_SCALER_CONFIG0 0x086d
+#define mmHUBPXFC2_HUBP_XFC_SLV_SCALER_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC2_HUBP_XFC_SLV_SCALER_CONFIG1 0x086e
+#define mmHUBPXFC2_HUBP_XFC_SLV_SCALER_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC2_HUBP_XFC_MPC_CONFIG 0x086f
+#define mmHUBPXFC2_HUBP_XFC_MPC_CONFIG_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp3_dispdec_hubp_dispdec
+// base address: 0xa50
+#define mmHUBP3_DCSURF_SURFACE_CONFIG 0x0879
+#define mmHUBP3_DCSURF_SURFACE_CONFIG_BASE_IDX 2
+#define mmHUBP3_DCSURF_ADDR_CONFIG 0x087a
+#define mmHUBP3_DCSURF_ADDR_CONFIG_BASE_IDX 2
+#define mmHUBP3_DCSURF_TILING_CONFIG 0x087b
+#define mmHUBP3_DCSURF_TILING_CONFIG_BASE_IDX 2
+#define mmHUBP3_DCSURF_PRI_VIEWPORT_START 0x087d
+#define mmHUBP3_DCSURF_PRI_VIEWPORT_START_BASE_IDX 2
+#define mmHUBP3_DCSURF_PRI_VIEWPORT_DIMENSION 0x087e
+#define mmHUBP3_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2
+#define mmHUBP3_DCSURF_PRI_VIEWPORT_START_C 0x087f
+#define mmHUBP3_DCSURF_PRI_VIEWPORT_START_C_BASE_IDX 2
+#define mmHUBP3_DCSURF_PRI_VIEWPORT_DIMENSION_C 0x0880
+#define mmHUBP3_DCSURF_PRI_VIEWPORT_DIMENSION_C_BASE_IDX 2
+#define mmHUBP3_DCSURF_SEC_VIEWPORT_START 0x0881
+#define mmHUBP3_DCSURF_SEC_VIEWPORT_START_BASE_IDX 2
+#define mmHUBP3_DCSURF_SEC_VIEWPORT_DIMENSION 0x0882
+#define mmHUBP3_DCSURF_SEC_VIEWPORT_DIMENSION_BASE_IDX 2
+#define mmHUBP3_DCSURF_SEC_VIEWPORT_START_C 0x0883
+#define mmHUBP3_DCSURF_SEC_VIEWPORT_START_C_BASE_IDX 2
+#define mmHUBP3_DCSURF_SEC_VIEWPORT_DIMENSION_C 0x0884
+#define mmHUBP3_DCSURF_SEC_VIEWPORT_DIMENSION_C_BASE_IDX 2
+#define mmHUBP3_DCHUBP_REQ_SIZE_CONFIG 0x0885
+#define mmHUBP3_DCHUBP_REQ_SIZE_CONFIG_BASE_IDX 2
+#define mmHUBP3_DCHUBP_REQ_SIZE_CONFIG_C 0x0886
+#define mmHUBP3_DCHUBP_REQ_SIZE_CONFIG_C_BASE_IDX 2
+#define mmHUBP3_DCHUBP_CNTL 0x0887
+#define mmHUBP3_DCHUBP_CNTL_BASE_IDX 2
+#define mmHUBP3_HUBP_CLK_CNTL 0x0888
+#define mmHUBP3_HUBP_CLK_CNTL_BASE_IDX 2
+#define mmHUBP3_DCHUBP_VMPG_CONFIG 0x0889
+#define mmHUBP3_DCHUBP_VMPG_CONFIG_BASE_IDX 2
+#define mmHUBP3_HUBPREQ_DEBUG_DB 0x088a
+#define mmHUBP3_HUBPREQ_DEBUG_DB_BASE_IDX 2
+#define mmHUBP3_HUBPREQ_DEBUG 0x088b
+#define mmHUBP3_HUBPREQ_DEBUG_BASE_IDX 2
+#define mmHUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK 0x088f
+#define mmHUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK_BASE_IDX 2
+#define mmHUBP3_HUBP_MEASURE_WIN_CTRL_DPPCLK 0x0890
+#define mmHUBP3_HUBP_MEASURE_WIN_CTRL_DPPCLK_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp3_dispdec_hubpreq_dispdec
+// base address: 0xa50
+#define mmHUBPREQ3_DCSURF_SURFACE_PITCH 0x089b
+#define mmHUBPREQ3_DCSURF_SURFACE_PITCH_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SURFACE_PITCH_C 0x089c
+#define mmHUBPREQ3_DCSURF_SURFACE_PITCH_C_BASE_IDX 2
+#define mmHUBPREQ3_VMID_SETTINGS_0 0x089d
+#define mmHUBPREQ3_VMID_SETTINGS_0_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS 0x089e
+#define mmHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH 0x089f
+#define mmHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_C 0x08a0
+#define mmHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C 0x08a1
+#define mmHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS 0x08a2
+#define mmHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH 0x08a3
+#define mmHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_C 0x08a4
+#define mmHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C 0x08a5
+#define mmHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS 0x08a6
+#define mmHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH 0x08a7
+#define mmHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C 0x08a8
+#define mmHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C 0x08a9
+#define mmHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS 0x08aa
+#define mmHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH 0x08ab
+#define mmHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C 0x08ac
+#define mmHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C 0x08ad
+#define mmHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SURFACE_CONTROL 0x08ae
+#define mmHUBPREQ3_DCSURF_SURFACE_CONTROL_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_FLIP_CONTROL 0x08af
+#define mmHUBPREQ3_DCSURF_FLIP_CONTROL_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_FLIP_CONTROL2 0x08b0
+#define mmHUBPREQ3_DCSURF_FLIP_CONTROL2_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_QUEUE_CONTROL 0x08b1
+#define mmHUBPREQ3_DCSURF_QUEUE_CONTROL_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_FRAME_PACING_TIME 0x08b2
+#define mmHUBPREQ3_DCSURF_FRAME_PACING_TIME_BASE_IDX 2
+#define mmHUBPREQ3_SURFACE_CURRENT_PACING_COUNTER 0x08b3
+#define mmHUBPREQ3_SURFACE_CURRENT_PACING_COUNTER_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT 0x08b4
+#define mmHUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SURFACE_INUSE 0x08b5
+#define mmHUBPREQ3_DCSURF_SURFACE_INUSE_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SURFACE_INUSE_HIGH 0x08b6
+#define mmHUBPREQ3_DCSURF_SURFACE_INUSE_HIGH_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SURFACE_INUSE_C 0x08b7
+#define mmHUBPREQ3_DCSURF_SURFACE_INUSE_C_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SURFACE_INUSE_HIGH_C 0x08b8
+#define mmHUBPREQ3_DCSURF_SURFACE_INUSE_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE 0x08b9
+#define mmHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH 0x08ba
+#define mmHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_C 0x08bb
+#define mmHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_C_BASE_IDX 2
+#define mmHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C 0x08bc
+#define mmHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ3_DCN_EXPANSION_MODE 0x08c0
+#define mmHUBPREQ3_DCN_EXPANSION_MODE_BASE_IDX 2
+#define mmHUBPREQ3_DCN_TTU_QOS_WM 0x08c1
+#define mmHUBPREQ3_DCN_TTU_QOS_WM_BASE_IDX 2
+#define mmHUBPREQ3_DCN_GLOBAL_TTU_CNTL 0x08c2
+#define mmHUBPREQ3_DCN_GLOBAL_TTU_CNTL_BASE_IDX 2
+#define mmHUBPREQ3_DCN_SURF0_TTU_CNTL0 0x08c3
+#define mmHUBPREQ3_DCN_SURF0_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ3_DCN_SURF0_TTU_CNTL1 0x08c4
+#define mmHUBPREQ3_DCN_SURF0_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ3_DCN_SURF1_TTU_CNTL0 0x08c5
+#define mmHUBPREQ3_DCN_SURF1_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ3_DCN_SURF1_TTU_CNTL1 0x08c6
+#define mmHUBPREQ3_DCN_SURF1_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ3_DCN_CUR0_TTU_CNTL0 0x08c7
+#define mmHUBPREQ3_DCN_CUR0_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ3_DCN_CUR0_TTU_CNTL1 0x08c8
+#define mmHUBPREQ3_DCN_CUR0_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ3_DCN_CUR1_TTU_CNTL0 0x08c9
+#define mmHUBPREQ3_DCN_CUR1_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ3_DCN_CUR1_TTU_CNTL1 0x08ca
+#define mmHUBPREQ3_DCN_CUR1_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ3_DCN_VM_SYSTEM_APERTURE_LOW_ADDR 0x08cb
+#define mmHUBPREQ3_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_BASE_IDX 2
+#define mmHUBPREQ3_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR 0x08cc
+#define mmHUBPREQ3_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_BASE_IDX 2
+#define mmHUBPREQ3_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB 0x08cd
+#define mmHUBPREQ3_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ3_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB 0x08ce
+#define mmHUBPREQ3_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB 0x08cf
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB 0x08d0
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB 0x08d1
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB 0x08d2
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB 0x08d3
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB 0x08d4
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB 0x08d5
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB 0x08d6
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PROTECTION_FAULT_ADDR_LSB 0x08d7
+#define mmHUBPREQ3_DC_VM_CONTEXT0_PROTECTION_FAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ3_DC_VM_CONTEXT0_CNTL 0x08d8
+#define mmHUBPREQ3_DC_VM_CONTEXT0_CNTL_BASE_IDX 2
+#define mmHUBPREQ3_DCN_VM_MX_L1_TLB_CNTL 0x08d9
+#define mmHUBPREQ3_DCN_VM_MX_L1_TLB_CNTL_BASE_IDX 2
+#define mmHUBPREQ3_BLANK_OFFSET_0 0x08da
+#define mmHUBPREQ3_BLANK_OFFSET_0_BASE_IDX 2
+#define mmHUBPREQ3_BLANK_OFFSET_1 0x08db
+#define mmHUBPREQ3_BLANK_OFFSET_1_BASE_IDX 2
+#define mmHUBPREQ3_DST_DIMENSIONS 0x08dc
+#define mmHUBPREQ3_DST_DIMENSIONS_BASE_IDX 2
+#define mmHUBPREQ3_DST_AFTER_SCALER 0x08dd
+#define mmHUBPREQ3_DST_AFTER_SCALER_BASE_IDX 2
+#define mmHUBPREQ3_PREFETCH_SETTINGS 0x08de
+#define mmHUBPREQ3_PREFETCH_SETTINGS_BASE_IDX 2
+#define mmHUBPREQ3_PREFETCH_SETTINGS_C 0x08df
+#define mmHUBPREQ3_PREFETCH_SETTINGS_C_BASE_IDX 2
+#define mmHUBPREQ3_VBLANK_PARAMETERS_0 0x08e0
+#define mmHUBPREQ3_VBLANK_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ3_VBLANK_PARAMETERS_1 0x08e1
+#define mmHUBPREQ3_VBLANK_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ3_VBLANK_PARAMETERS_2 0x08e2
+#define mmHUBPREQ3_VBLANK_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ3_VBLANK_PARAMETERS_3 0x08e3
+#define mmHUBPREQ3_VBLANK_PARAMETERS_3_BASE_IDX 2
+#define mmHUBPREQ3_VBLANK_PARAMETERS_4 0x08e4
+#define mmHUBPREQ3_VBLANK_PARAMETERS_4_BASE_IDX 2
+#define mmHUBPREQ3_FLIP_PARAMETERS_0 0x08e5
+#define mmHUBPREQ3_FLIP_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ3_FLIP_PARAMETERS_1 0x08e6
+#define mmHUBPREQ3_FLIP_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ3_FLIP_PARAMETERS_2 0x08e7
+#define mmHUBPREQ3_FLIP_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ3_NOM_PARAMETERS_0 0x08e8
+#define mmHUBPREQ3_NOM_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ3_NOM_PARAMETERS_1 0x08e9
+#define mmHUBPREQ3_NOM_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ3_NOM_PARAMETERS_2 0x08ea
+#define mmHUBPREQ3_NOM_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ3_NOM_PARAMETERS_3 0x08eb
+#define mmHUBPREQ3_NOM_PARAMETERS_3_BASE_IDX 2
+#define mmHUBPREQ3_NOM_PARAMETERS_4 0x08ec
+#define mmHUBPREQ3_NOM_PARAMETERS_4_BASE_IDX 2
+#define mmHUBPREQ3_NOM_PARAMETERS_5 0x08ed
+#define mmHUBPREQ3_NOM_PARAMETERS_5_BASE_IDX 2
+#define mmHUBPREQ3_NOM_PARAMETERS_6 0x08ee
+#define mmHUBPREQ3_NOM_PARAMETERS_6_BASE_IDX 2
+#define mmHUBPREQ3_NOM_PARAMETERS_7 0x08ef
+#define mmHUBPREQ3_NOM_PARAMETERS_7_BASE_IDX 2
+#define mmHUBPREQ3_PER_LINE_DELIVERY_PRE 0x08f0
+#define mmHUBPREQ3_PER_LINE_DELIVERY_PRE_BASE_IDX 2
+#define mmHUBPREQ3_PER_LINE_DELIVERY 0x08f1
+#define mmHUBPREQ3_PER_LINE_DELIVERY_BASE_IDX 2
+#define mmHUBPREQ3_CURSOR_SETTINGS 0x08f2
+#define mmHUBPREQ3_CURSOR_SETTINGS_BASE_IDX 2
+#define mmHUBPREQ3_REF_FREQ_TO_PIX_FREQ 0x08f3
+#define mmHUBPREQ3_REF_FREQ_TO_PIX_FREQ_BASE_IDX 2
+#define mmHUBPREQ3_DST_Y_DELTA_DRQ_LIMIT 0x08f4
+#define mmHUBPREQ3_DST_Y_DELTA_DRQ_LIMIT_BASE_IDX 2
+#define mmHUBPREQ3_HUBPREQ_MEM_PWR_CTRL 0x08f5
+#define mmHUBPREQ3_HUBPREQ_MEM_PWR_CTRL_BASE_IDX 2
+#define mmHUBPREQ3_HUBPREQ_MEM_PWR_STATUS 0x08f6
+#define mmHUBPREQ3_HUBPREQ_MEM_PWR_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp3_dispdec_hubpret_dispdec
+// base address: 0xa50
+#define mmHUBPRET3_HUBPRET_CONTROL 0x0900
+#define mmHUBPRET3_HUBPRET_CONTROL_BASE_IDX 2
+#define mmHUBPRET3_HUBPRET_MEM_PWR_CTRL 0x0901
+#define mmHUBPRET3_HUBPRET_MEM_PWR_CTRL_BASE_IDX 2
+#define mmHUBPRET3_HUBPRET_MEM_PWR_STATUS 0x0902
+#define mmHUBPRET3_HUBPRET_MEM_PWR_STATUS_BASE_IDX 2
+#define mmHUBPRET3_HUBPRET_READ_LINE_CTRL0 0x0903
+#define mmHUBPRET3_HUBPRET_READ_LINE_CTRL0_BASE_IDX 2
+#define mmHUBPRET3_HUBPRET_READ_LINE_CTRL1 0x0904
+#define mmHUBPRET3_HUBPRET_READ_LINE_CTRL1_BASE_IDX 2
+#define mmHUBPRET3_HUBPRET_READ_LINE0 0x0905
+#define mmHUBPRET3_HUBPRET_READ_LINE0_BASE_IDX 2
+#define mmHUBPRET3_HUBPRET_READ_LINE1 0x0906
+#define mmHUBPRET3_HUBPRET_READ_LINE1_BASE_IDX 2
+#define mmHUBPRET3_HUBPRET_INTERRUPT 0x0907
+#define mmHUBPRET3_HUBPRET_INTERRUPT_BASE_IDX 2
+#define mmHUBPRET3_HUBPRET_READ_LINE_VALUE 0x0908
+#define mmHUBPRET3_HUBPRET_READ_LINE_VALUE_BASE_IDX 2
+#define mmHUBPRET3_HUBPRET_READ_LINE_STATUS 0x0909
+#define mmHUBPRET3_HUBPRET_READ_LINE_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp3_dispdec_cursor0_dispdec
+// base address: 0xa50
+#define mmCURSOR0_3_CURSOR_CONTROL 0x090c
+#define mmCURSOR0_3_CURSOR_CONTROL_BASE_IDX 2
+#define mmCURSOR0_3_CURSOR_SURFACE_ADDRESS 0x090d
+#define mmCURSOR0_3_CURSOR_SURFACE_ADDRESS_BASE_IDX 2
+#define mmCURSOR0_3_CURSOR_SURFACE_ADDRESS_HIGH 0x090e
+#define mmCURSOR0_3_CURSOR_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmCURSOR0_3_CURSOR_SIZE 0x090f
+#define mmCURSOR0_3_CURSOR_SIZE_BASE_IDX 2
+#define mmCURSOR0_3_CURSOR_POSITION 0x0910
+#define mmCURSOR0_3_CURSOR_POSITION_BASE_IDX 2
+#define mmCURSOR0_3_CURSOR_HOT_SPOT 0x0911
+#define mmCURSOR0_3_CURSOR_HOT_SPOT_BASE_IDX 2
+#define mmCURSOR0_3_CURSOR_STEREO_CONTROL 0x0912
+#define mmCURSOR0_3_CURSOR_STEREO_CONTROL_BASE_IDX 2
+#define mmCURSOR0_3_CURSOR_DST_OFFSET 0x0913
+#define mmCURSOR0_3_CURSOR_DST_OFFSET_BASE_IDX 2
+#define mmCURSOR0_3_CURSOR_MEM_PWR_CTRL 0x0914
+#define mmCURSOR0_3_CURSOR_MEM_PWR_CTRL_BASE_IDX 2
+#define mmCURSOR0_3_CURSOR_MEM_PWR_STATUS 0x0915
+#define mmCURSOR0_3_CURSOR_MEM_PWR_STATUS_BASE_IDX 2
+#define mmCURSOR0_3_DMDATA_ADDRESS_HIGH 0x0916
+#define mmCURSOR0_3_DMDATA_ADDRESS_HIGH_BASE_IDX 2
+#define mmCURSOR0_3_DMDATA_ADDRESS_LOW 0x0917
+#define mmCURSOR0_3_DMDATA_ADDRESS_LOW_BASE_IDX 2
+#define mmCURSOR0_3_DMDATA_CNTL 0x0918
+#define mmCURSOR0_3_DMDATA_CNTL_BASE_IDX 2
+#define mmCURSOR0_3_DMDATA_QOS_CNTL 0x0919
+#define mmCURSOR0_3_DMDATA_QOS_CNTL_BASE_IDX 2
+#define mmCURSOR0_3_DMDATA_STATUS 0x091a
+#define mmCURSOR0_3_DMDATA_STATUS_BASE_IDX 2
+#define mmCURSOR0_3_DMDATA_SW_CNTL 0x091b
+#define mmCURSOR0_3_DMDATA_SW_CNTL_BASE_IDX 2
+#define mmCURSOR0_3_DMDATA_SW_DATA 0x091c
+#define mmCURSOR0_3_DMDATA_SW_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp3_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x24c4
+#define mmDC_PERFMON10_PERFCOUNTER_CNTL 0x0931
+#define mmDC_PERFMON10_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON10_PERFCOUNTER_CNTL2 0x0932
+#define mmDC_PERFMON10_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON10_PERFCOUNTER_STATE 0x0933
+#define mmDC_PERFMON10_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON10_PERFMON_CNTL 0x0934
+#define mmDC_PERFMON10_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON10_PERFMON_CNTL2 0x0935
+#define mmDC_PERFMON10_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON10_PERFMON_CVALUE_INT_MISC 0x0936
+#define mmDC_PERFMON10_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON10_PERFMON_CVALUE_LOW 0x0937
+#define mmDC_PERFMON10_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON10_PERFMON_HI 0x0938
+#define mmDC_PERFMON10_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON10_PERFMON_LOW 0x0939
+#define mmDC_PERFMON10_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp3_dispdec_hubpxfc_dispdec
+// base address: 0xa50
+#define mmHUBPXFC3_HUBP_XFC_CNTL 0x093d
+#define mmHUBPXFC3_HUBP_XFC_CNTL_BASE_IDX 2
+#define mmHUBPXFC3_HUBP_XFC_XBUF_RD_BASE0_ADDR_LSB 0x093e
+#define mmHUBPXFC3_HUBP_XFC_XBUF_RD_BASE0_ADDR_LSB_BASE_IDX 2
+#define mmHUBPXFC3_HUBP_XFC_XBUF_RD_BASE0_ADDR_MSB 0x093f
+#define mmHUBPXFC3_HUBP_XFC_XBUF_RD_BASE0_ADDR_MSB_BASE_IDX 2
+#define mmHUBPXFC3_HUBP_XFC_XBUF_RD_BASE1_ADDR_LSB 0x0940
+#define mmHUBPXFC3_HUBP_XFC_XBUF_RD_BASE1_ADDR_LSB_BASE_IDX 2
+#define mmHUBPXFC3_HUBP_XFC_XBUF_RD_BASE1_ADDR_MSB 0x0941
+#define mmHUBPXFC3_HUBP_XFC_XBUF_RD_BASE1_ADDR_MSB_BASE_IDX 2
+#define mmHUBPXFC3_HUBP_XFC_XBUF_RD_PITCH 0x0942
+#define mmHUBPXFC3_HUBP_XFC_XBUF_RD_PITCH_BASE_IDX 2
+#define mmHUBPXFC3_HUBP_XFC_DELAY_CONFIG0 0x0943
+#define mmHUBPXFC3_HUBP_XFC_DELAY_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC3_HUBP_XFC_DELAY_CONFIG1 0x0944
+#define mmHUBPXFC3_HUBP_XFC_DELAY_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC3_HUBP_XFC_DELAY_CONFIG2 0x0945
+#define mmHUBPXFC3_HUBP_XFC_DELAY_CONFIG2_BASE_IDX 2
+#define mmHUBPXFC3_HUBP_XFC_UNDERFLOW_STATUS 0x0946
+#define mmHUBPXFC3_HUBP_XFC_UNDERFLOW_STATUS_BASE_IDX 2
+#define mmHUBPXFC3_HUBP_XFC_SLV_VTG_CONFIG0 0x0947
+#define mmHUBPXFC3_HUBP_XFC_SLV_VTG_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC3_HUBP_XFC_SLV_VTG_CONFIG1 0x0948
+#define mmHUBPXFC3_HUBP_XFC_SLV_VTG_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC3_HUBP_XFC_SLV_SCALER_CONFIG0 0x0949
+#define mmHUBPXFC3_HUBP_XFC_SLV_SCALER_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC3_HUBP_XFC_SLV_SCALER_CONFIG1 0x094a
+#define mmHUBPXFC3_HUBP_XFC_SLV_SCALER_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC3_HUBP_XFC_MPC_CONFIG 0x094b
+#define mmHUBPXFC3_HUBP_XFC_MPC_CONFIG_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp4_dispdec_hubp_dispdec
+// base address: 0xdc0
+#define mmHUBP4_DCSURF_SURFACE_CONFIG 0x0955
+#define mmHUBP4_DCSURF_SURFACE_CONFIG_BASE_IDX 2
+#define mmHUBP4_DCSURF_ADDR_CONFIG 0x0956
+#define mmHUBP4_DCSURF_ADDR_CONFIG_BASE_IDX 2
+#define mmHUBP4_DCSURF_TILING_CONFIG 0x0957
+#define mmHUBP4_DCSURF_TILING_CONFIG_BASE_IDX 2
+#define mmHUBP4_DCSURF_PRI_VIEWPORT_START 0x0959
+#define mmHUBP4_DCSURF_PRI_VIEWPORT_START_BASE_IDX 2
+#define mmHUBP4_DCSURF_PRI_VIEWPORT_DIMENSION 0x095a
+#define mmHUBP4_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2
+#define mmHUBP4_DCSURF_PRI_VIEWPORT_START_C 0x095b
+#define mmHUBP4_DCSURF_PRI_VIEWPORT_START_C_BASE_IDX 2
+#define mmHUBP4_DCSURF_PRI_VIEWPORT_DIMENSION_C 0x095c
+#define mmHUBP4_DCSURF_PRI_VIEWPORT_DIMENSION_C_BASE_IDX 2
+#define mmHUBP4_DCSURF_SEC_VIEWPORT_START 0x095d
+#define mmHUBP4_DCSURF_SEC_VIEWPORT_START_BASE_IDX 2
+#define mmHUBP4_DCSURF_SEC_VIEWPORT_DIMENSION 0x095e
+#define mmHUBP4_DCSURF_SEC_VIEWPORT_DIMENSION_BASE_IDX 2
+#define mmHUBP4_DCSURF_SEC_VIEWPORT_START_C 0x095f
+#define mmHUBP4_DCSURF_SEC_VIEWPORT_START_C_BASE_IDX 2
+#define mmHUBP4_DCSURF_SEC_VIEWPORT_DIMENSION_C 0x0960
+#define mmHUBP4_DCSURF_SEC_VIEWPORT_DIMENSION_C_BASE_IDX 2
+#define mmHUBP4_DCHUBP_REQ_SIZE_CONFIG 0x0961
+#define mmHUBP4_DCHUBP_REQ_SIZE_CONFIG_BASE_IDX 2
+#define mmHUBP4_DCHUBP_REQ_SIZE_CONFIG_C 0x0962
+#define mmHUBP4_DCHUBP_REQ_SIZE_CONFIG_C_BASE_IDX 2
+#define mmHUBP4_DCHUBP_CNTL 0x0963
+#define mmHUBP4_DCHUBP_CNTL_BASE_IDX 2
+#define mmHUBP4_HUBP_CLK_CNTL 0x0964
+#define mmHUBP4_HUBP_CLK_CNTL_BASE_IDX 2
+#define mmHUBP4_DCHUBP_VMPG_CONFIG 0x0965
+#define mmHUBP4_DCHUBP_VMPG_CONFIG_BASE_IDX 2
+#define mmHUBP4_HUBPREQ_DEBUG_DB 0x0966
+#define mmHUBP4_HUBPREQ_DEBUG_DB_BASE_IDX 2
+#define mmHUBP4_HUBPREQ_DEBUG 0x0967
+#define mmHUBP4_HUBPREQ_DEBUG_BASE_IDX 2
+#define mmHUBP4_HUBP_MEASURE_WIN_CTRL_DCFCLK 0x096b
+#define mmHUBP4_HUBP_MEASURE_WIN_CTRL_DCFCLK_BASE_IDX 2
+#define mmHUBP4_HUBP_MEASURE_WIN_CTRL_DPPCLK 0x096c
+#define mmHUBP4_HUBP_MEASURE_WIN_CTRL_DPPCLK_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp4_dispdec_hubpreq_dispdec
+// base address: 0xdc0
+#define mmHUBPREQ4_DCSURF_SURFACE_PITCH 0x0977
+#define mmHUBPREQ4_DCSURF_SURFACE_PITCH_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SURFACE_PITCH_C 0x0978
+#define mmHUBPREQ4_DCSURF_SURFACE_PITCH_C_BASE_IDX 2
+#define mmHUBPREQ4_VMID_SETTINGS_0 0x0979
+#define mmHUBPREQ4_VMID_SETTINGS_0_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_PRIMARY_SURFACE_ADDRESS 0x097a
+#define mmHUBPREQ4_DCSURF_PRIMARY_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH 0x097b
+#define mmHUBPREQ4_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_PRIMARY_SURFACE_ADDRESS_C 0x097c
+#define mmHUBPREQ4_DCSURF_PRIMARY_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C 0x097d
+#define mmHUBPREQ4_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SECONDARY_SURFACE_ADDRESS 0x097e
+#define mmHUBPREQ4_DCSURF_SECONDARY_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH 0x097f
+#define mmHUBPREQ4_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SECONDARY_SURFACE_ADDRESS_C 0x0980
+#define mmHUBPREQ4_DCSURF_SECONDARY_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C 0x0981
+#define mmHUBPREQ4_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_PRIMARY_META_SURFACE_ADDRESS 0x0982
+#define mmHUBPREQ4_DCSURF_PRIMARY_META_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH 0x0983
+#define mmHUBPREQ4_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C 0x0984
+#define mmHUBPREQ4_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C 0x0985
+#define mmHUBPREQ4_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SECONDARY_META_SURFACE_ADDRESS 0x0986
+#define mmHUBPREQ4_DCSURF_SECONDARY_META_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH 0x0987
+#define mmHUBPREQ4_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C 0x0988
+#define mmHUBPREQ4_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C 0x0989
+#define mmHUBPREQ4_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SURFACE_CONTROL 0x098a
+#define mmHUBPREQ4_DCSURF_SURFACE_CONTROL_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_FLIP_CONTROL 0x098b
+#define mmHUBPREQ4_DCSURF_FLIP_CONTROL_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_FLIP_CONTROL2 0x098c
+#define mmHUBPREQ4_DCSURF_FLIP_CONTROL2_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_QUEUE_CONTROL 0x098d
+#define mmHUBPREQ4_DCSURF_QUEUE_CONTROL_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_FRAME_PACING_TIME 0x098e
+#define mmHUBPREQ4_DCSURF_FRAME_PACING_TIME_BASE_IDX 2
+#define mmHUBPREQ4_SURFACE_CURRENT_PACING_COUNTER 0x098f
+#define mmHUBPREQ4_SURFACE_CURRENT_PACING_COUNTER_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SURFACE_FLIP_INTERRUPT 0x0990
+#define mmHUBPREQ4_DCSURF_SURFACE_FLIP_INTERRUPT_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SURFACE_INUSE 0x0991
+#define mmHUBPREQ4_DCSURF_SURFACE_INUSE_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SURFACE_INUSE_HIGH 0x0992
+#define mmHUBPREQ4_DCSURF_SURFACE_INUSE_HIGH_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SURFACE_INUSE_C 0x0993
+#define mmHUBPREQ4_DCSURF_SURFACE_INUSE_C_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SURFACE_INUSE_HIGH_C 0x0994
+#define mmHUBPREQ4_DCSURF_SURFACE_INUSE_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SURFACE_EARLIEST_INUSE 0x0995
+#define mmHUBPREQ4_DCSURF_SURFACE_EARLIEST_INUSE_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SURFACE_EARLIEST_INUSE_HIGH 0x0996
+#define mmHUBPREQ4_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SURFACE_EARLIEST_INUSE_C 0x0997
+#define mmHUBPREQ4_DCSURF_SURFACE_EARLIEST_INUSE_C_BASE_IDX 2
+#define mmHUBPREQ4_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C 0x0998
+#define mmHUBPREQ4_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ4_DCN_EXPANSION_MODE 0x099c
+#define mmHUBPREQ4_DCN_EXPANSION_MODE_BASE_IDX 2
+#define mmHUBPREQ4_DCN_TTU_QOS_WM 0x099d
+#define mmHUBPREQ4_DCN_TTU_QOS_WM_BASE_IDX 2
+#define mmHUBPREQ4_DCN_GLOBAL_TTU_CNTL 0x099e
+#define mmHUBPREQ4_DCN_GLOBAL_TTU_CNTL_BASE_IDX 2
+#define mmHUBPREQ4_DCN_SURF0_TTU_CNTL0 0x099f
+#define mmHUBPREQ4_DCN_SURF0_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ4_DCN_SURF0_TTU_CNTL1 0x09a0
+#define mmHUBPREQ4_DCN_SURF0_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ4_DCN_SURF1_TTU_CNTL0 0x09a1
+#define mmHUBPREQ4_DCN_SURF1_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ4_DCN_SURF1_TTU_CNTL1 0x09a2
+#define mmHUBPREQ4_DCN_SURF1_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ4_DCN_CUR0_TTU_CNTL0 0x09a3
+#define mmHUBPREQ4_DCN_CUR0_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ4_DCN_CUR0_TTU_CNTL1 0x09a4
+#define mmHUBPREQ4_DCN_CUR0_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ4_DCN_CUR1_TTU_CNTL0 0x09a5
+#define mmHUBPREQ4_DCN_CUR1_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ4_DCN_CUR1_TTU_CNTL1 0x09a6
+#define mmHUBPREQ4_DCN_CUR1_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ4_DCN_VM_SYSTEM_APERTURE_LOW_ADDR 0x09a7
+#define mmHUBPREQ4_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_BASE_IDX 2
+#define mmHUBPREQ4_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR 0x09a8
+#define mmHUBPREQ4_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_BASE_IDX 2
+#define mmHUBPREQ4_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB 0x09a9
+#define mmHUBPREQ4_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ4_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB 0x09aa
+#define mmHUBPREQ4_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB 0x09ab
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB 0x09ac
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB 0x09ad
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB 0x09ae
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB 0x09af
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB 0x09b0
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB 0x09b1
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB 0x09b2
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PROTECTION_FAULT_ADDR_LSB 0x09b3
+#define mmHUBPREQ4_DC_VM_CONTEXT0_PROTECTION_FAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ4_DC_VM_CONTEXT0_CNTL 0x09b4
+#define mmHUBPREQ4_DC_VM_CONTEXT0_CNTL_BASE_IDX 2
+#define mmHUBPREQ4_DCN_VM_MX_L1_TLB_CNTL 0x09b5
+#define mmHUBPREQ4_DCN_VM_MX_L1_TLB_CNTL_BASE_IDX 2
+#define mmHUBPREQ4_BLANK_OFFSET_0 0x09b6
+#define mmHUBPREQ4_BLANK_OFFSET_0_BASE_IDX 2
+#define mmHUBPREQ4_BLANK_OFFSET_1 0x09b7
+#define mmHUBPREQ4_BLANK_OFFSET_1_BASE_IDX 2
+#define mmHUBPREQ4_DST_DIMENSIONS 0x09b8
+#define mmHUBPREQ4_DST_DIMENSIONS_BASE_IDX 2
+#define mmHUBPREQ4_DST_AFTER_SCALER 0x09b9
+#define mmHUBPREQ4_DST_AFTER_SCALER_BASE_IDX 2
+#define mmHUBPREQ4_PREFETCH_SETTINGS 0x09ba
+#define mmHUBPREQ4_PREFETCH_SETTINGS_BASE_IDX 2
+#define mmHUBPREQ4_PREFETCH_SETTINGS_C 0x09bb
+#define mmHUBPREQ4_PREFETCH_SETTINGS_C_BASE_IDX 2
+#define mmHUBPREQ4_VBLANK_PARAMETERS_0 0x09bc
+#define mmHUBPREQ4_VBLANK_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ4_VBLANK_PARAMETERS_1 0x09bd
+#define mmHUBPREQ4_VBLANK_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ4_VBLANK_PARAMETERS_2 0x09be
+#define mmHUBPREQ4_VBLANK_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ4_VBLANK_PARAMETERS_3 0x09bf
+#define mmHUBPREQ4_VBLANK_PARAMETERS_3_BASE_IDX 2
+#define mmHUBPREQ4_VBLANK_PARAMETERS_4 0x09c0
+#define mmHUBPREQ4_VBLANK_PARAMETERS_4_BASE_IDX 2
+#define mmHUBPREQ4_FLIP_PARAMETERS_0 0x09c1
+#define mmHUBPREQ4_FLIP_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ4_FLIP_PARAMETERS_1 0x09c2
+#define mmHUBPREQ4_FLIP_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ4_FLIP_PARAMETERS_2 0x09c3
+#define mmHUBPREQ4_FLIP_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ4_NOM_PARAMETERS_0 0x09c4
+#define mmHUBPREQ4_NOM_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ4_NOM_PARAMETERS_1 0x09c5
+#define mmHUBPREQ4_NOM_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ4_NOM_PARAMETERS_2 0x09c6
+#define mmHUBPREQ4_NOM_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ4_NOM_PARAMETERS_3 0x09c7
+#define mmHUBPREQ4_NOM_PARAMETERS_3_BASE_IDX 2
+#define mmHUBPREQ4_NOM_PARAMETERS_4 0x09c8
+#define mmHUBPREQ4_NOM_PARAMETERS_4_BASE_IDX 2
+#define mmHUBPREQ4_NOM_PARAMETERS_5 0x09c9
+#define mmHUBPREQ4_NOM_PARAMETERS_5_BASE_IDX 2
+#define mmHUBPREQ4_NOM_PARAMETERS_6 0x09ca
+#define mmHUBPREQ4_NOM_PARAMETERS_6_BASE_IDX 2
+#define mmHUBPREQ4_NOM_PARAMETERS_7 0x09cb
+#define mmHUBPREQ4_NOM_PARAMETERS_7_BASE_IDX 2
+#define mmHUBPREQ4_PER_LINE_DELIVERY_PRE 0x09cc
+#define mmHUBPREQ4_PER_LINE_DELIVERY_PRE_BASE_IDX 2
+#define mmHUBPREQ4_PER_LINE_DELIVERY 0x09cd
+#define mmHUBPREQ4_PER_LINE_DELIVERY_BASE_IDX 2
+#define mmHUBPREQ4_CURSOR_SETTINGS 0x09ce
+#define mmHUBPREQ4_CURSOR_SETTINGS_BASE_IDX 2
+#define mmHUBPREQ4_REF_FREQ_TO_PIX_FREQ 0x09cf
+#define mmHUBPREQ4_REF_FREQ_TO_PIX_FREQ_BASE_IDX 2
+#define mmHUBPREQ4_DST_Y_DELTA_DRQ_LIMIT 0x09d0
+#define mmHUBPREQ4_DST_Y_DELTA_DRQ_LIMIT_BASE_IDX 2
+#define mmHUBPREQ4_HUBPREQ_MEM_PWR_CTRL 0x09d1
+#define mmHUBPREQ4_HUBPREQ_MEM_PWR_CTRL_BASE_IDX 2
+#define mmHUBPREQ4_HUBPREQ_MEM_PWR_STATUS 0x09d2
+#define mmHUBPREQ4_HUBPREQ_MEM_PWR_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp4_dispdec_hubpret_dispdec
+// base address: 0xdc0
+#define mmHUBPRET4_HUBPRET_CONTROL 0x09dc
+#define mmHUBPRET4_HUBPRET_CONTROL_BASE_IDX 2
+#define mmHUBPRET4_HUBPRET_MEM_PWR_CTRL 0x09dd
+#define mmHUBPRET4_HUBPRET_MEM_PWR_CTRL_BASE_IDX 2
+#define mmHUBPRET4_HUBPRET_MEM_PWR_STATUS 0x09de
+#define mmHUBPRET4_HUBPRET_MEM_PWR_STATUS_BASE_IDX 2
+#define mmHUBPRET4_HUBPRET_READ_LINE_CTRL0 0x09df
+#define mmHUBPRET4_HUBPRET_READ_LINE_CTRL0_BASE_IDX 2
+#define mmHUBPRET4_HUBPRET_READ_LINE_CTRL1 0x09e0
+#define mmHUBPRET4_HUBPRET_READ_LINE_CTRL1_BASE_IDX 2
+#define mmHUBPRET4_HUBPRET_READ_LINE0 0x09e1
+#define mmHUBPRET4_HUBPRET_READ_LINE0_BASE_IDX 2
+#define mmHUBPRET4_HUBPRET_READ_LINE1 0x09e2
+#define mmHUBPRET4_HUBPRET_READ_LINE1_BASE_IDX 2
+#define mmHUBPRET4_HUBPRET_INTERRUPT 0x09e3
+#define mmHUBPRET4_HUBPRET_INTERRUPT_BASE_IDX 2
+#define mmHUBPRET4_HUBPRET_READ_LINE_VALUE 0x09e4
+#define mmHUBPRET4_HUBPRET_READ_LINE_VALUE_BASE_IDX 2
+#define mmHUBPRET4_HUBPRET_READ_LINE_STATUS 0x09e5
+#define mmHUBPRET4_HUBPRET_READ_LINE_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp4_dispdec_cursor0_dispdec
+// base address: 0xdc0
+#define mmCURSOR0_4_CURSOR_CONTROL 0x09e8
+#define mmCURSOR0_4_CURSOR_CONTROL_BASE_IDX 2
+#define mmCURSOR0_4_CURSOR_SURFACE_ADDRESS 0x09e9
+#define mmCURSOR0_4_CURSOR_SURFACE_ADDRESS_BASE_IDX 2
+#define mmCURSOR0_4_CURSOR_SURFACE_ADDRESS_HIGH 0x09ea
+#define mmCURSOR0_4_CURSOR_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmCURSOR0_4_CURSOR_SIZE 0x09eb
+#define mmCURSOR0_4_CURSOR_SIZE_BASE_IDX 2
+#define mmCURSOR0_4_CURSOR_POSITION 0x09ec
+#define mmCURSOR0_4_CURSOR_POSITION_BASE_IDX 2
+#define mmCURSOR0_4_CURSOR_HOT_SPOT 0x09ed
+#define mmCURSOR0_4_CURSOR_HOT_SPOT_BASE_IDX 2
+#define mmCURSOR0_4_CURSOR_STEREO_CONTROL 0x09ee
+#define mmCURSOR0_4_CURSOR_STEREO_CONTROL_BASE_IDX 2
+#define mmCURSOR0_4_CURSOR_DST_OFFSET 0x09ef
+#define mmCURSOR0_4_CURSOR_DST_OFFSET_BASE_IDX 2
+#define mmCURSOR0_4_CURSOR_MEM_PWR_CTRL 0x09f0
+#define mmCURSOR0_4_CURSOR_MEM_PWR_CTRL_BASE_IDX 2
+#define mmCURSOR0_4_CURSOR_MEM_PWR_STATUS 0x09f1
+#define mmCURSOR0_4_CURSOR_MEM_PWR_STATUS_BASE_IDX 2
+#define mmCURSOR0_4_DMDATA_ADDRESS_HIGH 0x09f2
+#define mmCURSOR0_4_DMDATA_ADDRESS_HIGH_BASE_IDX 2
+#define mmCURSOR0_4_DMDATA_ADDRESS_LOW 0x09f3
+#define mmCURSOR0_4_DMDATA_ADDRESS_LOW_BASE_IDX 2
+#define mmCURSOR0_4_DMDATA_CNTL 0x09f4
+#define mmCURSOR0_4_DMDATA_CNTL_BASE_IDX 2
+#define mmCURSOR0_4_DMDATA_QOS_CNTL 0x09f5
+#define mmCURSOR0_4_DMDATA_QOS_CNTL_BASE_IDX 2
+#define mmCURSOR0_4_DMDATA_STATUS 0x09f6
+#define mmCURSOR0_4_DMDATA_STATUS_BASE_IDX 2
+#define mmCURSOR0_4_DMDATA_SW_CNTL 0x09f7
+#define mmCURSOR0_4_DMDATA_SW_CNTL_BASE_IDX 2
+#define mmCURSOR0_4_DMDATA_SW_DATA 0x09f8
+#define mmCURSOR0_4_DMDATA_SW_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp4_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x2834
+#define mmDC_PERFMON11_PERFCOUNTER_CNTL 0x0a0d
+#define mmDC_PERFMON11_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON11_PERFCOUNTER_CNTL2 0x0a0e
+#define mmDC_PERFMON11_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON11_PERFCOUNTER_STATE 0x0a0f
+#define mmDC_PERFMON11_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON11_PERFMON_CNTL 0x0a10
+#define mmDC_PERFMON11_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON11_PERFMON_CNTL2 0x0a11
+#define mmDC_PERFMON11_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON11_PERFMON_CVALUE_INT_MISC 0x0a12
+#define mmDC_PERFMON11_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON11_PERFMON_CVALUE_LOW 0x0a13
+#define mmDC_PERFMON11_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON11_PERFMON_HI 0x0a14
+#define mmDC_PERFMON11_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON11_PERFMON_LOW 0x0a15
+#define mmDC_PERFMON11_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp4_dispdec_hubpxfc_dispdec
+// base address: 0xdc0
+#define mmHUBPXFC4_HUBP_XFC_CNTL 0x0a19
+#define mmHUBPXFC4_HUBP_XFC_CNTL_BASE_IDX 2
+#define mmHUBPXFC4_HUBP_XFC_XBUF_RD_BASE0_ADDR_LSB 0x0a1a
+#define mmHUBPXFC4_HUBP_XFC_XBUF_RD_BASE0_ADDR_LSB_BASE_IDX 2
+#define mmHUBPXFC4_HUBP_XFC_XBUF_RD_BASE0_ADDR_MSB 0x0a1b
+#define mmHUBPXFC4_HUBP_XFC_XBUF_RD_BASE0_ADDR_MSB_BASE_IDX 2
+#define mmHUBPXFC4_HUBP_XFC_XBUF_RD_BASE1_ADDR_LSB 0x0a1c
+#define mmHUBPXFC4_HUBP_XFC_XBUF_RD_BASE1_ADDR_LSB_BASE_IDX 2
+#define mmHUBPXFC4_HUBP_XFC_XBUF_RD_BASE1_ADDR_MSB 0x0a1d
+#define mmHUBPXFC4_HUBP_XFC_XBUF_RD_BASE1_ADDR_MSB_BASE_IDX 2
+#define mmHUBPXFC4_HUBP_XFC_XBUF_RD_PITCH 0x0a1e
+#define mmHUBPXFC4_HUBP_XFC_XBUF_RD_PITCH_BASE_IDX 2
+#define mmHUBPXFC4_HUBP_XFC_DELAY_CONFIG0 0x0a1f
+#define mmHUBPXFC4_HUBP_XFC_DELAY_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC4_HUBP_XFC_DELAY_CONFIG1 0x0a20
+#define mmHUBPXFC4_HUBP_XFC_DELAY_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC4_HUBP_XFC_DELAY_CONFIG2 0x0a21
+#define mmHUBPXFC4_HUBP_XFC_DELAY_CONFIG2_BASE_IDX 2
+#define mmHUBPXFC4_HUBP_XFC_UNDERFLOW_STATUS 0x0a22
+#define mmHUBPXFC4_HUBP_XFC_UNDERFLOW_STATUS_BASE_IDX 2
+#define mmHUBPXFC4_HUBP_XFC_SLV_VTG_CONFIG0 0x0a23
+#define mmHUBPXFC4_HUBP_XFC_SLV_VTG_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC4_HUBP_XFC_SLV_VTG_CONFIG1 0x0a24
+#define mmHUBPXFC4_HUBP_XFC_SLV_VTG_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC4_HUBP_XFC_SLV_SCALER_CONFIG0 0x0a25
+#define mmHUBPXFC4_HUBP_XFC_SLV_SCALER_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC4_HUBP_XFC_SLV_SCALER_CONFIG1 0x0a26
+#define mmHUBPXFC4_HUBP_XFC_SLV_SCALER_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC4_HUBP_XFC_MPC_CONFIG 0x0a27
+#define mmHUBPXFC4_HUBP_XFC_MPC_CONFIG_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp5_dispdec_hubp_dispdec
+// base address: 0x1130
+#define mmHUBP5_DCSURF_SURFACE_CONFIG 0x0a31
+#define mmHUBP5_DCSURF_SURFACE_CONFIG_BASE_IDX 2
+#define mmHUBP5_DCSURF_ADDR_CONFIG 0x0a32
+#define mmHUBP5_DCSURF_ADDR_CONFIG_BASE_IDX 2
+#define mmHUBP5_DCSURF_TILING_CONFIG 0x0a33
+#define mmHUBP5_DCSURF_TILING_CONFIG_BASE_IDX 2
+#define mmHUBP5_DCSURF_PRI_VIEWPORT_START 0x0a35
+#define mmHUBP5_DCSURF_PRI_VIEWPORT_START_BASE_IDX 2
+#define mmHUBP5_DCSURF_PRI_VIEWPORT_DIMENSION 0x0a36
+#define mmHUBP5_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2
+#define mmHUBP5_DCSURF_PRI_VIEWPORT_START_C 0x0a37
+#define mmHUBP5_DCSURF_PRI_VIEWPORT_START_C_BASE_IDX 2
+#define mmHUBP5_DCSURF_PRI_VIEWPORT_DIMENSION_C 0x0a38
+#define mmHUBP5_DCSURF_PRI_VIEWPORT_DIMENSION_C_BASE_IDX 2
+#define mmHUBP5_DCSURF_SEC_VIEWPORT_START 0x0a39
+#define mmHUBP5_DCSURF_SEC_VIEWPORT_START_BASE_IDX 2
+#define mmHUBP5_DCSURF_SEC_VIEWPORT_DIMENSION 0x0a3a
+#define mmHUBP5_DCSURF_SEC_VIEWPORT_DIMENSION_BASE_IDX 2
+#define mmHUBP5_DCSURF_SEC_VIEWPORT_START_C 0x0a3b
+#define mmHUBP5_DCSURF_SEC_VIEWPORT_START_C_BASE_IDX 2
+#define mmHUBP5_DCSURF_SEC_VIEWPORT_DIMENSION_C 0x0a3c
+#define mmHUBP5_DCSURF_SEC_VIEWPORT_DIMENSION_C_BASE_IDX 2
+#define mmHUBP5_DCHUBP_REQ_SIZE_CONFIG 0x0a3d
+#define mmHUBP5_DCHUBP_REQ_SIZE_CONFIG_BASE_IDX 2
+#define mmHUBP5_DCHUBP_REQ_SIZE_CONFIG_C 0x0a3e
+#define mmHUBP5_DCHUBP_REQ_SIZE_CONFIG_C_BASE_IDX 2
+#define mmHUBP5_DCHUBP_CNTL 0x0a3f
+#define mmHUBP5_DCHUBP_CNTL_BASE_IDX 2
+#define mmHUBP5_HUBP_CLK_CNTL 0x0a40
+#define mmHUBP5_HUBP_CLK_CNTL_BASE_IDX 2
+#define mmHUBP5_DCHUBP_VMPG_CONFIG 0x0a41
+#define mmHUBP5_DCHUBP_VMPG_CONFIG_BASE_IDX 2
+#define mmHUBP5_HUBPREQ_DEBUG_DB 0x0a42
+#define mmHUBP5_HUBPREQ_DEBUG_DB_BASE_IDX 2
+#define mmHUBP5_HUBPREQ_DEBUG 0x0a43
+#define mmHUBP5_HUBPREQ_DEBUG_BASE_IDX 2
+#define mmHUBP5_HUBP_MEASURE_WIN_CTRL_DCFCLK 0x0a47
+#define mmHUBP5_HUBP_MEASURE_WIN_CTRL_DCFCLK_BASE_IDX 2
+#define mmHUBP5_HUBP_MEASURE_WIN_CTRL_DPPCLK 0x0a48
+#define mmHUBP5_HUBP_MEASURE_WIN_CTRL_DPPCLK_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp5_dispdec_hubpreq_dispdec
+// base address: 0x1130
+#define mmHUBPREQ5_DCSURF_SURFACE_PITCH 0x0a53
+#define mmHUBPREQ5_DCSURF_SURFACE_PITCH_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SURFACE_PITCH_C 0x0a54
+#define mmHUBPREQ5_DCSURF_SURFACE_PITCH_C_BASE_IDX 2
+#define mmHUBPREQ5_VMID_SETTINGS_0 0x0a55
+#define mmHUBPREQ5_VMID_SETTINGS_0_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_PRIMARY_SURFACE_ADDRESS 0x0a56
+#define mmHUBPREQ5_DCSURF_PRIMARY_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH 0x0a57
+#define mmHUBPREQ5_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_PRIMARY_SURFACE_ADDRESS_C 0x0a58
+#define mmHUBPREQ5_DCSURF_PRIMARY_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C 0x0a59
+#define mmHUBPREQ5_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SECONDARY_SURFACE_ADDRESS 0x0a5a
+#define mmHUBPREQ5_DCSURF_SECONDARY_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH 0x0a5b
+#define mmHUBPREQ5_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SECONDARY_SURFACE_ADDRESS_C 0x0a5c
+#define mmHUBPREQ5_DCSURF_SECONDARY_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C 0x0a5d
+#define mmHUBPREQ5_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_PRIMARY_META_SURFACE_ADDRESS 0x0a5e
+#define mmHUBPREQ5_DCSURF_PRIMARY_META_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH 0x0a5f
+#define mmHUBPREQ5_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C 0x0a60
+#define mmHUBPREQ5_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C 0x0a61
+#define mmHUBPREQ5_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SECONDARY_META_SURFACE_ADDRESS 0x0a62
+#define mmHUBPREQ5_DCSURF_SECONDARY_META_SURFACE_ADDRESS_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH 0x0a63
+#define mmHUBPREQ5_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C 0x0a64
+#define mmHUBPREQ5_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C 0x0a65
+#define mmHUBPREQ5_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SURFACE_CONTROL 0x0a66
+#define mmHUBPREQ5_DCSURF_SURFACE_CONTROL_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_FLIP_CONTROL 0x0a67
+#define mmHUBPREQ5_DCSURF_FLIP_CONTROL_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_FLIP_CONTROL2 0x0a68
+#define mmHUBPREQ5_DCSURF_FLIP_CONTROL2_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_QUEUE_CONTROL 0x0a69
+#define mmHUBPREQ5_DCSURF_QUEUE_CONTROL_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_FRAME_PACING_TIME 0x0a6a
+#define mmHUBPREQ5_DCSURF_FRAME_PACING_TIME_BASE_IDX 2
+#define mmHUBPREQ5_SURFACE_CURRENT_PACING_COUNTER 0x0a6b
+#define mmHUBPREQ5_SURFACE_CURRENT_PACING_COUNTER_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SURFACE_FLIP_INTERRUPT 0x0a6c
+#define mmHUBPREQ5_DCSURF_SURFACE_FLIP_INTERRUPT_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SURFACE_INUSE 0x0a6d
+#define mmHUBPREQ5_DCSURF_SURFACE_INUSE_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SURFACE_INUSE_HIGH 0x0a6e
+#define mmHUBPREQ5_DCSURF_SURFACE_INUSE_HIGH_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SURFACE_INUSE_C 0x0a6f
+#define mmHUBPREQ5_DCSURF_SURFACE_INUSE_C_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SURFACE_INUSE_HIGH_C 0x0a70
+#define mmHUBPREQ5_DCSURF_SURFACE_INUSE_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SURFACE_EARLIEST_INUSE 0x0a71
+#define mmHUBPREQ5_DCSURF_SURFACE_EARLIEST_INUSE_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SURFACE_EARLIEST_INUSE_HIGH 0x0a72
+#define mmHUBPREQ5_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SURFACE_EARLIEST_INUSE_C 0x0a73
+#define mmHUBPREQ5_DCSURF_SURFACE_EARLIEST_INUSE_C_BASE_IDX 2
+#define mmHUBPREQ5_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C 0x0a74
+#define mmHUBPREQ5_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C_BASE_IDX 2
+#define mmHUBPREQ5_DCN_EXPANSION_MODE 0x0a78
+#define mmHUBPREQ5_DCN_EXPANSION_MODE_BASE_IDX 2
+#define mmHUBPREQ5_DCN_TTU_QOS_WM 0x0a79
+#define mmHUBPREQ5_DCN_TTU_QOS_WM_BASE_IDX 2
+#define mmHUBPREQ5_DCN_GLOBAL_TTU_CNTL 0x0a7a
+#define mmHUBPREQ5_DCN_GLOBAL_TTU_CNTL_BASE_IDX 2
+#define mmHUBPREQ5_DCN_SURF0_TTU_CNTL0 0x0a7b
+#define mmHUBPREQ5_DCN_SURF0_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ5_DCN_SURF0_TTU_CNTL1 0x0a7c
+#define mmHUBPREQ5_DCN_SURF0_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ5_DCN_SURF1_TTU_CNTL0 0x0a7d
+#define mmHUBPREQ5_DCN_SURF1_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ5_DCN_SURF1_TTU_CNTL1 0x0a7e
+#define mmHUBPREQ5_DCN_SURF1_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ5_DCN_CUR0_TTU_CNTL0 0x0a7f
+#define mmHUBPREQ5_DCN_CUR0_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ5_DCN_CUR0_TTU_CNTL1 0x0a80
+#define mmHUBPREQ5_DCN_CUR0_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ5_DCN_CUR1_TTU_CNTL0 0x0a81
+#define mmHUBPREQ5_DCN_CUR1_TTU_CNTL0_BASE_IDX 2
+#define mmHUBPREQ5_DCN_CUR1_TTU_CNTL1 0x0a82
+#define mmHUBPREQ5_DCN_CUR1_TTU_CNTL1_BASE_IDX 2
+#define mmHUBPREQ5_DCN_VM_SYSTEM_APERTURE_LOW_ADDR 0x0a83
+#define mmHUBPREQ5_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_BASE_IDX 2
+#define mmHUBPREQ5_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR 0x0a84
+#define mmHUBPREQ5_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_BASE_IDX 2
+#define mmHUBPREQ5_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB 0x0a85
+#define mmHUBPREQ5_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ5_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB 0x0a86
+#define mmHUBPREQ5_DC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB 0x0a87
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB 0x0a88
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB 0x0a89
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB 0x0a8a
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB 0x0a8b
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB 0x0a8c
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB 0x0a8d
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB 0x0a8e
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB_BASE_IDX 2
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PROTECTION_FAULT_ADDR_LSB 0x0a8f
+#define mmHUBPREQ5_DC_VM_CONTEXT0_PROTECTION_FAULT_ADDR_LSB_BASE_IDX 2
+#define mmHUBPREQ5_DC_VM_CONTEXT0_CNTL 0x0a90
+#define mmHUBPREQ5_DC_VM_CONTEXT0_CNTL_BASE_IDX 2
+#define mmHUBPREQ5_DCN_VM_MX_L1_TLB_CNTL 0x0a91
+#define mmHUBPREQ5_DCN_VM_MX_L1_TLB_CNTL_BASE_IDX 2
+#define mmHUBPREQ5_BLANK_OFFSET_0 0x0a92
+#define mmHUBPREQ5_BLANK_OFFSET_0_BASE_IDX 2
+#define mmHUBPREQ5_BLANK_OFFSET_1 0x0a93
+#define mmHUBPREQ5_BLANK_OFFSET_1_BASE_IDX 2
+#define mmHUBPREQ5_DST_DIMENSIONS 0x0a94
+#define mmHUBPREQ5_DST_DIMENSIONS_BASE_IDX 2
+#define mmHUBPREQ5_DST_AFTER_SCALER 0x0a95
+#define mmHUBPREQ5_DST_AFTER_SCALER_BASE_IDX 2
+#define mmHUBPREQ5_PREFETCH_SETTINGS 0x0a96
+#define mmHUBPREQ5_PREFETCH_SETTINGS_BASE_IDX 2
+#define mmHUBPREQ5_PREFETCH_SETTINGS_C 0x0a97
+#define mmHUBPREQ5_PREFETCH_SETTINGS_C_BASE_IDX 2
+#define mmHUBPREQ5_VBLANK_PARAMETERS_0 0x0a98
+#define mmHUBPREQ5_VBLANK_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ5_VBLANK_PARAMETERS_1 0x0a99
+#define mmHUBPREQ5_VBLANK_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ5_VBLANK_PARAMETERS_2 0x0a9a
+#define mmHUBPREQ5_VBLANK_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ5_VBLANK_PARAMETERS_3 0x0a9b
+#define mmHUBPREQ5_VBLANK_PARAMETERS_3_BASE_IDX 2
+#define mmHUBPREQ5_VBLANK_PARAMETERS_4 0x0a9c
+#define mmHUBPREQ5_VBLANK_PARAMETERS_4_BASE_IDX 2
+#define mmHUBPREQ5_FLIP_PARAMETERS_0 0x0a9d
+#define mmHUBPREQ5_FLIP_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ5_FLIP_PARAMETERS_1 0x0a9e
+#define mmHUBPREQ5_FLIP_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ5_FLIP_PARAMETERS_2 0x0a9f
+#define mmHUBPREQ5_FLIP_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ5_NOM_PARAMETERS_0 0x0aa0
+#define mmHUBPREQ5_NOM_PARAMETERS_0_BASE_IDX 2
+#define mmHUBPREQ5_NOM_PARAMETERS_1 0x0aa1
+#define mmHUBPREQ5_NOM_PARAMETERS_1_BASE_IDX 2
+#define mmHUBPREQ5_NOM_PARAMETERS_2 0x0aa2
+#define mmHUBPREQ5_NOM_PARAMETERS_2_BASE_IDX 2
+#define mmHUBPREQ5_NOM_PARAMETERS_3 0x0aa3
+#define mmHUBPREQ5_NOM_PARAMETERS_3_BASE_IDX 2
+#define mmHUBPREQ5_NOM_PARAMETERS_4 0x0aa4
+#define mmHUBPREQ5_NOM_PARAMETERS_4_BASE_IDX 2
+#define mmHUBPREQ5_NOM_PARAMETERS_5 0x0aa5
+#define mmHUBPREQ5_NOM_PARAMETERS_5_BASE_IDX 2
+#define mmHUBPREQ5_NOM_PARAMETERS_6 0x0aa6
+#define mmHUBPREQ5_NOM_PARAMETERS_6_BASE_IDX 2
+#define mmHUBPREQ5_NOM_PARAMETERS_7 0x0aa7
+#define mmHUBPREQ5_NOM_PARAMETERS_7_BASE_IDX 2
+#define mmHUBPREQ5_PER_LINE_DELIVERY_PRE 0x0aa8
+#define mmHUBPREQ5_PER_LINE_DELIVERY_PRE_BASE_IDX 2
+#define mmHUBPREQ5_PER_LINE_DELIVERY 0x0aa9
+#define mmHUBPREQ5_PER_LINE_DELIVERY_BASE_IDX 2
+#define mmHUBPREQ5_CURSOR_SETTINGS 0x0aaa
+#define mmHUBPREQ5_CURSOR_SETTINGS_BASE_IDX 2
+#define mmHUBPREQ5_REF_FREQ_TO_PIX_FREQ 0x0aab
+#define mmHUBPREQ5_REF_FREQ_TO_PIX_FREQ_BASE_IDX 2
+#define mmHUBPREQ5_DST_Y_DELTA_DRQ_LIMIT 0x0aac
+#define mmHUBPREQ5_DST_Y_DELTA_DRQ_LIMIT_BASE_IDX 2
+#define mmHUBPREQ5_HUBPREQ_MEM_PWR_CTRL 0x0aad
+#define mmHUBPREQ5_HUBPREQ_MEM_PWR_CTRL_BASE_IDX 2
+#define mmHUBPREQ5_HUBPREQ_MEM_PWR_STATUS 0x0aae
+#define mmHUBPREQ5_HUBPREQ_MEM_PWR_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp5_dispdec_hubpret_dispdec
+// base address: 0x1130
+#define mmHUBPRET5_HUBPRET_CONTROL 0x0ab8
+#define mmHUBPRET5_HUBPRET_CONTROL_BASE_IDX 2
+#define mmHUBPRET5_HUBPRET_MEM_PWR_CTRL 0x0ab9
+#define mmHUBPRET5_HUBPRET_MEM_PWR_CTRL_BASE_IDX 2
+#define mmHUBPRET5_HUBPRET_MEM_PWR_STATUS 0x0aba
+#define mmHUBPRET5_HUBPRET_MEM_PWR_STATUS_BASE_IDX 2
+#define mmHUBPRET5_HUBPRET_READ_LINE_CTRL0 0x0abb
+#define mmHUBPRET5_HUBPRET_READ_LINE_CTRL0_BASE_IDX 2
+#define mmHUBPRET5_HUBPRET_READ_LINE_CTRL1 0x0abc
+#define mmHUBPRET5_HUBPRET_READ_LINE_CTRL1_BASE_IDX 2
+#define mmHUBPRET5_HUBPRET_READ_LINE0 0x0abd
+#define mmHUBPRET5_HUBPRET_READ_LINE0_BASE_IDX 2
+#define mmHUBPRET5_HUBPRET_READ_LINE1 0x0abe
+#define mmHUBPRET5_HUBPRET_READ_LINE1_BASE_IDX 2
+#define mmHUBPRET5_HUBPRET_INTERRUPT 0x0abf
+#define mmHUBPRET5_HUBPRET_INTERRUPT_BASE_IDX 2
+#define mmHUBPRET5_HUBPRET_READ_LINE_VALUE 0x0ac0
+#define mmHUBPRET5_HUBPRET_READ_LINE_VALUE_BASE_IDX 2
+#define mmHUBPRET5_HUBPRET_READ_LINE_STATUS 0x0ac1
+#define mmHUBPRET5_HUBPRET_READ_LINE_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp5_dispdec_cursor0_dispdec
+// base address: 0x1130
+#define mmCURSOR0_5_CURSOR_CONTROL 0x0ac4
+#define mmCURSOR0_5_CURSOR_CONTROL_BASE_IDX 2
+#define mmCURSOR0_5_CURSOR_SURFACE_ADDRESS 0x0ac5
+#define mmCURSOR0_5_CURSOR_SURFACE_ADDRESS_BASE_IDX 2
+#define mmCURSOR0_5_CURSOR_SURFACE_ADDRESS_HIGH 0x0ac6
+#define mmCURSOR0_5_CURSOR_SURFACE_ADDRESS_HIGH_BASE_IDX 2
+#define mmCURSOR0_5_CURSOR_SIZE 0x0ac7
+#define mmCURSOR0_5_CURSOR_SIZE_BASE_IDX 2
+#define mmCURSOR0_5_CURSOR_POSITION 0x0ac8
+#define mmCURSOR0_5_CURSOR_POSITION_BASE_IDX 2
+#define mmCURSOR0_5_CURSOR_HOT_SPOT 0x0ac9
+#define mmCURSOR0_5_CURSOR_HOT_SPOT_BASE_IDX 2
+#define mmCURSOR0_5_CURSOR_STEREO_CONTROL 0x0aca
+#define mmCURSOR0_5_CURSOR_STEREO_CONTROL_BASE_IDX 2
+#define mmCURSOR0_5_CURSOR_DST_OFFSET 0x0acb
+#define mmCURSOR0_5_CURSOR_DST_OFFSET_BASE_IDX 2
+#define mmCURSOR0_5_CURSOR_MEM_PWR_CTRL 0x0acc
+#define mmCURSOR0_5_CURSOR_MEM_PWR_CTRL_BASE_IDX 2
+#define mmCURSOR0_5_CURSOR_MEM_PWR_STATUS 0x0acd
+#define mmCURSOR0_5_CURSOR_MEM_PWR_STATUS_BASE_IDX 2
+#define mmCURSOR0_5_DMDATA_ADDRESS_HIGH 0x0ace
+#define mmCURSOR0_5_DMDATA_ADDRESS_HIGH_BASE_IDX 2
+#define mmCURSOR0_5_DMDATA_ADDRESS_LOW 0x0acf
+#define mmCURSOR0_5_DMDATA_ADDRESS_LOW_BASE_IDX 2
+#define mmCURSOR0_5_DMDATA_CNTL 0x0ad0
+#define mmCURSOR0_5_DMDATA_CNTL_BASE_IDX 2
+#define mmCURSOR0_5_DMDATA_QOS_CNTL 0x0ad1
+#define mmCURSOR0_5_DMDATA_QOS_CNTL_BASE_IDX 2
+#define mmCURSOR0_5_DMDATA_STATUS 0x0ad2
+#define mmCURSOR0_5_DMDATA_STATUS_BASE_IDX 2
+#define mmCURSOR0_5_DMDATA_SW_CNTL 0x0ad3
+#define mmCURSOR0_5_DMDATA_SW_CNTL_BASE_IDX 2
+#define mmCURSOR0_5_DMDATA_SW_DATA 0x0ad4
+#define mmCURSOR0_5_DMDATA_SW_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp5_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x2ba4
+#define mmDC_PERFMON12_PERFCOUNTER_CNTL 0x0ae9
+#define mmDC_PERFMON12_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON12_PERFCOUNTER_CNTL2 0x0aea
+#define mmDC_PERFMON12_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON12_PERFCOUNTER_STATE 0x0aeb
+#define mmDC_PERFMON12_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON12_PERFMON_CNTL 0x0aec
+#define mmDC_PERFMON12_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON12_PERFMON_CNTL2 0x0aed
+#define mmDC_PERFMON12_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON12_PERFMON_CVALUE_INT_MISC 0x0aee
+#define mmDC_PERFMON12_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON12_PERFMON_CVALUE_LOW 0x0aef
+#define mmDC_PERFMON12_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON12_PERFMON_HI 0x0af0
+#define mmDC_PERFMON12_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON12_PERFMON_LOW 0x0af1
+#define mmDC_PERFMON12_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcbubp5_dispdec_hubpxfc_dispdec
+// base address: 0x1130
+#define mmHUBPXFC5_HUBP_XFC_CNTL 0x0af5
+#define mmHUBPXFC5_HUBP_XFC_CNTL_BASE_IDX 2
+#define mmHUBPXFC5_HUBP_XFC_XBUF_RD_BASE0_ADDR_LSB 0x0af6
+#define mmHUBPXFC5_HUBP_XFC_XBUF_RD_BASE0_ADDR_LSB_BASE_IDX 2
+#define mmHUBPXFC5_HUBP_XFC_XBUF_RD_BASE0_ADDR_MSB 0x0af7
+#define mmHUBPXFC5_HUBP_XFC_XBUF_RD_BASE0_ADDR_MSB_BASE_IDX 2
+#define mmHUBPXFC5_HUBP_XFC_XBUF_RD_BASE1_ADDR_LSB 0x0af8
+#define mmHUBPXFC5_HUBP_XFC_XBUF_RD_BASE1_ADDR_LSB_BASE_IDX 2
+#define mmHUBPXFC5_HUBP_XFC_XBUF_RD_BASE1_ADDR_MSB 0x0af9
+#define mmHUBPXFC5_HUBP_XFC_XBUF_RD_BASE1_ADDR_MSB_BASE_IDX 2
+#define mmHUBPXFC5_HUBP_XFC_XBUF_RD_PITCH 0x0afa
+#define mmHUBPXFC5_HUBP_XFC_XBUF_RD_PITCH_BASE_IDX 2
+#define mmHUBPXFC5_HUBP_XFC_DELAY_CONFIG0 0x0afb
+#define mmHUBPXFC5_HUBP_XFC_DELAY_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC5_HUBP_XFC_DELAY_CONFIG1 0x0afc
+#define mmHUBPXFC5_HUBP_XFC_DELAY_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC5_HUBP_XFC_DELAY_CONFIG2 0x0afd
+#define mmHUBPXFC5_HUBP_XFC_DELAY_CONFIG2_BASE_IDX 2
+#define mmHUBPXFC5_HUBP_XFC_UNDERFLOW_STATUS 0x0afe
+#define mmHUBPXFC5_HUBP_XFC_UNDERFLOW_STATUS_BASE_IDX 2
+#define mmHUBPXFC5_HUBP_XFC_SLV_VTG_CONFIG0 0x0aff
+#define mmHUBPXFC5_HUBP_XFC_SLV_VTG_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC5_HUBP_XFC_SLV_VTG_CONFIG1 0x0b00
+#define mmHUBPXFC5_HUBP_XFC_SLV_VTG_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC5_HUBP_XFC_SLV_SCALER_CONFIG0 0x0b01
+#define mmHUBPXFC5_HUBP_XFC_SLV_SCALER_CONFIG0_BASE_IDX 2
+#define mmHUBPXFC5_HUBP_XFC_SLV_SCALER_CONFIG1 0x0b02
+#define mmHUBPXFC5_HUBP_XFC_SLV_SCALER_CONFIG1_BASE_IDX 2
+#define mmHUBPXFC5_HUBP_XFC_MPC_CONFIG 0x0b03
+#define mmHUBPXFC5_HUBP_XFC_MPC_CONFIG_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp0_dispdec_dpp_top_dispdec
+// base address: 0x0
+#define mmDPP_TOP0_DPP_CONTROL 0x0cc5
+#define mmDPP_TOP0_DPP_CONTROL_BASE_IDX 2
+#define mmDPP_TOP0_DPP_SOFT_RESET 0x0cc6
+#define mmDPP_TOP0_DPP_SOFT_RESET_BASE_IDX 2
+#define mmDPP_TOP0_DPP_CRC_VAL_R_G 0x0cc7
+#define mmDPP_TOP0_DPP_CRC_VAL_R_G_BASE_IDX 2
+#define mmDPP_TOP0_DPP_CRC_VAL_B_A 0x0cc8
+#define mmDPP_TOP0_DPP_CRC_VAL_B_A_BASE_IDX 2
+#define mmDPP_TOP0_DPP_CRC_CTRL 0x0cc9
+#define mmDPP_TOP0_DPP_CRC_CTRL_BASE_IDX 2
+#define mmDPP_TOP0_HOST_READ_CONTROL 0x0cca
+#define mmDPP_TOP0_HOST_READ_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp0_dispdec_cnvc_cfg_dispdec
+// base address: 0x0
+#define mmCNVC_CFG0_CNVC_SURFACE_PIXEL_FORMAT 0x0ccf
+#define mmCNVC_CFG0_CNVC_SURFACE_PIXEL_FORMAT_BASE_IDX 2
+#define mmCNVC_CFG0_FORMAT_CONTROL 0x0cd0
+#define mmCNVC_CFG0_FORMAT_CONTROL_BASE_IDX 2
+#define mmCNVC_CFG0_FCNV_FP_BIAS_R 0x0cd1
+#define mmCNVC_CFG0_FCNV_FP_BIAS_R_BASE_IDX 2
+#define mmCNVC_CFG0_FCNV_FP_BIAS_G 0x0cd2
+#define mmCNVC_CFG0_FCNV_FP_BIAS_G_BASE_IDX 2
+#define mmCNVC_CFG0_FCNV_FP_BIAS_B 0x0cd3
+#define mmCNVC_CFG0_FCNV_FP_BIAS_B_BASE_IDX 2
+#define mmCNVC_CFG0_FCNV_FP_SCALE_R 0x0cd4
+#define mmCNVC_CFG0_FCNV_FP_SCALE_R_BASE_IDX 2
+#define mmCNVC_CFG0_FCNV_FP_SCALE_G 0x0cd5
+#define mmCNVC_CFG0_FCNV_FP_SCALE_G_BASE_IDX 2
+#define mmCNVC_CFG0_FCNV_FP_SCALE_B 0x0cd6
+#define mmCNVC_CFG0_FCNV_FP_SCALE_B_BASE_IDX 2
+#define mmCNVC_CFG0_COLOR_KEYER_CONTROL 0x0cd7
+#define mmCNVC_CFG0_COLOR_KEYER_CONTROL_BASE_IDX 2
+#define mmCNVC_CFG0_COLOR_KEYER_ALPHA 0x0cd8
+#define mmCNVC_CFG0_COLOR_KEYER_ALPHA_BASE_IDX 2
+#define mmCNVC_CFG0_COLOR_KEYER_RED 0x0cd9
+#define mmCNVC_CFG0_COLOR_KEYER_RED_BASE_IDX 2
+#define mmCNVC_CFG0_COLOR_KEYER_GREEN 0x0cda
+#define mmCNVC_CFG0_COLOR_KEYER_GREEN_BASE_IDX 2
+#define mmCNVC_CFG0_COLOR_KEYER_BLUE 0x0cdb
+#define mmCNVC_CFG0_COLOR_KEYER_BLUE_BASE_IDX 2
+#define mmCNVC_CFG0_ALPHA_2BIT_LUT 0x0cdd
+#define mmCNVC_CFG0_ALPHA_2BIT_LUT_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp0_dispdec_cnvc_cur_dispdec
+// base address: 0x0
+#define mmCNVC_CUR0_CURSOR0_CONTROL 0x0ce0
+#define mmCNVC_CUR0_CURSOR0_CONTROL_BASE_IDX 2
+#define mmCNVC_CUR0_CURSOR0_COLOR0 0x0ce1
+#define mmCNVC_CUR0_CURSOR0_COLOR0_BASE_IDX 2
+#define mmCNVC_CUR0_CURSOR0_COLOR1 0x0ce2
+#define mmCNVC_CUR0_CURSOR0_COLOR1_BASE_IDX 2
+#define mmCNVC_CUR0_CURSOR0_FP_SCALE_BIAS 0x0ce3
+#define mmCNVC_CUR0_CURSOR0_FP_SCALE_BIAS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp0_dispdec_dscl_dispdec
+// base address: 0x0
+#define mmDSCL0_SCL_COEF_RAM_TAP_SELECT 0x0cea
+#define mmDSCL0_SCL_COEF_RAM_TAP_SELECT_BASE_IDX 2
+#define mmDSCL0_SCL_COEF_RAM_TAP_DATA 0x0ceb
+#define mmDSCL0_SCL_COEF_RAM_TAP_DATA_BASE_IDX 2
+#define mmDSCL0_SCL_MODE 0x0cec
+#define mmDSCL0_SCL_MODE_BASE_IDX 2
+#define mmDSCL0_SCL_TAP_CONTROL 0x0ced
+#define mmDSCL0_SCL_TAP_CONTROL_BASE_IDX 2
+#define mmDSCL0_DSCL_CONTROL 0x0cee
+#define mmDSCL0_DSCL_CONTROL_BASE_IDX 2
+#define mmDSCL0_DSCL_2TAP_CONTROL 0x0cef
+#define mmDSCL0_DSCL_2TAP_CONTROL_BASE_IDX 2
+#define mmDSCL0_SCL_MANUAL_REPLICATE_CONTROL 0x0cf0
+#define mmDSCL0_SCL_MANUAL_REPLICATE_CONTROL_BASE_IDX 2
+#define mmDSCL0_SCL_HORZ_FILTER_SCALE_RATIO 0x0cf1
+#define mmDSCL0_SCL_HORZ_FILTER_SCALE_RATIO_BASE_IDX 2
+#define mmDSCL0_SCL_HORZ_FILTER_INIT 0x0cf2
+#define mmDSCL0_SCL_HORZ_FILTER_INIT_BASE_IDX 2
+#define mmDSCL0_SCL_HORZ_FILTER_SCALE_RATIO_C 0x0cf3
+#define mmDSCL0_SCL_HORZ_FILTER_SCALE_RATIO_C_BASE_IDX 2
+#define mmDSCL0_SCL_HORZ_FILTER_INIT_C 0x0cf4
+#define mmDSCL0_SCL_HORZ_FILTER_INIT_C_BASE_IDX 2
+#define mmDSCL0_SCL_VERT_FILTER_SCALE_RATIO 0x0cf5
+#define mmDSCL0_SCL_VERT_FILTER_SCALE_RATIO_BASE_IDX 2
+#define mmDSCL0_SCL_VERT_FILTER_INIT 0x0cf6
+#define mmDSCL0_SCL_VERT_FILTER_INIT_BASE_IDX 2
+#define mmDSCL0_SCL_VERT_FILTER_INIT_BOT 0x0cf7
+#define mmDSCL0_SCL_VERT_FILTER_INIT_BOT_BASE_IDX 2
+#define mmDSCL0_SCL_VERT_FILTER_SCALE_RATIO_C 0x0cf8
+#define mmDSCL0_SCL_VERT_FILTER_SCALE_RATIO_C_BASE_IDX 2
+#define mmDSCL0_SCL_VERT_FILTER_INIT_C 0x0cf9
+#define mmDSCL0_SCL_VERT_FILTER_INIT_C_BASE_IDX 2
+#define mmDSCL0_SCL_VERT_FILTER_INIT_BOT_C 0x0cfa
+#define mmDSCL0_SCL_VERT_FILTER_INIT_BOT_C_BASE_IDX 2
+#define mmDSCL0_SCL_BLACK_OFFSET 0x0cfb
+#define mmDSCL0_SCL_BLACK_OFFSET_BASE_IDX 2
+#define mmDSCL0_DSCL_UPDATE 0x0cfc
+#define mmDSCL0_DSCL_UPDATE_BASE_IDX 2
+#define mmDSCL0_DSCL_AUTOCAL 0x0cfd
+#define mmDSCL0_DSCL_AUTOCAL_BASE_IDX 2
+#define mmDSCL0_DSCL_EXT_OVERSCAN_LEFT_RIGHT 0x0cfe
+#define mmDSCL0_DSCL_EXT_OVERSCAN_LEFT_RIGHT_BASE_IDX 2
+#define mmDSCL0_DSCL_EXT_OVERSCAN_TOP_BOTTOM 0x0cff
+#define mmDSCL0_DSCL_EXT_OVERSCAN_TOP_BOTTOM_BASE_IDX 2
+#define mmDSCL0_OTG_H_BLANK 0x0d00
+#define mmDSCL0_OTG_H_BLANK_BASE_IDX 2
+#define mmDSCL0_OTG_V_BLANK 0x0d01
+#define mmDSCL0_OTG_V_BLANK_BASE_IDX 2
+#define mmDSCL0_RECOUT_START 0x0d02
+#define mmDSCL0_RECOUT_START_BASE_IDX 2
+#define mmDSCL0_RECOUT_SIZE 0x0d03
+#define mmDSCL0_RECOUT_SIZE_BASE_IDX 2
+#define mmDSCL0_MPC_SIZE 0x0d04
+#define mmDSCL0_MPC_SIZE_BASE_IDX 2
+#define mmDSCL0_LB_DATA_FORMAT 0x0d05
+#define mmDSCL0_LB_DATA_FORMAT_BASE_IDX 2
+#define mmDSCL0_LB_MEMORY_CTRL 0x0d06
+#define mmDSCL0_LB_MEMORY_CTRL_BASE_IDX 2
+#define mmDSCL0_LB_V_COUNTER 0x0d07
+#define mmDSCL0_LB_V_COUNTER_BASE_IDX 2
+#define mmDSCL0_DSCL_MEM_PWR_CTRL 0x0d08
+#define mmDSCL0_DSCL_MEM_PWR_CTRL_BASE_IDX 2
+#define mmDSCL0_DSCL_MEM_PWR_STATUS 0x0d09
+#define mmDSCL0_DSCL_MEM_PWR_STATUS_BASE_IDX 2
+#define mmDSCL0_OBUF_CONTROL 0x0d0a
+#define mmDSCL0_OBUF_CONTROL_BASE_IDX 2
+#define mmDSCL0_OBUF_MEM_PWR_CTRL 0x0d0b
+#define mmDSCL0_OBUF_MEM_PWR_CTRL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp0_dispdec_cm_dispdec
+// base address: 0x0
+#define mmCM0_CM_CONTROL 0x0d1a
+#define mmCM0_CM_CONTROL_BASE_IDX 2
+#define mmCM0_CM_ICSC_CONTROL 0x0d1b
+#define mmCM0_CM_ICSC_CONTROL_BASE_IDX 2
+#define mmCM0_CM_ICSC_C11_C12 0x0d1c
+#define mmCM0_CM_ICSC_C11_C12_BASE_IDX 2
+#define mmCM0_CM_ICSC_C13_C14 0x0d1d
+#define mmCM0_CM_ICSC_C13_C14_BASE_IDX 2
+#define mmCM0_CM_ICSC_C21_C22 0x0d1e
+#define mmCM0_CM_ICSC_C21_C22_BASE_IDX 2
+#define mmCM0_CM_ICSC_C23_C24 0x0d1f
+#define mmCM0_CM_ICSC_C23_C24_BASE_IDX 2
+#define mmCM0_CM_ICSC_C31_C32 0x0d20
+#define mmCM0_CM_ICSC_C31_C32_BASE_IDX 2
+#define mmCM0_CM_ICSC_C33_C34 0x0d21
+#define mmCM0_CM_ICSC_C33_C34_BASE_IDX 2
+#define mmCM0_CM_ICSC_B_C11_C12 0x0d22
+#define mmCM0_CM_ICSC_B_C11_C12_BASE_IDX 2
+#define mmCM0_CM_ICSC_B_C13_C14 0x0d23
+#define mmCM0_CM_ICSC_B_C13_C14_BASE_IDX 2
+#define mmCM0_CM_ICSC_B_C21_C22 0x0d24
+#define mmCM0_CM_ICSC_B_C21_C22_BASE_IDX 2
+#define mmCM0_CM_ICSC_B_C23_C24 0x0d25
+#define mmCM0_CM_ICSC_B_C23_C24_BASE_IDX 2
+#define mmCM0_CM_ICSC_B_C31_C32 0x0d26
+#define mmCM0_CM_ICSC_B_C31_C32_BASE_IDX 2
+#define mmCM0_CM_ICSC_B_C33_C34 0x0d27
+#define mmCM0_CM_ICSC_B_C33_C34_BASE_IDX 2
+#define mmCM0_CM_GAMUT_REMAP_CONTROL 0x0d28
+#define mmCM0_CM_GAMUT_REMAP_CONTROL_BASE_IDX 2
+#define mmCM0_CM_GAMUT_REMAP_C11_C12 0x0d29
+#define mmCM0_CM_GAMUT_REMAP_C11_C12_BASE_IDX 2
+#define mmCM0_CM_GAMUT_REMAP_C13_C14 0x0d2a
+#define mmCM0_CM_GAMUT_REMAP_C13_C14_BASE_IDX 2
+#define mmCM0_CM_GAMUT_REMAP_C21_C22 0x0d2b
+#define mmCM0_CM_GAMUT_REMAP_C21_C22_BASE_IDX 2
+#define mmCM0_CM_GAMUT_REMAP_C23_C24 0x0d2c
+#define mmCM0_CM_GAMUT_REMAP_C23_C24_BASE_IDX 2
+#define mmCM0_CM_GAMUT_REMAP_C31_C32 0x0d2d
+#define mmCM0_CM_GAMUT_REMAP_C31_C32_BASE_IDX 2
+#define mmCM0_CM_GAMUT_REMAP_C33_C34 0x0d2e
+#define mmCM0_CM_GAMUT_REMAP_C33_C34_BASE_IDX 2
+#define mmCM0_CM_GAMUT_REMAP_B_C11_C12 0x0d2f
+#define mmCM0_CM_GAMUT_REMAP_B_C11_C12_BASE_IDX 2
+#define mmCM0_CM_GAMUT_REMAP_B_C13_C14 0x0d30
+#define mmCM0_CM_GAMUT_REMAP_B_C13_C14_BASE_IDX 2
+#define mmCM0_CM_GAMUT_REMAP_B_C21_C22 0x0d31
+#define mmCM0_CM_GAMUT_REMAP_B_C21_C22_BASE_IDX 2
+#define mmCM0_CM_GAMUT_REMAP_B_C23_C24 0x0d32
+#define mmCM0_CM_GAMUT_REMAP_B_C23_C24_BASE_IDX 2
+#define mmCM0_CM_GAMUT_REMAP_B_C31_C32 0x0d33
+#define mmCM0_CM_GAMUT_REMAP_B_C31_C32_BASE_IDX 2
+#define mmCM0_CM_GAMUT_REMAP_B_C33_C34 0x0d34
+#define mmCM0_CM_GAMUT_REMAP_B_C33_C34_BASE_IDX 2
+#define mmCM0_CM_BIAS_CR_R 0x0d35
+#define mmCM0_CM_BIAS_CR_R_BASE_IDX 2
+#define mmCM0_CM_BIAS_Y_G_CB_B 0x0d36
+#define mmCM0_CM_BIAS_Y_G_CB_B_BASE_IDX 2
+#define mmCM0_CM_DGAM_CONTROL 0x0d37
+#define mmCM0_CM_DGAM_CONTROL_BASE_IDX 2
+#define mmCM0_CM_DGAM_LUT_INDEX 0x0d38
+#define mmCM0_CM_DGAM_LUT_INDEX_BASE_IDX 2
+#define mmCM0_CM_DGAM_LUT_DATA 0x0d39
+#define mmCM0_CM_DGAM_LUT_DATA_BASE_IDX 2
+#define mmCM0_CM_DGAM_LUT_WRITE_EN_MASK 0x0d3a
+#define mmCM0_CM_DGAM_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_START_CNTL_B 0x0d3b
+#define mmCM0_CM_DGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_START_CNTL_G 0x0d3c
+#define mmCM0_CM_DGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_START_CNTL_R 0x0d3d
+#define mmCM0_CM_DGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_SLOPE_CNTL_B 0x0d3e
+#define mmCM0_CM_DGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_SLOPE_CNTL_G 0x0d3f
+#define mmCM0_CM_DGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_SLOPE_CNTL_R 0x0d40
+#define mmCM0_CM_DGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_END_CNTL1_B 0x0d41
+#define mmCM0_CM_DGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_END_CNTL2_B 0x0d42
+#define mmCM0_CM_DGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_END_CNTL1_G 0x0d43
+#define mmCM0_CM_DGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_END_CNTL2_G 0x0d44
+#define mmCM0_CM_DGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_END_CNTL1_R 0x0d45
+#define mmCM0_CM_DGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_END_CNTL2_R 0x0d46
+#define mmCM0_CM_DGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_REGION_0_1 0x0d47
+#define mmCM0_CM_DGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_REGION_2_3 0x0d48
+#define mmCM0_CM_DGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_REGION_4_5 0x0d49
+#define mmCM0_CM_DGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_REGION_6_7 0x0d4a
+#define mmCM0_CM_DGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_REGION_8_9 0x0d4b
+#define mmCM0_CM_DGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_REGION_10_11 0x0d4c
+#define mmCM0_CM_DGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_REGION_12_13 0x0d4d
+#define mmCM0_CM_DGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMA_REGION_14_15 0x0d4e
+#define mmCM0_CM_DGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_START_CNTL_B 0x0d4f
+#define mmCM0_CM_DGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_START_CNTL_G 0x0d50
+#define mmCM0_CM_DGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_START_CNTL_R 0x0d51
+#define mmCM0_CM_DGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_SLOPE_CNTL_B 0x0d52
+#define mmCM0_CM_DGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_SLOPE_CNTL_G 0x0d53
+#define mmCM0_CM_DGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_SLOPE_CNTL_R 0x0d54
+#define mmCM0_CM_DGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_END_CNTL1_B 0x0d55
+#define mmCM0_CM_DGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_END_CNTL2_B 0x0d56
+#define mmCM0_CM_DGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_END_CNTL1_G 0x0d57
+#define mmCM0_CM_DGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_END_CNTL2_G 0x0d58
+#define mmCM0_CM_DGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_END_CNTL1_R 0x0d59
+#define mmCM0_CM_DGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_END_CNTL2_R 0x0d5a
+#define mmCM0_CM_DGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_REGION_0_1 0x0d5b
+#define mmCM0_CM_DGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_REGION_2_3 0x0d5c
+#define mmCM0_CM_DGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_REGION_4_5 0x0d5d
+#define mmCM0_CM_DGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_REGION_6_7 0x0d5e
+#define mmCM0_CM_DGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_REGION_8_9 0x0d5f
+#define mmCM0_CM_DGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_REGION_10_11 0x0d60
+#define mmCM0_CM_DGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_REGION_12_13 0x0d61
+#define mmCM0_CM_DGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM0_CM_DGAM_RAMB_REGION_14_15 0x0d62
+#define mmCM0_CM_DGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_CONTROL 0x0d63
+#define mmCM0_CM_BLNDGAM_CONTROL_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_LUT_INDEX 0x0d64
+#define mmCM0_CM_BLNDGAM_LUT_INDEX_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_LUT_DATA 0x0d65
+#define mmCM0_CM_BLNDGAM_LUT_DATA_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_LUT_WRITE_EN_MASK 0x0d66
+#define mmCM0_CM_BLNDGAM_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_START_CNTL_B 0x0d67
+#define mmCM0_CM_BLNDGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_START_CNTL_G 0x0d68
+#define mmCM0_CM_BLNDGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_START_CNTL_R 0x0d69
+#define mmCM0_CM_BLNDGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_SLOPE_CNTL_B 0x0d6a
+#define mmCM0_CM_BLNDGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_SLOPE_CNTL_G 0x0d6b
+#define mmCM0_CM_BLNDGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_SLOPE_CNTL_R 0x0d6c
+#define mmCM0_CM_BLNDGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_END_CNTL1_B 0x0d6d
+#define mmCM0_CM_BLNDGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_END_CNTL2_B 0x0d6e
+#define mmCM0_CM_BLNDGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_END_CNTL1_G 0x0d6f
+#define mmCM0_CM_BLNDGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_END_CNTL2_G 0x0d70
+#define mmCM0_CM_BLNDGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_END_CNTL1_R 0x0d71
+#define mmCM0_CM_BLNDGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_END_CNTL2_R 0x0d72
+#define mmCM0_CM_BLNDGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_0_1 0x0d73
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_2_3 0x0d74
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_4_5 0x0d75
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_6_7 0x0d76
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_8_9 0x0d77
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_10_11 0x0d78
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_12_13 0x0d79
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_14_15 0x0d7a
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_16_17 0x0d7b
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_16_17_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_18_19 0x0d7c
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_18_19_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_20_21 0x0d7d
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_20_21_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_22_23 0x0d7e
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_22_23_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_24_25 0x0d7f
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_24_25_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_26_27 0x0d80
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_26_27_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_28_29 0x0d81
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_28_29_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_30_31 0x0d82
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_30_31_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_32_33 0x0d83
+#define mmCM0_CM_BLNDGAM_RAMA_REGION_32_33_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_START_CNTL_B 0x0d84
+#define mmCM0_CM_BLNDGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_START_CNTL_G 0x0d85
+#define mmCM0_CM_BLNDGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_START_CNTL_R 0x0d86
+#define mmCM0_CM_BLNDGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_SLOPE_CNTL_B 0x0d87
+#define mmCM0_CM_BLNDGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_SLOPE_CNTL_G 0x0d88
+#define mmCM0_CM_BLNDGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_SLOPE_CNTL_R 0x0d89
+#define mmCM0_CM_BLNDGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_END_CNTL1_B 0x0d8a
+#define mmCM0_CM_BLNDGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_END_CNTL2_B 0x0d8b
+#define mmCM0_CM_BLNDGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_END_CNTL1_G 0x0d8c
+#define mmCM0_CM_BLNDGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_END_CNTL2_G 0x0d8d
+#define mmCM0_CM_BLNDGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_END_CNTL1_R 0x0d8e
+#define mmCM0_CM_BLNDGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_END_CNTL2_R 0x0d8f
+#define mmCM0_CM_BLNDGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_0_1 0x0d90
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_2_3 0x0d91
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_4_5 0x0d92
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_6_7 0x0d93
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_8_9 0x0d94
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_10_11 0x0d95
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_12_13 0x0d96
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_14_15 0x0d97
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_16_17 0x0d98
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_16_17_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_18_19 0x0d99
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_18_19_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_20_21 0x0d9a
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_20_21_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_22_23 0x0d9b
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_22_23_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_24_25 0x0d9c
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_24_25_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_26_27 0x0d9d
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_26_27_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_28_29 0x0d9e
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_28_29_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_30_31 0x0d9f
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_30_31_BASE_IDX 2
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_32_33 0x0da0
+#define mmCM0_CM_BLNDGAM_RAMB_REGION_32_33_BASE_IDX 2
+#define mmCM0_CM_HDR_MULT_COEF 0x0da1
+#define mmCM0_CM_HDR_MULT_COEF_BASE_IDX 2
+#define mmCM0_CM_MEM_PWR_CTRL 0x0da2
+#define mmCM0_CM_MEM_PWR_CTRL_BASE_IDX 2
+#define mmCM0_CM_MEM_PWR_STATUS 0x0da3
+#define mmCM0_CM_MEM_PWR_STATUS_BASE_IDX 2
+#define mmCM0_CM_DEALPHA 0x0da5
+#define mmCM0_CM_DEALPHA_BASE_IDX 2
+#define mmCM0_CM_COEF_FORMAT 0x0da6
+#define mmCM0_CM_COEF_FORMAT_BASE_IDX 2
+#define mmCM0_CM_SHAPER_CONTROL 0x0da7
+#define mmCM0_CM_SHAPER_CONTROL_BASE_IDX 2
+#define mmCM0_CM_SHAPER_OFFSET_R 0x0da8
+#define mmCM0_CM_SHAPER_OFFSET_R_BASE_IDX 2
+#define mmCM0_CM_SHAPER_OFFSET_G 0x0da9
+#define mmCM0_CM_SHAPER_OFFSET_G_BASE_IDX 2
+#define mmCM0_CM_SHAPER_OFFSET_B 0x0daa
+#define mmCM0_CM_SHAPER_OFFSET_B_BASE_IDX 2
+#define mmCM0_CM_SHAPER_SCALE_R 0x0dab
+#define mmCM0_CM_SHAPER_SCALE_R_BASE_IDX 2
+#define mmCM0_CM_SHAPER_SCALE_G_B 0x0dac
+#define mmCM0_CM_SHAPER_SCALE_G_B_BASE_IDX 2
+#define mmCM0_CM_SHAPER_LUT_INDEX 0x0dad
+#define mmCM0_CM_SHAPER_LUT_INDEX_BASE_IDX 2
+#define mmCM0_CM_SHAPER_LUT_DATA 0x0dae
+#define mmCM0_CM_SHAPER_LUT_DATA_BASE_IDX 2
+#define mmCM0_CM_SHAPER_LUT_WRITE_EN_MASK 0x0daf
+#define mmCM0_CM_SHAPER_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_START_CNTL_B 0x0db0
+#define mmCM0_CM_SHAPER_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_START_CNTL_G 0x0db1
+#define mmCM0_CM_SHAPER_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_START_CNTL_R 0x0db2
+#define mmCM0_CM_SHAPER_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_END_CNTL_B 0x0db3
+#define mmCM0_CM_SHAPER_RAMA_END_CNTL_B_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_END_CNTL_G 0x0db4
+#define mmCM0_CM_SHAPER_RAMA_END_CNTL_G_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_END_CNTL_R 0x0db5
+#define mmCM0_CM_SHAPER_RAMA_END_CNTL_R_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_REGION_0_1 0x0db6
+#define mmCM0_CM_SHAPER_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_REGION_2_3 0x0db7
+#define mmCM0_CM_SHAPER_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_REGION_4_5 0x0db8
+#define mmCM0_CM_SHAPER_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_REGION_6_7 0x0db9
+#define mmCM0_CM_SHAPER_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_REGION_8_9 0x0dba
+#define mmCM0_CM_SHAPER_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_REGION_10_11 0x0dbb
+#define mmCM0_CM_SHAPER_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_REGION_12_13 0x0dbc
+#define mmCM0_CM_SHAPER_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_REGION_14_15 0x0dbd
+#define mmCM0_CM_SHAPER_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_REGION_16_17 0x0dbe
+#define mmCM0_CM_SHAPER_RAMA_REGION_16_17_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_REGION_18_19 0x0dbf
+#define mmCM0_CM_SHAPER_RAMA_REGION_18_19_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_REGION_20_21 0x0dc0
+#define mmCM0_CM_SHAPER_RAMA_REGION_20_21_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_REGION_22_23 0x0dc1
+#define mmCM0_CM_SHAPER_RAMA_REGION_22_23_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_REGION_24_25 0x0dc2
+#define mmCM0_CM_SHAPER_RAMA_REGION_24_25_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_REGION_26_27 0x0dc3
+#define mmCM0_CM_SHAPER_RAMA_REGION_26_27_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_REGION_28_29 0x0dc4
+#define mmCM0_CM_SHAPER_RAMA_REGION_28_29_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_REGION_30_31 0x0dc5
+#define mmCM0_CM_SHAPER_RAMA_REGION_30_31_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMA_REGION_32_33 0x0dc6
+#define mmCM0_CM_SHAPER_RAMA_REGION_32_33_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_START_CNTL_B 0x0dc7
+#define mmCM0_CM_SHAPER_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_START_CNTL_G 0x0dc8
+#define mmCM0_CM_SHAPER_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_START_CNTL_R 0x0dc9
+#define mmCM0_CM_SHAPER_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_END_CNTL_B 0x0dca
+#define mmCM0_CM_SHAPER_RAMB_END_CNTL_B_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_END_CNTL_G 0x0dcb
+#define mmCM0_CM_SHAPER_RAMB_END_CNTL_G_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_END_CNTL_R 0x0dcc
+#define mmCM0_CM_SHAPER_RAMB_END_CNTL_R_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_REGION_0_1 0x0dcd
+#define mmCM0_CM_SHAPER_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_REGION_2_3 0x0dce
+#define mmCM0_CM_SHAPER_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_REGION_4_5 0x0dcf
+#define mmCM0_CM_SHAPER_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_REGION_6_7 0x0dd0
+#define mmCM0_CM_SHAPER_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_REGION_8_9 0x0dd1
+#define mmCM0_CM_SHAPER_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_REGION_10_11 0x0dd2
+#define mmCM0_CM_SHAPER_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_REGION_12_13 0x0dd3
+#define mmCM0_CM_SHAPER_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_REGION_14_15 0x0dd4
+#define mmCM0_CM_SHAPER_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_REGION_16_17 0x0dd5
+#define mmCM0_CM_SHAPER_RAMB_REGION_16_17_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_REGION_18_19 0x0dd6
+#define mmCM0_CM_SHAPER_RAMB_REGION_18_19_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_REGION_20_21 0x0dd7
+#define mmCM0_CM_SHAPER_RAMB_REGION_20_21_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_REGION_22_23 0x0dd8
+#define mmCM0_CM_SHAPER_RAMB_REGION_22_23_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_REGION_24_25 0x0dd9
+#define mmCM0_CM_SHAPER_RAMB_REGION_24_25_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_REGION_26_27 0x0dda
+#define mmCM0_CM_SHAPER_RAMB_REGION_26_27_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_REGION_28_29 0x0ddb
+#define mmCM0_CM_SHAPER_RAMB_REGION_28_29_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_REGION_30_31 0x0ddc
+#define mmCM0_CM_SHAPER_RAMB_REGION_30_31_BASE_IDX 2
+#define mmCM0_CM_SHAPER_RAMB_REGION_32_33 0x0ddd
+#define mmCM0_CM_SHAPER_RAMB_REGION_32_33_BASE_IDX 2
+#define mmCM0_CM_MEM_PWR_CTRL2 0x0dde
+#define mmCM0_CM_MEM_PWR_CTRL2_BASE_IDX 2
+#define mmCM0_CM_MEM_PWR_STATUS2 0x0ddf
+#define mmCM0_CM_MEM_PWR_STATUS2_BASE_IDX 2
+#define mmCM0_CM_3DLUT_MODE 0x0de0
+#define mmCM0_CM_3DLUT_MODE_BASE_IDX 2
+#define mmCM0_CM_3DLUT_INDEX 0x0de1
+#define mmCM0_CM_3DLUT_INDEX_BASE_IDX 2
+#define mmCM0_CM_3DLUT_DATA 0x0de2
+#define mmCM0_CM_3DLUT_DATA_BASE_IDX 2
+#define mmCM0_CM_3DLUT_DATA_30BIT 0x0de3
+#define mmCM0_CM_3DLUT_DATA_30BIT_BASE_IDX 2
+#define mmCM0_CM_3DLUT_READ_WRITE_CONTROL 0x0de4
+#define mmCM0_CM_3DLUT_READ_WRITE_CONTROL_BASE_IDX 2
+#define mmCM0_CM_3DLUT_OUT_NORM_FACTOR 0x0de5
+#define mmCM0_CM_3DLUT_OUT_NORM_FACTOR_BASE_IDX 2
+#define mmCM0_CM_3DLUT_OUT_OFFSET_R 0x0de6
+#define mmCM0_CM_3DLUT_OUT_OFFSET_R_BASE_IDX 2
+#define mmCM0_CM_3DLUT_OUT_OFFSET_G 0x0de7
+#define mmCM0_CM_3DLUT_OUT_OFFSET_G_BASE_IDX 2
+#define mmCM0_CM_3DLUT_OUT_OFFSET_B 0x0de8
+#define mmCM0_CM_3DLUT_OUT_OFFSET_B_BASE_IDX 2
+#define mmCM0_CM_TEST_DEBUG_INDEX 0x0de9
+#define mmCM0_CM_TEST_DEBUG_INDEX_BASE_IDX 2
+#define mmCM0_CM_TEST_DEBUG_DATA 0x0dea
+#define mmCM0_CM_TEST_DEBUG_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp0_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x3890
+#define mmDC_PERFMON13_PERFCOUNTER_CNTL 0x0e24
+#define mmDC_PERFMON13_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON13_PERFCOUNTER_CNTL2 0x0e25
+#define mmDC_PERFMON13_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON13_PERFCOUNTER_STATE 0x0e26
+#define mmDC_PERFMON13_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON13_PERFMON_CNTL 0x0e27
+#define mmDC_PERFMON13_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON13_PERFMON_CNTL2 0x0e28
+#define mmDC_PERFMON13_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON13_PERFMON_CVALUE_INT_MISC 0x0e29
+#define mmDC_PERFMON13_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON13_PERFMON_CVALUE_LOW 0x0e2a
+#define mmDC_PERFMON13_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON13_PERFMON_HI 0x0e2b
+#define mmDC_PERFMON13_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON13_PERFMON_LOW 0x0e2c
+#define mmDC_PERFMON13_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp1_dispdec_dpp_top_dispdec
+// base address: 0x5ac
+#define mmDPP_TOP1_DPP_CONTROL 0x0e30
+#define mmDPP_TOP1_DPP_CONTROL_BASE_IDX 2
+#define mmDPP_TOP1_DPP_SOFT_RESET 0x0e31
+#define mmDPP_TOP1_DPP_SOFT_RESET_BASE_IDX 2
+#define mmDPP_TOP1_DPP_CRC_VAL_R_G 0x0e32
+#define mmDPP_TOP1_DPP_CRC_VAL_R_G_BASE_IDX 2
+#define mmDPP_TOP1_DPP_CRC_VAL_B_A 0x0e33
+#define mmDPP_TOP1_DPP_CRC_VAL_B_A_BASE_IDX 2
+#define mmDPP_TOP1_DPP_CRC_CTRL 0x0e34
+#define mmDPP_TOP1_DPP_CRC_CTRL_BASE_IDX 2
+#define mmDPP_TOP1_HOST_READ_CONTROL 0x0e35
+#define mmDPP_TOP1_HOST_READ_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp1_dispdec_cnvc_cfg_dispdec
+// base address: 0x5ac
+#define mmCNVC_CFG1_CNVC_SURFACE_PIXEL_FORMAT 0x0e3a
+#define mmCNVC_CFG1_CNVC_SURFACE_PIXEL_FORMAT_BASE_IDX 2
+#define mmCNVC_CFG1_FORMAT_CONTROL 0x0e3b
+#define mmCNVC_CFG1_FORMAT_CONTROL_BASE_IDX 2
+#define mmCNVC_CFG1_FCNV_FP_BIAS_R 0x0e3c
+#define mmCNVC_CFG1_FCNV_FP_BIAS_R_BASE_IDX 2
+#define mmCNVC_CFG1_FCNV_FP_BIAS_G 0x0e3d
+#define mmCNVC_CFG1_FCNV_FP_BIAS_G_BASE_IDX 2
+#define mmCNVC_CFG1_FCNV_FP_BIAS_B 0x0e3e
+#define mmCNVC_CFG1_FCNV_FP_BIAS_B_BASE_IDX 2
+#define mmCNVC_CFG1_FCNV_FP_SCALE_R 0x0e3f
+#define mmCNVC_CFG1_FCNV_FP_SCALE_R_BASE_IDX 2
+#define mmCNVC_CFG1_FCNV_FP_SCALE_G 0x0e40
+#define mmCNVC_CFG1_FCNV_FP_SCALE_G_BASE_IDX 2
+#define mmCNVC_CFG1_FCNV_FP_SCALE_B 0x0e41
+#define mmCNVC_CFG1_FCNV_FP_SCALE_B_BASE_IDX 2
+#define mmCNVC_CFG1_COLOR_KEYER_CONTROL 0x0e42
+#define mmCNVC_CFG1_COLOR_KEYER_CONTROL_BASE_IDX 2
+#define mmCNVC_CFG1_COLOR_KEYER_ALPHA 0x0e43
+#define mmCNVC_CFG1_COLOR_KEYER_ALPHA_BASE_IDX 2
+#define mmCNVC_CFG1_COLOR_KEYER_RED 0x0e44
+#define mmCNVC_CFG1_COLOR_KEYER_RED_BASE_IDX 2
+#define mmCNVC_CFG1_COLOR_KEYER_GREEN 0x0e45
+#define mmCNVC_CFG1_COLOR_KEYER_GREEN_BASE_IDX 2
+#define mmCNVC_CFG1_COLOR_KEYER_BLUE 0x0e46
+#define mmCNVC_CFG1_COLOR_KEYER_BLUE_BASE_IDX 2
+#define mmCNVC_CFG1_ALPHA_2BIT_LUT 0x0e48
+#define mmCNVC_CFG1_ALPHA_2BIT_LUT_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp1_dispdec_cnvc_cur_dispdec
+// base address: 0x5ac
+#define mmCNVC_CUR1_CURSOR0_CONTROL 0x0e4b
+#define mmCNVC_CUR1_CURSOR0_CONTROL_BASE_IDX 2
+#define mmCNVC_CUR1_CURSOR0_COLOR0 0x0e4c
+#define mmCNVC_CUR1_CURSOR0_COLOR0_BASE_IDX 2
+#define mmCNVC_CUR1_CURSOR0_COLOR1 0x0e4d
+#define mmCNVC_CUR1_CURSOR0_COLOR1_BASE_IDX 2
+#define mmCNVC_CUR1_CURSOR0_FP_SCALE_BIAS 0x0e4e
+#define mmCNVC_CUR1_CURSOR0_FP_SCALE_BIAS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp1_dispdec_dscl_dispdec
+// base address: 0x5ac
+#define mmDSCL1_SCL_COEF_RAM_TAP_SELECT 0x0e55
+#define mmDSCL1_SCL_COEF_RAM_TAP_SELECT_BASE_IDX 2
+#define mmDSCL1_SCL_COEF_RAM_TAP_DATA 0x0e56
+#define mmDSCL1_SCL_COEF_RAM_TAP_DATA_BASE_IDX 2
+#define mmDSCL1_SCL_MODE 0x0e57
+#define mmDSCL1_SCL_MODE_BASE_IDX 2
+#define mmDSCL1_SCL_TAP_CONTROL 0x0e58
+#define mmDSCL1_SCL_TAP_CONTROL_BASE_IDX 2
+#define mmDSCL1_DSCL_CONTROL 0x0e59
+#define mmDSCL1_DSCL_CONTROL_BASE_IDX 2
+#define mmDSCL1_DSCL_2TAP_CONTROL 0x0e5a
+#define mmDSCL1_DSCL_2TAP_CONTROL_BASE_IDX 2
+#define mmDSCL1_SCL_MANUAL_REPLICATE_CONTROL 0x0e5b
+#define mmDSCL1_SCL_MANUAL_REPLICATE_CONTROL_BASE_IDX 2
+#define mmDSCL1_SCL_HORZ_FILTER_SCALE_RATIO 0x0e5c
+#define mmDSCL1_SCL_HORZ_FILTER_SCALE_RATIO_BASE_IDX 2
+#define mmDSCL1_SCL_HORZ_FILTER_INIT 0x0e5d
+#define mmDSCL1_SCL_HORZ_FILTER_INIT_BASE_IDX 2
+#define mmDSCL1_SCL_HORZ_FILTER_SCALE_RATIO_C 0x0e5e
+#define mmDSCL1_SCL_HORZ_FILTER_SCALE_RATIO_C_BASE_IDX 2
+#define mmDSCL1_SCL_HORZ_FILTER_INIT_C 0x0e5f
+#define mmDSCL1_SCL_HORZ_FILTER_INIT_C_BASE_IDX 2
+#define mmDSCL1_SCL_VERT_FILTER_SCALE_RATIO 0x0e60
+#define mmDSCL1_SCL_VERT_FILTER_SCALE_RATIO_BASE_IDX 2
+#define mmDSCL1_SCL_VERT_FILTER_INIT 0x0e61
+#define mmDSCL1_SCL_VERT_FILTER_INIT_BASE_IDX 2
+#define mmDSCL1_SCL_VERT_FILTER_INIT_BOT 0x0e62
+#define mmDSCL1_SCL_VERT_FILTER_INIT_BOT_BASE_IDX 2
+#define mmDSCL1_SCL_VERT_FILTER_SCALE_RATIO_C 0x0e63
+#define mmDSCL1_SCL_VERT_FILTER_SCALE_RATIO_C_BASE_IDX 2
+#define mmDSCL1_SCL_VERT_FILTER_INIT_C 0x0e64
+#define mmDSCL1_SCL_VERT_FILTER_INIT_C_BASE_IDX 2
+#define mmDSCL1_SCL_VERT_FILTER_INIT_BOT_C 0x0e65
+#define mmDSCL1_SCL_VERT_FILTER_INIT_BOT_C_BASE_IDX 2
+#define mmDSCL1_SCL_BLACK_OFFSET 0x0e66
+#define mmDSCL1_SCL_BLACK_OFFSET_BASE_IDX 2
+#define mmDSCL1_DSCL_UPDATE 0x0e67
+#define mmDSCL1_DSCL_UPDATE_BASE_IDX 2
+#define mmDSCL1_DSCL_AUTOCAL 0x0e68
+#define mmDSCL1_DSCL_AUTOCAL_BASE_IDX 2
+#define mmDSCL1_DSCL_EXT_OVERSCAN_LEFT_RIGHT 0x0e69
+#define mmDSCL1_DSCL_EXT_OVERSCAN_LEFT_RIGHT_BASE_IDX 2
+#define mmDSCL1_DSCL_EXT_OVERSCAN_TOP_BOTTOM 0x0e6a
+#define mmDSCL1_DSCL_EXT_OVERSCAN_TOP_BOTTOM_BASE_IDX 2
+#define mmDSCL1_OTG_H_BLANK 0x0e6b
+#define mmDSCL1_OTG_H_BLANK_BASE_IDX 2
+#define mmDSCL1_OTG_V_BLANK 0x0e6c
+#define mmDSCL1_OTG_V_BLANK_BASE_IDX 2
+#define mmDSCL1_RECOUT_START 0x0e6d
+#define mmDSCL1_RECOUT_START_BASE_IDX 2
+#define mmDSCL1_RECOUT_SIZE 0x0e6e
+#define mmDSCL1_RECOUT_SIZE_BASE_IDX 2
+#define mmDSCL1_MPC_SIZE 0x0e6f
+#define mmDSCL1_MPC_SIZE_BASE_IDX 2
+#define mmDSCL1_LB_DATA_FORMAT 0x0e70
+#define mmDSCL1_LB_DATA_FORMAT_BASE_IDX 2
+#define mmDSCL1_LB_MEMORY_CTRL 0x0e71
+#define mmDSCL1_LB_MEMORY_CTRL_BASE_IDX 2
+#define mmDSCL1_LB_V_COUNTER 0x0e72
+#define mmDSCL1_LB_V_COUNTER_BASE_IDX 2
+#define mmDSCL1_DSCL_MEM_PWR_CTRL 0x0e73
+#define mmDSCL1_DSCL_MEM_PWR_CTRL_BASE_IDX 2
+#define mmDSCL1_DSCL_MEM_PWR_STATUS 0x0e74
+#define mmDSCL1_DSCL_MEM_PWR_STATUS_BASE_IDX 2
+#define mmDSCL1_OBUF_CONTROL 0x0e75
+#define mmDSCL1_OBUF_CONTROL_BASE_IDX 2
+#define mmDSCL1_OBUF_MEM_PWR_CTRL 0x0e76
+#define mmDSCL1_OBUF_MEM_PWR_CTRL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp1_dispdec_cm_dispdec
+// base address: 0x5ac
+#define mmCM1_CM_CONTROL 0x0e85
+#define mmCM1_CM_CONTROL_BASE_IDX 2
+#define mmCM1_CM_ICSC_CONTROL 0x0e86
+#define mmCM1_CM_ICSC_CONTROL_BASE_IDX 2
+#define mmCM1_CM_ICSC_C11_C12 0x0e87
+#define mmCM1_CM_ICSC_C11_C12_BASE_IDX 2
+#define mmCM1_CM_ICSC_C13_C14 0x0e88
+#define mmCM1_CM_ICSC_C13_C14_BASE_IDX 2
+#define mmCM1_CM_ICSC_C21_C22 0x0e89
+#define mmCM1_CM_ICSC_C21_C22_BASE_IDX 2
+#define mmCM1_CM_ICSC_C23_C24 0x0e8a
+#define mmCM1_CM_ICSC_C23_C24_BASE_IDX 2
+#define mmCM1_CM_ICSC_C31_C32 0x0e8b
+#define mmCM1_CM_ICSC_C31_C32_BASE_IDX 2
+#define mmCM1_CM_ICSC_C33_C34 0x0e8c
+#define mmCM1_CM_ICSC_C33_C34_BASE_IDX 2
+#define mmCM1_CM_ICSC_B_C11_C12 0x0e8d
+#define mmCM1_CM_ICSC_B_C11_C12_BASE_IDX 2
+#define mmCM1_CM_ICSC_B_C13_C14 0x0e8e
+#define mmCM1_CM_ICSC_B_C13_C14_BASE_IDX 2
+#define mmCM1_CM_ICSC_B_C21_C22 0x0e8f
+#define mmCM1_CM_ICSC_B_C21_C22_BASE_IDX 2
+#define mmCM1_CM_ICSC_B_C23_C24 0x0e90
+#define mmCM1_CM_ICSC_B_C23_C24_BASE_IDX 2
+#define mmCM1_CM_ICSC_B_C31_C32 0x0e91
+#define mmCM1_CM_ICSC_B_C31_C32_BASE_IDX 2
+#define mmCM1_CM_ICSC_B_C33_C34 0x0e92
+#define mmCM1_CM_ICSC_B_C33_C34_BASE_IDX 2
+#define mmCM1_CM_GAMUT_REMAP_CONTROL 0x0e93
+#define mmCM1_CM_GAMUT_REMAP_CONTROL_BASE_IDX 2
+#define mmCM1_CM_GAMUT_REMAP_C11_C12 0x0e94
+#define mmCM1_CM_GAMUT_REMAP_C11_C12_BASE_IDX 2
+#define mmCM1_CM_GAMUT_REMAP_C13_C14 0x0e95
+#define mmCM1_CM_GAMUT_REMAP_C13_C14_BASE_IDX 2
+#define mmCM1_CM_GAMUT_REMAP_C21_C22 0x0e96
+#define mmCM1_CM_GAMUT_REMAP_C21_C22_BASE_IDX 2
+#define mmCM1_CM_GAMUT_REMAP_C23_C24 0x0e97
+#define mmCM1_CM_GAMUT_REMAP_C23_C24_BASE_IDX 2
+#define mmCM1_CM_GAMUT_REMAP_C31_C32 0x0e98
+#define mmCM1_CM_GAMUT_REMAP_C31_C32_BASE_IDX 2
+#define mmCM1_CM_GAMUT_REMAP_C33_C34 0x0e99
+#define mmCM1_CM_GAMUT_REMAP_C33_C34_BASE_IDX 2
+#define mmCM1_CM_GAMUT_REMAP_B_C11_C12 0x0e9a
+#define mmCM1_CM_GAMUT_REMAP_B_C11_C12_BASE_IDX 2
+#define mmCM1_CM_GAMUT_REMAP_B_C13_C14 0x0e9b
+#define mmCM1_CM_GAMUT_REMAP_B_C13_C14_BASE_IDX 2
+#define mmCM1_CM_GAMUT_REMAP_B_C21_C22 0x0e9c
+#define mmCM1_CM_GAMUT_REMAP_B_C21_C22_BASE_IDX 2
+#define mmCM1_CM_GAMUT_REMAP_B_C23_C24 0x0e9d
+#define mmCM1_CM_GAMUT_REMAP_B_C23_C24_BASE_IDX 2
+#define mmCM1_CM_GAMUT_REMAP_B_C31_C32 0x0e9e
+#define mmCM1_CM_GAMUT_REMAP_B_C31_C32_BASE_IDX 2
+#define mmCM1_CM_GAMUT_REMAP_B_C33_C34 0x0e9f
+#define mmCM1_CM_GAMUT_REMAP_B_C33_C34_BASE_IDX 2
+#define mmCM1_CM_BIAS_CR_R 0x0ea0
+#define mmCM1_CM_BIAS_CR_R_BASE_IDX 2
+#define mmCM1_CM_BIAS_Y_G_CB_B 0x0ea1
+#define mmCM1_CM_BIAS_Y_G_CB_B_BASE_IDX 2
+#define mmCM1_CM_DGAM_CONTROL 0x0ea2
+#define mmCM1_CM_DGAM_CONTROL_BASE_IDX 2
+#define mmCM1_CM_DGAM_LUT_INDEX 0x0ea3
+#define mmCM1_CM_DGAM_LUT_INDEX_BASE_IDX 2
+#define mmCM1_CM_DGAM_LUT_DATA 0x0ea4
+#define mmCM1_CM_DGAM_LUT_DATA_BASE_IDX 2
+#define mmCM1_CM_DGAM_LUT_WRITE_EN_MASK 0x0ea5
+#define mmCM1_CM_DGAM_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_START_CNTL_B 0x0ea6
+#define mmCM1_CM_DGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_START_CNTL_G 0x0ea7
+#define mmCM1_CM_DGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_START_CNTL_R 0x0ea8
+#define mmCM1_CM_DGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_SLOPE_CNTL_B 0x0ea9
+#define mmCM1_CM_DGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_SLOPE_CNTL_G 0x0eaa
+#define mmCM1_CM_DGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_SLOPE_CNTL_R 0x0eab
+#define mmCM1_CM_DGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_END_CNTL1_B 0x0eac
+#define mmCM1_CM_DGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_END_CNTL2_B 0x0ead
+#define mmCM1_CM_DGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_END_CNTL1_G 0x0eae
+#define mmCM1_CM_DGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_END_CNTL2_G 0x0eaf
+#define mmCM1_CM_DGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_END_CNTL1_R 0x0eb0
+#define mmCM1_CM_DGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_END_CNTL2_R 0x0eb1
+#define mmCM1_CM_DGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_REGION_0_1 0x0eb2
+#define mmCM1_CM_DGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_REGION_2_3 0x0eb3
+#define mmCM1_CM_DGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_REGION_4_5 0x0eb4
+#define mmCM1_CM_DGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_REGION_6_7 0x0eb5
+#define mmCM1_CM_DGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_REGION_8_9 0x0eb6
+#define mmCM1_CM_DGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_REGION_10_11 0x0eb7
+#define mmCM1_CM_DGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_REGION_12_13 0x0eb8
+#define mmCM1_CM_DGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMA_REGION_14_15 0x0eb9
+#define mmCM1_CM_DGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_START_CNTL_B 0x0eba
+#define mmCM1_CM_DGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_START_CNTL_G 0x0ebb
+#define mmCM1_CM_DGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_START_CNTL_R 0x0ebc
+#define mmCM1_CM_DGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_SLOPE_CNTL_B 0x0ebd
+#define mmCM1_CM_DGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_SLOPE_CNTL_G 0x0ebe
+#define mmCM1_CM_DGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_SLOPE_CNTL_R 0x0ebf
+#define mmCM1_CM_DGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_END_CNTL1_B 0x0ec0
+#define mmCM1_CM_DGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_END_CNTL2_B 0x0ec1
+#define mmCM1_CM_DGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_END_CNTL1_G 0x0ec2
+#define mmCM1_CM_DGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_END_CNTL2_G 0x0ec3
+#define mmCM1_CM_DGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_END_CNTL1_R 0x0ec4
+#define mmCM1_CM_DGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_END_CNTL2_R 0x0ec5
+#define mmCM1_CM_DGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_REGION_0_1 0x0ec6
+#define mmCM1_CM_DGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_REGION_2_3 0x0ec7
+#define mmCM1_CM_DGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_REGION_4_5 0x0ec8
+#define mmCM1_CM_DGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_REGION_6_7 0x0ec9
+#define mmCM1_CM_DGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_REGION_8_9 0x0eca
+#define mmCM1_CM_DGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_REGION_10_11 0x0ecb
+#define mmCM1_CM_DGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_REGION_12_13 0x0ecc
+#define mmCM1_CM_DGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM1_CM_DGAM_RAMB_REGION_14_15 0x0ecd
+#define mmCM1_CM_DGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_CONTROL 0x0ece
+#define mmCM1_CM_BLNDGAM_CONTROL_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_LUT_INDEX 0x0ecf
+#define mmCM1_CM_BLNDGAM_LUT_INDEX_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_LUT_DATA 0x0ed0
+#define mmCM1_CM_BLNDGAM_LUT_DATA_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_LUT_WRITE_EN_MASK 0x0ed1
+#define mmCM1_CM_BLNDGAM_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_START_CNTL_B 0x0ed2
+#define mmCM1_CM_BLNDGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_START_CNTL_G 0x0ed3
+#define mmCM1_CM_BLNDGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_START_CNTL_R 0x0ed4
+#define mmCM1_CM_BLNDGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_SLOPE_CNTL_B 0x0ed5
+#define mmCM1_CM_BLNDGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_SLOPE_CNTL_G 0x0ed6
+#define mmCM1_CM_BLNDGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_SLOPE_CNTL_R 0x0ed7
+#define mmCM1_CM_BLNDGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_END_CNTL1_B 0x0ed8
+#define mmCM1_CM_BLNDGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_END_CNTL2_B 0x0ed9
+#define mmCM1_CM_BLNDGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_END_CNTL1_G 0x0eda
+#define mmCM1_CM_BLNDGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_END_CNTL2_G 0x0edb
+#define mmCM1_CM_BLNDGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_END_CNTL1_R 0x0edc
+#define mmCM1_CM_BLNDGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_END_CNTL2_R 0x0edd
+#define mmCM1_CM_BLNDGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_0_1 0x0ede
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_2_3 0x0edf
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_4_5 0x0ee0
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_6_7 0x0ee1
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_8_9 0x0ee2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_10_11 0x0ee3
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_12_13 0x0ee4
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_14_15 0x0ee5
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_16_17 0x0ee6
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_16_17_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_18_19 0x0ee7
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_18_19_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_20_21 0x0ee8
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_20_21_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_22_23 0x0ee9
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_22_23_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_24_25 0x0eea
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_24_25_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_26_27 0x0eeb
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_26_27_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_28_29 0x0eec
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_28_29_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_30_31 0x0eed
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_30_31_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_32_33 0x0eee
+#define mmCM1_CM_BLNDGAM_RAMA_REGION_32_33_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_START_CNTL_B 0x0eef
+#define mmCM1_CM_BLNDGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_START_CNTL_G 0x0ef0
+#define mmCM1_CM_BLNDGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_START_CNTL_R 0x0ef1
+#define mmCM1_CM_BLNDGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_SLOPE_CNTL_B 0x0ef2
+#define mmCM1_CM_BLNDGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_SLOPE_CNTL_G 0x0ef3
+#define mmCM1_CM_BLNDGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_SLOPE_CNTL_R 0x0ef4
+#define mmCM1_CM_BLNDGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_END_CNTL1_B 0x0ef5
+#define mmCM1_CM_BLNDGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_END_CNTL2_B 0x0ef6
+#define mmCM1_CM_BLNDGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_END_CNTL1_G 0x0ef7
+#define mmCM1_CM_BLNDGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_END_CNTL2_G 0x0ef8
+#define mmCM1_CM_BLNDGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_END_CNTL1_R 0x0ef9
+#define mmCM1_CM_BLNDGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_END_CNTL2_R 0x0efa
+#define mmCM1_CM_BLNDGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_0_1 0x0efb
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_2_3 0x0efc
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_4_5 0x0efd
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_6_7 0x0efe
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_8_9 0x0eff
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_10_11 0x0f00
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_12_13 0x0f01
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_14_15 0x0f02
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_16_17 0x0f03
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_16_17_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_18_19 0x0f04
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_18_19_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_20_21 0x0f05
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_20_21_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_22_23 0x0f06
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_22_23_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_24_25 0x0f07
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_24_25_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_26_27 0x0f08
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_26_27_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_28_29 0x0f09
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_28_29_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_30_31 0x0f0a
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_30_31_BASE_IDX 2
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_32_33 0x0f0b
+#define mmCM1_CM_BLNDGAM_RAMB_REGION_32_33_BASE_IDX 2
+#define mmCM1_CM_HDR_MULT_COEF 0x0f0c
+#define mmCM1_CM_HDR_MULT_COEF_BASE_IDX 2
+#define mmCM1_CM_MEM_PWR_CTRL 0x0f0d
+#define mmCM1_CM_MEM_PWR_CTRL_BASE_IDX 2
+#define mmCM1_CM_MEM_PWR_STATUS 0x0f0e
+#define mmCM1_CM_MEM_PWR_STATUS_BASE_IDX 2
+#define mmCM1_CM_DEALPHA 0x0f10
+#define mmCM1_CM_DEALPHA_BASE_IDX 2
+#define mmCM1_CM_COEF_FORMAT 0x0f11
+#define mmCM1_CM_COEF_FORMAT_BASE_IDX 2
+#define mmCM1_CM_SHAPER_CONTROL 0x0f12
+#define mmCM1_CM_SHAPER_CONTROL_BASE_IDX 2
+#define mmCM1_CM_SHAPER_OFFSET_R 0x0f13
+#define mmCM1_CM_SHAPER_OFFSET_R_BASE_IDX 2
+#define mmCM1_CM_SHAPER_OFFSET_G 0x0f14
+#define mmCM1_CM_SHAPER_OFFSET_G_BASE_IDX 2
+#define mmCM1_CM_SHAPER_OFFSET_B 0x0f15
+#define mmCM1_CM_SHAPER_OFFSET_B_BASE_IDX 2
+#define mmCM1_CM_SHAPER_SCALE_R 0x0f16
+#define mmCM1_CM_SHAPER_SCALE_R_BASE_IDX 2
+#define mmCM1_CM_SHAPER_SCALE_G_B 0x0f17
+#define mmCM1_CM_SHAPER_SCALE_G_B_BASE_IDX 2
+#define mmCM1_CM_SHAPER_LUT_INDEX 0x0f18
+#define mmCM1_CM_SHAPER_LUT_INDEX_BASE_IDX 2
+#define mmCM1_CM_SHAPER_LUT_DATA 0x0f19
+#define mmCM1_CM_SHAPER_LUT_DATA_BASE_IDX 2
+#define mmCM1_CM_SHAPER_LUT_WRITE_EN_MASK 0x0f1a
+#define mmCM1_CM_SHAPER_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_START_CNTL_B 0x0f1b
+#define mmCM1_CM_SHAPER_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_START_CNTL_G 0x0f1c
+#define mmCM1_CM_SHAPER_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_START_CNTL_R 0x0f1d
+#define mmCM1_CM_SHAPER_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_END_CNTL_B 0x0f1e
+#define mmCM1_CM_SHAPER_RAMA_END_CNTL_B_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_END_CNTL_G 0x0f1f
+#define mmCM1_CM_SHAPER_RAMA_END_CNTL_G_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_END_CNTL_R 0x0f20
+#define mmCM1_CM_SHAPER_RAMA_END_CNTL_R_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_REGION_0_1 0x0f21
+#define mmCM1_CM_SHAPER_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_REGION_2_3 0x0f22
+#define mmCM1_CM_SHAPER_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_REGION_4_5 0x0f23
+#define mmCM1_CM_SHAPER_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_REGION_6_7 0x0f24
+#define mmCM1_CM_SHAPER_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_REGION_8_9 0x0f25
+#define mmCM1_CM_SHAPER_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_REGION_10_11 0x0f26
+#define mmCM1_CM_SHAPER_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_REGION_12_13 0x0f27
+#define mmCM1_CM_SHAPER_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_REGION_14_15 0x0f28
+#define mmCM1_CM_SHAPER_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_REGION_16_17 0x0f29
+#define mmCM1_CM_SHAPER_RAMA_REGION_16_17_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_REGION_18_19 0x0f2a
+#define mmCM1_CM_SHAPER_RAMA_REGION_18_19_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_REGION_20_21 0x0f2b
+#define mmCM1_CM_SHAPER_RAMA_REGION_20_21_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_REGION_22_23 0x0f2c
+#define mmCM1_CM_SHAPER_RAMA_REGION_22_23_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_REGION_24_25 0x0f2d
+#define mmCM1_CM_SHAPER_RAMA_REGION_24_25_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_REGION_26_27 0x0f2e
+#define mmCM1_CM_SHAPER_RAMA_REGION_26_27_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_REGION_28_29 0x0f2f
+#define mmCM1_CM_SHAPER_RAMA_REGION_28_29_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_REGION_30_31 0x0f30
+#define mmCM1_CM_SHAPER_RAMA_REGION_30_31_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMA_REGION_32_33 0x0f31
+#define mmCM1_CM_SHAPER_RAMA_REGION_32_33_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_START_CNTL_B 0x0f32
+#define mmCM1_CM_SHAPER_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_START_CNTL_G 0x0f33
+#define mmCM1_CM_SHAPER_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_START_CNTL_R 0x0f34
+#define mmCM1_CM_SHAPER_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_END_CNTL_B 0x0f35
+#define mmCM1_CM_SHAPER_RAMB_END_CNTL_B_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_END_CNTL_G 0x0f36
+#define mmCM1_CM_SHAPER_RAMB_END_CNTL_G_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_END_CNTL_R 0x0f37
+#define mmCM1_CM_SHAPER_RAMB_END_CNTL_R_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_REGION_0_1 0x0f38
+#define mmCM1_CM_SHAPER_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_REGION_2_3 0x0f39
+#define mmCM1_CM_SHAPER_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_REGION_4_5 0x0f3a
+#define mmCM1_CM_SHAPER_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_REGION_6_7 0x0f3b
+#define mmCM1_CM_SHAPER_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_REGION_8_9 0x0f3c
+#define mmCM1_CM_SHAPER_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_REGION_10_11 0x0f3d
+#define mmCM1_CM_SHAPER_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_REGION_12_13 0x0f3e
+#define mmCM1_CM_SHAPER_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_REGION_14_15 0x0f3f
+#define mmCM1_CM_SHAPER_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_REGION_16_17 0x0f40
+#define mmCM1_CM_SHAPER_RAMB_REGION_16_17_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_REGION_18_19 0x0f41
+#define mmCM1_CM_SHAPER_RAMB_REGION_18_19_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_REGION_20_21 0x0f42
+#define mmCM1_CM_SHAPER_RAMB_REGION_20_21_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_REGION_22_23 0x0f43
+#define mmCM1_CM_SHAPER_RAMB_REGION_22_23_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_REGION_24_25 0x0f44
+#define mmCM1_CM_SHAPER_RAMB_REGION_24_25_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_REGION_26_27 0x0f45
+#define mmCM1_CM_SHAPER_RAMB_REGION_26_27_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_REGION_28_29 0x0f46
+#define mmCM1_CM_SHAPER_RAMB_REGION_28_29_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_REGION_30_31 0x0f47
+#define mmCM1_CM_SHAPER_RAMB_REGION_30_31_BASE_IDX 2
+#define mmCM1_CM_SHAPER_RAMB_REGION_32_33 0x0f48
+#define mmCM1_CM_SHAPER_RAMB_REGION_32_33_BASE_IDX 2
+#define mmCM1_CM_MEM_PWR_CTRL2 0x0f49
+#define mmCM1_CM_MEM_PWR_CTRL2_BASE_IDX 2
+#define mmCM1_CM_MEM_PWR_STATUS2 0x0f4a
+#define mmCM1_CM_MEM_PWR_STATUS2_BASE_IDX 2
+#define mmCM1_CM_3DLUT_MODE 0x0f4b
+#define mmCM1_CM_3DLUT_MODE_BASE_IDX 2
+#define mmCM1_CM_3DLUT_INDEX 0x0f4c
+#define mmCM1_CM_3DLUT_INDEX_BASE_IDX 2
+#define mmCM1_CM_3DLUT_DATA 0x0f4d
+#define mmCM1_CM_3DLUT_DATA_BASE_IDX 2
+#define mmCM1_CM_3DLUT_DATA_30BIT 0x0f4e
+#define mmCM1_CM_3DLUT_DATA_30BIT_BASE_IDX 2
+#define mmCM1_CM_3DLUT_READ_WRITE_CONTROL 0x0f4f
+#define mmCM1_CM_3DLUT_READ_WRITE_CONTROL_BASE_IDX 2
+#define mmCM1_CM_3DLUT_OUT_NORM_FACTOR 0x0f50
+#define mmCM1_CM_3DLUT_OUT_NORM_FACTOR_BASE_IDX 2
+#define mmCM1_CM_3DLUT_OUT_OFFSET_R 0x0f51
+#define mmCM1_CM_3DLUT_OUT_OFFSET_R_BASE_IDX 2
+#define mmCM1_CM_3DLUT_OUT_OFFSET_G 0x0f52
+#define mmCM1_CM_3DLUT_OUT_OFFSET_G_BASE_IDX 2
+#define mmCM1_CM_3DLUT_OUT_OFFSET_B 0x0f53
+#define mmCM1_CM_3DLUT_OUT_OFFSET_B_BASE_IDX 2
+#define mmCM1_CM_TEST_DEBUG_INDEX 0x0f54
+#define mmCM1_CM_TEST_DEBUG_INDEX_BASE_IDX 2
+#define mmCM1_CM_TEST_DEBUG_DATA 0x0f55
+#define mmCM1_CM_TEST_DEBUG_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp1_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x3e3c
+#define mmDC_PERFMON14_PERFCOUNTER_CNTL 0x0f8f
+#define mmDC_PERFMON14_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON14_PERFCOUNTER_CNTL2 0x0f90
+#define mmDC_PERFMON14_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON14_PERFCOUNTER_STATE 0x0f91
+#define mmDC_PERFMON14_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON14_PERFMON_CNTL 0x0f92
+#define mmDC_PERFMON14_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON14_PERFMON_CNTL2 0x0f93
+#define mmDC_PERFMON14_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON14_PERFMON_CVALUE_INT_MISC 0x0f94
+#define mmDC_PERFMON14_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON14_PERFMON_CVALUE_LOW 0x0f95
+#define mmDC_PERFMON14_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON14_PERFMON_HI 0x0f96
+#define mmDC_PERFMON14_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON14_PERFMON_LOW 0x0f97
+#define mmDC_PERFMON14_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp2_dispdec_dpp_top_dispdec
+// base address: 0xb58
+#define mmDPP_TOP2_DPP_CONTROL 0x0f9b
+#define mmDPP_TOP2_DPP_CONTROL_BASE_IDX 2
+#define mmDPP_TOP2_DPP_SOFT_RESET 0x0f9c
+#define mmDPP_TOP2_DPP_SOFT_RESET_BASE_IDX 2
+#define mmDPP_TOP2_DPP_CRC_VAL_R_G 0x0f9d
+#define mmDPP_TOP2_DPP_CRC_VAL_R_G_BASE_IDX 2
+#define mmDPP_TOP2_DPP_CRC_VAL_B_A 0x0f9e
+#define mmDPP_TOP2_DPP_CRC_VAL_B_A_BASE_IDX 2
+#define mmDPP_TOP2_DPP_CRC_CTRL 0x0f9f
+#define mmDPP_TOP2_DPP_CRC_CTRL_BASE_IDX 2
+#define mmDPP_TOP2_HOST_READ_CONTROL 0x0fa0
+#define mmDPP_TOP2_HOST_READ_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp2_dispdec_cnvc_cfg_dispdec
+// base address: 0xb58
+#define mmCNVC_CFG2_CNVC_SURFACE_PIXEL_FORMAT 0x0fa5
+#define mmCNVC_CFG2_CNVC_SURFACE_PIXEL_FORMAT_BASE_IDX 2
+#define mmCNVC_CFG2_FORMAT_CONTROL 0x0fa6
+#define mmCNVC_CFG2_FORMAT_CONTROL_BASE_IDX 2
+#define mmCNVC_CFG2_FCNV_FP_BIAS_R 0x0fa7
+#define mmCNVC_CFG2_FCNV_FP_BIAS_R_BASE_IDX 2
+#define mmCNVC_CFG2_FCNV_FP_BIAS_G 0x0fa8
+#define mmCNVC_CFG2_FCNV_FP_BIAS_G_BASE_IDX 2
+#define mmCNVC_CFG2_FCNV_FP_BIAS_B 0x0fa9
+#define mmCNVC_CFG2_FCNV_FP_BIAS_B_BASE_IDX 2
+#define mmCNVC_CFG2_FCNV_FP_SCALE_R 0x0faa
+#define mmCNVC_CFG2_FCNV_FP_SCALE_R_BASE_IDX 2
+#define mmCNVC_CFG2_FCNV_FP_SCALE_G 0x0fab
+#define mmCNVC_CFG2_FCNV_FP_SCALE_G_BASE_IDX 2
+#define mmCNVC_CFG2_FCNV_FP_SCALE_B 0x0fac
+#define mmCNVC_CFG2_FCNV_FP_SCALE_B_BASE_IDX 2
+#define mmCNVC_CFG2_COLOR_KEYER_CONTROL 0x0fad
+#define mmCNVC_CFG2_COLOR_KEYER_CONTROL_BASE_IDX 2
+#define mmCNVC_CFG2_COLOR_KEYER_ALPHA 0x0fae
+#define mmCNVC_CFG2_COLOR_KEYER_ALPHA_BASE_IDX 2
+#define mmCNVC_CFG2_COLOR_KEYER_RED 0x0faf
+#define mmCNVC_CFG2_COLOR_KEYER_RED_BASE_IDX 2
+#define mmCNVC_CFG2_COLOR_KEYER_GREEN 0x0fb0
+#define mmCNVC_CFG2_COLOR_KEYER_GREEN_BASE_IDX 2
+#define mmCNVC_CFG2_COLOR_KEYER_BLUE 0x0fb1
+#define mmCNVC_CFG2_COLOR_KEYER_BLUE_BASE_IDX 2
+#define mmCNVC_CFG2_ALPHA_2BIT_LUT 0x0fb3
+#define mmCNVC_CFG2_ALPHA_2BIT_LUT_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp2_dispdec_cnvc_cur_dispdec
+// base address: 0xb58
+#define mmCNVC_CUR2_CURSOR0_CONTROL 0x0fb6
+#define mmCNVC_CUR2_CURSOR0_CONTROL_BASE_IDX 2
+#define mmCNVC_CUR2_CURSOR0_COLOR0 0x0fb7
+#define mmCNVC_CUR2_CURSOR0_COLOR0_BASE_IDX 2
+#define mmCNVC_CUR2_CURSOR0_COLOR1 0x0fb8
+#define mmCNVC_CUR2_CURSOR0_COLOR1_BASE_IDX 2
+#define mmCNVC_CUR2_CURSOR0_FP_SCALE_BIAS 0x0fb9
+#define mmCNVC_CUR2_CURSOR0_FP_SCALE_BIAS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp2_dispdec_dscl_dispdec
+// base address: 0xb58
+#define mmDSCL2_SCL_COEF_RAM_TAP_SELECT 0x0fc0
+#define mmDSCL2_SCL_COEF_RAM_TAP_SELECT_BASE_IDX 2
+#define mmDSCL2_SCL_COEF_RAM_TAP_DATA 0x0fc1
+#define mmDSCL2_SCL_COEF_RAM_TAP_DATA_BASE_IDX 2
+#define mmDSCL2_SCL_MODE 0x0fc2
+#define mmDSCL2_SCL_MODE_BASE_IDX 2
+#define mmDSCL2_SCL_TAP_CONTROL 0x0fc3
+#define mmDSCL2_SCL_TAP_CONTROL_BASE_IDX 2
+#define mmDSCL2_DSCL_CONTROL 0x0fc4
+#define mmDSCL2_DSCL_CONTROL_BASE_IDX 2
+#define mmDSCL2_DSCL_2TAP_CONTROL 0x0fc5
+#define mmDSCL2_DSCL_2TAP_CONTROL_BASE_IDX 2
+#define mmDSCL2_SCL_MANUAL_REPLICATE_CONTROL 0x0fc6
+#define mmDSCL2_SCL_MANUAL_REPLICATE_CONTROL_BASE_IDX 2
+#define mmDSCL2_SCL_HORZ_FILTER_SCALE_RATIO 0x0fc7
+#define mmDSCL2_SCL_HORZ_FILTER_SCALE_RATIO_BASE_IDX 2
+#define mmDSCL2_SCL_HORZ_FILTER_INIT 0x0fc8
+#define mmDSCL2_SCL_HORZ_FILTER_INIT_BASE_IDX 2
+#define mmDSCL2_SCL_HORZ_FILTER_SCALE_RATIO_C 0x0fc9
+#define mmDSCL2_SCL_HORZ_FILTER_SCALE_RATIO_C_BASE_IDX 2
+#define mmDSCL2_SCL_HORZ_FILTER_INIT_C 0x0fca
+#define mmDSCL2_SCL_HORZ_FILTER_INIT_C_BASE_IDX 2
+#define mmDSCL2_SCL_VERT_FILTER_SCALE_RATIO 0x0fcb
+#define mmDSCL2_SCL_VERT_FILTER_SCALE_RATIO_BASE_IDX 2
+#define mmDSCL2_SCL_VERT_FILTER_INIT 0x0fcc
+#define mmDSCL2_SCL_VERT_FILTER_INIT_BASE_IDX 2
+#define mmDSCL2_SCL_VERT_FILTER_INIT_BOT 0x0fcd
+#define mmDSCL2_SCL_VERT_FILTER_INIT_BOT_BASE_IDX 2
+#define mmDSCL2_SCL_VERT_FILTER_SCALE_RATIO_C 0x0fce
+#define mmDSCL2_SCL_VERT_FILTER_SCALE_RATIO_C_BASE_IDX 2
+#define mmDSCL2_SCL_VERT_FILTER_INIT_C 0x0fcf
+#define mmDSCL2_SCL_VERT_FILTER_INIT_C_BASE_IDX 2
+#define mmDSCL2_SCL_VERT_FILTER_INIT_BOT_C 0x0fd0
+#define mmDSCL2_SCL_VERT_FILTER_INIT_BOT_C_BASE_IDX 2
+#define mmDSCL2_SCL_BLACK_OFFSET 0x0fd1
+#define mmDSCL2_SCL_BLACK_OFFSET_BASE_IDX 2
+#define mmDSCL2_DSCL_UPDATE 0x0fd2
+#define mmDSCL2_DSCL_UPDATE_BASE_IDX 2
+#define mmDSCL2_DSCL_AUTOCAL 0x0fd3
+#define mmDSCL2_DSCL_AUTOCAL_BASE_IDX 2
+#define mmDSCL2_DSCL_EXT_OVERSCAN_LEFT_RIGHT 0x0fd4
+#define mmDSCL2_DSCL_EXT_OVERSCAN_LEFT_RIGHT_BASE_IDX 2
+#define mmDSCL2_DSCL_EXT_OVERSCAN_TOP_BOTTOM 0x0fd5
+#define mmDSCL2_DSCL_EXT_OVERSCAN_TOP_BOTTOM_BASE_IDX 2
+#define mmDSCL2_OTG_H_BLANK 0x0fd6
+#define mmDSCL2_OTG_H_BLANK_BASE_IDX 2
+#define mmDSCL2_OTG_V_BLANK 0x0fd7
+#define mmDSCL2_OTG_V_BLANK_BASE_IDX 2
+#define mmDSCL2_RECOUT_START 0x0fd8
+#define mmDSCL2_RECOUT_START_BASE_IDX 2
+#define mmDSCL2_RECOUT_SIZE 0x0fd9
+#define mmDSCL2_RECOUT_SIZE_BASE_IDX 2
+#define mmDSCL2_MPC_SIZE 0x0fda
+#define mmDSCL2_MPC_SIZE_BASE_IDX 2
+#define mmDSCL2_LB_DATA_FORMAT 0x0fdb
+#define mmDSCL2_LB_DATA_FORMAT_BASE_IDX 2
+#define mmDSCL2_LB_MEMORY_CTRL 0x0fdc
+#define mmDSCL2_LB_MEMORY_CTRL_BASE_IDX 2
+#define mmDSCL2_LB_V_COUNTER 0x0fdd
+#define mmDSCL2_LB_V_COUNTER_BASE_IDX 2
+#define mmDSCL2_DSCL_MEM_PWR_CTRL 0x0fde
+#define mmDSCL2_DSCL_MEM_PWR_CTRL_BASE_IDX 2
+#define mmDSCL2_DSCL_MEM_PWR_STATUS 0x0fdf
+#define mmDSCL2_DSCL_MEM_PWR_STATUS_BASE_IDX 2
+#define mmDSCL2_OBUF_CONTROL 0x0fe0
+#define mmDSCL2_OBUF_CONTROL_BASE_IDX 2
+#define mmDSCL2_OBUF_MEM_PWR_CTRL 0x0fe1
+#define mmDSCL2_OBUF_MEM_PWR_CTRL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp2_dispdec_cm_dispdec
+// base address: 0xb58
+#define mmCM2_CM_CONTROL 0x0ff0
+#define mmCM2_CM_CONTROL_BASE_IDX 2
+#define mmCM2_CM_ICSC_CONTROL 0x0ff1
+#define mmCM2_CM_ICSC_CONTROL_BASE_IDX 2
+#define mmCM2_CM_ICSC_C11_C12 0x0ff2
+#define mmCM2_CM_ICSC_C11_C12_BASE_IDX 2
+#define mmCM2_CM_ICSC_C13_C14 0x0ff3
+#define mmCM2_CM_ICSC_C13_C14_BASE_IDX 2
+#define mmCM2_CM_ICSC_C21_C22 0x0ff4
+#define mmCM2_CM_ICSC_C21_C22_BASE_IDX 2
+#define mmCM2_CM_ICSC_C23_C24 0x0ff5
+#define mmCM2_CM_ICSC_C23_C24_BASE_IDX 2
+#define mmCM2_CM_ICSC_C31_C32 0x0ff6
+#define mmCM2_CM_ICSC_C31_C32_BASE_IDX 2
+#define mmCM2_CM_ICSC_C33_C34 0x0ff7
+#define mmCM2_CM_ICSC_C33_C34_BASE_IDX 2
+#define mmCM2_CM_ICSC_B_C11_C12 0x0ff8
+#define mmCM2_CM_ICSC_B_C11_C12_BASE_IDX 2
+#define mmCM2_CM_ICSC_B_C13_C14 0x0ff9
+#define mmCM2_CM_ICSC_B_C13_C14_BASE_IDX 2
+#define mmCM2_CM_ICSC_B_C21_C22 0x0ffa
+#define mmCM2_CM_ICSC_B_C21_C22_BASE_IDX 2
+#define mmCM2_CM_ICSC_B_C23_C24 0x0ffb
+#define mmCM2_CM_ICSC_B_C23_C24_BASE_IDX 2
+#define mmCM2_CM_ICSC_B_C31_C32 0x0ffc
+#define mmCM2_CM_ICSC_B_C31_C32_BASE_IDX 2
+#define mmCM2_CM_ICSC_B_C33_C34 0x0ffd
+#define mmCM2_CM_ICSC_B_C33_C34_BASE_IDX 2
+#define mmCM2_CM_GAMUT_REMAP_CONTROL 0x0ffe
+#define mmCM2_CM_GAMUT_REMAP_CONTROL_BASE_IDX 2
+#define mmCM2_CM_GAMUT_REMAP_C11_C12 0x0fff
+#define mmCM2_CM_GAMUT_REMAP_C11_C12_BASE_IDX 2
+#define mmCM2_CM_GAMUT_REMAP_C13_C14 0x1000
+#define mmCM2_CM_GAMUT_REMAP_C13_C14_BASE_IDX 2
+#define mmCM2_CM_GAMUT_REMAP_C21_C22 0x1001
+#define mmCM2_CM_GAMUT_REMAP_C21_C22_BASE_IDX 2
+#define mmCM2_CM_GAMUT_REMAP_C23_C24 0x1002
+#define mmCM2_CM_GAMUT_REMAP_C23_C24_BASE_IDX 2
+#define mmCM2_CM_GAMUT_REMAP_C31_C32 0x1003
+#define mmCM2_CM_GAMUT_REMAP_C31_C32_BASE_IDX 2
+#define mmCM2_CM_GAMUT_REMAP_C33_C34 0x1004
+#define mmCM2_CM_GAMUT_REMAP_C33_C34_BASE_IDX 2
+#define mmCM2_CM_GAMUT_REMAP_B_C11_C12 0x1005
+#define mmCM2_CM_GAMUT_REMAP_B_C11_C12_BASE_IDX 2
+#define mmCM2_CM_GAMUT_REMAP_B_C13_C14 0x1006
+#define mmCM2_CM_GAMUT_REMAP_B_C13_C14_BASE_IDX 2
+#define mmCM2_CM_GAMUT_REMAP_B_C21_C22 0x1007
+#define mmCM2_CM_GAMUT_REMAP_B_C21_C22_BASE_IDX 2
+#define mmCM2_CM_GAMUT_REMAP_B_C23_C24 0x1008
+#define mmCM2_CM_GAMUT_REMAP_B_C23_C24_BASE_IDX 2
+#define mmCM2_CM_GAMUT_REMAP_B_C31_C32 0x1009
+#define mmCM2_CM_GAMUT_REMAP_B_C31_C32_BASE_IDX 2
+#define mmCM2_CM_GAMUT_REMAP_B_C33_C34 0x100a
+#define mmCM2_CM_GAMUT_REMAP_B_C33_C34_BASE_IDX 2
+#define mmCM2_CM_BIAS_CR_R 0x100b
+#define mmCM2_CM_BIAS_CR_R_BASE_IDX 2
+#define mmCM2_CM_BIAS_Y_G_CB_B 0x100c
+#define mmCM2_CM_BIAS_Y_G_CB_B_BASE_IDX 2
+#define mmCM2_CM_DGAM_CONTROL 0x100d
+#define mmCM2_CM_DGAM_CONTROL_BASE_IDX 2
+#define mmCM2_CM_DGAM_LUT_INDEX 0x100e
+#define mmCM2_CM_DGAM_LUT_INDEX_BASE_IDX 2
+#define mmCM2_CM_DGAM_LUT_DATA 0x100f
+#define mmCM2_CM_DGAM_LUT_DATA_BASE_IDX 2
+#define mmCM2_CM_DGAM_LUT_WRITE_EN_MASK 0x1010
+#define mmCM2_CM_DGAM_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_START_CNTL_B 0x1011
+#define mmCM2_CM_DGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_START_CNTL_G 0x1012
+#define mmCM2_CM_DGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_START_CNTL_R 0x1013
+#define mmCM2_CM_DGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_SLOPE_CNTL_B 0x1014
+#define mmCM2_CM_DGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_SLOPE_CNTL_G 0x1015
+#define mmCM2_CM_DGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_SLOPE_CNTL_R 0x1016
+#define mmCM2_CM_DGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_END_CNTL1_B 0x1017
+#define mmCM2_CM_DGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_END_CNTL2_B 0x1018
+#define mmCM2_CM_DGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_END_CNTL1_G 0x1019
+#define mmCM2_CM_DGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_END_CNTL2_G 0x101a
+#define mmCM2_CM_DGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_END_CNTL1_R 0x101b
+#define mmCM2_CM_DGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_END_CNTL2_R 0x101c
+#define mmCM2_CM_DGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_REGION_0_1 0x101d
+#define mmCM2_CM_DGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_REGION_2_3 0x101e
+#define mmCM2_CM_DGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_REGION_4_5 0x101f
+#define mmCM2_CM_DGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_REGION_6_7 0x1020
+#define mmCM2_CM_DGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_REGION_8_9 0x1021
+#define mmCM2_CM_DGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_REGION_10_11 0x1022
+#define mmCM2_CM_DGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_REGION_12_13 0x1023
+#define mmCM2_CM_DGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMA_REGION_14_15 0x1024
+#define mmCM2_CM_DGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_START_CNTL_B 0x1025
+#define mmCM2_CM_DGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_START_CNTL_G 0x1026
+#define mmCM2_CM_DGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_START_CNTL_R 0x1027
+#define mmCM2_CM_DGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_SLOPE_CNTL_B 0x1028
+#define mmCM2_CM_DGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_SLOPE_CNTL_G 0x1029
+#define mmCM2_CM_DGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_SLOPE_CNTL_R 0x102a
+#define mmCM2_CM_DGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_END_CNTL1_B 0x102b
+#define mmCM2_CM_DGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_END_CNTL2_B 0x102c
+#define mmCM2_CM_DGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_END_CNTL1_G 0x102d
+#define mmCM2_CM_DGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_END_CNTL2_G 0x102e
+#define mmCM2_CM_DGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_END_CNTL1_R 0x102f
+#define mmCM2_CM_DGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_END_CNTL2_R 0x1030
+#define mmCM2_CM_DGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_REGION_0_1 0x1031
+#define mmCM2_CM_DGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_REGION_2_3 0x1032
+#define mmCM2_CM_DGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_REGION_4_5 0x1033
+#define mmCM2_CM_DGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_REGION_6_7 0x1034
+#define mmCM2_CM_DGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_REGION_8_9 0x1035
+#define mmCM2_CM_DGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_REGION_10_11 0x1036
+#define mmCM2_CM_DGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_REGION_12_13 0x1037
+#define mmCM2_CM_DGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM2_CM_DGAM_RAMB_REGION_14_15 0x1038
+#define mmCM2_CM_DGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_CONTROL 0x1039
+#define mmCM2_CM_BLNDGAM_CONTROL_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_LUT_INDEX 0x103a
+#define mmCM2_CM_BLNDGAM_LUT_INDEX_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_LUT_DATA 0x103b
+#define mmCM2_CM_BLNDGAM_LUT_DATA_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_LUT_WRITE_EN_MASK 0x103c
+#define mmCM2_CM_BLNDGAM_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_START_CNTL_B 0x103d
+#define mmCM2_CM_BLNDGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_START_CNTL_G 0x103e
+#define mmCM2_CM_BLNDGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_START_CNTL_R 0x103f
+#define mmCM2_CM_BLNDGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_SLOPE_CNTL_B 0x1040
+#define mmCM2_CM_BLNDGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_SLOPE_CNTL_G 0x1041
+#define mmCM2_CM_BLNDGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_SLOPE_CNTL_R 0x1042
+#define mmCM2_CM_BLNDGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_END_CNTL1_B 0x1043
+#define mmCM2_CM_BLNDGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_END_CNTL2_B 0x1044
+#define mmCM2_CM_BLNDGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_END_CNTL1_G 0x1045
+#define mmCM2_CM_BLNDGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_END_CNTL2_G 0x1046
+#define mmCM2_CM_BLNDGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_END_CNTL1_R 0x1047
+#define mmCM2_CM_BLNDGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_END_CNTL2_R 0x1048
+#define mmCM2_CM_BLNDGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_0_1 0x1049
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_2_3 0x104a
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_4_5 0x104b
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_6_7 0x104c
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_8_9 0x104d
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_10_11 0x104e
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_12_13 0x104f
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_14_15 0x1050
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_16_17 0x1051
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_16_17_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_18_19 0x1052
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_18_19_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_20_21 0x1053
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_20_21_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_22_23 0x1054
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_22_23_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_24_25 0x1055
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_24_25_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_26_27 0x1056
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_26_27_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_28_29 0x1057
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_28_29_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_30_31 0x1058
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_30_31_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_32_33 0x1059
+#define mmCM2_CM_BLNDGAM_RAMA_REGION_32_33_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_START_CNTL_B 0x105a
+#define mmCM2_CM_BLNDGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_START_CNTL_G 0x105b
+#define mmCM2_CM_BLNDGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_START_CNTL_R 0x105c
+#define mmCM2_CM_BLNDGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_SLOPE_CNTL_B 0x105d
+#define mmCM2_CM_BLNDGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_SLOPE_CNTL_G 0x105e
+#define mmCM2_CM_BLNDGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_SLOPE_CNTL_R 0x105f
+#define mmCM2_CM_BLNDGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_END_CNTL1_B 0x1060
+#define mmCM2_CM_BLNDGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_END_CNTL2_B 0x1061
+#define mmCM2_CM_BLNDGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_END_CNTL1_G 0x1062
+#define mmCM2_CM_BLNDGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_END_CNTL2_G 0x1063
+#define mmCM2_CM_BLNDGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_END_CNTL1_R 0x1064
+#define mmCM2_CM_BLNDGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_END_CNTL2_R 0x1065
+#define mmCM2_CM_BLNDGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_0_1 0x1066
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_2_3 0x1067
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_4_5 0x1068
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_6_7 0x1069
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_8_9 0x106a
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_10_11 0x106b
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_12_13 0x106c
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_14_15 0x106d
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_16_17 0x106e
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_16_17_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_18_19 0x106f
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_18_19_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_20_21 0x1070
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_20_21_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_22_23 0x1071
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_22_23_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_24_25 0x1072
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_24_25_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_26_27 0x1073
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_26_27_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_28_29 0x1074
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_28_29_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_30_31 0x1075
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_30_31_BASE_IDX 2
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_32_33 0x1076
+#define mmCM2_CM_BLNDGAM_RAMB_REGION_32_33_BASE_IDX 2
+#define mmCM2_CM_HDR_MULT_COEF 0x1077
+#define mmCM2_CM_HDR_MULT_COEF_BASE_IDX 2
+#define mmCM2_CM_MEM_PWR_CTRL 0x1078
+#define mmCM2_CM_MEM_PWR_CTRL_BASE_IDX 2
+#define mmCM2_CM_MEM_PWR_STATUS 0x1079
+#define mmCM2_CM_MEM_PWR_STATUS_BASE_IDX 2
+#define mmCM2_CM_DEALPHA 0x107b
+#define mmCM2_CM_DEALPHA_BASE_IDX 2
+#define mmCM2_CM_COEF_FORMAT 0x107c
+#define mmCM2_CM_COEF_FORMAT_BASE_IDX 2
+#define mmCM2_CM_SHAPER_CONTROL 0x107d
+#define mmCM2_CM_SHAPER_CONTROL_BASE_IDX 2
+#define mmCM2_CM_SHAPER_OFFSET_R 0x107e
+#define mmCM2_CM_SHAPER_OFFSET_R_BASE_IDX 2
+#define mmCM2_CM_SHAPER_OFFSET_G 0x107f
+#define mmCM2_CM_SHAPER_OFFSET_G_BASE_IDX 2
+#define mmCM2_CM_SHAPER_OFFSET_B 0x1080
+#define mmCM2_CM_SHAPER_OFFSET_B_BASE_IDX 2
+#define mmCM2_CM_SHAPER_SCALE_R 0x1081
+#define mmCM2_CM_SHAPER_SCALE_R_BASE_IDX 2
+#define mmCM2_CM_SHAPER_SCALE_G_B 0x1082
+#define mmCM2_CM_SHAPER_SCALE_G_B_BASE_IDX 2
+#define mmCM2_CM_SHAPER_LUT_INDEX 0x1083
+#define mmCM2_CM_SHAPER_LUT_INDEX_BASE_IDX 2
+#define mmCM2_CM_SHAPER_LUT_DATA 0x1084
+#define mmCM2_CM_SHAPER_LUT_DATA_BASE_IDX 2
+#define mmCM2_CM_SHAPER_LUT_WRITE_EN_MASK 0x1085
+#define mmCM2_CM_SHAPER_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_START_CNTL_B 0x1086
+#define mmCM2_CM_SHAPER_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_START_CNTL_G 0x1087
+#define mmCM2_CM_SHAPER_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_START_CNTL_R 0x1088
+#define mmCM2_CM_SHAPER_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_END_CNTL_B 0x1089
+#define mmCM2_CM_SHAPER_RAMA_END_CNTL_B_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_END_CNTL_G 0x108a
+#define mmCM2_CM_SHAPER_RAMA_END_CNTL_G_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_END_CNTL_R 0x108b
+#define mmCM2_CM_SHAPER_RAMA_END_CNTL_R_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_REGION_0_1 0x108c
+#define mmCM2_CM_SHAPER_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_REGION_2_3 0x108d
+#define mmCM2_CM_SHAPER_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_REGION_4_5 0x108e
+#define mmCM2_CM_SHAPER_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_REGION_6_7 0x108f
+#define mmCM2_CM_SHAPER_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_REGION_8_9 0x1090
+#define mmCM2_CM_SHAPER_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_REGION_10_11 0x1091
+#define mmCM2_CM_SHAPER_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_REGION_12_13 0x1092
+#define mmCM2_CM_SHAPER_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_REGION_14_15 0x1093
+#define mmCM2_CM_SHAPER_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_REGION_16_17 0x1094
+#define mmCM2_CM_SHAPER_RAMA_REGION_16_17_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_REGION_18_19 0x1095
+#define mmCM2_CM_SHAPER_RAMA_REGION_18_19_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_REGION_20_21 0x1096
+#define mmCM2_CM_SHAPER_RAMA_REGION_20_21_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_REGION_22_23 0x1097
+#define mmCM2_CM_SHAPER_RAMA_REGION_22_23_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_REGION_24_25 0x1098
+#define mmCM2_CM_SHAPER_RAMA_REGION_24_25_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_REGION_26_27 0x1099
+#define mmCM2_CM_SHAPER_RAMA_REGION_26_27_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_REGION_28_29 0x109a
+#define mmCM2_CM_SHAPER_RAMA_REGION_28_29_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_REGION_30_31 0x109b
+#define mmCM2_CM_SHAPER_RAMA_REGION_30_31_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMA_REGION_32_33 0x109c
+#define mmCM2_CM_SHAPER_RAMA_REGION_32_33_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_START_CNTL_B 0x109d
+#define mmCM2_CM_SHAPER_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_START_CNTL_G 0x109e
+#define mmCM2_CM_SHAPER_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_START_CNTL_R 0x109f
+#define mmCM2_CM_SHAPER_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_END_CNTL_B 0x10a0
+#define mmCM2_CM_SHAPER_RAMB_END_CNTL_B_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_END_CNTL_G 0x10a1
+#define mmCM2_CM_SHAPER_RAMB_END_CNTL_G_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_END_CNTL_R 0x10a2
+#define mmCM2_CM_SHAPER_RAMB_END_CNTL_R_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_REGION_0_1 0x10a3
+#define mmCM2_CM_SHAPER_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_REGION_2_3 0x10a4
+#define mmCM2_CM_SHAPER_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_REGION_4_5 0x10a5
+#define mmCM2_CM_SHAPER_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_REGION_6_7 0x10a6
+#define mmCM2_CM_SHAPER_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_REGION_8_9 0x10a7
+#define mmCM2_CM_SHAPER_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_REGION_10_11 0x10a8
+#define mmCM2_CM_SHAPER_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_REGION_12_13 0x10a9
+#define mmCM2_CM_SHAPER_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_REGION_14_15 0x10aa
+#define mmCM2_CM_SHAPER_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_REGION_16_17 0x10ab
+#define mmCM2_CM_SHAPER_RAMB_REGION_16_17_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_REGION_18_19 0x10ac
+#define mmCM2_CM_SHAPER_RAMB_REGION_18_19_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_REGION_20_21 0x10ad
+#define mmCM2_CM_SHAPER_RAMB_REGION_20_21_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_REGION_22_23 0x10ae
+#define mmCM2_CM_SHAPER_RAMB_REGION_22_23_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_REGION_24_25 0x10af
+#define mmCM2_CM_SHAPER_RAMB_REGION_24_25_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_REGION_26_27 0x10b0
+#define mmCM2_CM_SHAPER_RAMB_REGION_26_27_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_REGION_28_29 0x10b1
+#define mmCM2_CM_SHAPER_RAMB_REGION_28_29_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_REGION_30_31 0x10b2
+#define mmCM2_CM_SHAPER_RAMB_REGION_30_31_BASE_IDX 2
+#define mmCM2_CM_SHAPER_RAMB_REGION_32_33 0x10b3
+#define mmCM2_CM_SHAPER_RAMB_REGION_32_33_BASE_IDX 2
+#define mmCM2_CM_MEM_PWR_CTRL2 0x10b4
+#define mmCM2_CM_MEM_PWR_CTRL2_BASE_IDX 2
+#define mmCM2_CM_MEM_PWR_STATUS2 0x10b5
+#define mmCM2_CM_MEM_PWR_STATUS2_BASE_IDX 2
+#define mmCM2_CM_3DLUT_MODE 0x10b6
+#define mmCM2_CM_3DLUT_MODE_BASE_IDX 2
+#define mmCM2_CM_3DLUT_INDEX 0x10b7
+#define mmCM2_CM_3DLUT_INDEX_BASE_IDX 2
+#define mmCM2_CM_3DLUT_DATA 0x10b8
+#define mmCM2_CM_3DLUT_DATA_BASE_IDX 2
+#define mmCM2_CM_3DLUT_DATA_30BIT 0x10b9
+#define mmCM2_CM_3DLUT_DATA_30BIT_BASE_IDX 2
+#define mmCM2_CM_3DLUT_READ_WRITE_CONTROL 0x10ba
+#define mmCM2_CM_3DLUT_READ_WRITE_CONTROL_BASE_IDX 2
+#define mmCM2_CM_3DLUT_OUT_NORM_FACTOR 0x10bb
+#define mmCM2_CM_3DLUT_OUT_NORM_FACTOR_BASE_IDX 2
+#define mmCM2_CM_3DLUT_OUT_OFFSET_R 0x10bc
+#define mmCM2_CM_3DLUT_OUT_OFFSET_R_BASE_IDX 2
+#define mmCM2_CM_3DLUT_OUT_OFFSET_G 0x10bd
+#define mmCM2_CM_3DLUT_OUT_OFFSET_G_BASE_IDX 2
+#define mmCM2_CM_3DLUT_OUT_OFFSET_B 0x10be
+#define mmCM2_CM_3DLUT_OUT_OFFSET_B_BASE_IDX 2
+#define mmCM2_CM_TEST_DEBUG_INDEX 0x10bf
+#define mmCM2_CM_TEST_DEBUG_INDEX_BASE_IDX 2
+#define mmCM2_CM_TEST_DEBUG_DATA 0x10c0
+#define mmCM2_CM_TEST_DEBUG_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp2_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x43e8
+#define mmDC_PERFMON15_PERFCOUNTER_CNTL 0x10fa
+#define mmDC_PERFMON15_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON15_PERFCOUNTER_CNTL2 0x10fb
+#define mmDC_PERFMON15_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON15_PERFCOUNTER_STATE 0x10fc
+#define mmDC_PERFMON15_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON15_PERFMON_CNTL 0x10fd
+#define mmDC_PERFMON15_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON15_PERFMON_CNTL2 0x10fe
+#define mmDC_PERFMON15_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON15_PERFMON_CVALUE_INT_MISC 0x10ff
+#define mmDC_PERFMON15_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON15_PERFMON_CVALUE_LOW 0x1100
+#define mmDC_PERFMON15_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON15_PERFMON_HI 0x1101
+#define mmDC_PERFMON15_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON15_PERFMON_LOW 0x1102
+#define mmDC_PERFMON15_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp3_dispdec_dpp_top_dispdec
+// base address: 0x1104
+#define mmDPP_TOP3_DPP_CONTROL 0x1106
+#define mmDPP_TOP3_DPP_CONTROL_BASE_IDX 2
+#define mmDPP_TOP3_DPP_SOFT_RESET 0x1107
+#define mmDPP_TOP3_DPP_SOFT_RESET_BASE_IDX 2
+#define mmDPP_TOP3_DPP_CRC_VAL_R_G 0x1108
+#define mmDPP_TOP3_DPP_CRC_VAL_R_G_BASE_IDX 2
+#define mmDPP_TOP3_DPP_CRC_VAL_B_A 0x1109
+#define mmDPP_TOP3_DPP_CRC_VAL_B_A_BASE_IDX 2
+#define mmDPP_TOP3_DPP_CRC_CTRL 0x110a
+#define mmDPP_TOP3_DPP_CRC_CTRL_BASE_IDX 2
+#define mmDPP_TOP3_HOST_READ_CONTROL 0x110b
+#define mmDPP_TOP3_HOST_READ_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp3_dispdec_cnvc_cfg_dispdec
+// base address: 0x1104
+#define mmCNVC_CFG3_CNVC_SURFACE_PIXEL_FORMAT 0x1110
+#define mmCNVC_CFG3_CNVC_SURFACE_PIXEL_FORMAT_BASE_IDX 2
+#define mmCNVC_CFG3_FORMAT_CONTROL 0x1111
+#define mmCNVC_CFG3_FORMAT_CONTROL_BASE_IDX 2
+#define mmCNVC_CFG3_FCNV_FP_BIAS_R 0x1112
+#define mmCNVC_CFG3_FCNV_FP_BIAS_R_BASE_IDX 2
+#define mmCNVC_CFG3_FCNV_FP_BIAS_G 0x1113
+#define mmCNVC_CFG3_FCNV_FP_BIAS_G_BASE_IDX 2
+#define mmCNVC_CFG3_FCNV_FP_BIAS_B 0x1114
+#define mmCNVC_CFG3_FCNV_FP_BIAS_B_BASE_IDX 2
+#define mmCNVC_CFG3_FCNV_FP_SCALE_R 0x1115
+#define mmCNVC_CFG3_FCNV_FP_SCALE_R_BASE_IDX 2
+#define mmCNVC_CFG3_FCNV_FP_SCALE_G 0x1116
+#define mmCNVC_CFG3_FCNV_FP_SCALE_G_BASE_IDX 2
+#define mmCNVC_CFG3_FCNV_FP_SCALE_B 0x1117
+#define mmCNVC_CFG3_FCNV_FP_SCALE_B_BASE_IDX 2
+#define mmCNVC_CFG3_COLOR_KEYER_CONTROL 0x1118
+#define mmCNVC_CFG3_COLOR_KEYER_CONTROL_BASE_IDX 2
+#define mmCNVC_CFG3_COLOR_KEYER_ALPHA 0x1119
+#define mmCNVC_CFG3_COLOR_KEYER_ALPHA_BASE_IDX 2
+#define mmCNVC_CFG3_COLOR_KEYER_RED 0x111a
+#define mmCNVC_CFG3_COLOR_KEYER_RED_BASE_IDX 2
+#define mmCNVC_CFG3_COLOR_KEYER_GREEN 0x111b
+#define mmCNVC_CFG3_COLOR_KEYER_GREEN_BASE_IDX 2
+#define mmCNVC_CFG3_COLOR_KEYER_BLUE 0x111c
+#define mmCNVC_CFG3_COLOR_KEYER_BLUE_BASE_IDX 2
+#define mmCNVC_CFG3_ALPHA_2BIT_LUT 0x111e
+#define mmCNVC_CFG3_ALPHA_2BIT_LUT_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp3_dispdec_cnvc_cur_dispdec
+// base address: 0x1104
+#define mmCNVC_CUR3_CURSOR0_CONTROL 0x1121
+#define mmCNVC_CUR3_CURSOR0_CONTROL_BASE_IDX 2
+#define mmCNVC_CUR3_CURSOR0_COLOR0 0x1122
+#define mmCNVC_CUR3_CURSOR0_COLOR0_BASE_IDX 2
+#define mmCNVC_CUR3_CURSOR0_COLOR1 0x1123
+#define mmCNVC_CUR3_CURSOR0_COLOR1_BASE_IDX 2
+#define mmCNVC_CUR3_CURSOR0_FP_SCALE_BIAS 0x1124
+#define mmCNVC_CUR3_CURSOR0_FP_SCALE_BIAS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp3_dispdec_dscl_dispdec
+// base address: 0x1104
+#define mmDSCL3_SCL_COEF_RAM_TAP_SELECT 0x112b
+#define mmDSCL3_SCL_COEF_RAM_TAP_SELECT_BASE_IDX 2
+#define mmDSCL3_SCL_COEF_RAM_TAP_DATA 0x112c
+#define mmDSCL3_SCL_COEF_RAM_TAP_DATA_BASE_IDX 2
+#define mmDSCL3_SCL_MODE 0x112d
+#define mmDSCL3_SCL_MODE_BASE_IDX 2
+#define mmDSCL3_SCL_TAP_CONTROL 0x112e
+#define mmDSCL3_SCL_TAP_CONTROL_BASE_IDX 2
+#define mmDSCL3_DSCL_CONTROL 0x112f
+#define mmDSCL3_DSCL_CONTROL_BASE_IDX 2
+#define mmDSCL3_DSCL_2TAP_CONTROL 0x1130
+#define mmDSCL3_DSCL_2TAP_CONTROL_BASE_IDX 2
+#define mmDSCL3_SCL_MANUAL_REPLICATE_CONTROL 0x1131
+#define mmDSCL3_SCL_MANUAL_REPLICATE_CONTROL_BASE_IDX 2
+#define mmDSCL3_SCL_HORZ_FILTER_SCALE_RATIO 0x1132
+#define mmDSCL3_SCL_HORZ_FILTER_SCALE_RATIO_BASE_IDX 2
+#define mmDSCL3_SCL_HORZ_FILTER_INIT 0x1133
+#define mmDSCL3_SCL_HORZ_FILTER_INIT_BASE_IDX 2
+#define mmDSCL3_SCL_HORZ_FILTER_SCALE_RATIO_C 0x1134
+#define mmDSCL3_SCL_HORZ_FILTER_SCALE_RATIO_C_BASE_IDX 2
+#define mmDSCL3_SCL_HORZ_FILTER_INIT_C 0x1135
+#define mmDSCL3_SCL_HORZ_FILTER_INIT_C_BASE_IDX 2
+#define mmDSCL3_SCL_VERT_FILTER_SCALE_RATIO 0x1136
+#define mmDSCL3_SCL_VERT_FILTER_SCALE_RATIO_BASE_IDX 2
+#define mmDSCL3_SCL_VERT_FILTER_INIT 0x1137
+#define mmDSCL3_SCL_VERT_FILTER_INIT_BASE_IDX 2
+#define mmDSCL3_SCL_VERT_FILTER_INIT_BOT 0x1138
+#define mmDSCL3_SCL_VERT_FILTER_INIT_BOT_BASE_IDX 2
+#define mmDSCL3_SCL_VERT_FILTER_SCALE_RATIO_C 0x1139
+#define mmDSCL3_SCL_VERT_FILTER_SCALE_RATIO_C_BASE_IDX 2
+#define mmDSCL3_SCL_VERT_FILTER_INIT_C 0x113a
+#define mmDSCL3_SCL_VERT_FILTER_INIT_C_BASE_IDX 2
+#define mmDSCL3_SCL_VERT_FILTER_INIT_BOT_C 0x113b
+#define mmDSCL3_SCL_VERT_FILTER_INIT_BOT_C_BASE_IDX 2
+#define mmDSCL3_SCL_BLACK_OFFSET 0x113c
+#define mmDSCL3_SCL_BLACK_OFFSET_BASE_IDX 2
+#define mmDSCL3_DSCL_UPDATE 0x113d
+#define mmDSCL3_DSCL_UPDATE_BASE_IDX 2
+#define mmDSCL3_DSCL_AUTOCAL 0x113e
+#define mmDSCL3_DSCL_AUTOCAL_BASE_IDX 2
+#define mmDSCL3_DSCL_EXT_OVERSCAN_LEFT_RIGHT 0x113f
+#define mmDSCL3_DSCL_EXT_OVERSCAN_LEFT_RIGHT_BASE_IDX 2
+#define mmDSCL3_DSCL_EXT_OVERSCAN_TOP_BOTTOM 0x1140
+#define mmDSCL3_DSCL_EXT_OVERSCAN_TOP_BOTTOM_BASE_IDX 2
+#define mmDSCL3_OTG_H_BLANK 0x1141
+#define mmDSCL3_OTG_H_BLANK_BASE_IDX 2
+#define mmDSCL3_OTG_V_BLANK 0x1142
+#define mmDSCL3_OTG_V_BLANK_BASE_IDX 2
+#define mmDSCL3_RECOUT_START 0x1143
+#define mmDSCL3_RECOUT_START_BASE_IDX 2
+#define mmDSCL3_RECOUT_SIZE 0x1144
+#define mmDSCL3_RECOUT_SIZE_BASE_IDX 2
+#define mmDSCL3_MPC_SIZE 0x1145
+#define mmDSCL3_MPC_SIZE_BASE_IDX 2
+#define mmDSCL3_LB_DATA_FORMAT 0x1146
+#define mmDSCL3_LB_DATA_FORMAT_BASE_IDX 2
+#define mmDSCL3_LB_MEMORY_CTRL 0x1147
+#define mmDSCL3_LB_MEMORY_CTRL_BASE_IDX 2
+#define mmDSCL3_LB_V_COUNTER 0x1148
+#define mmDSCL3_LB_V_COUNTER_BASE_IDX 2
+#define mmDSCL3_DSCL_MEM_PWR_CTRL 0x1149
+#define mmDSCL3_DSCL_MEM_PWR_CTRL_BASE_IDX 2
+#define mmDSCL3_DSCL_MEM_PWR_STATUS 0x114a
+#define mmDSCL3_DSCL_MEM_PWR_STATUS_BASE_IDX 2
+#define mmDSCL3_OBUF_CONTROL 0x114b
+#define mmDSCL3_OBUF_CONTROL_BASE_IDX 2
+#define mmDSCL3_OBUF_MEM_PWR_CTRL 0x114c
+#define mmDSCL3_OBUF_MEM_PWR_CTRL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp3_dispdec_cm_dispdec
+// base address: 0x1104
+#define mmCM3_CM_CONTROL 0x115b
+#define mmCM3_CM_CONTROL_BASE_IDX 2
+#define mmCM3_CM_ICSC_CONTROL 0x115c
+#define mmCM3_CM_ICSC_CONTROL_BASE_IDX 2
+#define mmCM3_CM_ICSC_C11_C12 0x115d
+#define mmCM3_CM_ICSC_C11_C12_BASE_IDX 2
+#define mmCM3_CM_ICSC_C13_C14 0x115e
+#define mmCM3_CM_ICSC_C13_C14_BASE_IDX 2
+#define mmCM3_CM_ICSC_C21_C22 0x115f
+#define mmCM3_CM_ICSC_C21_C22_BASE_IDX 2
+#define mmCM3_CM_ICSC_C23_C24 0x1160
+#define mmCM3_CM_ICSC_C23_C24_BASE_IDX 2
+#define mmCM3_CM_ICSC_C31_C32 0x1161
+#define mmCM3_CM_ICSC_C31_C32_BASE_IDX 2
+#define mmCM3_CM_ICSC_C33_C34 0x1162
+#define mmCM3_CM_ICSC_C33_C34_BASE_IDX 2
+#define mmCM3_CM_ICSC_B_C11_C12 0x1163
+#define mmCM3_CM_ICSC_B_C11_C12_BASE_IDX 2
+#define mmCM3_CM_ICSC_B_C13_C14 0x1164
+#define mmCM3_CM_ICSC_B_C13_C14_BASE_IDX 2
+#define mmCM3_CM_ICSC_B_C21_C22 0x1165
+#define mmCM3_CM_ICSC_B_C21_C22_BASE_IDX 2
+#define mmCM3_CM_ICSC_B_C23_C24 0x1166
+#define mmCM3_CM_ICSC_B_C23_C24_BASE_IDX 2
+#define mmCM3_CM_ICSC_B_C31_C32 0x1167
+#define mmCM3_CM_ICSC_B_C31_C32_BASE_IDX 2
+#define mmCM3_CM_ICSC_B_C33_C34 0x1168
+#define mmCM3_CM_ICSC_B_C33_C34_BASE_IDX 2
+#define mmCM3_CM_GAMUT_REMAP_CONTROL 0x1169
+#define mmCM3_CM_GAMUT_REMAP_CONTROL_BASE_IDX 2
+#define mmCM3_CM_GAMUT_REMAP_C11_C12 0x116a
+#define mmCM3_CM_GAMUT_REMAP_C11_C12_BASE_IDX 2
+#define mmCM3_CM_GAMUT_REMAP_C13_C14 0x116b
+#define mmCM3_CM_GAMUT_REMAP_C13_C14_BASE_IDX 2
+#define mmCM3_CM_GAMUT_REMAP_C21_C22 0x116c
+#define mmCM3_CM_GAMUT_REMAP_C21_C22_BASE_IDX 2
+#define mmCM3_CM_GAMUT_REMAP_C23_C24 0x116d
+#define mmCM3_CM_GAMUT_REMAP_C23_C24_BASE_IDX 2
+#define mmCM3_CM_GAMUT_REMAP_C31_C32 0x116e
+#define mmCM3_CM_GAMUT_REMAP_C31_C32_BASE_IDX 2
+#define mmCM3_CM_GAMUT_REMAP_C33_C34 0x116f
+#define mmCM3_CM_GAMUT_REMAP_C33_C34_BASE_IDX 2
+#define mmCM3_CM_GAMUT_REMAP_B_C11_C12 0x1170
+#define mmCM3_CM_GAMUT_REMAP_B_C11_C12_BASE_IDX 2
+#define mmCM3_CM_GAMUT_REMAP_B_C13_C14 0x1171
+#define mmCM3_CM_GAMUT_REMAP_B_C13_C14_BASE_IDX 2
+#define mmCM3_CM_GAMUT_REMAP_B_C21_C22 0x1172
+#define mmCM3_CM_GAMUT_REMAP_B_C21_C22_BASE_IDX 2
+#define mmCM3_CM_GAMUT_REMAP_B_C23_C24 0x1173
+#define mmCM3_CM_GAMUT_REMAP_B_C23_C24_BASE_IDX 2
+#define mmCM3_CM_GAMUT_REMAP_B_C31_C32 0x1174
+#define mmCM3_CM_GAMUT_REMAP_B_C31_C32_BASE_IDX 2
+#define mmCM3_CM_GAMUT_REMAP_B_C33_C34 0x1175
+#define mmCM3_CM_GAMUT_REMAP_B_C33_C34_BASE_IDX 2
+#define mmCM3_CM_BIAS_CR_R 0x1176
+#define mmCM3_CM_BIAS_CR_R_BASE_IDX 2
+#define mmCM3_CM_BIAS_Y_G_CB_B 0x1177
+#define mmCM3_CM_BIAS_Y_G_CB_B_BASE_IDX 2
+#define mmCM3_CM_DGAM_CONTROL 0x1178
+#define mmCM3_CM_DGAM_CONTROL_BASE_IDX 2
+#define mmCM3_CM_DGAM_LUT_INDEX 0x1179
+#define mmCM3_CM_DGAM_LUT_INDEX_BASE_IDX 2
+#define mmCM3_CM_DGAM_LUT_DATA 0x117a
+#define mmCM3_CM_DGAM_LUT_DATA_BASE_IDX 2
+#define mmCM3_CM_DGAM_LUT_WRITE_EN_MASK 0x117b
+#define mmCM3_CM_DGAM_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_START_CNTL_B 0x117c
+#define mmCM3_CM_DGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_START_CNTL_G 0x117d
+#define mmCM3_CM_DGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_START_CNTL_R 0x117e
+#define mmCM3_CM_DGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_SLOPE_CNTL_B 0x117f
+#define mmCM3_CM_DGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_SLOPE_CNTL_G 0x1180
+#define mmCM3_CM_DGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_SLOPE_CNTL_R 0x1181
+#define mmCM3_CM_DGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_END_CNTL1_B 0x1182
+#define mmCM3_CM_DGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_END_CNTL2_B 0x1183
+#define mmCM3_CM_DGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_END_CNTL1_G 0x1184
+#define mmCM3_CM_DGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_END_CNTL2_G 0x1185
+#define mmCM3_CM_DGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_END_CNTL1_R 0x1186
+#define mmCM3_CM_DGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_END_CNTL2_R 0x1187
+#define mmCM3_CM_DGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_REGION_0_1 0x1188
+#define mmCM3_CM_DGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_REGION_2_3 0x1189
+#define mmCM3_CM_DGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_REGION_4_5 0x118a
+#define mmCM3_CM_DGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_REGION_6_7 0x118b
+#define mmCM3_CM_DGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_REGION_8_9 0x118c
+#define mmCM3_CM_DGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_REGION_10_11 0x118d
+#define mmCM3_CM_DGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_REGION_12_13 0x118e
+#define mmCM3_CM_DGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMA_REGION_14_15 0x118f
+#define mmCM3_CM_DGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_START_CNTL_B 0x1190
+#define mmCM3_CM_DGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_START_CNTL_G 0x1191
+#define mmCM3_CM_DGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_START_CNTL_R 0x1192
+#define mmCM3_CM_DGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_SLOPE_CNTL_B 0x1193
+#define mmCM3_CM_DGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_SLOPE_CNTL_G 0x1194
+#define mmCM3_CM_DGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_SLOPE_CNTL_R 0x1195
+#define mmCM3_CM_DGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_END_CNTL1_B 0x1196
+#define mmCM3_CM_DGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_END_CNTL2_B 0x1197
+#define mmCM3_CM_DGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_END_CNTL1_G 0x1198
+#define mmCM3_CM_DGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_END_CNTL2_G 0x1199
+#define mmCM3_CM_DGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_END_CNTL1_R 0x119a
+#define mmCM3_CM_DGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_END_CNTL2_R 0x119b
+#define mmCM3_CM_DGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_REGION_0_1 0x119c
+#define mmCM3_CM_DGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_REGION_2_3 0x119d
+#define mmCM3_CM_DGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_REGION_4_5 0x119e
+#define mmCM3_CM_DGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_REGION_6_7 0x119f
+#define mmCM3_CM_DGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_REGION_8_9 0x11a0
+#define mmCM3_CM_DGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_REGION_10_11 0x11a1
+#define mmCM3_CM_DGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_REGION_12_13 0x11a2
+#define mmCM3_CM_DGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM3_CM_DGAM_RAMB_REGION_14_15 0x11a3
+#define mmCM3_CM_DGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_CONTROL 0x11a4
+#define mmCM3_CM_BLNDGAM_CONTROL_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_LUT_INDEX 0x11a5
+#define mmCM3_CM_BLNDGAM_LUT_INDEX_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_LUT_DATA 0x11a6
+#define mmCM3_CM_BLNDGAM_LUT_DATA_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_LUT_WRITE_EN_MASK 0x11a7
+#define mmCM3_CM_BLNDGAM_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_START_CNTL_B 0x11a8
+#define mmCM3_CM_BLNDGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_START_CNTL_G 0x11a9
+#define mmCM3_CM_BLNDGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_START_CNTL_R 0x11aa
+#define mmCM3_CM_BLNDGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_SLOPE_CNTL_B 0x11ab
+#define mmCM3_CM_BLNDGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_SLOPE_CNTL_G 0x11ac
+#define mmCM3_CM_BLNDGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_SLOPE_CNTL_R 0x11ad
+#define mmCM3_CM_BLNDGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_END_CNTL1_B 0x11ae
+#define mmCM3_CM_BLNDGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_END_CNTL2_B 0x11af
+#define mmCM3_CM_BLNDGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_END_CNTL1_G 0x11b0
+#define mmCM3_CM_BLNDGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_END_CNTL2_G 0x11b1
+#define mmCM3_CM_BLNDGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_END_CNTL1_R 0x11b2
+#define mmCM3_CM_BLNDGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_END_CNTL2_R 0x11b3
+#define mmCM3_CM_BLNDGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_0_1 0x11b4
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_2_3 0x11b5
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_4_5 0x11b6
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_6_7 0x11b7
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_8_9 0x11b8
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_10_11 0x11b9
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_12_13 0x11ba
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_14_15 0x11bb
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_16_17 0x11bc
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_16_17_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_18_19 0x11bd
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_18_19_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_20_21 0x11be
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_20_21_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_22_23 0x11bf
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_22_23_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_24_25 0x11c0
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_24_25_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_26_27 0x11c1
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_26_27_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_28_29 0x11c2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_28_29_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_30_31 0x11c3
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_30_31_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_32_33 0x11c4
+#define mmCM3_CM_BLNDGAM_RAMA_REGION_32_33_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_START_CNTL_B 0x11c5
+#define mmCM3_CM_BLNDGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_START_CNTL_G 0x11c6
+#define mmCM3_CM_BLNDGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_START_CNTL_R 0x11c7
+#define mmCM3_CM_BLNDGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_SLOPE_CNTL_B 0x11c8
+#define mmCM3_CM_BLNDGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_SLOPE_CNTL_G 0x11c9
+#define mmCM3_CM_BLNDGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_SLOPE_CNTL_R 0x11ca
+#define mmCM3_CM_BLNDGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_END_CNTL1_B 0x11cb
+#define mmCM3_CM_BLNDGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_END_CNTL2_B 0x11cc
+#define mmCM3_CM_BLNDGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_END_CNTL1_G 0x11cd
+#define mmCM3_CM_BLNDGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_END_CNTL2_G 0x11ce
+#define mmCM3_CM_BLNDGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_END_CNTL1_R 0x11cf
+#define mmCM3_CM_BLNDGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_END_CNTL2_R 0x11d0
+#define mmCM3_CM_BLNDGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_0_1 0x11d1
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_2_3 0x11d2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_4_5 0x11d3
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_6_7 0x11d4
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_8_9 0x11d5
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_10_11 0x11d6
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_12_13 0x11d7
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_14_15 0x11d8
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_16_17 0x11d9
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_16_17_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_18_19 0x11da
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_18_19_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_20_21 0x11db
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_20_21_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_22_23 0x11dc
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_22_23_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_24_25 0x11dd
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_24_25_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_26_27 0x11de
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_26_27_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_28_29 0x11df
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_28_29_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_30_31 0x11e0
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_30_31_BASE_IDX 2
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_32_33 0x11e1
+#define mmCM3_CM_BLNDGAM_RAMB_REGION_32_33_BASE_IDX 2
+#define mmCM3_CM_HDR_MULT_COEF 0x11e2
+#define mmCM3_CM_HDR_MULT_COEF_BASE_IDX 2
+#define mmCM3_CM_MEM_PWR_CTRL 0x11e3
+#define mmCM3_CM_MEM_PWR_CTRL_BASE_IDX 2
+#define mmCM3_CM_MEM_PWR_STATUS 0x11e4
+#define mmCM3_CM_MEM_PWR_STATUS_BASE_IDX 2
+#define mmCM3_CM_DEALPHA 0x11e6
+#define mmCM3_CM_DEALPHA_BASE_IDX 2
+#define mmCM3_CM_COEF_FORMAT 0x11e7
+#define mmCM3_CM_COEF_FORMAT_BASE_IDX 2
+#define mmCM3_CM_SHAPER_CONTROL 0x11e8
+#define mmCM3_CM_SHAPER_CONTROL_BASE_IDX 2
+#define mmCM3_CM_SHAPER_OFFSET_R 0x11e9
+#define mmCM3_CM_SHAPER_OFFSET_R_BASE_IDX 2
+#define mmCM3_CM_SHAPER_OFFSET_G 0x11ea
+#define mmCM3_CM_SHAPER_OFFSET_G_BASE_IDX 2
+#define mmCM3_CM_SHAPER_OFFSET_B 0x11eb
+#define mmCM3_CM_SHAPER_OFFSET_B_BASE_IDX 2
+#define mmCM3_CM_SHAPER_SCALE_R 0x11ec
+#define mmCM3_CM_SHAPER_SCALE_R_BASE_IDX 2
+#define mmCM3_CM_SHAPER_SCALE_G_B 0x11ed
+#define mmCM3_CM_SHAPER_SCALE_G_B_BASE_IDX 2
+#define mmCM3_CM_SHAPER_LUT_INDEX 0x11ee
+#define mmCM3_CM_SHAPER_LUT_INDEX_BASE_IDX 2
+#define mmCM3_CM_SHAPER_LUT_DATA 0x11ef
+#define mmCM3_CM_SHAPER_LUT_DATA_BASE_IDX 2
+#define mmCM3_CM_SHAPER_LUT_WRITE_EN_MASK 0x11f0
+#define mmCM3_CM_SHAPER_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_START_CNTL_B 0x11f1
+#define mmCM3_CM_SHAPER_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_START_CNTL_G 0x11f2
+#define mmCM3_CM_SHAPER_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_START_CNTL_R 0x11f3
+#define mmCM3_CM_SHAPER_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_END_CNTL_B 0x11f4
+#define mmCM3_CM_SHAPER_RAMA_END_CNTL_B_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_END_CNTL_G 0x11f5
+#define mmCM3_CM_SHAPER_RAMA_END_CNTL_G_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_END_CNTL_R 0x11f6
+#define mmCM3_CM_SHAPER_RAMA_END_CNTL_R_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_REGION_0_1 0x11f7
+#define mmCM3_CM_SHAPER_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_REGION_2_3 0x11f8
+#define mmCM3_CM_SHAPER_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_REGION_4_5 0x11f9
+#define mmCM3_CM_SHAPER_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_REGION_6_7 0x11fa
+#define mmCM3_CM_SHAPER_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_REGION_8_9 0x11fb
+#define mmCM3_CM_SHAPER_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_REGION_10_11 0x11fc
+#define mmCM3_CM_SHAPER_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_REGION_12_13 0x11fd
+#define mmCM3_CM_SHAPER_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_REGION_14_15 0x11fe
+#define mmCM3_CM_SHAPER_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_REGION_16_17 0x11ff
+#define mmCM3_CM_SHAPER_RAMA_REGION_16_17_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_REGION_18_19 0x1200
+#define mmCM3_CM_SHAPER_RAMA_REGION_18_19_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_REGION_20_21 0x1201
+#define mmCM3_CM_SHAPER_RAMA_REGION_20_21_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_REGION_22_23 0x1202
+#define mmCM3_CM_SHAPER_RAMA_REGION_22_23_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_REGION_24_25 0x1203
+#define mmCM3_CM_SHAPER_RAMA_REGION_24_25_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_REGION_26_27 0x1204
+#define mmCM3_CM_SHAPER_RAMA_REGION_26_27_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_REGION_28_29 0x1205
+#define mmCM3_CM_SHAPER_RAMA_REGION_28_29_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_REGION_30_31 0x1206
+#define mmCM3_CM_SHAPER_RAMA_REGION_30_31_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMA_REGION_32_33 0x1207
+#define mmCM3_CM_SHAPER_RAMA_REGION_32_33_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_START_CNTL_B 0x1208
+#define mmCM3_CM_SHAPER_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_START_CNTL_G 0x1209
+#define mmCM3_CM_SHAPER_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_START_CNTL_R 0x120a
+#define mmCM3_CM_SHAPER_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_END_CNTL_B 0x120b
+#define mmCM3_CM_SHAPER_RAMB_END_CNTL_B_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_END_CNTL_G 0x120c
+#define mmCM3_CM_SHAPER_RAMB_END_CNTL_G_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_END_CNTL_R 0x120d
+#define mmCM3_CM_SHAPER_RAMB_END_CNTL_R_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_REGION_0_1 0x120e
+#define mmCM3_CM_SHAPER_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_REGION_2_3 0x120f
+#define mmCM3_CM_SHAPER_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_REGION_4_5 0x1210
+#define mmCM3_CM_SHAPER_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_REGION_6_7 0x1211
+#define mmCM3_CM_SHAPER_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_REGION_8_9 0x1212
+#define mmCM3_CM_SHAPER_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_REGION_10_11 0x1213
+#define mmCM3_CM_SHAPER_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_REGION_12_13 0x1214
+#define mmCM3_CM_SHAPER_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_REGION_14_15 0x1215
+#define mmCM3_CM_SHAPER_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_REGION_16_17 0x1216
+#define mmCM3_CM_SHAPER_RAMB_REGION_16_17_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_REGION_18_19 0x1217
+#define mmCM3_CM_SHAPER_RAMB_REGION_18_19_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_REGION_20_21 0x1218
+#define mmCM3_CM_SHAPER_RAMB_REGION_20_21_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_REGION_22_23 0x1219
+#define mmCM3_CM_SHAPER_RAMB_REGION_22_23_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_REGION_24_25 0x121a
+#define mmCM3_CM_SHAPER_RAMB_REGION_24_25_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_REGION_26_27 0x121b
+#define mmCM3_CM_SHAPER_RAMB_REGION_26_27_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_REGION_28_29 0x121c
+#define mmCM3_CM_SHAPER_RAMB_REGION_28_29_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_REGION_30_31 0x121d
+#define mmCM3_CM_SHAPER_RAMB_REGION_30_31_BASE_IDX 2
+#define mmCM3_CM_SHAPER_RAMB_REGION_32_33 0x121e
+#define mmCM3_CM_SHAPER_RAMB_REGION_32_33_BASE_IDX 2
+#define mmCM3_CM_MEM_PWR_CTRL2 0x121f
+#define mmCM3_CM_MEM_PWR_CTRL2_BASE_IDX 2
+#define mmCM3_CM_MEM_PWR_STATUS2 0x1220
+#define mmCM3_CM_MEM_PWR_STATUS2_BASE_IDX 2
+#define mmCM3_CM_3DLUT_MODE 0x1221
+#define mmCM3_CM_3DLUT_MODE_BASE_IDX 2
+#define mmCM3_CM_3DLUT_INDEX 0x1222
+#define mmCM3_CM_3DLUT_INDEX_BASE_IDX 2
+#define mmCM3_CM_3DLUT_DATA 0x1223
+#define mmCM3_CM_3DLUT_DATA_BASE_IDX 2
+#define mmCM3_CM_3DLUT_DATA_30BIT 0x1224
+#define mmCM3_CM_3DLUT_DATA_30BIT_BASE_IDX 2
+#define mmCM3_CM_3DLUT_READ_WRITE_CONTROL 0x1225
+#define mmCM3_CM_3DLUT_READ_WRITE_CONTROL_BASE_IDX 2
+#define mmCM3_CM_3DLUT_OUT_NORM_FACTOR 0x1226
+#define mmCM3_CM_3DLUT_OUT_NORM_FACTOR_BASE_IDX 2
+#define mmCM3_CM_3DLUT_OUT_OFFSET_R 0x1227
+#define mmCM3_CM_3DLUT_OUT_OFFSET_R_BASE_IDX 2
+#define mmCM3_CM_3DLUT_OUT_OFFSET_G 0x1228
+#define mmCM3_CM_3DLUT_OUT_OFFSET_G_BASE_IDX 2
+#define mmCM3_CM_3DLUT_OUT_OFFSET_B 0x1229
+#define mmCM3_CM_3DLUT_OUT_OFFSET_B_BASE_IDX 2
+#define mmCM3_CM_TEST_DEBUG_INDEX 0x122a
+#define mmCM3_CM_TEST_DEBUG_INDEX_BASE_IDX 2
+#define mmCM3_CM_TEST_DEBUG_DATA 0x122b
+#define mmCM3_CM_TEST_DEBUG_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp3_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x4994
+#define mmDC_PERFMON16_PERFCOUNTER_CNTL 0x1265
+#define mmDC_PERFMON16_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON16_PERFCOUNTER_CNTL2 0x1266
+#define mmDC_PERFMON16_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON16_PERFCOUNTER_STATE 0x1267
+#define mmDC_PERFMON16_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON16_PERFMON_CNTL 0x1268
+#define mmDC_PERFMON16_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON16_PERFMON_CNTL2 0x1269
+#define mmDC_PERFMON16_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON16_PERFMON_CVALUE_INT_MISC 0x126a
+#define mmDC_PERFMON16_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON16_PERFMON_CVALUE_LOW 0x126b
+#define mmDC_PERFMON16_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON16_PERFMON_HI 0x126c
+#define mmDC_PERFMON16_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON16_PERFMON_LOW 0x126d
+#define mmDC_PERFMON16_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpcc0_dispdec
+// base address: 0x0
+#define mmMPCC0_MPCC_TOP_SEL 0x1271
+#define mmMPCC0_MPCC_TOP_SEL_BASE_IDX 2
+#define mmMPCC0_MPCC_BOT_SEL 0x1272
+#define mmMPCC0_MPCC_BOT_SEL_BASE_IDX 2
+#define mmMPCC0_MPCC_OPP_ID 0x1273
+#define mmMPCC0_MPCC_OPP_ID_BASE_IDX 2
+#define mmMPCC0_MPCC_CONTROL 0x1274
+#define mmMPCC0_MPCC_CONTROL_BASE_IDX 2
+#define mmMPCC0_MPCC_SM_CONTROL 0x1275
+#define mmMPCC0_MPCC_SM_CONTROL_BASE_IDX 2
+#define mmMPCC0_MPCC_UPDATE_LOCK_SEL 0x1276
+#define mmMPCC0_MPCC_UPDATE_LOCK_SEL_BASE_IDX 2
+#define mmMPCC0_MPCC_TOP_GAIN 0x1277
+#define mmMPCC0_MPCC_TOP_GAIN_BASE_IDX 2
+#define mmMPCC0_MPCC_BOT_GAIN_INSIDE 0x1278
+#define mmMPCC0_MPCC_BOT_GAIN_INSIDE_BASE_IDX 2
+#define mmMPCC0_MPCC_BOT_GAIN_OUTSIDE 0x1279
+#define mmMPCC0_MPCC_BOT_GAIN_OUTSIDE_BASE_IDX 2
+#define mmMPCC0_MPCC_BG_R_CR 0x127a
+#define mmMPCC0_MPCC_BG_R_CR_BASE_IDX 2
+#define mmMPCC0_MPCC_BG_G_Y 0x127b
+#define mmMPCC0_MPCC_BG_G_Y_BASE_IDX 2
+#define mmMPCC0_MPCC_BG_B_CB 0x127c
+#define mmMPCC0_MPCC_BG_B_CB_BASE_IDX 2
+#define mmMPCC0_MPCC_MEM_PWR_CTRL 0x127d
+#define mmMPCC0_MPCC_MEM_PWR_CTRL_BASE_IDX 2
+#define mmMPCC0_MPCC_STALL_STATUS 0x127e
+#define mmMPCC0_MPCC_STALL_STATUS_BASE_IDX 2
+#define mmMPCC0_MPCC_STATUS 0x127f
+#define mmMPCC0_MPCC_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpcc1_dispdec
+// base address: 0x6c
+#define mmMPCC1_MPCC_TOP_SEL 0x128c
+#define mmMPCC1_MPCC_TOP_SEL_BASE_IDX 2
+#define mmMPCC1_MPCC_BOT_SEL 0x128d
+#define mmMPCC1_MPCC_BOT_SEL_BASE_IDX 2
+#define mmMPCC1_MPCC_OPP_ID 0x128e
+#define mmMPCC1_MPCC_OPP_ID_BASE_IDX 2
+#define mmMPCC1_MPCC_CONTROL 0x128f
+#define mmMPCC1_MPCC_CONTROL_BASE_IDX 2
+#define mmMPCC1_MPCC_SM_CONTROL 0x1290
+#define mmMPCC1_MPCC_SM_CONTROL_BASE_IDX 2
+#define mmMPCC1_MPCC_UPDATE_LOCK_SEL 0x1291
+#define mmMPCC1_MPCC_UPDATE_LOCK_SEL_BASE_IDX 2
+#define mmMPCC1_MPCC_TOP_GAIN 0x1292
+#define mmMPCC1_MPCC_TOP_GAIN_BASE_IDX 2
+#define mmMPCC1_MPCC_BOT_GAIN_INSIDE 0x1293
+#define mmMPCC1_MPCC_BOT_GAIN_INSIDE_BASE_IDX 2
+#define mmMPCC1_MPCC_BOT_GAIN_OUTSIDE 0x1294
+#define mmMPCC1_MPCC_BOT_GAIN_OUTSIDE_BASE_IDX 2
+#define mmMPCC1_MPCC_BG_R_CR 0x1295
+#define mmMPCC1_MPCC_BG_R_CR_BASE_IDX 2
+#define mmMPCC1_MPCC_BG_G_Y 0x1296
+#define mmMPCC1_MPCC_BG_G_Y_BASE_IDX 2
+#define mmMPCC1_MPCC_BG_B_CB 0x1297
+#define mmMPCC1_MPCC_BG_B_CB_BASE_IDX 2
+#define mmMPCC1_MPCC_MEM_PWR_CTRL 0x1298
+#define mmMPCC1_MPCC_MEM_PWR_CTRL_BASE_IDX 2
+#define mmMPCC1_MPCC_STALL_STATUS 0x1299
+#define mmMPCC1_MPCC_STALL_STATUS_BASE_IDX 2
+#define mmMPCC1_MPCC_STATUS 0x129a
+#define mmMPCC1_MPCC_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpcc2_dispdec
+// base address: 0xd8
+#define mmMPCC2_MPCC_TOP_SEL 0x12a7
+#define mmMPCC2_MPCC_TOP_SEL_BASE_IDX 2
+#define mmMPCC2_MPCC_BOT_SEL 0x12a8
+#define mmMPCC2_MPCC_BOT_SEL_BASE_IDX 2
+#define mmMPCC2_MPCC_OPP_ID 0x12a9
+#define mmMPCC2_MPCC_OPP_ID_BASE_IDX 2
+#define mmMPCC2_MPCC_CONTROL 0x12aa
+#define mmMPCC2_MPCC_CONTROL_BASE_IDX 2
+#define mmMPCC2_MPCC_SM_CONTROL 0x12ab
+#define mmMPCC2_MPCC_SM_CONTROL_BASE_IDX 2
+#define mmMPCC2_MPCC_UPDATE_LOCK_SEL 0x12ac
+#define mmMPCC2_MPCC_UPDATE_LOCK_SEL_BASE_IDX 2
+#define mmMPCC2_MPCC_TOP_GAIN 0x12ad
+#define mmMPCC2_MPCC_TOP_GAIN_BASE_IDX 2
+#define mmMPCC2_MPCC_BOT_GAIN_INSIDE 0x12ae
+#define mmMPCC2_MPCC_BOT_GAIN_INSIDE_BASE_IDX 2
+#define mmMPCC2_MPCC_BOT_GAIN_OUTSIDE 0x12af
+#define mmMPCC2_MPCC_BOT_GAIN_OUTSIDE_BASE_IDX 2
+#define mmMPCC2_MPCC_BG_R_CR 0x12b0
+#define mmMPCC2_MPCC_BG_R_CR_BASE_IDX 2
+#define mmMPCC2_MPCC_BG_G_Y 0x12b1
+#define mmMPCC2_MPCC_BG_G_Y_BASE_IDX 2
+#define mmMPCC2_MPCC_BG_B_CB 0x12b2
+#define mmMPCC2_MPCC_BG_B_CB_BASE_IDX 2
+#define mmMPCC2_MPCC_MEM_PWR_CTRL 0x12b3
+#define mmMPCC2_MPCC_MEM_PWR_CTRL_BASE_IDX 2
+#define mmMPCC2_MPCC_STALL_STATUS 0x12b4
+#define mmMPCC2_MPCC_STALL_STATUS_BASE_IDX 2
+#define mmMPCC2_MPCC_STATUS 0x12b5
+#define mmMPCC2_MPCC_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpcc3_dispdec
+// base address: 0x144
+#define mmMPCC3_MPCC_TOP_SEL 0x12c2
+#define mmMPCC3_MPCC_TOP_SEL_BASE_IDX 2
+#define mmMPCC3_MPCC_BOT_SEL 0x12c3
+#define mmMPCC3_MPCC_BOT_SEL_BASE_IDX 2
+#define mmMPCC3_MPCC_OPP_ID 0x12c4
+#define mmMPCC3_MPCC_OPP_ID_BASE_IDX 2
+#define mmMPCC3_MPCC_CONTROL 0x12c5
+#define mmMPCC3_MPCC_CONTROL_BASE_IDX 2
+#define mmMPCC3_MPCC_SM_CONTROL 0x12c6
+#define mmMPCC3_MPCC_SM_CONTROL_BASE_IDX 2
+#define mmMPCC3_MPCC_UPDATE_LOCK_SEL 0x12c7
+#define mmMPCC3_MPCC_UPDATE_LOCK_SEL_BASE_IDX 2
+#define mmMPCC3_MPCC_TOP_GAIN 0x12c8
+#define mmMPCC3_MPCC_TOP_GAIN_BASE_IDX 2
+#define mmMPCC3_MPCC_BOT_GAIN_INSIDE 0x12c9
+#define mmMPCC3_MPCC_BOT_GAIN_INSIDE_BASE_IDX 2
+#define mmMPCC3_MPCC_BOT_GAIN_OUTSIDE 0x12ca
+#define mmMPCC3_MPCC_BOT_GAIN_OUTSIDE_BASE_IDX 2
+#define mmMPCC3_MPCC_BG_R_CR 0x12cb
+#define mmMPCC3_MPCC_BG_R_CR_BASE_IDX 2
+#define mmMPCC3_MPCC_BG_G_Y 0x12cc
+#define mmMPCC3_MPCC_BG_G_Y_BASE_IDX 2
+#define mmMPCC3_MPCC_BG_B_CB 0x12cd
+#define mmMPCC3_MPCC_BG_B_CB_BASE_IDX 2
+#define mmMPCC3_MPCC_MEM_PWR_CTRL 0x12ce
+#define mmMPCC3_MPCC_MEM_PWR_CTRL_BASE_IDX 2
+#define mmMPCC3_MPCC_STALL_STATUS 0x12cf
+#define mmMPCC3_MPCC_STALL_STATUS_BASE_IDX 2
+#define mmMPCC3_MPCC_STATUS 0x12d0
+#define mmMPCC3_MPCC_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpcc4_dispdec
+// base address: 0x1b0
+#define mmMPCC4_MPCC_TOP_SEL 0x12dd
+#define mmMPCC4_MPCC_TOP_SEL_BASE_IDX 2
+#define mmMPCC4_MPCC_BOT_SEL 0x12de
+#define mmMPCC4_MPCC_BOT_SEL_BASE_IDX 2
+#define mmMPCC4_MPCC_OPP_ID 0x12df
+#define mmMPCC4_MPCC_OPP_ID_BASE_IDX 2
+#define mmMPCC4_MPCC_CONTROL 0x12e0
+#define mmMPCC4_MPCC_CONTROL_BASE_IDX 2
+#define mmMPCC4_MPCC_SM_CONTROL 0x12e1
+#define mmMPCC4_MPCC_SM_CONTROL_BASE_IDX 2
+#define mmMPCC4_MPCC_UPDATE_LOCK_SEL 0x12e2
+#define mmMPCC4_MPCC_UPDATE_LOCK_SEL_BASE_IDX 2
+#define mmMPCC4_MPCC_TOP_GAIN 0x12e3
+#define mmMPCC4_MPCC_TOP_GAIN_BASE_IDX 2
+#define mmMPCC4_MPCC_BOT_GAIN_INSIDE 0x12e4
+#define mmMPCC4_MPCC_BOT_GAIN_INSIDE_BASE_IDX 2
+#define mmMPCC4_MPCC_BOT_GAIN_OUTSIDE 0x12e5
+#define mmMPCC4_MPCC_BOT_GAIN_OUTSIDE_BASE_IDX 2
+#define mmMPCC4_MPCC_BG_R_CR 0x12e6
+#define mmMPCC4_MPCC_BG_R_CR_BASE_IDX 2
+#define mmMPCC4_MPCC_BG_G_Y 0x12e7
+#define mmMPCC4_MPCC_BG_G_Y_BASE_IDX 2
+#define mmMPCC4_MPCC_BG_B_CB 0x12e8
+#define mmMPCC4_MPCC_BG_B_CB_BASE_IDX 2
+#define mmMPCC4_MPCC_MEM_PWR_CTRL 0x12e9
+#define mmMPCC4_MPCC_MEM_PWR_CTRL_BASE_IDX 2
+#define mmMPCC4_MPCC_STALL_STATUS 0x12ea
+#define mmMPCC4_MPCC_STALL_STATUS_BASE_IDX 2
+#define mmMPCC4_MPCC_STATUS 0x12eb
+#define mmMPCC4_MPCC_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpcc5_dispdec
+// base address: 0x21c
+#define mmMPCC5_MPCC_TOP_SEL 0x12f8
+#define mmMPCC5_MPCC_TOP_SEL_BASE_IDX 2
+#define mmMPCC5_MPCC_BOT_SEL 0x12f9
+#define mmMPCC5_MPCC_BOT_SEL_BASE_IDX 2
+#define mmMPCC5_MPCC_OPP_ID 0x12fa
+#define mmMPCC5_MPCC_OPP_ID_BASE_IDX 2
+#define mmMPCC5_MPCC_CONTROL 0x12fb
+#define mmMPCC5_MPCC_CONTROL_BASE_IDX 2
+#define mmMPCC5_MPCC_SM_CONTROL 0x12fc
+#define mmMPCC5_MPCC_SM_CONTROL_BASE_IDX 2
+#define mmMPCC5_MPCC_UPDATE_LOCK_SEL 0x12fd
+#define mmMPCC5_MPCC_UPDATE_LOCK_SEL_BASE_IDX 2
+#define mmMPCC5_MPCC_TOP_GAIN 0x12fe
+#define mmMPCC5_MPCC_TOP_GAIN_BASE_IDX 2
+#define mmMPCC5_MPCC_BOT_GAIN_INSIDE 0x12ff
+#define mmMPCC5_MPCC_BOT_GAIN_INSIDE_BASE_IDX 2
+#define mmMPCC5_MPCC_BOT_GAIN_OUTSIDE 0x1300
+#define mmMPCC5_MPCC_BOT_GAIN_OUTSIDE_BASE_IDX 2
+#define mmMPCC5_MPCC_BG_R_CR 0x1301
+#define mmMPCC5_MPCC_BG_R_CR_BASE_IDX 2
+#define mmMPCC5_MPCC_BG_G_Y 0x1302
+#define mmMPCC5_MPCC_BG_G_Y_BASE_IDX 2
+#define mmMPCC5_MPCC_BG_B_CB 0x1303
+#define mmMPCC5_MPCC_BG_B_CB_BASE_IDX 2
+#define mmMPCC5_MPCC_MEM_PWR_CTRL 0x1304
+#define mmMPCC5_MPCC_MEM_PWR_CTRL_BASE_IDX 2
+#define mmMPCC5_MPCC_STALL_STATUS 0x1305
+#define mmMPCC5_MPCC_STALL_STATUS_BASE_IDX 2
+#define mmMPCC5_MPCC_STATUS 0x1306
+#define mmMPCC5_MPCC_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpcc6_dispdec
+// base address: 0x288
+#define mmMPCC6_MPCC_TOP_SEL 0x1313
+#define mmMPCC6_MPCC_TOP_SEL_BASE_IDX 2
+#define mmMPCC6_MPCC_BOT_SEL 0x1314
+#define mmMPCC6_MPCC_BOT_SEL_BASE_IDX 2
+#define mmMPCC6_MPCC_OPP_ID 0x1315
+#define mmMPCC6_MPCC_OPP_ID_BASE_IDX 2
+#define mmMPCC6_MPCC_CONTROL 0x1316
+#define mmMPCC6_MPCC_CONTROL_BASE_IDX 2
+#define mmMPCC6_MPCC_SM_CONTROL 0x1317
+#define mmMPCC6_MPCC_SM_CONTROL_BASE_IDX 2
+#define mmMPCC6_MPCC_UPDATE_LOCK_SEL 0x1318
+#define mmMPCC6_MPCC_UPDATE_LOCK_SEL_BASE_IDX 2
+#define mmMPCC6_MPCC_TOP_GAIN 0x1319
+#define mmMPCC6_MPCC_TOP_GAIN_BASE_IDX 2
+#define mmMPCC6_MPCC_BOT_GAIN_INSIDE 0x131a
+#define mmMPCC6_MPCC_BOT_GAIN_INSIDE_BASE_IDX 2
+#define mmMPCC6_MPCC_BOT_GAIN_OUTSIDE 0x131b
+#define mmMPCC6_MPCC_BOT_GAIN_OUTSIDE_BASE_IDX 2
+#define mmMPCC6_MPCC_BG_R_CR 0x131c
+#define mmMPCC6_MPCC_BG_R_CR_BASE_IDX 2
+#define mmMPCC6_MPCC_BG_G_Y 0x131d
+#define mmMPCC6_MPCC_BG_G_Y_BASE_IDX 2
+#define mmMPCC6_MPCC_BG_B_CB 0x131e
+#define mmMPCC6_MPCC_BG_B_CB_BASE_IDX 2
+#define mmMPCC6_MPCC_MEM_PWR_CTRL 0x131f
+#define mmMPCC6_MPCC_MEM_PWR_CTRL_BASE_IDX 2
+#define mmMPCC6_MPCC_STALL_STATUS 0x1320
+#define mmMPCC6_MPCC_STALL_STATUS_BASE_IDX 2
+#define mmMPCC6_MPCC_STATUS 0x1321
+#define mmMPCC6_MPCC_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpcc7_dispdec
+// base address: 0x2f4
+#define mmMPCC7_MPCC_TOP_SEL 0x132e
+#define mmMPCC7_MPCC_TOP_SEL_BASE_IDX 2
+#define mmMPCC7_MPCC_BOT_SEL 0x132f
+#define mmMPCC7_MPCC_BOT_SEL_BASE_IDX 2
+#define mmMPCC7_MPCC_OPP_ID 0x1330
+#define mmMPCC7_MPCC_OPP_ID_BASE_IDX 2
+#define mmMPCC7_MPCC_CONTROL 0x1331
+#define mmMPCC7_MPCC_CONTROL_BASE_IDX 2
+#define mmMPCC7_MPCC_SM_CONTROL 0x1332
+#define mmMPCC7_MPCC_SM_CONTROL_BASE_IDX 2
+#define mmMPCC7_MPCC_UPDATE_LOCK_SEL 0x1333
+#define mmMPCC7_MPCC_UPDATE_LOCK_SEL_BASE_IDX 2
+#define mmMPCC7_MPCC_TOP_GAIN 0x1334
+#define mmMPCC7_MPCC_TOP_GAIN_BASE_IDX 2
+#define mmMPCC7_MPCC_BOT_GAIN_INSIDE 0x1335
+#define mmMPCC7_MPCC_BOT_GAIN_INSIDE_BASE_IDX 2
+#define mmMPCC7_MPCC_BOT_GAIN_OUTSIDE 0x1336
+#define mmMPCC7_MPCC_BOT_GAIN_OUTSIDE_BASE_IDX 2
+#define mmMPCC7_MPCC_BG_R_CR 0x1337
+#define mmMPCC7_MPCC_BG_R_CR_BASE_IDX 2
+#define mmMPCC7_MPCC_BG_G_Y 0x1338
+#define mmMPCC7_MPCC_BG_G_Y_BASE_IDX 2
+#define mmMPCC7_MPCC_BG_B_CB 0x1339
+#define mmMPCC7_MPCC_BG_B_CB_BASE_IDX 2
+#define mmMPCC7_MPCC_MEM_PWR_CTRL 0x133a
+#define mmMPCC7_MPCC_MEM_PWR_CTRL_BASE_IDX 2
+#define mmMPCC7_MPCC_STALL_STATUS 0x133b
+#define mmMPCC7_MPCC_STALL_STATUS_BASE_IDX 2
+#define mmMPCC7_MPCC_STATUS 0x133c
+#define mmMPCC7_MPCC_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpc_cfg_dispdec
+// base address: 0x0
+#define mmMPC_CLOCK_CONTROL 0x1349
+#define mmMPC_CLOCK_CONTROL_BASE_IDX 2
+#define mmMPC_SOFT_RESET 0x134a
+#define mmMPC_SOFT_RESET_BASE_IDX 2
+#define mmMPC_CRC_CTRL 0x134b
+#define mmMPC_CRC_CTRL_BASE_IDX 2
+#define mmMPC_CRC_SEL_CONTROL 0x134c
+#define mmMPC_CRC_SEL_CONTROL_BASE_IDX 2
+#define mmMPC_CRC_RESULT_AR 0x134d
+#define mmMPC_CRC_RESULT_AR_BASE_IDX 2
+#define mmMPC_CRC_RESULT_GB 0x134e
+#define mmMPC_CRC_RESULT_GB_BASE_IDX 2
+#define mmMPC_CRC_RESULT_C 0x134f
+#define mmMPC_CRC_RESULT_C_BASE_IDX 2
+#define mmMPC_PERFMON_EVENT_CTRL 0x1352
+#define mmMPC_PERFMON_EVENT_CTRL_BASE_IDX 2
+#define mmMPC_BYPASS_BG_AR 0x1353
+#define mmMPC_BYPASS_BG_AR_BASE_IDX 2
+#define mmMPC_BYPASS_BG_GB 0x1354
+#define mmMPC_BYPASS_BG_GB_BASE_IDX 2
+#define mmMPC_STALL_GRACE_WINDOW 0x1355
+#define mmMPC_STALL_GRACE_WINDOW_BASE_IDX 2
+#define mmMPC_HOST_READ_CONTROL 0x1356
+#define mmMPC_HOST_READ_CONTROL_BASE_IDX 2
+#define mmMPC_PENDING_TAKEN_STATUS_REG1 0x1357
+#define mmMPC_PENDING_TAKEN_STATUS_REG1_BASE_IDX 2
+#define mmMPC_PENDING_TAKEN_STATUS_REG2 0x1358
+#define mmMPC_PENDING_TAKEN_STATUS_REG2_BASE_IDX 2
+#define mmMPC_PENDING_TAKEN_STATUS_REG3 0x1359
+#define mmMPC_PENDING_TAKEN_STATUS_REG3_BASE_IDX 2
+#define mmMPC_UPDATE_ACK_REG5 0x135b
+#define mmMPC_UPDATE_ACK_REG5_BASE_IDX 2
+#define mmMPC_UPDATE_ACK_REG6 0x135c
+#define mmMPC_UPDATE_ACK_REG6_BASE_IDX 2
+#define mmADR_CFG_CUR_VUPDATE_LOCK_SET0 0x135d
+#define mmADR_CFG_CUR_VUPDATE_LOCK_SET0_BASE_IDX 2
+#define mmADR_CFG_VUPDATE_LOCK_SET0 0x135e
+#define mmADR_CFG_VUPDATE_LOCK_SET0_BASE_IDX 2
+#define mmADR_VUPDATE_LOCK_SET0 0x135f
+#define mmADR_VUPDATE_LOCK_SET0_BASE_IDX 2
+#define mmCFG_VUPDATE_LOCK_SET0 0x1360
+#define mmCFG_VUPDATE_LOCK_SET0_BASE_IDX 2
+#define mmCUR_VUPDATE_LOCK_SET0 0x1361
+#define mmCUR_VUPDATE_LOCK_SET0_BASE_IDX 2
+#define mmADR_CFG_CUR_VUPDATE_LOCK_SET1 0x1362
+#define mmADR_CFG_CUR_VUPDATE_LOCK_SET1_BASE_IDX 2
+#define mmADR_CFG_VUPDATE_LOCK_SET1 0x1363
+#define mmADR_CFG_VUPDATE_LOCK_SET1_BASE_IDX 2
+#define mmADR_VUPDATE_LOCK_SET1 0x1364
+#define mmADR_VUPDATE_LOCK_SET1_BASE_IDX 2
+#define mmCFG_VUPDATE_LOCK_SET1 0x1365
+#define mmCFG_VUPDATE_LOCK_SET1_BASE_IDX 2
+#define mmCUR_VUPDATE_LOCK_SET1 0x1366
+#define mmCUR_VUPDATE_LOCK_SET1_BASE_IDX 2
+#define mmADR_CFG_CUR_VUPDATE_LOCK_SET2 0x1367
+#define mmADR_CFG_CUR_VUPDATE_LOCK_SET2_BASE_IDX 2
+#define mmADR_CFG_VUPDATE_LOCK_SET2 0x1368
+#define mmADR_CFG_VUPDATE_LOCK_SET2_BASE_IDX 2
+#define mmADR_VUPDATE_LOCK_SET2 0x1369
+#define mmADR_VUPDATE_LOCK_SET2_BASE_IDX 2
+#define mmCFG_VUPDATE_LOCK_SET2 0x136a
+#define mmCFG_VUPDATE_LOCK_SET2_BASE_IDX 2
+#define mmCUR_VUPDATE_LOCK_SET2 0x136b
+#define mmCUR_VUPDATE_LOCK_SET2_BASE_IDX 2
+#define mmADR_CFG_CUR_VUPDATE_LOCK_SET3 0x136c
+#define mmADR_CFG_CUR_VUPDATE_LOCK_SET3_BASE_IDX 2
+#define mmADR_CFG_VUPDATE_LOCK_SET3 0x136d
+#define mmADR_CFG_VUPDATE_LOCK_SET3_BASE_IDX 2
+#define mmADR_VUPDATE_LOCK_SET3 0x136e
+#define mmADR_VUPDATE_LOCK_SET3_BASE_IDX 2
+#define mmCFG_VUPDATE_LOCK_SET3 0x136f
+#define mmCFG_VUPDATE_LOCK_SET3_BASE_IDX 2
+#define mmCUR_VUPDATE_LOCK_SET3 0x1370
+#define mmCUR_VUPDATE_LOCK_SET3_BASE_IDX 2
+#define mmADR_CFG_CUR_VUPDATE_LOCK_SET4 0x1371
+#define mmADR_CFG_CUR_VUPDATE_LOCK_SET4_BASE_IDX 2
+#define mmADR_CFG_VUPDATE_LOCK_SET4 0x1372
+#define mmADR_CFG_VUPDATE_LOCK_SET4_BASE_IDX 2
+#define mmADR_VUPDATE_LOCK_SET4 0x1373
+#define mmADR_VUPDATE_LOCK_SET4_BASE_IDX 2
+#define mmCFG_VUPDATE_LOCK_SET4 0x1374
+#define mmCFG_VUPDATE_LOCK_SET4_BASE_IDX 2
+#define mmCUR_VUPDATE_LOCK_SET4 0x1375
+#define mmCUR_VUPDATE_LOCK_SET4_BASE_IDX 2
+#define mmADR_CFG_CUR_VUPDATE_LOCK_SET5 0x1376
+#define mmADR_CFG_CUR_VUPDATE_LOCK_SET5_BASE_IDX 2
+#define mmADR_CFG_VUPDATE_LOCK_SET5 0x1377
+#define mmADR_CFG_VUPDATE_LOCK_SET5_BASE_IDX 2
+#define mmADR_VUPDATE_LOCK_SET5 0x1378
+#define mmADR_VUPDATE_LOCK_SET5_BASE_IDX 2
+#define mmCFG_VUPDATE_LOCK_SET5 0x1379
+#define mmCFG_VUPDATE_LOCK_SET5_BASE_IDX 2
+#define mmCUR_VUPDATE_LOCK_SET5 0x137a
+#define mmCUR_VUPDATE_LOCK_SET5_BASE_IDX 2
+#define mmMPC_OUT0_MUX 0x1385
+#define mmMPC_OUT0_MUX_BASE_IDX 2
+#define mmMPC_OUT0_DENORM_CONTROL 0x1386
+#define mmMPC_OUT0_DENORM_CONTROL_BASE_IDX 2
+#define mmMPC_OUT0_DENORM_CLAMP_G_Y 0x1387
+#define mmMPC_OUT0_DENORM_CLAMP_G_Y_BASE_IDX 2
+#define mmMPC_OUT0_DENORM_CLAMP_B_CB 0x1388
+#define mmMPC_OUT0_DENORM_CLAMP_B_CB_BASE_IDX 2
+#define mmMPC_OUT1_MUX 0x1389
+#define mmMPC_OUT1_MUX_BASE_IDX 2
+#define mmMPC_OUT1_DENORM_CONTROL 0x138a
+#define mmMPC_OUT1_DENORM_CONTROL_BASE_IDX 2
+#define mmMPC_OUT1_DENORM_CLAMP_G_Y 0x138b
+#define mmMPC_OUT1_DENORM_CLAMP_G_Y_BASE_IDX 2
+#define mmMPC_OUT1_DENORM_CLAMP_B_CB 0x138c
+#define mmMPC_OUT1_DENORM_CLAMP_B_CB_BASE_IDX 2
+#define mmMPC_OUT2_MUX 0x138d
+#define mmMPC_OUT2_MUX_BASE_IDX 2
+#define mmMPC_OUT2_DENORM_CONTROL 0x138e
+#define mmMPC_OUT2_DENORM_CONTROL_BASE_IDX 2
+#define mmMPC_OUT2_DENORM_CLAMP_G_Y 0x138f
+#define mmMPC_OUT2_DENORM_CLAMP_G_Y_BASE_IDX 2
+#define mmMPC_OUT2_DENORM_CLAMP_B_CB 0x1390
+#define mmMPC_OUT2_DENORM_CLAMP_B_CB_BASE_IDX 2
+#define mmMPC_OUT3_MUX 0x1391
+#define mmMPC_OUT3_MUX_BASE_IDX 2
+#define mmMPC_OUT3_DENORM_CONTROL 0x1392
+#define mmMPC_OUT3_DENORM_CONTROL_BASE_IDX 2
+#define mmMPC_OUT3_DENORM_CLAMP_G_Y 0x1393
+#define mmMPC_OUT3_DENORM_CLAMP_G_Y_BASE_IDX 2
+#define mmMPC_OUT3_DENORM_CLAMP_B_CB 0x1394
+#define mmMPC_OUT3_DENORM_CLAMP_B_CB_BASE_IDX 2
+#define mmMPC_OUT4_MUX 0x1395
+#define mmMPC_OUT4_MUX_BASE_IDX 2
+#define mmMPC_OUT4_DENORM_CONTROL 0x1396
+#define mmMPC_OUT4_DENORM_CONTROL_BASE_IDX 2
+#define mmMPC_OUT4_DENORM_CLAMP_G_Y 0x1397
+#define mmMPC_OUT4_DENORM_CLAMP_G_Y_BASE_IDX 2
+#define mmMPC_OUT4_DENORM_CLAMP_B_CB 0x1398
+#define mmMPC_OUT4_DENORM_CLAMP_B_CB_BASE_IDX 2
+#define mmMPC_OUT5_MUX 0x1399
+#define mmMPC_OUT5_MUX_BASE_IDX 2
+#define mmMPC_OUT5_DENORM_CONTROL 0x139a
+#define mmMPC_OUT5_DENORM_CONTROL_BASE_IDX 2
+#define mmMPC_OUT5_DENORM_CLAMP_G_Y 0x139b
+#define mmMPC_OUT5_DENORM_CLAMP_G_Y_BASE_IDX 2
+#define mmMPC_OUT5_DENORM_CLAMP_B_CB 0x139c
+#define mmMPC_OUT5_DENORM_CLAMP_B_CB_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpcc_ogam0_dispdec
+// base address: 0x0
+#define mmMPCC_OGAM0_MPCC_OGAM_MODE 0x13ae
+#define mmMPCC_OGAM0_MPCC_OGAM_MODE_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_LUT_INDEX 0x13af
+#define mmMPCC_OGAM0_MPCC_OGAM_LUT_INDEX_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_LUT_DATA 0x13b0
+#define mmMPCC_OGAM0_MPCC_OGAM_LUT_DATA_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_LUT_RAM_CONTROL 0x13b1
+#define mmMPCC_OGAM0_MPCC_OGAM_LUT_RAM_CONTROL_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_B 0x13b2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_G 0x13b3
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_R 0x13b4
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_SLOPE_CNTL_B 0x13b5
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_SLOPE_CNTL_G 0x13b6
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_SLOPE_CNTL_R 0x13b7
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_B 0x13b8
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_B 0x13b9
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_G 0x13ba
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_G 0x13bb
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_R 0x13bc
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_R 0x13bd
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1 0x13be
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_2_3 0x13bf
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_4_5 0x13c0
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_6_7 0x13c1
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_8_9 0x13c2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_10_11 0x13c3
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_12_13 0x13c4
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_14_15 0x13c5
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_16_17 0x13c6
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_16_17_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_18_19 0x13c7
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_18_19_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_20_21 0x13c8
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_20_21_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_22_23 0x13c9
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_22_23_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_24_25 0x13ca
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_24_25_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_26_27 0x13cb
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_26_27_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_28_29 0x13cc
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_28_29_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_30_31 0x13cd
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_30_31_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_32_33 0x13ce
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMA_REGION_32_33_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_B 0x13cf
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_G 0x13d0
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_R 0x13d1
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_SLOPE_CNTL_B 0x13d2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_SLOPE_CNTL_G 0x13d3
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_SLOPE_CNTL_R 0x13d4
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_B 0x13d5
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_B 0x13d6
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_G 0x13d7
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_G 0x13d8
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_R 0x13d9
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_R 0x13da
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_0_1 0x13db
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_2_3 0x13dc
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_4_5 0x13dd
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_6_7 0x13de
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_8_9 0x13df
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_10_11 0x13e0
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_12_13 0x13e1
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_14_15 0x13e2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_16_17 0x13e3
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_16_17_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_18_19 0x13e4
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_18_19_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_20_21 0x13e5
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_20_21_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_22_23 0x13e6
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_22_23_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_24_25 0x13e7
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_24_25_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_26_27 0x13e8
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_26_27_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_28_29 0x13e9
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_28_29_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_30_31 0x13ea
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_30_31_BASE_IDX 2
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_32_33 0x13eb
+#define mmMPCC_OGAM0_MPCC_OGAM_RAMB_REGION_32_33_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpcc_ogam1_dispdec
+// base address: 0x104
+#define mmMPCC_OGAM1_MPCC_OGAM_MODE 0x13ef
+#define mmMPCC_OGAM1_MPCC_OGAM_MODE_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_LUT_INDEX 0x13f0
+#define mmMPCC_OGAM1_MPCC_OGAM_LUT_INDEX_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_LUT_DATA 0x13f1
+#define mmMPCC_OGAM1_MPCC_OGAM_LUT_DATA_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_LUT_RAM_CONTROL 0x13f2
+#define mmMPCC_OGAM1_MPCC_OGAM_LUT_RAM_CONTROL_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_B 0x13f3
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_G 0x13f4
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_R 0x13f5
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_SLOPE_CNTL_B 0x13f6
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_SLOPE_CNTL_G 0x13f7
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_SLOPE_CNTL_R 0x13f8
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_B 0x13f9
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_B 0x13fa
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_G 0x13fb
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_G 0x13fc
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_R 0x13fd
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_R 0x13fe
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_0_1 0x13ff
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_2_3 0x1400
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_4_5 0x1401
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_6_7 0x1402
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_8_9 0x1403
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_10_11 0x1404
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_12_13 0x1405
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_14_15 0x1406
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_16_17 0x1407
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_16_17_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_18_19 0x1408
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_18_19_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_20_21 0x1409
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_20_21_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_22_23 0x140a
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_22_23_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_24_25 0x140b
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_24_25_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_26_27 0x140c
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_26_27_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_28_29 0x140d
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_28_29_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_30_31 0x140e
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_30_31_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_32_33 0x140f
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMA_REGION_32_33_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_B 0x1410
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_G 0x1411
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_R 0x1412
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_SLOPE_CNTL_B 0x1413
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_SLOPE_CNTL_G 0x1414
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_SLOPE_CNTL_R 0x1415
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_B 0x1416
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_B 0x1417
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_G 0x1418
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_G 0x1419
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_R 0x141a
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_R 0x141b
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_0_1 0x141c
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_2_3 0x141d
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_4_5 0x141e
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_6_7 0x141f
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_8_9 0x1420
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_10_11 0x1421
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_12_13 0x1422
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_14_15 0x1423
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_16_17 0x1424
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_16_17_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_18_19 0x1425
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_18_19_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_20_21 0x1426
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_20_21_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_22_23 0x1427
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_22_23_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_24_25 0x1428
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_24_25_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_26_27 0x1429
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_26_27_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_28_29 0x142a
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_28_29_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_30_31 0x142b
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_30_31_BASE_IDX 2
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_32_33 0x142c
+#define mmMPCC_OGAM1_MPCC_OGAM_RAMB_REGION_32_33_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpcc_ogam2_dispdec
+// base address: 0x208
+#define mmMPCC_OGAM2_MPCC_OGAM_MODE 0x1430
+#define mmMPCC_OGAM2_MPCC_OGAM_MODE_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_LUT_INDEX 0x1431
+#define mmMPCC_OGAM2_MPCC_OGAM_LUT_INDEX_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_LUT_DATA 0x1432
+#define mmMPCC_OGAM2_MPCC_OGAM_LUT_DATA_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_LUT_RAM_CONTROL 0x1433
+#define mmMPCC_OGAM2_MPCC_OGAM_LUT_RAM_CONTROL_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_B 0x1434
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_G 0x1435
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_R 0x1436
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_SLOPE_CNTL_B 0x1437
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_SLOPE_CNTL_G 0x1438
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_SLOPE_CNTL_R 0x1439
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_B 0x143a
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_B 0x143b
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_G 0x143c
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_G 0x143d
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_R 0x143e
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_R 0x143f
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_0_1 0x1440
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_2_3 0x1441
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_4_5 0x1442
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_6_7 0x1443
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_8_9 0x1444
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_10_11 0x1445
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_12_13 0x1446
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_14_15 0x1447
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_16_17 0x1448
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_16_17_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_18_19 0x1449
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_18_19_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_20_21 0x144a
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_20_21_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_22_23 0x144b
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_22_23_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_24_25 0x144c
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_24_25_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_26_27 0x144d
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_26_27_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_28_29 0x144e
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_28_29_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_30_31 0x144f
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_30_31_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_32_33 0x1450
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMA_REGION_32_33_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_B 0x1451
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_G 0x1452
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_R 0x1453
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_SLOPE_CNTL_B 0x1454
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_SLOPE_CNTL_G 0x1455
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_SLOPE_CNTL_R 0x1456
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_B 0x1457
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_B 0x1458
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_G 0x1459
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_G 0x145a
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_R 0x145b
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_R 0x145c
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_0_1 0x145d
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_2_3 0x145e
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_4_5 0x145f
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_6_7 0x1460
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_8_9 0x1461
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_10_11 0x1462
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_12_13 0x1463
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_14_15 0x1464
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_16_17 0x1465
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_16_17_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_18_19 0x1466
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_18_19_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_20_21 0x1467
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_20_21_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_22_23 0x1468
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_22_23_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_24_25 0x1469
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_24_25_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_26_27 0x146a
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_26_27_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_28_29 0x146b
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_28_29_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_30_31 0x146c
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_30_31_BASE_IDX 2
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_32_33 0x146d
+#define mmMPCC_OGAM2_MPCC_OGAM_RAMB_REGION_32_33_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpcc_ogam3_dispdec
+// base address: 0x30c
+#define mmMPCC_OGAM3_MPCC_OGAM_MODE 0x1471
+#define mmMPCC_OGAM3_MPCC_OGAM_MODE_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_LUT_INDEX 0x1472
+#define mmMPCC_OGAM3_MPCC_OGAM_LUT_INDEX_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_LUT_DATA 0x1473
+#define mmMPCC_OGAM3_MPCC_OGAM_LUT_DATA_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_LUT_RAM_CONTROL 0x1474
+#define mmMPCC_OGAM3_MPCC_OGAM_LUT_RAM_CONTROL_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_B 0x1475
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_G 0x1476
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_R 0x1477
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_SLOPE_CNTL_B 0x1478
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_SLOPE_CNTL_G 0x1479
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_SLOPE_CNTL_R 0x147a
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_B 0x147b
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_B 0x147c
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_G 0x147d
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_G 0x147e
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_R 0x147f
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_R 0x1480
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_0_1 0x1481
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_2_3 0x1482
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_4_5 0x1483
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_6_7 0x1484
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_8_9 0x1485
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_10_11 0x1486
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_12_13 0x1487
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_14_15 0x1488
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_16_17 0x1489
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_16_17_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_18_19 0x148a
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_18_19_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_20_21 0x148b
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_20_21_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_22_23 0x148c
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_22_23_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_24_25 0x148d
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_24_25_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_26_27 0x148e
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_26_27_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_28_29 0x148f
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_28_29_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_30_31 0x1490
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_30_31_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_32_33 0x1491
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMA_REGION_32_33_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_B 0x1492
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_G 0x1493
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_R 0x1494
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_SLOPE_CNTL_B 0x1495
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_SLOPE_CNTL_G 0x1496
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_SLOPE_CNTL_R 0x1497
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_B 0x1498
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_B 0x1499
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_G 0x149a
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_G 0x149b
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_R 0x149c
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_R 0x149d
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_0_1 0x149e
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_2_3 0x149f
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_4_5 0x14a0
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_6_7 0x14a1
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_8_9 0x14a2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_10_11 0x14a3
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_12_13 0x14a4
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_14_15 0x14a5
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_16_17 0x14a6
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_16_17_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_18_19 0x14a7
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_18_19_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_20_21 0x14a8
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_20_21_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_22_23 0x14a9
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_22_23_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_24_25 0x14aa
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_24_25_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_26_27 0x14ab
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_26_27_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_28_29 0x14ac
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_28_29_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_30_31 0x14ad
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_30_31_BASE_IDX 2
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_32_33 0x14ae
+#define mmMPCC_OGAM3_MPCC_OGAM_RAMB_REGION_32_33_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpcc_ogam4_dispdec
+// base address: 0x410
+#define mmMPCC_OGAM4_MPCC_OGAM_MODE 0x14b2
+#define mmMPCC_OGAM4_MPCC_OGAM_MODE_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_LUT_INDEX 0x14b3
+#define mmMPCC_OGAM4_MPCC_OGAM_LUT_INDEX_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_LUT_DATA 0x14b4
+#define mmMPCC_OGAM4_MPCC_OGAM_LUT_DATA_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_LUT_RAM_CONTROL 0x14b5
+#define mmMPCC_OGAM4_MPCC_OGAM_LUT_RAM_CONTROL_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_START_CNTL_B 0x14b6
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_START_CNTL_G 0x14b7
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_START_CNTL_R 0x14b8
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_SLOPE_CNTL_B 0x14b9
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_SLOPE_CNTL_G 0x14ba
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_SLOPE_CNTL_R 0x14bb
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_END_CNTL1_B 0x14bc
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_END_CNTL2_B 0x14bd
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_END_CNTL1_G 0x14be
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_END_CNTL2_G 0x14bf
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_END_CNTL1_R 0x14c0
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_END_CNTL2_R 0x14c1
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_0_1 0x14c2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_2_3 0x14c3
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_4_5 0x14c4
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_6_7 0x14c5
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_8_9 0x14c6
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_10_11 0x14c7
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_12_13 0x14c8
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_14_15 0x14c9
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_16_17 0x14ca
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_16_17_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_18_19 0x14cb
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_18_19_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_20_21 0x14cc
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_20_21_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_22_23 0x14cd
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_22_23_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_24_25 0x14ce
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_24_25_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_26_27 0x14cf
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_26_27_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_28_29 0x14d0
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_28_29_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_30_31 0x14d1
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_30_31_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_32_33 0x14d2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMA_REGION_32_33_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_START_CNTL_B 0x14d3
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_START_CNTL_G 0x14d4
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_START_CNTL_R 0x14d5
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_SLOPE_CNTL_B 0x14d6
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_SLOPE_CNTL_G 0x14d7
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_SLOPE_CNTL_R 0x14d8
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_END_CNTL1_B 0x14d9
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_END_CNTL2_B 0x14da
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_END_CNTL1_G 0x14db
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_END_CNTL2_G 0x14dc
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_END_CNTL1_R 0x14dd
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_END_CNTL2_R 0x14de
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_0_1 0x14df
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_2_3 0x14e0
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_4_5 0x14e1
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_6_7 0x14e2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_8_9 0x14e3
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_10_11 0x14e4
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_12_13 0x14e5
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_14_15 0x14e6
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_16_17 0x14e7
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_16_17_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_18_19 0x14e8
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_18_19_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_20_21 0x14e9
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_20_21_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_22_23 0x14ea
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_22_23_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_24_25 0x14eb
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_24_25_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_26_27 0x14ec
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_26_27_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_28_29 0x14ed
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_28_29_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_30_31 0x14ee
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_30_31_BASE_IDX 2
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_32_33 0x14ef
+#define mmMPCC_OGAM4_MPCC_OGAM_RAMB_REGION_32_33_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpcc_ogam5_dispdec
+// base address: 0x514
+#define mmMPCC_OGAM5_MPCC_OGAM_MODE 0x14f3
+#define mmMPCC_OGAM5_MPCC_OGAM_MODE_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_LUT_INDEX 0x14f4
+#define mmMPCC_OGAM5_MPCC_OGAM_LUT_INDEX_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_LUT_DATA 0x14f5
+#define mmMPCC_OGAM5_MPCC_OGAM_LUT_DATA_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_LUT_RAM_CONTROL 0x14f6
+#define mmMPCC_OGAM5_MPCC_OGAM_LUT_RAM_CONTROL_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_START_CNTL_B 0x14f7
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_START_CNTL_G 0x14f8
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_START_CNTL_R 0x14f9
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_SLOPE_CNTL_B 0x14fa
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_SLOPE_CNTL_G 0x14fb
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_SLOPE_CNTL_R 0x14fc
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_END_CNTL1_B 0x14fd
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_END_CNTL2_B 0x14fe
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_END_CNTL1_G 0x14ff
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_END_CNTL2_G 0x1500
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_END_CNTL1_R 0x1501
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_END_CNTL2_R 0x1502
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_0_1 0x1503
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_2_3 0x1504
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_4_5 0x1505
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_6_7 0x1506
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_8_9 0x1507
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_10_11 0x1508
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_12_13 0x1509
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_14_15 0x150a
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_16_17 0x150b
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_16_17_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_18_19 0x150c
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_18_19_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_20_21 0x150d
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_20_21_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_22_23 0x150e
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_22_23_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_24_25 0x150f
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_24_25_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_26_27 0x1510
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_26_27_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_28_29 0x1511
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_28_29_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_30_31 0x1512
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_30_31_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_32_33 0x1513
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMA_REGION_32_33_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_START_CNTL_B 0x1514
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_START_CNTL_G 0x1515
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_START_CNTL_R 0x1516
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_SLOPE_CNTL_B 0x1517
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_SLOPE_CNTL_G 0x1518
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_SLOPE_CNTL_R 0x1519
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_END_CNTL1_B 0x151a
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_END_CNTL2_B 0x151b
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_END_CNTL1_G 0x151c
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_END_CNTL2_G 0x151d
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_END_CNTL1_R 0x151e
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_END_CNTL2_R 0x151f
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_0_1 0x1520
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_2_3 0x1521
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_4_5 0x1522
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_6_7 0x1523
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_8_9 0x1524
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_10_11 0x1525
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_12_13 0x1526
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_14_15 0x1527
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_16_17 0x1528
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_16_17_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_18_19 0x1529
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_18_19_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_20_21 0x152a
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_20_21_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_22_23 0x152b
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_22_23_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_24_25 0x152c
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_24_25_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_26_27 0x152d
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_26_27_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_28_29 0x152e
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_28_29_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_30_31 0x152f
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_30_31_BASE_IDX 2
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_32_33 0x1530
+#define mmMPCC_OGAM5_MPCC_OGAM_RAMB_REGION_32_33_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpcc_ogam6_dispdec
+// base address: 0x618
+#define mmMPCC_OGAM6_MPCC_OGAM_MODE 0x1534
+#define mmMPCC_OGAM6_MPCC_OGAM_MODE_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_LUT_INDEX 0x1535
+#define mmMPCC_OGAM6_MPCC_OGAM_LUT_INDEX_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_LUT_DATA 0x1536
+#define mmMPCC_OGAM6_MPCC_OGAM_LUT_DATA_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_LUT_RAM_CONTROL 0x1537
+#define mmMPCC_OGAM6_MPCC_OGAM_LUT_RAM_CONTROL_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_START_CNTL_B 0x1538
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_START_CNTL_G 0x1539
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_START_CNTL_R 0x153a
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_SLOPE_CNTL_B 0x153b
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_SLOPE_CNTL_G 0x153c
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_SLOPE_CNTL_R 0x153d
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_END_CNTL1_B 0x153e
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_END_CNTL2_B 0x153f
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_END_CNTL1_G 0x1540
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_END_CNTL2_G 0x1541
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_END_CNTL1_R 0x1542
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_END_CNTL2_R 0x1543
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_0_1 0x1544
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_2_3 0x1545
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_4_5 0x1546
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_6_7 0x1547
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_8_9 0x1548
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_10_11 0x1549
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_12_13 0x154a
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_14_15 0x154b
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_16_17 0x154c
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_16_17_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_18_19 0x154d
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_18_19_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_20_21 0x154e
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_20_21_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_22_23 0x154f
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_22_23_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_24_25 0x1550
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_24_25_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_26_27 0x1551
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_26_27_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_28_29 0x1552
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_28_29_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_30_31 0x1553
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_30_31_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_32_33 0x1554
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMA_REGION_32_33_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_START_CNTL_B 0x1555
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_START_CNTL_G 0x1556
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_START_CNTL_R 0x1557
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_SLOPE_CNTL_B 0x1558
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_SLOPE_CNTL_G 0x1559
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_SLOPE_CNTL_R 0x155a
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_END_CNTL1_B 0x155b
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_END_CNTL2_B 0x155c
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_END_CNTL1_G 0x155d
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_END_CNTL2_G 0x155e
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_END_CNTL1_R 0x155f
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_END_CNTL2_R 0x1560
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_0_1 0x1561
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_2_3 0x1562
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_4_5 0x1563
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_6_7 0x1564
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_8_9 0x1565
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_10_11 0x1566
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_12_13 0x1567
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_14_15 0x1568
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_16_17 0x1569
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_16_17_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_18_19 0x156a
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_18_19_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_20_21 0x156b
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_20_21_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_22_23 0x156c
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_22_23_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_24_25 0x156d
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_24_25_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_26_27 0x156e
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_26_27_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_28_29 0x156f
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_28_29_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_30_31 0x1570
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_30_31_BASE_IDX 2
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_32_33 0x1571
+#define mmMPCC_OGAM6_MPCC_OGAM_RAMB_REGION_32_33_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpcc_ogam7_dispdec
+// base address: 0x71c
+#define mmMPCC_OGAM7_MPCC_OGAM_MODE 0x1575
+#define mmMPCC_OGAM7_MPCC_OGAM_MODE_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_LUT_INDEX 0x1576
+#define mmMPCC_OGAM7_MPCC_OGAM_LUT_INDEX_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_LUT_DATA 0x1577
+#define mmMPCC_OGAM7_MPCC_OGAM_LUT_DATA_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_LUT_RAM_CONTROL 0x1578
+#define mmMPCC_OGAM7_MPCC_OGAM_LUT_RAM_CONTROL_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_START_CNTL_B 0x1579
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_START_CNTL_G 0x157a
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_START_CNTL_R 0x157b
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_SLOPE_CNTL_B 0x157c
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_SLOPE_CNTL_G 0x157d
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_SLOPE_CNTL_R 0x157e
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_END_CNTL1_B 0x157f
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_END_CNTL2_B 0x1580
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_END_CNTL1_G 0x1581
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_END_CNTL2_G 0x1582
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_END_CNTL1_R 0x1583
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_END_CNTL2_R 0x1584
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_0_1 0x1585
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_2_3 0x1586
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_4_5 0x1587
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_6_7 0x1588
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_8_9 0x1589
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_10_11 0x158a
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_12_13 0x158b
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_14_15 0x158c
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_16_17 0x158d
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_16_17_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_18_19 0x158e
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_18_19_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_20_21 0x158f
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_20_21_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_22_23 0x1590
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_22_23_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_24_25 0x1591
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_24_25_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_26_27 0x1592
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_26_27_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_28_29 0x1593
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_28_29_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_30_31 0x1594
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_30_31_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_32_33 0x1595
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMA_REGION_32_33_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_START_CNTL_B 0x1596
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_START_CNTL_G 0x1597
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_START_CNTL_R 0x1598
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_SLOPE_CNTL_B 0x1599
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_SLOPE_CNTL_G 0x159a
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_SLOPE_CNTL_R 0x159b
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_END_CNTL1_B 0x159c
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_END_CNTL2_B 0x159d
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_END_CNTL1_G 0x159e
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_END_CNTL2_G 0x159f
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_END_CNTL1_R 0x15a0
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_END_CNTL2_R 0x15a1
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_0_1 0x15a2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_2_3 0x15a3
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_4_5 0x15a4
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_6_7 0x15a5
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_8_9 0x15a6
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_10_11 0x15a7
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_12_13 0x15a8
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_14_15 0x15a9
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_16_17 0x15aa
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_16_17_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_18_19 0x15ab
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_18_19_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_20_21 0x15ac
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_20_21_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_22_23 0x15ad
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_22_23_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_24_25 0x15ae
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_24_25_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_26_27 0x15af
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_26_27_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_28_29 0x15b0
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_28_29_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_30_31 0x15b1
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_30_31_BASE_IDX 2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_32_33 0x15b2
+#define mmMPCC_OGAM7_MPCC_OGAM_RAMB_REGION_32_33_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpc_ocsc_dispdec
+// base address: 0x0
+#define mmMPC_OUT_CSC_COEF_FORMAT 0x15b6
+#define mmMPC_OUT_CSC_COEF_FORMAT_BASE_IDX 2
+#define mmMPC_OUT0_CSC_MODE 0x15b7
+#define mmMPC_OUT0_CSC_MODE_BASE_IDX 2
+#define mmMPC_OUT0_CSC_C11_C12_A 0x15b8
+#define mmMPC_OUT0_CSC_C11_C12_A_BASE_IDX 2
+#define mmMPC_OUT0_CSC_C13_C14_A 0x15b9
+#define mmMPC_OUT0_CSC_C13_C14_A_BASE_IDX 2
+#define mmMPC_OUT0_CSC_C21_C22_A 0x15ba
+#define mmMPC_OUT0_CSC_C21_C22_A_BASE_IDX 2
+#define mmMPC_OUT0_CSC_C23_C24_A 0x15bb
+#define mmMPC_OUT0_CSC_C23_C24_A_BASE_IDX 2
+#define mmMPC_OUT0_CSC_C31_C32_A 0x15bc
+#define mmMPC_OUT0_CSC_C31_C32_A_BASE_IDX 2
+#define mmMPC_OUT0_CSC_C33_C34_A 0x15bd
+#define mmMPC_OUT0_CSC_C33_C34_A_BASE_IDX 2
+#define mmMPC_OUT0_CSC_C11_C12_B 0x15be
+#define mmMPC_OUT0_CSC_C11_C12_B_BASE_IDX 2
+#define mmMPC_OUT0_CSC_C13_C14_B 0x15bf
+#define mmMPC_OUT0_CSC_C13_C14_B_BASE_IDX 2
+#define mmMPC_OUT0_CSC_C21_C22_B 0x15c0
+#define mmMPC_OUT0_CSC_C21_C22_B_BASE_IDX 2
+#define mmMPC_OUT0_CSC_C23_C24_B 0x15c1
+#define mmMPC_OUT0_CSC_C23_C24_B_BASE_IDX 2
+#define mmMPC_OUT0_CSC_C31_C32_B 0x15c2
+#define mmMPC_OUT0_CSC_C31_C32_B_BASE_IDX 2
+#define mmMPC_OUT0_CSC_C33_C34_B 0x15c3
+#define mmMPC_OUT0_CSC_C33_C34_B_BASE_IDX 2
+#define mmMPC_OUT1_CSC_MODE 0x15c4
+#define mmMPC_OUT1_CSC_MODE_BASE_IDX 2
+#define mmMPC_OUT1_CSC_C11_C12_A 0x15c5
+#define mmMPC_OUT1_CSC_C11_C12_A_BASE_IDX 2
+#define mmMPC_OUT1_CSC_C13_C14_A 0x15c6
+#define mmMPC_OUT1_CSC_C13_C14_A_BASE_IDX 2
+#define mmMPC_OUT1_CSC_C21_C22_A 0x15c7
+#define mmMPC_OUT1_CSC_C21_C22_A_BASE_IDX 2
+#define mmMPC_OUT1_CSC_C23_C24_A 0x15c8
+#define mmMPC_OUT1_CSC_C23_C24_A_BASE_IDX 2
+#define mmMPC_OUT1_CSC_C31_C32_A 0x15c9
+#define mmMPC_OUT1_CSC_C31_C32_A_BASE_IDX 2
+#define mmMPC_OUT1_CSC_C33_C34_A 0x15ca
+#define mmMPC_OUT1_CSC_C33_C34_A_BASE_IDX 2
+#define mmMPC_OUT1_CSC_C11_C12_B 0x15cb
+#define mmMPC_OUT1_CSC_C11_C12_B_BASE_IDX 2
+#define mmMPC_OUT1_CSC_C13_C14_B 0x15cc
+#define mmMPC_OUT1_CSC_C13_C14_B_BASE_IDX 2
+#define mmMPC_OUT1_CSC_C21_C22_B 0x15cd
+#define mmMPC_OUT1_CSC_C21_C22_B_BASE_IDX 2
+#define mmMPC_OUT1_CSC_C23_C24_B 0x15ce
+#define mmMPC_OUT1_CSC_C23_C24_B_BASE_IDX 2
+#define mmMPC_OUT1_CSC_C31_C32_B 0x15cf
+#define mmMPC_OUT1_CSC_C31_C32_B_BASE_IDX 2
+#define mmMPC_OUT1_CSC_C33_C34_B 0x15d0
+#define mmMPC_OUT1_CSC_C33_C34_B_BASE_IDX 2
+#define mmMPC_OUT2_CSC_MODE 0x15d1
+#define mmMPC_OUT2_CSC_MODE_BASE_IDX 2
+#define mmMPC_OUT2_CSC_C11_C12_A 0x15d2
+#define mmMPC_OUT2_CSC_C11_C12_A_BASE_IDX 2
+#define mmMPC_OUT2_CSC_C13_C14_A 0x15d3
+#define mmMPC_OUT2_CSC_C13_C14_A_BASE_IDX 2
+#define mmMPC_OUT2_CSC_C21_C22_A 0x15d4
+#define mmMPC_OUT2_CSC_C21_C22_A_BASE_IDX 2
+#define mmMPC_OUT2_CSC_C23_C24_A 0x15d5
+#define mmMPC_OUT2_CSC_C23_C24_A_BASE_IDX 2
+#define mmMPC_OUT2_CSC_C31_C32_A 0x15d6
+#define mmMPC_OUT2_CSC_C31_C32_A_BASE_IDX 2
+#define mmMPC_OUT2_CSC_C33_C34_A 0x15d7
+#define mmMPC_OUT2_CSC_C33_C34_A_BASE_IDX 2
+#define mmMPC_OUT2_CSC_C11_C12_B 0x15d8
+#define mmMPC_OUT2_CSC_C11_C12_B_BASE_IDX 2
+#define mmMPC_OUT2_CSC_C13_C14_B 0x15d9
+#define mmMPC_OUT2_CSC_C13_C14_B_BASE_IDX 2
+#define mmMPC_OUT2_CSC_C21_C22_B 0x15da
+#define mmMPC_OUT2_CSC_C21_C22_B_BASE_IDX 2
+#define mmMPC_OUT2_CSC_C23_C24_B 0x15db
+#define mmMPC_OUT2_CSC_C23_C24_B_BASE_IDX 2
+#define mmMPC_OUT2_CSC_C31_C32_B 0x15dc
+#define mmMPC_OUT2_CSC_C31_C32_B_BASE_IDX 2
+#define mmMPC_OUT2_CSC_C33_C34_B 0x15dd
+#define mmMPC_OUT2_CSC_C33_C34_B_BASE_IDX 2
+#define mmMPC_OUT3_CSC_MODE 0x15de
+#define mmMPC_OUT3_CSC_MODE_BASE_IDX 2
+#define mmMPC_OUT3_CSC_C11_C12_A 0x15df
+#define mmMPC_OUT3_CSC_C11_C12_A_BASE_IDX 2
+#define mmMPC_OUT3_CSC_C13_C14_A 0x15e0
+#define mmMPC_OUT3_CSC_C13_C14_A_BASE_IDX 2
+#define mmMPC_OUT3_CSC_C21_C22_A 0x15e1
+#define mmMPC_OUT3_CSC_C21_C22_A_BASE_IDX 2
+#define mmMPC_OUT3_CSC_C23_C24_A 0x15e2
+#define mmMPC_OUT3_CSC_C23_C24_A_BASE_IDX 2
+#define mmMPC_OUT3_CSC_C31_C32_A 0x15e3
+#define mmMPC_OUT3_CSC_C31_C32_A_BASE_IDX 2
+#define mmMPC_OUT3_CSC_C33_C34_A 0x15e4
+#define mmMPC_OUT3_CSC_C33_C34_A_BASE_IDX 2
+#define mmMPC_OUT3_CSC_C11_C12_B 0x15e5
+#define mmMPC_OUT3_CSC_C11_C12_B_BASE_IDX 2
+#define mmMPC_OUT3_CSC_C13_C14_B 0x15e6
+#define mmMPC_OUT3_CSC_C13_C14_B_BASE_IDX 2
+#define mmMPC_OUT3_CSC_C21_C22_B 0x15e7
+#define mmMPC_OUT3_CSC_C21_C22_B_BASE_IDX 2
+#define mmMPC_OUT3_CSC_C23_C24_B 0x15e8
+#define mmMPC_OUT3_CSC_C23_C24_B_BASE_IDX 2
+#define mmMPC_OUT3_CSC_C31_C32_B 0x15e9
+#define mmMPC_OUT3_CSC_C31_C32_B_BASE_IDX 2
+#define mmMPC_OUT3_CSC_C33_C34_B 0x15ea
+#define mmMPC_OUT3_CSC_C33_C34_B_BASE_IDX 2
+#define mmMPC_OUT4_CSC_MODE 0x15eb
+#define mmMPC_OUT4_CSC_MODE_BASE_IDX 2
+#define mmMPC_OUT4_CSC_C11_C12_A 0x15ec
+#define mmMPC_OUT4_CSC_C11_C12_A_BASE_IDX 2
+#define mmMPC_OUT4_CSC_C13_C14_A 0x15ed
+#define mmMPC_OUT4_CSC_C13_C14_A_BASE_IDX 2
+#define mmMPC_OUT4_CSC_C21_C22_A 0x15ee
+#define mmMPC_OUT4_CSC_C21_C22_A_BASE_IDX 2
+#define mmMPC_OUT4_CSC_C23_C24_A 0x15ef
+#define mmMPC_OUT4_CSC_C23_C24_A_BASE_IDX 2
+#define mmMPC_OUT4_CSC_C31_C32_A 0x15f0
+#define mmMPC_OUT4_CSC_C31_C32_A_BASE_IDX 2
+#define mmMPC_OUT4_CSC_C33_C34_A 0x15f1
+#define mmMPC_OUT4_CSC_C33_C34_A_BASE_IDX 2
+#define mmMPC_OUT4_CSC_C11_C12_B 0x15f2
+#define mmMPC_OUT4_CSC_C11_C12_B_BASE_IDX 2
+#define mmMPC_OUT4_CSC_C13_C14_B 0x15f3
+#define mmMPC_OUT4_CSC_C13_C14_B_BASE_IDX 2
+#define mmMPC_OUT4_CSC_C21_C22_B 0x15f4
+#define mmMPC_OUT4_CSC_C21_C22_B_BASE_IDX 2
+#define mmMPC_OUT4_CSC_C23_C24_B 0x15f5
+#define mmMPC_OUT4_CSC_C23_C24_B_BASE_IDX 2
+#define mmMPC_OUT4_CSC_C31_C32_B 0x15f6
+#define mmMPC_OUT4_CSC_C31_C32_B_BASE_IDX 2
+#define mmMPC_OUT4_CSC_C33_C34_B 0x15f7
+#define mmMPC_OUT4_CSC_C33_C34_B_BASE_IDX 2
+#define mmMPC_OUT5_CSC_MODE 0x15f8
+#define mmMPC_OUT5_CSC_MODE_BASE_IDX 2
+#define mmMPC_OUT5_CSC_C11_C12_A 0x15f9
+#define mmMPC_OUT5_CSC_C11_C12_A_BASE_IDX 2
+#define mmMPC_OUT5_CSC_C13_C14_A 0x15fa
+#define mmMPC_OUT5_CSC_C13_C14_A_BASE_IDX 2
+#define mmMPC_OUT5_CSC_C21_C22_A 0x15fb
+#define mmMPC_OUT5_CSC_C21_C22_A_BASE_IDX 2
+#define mmMPC_OUT5_CSC_C23_C24_A 0x15fc
+#define mmMPC_OUT5_CSC_C23_C24_A_BASE_IDX 2
+#define mmMPC_OUT5_CSC_C31_C32_A 0x15fd
+#define mmMPC_OUT5_CSC_C31_C32_A_BASE_IDX 2
+#define mmMPC_OUT5_CSC_C33_C34_A 0x15fe
+#define mmMPC_OUT5_CSC_C33_C34_A_BASE_IDX 2
+#define mmMPC_OUT5_CSC_C11_C12_B 0x15ff
+#define mmMPC_OUT5_CSC_C11_C12_B_BASE_IDX 2
+#define mmMPC_OUT5_CSC_C13_C14_B 0x1600
+#define mmMPC_OUT5_CSC_C13_C14_B_BASE_IDX 2
+#define mmMPC_OUT5_CSC_C21_C22_B 0x1601
+#define mmMPC_OUT5_CSC_C21_C22_B_BASE_IDX 2
+#define mmMPC_OUT5_CSC_C23_C24_B 0x1602
+#define mmMPC_OUT5_CSC_C23_C24_B_BASE_IDX 2
+#define mmMPC_OUT5_CSC_C31_C32_B 0x1603
+#define mmMPC_OUT5_CSC_C31_C32_B_BASE_IDX 2
+#define mmMPC_OUT5_CSC_C33_C34_B 0x1604
+#define mmMPC_OUT5_CSC_C33_C34_B_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mpc_mpc_dcperfmon_dc_perfmon_dispdec
+// base address: 0x5964
+#define mmDC_PERFMON17_PERFCOUNTER_CNTL 0x1659
+#define mmDC_PERFMON17_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON17_PERFCOUNTER_CNTL2 0x165a
+#define mmDC_PERFMON17_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON17_PERFCOUNTER_STATE 0x165b
+#define mmDC_PERFMON17_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON17_PERFMON_CNTL 0x165c
+#define mmDC_PERFMON17_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON17_PERFMON_CNTL2 0x165d
+#define mmDC_PERFMON17_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON17_PERFMON_CVALUE_INT_MISC 0x165e
+#define mmDC_PERFMON17_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON17_PERFMON_CVALUE_LOW 0x165f
+#define mmDC_PERFMON17_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON17_PERFMON_HI 0x1660
+#define mmDC_PERFMON17_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON17_PERFMON_LOW 0x1661
+#define mmDC_PERFMON17_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_abm0_dispdec
+// base address: 0x0
+#define mmBL1_PWM_AMBIENT_LIGHT_LEVEL 0x17b0
+#define mmBL1_PWM_AMBIENT_LIGHT_LEVEL_BASE_IDX 2
+#define mmBL1_PWM_USER_LEVEL 0x17b1
+#define mmBL1_PWM_USER_LEVEL_BASE_IDX 2
+#define mmBL1_PWM_TARGET_ABM_LEVEL 0x17b2
+#define mmBL1_PWM_TARGET_ABM_LEVEL_BASE_IDX 2
+#define mmBL1_PWM_CURRENT_ABM_LEVEL 0x17b3
+#define mmBL1_PWM_CURRENT_ABM_LEVEL_BASE_IDX 2
+#define mmBL1_PWM_FINAL_DUTY_CYCLE 0x17b4
+#define mmBL1_PWM_FINAL_DUTY_CYCLE_BASE_IDX 2
+#define mmBL1_PWM_MINIMUM_DUTY_CYCLE 0x17b5
+#define mmBL1_PWM_MINIMUM_DUTY_CYCLE_BASE_IDX 2
+#define mmBL1_PWM_ABM_CNTL 0x17b6
+#define mmBL1_PWM_ABM_CNTL_BASE_IDX 2
+#define mmBL1_PWM_BL_UPDATE_SAMPLE_RATE 0x17b7
+#define mmBL1_PWM_BL_UPDATE_SAMPLE_RATE_BASE_IDX 2
+#define mmBL1_PWM_GRP2_REG_LOCK 0x17b8
+#define mmBL1_PWM_GRP2_REG_LOCK_BASE_IDX 2
+#define mmDC_ABM1_CNTL 0x17b9
+#define mmDC_ABM1_CNTL_BASE_IDX 2
+#define mmDC_ABM1_IPCSC_COEFF_SEL 0x17ba
+#define mmDC_ABM1_IPCSC_COEFF_SEL_BASE_IDX 2
+#define mmDC_ABM1_ACE_OFFSET_SLOPE_0 0x17bb
+#define mmDC_ABM1_ACE_OFFSET_SLOPE_0_BASE_IDX 2
+#define mmDC_ABM1_ACE_OFFSET_SLOPE_1 0x17bc
+#define mmDC_ABM1_ACE_OFFSET_SLOPE_1_BASE_IDX 2
+#define mmDC_ABM1_ACE_OFFSET_SLOPE_2 0x17bd
+#define mmDC_ABM1_ACE_OFFSET_SLOPE_2_BASE_IDX 2
+#define mmDC_ABM1_ACE_OFFSET_SLOPE_3 0x17be
+#define mmDC_ABM1_ACE_OFFSET_SLOPE_3_BASE_IDX 2
+#define mmDC_ABM1_ACE_OFFSET_SLOPE_4 0x17bf
+#define mmDC_ABM1_ACE_OFFSET_SLOPE_4_BASE_IDX 2
+#define mmDC_ABM1_ACE_THRES_12 0x17c0
+#define mmDC_ABM1_ACE_THRES_12_BASE_IDX 2
+#define mmDC_ABM1_ACE_THRES_34 0x17c1
+#define mmDC_ABM1_ACE_THRES_34_BASE_IDX 2
+#define mmDC_ABM1_ACE_CNTL_MISC 0x17c2
+#define mmDC_ABM1_ACE_CNTL_MISC_BASE_IDX 2
+#define mmDC_ABM1_HGLS_REG_READ_PROGRESS 0x17c4
+#define mmDC_ABM1_HGLS_REG_READ_PROGRESS_BASE_IDX 2
+#define mmDC_ABM1_HG_MISC_CTRL 0x17c5
+#define mmDC_ABM1_HG_MISC_CTRL_BASE_IDX 2
+#define mmDC_ABM1_LS_SUM_OF_LUMA 0x17c6
+#define mmDC_ABM1_LS_SUM_OF_LUMA_BASE_IDX 2
+#define mmDC_ABM1_LS_MIN_MAX_LUMA 0x17c7
+#define mmDC_ABM1_LS_MIN_MAX_LUMA_BASE_IDX 2
+#define mmDC_ABM1_LS_FILTERED_MIN_MAX_LUMA 0x17c8
+#define mmDC_ABM1_LS_FILTERED_MIN_MAX_LUMA_BASE_IDX 2
+#define mmDC_ABM1_LS_PIXEL_COUNT 0x17c9
+#define mmDC_ABM1_LS_PIXEL_COUNT_BASE_IDX 2
+#define mmDC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES 0x17ca
+#define mmDC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES_BASE_IDX 2
+#define mmDC_ABM1_LS_MIN_PIXEL_VALUE_COUNT 0x17cb
+#define mmDC_ABM1_LS_MIN_PIXEL_VALUE_COUNT_BASE_IDX 2
+#define mmDC_ABM1_LS_MAX_PIXEL_VALUE_COUNT 0x17cc
+#define mmDC_ABM1_LS_MAX_PIXEL_VALUE_COUNT_BASE_IDX 2
+#define mmDC_ABM1_HG_SAMPLE_RATE 0x17cd
+#define mmDC_ABM1_HG_SAMPLE_RATE_BASE_IDX 2
+#define mmDC_ABM1_LS_SAMPLE_RATE 0x17ce
+#define mmDC_ABM1_LS_SAMPLE_RATE_BASE_IDX 2
+#define mmDC_ABM1_HG_BIN_1_32_SHIFT_FLAG 0x17cf
+#define mmDC_ABM1_HG_BIN_1_32_SHIFT_FLAG_BASE_IDX 2
+#define mmDC_ABM1_HG_BIN_1_8_SHIFT_INDEX 0x17d0
+#define mmDC_ABM1_HG_BIN_1_8_SHIFT_INDEX_BASE_IDX 2
+#define mmDC_ABM1_HG_BIN_9_16_SHIFT_INDEX 0x17d1
+#define mmDC_ABM1_HG_BIN_9_16_SHIFT_INDEX_BASE_IDX 2
+#define mmDC_ABM1_HG_BIN_17_24_SHIFT_INDEX 0x17d2
+#define mmDC_ABM1_HG_BIN_17_24_SHIFT_INDEX_BASE_IDX 2
+#define mmDC_ABM1_HG_BIN_25_32_SHIFT_INDEX 0x17d3
+#define mmDC_ABM1_HG_BIN_25_32_SHIFT_INDEX_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_1 0x17d4
+#define mmDC_ABM1_HG_RESULT_1_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_2 0x17d5
+#define mmDC_ABM1_HG_RESULT_2_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_3 0x17d6
+#define mmDC_ABM1_HG_RESULT_3_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_4 0x17d7
+#define mmDC_ABM1_HG_RESULT_4_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_5 0x17d8
+#define mmDC_ABM1_HG_RESULT_5_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_6 0x17d9
+#define mmDC_ABM1_HG_RESULT_6_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_7 0x17da
+#define mmDC_ABM1_HG_RESULT_7_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_8 0x17db
+#define mmDC_ABM1_HG_RESULT_8_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_9 0x17dc
+#define mmDC_ABM1_HG_RESULT_9_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_10 0x17dd
+#define mmDC_ABM1_HG_RESULT_10_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_11 0x17de
+#define mmDC_ABM1_HG_RESULT_11_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_12 0x17df
+#define mmDC_ABM1_HG_RESULT_12_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_13 0x17e0
+#define mmDC_ABM1_HG_RESULT_13_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_14 0x17e1
+#define mmDC_ABM1_HG_RESULT_14_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_15 0x17e2
+#define mmDC_ABM1_HG_RESULT_15_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_16 0x17e3
+#define mmDC_ABM1_HG_RESULT_16_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_17 0x17e4
+#define mmDC_ABM1_HG_RESULT_17_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_18 0x17e5
+#define mmDC_ABM1_HG_RESULT_18_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_19 0x17e6
+#define mmDC_ABM1_HG_RESULT_19_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_20 0x17e7
+#define mmDC_ABM1_HG_RESULT_20_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_21 0x17e8
+#define mmDC_ABM1_HG_RESULT_21_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_22 0x17e9
+#define mmDC_ABM1_HG_RESULT_22_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_23 0x17ea
+#define mmDC_ABM1_HG_RESULT_23_BASE_IDX 2
+#define mmDC_ABM1_HG_RESULT_24 0x17eb
+#define mmDC_ABM1_HG_RESULT_24_BASE_IDX 2
+#define mmDC_ABM1_BL_MASTER_LOCK 0x17ec
+#define mmDC_ABM1_BL_MASTER_LOCK_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_fmt0_dispdec
+// base address: 0x0
+#define mmFMT0_FMT_CLAMP_COMPONENT_R 0x183c
+#define mmFMT0_FMT_CLAMP_COMPONENT_R_BASE_IDX 2
+#define mmFMT0_FMT_CLAMP_COMPONENT_G 0x183d
+#define mmFMT0_FMT_CLAMP_COMPONENT_G_BASE_IDX 2
+#define mmFMT0_FMT_CLAMP_COMPONENT_B 0x183e
+#define mmFMT0_FMT_CLAMP_COMPONENT_B_BASE_IDX 2
+#define mmFMT0_FMT_DYNAMIC_EXP_CNTL 0x183f
+#define mmFMT0_FMT_DYNAMIC_EXP_CNTL_BASE_IDX 2
+#define mmFMT0_FMT_CONTROL 0x1840
+#define mmFMT0_FMT_CONTROL_BASE_IDX 2
+#define mmFMT0_FMT_BIT_DEPTH_CONTROL 0x1841
+#define mmFMT0_FMT_BIT_DEPTH_CONTROL_BASE_IDX 2
+#define mmFMT0_FMT_DITHER_RAND_R_SEED 0x1842
+#define mmFMT0_FMT_DITHER_RAND_R_SEED_BASE_IDX 2
+#define mmFMT0_FMT_DITHER_RAND_G_SEED 0x1843
+#define mmFMT0_FMT_DITHER_RAND_G_SEED_BASE_IDX 2
+#define mmFMT0_FMT_DITHER_RAND_B_SEED 0x1844
+#define mmFMT0_FMT_DITHER_RAND_B_SEED_BASE_IDX 2
+#define mmFMT0_FMT_CLAMP_CNTL 0x1845
+#define mmFMT0_FMT_CLAMP_CNTL_BASE_IDX 2
+#define mmFMT0_FMT_SIDE_BY_SIDE_STEREO_CONTROL 0x1846
+#define mmFMT0_FMT_SIDE_BY_SIDE_STEREO_CONTROL_BASE_IDX 2
+#define mmFMT0_FMT_MAP420_MEMORY_CONTROL 0x1847
+#define mmFMT0_FMT_MAP420_MEMORY_CONTROL_BASE_IDX 2
+#define mmFMT0_FMT_422_CONTROL 0x1849
+#define mmFMT0_FMT_422_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_dpg0_dispdec
+// base address: 0x0
+#define mmDPG0_DPG_CONTROL 0x1854
+#define mmDPG0_DPG_CONTROL_BASE_IDX 2
+#define mmDPG0_DPG_RAMP_CONTROL 0x1855
+#define mmDPG0_DPG_RAMP_CONTROL_BASE_IDX 2
+#define mmDPG0_DPG_DIMENSIONS 0x1856
+#define mmDPG0_DPG_DIMENSIONS_BASE_IDX 2
+#define mmDPG0_DPG_COLOUR_R_CR 0x1857
+#define mmDPG0_DPG_COLOUR_R_CR_BASE_IDX 2
+#define mmDPG0_DPG_COLOUR_G_Y 0x1858
+#define mmDPG0_DPG_COLOUR_G_Y_BASE_IDX 2
+#define mmDPG0_DPG_COLOUR_B_CB 0x1859
+#define mmDPG0_DPG_COLOUR_B_CB_BASE_IDX 2
+#define mmDPG0_DPG_OFFSET_SEGMENT 0x185a
+#define mmDPG0_DPG_OFFSET_SEGMENT_BASE_IDX 2
+#define mmDPG0_DPG_STATUS 0x185b
+#define mmDPG0_DPG_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_oppbuf0_dispdec
+// base address: 0x0
+#define mmOPPBUF0_OPPBUF_CONTROL 0x1884
+#define mmOPPBUF0_OPPBUF_CONTROL_BASE_IDX 2
+#define mmOPPBUF0_OPPBUF_3D_PARAMETERS_0 0x1885
+#define mmOPPBUF0_OPPBUF_3D_PARAMETERS_0_BASE_IDX 2
+#define mmOPPBUF0_OPPBUF_3D_PARAMETERS_1 0x1886
+#define mmOPPBUF0_OPPBUF_3D_PARAMETERS_1_BASE_IDX 2
+#define mmOPPBUF0_OPPBUF_CONTROL1 0x1889
+#define mmOPPBUF0_OPPBUF_CONTROL1_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_opp_pipe0_dispdec
+// base address: 0x0
+#define mmOPP_PIPE0_OPP_PIPE_CONTROL 0x188c
+#define mmOPP_PIPE0_OPP_PIPE_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_opp_pipe_crc0_dispdec
+// base address: 0x0
+#define mmOPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL 0x1891
+#define mmOPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL_BASE_IDX 2
+#define mmOPP_PIPE_CRC0_OPP_PIPE_CRC_MASK 0x1892
+#define mmOPP_PIPE_CRC0_OPP_PIPE_CRC_MASK_BASE_IDX 2
+#define mmOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULT0 0x1893
+#define mmOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULT0_BASE_IDX 2
+#define mmOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULT1 0x1894
+#define mmOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULT1_BASE_IDX 2
+#define mmOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULT2 0x1895
+#define mmOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULT2_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_fmt1_dispdec
+// base address: 0x168
+#define mmFMT1_FMT_CLAMP_COMPONENT_R 0x1896
+#define mmFMT1_FMT_CLAMP_COMPONENT_R_BASE_IDX 2
+#define mmFMT1_FMT_CLAMP_COMPONENT_G 0x1897
+#define mmFMT1_FMT_CLAMP_COMPONENT_G_BASE_IDX 2
+#define mmFMT1_FMT_CLAMP_COMPONENT_B 0x1898
+#define mmFMT1_FMT_CLAMP_COMPONENT_B_BASE_IDX 2
+#define mmFMT1_FMT_DYNAMIC_EXP_CNTL 0x1899
+#define mmFMT1_FMT_DYNAMIC_EXP_CNTL_BASE_IDX 2
+#define mmFMT1_FMT_CONTROL 0x189a
+#define mmFMT1_FMT_CONTROL_BASE_IDX 2
+#define mmFMT1_FMT_BIT_DEPTH_CONTROL 0x189b
+#define mmFMT1_FMT_BIT_DEPTH_CONTROL_BASE_IDX 2
+#define mmFMT1_FMT_DITHER_RAND_R_SEED 0x189c
+#define mmFMT1_FMT_DITHER_RAND_R_SEED_BASE_IDX 2
+#define mmFMT1_FMT_DITHER_RAND_G_SEED 0x189d
+#define mmFMT1_FMT_DITHER_RAND_G_SEED_BASE_IDX 2
+#define mmFMT1_FMT_DITHER_RAND_B_SEED 0x189e
+#define mmFMT1_FMT_DITHER_RAND_B_SEED_BASE_IDX 2
+#define mmFMT1_FMT_CLAMP_CNTL 0x189f
+#define mmFMT1_FMT_CLAMP_CNTL_BASE_IDX 2
+#define mmFMT1_FMT_SIDE_BY_SIDE_STEREO_CONTROL 0x18a0
+#define mmFMT1_FMT_SIDE_BY_SIDE_STEREO_CONTROL_BASE_IDX 2
+#define mmFMT1_FMT_MAP420_MEMORY_CONTROL 0x18a1
+#define mmFMT1_FMT_MAP420_MEMORY_CONTROL_BASE_IDX 2
+#define mmFMT1_FMT_422_CONTROL 0x18a3
+#define mmFMT1_FMT_422_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_dpg1_dispdec
+// base address: 0x168
+#define mmDPG1_DPG_CONTROL 0x18ae
+#define mmDPG1_DPG_CONTROL_BASE_IDX 2
+#define mmDPG1_DPG_RAMP_CONTROL 0x18af
+#define mmDPG1_DPG_RAMP_CONTROL_BASE_IDX 2
+#define mmDPG1_DPG_DIMENSIONS 0x18b0
+#define mmDPG1_DPG_DIMENSIONS_BASE_IDX 2
+#define mmDPG1_DPG_COLOUR_R_CR 0x18b1
+#define mmDPG1_DPG_COLOUR_R_CR_BASE_IDX 2
+#define mmDPG1_DPG_COLOUR_G_Y 0x18b2
+#define mmDPG1_DPG_COLOUR_G_Y_BASE_IDX 2
+#define mmDPG1_DPG_COLOUR_B_CB 0x18b3
+#define mmDPG1_DPG_COLOUR_B_CB_BASE_IDX 2
+#define mmDPG1_DPG_OFFSET_SEGMENT 0x18b4
+#define mmDPG1_DPG_OFFSET_SEGMENT_BASE_IDX 2
+#define mmDPG1_DPG_STATUS 0x18b5
+#define mmDPG1_DPG_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_oppbuf1_dispdec
+// base address: 0x168
+#define mmOPPBUF1_OPPBUF_CONTROL 0x18de
+#define mmOPPBUF1_OPPBUF_CONTROL_BASE_IDX 2
+#define mmOPPBUF1_OPPBUF_3D_PARAMETERS_0 0x18df
+#define mmOPPBUF1_OPPBUF_3D_PARAMETERS_0_BASE_IDX 2
+#define mmOPPBUF1_OPPBUF_3D_PARAMETERS_1 0x18e0
+#define mmOPPBUF1_OPPBUF_3D_PARAMETERS_1_BASE_IDX 2
+#define mmOPPBUF1_OPPBUF_CONTROL1 0x18e3
+#define mmOPPBUF1_OPPBUF_CONTROL1_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_opp_pipe1_dispdec
+// base address: 0x168
+#define mmOPP_PIPE1_OPP_PIPE_CONTROL 0x18e6
+#define mmOPP_PIPE1_OPP_PIPE_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_opp_pipe_crc1_dispdec
+// base address: 0x168
+#define mmOPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL 0x18eb
+#define mmOPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL_BASE_IDX 2
+#define mmOPP_PIPE_CRC1_OPP_PIPE_CRC_MASK 0x18ec
+#define mmOPP_PIPE_CRC1_OPP_PIPE_CRC_MASK_BASE_IDX 2
+#define mmOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULT0 0x18ed
+#define mmOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULT0_BASE_IDX 2
+#define mmOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULT1 0x18ee
+#define mmOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULT1_BASE_IDX 2
+#define mmOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULT2 0x18ef
+#define mmOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULT2_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_fmt2_dispdec
+// base address: 0x2d0
+#define mmFMT2_FMT_CLAMP_COMPONENT_R 0x18f0
+#define mmFMT2_FMT_CLAMP_COMPONENT_R_BASE_IDX 2
+#define mmFMT2_FMT_CLAMP_COMPONENT_G 0x18f1
+#define mmFMT2_FMT_CLAMP_COMPONENT_G_BASE_IDX 2
+#define mmFMT2_FMT_CLAMP_COMPONENT_B 0x18f2
+#define mmFMT2_FMT_CLAMP_COMPONENT_B_BASE_IDX 2
+#define mmFMT2_FMT_DYNAMIC_EXP_CNTL 0x18f3
+#define mmFMT2_FMT_DYNAMIC_EXP_CNTL_BASE_IDX 2
+#define mmFMT2_FMT_CONTROL 0x18f4
+#define mmFMT2_FMT_CONTROL_BASE_IDX 2
+#define mmFMT2_FMT_BIT_DEPTH_CONTROL 0x18f5
+#define mmFMT2_FMT_BIT_DEPTH_CONTROL_BASE_IDX 2
+#define mmFMT2_FMT_DITHER_RAND_R_SEED 0x18f6
+#define mmFMT2_FMT_DITHER_RAND_R_SEED_BASE_IDX 2
+#define mmFMT2_FMT_DITHER_RAND_G_SEED 0x18f7
+#define mmFMT2_FMT_DITHER_RAND_G_SEED_BASE_IDX 2
+#define mmFMT2_FMT_DITHER_RAND_B_SEED 0x18f8
+#define mmFMT2_FMT_DITHER_RAND_B_SEED_BASE_IDX 2
+#define mmFMT2_FMT_CLAMP_CNTL 0x18f9
+#define mmFMT2_FMT_CLAMP_CNTL_BASE_IDX 2
+#define mmFMT2_FMT_SIDE_BY_SIDE_STEREO_CONTROL 0x18fa
+#define mmFMT2_FMT_SIDE_BY_SIDE_STEREO_CONTROL_BASE_IDX 2
+#define mmFMT2_FMT_MAP420_MEMORY_CONTROL 0x18fb
+#define mmFMT2_FMT_MAP420_MEMORY_CONTROL_BASE_IDX 2
+#define mmFMT2_FMT_422_CONTROL 0x18fd
+#define mmFMT2_FMT_422_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_dpg2_dispdec
+// base address: 0x2d0
+#define mmDPG2_DPG_CONTROL 0x1908
+#define mmDPG2_DPG_CONTROL_BASE_IDX 2
+#define mmDPG2_DPG_RAMP_CONTROL 0x1909
+#define mmDPG2_DPG_RAMP_CONTROL_BASE_IDX 2
+#define mmDPG2_DPG_DIMENSIONS 0x190a
+#define mmDPG2_DPG_DIMENSIONS_BASE_IDX 2
+#define mmDPG2_DPG_COLOUR_R_CR 0x190b
+#define mmDPG2_DPG_COLOUR_R_CR_BASE_IDX 2
+#define mmDPG2_DPG_COLOUR_G_Y 0x190c
+#define mmDPG2_DPG_COLOUR_G_Y_BASE_IDX 2
+#define mmDPG2_DPG_COLOUR_B_CB 0x190d
+#define mmDPG2_DPG_COLOUR_B_CB_BASE_IDX 2
+#define mmDPG2_DPG_OFFSET_SEGMENT 0x190e
+#define mmDPG2_DPG_OFFSET_SEGMENT_BASE_IDX 2
+#define mmDPG2_DPG_STATUS 0x190f
+#define mmDPG2_DPG_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_oppbuf2_dispdec
+// base address: 0x2d0
+#define mmOPPBUF2_OPPBUF_CONTROL 0x1938
+#define mmOPPBUF2_OPPBUF_CONTROL_BASE_IDX 2
+#define mmOPPBUF2_OPPBUF_3D_PARAMETERS_0 0x1939
+#define mmOPPBUF2_OPPBUF_3D_PARAMETERS_0_BASE_IDX 2
+#define mmOPPBUF2_OPPBUF_3D_PARAMETERS_1 0x193a
+#define mmOPPBUF2_OPPBUF_3D_PARAMETERS_1_BASE_IDX 2
+#define mmOPPBUF2_OPPBUF_CONTROL1 0x193d
+#define mmOPPBUF2_OPPBUF_CONTROL1_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_opp_pipe2_dispdec
+// base address: 0x2d0
+#define mmOPP_PIPE2_OPP_PIPE_CONTROL 0x1940
+#define mmOPP_PIPE2_OPP_PIPE_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_opp_pipe_crc2_dispdec
+// base address: 0x2d0
+#define mmOPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL 0x1945
+#define mmOPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL_BASE_IDX 2
+#define mmOPP_PIPE_CRC2_OPP_PIPE_CRC_MASK 0x1946
+#define mmOPP_PIPE_CRC2_OPP_PIPE_CRC_MASK_BASE_IDX 2
+#define mmOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULT0 0x1947
+#define mmOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULT0_BASE_IDX 2
+#define mmOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULT1 0x1948
+#define mmOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULT1_BASE_IDX 2
+#define mmOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULT2 0x1949
+#define mmOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULT2_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_fmt3_dispdec
+// base address: 0x438
+#define mmFMT3_FMT_CLAMP_COMPONENT_R 0x194a
+#define mmFMT3_FMT_CLAMP_COMPONENT_R_BASE_IDX 2
+#define mmFMT3_FMT_CLAMP_COMPONENT_G 0x194b
+#define mmFMT3_FMT_CLAMP_COMPONENT_G_BASE_IDX 2
+#define mmFMT3_FMT_CLAMP_COMPONENT_B 0x194c
+#define mmFMT3_FMT_CLAMP_COMPONENT_B_BASE_IDX 2
+#define mmFMT3_FMT_DYNAMIC_EXP_CNTL 0x194d
+#define mmFMT3_FMT_DYNAMIC_EXP_CNTL_BASE_IDX 2
+#define mmFMT3_FMT_CONTROL 0x194e
+#define mmFMT3_FMT_CONTROL_BASE_IDX 2
+#define mmFMT3_FMT_BIT_DEPTH_CONTROL 0x194f
+#define mmFMT3_FMT_BIT_DEPTH_CONTROL_BASE_IDX 2
+#define mmFMT3_FMT_DITHER_RAND_R_SEED 0x1950
+#define mmFMT3_FMT_DITHER_RAND_R_SEED_BASE_IDX 2
+#define mmFMT3_FMT_DITHER_RAND_G_SEED 0x1951
+#define mmFMT3_FMT_DITHER_RAND_G_SEED_BASE_IDX 2
+#define mmFMT3_FMT_DITHER_RAND_B_SEED 0x1952
+#define mmFMT3_FMT_DITHER_RAND_B_SEED_BASE_IDX 2
+#define mmFMT3_FMT_CLAMP_CNTL 0x1953
+#define mmFMT3_FMT_CLAMP_CNTL_BASE_IDX 2
+#define mmFMT3_FMT_SIDE_BY_SIDE_STEREO_CONTROL 0x1954
+#define mmFMT3_FMT_SIDE_BY_SIDE_STEREO_CONTROL_BASE_IDX 2
+#define mmFMT3_FMT_MAP420_MEMORY_CONTROL 0x1955
+#define mmFMT3_FMT_MAP420_MEMORY_CONTROL_BASE_IDX 2
+#define mmFMT3_FMT_422_CONTROL 0x1957
+#define mmFMT3_FMT_422_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_dpg3_dispdec
+// base address: 0x438
+#define mmDPG3_DPG_CONTROL 0x1962
+#define mmDPG3_DPG_CONTROL_BASE_IDX 2
+#define mmDPG3_DPG_RAMP_CONTROL 0x1963
+#define mmDPG3_DPG_RAMP_CONTROL_BASE_IDX 2
+#define mmDPG3_DPG_DIMENSIONS 0x1964
+#define mmDPG3_DPG_DIMENSIONS_BASE_IDX 2
+#define mmDPG3_DPG_COLOUR_R_CR 0x1965
+#define mmDPG3_DPG_COLOUR_R_CR_BASE_IDX 2
+#define mmDPG3_DPG_COLOUR_G_Y 0x1966
+#define mmDPG3_DPG_COLOUR_G_Y_BASE_IDX 2
+#define mmDPG3_DPG_COLOUR_B_CB 0x1967
+#define mmDPG3_DPG_COLOUR_B_CB_BASE_IDX 2
+#define mmDPG3_DPG_OFFSET_SEGMENT 0x1968
+#define mmDPG3_DPG_OFFSET_SEGMENT_BASE_IDX 2
+#define mmDPG3_DPG_STATUS 0x1969
+#define mmDPG3_DPG_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_oppbuf3_dispdec
+// base address: 0x438
+#define mmOPPBUF3_OPPBUF_CONTROL 0x1992
+#define mmOPPBUF3_OPPBUF_CONTROL_BASE_IDX 2
+#define mmOPPBUF3_OPPBUF_3D_PARAMETERS_0 0x1993
+#define mmOPPBUF3_OPPBUF_3D_PARAMETERS_0_BASE_IDX 2
+#define mmOPPBUF3_OPPBUF_3D_PARAMETERS_1 0x1994
+#define mmOPPBUF3_OPPBUF_3D_PARAMETERS_1_BASE_IDX 2
+#define mmOPPBUF3_OPPBUF_CONTROL1 0x1997
+#define mmOPPBUF3_OPPBUF_CONTROL1_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_opp_pipe3_dispdec
+// base address: 0x438
+#define mmOPP_PIPE3_OPP_PIPE_CONTROL 0x199a
+#define mmOPP_PIPE3_OPP_PIPE_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_opp_pipe_crc3_dispdec
+// base address: 0x438
+#define mmOPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL 0x199f
+#define mmOPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL_BASE_IDX 2
+#define mmOPP_PIPE_CRC3_OPP_PIPE_CRC_MASK 0x19a0
+#define mmOPP_PIPE_CRC3_OPP_PIPE_CRC_MASK_BASE_IDX 2
+#define mmOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULT0 0x19a1
+#define mmOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULT0_BASE_IDX 2
+#define mmOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULT1 0x19a2
+#define mmOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULT1_BASE_IDX 2
+#define mmOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULT2 0x19a3
+#define mmOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULT2_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_fmt4_dispdec
+// base address: 0x5a0
+#define mmFMT4_FMT_CLAMP_COMPONENT_R 0x19a4
+#define mmFMT4_FMT_CLAMP_COMPONENT_R_BASE_IDX 2
+#define mmFMT4_FMT_CLAMP_COMPONENT_G 0x19a5
+#define mmFMT4_FMT_CLAMP_COMPONENT_G_BASE_IDX 2
+#define mmFMT4_FMT_CLAMP_COMPONENT_B 0x19a6
+#define mmFMT4_FMT_CLAMP_COMPONENT_B_BASE_IDX 2
+#define mmFMT4_FMT_DYNAMIC_EXP_CNTL 0x19a7
+#define mmFMT4_FMT_DYNAMIC_EXP_CNTL_BASE_IDX 2
+#define mmFMT4_FMT_CONTROL 0x19a8
+#define mmFMT4_FMT_CONTROL_BASE_IDX 2
+#define mmFMT4_FMT_BIT_DEPTH_CONTROL 0x19a9
+#define mmFMT4_FMT_BIT_DEPTH_CONTROL_BASE_IDX 2
+#define mmFMT4_FMT_DITHER_RAND_R_SEED 0x19aa
+#define mmFMT4_FMT_DITHER_RAND_R_SEED_BASE_IDX 2
+#define mmFMT4_FMT_DITHER_RAND_G_SEED 0x19ab
+#define mmFMT4_FMT_DITHER_RAND_G_SEED_BASE_IDX 2
+#define mmFMT4_FMT_DITHER_RAND_B_SEED 0x19ac
+#define mmFMT4_FMT_DITHER_RAND_B_SEED_BASE_IDX 2
+#define mmFMT4_FMT_CLAMP_CNTL 0x19ad
+#define mmFMT4_FMT_CLAMP_CNTL_BASE_IDX 2
+#define mmFMT4_FMT_SIDE_BY_SIDE_STEREO_CONTROL 0x19ae
+#define mmFMT4_FMT_SIDE_BY_SIDE_STEREO_CONTROL_BASE_IDX 2
+#define mmFMT4_FMT_MAP420_MEMORY_CONTROL 0x19af
+#define mmFMT4_FMT_MAP420_MEMORY_CONTROL_BASE_IDX 2
+#define mmFMT4_FMT_422_CONTROL 0x19b1
+#define mmFMT4_FMT_422_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_dpg4_dispdec
+// base address: 0x5a0
+#define mmDPG4_DPG_CONTROL 0x19bc
+#define mmDPG4_DPG_CONTROL_BASE_IDX 2
+#define mmDPG4_DPG_RAMP_CONTROL 0x19bd
+#define mmDPG4_DPG_RAMP_CONTROL_BASE_IDX 2
+#define mmDPG4_DPG_DIMENSIONS 0x19be
+#define mmDPG4_DPG_DIMENSIONS_BASE_IDX 2
+#define mmDPG4_DPG_COLOUR_R_CR 0x19bf
+#define mmDPG4_DPG_COLOUR_R_CR_BASE_IDX 2
+#define mmDPG4_DPG_COLOUR_G_Y 0x19c0
+#define mmDPG4_DPG_COLOUR_G_Y_BASE_IDX 2
+#define mmDPG4_DPG_COLOUR_B_CB 0x19c1
+#define mmDPG4_DPG_COLOUR_B_CB_BASE_IDX 2
+#define mmDPG4_DPG_OFFSET_SEGMENT 0x19c2
+#define mmDPG4_DPG_OFFSET_SEGMENT_BASE_IDX 2
+#define mmDPG4_DPG_STATUS 0x19c3
+#define mmDPG4_DPG_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_oppbuf4_dispdec
+// base address: 0x5a0
+#define mmOPPBUF4_OPPBUF_CONTROL 0x19ec
+#define mmOPPBUF4_OPPBUF_CONTROL_BASE_IDX 2
+#define mmOPPBUF4_OPPBUF_3D_PARAMETERS_0 0x19ed
+#define mmOPPBUF4_OPPBUF_3D_PARAMETERS_0_BASE_IDX 2
+#define mmOPPBUF4_OPPBUF_3D_PARAMETERS_1 0x19ee
+#define mmOPPBUF4_OPPBUF_3D_PARAMETERS_1_BASE_IDX 2
+#define mmOPPBUF4_OPPBUF_CONTROL1 0x19f1
+#define mmOPPBUF4_OPPBUF_CONTROL1_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_opp_pipe4_dispdec
+// base address: 0x5a0
+#define mmOPP_PIPE4_OPP_PIPE_CONTROL 0x19f4
+#define mmOPP_PIPE4_OPP_PIPE_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_opp_pipe_crc4_dispdec
+// base address: 0x5a0
+#define mmOPP_PIPE_CRC4_OPP_PIPE_CRC_CONTROL 0x19f9
+#define mmOPP_PIPE_CRC4_OPP_PIPE_CRC_CONTROL_BASE_IDX 2
+#define mmOPP_PIPE_CRC4_OPP_PIPE_CRC_MASK 0x19fa
+#define mmOPP_PIPE_CRC4_OPP_PIPE_CRC_MASK_BASE_IDX 2
+#define mmOPP_PIPE_CRC4_OPP_PIPE_CRC_RESULT0 0x19fb
+#define mmOPP_PIPE_CRC4_OPP_PIPE_CRC_RESULT0_BASE_IDX 2
+#define mmOPP_PIPE_CRC4_OPP_PIPE_CRC_RESULT1 0x19fc
+#define mmOPP_PIPE_CRC4_OPP_PIPE_CRC_RESULT1_BASE_IDX 2
+#define mmOPP_PIPE_CRC4_OPP_PIPE_CRC_RESULT2 0x19fd
+#define mmOPP_PIPE_CRC4_OPP_PIPE_CRC_RESULT2_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_fmt5_dispdec
+// base address: 0x708
+#define mmFMT5_FMT_CLAMP_COMPONENT_R 0x19fe
+#define mmFMT5_FMT_CLAMP_COMPONENT_R_BASE_IDX 2
+#define mmFMT5_FMT_CLAMP_COMPONENT_G 0x19ff
+#define mmFMT5_FMT_CLAMP_COMPONENT_G_BASE_IDX 2
+#define mmFMT5_FMT_CLAMP_COMPONENT_B 0x1a00
+#define mmFMT5_FMT_CLAMP_COMPONENT_B_BASE_IDX 2
+#define mmFMT5_FMT_DYNAMIC_EXP_CNTL 0x1a01
+#define mmFMT5_FMT_DYNAMIC_EXP_CNTL_BASE_IDX 2
+#define mmFMT5_FMT_CONTROL 0x1a02
+#define mmFMT5_FMT_CONTROL_BASE_IDX 2
+#define mmFMT5_FMT_BIT_DEPTH_CONTROL 0x1a03
+#define mmFMT5_FMT_BIT_DEPTH_CONTROL_BASE_IDX 2
+#define mmFMT5_FMT_DITHER_RAND_R_SEED 0x1a04
+#define mmFMT5_FMT_DITHER_RAND_R_SEED_BASE_IDX 2
+#define mmFMT5_FMT_DITHER_RAND_G_SEED 0x1a05
+#define mmFMT5_FMT_DITHER_RAND_G_SEED_BASE_IDX 2
+#define mmFMT5_FMT_DITHER_RAND_B_SEED 0x1a06
+#define mmFMT5_FMT_DITHER_RAND_B_SEED_BASE_IDX 2
+#define mmFMT5_FMT_CLAMP_CNTL 0x1a07
+#define mmFMT5_FMT_CLAMP_CNTL_BASE_IDX 2
+#define mmFMT5_FMT_SIDE_BY_SIDE_STEREO_CONTROL 0x1a08
+#define mmFMT5_FMT_SIDE_BY_SIDE_STEREO_CONTROL_BASE_IDX 2
+#define mmFMT5_FMT_MAP420_MEMORY_CONTROL 0x1a09
+#define mmFMT5_FMT_MAP420_MEMORY_CONTROL_BASE_IDX 2
+#define mmFMT5_FMT_422_CONTROL 0x1a0b
+#define mmFMT5_FMT_422_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_dpg5_dispdec
+// base address: 0x708
+#define mmDPG5_DPG_CONTROL 0x1a16
+#define mmDPG5_DPG_CONTROL_BASE_IDX 2
+#define mmDPG5_DPG_RAMP_CONTROL 0x1a17
+#define mmDPG5_DPG_RAMP_CONTROL_BASE_IDX 2
+#define mmDPG5_DPG_DIMENSIONS 0x1a18
+#define mmDPG5_DPG_DIMENSIONS_BASE_IDX 2
+#define mmDPG5_DPG_COLOUR_R_CR 0x1a19
+#define mmDPG5_DPG_COLOUR_R_CR_BASE_IDX 2
+#define mmDPG5_DPG_COLOUR_G_Y 0x1a1a
+#define mmDPG5_DPG_COLOUR_G_Y_BASE_IDX 2
+#define mmDPG5_DPG_COLOUR_B_CB 0x1a1b
+#define mmDPG5_DPG_COLOUR_B_CB_BASE_IDX 2
+#define mmDPG5_DPG_OFFSET_SEGMENT 0x1a1c
+#define mmDPG5_DPG_OFFSET_SEGMENT_BASE_IDX 2
+#define mmDPG5_DPG_STATUS 0x1a1d
+#define mmDPG5_DPG_STATUS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_oppbuf5_dispdec
+// base address: 0x708
+#define mmOPPBUF5_OPPBUF_CONTROL 0x1a46
+#define mmOPPBUF5_OPPBUF_CONTROL_BASE_IDX 2
+#define mmOPPBUF5_OPPBUF_3D_PARAMETERS_0 0x1a47
+#define mmOPPBUF5_OPPBUF_3D_PARAMETERS_0_BASE_IDX 2
+#define mmOPPBUF5_OPPBUF_3D_PARAMETERS_1 0x1a48
+#define mmOPPBUF5_OPPBUF_3D_PARAMETERS_1_BASE_IDX 2
+#define mmOPPBUF5_OPPBUF_CONTROL1 0x1a4b
+#define mmOPPBUF5_OPPBUF_CONTROL1_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_opp_pipe5_dispdec
+// base address: 0x708
+#define mmOPP_PIPE5_OPP_PIPE_CONTROL 0x1a4e
+#define mmOPP_PIPE5_OPP_PIPE_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_opp_pipe_crc5_dispdec
+// base address: 0x708
+#define mmOPP_PIPE_CRC5_OPP_PIPE_CRC_CONTROL 0x1a53
+#define mmOPP_PIPE_CRC5_OPP_PIPE_CRC_CONTROL_BASE_IDX 2
+#define mmOPP_PIPE_CRC5_OPP_PIPE_CRC_MASK 0x1a54
+#define mmOPP_PIPE_CRC5_OPP_PIPE_CRC_MASK_BASE_IDX 2
+#define mmOPP_PIPE_CRC5_OPP_PIPE_CRC_RESULT0 0x1a55
+#define mmOPP_PIPE_CRC5_OPP_PIPE_CRC_RESULT0_BASE_IDX 2
+#define mmOPP_PIPE_CRC5_OPP_PIPE_CRC_RESULT1 0x1a56
+#define mmOPP_PIPE_CRC5_OPP_PIPE_CRC_RESULT1_BASE_IDX 2
+#define mmOPP_PIPE_CRC5_OPP_PIPE_CRC_RESULT2 0x1a57
+#define mmOPP_PIPE_CRC5_OPP_PIPE_CRC_RESULT2_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_opp_top_dispdec
+// base address: 0x0
+#define mmOPP_TOP_CLK_CONTROL 0x1a5e
+#define mmOPP_TOP_CLK_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_dscrm0_dispdec
+// base address: 0x0
+#define mmDSCRM0_DSCRM_DSC_FORWARD_CONFIG 0x1a64
+#define mmDSCRM0_DSCRM_DSC_FORWARD_CONFIG_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_dscrm1_dispdec
+// base address: 0x4
+#define mmDSCRM1_DSCRM_DSC_FORWARD_CONFIG 0x1a65
+#define mmDSCRM1_DSCRM_DSC_FORWARD_CONFIG_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_dscrm2_dispdec
+// base address: 0x8
+#define mmDSCRM2_DSCRM_DSC_FORWARD_CONFIG 0x1a66
+#define mmDSCRM2_DSCRM_DSC_FORWARD_CONFIG_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_dscrm3_dispdec
+// base address: 0xc
+#define mmDSCRM3_DSCRM_DSC_FORWARD_CONFIG 0x1a67
+#define mmDSCRM3_DSCRM_DSC_FORWARD_CONFIG_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_dscrm4_dispdec
+// base address: 0x10
+#define mmDSCRM4_DSCRM_DSC_FORWARD_CONFIG 0x1a68
+#define mmDSCRM4_DSCRM_DSC_FORWARD_CONFIG_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_dscrm5_dispdec
+// base address: 0x14
+#define mmDSCRM5_DSCRM_DSC_FORWARD_CONFIG 0x1a69
+#define mmDSCRM5_DSCRM_DSC_FORWARD_CONFIG_BASE_IDX 2
+
+
+// addressBlock: dce_dc_opp_opp_dcperfmon_dc_perfmon_dispdec
+// base address: 0x6af8
+#define mmDC_PERFMON18_PERFCOUNTER_CNTL 0x1abe
+#define mmDC_PERFMON18_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON18_PERFCOUNTER_CNTL2 0x1abf
+#define mmDC_PERFMON18_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON18_PERFCOUNTER_STATE 0x1ac0
+#define mmDC_PERFMON18_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON18_PERFMON_CNTL 0x1ac1
+#define mmDC_PERFMON18_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON18_PERFMON_CNTL2 0x1ac2
+#define mmDC_PERFMON18_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON18_PERFMON_CVALUE_INT_MISC 0x1ac3
+#define mmDC_PERFMON18_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON18_PERFMON_CVALUE_LOW 0x1ac4
+#define mmDC_PERFMON18_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON18_PERFMON_HI 0x1ac5
+#define mmDC_PERFMON18_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON18_PERFMON_LOW 0x1ac6
+#define mmDC_PERFMON18_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_optc_odm0_dispdec
+// base address: 0x0
+#define mmODM0_OPTC_INPUT_GLOBAL_CONTROL 0x1aca
+#define mmODM0_OPTC_INPUT_GLOBAL_CONTROL_BASE_IDX 2
+#define mmODM0_OPTC_DATA_SOURCE_SELECT 0x1acb
+#define mmODM0_OPTC_DATA_SOURCE_SELECT_BASE_IDX 2
+#define mmODM0_OPTC_DATA_FORMAT_CONTROL 0x1acc
+#define mmODM0_OPTC_DATA_FORMAT_CONTROL_BASE_IDX 2
+#define mmODM0_OPTC_BYTES_PER_PIXEL 0x1acd
+#define mmODM0_OPTC_BYTES_PER_PIXEL_BASE_IDX 2
+#define mmODM0_OPTC_WIDTH_CONTROL 0x1ace
+#define mmODM0_OPTC_WIDTH_CONTROL_BASE_IDX 2
+#define mmODM0_OPTC_INPUT_CLOCK_CONTROL 0x1acf
+#define mmODM0_OPTC_INPUT_CLOCK_CONTROL_BASE_IDX 2
+#define mmODM0_OPTC_MEMORY_CONFIG 0x1ad0
+#define mmODM0_OPTC_MEMORY_CONFIG_BASE_IDX 2
+#define mmODM0_OPTC_INPUT_SPARE_REGISTER 0x1ad1
+#define mmODM0_OPTC_INPUT_SPARE_REGISTER_BASE_IDX 2
+
+
+// addressBlock: dce_dc_optc_odm1_dispdec
+// base address: 0x40
+#define mmODM1_OPTC_INPUT_GLOBAL_CONTROL 0x1ada
+#define mmODM1_OPTC_INPUT_GLOBAL_CONTROL_BASE_IDX 2
+#define mmODM1_OPTC_DATA_SOURCE_SELECT 0x1adb
+#define mmODM1_OPTC_DATA_SOURCE_SELECT_BASE_IDX 2
+#define mmODM1_OPTC_DATA_FORMAT_CONTROL 0x1adc
+#define mmODM1_OPTC_DATA_FORMAT_CONTROL_BASE_IDX 2
+#define mmODM1_OPTC_BYTES_PER_PIXEL 0x1add
+#define mmODM1_OPTC_BYTES_PER_PIXEL_BASE_IDX 2
+#define mmODM1_OPTC_WIDTH_CONTROL 0x1ade
+#define mmODM1_OPTC_WIDTH_CONTROL_BASE_IDX 2
+#define mmODM1_OPTC_INPUT_CLOCK_CONTROL 0x1adf
+#define mmODM1_OPTC_INPUT_CLOCK_CONTROL_BASE_IDX 2
+#define mmODM1_OPTC_MEMORY_CONFIG 0x1ae0
+#define mmODM1_OPTC_MEMORY_CONFIG_BASE_IDX 2
+#define mmODM1_OPTC_INPUT_SPARE_REGISTER 0x1ae1
+#define mmODM1_OPTC_INPUT_SPARE_REGISTER_BASE_IDX 2
+
+
+// addressBlock: dce_dc_optc_odm2_dispdec
+// base address: 0x80
+#define mmODM2_OPTC_INPUT_GLOBAL_CONTROL 0x1aea
+#define mmODM2_OPTC_INPUT_GLOBAL_CONTROL_BASE_IDX 2
+#define mmODM2_OPTC_DATA_SOURCE_SELECT 0x1aeb
+#define mmODM2_OPTC_DATA_SOURCE_SELECT_BASE_IDX 2
+#define mmODM2_OPTC_DATA_FORMAT_CONTROL 0x1aec
+#define mmODM2_OPTC_DATA_FORMAT_CONTROL_BASE_IDX 2
+#define mmODM2_OPTC_BYTES_PER_PIXEL 0x1aed
+#define mmODM2_OPTC_BYTES_PER_PIXEL_BASE_IDX 2
+#define mmODM2_OPTC_WIDTH_CONTROL 0x1aee
+#define mmODM2_OPTC_WIDTH_CONTROL_BASE_IDX 2
+#define mmODM2_OPTC_INPUT_CLOCK_CONTROL 0x1aef
+#define mmODM2_OPTC_INPUT_CLOCK_CONTROL_BASE_IDX 2
+#define mmODM2_OPTC_MEMORY_CONFIG 0x1af0
+#define mmODM2_OPTC_MEMORY_CONFIG_BASE_IDX 2
+#define mmODM2_OPTC_INPUT_SPARE_REGISTER 0x1af1
+#define mmODM2_OPTC_INPUT_SPARE_REGISTER_BASE_IDX 2
+
+
+// addressBlock: dce_dc_optc_odm3_dispdec
+// base address: 0xc0
+#define mmODM3_OPTC_INPUT_GLOBAL_CONTROL 0x1afa
+#define mmODM3_OPTC_INPUT_GLOBAL_CONTROL_BASE_IDX 2
+#define mmODM3_OPTC_DATA_SOURCE_SELECT 0x1afb
+#define mmODM3_OPTC_DATA_SOURCE_SELECT_BASE_IDX 2
+#define mmODM3_OPTC_DATA_FORMAT_CONTROL 0x1afc
+#define mmODM3_OPTC_DATA_FORMAT_CONTROL_BASE_IDX 2
+#define mmODM3_OPTC_BYTES_PER_PIXEL 0x1afd
+#define mmODM3_OPTC_BYTES_PER_PIXEL_BASE_IDX 2
+#define mmODM3_OPTC_WIDTH_CONTROL 0x1afe
+#define mmODM3_OPTC_WIDTH_CONTROL_BASE_IDX 2
+#define mmODM3_OPTC_INPUT_CLOCK_CONTROL 0x1aff
+#define mmODM3_OPTC_INPUT_CLOCK_CONTROL_BASE_IDX 2
+#define mmODM3_OPTC_MEMORY_CONFIG 0x1b00
+#define mmODM3_OPTC_MEMORY_CONFIG_BASE_IDX 2
+#define mmODM3_OPTC_INPUT_SPARE_REGISTER 0x1b01
+#define mmODM3_OPTC_INPUT_SPARE_REGISTER_BASE_IDX 2
+
+
+// addressBlock: dce_dc_optc_odm4_dispdec
+// base address: 0x100
+#define mmODM4_OPTC_INPUT_GLOBAL_CONTROL 0x1b0a
+#define mmODM4_OPTC_INPUT_GLOBAL_CONTROL_BASE_IDX 2
+#define mmODM4_OPTC_DATA_SOURCE_SELECT 0x1b0b
+#define mmODM4_OPTC_DATA_SOURCE_SELECT_BASE_IDX 2
+#define mmODM4_OPTC_DATA_FORMAT_CONTROL 0x1b0c
+#define mmODM4_OPTC_DATA_FORMAT_CONTROL_BASE_IDX 2
+#define mmODM4_OPTC_BYTES_PER_PIXEL 0x1b0d
+#define mmODM4_OPTC_BYTES_PER_PIXEL_BASE_IDX 2
+#define mmODM4_OPTC_WIDTH_CONTROL 0x1b0e
+#define mmODM4_OPTC_WIDTH_CONTROL_BASE_IDX 2
+#define mmODM4_OPTC_INPUT_CLOCK_CONTROL 0x1b0f
+#define mmODM4_OPTC_INPUT_CLOCK_CONTROL_BASE_IDX 2
+#define mmODM4_OPTC_MEMORY_CONFIG 0x1b10
+#define mmODM4_OPTC_MEMORY_CONFIG_BASE_IDX 2
+#define mmODM4_OPTC_INPUT_SPARE_REGISTER 0x1b11
+#define mmODM4_OPTC_INPUT_SPARE_REGISTER_BASE_IDX 2
+
+
+// addressBlock: dce_dc_optc_odm5_dispdec
+// base address: 0x140
+#define mmODM5_OPTC_INPUT_GLOBAL_CONTROL 0x1b1a
+#define mmODM5_OPTC_INPUT_GLOBAL_CONTROL_BASE_IDX 2
+#define mmODM5_OPTC_DATA_SOURCE_SELECT 0x1b1b
+#define mmODM5_OPTC_DATA_SOURCE_SELECT_BASE_IDX 2
+#define mmODM5_OPTC_DATA_FORMAT_CONTROL 0x1b1c
+#define mmODM5_OPTC_DATA_FORMAT_CONTROL_BASE_IDX 2
+#define mmODM5_OPTC_BYTES_PER_PIXEL 0x1b1d
+#define mmODM5_OPTC_BYTES_PER_PIXEL_BASE_IDX 2
+#define mmODM5_OPTC_WIDTH_CONTROL 0x1b1e
+#define mmODM5_OPTC_WIDTH_CONTROL_BASE_IDX 2
+#define mmODM5_OPTC_INPUT_CLOCK_CONTROL 0x1b1f
+#define mmODM5_OPTC_INPUT_CLOCK_CONTROL_BASE_IDX 2
+#define mmODM5_OPTC_MEMORY_CONFIG 0x1b20
+#define mmODM5_OPTC_MEMORY_CONFIG_BASE_IDX 2
+#define mmODM5_OPTC_INPUT_SPARE_REGISTER 0x1b21
+#define mmODM5_OPTC_INPUT_SPARE_REGISTER_BASE_IDX 2
+
+
+// addressBlock: dce_dc_optc_otg0_dispdec
+// base address: 0x0
+#define mmOTG0_OTG_H_TOTAL 0x1b2a
+#define mmOTG0_OTG_H_TOTAL_BASE_IDX 2
+#define mmOTG0_OTG_H_BLANK_START_END 0x1b2b
+#define mmOTG0_OTG_H_BLANK_START_END_BASE_IDX 2
+#define mmOTG0_OTG_H_SYNC_A 0x1b2c
+#define mmOTG0_OTG_H_SYNC_A_BASE_IDX 2
+#define mmOTG0_OTG_H_SYNC_A_CNTL 0x1b2d
+#define mmOTG0_OTG_H_SYNC_A_CNTL_BASE_IDX 2
+#define mmOTG0_OTG_H_TIMING_CNTL 0x1b2e
+#define mmOTG0_OTG_H_TIMING_CNTL_BASE_IDX 2
+#define mmOTG0_OTG_V_TOTAL 0x1b2f
+#define mmOTG0_OTG_V_TOTAL_BASE_IDX 2
+#define mmOTG0_OTG_V_TOTAL_MIN 0x1b30
+#define mmOTG0_OTG_V_TOTAL_MIN_BASE_IDX 2
+#define mmOTG0_OTG_V_TOTAL_MAX 0x1b31
+#define mmOTG0_OTG_V_TOTAL_MAX_BASE_IDX 2
+#define mmOTG0_OTG_V_TOTAL_MID 0x1b32
+#define mmOTG0_OTG_V_TOTAL_MID_BASE_IDX 2
+#define mmOTG0_OTG_V_TOTAL_CONTROL 0x1b33
+#define mmOTG0_OTG_V_TOTAL_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_V_TOTAL_INT_STATUS 0x1b34
+#define mmOTG0_OTG_V_TOTAL_INT_STATUS_BASE_IDX 2
+#define mmOTG0_OTG_VSYNC_NOM_INT_STATUS 0x1b35
+#define mmOTG0_OTG_VSYNC_NOM_INT_STATUS_BASE_IDX 2
+#define mmOTG0_OTG_V_BLANK_START_END 0x1b36
+#define mmOTG0_OTG_V_BLANK_START_END_BASE_IDX 2
+#define mmOTG0_OTG_V_SYNC_A 0x1b37
+#define mmOTG0_OTG_V_SYNC_A_BASE_IDX 2
+#define mmOTG0_OTG_V_SYNC_A_CNTL 0x1b38
+#define mmOTG0_OTG_V_SYNC_A_CNTL_BASE_IDX 2
+#define mmOTG0_OTG_TRIGA_CNTL 0x1b39
+#define mmOTG0_OTG_TRIGA_CNTL_BASE_IDX 2
+#define mmOTG0_OTG_TRIGA_MANUAL_TRIG 0x1b3a
+#define mmOTG0_OTG_TRIGA_MANUAL_TRIG_BASE_IDX 2
+#define mmOTG0_OTG_TRIGB_CNTL 0x1b3b
+#define mmOTG0_OTG_TRIGB_CNTL_BASE_IDX 2
+#define mmOTG0_OTG_TRIGB_MANUAL_TRIG 0x1b3c
+#define mmOTG0_OTG_TRIGB_MANUAL_TRIG_BASE_IDX 2
+#define mmOTG0_OTG_FORCE_COUNT_NOW_CNTL 0x1b3d
+#define mmOTG0_OTG_FORCE_COUNT_NOW_CNTL_BASE_IDX 2
+#define mmOTG0_OTG_FLOW_CONTROL 0x1b3e
+#define mmOTG0_OTG_FLOW_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_STEREO_FORCE_NEXT_EYE 0x1b3f
+#define mmOTG0_OTG_STEREO_FORCE_NEXT_EYE_BASE_IDX 2
+#define mmOTG0_OTG_CONTROL 0x1b41
+#define mmOTG0_OTG_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_BLANK_CONTROL 0x1b42
+#define mmOTG0_OTG_BLANK_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_PIPE_ABORT_CONTROL 0x1b43
+#define mmOTG0_OTG_PIPE_ABORT_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_INTERLACE_CONTROL 0x1b44
+#define mmOTG0_OTG_INTERLACE_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_INTERLACE_STATUS 0x1b45
+#define mmOTG0_OTG_INTERLACE_STATUS_BASE_IDX 2
+#define mmOTG0_OTG_PIXEL_DATA_READBACK0 0x1b47
+#define mmOTG0_OTG_PIXEL_DATA_READBACK0_BASE_IDX 2
+#define mmOTG0_OTG_PIXEL_DATA_READBACK1 0x1b48
+#define mmOTG0_OTG_PIXEL_DATA_READBACK1_BASE_IDX 2
+#define mmOTG0_OTG_STATUS 0x1b49
+#define mmOTG0_OTG_STATUS_BASE_IDX 2
+#define mmOTG0_OTG_STATUS_POSITION 0x1b4a
+#define mmOTG0_OTG_STATUS_POSITION_BASE_IDX 2
+#define mmOTG0_OTG_NOM_VERT_POSITION 0x1b4b
+#define mmOTG0_OTG_NOM_VERT_POSITION_BASE_IDX 2
+#define mmOTG0_OTG_STATUS_FRAME_COUNT 0x1b4c
+#define mmOTG0_OTG_STATUS_FRAME_COUNT_BASE_IDX 2
+#define mmOTG0_OTG_STATUS_VF_COUNT 0x1b4d
+#define mmOTG0_OTG_STATUS_VF_COUNT_BASE_IDX 2
+#define mmOTG0_OTG_STATUS_HV_COUNT 0x1b4e
+#define mmOTG0_OTG_STATUS_HV_COUNT_BASE_IDX 2
+#define mmOTG0_OTG_COUNT_CONTROL 0x1b4f
+#define mmOTG0_OTG_COUNT_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_COUNT_RESET 0x1b50
+#define mmOTG0_OTG_COUNT_RESET_BASE_IDX 2
+#define mmOTG0_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE 0x1b51
+#define mmOTG0_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_BASE_IDX 2
+#define mmOTG0_OTG_VERT_SYNC_CONTROL 0x1b52
+#define mmOTG0_OTG_VERT_SYNC_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_STEREO_STATUS 0x1b53
+#define mmOTG0_OTG_STEREO_STATUS_BASE_IDX 2
+#define mmOTG0_OTG_STEREO_CONTROL 0x1b54
+#define mmOTG0_OTG_STEREO_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_SNAPSHOT_STATUS 0x1b55
+#define mmOTG0_OTG_SNAPSHOT_STATUS_BASE_IDX 2
+#define mmOTG0_OTG_SNAPSHOT_CONTROL 0x1b56
+#define mmOTG0_OTG_SNAPSHOT_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_SNAPSHOT_POSITION 0x1b57
+#define mmOTG0_OTG_SNAPSHOT_POSITION_BASE_IDX 2
+#define mmOTG0_OTG_SNAPSHOT_FRAME 0x1b58
+#define mmOTG0_OTG_SNAPSHOT_FRAME_BASE_IDX 2
+#define mmOTG0_OTG_INTERRUPT_CONTROL 0x1b59
+#define mmOTG0_OTG_INTERRUPT_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_UPDATE_LOCK 0x1b5a
+#define mmOTG0_OTG_UPDATE_LOCK_BASE_IDX 2
+#define mmOTG0_OTG_DOUBLE_BUFFER_CONTROL 0x1b5b
+#define mmOTG0_OTG_DOUBLE_BUFFER_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_MASTER_EN 0x1b5c
+#define mmOTG0_OTG_MASTER_EN_BASE_IDX 2
+#define mmOTG0_OTG_BLANK_DATA_COLOR 0x1b5e
+#define mmOTG0_OTG_BLANK_DATA_COLOR_BASE_IDX 2
+#define mmOTG0_OTG_BLANK_DATA_COLOR_EXT 0x1b5f
+#define mmOTG0_OTG_BLANK_DATA_COLOR_EXT_BASE_IDX 2
+#define mmOTG0_OTG_BLACK_COLOR 0x1b60
+#define mmOTG0_OTG_BLACK_COLOR_BASE_IDX 2
+#define mmOTG0_OTG_BLACK_COLOR_EXT 0x1b61
+#define mmOTG0_OTG_BLACK_COLOR_EXT_BASE_IDX 2
+#define mmOTG0_OTG_VERTICAL_INTERRUPT0_POSITION 0x1b62
+#define mmOTG0_OTG_VERTICAL_INTERRUPT0_POSITION_BASE_IDX 2
+#define mmOTG0_OTG_VERTICAL_INTERRUPT0_CONTROL 0x1b63
+#define mmOTG0_OTG_VERTICAL_INTERRUPT0_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_VERTICAL_INTERRUPT1_POSITION 0x1b64
+#define mmOTG0_OTG_VERTICAL_INTERRUPT1_POSITION_BASE_IDX 2
+#define mmOTG0_OTG_VERTICAL_INTERRUPT1_CONTROL 0x1b65
+#define mmOTG0_OTG_VERTICAL_INTERRUPT1_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_VERTICAL_INTERRUPT2_POSITION 0x1b66
+#define mmOTG0_OTG_VERTICAL_INTERRUPT2_POSITION_BASE_IDX 2
+#define mmOTG0_OTG_VERTICAL_INTERRUPT2_CONTROL 0x1b67
+#define mmOTG0_OTG_VERTICAL_INTERRUPT2_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_CRC_CNTL 0x1b68
+#define mmOTG0_OTG_CRC_CNTL_BASE_IDX 2
+#define mmOTG0_OTG_CRC_CNTL2 0x1b69
+#define mmOTG0_OTG_CRC_CNTL2_BASE_IDX 2
+#define mmOTG0_OTG_CRC0_WINDOWA_X_CONTROL 0x1b6a
+#define mmOTG0_OTG_CRC0_WINDOWA_X_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_CRC0_WINDOWA_Y_CONTROL 0x1b6b
+#define mmOTG0_OTG_CRC0_WINDOWA_Y_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_CRC0_WINDOWB_X_CONTROL 0x1b6c
+#define mmOTG0_OTG_CRC0_WINDOWB_X_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_CRC0_WINDOWB_Y_CONTROL 0x1b6d
+#define mmOTG0_OTG_CRC0_WINDOWB_Y_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_CRC0_DATA_RG 0x1b6e
+#define mmOTG0_OTG_CRC0_DATA_RG_BASE_IDX 2
+#define mmOTG0_OTG_CRC0_DATA_B 0x1b6f
+#define mmOTG0_OTG_CRC0_DATA_B_BASE_IDX 2
+#define mmOTG0_OTG_CRC1_WINDOWA_X_CONTROL 0x1b70
+#define mmOTG0_OTG_CRC1_WINDOWA_X_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_CRC1_WINDOWA_Y_CONTROL 0x1b71
+#define mmOTG0_OTG_CRC1_WINDOWA_Y_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_CRC1_WINDOWB_X_CONTROL 0x1b72
+#define mmOTG0_OTG_CRC1_WINDOWB_X_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_CRC1_WINDOWB_Y_CONTROL 0x1b73
+#define mmOTG0_OTG_CRC1_WINDOWB_Y_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_CRC1_DATA_RG 0x1b74
+#define mmOTG0_OTG_CRC1_DATA_RG_BASE_IDX 2
+#define mmOTG0_OTG_CRC1_DATA_B 0x1b75
+#define mmOTG0_OTG_CRC1_DATA_B_BASE_IDX 2
+#define mmOTG0_OTG_CRC2_DATA_RG 0x1b76
+#define mmOTG0_OTG_CRC2_DATA_RG_BASE_IDX 2
+#define mmOTG0_OTG_CRC2_DATA_B 0x1b77
+#define mmOTG0_OTG_CRC2_DATA_B_BASE_IDX 2
+#define mmOTG0_OTG_CRC3_DATA_RG 0x1b78
+#define mmOTG0_OTG_CRC3_DATA_RG_BASE_IDX 2
+#define mmOTG0_OTG_CRC3_DATA_B 0x1b79
+#define mmOTG0_OTG_CRC3_DATA_B_BASE_IDX 2
+#define mmOTG0_OTG_CRC_SIG_RED_GREEN_MASK 0x1b7a
+#define mmOTG0_OTG_CRC_SIG_RED_GREEN_MASK_BASE_IDX 2
+#define mmOTG0_OTG_CRC_SIG_BLUE_CONTROL_MASK 0x1b7b
+#define mmOTG0_OTG_CRC_SIG_BLUE_CONTROL_MASK_BASE_IDX 2
+#define mmOTG0_OTG_STATIC_SCREEN_CONTROL 0x1b82
+#define mmOTG0_OTG_STATIC_SCREEN_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_3D_STRUCTURE_CONTROL 0x1b83
+#define mmOTG0_OTG_3D_STRUCTURE_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_GSL_VSYNC_GAP 0x1b84
+#define mmOTG0_OTG_GSL_VSYNC_GAP_BASE_IDX 2
+#define mmOTG0_OTG_MASTER_UPDATE_MODE 0x1b85
+#define mmOTG0_OTG_MASTER_UPDATE_MODE_BASE_IDX 2
+#define mmOTG0_OTG_CLOCK_CONTROL 0x1b86
+#define mmOTG0_OTG_CLOCK_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_VSTARTUP_PARAM 0x1b87
+#define mmOTG0_OTG_VSTARTUP_PARAM_BASE_IDX 2
+#define mmOTG0_OTG_VUPDATE_PARAM 0x1b88
+#define mmOTG0_OTG_VUPDATE_PARAM_BASE_IDX 2
+#define mmOTG0_OTG_VREADY_PARAM 0x1b89
+#define mmOTG0_OTG_VREADY_PARAM_BASE_IDX 2
+#define mmOTG0_OTG_GLOBAL_SYNC_STATUS 0x1b8a
+#define mmOTG0_OTG_GLOBAL_SYNC_STATUS_BASE_IDX 2
+#define mmOTG0_OTG_MASTER_UPDATE_LOCK 0x1b8b
+#define mmOTG0_OTG_MASTER_UPDATE_LOCK_BASE_IDX 2
+#define mmOTG0_OTG_GSL_CONTROL 0x1b8c
+#define mmOTG0_OTG_GSL_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_GSL_WINDOW_X 0x1b8d
+#define mmOTG0_OTG_GSL_WINDOW_X_BASE_IDX 2
+#define mmOTG0_OTG_GSL_WINDOW_Y 0x1b8e
+#define mmOTG0_OTG_GSL_WINDOW_Y_BASE_IDX 2
+#define mmOTG0_OTG_VUPDATE_KEEPOUT 0x1b8f
+#define mmOTG0_OTG_VUPDATE_KEEPOUT_BASE_IDX 2
+#define mmOTG0_OTG_GLOBAL_CONTROL0 0x1b90
+#define mmOTG0_OTG_GLOBAL_CONTROL0_BASE_IDX 2
+#define mmOTG0_OTG_GLOBAL_CONTROL1 0x1b91
+#define mmOTG0_OTG_GLOBAL_CONTROL1_BASE_IDX 2
+#define mmOTG0_OTG_GLOBAL_CONTROL2 0x1b92
+#define mmOTG0_OTG_GLOBAL_CONTROL2_BASE_IDX 2
+#define mmOTG0_OTG_GLOBAL_CONTROL3 0x1b93
+#define mmOTG0_OTG_GLOBAL_CONTROL3_BASE_IDX 2
+#define mmOTG0_OTG_TRIG_MANUAL_CONTROL 0x1b94
+#define mmOTG0_OTG_TRIG_MANUAL_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_MANUAL_FLOW_CONTROL 0x1b95
+#define mmOTG0_OTG_MANUAL_FLOW_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_RANGE_TIMING_INT_STATUS 0x1b96
+#define mmOTG0_OTG_RANGE_TIMING_INT_STATUS_BASE_IDX 2
+#define mmOTG0_OTG_DRR_CONTROL 0x1b97
+#define mmOTG0_OTG_DRR_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_REQUEST_CONTROL 0x1b98
+#define mmOTG0_OTG_REQUEST_CONTROL_BASE_IDX 2
+#define mmOTG0_OTG_DSC_START_POSITION 0x1b99
+#define mmOTG0_OTG_DSC_START_POSITION_BASE_IDX 2
+#define mmOTG0_OTG_PIPE_UPDATE_STATUS 0x1b9a
+#define mmOTG0_OTG_PIPE_UPDATE_STATUS_BASE_IDX 2
+#define mmOTG0_OTG_SPARE_REGISTER 0x1b9c
+#define mmOTG0_OTG_SPARE_REGISTER_BASE_IDX 2
+
+
+// addressBlock: dce_dc_optc_otg1_dispdec
+// base address: 0x200
+#define mmOTG1_OTG_H_TOTAL 0x1baa
+#define mmOTG1_OTG_H_TOTAL_BASE_IDX 2
+#define mmOTG1_OTG_H_BLANK_START_END 0x1bab
+#define mmOTG1_OTG_H_BLANK_START_END_BASE_IDX 2
+#define mmOTG1_OTG_H_SYNC_A 0x1bac
+#define mmOTG1_OTG_H_SYNC_A_BASE_IDX 2
+#define mmOTG1_OTG_H_SYNC_A_CNTL 0x1bad
+#define mmOTG1_OTG_H_SYNC_A_CNTL_BASE_IDX 2
+#define mmOTG1_OTG_H_TIMING_CNTL 0x1bae
+#define mmOTG1_OTG_H_TIMING_CNTL_BASE_IDX 2
+#define mmOTG1_OTG_V_TOTAL 0x1baf
+#define mmOTG1_OTG_V_TOTAL_BASE_IDX 2
+#define mmOTG1_OTG_V_TOTAL_MIN 0x1bb0
+#define mmOTG1_OTG_V_TOTAL_MIN_BASE_IDX 2
+#define mmOTG1_OTG_V_TOTAL_MAX 0x1bb1
+#define mmOTG1_OTG_V_TOTAL_MAX_BASE_IDX 2
+#define mmOTG1_OTG_V_TOTAL_MID 0x1bb2
+#define mmOTG1_OTG_V_TOTAL_MID_BASE_IDX 2
+#define mmOTG1_OTG_V_TOTAL_CONTROL 0x1bb3
+#define mmOTG1_OTG_V_TOTAL_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_V_TOTAL_INT_STATUS 0x1bb4
+#define mmOTG1_OTG_V_TOTAL_INT_STATUS_BASE_IDX 2
+#define mmOTG1_OTG_VSYNC_NOM_INT_STATUS 0x1bb5
+#define mmOTG1_OTG_VSYNC_NOM_INT_STATUS_BASE_IDX 2
+#define mmOTG1_OTG_V_BLANK_START_END 0x1bb6
+#define mmOTG1_OTG_V_BLANK_START_END_BASE_IDX 2
+#define mmOTG1_OTG_V_SYNC_A 0x1bb7
+#define mmOTG1_OTG_V_SYNC_A_BASE_IDX 2
+#define mmOTG1_OTG_V_SYNC_A_CNTL 0x1bb8
+#define mmOTG1_OTG_V_SYNC_A_CNTL_BASE_IDX 2
+#define mmOTG1_OTG_TRIGA_CNTL 0x1bb9
+#define mmOTG1_OTG_TRIGA_CNTL_BASE_IDX 2
+#define mmOTG1_OTG_TRIGA_MANUAL_TRIG 0x1bba
+#define mmOTG1_OTG_TRIGA_MANUAL_TRIG_BASE_IDX 2
+#define mmOTG1_OTG_TRIGB_CNTL 0x1bbb
+#define mmOTG1_OTG_TRIGB_CNTL_BASE_IDX 2
+#define mmOTG1_OTG_TRIGB_MANUAL_TRIG 0x1bbc
+#define mmOTG1_OTG_TRIGB_MANUAL_TRIG_BASE_IDX 2
+#define mmOTG1_OTG_FORCE_COUNT_NOW_CNTL 0x1bbd
+#define mmOTG1_OTG_FORCE_COUNT_NOW_CNTL_BASE_IDX 2
+#define mmOTG1_OTG_FLOW_CONTROL 0x1bbe
+#define mmOTG1_OTG_FLOW_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_STEREO_FORCE_NEXT_EYE 0x1bbf
+#define mmOTG1_OTG_STEREO_FORCE_NEXT_EYE_BASE_IDX 2
+#define mmOTG1_OTG_CONTROL 0x1bc1
+#define mmOTG1_OTG_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_BLANK_CONTROL 0x1bc2
+#define mmOTG1_OTG_BLANK_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_PIPE_ABORT_CONTROL 0x1bc3
+#define mmOTG1_OTG_PIPE_ABORT_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_INTERLACE_CONTROL 0x1bc4
+#define mmOTG1_OTG_INTERLACE_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_INTERLACE_STATUS 0x1bc5
+#define mmOTG1_OTG_INTERLACE_STATUS_BASE_IDX 2
+#define mmOTG1_OTG_PIXEL_DATA_READBACK0 0x1bc7
+#define mmOTG1_OTG_PIXEL_DATA_READBACK0_BASE_IDX 2
+#define mmOTG1_OTG_PIXEL_DATA_READBACK1 0x1bc8
+#define mmOTG1_OTG_PIXEL_DATA_READBACK1_BASE_IDX 2
+#define mmOTG1_OTG_STATUS 0x1bc9
+#define mmOTG1_OTG_STATUS_BASE_IDX 2
+#define mmOTG1_OTG_STATUS_POSITION 0x1bca
+#define mmOTG1_OTG_STATUS_POSITION_BASE_IDX 2
+#define mmOTG1_OTG_NOM_VERT_POSITION 0x1bcb
+#define mmOTG1_OTG_NOM_VERT_POSITION_BASE_IDX 2
+#define mmOTG1_OTG_STATUS_FRAME_COUNT 0x1bcc
+#define mmOTG1_OTG_STATUS_FRAME_COUNT_BASE_IDX 2
+#define mmOTG1_OTG_STATUS_VF_COUNT 0x1bcd
+#define mmOTG1_OTG_STATUS_VF_COUNT_BASE_IDX 2
+#define mmOTG1_OTG_STATUS_HV_COUNT 0x1bce
+#define mmOTG1_OTG_STATUS_HV_COUNT_BASE_IDX 2
+#define mmOTG1_OTG_COUNT_CONTROL 0x1bcf
+#define mmOTG1_OTG_COUNT_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_COUNT_RESET 0x1bd0
+#define mmOTG1_OTG_COUNT_RESET_BASE_IDX 2
+#define mmOTG1_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE 0x1bd1
+#define mmOTG1_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_BASE_IDX 2
+#define mmOTG1_OTG_VERT_SYNC_CONTROL 0x1bd2
+#define mmOTG1_OTG_VERT_SYNC_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_STEREO_STATUS 0x1bd3
+#define mmOTG1_OTG_STEREO_STATUS_BASE_IDX 2
+#define mmOTG1_OTG_STEREO_CONTROL 0x1bd4
+#define mmOTG1_OTG_STEREO_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_SNAPSHOT_STATUS 0x1bd5
+#define mmOTG1_OTG_SNAPSHOT_STATUS_BASE_IDX 2
+#define mmOTG1_OTG_SNAPSHOT_CONTROL 0x1bd6
+#define mmOTG1_OTG_SNAPSHOT_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_SNAPSHOT_POSITION 0x1bd7
+#define mmOTG1_OTG_SNAPSHOT_POSITION_BASE_IDX 2
+#define mmOTG1_OTG_SNAPSHOT_FRAME 0x1bd8
+#define mmOTG1_OTG_SNAPSHOT_FRAME_BASE_IDX 2
+#define mmOTG1_OTG_INTERRUPT_CONTROL 0x1bd9
+#define mmOTG1_OTG_INTERRUPT_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_UPDATE_LOCK 0x1bda
+#define mmOTG1_OTG_UPDATE_LOCK_BASE_IDX 2
+#define mmOTG1_OTG_DOUBLE_BUFFER_CONTROL 0x1bdb
+#define mmOTG1_OTG_DOUBLE_BUFFER_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_MASTER_EN 0x1bdc
+#define mmOTG1_OTG_MASTER_EN_BASE_IDX 2
+#define mmOTG1_OTG_BLANK_DATA_COLOR 0x1bde
+#define mmOTG1_OTG_BLANK_DATA_COLOR_BASE_IDX 2
+#define mmOTG1_OTG_BLANK_DATA_COLOR_EXT 0x1bdf
+#define mmOTG1_OTG_BLANK_DATA_COLOR_EXT_BASE_IDX 2
+#define mmOTG1_OTG_BLACK_COLOR 0x1be0
+#define mmOTG1_OTG_BLACK_COLOR_BASE_IDX 2
+#define mmOTG1_OTG_BLACK_COLOR_EXT 0x1be1
+#define mmOTG1_OTG_BLACK_COLOR_EXT_BASE_IDX 2
+#define mmOTG1_OTG_VERTICAL_INTERRUPT0_POSITION 0x1be2
+#define mmOTG1_OTG_VERTICAL_INTERRUPT0_POSITION_BASE_IDX 2
+#define mmOTG1_OTG_VERTICAL_INTERRUPT0_CONTROL 0x1be3
+#define mmOTG1_OTG_VERTICAL_INTERRUPT0_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_VERTICAL_INTERRUPT1_POSITION 0x1be4
+#define mmOTG1_OTG_VERTICAL_INTERRUPT1_POSITION_BASE_IDX 2
+#define mmOTG1_OTG_VERTICAL_INTERRUPT1_CONTROL 0x1be5
+#define mmOTG1_OTG_VERTICAL_INTERRUPT1_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_VERTICAL_INTERRUPT2_POSITION 0x1be6
+#define mmOTG1_OTG_VERTICAL_INTERRUPT2_POSITION_BASE_IDX 2
+#define mmOTG1_OTG_VERTICAL_INTERRUPT2_CONTROL 0x1be7
+#define mmOTG1_OTG_VERTICAL_INTERRUPT2_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_CRC_CNTL 0x1be8
+#define mmOTG1_OTG_CRC_CNTL_BASE_IDX 2
+#define mmOTG1_OTG_CRC_CNTL2 0x1be9
+#define mmOTG1_OTG_CRC_CNTL2_BASE_IDX 2
+#define mmOTG1_OTG_CRC0_WINDOWA_X_CONTROL 0x1bea
+#define mmOTG1_OTG_CRC0_WINDOWA_X_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_CRC0_WINDOWA_Y_CONTROL 0x1beb
+#define mmOTG1_OTG_CRC0_WINDOWA_Y_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_CRC0_WINDOWB_X_CONTROL 0x1bec
+#define mmOTG1_OTG_CRC0_WINDOWB_X_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_CRC0_WINDOWB_Y_CONTROL 0x1bed
+#define mmOTG1_OTG_CRC0_WINDOWB_Y_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_CRC0_DATA_RG 0x1bee
+#define mmOTG1_OTG_CRC0_DATA_RG_BASE_IDX 2
+#define mmOTG1_OTG_CRC0_DATA_B 0x1bef
+#define mmOTG1_OTG_CRC0_DATA_B_BASE_IDX 2
+#define mmOTG1_OTG_CRC1_WINDOWA_X_CONTROL 0x1bf0
+#define mmOTG1_OTG_CRC1_WINDOWA_X_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_CRC1_WINDOWA_Y_CONTROL 0x1bf1
+#define mmOTG1_OTG_CRC1_WINDOWA_Y_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_CRC1_WINDOWB_X_CONTROL 0x1bf2
+#define mmOTG1_OTG_CRC1_WINDOWB_X_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_CRC1_WINDOWB_Y_CONTROL 0x1bf3
+#define mmOTG1_OTG_CRC1_WINDOWB_Y_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_CRC1_DATA_RG 0x1bf4
+#define mmOTG1_OTG_CRC1_DATA_RG_BASE_IDX 2
+#define mmOTG1_OTG_CRC1_DATA_B 0x1bf5
+#define mmOTG1_OTG_CRC1_DATA_B_BASE_IDX 2
+#define mmOTG1_OTG_CRC2_DATA_RG 0x1bf6
+#define mmOTG1_OTG_CRC2_DATA_RG_BASE_IDX 2
+#define mmOTG1_OTG_CRC2_DATA_B 0x1bf7
+#define mmOTG1_OTG_CRC2_DATA_B_BASE_IDX 2
+#define mmOTG1_OTG_CRC3_DATA_RG 0x1bf8
+#define mmOTG1_OTG_CRC3_DATA_RG_BASE_IDX 2
+#define mmOTG1_OTG_CRC3_DATA_B 0x1bf9
+#define mmOTG1_OTG_CRC3_DATA_B_BASE_IDX 2
+#define mmOTG1_OTG_CRC_SIG_RED_GREEN_MASK 0x1bfa
+#define mmOTG1_OTG_CRC_SIG_RED_GREEN_MASK_BASE_IDX 2
+#define mmOTG1_OTG_CRC_SIG_BLUE_CONTROL_MASK 0x1bfb
+#define mmOTG1_OTG_CRC_SIG_BLUE_CONTROL_MASK_BASE_IDX 2
+#define mmOTG1_OTG_STATIC_SCREEN_CONTROL 0x1c02
+#define mmOTG1_OTG_STATIC_SCREEN_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_3D_STRUCTURE_CONTROL 0x1c03
+#define mmOTG1_OTG_3D_STRUCTURE_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_GSL_VSYNC_GAP 0x1c04
+#define mmOTG1_OTG_GSL_VSYNC_GAP_BASE_IDX 2
+#define mmOTG1_OTG_MASTER_UPDATE_MODE 0x1c05
+#define mmOTG1_OTG_MASTER_UPDATE_MODE_BASE_IDX 2
+#define mmOTG1_OTG_CLOCK_CONTROL 0x1c06
+#define mmOTG1_OTG_CLOCK_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_VSTARTUP_PARAM 0x1c07
+#define mmOTG1_OTG_VSTARTUP_PARAM_BASE_IDX 2
+#define mmOTG1_OTG_VUPDATE_PARAM 0x1c08
+#define mmOTG1_OTG_VUPDATE_PARAM_BASE_IDX 2
+#define mmOTG1_OTG_VREADY_PARAM 0x1c09
+#define mmOTG1_OTG_VREADY_PARAM_BASE_IDX 2
+#define mmOTG1_OTG_GLOBAL_SYNC_STATUS 0x1c0a
+#define mmOTG1_OTG_GLOBAL_SYNC_STATUS_BASE_IDX 2
+#define mmOTG1_OTG_MASTER_UPDATE_LOCK 0x1c0b
+#define mmOTG1_OTG_MASTER_UPDATE_LOCK_BASE_IDX 2
+#define mmOTG1_OTG_GSL_CONTROL 0x1c0c
+#define mmOTG1_OTG_GSL_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_GSL_WINDOW_X 0x1c0d
+#define mmOTG1_OTG_GSL_WINDOW_X_BASE_IDX 2
+#define mmOTG1_OTG_GSL_WINDOW_Y 0x1c0e
+#define mmOTG1_OTG_GSL_WINDOW_Y_BASE_IDX 2
+#define mmOTG1_OTG_VUPDATE_KEEPOUT 0x1c0f
+#define mmOTG1_OTG_VUPDATE_KEEPOUT_BASE_IDX 2
+#define mmOTG1_OTG_GLOBAL_CONTROL0 0x1c10
+#define mmOTG1_OTG_GLOBAL_CONTROL0_BASE_IDX 2
+#define mmOTG1_OTG_GLOBAL_CONTROL1 0x1c11
+#define mmOTG1_OTG_GLOBAL_CONTROL1_BASE_IDX 2
+#define mmOTG1_OTG_GLOBAL_CONTROL2 0x1c12
+#define mmOTG1_OTG_GLOBAL_CONTROL2_BASE_IDX 2
+#define mmOTG1_OTG_GLOBAL_CONTROL3 0x1c13
+#define mmOTG1_OTG_GLOBAL_CONTROL3_BASE_IDX 2
+#define mmOTG1_OTG_TRIG_MANUAL_CONTROL 0x1c14
+#define mmOTG1_OTG_TRIG_MANUAL_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_MANUAL_FLOW_CONTROL 0x1c15
+#define mmOTG1_OTG_MANUAL_FLOW_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_RANGE_TIMING_INT_STATUS 0x1c16
+#define mmOTG1_OTG_RANGE_TIMING_INT_STATUS_BASE_IDX 2
+#define mmOTG1_OTG_DRR_CONTROL 0x1c17
+#define mmOTG1_OTG_DRR_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_REQUEST_CONTROL 0x1c18
+#define mmOTG1_OTG_REQUEST_CONTROL_BASE_IDX 2
+#define mmOTG1_OTG_DSC_START_POSITION 0x1c19
+#define mmOTG1_OTG_DSC_START_POSITION_BASE_IDX 2
+#define mmOTG1_OTG_PIPE_UPDATE_STATUS 0x1c1a
+#define mmOTG1_OTG_PIPE_UPDATE_STATUS_BASE_IDX 2
+#define mmOTG1_OTG_SPARE_REGISTER 0x1c1c
+#define mmOTG1_OTG_SPARE_REGISTER_BASE_IDX 2
+
+
+// addressBlock: dce_dc_optc_otg2_dispdec
+// base address: 0x400
+#define mmOTG2_OTG_H_TOTAL 0x1c2a
+#define mmOTG2_OTG_H_TOTAL_BASE_IDX 2
+#define mmOTG2_OTG_H_BLANK_START_END 0x1c2b
+#define mmOTG2_OTG_H_BLANK_START_END_BASE_IDX 2
+#define mmOTG2_OTG_H_SYNC_A 0x1c2c
+#define mmOTG2_OTG_H_SYNC_A_BASE_IDX 2
+#define mmOTG2_OTG_H_SYNC_A_CNTL 0x1c2d
+#define mmOTG2_OTG_H_SYNC_A_CNTL_BASE_IDX 2
+#define mmOTG2_OTG_H_TIMING_CNTL 0x1c2e
+#define mmOTG2_OTG_H_TIMING_CNTL_BASE_IDX 2
+#define mmOTG2_OTG_V_TOTAL 0x1c2f
+#define mmOTG2_OTG_V_TOTAL_BASE_IDX 2
+#define mmOTG2_OTG_V_TOTAL_MIN 0x1c30
+#define mmOTG2_OTG_V_TOTAL_MIN_BASE_IDX 2
+#define mmOTG2_OTG_V_TOTAL_MAX 0x1c31
+#define mmOTG2_OTG_V_TOTAL_MAX_BASE_IDX 2
+#define mmOTG2_OTG_V_TOTAL_MID 0x1c32
+#define mmOTG2_OTG_V_TOTAL_MID_BASE_IDX 2
+#define mmOTG2_OTG_V_TOTAL_CONTROL 0x1c33
+#define mmOTG2_OTG_V_TOTAL_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_V_TOTAL_INT_STATUS 0x1c34
+#define mmOTG2_OTG_V_TOTAL_INT_STATUS_BASE_IDX 2
+#define mmOTG2_OTG_VSYNC_NOM_INT_STATUS 0x1c35
+#define mmOTG2_OTG_VSYNC_NOM_INT_STATUS_BASE_IDX 2
+#define mmOTG2_OTG_V_BLANK_START_END 0x1c36
+#define mmOTG2_OTG_V_BLANK_START_END_BASE_IDX 2
+#define mmOTG2_OTG_V_SYNC_A 0x1c37
+#define mmOTG2_OTG_V_SYNC_A_BASE_IDX 2
+#define mmOTG2_OTG_V_SYNC_A_CNTL 0x1c38
+#define mmOTG2_OTG_V_SYNC_A_CNTL_BASE_IDX 2
+#define mmOTG2_OTG_TRIGA_CNTL 0x1c39
+#define mmOTG2_OTG_TRIGA_CNTL_BASE_IDX 2
+#define mmOTG2_OTG_TRIGA_MANUAL_TRIG 0x1c3a
+#define mmOTG2_OTG_TRIGA_MANUAL_TRIG_BASE_IDX 2
+#define mmOTG2_OTG_TRIGB_CNTL 0x1c3b
+#define mmOTG2_OTG_TRIGB_CNTL_BASE_IDX 2
+#define mmOTG2_OTG_TRIGB_MANUAL_TRIG 0x1c3c
+#define mmOTG2_OTG_TRIGB_MANUAL_TRIG_BASE_IDX 2
+#define mmOTG2_OTG_FORCE_COUNT_NOW_CNTL 0x1c3d
+#define mmOTG2_OTG_FORCE_COUNT_NOW_CNTL_BASE_IDX 2
+#define mmOTG2_OTG_FLOW_CONTROL 0x1c3e
+#define mmOTG2_OTG_FLOW_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_STEREO_FORCE_NEXT_EYE 0x1c3f
+#define mmOTG2_OTG_STEREO_FORCE_NEXT_EYE_BASE_IDX 2
+#define mmOTG2_OTG_CONTROL 0x1c41
+#define mmOTG2_OTG_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_BLANK_CONTROL 0x1c42
+#define mmOTG2_OTG_BLANK_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_PIPE_ABORT_CONTROL 0x1c43
+#define mmOTG2_OTG_PIPE_ABORT_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_INTERLACE_CONTROL 0x1c44
+#define mmOTG2_OTG_INTERLACE_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_INTERLACE_STATUS 0x1c45
+#define mmOTG2_OTG_INTERLACE_STATUS_BASE_IDX 2
+#define mmOTG2_OTG_PIXEL_DATA_READBACK0 0x1c47
+#define mmOTG2_OTG_PIXEL_DATA_READBACK0_BASE_IDX 2
+#define mmOTG2_OTG_PIXEL_DATA_READBACK1 0x1c48
+#define mmOTG2_OTG_PIXEL_DATA_READBACK1_BASE_IDX 2
+#define mmOTG2_OTG_STATUS 0x1c49
+#define mmOTG2_OTG_STATUS_BASE_IDX 2
+#define mmOTG2_OTG_STATUS_POSITION 0x1c4a
+#define mmOTG2_OTG_STATUS_POSITION_BASE_IDX 2
+#define mmOTG2_OTG_NOM_VERT_POSITION 0x1c4b
+#define mmOTG2_OTG_NOM_VERT_POSITION_BASE_IDX 2
+#define mmOTG2_OTG_STATUS_FRAME_COUNT 0x1c4c
+#define mmOTG2_OTG_STATUS_FRAME_COUNT_BASE_IDX 2
+#define mmOTG2_OTG_STATUS_VF_COUNT 0x1c4d
+#define mmOTG2_OTG_STATUS_VF_COUNT_BASE_IDX 2
+#define mmOTG2_OTG_STATUS_HV_COUNT 0x1c4e
+#define mmOTG2_OTG_STATUS_HV_COUNT_BASE_IDX 2
+#define mmOTG2_OTG_COUNT_CONTROL 0x1c4f
+#define mmOTG2_OTG_COUNT_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_COUNT_RESET 0x1c50
+#define mmOTG2_OTG_COUNT_RESET_BASE_IDX 2
+#define mmOTG2_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE 0x1c51
+#define mmOTG2_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_BASE_IDX 2
+#define mmOTG2_OTG_VERT_SYNC_CONTROL 0x1c52
+#define mmOTG2_OTG_VERT_SYNC_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_STEREO_STATUS 0x1c53
+#define mmOTG2_OTG_STEREO_STATUS_BASE_IDX 2
+#define mmOTG2_OTG_STEREO_CONTROL 0x1c54
+#define mmOTG2_OTG_STEREO_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_SNAPSHOT_STATUS 0x1c55
+#define mmOTG2_OTG_SNAPSHOT_STATUS_BASE_IDX 2
+#define mmOTG2_OTG_SNAPSHOT_CONTROL 0x1c56
+#define mmOTG2_OTG_SNAPSHOT_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_SNAPSHOT_POSITION 0x1c57
+#define mmOTG2_OTG_SNAPSHOT_POSITION_BASE_IDX 2
+#define mmOTG2_OTG_SNAPSHOT_FRAME 0x1c58
+#define mmOTG2_OTG_SNAPSHOT_FRAME_BASE_IDX 2
+#define mmOTG2_OTG_INTERRUPT_CONTROL 0x1c59
+#define mmOTG2_OTG_INTERRUPT_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_UPDATE_LOCK 0x1c5a
+#define mmOTG2_OTG_UPDATE_LOCK_BASE_IDX 2
+#define mmOTG2_OTG_DOUBLE_BUFFER_CONTROL 0x1c5b
+#define mmOTG2_OTG_DOUBLE_BUFFER_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_MASTER_EN 0x1c5c
+#define mmOTG2_OTG_MASTER_EN_BASE_IDX 2
+#define mmOTG2_OTG_BLANK_DATA_COLOR 0x1c5e
+#define mmOTG2_OTG_BLANK_DATA_COLOR_BASE_IDX 2
+#define mmOTG2_OTG_BLANK_DATA_COLOR_EXT 0x1c5f
+#define mmOTG2_OTG_BLANK_DATA_COLOR_EXT_BASE_IDX 2
+#define mmOTG2_OTG_BLACK_COLOR 0x1c60
+#define mmOTG2_OTG_BLACK_COLOR_BASE_IDX 2
+#define mmOTG2_OTG_BLACK_COLOR_EXT 0x1c61
+#define mmOTG2_OTG_BLACK_COLOR_EXT_BASE_IDX 2
+#define mmOTG2_OTG_VERTICAL_INTERRUPT0_POSITION 0x1c62
+#define mmOTG2_OTG_VERTICAL_INTERRUPT0_POSITION_BASE_IDX 2
+#define mmOTG2_OTG_VERTICAL_INTERRUPT0_CONTROL 0x1c63
+#define mmOTG2_OTG_VERTICAL_INTERRUPT0_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_VERTICAL_INTERRUPT1_POSITION 0x1c64
+#define mmOTG2_OTG_VERTICAL_INTERRUPT1_POSITION_BASE_IDX 2
+#define mmOTG2_OTG_VERTICAL_INTERRUPT1_CONTROL 0x1c65
+#define mmOTG2_OTG_VERTICAL_INTERRUPT1_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_VERTICAL_INTERRUPT2_POSITION 0x1c66
+#define mmOTG2_OTG_VERTICAL_INTERRUPT2_POSITION_BASE_IDX 2
+#define mmOTG2_OTG_VERTICAL_INTERRUPT2_CONTROL 0x1c67
+#define mmOTG2_OTG_VERTICAL_INTERRUPT2_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_CRC_CNTL 0x1c68
+#define mmOTG2_OTG_CRC_CNTL_BASE_IDX 2
+#define mmOTG2_OTG_CRC_CNTL2 0x1c69
+#define mmOTG2_OTG_CRC_CNTL2_BASE_IDX 2
+#define mmOTG2_OTG_CRC0_WINDOWA_X_CONTROL 0x1c6a
+#define mmOTG2_OTG_CRC0_WINDOWA_X_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_CRC0_WINDOWA_Y_CONTROL 0x1c6b
+#define mmOTG2_OTG_CRC0_WINDOWA_Y_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_CRC0_WINDOWB_X_CONTROL 0x1c6c
+#define mmOTG2_OTG_CRC0_WINDOWB_X_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_CRC0_WINDOWB_Y_CONTROL 0x1c6d
+#define mmOTG2_OTG_CRC0_WINDOWB_Y_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_CRC0_DATA_RG 0x1c6e
+#define mmOTG2_OTG_CRC0_DATA_RG_BASE_IDX 2
+#define mmOTG2_OTG_CRC0_DATA_B 0x1c6f
+#define mmOTG2_OTG_CRC0_DATA_B_BASE_IDX 2
+#define mmOTG2_OTG_CRC1_WINDOWA_X_CONTROL 0x1c70
+#define mmOTG2_OTG_CRC1_WINDOWA_X_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_CRC1_WINDOWA_Y_CONTROL 0x1c71
+#define mmOTG2_OTG_CRC1_WINDOWA_Y_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_CRC1_WINDOWB_X_CONTROL 0x1c72
+#define mmOTG2_OTG_CRC1_WINDOWB_X_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_CRC1_WINDOWB_Y_CONTROL 0x1c73
+#define mmOTG2_OTG_CRC1_WINDOWB_Y_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_CRC1_DATA_RG 0x1c74
+#define mmOTG2_OTG_CRC1_DATA_RG_BASE_IDX 2
+#define mmOTG2_OTG_CRC1_DATA_B 0x1c75
+#define mmOTG2_OTG_CRC1_DATA_B_BASE_IDX 2
+#define mmOTG2_OTG_CRC2_DATA_RG 0x1c76
+#define mmOTG2_OTG_CRC2_DATA_RG_BASE_IDX 2
+#define mmOTG2_OTG_CRC2_DATA_B 0x1c77
+#define mmOTG2_OTG_CRC2_DATA_B_BASE_IDX 2
+#define mmOTG2_OTG_CRC3_DATA_RG 0x1c78
+#define mmOTG2_OTG_CRC3_DATA_RG_BASE_IDX 2
+#define mmOTG2_OTG_CRC3_DATA_B 0x1c79
+#define mmOTG2_OTG_CRC3_DATA_B_BASE_IDX 2
+#define mmOTG2_OTG_CRC_SIG_RED_GREEN_MASK 0x1c7a
+#define mmOTG2_OTG_CRC_SIG_RED_GREEN_MASK_BASE_IDX 2
+#define mmOTG2_OTG_CRC_SIG_BLUE_CONTROL_MASK 0x1c7b
+#define mmOTG2_OTG_CRC_SIG_BLUE_CONTROL_MASK_BASE_IDX 2
+#define mmOTG2_OTG_STATIC_SCREEN_CONTROL 0x1c82
+#define mmOTG2_OTG_STATIC_SCREEN_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_3D_STRUCTURE_CONTROL 0x1c83
+#define mmOTG2_OTG_3D_STRUCTURE_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_GSL_VSYNC_GAP 0x1c84
+#define mmOTG2_OTG_GSL_VSYNC_GAP_BASE_IDX 2
+#define mmOTG2_OTG_MASTER_UPDATE_MODE 0x1c85
+#define mmOTG2_OTG_MASTER_UPDATE_MODE_BASE_IDX 2
+#define mmOTG2_OTG_CLOCK_CONTROL 0x1c86
+#define mmOTG2_OTG_CLOCK_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_VSTARTUP_PARAM 0x1c87
+#define mmOTG2_OTG_VSTARTUP_PARAM_BASE_IDX 2
+#define mmOTG2_OTG_VUPDATE_PARAM 0x1c88
+#define mmOTG2_OTG_VUPDATE_PARAM_BASE_IDX 2
+#define mmOTG2_OTG_VREADY_PARAM 0x1c89
+#define mmOTG2_OTG_VREADY_PARAM_BASE_IDX 2
+#define mmOTG2_OTG_GLOBAL_SYNC_STATUS 0x1c8a
+#define mmOTG2_OTG_GLOBAL_SYNC_STATUS_BASE_IDX 2
+#define mmOTG2_OTG_MASTER_UPDATE_LOCK 0x1c8b
+#define mmOTG2_OTG_MASTER_UPDATE_LOCK_BASE_IDX 2
+#define mmOTG2_OTG_GSL_CONTROL 0x1c8c
+#define mmOTG2_OTG_GSL_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_GSL_WINDOW_X 0x1c8d
+#define mmOTG2_OTG_GSL_WINDOW_X_BASE_IDX 2
+#define mmOTG2_OTG_GSL_WINDOW_Y 0x1c8e
+#define mmOTG2_OTG_GSL_WINDOW_Y_BASE_IDX 2
+#define mmOTG2_OTG_VUPDATE_KEEPOUT 0x1c8f
+#define mmOTG2_OTG_VUPDATE_KEEPOUT_BASE_IDX 2
+#define mmOTG2_OTG_GLOBAL_CONTROL0 0x1c90
+#define mmOTG2_OTG_GLOBAL_CONTROL0_BASE_IDX 2
+#define mmOTG2_OTG_GLOBAL_CONTROL1 0x1c91
+#define mmOTG2_OTG_GLOBAL_CONTROL1_BASE_IDX 2
+#define mmOTG2_OTG_GLOBAL_CONTROL2 0x1c92
+#define mmOTG2_OTG_GLOBAL_CONTROL2_BASE_IDX 2
+#define mmOTG2_OTG_GLOBAL_CONTROL3 0x1c93
+#define mmOTG2_OTG_GLOBAL_CONTROL3_BASE_IDX 2
+#define mmOTG2_OTG_TRIG_MANUAL_CONTROL 0x1c94
+#define mmOTG2_OTG_TRIG_MANUAL_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_MANUAL_FLOW_CONTROL 0x1c95
+#define mmOTG2_OTG_MANUAL_FLOW_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_RANGE_TIMING_INT_STATUS 0x1c96
+#define mmOTG2_OTG_RANGE_TIMING_INT_STATUS_BASE_IDX 2
+#define mmOTG2_OTG_DRR_CONTROL 0x1c97
+#define mmOTG2_OTG_DRR_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_REQUEST_CONTROL 0x1c98
+#define mmOTG2_OTG_REQUEST_CONTROL_BASE_IDX 2
+#define mmOTG2_OTG_DSC_START_POSITION 0x1c99
+#define mmOTG2_OTG_DSC_START_POSITION_BASE_IDX 2
+#define mmOTG2_OTG_PIPE_UPDATE_STATUS 0x1c9a
+#define mmOTG2_OTG_PIPE_UPDATE_STATUS_BASE_IDX 2
+#define mmOTG2_OTG_SPARE_REGISTER 0x1c9c
+#define mmOTG2_OTG_SPARE_REGISTER_BASE_IDX 2
+
+
+// addressBlock: dce_dc_optc_otg3_dispdec
+// base address: 0x600
+#define mmOTG3_OTG_H_TOTAL 0x1caa
+#define mmOTG3_OTG_H_TOTAL_BASE_IDX 2
+#define mmOTG3_OTG_H_BLANK_START_END 0x1cab
+#define mmOTG3_OTG_H_BLANK_START_END_BASE_IDX 2
+#define mmOTG3_OTG_H_SYNC_A 0x1cac
+#define mmOTG3_OTG_H_SYNC_A_BASE_IDX 2
+#define mmOTG3_OTG_H_SYNC_A_CNTL 0x1cad
+#define mmOTG3_OTG_H_SYNC_A_CNTL_BASE_IDX 2
+#define mmOTG3_OTG_H_TIMING_CNTL 0x1cae
+#define mmOTG3_OTG_H_TIMING_CNTL_BASE_IDX 2
+#define mmOTG3_OTG_V_TOTAL 0x1caf
+#define mmOTG3_OTG_V_TOTAL_BASE_IDX 2
+#define mmOTG3_OTG_V_TOTAL_MIN 0x1cb0
+#define mmOTG3_OTG_V_TOTAL_MIN_BASE_IDX 2
+#define mmOTG3_OTG_V_TOTAL_MAX 0x1cb1
+#define mmOTG3_OTG_V_TOTAL_MAX_BASE_IDX 2
+#define mmOTG3_OTG_V_TOTAL_MID 0x1cb2
+#define mmOTG3_OTG_V_TOTAL_MID_BASE_IDX 2
+#define mmOTG3_OTG_V_TOTAL_CONTROL 0x1cb3
+#define mmOTG3_OTG_V_TOTAL_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_V_TOTAL_INT_STATUS 0x1cb4
+#define mmOTG3_OTG_V_TOTAL_INT_STATUS_BASE_IDX 2
+#define mmOTG3_OTG_VSYNC_NOM_INT_STATUS 0x1cb5
+#define mmOTG3_OTG_VSYNC_NOM_INT_STATUS_BASE_IDX 2
+#define mmOTG3_OTG_V_BLANK_START_END 0x1cb6
+#define mmOTG3_OTG_V_BLANK_START_END_BASE_IDX 2
+#define mmOTG3_OTG_V_SYNC_A 0x1cb7
+#define mmOTG3_OTG_V_SYNC_A_BASE_IDX 2
+#define mmOTG3_OTG_V_SYNC_A_CNTL 0x1cb8
+#define mmOTG3_OTG_V_SYNC_A_CNTL_BASE_IDX 2
+#define mmOTG3_OTG_TRIGA_CNTL 0x1cb9
+#define mmOTG3_OTG_TRIGA_CNTL_BASE_IDX 2
+#define mmOTG3_OTG_TRIGA_MANUAL_TRIG 0x1cba
+#define mmOTG3_OTG_TRIGA_MANUAL_TRIG_BASE_IDX 2
+#define mmOTG3_OTG_TRIGB_CNTL 0x1cbb
+#define mmOTG3_OTG_TRIGB_CNTL_BASE_IDX 2
+#define mmOTG3_OTG_TRIGB_MANUAL_TRIG 0x1cbc
+#define mmOTG3_OTG_TRIGB_MANUAL_TRIG_BASE_IDX 2
+#define mmOTG3_OTG_FORCE_COUNT_NOW_CNTL 0x1cbd
+#define mmOTG3_OTG_FORCE_COUNT_NOW_CNTL_BASE_IDX 2
+#define mmOTG3_OTG_FLOW_CONTROL 0x1cbe
+#define mmOTG3_OTG_FLOW_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_STEREO_FORCE_NEXT_EYE 0x1cbf
+#define mmOTG3_OTG_STEREO_FORCE_NEXT_EYE_BASE_IDX 2
+#define mmOTG3_OTG_CONTROL 0x1cc1
+#define mmOTG3_OTG_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_BLANK_CONTROL 0x1cc2
+#define mmOTG3_OTG_BLANK_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_PIPE_ABORT_CONTROL 0x1cc3
+#define mmOTG3_OTG_PIPE_ABORT_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_INTERLACE_CONTROL 0x1cc4
+#define mmOTG3_OTG_INTERLACE_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_INTERLACE_STATUS 0x1cc5
+#define mmOTG3_OTG_INTERLACE_STATUS_BASE_IDX 2
+#define mmOTG3_OTG_PIXEL_DATA_READBACK0 0x1cc7
+#define mmOTG3_OTG_PIXEL_DATA_READBACK0_BASE_IDX 2
+#define mmOTG3_OTG_PIXEL_DATA_READBACK1 0x1cc8
+#define mmOTG3_OTG_PIXEL_DATA_READBACK1_BASE_IDX 2
+#define mmOTG3_OTG_STATUS 0x1cc9
+#define mmOTG3_OTG_STATUS_BASE_IDX 2
+#define mmOTG3_OTG_STATUS_POSITION 0x1cca
+#define mmOTG3_OTG_STATUS_POSITION_BASE_IDX 2
+#define mmOTG3_OTG_NOM_VERT_POSITION 0x1ccb
+#define mmOTG3_OTG_NOM_VERT_POSITION_BASE_IDX 2
+#define mmOTG3_OTG_STATUS_FRAME_COUNT 0x1ccc
+#define mmOTG3_OTG_STATUS_FRAME_COUNT_BASE_IDX 2
+#define mmOTG3_OTG_STATUS_VF_COUNT 0x1ccd
+#define mmOTG3_OTG_STATUS_VF_COUNT_BASE_IDX 2
+#define mmOTG3_OTG_STATUS_HV_COUNT 0x1cce
+#define mmOTG3_OTG_STATUS_HV_COUNT_BASE_IDX 2
+#define mmOTG3_OTG_COUNT_CONTROL 0x1ccf
+#define mmOTG3_OTG_COUNT_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_COUNT_RESET 0x1cd0
+#define mmOTG3_OTG_COUNT_RESET_BASE_IDX 2
+#define mmOTG3_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE 0x1cd1
+#define mmOTG3_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_BASE_IDX 2
+#define mmOTG3_OTG_VERT_SYNC_CONTROL 0x1cd2
+#define mmOTG3_OTG_VERT_SYNC_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_STEREO_STATUS 0x1cd3
+#define mmOTG3_OTG_STEREO_STATUS_BASE_IDX 2
+#define mmOTG3_OTG_STEREO_CONTROL 0x1cd4
+#define mmOTG3_OTG_STEREO_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_SNAPSHOT_STATUS 0x1cd5
+#define mmOTG3_OTG_SNAPSHOT_STATUS_BASE_IDX 2
+#define mmOTG3_OTG_SNAPSHOT_CONTROL 0x1cd6
+#define mmOTG3_OTG_SNAPSHOT_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_SNAPSHOT_POSITION 0x1cd7
+#define mmOTG3_OTG_SNAPSHOT_POSITION_BASE_IDX 2
+#define mmOTG3_OTG_SNAPSHOT_FRAME 0x1cd8
+#define mmOTG3_OTG_SNAPSHOT_FRAME_BASE_IDX 2
+#define mmOTG3_OTG_INTERRUPT_CONTROL 0x1cd9
+#define mmOTG3_OTG_INTERRUPT_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_UPDATE_LOCK 0x1cda
+#define mmOTG3_OTG_UPDATE_LOCK_BASE_IDX 2
+#define mmOTG3_OTG_DOUBLE_BUFFER_CONTROL 0x1cdb
+#define mmOTG3_OTG_DOUBLE_BUFFER_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_MASTER_EN 0x1cdc
+#define mmOTG3_OTG_MASTER_EN_BASE_IDX 2
+#define mmOTG3_OTG_BLANK_DATA_COLOR 0x1cde
+#define mmOTG3_OTG_BLANK_DATA_COLOR_BASE_IDX 2
+#define mmOTG3_OTG_BLANK_DATA_COLOR_EXT 0x1cdf
+#define mmOTG3_OTG_BLANK_DATA_COLOR_EXT_BASE_IDX 2
+#define mmOTG3_OTG_BLACK_COLOR 0x1ce0
+#define mmOTG3_OTG_BLACK_COLOR_BASE_IDX 2
+#define mmOTG3_OTG_BLACK_COLOR_EXT 0x1ce1
+#define mmOTG3_OTG_BLACK_COLOR_EXT_BASE_IDX 2
+#define mmOTG3_OTG_VERTICAL_INTERRUPT0_POSITION 0x1ce2
+#define mmOTG3_OTG_VERTICAL_INTERRUPT0_POSITION_BASE_IDX 2
+#define mmOTG3_OTG_VERTICAL_INTERRUPT0_CONTROL 0x1ce3
+#define mmOTG3_OTG_VERTICAL_INTERRUPT0_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_VERTICAL_INTERRUPT1_POSITION 0x1ce4
+#define mmOTG3_OTG_VERTICAL_INTERRUPT1_POSITION_BASE_IDX 2
+#define mmOTG3_OTG_VERTICAL_INTERRUPT1_CONTROL 0x1ce5
+#define mmOTG3_OTG_VERTICAL_INTERRUPT1_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_VERTICAL_INTERRUPT2_POSITION 0x1ce6
+#define mmOTG3_OTG_VERTICAL_INTERRUPT2_POSITION_BASE_IDX 2
+#define mmOTG3_OTG_VERTICAL_INTERRUPT2_CONTROL 0x1ce7
+#define mmOTG3_OTG_VERTICAL_INTERRUPT2_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_CRC_CNTL 0x1ce8
+#define mmOTG3_OTG_CRC_CNTL_BASE_IDX 2
+#define mmOTG3_OTG_CRC_CNTL2 0x1ce9
+#define mmOTG3_OTG_CRC_CNTL2_BASE_IDX 2
+#define mmOTG3_OTG_CRC0_WINDOWA_X_CONTROL 0x1cea
+#define mmOTG3_OTG_CRC0_WINDOWA_X_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_CRC0_WINDOWA_Y_CONTROL 0x1ceb
+#define mmOTG3_OTG_CRC0_WINDOWA_Y_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_CRC0_WINDOWB_X_CONTROL 0x1cec
+#define mmOTG3_OTG_CRC0_WINDOWB_X_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_CRC0_WINDOWB_Y_CONTROL 0x1ced
+#define mmOTG3_OTG_CRC0_WINDOWB_Y_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_CRC0_DATA_RG 0x1cee
+#define mmOTG3_OTG_CRC0_DATA_RG_BASE_IDX 2
+#define mmOTG3_OTG_CRC0_DATA_B 0x1cef
+#define mmOTG3_OTG_CRC0_DATA_B_BASE_IDX 2
+#define mmOTG3_OTG_CRC1_WINDOWA_X_CONTROL 0x1cf0
+#define mmOTG3_OTG_CRC1_WINDOWA_X_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_CRC1_WINDOWA_Y_CONTROL 0x1cf1
+#define mmOTG3_OTG_CRC1_WINDOWA_Y_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_CRC1_WINDOWB_X_CONTROL 0x1cf2
+#define mmOTG3_OTG_CRC1_WINDOWB_X_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_CRC1_WINDOWB_Y_CONTROL 0x1cf3
+#define mmOTG3_OTG_CRC1_WINDOWB_Y_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_CRC1_DATA_RG 0x1cf4
+#define mmOTG3_OTG_CRC1_DATA_RG_BASE_IDX 2
+#define mmOTG3_OTG_CRC1_DATA_B 0x1cf5
+#define mmOTG3_OTG_CRC1_DATA_B_BASE_IDX 2
+#define mmOTG3_OTG_CRC2_DATA_RG 0x1cf6
+#define mmOTG3_OTG_CRC2_DATA_RG_BASE_IDX 2
+#define mmOTG3_OTG_CRC2_DATA_B 0x1cf7
+#define mmOTG3_OTG_CRC2_DATA_B_BASE_IDX 2
+#define mmOTG3_OTG_CRC3_DATA_RG 0x1cf8
+#define mmOTG3_OTG_CRC3_DATA_RG_BASE_IDX 2
+#define mmOTG3_OTG_CRC3_DATA_B 0x1cf9
+#define mmOTG3_OTG_CRC3_DATA_B_BASE_IDX 2
+#define mmOTG3_OTG_CRC_SIG_RED_GREEN_MASK 0x1cfa
+#define mmOTG3_OTG_CRC_SIG_RED_GREEN_MASK_BASE_IDX 2
+#define mmOTG3_OTG_CRC_SIG_BLUE_CONTROL_MASK 0x1cfb
+#define mmOTG3_OTG_CRC_SIG_BLUE_CONTROL_MASK_BASE_IDX 2
+#define mmOTG3_OTG_STATIC_SCREEN_CONTROL 0x1d02
+#define mmOTG3_OTG_STATIC_SCREEN_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_3D_STRUCTURE_CONTROL 0x1d03
+#define mmOTG3_OTG_3D_STRUCTURE_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_GSL_VSYNC_GAP 0x1d04
+#define mmOTG3_OTG_GSL_VSYNC_GAP_BASE_IDX 2
+#define mmOTG3_OTG_MASTER_UPDATE_MODE 0x1d05
+#define mmOTG3_OTG_MASTER_UPDATE_MODE_BASE_IDX 2
+#define mmOTG3_OTG_CLOCK_CONTROL 0x1d06
+#define mmOTG3_OTG_CLOCK_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_VSTARTUP_PARAM 0x1d07
+#define mmOTG3_OTG_VSTARTUP_PARAM_BASE_IDX 2
+#define mmOTG3_OTG_VUPDATE_PARAM 0x1d08
+#define mmOTG3_OTG_VUPDATE_PARAM_BASE_IDX 2
+#define mmOTG3_OTG_VREADY_PARAM 0x1d09
+#define mmOTG3_OTG_VREADY_PARAM_BASE_IDX 2
+#define mmOTG3_OTG_GLOBAL_SYNC_STATUS 0x1d0a
+#define mmOTG3_OTG_GLOBAL_SYNC_STATUS_BASE_IDX 2
+#define mmOTG3_OTG_MASTER_UPDATE_LOCK 0x1d0b
+#define mmOTG3_OTG_MASTER_UPDATE_LOCK_BASE_IDX 2
+#define mmOTG3_OTG_GSL_CONTROL 0x1d0c
+#define mmOTG3_OTG_GSL_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_GSL_WINDOW_X 0x1d0d
+#define mmOTG3_OTG_GSL_WINDOW_X_BASE_IDX 2
+#define mmOTG3_OTG_GSL_WINDOW_Y 0x1d0e
+#define mmOTG3_OTG_GSL_WINDOW_Y_BASE_IDX 2
+#define mmOTG3_OTG_VUPDATE_KEEPOUT 0x1d0f
+#define mmOTG3_OTG_VUPDATE_KEEPOUT_BASE_IDX 2
+#define mmOTG3_OTG_GLOBAL_CONTROL0 0x1d10
+#define mmOTG3_OTG_GLOBAL_CONTROL0_BASE_IDX 2
+#define mmOTG3_OTG_GLOBAL_CONTROL1 0x1d11
+#define mmOTG3_OTG_GLOBAL_CONTROL1_BASE_IDX 2
+#define mmOTG3_OTG_GLOBAL_CONTROL2 0x1d12
+#define mmOTG3_OTG_GLOBAL_CONTROL2_BASE_IDX 2
+#define mmOTG3_OTG_GLOBAL_CONTROL3 0x1d13
+#define mmOTG3_OTG_GLOBAL_CONTROL3_BASE_IDX 2
+#define mmOTG3_OTG_TRIG_MANUAL_CONTROL 0x1d14
+#define mmOTG3_OTG_TRIG_MANUAL_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_MANUAL_FLOW_CONTROL 0x1d15
+#define mmOTG3_OTG_MANUAL_FLOW_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_RANGE_TIMING_INT_STATUS 0x1d16
+#define mmOTG3_OTG_RANGE_TIMING_INT_STATUS_BASE_IDX 2
+#define mmOTG3_OTG_DRR_CONTROL 0x1d17
+#define mmOTG3_OTG_DRR_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_REQUEST_CONTROL 0x1d18
+#define mmOTG3_OTG_REQUEST_CONTROL_BASE_IDX 2
+#define mmOTG3_OTG_DSC_START_POSITION 0x1d19
+#define mmOTG3_OTG_DSC_START_POSITION_BASE_IDX 2
+#define mmOTG3_OTG_PIPE_UPDATE_STATUS 0x1d1a
+#define mmOTG3_OTG_PIPE_UPDATE_STATUS_BASE_IDX 2
+#define mmOTG3_OTG_SPARE_REGISTER 0x1d1c
+#define mmOTG3_OTG_SPARE_REGISTER_BASE_IDX 2
+
+
+// addressBlock: dce_dc_optc_otg4_dispdec
+// base address: 0x800
+#define mmOTG4_OTG_H_TOTAL 0x1d2a
+#define mmOTG4_OTG_H_TOTAL_BASE_IDX 2
+#define mmOTG4_OTG_H_BLANK_START_END 0x1d2b
+#define mmOTG4_OTG_H_BLANK_START_END_BASE_IDX 2
+#define mmOTG4_OTG_H_SYNC_A 0x1d2c
+#define mmOTG4_OTG_H_SYNC_A_BASE_IDX 2
+#define mmOTG4_OTG_H_SYNC_A_CNTL 0x1d2d
+#define mmOTG4_OTG_H_SYNC_A_CNTL_BASE_IDX 2
+#define mmOTG4_OTG_H_TIMING_CNTL 0x1d2e
+#define mmOTG4_OTG_H_TIMING_CNTL_BASE_IDX 2
+#define mmOTG4_OTG_V_TOTAL 0x1d2f
+#define mmOTG4_OTG_V_TOTAL_BASE_IDX 2
+#define mmOTG4_OTG_V_TOTAL_MIN 0x1d30
+#define mmOTG4_OTG_V_TOTAL_MIN_BASE_IDX 2
+#define mmOTG4_OTG_V_TOTAL_MAX 0x1d31
+#define mmOTG4_OTG_V_TOTAL_MAX_BASE_IDX 2
+#define mmOTG4_OTG_V_TOTAL_MID 0x1d32
+#define mmOTG4_OTG_V_TOTAL_MID_BASE_IDX 2
+#define mmOTG4_OTG_V_TOTAL_CONTROL 0x1d33
+#define mmOTG4_OTG_V_TOTAL_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_V_TOTAL_INT_STATUS 0x1d34
+#define mmOTG4_OTG_V_TOTAL_INT_STATUS_BASE_IDX 2
+#define mmOTG4_OTG_VSYNC_NOM_INT_STATUS 0x1d35
+#define mmOTG4_OTG_VSYNC_NOM_INT_STATUS_BASE_IDX 2
+#define mmOTG4_OTG_V_BLANK_START_END 0x1d36
+#define mmOTG4_OTG_V_BLANK_START_END_BASE_IDX 2
+#define mmOTG4_OTG_V_SYNC_A 0x1d37
+#define mmOTG4_OTG_V_SYNC_A_BASE_IDX 2
+#define mmOTG4_OTG_V_SYNC_A_CNTL 0x1d38
+#define mmOTG4_OTG_V_SYNC_A_CNTL_BASE_IDX 2
+#define mmOTG4_OTG_TRIGA_CNTL 0x1d39
+#define mmOTG4_OTG_TRIGA_CNTL_BASE_IDX 2
+#define mmOTG4_OTG_TRIGA_MANUAL_TRIG 0x1d3a
+#define mmOTG4_OTG_TRIGA_MANUAL_TRIG_BASE_IDX 2
+#define mmOTG4_OTG_TRIGB_CNTL 0x1d3b
+#define mmOTG4_OTG_TRIGB_CNTL_BASE_IDX 2
+#define mmOTG4_OTG_TRIGB_MANUAL_TRIG 0x1d3c
+#define mmOTG4_OTG_TRIGB_MANUAL_TRIG_BASE_IDX 2
+#define mmOTG4_OTG_FORCE_COUNT_NOW_CNTL 0x1d3d
+#define mmOTG4_OTG_FORCE_COUNT_NOW_CNTL_BASE_IDX 2
+#define mmOTG4_OTG_FLOW_CONTROL 0x1d3e
+#define mmOTG4_OTG_FLOW_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_STEREO_FORCE_NEXT_EYE 0x1d3f
+#define mmOTG4_OTG_STEREO_FORCE_NEXT_EYE_BASE_IDX 2
+#define mmOTG4_OTG_CONTROL 0x1d41
+#define mmOTG4_OTG_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_BLANK_CONTROL 0x1d42
+#define mmOTG4_OTG_BLANK_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_PIPE_ABORT_CONTROL 0x1d43
+#define mmOTG4_OTG_PIPE_ABORT_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_INTERLACE_CONTROL 0x1d44
+#define mmOTG4_OTG_INTERLACE_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_INTERLACE_STATUS 0x1d45
+#define mmOTG4_OTG_INTERLACE_STATUS_BASE_IDX 2
+#define mmOTG4_OTG_PIXEL_DATA_READBACK0 0x1d47
+#define mmOTG4_OTG_PIXEL_DATA_READBACK0_BASE_IDX 2
+#define mmOTG4_OTG_PIXEL_DATA_READBACK1 0x1d48
+#define mmOTG4_OTG_PIXEL_DATA_READBACK1_BASE_IDX 2
+#define mmOTG4_OTG_STATUS 0x1d49
+#define mmOTG4_OTG_STATUS_BASE_IDX 2
+#define mmOTG4_OTG_STATUS_POSITION 0x1d4a
+#define mmOTG4_OTG_STATUS_POSITION_BASE_IDX 2
+#define mmOTG4_OTG_NOM_VERT_POSITION 0x1d4b
+#define mmOTG4_OTG_NOM_VERT_POSITION_BASE_IDX 2
+#define mmOTG4_OTG_STATUS_FRAME_COUNT 0x1d4c
+#define mmOTG4_OTG_STATUS_FRAME_COUNT_BASE_IDX 2
+#define mmOTG4_OTG_STATUS_VF_COUNT 0x1d4d
+#define mmOTG4_OTG_STATUS_VF_COUNT_BASE_IDX 2
+#define mmOTG4_OTG_STATUS_HV_COUNT 0x1d4e
+#define mmOTG4_OTG_STATUS_HV_COUNT_BASE_IDX 2
+#define mmOTG4_OTG_COUNT_CONTROL 0x1d4f
+#define mmOTG4_OTG_COUNT_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_COUNT_RESET 0x1d50
+#define mmOTG4_OTG_COUNT_RESET_BASE_IDX 2
+#define mmOTG4_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE 0x1d51
+#define mmOTG4_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_BASE_IDX 2
+#define mmOTG4_OTG_VERT_SYNC_CONTROL 0x1d52
+#define mmOTG4_OTG_VERT_SYNC_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_STEREO_STATUS 0x1d53
+#define mmOTG4_OTG_STEREO_STATUS_BASE_IDX 2
+#define mmOTG4_OTG_STEREO_CONTROL 0x1d54
+#define mmOTG4_OTG_STEREO_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_SNAPSHOT_STATUS 0x1d55
+#define mmOTG4_OTG_SNAPSHOT_STATUS_BASE_IDX 2
+#define mmOTG4_OTG_SNAPSHOT_CONTROL 0x1d56
+#define mmOTG4_OTG_SNAPSHOT_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_SNAPSHOT_POSITION 0x1d57
+#define mmOTG4_OTG_SNAPSHOT_POSITION_BASE_IDX 2
+#define mmOTG4_OTG_SNAPSHOT_FRAME 0x1d58
+#define mmOTG4_OTG_SNAPSHOT_FRAME_BASE_IDX 2
+#define mmOTG4_OTG_INTERRUPT_CONTROL 0x1d59
+#define mmOTG4_OTG_INTERRUPT_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_UPDATE_LOCK 0x1d5a
+#define mmOTG4_OTG_UPDATE_LOCK_BASE_IDX 2
+#define mmOTG4_OTG_DOUBLE_BUFFER_CONTROL 0x1d5b
+#define mmOTG4_OTG_DOUBLE_BUFFER_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_MASTER_EN 0x1d5c
+#define mmOTG4_OTG_MASTER_EN_BASE_IDX 2
+#define mmOTG4_OTG_BLANK_DATA_COLOR 0x1d5e
+#define mmOTG4_OTG_BLANK_DATA_COLOR_BASE_IDX 2
+#define mmOTG4_OTG_BLANK_DATA_COLOR_EXT 0x1d5f
+#define mmOTG4_OTG_BLANK_DATA_COLOR_EXT_BASE_IDX 2
+#define mmOTG4_OTG_BLACK_COLOR 0x1d60
+#define mmOTG4_OTG_BLACK_COLOR_BASE_IDX 2
+#define mmOTG4_OTG_BLACK_COLOR_EXT 0x1d61
+#define mmOTG4_OTG_BLACK_COLOR_EXT_BASE_IDX 2
+#define mmOTG4_OTG_VERTICAL_INTERRUPT0_POSITION 0x1d62
+#define mmOTG4_OTG_VERTICAL_INTERRUPT0_POSITION_BASE_IDX 2
+#define mmOTG4_OTG_VERTICAL_INTERRUPT0_CONTROL 0x1d63
+#define mmOTG4_OTG_VERTICAL_INTERRUPT0_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_VERTICAL_INTERRUPT1_POSITION 0x1d64
+#define mmOTG4_OTG_VERTICAL_INTERRUPT1_POSITION_BASE_IDX 2
+#define mmOTG4_OTG_VERTICAL_INTERRUPT1_CONTROL 0x1d65
+#define mmOTG4_OTG_VERTICAL_INTERRUPT1_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_VERTICAL_INTERRUPT2_POSITION 0x1d66
+#define mmOTG4_OTG_VERTICAL_INTERRUPT2_POSITION_BASE_IDX 2
+#define mmOTG4_OTG_VERTICAL_INTERRUPT2_CONTROL 0x1d67
+#define mmOTG4_OTG_VERTICAL_INTERRUPT2_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_CRC_CNTL 0x1d68
+#define mmOTG4_OTG_CRC_CNTL_BASE_IDX 2
+#define mmOTG4_OTG_CRC_CNTL2 0x1d69
+#define mmOTG4_OTG_CRC_CNTL2_BASE_IDX 2
+#define mmOTG4_OTG_CRC0_WINDOWA_X_CONTROL 0x1d6a
+#define mmOTG4_OTG_CRC0_WINDOWA_X_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_CRC0_WINDOWA_Y_CONTROL 0x1d6b
+#define mmOTG4_OTG_CRC0_WINDOWA_Y_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_CRC0_WINDOWB_X_CONTROL 0x1d6c
+#define mmOTG4_OTG_CRC0_WINDOWB_X_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_CRC0_WINDOWB_Y_CONTROL 0x1d6d
+#define mmOTG4_OTG_CRC0_WINDOWB_Y_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_CRC0_DATA_RG 0x1d6e
+#define mmOTG4_OTG_CRC0_DATA_RG_BASE_IDX 2
+#define mmOTG4_OTG_CRC0_DATA_B 0x1d6f
+#define mmOTG4_OTG_CRC0_DATA_B_BASE_IDX 2
+#define mmOTG4_OTG_CRC1_WINDOWA_X_CONTROL 0x1d70
+#define mmOTG4_OTG_CRC1_WINDOWA_X_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_CRC1_WINDOWA_Y_CONTROL 0x1d71
+#define mmOTG4_OTG_CRC1_WINDOWA_Y_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_CRC1_WINDOWB_X_CONTROL 0x1d72
+#define mmOTG4_OTG_CRC1_WINDOWB_X_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_CRC1_WINDOWB_Y_CONTROL 0x1d73
+#define mmOTG4_OTG_CRC1_WINDOWB_Y_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_CRC1_DATA_RG 0x1d74
+#define mmOTG4_OTG_CRC1_DATA_RG_BASE_IDX 2
+#define mmOTG4_OTG_CRC1_DATA_B 0x1d75
+#define mmOTG4_OTG_CRC1_DATA_B_BASE_IDX 2
+#define mmOTG4_OTG_CRC2_DATA_RG 0x1d76
+#define mmOTG4_OTG_CRC2_DATA_RG_BASE_IDX 2
+#define mmOTG4_OTG_CRC2_DATA_B 0x1d77
+#define mmOTG4_OTG_CRC2_DATA_B_BASE_IDX 2
+#define mmOTG4_OTG_CRC3_DATA_RG 0x1d78
+#define mmOTG4_OTG_CRC3_DATA_RG_BASE_IDX 2
+#define mmOTG4_OTG_CRC3_DATA_B 0x1d79
+#define mmOTG4_OTG_CRC3_DATA_B_BASE_IDX 2
+#define mmOTG4_OTG_CRC_SIG_RED_GREEN_MASK 0x1d7a
+#define mmOTG4_OTG_CRC_SIG_RED_GREEN_MASK_BASE_IDX 2
+#define mmOTG4_OTG_CRC_SIG_BLUE_CONTROL_MASK 0x1d7b
+#define mmOTG4_OTG_CRC_SIG_BLUE_CONTROL_MASK_BASE_IDX 2
+#define mmOTG4_OTG_STATIC_SCREEN_CONTROL 0x1d82
+#define mmOTG4_OTG_STATIC_SCREEN_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_3D_STRUCTURE_CONTROL 0x1d83
+#define mmOTG4_OTG_3D_STRUCTURE_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_GSL_VSYNC_GAP 0x1d84
+#define mmOTG4_OTG_GSL_VSYNC_GAP_BASE_IDX 2
+#define mmOTG4_OTG_MASTER_UPDATE_MODE 0x1d85
+#define mmOTG4_OTG_MASTER_UPDATE_MODE_BASE_IDX 2
+#define mmOTG4_OTG_CLOCK_CONTROL 0x1d86
+#define mmOTG4_OTG_CLOCK_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_VSTARTUP_PARAM 0x1d87
+#define mmOTG4_OTG_VSTARTUP_PARAM_BASE_IDX 2
+#define mmOTG4_OTG_VUPDATE_PARAM 0x1d88
+#define mmOTG4_OTG_VUPDATE_PARAM_BASE_IDX 2
+#define mmOTG4_OTG_VREADY_PARAM 0x1d89
+#define mmOTG4_OTG_VREADY_PARAM_BASE_IDX 2
+#define mmOTG4_OTG_GLOBAL_SYNC_STATUS 0x1d8a
+#define mmOTG4_OTG_GLOBAL_SYNC_STATUS_BASE_IDX 2
+#define mmOTG4_OTG_MASTER_UPDATE_LOCK 0x1d8b
+#define mmOTG4_OTG_MASTER_UPDATE_LOCK_BASE_IDX 2
+#define mmOTG4_OTG_GSL_CONTROL 0x1d8c
+#define mmOTG4_OTG_GSL_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_GSL_WINDOW_X 0x1d8d
+#define mmOTG4_OTG_GSL_WINDOW_X_BASE_IDX 2
+#define mmOTG4_OTG_GSL_WINDOW_Y 0x1d8e
+#define mmOTG4_OTG_GSL_WINDOW_Y_BASE_IDX 2
+#define mmOTG4_OTG_VUPDATE_KEEPOUT 0x1d8f
+#define mmOTG4_OTG_VUPDATE_KEEPOUT_BASE_IDX 2
+#define mmOTG4_OTG_GLOBAL_CONTROL0 0x1d90
+#define mmOTG4_OTG_GLOBAL_CONTROL0_BASE_IDX 2
+#define mmOTG4_OTG_GLOBAL_CONTROL1 0x1d91
+#define mmOTG4_OTG_GLOBAL_CONTROL1_BASE_IDX 2
+#define mmOTG4_OTG_GLOBAL_CONTROL2 0x1d92
+#define mmOTG4_OTG_GLOBAL_CONTROL2_BASE_IDX 2
+#define mmOTG4_OTG_GLOBAL_CONTROL3 0x1d93
+#define mmOTG4_OTG_GLOBAL_CONTROL3_BASE_IDX 2
+#define mmOTG4_OTG_TRIG_MANUAL_CONTROL 0x1d94
+#define mmOTG4_OTG_TRIG_MANUAL_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_MANUAL_FLOW_CONTROL 0x1d95
+#define mmOTG4_OTG_MANUAL_FLOW_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_RANGE_TIMING_INT_STATUS 0x1d96
+#define mmOTG4_OTG_RANGE_TIMING_INT_STATUS_BASE_IDX 2
+#define mmOTG4_OTG_DRR_CONTROL 0x1d97
+#define mmOTG4_OTG_DRR_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_REQUEST_CONTROL 0x1d98
+#define mmOTG4_OTG_REQUEST_CONTROL_BASE_IDX 2
+#define mmOTG4_OTG_DSC_START_POSITION 0x1d99
+#define mmOTG4_OTG_DSC_START_POSITION_BASE_IDX 2
+#define mmOTG4_OTG_PIPE_UPDATE_STATUS 0x1d9a
+#define mmOTG4_OTG_PIPE_UPDATE_STATUS_BASE_IDX 2
+#define mmOTG4_OTG_SPARE_REGISTER 0x1d9c
+#define mmOTG4_OTG_SPARE_REGISTER_BASE_IDX 2
+
+
+// addressBlock: dce_dc_optc_otg5_dispdec
+// base address: 0xa00
+#define mmOTG5_OTG_H_TOTAL 0x1daa
+#define mmOTG5_OTG_H_TOTAL_BASE_IDX 2
+#define mmOTG5_OTG_H_BLANK_START_END 0x1dab
+#define mmOTG5_OTG_H_BLANK_START_END_BASE_IDX 2
+#define mmOTG5_OTG_H_SYNC_A 0x1dac
+#define mmOTG5_OTG_H_SYNC_A_BASE_IDX 2
+#define mmOTG5_OTG_H_SYNC_A_CNTL 0x1dad
+#define mmOTG5_OTG_H_SYNC_A_CNTL_BASE_IDX 2
+#define mmOTG5_OTG_H_TIMING_CNTL 0x1dae
+#define mmOTG5_OTG_H_TIMING_CNTL_BASE_IDX 2
+#define mmOTG5_OTG_V_TOTAL 0x1daf
+#define mmOTG5_OTG_V_TOTAL_BASE_IDX 2
+#define mmOTG5_OTG_V_TOTAL_MIN 0x1db0
+#define mmOTG5_OTG_V_TOTAL_MIN_BASE_IDX 2
+#define mmOTG5_OTG_V_TOTAL_MAX 0x1db1
+#define mmOTG5_OTG_V_TOTAL_MAX_BASE_IDX 2
+#define mmOTG5_OTG_V_TOTAL_MID 0x1db2
+#define mmOTG5_OTG_V_TOTAL_MID_BASE_IDX 2
+#define mmOTG5_OTG_V_TOTAL_CONTROL 0x1db3
+#define mmOTG5_OTG_V_TOTAL_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_V_TOTAL_INT_STATUS 0x1db4
+#define mmOTG5_OTG_V_TOTAL_INT_STATUS_BASE_IDX 2
+#define mmOTG5_OTG_VSYNC_NOM_INT_STATUS 0x1db5
+#define mmOTG5_OTG_VSYNC_NOM_INT_STATUS_BASE_IDX 2
+#define mmOTG5_OTG_V_BLANK_START_END 0x1db6
+#define mmOTG5_OTG_V_BLANK_START_END_BASE_IDX 2
+#define mmOTG5_OTG_V_SYNC_A 0x1db7
+#define mmOTG5_OTG_V_SYNC_A_BASE_IDX 2
+#define mmOTG5_OTG_V_SYNC_A_CNTL 0x1db8
+#define mmOTG5_OTG_V_SYNC_A_CNTL_BASE_IDX 2
+#define mmOTG5_OTG_TRIGA_CNTL 0x1db9
+#define mmOTG5_OTG_TRIGA_CNTL_BASE_IDX 2
+#define mmOTG5_OTG_TRIGA_MANUAL_TRIG 0x1dba
+#define mmOTG5_OTG_TRIGA_MANUAL_TRIG_BASE_IDX 2
+#define mmOTG5_OTG_TRIGB_CNTL 0x1dbb
+#define mmOTG5_OTG_TRIGB_CNTL_BASE_IDX 2
+#define mmOTG5_OTG_TRIGB_MANUAL_TRIG 0x1dbc
+#define mmOTG5_OTG_TRIGB_MANUAL_TRIG_BASE_IDX 2
+#define mmOTG5_OTG_FORCE_COUNT_NOW_CNTL 0x1dbd
+#define mmOTG5_OTG_FORCE_COUNT_NOW_CNTL_BASE_IDX 2
+#define mmOTG5_OTG_FLOW_CONTROL 0x1dbe
+#define mmOTG5_OTG_FLOW_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_STEREO_FORCE_NEXT_EYE 0x1dbf
+#define mmOTG5_OTG_STEREO_FORCE_NEXT_EYE_BASE_IDX 2
+#define mmOTG5_OTG_CONTROL 0x1dc1
+#define mmOTG5_OTG_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_BLANK_CONTROL 0x1dc2
+#define mmOTG5_OTG_BLANK_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_PIPE_ABORT_CONTROL 0x1dc3
+#define mmOTG5_OTG_PIPE_ABORT_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_INTERLACE_CONTROL 0x1dc4
+#define mmOTG5_OTG_INTERLACE_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_INTERLACE_STATUS 0x1dc5
+#define mmOTG5_OTG_INTERLACE_STATUS_BASE_IDX 2
+#define mmOTG5_OTG_PIXEL_DATA_READBACK0 0x1dc7
+#define mmOTG5_OTG_PIXEL_DATA_READBACK0_BASE_IDX 2
+#define mmOTG5_OTG_PIXEL_DATA_READBACK1 0x1dc8
+#define mmOTG5_OTG_PIXEL_DATA_READBACK1_BASE_IDX 2
+#define mmOTG5_OTG_STATUS 0x1dc9
+#define mmOTG5_OTG_STATUS_BASE_IDX 2
+#define mmOTG5_OTG_STATUS_POSITION 0x1dca
+#define mmOTG5_OTG_STATUS_POSITION_BASE_IDX 2
+#define mmOTG5_OTG_NOM_VERT_POSITION 0x1dcb
+#define mmOTG5_OTG_NOM_VERT_POSITION_BASE_IDX 2
+#define mmOTG5_OTG_STATUS_FRAME_COUNT 0x1dcc
+#define mmOTG5_OTG_STATUS_FRAME_COUNT_BASE_IDX 2
+#define mmOTG5_OTG_STATUS_VF_COUNT 0x1dcd
+#define mmOTG5_OTG_STATUS_VF_COUNT_BASE_IDX 2
+#define mmOTG5_OTG_STATUS_HV_COUNT 0x1dce
+#define mmOTG5_OTG_STATUS_HV_COUNT_BASE_IDX 2
+#define mmOTG5_OTG_COUNT_CONTROL 0x1dcf
+#define mmOTG5_OTG_COUNT_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_COUNT_RESET 0x1dd0
+#define mmOTG5_OTG_COUNT_RESET_BASE_IDX 2
+#define mmOTG5_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE 0x1dd1
+#define mmOTG5_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_BASE_IDX 2
+#define mmOTG5_OTG_VERT_SYNC_CONTROL 0x1dd2
+#define mmOTG5_OTG_VERT_SYNC_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_STEREO_STATUS 0x1dd3
+#define mmOTG5_OTG_STEREO_STATUS_BASE_IDX 2
+#define mmOTG5_OTG_STEREO_CONTROL 0x1dd4
+#define mmOTG5_OTG_STEREO_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_SNAPSHOT_STATUS 0x1dd5
+#define mmOTG5_OTG_SNAPSHOT_STATUS_BASE_IDX 2
+#define mmOTG5_OTG_SNAPSHOT_CONTROL 0x1dd6
+#define mmOTG5_OTG_SNAPSHOT_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_SNAPSHOT_POSITION 0x1dd7
+#define mmOTG5_OTG_SNAPSHOT_POSITION_BASE_IDX 2
+#define mmOTG5_OTG_SNAPSHOT_FRAME 0x1dd8
+#define mmOTG5_OTG_SNAPSHOT_FRAME_BASE_IDX 2
+#define mmOTG5_OTG_INTERRUPT_CONTROL 0x1dd9
+#define mmOTG5_OTG_INTERRUPT_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_UPDATE_LOCK 0x1dda
+#define mmOTG5_OTG_UPDATE_LOCK_BASE_IDX 2
+#define mmOTG5_OTG_DOUBLE_BUFFER_CONTROL 0x1ddb
+#define mmOTG5_OTG_DOUBLE_BUFFER_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_MASTER_EN 0x1ddc
+#define mmOTG5_OTG_MASTER_EN_BASE_IDX 2
+#define mmOTG5_OTG_BLANK_DATA_COLOR 0x1dde
+#define mmOTG5_OTG_BLANK_DATA_COLOR_BASE_IDX 2
+#define mmOTG5_OTG_BLANK_DATA_COLOR_EXT 0x1ddf
+#define mmOTG5_OTG_BLANK_DATA_COLOR_EXT_BASE_IDX 2
+#define mmOTG5_OTG_BLACK_COLOR 0x1de0
+#define mmOTG5_OTG_BLACK_COLOR_BASE_IDX 2
+#define mmOTG5_OTG_BLACK_COLOR_EXT 0x1de1
+#define mmOTG5_OTG_BLACK_COLOR_EXT_BASE_IDX 2
+#define mmOTG5_OTG_VERTICAL_INTERRUPT0_POSITION 0x1de2
+#define mmOTG5_OTG_VERTICAL_INTERRUPT0_POSITION_BASE_IDX 2
+#define mmOTG5_OTG_VERTICAL_INTERRUPT0_CONTROL 0x1de3
+#define mmOTG5_OTG_VERTICAL_INTERRUPT0_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_VERTICAL_INTERRUPT1_POSITION 0x1de4
+#define mmOTG5_OTG_VERTICAL_INTERRUPT1_POSITION_BASE_IDX 2
+#define mmOTG5_OTG_VERTICAL_INTERRUPT1_CONTROL 0x1de5
+#define mmOTG5_OTG_VERTICAL_INTERRUPT1_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_VERTICAL_INTERRUPT2_POSITION 0x1de6
+#define mmOTG5_OTG_VERTICAL_INTERRUPT2_POSITION_BASE_IDX 2
+#define mmOTG5_OTG_VERTICAL_INTERRUPT2_CONTROL 0x1de7
+#define mmOTG5_OTG_VERTICAL_INTERRUPT2_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_CRC_CNTL 0x1de8
+#define mmOTG5_OTG_CRC_CNTL_BASE_IDX 2
+#define mmOTG5_OTG_CRC_CNTL2 0x1de9
+#define mmOTG5_OTG_CRC_CNTL2_BASE_IDX 2
+#define mmOTG5_OTG_CRC0_WINDOWA_X_CONTROL 0x1dea
+#define mmOTG5_OTG_CRC0_WINDOWA_X_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_CRC0_WINDOWA_Y_CONTROL 0x1deb
+#define mmOTG5_OTG_CRC0_WINDOWA_Y_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_CRC0_WINDOWB_X_CONTROL 0x1dec
+#define mmOTG5_OTG_CRC0_WINDOWB_X_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_CRC0_WINDOWB_Y_CONTROL 0x1ded
+#define mmOTG5_OTG_CRC0_WINDOWB_Y_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_CRC0_DATA_RG 0x1dee
+#define mmOTG5_OTG_CRC0_DATA_RG_BASE_IDX 2
+#define mmOTG5_OTG_CRC0_DATA_B 0x1def
+#define mmOTG5_OTG_CRC0_DATA_B_BASE_IDX 2
+#define mmOTG5_OTG_CRC1_WINDOWA_X_CONTROL 0x1df0
+#define mmOTG5_OTG_CRC1_WINDOWA_X_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_CRC1_WINDOWA_Y_CONTROL 0x1df1
+#define mmOTG5_OTG_CRC1_WINDOWA_Y_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_CRC1_WINDOWB_X_CONTROL 0x1df2
+#define mmOTG5_OTG_CRC1_WINDOWB_X_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_CRC1_WINDOWB_Y_CONTROL 0x1df3
+#define mmOTG5_OTG_CRC1_WINDOWB_Y_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_CRC1_DATA_RG 0x1df4
+#define mmOTG5_OTG_CRC1_DATA_RG_BASE_IDX 2
+#define mmOTG5_OTG_CRC1_DATA_B 0x1df5
+#define mmOTG5_OTG_CRC1_DATA_B_BASE_IDX 2
+#define mmOTG5_OTG_CRC2_DATA_RG 0x1df6
+#define mmOTG5_OTG_CRC2_DATA_RG_BASE_IDX 2
+#define mmOTG5_OTG_CRC2_DATA_B 0x1df7
+#define mmOTG5_OTG_CRC2_DATA_B_BASE_IDX 2
+#define mmOTG5_OTG_CRC3_DATA_RG 0x1df8
+#define mmOTG5_OTG_CRC3_DATA_RG_BASE_IDX 2
+#define mmOTG5_OTG_CRC3_DATA_B 0x1df9
+#define mmOTG5_OTG_CRC3_DATA_B_BASE_IDX 2
+#define mmOTG5_OTG_CRC_SIG_RED_GREEN_MASK 0x1dfa
+#define mmOTG5_OTG_CRC_SIG_RED_GREEN_MASK_BASE_IDX 2
+#define mmOTG5_OTG_CRC_SIG_BLUE_CONTROL_MASK 0x1dfb
+#define mmOTG5_OTG_CRC_SIG_BLUE_CONTROL_MASK_BASE_IDX 2
+#define mmOTG5_OTG_STATIC_SCREEN_CONTROL 0x1e02
+#define mmOTG5_OTG_STATIC_SCREEN_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_3D_STRUCTURE_CONTROL 0x1e03
+#define mmOTG5_OTG_3D_STRUCTURE_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_GSL_VSYNC_GAP 0x1e04
+#define mmOTG5_OTG_GSL_VSYNC_GAP_BASE_IDX 2
+#define mmOTG5_OTG_MASTER_UPDATE_MODE 0x1e05
+#define mmOTG5_OTG_MASTER_UPDATE_MODE_BASE_IDX 2
+#define mmOTG5_OTG_CLOCK_CONTROL 0x1e06
+#define mmOTG5_OTG_CLOCK_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_VSTARTUP_PARAM 0x1e07
+#define mmOTG5_OTG_VSTARTUP_PARAM_BASE_IDX 2
+#define mmOTG5_OTG_VUPDATE_PARAM 0x1e08
+#define mmOTG5_OTG_VUPDATE_PARAM_BASE_IDX 2
+#define mmOTG5_OTG_VREADY_PARAM 0x1e09
+#define mmOTG5_OTG_VREADY_PARAM_BASE_IDX 2
+#define mmOTG5_OTG_GLOBAL_SYNC_STATUS 0x1e0a
+#define mmOTG5_OTG_GLOBAL_SYNC_STATUS_BASE_IDX 2
+#define mmOTG5_OTG_MASTER_UPDATE_LOCK 0x1e0b
+#define mmOTG5_OTG_MASTER_UPDATE_LOCK_BASE_IDX 2
+#define mmOTG5_OTG_GSL_CONTROL 0x1e0c
+#define mmOTG5_OTG_GSL_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_GSL_WINDOW_X 0x1e0d
+#define mmOTG5_OTG_GSL_WINDOW_X_BASE_IDX 2
+#define mmOTG5_OTG_GSL_WINDOW_Y 0x1e0e
+#define mmOTG5_OTG_GSL_WINDOW_Y_BASE_IDX 2
+#define mmOTG5_OTG_VUPDATE_KEEPOUT 0x1e0f
+#define mmOTG5_OTG_VUPDATE_KEEPOUT_BASE_IDX 2
+#define mmOTG5_OTG_GLOBAL_CONTROL0 0x1e10
+#define mmOTG5_OTG_GLOBAL_CONTROL0_BASE_IDX 2
+#define mmOTG5_OTG_GLOBAL_CONTROL1 0x1e11
+#define mmOTG5_OTG_GLOBAL_CONTROL1_BASE_IDX 2
+#define mmOTG5_OTG_GLOBAL_CONTROL2 0x1e12
+#define mmOTG5_OTG_GLOBAL_CONTROL2_BASE_IDX 2
+#define mmOTG5_OTG_GLOBAL_CONTROL3 0x1e13
+#define mmOTG5_OTG_GLOBAL_CONTROL3_BASE_IDX 2
+#define mmOTG5_OTG_TRIG_MANUAL_CONTROL 0x1e14
+#define mmOTG5_OTG_TRIG_MANUAL_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_MANUAL_FLOW_CONTROL 0x1e15
+#define mmOTG5_OTG_MANUAL_FLOW_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_RANGE_TIMING_INT_STATUS 0x1e16
+#define mmOTG5_OTG_RANGE_TIMING_INT_STATUS_BASE_IDX 2
+#define mmOTG5_OTG_DRR_CONTROL 0x1e17
+#define mmOTG5_OTG_DRR_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_REQUEST_CONTROL 0x1e18
+#define mmOTG5_OTG_REQUEST_CONTROL_BASE_IDX 2
+#define mmOTG5_OTG_DSC_START_POSITION 0x1e19
+#define mmOTG5_OTG_DSC_START_POSITION_BASE_IDX 2
+#define mmOTG5_OTG_PIPE_UPDATE_STATUS 0x1e1a
+#define mmOTG5_OTG_PIPE_UPDATE_STATUS_BASE_IDX 2
+#define mmOTG5_OTG_SPARE_REGISTER 0x1e1c
+#define mmOTG5_OTG_SPARE_REGISTER_BASE_IDX 2
+
+
+// addressBlock: dce_dc_optc_optc_misc_dispdec
+// base address: 0x0
+#define mmDWB_SOURCE_SELECT 0x1e2a
+#define mmDWB_SOURCE_SELECT_BASE_IDX 2
+#define mmGSL_SOURCE_SELECT 0x1e2b
+#define mmGSL_SOURCE_SELECT_BASE_IDX 2
+#define mmOPTC_CLOCK_CONTROL 0x1e2c
+#define mmOPTC_CLOCK_CONTROL_BASE_IDX 2
+#define mmODM_MEM_PWR_CTRL 0x1e2d
+#define mmODM_MEM_PWR_CTRL_BASE_IDX 2
+#define mmODM_MEM_PWR_CTRL2 0x1e2e
+#define mmODM_MEM_PWR_CTRL2_BASE_IDX 2
+#define mmODM_MEM_PWR_CTRL3 0x1e2f
+#define mmODM_MEM_PWR_CTRL3_BASE_IDX 2
+#define mmODM_MEM_PWR_STATUS 0x1e30
+#define mmODM_MEM_PWR_STATUS_BASE_IDX 2
+#define mmOPTC_MISC_SPARE_REGISTER 0x1e31
+#define mmOPTC_MISC_SPARE_REGISTER_BASE_IDX 2
+
+
+// addressBlock: dce_dc_optc_optc_dcperfmon_dc_perfmon_dispdec
+// base address: 0x79a8
+#define mmDC_PERFMON19_PERFCOUNTER_CNTL 0x1e6a
+#define mmDC_PERFMON19_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON19_PERFCOUNTER_CNTL2 0x1e6b
+#define mmDC_PERFMON19_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON19_PERFCOUNTER_STATE 0x1e6c
+#define mmDC_PERFMON19_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON19_PERFMON_CNTL 0x1e6d
+#define mmDC_PERFMON19_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON19_PERFMON_CNTL2 0x1e6e
+#define mmDC_PERFMON19_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON19_PERFMON_CVALUE_INT_MISC 0x1e6f
+#define mmDC_PERFMON19_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON19_PERFMON_CVALUE_LOW 0x1e70
+#define mmDC_PERFMON19_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON19_PERFMON_HI 0x1e71
+#define mmDC_PERFMON19_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON19_PERFMON_LOW 0x1e72
+#define mmDC_PERFMON19_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dout_i2c_dispdec
+// base address: 0x0
+#define mmDC_I2C_CONTROL 0x1e98
+#define mmDC_I2C_CONTROL_BASE_IDX 2
+#define mmDC_I2C_ARBITRATION 0x1e99
+#define mmDC_I2C_ARBITRATION_BASE_IDX 2
+#define mmDC_I2C_INTERRUPT_CONTROL 0x1e9a
+#define mmDC_I2C_INTERRUPT_CONTROL_BASE_IDX 2
+#define mmDC_I2C_SW_STATUS 0x1e9b
+#define mmDC_I2C_SW_STATUS_BASE_IDX 2
+#define mmDC_I2C_DDC1_HW_STATUS 0x1e9c
+#define mmDC_I2C_DDC1_HW_STATUS_BASE_IDX 2
+#define mmDC_I2C_DDC2_HW_STATUS 0x1e9d
+#define mmDC_I2C_DDC2_HW_STATUS_BASE_IDX 2
+#define mmDC_I2C_DDC3_HW_STATUS 0x1e9e
+#define mmDC_I2C_DDC3_HW_STATUS_BASE_IDX 2
+#define mmDC_I2C_DDC4_HW_STATUS 0x1e9f
+#define mmDC_I2C_DDC4_HW_STATUS_BASE_IDX 2
+#define mmDC_I2C_DDC5_HW_STATUS 0x1ea0
+#define mmDC_I2C_DDC5_HW_STATUS_BASE_IDX 2
+#define mmDC_I2C_DDC6_HW_STATUS 0x1ea1
+#define mmDC_I2C_DDC6_HW_STATUS_BASE_IDX 2
+#define mmDC_I2C_DDC1_SPEED 0x1ea2
+#define mmDC_I2C_DDC1_SPEED_BASE_IDX 2
+#define mmDC_I2C_DDC1_SETUP 0x1ea3
+#define mmDC_I2C_DDC1_SETUP_BASE_IDX 2
+#define mmDC_I2C_DDC2_SPEED 0x1ea4
+#define mmDC_I2C_DDC2_SPEED_BASE_IDX 2
+#define mmDC_I2C_DDC2_SETUP 0x1ea5
+#define mmDC_I2C_DDC2_SETUP_BASE_IDX 2
+#define mmDC_I2C_DDC3_SPEED 0x1ea6
+#define mmDC_I2C_DDC3_SPEED_BASE_IDX 2
+#define mmDC_I2C_DDC3_SETUP 0x1ea7
+#define mmDC_I2C_DDC3_SETUP_BASE_IDX 2
+#define mmDC_I2C_DDC4_SPEED 0x1ea8
+#define mmDC_I2C_DDC4_SPEED_BASE_IDX 2
+#define mmDC_I2C_DDC4_SETUP 0x1ea9
+#define mmDC_I2C_DDC4_SETUP_BASE_IDX 2
+#define mmDC_I2C_DDC5_SPEED 0x1eaa
+#define mmDC_I2C_DDC5_SPEED_BASE_IDX 2
+#define mmDC_I2C_DDC5_SETUP 0x1eab
+#define mmDC_I2C_DDC5_SETUP_BASE_IDX 2
+#define mmDC_I2C_DDC6_SPEED 0x1eac
+#define mmDC_I2C_DDC6_SPEED_BASE_IDX 2
+#define mmDC_I2C_DDC6_SETUP 0x1ead
+#define mmDC_I2C_DDC6_SETUP_BASE_IDX 2
+#define mmDC_I2C_TRANSACTION0 0x1eae
+#define mmDC_I2C_TRANSACTION0_BASE_IDX 2
+#define mmDC_I2C_TRANSACTION1 0x1eaf
+#define mmDC_I2C_TRANSACTION1_BASE_IDX 2
+#define mmDC_I2C_TRANSACTION2 0x1eb0
+#define mmDC_I2C_TRANSACTION2_BASE_IDX 2
+#define mmDC_I2C_TRANSACTION3 0x1eb1
+#define mmDC_I2C_TRANSACTION3_BASE_IDX 2
+#define mmDC_I2C_DATA 0x1eb2
+#define mmDC_I2C_DATA_BASE_IDX 2
+#define mmDC_I2C_DDCVGA_SETUP 0x1eb5
+#define mmDC_I2C_DDCVGA_SETUP_BASE_IDX 2
+#define mmDC_I2C_EDID_DETECT_CTRL 0x1eb6
+#define mmDC_I2C_EDID_DETECT_CTRL_BASE_IDX 2
+#define mmDC_I2C_READ_REQUEST_INTERRUPT 0x1eb7
+#define mmDC_I2C_READ_REQUEST_INTERRUPT_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dio_misc_dispdec
+// base address: 0x0
+#define mmDIO_SCRATCH0 0x1eca
+#define mmDIO_SCRATCH0_BASE_IDX 2
+#define mmDIO_SCRATCH1 0x1ecb
+#define mmDIO_SCRATCH1_BASE_IDX 2
+#define mmDIO_SCRATCH2 0x1ecc
+#define mmDIO_SCRATCH2_BASE_IDX 2
+#define mmDIO_SCRATCH3 0x1ecd
+#define mmDIO_SCRATCH3_BASE_IDX 2
+#define mmDIO_SCRATCH4 0x1ece
+#define mmDIO_SCRATCH4_BASE_IDX 2
+#define mmDIO_SCRATCH5 0x1ecf
+#define mmDIO_SCRATCH5_BASE_IDX 2
+#define mmDIO_SCRATCH6 0x1ed0
+#define mmDIO_SCRATCH6_BASE_IDX 2
+#define mmDIO_SCRATCH7 0x1ed1
+#define mmDIO_SCRATCH7_BASE_IDX 2
+#define mmDCE_VCE_CONTROL 0x1ed2
+#define mmDCE_VCE_CONTROL_BASE_IDX 2
+#define mmDIO_MEM_PWR_STATUS 0x1edd
+#define mmDIO_MEM_PWR_STATUS_BASE_IDX 2
+#define mmDIO_MEM_PWR_CTRL 0x1ede
+#define mmDIO_MEM_PWR_CTRL_BASE_IDX 2
+#define mmDIO_MEM_PWR_CTRL2 0x1edf
+#define mmDIO_MEM_PWR_CTRL2_BASE_IDX 2
+#define mmDIO_CLK_CNTL 0x1ee0
+#define mmDIO_CLK_CNTL_BASE_IDX 2
+#define mmDIO_MEM_PWR_CTRL3 0x1ee1
+#define mmDIO_MEM_PWR_CTRL3_BASE_IDX 2
+#define mmDIO_POWER_MANAGEMENT_CNTL 0x1ee4
+#define mmDIO_POWER_MANAGEMENT_CNTL_BASE_IDX 2
+#define mmDIG_SOFT_RESET 0x1eee
+#define mmDIG_SOFT_RESET_BASE_IDX 2
+#define mmDIO_MEM_PWR_STATUS1 0x1ef0
+#define mmDIO_MEM_PWR_STATUS1_BASE_IDX 2
+#define mmDIO_CLK_CNTL2 0x1ef2
+#define mmDIO_CLK_CNTL2_BASE_IDX 2
+#define mmDIO_CLK_CNTL3 0x1ef3
+#define mmDIO_CLK_CNTL3_BASE_IDX 2
+#define mmDIO_HDMI_RXSTATUS_TIMER_CONTROL 0x1eff
+#define mmDIO_HDMI_RXSTATUS_TIMER_CONTROL_BASE_IDX 2
+#define mmDIO_PSP_INTERRUPT_STATUS 0x1f00
+#define mmDIO_PSP_INTERRUPT_STATUS_BASE_IDX 2
+#define mmDIO_PSP_INTERRUPT_CLEAR 0x1f01
+#define mmDIO_PSP_INTERRUPT_CLEAR_BASE_IDX 2
+#define mmDIO_GENERIC_INTERRUPT_MESSAGE 0x1f02
+#define mmDIO_GENERIC_INTERRUPT_MESSAGE_BASE_IDX 2
+#define mmDIO_GENERIC_INTERRUPT_CLEAR 0x1f03
+#define mmDIO_GENERIC_INTERRUPT_CLEAR_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_hpd0_dispdec
+// base address: 0x0
+#define mmHPD0_DC_HPD_INT_STATUS 0x1f14
+#define mmHPD0_DC_HPD_INT_STATUS_BASE_IDX 2
+#define mmHPD0_DC_HPD_INT_CONTROL 0x1f15
+#define mmHPD0_DC_HPD_INT_CONTROL_BASE_IDX 2
+#define mmHPD0_DC_HPD_CONTROL 0x1f16
+#define mmHPD0_DC_HPD_CONTROL_BASE_IDX 2
+#define mmHPD0_DC_HPD_FAST_TRAIN_CNTL 0x1f17
+#define mmHPD0_DC_HPD_FAST_TRAIN_CNTL_BASE_IDX 2
+#define mmHPD0_DC_HPD_TOGGLE_FILT_CNTL 0x1f18
+#define mmHPD0_DC_HPD_TOGGLE_FILT_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_hpd1_dispdec
+// base address: 0x20
+#define mmHPD1_DC_HPD_INT_STATUS 0x1f1c
+#define mmHPD1_DC_HPD_INT_STATUS_BASE_IDX 2
+#define mmHPD1_DC_HPD_INT_CONTROL 0x1f1d
+#define mmHPD1_DC_HPD_INT_CONTROL_BASE_IDX 2
+#define mmHPD1_DC_HPD_CONTROL 0x1f1e
+#define mmHPD1_DC_HPD_CONTROL_BASE_IDX 2
+#define mmHPD1_DC_HPD_FAST_TRAIN_CNTL 0x1f1f
+#define mmHPD1_DC_HPD_FAST_TRAIN_CNTL_BASE_IDX 2
+#define mmHPD1_DC_HPD_TOGGLE_FILT_CNTL 0x1f20
+#define mmHPD1_DC_HPD_TOGGLE_FILT_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_hpd2_dispdec
+// base address: 0x40
+#define mmHPD2_DC_HPD_INT_STATUS 0x1f24
+#define mmHPD2_DC_HPD_INT_STATUS_BASE_IDX 2
+#define mmHPD2_DC_HPD_INT_CONTROL 0x1f25
+#define mmHPD2_DC_HPD_INT_CONTROL_BASE_IDX 2
+#define mmHPD2_DC_HPD_CONTROL 0x1f26
+#define mmHPD2_DC_HPD_CONTROL_BASE_IDX 2
+#define mmHPD2_DC_HPD_FAST_TRAIN_CNTL 0x1f27
+#define mmHPD2_DC_HPD_FAST_TRAIN_CNTL_BASE_IDX 2
+#define mmHPD2_DC_HPD_TOGGLE_FILT_CNTL 0x1f28
+#define mmHPD2_DC_HPD_TOGGLE_FILT_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_hpd3_dispdec
+// base address: 0x60
+#define mmHPD3_DC_HPD_INT_STATUS 0x1f2c
+#define mmHPD3_DC_HPD_INT_STATUS_BASE_IDX 2
+#define mmHPD3_DC_HPD_INT_CONTROL 0x1f2d
+#define mmHPD3_DC_HPD_INT_CONTROL_BASE_IDX 2
+#define mmHPD3_DC_HPD_CONTROL 0x1f2e
+#define mmHPD3_DC_HPD_CONTROL_BASE_IDX 2
+#define mmHPD3_DC_HPD_FAST_TRAIN_CNTL 0x1f2f
+#define mmHPD3_DC_HPD_FAST_TRAIN_CNTL_BASE_IDX 2
+#define mmHPD3_DC_HPD_TOGGLE_FILT_CNTL 0x1f30
+#define mmHPD3_DC_HPD_TOGGLE_FILT_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_hpd4_dispdec
+// base address: 0x80
+#define mmHPD4_DC_HPD_INT_STATUS 0x1f34
+#define mmHPD4_DC_HPD_INT_STATUS_BASE_IDX 2
+#define mmHPD4_DC_HPD_INT_CONTROL 0x1f35
+#define mmHPD4_DC_HPD_INT_CONTROL_BASE_IDX 2
+#define mmHPD4_DC_HPD_CONTROL 0x1f36
+#define mmHPD4_DC_HPD_CONTROL_BASE_IDX 2
+#define mmHPD4_DC_HPD_FAST_TRAIN_CNTL 0x1f37
+#define mmHPD4_DC_HPD_FAST_TRAIN_CNTL_BASE_IDX 2
+#define mmHPD4_DC_HPD_TOGGLE_FILT_CNTL 0x1f38
+#define mmHPD4_DC_HPD_TOGGLE_FILT_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_hpd5_dispdec
+// base address: 0xa0
+#define mmHPD5_DC_HPD_INT_STATUS 0x1f3c
+#define mmHPD5_DC_HPD_INT_STATUS_BASE_IDX 2
+#define mmHPD5_DC_HPD_INT_CONTROL 0x1f3d
+#define mmHPD5_DC_HPD_INT_CONTROL_BASE_IDX 2
+#define mmHPD5_DC_HPD_CONTROL 0x1f3e
+#define mmHPD5_DC_HPD_CONTROL_BASE_IDX 2
+#define mmHPD5_DC_HPD_FAST_TRAIN_CNTL 0x1f3f
+#define mmHPD5_DC_HPD_FAST_TRAIN_CNTL_BASE_IDX 2
+#define mmHPD5_DC_HPD_TOGGLE_FILT_CNTL 0x1f40
+#define mmHPD5_DC_HPD_TOGGLE_FILT_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dio_dcperfmon_dc_perfmon_dispdec
+// base address: 0x7d10
+#define mmDC_PERFMON20_PERFCOUNTER_CNTL 0x1f44
+#define mmDC_PERFMON20_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON20_PERFCOUNTER_CNTL2 0x1f45
+#define mmDC_PERFMON20_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON20_PERFCOUNTER_STATE 0x1f46
+#define mmDC_PERFMON20_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON20_PERFMON_CNTL 0x1f47
+#define mmDC_PERFMON20_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON20_PERFMON_CNTL2 0x1f48
+#define mmDC_PERFMON20_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON20_PERFMON_CVALUE_INT_MISC 0x1f49
+#define mmDC_PERFMON20_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON20_PERFMON_CVALUE_LOW 0x1f4a
+#define mmDC_PERFMON20_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON20_PERFMON_HI 0x1f4b
+#define mmDC_PERFMON20_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON20_PERFMON_LOW 0x1f4c
+#define mmDC_PERFMON20_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dp_aux0_dispdec
+// base address: 0x0
+#define mmDP_AUX0_AUX_CONTROL 0x1f50
+#define mmDP_AUX0_AUX_CONTROL_BASE_IDX 2
+#define mmDP_AUX0_AUX_SW_CONTROL 0x1f51
+#define mmDP_AUX0_AUX_SW_CONTROL_BASE_IDX 2
+#define mmDP_AUX0_AUX_ARB_CONTROL 0x1f52
+#define mmDP_AUX0_AUX_ARB_CONTROL_BASE_IDX 2
+#define mmDP_AUX0_AUX_INTERRUPT_CONTROL 0x1f53
+#define mmDP_AUX0_AUX_INTERRUPT_CONTROL_BASE_IDX 2
+#define mmDP_AUX0_AUX_SW_STATUS 0x1f54
+#define mmDP_AUX0_AUX_SW_STATUS_BASE_IDX 2
+#define mmDP_AUX0_AUX_LS_STATUS 0x1f55
+#define mmDP_AUX0_AUX_LS_STATUS_BASE_IDX 2
+#define mmDP_AUX0_AUX_SW_DATA 0x1f56
+#define mmDP_AUX0_AUX_SW_DATA_BASE_IDX 2
+#define mmDP_AUX0_AUX_LS_DATA 0x1f57
+#define mmDP_AUX0_AUX_LS_DATA_BASE_IDX 2
+#define mmDP_AUX0_AUX_DPHY_TX_REF_CONTROL 0x1f58
+#define mmDP_AUX0_AUX_DPHY_TX_REF_CONTROL_BASE_IDX 2
+#define mmDP_AUX0_AUX_DPHY_TX_CONTROL 0x1f59
+#define mmDP_AUX0_AUX_DPHY_TX_CONTROL_BASE_IDX 2
+#define mmDP_AUX0_AUX_DPHY_RX_CONTROL0 0x1f5a
+#define mmDP_AUX0_AUX_DPHY_RX_CONTROL0_BASE_IDX 2
+#define mmDP_AUX0_AUX_DPHY_RX_CONTROL1 0x1f5b
+#define mmDP_AUX0_AUX_DPHY_RX_CONTROL1_BASE_IDX 2
+#define mmDP_AUX0_AUX_DPHY_TX_STATUS 0x1f5c
+#define mmDP_AUX0_AUX_DPHY_TX_STATUS_BASE_IDX 2
+#define mmDP_AUX0_AUX_DPHY_RX_STATUS 0x1f5d
+#define mmDP_AUX0_AUX_DPHY_RX_STATUS_BASE_IDX 2
+#define mmDP_AUX0_AUX_GTC_SYNC_CONTROL 0x1f5e
+#define mmDP_AUX0_AUX_GTC_SYNC_CONTROL_BASE_IDX 2
+#define mmDP_AUX0_AUX_GTC_SYNC_ERROR_CONTROL 0x1f5f
+#define mmDP_AUX0_AUX_GTC_SYNC_ERROR_CONTROL_BASE_IDX 2
+#define mmDP_AUX0_AUX_GTC_SYNC_CONTROLLER_STATUS 0x1f60
+#define mmDP_AUX0_AUX_GTC_SYNC_CONTROLLER_STATUS_BASE_IDX 2
+#define mmDP_AUX0_AUX_GTC_SYNC_STATUS 0x1f61
+#define mmDP_AUX0_AUX_GTC_SYNC_STATUS_BASE_IDX 2
+#define mmDP_AUX0_AUX_PHY_WAKE_CNTL 0x1f66
+#define mmDP_AUX0_AUX_PHY_WAKE_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dp_aux1_dispdec
+// base address: 0x70
+#define mmDP_AUX1_AUX_CONTROL 0x1f6c
+#define mmDP_AUX1_AUX_CONTROL_BASE_IDX 2
+#define mmDP_AUX1_AUX_SW_CONTROL 0x1f6d
+#define mmDP_AUX1_AUX_SW_CONTROL_BASE_IDX 2
+#define mmDP_AUX1_AUX_ARB_CONTROL 0x1f6e
+#define mmDP_AUX1_AUX_ARB_CONTROL_BASE_IDX 2
+#define mmDP_AUX1_AUX_INTERRUPT_CONTROL 0x1f6f
+#define mmDP_AUX1_AUX_INTERRUPT_CONTROL_BASE_IDX 2
+#define mmDP_AUX1_AUX_SW_STATUS 0x1f70
+#define mmDP_AUX1_AUX_SW_STATUS_BASE_IDX 2
+#define mmDP_AUX1_AUX_LS_STATUS 0x1f71
+#define mmDP_AUX1_AUX_LS_STATUS_BASE_IDX 2
+#define mmDP_AUX1_AUX_SW_DATA 0x1f72
+#define mmDP_AUX1_AUX_SW_DATA_BASE_IDX 2
+#define mmDP_AUX1_AUX_LS_DATA 0x1f73
+#define mmDP_AUX1_AUX_LS_DATA_BASE_IDX 2
+#define mmDP_AUX1_AUX_DPHY_TX_REF_CONTROL 0x1f74
+#define mmDP_AUX1_AUX_DPHY_TX_REF_CONTROL_BASE_IDX 2
+#define mmDP_AUX1_AUX_DPHY_TX_CONTROL 0x1f75
+#define mmDP_AUX1_AUX_DPHY_TX_CONTROL_BASE_IDX 2
+#define mmDP_AUX1_AUX_DPHY_RX_CONTROL0 0x1f76
+#define mmDP_AUX1_AUX_DPHY_RX_CONTROL0_BASE_IDX 2
+#define mmDP_AUX1_AUX_DPHY_RX_CONTROL1 0x1f77
+#define mmDP_AUX1_AUX_DPHY_RX_CONTROL1_BASE_IDX 2
+#define mmDP_AUX1_AUX_DPHY_TX_STATUS 0x1f78
+#define mmDP_AUX1_AUX_DPHY_TX_STATUS_BASE_IDX 2
+#define mmDP_AUX1_AUX_DPHY_RX_STATUS 0x1f79
+#define mmDP_AUX1_AUX_DPHY_RX_STATUS_BASE_IDX 2
+#define mmDP_AUX1_AUX_GTC_SYNC_CONTROL 0x1f7a
+#define mmDP_AUX1_AUX_GTC_SYNC_CONTROL_BASE_IDX 2
+#define mmDP_AUX1_AUX_GTC_SYNC_ERROR_CONTROL 0x1f7b
+#define mmDP_AUX1_AUX_GTC_SYNC_ERROR_CONTROL_BASE_IDX 2
+#define mmDP_AUX1_AUX_GTC_SYNC_CONTROLLER_STATUS 0x1f7c
+#define mmDP_AUX1_AUX_GTC_SYNC_CONTROLLER_STATUS_BASE_IDX 2
+#define mmDP_AUX1_AUX_GTC_SYNC_STATUS 0x1f7d
+#define mmDP_AUX1_AUX_GTC_SYNC_STATUS_BASE_IDX 2
+#define mmDP_AUX1_AUX_PHY_WAKE_CNTL 0x1f82
+#define mmDP_AUX1_AUX_PHY_WAKE_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dp_aux2_dispdec
+// base address: 0xe0
+#define mmDP_AUX2_AUX_CONTROL 0x1f88
+#define mmDP_AUX2_AUX_CONTROL_BASE_IDX 2
+#define mmDP_AUX2_AUX_SW_CONTROL 0x1f89
+#define mmDP_AUX2_AUX_SW_CONTROL_BASE_IDX 2
+#define mmDP_AUX2_AUX_ARB_CONTROL 0x1f8a
+#define mmDP_AUX2_AUX_ARB_CONTROL_BASE_IDX 2
+#define mmDP_AUX2_AUX_INTERRUPT_CONTROL 0x1f8b
+#define mmDP_AUX2_AUX_INTERRUPT_CONTROL_BASE_IDX 2
+#define mmDP_AUX2_AUX_SW_STATUS 0x1f8c
+#define mmDP_AUX2_AUX_SW_STATUS_BASE_IDX 2
+#define mmDP_AUX2_AUX_LS_STATUS 0x1f8d
+#define mmDP_AUX2_AUX_LS_STATUS_BASE_IDX 2
+#define mmDP_AUX2_AUX_SW_DATA 0x1f8e
+#define mmDP_AUX2_AUX_SW_DATA_BASE_IDX 2
+#define mmDP_AUX2_AUX_LS_DATA 0x1f8f
+#define mmDP_AUX2_AUX_LS_DATA_BASE_IDX 2
+#define mmDP_AUX2_AUX_DPHY_TX_REF_CONTROL 0x1f90
+#define mmDP_AUX2_AUX_DPHY_TX_REF_CONTROL_BASE_IDX 2
+#define mmDP_AUX2_AUX_DPHY_TX_CONTROL 0x1f91
+#define mmDP_AUX2_AUX_DPHY_TX_CONTROL_BASE_IDX 2
+#define mmDP_AUX2_AUX_DPHY_RX_CONTROL0 0x1f92
+#define mmDP_AUX2_AUX_DPHY_RX_CONTROL0_BASE_IDX 2
+#define mmDP_AUX2_AUX_DPHY_RX_CONTROL1 0x1f93
+#define mmDP_AUX2_AUX_DPHY_RX_CONTROL1_BASE_IDX 2
+#define mmDP_AUX2_AUX_DPHY_TX_STATUS 0x1f94
+#define mmDP_AUX2_AUX_DPHY_TX_STATUS_BASE_IDX 2
+#define mmDP_AUX2_AUX_DPHY_RX_STATUS 0x1f95
+#define mmDP_AUX2_AUX_DPHY_RX_STATUS_BASE_IDX 2
+#define mmDP_AUX2_AUX_GTC_SYNC_CONTROL 0x1f96
+#define mmDP_AUX2_AUX_GTC_SYNC_CONTROL_BASE_IDX 2
+#define mmDP_AUX2_AUX_GTC_SYNC_ERROR_CONTROL 0x1f97
+#define mmDP_AUX2_AUX_GTC_SYNC_ERROR_CONTROL_BASE_IDX 2
+#define mmDP_AUX2_AUX_GTC_SYNC_CONTROLLER_STATUS 0x1f98
+#define mmDP_AUX2_AUX_GTC_SYNC_CONTROLLER_STATUS_BASE_IDX 2
+#define mmDP_AUX2_AUX_GTC_SYNC_STATUS 0x1f99
+#define mmDP_AUX2_AUX_GTC_SYNC_STATUS_BASE_IDX 2
+#define mmDP_AUX2_AUX_PHY_WAKE_CNTL 0x1f9e
+#define mmDP_AUX2_AUX_PHY_WAKE_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dp_aux3_dispdec
+// base address: 0x150
+#define mmDP_AUX3_AUX_CONTROL 0x1fa4
+#define mmDP_AUX3_AUX_CONTROL_BASE_IDX 2
+#define mmDP_AUX3_AUX_SW_CONTROL 0x1fa5
+#define mmDP_AUX3_AUX_SW_CONTROL_BASE_IDX 2
+#define mmDP_AUX3_AUX_ARB_CONTROL 0x1fa6
+#define mmDP_AUX3_AUX_ARB_CONTROL_BASE_IDX 2
+#define mmDP_AUX3_AUX_INTERRUPT_CONTROL 0x1fa7
+#define mmDP_AUX3_AUX_INTERRUPT_CONTROL_BASE_IDX 2
+#define mmDP_AUX3_AUX_SW_STATUS 0x1fa8
+#define mmDP_AUX3_AUX_SW_STATUS_BASE_IDX 2
+#define mmDP_AUX3_AUX_LS_STATUS 0x1fa9
+#define mmDP_AUX3_AUX_LS_STATUS_BASE_IDX 2
+#define mmDP_AUX3_AUX_SW_DATA 0x1faa
+#define mmDP_AUX3_AUX_SW_DATA_BASE_IDX 2
+#define mmDP_AUX3_AUX_LS_DATA 0x1fab
+#define mmDP_AUX3_AUX_LS_DATA_BASE_IDX 2
+#define mmDP_AUX3_AUX_DPHY_TX_REF_CONTROL 0x1fac
+#define mmDP_AUX3_AUX_DPHY_TX_REF_CONTROL_BASE_IDX 2
+#define mmDP_AUX3_AUX_DPHY_TX_CONTROL 0x1fad
+#define mmDP_AUX3_AUX_DPHY_TX_CONTROL_BASE_IDX 2
+#define mmDP_AUX3_AUX_DPHY_RX_CONTROL0 0x1fae
+#define mmDP_AUX3_AUX_DPHY_RX_CONTROL0_BASE_IDX 2
+#define mmDP_AUX3_AUX_DPHY_RX_CONTROL1 0x1faf
+#define mmDP_AUX3_AUX_DPHY_RX_CONTROL1_BASE_IDX 2
+#define mmDP_AUX3_AUX_DPHY_TX_STATUS 0x1fb0
+#define mmDP_AUX3_AUX_DPHY_TX_STATUS_BASE_IDX 2
+#define mmDP_AUX3_AUX_DPHY_RX_STATUS 0x1fb1
+#define mmDP_AUX3_AUX_DPHY_RX_STATUS_BASE_IDX 2
+#define mmDP_AUX3_AUX_GTC_SYNC_CONTROL 0x1fb2
+#define mmDP_AUX3_AUX_GTC_SYNC_CONTROL_BASE_IDX 2
+#define mmDP_AUX3_AUX_GTC_SYNC_ERROR_CONTROL 0x1fb3
+#define mmDP_AUX3_AUX_GTC_SYNC_ERROR_CONTROL_BASE_IDX 2
+#define mmDP_AUX3_AUX_GTC_SYNC_CONTROLLER_STATUS 0x1fb4
+#define mmDP_AUX3_AUX_GTC_SYNC_CONTROLLER_STATUS_BASE_IDX 2
+#define mmDP_AUX3_AUX_GTC_SYNC_STATUS 0x1fb5
+#define mmDP_AUX3_AUX_GTC_SYNC_STATUS_BASE_IDX 2
+#define mmDP_AUX3_AUX_PHY_WAKE_CNTL 0x1fba
+#define mmDP_AUX3_AUX_PHY_WAKE_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dp_aux4_dispdec
+// base address: 0x1c0
+#define mmDP_AUX4_AUX_CONTROL 0x1fc0
+#define mmDP_AUX4_AUX_CONTROL_BASE_IDX 2
+#define mmDP_AUX4_AUX_SW_CONTROL 0x1fc1
+#define mmDP_AUX4_AUX_SW_CONTROL_BASE_IDX 2
+#define mmDP_AUX4_AUX_ARB_CONTROL 0x1fc2
+#define mmDP_AUX4_AUX_ARB_CONTROL_BASE_IDX 2
+#define mmDP_AUX4_AUX_INTERRUPT_CONTROL 0x1fc3
+#define mmDP_AUX4_AUX_INTERRUPT_CONTROL_BASE_IDX 2
+#define mmDP_AUX4_AUX_SW_STATUS 0x1fc4
+#define mmDP_AUX4_AUX_SW_STATUS_BASE_IDX 2
+#define mmDP_AUX4_AUX_LS_STATUS 0x1fc5
+#define mmDP_AUX4_AUX_LS_STATUS_BASE_IDX 2
+#define mmDP_AUX4_AUX_SW_DATA 0x1fc6
+#define mmDP_AUX4_AUX_SW_DATA_BASE_IDX 2
+#define mmDP_AUX4_AUX_LS_DATA 0x1fc7
+#define mmDP_AUX4_AUX_LS_DATA_BASE_IDX 2
+#define mmDP_AUX4_AUX_DPHY_TX_REF_CONTROL 0x1fc8
+#define mmDP_AUX4_AUX_DPHY_TX_REF_CONTROL_BASE_IDX 2
+#define mmDP_AUX4_AUX_DPHY_TX_CONTROL 0x1fc9
+#define mmDP_AUX4_AUX_DPHY_TX_CONTROL_BASE_IDX 2
+#define mmDP_AUX4_AUX_DPHY_RX_CONTROL0 0x1fca
+#define mmDP_AUX4_AUX_DPHY_RX_CONTROL0_BASE_IDX 2
+#define mmDP_AUX4_AUX_DPHY_RX_CONTROL1 0x1fcb
+#define mmDP_AUX4_AUX_DPHY_RX_CONTROL1_BASE_IDX 2
+#define mmDP_AUX4_AUX_DPHY_TX_STATUS 0x1fcc
+#define mmDP_AUX4_AUX_DPHY_TX_STATUS_BASE_IDX 2
+#define mmDP_AUX4_AUX_DPHY_RX_STATUS 0x1fcd
+#define mmDP_AUX4_AUX_DPHY_RX_STATUS_BASE_IDX 2
+#define mmDP_AUX4_AUX_GTC_SYNC_CONTROL 0x1fce
+#define mmDP_AUX4_AUX_GTC_SYNC_CONTROL_BASE_IDX 2
+#define mmDP_AUX4_AUX_GTC_SYNC_ERROR_CONTROL 0x1fcf
+#define mmDP_AUX4_AUX_GTC_SYNC_ERROR_CONTROL_BASE_IDX 2
+#define mmDP_AUX4_AUX_GTC_SYNC_CONTROLLER_STATUS 0x1fd0
+#define mmDP_AUX4_AUX_GTC_SYNC_CONTROLLER_STATUS_BASE_IDX 2
+#define mmDP_AUX4_AUX_GTC_SYNC_STATUS 0x1fd1
+#define mmDP_AUX4_AUX_GTC_SYNC_STATUS_BASE_IDX 2
+#define mmDP_AUX4_AUX_PHY_WAKE_CNTL 0x1fd6
+#define mmDP_AUX4_AUX_PHY_WAKE_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dp_aux5_dispdec
+// base address: 0x230
+#define mmDP_AUX5_AUX_CONTROL 0x1fdc
+#define mmDP_AUX5_AUX_CONTROL_BASE_IDX 2
+#define mmDP_AUX5_AUX_SW_CONTROL 0x1fdd
+#define mmDP_AUX5_AUX_SW_CONTROL_BASE_IDX 2
+#define mmDP_AUX5_AUX_ARB_CONTROL 0x1fde
+#define mmDP_AUX5_AUX_ARB_CONTROL_BASE_IDX 2
+#define mmDP_AUX5_AUX_INTERRUPT_CONTROL 0x1fdf
+#define mmDP_AUX5_AUX_INTERRUPT_CONTROL_BASE_IDX 2
+#define mmDP_AUX5_AUX_SW_STATUS 0x1fe0
+#define mmDP_AUX5_AUX_SW_STATUS_BASE_IDX 2
+#define mmDP_AUX5_AUX_LS_STATUS 0x1fe1
+#define mmDP_AUX5_AUX_LS_STATUS_BASE_IDX 2
+#define mmDP_AUX5_AUX_SW_DATA 0x1fe2
+#define mmDP_AUX5_AUX_SW_DATA_BASE_IDX 2
+#define mmDP_AUX5_AUX_LS_DATA 0x1fe3
+#define mmDP_AUX5_AUX_LS_DATA_BASE_IDX 2
+#define mmDP_AUX5_AUX_DPHY_TX_REF_CONTROL 0x1fe4
+#define mmDP_AUX5_AUX_DPHY_TX_REF_CONTROL_BASE_IDX 2
+#define mmDP_AUX5_AUX_DPHY_TX_CONTROL 0x1fe5
+#define mmDP_AUX5_AUX_DPHY_TX_CONTROL_BASE_IDX 2
+#define mmDP_AUX5_AUX_DPHY_RX_CONTROL0 0x1fe6
+#define mmDP_AUX5_AUX_DPHY_RX_CONTROL0_BASE_IDX 2
+#define mmDP_AUX5_AUX_DPHY_RX_CONTROL1 0x1fe7
+#define mmDP_AUX5_AUX_DPHY_RX_CONTROL1_BASE_IDX 2
+#define mmDP_AUX5_AUX_DPHY_TX_STATUS 0x1fe8
+#define mmDP_AUX5_AUX_DPHY_TX_STATUS_BASE_IDX 2
+#define mmDP_AUX5_AUX_DPHY_RX_STATUS 0x1fe9
+#define mmDP_AUX5_AUX_DPHY_RX_STATUS_BASE_IDX 2
+#define mmDP_AUX5_AUX_GTC_SYNC_CONTROL 0x1fea
+#define mmDP_AUX5_AUX_GTC_SYNC_CONTROL_BASE_IDX 2
+#define mmDP_AUX5_AUX_GTC_SYNC_ERROR_CONTROL 0x1feb
+#define mmDP_AUX5_AUX_GTC_SYNC_ERROR_CONTROL_BASE_IDX 2
+#define mmDP_AUX5_AUX_GTC_SYNC_CONTROLLER_STATUS 0x1fec
+#define mmDP_AUX5_AUX_GTC_SYNC_CONTROLLER_STATUS_BASE_IDX 2
+#define mmDP_AUX5_AUX_GTC_SYNC_STATUS 0x1fed
+#define mmDP_AUX5_AUX_GTC_SYNC_STATUS_BASE_IDX 2
+#define mmDP_AUX5_AUX_PHY_WAKE_CNTL 0x1ff2
+#define mmDP_AUX5_AUX_PHY_WAKE_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dig0_dispdec
+// base address: 0x0
+#define mmDIG0_DIG_FE_CNTL 0x2068
+#define mmDIG0_DIG_FE_CNTL_BASE_IDX 2
+#define mmDIG0_DIG_OUTPUT_CRC_CNTL 0x2069
+#define mmDIG0_DIG_OUTPUT_CRC_CNTL_BASE_IDX 2
+#define mmDIG0_DIG_OUTPUT_CRC_RESULT 0x206a
+#define mmDIG0_DIG_OUTPUT_CRC_RESULT_BASE_IDX 2
+#define mmDIG0_DIG_CLOCK_PATTERN 0x206b
+#define mmDIG0_DIG_CLOCK_PATTERN_BASE_IDX 2
+#define mmDIG0_DIG_TEST_PATTERN 0x206c
+#define mmDIG0_DIG_TEST_PATTERN_BASE_IDX 2
+#define mmDIG0_DIG_RANDOM_PATTERN_SEED 0x206d
+#define mmDIG0_DIG_RANDOM_PATTERN_SEED_BASE_IDX 2
+#define mmDIG0_DIG_FIFO_STATUS 0x206e
+#define mmDIG0_DIG_FIFO_STATUS_BASE_IDX 2
+#define mmDIG0_HDMI_METADATA_PACKET_CONTROL 0x206f
+#define mmDIG0_HDMI_METADATA_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL4 0x2070
+#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL4_BASE_IDX 2
+#define mmDIG0_HDMI_CONTROL 0x2071
+#define mmDIG0_HDMI_CONTROL_BASE_IDX 2
+#define mmDIG0_HDMI_STATUS 0x2072
+#define mmDIG0_HDMI_STATUS_BASE_IDX 2
+#define mmDIG0_HDMI_AUDIO_PACKET_CONTROL 0x2073
+#define mmDIG0_HDMI_AUDIO_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG0_HDMI_ACR_PACKET_CONTROL 0x2074
+#define mmDIG0_HDMI_ACR_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG0_HDMI_VBI_PACKET_CONTROL 0x2075
+#define mmDIG0_HDMI_VBI_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG0_HDMI_INFOFRAME_CONTROL0 0x2076
+#define mmDIG0_HDMI_INFOFRAME_CONTROL0_BASE_IDX 2
+#define mmDIG0_HDMI_INFOFRAME_CONTROL1 0x2077
+#define mmDIG0_HDMI_INFOFRAME_CONTROL1_BASE_IDX 2
+#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL0 0x2078
+#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL0_BASE_IDX 2
+#define mmDIG0_AFMT_INTERRUPT_STATUS 0x2079
+#define mmDIG0_AFMT_INTERRUPT_STATUS_BASE_IDX 2
+#define mmDIG0_HDMI_GC 0x207b
+#define mmDIG0_HDMI_GC_BASE_IDX 2
+#define mmDIG0_AFMT_AUDIO_PACKET_CONTROL2 0x207c
+#define mmDIG0_AFMT_AUDIO_PACKET_CONTROL2_BASE_IDX 2
+#define mmDIG0_AFMT_ISRC1_0 0x207d
+#define mmDIG0_AFMT_ISRC1_0_BASE_IDX 2
+#define mmDIG0_AFMT_ISRC1_1 0x207e
+#define mmDIG0_AFMT_ISRC1_1_BASE_IDX 2
+#define mmDIG0_AFMT_ISRC1_2 0x207f
+#define mmDIG0_AFMT_ISRC1_2_BASE_IDX 2
+#define mmDIG0_AFMT_ISRC1_3 0x2080
+#define mmDIG0_AFMT_ISRC1_3_BASE_IDX 2
+#define mmDIG0_AFMT_ISRC1_4 0x2081
+#define mmDIG0_AFMT_ISRC1_4_BASE_IDX 2
+#define mmDIG0_AFMT_ISRC2_0 0x2082
+#define mmDIG0_AFMT_ISRC2_0_BASE_IDX 2
+#define mmDIG0_AFMT_ISRC2_1 0x2083
+#define mmDIG0_AFMT_ISRC2_1_BASE_IDX 2
+#define mmDIG0_AFMT_ISRC2_2 0x2084
+#define mmDIG0_AFMT_ISRC2_2_BASE_IDX 2
+#define mmDIG0_AFMT_ISRC2_3 0x2085
+#define mmDIG0_AFMT_ISRC2_3_BASE_IDX 2
+#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL2 0x2086
+#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL2_BASE_IDX 2
+#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL3 0x2087
+#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL3_BASE_IDX 2
+#define mmDIG0_HDMI_DB_CONTROL 0x2088
+#define mmDIG0_HDMI_DB_CONTROL_BASE_IDX 2
+#define mmDIG0_DME_CONTROL 0x2089
+#define mmDIG0_DME_CONTROL_BASE_IDX 2
+#define mmDIG0_AFMT_MPEG_INFO0 0x208a
+#define mmDIG0_AFMT_MPEG_INFO0_BASE_IDX 2
+#define mmDIG0_AFMT_MPEG_INFO1 0x208b
+#define mmDIG0_AFMT_MPEG_INFO1_BASE_IDX 2
+#define mmDIG0_AFMT_GENERIC_HDR 0x208c
+#define mmDIG0_AFMT_GENERIC_HDR_BASE_IDX 2
+#define mmDIG0_AFMT_GENERIC_0 0x208d
+#define mmDIG0_AFMT_GENERIC_0_BASE_IDX 2
+#define mmDIG0_AFMT_GENERIC_1 0x208e
+#define mmDIG0_AFMT_GENERIC_1_BASE_IDX 2
+#define mmDIG0_AFMT_GENERIC_2 0x208f
+#define mmDIG0_AFMT_GENERIC_2_BASE_IDX 2
+#define mmDIG0_AFMT_GENERIC_3 0x2090
+#define mmDIG0_AFMT_GENERIC_3_BASE_IDX 2
+#define mmDIG0_AFMT_GENERIC_4 0x2091
+#define mmDIG0_AFMT_GENERIC_4_BASE_IDX 2
+#define mmDIG0_AFMT_GENERIC_5 0x2092
+#define mmDIG0_AFMT_GENERIC_5_BASE_IDX 2
+#define mmDIG0_AFMT_GENERIC_6 0x2093
+#define mmDIG0_AFMT_GENERIC_6_BASE_IDX 2
+#define mmDIG0_AFMT_GENERIC_7 0x2094
+#define mmDIG0_AFMT_GENERIC_7_BASE_IDX 2
+#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL1 0x2095
+#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL1_BASE_IDX 2
+#define mmDIG0_HDMI_ACR_32_0 0x2096
+#define mmDIG0_HDMI_ACR_32_0_BASE_IDX 2
+#define mmDIG0_HDMI_ACR_32_1 0x2097
+#define mmDIG0_HDMI_ACR_32_1_BASE_IDX 2
+#define mmDIG0_HDMI_ACR_44_0 0x2098
+#define mmDIG0_HDMI_ACR_44_0_BASE_IDX 2
+#define mmDIG0_HDMI_ACR_44_1 0x2099
+#define mmDIG0_HDMI_ACR_44_1_BASE_IDX 2
+#define mmDIG0_HDMI_ACR_48_0 0x209a
+#define mmDIG0_HDMI_ACR_48_0_BASE_IDX 2
+#define mmDIG0_HDMI_ACR_48_1 0x209b
+#define mmDIG0_HDMI_ACR_48_1_BASE_IDX 2
+#define mmDIG0_HDMI_ACR_STATUS_0 0x209c
+#define mmDIG0_HDMI_ACR_STATUS_0_BASE_IDX 2
+#define mmDIG0_HDMI_ACR_STATUS_1 0x209d
+#define mmDIG0_HDMI_ACR_STATUS_1_BASE_IDX 2
+#define mmDIG0_AFMT_AUDIO_INFO0 0x209e
+#define mmDIG0_AFMT_AUDIO_INFO0_BASE_IDX 2
+#define mmDIG0_AFMT_AUDIO_INFO1 0x209f
+#define mmDIG0_AFMT_AUDIO_INFO1_BASE_IDX 2
+#define mmDIG0_AFMT_60958_0 0x20a0
+#define mmDIG0_AFMT_60958_0_BASE_IDX 2
+#define mmDIG0_AFMT_60958_1 0x20a1
+#define mmDIG0_AFMT_60958_1_BASE_IDX 2
+#define mmDIG0_AFMT_AUDIO_CRC_CONTROL 0x20a2
+#define mmDIG0_AFMT_AUDIO_CRC_CONTROL_BASE_IDX 2
+#define mmDIG0_AFMT_RAMP_CONTROL0 0x20a3
+#define mmDIG0_AFMT_RAMP_CONTROL0_BASE_IDX 2
+#define mmDIG0_AFMT_RAMP_CONTROL1 0x20a4
+#define mmDIG0_AFMT_RAMP_CONTROL1_BASE_IDX 2
+#define mmDIG0_AFMT_RAMP_CONTROL2 0x20a5
+#define mmDIG0_AFMT_RAMP_CONTROL2_BASE_IDX 2
+#define mmDIG0_AFMT_RAMP_CONTROL3 0x20a6
+#define mmDIG0_AFMT_RAMP_CONTROL3_BASE_IDX 2
+#define mmDIG0_AFMT_60958_2 0x20a7
+#define mmDIG0_AFMT_60958_2_BASE_IDX 2
+#define mmDIG0_AFMT_AUDIO_CRC_RESULT 0x20a8
+#define mmDIG0_AFMT_AUDIO_CRC_RESULT_BASE_IDX 2
+#define mmDIG0_AFMT_STATUS 0x20a9
+#define mmDIG0_AFMT_STATUS_BASE_IDX 2
+#define mmDIG0_AFMT_AUDIO_PACKET_CONTROL 0x20aa
+#define mmDIG0_AFMT_AUDIO_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG0_AFMT_VBI_PACKET_CONTROL 0x20ab
+#define mmDIG0_AFMT_VBI_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG0_AFMT_INFOFRAME_CONTROL0 0x20ac
+#define mmDIG0_AFMT_INFOFRAME_CONTROL0_BASE_IDX 2
+#define mmDIG0_AFMT_AUDIO_SRC_CONTROL 0x20ad
+#define mmDIG0_AFMT_AUDIO_SRC_CONTROL_BASE_IDX 2
+#define mmDIG0_DIG_BE_CNTL 0x20af
+#define mmDIG0_DIG_BE_CNTL_BASE_IDX 2
+#define mmDIG0_DIG_BE_EN_CNTL 0x20b0
+#define mmDIG0_DIG_BE_EN_CNTL_BASE_IDX 2
+#define mmDIG0_TMDS_CNTL 0x20d3
+#define mmDIG0_TMDS_CNTL_BASE_IDX 2
+#define mmDIG0_TMDS_CONTROL_CHAR 0x20d4
+#define mmDIG0_TMDS_CONTROL_CHAR_BASE_IDX 2
+#define mmDIG0_TMDS_CONTROL0_FEEDBACK 0x20d5
+#define mmDIG0_TMDS_CONTROL0_FEEDBACK_BASE_IDX 2
+#define mmDIG0_TMDS_STEREOSYNC_CTL_SEL 0x20d6
+#define mmDIG0_TMDS_STEREOSYNC_CTL_SEL_BASE_IDX 2
+#define mmDIG0_TMDS_SYNC_CHAR_PATTERN_0_1 0x20d7
+#define mmDIG0_TMDS_SYNC_CHAR_PATTERN_0_1_BASE_IDX 2
+#define mmDIG0_TMDS_SYNC_CHAR_PATTERN_2_3 0x20d8
+#define mmDIG0_TMDS_SYNC_CHAR_PATTERN_2_3_BASE_IDX 2
+#define mmDIG0_TMDS_CTL_BITS 0x20da
+#define mmDIG0_TMDS_CTL_BITS_BASE_IDX 2
+#define mmDIG0_TMDS_DCBALANCER_CONTROL 0x20db
+#define mmDIG0_TMDS_DCBALANCER_CONTROL_BASE_IDX 2
+#define mmDIG0_TMDS_SYNC_DCBALANCE_CHAR 0x20dc
+#define mmDIG0_TMDS_SYNC_DCBALANCE_CHAR_BASE_IDX 2
+#define mmDIG0_TMDS_CTL0_1_GEN_CNTL 0x20dd
+#define mmDIG0_TMDS_CTL0_1_GEN_CNTL_BASE_IDX 2
+#define mmDIG0_TMDS_CTL2_3_GEN_CNTL 0x20de
+#define mmDIG0_TMDS_CTL2_3_GEN_CNTL_BASE_IDX 2
+#define mmDIG0_DIG_VERSION 0x20e0
+#define mmDIG0_DIG_VERSION_BASE_IDX 2
+#define mmDIG0_DIG_LANE_ENABLE 0x20e1
+#define mmDIG0_DIG_LANE_ENABLE_BASE_IDX 2
+#define mmDIG0_AFMT_CNTL 0x20e6
+#define mmDIG0_AFMT_CNTL_BASE_IDX 2
+#define mmDIG0_AFMT_VBI_PACKET_CONTROL1 0x20e7
+#define mmDIG0_AFMT_VBI_PACKET_CONTROL1_BASE_IDX 2
+#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL5 0x20f6
+#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL5_BASE_IDX 2
+#define mmDIG0_FORCE_DIG_DISABLE 0x20f7
+#define mmDIG0_FORCE_DIG_DISABLE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dp0_dispdec
+// base address: 0x0
+#define mmDP0_DP_LINK_CNTL 0x2108
+#define mmDP0_DP_LINK_CNTL_BASE_IDX 2
+#define mmDP0_DP_PIXEL_FORMAT 0x2109
+#define mmDP0_DP_PIXEL_FORMAT_BASE_IDX 2
+#define mmDP0_DP_MSA_COLORIMETRY 0x210a
+#define mmDP0_DP_MSA_COLORIMETRY_BASE_IDX 2
+#define mmDP0_DP_CONFIG 0x210b
+#define mmDP0_DP_CONFIG_BASE_IDX 2
+#define mmDP0_DP_VID_STREAM_CNTL 0x210c
+#define mmDP0_DP_VID_STREAM_CNTL_BASE_IDX 2
+#define mmDP0_DP_STEER_FIFO 0x210d
+#define mmDP0_DP_STEER_FIFO_BASE_IDX 2
+#define mmDP0_DP_MSA_MISC 0x210e
+#define mmDP0_DP_MSA_MISC_BASE_IDX 2
+#define mmDP0_DP_VID_TIMING 0x2110
+#define mmDP0_DP_VID_TIMING_BASE_IDX 2
+#define mmDP0_DP_VID_N 0x2111
+#define mmDP0_DP_VID_N_BASE_IDX 2
+#define mmDP0_DP_VID_M 0x2112
+#define mmDP0_DP_VID_M_BASE_IDX 2
+#define mmDP0_DP_LINK_FRAMING_CNTL 0x2113
+#define mmDP0_DP_LINK_FRAMING_CNTL_BASE_IDX 2
+#define mmDP0_DP_HBR2_EYE_PATTERN 0x2114
+#define mmDP0_DP_HBR2_EYE_PATTERN_BASE_IDX 2
+#define mmDP0_DP_VID_MSA_VBID 0x2115
+#define mmDP0_DP_VID_MSA_VBID_BASE_IDX 2
+#define mmDP0_DP_VID_INTERRUPT_CNTL 0x2116
+#define mmDP0_DP_VID_INTERRUPT_CNTL_BASE_IDX 2
+#define mmDP0_DP_DPHY_CNTL 0x2117
+#define mmDP0_DP_DPHY_CNTL_BASE_IDX 2
+#define mmDP0_DP_DPHY_TRAINING_PATTERN_SEL 0x2118
+#define mmDP0_DP_DPHY_TRAINING_PATTERN_SEL_BASE_IDX 2
+#define mmDP0_DP_DPHY_SYM0 0x2119
+#define mmDP0_DP_DPHY_SYM0_BASE_IDX 2
+#define mmDP0_DP_DPHY_SYM1 0x211a
+#define mmDP0_DP_DPHY_SYM1_BASE_IDX 2
+#define mmDP0_DP_DPHY_SYM2 0x211b
+#define mmDP0_DP_DPHY_SYM2_BASE_IDX 2
+#define mmDP0_DP_DPHY_8B10B_CNTL 0x211c
+#define mmDP0_DP_DPHY_8B10B_CNTL_BASE_IDX 2
+#define mmDP0_DP_DPHY_PRBS_CNTL 0x211d
+#define mmDP0_DP_DPHY_PRBS_CNTL_BASE_IDX 2
+#define mmDP0_DP_DPHY_SCRAM_CNTL 0x211e
+#define mmDP0_DP_DPHY_SCRAM_CNTL_BASE_IDX 2
+#define mmDP0_DP_DPHY_CRC_EN 0x211f
+#define mmDP0_DP_DPHY_CRC_EN_BASE_IDX 2
+#define mmDP0_DP_DPHY_CRC_CNTL 0x2120
+#define mmDP0_DP_DPHY_CRC_CNTL_BASE_IDX 2
+#define mmDP0_DP_DPHY_CRC_RESULT 0x2121
+#define mmDP0_DP_DPHY_CRC_RESULT_BASE_IDX 2
+#define mmDP0_DP_DPHY_CRC_MST_CNTL 0x2122
+#define mmDP0_DP_DPHY_CRC_MST_CNTL_BASE_IDX 2
+#define mmDP0_DP_DPHY_CRC_MST_STATUS 0x2123
+#define mmDP0_DP_DPHY_CRC_MST_STATUS_BASE_IDX 2
+#define mmDP0_DP_DPHY_FAST_TRAINING 0x2124
+#define mmDP0_DP_DPHY_FAST_TRAINING_BASE_IDX 2
+#define mmDP0_DP_DPHY_FAST_TRAINING_STATUS 0x2125
+#define mmDP0_DP_DPHY_FAST_TRAINING_STATUS_BASE_IDX 2
+#define mmDP0_DP_SEC_CNTL 0x212b
+#define mmDP0_DP_SEC_CNTL_BASE_IDX 2
+#define mmDP0_DP_SEC_CNTL1 0x212c
+#define mmDP0_DP_SEC_CNTL1_BASE_IDX 2
+#define mmDP0_DP_SEC_FRAMING1 0x212d
+#define mmDP0_DP_SEC_FRAMING1_BASE_IDX 2
+#define mmDP0_DP_SEC_FRAMING2 0x212e
+#define mmDP0_DP_SEC_FRAMING2_BASE_IDX 2
+#define mmDP0_DP_SEC_FRAMING3 0x212f
+#define mmDP0_DP_SEC_FRAMING3_BASE_IDX 2
+#define mmDP0_DP_SEC_FRAMING4 0x2130
+#define mmDP0_DP_SEC_FRAMING4_BASE_IDX 2
+#define mmDP0_DP_SEC_AUD_N 0x2131
+#define mmDP0_DP_SEC_AUD_N_BASE_IDX 2
+#define mmDP0_DP_SEC_AUD_N_READBACK 0x2132
+#define mmDP0_DP_SEC_AUD_N_READBACK_BASE_IDX 2
+#define mmDP0_DP_SEC_AUD_M 0x2133
+#define mmDP0_DP_SEC_AUD_M_BASE_IDX 2
+#define mmDP0_DP_SEC_AUD_M_READBACK 0x2134
+#define mmDP0_DP_SEC_AUD_M_READBACK_BASE_IDX 2
+#define mmDP0_DP_SEC_TIMESTAMP 0x2135
+#define mmDP0_DP_SEC_TIMESTAMP_BASE_IDX 2
+#define mmDP0_DP_SEC_PACKET_CNTL 0x2136
+#define mmDP0_DP_SEC_PACKET_CNTL_BASE_IDX 2
+#define mmDP0_DP_MSE_RATE_CNTL 0x2137
+#define mmDP0_DP_MSE_RATE_CNTL_BASE_IDX 2
+#define mmDP0_DP_MSE_RATE_UPDATE 0x2139
+#define mmDP0_DP_MSE_RATE_UPDATE_BASE_IDX 2
+#define mmDP0_DP_MSE_SAT0 0x213a
+#define mmDP0_DP_MSE_SAT0_BASE_IDX 2
+#define mmDP0_DP_MSE_SAT1 0x213b
+#define mmDP0_DP_MSE_SAT1_BASE_IDX 2
+#define mmDP0_DP_MSE_SAT2 0x213c
+#define mmDP0_DP_MSE_SAT2_BASE_IDX 2
+#define mmDP0_DP_MSE_SAT_UPDATE 0x213d
+#define mmDP0_DP_MSE_SAT_UPDATE_BASE_IDX 2
+#define mmDP0_DP_MSE_LINK_TIMING 0x213e
+#define mmDP0_DP_MSE_LINK_TIMING_BASE_IDX 2
+#define mmDP0_DP_MSE_MISC_CNTL 0x213f
+#define mmDP0_DP_MSE_MISC_CNTL_BASE_IDX 2
+#define mmDP0_DP_DPHY_BS_SR_SWAP_CNTL 0x2144
+#define mmDP0_DP_DPHY_BS_SR_SWAP_CNTL_BASE_IDX 2
+#define mmDP0_DP_DPHY_HBR2_PATTERN_CONTROL 0x2145
+#define mmDP0_DP_DPHY_HBR2_PATTERN_CONTROL_BASE_IDX 2
+#define mmDP0_DP_MSE_SAT0_STATUS 0x2147
+#define mmDP0_DP_MSE_SAT0_STATUS_BASE_IDX 2
+#define mmDP0_DP_MSE_SAT1_STATUS 0x2148
+#define mmDP0_DP_MSE_SAT1_STATUS_BASE_IDX 2
+#define mmDP0_DP_MSE_SAT2_STATUS 0x2149
+#define mmDP0_DP_MSE_SAT2_STATUS_BASE_IDX 2
+#define mmDP0_DP_MSA_TIMING_PARAM1 0x214c
+#define mmDP0_DP_MSA_TIMING_PARAM1_BASE_IDX 2
+#define mmDP0_DP_MSA_TIMING_PARAM2 0x214d
+#define mmDP0_DP_MSA_TIMING_PARAM2_BASE_IDX 2
+#define mmDP0_DP_MSA_TIMING_PARAM3 0x214e
+#define mmDP0_DP_MSA_TIMING_PARAM3_BASE_IDX 2
+#define mmDP0_DP_MSA_TIMING_PARAM4 0x214f
+#define mmDP0_DP_MSA_TIMING_PARAM4_BASE_IDX 2
+#define mmDP0_DP_MSO_CNTL 0x2150
+#define mmDP0_DP_MSO_CNTL_BASE_IDX 2
+#define mmDP0_DP_MSO_CNTL1 0x2151
+#define mmDP0_DP_MSO_CNTL1_BASE_IDX 2
+#define mmDP0_DP_DSC_CNTL 0x2152
+#define mmDP0_DP_DSC_CNTL_BASE_IDX 2
+#define mmDP0_DP_SEC_CNTL2 0x2153
+#define mmDP0_DP_SEC_CNTL2_BASE_IDX 2
+#define mmDP0_DP_SEC_CNTL3 0x2154
+#define mmDP0_DP_SEC_CNTL3_BASE_IDX 2
+#define mmDP0_DP_SEC_CNTL4 0x2155
+#define mmDP0_DP_SEC_CNTL4_BASE_IDX 2
+#define mmDP0_DP_SEC_CNTL5 0x2156
+#define mmDP0_DP_SEC_CNTL5_BASE_IDX 2
+#define mmDP0_DP_SEC_CNTL6 0x2157
+#define mmDP0_DP_SEC_CNTL6_BASE_IDX 2
+#define mmDP0_DP_SEC_CNTL7 0x2158
+#define mmDP0_DP_SEC_CNTL7_BASE_IDX 2
+#define mmDP0_DP_DB_CNTL 0x2159
+#define mmDP0_DP_DB_CNTL_BASE_IDX 2
+#define mmDP0_DP_MSA_VBID_MISC 0x215a
+#define mmDP0_DP_MSA_VBID_MISC_BASE_IDX 2
+#define mmDP0_DP_SEC_METADATA_TRANSMISSION 0x215b
+#define mmDP0_DP_SEC_METADATA_TRANSMISSION_BASE_IDX 2
+#define mmDP0_DP_DSC_BYTES_PER_PIXEL 0x215c
+#define mmDP0_DP_DSC_BYTES_PER_PIXEL_BASE_IDX 2
+#define mmDP0_DP_ALPM_CNTL 0x215d
+#define mmDP0_DP_ALPM_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dig1_dispdec
+// base address: 0x400
+#define mmDIG1_DIG_FE_CNTL 0x2168
+#define mmDIG1_DIG_FE_CNTL_BASE_IDX 2
+#define mmDIG1_DIG_OUTPUT_CRC_CNTL 0x2169
+#define mmDIG1_DIG_OUTPUT_CRC_CNTL_BASE_IDX 2
+#define mmDIG1_DIG_OUTPUT_CRC_RESULT 0x216a
+#define mmDIG1_DIG_OUTPUT_CRC_RESULT_BASE_IDX 2
+#define mmDIG1_DIG_CLOCK_PATTERN 0x216b
+#define mmDIG1_DIG_CLOCK_PATTERN_BASE_IDX 2
+#define mmDIG1_DIG_TEST_PATTERN 0x216c
+#define mmDIG1_DIG_TEST_PATTERN_BASE_IDX 2
+#define mmDIG1_DIG_RANDOM_PATTERN_SEED 0x216d
+#define mmDIG1_DIG_RANDOM_PATTERN_SEED_BASE_IDX 2
+#define mmDIG1_DIG_FIFO_STATUS 0x216e
+#define mmDIG1_DIG_FIFO_STATUS_BASE_IDX 2
+#define mmDIG1_HDMI_METADATA_PACKET_CONTROL 0x216f
+#define mmDIG1_HDMI_METADATA_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL4 0x2170
+#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL4_BASE_IDX 2
+#define mmDIG1_HDMI_CONTROL 0x2171
+#define mmDIG1_HDMI_CONTROL_BASE_IDX 2
+#define mmDIG1_HDMI_STATUS 0x2172
+#define mmDIG1_HDMI_STATUS_BASE_IDX 2
+#define mmDIG1_HDMI_AUDIO_PACKET_CONTROL 0x2173
+#define mmDIG1_HDMI_AUDIO_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG1_HDMI_ACR_PACKET_CONTROL 0x2174
+#define mmDIG1_HDMI_ACR_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG1_HDMI_VBI_PACKET_CONTROL 0x2175
+#define mmDIG1_HDMI_VBI_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG1_HDMI_INFOFRAME_CONTROL0 0x2176
+#define mmDIG1_HDMI_INFOFRAME_CONTROL0_BASE_IDX 2
+#define mmDIG1_HDMI_INFOFRAME_CONTROL1 0x2177
+#define mmDIG1_HDMI_INFOFRAME_CONTROL1_BASE_IDX 2
+#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL0 0x2178
+#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL0_BASE_IDX 2
+#define mmDIG1_AFMT_INTERRUPT_STATUS 0x2179
+#define mmDIG1_AFMT_INTERRUPT_STATUS_BASE_IDX 2
+#define mmDIG1_HDMI_GC 0x217b
+#define mmDIG1_HDMI_GC_BASE_IDX 2
+#define mmDIG1_AFMT_AUDIO_PACKET_CONTROL2 0x217c
+#define mmDIG1_AFMT_AUDIO_PACKET_CONTROL2_BASE_IDX 2
+#define mmDIG1_AFMT_ISRC1_0 0x217d
+#define mmDIG1_AFMT_ISRC1_0_BASE_IDX 2
+#define mmDIG1_AFMT_ISRC1_1 0x217e
+#define mmDIG1_AFMT_ISRC1_1_BASE_IDX 2
+#define mmDIG1_AFMT_ISRC1_2 0x217f
+#define mmDIG1_AFMT_ISRC1_2_BASE_IDX 2
+#define mmDIG1_AFMT_ISRC1_3 0x2180
+#define mmDIG1_AFMT_ISRC1_3_BASE_IDX 2
+#define mmDIG1_AFMT_ISRC1_4 0x2181
+#define mmDIG1_AFMT_ISRC1_4_BASE_IDX 2
+#define mmDIG1_AFMT_ISRC2_0 0x2182
+#define mmDIG1_AFMT_ISRC2_0_BASE_IDX 2
+#define mmDIG1_AFMT_ISRC2_1 0x2183
+#define mmDIG1_AFMT_ISRC2_1_BASE_IDX 2
+#define mmDIG1_AFMT_ISRC2_2 0x2184
+#define mmDIG1_AFMT_ISRC2_2_BASE_IDX 2
+#define mmDIG1_AFMT_ISRC2_3 0x2185
+#define mmDIG1_AFMT_ISRC2_3_BASE_IDX 2
+#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL2 0x2186
+#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL2_BASE_IDX 2
+#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL3 0x2187
+#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL3_BASE_IDX 2
+#define mmDIG1_HDMI_DB_CONTROL 0x2188
+#define mmDIG1_HDMI_DB_CONTROL_BASE_IDX 2
+#define mmDIG1_DME_CONTROL 0x2189
+#define mmDIG1_DME_CONTROL_BASE_IDX 2
+#define mmDIG1_AFMT_MPEG_INFO0 0x218a
+#define mmDIG1_AFMT_MPEG_INFO0_BASE_IDX 2
+#define mmDIG1_AFMT_MPEG_INFO1 0x218b
+#define mmDIG1_AFMT_MPEG_INFO1_BASE_IDX 2
+#define mmDIG1_AFMT_GENERIC_HDR 0x218c
+#define mmDIG1_AFMT_GENERIC_HDR_BASE_IDX 2
+#define mmDIG1_AFMT_GENERIC_0 0x218d
+#define mmDIG1_AFMT_GENERIC_0_BASE_IDX 2
+#define mmDIG1_AFMT_GENERIC_1 0x218e
+#define mmDIG1_AFMT_GENERIC_1_BASE_IDX 2
+#define mmDIG1_AFMT_GENERIC_2 0x218f
+#define mmDIG1_AFMT_GENERIC_2_BASE_IDX 2
+#define mmDIG1_AFMT_GENERIC_3 0x2190
+#define mmDIG1_AFMT_GENERIC_3_BASE_IDX 2
+#define mmDIG1_AFMT_GENERIC_4 0x2191
+#define mmDIG1_AFMT_GENERIC_4_BASE_IDX 2
+#define mmDIG1_AFMT_GENERIC_5 0x2192
+#define mmDIG1_AFMT_GENERIC_5_BASE_IDX 2
+#define mmDIG1_AFMT_GENERIC_6 0x2193
+#define mmDIG1_AFMT_GENERIC_6_BASE_IDX 2
+#define mmDIG1_AFMT_GENERIC_7 0x2194
+#define mmDIG1_AFMT_GENERIC_7_BASE_IDX 2
+#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL1 0x2195
+#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL1_BASE_IDX 2
+#define mmDIG1_HDMI_ACR_32_0 0x2196
+#define mmDIG1_HDMI_ACR_32_0_BASE_IDX 2
+#define mmDIG1_HDMI_ACR_32_1 0x2197
+#define mmDIG1_HDMI_ACR_32_1_BASE_IDX 2
+#define mmDIG1_HDMI_ACR_44_0 0x2198
+#define mmDIG1_HDMI_ACR_44_0_BASE_IDX 2
+#define mmDIG1_HDMI_ACR_44_1 0x2199
+#define mmDIG1_HDMI_ACR_44_1_BASE_IDX 2
+#define mmDIG1_HDMI_ACR_48_0 0x219a
+#define mmDIG1_HDMI_ACR_48_0_BASE_IDX 2
+#define mmDIG1_HDMI_ACR_48_1 0x219b
+#define mmDIG1_HDMI_ACR_48_1_BASE_IDX 2
+#define mmDIG1_HDMI_ACR_STATUS_0 0x219c
+#define mmDIG1_HDMI_ACR_STATUS_0_BASE_IDX 2
+#define mmDIG1_HDMI_ACR_STATUS_1 0x219d
+#define mmDIG1_HDMI_ACR_STATUS_1_BASE_IDX 2
+#define mmDIG1_AFMT_AUDIO_INFO0 0x219e
+#define mmDIG1_AFMT_AUDIO_INFO0_BASE_IDX 2
+#define mmDIG1_AFMT_AUDIO_INFO1 0x219f
+#define mmDIG1_AFMT_AUDIO_INFO1_BASE_IDX 2
+#define mmDIG1_AFMT_60958_0 0x21a0
+#define mmDIG1_AFMT_60958_0_BASE_IDX 2
+#define mmDIG1_AFMT_60958_1 0x21a1
+#define mmDIG1_AFMT_60958_1_BASE_IDX 2
+#define mmDIG1_AFMT_AUDIO_CRC_CONTROL 0x21a2
+#define mmDIG1_AFMT_AUDIO_CRC_CONTROL_BASE_IDX 2
+#define mmDIG1_AFMT_RAMP_CONTROL0 0x21a3
+#define mmDIG1_AFMT_RAMP_CONTROL0_BASE_IDX 2
+#define mmDIG1_AFMT_RAMP_CONTROL1 0x21a4
+#define mmDIG1_AFMT_RAMP_CONTROL1_BASE_IDX 2
+#define mmDIG1_AFMT_RAMP_CONTROL2 0x21a5
+#define mmDIG1_AFMT_RAMP_CONTROL2_BASE_IDX 2
+#define mmDIG1_AFMT_RAMP_CONTROL3 0x21a6
+#define mmDIG1_AFMT_RAMP_CONTROL3_BASE_IDX 2
+#define mmDIG1_AFMT_60958_2 0x21a7
+#define mmDIG1_AFMT_60958_2_BASE_IDX 2
+#define mmDIG1_AFMT_AUDIO_CRC_RESULT 0x21a8
+#define mmDIG1_AFMT_AUDIO_CRC_RESULT_BASE_IDX 2
+#define mmDIG1_AFMT_STATUS 0x21a9
+#define mmDIG1_AFMT_STATUS_BASE_IDX 2
+#define mmDIG1_AFMT_AUDIO_PACKET_CONTROL 0x21aa
+#define mmDIG1_AFMT_AUDIO_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG1_AFMT_VBI_PACKET_CONTROL 0x21ab
+#define mmDIG1_AFMT_VBI_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG1_AFMT_INFOFRAME_CONTROL0 0x21ac
+#define mmDIG1_AFMT_INFOFRAME_CONTROL0_BASE_IDX 2
+#define mmDIG1_AFMT_AUDIO_SRC_CONTROL 0x21ad
+#define mmDIG1_AFMT_AUDIO_SRC_CONTROL_BASE_IDX 2
+#define mmDIG1_DIG_BE_CNTL 0x21af
+#define mmDIG1_DIG_BE_CNTL_BASE_IDX 2
+#define mmDIG1_DIG_BE_EN_CNTL 0x21b0
+#define mmDIG1_DIG_BE_EN_CNTL_BASE_IDX 2
+#define mmDIG1_TMDS_CNTL 0x21d3
+#define mmDIG1_TMDS_CNTL_BASE_IDX 2
+#define mmDIG1_TMDS_CONTROL_CHAR 0x21d4
+#define mmDIG1_TMDS_CONTROL_CHAR_BASE_IDX 2
+#define mmDIG1_TMDS_CONTROL0_FEEDBACK 0x21d5
+#define mmDIG1_TMDS_CONTROL0_FEEDBACK_BASE_IDX 2
+#define mmDIG1_TMDS_STEREOSYNC_CTL_SEL 0x21d6
+#define mmDIG1_TMDS_STEREOSYNC_CTL_SEL_BASE_IDX 2
+#define mmDIG1_TMDS_SYNC_CHAR_PATTERN_0_1 0x21d7
+#define mmDIG1_TMDS_SYNC_CHAR_PATTERN_0_1_BASE_IDX 2
+#define mmDIG1_TMDS_SYNC_CHAR_PATTERN_2_3 0x21d8
+#define mmDIG1_TMDS_SYNC_CHAR_PATTERN_2_3_BASE_IDX 2
+#define mmDIG1_TMDS_CTL_BITS 0x21da
+#define mmDIG1_TMDS_CTL_BITS_BASE_IDX 2
+#define mmDIG1_TMDS_DCBALANCER_CONTROL 0x21db
+#define mmDIG1_TMDS_DCBALANCER_CONTROL_BASE_IDX 2
+#define mmDIG1_TMDS_SYNC_DCBALANCE_CHAR 0x21dc
+#define mmDIG1_TMDS_SYNC_DCBALANCE_CHAR_BASE_IDX 2
+#define mmDIG1_TMDS_CTL0_1_GEN_CNTL 0x21dd
+#define mmDIG1_TMDS_CTL0_1_GEN_CNTL_BASE_IDX 2
+#define mmDIG1_TMDS_CTL2_3_GEN_CNTL 0x21de
+#define mmDIG1_TMDS_CTL2_3_GEN_CNTL_BASE_IDX 2
+#define mmDIG1_DIG_VERSION 0x21e0
+#define mmDIG1_DIG_VERSION_BASE_IDX 2
+#define mmDIG1_DIG_LANE_ENABLE 0x21e1
+#define mmDIG1_DIG_LANE_ENABLE_BASE_IDX 2
+#define mmDIG1_AFMT_CNTL 0x21e6
+#define mmDIG1_AFMT_CNTL_BASE_IDX 2
+#define mmDIG1_AFMT_VBI_PACKET_CONTROL1 0x21e7
+#define mmDIG1_AFMT_VBI_PACKET_CONTROL1_BASE_IDX 2
+#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL5 0x21f6
+#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL5_BASE_IDX 2
+#define mmDIG1_FORCE_DIG_DISABLE 0x21f7
+#define mmDIG1_FORCE_DIG_DISABLE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dp1_dispdec
+// base address: 0x400
+#define mmDP1_DP_LINK_CNTL 0x2208
+#define mmDP1_DP_LINK_CNTL_BASE_IDX 2
+#define mmDP1_DP_PIXEL_FORMAT 0x2209
+#define mmDP1_DP_PIXEL_FORMAT_BASE_IDX 2
+#define mmDP1_DP_MSA_COLORIMETRY 0x220a
+#define mmDP1_DP_MSA_COLORIMETRY_BASE_IDX 2
+#define mmDP1_DP_CONFIG 0x220b
+#define mmDP1_DP_CONFIG_BASE_IDX 2
+#define mmDP1_DP_VID_STREAM_CNTL 0x220c
+#define mmDP1_DP_VID_STREAM_CNTL_BASE_IDX 2
+#define mmDP1_DP_STEER_FIFO 0x220d
+#define mmDP1_DP_STEER_FIFO_BASE_IDX 2
+#define mmDP1_DP_MSA_MISC 0x220e
+#define mmDP1_DP_MSA_MISC_BASE_IDX 2
+#define mmDP1_DP_VID_TIMING 0x2210
+#define mmDP1_DP_VID_TIMING_BASE_IDX 2
+#define mmDP1_DP_VID_N 0x2211
+#define mmDP1_DP_VID_N_BASE_IDX 2
+#define mmDP1_DP_VID_M 0x2212
+#define mmDP1_DP_VID_M_BASE_IDX 2
+#define mmDP1_DP_LINK_FRAMING_CNTL 0x2213
+#define mmDP1_DP_LINK_FRAMING_CNTL_BASE_IDX 2
+#define mmDP1_DP_HBR2_EYE_PATTERN 0x2214
+#define mmDP1_DP_HBR2_EYE_PATTERN_BASE_IDX 2
+#define mmDP1_DP_VID_MSA_VBID 0x2215
+#define mmDP1_DP_VID_MSA_VBID_BASE_IDX 2
+#define mmDP1_DP_VID_INTERRUPT_CNTL 0x2216
+#define mmDP1_DP_VID_INTERRUPT_CNTL_BASE_IDX 2
+#define mmDP1_DP_DPHY_CNTL 0x2217
+#define mmDP1_DP_DPHY_CNTL_BASE_IDX 2
+#define mmDP1_DP_DPHY_TRAINING_PATTERN_SEL 0x2218
+#define mmDP1_DP_DPHY_TRAINING_PATTERN_SEL_BASE_IDX 2
+#define mmDP1_DP_DPHY_SYM0 0x2219
+#define mmDP1_DP_DPHY_SYM0_BASE_IDX 2
+#define mmDP1_DP_DPHY_SYM1 0x221a
+#define mmDP1_DP_DPHY_SYM1_BASE_IDX 2
+#define mmDP1_DP_DPHY_SYM2 0x221b
+#define mmDP1_DP_DPHY_SYM2_BASE_IDX 2
+#define mmDP1_DP_DPHY_8B10B_CNTL 0x221c
+#define mmDP1_DP_DPHY_8B10B_CNTL_BASE_IDX 2
+#define mmDP1_DP_DPHY_PRBS_CNTL 0x221d
+#define mmDP1_DP_DPHY_PRBS_CNTL_BASE_IDX 2
+#define mmDP1_DP_DPHY_SCRAM_CNTL 0x221e
+#define mmDP1_DP_DPHY_SCRAM_CNTL_BASE_IDX 2
+#define mmDP1_DP_DPHY_CRC_EN 0x221f
+#define mmDP1_DP_DPHY_CRC_EN_BASE_IDX 2
+#define mmDP1_DP_DPHY_CRC_CNTL 0x2220
+#define mmDP1_DP_DPHY_CRC_CNTL_BASE_IDX 2
+#define mmDP1_DP_DPHY_CRC_RESULT 0x2221
+#define mmDP1_DP_DPHY_CRC_RESULT_BASE_IDX 2
+#define mmDP1_DP_DPHY_CRC_MST_CNTL 0x2222
+#define mmDP1_DP_DPHY_CRC_MST_CNTL_BASE_IDX 2
+#define mmDP1_DP_DPHY_CRC_MST_STATUS 0x2223
+#define mmDP1_DP_DPHY_CRC_MST_STATUS_BASE_IDX 2
+#define mmDP1_DP_DPHY_FAST_TRAINING 0x2224
+#define mmDP1_DP_DPHY_FAST_TRAINING_BASE_IDX 2
+#define mmDP1_DP_DPHY_FAST_TRAINING_STATUS 0x2225
+#define mmDP1_DP_DPHY_FAST_TRAINING_STATUS_BASE_IDX 2
+#define mmDP1_DP_SEC_CNTL 0x222b
+#define mmDP1_DP_SEC_CNTL_BASE_IDX 2
+#define mmDP1_DP_SEC_CNTL1 0x222c
+#define mmDP1_DP_SEC_CNTL1_BASE_IDX 2
+#define mmDP1_DP_SEC_FRAMING1 0x222d
+#define mmDP1_DP_SEC_FRAMING1_BASE_IDX 2
+#define mmDP1_DP_SEC_FRAMING2 0x222e
+#define mmDP1_DP_SEC_FRAMING2_BASE_IDX 2
+#define mmDP1_DP_SEC_FRAMING3 0x222f
+#define mmDP1_DP_SEC_FRAMING3_BASE_IDX 2
+#define mmDP1_DP_SEC_FRAMING4 0x2230
+#define mmDP1_DP_SEC_FRAMING4_BASE_IDX 2
+#define mmDP1_DP_SEC_AUD_N 0x2231
+#define mmDP1_DP_SEC_AUD_N_BASE_IDX 2
+#define mmDP1_DP_SEC_AUD_N_READBACK 0x2232
+#define mmDP1_DP_SEC_AUD_N_READBACK_BASE_IDX 2
+#define mmDP1_DP_SEC_AUD_M 0x2233
+#define mmDP1_DP_SEC_AUD_M_BASE_IDX 2
+#define mmDP1_DP_SEC_AUD_M_READBACK 0x2234
+#define mmDP1_DP_SEC_AUD_M_READBACK_BASE_IDX 2
+#define mmDP1_DP_SEC_TIMESTAMP 0x2235
+#define mmDP1_DP_SEC_TIMESTAMP_BASE_IDX 2
+#define mmDP1_DP_SEC_PACKET_CNTL 0x2236
+#define mmDP1_DP_SEC_PACKET_CNTL_BASE_IDX 2
+#define mmDP1_DP_MSE_RATE_CNTL 0x2237
+#define mmDP1_DP_MSE_RATE_CNTL_BASE_IDX 2
+#define mmDP1_DP_MSE_RATE_UPDATE 0x2239
+#define mmDP1_DP_MSE_RATE_UPDATE_BASE_IDX 2
+#define mmDP1_DP_MSE_SAT0 0x223a
+#define mmDP1_DP_MSE_SAT0_BASE_IDX 2
+#define mmDP1_DP_MSE_SAT1 0x223b
+#define mmDP1_DP_MSE_SAT1_BASE_IDX 2
+#define mmDP1_DP_MSE_SAT2 0x223c
+#define mmDP1_DP_MSE_SAT2_BASE_IDX 2
+#define mmDP1_DP_MSE_SAT_UPDATE 0x223d
+#define mmDP1_DP_MSE_SAT_UPDATE_BASE_IDX 2
+#define mmDP1_DP_MSE_LINK_TIMING 0x223e
+#define mmDP1_DP_MSE_LINK_TIMING_BASE_IDX 2
+#define mmDP1_DP_MSE_MISC_CNTL 0x223f
+#define mmDP1_DP_MSE_MISC_CNTL_BASE_IDX 2
+#define mmDP1_DP_DPHY_BS_SR_SWAP_CNTL 0x2244
+#define mmDP1_DP_DPHY_BS_SR_SWAP_CNTL_BASE_IDX 2
+#define mmDP1_DP_DPHY_HBR2_PATTERN_CONTROL 0x2245
+#define mmDP1_DP_DPHY_HBR2_PATTERN_CONTROL_BASE_IDX 2
+#define mmDP1_DP_MSE_SAT0_STATUS 0x2247
+#define mmDP1_DP_MSE_SAT0_STATUS_BASE_IDX 2
+#define mmDP1_DP_MSE_SAT1_STATUS 0x2248
+#define mmDP1_DP_MSE_SAT1_STATUS_BASE_IDX 2
+#define mmDP1_DP_MSE_SAT2_STATUS 0x2249
+#define mmDP1_DP_MSE_SAT2_STATUS_BASE_IDX 2
+#define mmDP1_DP_MSA_TIMING_PARAM1 0x224c
+#define mmDP1_DP_MSA_TIMING_PARAM1_BASE_IDX 2
+#define mmDP1_DP_MSA_TIMING_PARAM2 0x224d
+#define mmDP1_DP_MSA_TIMING_PARAM2_BASE_IDX 2
+#define mmDP1_DP_MSA_TIMING_PARAM3 0x224e
+#define mmDP1_DP_MSA_TIMING_PARAM3_BASE_IDX 2
+#define mmDP1_DP_MSA_TIMING_PARAM4 0x224f
+#define mmDP1_DP_MSA_TIMING_PARAM4_BASE_IDX 2
+#define mmDP1_DP_MSO_CNTL 0x2250
+#define mmDP1_DP_MSO_CNTL_BASE_IDX 2
+#define mmDP1_DP_MSO_CNTL1 0x2251
+#define mmDP1_DP_MSO_CNTL1_BASE_IDX 2
+#define mmDP1_DP_DSC_CNTL 0x2252
+#define mmDP1_DP_DSC_CNTL_BASE_IDX 2
+#define mmDP1_DP_SEC_CNTL2 0x2253
+#define mmDP1_DP_SEC_CNTL2_BASE_IDX 2
+#define mmDP1_DP_SEC_CNTL3 0x2254
+#define mmDP1_DP_SEC_CNTL3_BASE_IDX 2
+#define mmDP1_DP_SEC_CNTL4 0x2255
+#define mmDP1_DP_SEC_CNTL4_BASE_IDX 2
+#define mmDP1_DP_SEC_CNTL5 0x2256
+#define mmDP1_DP_SEC_CNTL5_BASE_IDX 2
+#define mmDP1_DP_SEC_CNTL6 0x2257
+#define mmDP1_DP_SEC_CNTL6_BASE_IDX 2
+#define mmDP1_DP_SEC_CNTL7 0x2258
+#define mmDP1_DP_SEC_CNTL7_BASE_IDX 2
+#define mmDP1_DP_DB_CNTL 0x2259
+#define mmDP1_DP_DB_CNTL_BASE_IDX 2
+#define mmDP1_DP_MSA_VBID_MISC 0x225a
+#define mmDP1_DP_MSA_VBID_MISC_BASE_IDX 2
+#define mmDP1_DP_SEC_METADATA_TRANSMISSION 0x225b
+#define mmDP1_DP_SEC_METADATA_TRANSMISSION_BASE_IDX 2
+#define mmDP1_DP_DSC_BYTES_PER_PIXEL 0x225c
+#define mmDP1_DP_DSC_BYTES_PER_PIXEL_BASE_IDX 2
+#define mmDP1_DP_ALPM_CNTL 0x225d
+#define mmDP1_DP_ALPM_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dig2_dispdec
+// base address: 0x800
+#define mmDIG2_DIG_FE_CNTL 0x2268
+#define mmDIG2_DIG_FE_CNTL_BASE_IDX 2
+#define mmDIG2_DIG_OUTPUT_CRC_CNTL 0x2269
+#define mmDIG2_DIG_OUTPUT_CRC_CNTL_BASE_IDX 2
+#define mmDIG2_DIG_OUTPUT_CRC_RESULT 0x226a
+#define mmDIG2_DIG_OUTPUT_CRC_RESULT_BASE_IDX 2
+#define mmDIG2_DIG_CLOCK_PATTERN 0x226b
+#define mmDIG2_DIG_CLOCK_PATTERN_BASE_IDX 2
+#define mmDIG2_DIG_TEST_PATTERN 0x226c
+#define mmDIG2_DIG_TEST_PATTERN_BASE_IDX 2
+#define mmDIG2_DIG_RANDOM_PATTERN_SEED 0x226d
+#define mmDIG2_DIG_RANDOM_PATTERN_SEED_BASE_IDX 2
+#define mmDIG2_DIG_FIFO_STATUS 0x226e
+#define mmDIG2_DIG_FIFO_STATUS_BASE_IDX 2
+#define mmDIG2_HDMI_METADATA_PACKET_CONTROL 0x226f
+#define mmDIG2_HDMI_METADATA_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL4 0x2270
+#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL4_BASE_IDX 2
+#define mmDIG2_HDMI_CONTROL 0x2271
+#define mmDIG2_HDMI_CONTROL_BASE_IDX 2
+#define mmDIG2_HDMI_STATUS 0x2272
+#define mmDIG2_HDMI_STATUS_BASE_IDX 2
+#define mmDIG2_HDMI_AUDIO_PACKET_CONTROL 0x2273
+#define mmDIG2_HDMI_AUDIO_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG2_HDMI_ACR_PACKET_CONTROL 0x2274
+#define mmDIG2_HDMI_ACR_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG2_HDMI_VBI_PACKET_CONTROL 0x2275
+#define mmDIG2_HDMI_VBI_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG2_HDMI_INFOFRAME_CONTROL0 0x2276
+#define mmDIG2_HDMI_INFOFRAME_CONTROL0_BASE_IDX 2
+#define mmDIG2_HDMI_INFOFRAME_CONTROL1 0x2277
+#define mmDIG2_HDMI_INFOFRAME_CONTROL1_BASE_IDX 2
+#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL0 0x2278
+#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL0_BASE_IDX 2
+#define mmDIG2_AFMT_INTERRUPT_STATUS 0x2279
+#define mmDIG2_AFMT_INTERRUPT_STATUS_BASE_IDX 2
+#define mmDIG2_HDMI_GC 0x227b
+#define mmDIG2_HDMI_GC_BASE_IDX 2
+#define mmDIG2_AFMT_AUDIO_PACKET_CONTROL2 0x227c
+#define mmDIG2_AFMT_AUDIO_PACKET_CONTROL2_BASE_IDX 2
+#define mmDIG2_AFMT_ISRC1_0 0x227d
+#define mmDIG2_AFMT_ISRC1_0_BASE_IDX 2
+#define mmDIG2_AFMT_ISRC1_1 0x227e
+#define mmDIG2_AFMT_ISRC1_1_BASE_IDX 2
+#define mmDIG2_AFMT_ISRC1_2 0x227f
+#define mmDIG2_AFMT_ISRC1_2_BASE_IDX 2
+#define mmDIG2_AFMT_ISRC1_3 0x2280
+#define mmDIG2_AFMT_ISRC1_3_BASE_IDX 2
+#define mmDIG2_AFMT_ISRC1_4 0x2281
+#define mmDIG2_AFMT_ISRC1_4_BASE_IDX 2
+#define mmDIG2_AFMT_ISRC2_0 0x2282
+#define mmDIG2_AFMT_ISRC2_0_BASE_IDX 2
+#define mmDIG2_AFMT_ISRC2_1 0x2283
+#define mmDIG2_AFMT_ISRC2_1_BASE_IDX 2
+#define mmDIG2_AFMT_ISRC2_2 0x2284
+#define mmDIG2_AFMT_ISRC2_2_BASE_IDX 2
+#define mmDIG2_AFMT_ISRC2_3 0x2285
+#define mmDIG2_AFMT_ISRC2_3_BASE_IDX 2
+#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL2 0x2286
+#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL2_BASE_IDX 2
+#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL3 0x2287
+#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL3_BASE_IDX 2
+#define mmDIG2_HDMI_DB_CONTROL 0x2288
+#define mmDIG2_HDMI_DB_CONTROL_BASE_IDX 2
+#define mmDIG2_DME_CONTROL 0x2289
+#define mmDIG2_DME_CONTROL_BASE_IDX 2
+#define mmDIG2_AFMT_MPEG_INFO0 0x228a
+#define mmDIG2_AFMT_MPEG_INFO0_BASE_IDX 2
+#define mmDIG2_AFMT_MPEG_INFO1 0x228b
+#define mmDIG2_AFMT_MPEG_INFO1_BASE_IDX 2
+#define mmDIG2_AFMT_GENERIC_HDR 0x228c
+#define mmDIG2_AFMT_GENERIC_HDR_BASE_IDX 2
+#define mmDIG2_AFMT_GENERIC_0 0x228d
+#define mmDIG2_AFMT_GENERIC_0_BASE_IDX 2
+#define mmDIG2_AFMT_GENERIC_1 0x228e
+#define mmDIG2_AFMT_GENERIC_1_BASE_IDX 2
+#define mmDIG2_AFMT_GENERIC_2 0x228f
+#define mmDIG2_AFMT_GENERIC_2_BASE_IDX 2
+#define mmDIG2_AFMT_GENERIC_3 0x2290
+#define mmDIG2_AFMT_GENERIC_3_BASE_IDX 2
+#define mmDIG2_AFMT_GENERIC_4 0x2291
+#define mmDIG2_AFMT_GENERIC_4_BASE_IDX 2
+#define mmDIG2_AFMT_GENERIC_5 0x2292
+#define mmDIG2_AFMT_GENERIC_5_BASE_IDX 2
+#define mmDIG2_AFMT_GENERIC_6 0x2293
+#define mmDIG2_AFMT_GENERIC_6_BASE_IDX 2
+#define mmDIG2_AFMT_GENERIC_7 0x2294
+#define mmDIG2_AFMT_GENERIC_7_BASE_IDX 2
+#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL1 0x2295
+#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL1_BASE_IDX 2
+#define mmDIG2_HDMI_ACR_32_0 0x2296
+#define mmDIG2_HDMI_ACR_32_0_BASE_IDX 2
+#define mmDIG2_HDMI_ACR_32_1 0x2297
+#define mmDIG2_HDMI_ACR_32_1_BASE_IDX 2
+#define mmDIG2_HDMI_ACR_44_0 0x2298
+#define mmDIG2_HDMI_ACR_44_0_BASE_IDX 2
+#define mmDIG2_HDMI_ACR_44_1 0x2299
+#define mmDIG2_HDMI_ACR_44_1_BASE_IDX 2
+#define mmDIG2_HDMI_ACR_48_0 0x229a
+#define mmDIG2_HDMI_ACR_48_0_BASE_IDX 2
+#define mmDIG2_HDMI_ACR_48_1 0x229b
+#define mmDIG2_HDMI_ACR_48_1_BASE_IDX 2
+#define mmDIG2_HDMI_ACR_STATUS_0 0x229c
+#define mmDIG2_HDMI_ACR_STATUS_0_BASE_IDX 2
+#define mmDIG2_HDMI_ACR_STATUS_1 0x229d
+#define mmDIG2_HDMI_ACR_STATUS_1_BASE_IDX 2
+#define mmDIG2_AFMT_AUDIO_INFO0 0x229e
+#define mmDIG2_AFMT_AUDIO_INFO0_BASE_IDX 2
+#define mmDIG2_AFMT_AUDIO_INFO1 0x229f
+#define mmDIG2_AFMT_AUDIO_INFO1_BASE_IDX 2
+#define mmDIG2_AFMT_60958_0 0x22a0
+#define mmDIG2_AFMT_60958_0_BASE_IDX 2
+#define mmDIG2_AFMT_60958_1 0x22a1
+#define mmDIG2_AFMT_60958_1_BASE_IDX 2
+#define mmDIG2_AFMT_AUDIO_CRC_CONTROL 0x22a2
+#define mmDIG2_AFMT_AUDIO_CRC_CONTROL_BASE_IDX 2
+#define mmDIG2_AFMT_RAMP_CONTROL0 0x22a3
+#define mmDIG2_AFMT_RAMP_CONTROL0_BASE_IDX 2
+#define mmDIG2_AFMT_RAMP_CONTROL1 0x22a4
+#define mmDIG2_AFMT_RAMP_CONTROL1_BASE_IDX 2
+#define mmDIG2_AFMT_RAMP_CONTROL2 0x22a5
+#define mmDIG2_AFMT_RAMP_CONTROL2_BASE_IDX 2
+#define mmDIG2_AFMT_RAMP_CONTROL3 0x22a6
+#define mmDIG2_AFMT_RAMP_CONTROL3_BASE_IDX 2
+#define mmDIG2_AFMT_60958_2 0x22a7
+#define mmDIG2_AFMT_60958_2_BASE_IDX 2
+#define mmDIG2_AFMT_AUDIO_CRC_RESULT 0x22a8
+#define mmDIG2_AFMT_AUDIO_CRC_RESULT_BASE_IDX 2
+#define mmDIG2_AFMT_STATUS 0x22a9
+#define mmDIG2_AFMT_STATUS_BASE_IDX 2
+#define mmDIG2_AFMT_AUDIO_PACKET_CONTROL 0x22aa
+#define mmDIG2_AFMT_AUDIO_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG2_AFMT_VBI_PACKET_CONTROL 0x22ab
+#define mmDIG2_AFMT_VBI_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG2_AFMT_INFOFRAME_CONTROL0 0x22ac
+#define mmDIG2_AFMT_INFOFRAME_CONTROL0_BASE_IDX 2
+#define mmDIG2_AFMT_AUDIO_SRC_CONTROL 0x22ad
+#define mmDIG2_AFMT_AUDIO_SRC_CONTROL_BASE_IDX 2
+#define mmDIG2_DIG_BE_CNTL 0x22af
+#define mmDIG2_DIG_BE_CNTL_BASE_IDX 2
+#define mmDIG2_DIG_BE_EN_CNTL 0x22b0
+#define mmDIG2_DIG_BE_EN_CNTL_BASE_IDX 2
+#define mmDIG2_TMDS_CNTL 0x22d3
+#define mmDIG2_TMDS_CNTL_BASE_IDX 2
+#define mmDIG2_TMDS_CONTROL_CHAR 0x22d4
+#define mmDIG2_TMDS_CONTROL_CHAR_BASE_IDX 2
+#define mmDIG2_TMDS_CONTROL0_FEEDBACK 0x22d5
+#define mmDIG2_TMDS_CONTROL0_FEEDBACK_BASE_IDX 2
+#define mmDIG2_TMDS_STEREOSYNC_CTL_SEL 0x22d6
+#define mmDIG2_TMDS_STEREOSYNC_CTL_SEL_BASE_IDX 2
+#define mmDIG2_TMDS_SYNC_CHAR_PATTERN_0_1 0x22d7
+#define mmDIG2_TMDS_SYNC_CHAR_PATTERN_0_1_BASE_IDX 2
+#define mmDIG2_TMDS_SYNC_CHAR_PATTERN_2_3 0x22d8
+#define mmDIG2_TMDS_SYNC_CHAR_PATTERN_2_3_BASE_IDX 2
+#define mmDIG2_TMDS_CTL_BITS 0x22da
+#define mmDIG2_TMDS_CTL_BITS_BASE_IDX 2
+#define mmDIG2_TMDS_DCBALANCER_CONTROL 0x22db
+#define mmDIG2_TMDS_DCBALANCER_CONTROL_BASE_IDX 2
+#define mmDIG2_TMDS_SYNC_DCBALANCE_CHAR 0x22dc
+#define mmDIG2_TMDS_SYNC_DCBALANCE_CHAR_BASE_IDX 2
+#define mmDIG2_TMDS_CTL0_1_GEN_CNTL 0x22dd
+#define mmDIG2_TMDS_CTL0_1_GEN_CNTL_BASE_IDX 2
+#define mmDIG2_TMDS_CTL2_3_GEN_CNTL 0x22de
+#define mmDIG2_TMDS_CTL2_3_GEN_CNTL_BASE_IDX 2
+#define mmDIG2_DIG_VERSION 0x22e0
+#define mmDIG2_DIG_VERSION_BASE_IDX 2
+#define mmDIG2_DIG_LANE_ENABLE 0x22e1
+#define mmDIG2_DIG_LANE_ENABLE_BASE_IDX 2
+#define mmDIG2_AFMT_CNTL 0x22e6
+#define mmDIG2_AFMT_CNTL_BASE_IDX 2
+#define mmDIG2_AFMT_VBI_PACKET_CONTROL1 0x22e7
+#define mmDIG2_AFMT_VBI_PACKET_CONTROL1_BASE_IDX 2
+#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL5 0x22f6
+#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL5_BASE_IDX 2
+#define mmDIG2_FORCE_DIG_DISABLE 0x22f7
+#define mmDIG2_FORCE_DIG_DISABLE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dp2_dispdec
+// base address: 0x800
+#define mmDP2_DP_LINK_CNTL 0x2308
+#define mmDP2_DP_LINK_CNTL_BASE_IDX 2
+#define mmDP2_DP_PIXEL_FORMAT 0x2309
+#define mmDP2_DP_PIXEL_FORMAT_BASE_IDX 2
+#define mmDP2_DP_MSA_COLORIMETRY 0x230a
+#define mmDP2_DP_MSA_COLORIMETRY_BASE_IDX 2
+#define mmDP2_DP_CONFIG 0x230b
+#define mmDP2_DP_CONFIG_BASE_IDX 2
+#define mmDP2_DP_VID_STREAM_CNTL 0x230c
+#define mmDP2_DP_VID_STREAM_CNTL_BASE_IDX 2
+#define mmDP2_DP_STEER_FIFO 0x230d
+#define mmDP2_DP_STEER_FIFO_BASE_IDX 2
+#define mmDP2_DP_MSA_MISC 0x230e
+#define mmDP2_DP_MSA_MISC_BASE_IDX 2
+#define mmDP2_DP_VID_TIMING 0x2310
+#define mmDP2_DP_VID_TIMING_BASE_IDX 2
+#define mmDP2_DP_VID_N 0x2311
+#define mmDP2_DP_VID_N_BASE_IDX 2
+#define mmDP2_DP_VID_M 0x2312
+#define mmDP2_DP_VID_M_BASE_IDX 2
+#define mmDP2_DP_LINK_FRAMING_CNTL 0x2313
+#define mmDP2_DP_LINK_FRAMING_CNTL_BASE_IDX 2
+#define mmDP2_DP_HBR2_EYE_PATTERN 0x2314
+#define mmDP2_DP_HBR2_EYE_PATTERN_BASE_IDX 2
+#define mmDP2_DP_VID_MSA_VBID 0x2315
+#define mmDP2_DP_VID_MSA_VBID_BASE_IDX 2
+#define mmDP2_DP_VID_INTERRUPT_CNTL 0x2316
+#define mmDP2_DP_VID_INTERRUPT_CNTL_BASE_IDX 2
+#define mmDP2_DP_DPHY_CNTL 0x2317
+#define mmDP2_DP_DPHY_CNTL_BASE_IDX 2
+#define mmDP2_DP_DPHY_TRAINING_PATTERN_SEL 0x2318
+#define mmDP2_DP_DPHY_TRAINING_PATTERN_SEL_BASE_IDX 2
+#define mmDP2_DP_DPHY_SYM0 0x2319
+#define mmDP2_DP_DPHY_SYM0_BASE_IDX 2
+#define mmDP2_DP_DPHY_SYM1 0x231a
+#define mmDP2_DP_DPHY_SYM1_BASE_IDX 2
+#define mmDP2_DP_DPHY_SYM2 0x231b
+#define mmDP2_DP_DPHY_SYM2_BASE_IDX 2
+#define mmDP2_DP_DPHY_8B10B_CNTL 0x231c
+#define mmDP2_DP_DPHY_8B10B_CNTL_BASE_IDX 2
+#define mmDP2_DP_DPHY_PRBS_CNTL 0x231d
+#define mmDP2_DP_DPHY_PRBS_CNTL_BASE_IDX 2
+#define mmDP2_DP_DPHY_SCRAM_CNTL 0x231e
+#define mmDP2_DP_DPHY_SCRAM_CNTL_BASE_IDX 2
+#define mmDP2_DP_DPHY_CRC_EN 0x231f
+#define mmDP2_DP_DPHY_CRC_EN_BASE_IDX 2
+#define mmDP2_DP_DPHY_CRC_CNTL 0x2320
+#define mmDP2_DP_DPHY_CRC_CNTL_BASE_IDX 2
+#define mmDP2_DP_DPHY_CRC_RESULT 0x2321
+#define mmDP2_DP_DPHY_CRC_RESULT_BASE_IDX 2
+#define mmDP2_DP_DPHY_CRC_MST_CNTL 0x2322
+#define mmDP2_DP_DPHY_CRC_MST_CNTL_BASE_IDX 2
+#define mmDP2_DP_DPHY_CRC_MST_STATUS 0x2323
+#define mmDP2_DP_DPHY_CRC_MST_STATUS_BASE_IDX 2
+#define mmDP2_DP_DPHY_FAST_TRAINING 0x2324
+#define mmDP2_DP_DPHY_FAST_TRAINING_BASE_IDX 2
+#define mmDP2_DP_DPHY_FAST_TRAINING_STATUS 0x2325
+#define mmDP2_DP_DPHY_FAST_TRAINING_STATUS_BASE_IDX 2
+#define mmDP2_DP_SEC_CNTL 0x232b
+#define mmDP2_DP_SEC_CNTL_BASE_IDX 2
+#define mmDP2_DP_SEC_CNTL1 0x232c
+#define mmDP2_DP_SEC_CNTL1_BASE_IDX 2
+#define mmDP2_DP_SEC_FRAMING1 0x232d
+#define mmDP2_DP_SEC_FRAMING1_BASE_IDX 2
+#define mmDP2_DP_SEC_FRAMING2 0x232e
+#define mmDP2_DP_SEC_FRAMING2_BASE_IDX 2
+#define mmDP2_DP_SEC_FRAMING3 0x232f
+#define mmDP2_DP_SEC_FRAMING3_BASE_IDX 2
+#define mmDP2_DP_SEC_FRAMING4 0x2330
+#define mmDP2_DP_SEC_FRAMING4_BASE_IDX 2
+#define mmDP2_DP_SEC_AUD_N 0x2331
+#define mmDP2_DP_SEC_AUD_N_BASE_IDX 2
+#define mmDP2_DP_SEC_AUD_N_READBACK 0x2332
+#define mmDP2_DP_SEC_AUD_N_READBACK_BASE_IDX 2
+#define mmDP2_DP_SEC_AUD_M 0x2333
+#define mmDP2_DP_SEC_AUD_M_BASE_IDX 2
+#define mmDP2_DP_SEC_AUD_M_READBACK 0x2334
+#define mmDP2_DP_SEC_AUD_M_READBACK_BASE_IDX 2
+#define mmDP2_DP_SEC_TIMESTAMP 0x2335
+#define mmDP2_DP_SEC_TIMESTAMP_BASE_IDX 2
+#define mmDP2_DP_SEC_PACKET_CNTL 0x2336
+#define mmDP2_DP_SEC_PACKET_CNTL_BASE_IDX 2
+#define mmDP2_DP_MSE_RATE_CNTL 0x2337
+#define mmDP2_DP_MSE_RATE_CNTL_BASE_IDX 2
+#define mmDP2_DP_MSE_RATE_UPDATE 0x2339
+#define mmDP2_DP_MSE_RATE_UPDATE_BASE_IDX 2
+#define mmDP2_DP_MSE_SAT0 0x233a
+#define mmDP2_DP_MSE_SAT0_BASE_IDX 2
+#define mmDP2_DP_MSE_SAT1 0x233b
+#define mmDP2_DP_MSE_SAT1_BASE_IDX 2
+#define mmDP2_DP_MSE_SAT2 0x233c
+#define mmDP2_DP_MSE_SAT2_BASE_IDX 2
+#define mmDP2_DP_MSE_SAT_UPDATE 0x233d
+#define mmDP2_DP_MSE_SAT_UPDATE_BASE_IDX 2
+#define mmDP2_DP_MSE_LINK_TIMING 0x233e
+#define mmDP2_DP_MSE_LINK_TIMING_BASE_IDX 2
+#define mmDP2_DP_MSE_MISC_CNTL 0x233f
+#define mmDP2_DP_MSE_MISC_CNTL_BASE_IDX 2
+#define mmDP2_DP_DPHY_BS_SR_SWAP_CNTL 0x2344
+#define mmDP2_DP_DPHY_BS_SR_SWAP_CNTL_BASE_IDX 2
+#define mmDP2_DP_DPHY_HBR2_PATTERN_CONTROL 0x2345
+#define mmDP2_DP_DPHY_HBR2_PATTERN_CONTROL_BASE_IDX 2
+#define mmDP2_DP_MSE_SAT0_STATUS 0x2347
+#define mmDP2_DP_MSE_SAT0_STATUS_BASE_IDX 2
+#define mmDP2_DP_MSE_SAT1_STATUS 0x2348
+#define mmDP2_DP_MSE_SAT1_STATUS_BASE_IDX 2
+#define mmDP2_DP_MSE_SAT2_STATUS 0x2349
+#define mmDP2_DP_MSE_SAT2_STATUS_BASE_IDX 2
+#define mmDP2_DP_MSA_TIMING_PARAM1 0x234c
+#define mmDP2_DP_MSA_TIMING_PARAM1_BASE_IDX 2
+#define mmDP2_DP_MSA_TIMING_PARAM2 0x234d
+#define mmDP2_DP_MSA_TIMING_PARAM2_BASE_IDX 2
+#define mmDP2_DP_MSA_TIMING_PARAM3 0x234e
+#define mmDP2_DP_MSA_TIMING_PARAM3_BASE_IDX 2
+#define mmDP2_DP_MSA_TIMING_PARAM4 0x234f
+#define mmDP2_DP_MSA_TIMING_PARAM4_BASE_IDX 2
+#define mmDP2_DP_MSO_CNTL 0x2350
+#define mmDP2_DP_MSO_CNTL_BASE_IDX 2
+#define mmDP2_DP_MSO_CNTL1 0x2351
+#define mmDP2_DP_MSO_CNTL1_BASE_IDX 2
+#define mmDP2_DP_DSC_CNTL 0x2352
+#define mmDP2_DP_DSC_CNTL_BASE_IDX 2
+#define mmDP2_DP_SEC_CNTL2 0x2353
+#define mmDP2_DP_SEC_CNTL2_BASE_IDX 2
+#define mmDP2_DP_SEC_CNTL3 0x2354
+#define mmDP2_DP_SEC_CNTL3_BASE_IDX 2
+#define mmDP2_DP_SEC_CNTL4 0x2355
+#define mmDP2_DP_SEC_CNTL4_BASE_IDX 2
+#define mmDP2_DP_SEC_CNTL5 0x2356
+#define mmDP2_DP_SEC_CNTL5_BASE_IDX 2
+#define mmDP2_DP_SEC_CNTL6 0x2357
+#define mmDP2_DP_SEC_CNTL6_BASE_IDX 2
+#define mmDP2_DP_SEC_CNTL7 0x2358
+#define mmDP2_DP_SEC_CNTL7_BASE_IDX 2
+#define mmDP2_DP_DB_CNTL 0x2359
+#define mmDP2_DP_DB_CNTL_BASE_IDX 2
+#define mmDP2_DP_MSA_VBID_MISC 0x235a
+#define mmDP2_DP_MSA_VBID_MISC_BASE_IDX 2
+#define mmDP2_DP_SEC_METADATA_TRANSMISSION 0x235b
+#define mmDP2_DP_SEC_METADATA_TRANSMISSION_BASE_IDX 2
+#define mmDP2_DP_DSC_BYTES_PER_PIXEL 0x235c
+#define mmDP2_DP_DSC_BYTES_PER_PIXEL_BASE_IDX 2
+#define mmDP2_DP_ALPM_CNTL 0x235d
+#define mmDP2_DP_ALPM_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dig3_dispdec
+// base address: 0xc00
+#define mmDIG3_DIG_FE_CNTL 0x2368
+#define mmDIG3_DIG_FE_CNTL_BASE_IDX 2
+#define mmDIG3_DIG_OUTPUT_CRC_CNTL 0x2369
+#define mmDIG3_DIG_OUTPUT_CRC_CNTL_BASE_IDX 2
+#define mmDIG3_DIG_OUTPUT_CRC_RESULT 0x236a
+#define mmDIG3_DIG_OUTPUT_CRC_RESULT_BASE_IDX 2
+#define mmDIG3_DIG_CLOCK_PATTERN 0x236b
+#define mmDIG3_DIG_CLOCK_PATTERN_BASE_IDX 2
+#define mmDIG3_DIG_TEST_PATTERN 0x236c
+#define mmDIG3_DIG_TEST_PATTERN_BASE_IDX 2
+#define mmDIG3_DIG_RANDOM_PATTERN_SEED 0x236d
+#define mmDIG3_DIG_RANDOM_PATTERN_SEED_BASE_IDX 2
+#define mmDIG3_DIG_FIFO_STATUS 0x236e
+#define mmDIG3_DIG_FIFO_STATUS_BASE_IDX 2
+#define mmDIG3_HDMI_METADATA_PACKET_CONTROL 0x236f
+#define mmDIG3_HDMI_METADATA_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL4 0x2370
+#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL4_BASE_IDX 2
+#define mmDIG3_HDMI_CONTROL 0x2371
+#define mmDIG3_HDMI_CONTROL_BASE_IDX 2
+#define mmDIG3_HDMI_STATUS 0x2372
+#define mmDIG3_HDMI_STATUS_BASE_IDX 2
+#define mmDIG3_HDMI_AUDIO_PACKET_CONTROL 0x2373
+#define mmDIG3_HDMI_AUDIO_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG3_HDMI_ACR_PACKET_CONTROL 0x2374
+#define mmDIG3_HDMI_ACR_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG3_HDMI_VBI_PACKET_CONTROL 0x2375
+#define mmDIG3_HDMI_VBI_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG3_HDMI_INFOFRAME_CONTROL0 0x2376
+#define mmDIG3_HDMI_INFOFRAME_CONTROL0_BASE_IDX 2
+#define mmDIG3_HDMI_INFOFRAME_CONTROL1 0x2377
+#define mmDIG3_HDMI_INFOFRAME_CONTROL1_BASE_IDX 2
+#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL0 0x2378
+#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL0_BASE_IDX 2
+#define mmDIG3_AFMT_INTERRUPT_STATUS 0x2379
+#define mmDIG3_AFMT_INTERRUPT_STATUS_BASE_IDX 2
+#define mmDIG3_HDMI_GC 0x237b
+#define mmDIG3_HDMI_GC_BASE_IDX 2
+#define mmDIG3_AFMT_AUDIO_PACKET_CONTROL2 0x237c
+#define mmDIG3_AFMT_AUDIO_PACKET_CONTROL2_BASE_IDX 2
+#define mmDIG3_AFMT_ISRC1_0 0x237d
+#define mmDIG3_AFMT_ISRC1_0_BASE_IDX 2
+#define mmDIG3_AFMT_ISRC1_1 0x237e
+#define mmDIG3_AFMT_ISRC1_1_BASE_IDX 2
+#define mmDIG3_AFMT_ISRC1_2 0x237f
+#define mmDIG3_AFMT_ISRC1_2_BASE_IDX 2
+#define mmDIG3_AFMT_ISRC1_3 0x2380
+#define mmDIG3_AFMT_ISRC1_3_BASE_IDX 2
+#define mmDIG3_AFMT_ISRC1_4 0x2381
+#define mmDIG3_AFMT_ISRC1_4_BASE_IDX 2
+#define mmDIG3_AFMT_ISRC2_0 0x2382
+#define mmDIG3_AFMT_ISRC2_0_BASE_IDX 2
+#define mmDIG3_AFMT_ISRC2_1 0x2383
+#define mmDIG3_AFMT_ISRC2_1_BASE_IDX 2
+#define mmDIG3_AFMT_ISRC2_2 0x2384
+#define mmDIG3_AFMT_ISRC2_2_BASE_IDX 2
+#define mmDIG3_AFMT_ISRC2_3 0x2385
+#define mmDIG3_AFMT_ISRC2_3_BASE_IDX 2
+#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL2 0x2386
+#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL2_BASE_IDX 2
+#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL3 0x2387
+#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL3_BASE_IDX 2
+#define mmDIG3_HDMI_DB_CONTROL 0x2388
+#define mmDIG3_HDMI_DB_CONTROL_BASE_IDX 2
+#define mmDIG3_DME_CONTROL 0x2389
+#define mmDIG3_DME_CONTROL_BASE_IDX 2
+#define mmDIG3_AFMT_MPEG_INFO0 0x238a
+#define mmDIG3_AFMT_MPEG_INFO0_BASE_IDX 2
+#define mmDIG3_AFMT_MPEG_INFO1 0x238b
+#define mmDIG3_AFMT_MPEG_INFO1_BASE_IDX 2
+#define mmDIG3_AFMT_GENERIC_HDR 0x238c
+#define mmDIG3_AFMT_GENERIC_HDR_BASE_IDX 2
+#define mmDIG3_AFMT_GENERIC_0 0x238d
+#define mmDIG3_AFMT_GENERIC_0_BASE_IDX 2
+#define mmDIG3_AFMT_GENERIC_1 0x238e
+#define mmDIG3_AFMT_GENERIC_1_BASE_IDX 2
+#define mmDIG3_AFMT_GENERIC_2 0x238f
+#define mmDIG3_AFMT_GENERIC_2_BASE_IDX 2
+#define mmDIG3_AFMT_GENERIC_3 0x2390
+#define mmDIG3_AFMT_GENERIC_3_BASE_IDX 2
+#define mmDIG3_AFMT_GENERIC_4 0x2391
+#define mmDIG3_AFMT_GENERIC_4_BASE_IDX 2
+#define mmDIG3_AFMT_GENERIC_5 0x2392
+#define mmDIG3_AFMT_GENERIC_5_BASE_IDX 2
+#define mmDIG3_AFMT_GENERIC_6 0x2393
+#define mmDIG3_AFMT_GENERIC_6_BASE_IDX 2
+#define mmDIG3_AFMT_GENERIC_7 0x2394
+#define mmDIG3_AFMT_GENERIC_7_BASE_IDX 2
+#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL1 0x2395
+#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL1_BASE_IDX 2
+#define mmDIG3_HDMI_ACR_32_0 0x2396
+#define mmDIG3_HDMI_ACR_32_0_BASE_IDX 2
+#define mmDIG3_HDMI_ACR_32_1 0x2397
+#define mmDIG3_HDMI_ACR_32_1_BASE_IDX 2
+#define mmDIG3_HDMI_ACR_44_0 0x2398
+#define mmDIG3_HDMI_ACR_44_0_BASE_IDX 2
+#define mmDIG3_HDMI_ACR_44_1 0x2399
+#define mmDIG3_HDMI_ACR_44_1_BASE_IDX 2
+#define mmDIG3_HDMI_ACR_48_0 0x239a
+#define mmDIG3_HDMI_ACR_48_0_BASE_IDX 2
+#define mmDIG3_HDMI_ACR_48_1 0x239b
+#define mmDIG3_HDMI_ACR_48_1_BASE_IDX 2
+#define mmDIG3_HDMI_ACR_STATUS_0 0x239c
+#define mmDIG3_HDMI_ACR_STATUS_0_BASE_IDX 2
+#define mmDIG3_HDMI_ACR_STATUS_1 0x239d
+#define mmDIG3_HDMI_ACR_STATUS_1_BASE_IDX 2
+#define mmDIG3_AFMT_AUDIO_INFO0 0x239e
+#define mmDIG3_AFMT_AUDIO_INFO0_BASE_IDX 2
+#define mmDIG3_AFMT_AUDIO_INFO1 0x239f
+#define mmDIG3_AFMT_AUDIO_INFO1_BASE_IDX 2
+#define mmDIG3_AFMT_60958_0 0x23a0
+#define mmDIG3_AFMT_60958_0_BASE_IDX 2
+#define mmDIG3_AFMT_60958_1 0x23a1
+#define mmDIG3_AFMT_60958_1_BASE_IDX 2
+#define mmDIG3_AFMT_AUDIO_CRC_CONTROL 0x23a2
+#define mmDIG3_AFMT_AUDIO_CRC_CONTROL_BASE_IDX 2
+#define mmDIG3_AFMT_RAMP_CONTROL0 0x23a3
+#define mmDIG3_AFMT_RAMP_CONTROL0_BASE_IDX 2
+#define mmDIG3_AFMT_RAMP_CONTROL1 0x23a4
+#define mmDIG3_AFMT_RAMP_CONTROL1_BASE_IDX 2
+#define mmDIG3_AFMT_RAMP_CONTROL2 0x23a5
+#define mmDIG3_AFMT_RAMP_CONTROL2_BASE_IDX 2
+#define mmDIG3_AFMT_RAMP_CONTROL3 0x23a6
+#define mmDIG3_AFMT_RAMP_CONTROL3_BASE_IDX 2
+#define mmDIG3_AFMT_60958_2 0x23a7
+#define mmDIG3_AFMT_60958_2_BASE_IDX 2
+#define mmDIG3_AFMT_AUDIO_CRC_RESULT 0x23a8
+#define mmDIG3_AFMT_AUDIO_CRC_RESULT_BASE_IDX 2
+#define mmDIG3_AFMT_STATUS 0x23a9
+#define mmDIG3_AFMT_STATUS_BASE_IDX 2
+#define mmDIG3_AFMT_AUDIO_PACKET_CONTROL 0x23aa
+#define mmDIG3_AFMT_AUDIO_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG3_AFMT_VBI_PACKET_CONTROL 0x23ab
+#define mmDIG3_AFMT_VBI_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG3_AFMT_INFOFRAME_CONTROL0 0x23ac
+#define mmDIG3_AFMT_INFOFRAME_CONTROL0_BASE_IDX 2
+#define mmDIG3_AFMT_AUDIO_SRC_CONTROL 0x23ad
+#define mmDIG3_AFMT_AUDIO_SRC_CONTROL_BASE_IDX 2
+#define mmDIG3_DIG_BE_CNTL 0x23af
+#define mmDIG3_DIG_BE_CNTL_BASE_IDX 2
+#define mmDIG3_DIG_BE_EN_CNTL 0x23b0
+#define mmDIG3_DIG_BE_EN_CNTL_BASE_IDX 2
+#define mmDIG3_TMDS_CNTL 0x23d3
+#define mmDIG3_TMDS_CNTL_BASE_IDX 2
+#define mmDIG3_TMDS_CONTROL_CHAR 0x23d4
+#define mmDIG3_TMDS_CONTROL_CHAR_BASE_IDX 2
+#define mmDIG3_TMDS_CONTROL0_FEEDBACK 0x23d5
+#define mmDIG3_TMDS_CONTROL0_FEEDBACK_BASE_IDX 2
+#define mmDIG3_TMDS_STEREOSYNC_CTL_SEL 0x23d6
+#define mmDIG3_TMDS_STEREOSYNC_CTL_SEL_BASE_IDX 2
+#define mmDIG3_TMDS_SYNC_CHAR_PATTERN_0_1 0x23d7
+#define mmDIG3_TMDS_SYNC_CHAR_PATTERN_0_1_BASE_IDX 2
+#define mmDIG3_TMDS_SYNC_CHAR_PATTERN_2_3 0x23d8
+#define mmDIG3_TMDS_SYNC_CHAR_PATTERN_2_3_BASE_IDX 2
+#define mmDIG3_TMDS_CTL_BITS 0x23da
+#define mmDIG3_TMDS_CTL_BITS_BASE_IDX 2
+#define mmDIG3_TMDS_DCBALANCER_CONTROL 0x23db
+#define mmDIG3_TMDS_DCBALANCER_CONTROL_BASE_IDX 2
+#define mmDIG3_TMDS_SYNC_DCBALANCE_CHAR 0x23dc
+#define mmDIG3_TMDS_SYNC_DCBALANCE_CHAR_BASE_IDX 2
+#define mmDIG3_TMDS_CTL0_1_GEN_CNTL 0x23dd
+#define mmDIG3_TMDS_CTL0_1_GEN_CNTL_BASE_IDX 2
+#define mmDIG3_TMDS_CTL2_3_GEN_CNTL 0x23de
+#define mmDIG3_TMDS_CTL2_3_GEN_CNTL_BASE_IDX 2
+#define mmDIG3_DIG_VERSION 0x23e0
+#define mmDIG3_DIG_VERSION_BASE_IDX 2
+#define mmDIG3_DIG_LANE_ENABLE 0x23e1
+#define mmDIG3_DIG_LANE_ENABLE_BASE_IDX 2
+#define mmDIG3_AFMT_CNTL 0x23e6
+#define mmDIG3_AFMT_CNTL_BASE_IDX 2
+#define mmDIG3_AFMT_VBI_PACKET_CONTROL1 0x23e7
+#define mmDIG3_AFMT_VBI_PACKET_CONTROL1_BASE_IDX 2
+#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL5 0x23f6
+#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL5_BASE_IDX 2
+#define mmDIG3_FORCE_DIG_DISABLE 0x23f7
+#define mmDIG3_FORCE_DIG_DISABLE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dp3_dispdec
+// base address: 0xc00
+#define mmDP3_DP_LINK_CNTL 0x2408
+#define mmDP3_DP_LINK_CNTL_BASE_IDX 2
+#define mmDP3_DP_PIXEL_FORMAT 0x2409
+#define mmDP3_DP_PIXEL_FORMAT_BASE_IDX 2
+#define mmDP3_DP_MSA_COLORIMETRY 0x240a
+#define mmDP3_DP_MSA_COLORIMETRY_BASE_IDX 2
+#define mmDP3_DP_CONFIG 0x240b
+#define mmDP3_DP_CONFIG_BASE_IDX 2
+#define mmDP3_DP_VID_STREAM_CNTL 0x240c
+#define mmDP3_DP_VID_STREAM_CNTL_BASE_IDX 2
+#define mmDP3_DP_STEER_FIFO 0x240d
+#define mmDP3_DP_STEER_FIFO_BASE_IDX 2
+#define mmDP3_DP_MSA_MISC 0x240e
+#define mmDP3_DP_MSA_MISC_BASE_IDX 2
+#define mmDP3_DP_VID_TIMING 0x2410
+#define mmDP3_DP_VID_TIMING_BASE_IDX 2
+#define mmDP3_DP_VID_N 0x2411
+#define mmDP3_DP_VID_N_BASE_IDX 2
+#define mmDP3_DP_VID_M 0x2412
+#define mmDP3_DP_VID_M_BASE_IDX 2
+#define mmDP3_DP_LINK_FRAMING_CNTL 0x2413
+#define mmDP3_DP_LINK_FRAMING_CNTL_BASE_IDX 2
+#define mmDP3_DP_HBR2_EYE_PATTERN 0x2414
+#define mmDP3_DP_HBR2_EYE_PATTERN_BASE_IDX 2
+#define mmDP3_DP_VID_MSA_VBID 0x2415
+#define mmDP3_DP_VID_MSA_VBID_BASE_IDX 2
+#define mmDP3_DP_VID_INTERRUPT_CNTL 0x2416
+#define mmDP3_DP_VID_INTERRUPT_CNTL_BASE_IDX 2
+#define mmDP3_DP_DPHY_CNTL 0x2417
+#define mmDP3_DP_DPHY_CNTL_BASE_IDX 2
+#define mmDP3_DP_DPHY_TRAINING_PATTERN_SEL 0x2418
+#define mmDP3_DP_DPHY_TRAINING_PATTERN_SEL_BASE_IDX 2
+#define mmDP3_DP_DPHY_SYM0 0x2419
+#define mmDP3_DP_DPHY_SYM0_BASE_IDX 2
+#define mmDP3_DP_DPHY_SYM1 0x241a
+#define mmDP3_DP_DPHY_SYM1_BASE_IDX 2
+#define mmDP3_DP_DPHY_SYM2 0x241b
+#define mmDP3_DP_DPHY_SYM2_BASE_IDX 2
+#define mmDP3_DP_DPHY_8B10B_CNTL 0x241c
+#define mmDP3_DP_DPHY_8B10B_CNTL_BASE_IDX 2
+#define mmDP3_DP_DPHY_PRBS_CNTL 0x241d
+#define mmDP3_DP_DPHY_PRBS_CNTL_BASE_IDX 2
+#define mmDP3_DP_DPHY_SCRAM_CNTL 0x241e
+#define mmDP3_DP_DPHY_SCRAM_CNTL_BASE_IDX 2
+#define mmDP3_DP_DPHY_CRC_EN 0x241f
+#define mmDP3_DP_DPHY_CRC_EN_BASE_IDX 2
+#define mmDP3_DP_DPHY_CRC_CNTL 0x2420
+#define mmDP3_DP_DPHY_CRC_CNTL_BASE_IDX 2
+#define mmDP3_DP_DPHY_CRC_RESULT 0x2421
+#define mmDP3_DP_DPHY_CRC_RESULT_BASE_IDX 2
+#define mmDP3_DP_DPHY_CRC_MST_CNTL 0x2422
+#define mmDP3_DP_DPHY_CRC_MST_CNTL_BASE_IDX 2
+#define mmDP3_DP_DPHY_CRC_MST_STATUS 0x2423
+#define mmDP3_DP_DPHY_CRC_MST_STATUS_BASE_IDX 2
+#define mmDP3_DP_DPHY_FAST_TRAINING 0x2424
+#define mmDP3_DP_DPHY_FAST_TRAINING_BASE_IDX 2
+#define mmDP3_DP_DPHY_FAST_TRAINING_STATUS 0x2425
+#define mmDP3_DP_DPHY_FAST_TRAINING_STATUS_BASE_IDX 2
+#define mmDP3_DP_SEC_CNTL 0x242b
+#define mmDP3_DP_SEC_CNTL_BASE_IDX 2
+#define mmDP3_DP_SEC_CNTL1 0x242c
+#define mmDP3_DP_SEC_CNTL1_BASE_IDX 2
+#define mmDP3_DP_SEC_FRAMING1 0x242d
+#define mmDP3_DP_SEC_FRAMING1_BASE_IDX 2
+#define mmDP3_DP_SEC_FRAMING2 0x242e
+#define mmDP3_DP_SEC_FRAMING2_BASE_IDX 2
+#define mmDP3_DP_SEC_FRAMING3 0x242f
+#define mmDP3_DP_SEC_FRAMING3_BASE_IDX 2
+#define mmDP3_DP_SEC_FRAMING4 0x2430
+#define mmDP3_DP_SEC_FRAMING4_BASE_IDX 2
+#define mmDP3_DP_SEC_AUD_N 0x2431
+#define mmDP3_DP_SEC_AUD_N_BASE_IDX 2
+#define mmDP3_DP_SEC_AUD_N_READBACK 0x2432
+#define mmDP3_DP_SEC_AUD_N_READBACK_BASE_IDX 2
+#define mmDP3_DP_SEC_AUD_M 0x2433
+#define mmDP3_DP_SEC_AUD_M_BASE_IDX 2
+#define mmDP3_DP_SEC_AUD_M_READBACK 0x2434
+#define mmDP3_DP_SEC_AUD_M_READBACK_BASE_IDX 2
+#define mmDP3_DP_SEC_TIMESTAMP 0x2435
+#define mmDP3_DP_SEC_TIMESTAMP_BASE_IDX 2
+#define mmDP3_DP_SEC_PACKET_CNTL 0x2436
+#define mmDP3_DP_SEC_PACKET_CNTL_BASE_IDX 2
+#define mmDP3_DP_MSE_RATE_CNTL 0x2437
+#define mmDP3_DP_MSE_RATE_CNTL_BASE_IDX 2
+#define mmDP3_DP_MSE_RATE_UPDATE 0x2439
+#define mmDP3_DP_MSE_RATE_UPDATE_BASE_IDX 2
+#define mmDP3_DP_MSE_SAT0 0x243a
+#define mmDP3_DP_MSE_SAT0_BASE_IDX 2
+#define mmDP3_DP_MSE_SAT1 0x243b
+#define mmDP3_DP_MSE_SAT1_BASE_IDX 2
+#define mmDP3_DP_MSE_SAT2 0x243c
+#define mmDP3_DP_MSE_SAT2_BASE_IDX 2
+#define mmDP3_DP_MSE_SAT_UPDATE 0x243d
+#define mmDP3_DP_MSE_SAT_UPDATE_BASE_IDX 2
+#define mmDP3_DP_MSE_LINK_TIMING 0x243e
+#define mmDP3_DP_MSE_LINK_TIMING_BASE_IDX 2
+#define mmDP3_DP_MSE_MISC_CNTL 0x243f
+#define mmDP3_DP_MSE_MISC_CNTL_BASE_IDX 2
+#define mmDP3_DP_DPHY_BS_SR_SWAP_CNTL 0x2444
+#define mmDP3_DP_DPHY_BS_SR_SWAP_CNTL_BASE_IDX 2
+#define mmDP3_DP_DPHY_HBR2_PATTERN_CONTROL 0x2445
+#define mmDP3_DP_DPHY_HBR2_PATTERN_CONTROL_BASE_IDX 2
+#define mmDP3_DP_MSE_SAT0_STATUS 0x2447
+#define mmDP3_DP_MSE_SAT0_STATUS_BASE_IDX 2
+#define mmDP3_DP_MSE_SAT1_STATUS 0x2448
+#define mmDP3_DP_MSE_SAT1_STATUS_BASE_IDX 2
+#define mmDP3_DP_MSE_SAT2_STATUS 0x2449
+#define mmDP3_DP_MSE_SAT2_STATUS_BASE_IDX 2
+#define mmDP3_DP_MSA_TIMING_PARAM1 0x244c
+#define mmDP3_DP_MSA_TIMING_PARAM1_BASE_IDX 2
+#define mmDP3_DP_MSA_TIMING_PARAM2 0x244d
+#define mmDP3_DP_MSA_TIMING_PARAM2_BASE_IDX 2
+#define mmDP3_DP_MSA_TIMING_PARAM3 0x244e
+#define mmDP3_DP_MSA_TIMING_PARAM3_BASE_IDX 2
+#define mmDP3_DP_MSA_TIMING_PARAM4 0x244f
+#define mmDP3_DP_MSA_TIMING_PARAM4_BASE_IDX 2
+#define mmDP3_DP_MSO_CNTL 0x2450
+#define mmDP3_DP_MSO_CNTL_BASE_IDX 2
+#define mmDP3_DP_MSO_CNTL1 0x2451
+#define mmDP3_DP_MSO_CNTL1_BASE_IDX 2
+#define mmDP3_DP_DSC_CNTL 0x2452
+#define mmDP3_DP_DSC_CNTL_BASE_IDX 2
+#define mmDP3_DP_SEC_CNTL2 0x2453
+#define mmDP3_DP_SEC_CNTL2_BASE_IDX 2
+#define mmDP3_DP_SEC_CNTL3 0x2454
+#define mmDP3_DP_SEC_CNTL3_BASE_IDX 2
+#define mmDP3_DP_SEC_CNTL4 0x2455
+#define mmDP3_DP_SEC_CNTL4_BASE_IDX 2
+#define mmDP3_DP_SEC_CNTL5 0x2456
+#define mmDP3_DP_SEC_CNTL5_BASE_IDX 2
+#define mmDP3_DP_SEC_CNTL6 0x2457
+#define mmDP3_DP_SEC_CNTL6_BASE_IDX 2
+#define mmDP3_DP_SEC_CNTL7 0x2458
+#define mmDP3_DP_SEC_CNTL7_BASE_IDX 2
+#define mmDP3_DP_DB_CNTL 0x2459
+#define mmDP3_DP_DB_CNTL_BASE_IDX 2
+#define mmDP3_DP_MSA_VBID_MISC 0x245a
+#define mmDP3_DP_MSA_VBID_MISC_BASE_IDX 2
+#define mmDP3_DP_SEC_METADATA_TRANSMISSION 0x245b
+#define mmDP3_DP_SEC_METADATA_TRANSMISSION_BASE_IDX 2
+#define mmDP3_DP_DSC_BYTES_PER_PIXEL 0x245c
+#define mmDP3_DP_DSC_BYTES_PER_PIXEL_BASE_IDX 2
+#define mmDP3_DP_ALPM_CNTL 0x245d
+#define mmDP3_DP_ALPM_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dig4_dispdec
+// base address: 0x1000
+#define mmDIG4_DIG_FE_CNTL 0x2468
+#define mmDIG4_DIG_FE_CNTL_BASE_IDX 2
+#define mmDIG4_DIG_OUTPUT_CRC_CNTL 0x2469
+#define mmDIG4_DIG_OUTPUT_CRC_CNTL_BASE_IDX 2
+#define mmDIG4_DIG_OUTPUT_CRC_RESULT 0x246a
+#define mmDIG4_DIG_OUTPUT_CRC_RESULT_BASE_IDX 2
+#define mmDIG4_DIG_CLOCK_PATTERN 0x246b
+#define mmDIG4_DIG_CLOCK_PATTERN_BASE_IDX 2
+#define mmDIG4_DIG_TEST_PATTERN 0x246c
+#define mmDIG4_DIG_TEST_PATTERN_BASE_IDX 2
+#define mmDIG4_DIG_RANDOM_PATTERN_SEED 0x246d
+#define mmDIG4_DIG_RANDOM_PATTERN_SEED_BASE_IDX 2
+#define mmDIG4_DIG_FIFO_STATUS 0x246e
+#define mmDIG4_DIG_FIFO_STATUS_BASE_IDX 2
+#define mmDIG4_HDMI_METADATA_PACKET_CONTROL 0x246f
+#define mmDIG4_HDMI_METADATA_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL4 0x2470
+#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL4_BASE_IDX 2
+#define mmDIG4_HDMI_CONTROL 0x2471
+#define mmDIG4_HDMI_CONTROL_BASE_IDX 2
+#define mmDIG4_HDMI_STATUS 0x2472
+#define mmDIG4_HDMI_STATUS_BASE_IDX 2
+#define mmDIG4_HDMI_AUDIO_PACKET_CONTROL 0x2473
+#define mmDIG4_HDMI_AUDIO_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG4_HDMI_ACR_PACKET_CONTROL 0x2474
+#define mmDIG4_HDMI_ACR_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG4_HDMI_VBI_PACKET_CONTROL 0x2475
+#define mmDIG4_HDMI_VBI_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG4_HDMI_INFOFRAME_CONTROL0 0x2476
+#define mmDIG4_HDMI_INFOFRAME_CONTROL0_BASE_IDX 2
+#define mmDIG4_HDMI_INFOFRAME_CONTROL1 0x2477
+#define mmDIG4_HDMI_INFOFRAME_CONTROL1_BASE_IDX 2
+#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL0 0x2478
+#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL0_BASE_IDX 2
+#define mmDIG4_AFMT_INTERRUPT_STATUS 0x2479
+#define mmDIG4_AFMT_INTERRUPT_STATUS_BASE_IDX 2
+#define mmDIG4_HDMI_GC 0x247b
+#define mmDIG4_HDMI_GC_BASE_IDX 2
+#define mmDIG4_AFMT_AUDIO_PACKET_CONTROL2 0x247c
+#define mmDIG4_AFMT_AUDIO_PACKET_CONTROL2_BASE_IDX 2
+#define mmDIG4_AFMT_ISRC1_0 0x247d
+#define mmDIG4_AFMT_ISRC1_0_BASE_IDX 2
+#define mmDIG4_AFMT_ISRC1_1 0x247e
+#define mmDIG4_AFMT_ISRC1_1_BASE_IDX 2
+#define mmDIG4_AFMT_ISRC1_2 0x247f
+#define mmDIG4_AFMT_ISRC1_2_BASE_IDX 2
+#define mmDIG4_AFMT_ISRC1_3 0x2480
+#define mmDIG4_AFMT_ISRC1_3_BASE_IDX 2
+#define mmDIG4_AFMT_ISRC1_4 0x2481
+#define mmDIG4_AFMT_ISRC1_4_BASE_IDX 2
+#define mmDIG4_AFMT_ISRC2_0 0x2482
+#define mmDIG4_AFMT_ISRC2_0_BASE_IDX 2
+#define mmDIG4_AFMT_ISRC2_1 0x2483
+#define mmDIG4_AFMT_ISRC2_1_BASE_IDX 2
+#define mmDIG4_AFMT_ISRC2_2 0x2484
+#define mmDIG4_AFMT_ISRC2_2_BASE_IDX 2
+#define mmDIG4_AFMT_ISRC2_3 0x2485
+#define mmDIG4_AFMT_ISRC2_3_BASE_IDX 2
+#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL2 0x2486
+#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL2_BASE_IDX 2
+#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL3 0x2487
+#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL3_BASE_IDX 2
+#define mmDIG4_HDMI_DB_CONTROL 0x2488
+#define mmDIG4_HDMI_DB_CONTROL_BASE_IDX 2
+#define mmDIG4_DME_CONTROL 0x2489
+#define mmDIG4_DME_CONTROL_BASE_IDX 2
+#define mmDIG4_AFMT_MPEG_INFO0 0x248a
+#define mmDIG4_AFMT_MPEG_INFO0_BASE_IDX 2
+#define mmDIG4_AFMT_MPEG_INFO1 0x248b
+#define mmDIG4_AFMT_MPEG_INFO1_BASE_IDX 2
+#define mmDIG4_AFMT_GENERIC_HDR 0x248c
+#define mmDIG4_AFMT_GENERIC_HDR_BASE_IDX 2
+#define mmDIG4_AFMT_GENERIC_0 0x248d
+#define mmDIG4_AFMT_GENERIC_0_BASE_IDX 2
+#define mmDIG4_AFMT_GENERIC_1 0x248e
+#define mmDIG4_AFMT_GENERIC_1_BASE_IDX 2
+#define mmDIG4_AFMT_GENERIC_2 0x248f
+#define mmDIG4_AFMT_GENERIC_2_BASE_IDX 2
+#define mmDIG4_AFMT_GENERIC_3 0x2490
+#define mmDIG4_AFMT_GENERIC_3_BASE_IDX 2
+#define mmDIG4_AFMT_GENERIC_4 0x2491
+#define mmDIG4_AFMT_GENERIC_4_BASE_IDX 2
+#define mmDIG4_AFMT_GENERIC_5 0x2492
+#define mmDIG4_AFMT_GENERIC_5_BASE_IDX 2
+#define mmDIG4_AFMT_GENERIC_6 0x2493
+#define mmDIG4_AFMT_GENERIC_6_BASE_IDX 2
+#define mmDIG4_AFMT_GENERIC_7 0x2494
+#define mmDIG4_AFMT_GENERIC_7_BASE_IDX 2
+#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL1 0x2495
+#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL1_BASE_IDX 2
+#define mmDIG4_HDMI_ACR_32_0 0x2496
+#define mmDIG4_HDMI_ACR_32_0_BASE_IDX 2
+#define mmDIG4_HDMI_ACR_32_1 0x2497
+#define mmDIG4_HDMI_ACR_32_1_BASE_IDX 2
+#define mmDIG4_HDMI_ACR_44_0 0x2498
+#define mmDIG4_HDMI_ACR_44_0_BASE_IDX 2
+#define mmDIG4_HDMI_ACR_44_1 0x2499
+#define mmDIG4_HDMI_ACR_44_1_BASE_IDX 2
+#define mmDIG4_HDMI_ACR_48_0 0x249a
+#define mmDIG4_HDMI_ACR_48_0_BASE_IDX 2
+#define mmDIG4_HDMI_ACR_48_1 0x249b
+#define mmDIG4_HDMI_ACR_48_1_BASE_IDX 2
+#define mmDIG4_HDMI_ACR_STATUS_0 0x249c
+#define mmDIG4_HDMI_ACR_STATUS_0_BASE_IDX 2
+#define mmDIG4_HDMI_ACR_STATUS_1 0x249d
+#define mmDIG4_HDMI_ACR_STATUS_1_BASE_IDX 2
+#define mmDIG4_AFMT_AUDIO_INFO0 0x249e
+#define mmDIG4_AFMT_AUDIO_INFO0_BASE_IDX 2
+#define mmDIG4_AFMT_AUDIO_INFO1 0x249f
+#define mmDIG4_AFMT_AUDIO_INFO1_BASE_IDX 2
+#define mmDIG4_AFMT_60958_0 0x24a0
+#define mmDIG4_AFMT_60958_0_BASE_IDX 2
+#define mmDIG4_AFMT_60958_1 0x24a1
+#define mmDIG4_AFMT_60958_1_BASE_IDX 2
+#define mmDIG4_AFMT_AUDIO_CRC_CONTROL 0x24a2
+#define mmDIG4_AFMT_AUDIO_CRC_CONTROL_BASE_IDX 2
+#define mmDIG4_AFMT_RAMP_CONTROL0 0x24a3
+#define mmDIG4_AFMT_RAMP_CONTROL0_BASE_IDX 2
+#define mmDIG4_AFMT_RAMP_CONTROL1 0x24a4
+#define mmDIG4_AFMT_RAMP_CONTROL1_BASE_IDX 2
+#define mmDIG4_AFMT_RAMP_CONTROL2 0x24a5
+#define mmDIG4_AFMT_RAMP_CONTROL2_BASE_IDX 2
+#define mmDIG4_AFMT_RAMP_CONTROL3 0x24a6
+#define mmDIG4_AFMT_RAMP_CONTROL3_BASE_IDX 2
+#define mmDIG4_AFMT_60958_2 0x24a7
+#define mmDIG4_AFMT_60958_2_BASE_IDX 2
+#define mmDIG4_AFMT_AUDIO_CRC_RESULT 0x24a8
+#define mmDIG4_AFMT_AUDIO_CRC_RESULT_BASE_IDX 2
+#define mmDIG4_AFMT_STATUS 0x24a9
+#define mmDIG4_AFMT_STATUS_BASE_IDX 2
+#define mmDIG4_AFMT_AUDIO_PACKET_CONTROL 0x24aa
+#define mmDIG4_AFMT_AUDIO_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG4_AFMT_VBI_PACKET_CONTROL 0x24ab
+#define mmDIG4_AFMT_VBI_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG4_AFMT_INFOFRAME_CONTROL0 0x24ac
+#define mmDIG4_AFMT_INFOFRAME_CONTROL0_BASE_IDX 2
+#define mmDIG4_AFMT_AUDIO_SRC_CONTROL 0x24ad
+#define mmDIG4_AFMT_AUDIO_SRC_CONTROL_BASE_IDX 2
+#define mmDIG4_DIG_BE_CNTL 0x24af
+#define mmDIG4_DIG_BE_CNTL_BASE_IDX 2
+#define mmDIG4_DIG_BE_EN_CNTL 0x24b0
+#define mmDIG4_DIG_BE_EN_CNTL_BASE_IDX 2
+#define mmDIG4_TMDS_CNTL 0x24d3
+#define mmDIG4_TMDS_CNTL_BASE_IDX 2
+#define mmDIG4_TMDS_CONTROL_CHAR 0x24d4
+#define mmDIG4_TMDS_CONTROL_CHAR_BASE_IDX 2
+#define mmDIG4_TMDS_CONTROL0_FEEDBACK 0x24d5
+#define mmDIG4_TMDS_CONTROL0_FEEDBACK_BASE_IDX 2
+#define mmDIG4_TMDS_STEREOSYNC_CTL_SEL 0x24d6
+#define mmDIG4_TMDS_STEREOSYNC_CTL_SEL_BASE_IDX 2
+#define mmDIG4_TMDS_SYNC_CHAR_PATTERN_0_1 0x24d7
+#define mmDIG4_TMDS_SYNC_CHAR_PATTERN_0_1_BASE_IDX 2
+#define mmDIG4_TMDS_SYNC_CHAR_PATTERN_2_3 0x24d8
+#define mmDIG4_TMDS_SYNC_CHAR_PATTERN_2_3_BASE_IDX 2
+#define mmDIG4_TMDS_CTL_BITS 0x24da
+#define mmDIG4_TMDS_CTL_BITS_BASE_IDX 2
+#define mmDIG4_TMDS_DCBALANCER_CONTROL 0x24db
+#define mmDIG4_TMDS_DCBALANCER_CONTROL_BASE_IDX 2
+#define mmDIG4_TMDS_SYNC_DCBALANCE_CHAR 0x24dc
+#define mmDIG4_TMDS_SYNC_DCBALANCE_CHAR_BASE_IDX 2
+#define mmDIG4_TMDS_CTL0_1_GEN_CNTL 0x24dd
+#define mmDIG4_TMDS_CTL0_1_GEN_CNTL_BASE_IDX 2
+#define mmDIG4_TMDS_CTL2_3_GEN_CNTL 0x24de
+#define mmDIG4_TMDS_CTL2_3_GEN_CNTL_BASE_IDX 2
+#define mmDIG4_DIG_VERSION 0x24e0
+#define mmDIG4_DIG_VERSION_BASE_IDX 2
+#define mmDIG4_DIG_LANE_ENABLE 0x24e1
+#define mmDIG4_DIG_LANE_ENABLE_BASE_IDX 2
+#define mmDIG4_AFMT_CNTL 0x24e6
+#define mmDIG4_AFMT_CNTL_BASE_IDX 2
+#define mmDIG4_AFMT_VBI_PACKET_CONTROL1 0x24e7
+#define mmDIG4_AFMT_VBI_PACKET_CONTROL1_BASE_IDX 2
+#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL5 0x24f6
+#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL5_BASE_IDX 2
+#define mmDIG4_FORCE_DIG_DISABLE 0x24f7
+#define mmDIG4_FORCE_DIG_DISABLE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dp4_dispdec
+// base address: 0x1000
+#define mmDP4_DP_LINK_CNTL 0x2508
+#define mmDP4_DP_LINK_CNTL_BASE_IDX 2
+#define mmDP4_DP_PIXEL_FORMAT 0x2509
+#define mmDP4_DP_PIXEL_FORMAT_BASE_IDX 2
+#define mmDP4_DP_MSA_COLORIMETRY 0x250a
+#define mmDP4_DP_MSA_COLORIMETRY_BASE_IDX 2
+#define mmDP4_DP_CONFIG 0x250b
+#define mmDP4_DP_CONFIG_BASE_IDX 2
+#define mmDP4_DP_VID_STREAM_CNTL 0x250c
+#define mmDP4_DP_VID_STREAM_CNTL_BASE_IDX 2
+#define mmDP4_DP_STEER_FIFO 0x250d
+#define mmDP4_DP_STEER_FIFO_BASE_IDX 2
+#define mmDP4_DP_MSA_MISC 0x250e
+#define mmDP4_DP_MSA_MISC_BASE_IDX 2
+#define mmDP4_DP_VID_TIMING 0x2510
+#define mmDP4_DP_VID_TIMING_BASE_IDX 2
+#define mmDP4_DP_VID_N 0x2511
+#define mmDP4_DP_VID_N_BASE_IDX 2
+#define mmDP4_DP_VID_M 0x2512
+#define mmDP4_DP_VID_M_BASE_IDX 2
+#define mmDP4_DP_LINK_FRAMING_CNTL 0x2513
+#define mmDP4_DP_LINK_FRAMING_CNTL_BASE_IDX 2
+#define mmDP4_DP_HBR2_EYE_PATTERN 0x2514
+#define mmDP4_DP_HBR2_EYE_PATTERN_BASE_IDX 2
+#define mmDP4_DP_VID_MSA_VBID 0x2515
+#define mmDP4_DP_VID_MSA_VBID_BASE_IDX 2
+#define mmDP4_DP_VID_INTERRUPT_CNTL 0x2516
+#define mmDP4_DP_VID_INTERRUPT_CNTL_BASE_IDX 2
+#define mmDP4_DP_DPHY_CNTL 0x2517
+#define mmDP4_DP_DPHY_CNTL_BASE_IDX 2
+#define mmDP4_DP_DPHY_TRAINING_PATTERN_SEL 0x2518
+#define mmDP4_DP_DPHY_TRAINING_PATTERN_SEL_BASE_IDX 2
+#define mmDP4_DP_DPHY_SYM0 0x2519
+#define mmDP4_DP_DPHY_SYM0_BASE_IDX 2
+#define mmDP4_DP_DPHY_SYM1 0x251a
+#define mmDP4_DP_DPHY_SYM1_BASE_IDX 2
+#define mmDP4_DP_DPHY_SYM2 0x251b
+#define mmDP4_DP_DPHY_SYM2_BASE_IDX 2
+#define mmDP4_DP_DPHY_8B10B_CNTL 0x251c
+#define mmDP4_DP_DPHY_8B10B_CNTL_BASE_IDX 2
+#define mmDP4_DP_DPHY_PRBS_CNTL 0x251d
+#define mmDP4_DP_DPHY_PRBS_CNTL_BASE_IDX 2
+#define mmDP4_DP_DPHY_SCRAM_CNTL 0x251e
+#define mmDP4_DP_DPHY_SCRAM_CNTL_BASE_IDX 2
+#define mmDP4_DP_DPHY_CRC_EN 0x251f
+#define mmDP4_DP_DPHY_CRC_EN_BASE_IDX 2
+#define mmDP4_DP_DPHY_CRC_CNTL 0x2520
+#define mmDP4_DP_DPHY_CRC_CNTL_BASE_IDX 2
+#define mmDP4_DP_DPHY_CRC_RESULT 0x2521
+#define mmDP4_DP_DPHY_CRC_RESULT_BASE_IDX 2
+#define mmDP4_DP_DPHY_CRC_MST_CNTL 0x2522
+#define mmDP4_DP_DPHY_CRC_MST_CNTL_BASE_IDX 2
+#define mmDP4_DP_DPHY_CRC_MST_STATUS 0x2523
+#define mmDP4_DP_DPHY_CRC_MST_STATUS_BASE_IDX 2
+#define mmDP4_DP_DPHY_FAST_TRAINING 0x2524
+#define mmDP4_DP_DPHY_FAST_TRAINING_BASE_IDX 2
+#define mmDP4_DP_DPHY_FAST_TRAINING_STATUS 0x2525
+#define mmDP4_DP_DPHY_FAST_TRAINING_STATUS_BASE_IDX 2
+#define mmDP4_DP_SEC_CNTL 0x252b
+#define mmDP4_DP_SEC_CNTL_BASE_IDX 2
+#define mmDP4_DP_SEC_CNTL1 0x252c
+#define mmDP4_DP_SEC_CNTL1_BASE_IDX 2
+#define mmDP4_DP_SEC_FRAMING1 0x252d
+#define mmDP4_DP_SEC_FRAMING1_BASE_IDX 2
+#define mmDP4_DP_SEC_FRAMING2 0x252e
+#define mmDP4_DP_SEC_FRAMING2_BASE_IDX 2
+#define mmDP4_DP_SEC_FRAMING3 0x252f
+#define mmDP4_DP_SEC_FRAMING3_BASE_IDX 2
+#define mmDP4_DP_SEC_FRAMING4 0x2530
+#define mmDP4_DP_SEC_FRAMING4_BASE_IDX 2
+#define mmDP4_DP_SEC_AUD_N 0x2531
+#define mmDP4_DP_SEC_AUD_N_BASE_IDX 2
+#define mmDP4_DP_SEC_AUD_N_READBACK 0x2532
+#define mmDP4_DP_SEC_AUD_N_READBACK_BASE_IDX 2
+#define mmDP4_DP_SEC_AUD_M 0x2533
+#define mmDP4_DP_SEC_AUD_M_BASE_IDX 2
+#define mmDP4_DP_SEC_AUD_M_READBACK 0x2534
+#define mmDP4_DP_SEC_AUD_M_READBACK_BASE_IDX 2
+#define mmDP4_DP_SEC_TIMESTAMP 0x2535
+#define mmDP4_DP_SEC_TIMESTAMP_BASE_IDX 2
+#define mmDP4_DP_SEC_PACKET_CNTL 0x2536
+#define mmDP4_DP_SEC_PACKET_CNTL_BASE_IDX 2
+#define mmDP4_DP_MSE_RATE_CNTL 0x2537
+#define mmDP4_DP_MSE_RATE_CNTL_BASE_IDX 2
+#define mmDP4_DP_MSE_RATE_UPDATE 0x2539
+#define mmDP4_DP_MSE_RATE_UPDATE_BASE_IDX 2
+#define mmDP4_DP_MSE_SAT0 0x253a
+#define mmDP4_DP_MSE_SAT0_BASE_IDX 2
+#define mmDP4_DP_MSE_SAT1 0x253b
+#define mmDP4_DP_MSE_SAT1_BASE_IDX 2
+#define mmDP4_DP_MSE_SAT2 0x253c
+#define mmDP4_DP_MSE_SAT2_BASE_IDX 2
+#define mmDP4_DP_MSE_SAT_UPDATE 0x253d
+#define mmDP4_DP_MSE_SAT_UPDATE_BASE_IDX 2
+#define mmDP4_DP_MSE_LINK_TIMING 0x253e
+#define mmDP4_DP_MSE_LINK_TIMING_BASE_IDX 2
+#define mmDP4_DP_MSE_MISC_CNTL 0x253f
+#define mmDP4_DP_MSE_MISC_CNTL_BASE_IDX 2
+#define mmDP4_DP_DPHY_BS_SR_SWAP_CNTL 0x2544
+#define mmDP4_DP_DPHY_BS_SR_SWAP_CNTL_BASE_IDX 2
+#define mmDP4_DP_DPHY_HBR2_PATTERN_CONTROL 0x2545
+#define mmDP4_DP_DPHY_HBR2_PATTERN_CONTROL_BASE_IDX 2
+#define mmDP4_DP_MSE_SAT0_STATUS 0x2547
+#define mmDP4_DP_MSE_SAT0_STATUS_BASE_IDX 2
+#define mmDP4_DP_MSE_SAT1_STATUS 0x2548
+#define mmDP4_DP_MSE_SAT1_STATUS_BASE_IDX 2
+#define mmDP4_DP_MSE_SAT2_STATUS 0x2549
+#define mmDP4_DP_MSE_SAT2_STATUS_BASE_IDX 2
+#define mmDP4_DP_MSA_TIMING_PARAM1 0x254c
+#define mmDP4_DP_MSA_TIMING_PARAM1_BASE_IDX 2
+#define mmDP4_DP_MSA_TIMING_PARAM2 0x254d
+#define mmDP4_DP_MSA_TIMING_PARAM2_BASE_IDX 2
+#define mmDP4_DP_MSA_TIMING_PARAM3 0x254e
+#define mmDP4_DP_MSA_TIMING_PARAM3_BASE_IDX 2
+#define mmDP4_DP_MSA_TIMING_PARAM4 0x254f
+#define mmDP4_DP_MSA_TIMING_PARAM4_BASE_IDX 2
+#define mmDP4_DP_MSO_CNTL 0x2550
+#define mmDP4_DP_MSO_CNTL_BASE_IDX 2
+#define mmDP4_DP_MSO_CNTL1 0x2551
+#define mmDP4_DP_MSO_CNTL1_BASE_IDX 2
+#define mmDP4_DP_DSC_CNTL 0x2552
+#define mmDP4_DP_DSC_CNTL_BASE_IDX 2
+#define mmDP4_DP_SEC_CNTL2 0x2553
+#define mmDP4_DP_SEC_CNTL2_BASE_IDX 2
+#define mmDP4_DP_SEC_CNTL3 0x2554
+#define mmDP4_DP_SEC_CNTL3_BASE_IDX 2
+#define mmDP4_DP_SEC_CNTL4 0x2555
+#define mmDP4_DP_SEC_CNTL4_BASE_IDX 2
+#define mmDP4_DP_SEC_CNTL5 0x2556
+#define mmDP4_DP_SEC_CNTL5_BASE_IDX 2
+#define mmDP4_DP_SEC_CNTL6 0x2557
+#define mmDP4_DP_SEC_CNTL6_BASE_IDX 2
+#define mmDP4_DP_SEC_CNTL7 0x2558
+#define mmDP4_DP_SEC_CNTL7_BASE_IDX 2
+#define mmDP4_DP_DB_CNTL 0x2559
+#define mmDP4_DP_DB_CNTL_BASE_IDX 2
+#define mmDP4_DP_MSA_VBID_MISC 0x255a
+#define mmDP4_DP_MSA_VBID_MISC_BASE_IDX 2
+#define mmDP4_DP_SEC_METADATA_TRANSMISSION 0x255b
+#define mmDP4_DP_SEC_METADATA_TRANSMISSION_BASE_IDX 2
+#define mmDP4_DP_DSC_BYTES_PER_PIXEL 0x255c
+#define mmDP4_DP_DSC_BYTES_PER_PIXEL_BASE_IDX 2
+#define mmDP4_DP_ALPM_CNTL 0x255d
+#define mmDP4_DP_ALPM_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dig5_dispdec
+// base address: 0x1400
+#define mmDIG5_DIG_FE_CNTL 0x2568
+#define mmDIG5_DIG_FE_CNTL_BASE_IDX 2
+#define mmDIG5_DIG_OUTPUT_CRC_CNTL 0x2569
+#define mmDIG5_DIG_OUTPUT_CRC_CNTL_BASE_IDX 2
+#define mmDIG5_DIG_OUTPUT_CRC_RESULT 0x256a
+#define mmDIG5_DIG_OUTPUT_CRC_RESULT_BASE_IDX 2
+#define mmDIG5_DIG_CLOCK_PATTERN 0x256b
+#define mmDIG5_DIG_CLOCK_PATTERN_BASE_IDX 2
+#define mmDIG5_DIG_TEST_PATTERN 0x256c
+#define mmDIG5_DIG_TEST_PATTERN_BASE_IDX 2
+#define mmDIG5_DIG_RANDOM_PATTERN_SEED 0x256d
+#define mmDIG5_DIG_RANDOM_PATTERN_SEED_BASE_IDX 2
+#define mmDIG5_DIG_FIFO_STATUS 0x256e
+#define mmDIG5_DIG_FIFO_STATUS_BASE_IDX 2
+#define mmDIG5_HDMI_METADATA_PACKET_CONTROL 0x256f
+#define mmDIG5_HDMI_METADATA_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL4 0x2570
+#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL4_BASE_IDX 2
+#define mmDIG5_HDMI_CONTROL 0x2571
+#define mmDIG5_HDMI_CONTROL_BASE_IDX 2
+#define mmDIG5_HDMI_STATUS 0x2572
+#define mmDIG5_HDMI_STATUS_BASE_IDX 2
+#define mmDIG5_HDMI_AUDIO_PACKET_CONTROL 0x2573
+#define mmDIG5_HDMI_AUDIO_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG5_HDMI_ACR_PACKET_CONTROL 0x2574
+#define mmDIG5_HDMI_ACR_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG5_HDMI_VBI_PACKET_CONTROL 0x2575
+#define mmDIG5_HDMI_VBI_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG5_HDMI_INFOFRAME_CONTROL0 0x2576
+#define mmDIG5_HDMI_INFOFRAME_CONTROL0_BASE_IDX 2
+#define mmDIG5_HDMI_INFOFRAME_CONTROL1 0x2577
+#define mmDIG5_HDMI_INFOFRAME_CONTROL1_BASE_IDX 2
+#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL0 0x2578
+#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL0_BASE_IDX 2
+#define mmDIG5_AFMT_INTERRUPT_STATUS 0x2579
+#define mmDIG5_AFMT_INTERRUPT_STATUS_BASE_IDX 2
+#define mmDIG5_HDMI_GC 0x257b
+#define mmDIG5_HDMI_GC_BASE_IDX 2
+#define mmDIG5_AFMT_AUDIO_PACKET_CONTROL2 0x257c
+#define mmDIG5_AFMT_AUDIO_PACKET_CONTROL2_BASE_IDX 2
+#define mmDIG5_AFMT_ISRC1_0 0x257d
+#define mmDIG5_AFMT_ISRC1_0_BASE_IDX 2
+#define mmDIG5_AFMT_ISRC1_1 0x257e
+#define mmDIG5_AFMT_ISRC1_1_BASE_IDX 2
+#define mmDIG5_AFMT_ISRC1_2 0x257f
+#define mmDIG5_AFMT_ISRC1_2_BASE_IDX 2
+#define mmDIG5_AFMT_ISRC1_3 0x2580
+#define mmDIG5_AFMT_ISRC1_3_BASE_IDX 2
+#define mmDIG5_AFMT_ISRC1_4 0x2581
+#define mmDIG5_AFMT_ISRC1_4_BASE_IDX 2
+#define mmDIG5_AFMT_ISRC2_0 0x2582
+#define mmDIG5_AFMT_ISRC2_0_BASE_IDX 2
+#define mmDIG5_AFMT_ISRC2_1 0x2583
+#define mmDIG5_AFMT_ISRC2_1_BASE_IDX 2
+#define mmDIG5_AFMT_ISRC2_2 0x2584
+#define mmDIG5_AFMT_ISRC2_2_BASE_IDX 2
+#define mmDIG5_AFMT_ISRC2_3 0x2585
+#define mmDIG5_AFMT_ISRC2_3_BASE_IDX 2
+#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL2 0x2586
+#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL2_BASE_IDX 2
+#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL3 0x2587
+#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL3_BASE_IDX 2
+#define mmDIG5_HDMI_DB_CONTROL 0x2588
+#define mmDIG5_HDMI_DB_CONTROL_BASE_IDX 2
+#define mmDIG5_DME_CONTROL 0x2589
+#define mmDIG5_DME_CONTROL_BASE_IDX 2
+#define mmDIG5_AFMT_MPEG_INFO0 0x258a
+#define mmDIG5_AFMT_MPEG_INFO0_BASE_IDX 2
+#define mmDIG5_AFMT_MPEG_INFO1 0x258b
+#define mmDIG5_AFMT_MPEG_INFO1_BASE_IDX 2
+#define mmDIG5_AFMT_GENERIC_HDR 0x258c
+#define mmDIG5_AFMT_GENERIC_HDR_BASE_IDX 2
+#define mmDIG5_AFMT_GENERIC_0 0x258d
+#define mmDIG5_AFMT_GENERIC_0_BASE_IDX 2
+#define mmDIG5_AFMT_GENERIC_1 0x258e
+#define mmDIG5_AFMT_GENERIC_1_BASE_IDX 2
+#define mmDIG5_AFMT_GENERIC_2 0x258f
+#define mmDIG5_AFMT_GENERIC_2_BASE_IDX 2
+#define mmDIG5_AFMT_GENERIC_3 0x2590
+#define mmDIG5_AFMT_GENERIC_3_BASE_IDX 2
+#define mmDIG5_AFMT_GENERIC_4 0x2591
+#define mmDIG5_AFMT_GENERIC_4_BASE_IDX 2
+#define mmDIG5_AFMT_GENERIC_5 0x2592
+#define mmDIG5_AFMT_GENERIC_5_BASE_IDX 2
+#define mmDIG5_AFMT_GENERIC_6 0x2593
+#define mmDIG5_AFMT_GENERIC_6_BASE_IDX 2
+#define mmDIG5_AFMT_GENERIC_7 0x2594
+#define mmDIG5_AFMT_GENERIC_7_BASE_IDX 2
+#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL1 0x2595
+#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL1_BASE_IDX 2
+#define mmDIG5_HDMI_ACR_32_0 0x2596
+#define mmDIG5_HDMI_ACR_32_0_BASE_IDX 2
+#define mmDIG5_HDMI_ACR_32_1 0x2597
+#define mmDIG5_HDMI_ACR_32_1_BASE_IDX 2
+#define mmDIG5_HDMI_ACR_44_0 0x2598
+#define mmDIG5_HDMI_ACR_44_0_BASE_IDX 2
+#define mmDIG5_HDMI_ACR_44_1 0x2599
+#define mmDIG5_HDMI_ACR_44_1_BASE_IDX 2
+#define mmDIG5_HDMI_ACR_48_0 0x259a
+#define mmDIG5_HDMI_ACR_48_0_BASE_IDX 2
+#define mmDIG5_HDMI_ACR_48_1 0x259b
+#define mmDIG5_HDMI_ACR_48_1_BASE_IDX 2
+#define mmDIG5_HDMI_ACR_STATUS_0 0x259c
+#define mmDIG5_HDMI_ACR_STATUS_0_BASE_IDX 2
+#define mmDIG5_HDMI_ACR_STATUS_1 0x259d
+#define mmDIG5_HDMI_ACR_STATUS_1_BASE_IDX 2
+#define mmDIG5_AFMT_AUDIO_INFO0 0x259e
+#define mmDIG5_AFMT_AUDIO_INFO0_BASE_IDX 2
+#define mmDIG5_AFMT_AUDIO_INFO1 0x259f
+#define mmDIG5_AFMT_AUDIO_INFO1_BASE_IDX 2
+#define mmDIG5_AFMT_60958_0 0x25a0
+#define mmDIG5_AFMT_60958_0_BASE_IDX 2
+#define mmDIG5_AFMT_60958_1 0x25a1
+#define mmDIG5_AFMT_60958_1_BASE_IDX 2
+#define mmDIG5_AFMT_AUDIO_CRC_CONTROL 0x25a2
+#define mmDIG5_AFMT_AUDIO_CRC_CONTROL_BASE_IDX 2
+#define mmDIG5_AFMT_RAMP_CONTROL0 0x25a3
+#define mmDIG5_AFMT_RAMP_CONTROL0_BASE_IDX 2
+#define mmDIG5_AFMT_RAMP_CONTROL1 0x25a4
+#define mmDIG5_AFMT_RAMP_CONTROL1_BASE_IDX 2
+#define mmDIG5_AFMT_RAMP_CONTROL2 0x25a5
+#define mmDIG5_AFMT_RAMP_CONTROL2_BASE_IDX 2
+#define mmDIG5_AFMT_RAMP_CONTROL3 0x25a6
+#define mmDIG5_AFMT_RAMP_CONTROL3_BASE_IDX 2
+#define mmDIG5_AFMT_60958_2 0x25a7
+#define mmDIG5_AFMT_60958_2_BASE_IDX 2
+#define mmDIG5_AFMT_AUDIO_CRC_RESULT 0x25a8
+#define mmDIG5_AFMT_AUDIO_CRC_RESULT_BASE_IDX 2
+#define mmDIG5_AFMT_STATUS 0x25a9
+#define mmDIG5_AFMT_STATUS_BASE_IDX 2
+#define mmDIG5_AFMT_AUDIO_PACKET_CONTROL 0x25aa
+#define mmDIG5_AFMT_AUDIO_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG5_AFMT_VBI_PACKET_CONTROL 0x25ab
+#define mmDIG5_AFMT_VBI_PACKET_CONTROL_BASE_IDX 2
+#define mmDIG5_AFMT_INFOFRAME_CONTROL0 0x25ac
+#define mmDIG5_AFMT_INFOFRAME_CONTROL0_BASE_IDX 2
+#define mmDIG5_AFMT_AUDIO_SRC_CONTROL 0x25ad
+#define mmDIG5_AFMT_AUDIO_SRC_CONTROL_BASE_IDX 2
+#define mmDIG5_DIG_BE_CNTL 0x25af
+#define mmDIG5_DIG_BE_CNTL_BASE_IDX 2
+#define mmDIG5_DIG_BE_EN_CNTL 0x25b0
+#define mmDIG5_DIG_BE_EN_CNTL_BASE_IDX 2
+#define mmDIG5_TMDS_CNTL 0x25d3
+#define mmDIG5_TMDS_CNTL_BASE_IDX 2
+#define mmDIG5_TMDS_CONTROL_CHAR 0x25d4
+#define mmDIG5_TMDS_CONTROL_CHAR_BASE_IDX 2
+#define mmDIG5_TMDS_CONTROL0_FEEDBACK 0x25d5
+#define mmDIG5_TMDS_CONTROL0_FEEDBACK_BASE_IDX 2
+#define mmDIG5_TMDS_STEREOSYNC_CTL_SEL 0x25d6
+#define mmDIG5_TMDS_STEREOSYNC_CTL_SEL_BASE_IDX 2
+#define mmDIG5_TMDS_SYNC_CHAR_PATTERN_0_1 0x25d7
+#define mmDIG5_TMDS_SYNC_CHAR_PATTERN_0_1_BASE_IDX 2
+#define mmDIG5_TMDS_SYNC_CHAR_PATTERN_2_3 0x25d8
+#define mmDIG5_TMDS_SYNC_CHAR_PATTERN_2_3_BASE_IDX 2
+#define mmDIG5_TMDS_CTL_BITS 0x25da
+#define mmDIG5_TMDS_CTL_BITS_BASE_IDX 2
+#define mmDIG5_TMDS_DCBALANCER_CONTROL 0x25db
+#define mmDIG5_TMDS_DCBALANCER_CONTROL_BASE_IDX 2
+#define mmDIG5_TMDS_SYNC_DCBALANCE_CHAR 0x25dc
+#define mmDIG5_TMDS_SYNC_DCBALANCE_CHAR_BASE_IDX 2
+#define mmDIG5_TMDS_CTL0_1_GEN_CNTL 0x25dd
+#define mmDIG5_TMDS_CTL0_1_GEN_CNTL_BASE_IDX 2
+#define mmDIG5_TMDS_CTL2_3_GEN_CNTL 0x25de
+#define mmDIG5_TMDS_CTL2_3_GEN_CNTL_BASE_IDX 2
+#define mmDIG5_DIG_VERSION 0x25e0
+#define mmDIG5_DIG_VERSION_BASE_IDX 2
+#define mmDIG5_DIG_LANE_ENABLE 0x25e1
+#define mmDIG5_DIG_LANE_ENABLE_BASE_IDX 2
+#define mmDIG5_AFMT_CNTL 0x25e6
+#define mmDIG5_AFMT_CNTL_BASE_IDX 2
+#define mmDIG5_AFMT_VBI_PACKET_CONTROL1 0x25e7
+#define mmDIG5_AFMT_VBI_PACKET_CONTROL1_BASE_IDX 2
+#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL5 0x25f6
+#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL5_BASE_IDX 2
+#define mmDIG5_FORCE_DIG_DISABLE 0x25f7
+#define mmDIG5_FORCE_DIG_DISABLE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dio_dp5_dispdec
+// base address: 0x1400
+#define mmDP5_DP_LINK_CNTL 0x2608
+#define mmDP5_DP_LINK_CNTL_BASE_IDX 2
+#define mmDP5_DP_PIXEL_FORMAT 0x2609
+#define mmDP5_DP_PIXEL_FORMAT_BASE_IDX 2
+#define mmDP5_DP_MSA_COLORIMETRY 0x260a
+#define mmDP5_DP_MSA_COLORIMETRY_BASE_IDX 2
+#define mmDP5_DP_CONFIG 0x260b
+#define mmDP5_DP_CONFIG_BASE_IDX 2
+#define mmDP5_DP_VID_STREAM_CNTL 0x260c
+#define mmDP5_DP_VID_STREAM_CNTL_BASE_IDX 2
+#define mmDP5_DP_STEER_FIFO 0x260d
+#define mmDP5_DP_STEER_FIFO_BASE_IDX 2
+#define mmDP5_DP_MSA_MISC 0x260e
+#define mmDP5_DP_MSA_MISC_BASE_IDX 2
+#define mmDP5_DP_VID_TIMING 0x2610
+#define mmDP5_DP_VID_TIMING_BASE_IDX 2
+#define mmDP5_DP_VID_N 0x2611
+#define mmDP5_DP_VID_N_BASE_IDX 2
+#define mmDP5_DP_VID_M 0x2612
+#define mmDP5_DP_VID_M_BASE_IDX 2
+#define mmDP5_DP_LINK_FRAMING_CNTL 0x2613
+#define mmDP5_DP_LINK_FRAMING_CNTL_BASE_IDX 2
+#define mmDP5_DP_HBR2_EYE_PATTERN 0x2614
+#define mmDP5_DP_HBR2_EYE_PATTERN_BASE_IDX 2
+#define mmDP5_DP_VID_MSA_VBID 0x2615
+#define mmDP5_DP_VID_MSA_VBID_BASE_IDX 2
+#define mmDP5_DP_VID_INTERRUPT_CNTL 0x2616
+#define mmDP5_DP_VID_INTERRUPT_CNTL_BASE_IDX 2
+#define mmDP5_DP_DPHY_CNTL 0x2617
+#define mmDP5_DP_DPHY_CNTL_BASE_IDX 2
+#define mmDP5_DP_DPHY_TRAINING_PATTERN_SEL 0x2618
+#define mmDP5_DP_DPHY_TRAINING_PATTERN_SEL_BASE_IDX 2
+#define mmDP5_DP_DPHY_SYM0 0x2619
+#define mmDP5_DP_DPHY_SYM0_BASE_IDX 2
+#define mmDP5_DP_DPHY_SYM1 0x261a
+#define mmDP5_DP_DPHY_SYM1_BASE_IDX 2
+#define mmDP5_DP_DPHY_SYM2 0x261b
+#define mmDP5_DP_DPHY_SYM2_BASE_IDX 2
+#define mmDP5_DP_DPHY_8B10B_CNTL 0x261c
+#define mmDP5_DP_DPHY_8B10B_CNTL_BASE_IDX 2
+#define mmDP5_DP_DPHY_PRBS_CNTL 0x261d
+#define mmDP5_DP_DPHY_PRBS_CNTL_BASE_IDX 2
+#define mmDP5_DP_DPHY_SCRAM_CNTL 0x261e
+#define mmDP5_DP_DPHY_SCRAM_CNTL_BASE_IDX 2
+#define mmDP5_DP_DPHY_CRC_EN 0x261f
+#define mmDP5_DP_DPHY_CRC_EN_BASE_IDX 2
+#define mmDP5_DP_DPHY_CRC_CNTL 0x2620
+#define mmDP5_DP_DPHY_CRC_CNTL_BASE_IDX 2
+#define mmDP5_DP_DPHY_CRC_RESULT 0x2621
+#define mmDP5_DP_DPHY_CRC_RESULT_BASE_IDX 2
+#define mmDP5_DP_DPHY_CRC_MST_CNTL 0x2622
+#define mmDP5_DP_DPHY_CRC_MST_CNTL_BASE_IDX 2
+#define mmDP5_DP_DPHY_CRC_MST_STATUS 0x2623
+#define mmDP5_DP_DPHY_CRC_MST_STATUS_BASE_IDX 2
+#define mmDP5_DP_DPHY_FAST_TRAINING 0x2624
+#define mmDP5_DP_DPHY_FAST_TRAINING_BASE_IDX 2
+#define mmDP5_DP_DPHY_FAST_TRAINING_STATUS 0x2625
+#define mmDP5_DP_DPHY_FAST_TRAINING_STATUS_BASE_IDX 2
+#define mmDP5_DP_SEC_CNTL 0x262b
+#define mmDP5_DP_SEC_CNTL_BASE_IDX 2
+#define mmDP5_DP_SEC_CNTL1 0x262c
+#define mmDP5_DP_SEC_CNTL1_BASE_IDX 2
+#define mmDP5_DP_SEC_FRAMING1 0x262d
+#define mmDP5_DP_SEC_FRAMING1_BASE_IDX 2
+#define mmDP5_DP_SEC_FRAMING2 0x262e
+#define mmDP5_DP_SEC_FRAMING2_BASE_IDX 2
+#define mmDP5_DP_SEC_FRAMING3 0x262f
+#define mmDP5_DP_SEC_FRAMING3_BASE_IDX 2
+#define mmDP5_DP_SEC_FRAMING4 0x2630
+#define mmDP5_DP_SEC_FRAMING4_BASE_IDX 2
+#define mmDP5_DP_SEC_AUD_N 0x2631
+#define mmDP5_DP_SEC_AUD_N_BASE_IDX 2
+#define mmDP5_DP_SEC_AUD_N_READBACK 0x2632
+#define mmDP5_DP_SEC_AUD_N_READBACK_BASE_IDX 2
+#define mmDP5_DP_SEC_AUD_M 0x2633
+#define mmDP5_DP_SEC_AUD_M_BASE_IDX 2
+#define mmDP5_DP_SEC_AUD_M_READBACK 0x2634
+#define mmDP5_DP_SEC_AUD_M_READBACK_BASE_IDX 2
+#define mmDP5_DP_SEC_TIMESTAMP 0x2635
+#define mmDP5_DP_SEC_TIMESTAMP_BASE_IDX 2
+#define mmDP5_DP_SEC_PACKET_CNTL 0x2636
+#define mmDP5_DP_SEC_PACKET_CNTL_BASE_IDX 2
+#define mmDP5_DP_MSE_RATE_CNTL 0x2637
+#define mmDP5_DP_MSE_RATE_CNTL_BASE_IDX 2
+#define mmDP5_DP_MSE_RATE_UPDATE 0x2639
+#define mmDP5_DP_MSE_RATE_UPDATE_BASE_IDX 2
+#define mmDP5_DP_MSE_SAT0 0x263a
+#define mmDP5_DP_MSE_SAT0_BASE_IDX 2
+#define mmDP5_DP_MSE_SAT1 0x263b
+#define mmDP5_DP_MSE_SAT1_BASE_IDX 2
+#define mmDP5_DP_MSE_SAT2 0x263c
+#define mmDP5_DP_MSE_SAT2_BASE_IDX 2
+#define mmDP5_DP_MSE_SAT_UPDATE 0x263d
+#define mmDP5_DP_MSE_SAT_UPDATE_BASE_IDX 2
+#define mmDP5_DP_MSE_LINK_TIMING 0x263e
+#define mmDP5_DP_MSE_LINK_TIMING_BASE_IDX 2
+#define mmDP5_DP_MSE_MISC_CNTL 0x263f
+#define mmDP5_DP_MSE_MISC_CNTL_BASE_IDX 2
+#define mmDP5_DP_DPHY_BS_SR_SWAP_CNTL 0x2644
+#define mmDP5_DP_DPHY_BS_SR_SWAP_CNTL_BASE_IDX 2
+#define mmDP5_DP_DPHY_HBR2_PATTERN_CONTROL 0x2645
+#define mmDP5_DP_DPHY_HBR2_PATTERN_CONTROL_BASE_IDX 2
+#define mmDP5_DP_MSE_SAT0_STATUS 0x2647
+#define mmDP5_DP_MSE_SAT0_STATUS_BASE_IDX 2
+#define mmDP5_DP_MSE_SAT1_STATUS 0x2648
+#define mmDP5_DP_MSE_SAT1_STATUS_BASE_IDX 2
+#define mmDP5_DP_MSE_SAT2_STATUS 0x2649
+#define mmDP5_DP_MSE_SAT2_STATUS_BASE_IDX 2
+#define mmDP5_DP_MSA_TIMING_PARAM1 0x264c
+#define mmDP5_DP_MSA_TIMING_PARAM1_BASE_IDX 2
+#define mmDP5_DP_MSA_TIMING_PARAM2 0x264d
+#define mmDP5_DP_MSA_TIMING_PARAM2_BASE_IDX 2
+#define mmDP5_DP_MSA_TIMING_PARAM3 0x264e
+#define mmDP5_DP_MSA_TIMING_PARAM3_BASE_IDX 2
+#define mmDP5_DP_MSA_TIMING_PARAM4 0x264f
+#define mmDP5_DP_MSA_TIMING_PARAM4_BASE_IDX 2
+#define mmDP5_DP_MSO_CNTL 0x2650
+#define mmDP5_DP_MSO_CNTL_BASE_IDX 2
+#define mmDP5_DP_MSO_CNTL1 0x2651
+#define mmDP5_DP_MSO_CNTL1_BASE_IDX 2
+#define mmDP5_DP_DSC_CNTL 0x2652
+#define mmDP5_DP_DSC_CNTL_BASE_IDX 2
+#define mmDP5_DP_SEC_CNTL2 0x2653
+#define mmDP5_DP_SEC_CNTL2_BASE_IDX 2
+#define mmDP5_DP_SEC_CNTL3 0x2654
+#define mmDP5_DP_SEC_CNTL3_BASE_IDX 2
+#define mmDP5_DP_SEC_CNTL4 0x2655
+#define mmDP5_DP_SEC_CNTL4_BASE_IDX 2
+#define mmDP5_DP_SEC_CNTL5 0x2656
+#define mmDP5_DP_SEC_CNTL5_BASE_IDX 2
+#define mmDP5_DP_SEC_CNTL6 0x2657
+#define mmDP5_DP_SEC_CNTL6_BASE_IDX 2
+#define mmDP5_DP_SEC_CNTL7 0x2658
+#define mmDP5_DP_SEC_CNTL7_BASE_IDX 2
+#define mmDP5_DP_DB_CNTL 0x2659
+#define mmDP5_DP_DB_CNTL_BASE_IDX 2
+#define mmDP5_DP_MSA_VBID_MISC 0x265a
+#define mmDP5_DP_MSA_VBID_MISC_BASE_IDX 2
+#define mmDP5_DP_SEC_METADATA_TRANSMISSION 0x265b
+#define mmDP5_DP_SEC_METADATA_TRANSMISSION_BASE_IDX 2
+#define mmDP5_DP_DSC_BYTES_PER_PIXEL 0x265c
+#define mmDP5_DP_DSC_BYTES_PER_PIXEL_BASE_IDX 2
+#define mmDP5_DP_ALPM_CNTL 0x265d
+#define mmDP5_DP_ALPM_CNTL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcio_dcio_dispdec
+// base address: 0x0
+#define mmDC_GENERICA 0x2868
+#define mmDC_GENERICA_BASE_IDX 2
+#define mmDC_GENERICB 0x2869
+#define mmDC_GENERICB_BASE_IDX 2
+#define mmDC_REF_CLK_CNTL 0x286b
+#define mmDC_REF_CLK_CNTL_BASE_IDX 2
+#define mmUNIPHYA_LINK_CNTL 0x286d
+#define mmUNIPHYA_LINK_CNTL_BASE_IDX 2
+#define mmUNIPHYA_CHANNEL_XBAR_CNTL 0x286e
+#define mmUNIPHYA_CHANNEL_XBAR_CNTL_BASE_IDX 2
+#define mmUNIPHYB_LINK_CNTL 0x286f
+#define mmUNIPHYB_LINK_CNTL_BASE_IDX 2
+#define mmUNIPHYB_CHANNEL_XBAR_CNTL 0x2870
+#define mmUNIPHYB_CHANNEL_XBAR_CNTL_BASE_IDX 2
+#define mmUNIPHYC_LINK_CNTL 0x2871
+#define mmUNIPHYC_LINK_CNTL_BASE_IDX 2
+#define mmUNIPHYC_CHANNEL_XBAR_CNTL 0x2872
+#define mmUNIPHYC_CHANNEL_XBAR_CNTL_BASE_IDX 2
+#define mmUNIPHYD_LINK_CNTL 0x2873
+#define mmUNIPHYD_LINK_CNTL_BASE_IDX 2
+#define mmUNIPHYD_CHANNEL_XBAR_CNTL 0x2874
+#define mmUNIPHYD_CHANNEL_XBAR_CNTL_BASE_IDX 2
+#define mmUNIPHYE_LINK_CNTL 0x2875
+#define mmUNIPHYE_LINK_CNTL_BASE_IDX 2
+#define mmUNIPHYE_CHANNEL_XBAR_CNTL 0x2876
+#define mmUNIPHYE_CHANNEL_XBAR_CNTL_BASE_IDX 2
+#define mmUNIPHYF_LINK_CNTL 0x2877
+#define mmUNIPHYF_LINK_CNTL_BASE_IDX 2
+#define mmUNIPHYF_CHANNEL_XBAR_CNTL 0x2878
+#define mmUNIPHYF_CHANNEL_XBAR_CNTL_BASE_IDX 2
+#define mmDCIO_WRCMD_DELAY 0x287e
+#define mmDCIO_WRCMD_DELAY_BASE_IDX 2
+#define mmDC_PINSTRAPS 0x2880
+#define mmDC_PINSTRAPS_BASE_IDX 2
+#define mmLVTMA_PWRSEQ_CNTL 0x2883
+#define mmLVTMA_PWRSEQ_CNTL_BASE_IDX 2
+#define mmLVTMA_PWRSEQ_STATE 0x2884
+#define mmLVTMA_PWRSEQ_STATE_BASE_IDX 2
+#define mmLVTMA_PWRSEQ_REF_DIV 0x2885
+#define mmLVTMA_PWRSEQ_REF_DIV_BASE_IDX 2
+#define mmLVTMA_PWRSEQ_DELAY1 0x2886
+#define mmLVTMA_PWRSEQ_DELAY1_BASE_IDX 2
+#define mmLVTMA_PWRSEQ_DELAY2 0x2887
+#define mmLVTMA_PWRSEQ_DELAY2_BASE_IDX 2
+#define mmBL_PWM_CNTL 0x2888
+#define mmBL_PWM_CNTL_BASE_IDX 2
+#define mmBL_PWM_CNTL2 0x2889
+#define mmBL_PWM_CNTL2_BASE_IDX 2
+#define mmBL_PWM_PERIOD_CNTL 0x288a
+#define mmBL_PWM_PERIOD_CNTL_BASE_IDX 2
+#define mmBL_PWM_GRP1_REG_LOCK 0x288b
+#define mmBL_PWM_GRP1_REG_LOCK_BASE_IDX 2
+#define mmDCIO_GSL_GENLK_PAD_CNTL 0x288c
+#define mmDCIO_GSL_GENLK_PAD_CNTL_BASE_IDX 2
+#define mmDCIO_GSL_SWAPLOCK_PAD_CNTL 0x288d
+#define mmDCIO_GSL_SWAPLOCK_PAD_CNTL_BASE_IDX 2
+#define mmDCIO_CLOCK_CNTL 0x2895
+#define mmDCIO_CLOCK_CNTL_BASE_IDX 2
+#define mmDCIO_SOFT_RESET 0x289e
+#define mmDCIO_SOFT_RESET_BASE_IDX 2
+#define mmAUXP_IMPCAL 0x28a3
+#define mmAUXP_IMPCAL_BASE_IDX 2
+#define mmAUXN_IMPCAL 0x28a4
+#define mmAUXN_IMPCAL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcio_dcio_chip_dispdec
+// base address: 0x0
+#define mmDC_GPIO_GENERIC_MASK 0x28c8
+#define mmDC_GPIO_GENERIC_MASK_BASE_IDX 2
+#define mmDC_GPIO_GENERIC_A 0x28c9
+#define mmDC_GPIO_GENERIC_A_BASE_IDX 2
+#define mmDC_GPIO_GENERIC_EN 0x28ca
+#define mmDC_GPIO_GENERIC_EN_BASE_IDX 2
+#define mmDC_GPIO_GENERIC_Y 0x28cb
+#define mmDC_GPIO_GENERIC_Y_BASE_IDX 2
+#define mmDC_GPIO_DDC1_MASK 0x28d0
+#define mmDC_GPIO_DDC1_MASK_BASE_IDX 2
+#define mmDC_GPIO_DDC1_A 0x28d1
+#define mmDC_GPIO_DDC1_A_BASE_IDX 2
+#define mmDC_GPIO_DDC1_EN 0x28d2
+#define mmDC_GPIO_DDC1_EN_BASE_IDX 2
+#define mmDC_GPIO_DDC1_Y 0x28d3
+#define mmDC_GPIO_DDC1_Y_BASE_IDX 2
+#define mmDC_GPIO_DDC2_MASK 0x28d4
+#define mmDC_GPIO_DDC2_MASK_BASE_IDX 2
+#define mmDC_GPIO_DDC2_A 0x28d5
+#define mmDC_GPIO_DDC2_A_BASE_IDX 2
+#define mmDC_GPIO_DDC2_EN 0x28d6
+#define mmDC_GPIO_DDC2_EN_BASE_IDX 2
+#define mmDC_GPIO_DDC2_Y 0x28d7
+#define mmDC_GPIO_DDC2_Y_BASE_IDX 2
+#define mmDC_GPIO_DDC3_MASK 0x28d8
+#define mmDC_GPIO_DDC3_MASK_BASE_IDX 2
+#define mmDC_GPIO_DDC3_A 0x28d9
+#define mmDC_GPIO_DDC3_A_BASE_IDX 2
+#define mmDC_GPIO_DDC3_EN 0x28da
+#define mmDC_GPIO_DDC3_EN_BASE_IDX 2
+#define mmDC_GPIO_DDC3_Y 0x28db
+#define mmDC_GPIO_DDC3_Y_BASE_IDX 2
+#define mmDC_GPIO_DDC4_MASK 0x28dc
+#define mmDC_GPIO_DDC4_MASK_BASE_IDX 2
+#define mmDC_GPIO_DDC4_A 0x28dd
+#define mmDC_GPIO_DDC4_A_BASE_IDX 2
+#define mmDC_GPIO_DDC4_EN 0x28de
+#define mmDC_GPIO_DDC4_EN_BASE_IDX 2
+#define mmDC_GPIO_DDC4_Y 0x28df
+#define mmDC_GPIO_DDC4_Y_BASE_IDX 2
+#define mmDC_GPIO_DDC5_MASK 0x28e0
+#define mmDC_GPIO_DDC5_MASK_BASE_IDX 2
+#define mmDC_GPIO_DDC5_A 0x28e1
+#define mmDC_GPIO_DDC5_A_BASE_IDX 2
+#define mmDC_GPIO_DDC5_EN 0x28e2
+#define mmDC_GPIO_DDC5_EN_BASE_IDX 2
+#define mmDC_GPIO_DDC5_Y 0x28e3
+#define mmDC_GPIO_DDC5_Y_BASE_IDX 2
+#define mmDC_GPIO_DDC6_MASK 0x28e4
+#define mmDC_GPIO_DDC6_MASK_BASE_IDX 2
+#define mmDC_GPIO_DDC6_A 0x28e5
+#define mmDC_GPIO_DDC6_A_BASE_IDX 2
+#define mmDC_GPIO_DDC6_EN 0x28e6
+#define mmDC_GPIO_DDC6_EN_BASE_IDX 2
+#define mmDC_GPIO_DDC6_Y 0x28e7
+#define mmDC_GPIO_DDC6_Y_BASE_IDX 2
+#define mmDC_GPIO_DDCVGA_MASK 0x28e8
+#define mmDC_GPIO_DDCVGA_MASK_BASE_IDX 2
+#define mmDC_GPIO_DDCVGA_A 0x28e9
+#define mmDC_GPIO_DDCVGA_A_BASE_IDX 2
+#define mmDC_GPIO_DDCVGA_EN 0x28ea
+#define mmDC_GPIO_DDCVGA_EN_BASE_IDX 2
+#define mmDC_GPIO_DDCVGA_Y 0x28eb
+#define mmDC_GPIO_DDCVGA_Y_BASE_IDX 2
+#define mmDC_GPIO_GENLK_MASK 0x28f0
+#define mmDC_GPIO_GENLK_MASK_BASE_IDX 2
+#define mmDC_GPIO_GENLK_A 0x28f1
+#define mmDC_GPIO_GENLK_A_BASE_IDX 2
+#define mmDC_GPIO_GENLK_EN 0x28f2
+#define mmDC_GPIO_GENLK_EN_BASE_IDX 2
+#define mmDC_GPIO_GENLK_Y 0x28f3
+#define mmDC_GPIO_GENLK_Y_BASE_IDX 2
+#define mmDC_GPIO_HPD_MASK 0x28f4
+#define mmDC_GPIO_HPD_MASK_BASE_IDX 2
+#define mmDC_GPIO_HPD_A 0x28f5
+#define mmDC_GPIO_HPD_A_BASE_IDX 2
+#define mmDC_GPIO_HPD_EN 0x28f6
+#define mmDC_GPIO_HPD_EN_BASE_IDX 2
+#define mmDC_GPIO_HPD_Y 0x28f7
+#define mmDC_GPIO_HPD_Y_BASE_IDX 2
+#define mmDC_GPIO_PWRSEQ_MASK 0x28f8
+#define mmDC_GPIO_PWRSEQ_MASK_BASE_IDX 2
+#define mmDC_GPIO_PWRSEQ_A 0x28f9
+#define mmDC_GPIO_PWRSEQ_A_BASE_IDX 2
+#define mmDC_GPIO_PWRSEQ_EN 0x28fa
+#define mmDC_GPIO_PWRSEQ_EN_BASE_IDX 2
+#define mmDC_GPIO_PWRSEQ_Y 0x28fb
+#define mmDC_GPIO_PWRSEQ_Y_BASE_IDX 2
+#define mmDC_GPIO_PAD_STRENGTH_1 0x28fc
+#define mmDC_GPIO_PAD_STRENGTH_1_BASE_IDX 2
+#define mmDC_GPIO_PAD_STRENGTH_2 0x28fd
+#define mmDC_GPIO_PAD_STRENGTH_2_BASE_IDX 2
+#define mmPHY_AUX_CNTL 0x28ff
+#define mmPHY_AUX_CNTL_BASE_IDX 2
+#define mmDC_GPIO_TX12_EN 0x2915
+#define mmDC_GPIO_TX12_EN_BASE_IDX 2
+#define mmDC_GPIO_AUX_CTRL_0 0x2916
+#define mmDC_GPIO_AUX_CTRL_0_BASE_IDX 2
+#define mmDC_GPIO_AUX_CTRL_1 0x2917
+#define mmDC_GPIO_AUX_CTRL_1_BASE_IDX 2
+#define mmDC_GPIO_AUX_CTRL_2 0x2918
+#define mmDC_GPIO_AUX_CTRL_2_BASE_IDX 2
+#define mmDC_GPIO_RXEN 0x2919
+#define mmDC_GPIO_RXEN_BASE_IDX 2
+#define mmDC_GPIO_PULLUPEN 0x291a
+#define mmDC_GPIO_PULLUPEN_BASE_IDX 2
+#define mmDC_GPIO_AUX_CTRL_3 0x291b
+#define mmDC_GPIO_AUX_CTRL_3_BASE_IDX 2
+#define mmDC_GPIO_AUX_CTRL_4 0x291c
+#define mmDC_GPIO_AUX_CTRL_4_BASE_IDX 2
+#define mmDC_GPIO_AUX_CTRL_5 0x291d
+#define mmDC_GPIO_AUX_CTRL_5_BASE_IDX 2
+#define mmAUXI2C_PAD_ALL_PWR_OK 0x291e
+#define mmAUXI2C_PAD_ALL_PWR_OK_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcio_dcio_uniphy0_dispdec
+// base address: 0x0
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED0 0x2928
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED0_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED1 0x2929
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED1_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED2 0x292a
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED2_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED3 0x292b
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED3_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED4 0x292c
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED4_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED5 0x292d
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED5_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED6 0x292e
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED6_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED7 0x292f
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED7_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED8 0x2930
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED8_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED9 0x2931
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED9_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED10 0x2932
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED10_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED11 0x2933
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED11_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED12 0x2934
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED12_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED13 0x2935
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED13_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED14 0x2936
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED14_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED15 0x2937
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED15_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED16 0x2938
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED16_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED17 0x2939
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED17_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED18 0x293a
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED18_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED19 0x293b
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED19_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED20 0x293c
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED20_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED21 0x293d
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED21_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED22 0x293e
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED22_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED23 0x293f
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED23_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED24 0x2940
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED24_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED25 0x2941
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED25_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED26 0x2942
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED26_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED27 0x2943
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED27_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED28 0x2944
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED28_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED29 0x2945
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED29_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED30 0x2946
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED30_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED31 0x2947
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED31_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED32 0x2948
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED32_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED33 0x2949
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED33_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED34 0x294a
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED34_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED35 0x294b
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED35_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED36 0x294c
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED36_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED37 0x294d
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED37_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED38 0x294e
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED38_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED39 0x294f
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED39_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED40 0x2950
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED40_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED41 0x2951
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED41_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED42 0x2952
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED42_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED43 0x2953
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED43_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED44 0x2954
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED44_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED45 0x2955
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED45_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED46 0x2956
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED46_BASE_IDX 2
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED47 0x2957
+#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED47_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcio_dcio_uniphy1_dispdec
+// base address: 0x360
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED0 0x2a00
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED0_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED1 0x2a01
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED1_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED2 0x2a02
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED2_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED3 0x2a03
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED3_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED4 0x2a04
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED4_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED5 0x2a05
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED5_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED6 0x2a06
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED6_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED7 0x2a07
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED7_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED8 0x2a08
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED8_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED9 0x2a09
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED9_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED10 0x2a0a
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED10_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED11 0x2a0b
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED11_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED12 0x2a0c
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED12_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED13 0x2a0d
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED13_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED14 0x2a0e
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED14_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED15 0x2a0f
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED15_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED16 0x2a10
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED16_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED17 0x2a11
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED17_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED18 0x2a12
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED18_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED19 0x2a13
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED19_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED20 0x2a14
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED20_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED21 0x2a15
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED21_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED22 0x2a16
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED22_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED23 0x2a17
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED23_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED24 0x2a18
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED24_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED25 0x2a19
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED25_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED26 0x2a1a
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED26_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED27 0x2a1b
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED27_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED28 0x2a1c
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED28_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED29 0x2a1d
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED29_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED30 0x2a1e
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED30_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED31 0x2a1f
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED31_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED32 0x2a20
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED32_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED33 0x2a21
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED33_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED34 0x2a22
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED34_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED35 0x2a23
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED35_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED36 0x2a24
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED36_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED37 0x2a25
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED37_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED38 0x2a26
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED38_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED39 0x2a27
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED39_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED40 0x2a28
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED40_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED41 0x2a29
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED41_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED42 0x2a2a
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED42_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED43 0x2a2b
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED43_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED44 0x2a2c
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED44_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED45 0x2a2d
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED45_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED46 0x2a2e
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED46_BASE_IDX 2
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED47 0x2a2f
+#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED47_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcio_dcio_uniphy2_dispdec
+// base address: 0x6c0
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED0 0x2ad8
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED0_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED1 0x2ad9
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED1_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED2 0x2ada
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED2_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED3 0x2adb
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED3_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED4 0x2adc
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED4_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED5 0x2add
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED5_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED6 0x2ade
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED6_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED7 0x2adf
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED7_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED8 0x2ae0
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED8_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED9 0x2ae1
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED9_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED10 0x2ae2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED10_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED11 0x2ae3
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED11_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED12 0x2ae4
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED12_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED13 0x2ae5
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED13_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED14 0x2ae6
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED14_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED15 0x2ae7
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED15_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED16 0x2ae8
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED16_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED17 0x2ae9
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED17_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED18 0x2aea
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED18_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED19 0x2aeb
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED19_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED20 0x2aec
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED20_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED21 0x2aed
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED21_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED22 0x2aee
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED22_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED23 0x2aef
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED23_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED24 0x2af0
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED24_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED25 0x2af1
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED25_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED26 0x2af2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED26_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED27 0x2af3
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED27_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED28 0x2af4
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED28_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED29 0x2af5
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED29_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED30 0x2af6
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED30_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED31 0x2af7
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED31_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED32 0x2af8
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED32_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED33 0x2af9
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED33_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED34 0x2afa
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED34_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED35 0x2afb
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED35_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED36 0x2afc
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED36_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED37 0x2afd
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED37_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED38 0x2afe
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED38_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED39 0x2aff
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED39_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED40 0x2b00
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED40_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED41 0x2b01
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED41_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED42 0x2b02
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED42_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED43 0x2b03
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED43_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED44 0x2b04
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED44_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED45 0x2b05
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED45_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED46 0x2b06
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED46_BASE_IDX 2
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED47 0x2b07
+#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED47_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcio_dcio_uniphy3_dispdec
+// base address: 0xa20
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED0 0x2bb0
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED0_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED1 0x2bb1
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED1_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED2 0x2bb2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED2_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED3 0x2bb3
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED3_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED4 0x2bb4
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED4_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED5 0x2bb5
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED5_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED6 0x2bb6
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED6_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED7 0x2bb7
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED7_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED8 0x2bb8
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED8_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED9 0x2bb9
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED9_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED10 0x2bba
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED10_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED11 0x2bbb
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED11_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED12 0x2bbc
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED12_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED13 0x2bbd
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED13_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED14 0x2bbe
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED14_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED15 0x2bbf
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED15_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED16 0x2bc0
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED16_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED17 0x2bc1
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED17_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED18 0x2bc2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED18_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED19 0x2bc3
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED19_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED20 0x2bc4
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED20_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED21 0x2bc5
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED21_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED22 0x2bc6
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED22_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED23 0x2bc7
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED23_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED24 0x2bc8
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED24_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED25 0x2bc9
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED25_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED26 0x2bca
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED26_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED27 0x2bcb
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED27_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED28 0x2bcc
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED28_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED29 0x2bcd
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED29_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED30 0x2bce
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED30_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED31 0x2bcf
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED31_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED32 0x2bd0
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED32_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED33 0x2bd1
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED33_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED34 0x2bd2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED34_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED35 0x2bd3
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED35_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED36 0x2bd4
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED36_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED37 0x2bd5
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED37_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED38 0x2bd6
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED38_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED39 0x2bd7
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED39_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED40 0x2bd8
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED40_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED41 0x2bd9
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED41_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED42 0x2bda
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED42_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED43 0x2bdb
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED43_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED44 0x2bdc
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED44_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED45 0x2bdd
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED45_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED46 0x2bde
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED46_BASE_IDX 2
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED47 0x2bdf
+#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED47_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcio_dcio_uniphy4_dispdec
+// base address: 0xd80
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED0 0x2c88
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED0_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED1 0x2c89
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED1_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED2 0x2c8a
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED2_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED3 0x2c8b
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED3_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED4 0x2c8c
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED4_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED5 0x2c8d
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED5_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED6 0x2c8e
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED6_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED7 0x2c8f
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED7_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED8 0x2c90
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED8_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED9 0x2c91
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED9_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED10 0x2c92
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED10_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED11 0x2c93
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED11_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED12 0x2c94
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED12_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED13 0x2c95
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED13_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED14 0x2c96
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED14_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED15 0x2c97
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED15_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED16 0x2c98
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED16_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED17 0x2c99
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED17_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED18 0x2c9a
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED18_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED19 0x2c9b
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED19_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED20 0x2c9c
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED20_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED21 0x2c9d
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED21_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED22 0x2c9e
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED22_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED23 0x2c9f
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED23_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED24 0x2ca0
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED24_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED25 0x2ca1
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED25_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED26 0x2ca2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED26_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED27 0x2ca3
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED27_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED28 0x2ca4
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED28_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED29 0x2ca5
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED29_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED30 0x2ca6
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED30_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED31 0x2ca7
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED31_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED32 0x2ca8
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED32_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED33 0x2ca9
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED33_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED34 0x2caa
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED34_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED35 0x2cab
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED35_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED36 0x2cac
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED36_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED37 0x2cad
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED37_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED38 0x2cae
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED38_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED39 0x2caf
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED39_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED40 0x2cb0
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED40_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED41 0x2cb1
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED41_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED42 0x2cb2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED42_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED43 0x2cb3
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED43_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED44 0x2cb4
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED44_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED45 0x2cb5
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED45_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED46 0x2cb6
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED46_BASE_IDX 2
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED47 0x2cb7
+#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED47_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcio_dcio_uniphy5_dispdec
+// base address: 0x10e0
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED0 0x2d60
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED0_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED1 0x2d61
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED1_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED2 0x2d62
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED2_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED3 0x2d63
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED3_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED4 0x2d64
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED4_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED5 0x2d65
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED5_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED6 0x2d66
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED6_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED7 0x2d67
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED7_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED8 0x2d68
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED8_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED9 0x2d69
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED9_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED10 0x2d6a
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED10_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED11 0x2d6b
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED11_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED12 0x2d6c
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED12_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED13 0x2d6d
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED13_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED14 0x2d6e
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED14_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED15 0x2d6f
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED15_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED16 0x2d70
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED16_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED17 0x2d71
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED17_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED18 0x2d72
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED18_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED19 0x2d73
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED19_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED20 0x2d74
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED20_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED21 0x2d75
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED21_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED22 0x2d76
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED22_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED23 0x2d77
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED23_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED24 0x2d78
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED24_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED25 0x2d79
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED25_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED26 0x2d7a
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED26_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED27 0x2d7b
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED27_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED28 0x2d7c
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED28_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED29 0x2d7d
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED29_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED30 0x2d7e
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED30_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED31 0x2d7f
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED31_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED32 0x2d80
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED32_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED33 0x2d81
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED33_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED34 0x2d82
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED34_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED35 0x2d83
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED35_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED36 0x2d84
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED36_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED37 0x2d85
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED37_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED38 0x2d86
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED38_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED39 0x2d87
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED39_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED40 0x2d88
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED40_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED41 0x2d89
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED41_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED42 0x2d8a
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED42_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED43 0x2d8b
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED43_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED44 0x2d8c
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED44_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED45 0x2d8d
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED45_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED46 0x2d8e
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED46_BASE_IDX 2
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED47 0x2d8f
+#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED47_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dcio_dcio_uniphy6_dispdec
+// base address: 0x1440
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED0 0x2e38
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED0_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED1 0x2e39
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED1_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED2 0x2e3a
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED2_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED3 0x2e3b
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED3_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED4 0x2e3c
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED4_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED5 0x2e3d
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED5_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED6 0x2e3e
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED6_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED7 0x2e3f
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED7_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED8 0x2e40
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED8_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED9 0x2e41
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED9_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED10 0x2e42
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED10_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED11 0x2e43
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED11_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED12 0x2e44
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED12_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED13 0x2e45
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED13_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED14 0x2e46
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED14_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED15 0x2e47
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED15_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED16 0x2e48
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED16_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED17 0x2e49
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED17_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED18 0x2e4a
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED18_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED19 0x2e4b
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED19_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED20 0x2e4c
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED20_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED21 0x2e4d
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED21_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED22 0x2e4e
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED22_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED23 0x2e4f
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED23_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED24 0x2e50
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED24_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED25 0x2e51
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED25_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED26 0x2e52
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED26_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED27 0x2e53
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED27_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED28 0x2e54
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED28_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED29 0x2e55
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED29_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED30 0x2e56
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED30_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED31 0x2e57
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED31_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED32 0x2e58
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED32_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED33 0x2e59
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED33_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED34 0x2e5a
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED34_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED35 0x2e5b
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED35_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED36 0x2e5c
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED36_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED37 0x2e5d
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED37_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED38 0x2e5e
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED38_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED39 0x2e5f
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED39_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED40 0x2e60
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED40_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED41 0x2e61
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED41_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED42 0x2e62
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED42_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED43 0x2e63
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED43_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED44 0x2e64
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED44_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED45 0x2e65
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED45_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED46 0x2e66
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED46_BASE_IDX 2
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED47 0x2e67
+#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED47_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc0_dispdec_dsc_top_dispdec
+// base address: 0x0
+#define mmDSC_TOP0_DSC_TOP_CONTROL 0x3000
+#define mmDSC_TOP0_DSC_TOP_CONTROL_BASE_IDX 2
+#define mmDSC_TOP0_DSC_DEBUG_CONTROL 0x3001
+#define mmDSC_TOP0_DSC_DEBUG_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc0_dispdec_dsccif_dispdec
+// base address: 0x0
+#define mmDSCCIF0_DSCCIF_CONFIG0 0x3005
+#define mmDSCCIF0_DSCCIF_CONFIG0_BASE_IDX 2
+#define mmDSCCIF0_DSCCIF_CONFIG1 0x3006
+#define mmDSCCIF0_DSCCIF_CONFIG1_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc0_dispdec_dscc_dispdec
+// base address: 0x0
+#define mmDSCC0_DSCC_CONFIG0 0x300a
+#define mmDSCC0_DSCC_CONFIG0_BASE_IDX 2
+#define mmDSCC0_DSCC_CONFIG1 0x300b
+#define mmDSCC0_DSCC_CONFIG1_BASE_IDX 2
+#define mmDSCC0_DSCC_STATUS 0x300c
+#define mmDSCC0_DSCC_STATUS_BASE_IDX 2
+#define mmDSCC0_DSCC_INTERRUPT_CONTROL_STATUS 0x300d
+#define mmDSCC0_DSCC_INTERRUPT_CONTROL_STATUS_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG0 0x300e
+#define mmDSCC0_DSCC_PPS_CONFIG0_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG1 0x300f
+#define mmDSCC0_DSCC_PPS_CONFIG1_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG2 0x3010
+#define mmDSCC0_DSCC_PPS_CONFIG2_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG3 0x3011
+#define mmDSCC0_DSCC_PPS_CONFIG3_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG4 0x3012
+#define mmDSCC0_DSCC_PPS_CONFIG4_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG5 0x3013
+#define mmDSCC0_DSCC_PPS_CONFIG5_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG6 0x3014
+#define mmDSCC0_DSCC_PPS_CONFIG6_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG7 0x3015
+#define mmDSCC0_DSCC_PPS_CONFIG7_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG8 0x3016
+#define mmDSCC0_DSCC_PPS_CONFIG8_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG9 0x3017
+#define mmDSCC0_DSCC_PPS_CONFIG9_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG10 0x3018
+#define mmDSCC0_DSCC_PPS_CONFIG10_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG11 0x3019
+#define mmDSCC0_DSCC_PPS_CONFIG11_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG12 0x301a
+#define mmDSCC0_DSCC_PPS_CONFIG12_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG13 0x301b
+#define mmDSCC0_DSCC_PPS_CONFIG13_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG14 0x301c
+#define mmDSCC0_DSCC_PPS_CONFIG14_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG15 0x301d
+#define mmDSCC0_DSCC_PPS_CONFIG15_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG16 0x301e
+#define mmDSCC0_DSCC_PPS_CONFIG16_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG17 0x301f
+#define mmDSCC0_DSCC_PPS_CONFIG17_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG18 0x3020
+#define mmDSCC0_DSCC_PPS_CONFIG18_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG19 0x3021
+#define mmDSCC0_DSCC_PPS_CONFIG19_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG20 0x3022
+#define mmDSCC0_DSCC_PPS_CONFIG20_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG21 0x3023
+#define mmDSCC0_DSCC_PPS_CONFIG21_BASE_IDX 2
+#define mmDSCC0_DSCC_PPS_CONFIG22 0x3024
+#define mmDSCC0_DSCC_PPS_CONFIG22_BASE_IDX 2
+#define mmDSCC0_DSCC_MEM_POWER_CONTROL 0x3025
+#define mmDSCC0_DSCC_MEM_POWER_CONTROL_BASE_IDX 2
+#define mmDSCC0_DSCC_R_Y_SQUARED_ERROR_LOWER 0x3026
+#define mmDSCC0_DSCC_R_Y_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC0_DSCC_R_Y_SQUARED_ERROR_UPPER 0x3027
+#define mmDSCC0_DSCC_R_Y_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC0_DSCC_G_CB_SQUARED_ERROR_LOWER 0x3028
+#define mmDSCC0_DSCC_G_CB_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC0_DSCC_G_CB_SQUARED_ERROR_UPPER 0x3029
+#define mmDSCC0_DSCC_G_CB_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC0_DSCC_B_CR_SQUARED_ERROR_LOWER 0x302a
+#define mmDSCC0_DSCC_B_CR_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC0_DSCC_B_CR_SQUARED_ERROR_UPPER 0x302b
+#define mmDSCC0_DSCC_B_CR_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC0_DSCC_MAX_ABS_ERROR0 0x302c
+#define mmDSCC0_DSCC_MAX_ABS_ERROR0_BASE_IDX 2
+#define mmDSCC0_DSCC_MAX_ABS_ERROR1 0x302d
+#define mmDSCC0_DSCC_MAX_ABS_ERROR1_BASE_IDX 2
+#define mmDSCC0_DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL 0x302e
+#define mmDSCC0_DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC0_DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL 0x302f
+#define mmDSCC0_DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC0_DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL 0x3030
+#define mmDSCC0_DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC0_DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL 0x3031
+#define mmDSCC0_DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC0_DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL 0x3032
+#define mmDSCC0_DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC0_DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL 0x3033
+#define mmDSCC0_DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC0_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL 0x3034
+#define mmDSCC0_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL 0x3035
+#define mmDSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC0_DSCC_TEST_DEBUG_BUS_ROTATE 0x303a
+#define mmDSCC0_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc0_dispdec_dsc_dcperfmon_dc_perfmon_dispdec
+// base address: 0xc140
+#define mmDC_PERFMON21_PERFCOUNTER_CNTL 0x3050
+#define mmDC_PERFMON21_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON21_PERFCOUNTER_CNTL2 0x3051
+#define mmDC_PERFMON21_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON21_PERFCOUNTER_STATE 0x3052
+#define mmDC_PERFMON21_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON21_PERFMON_CNTL 0x3053
+#define mmDC_PERFMON21_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON21_PERFMON_CNTL2 0x3054
+#define mmDC_PERFMON21_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON21_PERFMON_CVALUE_INT_MISC 0x3055
+#define mmDC_PERFMON21_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON21_PERFMON_CVALUE_LOW 0x3056
+#define mmDC_PERFMON21_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON21_PERFMON_HI 0x3057
+#define mmDC_PERFMON21_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON21_PERFMON_LOW 0x3058
+#define mmDC_PERFMON21_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc1_dispdec_dsc_top_dispdec
+// base address: 0x170
+#define mmDSC_TOP1_DSC_TOP_CONTROL 0x305c
+#define mmDSC_TOP1_DSC_TOP_CONTROL_BASE_IDX 2
+#define mmDSC_TOP1_DSC_DEBUG_CONTROL 0x305d
+#define mmDSC_TOP1_DSC_DEBUG_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc1_dispdec_dsccif_dispdec
+// base address: 0x170
+#define mmDSCCIF1_DSCCIF_CONFIG0 0x3061
+#define mmDSCCIF1_DSCCIF_CONFIG0_BASE_IDX 2
+#define mmDSCCIF1_DSCCIF_CONFIG1 0x3062
+#define mmDSCCIF1_DSCCIF_CONFIG1_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc1_dispdec_dscc_dispdec
+// base address: 0x170
+#define mmDSCC1_DSCC_CONFIG0 0x3066
+#define mmDSCC1_DSCC_CONFIG0_BASE_IDX 2
+#define mmDSCC1_DSCC_CONFIG1 0x3067
+#define mmDSCC1_DSCC_CONFIG1_BASE_IDX 2
+#define mmDSCC1_DSCC_STATUS 0x3068
+#define mmDSCC1_DSCC_STATUS_BASE_IDX 2
+#define mmDSCC1_DSCC_INTERRUPT_CONTROL_STATUS 0x3069
+#define mmDSCC1_DSCC_INTERRUPT_CONTROL_STATUS_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG0 0x306a
+#define mmDSCC1_DSCC_PPS_CONFIG0_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG1 0x306b
+#define mmDSCC1_DSCC_PPS_CONFIG1_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG2 0x306c
+#define mmDSCC1_DSCC_PPS_CONFIG2_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG3 0x306d
+#define mmDSCC1_DSCC_PPS_CONFIG3_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG4 0x306e
+#define mmDSCC1_DSCC_PPS_CONFIG4_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG5 0x306f
+#define mmDSCC1_DSCC_PPS_CONFIG5_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG6 0x3070
+#define mmDSCC1_DSCC_PPS_CONFIG6_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG7 0x3071
+#define mmDSCC1_DSCC_PPS_CONFIG7_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG8 0x3072
+#define mmDSCC1_DSCC_PPS_CONFIG8_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG9 0x3073
+#define mmDSCC1_DSCC_PPS_CONFIG9_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG10 0x3074
+#define mmDSCC1_DSCC_PPS_CONFIG10_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG11 0x3075
+#define mmDSCC1_DSCC_PPS_CONFIG11_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG12 0x3076
+#define mmDSCC1_DSCC_PPS_CONFIG12_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG13 0x3077
+#define mmDSCC1_DSCC_PPS_CONFIG13_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG14 0x3078
+#define mmDSCC1_DSCC_PPS_CONFIG14_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG15 0x3079
+#define mmDSCC1_DSCC_PPS_CONFIG15_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG16 0x307a
+#define mmDSCC1_DSCC_PPS_CONFIG16_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG17 0x307b
+#define mmDSCC1_DSCC_PPS_CONFIG17_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG18 0x307c
+#define mmDSCC1_DSCC_PPS_CONFIG18_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG19 0x307d
+#define mmDSCC1_DSCC_PPS_CONFIG19_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG20 0x307e
+#define mmDSCC1_DSCC_PPS_CONFIG20_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG21 0x307f
+#define mmDSCC1_DSCC_PPS_CONFIG21_BASE_IDX 2
+#define mmDSCC1_DSCC_PPS_CONFIG22 0x3080
+#define mmDSCC1_DSCC_PPS_CONFIG22_BASE_IDX 2
+#define mmDSCC1_DSCC_MEM_POWER_CONTROL 0x3081
+#define mmDSCC1_DSCC_MEM_POWER_CONTROL_BASE_IDX 2
+#define mmDSCC1_DSCC_R_Y_SQUARED_ERROR_LOWER 0x3082
+#define mmDSCC1_DSCC_R_Y_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC1_DSCC_R_Y_SQUARED_ERROR_UPPER 0x3083
+#define mmDSCC1_DSCC_R_Y_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC1_DSCC_G_CB_SQUARED_ERROR_LOWER 0x3084
+#define mmDSCC1_DSCC_G_CB_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC1_DSCC_G_CB_SQUARED_ERROR_UPPER 0x3085
+#define mmDSCC1_DSCC_G_CB_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC1_DSCC_B_CR_SQUARED_ERROR_LOWER 0x3086
+#define mmDSCC1_DSCC_B_CR_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC1_DSCC_B_CR_SQUARED_ERROR_UPPER 0x3087
+#define mmDSCC1_DSCC_B_CR_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC1_DSCC_MAX_ABS_ERROR0 0x3088
+#define mmDSCC1_DSCC_MAX_ABS_ERROR0_BASE_IDX 2
+#define mmDSCC1_DSCC_MAX_ABS_ERROR1 0x3089
+#define mmDSCC1_DSCC_MAX_ABS_ERROR1_BASE_IDX 2
+#define mmDSCC1_DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL 0x308a
+#define mmDSCC1_DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC1_DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL 0x308b
+#define mmDSCC1_DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC1_DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL 0x308c
+#define mmDSCC1_DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC1_DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL 0x308d
+#define mmDSCC1_DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC1_DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL 0x308e
+#define mmDSCC1_DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC1_DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL 0x308f
+#define mmDSCC1_DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC1_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL 0x3090
+#define mmDSCC1_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC1_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL 0x3091
+#define mmDSCC1_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC1_DSCC_TEST_DEBUG_BUS_ROTATE 0x3096
+#define mmDSCC1_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc1_dispdec_dsc_dcperfmon_dc_perfmon_dispdec
+// base address: 0xc2b0
+#define mmDC_PERFMON22_PERFCOUNTER_CNTL 0x30ac
+#define mmDC_PERFMON22_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON22_PERFCOUNTER_CNTL2 0x30ad
+#define mmDC_PERFMON22_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON22_PERFCOUNTER_STATE 0x30ae
+#define mmDC_PERFMON22_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON22_PERFMON_CNTL 0x30af
+#define mmDC_PERFMON22_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON22_PERFMON_CNTL2 0x30b0
+#define mmDC_PERFMON22_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON22_PERFMON_CVALUE_INT_MISC 0x30b1
+#define mmDC_PERFMON22_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON22_PERFMON_CVALUE_LOW 0x30b2
+#define mmDC_PERFMON22_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON22_PERFMON_HI 0x30b3
+#define mmDC_PERFMON22_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON22_PERFMON_LOW 0x30b4
+#define mmDC_PERFMON22_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc2_dispdec_dsc_top_dispdec
+// base address: 0x2e0
+#define mmDSC_TOP2_DSC_TOP_CONTROL 0x30b8
+#define mmDSC_TOP2_DSC_TOP_CONTROL_BASE_IDX 2
+#define mmDSC_TOP2_DSC_DEBUG_CONTROL 0x30b9
+#define mmDSC_TOP2_DSC_DEBUG_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc2_dispdec_dsccif_dispdec
+// base address: 0x2e0
+#define mmDSCCIF2_DSCCIF_CONFIG0 0x30bd
+#define mmDSCCIF2_DSCCIF_CONFIG0_BASE_IDX 2
+#define mmDSCCIF2_DSCCIF_CONFIG1 0x30be
+#define mmDSCCIF2_DSCCIF_CONFIG1_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc2_dispdec_dscc_dispdec
+// base address: 0x2e0
+#define mmDSCC2_DSCC_CONFIG0 0x30c2
+#define mmDSCC2_DSCC_CONFIG0_BASE_IDX 2
+#define mmDSCC2_DSCC_CONFIG1 0x30c3
+#define mmDSCC2_DSCC_CONFIG1_BASE_IDX 2
+#define mmDSCC2_DSCC_STATUS 0x30c4
+#define mmDSCC2_DSCC_STATUS_BASE_IDX 2
+#define mmDSCC2_DSCC_INTERRUPT_CONTROL_STATUS 0x30c5
+#define mmDSCC2_DSCC_INTERRUPT_CONTROL_STATUS_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG0 0x30c6
+#define mmDSCC2_DSCC_PPS_CONFIG0_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG1 0x30c7
+#define mmDSCC2_DSCC_PPS_CONFIG1_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG2 0x30c8
+#define mmDSCC2_DSCC_PPS_CONFIG2_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG3 0x30c9
+#define mmDSCC2_DSCC_PPS_CONFIG3_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG4 0x30ca
+#define mmDSCC2_DSCC_PPS_CONFIG4_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG5 0x30cb
+#define mmDSCC2_DSCC_PPS_CONFIG5_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG6 0x30cc
+#define mmDSCC2_DSCC_PPS_CONFIG6_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG7 0x30cd
+#define mmDSCC2_DSCC_PPS_CONFIG7_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG8 0x30ce
+#define mmDSCC2_DSCC_PPS_CONFIG8_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG9 0x30cf
+#define mmDSCC2_DSCC_PPS_CONFIG9_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG10 0x30d0
+#define mmDSCC2_DSCC_PPS_CONFIG10_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG11 0x30d1
+#define mmDSCC2_DSCC_PPS_CONFIG11_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG12 0x30d2
+#define mmDSCC2_DSCC_PPS_CONFIG12_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG13 0x30d3
+#define mmDSCC2_DSCC_PPS_CONFIG13_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG14 0x30d4
+#define mmDSCC2_DSCC_PPS_CONFIG14_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG15 0x30d5
+#define mmDSCC2_DSCC_PPS_CONFIG15_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG16 0x30d6
+#define mmDSCC2_DSCC_PPS_CONFIG16_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG17 0x30d7
+#define mmDSCC2_DSCC_PPS_CONFIG17_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG18 0x30d8
+#define mmDSCC2_DSCC_PPS_CONFIG18_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG19 0x30d9
+#define mmDSCC2_DSCC_PPS_CONFIG19_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG20 0x30da
+#define mmDSCC2_DSCC_PPS_CONFIG20_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG21 0x30db
+#define mmDSCC2_DSCC_PPS_CONFIG21_BASE_IDX 2
+#define mmDSCC2_DSCC_PPS_CONFIG22 0x30dc
+#define mmDSCC2_DSCC_PPS_CONFIG22_BASE_IDX 2
+#define mmDSCC2_DSCC_MEM_POWER_CONTROL 0x30dd
+#define mmDSCC2_DSCC_MEM_POWER_CONTROL_BASE_IDX 2
+#define mmDSCC2_DSCC_R_Y_SQUARED_ERROR_LOWER 0x30de
+#define mmDSCC2_DSCC_R_Y_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC2_DSCC_R_Y_SQUARED_ERROR_UPPER 0x30df
+#define mmDSCC2_DSCC_R_Y_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC2_DSCC_G_CB_SQUARED_ERROR_LOWER 0x30e0
+#define mmDSCC2_DSCC_G_CB_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC2_DSCC_G_CB_SQUARED_ERROR_UPPER 0x30e1
+#define mmDSCC2_DSCC_G_CB_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC2_DSCC_B_CR_SQUARED_ERROR_LOWER 0x30e2
+#define mmDSCC2_DSCC_B_CR_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC2_DSCC_B_CR_SQUARED_ERROR_UPPER 0x30e3
+#define mmDSCC2_DSCC_B_CR_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC2_DSCC_MAX_ABS_ERROR0 0x30e4
+#define mmDSCC2_DSCC_MAX_ABS_ERROR0_BASE_IDX 2
+#define mmDSCC2_DSCC_MAX_ABS_ERROR1 0x30e5
+#define mmDSCC2_DSCC_MAX_ABS_ERROR1_BASE_IDX 2
+#define mmDSCC2_DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL 0x30e6
+#define mmDSCC2_DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC2_DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL 0x30e7
+#define mmDSCC2_DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC2_DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL 0x30e8
+#define mmDSCC2_DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC2_DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL 0x30e9
+#define mmDSCC2_DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC2_DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL 0x30ea
+#define mmDSCC2_DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC2_DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL 0x30eb
+#define mmDSCC2_DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC2_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL 0x30ec
+#define mmDSCC2_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC2_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL 0x30ed
+#define mmDSCC2_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC2_DSCC_TEST_DEBUG_BUS_ROTATE 0x30f2
+#define mmDSCC2_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc2_dispdec_dsc_dcperfmon_dc_perfmon_dispdec
+// base address: 0xc420
+#define mmDC_PERFMON23_PERFCOUNTER_CNTL 0x3108
+#define mmDC_PERFMON23_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON23_PERFCOUNTER_CNTL2 0x3109
+#define mmDC_PERFMON23_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON23_PERFCOUNTER_STATE 0x310a
+#define mmDC_PERFMON23_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON23_PERFMON_CNTL 0x310b
+#define mmDC_PERFMON23_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON23_PERFMON_CNTL2 0x310c
+#define mmDC_PERFMON23_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON23_PERFMON_CVALUE_INT_MISC 0x310d
+#define mmDC_PERFMON23_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON23_PERFMON_CVALUE_LOW 0x310e
+#define mmDC_PERFMON23_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON23_PERFMON_HI 0x310f
+#define mmDC_PERFMON23_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON23_PERFMON_LOW 0x3110
+#define mmDC_PERFMON23_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc3_dispdec_dsc_top_dispdec
+// base address: 0x450
+#define mmDSC_TOP3_DSC_TOP_CONTROL 0x3114
+#define mmDSC_TOP3_DSC_TOP_CONTROL_BASE_IDX 2
+#define mmDSC_TOP3_DSC_DEBUG_CONTROL 0x3115
+#define mmDSC_TOP3_DSC_DEBUG_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc3_dispdec_dsccif_dispdec
+// base address: 0x450
+#define mmDSCCIF3_DSCCIF_CONFIG0 0x3119
+#define mmDSCCIF3_DSCCIF_CONFIG0_BASE_IDX 2
+#define mmDSCCIF3_DSCCIF_CONFIG1 0x311a
+#define mmDSCCIF3_DSCCIF_CONFIG1_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc3_dispdec_dscc_dispdec
+// base address: 0x450
+#define mmDSCC3_DSCC_CONFIG0 0x311e
+#define mmDSCC3_DSCC_CONFIG0_BASE_IDX 2
+#define mmDSCC3_DSCC_CONFIG1 0x311f
+#define mmDSCC3_DSCC_CONFIG1_BASE_IDX 2
+#define mmDSCC3_DSCC_STATUS 0x3120
+#define mmDSCC3_DSCC_STATUS_BASE_IDX 2
+#define mmDSCC3_DSCC_INTERRUPT_CONTROL_STATUS 0x3121
+#define mmDSCC3_DSCC_INTERRUPT_CONTROL_STATUS_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG0 0x3122
+#define mmDSCC3_DSCC_PPS_CONFIG0_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG1 0x3123
+#define mmDSCC3_DSCC_PPS_CONFIG1_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG2 0x3124
+#define mmDSCC3_DSCC_PPS_CONFIG2_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG3 0x3125
+#define mmDSCC3_DSCC_PPS_CONFIG3_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG4 0x3126
+#define mmDSCC3_DSCC_PPS_CONFIG4_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG5 0x3127
+#define mmDSCC3_DSCC_PPS_CONFIG5_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG6 0x3128
+#define mmDSCC3_DSCC_PPS_CONFIG6_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG7 0x3129
+#define mmDSCC3_DSCC_PPS_CONFIG7_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG8 0x312a
+#define mmDSCC3_DSCC_PPS_CONFIG8_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG9 0x312b
+#define mmDSCC3_DSCC_PPS_CONFIG9_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG10 0x312c
+#define mmDSCC3_DSCC_PPS_CONFIG10_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG11 0x312d
+#define mmDSCC3_DSCC_PPS_CONFIG11_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG12 0x312e
+#define mmDSCC3_DSCC_PPS_CONFIG12_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG13 0x312f
+#define mmDSCC3_DSCC_PPS_CONFIG13_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG14 0x3130
+#define mmDSCC3_DSCC_PPS_CONFIG14_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG15 0x3131
+#define mmDSCC3_DSCC_PPS_CONFIG15_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG16 0x3132
+#define mmDSCC3_DSCC_PPS_CONFIG16_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG17 0x3133
+#define mmDSCC3_DSCC_PPS_CONFIG17_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG18 0x3134
+#define mmDSCC3_DSCC_PPS_CONFIG18_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG19 0x3135
+#define mmDSCC3_DSCC_PPS_CONFIG19_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG20 0x3136
+#define mmDSCC3_DSCC_PPS_CONFIG20_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG21 0x3137
+#define mmDSCC3_DSCC_PPS_CONFIG21_BASE_IDX 2
+#define mmDSCC3_DSCC_PPS_CONFIG22 0x3138
+#define mmDSCC3_DSCC_PPS_CONFIG22_BASE_IDX 2
+#define mmDSCC3_DSCC_MEM_POWER_CONTROL 0x3139
+#define mmDSCC3_DSCC_MEM_POWER_CONTROL_BASE_IDX 2
+#define mmDSCC3_DSCC_R_Y_SQUARED_ERROR_LOWER 0x313a
+#define mmDSCC3_DSCC_R_Y_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC3_DSCC_R_Y_SQUARED_ERROR_UPPER 0x313b
+#define mmDSCC3_DSCC_R_Y_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC3_DSCC_G_CB_SQUARED_ERROR_LOWER 0x313c
+#define mmDSCC3_DSCC_G_CB_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC3_DSCC_G_CB_SQUARED_ERROR_UPPER 0x313d
+#define mmDSCC3_DSCC_G_CB_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC3_DSCC_B_CR_SQUARED_ERROR_LOWER 0x313e
+#define mmDSCC3_DSCC_B_CR_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC3_DSCC_B_CR_SQUARED_ERROR_UPPER 0x313f
+#define mmDSCC3_DSCC_B_CR_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC3_DSCC_MAX_ABS_ERROR0 0x3140
+#define mmDSCC3_DSCC_MAX_ABS_ERROR0_BASE_IDX 2
+#define mmDSCC3_DSCC_MAX_ABS_ERROR1 0x3141
+#define mmDSCC3_DSCC_MAX_ABS_ERROR1_BASE_IDX 2
+#define mmDSCC3_DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL 0x3142
+#define mmDSCC3_DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC3_DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL 0x3143
+#define mmDSCC3_DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC3_DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL 0x3144
+#define mmDSCC3_DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC3_DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL 0x3145
+#define mmDSCC3_DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC3_DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL 0x3146
+#define mmDSCC3_DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC3_DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL 0x3147
+#define mmDSCC3_DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC3_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL 0x3148
+#define mmDSCC3_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC3_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL 0x3149
+#define mmDSCC3_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC3_DSCC_TEST_DEBUG_BUS_ROTATE 0x314e
+#define mmDSCC3_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc3_dispdec_dsc_dcperfmon_dc_perfmon_dispdec
+// base address: 0xc590
+#define mmDC_PERFMON24_PERFCOUNTER_CNTL 0x3164
+#define mmDC_PERFMON24_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON24_PERFCOUNTER_CNTL2 0x3165
+#define mmDC_PERFMON24_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON24_PERFCOUNTER_STATE 0x3166
+#define mmDC_PERFMON24_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON24_PERFMON_CNTL 0x3167
+#define mmDC_PERFMON24_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON24_PERFMON_CNTL2 0x3168
+#define mmDC_PERFMON24_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON24_PERFMON_CVALUE_INT_MISC 0x3169
+#define mmDC_PERFMON24_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON24_PERFMON_CVALUE_LOW 0x316a
+#define mmDC_PERFMON24_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON24_PERFMON_HI 0x316b
+#define mmDC_PERFMON24_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON24_PERFMON_LOW 0x316c
+#define mmDC_PERFMON24_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc4_dispdec_dsc_top_dispdec
+// base address: 0x5c0
+#define mmDSC_TOP4_DSC_TOP_CONTROL 0x3170
+#define mmDSC_TOP4_DSC_TOP_CONTROL_BASE_IDX 2
+#define mmDSC_TOP4_DSC_DEBUG_CONTROL 0x3171
+#define mmDSC_TOP4_DSC_DEBUG_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc4_dispdec_dsccif_dispdec
+// base address: 0x5c0
+#define mmDSCCIF4_DSCCIF_CONFIG0 0x3175
+#define mmDSCCIF4_DSCCIF_CONFIG0_BASE_IDX 2
+#define mmDSCCIF4_DSCCIF_CONFIG1 0x3176
+#define mmDSCCIF4_DSCCIF_CONFIG1_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc4_dispdec_dscc_dispdec
+// base address: 0x5c0
+#define mmDSCC4_DSCC_CONFIG0 0x317a
+#define mmDSCC4_DSCC_CONFIG0_BASE_IDX 2
+#define mmDSCC4_DSCC_CONFIG1 0x317b
+#define mmDSCC4_DSCC_CONFIG1_BASE_IDX 2
+#define mmDSCC4_DSCC_STATUS 0x317c
+#define mmDSCC4_DSCC_STATUS_BASE_IDX 2
+#define mmDSCC4_DSCC_INTERRUPT_CONTROL_STATUS 0x317d
+#define mmDSCC4_DSCC_INTERRUPT_CONTROL_STATUS_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG0 0x317e
+#define mmDSCC4_DSCC_PPS_CONFIG0_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG1 0x317f
+#define mmDSCC4_DSCC_PPS_CONFIG1_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG2 0x3180
+#define mmDSCC4_DSCC_PPS_CONFIG2_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG3 0x3181
+#define mmDSCC4_DSCC_PPS_CONFIG3_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG4 0x3182
+#define mmDSCC4_DSCC_PPS_CONFIG4_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG5 0x3183
+#define mmDSCC4_DSCC_PPS_CONFIG5_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG6 0x3184
+#define mmDSCC4_DSCC_PPS_CONFIG6_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG7 0x3185
+#define mmDSCC4_DSCC_PPS_CONFIG7_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG8 0x3186
+#define mmDSCC4_DSCC_PPS_CONFIG8_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG9 0x3187
+#define mmDSCC4_DSCC_PPS_CONFIG9_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG10 0x3188
+#define mmDSCC4_DSCC_PPS_CONFIG10_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG11 0x3189
+#define mmDSCC4_DSCC_PPS_CONFIG11_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG12 0x318a
+#define mmDSCC4_DSCC_PPS_CONFIG12_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG13 0x318b
+#define mmDSCC4_DSCC_PPS_CONFIG13_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG14 0x318c
+#define mmDSCC4_DSCC_PPS_CONFIG14_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG15 0x318d
+#define mmDSCC4_DSCC_PPS_CONFIG15_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG16 0x318e
+#define mmDSCC4_DSCC_PPS_CONFIG16_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG17 0x318f
+#define mmDSCC4_DSCC_PPS_CONFIG17_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG18 0x3190
+#define mmDSCC4_DSCC_PPS_CONFIG18_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG19 0x3191
+#define mmDSCC4_DSCC_PPS_CONFIG19_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG20 0x3192
+#define mmDSCC4_DSCC_PPS_CONFIG20_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG21 0x3193
+#define mmDSCC4_DSCC_PPS_CONFIG21_BASE_IDX 2
+#define mmDSCC4_DSCC_PPS_CONFIG22 0x3194
+#define mmDSCC4_DSCC_PPS_CONFIG22_BASE_IDX 2
+#define mmDSCC4_DSCC_MEM_POWER_CONTROL 0x3195
+#define mmDSCC4_DSCC_MEM_POWER_CONTROL_BASE_IDX 2
+#define mmDSCC4_DSCC_R_Y_SQUARED_ERROR_LOWER 0x3196
+#define mmDSCC4_DSCC_R_Y_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC4_DSCC_R_Y_SQUARED_ERROR_UPPER 0x3197
+#define mmDSCC4_DSCC_R_Y_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC4_DSCC_G_CB_SQUARED_ERROR_LOWER 0x3198
+#define mmDSCC4_DSCC_G_CB_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC4_DSCC_G_CB_SQUARED_ERROR_UPPER 0x3199
+#define mmDSCC4_DSCC_G_CB_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC4_DSCC_B_CR_SQUARED_ERROR_LOWER 0x319a
+#define mmDSCC4_DSCC_B_CR_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC4_DSCC_B_CR_SQUARED_ERROR_UPPER 0x319b
+#define mmDSCC4_DSCC_B_CR_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC4_DSCC_MAX_ABS_ERROR0 0x319c
+#define mmDSCC4_DSCC_MAX_ABS_ERROR0_BASE_IDX 2
+#define mmDSCC4_DSCC_MAX_ABS_ERROR1 0x319d
+#define mmDSCC4_DSCC_MAX_ABS_ERROR1_BASE_IDX 2
+#define mmDSCC4_DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL 0x319e
+#define mmDSCC4_DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC4_DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL 0x319f
+#define mmDSCC4_DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC4_DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL 0x31a0
+#define mmDSCC4_DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC4_DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL 0x31a1
+#define mmDSCC4_DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC4_DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL 0x31a2
+#define mmDSCC4_DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC4_DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL 0x31a3
+#define mmDSCC4_DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC4_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL 0x31a4
+#define mmDSCC4_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC4_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL 0x31a5
+#define mmDSCC4_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC4_DSCC_TEST_DEBUG_BUS_ROTATE 0x31aa
+#define mmDSCC4_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc4_dispdec_dsc_dcperfmon_dc_perfmon_dispdec
+// base address: 0xc700
+#define mmDC_PERFMON25_PERFCOUNTER_CNTL 0x31c0
+#define mmDC_PERFMON25_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON25_PERFCOUNTER_CNTL2 0x31c1
+#define mmDC_PERFMON25_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON25_PERFCOUNTER_STATE 0x31c2
+#define mmDC_PERFMON25_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON25_PERFMON_CNTL 0x31c3
+#define mmDC_PERFMON25_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON25_PERFMON_CNTL2 0x31c4
+#define mmDC_PERFMON25_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON25_PERFMON_CVALUE_INT_MISC 0x31c5
+#define mmDC_PERFMON25_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON25_PERFMON_CVALUE_LOW 0x31c6
+#define mmDC_PERFMON25_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON25_PERFMON_HI 0x31c7
+#define mmDC_PERFMON25_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON25_PERFMON_LOW 0x31c8
+#define mmDC_PERFMON25_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc5_dispdec_dsc_top_dispdec
+// base address: 0x730
+#define mmDSC_TOP5_DSC_TOP_CONTROL 0x31cc
+#define mmDSC_TOP5_DSC_TOP_CONTROL_BASE_IDX 2
+#define mmDSC_TOP5_DSC_DEBUG_CONTROL 0x31cd
+#define mmDSC_TOP5_DSC_DEBUG_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc5_dispdec_dsccif_dispdec
+// base address: 0x730
+#define mmDSCCIF5_DSCCIF_CONFIG0 0x31d1
+#define mmDSCCIF5_DSCCIF_CONFIG0_BASE_IDX 2
+#define mmDSCCIF5_DSCCIF_CONFIG1 0x31d2
+#define mmDSCCIF5_DSCCIF_CONFIG1_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc5_dispdec_dscc_dispdec
+// base address: 0x730
+#define mmDSCC5_DSCC_CONFIG0 0x31d6
+#define mmDSCC5_DSCC_CONFIG0_BASE_IDX 2
+#define mmDSCC5_DSCC_CONFIG1 0x31d7
+#define mmDSCC5_DSCC_CONFIG1_BASE_IDX 2
+#define mmDSCC5_DSCC_STATUS 0x31d8
+#define mmDSCC5_DSCC_STATUS_BASE_IDX 2
+#define mmDSCC5_DSCC_INTERRUPT_CONTROL_STATUS 0x31d9
+#define mmDSCC5_DSCC_INTERRUPT_CONTROL_STATUS_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG0 0x31da
+#define mmDSCC5_DSCC_PPS_CONFIG0_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG1 0x31db
+#define mmDSCC5_DSCC_PPS_CONFIG1_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG2 0x31dc
+#define mmDSCC5_DSCC_PPS_CONFIG2_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG3 0x31dd
+#define mmDSCC5_DSCC_PPS_CONFIG3_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG4 0x31de
+#define mmDSCC5_DSCC_PPS_CONFIG4_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG5 0x31df
+#define mmDSCC5_DSCC_PPS_CONFIG5_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG6 0x31e0
+#define mmDSCC5_DSCC_PPS_CONFIG6_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG7 0x31e1
+#define mmDSCC5_DSCC_PPS_CONFIG7_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG8 0x31e2
+#define mmDSCC5_DSCC_PPS_CONFIG8_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG9 0x31e3
+#define mmDSCC5_DSCC_PPS_CONFIG9_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG10 0x31e4
+#define mmDSCC5_DSCC_PPS_CONFIG10_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG11 0x31e5
+#define mmDSCC5_DSCC_PPS_CONFIG11_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG12 0x31e6
+#define mmDSCC5_DSCC_PPS_CONFIG12_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG13 0x31e7
+#define mmDSCC5_DSCC_PPS_CONFIG13_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG14 0x31e8
+#define mmDSCC5_DSCC_PPS_CONFIG14_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG15 0x31e9
+#define mmDSCC5_DSCC_PPS_CONFIG15_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG16 0x31ea
+#define mmDSCC5_DSCC_PPS_CONFIG16_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG17 0x31eb
+#define mmDSCC5_DSCC_PPS_CONFIG17_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG18 0x31ec
+#define mmDSCC5_DSCC_PPS_CONFIG18_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG19 0x31ed
+#define mmDSCC5_DSCC_PPS_CONFIG19_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG20 0x31ee
+#define mmDSCC5_DSCC_PPS_CONFIG20_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG21 0x31ef
+#define mmDSCC5_DSCC_PPS_CONFIG21_BASE_IDX 2
+#define mmDSCC5_DSCC_PPS_CONFIG22 0x31f0
+#define mmDSCC5_DSCC_PPS_CONFIG22_BASE_IDX 2
+#define mmDSCC5_DSCC_MEM_POWER_CONTROL 0x31f1
+#define mmDSCC5_DSCC_MEM_POWER_CONTROL_BASE_IDX 2
+#define mmDSCC5_DSCC_R_Y_SQUARED_ERROR_LOWER 0x31f2
+#define mmDSCC5_DSCC_R_Y_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC5_DSCC_R_Y_SQUARED_ERROR_UPPER 0x31f3
+#define mmDSCC5_DSCC_R_Y_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC5_DSCC_G_CB_SQUARED_ERROR_LOWER 0x31f4
+#define mmDSCC5_DSCC_G_CB_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC5_DSCC_G_CB_SQUARED_ERROR_UPPER 0x31f5
+#define mmDSCC5_DSCC_G_CB_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC5_DSCC_B_CR_SQUARED_ERROR_LOWER 0x31f6
+#define mmDSCC5_DSCC_B_CR_SQUARED_ERROR_LOWER_BASE_IDX 2
+#define mmDSCC5_DSCC_B_CR_SQUARED_ERROR_UPPER 0x31f7
+#define mmDSCC5_DSCC_B_CR_SQUARED_ERROR_UPPER_BASE_IDX 2
+#define mmDSCC5_DSCC_MAX_ABS_ERROR0 0x31f8
+#define mmDSCC5_DSCC_MAX_ABS_ERROR0_BASE_IDX 2
+#define mmDSCC5_DSCC_MAX_ABS_ERROR1 0x31f9
+#define mmDSCC5_DSCC_MAX_ABS_ERROR1_BASE_IDX 2
+#define mmDSCC5_DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL 0x31fa
+#define mmDSCC5_DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC5_DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL 0x31fb
+#define mmDSCC5_DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC5_DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL 0x31fc
+#define mmDSCC5_DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC5_DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL 0x31fd
+#define mmDSCC5_DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC5_DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL 0x31fe
+#define mmDSCC5_DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC5_DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL 0x31ff
+#define mmDSCC5_DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC5_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL 0x3200
+#define mmDSCC5_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC5_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL 0x3201
+#define mmDSCC5_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX 2
+#define mmDSCC5_DSCC_TEST_DEBUG_BUS_ROTATE 0x3206
+#define mmDSCC5_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dsc5_dispdec_dsc_dcperfmon_dc_perfmon_dispdec
+// base address: 0xc870
+#define mmDC_PERFMON26_PERFCOUNTER_CNTL 0x321c
+#define mmDC_PERFMON26_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON26_PERFCOUNTER_CNTL2 0x321d
+#define mmDC_PERFMON26_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON26_PERFCOUNTER_STATE 0x321e
+#define mmDC_PERFMON26_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON26_PERFMON_CNTL 0x321f
+#define mmDC_PERFMON26_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON26_PERFMON_CNTL2 0x3220
+#define mmDC_PERFMON26_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON26_PERFMON_CVALUE_INT_MISC 0x3221
+#define mmDC_PERFMON26_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON26_PERFMON_CVALUE_LOW 0x3222
+#define mmDC_PERFMON26_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON26_PERFMON_HI 0x3223
+#define mmDC_PERFMON26_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON26_PERFMON_LOW 0x3224
+#define mmDC_PERFMON26_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dmu_dmcub_dispdec
+// base address: 0x0
+#define mmDMCUB_REGION0_OFFSET 0x3238
+#define mmDMCUB_REGION0_OFFSET_BASE_IDX 2
+#define mmDMCUB_REGION0_OFFSET_HIGH 0x3239
+#define mmDMCUB_REGION0_OFFSET_HIGH_BASE_IDX 2
+#define mmDMCUB_REGION1_OFFSET 0x323a
+#define mmDMCUB_REGION1_OFFSET_BASE_IDX 2
+#define mmDMCUB_REGION1_OFFSET_HIGH 0x323b
+#define mmDMCUB_REGION1_OFFSET_HIGH_BASE_IDX 2
+#define mmDMCUB_REGION2_OFFSET 0x323c
+#define mmDMCUB_REGION2_OFFSET_BASE_IDX 2
+#define mmDMCUB_REGION2_OFFSET_HIGH 0x323d
+#define mmDMCUB_REGION2_OFFSET_HIGH_BASE_IDX 2
+#define mmDMCUB_REGION4_OFFSET 0x3240
+#define mmDMCUB_REGION4_OFFSET_BASE_IDX 2
+#define mmDMCUB_REGION4_OFFSET_HIGH 0x3241
+#define mmDMCUB_REGION4_OFFSET_HIGH_BASE_IDX 2
+#define mmDMCUB_REGION5_OFFSET 0x3242
+#define mmDMCUB_REGION5_OFFSET_BASE_IDX 2
+#define mmDMCUB_REGION5_OFFSET_HIGH 0x3243
+#define mmDMCUB_REGION5_OFFSET_HIGH_BASE_IDX 2
+#define mmDMCUB_REGION6_OFFSET 0x3244
+#define mmDMCUB_REGION6_OFFSET_BASE_IDX 2
+#define mmDMCUB_REGION6_OFFSET_HIGH 0x3245
+#define mmDMCUB_REGION6_OFFSET_HIGH_BASE_IDX 2
+#define mmDMCUB_REGION7_OFFSET 0x3246
+#define mmDMCUB_REGION7_OFFSET_BASE_IDX 2
+#define mmDMCUB_REGION7_OFFSET_HIGH 0x3247
+#define mmDMCUB_REGION7_OFFSET_HIGH_BASE_IDX 2
+#define mmDMCUB_REGION0_TOP_ADDRESS 0x3248
+#define mmDMCUB_REGION0_TOP_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION1_TOP_ADDRESS 0x3249
+#define mmDMCUB_REGION1_TOP_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION2_TOP_ADDRESS 0x324a
+#define mmDMCUB_REGION2_TOP_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION4_TOP_ADDRESS 0x324b
+#define mmDMCUB_REGION4_TOP_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION5_TOP_ADDRESS 0x324c
+#define mmDMCUB_REGION5_TOP_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION6_TOP_ADDRESS 0x324d
+#define mmDMCUB_REGION6_TOP_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION7_TOP_ADDRESS 0x324e
+#define mmDMCUB_REGION7_TOP_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION3_CW0_BASE_ADDRESS 0x324f
+#define mmDMCUB_REGION3_CW0_BASE_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION3_CW1_BASE_ADDRESS 0x3250
+#define mmDMCUB_REGION3_CW1_BASE_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION3_CW2_BASE_ADDRESS 0x3251
+#define mmDMCUB_REGION3_CW2_BASE_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION3_CW3_BASE_ADDRESS 0x3252
+#define mmDMCUB_REGION3_CW3_BASE_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION3_CW4_BASE_ADDRESS 0x3253
+#define mmDMCUB_REGION3_CW4_BASE_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION3_CW5_BASE_ADDRESS 0x3254
+#define mmDMCUB_REGION3_CW5_BASE_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION3_CW6_BASE_ADDRESS 0x3255
+#define mmDMCUB_REGION3_CW6_BASE_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION3_CW7_BASE_ADDRESS 0x3256
+#define mmDMCUB_REGION3_CW7_BASE_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION3_CW0_TOP_ADDRESS 0x3257
+#define mmDMCUB_REGION3_CW0_TOP_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION3_CW1_TOP_ADDRESS 0x3258
+#define mmDMCUB_REGION3_CW1_TOP_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION3_CW2_TOP_ADDRESS 0x3259
+#define mmDMCUB_REGION3_CW2_TOP_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION3_CW3_TOP_ADDRESS 0x325a
+#define mmDMCUB_REGION3_CW3_TOP_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION3_CW4_TOP_ADDRESS 0x325b
+#define mmDMCUB_REGION3_CW4_TOP_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION3_CW5_TOP_ADDRESS 0x325c
+#define mmDMCUB_REGION3_CW5_TOP_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION3_CW6_TOP_ADDRESS 0x325d
+#define mmDMCUB_REGION3_CW6_TOP_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION3_CW7_TOP_ADDRESS 0x325e
+#define mmDMCUB_REGION3_CW7_TOP_ADDRESS_BASE_IDX 2
+#define mmDMCUB_REGION3_CW0_OFFSET 0x325f
+#define mmDMCUB_REGION3_CW0_OFFSET_BASE_IDX 2
+#define mmDMCUB_REGION3_CW0_OFFSET_HIGH 0x3260
+#define mmDMCUB_REGION3_CW0_OFFSET_HIGH_BASE_IDX 2
+#define mmDMCUB_REGION3_CW1_OFFSET 0x3261
+#define mmDMCUB_REGION3_CW1_OFFSET_BASE_IDX 2
+#define mmDMCUB_REGION3_CW1_OFFSET_HIGH 0x3262
+#define mmDMCUB_REGION3_CW1_OFFSET_HIGH_BASE_IDX 2
+#define mmDMCUB_REGION3_CW2_OFFSET 0x3263
+#define mmDMCUB_REGION3_CW2_OFFSET_BASE_IDX 2
+#define mmDMCUB_REGION3_CW2_OFFSET_HIGH 0x3264
+#define mmDMCUB_REGION3_CW2_OFFSET_HIGH_BASE_IDX 2
+#define mmDMCUB_REGION3_CW3_OFFSET 0x3265
+#define mmDMCUB_REGION3_CW3_OFFSET_BASE_IDX 2
+#define mmDMCUB_REGION3_CW3_OFFSET_HIGH 0x3266
+#define mmDMCUB_REGION3_CW3_OFFSET_HIGH_BASE_IDX 2
+#define mmDMCUB_REGION3_CW4_OFFSET 0x3267
+#define mmDMCUB_REGION3_CW4_OFFSET_BASE_IDX 2
+#define mmDMCUB_REGION3_CW4_OFFSET_HIGH 0x3268
+#define mmDMCUB_REGION3_CW4_OFFSET_HIGH_BASE_IDX 2
+#define mmDMCUB_REGION3_CW5_OFFSET 0x3269
+#define mmDMCUB_REGION3_CW5_OFFSET_BASE_IDX 2
+#define mmDMCUB_REGION3_CW5_OFFSET_HIGH 0x326a
+#define mmDMCUB_REGION3_CW5_OFFSET_HIGH_BASE_IDX 2
+#define mmDMCUB_REGION3_CW6_OFFSET 0x326b
+#define mmDMCUB_REGION3_CW6_OFFSET_BASE_IDX 2
+#define mmDMCUB_REGION3_CW6_OFFSET_HIGH 0x326c
+#define mmDMCUB_REGION3_CW6_OFFSET_HIGH_BASE_IDX 2
+#define mmDMCUB_REGION3_CW7_OFFSET 0x326d
+#define mmDMCUB_REGION3_CW7_OFFSET_BASE_IDX 2
+#define mmDMCUB_REGION3_CW7_OFFSET_HIGH 0x326e
+#define mmDMCUB_REGION3_CW7_OFFSET_HIGH_BASE_IDX 2
+#define mmDMCUB_INTERRUPT_ENABLE 0x326f
+#define mmDMCUB_INTERRUPT_ENABLE_BASE_IDX 2
+#define mmDMCUB_INTERRUPT_ACK 0x3270
+#define mmDMCUB_INTERRUPT_ACK_BASE_IDX 2
+#define mmDMCUB_INTERRUPT_STATUS 0x3271
+#define mmDMCUB_INTERRUPT_STATUS_BASE_IDX 2
+#define mmDMCUB_INTERRUPT_TYPE 0x3272
+#define mmDMCUB_INTERRUPT_TYPE_BASE_IDX 2
+#define mmDMCUB_EXT_INTERRUPT_STATUS 0x3273
+#define mmDMCUB_EXT_INTERRUPT_STATUS_BASE_IDX 2
+#define mmDMCUB_EXT_INTERRUPT_CTXID 0x3274
+#define mmDMCUB_EXT_INTERRUPT_CTXID_BASE_IDX 2
+#define mmDMCUB_EXT_INTERRUPT_ACK 0x3275
+#define mmDMCUB_EXT_INTERRUPT_ACK_BASE_IDX 2
+#define mmDMCUB_INST_FETCH_FAULT_ADDR 0x3276
+#define mmDMCUB_INST_FETCH_FAULT_ADDR_BASE_IDX 2
+#define mmDMCUB_DATA_WRITE_FAULT_ADDR 0x3277
+#define mmDMCUB_DATA_WRITE_FAULT_ADDR_BASE_IDX 2
+#define mmDMCUB_SEC_CNTL 0x3278
+#define mmDMCUB_SEC_CNTL_BASE_IDX 2
+#define mmDMCUB_MEM_CNTL 0x3279
+#define mmDMCUB_MEM_CNTL_BASE_IDX 2
+#define mmDMCUB_INBOX0_BASE_ADDRESS 0x327a
+#define mmDMCUB_INBOX0_BASE_ADDRESS_BASE_IDX 2
+#define mmDMCUB_INBOX0_SIZE 0x327b
+#define mmDMCUB_INBOX0_SIZE_BASE_IDX 2
+#define mmDMCUB_INBOX0_WPTR 0x327c
+#define mmDMCUB_INBOX0_WPTR_BASE_IDX 2
+#define mmDMCUB_INBOX0_RPTR 0x327d
+#define mmDMCUB_INBOX0_RPTR_BASE_IDX 2
+#define mmDMCUB_INBOX1_BASE_ADDRESS 0x327e
+#define mmDMCUB_INBOX1_BASE_ADDRESS_BASE_IDX 2
+#define mmDMCUB_INBOX1_SIZE 0x327f
+#define mmDMCUB_INBOX1_SIZE_BASE_IDX 2
+#define mmDMCUB_INBOX1_WPTR 0x3280
+#define mmDMCUB_INBOX1_WPTR_BASE_IDX 2
+#define mmDMCUB_INBOX1_RPTR 0x3281
+#define mmDMCUB_INBOX1_RPTR_BASE_IDX 2
+#define mmDMCUB_OUTBOX0_BASE_ADDRESS 0x3282
+#define mmDMCUB_OUTBOX0_BASE_ADDRESS_BASE_IDX 2
+#define mmDMCUB_OUTBOX0_SIZE 0x3283
+#define mmDMCUB_OUTBOX0_SIZE_BASE_IDX 2
+#define mmDMCUB_OUTBOX0_WPTR 0x3284
+#define mmDMCUB_OUTBOX0_WPTR_BASE_IDX 2
+#define mmDMCUB_OUTBOX0_RPTR 0x3285
+#define mmDMCUB_OUTBOX0_RPTR_BASE_IDX 2
+#define mmDMCUB_OUTBOX1_BASE_ADDRESS 0x3286
+#define mmDMCUB_OUTBOX1_BASE_ADDRESS_BASE_IDX 2
+#define mmDMCUB_OUTBOX1_SIZE 0x3287
+#define mmDMCUB_OUTBOX1_SIZE_BASE_IDX 2
+#define mmDMCUB_OUTBOX1_WPTR 0x3288
+#define mmDMCUB_OUTBOX1_WPTR_BASE_IDX 2
+#define mmDMCUB_OUTBOX1_RPTR 0x3289
+#define mmDMCUB_OUTBOX1_RPTR_BASE_IDX 2
+#define mmDMCUB_TIMER_TRIGGER0 0x328a
+#define mmDMCUB_TIMER_TRIGGER0_BASE_IDX 2
+#define mmDMCUB_TIMER_TRIGGER1 0x328b
+#define mmDMCUB_TIMER_TRIGGER1_BASE_IDX 2
+#define mmDMCUB_TIMER_WINDOW 0x328c
+#define mmDMCUB_TIMER_WINDOW_BASE_IDX 2
+#define mmDMCUB_SCRATCH0 0x328d
+#define mmDMCUB_SCRATCH0_BASE_IDX 2
+#define mmDMCUB_SCRATCH1 0x328e
+#define mmDMCUB_SCRATCH1_BASE_IDX 2
+#define mmDMCUB_SCRATCH2 0x328f
+#define mmDMCUB_SCRATCH2_BASE_IDX 2
+#define mmDMCUB_SCRATCH3 0x3290
+#define mmDMCUB_SCRATCH3_BASE_IDX 2
+#define mmDMCUB_SCRATCH4 0x3291
+#define mmDMCUB_SCRATCH4_BASE_IDX 2
+#define mmDMCUB_SCRATCH5 0x3292
+#define mmDMCUB_SCRATCH5_BASE_IDX 2
+#define mmDMCUB_SCRATCH6 0x3293
+#define mmDMCUB_SCRATCH6_BASE_IDX 2
+#define mmDMCUB_SCRATCH7 0x3294
+#define mmDMCUB_SCRATCH7_BASE_IDX 2
+#define mmDMCUB_SCRATCH8 0x3295
+#define mmDMCUB_SCRATCH8_BASE_IDX 2
+#define mmDMCUB_SCRATCH9 0x3296
+#define mmDMCUB_SCRATCH9_BASE_IDX 2
+#define mmDMCUB_SCRATCH10 0x3297
+#define mmDMCUB_SCRATCH10_BASE_IDX 2
+#define mmDMCUB_SCRATCH11 0x3298
+#define mmDMCUB_SCRATCH11_BASE_IDX 2
+#define mmDMCUB_SCRATCH12 0x3299
+#define mmDMCUB_SCRATCH12_BASE_IDX 2
+#define mmDMCUB_SCRATCH13 0x329a
+#define mmDMCUB_SCRATCH13_BASE_IDX 2
+#define mmDMCUB_SCRATCH14 0x329b
+#define mmDMCUB_SCRATCH14_BASE_IDX 2
+#define mmDMCUB_SCRATCH15 0x329c
+#define mmDMCUB_SCRATCH15_BASE_IDX 2
+#define mmDMCUB_CNTL 0x32a0
+#define mmDMCUB_CNTL_BASE_IDX 2
+#define mmDMCUB_GPINT_DATAIN0 0x32a1
+#define mmDMCUB_GPINT_DATAIN0_BASE_IDX 2
+#define mmDMCUB_GPINT_DATAIN1 0x32a2
+#define mmDMCUB_GPINT_DATAIN1_BASE_IDX 2
+#define mmDMCUB_GPINT_DATAOUT 0x32a3
+#define mmDMCUB_GPINT_DATAOUT_BASE_IDX 2
+#define mmDMCUB_UNDEFINED_ADDRESS_FAULT_ADDR 0x32a4
+#define mmDMCUB_UNDEFINED_ADDRESS_FAULT_ADDR_BASE_IDX 2
+#define mmDMCUB_LS_WAKE_INT_ENABLE 0x32a5
+#define mmDMCUB_LS_WAKE_INT_ENABLE_BASE_IDX 2
+#define mmDMCUB_MEM_PWR_CNTL 0x32a6
+#define mmDMCUB_MEM_PWR_CNTL_BASE_IDX 2
+#define mmDMCUB_TIMER_CURRENT 0x32a7
+#define mmDMCUB_TIMER_CURRENT_BASE_IDX 2
+#define mmDMCUB_PROC_ID 0x32a9
+#define mmDMCUB_PROC_ID_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mmhubbub_mcif_wb2_dispdec
+// base address: 0xc6b8
+#define mmMCIF_WB2_MCIF_WB_BUFMGR_SW_CONTROL 0x3460
+#define mmMCIF_WB2_MCIF_WB_BUFMGR_SW_CONTROL_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUFMGR_CUR_LINE_R 0x3461
+#define mmMCIF_WB2_MCIF_WB_BUFMGR_CUR_LINE_R_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUFMGR_STATUS 0x3462
+#define mmMCIF_WB2_MCIF_WB_BUFMGR_STATUS_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_PITCH 0x3463
+#define mmMCIF_WB2_MCIF_WB_BUF_PITCH_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_1_STATUS 0x3464
+#define mmMCIF_WB2_MCIF_WB_BUF_1_STATUS_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_1_STATUS2 0x3465
+#define mmMCIF_WB2_MCIF_WB_BUF_1_STATUS2_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_2_STATUS 0x3466
+#define mmMCIF_WB2_MCIF_WB_BUF_2_STATUS_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_2_STATUS2 0x3467
+#define mmMCIF_WB2_MCIF_WB_BUF_2_STATUS2_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_3_STATUS 0x3468
+#define mmMCIF_WB2_MCIF_WB_BUF_3_STATUS_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_3_STATUS2 0x3469
+#define mmMCIF_WB2_MCIF_WB_BUF_3_STATUS2_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_4_STATUS 0x346a
+#define mmMCIF_WB2_MCIF_WB_BUF_4_STATUS_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_4_STATUS2 0x346b
+#define mmMCIF_WB2_MCIF_WB_BUF_4_STATUS2_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_ARBITRATION_CONTROL 0x346c
+#define mmMCIF_WB2_MCIF_WB_ARBITRATION_CONTROL_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_SCLK_CHANGE 0x346d
+#define mmMCIF_WB2_MCIF_WB_SCLK_CHANGE_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_TEST_DEBUG_INDEX 0x346e
+#define mmMCIF_WB2_MCIF_WB_TEST_DEBUG_INDEX_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_TEST_DEBUG_DATA 0x346f
+#define mmMCIF_WB2_MCIF_WB_TEST_DEBUG_DATA_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_1_ADDR_Y 0x3470
+#define mmMCIF_WB2_MCIF_WB_BUF_1_ADDR_Y_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_1_ADDR_Y_OFFSET 0x3471
+#define mmMCIF_WB2_MCIF_WB_BUF_1_ADDR_Y_OFFSET_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_1_ADDR_C 0x3472
+#define mmMCIF_WB2_MCIF_WB_BUF_1_ADDR_C_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_1_ADDR_C_OFFSET 0x3473
+#define mmMCIF_WB2_MCIF_WB_BUF_1_ADDR_C_OFFSET_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_2_ADDR_Y 0x3474
+#define mmMCIF_WB2_MCIF_WB_BUF_2_ADDR_Y_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_2_ADDR_Y_OFFSET 0x3475
+#define mmMCIF_WB2_MCIF_WB_BUF_2_ADDR_Y_OFFSET_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_2_ADDR_C 0x3476
+#define mmMCIF_WB2_MCIF_WB_BUF_2_ADDR_C_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_2_ADDR_C_OFFSET 0x3477
+#define mmMCIF_WB2_MCIF_WB_BUF_2_ADDR_C_OFFSET_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_3_ADDR_Y 0x3478
+#define mmMCIF_WB2_MCIF_WB_BUF_3_ADDR_Y_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_3_ADDR_Y_OFFSET 0x3479
+#define mmMCIF_WB2_MCIF_WB_BUF_3_ADDR_Y_OFFSET_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_3_ADDR_C 0x347a
+#define mmMCIF_WB2_MCIF_WB_BUF_3_ADDR_C_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_3_ADDR_C_OFFSET 0x347b
+#define mmMCIF_WB2_MCIF_WB_BUF_3_ADDR_C_OFFSET_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_4_ADDR_Y 0x347c
+#define mmMCIF_WB2_MCIF_WB_BUF_4_ADDR_Y_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_4_ADDR_Y_OFFSET 0x347d
+#define mmMCIF_WB2_MCIF_WB_BUF_4_ADDR_Y_OFFSET_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_4_ADDR_C 0x347e
+#define mmMCIF_WB2_MCIF_WB_BUF_4_ADDR_C_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_4_ADDR_C_OFFSET 0x347f
+#define mmMCIF_WB2_MCIF_WB_BUF_4_ADDR_C_OFFSET_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUFMGR_VCE_CONTROL 0x3480
+#define mmMCIF_WB2_MCIF_WB_BUFMGR_VCE_CONTROL_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_NB_PSTATE_LATENCY_WATERMARK 0x3481
+#define mmMCIF_WB2_MCIF_WB_NB_PSTATE_LATENCY_WATERMARK_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_NB_PSTATE_CONTROL 0x3482
+#define mmMCIF_WB2_MCIF_WB_NB_PSTATE_CONTROL_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_WATERMARK 0x3483
+#define mmMCIF_WB2_MCIF_WB_WATERMARK_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_CLOCK_GATER_CONTROL 0x3484
+#define mmMCIF_WB2_MCIF_WB_CLOCK_GATER_CONTROL_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_WARM_UP_CNTL 0x3485
+#define mmMCIF_WB2_MCIF_WB_WARM_UP_CNTL_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_SELF_REFRESH_CONTROL 0x3486
+#define mmMCIF_WB2_MCIF_WB_SELF_REFRESH_CONTROL_BASE_IDX 2
+#define mmMCIF_WB2_MULTI_LEVEL_QOS_CTRL 0x3487
+#define mmMCIF_WB2_MULTI_LEVEL_QOS_CTRL_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_LUMA_SIZE 0x3489
+#define mmMCIF_WB2_MCIF_WB_BUF_LUMA_SIZE_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_CHROMA_SIZE 0x348a
+#define mmMCIF_WB2_MCIF_WB_BUF_CHROMA_SIZE_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_1_ADDR_Y_HIGH 0x348b
+#define mmMCIF_WB2_MCIF_WB_BUF_1_ADDR_Y_HIGH_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_1_ADDR_C_HIGH 0x348c
+#define mmMCIF_WB2_MCIF_WB_BUF_1_ADDR_C_HIGH_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_2_ADDR_Y_HIGH 0x348d
+#define mmMCIF_WB2_MCIF_WB_BUF_2_ADDR_Y_HIGH_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_2_ADDR_C_HIGH 0x348e
+#define mmMCIF_WB2_MCIF_WB_BUF_2_ADDR_C_HIGH_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_3_ADDR_Y_HIGH 0x348f
+#define mmMCIF_WB2_MCIF_WB_BUF_3_ADDR_Y_HIGH_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_3_ADDR_C_HIGH 0x3490
+#define mmMCIF_WB2_MCIF_WB_BUF_3_ADDR_C_HIGH_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_4_ADDR_Y_HIGH 0x3491
+#define mmMCIF_WB2_MCIF_WB_BUF_4_ADDR_Y_HIGH_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_4_ADDR_C_HIGH 0x3492
+#define mmMCIF_WB2_MCIF_WB_BUF_4_ADDR_C_HIGH_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_1_RESOLUTION 0x3493
+#define mmMCIF_WB2_MCIF_WB_BUF_1_RESOLUTION_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_2_RESOLUTION 0x3494
+#define mmMCIF_WB2_MCIF_WB_BUF_2_RESOLUTION_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_3_RESOLUTION 0x3495
+#define mmMCIF_WB2_MCIF_WB_BUF_3_RESOLUTION_BASE_IDX 2
+#define mmMCIF_WB2_MCIF_WB_BUF_4_RESOLUTION 0x3496
+#define mmMCIF_WB2_MCIF_WB_BUF_4_RESOLUTION_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mmhubbub_xfcp0_dispdec
+// base address: 0x0
+#define mmXFCP0_MMHUBBUB_XFC_CNTL 0x34a0
+#define mmXFCP0_MMHUBBUB_XFC_CNTL_BASE_IDX 2
+#define mmXFCP0_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_LSB 0x34a1
+#define mmXFCP0_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_LSB_BASE_IDX 2
+#define mmXFCP0_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_MSB 0x34a2
+#define mmXFCP0_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_MSB_BASE_IDX 2
+#define mmXFCP0_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_LSB 0x34a3
+#define mmXFCP0_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_LSB_BASE_IDX 2
+#define mmXFCP0_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_MSB 0x34a4
+#define mmXFCP0_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_MSB_BASE_IDX 2
+#define mmXFCP0_MMHUBBUB_XFC_XBUF_CONFIG 0x34a5
+#define mmXFCP0_MMHUBBUB_XFC_XBUF_CONFIG_BASE_IDX 2
+#define mmXFCP0_MMHUBBUB_XFC_XBUF_SIZE 0x34a6
+#define mmXFCP0_MMHUBBUB_XFC_XBUF_SIZE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mmhubbub_xfcp1_dispdec
+// base address: 0x80
+#define mmXFCP1_MMHUBBUB_XFC_CNTL 0x34c0
+#define mmXFCP1_MMHUBBUB_XFC_CNTL_BASE_IDX 2
+#define mmXFCP1_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_LSB 0x34c1
+#define mmXFCP1_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_LSB_BASE_IDX 2
+#define mmXFCP1_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_MSB 0x34c2
+#define mmXFCP1_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_MSB_BASE_IDX 2
+#define mmXFCP1_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_LSB 0x34c3
+#define mmXFCP1_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_LSB_BASE_IDX 2
+#define mmXFCP1_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_MSB 0x34c4
+#define mmXFCP1_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_MSB_BASE_IDX 2
+#define mmXFCP1_MMHUBBUB_XFC_XBUF_CONFIG 0x34c5
+#define mmXFCP1_MMHUBBUB_XFC_XBUF_CONFIG_BASE_IDX 2
+#define mmXFCP1_MMHUBBUB_XFC_XBUF_SIZE 0x34c6
+#define mmXFCP1_MMHUBBUB_XFC_XBUF_SIZE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mmhubbub_xfcp2_dispdec
+// base address: 0x100
+#define mmXFCP2_MMHUBBUB_XFC_CNTL 0x34e0
+#define mmXFCP2_MMHUBBUB_XFC_CNTL_BASE_IDX 2
+#define mmXFCP2_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_LSB 0x34e1
+#define mmXFCP2_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_LSB_BASE_IDX 2
+#define mmXFCP2_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_MSB 0x34e2
+#define mmXFCP2_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_MSB_BASE_IDX 2
+#define mmXFCP2_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_LSB 0x34e3
+#define mmXFCP2_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_LSB_BASE_IDX 2
+#define mmXFCP2_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_MSB 0x34e4
+#define mmXFCP2_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_MSB_BASE_IDX 2
+#define mmXFCP2_MMHUBBUB_XFC_XBUF_CONFIG 0x34e5
+#define mmXFCP2_MMHUBBUB_XFC_XBUF_CONFIG_BASE_IDX 2
+#define mmXFCP2_MMHUBBUB_XFC_XBUF_SIZE 0x34e6
+#define mmXFCP2_MMHUBBUB_XFC_XBUF_SIZE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mmhubbub_xfcp3_dispdec
+// base address: 0x180
+#define mmXFCP3_MMHUBBUB_XFC_CNTL 0x3500
+#define mmXFCP3_MMHUBBUB_XFC_CNTL_BASE_IDX 2
+#define mmXFCP3_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_LSB 0x3501
+#define mmXFCP3_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_LSB_BASE_IDX 2
+#define mmXFCP3_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_MSB 0x3502
+#define mmXFCP3_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_MSB_BASE_IDX 2
+#define mmXFCP3_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_LSB 0x3503
+#define mmXFCP3_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_LSB_BASE_IDX 2
+#define mmXFCP3_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_MSB 0x3504
+#define mmXFCP3_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_MSB_BASE_IDX 2
+#define mmXFCP3_MMHUBBUB_XFC_XBUF_CONFIG 0x3505
+#define mmXFCP3_MMHUBBUB_XFC_XBUF_CONFIG_BASE_IDX 2
+#define mmXFCP3_MMHUBBUB_XFC_XBUF_SIZE 0x3506
+#define mmXFCP3_MMHUBBUB_XFC_XBUF_SIZE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mmhubbub_xfcp4_dispdec
+// base address: 0x200
+#define mmXFCP4_MMHUBBUB_XFC_CNTL 0x3520
+#define mmXFCP4_MMHUBBUB_XFC_CNTL_BASE_IDX 2
+#define mmXFCP4_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_LSB 0x3521
+#define mmXFCP4_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_LSB_BASE_IDX 2
+#define mmXFCP4_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_MSB 0x3522
+#define mmXFCP4_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_MSB_BASE_IDX 2
+#define mmXFCP4_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_LSB 0x3523
+#define mmXFCP4_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_LSB_BASE_IDX 2
+#define mmXFCP4_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_MSB 0x3524
+#define mmXFCP4_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_MSB_BASE_IDX 2
+#define mmXFCP4_MMHUBBUB_XFC_XBUF_CONFIG 0x3525
+#define mmXFCP4_MMHUBBUB_XFC_XBUF_CONFIG_BASE_IDX 2
+#define mmXFCP4_MMHUBBUB_XFC_XBUF_SIZE 0x3526
+#define mmXFCP4_MMHUBBUB_XFC_XBUF_SIZE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mmhubbub_xfcp5_dispdec
+// base address: 0x280
+#define mmXFCP5_MMHUBBUB_XFC_CNTL 0x3540
+#define mmXFCP5_MMHUBBUB_XFC_CNTL_BASE_IDX 2
+#define mmXFCP5_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_LSB 0x3541
+#define mmXFCP5_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_LSB_BASE_IDX 2
+#define mmXFCP5_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_MSB 0x3542
+#define mmXFCP5_MMHUBBUB_XFC_XBUF_WR_BASE0_ADDR_MSB_BASE_IDX 2
+#define mmXFCP5_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_LSB 0x3543
+#define mmXFCP5_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_LSB_BASE_IDX 2
+#define mmXFCP5_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_MSB 0x3544
+#define mmXFCP5_MMHUBBUB_XFC_XBUF_WR_BASE1_ADDR_MSB_BASE_IDX 2
+#define mmXFCP5_MMHUBBUB_XFC_XBUF_CONFIG 0x3545
+#define mmXFCP5_MMHUBBUB_XFC_XBUF_CONFIG_BASE_IDX 2
+#define mmXFCP5_MMHUBBUB_XFC_XBUF_SIZE 0x3546
+#define mmXFCP5_MMHUBBUB_XFC_XBUF_SIZE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_mmhubbub_xfc_dispdec
+// base address: 0x0
+#define mmXFC_MEM_PWR_CNTL 0x35a0
+#define mmXFC_MEM_PWR_CNTL_BASE_IDX 2
+#define mmMMHUBBUB_XFC_XBUF_WR_SURF_CONFIG 0x35a1
+#define mmMMHUBBUB_XFC_XBUF_WR_SURF_CONFIG_BASE_IDX 2
+#define mmMMHUBBUB_XFC_XBUF_WR_CONFIG 0x35a2
+#define mmMMHUBBUB_XFC_XBUF_WR_CONFIG_BASE_IDX 2
+#define mmMMHUBBUB_XFC_IO_BACKPRESSURE_RELEASE_TIMER 0x35a3
+#define mmMMHUBBUB_XFC_IO_BACKPRESSURE_RELEASE_TIMER_BASE_IDX 2
+#define mmMMHUBBUB_XFC_GPU_CTRL 0x35a4
+#define mmMMHUBBUB_XFC_GPU_CTRL_BASE_IDX 2
+#define mmMMHUBBUB_XFC_XBUF_VM_CTRL 0x35a5
+#define mmMMHUBBUB_XFC_XBUF_VM_CTRL_BASE_IDX 2
+#define mmMMHUBBUB_XFC_XBUF_VM_INIT_BASE_ADDR_LSB 0x35a6
+#define mmMMHUBBUB_XFC_XBUF_VM_INIT_BASE_ADDR_LSB_BASE_IDX 2
+#define mmMMHUBBUB_XFC_XBUF_VM_INIT_BASE_ADDR_MSB 0x35a7
+#define mmMMHUBBUB_XFC_XBUF_VM_INIT_BASE_ADDR_MSB_BASE_IDX 2
+#define mmMMHUBBUB_XFC_XBUF_VM_INIT_PIXEL_VALUE_LSB 0x35a8
+#define mmMMHUBBUB_XFC_XBUF_VM_INIT_PIXEL_VALUE_LSB_BASE_IDX 2
+#define mmMMHUBBUB_XFC_XBUF_VM_INIT_PIXEL_VALUE_MSB 0x35a9
+#define mmMMHUBBUB_XFC_XBUF_VM_INIT_PIXEL_VALUE_MSB_BASE_IDX 2
+#define mmMMHUBBUB_XFC_GPU0_BASE_ADDR 0x35aa
+#define mmMMHUBBUB_XFC_GPU0_BASE_ADDR_BASE_IDX 2
+#define mmMMHUBBUB_XFC_GPU1_BASE_ADDR 0x35ab
+#define mmMMHUBBUB_XFC_GPU1_BASE_ADDR_BASE_IDX 2
+#define mmMMHUBBUB_XFC_GPU2_BASE_ADDR 0x35ac
+#define mmMMHUBBUB_XFC_GPU2_BASE_ADDR_BASE_IDX 2
+#define mmMMHUBBUB_XFC_GPU3_BASE_ADDR 0x35ad
+#define mmMMHUBBUB_XFC_GPU3_BASE_ADDR_BASE_IDX 2
+#define mmMMHUBBUB_XFCMON_CTRL 0x35ae
+#define mmMMHUBBUB_XFCMON_CTRL_BASE_IDX 2
+#define mmMMHUBBUB_XFCMON_TIMER 0x35af
+#define mmMMHUBBUB_XFCMON_TIMER_BASE_IDX 2
+#define mmMMHUBBUB_XFCMON_STAT_REQUESTS 0x35b0
+#define mmMMHUBBUB_XFCMON_STAT_REQUESTS_BASE_IDX 2
+#define mmMMHUBBUB_XFCMON_STAT_BACKPRESSURE 0x35b1
+#define mmMMHUBBUB_XFCMON_STAT_BACKPRESSURE_BASE_IDX 2
+#define mmMMHUBBUB_XFCMON_STAT_MAX_REQUESTS 0x35b2
+#define mmMMHUBBUB_XFCMON_STAT_MAX_REQUESTS_BASE_IDX 2
+#define mmMMHUBBUB_XFCMON_STAT_BACKPRESSURE_AT_MAX_REQUESTS 0x35b3
+#define mmMMHUBBUB_XFCMON_STAT_BACKPRESSURE_AT_MAX_REQUESTS_BASE_IDX 2
+#define mmMMHUBBUB_XFCMON_STAT_MAX_BACKPRESSURE 0x35b4
+#define mmMMHUBBUB_XFCMON_STAT_MAX_BACKPRESSURE_BASE_IDX 2
+#define mmMMHUBBUB_XFCMON_STAT_REQUESTS_AT_MAX_BACKPRESSURE 0x35b5
+#define mmMMHUBBUB_XFCMON_STAT_REQUESTS_AT_MAX_BACKPRESSURE_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp4_dispdec_dpp_top_dispdec
+// base address: 0xa42c
+#define mmDPP_TOP4_DPP_CONTROL 0x35d0
+#define mmDPP_TOP4_DPP_CONTROL_BASE_IDX 2
+#define mmDPP_TOP4_DPP_SOFT_RESET 0x35d1
+#define mmDPP_TOP4_DPP_SOFT_RESET_BASE_IDX 2
+#define mmDPP_TOP4_DPP_CRC_VAL_R_G 0x35d2
+#define mmDPP_TOP4_DPP_CRC_VAL_R_G_BASE_IDX 2
+#define mmDPP_TOP4_DPP_CRC_VAL_B_A 0x35d3
+#define mmDPP_TOP4_DPP_CRC_VAL_B_A_BASE_IDX 2
+#define mmDPP_TOP4_DPP_CRC_CTRL 0x35d4
+#define mmDPP_TOP4_DPP_CRC_CTRL_BASE_IDX 2
+#define mmDPP_TOP4_HOST_READ_CONTROL 0x35d5
+#define mmDPP_TOP4_HOST_READ_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp4_dispdec_cnvc_cfg_dispdec
+// base address: 0xa42c
+#define mmCNVC_CFG4_CNVC_SURFACE_PIXEL_FORMAT 0x35da
+#define mmCNVC_CFG4_CNVC_SURFACE_PIXEL_FORMAT_BASE_IDX 2
+#define mmCNVC_CFG4_FORMAT_CONTROL 0x35db
+#define mmCNVC_CFG4_FORMAT_CONTROL_BASE_IDX 2
+#define mmCNVC_CFG4_FCNV_FP_BIAS_R 0x35dc
+#define mmCNVC_CFG4_FCNV_FP_BIAS_R_BASE_IDX 2
+#define mmCNVC_CFG4_FCNV_FP_BIAS_G 0x35dd
+#define mmCNVC_CFG4_FCNV_FP_BIAS_G_BASE_IDX 2
+#define mmCNVC_CFG4_FCNV_FP_BIAS_B 0x35de
+#define mmCNVC_CFG4_FCNV_FP_BIAS_B_BASE_IDX 2
+#define mmCNVC_CFG4_FCNV_FP_SCALE_R 0x35df
+#define mmCNVC_CFG4_FCNV_FP_SCALE_R_BASE_IDX 2
+#define mmCNVC_CFG4_FCNV_FP_SCALE_G 0x35e0
+#define mmCNVC_CFG4_FCNV_FP_SCALE_G_BASE_IDX 2
+#define mmCNVC_CFG4_FCNV_FP_SCALE_B 0x35e1
+#define mmCNVC_CFG4_FCNV_FP_SCALE_B_BASE_IDX 2
+#define mmCNVC_CFG4_COLOR_KEYER_CONTROL 0x35e2
+#define mmCNVC_CFG4_COLOR_KEYER_CONTROL_BASE_IDX 2
+#define mmCNVC_CFG4_COLOR_KEYER_ALPHA 0x35e3
+#define mmCNVC_CFG4_COLOR_KEYER_ALPHA_BASE_IDX 2
+#define mmCNVC_CFG4_COLOR_KEYER_RED 0x35e4
+#define mmCNVC_CFG4_COLOR_KEYER_RED_BASE_IDX 2
+#define mmCNVC_CFG4_COLOR_KEYER_GREEN 0x35e5
+#define mmCNVC_CFG4_COLOR_KEYER_GREEN_BASE_IDX 2
+#define mmCNVC_CFG4_COLOR_KEYER_BLUE 0x35e6
+#define mmCNVC_CFG4_COLOR_KEYER_BLUE_BASE_IDX 2
+#define mmCNVC_CFG4_ALPHA_2BIT_LUT 0x35e8
+#define mmCNVC_CFG4_ALPHA_2BIT_LUT_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp4_dispdec_cnvc_cur_dispdec
+// base address: 0xa42c
+#define mmCNVC_CUR4_CURSOR0_CONTROL 0x35eb
+#define mmCNVC_CUR4_CURSOR0_CONTROL_BASE_IDX 2
+#define mmCNVC_CUR4_CURSOR0_COLOR0 0x35ec
+#define mmCNVC_CUR4_CURSOR0_COLOR0_BASE_IDX 2
+#define mmCNVC_CUR4_CURSOR0_COLOR1 0x35ed
+#define mmCNVC_CUR4_CURSOR0_COLOR1_BASE_IDX 2
+#define mmCNVC_CUR4_CURSOR0_FP_SCALE_BIAS 0x35ee
+#define mmCNVC_CUR4_CURSOR0_FP_SCALE_BIAS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp4_dispdec_dscl_dispdec
+// base address: 0xa42c
+#define mmDSCL4_SCL_COEF_RAM_TAP_SELECT 0x35f5
+#define mmDSCL4_SCL_COEF_RAM_TAP_SELECT_BASE_IDX 2
+#define mmDSCL4_SCL_COEF_RAM_TAP_DATA 0x35f6
+#define mmDSCL4_SCL_COEF_RAM_TAP_DATA_BASE_IDX 2
+#define mmDSCL4_SCL_MODE 0x35f7
+#define mmDSCL4_SCL_MODE_BASE_IDX 2
+#define mmDSCL4_SCL_TAP_CONTROL 0x35f8
+#define mmDSCL4_SCL_TAP_CONTROL_BASE_IDX 2
+#define mmDSCL4_DSCL_CONTROL 0x35f9
+#define mmDSCL4_DSCL_CONTROL_BASE_IDX 2
+#define mmDSCL4_DSCL_2TAP_CONTROL 0x35fa
+#define mmDSCL4_DSCL_2TAP_CONTROL_BASE_IDX 2
+#define mmDSCL4_SCL_MANUAL_REPLICATE_CONTROL 0x35fb
+#define mmDSCL4_SCL_MANUAL_REPLICATE_CONTROL_BASE_IDX 2
+#define mmDSCL4_SCL_HORZ_FILTER_SCALE_RATIO 0x35fc
+#define mmDSCL4_SCL_HORZ_FILTER_SCALE_RATIO_BASE_IDX 2
+#define mmDSCL4_SCL_HORZ_FILTER_INIT 0x35fd
+#define mmDSCL4_SCL_HORZ_FILTER_INIT_BASE_IDX 2
+#define mmDSCL4_SCL_HORZ_FILTER_SCALE_RATIO_C 0x35fe
+#define mmDSCL4_SCL_HORZ_FILTER_SCALE_RATIO_C_BASE_IDX 2
+#define mmDSCL4_SCL_HORZ_FILTER_INIT_C 0x35ff
+#define mmDSCL4_SCL_HORZ_FILTER_INIT_C_BASE_IDX 2
+#define mmDSCL4_SCL_VERT_FILTER_SCALE_RATIO 0x3600
+#define mmDSCL4_SCL_VERT_FILTER_SCALE_RATIO_BASE_IDX 2
+#define mmDSCL4_SCL_VERT_FILTER_INIT 0x3601
+#define mmDSCL4_SCL_VERT_FILTER_INIT_BASE_IDX 2
+#define mmDSCL4_SCL_VERT_FILTER_INIT_BOT 0x3602
+#define mmDSCL4_SCL_VERT_FILTER_INIT_BOT_BASE_IDX 2
+#define mmDSCL4_SCL_VERT_FILTER_SCALE_RATIO_C 0x3603
+#define mmDSCL4_SCL_VERT_FILTER_SCALE_RATIO_C_BASE_IDX 2
+#define mmDSCL4_SCL_VERT_FILTER_INIT_C 0x3604
+#define mmDSCL4_SCL_VERT_FILTER_INIT_C_BASE_IDX 2
+#define mmDSCL4_SCL_VERT_FILTER_INIT_BOT_C 0x3605
+#define mmDSCL4_SCL_VERT_FILTER_INIT_BOT_C_BASE_IDX 2
+#define mmDSCL4_SCL_BLACK_OFFSET 0x3606
+#define mmDSCL4_SCL_BLACK_OFFSET_BASE_IDX 2
+#define mmDSCL4_DSCL_UPDATE 0x3607
+#define mmDSCL4_DSCL_UPDATE_BASE_IDX 2
+#define mmDSCL4_DSCL_AUTOCAL 0x3608
+#define mmDSCL4_DSCL_AUTOCAL_BASE_IDX 2
+#define mmDSCL4_DSCL_EXT_OVERSCAN_LEFT_RIGHT 0x3609
+#define mmDSCL4_DSCL_EXT_OVERSCAN_LEFT_RIGHT_BASE_IDX 2
+#define mmDSCL4_DSCL_EXT_OVERSCAN_TOP_BOTTOM 0x360a
+#define mmDSCL4_DSCL_EXT_OVERSCAN_TOP_BOTTOM_BASE_IDX 2
+#define mmDSCL4_OTG_H_BLANK 0x360b
+#define mmDSCL4_OTG_H_BLANK_BASE_IDX 2
+#define mmDSCL4_OTG_V_BLANK 0x360c
+#define mmDSCL4_OTG_V_BLANK_BASE_IDX 2
+#define mmDSCL4_RECOUT_START 0x360d
+#define mmDSCL4_RECOUT_START_BASE_IDX 2
+#define mmDSCL4_RECOUT_SIZE 0x360e
+#define mmDSCL4_RECOUT_SIZE_BASE_IDX 2
+#define mmDSCL4_MPC_SIZE 0x360f
+#define mmDSCL4_MPC_SIZE_BASE_IDX 2
+#define mmDSCL4_LB_DATA_FORMAT 0x3610
+#define mmDSCL4_LB_DATA_FORMAT_BASE_IDX 2
+#define mmDSCL4_LB_MEMORY_CTRL 0x3611
+#define mmDSCL4_LB_MEMORY_CTRL_BASE_IDX 2
+#define mmDSCL4_LB_V_COUNTER 0x3612
+#define mmDSCL4_LB_V_COUNTER_BASE_IDX 2
+#define mmDSCL4_DSCL_MEM_PWR_CTRL 0x3613
+#define mmDSCL4_DSCL_MEM_PWR_CTRL_BASE_IDX 2
+#define mmDSCL4_DSCL_MEM_PWR_STATUS 0x3614
+#define mmDSCL4_DSCL_MEM_PWR_STATUS_BASE_IDX 2
+#define mmDSCL4_OBUF_CONTROL 0x3615
+#define mmDSCL4_OBUF_CONTROL_BASE_IDX 2
+#define mmDSCL4_OBUF_MEM_PWR_CTRL 0x3616
+#define mmDSCL4_OBUF_MEM_PWR_CTRL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp4_dispdec_cm_dispdec
+// base address: 0xa42c
+#define mmCM4_CM_CONTROL 0x3625
+#define mmCM4_CM_CONTROL_BASE_IDX 2
+#define mmCM4_CM_ICSC_CONTROL 0x3626
+#define mmCM4_CM_ICSC_CONTROL_BASE_IDX 2
+#define mmCM4_CM_ICSC_C11_C12 0x3627
+#define mmCM4_CM_ICSC_C11_C12_BASE_IDX 2
+#define mmCM4_CM_ICSC_C13_C14 0x3628
+#define mmCM4_CM_ICSC_C13_C14_BASE_IDX 2
+#define mmCM4_CM_ICSC_C21_C22 0x3629
+#define mmCM4_CM_ICSC_C21_C22_BASE_IDX 2
+#define mmCM4_CM_ICSC_C23_C24 0x362a
+#define mmCM4_CM_ICSC_C23_C24_BASE_IDX 2
+#define mmCM4_CM_ICSC_C31_C32 0x362b
+#define mmCM4_CM_ICSC_C31_C32_BASE_IDX 2
+#define mmCM4_CM_ICSC_C33_C34 0x362c
+#define mmCM4_CM_ICSC_C33_C34_BASE_IDX 2
+#define mmCM4_CM_ICSC_B_C11_C12 0x362d
+#define mmCM4_CM_ICSC_B_C11_C12_BASE_IDX 2
+#define mmCM4_CM_ICSC_B_C13_C14 0x362e
+#define mmCM4_CM_ICSC_B_C13_C14_BASE_IDX 2
+#define mmCM4_CM_ICSC_B_C21_C22 0x362f
+#define mmCM4_CM_ICSC_B_C21_C22_BASE_IDX 2
+#define mmCM4_CM_ICSC_B_C23_C24 0x3630
+#define mmCM4_CM_ICSC_B_C23_C24_BASE_IDX 2
+#define mmCM4_CM_ICSC_B_C31_C32 0x3631
+#define mmCM4_CM_ICSC_B_C31_C32_BASE_IDX 2
+#define mmCM4_CM_ICSC_B_C33_C34 0x3632
+#define mmCM4_CM_ICSC_B_C33_C34_BASE_IDX 2
+#define mmCM4_CM_GAMUT_REMAP_CONTROL 0x3633
+#define mmCM4_CM_GAMUT_REMAP_CONTROL_BASE_IDX 2
+#define mmCM4_CM_GAMUT_REMAP_C11_C12 0x3634
+#define mmCM4_CM_GAMUT_REMAP_C11_C12_BASE_IDX 2
+#define mmCM4_CM_GAMUT_REMAP_C13_C14 0x3635
+#define mmCM4_CM_GAMUT_REMAP_C13_C14_BASE_IDX 2
+#define mmCM4_CM_GAMUT_REMAP_C21_C22 0x3636
+#define mmCM4_CM_GAMUT_REMAP_C21_C22_BASE_IDX 2
+#define mmCM4_CM_GAMUT_REMAP_C23_C24 0x3637
+#define mmCM4_CM_GAMUT_REMAP_C23_C24_BASE_IDX 2
+#define mmCM4_CM_GAMUT_REMAP_C31_C32 0x3638
+#define mmCM4_CM_GAMUT_REMAP_C31_C32_BASE_IDX 2
+#define mmCM4_CM_GAMUT_REMAP_C33_C34 0x3639
+#define mmCM4_CM_GAMUT_REMAP_C33_C34_BASE_IDX 2
+#define mmCM4_CM_GAMUT_REMAP_B_C11_C12 0x363a
+#define mmCM4_CM_GAMUT_REMAP_B_C11_C12_BASE_IDX 2
+#define mmCM4_CM_GAMUT_REMAP_B_C13_C14 0x363b
+#define mmCM4_CM_GAMUT_REMAP_B_C13_C14_BASE_IDX 2
+#define mmCM4_CM_GAMUT_REMAP_B_C21_C22 0x363c
+#define mmCM4_CM_GAMUT_REMAP_B_C21_C22_BASE_IDX 2
+#define mmCM4_CM_GAMUT_REMAP_B_C23_C24 0x363d
+#define mmCM4_CM_GAMUT_REMAP_B_C23_C24_BASE_IDX 2
+#define mmCM4_CM_GAMUT_REMAP_B_C31_C32 0x363e
+#define mmCM4_CM_GAMUT_REMAP_B_C31_C32_BASE_IDX 2
+#define mmCM4_CM_GAMUT_REMAP_B_C33_C34 0x363f
+#define mmCM4_CM_GAMUT_REMAP_B_C33_C34_BASE_IDX 2
+#define mmCM4_CM_BIAS_CR_R 0x3640
+#define mmCM4_CM_BIAS_CR_R_BASE_IDX 2
+#define mmCM4_CM_BIAS_Y_G_CB_B 0x3641
+#define mmCM4_CM_BIAS_Y_G_CB_B_BASE_IDX 2
+#define mmCM4_CM_DGAM_CONTROL 0x3642
+#define mmCM4_CM_DGAM_CONTROL_BASE_IDX 2
+#define mmCM4_CM_DGAM_LUT_INDEX 0x3643
+#define mmCM4_CM_DGAM_LUT_INDEX_BASE_IDX 2
+#define mmCM4_CM_DGAM_LUT_DATA 0x3644
+#define mmCM4_CM_DGAM_LUT_DATA_BASE_IDX 2
+#define mmCM4_CM_DGAM_LUT_WRITE_EN_MASK 0x3645
+#define mmCM4_CM_DGAM_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_START_CNTL_B 0x3646
+#define mmCM4_CM_DGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_START_CNTL_G 0x3647
+#define mmCM4_CM_DGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_START_CNTL_R 0x3648
+#define mmCM4_CM_DGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_SLOPE_CNTL_B 0x3649
+#define mmCM4_CM_DGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_SLOPE_CNTL_G 0x364a
+#define mmCM4_CM_DGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_SLOPE_CNTL_R 0x364b
+#define mmCM4_CM_DGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_END_CNTL1_B 0x364c
+#define mmCM4_CM_DGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_END_CNTL2_B 0x364d
+#define mmCM4_CM_DGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_END_CNTL1_G 0x364e
+#define mmCM4_CM_DGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_END_CNTL2_G 0x364f
+#define mmCM4_CM_DGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_END_CNTL1_R 0x3650
+#define mmCM4_CM_DGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_END_CNTL2_R 0x3651
+#define mmCM4_CM_DGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_REGION_0_1 0x3652
+#define mmCM4_CM_DGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_REGION_2_3 0x3653
+#define mmCM4_CM_DGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_REGION_4_5 0x3654
+#define mmCM4_CM_DGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_REGION_6_7 0x3655
+#define mmCM4_CM_DGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_REGION_8_9 0x3656
+#define mmCM4_CM_DGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_REGION_10_11 0x3657
+#define mmCM4_CM_DGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_REGION_12_13 0x3658
+#define mmCM4_CM_DGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMA_REGION_14_15 0x3659
+#define mmCM4_CM_DGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_START_CNTL_B 0x365a
+#define mmCM4_CM_DGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_START_CNTL_G 0x365b
+#define mmCM4_CM_DGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_START_CNTL_R 0x365c
+#define mmCM4_CM_DGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_SLOPE_CNTL_B 0x365d
+#define mmCM4_CM_DGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_SLOPE_CNTL_G 0x365e
+#define mmCM4_CM_DGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_SLOPE_CNTL_R 0x365f
+#define mmCM4_CM_DGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_END_CNTL1_B 0x3660
+#define mmCM4_CM_DGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_END_CNTL2_B 0x3661
+#define mmCM4_CM_DGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_END_CNTL1_G 0x3662
+#define mmCM4_CM_DGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_END_CNTL2_G 0x3663
+#define mmCM4_CM_DGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_END_CNTL1_R 0x3664
+#define mmCM4_CM_DGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_END_CNTL2_R 0x3665
+#define mmCM4_CM_DGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_REGION_0_1 0x3666
+#define mmCM4_CM_DGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_REGION_2_3 0x3667
+#define mmCM4_CM_DGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_REGION_4_5 0x3668
+#define mmCM4_CM_DGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_REGION_6_7 0x3669
+#define mmCM4_CM_DGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_REGION_8_9 0x366a
+#define mmCM4_CM_DGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_REGION_10_11 0x366b
+#define mmCM4_CM_DGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_REGION_12_13 0x366c
+#define mmCM4_CM_DGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM4_CM_DGAM_RAMB_REGION_14_15 0x366d
+#define mmCM4_CM_DGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_CONTROL 0x366e
+#define mmCM4_CM_BLNDGAM_CONTROL_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_LUT_INDEX 0x366f
+#define mmCM4_CM_BLNDGAM_LUT_INDEX_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_LUT_DATA 0x3670
+#define mmCM4_CM_BLNDGAM_LUT_DATA_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_LUT_WRITE_EN_MASK 0x3671
+#define mmCM4_CM_BLNDGAM_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_START_CNTL_B 0x3672
+#define mmCM4_CM_BLNDGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_START_CNTL_G 0x3673
+#define mmCM4_CM_BLNDGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_START_CNTL_R 0x3674
+#define mmCM4_CM_BLNDGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_SLOPE_CNTL_B 0x3675
+#define mmCM4_CM_BLNDGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_SLOPE_CNTL_G 0x3676
+#define mmCM4_CM_BLNDGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_SLOPE_CNTL_R 0x3677
+#define mmCM4_CM_BLNDGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_END_CNTL1_B 0x3678
+#define mmCM4_CM_BLNDGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_END_CNTL2_B 0x3679
+#define mmCM4_CM_BLNDGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_END_CNTL1_G 0x367a
+#define mmCM4_CM_BLNDGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_END_CNTL2_G 0x367b
+#define mmCM4_CM_BLNDGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_END_CNTL1_R 0x367c
+#define mmCM4_CM_BLNDGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_END_CNTL2_R 0x367d
+#define mmCM4_CM_BLNDGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_0_1 0x367e
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_2_3 0x367f
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_4_5 0x3680
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_6_7 0x3681
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_8_9 0x3682
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_10_11 0x3683
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_12_13 0x3684
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_14_15 0x3685
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_16_17 0x3686
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_16_17_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_18_19 0x3687
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_18_19_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_20_21 0x3688
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_20_21_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_22_23 0x3689
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_22_23_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_24_25 0x368a
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_24_25_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_26_27 0x368b
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_26_27_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_28_29 0x368c
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_28_29_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_30_31 0x368d
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_30_31_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_32_33 0x368e
+#define mmCM4_CM_BLNDGAM_RAMA_REGION_32_33_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_START_CNTL_B 0x368f
+#define mmCM4_CM_BLNDGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_START_CNTL_G 0x3690
+#define mmCM4_CM_BLNDGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_START_CNTL_R 0x3691
+#define mmCM4_CM_BLNDGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_SLOPE_CNTL_B 0x3692
+#define mmCM4_CM_BLNDGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_SLOPE_CNTL_G 0x3693
+#define mmCM4_CM_BLNDGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_SLOPE_CNTL_R 0x3694
+#define mmCM4_CM_BLNDGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_END_CNTL1_B 0x3695
+#define mmCM4_CM_BLNDGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_END_CNTL2_B 0x3696
+#define mmCM4_CM_BLNDGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_END_CNTL1_G 0x3697
+#define mmCM4_CM_BLNDGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_END_CNTL2_G 0x3698
+#define mmCM4_CM_BLNDGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_END_CNTL1_R 0x3699
+#define mmCM4_CM_BLNDGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_END_CNTL2_R 0x369a
+#define mmCM4_CM_BLNDGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_0_1 0x369b
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_2_3 0x369c
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_4_5 0x369d
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_6_7 0x369e
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_8_9 0x369f
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_10_11 0x36a0
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_12_13 0x36a1
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_14_15 0x36a2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_16_17 0x36a3
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_16_17_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_18_19 0x36a4
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_18_19_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_20_21 0x36a5
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_20_21_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_22_23 0x36a6
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_22_23_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_24_25 0x36a7
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_24_25_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_26_27 0x36a8
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_26_27_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_28_29 0x36a9
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_28_29_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_30_31 0x36aa
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_30_31_BASE_IDX 2
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_32_33 0x36ab
+#define mmCM4_CM_BLNDGAM_RAMB_REGION_32_33_BASE_IDX 2
+#define mmCM4_CM_HDR_MULT_COEF 0x36ac
+#define mmCM4_CM_HDR_MULT_COEF_BASE_IDX 2
+#define mmCM4_CM_MEM_PWR_CTRL 0x36ad
+#define mmCM4_CM_MEM_PWR_CTRL_BASE_IDX 2
+#define mmCM4_CM_MEM_PWR_STATUS 0x36ae
+#define mmCM4_CM_MEM_PWR_STATUS_BASE_IDX 2
+#define mmCM4_CM_DEALPHA 0x36b0
+#define mmCM4_CM_DEALPHA_BASE_IDX 2
+#define mmCM4_CM_COEF_FORMAT 0x36b1
+#define mmCM4_CM_COEF_FORMAT_BASE_IDX 2
+#define mmCM4_CM_SHAPER_CONTROL 0x36b2
+#define mmCM4_CM_SHAPER_CONTROL_BASE_IDX 2
+#define mmCM4_CM_SHAPER_OFFSET_R 0x36b3
+#define mmCM4_CM_SHAPER_OFFSET_R_BASE_IDX 2
+#define mmCM4_CM_SHAPER_OFFSET_G 0x36b4
+#define mmCM4_CM_SHAPER_OFFSET_G_BASE_IDX 2
+#define mmCM4_CM_SHAPER_OFFSET_B 0x36b5
+#define mmCM4_CM_SHAPER_OFFSET_B_BASE_IDX 2
+#define mmCM4_CM_SHAPER_SCALE_R 0x36b6
+#define mmCM4_CM_SHAPER_SCALE_R_BASE_IDX 2
+#define mmCM4_CM_SHAPER_SCALE_G_B 0x36b7
+#define mmCM4_CM_SHAPER_SCALE_G_B_BASE_IDX 2
+#define mmCM4_CM_SHAPER_LUT_INDEX 0x36b8
+#define mmCM4_CM_SHAPER_LUT_INDEX_BASE_IDX 2
+#define mmCM4_CM_SHAPER_LUT_DATA 0x36b9
+#define mmCM4_CM_SHAPER_LUT_DATA_BASE_IDX 2
+#define mmCM4_CM_SHAPER_LUT_WRITE_EN_MASK 0x36ba
+#define mmCM4_CM_SHAPER_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_START_CNTL_B 0x36bb
+#define mmCM4_CM_SHAPER_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_START_CNTL_G 0x36bc
+#define mmCM4_CM_SHAPER_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_START_CNTL_R 0x36bd
+#define mmCM4_CM_SHAPER_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_END_CNTL_B 0x36be
+#define mmCM4_CM_SHAPER_RAMA_END_CNTL_B_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_END_CNTL_G 0x36bf
+#define mmCM4_CM_SHAPER_RAMA_END_CNTL_G_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_END_CNTL_R 0x36c0
+#define mmCM4_CM_SHAPER_RAMA_END_CNTL_R_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_REGION_0_1 0x36c1
+#define mmCM4_CM_SHAPER_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_REGION_2_3 0x36c2
+#define mmCM4_CM_SHAPER_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_REGION_4_5 0x36c3
+#define mmCM4_CM_SHAPER_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_REGION_6_7 0x36c4
+#define mmCM4_CM_SHAPER_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_REGION_8_9 0x36c5
+#define mmCM4_CM_SHAPER_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_REGION_10_11 0x36c6
+#define mmCM4_CM_SHAPER_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_REGION_12_13 0x36c7
+#define mmCM4_CM_SHAPER_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_REGION_14_15 0x36c8
+#define mmCM4_CM_SHAPER_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_REGION_16_17 0x36c9
+#define mmCM4_CM_SHAPER_RAMA_REGION_16_17_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_REGION_18_19 0x36ca
+#define mmCM4_CM_SHAPER_RAMA_REGION_18_19_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_REGION_20_21 0x36cb
+#define mmCM4_CM_SHAPER_RAMA_REGION_20_21_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_REGION_22_23 0x36cc
+#define mmCM4_CM_SHAPER_RAMA_REGION_22_23_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_REGION_24_25 0x36cd
+#define mmCM4_CM_SHAPER_RAMA_REGION_24_25_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_REGION_26_27 0x36ce
+#define mmCM4_CM_SHAPER_RAMA_REGION_26_27_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_REGION_28_29 0x36cf
+#define mmCM4_CM_SHAPER_RAMA_REGION_28_29_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_REGION_30_31 0x36d0
+#define mmCM4_CM_SHAPER_RAMA_REGION_30_31_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMA_REGION_32_33 0x36d1
+#define mmCM4_CM_SHAPER_RAMA_REGION_32_33_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_START_CNTL_B 0x36d2
+#define mmCM4_CM_SHAPER_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_START_CNTL_G 0x36d3
+#define mmCM4_CM_SHAPER_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_START_CNTL_R 0x36d4
+#define mmCM4_CM_SHAPER_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_END_CNTL_B 0x36d5
+#define mmCM4_CM_SHAPER_RAMB_END_CNTL_B_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_END_CNTL_G 0x36d6
+#define mmCM4_CM_SHAPER_RAMB_END_CNTL_G_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_END_CNTL_R 0x36d7
+#define mmCM4_CM_SHAPER_RAMB_END_CNTL_R_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_REGION_0_1 0x36d8
+#define mmCM4_CM_SHAPER_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_REGION_2_3 0x36d9
+#define mmCM4_CM_SHAPER_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_REGION_4_5 0x36da
+#define mmCM4_CM_SHAPER_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_REGION_6_7 0x36db
+#define mmCM4_CM_SHAPER_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_REGION_8_9 0x36dc
+#define mmCM4_CM_SHAPER_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_REGION_10_11 0x36dd
+#define mmCM4_CM_SHAPER_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_REGION_12_13 0x36de
+#define mmCM4_CM_SHAPER_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_REGION_14_15 0x36df
+#define mmCM4_CM_SHAPER_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_REGION_16_17 0x36e0
+#define mmCM4_CM_SHAPER_RAMB_REGION_16_17_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_REGION_18_19 0x36e1
+#define mmCM4_CM_SHAPER_RAMB_REGION_18_19_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_REGION_20_21 0x36e2
+#define mmCM4_CM_SHAPER_RAMB_REGION_20_21_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_REGION_22_23 0x36e3
+#define mmCM4_CM_SHAPER_RAMB_REGION_22_23_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_REGION_24_25 0x36e4
+#define mmCM4_CM_SHAPER_RAMB_REGION_24_25_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_REGION_26_27 0x36e5
+#define mmCM4_CM_SHAPER_RAMB_REGION_26_27_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_REGION_28_29 0x36e6
+#define mmCM4_CM_SHAPER_RAMB_REGION_28_29_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_REGION_30_31 0x36e7
+#define mmCM4_CM_SHAPER_RAMB_REGION_30_31_BASE_IDX 2
+#define mmCM4_CM_SHAPER_RAMB_REGION_32_33 0x36e8
+#define mmCM4_CM_SHAPER_RAMB_REGION_32_33_BASE_IDX 2
+#define mmCM4_CM_MEM_PWR_CTRL2 0x36e9
+#define mmCM4_CM_MEM_PWR_CTRL2_BASE_IDX 2
+#define mmCM4_CM_MEM_PWR_STATUS2 0x36ea
+#define mmCM4_CM_MEM_PWR_STATUS2_BASE_IDX 2
+#define mmCM4_CM_3DLUT_MODE 0x36eb
+#define mmCM4_CM_3DLUT_MODE_BASE_IDX 2
+#define mmCM4_CM_3DLUT_INDEX 0x36ec
+#define mmCM4_CM_3DLUT_INDEX_BASE_IDX 2
+#define mmCM4_CM_3DLUT_DATA 0x36ed
+#define mmCM4_CM_3DLUT_DATA_BASE_IDX 2
+#define mmCM4_CM_3DLUT_DATA_30BIT 0x36ee
+#define mmCM4_CM_3DLUT_DATA_30BIT_BASE_IDX 2
+#define mmCM4_CM_3DLUT_READ_WRITE_CONTROL 0x36ef
+#define mmCM4_CM_3DLUT_READ_WRITE_CONTROL_BASE_IDX 2
+#define mmCM4_CM_3DLUT_OUT_NORM_FACTOR 0x36f0
+#define mmCM4_CM_3DLUT_OUT_NORM_FACTOR_BASE_IDX 2
+#define mmCM4_CM_3DLUT_OUT_OFFSET_R 0x36f1
+#define mmCM4_CM_3DLUT_OUT_OFFSET_R_BASE_IDX 2
+#define mmCM4_CM_3DLUT_OUT_OFFSET_G 0x36f2
+#define mmCM4_CM_3DLUT_OUT_OFFSET_G_BASE_IDX 2
+#define mmCM4_CM_3DLUT_OUT_OFFSET_B 0x36f3
+#define mmCM4_CM_3DLUT_OUT_OFFSET_B_BASE_IDX 2
+#define mmCM4_CM_TEST_DEBUG_INDEX 0x36f4
+#define mmCM4_CM_TEST_DEBUG_INDEX_BASE_IDX 2
+#define mmCM4_CM_TEST_DEBUG_DATA 0x36f5
+#define mmCM4_CM_TEST_DEBUG_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp4_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
+// base address: 0xdcbc
+#define mmDC_PERFMON27_PERFCOUNTER_CNTL 0x372f
+#define mmDC_PERFMON27_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON27_PERFCOUNTER_CNTL2 0x3730
+#define mmDC_PERFMON27_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON27_PERFCOUNTER_STATE 0x3731
+#define mmDC_PERFMON27_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON27_PERFMON_CNTL 0x3732
+#define mmDC_PERFMON27_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON27_PERFMON_CNTL2 0x3733
+#define mmDC_PERFMON27_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON27_PERFMON_CVALUE_INT_MISC 0x3734
+#define mmDC_PERFMON27_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON27_PERFMON_CVALUE_LOW 0x3735
+#define mmDC_PERFMON27_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON27_PERFMON_HI 0x3736
+#define mmDC_PERFMON27_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON27_PERFMON_LOW 0x3737
+#define mmDC_PERFMON27_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp5_dispdec_dpp_top_dispdec
+// base address: 0xa9d8
+#define mmDPP_TOP5_DPP_CONTROL 0x373b
+#define mmDPP_TOP5_DPP_CONTROL_BASE_IDX 2
+#define mmDPP_TOP5_DPP_SOFT_RESET 0x373c
+#define mmDPP_TOP5_DPP_SOFT_RESET_BASE_IDX 2
+#define mmDPP_TOP5_DPP_CRC_VAL_R_G 0x373d
+#define mmDPP_TOP5_DPP_CRC_VAL_R_G_BASE_IDX 2
+#define mmDPP_TOP5_DPP_CRC_VAL_B_A 0x373e
+#define mmDPP_TOP5_DPP_CRC_VAL_B_A_BASE_IDX 2
+#define mmDPP_TOP5_DPP_CRC_CTRL 0x373f
+#define mmDPP_TOP5_DPP_CRC_CTRL_BASE_IDX 2
+#define mmDPP_TOP5_HOST_READ_CONTROL 0x3740
+#define mmDPP_TOP5_HOST_READ_CONTROL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp5_dispdec_cnvc_cfg_dispdec
+// base address: 0xa9d8
+#define mmCNVC_CFG5_CNVC_SURFACE_PIXEL_FORMAT 0x3745
+#define mmCNVC_CFG5_CNVC_SURFACE_PIXEL_FORMAT_BASE_IDX 2
+#define mmCNVC_CFG5_FORMAT_CONTROL 0x3746
+#define mmCNVC_CFG5_FORMAT_CONTROL_BASE_IDX 2
+#define mmCNVC_CFG5_FCNV_FP_BIAS_R 0x3747
+#define mmCNVC_CFG5_FCNV_FP_BIAS_R_BASE_IDX 2
+#define mmCNVC_CFG5_FCNV_FP_BIAS_G 0x3748
+#define mmCNVC_CFG5_FCNV_FP_BIAS_G_BASE_IDX 2
+#define mmCNVC_CFG5_FCNV_FP_BIAS_B 0x3749
+#define mmCNVC_CFG5_FCNV_FP_BIAS_B_BASE_IDX 2
+#define mmCNVC_CFG5_FCNV_FP_SCALE_R 0x374a
+#define mmCNVC_CFG5_FCNV_FP_SCALE_R_BASE_IDX 2
+#define mmCNVC_CFG5_FCNV_FP_SCALE_G 0x374b
+#define mmCNVC_CFG5_FCNV_FP_SCALE_G_BASE_IDX 2
+#define mmCNVC_CFG5_FCNV_FP_SCALE_B 0x374c
+#define mmCNVC_CFG5_FCNV_FP_SCALE_B_BASE_IDX 2
+#define mmCNVC_CFG5_COLOR_KEYER_CONTROL 0x374d
+#define mmCNVC_CFG5_COLOR_KEYER_CONTROL_BASE_IDX 2
+#define mmCNVC_CFG5_COLOR_KEYER_ALPHA 0x374e
+#define mmCNVC_CFG5_COLOR_KEYER_ALPHA_BASE_IDX 2
+#define mmCNVC_CFG5_COLOR_KEYER_RED 0x374f
+#define mmCNVC_CFG5_COLOR_KEYER_RED_BASE_IDX 2
+#define mmCNVC_CFG5_COLOR_KEYER_GREEN 0x3750
+#define mmCNVC_CFG5_COLOR_KEYER_GREEN_BASE_IDX 2
+#define mmCNVC_CFG5_COLOR_KEYER_BLUE 0x3751
+#define mmCNVC_CFG5_COLOR_KEYER_BLUE_BASE_IDX 2
+#define mmCNVC_CFG5_ALPHA_2BIT_LUT 0x3753
+#define mmCNVC_CFG5_ALPHA_2BIT_LUT_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp5_dispdec_cnvc_cur_dispdec
+// base address: 0xa9d8
+#define mmCNVC_CUR5_CURSOR0_CONTROL 0x3756
+#define mmCNVC_CUR5_CURSOR0_CONTROL_BASE_IDX 2
+#define mmCNVC_CUR5_CURSOR0_COLOR0 0x3757
+#define mmCNVC_CUR5_CURSOR0_COLOR0_BASE_IDX 2
+#define mmCNVC_CUR5_CURSOR0_COLOR1 0x3758
+#define mmCNVC_CUR5_CURSOR0_COLOR1_BASE_IDX 2
+#define mmCNVC_CUR5_CURSOR0_FP_SCALE_BIAS 0x3759
+#define mmCNVC_CUR5_CURSOR0_FP_SCALE_BIAS_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp5_dispdec_dscl_dispdec
+// base address: 0xa9d8
+#define mmDSCL5_SCL_COEF_RAM_TAP_SELECT 0x3760
+#define mmDSCL5_SCL_COEF_RAM_TAP_SELECT_BASE_IDX 2
+#define mmDSCL5_SCL_COEF_RAM_TAP_DATA 0x3761
+#define mmDSCL5_SCL_COEF_RAM_TAP_DATA_BASE_IDX 2
+#define mmDSCL5_SCL_MODE 0x3762
+#define mmDSCL5_SCL_MODE_BASE_IDX 2
+#define mmDSCL5_SCL_TAP_CONTROL 0x3763
+#define mmDSCL5_SCL_TAP_CONTROL_BASE_IDX 2
+#define mmDSCL5_DSCL_CONTROL 0x3764
+#define mmDSCL5_DSCL_CONTROL_BASE_IDX 2
+#define mmDSCL5_DSCL_2TAP_CONTROL 0x3765
+#define mmDSCL5_DSCL_2TAP_CONTROL_BASE_IDX 2
+#define mmDSCL5_SCL_MANUAL_REPLICATE_CONTROL 0x3766
+#define mmDSCL5_SCL_MANUAL_REPLICATE_CONTROL_BASE_IDX 2
+#define mmDSCL5_SCL_HORZ_FILTER_SCALE_RATIO 0x3767
+#define mmDSCL5_SCL_HORZ_FILTER_SCALE_RATIO_BASE_IDX 2
+#define mmDSCL5_SCL_HORZ_FILTER_INIT 0x3768
+#define mmDSCL5_SCL_HORZ_FILTER_INIT_BASE_IDX 2
+#define mmDSCL5_SCL_HORZ_FILTER_SCALE_RATIO_C 0x3769
+#define mmDSCL5_SCL_HORZ_FILTER_SCALE_RATIO_C_BASE_IDX 2
+#define mmDSCL5_SCL_HORZ_FILTER_INIT_C 0x376a
+#define mmDSCL5_SCL_HORZ_FILTER_INIT_C_BASE_IDX 2
+#define mmDSCL5_SCL_VERT_FILTER_SCALE_RATIO 0x376b
+#define mmDSCL5_SCL_VERT_FILTER_SCALE_RATIO_BASE_IDX 2
+#define mmDSCL5_SCL_VERT_FILTER_INIT 0x376c
+#define mmDSCL5_SCL_VERT_FILTER_INIT_BASE_IDX 2
+#define mmDSCL5_SCL_VERT_FILTER_INIT_BOT 0x376d
+#define mmDSCL5_SCL_VERT_FILTER_INIT_BOT_BASE_IDX 2
+#define mmDSCL5_SCL_VERT_FILTER_SCALE_RATIO_C 0x376e
+#define mmDSCL5_SCL_VERT_FILTER_SCALE_RATIO_C_BASE_IDX 2
+#define mmDSCL5_SCL_VERT_FILTER_INIT_C 0x376f
+#define mmDSCL5_SCL_VERT_FILTER_INIT_C_BASE_IDX 2
+#define mmDSCL5_SCL_VERT_FILTER_INIT_BOT_C 0x3770
+#define mmDSCL5_SCL_VERT_FILTER_INIT_BOT_C_BASE_IDX 2
+#define mmDSCL5_SCL_BLACK_OFFSET 0x3771
+#define mmDSCL5_SCL_BLACK_OFFSET_BASE_IDX 2
+#define mmDSCL5_DSCL_UPDATE 0x3772
+#define mmDSCL5_DSCL_UPDATE_BASE_IDX 2
+#define mmDSCL5_DSCL_AUTOCAL 0x3773
+#define mmDSCL5_DSCL_AUTOCAL_BASE_IDX 2
+#define mmDSCL5_DSCL_EXT_OVERSCAN_LEFT_RIGHT 0x3774
+#define mmDSCL5_DSCL_EXT_OVERSCAN_LEFT_RIGHT_BASE_IDX 2
+#define mmDSCL5_DSCL_EXT_OVERSCAN_TOP_BOTTOM 0x3775
+#define mmDSCL5_DSCL_EXT_OVERSCAN_TOP_BOTTOM_BASE_IDX 2
+#define mmDSCL5_OTG_H_BLANK 0x3776
+#define mmDSCL5_OTG_H_BLANK_BASE_IDX 2
+#define mmDSCL5_OTG_V_BLANK 0x3777
+#define mmDSCL5_OTG_V_BLANK_BASE_IDX 2
+#define mmDSCL5_RECOUT_START 0x3778
+#define mmDSCL5_RECOUT_START_BASE_IDX 2
+#define mmDSCL5_RECOUT_SIZE 0x3779
+#define mmDSCL5_RECOUT_SIZE_BASE_IDX 2
+#define mmDSCL5_MPC_SIZE 0x377a
+#define mmDSCL5_MPC_SIZE_BASE_IDX 2
+#define mmDSCL5_LB_DATA_FORMAT 0x377b
+#define mmDSCL5_LB_DATA_FORMAT_BASE_IDX 2
+#define mmDSCL5_LB_MEMORY_CTRL 0x377c
+#define mmDSCL5_LB_MEMORY_CTRL_BASE_IDX 2
+#define mmDSCL5_LB_V_COUNTER 0x377d
+#define mmDSCL5_LB_V_COUNTER_BASE_IDX 2
+#define mmDSCL5_DSCL_MEM_PWR_CTRL 0x377e
+#define mmDSCL5_DSCL_MEM_PWR_CTRL_BASE_IDX 2
+#define mmDSCL5_DSCL_MEM_PWR_STATUS 0x377f
+#define mmDSCL5_DSCL_MEM_PWR_STATUS_BASE_IDX 2
+#define mmDSCL5_OBUF_CONTROL 0x3780
+#define mmDSCL5_OBUF_CONTROL_BASE_IDX 2
+#define mmDSCL5_OBUF_MEM_PWR_CTRL 0x3781
+#define mmDSCL5_OBUF_MEM_PWR_CTRL_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp5_dispdec_cm_dispdec
+// base address: 0xa9d8
+#define mmCM5_CM_CONTROL 0x3790
+#define mmCM5_CM_CONTROL_BASE_IDX 2
+#define mmCM5_CM_ICSC_CONTROL 0x3791
+#define mmCM5_CM_ICSC_CONTROL_BASE_IDX 2
+#define mmCM5_CM_ICSC_C11_C12 0x3792
+#define mmCM5_CM_ICSC_C11_C12_BASE_IDX 2
+#define mmCM5_CM_ICSC_C13_C14 0x3793
+#define mmCM5_CM_ICSC_C13_C14_BASE_IDX 2
+#define mmCM5_CM_ICSC_C21_C22 0x3794
+#define mmCM5_CM_ICSC_C21_C22_BASE_IDX 2
+#define mmCM5_CM_ICSC_C23_C24 0x3795
+#define mmCM5_CM_ICSC_C23_C24_BASE_IDX 2
+#define mmCM5_CM_ICSC_C31_C32 0x3796
+#define mmCM5_CM_ICSC_C31_C32_BASE_IDX 2
+#define mmCM5_CM_ICSC_C33_C34 0x3797
+#define mmCM5_CM_ICSC_C33_C34_BASE_IDX 2
+#define mmCM5_CM_ICSC_B_C11_C12 0x3798
+#define mmCM5_CM_ICSC_B_C11_C12_BASE_IDX 2
+#define mmCM5_CM_ICSC_B_C13_C14 0x3799
+#define mmCM5_CM_ICSC_B_C13_C14_BASE_IDX 2
+#define mmCM5_CM_ICSC_B_C21_C22 0x379a
+#define mmCM5_CM_ICSC_B_C21_C22_BASE_IDX 2
+#define mmCM5_CM_ICSC_B_C23_C24 0x379b
+#define mmCM5_CM_ICSC_B_C23_C24_BASE_IDX 2
+#define mmCM5_CM_ICSC_B_C31_C32 0x379c
+#define mmCM5_CM_ICSC_B_C31_C32_BASE_IDX 2
+#define mmCM5_CM_ICSC_B_C33_C34 0x379d
+#define mmCM5_CM_ICSC_B_C33_C34_BASE_IDX 2
+#define mmCM5_CM_GAMUT_REMAP_CONTROL 0x379e
+#define mmCM5_CM_GAMUT_REMAP_CONTROL_BASE_IDX 2
+#define mmCM5_CM_GAMUT_REMAP_C11_C12 0x379f
+#define mmCM5_CM_GAMUT_REMAP_C11_C12_BASE_IDX 2
+#define mmCM5_CM_GAMUT_REMAP_C13_C14 0x37a0
+#define mmCM5_CM_GAMUT_REMAP_C13_C14_BASE_IDX 2
+#define mmCM5_CM_GAMUT_REMAP_C21_C22 0x37a1
+#define mmCM5_CM_GAMUT_REMAP_C21_C22_BASE_IDX 2
+#define mmCM5_CM_GAMUT_REMAP_C23_C24 0x37a2
+#define mmCM5_CM_GAMUT_REMAP_C23_C24_BASE_IDX 2
+#define mmCM5_CM_GAMUT_REMAP_C31_C32 0x37a3
+#define mmCM5_CM_GAMUT_REMAP_C31_C32_BASE_IDX 2
+#define mmCM5_CM_GAMUT_REMAP_C33_C34 0x37a4
+#define mmCM5_CM_GAMUT_REMAP_C33_C34_BASE_IDX 2
+#define mmCM5_CM_GAMUT_REMAP_B_C11_C12 0x37a5
+#define mmCM5_CM_GAMUT_REMAP_B_C11_C12_BASE_IDX 2
+#define mmCM5_CM_GAMUT_REMAP_B_C13_C14 0x37a6
+#define mmCM5_CM_GAMUT_REMAP_B_C13_C14_BASE_IDX 2
+#define mmCM5_CM_GAMUT_REMAP_B_C21_C22 0x37a7
+#define mmCM5_CM_GAMUT_REMAP_B_C21_C22_BASE_IDX 2
+#define mmCM5_CM_GAMUT_REMAP_B_C23_C24 0x37a8
+#define mmCM5_CM_GAMUT_REMAP_B_C23_C24_BASE_IDX 2
+#define mmCM5_CM_GAMUT_REMAP_B_C31_C32 0x37a9
+#define mmCM5_CM_GAMUT_REMAP_B_C31_C32_BASE_IDX 2
+#define mmCM5_CM_GAMUT_REMAP_B_C33_C34 0x37aa
+#define mmCM5_CM_GAMUT_REMAP_B_C33_C34_BASE_IDX 2
+#define mmCM5_CM_BIAS_CR_R 0x37ab
+#define mmCM5_CM_BIAS_CR_R_BASE_IDX 2
+#define mmCM5_CM_BIAS_Y_G_CB_B 0x37ac
+#define mmCM5_CM_BIAS_Y_G_CB_B_BASE_IDX 2
+#define mmCM5_CM_DGAM_CONTROL 0x37ad
+#define mmCM5_CM_DGAM_CONTROL_BASE_IDX 2
+#define mmCM5_CM_DGAM_LUT_INDEX 0x37ae
+#define mmCM5_CM_DGAM_LUT_INDEX_BASE_IDX 2
+#define mmCM5_CM_DGAM_LUT_DATA 0x37af
+#define mmCM5_CM_DGAM_LUT_DATA_BASE_IDX 2
+#define mmCM5_CM_DGAM_LUT_WRITE_EN_MASK 0x37b0
+#define mmCM5_CM_DGAM_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_START_CNTL_B 0x37b1
+#define mmCM5_CM_DGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_START_CNTL_G 0x37b2
+#define mmCM5_CM_DGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_START_CNTL_R 0x37b3
+#define mmCM5_CM_DGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_SLOPE_CNTL_B 0x37b4
+#define mmCM5_CM_DGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_SLOPE_CNTL_G 0x37b5
+#define mmCM5_CM_DGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_SLOPE_CNTL_R 0x37b6
+#define mmCM5_CM_DGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_END_CNTL1_B 0x37b7
+#define mmCM5_CM_DGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_END_CNTL2_B 0x37b8
+#define mmCM5_CM_DGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_END_CNTL1_G 0x37b9
+#define mmCM5_CM_DGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_END_CNTL2_G 0x37ba
+#define mmCM5_CM_DGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_END_CNTL1_R 0x37bb
+#define mmCM5_CM_DGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_END_CNTL2_R 0x37bc
+#define mmCM5_CM_DGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_REGION_0_1 0x37bd
+#define mmCM5_CM_DGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_REGION_2_3 0x37be
+#define mmCM5_CM_DGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_REGION_4_5 0x37bf
+#define mmCM5_CM_DGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_REGION_6_7 0x37c0
+#define mmCM5_CM_DGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_REGION_8_9 0x37c1
+#define mmCM5_CM_DGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_REGION_10_11 0x37c2
+#define mmCM5_CM_DGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_REGION_12_13 0x37c3
+#define mmCM5_CM_DGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMA_REGION_14_15 0x37c4
+#define mmCM5_CM_DGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_START_CNTL_B 0x37c5
+#define mmCM5_CM_DGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_START_CNTL_G 0x37c6
+#define mmCM5_CM_DGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_START_CNTL_R 0x37c7
+#define mmCM5_CM_DGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_SLOPE_CNTL_B 0x37c8
+#define mmCM5_CM_DGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_SLOPE_CNTL_G 0x37c9
+#define mmCM5_CM_DGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_SLOPE_CNTL_R 0x37ca
+#define mmCM5_CM_DGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_END_CNTL1_B 0x37cb
+#define mmCM5_CM_DGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_END_CNTL2_B 0x37cc
+#define mmCM5_CM_DGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_END_CNTL1_G 0x37cd
+#define mmCM5_CM_DGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_END_CNTL2_G 0x37ce
+#define mmCM5_CM_DGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_END_CNTL1_R 0x37cf
+#define mmCM5_CM_DGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_END_CNTL2_R 0x37d0
+#define mmCM5_CM_DGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_REGION_0_1 0x37d1
+#define mmCM5_CM_DGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_REGION_2_3 0x37d2
+#define mmCM5_CM_DGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_REGION_4_5 0x37d3
+#define mmCM5_CM_DGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_REGION_6_7 0x37d4
+#define mmCM5_CM_DGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_REGION_8_9 0x37d5
+#define mmCM5_CM_DGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_REGION_10_11 0x37d6
+#define mmCM5_CM_DGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_REGION_12_13 0x37d7
+#define mmCM5_CM_DGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM5_CM_DGAM_RAMB_REGION_14_15 0x37d8
+#define mmCM5_CM_DGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_CONTROL 0x37d9
+#define mmCM5_CM_BLNDGAM_CONTROL_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_LUT_INDEX 0x37da
+#define mmCM5_CM_BLNDGAM_LUT_INDEX_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_LUT_DATA 0x37db
+#define mmCM5_CM_BLNDGAM_LUT_DATA_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_LUT_WRITE_EN_MASK 0x37dc
+#define mmCM5_CM_BLNDGAM_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_START_CNTL_B 0x37dd
+#define mmCM5_CM_BLNDGAM_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_START_CNTL_G 0x37de
+#define mmCM5_CM_BLNDGAM_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_START_CNTL_R 0x37df
+#define mmCM5_CM_BLNDGAM_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_SLOPE_CNTL_B 0x37e0
+#define mmCM5_CM_BLNDGAM_RAMA_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_SLOPE_CNTL_G 0x37e1
+#define mmCM5_CM_BLNDGAM_RAMA_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_SLOPE_CNTL_R 0x37e2
+#define mmCM5_CM_BLNDGAM_RAMA_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_END_CNTL1_B 0x37e3
+#define mmCM5_CM_BLNDGAM_RAMA_END_CNTL1_B_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_END_CNTL2_B 0x37e4
+#define mmCM5_CM_BLNDGAM_RAMA_END_CNTL2_B_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_END_CNTL1_G 0x37e5
+#define mmCM5_CM_BLNDGAM_RAMA_END_CNTL1_G_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_END_CNTL2_G 0x37e6
+#define mmCM5_CM_BLNDGAM_RAMA_END_CNTL2_G_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_END_CNTL1_R 0x37e7
+#define mmCM5_CM_BLNDGAM_RAMA_END_CNTL1_R_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_END_CNTL2_R 0x37e8
+#define mmCM5_CM_BLNDGAM_RAMA_END_CNTL2_R_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_0_1 0x37e9
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_2_3 0x37ea
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_4_5 0x37eb
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_6_7 0x37ec
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_8_9 0x37ed
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_10_11 0x37ee
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_12_13 0x37ef
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_14_15 0x37f0
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_16_17 0x37f1
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_16_17_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_18_19 0x37f2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_18_19_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_20_21 0x37f3
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_20_21_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_22_23 0x37f4
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_22_23_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_24_25 0x37f5
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_24_25_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_26_27 0x37f6
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_26_27_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_28_29 0x37f7
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_28_29_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_30_31 0x37f8
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_30_31_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_32_33 0x37f9
+#define mmCM5_CM_BLNDGAM_RAMA_REGION_32_33_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_START_CNTL_B 0x37fa
+#define mmCM5_CM_BLNDGAM_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_START_CNTL_G 0x37fb
+#define mmCM5_CM_BLNDGAM_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_START_CNTL_R 0x37fc
+#define mmCM5_CM_BLNDGAM_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_SLOPE_CNTL_B 0x37fd
+#define mmCM5_CM_BLNDGAM_RAMB_SLOPE_CNTL_B_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_SLOPE_CNTL_G 0x37fe
+#define mmCM5_CM_BLNDGAM_RAMB_SLOPE_CNTL_G_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_SLOPE_CNTL_R 0x37ff
+#define mmCM5_CM_BLNDGAM_RAMB_SLOPE_CNTL_R_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_END_CNTL1_B 0x3800
+#define mmCM5_CM_BLNDGAM_RAMB_END_CNTL1_B_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_END_CNTL2_B 0x3801
+#define mmCM5_CM_BLNDGAM_RAMB_END_CNTL2_B_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_END_CNTL1_G 0x3802
+#define mmCM5_CM_BLNDGAM_RAMB_END_CNTL1_G_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_END_CNTL2_G 0x3803
+#define mmCM5_CM_BLNDGAM_RAMB_END_CNTL2_G_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_END_CNTL1_R 0x3804
+#define mmCM5_CM_BLNDGAM_RAMB_END_CNTL1_R_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_END_CNTL2_R 0x3805
+#define mmCM5_CM_BLNDGAM_RAMB_END_CNTL2_R_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_0_1 0x3806
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_2_3 0x3807
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_4_5 0x3808
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_6_7 0x3809
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_8_9 0x380a
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_10_11 0x380b
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_12_13 0x380c
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_14_15 0x380d
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_16_17 0x380e
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_16_17_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_18_19 0x380f
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_18_19_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_20_21 0x3810
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_20_21_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_22_23 0x3811
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_22_23_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_24_25 0x3812
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_24_25_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_26_27 0x3813
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_26_27_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_28_29 0x3814
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_28_29_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_30_31 0x3815
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_30_31_BASE_IDX 2
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_32_33 0x3816
+#define mmCM5_CM_BLNDGAM_RAMB_REGION_32_33_BASE_IDX 2
+#define mmCM5_CM_HDR_MULT_COEF 0x3817
+#define mmCM5_CM_HDR_MULT_COEF_BASE_IDX 2
+#define mmCM5_CM_MEM_PWR_CTRL 0x3818
+#define mmCM5_CM_MEM_PWR_CTRL_BASE_IDX 2
+#define mmCM5_CM_MEM_PWR_STATUS 0x3819
+#define mmCM5_CM_MEM_PWR_STATUS_BASE_IDX 2
+#define mmCM5_CM_DEALPHA 0x381b
+#define mmCM5_CM_DEALPHA_BASE_IDX 2
+#define mmCM5_CM_COEF_FORMAT 0x381c
+#define mmCM5_CM_COEF_FORMAT_BASE_IDX 2
+#define mmCM5_CM_SHAPER_CONTROL 0x381d
+#define mmCM5_CM_SHAPER_CONTROL_BASE_IDX 2
+#define mmCM5_CM_SHAPER_OFFSET_R 0x381e
+#define mmCM5_CM_SHAPER_OFFSET_R_BASE_IDX 2
+#define mmCM5_CM_SHAPER_OFFSET_G 0x381f
+#define mmCM5_CM_SHAPER_OFFSET_G_BASE_IDX 2
+#define mmCM5_CM_SHAPER_OFFSET_B 0x3820
+#define mmCM5_CM_SHAPER_OFFSET_B_BASE_IDX 2
+#define mmCM5_CM_SHAPER_SCALE_R 0x3821
+#define mmCM5_CM_SHAPER_SCALE_R_BASE_IDX 2
+#define mmCM5_CM_SHAPER_SCALE_G_B 0x3822
+#define mmCM5_CM_SHAPER_SCALE_G_B_BASE_IDX 2
+#define mmCM5_CM_SHAPER_LUT_INDEX 0x3823
+#define mmCM5_CM_SHAPER_LUT_INDEX_BASE_IDX 2
+#define mmCM5_CM_SHAPER_LUT_DATA 0x3824
+#define mmCM5_CM_SHAPER_LUT_DATA_BASE_IDX 2
+#define mmCM5_CM_SHAPER_LUT_WRITE_EN_MASK 0x3825
+#define mmCM5_CM_SHAPER_LUT_WRITE_EN_MASK_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_START_CNTL_B 0x3826
+#define mmCM5_CM_SHAPER_RAMA_START_CNTL_B_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_START_CNTL_G 0x3827
+#define mmCM5_CM_SHAPER_RAMA_START_CNTL_G_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_START_CNTL_R 0x3828
+#define mmCM5_CM_SHAPER_RAMA_START_CNTL_R_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_END_CNTL_B 0x3829
+#define mmCM5_CM_SHAPER_RAMA_END_CNTL_B_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_END_CNTL_G 0x382a
+#define mmCM5_CM_SHAPER_RAMA_END_CNTL_G_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_END_CNTL_R 0x382b
+#define mmCM5_CM_SHAPER_RAMA_END_CNTL_R_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_REGION_0_1 0x382c
+#define mmCM5_CM_SHAPER_RAMA_REGION_0_1_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_REGION_2_3 0x382d
+#define mmCM5_CM_SHAPER_RAMA_REGION_2_3_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_REGION_4_5 0x382e
+#define mmCM5_CM_SHAPER_RAMA_REGION_4_5_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_REGION_6_7 0x382f
+#define mmCM5_CM_SHAPER_RAMA_REGION_6_7_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_REGION_8_9 0x3830
+#define mmCM5_CM_SHAPER_RAMA_REGION_8_9_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_REGION_10_11 0x3831
+#define mmCM5_CM_SHAPER_RAMA_REGION_10_11_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_REGION_12_13 0x3832
+#define mmCM5_CM_SHAPER_RAMA_REGION_12_13_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_REGION_14_15 0x3833
+#define mmCM5_CM_SHAPER_RAMA_REGION_14_15_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_REGION_16_17 0x3834
+#define mmCM5_CM_SHAPER_RAMA_REGION_16_17_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_REGION_18_19 0x3835
+#define mmCM5_CM_SHAPER_RAMA_REGION_18_19_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_REGION_20_21 0x3836
+#define mmCM5_CM_SHAPER_RAMA_REGION_20_21_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_REGION_22_23 0x3837
+#define mmCM5_CM_SHAPER_RAMA_REGION_22_23_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_REGION_24_25 0x3838
+#define mmCM5_CM_SHAPER_RAMA_REGION_24_25_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_REGION_26_27 0x3839
+#define mmCM5_CM_SHAPER_RAMA_REGION_26_27_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_REGION_28_29 0x383a
+#define mmCM5_CM_SHAPER_RAMA_REGION_28_29_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_REGION_30_31 0x383b
+#define mmCM5_CM_SHAPER_RAMA_REGION_30_31_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMA_REGION_32_33 0x383c
+#define mmCM5_CM_SHAPER_RAMA_REGION_32_33_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_START_CNTL_B 0x383d
+#define mmCM5_CM_SHAPER_RAMB_START_CNTL_B_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_START_CNTL_G 0x383e
+#define mmCM5_CM_SHAPER_RAMB_START_CNTL_G_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_START_CNTL_R 0x383f
+#define mmCM5_CM_SHAPER_RAMB_START_CNTL_R_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_END_CNTL_B 0x3840
+#define mmCM5_CM_SHAPER_RAMB_END_CNTL_B_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_END_CNTL_G 0x3841
+#define mmCM5_CM_SHAPER_RAMB_END_CNTL_G_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_END_CNTL_R 0x3842
+#define mmCM5_CM_SHAPER_RAMB_END_CNTL_R_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_REGION_0_1 0x3843
+#define mmCM5_CM_SHAPER_RAMB_REGION_0_1_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_REGION_2_3 0x3844
+#define mmCM5_CM_SHAPER_RAMB_REGION_2_3_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_REGION_4_5 0x3845
+#define mmCM5_CM_SHAPER_RAMB_REGION_4_5_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_REGION_6_7 0x3846
+#define mmCM5_CM_SHAPER_RAMB_REGION_6_7_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_REGION_8_9 0x3847
+#define mmCM5_CM_SHAPER_RAMB_REGION_8_9_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_REGION_10_11 0x3848
+#define mmCM5_CM_SHAPER_RAMB_REGION_10_11_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_REGION_12_13 0x3849
+#define mmCM5_CM_SHAPER_RAMB_REGION_12_13_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_REGION_14_15 0x384a
+#define mmCM5_CM_SHAPER_RAMB_REGION_14_15_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_REGION_16_17 0x384b
+#define mmCM5_CM_SHAPER_RAMB_REGION_16_17_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_REGION_18_19 0x384c
+#define mmCM5_CM_SHAPER_RAMB_REGION_18_19_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_REGION_20_21 0x384d
+#define mmCM5_CM_SHAPER_RAMB_REGION_20_21_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_REGION_22_23 0x384e
+#define mmCM5_CM_SHAPER_RAMB_REGION_22_23_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_REGION_24_25 0x384f
+#define mmCM5_CM_SHAPER_RAMB_REGION_24_25_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_REGION_26_27 0x3850
+#define mmCM5_CM_SHAPER_RAMB_REGION_26_27_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_REGION_28_29 0x3851
+#define mmCM5_CM_SHAPER_RAMB_REGION_28_29_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_REGION_30_31 0x3852
+#define mmCM5_CM_SHAPER_RAMB_REGION_30_31_BASE_IDX 2
+#define mmCM5_CM_SHAPER_RAMB_REGION_32_33 0x3853
+#define mmCM5_CM_SHAPER_RAMB_REGION_32_33_BASE_IDX 2
+#define mmCM5_CM_MEM_PWR_CTRL2 0x3854
+#define mmCM5_CM_MEM_PWR_CTRL2_BASE_IDX 2
+#define mmCM5_CM_MEM_PWR_STATUS2 0x3855
+#define mmCM5_CM_MEM_PWR_STATUS2_BASE_IDX 2
+#define mmCM5_CM_3DLUT_MODE 0x3856
+#define mmCM5_CM_3DLUT_MODE_BASE_IDX 2
+#define mmCM5_CM_3DLUT_INDEX 0x3857
+#define mmCM5_CM_3DLUT_INDEX_BASE_IDX 2
+#define mmCM5_CM_3DLUT_DATA 0x3858
+#define mmCM5_CM_3DLUT_DATA_BASE_IDX 2
+#define mmCM5_CM_3DLUT_DATA_30BIT 0x3859
+#define mmCM5_CM_3DLUT_DATA_30BIT_BASE_IDX 2
+#define mmCM5_CM_3DLUT_READ_WRITE_CONTROL 0x385a
+#define mmCM5_CM_3DLUT_READ_WRITE_CONTROL_BASE_IDX 2
+#define mmCM5_CM_3DLUT_OUT_NORM_FACTOR 0x385b
+#define mmCM5_CM_3DLUT_OUT_NORM_FACTOR_BASE_IDX 2
+#define mmCM5_CM_3DLUT_OUT_OFFSET_R 0x385c
+#define mmCM5_CM_3DLUT_OUT_OFFSET_R_BASE_IDX 2
+#define mmCM5_CM_3DLUT_OUT_OFFSET_G 0x385d
+#define mmCM5_CM_3DLUT_OUT_OFFSET_G_BASE_IDX 2
+#define mmCM5_CM_3DLUT_OUT_OFFSET_B 0x385e
+#define mmCM5_CM_3DLUT_OUT_OFFSET_B_BASE_IDX 2
+#define mmCM5_CM_TEST_DEBUG_INDEX 0x385f
+#define mmCM5_CM_TEST_DEBUG_INDEX_BASE_IDX 2
+#define mmCM5_CM_TEST_DEBUG_DATA 0x3860
+#define mmCM5_CM_TEST_DEBUG_DATA_BASE_IDX 2
+
+
+// addressBlock: dce_dc_dpp5_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
+// base address: 0xe268
+#define mmDC_PERFMON28_PERFCOUNTER_CNTL 0x389a
+#define mmDC_PERFMON28_PERFCOUNTER_CNTL_BASE_IDX 2
+#define mmDC_PERFMON28_PERFCOUNTER_CNTL2 0x389b
+#define mmDC_PERFMON28_PERFCOUNTER_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON28_PERFCOUNTER_STATE 0x389c
+#define mmDC_PERFMON28_PERFCOUNTER_STATE_BASE_IDX 2
+#define mmDC_PERFMON28_PERFMON_CNTL 0x389d
+#define mmDC_PERFMON28_PERFMON_CNTL_BASE_IDX 2
+#define mmDC_PERFMON28_PERFMON_CNTL2 0x389e
+#define mmDC_PERFMON28_PERFMON_CNTL2_BASE_IDX 2
+#define mmDC_PERFMON28_PERFMON_CVALUE_INT_MISC 0x389f
+#define mmDC_PERFMON28_PERFMON_CVALUE_INT_MISC_BASE_IDX 2
+#define mmDC_PERFMON28_PERFMON_CVALUE_LOW 0x38a0
+#define mmDC_PERFMON28_PERFMON_CVALUE_LOW_BASE_IDX 2
+#define mmDC_PERFMON28_PERFMON_HI 0x38a1
+#define mmDC_PERFMON28_PERFMON_HI_BASE_IDX 2
+#define mmDC_PERFMON28_PERFMON_LOW 0x38a2
+#define mmDC_PERFMON28_PERFMON_LOW_BASE_IDX 2
+
+
+// addressBlock: dce_dc_hda_azcontroller_azdec
+// base address: 0x0
+#define mmCORB_WRITE_POINTER 0x0000
+#define mmCORB_WRITE_POINTER_BASE_IDX 0
+#define mmCORB_READ_POINTER 0x0000
+#define mmCORB_READ_POINTER_BASE_IDX 0
+#define mmCORB_CONTROL 0x0001
+#define mmCORB_CONTROL_BASE_IDX 0
+#define mmCORB_STATUS 0x0001
+#define mmCORB_STATUS_BASE_IDX 0
+#define mmCORB_SIZE 0x0001
+#define mmCORB_SIZE_BASE_IDX 0
+#define mmRIRB_LOWER_BASE_ADDRESS 0x0002
+#define mmRIRB_LOWER_BASE_ADDRESS_BASE_IDX 0
+#define mmRIRB_UPPER_BASE_ADDRESS 0x0003
+#define mmRIRB_UPPER_BASE_ADDRESS_BASE_IDX 0
+#define mmRIRB_WRITE_POINTER 0x0004
+#define mmRIRB_WRITE_POINTER_BASE_IDX 0
+#define mmRESPONSE_INTERRUPT_COUNT 0x0004
+#define mmRESPONSE_INTERRUPT_COUNT_BASE_IDX 0
+#define mmRIRB_CONTROL 0x0005
+#define mmRIRB_CONTROL_BASE_IDX 0
+#define mmRIRB_STATUS 0x0005
+#define mmRIRB_STATUS_BASE_IDX 0
+#define mmRIRB_SIZE 0x0005
+#define mmRIRB_SIZE_BASE_IDX 0
+#define mmIMMEDIATE_COMMAND_OUTPUT_INTERFACE 0x0006
+#define mmIMMEDIATE_COMMAND_OUTPUT_INTERFACE_BASE_IDX 0
+#define mmIMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA 0x0006
+#define mmIMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_BASE_IDX 0
+#define mmIMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX 0x0006
+#define mmIMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_BASE_IDX 0
+#define mmIMMEDIATE_RESPONSE_INPUT_INTERFACE 0x0007
+#define mmIMMEDIATE_RESPONSE_INPUT_INTERFACE_BASE_IDX 0
+#define mmIMMEDIATE_COMMAND_STATUS 0x0008
+#define mmIMMEDIATE_COMMAND_STATUS_BASE_IDX 0
+#define mmDMA_POSITION_LOWER_BASE_ADDRESS 0x000a
+#define mmDMA_POSITION_LOWER_BASE_ADDRESS_BASE_IDX 0
+#define mmDMA_POSITION_UPPER_BASE_ADDRESS 0x000b
+#define mmDMA_POSITION_UPPER_BASE_ADDRESS_BASE_IDX 0
+#define mmWALL_CLOCK_COUNTER_ALIAS 0x074c
+#define mmWALL_CLOCK_COUNTER_ALIAS_BASE_IDX 1
+
+
+// addressBlock: dce_dc_hda_azendpoint_azdec
+// base address: 0x0
+#define mmAZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA 0x0006
+#define mmAZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_BASE_IDX 0
+#define mmAZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX 0x0006
+#define mmAZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_BASE_IDX 0
+
+
+// addressBlock: dce_dc_hda_azinputendpoint_azdec
+// base address: 0x0
+#define mmAZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_DATA 0x0006
+#define mmAZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_DATA_BASE_IDX 0
+#define mmAZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_INDEX 0x0006
+#define mmAZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_INDEX_BASE_IDX 0
+
+
+// addressBlock: dce_dc_hda_azroot_azdec
+// base address: 0x0
+#define mmAZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA 0x0006
+#define mmAZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_BASE_IDX 0
+#define mmAZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX 0x0006
+#define mmAZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_BASE_IDX 0
+
+
+// addressBlock: dce_dc_hda_azstream0_azdec
+// base address: 0x0
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS 0x000e
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX 0
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER 0x000f
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX 0
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH 0x0010
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX 0
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX 0x0011
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX 0
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE 0x0012
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX 0
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_FORMAT 0x0012
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX 0
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS 0x0014
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX 0
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS 0x0015
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX 0
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS 0x0761
+#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX 1
+
+
+// addressBlock: dce_dc_hda_azstream1_azdec
+// base address: 0x20
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS 0x0016
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX 0
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER 0x0017
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX 0
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH 0x0018
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX 0
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX 0x0019
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX 0
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE 0x001a
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX 0
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_FORMAT 0x001a
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX 0
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS 0x001c
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX 0
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS 0x001d
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX 0
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS 0x0769
+#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX 1
+
+
+// addressBlock: dce_dc_hda_azstream2_azdec
+// base address: 0x40
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS 0x001e
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX 0
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER 0x001f
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX 0
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH 0x0020
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX 0
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX 0x0021
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX 0
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE 0x0022
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX 0
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_FORMAT 0x0022
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX 0
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS 0x0024
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX 0
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS 0x0025
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX 0
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS 0x0771
+#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX 1
+
+
+// addressBlock: dce_dc_hda_azstream3_azdec
+// base address: 0x60
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS 0x0026
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX 0
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER 0x0027
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX 0
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH 0x0028
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX 0
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX 0x0029
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX 0
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE 0x002a
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX 0
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_FORMAT 0x002a
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX 0
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS 0x002c
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX 0
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS 0x002d
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX 0
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS 0x0779
+#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX 1
+
+
+// addressBlock: dce_dc_hda_azstream4_azdec
+// base address: 0x80
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS 0x002e
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX 0
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER 0x002f
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX 0
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH 0x0030
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX 0
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX 0x0031
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX 0
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE 0x0032
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX 0
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_FORMAT 0x0032
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX 0
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS 0x0034
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX 0
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS 0x0035
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX 0
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS 0x0781
+#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX 1
+
+
+// addressBlock: dce_dc_hda_azstream5_azdec
+// base address: 0xa0
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS 0x0036
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX 0
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER 0x0037
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX 0
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH 0x0038
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX 0
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX 0x0039
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX 0
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE 0x003a
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX 0
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_FORMAT 0x003a
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX 0
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS 0x003c
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX 0
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS 0x003d
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX 0
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS 0x0789
+#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX 1
+
+
+// addressBlock: dce_dc_hda_azstream6_azdec
+// base address: 0xc0
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS 0x003e
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX 0
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER 0x003f
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX 0
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH 0x0040
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX 0
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX 0x0041
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX 0
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE 0x0042
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX 0
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_FORMAT 0x0042
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX 0
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS 0x0044
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX 0
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS 0x0045
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX 0
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS 0x0791
+#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX 1
+
+
+// addressBlock: dce_dc_hda_azstream7_azdec
+// base address: 0xe0
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS 0x0046
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_BASE_IDX 0
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER 0x0047
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_BASE_IDX 0
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH 0x0048
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_BASE_IDX 0
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX 0x0049
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_BASE_IDX 0
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE 0x004a
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_BASE_IDX 0
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_FORMAT 0x004a
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_FORMAT_BASE_IDX 0
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS 0x004c
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_BASE_IDX 0
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS 0x004d
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_BASE_IDX 0
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS 0x0799
+#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_BASE_IDX 1
+
+
+// addressBlock: vga_vgaseqind
+// base address: 0x0
+#define ixSEQ00 0x0000
+#define ixSEQ01 0x0001
+#define ixSEQ02 0x0002
+#define ixSEQ03 0x0003
+#define ixSEQ04 0x0004
+
+
+// addressBlock: vga_vgacrtind
+// base address: 0x0
+#define ixCRT00 0x0000
+#define ixCRT01 0x0001
+#define ixCRT02 0x0002
+#define ixCRT03 0x0003
+#define ixCRT04 0x0004
+#define ixCRT05 0x0005
+#define ixCRT06 0x0006
+#define ixCRT07 0x0007
+#define ixCRT08 0x0008
+#define ixCRT09 0x0009
+#define ixCRT0A 0x000a
+#define ixCRT0B 0x000b
+#define ixCRT0C 0x000c
+#define ixCRT0D 0x000d
+#define ixCRT0E 0x000e
+#define ixCRT0F 0x000f
+#define ixCRT10 0x0010
+#define ixCRT11 0x0011
+#define ixCRT12 0x0012
+#define ixCRT13 0x0013
+#define ixCRT14 0x0014
+#define ixCRT15 0x0015
+#define ixCRT16 0x0016
+#define ixCRT17 0x0017
+#define ixCRT18 0x0018
+#define ixCRT1E 0x001e
+#define ixCRT1F 0x001f
+#define ixCRT22 0x0022
+
+
+// addressBlock: vga_vgagrphind
+// base address: 0x0
+#define ixGRA00 0x0000
+#define ixGRA01 0x0001
+#define ixGRA02 0x0002
+#define ixGRA03 0x0003
+#define ixGRA04 0x0004
+#define ixGRA05 0x0005
+#define ixGRA06 0x0006
+#define ixGRA07 0x0007
+#define ixGRA08 0x0008
+
+
+// addressBlock: vga_vgaattrind
+// base address: 0x0
+#define ixATTR00 0x0000
+#define ixATTR01 0x0001
+#define ixATTR02 0x0002
+#define ixATTR03 0x0003
+#define ixATTR04 0x0004
+#define ixATTR05 0x0005
+#define ixATTR06 0x0006
+#define ixATTR07 0x0007
+#define ixATTR08 0x0008
+#define ixATTR09 0x0009
+#define ixATTR0A 0x000a
+#define ixATTR0B 0x000b
+#define ixATTR0C 0x000c
+#define ixATTR0D 0x000d
+#define ixATTR0E 0x000e
+#define ixATTR0F 0x000f
+#define ixATTR10 0x0010
+#define ixATTR11 0x0011
+#define ixATTR12 0x0012
+#define ixATTR13 0x0013
+#define ixATTR14 0x0014
+
+
+// addressBlock: azendpoint_f2codecind
+// base address: 0x0
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT 0x2200
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x2706
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x270d
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_2 0x270e
+#define ixAZALIA_F2_CODEC_CONVERTER_STRIPE_CONTROL 0x2724
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_3 0x273e
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_RAMP_RATE 0x2770
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING 0x2771
+#define ixAZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x2f09
+#define ixAZALIA_F2_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x2f0a
+#define ixAZALIA_F2_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS 0x2f0b
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONNECTION_LIST_ENTRY 0x3702
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_WIDGET_CONTROL 0x3707
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE 0x3708
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE 0x3709
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x371c
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2 0x371d
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3 0x371e
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4 0x371f
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION 0x3770
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_CHANNEL_ALLOCATION 0x3771
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_DOWN_MIX_INFO 0x3772
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR 0x3776
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR_DATA 0x3776
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE 0x3777
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE 0x3778
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE 0x3779
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE 0x377a
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_LIPSYNC 0x377b
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_HBR 0x377c
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_INDEX 0x3780
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_DATA 0x3781
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE 0x3785
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE 0x3786
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE 0x3787
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE 0x3788
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL_MODE 0x3789
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_0 0x378a
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1 0x378b
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_2 0x378c
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_3 0x378d
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4 0x378e
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_5 0x378f
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_6 0x3790
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_7 0x3791
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_8 0x3792
+#define ixAZALIA_F2_CODEC_PIN_ASSOCIATION_INFO 0x3793
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS 0x3797
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x3798
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_LPIB 0x3799
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x379a
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_CODING_TYPE 0x379b
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_FORMAT_CHANGED 0x379c
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION 0x379d
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE 0x379e
+#define ixAZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x3f09
+#define ixAZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES 0x3f0c
+#define ixAZALIA_F2_CODEC_PIN_PARAMETER_CONNECTION_LIST_LENGTH 0x3f0e
+
+
+// addressBlock: azendpoint_descriptorind
+// base address: 0x0
+#define ixAUDIO_DESCRIPTOR0 0x0001
+#define ixAUDIO_DESCRIPTOR1 0x0002
+#define ixAUDIO_DESCRIPTOR2 0x0003
+#define ixAUDIO_DESCRIPTOR3 0x0004
+#define ixAUDIO_DESCRIPTOR4 0x0005
+#define ixAUDIO_DESCRIPTOR5 0x0006
+#define ixAUDIO_DESCRIPTOR6 0x0007
+#define ixAUDIO_DESCRIPTOR7 0x0008
+#define ixAUDIO_DESCRIPTOR8 0x0009
+#define ixAUDIO_DESCRIPTOR9 0x000a
+#define ixAUDIO_DESCRIPTOR10 0x000b
+#define ixAUDIO_DESCRIPTOR11 0x000c
+#define ixAUDIO_DESCRIPTOR12 0x000d
+#define ixAUDIO_DESCRIPTOR13 0x000e
+
+
+// addressBlock: azendpoint_sinkinfoind
+// base address: 0x0
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MANUFACTURER_ID 0x0000
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_PRODUCT_ID 0x0001
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_SINK_DESCRIPTION_LEN 0x0002
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_PORTID0 0x0003
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_PORTID1 0x0004
+#define ixSINK_DESCRIPTION0 0x0005
+#define ixSINK_DESCRIPTION1 0x0006
+#define ixSINK_DESCRIPTION2 0x0007
+#define ixSINK_DESCRIPTION3 0x0008
+#define ixSINK_DESCRIPTION4 0x0009
+#define ixSINK_DESCRIPTION5 0x000a
+#define ixSINK_DESCRIPTION6 0x000b
+#define ixSINK_DESCRIPTION7 0x000c
+#define ixSINK_DESCRIPTION8 0x000d
+#define ixSINK_DESCRIPTION9 0x000e
+#define ixSINK_DESCRIPTION10 0x000f
+#define ixSINK_DESCRIPTION11 0x0010
+#define ixSINK_DESCRIPTION12 0x0011
+#define ixSINK_DESCRIPTION13 0x0012
+#define ixSINK_DESCRIPTION14 0x0013
+#define ixSINK_DESCRIPTION15 0x0014
+#define ixSINK_DESCRIPTION16 0x0015
+#define ixSINK_DESCRIPTION17 0x0016
+
+
+// addressBlock: azf0controller_azinputcrc0resultind
+// base address: 0x0
+#define ixAZALIA_INPUT_CRC0_CHANNEL0 0x0000
+#define ixAZALIA_INPUT_CRC0_CHANNEL1 0x0001
+#define ixAZALIA_INPUT_CRC0_CHANNEL2 0x0002
+#define ixAZALIA_INPUT_CRC0_CHANNEL3 0x0003
+#define ixAZALIA_INPUT_CRC0_CHANNEL4 0x0004
+#define ixAZALIA_INPUT_CRC0_CHANNEL5 0x0005
+#define ixAZALIA_INPUT_CRC0_CHANNEL6 0x0006
+#define ixAZALIA_INPUT_CRC0_CHANNEL7 0x0007
+
+
+// addressBlock: azf0controller_azinputcrc1resultind
+// base address: 0x0
+#define ixAZALIA_INPUT_CRC1_CHANNEL0 0x0000
+#define ixAZALIA_INPUT_CRC1_CHANNEL1 0x0001
+#define ixAZALIA_INPUT_CRC1_CHANNEL2 0x0002
+#define ixAZALIA_INPUT_CRC1_CHANNEL3 0x0003
+#define ixAZALIA_INPUT_CRC1_CHANNEL4 0x0004
+#define ixAZALIA_INPUT_CRC1_CHANNEL5 0x0005
+#define ixAZALIA_INPUT_CRC1_CHANNEL6 0x0006
+#define ixAZALIA_INPUT_CRC1_CHANNEL7 0x0007
+
+
+// addressBlock: azf0controller_azcrc0resultind
+// base address: 0x0
+#define ixAZALIA_CRC0_CHANNEL0 0x0000
+#define ixAZALIA_CRC0_CHANNEL1 0x0001
+#define ixAZALIA_CRC0_CHANNEL2 0x0002
+#define ixAZALIA_CRC0_CHANNEL3 0x0003
+#define ixAZALIA_CRC0_CHANNEL4 0x0004
+#define ixAZALIA_CRC0_CHANNEL5 0x0005
+#define ixAZALIA_CRC0_CHANNEL6 0x0006
+#define ixAZALIA_CRC0_CHANNEL7 0x0007
+
+
+// addressBlock: azf0controller_azcrc1resultind
+// base address: 0x0
+#define ixAZALIA_CRC1_CHANNEL0 0x0000
+#define ixAZALIA_CRC1_CHANNEL1 0x0001
+#define ixAZALIA_CRC1_CHANNEL2 0x0002
+#define ixAZALIA_CRC1_CHANNEL3 0x0003
+#define ixAZALIA_CRC1_CHANNEL4 0x0004
+#define ixAZALIA_CRC1_CHANNEL5 0x0005
+#define ixAZALIA_CRC1_CHANNEL6 0x0006
+#define ixAZALIA_CRC1_CHANNEL7 0x0007
+
+
+// addressBlock: azinputendpoint_f2codecind
+// base address: 0x0
+#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT 0x6200
+#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x6706
+#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x670d
+#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x6f09
+#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x6f0a
+#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS 0x6f0b
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL 0x7707
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE 0x7708
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_PIN_SENSE 0x7709
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x771c
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2 0x771d
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3 0x771e
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4 0x771f
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION 0x7771
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL0_ENABLE 0x7777
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL2_ENABLE 0x7778
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL4_ENABLE 0x7779
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL6_ENABLE 0x777a
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_HBR 0x777c
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL1_ENABLE 0x7785
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL3_ENABLE 0x7786
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL5_ENABLE 0x7787
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL7_ENABLE 0x7788
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x7798
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB 0x7799
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x779a
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL 0x779b
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_INFOFRAME 0x779c
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_STATUS_L 0x779d
+#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_STATUS_H 0x779e
+#define ixAZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x7f09
+#define ixAZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES 0x7f0c
+
+
+// addressBlock: azroot_f2codecind
+// base address: 0x0
+#define ixAZALIA_F2_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID 0x0f00
+#define ixAZALIA_F2_CODEC_ROOT_PARAMETER_REVISION_ID 0x0f02
+#define ixAZALIA_F2_CODEC_ROOT_PARAMETER_SUBORDINATE_NODE_COUNT 0x0f04
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE 0x1705
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID 0x1720
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_2 0x1721
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_3 0x1722
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_4 0x1723
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION 0x1770
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESET 0x17ff
+#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_SUBORDINATE_NODE_COUNT 0x1f04
+#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_GROUP_TYPE 0x1f05
+#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES 0x1f0a
+#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS 0x1f0b
+#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES 0x1f0f
+
+
+// addressBlock: azf0stream0_streamind
+// base address: 0x0
+#define ixAZF0STREAM0_AZALIA_FIFO_SIZE_CONTROL 0x0000
+#define ixAZF0STREAM0_AZALIA_LATENCY_COUNTER_CONTROL 0x0001
+#define ixAZF0STREAM0_AZALIA_WORSTCASE_LATENCY_COUNT 0x0002
+#define ixAZF0STREAM0_AZALIA_CUMULATIVE_LATENCY_COUNT 0x0003
+#define ixAZF0STREAM0_AZALIA_CUMULATIVE_REQUEST_COUNT 0x0004
+
+
+// addressBlock: azf0stream1_streamind
+// base address: 0x0
+#define ixAZF0STREAM1_AZALIA_FIFO_SIZE_CONTROL 0x0000
+#define ixAZF0STREAM1_AZALIA_LATENCY_COUNTER_CONTROL 0x0001
+#define ixAZF0STREAM1_AZALIA_WORSTCASE_LATENCY_COUNT 0x0002
+#define ixAZF0STREAM1_AZALIA_CUMULATIVE_LATENCY_COUNT 0x0003
+#define ixAZF0STREAM1_AZALIA_CUMULATIVE_REQUEST_COUNT 0x0004
+
+
+// addressBlock: azf0stream2_streamind
+// base address: 0x0
+#define ixAZF0STREAM2_AZALIA_FIFO_SIZE_CONTROL 0x0000
+#define ixAZF0STREAM2_AZALIA_LATENCY_COUNTER_CONTROL 0x0001
+#define ixAZF0STREAM2_AZALIA_WORSTCASE_LATENCY_COUNT 0x0002
+#define ixAZF0STREAM2_AZALIA_CUMULATIVE_LATENCY_COUNT 0x0003
+#define ixAZF0STREAM2_AZALIA_CUMULATIVE_REQUEST_COUNT 0x0004
+
+
+// addressBlock: azf0stream3_streamind
+// base address: 0x0
+#define ixAZF0STREAM3_AZALIA_FIFO_SIZE_CONTROL 0x0000
+#define ixAZF0STREAM3_AZALIA_LATENCY_COUNTER_CONTROL 0x0001
+#define ixAZF0STREAM3_AZALIA_WORSTCASE_LATENCY_COUNT 0x0002
+#define ixAZF0STREAM3_AZALIA_CUMULATIVE_LATENCY_COUNT 0x0003
+#define ixAZF0STREAM3_AZALIA_CUMULATIVE_REQUEST_COUNT 0x0004
+
+
+// addressBlock: azf0stream4_streamind
+// base address: 0x0
+#define ixAZF0STREAM4_AZALIA_FIFO_SIZE_CONTROL 0x0000
+#define ixAZF0STREAM4_AZALIA_LATENCY_COUNTER_CONTROL 0x0001
+#define ixAZF0STREAM4_AZALIA_WORSTCASE_LATENCY_COUNT 0x0002
+#define ixAZF0STREAM4_AZALIA_CUMULATIVE_LATENCY_COUNT 0x0003
+#define ixAZF0STREAM4_AZALIA_CUMULATIVE_REQUEST_COUNT 0x0004
+
+
+// addressBlock: azf0stream5_streamind
+// base address: 0x0
+#define ixAZF0STREAM5_AZALIA_FIFO_SIZE_CONTROL 0x0000
+#define ixAZF0STREAM5_AZALIA_LATENCY_COUNTER_CONTROL 0x0001
+#define ixAZF0STREAM5_AZALIA_WORSTCASE_LATENCY_COUNT 0x0002
+#define ixAZF0STREAM5_AZALIA_CUMULATIVE_LATENCY_COUNT 0x0003
+#define ixAZF0STREAM5_AZALIA_CUMULATIVE_REQUEST_COUNT 0x0004
+
+
+// addressBlock: azf0stream6_streamind
+// base address: 0x0
+#define ixAZF0STREAM6_AZALIA_FIFO_SIZE_CONTROL 0x0000
+#define ixAZF0STREAM6_AZALIA_LATENCY_COUNTER_CONTROL 0x0001
+#define ixAZF0STREAM6_AZALIA_WORSTCASE_LATENCY_COUNT 0x0002
+#define ixAZF0STREAM6_AZALIA_CUMULATIVE_LATENCY_COUNT 0x0003
+#define ixAZF0STREAM6_AZALIA_CUMULATIVE_REQUEST_COUNT 0x0004
+
+
+// addressBlock: azf0stream7_streamind
+// base address: 0x0
+#define ixAZF0STREAM7_AZALIA_FIFO_SIZE_CONTROL 0x0000
+#define ixAZF0STREAM7_AZALIA_LATENCY_COUNTER_CONTROL 0x0001
+#define ixAZF0STREAM7_AZALIA_WORSTCASE_LATENCY_COUNT 0x0002
+#define ixAZF0STREAM7_AZALIA_CUMULATIVE_LATENCY_COUNT 0x0003
+#define ixAZF0STREAM7_AZALIA_CUMULATIVE_REQUEST_COUNT 0x0004
+
+
+// addressBlock: azf0stream8_streamind
+// base address: 0x0
+#define ixAZF0STREAM8_AZALIA_FIFO_SIZE_CONTROL 0x0000
+#define ixAZF0STREAM8_AZALIA_LATENCY_COUNTER_CONTROL 0x0001
+#define ixAZF0STREAM8_AZALIA_WORSTCASE_LATENCY_COUNT 0x0002
+#define ixAZF0STREAM8_AZALIA_CUMULATIVE_LATENCY_COUNT 0x0003
+#define ixAZF0STREAM8_AZALIA_CUMULATIVE_REQUEST_COUNT 0x0004
+
+
+// addressBlock: azf0stream9_streamind
+// base address: 0x0
+#define ixAZF0STREAM9_AZALIA_FIFO_SIZE_CONTROL 0x0000
+#define ixAZF0STREAM9_AZALIA_LATENCY_COUNTER_CONTROL 0x0001
+#define ixAZF0STREAM9_AZALIA_WORSTCASE_LATENCY_COUNT 0x0002
+#define ixAZF0STREAM9_AZALIA_CUMULATIVE_LATENCY_COUNT 0x0003
+#define ixAZF0STREAM9_AZALIA_CUMULATIVE_REQUEST_COUNT 0x0004
+
+
+// addressBlock: azf0stream10_streamind
+// base address: 0x0
+#define ixAZF0STREAM10_AZALIA_FIFO_SIZE_CONTROL 0x0000
+#define ixAZF0STREAM10_AZALIA_LATENCY_COUNTER_CONTROL 0x0001
+#define ixAZF0STREAM10_AZALIA_WORSTCASE_LATENCY_COUNT 0x0002
+#define ixAZF0STREAM10_AZALIA_CUMULATIVE_LATENCY_COUNT 0x0003
+#define ixAZF0STREAM10_AZALIA_CUMULATIVE_REQUEST_COUNT 0x0004
+
+
+// addressBlock: azf0stream11_streamind
+// base address: 0x0
+#define ixAZF0STREAM11_AZALIA_FIFO_SIZE_CONTROL 0x0000
+#define ixAZF0STREAM11_AZALIA_LATENCY_COUNTER_CONTROL 0x0001
+#define ixAZF0STREAM11_AZALIA_WORSTCASE_LATENCY_COUNT 0x0002
+#define ixAZF0STREAM11_AZALIA_CUMULATIVE_LATENCY_COUNT 0x0003
+#define ixAZF0STREAM11_AZALIA_CUMULATIVE_REQUEST_COUNT 0x0004
+
+
+// addressBlock: azf0stream12_streamind
+// base address: 0x0
+#define ixAZF0STREAM12_AZALIA_FIFO_SIZE_CONTROL 0x0000
+#define ixAZF0STREAM12_AZALIA_LATENCY_COUNTER_CONTROL 0x0001
+#define ixAZF0STREAM12_AZALIA_WORSTCASE_LATENCY_COUNT 0x0002
+#define ixAZF0STREAM12_AZALIA_CUMULATIVE_LATENCY_COUNT 0x0003
+#define ixAZF0STREAM12_AZALIA_CUMULATIVE_REQUEST_COUNT 0x0004
+
+
+// addressBlock: azf0stream13_streamind
+// base address: 0x0
+#define ixAZF0STREAM13_AZALIA_FIFO_SIZE_CONTROL 0x0000
+#define ixAZF0STREAM13_AZALIA_LATENCY_COUNTER_CONTROL 0x0001
+#define ixAZF0STREAM13_AZALIA_WORSTCASE_LATENCY_COUNT 0x0002
+#define ixAZF0STREAM13_AZALIA_CUMULATIVE_LATENCY_COUNT 0x0003
+#define ixAZF0STREAM13_AZALIA_CUMULATIVE_REQUEST_COUNT 0x0004
+
+
+// addressBlock: azf0stream14_streamind
+// base address: 0x0
+#define ixAZF0STREAM14_AZALIA_FIFO_SIZE_CONTROL 0x0000
+#define ixAZF0STREAM14_AZALIA_LATENCY_COUNTER_CONTROL 0x0001
+#define ixAZF0STREAM14_AZALIA_WORSTCASE_LATENCY_COUNT 0x0002
+#define ixAZF0STREAM14_AZALIA_CUMULATIVE_LATENCY_COUNT 0x0003
+#define ixAZF0STREAM14_AZALIA_CUMULATIVE_REQUEST_COUNT 0x0004
+
+
+// addressBlock: azf0stream15_streamind
+// base address: 0x0
+#define ixAZF0STREAM15_AZALIA_FIFO_SIZE_CONTROL 0x0000
+#define ixAZF0STREAM15_AZALIA_LATENCY_COUNTER_CONTROL 0x0001
+#define ixAZF0STREAM15_AZALIA_WORSTCASE_LATENCY_COUNT 0x0002
+#define ixAZF0STREAM15_AZALIA_CUMULATIVE_LATENCY_COUNT 0x0003
+#define ixAZF0STREAM15_AZALIA_CUMULATIVE_REQUEST_COUNT 0x0004
+
+
+// addressBlock: azf0endpoint0_endpointind
+// base address: 0x0
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0001
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT 0x0002
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x0003
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x0004
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS 0x0005
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x0006
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL 0x0007
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE 0x0008
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING 0x0009
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA 0x000c
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN 0x000d
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX 0x000e
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0020
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES 0x0021
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE 0x0022
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE 0x0023
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL 0x0024
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER 0x0025
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0 0x0028
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1 0x0029
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2 0x002a
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3 0x002b
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4 0x002c
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5 0x002d
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6 0x002e
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7 0x002f
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8 0x0030
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9 0x0031
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10 0x0032
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11 0x0033
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12 0x0034
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13 0x0035
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE 0x0036
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC 0x0037
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR 0x0038
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0 0x003a
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1 0x003b
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2 0x003c
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3 0x003d
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4 0x003e
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5 0x003f
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6 0x0040
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7 0x0041
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8 0x0042
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL 0x0054
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE 0x0055
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x0056
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2 0x0057
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE 0x0058
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0 0x0059
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1 0x005a
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2 0x005b
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3 0x005c
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4 0x005d
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5 0x005e
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6 0x005f
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7 0x0060
+#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8 0x0061
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO 0x0062
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS 0x0063
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x0064
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB 0x0065
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x0066
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE 0x0067
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED 0x0068
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION 0x0069
+#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE 0x006a
+#define ixAZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLE_STATUS 0x006b
+#define ixAZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLED_INT_STATUS 0x006c
+#define ixAZF0ENDPOINT0_AZALIA_F0_AUDIO_DISABLED_INT_STATUS 0x006d
+#define ixAZF0ENDPOINT0_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS 0x006e
+
+
+// addressBlock: azf0endpoint1_endpointind
+// base address: 0x0
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0001
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT 0x0002
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x0003
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x0004
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS 0x0005
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x0006
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL 0x0007
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE 0x0008
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING 0x0009
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA 0x000c
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN 0x000d
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX 0x000e
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0020
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES 0x0021
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE 0x0022
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE 0x0023
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL 0x0024
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER 0x0025
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0 0x0028
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1 0x0029
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2 0x002a
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3 0x002b
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4 0x002c
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5 0x002d
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6 0x002e
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7 0x002f
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8 0x0030
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9 0x0031
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10 0x0032
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11 0x0033
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12 0x0034
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13 0x0035
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE 0x0036
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC 0x0037
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR 0x0038
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0 0x003a
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1 0x003b
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2 0x003c
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3 0x003d
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4 0x003e
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5 0x003f
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6 0x0040
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7 0x0041
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8 0x0042
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL 0x0054
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE 0x0055
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x0056
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2 0x0057
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE 0x0058
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0 0x0059
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1 0x005a
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2 0x005b
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3 0x005c
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4 0x005d
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5 0x005e
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6 0x005f
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7 0x0060
+#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8 0x0061
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO 0x0062
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS 0x0063
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x0064
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB 0x0065
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x0066
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE 0x0067
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED 0x0068
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION 0x0069
+#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE 0x006a
+#define ixAZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLE_STATUS 0x006b
+#define ixAZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLED_INT_STATUS 0x006c
+#define ixAZF0ENDPOINT1_AZALIA_F0_AUDIO_DISABLED_INT_STATUS 0x006d
+#define ixAZF0ENDPOINT1_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS 0x006e
+
+
+// addressBlock: azf0endpoint2_endpointind
+// base address: 0x0
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0001
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT 0x0002
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x0003
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x0004
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS 0x0005
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x0006
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL 0x0007
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE 0x0008
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING 0x0009
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA 0x000c
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN 0x000d
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX 0x000e
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0020
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES 0x0021
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE 0x0022
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE 0x0023
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL 0x0024
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER 0x0025
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0 0x0028
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1 0x0029
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2 0x002a
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3 0x002b
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4 0x002c
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5 0x002d
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6 0x002e
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7 0x002f
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8 0x0030
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9 0x0031
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10 0x0032
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11 0x0033
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12 0x0034
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13 0x0035
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE 0x0036
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC 0x0037
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR 0x0038
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0 0x003a
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1 0x003b
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2 0x003c
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3 0x003d
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4 0x003e
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5 0x003f
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6 0x0040
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7 0x0041
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8 0x0042
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL 0x0054
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE 0x0055
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x0056
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2 0x0057
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE 0x0058
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0 0x0059
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1 0x005a
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2 0x005b
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3 0x005c
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4 0x005d
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5 0x005e
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6 0x005f
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7 0x0060
+#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8 0x0061
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO 0x0062
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS 0x0063
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x0064
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB 0x0065
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x0066
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE 0x0067
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED 0x0068
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION 0x0069
+#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE 0x006a
+#define ixAZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLE_STATUS 0x006b
+#define ixAZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLED_INT_STATUS 0x006c
+#define ixAZF0ENDPOINT2_AZALIA_F0_AUDIO_DISABLED_INT_STATUS 0x006d
+#define ixAZF0ENDPOINT2_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS 0x006e
+
+
+// addressBlock: azf0endpoint3_endpointind
+// base address: 0x0
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0001
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT 0x0002
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x0003
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x0004
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS 0x0005
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x0006
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL 0x0007
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE 0x0008
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING 0x0009
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA 0x000c
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN 0x000d
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX 0x000e
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0020
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES 0x0021
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE 0x0022
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE 0x0023
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL 0x0024
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER 0x0025
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0 0x0028
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1 0x0029
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2 0x002a
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3 0x002b
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4 0x002c
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5 0x002d
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6 0x002e
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7 0x002f
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8 0x0030
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9 0x0031
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10 0x0032
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11 0x0033
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12 0x0034
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13 0x0035
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE 0x0036
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC 0x0037
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR 0x0038
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0 0x003a
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1 0x003b
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2 0x003c
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3 0x003d
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4 0x003e
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5 0x003f
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6 0x0040
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7 0x0041
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8 0x0042
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL 0x0054
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE 0x0055
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x0056
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2 0x0057
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE 0x0058
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0 0x0059
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1 0x005a
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2 0x005b
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3 0x005c
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4 0x005d
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5 0x005e
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6 0x005f
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7 0x0060
+#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8 0x0061
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO 0x0062
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS 0x0063
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x0064
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB 0x0065
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x0066
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE 0x0067
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED 0x0068
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION 0x0069
+#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE 0x006a
+#define ixAZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLE_STATUS 0x006b
+#define ixAZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLED_INT_STATUS 0x006c
+#define ixAZF0ENDPOINT3_AZALIA_F0_AUDIO_DISABLED_INT_STATUS 0x006d
+#define ixAZF0ENDPOINT3_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS 0x006e
+
+
+// addressBlock: azf0endpoint4_endpointind
+// base address: 0x0
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0001
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT 0x0002
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x0003
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x0004
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS 0x0005
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x0006
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL 0x0007
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE 0x0008
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING 0x0009
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA 0x000c
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN 0x000d
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX 0x000e
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0020
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES 0x0021
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE 0x0022
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE 0x0023
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL 0x0024
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER 0x0025
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0 0x0028
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1 0x0029
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2 0x002a
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3 0x002b
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4 0x002c
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5 0x002d
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6 0x002e
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7 0x002f
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8 0x0030
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9 0x0031
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10 0x0032
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11 0x0033
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12 0x0034
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13 0x0035
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE 0x0036
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC 0x0037
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR 0x0038
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0 0x003a
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1 0x003b
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2 0x003c
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3 0x003d
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4 0x003e
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5 0x003f
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6 0x0040
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7 0x0041
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8 0x0042
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL 0x0054
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE 0x0055
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x0056
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2 0x0057
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE 0x0058
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0 0x0059
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1 0x005a
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2 0x005b
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3 0x005c
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4 0x005d
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5 0x005e
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6 0x005f
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7 0x0060
+#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8 0x0061
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO 0x0062
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS 0x0063
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x0064
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB 0x0065
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x0066
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE 0x0067
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED 0x0068
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION 0x0069
+#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE 0x006a
+#define ixAZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLE_STATUS 0x006b
+#define ixAZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLED_INT_STATUS 0x006c
+#define ixAZF0ENDPOINT4_AZALIA_F0_AUDIO_DISABLED_INT_STATUS 0x006d
+#define ixAZF0ENDPOINT4_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS 0x006e
+
+
+// addressBlock: azf0endpoint5_endpointind
+// base address: 0x0
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0001
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT 0x0002
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x0003
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x0004
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS 0x0005
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x0006
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL 0x0007
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE 0x0008
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING 0x0009
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA 0x000c
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN 0x000d
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX 0x000e
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0020
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES 0x0021
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE 0x0022
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE 0x0023
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL 0x0024
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER 0x0025
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0 0x0028
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1 0x0029
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2 0x002a
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3 0x002b
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4 0x002c
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5 0x002d
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6 0x002e
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7 0x002f
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8 0x0030
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9 0x0031
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10 0x0032
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11 0x0033
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12 0x0034
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13 0x0035
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE 0x0036
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC 0x0037
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR 0x0038
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0 0x003a
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1 0x003b
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2 0x003c
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3 0x003d
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4 0x003e
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5 0x003f
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6 0x0040
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7 0x0041
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8 0x0042
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL 0x0054
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE 0x0055
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x0056
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2 0x0057
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE 0x0058
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0 0x0059
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1 0x005a
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2 0x005b
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3 0x005c
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4 0x005d
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5 0x005e
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6 0x005f
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7 0x0060
+#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8 0x0061
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO 0x0062
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS 0x0063
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x0064
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB 0x0065
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x0066
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE 0x0067
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED 0x0068
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION 0x0069
+#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE 0x006a
+#define ixAZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLE_STATUS 0x006b
+#define ixAZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLED_INT_STATUS 0x006c
+#define ixAZF0ENDPOINT5_AZALIA_F0_AUDIO_DISABLED_INT_STATUS 0x006d
+#define ixAZF0ENDPOINT5_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS 0x006e
+
+
+// addressBlock: azf0endpoint6_endpointind
+// base address: 0x0
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0001
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT 0x0002
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x0003
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x0004
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS 0x0005
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x0006
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL 0x0007
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE 0x0008
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING 0x0009
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA 0x000c
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN 0x000d
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX 0x000e
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0020
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES 0x0021
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE 0x0022
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE 0x0023
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL 0x0024
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER 0x0025
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0 0x0028
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1 0x0029
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2 0x002a
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3 0x002b
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4 0x002c
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5 0x002d
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6 0x002e
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7 0x002f
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8 0x0030
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9 0x0031
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10 0x0032
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11 0x0033
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12 0x0034
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13 0x0035
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE 0x0036
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC 0x0037
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR 0x0038
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0 0x003a
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1 0x003b
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2 0x003c
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3 0x003d
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4 0x003e
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5 0x003f
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6 0x0040
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7 0x0041
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8 0x0042
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL 0x0054
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE 0x0055
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x0056
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2 0x0057
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE 0x0058
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0 0x0059
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1 0x005a
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2 0x005b
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3 0x005c
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4 0x005d
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5 0x005e
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6 0x005f
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7 0x0060
+#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8 0x0061
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO 0x0062
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS 0x0063
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x0064
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB 0x0065
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x0066
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE 0x0067
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED 0x0068
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION 0x0069
+#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE 0x006a
+#define ixAZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLE_STATUS 0x006b
+#define ixAZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLED_INT_STATUS 0x006c
+#define ixAZF0ENDPOINT6_AZALIA_F0_AUDIO_DISABLED_INT_STATUS 0x006d
+#define ixAZF0ENDPOINT6_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS 0x006e
+
+
+// addressBlock: azf0endpoint7_endpointind
+// base address: 0x0
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0001
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT 0x0002
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x0003
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x0004
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS 0x0005
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x0006
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL 0x0007
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE 0x0008
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING 0x0009
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA 0x000c
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN 0x000d
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX 0x000e
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0020
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES 0x0021
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE 0x0022
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE 0x0023
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL 0x0024
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER 0x0025
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0 0x0028
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1 0x0029
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2 0x002a
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3 0x002b
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4 0x002c
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5 0x002d
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6 0x002e
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7 0x002f
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8 0x0030
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9 0x0031
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10 0x0032
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11 0x0033
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12 0x0034
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13 0x0035
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE 0x0036
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC 0x0037
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR 0x0038
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0 0x003a
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1 0x003b
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2 0x003c
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3 0x003d
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4 0x003e
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5 0x003f
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6 0x0040
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7 0x0041
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8 0x0042
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL 0x0054
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE 0x0055
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x0056
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2 0x0057
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE 0x0058
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0 0x0059
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1 0x005a
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2 0x005b
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3 0x005c
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4 0x005d
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5 0x005e
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6 0x005f
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7 0x0060
+#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8 0x0061
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO 0x0062
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS 0x0063
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x0064
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB 0x0065
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x0066
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE 0x0067
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED 0x0068
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION 0x0069
+#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE 0x006a
+#define ixAZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLE_STATUS 0x006b
+#define ixAZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLED_INT_STATUS 0x006c
+#define ixAZF0ENDPOINT7_AZALIA_F0_AUDIO_DISABLED_INT_STATUS 0x006d
+#define ixAZF0ENDPOINT7_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS 0x006e
+
+
+// addressBlock: azf0inputendpoint0_inputendpointind
+// base address: 0x0
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0001
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT 0x0002
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x0003
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x0004
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS 0x0005
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x0006
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0020
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES 0x0021
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE 0x0022
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE 0x0023
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL 0x0024
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE 0x0036
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2 0x0037
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR 0x0038
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION 0x0053
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL 0x0054
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE 0x0055
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x0056
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x0064
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB 0x0065
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x0066
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL 0x0067
+#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME 0x0068
+
+
+// addressBlock: azf0inputendpoint1_inputendpointind
+// base address: 0x0
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0001
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT 0x0002
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x0003
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x0004
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS 0x0005
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x0006
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0020
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES 0x0021
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE 0x0022
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE 0x0023
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL 0x0024
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE 0x0036
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2 0x0037
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR 0x0038
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION 0x0053
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL 0x0054
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE 0x0055
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x0056
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x0064
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB 0x0065
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x0066
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL 0x0067
+#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME 0x0068
+
+
+// addressBlock: azf0inputendpoint2_inputendpointind
+// base address: 0x0
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0001
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT 0x0002
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x0003
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x0004
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS 0x0005
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x0006
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0020
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES 0x0021
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE 0x0022
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE 0x0023
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL 0x0024
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE 0x0036
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2 0x0037
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR 0x0038
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION 0x0053
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL 0x0054
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE 0x0055
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x0056
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x0064
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB 0x0065
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x0066
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL 0x0067
+#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME 0x0068
+
+
+// addressBlock: azf0inputendpoint3_inputendpointind
+// base address: 0x0
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0001
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT 0x0002
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x0003
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x0004
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS 0x0005
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x0006
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0020
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES 0x0021
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE 0x0022
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE 0x0023
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL 0x0024
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE 0x0036
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2 0x0037
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR 0x0038
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION 0x0053
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL 0x0054
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE 0x0055
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x0056
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x0064
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB 0x0065
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x0066
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL 0x0067
+#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME 0x0068
+
+
+// addressBlock: azf0inputendpoint4_inputendpointind
+// base address: 0x0
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0001
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT 0x0002
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x0003
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x0004
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS 0x0005
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x0006
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0020
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES 0x0021
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE 0x0022
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE 0x0023
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL 0x0024
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE 0x0036
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2 0x0037
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR 0x0038
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION 0x0053
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL 0x0054
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE 0x0055
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x0056
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x0064
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB 0x0065
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x0066
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL 0x0067
+#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME 0x0068
+
+
+// addressBlock: azf0inputendpoint5_inputendpointind
+// base address: 0x0
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0001
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT 0x0002
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x0003
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x0004
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS 0x0005
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x0006
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0020
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES 0x0021
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE 0x0022
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE 0x0023
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL 0x0024
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE 0x0036
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2 0x0037
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR 0x0038
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION 0x0053
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL 0x0054
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE 0x0055
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x0056
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x0064
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB 0x0065
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x0066
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL 0x0067
+#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME 0x0068
+
+
+// addressBlock: azf0inputendpoint6_inputendpointind
+// base address: 0x0
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0001
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT 0x0002
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x0003
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x0004
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS 0x0005
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x0006
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0020
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES 0x0021
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE 0x0022
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE 0x0023
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL 0x0024
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE 0x0036
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2 0x0037
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR 0x0038
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION 0x0053
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL 0x0054
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE 0x0055
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x0056
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x0064
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB 0x0065
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x0066
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL 0x0067
+#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME 0x0068
+
+
+// addressBlock: azf0inputendpoint7_inputendpointind
+// base address: 0x0
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0001
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT 0x0002
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x0003
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x0004
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS 0x0005
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x0006
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0020
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES 0x0021
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE 0x0022
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE 0x0023
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL 0x0024
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE 0x0036
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2 0x0037
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR 0x0038
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION 0x0053
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL 0x0054
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE 0x0055
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x0056
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL 0x0064
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB 0x0065
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT 0x0066
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL 0x0067
+#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME 0x0068
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_2_0_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_2_0_0_sh_mask.h
new file mode 100644
index 000000000000..10c83fecd147
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_2_0_0_sh_mask.h